From a41dae270174e41a38d1f134590f70c6339b50e8 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 2 Oct 2025 14:11:08 +0000 Subject: [PATCH 001/254] LTO-IR expt --- cpp/CMakeLists.txt | 173 ++++++ .../Modules/JitifyPreprocessKernels.cmake | 2 + cpp/include/cudf/jit/filter_params.cuh | 28 + cpp/include/cudf/jit/lto/library.cuh | 8 + cpp/include/cudf/jit/lto/operators.cuh | 467 +++++++++++++++ cpp/include/cudf/jit/lto/thunk.cuh | 135 +++++ cpp/include/cudf/jit/lto/types.cuh | 480 +++++++++++++++ cpp/include/cudf/jit/transform_params.cuh | 28 + cpp/include/cudf/transform.hpp | 20 + cpp/src/jit/build.hpp | 41 ++ cpp/src/jit/lto/library.cu | 9 + cpp/src/jit/lto/operators.inl.cuh | 566 ++++++++++++++++++ cpp/src/jit/lto/types.inl.cuh | 365 +++++++++++ cpp/src/jit/rtc/cache.cpp | 27 + cpp/src/jit/rtc/cache.hpp | 163 +++++ cpp/src/jit/rtc/rtc.cpp | 214 +++++++ cpp/src/jit/rtc/rtc.hpp | 120 ++++ cpp/src/jit/rtc/sha256.cpp | 42 ++ cpp/src/jit/rtc/sha256.hpp | 99 +++ .../filter/jit/lto_kernel.inl.cuh | 34 ++ cpp/src/transform/jit/lto_kernel.inl.cuh | 33 + cpp/src/transform/transform.cpp | 25 +- jit.md | 1 + lto-ir-notes.txt | 1 + todo.md | 5 + 25 files changed, 3085 insertions(+), 1 deletion(-) create mode 100644 cpp/include/cudf/jit/filter_params.cuh create mode 100644 cpp/include/cudf/jit/lto/library.cuh create mode 100644 cpp/include/cudf/jit/lto/operators.cuh create mode 100644 cpp/include/cudf/jit/lto/thunk.cuh create mode 100644 cpp/include/cudf/jit/lto/types.cuh create mode 100644 cpp/include/cudf/jit/transform_params.cuh create mode 100644 cpp/src/jit/build.hpp create mode 100644 cpp/src/jit/lto/library.cu create mode 100644 cpp/src/jit/lto/operators.inl.cuh create mode 100644 cpp/src/jit/lto/types.inl.cuh create mode 100644 cpp/src/jit/rtc/cache.cpp create mode 100644 cpp/src/jit/rtc/cache.hpp create mode 100644 cpp/src/jit/rtc/rtc.cpp create mode 100644 cpp/src/jit/rtc/rtc.hpp create mode 100644 cpp/src/jit/rtc/sha256.cpp create mode 100644 cpp/src/jit/rtc/sha256.hpp create mode 100644 cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh create mode 100644 cpp/src/transform/jit/lto_kernel.inl.cuh create mode 100644 jit.md create mode 100644 lto-ir-notes.txt create mode 100644 todo.md diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c601dd0ce1df..a3959bdb2153 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -140,6 +140,8 @@ if(NOT CUDF_GENERATED_INCLUDE_DIR) set(CUDF_GENERATED_INCLUDE_DIR ${CUDF_BINARY_DIR}) endif() +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/cudf) + # ################################################################################################## # * linter configuration --------------------------------------------------------------------------- if(CUDF_CLANG_TIDY) @@ -243,6 +245,8 @@ rapids_find_package( ) include(cmake/Modules/ConfigureCUDA.cmake) # set other CUDA compilation flags +find_program(BIN2C_EXE bin2c REQUIRED) + # ################################################################################################## # * dependencies ---------------------------------------------------------------------------------- @@ -329,6 +333,173 @@ if(NOT BUILD_SHARED_LIBS) endif() endif() +# CHANGE TO embed_binary_command +function(embed_fatbins_command target_name) + + set(DEST_FILE ${CUDF_GENERATED_INCLUDE_DIR}/include/${target_name}_fatbin_bytes.h) + add_custom_command( + OUTPUT ${DEST_FILE} + COMMAND ${BIN2C_EXE} --const --type char --static --name ${target_name}_fatbin_bytes + "$" > ${DEST_FILE} + DEPENDS ${target_name} + COMMENT + "Converting fatbins generated by ${target_name} to C source file (${target_name}_fatbin_bytes.h) with bin2c" + ) + + add_custom_target(${target_name}_fatbin_bytes DEPENDS ${DEST_FILE}) + +endfunction() + + +""" + +# Take a list of files to JIT-compile and run them through jitify_preprocess. +function(jit_preprocess_files) + cmake_parse_arguments(ARG "" "SOURCE_DIRECTORY" "FILES" ${ARGN}) + + get_target_property(libcudacxx_raw_includes CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) + set(includes) + foreach(inc IN LISTS libcudacxx_raw_includes CUDAToolkit_INCLUDE_DIRS) + list(APPEND includes "-I${inc}") + endforeach() + foreach(ARG_FILE ${ARG_FILES}) + set(ARG_OUTPUT ${CUDF_GENERATED_INCLUDE_DIR}/include/jit_preprocessed_files/${ARG_FILE}.jit.hpp) + get_filename_component(jit_output_directory "${ARG_OUTPUT}" DIRECTORY) + list(APPEND JIT_PREPROCESSED_FILES "${ARG_OUTPUT}") + + get_filename_component(ARG_OUTPUT_DIR "${ARG_OUTPUT}" DIRECTORY) + + # Note: need to pass _FILE_OFFSET_BITS=64 in COMMAND due to a limitation in how conda builds + # glibc + add_custom_command( + OUTPUT ${ARG_OUTPUT} + DEPENDS jitify_preprocess "${ARG_SOURCE_DIRECTORY}/${ARG_FILE}" + WORKING_DIRECTORY ${ARG_SOURCE_DIRECTORY} + VERBATIM + COMMAND ${CMAKE_COMMAND} -E make_directory "${jit_output_directory}" + COMMAND + "${CMAKE_COMMAND}" -E env LD_LIBRARY_PATH=${CUDAToolkit_LIBRARY_DIR} + $ ${ARG_FILE} -o ${ARG_OUTPUT_DIR} -i -std=c++20 + -remove-unused-globals -D_FILE_OFFSET_BITS=64 -D__CUDACC_RTC__ -DCUDF_RUNTIME_JIT + -I${CUDF_SOURCE_DIR}/include -I${CUDF_SOURCE_DIR}/src ${includes} + --no-preinclude-workarounds --no-replace-pragma-once --diag-suppress=47 --device-int128 + COMMENT "Custom command to JIT-compile files." + ) + endforeach() + set(JIT_PREPROCESSED_FILES + "${JIT_PREPROCESSED_FILES}" + PARENT_SCOPE + ) +endfunction() + +if(NOT (EXISTS "${CUDF_GENERATED_INCLUDE_DIR}/include")) + make_directory("${CUDF_GENERATED_INCLUDE_DIR}/include") +endif() + +""" +# needs to recursively bundle the actually used headers; not everything +# should we just use the install include dir? +# provided cudacxx will also be exported? can we get the installed include dir path? +# after installing, should be configure a file that points to the installed cudacxx include dir? + +set(CUDF_JIT_INCLUDE_HEADERS +include/cudf/table/table_device_view.cuh +include/cudf/jit/lto/types.cuh +include/cudf/jit/lto/operators.cuh +include/cudf/jit/lto/library.cuh +include/cudf/jit/lto/thunk.cuh +include/cudf/jit/transform_params.cuh +include/cudf/jit/filter_params.cuh +include/cudf/utilities/traits.cuh +include/cudf/strings/string_view.cuh +include/cudf/column/column_device_view_base.cuh +include/cudf/detail/null_mask.cuh +include/cudf/detail/utilities/grid_1d.cuh +include/cudf/detail/offsets_iterator.cuh +include/cudf/detail/indexalator.cuh +include/cudf/ast/detail/operator_functor.cuh +include/cudf/ast/detail/operators.cuh +src/jit/accessors.cuh +src/jit/span.cuh +) + +# CCCL::libcudacxx + + # we have to copy everything in the include dirs here + # get_target_property(libcudacxx_raw_includes CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) + +set(CUDF_JIT_LTO_INCLUDE_HEADERS +include/cudf/jit/lto/types.cuh +include/cudf/jit/lto/operators.cuh +include/cudf/jit/transform_params.cuh +include/cudf/jit/filter_params.cuh +) + + +# CUDA/STD headers + + +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CUDF_JIT_CUDA_FLAGS "-G") +else if(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + set(CUDF_JIT_CUDA_FLAGS "-O2;-DNDEBUG") +else if(CMAKE_BUILD_TYPE STREQUAL "Release") + set(CUDF_JIT_CUDA_FLAGS "-O3;-DNDEBUG") + endif() + endif() +endif() + + +if(CUDA_ENABLE_LINEINFO) + list(APPEND CUDF_JIT_CUDA_FLAGS "${CUDF_JIT_CUDA_FLAGS};-lineinfo") +endif() + + + +# use current project CUDA_ARCHITECTURES for JIT compilation +set(CUDF_CXX_STANDARD 20) +set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) +set(CUDF_JIT_ARCHITECTURES "75;80;86;90;100;120") +set(CUDF_JIT_DEFINITIONS "${CUDF_CUDA_DEFINITIONS};-DCUDF_RUNTIME_JIT;-D__CUDACC_RTC__") +# set(CUDF_JIT_) +set(CUDF_JIT_COMPILE_FLAGS "-O3;-std=c++${CUDF_CUDA_STANDARD};-Xcompiler=-fPIC;-Xcompiler=-DPIC;-fvisibility=hidden;-Xfatbin=--compress-all;--compress-mode=size") +set(CUDF_JIT_LINK_FLAGS) + +function(add_lto_ir_library target_name source_file) + + add_library(${target_name} OBJECT ${source_file}) + target_compile_options(${target_name} PRIVATE -Xfatbin=--compress-all --compress-mode=size) + set_target_properties( + ${target_name} + PROPERTIES CUDA_ARCHITECTURES "75-real;80-real;86-real;90-real;100-real;120-real" + CUDA_SEPARABLE_COMPILATION ON + CUDA_FATBIN_COMPILATION ON + POSITION_INDEPENDENT_CODE ON + INTERPROCEDURAL_OPTIMIZATION ON + ) + target_include_directories( + ${target_name} PUBLIC ${CUDF_SOURCE_DIR}/include ${CUDF_SOURCE_DIR}/src + ) + target_compile_definitions(${target_name} PUBLIC ${CUDF_CUDA_DEFINITIONS} -DCUDF_RUNTIME_JIT -D__CUDACC_RTC__) + target_link_libraries(${target_name} PUBLIC CCCL::libcudacxx) + set_target_properties( + ${target_name} + PROPERTIES CXX_STANDARD ${CUDF_CXX_STANDARD} + CXX_STANDARD_REQUIRED ON + # For std:: support of __int128_t. Can be removed once using cuda::std + CXX_EXTENSIONS ON + CXX_VISIBILITY_PRESET hidden + CUDA_STANDARD ${CUDF_CUDA_STANDARD} + CUDA_STANDARD_REQUIRED ON + CUDA_VISIBILITY_PRESET hidden + ) + + embed_fatbins_command(${target_name}) + +endfunction() + +add_lto_ir_library(cudf_lto_library src/jit/lto/library.cu) + # ################################################################################################## # * library targets ------------------------------------------------------------------------------- add_library( @@ -834,6 +1005,8 @@ add_library( src/utilities/type_dispatcher.cpp ) +add_dependencies(cudf cudf_lto_library_fatbin_bytes) + # Anything that includes jitify needs to be compiled with _FILE_OFFSET_BITS=64 due to a limitation # in how conda builds glibc set_source_files_properties( diff --git a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake index f21fdbe5ffb5..bd4169185b89 100644 --- a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake +++ b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake @@ -60,6 +60,8 @@ jit_preprocess_files( stream_compaction/filter/jit/kernel.cu transform/jit/kernel.cu ) +jit_preprocess_files(SOURCE_DIRECTORY ${CUDF_SOURCE_DIR}/include/cudf FILES jit/lto/library.cuh) + add_custom_target( jitify_preprocess_run DEPENDS ${JIT_PREPROCESSED_FILES} diff --git a/cpp/include/cudf/jit/filter_params.cuh b/cpp/include/cudf/jit/filter_params.cuh new file mode 100644 index 000000000000..8de94ffa3a16 --- /dev/null +++ b/cpp/include/cudf/jit/filter_params.cuh @@ -0,0 +1,28 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include +#include + +namespace CUDF_EXPORT cudf { + +namespace lto { + +struct filter_params { + void const* inputs = nullptr; + void* user_data = nullptr; + void const* outputs = nullptr; + void const* span_outputs = nullptr; + size_type row_index = 0; +}; + +} // namespace lto +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/library.cuh b/cpp/include/cudf/jit/lto/library.cuh new file mode 100644 index 000000000000..a50eb213f77e --- /dev/null +++ b/cpp/include/cudf/jit/lto/library.cuh @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include +#include +#include diff --git a/cpp/include/cudf/jit/lto/operators.cuh b/cpp/include/cudf/jit/lto/operators.cuh new file mode 100644 index 000000000000..361dae7a6024 --- /dev/null +++ b/cpp/include/cudf/jit/lto/operators.cuh @@ -0,0 +1,467 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace CUDF_LTO_EXPORT cudf { + +namespace lto { + +namespace operators { + +#define CUDF_LTO_DECL(op, type) \ + extern __device__ void op(type* out, type const* a, type const* b); \ + \ + extern __device__ void op(optional* out, optional const* a, optional const* b) + +CUDF_LTO_DECL(add, int32_t); +CUDF_LTO_DECL(add, int64_t); +CUDF_LTO_DECL(add, uint32_t); +CUDF_LTO_DECL(add, uint64_t); +CUDF_LTO_DECL(add, float32_t); +CUDF_LTO_DECL(add, float64_t); +CUDF_LTO_DECL(add, decimal32); +CUDF_LTO_DECL(add, decimal64); +CUDF_LTO_DECL(add, decimal128); +CUDF_LTO_DECL(add, duration_D); +CUDF_LTO_DECL(add, duration_s); +CUDF_LTO_DECL(add, duration_ms); +CUDF_LTO_DECL(add, duration_ns); + +CUDF_LTO_DECL(sub, int32_t); +CUDF_LTO_DECL(sub, int64_t); +CUDF_LTO_DECL(sub, uint32_t); +CUDF_LTO_DECL(sub, uint64_t); +CUDF_LTO_DECL(sub, float32_t); +CUDF_LTO_DECL(sub, float64_t); +CUDF_LTO_DECL(sub, decimal32); +CUDF_LTO_DECL(sub, decimal64); +CUDF_LTO_DECL(sub, decimal128); +CUDF_LTO_DECL(sub, duration_D); +CUDF_LTO_DECL(sub, duration_s); +CUDF_LTO_DECL(sub, duration_ms); +CUDF_LTO_DECL(sub, duration_ns); + +CUDF_LTO_DECL(mul, int32_t); +CUDF_LTO_DECL(mul, int64_t); +CUDF_LTO_DECL(mul, uint32_t); +CUDF_LTO_DECL(mul, uint64_t); +CUDF_LTO_DECL(mul, float32_t); +CUDF_LTO_DECL(mul, float64_t); +CUDF_LTO_DECL(mul, decimal32); +CUDF_LTO_DECL(mul, decimal64); +CUDF_LTO_DECL(mul, decimal128); + +CUDF_LTO_DECL(div, int32_t); +CUDF_LTO_DECL(div, int64_t); +CUDF_LTO_DECL(div, uint32_t); +CUDF_LTO_DECL(div, uint64_t); +CUDF_LTO_DECL(div, float32_t); +CUDF_LTO_DECL(div, float64_t); +CUDF_LTO_DECL(div, decimal32); +CUDF_LTO_DECL(div, decimal64); +CUDF_LTO_DECL(div, decimal128); + +CUDF_LTO_DECL(mod, float32_t); +CUDF_LTO_DECL(mod, float64_t); + +CUDF_LTO_DECL(pymod, float32_t); +CUDF_LTO_DECL(pymod, float64_t); + +CUDF_LTO_DECL(pow, float32_t); +CUDF_LTO_DECL(pow, float64_t); + +#undef CUDF_LTO_DECL + +#define CUDF_LTO_DECL(op, type) \ + extern __device__ void op(bool* out, type const* a, type const* b); \ + \ + extern __device__ void op(bool* out, optional const* a, optional const* b); \ + \ + extern __device__ void op(optional* out, optional const* a, optional const* b) + +CUDF_LTO_DECL(equal, bool); +CUDF_LTO_DECL(equal, int8_t); +CUDF_LTO_DECL(equal, int16_t); +CUDF_LTO_DECL(equal, int32_t); +CUDF_LTO_DECL(equal, int64_t); +CUDF_LTO_DECL(equal, uint8_t); +CUDF_LTO_DECL(equal, uint16_t); +CUDF_LTO_DECL(equal, uint32_t); +CUDF_LTO_DECL(equal, uint64_t); +CUDF_LTO_DECL(equal, float32_t); +CUDF_LTO_DECL(equal, float64_t); +CUDF_LTO_DECL(equal, decimal32); +CUDF_LTO_DECL(equal, decimal64); +CUDF_LTO_DECL(equal, decimal128); +CUDF_LTO_DECL(equal, timestamp_D); +CUDF_LTO_DECL(equal, timestamp_s); +CUDF_LTO_DECL(equal, timestamp_ms); +CUDF_LTO_DECL(equal, timestamp_us); +CUDF_LTO_DECL(equal, timestamp_ns); +CUDF_LTO_DECL(equal, duration_D); +CUDF_LTO_DECL(equal, duration_s); +CUDF_LTO_DECL(equal, duration_ms); +CUDF_LTO_DECL(equal, duration_ns); +CUDF_LTO_DECL(equal, string_view); + +CUDF_LTO_DECL(null_equal, bool); +CUDF_LTO_DECL(null_equal, int8_t); +CUDF_LTO_DECL(null_equal, int16_t); +CUDF_LTO_DECL(null_equal, int32_t); +CUDF_LTO_DECL(null_equal, int64_t); +CUDF_LTO_DECL(null_equal, uint8_t); +CUDF_LTO_DECL(null_equal, uint16_t); +CUDF_LTO_DECL(null_equal, uint32_t); +CUDF_LTO_DECL(null_equal, uint64_t); +CUDF_LTO_DECL(null_equal, float32_t); +CUDF_LTO_DECL(null_equal, float64_t); +CUDF_LTO_DECL(null_equal, decimal32); +CUDF_LTO_DECL(null_equal, decimal64); +CUDF_LTO_DECL(null_equal, decimal128); +CUDF_LTO_DECL(null_equal, timestamp_D); +CUDF_LTO_DECL(null_equal, timestamp_s); +CUDF_LTO_DECL(null_equal, timestamp_ms); +CUDF_LTO_DECL(null_equal, timestamp_us); +CUDF_LTO_DECL(null_equal, timestamp_ns); +CUDF_LTO_DECL(null_equal, duration_D); +CUDF_LTO_DECL(null_equal, duration_s); +CUDF_LTO_DECL(null_equal, duration_ms); +CUDF_LTO_DECL(null_equal, duration_ns); +CUDF_LTO_DECL(null_equal, string_view); + +#undef CUDF_LTO_DECL + +#define CUDF_LTO_DECL(op, type) \ + extern __device__ void op(bool* out, type const* a, type const* b); \ + \ + extern __device__ void op(optional* out, optional const* a, optional const* b) + +CUDF_LTO_DECL(less, bool); +CUDF_LTO_DECL(less, int8_t); +CUDF_LTO_DECL(less, int16_t); +CUDF_LTO_DECL(less, int32_t); +CUDF_LTO_DECL(less, int64_t); +CUDF_LTO_DECL(less, uint8_t); +CUDF_LTO_DECL(less, uint16_t); +CUDF_LTO_DECL(less, uint32_t); +CUDF_LTO_DECL(less, uint64_t); +CUDF_LTO_DECL(less, float32_t); +CUDF_LTO_DECL(less, float64_t); +CUDF_LTO_DECL(less, decimal32); +CUDF_LTO_DECL(less, decimal64); +CUDF_LTO_DECL(less, decimal128); +CUDF_LTO_DECL(less, timestamp_D); +CUDF_LTO_DECL(less, timestamp_s); +CUDF_LTO_DECL(less, timestamp_ms); +CUDF_LTO_DECL(less, timestamp_us); +CUDF_LTO_DECL(less, timestamp_ns); +CUDF_LTO_DECL(less, duration_D); +CUDF_LTO_DECL(less, duration_s); +CUDF_LTO_DECL(less, duration_ms); +CUDF_LTO_DECL(less, duration_ns); +CUDF_LTO_DECL(less, string_view); + +CUDF_LTO_DECL(greater, bool); +CUDF_LTO_DECL(greater, int8_t); +CUDF_LTO_DECL(greater, int16_t); +CUDF_LTO_DECL(greater, int32_t); +CUDF_LTO_DECL(greater, int64_t); +CUDF_LTO_DECL(greater, uint8_t); +CUDF_LTO_DECL(greater, uint16_t); +CUDF_LTO_DECL(greater, uint32_t); +CUDF_LTO_DECL(greater, uint64_t); +CUDF_LTO_DECL(greater, float32_t); +CUDF_LTO_DECL(greater, float64_t); +CUDF_LTO_DECL(greater, decimal32); +CUDF_LTO_DECL(greater, decimal64); +CUDF_LTO_DECL(greater, decimal128); +CUDF_LTO_DECL(greater, timestamp_D); +CUDF_LTO_DECL(greater, timestamp_s); +CUDF_LTO_DECL(greater, timestamp_ms); +CUDF_LTO_DECL(greater, timestamp_us); +CUDF_LTO_DECL(greater, timestamp_ns); +CUDF_LTO_DECL(greater, duration_D); +CUDF_LTO_DECL(greater, duration_s); +CUDF_LTO_DECL(greater, duration_ms); +CUDF_LTO_DECL(greater, duration_ns); +CUDF_LTO_DECL(greater, string_view); + +CUDF_LTO_DECL(less_equal, bool); +CUDF_LTO_DECL(less_equal, int8_t); +CUDF_LTO_DECL(less_equal, int16_t); +CUDF_LTO_DECL(less_equal, int32_t); +CUDF_LTO_DECL(less_equal, int64_t); +CUDF_LTO_DECL(less_equal, uint8_t); +CUDF_LTO_DECL(less_equal, uint16_t); +CUDF_LTO_DECL(less_equal, uint32_t); +CUDF_LTO_DECL(less_equal, uint64_t); +CUDF_LTO_DECL(less_equal, float32_t); +CUDF_LTO_DECL(less_equal, float64_t); +CUDF_LTO_DECL(less_equal, decimal32); +CUDF_LTO_DECL(less_equal, decimal64); +CUDF_LTO_DECL(less_equal, decimal128); +CUDF_LTO_DECL(less_equal, timestamp_D); +CUDF_LTO_DECL(less_equal, timestamp_s); +CUDF_LTO_DECL(less_equal, timestamp_ms); +CUDF_LTO_DECL(less_equal, timestamp_us); +CUDF_LTO_DECL(less_equal, timestamp_ns); +CUDF_LTO_DECL(less_equal, duration_D); +CUDF_LTO_DECL(less_equal, duration_s); +CUDF_LTO_DECL(less_equal, duration_ms); +CUDF_LTO_DECL(less_equal, duration_ns); +CUDF_LTO_DECL(less_equal, string_view); + +CUDF_LTO_DECL(greater_equal, bool); +CUDF_LTO_DECL(greater_equal, int8_t); +CUDF_LTO_DECL(greater_equal, int16_t); +CUDF_LTO_DECL(greater_equal, int32_t); +CUDF_LTO_DECL(greater_equal, int64_t); +CUDF_LTO_DECL(greater_equal, uint8_t); +CUDF_LTO_DECL(greater_equal, uint16_t); +CUDF_LTO_DECL(greater_equal, uint32_t); +CUDF_LTO_DECL(greater_equal, uint64_t); +CUDF_LTO_DECL(greater_equal, float32_t); +CUDF_LTO_DECL(greater_equal, float64_t); +CUDF_LTO_DECL(greater_equal, decimal32); +CUDF_LTO_DECL(greater_equal, decimal64); +CUDF_LTO_DECL(greater_equal, decimal128); +CUDF_LTO_DECL(greater_equal, timestamp_D); +CUDF_LTO_DECL(greater_equal, timestamp_s); +CUDF_LTO_DECL(greater_equal, timestamp_ms); +CUDF_LTO_DECL(greater_equal, timestamp_us); +CUDF_LTO_DECL(greater_equal, timestamp_ns); +CUDF_LTO_DECL(greater_equal, duration_D); +CUDF_LTO_DECL(greater_equal, duration_s); +CUDF_LTO_DECL(greater_equal, duration_ms); +CUDF_LTO_DECL(greater_equal, duration_ns); +CUDF_LTO_DECL(greater_equal, string_view); + +#undef CUDF_LTO_DECL + +#define CUDF_LTO_DECL(op, type) \ + extern __device__ void op(type* out, type const* a, type const* b); \ + \ + extern __device__ void op(optional* out, optional const* a, optional const* b) + +CUDF_LTO_DECL(bitwise_and, int32_t); +CUDF_LTO_DECL(bitwise_and, int64_t); +CUDF_LTO_DECL(bitwise_and, uint32_t); +CUDF_LTO_DECL(bitwise_and, uint64_t); + +CUDF_LTO_DECL(bitwise_or, int32_t); +CUDF_LTO_DECL(bitwise_or, int64_t); +CUDF_LTO_DECL(bitwise_or, uint32_t); +CUDF_LTO_DECL(bitwise_or, uint64_t); + +CUDF_LTO_DECL(bitwise_xor, int32_t); +CUDF_LTO_DECL(bitwise_xor, int64_t); +CUDF_LTO_DECL(bitwise_xor, uint32_t); +CUDF_LTO_DECL(bitwise_xor, uint64_t); + +#undef CUDF_LTO_DECL + +#define CUDF_LTO_DECL(op, type) \ + extern __device__ void op(type* out, type const* a, type const* b); \ + \ + extern __device__ void op(optional* out, optional const* a, optional const* b); + +CUDF_LTO_DECL(logical_and, bool); + +CUDF_LTO_DECL(null_logical_and, bool); + +CUDF_LTO_DECL(logical_or, bool); + +CUDF_LTO_DECL(null_logical_or, bool); + +#undef CUDF_LTO_DECL + +#define CUDF_LTO_DECL(op, type) \ + extern __device__ void op(type* out, type const* a); \ + \ + extern __device__ void op(optional* out, optional const* a) + +CUDF_LTO_DECL(identity, bool); +CUDF_LTO_DECL(identity, int8_t); +CUDF_LTO_DECL(identity, int16_t); +CUDF_LTO_DECL(identity, int32_t); +CUDF_LTO_DECL(identity, int64_t); +CUDF_LTO_DECL(identity, uint8_t); +CUDF_LTO_DECL(identity, uint16_t); +CUDF_LTO_DECL(identity, uint32_t); +CUDF_LTO_DECL(identity, uint64_t); +CUDF_LTO_DECL(identity, float32_t); +CUDF_LTO_DECL(identity, float64_t); +CUDF_LTO_DECL(identity, decimal32); +CUDF_LTO_DECL(identity, decimal64); +CUDF_LTO_DECL(identity, decimal128); +CUDF_LTO_DECL(identity, timestamp_D); +CUDF_LTO_DECL(identity, timestamp_s); +CUDF_LTO_DECL(identity, timestamp_ms); +CUDF_LTO_DECL(identity, timestamp_us); +CUDF_LTO_DECL(identity, timestamp_ns); +CUDF_LTO_DECL(identity, duration_D); +CUDF_LTO_DECL(identity, duration_s); +CUDF_LTO_DECL(identity, duration_ms); +CUDF_LTO_DECL(identity, duration_ns); +CUDF_LTO_DECL(identity, string_view); + +CUDF_LTO_DECL(sin, float32_t); +CUDF_LTO_DECL(sin, float64_t); + +CUDF_LTO_DECL(cos, float32_t); +CUDF_LTO_DECL(cos, float64_t); + +CUDF_LTO_DECL(tan, float32_t); +CUDF_LTO_DECL(tan, float64_t); + +CUDF_LTO_DECL(arcsin, float32_t); +CUDF_LTO_DECL(arcsin, float64_t); + +CUDF_LTO_DECL(arccos, float32_t); +CUDF_LTO_DECL(arccos, float64_t); + +CUDF_LTO_DECL(arctan, float32_t); +CUDF_LTO_DECL(arctan, float64_t); + +CUDF_LTO_DECL(sinh, float32_t); +CUDF_LTO_DECL(sinh, float64_t); + +CUDF_LTO_DECL(cosh, float32_t); +CUDF_LTO_DECL(cosh, float64_t); + +CUDF_LTO_DECL(tanh, float32_t); +CUDF_LTO_DECL(tanh, float64_t); + +CUDF_LTO_DECL(arcsinh, float32_t); +CUDF_LTO_DECL(arcsinh, float64_t); + +CUDF_LTO_DECL(arccosh, float32_t); +CUDF_LTO_DECL(arccosh, float64_t); + +CUDF_LTO_DECL(arctanh, float32_t); +CUDF_LTO_DECL(arctanh, float64_t); + +CUDF_LTO_DECL(exp, float32_t); +CUDF_LTO_DECL(exp, float64_t); + +CUDF_LTO_DECL(log, float32_t); +CUDF_LTO_DECL(log, float64_t); + +CUDF_LTO_DECL(cbrt, float32_t); +CUDF_LTO_DECL(cbrt, float64_t); + +CUDF_LTO_DECL(ceil, float32_t); +CUDF_LTO_DECL(ceil, float64_t); + +CUDF_LTO_DECL(floor, float32_t); +CUDF_LTO_DECL(floor, float64_t); + +CUDF_LTO_DECL(abs, int32_t); +CUDF_LTO_DECL(abs, int64_t); +CUDF_LTO_DECL(abs, float32_t); +CUDF_LTO_DECL(abs, float64_t); + +CUDF_LTO_DECL(rint, float32_t); +CUDF_LTO_DECL(rint, float64_t); + +CUDF_LTO_DECL(bit_invert, uint32_t); +CUDF_LTO_DECL(bit_invert, uint64_t); +CUDF_LTO_DECL(bit_invert, int32_t); +CUDF_LTO_DECL(bit_invert, int64_t); + +#undef CUDF_LTO_DECL + +#define CUDF_LTO_DECL(op, ret_type, type) \ + extern __device__ void op(ret_type* out, type const* a); \ + \ + extern __device__ void op(optional* out, optional const* a) + +CUDF_LTO_DECL(cast_to_int64, int64_t, bool); +CUDF_LTO_DECL(cast_to_int64, int64_t, int8_t); +CUDF_LTO_DECL(cast_to_int64, int64_t, int16_t); +CUDF_LTO_DECL(cast_to_int64, int64_t, int32_t); +CUDF_LTO_DECL(cast_to_int64, int64_t, int64_t); +CUDF_LTO_DECL(cast_to_int64, int64_t, uint8_t); +CUDF_LTO_DECL(cast_to_int64, int64_t, uint16_t); +CUDF_LTO_DECL(cast_to_int64, int64_t, uint32_t); +CUDF_LTO_DECL(cast_to_int64, int64_t, uint64_t); +CUDF_LTO_DECL(cast_to_int64, int64_t, float32_t); +CUDF_LTO_DECL(cast_to_int64, int64_t, float64_t); + +CUDF_LTO_DECL(cast_to_uint64, uint64_t, bool); +CUDF_LTO_DECL(cast_to_uint64, uint64_t, int8_t); +CUDF_LTO_DECL(cast_to_uint64, uint64_t, int16_t); +CUDF_LTO_DECL(cast_to_uint64, uint64_t, int32_t); +CUDF_LTO_DECL(cast_to_uint64, uint64_t, int64_t); +CUDF_LTO_DECL(cast_to_uint64, uint64_t, uint8_t); +CUDF_LTO_DECL(cast_to_uint64, uint64_t, uint16_t); +CUDF_LTO_DECL(cast_to_uint64, uint64_t, uint32_t); +CUDF_LTO_DECL(cast_to_uint64, uint64_t, uint64_t); +CUDF_LTO_DECL(cast_to_uint64, uint64_t, float32_t); +CUDF_LTO_DECL(cast_to_uint64, uint64_t, float64_t); + +CUDF_LTO_DECL(cast_to_float64, float64_t, bool); +CUDF_LTO_DECL(cast_to_float64, float64_t, int8_t); +CUDF_LTO_DECL(cast_to_float64, float64_t, int16_t); +CUDF_LTO_DECL(cast_to_float64, float64_t, int32_t); +CUDF_LTO_DECL(cast_to_float64, float64_t, int64_t); +CUDF_LTO_DECL(cast_to_float64, float64_t, uint8_t); +CUDF_LTO_DECL(cast_to_float64, float64_t, uint16_t); +CUDF_LTO_DECL(cast_to_float64, float64_t, uint32_t); +CUDF_LTO_DECL(cast_to_float64, float64_t, uint64_t); +CUDF_LTO_DECL(cast_to_float64, float64_t, float32_t); +CUDF_LTO_DECL(cast_to_float64, float64_t, float64_t); + +#undef CUDF_LTO_DECL + +#define CUDF_LTO_DECL(op, type) \ + extern __device__ void op(bool* out, type const* a); \ + \ + extern __device__ void op(bool* out, optional const* a); \ + \ + extern __device__ void op(optional* out, optional const* a) + +CUDF_LTO_DECL(is_null, bool); +CUDF_LTO_DECL(is_null, int8_t); +CUDF_LTO_DECL(is_null, int16_t); +CUDF_LTO_DECL(is_null, int32_t); +CUDF_LTO_DECL(is_null, int64_t); +CUDF_LTO_DECL(is_null, uint8_t); +CUDF_LTO_DECL(is_null, uint16_t); +CUDF_LTO_DECL(is_null, uint32_t); +CUDF_LTO_DECL(is_null, uint64_t); +CUDF_LTO_DECL(is_null, float32_t); +CUDF_LTO_DECL(is_null, float64_t); +CUDF_LTO_DECL(is_null, decimal32); +CUDF_LTO_DECL(is_null, decimal64); +CUDF_LTO_DECL(is_null, decimal128); +CUDF_LTO_DECL(is_null, timestamp_D); +CUDF_LTO_DECL(is_null, timestamp_s); +CUDF_LTO_DECL(is_null, timestamp_ms); +CUDF_LTO_DECL(is_null, timestamp_us); +CUDF_LTO_DECL(is_null, timestamp_ns); +CUDF_LTO_DECL(is_null, duration_D); +CUDF_LTO_DECL(is_null, duration_s); +CUDF_LTO_DECL(is_null, duration_ms); +CUDF_LTO_DECL(is_null, duration_ns); +CUDF_LTO_DECL(is_null, string_view); + +CUDF_LTO_DECL(logical_not, bool); +CUDF_LTO_DECL(logical_not, int8_t); +CUDF_LTO_DECL(logical_not, int16_t); +CUDF_LTO_DECL(logical_not, int32_t); +CUDF_LTO_DECL(logical_not, int64_t); +CUDF_LTO_DECL(logical_not, uint8_t); +CUDF_LTO_DECL(logical_not, uint16_t); +CUDF_LTO_DECL(logical_not, uint32_t); +CUDF_LTO_DECL(logical_not, uint64_t); + +#undef CUDF_LTO_DECL + +} // namespace operators + +} // namespace lto +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/thunk.cuh b/cpp/include/cudf/jit/lto/thunk.cuh new file mode 100644 index 000000000000..afeda2ef54b7 --- /dev/null +++ b/cpp/include/cudf/jit/lto/thunk.cuh @@ -0,0 +1,135 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace CUDF_EXPORT cudf { + +namespace lto { + +template +struct lowered_type_of_t; + +template +using lowered_type_of = typename lowered_type_of_t::type; + +template +struct lifted_type_of_t; + +template +using lifted_type_of = typename lifted_type_of_t::type; + +#define CUDF_LTO_MAP(lowered_type, lifted_type) \ + static_assert(sizeof(lowered_type) == sizeof(lifted_type), \ + "(1: size). Lowered and Lifted types must be bitwise-equivalent"); \ + static_assert(alignof(lowered_type) == alignof(lifted_type), \ + "(2: alignment). Lowered and Lifted types must be bitwise-equivalent"); \ + static_assert( \ + sizeof(lto::optional) == sizeof(cuda::std::optional), \ + "(1: size). Lowered and Lifted types must have bitwise-equivalent optional types"); \ + static_assert( \ + alignof(lto::optional) == alignof(cuda::std::optional), \ + "(2: alignment). Lowered and Lifted types must have bitwise-equivalent optional types"); \ + \ + template <> \ + struct lifted_type_of_t { \ + using type = lifted_type; \ + }; \ + \ + template <> \ + struct lowered_type_of_t { \ + using type = lowered_type; \ + }; \ + \ + __forceinline__ __device__ lowered_type* lower(lifted_type* p) \ + { \ + return reinterpret_cast(p); \ + } \ + \ + __forceinline__ __device__ lowered_type const* lower(lifted_type const* p) \ + { \ + return reinterpret_cast(p); \ + } \ + \ + __forceinline__ __device__ lto::optional* lower( \ + cuda::std::optional* p) \ + { \ + return reinterpret_cast*>(p); \ + } \ + \ + __forceinline__ __device__ lto::optional const* lower( \ + cuda::std::optional const* p) \ + { \ + return reinterpret_cast const*>(p); \ + } \ + \ + __forceinline__ __device__ lifted_type* lift(lowered_type* p) \ + { \ + return reinterpret_cast(p); \ + } \ + \ + __forceinline__ __device__ lifted_type const* lift(lowered_type const* p) \ + { \ + return reinterpret_cast(p); \ + } \ + \ + __forceinline__ __device__ cuda::std::optional* lift( \ + lto::optional* p) \ + { \ + return reinterpret_cast*>(p); \ + } \ + \ + __forceinline__ __device__ cuda::std::optional const* lift( \ + lto::optional const* p) \ + { \ + return reinterpret_cast const*>(p); \ + } + +CUDF_LTO_MAP(bool, bool); +CUDF_LTO_MAP(lto::data_type, cudf::data_type); +CUDF_LTO_MAP(lto::int8_t, std::int8_t); +CUDF_LTO_MAP(lto::int16_t, std::int16_t); +CUDF_LTO_MAP(lto::int32_t, std::int32_t); +CUDF_LTO_MAP(lto::int64_t, std::int64_t); +CUDF_LTO_MAP(lto::uint8_t, std::uint8_t); +CUDF_LTO_MAP(lto::uint16_t, std::uint16_t); +CUDF_LTO_MAP(lto::uint32_t, std::uint32_t); +CUDF_LTO_MAP(lto::uint64_t, std::uint64_t); +CUDF_LTO_MAP(lto::float32_t, float); +CUDF_LTO_MAP(lto::float64_t, double); +CUDF_LTO_MAP(lto::decimal32, numeric::decimal32); +CUDF_LTO_MAP(lto::decimal64, numeric::decimal64); +CUDF_LTO_MAP(lto::decimal128, numeric::decimal128); +CUDF_LTO_MAP(lto::string_view, cudf::string_view); +CUDF_LTO_MAP(lto::timestamp_D, cudf::timestamp_D); +CUDF_LTO_MAP(lto::timestamp_h, cudf::timestamp_h); +CUDF_LTO_MAP(lto::timestamp_m, cudf::timestamp_m); +CUDF_LTO_MAP(lto::timestamp_s, cudf::timestamp_s); +CUDF_LTO_MAP(lto::timestamp_ms, cudf::timestamp_ms); +CUDF_LTO_MAP(lto::timestamp_us, cudf::timestamp_us); +CUDF_LTO_MAP(lto::timestamp_ns, cudf::timestamp_ns); +CUDF_LTO_MAP(lto::duration_D, cudf::duration_D); +CUDF_LTO_MAP(lto::duration_h, cudf::duration_h); +CUDF_LTO_MAP(lto::duration_m, cudf::duration_m); +CUDF_LTO_MAP(lto::duration_s, cudf::duration_s); +CUDF_LTO_MAP(lto::duration_ms, cudf::duration_ms); +CUDF_LTO_MAP(lto::duration_us, cudf::duration_us); +CUDF_LTO_MAP(lto::duration_ns, cudf::duration_ns); +CUDF_LTO_MAP(lto::column_device_view_core, cudf::column_device_view_core); +CUDF_LTO_MAP(lto::mutable_column_device_view_core, cudf::mutable_column_device_view_core); + +#undef CUDF_LTO_MAP + +} // namespace lto + +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh new file mode 100644 index 000000000000..b04d48eb32f3 --- /dev/null +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -0,0 +1,480 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#define CUDF_LTO_EXPORT __attribute__((visibility("default"))) + +namespace CUDF_LTO_EXPORT cudf { + +namespace lto { + +/** + * @brief LTO-JIT functions and thunk types + * + * These are declarations for functions that will be used in LTO-JIT compiled code. + * They are pre-compiled into a device library that is linked at JIT compile time. + * This header should be minimal and only contain necessary types and function declarations as it + * will be included and compiled at JIT compile time. Including other headers will lead to longer + * JIT compile times which can be unbounded and cause slowdowns. + * + */ + +using int8_t = signed char; +using int16_t = signed short; +using int32_t = signed int; +using int64_t = signed long long; +using uint8_t = unsigned char; +using uint16_t = unsigned short; +using uint32_t = unsigned int; +using uint64_t = unsigned long long; + +using intptr_t = int64_t; +using uintptr_t = uint64_t; + +using intmax_t = int64_t; +using uintmax_t = uint64_t; + +using float32_t = float; +using float64_t = double; + +using size_type = int32_t; + +using bitmask_type = uint32_t; + +using char_utf8 = uint32_t; + +enum class type_id : int32_t {}; + +enum scale_type : int32_t {}; + +struct __attribute__((may_alias)) data_type { + private: + type_id __id = {}; + int32_t __fixed_point_scale = 0; +}; + +struct __attribute__((may_alias)) string_view { + private: + char const* __data = nullptr; + size_type __bytes = 0; + mutable size_type __length = 0; + + public: + [[nodiscard]] __device__ size_type size_bytes() const; + + [[nodiscard]] __device__ size_type length() const; + + [[nodiscard]] __device__ char const* data() const; + + [[nodiscard]] __device__ bool empty() const; + + __device__ char_utf8 operator[](size_type pos) const; + + [[nodiscard]] __device__ size_type byte_offset(size_type pos) const; + + [[nodiscard]] __device__ int compare(string_view const& str) const; + + __device__ int compare(char const* str, size_type bytes) const; + + __device__ bool operator==(string_view const& rhs) const; + + __device__ bool operator!=(string_view const& rhs) const; + + __device__ bool operator<(string_view const& rhs) const; + + __device__ bool operator>(string_view const& rhs) const; + + __device__ bool operator<=(string_view const& rhs) const; + + __device__ bool operator>=(string_view const& rhs) const; + + [[nodiscard]] __device__ size_type find(string_view const& str, + size_type pos = 0, + size_type count = -1) const; + + __device__ size_type find(char const* str, + size_type bytes, + size_type pos = 0, + size_type count = -1) const; + + [[nodiscard]] __device__ size_type find(char_utf8 character, + size_type pos = 0, + size_type count = -1) const; + + [[nodiscard]] __device__ size_type rfind(string_view const& str, + size_type pos = 0, + size_type count = -1) const; + + __device__ size_type rfind(char const* str, + size_type bytes, + size_type pos = 0, + size_type count = -1) const; + + [[nodiscard]] __device__ size_type rfind(char_utf8 character, + size_type pos = 0, + size_type count = -1) const; + + [[nodiscard]] __device__ string_view substr(size_type start, size_type length) const; + + static inline size_type const npos{-1}; +}; + +struct __attribute__((may_alias)) decimal32 { + private: + int32_t __value = 0; + scale_type __scale = scale_type{}; +}; + +struct __attribute__((may_alias)) decimal64 { + private: + int64_t __value = 0; + scale_type __scale = scale_type{}; +}; + +struct __attribute__((may_alias)) decimal128 { + private: + __int128_t __value = 0; + scale_type __scale = scale_type{}; +}; + +struct __attribute__((may_alias)) timestamp_D { + private: + int32_t __rep = 0; +}; + +struct __attribute__((may_alias)) timestamp_h { + private: + int32_t __rep = 0; +}; + +struct __attribute__((may_alias)) timestamp_m { + private: + int32_t __rep = 0; +}; + +struct __attribute__((may_alias)) timestamp_s { + private: + int64_t __rep = 0; +}; + +struct __attribute__((may_alias)) timestamp_ms { + private: + int64_t __rep = 0; +}; + +struct __attribute__((may_alias)) timestamp_us { + private: + int64_t __rep = 0; +}; + +struct __attribute__((may_alias)) timestamp_ns { + private: + int64_t __rep = 0; +}; + +struct __attribute__((may_alias)) duration_D { + private: + int32_t __rep = 0; +}; + +struct __attribute__((may_alias)) duration_h { + private: + int32_t __rep = 0; +}; + +struct __attribute__((may_alias)) duration_m { + private: + int32_t __rep = 0; +}; + +struct __attribute__((may_alias)) duration_s { + private: + int64_t __rep = 0; +}; + +struct __attribute__((may_alias)) duration_ms { + private: + int64_t __rep = 0; +}; + +struct __attribute__((may_alias)) duration_us { + private: + int64_t __rep = 0; +}; + +struct __attribute__((may_alias)) duration_ns { + private: + int64_t __rep = 0; +}; + +struct inplace_t {}; + +inline constexpr inplace_t inplace{}; + +// [ ] assumes T is trivially copyable +template +struct __attribute__((may_alias)) optional { + private: + T __val; + bool __engaged; + + public: + __device__ constexpr optional() : __val{}, __engaged{false} {} + + template + __device__ constexpr optional(inplace_t, Args&&... args) + : __val{static_cast(args)...}, __engaged{true} + { + } + + __device__ constexpr optional(T val) : __val{val}, __engaged{true} {} + + constexpr optional(optional const&) = default; + + constexpr optional(optional&&) = default; + + constexpr optional& operator=(optional const&) = default; + + constexpr optional& operator=(optional&&) = default; + + constexpr ~optional() = default; + + __device__ constexpr bool has_value() const { return __engaged; } + + __device__ constexpr void reset() { __engaged = false; } + + __device__ constexpr T const& get() const { return __val; } + + __device__ constexpr T& get() { return __val; } + + __device__ constexpr T const* operator->() const { return &__val; } + + __device__ constexpr T* operator->() { return &__val; } + + __device__ constexpr T const& operator*() const { return __val; } + + __device__ constexpr T& operator*() { return __val; } + + __device__ constexpr T const& value() const { return __val; } + + __device__ constexpr T& value() { return __val; } + + __device__ constexpr explicit operator bool() const { return __engaged; } + + __device__ constexpr T value_or(T __v) const { return __engaged ? __val : __v; } +}; + +template +optional(T) -> optional; + +struct alignas(16) __attribute__((may_alias)) column_device_view_core { + private: + data_type __type = {}; + size_type __size = 0; + void const* __data = nullptr; + bitmask_type const* __null_mask = nullptr; + size_type __offset = 0; + column_device_view_core* __d_children = nullptr; + size_type __num_children = 0; + + public: + template + __device__ T const* head() const; + + __device__ size_type size() const; + + __device__ bool nullable() const; + + __device__ bitmask_type const* null_mask() const; + + __device__ size_type offset() const; + + __device__ bool is_valid(size_type idx) const; + + __device__ bool is_valid_nocheck(size_type idx) const; + + __device__ bool is_null(size_type idx) const; + + __device__ bool is_null_nocheck(size_type idx) const; + + __device__ bitmask_type get_mask_word(size_type word_index) const; + + template + __device__ T element(size_type idx) const; + + __device__ size_type num_child_columns() const; +}; + +#define CUDF_LTO_DECL(Type) \ + extern template __device__ Type const* column_device_view_core::head() const; + +CUDF_LTO_DECL(bool) +CUDF_LTO_DECL(int8_t) +CUDF_LTO_DECL(int16_t) +CUDF_LTO_DECL(int32_t) +CUDF_LTO_DECL(int64_t) +CUDF_LTO_DECL(uint8_t) +CUDF_LTO_DECL(uint16_t) +CUDF_LTO_DECL(uint32_t) +CUDF_LTO_DECL(uint64_t) +CUDF_LTO_DECL(float32_t) +CUDF_LTO_DECL(float64_t) +CUDF_LTO_DECL(timestamp_D) +CUDF_LTO_DECL(timestamp_h) +CUDF_LTO_DECL(timestamp_m) +CUDF_LTO_DECL(timestamp_s) +CUDF_LTO_DECL(timestamp_ms) +CUDF_LTO_DECL(timestamp_us) +CUDF_LTO_DECL(timestamp_ns) +CUDF_LTO_DECL(duration_D) +CUDF_LTO_DECL(duration_h) +CUDF_LTO_DECL(duration_m) +CUDF_LTO_DECL(duration_s) +CUDF_LTO_DECL(duration_ms) +CUDF_LTO_DECL(duration_us) +CUDF_LTO_DECL(duration_ns) + +#undef CUDF_LTO_DECL + +#define CUDF_LTO_DECL(Type) \ + extern template __device__ Type column_device_view_core::element(size_type idx) const; + +CUDF_LTO_DECL(bool) +CUDF_LTO_DECL(int8_t) +CUDF_LTO_DECL(int16_t) +CUDF_LTO_DECL(int32_t) +CUDF_LTO_DECL(int64_t) +CUDF_LTO_DECL(uint8_t) +CUDF_LTO_DECL(uint16_t) +CUDF_LTO_DECL(uint32_t) +CUDF_LTO_DECL(uint64_t) +CUDF_LTO_DECL(decimal32) +CUDF_LTO_DECL(decimal64) +CUDF_LTO_DECL(decimal128) +CUDF_LTO_DECL(float32_t) +CUDF_LTO_DECL(float64_t) +CUDF_LTO_DECL(string_view) +CUDF_LTO_DECL(timestamp_D) +CUDF_LTO_DECL(timestamp_h) +CUDF_LTO_DECL(timestamp_m) +CUDF_LTO_DECL(timestamp_s) +CUDF_LTO_DECL(timestamp_ms) +CUDF_LTO_DECL(timestamp_us) +CUDF_LTO_DECL(timestamp_ns) +CUDF_LTO_DECL(duration_D) +CUDF_LTO_DECL(duration_h) +CUDF_LTO_DECL(duration_m) +CUDF_LTO_DECL(duration_s) +CUDF_LTO_DECL(duration_ms) +CUDF_LTO_DECL(duration_us) +CUDF_LTO_DECL(duration_ns) + +#undef CUDF_LTO_DECL + +struct alignas(16) __attribute__((may_alias)) mutable_column_device_view_core { + private: + data_type __type = {}; + size_type __size = 0; + void const* __data = nullptr; + bitmask_type const* __null_mask = nullptr; + size_type __offset = 0; + mutable_column_device_view_core* __d_children = nullptr; + size_type __num_children = 0; + + public: + template + __device__ T* head() const; + + __device__ size_type size() const; + + __device__ bool nullable() const; + + __device__ bitmask_type* null_mask() const; + + __device__ size_type offset() const; + + __device__ bool is_valid(size_type idx) const; + + __device__ bool is_valid_nocheck(size_type idx) const; + + __device__ bool is_null(size_type idx) const; + + __device__ bool is_null_nocheck(size_type idx) const; + + __device__ bitmask_type get_mask_word(size_type word_index) const; + + template + __device__ T element(size_type idx) const; +}; + +#define CUDF_LTO_DECL(Type) \ + extern template __device__ Type* mutable_column_device_view_core::head() const; + +CUDF_LTO_DECL(bool) +CUDF_LTO_DECL(int8_t) +CUDF_LTO_DECL(int16_t) +CUDF_LTO_DECL(int32_t) +CUDF_LTO_DECL(int64_t) +CUDF_LTO_DECL(uint8_t) +CUDF_LTO_DECL(uint16_t) +CUDF_LTO_DECL(uint32_t) +CUDF_LTO_DECL(uint64_t) +CUDF_LTO_DECL(float32_t) +CUDF_LTO_DECL(float64_t) +CUDF_LTO_DECL(timestamp_D) +CUDF_LTO_DECL(timestamp_h) +CUDF_LTO_DECL(timestamp_m) +CUDF_LTO_DECL(timestamp_s) +CUDF_LTO_DECL(timestamp_ms) +CUDF_LTO_DECL(timestamp_us) +CUDF_LTO_DECL(timestamp_ns) +CUDF_LTO_DECL(duration_D) +CUDF_LTO_DECL(duration_h) +CUDF_LTO_DECL(duration_m) +CUDF_LTO_DECL(duration_s) +CUDF_LTO_DECL(duration_ms) +CUDF_LTO_DECL(duration_us) +CUDF_LTO_DECL(duration_ns) + +#undef CUDF_LTO_DECL + +#define CUDF_LTO_DECL(Type) \ + extern template __device__ Type mutable_column_device_view_core::element(size_type idx) \ + const; + +CUDF_LTO_DECL(bool) +CUDF_LTO_DECL(int8_t) +CUDF_LTO_DECL(int16_t) +CUDF_LTO_DECL(int32_t) +CUDF_LTO_DECL(int64_t) +CUDF_LTO_DECL(uint8_t) +CUDF_LTO_DECL(uint16_t) +CUDF_LTO_DECL(uint32_t) +CUDF_LTO_DECL(uint64_t) +CUDF_LTO_DECL(decimal32) +CUDF_LTO_DECL(decimal64) +CUDF_LTO_DECL(decimal128) +CUDF_LTO_DECL(float32_t) +CUDF_LTO_DECL(float64_t) +CUDF_LTO_DECL(string_view) +CUDF_LTO_DECL(timestamp_D) +CUDF_LTO_DECL(timestamp_h) +CUDF_LTO_DECL(timestamp_m) +CUDF_LTO_DECL(timestamp_s) +CUDF_LTO_DECL(timestamp_ms) +CUDF_LTO_DECL(timestamp_us) +CUDF_LTO_DECL(timestamp_ns) +CUDF_LTO_DECL(duration_D) +CUDF_LTO_DECL(duration_h) +CUDF_LTO_DECL(duration_m) +CUDF_LTO_DECL(duration_s) +CUDF_LTO_DECL(duration_ms) +CUDF_LTO_DECL(duration_us) +CUDF_LTO_DECL(duration_ns) + +#undef CUDF_LTO_DECL + +} // namespace lto +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/transform_params.cuh b/cpp/include/cudf/jit/transform_params.cuh new file mode 100644 index 000000000000..7feb6b0801dd --- /dev/null +++ b/cpp/include/cudf/jit/transform_params.cuh @@ -0,0 +1,28 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include +#include + +namespace CUDF_EXPORT cudf { + +namespace lto { + +struct transform_params { + void const* inputs = nullptr; + void* user_data = nullptr; + void const* outputs = nullptr; + void const* span_outputs = nullptr; + size_type row_index = 0; +}; + +} // namespace lto +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index e919e40af4d1..2aab2bc54815 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -61,6 +61,26 @@ std::unique_ptr transform( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +std::unique_ptr transform_lto_cuda_udf( + std::vector const& inputs, + std::string_view source_code, + std::string_view transform_function, + data_type output_type, + std::optional user_data = std::nullopt, + null_aware is_null_aware = null_aware::NO, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + +std::unique_ptr transform_lto_binary_udf( + std::vector const& inputs, + std::span source_code, + std::string_view transform_function, + data_type output_type, + std::optional user_data = std::nullopt, + null_aware is_null_aware = null_aware::NO, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + /** * @brief Creates a null_mask from `input` by converting `NaN` to null and * preserving existing null values and also returns new null_count. diff --git a/cpp/src/jit/build.hpp b/cpp/src/jit/build.hpp new file mode 100644 index 000000000000..6fbd4b60ac13 --- /dev/null +++ b/cpp/src/jit/build.hpp @@ -0,0 +1,41 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include + +#include + +namespace cudf { +namespace rtc { + +header_map get_jit_headers(); + +header_map get_jit_lto_headers(); + +std::span get_jit_options(); + +std::span get_jit_lto_options(); + +// input: LTO IR data +// output: nvJitLink object +int32_t get_current_device_physical_model() +{ + int32_t device; + // cudaGetDeviceCount() + CUDF_EXPECTS(cudaGetDevice(&device) == cudaSuccess, "Failed to get current CUDA device"); + + cudaDeviceProp props; + CUDF_EXPECTS(cudaGetDeviceProperties(&props, device) == cudaSuccess, + "Failed to get device properties"); + + return props.major * 10 + props.minor; +} + +std::vector get_all_device_physical_models(); + +} // namespace rtc +} // namespace cudf diff --git a/cpp/src/jit/lto/library.cu b/cpp/src/jit/lto/library.cu new file mode 100644 index 000000000000..ae8cd137e7ab --- /dev/null +++ b/cpp/src/jit/lto/library.cu @@ -0,0 +1,9 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include diff --git a/cpp/src/jit/lto/operators.inl.cuh b/cpp/src/jit/lto/operators.inl.cuh new file mode 100644 index 000000000000..1841cb738c18 --- /dev/null +++ b/cpp/src/jit/lto/operators.inl.cuh @@ -0,0 +1,566 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include +#include +#include + +namespace CUDF_EXPORT cudf { +namespace lto { + +template +using func = ast::detail::operator_functor; + +template +using null_func = ast::detail::operator_functor; + +using opcode = ast::ast_operator; + +#define CUDF_LTO_DEF(op, OP, type) \ + __device__ void operators::op(type* out, type const* a, type const* b) \ + { \ + auto ret = func{}(*lto::lift(a), *lto::lift(b)); \ + *out = *lto::lower(&ret); \ + } \ + \ + __device__ void operators::op( \ + lto::optional* out, lto::optional const* a, lto::optional const* b) \ + { \ + auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ + *out = *lto::lower(&ret); \ + } + +CUDF_LTO_DEF(add, ADD, int32_t); +CUDF_LTO_DEF(add, ADD, int64_t); +CUDF_LTO_DEF(add, ADD, uint32_t); +CUDF_LTO_DEF(add, ADD, uint64_t); +CUDF_LTO_DEF(add, ADD, float32_t); +CUDF_LTO_DEF(add, ADD, float64_t); +CUDF_LTO_DEF(add, ADD, decimal32); +CUDF_LTO_DEF(add, ADD, decimal64); +CUDF_LTO_DEF(add, ADD, decimal128); +CUDF_LTO_DEF(add, ADD, duration_D); +CUDF_LTO_DEF(add, ADD, duration_s); +CUDF_LTO_DEF(add, ADD, duration_ms); +CUDF_LTO_DEF(add, ADD, duration_ns); + +CUDF_LTO_DEF(sub, SUB, int32_t); +CUDF_LTO_DEF(sub, SUB, int64_t); +CUDF_LTO_DEF(sub, SUB, uint32_t); +CUDF_LTO_DEF(sub, SUB, uint64_t); +CUDF_LTO_DEF(sub, SUB, float32_t); +CUDF_LTO_DEF(sub, SUB, float64_t); +CUDF_LTO_DEF(sub, SUB, decimal32); +CUDF_LTO_DEF(sub, SUB, decimal64); +CUDF_LTO_DEF(sub, SUB, decimal128); +CUDF_LTO_DEF(sub, SUB, duration_D); +CUDF_LTO_DEF(sub, SUB, duration_s); +CUDF_LTO_DEF(sub, SUB, duration_ms); +CUDF_LTO_DEF(sub, SUB, duration_ns); + +CUDF_LTO_DEF(mul, MUL, int32_t); +CUDF_LTO_DEF(mul, MUL, int64_t); +CUDF_LTO_DEF(mul, MUL, uint32_t); +CUDF_LTO_DEF(mul, MUL, uint64_t); +CUDF_LTO_DEF(mul, MUL, float32_t); +CUDF_LTO_DEF(mul, MUL, float64_t); +CUDF_LTO_DEF(mul, MUL, decimal32); +CUDF_LTO_DEF(mul, MUL, decimal64); +CUDF_LTO_DEF(mul, MUL, decimal128); + +CUDF_LTO_DEF(div, DIV, int32_t); +CUDF_LTO_DEF(div, DIV, int64_t); +CUDF_LTO_DEF(div, DIV, uint32_t); +CUDF_LTO_DEF(div, DIV, uint64_t); +CUDF_LTO_DEF(div, DIV, float32_t); +CUDF_LTO_DEF(div, DIV, float64_t); +CUDF_LTO_DEF(div, DIV, decimal32); +CUDF_LTO_DEF(div, DIV, decimal64); +CUDF_LTO_DEF(div, DIV, decimal128); + +CUDF_LTO_DEF(mod, MOD, float32_t); +CUDF_LTO_DEF(mod, MOD, float64_t); + +CUDF_LTO_DEF(pymod, PYMOD, float32_t); +CUDF_LTO_DEF(pymod, PYMOD, float64_t); + +CUDF_LTO_DEF(pow, POW, float32_t); +CUDF_LTO_DEF(pow, POW, float64_t); + +#undef CUDF_LTO_DEF + +#define CUDF_LTO_DEF(op, OP, type) \ + __device__ void operators::op(bool* out, type const* a, type const* b) \ + { \ + auto ret = func{}(*lto::lift(a), *lto::lift(b)); \ + *out = *lto::lower(&ret); \ + } \ + \ + __device__ void operators::op(bool* out, optional const* a, optional const* b) \ + { \ + auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ + *out = **lto::lower(&ret); \ + } \ + \ + __device__ void operators::op( \ + optional* out, optional const* a, optional const* b) \ + { \ + auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ + *out = *lto::lower(&ret); \ + } + +CUDF_LTO_DEF(equal, EQUAL, bool); +CUDF_LTO_DEF(equal, EQUAL, int8_t); +CUDF_LTO_DEF(equal, EQUAL, int16_t); +CUDF_LTO_DEF(equal, EQUAL, int32_t); +CUDF_LTO_DEF(equal, EQUAL, int64_t); +CUDF_LTO_DEF(equal, EQUAL, uint8_t); +CUDF_LTO_DEF(equal, EQUAL, uint16_t); +CUDF_LTO_DEF(equal, EQUAL, uint32_t); +CUDF_LTO_DEF(equal, EQUAL, uint64_t); +CUDF_LTO_DEF(equal, EQUAL, float32_t); +CUDF_LTO_DEF(equal, EQUAL, float64_t); +CUDF_LTO_DEF(equal, EQUAL, decimal32); +CUDF_LTO_DEF(equal, EQUAL, decimal64); +CUDF_LTO_DEF(equal, EQUAL, decimal128); +CUDF_LTO_DEF(equal, EQUAL, timestamp_D); +CUDF_LTO_DEF(equal, EQUAL, timestamp_s); +CUDF_LTO_DEF(equal, EQUAL, timestamp_ms); +CUDF_LTO_DEF(equal, EQUAL, timestamp_us); +CUDF_LTO_DEF(equal, EQUAL, timestamp_ns); +CUDF_LTO_DEF(equal, EQUAL, duration_D); +CUDF_LTO_DEF(equal, EQUAL, duration_s); +CUDF_LTO_DEF(equal, EQUAL, duration_ms); +CUDF_LTO_DEF(equal, EQUAL, duration_ns); +CUDF_LTO_DEF(equal, EQUAL, string_view); + +CUDF_LTO_DEF(null_equal, NULL_EQUAL, bool); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, int8_t); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, int16_t); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, int32_t); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, int64_t); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, uint8_t); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, uint16_t); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, uint32_t); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, uint64_t); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, float32_t); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, float64_t); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, decimal32); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, decimal64); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, decimal128); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, timestamp_D); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, timestamp_s); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, timestamp_ms); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, timestamp_us); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, timestamp_ns); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, duration_D); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, duration_s); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, duration_ms); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, duration_ns); +CUDF_LTO_DEF(null_equal, NULL_EQUAL, string_view); + +#undef CUDF_LTO_DEF + +#define CUDF_LTO_DEF(op, OP, type) \ + __device__ void operators::op(bool* out, type const* a, type const* b) \ + { \ + auto ret = func{}(*lto::lift(a), *lto::lift(b)); \ + *out = *lto::lower(&ret); \ + } \ + \ + __device__ void operators::op( \ + optional* out, optional const* a, optional const* b) \ + { \ + auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ + *out = *lto::lower(&ret); \ + } + +CUDF_LTO_DEF(less, LESS, bool); +CUDF_LTO_DEF(less, LESS, int8_t); +CUDF_LTO_DEF(less, LESS, int16_t); +CUDF_LTO_DEF(less, LESS, int32_t); +CUDF_LTO_DEF(less, LESS, int64_t); +CUDF_LTO_DEF(less, LESS, uint8_t); +CUDF_LTO_DEF(less, LESS, uint16_t); +CUDF_LTO_DEF(less, LESS, uint32_t); +CUDF_LTO_DEF(less, LESS, uint64_t); +CUDF_LTO_DEF(less, LESS, float32_t); +CUDF_LTO_DEF(less, LESS, float64_t); +CUDF_LTO_DEF(less, LESS, decimal32); +CUDF_LTO_DEF(less, LESS, decimal64); +CUDF_LTO_DEF(less, LESS, decimal128); +CUDF_LTO_DEF(less, LESS, timestamp_D); +CUDF_LTO_DEF(less, LESS, timestamp_s); +CUDF_LTO_DEF(less, LESS, timestamp_ms); +CUDF_LTO_DEF(less, LESS, timestamp_us); +CUDF_LTO_DEF(less, LESS, timestamp_ns); +CUDF_LTO_DEF(less, LESS, duration_D); +CUDF_LTO_DEF(less, LESS, duration_s); +CUDF_LTO_DEF(less, LESS, duration_ms); +CUDF_LTO_DEF(less, LESS, duration_ns); +CUDF_LTO_DEF(less, LESS, string_view); + +CUDF_LTO_DEF(greater, GREATER, bool); +CUDF_LTO_DEF(greater, GREATER, int8_t); +CUDF_LTO_DEF(greater, GREATER, int16_t); +CUDF_LTO_DEF(greater, GREATER, int32_t); +CUDF_LTO_DEF(greater, GREATER, int64_t); +CUDF_LTO_DEF(greater, GREATER, uint8_t); +CUDF_LTO_DEF(greater, GREATER, uint16_t); +CUDF_LTO_DEF(greater, GREATER, uint32_t); +CUDF_LTO_DEF(greater, GREATER, uint64_t); +CUDF_LTO_DEF(greater, GREATER, float32_t); +CUDF_LTO_DEF(greater, GREATER, float64_t); +CUDF_LTO_DEF(greater, GREATER, decimal32); +CUDF_LTO_DEF(greater, GREATER, decimal64); +CUDF_LTO_DEF(greater, GREATER, decimal128); +CUDF_LTO_DEF(greater, GREATER, timestamp_D); +CUDF_LTO_DEF(greater, GREATER, timestamp_s); +CUDF_LTO_DEF(greater, GREATER, timestamp_ms); +CUDF_LTO_DEF(greater, GREATER, timestamp_us); +CUDF_LTO_DEF(greater, GREATER, timestamp_ns); +CUDF_LTO_DEF(greater, GREATER, duration_D); +CUDF_LTO_DEF(greater, GREATER, duration_s); +CUDF_LTO_DEF(greater, GREATER, duration_ms); +CUDF_LTO_DEF(greater, GREATER, duration_ns); +CUDF_LTO_DEF(greater, GREATER, string_view); + +CUDF_LTO_DEF(less_equal, LESS_EQUAL, bool); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, int8_t); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, int16_t); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, int32_t); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, int64_t); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, uint8_t); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, uint16_t); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, uint32_t); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, uint64_t); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, float32_t); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, float64_t); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, decimal32); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, decimal64); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, decimal128); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, timestamp_D); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, timestamp_s); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, timestamp_ms); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, timestamp_us); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, timestamp_ns); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, duration_D); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, duration_s); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, duration_ms); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, duration_ns); +CUDF_LTO_DEF(less_equal, LESS_EQUAL, string_view); + +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, bool); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, int8_t); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, int16_t); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, int32_t); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, int64_t); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, uint8_t); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, uint16_t); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, uint32_t); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, uint64_t); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, float32_t); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, float64_t); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, decimal32); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, decimal64); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, decimal128); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, timestamp_D); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, timestamp_s); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, timestamp_ms); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, timestamp_us); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, timestamp_ns); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, duration_D); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, duration_s); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, duration_ms); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, duration_ns); +CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, string_view); + +#undef CUDF_LTO_DEF + +#define CUDF_LTO_DEF(op, OP, type) \ + __device__ void operators::op(type* out, type const* a, type const* b) \ + { \ + auto ret = func{}(*lto::lift(a), *lto::lift(b)); \ + *out = *lto::lower(&ret); \ + } \ + \ + __device__ void operators::op( \ + optional* out, optional const* a, optional const* b) \ + { \ + auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ + *out = *lto::lower(&ret); \ + } + +CUDF_LTO_DEF(bitwise_and, BITWISE_AND, int32_t); +CUDF_LTO_DEF(bitwise_and, BITWISE_AND, int64_t); +CUDF_LTO_DEF(bitwise_and, BITWISE_AND, uint32_t); +CUDF_LTO_DEF(bitwise_and, BITWISE_AND, uint64_t); + +CUDF_LTO_DEF(bitwise_or, BITWISE_OR, int32_t); +CUDF_LTO_DEF(bitwise_or, BITWISE_OR, int64_t); +CUDF_LTO_DEF(bitwise_or, BITWISE_OR, uint32_t); +CUDF_LTO_DEF(bitwise_or, BITWISE_OR, uint64_t); + +CUDF_LTO_DEF(bitwise_xor, BITWISE_XOR, int32_t); +CUDF_LTO_DEF(bitwise_xor, BITWISE_XOR, int64_t); +CUDF_LTO_DEF(bitwise_xor, BITWISE_XOR, uint32_t); +CUDF_LTO_DEF(bitwise_xor, BITWISE_XOR, uint64_t); + +#undef CUDF_LTO_DEF + +#define CUDF_LTO_DEF(op, OP, type) \ + __device__ void operators::op(type* out, type const* a, type const* b) \ + { \ + auto ret = func{}(*lto::lift(a), *lto::lift(b)); \ + *out = *lto::lower(&ret); \ + } \ + \ + __device__ void operators::op( \ + optional* out, optional const* a, optional const* b) \ + { \ + auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ + *out = *lto::lower(&ret); \ + } + +CUDF_LTO_DEF(logical_and, LOGICAL_AND, bool); + +CUDF_LTO_DEF(null_logical_and, NULL_LOGICAL_AND, bool); + +CUDF_LTO_DEF(logical_or, LOGICAL_OR, bool); + +CUDF_LTO_DEF(null_logical_or, NULL_LOGICAL_OR, bool); + +#undef CUDF_LTO_DEF + +#define CUDF_LTO_DEF(op, OP, type) \ + __device__ void operators::op(type* out, type const* a) \ + { \ + auto ret = func{}(*lto::lift(a)); \ + *out = *lto::lower(&ret); \ + } \ + \ + __device__ void operators::op(optional* out, optional const* a) \ + { \ + auto ret = null_func{}(*lto::lift(a)); \ + *out = *lto::lower(&ret); \ + } + +CUDF_LTO_DEF(identity, IDENTITY, bool); +CUDF_LTO_DEF(identity, IDENTITY, int8_t); +CUDF_LTO_DEF(identity, IDENTITY, int16_t); +CUDF_LTO_DEF(identity, IDENTITY, int32_t); +CUDF_LTO_DEF(identity, IDENTITY, int64_t); +CUDF_LTO_DEF(identity, IDENTITY, uint8_t); +CUDF_LTO_DEF(identity, IDENTITY, uint16_t); +CUDF_LTO_DEF(identity, IDENTITY, uint32_t); +CUDF_LTO_DEF(identity, IDENTITY, uint64_t); +CUDF_LTO_DEF(identity, IDENTITY, float32_t); +CUDF_LTO_DEF(identity, IDENTITY, float64_t); +CUDF_LTO_DEF(identity, IDENTITY, decimal32); +CUDF_LTO_DEF(identity, IDENTITY, decimal64); +CUDF_LTO_DEF(identity, IDENTITY, decimal128); +CUDF_LTO_DEF(identity, IDENTITY, timestamp_D); +CUDF_LTO_DEF(identity, IDENTITY, timestamp_s); +CUDF_LTO_DEF(identity, IDENTITY, timestamp_ms); +CUDF_LTO_DEF(identity, IDENTITY, timestamp_us); +CUDF_LTO_DEF(identity, IDENTITY, timestamp_ns); +CUDF_LTO_DEF(identity, IDENTITY, duration_D); +CUDF_LTO_DEF(identity, IDENTITY, duration_s); +CUDF_LTO_DEF(identity, IDENTITY, duration_ms); +CUDF_LTO_DEF(identity, IDENTITY, duration_ns); +CUDF_LTO_DEF(identity, IDENTITY, string_view); + +CUDF_LTO_DEF(sin, SIN, float32_t); +CUDF_LTO_DEF(sin, SIN, float64_t); + +CUDF_LTO_DEF(cos, COS, float32_t); +CUDF_LTO_DEF(cos, COS, float64_t); + +CUDF_LTO_DEF(tan, TAN, float32_t); +CUDF_LTO_DEF(tan, TAN, float64_t); + +CUDF_LTO_DEF(arcsin, ARCSIN, float32_t); +CUDF_LTO_DEF(arcsin, ARCSIN, float64_t); + +CUDF_LTO_DEF(arccos, ARCCOS, float32_t); +CUDF_LTO_DEF(arccos, ARCCOS, float64_t); + +CUDF_LTO_DEF(arctan, ARCTAN, float32_t); +CUDF_LTO_DEF(arctan, ARCTAN, float64_t); + +CUDF_LTO_DEF(sinh, SINH, float32_t); +CUDF_LTO_DEF(sinh, SINH, float64_t); + +CUDF_LTO_DEF(cosh, COSH, float32_t); +CUDF_LTO_DEF(cosh, COSH, float64_t); + +CUDF_LTO_DEF(tanh, TANH, float32_t); +CUDF_LTO_DEF(tanh, TANH, float64_t); + +CUDF_LTO_DEF(arcsinh, ARCSINH, float32_t); +CUDF_LTO_DEF(arcsinh, ARCSINH, float64_t); + +CUDF_LTO_DEF(arccosh, ARCCOSH, float32_t); +CUDF_LTO_DEF(arccosh, ARCCOSH, float64_t); + +CUDF_LTO_DEF(arctanh, ARCTANH, float32_t); +CUDF_LTO_DEF(arctanh, ARCTANH, float64_t); + +CUDF_LTO_DEF(exp, EXP, float32_t); +CUDF_LTO_DEF(exp, EXP, float64_t); + +CUDF_LTO_DEF(log, LOG, float32_t); +CUDF_LTO_DEF(log, LOG, float64_t); + +CUDF_LTO_DEF(cbrt, CBRT, float32_t); +CUDF_LTO_DEF(cbrt, CBRT, float64_t); + +CUDF_LTO_DEF(ceil, CEIL, float32_t); +CUDF_LTO_DEF(ceil, CEIL, float64_t); + +CUDF_LTO_DEF(floor, FLOOR, float32_t); +CUDF_LTO_DEF(floor, FLOOR, float64_t); + +CUDF_LTO_DEF(abs, ABS, int32_t); +CUDF_LTO_DEF(abs, ABS, int64_t); +CUDF_LTO_DEF(abs, ABS, float32_t); +CUDF_LTO_DEF(abs, ABS, float64_t); + +CUDF_LTO_DEF(rint, RINT, float32_t); +CUDF_LTO_DEF(rint, RINT, float64_t); + +CUDF_LTO_DEF(bit_invert, BIT_INVERT, uint32_t); +CUDF_LTO_DEF(bit_invert, BIT_INVERT, uint64_t); +CUDF_LTO_DEF(bit_invert, BIT_INVERT, int32_t); +CUDF_LTO_DEF(bit_invert, BIT_INVERT, int64_t); + +#undef CUDF_LTO_DEF + +#define CUDF_LTO_DEF(op, OP, ret_type, type) \ + extern __device__ void operators::op(ret_type* out, type const* a) \ + { \ + auto ret = func{}(*lto::lift(a)); \ + *out = *lto::lower(&ret); \ + } \ + \ + extern __device__ void operators::op(optional* out, optional const* a) \ + { \ + auto ret = null_func{}(*lto::lift(a)); \ + *out = *lto::lower(&ret); \ + } + +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, bool); +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, int8_t); +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, int16_t); +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, int32_t); +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, int64_t); +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, uint8_t); +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, uint16_t); +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, uint32_t); +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, uint64_t); +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, float32_t); +CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, float64_t); + +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, bool); +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, int8_t); +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, int16_t); +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, int32_t); +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, int64_t); +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, uint8_t); +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, uint16_t); +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, uint32_t); +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, uint64_t); +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, float32_t); +CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, float64_t); + +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, bool); +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, int8_t); +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, int16_t); +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, int32_t); +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, int64_t); +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, uint8_t); +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, uint16_t); +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, uint32_t); +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, uint64_t); +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, float32_t); +CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, float64_t); + +#undef CUDF_LTO_DEF + +#define CUDF_LTO_DEF(op, OP, type) \ + __device__ void operators::op(bool* out, type const* a) \ + { \ + auto ret = func{}(*lto::lift(a)); \ + *out = *lto::lower(&ret); \ + } \ + \ + __device__ void operators::op(bool* out, optional const* a) \ + { \ + auto ret = null_func{}(*lto::lift(a)); \ + *out = *lto::lower(&ret); \ + } \ + \ + __device__ void operators::op(optional* out, optional const* a) \ + { \ + auto ret = null_func{}(*lto::lift(a)); \ + *out = *lto::lower(&ret); \ + } + +CUDF_LTO_DEF(is_null, IS_NULL, bool); +CUDF_LTO_DEF(is_null, IS_NULL, int8_t); +CUDF_LTO_DEF(is_null, IS_NULL, int16_t); +CUDF_LTO_DEF(is_null, IS_NULL, int32_t); +CUDF_LTO_DEF(is_null, IS_NULL, int64_t); +CUDF_LTO_DEF(is_null, IS_NULL, uint8_t); +CUDF_LTO_DEF(is_null, IS_NULL, uint16_t); +CUDF_LTO_DEF(is_null, IS_NULL, uint32_t); +CUDF_LTO_DEF(is_null, IS_NULL, uint64_t); +CUDF_LTO_DEF(is_null, IS_NULL, float32_t); +CUDF_LTO_DEF(is_null, IS_NULL, float64_t); +CUDF_LTO_DEF(is_null, IS_NULL, decimal32); +CUDF_LTO_DEF(is_null, IS_NULL, decimal64); +CUDF_LTO_DEF(is_null, IS_NULL, decimal128); +CUDF_LTO_DEF(is_null, IS_NULL, timestamp_D); +CUDF_LTO_DEF(is_null, IS_NULL, timestamp_s); +CUDF_LTO_DEF(is_null, IS_NULL, timestamp_ms); +CUDF_LTO_DEF(is_null, IS_NULL, timestamp_us); +CUDF_LTO_DEF(is_null, IS_NULL, timestamp_ns); +CUDF_LTO_DEF(is_null, IS_NULL, duration_D); +CUDF_LTO_DEF(is_null, IS_NULL, duration_s); +CUDF_LTO_DEF(is_null, IS_NULL, duration_ms); +CUDF_LTO_DEF(is_null, IS_NULL, duration_ns); +CUDF_LTO_DEF(is_null, IS_NULL, string_view); + +#undef CUDF_LTO_DEF + +#define CUDF_LTO_DEF(op, OP, type) \ + __device__ void operators::op(bool* out, type const* a) \ + { \ + auto ret = func{}(*lto::lift(a)); \ + *out = *lto::lower(&ret); \ + } \ + \ + __device__ void operators::op(optional* out, optional const* a) \ + { \ + auto ret = null_func{}(*lto::lift(a)); \ + *out = *lto::lower(&ret); \ + } + +CUDF_LTO_DEF(logical_not, NOT, bool); +CUDF_LTO_DEF(logical_not, NOT, int8_t); +CUDF_LTO_DEF(logical_not, NOT, int16_t); +CUDF_LTO_DEF(logical_not, NOT, int32_t); +CUDF_LTO_DEF(logical_not, NOT, int64_t); +CUDF_LTO_DEF(logical_not, NOT, uint8_t); +CUDF_LTO_DEF(logical_not, NOT, uint16_t); +CUDF_LTO_DEF(logical_not, NOT, uint32_t); +CUDF_LTO_DEF(logical_not, NOT, uint64_t); + +#undef CUDF_LTO_DEF + +} // namespace lto +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/lto/types.inl.cuh b/cpp/src/jit/lto/types.inl.cuh new file mode 100644 index 000000000000..de727e56c686 --- /dev/null +++ b/cpp/src/jit/lto/types.inl.cuh @@ -0,0 +1,365 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include + +namespace CUDF_EXPORT cudf { +namespace lto { + +[[nodiscard]] __device__ size_type string_view::size_bytes() const +{ + return lto::lift(this)->size_bytes(); +} + +[[nodiscard]] __device__ size_type string_view::length() const { return lto::lift(this)->length(); } + +[[nodiscard]] __device__ char const* string_view::data() const { return lto::lift(this)->data(); } + +[[nodiscard]] __device__ bool string_view::empty() const { return lto::lift(this)->empty(); } + +__device__ char_utf8 string_view::operator[](size_type pos) const +{ + return lto::lift(this)->operator[](pos); +} + +[[nodiscard]] __device__ size_type string_view::byte_offset(size_type pos) const +{ + return lto::lift(this)->byte_offset(pos); +} + +[[nodiscard]] __device__ int string_view::compare(string_view const& str) const +{ + return lto::lift(this)->compare(*lto::lift(&str)); +} + +__device__ int string_view::compare(char const* str, size_type bytes) const +{ + return lto::lift(this)->compare(str, bytes); +} + +__device__ bool string_view::operator==(string_view const& rhs) const +{ + return lto::lift(this)->operator==(*lto::lift(&rhs)); +} + +__device__ bool string_view::operator!=(string_view const& rhs) const +{ + return lto::lift(this)->operator!=(*lto::lift(&rhs)); +} + +__device__ bool string_view::operator<(string_view const& rhs) const +{ + return lto::lift(this)->operator<(*lto::lift(&rhs)); +} + +__device__ bool string_view::operator>(string_view const& rhs) const +{ + return lto::lift(this)->operator>(*lto::lift(&rhs)); +} + +__device__ bool string_view::operator<=(string_view const& rhs) const +{ + return lto::lift(this)->operator<=(*lto::lift(&rhs)); +} + +__device__ bool string_view::operator>=(string_view const& rhs) const +{ + return lto::lift(this)->operator>=(*lto::lift(&rhs)); +} + +[[nodiscard]] __device__ size_type string_view::find(string_view const& str, + size_type pos, + size_type count) const +{ + return lto::lift(this)->find(*lto::lift(&str), pos, count); +} + +__device__ size_type string_view::find(char const* str, + size_type bytes, + size_type pos, + size_type count) const +{ + return lto::lift(this)->find(str, bytes, pos, count); +} + +[[nodiscard]] __device__ size_type string_view::find(char_utf8 character, + size_type pos, + size_type count) const +{ + return lto::lift(this)->find(character, pos, count); +} + +[[nodiscard]] __device__ size_type string_view::rfind(string_view const& str, + size_type pos, + size_type count) const +{ + return lto::lift(this)->rfind(*lto::lift(&str), pos, count); +} + +__device__ size_type string_view::rfind(char const* str, + size_type bytes, + size_type pos, + size_type count) const +{ + return lto::lift(this)->rfind(str, bytes, pos, count); +} + +[[nodiscard]] __device__ size_type string_view::rfind(char_utf8 character, + size_type pos, + size_type count) const +{ + return lto::lift(this)->rfind(character, pos, count); +} + +[[nodiscard]] __device__ string_view string_view::substr(size_type start, size_type length) const +{ + auto ret = lto::lift(this)->substr(start, length); + return *lto::lower(&ret); +} + +template +__device__ T const* column_device_view_core::head() const +{ + return lto::lower(lto::lift(this)->head>()); +} + +#define CUDF_LTO_INST(Type) \ + template __device__ Type const* column_device_view_core::head() const + +CUDF_LTO_INST(bool); +CUDF_LTO_INST(int8_t); +CUDF_LTO_INST(int16_t); +CUDF_LTO_INST(int32_t); +CUDF_LTO_INST(int64_t); +CUDF_LTO_INST(uint8_t); +CUDF_LTO_INST(uint16_t); +CUDF_LTO_INST(uint32_t); +CUDF_LTO_INST(uint64_t); +CUDF_LTO_INST(float32_t); +CUDF_LTO_INST(float64_t); +CUDF_LTO_INST(timestamp_D); +CUDF_LTO_INST(timestamp_h); +CUDF_LTO_INST(timestamp_m); +CUDF_LTO_INST(timestamp_s); +CUDF_LTO_INST(timestamp_ms); +CUDF_LTO_INST(timestamp_us); +CUDF_LTO_INST(timestamp_ns); +CUDF_LTO_INST(duration_D); +CUDF_LTO_INST(duration_h); +CUDF_LTO_INST(duration_m); +CUDF_LTO_INST(duration_s); +CUDF_LTO_INST(duration_ms); +CUDF_LTO_INST(duration_us); +CUDF_LTO_INST(duration_ns); + +#undef CUDF_LTO_INST + +__device__ size_type column_device_view_core::size() const { return lto::lift(this)->size(); } + +__device__ bool column_device_view_core::nullable() const { return lto::lift(this)->nullable(); } + +__device__ bitmask_type const* column_device_view_core::null_mask() const +{ + return lto::lift(this)->null_mask(); +} + +__device__ size_type column_device_view_core::offset() const { return lto::lift(this)->offset(); } + +__device__ bool column_device_view_core::is_valid(size_type index) const +{ + return lto::lift(this)->is_valid(index); +} + +__device__ bool column_device_view_core::is_valid_nocheck(size_type index) const +{ + return lto::lift(this)->is_valid_nocheck(index); +} + +__device__ bool column_device_view_core::is_null(size_type index) const +{ + return lto::lift(this)->is_null(index); +} + +__device__ bool column_device_view_core::is_null_nocheck(size_type index) const +{ + return lto::lift(this)->is_null_nocheck(index); +} + +__device__ bitmask_type column_device_view_core::get_mask_word(size_type index) const +{ + return lto::lift(this)->get_mask_word(index); +} + +template +__device__ T column_device_view_core::element(size_type index) const +{ + auto ret = lto::lift(this)->element>(index); + return *lto::lower(&ret); +} + +#define CUDF_LTO_INST(Type) \ + template __device__ Type column_device_view_core::element(size_type idx) const + +CUDF_LTO_INST(bool); +CUDF_LTO_INST(int8_t); +CUDF_LTO_INST(int16_t); +CUDF_LTO_INST(int32_t); +CUDF_LTO_INST(int64_t); +CUDF_LTO_INST(uint8_t); +CUDF_LTO_INST(uint16_t); +CUDF_LTO_INST(uint32_t); +CUDF_LTO_INST(uint64_t); +CUDF_LTO_INST(decimal32); +CUDF_LTO_INST(decimal64); +CUDF_LTO_INST(decimal128); +CUDF_LTO_INST(float32_t); +CUDF_LTO_INST(float64_t); +CUDF_LTO_INST(string_view); +CUDF_LTO_INST(timestamp_D); +CUDF_LTO_INST(timestamp_h); +CUDF_LTO_INST(timestamp_m); +CUDF_LTO_INST(timestamp_s); +CUDF_LTO_INST(timestamp_ms); +CUDF_LTO_INST(timestamp_us); +CUDF_LTO_INST(timestamp_ns); +CUDF_LTO_INST(duration_D); +CUDF_LTO_INST(duration_h); +CUDF_LTO_INST(duration_m); +CUDF_LTO_INST(duration_s); +CUDF_LTO_INST(duration_ms); +CUDF_LTO_INST(duration_us); +CUDF_LTO_INST(duration_ns); + +#undef CUDF_LTO_INST + +__device__ size_type column_device_view_core::num_child_columns() const +{ + return lto::lift(this)->num_child_columns(); +} + +template +__device__ T* mutable_column_device_view_core::head() const +{ + return lto::lower(lto::lift(this)->head>()); +} + +#define CUDF_LTO_INST(Type) \ + template __device__ Type* mutable_column_device_view_core::head() const + +CUDF_LTO_INST(bool); +CUDF_LTO_INST(int8_t); +CUDF_LTO_INST(int16_t); +CUDF_LTO_INST(int32_t); +CUDF_LTO_INST(int64_t); +CUDF_LTO_INST(uint8_t); +CUDF_LTO_INST(uint16_t); +CUDF_LTO_INST(uint32_t); +CUDF_LTO_INST(uint64_t); +CUDF_LTO_INST(float32_t); +CUDF_LTO_INST(float64_t); +CUDF_LTO_INST(timestamp_D); +CUDF_LTO_INST(timestamp_h); +CUDF_LTO_INST(timestamp_m); +CUDF_LTO_INST(timestamp_s); +CUDF_LTO_INST(timestamp_ms); +CUDF_LTO_INST(timestamp_us); +CUDF_LTO_INST(timestamp_ns); +CUDF_LTO_INST(duration_D); +CUDF_LTO_INST(duration_h); +CUDF_LTO_INST(duration_m); +CUDF_LTO_INST(duration_s); +CUDF_LTO_INST(duration_ms); +CUDF_LTO_INST(duration_us); +CUDF_LTO_INST(duration_ns); + +#undef CUDF_LTO_INST + +__device__ size_type mutable_column_device_view_core::size() const +{ + return lto::lift(this)->size(); +} + +__device__ bool mutable_column_device_view_core::nullable() const +{ + return lto::lift(this)->nullable(); +} + +__device__ bitmask_type* mutable_column_device_view_core::null_mask() const +{ + return lto::lift(this)->null_mask(); +} + +__device__ size_type mutable_column_device_view_core::offset() const +{ + return lto::lift(this)->offset(); +} + +__device__ bool mutable_column_device_view_core::is_valid(size_type index) const +{ + return lto::lift(this)->is_valid(index); +} + +__device__ bool mutable_column_device_view_core::is_valid_nocheck(size_type index) const +{ + return lto::lift(this)->is_valid_nocheck(index); +} + +__device__ bool mutable_column_device_view_core::is_null(size_type index) const +{ + return lto::lift(this)->is_null(index); +} + +__device__ bool mutable_column_device_view_core::is_null_nocheck(size_type index) const +{ + return lto::lift(this)->is_null_nocheck(index); +} + +__device__ bitmask_type mutable_column_device_view_core::get_mask_word(size_type index) const +{ + return lto::lift(this)->get_mask_word(index); +} + +template +__device__ T mutable_column_device_view_core::element(size_type index) const +{ + auto ret = lto::lift(this)->element>(index); + return *lto::lower(&ret); +} + +#define CUDF_LTO_INST(Type) \ + template __device__ Type mutable_column_device_view_core::element(size_type idx) const + +CUDF_LTO_INST(bool); +CUDF_LTO_INST(int8_t); +CUDF_LTO_INST(int16_t); +CUDF_LTO_INST(int32_t); +CUDF_LTO_INST(int64_t); +CUDF_LTO_INST(uint8_t); +CUDF_LTO_INST(uint16_t); +CUDF_LTO_INST(uint32_t); +CUDF_LTO_INST(uint64_t); +CUDF_LTO_INST(float32_t); +CUDF_LTO_INST(float64_t); +CUDF_LTO_INST(timestamp_D); +CUDF_LTO_INST(timestamp_h); +CUDF_LTO_INST(timestamp_m); +CUDF_LTO_INST(timestamp_s); +CUDF_LTO_INST(timestamp_ms); +CUDF_LTO_INST(timestamp_us); +CUDF_LTO_INST(timestamp_ns); +CUDF_LTO_INST(duration_D); +CUDF_LTO_INST(duration_h); +CUDF_LTO_INST(duration_m); +CUDF_LTO_INST(duration_s); +CUDF_LTO_INST(duration_ms); +CUDF_LTO_INST(duration_us); +CUDF_LTO_INST(duration_ns); + +#undef CUDF_LTO_INST + +} // namespace lto +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp new file mode 100644 index 000000000000..1a52e8629db5 --- /dev/null +++ b/cpp/src/jit/rtc/cache.cpp @@ -0,0 +1,27 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "jit/rtc/cache.hpp" + +#include + +namespace cudf { +namespace rtc { + +void cache_t::store_blob(sha256_hash const& sha, blob binary) { CUDF_FUNC_RANGE(); } + +blob cache_t::query_blob(sha256_hash const& sha) { CUDF_FUNC_RANGE(); } + +void cache_t::store_fragment(sha256_hash const& sha, fragment frag) { CUDF_FUNC_RANGE(); } + +fragment cache_t::query_fragment(sha256_hash const& sha) { CUDF_FUNC_RANGE(); } + +void cache_t::store_module(sha256_hash const& sha, module mod) { CUDF_FUNC_RANGE(); } + +module cache_t::query_module(sha256_hash const& sha) { CUDF_FUNC_RANGE(); } + +} // namespace rtc +} // namespace cudf \ No newline at end of file diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp new file mode 100644 index 000000000000..589d9755048d --- /dev/null +++ b/cpp/src/jit/rtc/cache.hpp @@ -0,0 +1,163 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include +#include +#include + +namespace cudf { +namespace rtc { + +struct rwlock_t { + static constexpr size_t WRITE_STATE = ~(size_t)0; + + private: + size_t state_; + + public: + rwlock_t() : state_{0} {} + rwlock_t(rwlock_t const&) = default; + rwlock_t& operator=(rwlock_t const&) = default; + rwlock_t(rwlock_t&&) = default; + rwlock_t& operator=(rwlock_t&&) = default; + ~rwlock_t() = default; + + void lock_read() {} + + void unlock_read() + { + std::atomic_ref state{state_}; + state.fetch_sub(1, std::memory_order_relaxed); + } + + void lock_write() {} + + void unlock_write() + { + std::atomic_ref state{state_}; + state.store(0, std::memory_order_release); + } +}; + +struct read_guard { + private: + rwlock_t& lock_; + + public: + read_guard(rwlock_t& lock) : lock_{lock} { lock_.lock_read(); } + + read_guard(read_guard const&) = delete; + read_guard& operator=(read_guard const&) = delete; + read_guard(read_guard&&) = delete; + read_guard& operator=(read_guard&&) = delete; + + ~read_guard() { lock_.unlock_read(); } +}; + +struct write_guard { + private: + rwlock_t& lock_; + + public: + write_guard(rwlock_t& lock) : lock_{lock} { lock_.lock_write(); } + + write_guard(write_guard const&) = delete; + write_guard& operator=(write_guard const&) = delete; + write_guard(write_guard&&) = delete; + write_guard& operator=(write_guard&&) = delete; + + ~write_guard() { lock_.unlock_write(); } +}; + +struct cache_t { + private: + struct statistics { + uint64_t memory_hits = 0; + uint64_t memory_misses = 0; + uint64_t disk_hits = 0; + uint64_t disk_misses = 0; + }; + + struct limits { + uint64_t max_blobs_memory_size = UINT64_MAX; + uint64_t max_num_fragments = UINT64_MAX; + uint64_t max_num_modules = UINT64_MAX; + }; + + alignas(std::hardware_destructive_interference_size) rwlock_t blobs_lock_; + + std::unordered_map blobs_; + + uint64_t blobs_memory_size_; + + alignas(std::hardware_destructive_interference_size) rwlock_t fragments_lock_; + + std::unordered_map fragments_; + + alignas(std::hardware_destructive_interference_size) rwlock_t modules_lock_; + + std::unordered_map modules_; + + alignas(std::hardware_destructive_interference_size) statistics stats_; + + public: + // [ ] memory usage + // [ ] the object or blob being cached + // [ ] best we can do is LRU? + + // [ ] when writing to path; store at target+".tmp${consistent_rand}"; then rename to target + // [ ] local in-memory cache with memory limit & LRU eviction policy + // + // [ ] disk cache + // + // [ ] to use key as file id, convert to hex string + cache_t() = default; + cache_t(cache_t const&) = delete; + cache_t& operator=(cache_t const&) = delete; + cache_t(cache_t&&) = delete; + cache_t& operator=(cache_t&&) = delete; + ~cache_t(); + + // [ ] file cache path? + void store_blob(sha256_hash const& sha, blob binary); + + blob query_blob(sha256_hash const& sha); + + void store_fragment(sha256_hash const& sha, fragment frag); + + fragment query_fragment(sha256_hash const& sha); + + void store_module(sha256_hash const& sha, module mod); + + module query_module(sha256_hash const& sha); +}; + +// [ ] environment variables to control: +// [ ] cache path +// [ ] cache entries limit +// [ ] disable caching +// [ ] cache statistics: hits, misses, etc. +// [ ] cache tree pre-loading at startup +// [ ] on startup, log cache path, loading information, etc. + +cache_t& global_cache(); + +void preload_cache(); + +// [ ] pre-load cuda +// [ ] if a user provides a key, use: USER_KEY+${key}, otherwise use sha256 of contents +// [ ] use resource type in key to avoid collisions +void compile_operator(); + +void link_operator(); + +} // namespace rtc +} // namespace cudf diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp new file mode 100644 index 000000000000..668f7c76723a --- /dev/null +++ b/cpp/src/jit/rtc/rtc.cpp @@ -0,0 +1,214 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "jit/rtc/rtc.hpp" + +#include +#include + +#include +#include + +#include +#include + +#define CUDF_CONCATENATE_DETAIL(x, y) x##y +#define CUDF_CONCATENATE(x, y) CUDF_CONCATENATE_DETAIL(x, y) + +#define CUDF_DEFER(...) ::cudf::rtc::defer CUDF_CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) + +namespace cudf { +namespace rtc { + +template +struct defer { + private: + T func_; + + public: + template + defer(Args&&... args) : func_{std::forward(args)...} + { + } + defer(defer const&) = delete; + defer& operator=(defer const&) = delete; + defer(defer&&) = delete; + defer& operator=(defer&&) = delete; + ~defer() { func_(); } +}; + +fragment fragment_t::load(fragment_t::load_params const& params) { CUDF_FUNC_RANGE(); } + +fragment fragment_t::compile(fragment_t::compile_params const& params) { CUDF_FUNC_RANGE(); } + +blob fragment_t::get_lto_ir() const { CUDF_FUNC_RANGE(); } + +blob fragment_t::get_cubin() const { CUDF_FUNC_RANGE(); } + +blob module_t::link_as_cubin(const link_params& params) { CUDF_FUNC_RANGE(); } + +module module_t::load(blob_view cubin) { CUDF_FUNC_RANGE(); } + +module module_t::link(const link_params& params) { CUDF_FUNC_RANGE(); } + +function_ref module_t::get_function(char const* name) const { CUDF_FUNC_RANGE(); } + +nvJitLinkHandle load_fragment_from(std::span source, + char const* name, + std::span link_options, + nvJitLinkInputType type) +{ + CUDF_FUNC_RANGE(); + + // auto sm = get_device_compute_model(); + // auto arch = std::format("-arch=sm_{}", sm); + // const char* link_options[] = {"-lto", arch.c_str()}; + + nvJitLinkHandle handle; + CUDF_EXPECTS( + nvJitLinkCreate(&handle, std::size(link_options), link_options.data()) == NVJITLINK_SUCCESS, + "Failed to create nvJitLink handle"); + CUDF_EXPECTS( + nvJitLinkAddData(handle, type, source.data(), source.size_bytes(), name) == NVJITLINK_SUCCESS, + "Failed to add LTO fatbin data to nvJitLink handle"); + CUDF_EXPECTS(nvJitLinkComplete(handle) == NVJITLINK_SUCCESS, + "Failed to complete nvJitLink handle"); + + return handle; +} + +// [ ] hash function: runtime? driver? rapids version? etc. + +// [ ] cudf_jit_header_map{"": "int x\0", "intval": "int x\0"} + cuda/std -- always +// available? [ ] when should they be hashed? + +// [ ] includes +// [ ] defines +// [ ] link flags +// [ ] compile flags +// [ ] compile for all CUDF architectures? +// input: CUDA code as string +// output: LTO IR code +std::vector compile_fragment(std::span headers, + std::span include_names, + std::span options) +{ + CUDF_FUNC_RANGE(); + + // [ ] create hash + // [ ] serialize options and headers + + // auto sm = get_device_compute_model(); + // auto arch = std::format("--gpu-architecture=compute_{}", sm); + // char const* opts[] = {arch.c_str(), "--dlto", "--relocatable-device-code=true"}; + + nvrtcProgram program; + nvrtcCreateProgram(&program, + "", + "program", + static_cast(headers.size()), + headers.data(), + include_names.data()); + nvrtcCompileProgram(program, static_cast(options.size()), options.data()); + + // nvrtcGetProgramLog(nvrtcProgram prog, char *log) + // nvrtcGetProgramLogSize(nvrtcProgram prog, size_t *logSizeRet) + + size_t lto_ir_size; + nvrtcGetLTOIRSize(program, <o_ir_size); + + std::vector lto_ir; + lto_ir.resize(lto_ir_size); + nvrtcGetLTOIR(program, (char*)lto_ir.data()); + + // nvrtcGetErrorString() + // nvrtcGetCUBIN() + // nvrtcGetCUBINSize() + + return lto_ir; +} + +// input: LTO IR fragments +// output: Linked/Finalized CUBIN. Contains kernel(s) from all fragments +std::vector link_fragments(std::span> fragments, + std::span fragment_types, + std::span names, + std::span link_options) +{ + CUDF_FUNC_RANGE(); + + // [ ] create hash + // [ ] serialize link options, names, and fragments + // [ ] span exact_hash; request from user + + // auto sm = get_device_compute_model(); + // auto arch = std::format("--gpu-architecture=compute_{}", sm); + // const char* link_options[] = {"-lto", arch.c_str()}; + + std::span ltoIR1 = fragments[0]; + char const* ltoIR1_name = names[0]; + std::span ltoIR2 = fragments[1]; + char const* ltoIR2_name = names[1]; + nvJitLinkHandle handle; + nvJitLinkCreate(&handle, static_cast(std::size(link_options)), link_options.data()); + nvJitLinkAddData(handle, NVJITLINK_INPUT_LTOIR, (void*)ltoIR1.data(), ltoIR1.size(), ltoIR1_name); + nvJitLinkAddData(handle, NVJITLINK_INPUT_LTOIR, (void*)ltoIR2.data(), ltoIR2.size(), ltoIR2_name); + + // Call to nvJitLinkComplete causes linker to link together the two LTO IR modules, do + // optimization on the linked LTO IR, and generate cubin from it. + nvJitLinkComplete(handle); + + // nvJitLinkGetErrorLog() + // nvJitLinkGetInfoLog() + + // get linked cubin + size_t cubinSize; + nvJitLinkGetLinkedCubinSize(handle, &cubinSize); + + std::vector cubin; + cubin.resize(cubinSize); + nvJitLinkGetLinkedCubin(handle, cubin.data()); + nvJitLinkDestroy(&handle); + return cubin; +} + +// input: linked CUBIN +// output: CUmodule with launchable kernels +CUmodule create_module(std::span cubin) +{ + CUDF_FUNC_RANGE(); + + CUmodule module; + // cubin is linked, so now load it + cuModuleLoadData(&module, cubin.data()); + // cuModuleGetFunctionCount(unsigned int *count, CUmodule mod); + // cuModuleEnumerateFunctions + // cuModuleGetFunction(CUfunction *hfunc, CUmodule hmod, const char *name) + // cuLibraryGetKernel(CUkernel *pKernel, CUlibrary library, const char *name) + // cuLibraryGetKernelCount(unsigned int *count, CUlibrary lib) + + return module; +} + +// SIMPLE KEY: from user +// COMPLEX KEY: sha256 of all parameters and blobs involved; + driver + runtime +// + +// [ ] All these functions should have wrappers that cache results and request an optional key from +// the user + +// [ ] Method to pre-compile library and reuse it across multiple operators; compile_library(); +// compile_library_cached() + +// [ ] jit_key(key) -> key+driver+CUDA_versions+CUDA_runtime_versions+device_compute_models +void setup_context() +{ + // #include +} + +} // namespace rtc + +} // namespace cudf diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp new file mode 100644 index 000000000000..bcc89f5c83cf --- /dev/null +++ b/cpp/src/jit/rtc/rtc.hpp @@ -0,0 +1,120 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include +#include +#include +#include + +extern "C" { +typedef struct nvJitLink* nvJitLinkHandle; +typedef struct CUmod_st* CUmodule; +} + +namespace cudf { +namespace rtc { + +enum class binary_type : int8_t { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; + +using blob_t = std::vector; +using blob = std::shared_ptr; + +using blob_view = std::span; + +struct [[nodiscard]] header_map { + std::span include_names = {}; // null-terminated header include names + std::span headers = {}; // null-terminated header contents + std::span header_sizes = {}; // sizes of each header (excluding null-terminator) +}; + +struct fragment_t; + +using fragment = std::shared_ptr; + +struct fragment_t { + private: + nvJitLinkHandle handle_; + binary_type type_; + + public: + struct load_params { + blob_view source = {}; + char const* name = nullptr; + binary_type type = binary_type::LTO_IR; + }; + + struct compile_params { + header_map headers = {}; + std::span options = {}; + binary_type target_type = binary_type::LTO_IR; + }; + + [[nodiscard]] static fragment load(load_params const& params); + + [[nodiscard]] static fragment compile(compile_params const& params); + + fragment_t(fragment_t const&) = delete; + fragment_t(fragment_t&&) = delete; + fragment_t& operator=(fragment_t const&) = delete; + fragment_t& operator=(fragment_t&&) = delete; + ~fragment_t(); + + [[nodiscard]] blob get_lto_ir() const; + + [[nodiscard]] blob get_cubin() const; + + private: + fragment_t(nvJitLinkHandle handle, binary_type type) : handle_(handle), type_(type) {} +}; + +struct module_t; + +using module = std::shared_ptr; + +void mangle_cxx_symbol(char const*); + +struct function_ref { + void launch(); +}; + +struct module_t { + private: + CUmodule handle_; + + public: + module_t(module_t const&) = delete; + module_t(module_t&&) = delete; + module_t& operator=(module_t const&) = delete; + module_t& operator=(module_t&&) = delete; + ~module_t(); + + struct load_params { + blob_view cubin = {}; + }; + + struct link_params { + std::span fragments = {}; + std::span binary_types = {}; + std::span names = {}; + std::span link_options = {}; + }; + + [[nodiscard]] static blob link_as_cubin(link_params const& params); + + [[nodiscard]] static module load(blob_view cubin); + + [[nodiscard]] static module link(link_params const& params); + + [[nodiscard]] function_ref get_function(char const* name) const; + + private: + module_t(CUmodule handle) : handle_(handle) {} +}; + +} // namespace rtc +} // namespace cudf diff --git a/cpp/src/jit/rtc/sha256.cpp b/cpp/src/jit/rtc/sha256.cpp new file mode 100644 index 000000000000..97c2d1215e1c --- /dev/null +++ b/cpp/src/jit/rtc/sha256.cpp @@ -0,0 +1,42 @@ +#include "jit/rtc/sha256.hpp" + +#include + +extern "C" { +#include +} + +namespace CUDF_EXPORT cudf { +namespace rtc { + +sha256_context::sha256_context() : ectx_(nullptr) +{ + const EVP_MD* type = EVP_sha256(); + ectx_ = EVP_MD_CTX_new(); + CUDF_EXPECTS(ectx_ != nullptr, "EVP_MD_CTX_new failed"); + CUDF_EXPECTS(EVP_DigestInit_ex(ectx_, type, nullptr) == 1, "EVP_DigestInit_ex failed"); +} + +sha256_context::~sha256_context() +{ + if (ectx_ != nullptr) { EVP_MD_CTX_free(ectx_); } +} + +void sha256_context::update(std::span data) +{ + CUDF_EXPECTS(EVP_DigestUpdate(ectx_, data.data(), data.size()) == 1, "EVP_DigestUpdate failed"); +} + +sha256_hash sha256_context::finalize() +{ + sha256_hash hash; + unsigned int length = 0; + CUDF_EXPECTS(EVP_DigestFinal_ex(ectx_, hash.data_, &length) == 1, "EVP_DigestFinal_ex failed"); + CUDF_EXPECTS(length == 64, "Unexpected SHA256 length"); + EVP_MD const* type = EVP_sha256(); + CUDF_EXPECTS(EVP_DigestInit_ex(ectx_, type, nullptr) == 1, "EVP_DigestInit_ex failed"); + return hash; +} + +} // namespace rtc +} // namespace CUDF_EXPORT cudf \ No newline at end of file diff --git a/cpp/src/jit/rtc/sha256.hpp b/cpp/src/jit/rtc/sha256.hpp new file mode 100644 index 000000000000..21c92384ffd3 --- /dev/null +++ b/cpp/src/jit/rtc/sha256.hpp @@ -0,0 +1,99 @@ +#pragma once +#include + +#include +#include +#include +#include + +extern "C" { +typedef struct evp_md_ctx_st EVP_MD_CTX; +} + +namespace CUDF_EXPORT cudf { +namespace rtc { + +struct sha256_hex_string { + char data_[65]; + + constexpr operator std::string_view() const { return std::string_view{data_, 64}; } + + char const* data() const { return data_; } + + char const* c_str() const { return data_; } + + static constexpr size_t size() { return 64; } +}; + +struct sha256_hash { + alignas(16) uint8_t data_[32]; + + constexpr bool operator==(sha256_hash const& hash) const + { + return std::equal(std::begin(data_), std::end(data_), std::begin(hash.data_)); + } + + constexpr bool operator!=(sha256_hash const& hash) const { return !(*this == hash); } + + constexpr sha256_hex_string to_hex() const + { + static constexpr char const HEX_CHARS[] = "0123456789abcdef"; + sha256_hex_string hex; + for (size_t i = 0; i < 32; ++i) { + hex.data_[i * 2] = HEX_CHARS[(data_[i] >> 4) & 0x0F]; + hex.data_[i * 2 + 1] = HEX_CHARS[data_[i] & 0x0F]; + } + hex.data_[64] = '\0'; + return hex; + } +}; + +struct sha256_hash_hasher { + constexpr uint64_t operator()(sha256_hash const& obj) const + { + struct u64x4 { + uint64_t v[4]; + }; + + auto value = std::bit_cast(obj); + auto const h0 = value.v[0]; + auto const h1 = value.v[1]; + auto const h2 = value.v[2]; + auto const h3 = value.v[3]; + + auto mix = [](uint64_t seed, uint64_t v) { + seed ^= v + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); + return seed; + }; + + return mix(mix(mix(h0, h1), h2), h3); + } +}; + +struct sha256_context { + private: + EVP_MD_CTX* ectx_; + + public: + sha256_context(); + sha256_context(sha256_context const& other) = delete; + sha256_context& operator=(sha256_context const& other) = delete; + sha256_context(sha256_context&& other) : ectx_(other.ectx_) { other.ectx_ = nullptr; } + + sha256_context& operator=(sha256_context&& other) + { + if (this == &other) [[unlikely]] { return *this; } + this->~sha256_context(); + new (this) sha256_context(std::move(other)); + return *this; + } + + ~sha256_context(); + + void update(std::span data); + + sha256_hash finalize(); +}; + +} // namespace rtc +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh b/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh new file mode 100644 index 000000000000..5b393a029685 --- /dev/null +++ b/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh @@ -0,0 +1,34 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include +#include + +extern "C" { + +__device__ void filter_operator(cudf::lto::filter_params const* params); + +__global__ void filter_kernel(void const* outputs, + void const* span_outputs, + void const* inputs, + void* user_data, + cudf::size_type num_rows) +{ + auto const start = cudf::detail::grid_1d::global_thread_id(); + auto const stride = cudf::detail::grid_1d::grid_stride(); + auto const size = num_rows; + + for (auto i = start; i < size; i += stride) { + cudf::lto::filter_params p{.inputs = inputs, + .user_data = user_data, + .outputs = outputs, + .span_outputs = span_outputs, + .row_index = static_cast(i)}; + filter_operator(&p); + } +} +} diff --git a/cpp/src/transform/jit/lto_kernel.inl.cuh b/cpp/src/transform/jit/lto_kernel.inl.cuh new file mode 100644 index 000000000000..2ced6aa914b8 --- /dev/null +++ b/cpp/src/transform/jit/lto_kernel.inl.cuh @@ -0,0 +1,33 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include +#include + +extern "C" { +__device__ void transform_operator(cudf::lto::transform_params const* params); + +__global__ void transform_kernel(void const* outputs, + void const* span_outputs, + void const* inputs, + void* user_data, + cudf::size_type num_rows) +{ + auto const start = cudf::detail::grid_1d::global_thread_id(); + auto const stride = cudf::detail::grid_1d::grid_stride(); + auto const size = num_rows; + + for (auto i = start; i < size; i += stride) { + cudf::lto::transform_params p{.inputs = inputs, + .user_data = user_data, + .outputs = outputs, + .span_outputs = span_outputs, + .row_index = static_cast(i)}; + transform_operator(&p); + } +} +} diff --git a/cpp/src/transform/transform.cpp b/cpp/src/transform/transform.cpp index 8f3ab5f0b764..4ba1efecea08 100644 --- a/cpp/src/transform/transform.cpp +++ b/cpp/src/transform/transform.cpp @@ -16,13 +16,14 @@ #include +#include #include #include #include #include #include #include -#include +#include namespace cudf { namespace transformation { @@ -36,6 +37,28 @@ jitify2::Kernel get_kernel(std::string const& kernel_name, std::string const& cu .get_kernel(kernel_name, {}, {{"cudf/detail/operation-udf.hpp", cuda_source}}, {"-arch=sm_."}); } +// [ ] load the cudf transformation LTO library; return Culibrary object representing it +// void get_transform_library(); + +// [ ] take CUDA/PTX code as input +// [ ] support different function signature styles +// [ ] handle include directories +// [ ] C++/C-ABI symbol name for locating function to link against +// [ ] take input and output data types as parameters +// [ ] take function_info struct as input; describe: input/output types, null-awareness, user-data +// requirement, etc. +// [ ] return Culibrary object representing compiled operator +// void compile_library(); +// void compile_transform_operator_thunk(); + +// [ ] take LTO-IR compiled binary of operator as input +// [ ] link it into the lto library; return +// void link_transform_thunk(); + +// [ ] if we can, we should not depend on jitify for LTO-IR compilation, linking, and caching; use +// nvJITLink directly +// void cache_transform_thunk(); + jitify2::ConfiguredKernel build_transform_kernel( std::string const& kernel_name, size_type base_column_size, diff --git a/jit.md b/jit.md new file mode 100644 index 000000000000..1e06dd7da261 --- /dev/null +++ b/jit.md @@ -0,0 +1 @@ +# Explain JIT compilation flow as it is in CUDF right now \ No newline at end of file diff --git a/lto-ir-notes.txt b/lto-ir-notes.txt new file mode 100644 index 000000000000..f5ab9944f873 --- /dev/null +++ b/lto-ir-notes.txt @@ -0,0 +1 @@ +Not compatible across major versions diff --git a/todo.md b/todo.md new file mode 100644 index 000000000000..ebcb160c99d0 --- /dev/null +++ b/todo.md @@ -0,0 +1,5 @@ +[ ] Use PCH +[ ] Add time trace to JIT and AOT + + + COMMAND = cd /home/coder/cudf/cpp/src && /home/coder/.conda/envs/rapids/bin/cmake -E make_directory /home/coder/cudf/cpp/build/conda/cuda-13.0/release/include/jit_preprocessed_files/binaryop/jit && /home/coder/.conda/envs/rapids/bin/cmake -E env LD_LIBRARY_PATH=/home/coder/.conda/envs/rapids/lib /home/coder/cudf/cpp/build/conda/cuda-13.0/release/bin/cudf/jitify_preprocess binaryop/jit/kernel.cu -o /home/coder/cudf/cpp/build/conda/cuda-13.0/release/include/jit_preprocessed_files/binaryop/jit -i -std=c++20 -remove-unused-globals -D_FILE_OFFSET_BITS=64 -D__CUDACC_RTC__ -DCUDF_RUNTIME_JIT -I/home/coder/cudf/cpp/include -I/home/coder/cudf/cpp/src -I/home/coder/cudf/cpp/build/conda/cuda-13.0/release/_deps/cccl-src/lib/cmake/libcudacxx/../../../libcudacxx/include -I/home/coder/.conda/envs/rapids/targets/x86_64-linux/include -I/home/coder/.conda/envs/rapids/targets/x86_64-linux/include/cccl --no-preinclude-workarounds --no-replace-pragma-once --diag-suppress=47 --device-int128 \ No newline at end of file From a16ba6cfd0980b1267156503d2d000419b60d248 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 6 Dec 2025 17:04:03 +0000 Subject: [PATCH 002/254] update --- cpp/CMakeLists.txt | 4 ++-- cpp/include/cudf/jit/filter_params.cuh | 8 +------- cpp/include/cudf/jit/transform_params.cuh | 9 ++------- cpp/src/jit/rtc/cache.cpp | 6 +++--- cpp/src/jit/rtc/rtc.cpp | 3 +-- cpp/src/jit/rtc/rtc.hpp | 1 - cpp/src/jit/rtc/sha256.cpp | 9 +++++++-- cpp/src/jit/rtc/sha256.hpp | 5 +++++ jit.md | 2 +- todo.md | 2 -- 10 files changed, 22 insertions(+), 27 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a3959bdb2153..bed2275ce135 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -453,8 +453,8 @@ endif() if(CUDA_ENABLE_LINEINFO) list(APPEND CUDF_JIT_CUDA_FLAGS "${CUDF_JIT_CUDA_FLAGS};-lineinfo") endif() - - + + # use current project CUDA_ARCHITECTURES for JIT compilation set(CUDF_CXX_STANDARD 20) diff --git a/cpp/include/cudf/jit/filter_params.cuh b/cpp/include/cudf/jit/filter_params.cuh index 8de94ffa3a16..c4576a1699c7 100644 --- a/cpp/include/cudf/jit/filter_params.cuh +++ b/cpp/include/cudf/jit/filter_params.cuh @@ -1,18 +1,12 @@ - /* * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ #pragma once #include -#include -namespace CUDF_EXPORT cudf { +namespace CUDF_LTO_EXPORT cudf { namespace lto { diff --git a/cpp/include/cudf/jit/transform_params.cuh b/cpp/include/cudf/jit/transform_params.cuh index 7feb6b0801dd..da1845446af3 100644 --- a/cpp/include/cudf/jit/transform_params.cuh +++ b/cpp/include/cudf/jit/transform_params.cuh @@ -4,15 +4,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ #pragma once #include -#include -namespace CUDF_EXPORT cudf { +namespace CUDF_LTO_EXPORT cudf { namespace lto { @@ -25,4 +20,4 @@ struct transform_params { }; } // namespace lto -} // namespace CUDF_EXPORT cudf +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index 1a52e8629db5..ffb1c1e6f239 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -4,10 +4,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "jit/rtc/cache.hpp" - #include +#include + namespace cudf { namespace rtc { @@ -24,4 +24,4 @@ void cache_t::store_module(sha256_hash const& sha, module mod) { CUDF_FUNC_RANGE module cache_t::query_module(sha256_hash const& sha) { CUDF_FUNC_RANGE(); } } // namespace rtc -} // namespace cudf \ No newline at end of file +} // namespace cudf diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 668f7c76723a..a3efcf7f0580 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -4,14 +4,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "jit/rtc/rtc.hpp" - #include #include #include #include +#include #include #include diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index bcc89f5c83cf..6e366ad43bc3 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -5,7 +5,6 @@ */ #pragma once -#include #include #include #include diff --git a/cpp/src/jit/rtc/sha256.cpp b/cpp/src/jit/rtc/sha256.cpp index 97c2d1215e1c..93be0476125c 100644 --- a/cpp/src/jit/rtc/sha256.cpp +++ b/cpp/src/jit/rtc/sha256.cpp @@ -1,7 +1,12 @@ -#include "jit/rtc/sha256.hpp" +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ #include +#include + extern "C" { #include } @@ -39,4 +44,4 @@ sha256_hash sha256_context::finalize() } } // namespace rtc -} // namespace CUDF_EXPORT cudf \ No newline at end of file +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/sha256.hpp b/cpp/src/jit/rtc/sha256.hpp index 21c92384ffd3..cec72cdc6865 100644 --- a/cpp/src/jit/rtc/sha256.hpp +++ b/cpp/src/jit/rtc/sha256.hpp @@ -1,3 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + #pragma once #include diff --git a/jit.md b/jit.md index 1e06dd7da261..6e18b46d2386 100644 --- a/jit.md +++ b/jit.md @@ -1 +1 @@ -# Explain JIT compilation flow as it is in CUDF right now \ No newline at end of file +# Explain JIT compilation flow as it is in CUDF right now diff --git a/todo.md b/todo.md index ebcb160c99d0..e6a84381b842 100644 --- a/todo.md +++ b/todo.md @@ -1,5 +1,3 @@ [ ] Use PCH [ ] Add time trace to JIT and AOT - - COMMAND = cd /home/coder/cudf/cpp/src && /home/coder/.conda/envs/rapids/bin/cmake -E make_directory /home/coder/cudf/cpp/build/conda/cuda-13.0/release/include/jit_preprocessed_files/binaryop/jit && /home/coder/.conda/envs/rapids/bin/cmake -E env LD_LIBRARY_PATH=/home/coder/.conda/envs/rapids/lib /home/coder/cudf/cpp/build/conda/cuda-13.0/release/bin/cudf/jitify_preprocess binaryop/jit/kernel.cu -o /home/coder/cudf/cpp/build/conda/cuda-13.0/release/include/jit_preprocessed_files/binaryop/jit -i -std=c++20 -remove-unused-globals -D_FILE_OFFSET_BITS=64 -D__CUDACC_RTC__ -DCUDF_RUNTIME_JIT -I/home/coder/cudf/cpp/include -I/home/coder/cudf/cpp/src -I/home/coder/cudf/cpp/build/conda/cuda-13.0/release/_deps/cccl-src/lib/cmake/libcudacxx/../../../libcudacxx/include -I/home/coder/.conda/envs/rapids/targets/x86_64-linux/include -I/home/coder/.conda/envs/rapids/targets/x86_64-linux/include/cccl --no-preinclude-workarounds --no-replace-pragma-once --diag-suppress=47 --device-int128 \ No newline at end of file From 6b0172ef32654fc6d34dbe1c7003c46b0be34144 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 9 Dec 2025 11:47:01 +0000 Subject: [PATCH 003/254] checkpoint --- cpp/include/cudf/jit/filter_params.cuh | 2 +- cpp/include/cudf/utilities/defer.hpp | 36 ++ cpp/src/jit/rtc/cache.cpp | 354 ++++++++++++++++- cpp/src/jit/rtc/cache.hpp | 311 +++++++++++---- cpp/src/jit/rtc/cudf.cpp | 2 + cpp/src/jit/rtc/cudf.hpp | 35 ++ cpp/src/jit/rtc/rtc.cpp | 529 ++++++++++++++++++------- cpp/src/jit/rtc/rtc.hpp | 128 ++++-- cpp/src/jit/rtc/sha256.hpp | 20 +- 9 files changed, 1155 insertions(+), 262 deletions(-) create mode 100644 cpp/include/cudf/utilities/defer.hpp create mode 100644 cpp/src/jit/rtc/cudf.cpp create mode 100644 cpp/src/jit/rtc/cudf.hpp diff --git a/cpp/include/cudf/jit/filter_params.cuh b/cpp/include/cudf/jit/filter_params.cuh index c4576a1699c7..d9b5c2dd1a1e 100644 --- a/cpp/include/cudf/jit/filter_params.cuh +++ b/cpp/include/cudf/jit/filter_params.cuh @@ -19,4 +19,4 @@ struct filter_params { }; } // namespace lto -} // namespace CUDF_EXPORT cudf +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/utilities/defer.hpp b/cpp/include/cudf/utilities/defer.hpp new file mode 100644 index 000000000000..d17607a29fb7 --- /dev/null +++ b/cpp/include/cudf/utilities/defer.hpp @@ -0,0 +1,36 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#define CUDF_DEFER__CONCATENATE_DETAIL(x, y) x##y +#define CUDF_DEFER__CONCATENATE(x, y) CUDF_DEFER__CONCATENATE_DETAIL(x, y) +#define CUDF_DEFER(...) ::cudf::defer CUDF_DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) + +namespace CUDF_EXPORT cudf { + +template +struct defer { + private: + T func_; + + public: + template + defer(Args&&... args) : func_{static_cast(args)...} + { + } + defer(defer const&) = delete; + defer& operator=(defer const&) = delete; + defer(defer&&) = delete; + defer& operator=(defer&&) = delete; + ~defer() { func_(); } +}; + +template +defer(T) -> defer; + +} // namespace CUDF_EXPORT cudf \ No newline at end of file diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index ffb1c1e6f239..30a4f9eaadb6 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -5,23 +5,365 @@ */ #include +#include +#include +#include #include +#include +#include +#include +#include + +#include +#include +#include namespace cudf { namespace rtc { -void cache_t::store_blob(sha256_hash const& sha, blob binary) { CUDF_FUNC_RANGE(); } +namespace { + +[[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) +{ + auto error_code = errno; + auto error_str = std::format( + "{}. `{}` failed with {} ({})", message, syscall_name, error_code, std::strerror(error_code)); + CUDF_FAIL(+error_str, std::runtime_error); +} + +} // namespace + +compile_cache_t::compile_cache_t(bool enabled, + std::string cache_dir, + compile_cache_limits const& limits) + : enabled_{enabled}, + cache_dir_{std::move(cache_dir)}, + limits_{limits}, + blobs_cache_{limits.num_blobs}, + fragments_cache_{limits.num_fragments}, + libraries_cache_{limits.num_libraries}, + tick_{0} +{ + // Create cache directory if it doesn't exist + if (mkdir(cache_dir_.c_str(), S_IRWXU | S_IRWXG | S_IRWXO) == -1) { + if (errno != EEXIST) { throw_posix("Failed to create RTC cache directory", "mkdir"); } + } +} + +bool compile_cache_t::is_enabled() const { return enabled_; } + +void compile_cache_t::enable() { enabled_ = true; } + +void compile_cache_t::disable() { enabled_ = false; } + +void compile_cache_t::store_blob_to_memory(sha256_hash const& sha, std::shared_future binary) +{ + CUDF_FUNC_RANGE(); + + if (!enabled_) { return; } + + std::atomic_ref tick{tick_}; + auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + + detail::write_guard guard{blobs_cache_.lock_}; + + blobs_cache_.insert(sha, std::move(binary), current_tick); +} + +void compile_cache_t::store_blob_to_disk(sha256_hash const& sha, blob_view binary) +{ + CUDF_FUNC_RANGE(); + + if (!enabled_) { return; } + + char temp_path[] = "/tmp/blob-XXXXXX"; + + { + int fd = mkstemp(temp_path); + if (fd == -1) { throw_posix("Failed to create temporary file for RTC cache", "mkstemp"); } + + CUDF_DEFER([&] { + if (remove(temp_path) == -1) { + throw_posix("Failed to remove temporary RTC cache file", "remove"); + } + }); + + if (write(fd, binary.data(), binary.size()) == -1) { + throw_posix("Failed to write RTC cache to temporary file", "write"); + } + } + + auto hex = sha.to_hex_string(); + char final_path[PATH_MAX + 1]; + auto result = std::format_to_n(final_path, PATH_MAX, "{}/{}.blob", cache_dir_, hex.view()); + CUDF_EXPECTS( + result.out != (final_path + PATH_MAX), "Path length exceeded PATH_MAX", std::runtime_error); + *result.out = '\0'; + + // rename is atomic, even if another process is performing the same operation + if (rename(temp_path, final_path) == -1) { + auto error_code = errno; + + if (error_code == EEXIST) { + // another process has already created the file, so just remove our temp file + if (remove(temp_path) == -1) { + throw_posix("Failed to remove temporary RTC cache file", "remove"); + } + return; + } + + throw_posix("Failed to move temporary RTC cache file to final location", "rename"); + } +} + +std::optional> compile_cache_t::query_blob_from_memory( + sha256_hash const& sha) +{ + CUDF_FUNC_RANGE(); + + if (!enabled_) { return std::nullopt; } + + std::atomic_ref tick{tick_}; + auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + + { + detail::read_guard guard{blobs_cache_.lock_}; + auto const it = blobs_cache_.entries_.find(sha); + + if (it != blobs_cache_.entries_.end()) { + counter_.hit_memory_blob(); + it->second.hit(current_tick); + return it->second.value; + } else { + counter_.miss_memory_blob(); + return std::nullopt; + } + } +} + +std::optional compile_cache_t::query_blob_from_disk(sha256_hash const& sha) +{ + CUDF_FUNC_RANGE(); + + if (!enabled_) { return std::nullopt; } + + auto hex = sha.to_hex_string(); + auto path = std::format("{}/{}.blob", cache_dir_, hex.view()); + + int fd = open(path.c_str(), O_RDONLY); + + if (fd == -1) { + if (errno == ENOENT) { + counter_.miss_disk_blob(); + return std::nullopt; + } else { + throw_posix("Failed to open RTC cache file from disk", "open"); + } + } + + auto file_size = lseek(fd, 0, SEEK_END); + if (file_size == -1) { throw_posix("Failed to determine size of RTC cache file", "lseek"); } + + void* map = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); + + if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTC cache file", "mmap"); } + + if (close(fd) == -1) { + throw_posix("Failed to close RTC cache file after memory-mapping", "close"); + } + + auto deleter = +[](void*, uint8_t const* buffer, size_t size) { + if (munmap(static_cast(const_cast(buffer)), size) == -1) { + throw_posix("Failed to unmap RTC cache file from memory", "munmap"); + } + }; + + counter_.hit_disk_blob(); + + return std::make_shared( + blob_t::from_parts(static_cast(map), file_size, nullptr, deleter)); +} + +void compile_cache_t::store_fragment(sha256_hash const& sha, std::shared_future frag) +{ + CUDF_FUNC_RANGE(); + + if (!enabled_) { return; } + + std::atomic_ref tick{tick_}; + auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + + { + detail::write_guard guard{fragments_cache_.lock_}; + + fragments_cache_.insert(sha, std::move(frag), current_tick); + } +} + +std::optional> compile_cache_t::query_fragment(sha256_hash const& sha) +{ + CUDF_FUNC_RANGE(); + + if (!enabled_) { return std::nullopt; } + + std::atomic_ref tick{tick_}; + auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + + { + detail::read_guard guard{fragments_cache_.lock_}; + + auto const it = fragments_cache_.entries_.find(sha); + if (it != fragments_cache_.entries_.end()) { + counter_.hit_memory_fragment(); + it->second.hit(current_tick); + return it->second.value; + } else { + counter_.miss_memory_fragment(); + return std::nullopt; + } + } +} + +void compile_cache_t::store_library(sha256_hash const& sha, std::shared_future mod) +{ + CUDF_FUNC_RANGE(); + + if (!enabled_) { return; } + + std::atomic_ref tick{tick_}; + auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + + { + detail::write_guard guard{libraries_cache_.lock_}; + + libraries_cache_.insert(sha, std::move(mod), current_tick); + } +} + +std::optional> compile_cache_t::query_library(sha256_hash const& sha) +{ + CUDF_FUNC_RANGE(); + + if (!enabled_) { return std::nullopt; } + + std::atomic_ref tick{tick_}; + auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + + { + detail::read_guard guard{libraries_cache_.lock_}; + + auto const it = libraries_cache_.entries_.find(sha); + if (it != libraries_cache_.entries_.end()) { + counter_.hit_memory_library(); + it->second.hit(current_tick); + return it->second.value; + } else { + counter_.miss_memory_library(); + return std::nullopt; + } + } +} + +compile_cache_statistics compile_cache_t::get_statistics() const { return counter_.get(); } + +void compile_cache_t::clear_statistics() { counter_.clear(); } + +compile_cache_limits compile_cache_t::get_limits() const { return limits_; } + +size_t compile_cache_t::get_blob_count() const +{ + CUDF_FUNC_RANGE(); + + { + detail::read_guard guard{blobs_cache_.lock_}; + return blobs_cache_.entries_.size(); + } +} + +size_t compile_cache_t::get_fragment_count() const +{ + CUDF_FUNC_RANGE(); + + { + detail::read_guard guard{fragments_cache_.lock_}; + return fragments_cache_.entries_.size(); + } +} + +size_t compile_cache_t::get_library_count() const +{ + CUDF_FUNC_RANGE(); + + { + detail::read_guard guard{libraries_cache_.lock_}; + return libraries_cache_.entries_.size(); + } +} + +void compile_cache_t::clear_memory_store() +{ + CUDF_FUNC_RANGE(); + + { + detail::write_guard guard{blobs_cache_.lock_}; + blobs_cache_.entries_.clear(); + } + + { + detail::write_guard guard{fragments_cache_.lock_}; + fragments_cache_.entries_.clear(); + } + + { + detail::write_guard guard{libraries_cache_.lock_}; + libraries_cache_.entries_.clear(); + } +} + +void compile_cache_t::clear_disk_store() +{ + CUDF_FUNC_RANGE(); + DIR* dir = opendir(cache_dir_.c_str()); + + if (dir == nullptr) { throw_posix("Failed to open RTC cache directory for clearing", "opendir"); } + + CUDF_DEFER([&] { closedir(dir); }); + + errno = 0; // reset errno before reading + + struct dirent* entry_iter; + + while (true) { + entry_iter = readdir(dir); + + if (entry_iter == nullptr) { + if (errno != 0) { + throw_posix("Failed to read RTC cache directory for clearing", "readdir"); + } else { + break; + } + } -blob cache_t::query_blob(sha256_hash const& sha) { CUDF_FUNC_RANGE(); } + struct stat st; + char path[PATH_MAX + 1]; -void cache_t::store_fragment(sha256_hash const& sha, fragment frag) { CUDF_FUNC_RANGE(); } + if (lstat(path, &st) == -1) { + throw_posix("Failed to get file status for RTC cache clearing", "lstat"); + } -fragment cache_t::query_fragment(sha256_hash const& sha) { CUDF_FUNC_RANGE(); } + if (S_ISREG(st.st_mode)) { + if (unlink(path) == -1) { + throw_posix("Failed to unlink RTC cache file during clearing", "unlink"); + } + } -void cache_t::store_module(sha256_hash const& sha, module mod) { CUDF_FUNC_RANGE(); } + // reset errno for next iteration + errno = 0; + } -module cache_t::query_module(sha256_hash const& sha) { CUDF_FUNC_RANGE(); } + return; +} } // namespace rtc } // namespace cudf diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index 589d9755048d..223bab584ec7 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -9,28 +9,67 @@ #include #include +#include #include #include +#include +#include #include namespace cudf { namespace rtc { -struct rwlock_t { - static constexpr size_t WRITE_STATE = ~(size_t)0; +struct [[nodiscard]] compile_cache_statistics { + uint64_t blob_memory_hits = 0; + uint64_t blob_memory_misses = 0; + uint64_t fragment_memory_hits = 0; + uint64_t fragment_memory_misses = 0; + uint64_t library_memory_hits = 0; + uint64_t library_memory_misses = 0; + uint64_t blob_disk_hits = 0; + uint64_t blob_disk_misses = 0; +}; + +struct [[nodiscard]] compile_cache_limits { + uint32_t num_blobs = 1024; + uint32_t num_fragments = 1024; + uint32_t num_libraries = 1024; +}; +namespace detail { + +struct rw_spinlock_t { private: + static constexpr size_t WRITE_STATE = ~size_t{0}; + static constexpr size_t IDLE_STATE = 0; + size_t state_; public: - rwlock_t() : state_{0} {} - rwlock_t(rwlock_t const&) = default; - rwlock_t& operator=(rwlock_t const&) = default; - rwlock_t(rwlock_t&&) = default; - rwlock_t& operator=(rwlock_t&&) = default; - ~rwlock_t() = default; + rw_spinlock_t() : state_{IDLE_STATE} {} + rw_spinlock_t(rw_spinlock_t const&) = default; + rw_spinlock_t& operator=(rw_spinlock_t const&) = default; + rw_spinlock_t(rw_spinlock_t&&) = default; + rw_spinlock_t& operator=(rw_spinlock_t&&) = default; + ~rw_spinlock_t() = default; + + void lock_read() + { + std::atomic_ref state{state_}; - void lock_read() {} + auto expected = IDLE_STATE; + auto target = size_t{1}; + + while (!state.compare_exchange_weak( + expected, target, std::memory_order_acquire, std::memory_order_relaxed)) { + if (expected == WRITE_STATE) { + expected = IDLE_STATE; + target = 1; + } else { + target = expected + 1; + } + } + } void unlock_read() { @@ -38,21 +77,32 @@ struct rwlock_t { state.fetch_sub(1, std::memory_order_relaxed); } - void lock_write() {} + void lock_write() + { + std::atomic_ref state{state_}; + + auto expected = IDLE_STATE; + + while (!state.compare_exchange_weak( + expected, WRITE_STATE, std::memory_order_acquire, std::memory_order_relaxed)) { + expected = IDLE_STATE; + } + } void unlock_write() { std::atomic_ref state{state_}; - state.store(0, std::memory_order_release); + state.store(IDLE_STATE, std::memory_order_release); } }; +template struct read_guard { private: - rwlock_t& lock_; + Lock& lock_; public: - read_guard(rwlock_t& lock) : lock_{lock} { lock_.lock_read(); } + read_guard(Lock& lock) : lock_{lock} { lock_.lock_read(); } read_guard(read_guard const&) = delete; read_guard& operator=(read_guard const&) = delete; @@ -62,12 +112,13 @@ struct read_guard { ~read_guard() { lock_.unlock_read(); } }; +template struct write_guard { private: - rwlock_t& lock_; + Lock& lock_; public: - write_guard(rwlock_t& lock) : lock_{lock} { lock_.lock_write(); } + write_guard(Lock& lock) : lock_{lock} { lock_.lock_write(); } write_guard(write_guard const&) = delete; write_guard& operator=(write_guard const&) = delete; @@ -77,87 +128,203 @@ struct write_guard { ~write_guard() { lock_.unlock_write(); } }; -struct cache_t { - private: - struct statistics { - uint64_t memory_hits = 0; - uint64_t memory_misses = 0; - uint64_t disk_hits = 0; - uint64_t disk_misses = 0; - }; +template +struct alignas(std::hardware_destructive_interference_size) lru_memory_cache { + struct entry { + uint64_t last_touched_tick; + T value; - struct limits { - uint64_t max_blobs_memory_size = UINT64_MAX; - uint64_t max_num_fragments = UINT64_MAX; - uint64_t max_num_modules = UINT64_MAX; + void hit(uint64_t tick) { last_touched_tick = tick; } }; - alignas(std::hardware_destructive_interference_size) rwlock_t blobs_lock_; + std::unordered_map entries_; + rw_spinlock_t lock_; + size_t limit_; - std::unordered_map blobs_; + explicit lru_memory_cache(size_t limit) : entries_{}, lock_{}, limit_{limit} + { + // reserve space to avoid rehashing + entries_.reserve(limit * 2); + } - uint64_t blobs_memory_size_; + void insert(sha256_hash const& sha, T&& value, uint64_t tick) + { + if ((entries_.size() + 1) > limit_) { + std::vector> rankings; + rankings.reserve(entries_.size()); - alignas(std::hardware_destructive_interference_size) rwlock_t fragments_lock_; + for (auto const& [key, ent] : entries_) { + rankings.emplace_back(key, ent.last_touched_tick); + } - std::unordered_map fragments_; + std::sort(rankings.begin(), rankings.end(), [](auto const& a, auto const& b) { + return a.second < b.second; + }); - alignas(std::hardware_destructive_interference_size) rwlock_t modules_lock_; + // purge least recently used half - std::unordered_map modules_; + auto num_to_purge = rankings.size() / 2; - alignas(std::hardware_destructive_interference_size) statistics stats_; + for (size_t i = 0; i < num_to_purge; ++i) { + entries_.erase(rankings[i].first); + } + } - public: - // [ ] memory usage - // [ ] the object or blob being cached - // [ ] best we can do is LRU? + entries_.emplace(sha, entry{tick, std::move(value)}); + } +}; + +struct alignas(std::hardware_destructive_interference_size) counter { + uint64_t value_ = 0; + + void increment() + { + std::atomic_ref c{value_}; + c.fetch_add(1, std::memory_order_relaxed); + } - // [ ] when writing to path; store at target+".tmp${consistent_rand}"; then rename to target - // [ ] local in-memory cache with memory limit & LRU eviction policy - // - // [ ] disk cache - // - // [ ] to use key as file id, convert to hex string - cache_t() = default; - cache_t(cache_t const&) = delete; - cache_t& operator=(cache_t const&) = delete; - cache_t(cache_t&&) = delete; - cache_t& operator=(cache_t&&) = delete; - ~cache_t(); + [[nodiscard]] uint64_t get() const + { + std::atomic_ref c{value_}; + return c.load(std::memory_order_relaxed); + } + + void reset() + { + std::atomic_ref c{value_}; + c.store(0, std::memory_order_relaxed); + } +}; - // [ ] file cache path? - void store_blob(sha256_hash const& sha, blob binary); +struct compile_cache_statistics_counter { + counter blob_memory_hits; + counter blob_memory_misses; + counter fragment_memory_hits; + counter fragment_memory_misses; + counter library_memory_hits; + counter library_memory_misses; + counter blob_disk_hits; + counter blob_disk_misses; + + void clear() + { + blob_memory_hits.reset(); + blob_memory_misses.reset(); + fragment_memory_hits.reset(); + fragment_memory_misses.reset(); + library_memory_hits.reset(); + library_memory_misses.reset(); + blob_disk_hits.reset(); + blob_disk_misses.reset(); + } - blob query_blob(sha256_hash const& sha); + void hit_memory_blob() { blob_memory_hits.increment(); } - void store_fragment(sha256_hash const& sha, fragment frag); + void miss_memory_blob() { blob_memory_misses.increment(); } - fragment query_fragment(sha256_hash const& sha); + void hit_memory_fragment() { fragment_memory_hits.increment(); } - void store_module(sha256_hash const& sha, module mod); + void miss_memory_fragment() { fragment_memory_misses.increment(); } - module query_module(sha256_hash const& sha); + void hit_memory_library() { library_memory_hits.increment(); } + + void miss_memory_library() { library_memory_misses.increment(); } + + void hit_disk_blob() { blob_disk_hits.increment(); } + + void miss_disk_blob() { blob_disk_misses.increment(); } + + compile_cache_statistics get() const + { + return compile_cache_statistics{.blob_memory_hits = blob_memory_hits.get(), + .blob_memory_misses = blob_memory_misses.get(), + .fragment_memory_hits = fragment_memory_hits.get(), + .fragment_memory_misses = fragment_memory_misses.get(), + .library_memory_hits = library_memory_hits.get(), + .library_memory_misses = library_memory_misses.get(), + .blob_disk_hits = blob_disk_hits.get(), + .blob_disk_misses = blob_disk_misses.get()}; + } }; -// [ ] environment variables to control: -// [ ] cache path -// [ ] cache entries limit -// [ ] disable caching -// [ ] cache statistics: hits, misses, etc. -// [ ] cache tree pre-loading at startup -// [ ] on startup, log cache path, loading information, etc. +} // namespace detail + +/// @brief Compile cache for RTC blobs, fragments, and libraries +/// @details Provides in-memory and on-disk caching of compiled RTC artifacts. +/// The cache uses an LRU eviction policy when the number of cached items +/// exceeds user-defined limits. +/// In-memory cache is implemented using a thread-safe LRU cache that supports concurrent reads. +/// The on-disk cache also allows concurrent access and stores cached items in files +/// within a specified directory. Writing to disk is atomic to prevent corruption from +/// concurrent writes or process interruptions. +/// In addition, the cache maintains statistics on cache hits and misses for both +/// in-memory and on-disk caches to help monitor cache performance in benchmarking and debugging. +/// The interface is zero-copy throughout, using shared pointers and spans to avoid unnecessary data +/// copying across threads and disk. +struct compile_cache_t { + private: + bool enabled_; + + std::string cache_dir_; + + compile_cache_limits limits_; + + detail::lru_memory_cache> blobs_cache_; + + detail::lru_memory_cache> fragments_cache_; + + detail::lru_memory_cache> libraries_cache_; + + detail::compile_cache_statistics_counter counter_; + + alignas(std::hardware_destructive_interference_size) uint64_t tick_; + + public: + compile_cache_t(bool enabled, std::string cache_dir, compile_cache_limits const& limits); + compile_cache_t(compile_cache_t const&) = delete; + compile_cache_t& operator=(compile_cache_t const&) = delete; + compile_cache_t(compile_cache_t&&) = delete; + compile_cache_t& operator=(compile_cache_t&&) = delete; + ~compile_cache_t() = default; + + void enable(); + + void disable(); + + [[nodiscard]] bool is_enabled(); + + void store_blob_to_memory(sha256_hash const& sha, std::shared_future binary); + + void store_blob_to_disk(sha256_hash const& sha, blob_view binary); + + std::optional> query_blob_from_memory(sha256_hash const& sha); + + std::optional query_blob_from_disk(sha256_hash const& sha); + + void store_fragment(sha256_hash const& sha, std::shared_future frag); -cache_t& global_cache(); + std::optional> query_fragment(sha256_hash const& sha); -void preload_cache(); + void store_library(sha256_hash const& sha, std::shared_future mod); -// [ ] pre-load cuda -// [ ] if a user provides a key, use: USER_KEY+${key}, otherwise use sha256 of contents -// [ ] use resource type in key to avoid collisions -void compile_operator(); + std::optional> query_library(sha256_hash const& sha); -void link_operator(); + compile_cache_statistics get_statistics() const; + + void clear_statistics(); + + compile_cache_limits get_limits() const; + + [[nodiscard]] size_t get_blob_count() const; + + [[nodiscard]] size_t get_fragment_count() const; + + [[nodiscard]] size_t get_library_count() const; + + void clear_memory_store(); + + void clear_disk_store(); +}; } // namespace rtc } // namespace cudf diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp new file mode 100644 index 000000000000..139597f9cb07 --- /dev/null +++ b/cpp/src/jit/rtc/cudf.cpp @@ -0,0 +1,2 @@ + + diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/rtc/cudf.hpp new file mode 100644 index 000000000000..bd138e83bf55 --- /dev/null +++ b/cpp/src/jit/rtc/cudf.hpp @@ -0,0 +1,35 @@ + + + + + +// SIMPLE KEY: from user +// COMPLEX KEY: sha256 of all parameters and blobs involved; + driver + runtime +// + +// [ ] All these functions should have wrappers that cache results and request an optional key from +// the user + +// [ ] Method to pre-compile library and reuse it across multiple operators; compile_library(); +// compile_library_cached() + +// [ ] jit_key(key) -> key+driver+CUDA_versions+CUDA_runtime_versions+device_compute_models + // #include + + +// [ ] environment variables to control: +// [ ] cache path +// [ ] cache entries limit +// [ ] disable caching +// [ ] cache statistics: hits, misses, etc. +// [ ] on startup, log cache path, loading information, etc. + +compile_cache_t& get_cache(); + +// [ ] if a user provides a key, use: USER_KEY+${key}, otherwise use sha256 of contents +// [ ] use resource type in key to avoid collisions +void compile_operator(); + +void make_pch(); + +void link_operator(); \ No newline at end of file diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index a3efcf7f0580..a0be6b3e0ed8 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -5,209 +5,454 @@ */ #include +#include +#include +#include #include #include #include +#include #include #include #include -#define CUDF_CONCATENATE_DETAIL(x, y) x##y -#define CUDF_CONCATENATE(x, y) CUDF_CONCATENATE_DETAIL(x, y) - -#define CUDF_DEFER(...) ::cudf::rtc::defer CUDF_CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) +#include + +#define CUDFRTC_CONCATENATE_DETAIL(x, y) x##y +#define CUDFRTC_CONCATENATE(x, y) CUDFRTC_CONCATENATE_DETAIL(x, y) + +#define CUDFRTC_CHECK_CUDA(...) \ + do { \ + CUresult result = (__VA_ARGS__); \ + if (result != CUDA_SUCCESS) { \ + char const* enum_str; \ + CUDF_EXPECTS(cuGetErrorString(result, &enum_str) == CUDA_SUCCESS, \ + "Unable to get CUDA error string"); \ + auto errstr = std::format("CUDA Call {} failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast(result), \ + enum_str); \ + CUDF_FAIL(+errstr, std::runtime_error); \ + } \ + } while (0) + +#define CUDFRTC_CHECK_NVRTC(...) \ + do { \ + nvrtcResult result = (__VA_ARGS__); \ + if (result != NVRTC_SUCCESS) { \ + auto errstr = std::format("NVRTC Call {} failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast(result), \ + nvrtcGetErrorString(result)); \ + CUDF_FAIL(+errstr, std::runtime_error); \ + } \ + } while (0) + +#define CUDFRTC_CHECK_NVJITLINK(...) \ + do { \ + nvJitLinkResult result = (__VA_ARGS__); \ + if (result != NVJITLINK_SUCCESS) { \ + auto errstr = std::format("nvJitLink Call {} failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast(result), \ + cudf_nvJitLinkResultString(result)); \ + CUDF_FAIL(+errstr, std::runtime_error); \ + } \ + } while (0) namespace cudf { namespace rtc { -template -struct defer { - private: - T func_; +char const* cudf_nvJitLinkResultString(nvJitLinkResult result) +{ + switch (result) { + case NVJITLINK_SUCCESS: return "NVJITLINK_SUCCESS"; + case NVJITLINK_ERROR_UNRECOGNIZED_OPTION: return "NVJITLINK_ERROR_UNRECOGNIZED_OPTION"; + case NVJITLINK_ERROR_MISSING_ARCH: return "NVJITLINK_ERROR_MISSING_ARCH"; + case NVJITLINK_ERROR_INVALID_INPUT: return "NVJITLINK_ERROR_INVALID_INPUT"; + case NVJITLINK_ERROR_PTX_COMPILE: return "NVJITLINK_ERROR_PTX_COMPILE"; + case NVJITLINK_ERROR_NVVM_COMPILE: return "NVJITLINK_ERROR_NVVM_COMPILE"; + case NVJITLINK_ERROR_INTERNAL: return "NVJITLINK_ERROR_INTERNAL"; + case NVJITLINK_ERROR_THREADPOOL: return "NVJITLINK_ERROR_THREADPOOL"; + case NVJITLINK_ERROR_UNRECOGNIZED_INPUT: return "NVJITLINK_ERROR_UNRECOGNIZED_INPUT"; + case NVJITLINK_ERROR_FINALIZE: return "NVJITLINK_ERROR_FINALIZE"; + case NVJITLINK_ERROR_NULL_INPUT: return "NVJITLINK_ERROR_NULL_INPUT"; + case NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS: return "NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS"; + case NVJITLINK_ERROR_INCORRECT_INPUT_TYPE: return "NVJITLINK_ERROR_INCORRECT_INPUT_TYPE"; + case NVJITLINK_ERROR_ARCH_MISMATCH: return "NVJITLINK_ERROR_ARCH_MISMATCH"; + case NVJITLINK_ERROR_OUTDATED_LIBRARY: return "NVJITLINK_ERROR_OUTDATED_LIBRARY"; + case NVJITLINK_ERROR_MISSING_FATBIN: return "NVJITLINK_ERROR_MISSING_FATBIN"; + case NVJITLINK_ERROR_UNRECOGNIZED_ARCH: return "NVJITLINK_ERROR_UNRECOGNIZED_ARCH"; + case NVJITLINK_ERROR_UNSUPPORTED_ARCH: return "NVJITLINK_ERROR_UNSUPPORTED_ARCH"; + case NVJITLINK_ERROR_LTO_NOT_ENABLED: return "NVJITLINK_ERROR_LTO_NOT_ENABLED"; + default: CUDF_FAIL("Unrecognized nvJitLinkResult type", std::runtime_error); + } +} - public: - template - defer(Args&&... args) : func_{std::forward(args)...} - { +char const* binary_type_string(binary_type type) +{ + switch (type) { + case binary_type::LTO_IR: return "LTO_IR"; + case binary_type::CUBIN: return "CUBIN"; + case binary_type::FATBIN: return "FATBIN"; + case binary_type::PTX: return "PTX"; + default: CUDF_FAIL("Unrecognized binary_type", std::runtime_error); } - defer(defer const&) = delete; - defer& operator=(defer const&) = delete; - defer(defer&&) = delete; - defer& operator=(defer&&) = delete; - ~defer() { func_(); } -}; +} + +blob_t blob_t::from_vector(std::vector&& data) +{ + auto ptr = new std::vector(std::move(data)); + return blob_t::from_parts( + ptr->data(), ptr->size(), ptr, [](void* user_data, uint8_t const*, size_t) { + delete reinterpret_cast*>(user_data); + }); +} + +blob_t blob_t::from_static_data(std::span data) +{ + return blob_t::from_parts( + data.data(), data.size(), nullptr, [](void*, uint8_t const*, size_t) {}); +} -fragment fragment_t::load(fragment_t::load_params const& params) { CUDF_FUNC_RANGE(); } +fragment fragment_t::load(load_params const& params) +{ + CUDF_FUNC_RANGE(); -fragment fragment_t::compile(fragment_t::compile_params const& params) { CUDF_FUNC_RANGE(); } + return std::make_shared(params.blob, params.type); +} -blob fragment_t::get_lto_ir() const { CUDF_FUNC_RANGE(); } +void log_nvrtc_compile_result(fragment_t::compile_params const& params, + nvrtcProgram program, + nvrtcResult compile_result) +{ + size_t log_size; + CUDFRTC_CHECK_NVRTC(nvrtcGetProgramLogSize(program, &log_size)); -blob fragment_t::get_cubin() const { CUDF_FUNC_RANGE(); } + std::vector log; + log.resize(log_size); + CUDFRTC_CHECK_NVRTC(nvrtcGetProgramLog(program, log.data())); -blob module_t::link_as_cubin(const link_params& params) { CUDF_FUNC_RANGE(); } + auto status_str = + (compile_result == NVRTC_SUCCESS) ? "completed with warning" : "failed with error"; -module module_t::load(blob_view cubin) { CUDF_FUNC_RANGE(); } + std::string headers_str; + for (auto const& header : params.headers.include_names) { + headers_str += std::format("\t{}\n", header); + } -module module_t::link(const link_params& params) { CUDF_FUNC_RANGE(); } + std::string options_str; + for (auto const& option : params.options) { + options_str += std::format("\t{}\n", option); + } -function_ref module_t::get_function(char const* name) const { CUDF_FUNC_RANGE(); } + auto str = std::format("NCRTC Compilation for {} {} ({}): {}.\nHeaders: {}\nOptions: {}\n\n{}", + params.name == nullptr ? "" : params.name, + status_str, + static_cast(compile_result), + nvrtcGetErrorString(compile_result), + headers_str, + options_str, + log.data()); + + if (compile_result != NVRTC_SUCCESS) { + CUDF_FAIL(+str, std::runtime_error); + } else if (!log.empty()) { + CUDF_LOG_WARN(str); + } +} -nvJitLinkHandle load_fragment_from(std::span source, - char const* name, - std::span link_options, - nvJitLinkInputType type) +void log_nvJitLink_link_result(library_t::link_params const& params, + nvJitLinkHandle handle, + nvJitLinkResult link_result) { - CUDF_FUNC_RANGE(); + size_t info_log_size; + CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetInfoLogSize(handle, &info_log_size)); - // auto sm = get_device_compute_model(); - // auto arch = std::format("-arch=sm_{}", sm); - // const char* link_options[] = {"-lto", arch.c_str()}; + std::vector info_log; + info_log.resize(info_log_size); + CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetInfoLog(handle, info_log.data())); - nvJitLinkHandle handle; - CUDF_EXPECTS( - nvJitLinkCreate(&handle, std::size(link_options), link_options.data()) == NVJITLINK_SUCCESS, - "Failed to create nvJitLink handle"); - CUDF_EXPECTS( - nvJitLinkAddData(handle, type, source.data(), source.size_bytes(), name) == NVJITLINK_SUCCESS, - "Failed to add LTO fatbin data to nvJitLink handle"); - CUDF_EXPECTS(nvJitLinkComplete(handle) == NVJITLINK_SUCCESS, - "Failed to complete nvJitLink handle"); - - return handle; -} - -// [ ] hash function: runtime? driver? rapids version? etc. - -// [ ] cudf_jit_header_map{"": "int x\0", "intval": "int x\0"} + cuda/std -- always -// available? [ ] when should they be hashed? - -// [ ] includes -// [ ] defines -// [ ] link flags -// [ ] compile flags -// [ ] compile for all CUDF architectures? -// input: CUDA code as string -// output: LTO IR code -std::vector compile_fragment(std::span headers, - std::span include_names, - std::span options) + size_t error_log_size; + CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetErrorLogSize(handle, &error_log_size)); + + std::vector error_log; + error_log.resize(error_log_size); + CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetErrorLog(handle, error_log.data())); + + std::string fragments_str; + for (auto const& fragment : params.names) { + fragments_str += std::format("\t{}\n", fragment); + } + + std::string link_options_str; + for (auto const& option : params.link_options) { + link_options_str += std::format("\t{}\n", option); + } + + char const* binary_type_str = binary_type_string(params.output_type); + + auto status_str = + (link_result == NVJITLINK_SUCCESS) ? "completed successfully" : "failed with error"; + + auto str = std::format( + "nvJitLink Linking for {} ({}) {} ({}): {}.\nFragments: {}\n" + "Link Options: {}\n\nInfo Log:\n{}\n\nError Log:\n{}", + params.name == nullptr ? "" : params.name, + binary_type_str, + status_str, + static_cast(link_result), + cudf_nvJitLinkResultString(link_result), + fragments_str, + link_options_str, + info_log.data(), + error_log.data()); + + if (link_result != NVJITLINK_SUCCESS) { + CUDF_FAIL(+str, std::runtime_error); + } else if (!info_log.empty() || !error_log.empty()) { + CUDF_LOG_WARN(str); + } +} + +fragment fragment_t::compile(compile_params const& params) { CUDF_FUNC_RANGE(); - // [ ] create hash - // [ ] serialize options and headers + nvrtcProgram program; + CUDFRTC_CHECK_NVRTC(nvrtcCreateProgram(&program, + params.source, + params.name, + static_cast(params.headers.headers.size()), + params.headers.headers.data(), + params.headers.include_names.data())); - // auto sm = get_device_compute_model(); - // auto arch = std::format("--gpu-architecture=compute_{}", sm); - // char const* opts[] = {arch.c_str(), "--dlto", "--relocatable-device-code=true"}; + CUDF_DEFER([&] { nvrtcDestroyProgram(&program); }); - nvrtcProgram program; - nvrtcCreateProgram(&program, - "", - "program", - static_cast(headers.size()), - headers.data(), - include_names.data()); - nvrtcCompileProgram(program, static_cast(options.size()), options.data()); + auto compile_result = + nvrtcCompileProgram(program, static_cast(params.options.size()), params.options.data()); + + log_nvrtc_compile_result(params, program, compile_result); + + switch (params.target_type) { + case binary_type::LTO_IR: { + size_t lto_ir_size; + CUDFRTC_CHECK_NVRTC(nvrtcGetLTOIRSize(program, <o_ir_size)); + + std::vector lto_ir; + lto_ir.resize(lto_ir_size); + CUDFRTC_CHECK_NVRTC(nvrtcGetLTOIR(program, reinterpret_cast(lto_ir.data()))); + + auto shared_blob = std::make_shared(blob_t::from_vector(std::move(lto_ir))); + + return std::make_shared(std::move(shared_blob), binary_type::LTO_IR); + + } break; + case binary_type::CUBIN: { + size_t cubin_size; + CUDFRTC_CHECK_NVRTC(nvrtcGetCUBINSize(program, &cubin_size)); + + std::vector cubin; + cubin.resize(cubin_size); + CUDFRTC_CHECK_NVRTC(nvrtcGetCUBIN(program, reinterpret_cast(cubin.data()))); + + auto shared_blob = std::make_shared(blob_t::from_vector(std::move(cubin))); + + return std::make_shared(std::move(shared_blob), binary_type::CUBIN); + + } break; + default: CUDF_FAIL("Unsupported binary type for compiling fragment"); + } +} - // nvrtcGetProgramLog(nvrtcProgram prog, char *log) - // nvrtcGetProgramLogSize(nvrtcProgram prog, size_t *logSizeRet) +blob const& fragment_t::get_lto_ir() const +{ + CUDF_FUNC_RANGE(); + CUDF_EXPECTS(type_ == binary_type::LTO_IR, "Fragment does not contain LTO IR"); + return blob_; +} - size_t lto_ir_size; - nvrtcGetLTOIRSize(program, <o_ir_size); +blob const& fragment_t::get_cubin() const +{ + CUDF_FUNC_RANGE(); + CUDF_EXPECTS(type_ == binary_type::CUBIN, "Fragment does not contain CUBIN"); + return blob_; +} - std::vector lto_ir; - lto_ir.resize(lto_ir_size); - nvrtcGetLTOIR(program, (char*)lto_ir.data()); +void kernel_ref::launch(uint32_t grid_dim_x, + uint32_t grid_dim_y, + uint32_t grid_dim_z, + uint32_t block_dim_x, + uint32_t block_dim_y, + uint32_t block_dim_z, + uint32_t shared_mem_bytes, + CUstream stream, + void** kernel_params) +{ + CUlaunchConfig cfg{.gridDimX = grid_dim_x, + .gridDimY = grid_dim_y, + .gridDimZ = grid_dim_z, + .blockDimX = block_dim_x, + .blockDimY = block_dim_y, + .blockDimZ = block_dim_z, + .sharedMemBytes = shared_mem_bytes, + .hStream = stream, + .attrs = nullptr, + .numAttrs = 0}; + CUDFRTC_CHECK_CUDA( + cuLaunchKernelEx(&cfg, reinterpret_cast(handle_), kernel_params, nullptr)); +} - // nvrtcGetErrorString() - // nvrtcGetCUBIN() - // nvrtcGetCUBINSize() +std::string_view kernel_ref::get_name() const +{ + CUDF_FUNC_RANGE(); + char const* name; + CUDFRTC_CHECK_CUDA(cuKernelGetName(&name, handle_)); + return std::string_view{name == nullptr ? "" : name}; +} - return lto_ir; +library_t::~library_t() +{ + if (handle_ != nullptr) { + if (cuLibraryUnload(handle_) != CUDA_SUCCESS) { std::terminate(); } + } } -// input: LTO IR fragments -// output: Linked/Finalized CUBIN. Contains kernel(s) from all fragments -std::vector link_fragments(std::span> fragments, - std::span fragment_types, - std::span names, - std::span link_options) +library library_t::load(load_params const& params) { CUDF_FUNC_RANGE(); + CUlibrary handle; + + CUDFRTC_CHECK_CUDA( + cuLibraryLoadData(&handle, params.binary.data(), nullptr, nullptr, 0, nullptr, nullptr, 0)); + + CUDF_DEFER([&] { + if (handle != nullptr) { CUDFRTC_CHECK_CUDA(cuLibraryUnload(handle)); } + }); + + auto module = std::make_shared(handle); - // [ ] create hash - // [ ] serialize link options, names, and fragments - // [ ] span exact_hash; request from user + handle = nullptr; + + return module; +} + +blob library_t::link_as_blob(link_params const& params) +{ + CUDF_FUNC_RANGE(); - // auto sm = get_device_compute_model(); - // auto arch = std::format("--gpu-architecture=compute_{}", sm); - // const char* link_options[] = {"-lto", arch.c_str()}; + CUDF_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, + "Only CUBIN and PTX output types are supported for linking modules", + std::logic_error); - std::span ltoIR1 = fragments[0]; - char const* ltoIR1_name = names[0]; - std::span ltoIR2 = fragments[1]; - char const* ltoIR2_name = names[1]; nvJitLinkHandle handle; - nvJitLinkCreate(&handle, static_cast(std::size(link_options)), link_options.data()); - nvJitLinkAddData(handle, NVJITLINK_INPUT_LTOIR, (void*)ltoIR1.data(), ltoIR1.size(), ltoIR1_name); - nvJitLinkAddData(handle, NVJITLINK_INPUT_LTOIR, (void*)ltoIR2.data(), ltoIR2.size(), ltoIR2_name); - // Call to nvJitLinkComplete causes linker to link together the two LTO IR modules, do - // optimization on the linked LTO IR, and generate cubin from it. - nvJitLinkComplete(handle); + CUDFRTC_CHECK_NVJITLINK(nvJitLinkCreate(&handle, + static_cast(params.link_options.size()), + const_cast(params.link_options.data()))); + + CUDF_DEFER([&] { nvJitLinkDestroy(&handle); }); + + for (size_t i = 0; i < params.fragments.size(); i++) { + auto name = params.names[i]; + auto fragment = params.fragments[i]; + auto bin_type = params.binary_types[i]; - // nvJitLinkGetErrorLog() - // nvJitLinkGetInfoLog() + nvJitLinkInputType nv_type; - // get linked cubin - size_t cubinSize; - nvJitLinkGetLinkedCubinSize(handle, &cubinSize); + switch (bin_type) { + case binary_type::LTO_IR: nv_type = NVJITLINK_INPUT_LTOIR; break; + case binary_type::CUBIN: nv_type = NVJITLINK_INPUT_CUBIN; break; + case binary_type::FATBIN: nv_type = NVJITLINK_INPUT_FATBIN; break; + case binary_type::PTX: nv_type = NVJITLINK_INPUT_PTX; break; + default: CUDF_FAIL("Unsupported binary type for loading fragment", std::logic_error); + } - std::vector cubin; - cubin.resize(cubinSize); - nvJitLinkGetLinkedCubin(handle, cubin.data()); - nvJitLinkDestroy(&handle); - return cubin; + CUDFRTC_CHECK_NVJITLINK( + nvJitLinkAddData(handle, nv_type, fragment.data(), fragment.size_bytes(), name)); + } + + nvJitLinkResult link_result = nvJitLinkComplete(handle); + + log_nvJitLink_link_result(params, handle, link_result); + + switch (params.output_type) { + case binary_type::CUBIN: { + size_t cubin_size; + CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedCubinSize(handle, &cubin_size)); + std::vector cubin; + cubin.resize(cubin_size); + CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedCubin(handle, cubin.data())); + return std::make_shared(blob_t::from_vector(std::move(cubin))); + } break; + + case binary_type::PTX: { + size_t ptx_size; + CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedPtxSize(handle, &ptx_size)); + std::vector ptx; + ptx.resize(ptx_size); + CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedPtx(handle, reinterpret_cast(ptx.data()))); + return std::make_shared(blob_t::from_vector(std::move(ptx))); + } break; + + default: + CUDF_FAIL("Unsupported output binary type for linking CUDA libraries", std::runtime_error); + } } -// input: linked CUBIN -// output: CUmodule with launchable kernels -CUmodule create_module(std::span cubin) +library library_t::link(link_params const& params) { CUDF_FUNC_RANGE(); + auto blob = link_as_blob(params); + return load(load_params{blob->view(), params.output_type}); +} - CUmodule module; - // cubin is linked, so now load it - cuModuleLoadData(&module, cubin.data()); - // cuModuleGetFunctionCount(unsigned int *count, CUmodule mod); - // cuModuleEnumerateFunctions - // cuModuleGetFunction(CUfunction *hfunc, CUmodule hmod, const char *name) - // cuLibraryGetKernel(CUkernel *pKernel, CUlibrary library, const char *name) - // cuLibraryGetKernelCount(unsigned int *count, CUlibrary lib) - - return module; +kernel_ref library_t::get_kernel(char const* name) const +{ + CUkernel kernel; + CUDFRTC_CHECK_CUDA(cuLibraryGetKernel(&kernel, handle_, name)); + return kernel_ref{kernel}; } -// SIMPLE KEY: from user -// COMPLEX KEY: sha256 of all parameters and blobs involved; + driver + runtime -// +std::vector library_t::enumerate_kernels() const +{ + uint32_t num_kernels; + CUDFRTC_CHECK_CUDA(cuLibraryGetKernelCount(&num_kernels, handle_)); + + std::vector kernels; + kernels.resize(num_kernels); -// [ ] All these functions should have wrappers that cache results and request an optional key from -// the user + CUDFRTC_CHECK_CUDA(cuLibraryEnumerateKernels(kernels.data(), num_kernels, handle_)); -// [ ] Method to pre-compile library and reuse it across multiple operators; compile_library(); -// compile_library_cached() + std::vector result; + for (CUkernel k : kernels) { + result.emplace_back(k); + } -// [ ] jit_key(key) -> key+driver+CUDA_versions+CUDA_runtime_versions+device_compute_models -void setup_context() -{ - // #include + return result; } } // namespace rtc +std::string rtc::demangle_cuda_symbol(char const* mangled_name) +{ + int status; + size_t length; + + char* demangled_name = abi::__cxa_demangle(mangled_name, nullptr, &length, &status); + + CUDF_EXPECTS(status == 0, "Demangling CUDA symbol name failed"); + CUDF_EXPECTS(demangled_name != nullptr, "Demangling CUDA symbol name failed"); + + CUDF_DEFER([&] { + if (demangled_name != nullptr) free(demangled_name); + }); + + std::string result{demangled_name}; + + return result; +} + } // namespace cudf diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 6e366ad43bc3..11098bbcfd77 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -11,8 +11,9 @@ #include extern "C" { -typedef struct nvJitLink* nvJitLinkHandle; -typedef struct CUmod_st* CUmodule; +typedef struct CUlib_st* CUlibrary; +typedef struct CUkern_st* CUkernel; +typedef struct CUstream_st* CUstream; } namespace cudf { @@ -20,11 +21,50 @@ namespace rtc { enum class binary_type : int8_t { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; -using blob_t = std::vector; -using blob = std::shared_ptr; - using blob_view = std::span; +/// @brief Represents a binary blob +/// @details Manages the lifetime of the binary data via a user-provided +/// deallocator function. This enables zero-copy usage of binary data stored +/// in various forms (e.g., std::vector, mmap'd file, etc.). +struct [[nodiscard]] blob_t { + using dealloctor_fn = void (*)(void*, uint8_t const*, size_t); + + private: + static void noop_deallocator(void*, uint8_t const*, size_t) {} + + uint8_t const* data_; + size_t size_; + void* user_data_; + dealloctor_fn deallocator_; + + blob_t(uint8_t const* data, size_t size, void* user_data, dealloctor_fn deallocator) + : data_(data), size_(size), user_data_(user_data), deallocator_(deallocator) + { + } + + public: + blob_t() : data_(nullptr), size_(0), user_data_(nullptr), deallocator_(noop_deallocator) {} + blob_t(blob_t const&) = delete; + blob_t& operator=(blob_t const&) = delete; + blob_t(blob_t&& other) noexcept; + blob_t& operator=(blob_t&& other) noexcept; + ~blob_t() { deallocator_(user_data_, data_, size_); } + + [[nodiscard]] blob_view view() const { return blob_view{data_, size_}; } + + static blob_t from_parts(uint8_t const* data, + size_t size, + void* user_data, + dealloctor_fn deallocator); + + static blob_t from_vector(std::vector&& data); + + static blob_t from_static_data(std::span data); +}; + +using blob = std::shared_ptr; + struct [[nodiscard]] header_map { std::span include_names = {}; // null-terminated header include names std::span headers = {}; // null-terminated header contents @@ -35,19 +75,21 @@ struct fragment_t; using fragment = std::shared_ptr; +/// @brief Represents a partially compiled RTC kernel (i.e. fragment) in LTO-IR or PTX struct fragment_t { private: - nvJitLinkHandle handle_; + blob blob_; binary_type type_; public: struct load_params { - blob_view source = {}; - char const* name = nullptr; + blob blob = {}; binary_type type = binary_type::LTO_IR; }; struct compile_params { + char const* name = nullptr; + char const* source = nullptr; header_map headers = {}; std::span options = {}; binary_type target_type = binary_type::LTO_IR; @@ -57,63 +99,85 @@ struct fragment_t { [[nodiscard]] static fragment compile(compile_params const& params); + fragment_t(blob blob, binary_type type) : blob_(std::move(blob)), type_(type) {} fragment_t(fragment_t const&) = delete; fragment_t(fragment_t&&) = delete; fragment_t& operator=(fragment_t const&) = delete; fragment_t& operator=(fragment_t&&) = delete; - ~fragment_t(); + ~fragment_t() = default; - [[nodiscard]] blob get_lto_ir() const; + [[nodiscard]] binary_type get_type() const { return type_; } - [[nodiscard]] blob get_cubin() const; + [[nodiscard]] blob const& get_lto_ir() const; - private: - fragment_t(nvJitLinkHandle handle, binary_type type) : handle_(handle), type_(type) {} + [[nodiscard]] blob const& get_cubin() const; }; -struct module_t; +struct library_t; -using module = std::shared_ptr; +using library = std::shared_ptr; -void mangle_cxx_symbol(char const*); +struct kernel_ref { + private: + CUkernel handle_; -struct function_ref { - void launch(); + public: + explicit kernel_ref(CUkernel handle) : handle_(handle) {} + + void launch(uint32_t grid_dim_x, + uint32_t grid_dim_y, + uint32_t grid_dim_z, + uint32_t block_dim_x, + uint32_t block_dim_y, + uint32_t block_dim_z, + uint32_t shared_mem_bytes, + CUstream stream, + void** kernel_params); + + [[nodiscard]] std::string_view get_name() const; }; -struct module_t { +/// @brief Represents a loaded RTC library containing compiled kernels +/// Input: CUBIN or PTX binary +/// Output: loaded library with launchable kernels +struct library_t { private: - CUmodule handle_; + CUlibrary handle_; public: - module_t(module_t const&) = delete; - module_t(module_t&&) = delete; - module_t& operator=(module_t const&) = delete; - module_t& operator=(module_t&&) = delete; - ~module_t(); + explicit library_t(CUlibrary handle) : handle_(handle) {} + library_t(library_t const&) = delete; + library_t(library_t&&) = delete; + library_t& operator=(library_t const&) = delete; + library_t& operator=(library_t&&) = delete; + ~library_t(); struct load_params { - blob_view cubin = {}; + blob_view binary = {}; + binary_type type = binary_type::CUBIN; }; struct link_params { + char const* name = nullptr; + binary_type output_type = binary_type::CUBIN; std::span fragments = {}; std::span binary_types = {}; std::span names = {}; std::span link_options = {}; }; - [[nodiscard]] static blob link_as_cubin(link_params const& params); + [[nodiscard]] static library load(load_params const& params); - [[nodiscard]] static module load(blob_view cubin); + [[nodiscard]] static blob link_as_blob(link_params const& params); - [[nodiscard]] static module link(link_params const& params); + [[nodiscard]] static library link(link_params const& params); - [[nodiscard]] function_ref get_function(char const* name) const; + [[nodiscard]] kernel_ref get_kernel(char const* name) const; - private: - module_t(CUmodule handle) : handle_(handle) {} + [[nodiscard]] std::vector enumerate_kernels() const; }; +std::string demangle_cuda_symbol(char const* mangled_name); + } // namespace rtc } // namespace cudf diff --git a/cpp/src/jit/rtc/sha256.hpp b/cpp/src/jit/rtc/sha256.hpp index cec72cdc6865..371d60a90172 100644 --- a/cpp/src/jit/rtc/sha256.hpp +++ b/cpp/src/jit/rtc/sha256.hpp @@ -18,19 +18,21 @@ typedef struct evp_md_ctx_st EVP_MD_CTX; namespace CUDF_EXPORT cudf { namespace rtc { -struct sha256_hex_string { +struct [[nodiscard]] sha256_hex_string { char data_[65]; - constexpr operator std::string_view() const { return std::string_view{data_, 64}; } + constexpr std::string_view view() const { return std::string_view{data_, 64}; } - char const* data() const { return data_; } + constexpr operator std::string_view() const { return view(); } - char const* c_str() const { return data_; } + [[nodiscard]] char const* data() const { return data_; } + + [[nodiscard]] char const* c_str() const { return data_; } static constexpr size_t size() { return 64; } }; -struct sha256_hash { +struct [[nodiscard]] sha256_hash { alignas(16) uint8_t data_[32]; constexpr bool operator==(sha256_hash const& hash) const @@ -40,7 +42,7 @@ struct sha256_hash { constexpr bool operator!=(sha256_hash const& hash) const { return !(*this == hash); } - constexpr sha256_hex_string to_hex() const + constexpr sha256_hex_string to_hex_string() const { static constexpr char const HEX_CHARS[] = "0123456789abcdef"; sha256_hex_string hex; @@ -53,11 +55,11 @@ struct sha256_hash { } }; -struct sha256_hash_hasher { +struct [[nodiscard]] sha256_hash_hasher { constexpr uint64_t operator()(sha256_hash const& obj) const { struct u64x4 { - uint64_t v[4]; + alignas(16) uint64_t v[4]; }; auto value = std::bit_cast(obj); @@ -75,7 +77,7 @@ struct sha256_hash_hasher { } }; -struct sha256_context { +struct [[nodiscard]] sha256_context { private: EVP_MD_CTX* ectx_; From 18b51b26f3c7ece603d0bf8b4b389012f20f8b76 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 9 Dec 2025 12:01:45 +0000 Subject: [PATCH 004/254] checkpoint --- cpp/src/jit/rtc/cache.cpp | 46 +++++++++++++++---------------- cpp/src/jit/rtc/cache.hpp | 58 +++++++++++++++++++-------------------- cpp/src/jit/rtc/cudf.hpp | 18 ++++++++---- cpp/src/jit/rtc/rtc.cpp | 2 +- cpp/src/jit/rtc/rtc.hpp | 2 +- 5 files changed, 65 insertions(+), 61 deletions(-) diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index 30a4f9eaadb6..bd6e4e85c9c0 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -34,9 +34,7 @@ namespace { } // namespace -compile_cache_t::compile_cache_t(bool enabled, - std::string cache_dir, - compile_cache_limits const& limits) +cache_t::cache_t(bool enabled, std::string cache_dir, cache_limits const& limits) : enabled_{enabled}, cache_dir_{std::move(cache_dir)}, limits_{limits}, @@ -51,13 +49,13 @@ compile_cache_t::compile_cache_t(bool enabled, } } -bool compile_cache_t::is_enabled() const { return enabled_; } +bool cache_t::is_enabled() const { return enabled_; } -void compile_cache_t::enable() { enabled_ = true; } +void cache_t::enable() { enabled_ = true; } -void compile_cache_t::disable() { enabled_ = false; } +void cache_t::disable() { enabled_ = false; } -void compile_cache_t::store_blob_to_memory(sha256_hash const& sha, std::shared_future binary) +void cache_t::store_blob_to_memory(sha256_hash const& sha, std::shared_future binary) { CUDF_FUNC_RANGE(); @@ -71,7 +69,7 @@ void compile_cache_t::store_blob_to_memory(sha256_hash const& sha, std::shared_f blobs_cache_.insert(sha, std::move(binary), current_tick); } -void compile_cache_t::store_blob_to_disk(sha256_hash const& sha, blob_view binary) +void cache_t::store_blob_to_disk(sha256_hash const& sha, blob_view binary) { CUDF_FUNC_RANGE(); @@ -117,8 +115,7 @@ void compile_cache_t::store_blob_to_disk(sha256_hash const& sha, blob_view binar } } -std::optional> compile_cache_t::query_blob_from_memory( - sha256_hash const& sha) +std::optional> cache_t::query_blob_from_memory(sha256_hash const& sha) { CUDF_FUNC_RANGE(); @@ -142,7 +139,7 @@ std::optional> compile_cache_t::query_blob_from_memory( } } -std::optional compile_cache_t::query_blob_from_disk(sha256_hash const& sha) +std::optional cache_t::query_blob_from_disk(sha256_hash const& sha) { CUDF_FUNC_RANGE(); @@ -185,7 +182,7 @@ std::optional compile_cache_t::query_blob_from_disk(sha256_hash const& sha blob_t::from_parts(static_cast(map), file_size, nullptr, deleter)); } -void compile_cache_t::store_fragment(sha256_hash const& sha, std::shared_future frag) +void cache_t::store_fragment(sha256_hash const& sha, std::shared_future frag) { CUDF_FUNC_RANGE(); @@ -201,7 +198,7 @@ void compile_cache_t::store_fragment(sha256_hash const& sha, std::shared_future< } } -std::optional> compile_cache_t::query_fragment(sha256_hash const& sha) +std::optional> cache_t::query_fragment(sha256_hash const& sha) { CUDF_FUNC_RANGE(); @@ -225,7 +222,7 @@ std::optional> compile_cache_t::query_fragment(sha2 } } -void compile_cache_t::store_library(sha256_hash const& sha, std::shared_future mod) +void cache_t::store_library(sha256_hash const& sha, std::shared_future mod) { CUDF_FUNC_RANGE(); @@ -241,7 +238,7 @@ void compile_cache_t::store_library(sha256_hash const& sha, std::shared_future> compile_cache_t::query_library(sha256_hash const& sha) +std::optional> cache_t::query_library(sha256_hash const& sha) { CUDF_FUNC_RANGE(); @@ -265,13 +262,13 @@ std::optional> compile_cache_t::query_library(sha256 } } -compile_cache_statistics compile_cache_t::get_statistics() const { return counter_.get(); } +cache_statistics cache_t::get_statistics() const { return counter_.get_statistics(); } -void compile_cache_t::clear_statistics() { counter_.clear(); } +void cache_t::clear_statistics() { counter_.clear(); } -compile_cache_limits compile_cache_t::get_limits() const { return limits_; } +cache_limits cache_t::get_limits() const { return limits_; } -size_t compile_cache_t::get_blob_count() const +size_t cache_t::get_blob_count() const { CUDF_FUNC_RANGE(); @@ -281,7 +278,7 @@ size_t compile_cache_t::get_blob_count() const } } -size_t compile_cache_t::get_fragment_count() const +size_t cache_t::get_fragment_count() const { CUDF_FUNC_RANGE(); @@ -291,7 +288,7 @@ size_t compile_cache_t::get_fragment_count() const } } -size_t compile_cache_t::get_library_count() const +size_t cache_t::get_library_count() const { CUDF_FUNC_RANGE(); @@ -301,7 +298,7 @@ size_t compile_cache_t::get_library_count() const } } -void compile_cache_t::clear_memory_store() +void cache_t::clear_memory_store() { CUDF_FUNC_RANGE(); @@ -321,9 +318,10 @@ void compile_cache_t::clear_memory_store() } } -void compile_cache_t::clear_disk_store() +void cache_t::clear_disk_store() { CUDF_FUNC_RANGE(); + DIR* dir = opendir(cache_dir_.c_str()); if (dir == nullptr) { throw_posix("Failed to open RTC cache directory for clearing", "opendir"); } diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index 223bab584ec7..fbe18f7f56d2 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -19,7 +19,7 @@ namespace cudf { namespace rtc { -struct [[nodiscard]] compile_cache_statistics { +struct [[nodiscard]] cache_statistics { uint64_t blob_memory_hits = 0; uint64_t blob_memory_misses = 0; uint64_t fragment_memory_hits = 0; @@ -30,7 +30,7 @@ struct [[nodiscard]] compile_cache_statistics { uint64_t blob_disk_misses = 0; }; -struct [[nodiscard]] compile_cache_limits { +struct [[nodiscard]] cache_limits { uint32_t num_blobs = 1024; uint32_t num_fragments = 1024; uint32_t num_libraries = 1024; @@ -196,7 +196,7 @@ struct alignas(std::hardware_destructive_interference_size) counter { } }; -struct compile_cache_statistics_counter { +struct cache_statistics_counter { counter blob_memory_hits; counter blob_memory_misses; counter fragment_memory_hits; @@ -234,22 +234,22 @@ struct compile_cache_statistics_counter { void miss_disk_blob() { blob_disk_misses.increment(); } - compile_cache_statistics get() const + cache_statistics get_statistics() const { - return compile_cache_statistics{.blob_memory_hits = blob_memory_hits.get(), - .blob_memory_misses = blob_memory_misses.get(), - .fragment_memory_hits = fragment_memory_hits.get(), - .fragment_memory_misses = fragment_memory_misses.get(), - .library_memory_hits = library_memory_hits.get(), - .library_memory_misses = library_memory_misses.get(), - .blob_disk_hits = blob_disk_hits.get(), - .blob_disk_misses = blob_disk_misses.get()}; + return cache_statistics{.blob_memory_hits = blob_memory_hits.get(), + .blob_memory_misses = blob_memory_misses.get(), + .fragment_memory_hits = fragment_memory_hits.get(), + .fragment_memory_misses = fragment_memory_misses.get(), + .library_memory_hits = library_memory_hits.get(), + .library_memory_misses = library_memory_misses.get(), + .blob_disk_hits = blob_disk_hits.get(), + .blob_disk_misses = blob_disk_misses.get()}; } }; } // namespace detail -/// @brief Compile cache for RTC blobs, fragments, and libraries +/// @brief Thread-safe compile cache for compiled blobs, fragments, and libraries /// @details Provides in-memory and on-disk caching of compiled RTC artifacts. /// The cache uses an LRU eviction policy when the number of cached items /// exceeds user-defined limits. @@ -261,13 +261,13 @@ struct compile_cache_statistics_counter { /// in-memory and on-disk caches to help monitor cache performance in benchmarking and debugging. /// The interface is zero-copy throughout, using shared pointers and spans to avoid unnecessary data /// copying across threads and disk. -struct compile_cache_t { +struct cache_t { private: bool enabled_; std::string cache_dir_; - compile_cache_limits limits_; + cache_limits limits_; detail::lru_memory_cache> blobs_cache_; @@ -275,24 +275,24 @@ struct compile_cache_t { detail::lru_memory_cache> libraries_cache_; - detail::compile_cache_statistics_counter counter_; + detail::cache_statistics_counter counter_; alignas(std::hardware_destructive_interference_size) uint64_t tick_; public: - compile_cache_t(bool enabled, std::string cache_dir, compile_cache_limits const& limits); - compile_cache_t(compile_cache_t const&) = delete; - compile_cache_t& operator=(compile_cache_t const&) = delete; - compile_cache_t(compile_cache_t&&) = delete; - compile_cache_t& operator=(compile_cache_t&&) = delete; - ~compile_cache_t() = default; + cache_t(bool enabled, std::string cache_dir, cache_limits const& limits); + cache_t(cache_t const&) = delete; + cache_t& operator=(cache_t const&) = delete; + cache_t(cache_t&&) = delete; + cache_t& operator=(cache_t&&) = delete; + ~cache_t() = default; + + [[nodiscard]] bool is_enabled() const; void enable(); void disable(); - [[nodiscard]] bool is_enabled(); - void store_blob_to_memory(sha256_hash const& sha, std::shared_future binary); void store_blob_to_disk(sha256_hash const& sha, blob_view binary); @@ -309,17 +309,17 @@ struct compile_cache_t { std::optional> query_library(sha256_hash const& sha); - compile_cache_statistics get_statistics() const; + cache_statistics get_statistics() const; void clear_statistics(); - compile_cache_limits get_limits() const; + cache_limits get_limits() const; - [[nodiscard]] size_t get_blob_count() const; + [[nodiscard]] size_t get_blob_count() const; [[nodiscard]] size_t get_fragment_count() const; - [[nodiscard]] size_t get_library_count() const; + [[nodiscard]] size_t get_library_count() const; void clear_memory_store(); diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/rtc/cudf.hpp index bd138e83bf55..e621ee5bfe08 100644 --- a/cpp/src/jit/rtc/cudf.hpp +++ b/cpp/src/jit/rtc/cudf.hpp @@ -1,7 +1,14 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +namespace cudf { +namespace rtc { // SIMPLE KEY: from user // COMPLEX KEY: sha256 of all parameters and blobs involved; + driver + runtime @@ -14,8 +21,7 @@ // compile_library_cached() // [ ] jit_key(key) -> key+driver+CUDA_versions+CUDA_runtime_versions+device_compute_models - // #include - +// #include // [ ] environment variables to control: // [ ] cache path @@ -24,12 +30,12 @@ // [ ] cache statistics: hits, misses, etc. // [ ] on startup, log cache path, loading information, etc. -compile_cache_t& get_cache(); - // [ ] if a user provides a key, use: USER_KEY+${key}, otherwise use sha256 of contents // [ ] use resource type in key to avoid collisions -void compile_operator(); void make_pch(); -void link_operator(); \ No newline at end of file +void link_operator(); + +} // namespace rtc +} // namespace cudf diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index a0be6b3e0ed8..24bf50edca15 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 11098bbcfd77..322709269b45 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ From 46d8dd46984ed2a7052758550ed9697f5cfaddb5 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 9 Dec 2025 18:43:34 +0000 Subject: [PATCH 005/254] checkpoint --- cpp/CMakeLists.txt | 173 +++++++++++---------------- cpp/embed.py | 44 +++++++ cpp/include/cudf/jit/lto/types.cuh | 2 +- cpp/include/cudf/utilities/defer.hpp | 6 +- cpp/src/jit/rtc/cudf.cpp | 9 ++ cpp/src/jit/rtc/rtc.cpp | 7 +- todo.md | 1 - 7 files changed, 130 insertions(+), 112 deletions(-) create mode 100644 cpp/embed.py diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bed2275ce135..08bafb2505ba 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -350,125 +350,36 @@ function(embed_fatbins_command target_name) endfunction() - -""" - -# Take a list of files to JIT-compile and run them through jitify_preprocess. -function(jit_preprocess_files) - cmake_parse_arguments(ARG "" "SOURCE_DIRECTORY" "FILES" ${ARGN}) - - get_target_property(libcudacxx_raw_includes CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) - set(includes) - foreach(inc IN LISTS libcudacxx_raw_includes CUDAToolkit_INCLUDE_DIRS) - list(APPEND includes "-I${inc}") - endforeach() - foreach(ARG_FILE ${ARG_FILES}) - set(ARG_OUTPUT ${CUDF_GENERATED_INCLUDE_DIR}/include/jit_preprocessed_files/${ARG_FILE}.jit.hpp) - get_filename_component(jit_output_directory "${ARG_OUTPUT}" DIRECTORY) - list(APPEND JIT_PREPROCESSED_FILES "${ARG_OUTPUT}") - - get_filename_component(ARG_OUTPUT_DIR "${ARG_OUTPUT}" DIRECTORY) - - # Note: need to pass _FILE_OFFSET_BITS=64 in COMMAND due to a limitation in how conda builds - # glibc - add_custom_command( - OUTPUT ${ARG_OUTPUT} - DEPENDS jitify_preprocess "${ARG_SOURCE_DIRECTORY}/${ARG_FILE}" - WORKING_DIRECTORY ${ARG_SOURCE_DIRECTORY} - VERBATIM - COMMAND ${CMAKE_COMMAND} -E make_directory "${jit_output_directory}" - COMMAND - "${CMAKE_COMMAND}" -E env LD_LIBRARY_PATH=${CUDAToolkit_LIBRARY_DIR} - $ ${ARG_FILE} -o ${ARG_OUTPUT_DIR} -i -std=c++20 - -remove-unused-globals -D_FILE_OFFSET_BITS=64 -D__CUDACC_RTC__ -DCUDF_RUNTIME_JIT - -I${CUDF_SOURCE_DIR}/include -I${CUDF_SOURCE_DIR}/src ${includes} - --no-preinclude-workarounds --no-replace-pragma-once --diag-suppress=47 --device-int128 - COMMENT "Custom command to JIT-compile files." - ) - endforeach() - set(JIT_PREPROCESSED_FILES - "${JIT_PREPROCESSED_FILES}" - PARENT_SCOPE - ) -endfunction() - -if(NOT (EXISTS "${CUDF_GENERATED_INCLUDE_DIR}/include")) - make_directory("${CUDF_GENERATED_INCLUDE_DIR}/include") -endif() - -""" -# needs to recursively bundle the actually used headers; not everything -# should we just use the install include dir? -# provided cudacxx will also be exported? can we get the installed include dir path? -# after installing, should be configure a file that points to the installed cudacxx include dir? - -set(CUDF_JIT_INCLUDE_HEADERS -include/cudf/table/table_device_view.cuh -include/cudf/jit/lto/types.cuh -include/cudf/jit/lto/operators.cuh -include/cudf/jit/lto/library.cuh -include/cudf/jit/lto/thunk.cuh -include/cudf/jit/transform_params.cuh -include/cudf/jit/filter_params.cuh -include/cudf/utilities/traits.cuh -include/cudf/strings/string_view.cuh -include/cudf/column/column_device_view_base.cuh -include/cudf/detail/null_mask.cuh -include/cudf/detail/utilities/grid_1d.cuh -include/cudf/detail/offsets_iterator.cuh -include/cudf/detail/indexalator.cuh -include/cudf/ast/detail/operator_functor.cuh -include/cudf/ast/detail/operators.cuh -src/jit/accessors.cuh -src/jit/span.cuh -) - -# CCCL::libcudacxx - - # we have to copy everything in the include dirs here - # get_target_property(libcudacxx_raw_includes CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) - -set(CUDF_JIT_LTO_INCLUDE_HEADERS -include/cudf/jit/lto/types.cuh -include/cudf/jit/lto/operators.cuh -include/cudf/jit/transform_params.cuh -include/cudf/jit/filter_params.cuh -) - - -# CUDA/STD headers - - if(CMAKE_BUILD_TYPE STREQUAL "Debug") set(CUDF_JIT_CUDA_FLAGS "-G") -else if(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") - set(CUDF_JIT_CUDA_FLAGS "-O2;-DNDEBUG") -else if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(CUDF_JIT_CUDA_FLAGS "-O3;-DNDEBUG") - endif() +else() + if(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + set(CUDF_JIT_CUDA_FLAGS "-O2;-DNDEBUG") + else() + if(CMAKE_BUILD_TYPE STREQUAL "Release") + set(CUDF_JIT_CUDA_FLAGS "-O3;-DNDEBUG") + endif() endif() endif() - if(CUDA_ENABLE_LINEINFO) list(APPEND CUDF_JIT_CUDA_FLAGS "${CUDF_JIT_CUDA_FLAGS};-lineinfo") endif() - - -# use current project CUDA_ARCHITECTURES for JIT compilation set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) set(CUDF_JIT_ARCHITECTURES "75;80;86;90;100;120") -set(CUDF_JIT_DEFINITIONS "${CUDF_CUDA_DEFINITIONS};-DCUDF_RUNTIME_JIT;-D__CUDACC_RTC__") -# set(CUDF_JIT_) -set(CUDF_JIT_COMPILE_FLAGS "-O3;-std=c++${CUDF_CUDA_STANDARD};-Xcompiler=-fPIC;-Xcompiler=-DPIC;-fvisibility=hidden;-Xfatbin=--compress-all;--compress-mode=size") -set(CUDF_JIT_LINK_FLAGS) +set(CUDF_JIT_DEFINITIONS "-DCUDF_RUNTIME_JIT;-D__CUDACC_RTC__") +set(CUDF_JIT_COMPILE_FLAGS + "-O3;-std=c++${CUDF_CUDA_STANDARD};-Xcompiler=-fPIC;-Xcompiler=-DPIC;-fvisibility=hidden;-Xfatbin=--compress-all;--compress-mode=size;--device-int128" +) function(add_lto_ir_library target_name source_file) add_library(${target_name} OBJECT ${source_file}) - target_compile_options(${target_name} PRIVATE -Xfatbin=--compress-all --compress-mode=size) + target_compile_options( + ${target_name} PRIVATE -Xfatbin=--compress-all --compress-mode=size --device-int128 + ) set_target_properties( ${target_name} PROPERTIES CUDA_ARCHITECTURES "75-real;80-real;86-real;90-real;100-real;120-real" @@ -480,7 +391,7 @@ function(add_lto_ir_library target_name source_file) target_include_directories( ${target_name} PUBLIC ${CUDF_SOURCE_DIR}/include ${CUDF_SOURCE_DIR}/src ) - target_compile_definitions(${target_name} PUBLIC ${CUDF_CUDA_DEFINITIONS} -DCUDF_RUNTIME_JIT -D__CUDACC_RTC__) + target_compile_definitions(${target_name} PUBLIC ${CUDF_CUDA_DEFINITIONS} -DCUDF_RUNTIME_JIT) target_link_libraries(${target_name} PUBLIC CCCL::libcudacxx) set_target_properties( ${target_name} @@ -494,11 +405,61 @@ function(add_lto_ir_library target_name source_file) CUDA_VISIBILITY_PRESET hidden ) - embed_fatbins_command(${target_name}) - endfunction() add_lto_ir_library(cudf_lto_library src/jit/lto/library.cu) +embed_fatbins_command(cudf_lto_library) + +function(embed_jit_sources_var) + +endfunction() + +function(embed_jit_options_var) + +endfunction() + +embed_jit_sources_var( + identifier_name + EXPLICIT + SOURCE_DIRECTORY + cudf/include/ + FILES + cudf/table/table_device_view.cuh + cudf/jit/lto/types.cuh + cudf/jit/lto/operators.cuh + cudf/jit/lto/library.cuh + cudf/jit/lto/thunk.cuh + cudf/jit/transform_params.cuh + cudf/jit/filter_params.cuh + cudf/utilities/traits.cuh + cudf/strings/string_view.cuh + cudf/column/column_device_view_base.cuh + cudf/detail/null_mask.cuh + cudf/detail/utilities/grid_1d.cuh + cudf/detail/offsets_iterator.cuh + cudf/detail/indexalator.cuh + cudf/ast/detail/operator_functor.cuh + cudf/ast/detail/operators.cuh + cudf/jit/lto/types.cuh + cudf/jit/lto/operators.cuh + cudf/jit/transform_params.cuh + cudf/jit/filter_params.cuh +) + +embed_jit_sources_var( + identifier_name EXPLICIT SOURCE_DIRECTORY src FILES src/jit/accessors.cuh src/jit/span.cuh +) + +get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) + +# CUDA/STD headers +# needs to recursively bundle the actually used headers; not everything should we just use the +# install include dir? provided cudacxx will also be exported? can we get the installed include dir +# path? after installing, should be configure a file that points to the installed cudacxx include +# dir? +embed_jit_sources_var(identifier_name RECURSIVE SOURCE_DIRECTORY ${LIBCUDACXX_RAW_INCLUDE_DIRS}) + +embed_jit_options_var(identifier_name OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- diff --git a/cpp/embed.py b/cpp/embed.py new file mode 100644 index 000000000000..57be24125e3b --- /dev/null +++ b/cpp/embed.py @@ -0,0 +1,44 @@ + + +# create header file to embed header map +import sys +import hashlib + +# [ ] include dir resolution +# [ ] get headers and options and outputs as JSON-encoded HEX strings? +# [ ] usage: python3 rtc_embed.py --embed --header --type=explicit|recursive header_map_id:reference_dir:[list of header files] --option option_id:[compiler options] --output output_file + + +""" headers + +struct { +unsigned char const * const * include_names; +unsigned char const * const * headers; +unsigned long long const * header_sizes; +unsigned char * headers_sha256; +unsigned long long num_includes; +unsigned char sha256[32]; +} ${variable_name} = { ... }; + +""" + +"""options + +struct { +unsigned char const * const * options; +unsigned char * options_sha256; +unsigned long long num_options; +unsigned char sha256[32]; +} ${variable_name} = { ... }; + +""" + +"""binary blob + +struct { +unsigned char const * data; +unsigned long long size; +unsigned char sha256[32]; +} ${variable_name} = { ... }; + +""" \ No newline at end of file diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index b04d48eb32f3..71c9b6e8d6e0 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -4,7 +4,7 @@ */ #pragma once -#define CUDF_LTO_EXPORT __attribute__((visibility("default"))) +#define CUDF_LTO_EXPORT namespace CUDF_LTO_EXPORT cudf { diff --git a/cpp/include/cudf/utilities/defer.hpp b/cpp/include/cudf/utilities/defer.hpp index d17607a29fb7..2f924eb25bb3 100644 --- a/cpp/include/cudf/utilities/defer.hpp +++ b/cpp/include/cudf/utilities/defer.hpp @@ -13,6 +13,10 @@ namespace CUDF_EXPORT cudf { +/// @brief RAII utility to execute a callable at the end of a scope. +/// This is useful for ensuring cleanup code is executed, even in the presence of exceptions. +/// And is intended for wrapping C APIs that require explicit resource management without having to +/// write custom wrapper types. template struct defer { private: @@ -33,4 +37,4 @@ struct defer { template defer(T) -> defer; -} // namespace CUDF_EXPORT cudf \ No newline at end of file +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 139597f9cb07..949a4ae9bf42 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -1,2 +1,11 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace cudf { +namespace rtc { + +} +} // namespace cudf diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 24bf50edca15..66660221f6b0 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -269,14 +269,12 @@ fragment fragment_t::compile(compile_params const& params) blob const& fragment_t::get_lto_ir() const { - CUDF_FUNC_RANGE(); CUDF_EXPECTS(type_ == binary_type::LTO_IR, "Fragment does not contain LTO IR"); return blob_; } blob const& fragment_t::get_cubin() const { - CUDF_FUNC_RANGE(); CUDF_EXPECTS(type_ == binary_type::CUBIN, "Fragment does not contain CUBIN"); return blob_; } @@ -291,6 +289,8 @@ void kernel_ref::launch(uint32_t grid_dim_x, CUstream stream, void** kernel_params) { + CUDF_FUNC_RANGE(); + CUlaunchConfig cfg{.gridDimX = grid_dim_x, .gridDimY = grid_dim_y, .gridDimZ = grid_dim_z, @@ -307,7 +307,6 @@ void kernel_ref::launch(uint32_t grid_dim_x, std::string_view kernel_ref::get_name() const { - CUDF_FUNC_RANGE(); char const* name; CUDFRTC_CHECK_CUDA(cuKernelGetName(&name, handle_)); return std::string_view{name == nullptr ? "" : name}; @@ -323,6 +322,7 @@ library_t::~library_t() library library_t::load(load_params const& params) { CUDF_FUNC_RANGE(); + CUlibrary handle; CUDFRTC_CHECK_CUDA( @@ -405,6 +405,7 @@ blob library_t::link_as_blob(link_params const& params) library library_t::link(link_params const& params) { CUDF_FUNC_RANGE(); + auto blob = link_as_blob(params); return load(load_params{blob->view(), params.output_type}); } diff --git a/todo.md b/todo.md index e6a84381b842..29922bc7b073 100644 --- a/todo.md +++ b/todo.md @@ -1,3 +1,2 @@ [ ] Use PCH [ ] Add time trace to JIT and AOT - From 2c5bf47a1f9e657c1889f64eda2a0d373670f43d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 9 Dec 2025 21:14:42 +0000 Subject: [PATCH 006/254] checkpoint --- cpp/CMakeLists.txt | 17 +++++++++-------- cpp/src/jit/rtc/cache.cpp | 28 ++++++++++++++++++++-------- cpp/src/jit/rtc/cache.hpp | 21 ++++++++++++--------- cpp/src/jit/rtc/rtc.cpp | 2 +- cpp/src/jit/rtc/rtc.hpp | 2 +- cpp/src/jit/rtc/sha256.hpp | 2 +- 6 files changed, 44 insertions(+), 28 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 08bafb2505ba..0d8d2407b0bf 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -377,9 +377,7 @@ set(CUDF_JIT_COMPILE_FLAGS function(add_lto_ir_library target_name source_file) add_library(${target_name} OBJECT ${source_file}) - target_compile_options( - ${target_name} PRIVATE -Xfatbin=--compress-all --compress-mode=size --device-int128 - ) + target_compile_options(${target_name} PRIVATE -Xfatbin=--compress-all --compress-mode=size) set_target_properties( ${target_name} PROPERTIES CUDA_ARCHITECTURES "75-real;80-real;86-real;90-real;100-real;120-real" @@ -452,11 +450,10 @@ embed_jit_sources_var( get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) -# CUDA/STD headers -# needs to recursively bundle the actually used headers; not everything should we just use the -# install include dir? provided cudacxx will also be exported? can we get the installed include dir -# path? after installing, should be configure a file that points to the installed cudacxx include -# dir? +# CUDA/STD headers needs to recursively bundle the actually used headers; not everything should we +# just use the install include dir? provided cudacxx will also be exported? can we get the installed +# include dir path? after installing, should be configure a file that points to the installed +# cudacxx include dir? embed_jit_sources_var(identifier_name RECURSIVE SOURCE_DIRECTORY ${LIBCUDACXX_RAW_INCLUDE_DIRS}) embed_jit_options_var(identifier_name OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) @@ -706,6 +703,10 @@ add_library( src/jit/parser.cpp src/jit/row_ir.cpp src/jit/util.cpp + src/jit/rtc/cache.cpp + src/jit/rtc/cudf.cpp + src/jit/rtc/rtc.cpp + src/jit/rtc/sha256.cpp src/join/conditional_join.cu src/join/cross_join.cu src/join/distinct_hash_join.cu diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index bd6e4e85c9c0..05a045ae84f4 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -49,11 +49,23 @@ cache_t::cache_t(bool enabled, std::string cache_dir, cache_limits const& limits } } -bool cache_t::is_enabled() const { return enabled_; } +bool cache_t::is_enabled() +{ + std::atomic_ref c{enabled_}; + return c.load(std::memory_order_relaxed); +} -void cache_t::enable() { enabled_ = true; } +void cache_t::enable() +{ + std::atomic_ref c{enabled_}; + c.store(true, std::memory_order_relaxed); +} -void cache_t::disable() { enabled_ = false; } +void cache_t::disable() +{ + std::atomic_ref c{enabled_}; + c.store(false, std::memory_order_relaxed); +} void cache_t::store_blob_to_memory(sha256_hash const& sha, std::shared_future binary) { @@ -262,13 +274,13 @@ std::optional> cache_t::query_library(sha256_hash co } } -cache_statistics cache_t::get_statistics() const { return counter_.get_statistics(); } +cache_statistics cache_t::get_statistics() { return counter_.get_statistics(); } void cache_t::clear_statistics() { counter_.clear(); } -cache_limits cache_t::get_limits() const { return limits_; } +cache_limits cache_t::get_limits() { return limits_; } -size_t cache_t::get_blob_count() const +size_t cache_t::get_blob_count() { CUDF_FUNC_RANGE(); @@ -278,7 +290,7 @@ size_t cache_t::get_blob_count() const } } -size_t cache_t::get_fragment_count() const +size_t cache_t::get_fragment_count() { CUDF_FUNC_RANGE(); @@ -288,7 +300,7 @@ size_t cache_t::get_fragment_count() const } } -size_t cache_t::get_library_count() const +size_t cache_t::get_library_count() { CUDF_FUNC_RANGE(); diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index fbe18f7f56d2..072713456fcf 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -128,8 +128,11 @@ struct write_guard { ~write_guard() { lock_.unlock_write(); } }; +inline constexpr size_t CACHELINE_ALIGNMENT = + 64; // = std::hardware_destructive_interference_size */ + template -struct alignas(std::hardware_destructive_interference_size) lru_memory_cache { +struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { struct entry { uint64_t last_touched_tick; T value; @@ -174,7 +177,7 @@ struct alignas(std::hardware_destructive_interference_size) lru_memory_cache { } }; -struct alignas(std::hardware_destructive_interference_size) counter { +struct alignas(CACHELINE_ALIGNMENT) counter { uint64_t value_ = 0; void increment() @@ -277,7 +280,7 @@ struct cache_t { detail::cache_statistics_counter counter_; - alignas(std::hardware_destructive_interference_size) uint64_t tick_; + alignas(detail::CACHELINE_ALIGNMENT) uint64_t tick_; public: cache_t(bool enabled, std::string cache_dir, cache_limits const& limits); @@ -287,7 +290,7 @@ struct cache_t { cache_t& operator=(cache_t&&) = delete; ~cache_t() = default; - [[nodiscard]] bool is_enabled() const; + [[nodiscard]] bool is_enabled(); void enable(); @@ -309,17 +312,17 @@ struct cache_t { std::optional> query_library(sha256_hash const& sha); - cache_statistics get_statistics() const; + cache_statistics get_statistics(); void clear_statistics(); - cache_limits get_limits() const; + cache_limits get_limits(); - [[nodiscard]] size_t get_blob_count() const; + [[nodiscard]] size_t get_blob_count(); - [[nodiscard]] size_t get_fragment_count() const; + [[nodiscard]] size_t get_fragment_count(); - [[nodiscard]] size_t get_library_count() const; + [[nodiscard]] size_t get_library_count(); void clear_memory_store(); diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 66660221f6b0..5674baba7fdf 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -121,7 +121,7 @@ fragment fragment_t::load(load_params const& params) { CUDF_FUNC_RANGE(); - return std::make_shared(params.blob, params.type); + return std::make_shared(params.binary, params.type); } void log_nvrtc_compile_result(fragment_t::compile_params const& params, diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 322709269b45..3f5c26120208 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -83,7 +83,7 @@ struct fragment_t { public: struct load_params { - blob blob = {}; + blob binary = {}; binary_type type = binary_type::LTO_IR; }; diff --git a/cpp/src/jit/rtc/sha256.hpp b/cpp/src/jit/rtc/sha256.hpp index 371d60a90172..ccce4f18b190 100644 --- a/cpp/src/jit/rtc/sha256.hpp +++ b/cpp/src/jit/rtc/sha256.hpp @@ -44,7 +44,7 @@ struct [[nodiscard]] sha256_hash { constexpr sha256_hex_string to_hex_string() const { - static constexpr char const HEX_CHARS[] = "0123456789abcdef"; + constexpr char const HEX_CHARS[] = "0123456789abcdef"; sha256_hex_string hex; for (size_t i = 0; i < 32; ++i) { hex.data_[i * 2] = HEX_CHARS[(data_[i] >> 4) & 0x0F]; From df6a9036a955cf503fa3aa69faa0ee14aa40509f Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 11 Dec 2025 14:59:02 +0000 Subject: [PATCH 007/254] checkpoint --- cpp/CMakeLists.txt | 64 +---- cpp/embed.py | 310 ++++++++++++++++++++-- cpp/include/cudf/jit/filter_params.cuh | 10 +- cpp/include/cudf/jit/lto/types.cuh | 45 ++-- cpp/include/cudf/jit/transform_params.cuh | 10 +- cpp/include/cudf/utilities/defer.hpp | 4 +- cpp/src/jit/rtc/cudf.hpp | 2 +- 7 files changed, 329 insertions(+), 116 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0d8d2407b0bf..a0396ab1ef3d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -333,22 +333,9 @@ if(NOT BUILD_SHARED_LIBS) endif() endif() -# CHANGE TO embed_binary_command -function(embed_fatbins_command target_name) - - set(DEST_FILE ${CUDF_GENERATED_INCLUDE_DIR}/include/${target_name}_fatbin_bytes.h) - add_custom_command( - OUTPUT ${DEST_FILE} - COMMAND ${BIN2C_EXE} --const --type char --static --name ${target_name}_fatbin_bytes - "$" > ${DEST_FILE} - DEPENDS ${target_name} - COMMENT - "Converting fatbins generated by ${target_name} to C source file (${target_name}_fatbin_bytes.h) with bin2c" - ) - - add_custom_target(${target_name}_fatbin_bytes DEPENDS ${DEST_FILE}) +# We can't precompile the LTO-IR as we need to generate fatbins for the specific target machine +# LTO-IR is not compatible across -endfunction() if(CMAKE_BUILD_TYPE STREQUAL "Debug") set(CUDF_JIT_CUDA_FLAGS "-G") @@ -374,51 +361,21 @@ set(CUDF_JIT_COMPILE_FLAGS "-O3;-std=c++${CUDF_CUDA_STANDARD};-Xcompiler=-fPIC;-Xcompiler=-DPIC;-fvisibility=hidden;-Xfatbin=--compress-all;--compress-mode=size;--device-int128" ) -function(add_lto_ir_library target_name source_file) - add_library(${target_name} OBJECT ${source_file}) - target_compile_options(${target_name} PRIVATE -Xfatbin=--compress-all --compress-mode=size) - set_target_properties( - ${target_name} - PROPERTIES CUDA_ARCHITECTURES "75-real;80-real;86-real;90-real;100-real;120-real" - CUDA_SEPARABLE_COMPILATION ON - CUDA_FATBIN_COMPILATION ON - POSITION_INDEPENDENT_CODE ON - INTERPROCEDURAL_OPTIMIZATION ON - ) - target_include_directories( - ${target_name} PUBLIC ${CUDF_SOURCE_DIR}/include ${CUDF_SOURCE_DIR}/src - ) - target_compile_definitions(${target_name} PUBLIC ${CUDF_CUDA_DEFINITIONS} -DCUDF_RUNTIME_JIT) - target_link_libraries(${target_name} PUBLIC CCCL::libcudacxx) - set_target_properties( - ${target_name} - PROPERTIES CXX_STANDARD ${CUDF_CXX_STANDARD} - CXX_STANDARD_REQUIRED ON - # For std:: support of __int128_t. Can be removed once using cuda::std - CXX_EXTENSIONS ON - CXX_VISIBILITY_PRESET hidden - CUDA_STANDARD ${CUDF_CUDA_STANDARD} - CUDA_STANDARD_REQUIRED ON - CUDA_VISIBILITY_PRESET hidden - ) +function(embed_jit_sources_var) endfunction() -add_lto_ir_library(cudf_lto_library src/jit/lto/library.cu) -embed_fatbins_command(cudf_lto_library) - -function(embed_jit_sources_var) +function(embed_jit_options_var) endfunction() -function(embed_jit_options_var) +function(embed_jit_blob) endfunction() -embed_jit_sources_var( +embed_jit_sources( identifier_name - EXPLICIT SOURCE_DIRECTORY cudf/include/ FILES @@ -444,8 +401,8 @@ embed_jit_sources_var( cudf/jit/filter_params.cuh ) -embed_jit_sources_var( - identifier_name EXPLICIT SOURCE_DIRECTORY src FILES src/jit/accessors.cuh src/jit/span.cuh +embed_jit_sources( + identifier_name SOURCE_DIRECTORY src FILES src/jit/accessors.cuh src/jit/span.cuh ) get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) @@ -454,9 +411,9 @@ get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLU # just use the install include dir? provided cudacxx will also be exported? can we get the installed # include dir path? after installing, should be configure a file that points to the installed # cudacxx include dir? -embed_jit_sources_var(identifier_name RECURSIVE SOURCE_DIRECTORY ${LIBCUDACXX_RAW_INCLUDE_DIRS}) +embed_jit_sources_directory(identifier_name SOURCE_DIRECTORY ${LIBCUDACXX_RAW_INCLUDE_DIRS}) -embed_jit_options_var(identifier_name OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) +embed_jit_options(identifier_name OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- @@ -967,7 +924,6 @@ add_library( src/utilities/type_dispatcher.cpp ) -add_dependencies(cudf cudf_lto_library_fatbin_bytes) # Anything that includes jitify needs to be compiled with _FILE_OFFSET_BITS=64 due to a limitation # in how conda builds glibc diff --git a/cpp/embed.py b/cpp/embed.py index 57be24125e3b..f364e6d61541 100644 --- a/cpp/embed.py +++ b/cpp/embed.py @@ -1,44 +1,298 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import argparse +import binascii +import json +import os +from typing import * + +"""headers C code + +typedef struct ${id}_t { + unsigned char const * const * include_names; + unsigned char const * const * headers; + unsigned long long const * header_sizes; + unsigned long long num_includes; +} ${id}_t; + +static ${id}_t const ${id} = { ... }; + +""" + +"""options C code + +typedef struct ${id}_t { + unsigned char const * const * options; + unsigned long long const * option_sizes; + unsigned long long num_options; +} ${id}_t; + +static ${id}_t const ${id} = { ... }; + +""" + +"""blobs C code + +typedef struct ${id}_t { + unsigned char const * const * blobs; + unsigned long long const * blob_sizes; + unsigned long long num_blobs; +} ${id}_t; + +static ${id}_t const ${id} = { ... }; + +""" + + +"""sources json schema + "sources": [ + { + "files": [{"source_directory": "string", "files": ["string", ... ]}], + "directories": ["string", ... ], + } + ] +""" + + +"""options json schema + "options": [ "string", ... ] +""" + +"""blobs json schema + "files": [ "string", ... ] +""" + + +"""entry schema +"id": { + "type": "sources" | "options" | "blobs", + "sources|options|blobs": list | dict +} +""" + +STR_CHAR_TYPE = "unsigned char" +BYTE_CHAR_TYPE = "unsigned char" +SIZE_TYPE = "unsigned long long" + + +class CArrayDecl(NamedTuple): + id: str + decl: str + size_id: str + size_decl: str # create header file to embed header map -import sys -import hashlib # [ ] include dir resolution # [ ] get headers and options and outputs as JSON-encoded HEX strings? -# [ ] usage: python3 rtc_embed.py --embed --header --type=explicit|recursive header_map_id:reference_dir:[list of header files] --option option_id:[compiler options] --output output_file +def generate_sources(entry: Dict): + id = entry["id"] + sources = entry["sources"] + decls: List[CArrayDecl] = [] -""" headers + for i, source_entry in enumerate(sources): + files = source_entry.get("files", None) + directories = source_entry.get("directories", None) -struct { -unsigned char const * const * include_names; -unsigned char const * const * headers; -unsigned long long const * header_sizes; -unsigned char * headers_sha256; -unsigned long long num_includes; -unsigned char sha256[32]; -} ${variable_name} = { ... }; + if files is not None: + base = files["base"] + files = files["files"] -""" + for j, file_path in enumerate(files): + pass + -"""options + if directories is not None: + for j, directory in enumerate(directories): + pass + -struct { -unsigned char const * const * options; -unsigned char * options_sha256; -unsigned long long num_options; -unsigned char sha256[32]; -} ${variable_name} = { ... }; -""" +def generate_options(entry: Dict): + id = entry["id"] + options = entry["options"] + + arrays: List[CArrayDecl] = [] + + for i, option in enumerate(options): + option_id = f"{id}_option_{i}" + option_size_id = f"{option_id}_size" + option_bytes = option.encode("ascii") + option_bytes_null_terminated = option_bytes + b"\0" + option_bytes_size = len( + option_bytes + ) # exclude null terminator from length + option_bytes_size_null_terminated = len(option_bytes_null_terminated) + option_byte_array = ", ".join( + [str(b) for b in option_bytes_null_terminated] + ) + option_decl = f"static {BYTE_CHAR_TYPE} const {option_id}[{option_bytes_size_null_terminated}] = {{ {option_byte_array} }};" + option_size_decl = f"static {SIZE_TYPE} const {option_size_id} = {option_bytes_size}ULL;" + arrays.append( + CArrayDecl( + id=option_id, + decl=option_decl, + size_id=option_size_id, + size_decl=option_size_decl, + ) + ) + + code: str = "" + + for array in arrays: + code += f"{array.decl}\n{array.size_decl}\n" + + num_options = len(arrays) + option_ids = ", ".join([d.id for d in arrays]) + option_size_ids = ", ".join([d.size_id for d in arrays]) + + code += f""" + static {BYTE_CHAR_TYPE} const * const {id}_options[{num_options}] = {{ {option_ids} }}; + static {SIZE_TYPE} const {id}_option_sizes[{num_options}] = {{ {option_size_ids} }}; + + typedef struct {id}_t {{ + {BYTE_CHAR_TYPE} const * const * options; + {SIZE_TYPE} const * option_sizes; + {SIZE_TYPE} num_options; + }} {id}_t; + + static {id}_t const {id} = {{ + {id}_options, + {id}_option_sizes, + {num_options}ULL + }}; + """ + + return code + + +def generate_blobs(entries: Any): + id = entries["id"] + files = entries["files"] + + arrays: List[CArrayDecl] = [] + + for i, file_path in enumerate(files): + blob_id = f"{id}_blob_{i}" + blob_size_id = f"{blob_id}_size" + with open(file_path, "rb") as f: + blob_bytes = f.read() + blob_bytes_size = len(blob_bytes) + blob_byte_array = ", ".join([str(b) for b in blob_bytes]) + blob_decl = f"static {BYTE_CHAR_TYPE} const {blob_id}[{blob_bytes_size}] = {{ {blob_byte_array} }};" + blob_size_decl = ( + f"static {SIZE_TYPE} const {blob_size_id} = {blob_bytes_size}ULL;" + ) + arrays.append( + CArrayDecl( + id=blob_id, + decl=blob_decl, + size_id=blob_size_id, + size_decl=blob_size_decl, + ) + ) + + code: str = "" + + for array in arrays: + code += f"{array.decl}\n{array.size_decl}\n" + + num_blobs = len(arrays) + blob_ids = ", ".join([d.id for d in arrays]) + blob_size_ids = ", ".join([d.size_id for d in arrays]) + + code += f""" + static {BYTE_CHAR_TYPE} const * const {id}_blobs[{num_blobs}] = {{ {blob_ids} }}; + static {SIZE_TYPE} const {id}_blob_sizes[{num_blobs}] = {{ {blob_size_ids} }}; + + typedef struct {id}_t {{ + {BYTE_CHAR_TYPE} const * const * blobs; + {SIZE_TYPE} const * blob_sizes; + {SIZE_TYPE} num_blobs; + }} {id}_t; + + static {id}_t const {id} = {{ + {id}_blobs, + {id}_blob_sizes, + {num_blobs}ULL + }}; + """ + + return code + + +def generate_embed_source(entries: Any): + merged = {} + + # Merge the entries for each id as they are specified in any order + + for id, entry in entries.items(): + entry_type = entry["type"] + + if entry_type == "sources": + sources: List = entry["sources"] + + if id not in merged: + merged[id] = {"type": "sources", "sources": sources} + else: + assert merged[id]["type"] == "sources" + merged[id]["sources"].extend(sources) + + elif entry_type == "options": + options: List = entry["options"] + + if id not in merged: + merged[id] = {"type": "options", "options": options} + else: + assert merged[id]["type"] == "options" + merged[id]["options"].extend(options) + + elif entry_type == "blobs": + files: List = entry["files"] + + if id not in merged: + merged[id] = {"type": "blobs", "files": files} + else: + assert merged[id]["type"] == "blobs" + merged[id]["files"].extend(files) + else: + raise ValueError(f"Unknown type: {entry_type}") + + for id, entry in merged.items(): + if entry["type"] == "sources": + pass + elif entry["type"] == "options": + pass + elif entry["type"] == "blobs": + pass + else: + raise ValueError(f"Unknown type: {entry['type']}") + + +# Usage: embed.py --hex "" --out +def main(): + # parse HEX-encoded string from CLI args + parser = argparse.ArgumentParser( + description="Embed headers, options, or binary blobs into C++ source code." + ) + parser.add_argument( + "--hex", + type=str, + required=True, + help="HEX-encoded JSON description of what to embed", + ) + parser.add_argument( + "--out", type=str, required=True, help="Output C++ source file" + ) + args = parser.parse_args() -"""binary blob + # Decode HEX-encoded JSON description + json_hex = binascii.unhexlify(args.hex) + description = json.loads(json_hex) -struct { -unsigned char const * data; -unsigned long long size; -unsigned char sha256[32]; -} ${variable_name} = { ... }; -""" \ No newline at end of file +# HEX-encoded JSON description of what to embed +def load_description(json_file): + pass diff --git a/cpp/include/cudf/jit/filter_params.cuh b/cpp/include/cudf/jit/filter_params.cuh index d9b5c2dd1a1e..cb2254c4ca7c 100644 --- a/cpp/include/cudf/jit/filter_params.cuh +++ b/cpp/include/cudf/jit/filter_params.cuh @@ -10,12 +10,12 @@ namespace CUDF_LTO_EXPORT cudf { namespace lto { +/// @brief Type-erased parameters for LTO-JIT-compiled filter operations. struct filter_params { - void const* inputs = nullptr; - void* user_data = nullptr; - void const* outputs = nullptr; - void const* span_outputs = nullptr; - size_type row_index = 0; + void const* inputs = nullptr; ///< Pointer to inputs data. + void* user_data = nullptr; ///< Pointer to user data / context. + void const* outputs = nullptr; ///< Pointer to outputs data. + size_type row_index = 0; ///< Current row index. }; } // namespace lto diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index 71c9b6e8d6e0..64f6e4b707c9 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -5,6 +5,7 @@ #pragma once #define CUDF_LTO_EXPORT +#define CUDF_LTO_ALIAS __attribute__((may_alias)) namespace CUDF_LTO_EXPORT cudf { @@ -49,13 +50,13 @@ enum class type_id : int32_t {}; enum scale_type : int32_t {}; -struct __attribute__((may_alias)) data_type { +struct CUDF_LTO_ALIAS data_type { private: type_id __id = {}; int32_t __fixed_point_scale = 0; }; -struct __attribute__((may_alias)) string_view { +struct CUDF_LTO_ALIAS string_view { private: char const* __data = nullptr; size_type __bytes = 0; @@ -121,90 +122,90 @@ struct __attribute__((may_alias)) string_view { static inline size_type const npos{-1}; }; -struct __attribute__((may_alias)) decimal32 { +struct CUDF_LTO_ALIAS decimal32 { private: int32_t __value = 0; scale_type __scale = scale_type{}; }; -struct __attribute__((may_alias)) decimal64 { +struct CUDF_LTO_ALIAS decimal64 { private: int64_t __value = 0; scale_type __scale = scale_type{}; }; -struct __attribute__((may_alias)) decimal128 { +struct CUDF_LTO_ALIAS decimal128 { private: __int128_t __value = 0; scale_type __scale = scale_type{}; }; -struct __attribute__((may_alias)) timestamp_D { +struct CUDF_LTO_ALIAS timestamp_D { private: int32_t __rep = 0; }; -struct __attribute__((may_alias)) timestamp_h { +struct CUDF_LTO_ALIAS timestamp_h { private: int32_t __rep = 0; }; -struct __attribute__((may_alias)) timestamp_m { +struct CUDF_LTO_ALIAS timestamp_m { private: int32_t __rep = 0; }; -struct __attribute__((may_alias)) timestamp_s { +struct CUDF_LTO_ALIAS timestamp_s { private: int64_t __rep = 0; }; -struct __attribute__((may_alias)) timestamp_ms { +struct CUDF_LTO_ALIAS timestamp_ms { private: int64_t __rep = 0; }; -struct __attribute__((may_alias)) timestamp_us { +struct CUDF_LTO_ALIAS timestamp_us { private: int64_t __rep = 0; }; -struct __attribute__((may_alias)) timestamp_ns { +struct CUDF_LTO_ALIAS timestamp_ns { private: int64_t __rep = 0; }; -struct __attribute__((may_alias)) duration_D { +struct CUDF_LTO_ALIAS duration_D { private: int32_t __rep = 0; }; -struct __attribute__((may_alias)) duration_h { +struct CUDF_LTO_ALIAS duration_h { private: int32_t __rep = 0; }; -struct __attribute__((may_alias)) duration_m { +struct CUDF_LTO_ALIAS duration_m { private: int32_t __rep = 0; }; -struct __attribute__((may_alias)) duration_s { +struct CUDF_LTO_ALIAS duration_s { private: int64_t __rep = 0; }; -struct __attribute__((may_alias)) duration_ms { +struct CUDF_LTO_ALIAS duration_ms { private: int64_t __rep = 0; }; -struct __attribute__((may_alias)) duration_us { +struct CUDF_LTO_ALIAS duration_us { private: int64_t __rep = 0; }; -struct __attribute__((may_alias)) duration_ns { +struct CUDF_LTO_ALIAS duration_ns { private: int64_t __rep = 0; }; @@ -215,7 +216,7 @@ inline constexpr inplace_t inplace{}; // [ ] assumes T is trivially copyable template -struct __attribute__((may_alias)) optional { +struct CUDF_LTO_ALIAS optional { private: T __val; bool __engaged; @@ -269,7 +270,7 @@ struct __attribute__((may_alias)) optional { template optional(T) -> optional; -struct alignas(16) __attribute__((may_alias)) column_device_view_core { +struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { private: data_type __type = {}; size_type __size = 0; @@ -373,7 +374,7 @@ CUDF_LTO_DECL(duration_ns) #undef CUDF_LTO_DECL -struct alignas(16) __attribute__((may_alias)) mutable_column_device_view_core { +struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { private: data_type __type = {}; size_type __size = 0; diff --git a/cpp/include/cudf/jit/transform_params.cuh b/cpp/include/cudf/jit/transform_params.cuh index da1845446af3..ca6419ca7dfb 100644 --- a/cpp/include/cudf/jit/transform_params.cuh +++ b/cpp/include/cudf/jit/transform_params.cuh @@ -11,12 +11,12 @@ namespace CUDF_LTO_EXPORT cudf { namespace lto { +/// @brief Type-erased parameters for LTO-JIT-compiled transform operations. struct transform_params { - void const* inputs = nullptr; - void* user_data = nullptr; - void const* outputs = nullptr; - void const* span_outputs = nullptr; - size_type row_index = 0; + void const* inputs = nullptr; ///< Pointer to inputs data. + void* user_data = nullptr; ///< Pointer to user data / context. + void const* outputs = nullptr; ///< Pointer to outputs data. + size_type row_index = 0; ///< Current row index. }; } // namespace lto diff --git a/cpp/include/cudf/utilities/defer.hpp b/cpp/include/cudf/utilities/defer.hpp index 2f924eb25bb3..079fbe64dab9 100644 --- a/cpp/include/cudf/utilities/defer.hpp +++ b/cpp/include/cudf/utilities/defer.hpp @@ -23,6 +23,8 @@ struct defer { T func_; public: + /// @brief Construct a `defer` object that will invoke the provided callable upon destruction. + /// @param args Arguments to forward to the callable's constructor. template defer(Args&&... args) : func_{static_cast(args)...} { @@ -35,6 +37,6 @@ struct defer { }; template -defer(T) -> defer; +defer(T) -> defer; ///< Class template argument deduction guide } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/rtc/cudf.hpp index e621ee5bfe08..894107f47c23 100644 --- a/cpp/src/jit/rtc/cudf.hpp +++ b/cpp/src/jit/rtc/cudf.hpp @@ -29,7 +29,7 @@ namespace rtc { // [ ] disable caching // [ ] cache statistics: hits, misses, etc. // [ ] on startup, log cache path, loading information, etc. - +// [ ] not compatible across major driver versions // [ ] if a user provides a key, use: USER_KEY+${key}, otherwise use sha256 of contents // [ ] use resource type in key to avoid collisions From e18983dd8080d7952f45c7ed52290ddb29e428ca Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 11 Dec 2025 20:53:30 +0000 Subject: [PATCH 008/254] checkpoint --- cpp/CMakeLists.txt | 7 +- cpp/cxx_embed.py | 377 +++++++++++++++++++++++++++++++++++++++++++++ cpp/embed.py | 298 ----------------------------------- 3 files changed, 378 insertions(+), 304 deletions(-) create mode 100644 cpp/cxx_embed.py delete mode 100644 cpp/embed.py diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8103e0068ad8..8352f7036ead 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -336,7 +336,6 @@ endif() # We can't precompile the LTO-IR as we need to generate fatbins for the specific target machine # LTO-IR is not compatible across - if(CMAKE_BUILD_TYPE STREQUAL "Debug") set(CUDF_JIT_CUDA_FLAGS "-G") else() @@ -361,7 +360,6 @@ set(CUDF_JIT_COMPILE_FLAGS "-O3;-std=c++${CUDF_CUDA_STANDARD};-Xcompiler=-fPIC;-Xcompiler=-DPIC;-fvisibility=hidden;-Xfatbin=--compress-all;--compress-mode=size;--device-int128" ) - function(embed_jit_sources_var) endfunction() @@ -401,9 +399,7 @@ embed_jit_sources( cudf/jit/filter_params.cuh ) -embed_jit_sources( - identifier_name SOURCE_DIRECTORY src FILES src/jit/accessors.cuh src/jit/span.cuh -) +embed_jit_sources(identifier_name SOURCE_DIRECTORY src FILES src/jit/accessors.cuh src/jit/span.cuh) get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) @@ -929,7 +925,6 @@ add_library( src/utilities/type_dispatcher.cpp ) - # Anything that includes jitify needs to be compiled with _FILE_OFFSET_BITS=64 due to a limitation # in how conda builds glibc set_source_files_properties( diff --git a/cpp/cxx_embed.py b/cpp/cxx_embed.py new file mode 100644 index 000000000000..bf7785555884 --- /dev/null +++ b/cpp/cxx_embed.py @@ -0,0 +1,377 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import argparse +import binascii +import json +import os +from typing import Any, NamedTuple, Self + +BYTE_TYPE = "unsigned char" +SIZE_TYPE = "unsigned long long" +STORAGE_SPEC = "static" + + +"""headers CXX code + +typedef struct ${id}_t { + unsigned char const * const * include_names; + unsigned char const * const * headers; + unsigned long long const * header_sizes; + unsigned long long num_includes; +} ${id}_t; + +static ${id}_t const ${id} = { ... }; + +""" + +"""options CXX code + +typedef struct ${id}_t { + unsigned char const * const * options; + unsigned long long const * option_sizes; + unsigned long long num_options; +} ${id}_t; + +static ${id}_t const ${id} = { ... }; + +""" + + +### json schema + +"""sources + "sources": + { + "files": [{"source_directory": "string", "files": ["string", ... ]}], + "directories": ["string", ... ] + } +""" + + +"""options + "options": [ "string", ... ] +""" + + +"""entries +{ + "id": { + "type": "sources" | "options", + "sources|options": list | dict + } +} +""" + + +class CXXVarDecl(NamedTuple): + id: str + decl: str + + def code(self: Self) -> str: + f""" + {self.decl} + """ + + @staticmethod + def of_bytes(id: str, data: bytes) -> Self: + byte_array = ", ".join([str(b) for b in data]) + decl = f"{STORAGE_SPEC} {BYTE_TYPE} const {id}[{len(data)}] = {{ {byte_array} }};" + return CXXVarDecl(id=id, decl=decl) + + @staticmethod + def of_size(id: str, size: int) -> Self: + decl = f"{STORAGE_SPEC} {SIZE_TYPE} const {id} = {size}ULL;" + return CXXVarDecl(id=id, decl=decl) + + +class CXXSizeArrayDecl(NamedTuple): + id: str + sizes: list[int] + + def code(self: Self) -> CXXVarDecl: + size_array = ", ".join([f"{size}ULL" for size in self.sizes]) + decl = f"{STORAGE_SPEC} {SIZE_TYPE} const {self.id}[{len(self.sizes)}] = {{ {size_array} }};" + return CXXVarDecl(id=self.id, decl=decl) + + @staticmethod + def of_sizes(id: str, sizes: list[int]) -> Self: + return CXXSizeArrayDecl(id=id, sizes=sizes) + + +class CXXByteArrayDecl(NamedTuple): + id: str + data: CXXVarDecl + size: CXXVarDecl + + def code(self: Self) -> CXXVarDecl: + return CXXVarDecl( + id=self.id, + decl=f""" + {self.data.code()} + {self.size.code()} + + typedef struct {self.id}_t {{ + {BYTE_TYPE} const * data; + {SIZE_TYPE} size; + }} {self.id}_t; + + {STORAGE_SPEC} {self.id}_t const {self.id} = {{ + {self.data.id}, + {self.size.id} + }}; + """, + ) + + @staticmethod + def of_bytes(id: str, data: bytes, null_terminate: bool) -> Self: + # exclude null terminator from length + size_decl = CXXVarDecl.of_size(id=f"{id}_size", size=len(data)) + + if null_terminate: + data += b"\0" + + data_decl = CXXVarDecl.of_bytes(id=f"{id}_data", data=data) + + return CXXByteArrayDecl(id=id, data=data_decl, size=size_decl) + + +class CXXArrayOfByteArraysDecl(NamedTuple): + id: str + elements: list[CXXByteArrayDecl] + size: CXXVarDecl + + def code(self: Self) -> CXXVarDecl: + elements_decl = ";\n".join(e.code() for e in self.elements) + count = len(self.elements) + data_ids = ", ".join([d.data.id for d in self.elements]) + size_ids = ", ".join([d.size.id for d in self.elements]) + + return CXXVarDecl( + id=self.id, + decl=f""" + {elements_decl}; + + {STORAGE_SPEC} {BYTE_TYPE} const * const {self.id}_elements[{count}] = {{ {data_ids} }}; + {STORAGE_SPEC} {SIZE_TYPE} const {self.id}_element_sizes[{count}] = {{ {size_ids} }}; + + typedef struct {self.id}_t {{ + {BYTE_TYPE} const * const * elements; + {SIZE_TYPE} const * element_sizes; + {SIZE_TYPE} size; + }} {self.id}_t; + + {STORAGE_SPEC} {self.id}_t const {self.id} = {{ + {self.id}_elements, + {self.id}_element_sizes, + {count}ULL + }}; + """, + ) + + @staticmethod + def of_bytes_array( + id: str, data_list: list[bytes], null_terminate: bool + ) -> Self: + size = len(data_list) + size_decl = CXXVarDecl.of_size(id=f"{id}_size", size=size) + array_decls: list[CXXByteArrayDecl] = [ + CXXByteArrayDecl.of_bytes( + id=f"{id}_element_{i}", + data=data, + null_terminate=null_terminate, + ) + for i, data in enumerate(data_list) + ] + + return CXXArrayOfByteArraysDecl( + id=id, elements=array_decls, size=size_decl + ) + + +def generate_blobs(entries: Any): + id = entries["id"] + files = entries["files"] + data_list: list[bytes] = [] + + for i, file_path in enumerate(files): + with open(file_path, "rb") as f: + blob_bytes = f.read() + data_list.append(blob_bytes) + + return CXXArrayOfByteArraysDecl.of_bytes_array( + id=f"{id}", data_list=data_list, null_terminate=False + ).code() + + +def generate_options(entry: dict): + id = entry["id"] + options = entry["options"] + arrays: list[bytes] = [opt.encode("utf-8") for opt in options] + + return CXXArrayOfByteArraysDecl.of_bytes_array( + id=f"{id}", data_list=arrays, null_terminate=True + ).code() + + +class Include(NamedTuple): + path: str + data: bytes + + +class IncludeMapDecl(NamedTuple): + name: str + include_names: CXXArrayOfByteArraysDecl + sources: CXXArrayOfByteArraysDecl + header_sizes: CXXSizeArrayDecl + + def code(self: Self) -> str: + return f""" + {self.include_names.code()} + {self.sources.code()} + {self.header_sizes.code()} + + typedef struct {self.name}_t {{ + {BYTE_TYPE} const * const * include_names; + {BYTE_TYPE} const * const * headers; + {SIZE_TYPE} const * header_sizes; + {SIZE_TYPE} num_includes; + }} {self.name}_t; + + {STORAGE_SPEC} {self.name}_t const {self.name} = {{ + {self.include_names.id}.elements, + {self.sources.id}.elements, + {self.header_sizes.id}, + {self.include_names.size.id} + }}; + """ + + +def generate_sources(entry: dict): + id = entry["id"] + sources = entry["sources"] + includes: list[Include] = [] + + for source_entry in sources: + files = source_entry.get("files", None) + directories = source_entry.get("directories", None) + + if files is not None: + source_directory = source_entry.get("source_directory") + files = files["files"] + + for include in files: + path = os.path.join(source_directory, include) + with open(path, "rb") as f: + header_bytes = f.read() + + includes.append(Include(path=include, data=header_bytes)) + + if directories is not None: + for directory in directories: + for root, _, files in os.walk(directory): + for file in files: + path = os.path.join(root, file) + with open(path, "rb") as f: + header_bytes = f.read() + + # make include path relative to directory + include_path = os.path.relpath(path, directory) + + includes.append( + Include(path=include_path, data=header_bytes) + ) + + source_data_decl: CXXArrayOfByteArraysDecl = ( + CXXArrayOfByteArraysDecl.of_bytes_array( + id=f"{id}_headers", + data_list=[include.data for include in includes], + null_terminate=True, + ) + ) + + header_size_decls: CXXSizeArrayDecl = CXXSizeArrayDecl.of_sizes( + id=f"{id}_header_sizes", + sizes=[len(include.data) for include in includes], + ) + + include_name_decls: CXXArrayOfByteArraysDecl = ( + CXXArrayOfByteArraysDecl.of_bytes_array( + id=f"{id}_include_names", + data_list=[include.path.encode("utf-8") for include in includes], + null_terminate=True, + ) + ) + + return IncludeMapDecl( + name=id, + include_names=include_name_decls, + sources=source_data_decl, + header_sizes=header_size_decls, + ).code() + + +def generate_embed_source(entries: Any): + merged = {} + + for id, entry in entries.items(): + entry_type = entry["type"] + + if entry_type == "sources": + sources: list = entry["sources"] + + if id not in merged: + merged[id] = {"type": "sources", "sources": sources} + else: + assert merged[id]["type"] == "sources" + merged[id]["sources"].extend(sources) + + elif entry_type == "options": + options: list = entry["options"] + + if id not in merged: + merged[id] = {"type": "options", "options": options} + else: + assert merged[id]["type"] == "options" + merged[id]["options"].extend(options) + + else: + raise ValueError(f"Unknown type: {entry_type}") + + code: str = "" + + for id, entry in merged.items(): + if entry["type"] == "sources": + code += generate_sources(entry) + elif entry["type"] == "options": + code += generate_options(entry) + else: + raise ValueError(f"Unknown type: {entry['type']}") + + return code + + +# Usage: embed.py --hex "" --out +def main(): + # parse HEX-encoded string from CLI args + parser = argparse.ArgumentParser( + description="Embed headers, options, or binary blobs into C++ source code." + ) + parser.add_argument( + "--hex", + type=str, + required=True, + help="HEX-encoded JSON description of what to embed", + ) + parser.add_argument( + "--out", type=str, required=True, help="Output C++ source file" + ) + args = parser.parse_args() + + # Decode HEX-encoded JSON description + json_hex = binascii.unhexlify(args.hex) + description = json.loads(json_hex) + + +# HEX-encoded JSON description of what to embed +def load_description(json_file): + pass diff --git a/cpp/embed.py b/cpp/embed.py deleted file mode 100644 index f364e6d61541..000000000000 --- a/cpp/embed.py +++ /dev/null @@ -1,298 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -import argparse -import binascii -import json -import os -from typing import * - -"""headers C code - -typedef struct ${id}_t { - unsigned char const * const * include_names; - unsigned char const * const * headers; - unsigned long long const * header_sizes; - unsigned long long num_includes; -} ${id}_t; - -static ${id}_t const ${id} = { ... }; - -""" - -"""options C code - -typedef struct ${id}_t { - unsigned char const * const * options; - unsigned long long const * option_sizes; - unsigned long long num_options; -} ${id}_t; - -static ${id}_t const ${id} = { ... }; - -""" - -"""blobs C code - -typedef struct ${id}_t { - unsigned char const * const * blobs; - unsigned long long const * blob_sizes; - unsigned long long num_blobs; -} ${id}_t; - -static ${id}_t const ${id} = { ... }; - -""" - - -"""sources json schema - "sources": [ - { - "files": [{"source_directory": "string", "files": ["string", ... ]}], - "directories": ["string", ... ], - } - ] -""" - - -"""options json schema - "options": [ "string", ... ] -""" - -"""blobs json schema - "files": [ "string", ... ] -""" - - -"""entry schema -"id": { - "type": "sources" | "options" | "blobs", - "sources|options|blobs": list | dict -} -""" - -STR_CHAR_TYPE = "unsigned char" -BYTE_CHAR_TYPE = "unsigned char" -SIZE_TYPE = "unsigned long long" - - -class CArrayDecl(NamedTuple): - id: str - decl: str - size_id: str - size_decl: str - - -# create header file to embed header map - -# [ ] include dir resolution -# [ ] get headers and options and outputs as JSON-encoded HEX strings? -def generate_sources(entry: Dict): - id = entry["id"] - sources = entry["sources"] - - decls: List[CArrayDecl] = [] - - for i, source_entry in enumerate(sources): - files = source_entry.get("files", None) - directories = source_entry.get("directories", None) - - if files is not None: - base = files["base"] - files = files["files"] - - for j, file_path in enumerate(files): - pass - - - if directories is not None: - for j, directory in enumerate(directories): - pass - - - -def generate_options(entry: Dict): - id = entry["id"] - options = entry["options"] - - arrays: List[CArrayDecl] = [] - - for i, option in enumerate(options): - option_id = f"{id}_option_{i}" - option_size_id = f"{option_id}_size" - option_bytes = option.encode("ascii") - option_bytes_null_terminated = option_bytes + b"\0" - option_bytes_size = len( - option_bytes - ) # exclude null terminator from length - option_bytes_size_null_terminated = len(option_bytes_null_terminated) - option_byte_array = ", ".join( - [str(b) for b in option_bytes_null_terminated] - ) - option_decl = f"static {BYTE_CHAR_TYPE} const {option_id}[{option_bytes_size_null_terminated}] = {{ {option_byte_array} }};" - option_size_decl = f"static {SIZE_TYPE} const {option_size_id} = {option_bytes_size}ULL;" - arrays.append( - CArrayDecl( - id=option_id, - decl=option_decl, - size_id=option_size_id, - size_decl=option_size_decl, - ) - ) - - code: str = "" - - for array in arrays: - code += f"{array.decl}\n{array.size_decl}\n" - - num_options = len(arrays) - option_ids = ", ".join([d.id for d in arrays]) - option_size_ids = ", ".join([d.size_id for d in arrays]) - - code += f""" - static {BYTE_CHAR_TYPE} const * const {id}_options[{num_options}] = {{ {option_ids} }}; - static {SIZE_TYPE} const {id}_option_sizes[{num_options}] = {{ {option_size_ids} }}; - - typedef struct {id}_t {{ - {BYTE_CHAR_TYPE} const * const * options; - {SIZE_TYPE} const * option_sizes; - {SIZE_TYPE} num_options; - }} {id}_t; - - static {id}_t const {id} = {{ - {id}_options, - {id}_option_sizes, - {num_options}ULL - }}; - """ - - return code - - -def generate_blobs(entries: Any): - id = entries["id"] - files = entries["files"] - - arrays: List[CArrayDecl] = [] - - for i, file_path in enumerate(files): - blob_id = f"{id}_blob_{i}" - blob_size_id = f"{blob_id}_size" - with open(file_path, "rb") as f: - blob_bytes = f.read() - blob_bytes_size = len(blob_bytes) - blob_byte_array = ", ".join([str(b) for b in blob_bytes]) - blob_decl = f"static {BYTE_CHAR_TYPE} const {blob_id}[{blob_bytes_size}] = {{ {blob_byte_array} }};" - blob_size_decl = ( - f"static {SIZE_TYPE} const {blob_size_id} = {blob_bytes_size}ULL;" - ) - arrays.append( - CArrayDecl( - id=blob_id, - decl=blob_decl, - size_id=blob_size_id, - size_decl=blob_size_decl, - ) - ) - - code: str = "" - - for array in arrays: - code += f"{array.decl}\n{array.size_decl}\n" - - num_blobs = len(arrays) - blob_ids = ", ".join([d.id for d in arrays]) - blob_size_ids = ", ".join([d.size_id for d in arrays]) - - code += f""" - static {BYTE_CHAR_TYPE} const * const {id}_blobs[{num_blobs}] = {{ {blob_ids} }}; - static {SIZE_TYPE} const {id}_blob_sizes[{num_blobs}] = {{ {blob_size_ids} }}; - - typedef struct {id}_t {{ - {BYTE_CHAR_TYPE} const * const * blobs; - {SIZE_TYPE} const * blob_sizes; - {SIZE_TYPE} num_blobs; - }} {id}_t; - - static {id}_t const {id} = {{ - {id}_blobs, - {id}_blob_sizes, - {num_blobs}ULL - }}; - """ - - return code - - -def generate_embed_source(entries: Any): - merged = {} - - # Merge the entries for each id as they are specified in any order - - for id, entry in entries.items(): - entry_type = entry["type"] - - if entry_type == "sources": - sources: List = entry["sources"] - - if id not in merged: - merged[id] = {"type": "sources", "sources": sources} - else: - assert merged[id]["type"] == "sources" - merged[id]["sources"].extend(sources) - - elif entry_type == "options": - options: List = entry["options"] - - if id not in merged: - merged[id] = {"type": "options", "options": options} - else: - assert merged[id]["type"] == "options" - merged[id]["options"].extend(options) - - elif entry_type == "blobs": - files: List = entry["files"] - - if id not in merged: - merged[id] = {"type": "blobs", "files": files} - else: - assert merged[id]["type"] == "blobs" - merged[id]["files"].extend(files) - else: - raise ValueError(f"Unknown type: {entry_type}") - - for id, entry in merged.items(): - if entry["type"] == "sources": - pass - elif entry["type"] == "options": - pass - elif entry["type"] == "blobs": - pass - else: - raise ValueError(f"Unknown type: {entry['type']}") - - -# Usage: embed.py --hex "" --out -def main(): - # parse HEX-encoded string from CLI args - parser = argparse.ArgumentParser( - description="Embed headers, options, or binary blobs into C++ source code." - ) - parser.add_argument( - "--hex", - type=str, - required=True, - help="HEX-encoded JSON description of what to embed", - ) - parser.add_argument( - "--out", type=str, required=True, help="Output C++ source file" - ) - args = parser.parse_args() - - # Decode HEX-encoded JSON description - json_hex = binascii.unhexlify(args.hex) - description = json.loads(json_hex) - - -# HEX-encoded JSON description of what to embed -def load_description(json_file): - pass From 099e6dea8bebca10f93bd1fcf24e77bebcecf27d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 6 Jan 2026 11:51:24 +0000 Subject: [PATCH 009/254] Add JIT embedding support and update copyright years - Introduced JitEmbed.cmake for managing JIT includes and options. - Added jit_embed.py script for generating C++ source code from YAML configurations. - Removed deprecated cxx_embed.py script. - Updated copyright years in various files from 2025 to 2026. - Added null_mask_opt.cuh for optimized null mask operations. - Updated filter_params.cuh, library.cuh, operators.cuh, thunk.cuh, and types.cuh with new copyright years. - Modified transform.hpp and defer.hpp to reflect updated copyright years. - Updated JIT-related source files to maintain consistency in copyright years. --- cpp/CMakeLists.txt | 57 ++- cpp/cmake/Modules/JitEmbed.cmake | 141 +++++++ .../Modules/JitifyPreprocessKernels.cmake | 2 +- cpp/cmake/Modules/jit_embed.py | 350 ++++++++++++++++ cpp/cxx_embed.py | 377 ------------------ cpp/include/cudf/detail/null_mask_opt.cuh | 90 +++++ cpp/include/cudf/jit/filter_params.cuh | 2 +- cpp/include/cudf/jit/lto/library.cuh | 2 +- cpp/include/cudf/jit/lto/operators.cuh | 2 +- cpp/include/cudf/jit/lto/thunk.cuh | 2 +- cpp/include/cudf/jit/lto/types.cuh | 2 +- cpp/include/cudf/jit/transform_params.cuh | 2 +- cpp/include/cudf/transform.hpp | 2 +- cpp/include/cudf/utilities/defer.hpp | 2 +- cpp/src/jit/build.hpp | 2 +- cpp/src/jit/lto/library.cu | 2 +- cpp/src/jit/lto/operators.inl.cuh | 2 +- cpp/src/jit/lto/types.inl.cuh | 2 +- cpp/src/jit/rtc/cache.cpp | 2 +- cpp/src/jit/rtc/cache.hpp | 2 +- cpp/src/jit/rtc/cudf.cpp | 2 +- cpp/src/jit/rtc/cudf.hpp | 2 +- cpp/src/jit/rtc/rtc.cpp | 2 +- cpp/src/jit/rtc/rtc.hpp | 2 +- cpp/src/jit/rtc/sha256.cpp | 2 +- cpp/src/jit/rtc/sha256.hpp | 2 +- .../filter/jit/lto_kernel.inl.cuh | 2 +- cpp/src/transform/jit/lto_kernel.inl.cuh | 2 +- cpp/src/transform/transform.cpp | 2 +- 29 files changed, 631 insertions(+), 432 deletions(-) create mode 100644 cpp/cmake/Modules/JitEmbed.cmake create mode 100644 cpp/cmake/Modules/jit_embed.py delete mode 100644 cpp/cxx_embed.py create mode 100644 cpp/include/cudf/detail/null_mask_opt.cuh diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8352f7036ead..884d965fe92f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2018-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -304,6 +304,9 @@ endif() # preprocess jitify-able kernels include(cmake/Modules/JitifyPreprocessKernels.cmake) +# JIT Embedding helper functions +include(cmake/Modules/JitEmbed.cmake) + # find KvikIO include(cmake/thirdparty/get_kvikio.cmake) @@ -333,9 +336,6 @@ if(NOT BUILD_SHARED_LIBS) endif() endif() -# We can't precompile the LTO-IR as we need to generate fatbins for the specific target machine -# LTO-IR is not compatible across - if(CMAKE_BUILD_TYPE STREQUAL "Debug") set(CUDF_JIT_CUDA_FLAGS "-G") else() @@ -354,28 +354,20 @@ endif() set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) -set(CUDF_JIT_ARCHITECTURES "75;80;86;90;100;120") -set(CUDF_JIT_DEFINITIONS "-DCUDF_RUNTIME_JIT;-D__CUDACC_RTC__") -set(CUDF_JIT_COMPILE_FLAGS - "-O3;-std=c++${CUDF_CUDA_STANDARD};-Xcompiler=-fPIC;-Xcompiler=-DPIC;-fvisibility=hidden;-Xfatbin=--compress-all;--compress-mode=size;--device-int128" -) - -function(embed_jit_sources_var) -endfunction() - -function(embed_jit_options_var) - -endfunction() - -function(embed_jit_blob) +list(APPEND CUDF_JIT_ARCHITECTURES 75 80 86 90 100 120) +list(APPEND CUDF_JIT_DEFINITIONS -DCUDF_RUNTIME_JIT -D__CUDACC_RTC__) +list( + APPEND CUDF_JIT_COMPILE_FLAGS -O3 -std=c++${CUDF_CUDA_STANDARD} -Xcompiler=-fPIC -Xcompiler=-DPIC + -fvisibility=hidden -Xfatbin=--compress-all --compress-mode=size --device-int128 +) -endfunction() +get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) -embed_jit_sources( - identifier_name - SOURCE_DIRECTORY - cudf/include/ +add_jit_includes( + cudf_jit + DIRECTORY + ${CMAKE_CURRENT_SOURCE_DIR}/include FILES cudf/table/table_device_view.cuh cudf/jit/lto/types.cuh @@ -399,17 +391,18 @@ embed_jit_sources( cudf/jit/filter_params.cuh ) -embed_jit_sources(identifier_name SOURCE_DIRECTORY src FILES src/jit/accessors.cuh src/jit/span.cuh) +# TODO: add library.cu LTO type checks need to be compiled as part of the AOT workflow to ensure +# type consistency between AOT and JIT -get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) +add_jit_includes( + cudf_jit DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} FILES src/jit/accessors.cuh src/jit/span.cuh +) + +add_jit_includes(cudf_jit INCLUDE_DIRECTORY ${LIBCUDACXX_RAW_INCLUDE_DIRS}) -# CUDA/STD headers needs to recursively bundle the actually used headers; not everything should we -# just use the install include dir? provided cudacxx will also be exported? can we get the installed -# include dir path? after installing, should be configure a file that points to the installed -# cudacxx include dir? -embed_jit_sources_directory(identifier_name SOURCE_DIRECTORY ${LIBCUDACXX_RAW_INCLUDE_DIRS}) +add_jit_options(cudf_jit OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) -embed_jit_options(identifier_name OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) +generate_jit_source_map(TARGET cudf_jit) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- @@ -924,6 +917,8 @@ add_library( src/utilities/type_checks.cpp src/utilities/type_dispatcher.cpp ) +add_dependencies(cudf cudf_jit) +target_include_directories(cudf PRIVATE ${cudf_jit_INCLUDE_DIR}) # Anything that includes jitify needs to be compiled with _FILE_OFFSET_BITS=64 due to a limitation # in how conda builds glibc diff --git a/cpp/cmake/Modules/JitEmbed.cmake b/cpp/cmake/Modules/JitEmbed.cmake new file mode 100644 index 000000000000..9980729440de --- /dev/null +++ b/cpp/cmake/Modules/JitEmbed.cmake @@ -0,0 +1,141 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +find_package(Python3 REQUIRED COMPONENTS Interpreter) + +function(add_jit_includes) + set(IDENTIFIER ${ARGV0}) + set(OPTIONS "") + set(ONE_VALUE_ARGS DIRECTORY INCLUDE_DIRECTORY) + set(MULTI_VALUE_ARGS FILES) + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT ARG_DIRECTORY AND NOT ARG_INCLUDE_DIRECTORY) + message(FATAL_ERROR "Either DIRECTORY or INCLUDE_DIRECTORY must be specified") + endif() + + if(ARG_DIRECTORY AND ARG_INCLUDE_DIRECTORY) + message(FATAL_ERROR "Only one of DIRECTORY or INCLUDE_DIRECTORY can be specified") + endif() + + if(ARG_INCLUDE_DIRECTORY AND ARG_FILES) + message(FATAL_ERROR "FILES cannot be specified with INCLUDE_DIRECTORY") + endif() + + # recursively gather all include files under the specified directory and get their paths relative + # to the base include directory + if(ARG_INCLUDE_DIRECTORY) + file(GLOB_RECURSE INCLUDE_FILES "${ARG_INCLUDE_DIRECTORY}/*") + set(INCLUDE_FILES_RELATIVE_PATHS "") + foreach(INCLUDE_FILE IN LISTS INCLUDE_FILES) + file(RELATIVE_PATH REL_PATH "${ARG_INCLUDE_DIRECTORY}" "${INCLUDE_FILE}") + list(APPEND INCLUDE_FILES_RELATIVE_PATHS "${REL_PATH}") + endforeach() + set(ARG_DIRECTORY ${ARG_INCLUDE_DIRECTORY}) + set(ARG_FILES ${INCLUDE_FILES_RELATIVE_PATHS}) + endif() + + # check that each source file exists + foreach(SOURCE_FILE IN LISTS ARG_FILES) + if(NOT EXISTS "${ARG_DIRECTORY}/${SOURCE_FILE}") + message(FATAL_ERROR "Source file '${ARG_DIRECTORY}/${SOURCE_FILE}' does not exist") + endif() + endforeach(SOURCE_FILE) + + set(SOURCE_FILES ${${IDENTIFIER}_sources_file_paths}) + set(INCLUDE_NAMES ${${IDENTIFIER}_sources_include_names}) + + foreach(SOURCE_FILE IN LISTS ARG_FILES) + list(APPEND SOURCE_FILES "${ARG_DIRECTORY}/${SOURCE_FILE}") + list(APPEND INCLUDE_NAMES "${SOURCE_FILE}") + endforeach() + + set(${IDENTIFIER}_sources_file_paths + ${SOURCE_FILES} + PARENT_SCOPE + ) + set(${IDENTIFIER}_sources_include_names + ${INCLUDE_NAMES} + PARENT_SCOPE + ) + +endfunction() + +function(add_jit_options) + set(IDENTIFIER ${ARGV0}) + set(OPTIONS "") + set(ONE_VALUE_ARGS "") + set(MULTI_VALUE_ARGS "OPTIONS") + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + set(${IDENTIFIER}_options + "${ARG_OPTIONS}" + PARENT_SCOPE + ) + +endfunction() + +# pass the encoded args to the jit_embed.py script to generate the source and options maps +function(generate_jit_source_map) + set(ONE_VALUE_ARGS "TARGET") + cmake_parse_arguments(ARG "" "${ONE_VALUE_ARGS}" "" ${ARGN}) + + list(LENGTH ${ARG_TARGET}_sources_include_names NUM_SOURCES) + list(LENGTH ${ARG_TARGET}_options NUM_OPTIONS) + + set(TARGET_YAML " - id: \"${ARG_TARGET}_sources\"\n type: \"sources\"\n sources:\n") + + math(EXPR LAST_SOURCE_INDEX "${NUM_SOURCES} - 1") + foreach(i RANGE 0 ${LAST_SOURCE_INDEX}) + list(GET ${ARG_TARGET}_sources_include_names ${i} INCLUDE_NAME) + list(GET ${ARG_TARGET}_sources_file_paths ${i} SOURCE_FILE_PATH) + set(TARGET_YAML "${TARGET_YAML} - include_name: \"${INCLUDE_NAME}\"\n") + set(TARGET_YAML "${TARGET_YAML} file_path: \"${SOURCE_FILE_PATH}\"\n") + endforeach() + + set(TARGET_YAML "${TARGET_YAML}\n\n") + set(TARGET_YAML + "${TARGET_YAML} - id: \"${ARG_TARGET}_options\"\n type: \"options\"\n options:\n" + ) + + math(EXPR LAST_OPTION_INDEX "${NUM_OPTIONS} - 1") + foreach(i RANGE 0 ${LAST_OPTION_INDEX}) + list(GET ${ARG_TARGET}_options ${i} OPTION) + set(TARGET_YAML "${TARGET_YAML} - \"${OPTION}\"\n") + endforeach() + + # write CONFIG to temp file and pass file path to script + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}.yaml" "${TARGET_YAML}") + + set(INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/jit_embed") + set(HEADER "${INCLUDE_DIR}/${ARG_TARGET}.h") + + add_custom_command( + OUTPUT ${HEADER} + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" --output + "${HEADER}" --input-file "${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}.yaml" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" + ${${ARG_TARGET}_sources_file_paths} + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + COMMENT "Generating JIT source map for ${ARG_TARGET} into ${HEADER}" + VERBATIM + ) + + add_custom_target(${ARG_TARGET} ALL DEPENDS "${HEADER}") + + message("Generated into: ${HEADER}") + + set(${ARG_TARGET}_INCLUDE_DIR + ${INCLUDE_DIR} + PARENT_SCOPE + ) + set(${ARG_TARGET}_HEADER + ${HEADER} + PARENT_SCOPE + ) + +endfunction() diff --git a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake index bd4169185b89..8b09bf71243f 100644 --- a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake +++ b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py new file mode 100644 index 000000000000..06ee7222cfb7 --- /dev/null +++ b/cpp/cmake/Modules/jit_embed.py @@ -0,0 +1,350 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import argparse +from typing import Any, NamedTuple, Self + +import yaml + +BYTE_TYPE = "unsigned char" +SIZE_TYPE = "unsigned long long" +STORAGE_SPEC = "static constexpr" +LIST_LINE_WIDTH = 32 +NAMESPACE_PREFIX = "jit_" + + +### json schema + +"""entries +[ + { + "id": string, + "type": "sources", + "sources": [ + { + "include_name": string, + "file_path": string + } + ] + }, + { + "id": string, + "type": "options", + "options": list[string] + } +] +""" + + +def list_string(strings: list[str]) -> str: + lines = [] + for i in range(0, len(strings), LIST_LINE_WIDTH): + line = ", ".join(strings[i : i + LIST_LINE_WIDTH]) + lines.append(line) + return ",\n".join(lines) + + +def hex_string(value: int) -> str: + return f"0x{value:02X}" + + +class CXXVarDecl(NamedTuple): + id: str + expr: str + + def code(self: Self) -> str: + return f"""{self.expr}""" + + @staticmethod + def of_bytes(id: str, data: bytes) -> Self: + byte_array = list_string([hex_string(b) for b in data]) + expr = f"""{STORAGE_SPEC} {BYTE_TYPE} const {id}[{len(data)}] = {{ +{byte_array} +}};""" + return CXXVarDecl(id=id, expr=expr) + + @staticmethod + def of_size(id: str, size: int) -> Self: + expr = f"{STORAGE_SPEC} {SIZE_TYPE} const {id} = {size}ULL;" + return CXXVarDecl(id=id, expr=expr) + + +class CXXSizeArrayDecl(NamedTuple): + id: str + sizes: list[int] + + def decl(self: Self) -> CXXVarDecl: + size_array = list_string([f"{size}ULL" for size in self.sizes]) + expr = f"""{STORAGE_SPEC} {SIZE_TYPE} const {self.id}[{len(self.sizes)}] = {{ +{size_array} +}};""" + return CXXVarDecl(id=self.id, expr=expr) + + @staticmethod + def of_sizes(id: str, sizes: list[int]) -> Self: + return CXXSizeArrayDecl(id=id, sizes=sizes) + + +class CXXByteArrayDecl(NamedTuple): + id: str + data: CXXVarDecl + size: CXXVarDecl + + def decl(self: Self) -> CXXVarDecl: + return CXXVarDecl( + id=self.id, + expr=f""" +{self.data.code()} + +{self.size.code()} + + +{STORAGE_SPEC} {NAMESPACE_PREFIX}byte_array_t const {self.id} = {{ + {self.data.id}, + {self.size.id} +}}; +""", + ) + + @staticmethod + def of_bytes(id: str, data: bytes, null_terminate: bool) -> Self: + # exclude null terminator from length + size_decl = CXXVarDecl.of_size(id=f"{id}_size", size=len(data)) + + if null_terminate: + data += b"\0" + + data_decl = CXXVarDecl.of_bytes(id=f"{id}_data", data=data) + + return CXXByteArrayDecl(id=id, data=data_decl, size=size_decl) + + +class CXXArrayOfByteArraysDecl(NamedTuple): + id: str + elements: list[CXXByteArrayDecl] + size: CXXVarDecl + + def decl(self: Self) -> CXXVarDecl: + elements_decl = "\n".join(e.decl().code() for e in self.elements) + count = len(self.elements) + data_ids = ", ".join([d.data.id for d in self.elements]) + size_ids = ", ".join([d.size.id for d in self.elements]) + + return CXXVarDecl( + id=self.id, + expr=f""" +{elements_decl} + +{STORAGE_SPEC} {BYTE_TYPE} const * const {self.id}_elements[{count}] = {{ {data_ids} }}; +{STORAGE_SPEC} {SIZE_TYPE} const {self.id}_element_sizes[{count}] = {{ {size_ids} }}; + + +{STORAGE_SPEC} {NAMESPACE_PREFIX}array_of_byte_arrays_t const {self.id} = {{ + {self.id}_elements, + {self.id}_element_sizes, + {count}ULL +}}; +""", + ) + + @staticmethod + def of_bytes_array( + id: str, data_list: list[bytes], null_terminate: bool + ) -> Self: + size = len(data_list) + size_decl = CXXVarDecl.of_size(id=f"{id}_size", size=size) + array_decls: list[CXXByteArrayDecl] = [ + CXXByteArrayDecl.of_bytes( + id=f"{id}_element_{i}", + data=data, + null_terminate=null_terminate, + ) + for i, data in enumerate(data_list) + ] + + return CXXArrayOfByteArraysDecl( + id=id, elements=array_decls, size=size_decl + ) + + +def generate_cxx_options(id: str, entry: dict) -> str: + options = entry["options"] + arrays: list[bytes] = [opt.encode("utf-8") for opt in options] + + return ( + CXXArrayOfByteArraysDecl.of_bytes_array( + id=f"{id}", data_list=arrays, null_terminate=True + ) + .decl() + .code() + ) + + +class Include(NamedTuple): + name: str + data: bytes + + +class IncludeMapDecl(NamedTuple): + name: str + include_names: CXXArrayOfByteArraysDecl + sources: CXXArrayOfByteArraysDecl + header_sizes: CXXSizeArrayDecl + + def code(self: Self) -> str: + return f""" +{self.include_names.decl().code()} + +{self.sources.decl().code()} + +{self.header_sizes.decl().code()} + +{self.include_names.size.code()} + + +{STORAGE_SPEC} {NAMESPACE_PREFIX}include_map_t const {self.name} = {{ + {self.include_names.id}.elements, + {self.sources.id}.elements, + {self.header_sizes.id}, + {self.include_names.size.id} +}}; +""" + + +def generate_cxx_source_map(id: str, entry: dict): + sources = entry["sources"] + includes: list[Include] = [] + + for source_entry in sources: + include_name = source_entry["include_name"] + file_path = source_entry["file_path"] + with open(file_path, "rb") as f: + header_bytes = f.read() + includes.append(Include(name=include_name, data=header_bytes)) + + source_data_decl: CXXArrayOfByteArraysDecl = ( + CXXArrayOfByteArraysDecl.of_bytes_array( + id=f"{id}_headers", + data_list=[include.data for include in includes], + null_terminate=True, + ) + ) + + header_size_decls: CXXSizeArrayDecl = CXXSizeArrayDecl.of_sizes( + id=f"{id}_header_sizes", + sizes=[len(include.data) for include in includes], + ) + + include_name_decls: CXXArrayOfByteArraysDecl = ( + CXXArrayOfByteArraysDecl.of_bytes_array( + id=f"{id}_include_names", + data_list=[include.name.encode("utf-8") for include in includes], + null_terminate=True, + ) + ) + + return IncludeMapDecl( + name=id, + include_names=include_name_decls, + sources=source_data_decl, + header_sizes=header_size_decls, + ).code() + + +def generate_embed_source(entries: list[dict[str, Any]]) -> str: + merged = {} + + for entry in entries: + entry_type = entry["type"] + id = entry["id"] + + if entry_type == "sources": + sources: list = entry["sources"] + + if id not in merged: + merged[id] = {"type": "sources", "sources": sources} + else: + assert merged[id]["type"] == "sources" + merged[id]["sources"].extend(sources) + + elif entry_type == "options": + options: list = entry["options"] + + if id not in merged: + merged[id] = {"type": "options", "options": options} + else: + assert merged[id]["type"] == "options" + merged[id]["options"].extend(options) + + else: + raise ValueError(f"Unknown type: {entry_type}") + + code: str = "" + + for id, entry in merged.items(): + if entry["type"] == "sources": + code += generate_cxx_source_map(id, entry) + elif entry["type"] == "options": + code += generate_cxx_options(id, entry) + else: + raise ValueError(f"Unknown type: {entry['type']}") + + return f""" +#pragma once + +extern "C" {{ + +typedef struct {NAMESPACE_PREFIX}byte_array_t {{ + {BYTE_TYPE} const * data; + {SIZE_TYPE} size; +}} {NAMESPACE_PREFIX}byte_array_t; + +typedef struct {NAMESPACE_PREFIX}array_of_byte_arrays_t {{ + {BYTE_TYPE} const * const * elements; + {SIZE_TYPE} const * element_sizes; + {SIZE_TYPE} size; +}} {NAMESPACE_PREFIX}array_of_byte_arrays_t; + +typedef struct {NAMESPACE_PREFIX}include_map_t {{ + {BYTE_TYPE} const * const * include_names; + {BYTE_TYPE} const * const * headers; + {SIZE_TYPE} const * header_sizes; + {SIZE_TYPE} num_includes; +}} {NAMESPACE_PREFIX}include_map_t; + +{code} + +}} +""" + + +# Usage: embed.py --input-file --output +def main(): + # parse HEX-encoded string from CLI args + parser = argparse.ArgumentParser( + description="Embed headers, options, or binary blobs into C++ source code." + ) + + # Use CMAKE-encoded options string instead + parser.add_argument( + "--input-file", + type=str, + required=True, + help="JSON description of what to embed", + ) + + parser.add_argument( + "--output", type=str, required=True, help="Output C++ source file" + ) + + args = parser.parse_args() + + with open(args.input_file, "rb") as f: + description = yaml.safe_load(f) + code = generate_embed_source(description) + + with open(args.output, "w") as f: + f.write(code) + + +if __name__ == "__main__": + main() diff --git a/cpp/cxx_embed.py b/cpp/cxx_embed.py deleted file mode 100644 index bf7785555884..000000000000 --- a/cpp/cxx_embed.py +++ /dev/null @@ -1,377 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -import argparse -import binascii -import json -import os -from typing import Any, NamedTuple, Self - -BYTE_TYPE = "unsigned char" -SIZE_TYPE = "unsigned long long" -STORAGE_SPEC = "static" - - -"""headers CXX code - -typedef struct ${id}_t { - unsigned char const * const * include_names; - unsigned char const * const * headers; - unsigned long long const * header_sizes; - unsigned long long num_includes; -} ${id}_t; - -static ${id}_t const ${id} = { ... }; - -""" - -"""options CXX code - -typedef struct ${id}_t { - unsigned char const * const * options; - unsigned long long const * option_sizes; - unsigned long long num_options; -} ${id}_t; - -static ${id}_t const ${id} = { ... }; - -""" - - -### json schema - -"""sources - "sources": - { - "files": [{"source_directory": "string", "files": ["string", ... ]}], - "directories": ["string", ... ] - } -""" - - -"""options - "options": [ "string", ... ] -""" - - -"""entries -{ - "id": { - "type": "sources" | "options", - "sources|options": list | dict - } -} -""" - - -class CXXVarDecl(NamedTuple): - id: str - decl: str - - def code(self: Self) -> str: - f""" - {self.decl} - """ - - @staticmethod - def of_bytes(id: str, data: bytes) -> Self: - byte_array = ", ".join([str(b) for b in data]) - decl = f"{STORAGE_SPEC} {BYTE_TYPE} const {id}[{len(data)}] = {{ {byte_array} }};" - return CXXVarDecl(id=id, decl=decl) - - @staticmethod - def of_size(id: str, size: int) -> Self: - decl = f"{STORAGE_SPEC} {SIZE_TYPE} const {id} = {size}ULL;" - return CXXVarDecl(id=id, decl=decl) - - -class CXXSizeArrayDecl(NamedTuple): - id: str - sizes: list[int] - - def code(self: Self) -> CXXVarDecl: - size_array = ", ".join([f"{size}ULL" for size in self.sizes]) - decl = f"{STORAGE_SPEC} {SIZE_TYPE} const {self.id}[{len(self.sizes)}] = {{ {size_array} }};" - return CXXVarDecl(id=self.id, decl=decl) - - @staticmethod - def of_sizes(id: str, sizes: list[int]) -> Self: - return CXXSizeArrayDecl(id=id, sizes=sizes) - - -class CXXByteArrayDecl(NamedTuple): - id: str - data: CXXVarDecl - size: CXXVarDecl - - def code(self: Self) -> CXXVarDecl: - return CXXVarDecl( - id=self.id, - decl=f""" - {self.data.code()} - {self.size.code()} - - typedef struct {self.id}_t {{ - {BYTE_TYPE} const * data; - {SIZE_TYPE} size; - }} {self.id}_t; - - {STORAGE_SPEC} {self.id}_t const {self.id} = {{ - {self.data.id}, - {self.size.id} - }}; - """, - ) - - @staticmethod - def of_bytes(id: str, data: bytes, null_terminate: bool) -> Self: - # exclude null terminator from length - size_decl = CXXVarDecl.of_size(id=f"{id}_size", size=len(data)) - - if null_terminate: - data += b"\0" - - data_decl = CXXVarDecl.of_bytes(id=f"{id}_data", data=data) - - return CXXByteArrayDecl(id=id, data=data_decl, size=size_decl) - - -class CXXArrayOfByteArraysDecl(NamedTuple): - id: str - elements: list[CXXByteArrayDecl] - size: CXXVarDecl - - def code(self: Self) -> CXXVarDecl: - elements_decl = ";\n".join(e.code() for e in self.elements) - count = len(self.elements) - data_ids = ", ".join([d.data.id for d in self.elements]) - size_ids = ", ".join([d.size.id for d in self.elements]) - - return CXXVarDecl( - id=self.id, - decl=f""" - {elements_decl}; - - {STORAGE_SPEC} {BYTE_TYPE} const * const {self.id}_elements[{count}] = {{ {data_ids} }}; - {STORAGE_SPEC} {SIZE_TYPE} const {self.id}_element_sizes[{count}] = {{ {size_ids} }}; - - typedef struct {self.id}_t {{ - {BYTE_TYPE} const * const * elements; - {SIZE_TYPE} const * element_sizes; - {SIZE_TYPE} size; - }} {self.id}_t; - - {STORAGE_SPEC} {self.id}_t const {self.id} = {{ - {self.id}_elements, - {self.id}_element_sizes, - {count}ULL - }}; - """, - ) - - @staticmethod - def of_bytes_array( - id: str, data_list: list[bytes], null_terminate: bool - ) -> Self: - size = len(data_list) - size_decl = CXXVarDecl.of_size(id=f"{id}_size", size=size) - array_decls: list[CXXByteArrayDecl] = [ - CXXByteArrayDecl.of_bytes( - id=f"{id}_element_{i}", - data=data, - null_terminate=null_terminate, - ) - for i, data in enumerate(data_list) - ] - - return CXXArrayOfByteArraysDecl( - id=id, elements=array_decls, size=size_decl - ) - - -def generate_blobs(entries: Any): - id = entries["id"] - files = entries["files"] - data_list: list[bytes] = [] - - for i, file_path in enumerate(files): - with open(file_path, "rb") as f: - blob_bytes = f.read() - data_list.append(blob_bytes) - - return CXXArrayOfByteArraysDecl.of_bytes_array( - id=f"{id}", data_list=data_list, null_terminate=False - ).code() - - -def generate_options(entry: dict): - id = entry["id"] - options = entry["options"] - arrays: list[bytes] = [opt.encode("utf-8") for opt in options] - - return CXXArrayOfByteArraysDecl.of_bytes_array( - id=f"{id}", data_list=arrays, null_terminate=True - ).code() - - -class Include(NamedTuple): - path: str - data: bytes - - -class IncludeMapDecl(NamedTuple): - name: str - include_names: CXXArrayOfByteArraysDecl - sources: CXXArrayOfByteArraysDecl - header_sizes: CXXSizeArrayDecl - - def code(self: Self) -> str: - return f""" - {self.include_names.code()} - {self.sources.code()} - {self.header_sizes.code()} - - typedef struct {self.name}_t {{ - {BYTE_TYPE} const * const * include_names; - {BYTE_TYPE} const * const * headers; - {SIZE_TYPE} const * header_sizes; - {SIZE_TYPE} num_includes; - }} {self.name}_t; - - {STORAGE_SPEC} {self.name}_t const {self.name} = {{ - {self.include_names.id}.elements, - {self.sources.id}.elements, - {self.header_sizes.id}, - {self.include_names.size.id} - }}; - """ - - -def generate_sources(entry: dict): - id = entry["id"] - sources = entry["sources"] - includes: list[Include] = [] - - for source_entry in sources: - files = source_entry.get("files", None) - directories = source_entry.get("directories", None) - - if files is not None: - source_directory = source_entry.get("source_directory") - files = files["files"] - - for include in files: - path = os.path.join(source_directory, include) - with open(path, "rb") as f: - header_bytes = f.read() - - includes.append(Include(path=include, data=header_bytes)) - - if directories is not None: - for directory in directories: - for root, _, files in os.walk(directory): - for file in files: - path = os.path.join(root, file) - with open(path, "rb") as f: - header_bytes = f.read() - - # make include path relative to directory - include_path = os.path.relpath(path, directory) - - includes.append( - Include(path=include_path, data=header_bytes) - ) - - source_data_decl: CXXArrayOfByteArraysDecl = ( - CXXArrayOfByteArraysDecl.of_bytes_array( - id=f"{id}_headers", - data_list=[include.data for include in includes], - null_terminate=True, - ) - ) - - header_size_decls: CXXSizeArrayDecl = CXXSizeArrayDecl.of_sizes( - id=f"{id}_header_sizes", - sizes=[len(include.data) for include in includes], - ) - - include_name_decls: CXXArrayOfByteArraysDecl = ( - CXXArrayOfByteArraysDecl.of_bytes_array( - id=f"{id}_include_names", - data_list=[include.path.encode("utf-8") for include in includes], - null_terminate=True, - ) - ) - - return IncludeMapDecl( - name=id, - include_names=include_name_decls, - sources=source_data_decl, - header_sizes=header_size_decls, - ).code() - - -def generate_embed_source(entries: Any): - merged = {} - - for id, entry in entries.items(): - entry_type = entry["type"] - - if entry_type == "sources": - sources: list = entry["sources"] - - if id not in merged: - merged[id] = {"type": "sources", "sources": sources} - else: - assert merged[id]["type"] == "sources" - merged[id]["sources"].extend(sources) - - elif entry_type == "options": - options: list = entry["options"] - - if id not in merged: - merged[id] = {"type": "options", "options": options} - else: - assert merged[id]["type"] == "options" - merged[id]["options"].extend(options) - - else: - raise ValueError(f"Unknown type: {entry_type}") - - code: str = "" - - for id, entry in merged.items(): - if entry["type"] == "sources": - code += generate_sources(entry) - elif entry["type"] == "options": - code += generate_options(entry) - else: - raise ValueError(f"Unknown type: {entry['type']}") - - return code - - -# Usage: embed.py --hex "" --out -def main(): - # parse HEX-encoded string from CLI args - parser = argparse.ArgumentParser( - description="Embed headers, options, or binary blobs into C++ source code." - ) - parser.add_argument( - "--hex", - type=str, - required=True, - help="HEX-encoded JSON description of what to embed", - ) - parser.add_argument( - "--out", type=str, required=True, help="Output C++ source file" - ) - args = parser.parse_args() - - # Decode HEX-encoded JSON description - json_hex = binascii.unhexlify(args.hex) - description = json.loads(json_hex) - - -# HEX-encoded JSON description of what to embed -def load_description(json_file): - pass diff --git a/cpp/include/cudf/detail/null_mask_opt.cuh b/cpp/include/cudf/detail/null_mask_opt.cuh new file mode 100644 index 000000000000..cbae41d43cca --- /dev/null +++ b/cpp/include/cudf/detail/null_mask_opt.cuh @@ -0,0 +1,90 @@ + +#include +#include + +namespace cudf { +namespace detail { + +// These will be written together to global memory as the output of a null mask operation. +// Assumes output null_mask buffer is a temporary buffer that can be overwritten. +struct null_mask_chunk { + cudf::bitmask_type word; + cudf::size_type index; + cudf::size_type valid_count; + + // num_chunks and src_size must be >= 0 + template + static __device__ null_mask_chunk load(cudf::size_type current_chunk, + BitMaskWordFunc&& src_null_mask, + cudf::size_type src_offset, + cudf::size_type src_size) + { + auto chunk_bit_begin = current_chunk * static_cast(sizeof(bitmask_type) * 8); + constexpr auto num_chunk_bits = static_cast(sizeof(bitmask_type) * 8); + + auto src_bit_begin = src_offset + chunk_bit_begin; + auto const num_src_words = (src_size + (num_chunk_bits - 1)) / num_chunk_bits; + + auto leading_word_index = src_bit_begin / num_chunk_bits; + auto trailing_word_index = (src_bit_begin + (num_chunk_bits - 1)) / num_chunk_bits; + + if (trailing_word_index < num_src_words) [[likely]] { + auto leading_bits = src_null_mask(leading_word_index); + auto trailing_bits = src_null_mask(trailing_word_index); + auto bit_shift = src_bit_begin % num_chunk_bits; + auto merged = (cudf::bitmask_type)__funnelshift_r(leading_bits, trailing_bits, bit_shift); + auto valid_count = (cudf::size_type)__popc(merged); + return null_mask_chunk{merged, current_chunk, valid_count}; + } else { + auto leading_bits = src_null_mask(leading_word_index); + auto bit_shift = src_bit_begin % num_chunk_bits; + auto num_discard_bits = (src_bit_begin + num_chunk_bits) - (src_offset + src_size); + auto mask = (~bitmask_type{0}) >> num_discard_bits; + auto output = (leading_bits >> bit_shift) & mask; + auto valid_count = (cudf::size_type)__popc(output); + return null_mask_chunk{output, current_chunk, valid_count}; + } + } +}; + +template +CUDF_KERNEL void chunked_bitmask_binop(Binop op, + device_span destination, + device_span source, + device_span source_begin_bits, + size_type source_size_bits, + size_type num_chunks, + size_type* valid_count_ptr) +{ + auto const tid = cudf::detail::grid_1d::global_thread_id(); + cudf::size_type valid_count = 0; + + // for(auto i = tid;) + // [ ] can we reduce the number of blocks so we can have more warps per block? + + // [ ] let the CPU sum the valid counts from each block + + //[ ] use duff's device to unroll the sources + // switch(src.size() / 16){ + // [ ] use function to load + // 16 pairs + reduce + // 8 pairs + reduce + // 4 pairs + reduce + // 2 pairs + reduce + // 1 pair + reduce + // } + + using BlockReduce = cub::BlockReduce; + __shared__ typename BlockReduce::TempStorage temp_storage; + size_type block_valid_count = BlockReduce(temp_storage).Sum(valid_count); + + if (threadIdx.x == 0) { atomicAdd(valid_count_ptr, block_valid_count); } +} + +// to embed in transform kernel + +// write output to uint32_t +// once full or at end, write to global memory; using null_mask_chunk + +} // namespace detail +} // namespace cudf diff --git a/cpp/include/cudf/jit/filter_params.cuh b/cpp/include/cudf/jit/filter_params.cuh index cb2254c4ca7c..3de139a70d4d 100644 --- a/cpp/include/cudf/jit/filter_params.cuh +++ b/cpp/include/cudf/jit/filter_params.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cudf/jit/lto/library.cuh b/cpp/include/cudf/jit/lto/library.cuh index a50eb213f77e..ca733b905393 100644 --- a/cpp/include/cudf/jit/lto/library.cuh +++ b/cpp/include/cudf/jit/lto/library.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/include/cudf/jit/lto/operators.cuh b/cpp/include/cudf/jit/lto/operators.cuh index 361dae7a6024..911ac51bed38 100644 --- a/cpp/include/cudf/jit/lto/operators.cuh +++ b/cpp/include/cudf/jit/lto/operators.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/include/cudf/jit/lto/thunk.cuh b/cpp/include/cudf/jit/lto/thunk.cuh index afeda2ef54b7..36f547c4a372 100644 --- a/cpp/include/cudf/jit/lto/thunk.cuh +++ b/cpp/include/cudf/jit/lto/thunk.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index 64f6e4b707c9..1e04c449199a 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/include/cudf/jit/transform_params.cuh b/cpp/include/cudf/jit/transform_params.cuh index ca6419ca7dfb..a24c00b13103 100644 --- a/cpp/include/cudf/jit/transform_params.cuh +++ b/cpp/include/cudf/jit/transform_params.cuh @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 2aab2bc54815..8cc52a4f4252 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cudf/utilities/defer.hpp b/cpp/include/cudf/utilities/defer.hpp index 079fbe64dab9..075d848b4272 100644 --- a/cpp/include/cudf/utilities/defer.hpp +++ b/cpp/include/cudf/utilities/defer.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/build.hpp b/cpp/src/jit/build.hpp index 6fbd4b60ac13..6dac86982a33 100644 --- a/cpp/src/jit/build.hpp +++ b/cpp/src/jit/build.hpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/lto/library.cu b/cpp/src/jit/lto/library.cu index ae8cd137e7ab..693461a06510 100644 --- a/cpp/src/jit/lto/library.cu +++ b/cpp/src/jit/lto/library.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/lto/operators.inl.cuh b/cpp/src/jit/lto/operators.inl.cuh index 1841cb738c18..3bb35bcd67f5 100644 --- a/cpp/src/jit/lto/operators.inl.cuh +++ b/cpp/src/jit/lto/operators.inl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/jit/lto/types.inl.cuh b/cpp/src/jit/lto/types.inl.cuh index de727e56c686..340496ec263a 100644 --- a/cpp/src/jit/lto/types.inl.cuh +++ b/cpp/src/jit/lto/types.inl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index 05a045ae84f4..35ba831a0719 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index 072713456fcf..60e90a28ad18 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 949a4ae9bf42..010c483510df 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/rtc/cudf.hpp index 894107f47c23..4e6bb328303f 100644 --- a/cpp/src/jit/rtc/cudf.hpp +++ b/cpp/src/jit/rtc/cudf.hpp @@ -1,7 +1,7 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 5674baba7fdf..6d5d60e3bda3 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 3f5c26120208..f9d594987f74 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/rtc/sha256.cpp b/cpp/src/jit/rtc/sha256.cpp index 93be0476125c..d6b752c55cec 100644 --- a/cpp/src/jit/rtc/sha256.cpp +++ b/cpp/src/jit/rtc/sha256.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/rtc/sha256.hpp b/cpp/src/jit/rtc/sha256.hpp index ccce4f18b190..7eef7dd53049 100644 --- a/cpp/src/jit/rtc/sha256.hpp +++ b/cpp/src/jit/rtc/sha256.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh b/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh index 5b393a029685..cd7be58c5ce9 100644 --- a/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh +++ b/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/transform/jit/lto_kernel.inl.cuh b/cpp/src/transform/jit/lto_kernel.inl.cuh index 2ced6aa914b8..e503198f64cd 100644 --- a/cpp/src/transform/jit/lto_kernel.inl.cuh +++ b/cpp/src/transform/jit/lto_kernel.inl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/transform/transform.cpp b/cpp/src/transform/transform.cpp index 4ba1efecea08..9b4306470158 100644 --- a/cpp/src/transform/transform.cpp +++ b/cpp/src/transform/transform.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ From ea6a7f0dd9e5e19f9550d0a5d2e6b8e5172963c0 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 6 Jan 2026 22:51:15 +0000 Subject: [PATCH 010/254] Implement JIT embedding support and update related components --- cpp/CMakeLists.txt | 19 +- cpp/cmake/Modules/JitEmbed.cmake | 2 +- cpp/cmake/Modules/jit_embed.py | 2 +- cpp/src/jit/build.hpp | 41 ---- .../jit/lto/{library.cu => library.inl.cuh} | 0 cpp/src/jit/rtc/cache.cpp | 11 +- cpp/src/jit/rtc/cudf.cpp | 199 ++++++++++++++++++ cpp/src/jit/rtc/cudf.hpp | 32 +-- cpp/src/jit/rtc/rtc.cpp | 14 +- cpp/src/jit/rtc/rtc.hpp | 12 +- cpp/src/runtime/context.cpp | 13 +- cpp/src/runtime/context.hpp | 9 +- .../filter/jit/lto_kernel.inl.cuh | 10 +- cpp/src/transform/jit/lto_kernel.inl.cuh | 10 +- cpp/src/transform/transform.cu | 1 - cpp/tests/CMakeLists.txt | 3 +- cpp/tests/jit/rtc.cpp | 36 ++++ 17 files changed, 300 insertions(+), 114 deletions(-) delete mode 100644 cpp/src/jit/build.hpp rename cpp/src/jit/lto/{library.cu => library.inl.cuh} (100%) create mode 100644 cpp/tests/jit/rtc.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 917d7afbf5d7..011fedc0e901 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -139,8 +139,6 @@ if(NOT CUDF_GENERATED_INCLUDE_DIR) set(CUDF_GENERATED_INCLUDE_DIR ${CUDF_BINARY_DIR}) endif() -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/cudf) - # ################################################################################################## # * linter configuration --------------------------------------------------------------------------- if(CUDF_CLANG_TIDY) @@ -244,8 +242,6 @@ rapids_find_package( ) include(cmake/Modules/ConfigureCUDA.cmake) # set other CUDA compilation flags -find_program(BIN2C_EXE bin2c REQUIRED) - # ################################################################################################## # * dependencies ---------------------------------------------------------------------------------- @@ -364,7 +360,7 @@ list( get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) add_jit_includes( - cudf_jit + cudf_jit_embed DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include FILES @@ -394,14 +390,15 @@ add_jit_includes( # type consistency between AOT and JIT add_jit_includes( - cudf_jit DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} FILES src/jit/accessors.cuh src/jit/span.cuh + cudf_jit_embed DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} FILES src/jit/lto/library.inl.cuh + src/jit/lto/operators.inl.cuh src/jit/lto/types.inl.cuh src/jit/accessors.cuh src/jit/span.cuh ) -add_jit_includes(cudf_jit INCLUDE_DIRECTORY ${LIBCUDACXX_RAW_INCLUDE_DIRS}) +add_jit_includes(cudf_jit_embed INCLUDE_DIRECTORY ${LIBCUDACXX_RAW_INCLUDE_DIRS}) -add_jit_options(cudf_jit OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) +add_jit_options(cudf_jit_embed OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) -generate_jit_source_map(TARGET cudf_jit) +generate_jit_source_map(TARGET cudf_jit_embed) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- @@ -918,8 +915,8 @@ add_library( src/utilities/type_checks.cpp src/utilities/type_dispatcher.cpp ) -add_dependencies(cudf cudf_jit) -target_include_directories(cudf PRIVATE ${cudf_jit_INCLUDE_DIR}) +add_dependencies(cudf cudf_jit_embed) +target_include_directories(cudf PRIVATE ${cudf_jit_embed_INCLUDE_DIR}) # Anything that includes jitify needs to be compiled with _FILE_OFFSET_BITS=64 due to a limitation # in how conda builds glibc diff --git a/cpp/cmake/Modules/JitEmbed.cmake b/cpp/cmake/Modules/JitEmbed.cmake index 9980729440de..47c8fd641d5e 100644 --- a/cpp/cmake/Modules/JitEmbed.cmake +++ b/cpp/cmake/Modules/JitEmbed.cmake @@ -111,7 +111,7 @@ function(generate_jit_source_map) # write CONFIG to temp file and pass file path to script file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}.yaml" "${TARGET_YAML}") - set(INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/jit_embed") + set(INCLUDE_DIR "${CUDF_GENERATED_INCLUDE_DIR}/include/jit_embed") set(HEADER "${INCLUDE_DIR}/${ARG_TARGET}.h") add_custom_command( diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py index 06ee7222cfb7..c91270f82606 100644 --- a/cpp/cmake/Modules/jit_embed.py +++ b/cpp/cmake/Modules/jit_embed.py @@ -6,7 +6,7 @@ import yaml BYTE_TYPE = "unsigned char" -SIZE_TYPE = "unsigned long long" +SIZE_TYPE = "unsigned long" STORAGE_SPEC = "static constexpr" LIST_LINE_WIDTH = 32 NAMESPACE_PREFIX = "jit_" diff --git a/cpp/src/jit/build.hpp b/cpp/src/jit/build.hpp deleted file mode 100644 index 6dac86982a33..000000000000 --- a/cpp/src/jit/build.hpp +++ /dev/null @@ -1,41 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include - -#include - -namespace cudf { -namespace rtc { - -header_map get_jit_headers(); - -header_map get_jit_lto_headers(); - -std::span get_jit_options(); - -std::span get_jit_lto_options(); - -// input: LTO IR data -// output: nvJitLink object -int32_t get_current_device_physical_model() -{ - int32_t device; - // cudaGetDeviceCount() - CUDF_EXPECTS(cudaGetDevice(&device) == cudaSuccess, "Failed to get current CUDA device"); - - cudaDeviceProp props; - CUDF_EXPECTS(cudaGetDeviceProperties(&props, device) == cudaSuccess, - "Failed to get device properties"); - - return props.major * 10 + props.minor; -} - -std::vector get_all_device_physical_models(); - -} // namespace rtc -} // namespace cudf diff --git a/cpp/src/jit/lto/library.cu b/cpp/src/jit/lto/library.inl.cuh similarity index 100% rename from cpp/src/jit/lto/library.cu rename to cpp/src/jit/lto/library.inl.cuh diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index 35ba831a0719..19abcd4951ef 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -43,6 +43,9 @@ cache_t::cache_t(bool enabled, std::string cache_dir, cache_limits const& limits libraries_cache_{limits.num_libraries}, tick_{0} { + CUDF_EXPECTS(limits.num_blobs >= 2, "Blob cache limit must be at least 2"); + CUDF_EXPECTS(limits.num_fragments >= 2, "Fragment cache limit must be at least 2"); + CUDF_EXPECTS(limits.num_libraries >= 2, "Library cache limit must be at least 2"); // Create cache directory if it doesn't exist if (mkdir(cache_dir_.c_str(), S_IRWXU | S_IRWXG | S_IRWXO) == -1) { if (errno != EEXIST) { throw_posix("Failed to create RTC cache directory", "mkdir"); } @@ -342,7 +345,7 @@ void cache_t::clear_disk_store() errno = 0; // reset errno before reading - struct dirent* entry_iter; + struct dirent* entry_iter = nullptr; while (true) { entry_iter = readdir(dir); @@ -355,14 +358,14 @@ void cache_t::clear_disk_store() } } - struct stat st; + struct stat entry_stat; char path[PATH_MAX + 1]; - if (lstat(path, &st) == -1) { + if (lstat(path, &entry_stat) == -1) { throw_posix("Failed to get file status for RTC cache clearing", "lstat"); } - if (S_ISREG(st.st_mode)) { + if (S_ISREG(entry_stat.st_mode)) { if (unlink(path) == -1) { throw_posix("Failed to unlink RTC cache file during clearing", "unlink"); } diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 010c483510df..6953bddd0d97 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -4,8 +4,207 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include + namespace cudf { namespace rtc { +int32_t get_driver_version() +{ + int32_t driver_version; + CUDF_EXPECTS(cudaDriverGetVersion(&driver_version) == cudaSuccess, + "Failed to get CUDA driver version"); + return driver_version; +} + +int32_t get_runtime_version() +{ + int32_t runtime_version; + CUDF_EXPECTS(cudaRuntimeGetVersion(&runtime_version) == cudaSuccess, + "Failed to get CUDA runtime version"); + return runtime_version; +} + +int32_t get_current_device_physical_model() +{ + int32_t device; + CUDF_EXPECTS(cudaGetDevice(&device) == cudaSuccess, "Failed to get current CUDA device"); + + cudaDeviceProp props; + CUDF_EXPECTS(cudaGetDeviceProperties(&props, device) == cudaSuccess, + "Failed to get device properties"); + + return props.major * 10 + props.minor; +} + +sha256_hash hash_string(std::span input) +{ + sha256_context ctx; + ctx.update(std::span{reinterpret_cast(input.data()), input.size()}); + return ctx.finalize(); +} + +cache_t& get_rtc_cache() { return cudf::get_context().rtc_cache(); } + +fragment_t& compile_fragment(char const* name, char const* source_code_cstr, char const* key) +{ + auto sm = get_current_device_physical_model(); + auto const cache_key = std::format(R"***( + fragment_type=LTO_IR, + key={}, + cuda_runtime={}, + cuda_driver={}, + arch={})***", + key, + get_runtime_version(), + get_driver_version(), + sm); + + auto const cache_key_sha256 = hash_string(cache_key); + + auto& cache = get_rtc_cache(); + + if (auto frag = cache.query_fragment(cache_key_sha256); frag.has_value()) { + return *frag->get(); + } else if (auto disk_frag = cache.query_blob_from_disk(cache_key_sha256); disk_frag.has_value()) { + std::promise prom; + auto fut = prom.get_future(); + { + cache.store_fragment(cache_key_sha256, prom.get_future()); + fragment_t::load_params load_params{.binary = *disk_frag, .type = binary_type::LTO_IR}; + auto frag = fragment_t::load(load_params); + prom.set_value(std::move(frag)); + } + return *fut.get(); + } + + std::promise prom; + cache.store_fragment(cache_key_sha256, prom.get_future()); + auto fut = prom.get_future(); + + auto const headers = header_map{ + .include_names = + std::span{reinterpret_cast(cudf_jit_embed_sources.include_names), + cudf_jit_embed_sources.num_includes}, + .headers = std::span{reinterpret_cast(cudf_jit_embed_sources.headers), + cudf_jit_embed_sources.num_includes}, + .header_sizes = + std::span{cudf_jit_embed_sources.header_sizes, cudf_jit_embed_sources.num_includes}}; + + std::vector options; + auto embed_options = reinterpret_cast(cudf_jit_embed_options.elements); + std::copy( + embed_options, embed_options + cudf_jit_embed_options.size, std::back_inserter(options)); + auto arch_flag = std::format("-arch=sm_{}", sm); + options.push_back(arch_flag.c_str()); + options.push_back("-dlto"); + options.push_back("-rdc=true"); + + auto const params = fragment_t::compile_params{.name = name, + .source = source_code_cstr, + .headers = headers, + .options = options, + .target_type = binary_type::LTO_IR}; + + auto frag = fragment_t::compile(params); + + prom.set_value(std::move(frag)); + return *fut.get(); } + +fragment_t& compile_library_fragment() +{ + return compile_fragment("cudf_lto_library", + R"***( + #include "jit/lto/library.inl.cuh" + )***", + "cudf_lto_library"); +} + +fragment_t& compile_udf_fragment(char const* source_code_cstr, char const* key) +{ + return compile_fragment("cudf_udf_fragment", source_code_cstr, key); +} + +kernel_ref compile_and_link_udf(char const* name, + char const* kernel_name, + char const* kernel_key, + char const* udf_code, + char const* udf_key) +{ + auto sm = get_current_device_physical_model(); + auto library_key = std::format(R"***( + fragment_types=LTO_IR, + target=CUBIN, + kernel={}, + udf={}, + cuda_runtime={}, + cuda_driver={}, + arch={})***", + kernel_key, + udf_key, + get_runtime_version(), + get_driver_version(), + sm); + auto const library_key_sha256 = hash_string(library_key); + + auto& cache = get_rtc_cache(); + + // [ ] should probably use query_or_insert + if (auto lib = cache.query_library(library_key_sha256); lib.has_value()) { + return lib->get()->get_kernel(kernel_name); + } else if (auto disk_lib = cache.query_blob_from_disk(library_key_sha256); disk_lib.has_value()) { + std::promise prom; + auto fut = prom.get_future(); + + { + cache.store_library(library_key_sha256, prom.get_future()); + library_t::load_params load_params{.binary = (*disk_lib)->view(), .type = binary_type::CUBIN}; + auto lib = library_t::load(load_params); + prom.set_value(std::move(lib)); + } + + return fut.get()->get_kernel(kernel_name); + } + + auto& library_frag = compile_library_fragment(); + auto& udf_frag = compile_udf_fragment(udf_code, udf_key); + + std::promise prom; + cache.store_library(library_key_sha256, prom.get_future()); + auto fut = prom.get_future(); + + blob_view const link_fragments[] = {library_frag.get_cubin()->view(), + udf_frag.get_cubin()->view()}; + binary_type const fragment_binary_types[] = {binary_type::LTO_IR, binary_type::LTO_IR}; + + char const* const fragment_names[] = {"cudf_lto_library", "cudf_udf_fragment"}; + + auto arch_flag = std::format("-arch=sm_{}", sm); + char const* const link_options[] = {"-lto", arch_flag.c_str()}; + + auto const params = library_t::link_params{.name = name, + .output_type = binary_type::CUBIN, + .fragments = link_fragments, + .fragment_binary_types = fragment_binary_types, + .fragment_names = fragment_names, + .link_options = link_options}; + + auto lib = library_t::link(params); + prom.set_value(std::move(lib)); + return fut.get()->get_kernel(kernel_name); +} + +} // namespace rtc } // namespace cudf diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/rtc/cudf.hpp index 4e6bb328303f..93666c118c72 100644 --- a/cpp/src/jit/rtc/cudf.hpp +++ b/cpp/src/jit/rtc/cudf.hpp @@ -6,36 +6,16 @@ */ #pragma once +#include namespace cudf { namespace rtc { -// SIMPLE KEY: from user -// COMPLEX KEY: sha256 of all parameters and blobs involved; + driver + runtime -// - -// [ ] All these functions should have wrappers that cache results and request an optional key from -// the user - -// [ ] Method to pre-compile library and reuse it across multiple operators; compile_library(); -// compile_library_cached() - -// [ ] jit_key(key) -> key+driver+CUDA_versions+CUDA_runtime_versions+device_compute_models -// #include - -// [ ] environment variables to control: -// [ ] cache path -// [ ] cache entries limit -// [ ] disable caching -// [ ] cache statistics: hits, misses, etc. -// [ ] on startup, log cache path, loading information, etc. -// [ ] not compatible across major driver versions -// [ ] if a user provides a key, use: USER_KEY+${key}, otherwise use sha256 of contents -// [ ] use resource type in key to avoid collisions - -void make_pch(); - -void link_operator(); +kernel_ref compile_and_link_udf(char const* name, + char const* kernel_name, + char const* kernel_key, + char const* udf_code, + char const* udf_key); } // namespace rtc } // namespace cudf diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 6d5d60e3bda3..5ce02dacdd24 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -140,12 +140,12 @@ void log_nvrtc_compile_result(fragment_t::compile_params const& params, std::string headers_str; for (auto const& header : params.headers.include_names) { - headers_str += std::format("\t{}\n", header); + headers_str = std::format("{}\t{}\n", headers_str, header); } std::string options_str; for (auto const& option : params.options) { - options_str += std::format("\t{}\n", option); + options_str = std::format("{}\t{}\n", options_str, option); } auto str = std::format("NCRTC Compilation for {} {} ({}): {}.\nHeaders: {}\nOptions: {}\n\n{}", @@ -183,13 +183,13 @@ void log_nvJitLink_link_result(library_t::link_params const& params, CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetErrorLog(handle, error_log.data())); std::string fragments_str; - for (auto const& fragment : params.names) { - fragments_str += std::format("\t{}\n", fragment); + for (auto const& fragment_name : params.fragment_names) { + fragments_str = std::format("{}\t{}\n", fragments_str, fragment_name); } std::string link_options_str; for (auto const& option : params.link_options) { - link_options_str += std::format("\t{}\n", option); + link_options_str = std::format("{}\t{}\n", link_options_str, option); } char const* binary_type_str = binary_type_string(params.output_type); @@ -356,9 +356,9 @@ blob library_t::link_as_blob(link_params const& params) CUDF_DEFER([&] { nvJitLinkDestroy(&handle); }); for (size_t i = 0; i < params.fragments.size(); i++) { - auto name = params.names[i]; + auto name = params.fragment_names[i]; auto fragment = params.fragments[i]; - auto bin_type = params.binary_types[i]; + auto bin_type = params.fragment_binary_types[i]; nvJitLinkInputType nv_type; diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index f9d594987f74..8688e1547b72 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -158,12 +158,12 @@ struct library_t { }; struct link_params { - char const* name = nullptr; - binary_type output_type = binary_type::CUBIN; - std::span fragments = {}; - std::span binary_types = {}; - std::span names = {}; - std::span link_options = {}; + char const* name = nullptr; + binary_type output_type = binary_type::CUBIN; + std::span fragments = {}; + std::span fragment_binary_types = {}; + std::span fragment_names = {}; + std::span link_options = {}; }; [[nodiscard]] static library load(load_params const& params); diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index d973d6fb9628..51e935a81f0b 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -8,6 +8,7 @@ #include "io/comp/nvcomp_adapter.hpp" #include "io/utilities/getenv_or.hpp" #include "jit/cache.hpp" +#include "jit/rtc/cache.hpp" #include #include @@ -16,7 +17,7 @@ namespace cudf { -context::context(init_flags flags) : _program_cache{nullptr} +context::context(init_flags flags) : _program_cache{nullptr}, _rtc_cache{} { auto dump_codegen_flag = getenv_or("LIBCUDF_JIT_DUMP_CODEGEN", std::string{"OFF"}); _dump_codegen = (dump_codegen_flag == "ON" || dump_codegen_flag == "1"); @@ -33,6 +34,12 @@ jit::program_cache& context::program_cache() return *_program_cache; } +rtc::cache_t& context::rtc_cache() +{ + CUDF_EXPECTS(_rtc_cache != nullptr, "RTC cache not initialized", std::runtime_error); + return *_rtc_cache; +} + bool context::dump_codegen() const { return _dump_codegen; } void context::initialize_components(init_flags flags) @@ -42,6 +49,8 @@ void context::initialize_components(init_flags flags) if (has_flag(new_flags, init_flags::INIT_JIT_CACHE)) { _program_cache = std::make_unique(); + // TODO: Make cache directory configurable + _rtc_cache = std::make_unique(true, "/tmp/cudf_rtc_cache", rtc::cache_limits{}); } if (has_flag(new_flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index c0726d986337..9df6f24a8cb1 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -16,12 +16,17 @@ namespace jit { class program_cache; } +namespace rtc { +class cache_t; +} + /// @brief The context object contains global state internal to CUDF. /// It helps to ensure structured and well-defined construction and destruction of global /// objects/state across translation units. class context { private: std::unique_ptr _program_cache; + std::unique_ptr _rtc_cache; init_flags _initialized_flags = init_flags::NONE; bool _dump_codegen = false; bool _use_jit = false; @@ -36,6 +41,8 @@ class context { jit::program_cache& program_cache(); + rtc::cache_t& rtc_cache(); + [[nodiscard]] bool dump_codegen() const; /// @brief Initialize additional components based on the provided flags diff --git a/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh b/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh index cd7be58c5ce9..26d82f7c4bc1 100644 --- a/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh +++ b/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh @@ -13,7 +13,6 @@ extern "C" { __device__ void filter_operator(cudf::lto::filter_params const* params); __global__ void filter_kernel(void const* outputs, - void const* span_outputs, void const* inputs, void* user_data, cudf::size_type num_rows) @@ -23,11 +22,10 @@ __global__ void filter_kernel(void const* outputs, auto const size = num_rows; for (auto i = start; i < size; i += stride) { - cudf::lto::filter_params p{.inputs = inputs, - .user_data = user_data, - .outputs = outputs, - .span_outputs = span_outputs, - .row_index = static_cast(i)}; + cudf::lto::filter_params p{.inputs = inputs, + .user_data = user_data, + .outputs = outputs, + .row_index = static_cast(i)}; filter_operator(&p); } } diff --git a/cpp/src/transform/jit/lto_kernel.inl.cuh b/cpp/src/transform/jit/lto_kernel.inl.cuh index e503198f64cd..da41bd3babe1 100644 --- a/cpp/src/transform/jit/lto_kernel.inl.cuh +++ b/cpp/src/transform/jit/lto_kernel.inl.cuh @@ -12,7 +12,6 @@ extern "C" { __device__ void transform_operator(cudf::lto::transform_params const* params); __global__ void transform_kernel(void const* outputs, - void const* span_outputs, void const* inputs, void* user_data, cudf::size_type num_rows) @@ -22,11 +21,10 @@ __global__ void transform_kernel(void const* outputs, auto const size = num_rows; for (auto i = start; i < size; i += stride) { - cudf::lto::transform_params p{.inputs = inputs, - .user_data = user_data, - .outputs = outputs, - .span_outputs = span_outputs, - .row_index = static_cast(i)}; + cudf::lto::transform_params p{.inputs = inputs, + .user_data = user_data, + .outputs = outputs, + .row_index = static_cast(i)}; transform_operator(&p); } } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 54bc1ed5de61..1122deedf667 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -17,7 +17,6 @@ #include -#include #include #include #include diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index c931f954ca34..65f11242670b 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2018-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -719,6 +719,7 @@ ConfigureTest(JIT_PARSER_TEST jit/parse_ptx_function.cpp) target_include_directories(JIT_PARSER_TEST PRIVATE "$") ConfigureTest(ROW_IR_TEST jit/row_ir.cpp) +ConfigureTest(RTC_TEST jit/rtc.cpp) # ################################################################################################## # * stream testing --------------------------------------------------------------------------------- diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp new file mode 100644 index 000000000000..05495b2ffd19 --- /dev/null +++ b/cpp/tests/jit/rtc.cpp @@ -0,0 +1,36 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "cudf_test/column_wrapper.hpp" +#include "jit/row_ir.hpp" + +#include +#include + +#include + +using namespace cudf; + +struct RTCTest : public ::testing::Test {}; + +TEST_F(RTCTest, CreateFragment) +{ + rtc::compile_and_link_udf("test_fragment", + "test_kernel_name", + "test_kernel_key", + R"***( + + #include "cudf/jit/transform_params.cuh" + #include "cudf/jit/lto/operators.inl.cuh" + #include "cudf/jit/lto/types.inl.cuh" + + extern "C" __device__ transform_operation(){ + } + + )***", + "test_udf_key"); +} + +CUDF_TEST_PROGRAM_MAIN() From ae34b91b36e08bf1e5c29a3700ef9e3bbfb020de Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 7 Jan 2026 00:03:04 +0000 Subject: [PATCH 011/254] Enhance JIT support by adding CUDA JIT linking and updating fragment compilation methods --- cpp/CMakeLists.txt | 2 +- cpp/src/jit/rtc/cudf.cpp | 15 ++++++++++----- cpp/src/jit/rtc/cudf.hpp | 6 ++++-- cpp/src/jit/rtc/rtc.hpp | 16 ++++++++-------- cpp/tests/jit/rtc.cpp | 12 +++++++----- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 011fedc0e901..691848328872 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1060,7 +1060,7 @@ target_link_libraries( cudf PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ PRIVATE $ cuco::cuco ZLIB::ZLIB nvcomp::nvcomp - kvikio::kvikio nanoarrow::nanoarrow zstd + kvikio::kvikio nanoarrow::nanoarrow zstd CUDA::nvJitLink ) # Add Conda library, and include paths if specified diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 6953bddd0d97..7dda64ead023 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -48,6 +48,11 @@ int32_t get_current_device_physical_model() return props.major * 10 + props.minor; } +void max_occupancy_config() +{ + // [ ] Same as configure_1d_max_occupancy +} + sha256_hash hash_string(std::span input) { sha256_context ctx; @@ -57,7 +62,7 @@ sha256_hash hash_string(std::span input) cache_t& get_rtc_cache() { return cudf::get_context().rtc_cache(); } -fragment_t& compile_fragment(char const* name, char const* source_code_cstr, char const* key) +fragment_t const& compile_fragment(char const* name, char const* source_code_cstr, char const* key) { auto sm = get_current_device_physical_model(); auto const cache_key = std::format(R"***( @@ -123,7 +128,7 @@ fragment_t& compile_fragment(char const* name, char const* source_code_cstr, cha return *fut.get(); } -fragment_t& compile_library_fragment() +fragment_t const& compile_library_fragment() { return compile_fragment("cudf_lto_library", R"***( @@ -132,7 +137,7 @@ fragment_t& compile_library_fragment() "cudf_lto_library"); } -fragment_t& compile_udf_fragment(char const* source_code_cstr, char const* key) +fragment_t const& compile_udf_fragment(char const* source_code_cstr, char const* key) { return compile_fragment("cudf_udf_fragment", source_code_cstr, key); } @@ -178,8 +183,8 @@ kernel_ref compile_and_link_udf(char const* name, return fut.get()->get_kernel(kernel_name); } - auto& library_frag = compile_library_fragment(); - auto& udf_frag = compile_udf_fragment(udf_code, udf_key); + auto const& library_frag = compile_library_fragment(); + auto const& udf_frag = compile_udf_fragment(udf_code, udf_key); std::promise prom; cache.store_library(library_key_sha256, prom.get_future()); diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/rtc/cudf.hpp index 93666c118c72..08b890587813 100644 --- a/cpp/src/jit/rtc/cudf.hpp +++ b/cpp/src/jit/rtc/cudf.hpp @@ -6,9 +6,11 @@ */ #pragma once +#include + #include -namespace cudf { +namespace CUDF_EXPORT cudf { namespace rtc { kernel_ref compile_and_link_udf(char const* name, @@ -18,4 +20,4 @@ kernel_ref compile_and_link_udf(char const* name, char const* udf_key); } // namespace rtc -} // namespace cudf +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 8688e1547b72..2d56a42a3662 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -76,18 +76,18 @@ struct fragment_t; using fragment = std::shared_ptr; /// @brief Represents a partially compiled RTC kernel (i.e. fragment) in LTO-IR or PTX -struct fragment_t { +struct [[nodiscard]] fragment_t { private: blob blob_; binary_type type_; public: - struct load_params { + struct [[nodiscard]] load_params { blob binary = {}; binary_type type = binary_type::LTO_IR; }; - struct compile_params { + struct [[nodiscard]] compile_params { char const* name = nullptr; char const* source = nullptr; header_map headers = {}; @@ -117,7 +117,7 @@ struct library_t; using library = std::shared_ptr; -struct kernel_ref { +struct [[nodiscard]] kernel_ref { private: CUkernel handle_; @@ -140,7 +140,7 @@ struct kernel_ref { /// @brief Represents a loaded RTC library containing compiled kernels /// Input: CUBIN or PTX binary /// Output: loaded library with launchable kernels -struct library_t { +struct [[nodiscard]] library_t { private: CUlibrary handle_; @@ -152,12 +152,12 @@ struct library_t { library_t& operator=(library_t&&) = delete; ~library_t(); - struct load_params { + struct [[nodiscard]] load_params { blob_view binary = {}; binary_type type = binary_type::CUBIN; }; - struct link_params { + struct [[nodiscard]] link_params { char const* name = nullptr; binary_type output_type = binary_type::CUBIN; std::span fragments = {}; @@ -177,7 +177,7 @@ struct library_t { [[nodiscard]] std::vector enumerate_kernels() const; }; -std::string demangle_cuda_symbol(char const* mangled_name); +[[nodiscard]] std::string demangle_cuda_symbol(char const* mangled_name); } // namespace rtc } // namespace cudf diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index 05495b2ffd19..e2968a71ef73 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -17,10 +17,10 @@ struct RTCTest : public ::testing::Test {}; TEST_F(RTCTest, CreateFragment) { - rtc::compile_and_link_udf("test_fragment", - "test_kernel_name", - "test_kernel_key", - R"***( + auto kern = rtc::compile_and_link_udf("test_fragment", + "transform_kernel", + "test_kernel_key", + R"***( #include "cudf/jit/transform_params.cuh" #include "cudf/jit/lto/operators.inl.cuh" @@ -30,7 +30,9 @@ TEST_F(RTCTest, CreateFragment) } )***", - "test_udf_key"); + "test_udf_key"); + + (void)kern; } CUDF_TEST_PROGRAM_MAIN() From 450904c534c177f0b80dea3b364790caefd065c7 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 8 Jan 2026 11:51:00 +0000 Subject: [PATCH 012/254] Refactor JIT embedding and caching mechanisms - Updated `jit_embed.py` to enhance the structure of embedded data, including the addition of SHA256 hashing for string data and file sources. - Introduced alignment parameters for byte arrays and improved handling of null terminators. - Added methods to retrieve cache directory and manage file operations in `cache.cpp` and `cache.hpp`. - Implemented file reading and writing utilities with error handling in `cudf.cpp`. - Enhanced error reporting in CUDA and NVRTC checks for better debugging. - Updated the blob management in `blob.hpp` to support move semantics. - Fixed SHA256 length validation in `sha256.cpp` to use the correct size. - Removed outdated comments and TODOs in `transform.cu` to clean up the codebase. --- cpp/CMakeLists.txt | 73 ++++---- cpp/cmake/Modules/JitEmbed.cmake | 189 ++++++++++++++------- cpp/cmake/Modules/jit_embed.py | 274 ++++++++++++++++--------------- cpp/src/jit/rtc/cache.cpp | 2 + cpp/src/jit/rtc/cache.hpp | 2 + cpp/src/jit/rtc/cudf.cpp | 185 ++++++++++++++++++--- cpp/src/jit/rtc/rtc.cpp | 137 +++++++++------- cpp/src/jit/rtc/rtc.hpp | 29 +++- cpp/src/jit/rtc/sha256.cpp | 2 +- cpp/src/transform/transform.cu | 22 --- 10 files changed, 573 insertions(+), 342 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 691848328872..15ad9215c16e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -350,55 +350,50 @@ endif() set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) -list(APPEND CUDF_JIT_ARCHITECTURES 75 80 86 90 100 120) list(APPEND CUDF_JIT_DEFINITIONS -DCUDF_RUNTIME_JIT -D__CUDACC_RTC__) list( - APPEND CUDF_JIT_COMPILE_FLAGS -O3 -std=c++${CUDF_CUDA_STANDARD} -Xcompiler=-fPIC -Xcompiler=-DPIC - -fvisibility=hidden -Xfatbin=--compress-all --compress-mode=size --device-int128 + APPEND + CUDF_JIT_COMPILE_FLAGS + -std=c++${CUDF_CUDA_STANDARD} + # -Xcompiler=-fPIC -Xcompiler=-DPIC -fvisibility=hidden -Xfatbin=--compress-all + # --compress-mode=size + --device-int128 + ${CUDF_JIT_DEFINITIONS} ) -get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) +jit_add_include_directory( + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include DEST_DIRECTORY cudf/include + INCLUDE_DIRECTORIES cudf/include +) -add_jit_includes( - cudf_jit_embed - DIRECTORY - ${CMAKE_CURRENT_SOURCE_DIR}/include - FILES - cudf/table/table_device_view.cuh - cudf/jit/lto/types.cuh - cudf/jit/lto/operators.cuh - cudf/jit/lto/library.cuh - cudf/jit/lto/thunk.cuh - cudf/jit/transform_params.cuh - cudf/jit/filter_params.cuh - cudf/utilities/traits.cuh - cudf/strings/string_view.cuh - cudf/column/column_device_view_base.cuh - cudf/detail/null_mask.cuh - cudf/detail/utilities/grid_1d.cuh - cudf/detail/offsets_iterator.cuh - cudf/detail/indexalator.cuh - cudf/ast/detail/operator_functor.cuh - cudf/ast/detail/operators.cuh - cudf/jit/lto/types.cuh - cudf/jit/lto/operators.cuh - cudf/jit/transform_params.cuh - cudf/jit/filter_params.cuh +jit_add_include_directory( + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src DEST_DIRECTORY cudf/src + INCLUDE_DIRECTORIES cudf/src ) # TODO: add library.cu LTO type checks need to be compiled as part of the AOT workflow to ensure -# type consistency between AOT and JIT +# type consistency between AOT and JIT Must be in test suite -add_jit_includes( - cudf_jit_embed DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} FILES src/jit/lto/library.inl.cuh - src/jit/lto/operators.inl.cuh src/jit/lto/types.inl.cuh src/jit/accessors.cuh src/jit/span.cuh -) +get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) -add_jit_includes(cudf_jit_embed INCLUDE_DIRECTORY ${LIBCUDACXX_RAW_INCLUDE_DIRS}) +foreach(CUDACXX_INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) + jit_add_include_directory( + cudf_jit_embed COPY_DIRECTORY ${CUDACXX_INC_DIR} DEST_DIRECTORY libcudacxx INCLUDE_DIRECTORIES + libcudacxx + ) + +endforeach() + +foreach(CTK_INC_DIR IN LISTS CUDAToolkit_INCLUDE_DIRS) + jit_add_include_directory( + cudf_jit_embed COPY_DIRECTORY ${CTK_INC_DIR} DEST_DIRECTORY CUDAToolKit INCLUDE_DIRECTORIES + CUDAToolKit CUDAToolKit/cuda/std + ) +endforeach() -add_jit_options(cudf_jit_embed OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) +jit_add_options(cudf_jit_embed OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) -generate_jit_source_map(TARGET cudf_jit_embed) +jit_embed(cudf_jit_embed) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- @@ -1055,12 +1050,12 @@ target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_A # Compile stringified JIT sources first add_dependencies(cudf jitify_preprocess_run) -# Specify the target module library dependencies +# Specify the target module library dependencies TODO: use runtime nvrtc and nvjitlink and driver target_link_libraries( cudf PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ PRIVATE $ cuco::cuco ZLIB::ZLIB nvcomp::nvcomp - kvikio::kvikio nanoarrow::nanoarrow zstd CUDA::nvJitLink + kvikio::kvikio nanoarrow::nanoarrow zstd CUDA::nvJitLink CUDA::nvrtc CUDA::cuda_driver ) # Add Conda library, and include paths if specified diff --git a/cpp/cmake/Modules/JitEmbed.cmake b/cpp/cmake/Modules/JitEmbed.cmake index 47c8fd641d5e..04cbe5d6e7b9 100644 --- a/cpp/cmake/Modules/JitEmbed.cmake +++ b/cpp/cmake/Modules/JitEmbed.cmake @@ -7,133 +7,198 @@ find_package(Python3 REQUIRED COMPONENTS Interpreter) -function(add_jit_includes) - set(IDENTIFIER ${ARGV0}) +function(jit_add_include_directory) + set(TARGET ${ARGV0}) set(OPTIONS "") - set(ONE_VALUE_ARGS DIRECTORY INCLUDE_DIRECTORY) - set(MULTI_VALUE_ARGS FILES) + set(ONE_VALUE_ARGS COPY_DIRECTORY # Source directory where files will be copied from + DEST_DIRECTORY # Destination directory where files will be copied to + ) + set(MULTI_VALUE_ARGS + FILES # Source files relative to COPY_DIRECTORY (optional, if not provided, all files under + # COPY_DIRECTORY will be used) + INCLUDE_DIRECTORIES # Include directories to be used when compiling with these files + ) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - if(NOT ARG_DIRECTORY AND NOT ARG_INCLUDE_DIRECTORY) - message(FATAL_ERROR "Either DIRECTORY or INCLUDE_DIRECTORY must be specified") + if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET argument is required") endif() - if(ARG_DIRECTORY AND ARG_INCLUDE_DIRECTORY) - message(FATAL_ERROR "Only one of DIRECTORY or INCLUDE_DIRECTORY can be specified") + if(NOT ARG_COPY_DIRECTORY) + message(FATAL_ERROR "COPY_DIRECTORY argument is required") endif() - if(ARG_INCLUDE_DIRECTORY AND ARG_FILES) - message(FATAL_ERROR "FILES cannot be specified with INCLUDE_DIRECTORY") + if(NOT ARG_DEST_DIRECTORY) + message(FATAL_ERROR "DEST_DIRECTORY argument is required") endif() - # recursively gather all include files under the specified directory and get their paths relative - # to the base include directory - if(ARG_INCLUDE_DIRECTORY) - file(GLOB_RECURSE INCLUDE_FILES "${ARG_INCLUDE_DIRECTORY}/*") + if(NOT ARG_INCLUDE_DIRECTORIES) + message(FATAL_ERROR "INCLUDE_DIRECTORIES argument is required") + endif() + + if(NOT ARG_FILES) + # gather all include files under the specified directory + file(GLOB_RECURSE INCLUDE_FILES "${ARG_COPY_DIRECTORY}/*") + + # get their paths relative to the base include directory set(INCLUDE_FILES_RELATIVE_PATHS "") foreach(INCLUDE_FILE IN LISTS INCLUDE_FILES) - file(RELATIVE_PATH REL_PATH "${ARG_INCLUDE_DIRECTORY}" "${INCLUDE_FILE}") - list(APPEND INCLUDE_FILES_RELATIVE_PATHS "${REL_PATH}") + file(RELATIVE_PATH INCLUDE_FILE_REL_PATH "${ARG_COPY_DIRECTORY}" "${INCLUDE_FILE}") + list(APPEND INCLUDE_FILES_RELATIVE_PATHS "${INCLUDE_FILE_REL_PATH}") endforeach() - set(ARG_DIRECTORY ${ARG_INCLUDE_DIRECTORY}) + set(ARG_FILES ${INCLUDE_FILES_RELATIVE_PATHS}) endif() # check that each source file exists foreach(SOURCE_FILE IN LISTS ARG_FILES) - if(NOT EXISTS "${ARG_DIRECTORY}/${SOURCE_FILE}") - message(FATAL_ERROR "Source file '${ARG_DIRECTORY}/${SOURCE_FILE}' does not exist") + if(NOT EXISTS "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") + message(FATAL_ERROR "Source file '${ARG_COPY_DIRECTORY}/${SOURCE_FILE}' does not exist") endif() endforeach(SOURCE_FILE) - set(SOURCE_FILES ${${IDENTIFIER}_sources_file_paths}) - set(INCLUDE_NAMES ${${IDENTIFIER}_sources_include_names}) + # Set scope variables to accumulate results + + set(SOURCE_FILES ${jitembed_${TARGET}_incdir__source_files}) + set(SOURCE_FILE_DESTS ${jitembed_${TARGET}_incdir__source_file_dests}) + set(INCLUDE_DIRECTORIES ${jitembed_${TARGET}_incdir__include_directories}) foreach(SOURCE_FILE IN LISTS ARG_FILES) - list(APPEND SOURCE_FILES "${ARG_DIRECTORY}/${SOURCE_FILE}") - list(APPEND INCLUDE_NAMES "${SOURCE_FILE}") + list(APPEND SOURCE_FILES "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") + list(APPEND SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}") endforeach() - set(${IDENTIFIER}_sources_file_paths + list(APPEND INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES}) + + set(jitembed_${TARGET}_incdir__source_files ${SOURCE_FILES} PARENT_SCOPE ) - set(${IDENTIFIER}_sources_include_names - ${INCLUDE_NAMES} + set(jitembed_${TARGET}_incdir__source_file_dests + ${SOURCE_FILE_DESTS} + PARENT_SCOPE + ) + set(jitembed_${TARGET}_incdir__include_directories + ${INCLUDE_DIRECTORIES} PARENT_SCOPE ) endfunction() -function(add_jit_options) - set(IDENTIFIER ${ARGV0}) +function(jit_add_options) + set(TARGET ${ARGV0}) set(OPTIONS "") set(ONE_VALUE_ARGS "") set(MULTI_VALUE_ARGS "OPTIONS") cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - set(${IDENTIFIER}_options - "${ARG_OPTIONS}" + if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET argument is required") + endif() + + if(NOT ARG_OPTIONS) + message(FATAL_ERROR "OPTIONS argument is required") + endif() + + set(options ${jitembed_options_${TARGET}_options}) + foreach(OPTION IN LISTS ARG_OPTIONS) + list(APPEND options "${OPTION}") + endforeach() + + set(jitembed_options_${TARGET}_options + ${options} PARENT_SCOPE ) endfunction() # pass the encoded args to the jit_embed.py script to generate the source and options maps -function(generate_jit_source_map) - set(ONE_VALUE_ARGS "TARGET") +function(jit_embed) + set(TARGET ${ARGV0}) cmake_parse_arguments(ARG "" "${ONE_VALUE_ARGS}" "" ${ARGN}) - list(LENGTH ${ARG_TARGET}_sources_include_names NUM_SOURCES) - list(LENGTH ${ARG_TARGET}_options NUM_OPTIONS) + if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET argument is required") + endif() - set(TARGET_YAML " - id: \"${ARG_TARGET}_sources\"\n type: \"sources\"\n sources:\n") + string(APPEND TARGET_YAML "\"${TARGET}_sources\":\n") + string(APPEND TARGET_YAML " type: \"sources\"\n") - math(EXPR LAST_SOURCE_INDEX "${NUM_SOURCES} - 1") - foreach(i RANGE 0 ${LAST_SOURCE_INDEX}) - list(GET ${ARG_TARGET}_sources_include_names ${i} INCLUDE_NAME) - list(GET ${ARG_TARGET}_sources_file_paths ${i} SOURCE_FILE_PATH) - set(TARGET_YAML "${TARGET_YAML} - include_name: \"${INCLUDE_NAME}\"\n") - set(TARGET_YAML "${TARGET_YAML} file_path: \"${SOURCE_FILE_PATH}\"\n") - endforeach() + if(DEFINED jitembed_${TARGET}_incdir__source_files) - set(TARGET_YAML "${TARGET_YAML}\n\n") - set(TARGET_YAML - "${TARGET_YAML} - id: \"${ARG_TARGET}_options\"\n type: \"options\"\n options:\n" - ) + # gather source files + string(APPEND TARGET_YAML " sources:\n") + list(LENGTH jitembed_${TARGET}_incdir__source_files NUM_SOURCES) - math(EXPR LAST_OPTION_INDEX "${NUM_OPTIONS} - 1") - foreach(i RANGE 0 ${LAST_OPTION_INDEX}) - list(GET ${ARG_TARGET}_options ${i} OPTION) - set(TARGET_YAML "${TARGET_YAML} - \"${OPTION}\"\n") - endforeach() + math(EXPR LAST_SOURCE_INDEX "${NUM_SOURCES} - 1") + foreach(i RANGE 0 ${LAST_SOURCE_INDEX}) + list(GET jitembed_${TARGET}_incdir__source_files ${i} SOURCE_FILE) + list(GET jitembed_${TARGET}_incdir__source_file_dests ${i} SOURCE_FILE_DEST) + string(APPEND TARGET_YAML " - file: \"${SOURCE_FILE}\"\n") + string(APPEND TARGET_YAML " dest: \"${SOURCE_FILE_DEST}\"\n") + endforeach() - # write CONFIG to temp file and pass file path to script - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}.yaml" "${TARGET_YAML}") + # gather include directories + + string(APPEND TARGET_YAML " include_directories:\n") + list(LENGTH jitembed_${TARGET}_incdir__include_directories NUM_INCLUDE_DIRS) + math(EXPR LAST_INCLUDE_DIR_INDEX "${NUM_INCLUDE_DIRS} - 1") + foreach(i RANGE 0 ${LAST_INCLUDE_DIR_INDEX}) + list(GET jitembed_${TARGET}_incdir__include_directories ${i} INCLUDE_DIR) + string(APPEND TARGET_YAML " - \"${INCLUDE_DIR}\"\n") + endforeach() + + endif() + + string(APPEND TARGET_YAML "\n\n") + + if(DEFINED jitembed_options_${TARGET}_options) + string(APPEND TARGET_YAML "\"${TARGET}_options\":\n") + string(APPEND TARGET_YAML " type: \"strings\"\n") + + # gather options + string(APPEND TARGET_YAML " strings:\n") + + list(LENGTH jitembed_options_${TARGET}_options NUM_OPTIONS) + math(EXPR LAST_OPTION_INDEX "${NUM_OPTIONS} - 1") + foreach(i RANGE 0 ${LAST_OPTION_INDEX}) + list(GET jitembed_options_${TARGET}_options ${i} OPTION) + string(APPEND TARGET_YAML " - \"${OPTION}\"\n") + endforeach() + + endif() + + set(YAML_FILE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.yaml") set(INCLUDE_DIR "${CUDF_GENERATED_INCLUDE_DIR}/include/jit_embed") - set(HEADER "${INCLUDE_DIR}/${ARG_TARGET}.h") + set(HEADER "${INCLUDE_DIR}/${TARGET}.h") + + # write CONFIG to temp file and pass file path to script + file(WRITE "${YAML_FILE_PATH}" "${TARGET_YAML}") add_custom_command( OUTPUT ${HEADER} COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" --output - "${HEADER}" --input-file "${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}.yaml" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" - ${${ARG_TARGET}_sources_file_paths} + "${HEADER}" --input "${YAML_FILE_PATH}" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" "${YAML_FILE_PATH}" + ${jitembed_${TARGET}_incdir__source_files} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - COMMENT "Generating JIT source map for ${ARG_TARGET} into ${HEADER}" + COMMENT "Generating JIT embed for ${TARGET} (YAML: ${YAML_FILE_PATH}) into ${HEADER}" VERBATIM ) - add_custom_target(${ARG_TARGET} ALL DEPENDS "${HEADER}") + add_custom_target(${TARGET} ALL DEPENDS "${HEADER}") - message("Generated into: ${HEADER}") + message( + STATUS + "JIT embed for target ${TARGET} (YAML: ${YAML_FILE_PATH}) will be generated into: ${HEADER}" + ) - set(${ARG_TARGET}_INCLUDE_DIR + set(${TARGET}_INCLUDE_DIR ${INCLUDE_DIR} PARENT_SCOPE ) - set(${ARG_TARGET}_HEADER + set(${TARGET}_HEADER ${HEADER} PARENT_SCOPE ) diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py index c91270f82606..dd62cd7b4800 100644 --- a/cpp/cmake/Modules/jit_embed.py +++ b/cpp/cmake/Modules/jit_embed.py @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import argparse -from typing import Any, NamedTuple, Self +import hashlib +from typing import NamedTuple, Self import yaml @@ -11,30 +12,50 @@ LIST_LINE_WIDTH = 32 NAMESPACE_PREFIX = "jit_" - ### json schema """entries [ - { - "id": string, + $id: { "type": "sources", "sources": [ { - "include_name": string, - "file_path": string + "file": string, + "dest": string } - ] + ], + "include_directories": [string] }, - { - "id": string, - "type": "options", - "options": list[string] + $id: { + "type": "strings", + "strings": list[string] } ] """ +PREAMBLE = f""" +/// Auto-generated by jit_embed.py. Do not edit directly. +#pragma once + +extern "C" {{ + +typedef struct {NAMESPACE_PREFIX}byte_array_t {{ + {BYTE_TYPE} const * data; + {SIZE_TYPE} size; +}} {NAMESPACE_PREFIX}byte_array_t; + +typedef struct {NAMESPACE_PREFIX}array_of_byte_arrays_t {{ + {BYTE_TYPE} const * const * elements; + {SIZE_TYPE} const * element_sizes; + {SIZE_TYPE} size; +}} {NAMESPACE_PREFIX}array_of_byte_arrays_t; + +}} + +""" + + def list_string(strings: list[str]) -> str: lines = [] for i in range(0, len(strings), LIST_LINE_WIDTH): @@ -55,9 +76,9 @@ def code(self: Self) -> str: return f"""{self.expr}""" @staticmethod - def of_bytes(id: str, data: bytes) -> Self: + def of_bytes(id: str, data: bytes, alignment: int) -> Self: byte_array = list_string([hex_string(b) for b in data]) - expr = f"""{STORAGE_SPEC} {BYTE_TYPE} const {id}[{len(data)}] = {{ + expr = f"""alignas({alignment}) {STORAGE_SPEC} {BYTE_TYPE} const {id}[{len(data)}] = {{ {byte_array} }};""" return CXXVarDecl(id=id, expr=expr) @@ -106,14 +127,17 @@ def decl(self: Self) -> CXXVarDecl: ) @staticmethod - def of_bytes(id: str, data: bytes, null_terminate: bool) -> Self: + def of_bytes( + id: str, data: bytes, alignment: int, num_null_terminators: int + ) -> Self: # exclude null terminator from length size_decl = CXXVarDecl.of_size(id=f"{id}_size", size=len(data)) - if null_terminate: - data += b"\0" + data += b"\0" * num_null_terminators - data_decl = CXXVarDecl.of_bytes(id=f"{id}_data", data=data) + data_decl = CXXVarDecl.of_bytes( + id=f"{id}_data", data=data, alignment=alignment + ) return CXXByteArrayDecl(id=id, data=data_decl, size=size_decl) @@ -148,7 +172,10 @@ def decl(self: Self) -> CXXVarDecl: @staticmethod def of_bytes_array( - id: str, data_list: list[bytes], null_terminate: bool + id: str, + data_list: list[bytes], + data_alignment: int, + num_null_terminators: int, ) -> Self: size = len(data_list) size_decl = CXXVarDecl.of_size(id=f"{id}_size", size=size) @@ -156,7 +183,8 @@ def of_bytes_array( CXXByteArrayDecl.of_bytes( id=f"{id}_element_{i}", data=data, - null_terminate=null_terminate, + alignment=data_alignment, + num_null_terminators=num_null_terminators, ) for i, data in enumerate(data_list) ] @@ -166,170 +194,152 @@ def of_bytes_array( ) -def generate_cxx_options(id: str, entry: dict) -> str: - options = entry["options"] - arrays: list[bytes] = [opt.encode("utf-8") for opt in options] - - return ( - CXXArrayOfByteArraysDecl.of_bytes_array( - id=f"{id}", data_list=arrays, null_terminate=True - ) - .decl() - .code() - ) +def generate_cxx_string_data(id: str, strings: list[str]) -> str: + arrays: list[bytes] = [s.encode("utf-8") for s in strings] + # compute combined sha256 hash of all strings + sha = hashlib.sha256() + for arr in arrays: + sha.update(arr) + data_hash: bytes = sha.digest() -class Include(NamedTuple): - name: str - data: bytes - + arrays_decl = CXXArrayOfByteArraysDecl.of_bytes_array( + id=f"{id}", data_list=arrays, data_alignment=1, num_null_terminators=1 + ) -class IncludeMapDecl(NamedTuple): - name: str - include_names: CXXArrayOfByteArraysDecl - sources: CXXArrayOfByteArraysDecl - header_sizes: CXXSizeArrayDecl + data_hash_decl: CXXByteArrayDecl = CXXByteArrayDecl.of_bytes( + id=f"{id}_data_hash", + data=data_hash, + alignment=1, + num_null_terminators=0, + ) - def code(self: Self) -> str: - return f""" -{self.include_names.decl().code()} + return f""" +{arrays_decl.decl().code()} -{self.sources.decl().code()} +{data_hash_decl.decl().code()} +""" -{self.header_sizes.decl().code()} -{self.include_names.size.code()} +def generate_cxx_source_files_data( + id: str, + file_paths: list[str], + dests: list[str], + include_directories: list[str] = [], +) -> str: + files_bytes: list[bytes] = [] + # compute combined sha256 hash of all files + sha = hashlib.sha256() -{STORAGE_SPEC} {NAMESPACE_PREFIX}include_map_t const {self.name} = {{ - {self.include_names.id}.elements, - {self.sources.id}.elements, - {self.header_sizes.id}, - {self.include_names.size.id} -}}; -""" - + for file_path in file_paths: + with open(file_path, "rb") as f: + data = f.read() + sha.update(data) + files_bytes.append(data) -def generate_cxx_source_map(id: str, entry: dict): - sources = entry["sources"] - includes: list[Include] = [] + file_data_hash: bytes = sha.digest() - for source_entry in sources: - include_name = source_entry["include_name"] - file_path = source_entry["file_path"] - with open(file_path, "rb") as f: - header_bytes = f.read() - includes.append(Include(name=include_name, data=header_bytes)) + file_destinations_decls: CXXArrayOfByteArraysDecl = ( + CXXArrayOfByteArraysDecl.of_bytes_array( + id=f"{id}_file_destinations", + data_list=[f.encode("utf-8") for f in dests], + data_alignment=1, + num_null_terminators=1, + ) + ) - source_data_decl: CXXArrayOfByteArraysDecl = ( + file_data_decl: CXXArrayOfByteArraysDecl = ( CXXArrayOfByteArraysDecl.of_bytes_array( - id=f"{id}_headers", - data_list=[include.data for include in includes], - null_terminate=True, + id=f"{id}_file_data", + data_list=files_bytes, + data_alignment=1, + num_null_terminators=1, ) ) - header_size_decls: CXXSizeArrayDecl = CXXSizeArrayDecl.of_sizes( - id=f"{id}_header_sizes", - sizes=[len(include.data) for include in includes], + file_data_hash_decl: CXXByteArrayDecl = CXXByteArrayDecl.of_bytes( + id=f"{id}_file_data_hash", + data=file_data_hash, + alignment=1, + num_null_terminators=0, + ) + + file_size_decls: CXXSizeArrayDecl = CXXSizeArrayDecl.of_sizes( + id=f"{id}_file_sizes", + sizes=[len(f) for f in files_bytes], ) - include_name_decls: CXXArrayOfByteArraysDecl = ( + include_directories_decls: CXXArrayOfByteArraysDecl = ( CXXArrayOfByteArraysDecl.of_bytes_array( - id=f"{id}_include_names", - data_list=[include.name.encode("utf-8") for include in includes], - null_terminate=True, + id=f"{id}_include_directories", + data_list=[d.encode("utf-8") for d in include_directories], + data_alignment=1, + num_null_terminators=1, ) ) - return IncludeMapDecl( - name=id, - include_names=include_name_decls, - sources=source_data_decl, - header_sizes=header_size_decls, - ).code() - + return f""" +{file_destinations_decls.decl().code()} -def generate_embed_source(entries: list[dict[str, Any]]) -> str: - merged = {} +{file_data_decl.decl().code()} - for entry in entries: - entry_type = entry["type"] - id = entry["id"] +{file_data_hash_decl.decl().code()} - if entry_type == "sources": - sources: list = entry["sources"] +{file_size_decls.decl().code()} - if id not in merged: - merged[id] = {"type": "sources", "sources": sources} - else: - assert merged[id]["type"] == "sources" - merged[id]["sources"].extend(sources) +{include_directories_decls.decl().code()} +""" - elif entry_type == "options": - options: list = entry["options"] - if id not in merged: - merged[id] = {"type": "options", "options": options} - else: - assert merged[id]["type"] == "options" - merged[id]["options"].extend(options) +# TODO: write a schema validator for the input YAML - else: - raise ValueError(f"Unknown type: {entry_type}") +def generate_embed_source(entries: dict[str, dict[str, dict]]) -> str: code: str = "" - for id, entry in merged.items(): - if entry["type"] == "sources": - code += generate_cxx_source_map(id, entry) - elif entry["type"] == "options": - code += generate_cxx_options(id, entry) - else: - raise ValueError(f"Unknown type: {entry['type']}") + for entry_id, entry_value in entries.items(): + entry_type = entry_value["type"] - return f""" -#pragma once + if entry_type == "sources": + sources: list[dict] = entry_value["sources"] + file_paths = [s["file"] for s in sources] + dests = [s["dest"] for s in sources] + include_directories: list[str] = entry_value["include_directories"] + code += generate_cxx_source_files_data( + entry_id, file_paths, dests, include_directories + ) -extern "C" {{ + elif entry_type == "strings": + options: list[str] = entry_value["strings"] + code += generate_cxx_string_data(entry_id, options) -typedef struct {NAMESPACE_PREFIX}byte_array_t {{ - {BYTE_TYPE} const * data; - {SIZE_TYPE} size; -}} {NAMESPACE_PREFIX}byte_array_t; + else: + raise ValueError(f"Unknown type: {entry_type}") -typedef struct {NAMESPACE_PREFIX}array_of_byte_arrays_t {{ - {BYTE_TYPE} const * const * elements; - {SIZE_TYPE} const * element_sizes; - {SIZE_TYPE} size; -}} {NAMESPACE_PREFIX}array_of_byte_arrays_t; + return f""" +{PREAMBLE} -typedef struct {NAMESPACE_PREFIX}include_map_t {{ - {BYTE_TYPE} const * const * include_names; - {BYTE_TYPE} const * const * headers; - {SIZE_TYPE} const * header_sizes; - {SIZE_TYPE} num_includes; -}} {NAMESPACE_PREFIX}include_map_t; +extern "C" {{ {code} }} + """ -# Usage: embed.py --input-file --output +# Usage: embed.py --input --output def main(): - # parse HEX-encoded string from CLI args parser = argparse.ArgumentParser( description="Embed headers, options, or binary blobs into C++ source code." ) - # Use CMAKE-encoded options string instead parser.add_argument( - "--input-file", + "--input", type=str, required=True, - help="JSON description of what to embed", + help="YAML description of what to embed", ) parser.add_argument( @@ -338,7 +348,7 @@ def main(): args = parser.parse_args() - with open(args.input_file, "rb") as f: + with open(args.input, "rb") as f: description = yaml.safe_load(f) code = generate_embed_source(description) diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index 19abcd4951ef..5a9d6dff70a8 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -70,6 +70,8 @@ void cache_t::disable() c.store(false, std::memory_order_relaxed); } +std::string const& cache_t::get_cache_dir() { return cache_dir_; } + void cache_t::store_blob_to_memory(sha256_hash const& sha, std::shared_future binary) { CUDF_FUNC_RANGE(); diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index 60e90a28ad18..517300e23c09 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -296,6 +296,8 @@ struct cache_t { void disable(); + [[nodiscard]] std::string const& get_cache_dir(); + void store_blob_to_memory(sha256_hash const& sha, std::shared_future binary); void store_blob_to_disk(sha256_hash const& sha, blob_view binary); diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 7dda64ead023..0acb34a3a876 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -4,16 +4,22 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include #include #include #include +#include #include #include #include #include +#include +#include +#include +#include #include #include @@ -50,7 +56,7 @@ int32_t get_current_device_physical_model() void max_occupancy_config() { - // [ ] Same as configure_1d_max_occupancy + // TODO: Same as configure_1d_max_occupancy } sha256_hash hash_string(std::span input) @@ -62,6 +68,134 @@ sha256_hash hash_string(std::span input) cache_t& get_rtc_cache() { return cudf::get_context().rtc_cache(); } +[[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) +{ + auto error_code = errno; + auto error_str = std::format( + "{}. `{}` failed with {} ({})", message, syscall_name, error_code, std::strerror(error_code)); + CUDF_FAIL(+error_str, std::runtime_error); +} + +void add_file(char const* dst_path, unsigned char const* data, size_t data_size) +{ + int dst_file = open(dst_path, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (dst_file == -1) { + if (errno == EEXIST) { + // file already exists (repeated include) + return; + } + throw_posix(std::format("Failed to create file ({})", dst_path), "open"); + } + + CUDF_DEFER([&] { + if (close(dst_file) != 0) { + throw_posix(std::format("Failed to close file ({})", dst_path), "close"); + } + }); + + if (write(dst_file, data, data_size) == -1) { + throw_posix(std::format("Failed to write file ({})", dst_path), "write"); + } +} + +std::vector read_file(char const* path) +{ + int fd = open(path, O_RDONLY); + + if (fd == -1) { throw_posix(std::format("Failed to open file ({})", path), "open"); } + + CUDF_DEFER([&] { + if (close(fd) == -1) { throw_posix(std::format("Failed to close file ({})", path), "close"); } + }); + + // get file size + struct stat file_stat; + if (fstat(fd, &file_stat) == -1) { + throw_posix(std::format("Failed to get file status for file ({})", path), "fstat"); + } + + std::vector contents; + contents.resize(file_stat.st_size); + + if (read(fd, contents.data(), contents.size()) == -1) { + throw_posix(std::format("Failed to read file ({})", path), "read"); + } + + return contents; +} + +void copy_includes_to_dir(char const* dir) +{ + for (size_t i = 0; i < cudf_jit_embed_sources_file_data.size; ++i) { + auto const data = cudf_jit_embed_sources_file_data.elements[i]; + auto const data_size = cudf_jit_embed_sources_file_data.element_sizes[i]; + auto const dst = + reinterpret_cast(cudf_jit_embed_sources_file_destinations.elements[i]); + auto const dst_path = std::format("{}/{}", dir, dst); + + std::filesystem::create_directories(std::filesystem::path{dst_path}.parent_path()); + add_file(dst_path.c_str(), data, data_size); + } + + auto hash_path = std::format("{}/state.hash", dir); + add_file(hash_path.c_str(), + cudf_jit_embed_sources_file_data_hash.data, + cudf_jit_embed_sources_file_data_hash.size); +} + +void create_new_include_dir(std::string const& include_path) +{ + // directory does not exist, so create it + char tmp_dir_data[] = "/tmp/jit_includes_XXXXXX"; + char* tmp_dir = mkdtemp(tmp_dir_data); + if (tmp_dir == nullptr) { + throw_posix( + std::format("Failed to create temporary RTC include directory for ({})", include_path), + "mkdtemp"); + } + + copy_includes_to_dir(tmp_dir); + + // rename the temporary directory to the target include_path + if (rename(tmp_dir, include_path.c_str()) == -1) { + throw_posix( + std::format("Failed to rename temporary RTC include directory to ({})", include_path), + "rename"); + } +} + +void create_include_dir(std::string const& include_path) +{ + struct stat path_info; + + if (lstat(include_path.c_str(), &path_info) == -1) { + if (errno != ENOENT) { + throw_posix(std::format("Failed to get stat for jit_include directory ({})", include_path), + "lstat"); + } else { + create_new_include_dir(include_path); + } + } else { + if (!S_ISDIR(path_info.st_mode)) { + CUDF_FAIL(+std::format("RTC include path ({}) exists but is not a directory", include_path), + std::runtime_error); + } else { + // verify contents match expected headers + auto hash_path = std::format("{}/state.hash", include_path); + auto hash_data = read_file(hash_path.c_str()); + + if (hash_data.size() != cudf_jit_embed_sources_file_data_hash.size || + std::memcmp(hash_data.data(), + cudf_jit_embed_sources_file_data_hash.data, + cudf_jit_embed_sources_file_data_hash.size) != 0) { + CUDF_FAIL(+std::format("RTC include directory ({}) contents do not match expected headers", + include_path), + std::runtime_error); + } + } + } +} + fragment_t const& compile_fragment(char const* name, char const* source_code_cstr, char const* key) { auto sm = get_current_device_physical_model(); @@ -84,9 +218,9 @@ fragment_t const& compile_fragment(char const* name, char const* source_code_cst return *frag->get(); } else if (auto disk_frag = cache.query_blob_from_disk(cache_key_sha256); disk_frag.has_value()) { std::promise prom; - auto fut = prom.get_future(); + auto fut = std::shared_future{prom.get_future()}; { - cache.store_fragment(cache_key_sha256, prom.get_future()); + cache.store_fragment(cache_key_sha256, fut); fragment_t::load_params load_params{.binary = *disk_frag, .type = binary_type::LTO_IR}; auto frag = fragment_t::load(load_params); prom.set_value(std::move(frag)); @@ -95,17 +229,21 @@ fragment_t const& compile_fragment(char const* name, char const* source_code_cst } std::promise prom; - cache.store_fragment(cache_key_sha256, prom.get_future()); - auto fut = prom.get_future(); - - auto const headers = header_map{ - .include_names = - std::span{reinterpret_cast(cudf_jit_embed_sources.include_names), - cudf_jit_embed_sources.num_includes}, - .headers = std::span{reinterpret_cast(cudf_jit_embed_sources.headers), - cudf_jit_embed_sources.num_includes}, - .header_sizes = - std::span{cudf_jit_embed_sources.header_sizes, cudf_jit_embed_sources.num_includes}}; + auto fut = std::shared_future{prom.get_future()}; + cache.store_fragment(cache_key_sha256, fut); + + auto cache_dir = cache.get_cache_dir(); + auto include_dir = std::format("{}/jit_includes", cache_dir); + + create_include_dir(include_dir); + + std::vector include_options; + include_options.push_back(std::format("-I{}", include_dir)); + for (size_t i = 0; i < cudf_jit_embed_sources_include_directories.size; i++) { + auto include_path = + reinterpret_cast(cudf_jit_embed_sources_include_directories.elements[i]); + include_options.push_back(std::format("-I{}/{}", include_dir, include_path)); + } std::vector options; auto embed_options = reinterpret_cast(cudf_jit_embed_options.elements); @@ -115,10 +253,16 @@ fragment_t const& compile_fragment(char const* name, char const* source_code_cst options.push_back(arch_flag.c_str()); options.push_back("-dlto"); options.push_back("-rdc=true"); + options.push_back("--split-compile=0"); + options.push_back("-default-device"); + + for (auto const& include_option : include_options) { + options.push_back(include_option.c_str()); + } auto const params = fragment_t::compile_params{.name = name, .source = source_code_cstr, - .headers = headers, + .headers = {}, .options = options, .target_type = binary_type::LTO_IR}; @@ -166,15 +310,15 @@ kernel_ref compile_and_link_udf(char const* name, auto& cache = get_rtc_cache(); - // [ ] should probably use query_or_insert + // TODO: (atomicity) should probably use query_or_insert if (auto lib = cache.query_library(library_key_sha256); lib.has_value()) { return lib->get()->get_kernel(kernel_name); } else if (auto disk_lib = cache.query_blob_from_disk(library_key_sha256); disk_lib.has_value()) { std::promise prom; - auto fut = prom.get_future(); + auto fut = std::shared_future{prom.get_future()}; { - cache.store_library(library_key_sha256, prom.get_future()); + cache.store_library(library_key_sha256, fut); library_t::load_params load_params{.binary = (*disk_lib)->view(), .type = binary_type::CUBIN}; auto lib = library_t::load(load_params); prom.set_value(std::move(lib)); @@ -187,8 +331,8 @@ kernel_ref compile_and_link_udf(char const* name, auto const& udf_frag = compile_udf_fragment(udf_code, udf_key); std::promise prom; - cache.store_library(library_key_sha256, prom.get_future()); - auto fut = prom.get_future(); + auto fut = std::shared_future{prom.get_future()}; + cache.store_library(library_key_sha256, fut); blob_view const link_fragments[] = {library_frag.get_cubin()->view(), udf_frag.get_cubin()->view()}; @@ -208,6 +352,7 @@ kernel_ref compile_and_link_udf(char const* name, auto lib = library_t::link(params); prom.set_value(std::move(lib)); + return fut.get()->get_kernel(kernel_name); } diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 5ce02dacdd24..0967918d05be 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -23,49 +23,51 @@ #define CUDFRTC_CONCATENATE_DETAIL(x, y) x##y #define CUDFRTC_CONCATENATE(x, y) CUDFRTC_CONCATENATE_DETAIL(x, y) -#define CUDFRTC_CHECK_CUDA(...) \ - do { \ - CUresult result = (__VA_ARGS__); \ - if (result != CUDA_SUCCESS) { \ - char const* enum_str; \ - CUDF_EXPECTS(cuGetErrorString(result, &enum_str) == CUDA_SUCCESS, \ - "Unable to get CUDA error string"); \ - auto errstr = std::format("CUDA Call {} failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast(result), \ - enum_str); \ - CUDF_FAIL(+errstr, std::runtime_error); \ - } \ +#define CUDFRTC_CHECK_CUDA(...) \ + do { \ + ::CUresult result = (__VA_ARGS__); \ + if (result != CUDA_SUCCESS) { \ + char const* enum_str; \ + CUDF_EXPECTS(::cuGetErrorString(result, &enum_str) == CUDA_SUCCESS, \ + "Unable to get CUDA error string"); \ + auto errstr = ::std::format("CUDA Call {} failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast(result), \ + enum_str); \ + CUDF_FAIL(+errstr, ::std::runtime_error); \ + } \ } while (0) -#define CUDFRTC_CHECK_NVRTC(...) \ - do { \ - nvrtcResult result = (__VA_ARGS__); \ - if (result != NVRTC_SUCCESS) { \ - auto errstr = std::format("NVRTC Call {} failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast(result), \ - nvrtcGetErrorString(result)); \ - CUDF_FAIL(+errstr, std::runtime_error); \ - } \ +#define CUDFRTC_CHECK_NVRTC(params, program, ...) \ + do { \ + ::nvrtcResult result = (__VA_ARGS__); \ + if (result != NVRTC_SUCCESS) { \ + auto errstr = ::std::format("NVRTC Call {} failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast(result), \ + ::nvrtcGetErrorString(result)); \ + CUDF_FAIL(+errstr + "\n" + \ + (program == nullptr ? "" : ::cudf::rtc::get_nvrtc_log(params, program, result)), \ + ::std::runtime_error); \ + } \ } while (0) -#define CUDFRTC_CHECK_NVJITLINK(...) \ - do { \ - nvJitLinkResult result = (__VA_ARGS__); \ - if (result != NVJITLINK_SUCCESS) { \ - auto errstr = std::format("nvJitLink Call {} failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast(result), \ - cudf_nvJitLinkResultString(result)); \ - CUDF_FAIL(+errstr, std::runtime_error); \ - } \ +#define CUDFRTC_CHECK_NVJITLINK(...) \ + do { \ + ::nvJitLinkResult result = (__VA_ARGS__); \ + if (result != NVJITLINK_SUCCESS) { \ + auto errstr = ::std::format("nvJitLink Call {} failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast(result), \ + ::cudf::rtc::get_nvJitLinkResultString(result)); \ + CUDF_FAIL(+errstr, std::runtime_error); \ + } \ } while (0) namespace cudf { namespace rtc { -char const* cudf_nvJitLinkResultString(nvJitLinkResult result) +char const* get_nvJitLinkResultString(nvJitLinkResult result) { switch (result) { case NVJITLINK_SUCCESS: return "NVJITLINK_SUCCESS"; @@ -124,16 +126,17 @@ fragment fragment_t::load(load_params const& params) return std::make_shared(params.binary, params.type); } -void log_nvrtc_compile_result(fragment_t::compile_params const& params, - nvrtcProgram program, - nvrtcResult compile_result) +std::string get_nvrtc_log(fragment_t::compile_params const& params, + nvrtcProgram program, + nvrtcResult compile_result) { size_t log_size; - CUDFRTC_CHECK_NVRTC(nvrtcGetProgramLogSize(program, &log_size)); + if (nvrtcGetProgramLogSize(program, &log_size) != NVRTC_SUCCESS) { return std::string{}; } std::vector log; log.resize(log_size); - CUDFRTC_CHECK_NVRTC(nvrtcGetProgramLog(program, log.data())); + + if (nvrtcGetProgramLog(program, log.data()) != NVRTC_SUCCESS) { return std::string{}; } auto status_str = (compile_result == NVRTC_SUCCESS) ? "completed with warning" : "failed with error"; @@ -148,20 +151,14 @@ void log_nvrtc_compile_result(fragment_t::compile_params const& params, options_str = std::format("{}\t{}\n", options_str, option); } - auto str = std::format("NCRTC Compilation for {} {} ({}): {}.\nHeaders: {}\nOptions: {}\n\n{}", - params.name == nullptr ? "" : params.name, - status_str, - static_cast(compile_result), - nvrtcGetErrorString(compile_result), - headers_str, - options_str, - log.data()); - - if (compile_result != NVRTC_SUCCESS) { - CUDF_FAIL(+str, std::runtime_error); - } else if (!log.empty()) { - CUDF_LOG_WARN(str); - } + return std::format("NCRTC Compilation for {} {} ({}): {}.\nHeaders: {}\nOptions:\t{}\n\n{}", + params.name == nullptr ? "" : params.name, + status_str, + static_cast(compile_result), + nvrtcGetErrorString(compile_result), + headers_str, + options_str, + log.data()); } void log_nvJitLink_link_result(library_t::link_params const& params, @@ -204,7 +201,7 @@ void log_nvJitLink_link_result(library_t::link_params const& params, binary_type_str, status_str, static_cast(link_result), - cudf_nvJitLinkResultString(link_result), + get_nvJitLinkResultString(link_result), fragments_str, link_options_str, info_log.data(), @@ -221,8 +218,11 @@ fragment fragment_t::compile(compile_params const& params) { CUDF_FUNC_RANGE(); - nvrtcProgram program; - CUDFRTC_CHECK_NVRTC(nvrtcCreateProgram(&program, + nvrtcProgram program = nullptr; + + CUDFRTC_CHECK_NVRTC(params, + program, + nvrtcCreateProgram(&program, params.source, params.name, static_cast(params.headers.headers.size()), @@ -231,19 +231,25 @@ fragment fragment_t::compile(compile_params const& params) CUDF_DEFER([&] { nvrtcDestroyProgram(&program); }); - auto compile_result = - nvrtcCompileProgram(program, static_cast(params.options.size()), params.options.data()); + CUDFRTC_CHECK_NVRTC( + params, + program, + nvrtcCompileProgram(program, static_cast(params.options.size()), params.options.data())); - log_nvrtc_compile_result(params, program, compile_result); + if (auto log = get_nvrtc_log(params, program, NVRTC_SUCCESS); !log.empty()) { + CUDF_LOG_WARN(log); + } switch (params.target_type) { case binary_type::LTO_IR: { size_t lto_ir_size; - CUDFRTC_CHECK_NVRTC(nvrtcGetLTOIRSize(program, <o_ir_size)); + CUDFRTC_CHECK_NVRTC(params, program, nvrtcGetLTOIRSize(program, <o_ir_size)); std::vector lto_ir; lto_ir.resize(lto_ir_size); - CUDFRTC_CHECK_NVRTC(nvrtcGetLTOIR(program, reinterpret_cast(lto_ir.data()))); + + CUDFRTC_CHECK_NVRTC( + params, program, nvrtcGetLTOIR(program, reinterpret_cast(lto_ir.data()))); auto shared_blob = std::make_shared(blob_t::from_vector(std::move(lto_ir))); @@ -252,11 +258,12 @@ fragment fragment_t::compile(compile_params const& params) } break; case binary_type::CUBIN: { size_t cubin_size; - CUDFRTC_CHECK_NVRTC(nvrtcGetCUBINSize(program, &cubin_size)); + CUDFRTC_CHECK_NVRTC(params, program, nvrtcGetCUBINSize(program, &cubin_size)); std::vector cubin; cubin.resize(cubin_size); - CUDFRTC_CHECK_NVRTC(nvrtcGetCUBIN(program, reinterpret_cast(cubin.data()))); + CUDFRTC_CHECK_NVRTC( + params, program, nvrtcGetCUBIN(program, reinterpret_cast(cubin.data()))); auto shared_blob = std::make_shared(blob_t::from_vector(std::move(cubin))); @@ -381,19 +388,23 @@ blob library_t::link_as_blob(link_params const& params) switch (params.output_type) { case binary_type::CUBIN: { size_t cubin_size; + CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedCubinSize(handle, &cubin_size)); std::vector cubin; cubin.resize(cubin_size); CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedCubin(handle, cubin.data())); + return std::make_shared(blob_t::from_vector(std::move(cubin))); } break; case binary_type::PTX: { size_t ptx_size; + CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedPtxSize(handle, &ptx_size)); std::vector ptx; ptx.resize(ptx_size); CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedPtx(handle, reinterpret_cast(ptx.data()))); + return std::make_shared(blob_t::from_vector(std::move(ptx))); } break; diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 2d56a42a3662..f25b9a30955e 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -45,10 +45,30 @@ struct [[nodiscard]] blob_t { public: blob_t() : data_(nullptr), size_(0), user_data_(nullptr), deallocator_(noop_deallocator) {} + blob_t(blob_t const&) = delete; blob_t& operator=(blob_t const&) = delete; - blob_t(blob_t&& other) noexcept; - blob_t& operator=(blob_t&& other) noexcept; + + blob_t(blob_t&& other) noexcept + : data_(other.data_), + size_(other.size_), + user_data_(other.user_data_), + deallocator_(other.deallocator_) + { + other.data_ = nullptr; + other.size_ = 0; + other.user_data_ = nullptr; + other.deallocator_ = noop_deallocator; + } + + blob_t& operator=(blob_t&& other) noexcept + { + if (this == &other) [[unlikely]] { return *this; } + this->~blob_t(); + new (this) blob_t(std::move(other)); + return *this; + } + ~blob_t() { deallocator_(user_data_, data_, size_); } [[nodiscard]] blob_view view() const { return blob_view{data_, size_}; } @@ -56,7 +76,10 @@ struct [[nodiscard]] blob_t { static blob_t from_parts(uint8_t const* data, size_t size, void* user_data, - dealloctor_fn deallocator); + dealloctor_fn deallocator) + { + return blob_t{data, size, user_data, deallocator}; + } static blob_t from_vector(std::vector&& data); diff --git a/cpp/src/jit/rtc/sha256.cpp b/cpp/src/jit/rtc/sha256.cpp index d6b752c55cec..1cd36d23187c 100644 --- a/cpp/src/jit/rtc/sha256.cpp +++ b/cpp/src/jit/rtc/sha256.cpp @@ -37,7 +37,7 @@ sha256_hash sha256_context::finalize() sha256_hash hash; unsigned int length = 0; CUDF_EXPECTS(EVP_DigestFinal_ex(ectx_, hash.data_, &length) == 1, "EVP_DigestFinal_ex failed"); - CUDF_EXPECTS(length == 64, "Unexpected SHA256 length"); + CUDF_EXPECTS(length == sizeof(sha256_hash::data_), "Unexpected SHA256 length"); EVP_MD const* type = EVP_sha256(); CUDF_EXPECTS(EVP_DigestInit_ex(ectx_, type, nullptr) == 1, "EVP_DigestInit_ex failed"); return hash; diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 1122deedf667..6540de4f8f92 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -37,28 +37,6 @@ jitify2::Kernel get_kernel(std::string const& kernel_name, std::string const& cu .get_kernel(kernel_name, {}, {{"cudf/detail/operation-udf.hpp", cuda_source}}, {"-arch=sm_."}); } -// [ ] load the cudf transformation LTO library; return Culibrary object representing it -// void get_transform_library(); - -// [ ] take CUDA/PTX code as input -// [ ] support different function signature styles -// [ ] handle include directories -// [ ] C++/C-ABI symbol name for locating function to link against -// [ ] take input and output data types as parameters -// [ ] take function_info struct as input; describe: input/output types, null-awareness, user-data -// requirement, etc. -// [ ] return Culibrary object representing compiled operator -// void compile_library(); -// void compile_transform_operator_thunk(); - -// [ ] take LTO-IR compiled binary of operator as input -// [ ] link it into the lto library; return -// void link_transform_thunk(); - -// [ ] if we can, we should not depend on jitify for LTO-IR compilation, linking, and caching; use -// nvJITLink directly -// void cache_transform_thunk(); - jitify2::StringVec build_jit_template_params( null_aware is_null_aware, bool may_evaluate_null, From 30e25b872ac39bd37591503f3dc19fd8c2a58ff2 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 8 Jan 2026 15:58:17 +0000 Subject: [PATCH 013/254] Refactor and Update CUDA JIT Runtime Components - Removed the `null_mask_opt.cuh` file as it is no longer needed. - Updated copyright years in various header files to 2026. - Replaced `` with `` in `assert.cuh`. - Added conditional includes for `` in `fixed_point.hpp` and `temporary.hpp` to avoid compilation issues in certain contexts. - Changed temporary directory naming convention from `jit_includes` to `jit-includes` for consistency. - Introduced new header files in the JIT runtime includes directory to wrap standard library components from `cuda/std`. - Modified logging format in `rtc.cpp` for improved readability. - Updated cache directory naming in `context.cpp` to match the new convention. - Cleaned up test cases in `rtc.cpp` by removing unnecessary whitespace. --- cpp/CMakeLists.txt | 27 +++--- cpp/include/cudf/detail/null_mask_opt.cuh | 90 ------------------- cpp/include/cudf/detail/utilities/assert.cuh | 4 +- cpp/include/cudf/fixed_point/fixed_point.hpp | 9 +- cpp/include/cudf/fixed_point/temporary.hpp | 6 +- cpp/include/cudf/wrappers/dictionary.hpp | 4 +- cpp/src/jit/rtc/cudf.cpp | 4 +- cpp/src/jit/rtc/includes/system/algorithm | 12 +++ cpp/src/jit/rtc/includes/system/array | 12 +++ cpp/src/jit/rtc/includes/system/atomic | 12 +++ cpp/src/jit/rtc/includes/system/barrier | 12 +++ cpp/src/jit/rtc/includes/system/bit | 12 +++ cpp/src/jit/rtc/includes/system/cassert | 12 +++ cpp/src/jit/rtc/includes/system/cassert.h | 10 +++ cpp/src/jit/rtc/includes/system/ccomplex | 12 +++ cpp/src/jit/rtc/includes/system/ccomplex.h | 12 +++ cpp/src/jit/rtc/includes/system/cfloat | 11 +++ cpp/src/jit/rtc/includes/system/cfloat.h | 10 +++ cpp/src/jit/rtc/includes/system/chrono | 12 +++ cpp/src/jit/rtc/includes/system/climits | 11 +++ cpp/src/jit/rtc/includes/system/cmath | 12 +++ cpp/src/jit/rtc/includes/system/cmath.h | 12 +++ cpp/src/jit/rtc/includes/system/complex | 12 +++ cpp/src/jit/rtc/includes/system/concepts | 12 +++ cpp/src/jit/rtc/includes/system/cstddef | 12 +++ cpp/src/jit/rtc/includes/system/cstdint | 12 +++ cpp/src/jit/rtc/includes/system/cstdlib | 12 +++ cpp/src/jit/rtc/includes/system/cstring | 12 +++ cpp/src/jit/rtc/includes/system/ctime | 12 +++ cpp/src/jit/rtc/includes/system/expected | 12 +++ cpp/src/jit/rtc/includes/system/functional | 12 +++ .../jit/rtc/includes/system/initializer_list | 12 +++ .../jit/rtc/includes/system/inplace_vector | 12 +++ cpp/src/jit/rtc/includes/system/iterator | 12 +++ cpp/src/jit/rtc/includes/system/latch | 12 +++ cpp/src/jit/rtc/includes/system/limits | 12 +++ cpp/src/jit/rtc/includes/system/linalg | 12 +++ cpp/src/jit/rtc/includes/system/mdspan | 12 +++ cpp/src/jit/rtc/includes/system/memory | 12 +++ cpp/src/jit/rtc/includes/system/numbers | 12 +++ cpp/src/jit/rtc/includes/system/numeric | 12 +++ cpp/src/jit/rtc/includes/system/optional | 12 +++ cpp/src/jit/rtc/includes/system/ranges | 12 +++ cpp/src/jit/rtc/includes/system/ratio | 12 +++ cpp/src/jit/rtc/includes/system/semaphore | 12 +++ .../jit/rtc/includes/system/source_location | 12 +++ cpp/src/jit/rtc/includes/system/span | 12 +++ cpp/src/jit/rtc/includes/system/stddef.h | 12 +++ cpp/src/jit/rtc/includes/system/stdint.h | 12 +++ cpp/src/jit/rtc/includes/system/stdlib.h | 12 +++ cpp/src/jit/rtc/includes/system/string.h | 12 +++ cpp/src/jit/rtc/includes/system/string_view | 12 +++ cpp/src/jit/rtc/includes/system/time.h | 12 +++ cpp/src/jit/rtc/includes/system/tuple | 12 +++ cpp/src/jit/rtc/includes/system/type_traits | 12 +++ cpp/src/jit/rtc/includes/system/utility | 12 +++ cpp/src/jit/rtc/includes/system/variant | 12 +++ cpp/src/jit/rtc/includes/system/version | 6 ++ cpp/src/jit/rtc/rtc.cpp | 6 +- cpp/src/runtime/context.cpp | 2 +- cpp/tests/jit/rtc.cpp | 1 - 61 files changed, 638 insertions(+), 115 deletions(-) delete mode 100644 cpp/include/cudf/detail/null_mask_opt.cuh create mode 100644 cpp/src/jit/rtc/includes/system/algorithm create mode 100644 cpp/src/jit/rtc/includes/system/array create mode 100644 cpp/src/jit/rtc/includes/system/atomic create mode 100644 cpp/src/jit/rtc/includes/system/barrier create mode 100644 cpp/src/jit/rtc/includes/system/bit create mode 100644 cpp/src/jit/rtc/includes/system/cassert create mode 100644 cpp/src/jit/rtc/includes/system/cassert.h create mode 100644 cpp/src/jit/rtc/includes/system/ccomplex create mode 100644 cpp/src/jit/rtc/includes/system/ccomplex.h create mode 100644 cpp/src/jit/rtc/includes/system/cfloat create mode 100644 cpp/src/jit/rtc/includes/system/cfloat.h create mode 100644 cpp/src/jit/rtc/includes/system/chrono create mode 100644 cpp/src/jit/rtc/includes/system/climits create mode 100644 cpp/src/jit/rtc/includes/system/cmath create mode 100644 cpp/src/jit/rtc/includes/system/cmath.h create mode 100644 cpp/src/jit/rtc/includes/system/complex create mode 100644 cpp/src/jit/rtc/includes/system/concepts create mode 100644 cpp/src/jit/rtc/includes/system/cstddef create mode 100644 cpp/src/jit/rtc/includes/system/cstdint create mode 100644 cpp/src/jit/rtc/includes/system/cstdlib create mode 100644 cpp/src/jit/rtc/includes/system/cstring create mode 100644 cpp/src/jit/rtc/includes/system/ctime create mode 100644 cpp/src/jit/rtc/includes/system/expected create mode 100644 cpp/src/jit/rtc/includes/system/functional create mode 100644 cpp/src/jit/rtc/includes/system/initializer_list create mode 100644 cpp/src/jit/rtc/includes/system/inplace_vector create mode 100644 cpp/src/jit/rtc/includes/system/iterator create mode 100644 cpp/src/jit/rtc/includes/system/latch create mode 100644 cpp/src/jit/rtc/includes/system/limits create mode 100644 cpp/src/jit/rtc/includes/system/linalg create mode 100644 cpp/src/jit/rtc/includes/system/mdspan create mode 100644 cpp/src/jit/rtc/includes/system/memory create mode 100644 cpp/src/jit/rtc/includes/system/numbers create mode 100644 cpp/src/jit/rtc/includes/system/numeric create mode 100644 cpp/src/jit/rtc/includes/system/optional create mode 100644 cpp/src/jit/rtc/includes/system/ranges create mode 100644 cpp/src/jit/rtc/includes/system/ratio create mode 100644 cpp/src/jit/rtc/includes/system/semaphore create mode 100644 cpp/src/jit/rtc/includes/system/source_location create mode 100644 cpp/src/jit/rtc/includes/system/span create mode 100644 cpp/src/jit/rtc/includes/system/stddef.h create mode 100644 cpp/src/jit/rtc/includes/system/stdint.h create mode 100644 cpp/src/jit/rtc/includes/system/stdlib.h create mode 100644 cpp/src/jit/rtc/includes/system/string.h create mode 100644 cpp/src/jit/rtc/includes/system/string_view create mode 100644 cpp/src/jit/rtc/includes/system/time.h create mode 100644 cpp/src/jit/rtc/includes/system/tuple create mode 100644 cpp/src/jit/rtc/includes/system/type_traits create mode 100644 cpp/src/jit/rtc/includes/system/utility create mode 100644 cpp/src/jit/rtc/includes/system/variant create mode 100644 cpp/src/jit/rtc/includes/system/version diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 15ad9215c16e..bd3ddbb11b6a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -368,28 +368,33 @@ jit_add_include_directory( jit_add_include_directory( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src DEST_DIRECTORY cudf/src - INCLUDE_DIRECTORIES cudf/src + INCLUDE_DIRECTORIES cudf/src cudf/src/jit/rtc/includes/system ) # TODO: add library.cu LTO type checks need to be compiled as part of the AOT workflow to ensure -# type consistency between AOT and JIT Must be in test suite +# type consistency between AOT and JIT. +# +# Must be in test suite get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) -foreach(CUDACXX_INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) +foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) + cmake_path(GET INC_DIR FILENAME INC_DIR_NAME) + jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CUDACXX_INC_DIR} DEST_DIRECTORY libcudacxx INCLUDE_DIRECTORIES - libcudacxx + cudf_jit_embed COPY_DIRECTORY ${INC_DIR} DEST_DIRECTORY CCCL/libcudacxx/${INC_DIR_NAME} + INCLUDE_DIRECTORIES CCCL/libcudacxx/${INC_DIR_NAME} ) endforeach() -foreach(CTK_INC_DIR IN LISTS CUDAToolkit_INCLUDE_DIRS) - jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CTK_INC_DIR} DEST_DIRECTORY CUDAToolKit INCLUDE_DIRECTORIES - CUDAToolKit CUDAToolKit/cuda/std - ) -endforeach() +# TODO: remove foreach(INC_DIR IN LISTS CUDAToolkit_INCLUDE_DIRS) message("CTKKK dir: ${INC_DIR}") +# cmake_path(GET INC_DIR FILENAME INC_DIR_NAME) + +# jit_add_include_directory( cudf_jit_embed COPY_DIRECTORY ${INC_DIR} DEST_DIRECTORY +# CUDAToolKit/${INC_DIR_NAME} INCLUDE_DIRECTORIES CUDAToolKit/${INC_DIR_NAME} ) + +# endforeach() jit_add_options(cudf_jit_embed OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) diff --git a/cpp/include/cudf/detail/null_mask_opt.cuh b/cpp/include/cudf/detail/null_mask_opt.cuh deleted file mode 100644 index cbae41d43cca..000000000000 --- a/cpp/include/cudf/detail/null_mask_opt.cuh +++ /dev/null @@ -1,90 +0,0 @@ - -#include -#include - -namespace cudf { -namespace detail { - -// These will be written together to global memory as the output of a null mask operation. -// Assumes output null_mask buffer is a temporary buffer that can be overwritten. -struct null_mask_chunk { - cudf::bitmask_type word; - cudf::size_type index; - cudf::size_type valid_count; - - // num_chunks and src_size must be >= 0 - template - static __device__ null_mask_chunk load(cudf::size_type current_chunk, - BitMaskWordFunc&& src_null_mask, - cudf::size_type src_offset, - cudf::size_type src_size) - { - auto chunk_bit_begin = current_chunk * static_cast(sizeof(bitmask_type) * 8); - constexpr auto num_chunk_bits = static_cast(sizeof(bitmask_type) * 8); - - auto src_bit_begin = src_offset + chunk_bit_begin; - auto const num_src_words = (src_size + (num_chunk_bits - 1)) / num_chunk_bits; - - auto leading_word_index = src_bit_begin / num_chunk_bits; - auto trailing_word_index = (src_bit_begin + (num_chunk_bits - 1)) / num_chunk_bits; - - if (trailing_word_index < num_src_words) [[likely]] { - auto leading_bits = src_null_mask(leading_word_index); - auto trailing_bits = src_null_mask(trailing_word_index); - auto bit_shift = src_bit_begin % num_chunk_bits; - auto merged = (cudf::bitmask_type)__funnelshift_r(leading_bits, trailing_bits, bit_shift); - auto valid_count = (cudf::size_type)__popc(merged); - return null_mask_chunk{merged, current_chunk, valid_count}; - } else { - auto leading_bits = src_null_mask(leading_word_index); - auto bit_shift = src_bit_begin % num_chunk_bits; - auto num_discard_bits = (src_bit_begin + num_chunk_bits) - (src_offset + src_size); - auto mask = (~bitmask_type{0}) >> num_discard_bits; - auto output = (leading_bits >> bit_shift) & mask; - auto valid_count = (cudf::size_type)__popc(output); - return null_mask_chunk{output, current_chunk, valid_count}; - } - } -}; - -template -CUDF_KERNEL void chunked_bitmask_binop(Binop op, - device_span destination, - device_span source, - device_span source_begin_bits, - size_type source_size_bits, - size_type num_chunks, - size_type* valid_count_ptr) -{ - auto const tid = cudf::detail::grid_1d::global_thread_id(); - cudf::size_type valid_count = 0; - - // for(auto i = tid;) - // [ ] can we reduce the number of blocks so we can have more warps per block? - - // [ ] let the CPU sum the valid counts from each block - - //[ ] use duff's device to unroll the sources - // switch(src.size() / 16){ - // [ ] use function to load - // 16 pairs + reduce - // 8 pairs + reduce - // 4 pairs + reduce - // 2 pairs + reduce - // 1 pair + reduce - // } - - using BlockReduce = cub::BlockReduce; - __shared__ typename BlockReduce::TempStorage temp_storage; - size_type block_valid_count = BlockReduce(temp_storage).Sum(valid_count); - - if (threadIdx.x == 0) { atomicAdd(valid_count_ptr, block_valid_count); } -} - -// to embed in transform kernel - -// write output to uint32_t -// once full or at end, write to global memory; using null_mask_chunk - -} // namespace detail -} // namespace cudf diff --git a/cpp/include/cudf/detail/utilities/assert.cuh b/cpp/include/cudf/detail/utilities/assert.cuh index 5ecd00db4fba..077fb7b4fc5b 100644 --- a/cpp/include/cudf/detail/utilities/assert.cuh +++ b/cpp/include/cudf/detail/utilities/assert.cuh @@ -1,11 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2023, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#include +#include /** * @brief `assert`-like macro for device code diff --git a/cpp/include/cudf/fixed_point/fixed_point.hpp b/cpp/include/cudf/fixed_point/fixed_point.hpp index 6dcc4aed20a1..5fde736dc798 100644 --- a/cpp/include/cudf/fixed_point/fixed_point.hpp +++ b/cpp/include/cudf/fixed_point/fixed_point.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -17,7 +17,10 @@ #include #include #include + +#if (!defined(__CUDACC_RTC__)) && !defined(CUDF_RUNTIME_JIT) #include +#endif /// `fixed_point` and supporting types namespace CUDF_EXPORT numeric { @@ -570,6 +573,8 @@ class fixed_point { return fixed_point{scaled_integer{value, scale}}; } +#if (!defined(__CUDACC_RTC__)) && !defined(CUDF_RUNTIME_JIT) + /** * @brief Returns a string representation of the fixed_point value. */ @@ -589,6 +594,8 @@ class fixed_point { auto const zeros = std::string(_scale, '0'); return detail::to_string(_value) + zeros; } + +#endif }; /** diff --git a/cpp/include/cudf/fixed_point/temporary.hpp b/cpp/include/cudf/fixed_point/temporary.hpp index 097f0cb061d8..7ef615c5192f 100644 --- a/cpp/include/cudf/fixed_point/temporary.hpp +++ b/cpp/include/cudf/fixed_point/temporary.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,12 +10,15 @@ #include #include +#if (!defined(__CUDACC_RTC__)) && !defined(CUDF_RUNTIME_JIT) #include #include +#endif namespace CUDF_EXPORT numeric { namespace detail { +#if (!defined(__CUDACC_RTC__)) && !defined(CUDF_RUNTIME_JIT) template auto to_string(T value) -> std::string { @@ -41,6 +44,7 @@ auto to_string(T value) -> std::string } return std::string{}; // won't ever hit here, need to suppress warning though } +#endif template CUDF_HOST_DEVICE constexpr auto abs(T value) diff --git a/cpp/include/cudf/wrappers/dictionary.hpp b/cpp/include/cudf/wrappers/dictionary.hpp index 9e3e8a0acdc6..73a95ed7b249 100644 --- a/cpp/include/cudf/wrappers/dictionary.hpp +++ b/cpp/include/cudf/wrappers/dictionary.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,8 +7,6 @@ #include -#include - #include /** diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 0acb34a3a876..73d69871846d 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -146,7 +146,7 @@ void copy_includes_to_dir(char const* dir) void create_new_include_dir(std::string const& include_path) { // directory does not exist, so create it - char tmp_dir_data[] = "/tmp/jit_includes_XXXXXX"; + char tmp_dir_data[] = "/tmp/jit-includes_XXXXXX"; char* tmp_dir = mkdtemp(tmp_dir_data); if (tmp_dir == nullptr) { throw_posix( @@ -233,7 +233,7 @@ fragment_t const& compile_fragment(char const* name, char const* source_code_cst cache.store_fragment(cache_key_sha256, fut); auto cache_dir = cache.get_cache_dir(); - auto include_dir = std::format("{}/jit_includes", cache_dir); + auto include_dir = std::format("{}/jit-includes", cache_dir); create_include_dir(include_dir); diff --git a/cpp/src/jit/rtc/includes/system/algorithm b/cpp/src/jit/rtc/includes/system/algorithm new file mode 100644 index 000000000000..6f7d2aefd31f --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/algorithm @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/array b/cpp/src/jit/rtc/includes/system/array new file mode 100644 index 000000000000..3b264123dacd --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/array @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/atomic b/cpp/src/jit/rtc/includes/system/atomic new file mode 100644 index 000000000000..5c2dc1d81400 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/atomic @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/barrier b/cpp/src/jit/rtc/includes/system/barrier new file mode 100644 index 000000000000..95cd35657daa --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/barrier @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/bit b/cpp/src/jit/rtc/includes/system/bit new file mode 100644 index 000000000000..493dc2115366 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/bit @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/cassert b/cpp/src/jit/rtc/includes/system/cassert new file mode 100644 index 000000000000..1e2b85df5dc4 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/cassert @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + + +} diff --git a/cpp/src/jit/rtc/includes/system/cassert.h b/cpp/src/jit/rtc/includes/system/cassert.h new file mode 100644 index 000000000000..c7d847aafdc4 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/cassert.h @@ -0,0 +1,10 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std { + +} diff --git a/cpp/src/jit/rtc/includes/system/ccomplex b/cpp/src/jit/rtc/includes/system/ccomplex new file mode 100644 index 000000000000..dbb5e678e49f --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/ccomplex @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/ccomplex.h b/cpp/src/jit/rtc/includes/system/ccomplex.h new file mode 100644 index 000000000000..a2eded0c8289 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/ccomplex.h @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std { + +using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/cfloat b/cpp/src/jit/rtc/includes/system/cfloat new file mode 100644 index 000000000000..ac41c2724b44 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/cfloat @@ -0,0 +1,11 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + +} diff --git a/cpp/src/jit/rtc/includes/system/cfloat.h b/cpp/src/jit/rtc/includes/system/cfloat.h new file mode 100644 index 000000000000..0c23d7ca72f4 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/cfloat.h @@ -0,0 +1,10 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std { + +} diff --git a/cpp/src/jit/rtc/includes/system/chrono b/cpp/src/jit/rtc/includes/system/chrono new file mode 100644 index 000000000000..63b2e7e4a183 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/chrono @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/climits b/cpp/src/jit/rtc/includes/system/climits new file mode 100644 index 000000000000..a555fb339dfa --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/climits @@ -0,0 +1,11 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + +} diff --git a/cpp/src/jit/rtc/includes/system/cmath b/cpp/src/jit/rtc/includes/system/cmath new file mode 100644 index 000000000000..8eff37d3a688 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/cmath @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/cmath.h b/cpp/src/jit/rtc/includes/system/cmath.h new file mode 100644 index 000000000000..97ed927ef2a9 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/cmath.h @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std { + +using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/complex b/cpp/src/jit/rtc/includes/system/complex new file mode 100644 index 000000000000..8b3855b16825 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/complex @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/concepts b/cpp/src/jit/rtc/includes/system/concepts new file mode 100644 index 000000000000..fadf29d68552 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/concepts @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/cstddef b/cpp/src/jit/rtc/includes/system/cstddef new file mode 100644 index 000000000000..7e234744ace1 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/cstddef @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/cstdint b/cpp/src/jit/rtc/includes/system/cstdint new file mode 100644 index 000000000000..f5bdc5e318e9 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/cstdint @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/cstdlib b/cpp/src/jit/rtc/includes/system/cstdlib new file mode 100644 index 000000000000..d4c5a263926b --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/cstdlib @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/cstring b/cpp/src/jit/rtc/includes/system/cstring new file mode 100644 index 000000000000..caec3d64ae06 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/cstring @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/ctime b/cpp/src/jit/rtc/includes/system/ctime new file mode 100644 index 000000000000..eb245dd5de63 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/ctime @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/expected b/cpp/src/jit/rtc/includes/system/expected new file mode 100644 index 000000000000..32553c615b9c --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/expected @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/functional b/cpp/src/jit/rtc/includes/system/functional new file mode 100644 index 000000000000..c2d1afd9da3a --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/functional @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/initializer_list b/cpp/src/jit/rtc/includes/system/initializer_list new file mode 100644 index 000000000000..f69a6c36c12c --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/initializer_list @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/inplace_vector b/cpp/src/jit/rtc/includes/system/inplace_vector new file mode 100644 index 000000000000..dc88a71ce59e --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/inplace_vector @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/iterator b/cpp/src/jit/rtc/includes/system/iterator new file mode 100644 index 000000000000..cdaa28d88d47 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/iterator @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/latch b/cpp/src/jit/rtc/includes/system/latch new file mode 100644 index 000000000000..906df15bf987 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/latch @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/limits b/cpp/src/jit/rtc/includes/system/limits new file mode 100644 index 000000000000..58aad6743f75 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/limits @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/linalg b/cpp/src/jit/rtc/includes/system/linalg new file mode 100644 index 000000000000..0db2c0558620 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/linalg @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/mdspan b/cpp/src/jit/rtc/includes/system/mdspan new file mode 100644 index 000000000000..6634dbe85bb6 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/mdspan @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/memory b/cpp/src/jit/rtc/includes/system/memory new file mode 100644 index 000000000000..bb8634824332 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/memory @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/numbers b/cpp/src/jit/rtc/includes/system/numbers new file mode 100644 index 000000000000..90c49d6159f3 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/numbers @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/numeric b/cpp/src/jit/rtc/includes/system/numeric new file mode 100644 index 000000000000..501c41b53141 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/numeric @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/optional b/cpp/src/jit/rtc/includes/system/optional new file mode 100644 index 000000000000..221670774c11 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/optional @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/ranges b/cpp/src/jit/rtc/includes/system/ranges new file mode 100644 index 000000000000..b2e1e4b88f60 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/ranges @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/ratio b/cpp/src/jit/rtc/includes/system/ratio new file mode 100644 index 000000000000..c7a79cf3be4b --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/ratio @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/semaphore b/cpp/src/jit/rtc/includes/system/semaphore new file mode 100644 index 000000000000..a27c6a001d80 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/semaphore @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/source_location b/cpp/src/jit/rtc/includes/system/source_location new file mode 100644 index 000000000000..d8d4ffdd51f3 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/source_location @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/span b/cpp/src/jit/rtc/includes/system/span new file mode 100644 index 000000000000..37a20bfe8422 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/span @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/stddef.h b/cpp/src/jit/rtc/includes/system/stddef.h new file mode 100644 index 000000000000..71352d667749 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/stddef.h @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std { + +using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/stdint.h b/cpp/src/jit/rtc/includes/system/stdint.h new file mode 100644 index 000000000000..03c8eb0a9dd1 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/stdint.h @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std { + +using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/stdlib.h b/cpp/src/jit/rtc/includes/system/stdlib.h new file mode 100644 index 000000000000..c216518c9c29 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/stdlib.h @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std { + +using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/string.h b/cpp/src/jit/rtc/includes/system/string.h new file mode 100644 index 000000000000..83275541eb0a --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/string.h @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std { + +using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/string_view b/cpp/src/jit/rtc/includes/system/string_view new file mode 100644 index 000000000000..2571122e34d4 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/string_view @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/time.h b/cpp/src/jit/rtc/includes/system/time.h new file mode 100644 index 000000000000..05e4eab7bd9f --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/time.h @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std { + +using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/tuple b/cpp/src/jit/rtc/includes/system/tuple new file mode 100644 index 000000000000..986ab80ee5a8 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/tuple @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/type_traits b/cpp/src/jit/rtc/includes/system/type_traits new file mode 100644 index 000000000000..63e4f60aebf1 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/type_traits @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/utility b/cpp/src/jit/rtc/includes/system/utility new file mode 100644 index 000000000000..db465bfdd181 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/utility @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/variant b/cpp/src/jit/rtc/includes/system/variant new file mode 100644 index 000000000000..cefd94b3e8a7 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/variant @@ -0,0 +1,12 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include + +namespace std{ + + using namespace ::cuda::std; + +} diff --git a/cpp/src/jit/rtc/includes/system/version b/cpp/src/jit/rtc/includes/system/version new file mode 100644 index 000000000000..56a7a1d15cec --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/version @@ -0,0 +1,6 @@ + /* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once +#include diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 0967918d05be..e4754347dd47 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -151,7 +151,7 @@ std::string get_nvrtc_log(fragment_t::compile_params const& params, options_str = std::format("{}\t{}\n", options_str, option); } - return std::format("NCRTC Compilation for {} {} ({}): {}.\nHeaders: {}\nOptions:\t{}\n\n{}", + return std::format("NCRTC Compilation for {} {} ({}): {}.\nHeaders:\n{}\n\nOptions:\n{}\n\n{}", params.name == nullptr ? "" : params.name, status_str, static_cast(compile_result), @@ -195,8 +195,8 @@ void log_nvJitLink_link_result(library_t::link_params const& params, (link_result == NVJITLINK_SUCCESS) ? "completed successfully" : "failed with error"; auto str = std::format( - "nvJitLink Linking for {} ({}) {} ({}): {}.\nFragments: {}\n" - "Link Options: {}\n\nInfo Log:\n{}\n\nError Log:\n{}", + "nvJitLink Linking for {} ({}) {} ({}): {}.\nFragments: \n{}\n" + "Link Options: \n{}\n\nInfo Log:\n{}\n\nError Log:\n{}\n\n", params.name == nullptr ? "" : params.name, binary_type_str, status_str, diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 51e935a81f0b..417950c37130 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -50,7 +50,7 @@ void context::initialize_components(init_flags flags) if (has_flag(new_flags, init_flags::INIT_JIT_CACHE)) { _program_cache = std::make_unique(); // TODO: Make cache directory configurable - _rtc_cache = std::make_unique(true, "/tmp/cudf_rtc_cache", rtc::cache_limits{}); + _rtc_cache = std::make_unique(true, "/tmp/cudf-rtc-cache", rtc::cache_limits{}); } if (has_flag(new_flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index e2968a71ef73..2c633beb5cd5 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -21,7 +21,6 @@ TEST_F(RTCTest, CreateFragment) "transform_kernel", "test_kernel_key", R"***( - #include "cudf/jit/transform_params.cuh" #include "cudf/jit/lto/operators.inl.cuh" #include "cudf/jit/lto/types.inl.cuh" From 46e84efde08886f2c6a435ced12ff35468f697dd Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 9 Jan 2026 04:30:34 +0000 Subject: [PATCH 014/254] Refactor JIT operators and error handling - Updated the declaration of JIT operators in `operators.cuh` to use inline functions for better optimization. - Modified the `thunk.cuh` file to streamline the lower and lift functions for type conversions. - Enhanced the `string_view` structure in `types.cuh` by adding inline specifiers to member functions for lto-linking - Adjusted the error handling macros in `error.hpp` to provide clearer error messages. - Improved logging in the RTC compilation process in `rtc.cpp` to include more detailed information about NVRTC and nvJitLink results. - Updated test cases to reflect changes in header file paths for JIT operators and types. --- cpp/include/cudf/jit/lto/operators.cuh | 77 +++--- cpp/include/cudf/jit/lto/thunk.cuh | 16 +- cpp/include/cudf/jit/lto/types.cuh | 116 ++++----- cpp/include/cudf/strings/string_view.hpp | 4 +- cpp/include/cudf/utilities/error.hpp | 4 +- cpp/src/jit/row_ir.cpp | 3 +- cpp/src/jit/rtc/cudf.cpp | 46 +++- cpp/src/jit/rtc/rtc.cpp | 316 ++++++++++++++--------- cpp/tests/jit/rtc.cpp | 6 +- 9 files changed, 356 insertions(+), 232 deletions(-) diff --git a/cpp/include/cudf/jit/lto/operators.cuh b/cpp/include/cudf/jit/lto/operators.cuh index 911ac51bed38..49445d418ea6 100644 --- a/cpp/include/cudf/jit/lto/operators.cuh +++ b/cpp/include/cudf/jit/lto/operators.cuh @@ -11,10 +11,11 @@ namespace lto { namespace operators { -#define CUDF_LTO_DECL(op, type) \ - extern __device__ void op(type* out, type const* a, type const* b); \ - \ - extern __device__ void op(optional* out, optional const* a, optional const* b) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern inline void op(type* out, type const* a, type const* b); \ + \ + __device__ extern inline void op( \ + optional* out, optional const* a, optional const* b) CUDF_LTO_DECL(add, int32_t); CUDF_LTO_DECL(add, int64_t); @@ -75,12 +76,13 @@ CUDF_LTO_DECL(pow, float64_t); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - extern __device__ void op(bool* out, type const* a, type const* b); \ - \ - extern __device__ void op(bool* out, optional const* a, optional const* b); \ - \ - extern __device__ void op(optional* out, optional const* a, optional const* b) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern inline void op(bool* out, type const* a, type const* b); \ + \ + __device__ extern inline void op(bool* out, optional const* a, optional const* b); \ + \ + __device__ extern inline void op( \ + optional* out, optional const* a, optional const* b) CUDF_LTO_DECL(equal, bool); CUDF_LTO_DECL(equal, int8_t); @@ -134,10 +136,11 @@ CUDF_LTO_DECL(null_equal, string_view); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - extern __device__ void op(bool* out, type const* a, type const* b); \ - \ - extern __device__ void op(optional* out, optional const* a, optional const* b) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern inline void op(bool* out, type const* a, type const* b); \ + \ + __device__ extern inline void op( \ + optional* out, optional const* a, optional const* b) CUDF_LTO_DECL(less, bool); CUDF_LTO_DECL(less, int8_t); @@ -241,10 +244,11 @@ CUDF_LTO_DECL(greater_equal, string_view); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - extern __device__ void op(type* out, type const* a, type const* b); \ - \ - extern __device__ void op(optional* out, optional const* a, optional const* b) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern inline void op(type* out, type const* a, type const* b); \ + \ + __device__ extern inline void op( \ + optional* out, optional const* a, optional const* b) CUDF_LTO_DECL(bitwise_and, int32_t); CUDF_LTO_DECL(bitwise_and, int64_t); @@ -263,10 +267,11 @@ CUDF_LTO_DECL(bitwise_xor, uint64_t); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - extern __device__ void op(type* out, type const* a, type const* b); \ - \ - extern __device__ void op(optional* out, optional const* a, optional const* b); +#define CUDF_LTO_DECL(op, type) \ + __device__ extern inline void op(type* out, type const* a, type const* b); \ + \ + __device__ extern inline void op( \ + optional* out, optional const* a, optional const* b); CUDF_LTO_DECL(logical_and, bool); @@ -278,10 +283,10 @@ CUDF_LTO_DECL(null_logical_or, bool); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - extern __device__ void op(type* out, type const* a); \ - \ - extern __device__ void op(optional* out, optional const* a) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern inline void op(type* out, type const* a); \ + \ + __device__ extern inline void op(optional* out, optional const* a) CUDF_LTO_DECL(identity, bool); CUDF_LTO_DECL(identity, int8_t); @@ -374,10 +379,10 @@ CUDF_LTO_DECL(bit_invert, int64_t); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, ret_type, type) \ - extern __device__ void op(ret_type* out, type const* a); \ - \ - extern __device__ void op(optional* out, optional const* a) +#define CUDF_LTO_DECL(op, ret_type, type) \ + __device__ extern inline void op(ret_type* out, type const* a); \ + \ + __device__ extern inline void op(optional* out, optional const* a) CUDF_LTO_DECL(cast_to_int64, int64_t, bool); CUDF_LTO_DECL(cast_to_int64, int64_t, int8_t); @@ -417,12 +422,12 @@ CUDF_LTO_DECL(cast_to_float64, float64_t, float64_t); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - extern __device__ void op(bool* out, type const* a); \ - \ - extern __device__ void op(bool* out, optional const* a); \ - \ - extern __device__ void op(optional* out, optional const* a) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern inline void op(bool* out, type const* a); \ + \ + __device__ extern inline void op(bool* out, optional const* a); \ + \ + __device__ extern inline void op(optional* out, optional const* a) CUDF_LTO_DECL(is_null, bool); CUDF_LTO_DECL(is_null, int8_t); diff --git a/cpp/include/cudf/jit/lto/thunk.cuh b/cpp/include/cudf/jit/lto/thunk.cuh index 36f547c4a372..699ae1388fe2 100644 --- a/cpp/include/cudf/jit/lto/thunk.cuh +++ b/cpp/include/cudf/jit/lto/thunk.cuh @@ -51,45 +51,45 @@ using lifted_type_of = typename lifted_type_of_t::type; using type = lowered_type; \ }; \ \ - __forceinline__ __device__ lowered_type* lower(lifted_type* p) \ + __device__ __forceinline__ lowered_type* lower(lifted_type* p) \ { \ return reinterpret_cast(p); \ } \ \ - __forceinline__ __device__ lowered_type const* lower(lifted_type const* p) \ + __device__ __forceinline__ lowered_type const* lower(lifted_type const* p) \ { \ return reinterpret_cast(p); \ } \ \ - __forceinline__ __device__ lto::optional* lower( \ + __device__ __forceinline__ lto::optional* lower( \ cuda::std::optional* p) \ { \ return reinterpret_cast*>(p); \ } \ \ - __forceinline__ __device__ lto::optional const* lower( \ + __device__ __forceinline__ lto::optional const* lower( \ cuda::std::optional const* p) \ { \ return reinterpret_cast const*>(p); \ } \ \ - __forceinline__ __device__ lifted_type* lift(lowered_type* p) \ + __device__ __forceinline__ lifted_type* lift(lowered_type* p) \ { \ return reinterpret_cast(p); \ } \ \ - __forceinline__ __device__ lifted_type const* lift(lowered_type const* p) \ + __device__ __forceinline__ lifted_type const* lift(lowered_type const* p) \ { \ return reinterpret_cast(p); \ } \ \ - __forceinline__ __device__ cuda::std::optional* lift( \ + __device__ __forceinline__ cuda::std::optional* lift( \ lto::optional* p) \ { \ return reinterpret_cast*>(p); \ } \ \ - __forceinline__ __device__ cuda::std::optional const* lift( \ + __device__ __forceinline__ cuda::std::optional const* lift( \ lto::optional const* p) \ { \ return reinterpret_cast const*>(p); \ diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index 1e04c449199a..fb4a47ebc39c 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -63,61 +63,61 @@ struct CUDF_LTO_ALIAS string_view { mutable size_type __length = 0; public: - [[nodiscard]] __device__ size_type size_bytes() const; + [[nodiscard]] __device__ inline size_type size_bytes() const; - [[nodiscard]] __device__ size_type length() const; + [[nodiscard]] __device__ inline size_type length() const; - [[nodiscard]] __device__ char const* data() const; + [[nodiscard]] __device__ inline char const* data() const; - [[nodiscard]] __device__ bool empty() const; + [[nodiscard]] __device__ inline bool empty() const; - __device__ char_utf8 operator[](size_type pos) const; + [[nodiscard]] __device__ inline char_utf8 operator[](size_type pos) const; - [[nodiscard]] __device__ size_type byte_offset(size_type pos) const; + [[nodiscard]] __device__ inline size_type byte_offset(size_type pos) const; - [[nodiscard]] __device__ int compare(string_view const& str) const; + [[nodiscard]] __device__ inline int compare(string_view const& str) const; - __device__ int compare(char const* str, size_type bytes) const; + [[nodiscard]] __device__ inline int compare(char const* str, size_type bytes) const; - __device__ bool operator==(string_view const& rhs) const; + [[nodiscard]] __device__ inline bool operator==(string_view const& rhs) const; - __device__ bool operator!=(string_view const& rhs) const; + [[nodiscard]] __device__ inline bool operator!=(string_view const& rhs) const; - __device__ bool operator<(string_view const& rhs) const; + [[nodiscard]] __device__ inline bool operator<(string_view const& rhs) const; - __device__ bool operator>(string_view const& rhs) const; + [[nodiscard]] __device__ inline bool operator>(string_view const& rhs) const; - __device__ bool operator<=(string_view const& rhs) const; + [[nodiscard]] __device__ inline bool operator<=(string_view const& rhs) const; - __device__ bool operator>=(string_view const& rhs) const; + [[nodiscard]] __device__ inline bool operator>=(string_view const& rhs) const; - [[nodiscard]] __device__ size_type find(string_view const& str, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ inline size_type find(string_view const& str, + size_type pos = 0, + size_type count = -1) const; - __device__ size_type find(char const* str, - size_type bytes, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ inline size_type find(char const* str, + size_type bytes, + size_type pos = 0, + size_type count = -1) const; - [[nodiscard]] __device__ size_type find(char_utf8 character, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ inline size_type find(char_utf8 character, + size_type pos = 0, + size_type count = -1) const; - [[nodiscard]] __device__ size_type rfind(string_view const& str, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ inline size_type rfind(string_view const& str, + size_type pos = 0, + size_type count = -1) const; - __device__ size_type rfind(char const* str, - size_type bytes, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ inline size_type rfind(char const* str, + size_type bytes, + size_type pos = 0, + size_type count = -1) const; - [[nodiscard]] __device__ size_type rfind(char_utf8 character, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ inline size_type rfind(char_utf8 character, + size_type pos = 0, + size_type count = -1) const; - [[nodiscard]] __device__ string_view substr(size_type start, size_type length) const; + [[nodiscard]] __device__ inline string_view substr(size_type start, size_type length) const; static inline size_type const npos{-1}; }; @@ -282,30 +282,30 @@ struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { public: template - __device__ T const* head() const; + __device__ inline T const* head() const; - __device__ size_type size() const; + __device__ inline size_type size() const; - __device__ bool nullable() const; + __device__ inline bool nullable() const; - __device__ bitmask_type const* null_mask() const; + __device__ inline bitmask_type const* null_mask() const; - __device__ size_type offset() const; + __device__ inline size_type offset() const; - __device__ bool is_valid(size_type idx) const; + __device__ inline bool is_valid(size_type idx) const; - __device__ bool is_valid_nocheck(size_type idx) const; + __device__ inline bool is_valid_nocheck(size_type idx) const; - __device__ bool is_null(size_type idx) const; + __device__ inline bool is_null(size_type idx) const; - __device__ bool is_null_nocheck(size_type idx) const; + __device__ inline bool is_null_nocheck(size_type idx) const; - __device__ bitmask_type get_mask_word(size_type word_index) const; + __device__ inline bitmask_type get_mask_word(size_type word_index) const; template - __device__ T element(size_type idx) const; + __device__ inline T element(size_type idx) const; - __device__ size_type num_child_columns() const; + __device__ inline size_type num_child_columns() const; }; #define CUDF_LTO_DECL(Type) \ @@ -386,28 +386,28 @@ struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { public: template - __device__ T* head() const; + __device__ inline T* head() const; - __device__ size_type size() const; + __device__ inline size_type size() const; - __device__ bool nullable() const; + __device__ inline bool nullable() const; - __device__ bitmask_type* null_mask() const; + __device__ inline bitmask_type* null_mask() const; - __device__ size_type offset() const; + __device__ inline size_type offset() const; - __device__ bool is_valid(size_type idx) const; + __device__ inline bool is_valid(size_type idx) const; - __device__ bool is_valid_nocheck(size_type idx) const; + __device__ inline bool is_valid_nocheck(size_type idx) const; - __device__ bool is_null(size_type idx) const; + __device__ inline bool is_null(size_type idx) const; - __device__ bool is_null_nocheck(size_type idx) const; + __device__ inline bool is_null_nocheck(size_type idx) const; - __device__ bitmask_type get_mask_word(size_type word_index) const; + __device__ inline bitmask_type get_mask_word(size_type word_index) const; template - __device__ T element(size_type idx) const; + __device__ inline T element(size_type idx) const; }; #define CUDF_LTO_DECL(Type) \ diff --git a/cpp/include/cudf/strings/string_view.hpp b/cpp/include/cudf/strings/string_view.hpp index 2a1db81ec7a3..dfaf54e4f415 100644 --- a/cpp/include/cudf/strings/string_view.hpp +++ b/cpp/include/cudf/strings/string_view.hpp @@ -1,12 +1,14 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include +#if !defined(__CUDACC_RTC__) && !defined(CUDF_RUNTIME_JIT) #include +#endif #include diff --git a/cpp/include/cudf/utilities/error.hpp b/cpp/include/cudf/utilities/error.hpp index abf2f80330c8..f924af36943b 100644 --- a/cpp/include/cudf/utilities/error.hpp +++ b/cpp/include/cudf/utilities/error.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -189,7 +189,7 @@ struct data_type_error : std::invalid_argument { #define CUDF_FAIL_2(_what, _exception_type) \ /*NOLINTNEXTLINE(bugprone-macro-parentheses)*/ \ - throw _exception_type { "CUDF failure at:" __FILE__ ":" CUDF_STRINGIFY(__LINE__) ": " _what } + throw _exception_type { "CUDF failure at: " __FILE__ ":" CUDF_STRINGIFY(__LINE__) ": " _what } #define CUDF_FAIL_1(_what) CUDF_FAIL_2(_what, cudf::logic_error) diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index 7f6af7e115de..9b21d6d12731 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -11,7 +11,6 @@ #include #include -#include #include namespace cudf { diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 73d69871846d..7d47e51af052 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include #include #include @@ -26,6 +27,8 @@ namespace cudf { namespace rtc { +namespace { + int32_t get_driver_version() { int32_t driver_version; @@ -54,10 +57,12 @@ int32_t get_current_device_physical_model() return props.major * 10 + props.minor; } +/* void max_occupancy_config() { + CUDF_FAIL("Not implemented", std::logic_error); // TODO: Same as configure_1d_max_occupancy -} +}*/ sha256_hash hash_string(std::span input) { @@ -126,6 +131,8 @@ std::vector read_file(char const* path) void copy_includes_to_dir(char const* dir) { + CUDF_FUNC_RANGE(); + for (size_t i = 0; i < cudf_jit_embed_sources_file_data.size; ++i) { auto const data = cudf_jit_embed_sources_file_data.elements[i]; auto const data_size = cudf_jit_embed_sources_file_data.element_sizes[i]; @@ -145,6 +152,8 @@ void copy_includes_to_dir(char const* dir) void create_new_include_dir(std::string const& include_path) { + CUDF_FUNC_RANGE(); + // directory does not exist, so create it char tmp_dir_data[] = "/tmp/jit-includes_XXXXXX"; char* tmp_dir = mkdtemp(tmp_dir_data); @@ -156,6 +165,8 @@ void create_new_include_dir(std::string const& include_path) copy_includes_to_dir(tmp_dir); + // [ ] use flockdir and use locks when accessing? + // rename the temporary directory to the target include_path if (rename(tmp_dir, include_path.c_str()) == -1) { throw_posix( @@ -166,6 +177,8 @@ void create_new_include_dir(std::string const& include_path) void create_include_dir(std::string const& include_path) { + CUDF_FUNC_RANGE(); + struct stat path_info; if (lstat(include_path.c_str(), &path_info) == -1) { @@ -180,6 +193,7 @@ void create_include_dir(std::string const& include_path) CUDF_FAIL(+std::format("RTC include path ({}) exists but is not a directory", include_path), std::runtime_error); } else { + // TODO: address, should not error out here // verify contents match expected headers auto hash_path = std::format("{}/state.hash", include_path); auto hash_data = read_file(hash_path.c_str()); @@ -196,8 +210,12 @@ void create_include_dir(std::string const& include_path) } } +} // namespace + fragment_t const& compile_fragment(char const* name, char const* source_code_cstr, char const* key) { + CUDF_FUNC_RANGE(); + auto sm = get_current_device_physical_model(); auto const cache_key = std::format(R"***( fragment_type=LTO_IR, @@ -274,6 +292,8 @@ fragment_t const& compile_fragment(char const* name, char const* source_code_cst fragment_t const& compile_library_fragment() { + CUDF_FUNC_RANGE(); + return compile_fragment("cudf_lto_library", R"***( #include "jit/lto/library.inl.cuh" @@ -283,6 +303,8 @@ fragment_t const& compile_library_fragment() fragment_t const& compile_udf_fragment(char const* source_code_cstr, char const* key) { + CUDF_FUNC_RANGE(); + return compile_fragment("cudf_udf_fragment", source_code_cstr, key); } @@ -292,6 +314,8 @@ kernel_ref compile_and_link_udf(char const* name, char const* udf_code, char const* udf_key) { + CUDF_FUNC_RANGE(); + auto sm = get_current_device_physical_model(); auto library_key = std::format(R"***( fragment_types=LTO_IR, @@ -334,14 +358,28 @@ kernel_ref compile_and_link_udf(char const* name, auto fut = std::shared_future{prom.get_future()}; cache.store_library(library_key_sha256, fut); - blob_view const link_fragments[] = {library_frag.get_cubin()->view(), - udf_frag.get_cubin()->view()}; + blob_view const link_fragments[] = {library_frag.get_lto_ir()->view(), + udf_frag.get_lto_ir()->view()}; binary_type const fragment_binary_types[] = {binary_type::LTO_IR, binary_type::LTO_IR}; char const* const fragment_names[] = {"cudf_lto_library", "cudf_udf_fragment"}; + // TODO: optimization flags + // TODO: split compile + // TODO: split-compile-extended + // TODO: lineinfo and debug info options + // TODO: -kernels-used= + // TODO: sass dump + // TODO: time dump + // TODO: env variable to control options + // TODO: fma + // TODO: variables-used + // TODO: -optimize-unused-variables + // TODO: -nocache + // TODO: -device-stack-protector auto arch_flag = std::format("-arch=sm_{}", sm); - char const* const link_options[] = {"-lto", arch_flag.c_str()}; + char const* const link_options[] = { + "-lto", "-optimize-unused-variables", "-kernels-used=transform_kernel", arch_flag.c_str()}; auto const params = library_t::link_params{.name = name, .output_type = binary_type::CUBIN, diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index e4754347dd47..63840bc5103c 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -23,50 +23,51 @@ #define CUDFRTC_CONCATENATE_DETAIL(x, y) x##y #define CUDFRTC_CONCATENATE(x, y) CUDFRTC_CONCATENATE_DETAIL(x, y) -#define CUDFRTC_CHECK_CUDA(...) \ - do { \ - ::CUresult result = (__VA_ARGS__); \ - if (result != CUDA_SUCCESS) { \ - char const* enum_str; \ - CUDF_EXPECTS(::cuGetErrorString(result, &enum_str) == CUDA_SUCCESS, \ - "Unable to get CUDA error string"); \ - auto errstr = ::std::format("CUDA Call {} failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast(result), \ - enum_str); \ - CUDF_FAIL(+errstr, ::std::runtime_error); \ - } \ +#define CUDFRTC_CHECK_CUDA(...) \ + do { \ + ::CUresult __result = (__VA_ARGS__); \ + if (__result != ::CUDA_SUCCESS) { \ + char const* __enum_str; \ + CUDF_EXPECTS(::cuGetErrorString(__result, &__enum_str) == ::CUDA_SUCCESS, \ + "Unable to get CUDA error string"); \ + auto __errstr = ::std::format("(cuda) Call {} failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::int64_t>(__result), \ + __enum_str); \ + CUDF_FAIL(+__errstr, ::std::runtime_error); \ + } \ } while (0) -#define CUDFRTC_CHECK_NVRTC(params, program, ...) \ - do { \ - ::nvrtcResult result = (__VA_ARGS__); \ - if (result != NVRTC_SUCCESS) { \ - auto errstr = ::std::format("NVRTC Call {} failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast(result), \ - ::nvrtcGetErrorString(result)); \ - CUDF_FAIL(+errstr + "\n" + \ - (program == nullptr ? "" : ::cudf::rtc::get_nvrtc_log(params, program, result)), \ - ::std::runtime_error); \ - } \ - } while (0) - -#define CUDFRTC_CHECK_NVJITLINK(...) \ +#define CUDFRTC_CHECK_NVRTC(params, program, ...) \ do { \ - ::nvJitLinkResult result = (__VA_ARGS__); \ - if (result != NVJITLINK_SUCCESS) { \ - auto errstr = ::std::format("nvJitLink Call {} failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast(result), \ - ::cudf::rtc::get_nvJitLinkResultString(result)); \ - CUDF_FAIL(+errstr, std::runtime_error); \ + ::nvrtcResult __result = (__VA_ARGS__); \ + ::cudf::rtc::log_nvrtc_result(params, program, __result); \ + if (__result != ::NVRTC_SUCCESS) { \ + auto __errstr = ::std::format("(nvrtc) Call {} failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::int64_t>(__result), \ + ::nvrtcGetErrorString(__result)); \ + CUDF_FAIL(+__errstr, ::std::runtime_error); \ } \ } while (0) +#define CUDFRTC_CHECK_NVJITLINK(params, handle, ...) \ + do { \ + ::nvJitLinkResult __result = (__VA_ARGS__); \ + ::cudf::rtc::log_nvJitLink_result(params, handle, __result); \ + if (__result != ::NVJITLINK_SUCCESS) { \ + auto __errstr = ::std::format("(nvJitLink) Call {} failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::int64_t>(__result), \ + ::cudf::rtc::get_nvJitLinkResultString(__result)); \ + CUDF_FAIL(+__errstr, ::std::runtime_error); \ + } \ + } while (0) + namespace cudf { namespace rtc { +namespace { char const* get_nvJitLinkResultString(nvJitLinkResult result) { switch (result) { @@ -89,7 +90,10 @@ char const* get_nvJitLinkResultString(nvJitLinkResult result) case NVJITLINK_ERROR_UNRECOGNIZED_ARCH: return "NVJITLINK_ERROR_UNRECOGNIZED_ARCH"; case NVJITLINK_ERROR_UNSUPPORTED_ARCH: return "NVJITLINK_ERROR_UNSUPPORTED_ARCH"; case NVJITLINK_ERROR_LTO_NOT_ENABLED: return "NVJITLINK_ERROR_LTO_NOT_ENABLED"; - default: CUDF_FAIL("Unrecognized nvJitLinkResult type", std::runtime_error); + default: + CUDF_FAIL( + +std::format("Unrecognized nvJitLinkResult type: ({})", static_cast(result)), + std::runtime_error); } } @@ -100,46 +104,56 @@ char const* binary_type_string(binary_type type) case binary_type::CUBIN: return "CUBIN"; case binary_type::FATBIN: return "FATBIN"; case binary_type::PTX: return "PTX"; - default: CUDF_FAIL("Unrecognized binary_type", std::runtime_error); + default: + CUDF_FAIL(+std::format("Unrecognized binary_type: ({})", static_cast(type)), + std::runtime_error); } } -blob_t blob_t::from_vector(std::vector&& data) +nvJitLinkInputType to_nvjitlink_input_type(binary_type bin_type) { - auto ptr = new std::vector(std::move(data)); - return blob_t::from_parts( - ptr->data(), ptr->size(), ptr, [](void* user_data, uint8_t const*, size_t) { - delete reinterpret_cast*>(user_data); - }); -} - -blob_t blob_t::from_static_data(std::span data) -{ - return blob_t::from_parts( - data.data(), data.size(), nullptr, [](void*, uint8_t const*, size_t) {}); + switch (bin_type) { + case binary_type::LTO_IR: return NVJITLINK_INPUT_LTOIR; + case binary_type::CUBIN: return NVJITLINK_INPUT_CUBIN; + case binary_type::FATBIN: return NVJITLINK_INPUT_FATBIN; + case binary_type::PTX: return NVJITLINK_INPUT_PTX; + default: + CUDF_FAIL( + +std::format("Unrecognized binary type for linking: ({}) ", static_cast(bin_type)), + std::logic_error); + } } -fragment fragment_t::load(load_params const& params) +void log_nvrtc_result(fragment_t::compile_params const& params, + nvrtcProgram program, + nvrtcResult compile_result) { - CUDF_FUNC_RANGE(); + if (program == nullptr) { return; } - return std::make_shared(params.binary, params.type); -} - -std::string get_nvrtc_log(fragment_t::compile_params const& params, - nvrtcProgram program, - nvrtcResult compile_result) -{ size_t log_size; - if (nvrtcGetProgramLogSize(program, &log_size) != NVRTC_SUCCESS) { return std::string{}; } + if (auto errc = nvrtcGetProgramLogSize(program, &log_size); errc != NVRTC_SUCCESS) { + CUDF_FAIL(+std::format("Failed to get NVRTC program log size with error ({}): {}", + static_cast(errc), + nvrtcGetErrorString(errc)), + std::runtime_error); + } + + if (log_size <= 1) { return; } std::vector log; log.resize(log_size); - if (nvrtcGetProgramLog(program, log.data()) != NVRTC_SUCCESS) { return std::string{}; } + if (auto errc = nvrtcGetProgramLog(program, log.data()); errc != NVRTC_SUCCESS) { + CUDF_FAIL(+std::format("Failed to get NVRTC program log with error ({}): {}", + static_cast(errc), + nvrtcGetErrorString(errc)), + std::runtime_error); + } + + log.resize(log_size == 0 ? 0 : (log_size - 1)); - auto status_str = - (compile_result == NVRTC_SUCCESS) ? "completed with warning" : "failed with error"; + auto status_str = (compile_result == NVRTC_SUCCESS && !log.empty()) ? "completed with warning" + : "failed with error"; std::string headers_str; for (auto const& header : params.headers.include_names) { @@ -151,33 +165,71 @@ std::string get_nvrtc_log(fragment_t::compile_params const& params, options_str = std::format("{}\t{}\n", options_str, option); } - return std::format("NCRTC Compilation for {} {} ({}): {}.\nHeaders:\n{}\n\nOptions:\n{}\n\n{}", - params.name == nullptr ? "" : params.name, - status_str, - static_cast(compile_result), - nvrtcGetErrorString(compile_result), - headers_str, - options_str, - log.data()); + auto msg = std::format( + "NVRTC Compilation for `{}` {} ({}): {}.\nHeaders:\n{}\n\nOptions:\n{}\n\nLog:\n\t{}", + params.name == nullptr ? "" : params.name, + status_str, + static_cast(compile_result), + nvrtcGetErrorString(compile_result), + headers_str, + options_str, + std::string_view{log.data(), log.size()}); + + if (compile_result != NVRTC_SUCCESS) { + CUDF_LOG_ERROR(msg); + } else { + CUDF_LOG_WARN(msg); + } } -void log_nvJitLink_link_result(library_t::link_params const& params, - nvJitLinkHandle handle, - nvJitLinkResult link_result) +void log_nvJitLink_result(library_t::link_params const& params, + nvJitLinkHandle handle, + nvJitLinkResult link_result) { + if (handle == nullptr) { return; } + size_t info_log_size; - CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetInfoLogSize(handle, &info_log_size)); + if (auto errc = nvJitLinkGetInfoLogSize(handle, &info_log_size); errc != NVJITLINK_SUCCESS) { + CUDF_FAIL(+std::format("Failed to get nvJitLink info log size with error ({}): {}", + static_cast(errc), + get_nvJitLinkResultString(errc)), + std::runtime_error); + } std::vector info_log; - info_log.resize(info_log_size); - CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetInfoLog(handle, info_log.data())); + if (info_log_size > 1) { + info_log.resize(info_log_size); + if (auto errc = nvJitLinkGetInfoLog(handle, info_log.data()); errc != NVJITLINK_SUCCESS) { + CUDF_FAIL(+std::format("Failed to get nvJitLink info log with error ({}): {}", + static_cast(errc), + get_nvJitLinkResultString(errc)), + std::runtime_error); + } + } + info_log.resize(info_log_size == 0 ? 0 : (info_log_size - 1)); size_t error_log_size; - CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetErrorLogSize(handle, &error_log_size)); + if (auto errc = nvJitLinkGetErrorLogSize(handle, &error_log_size); errc != NVJITLINK_SUCCESS) { + CUDF_FAIL(+std::format("Failed to get nvJitLink error log size with error ({}): {}", + static_cast(errc), + get_nvJitLinkResultString(errc)), + std::runtime_error); + } std::vector error_log; - error_log.resize(error_log_size); - CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetErrorLog(handle, error_log.data())); + + if (error_log_size > 1) { + error_log.resize(error_log_size); + if (auto errc = nvJitLinkGetErrorLog(handle, error_log.data()); errc != NVJITLINK_SUCCESS) { + CUDF_FAIL(+std::format("Failed to get nvJitLink error log with error ({}): {}", + static_cast(errc), + get_nvJitLinkResultString(errc)), + std::runtime_error); + } + } + error_log.resize(error_log_size == 0 ? 0 : (error_log_size - 1)); + + if (info_log.empty() && error_log.empty()) { return; } std::string fragments_str; for (auto const& fragment_name : params.fragment_names) { @@ -191,12 +243,11 @@ void log_nvJitLink_link_result(library_t::link_params const& params, char const* binary_type_str = binary_type_string(params.output_type); - auto status_str = - (link_result == NVJITLINK_SUCCESS) ? "completed successfully" : "failed with error"; + auto status_str = error_log.empty() ? "completed with warnings" : "failed with errors"; - auto str = std::format( - "nvJitLink Linking for {} ({}) {} ({}): {}.\nFragments: \n{}\n" - "Link Options: \n{}\n\nInfo Log:\n{}\n\nError Log:\n{}\n\n", + auto msg = std::format( + "(nvJitLink) Linking for `{}` ({}) {}, error code ({}): {}.\nFragments: \n{}\n" + "Link Options: \n{}\n\nInfo Log:\n\t{}\n\nError Log:\n\t{}\n\n", params.name == nullptr ? "" : params.name, binary_type_str, status_str, @@ -204,18 +255,43 @@ void log_nvJitLink_link_result(library_t::link_params const& params, get_nvJitLinkResultString(link_result), fragments_str, link_options_str, - info_log.data(), - error_log.data()); + std::string_view{info_log.data(), info_log.size()}, + std::string_view{error_log.data(), error_log.size()}); - if (link_result != NVJITLINK_SUCCESS) { - CUDF_FAIL(+str, std::runtime_error); - } else if (!info_log.empty() || !error_log.empty()) { - CUDF_LOG_WARN(str); + if (!error_log.empty()) { + CUDF_LOG_ERROR(msg); + } else { + CUDF_LOG_WARN(msg); } } +} // namespace + +blob_t blob_t::from_vector(std::vector&& data) +{ + auto ptr = new std::vector(std::move(data)); + return blob_t::from_parts( + ptr->data(), ptr->size(), ptr, [](void* user_data, uint8_t const*, size_t) { + delete reinterpret_cast*>(user_data); + }); +} + +blob_t blob_t::from_static_data(std::span data) +{ + return blob_t::from_parts( + data.data(), data.size(), nullptr, [](void*, uint8_t const*, size_t) {}); +} + +fragment fragment_t::load(load_params const& params) +{ + CUDF_FUNC_RANGE(); + // TODO: check + + return std::make_shared(params.binary, params.type); +} fragment fragment_t::compile(compile_params const& params) { + // TODO: check CUDF_FUNC_RANGE(); nvrtcProgram program = nullptr; @@ -236,10 +312,6 @@ fragment fragment_t::compile(compile_params const& params) program, nvrtcCompileProgram(program, static_cast(params.options.size()), params.options.data())); - if (auto log = get_nvrtc_log(params, program, NVRTC_SUCCESS); !log.empty()) { - CUDF_LOG_WARN(log); - } - switch (params.target_type) { case binary_type::LTO_IR: { size_t lto_ir_size; @@ -328,6 +400,7 @@ library_t::~library_t() library library_t::load(load_params const& params) { + // TODO: check CUDF_FUNC_RANGE(); CUlibrary handle; @@ -353,46 +426,49 @@ blob library_t::link_as_blob(link_params const& params) CUDF_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, "Only CUBIN and PTX output types are supported for linking modules", std::logic_error); + CUDF_EXPECTS(params.fragments.size() == params.fragment_binary_types.size(), + "Mismatched number of fragments and fragment binary types", + std::logic_error); + CUDF_EXPECTS(params.fragments.size() == params.fragment_names.size(), + "Mismatched number of fragments and fragment names", + std::logic_error); + CUDF_EXPECTS(params.fragments.size() > 0, "No fragments provided for linking", std::logic_error); + + for (auto& frag : params.fragments) { + CUDF_EXPECTS(frag.size_bytes() > 0, "Fragment binary data must be non-empty", std::logic_error); + } - nvJitLinkHandle handle; + nvJitLinkHandle handle = nullptr; - CUDFRTC_CHECK_NVJITLINK(nvJitLinkCreate(&handle, + CUDFRTC_CHECK_NVJITLINK(params, + handle, + nvJitLinkCreate(&handle, static_cast(params.link_options.size()), const_cast(params.link_options.data()))); CUDF_DEFER([&] { nvJitLinkDestroy(&handle); }); for (size_t i = 0; i < params.fragments.size(); i++) { - auto name = params.fragment_names[i]; - auto fragment = params.fragments[i]; - auto bin_type = params.fragment_binary_types[i]; - - nvJitLinkInputType nv_type; - - switch (bin_type) { - case binary_type::LTO_IR: nv_type = NVJITLINK_INPUT_LTOIR; break; - case binary_type::CUBIN: nv_type = NVJITLINK_INPUT_CUBIN; break; - case binary_type::FATBIN: nv_type = NVJITLINK_INPUT_FATBIN; break; - case binary_type::PTX: nv_type = NVJITLINK_INPUT_PTX; break; - default: CUDF_FAIL("Unsupported binary type for loading fragment", std::logic_error); - } + auto name = params.fragment_names[i]; + auto fragment = params.fragments[i]; + auto bin_type = params.fragment_binary_types[i]; + nvJitLinkInputType nv_type = to_nvjitlink_input_type(bin_type); CUDFRTC_CHECK_NVJITLINK( + params, + handle, nvJitLinkAddData(handle, nv_type, fragment.data(), fragment.size_bytes(), name)); } - nvJitLinkResult link_result = nvJitLinkComplete(handle); - - log_nvJitLink_link_result(params, handle, link_result); + CUDFRTC_CHECK_NVJITLINK(params, handle, nvJitLinkComplete(handle)); switch (params.output_type) { case binary_type::CUBIN: { size_t cubin_size; - - CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedCubinSize(handle, &cubin_size)); + CUDFRTC_CHECK_NVJITLINK(params, handle, nvJitLinkGetLinkedCubinSize(handle, &cubin_size)); std::vector cubin; cubin.resize(cubin_size); - CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedCubin(handle, cubin.data())); + CUDFRTC_CHECK_NVJITLINK(params, handle, nvJitLinkGetLinkedCubin(handle, cubin.data())); return std::make_shared(blob_t::from_vector(std::move(cubin))); } break; @@ -400,16 +476,20 @@ blob library_t::link_as_blob(link_params const& params) case binary_type::PTX: { size_t ptx_size; - CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedPtxSize(handle, &ptx_size)); + CUDFRTC_CHECK_NVJITLINK(params, handle, nvJitLinkGetLinkedPtxSize(handle, &ptx_size)); std::vector ptx; ptx.resize(ptx_size); - CUDFRTC_CHECK_NVJITLINK(nvJitLinkGetLinkedPtx(handle, reinterpret_cast(ptx.data()))); + + CUDFRTC_CHECK_NVJITLINK( + params, handle, nvJitLinkGetLinkedPtx(handle, reinterpret_cast(ptx.data()))); return std::make_shared(blob_t::from_vector(std::move(ptx))); } break; default: - CUDF_FAIL("Unsupported output binary type for linking CUDA libraries", std::runtime_error); + CUDF_FAIL(+std::format("Unsupported output binary type for linking CUDA libraries: ({})", + static_cast(params.output_type)), + std::runtime_error); } } diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index 2c633beb5cd5..ea9adfa58706 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -22,10 +22,10 @@ TEST_F(RTCTest, CreateFragment) "test_kernel_key", R"***( #include "cudf/jit/transform_params.cuh" - #include "cudf/jit/lto/operators.inl.cuh" - #include "cudf/jit/lto/types.inl.cuh" + #include "jit/lto/operators.inl.cuh" + #include "jit/lto/types.inl.cuh" - extern "C" __device__ transform_operation(){ + extern "C" __device__ void transform_operator(cudf::lto::transform_params const*){ } )***", From 2734e897afadf00ca6a36b5e1cf787798130ccae Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 10 Jan 2026 06:37:50 +0000 Subject: [PATCH 015/254] Refactor JIT compilation code for improved performance and maintainability - Changed function declarations in `operators.cuh` from inline to non-inline for better optimization. - Updated `string_view` methods in `types.cuh` to remove inline specifiers, enhancing clarity. - Added a README file in the RTC directory to document design and compilation strategies. - Enhanced error handling in `cache.cpp` with improved error messages and added filesystem support. - Modified `cudf.cpp` to streamline include directory management and added SHA256 hashing for file integrity. - Implemented structured paths for cache storage in `cache.hpp`. - Introduced new functions for managing include directories and file operations in `cudf.cpp`. - Updated tests in `rtc.cpp` to measure and report compilation times for better performance tracking. --- cpp/cmake/Modules/jit_embed.py | 309 +++++++++++++--------- cpp/include/cudf/jit/lto/operators.cuh | 77 +++--- cpp/include/cudf/jit/lto/types.cuh | 116 ++++---- cpp/src/jit/rtc/README.md | 7 + cpp/src/jit/rtc/cache.cpp | 35 ++- cpp/src/jit/rtc/cache.hpp | 2 + cpp/src/jit/rtc/cudf.cpp | 223 +++++++++++----- cpp/src/jit/rtc/cudf.hpp | 2 + cpp/src/jit/rtc/includes/system/README.md | 1 + cpp/src/jit/rtc/sha256.hpp | 25 +- cpp/src/runtime/context.cpp | 2 + cpp/tests/jit/rtc.cpp | 40 ++- 12 files changed, 508 insertions(+), 331 deletions(-) create mode 100644 cpp/src/jit/rtc/README.md create mode 100644 cpp/src/jit/rtc/includes/system/README.md diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py index dd62cd7b4800..722894fa4bc4 100644 --- a/cpp/cmake/Modules/jit_embed.py +++ b/cpp/cmake/Modules/jit_embed.py @@ -40,16 +40,21 @@ extern "C" {{ -typedef struct {NAMESPACE_PREFIX}byte_array_t {{ +typedef struct {NAMESPACE_PREFIX}bytes_t {{ {BYTE_TYPE} const * data; {SIZE_TYPE} size; -}} {NAMESPACE_PREFIX}byte_array_t; +}} {NAMESPACE_PREFIX}bytes_t; -typedef struct {NAMESPACE_PREFIX}array_of_byte_arrays_t {{ - {BYTE_TYPE} const * const * elements; - {SIZE_TYPE} const * element_sizes; +typedef struct {NAMESPACE_PREFIX}byte_range_t {{ + {SIZE_TYPE} offset; {SIZE_TYPE} size; -}} {NAMESPACE_PREFIX}array_of_byte_arrays_t; +}} {NAMESPACE_PREFIX}byte_range_t; + +typedef struct {NAMESPACE_PREFIX}bytes_array_t {{ + {NAMESPACE_PREFIX}bytes_t bytes; + {NAMESPACE_PREFIX}byte_range_t const * ranges; + {SIZE_TYPE} num_ranges; +}} {NAMESPACE_PREFIX}bytes_array_t; }} @@ -64,7 +69,7 @@ def list_string(strings: list[str]) -> str: return ",\n".join(lines) -def hex_string(value: int) -> str: +def byte_hex_string(value: int) -> str: return f"0x{value:02X}" @@ -72,12 +77,9 @@ class CXXVarDecl(NamedTuple): id: str expr: str - def code(self: Self) -> str: - return f"""{self.expr}""" - @staticmethod def of_bytes(id: str, data: bytes, alignment: int) -> Self: - byte_array = list_string([hex_string(b) for b in data]) + byte_array = list_string([byte_hex_string(b) for b in data]) expr = f"""alignas({alignment}) {STORAGE_SPEC} {BYTE_TYPE} const {id}[{len(data)}] = {{ {byte_array} }};""" @@ -88,207 +90,258 @@ def of_size(id: str, size: int) -> Self: expr = f"{STORAGE_SPEC} {SIZE_TYPE} const {id} = {size}ULL;" return CXXVarDecl(id=id, expr=expr) + def decl(self: Self) -> str: + return f"""{self.expr}""" + class CXXSizeArrayDecl(NamedTuple): id: str sizes: list[int] - def decl(self: Self) -> CXXVarDecl: + @staticmethod + def of_sizes(id: str, sizes: list[int]) -> Self: + return CXXSizeArrayDecl(id=id, sizes=sizes) + + def var(self: Self) -> CXXVarDecl: size_array = list_string([f"{size}ULL" for size in self.sizes]) expr = f"""{STORAGE_SPEC} {SIZE_TYPE} const {self.id}[{len(self.sizes)}] = {{ {size_array} }};""" return CXXVarDecl(id=self.id, expr=expr) + +class CXXBytesDecl(NamedTuple): + id: str + data: bytes + alignment: int + num_null_terminators: int + @staticmethod - def of_sizes(id: str, sizes: list[int]) -> Self: - return CXXSizeArrayDecl(id=id, sizes=sizes) + def of_bytes( + id: str, data: bytes, alignment: int, num_null_terminators: int + ) -> Self: + return CXXBytesDecl( + id=id, + data=data, + alignment=alignment, + num_null_terminators=num_null_terminators, + ) + def var(self: Self) -> CXXVarDecl: + # exclude null terminator from length + size_decl = CXXVarDecl.of_size( + id=f"{self.id}_size", size=len(self.data) + ) -class CXXByteArrayDecl(NamedTuple): - id: str - data: CXXVarDecl - size: CXXVarDecl + data = self.data + b"\0" * self.num_null_terminators + + data_decl = CXXVarDecl.of_bytes( + id=f"{self.id}_data", data=data, alignment=self.alignment + ) - def decl(self: Self) -> CXXVarDecl: return CXXVarDecl( id=self.id, expr=f""" -{self.data.code()} +{data_decl.decl()} -{self.size.code()} +{size_decl.decl()} -{STORAGE_SPEC} {NAMESPACE_PREFIX}byte_array_t const {self.id} = {{ - {self.data.id}, - {self.size.id} +{STORAGE_SPEC} {NAMESPACE_PREFIX}bytes_t const {self.id} = {{ + .data = {data_decl.id}, + .size = {size_decl.id} }}; """, ) + +class CXXRangesDecl(NamedTuple): + id: str + ranges: list[tuple[int, int]] # list of (offset, size) + @staticmethod - def of_bytes( - id: str, data: bytes, alignment: int, num_null_terminators: int - ) -> Self: - # exclude null terminator from length - size_decl = CXXVarDecl.of_size(id=f"{id}_size", size=len(data)) + def of_ranges(id: str, ranges: list[tuple[int, int]]) -> Self: + return CXXRangesDecl(id=id, ranges=ranges) - data += b"\0" * num_null_terminators + def var(self: Self) -> CXXVarDecl: + ranges_str = [ + f"{{{offset}UL, {size}UL}}" for offset, size in self.ranges + ] - data_decl = CXXVarDecl.of_bytes( - id=f"{id}_data", data=data, alignment=alignment - ) + ranges_str_formatted = list_string(ranges_str) - return CXXByteArrayDecl(id=id, data=data_decl, size=size_decl) + expr = f"""{STORAGE_SPEC} {NAMESPACE_PREFIX}byte_range_t const {self.id}[{len(self.ranges)}] = {{ +{ranges_str_formatted} +}};""" + return CXXVarDecl(id=self.id, expr=expr) -class CXXArrayOfByteArraysDecl(NamedTuple): +class CXXArrayOfBytesDecl(NamedTuple): id: str - elements: list[CXXByteArrayDecl] - size: CXXVarDecl + data: bytes + alignment: int + ranges: list[tuple[int, int]] # list of (offset, size) - def decl(self: Self) -> CXXVarDecl: - elements_decl = "\n".join(e.decl().code() for e in self.elements) - count = len(self.elements) - data_ids = ", ".join([d.data.id for d in self.elements]) - size_ids = ", ".join([d.size.id for d in self.elements]) + @staticmethod + def of_byte_ranges( + id: str, + data: bytes, + ranges: list[tuple[int, int]], + alignment: int, + ) -> Self: + return CXXArrayOfBytesDecl( + id=id, + data=data, + alignment=alignment, + ranges=ranges, + ) - return CXXVarDecl( - id=self.id, - expr=f""" -{elements_decl} + def var(self: Self) -> CXXVarDecl: + bytes_decl = CXXBytesDecl.of_bytes( + id=f"{self.id}_bytes", + data=self.data, + alignment=self.alignment, + num_null_terminators=0, + ) + + ranges_decl = CXXRangesDecl.of_ranges( + id=f"{self.id}_ranges", ranges=self.ranges + ) -{STORAGE_SPEC} {BYTE_TYPE} const * const {self.id}_elements[{count}] = {{ {data_ids} }}; -{STORAGE_SPEC} {SIZE_TYPE} const {self.id}_element_sizes[{count}] = {{ {size_ids} }}; + expr = f""" +{bytes_decl.var().decl()} +{ranges_decl.var().decl()} -{STORAGE_SPEC} {NAMESPACE_PREFIX}array_of_byte_arrays_t const {self.id} = {{ - {self.id}_elements, - {self.id}_element_sizes, - {count}ULL +{STORAGE_SPEC} {NAMESPACE_PREFIX}bytes_array_t const {self.id} = {{ + .bytes = {bytes_decl.id}, + .ranges = {ranges_decl.id}, + .num_ranges = {len(self.ranges)} }}; -""", +""" + + return CXXVarDecl( + id=self.id, + expr=expr, ) - @staticmethod - def of_bytes_array( - id: str, - data_list: list[bytes], - data_alignment: int, - num_null_terminators: int, - ) -> Self: - size = len(data_list) - size_decl = CXXVarDecl.of_size(id=f"{id}_size", size=size) - array_decls: list[CXXByteArrayDecl] = [ - CXXByteArrayDecl.of_bytes( - id=f"{id}_element_{i}", - data=data, - alignment=data_alignment, - num_null_terminators=num_null_terminators, - ) - for i, data in enumerate(data_list) - ] - return CXXArrayOfByteArraysDecl( - id=id, elements=array_decls, size=size_decl - ) +def merge_bytes_with_null_terminators( + bytes_lists: list[bytes], +) -> tuple[bytes, list[tuple[int, int]]]: + merged: bytes = bytes() + ranges: list[tuple[int, int]] = [] + + for byte_data in bytes_lists: + ranges.append((len(merged), len(byte_data))) + merged += byte_data + b"\0" + return merged, ranges -def generate_cxx_string_data(id: str, strings: list[str]) -> str: - arrays: list[bytes] = [s.encode("utf-8") for s in strings] - # compute combined sha256 hash of all strings +def generate_cxx_strings_data(id: str, strings: list[str]) -> str: + data, ranges = merge_bytes_with_null_terminators( + [s.encode("utf-8") for s in strings] + ) + sha = hashlib.sha256() - for arr in arrays: - sha.update(arr) + sha.update(data) data_hash: bytes = sha.digest() - arrays_decl = CXXArrayOfByteArraysDecl.of_bytes_array( - id=f"{id}", data_list=arrays, data_alignment=1, num_null_terminators=1 + arrays_decl = CXXArrayOfBytesDecl.of_byte_ranges( + id=f"{id}", + data=data, + alignment=1, + ranges=ranges, ) - data_hash_decl: CXXByteArrayDecl = CXXByteArrayDecl.of_bytes( - id=f"{id}_data_hash", + data_hash_decl: CXXBytesDecl = CXXBytesDecl.of_bytes( + id=f"{id}_hash", data=data_hash, alignment=1, num_null_terminators=0, ) return f""" -{arrays_decl.decl().code()} +{arrays_decl.var().decl()} -{data_hash_decl.decl().code()} +{data_hash_decl.var().decl()} """ +def load_file_bytes(file_path: str) -> bytes: + with open(file_path, "rb") as f: + return f.read() + + def generate_cxx_source_files_data( id: str, file_paths: list[str], dests: list[str], include_directories: list[str] = [], ) -> str: - files_bytes: list[bytes] = [] + files_bytes, files_ranges = merge_bytes_with_null_terminators( + [load_file_bytes(p) for p in file_paths] + ) + + merged_dests_bytes, merged_dests_ranges = ( + merge_bytes_with_null_terminators([d.encode("utf-8") for d in dests]) + ) + + merged_include_directories_bytes, merged_include_directories_ranges = ( + merge_bytes_with_null_terminators( + [d.encode("utf-8") for d in include_directories] + ) + ) # compute combined sha256 hash of all files sha = hashlib.sha256() + sha.update(files_bytes) + sha.update(merged_dests_bytes) + sha.update(merged_include_directories_bytes) - for file_path in file_paths: - with open(file_path, "rb") as f: - data = f.read() - sha.update(data) - files_bytes.append(data) + hash: bytes = sha.digest() - file_data_hash: bytes = sha.digest() - - file_destinations_decls: CXXArrayOfByteArraysDecl = ( - CXXArrayOfByteArraysDecl.of_bytes_array( + file_destinations_decls: CXXArrayOfBytesDecl = ( + CXXArrayOfBytesDecl.of_byte_ranges( id=f"{id}_file_destinations", - data_list=[f.encode("utf-8") for f in dests], - data_alignment=1, - num_null_terminators=1, - ) - ) - - file_data_decl: CXXArrayOfByteArraysDecl = ( - CXXArrayOfByteArraysDecl.of_bytes_array( - id=f"{id}_file_data", - data_list=files_bytes, - data_alignment=1, - num_null_terminators=1, + data=merged_dests_bytes, + ranges=merged_dests_ranges, + alignment=1, ) ) - file_data_hash_decl: CXXByteArrayDecl = CXXByteArrayDecl.of_bytes( - id=f"{id}_file_data_hash", - data=file_data_hash, + file_data_decl: CXXArrayOfBytesDecl = CXXArrayOfBytesDecl.of_byte_ranges( + id=f"{id}_file_data", + data=files_bytes, + ranges=files_ranges, alignment=1, - num_null_terminators=0, - ) - - file_size_decls: CXXSizeArrayDecl = CXXSizeArrayDecl.of_sizes( - id=f"{id}_file_sizes", - sizes=[len(f) for f in files_bytes], ) - include_directories_decls: CXXArrayOfByteArraysDecl = ( - CXXArrayOfByteArraysDecl.of_bytes_array( + include_directories_decls: CXXArrayOfBytesDecl = ( + CXXArrayOfBytesDecl.of_byte_ranges( id=f"{id}_include_directories", - data_list=[d.encode("utf-8") for d in include_directories], - data_alignment=1, - num_null_terminators=1, + data=merged_include_directories_bytes, + ranges=merged_include_directories_ranges, + alignment=1, ) ) - return f""" -{file_destinations_decls.decl().code()} + hash_decl: CXXBytesDecl = CXXBytesDecl.of_bytes( + id=f"{id}_hash", + data=hash, + alignment=1, + num_null_terminators=0, + ) -{file_data_decl.decl().code()} + return f""" +{file_destinations_decls.var().decl()} -{file_data_hash_decl.decl().code()} +{file_data_decl.var().decl()} -{file_size_decls.decl().code()} +{include_directories_decls.var().decl()} -{include_directories_decls.decl().code()} +{hash_decl.var().decl()} """ @@ -312,7 +365,7 @@ def generate_embed_source(entries: dict[str, dict[str, dict]]) -> str: elif entry_type == "strings": options: list[str] = entry_value["strings"] - code += generate_cxx_string_data(entry_id, options) + code += generate_cxx_strings_data(entry_id, options) else: raise ValueError(f"Unknown type: {entry_type}") diff --git a/cpp/include/cudf/jit/lto/operators.cuh b/cpp/include/cudf/jit/lto/operators.cuh index 49445d418ea6..135d20f5c155 100644 --- a/cpp/include/cudf/jit/lto/operators.cuh +++ b/cpp/include/cudf/jit/lto/operators.cuh @@ -11,11 +11,10 @@ namespace lto { namespace operators { -#define CUDF_LTO_DECL(op, type) \ - __device__ extern inline void op(type* out, type const* a, type const* b); \ - \ - __device__ extern inline void op( \ - optional* out, optional const* a, optional const* b) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern void op(type* out, type const* a, type const* b); \ + \ + __device__ extern void op(optional* out, optional const* a, optional const* b) CUDF_LTO_DECL(add, int32_t); CUDF_LTO_DECL(add, int64_t); @@ -76,13 +75,12 @@ CUDF_LTO_DECL(pow, float64_t); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - __device__ extern inline void op(bool* out, type const* a, type const* b); \ - \ - __device__ extern inline void op(bool* out, optional const* a, optional const* b); \ - \ - __device__ extern inline void op( \ - optional* out, optional const* a, optional const* b) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern void op(bool* out, type const* a, type const* b); \ + \ + __device__ extern void op(bool* out, optional const* a, optional const* b); \ + \ + __device__ extern void op(optional* out, optional const* a, optional const* b) CUDF_LTO_DECL(equal, bool); CUDF_LTO_DECL(equal, int8_t); @@ -136,11 +134,10 @@ CUDF_LTO_DECL(null_equal, string_view); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - __device__ extern inline void op(bool* out, type const* a, type const* b); \ - \ - __device__ extern inline void op( \ - optional* out, optional const* a, optional const* b) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern void op(bool* out, type const* a, type const* b); \ + \ + __device__ extern void op(optional* out, optional const* a, optional const* b) CUDF_LTO_DECL(less, bool); CUDF_LTO_DECL(less, int8_t); @@ -244,11 +241,10 @@ CUDF_LTO_DECL(greater_equal, string_view); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - __device__ extern inline void op(type* out, type const* a, type const* b); \ - \ - __device__ extern inline void op( \ - optional* out, optional const* a, optional const* b) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern void op(type* out, type const* a, type const* b); \ + \ + __device__ extern void op(optional* out, optional const* a, optional const* b) CUDF_LTO_DECL(bitwise_and, int32_t); CUDF_LTO_DECL(bitwise_and, int64_t); @@ -267,11 +263,10 @@ CUDF_LTO_DECL(bitwise_xor, uint64_t); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - __device__ extern inline void op(type* out, type const* a, type const* b); \ - \ - __device__ extern inline void op( \ - optional* out, optional const* a, optional const* b); +#define CUDF_LTO_DECL(op, type) \ + __device__ extern void op(type* out, type const* a, type const* b); \ + \ + __device__ extern void op(optional* out, optional const* a, optional const* b); CUDF_LTO_DECL(logical_and, bool); @@ -283,10 +278,10 @@ CUDF_LTO_DECL(null_logical_or, bool); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - __device__ extern inline void op(type* out, type const* a); \ - \ - __device__ extern inline void op(optional* out, optional const* a) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern void op(type* out, type const* a); \ + \ + __device__ extern void op(optional* out, optional const* a) CUDF_LTO_DECL(identity, bool); CUDF_LTO_DECL(identity, int8_t); @@ -379,10 +374,10 @@ CUDF_LTO_DECL(bit_invert, int64_t); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, ret_type, type) \ - __device__ extern inline void op(ret_type* out, type const* a); \ - \ - __device__ extern inline void op(optional* out, optional const* a) +#define CUDF_LTO_DECL(op, ret_type, type) \ + __device__ extern void op(ret_type* out, type const* a); \ + \ + __device__ extern void op(optional* out, optional const* a) CUDF_LTO_DECL(cast_to_int64, int64_t, bool); CUDF_LTO_DECL(cast_to_int64, int64_t, int8_t); @@ -422,12 +417,12 @@ CUDF_LTO_DECL(cast_to_float64, float64_t, float64_t); #undef CUDF_LTO_DECL -#define CUDF_LTO_DECL(op, type) \ - __device__ extern inline void op(bool* out, type const* a); \ - \ - __device__ extern inline void op(bool* out, optional const* a); \ - \ - __device__ extern inline void op(optional* out, optional const* a) +#define CUDF_LTO_DECL(op, type) \ + __device__ extern void op(bool* out, type const* a); \ + \ + __device__ extern void op(bool* out, optional const* a); \ + \ + __device__ extern void op(optional* out, optional const* a) CUDF_LTO_DECL(is_null, bool); CUDF_LTO_DECL(is_null, int8_t); diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index fb4a47ebc39c..584d83f8e904 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -63,61 +63,61 @@ struct CUDF_LTO_ALIAS string_view { mutable size_type __length = 0; public: - [[nodiscard]] __device__ inline size_type size_bytes() const; + [[nodiscard]] __device__ size_type size_bytes() const; - [[nodiscard]] __device__ inline size_type length() const; + [[nodiscard]] __device__ size_type length() const; - [[nodiscard]] __device__ inline char const* data() const; + [[nodiscard]] __device__ char const* data() const; - [[nodiscard]] __device__ inline bool empty() const; + [[nodiscard]] __device__ bool empty() const; - [[nodiscard]] __device__ inline char_utf8 operator[](size_type pos) const; + [[nodiscard]] __device__ char_utf8 operator[](size_type pos) const; - [[nodiscard]] __device__ inline size_type byte_offset(size_type pos) const; + [[nodiscard]] __device__ size_type byte_offset(size_type pos) const; - [[nodiscard]] __device__ inline int compare(string_view const& str) const; + [[nodiscard]] __device__ int compare(string_view const& str) const; - [[nodiscard]] __device__ inline int compare(char const* str, size_type bytes) const; + [[nodiscard]] __device__ int compare(char const* str, size_type bytes) const; - [[nodiscard]] __device__ inline bool operator==(string_view const& rhs) const; + [[nodiscard]] __device__ bool operator==(string_view const& rhs) const; - [[nodiscard]] __device__ inline bool operator!=(string_view const& rhs) const; + [[nodiscard]] __device__ bool operator!=(string_view const& rhs) const; - [[nodiscard]] __device__ inline bool operator<(string_view const& rhs) const; + [[nodiscard]] __device__ bool operator<(string_view const& rhs) const; - [[nodiscard]] __device__ inline bool operator>(string_view const& rhs) const; + [[nodiscard]] __device__ bool operator>(string_view const& rhs) const; - [[nodiscard]] __device__ inline bool operator<=(string_view const& rhs) const; + [[nodiscard]] __device__ bool operator<=(string_view const& rhs) const; - [[nodiscard]] __device__ inline bool operator>=(string_view const& rhs) const; + [[nodiscard]] __device__ bool operator>=(string_view const& rhs) const; - [[nodiscard]] __device__ inline size_type find(string_view const& str, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ size_type find(string_view const& str, + size_type pos = 0, + size_type count = -1) const; - [[nodiscard]] __device__ inline size_type find(char const* str, - size_type bytes, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ size_type find(char const* str, + size_type bytes, + size_type pos = 0, + size_type count = -1) const; - [[nodiscard]] __device__ inline size_type find(char_utf8 character, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ size_type find(char_utf8 character, + size_type pos = 0, + size_type count = -1) const; - [[nodiscard]] __device__ inline size_type rfind(string_view const& str, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ size_type rfind(string_view const& str, + size_type pos = 0, + size_type count = -1) const; - [[nodiscard]] __device__ inline size_type rfind(char const* str, - size_type bytes, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ size_type rfind(char const* str, + size_type bytes, + size_type pos = 0, + size_type count = -1) const; - [[nodiscard]] __device__ inline size_type rfind(char_utf8 character, - size_type pos = 0, - size_type count = -1) const; + [[nodiscard]] __device__ size_type rfind(char_utf8 character, + size_type pos = 0, + size_type count = -1) const; - [[nodiscard]] __device__ inline string_view substr(size_type start, size_type length) const; + [[nodiscard]] __device__ string_view substr(size_type start, size_type length) const; static inline size_type const npos{-1}; }; @@ -282,30 +282,30 @@ struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { public: template - __device__ inline T const* head() const; + __device__ T const* head() const; - __device__ inline size_type size() const; + __device__ size_type size() const; - __device__ inline bool nullable() const; + __device__ bool nullable() const; - __device__ inline bitmask_type const* null_mask() const; + __device__ bitmask_type const* null_mask() const; - __device__ inline size_type offset() const; + __device__ size_type offset() const; - __device__ inline bool is_valid(size_type idx) const; + __device__ bool is_valid(size_type idx) const; - __device__ inline bool is_valid_nocheck(size_type idx) const; + __device__ bool is_valid_nocheck(size_type idx) const; - __device__ inline bool is_null(size_type idx) const; + __device__ bool is_null(size_type idx) const; - __device__ inline bool is_null_nocheck(size_type idx) const; + __device__ bool is_null_nocheck(size_type idx) const; - __device__ inline bitmask_type get_mask_word(size_type word_index) const; + __device__ bitmask_type get_mask_word(size_type word_index) const; template - __device__ inline T element(size_type idx) const; + __device__ T element(size_type idx) const; - __device__ inline size_type num_child_columns() const; + __device__ size_type num_child_columns() const; }; #define CUDF_LTO_DECL(Type) \ @@ -386,28 +386,28 @@ struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { public: template - __device__ inline T* head() const; + __device__ T* head() const; - __device__ inline size_type size() const; + __device__ size_type size() const; - __device__ inline bool nullable() const; + __device__ bool nullable() const; - __device__ inline bitmask_type* null_mask() const; + __device__ bitmask_type* null_mask() const; - __device__ inline size_type offset() const; + __device__ size_type offset() const; - __device__ inline bool is_valid(size_type idx) const; + __device__ bool is_valid(size_type idx) const; - __device__ inline bool is_valid_nocheck(size_type idx) const; + __device__ bool is_valid_nocheck(size_type idx) const; - __device__ inline bool is_null(size_type idx) const; + __device__ bool is_null(size_type idx) const; - __device__ inline bool is_null_nocheck(size_type idx) const; + __device__ bool is_null_nocheck(size_type idx) const; - __device__ inline bitmask_type get_mask_word(size_type word_index) const; + __device__ bitmask_type get_mask_word(size_type word_index) const; template - __device__ inline T element(size_type idx) const; + __device__ T element(size_type idx) const; }; #define CUDF_LTO_DECL(Type) \ diff --git a/cpp/src/jit/rtc/README.md b/cpp/src/jit/rtc/README.md new file mode 100644 index 000000000000..8e20dbcbf48b --- /dev/null +++ b/cpp/src/jit/rtc/README.md @@ -0,0 +1,7 @@ +# Doc + +# vs JITIFY + +# Design + +# How to Achieve Fast Compilation with LTO-IR \ No newline at end of file diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index 5a9d6dff70a8..3183254f3896 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -17,6 +17,7 @@ #include #include +#include #include namespace cudf { @@ -26,10 +27,10 @@ namespace { [[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) { - auto error_code = errno; - auto error_str = std::format( - "{}. `{}` failed with {} ({})", message, syscall_name, error_code, std::strerror(error_code)); - CUDF_FAIL(+error_str, std::runtime_error); + auto errc = errno; + auto err_str = + std::format("{}. `{}` failed with {} ({})", message, syscall_name, errc, std::strerror(errc)); + CUDF_FAIL(+err_str, std::runtime_error); } } // namespace @@ -99,9 +100,7 @@ void cache_t::store_blob_to_disk(sha256_hash const& sha, blob_view binary) if (fd == -1) { throw_posix("Failed to create temporary file for RTC cache", "mkstemp"); } CUDF_DEFER([&] { - if (remove(temp_path) == -1) { - throw_posix("Failed to remove temporary RTC cache file", "remove"); - } + if (close(fd) == -1) { throw_posix("Failed to close temporary RTC cache file", "close"); } }); if (write(fd, binary.data(), binary.size()) == -1) { @@ -109,26 +108,26 @@ void cache_t::store_blob_to_disk(sha256_hash const& sha, blob_view binary) } } - auto hex = sha.to_hex_string(); - char final_path[PATH_MAX + 1]; - auto result = std::format_to_n(final_path, PATH_MAX, "{}/{}.blob", cache_dir_, hex.view()); - CUDF_EXPECTS( - result.out != (final_path + PATH_MAX), "Path length exceeded PATH_MAX", std::runtime_error); - *result.out = '\0'; + auto hex = sha.to_hex_string(); + auto final_path = std::format("{}/{}.blob", cache_dir_, hex.view()); + + std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); // rename is atomic, even if another process is performing the same operation - if (rename(temp_path, final_path) == -1) { - auto error_code = errno; + if (rename(temp_path, final_path.c_str()) == -1) { + auto errc = errno; - if (error_code == EEXIST) { + if (errc == EEXIST) { // another process has already created the file, so just remove our temp file if (remove(temp_path) == -1) { throw_posix("Failed to remove temporary RTC cache file", "remove"); } return; + } else { + throw_posix( + std::format("Failed to move temporary RTC cache file to final location ({})", final_path), + "rename"); } - - throw_posix("Failed to move temporary RTC cache file to final location", "rename"); } } diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index 517300e23c09..ca3b9bbfeca6 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -267,6 +267,8 @@ struct cache_statistics_counter { struct cache_t { private: bool enabled_; + // TODO: store under more structured pathsi, i.e. /cache. + // TODO: stamp binaries so we can pre-load? std::string cache_dir_; diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 7d47e51af052..572c9ec1ff2e 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -15,14 +15,30 @@ #include #include #include +#include #include +#include #include #include +#include #include #include #include #include +#include + +#define CUDFRTC_CHECK_CUDART(msg, ...) \ + do { \ + ::cudaError_t __result = (__VA_ARGS__); \ + if (__result != ::cudaSuccess) { \ + auto __errstr = ::std::format("(cudart) Call {} failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::int64_t>(__result), \ + ::cudaGetErrorString(__result)); \ + CUDF_FAIL(+std::format("{}. {}", msg, __errstr), ::std::runtime_error); \ + } \ + } while (0) namespace cudf { namespace rtc { @@ -32,27 +48,27 @@ namespace { int32_t get_driver_version() { int32_t driver_version; - CUDF_EXPECTS(cudaDriverGetVersion(&driver_version) == cudaSuccess, - "Failed to get CUDA driver version"); + CUDFRTC_CHECK_CUDART("Failed to get CUDA driver version", cudaDriverGetVersion(&driver_version)); + return driver_version; } int32_t get_runtime_version() { int32_t runtime_version; - CUDF_EXPECTS(cudaRuntimeGetVersion(&runtime_version) == cudaSuccess, - "Failed to get CUDA runtime version"); + CUDFRTC_CHECK_CUDART("Failed to get CUDA runtime version", + cudaRuntimeGetVersion(&runtime_version)); + return runtime_version; } int32_t get_current_device_physical_model() { int32_t device; - CUDF_EXPECTS(cudaGetDevice(&device) == cudaSuccess, "Failed to get current CUDA device"); + CUDFRTC_CHECK_CUDART("Failed to get current CUDA device", cudaGetDevice(&device)); cudaDeviceProp props; - CUDF_EXPECTS(cudaGetDeviceProperties(&props, device) == cudaSuccess, - "Failed to get device properties"); + CUDFRTC_CHECK_CUDART("Failed to get device properties", cudaGetDeviceProperties(&props, device)); return props.major * 10 + props.minor; } @@ -81,7 +97,7 @@ cache_t& get_rtc_cache() { return cudf::get_context().rtc_cache(); } CUDF_FAIL(+error_str, std::runtime_error); } -void add_file(char const* dst_path, unsigned char const* data, size_t data_size) +void add_file(char const* dst_path, std::span contents) { int dst_file = open(dst_path, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (dst_file == -1) { @@ -98,7 +114,7 @@ void add_file(char const* dst_path, unsigned char const* data, size_t data_size) } }); - if (write(dst_file, data, data_size) == -1) { + if (write(dst_file, contents.data(), contents.size()) == -1) { throw_posix(std::format("Failed to write file ({})", dst_path), "write"); } } @@ -129,28 +145,46 @@ std::vector read_file(char const* path) return contents; } -void copy_includes_to_dir(char const* dir) +static constexpr char const* HASH_FILENAME = ".sha256.hash"; + +void copy_includes_to_dir(char const* dst_dir) { CUDF_FUNC_RANGE(); - for (size_t i = 0; i < cudf_jit_embed_sources_file_data.size; ++i) { - auto const data = cudf_jit_embed_sources_file_data.elements[i]; - auto const data_size = cudf_jit_embed_sources_file_data.element_sizes[i]; - auto const dst = - reinterpret_cast(cudf_jit_embed_sources_file_destinations.elements[i]); - auto const dst_path = std::format("{}/{}", dir, dst); + auto const files_data = cudf_jit_embed_sources_file_data.bytes.data; + auto const destinations_data = + reinterpret_cast(cudf_jit_embed_sources_file_destinations.bytes.data); + for (size_t i = 0; i < cudf_jit_embed_sources_file_data.num_ranges; ++i) { + auto const file_data_range = cudf_jit_embed_sources_file_data.ranges[i]; + auto const destination_range = cudf_jit_embed_sources_file_destinations.ranges[i]; + + auto const file_data = std::span{files_data + file_data_range.offset, file_data_range.size}; + auto const destination = + std::string_view{destinations_data + destination_range.offset, destination_range.size}; + + auto const destination_path = std::format("{}/{}", dst_dir, destination); - std::filesystem::create_directories(std::filesystem::path{dst_path}.parent_path()); - add_file(dst_path.c_str(), data, data_size); + std::filesystem::create_directories(std::filesystem::path{destination_path}.parent_path()); + add_file(destination_path.c_str(), file_data); } - auto hash_path = std::format("{}/state.hash", dir); - add_file(hash_path.c_str(), - cudf_jit_embed_sources_file_data_hash.data, - cudf_jit_embed_sources_file_data_hash.size); + { + // write out the state hash file + auto hash_path = std::format("{}/{}", dst_dir, HASH_FILENAME); + add_file(hash_path.c_str(), + std::span{cudf_jit_embed_sources_hash.data, cudf_jit_embed_sources_hash.size}); + } } -void create_new_include_dir(std::string const& include_path) +std::string get_include_dir(char const* base_dir) +{ + auto sha256_str = sha256_hex_string::make( + std::span{cudf_jit_embed_sources_hash.data, cudf_jit_embed_sources_hash.size}); + + return std::format("{}/{}", base_dir, sha256_str.view()); +} + +void create_new_include_dir(char const* base_dir) { CUDF_FUNC_RANGE(); @@ -158,60 +192,68 @@ void create_new_include_dir(std::string const& include_path) char tmp_dir_data[] = "/tmp/jit-includes_XXXXXX"; char* tmp_dir = mkdtemp(tmp_dir_data); if (tmp_dir == nullptr) { - throw_posix( - std::format("Failed to create temporary RTC include directory for ({})", include_path), - "mkdtemp"); + throw_posix(std::format("Failed to create temporary RTC include directory for ({})", base_dir), + "mkdtemp"); } copy_includes_to_dir(tmp_dir); - // [ ] use flockdir and use locks when accessing? + auto include_dir = get_include_dir(base_dir); - // rename the temporary directory to the target include_path - if (rename(tmp_dir, include_path.c_str()) == -1) { + // rename the temporary directory to the target include_dir + if (rename(tmp_dir, include_dir.c_str()) == -1) { throw_posix( - std::format("Failed to rename temporary RTC include directory to ({})", include_path), + std::format("Failed to rename temporary RTC include directory to ({})", include_dir), "rename"); } } -void create_include_dir(std::string const& include_path) +std::string install_includes_to(char const* base_dir) { CUDF_FUNC_RANGE(); - struct stat path_info; + auto include_dir = get_include_dir(base_dir); - if (lstat(include_path.c_str(), &path_info) == -1) { + struct stat path_info; + if (lstat(include_dir.c_str(), &path_info) == -1) { if (errno != ENOENT) { - throw_posix(std::format("Failed to get stat for jit_include directory ({})", include_path), - "lstat"); + throw_posix(std::format("Failed to get stat for directory ({})", include_dir), "lstat"); } else { - create_new_include_dir(include_path); + std::filesystem::create_directories(base_dir); + create_new_include_dir(base_dir); } } else { + // directory exists, perform important sanity checks if (!S_ISDIR(path_info.st_mode)) { - CUDF_FAIL(+std::format("RTC include path ({}) exists but is not a directory", include_path), + CUDF_FAIL(+std::format("Include dir ({}) exists but is not a directory", include_dir), std::runtime_error); } else { - // TODO: address, should not error out here // verify contents match expected headers - auto hash_path = std::format("{}/state.hash", include_path); + auto hash_path = std::format("{}/{}", include_dir, HASH_FILENAME); auto hash_data = read_file(hash_path.c_str()); - if (hash_data.size() != cudf_jit_embed_sources_file_data_hash.size || - std::memcmp(hash_data.data(), - cudf_jit_embed_sources_file_data_hash.data, - cudf_jit_embed_sources_file_data_hash.size) != 0) { - CUDF_FAIL(+std::format("RTC include directory ({}) contents do not match expected headers", - include_path), - std::runtime_error); - } + CUDF_EXPECTS(std::equal(hash_data.begin(), + hash_data.end(), + cudf_jit_embed_sources_hash.data, + cudf_jit_embed_sources_hash.data + cudf_jit_embed_sources_hash.size), + +std::format("RTC include dir ({}) is corrupted", include_dir), + std::runtime_error); } } + + return include_dir; } } // namespace +void install_includes(char const* cache_dir) +{ + CUDF_FUNC_RANGE(); + + auto install_dir = std::format("{}/jit-install", cache_dir); + install_includes_to(install_dir.c_str()); +} + fragment_t const& compile_fragment(char const* name, char const* source_code_cstr, char const* key) { CUDF_FUNC_RANGE(); @@ -233,8 +275,10 @@ fragment_t const& compile_fragment(char const* name, char const* source_code_cst auto& cache = get_rtc_cache(); if (auto frag = cache.query_fragment(cache_key_sha256); frag.has_value()) { + std::cout << "Loading RTC base library from memory\n"; return *frag->get(); } else if (auto disk_frag = cache.query_blob_from_disk(cache_key_sha256); disk_frag.has_value()) { + std::cout << "Loading RTC base library from disk cache\n"; std::promise prom; auto fut = std::shared_future{prom.get_future()}; { @@ -246,33 +290,50 @@ fragment_t const& compile_fragment(char const* name, char const* source_code_cst return *fut.get(); } + std::cout << "Compiling and linking RTC base library\n"; std::promise prom; auto fut = std::shared_future{prom.get_future()}; cache.store_fragment(cache_key_sha256, fut); + auto begin = std::chrono::high_resolution_clock::now(); auto cache_dir = cache.get_cache_dir(); - auto include_dir = std::format("{}/jit-includes", cache_dir); - - create_include_dir(include_dir); + auto install_dir = std::format("{}/jit-install", cache_dir); + auto include_dir = get_include_dir(install_dir.c_str()); std::vector include_options; include_options.push_back(std::format("-I{}", include_dir)); - for (size_t i = 0; i < cudf_jit_embed_sources_include_directories.size; i++) { - auto include_path = - reinterpret_cast(cudf_jit_embed_sources_include_directories.elements[i]); - include_options.push_back(std::format("-I{}/{}", include_dir, include_path)); + + auto include_directories_data = + reinterpret_cast(cudf_jit_embed_sources_include_directories.bytes.data); + for (size_t i = 0; i < cudf_jit_embed_sources_include_directories.num_ranges; i++) { + auto range = cudf_jit_embed_sources_include_directories.ranges[i]; + auto dest_include_directory = include_directories_data + range.offset; + include_options.push_back(std::format("-I{}/{}", include_dir, dest_include_directory)); } std::vector options; - auto embed_options = reinterpret_cast(cudf_jit_embed_options.elements); - std::copy( - embed_options, embed_options + cudf_jit_embed_options.size, std::back_inserter(options)); - auto arch_flag = std::format("-arch=sm_{}", sm); + auto embed_options_data = reinterpret_cast(cudf_jit_embed_options.bytes.data); + + for (size_t i = 0; i < cudf_jit_embed_options.num_ranges; i++) { + auto range = cudf_jit_embed_options.ranges[i]; + auto option = embed_options_data + range.offset; + options.push_back(option); + } + + auto arch_flag = std::format("--gpu-architecture=sm_{}", sm); options.push_back(arch_flag.c_str()); - options.push_back("-dlto"); - options.push_back("-rdc=true"); - options.push_back("--split-compile=0"); - options.push_back("-default-device"); + options.push_back("--dlink-time-opt"); + options.push_back("--relocatable-device-code=true"); + // options.push_back("--split-compile=0"); + // options.push_back("--fdevice-time-trace=jit_comp_trace.json"); + // options.push_back("--minimal"); + // options.push_back("--time=compile_trace.json"); + // options.push_back("-time"); + // --fast-compile + options.push_back("--pch"); + options.push_back("--pch-dir=/tmp/cudf-rtc-pch"); // [ ] fix; make it consistent (hashing of header contents?) + + options.push_back("--device-as-default-execution-space"); for (auto const& include_option : include_options) { options.push_back(include_option.c_str()); @@ -286,6 +347,15 @@ fragment_t const& compile_fragment(char const* name, char const* source_code_cst auto frag = fragment_t::compile(params); + auto view = frag->get_lto_ir()->view(); + + auto end = std::chrono::high_resolution_clock::now(); + auto dur = end - begin; + std::cout << "RTC fragment compilation for `" << name << "` took " + << std::chrono::duration_cast(dur).count() << " us\n"; + + cache.store_blob_to_disk(cache_key_sha256, view); + prom.set_value(std::move(frag)); return *fut.get(); } @@ -332,12 +402,15 @@ kernel_ref compile_and_link_udf(char const* name, sm); auto const library_key_sha256 = hash_string(library_key); + // [ ] we also need to use the include dirs as part of the key auto& cache = get_rtc_cache(); // TODO: (atomicity) should probably use query_or_insert if (auto lib = cache.query_library(library_key_sha256); lib.has_value()) { + std::cout << "Loading kernel from memory\n"; return lib->get()->get_kernel(kernel_name); } else if (auto disk_lib = cache.query_blob_from_disk(library_key_sha256); disk_lib.has_value()) { + std::cout << "Loading kernel from disk cache\n"; std::promise prom; auto fut = std::shared_future{prom.get_future()}; @@ -351,12 +424,16 @@ kernel_ref compile_and_link_udf(char const* name, return fut.get()->get_kernel(kernel_name); } + std::cout << "Compiling and linking library\n"; auto const& library_frag = compile_library_fragment(); - auto const& udf_frag = compile_udf_fragment(udf_code, udf_key); + + auto const& udf_frag = compile_udf_fragment(udf_code, udf_key); std::promise prom; auto fut = std::shared_future{prom.get_future()}; - cache.store_library(library_key_sha256, fut); + // cache.store_library(library_key_sha256, fut); + + auto begin = std::chrono::high_resolution_clock::now(); blob_view const link_fragments[] = {library_frag.get_lto_ir()->view(), udf_frag.get_lto_ir()->view()}; @@ -364,6 +441,8 @@ kernel_ref compile_and_link_udf(char const* name, char const* const fragment_names[] = {"cudf_lto_library", "cudf_udf_fragment"}; + // TODO: run compilation tests at program startup + // TODO: optimization flags // TODO: split compile // TODO: split-compile-extended @@ -378,8 +457,12 @@ kernel_ref compile_and_link_udf(char const* name, // TODO: -nocache // TODO: -device-stack-protector auto arch_flag = std::format("-arch=sm_{}", sm); - char const* const link_options[] = { - "-lto", "-optimize-unused-variables", "-kernels-used=transform_kernel", arch_flag.c_str()}; + char const* const link_options[] = { // "-split-compile=0", + "-lto", // TODO: full flag names + + // "-optimize-unused-variables", + "-kernels-used=transform_kernel", + arch_flag.c_str()}; auto const params = library_t::link_params{.name = name, .output_type = binary_type::CUBIN, @@ -387,8 +470,14 @@ kernel_ref compile_and_link_udf(char const* name, .fragment_binary_types = fragment_binary_types, .fragment_names = fragment_names, .link_options = link_options}; - + // TODO: compilation flow logging with time taken, should be disabled when not in use auto lib = library_t::link(params); + + auto end = std::chrono::high_resolution_clock::now(); + auto dur = end - begin; + std::cout << "RTC library linking for `" << name << "` took " + << std::chrono::duration_cast(dur).count() << " us\n"; + // TODO: store to disk cache prom.set_value(std::move(lib)); return fut.get()->get_kernel(kernel_name); diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/rtc/cudf.hpp index 08b890587813..5ae4c3963c27 100644 --- a/cpp/src/jit/rtc/cudf.hpp +++ b/cpp/src/jit/rtc/cudf.hpp @@ -13,6 +13,8 @@ namespace CUDF_EXPORT cudf { namespace rtc { +void install_includes(char const* cache_dir); + kernel_ref compile_and_link_udf(char const* name, char const* kernel_name, char const* kernel_key, diff --git a/cpp/src/jit/rtc/includes/system/README.md b/cpp/src/jit/rtc/includes/system/README.md new file mode 100644 index 000000000000..60a941276325 --- /dev/null +++ b/cpp/src/jit/rtc/includes/system/README.md @@ -0,0 +1 @@ +# Doc \ No newline at end of file diff --git a/cpp/src/jit/rtc/sha256.hpp b/cpp/src/jit/rtc/sha256.hpp index 7eef7dd53049..51763b1bc9fc 100644 --- a/cpp/src/jit/rtc/sha256.hpp +++ b/cpp/src/jit/rtc/sha256.hpp @@ -4,6 +4,7 @@ */ #pragma once +#include #include #include @@ -30,6 +31,19 @@ struct [[nodiscard]] sha256_hex_string { [[nodiscard]] char const* c_str() const { return data_; } static constexpr size_t size() { return 64; } + + static sha256_hex_string make(std::span input) + { + CUDF_EXPECTS(input.size() == 32, "Input size must be 32 bytes for SHA256 hash"); + constexpr char const HEX_CHARS[] = "0123456789abcdef"; + sha256_hex_string hex; + for (size_t i = 0; i < 32; ++i) { + hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; + hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; + } + hex.data_[64] = '\0'; + return hex; + } }; struct [[nodiscard]] sha256_hash { @@ -42,16 +56,9 @@ struct [[nodiscard]] sha256_hash { constexpr bool operator!=(sha256_hash const& hash) const { return !(*this == hash); } - constexpr sha256_hex_string to_hex_string() const + sha256_hex_string to_hex_string() const { - constexpr char const HEX_CHARS[] = "0123456789abcdef"; - sha256_hex_string hex; - for (size_t i = 0; i < 32; ++i) { - hex.data_[i * 2] = HEX_CHARS[(data_[i] >> 4) & 0x0F]; - hex.data_[i * 2 + 1] = HEX_CHARS[data_[i] & 0x0F]; - } - hex.data_[64] = '\0'; - return hex; + return sha256_hex_string::make(std::span{data_, sizeof(data_)}); } }; diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 417950c37130..3b1608559e15 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -9,6 +9,7 @@ #include "io/utilities/getenv_or.hpp" #include "jit/cache.hpp" #include "jit/rtc/cache.hpp" +#include "jit/rtc/cudf.hpp" #include #include @@ -51,6 +52,7 @@ void context::initialize_components(init_flags flags) _program_cache = std::make_unique(); // TODO: Make cache directory configurable _rtc_cache = std::make_unique(true, "/tmp/cudf-rtc-cache", rtc::cache_limits{}); + cudf::rtc::install_includes("/tmp/cudf-rtc-cache"); } if (has_flag(new_flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index ea9adfa58706..c6e9a422779d 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -11,27 +11,47 @@ #include +#include + using namespace cudf; struct RTCTest : public ::testing::Test {}; TEST_F(RTCTest, CreateFragment) { - auto kern = rtc::compile_and_link_udf("test_fragment", - "transform_kernel", - "test_kernel_key", - R"***( + auto fn = []() { + auto begin = std::chrono::high_resolution_clock::now(); + auto kern = rtc::compile_and_link_udf("test_fragment", + "transform_kernel", + "test_kernel_key", + R"***( #include "cudf/jit/transform_params.cuh" - #include "jit/lto/operators.inl.cuh" - #include "jit/lto/types.inl.cuh" - - extern "C" __device__ void transform_operator(cudf::lto::transform_params const*){ + // #include "cudf/jit/lto/operators.cuh" + + extern "C" __device__ void transform_operator(cudf::lto::transform_params const* p){ + int a = 1; + int b = 2; + int c = 3; + int * out = (int *)p->outputs; + // cudf::lto::operators::add(&c, &a, &b); + // cudf::lto::operators::sub(&c, &a, &b); + // cudf::lto::operators::mul(&c, &a, &b); + // cudf::lto::operators::mul(&c, &a, &b); + *out = a + b * c; } )***", - "test_udf_key"); + "test_udf_key"); + + (void)kern; + auto end = std::chrono::high_resolution_clock::now(); + auto dur = end - begin; + std::cout << "RTC compilation took " + << std::chrono::duration_cast(dur).count() << " us\n"; + }; - (void)kern; + fn(); + fn(); } CUDF_TEST_PROGRAM_MAIN() From 2beabf0873ad468f4638a19553e34b7d6874aa40 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 12 Jan 2026 16:09:50 +0000 Subject: [PATCH 016/254] make cudf's context mt-safe on initialization --- cpp/include/cudf/context.hpp | 12 ++--- cpp/include/cudf_test/testing_main.hpp | 3 -- cpp/src/io/parquet/reader_impl.cpp | 2 +- cpp/src/jit/cache.cpp | 2 +- cpp/src/jit/row_ir.cpp | 4 +- cpp/src/runtime/context.cpp | 49 +++++++++------------ cpp/src/runtime/context.hpp | 13 +++--- cpp/tests/utilities_tests/context_tests.cpp | 31 ++----------- 8 files changed, 40 insertions(+), 76 deletions(-) diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index 57ba95f81078..e1d5d017924d 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -64,14 +64,10 @@ constexpr bool has_flag(init_flags flags, init_flags flag) noexcept return (flags | flag) == flags; } -/// @brief Initialize the cudf global context +/// @brief Initialize the cuDF global context /// @param flags Optional flags to control which initialization steps to perform. -/// Can be called multiple times to initialize additional components. If all selected -/// steps are already performed, the call has no effect. -void initialize(init_flags flags = init_flags::INIT_JIT_CACHE); - -/// @brief de-initialize the cudf global context -/// @throws std::runtime_error if the context is already de-initialized -void deinitialize(); +/// Can be called multiple times. Only the first call will have an effect. +/// @thread-safety This function is thread-safe. +void initialize(init_flags flags = init_flags::ALL); } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf_test/testing_main.hpp b/cpp/include/cudf_test/testing_main.hpp index c43baa916a90..5505c1639554 100644 --- a/cpp/include/cudf_test/testing_main.hpp +++ b/cpp/include/cudf_test/testing_main.hpp @@ -222,7 +222,6 @@ inline void init_cudf_test(int argc, char** argv, cudf::test::config const& conf #define CUDF_TEST_PROGRAM_MAIN() \ int main(int argc, char** argv) \ { \ - cudf::initialize(); \ ::testing::InitGoogleTest(&argc, argv); \ init_cudf_test(argc, argv); \ if (std::getenv("GTEST_CUDF_MEMORY_PEAK")) { \ @@ -231,11 +230,9 @@ inline void init_cudf_test(int argc, char** argv, cudf::test::config const& conf cudf::set_current_device_resource(&mr); \ auto rc = RUN_ALL_TESTS(); \ std::cout << "Peak memory usage " << mr.get_bytes_counter().peak << " bytes" << std::endl; \ - cudf::deinitialize(); \ return rc; \ } else { \ auto rc = RUN_ALL_TESTS(); \ - cudf::deinitialize(); \ return rc; \ } \ } diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index 1c612de61df5..abbbe73990ee 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -858,7 +858,7 @@ table_with_metadata reader_impl::finalize_output(read_mode mode, if (_num_filter_only_columns > 0) { out_metadata.schema_info.resize(output_count); } - bool use_jit = cudf::get_context().use_jit() || _options.use_jit_filter; + bool use_jit = cudf::get_context(init_flags::NONE).use_jit() || _options.use_jit_filter; if (!use_jit) { auto predicate = cudf::detail::compute_column( diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 7d03ed8fdbca..4d9ff9a0ec14 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -131,6 +131,6 @@ jitify2::ProgramCache<>& jit::program_cache::get(jitify2::PreprocessedProgramDat jitify2::ProgramCache<>& jit::get_program_cache(jitify2::PreprocessedProgramData const& preprog) { - return cudf::get_context().program_cache().get(preprog); + return cudf::get_context(init_flags::INIT_JIT_CACHE).program_cache().get(preprog); } } // namespace cudf diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index a0adc1489d78..c46a441b2818 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -586,7 +586,7 @@ transform_args ast_converter::compute_column(target target_id, is_null_aware, output_nullability}; - if (get_context().dump_codegen()) { + if (get_context(init_flags::NONE).dump_codegen()) { std::cout << "Generated code for transform: " << transform.udf << std::endl; } @@ -643,7 +643,7 @@ filter_args ast_converter::filter(target target_id, std::nullopt, is_null_aware}; - if (get_context().dump_codegen()) { + if (get_context(init_flags::NONE).dump_codegen()) { std::cout << "Generated code for filter: " << filter.predicate_udf << std::endl; } diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index d973d6fb9628..f49ca672e609 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -29,7 +29,7 @@ context::context(init_flags flags) : _program_cache{nullptr} jit::program_cache& context::program_cache() { - CUDF_EXPECTS(_program_cache != nullptr, "JIT cache not initialized", std::runtime_error); + CUDF_EXPECTS(_program_cache != nullptr, "JIT cache is not initialized", std::runtime_error); return *_program_cache; } @@ -37,32 +37,19 @@ bool context::dump_codegen() const { return _dump_codegen; } void context::initialize_components(init_flags flags) { - // Only initialize components that haven't been initialized yet - auto const new_flags = flags & ~_initialized_flags; - - if (has_flag(new_flags, init_flags::INIT_JIT_CACHE)) { + if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { _program_cache = std::make_unique(); } - if (has_flag(new_flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } + if (has_flag(flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } - _initialized_flags = _initialized_flags | new_flags; + _initialized_flags = flags; } bool context::use_jit() const { return _use_jit; } -std::unique_ptr& get_context_ptr_ref() -{ - static std::unique_ptr context; - return context; -} - -context& get_context() -{ - auto& ctx = get_context_ptr_ref(); - if (ctx == nullptr) { cudf::initialize(); } - return *ctx; -} +static std::unique_ptr ctx; +static std::once_flag ctx_init_flag{}; } // namespace cudf @@ -70,15 +57,21 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { - auto& ctx = get_context_ptr_ref(); - if (ctx == nullptr) { - // First initialization - create the context - ctx = std::make_unique(flags); - } else { - // Context already exists - initialize additional components - ctx->initialize_components(flags); - } + std::call_once(ctx_init_flag, [&]() { + auto c = std::make_unique(); + c->initialize_components(flags); + ctx = std::move(c); + }); + + CUDF_EXPECTS(has_flag(ctx->_initialized_flags, flags), + "CUDF's context has already been initialized with incompatible flags", + std::runtime_error); +} + +context& get_context(init_flags expected) +{ + initialize(expected); + return *ctx; } -void deinitialize() { get_context_ptr_ref().reset(); } } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index c0726d986337..d74bd7f924e8 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -38,15 +38,18 @@ class context { [[nodiscard]] bool dump_codegen() const; - /// @brief Initialize additional components based on the provided flags + [[nodiscard]] bool use_jit() const; + + private: + /// @brief Initialize components based on the provided flags /// @param flags The initialization flags to process void initialize_components(init_flags flags); - [[nodiscard]] bool use_jit() const; + friend void initialize(init_flags flags); }; -std::unique_ptr& get_context_ptr_ref(); - -context& get_context(); +/// @brief Get the cuDF global context +/// @param expected the initialization flags that are expected +context& get_context(init_flags expected); } // namespace cudf diff --git a/cpp/tests/utilities_tests/context_tests.cpp b/cpp/tests/utilities_tests/context_tests.cpp index 62e0c45c0145..37e15e3f6530 100644 --- a/cpp/tests/utilities_tests/context_tests.cpp +++ b/cpp/tests/utilities_tests/context_tests.cpp @@ -9,38 +9,13 @@ #include -struct ContextTest : public cudf::test::BaseFixture { - ~ContextTest() override - { - try { - cudf::deinitialize(); - } catch (...) { - } - } -}; +struct ContextTest : public cudf::test::BaseFixture {}; TEST_F(ContextTest, MultipleInitializeCalls) { cudf::initialize(cudf::init_flags::INIT_JIT_CACHE); - EXPECT_NO_THROW(cudf::initialize(cudf::init_flags::LOAD_NVCOMP)); - EXPECT_NO_THROW(cudf::initialize(cudf::init_flags::ALL)); -} - -TEST_F(ContextTest, InitializeAfterDeinitialize) -{ - cudf::initialize(cudf::init_flags::ALL); - cudf::deinitialize(); - + EXPECT_THROW(cudf::initialize(cudf::init_flags::LOAD_NVCOMP), std::runtime_error); + EXPECT_THROW(cudf::initialize(cudf::init_flags::ALL), std::runtime_error); EXPECT_NO_THROW(cudf::initialize(cudf::init_flags::INIT_JIT_CACHE)); } - -TEST_F(ContextTest, DeinitializeWithoutInitialize) { EXPECT_NO_THROW(cudf::deinitialize()); } - -TEST_F(ContextTest, MultipleDeinitializeCalls) -{ - cudf::initialize(cudf::init_flags::ALL); - cudf::deinitialize(); - - EXPECT_NO_THROW(cudf::deinitialize()); -} From 06136aa6c7bf5a319694b8b445c23f8f3803f885 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 12 Jan 2026 16:12:30 +0000 Subject: [PATCH 017/254] formatting --- cpp/include/cudf/context.hpp | 2 +- cpp/include/cudf_test/testing_main.hpp | 2 +- cpp/src/jit/cache.cpp | 2 +- cpp/src/runtime/context.cpp | 2 +- cpp/src/runtime/context.hpp | 2 +- cpp/tests/utilities_tests/context_tests.cpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index e1d5d017924d..6705deee0b69 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cudf_test/testing_main.hpp b/cpp/include/cudf_test/testing_main.hpp index 5505c1639554..4c22a17bed6f 100644 --- a/cpp/include/cudf_test/testing_main.hpp +++ b/cpp/include/cudf_test/testing_main.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 4d9ff9a0ec14..3d8ffc757de9 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index f49ca672e609..77d172b4cf0a 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index d74bd7f924e8..38fbc7d67cbe 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/utilities_tests/context_tests.cpp b/cpp/tests/utilities_tests/context_tests.cpp index 37e15e3f6530..25af0649c844 100644 --- a/cpp/tests/utilities_tests/context_tests.cpp +++ b/cpp/tests/utilities_tests/context_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ From a29e57b9c704ddd5388af5a4cb44305c66153a55 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 12 Jan 2026 16:25:47 +0000 Subject: [PATCH 018/254] docfix --- cpp/include/cudf/context.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index 6705deee0b69..76f279dc5fc2 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -67,7 +67,7 @@ constexpr bool has_flag(init_flags flags, init_flags flag) noexcept /// @brief Initialize the cuDF global context /// @param flags Optional flags to control which initialization steps to perform. /// Can be called multiple times. Only the first call will have an effect. -/// @thread-safety This function is thread-safe. +/// @note Thread-safety: This function is thread-safe. void initialize(init_flags flags = init_flags::ALL); } // namespace CUDF_EXPORT cudf From 71b376152c3be40a50199522d964673c9ef8052c Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 12 Jan 2026 17:49:03 +0000 Subject: [PATCH 019/254] fixed initialization of jit cache --- cpp/src/runtime/context.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 77d172b4cf0a..46e89c2cddd8 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -18,11 +18,16 @@ namespace cudf { context::context(init_flags flags) : _program_cache{nullptr} { - auto dump_codegen_flag = getenv_or("LIBCUDF_JIT_DUMP_CODEGEN", std::string{"OFF"}); - _dump_codegen = (dump_codegen_flag == "ON" || dump_codegen_flag == "1"); + auto dump_codegen_env = getenv_or("LIBCUDF_JIT_DUMP_CODEGEN", std::string{"OFF"}); + bool dump_codegen = + (dump_codegen_env == "ON" || dump_codegen_env == "on" || dump_codegen_env == "1"); - auto use_jit_flag = getenv_or("LIBCUDF_JIT_ENABLED", std::string{"OFF"}); - _use_jit = (use_jit_flag == "ON" || use_jit_flag == "1"); + auto use_jit_env = getenv_or("LIBCUDF_JIT_ENABLED", std::string{"OFF"}); + bool use_jit = (use_jit_env == "ON" || use_jit_env == "on" || use_jit_env == "1"); + + _dump_codegen = dump_codegen; + + flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); initialize_components(flags); } From 5df019b8f5cc84c2a36fd3ad3781136d965719ad Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 12 Jan 2026 22:18:21 +0000 Subject: [PATCH 020/254] added todo --- cpp/cmake/Modules/jit_embed.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py index 722894fa4bc4..7aaefa7dfa6b 100644 --- a/cpp/cmake/Modules/jit_embed.py +++ b/cpp/cmake/Modules/jit_embed.py @@ -334,6 +334,8 @@ def generate_cxx_source_files_data( num_null_terminators=0, ) + # TODO: add lz4 compression and decompression as options (default) + return f""" {file_destinations_decls.var().decl()} From b42b9988aa16482d0e605c993cc12b9aceff4409 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 13 Jan 2026 09:10:13 +0000 Subject: [PATCH 021/254] todos and refactoring --- cpp/src/jit/rtc/cache.cpp | 7 ++++--- cpp/src/jit/rtc/cudf.cpp | 9 +++++---- cpp/src/jit/rtc/rtc.cpp | 10 ++-------- cpp/src/jit/rtc/rtc.hpp | 5 ++--- 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index 3183254f3896..d0dfd8cc3c97 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -347,6 +347,8 @@ void cache_t::clear_disk_store() errno = 0; // reset errno before reading struct dirent* entry_iter = nullptr; + std::vector entry_path; + entry_path.resize(PATH_MAX + 1); while (true) { entry_iter = readdir(dir); @@ -360,14 +362,13 @@ void cache_t::clear_disk_store() } struct stat entry_stat; - char path[PATH_MAX + 1]; - if (lstat(path, &entry_stat) == -1) { + if (lstat(entry_path.data(), &entry_stat) == -1) { throw_posix("Failed to get file status for RTC cache clearing", "lstat"); } if (S_ISREG(entry_stat.st_mode)) { - if (unlink(path) == -1) { + if (unlink(entry_path.data()) == -1) { throw_posix("Failed to unlink RTC cache file during clearing", "unlink"); } } diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 572c9ec1ff2e..7a46d408ae63 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -331,7 +331,8 @@ fragment_t const& compile_fragment(char const* name, char const* source_code_cst // options.push_back("-time"); // --fast-compile options.push_back("--pch"); - options.push_back("--pch-dir=/tmp/cudf-rtc-pch"); // [ ] fix; make it consistent (hashing of header contents?) + options.push_back( + "--pch-dir=/tmp/cudf-rtc-pch"); // [ ] fix; make it consistent (hashing of header contents?) options.push_back("--device-as-default-execution-space"); @@ -347,7 +348,7 @@ fragment_t const& compile_fragment(char const* name, char const* source_code_cst auto frag = fragment_t::compile(params); - auto view = frag->get_lto_ir()->view(); + auto view = frag->get(binary_type::LTO_IR)->view(); auto end = std::chrono::high_resolution_clock::now(); auto dur = end - begin; @@ -435,8 +436,8 @@ kernel_ref compile_and_link_udf(char const* name, auto begin = std::chrono::high_resolution_clock::now(); - blob_view const link_fragments[] = {library_frag.get_lto_ir()->view(), - udf_frag.get_lto_ir()->view()}; + blob_view const link_fragments[] = {library_frag.get(binary_type::LTO_IR)->view(), + udf_frag.get(binary_type::LTO_IR)->view()}; binary_type const fragment_binary_types[] = {binary_type::LTO_IR, binary_type::LTO_IR}; char const* const fragment_names[] = {"cudf_lto_library", "cudf_udf_fragment"}; diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 63840bc5103c..fdb018637068 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -346,15 +346,9 @@ fragment fragment_t::compile(compile_params const& params) } } -blob const& fragment_t::get_lto_ir() const +blob const& fragment_t::get(binary_type type) const { - CUDF_EXPECTS(type_ == binary_type::LTO_IR, "Fragment does not contain LTO IR"); - return blob_; -} - -blob const& fragment_t::get_cubin() const -{ - CUDF_EXPECTS(type_ == binary_type::CUBIN, "Fragment does not contain CUBIN"); + CUDF_EXPECTS(type_ == type, "Fragment does not contain expected binary type"); return blob_; } diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index f25b9a30955e..130b30c8ad8c 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -131,9 +131,8 @@ struct [[nodiscard]] fragment_t { [[nodiscard]] binary_type get_type() const { return type_; } - [[nodiscard]] blob const& get_lto_ir() const; - - [[nodiscard]] blob const& get_cubin() const; + // TODO: lamarrr dumping of the byte code, LTO-IR: NVVM?, PTX: PTX Bytecode + [[nodiscard]] blob const& get(binary_type type) const; }; struct library_t; From ff2f88f05966d73503d0182e12067dece9384a8b Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 4 Feb 2026 13:46:51 +0000 Subject: [PATCH 022/254] update --- cpp/CMakeLists.txt | 7 +++++++ cpp/cmake/Modules/jit_embed.py | 2 ++ cpp/src/jit/rtc/cache.cpp | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bd3ddbb11b6a..aad5be31bf7c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -350,6 +350,13 @@ endif() set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) + +function(add_lto_ir_target) + +# TODO: implement LTO target addition + +endfunction() + list(APPEND CUDF_JIT_DEFINITIONS -DCUDF_RUNTIME_JIT -D__CUDACC_RTC__) list( APPEND diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py index 7aaefa7dfa6b..6d03c53f9041 100644 --- a/cpp/cmake/Modules/jit_embed.py +++ b/cpp/cmake/Modules/jit_embed.py @@ -383,6 +383,8 @@ def generate_embed_source(entries: dict[str, dict[str, dict]]) -> str: """ +# TODO: use ASM embed for blobs + # Usage: embed.py --input --output def main(): diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index d0dfd8cc3c97..a65731f7d516 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -93,7 +93,7 @@ void cache_t::store_blob_to_disk(sha256_hash const& sha, blob_view binary) if (!enabled_) { return; } - char temp_path[] = "/tmp/blob-XXXXXX"; + char temp_path[] = "/tmp/cudf-blob-XXXXXX"; { int fd = mkstemp(temp_path); From 44c1344e180ca8aa6ca0415c701779b7f067a598 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 6 Feb 2026 01:33:04 +0000 Subject: [PATCH 023/254] Refactor JIT compilation and caching system - Introduced a new `jit_bundle` class to encapsulate JIT installation and management. - Updated `cache_statistics` to include disk hit/miss counters for blobs and fragments. - Removed the custom `rw_spinlock_t` implementation in favor of standard synchronization mechanisms. - Enhanced the `cache_t` class to manage JIT bundle installation and caching more effectively. - Refactored file handling functions to improve clarity and error handling. - Updated kernel compilation logic to utilize the new JIT bundle structure. - Improved test cases for fragment creation to reflect changes in the JIT compilation process. --- cpp/CMakeLists.txt | 66 +-- cpp/cmake/Modules/JitEmbed.cmake | 110 +++- cpp/cmake/Modules/jit_embed.py | 378 +++++++++++-- cpp/src/jit/lto/library.cu | 5 + cpp/src/jit/rtc/README.md | 2 +- cpp/src/jit/rtc/cache.cpp | 360 ++++++------ cpp/src/jit/rtc/cache.hpp | 169 ++---- cpp/src/jit/rtc/cudf.cpp | 639 +++++++++++----------- cpp/src/jit/rtc/cudf.hpp | 34 +- cpp/src/jit/rtc/includes/system/README.md | 2 +- cpp/src/jit/rtc/rtc.cpp | 14 + cpp/src/jit/rtc/rtc.hpp | 6 + cpp/src/runtime/context.cpp | 13 +- cpp/src/runtime/context.hpp | 6 +- cpp/tests/jit/rtc.cpp | 15 +- 15 files changed, 1100 insertions(+), 719 deletions(-) create mode 100644 cpp/src/jit/lto/library.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index aad5be31bf7c..edc9b608f4de 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -350,22 +350,33 @@ endif() set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) - -function(add_lto_ir_target) - -# TODO: implement LTO target addition - -endfunction() - list(APPEND CUDF_JIT_DEFINITIONS -DCUDF_RUNTIME_JIT -D__CUDACC_RTC__) -list( - APPEND - CUDF_JIT_COMPILE_FLAGS - -std=c++${CUDF_CUDA_STANDARD} - # -Xcompiler=-fPIC -Xcompiler=-DPIC -fvisibility=hidden -Xfatbin=--compress-all - # --compress-mode=size - --device-int128 - ${CUDF_JIT_DEFINITIONS} +list(APPEND CUDF_JIT_COMPILE_FLAGS -std=c++${CUDF_CUDA_STANDARD} --device-int128 + ${CUDF_JIT_DEFINITIONS} +) + +set(CUDF_LTO_JIT_ARCHITECTURES "80-real;90-real;100-real;120-real") +add_library(cudf_lto_library OBJECT src/jit/lto/library.cu) +target_compile_options(cudf_lto_library PRIVATE -Xfatbin=--compress-all --compress-mode=size) +set_target_properties( + cudf_lto_library + PROPERTIES CUDA_ARCHITECTURES "${CUDF_LTO_JIT_ARCHITECTURES}" + CUDA_SEPARABLE_COMPILATION ON + CUDA_FATBIN_COMPILATION ON + POSITION_INDEPENDENT_CODE ON + INTERPROCEDURAL_OPTIMIZATION ON + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS ON + CXX_VISIBILITY_PRESET hidden + CUDA_STANDARD 20 + CUDA_STANDARD_REQUIRED ON + CUDA_VISIBILITY_PRESET hidden +) +target_link_libraries(cudf_lto_library PRIVATE CCCL::CCCL) +target_include_directories( + cudf_lto_library PRIVATE "$" + "$" ) jit_add_include_directory( @@ -378,11 +389,6 @@ jit_add_include_directory( INCLUDE_DIRECTORIES cudf/src cudf/src/jit/rtc/includes/system ) -# TODO: add library.cu LTO type checks need to be compiled as part of the AOT workflow to ensure -# type consistency between AOT and JIT. -# -# Must be in test suite - get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) @@ -392,20 +398,12 @@ foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) cudf_jit_embed COPY_DIRECTORY ${INC_DIR} DEST_DIRECTORY CCCL/libcudacxx/${INC_DIR_NAME} INCLUDE_DIRECTORIES CCCL/libcudacxx/${INC_DIR_NAME} ) - endforeach() - -# TODO: remove foreach(INC_DIR IN LISTS CUDAToolkit_INCLUDE_DIRS) message("CTKKK dir: ${INC_DIR}") -# cmake_path(GET INC_DIR FILENAME INC_DIR_NAME) - -# jit_add_include_directory( cudf_jit_embed COPY_DIRECTORY ${INC_DIR} DEST_DIRECTORY -# CUDAToolKit/${INC_DIR_NAME} INCLUDE_DIRECTORIES CUDAToolKit/${INC_DIR_NAME} ) - -# endforeach() - +# TODO(lamarrr): inspect the generated cubin and make sure it doesn't contain +# any unneeded symbols. it should be as small as possible jit_add_options(cudf_jit_embed OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) - -jit_embed(cudf_jit_embed) +jit_add_blob(cudf_jit_embed FILE "$" DEST cudf_lto_library.fatbin) +jit_embed(cudf_jit_embed COMPRESSION lz4) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- @@ -921,9 +919,11 @@ add_library( src/utilities/traits.cpp src/utilities/type_checks.cpp src/utilities/type_dispatcher.cpp + ${cudf_jit_embed_SOURCE_DIR}/embed.cpp ) + add_dependencies(cudf cudf_jit_embed) -target_include_directories(cudf PRIVATE ${cudf_jit_embed_INCLUDE_DIR}) +target_include_directories(cudf PRIVATE ${cudf_jit_embed_INCLUDE_DIRS}) # Anything that includes jitify needs to be compiled with _FILE_OFFSET_BITS=64 due to a limitation # in how conda builds glibc diff --git a/cpp/cmake/Modules/JitEmbed.cmake b/cpp/cmake/Modules/JitEmbed.cmake index 04cbe5d6e7b9..fc5033b4eacd 100644 --- a/cpp/cmake/Modules/JitEmbed.cmake +++ b/cpp/cmake/Modules/JitEmbed.cmake @@ -112,15 +112,62 @@ function(jit_add_options) endfunction() +function(jit_add_blob) + set(TARGET ${ARGV0}) + set(OPTIONS "") + set(ONE_VALUE_ARGS "FILE;DEST") + set(MULTI_VALUE_ARGS "") + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET argument is required") + endif() + + if(NOT ARG_FILE) + message(FATAL_ERROR "FILE argument is required") + endif() + + if(NOT ARG_DEST) + message(FATAL_ERROR "DEST argument is required") + endif() + + set(blob_files ${jitembed_${TARGET}_blob__files}) + set(blob_dests ${jitembed_${TARGET}_blob__dests}) + + list(APPEND blob_files "${ARG_FILE}") + list(APPEND blob_dests "${ARG_DEST}") + + set(jitembed_${TARGET}_blob__files + ${blob_files} + PARENT_SCOPE + ) + + set(jitembed_${TARGET}_blob__dests + ${blob_dests} + PARENT_SCOPE + ) +endfunction() + # pass the encoded args to the jit_embed.py script to generate the source and options maps function(jit_embed) set(TARGET ${ARGV0}) - cmake_parse_arguments(ARG "" "${ONE_VALUE_ARGS}" "" ${ARGN}) + set(OPTIONS "") + set(ONE_VALUE_ARGS "COMPRESSION") + set(MULTI_VALUE_ARGS "") + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) if(NOT DEFINED TARGET) message(FATAL_ERROR "TARGET argument is required") endif() + if(NOT DEFINED ARG_COMPRESSION) + message(FATAL_ERROR "COMPRESSION argument is required") + endif() + + if(NOT ARG_COMPRESSION STREQUAL "none" AND NOT ARG_COMPRESSION STREQUAL "lz4") + message(FATAL_ERROR "COMPRESSION argument must be either none or lz4") + endif() + string(APPEND TARGET_YAML "\"${TARGET}_sources\":\n") string(APPEND TARGET_YAML " type: \"sources\"\n") @@ -148,6 +195,8 @@ function(jit_embed) string(APPEND TARGET_YAML " - \"${INCLUDE_DIR}\"\n") endforeach() + string(APPEND TARGET_YAML " compression: ${ARG_COMPRESSION}\n") + endif() string(APPEND TARGET_YAML "\n\n") @@ -169,37 +218,68 @@ function(jit_embed) endif() + string(APPEND TARGET_YAML "\n\n") + + if(DEFINED jitembed_${TARGET}_blob__files) + + string(APPEND TARGET_YAML "\"${TARGET}_blobs\":\n") + string(APPEND TARGET_YAML " type: \"blobs\"\n") + + # gather blobs + string(APPEND TARGET_YAML " blobs:\n") + + list(LENGTH jitembed_${TARGET}_blob__files NUM_BLOBS) + math(EXPR LAST_BLOB_INDEX "${NUM_BLOBS} - 1") + foreach(i RANGE 0 ${LAST_BLOB_INDEX}) + list(GET jitembed_${TARGET}_blob__files ${i} BLOB_FILE) + list(GET jitembed_${TARGET}_blob__dests ${i} BLOB_DEST) + string(APPEND TARGET_YAML " - file: \"${BLOB_FILE}\"\n") + string(APPEND TARGET_YAML " dest: \"${BLOB_DEST}\"\n") + endforeach() + + string(APPEND TARGET_YAML " compression: ${ARG_COMPRESSION}\n") + + endif() + set(YAML_FILE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.yaml") - set(INCLUDE_DIR "${CUDF_GENERATED_INCLUDE_DIR}/include/jit_embed") - set(HEADER "${INCLUDE_DIR}/${TARGET}.h") + set(JIT_EMBED_DIR "${CUDF_GENERATED_INCLUDE_DIR}/jit_embed") + set(OUTPUT_DIR "${JIT_EMBED_DIR}/${TARGET}") # write CONFIG to temp file and pass file path to script - file(WRITE "${YAML_FILE_PATH}" "${TARGET_YAML}") + file( + GENERATE + OUTPUT "${YAML_FILE_PATH}" + CONTENT "${TARGET_YAML}" + ) add_custom_command( - OUTPUT ${HEADER} - COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" --output - "${HEADER}" --input "${YAML_FILE_PATH}" + OUTPUT ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.cpp ${OUTPUT_DIR}/embed.bin + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" --id + "${TARGET}" --output-dir "${OUTPUT_DIR}" --input "${YAML_FILE_PATH}" DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" "${YAML_FILE_PATH}" - ${jitembed_${TARGET}_incdir__source_files} + ${jitembed_${TARGET}_incdir__source_files} ${jitembed_${TARGET}_blob__files} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - COMMENT "Generating JIT embed for ${TARGET} (YAML: ${YAML_FILE_PATH}) into ${HEADER}" + COMMENT + "Generating JIT embed for ${TARGET} (YAML: ${YAML_FILE_PATH}) into ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.cpp ${OUTPUT_DIR}/embed.bin" VERBATIM ) - add_custom_target(${TARGET} ALL DEPENDS "${HEADER}") + add_custom_target( + ${TARGET} ALL DEPENDS ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.cpp ${OUTPUT_DIR}/embed.bin + ) message( STATUS - "JIT embed for target ${TARGET} (YAML: ${YAML_FILE_PATH}) will be generated into: ${HEADER}" + "JIT embed for target ${TARGET} (YAML: ${YAML_FILE_PATH}) will be generated into: ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.cpp ${OUTPUT_DIR}/embed.bin" ) - set(${TARGET}_INCLUDE_DIR - ${INCLUDE_DIR} + set(${TARGET}_INCLUDE_DIRS + "${CUDF_GENERATED_INCLUDE_DIR};${OUTPUT_DIR}" PARENT_SCOPE ) - set(${TARGET}_HEADER - ${HEADER} + + set(${TARGET}_SOURCE_DIR + ${OUTPUT_DIR} PARENT_SCOPE ) diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py index 6d03c53f9041..5dd23d73f494 100644 --- a/cpp/cmake/Modules/jit_embed.py +++ b/cpp/cmake/Modules/jit_embed.py @@ -2,16 +2,20 @@ # SPDX-License-Identifier: Apache-2.0 import argparse import hashlib +import logging +import os from typing import NamedTuple, Self +import lz4.frame import yaml -BYTE_TYPE = "unsigned char" -SIZE_TYPE = "unsigned long" -STORAGE_SPEC = "static constexpr" LIST_LINE_WIDTH = 32 NAMESPACE_PREFIX = "jit_" + +# TODO: write a schema validator for the input YAML + + ### json schema """entries @@ -25,10 +29,21 @@ } ], "include_directories": [string] + "compression": string }, $id: { "type": "strings", "strings": list[string] + }, + $id: { + "type": "blobs", + "blobs": [ + { + "file": string, + "dest": string + } + ], + "compression": string } ] """ @@ -41,25 +56,27 @@ extern "C" {{ typedef struct {NAMESPACE_PREFIX}bytes_t {{ - {BYTE_TYPE} const * data; - {SIZE_TYPE} size; + unsigned char const * data; + unsigned long size; }} {NAMESPACE_PREFIX}bytes_t; typedef struct {NAMESPACE_PREFIX}byte_range_t {{ - {SIZE_TYPE} offset; - {SIZE_TYPE} size; + unsigned long offset; + unsigned long size; }} {NAMESPACE_PREFIX}byte_range_t; typedef struct {NAMESPACE_PREFIX}bytes_array_t {{ {NAMESPACE_PREFIX}bytes_t bytes; {NAMESPACE_PREFIX}byte_range_t const * ranges; - {SIZE_TYPE} num_ranges; + unsigned long num_ranges; }} {NAMESPACE_PREFIX}bytes_array_t; }} """ +logger = logging.getLogger(__name__) + def list_string(strings: list[str]) -> str: lines = [] @@ -80,20 +97,33 @@ class CXXVarDecl(NamedTuple): @staticmethod def of_bytes(id: str, data: bytes, alignment: int) -> Self: byte_array = list_string([byte_hex_string(b) for b in data]) - expr = f"""alignas({alignment}) {STORAGE_SPEC} {BYTE_TYPE} const {id}[{len(data)}] = {{ + expr = f"""alignas({alignment}) static unsigned char const {id}[{len(data)}] = {{ {byte_array} }};""" return CXXVarDecl(id=id, expr=expr) @staticmethod def of_size(id: str, size: int) -> Self: - expr = f"{STORAGE_SPEC} {SIZE_TYPE} const {id} = {size}ULL;" + expr = f"static long const {id} = {size}L;" return CXXVarDecl(id=id, expr=expr) def decl(self: Self) -> str: return f"""{self.expr}""" +class CXXSizeDecl(NamedTuple): + id: str + size: int + + @staticmethod + def of_size(id: str, size: int) -> Self: + return CXXSizeDecl(id=id, size=size) + + def var(self: Self) -> CXXVarDecl: + expr = f"static unsigned long const {self.id} = {self.size}UL;" + return CXXVarDecl(id=self.id, expr=expr) + + class CXXSizeArrayDecl(NamedTuple): id: str sizes: list[int] @@ -103,8 +133,8 @@ def of_sizes(id: str, sizes: list[int]) -> Self: return CXXSizeArrayDecl(id=id, sizes=sizes) def var(self: Self) -> CXXVarDecl: - size_array = list_string([f"{size}ULL" for size in self.sizes]) - expr = f"""{STORAGE_SPEC} {SIZE_TYPE} const {self.id}[{len(self.sizes)}] = {{ + size_array = list_string([f"{size}UL" for size in self.sizes]) + expr = f"""static unsigned long const {self.id}[{len(self.sizes)}] = {{ {size_array} }};""" return CXXVarDecl(id=self.id, expr=expr) @@ -147,7 +177,7 @@ def var(self: Self) -> CXXVarDecl: {size_decl.decl()} -{STORAGE_SPEC} {NAMESPACE_PREFIX}bytes_t const {self.id} = {{ +static {NAMESPACE_PREFIX}bytes_t const {self.id} = {{ .data = {data_decl.id}, .size = {size_decl.id} }}; @@ -170,7 +200,7 @@ def var(self: Self) -> CXXVarDecl: ranges_str_formatted = list_string(ranges_str) - expr = f"""{STORAGE_SPEC} {NAMESPACE_PREFIX}byte_range_t const {self.id}[{len(self.ranges)}] = {{ + expr = f"""static {NAMESPACE_PREFIX}byte_range_t const {self.id}[{len(self.ranges)}] = {{ {ranges_str_formatted} }};""" return CXXVarDecl(id=self.id, expr=expr) @@ -213,7 +243,7 @@ def var(self: Self) -> CXXVarDecl: {ranges_decl.var().decl()} -{STORAGE_SPEC} {NAMESPACE_PREFIX}bytes_array_t const {self.id} = {{ +static {NAMESPACE_PREFIX}bytes_array_t const {self.id} = {{ .bytes = {bytes_decl.id}, .ranges = {ranges_decl.id}, .num_ranges = {len(self.ranges)} @@ -226,6 +256,53 @@ def var(self: Self) -> CXXVarDecl: ) +class CXXBinEmbedDecl(NamedTuple): + id: str + + @staticmethod + def of(id: str) -> Self: + return CXXBinEmbedDecl(id=id) + + def var(self: Self) -> CXXVarDecl: + return CXXVarDecl( + id=self.id, + expr=f""" + extern unsigned char const {self.id}_begin[]; + extern unsigned char const {self.id}_end[]; + +static {NAMESPACE_PREFIX}bytes_t const {self.id} = {{ + .data = {self.id}_begin, + .size = (unsigned long)({self.id}_end - {self.id}_begin) +}}; + + """, + ) + + +class CXXAsmEmbedDecl(NamedTuple): + id: str + file: str + + @staticmethod + def of_file(id: str, file: str) -> Self: + return CXXAsmEmbedDecl(id=id, file=file) + + def var(self: Self) -> CXXVarDecl: + return CXXVarDecl( + id=self.id, + expr=f""" + asm( + ".section .rodata\\n" + ".global {self.id}_begin\\n" + ".global {self.id}_end\\n" + "{self.id}_begin:\\n" + ".incbin \\"{self.file}\\"\\n" + "{self.id}_end:\\n" + ); +""", + ) + + def merge_bytes_with_null_terminators( bytes_lists: list[bytes], ) -> tuple[bytes, list[tuple[int, int]]]: @@ -239,14 +316,22 @@ def merge_bytes_with_null_terminators( return merged, ranges -def generate_cxx_strings_data(id: str, strings: list[str]) -> str: +class EmbedOutput(NamedTuple): + cxx_header: str + cxx_source: str + bin_file_name: str | None + bin_file_data: bytes | None + hash: bytes + + +def generate_cxx_strings_data(id: str, strings: list[str]) -> EmbedOutput: data, ranges = merge_bytes_with_null_terminators( [s.encode("utf-8") for s in strings] ) sha = hashlib.sha256() sha.update(data) - data_hash: bytes = sha.digest() + hash = sha.digest() arrays_decl = CXXArrayOfBytesDecl.of_byte_ranges( id=f"{id}", @@ -255,19 +340,18 @@ def generate_cxx_strings_data(id: str, strings: list[str]) -> str: ranges=ranges, ) - data_hash_decl: CXXBytesDecl = CXXBytesDecl.of_bytes( - id=f"{id}_hash", - data=data_hash, - alignment=1, - num_null_terminators=0, - ) - - return f""" + cxx_header = f""" {arrays_decl.var().decl()} - -{data_hash_decl.var().decl()} """ + return EmbedOutput( + cxx_header=cxx_header, + cxx_source="", + bin_file_name=None, + bin_file_data=None, + hash=hash, + ) + def load_file_bytes(file_path: str) -> bytes: with open(file_path, "rb") as f: @@ -278,12 +362,30 @@ def generate_cxx_source_files_data( id: str, file_paths: list[str], dests: list[str], - include_directories: list[str] = [], -) -> str: - files_bytes, files_ranges = merge_bytes_with_null_terminators( + include_directories: list[str], + compression: str, +) -> EmbedOutput: + uncompressed_files_bytes, files_ranges = merge_bytes_with_null_terminators( [load_file_bytes(p) for p in file_paths] ) + assert compression in ("none", "lz4"), "Invalid compression type" + compress = compression != "none" + + compressed_files_bytes = ( + lz4.frame.compress( + uncompressed_files_bytes, + compression_level=lz4.frame.COMPRESSIONLEVEL_MAX, + ) + if compress + else None + ) + + if compress: + logger.info( + f"{id}'s uncompressed size is {len(uncompressed_files_bytes)} bytes, compressed size is {len(compressed_files_bytes)} bytes" + ) + merged_dests_bytes, merged_dests_ranges = ( merge_bytes_with_null_terminators([d.encode("utf-8") for d in dests]) ) @@ -296,9 +398,10 @@ def generate_cxx_source_files_data( # compute combined sha256 hash of all files sha = hashlib.sha256() - sha.update(files_bytes) + sha.update(uncompressed_files_bytes) sha.update(merged_dests_bytes) sha.update(merged_include_directories_bytes) + sha.update(compression.encode("utf-8")) hash: bytes = sha.digest() @@ -311,11 +414,20 @@ def generate_cxx_source_files_data( ) ) - file_data_decl: CXXArrayOfBytesDecl = CXXArrayOfBytesDecl.of_byte_ranges( - id=f"{id}_file_data", - data=files_bytes, - ranges=files_ranges, - alignment=1, + binary_file_name = f"{id}_binary.bin" + + binary_decl: CXXBinEmbedDecl = CXXBinEmbedDecl.of(id=f"{id}_binary") + + binary_size_decl: CXXSizeDecl = CXXSizeDecl.of_size( + id=f"{id}_uncompressed_size", size=len(uncompressed_files_bytes) + ) + + binary_embed_decl: CXXAsmEmbedDecl = CXXAsmEmbedDecl.of_file( + id=f"{id}_binary", file=binary_file_name + ) + + binary_ranges_decl = CXXRangesDecl.of_ranges( + id=f"{id}_ranges", ranges=files_ranges ) include_directories_decls: CXXArrayOfBytesDecl = ( @@ -327,31 +439,126 @@ def generate_cxx_source_files_data( ) ) - hash_decl: CXXBytesDecl = CXXBytesDecl.of_bytes( - id=f"{id}_hash", - data=hash, - alignment=1, - num_null_terminators=0, + cxx_header = f""" +{file_destinations_decls.var().decl()} + +{binary_decl.var().decl()} + +{binary_size_decl.var().decl()} + +{binary_ranges_decl.var().decl()} + +{include_directories_decls.var().decl()} + +""" + + cxx_source = f""" +{binary_embed_decl.var().decl()} +""" + + return EmbedOutput( + cxx_header=cxx_header, + cxx_source=cxx_source, + bin_file_name=binary_file_name, + bin_file_data=compressed_files_bytes + if compress + else uncompressed_files_bytes, + hash=hash, + ) + + +def generate_cxx_blobs_data( + id: str, blob_paths: list[str], dests: list[str], compression: str +) -> EmbedOutput: + uncompressed_blob_bytes, blob_ranges = merge_bytes_with_null_terminators( + [load_file_bytes(p) for p in blob_paths] + ) + + assert compression in ("none", "lz4"), "Invalid compression type" + compress = compression != "none" + + compressed_blob_bytes = ( + lz4.frame.compress( + uncompressed_blob_bytes, + compression_level=lz4.frame.COMPRESSIONLEVEL_MAX, + ) + if compress + else None + ) + + merged_dests_bytes, merged_dests_ranges = ( + merge_bytes_with_null_terminators([d.encode("utf-8") for d in dests]) + ) + + if compress: + logger.info( + f"{id}'s uncompressed size is {len(uncompressed_blob_bytes)} bytes, compressed size is {len(compressed_blob_bytes)} bytes" + ) + + # compute combined sha256 hash of all files + sha = hashlib.sha256() + sha.update(uncompressed_blob_bytes) + sha.update(merged_dests_bytes) + sha.update(compression.encode("utf-8")) + + hash: bytes = sha.digest() + + file_destinations_decls: CXXArrayOfBytesDecl = ( + CXXArrayOfBytesDecl.of_byte_ranges( + id=f"{id}_file_destinations", + data=merged_dests_bytes, + ranges=merged_dests_ranges, + alignment=1, + ) ) - # TODO: add lz4 compression and decompression as options (default) + binary_file_name = f"{id}_binary.bin" + + binary_decl: CXXBinEmbedDecl = CXXBinEmbedDecl.of(id=f"{id}_binary") + + binary_size_decl: CXXSizeDecl = CXXSizeDecl.of_size( + id=f"{id}_uncompressed_size", size=len(uncompressed_blob_bytes) + ) + + binary_embed_decl: CXXAsmEmbedDecl = CXXAsmEmbedDecl.of_file( + id=f"{id}_binary", file=binary_file_name + ) + + binary_ranges_decl = CXXRangesDecl.of_ranges( + id=f"{id}_ranges", ranges=blob_ranges + ) + + cxx_header = f""" - return f""" {file_destinations_decls.var().decl()} -{file_data_decl.var().decl()} +{binary_decl.var().decl()} -{include_directories_decls.var().decl()} +{binary_size_decl.var().decl()} -{hash_decl.var().decl()} +{binary_ranges_decl.var().decl()} """ + cxx_source = f""" +{binary_embed_decl.var().decl()} +""" -# TODO: write a schema validator for the input YAML + return EmbedOutput( + cxx_header=cxx_header, + cxx_source=cxx_source, + bin_file_name=binary_file_name, + bin_file_data=compressed_blob_bytes + if compress + else uncompressed_blob_bytes, + hash=hash, + ) -def generate_embed_source(entries: dict[str, dict[str, dict]]) -> str: - code: str = "" +def generate_embed( + id: str, entries: dict[str, dict[str, dict]], output_dir: str +): + outputs: list[EmbedOutput] = [] + sha = hashlib.sha256() for entry_id, entry_value in entries.items(): entry_type = entry_value["type"] @@ -361,37 +568,87 @@ def generate_embed_source(entries: dict[str, dict[str, dict]]) -> str: file_paths = [s["file"] for s in sources] dests = [s["dest"] for s in sources] include_directories: list[str] = entry_value["include_directories"] - code += generate_cxx_source_files_data( - entry_id, file_paths, dests, include_directories + compression = entry_value["compression"] + output = generate_cxx_source_files_data( + entry_id, file_paths, dests, include_directories, compression ) + sha.update(output.hash) + outputs.append(output) elif entry_type == "strings": options: list[str] = entry_value["strings"] - code += generate_cxx_strings_data(entry_id, options) + output = generate_cxx_strings_data(entry_id, options) + sha.update(output.hash) + outputs.append(output) + + elif entry_type == "blobs": + blobs: list[str] = entry_value["blobs"] + file_paths = [s["file"] for s in blobs] + dests = [s["dest"] for s in blobs] + compression = entry_value["compression"] + output = generate_cxx_blobs_data( + entry_id, file_paths, dests, compression + ) + sha.update(output.hash) + outputs.append(output) else: raise ValueError(f"Unknown type: {entry_type}") - return f""" + hash = sha.digest() + + hash_decl: CXXBytesDecl = CXXBytesDecl.of_bytes( + id=f"{id}_hash", + data=hash, + alignment=1, + num_null_terminators=0, + ) + + cxx_header = f""" {PREAMBLE} extern "C" {{ -{code} + +{hash_decl.var().decl()} + +{"\n\n".join([output.cxx_header for output in outputs])} }} """ -# TODO: use ASM embed for blobs + cxx_source = f""" +{"\n\n".join([output.cxx_source for output in outputs])} +""" + + os.makedirs(output_dir, exist_ok=True) + + with open(f"{output_dir}/embed.hpp", "w") as f: + f.write(cxx_header) + with open(f"{output_dir}/embed.cpp", "w") as f: + f.write(cxx_source) -# Usage: embed.py --input --output + for output in outputs: + if output.bin_file_name and output.bin_file_data: + with open(f"{output_dir}/{output.bin_file_name}", "wb") as f: + f.write(output.bin_file_data) + + +# Usage: embed.py --id --input --output-dir def main(): parser = argparse.ArgumentParser( description="Embed headers, options, or binary blobs into C++ source code." ) + parser.add_argument( + "--id", + type=str, + required=True, + help="Identifier for the output", + ) + parser.add_argument( "--input", type=str, @@ -400,17 +657,18 @@ def main(): ) parser.add_argument( - "--output", type=str, required=True, help="Output C++ source file" + "--output-dir", + type=str, + required=True, + help="Output directory for generated files", ) args = parser.parse_args() with open(args.input, "rb") as f: description = yaml.safe_load(f) - code = generate_embed_source(description) - with open(args.output, "w") as f: - f.write(code) + generate_embed(args.id, description, args.output_dir) if __name__ == "__main__": diff --git a/cpp/src/jit/lto/library.cu b/cpp/src/jit/lto/library.cu new file mode 100644 index 000000000000..7865c2acbad3 --- /dev/null +++ b/cpp/src/jit/lto/library.cu @@ -0,0 +1,5 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#include diff --git a/cpp/src/jit/rtc/README.md b/cpp/src/jit/rtc/README.md index 8e20dbcbf48b..4f33fa6b665d 100644 --- a/cpp/src/jit/rtc/README.md +++ b/cpp/src/jit/rtc/README.md @@ -4,4 +4,4 @@ # Design -# How to Achieve Fast Compilation with LTO-IR \ No newline at end of file +# How to Achieve Fast Compilation with LTO-IR diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index a65731f7d516..4cb0b8b6d7da 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -35,9 +35,8 @@ namespace { } // namespace -cache_t::cache_t(bool enabled, std::string cache_dir, cache_limits const& limits) - : enabled_{enabled}, - cache_dir_{std::move(cache_dir)}, +cache_t::cache_t(std::string cache_dir, cache_limits const& limits) + : cache_dir_{std::move(cache_dir)}, limits_{limits}, blobs_cache_{limits.num_blobs}, fragments_cache_{limits.num_fragments}, @@ -53,46 +52,62 @@ cache_t::cache_t(bool enabled, std::string cache_dir, cache_limits const& limits } } -bool cache_t::is_enabled() -{ - std::atomic_ref c{enabled_}; - return c.load(std::memory_order_relaxed); -} +std::string const& cache_t::get_cache_dir() { return cache_dir_; } -void cache_t::enable() +std::optional blob_t::from_file(char const* path) { - std::atomic_ref c{enabled_}; - c.store(true, std::memory_order_relaxed); -} + int fd = open(path, O_RDONLY); -void cache_t::disable() -{ - std::atomic_ref c{enabled_}; - c.store(false, std::memory_order_relaxed); -} + if (fd == -1) { + if (errno == ENOENT) { + return std::nullopt; + } else { + throw_posix("Failed to open RTC cache file from disk", "open"); + } + } -std::string const& cache_t::get_cache_dir() { return cache_dir_; } + auto file_size = lseek(fd, 0, SEEK_END); + if (file_size == -1) { throw_posix("Failed to determine size of RTC cache file", "lseek"); } -void cache_t::store_blob_to_memory(sha256_hash const& sha, std::shared_future binary) -{ - CUDF_FUNC_RANGE(); + void* map = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); - if (!enabled_) { return; } + if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTC cache file", "mmap"); } - std::atomic_ref tick{tick_}; - auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + if (close(fd) == -1) { + throw_posix("Failed to close RTC cache file after memory-mapping", "close"); + } - detail::write_guard guard{blobs_cache_.lock_}; + auto deleter = +[](void*, uint8_t const* buffer, size_t size) { + if (munmap(static_cast(const_cast(buffer)), size) == -1) { + throw_posix("Failed to unmap RTC cache file from memory", "munmap"); + } + }; - blobs_cache_.insert(sha, std::move(binary), current_tick); + return blob_t::from_parts(static_cast(map), file_size, nullptr, deleter); } -void cache_t::store_blob_to_disk(sha256_hash const& sha, blob_view binary) +namespace { + +std::optional get_disk_blob(std::string const& cache_dir, + std::string const& object_type, + sha256_hash const& sha) { - CUDF_FUNC_RANGE(); + auto hex = sha.to_hex_string(); + auto path = std::format("{}/{}.{}.bin", cache_dir, object_type, hex.view()); - if (!enabled_) { return; } + auto blob = blob_t::from_file(path.c_str()); + if (!blob.has_value()) { return std::nullopt; } + { + return std::make_shared(std::move(*blob)); + } +} + +void add_blob_to_disk(std::string const& cache_dir, + std::string const& object_type, + sha256_hash const& sha, + blob_view binary) +{ char temp_path[] = "/tmp/cudf-blob-XXXXXX"; { @@ -109,7 +124,7 @@ void cache_t::store_blob_to_disk(sha256_hash const& sha, blob_view binary) } auto hex = sha.to_hex_string(); - auto final_path = std::format("{}/{}.blob", cache_dir_, hex.view()); + auto final_path = std::format("{}/{}.{}.bin", cache_dir, object_type, hex.view()); std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); @@ -131,149 +146,209 @@ void cache_t::store_blob_to_disk(sha256_hash const& sha, blob_view binary) } } -std::optional> cache_t::query_blob_from_memory(sha256_hash const& sha) +} // namespace + +std::shared_future cache_t::query_or_insert_blob(sha256_hash const& sha, + std::function maker) { CUDF_FUNC_RANGE(); - if (!enabled_) { return std::nullopt; } - std::atomic_ref tick{tick_}; auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); - { - detail::read_guard guard{blobs_cache_.lock_}; - auto const it = blobs_cache_.entries_.find(sha); + bool unlocked = false; + lock_.lock(); - if (it != blobs_cache_.entries_.end()) { - counter_.hit_memory_blob(); - it->second.hit(current_tick); - return it->second.value; - } else { - counter_.miss_memory_blob(); - return std::nullopt; - } - } -} + CUDF_DEFER([&] { + if (!unlocked) { lock_.unlock(); } + }); -std::optional cache_t::query_blob_from_disk(sha256_hash const& sha) -{ - CUDF_FUNC_RANGE(); + auto const it = blobs_cache_.entries_.find(sha); + // check memory cache + if (it != blobs_cache_.entries_.end()) { + counter_.hit_memory_blob(); - if (!enabled_) { return std::nullopt; } + // update LRU tick + it->second.hit(current_tick); + return it->second.value; - auto hex = sha.to_hex_string(); - auto path = std::format("{}/{}.blob", cache_dir_, hex.view()); + } else { + counter_.miss_memory_blob(); - int fd = open(path.c_str(), O_RDONLY); + // check disk cache + auto disk_blob = get_disk_blob(cache_dir_, "blob", sha); - if (fd == -1) { - if (errno == ENOENT) { - counter_.miss_disk_blob(); - return std::nullopt; - } else { - throw_posix("Failed to open RTC cache file from disk", "open"); - } - } + if (disk_blob.has_value()) { + counter_.hit_disk_blob(); + std::promise promise; + promise.set_value(std::move(*disk_blob)); - auto file_size = lseek(fd, 0, SEEK_END); - if (file_size == -1) { throw_posix("Failed to determine size of RTC cache file", "lseek"); } + auto fut = promise.get_future(); - void* map = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); + // insert into cache + blobs_cache_.insert(sha, fut.share(), current_tick); + return fut.share(); - if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTC cache file", "mmap"); } + } else { + counter_.miss_disk_blob(); - if (close(fd) == -1) { - throw_posix("Failed to close RTC cache file after memory-mapping", "close"); - } + std::promise promise; + auto fut = promise.get_future(); + blobs_cache_.insert(sha, fut.share(), current_tick); - auto deleter = +[](void*, uint8_t const* buffer, size_t size) { - if (munmap(static_cast(const_cast(buffer)), size) == -1) { - throw_posix("Failed to unmap RTC cache file from memory", "munmap"); - } - }; + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this sha + lock_.unlock(); + unlocked = true; + + auto result = maker(); + promise.set_value(result); - counter_.hit_disk_blob(); + // store result to disk + add_blob_to_disk(cache_dir_, "blob", sha, result->view()); - return std::make_shared( - blob_t::from_parts(static_cast(map), file_size, nullptr, deleter)); + return fut.share(); + } + } } -void cache_t::store_fragment(sha256_hash const& sha, std::shared_future frag) +std::shared_future cache_t::query_or_insert_fragment(sha256_hash const& sha, + binary_type type, + std::function maker) { CUDF_FUNC_RANGE(); - if (!enabled_) { return; } - std::atomic_ref tick{tick_}; auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); - { - detail::write_guard guard{fragments_cache_.lock_}; + bool unlocked = false; + lock_.lock(); - fragments_cache_.insert(sha, std::move(frag), current_tick); - } -} + CUDF_DEFER([&] { + if (!unlocked) { lock_.unlock(); } + }); -std::optional> cache_t::query_fragment(sha256_hash const& sha) -{ - CUDF_FUNC_RANGE(); + auto const it = fragments_cache_.entries_.find(sha); + // check memory cache + if (it != fragments_cache_.entries_.end()) { + counter_.hit_memory_fragment(); + // update LRU tick + it->second.hit(current_tick); + return it->second.value; + } else { + counter_.miss_memory_fragment(); - if (!enabled_) { return std::nullopt; } + // check disk cache + auto disk_blob = get_disk_blob(cache_dir_, "fragment", sha); + if (disk_blob.has_value()) { + counter_.hit_disk_fragment(); - std::atomic_ref tick{tick_}; - auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + std::promise promise; + auto fut = promise.get_future(); + fragments_cache_.insert(sha, fut.share(), current_tick); - { - detail::read_guard guard{fragments_cache_.lock_}; + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this sha + lock_.unlock(); + unlocked = true; + + fragment_t::load_params load_params{.binary = *disk_blob, .type = type}; + + auto frag = fragment_t::load(load_params); + promise.set_value(std::move(frag)); + + return fut.share(); - auto const it = fragments_cache_.entries_.find(sha); - if (it != fragments_cache_.entries_.end()) { - counter_.hit_memory_fragment(); - it->second.hit(current_tick); - return it->second.value; } else { - counter_.miss_memory_fragment(); - return std::nullopt; + counter_.miss_disk_fragment(); + + std::promise promise; + + auto fut = promise.get_future(); + fragments_cache_.insert(sha, fut.share(), current_tick); + + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this sha + lock_.unlock(); + unlocked = true; + + auto result = maker(); + promise.set_value(result); + + // store result to disk + add_blob_to_disk(cache_dir_, "fragment", sha, result->get(type)->view()); + + return fut.share(); } } } -void cache_t::store_library(sha256_hash const& sha, std::shared_future mod) +std::shared_future cache_t::query_or_insert_library( + sha256_hash const& sha, binary_type type, std::function()> maker) { CUDF_FUNC_RANGE(); - if (!enabled_) { return; } - std::atomic_ref tick{tick_}; auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); - { - detail::write_guard guard{libraries_cache_.lock_}; + bool unlocked = false; + lock_.lock(); - libraries_cache_.insert(sha, std::move(mod), current_tick); - } -} + CUDF_DEFER([&] { + if (!unlocked) { lock_.unlock(); } + }); -std::optional> cache_t::query_library(sha256_hash const& sha) -{ - CUDF_FUNC_RANGE(); + auto const it = libraries_cache_.entries_.find(sha); + // check memory cache + if (it != libraries_cache_.entries_.end()) { + counter_.hit_memory_library(); + // update LRU tick + it->second.hit(current_tick); + return it->second.value; + } else { + counter_.miss_memory_library(); - if (!enabled_) { return std::nullopt; } + // check disk cache + auto disk_blob = get_disk_blob(cache_dir_, "library", sha); + if (disk_blob.has_value()) { + counter_.hit_disk_library(); - std::atomic_ref tick{tick_}; - auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + std::promise promise; + auto fut = promise.get_future(); + libraries_cache_.insert(sha, fut.share(), current_tick); - { - detail::read_guard guard{libraries_cache_.lock_}; + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this sha + lock_.unlock(); + unlocked = true; + + library_t::load_params load_params{.binary = (*disk_blob)->view(), .type = type}; + + auto lib = library_t::load(load_params); + promise.set_value(std::move(lib)); + + return fut.share(); - auto const it = libraries_cache_.entries_.find(sha); - if (it != libraries_cache_.entries_.end()) { - counter_.hit_memory_library(); - it->second.hit(current_tick); - return it->second.value; } else { - counter_.miss_memory_library(); - return std::nullopt; + counter_.miss_disk_library(); + + std::promise promise; + + auto fut = promise.get_future(); + libraries_cache_.insert(sha, fut.share(), current_tick); + + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this sha + lock_.unlock(); + unlocked = true; + + auto [library, blob] = maker(); + promise.set_value(library); + + // store result to disk + add_blob_to_disk(cache_dir_, "library", sha, blob->view()); + + return fut.share(); } } } @@ -286,52 +361,31 @@ cache_limits cache_t::get_limits() { return limits_; } size_t cache_t::get_blob_count() { - CUDF_FUNC_RANGE(); - - { - detail::read_guard guard{blobs_cache_.lock_}; - return blobs_cache_.entries_.size(); - } + std::lock_guard guard{lock_}; + return blobs_cache_.entries_.size(); } size_t cache_t::get_fragment_count() { - CUDF_FUNC_RANGE(); - - { - detail::read_guard guard{fragments_cache_.lock_}; - return fragments_cache_.entries_.size(); - } + std::lock_guard guard{lock_}; + return fragments_cache_.entries_.size(); } size_t cache_t::get_library_count() { - CUDF_FUNC_RANGE(); - - { - detail::read_guard guard{libraries_cache_.lock_}; - return libraries_cache_.entries_.size(); - } + std::lock_guard guard{lock_}; + return libraries_cache_.entries_.size(); } void cache_t::clear_memory_store() { CUDF_FUNC_RANGE(); - { - detail::write_guard guard{blobs_cache_.lock_}; - blobs_cache_.entries_.clear(); - } - - { - detail::write_guard guard{fragments_cache_.lock_}; - fragments_cache_.entries_.clear(); - } + std::lock_guard guard{lock_}; - { - detail::write_guard guard{libraries_cache_.lock_}; - libraries_cache_.entries_.clear(); - } + blobs_cache_.entries_.clear(); + fragments_cache_.entries_.clear(); + libraries_cache_.entries_.clear(); } void cache_t::clear_disk_store() diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index ca3b9bbfeca6..4e52b2f0cadf 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -13,7 +13,6 @@ #include #include #include -#include #include namespace cudf { @@ -22,12 +21,16 @@ namespace rtc { struct [[nodiscard]] cache_statistics { uint64_t blob_memory_hits = 0; uint64_t blob_memory_misses = 0; + uint64_t blob_disk_hits = 0; + uint64_t blob_disk_misses = 0; uint64_t fragment_memory_hits = 0; uint64_t fragment_memory_misses = 0; + uint64_t fragment_disk_hits = 0; + uint64_t fragment_disk_misses = 0; uint64_t library_memory_hits = 0; uint64_t library_memory_misses = 0; - uint64_t blob_disk_hits = 0; - uint64_t blob_disk_misses = 0; + uint64_t library_disk_hits = 0; + uint64_t library_disk_misses = 0; }; struct [[nodiscard]] cache_limits { @@ -38,96 +41,6 @@ struct [[nodiscard]] cache_limits { namespace detail { -struct rw_spinlock_t { - private: - static constexpr size_t WRITE_STATE = ~size_t{0}; - static constexpr size_t IDLE_STATE = 0; - - size_t state_; - - public: - rw_spinlock_t() : state_{IDLE_STATE} {} - rw_spinlock_t(rw_spinlock_t const&) = default; - rw_spinlock_t& operator=(rw_spinlock_t const&) = default; - rw_spinlock_t(rw_spinlock_t&&) = default; - rw_spinlock_t& operator=(rw_spinlock_t&&) = default; - ~rw_spinlock_t() = default; - - void lock_read() - { - std::atomic_ref state{state_}; - - auto expected = IDLE_STATE; - auto target = size_t{1}; - - while (!state.compare_exchange_weak( - expected, target, std::memory_order_acquire, std::memory_order_relaxed)) { - if (expected == WRITE_STATE) { - expected = IDLE_STATE; - target = 1; - } else { - target = expected + 1; - } - } - } - - void unlock_read() - { - std::atomic_ref state{state_}; - state.fetch_sub(1, std::memory_order_relaxed); - } - - void lock_write() - { - std::atomic_ref state{state_}; - - auto expected = IDLE_STATE; - - while (!state.compare_exchange_weak( - expected, WRITE_STATE, std::memory_order_acquire, std::memory_order_relaxed)) { - expected = IDLE_STATE; - } - } - - void unlock_write() - { - std::atomic_ref state{state_}; - state.store(IDLE_STATE, std::memory_order_release); - } -}; - -template -struct read_guard { - private: - Lock& lock_; - - public: - read_guard(Lock& lock) : lock_{lock} { lock_.lock_read(); } - - read_guard(read_guard const&) = delete; - read_guard& operator=(read_guard const&) = delete; - read_guard(read_guard&&) = delete; - read_guard& operator=(read_guard&&) = delete; - - ~read_guard() { lock_.unlock_read(); } -}; - -template -struct write_guard { - private: - Lock& lock_; - - public: - write_guard(Lock& lock) : lock_{lock} { lock_.lock_write(); } - - write_guard(write_guard const&) = delete; - write_guard& operator=(write_guard const&) = delete; - write_guard(write_guard&&) = delete; - write_guard& operator=(write_guard&&) = delete; - - ~write_guard() { lock_.unlock_write(); } -}; - inline constexpr size_t CACHELINE_ALIGNMENT = 64; // = std::hardware_destructive_interference_size */ @@ -141,10 +54,9 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { }; std::unordered_map entries_; - rw_spinlock_t lock_; size_t limit_; - explicit lru_memory_cache(size_t limit) : entries_{}, lock_{}, limit_{limit} + explicit lru_memory_cache(size_t limit) : entries_{}, limit_{limit} { // reserve space to avoid rehashing entries_.reserve(limit * 2); @@ -202,51 +114,71 @@ struct alignas(CACHELINE_ALIGNMENT) counter { struct cache_statistics_counter { counter blob_memory_hits; counter blob_memory_misses; + counter blob_disk_hits; + counter blob_disk_misses; counter fragment_memory_hits; counter fragment_memory_misses; + counter fragment_disk_hits; + counter fragment_disk_misses; counter library_memory_hits; counter library_memory_misses; - counter blob_disk_hits; - counter blob_disk_misses; + counter library_disk_hits; + counter library_disk_misses; void clear() { blob_memory_hits.reset(); blob_memory_misses.reset(); + blob_disk_hits.reset(); + blob_disk_misses.reset(); fragment_memory_hits.reset(); fragment_memory_misses.reset(); + fragment_disk_hits.reset(); + fragment_disk_misses.reset(); library_memory_hits.reset(); library_memory_misses.reset(); - blob_disk_hits.reset(); - blob_disk_misses.reset(); + library_disk_hits.reset(); + library_disk_misses.reset(); } void hit_memory_blob() { blob_memory_hits.increment(); } void miss_memory_blob() { blob_memory_misses.increment(); } + void hit_disk_blob() { blob_disk_hits.increment(); } + + void miss_disk_blob() { blob_disk_misses.increment(); } + void hit_memory_fragment() { fragment_memory_hits.increment(); } void miss_memory_fragment() { fragment_memory_misses.increment(); } + void hit_disk_fragment() { fragment_disk_hits.increment(); } + + void miss_disk_fragment() { fragment_disk_misses.increment(); } + void hit_memory_library() { library_memory_hits.increment(); } void miss_memory_library() { library_memory_misses.increment(); } - void hit_disk_blob() { blob_disk_hits.increment(); } + void hit_disk_library() { library_disk_hits.increment(); } - void miss_disk_blob() { blob_disk_misses.increment(); } + void miss_disk_library() { library_disk_misses.increment(); } cache_statistics get_statistics() const { return cache_statistics{.blob_memory_hits = blob_memory_hits.get(), .blob_memory_misses = blob_memory_misses.get(), + .blob_disk_hits = blob_disk_hits.get(), + .blob_disk_misses = blob_disk_misses.get(), .fragment_memory_hits = fragment_memory_hits.get(), .fragment_memory_misses = fragment_memory_misses.get(), + .fragment_disk_hits = fragment_disk_hits.get(), + .fragment_disk_misses = fragment_disk_misses.get(), .library_memory_hits = library_memory_hits.get(), .library_memory_misses = library_memory_misses.get(), - .blob_disk_hits = blob_disk_hits.get(), - .blob_disk_misses = blob_disk_misses.get()}; + .library_disk_hits = library_disk_hits.get(), + .library_disk_misses = library_disk_misses.get()}; } }; @@ -266,7 +198,6 @@ struct cache_statistics_counter { /// copying across threads and disk. struct cache_t { private: - bool enabled_; // TODO: store under more structured pathsi, i.e. /cache. // TODO: stamp binaries so we can pre-load? @@ -274,6 +205,8 @@ struct cache_t { cache_limits limits_; + std::mutex lock_; + detail::lru_memory_cache> blobs_cache_; detail::lru_memory_cache> fragments_cache_; @@ -285,36 +218,24 @@ struct cache_t { alignas(detail::CACHELINE_ALIGNMENT) uint64_t tick_; public: - cache_t(bool enabled, std::string cache_dir, cache_limits const& limits); + cache_t(std::string cache_dir, cache_limits const& limits); cache_t(cache_t const&) = delete; cache_t& operator=(cache_t const&) = delete; cache_t(cache_t&&) = delete; cache_t& operator=(cache_t&&) = delete; ~cache_t() = default; - [[nodiscard]] bool is_enabled(); - - void enable(); - - void disable(); - [[nodiscard]] std::string const& get_cache_dir(); - void store_blob_to_memory(sha256_hash const& sha, std::shared_future binary); - - void store_blob_to_disk(sha256_hash const& sha, blob_view binary); - - std::optional> query_blob_from_memory(sha256_hash const& sha); - - std::optional query_blob_from_disk(sha256_hash const& sha); - - void store_fragment(sha256_hash const& sha, std::shared_future frag); - - std::optional> query_fragment(sha256_hash const& sha); + std::shared_future query_or_insert_blob(sha256_hash const& sha, + std::function maker); - void store_library(sha256_hash const& sha, std::shared_future mod); + std::shared_future query_or_insert_fragment(sha256_hash const& sha, + binary_type type, + std::function maker); - std::optional> query_library(sha256_hash const& sha); + std::shared_future query_or_insert_library( + sha256_hash const& sha, binary_type type, std::function()> maker); cache_statistics get_statistics(); diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 7a46d408ae63..dd62c217b667 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -5,23 +5,24 @@ */ #include +#include #include #include #include -#include #include #include #include #include #include +#include +#include #include #include #include #include -#include #include #include #include @@ -45,41 +46,6 @@ namespace rtc { namespace { -int32_t get_driver_version() -{ - int32_t driver_version; - CUDFRTC_CHECK_CUDART("Failed to get CUDA driver version", cudaDriverGetVersion(&driver_version)); - - return driver_version; -} - -int32_t get_runtime_version() -{ - int32_t runtime_version; - CUDFRTC_CHECK_CUDART("Failed to get CUDA runtime version", - cudaRuntimeGetVersion(&runtime_version)); - - return runtime_version; -} - -int32_t get_current_device_physical_model() -{ - int32_t device; - CUDFRTC_CHECK_CUDART("Failed to get current CUDA device", cudaGetDevice(&device)); - - cudaDeviceProp props; - CUDFRTC_CHECK_CUDART("Failed to get device properties", cudaGetDeviceProperties(&props, device)); - - return props.major * 10 + props.minor; -} - -/* -void max_occupancy_config() -{ - CUDF_FAIL("Not implemented", std::logic_error); - // TODO: Same as configure_1d_max_occupancy -}*/ - sha256_hash hash_string(std::span input) { sha256_context ctx; @@ -97,12 +63,12 @@ cache_t& get_rtc_cache() { return cudf::get_context().rtc_cache(); } CUDF_FAIL(+error_str, std::runtime_error); } -void add_file(char const* dst_path, std::span contents) +void install_file(char const* dst_path, std::span contents) { int dst_file = open(dst_path, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (dst_file == -1) { if (errno == EEXIST) { - // file already exists (repeated include) + // file already exists, so just return return; } throw_posix(std::format("Failed to create file ({})", dst_path), "open"); @@ -119,369 +85,414 @@ void add_file(char const* dst_path, std::span contents) } } -std::vector read_file(char const* path) +void install_file_set(char const* target_dir, + jit_bytes_t const& compressed_binary, + size_t uncompressed_size, + std::span file_ranges, + jit_bytes_array_t const& dst) { - int fd = open(path, O_RDONLY); + std::vector decompressed; + decompressed.resize(uncompressed_size); - if (fd == -1) { throw_posix(std::format("Failed to open file ({})", path), "open"); } + int errc = LZ4_decompress_safe(reinterpret_cast(compressed_binary.data), + reinterpret_cast(decompressed.data()), + compressed_binary.size, + uncompressed_size); - CUDF_DEFER([&] { - if (close(fd) == -1) { throw_posix(std::format("Failed to close file ({})", path), "close"); } - }); - - // get file size - struct stat file_stat; - if (fstat(fd, &file_stat) == -1) { - throw_posix(std::format("Failed to get file status for file ({})", path), "fstat"); - } - - std::vector contents; - contents.resize(file_stat.st_size); + CUDF_EXPECTS(errc == static_cast(uncompressed_size), + "Failed to decompress embedded RTC source files", + std::runtime_error); - if (read(fd, contents.data(), contents.size()) == -1) { - throw_posix(std::format("Failed to read file ({})", path), "read"); - } - - return contents; -} - -static constexpr char const* HASH_FILENAME = ".sha256.hash"; - -void copy_includes_to_dir(char const* dst_dir) -{ - CUDF_FUNC_RANGE(); - - auto const files_data = cudf_jit_embed_sources_file_data.bytes.data; - auto const destinations_data = - reinterpret_cast(cudf_jit_embed_sources_file_destinations.bytes.data); - for (size_t i = 0; i < cudf_jit_embed_sources_file_data.num_ranges; ++i) { - auto const file_data_range = cudf_jit_embed_sources_file_data.ranges[i]; - auto const destination_range = cudf_jit_embed_sources_file_destinations.ranges[i]; + auto const files_data = decompressed.data(); + for (size_t i = 0; i < file_ranges.size(); ++i) { + auto const file_data_range = file_ranges[i]; + auto const dst_range = dst.ranges[i]; auto const file_data = std::span{files_data + file_data_range.offset, file_data_range.size}; - auto const destination = - std::string_view{destinations_data + destination_range.offset, destination_range.size}; + auto const dst_path = std::string_view{ + reinterpret_cast(dst.bytes.data) + dst_range.offset, dst_range.size}; - auto const destination_path = std::format("{}/{}", dst_dir, destination); - - std::filesystem::create_directories(std::filesystem::path{destination_path}.parent_path()); - add_file(destination_path.c_str(), file_data); - } + auto const target_path = std::format("{}/{}", target_dir, dst_path); - { - // write out the state hash file - auto hash_path = std::format("{}/{}", dst_dir, HASH_FILENAME); - add_file(hash_path.c_str(), - std::span{cudf_jit_embed_sources_hash.data, cudf_jit_embed_sources_hash.size}); + std::filesystem::create_directories(std::filesystem::path{target_path}.parent_path()); + install_file(target_path.c_str(), file_data); } } -std::string get_include_dir(char const* base_dir) +void install_cudf_jit(char const* target_dir) { - auto sha256_str = sha256_hex_string::make( - std::span{cudf_jit_embed_sources_hash.data, cudf_jit_embed_sources_hash.size}); - - return std::format("{}/{}", base_dir, sha256_str.view()); + install_file_set(target_dir, + cudf_jit_embed_blobs_binary, + cudf_jit_embed_blobs_uncompressed_size, + cudf_jit_embed_blobs_ranges, + cudf_jit_embed_blobs_file_destinations); + + install_file_set(target_dir, + cudf_jit_embed_sources_binary, + cudf_jit_embed_sources_uncompressed_size, + cudf_jit_embed_sources_ranges, + cudf_jit_embed_sources_file_destinations); } -void create_new_include_dir(char const* base_dir) +void create_and_install_cudf_jit(char const* target_dir) { - CUDF_FUNC_RANGE(); - // directory does not exist, so create it - char tmp_dir_data[] = "/tmp/jit-includes_XXXXXX"; - char* tmp_dir = mkdtemp(tmp_dir_data); + char tmp_dir_[] = "/tmp/cudf-jit-tmpdir_XXXXXX"; + char* tmp_dir = mkdtemp(tmp_dir_); if (tmp_dir == nullptr) { - throw_posix(std::format("Failed to create temporary RTC include directory for ({})", base_dir), - "mkdtemp"); + throw_posix( + std::format("Failed to create temporary JIT install directory for ({})", target_dir), + "mkdtemp"); } - copy_includes_to_dir(tmp_dir); - - auto include_dir = get_include_dir(base_dir); + install_cudf_jit(tmp_dir); - // rename the temporary directory to the target include_dir - if (rename(tmp_dir, include_dir.c_str()) == -1) { - throw_posix( - std::format("Failed to rename temporary RTC include directory to ({})", include_dir), - "rename"); + // rename the temporary directory to the target install directory + if (rename(tmp_dir, target_dir) == -1) { + throw_posix(std::format("Failed to rename temporary JIT install directory to ({})", target_dir), + "rename"); } } -std::string install_includes_to(char const* base_dir) +} // namespace + +jit_bundle::jit_bundle(std::string install_dir) : install_dir_{std::move(install_dir)} +{ + ensure_installed(); + preload_lto_library(); +} + +void jit_bundle::ensure_installed() const { CUDF_FUNC_RANGE(); - auto include_dir = get_include_dir(base_dir); + auto expected_hash = get_hash(); + auto expected_path = std::format("{}/{}", install_dir_, expected_hash); struct stat path_info; - if (lstat(include_dir.c_str(), &path_info) == -1) { + + if (lstat(expected_path.c_str(), &path_info) == -1) { if (errno != ENOENT) { - throw_posix(std::format("Failed to get stat for directory ({})", include_dir), "lstat"); + throw_posix(std::format("Failed to get stat for directory ({})", expected_path), "lstat"); } else { - std::filesystem::create_directories(base_dir); - create_new_include_dir(base_dir); + // ensure base install directory exists + std::filesystem::create_directories(install_dir_); + create_and_install_cudf_jit(expected_path.c_str()); } } else { - // directory exists, perform important sanity checks - if (!S_ISDIR(path_info.st_mode)) { - CUDF_FAIL(+std::format("Include dir ({}) exists but is not a directory", include_dir), - std::runtime_error); - } else { - // verify contents match expected headers - auto hash_path = std::format("{}/{}", include_dir, HASH_FILENAME); - auto hash_data = read_file(hash_path.c_str()); - - CUDF_EXPECTS(std::equal(hash_data.begin(), - hash_data.end(), - cudf_jit_embed_sources_hash.data, - cudf_jit_embed_sources_hash.data + cudf_jit_embed_sources_hash.size), - +std::format("RTC include dir ({}) is corrupted", include_dir), - std::runtime_error); - } + // directory exists, perform minor sanity check + CUDF_EXPECTS(S_ISDIR(path_info.st_mode), + +std::format("JIT install path ({}) exists but is not a directory", expected_path), + std::runtime_error); } - - return include_dir; } -} // namespace - -void install_includes(char const* cache_dir) +void jit_bundle::preload_lto_library() { - CUDF_FUNC_RANGE(); - - auto install_dir = std::format("{}/jit-install", cache_dir); - install_includes_to(install_dir.c_str()); -} + auto& cache = get_rtc_cache(); -fragment_t const& compile_fragment(char const* name, char const* source_code_cstr, char const* key) -{ - CUDF_FUNC_RANGE(); + auto bundle_hash = get_hash(); - auto sm = get_current_device_physical_model(); auto const cache_key = std::format(R"***( - fragment_type=LTO_IR, + fragment_type=CUBIN, key={}, - cuda_runtime={}, - cuda_driver={}, - arch={})***", - key, - get_runtime_version(), - get_driver_version(), - sm); + bundle={} + )***", + "cudf_lto_library", + bundle_hash); auto const cache_key_sha256 = hash_string(cache_key); - auto& cache = get_rtc_cache(); + auto compile = [&] { + auto directory = get_directory(); + auto path = std::format("{}/{}", directory, "cudf_lto_library.cubin"); + auto cubin = blob_t::from_file(path.c_str()); + CUDF_EXPECTS(cubin.has_value(), + +std::format("Failed to load LTO library cubin from disk at ({})", path), + std::runtime_error); + fragment_t::load_params load_params{.binary = std::make_shared(std::move(*cubin)), + .type = binary_type::LTO_IR}; + return fragment_t::load(load_params); + }; + + auto fut = cache.query_or_insert_fragment(cache_key_sha256, binary_type::LTO_IR, compile); + + lto_library_ = fut.get(); +} - if (auto frag = cache.query_fragment(cache_key_sha256); frag.has_value()) { - std::cout << "Loading RTC base library from memory\n"; - return *frag->get(); - } else if (auto disk_frag = cache.query_blob_from_disk(cache_key_sha256); disk_frag.has_value()) { - std::cout << "Loading RTC base library from disk cache\n"; - std::promise prom; - auto fut = std::shared_future{prom.get_future()}; - { - cache.store_fragment(cache_key_sha256, fut); - fragment_t::load_params load_params{.binary = *disk_frag, .type = binary_type::LTO_IR}; - auto frag = fragment_t::load(load_params); - prom.set_value(std::move(frag)); - } - return *fut.get(); - } +std::string jit_bundle::get_hash() const +{ + auto str = sha256_hex_string::make( + std::span{cudf_jit_embed_hash.data, static_cast(cudf_jit_embed_hash.size)}); + return std::string{str.view()}; +} - std::cout << "Compiling and linking RTC base library\n"; - std::promise prom; - auto fut = std::shared_future{prom.get_future()}; - cache.store_fragment(cache_key_sha256, fut); +std::string jit_bundle::get_directory() const +{ + auto hash = get_hash(); + return std::format("{}/{}", install_dir_, hash); +} - auto begin = std::chrono::high_resolution_clock::now(); - auto cache_dir = cache.get_cache_dir(); - auto install_dir = std::format("{}/jit-install", cache_dir); - auto include_dir = get_include_dir(install_dir.c_str()); +fragment jit_bundle::get_lto_library() const { return lto_library_; } - std::vector include_options; - include_options.push_back(std::format("-I{}", include_dir)); +std::vector jit_bundle::get_include_directories() const +{ + std::vector directories; + auto base_dir = get_directory(); auto include_directories_data = reinterpret_cast(cudf_jit_embed_sources_include_directories.bytes.data); + for (size_t i = 0; i < cudf_jit_embed_sources_include_directories.num_ranges; i++) { auto range = cudf_jit_embed_sources_include_directories.ranges[i]; auto dest_include_directory = include_directories_data + range.offset; - include_options.push_back(std::format("-I{}/{}", include_dir, dest_include_directory)); + directories.emplace_back(std::format("{}/{}", base_dir, dest_include_directory)); } - std::vector options; + return directories; +} + +std::vector jit_bundle::get_compile_options() const +{ + std::vector options; + auto embed_options_data = reinterpret_cast(cudf_jit_embed_options.bytes.data); for (size_t i = 0; i < cudf_jit_embed_options.num_ranges; i++) { auto range = cudf_jit_embed_options.ranges[i]; auto option = embed_options_data + range.offset; - options.push_back(option); - } - - auto arch_flag = std::format("--gpu-architecture=sm_{}", sm); - options.push_back(arch_flag.c_str()); - options.push_back("--dlink-time-opt"); - options.push_back("--relocatable-device-code=true"); - // options.push_back("--split-compile=0"); - // options.push_back("--fdevice-time-trace=jit_comp_trace.json"); - // options.push_back("--minimal"); - // options.push_back("--time=compile_trace.json"); - // options.push_back("-time"); - // --fast-compile - options.push_back("--pch"); - options.push_back( - "--pch-dir=/tmp/cudf-rtc-pch"); // [ ] fix; make it consistent (hashing of header contents?) - - options.push_back("--device-as-default-execution-space"); - - for (auto const& include_option : include_options) { - options.push_back(include_option.c_str()); + options.emplace_back(option); } - auto const params = fragment_t::compile_params{.name = name, - .source = source_code_cstr, - .headers = {}, - .options = options, - .target_type = binary_type::LTO_IR}; + return options; +} - auto frag = fragment_t::compile(params); +namespace { - auto view = frag->get(binary_type::LTO_IR)->view(); +int32_t get_driver_version() +{ + int32_t driver_version; + CUDFRTC_CHECK_CUDART("Failed to get CUDA driver version", cudaDriverGetVersion(&driver_version)); - auto end = std::chrono::high_resolution_clock::now(); - auto dur = end - begin; - std::cout << "RTC fragment compilation for `" << name << "` took " - << std::chrono::duration_cast(dur).count() << " us\n"; + return driver_version; +} - cache.store_blob_to_disk(cache_key_sha256, view); +int32_t get_runtime_version() +{ + int32_t runtime_version; + CUDFRTC_CHECK_CUDART("Failed to get CUDA runtime version", + cudaRuntimeGetVersion(&runtime_version)); - prom.set_value(std::move(frag)); - return *fut.get(); + return runtime_version; } -fragment_t const& compile_library_fragment() +int32_t get_current_device_physical_model() { - CUDF_FUNC_RANGE(); + int32_t device; + CUDFRTC_CHECK_CUDART("Failed to get current CUDA device", cudaGetDevice(&device)); - return compile_fragment("cudf_lto_library", - R"***( - #include "jit/lto/library.inl.cuh" - )***", - "cudf_lto_library"); + cudaDeviceProp props; + CUDFRTC_CHECK_CUDART("Failed to get device properties", cudaGetDeviceProperties(&props, device)); + + return props.major * 10 + props.minor; } -fragment_t const& compile_udf_fragment(char const* source_code_cstr, char const* key) +} // namespace + +fragment get_or_compile_fragment(char const* name, char const* source_code_cstr, char const* key) { CUDF_FUNC_RANGE(); - return compile_fragment("cudf_udf_fragment", source_code_cstr, key); + auto& bundle = get_bundle(); + auto& cache = get_rtc_cache(); + + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_physical_model(); + auto bundle_hash = bundle.get_hash(); + + auto const cache_key = std::format(R"***( + fragment_type=LTO_IR, + key={}, + cuda_runtime={}, + cuda_driver={}, + arch={}, + bundle={} + )***", + key, + runtime, + driver, + sm, + bundle_hash); + + auto const cache_key_sha256 = hash_string(cache_key); + + // TODO: add time function in cache + + auto compile = [&] { + auto begin = std::chrono::high_resolution_clock::now(); + auto cache_dir = cache.get_cache_dir(); + auto install_dir = std::format("{}/jit-install", cache_dir); + + auto include_dirs = bundle.get_include_directories(); + auto compile_options = bundle.get_compile_options(); + + std::vector options; + + for (auto const& include_dir : include_dirs) { + options.emplace_back(std::format("-I{}", include_dir)); + } + + for (auto const& compile_option : compile_options) { + options.emplace_back(compile_option); + } + + options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); + options.emplace_back("--dlink-time-opt"); + options.emplace_back("--relocatable-device-code=true"); + options.emplace_back("--device-as-default-execution-space"); + + // --split-compile=0 + // --fdevice-time-trace=jit_comp_trace.json + // --minimal + // --time=compile_trace.json + // -time + // --fast-compile + // --pch + // --pch-dir=/tmp/cudf-rtc-pch + + std::vector options_cstr; + for (auto const& option : options) { + options_cstr.emplace_back(option.c_str()); + } + + auto const params = fragment_t::compile_params{.name = name, + .source = source_code_cstr, + .headers = {}, + .options = options_cstr, + .target_type = binary_type::LTO_IR}; + + auto frag = fragment_t::compile(params); + + auto end = std::chrono::high_resolution_clock::now(); + + auto duration = end - begin; + + CUDF_LOG_INFO( + "Compiled fragment `{}` in {} ms", + name, + std::chrono::duration_cast>(duration).count()); + + return frag; + }; + + auto fut = cache.query_or_insert_fragment(cache_key_sha256, binary_type::LTO_IR, compile); + + return fut.get(); } -kernel_ref compile_and_link_udf(char const* name, - char const* kernel_name, - char const* kernel_key, - char const* udf_code, - char const* udf_key) +jit_bundle& get_bundle() +{ + auto& context = cudf::get_context(); + return context.jit_bundle(); +} + +library compile_and_link_udf(char const* name, + char const* udf_code, + char const* udf_key, + char const* kernel_symbol) { CUDF_FUNC_RANGE(); - auto sm = get_current_device_physical_model(); - auto library_key = std::format(R"***( - fragment_types=LTO_IR, + auto& cache = get_rtc_cache(); + auto& bundle = get_bundle(); + + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_physical_model(); + auto bundle_hash = bundle.get_hash(); + + auto compile = [&] { + auto begin = std::chrono::high_resolution_clock::now(); + auto library = bundle.get_lto_library(); + auto fragment = get_or_compile_fragment(name, udf_code, udf_key); + + // TODO: experiment with: + // optimization flags + // split compile + // split-compile-extended + // lineinfo and debug info options + // -kernels-used= + // sass dump + // time dump + // env variable to control options + // fma + // variables-used + // -optimize-unused-variables + // -nocache + // -device-stack-protector + + std::vector options; + + options.emplace_back("-lto"); + options.emplace_back(std::format("-arch=sm_{}", sm)); + options.emplace_back(std::format("-kernels-used={}", kernel_symbol)); + + std::vector options_cstr; + for (auto const& option : options) { + options_cstr.emplace_back(option.c_str()); + } + + blob_view const link_fragments[] = {library->get(binary_type::LTO_IR)->view(), + fragment->get(binary_type::LTO_IR)->view()}; + + binary_type const fragment_binary_types[] = {binary_type::LTO_IR, binary_type::LTO_IR}; + + char const* const fragment_names[] = {"cudf_lto_library", "cudf_udf_fragment"}; + + auto const params = library_t::link_params{.name = name, + .output_type = binary_type::CUBIN, + .fragments = link_fragments, + .fragment_binary_types = fragment_binary_types, + .fragment_names = fragment_names, + .link_options = options_cstr}; + + auto blob = library_t::link_as_blob(params); + + auto load_params = library_t::load_params{.binary = blob->view(), .type = binary_type::CUBIN}; + + auto linked_library = library_t::load(load_params); + + auto end = std::chrono::high_resolution_clock::now(); + + auto duration = end - begin; + + CUDF_LOG_INFO( + "Compiled fragment `{}` in {} ms", + name, + std::chrono::duration_cast>(duration).count()); + + return std::make_tuple(linked_library, blob); + }; + + auto library_cache_key = std::format(R"***( target=CUBIN, - kernel={}, + kernels={}, udf={}, cuda_runtime={}, cuda_driver={}, arch={})***", - kernel_key, - udf_key, - get_runtime_version(), - get_driver_version(), - sm); - auto const library_key_sha256 = hash_string(library_key); - - // [ ] we also need to use the include dirs as part of the key - auto& cache = get_rtc_cache(); + kernel_symbol, + udf_key, + runtime, + driver, + sm); + auto const library_cache_key_sha256 = hash_string(library_cache_key); - // TODO: (atomicity) should probably use query_or_insert - if (auto lib = cache.query_library(library_key_sha256); lib.has_value()) { - std::cout << "Loading kernel from memory\n"; - return lib->get()->get_kernel(kernel_name); - } else if (auto disk_lib = cache.query_blob_from_disk(library_key_sha256); disk_lib.has_value()) { - std::cout << "Loading kernel from disk cache\n"; - std::promise prom; - auto fut = std::shared_future{prom.get_future()}; - - { - cache.store_library(library_key_sha256, fut); - library_t::load_params load_params{.binary = (*disk_lib)->view(), .type = binary_type::CUBIN}; - auto lib = library_t::load(load_params); - prom.set_value(std::move(lib)); - } - - return fut.get()->get_kernel(kernel_name); - } + auto library = + cache.query_or_insert_library(library_cache_key_sha256, binary_type::CUBIN, compile); - std::cout << "Compiling and linking library\n"; - auto const& library_frag = compile_library_fragment(); - - auto const& udf_frag = compile_udf_fragment(udf_code, udf_key); - - std::promise prom; - auto fut = std::shared_future{prom.get_future()}; - // cache.store_library(library_key_sha256, fut); - - auto begin = std::chrono::high_resolution_clock::now(); - - blob_view const link_fragments[] = {library_frag.get(binary_type::LTO_IR)->view(), - udf_frag.get(binary_type::LTO_IR)->view()}; - binary_type const fragment_binary_types[] = {binary_type::LTO_IR, binary_type::LTO_IR}; - - char const* const fragment_names[] = {"cudf_lto_library", "cudf_udf_fragment"}; - - // TODO: run compilation tests at program startup - - // TODO: optimization flags - // TODO: split compile - // TODO: split-compile-extended - // TODO: lineinfo and debug info options - // TODO: -kernels-used= - // TODO: sass dump - // TODO: time dump - // TODO: env variable to control options - // TODO: fma - // TODO: variables-used - // TODO: -optimize-unused-variables - // TODO: -nocache - // TODO: -device-stack-protector - auto arch_flag = std::format("-arch=sm_{}", sm); - char const* const link_options[] = { // "-split-compile=0", - "-lto", // TODO: full flag names - - // "-optimize-unused-variables", - "-kernels-used=transform_kernel", - arch_flag.c_str()}; - - auto const params = library_t::link_params{.name = name, - .output_type = binary_type::CUBIN, - .fragments = link_fragments, - .fragment_binary_types = fragment_binary_types, - .fragment_names = fragment_names, - .link_options = link_options}; - // TODO: compilation flow logging with time taken, should be disabled when not in use - auto lib = library_t::link(params); - - auto end = std::chrono::high_resolution_clock::now(); - auto dur = end - begin; - std::cout << "RTC library linking for `" << name << "` took " - << std::chrono::duration_cast(dur).count() << " us\n"; - // TODO: store to disk cache - prom.set_value(std::move(lib)); - - return fut.get()->get_kernel(kernel_name); + return library.get(); } } // namespace rtc diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/rtc/cudf.hpp index 5ae4c3963c27..e8422b42e487 100644 --- a/cpp/src/jit/rtc/cudf.hpp +++ b/cpp/src/jit/rtc/cudf.hpp @@ -13,13 +13,35 @@ namespace CUDF_EXPORT cudf { namespace rtc { -void install_includes(char const* cache_dir); +class jit_bundle { + std::string install_dir_; + fragment lto_library_; -kernel_ref compile_and_link_udf(char const* name, - char const* kernel_name, - char const* kernel_key, - char const* udf_code, - char const* udf_key); + private: + void ensure_installed() const; + + void preload_lto_library(); + + public: + jit_bundle(std::string install_dir); + + std::string get_hash() const; + + std::string get_directory() const; + + fragment get_lto_library() const; + + std::vector get_include_directories() const; + + std::vector get_compile_options() const; +}; + +jit_bundle& get_bundle(); + +library compile_and_link_udf(char const* name, + char const* udf_code, + char const* udf_key, + char const* kernel_symbol); } // namespace rtc } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/includes/system/README.md b/cpp/src/jit/rtc/includes/system/README.md index 60a941276325..14d14195d2c8 100644 --- a/cpp/src/jit/rtc/includes/system/README.md +++ b/cpp/src/jit/rtc/includes/system/README.md @@ -1 +1 @@ -# Doc \ No newline at end of file +# Doc diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index fdb018637068..f8e49eb72324 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -352,6 +352,20 @@ blob const& fragment_t::get(binary_type type) const return blob_; } +std::tuple kernel_ref::max_occupancy_config(size_t dynamic_shared_memory_bytes, + int32_t block_size_limit) const +{ + int32_t min_grid_size; + int32_t block_size; + CUDFRTC_CHECK_CUDA(cuOccupancyMaxPotentialBlockSize(&min_grid_size, + &block_size, + reinterpret_cast(handle_), + nullptr, + dynamic_shared_memory_bytes, + block_size_limit)); + return {min_grid_size, block_size}; +} + void kernel_ref::launch(uint32_t grid_dim_x, uint32_t grid_dim_y, uint32_t grid_dim_z, diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 130b30c8ad8c..64d41d3c22db 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -9,6 +9,7 @@ #include #include #include +#include extern "C" { typedef struct CUlib_st* CUlibrary; @@ -84,6 +85,8 @@ struct [[nodiscard]] blob_t { static blob_t from_vector(std::vector&& data); static blob_t from_static_data(std::span data); + + static std::optional from_file(char const* path); }; using blob = std::shared_ptr; @@ -146,6 +149,9 @@ struct [[nodiscard]] kernel_ref { public: explicit kernel_ref(CUkernel handle) : handle_(handle) {} + std::tuple max_occupancy_config(size_t dynamic_shared_memory_bytes, + int32_t block_size_limit) const; + void launch(uint32_t grid_dim_x, uint32_t grid_dim_y, uint32_t grid_dim_z, diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 3b1608559e15..ee9a83356b9e 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -18,7 +18,8 @@ namespace cudf { -context::context(init_flags flags) : _program_cache{nullptr}, _rtc_cache{} +context::context(init_flags flags) + : _program_cache{nullptr}, _rtc_cache{nullptr}, _jit_bundle{nullptr} { auto dump_codegen_flag = getenv_or("LIBCUDF_JIT_DUMP_CODEGEN", std::string{"OFF"}); _dump_codegen = (dump_codegen_flag == "ON" || dump_codegen_flag == "1"); @@ -41,6 +42,12 @@ rtc::cache_t& context::rtc_cache() return *_rtc_cache; } +rtc::jit_bundle& context::jit_bundle() +{ + CUDF_EXPECTS(_jit_bundle != nullptr, "JIT bundle not initialized", std::runtime_error); + return *_jit_bundle; +} + bool context::dump_codegen() const { return _dump_codegen; } void context::initialize_components(init_flags flags) @@ -51,8 +58,8 @@ void context::initialize_components(init_flags flags) if (has_flag(new_flags, init_flags::INIT_JIT_CACHE)) { _program_cache = std::make_unique(); // TODO: Make cache directory configurable - _rtc_cache = std::make_unique(true, "/tmp/cudf-rtc-cache", rtc::cache_limits{}); - cudf::rtc::install_includes("/tmp/cudf-rtc-cache"); + _rtc_cache = std::make_unique("/tmp/cudf-rtc-cache", rtc::cache_limits{}); + _jit_bundle = std::make_unique("/tmp/cudf-jit-install"); } if (has_flag(new_flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 9df6f24a8cb1..a3aae079be7f 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -18,7 +18,8 @@ class program_cache; namespace rtc { class cache_t; -} +class jit_bundle; +} // namespace rtc /// @brief The context object contains global state internal to CUDF. /// It helps to ensure structured and well-defined construction and destruction of global @@ -27,6 +28,7 @@ class context { private: std::unique_ptr _program_cache; std::unique_ptr _rtc_cache; + std::unique_ptr _jit_bundle; init_flags _initialized_flags = init_flags::NONE; bool _dump_codegen = false; bool _use_jit = false; @@ -43,6 +45,8 @@ class context { rtc::cache_t& rtc_cache(); + rtc::jit_bundle& jit_bundle(); + [[nodiscard]] bool dump_codegen() const; /// @brief Initialize additional components based on the provided flags diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index c6e9a422779d..d7efc0e6ce0d 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -22,26 +22,25 @@ TEST_F(RTCTest, CreateFragment) auto fn = []() { auto begin = std::chrono::high_resolution_clock::now(); auto kern = rtc::compile_and_link_udf("test_fragment", - "transform_kernel", - "test_kernel_key", R"***( #include "cudf/jit/transform_params.cuh" - // #include "cudf/jit/lto/operators.cuh" + #include "cudf/jit/lto/operators.cuh" extern "C" __device__ void transform_operator(cudf::lto::transform_params const* p){ int a = 1; int b = 2; int c = 3; int * out = (int *)p->outputs; - // cudf::lto::operators::add(&c, &a, &b); - // cudf::lto::operators::sub(&c, &a, &b); - // cudf::lto::operators::mul(&c, &a, &b); - // cudf::lto::operators::mul(&c, &a, &b); + cudf::lto::operators::add(&c, &a, &b); + cudf::lto::operators::sub(&c, &a, &b); + cudf::lto::operators::mul(&c, &a, &b); + cudf::lto::operators::mul(&c, &a, &b); *out = a + b * c; } )***", - "test_udf_key"); + "test_udf_key", + "transform_kernel"); (void)kern; auto end = std::chrono::high_resolution_clock::now(); From 86c0010ab80b5430bc128b6fe989cdd054d42931 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 6 Feb 2026 16:16:54 +0000 Subject: [PATCH 024/254] Enhance JIT compilation with ASM support and integrate LZ4 compression --- cpp/CMakeLists.txt | 28 ++++++++++--- cpp/cmake/Modules/JitEmbed.cmake | 10 ++--- cpp/cmake/Modules/jit_embed.py | 65 +++++++++++++++--------------- cpp/cmake/thirdparty/get_lz4.cmake | 35 ++++++++++++++++ cpp/src/jit/rtc/cudf.cpp | 11 ++--- cpp/src/jit/rtc/rtc.cpp | 2 +- cpp/src/jit/rtc/rtc.hpp | 2 +- 7 files changed, 102 insertions(+), 51 deletions(-) create mode 100644 cpp/cmake/thirdparty/get_lz4.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index edc9b608f4de..38c6bbed7e7b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -20,7 +20,7 @@ rapids_cuda_init_architectures(CUDF) project( CUDF VERSION "${RAPIDS_VERSION}" - LANGUAGES C CXX CUDA + LANGUAGES C CXX CUDA ASM ) if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 12.2) @@ -314,6 +314,9 @@ include(cmake/thirdparty/get_thread_pool.cmake) # find zstd include(cmake/thirdparty/get_zstd.cmake) +# find lz4 +include(cmake/thirdparty/get_lz4.cmake) + # Workaround until https://github.com/rapidsai/rapids-cmake/issues/176 is resolved if(NOT BUILD_SHARED_LIBS) include("${rapids-cmake-dir}/export/find_package_file.cmake") @@ -399,8 +402,8 @@ foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) INCLUDE_DIRECTORIES CCCL/libcudacxx/${INC_DIR_NAME} ) endforeach() -# TODO(lamarrr): inspect the generated cubin and make sure it doesn't contain -# any unneeded symbols. it should be as small as possible +# TODO(lamarrr): inspect the generated cubin and make sure it doesn't contain any unneeded symbols. +# it should be as small as possible jit_add_options(cudf_jit_embed OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) jit_add_blob(cudf_jit_embed FILE "$" DEST cudf_lto_library.fatbin) jit_embed(cudf_jit_embed COMPRESSION lz4) @@ -919,12 +922,16 @@ add_library( src/utilities/traits.cpp src/utilities/type_checks.cpp src/utilities/type_dispatcher.cpp - ${cudf_jit_embed_SOURCE_DIR}/embed.cpp + ${cudf_jit_embed_SOURCE_DIR}/embed.s ) add_dependencies(cudf cudf_jit_embed) target_include_directories(cudf PRIVATE ${cudf_jit_embed_INCLUDE_DIRS}) +foreach(incbin_dir IN LISTS cudf_jit_embed_INCLUDE_DIRS) + target_compile_options(cudf PRIVATE $<$:-Wa,-I${incbin_dir}>) +endforeach() + # Anything that includes jitify needs to be compiled with _FILE_OFFSET_BITS=64 due to a limitation # in how conda builds glibc set_source_files_properties( @@ -1066,8 +1073,17 @@ add_dependencies(cudf jitify_preprocess_run) target_link_libraries( cudf PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ - PRIVATE $ cuco::cuco ZLIB::ZLIB nvcomp::nvcomp - kvikio::kvikio nanoarrow::nanoarrow zstd CUDA::nvJitLink CUDA::nvrtc CUDA::cuda_driver + PRIVATE $ + cuco::cuco + ZLIB::ZLIB + nvcomp::nvcomp + kvikio::kvikio + nanoarrow::nanoarrow + zstd + CUDA::nvJitLink + CUDA::nvrtc + CUDA::cuda_driver + lz4_static ) # Add Conda library, and include paths if specified diff --git a/cpp/cmake/Modules/JitEmbed.cmake b/cpp/cmake/Modules/JitEmbed.cmake index fc5033b4eacd..40cac0bd0016 100644 --- a/cpp/cmake/Modules/JitEmbed.cmake +++ b/cpp/cmake/Modules/JitEmbed.cmake @@ -253,24 +253,22 @@ function(jit_embed) ) add_custom_command( - OUTPUT ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.cpp ${OUTPUT_DIR}/embed.bin + OUTPUT ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s ${OUTPUT_DIR}/embed.bin COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" --id "${TARGET}" --output-dir "${OUTPUT_DIR}" --input "${YAML_FILE_PATH}" DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" "${YAML_FILE_PATH}" ${jitembed_${TARGET}_incdir__source_files} ${jitembed_${TARGET}_blob__files} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" COMMENT - "Generating JIT embed for ${TARGET} (YAML: ${YAML_FILE_PATH}) into ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.cpp ${OUTPUT_DIR}/embed.bin" + "Generating JIT embed for ${TARGET} (YAML: ${YAML_FILE_PATH}) into ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s" VERBATIM ) - add_custom_target( - ${TARGET} ALL DEPENDS ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.cpp ${OUTPUT_DIR}/embed.bin - ) + add_custom_target(${TARGET} ALL DEPENDS ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s) message( STATUS - "JIT embed for target ${TARGET} (YAML: ${YAML_FILE_PATH}) will be generated into: ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.cpp ${OUTPUT_DIR}/embed.bin" + "JIT embed for target ${TARGET} (YAML: ${YAML_FILE_PATH}) will be generated into: ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s" ) set(${TARGET}_INCLUDE_DIRS diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py index 5dd23d73f494..7eb952900f9e 100644 --- a/cpp/cmake/Modules/jit_embed.py +++ b/cpp/cmake/Modules/jit_embed.py @@ -279,28 +279,23 @@ def var(self: Self) -> CXXVarDecl: ) -class CXXAsmEmbedDecl(NamedTuple): +class AsmEmbedDecl(NamedTuple): id: str file: str @staticmethod def of_file(id: str, file: str) -> Self: - return CXXAsmEmbedDecl(id=id, file=file) + return AsmEmbedDecl(id=id, file=file) - def var(self: Self) -> CXXVarDecl: - return CXXVarDecl( - id=self.id, - expr=f""" - asm( - ".section .rodata\\n" - ".global {self.id}_begin\\n" - ".global {self.id}_end\\n" - "{self.id}_begin:\\n" - ".incbin \\"{self.file}\\"\\n" - "{self.id}_end:\\n" - ); -""", - ) + def decl(self: Self) -> str: + return f""" +.section .rodata +.global {self.id}_begin +.global {self.id}_end +{self.id}_begin: +.incbin "{self.file}" +{self.id}_end: +""" def merge_bytes_with_null_terminators( @@ -317,8 +312,9 @@ def merge_bytes_with_null_terminators( class EmbedOutput(NamedTuple): - cxx_header: str - cxx_source: str + cxx_header: str | None + cxx_source: str | None + asm_source: str | None bin_file_name: str | None bin_file_data: bytes | None hash: bytes @@ -346,7 +342,8 @@ def generate_cxx_strings_data(id: str, strings: list[str]) -> EmbedOutput: return EmbedOutput( cxx_header=cxx_header, - cxx_source="", + cxx_source=None, + asm_source=None, bin_file_name=None, bin_file_data=None, hash=hash, @@ -422,7 +419,7 @@ def generate_cxx_source_files_data( id=f"{id}_uncompressed_size", size=len(uncompressed_files_bytes) ) - binary_embed_decl: CXXAsmEmbedDecl = CXXAsmEmbedDecl.of_file( + binary_embed_decl: AsmEmbedDecl = AsmEmbedDecl.of_file( id=f"{id}_binary", file=binary_file_name ) @@ -452,13 +449,14 @@ def generate_cxx_source_files_data( """ - cxx_source = f""" -{binary_embed_decl.var().decl()} + asm_source = f""" +{binary_embed_decl.decl()} """ return EmbedOutput( cxx_header=cxx_header, - cxx_source=cxx_source, + cxx_source=None, + asm_source=asm_source, bin_file_name=binary_file_name, bin_file_data=compressed_files_bytes if compress @@ -520,7 +518,7 @@ def generate_cxx_blobs_data( id=f"{id}_uncompressed_size", size=len(uncompressed_blob_bytes) ) - binary_embed_decl: CXXAsmEmbedDecl = CXXAsmEmbedDecl.of_file( + binary_embed_decl: AsmEmbedDecl = AsmEmbedDecl.of_file( id=f"{id}_binary", file=binary_file_name ) @@ -539,13 +537,14 @@ def generate_cxx_blobs_data( {binary_ranges_decl.var().decl()} """ - cxx_source = f""" -{binary_embed_decl.var().decl()} + asm_source = f""" +{binary_embed_decl.decl()} """ return EmbedOutput( cxx_header=cxx_header, - cxx_source=cxx_source, + cxx_source=None, + asm_source=asm_source, bin_file_name=binary_file_name, bin_file_data=compressed_blob_bytes if compress @@ -612,14 +611,16 @@ def generate_embed( {hash_decl.var().decl()} -{"\n\n".join([output.cxx_header for output in outputs])} +{"\n\n".join([output.cxx_header if output.cxx_header is not None else "" for output in outputs])} }} """ - cxx_source = f""" -{"\n\n".join([output.cxx_source for output in outputs])} + asm_source = f""" +{"\n\n".join([output.asm_source if output.asm_source is not None else "" for output in outputs])} + +.section .note.GNU-stack,"",@progbits """ os.makedirs(output_dir, exist_ok=True) @@ -627,8 +628,8 @@ def generate_embed( with open(f"{output_dir}/embed.hpp", "w") as f: f.write(cxx_header) - with open(f"{output_dir}/embed.cpp", "w") as f: - f.write(cxx_source) + with open(f"{output_dir}/embed.s", "w") as f: + f.write(asm_source) for output in outputs: if output.bin_file_name and output.bin_file_data: diff --git a/cpp/cmake/thirdparty/get_lz4.cmake b/cpp/cmake/thirdparty/get_lz4.cmake new file mode 100644 index 000000000000..5e06f1414dcb --- /dev/null +++ b/cpp/cmake/thirdparty/get_lz4.cmake @@ -0,0 +1,35 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +# Use CPM to find or clone lz4 +function(find_and_configure_lz4) + + set(CPM_DOWNLOAD_lz4 ON) + rapids_cpm_find( + lz4 + 1.5.7 + GLOBAL_TARGETS + lz4 + CPM_ARGS + GIT_REPOSITORY + https://github.com/lz4/lz4.git + GIT_TAG + 446a35f + GIT_SHALLOW + TRUE) + + if(lz4_ADDED) + add_library(lz4_static STATIC lz4file.c lz4frame.c lz4hc.c xxhash.c) + target_include_directories(lz4_static + PUBLIC $) + + set_target_properties(lz4_static PROPERTIES POSITION_INDEPENDENT_CODE ON) + endif() + +endfunction() + +find_and_configure_lz4() diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index dd62c217b667..d6b675dc73d7 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -194,7 +194,7 @@ void jit_bundle::preload_lto_library() auto bundle_hash = get_hash(); auto const cache_key = std::format(R"***( - fragment_type=CUBIN, + fragment_type=FATBIN, key={}, bundle={} )***", @@ -205,17 +205,17 @@ void jit_bundle::preload_lto_library() auto compile = [&] { auto directory = get_directory(); - auto path = std::format("{}/{}", directory, "cudf_lto_library.cubin"); + auto path = std::format("{}/{}", directory, "cudf_lto_library.fatbin"); auto cubin = blob_t::from_file(path.c_str()); CUDF_EXPECTS(cubin.has_value(), +std::format("Failed to load LTO library cubin from disk at ({})", path), std::runtime_error); fragment_t::load_params load_params{.binary = std::make_shared(std::move(*cubin)), - .type = binary_type::LTO_IR}; + .type = binary_type::FATBIN}; return fragment_t::load(load_params); }; - auto fut = cache.query_or_insert_fragment(cache_key_sha256, binary_type::LTO_IR, compile); + auto fut = cache.query_or_insert_fragment(cache_key_sha256, binary_type::FATBIN, compile); lto_library_ = fut.get(); } @@ -352,6 +352,7 @@ fragment get_or_compile_fragment(char const* name, char const* source_code_cstr, options.emplace_back("--relocatable-device-code=true"); options.emplace_back("--device-as-default-execution-space"); + // TODO: experiment with: // --split-compile=0 // --fdevice-time-trace=jit_comp_trace.json // --minimal @@ -476,7 +477,7 @@ library compile_and_link_udf(char const* name, }; auto library_cache_key = std::format(R"***( - target=CUBIN, + library_type=CUBIN, kernels={}, udf={}, cuda_runtime={}, diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index f8e49eb72324..5f0e56caa845 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -284,7 +284,7 @@ blob_t blob_t::from_static_data(std::span data) fragment fragment_t::load(load_params const& params) { CUDF_FUNC_RANGE(); - // TODO: check + // TODO: validate parameters return std::make_shared(params.binary, params.type); } diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 64d41d3c22db..8c020f3681df 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -7,9 +7,9 @@ #pragma once #include #include +#include #include #include -#include extern "C" { typedef struct CUlib_st* CUlibrary; From a955a0462102c42f7513cec9f58307a9dfec2af8 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 6 Feb 2026 16:50:17 +0000 Subject: [PATCH 025/254] Update copyright year in get_lz4.cmake and improve formatting --- cpp/cmake/thirdparty/get_lz4.cmake | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/cpp/cmake/thirdparty/get_lz4.cmake b/cpp/cmake/thirdparty/get_lz4.cmake index 5e06f1414dcb..5f2f5d1b8e46 100644 --- a/cpp/cmake/thirdparty/get_lz4.cmake +++ b/cpp/cmake/thirdparty/get_lz4.cmake @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -10,22 +10,20 @@ function(find_and_configure_lz4) set(CPM_DOWNLOAD_lz4 ON) rapids_cpm_find( - lz4 - 1.5.7 - GLOBAL_TARGETS - lz4 + lz4 1.5.7 + GLOBAL_TARGETS lz4 CPM_ARGS - GIT_REPOSITORY - https://github.com/lz4/lz4.git - GIT_TAG - 446a35f - GIT_SHALLOW - TRUE) + GIT_REPOSITORY https://github.com/lz4/lz4.git + GIT_TAG 446a35f + GIT_SHALLOW TRUE + ) if(lz4_ADDED) - add_library(lz4_static STATIC lz4file.c lz4frame.c lz4hc.c xxhash.c) - target_include_directories(lz4_static - PUBLIC $) + add_library( + lz4_static STATIC ${lz4_SOURCE_DIR}/lib/lz4file.c ${lz4_SOURCE_DIR}/lib/lz4frame.c + ${lz4_SOURCE_DIR}/lib/lz4hc.c ${lz4_SOURCE_DIR}/lib/xxhash.c + ) + target_include_directories(lz4_static PUBLIC $) set_target_properties(lz4_static PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() From 7d2a2bb8c95dc0dbe21c84a429d2cafea3f9901e Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 7 Feb 2026 03:04:16 +0000 Subject: [PATCH 026/254] Refactor LZ4 integration and improve JIT logging - Change LZ4 library linking from static to object files for better flexibility. - Update LZ4 compression method in Python script to use block compression. - Enhance JIT installation logging for better debugging. - Modify CMake configuration to use the latest LZ4 development version. --- cpp/CMakeLists.txt | 2 +- cpp/cmake/Modules/JitEmbed.cmake | 1 + cpp/cmake/Modules/jit_embed.py | 14 +++++++++----- cpp/cmake/thirdparty/get_lz4.cmake | 11 +++++------ cpp/src/jit/rtc/cudf.cpp | 5 +++-- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 196fbb8ce45f..f21420a5617e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1085,7 +1085,7 @@ target_link_libraries( CUDA::nvJitLink CUDA::nvrtc CUDA::cuda_driver - lz4_static + lz4_objects ) # Add Conda library, and include paths if specified diff --git a/cpp/cmake/Modules/JitEmbed.cmake b/cpp/cmake/Modules/JitEmbed.cmake index 40cac0bd0016..f799f4f97654 100644 --- a/cpp/cmake/Modules/JitEmbed.cmake +++ b/cpp/cmake/Modules/JitEmbed.cmake @@ -254,6 +254,7 @@ function(jit_embed) add_custom_command( OUTPUT ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s ${OUTPUT_DIR}/embed.bin + BYPRODUCTS ${OUTPUT_DIR}/* COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" --id "${TARGET}" --output-dir "${OUTPUT_DIR}" --input "${YAML_FILE_PATH}" DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" "${YAML_FILE_PATH}" diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py index 7eb952900f9e..bc00ee917ae3 100644 --- a/cpp/cmake/Modules/jit_embed.py +++ b/cpp/cmake/Modules/jit_embed.py @@ -6,7 +6,7 @@ import os from typing import NamedTuple, Self -import lz4.frame +import lz4.block import yaml LIST_LINE_WIDTH = 32 @@ -370,9 +370,11 @@ def generate_cxx_source_files_data( compress = compression != "none" compressed_files_bytes = ( - lz4.frame.compress( + lz4.block.compress( uncompressed_files_bytes, - compression_level=lz4.frame.COMPRESSIONLEVEL_MAX, + mode="high_compression", + compression = 12, + store_size = False ) if compress else None @@ -476,9 +478,11 @@ def generate_cxx_blobs_data( compress = compression != "none" compressed_blob_bytes = ( - lz4.frame.compress( + lz4.block.compress( uncompressed_blob_bytes, - compression_level=lz4.frame.COMPRESSIONLEVEL_MAX, + mode="high_compression", + compression = 12, + store_size = False ) if compress else None diff --git a/cpp/cmake/thirdparty/get_lz4.cmake b/cpp/cmake/thirdparty/get_lz4.cmake index 5f2f5d1b8e46..42c79f9af425 100644 --- a/cpp/cmake/thirdparty/get_lz4.cmake +++ b/cpp/cmake/thirdparty/get_lz4.cmake @@ -10,7 +10,7 @@ function(find_and_configure_lz4) set(CPM_DOWNLOAD_lz4 ON) rapids_cpm_find( - lz4 1.5.7 + lz4 dev GLOBAL_TARGETS lz4 CPM_ARGS GIT_REPOSITORY https://github.com/lz4/lz4.git @@ -20,12 +20,11 @@ function(find_and_configure_lz4) if(lz4_ADDED) add_library( - lz4_static STATIC ${lz4_SOURCE_DIR}/lib/lz4file.c ${lz4_SOURCE_DIR}/lib/lz4frame.c - ${lz4_SOURCE_DIR}/lib/lz4hc.c ${lz4_SOURCE_DIR}/lib/xxhash.c + lz4_objects OBJECT + ${lz4_SOURCE_DIR}/lib/lz4.c ${lz4_SOURCE_DIR}/lib/lz4file.c ${lz4_SOURCE_DIR}/lib/lz4frame.c + ${lz4_SOURCE_DIR}/lib/lz4hc.c ${lz4_SOURCE_DIR}/lib/xxhash.c ) - target_include_directories(lz4_static PUBLIC $) - - set_target_properties(lz4_static PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_include_directories(lz4_objects PUBLIC $) endif() endfunction() diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index d6b675dc73d7..47e2a11e3de5 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -176,6 +176,7 @@ void jit_bundle::ensure_installed() const throw_posix(std::format("Failed to get stat for directory ({})", expected_path), "lstat"); } else { // ensure base install directory exists + CUDF_LOG_INFO("Creating JIT install directory at ({})", expected_path); std::filesystem::create_directories(install_dir_); create_and_install_cudf_jit(expected_path.c_str()); } @@ -444,10 +445,10 @@ library compile_and_link_udf(char const* name, options_cstr.emplace_back(option.c_str()); } - blob_view const link_fragments[] = {library->get(binary_type::LTO_IR)->view(), + blob_view const link_fragments[] = {library->get(binary_type::FATBIN)->view(), fragment->get(binary_type::LTO_IR)->view()}; - binary_type const fragment_binary_types[] = {binary_type::LTO_IR, binary_type::LTO_IR}; + binary_type const fragment_binary_types[] = {binary_type::FATBIN, binary_type::LTO_IR}; char const* const fragment_names[] = {"cudf_lto_library", "cudf_udf_fragment"}; From 82ddae09cb7e8fb1edcc6540ab2007e5e23e36e8 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Feb 2026 07:18:55 +0000 Subject: [PATCH 027/254] Integrate Zstandard (zstd) compression support in JIT embedding and update related functions --- cpp/CMakeLists.txt | 2 +- cpp/cmake/Modules/JitEmbed.cmake | 7 +++- cpp/cmake/Modules/jit_embed.py | 63 +++++++++++++++++++++++------- cpp/include/cudf/jit/lto/thunk.cuh | 4 +- cpp/include/cudf/jit/lto/types.cuh | 4 +- cpp/src/jit/lto/operators.inl.cuh | 4 +- cpp/src/jit/lto/types.inl.cuh | 4 +- cpp/src/jit/rtc/cudf.cpp | 62 +++++++++++++++++++++++------ dependencies.yaml | 1 + 9 files changed, 113 insertions(+), 38 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f21420a5617e..8e7a76263b5d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -406,7 +406,7 @@ endforeach() # it should be as small as possible jit_add_options(cudf_jit_embed OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) jit_add_blob(cudf_jit_embed FILE "$" DEST cudf_lto_library.fatbin) -jit_embed(cudf_jit_embed COMPRESSION lz4) +jit_embed(cudf_jit_embed COMPRESSION zstd) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- diff --git a/cpp/cmake/Modules/JitEmbed.cmake b/cpp/cmake/Modules/JitEmbed.cmake index f799f4f97654..bed1e0a47459 100644 --- a/cpp/cmake/Modules/JitEmbed.cmake +++ b/cpp/cmake/Modules/JitEmbed.cmake @@ -164,8 +164,11 @@ function(jit_embed) message(FATAL_ERROR "COMPRESSION argument is required") endif() - if(NOT ARG_COMPRESSION STREQUAL "none" AND NOT ARG_COMPRESSION STREQUAL "lz4") - message(FATAL_ERROR "COMPRESSION argument must be either none or lz4") + if(NOT ARG_COMPRESSION STREQUAL "none" + AND NOT ARG_COMPRESSION STREQUAL "lz4" + AND NOT ARG_COMPRESSION STREQUAL "zstd" + ) + message(FATAL_ERROR "COMPRESSION argument must be either none, lz4, or, zstd") endif() string(APPEND TARGET_YAML "\"${TARGET}_sources\":\n") diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py index bc00ee917ae3..b5a5f732e8cf 100644 --- a/cpp/cmake/Modules/jit_embed.py +++ b/cpp/cmake/Modules/jit_embed.py @@ -8,6 +8,7 @@ import lz4.block import yaml +import zstd LIST_LINE_WIDTH = 32 NAMESPACE_PREFIX = "jit_" @@ -206,6 +207,19 @@ def var(self: Self) -> CXXVarDecl: return CXXVarDecl(id=self.id, expr=expr) +class CXXAsciiStringDecl(NamedTuple): + id: str + string: str + + @staticmethod + def of_string(id: str, string: str) -> Self: + return CXXAsciiStringDecl(id=id, string=string) + + def var(self: Self) -> CXXVarDecl: + expr = f"""static char const {self.id}[] = "{self.string}";""" + return CXXVarDecl(id=self.id, expr=expr) + + class CXXArrayOfBytesDecl(NamedTuple): id: str data: bytes @@ -355,6 +369,27 @@ def load_file_bytes(file_path: str) -> bytes: return f.read() +def compress_bytes(data: bytes, compression: str) -> bytes: + assert compression in ("none", "lz4", "zstd"), "Invalid compression type" + + if compression == "none": + return data + elif compression == "lz4": + compressed_data = lz4.block.compress( + data, mode="high_compression", compression=12, store_size=False + ) + logger.info( + f"Uncompressed size is {len(data)} bytes, compressed size is {len(compressed_data)} bytes" + ) + return compressed_data + elif compression == "zstd": + compressed_data = zstd.compress(data, 22) + logger.info( + f"Uncompressed size is {len(data)} bytes, compressed size is {len(compressed_data)} bytes" + ) + return compressed_data + + def generate_cxx_source_files_data( id: str, file_paths: list[str], @@ -366,16 +401,10 @@ def generate_cxx_source_files_data( [load_file_bytes(p) for p in file_paths] ) - assert compression in ("none", "lz4"), "Invalid compression type" compress = compression != "none" compressed_files_bytes = ( - lz4.block.compress( - uncompressed_files_bytes, - mode="high_compression", - compression = 12, - store_size = False - ) + compress_bytes(uncompressed_files_bytes, compression) if compress else None ) @@ -429,6 +458,10 @@ def generate_cxx_source_files_data( id=f"{id}_ranges", ranges=files_ranges ) + binary_compression_decl = CXXAsciiStringDecl.of_string( + id=f"{id}_compression", string=compression + ) + include_directories_decls: CXXArrayOfBytesDecl = ( CXXArrayOfBytesDecl.of_byte_ranges( id=f"{id}_include_directories", @@ -445,6 +478,8 @@ def generate_cxx_source_files_data( {binary_size_decl.var().decl()} +{binary_compression_decl.var().decl()} + {binary_ranges_decl.var().decl()} {include_directories_decls.var().decl()} @@ -474,16 +509,10 @@ def generate_cxx_blobs_data( [load_file_bytes(p) for p in blob_paths] ) - assert compression in ("none", "lz4"), "Invalid compression type" compress = compression != "none" compressed_blob_bytes = ( - lz4.block.compress( - uncompressed_blob_bytes, - mode="high_compression", - compression = 12, - store_size = False - ) + compress_bytes(uncompressed_blob_bytes, compression) if compress else None ) @@ -522,6 +551,10 @@ def generate_cxx_blobs_data( id=f"{id}_uncompressed_size", size=len(uncompressed_blob_bytes) ) + binary_compression_decl = CXXAsciiStringDecl.of_string( + id=f"{id}_compression", string=compression + ) + binary_embed_decl: AsmEmbedDecl = AsmEmbedDecl.of_file( id=f"{id}_binary", file=binary_file_name ) @@ -536,6 +569,8 @@ def generate_cxx_blobs_data( {binary_decl.var().decl()} +{binary_compression_decl.var().decl()} + {binary_size_decl.var().decl()} {binary_ranges_decl.var().decl()} diff --git a/cpp/include/cudf/jit/lto/thunk.cuh b/cpp/include/cudf/jit/lto/thunk.cuh index 699ae1388fe2..7af61eca0f39 100644 --- a/cpp/include/cudf/jit/lto/thunk.cuh +++ b/cpp/include/cudf/jit/lto/thunk.cuh @@ -13,7 +13,7 @@ #include -namespace CUDF_EXPORT cudf { +namespace CUDF_LTO_EXPORT cudf { namespace lto { @@ -132,4 +132,4 @@ CUDF_LTO_MAP(lto::mutable_column_device_view_core, cudf::mutable_column_device_v } // namespace lto -} // namespace CUDF_EXPORT cudf +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index 584d83f8e904..e71a765ca902 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -4,8 +4,8 @@ */ #pragma once -#define CUDF_LTO_EXPORT -#define CUDF_LTO_ALIAS __attribute__((may_alias)) +#define CUDF_LTO_EXPORT __attribute__((visibility("default"))) +#define CUDF_LTO_ALIAS __attribute__((may_alias)) namespace CUDF_LTO_EXPORT cudf { diff --git a/cpp/src/jit/lto/operators.inl.cuh b/cpp/src/jit/lto/operators.inl.cuh index 3bb35bcd67f5..e0ae566f6ff8 100644 --- a/cpp/src/jit/lto/operators.inl.cuh +++ b/cpp/src/jit/lto/operators.inl.cuh @@ -9,7 +9,7 @@ #include #include -namespace CUDF_EXPORT cudf { +namespace CUDF_LTO_EXPORT cudf { namespace lto { template @@ -563,4 +563,4 @@ CUDF_LTO_DEF(logical_not, NOT, uint64_t); #undef CUDF_LTO_DEF } // namespace lto -} // namespace CUDF_EXPORT cudf +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/jit/lto/types.inl.cuh b/cpp/src/jit/lto/types.inl.cuh index 340496ec263a..92aeab8eca3f 100644 --- a/cpp/src/jit/lto/types.inl.cuh +++ b/cpp/src/jit/lto/types.inl.cuh @@ -7,7 +7,7 @@ #include #include -namespace CUDF_EXPORT cudf { +namespace CUDF_LTO_EXPORT cudf { namespace lto { [[nodiscard]] __device__ size_type string_view::size_bytes() const @@ -362,4 +362,4 @@ CUDF_LTO_INST(duration_ns); #undef CUDF_LTO_INST } // namespace lto -} // namespace CUDF_EXPORT cudf +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 47e2a11e3de5..66f1fef7ffb1 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -85,24 +86,57 @@ void install_file(char const* dst_path, std::span contents) } } -void install_file_set(char const* target_dir, - jit_bytes_t const& compressed_binary, - size_t uncompressed_size, - std::span file_ranges, - jit_bytes_array_t const& dst) +std::vector decompress_blob(jit_bytes_t const& compressed_binary, + size_t uncompressed_size, + char const* compression) { std::vector decompressed; decompressed.resize(uncompressed_size); - int errc = LZ4_decompress_safe(reinterpret_cast(compressed_binary.data), - reinterpret_cast(decompressed.data()), - compressed_binary.size, - uncompressed_size); + if (std::string_view{compression} == "lz4") { + int errc = LZ4_decompress_safe(reinterpret_cast(compressed_binary.data), + reinterpret_cast(decompressed.data()), + compressed_binary.size, + uncompressed_size); + + CUDF_EXPECTS( + errc == static_cast(uncompressed_size), + +std::format("Failed to decompress embedded RTC source files with LZ4, error code {}", errc), + std::runtime_error); + + } else if (std::string_view{compression} == "zstd") { + size_t const errc = ZSTD_decompress( + decompressed.data(), uncompressed_size, compressed_binary.data, compressed_binary.size); + + CUDF_EXPECTS( + !ZSTD_isError(errc) && errc == uncompressed_size, + +std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : ", + errc, + ZSTD_getErrorName(errc)), + std::runtime_error); + } else { + // compression is "none", so just copy the data + std::copy( + compressed_binary.data, compressed_binary.data + compressed_binary.size, decompressed.data()); + } - CUDF_EXPECTS(errc == static_cast(uncompressed_size), - "Failed to decompress embedded RTC source files", + return decompressed; +} + +void install_file_set(char const* target_dir, + jit_bytes_t const& compressed_binary, + size_t uncompressed_size, + std::span file_ranges, + jit_bytes_array_t const& dst, + char const* compression) +{ + CUDF_EXPECTS(compression != nullptr, "Compression type must be specified", std::runtime_error); + CUDF_EXPECTS(compression == std::string_view{"none"} || compression == std::string_view{"lz4"} || + compression == std::string_view{"zstd"}, + +std::format("Unsupported compression type specified: {}", compression), std::runtime_error); + auto decompressed = decompress_blob(compressed_binary, uncompressed_size, compression); auto const files_data = decompressed.data(); for (size_t i = 0; i < file_ranges.size(); ++i) { @@ -125,13 +159,15 @@ void install_cudf_jit(char const* target_dir) cudf_jit_embed_blobs_binary, cudf_jit_embed_blobs_uncompressed_size, cudf_jit_embed_blobs_ranges, - cudf_jit_embed_blobs_file_destinations); + cudf_jit_embed_blobs_file_destinations, + cudf_jit_embed_blobs_compression); install_file_set(target_dir, cudf_jit_embed_sources_binary, cudf_jit_embed_sources_uncompressed_size, cudf_jit_embed_sources_ranges, - cudf_jit_embed_sources_file_destinations); + cudf_jit_embed_sources_file_destinations, + cudf_jit_embed_sources_compression); } void create_and_install_cudf_jit(char const* target_dir) diff --git a/dependencies.yaml b/dependencies.yaml index 0aae110609ee..110c1218900b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -480,6 +480,7 @@ dependencies: - cxx-compiler - dlpack>=0.8,<1.0 - zlib>=1.2.13 + - python-zstd>=1.5.7.2 specific: - output_types: conda matrices: From 43a4e0894ea19e3e137ba3cc256cbaebe2b0ff94 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Feb 2026 07:19:28 +0000 Subject: [PATCH 028/254] Add python-zstd dependency to environment YAML files --- conda/environments/all_cuda-129_arch-aarch64.yaml | 1 + conda/environments/all_cuda-129_arch-x86_64.yaml | 1 + conda/environments/all_cuda-131_arch-aarch64.yaml | 1 + conda/environments/all_cuda-131_arch-x86_64.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 0115a6f6feea..1dfba26d5e41 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -80,6 +80,7 @@ dependencies: - pytest-xdist - python-confluent-kafka>=2.8.0,<2.9.0 - python-xxhash +- python-zstd>=1.5.7.2 - python>=3.11,<3.14 - pytorch>=2.4.0 - rapids-build-backend>=0.4.0,<0.5.0 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 9113a7a1b92a..be2cc8909a8c 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -80,6 +80,7 @@ dependencies: - pytest-xdist - python-confluent-kafka>=2.8.0,<2.9.0 - python-xxhash +- python-zstd>=1.5.7.2 - python>=3.11,<3.14 - pytorch>=2.4.0 - rapids-build-backend>=0.4.0,<0.5.0 diff --git a/conda/environments/all_cuda-131_arch-aarch64.yaml b/conda/environments/all_cuda-131_arch-aarch64.yaml index ed350adcb74c..eec9292a1314 100644 --- a/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -80,6 +80,7 @@ dependencies: - pytest-xdist - python-confluent-kafka>=2.8.0,<2.9.0 - python-xxhash +- python-zstd>=1.5.7.2 - python>=3.11,<3.14 - rapids-build-backend>=0.4.0,<0.5.0 - rapids-dask-dependency==26.4.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-131_arch-x86_64.yaml b/conda/environments/all_cuda-131_arch-x86_64.yaml index 5c14dfca99bb..7e5c62425e4b 100644 --- a/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -80,6 +80,7 @@ dependencies: - pytest-xdist - python-confluent-kafka>=2.8.0,<2.9.0 - python-xxhash +- python-zstd>=1.5.7.2 - python>=3.11,<3.14 - rapids-build-backend>=0.4.0,<0.5.0 - rapids-dask-dependency==26.4.*,>=0.0.0a0 From 37976fca295ac8ca8b58348587c42243092fc698 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Feb 2026 10:03:29 +0000 Subject: [PATCH 029/254] update --- cpp/src/jit/rtc/cudf.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 66f1fef7ffb1..1cf096909555 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -30,6 +30,9 @@ #include #include +// TODO: remove +#define CHECKPOINT std::cout << "Checkpoint: " << __FILE__ << ":" << __LINE__ << std::endl + #define CUDFRTC_CHECK_CUDART(msg, ...) \ do { \ ::cudaError_t __result = (__VA_ARGS__); \ @@ -196,6 +199,9 @@ jit_bundle::jit_bundle(std::string install_dir) : install_dir_{std::move(install { ensure_installed(); preload_lto_library(); + + // TODO: recursive because of call to get_bundle() and get_cache() + // TODO: fix cmake change tracking } void jit_bundle::ensure_installed() const From 17ebfa0c47ef989ea27cc67b0bc7d4035dddc9a5 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Feb 2026 11:41:31 +0000 Subject: [PATCH 030/254] refactoring + multi-threaded setup fix --- cpp/include/cudf/context.hpp | 14 ++-- cpp/include/cudf_test/testing_main.hpp | 3 + cpp/src/io/parquet/reader_impl.cpp | 2 +- cpp/src/jit/cache.cpp | 4 +- cpp/src/jit/row_ir.cpp | 4 +- cpp/src/runtime/context.cpp | 80 ++++++++++++--------- cpp/src/runtime/context.hpp | 27 ++++--- cpp/tests/utilities_tests/context_tests.cpp | 63 ++++++++++++++-- 8 files changed, 141 insertions(+), 56 deletions(-) diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index 76f279dc5fc2..5a892853ba19 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -64,10 +64,14 @@ constexpr bool has_flag(init_flags flags, init_flags flag) noexcept return (flags | flag) == flags; } -/// @brief Initialize the cuDF global context +/// @brief Initialize the cudf global context /// @param flags Optional flags to control which initialization steps to perform. -/// Can be called multiple times. Only the first call will have an effect. -/// @note Thread-safety: This function is thread-safe. -void initialize(init_flags flags = init_flags::ALL); +/// Can be called multiple times to initialize additional components. If all selected +/// steps are already performed, the call has no effect. +void initialize(init_flags flags = init_flags::INIT_JIT_CACHE); + +/// @brief teardown the cudf global context, resetting it to an uninitialized state. This is primarily +/// intended for testing purposes, allowing for re-initialization of the context after teardown. +void teardown(); } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf_test/testing_main.hpp b/cpp/include/cudf_test/testing_main.hpp index 4c22a17bed6f..11e5c3f2bf7b 100644 --- a/cpp/include/cudf_test/testing_main.hpp +++ b/cpp/include/cudf_test/testing_main.hpp @@ -222,6 +222,7 @@ inline void init_cudf_test(int argc, char** argv, cudf::test::config const& conf #define CUDF_TEST_PROGRAM_MAIN() \ int main(int argc, char** argv) \ { \ + cudf::initialize(); \ ::testing::InitGoogleTest(&argc, argv); \ init_cudf_test(argc, argv); \ if (std::getenv("GTEST_CUDF_MEMORY_PEAK")) { \ @@ -230,9 +231,11 @@ inline void init_cudf_test(int argc, char** argv, cudf::test::config const& conf cudf::set_current_device_resource(&mr); \ auto rc = RUN_ALL_TESTS(); \ std::cout << "Peak memory usage " << mr.get_bytes_counter().peak << " bytes" << std::endl; \ + cudf::teardown(); \ return rc; \ } else { \ auto rc = RUN_ALL_TESTS(); \ + cudf::teardown(); \ return rc; \ } \ } diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index abbbe73990ee..1c612de61df5 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -858,7 +858,7 @@ table_with_metadata reader_impl::finalize_output(read_mode mode, if (_num_filter_only_columns > 0) { out_metadata.schema_info.resize(output_count); } - bool use_jit = cudf::get_context(init_flags::NONE).use_jit() || _options.use_jit_filter; + bool use_jit = cudf::get_context().use_jit() || _options.use_jit_filter; if (!use_jit) { auto predicate = cudf::detail::compute_column( diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 3d8ffc757de9..7d03ed8fdbca 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -131,6 +131,6 @@ jitify2::ProgramCache<>& jit::program_cache::get(jitify2::PreprocessedProgramDat jitify2::ProgramCache<>& jit::get_program_cache(jitify2::PreprocessedProgramData const& preprog) { - return cudf::get_context(init_flags::INIT_JIT_CACHE).program_cache().get(preprog); + return cudf::get_context().program_cache().get(preprog); } } // namespace cudf diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index c46a441b2818..a0adc1489d78 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -586,7 +586,7 @@ transform_args ast_converter::compute_column(target target_id, is_null_aware, output_nullability}; - if (get_context(init_flags::NONE).dump_codegen()) { + if (get_context().dump_codegen()) { std::cout << "Generated code for transform: " << transform.udf << std::endl; } @@ -643,7 +643,7 @@ filter_args ast_converter::filter(target target_id, std::nullopt, is_null_aware}; - if (get_context(init_flags::NONE).dump_codegen()) { + if (get_context().dump_codegen()) { std::cout << "Generated code for filter: " << filter.predicate_udf << std::endl; } diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 46e89c2cddd8..a5d5028f8081 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -16,45 +16,40 @@ namespace cudf { -context::context(init_flags flags) : _program_cache{nullptr} +context::context(context_config const& cfg, init_flags flags) + : _config{cfg}, _program_cache_init_flag{}, _program_cache{nullptr} { - auto dump_codegen_env = getenv_or("LIBCUDF_JIT_DUMP_CODEGEN", std::string{"OFF"}); - bool dump_codegen = - (dump_codegen_env == "ON" || dump_codegen_env == "on" || dump_codegen_env == "1"); - - auto use_jit_env = getenv_or("LIBCUDF_JIT_ENABLED", std::string{"OFF"}); - bool use_jit = (use_jit_env == "ON" || use_jit_env == "on" || use_jit_env == "1"); - - _dump_codegen = dump_codegen; + initialize_components(flags); +} - flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); +void context::ensure_nvcomp_loaded() { io::detail::nvcomp::load_nvcomp_library(); } - initialize_components(flags); +void context::ensure_jit_cache_initialized() +{ + std::call_once(_program_cache_init_flag, + [&]() { _program_cache = std::make_unique(); }); } jit::program_cache& context::program_cache() { - CUDF_EXPECTS(_program_cache != nullptr, "JIT cache is not initialized", std::runtime_error); + ensure_jit_cache_initialized(); return *_program_cache; } -bool context::dump_codegen() const { return _dump_codegen; } +bool context::dump_codegen() const { return _config.dump_codegen; } + +bool context::use_jit() const { return _config.use_jit; } void context::initialize_components(init_flags flags) { - if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { - _program_cache = std::make_unique(); - } + if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { ensure_jit_cache_initialized(); } if (has_flag(flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } - - _initialized_flags = flags; } -bool context::use_jit() const { return _use_jit; } - -static std::unique_ptr ctx; -static std::once_flag ctx_init_flag{}; +static std::optional _context{std::nullopt}; +static std::optional _context_init_flag{std::in_place}; +static std::optional _context_deinit_flag{std::in_place}; } // namespace cudf @@ -62,21 +57,42 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { - std::call_once(ctx_init_flag, [&]() { - auto c = std::make_unique(); - c->initialize_components(flags); - ctx = std::move(c); + std::call_once(*_context_init_flag, [&]() { + auto dump_codegen_env = getenv_or("LIBCUDF_JIT_DUMP_CODEGEN", std::string{"OFF"}); + bool dump_codegen = + (dump_codegen_env == "ON" || dump_codegen_env == "on" || dump_codegen_env == "1"); + + auto use_jit_env = getenv_or("LIBCUDF_JIT_ENABLED", std::string{"OFF"}); + bool use_jit = (use_jit_env == "ON" || use_jit_env == "on" || use_jit_env == "1"); + + flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); + + context_config cfg{ + .dump_codegen = dump_codegen, + .use_jit = use_jit, + }; + + _context.emplace(cfg, flags); }); - CUDF_EXPECTS(has_flag(ctx->_initialized_flags, flags), - "CUDF's context has already been initialized with incompatible flags", - std::runtime_error); + _context->initialize_components(flags); +} + +void teardown() +{ + std::call_once(*_context_deinit_flag, [&]() { + _context.reset(); + _context_init_flag.reset(); + _context_deinit_flag.reset(); + _context_init_flag.emplace(); + _context_deinit_flag.emplace(); + }); } -context& get_context(init_flags expected) +context& get_context() { - initialize(expected); - return *ctx; + cudf::initialize(); + return *_context; } } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 38fbc7d67cbe..f95b9684034e 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -9,6 +9,7 @@ #include #include +#include namespace cudf { @@ -16,18 +17,28 @@ namespace jit { class program_cache; } +struct [[nodiscard]] context_config { + bool dump_codegen = false; + bool use_jit = false; +}; + /// @brief The context object contains global state internal to CUDF. /// It helps to ensure structured and well-defined construction and destruction of global /// objects/state across translation units. class context { + public: private: + context_config _config; + std::once_flag _program_cache_init_flag; std::unique_ptr _program_cache; - init_flags _initialized_flags = init_flags::NONE; - bool _dump_codegen = false; - bool _use_jit = false; + + private: + void ensure_nvcomp_loaded(); + + void ensure_jit_cache_initialized(); public: - context(init_flags flags = init_flags::INIT_JIT_CACHE); + context(context_config const& cfg = {}, init_flags flags = init_flags::INIT_JIT_CACHE); context(context const&) = delete; context& operator=(context const&) = delete; context(context&&) = delete; @@ -40,16 +51,12 @@ class context { [[nodiscard]] bool use_jit() const; - private: - /// @brief Initialize components based on the provided flags + /// @brief Initialize additional components based on the provided flags /// @param flags The initialization flags to process void initialize_components(init_flags flags); - - friend void initialize(init_flags flags); }; /// @brief Get the cuDF global context -/// @param expected the initialization flags that are expected -context& get_context(init_flags expected); +context& get_context(); } // namespace cudf diff --git a/cpp/tests/utilities_tests/context_tests.cpp b/cpp/tests/utilities_tests/context_tests.cpp index 25af0649c844..24b7293c9b2e 100644 --- a/cpp/tests/utilities_tests/context_tests.cpp +++ b/cpp/tests/utilities_tests/context_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,13 +9,68 @@ #include -struct ContextTest : public cudf::test::BaseFixture {}; +struct ContextTest : public cudf::test::BaseFixture { + ~ContextTest() override + { + try { + cudf::teardown(); + } catch (...) { + } + } +}; TEST_F(ContextTest, MultipleInitializeCalls) { cudf::initialize(cudf::init_flags::INIT_JIT_CACHE); - EXPECT_THROW(cudf::initialize(cudf::init_flags::LOAD_NVCOMP), std::runtime_error); - EXPECT_THROW(cudf::initialize(cudf::init_flags::ALL), std::runtime_error); + EXPECT_NO_THROW(cudf::initialize(cudf::init_flags::LOAD_NVCOMP)); + EXPECT_NO_THROW(cudf::initialize(cudf::init_flags::ALL)); +} + +TEST_F(ContextTest, InitializeAfterTeardown) +{ + cudf::initialize(cudf::init_flags::ALL); + cudf::teardown(); + EXPECT_NO_THROW(cudf::initialize(cudf::init_flags::INIT_JIT_CACHE)); } + +TEST_F(ContextTest, TeardownWithoutInitialize) { EXPECT_NO_THROW(cudf::teardown()); } + +TEST_F(ContextTest, MultipleTeardownCalls) +{ + cudf::initialize(cudf::init_flags::ALL); + cudf::teardown(); + + EXPECT_NO_THROW(cudf::teardown()); +} + +template +void run_multithreaded(Lambda func) +{ + std::vector threads; + auto concurrency = std::thread::hardware_concurrency(); + + for (size_t i = 0; i < concurrency; ++i) { + threads.emplace_back([i, f = std::move(func)]() { f(i); }); + } + + for (auto& t : threads) { + t.join(); + } +} + +TEST_F(ContextTest, MultipleInitializeCallsMultiThreaded) +{ + auto init_task = [](size_t thread_id) { + auto role = thread_id % 3; + if (role == 0) { + EXPECT_NO_THROW(cudf::initialize(cudf::init_flags::INIT_JIT_CACHE)); + } else if (role == 1) { + EXPECT_NO_THROW(cudf::initialize(cudf::init_flags::LOAD_NVCOMP)); + } else { + EXPECT_NO_THROW(cudf::initialize(cudf::init_flags::ALL)); + } + }; + EXPECT_NO_FATAL_FAILURE(run_multithreaded(init_task)); +} From 6fbc5a4bd33deedd44683ea400c2efc7c024d60e Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Feb 2026 20:18:18 +0000 Subject: [PATCH 031/254] enhance teardown documentation with usage warnings and clarify purpose --- cpp/include/cudf/context.hpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index 5a892853ba19..6caa99fd28a9 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -70,8 +70,12 @@ constexpr bool has_flag(init_flags flags, init_flags flag) noexcept /// steps are already performed, the call has no effect. void initialize(init_flags flags = init_flags::INIT_JIT_CACHE); -/// @brief teardown the cudf global context, resetting it to an uninitialized state. This is primarily -/// intended for testing purposes, allowing for re-initialization of the context after teardown. +/// @brief Destroy the cudf global context, resetting it to an uninitialized state. This is +/// primarily intended for testing purposes, allowing for re-initialization of the context after +/// teardown. +/// @warning This is not intended for general use and may lead to undefined behavior if used +/// improperly. The caller must ensure that no threads are concurrently accessing the context during +/// teardown and that only one thread calls teardown at a time. void teardown(); } // namespace CUDF_EXPORT cudf From 5aa1ea5881626d17944b7129ca23037d43e4a93f Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Feb 2026 20:23:36 +0000 Subject: [PATCH 032/254] update copyright year to 2026 in header files --- cpp/include/cudf/context.hpp | 2 +- cpp/include/cudf_test/testing_main.hpp | 4 ++-- cpp/tests/utilities_tests/context_tests.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index 6caa99fd28a9..e9172c91cdbf 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cudf_test/testing_main.hpp b/cpp/include/cudf_test/testing_main.hpp index 11e5c3f2bf7b..d1994be5723b 100644 --- a/cpp/include/cudf_test/testing_main.hpp +++ b/cpp/include/cudf_test/testing_main.hpp @@ -231,11 +231,11 @@ inline void init_cudf_test(int argc, char** argv, cudf::test::config const& conf cudf::set_current_device_resource(&mr); \ auto rc = RUN_ALL_TESTS(); \ std::cout << "Peak memory usage " << mr.get_bytes_counter().peak << " bytes" << std::endl; \ - cudf::teardown(); \ + cudf::teardown(); \ return rc; \ } else { \ auto rc = RUN_ALL_TESTS(); \ - cudf::teardown(); \ + cudf::teardown(); \ return rc; \ } \ } diff --git a/cpp/tests/utilities_tests/context_tests.cpp b/cpp/tests/utilities_tests/context_tests.cpp index 24b7293c9b2e..eaf370999e45 100644 --- a/cpp/tests/utilities_tests/context_tests.cpp +++ b/cpp/tests/utilities_tests/context_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ From fab6e6eb27c9a191f00111edcede0e3967bdc104 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Feb 2026 20:29:50 +0000 Subject: [PATCH 033/254] enhance teardown function to reset context and improve resource management --- cpp/src/runtime/context.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index a5d5028f8081..4a7fda8e0dae 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -81,9 +81,9 @@ void initialize(init_flags flags) void teardown() { std::call_once(*_context_deinit_flag, [&]() { + // reset the context to destroy all global objects and release resources, allowing for clean + // re-initialization in the future if desired. _context.reset(); - _context_init_flag.reset(); - _context_deinit_flag.reset(); _context_init_flag.emplace(); _context_deinit_flag.emplace(); }); From 57382182d9ed8c4cabe249938da77fe129038083 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Feb 2026 20:31:26 +0000 Subject: [PATCH 034/254] fix: adjust concurrency calculation in multithreaded test utility --- cpp/tests/utilities_tests/context_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/tests/utilities_tests/context_tests.cpp b/cpp/tests/utilities_tests/context_tests.cpp index eaf370999e45..c0192b3a5589 100644 --- a/cpp/tests/utilities_tests/context_tests.cpp +++ b/cpp/tests/utilities_tests/context_tests.cpp @@ -49,7 +49,7 @@ template void run_multithreaded(Lambda func) { std::vector threads; - auto concurrency = std::thread::hardware_concurrency(); + auto concurrency = std::max(std::thread::hardware_concurrency(), 1U) - 1U; for (size_t i = 0; i < concurrency; ++i) { threads.emplace_back([i, f = std::move(func)]() { f(i); }); From c34420afc358927e8bdda5685f7a0abdf4756e2d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Feb 2026 21:09:12 +0000 Subject: [PATCH 035/254] fix: adjust concurrency calculation in multithreaded utility --- cpp/tests/utilities_tests/context_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/tests/utilities_tests/context_tests.cpp b/cpp/tests/utilities_tests/context_tests.cpp index c0192b3a5589..32ff707e3beb 100644 --- a/cpp/tests/utilities_tests/context_tests.cpp +++ b/cpp/tests/utilities_tests/context_tests.cpp @@ -49,7 +49,7 @@ template void run_multithreaded(Lambda func) { std::vector threads; - auto concurrency = std::max(std::thread::hardware_concurrency(), 1U) - 1U; + auto concurrency = std::max(std::thread::hardware_concurrency(), 2U) - 1U; for (size_t i = 0; i < concurrency; ++i) { threads.emplace_back([i, f = std::move(func)]() { f(i); }); From 8b583920935bdaa9bbd19464f888a5f40c95625f Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 9 Feb 2026 05:19:49 +0000 Subject: [PATCH 036/254] feat: enhance initialization flags and context management for JIT and RTC caches --- cpp/include/cudf/context.hpp | 12 ++++-- cpp/src/jit/rtc/cache.hpp | 5 +-- cpp/src/jit/rtc/cudf.cpp | 39 ++++++++----------- cpp/src/jit/rtc/cudf.hpp | 8 ++-- cpp/src/runtime/context.cpp | 73 ++++++++++++++++++++++++++++-------- cpp/src/runtime/context.hpp | 14 ++++--- 6 files changed, 95 insertions(+), 56 deletions(-) diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index e9172c91cdbf..e56862520034 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -20,8 +20,14 @@ enum class init_flags : std::uint32_t { LOAD_NVCOMP = 1 << 0, /// @brief Initialize the JIT program cache during initialization INIT_JIT_CACHE = 1 << 1, - /// @brief All initialization steps (default behavior) - ALL = LOAD_NVCOMP | INIT_JIT_CACHE + /// @brief Initialize the RTC cache during initialization + INIT_RTC_CACHE = 1 << 2, + /// @brief Initialize the JIT bundle during initialization + INIT_JIT_BUNDLE = 1 << 3, + /// @brief The default initialization steps performed by `initialize()` if no flags are provided + DEFAULT = INIT_JIT_CACHE | INIT_RTC_CACHE | INIT_JIT_BUNDLE, + /// @brief All initialization steps + ALL = LOAD_NVCOMP | INIT_JIT_CACHE | INIT_RTC_CACHE | INIT_JIT_BUNDLE }; /// @brief Bitwise OR operator for init_flags @@ -68,7 +74,7 @@ constexpr bool has_flag(init_flags flags, init_flags flag) noexcept /// @param flags Optional flags to control which initialization steps to perform. /// Can be called multiple times to initialize additional components. If all selected /// steps are already performed, the call has no effect. -void initialize(init_flags flags = init_flags::INIT_JIT_CACHE); +void initialize(init_flags flags = init_flags::DEFAULT); /// @brief Destroy the cudf global context, resetting it to an uninitialized state. This is /// primarily intended for testing purposes, allowing for re-initialization of the context after diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index 4e52b2f0cadf..0b2a0623aa15 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -37,6 +37,8 @@ struct [[nodiscard]] cache_limits { uint32_t num_blobs = 1024; uint32_t num_fragments = 1024; uint32_t num_libraries = 1024; + + static constexpr cache_limits default_limits() { return cache_limits{}; } }; namespace detail { @@ -198,9 +200,6 @@ struct cache_statistics_counter { /// copying across threads and disk. struct cache_t { private: - // TODO: store under more structured pathsi, i.e. /cache. - // TODO: stamp binaries so we can pre-load? - std::string cache_dir_; cache_limits limits_; diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 1cf096909555..72fb08c9a88f 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -57,8 +57,6 @@ sha256_hash hash_string(std::span input) return ctx.finalize(); } -cache_t& get_rtc_cache() { return cudf::get_context().rtc_cache(); } - [[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) { auto error_code = errno; @@ -195,16 +193,15 @@ void create_and_install_cudf_jit(char const* target_dir) } // namespace -jit_bundle::jit_bundle(std::string install_dir) : install_dir_{std::move(install_dir)} +jit_bundle_t::jit_bundle_t(std::string install_dir, cache_t& cache) + : install_dir_{std::move(install_dir)}, cache_{&cache} { ensure_installed(); preload_lto_library(); - - // TODO: recursive because of call to get_bundle() and get_cache() - // TODO: fix cmake change tracking + // TODO: fix cmake tracking of the scripts and embedded files } -void jit_bundle::ensure_installed() const +void jit_bundle_t::ensure_installed() const { CUDF_FUNC_RANGE(); @@ -230,9 +227,9 @@ void jit_bundle::ensure_installed() const } } -void jit_bundle::preload_lto_library() +void jit_bundle_t::preload_lto_library() { - auto& cache = get_rtc_cache(); + auto& cache = *cache_; auto bundle_hash = get_hash(); @@ -263,22 +260,22 @@ void jit_bundle::preload_lto_library() lto_library_ = fut.get(); } -std::string jit_bundle::get_hash() const +std::string jit_bundle_t::get_hash() const { auto str = sha256_hex_string::make( std::span{cudf_jit_embed_hash.data, static_cast(cudf_jit_embed_hash.size)}); return std::string{str.view()}; } -std::string jit_bundle::get_directory() const +std::string jit_bundle_t::get_directory() const { auto hash = get_hash(); return std::format("{}/{}", install_dir_, hash); } -fragment jit_bundle::get_lto_library() const { return lto_library_; } +fragment jit_bundle_t::get_lto_library() const { return lto_library_; } -std::vector jit_bundle::get_include_directories() const +std::vector jit_bundle_t::get_include_directories() const { std::vector directories; auto base_dir = get_directory(); @@ -295,7 +292,7 @@ std::vector jit_bundle::get_include_directories() const return directories; } -std::vector jit_bundle::get_compile_options() const +std::vector jit_bundle_t::get_compile_options() const { std::vector options; @@ -346,8 +343,8 @@ fragment get_or_compile_fragment(char const* name, char const* source_code_cstr, { CUDF_FUNC_RANGE(); - auto& bundle = get_bundle(); - auto& cache = get_rtc_cache(); + auto& bundle = cudf::get_context().jit_bundle(); + auto& cache = cudf::get_context().rtc_cache(); auto runtime = get_runtime_version(); auto driver = get_driver_version(); @@ -435,12 +432,6 @@ fragment get_or_compile_fragment(char const* name, char const* source_code_cstr, return fut.get(); } -jit_bundle& get_bundle() -{ - auto& context = cudf::get_context(); - return context.jit_bundle(); -} - library compile_and_link_udf(char const* name, char const* udf_code, char const* udf_key, @@ -448,8 +439,8 @@ library compile_and_link_udf(char const* name, { CUDF_FUNC_RANGE(); - auto& cache = get_rtc_cache(); - auto& bundle = get_bundle(); + auto& cache = cudf::get_context().rtc_cache(); + auto& bundle = cudf::get_context().jit_bundle(); auto runtime = get_runtime_version(); auto driver = get_driver_version(); diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/rtc/cudf.hpp index e8422b42e487..638bf0010d65 100644 --- a/cpp/src/jit/rtc/cudf.hpp +++ b/cpp/src/jit/rtc/cudf.hpp @@ -8,14 +8,16 @@ #pragma once #include +#include #include namespace CUDF_EXPORT cudf { namespace rtc { -class jit_bundle { +class jit_bundle_t { std::string install_dir_; fragment lto_library_; + cache_t* cache_; private: void ensure_installed() const; @@ -23,7 +25,7 @@ class jit_bundle { void preload_lto_library(); public: - jit_bundle(std::string install_dir); + jit_bundle_t(std::string install_dir, cache_t& cache); std::string get_hash() const; @@ -36,8 +38,6 @@ class jit_bundle { std::vector get_compile_options() const; }; -jit_bundle& get_bundle(); - library compile_and_link_udf(char const* name, char const* udf_code, char const* udf_key, diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 71117c5a9d7f..87ef50616305 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -18,8 +18,8 @@ namespace cudf { -context::context(context_config const& cfg, init_flags flags) - : _config{cfg}, _program_cache_init_flag{}, _program_cache{nullptr} +context::context(context_config cfg, init_flags flags) + : _config{std::move(cfg)}, _program_cache_init_flag{}, _program_cache{nullptr} { initialize_components(flags); } @@ -32,33 +32,42 @@ void context::ensure_jit_cache_initialized() [&]() { _program_cache = std::make_unique(); }); } -jit::program_cache& context::program_cache() +void context::ensure_rtc_cache_initialized() { - ensure_jit_cache_initialized(); - return *_program_cache; + std::call_once(_rtc_cache_init_flag, [&]() { + // make sure the rtc cache directory exists + std::filesystem::create_directories(_config.rtc_cache_dir); + _rtc_cache = + std::make_unique(_config.rtc_cache_dir, rtc::cache_limits::default_limits()); + }); } -rtc::cache_t& context::rtc_cache() +void context::ensure_jit_bundle_initialized() { - CUDF_EXPECTS(_rtc_cache != nullptr, "RTC cache not initialized", std::runtime_error); - return *_rtc_cache; + // note that jit_bundle depends on rtc_cache, so we ensure rtc_cache is initialized first. + ensure_rtc_cache_initialized(); + std::call_once(_jit_bundle_init_flag, [&]() { + // make sure the jit bundle directory exists + std::filesystem::create_directories(_config.jit_bundle_dir); + _jit_bundle = std::make_unique(_config.jit_bundle_dir, *_rtc_cache); + }); } -rtc::jit_bundle& context::jit_bundle() +jit::program_cache& context::program_cache() { - CUDF_EXPECTS(_jit_bundle != nullptr, "JIT bundle not initialized", std::runtime_error); - return *_jit_bundle; + ensure_jit_cache_initialized(); + return *_program_cache; } rtc::cache_t& context::rtc_cache() { - CUDF_EXPECTS(_rtc_cache != nullptr, "RTC cache not initialized", std::runtime_error); + ensure_rtc_cache_initialized(); return *_rtc_cache; } -rtc::jit_bundle& context::jit_bundle() +rtc::jit_bundle_t& context::jit_bundle() { - CUDF_EXPECTS(_jit_bundle != nullptr, "JIT bundle not initialized", std::runtime_error); + ensure_jit_bundle_initialized(); return *_jit_bundle; } @@ -71,12 +80,39 @@ void context::initialize_components(init_flags flags) if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { ensure_jit_cache_initialized(); } if (has_flag(flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } + + if (has_flag(flags, init_flags::INIT_RTC_CACHE)) { ensure_rtc_cache_initialized(); } + + if (has_flag(flags, init_flags::INIT_JIT_BUNDLE)) { ensure_jit_bundle_initialized(); } } static std::optional _context{std::nullopt}; static std::optional _context_init_flag{std::in_place}; static std::optional _context_deinit_flag{std::in_place}; +std::filesystem::path get_cudf_dir() +{ + auto cudf_dir_env = std::getenv("LIBCUDF_DIR"); + if (cudf_dir_env != nullptr) { + return std::filesystem::path(cudf_dir_env); + } else { + auto home_dir = std::getenv("HOME"); + CUDF_EXPECTS(home_dir != nullptr, "HOME environment variable is not set", std::runtime_error); + auto cudf_dir = std::filesystem::path(home_dir) / ".cudf"; + return cudf_dir; + } +} + +std::filesystem::path get_jit_bundle_dir() +{ + return getenv_or("LIBCUDF_JIT_BUNDLE_DIR", get_cudf_dir() / "jit_bundle"); +} + +std::filesystem::path get_rtc_cache_dir() +{ + return getenv_or("LIBCUDF_RTC_CACHE_DIR", get_cudf_dir() / "rtc_cache"); +} + } // namespace cudf namespace CUDF_EXPORT cudf { @@ -93,9 +129,14 @@ void initialize(init_flags flags) flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); + auto jit_bundle_dir = get_jit_bundle_dir(); + auto rtc_cache_dir = get_rtc_cache_dir(); + context_config cfg{ - .dump_codegen = dump_codegen, - .use_jit = use_jit, + .dump_codegen = dump_codegen, + .use_jit = use_jit, + .rtc_cache_dir = rtc_cache_dir, + .jit_bundle_dir = jit_bundle_dir, }; _context.emplace(cfg, flags); diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index d7905dd76d01..4de008e5ebcb 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -19,12 +19,14 @@ class program_cache; namespace rtc { class cache_t; -class jit_bundle; +class jit_bundle_t; } // namespace rtc struct [[nodiscard]] context_config { - bool dump_codegen = false; - bool use_jit = false; + bool dump_codegen = false; + bool use_jit = false; + std::string rtc_cache_dir = {}; + std::string jit_bundle_dir = {}; }; /// @brief The context object contains global state internal to CUDF. @@ -39,7 +41,7 @@ class context { std::once_flag _rtc_cache_init_flag; std::unique_ptr _rtc_cache; std::once_flag _jit_bundle_init_flag; - std::unique_ptr _jit_bundle; + std::unique_ptr _jit_bundle; private: void ensure_nvcomp_loaded(); @@ -51,7 +53,7 @@ class context { void ensure_jit_bundle_initialized(); public: - context(context_config const& cfg = {}, init_flags flags = init_flags::INIT_JIT_CACHE); + context(context_config cfg = {}, init_flags flags = init_flags::DEFAULT); context(context const&) = delete; context& operator=(context const&) = delete; context(context&&) = delete; @@ -62,7 +64,7 @@ class context { rtc::cache_t& rtc_cache(); - rtc::jit_bundle& jit_bundle(); + rtc::jit_bundle_t& jit_bundle(); [[nodiscard]] bool dump_codegen() const; From b2be86b08de79cca1e252cf79a030e07f8b13a7c Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 9 Feb 2026 05:27:00 +0000 Subject: [PATCH 037/254] feat: add get_bool_env_or specialization for boolean environment variable handling --- cpp/src/io/utilities/getenv_or.hpp | 10 ++++++++++ cpp/src/runtime/context.cpp | 8 ++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cpp/src/io/utilities/getenv_or.hpp b/cpp/src/io/utilities/getenv_or.hpp index f6137cbd968b..fc9f5f8bc328 100644 --- a/cpp/src/io/utilities/getenv_or.hpp +++ b/cpp/src/io/utilities/getenv_or.hpp @@ -42,4 +42,14 @@ T getenv_or(std::string_view env_var_name, T default_val) return converted_val; } +/** + * @brief Specialization of getenv_or for bool, to allow common "ON"/"OFF" string values. + */ +inline bool get_bool_env_or(std::string_view env_var_name, bool default_val) +{ + auto val = getenv_or(env_var_name, default_val ? std::string{"ON"} : std::string{"OFF"}); + return val == "ON" || val == "on" || val == "1" || val == "true" || val == "TRUE" || + val == "True"; +} + } // namespace diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 4a7fda8e0dae..561840b9a587 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -58,12 +58,8 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { std::call_once(*_context_init_flag, [&]() { - auto dump_codegen_env = getenv_or("LIBCUDF_JIT_DUMP_CODEGEN", std::string{"OFF"}); - bool dump_codegen = - (dump_codegen_env == "ON" || dump_codegen_env == "on" || dump_codegen_env == "1"); - - auto use_jit_env = getenv_or("LIBCUDF_JIT_ENABLED", std::string{"OFF"}); - bool use_jit = (use_jit_env == "ON" || use_jit_env == "on" || use_jit_env == "1"); + bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); + bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); From 925171d89472dcff037dae77b150957015b26c59 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 9 Feb 2026 08:16:20 +0000 Subject: [PATCH 038/254] feat: improve cache management and blob handling in RTC --- cpp/src/jit/rtc/cache.cpp | 90 +++++++++++++++------------- cpp/src/jit/rtc/cache.hpp | 37 +++++++----- cpp/src/jit/rtc/cudf.cpp | 122 +++++++++++++++++--------------------- cpp/src/jit/rtc/rtc.cpp | 2 +- cpp/src/jit/rtc/rtc.hpp | 2 +- 5 files changed, 129 insertions(+), 124 deletions(-) diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index 4cb0b8b6d7da..3a6e6d373d22 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -46,10 +46,6 @@ cache_t::cache_t(std::string cache_dir, cache_limits const& limits) CUDF_EXPECTS(limits.num_blobs >= 2, "Blob cache limit must be at least 2"); CUDF_EXPECTS(limits.num_fragments >= 2, "Fragment cache limit must be at least 2"); CUDF_EXPECTS(limits.num_libraries >= 2, "Library cache limit must be at least 2"); - // Create cache directory if it doesn't exist - if (mkdir(cache_dir_.c_str(), S_IRWXU | S_IRWXG | S_IRWXO) == -1) { - if (errno != EEXIST) { throw_posix("Failed to create RTC cache directory", "mkdir"); } - } } std::string const& cache_t::get_cache_dir() { return cache_dir_; } @@ -88,12 +84,15 @@ std::optional blob_t::from_file(char const* path) namespace { +/// @brief retrieves a blob from disk based on the given sha256 hash and object type (e.g. "blob", +/// "fragment", "library"). Returns nullopt if the file doesn't exist on disk, and throws if any +/// other error occurs. std::optional get_disk_blob(std::string const& cache_dir, std::string const& object_type, sha256_hash const& sha) { auto hex = sha.to_hex_string(); - auto path = std::format("{}/{}.{}.bin", cache_dir, object_type, hex.view()); + auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); auto blob = blob_t::from_file(path.c_str()); @@ -103,12 +102,14 @@ std::optional get_disk_blob(std::string const& cache_dir, } } +/// @brief atomically writes a blob to disk by first writing to a temporary file and then renaming +/// it to the final path. void add_blob_to_disk(std::string const& cache_dir, std::string const& object_type, sha256_hash const& sha, blob_view binary) { - char temp_path[] = "/tmp/cudf-blob-XXXXXX"; + char temp_path[] = "/tmp/cudf-bin-XXXXXX"; { int fd = mkstemp(temp_path); @@ -124,7 +125,7 @@ void add_blob_to_disk(std::string const& cache_dir, } auto hex = sha.to_hex_string(); - auto final_path = std::format("{}/{}.{}.bin", cache_dir, object_type, hex.view()); + auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); @@ -163,13 +164,13 @@ std::shared_future cache_t::query_or_insert_blob(sha256_hash const& sha, if (!unlocked) { lock_.unlock(); } }); - auto const it = blobs_cache_.entries_.find(sha); // check memory cache - if (it != blobs_cache_.entries_.end()) { + if (auto it = blobs_cache_.entries_.find(sha); it != blobs_cache_.entries_.end()) { counter_.hit_memory_blob(); // update LRU tick it->second.hit(current_tick); + return it->second.value; } else { @@ -178,23 +179,25 @@ std::shared_future cache_t::query_or_insert_blob(sha256_hash const& sha, // check disk cache auto disk_blob = get_disk_blob(cache_dir_, "blob", sha); + std::promise promise; + auto fut = promise.get_future().share(); + auto cache_fut = fut; + auto ret_fut = fut; + if (disk_blob.has_value()) { counter_.hit_disk_blob(); - std::promise promise; - promise.set_value(std::move(*disk_blob)); - auto fut = promise.get_future(); + promise.set_value(std::move(*disk_blob)); // insert into cache - blobs_cache_.insert(sha, fut.share(), current_tick); - return fut.share(); + blobs_cache_.insert(sha, std::move(cache_fut), current_tick); + + return ret_fut; } else { counter_.miss_disk_blob(); - std::promise promise; - auto fut = promise.get_future(); - blobs_cache_.insert(sha, fut.share(), current_tick); + blobs_cache_.insert(sha, std::move(cache_fut), current_tick); // we can release the lock while calling the maker function since it may be expensive and we // have already reserved a spot in the cache for this sha @@ -207,7 +210,7 @@ std::shared_future cache_t::query_or_insert_blob(sha256_hash const& sha, // store result to disk add_blob_to_disk(cache_dir_, "blob", sha, result->view()); - return fut.share(); + return ret_fut; } } } @@ -228,24 +231,31 @@ std::shared_future cache_t::query_or_insert_fragment(sha256_hash const if (!unlocked) { lock_.unlock(); } }); - auto const it = fragments_cache_.entries_.find(sha); // check memory cache - if (it != fragments_cache_.entries_.end()) { + if (auto it = fragments_cache_.entries_.find(sha); it != fragments_cache_.entries_.end()) { counter_.hit_memory_fragment(); + // update LRU tick + it->second.hit(current_tick); + return it->second.value; + } else { counter_.miss_memory_fragment(); // check disk cache auto disk_blob = get_disk_blob(cache_dir_, "fragment", sha); + + std::promise promise; + auto fut = promise.get_future().share(); + auto cache_fut = fut; + auto ret_fut = fut; + if (disk_blob.has_value()) { counter_.hit_disk_fragment(); - std::promise promise; - auto fut = promise.get_future(); - fragments_cache_.insert(sha, fut.share(), current_tick); + fragments_cache_.insert(sha, std::move(cache_fut), current_tick); // we can release the lock while calling the maker function since it may be expensive and we // have already reserved a spot in the cache for this sha @@ -257,15 +267,12 @@ std::shared_future cache_t::query_or_insert_fragment(sha256_hash const auto frag = fragment_t::load(load_params); promise.set_value(std::move(frag)); - return fut.share(); + return ret_fut; } else { counter_.miss_disk_fragment(); - std::promise promise; - - auto fut = promise.get_future(); - fragments_cache_.insert(sha, fut.share(), current_tick); + fragments_cache_.insert(sha, std::move(cache_fut), current_tick); // we can release the lock while calling the maker function since it may be expensive and we // have already reserved a spot in the cache for this sha @@ -278,7 +285,7 @@ std::shared_future cache_t::query_or_insert_fragment(sha256_hash const // store result to disk add_blob_to_disk(cache_dir_, "fragment", sha, result->get(type)->view()); - return fut.share(); + return ret_fut; } } } @@ -298,24 +305,30 @@ std::shared_future cache_t::query_or_insert_library( if (!unlocked) { lock_.unlock(); } }); - auto const it = libraries_cache_.entries_.find(sha); // check memory cache - if (it != libraries_cache_.entries_.end()) { + if (auto it = libraries_cache_.entries_.find(sha); it != libraries_cache_.entries_.end()) { counter_.hit_memory_library(); + // update LRU tick it->second.hit(current_tick); + return it->second.value; + } else { counter_.miss_memory_library(); // check disk cache auto disk_blob = get_disk_blob(cache_dir_, "library", sha); + + std::promise promise; + auto fut = promise.get_future().share(); + auto cache_fut = fut; + auto ret_fut = fut; + if (disk_blob.has_value()) { counter_.hit_disk_library(); - std::promise promise; - auto fut = promise.get_future(); - libraries_cache_.insert(sha, fut.share(), current_tick); + libraries_cache_.insert(sha, std::move(cache_fut), current_tick); // we can release the lock while calling the maker function since it may be expensive and we // have already reserved a spot in the cache for this sha @@ -327,15 +340,12 @@ std::shared_future cache_t::query_or_insert_library( auto lib = library_t::load(load_params); promise.set_value(std::move(lib)); - return fut.share(); + return ret_fut; } else { counter_.miss_disk_library(); - std::promise promise; - - auto fut = promise.get_future(); - libraries_cache_.insert(sha, fut.share(), current_tick); + libraries_cache_.insert(sha, std::move(cache_fut), current_tick); // we can release the lock while calling the maker function since it may be expensive and we // have already reserved a spot in the cache for this sha @@ -348,7 +358,7 @@ std::shared_future cache_t::query_or_insert_library( // store result to disk add_blob_to_disk(cache_dir_, "library", sha, blob->view()); - return fut.share(); + return ret_fut; } } } diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index 0b2a0623aa15..28dd796fe748 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -61,31 +61,38 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { explicit lru_memory_cache(size_t limit) : entries_{}, limit_{limit} { // reserve space to avoid rehashing + CUDF_EXPECTS(limit > 0, "Cache limit must be greater than 0"); entries_.reserve(limit * 2); } - void insert(sha256_hash const& sha, T&& value, uint64_t tick) + void purge() { - if ((entries_.size() + 1) > limit_) { - std::vector> rankings; - rankings.reserve(entries_.size()); + if (entries_.empty()) { return; } + + auto num_to_purge = std::max(entries_.size() / 2, static_cast(1)); - for (auto const& [key, ent] : entries_) { - rankings.emplace_back(key, ent.last_touched_tick); - } + std::vector> rankings; + rankings.reserve(entries_.size()); - std::sort(rankings.begin(), rankings.end(), [](auto const& a, auto const& b) { - return a.second < b.second; - }); + for (auto const& [key, entry] : entries_) { + rankings.emplace_back(key, entry.last_touched_tick); + } - // purge least recently used half + std::sort(rankings.begin(), rankings.end(), [](auto const& a, auto const& b) { + return a.second < b.second; + }); - auto num_to_purge = rankings.size() / 2; + // purge least recently used half + rankings.resize(num_to_purge); - for (size_t i = 0; i < num_to_purge; ++i) { - entries_.erase(rankings[i].first); - } + for (auto [key, _] : rankings) { + entries_.erase(key); } + } + + void insert(sha256_hash const& sha, T&& value, uint64_t tick) + { + if ((entries_.size() + 1) > limit_) { purge(); } entries_.emplace(sha, entry{tick, std::move(value)}); } diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 72fb08c9a88f..bee4d6e7ae3f 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -28,10 +28,6 @@ #include #include #include -#include - -// TODO: remove -#define CHECKPOINT std::cout << "Checkpoint: " << __FILE__ << ":" << __LINE__ << std::endl #define CUDFRTC_CHECK_CUDART(msg, ...) \ do { \ @@ -137,17 +133,17 @@ void install_file_set(char const* target_dir, +std::format("Unsupported compression type specified: {}", compression), std::runtime_error); - auto decompressed = decompress_blob(compressed_binary, uncompressed_size, compression); - auto const files_data = decompressed.data(); + auto decompressed = decompress_blob(compressed_binary, uncompressed_size, compression); + auto files_data = decompressed.data(); for (size_t i = 0; i < file_ranges.size(); ++i) { - auto const file_data_range = file_ranges[i]; - auto const dst_range = dst.ranges[i]; - auto const file_data = std::span{files_data + file_data_range.offset, file_data_range.size}; - auto const dst_path = std::string_view{ + auto file_data_range = file_ranges[i]; + auto dst_range = dst.ranges[i]; + auto file_data = std::span{files_data + file_data_range.offset, file_data_range.size}; + auto dst_path = std::string_view{ reinterpret_cast(dst.bytes.data) + dst_range.offset, dst_range.size}; - auto const target_path = std::format("{}/{}", target_dir, dst_path); + auto target_path = std::format("{}/{}", target_dir, dst_path); std::filesystem::create_directories(std::filesystem::path{target_path}.parent_path()); install_file(target_path.c_str(), file_data); @@ -233,20 +229,17 @@ void jit_bundle_t::preload_lto_library() auto bundle_hash = get_hash(); - auto const cache_key = std::format(R"***( - fragment_type=FATBIN, - key={}, - bundle={} - )***", - "cudf_lto_library", - bundle_hash); + auto cache_key = std::format(R"***(fragment_type=FATBIN +key={} +bundle={})***", + "cudf_lto_library", + bundle_hash); - auto const cache_key_sha256 = hash_string(cache_key); + auto cache_key_sha256 = hash_string(cache_key); auto compile = [&] { - auto directory = get_directory(); - auto path = std::format("{}/{}", directory, "cudf_lto_library.fatbin"); - auto cubin = blob_t::from_file(path.c_str()); + auto path = std::format("{}/{}", get_directory(), "cudf_lto_library.fatbin"); + auto cubin = blob_t::from_file(path.c_str()); CUDF_EXPECTS(cubin.has_value(), +std::format("Failed to load LTO library cubin from disk at ({})", path), std::runtime_error); @@ -269,8 +262,7 @@ std::string jit_bundle_t::get_hash() const std::string jit_bundle_t::get_directory() const { - auto hash = get_hash(); - return std::format("{}/{}", install_dir_, hash); + return std::format("{}/{}", install_dir_, get_hash()); } fragment jit_bundle_t::get_lto_library() const { return lto_library_; } @@ -351,28 +343,24 @@ fragment get_or_compile_fragment(char const* name, char const* source_code_cstr, auto sm = get_current_device_physical_model(); auto bundle_hash = bundle.get_hash(); - auto const cache_key = std::format(R"***( - fragment_type=LTO_IR, - key={}, - cuda_runtime={}, - cuda_driver={}, - arch={}, - bundle={} - )***", - key, - runtime, - driver, - sm, - bundle_hash); - - auto const cache_key_sha256 = hash_string(cache_key); + auto cache_key = std::format(R"***(fragment_type=LTO_IR +key={} +cuda_runtime={} +cuda_driver={} +arch={} +bundle={})***", + key, + runtime, + driver, + sm, + bundle_hash); + + auto cache_key_sha256 = hash_string(cache_key); // TODO: add time function in cache auto compile = [&] { - auto begin = std::chrono::high_resolution_clock::now(); - auto cache_dir = cache.get_cache_dir(); - auto install_dir = std::format("{}/jit-install", cache_dir); + auto begin = std::chrono::high_resolution_clock::now(); auto include_dirs = bundle.get_include_directories(); auto compile_options = bundle.get_compile_options(); @@ -407,11 +395,11 @@ fragment get_or_compile_fragment(char const* name, char const* source_code_cstr, options_cstr.emplace_back(option.c_str()); } - auto const params = fragment_t::compile_params{.name = name, - .source = source_code_cstr, - .headers = {}, - .options = options_cstr, - .target_type = binary_type::LTO_IR}; + auto params = fragment_t::compile_params{.name = name, + .source = source_code_cstr, + .headers = {}, + .options = options_cstr, + .target_type = binary_type::LTO_IR}; auto frag = fragment_t::compile(params); @@ -452,14 +440,15 @@ library compile_and_link_udf(char const* name, auto library = bundle.get_lto_library(); auto fragment = get_or_compile_fragment(name, udf_code, udf_key); + // TODO: sass dump + // TODO: time dump + // TODO: experiment with: // optimization flags // split compile // split-compile-extended // lineinfo and debug info options // -kernels-used= - // sass dump - // time dump // env variable to control options // fma // variables-used @@ -478,19 +467,19 @@ library compile_and_link_udf(char const* name, options_cstr.emplace_back(option.c_str()); } - blob_view const link_fragments[] = {library->get(binary_type::FATBIN)->view(), - fragment->get(binary_type::LTO_IR)->view()}; + blob_view link_fragments[] = {library->get(binary_type::FATBIN)->view(), + fragment->get(binary_type::LTO_IR)->view()}; - binary_type const fragment_binary_types[] = {binary_type::FATBIN, binary_type::LTO_IR}; + binary_type fragment_binary_types[] = {binary_type::FATBIN, binary_type::LTO_IR}; - char const* const fragment_names[] = {"cudf_lto_library", "cudf_udf_fragment"}; + char const* fragment_names[] = {"cudf_lto_library", name}; - auto const params = library_t::link_params{.name = name, - .output_type = binary_type::CUBIN, - .fragments = link_fragments, - .fragment_binary_types = fragment_binary_types, - .fragment_names = fragment_names, - .link_options = options_cstr}; + auto params = library_t::link_params{.name = name, + .output_type = binary_type::CUBIN, + .fragments = link_fragments, + .fragment_binary_types = fragment_binary_types, + .fragment_names = fragment_names, + .link_options = options_cstr}; auto blob = library_t::link_as_blob(params); @@ -510,19 +499,18 @@ library compile_and_link_udf(char const* name, return std::make_tuple(linked_library, blob); }; - auto library_cache_key = std::format(R"***( - library_type=CUBIN, - kernels={}, - udf={}, - cuda_runtime={}, - cuda_driver={}, - arch={})***", + auto library_cache_key = std::format(R"***(library_type=CUBIN +kernels={} +udf={} +cuda_runtime={} +cuda_driver={} +arch={})***", kernel_symbol, udf_key, runtime, driver, sm); - auto const library_cache_key_sha256 = hash_string(library_cache_key); + auto library_cache_key_sha256 = hash_string(library_cache_key); auto library = cache.query_or_insert_library(library_cache_key_sha256, binary_type::CUBIN, compile); diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 5f0e56caa845..10a1956ff570 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -374,7 +374,7 @@ void kernel_ref::launch(uint32_t grid_dim_x, uint32_t block_dim_z, uint32_t shared_mem_bytes, CUstream stream, - void** kernel_params) + void** kernel_params) const { CUDF_FUNC_RANGE(); diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 8c020f3681df..ae6000b83a32 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -160,7 +160,7 @@ struct [[nodiscard]] kernel_ref { uint32_t block_dim_z, uint32_t shared_mem_bytes, CUstream stream, - void** kernel_params); + void** kernel_params) const; [[nodiscard]] std::string_view get_name() const; }; From 8fd14de6126df695987264c57cb883559e48f381 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 9 Feb 2026 08:21:15 +0000 Subject: [PATCH 039/254] fix: replace high_resolution_clock with steady_clock for more stable timing in JIT compilation --- cpp/src/jit/rtc/cudf.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index bee4d6e7ae3f..0f1efa77b332 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -360,7 +360,7 @@ bundle={})***", // TODO: add time function in cache auto compile = [&] { - auto begin = std::chrono::high_resolution_clock::now(); + auto begin = std::chrono::steady_clock::now(); auto include_dirs = bundle.get_include_directories(); auto compile_options = bundle.get_compile_options(); @@ -403,7 +403,7 @@ bundle={})***", auto frag = fragment_t::compile(params); - auto end = std::chrono::high_resolution_clock::now(); + auto end = std::chrono::steady_clock::now(); auto duration = end - begin; @@ -436,7 +436,7 @@ library compile_and_link_udf(char const* name, auto bundle_hash = bundle.get_hash(); auto compile = [&] { - auto begin = std::chrono::high_resolution_clock::now(); + auto begin = std::chrono::steady_clock::now(); auto library = bundle.get_lto_library(); auto fragment = get_or_compile_fragment(name, udf_code, udf_key); @@ -487,7 +487,7 @@ library compile_and_link_udf(char const* name, auto linked_library = library_t::load(load_params); - auto end = std::chrono::high_resolution_clock::now(); + auto end = std::chrono::steady_clock::now(); auto duration = end - begin; From c9aedc923300faee1763d1d75f64897f70d982ff Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 9 Feb 2026 10:30:14 +0000 Subject: [PATCH 040/254] formatting --- cpp/include/cudf/jit/filter_params.cuh | 22 ----------- cpp/include/cudf/jit/lto/filter_params.cuh | 17 +++++++++ .../cudf/jit/{ => lto}/transform_params.cuh | 9 ++--- cpp/src/jit/rtc/cache.cpp | 15 ++++---- cpp/src/jit/rtc/cache.hpp | 13 +++++-- cpp/src/jit/rtc/cudf.cpp | 13 ++++--- cpp/src/jit/rtc/includes/system/algorithm | 6 +-- cpp/src/jit/rtc/includes/system/array | 6 +-- cpp/src/jit/rtc/includes/system/atomic | 6 +-- cpp/src/jit/rtc/includes/system/barrier | 6 +-- cpp/src/jit/rtc/includes/system/bit | 6 +-- cpp/src/jit/rtc/includes/system/cassert | 6 +-- cpp/src/jit/rtc/includes/system/ccomplex | 6 +-- cpp/src/jit/rtc/includes/system/cfloat | 5 +-- cpp/src/jit/rtc/includes/system/chrono | 6 +-- cpp/src/jit/rtc/includes/system/climits | 5 +-- cpp/src/jit/rtc/includes/system/cmath | 6 +-- cpp/src/jit/rtc/includes/system/complex | 6 +-- cpp/src/jit/rtc/includes/system/concepts | 6 +-- cpp/src/jit/rtc/includes/system/cstddef | 6 +-- cpp/src/jit/rtc/includes/system/cstdint | 6 +-- cpp/src/jit/rtc/includes/system/cstdlib | 6 +-- cpp/src/jit/rtc/includes/system/cstring | 6 +-- cpp/src/jit/rtc/includes/system/ctime | 6 +-- cpp/src/jit/rtc/includes/system/expected | 6 +-- cpp/src/jit/rtc/includes/system/functional | 6 +-- .../jit/rtc/includes/system/initializer_list | 6 +-- .../jit/rtc/includes/system/inplace_vector | 6 +-- cpp/src/jit/rtc/includes/system/iterator | 6 +-- cpp/src/jit/rtc/includes/system/latch | 6 +-- cpp/src/jit/rtc/includes/system/limits | 6 +-- cpp/src/jit/rtc/includes/system/linalg | 6 +-- cpp/src/jit/rtc/includes/system/mdspan | 6 +-- cpp/src/jit/rtc/includes/system/memory | 6 +-- cpp/src/jit/rtc/includes/system/numbers | 6 +-- cpp/src/jit/rtc/includes/system/numeric | 6 +-- cpp/src/jit/rtc/includes/system/optional | 6 +-- cpp/src/jit/rtc/includes/system/ranges | 6 +-- cpp/src/jit/rtc/includes/system/ratio | 6 +-- cpp/src/jit/rtc/includes/system/semaphore | 6 +-- .../jit/rtc/includes/system/source_location | 6 +-- cpp/src/jit/rtc/includes/system/span | 6 +-- cpp/src/jit/rtc/includes/system/string_view | 6 +-- cpp/src/jit/rtc/includes/system/tuple | 6 +-- cpp/src/jit/rtc/includes/system/type_traits | 6 +-- cpp/src/jit/rtc/includes/system/utility | 6 +-- cpp/src/jit/rtc/includes/system/variant | 6 +-- cpp/src/jit/rtc/includes/system/version | 2 +- cpp/src/jit/rtc/rtc.cpp | 10 +++++ cpp/src/jit/rtc/rtc.hpp | 37 +++++++++++++++++++ cpp/src/transform/jit/lto_kernel.inl.cuh | 23 +++++------- 51 files changed, 224 insertions(+), 181 deletions(-) delete mode 100644 cpp/include/cudf/jit/filter_params.cuh create mode 100644 cpp/include/cudf/jit/lto/filter_params.cuh rename cpp/include/cudf/jit/{ => lto}/transform_params.cuh (55%) diff --git a/cpp/include/cudf/jit/filter_params.cuh b/cpp/include/cudf/jit/filter_params.cuh deleted file mode 100644 index 3de139a70d4d..000000000000 --- a/cpp/include/cudf/jit/filter_params.cuh +++ /dev/null @@ -1,22 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include - -namespace CUDF_LTO_EXPORT cudf { - -namespace lto { - -/// @brief Type-erased parameters for LTO-JIT-compiled filter operations. -struct filter_params { - void const* inputs = nullptr; ///< Pointer to inputs data. - void* user_data = nullptr; ///< Pointer to user data / context. - void const* outputs = nullptr; ///< Pointer to outputs data. - size_type row_index = 0; ///< Current row index. -}; - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/filter_params.cuh b/cpp/include/cudf/jit/lto/filter_params.cuh new file mode 100644 index 000000000000..a733314c4f11 --- /dev/null +++ b/cpp/include/cudf/jit/lto/filter_params.cuh @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include + +namespace CUDF_LTO_EXPORT cudf { + +namespace lto { + +using filter_params = transform_params; + +} // namespace lto +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/transform_params.cuh b/cpp/include/cudf/jit/lto/transform_params.cuh similarity index 55% rename from cpp/include/cudf/jit/transform_params.cuh rename to cpp/include/cudf/jit/lto/transform_params.cuh index a24c00b13103..c81a2cee7c67 100644 --- a/cpp/include/cudf/jit/transform_params.cuh +++ b/cpp/include/cudf/jit/lto/transform_params.cuh @@ -12,11 +12,10 @@ namespace CUDF_LTO_EXPORT cudf { namespace lto { /// @brief Type-erased parameters for LTO-JIT-compiled transform operations. -struct transform_params { - void const* inputs = nullptr; ///< Pointer to inputs data. - void* user_data = nullptr; ///< Pointer to user data / context. - void const* outputs = nullptr; ///< Pointer to outputs data. - size_type row_index = 0; ///< Current row index. +struct [[nodiscard]] transform_params { + void* __restrict__ const* __restrict__ scope = + nullptr; ///< Pointer to scope data (e.g. column views, scalars, etc.). + size_type row_index = 0; ///< Current row index. }; } // namespace lto diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index 3a6e6d373d22..462f3357d9da 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -150,7 +150,7 @@ void add_blob_to_disk(std::string const& cache_dir, } // namespace std::shared_future cache_t::query_or_insert_blob(sha256_hash const& sha, - std::function maker) + blob_compile_function_t compile) { CUDF_FUNC_RANGE(); @@ -204,7 +204,7 @@ std::shared_future cache_t::query_or_insert_blob(sha256_hash const& sha, lock_.unlock(); unlocked = true; - auto result = maker(); + auto result = compile(); promise.set_value(result); // store result to disk @@ -217,7 +217,7 @@ std::shared_future cache_t::query_or_insert_blob(sha256_hash const& sha, std::shared_future cache_t::query_or_insert_fragment(sha256_hash const& sha, binary_type type, - std::function maker) + fragment_compile_function_t compile) { CUDF_FUNC_RANGE(); @@ -279,7 +279,7 @@ std::shared_future cache_t::query_or_insert_fragment(sha256_hash const lock_.unlock(); unlocked = true; - auto result = maker(); + auto result = compile(); promise.set_value(result); // store result to disk @@ -290,8 +290,9 @@ std::shared_future cache_t::query_or_insert_fragment(sha256_hash const } } -std::shared_future cache_t::query_or_insert_library( - sha256_hash const& sha, binary_type type, std::function()> maker) +std::shared_future cache_t::query_or_insert_library(sha256_hash const& sha, + binary_type type, + library_compile_function_t compile) { CUDF_FUNC_RANGE(); @@ -352,7 +353,7 @@ std::shared_future cache_t::query_or_insert_library( lock_.unlock(); unlocked = true; - auto [library, blob] = maker(); + auto [library, blob] = compile(); promise.set_value(library); // store result to disk diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index 28dd796fe748..276880ae0eac 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -193,6 +193,10 @@ struct cache_statistics_counter { } // namespace detail +using blob_compile_function_t = function_ref; +using fragment_compile_function_t = function_ref; +using library_compile_function_t = function_ref()>; + /// @brief Thread-safe compile cache for compiled blobs, fragments, and libraries /// @details Provides in-memory and on-disk caching of compiled RTC artifacts. /// The cache uses an LRU eviction policy when the number of cached items @@ -234,14 +238,15 @@ struct cache_t { [[nodiscard]] std::string const& get_cache_dir(); std::shared_future query_or_insert_blob(sha256_hash const& sha, - std::function maker); + blob_compile_function_t compile); std::shared_future query_or_insert_fragment(sha256_hash const& sha, binary_type type, - std::function maker); + fragment_compile_function_t compile); - std::shared_future query_or_insert_library( - sha256_hash const& sha, binary_type type, std::function()> maker); + std::shared_future query_or_insert_library(sha256_hash const& sha, + binary_type type, + library_compile_function_t compile); cache_statistics get_statistics(); diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 0f1efa77b332..9ebabc8f491b 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -248,7 +248,8 @@ bundle={})***", return fragment_t::load(load_params); }; - auto fut = cache.query_or_insert_fragment(cache_key_sha256, binary_type::FATBIN, compile); + auto fut = cache.query_or_insert_fragment( + cache_key_sha256, binary_type::FATBIN, fragment_compile_function_t::from_functor(compile)); lto_library_ = fut.get(); } @@ -415,7 +416,8 @@ bundle={})***", return frag; }; - auto fut = cache.query_or_insert_fragment(cache_key_sha256, binary_type::LTO_IR, compile); + auto fut = cache.query_or_insert_fragment( + cache_key_sha256, binary_type::LTO_IR, fragment_compile_function_t::from_functor(compile)); return fut.get(); } @@ -445,7 +447,7 @@ library compile_and_link_udf(char const* name, // TODO: experiment with: // optimization flags - // split compile + // split-compile // split-compile-extended // lineinfo and debug info options // -kernels-used= @@ -512,8 +514,9 @@ arch={})***", sm); auto library_cache_key_sha256 = hash_string(library_cache_key); - auto library = - cache.query_or_insert_library(library_cache_key_sha256, binary_type::CUBIN, compile); + auto library = cache.query_or_insert_library(library_cache_key_sha256, + binary_type::CUBIN, + library_compile_function_t::from_functor(compile)); return library.get(); } diff --git a/cpp/src/jit/rtc/includes/system/algorithm b/cpp/src/jit/rtc/includes/system/algorithm index 6f7d2aefd31f..8cf34e5afc0a 100644 --- a/cpp/src/jit/rtc/includes/system/algorithm +++ b/cpp/src/jit/rtc/includes/system/algorithm @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/array b/cpp/src/jit/rtc/includes/system/array index 3b264123dacd..fe2d963aba07 100644 --- a/cpp/src/jit/rtc/includes/system/array +++ b/cpp/src/jit/rtc/includes/system/array @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/atomic b/cpp/src/jit/rtc/includes/system/atomic index 5c2dc1d81400..9f32c5729abb 100644 --- a/cpp/src/jit/rtc/includes/system/atomic +++ b/cpp/src/jit/rtc/includes/system/atomic @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/barrier b/cpp/src/jit/rtc/includes/system/barrier index 95cd35657daa..54dca7534e0f 100644 --- a/cpp/src/jit/rtc/includes/system/barrier +++ b/cpp/src/jit/rtc/includes/system/barrier @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/bit b/cpp/src/jit/rtc/includes/system/bit index 493dc2115366..e6aeda22d2b1 100644 --- a/cpp/src/jit/rtc/includes/system/bit +++ b/cpp/src/jit/rtc/includes/system/bit @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/cassert b/cpp/src/jit/rtc/includes/system/cassert index 1e2b85df5dc4..c7d847aafdc4 100644 --- a/cpp/src/jit/rtc/includes/system/cassert +++ b/cpp/src/jit/rtc/includes/system/cassert @@ -1,12 +1,10 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ - - +namespace std { } diff --git a/cpp/src/jit/rtc/includes/system/ccomplex b/cpp/src/jit/rtc/includes/system/ccomplex index dbb5e678e49f..a2eded0c8289 100644 --- a/cpp/src/jit/rtc/includes/system/ccomplex +++ b/cpp/src/jit/rtc/includes/system/ccomplex @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/cfloat b/cpp/src/jit/rtc/includes/system/cfloat index ac41c2724b44..0c23d7ca72f4 100644 --- a/cpp/src/jit/rtc/includes/system/cfloat +++ b/cpp/src/jit/rtc/includes/system/cfloat @@ -1,11 +1,10 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ - +namespace std { } diff --git a/cpp/src/jit/rtc/includes/system/chrono b/cpp/src/jit/rtc/includes/system/chrono index 63b2e7e4a183..4625526096eb 100644 --- a/cpp/src/jit/rtc/includes/system/chrono +++ b/cpp/src/jit/rtc/includes/system/chrono @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/climits b/cpp/src/jit/rtc/includes/system/climits index a555fb339dfa..e7ddbe78a70e 100644 --- a/cpp/src/jit/rtc/includes/system/climits +++ b/cpp/src/jit/rtc/includes/system/climits @@ -1,11 +1,10 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ - +namespace std { } diff --git a/cpp/src/jit/rtc/includes/system/cmath b/cpp/src/jit/rtc/includes/system/cmath index 8eff37d3a688..97ed927ef2a9 100644 --- a/cpp/src/jit/rtc/includes/system/cmath +++ b/cpp/src/jit/rtc/includes/system/cmath @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/complex b/cpp/src/jit/rtc/includes/system/complex index 8b3855b16825..530e7601adf6 100644 --- a/cpp/src/jit/rtc/includes/system/complex +++ b/cpp/src/jit/rtc/includes/system/complex @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/concepts b/cpp/src/jit/rtc/includes/system/concepts index fadf29d68552..41357406d9ef 100644 --- a/cpp/src/jit/rtc/includes/system/concepts +++ b/cpp/src/jit/rtc/includes/system/concepts @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/cstddef b/cpp/src/jit/rtc/includes/system/cstddef index 7e234744ace1..71352d667749 100644 --- a/cpp/src/jit/rtc/includes/system/cstddef +++ b/cpp/src/jit/rtc/includes/system/cstddef @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/cstdint b/cpp/src/jit/rtc/includes/system/cstdint index f5bdc5e318e9..03c8eb0a9dd1 100644 --- a/cpp/src/jit/rtc/includes/system/cstdint +++ b/cpp/src/jit/rtc/includes/system/cstdint @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/cstdlib b/cpp/src/jit/rtc/includes/system/cstdlib index d4c5a263926b..c216518c9c29 100644 --- a/cpp/src/jit/rtc/includes/system/cstdlib +++ b/cpp/src/jit/rtc/includes/system/cstdlib @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/cstring b/cpp/src/jit/rtc/includes/system/cstring index caec3d64ae06..83275541eb0a 100644 --- a/cpp/src/jit/rtc/includes/system/cstring +++ b/cpp/src/jit/rtc/includes/system/cstring @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/ctime b/cpp/src/jit/rtc/includes/system/ctime index eb245dd5de63..05e4eab7bd9f 100644 --- a/cpp/src/jit/rtc/includes/system/ctime +++ b/cpp/src/jit/rtc/includes/system/ctime @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/expected b/cpp/src/jit/rtc/includes/system/expected index 32553c615b9c..eea942704ceb 100644 --- a/cpp/src/jit/rtc/includes/system/expected +++ b/cpp/src/jit/rtc/includes/system/expected @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/functional b/cpp/src/jit/rtc/includes/system/functional index c2d1afd9da3a..b7b94f351aee 100644 --- a/cpp/src/jit/rtc/includes/system/functional +++ b/cpp/src/jit/rtc/includes/system/functional @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/initializer_list b/cpp/src/jit/rtc/includes/system/initializer_list index f69a6c36c12c..0ab9115d43c1 100644 --- a/cpp/src/jit/rtc/includes/system/initializer_list +++ b/cpp/src/jit/rtc/includes/system/initializer_list @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/inplace_vector b/cpp/src/jit/rtc/includes/system/inplace_vector index dc88a71ce59e..0d60464a347f 100644 --- a/cpp/src/jit/rtc/includes/system/inplace_vector +++ b/cpp/src/jit/rtc/includes/system/inplace_vector @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/iterator b/cpp/src/jit/rtc/includes/system/iterator index cdaa28d88d47..a5c45ae5f9d5 100644 --- a/cpp/src/jit/rtc/includes/system/iterator +++ b/cpp/src/jit/rtc/includes/system/iterator @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/latch b/cpp/src/jit/rtc/includes/system/latch index 906df15bf987..64f1e2365895 100644 --- a/cpp/src/jit/rtc/includes/system/latch +++ b/cpp/src/jit/rtc/includes/system/latch @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/limits b/cpp/src/jit/rtc/includes/system/limits index 58aad6743f75..95d9c27cf00d 100644 --- a/cpp/src/jit/rtc/includes/system/limits +++ b/cpp/src/jit/rtc/includes/system/limits @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/linalg b/cpp/src/jit/rtc/includes/system/linalg index 0db2c0558620..59e85675b30a 100644 --- a/cpp/src/jit/rtc/includes/system/linalg +++ b/cpp/src/jit/rtc/includes/system/linalg @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/mdspan b/cpp/src/jit/rtc/includes/system/mdspan index 6634dbe85bb6..94e48d35185a 100644 --- a/cpp/src/jit/rtc/includes/system/mdspan +++ b/cpp/src/jit/rtc/includes/system/mdspan @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/memory b/cpp/src/jit/rtc/includes/system/memory index bb8634824332..4706fe6f48d2 100644 --- a/cpp/src/jit/rtc/includes/system/memory +++ b/cpp/src/jit/rtc/includes/system/memory @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/numbers b/cpp/src/jit/rtc/includes/system/numbers index 90c49d6159f3..a3df0d809d29 100644 --- a/cpp/src/jit/rtc/includes/system/numbers +++ b/cpp/src/jit/rtc/includes/system/numbers @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/numeric b/cpp/src/jit/rtc/includes/system/numeric index 501c41b53141..bc8958976325 100644 --- a/cpp/src/jit/rtc/includes/system/numeric +++ b/cpp/src/jit/rtc/includes/system/numeric @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/optional b/cpp/src/jit/rtc/includes/system/optional index 221670774c11..8f7e5862a0ea 100644 --- a/cpp/src/jit/rtc/includes/system/optional +++ b/cpp/src/jit/rtc/includes/system/optional @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/ranges b/cpp/src/jit/rtc/includes/system/ranges index b2e1e4b88f60..e45db1965fe5 100644 --- a/cpp/src/jit/rtc/includes/system/ranges +++ b/cpp/src/jit/rtc/includes/system/ranges @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/ratio b/cpp/src/jit/rtc/includes/system/ratio index c7a79cf3be4b..f6ed2d1262ef 100644 --- a/cpp/src/jit/rtc/includes/system/ratio +++ b/cpp/src/jit/rtc/includes/system/ratio @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/semaphore b/cpp/src/jit/rtc/includes/system/semaphore index a27c6a001d80..60d7f0bca92b 100644 --- a/cpp/src/jit/rtc/includes/system/semaphore +++ b/cpp/src/jit/rtc/includes/system/semaphore @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/source_location b/cpp/src/jit/rtc/includes/system/source_location index d8d4ffdd51f3..0f9c293abf9e 100644 --- a/cpp/src/jit/rtc/includes/system/source_location +++ b/cpp/src/jit/rtc/includes/system/source_location @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/span b/cpp/src/jit/rtc/includes/system/span index 37a20bfe8422..f82a9a0e6e6d 100644 --- a/cpp/src/jit/rtc/includes/system/span +++ b/cpp/src/jit/rtc/includes/system/span @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/string_view b/cpp/src/jit/rtc/includes/system/string_view index 2571122e34d4..826afab8c607 100644 --- a/cpp/src/jit/rtc/includes/system/string_view +++ b/cpp/src/jit/rtc/includes/system/string_view @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/tuple b/cpp/src/jit/rtc/includes/system/tuple index 986ab80ee5a8..1b330a8db160 100644 --- a/cpp/src/jit/rtc/includes/system/tuple +++ b/cpp/src/jit/rtc/includes/system/tuple @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/type_traits b/cpp/src/jit/rtc/includes/system/type_traits index 63e4f60aebf1..2b2c1a21ba9b 100644 --- a/cpp/src/jit/rtc/includes/system/type_traits +++ b/cpp/src/jit/rtc/includes/system/type_traits @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/utility b/cpp/src/jit/rtc/includes/system/utility index db465bfdd181..78cf8d508129 100644 --- a/cpp/src/jit/rtc/includes/system/utility +++ b/cpp/src/jit/rtc/includes/system/utility @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/variant b/cpp/src/jit/rtc/includes/system/variant index cefd94b3e8a7..142842ce87ad 100644 --- a/cpp/src/jit/rtc/includes/system/variant +++ b/cpp/src/jit/rtc/includes/system/variant @@ -1,12 +1,12 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -namespace std{ +namespace std { - using namespace ::cuda::std; +using namespace ::cuda::std; } diff --git a/cpp/src/jit/rtc/includes/system/version b/cpp/src/jit/rtc/includes/system/version index 56a7a1d15cec..a24af52c843d 100644 --- a/cpp/src/jit/rtc/includes/system/version +++ b/cpp/src/jit/rtc/includes/system/version @@ -1,4 +1,4 @@ - /* +/* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 10a1956ff570..a287755c4b0f 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -378,6 +378,16 @@ void kernel_ref::launch(uint32_t grid_dim_x, { CUDF_FUNC_RANGE(); + CUDF_EXPECTS(grid_dim_x > 0 && grid_dim_y > 0 && grid_dim_z > 0, + "Grid dimensions must be greater than zero", + std::logic_error); + CUDF_EXPECTS(block_dim_x > 0 && block_dim_y > 0 && block_dim_z > 0, + "Block dimensions must be greater than zero", + std::logic_error); + CUDF_EXPECTS(stream != nullptr, "CUDA stream must not be null", std::logic_error); + CUDF_EXPECTS( + kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); + CUlaunchConfig cfg{.gridDimX = grid_dim_x, .gridDimY = grid_dim_y, .gridDimZ = grid_dim_z, diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index ae6000b83a32..7e56b8c9a2c0 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -91,6 +91,43 @@ struct [[nodiscard]] blob_t { using blob = std::shared_ptr; +template +struct function_ref; + +/// @brief Zero-copy, type-erased reference to a callable entity (e.g. lambda, function pointer) +/// that can be invoked with the given signature. +template +struct function_ref { + private: + void* _user_data; + R (*_thunk)(void*, Args...); + + public: + function_ref(void* user_data, R (*thunk)(void*, Args...)) : _user_data{user_data}, _thunk{thunk} + { + } + + R operator()(Args... args) const { return _thunk(_user_data, std::forward(args)...); } + + template + static function_ref from_functor(Lambda& func) + { + return function_ref{static_cast(std::addressof(func)), + +[](void* user_data, Args... args) -> R { + auto& lambda = *static_cast*>(user_data); + return lambda(std::forward(args)...); + }}; + } + + static function_ref from_function_pointer(R (*func)(Args...)) + { + return function_ref{func, +[](void* user_data, Args... args) -> R { + auto func = reinterpret_cast(user_data); + return func(std::forward(args)...); + }}; + } +}; + struct [[nodiscard]] header_map { std::span include_names = {}; // null-terminated header include names std::span headers = {}; // null-terminated header contents diff --git a/cpp/src/transform/jit/lto_kernel.inl.cuh b/cpp/src/transform/jit/lto_kernel.inl.cuh index da41bd3babe1..55da5a3fc3a1 100644 --- a/cpp/src/transform/jit/lto_kernel.inl.cuh +++ b/cpp/src/transform/jit/lto_kernel.inl.cuh @@ -9,23 +9,20 @@ #include extern "C" { -__device__ void transform_operator(cudf::lto::transform_params const* params); -__global__ void transform_kernel(void const* outputs, - void const* inputs, - void* user_data, - cudf::size_type num_rows) +__device__ void transform_operator(cudf::lto::transform_params params); + +__global__ void transform_kernel(void* __restrict__ const* __restrict__ scope, + int32_t num_rows) { - auto const start = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); - auto const size = num_rows; + auto start = cudf::detail::grid_1d::global_thread_id(); + auto stride = cudf::detail::grid_1d::grid_stride(); + auto size = num_rows; for (auto i = start; i < size; i += stride) { - cudf::lto::transform_params p{.inputs = inputs, - .user_data = user_data, - .outputs = outputs, - .row_index = static_cast(i)}; - transform_operator(&p); + cudf::lto::transform_params params{.scope = scope, + .row_index = static_cast(i)}; + transform_operator(params); } } } From bae7fde7caaf7233326b701580da013dbfa3a967 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 10 Feb 2026 15:50:49 +0000 Subject: [PATCH 041/254] feat: add new accessors and device span structures for improved LTO functionality --- cpp/include/cudf/jit/lto/accessors.cuh | 119 +++++++++ cpp/include/cudf/jit/lto/types.cuh | 321 +++++++++++++++---------- cpp/src/jit/lto/types.inl.cuh | 176 +++++--------- cpp/tests/jit/rtc.cpp | 42 ++-- 4 files changed, 402 insertions(+), 256 deletions(-) create mode 100644 cpp/include/cudf/jit/lto/accessors.cuh diff --git a/cpp/include/cudf/jit/lto/accessors.cuh b/cpp/include/cudf/jit/lto/accessors.cuh new file mode 100644 index 000000000000..302f13fc9730 --- /dev/null +++ b/cpp/include/cudf/jit/lto/accessors.cuh @@ -0,0 +1,119 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +// TODO: cuda std optional? +// TODO: how to convert to our optional? +// TODO: handle scalar +// TODO: scope variables should be aligned to avoid uncoalesced reads/writes +// TODO: how will expression evaluation work to handle multiple outputs? +// TODO: make fixed_point be able to assign values to columns +// TODO: is_null and is_valid + +namespace cudf { +namespace lto { + +using scope_type = void* const __restrict__* __restrict__; + +template +struct column { + static constexpr bool IS_SCALAR = IsScalar; + static constexpr bool IS_NULLABLE = IsNullable; + + using Type = T; + using Arg = ColumnType const* __restrict__; + + static __device__ auto get(scope_type scope, size_type i) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + if constexpr (!IsNullable) { + return p->template element(index); + } else { + return p->template nullable_element(index); + } + } + + static __device__ void assign(scope_type scope, size_type i, auto const& value) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + p->template assign(index, value); + } + + static __device__ auto* null_mask(scope_type scope) + { + auto p = static_cast(scope[ScopeIndex]); + return p->null_mask(); + } +}; + +template +struct span { + static constexpr bool IS_SCALAR = IsScalar; + static constexpr bool IS_NULLABLE = IsNullable; + + using Type = T; + using Arg = SpanType const* __restrict__; + + static __device__ auto get(scope_type scope, size_type i) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + if constexpr (!IsNullable) { + return p->template element(index); + } else { + return p->template nullable_element(index); + } + } + + static __device__ void assign(scope_type scope, size_type i, auto const& value) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + p->template assign(index, value); + } + + static __device__ auto* null_mask(scope_type scope) + { + auto p = static_cast(scope[ScopeIndex]); + return p->null_mask(); + } +}; + +} // namespace lto +} // namespace cudf + +template +struct element_operation { + template + static __device__ void evaluate(scope_type scope, cudf::size_type i, Operator&& op) + { + if constexpr (UserDataIndex >= 0) { + auto output_args; + GENERIC_TRANSFORM_OP(user_data, i, &res, In::element(inputs, i)...); + } else { + GENERIC_TRANSFORM_OP(&res, In::element(inputs, i)...); + } + } +}; diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index e71a765ca902..adc34412c57f 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -31,6 +31,7 @@ using uint16_t = unsigned short; using uint32_t = unsigned int; using uint64_t = unsigned long long; +using size_t = unsigned long; using intptr_t = int64_t; using uintptr_t = uint64_t; @@ -270,6 +271,153 @@ struct CUDF_LTO_ALIAS optional { template optional(T) -> optional; +template +struct [[nodiscard]] device_span { + private: + T* __data = nullptr; + size_t __size = 0; +}; + +template +struct [[nodiscard]] device_optional_span { + private: + T* __data = nullptr; + size_t __size = 0; + bitmask_type* __null_mask = nullptr; + + public: + __device__ T* data() const { return __data; } + + __device__ size_t size() const { return __size; } + + __device__ bool empty() const { return __size == 0; } + + __device__ T& operator[](size_t pos) const { return __data[pos]; } + + __device__ T* begin() const { return __data; } + + __device__ T* end() const { return __data + __size; } + + __device__ device_optional_span as_const() const + { + return device_optional_span{__data, __size, __null_mask}; + } + + __device__ bool nullable() const { return __null_mask != nullptr; } + + __device__ static constexpr bool bit_is_set(bitmask_type const* bitmask, size_t bit_index) + { + constexpr auto bits_per_word = sizeof(bitmask_type) * 8; + return bitmask[bit_index / bits_per_word] & (bitmask_type{1} << (bit_index % bits_per_word)); + } + + [[nodiscard]] __device__ bool is_valid_nocheck(size_t element_index) const + { + return bit_is_set(__null_mask, element_index); + } + + __device__ bool is_valid(size_t element_index) const + { + return not nullable() or is_valid_nocheck(element_index); + } + + __device__ bool is_null(size_t element_index) const { return !is_valid(element_index); } + + __device__ T& element(size_t idx) const { return __data[idx]; } + + __device__ optional nullable_element(size_t idx) const; + + __device__ void assign(size_t idx, T value) const { __data[idx] = value; } +}; + +#define FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE \ + DO_IT(bool) \ + DO_IT(int8_t) \ + DO_IT(int16_t) \ + DO_IT(int32_t) \ + DO_IT(int64_t) \ + DO_IT(uint8_t) \ + DO_IT(uint16_t) \ + DO_IT(uint32_t) \ + DO_IT(uint64_t) \ + DO_IT(float32_t) \ + DO_IT(float64_t) \ + DO_IT(timestamp_D) \ + DO_IT(timestamp_h) \ + DO_IT(timestamp_m) \ + DO_IT(timestamp_s) \ + DO_IT(timestamp_ms) \ + DO_IT(timestamp_us) \ + DO_IT(timestamp_ns) \ + DO_IT(duration_D) \ + DO_IT(duration_h) \ + DO_IT(duration_m) \ + DO_IT(duration_s) \ + DO_IT(duration_ms) \ + DO_IT(duration_us) \ + DO_IT(duration_ns) + +#define FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE \ + DO_IT(bool) \ + DO_IT(int8_t) \ + DO_IT(int16_t) \ + DO_IT(int32_t) \ + DO_IT(int64_t) \ + DO_IT(uint8_t) \ + DO_IT(uint16_t) \ + DO_IT(uint32_t) \ + DO_IT(uint64_t) \ + DO_IT(decimal32) \ + DO_IT(decimal64) \ + DO_IT(decimal128) \ + DO_IT(float32_t) \ + DO_IT(float64_t) \ + DO_IT(string_view) \ + DO_IT(timestamp_D) \ + DO_IT(timestamp_h) \ + DO_IT(timestamp_m) \ + DO_IT(timestamp_s) \ + DO_IT(timestamp_ms) \ + DO_IT(timestamp_us) \ + DO_IT(timestamp_ns) \ + DO_IT(duration_D) \ + DO_IT(duration_h) \ + DO_IT(duration_m) \ + DO_IT(duration_s) \ + DO_IT(duration_ms) \ + DO_IT(duration_us) \ + DO_IT(duration_ns) + +#define FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE \ + DO_IT(bool) \ + DO_IT(int8_t) \ + DO_IT(int16_t) \ + DO_IT(int32_t) \ + DO_IT(int64_t) \ + DO_IT(uint8_t) \ + DO_IT(uint16_t) \ + DO_IT(uint32_t) \ + DO_IT(uint64_t) \ + DO_IT(decimal32) \ + DO_IT(decimal64) \ + DO_IT(decimal128) \ + DO_IT(float32_t) \ + DO_IT(float64_t) \ + DO_IT(timestamp_D) \ + DO_IT(timestamp_h) \ + DO_IT(timestamp_m) \ + DO_IT(timestamp_s) \ + DO_IT(timestamp_ms) \ + DO_IT(timestamp_us) \ + DO_IT(timestamp_ns) \ + DO_IT(duration_D) \ + DO_IT(duration_h) \ + DO_IT(duration_m) \ + DO_IT(duration_s) \ + DO_IT(duration_ms) \ + DO_IT(duration_us) \ + DO_IT(duration_ns) + struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { private: data_type __type = {}; @@ -305,74 +453,33 @@ struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { template __device__ T element(size_type idx) const; + template + __device__ optional nullable_element(size_type idx) const; + __device__ size_type num_child_columns() const; }; -#define CUDF_LTO_DECL(Type) \ +#define DO_IT(Type) \ extern template __device__ Type const* column_device_view_core::head() const; -CUDF_LTO_DECL(bool) -CUDF_LTO_DECL(int8_t) -CUDF_LTO_DECL(int16_t) -CUDF_LTO_DECL(int32_t) -CUDF_LTO_DECL(int64_t) -CUDF_LTO_DECL(uint8_t) -CUDF_LTO_DECL(uint16_t) -CUDF_LTO_DECL(uint32_t) -CUDF_LTO_DECL(uint64_t) -CUDF_LTO_DECL(float32_t) -CUDF_LTO_DECL(float64_t) -CUDF_LTO_DECL(timestamp_D) -CUDF_LTO_DECL(timestamp_h) -CUDF_LTO_DECL(timestamp_m) -CUDF_LTO_DECL(timestamp_s) -CUDF_LTO_DECL(timestamp_ms) -CUDF_LTO_DECL(timestamp_us) -CUDF_LTO_DECL(timestamp_ns) -CUDF_LTO_DECL(duration_D) -CUDF_LTO_DECL(duration_h) -CUDF_LTO_DECL(duration_m) -CUDF_LTO_DECL(duration_s) -CUDF_LTO_DECL(duration_ms) -CUDF_LTO_DECL(duration_us) -CUDF_LTO_DECL(duration_ns) - -#undef CUDF_LTO_DECL - -#define CUDF_LTO_DECL(Type) \ +FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE + +#undef DO_IT + +#define DO_IT(Type) \ extern template __device__ Type column_device_view_core::element(size_type idx) const; -CUDF_LTO_DECL(bool) -CUDF_LTO_DECL(int8_t) -CUDF_LTO_DECL(int16_t) -CUDF_LTO_DECL(int32_t) -CUDF_LTO_DECL(int64_t) -CUDF_LTO_DECL(uint8_t) -CUDF_LTO_DECL(uint16_t) -CUDF_LTO_DECL(uint32_t) -CUDF_LTO_DECL(uint64_t) -CUDF_LTO_DECL(decimal32) -CUDF_LTO_DECL(decimal64) -CUDF_LTO_DECL(decimal128) -CUDF_LTO_DECL(float32_t) -CUDF_LTO_DECL(float64_t) -CUDF_LTO_DECL(string_view) -CUDF_LTO_DECL(timestamp_D) -CUDF_LTO_DECL(timestamp_h) -CUDF_LTO_DECL(timestamp_m) -CUDF_LTO_DECL(timestamp_s) -CUDF_LTO_DECL(timestamp_ms) -CUDF_LTO_DECL(timestamp_us) -CUDF_LTO_DECL(timestamp_ns) -CUDF_LTO_DECL(duration_D) -CUDF_LTO_DECL(duration_h) -CUDF_LTO_DECL(duration_m) -CUDF_LTO_DECL(duration_s) -CUDF_LTO_DECL(duration_ms) -CUDF_LTO_DECL(duration_us) -CUDF_LTO_DECL(duration_ns) - -#undef CUDF_LTO_DECL +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(Type) \ + extern template __device__ optional column_device_view_core::nullable_element( \ + size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { private: @@ -408,74 +515,44 @@ struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { template __device__ T element(size_type idx) const; + + template + __device__ optional nullable_element(size_type idx) const; + + template + __device__ void assign(size_type idx, T value) const; }; -#define CUDF_LTO_DECL(Type) \ +#define DO_IT(Type) \ extern template __device__ Type* mutable_column_device_view_core::head() const; -CUDF_LTO_DECL(bool) -CUDF_LTO_DECL(int8_t) -CUDF_LTO_DECL(int16_t) -CUDF_LTO_DECL(int32_t) -CUDF_LTO_DECL(int64_t) -CUDF_LTO_DECL(uint8_t) -CUDF_LTO_DECL(uint16_t) -CUDF_LTO_DECL(uint32_t) -CUDF_LTO_DECL(uint64_t) -CUDF_LTO_DECL(float32_t) -CUDF_LTO_DECL(float64_t) -CUDF_LTO_DECL(timestamp_D) -CUDF_LTO_DECL(timestamp_h) -CUDF_LTO_DECL(timestamp_m) -CUDF_LTO_DECL(timestamp_s) -CUDF_LTO_DECL(timestamp_ms) -CUDF_LTO_DECL(timestamp_us) -CUDF_LTO_DECL(timestamp_ns) -CUDF_LTO_DECL(duration_D) -CUDF_LTO_DECL(duration_h) -CUDF_LTO_DECL(duration_m) -CUDF_LTO_DECL(duration_s) -CUDF_LTO_DECL(duration_ms) -CUDF_LTO_DECL(duration_us) -CUDF_LTO_DECL(duration_ns) - -#undef CUDF_LTO_DECL - -#define CUDF_LTO_DECL(Type) \ +FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE + +#undef DO_IT + +#define DO_IT(Type) \ extern template __device__ Type mutable_column_device_view_core::element(size_type idx) \ const; -CUDF_LTO_DECL(bool) -CUDF_LTO_DECL(int8_t) -CUDF_LTO_DECL(int16_t) -CUDF_LTO_DECL(int32_t) -CUDF_LTO_DECL(int64_t) -CUDF_LTO_DECL(uint8_t) -CUDF_LTO_DECL(uint16_t) -CUDF_LTO_DECL(uint32_t) -CUDF_LTO_DECL(uint64_t) -CUDF_LTO_DECL(decimal32) -CUDF_LTO_DECL(decimal64) -CUDF_LTO_DECL(decimal128) -CUDF_LTO_DECL(float32_t) -CUDF_LTO_DECL(float64_t) -CUDF_LTO_DECL(string_view) -CUDF_LTO_DECL(timestamp_D) -CUDF_LTO_DECL(timestamp_h) -CUDF_LTO_DECL(timestamp_m) -CUDF_LTO_DECL(timestamp_s) -CUDF_LTO_DECL(timestamp_ms) -CUDF_LTO_DECL(timestamp_us) -CUDF_LTO_DECL(timestamp_ns) -CUDF_LTO_DECL(duration_D) -CUDF_LTO_DECL(duration_h) -CUDF_LTO_DECL(duration_m) -CUDF_LTO_DECL(duration_s) -CUDF_LTO_DECL(duration_ms) -CUDF_LTO_DECL(duration_us) -CUDF_LTO_DECL(duration_ns) - -#undef CUDF_LTO_DECL +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(Type) \ + extern template __device__ optional \ + mutable_column_device_view_core::nullable_element(size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(Type) \ + extern template __device__ void mutable_column_device_view_core::assign(size_type idx, \ + Type value) const; + +FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE + +#undef DO_IT } // namespace lto } // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/jit/lto/types.inl.cuh b/cpp/src/jit/lto/types.inl.cuh index 92aeab8eca3f..2960946e89c7 100644 --- a/cpp/src/jit/lto/types.inl.cuh +++ b/cpp/src/jit/lto/types.inl.cuh @@ -127,36 +127,11 @@ __device__ T const* column_device_view_core::head() const return lto::lower(lto::lift(this)->head>()); } -#define CUDF_LTO_INST(Type) \ - template __device__ Type const* column_device_view_core::head() const - -CUDF_LTO_INST(bool); -CUDF_LTO_INST(int8_t); -CUDF_LTO_INST(int16_t); -CUDF_LTO_INST(int32_t); -CUDF_LTO_INST(int64_t); -CUDF_LTO_INST(uint8_t); -CUDF_LTO_INST(uint16_t); -CUDF_LTO_INST(uint32_t); -CUDF_LTO_INST(uint64_t); -CUDF_LTO_INST(float32_t); -CUDF_LTO_INST(float64_t); -CUDF_LTO_INST(timestamp_D); -CUDF_LTO_INST(timestamp_h); -CUDF_LTO_INST(timestamp_m); -CUDF_LTO_INST(timestamp_s); -CUDF_LTO_INST(timestamp_ms); -CUDF_LTO_INST(timestamp_us); -CUDF_LTO_INST(timestamp_ns); -CUDF_LTO_INST(duration_D); -CUDF_LTO_INST(duration_h); -CUDF_LTO_INST(duration_m); -CUDF_LTO_INST(duration_s); -CUDF_LTO_INST(duration_ms); -CUDF_LTO_INST(duration_us); -CUDF_LTO_INST(duration_ns); - -#undef CUDF_LTO_INST +#define DO_IT(Type) template __device__ Type const* column_device_view_core::head() const + +FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE + +#undef DO_IT __device__ size_type column_device_view_core::size() const { return lto::lift(this)->size(); } @@ -201,40 +176,27 @@ __device__ T column_device_view_core::element(size_type index) const return *lto::lower(&ret); } -#define CUDF_LTO_INST(Type) \ +#define DO_IT(Type) \ template __device__ Type column_device_view_core::element(size_type idx) const -CUDF_LTO_INST(bool); -CUDF_LTO_INST(int8_t); -CUDF_LTO_INST(int16_t); -CUDF_LTO_INST(int32_t); -CUDF_LTO_INST(int64_t); -CUDF_LTO_INST(uint8_t); -CUDF_LTO_INST(uint16_t); -CUDF_LTO_INST(uint32_t); -CUDF_LTO_INST(uint64_t); -CUDF_LTO_INST(decimal32); -CUDF_LTO_INST(decimal64); -CUDF_LTO_INST(decimal128); -CUDF_LTO_INST(float32_t); -CUDF_LTO_INST(float64_t); -CUDF_LTO_INST(string_view); -CUDF_LTO_INST(timestamp_D); -CUDF_LTO_INST(timestamp_h); -CUDF_LTO_INST(timestamp_m); -CUDF_LTO_INST(timestamp_s); -CUDF_LTO_INST(timestamp_ms); -CUDF_LTO_INST(timestamp_us); -CUDF_LTO_INST(timestamp_ns); -CUDF_LTO_INST(duration_D); -CUDF_LTO_INST(duration_h); -CUDF_LTO_INST(duration_m); -CUDF_LTO_INST(duration_s); -CUDF_LTO_INST(duration_ms); -CUDF_LTO_INST(duration_us); -CUDF_LTO_INST(duration_ns); - -#undef CUDF_LTO_INST +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +template +__device__ optional column_device_view_core::nullable_element(size_type index) const +{ + auto ret = lto::lift(this)->nullable_element>(index); + return *lto::lower(&ret); +} + +#define DO_IT(Type) \ + template __device__ optional column_device_view_core::nullable_element( \ + size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT __device__ size_type column_device_view_core::num_child_columns() const { @@ -247,36 +209,11 @@ __device__ T* mutable_column_device_view_core::head() const return lto::lower(lto::lift(this)->head>()); } -#define CUDF_LTO_INST(Type) \ - template __device__ Type* mutable_column_device_view_core::head() const - -CUDF_LTO_INST(bool); -CUDF_LTO_INST(int8_t); -CUDF_LTO_INST(int16_t); -CUDF_LTO_INST(int32_t); -CUDF_LTO_INST(int64_t); -CUDF_LTO_INST(uint8_t); -CUDF_LTO_INST(uint16_t); -CUDF_LTO_INST(uint32_t); -CUDF_LTO_INST(uint64_t); -CUDF_LTO_INST(float32_t); -CUDF_LTO_INST(float64_t); -CUDF_LTO_INST(timestamp_D); -CUDF_LTO_INST(timestamp_h); -CUDF_LTO_INST(timestamp_m); -CUDF_LTO_INST(timestamp_s); -CUDF_LTO_INST(timestamp_ms); -CUDF_LTO_INST(timestamp_us); -CUDF_LTO_INST(timestamp_ns); -CUDF_LTO_INST(duration_D); -CUDF_LTO_INST(duration_h); -CUDF_LTO_INST(duration_m); -CUDF_LTO_INST(duration_s); -CUDF_LTO_INST(duration_ms); -CUDF_LTO_INST(duration_us); -CUDF_LTO_INST(duration_ns); - -#undef CUDF_LTO_INST +#define DO_IT(Type) template __device__ Type* mutable_column_device_view_core::head() const + +FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE + +#undef DO_IT __device__ size_type mutable_column_device_view_core::size() const { @@ -330,36 +267,35 @@ __device__ T mutable_column_device_view_core::element(size_type index) const return *lto::lower(&ret); } -#define CUDF_LTO_INST(Type) \ +#define DO_IT(Type) \ template __device__ Type mutable_column_device_view_core::element(size_type idx) const -CUDF_LTO_INST(bool); -CUDF_LTO_INST(int8_t); -CUDF_LTO_INST(int16_t); -CUDF_LTO_INST(int32_t); -CUDF_LTO_INST(int64_t); -CUDF_LTO_INST(uint8_t); -CUDF_LTO_INST(uint16_t); -CUDF_LTO_INST(uint32_t); -CUDF_LTO_INST(uint64_t); -CUDF_LTO_INST(float32_t); -CUDF_LTO_INST(float64_t); -CUDF_LTO_INST(timestamp_D); -CUDF_LTO_INST(timestamp_h); -CUDF_LTO_INST(timestamp_m); -CUDF_LTO_INST(timestamp_s); -CUDF_LTO_INST(timestamp_ms); -CUDF_LTO_INST(timestamp_us); -CUDF_LTO_INST(timestamp_ns); -CUDF_LTO_INST(duration_D); -CUDF_LTO_INST(duration_h); -CUDF_LTO_INST(duration_m); -CUDF_LTO_INST(duration_s); -CUDF_LTO_INST(duration_ms); -CUDF_LTO_INST(duration_us); -CUDF_LTO_INST(duration_ns); - -#undef CUDF_LTO_INST +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +template +__device__ optional mutable_column_device_view_core::nullable_element(size_type index) const +{ + auto ret = lto::lift(this)->nullable_element>(index); + return *lto::lower(&ret); +} + +#define DO_IT(Type) \ + template __device__ optional mutable_column_device_view_core::nullable_element( \ + size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(Type) \ + template __device__ void mutable_column_device_view_core::assign(size_type idx, \ + Type value) const; + +FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE + +#undef DO_IT } // namespace lto } // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index d7efc0e6ce0d..717fca99be03 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -9,6 +9,8 @@ #include #include +#include + #include #include @@ -19,38 +21,50 @@ struct RTCTest : public ::testing::Test {}; TEST_F(RTCTest, CreateFragment) { + + // TODO: add configuration parameters necessary for testing + // and profiling cache behaviour auto fn = []() { - auto begin = std::chrono::high_resolution_clock::now(); - auto kern = rtc::compile_and_link_udf("test_fragment", - R"***( - #include "cudf/jit/transform_params.cuh" + auto lib = rtc::compile_and_link_udf("test_fragment", + R"***( + #include "cudf/jit/lto/transform_params.cuh" #include "cudf/jit/lto/operators.cuh" - extern "C" __device__ void transform_operator(cudf::lto::transform_params const* p){ + // TODO: declare getters and setters required for the specific LTO context of the operator + // they should use the provided LTO functions; might need a planner + + extern "C" __device__ void transform_operator(cudf::lto::transform_params p){ int a = 1; int b = 2; int c = 3; - int * out = (int *)p->outputs; + double d; + int * out = (int *)p.scope[0]; cudf::lto::operators::add(&c, &a, &b); cudf::lto::operators::sub(&c, &a, &b); cudf::lto::operators::mul(&c, &a, &b); cudf::lto::operators::mul(&c, &a, &b); + cudf::lto::operators::cast_to_float64(&d, &c); + c = (int)a; + c = c * c; *out = a + b * c; } )***", - "test_udf_key", - "transform_kernel"); - - (void)kern; - auto end = std::chrono::high_resolution_clock::now(); - auto dur = end - begin; - std::cout << "RTC compilation took " - << std::chrono::duration_cast(dur).count() << " us\n"; + "test_udf_key", + "transform_kernel"); + + auto kernel = lib->get_kernel("transform_kernel"); + + EXPECT_EQ("transform_kernel", kernel.get_name()); + void** scope = nullptr; + int32_t num_rows = 0; + void* args[] = {&scope, &num_rows}; + kernel.launch(1, 1, 1, 1, 1, 1, 0, nullptr, args); }; fn(); fn(); + fn(); } CUDF_TEST_PROGRAM_MAIN() From e2915e7166826495d1b64df49b3b9c6bd0179d50 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 11 Feb 2026 03:10:37 +0000 Subject: [PATCH 042/254] feaute-parity for column_device_view core types --- .../cudf/column/column_device_view_base.cuh | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/cpp/include/cudf/column/column_device_view_base.cuh b/cpp/include/cudf/column/column_device_view_base.cuh index 111361b10961..8a283351f7a1 100644 --- a/cpp/include/cudf/column/column_device_view_base.cuh +++ b/cpp/include/cudf/column/column_device_view_base.cuh @@ -411,6 +411,30 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba return data()[element_index]; } + /** + * @brief Returns an optional of the element at the specified index. + * + * This function accounts for the offset. + * + * This function does not participate in overload resolution if `is_rep_layout_compatible` is + * false. Specializations of this function may exist for types `T` where + *`is_rep_layout_compatible` is false. + * + * + * @tparam T The element type + * @param element_index Position of the desired element + * @return Optional containing the element at the specified index, or `nullopt` if the element is + *null + */ + template ())> + [[nodiscard]] __device__ cuda::std::optional nullable_element( + size_type element_index) const noexcept + { + if (is_null(element_index)) { return cuda::std::nullopt; } + + return element(element_index); + } + /** * @brief Returns `string_view` to the string element at the specified index. * @@ -433,6 +457,27 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba return string_view{d_strings + offset, static_cast(itr[index + 1] - offset)}; } + /** + * @brief Returns optional `string_view` to the string element at the specified index. + * + * If the element at the specified index is NULL, i.e., `is_null(element_index) + * == true`, then nullopt is returned. + * + * This function accounts for the offset. + * + * @param element_index Position of the desired string element + * @return Optional containing the string_view instance representing this element at this index, + * or `nullopt` if the element is null + */ + template )> + [[nodiscard]] __device__ cuda::std::optional nullable_element( + size_type element_index) const noexcept + { + if (is_null(element_index)) { return cuda::std::nullopt; } + + return element(element_index); + } + public: /** * @brief Returns a `numeric::fixed_point` element at the specified index for a `fixed_point` @@ -453,6 +498,25 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba return T{scaled_integer{data()[element_index], scale}}; } + /** + * @brief Returns a `numeric::fixed_point` element at the specified index for a `fixed_point` + * column. + * + * If the element at the specified index is NULL, i.e., `is_null(element_index) == true`, + * then any attempt to use the result will lead to undefined behavior. + * + * @param element_index Position of the desired element + * @return numeric::fixed_point representing the element at this index + */ + template ())> + [[nodiscard]] __device__ cuda::std::optional nullable_element( + size_type element_index) const noexcept + { + if (is_null(element_index)) { return cuda::std::nullopt; } + + return element(element_index); + } + /** * @brief Returns the specified child * @@ -596,6 +660,111 @@ class alignas(16) mutable_column_device_view_core : public detail::column_device return data()[element_index]; } + /** + * @brief Returns an optional of the element at the specified index. + * + * This function accounts for the offset. + * + * This function does not participate in overload resolution if `is_rep_layout_compatible` is + * false. Specializations of this function may exist for types `T` where + *`is_rep_layout_compatible` is false. + * + * + * @tparam T The element type + * @param element_index Position of the desired element + * @return Optional containing the element at the specified index, or `nullopt` if the element is + *null + */ + template ())> + [[nodiscard]] __device__ cuda::std::optional nullable_element( + size_type element_index) const noexcept + { + if (is_null(element_index)) { return cuda::std::nullopt; } + + return element(element_index); + } + + /** + * @brief Returns `string_view` to the string element at the specified index. + * + * If the element at the specified index is NULL, i.e., `is_null(element_index) + * == true`, then any attempt to use the result will lead to undefined behavior. + * + * This function accounts for the offset. + * + * @param element_index Position of the desired string element + * @return string_view instance representing this element at this index + */ + template )> + [[nodiscard]] __device__ T element(size_type element_index) const noexcept + { + size_type index = element_index + offset(); // account for this view's _offset + char const* d_strings = static_cast(_data); + auto const offsets = child(offsets_column_index); + auto const itr = cudf::detail::input_offsetalator(offsets.head(), offsets.type()); + auto const offset = itr[index]; + return string_view{d_strings + offset, static_cast(itr[index + 1] - offset)}; + } + + /** + * @brief Returns optional `string_view` to the string element at the specified index. + * + * If the element at the specified index is NULL, i.e., `is_null(element_index) + * == true`, then nullopt is returned. + * + * This function accounts for the offset. + * + * @param element_index Position of the desired string element + * @return Optional containing the string_view instance representing this element at this index, + * or `nullopt` if the element is null + */ + template )> + [[nodiscard]] __device__ cuda::std::optional nullable_element( + size_type element_index) const noexcept + { + if (is_null(element_index)) { return cuda::std::nullopt; } + + return element(element_index); + } + + /** + * @brief Returns a `numeric::fixed_point` element at the specified index for a `fixed_point` + * column. + * + * If the element at the specified index is NULL, i.e., `is_null(element_index) == true`, + * then any attempt to use the result will lead to undefined behavior. + * + * @param element_index Position of the desired element + * @return numeric::fixed_point representing the element at this index + */ + template ())> + [[nodiscard]] __device__ T element(size_type element_index) const noexcept + { + using namespace numeric; + using rep = typename T::rep; + auto const scale = scale_type{_type.scale()}; + return T{scaled_integer{data()[element_index], scale}}; + } + + /** + * @brief Returns a `numeric::fixed_point` element at the specified index for a `fixed_point` + * column. + * + * If the element at the specified index is NULL, i.e., `is_null(element_index) == true`, + * then any attempt to use the result will lead to undefined behavior. + * + * @param element_index Position of the desired element + * @return numeric::fixed_point representing the element at this index + */ + template ())> + [[nodiscard]] __device__ cuda::std::optional nullable_element( + size_type element_index) const noexcept + { + if (is_null(element_index)) { return cuda::std::nullopt; } + + return element(element_index); + } + /** * @brief Assigns `value` to the element at `element_index` * From 4480033a6f3d31521a308c597ebd5c998e2a6fd9 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 11 Feb 2026 03:14:27 +0000 Subject: [PATCH 043/254] refactoring --- cpp/include/cudf/jit/lto/accessors.cuh | 119 ----------- cpp/include/cudf/jit/lto/transform_params.cuh | 1 - cpp/include/cudf/jit/lto/types.cuh | 198 +++++++++--------- cpp/src/jit/lto/types.inl.cuh | 189 +++++++++-------- cpp/src/transform/jit/lto_kernel.inl.cuh | 5 +- 5 files changed, 199 insertions(+), 313 deletions(-) delete mode 100644 cpp/include/cudf/jit/lto/accessors.cuh diff --git a/cpp/include/cudf/jit/lto/accessors.cuh b/cpp/include/cudf/jit/lto/accessors.cuh deleted file mode 100644 index 302f13fc9730..000000000000 --- a/cpp/include/cudf/jit/lto/accessors.cuh +++ /dev/null @@ -1,119 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include - -// TODO: cuda std optional? -// TODO: how to convert to our optional? -// TODO: handle scalar -// TODO: scope variables should be aligned to avoid uncoalesced reads/writes -// TODO: how will expression evaluation work to handle multiple outputs? -// TODO: make fixed_point be able to assign values to columns -// TODO: is_null and is_valid - -namespace cudf { -namespace lto { - -using scope_type = void* const __restrict__* __restrict__; - -template -struct column { - static constexpr bool IS_SCALAR = IsScalar; - static constexpr bool IS_NULLABLE = IsNullable; - - using Type = T; - using Arg = ColumnType const* __restrict__; - - static __device__ auto get(scope_type scope, size_type i) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - if constexpr (!IsNullable) { - return p->template element(index); - } else { - return p->template nullable_element(index); - } - } - - static __device__ void assign(scope_type scope, size_type i, auto const& value) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - p->template assign(index, value); - } - - static __device__ auto* null_mask(scope_type scope) - { - auto p = static_cast(scope[ScopeIndex]); - return p->null_mask(); - } -}; - -template -struct span { - static constexpr bool IS_SCALAR = IsScalar; - static constexpr bool IS_NULLABLE = IsNullable; - - using Type = T; - using Arg = SpanType const* __restrict__; - - static __device__ auto get(scope_type scope, size_type i) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - if constexpr (!IsNullable) { - return p->template element(index); - } else { - return p->template nullable_element(index); - } - } - - static __device__ void assign(scope_type scope, size_type i, auto const& value) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - p->template assign(index, value); - } - - static __device__ auto* null_mask(scope_type scope) - { - auto p = static_cast(scope[ScopeIndex]); - return p->null_mask(); - } -}; - -} // namespace lto -} // namespace cudf - -template -struct element_operation { - template - static __device__ void evaluate(scope_type scope, cudf::size_type i, Operator&& op) - { - if constexpr (UserDataIndex >= 0) { - auto output_args; - GENERIC_TRANSFORM_OP(user_data, i, &res, In::element(inputs, i)...); - } else { - GENERIC_TRANSFORM_OP(&res, In::element(inputs, i)...); - } - } -}; diff --git a/cpp/include/cudf/jit/lto/transform_params.cuh b/cpp/include/cudf/jit/lto/transform_params.cuh index c81a2cee7c67..38ded95bc695 100644 --- a/cpp/include/cudf/jit/lto/transform_params.cuh +++ b/cpp/include/cudf/jit/lto/transform_params.cuh @@ -8,7 +8,6 @@ #include namespace CUDF_LTO_EXPORT cudf { - namespace lto { /// @brief Type-erased parameters for LTO-JIT-compiled transform operations. diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index adc34412c57f..0bb5561a2614 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -20,6 +20,17 @@ namespace lto { * will be included and compiled at JIT compile time. Including other headers will lead to longer * JIT compile times which can be unbounded and cause slowdowns. * + * This essentially serves as the ABI for LTO-JIT compiled code to interact with the rest of cuDF. + * Any changes to this header should be made with ABI stability in mind as it can break existing + * LTO-JIT compiled code and lead to undefined behavior. For example, adding new member variables to + * these structs will change their size and layout which can break existing code. Adding new + * functions is generally safe as long as they don't change the existing function signatures, but it + * can still lead to issues if the new functions are called from existing code that wasn't compiled + * with them. Removing or changing existing functions is not safe and will break existing code. + * Changing the types of existing member variables can also break existing code if it changes the + * size or layout of the structs. In general, any change to this header should be made with caution + * and thorough testing to ensure ABI compatibility. + * */ using int8_t = signed char; @@ -53,15 +64,15 @@ enum scale_type : int32_t {}; struct CUDF_LTO_ALIAS data_type { private: - type_id __id = {}; - int32_t __fixed_point_scale = 0; + type_id _id = {}; + int32_t _fixed_point_scale = 0; }; struct CUDF_LTO_ALIAS string_view { private: - char const* __data = nullptr; - size_type __bytes = 0; - mutable size_type __length = 0; + char const* _data = nullptr; + size_type _bytes = 0; + mutable size_type _length = 0; public: [[nodiscard]] __device__ size_type size_bytes() const; @@ -125,97 +136,97 @@ struct CUDF_LTO_ALIAS string_view { struct CUDF_LTO_ALIAS decimal32 { private: - int32_t __value = 0; - scale_type __scale = scale_type{}; + int32_t _value = 0; + scale_type _scale = scale_type{}; }; struct CUDF_LTO_ALIAS decimal64 { private: - int64_t __value = 0; - scale_type __scale = scale_type{}; + int64_t _value = 0; + scale_type _scale = scale_type{}; }; struct CUDF_LTO_ALIAS decimal128 { private: - __int128_t __value = 0; - scale_type __scale = scale_type{}; + __int128_t _value = 0; + scale_type _scale = scale_type{}; }; struct CUDF_LTO_ALIAS timestamp_D { private: - int32_t __rep = 0; + int32_t _rep = 0; }; struct CUDF_LTO_ALIAS timestamp_h { private: - int32_t __rep = 0; + int32_t _rep = 0; }; struct CUDF_LTO_ALIAS timestamp_m { private: - int32_t __rep = 0; + int32_t _rep = 0; }; struct CUDF_LTO_ALIAS timestamp_s { private: - int64_t __rep = 0; + int64_t _rep = 0; }; struct CUDF_LTO_ALIAS timestamp_ms { private: - int64_t __rep = 0; + int64_t _rep = 0; }; struct CUDF_LTO_ALIAS timestamp_us { private: - int64_t __rep = 0; + int64_t _rep = 0; }; struct CUDF_LTO_ALIAS timestamp_ns { private: - int64_t __rep = 0; + int64_t _rep = 0; }; struct CUDF_LTO_ALIAS duration_D { private: - int32_t __rep = 0; + int32_t _rep = 0; }; struct CUDF_LTO_ALIAS duration_h { private: - int32_t __rep = 0; + int32_t _rep = 0; }; struct CUDF_LTO_ALIAS duration_m { private: - int32_t __rep = 0; + int32_t _rep = 0; }; struct CUDF_LTO_ALIAS duration_s { private: - int64_t __rep = 0; + int64_t _rep = 0; }; struct CUDF_LTO_ALIAS duration_ms { private: - int64_t __rep = 0; + int64_t _rep = 0; }; struct CUDF_LTO_ALIAS duration_us { private: - int64_t __rep = 0; + int64_t _rep = 0; }; struct CUDF_LTO_ALIAS duration_ns { private: - int64_t __rep = 0; + int64_t _rep = 0; }; struct inplace_t {}; inline constexpr inplace_t inplace{}; -// [ ] assumes T is trivially copyable +// TODO: assumes T is trivially copyable template struct CUDF_LTO_ALIAS optional { private: @@ -274,46 +285,46 @@ optional(T) -> optional; template struct [[nodiscard]] device_span { private: - T* __data = nullptr; - size_t __size = 0; + T* _data = nullptr; + size_t _size = 0; }; +__device__ constexpr bool bit_is_set(bitmask_type const* bitmask, size_t bit_index) +{ + constexpr auto bits_per_word = sizeof(bitmask_type) * 8; + return bitmask[bit_index / bits_per_word] & (bitmask_type{1} << (bit_index % bits_per_word)); +} + template struct [[nodiscard]] device_optional_span { private: - T* __data = nullptr; - size_t __size = 0; - bitmask_type* __null_mask = nullptr; + T* _data = nullptr; + size_t _size = 0; + bitmask_type* _null_mask = nullptr; public: - __device__ T* data() const { return __data; } + __device__ T* data() const { return _data; } - __device__ size_t size() const { return __size; } + __device__ size_t size() const { return _size; } - __device__ bool empty() const { return __size == 0; } + __device__ bool empty() const { return _size == 0; } - __device__ T& operator[](size_t pos) const { return __data[pos]; } + __device__ T& operator[](size_t pos) const { return _data[pos]; } - __device__ T* begin() const { return __data; } + __device__ T* begin() const { return _data; } - __device__ T* end() const { return __data + __size; } + __device__ T* end() const { return _data + _size; } __device__ device_optional_span as_const() const { - return device_optional_span{__data, __size, __null_mask}; + return device_optional_span{_data, _size, _null_mask}; } - __device__ bool nullable() const { return __null_mask != nullptr; } - - __device__ static constexpr bool bit_is_set(bitmask_type const* bitmask, size_t bit_index) - { - constexpr auto bits_per_word = sizeof(bitmask_type) * 8; - return bitmask[bit_index / bits_per_word] & (bitmask_type{1} << (bit_index % bits_per_word)); - } + __device__ bool nullable() const { return _null_mask != nullptr; } [[nodiscard]] __device__ bool is_valid_nocheck(size_t element_index) const { - return bit_is_set(__null_mask, element_index); + return bit_is_set(_null_mask, element_index); } __device__ bool is_valid(size_t element_index) const @@ -323,11 +334,11 @@ struct [[nodiscard]] device_optional_span { __device__ bool is_null(size_t element_index) const { return !is_valid(element_index); } - __device__ T& element(size_t idx) const { return __data[idx]; } + __device__ T& element(size_t idx) const { return _data[idx]; } __device__ optional nullable_element(size_t idx) const; - __device__ void assign(size_t idx, T value) const { __data[idx] = value; } + __device__ void assign(size_t idx, T value) const { _data[idx] = value; } }; #define FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE \ @@ -420,13 +431,13 @@ struct [[nodiscard]] device_optional_span { struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { private: - data_type __type = {}; - size_type __size = 0; - void const* __data = nullptr; - bitmask_type const* __null_mask = nullptr; - size_type __offset = 0; - column_device_view_core* __d_children = nullptr; - size_type __num_children = 0; + data_type _type = {}; + size_type _size = 0; + void const* _data = nullptr; + bitmask_type const* _null_mask = nullptr; + size_type _offset = 0; + column_device_view_core* _d_children = nullptr; + size_type _num_children = 0; public: template @@ -459,37 +470,15 @@ struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { __device__ size_type num_child_columns() const; }; -#define DO_IT(Type) \ - extern template __device__ Type const* column_device_view_core::head() const; - -FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE - -#undef DO_IT - -#define DO_IT(Type) \ - extern template __device__ Type column_device_view_core::element(size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(Type) \ - extern template __device__ optional column_device_view_core::nullable_element( \ - size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { private: - data_type __type = {}; - size_type __size = 0; - void const* __data = nullptr; - bitmask_type const* __null_mask = nullptr; - size_type __offset = 0; - mutable_column_device_view_core* __d_children = nullptr; - size_type __num_children = 0; + data_type _type = {}; + size_type _size = 0; + void const* _data = nullptr; + bitmask_type const* _null_mask = nullptr; + size_type _offset = 0; + mutable_column_device_view_core* _d_children = nullptr; + size_type _num_children = 0; public: template @@ -523,32 +512,51 @@ struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { __device__ void assign(size_type idx, T value) const; }; -#define DO_IT(Type) \ - extern template __device__ Type* mutable_column_device_view_core::head() const; +#define DO_IT(T) extern template __device__ T const* column_device_view_core::head() const; FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE #undef DO_IT -#define DO_IT(Type) \ - extern template __device__ Type mutable_column_device_view_core::element(size_type idx) \ - const; +#define DO_IT(T) \ + extern template __device__ T column_device_view_core::element(size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(T) \ + extern template __device__ optional column_device_view_core::nullable_element( \ + size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -#define DO_IT(Type) \ - extern template __device__ optional \ - mutable_column_device_view_core::nullable_element(size_type idx) const; +#define DO_IT(T) extern template __device__ T* mutable_column_device_view_core::head() const; + +FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE + +#undef DO_IT + +#define DO_IT(T) \ + extern template __device__ T mutable_column_device_view_core::element(size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(T) \ + extern template __device__ optional mutable_column_device_view_core::nullable_element( \ + size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -#define DO_IT(Type) \ - extern template __device__ void mutable_column_device_view_core::assign(size_type idx, \ - Type value) const; +#define DO_IT(T) \ + extern template __device__ void mutable_column_device_view_core::assign(size_type idx, \ + T value) const; FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE diff --git a/cpp/src/jit/lto/types.inl.cuh b/cpp/src/jit/lto/types.inl.cuh index 2960946e89c7..83a7474870df 100644 --- a/cpp/src/jit/lto/types.inl.cuh +++ b/cpp/src/jit/lto/types.inl.cuh @@ -12,70 +12,70 @@ namespace lto { [[nodiscard]] __device__ size_type string_view::size_bytes() const { - return lto::lift(this)->size_bytes(); + return lift(this)->size_bytes(); } -[[nodiscard]] __device__ size_type string_view::length() const { return lto::lift(this)->length(); } +[[nodiscard]] __device__ size_type string_view::length() const { return lift(this)->length(); } -[[nodiscard]] __device__ char const* string_view::data() const { return lto::lift(this)->data(); } +[[nodiscard]] __device__ char const* string_view::data() const { return lift(this)->data(); } -[[nodiscard]] __device__ bool string_view::empty() const { return lto::lift(this)->empty(); } +[[nodiscard]] __device__ bool string_view::empty() const { return lift(this)->empty(); } __device__ char_utf8 string_view::operator[](size_type pos) const { - return lto::lift(this)->operator[](pos); + return lift(this)->operator[](pos); } [[nodiscard]] __device__ size_type string_view::byte_offset(size_type pos) const { - return lto::lift(this)->byte_offset(pos); + return lift(this)->byte_offset(pos); } [[nodiscard]] __device__ int string_view::compare(string_view const& str) const { - return lto::lift(this)->compare(*lto::lift(&str)); + return lift(this)->compare(*lift(&str)); } __device__ int string_view::compare(char const* str, size_type bytes) const { - return lto::lift(this)->compare(str, bytes); + return lift(this)->compare(str, bytes); } __device__ bool string_view::operator==(string_view const& rhs) const { - return lto::lift(this)->operator==(*lto::lift(&rhs)); + return lift(this)->operator==(*lift(&rhs)); } __device__ bool string_view::operator!=(string_view const& rhs) const { - return lto::lift(this)->operator!=(*lto::lift(&rhs)); + return lift(this)->operator!=(*lift(&rhs)); } __device__ bool string_view::operator<(string_view const& rhs) const { - return lto::lift(this)->operator<(*lto::lift(&rhs)); + return lift(this)->operator<(*lift(&rhs)); } __device__ bool string_view::operator>(string_view const& rhs) const { - return lto::lift(this)->operator>(*lto::lift(&rhs)); + return lift(this)->operator>(*lift(&rhs)); } __device__ bool string_view::operator<=(string_view const& rhs) const { - return lto::lift(this)->operator<=(*lto::lift(&rhs)); + return lift(this)->operator<=(*lift(&rhs)); } __device__ bool string_view::operator>=(string_view const& rhs) const { - return lto::lift(this)->operator>=(*lto::lift(&rhs)); + return lift(this)->operator>=(*lift(&rhs)); } [[nodiscard]] __device__ size_type string_view::find(string_view const& str, size_type pos, size_type count) const { - return lto::lift(this)->find(*lto::lift(&str), pos, count); + return lift(this)->find(*lift(&str), pos, count); } __device__ size_type string_view::find(char const* str, @@ -83,21 +83,21 @@ __device__ size_type string_view::find(char const* str, size_type pos, size_type count) const { - return lto::lift(this)->find(str, bytes, pos, count); + return lift(this)->find(str, bytes, pos, count); } [[nodiscard]] __device__ size_type string_view::find(char_utf8 character, size_type pos, size_type count) const { - return lto::lift(this)->find(character, pos, count); + return lift(this)->find(character, pos, count); } [[nodiscard]] __device__ size_type string_view::rfind(string_view const& str, size_type pos, size_type count) const { - return lto::lift(this)->rfind(*lto::lift(&str), pos, count); + return lift(this)->rfind(*lift(&str), pos, count); } __device__ size_type string_view::rfind(char const* str, @@ -105,193 +105,192 @@ __device__ size_type string_view::rfind(char const* str, size_type pos, size_type count) const { - return lto::lift(this)->rfind(str, bytes, pos, count); + return lift(this)->rfind(str, bytes, pos, count); } [[nodiscard]] __device__ size_type string_view::rfind(char_utf8 character, size_type pos, size_type count) const { - return lto::lift(this)->rfind(character, pos, count); + return lift(this)->rfind(character, pos, count); } [[nodiscard]] __device__ string_view string_view::substr(size_type start, size_type length) const { - auto ret = lto::lift(this)->substr(start, length); - return *lto::lower(&ret); + auto ret = lift(this)->substr(start, length); + return *lower(&ret); } template __device__ T const* column_device_view_core::head() const { - return lto::lower(lto::lift(this)->head>()); + return lower(lift(this)->head>()); } -#define DO_IT(Type) template __device__ Type const* column_device_view_core::head() const +__device__ size_type column_device_view_core::size() const { return lift(this)->size(); } -FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE - -#undef DO_IT - -__device__ size_type column_device_view_core::size() const { return lto::lift(this)->size(); } - -__device__ bool column_device_view_core::nullable() const { return lto::lift(this)->nullable(); } +__device__ bool column_device_view_core::nullable() const { return lift(this)->nullable(); } __device__ bitmask_type const* column_device_view_core::null_mask() const { - return lto::lift(this)->null_mask(); + return lift(this)->null_mask(); } -__device__ size_type column_device_view_core::offset() const { return lto::lift(this)->offset(); } +__device__ size_type column_device_view_core::offset() const { return lift(this)->offset(); } __device__ bool column_device_view_core::is_valid(size_type index) const { - return lto::lift(this)->is_valid(index); + return lift(this)->is_valid(index); } __device__ bool column_device_view_core::is_valid_nocheck(size_type index) const { - return lto::lift(this)->is_valid_nocheck(index); + return lift(this)->is_valid_nocheck(index); } __device__ bool column_device_view_core::is_null(size_type index) const { - return lto::lift(this)->is_null(index); + return lift(this)->is_null(index); } __device__ bool column_device_view_core::is_null_nocheck(size_type index) const { - return lto::lift(this)->is_null_nocheck(index); + return lift(this)->is_null_nocheck(index); } __device__ bitmask_type column_device_view_core::get_mask_word(size_type index) const { - return lto::lift(this)->get_mask_word(index); + return lift(this)->get_mask_word(index); } template __device__ T column_device_view_core::element(size_type index) const { - auto ret = lto::lift(this)->element>(index); - return *lto::lower(&ret); + auto ret = lift(this)->element>(index); + return *lower(&ret); } -#define DO_IT(Type) \ - template __device__ Type column_device_view_core::element(size_type idx) const - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - template __device__ optional column_device_view_core::nullable_element(size_type index) const { - auto ret = lto::lift(this)->nullable_element>(index); - return *lto::lower(&ret); + auto ret = lift(this)->nullable_element>(index); + return *lower(&ret); } -#define DO_IT(Type) \ - template __device__ optional column_device_view_core::nullable_element( \ - size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - __device__ size_type column_device_view_core::num_child_columns() const { - return lto::lift(this)->num_child_columns(); + return lift(this)->num_child_columns(); } template __device__ T* mutable_column_device_view_core::head() const { - return lto::lower(lto::lift(this)->head>()); + return lower(lift(this)->head>()); } -#define DO_IT(Type) template __device__ Type* mutable_column_device_view_core::head() const - -FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE - -#undef DO_IT - -__device__ size_type mutable_column_device_view_core::size() const -{ - return lto::lift(this)->size(); -} +__device__ size_type mutable_column_device_view_core::size() const { return lift(this)->size(); } -__device__ bool mutable_column_device_view_core::nullable() const -{ - return lto::lift(this)->nullable(); -} +__device__ bool mutable_column_device_view_core::nullable() const { return lift(this)->nullable(); } __device__ bitmask_type* mutable_column_device_view_core::null_mask() const { - return lto::lift(this)->null_mask(); + return lift(this)->null_mask(); } __device__ size_type mutable_column_device_view_core::offset() const { - return lto::lift(this)->offset(); + return lift(this)->offset(); } __device__ bool mutable_column_device_view_core::is_valid(size_type index) const { - return lto::lift(this)->is_valid(index); + return lift(this)->is_valid(index); } __device__ bool mutable_column_device_view_core::is_valid_nocheck(size_type index) const { - return lto::lift(this)->is_valid_nocheck(index); + return lift(this)->is_valid_nocheck(index); } __device__ bool mutable_column_device_view_core::is_null(size_type index) const { - return lto::lift(this)->is_null(index); + return lift(this)->is_null(index); } __device__ bool mutable_column_device_view_core::is_null_nocheck(size_type index) const { - return lto::lift(this)->is_null_nocheck(index); + return lift(this)->is_null_nocheck(index); } __device__ bitmask_type mutable_column_device_view_core::get_mask_word(size_type index) const { - return lto::lift(this)->get_mask_word(index); + return lift(this)->get_mask_word(index); } template __device__ T mutable_column_device_view_core::element(size_type index) const { - auto ret = lto::lift(this)->element>(index); - return *lto::lower(&ret); + auto ret = lift(this)->element>(index); + return *lower(&ret); +} + +template +__device__ optional mutable_column_device_view_core::nullable_element(size_type index) const +{ + auto ret = lift(this)->nullable_element>(index); + return *lower(&ret); +} + +template +__device__ void mutable_column_device_view_core::assign(size_type index, T value) const +{ + lift(this)->assign>(index, *lift(&value)); } -#define DO_IT(Type) \ - template __device__ Type mutable_column_device_view_core::element(size_type idx) const +/// Explicit template instantiations + +#define DO_IT(T) template __device__ T const* column_device_view_core::head() const; + +FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE + +#undef DO_IT + +#define DO_IT(T) template __device__ T column_device_view_core::element(size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -template -__device__ optional mutable_column_device_view_core::nullable_element(size_type index) const -{ - auto ret = lto::lift(this)->nullable_element>(index); - return *lto::lower(&ret); -} +#define DO_IT(T) \ + template __device__ optional column_device_view_core::nullable_element(size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(T) template __device__ T* mutable_column_device_view_core::head() const; + +FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE + +#undef DO_IT + +#define DO_IT(T) \ + template __device__ T mutable_column_device_view_core::element(size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT -#define DO_IT(Type) \ - template __device__ optional mutable_column_device_view_core::nullable_element( \ +#define DO_IT(T) \ + template __device__ optional mutable_column_device_view_core::nullable_element( \ size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -#define DO_IT(Type) \ - template __device__ void mutable_column_device_view_core::assign(size_type idx, \ - Type value) const; +#define DO_IT(T) \ + template __device__ void mutable_column_device_view_core::assign(size_type idx, T value) const; FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE diff --git a/cpp/src/transform/jit/lto_kernel.inl.cuh b/cpp/src/transform/jit/lto_kernel.inl.cuh index 55da5a3fc3a1..3bc2a734d45a 100644 --- a/cpp/src/transform/jit/lto_kernel.inl.cuh +++ b/cpp/src/transform/jit/lto_kernel.inl.cuh @@ -5,15 +5,14 @@ #pragma once #include -#include +#include #include extern "C" { __device__ void transform_operator(cudf::lto::transform_params params); -__global__ void transform_kernel(void* __restrict__ const* __restrict__ scope, - int32_t num_rows) +__global__ void transform_kernel(void* __restrict__ const* __restrict__ scope, int32_t num_rows) { auto start = cudf::detail::grid_1d::global_thread_id(); auto stride = cudf::detail::grid_1d::grid_stride(); From ba8f921b568e64fb74a0be9b6fa7519e760de2a5 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 11 Feb 2026 03:15:43 +0000 Subject: [PATCH 044/254] update --- cpp/include/cudf/jit/lto/scope.cuh | 138 +++++++++++++++++++++++++++++ cpp/src/jit/rtc/rtc.cpp | 1 + 2 files changed, 139 insertions(+) create mode 100644 cpp/include/cudf/jit/lto/scope.cuh diff --git a/cpp/include/cudf/jit/lto/scope.cuh b/cpp/include/cudf/jit/lto/scope.cuh new file mode 100644 index 000000000000..0a570f190c8d --- /dev/null +++ b/cpp/include/cudf/jit/lto/scope.cuh @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +// TODO: scope variables should be aligned to avoid uncoalesced reads/writes + +namespace cudf { +namespace lto { +namespace scope { + +using args = void* const __restrict__* __restrict__; + +template +struct column { + static constexpr bool IS_SCALAR = IsScalar; + static constexpr bool IS_NULLABLE = IsNullable; + + using Type = T; + using Arg = ColumnType const* __restrict__; + + static __device__ auto get(args scope, size_type i) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + if constexpr (!IsNullable) { + return p->template element(index); + } else { + return p->template nullable_element(index); + } + } + + static __device__ void assign(args scope, size_type i, auto const& value) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + p->template assign(index, value); + } + + static __device__ auto* null_mask(args scope) + { + auto p = static_cast(scope[ScopeIndex]); + return p->null_mask(); + } + + static __device__ bool is_null(args scope, size_type i) + { + if constexpr (!IsNullable) { return false; } + + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + return p->is_null(index); + } + + static __device__ bool is_valid(args scope, size_type i) { return !is_null(scope, i); } +}; + +template +struct span { + static constexpr bool IS_SCALAR = IsScalar; + static constexpr bool IS_NULLABLE = IsNullable; + + using Type = T; + using Arg = SpanType const* __restrict__; + + static __device__ auto get(args scope, size_type i) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + if constexpr (!IsNullable) { + return p->template element(index); + } else { + return p->template nullable_element(index); + } + } + + static __device__ void assign(args scope, size_type i, auto const& value) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + p->template assign(index, value); + } + + static __device__ auto* null_mask(args scope) + { + auto p = static_cast(scope[ScopeIndex]); + return p->null_mask(); + } + + static __device__ bool is_null(args scope, size_type i) + { + if constexpr (!IsNullable) { return false; } + + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + return p->is_null(index); + } + + static __device__ bool is_valid(args scope, size_type i) { return !is_null(scope, i); } +}; + +} // namespace scope +} // namespace lto +} // namespace cudf + +// TODO: use this to document how operators can use the accessors +template +struct element_operation { + template + static __device__ void evaluate(args scope, cudf::size_type i, Operator&& op) + { + if constexpr (UserDataIndex >= 0) { + auto output_args; + GENERIC_TRANSFORM_OP(user_data, i, &res, In::element(inputs, i)...); + } else { + GENERIC_TRANSFORM_OP(&res, In::element(inputs, i)...); + } + } +}; diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index a287755c4b0f..6eca8355a5e4 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -307,6 +307,7 @@ fragment fragment_t::compile(compile_params const& params) CUDF_DEFER([&] { nvrtcDestroyProgram(&program); }); + // TODO: log is printed twice when warnings are raised CUDFRTC_CHECK_NVRTC( params, program, From dca589e7d64d8f53f8891e68bc92b5f1a9a3b188 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 11 Feb 2026 03:24:54 +0000 Subject: [PATCH 045/254] update --- cpp/src/runtime/context.cpp | 1 - cpp/src/runtime/context.hpp | 2 -- 2 files changed, 3 deletions(-) diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 05c3eb6f29b9..c48c78e6c67a 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -55,7 +55,6 @@ void context::ensure_jit_bundle_initialized() jit::program_cache& context::program_cache() { - ensure_jit_cache_initialized(); ensure_jit_cache_initialized(); return *_program_cache; } diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 26f3c1ea9d99..ac8b780c92ba 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -35,8 +35,6 @@ struct [[nodiscard]] context_config { /// It helps to ensure structured and well-defined construction and destruction of global /// objects/state across translation units. class context { - public: - public: private: context_config _config; std::once_flag _program_cache_init_flag; From def98337dfed4e7f9b69717a74d7cbcad34350cb Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 11 Feb 2026 06:43:11 +0000 Subject: [PATCH 046/254] Refactor LTO library structure and update operators --- .../cudf/column/column_device_view_base.cuh | 2 +- cpp/include/cudf/context.hpp | 4 +- cpp/include/cudf/jit/lto/column_view.cuh | 237 ++++++++++ cpp/include/cudf/jit/lto/export.cuh | 10 + cpp/include/cudf/jit/lto/filter_params.cuh | 17 - cpp/include/cudf/jit/lto/library.cuh | 8 + cpp/include/cudf/jit/lto/optional.cuh | 74 ++++ cpp/include/cudf/jit/lto/optional_span.cuh | 69 +++ cpp/include/cudf/jit/lto/scope.cuh | 86 +--- cpp/include/cudf/jit/lto/span.cuh | 40 ++ cpp/include/cudf/jit/lto/string_view.cuh | 79 ++++ cpp/include/cudf/jit/lto/thunk.cuh | 188 ++++---- cpp/include/cudf/jit/lto/transform_params.cuh | 1 + cpp/include/cudf/jit/lto/types.cuh | 408 +----------------- .../{types.inl.cuh => column_view.def.cuh} | 113 +---- cpp/src/jit/lto/library.cu | 6 +- cpp/src/jit/lto/library.inl.cuh | 9 - .../{operators.inl.cuh => operators.def.cuh} | 4 + cpp/src/jit/lto/string_view.def.cuh | 125 ++++++ cpp/src/runtime/context.cpp | 4 +- cpp/src/runtime/context.hpp | 2 - .../filter/jit/lto_kernel.inl.cuh | 32 -- cpp/tests/jit/rtc.cpp | 106 ++++- 23 files changed, 856 insertions(+), 768 deletions(-) create mode 100644 cpp/include/cudf/jit/lto/column_view.cuh create mode 100644 cpp/include/cudf/jit/lto/export.cuh delete mode 100644 cpp/include/cudf/jit/lto/filter_params.cuh create mode 100644 cpp/include/cudf/jit/lto/optional.cuh create mode 100644 cpp/include/cudf/jit/lto/optional_span.cuh create mode 100644 cpp/include/cudf/jit/lto/span.cuh create mode 100644 cpp/include/cudf/jit/lto/string_view.cuh rename cpp/src/jit/lto/{types.inl.cuh => column_view.def.cuh} (58%) delete mode 100644 cpp/src/jit/lto/library.inl.cuh rename cpp/src/jit/lto/{operators.inl.cuh => operators.def.cuh} (99%) create mode 100644 cpp/src/jit/lto/string_view.def.cuh delete mode 100644 cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh diff --git a/cpp/include/cudf/column/column_device_view_base.cuh b/cpp/include/cudf/column/column_device_view_base.cuh index 8a283351f7a1..ddc07ed8aae8 100644 --- a/cpp/include/cudf/column/column_device_view_base.cuh +++ b/cpp/include/cudf/column/column_device_view_base.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index e56862520034..62665787b023 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -25,8 +25,8 @@ enum class init_flags : std::uint32_t { /// @brief Initialize the JIT bundle during initialization INIT_JIT_BUNDLE = 1 << 3, /// @brief The default initialization steps performed by `initialize()` if no flags are provided - DEFAULT = INIT_JIT_CACHE | INIT_RTC_CACHE | INIT_JIT_BUNDLE, - /// @brief All initialization steps + DEFAULT = INIT_JIT_CACHE | INIT_RTC_CACHE | INIT_JIT_BUNDLE, + /// @brief All initialization steps ALL = LOAD_NVCOMP | INIT_JIT_CACHE | INIT_RTC_CACHE | INIT_JIT_BUNDLE }; diff --git a/cpp/include/cudf/jit/lto/column_view.cuh b/cpp/include/cudf/jit/lto/column_view.cuh new file mode 100644 index 000000000000..a225383500e7 --- /dev/null +++ b/cpp/include/cudf/jit/lto/column_view.cuh @@ -0,0 +1,237 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include +#include +#include + +namespace CUDF_LTO_EXPORT cudf { +namespace lto { + +struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { + private: + data_type _type = {}; + size_type _size = 0; + void const* _data = nullptr; + bitmask_type const* _null_mask = nullptr; + size_type _offset = 0; + column_device_view_core* _d_children = nullptr; + size_type _num_children = 0; + + public: + template + __device__ T const* head() const; + + __device__ size_type size() const; + + __device__ bool nullable() const; + + __device__ bitmask_type const* null_mask() const; + + __device__ size_type offset() const; + + __device__ bool is_valid(size_type idx) const; + + __device__ bool is_valid_nocheck(size_type idx) const; + + __device__ bool is_null(size_type idx) const; + + __device__ bool is_null_nocheck(size_type idx) const; + + __device__ bitmask_type get_mask_word(size_type word_index) const; + + template + __device__ T element(size_type idx) const; + + template + __device__ optional nullable_element(size_type idx) const; + + __device__ size_type num_child_columns() const; +}; + +struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { + private: + data_type _type = {}; + size_type _size = 0; + void const* _data = nullptr; + bitmask_type const* _null_mask = nullptr; + size_type _offset = 0; + mutable_column_device_view_core* _d_children = nullptr; + size_type _num_children = 0; + + public: + template + __device__ T* head() const; + + __device__ size_type size() const; + + __device__ bool nullable() const; + + __device__ bitmask_type* null_mask() const; + + __device__ size_type offset() const; + + __device__ bool is_valid(size_type idx) const; + + __device__ bool is_valid_nocheck(size_type idx) const; + + __device__ bool is_null(size_type idx) const; + + __device__ bool is_null_nocheck(size_type idx) const; + + __device__ bitmask_type get_mask_word(size_type word_index) const; + + template + __device__ T element(size_type idx) const; + + template + __device__ optional nullable_element(size_type idx) const; + + template + __device__ void assign(size_type idx, T value) const; +}; + +#define FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE \ + DO_IT(bool) \ + DO_IT(int8_t) \ + DO_IT(int16_t) \ + DO_IT(int32_t) \ + DO_IT(int64_t) \ + DO_IT(uint8_t) \ + DO_IT(uint16_t) \ + DO_IT(uint32_t) \ + DO_IT(uint64_t) \ + DO_IT(float32_t) \ + DO_IT(float64_t) \ + DO_IT(timestamp_D) \ + DO_IT(timestamp_h) \ + DO_IT(timestamp_m) \ + DO_IT(timestamp_s) \ + DO_IT(timestamp_ms) \ + DO_IT(timestamp_us) \ + DO_IT(timestamp_ns) \ + DO_IT(duration_D) \ + DO_IT(duration_h) \ + DO_IT(duration_m) \ + DO_IT(duration_s) \ + DO_IT(duration_ms) \ + DO_IT(duration_us) \ + DO_IT(duration_ns) + +#define FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE \ + DO_IT(bool) \ + DO_IT(int8_t) \ + DO_IT(int16_t) \ + DO_IT(int32_t) \ + DO_IT(int64_t) \ + DO_IT(uint8_t) \ + DO_IT(uint16_t) \ + DO_IT(uint32_t) \ + DO_IT(uint64_t) \ + DO_IT(decimal32) \ + DO_IT(decimal64) \ + DO_IT(decimal128) \ + DO_IT(float32_t) \ + DO_IT(float64_t) \ + DO_IT(string_view) \ + DO_IT(timestamp_D) \ + DO_IT(timestamp_h) \ + DO_IT(timestamp_m) \ + DO_IT(timestamp_s) \ + DO_IT(timestamp_ms) \ + DO_IT(timestamp_us) \ + DO_IT(timestamp_ns) \ + DO_IT(duration_D) \ + DO_IT(duration_h) \ + DO_IT(duration_m) \ + DO_IT(duration_s) \ + DO_IT(duration_ms) \ + DO_IT(duration_us) \ + DO_IT(duration_ns) + +#define FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE \ + DO_IT(bool) \ + DO_IT(int8_t) \ + DO_IT(int16_t) \ + DO_IT(int32_t) \ + DO_IT(int64_t) \ + DO_IT(uint8_t) \ + DO_IT(uint16_t) \ + DO_IT(uint32_t) \ + DO_IT(uint64_t) \ + DO_IT(decimal32) \ + DO_IT(decimal64) \ + DO_IT(decimal128) \ + DO_IT(float32_t) \ + DO_IT(float64_t) \ + DO_IT(timestamp_D) \ + DO_IT(timestamp_h) \ + DO_IT(timestamp_m) \ + DO_IT(timestamp_s) \ + DO_IT(timestamp_ms) \ + DO_IT(timestamp_us) \ + DO_IT(timestamp_ns) \ + DO_IT(duration_D) \ + DO_IT(duration_h) \ + DO_IT(duration_m) \ + DO_IT(duration_s) \ + DO_IT(duration_ms) \ + DO_IT(duration_us) \ + DO_IT(duration_ns) + +#define DO_IT(T) extern template __device__ T const* column_device_view_core::head() const; + +FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE + +#undef DO_IT + +#define DO_IT(T) \ + extern template __device__ T column_device_view_core::element(size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(T) \ + extern template __device__ optional column_device_view_core::nullable_element( \ + size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(T) extern template __device__ T* mutable_column_device_view_core::head() const; + +FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE + +#undef DO_IT + +#define DO_IT(T) \ + extern template __device__ T mutable_column_device_view_core::element(size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(T) \ + extern template __device__ optional mutable_column_device_view_core::nullable_element( \ + size_type idx) const; + +FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE + +#undef DO_IT + +#define DO_IT(T) \ + extern template __device__ void mutable_column_device_view_core::assign(size_type idx, \ + T value) const; + +FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE + +#undef DO_IT + +} // namespace lto +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/export.cuh b/cpp/include/cudf/jit/lto/export.cuh new file mode 100644 index 000000000000..b91aa226d625 --- /dev/null +++ b/cpp/include/cudf/jit/lto/export.cuh @@ -0,0 +1,10 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#define CUDF_LTO_EXPORT __attribute__((visibility("default"))) + +#define CUDF_LTO_ALIAS __attribute__((may_alias)) diff --git a/cpp/include/cudf/jit/lto/filter_params.cuh b/cpp/include/cudf/jit/lto/filter_params.cuh deleted file mode 100644 index a733314c4f11..000000000000 --- a/cpp/include/cudf/jit/lto/filter_params.cuh +++ /dev/null @@ -1,17 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include -#include - -namespace CUDF_LTO_EXPORT cudf { - -namespace lto { - -using filter_params = transform_params; - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/library.cuh b/cpp/include/cudf/jit/lto/library.cuh index ca733b905393..821048b09a46 100644 --- a/cpp/include/cudf/jit/lto/library.cuh +++ b/cpp/include/cudf/jit/lto/library.cuh @@ -3,6 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ #pragma once +#include +#include #include +#include +#include +#include +#include +#include #include +#include #include diff --git a/cpp/include/cudf/jit/lto/optional.cuh b/cpp/include/cudf/jit/lto/optional.cuh new file mode 100644 index 000000000000..33bcbc79a792 --- /dev/null +++ b/cpp/include/cudf/jit/lto/optional.cuh @@ -0,0 +1,74 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace CUDF_LTO_EXPORT cudf { +namespace lto { + +struct inplace_t {}; + +inline constexpr inplace_t inplace{}; + +// TODO: assumes T is trivially copyable +template +struct CUDF_LTO_ALIAS optional { + private: + T __val; + bool __engaged; + + public: + __device__ constexpr optional() : __val{}, __engaged{false} {} + + template + __device__ constexpr optional(inplace_t, Args&&... args) + : __val{static_cast(args)...}, __engaged{true} + { + } + + __device__ constexpr optional(T val) : __val{val}, __engaged{true} {} + + constexpr optional(optional const&) = default; + + constexpr optional(optional&&) = default; + + constexpr optional& operator=(optional const&) = default; + + constexpr optional& operator=(optional&&) = default; + + constexpr ~optional() = default; + + __device__ constexpr bool has_value() const { return __engaged; } + + __device__ constexpr void reset() { __engaged = false; } + + __device__ constexpr T const& get() const { return __val; } + + __device__ constexpr T& get() { return __val; } + + __device__ constexpr T const* operator->() const { return &__val; } + + __device__ constexpr T* operator->() { return &__val; } + + __device__ constexpr T const& operator*() const { return __val; } + + __device__ constexpr T& operator*() { return __val; } + + __device__ constexpr T const& value() const { return __val; } + + __device__ constexpr T& value() { return __val; } + + __device__ constexpr explicit operator bool() const { return __engaged; } + + __device__ constexpr T value_or(T __v) const { return __engaged ? __val : __v; } +}; + +template +optional(T) -> optional; + +} // namespace lto +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/optional_span.cuh b/cpp/include/cudf/jit/lto/optional_span.cuh new file mode 100644 index 000000000000..1dad8ea25669 --- /dev/null +++ b/cpp/include/cudf/jit/lto/optional_span.cuh @@ -0,0 +1,69 @@ + + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include +#include + +namespace CUDF_LTO_EXPORT cudf { +namespace lto { + +__device__ constexpr bool bit_is_set(bitmask_type const* bitmask, size_t bit_index) +{ + constexpr auto bits_per_word = sizeof(bitmask_type) * 8; + return bitmask[bit_index / bits_per_word] & (bitmask_type{1} << (bit_index % bits_per_word)); +} + +template +struct [[nodiscard]] optional_span { + private: + T* _data = nullptr; + size_t _size = 0; + bitmask_type* _null_mask = nullptr; + + public: + __device__ T* data() const { return _data; } + + __device__ size_t size() const { return _size; } + + __device__ bool empty() const { return _size == 0; } + + __device__ T& operator[](size_t pos) const { return _data[pos]; } + + __device__ T* begin() const { return _data; } + + __device__ T* end() const { return _data + _size; } + + __device__ optional_span as_const() const + { + return optional_span{_data, _size, _null_mask}; + } + + __device__ bool nullable() const { return _null_mask != nullptr; } + + [[nodiscard]] __device__ bool is_valid_nocheck(size_t element_index) const + { + return bit_is_set(_null_mask, element_index); + } + + __device__ bool is_valid(size_t element_index) const + { + return not nullable() or is_valid_nocheck(element_index); + } + + __device__ bool is_null(size_t element_index) const { return !is_valid(element_index); } + + __device__ T& element(size_t idx) const { return _data[idx]; } + + __device__ optional nullable_element(size_t idx) const; + + __device__ void assign(size_t idx, T value) const { _data[idx] = value; } +}; + +} // namespace lto +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/scope.cuh b/cpp/include/cudf/jit/lto/scope.cuh index 0a570f190c8d..9876faa94da4 100644 --- a/cpp/include/cudf/jit/lto/scope.cuh +++ b/cpp/include/cudf/jit/lto/scope.cuh @@ -8,15 +8,17 @@ // TODO: scope variables should be aligned to avoid uncoalesced reads/writes -namespace cudf { +namespace CUDF_LTO_EXPORT cudf { namespace lto { namespace scope { using args = void* const __restrict__* __restrict__; -template struct column { @@ -26,7 +28,7 @@ struct column { using Type = T; using Arg = ColumnType const* __restrict__; - static __device__ auto get(args scope, size_type i) + static __device__ decltype(auto) element(args scope, size_type i) { auto p = static_cast(scope[ScopeIndex]); auto index = IsScalar ? 0 : i; @@ -38,7 +40,7 @@ struct column { } } - static __device__ void assign(args scope, size_type i, auto const& value) + static __device__ void assign(args scope, size_type i, T value) { auto p = static_cast(scope[ScopeIndex]); auto index = IsScalar ? 0 : i; @@ -58,81 +60,23 @@ struct column { auto p = static_cast(scope[ScopeIndex]); auto index = IsScalar ? 0 : i; + return p->is_null(index); } static __device__ bool is_valid(args scope, size_type i) { return !is_null(scope, i); } }; -template -struct span { - static constexpr bool IS_SCALAR = IsScalar; - static constexpr bool IS_NULLABLE = IsNullable; - - using Type = T; - using Arg = SpanType const* __restrict__; - - static __device__ auto get(args scope, size_type i) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - if constexpr (!IsNullable) { - return p->template element(index); - } else { - return p->template nullable_element(index); - } - } - - static __device__ void assign(args scope, size_type i, auto const& value) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - p->template assign(index, value); - } +template +struct user_data { + using Arg = void* __restrict__; - static __device__ auto* null_mask(args scope) + static __device__ decltype(auto) element(args scope, [[maybe_unused]] size_type i) { - auto p = static_cast(scope[ScopeIndex]); - return p->null_mask(); - } - - static __device__ bool is_null(args scope, size_type i) - { - if constexpr (!IsNullable) { return false; } - - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - return p->is_null(index); + return static_cast(scope[ScopeIndex]); } - - static __device__ bool is_valid(args scope, size_type i) { return !is_null(scope, i); } }; } // namespace scope } // namespace lto -} // namespace cudf - -// TODO: use this to document how operators can use the accessors -template -struct element_operation { - template - static __device__ void evaluate(args scope, cudf::size_type i, Operator&& op) - { - if constexpr (UserDataIndex >= 0) { - auto output_args; - GENERIC_TRANSFORM_OP(user_data, i, &res, In::element(inputs, i)...); - } else { - GENERIC_TRANSFORM_OP(&res, In::element(inputs, i)...); - } - } -}; +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/span.cuh b/cpp/include/cudf/jit/lto/span.cuh new file mode 100644 index 000000000000..75912e57981b --- /dev/null +++ b/cpp/include/cudf/jit/lto/span.cuh @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include + +namespace CUDF_LTO_EXPORT cudf { +namespace lto { + +template +struct [[nodiscard]] span { + private: + T* _data = nullptr; + size_t _size = 0; + + public: + __device__ T* data() const { return _data; } + + __device__ size_t size() const { return _size; } + + __device__ bool empty() const { return _size == 0; } + + __device__ T& operator[](size_t pos) const { return _data[pos]; } + + __device__ T* begin() const { return _data; } + + __device__ T* end() const { return _data + _size; } + + __device__ span as_const() const { return span{_data, _size}; } + + __device__ T& element(size_t idx) const { return _data[idx]; } + + __device__ void assign(size_t idx, T value) const { _data[idx] = value; } +}; + +} // namespace lto +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/string_view.cuh b/cpp/include/cudf/jit/lto/string_view.cuh new file mode 100644 index 000000000000..abaca66d741d --- /dev/null +++ b/cpp/include/cudf/jit/lto/string_view.cuh @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include + +namespace CUDF_LTO_EXPORT cudf { +namespace lto { + +struct CUDF_LTO_ALIAS string_view { + private: + char const* _data = nullptr; + size_type _bytes = 0; + mutable size_type _length = 0; + + public: + [[nodiscard]] __device__ size_type size_bytes() const; + + [[nodiscard]] __device__ size_type length() const; + + [[nodiscard]] __device__ char const* data() const; + + [[nodiscard]] __device__ bool empty() const; + + [[nodiscard]] __device__ char_utf8 operator[](size_type pos) const; + + [[nodiscard]] __device__ size_type byte_offset(size_type pos) const; + + [[nodiscard]] __device__ int compare(string_view const& str) const; + + [[nodiscard]] __device__ int compare(char const* str, size_type bytes) const; + + [[nodiscard]] __device__ bool operator==(string_view const& rhs) const; + + [[nodiscard]] __device__ bool operator!=(string_view const& rhs) const; + + [[nodiscard]] __device__ bool operator<(string_view const& rhs) const; + + [[nodiscard]] __device__ bool operator>(string_view const& rhs) const; + + [[nodiscard]] __device__ bool operator<=(string_view const& rhs) const; + + [[nodiscard]] __device__ bool operator>=(string_view const& rhs) const; + + [[nodiscard]] __device__ size_type find(string_view const& str, + size_type pos = 0, + size_type count = -1) const; + + [[nodiscard]] __device__ size_type find(char const* str, + size_type bytes, + size_type pos = 0, + size_type count = -1) const; + + [[nodiscard]] __device__ size_type find(char_utf8 character, + size_type pos = 0, + size_type count = -1) const; + + [[nodiscard]] __device__ size_type rfind(string_view const& str, + size_type pos = 0, + size_type count = -1) const; + + [[nodiscard]] __device__ size_type rfind(char const* str, + size_type bytes, + size_type pos = 0, + size_type count = -1) const; + + [[nodiscard]] __device__ size_type rfind(char_utf8 character, + size_type pos = 0, + size_type count = -1) const; + + [[nodiscard]] __device__ string_view substr(size_type start, size_type length) const; + + static inline size_type const npos{-1}; +}; + +} // namespace lto +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/thunk.cuh b/cpp/include/cudf/jit/lto/thunk.cuh index 7af61eca0f39..9d3ba4ab87ae 100644 --- a/cpp/include/cudf/jit/lto/thunk.cuh +++ b/cpp/include/cudf/jit/lto/thunk.cuh @@ -29,104 +29,102 @@ struct lifted_type_of_t; template using lifted_type_of = typename lifted_type_of_t::type; -#define CUDF_LTO_MAP(lowered_type, lifted_type) \ - static_assert(sizeof(lowered_type) == sizeof(lifted_type), \ - "(1: size). Lowered and Lifted types must be bitwise-equivalent"); \ - static_assert(alignof(lowered_type) == alignof(lifted_type), \ - "(2: alignment). Lowered and Lifted types must be bitwise-equivalent"); \ - static_assert( \ - sizeof(lto::optional) == sizeof(cuda::std::optional), \ - "(1: size). Lowered and Lifted types must have bitwise-equivalent optional types"); \ - static_assert( \ - alignof(lto::optional) == alignof(cuda::std::optional), \ - "(2: alignment). Lowered and Lifted types must have bitwise-equivalent optional types"); \ - \ - template <> \ - struct lifted_type_of_t { \ - using type = lifted_type; \ - }; \ - \ - template <> \ - struct lowered_type_of_t { \ - using type = lowered_type; \ - }; \ - \ - __device__ __forceinline__ lowered_type* lower(lifted_type* p) \ - { \ - return reinterpret_cast(p); \ - } \ - \ - __device__ __forceinline__ lowered_type const* lower(lifted_type const* p) \ - { \ - return reinterpret_cast(p); \ - } \ - \ - __device__ __forceinline__ lto::optional* lower( \ - cuda::std::optional* p) \ - { \ - return reinterpret_cast*>(p); \ - } \ - \ - __device__ __forceinline__ lto::optional const* lower( \ - cuda::std::optional const* p) \ - { \ - return reinterpret_cast const*>(p); \ - } \ - \ - __device__ __forceinline__ lifted_type* lift(lowered_type* p) \ - { \ - return reinterpret_cast(p); \ - } \ - \ - __device__ __forceinline__ lifted_type const* lift(lowered_type const* p) \ - { \ - return reinterpret_cast(p); \ - } \ - \ - __device__ __forceinline__ cuda::std::optional* lift( \ - lto::optional* p) \ - { \ - return reinterpret_cast*>(p); \ - } \ - \ - __device__ __forceinline__ cuda::std::optional const* lift( \ - lto::optional const* p) \ - { \ - return reinterpret_cast const*>(p); \ +#define CUDF_LTO_MAP(lowered_type, lifted_type) \ + static_assert(sizeof(lowered_type) == sizeof(lifted_type), \ + "(1: size). Lowered and Lifted types must be bitwise-equivalent"); \ + static_assert(alignof(lowered_type) == alignof(lifted_type), \ + "(2: alignment). Lowered and Lifted types must be bitwise-equivalent"); \ + static_assert( \ + sizeof(optional) == sizeof(cuda::std::optional), \ + "(1: size). Lowered and Lifted types must have bitwise-equivalent optional types"); \ + static_assert( \ + alignof(optional) == alignof(cuda::std::optional), \ + "(2: alignment). Lowered and Lifted types must have bitwise-equivalent optional types"); \ + \ + template <> \ + struct lifted_type_of_t { \ + using type = lifted_type; \ + }; \ + \ + template <> \ + struct lowered_type_of_t { \ + using type = lowered_type; \ + }; \ + \ + __device__ __forceinline__ lowered_type* lower(lifted_type* p) \ + { \ + return reinterpret_cast(p); \ + } \ + \ + __device__ __forceinline__ lowered_type const* lower(lifted_type const* p) \ + { \ + return reinterpret_cast(p); \ + } \ + \ + __device__ __forceinline__ optional* lower(cuda::std::optional* p) \ + { \ + return reinterpret_cast*>(p); \ + } \ + \ + __device__ __forceinline__ optional const* lower( \ + cuda::std::optional const* p) \ + { \ + return reinterpret_cast const*>(p); \ + } \ + \ + __device__ __forceinline__ lifted_type* lift(lowered_type* p) \ + { \ + return reinterpret_cast(p); \ + } \ + \ + __device__ __forceinline__ lifted_type const* lift(lowered_type const* p) \ + { \ + return reinterpret_cast(p); \ + } \ + \ + __device__ __forceinline__ cuda::std::optional* lift(optional* p) \ + { \ + return reinterpret_cast*>(p); \ + } \ + \ + __device__ __forceinline__ cuda::std::optional const* lift( \ + optional const* p) \ + { \ + return reinterpret_cast const*>(p); \ } CUDF_LTO_MAP(bool, bool); -CUDF_LTO_MAP(lto::data_type, cudf::data_type); -CUDF_LTO_MAP(lto::int8_t, std::int8_t); -CUDF_LTO_MAP(lto::int16_t, std::int16_t); -CUDF_LTO_MAP(lto::int32_t, std::int32_t); -CUDF_LTO_MAP(lto::int64_t, std::int64_t); -CUDF_LTO_MAP(lto::uint8_t, std::uint8_t); -CUDF_LTO_MAP(lto::uint16_t, std::uint16_t); -CUDF_LTO_MAP(lto::uint32_t, std::uint32_t); -CUDF_LTO_MAP(lto::uint64_t, std::uint64_t); -CUDF_LTO_MAP(lto::float32_t, float); -CUDF_LTO_MAP(lto::float64_t, double); -CUDF_LTO_MAP(lto::decimal32, numeric::decimal32); -CUDF_LTO_MAP(lto::decimal64, numeric::decimal64); -CUDF_LTO_MAP(lto::decimal128, numeric::decimal128); -CUDF_LTO_MAP(lto::string_view, cudf::string_view); -CUDF_LTO_MAP(lto::timestamp_D, cudf::timestamp_D); -CUDF_LTO_MAP(lto::timestamp_h, cudf::timestamp_h); -CUDF_LTO_MAP(lto::timestamp_m, cudf::timestamp_m); -CUDF_LTO_MAP(lto::timestamp_s, cudf::timestamp_s); -CUDF_LTO_MAP(lto::timestamp_ms, cudf::timestamp_ms); -CUDF_LTO_MAP(lto::timestamp_us, cudf::timestamp_us); -CUDF_LTO_MAP(lto::timestamp_ns, cudf::timestamp_ns); -CUDF_LTO_MAP(lto::duration_D, cudf::duration_D); -CUDF_LTO_MAP(lto::duration_h, cudf::duration_h); -CUDF_LTO_MAP(lto::duration_m, cudf::duration_m); -CUDF_LTO_MAP(lto::duration_s, cudf::duration_s); -CUDF_LTO_MAP(lto::duration_ms, cudf::duration_ms); -CUDF_LTO_MAP(lto::duration_us, cudf::duration_us); -CUDF_LTO_MAP(lto::duration_ns, cudf::duration_ns); -CUDF_LTO_MAP(lto::column_device_view_core, cudf::column_device_view_core); -CUDF_LTO_MAP(lto::mutable_column_device_view_core, cudf::mutable_column_device_view_core); +CUDF_LTO_MAP(data_type, cudf::data_type); +CUDF_LTO_MAP(int8_t, std::int8_t); +CUDF_LTO_MAP(int16_t, std::int16_t); +CUDF_LTO_MAP(int32_t, std::int32_t); +CUDF_LTO_MAP(int64_t, std::int64_t); +CUDF_LTO_MAP(uint8_t, std::uint8_t); +CUDF_LTO_MAP(uint16_t, std::uint16_t); +CUDF_LTO_MAP(uint32_t, std::uint32_t); +CUDF_LTO_MAP(uint64_t, std::uint64_t); +CUDF_LTO_MAP(float32_t, float); +CUDF_LTO_MAP(float64_t, double); +CUDF_LTO_MAP(decimal32, numeric::decimal32); +CUDF_LTO_MAP(decimal64, numeric::decimal64); +CUDF_LTO_MAP(decimal128, numeric::decimal128); +CUDF_LTO_MAP(string_view, cudf::string_view); +CUDF_LTO_MAP(timestamp_D, cudf::timestamp_D); +CUDF_LTO_MAP(timestamp_h, cudf::timestamp_h); +CUDF_LTO_MAP(timestamp_m, cudf::timestamp_m); +CUDF_LTO_MAP(timestamp_s, cudf::timestamp_s); +CUDF_LTO_MAP(timestamp_ms, cudf::timestamp_ms); +CUDF_LTO_MAP(timestamp_us, cudf::timestamp_us); +CUDF_LTO_MAP(timestamp_ns, cudf::timestamp_ns); +CUDF_LTO_MAP(duration_D, cudf::duration_D); +CUDF_LTO_MAP(duration_h, cudf::duration_h); +CUDF_LTO_MAP(duration_m, cudf::duration_m); +CUDF_LTO_MAP(duration_s, cudf::duration_s); +CUDF_LTO_MAP(duration_ms, cudf::duration_ms); +CUDF_LTO_MAP(duration_us, cudf::duration_us); +CUDF_LTO_MAP(duration_ns, cudf::duration_ns); +CUDF_LTO_MAP(column_device_view_core, cudf::column_device_view_core); +CUDF_LTO_MAP(mutable_column_device_view_core, cudf::mutable_column_device_view_core); #undef CUDF_LTO_MAP diff --git a/cpp/include/cudf/jit/lto/transform_params.cuh b/cpp/include/cudf/jit/lto/transform_params.cuh index 38ded95bc695..4decb57be7b0 100644 --- a/cpp/include/cudf/jit/lto/transform_params.cuh +++ b/cpp/include/cudf/jit/lto/transform_params.cuh @@ -14,6 +14,7 @@ namespace lto { struct [[nodiscard]] transform_params { void* __restrict__ const* __restrict__ scope = nullptr; ///< Pointer to scope data (e.g. column views, scalars, etc.). + size_type num_rows = 0; ///< Total number of rows to process. size_type row_index = 0; ///< Current row index. }; diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index 0bb5561a2614..69cb1d13734b 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -4,8 +4,7 @@ */ #pragma once -#define CUDF_LTO_EXPORT __attribute__((visibility("default"))) -#define CUDF_LTO_ALIAS __attribute__((may_alias)) +#include namespace CUDF_LTO_EXPORT cudf { @@ -68,72 +67,6 @@ struct CUDF_LTO_ALIAS data_type { int32_t _fixed_point_scale = 0; }; -struct CUDF_LTO_ALIAS string_view { - private: - char const* _data = nullptr; - size_type _bytes = 0; - mutable size_type _length = 0; - - public: - [[nodiscard]] __device__ size_type size_bytes() const; - - [[nodiscard]] __device__ size_type length() const; - - [[nodiscard]] __device__ char const* data() const; - - [[nodiscard]] __device__ bool empty() const; - - [[nodiscard]] __device__ char_utf8 operator[](size_type pos) const; - - [[nodiscard]] __device__ size_type byte_offset(size_type pos) const; - - [[nodiscard]] __device__ int compare(string_view const& str) const; - - [[nodiscard]] __device__ int compare(char const* str, size_type bytes) const; - - [[nodiscard]] __device__ bool operator==(string_view const& rhs) const; - - [[nodiscard]] __device__ bool operator!=(string_view const& rhs) const; - - [[nodiscard]] __device__ bool operator<(string_view const& rhs) const; - - [[nodiscard]] __device__ bool operator>(string_view const& rhs) const; - - [[nodiscard]] __device__ bool operator<=(string_view const& rhs) const; - - [[nodiscard]] __device__ bool operator>=(string_view const& rhs) const; - - [[nodiscard]] __device__ size_type find(string_view const& str, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ size_type find(char const* str, - size_type bytes, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ size_type find(char_utf8 character, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ size_type rfind(string_view const& str, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ size_type rfind(char const* str, - size_type bytes, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ size_type rfind(char_utf8 character, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ string_view substr(size_type start, size_type length) const; - - static inline size_type const npos{-1}; -}; - struct CUDF_LTO_ALIAS decimal32 { private: int32_t _value = 0; @@ -222,345 +155,14 @@ struct CUDF_LTO_ALIAS duration_ns { int64_t _rep = 0; }; -struct inplace_t {}; - -inline constexpr inplace_t inplace{}; - -// TODO: assumes T is trivially copyable -template -struct CUDF_LTO_ALIAS optional { - private: - T __val; - bool __engaged; - - public: - __device__ constexpr optional() : __val{}, __engaged{false} {} - - template - __device__ constexpr optional(inplace_t, Args&&... args) - : __val{static_cast(args)...}, __engaged{true} - { - } - - __device__ constexpr optional(T val) : __val{val}, __engaged{true} {} - - constexpr optional(optional const&) = default; - - constexpr optional(optional&&) = default; - - constexpr optional& operator=(optional const&) = default; - - constexpr optional& operator=(optional&&) = default; - - constexpr ~optional() = default; - - __device__ constexpr bool has_value() const { return __engaged; } - - __device__ constexpr void reset() { __engaged = false; } - - __device__ constexpr T const& get() const { return __val; } - - __device__ constexpr T& get() { return __val; } - - __device__ constexpr T const* operator->() const { return &__val; } - - __device__ constexpr T* operator->() { return &__val; } - - __device__ constexpr T const& operator*() const { return __val; } - - __device__ constexpr T& operator*() { return __val; } - - __device__ constexpr T const& value() const { return __val; } - - __device__ constexpr T& value() { return __val; } - - __device__ constexpr explicit operator bool() const { return __engaged; } - - __device__ constexpr T value_or(T __v) const { return __engaged ? __val : __v; } -}; - template -optional(T) -> optional; - -template -struct [[nodiscard]] device_span { - private: - T* _data = nullptr; - size_t _size = 0; -}; - -__device__ constexpr bool bit_is_set(bitmask_type const* bitmask, size_t bit_index) -{ - constexpr auto bits_per_word = sizeof(bitmask_type) * 8; - return bitmask[bit_index / bits_per_word] & (bitmask_type{1} << (bit_index % bits_per_word)); -} - -template -struct [[nodiscard]] device_optional_span { - private: - T* _data = nullptr; - size_t _size = 0; - bitmask_type* _null_mask = nullptr; - - public: - __device__ T* data() const { return _data; } - - __device__ size_t size() const { return _size; } - - __device__ bool empty() const { return _size == 0; } - - __device__ T& operator[](size_t pos) const { return _data[pos]; } - - __device__ T* begin() const { return _data; } - - __device__ T* end() const { return _data + _size; } - - __device__ device_optional_span as_const() const - { - return device_optional_span{_data, _size, _null_mask}; - } - - __device__ bool nullable() const { return _null_mask != nullptr; } - - [[nodiscard]] __device__ bool is_valid_nocheck(size_t element_index) const - { - return bit_is_set(_null_mask, element_index); - } - - __device__ bool is_valid(size_t element_index) const - { - return not nullable() or is_valid_nocheck(element_index); - } - - __device__ bool is_null(size_t element_index) const { return !is_valid(element_index); } - - __device__ T& element(size_t idx) const { return _data[idx]; } - - __device__ optional nullable_element(size_t idx) const; - - __device__ void assign(size_t idx, T value) const { _data[idx] = value; } -}; - -#define FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE \ - DO_IT(bool) \ - DO_IT(int8_t) \ - DO_IT(int16_t) \ - DO_IT(int32_t) \ - DO_IT(int64_t) \ - DO_IT(uint8_t) \ - DO_IT(uint16_t) \ - DO_IT(uint32_t) \ - DO_IT(uint64_t) \ - DO_IT(float32_t) \ - DO_IT(float64_t) \ - DO_IT(timestamp_D) \ - DO_IT(timestamp_h) \ - DO_IT(timestamp_m) \ - DO_IT(timestamp_s) \ - DO_IT(timestamp_ms) \ - DO_IT(timestamp_us) \ - DO_IT(timestamp_ns) \ - DO_IT(duration_D) \ - DO_IT(duration_h) \ - DO_IT(duration_m) \ - DO_IT(duration_s) \ - DO_IT(duration_ms) \ - DO_IT(duration_us) \ - DO_IT(duration_ns) - -#define FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE \ - DO_IT(bool) \ - DO_IT(int8_t) \ - DO_IT(int16_t) \ - DO_IT(int32_t) \ - DO_IT(int64_t) \ - DO_IT(uint8_t) \ - DO_IT(uint16_t) \ - DO_IT(uint32_t) \ - DO_IT(uint64_t) \ - DO_IT(decimal32) \ - DO_IT(decimal64) \ - DO_IT(decimal128) \ - DO_IT(float32_t) \ - DO_IT(float64_t) \ - DO_IT(string_view) \ - DO_IT(timestamp_D) \ - DO_IT(timestamp_h) \ - DO_IT(timestamp_m) \ - DO_IT(timestamp_s) \ - DO_IT(timestamp_ms) \ - DO_IT(timestamp_us) \ - DO_IT(timestamp_ns) \ - DO_IT(duration_D) \ - DO_IT(duration_h) \ - DO_IT(duration_m) \ - DO_IT(duration_s) \ - DO_IT(duration_ms) \ - DO_IT(duration_us) \ - DO_IT(duration_ns) - -#define FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE \ - DO_IT(bool) \ - DO_IT(int8_t) \ - DO_IT(int16_t) \ - DO_IT(int32_t) \ - DO_IT(int64_t) \ - DO_IT(uint8_t) \ - DO_IT(uint16_t) \ - DO_IT(uint32_t) \ - DO_IT(uint64_t) \ - DO_IT(decimal32) \ - DO_IT(decimal64) \ - DO_IT(decimal128) \ - DO_IT(float32_t) \ - DO_IT(float64_t) \ - DO_IT(timestamp_D) \ - DO_IT(timestamp_h) \ - DO_IT(timestamp_m) \ - DO_IT(timestamp_s) \ - DO_IT(timestamp_ms) \ - DO_IT(timestamp_us) \ - DO_IT(timestamp_ns) \ - DO_IT(duration_D) \ - DO_IT(duration_h) \ - DO_IT(duration_m) \ - DO_IT(duration_s) \ - DO_IT(duration_ms) \ - DO_IT(duration_us) \ - DO_IT(duration_ns) - -struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { - private: - data_type _type = {}; - size_type _size = 0; - void const* _data = nullptr; - bitmask_type const* _null_mask = nullptr; - size_type _offset = 0; - column_device_view_core* _d_children = nullptr; - size_type _num_children = 0; - - public: - template - __device__ T const* head() const; - - __device__ size_type size() const; - - __device__ bool nullable() const; - - __device__ bitmask_type const* null_mask() const; - - __device__ size_type offset() const; - - __device__ bool is_valid(size_type idx) const; - - __device__ bool is_valid_nocheck(size_type idx) const; - - __device__ bool is_null(size_type idx) const; - - __device__ bool is_null_nocheck(size_type idx) const; - - __device__ bitmask_type get_mask_word(size_type word_index) const; - - template - __device__ T element(size_type idx) const; - - template - __device__ optional nullable_element(size_type idx) const; - - __device__ size_type num_child_columns() const; -}; - -struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { - private: - data_type _type = {}; - size_type _size = 0; - void const* _data = nullptr; - bitmask_type const* _null_mask = nullptr; - size_type _offset = 0; - mutable_column_device_view_core* _d_children = nullptr; - size_type _num_children = 0; - - public: - template - __device__ T* head() const; - - __device__ size_type size() const; - - __device__ bool nullable() const; - - __device__ bitmask_type* null_mask() const; - - __device__ size_type offset() const; - - __device__ bool is_valid(size_type idx) const; - - __device__ bool is_valid_nocheck(size_type idx) const; - - __device__ bool is_null(size_type idx) const; - - __device__ bool is_null_nocheck(size_type idx) const; - - __device__ bitmask_type get_mask_word(size_type word_index) const; - - template - __device__ T element(size_type idx) const; - - template - __device__ optional nullable_element(size_type idx) const; - - template - __device__ void assign(size_type idx, T value) const; -}; - -#define DO_IT(T) extern template __device__ T const* column_device_view_core::head() const; - -FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - extern template __device__ T column_device_view_core::element(size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - extern template __device__ optional column_device_view_core::nullable_element( \ - size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) extern template __device__ T* mutable_column_device_view_core::head() const; - -FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - extern template __device__ T mutable_column_device_view_core::element(size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - extern template __device__ optional mutable_column_device_view_core::nullable_element( \ - size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT +struct CUDF_LTO_ALIAS optional; -#define DO_IT(T) \ - extern template __device__ void mutable_column_device_view_core::assign(size_type idx, \ - T value) const; +struct CUDF_LTO_ALIAS string_view; -FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE +struct CUDF_LTO_ALIAS column_device_view_core; -#undef DO_IT +struct CUDF_LTO_ALIAS mutable_column_device_view_core; } // namespace lto } // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/jit/lto/types.inl.cuh b/cpp/src/jit/lto/column_view.def.cuh similarity index 58% rename from cpp/src/jit/lto/types.inl.cuh rename to cpp/src/jit/lto/column_view.def.cuh index 83a7474870df..bd4df0b3b3a5 100644 --- a/cpp/src/jit/lto/types.inl.cuh +++ b/cpp/src/jit/lto/column_view.def.cuh @@ -4,123 +4,12 @@ */ #pragma once +#include #include -#include namespace CUDF_LTO_EXPORT cudf { namespace lto { -[[nodiscard]] __device__ size_type string_view::size_bytes() const -{ - return lift(this)->size_bytes(); -} - -[[nodiscard]] __device__ size_type string_view::length() const { return lift(this)->length(); } - -[[nodiscard]] __device__ char const* string_view::data() const { return lift(this)->data(); } - -[[nodiscard]] __device__ bool string_view::empty() const { return lift(this)->empty(); } - -__device__ char_utf8 string_view::operator[](size_type pos) const -{ - return lift(this)->operator[](pos); -} - -[[nodiscard]] __device__ size_type string_view::byte_offset(size_type pos) const -{ - return lift(this)->byte_offset(pos); -} - -[[nodiscard]] __device__ int string_view::compare(string_view const& str) const -{ - return lift(this)->compare(*lift(&str)); -} - -__device__ int string_view::compare(char const* str, size_type bytes) const -{ - return lift(this)->compare(str, bytes); -} - -__device__ bool string_view::operator==(string_view const& rhs) const -{ - return lift(this)->operator==(*lift(&rhs)); -} - -__device__ bool string_view::operator!=(string_view const& rhs) const -{ - return lift(this)->operator!=(*lift(&rhs)); -} - -__device__ bool string_view::operator<(string_view const& rhs) const -{ - return lift(this)->operator<(*lift(&rhs)); -} - -__device__ bool string_view::operator>(string_view const& rhs) const -{ - return lift(this)->operator>(*lift(&rhs)); -} - -__device__ bool string_view::operator<=(string_view const& rhs) const -{ - return lift(this)->operator<=(*lift(&rhs)); -} - -__device__ bool string_view::operator>=(string_view const& rhs) const -{ - return lift(this)->operator>=(*lift(&rhs)); -} - -[[nodiscard]] __device__ size_type string_view::find(string_view const& str, - size_type pos, - size_type count) const -{ - return lift(this)->find(*lift(&str), pos, count); -} - -__device__ size_type string_view::find(char const* str, - size_type bytes, - size_type pos, - size_type count) const -{ - return lift(this)->find(str, bytes, pos, count); -} - -[[nodiscard]] __device__ size_type string_view::find(char_utf8 character, - size_type pos, - size_type count) const -{ - return lift(this)->find(character, pos, count); -} - -[[nodiscard]] __device__ size_type string_view::rfind(string_view const& str, - size_type pos, - size_type count) const -{ - return lift(this)->rfind(*lift(&str), pos, count); -} - -__device__ size_type string_view::rfind(char const* str, - size_type bytes, - size_type pos, - size_type count) const -{ - return lift(this)->rfind(str, bytes, pos, count); -} - -[[nodiscard]] __device__ size_type string_view::rfind(char_utf8 character, - size_type pos, - size_type count) const -{ - return lift(this)->rfind(character, pos, count); -} - -[[nodiscard]] __device__ string_view string_view::substr(size_type start, size_type length) const -{ - auto ret = lift(this)->substr(start, length); - return *lower(&ret); -} - template __device__ T const* column_device_view_core::head() const { diff --git a/cpp/src/jit/lto/library.cu b/cpp/src/jit/lto/library.cu index 7865c2acbad3..3776502185b6 100644 --- a/cpp/src/jit/lto/library.cu +++ b/cpp/src/jit/lto/library.cu @@ -2,4 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -#include + +#include +#include +#include +#include diff --git a/cpp/src/jit/lto/library.inl.cuh b/cpp/src/jit/lto/library.inl.cuh deleted file mode 100644 index 693461a06510..000000000000 --- a/cpp/src/jit/lto/library.inl.cuh +++ /dev/null @@ -1,9 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include diff --git a/cpp/src/jit/lto/operators.inl.cuh b/cpp/src/jit/lto/operators.def.cuh similarity index 99% rename from cpp/src/jit/lto/operators.inl.cuh rename to cpp/src/jit/lto/operators.def.cuh index e0ae566f6ff8..dbaf95ec5141 100644 --- a/cpp/src/jit/lto/operators.inl.cuh +++ b/cpp/src/jit/lto/operators.def.cuh @@ -537,6 +537,10 @@ CUDF_LTO_DEF(is_null, IS_NULL, string_view); #undef CUDF_LTO_DEF +// TODO: lamarrr, remove ABI-lock to cuda::std::optional and other types. +// CUDF should probably have its own set of types used in kernels for columns. +// JIT should have its own type set + #define CUDF_LTO_DEF(op, OP, type) \ __device__ void operators::op(bool* out, type const* a) \ { \ diff --git a/cpp/src/jit/lto/string_view.def.cuh b/cpp/src/jit/lto/string_view.def.cuh new file mode 100644 index 000000000000..957eb6e07b96 --- /dev/null +++ b/cpp/src/jit/lto/string_view.def.cuh @@ -0,0 +1,125 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include + +namespace CUDF_LTO_EXPORT cudf { +namespace lto { + +[[nodiscard]] __device__ size_type string_view::size_bytes() const +{ + return lift(this)->size_bytes(); +} + +[[nodiscard]] __device__ size_type string_view::length() const { return lift(this)->length(); } + +[[nodiscard]] __device__ char const* string_view::data() const { return lift(this)->data(); } + +[[nodiscard]] __device__ bool string_view::empty() const { return lift(this)->empty(); } + +__device__ char_utf8 string_view::operator[](size_type pos) const +{ + return lift(this)->operator[](pos); +} + +[[nodiscard]] __device__ size_type string_view::byte_offset(size_type pos) const +{ + return lift(this)->byte_offset(pos); +} + +[[nodiscard]] __device__ int string_view::compare(string_view const& str) const +{ + return lift(this)->compare(*lift(&str)); +} + +__device__ int string_view::compare(char const* str, size_type bytes) const +{ + return lift(this)->compare(str, bytes); +} + +__device__ bool string_view::operator==(string_view const& rhs) const +{ + return lift(this)->operator==(*lift(&rhs)); +} + +__device__ bool string_view::operator!=(string_view const& rhs) const +{ + return lift(this)->operator!=(*lift(&rhs)); +} + +__device__ bool string_view::operator<(string_view const& rhs) const +{ + return lift(this)->operator<(*lift(&rhs)); +} + +__device__ bool string_view::operator>(string_view const& rhs) const +{ + return lift(this)->operator>(*lift(&rhs)); +} + +__device__ bool string_view::operator<=(string_view const& rhs) const +{ + return lift(this)->operator<=(*lift(&rhs)); +} + +__device__ bool string_view::operator>=(string_view const& rhs) const +{ + return lift(this)->operator>=(*lift(&rhs)); +} + +[[nodiscard]] __device__ size_type string_view::find(string_view const& str, + size_type pos, + size_type count) const +{ + return lift(this)->find(*lift(&str), pos, count); +} + +__device__ size_type string_view::find(char const* str, + size_type bytes, + size_type pos, + size_type count) const +{ + return lift(this)->find(str, bytes, pos, count); +} + +[[nodiscard]] __device__ size_type string_view::find(char_utf8 character, + size_type pos, + size_type count) const +{ + return lift(this)->find(character, pos, count); +} + +[[nodiscard]] __device__ size_type string_view::rfind(string_view const& str, + size_type pos, + size_type count) const +{ + return lift(this)->rfind(*lift(&str), pos, count); +} + +__device__ size_type string_view::rfind(char const* str, + size_type bytes, + size_type pos, + size_type count) const +{ + return lift(this)->rfind(str, bytes, pos, count); +} + +[[nodiscard]] __device__ size_type string_view::rfind(char_utf8 character, + size_type pos, + size_type count) const +{ + return lift(this)->rfind(character, pos, count); +} + +[[nodiscard]] __device__ string_view string_view::substr(size_type start, size_type length) const +{ + auto ret = lift(this)->substr(start, length); + return *lower(&ret); +} + +} // namespace lto +} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index c48c78e6c67a..c83770cc1b65 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 */ #include "runtime/context.hpp" diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index ac8b780c92ba..a1c90c6e55cd 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -1,5 +1,4 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -11,7 +10,6 @@ #include #include -#include namespace cudf { diff --git a/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh b/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh deleted file mode 100644 index 26d82f7c4bc1..000000000000 --- a/cpp/src/stream_compaction/filter/jit/lto_kernel.inl.cuh +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include -#include -#include - -extern "C" { - -__device__ void filter_operator(cudf::lto::filter_params const* params); - -__global__ void filter_kernel(void const* outputs, - void const* inputs, - void* user_data, - cudf::size_type num_rows) -{ - auto const start = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); - auto const size = num_rows; - - for (auto i = start; i < size; i += stride) { - cudf::lto::filter_params p{.inputs = inputs, - .user_data = user_data, - .outputs = outputs, - .row_index = static_cast(i)}; - filter_operator(&p); - } -} -} diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index 717fca99be03..cbc242ee568c 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -19,36 +19,100 @@ using namespace cudf; struct RTCTest : public ::testing::Test {}; +// TODO: use this to document how operators can use the accessors +/* +template +struct element_operation { + template + static __device__ void evaluate(args scope, cudf::size_type i, Operator&& op) + { + if constexpr (UserDataIndex >= 0) { + auto output_args; + GENERIC_TRANSFORM_OP(user_data, i, &res, In::element(inputs, i)...); + } else { + GENERIC_TRANSFORM_OP(&res, In::element(inputs, i)...); + } + } +}; +*/ + TEST_F(RTCTest, CreateFragment) { - // TODO: add configuration parameters necessary for testing // and profiling cache behaviour + // TODO: declare getters and setters required for the specific LTO context of the operator + // they should use the provided LTO functions; might need a planner + auto fn = []() { auto lib = rtc::compile_and_link_udf("test_fragment", R"***( #include "cudf/jit/lto/transform_params.cuh" #include "cudf/jit/lto/operators.cuh" + #include "cudf/jit/lto/scope.cuh" + #include "cudf/jit/lto/column_view.cuh" // for column_view_core, mutable_column_view_core - // TODO: declare getters and setters required for the specific LTO context of the operator - // they should use the provided LTO functions; might need a planner + // if we detect that all types are simple types + // we can exclude some of the getters, setters, and operators + // for example, we have span, optional_span, and column_view_core extern "C" __device__ void transform_operator(cudf::lto::transform_params p){ - int a = 1; - int b = 2; - int c = 3; - double d; - int * out = (int *)p.scope[0]; - cudf::lto::operators::add(&c, &a, &b); - cudf::lto::operators::sub(&c, &a, &b); - cudf::lto::operators::mul(&c, &a, &b); - cudf::lto::operators::mul(&c, &a, &b); - cudf::lto::operators::cast_to_float64(&d, &c); - c = (int)a; - c = c * c; - *out = a + b * c; - } + using namespace cudf::lto; + + /// <-- BEGIN OF INPUT UNPACKING: Defined by input planner + + // unpack inputs from scope using the appropriate getters based on the LTO context + using col_user_data = scope::user_data<0>; + using col0 = scope::column<1, column_view_core, int, false, false>; + using col1 = scope::column<2, column_view_core, int, false, false>; + using col2 = scope::column<3, column_view_core, double, false, false>; + using col3 = scope::column<4, column_view_core, float, false, false>; + using col4 = scope::column<5, span, float, false, false>; + using col5 = scope::column<6, optional_span, float, false, true>; + using out_col = scope::column<7, mutable_column_view_core, double, false, false>; + + auto user_data = col_user_data::element(p.scope, p.row_index); + auto in0 = col0::element(p.scope, p.row_index); + auto in1 = col1::element(p.scope, p.row_index); + auto in2 = col2::element(p.scope, p.row_index); + auto in3 = col3::element(p.scope, p.row_index); + auto in4 = col4::element(p.scope, p.row_index); + auto in5 = col5::element(p.scope, p.row_index); + + auto result = 0.0; + + /// <-- END OF INPUT UNPACKING + + /// <-- BEGIN OF OPERATOR: Derived from user + + // run operation using the LTO-compiled operators; these should be inlined into the final kernel and optimized together by NVJITLink + + operators::add(&c, &a, &b); + operators::sub(&c, &a, &b); + operators::mul(&c, &a, &b); + operators::mul(&c, &a, &b); + operators::arctan(&c, &a, &b); + operators::sqrt(&c, &a, &b); + operators::cbrt(&c, &a, &b); + operators::arccos(&c, &a, &b); + operators::cast_to_float64(&d, &c); + + /// <-- END OF USER-DEFINED OPERATOR + + + /// <-- BEGIN OF OUTPUT PACKING: Defined by planner + + // write output to global memory + + out_col::assign(p.scope, p.row_index, result); + + + /// <-- END OF OUTPUT PACKING + } )***", "test_udf_key", "transform_kernel"); @@ -56,10 +120,10 @@ TEST_F(RTCTest, CreateFragment) auto kernel = lib->get_kernel("transform_kernel"); EXPECT_EQ("transform_kernel", kernel.get_name()); - void** scope = nullptr; - int32_t num_rows = 0; - void* args[] = {&scope, &num_rows}; - kernel.launch(1, 1, 1, 1, 1, 1, 0, nullptr, args); + // void** scope = nullptr; + // int32_t num_rows = 0; + // void* args[] = {&scope, &num_rows}; + // kernel.launch(1, 1, 1, 1, 1, 1, 0, nullptr, args); }; fn(); From 22eef222c911eee437762a1c48649e023b6ea564 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 11 Feb 2026 06:46:12 +0000 Subject: [PATCH 047/254] Reformat comments and code layout in transform_params structure --- cpp/include/cudf/jit/lto/transform_params.cuh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/include/cudf/jit/lto/transform_params.cuh b/cpp/include/cudf/jit/lto/transform_params.cuh index 4decb57be7b0..9e1789e05865 100644 --- a/cpp/include/cudf/jit/lto/transform_params.cuh +++ b/cpp/include/cudf/jit/lto/transform_params.cuh @@ -12,10 +12,14 @@ namespace lto { /// @brief Type-erased parameters for LTO-JIT-compiled transform operations. struct [[nodiscard]] transform_params { - void* __restrict__ const* __restrict__ scope = - nullptr; ///< Pointer to scope data (e.g. column views, scalars, etc.). - size_type num_rows = 0; ///< Total number of rows to process. - size_type row_index = 0; ///< Current row index. + /// @brief Pointer to scope data (e.g. column views, scalars, etc.). + void* __restrict__ const* __restrict__ scope = nullptr; + + /// @brief Total number of rows to process. + size_type num_rows = 0; + + /// @brief Current row index. + size_type row_index = 0; }; } // namespace lto From 613eb3798c0382781c48d79889234fbc83519332 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 11 Feb 2026 08:46:43 +0000 Subject: [PATCH 048/254] changed symbol visibility --- cpp/CMakeLists.txt | 4 +++- cpp/src/jit/rtc/cache.cpp | 4 ++-- cpp/src/jit/rtc/cache.hpp | 4 ++-- cpp/src/jit/rtc/cudf.cpp | 2 +- cpp/src/jit/rtc/rtc.cpp | 2 +- cpp/src/jit/rtc/rtc.hpp | 3 ++- cpp/tests/jit/rtc.cpp | 13 ++++++++++--- 7 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 22ecd30c099e..8b1bb5816cb5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1073,7 +1073,9 @@ target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_A # Compile stringified JIT sources first add_dependencies(cudf jitify_preprocess_run) -# Specify the target module library dependencies TODO: use runtime nvrtc and nvjitlink and driver +# Specify the target module library dependencies + +# TODO: use runtime nvrtc and nvjitlink and driver target_link_libraries( cudf PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp index 462f3357d9da..1d82292b1629 100644 --- a/cpp/src/jit/rtc/cache.cpp +++ b/cpp/src/jit/rtc/cache.cpp @@ -20,7 +20,7 @@ #include #include -namespace cudf { +namespace CUDF_EXPORT cudf { namespace rtc { namespace { @@ -446,4 +446,4 @@ void cache_t::clear_disk_store() } } // namespace rtc -} // namespace cudf +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index 276880ae0eac..b164d2d1347c 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -15,7 +15,7 @@ #include #include -namespace cudf { +namespace CUDF_EXPORT cudf { namespace rtc { struct [[nodiscard]] cache_statistics { @@ -266,4 +266,4 @@ struct cache_t { }; } // namespace rtc -} // namespace cudf +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/rtc/cudf.cpp index 9ebabc8f491b..79b0112242c7 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/rtc/cudf.cpp @@ -41,7 +41,7 @@ } \ } while (0) -namespace cudf { +namespace CUDF_EXPORT cudf { namespace rtc { namespace { diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 6eca8355a5e4..f3069f0154d5 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -64,7 +64,7 @@ } \ } while (0) -namespace cudf { +namespace CUDF_EXPORT cudf { namespace rtc { namespace { diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 7e56b8c9a2c0..abebabde036a 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -10,6 +10,7 @@ #include #include #include +#include extern "C" { typedef struct CUlib_st* CUlibrary; @@ -17,7 +18,7 @@ typedef struct CUkern_st* CUkernel; typedef struct CUstream_st* CUstream; } -namespace cudf { +namespace CUDF_EXPORT cudf { namespace rtc { enum class binary_type : int8_t { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index cbc242ee568c..9f6f7811fb4e 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -8,9 +8,6 @@ #include #include - -#include - #include #include @@ -40,6 +37,16 @@ struct element_operation { }; */ + // TODO: take optional pointer to this for it to be written to + // TODO: take optional list of extra compile flags for linking and compiling +struct jit_statistics{ + std::chrono::nanoseconds cpp_compile_time{}; + std::chrono::nanoseconds fragment_link_time{}; + std::chrono::nanoseconds total_time{}; + // TODO: cache hit statistics + // global cache statistics before and after? +}; + TEST_F(RTCTest, CreateFragment) { // TODO: add configuration parameters necessary for testing From 1d53fc836da2de83666b16c846d0135497a35dd1 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 11 Feb 2026 08:48:38 +0000 Subject: [PATCH 049/254] Rename struct `jit_statistics` to `jit_compilation_stats` for clarity --- cpp/tests/jit/rtc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index 9f6f7811fb4e..442cae985848 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -39,7 +39,7 @@ struct element_operation { // TODO: take optional pointer to this for it to be written to // TODO: take optional list of extra compile flags for linking and compiling -struct jit_statistics{ +struct jit_compilation_stats{ std::chrono::nanoseconds cpp_compile_time{}; std::chrono::nanoseconds fragment_link_time{}; std::chrono::nanoseconds total_time{}; From 112ae322c4b7202732999f2887ded9c18d3798fe Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 11 Feb 2026 08:53:34 +0000 Subject: [PATCH 050/254] cmake dependency fix --- cpp/cmake/Modules/JitEmbed.cmake | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/Modules/JitEmbed.cmake b/cpp/cmake/Modules/JitEmbed.cmake index bed1e0a47459..d80aecb51ba1 100644 --- a/cpp/cmake/Modules/JitEmbed.cmake +++ b/cpp/cmake/Modules/JitEmbed.cmake @@ -133,10 +133,23 @@ function(jit_add_blob) set(blob_files ${jitembed_${TARGET}_blob__files}) set(blob_dests ${jitembed_${TARGET}_blob__dests}) + set(blob_target_deps ${jitembed_${TARGET}_blob__target_deps}) list(APPEND blob_files "${ARG_FILE}") list(APPEND blob_dests "${ARG_DEST}") + # If FILE is a $ generator expression, record the target + # name so jit_embed() can add an explicit ordering dependency via + # add_dependencies(). A plain file dependency on the evaluated paths is not + # sufficient to guarantee build ordering in parallel builds. + if(ARG_FILE MATCHES "\\$]+)>") + list(APPEND blob_target_deps "${CMAKE_MATCH_1}") + set(jitembed_${TARGET}_blob__target_deps + ${blob_target_deps} + PARENT_SCOPE + ) + endif() + set(jitembed_${TARGET}_blob__files ${blob_files} PARENT_SCOPE @@ -268,7 +281,15 @@ function(jit_embed) VERBATIM ) - add_custom_target(${TARGET} ALL DEPENDS ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s) + add_custom_target(${TARGET} ALL DEPENDS ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s + ${OUTPUT_DIR}/embed.bin) + + # Ensure any CMake targets whose objects are embedded are fully built before + # the custom command runs. File-level DEPENDS on $ paths + # does not guarantee target build ordering in parallel builds. + if(DEFINED jitembed_${TARGET}_blob__target_deps) + add_dependencies(${TARGET} ${jitembed_${TARGET}_blob__target_deps}) + endif() message( STATUS From 32383c006deb18e88d4f50d2729a5def07929682 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 12 Feb 2026 13:33:38 +0000 Subject: [PATCH 051/254] refactoring --- cpp/include/cudf/jit/lto/types.cuh | 1 + cpp/include/cudf/transform.hpp | 44 +++++++++------ cpp/src/jit/rtc/cudf.hpp | 35 ++++++++++-- cpp/tests/jit/rtc.cpp | 91 +++++++++++++++++++----------- 4 files changed, 116 insertions(+), 55 deletions(-) diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index 69cb1d13734b..a8b1d0932bcf 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -10,6 +10,7 @@ namespace CUDF_LTO_EXPORT cudf { namespace lto { + // TODO: update doc, and specify this is our ABI for JIT code and should be kept different from normal code /** * @brief LTO-JIT functions and thunk types * diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index fdf505e328bf..0874bb5bace6 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -11,6 +11,8 @@ #include #include +#include +#include namespace CUDF_EXPORT cudf { @@ -63,25 +65,31 @@ std::unique_ptr transform( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); -std::unique_ptr transform_lto_cuda_udf( - std::vector const& inputs, - std::string_view source_code, - std::string_view transform_function, - data_type output_type, - std::optional user_data = std::nullopt, - null_aware is_null_aware = null_aware::NO, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +enum class [[nodiscard]] udf_source_type : uint8_t { LTOIR_BINARY = 0, PTX_BINARY = 1, CUDA = 2 }; -std::unique_ptr transform_lto_binary_udf( - std::vector const& inputs, - std::span source_code, - std::string_view transform_function, - data_type output_type, - std::optional user_data = std::nullopt, - null_aware is_null_aware = null_aware::NO, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +struct [[nodiscard]] transform_params { + using input_type = std::variant; + + std::vector inputs = {}; + + std::optional output_size = std::nullopt; + + std::vector output_types = {}; + + null_aware is_null_aware = null_aware::NO; + + std::span udf = {}; + + udf_source_type type = udf_source_type::LTOIR_BINARY; + + output_nullability null_policy = output_nullability::PRESERVE; + + rmm::cuda_stream_view stream = cudf::get_default_stream(); + + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref(); +}; + +std::unique_ptr transform2(transform_params const& params); /** * @brief Creates a null_mask from `input` by converting `NaN` to null and diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/rtc/cudf.hpp index 638bf0010d65..acb33ac6fd97 100644 --- a/cpp/src/jit/rtc/cudf.hpp +++ b/cpp/src/jit/rtc/cudf.hpp @@ -38,10 +38,37 @@ class jit_bundle_t { std::vector get_compile_options() const; }; -library compile_and_link_udf(char const* name, - char const* udf_code, - char const* udf_key, - char const* kernel_symbol); +struct [[nodiscard]] udf_compile_params { + std::string_view name = {}; + + std::span udf = {}; + + std::string_view key = {}; + + std::string_view kernel_symbol = {}; + + std::span extra_compile_flags = {}; + + std::span extra_link_flags = {}; +}; + +library compile_and_link_cuda_udf(udf_compile_params const& params); + +struct [[nodiscard]] udf_link_params { + std::string_view name = {}; + + std::span udf_binary = {}; + + binary_type type = binary_type::LTO_IR; + + std::string_view key = {}; + + std::string_view kernel_symbol = {}; + + std::span extra_link_flags = {}; +}; + +library link_udf(udf_link_params const& params); } // namespace rtc } // namespace CUDF_EXPORT cudf diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index 442cae985848..153f7ea2c025 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -8,6 +8,7 @@ #include #include + #include #include @@ -37,9 +38,15 @@ struct element_operation { }; */ - // TODO: take optional pointer to this for it to be written to - // TODO: take optional list of extra compile flags for linking and compiling -struct jit_compilation_stats{ +// TODO: cache control policy: ignore cache +// TODO: statistics callback, in cudf layer? +// TODO: extra compile and link flags for LTO library and final kernel, use in CUDF layer, not +// here? +// +// TODO: ?? +// TODO: take optional pointer/callback to this for it to be written to +// TODO: take optional list of extra compile flags for linking and compiling +struct jit_compile_stats { std::chrono::nanoseconds cpp_compile_time{}; std::chrono::nanoseconds fragment_link_time{}; std::chrono::nanoseconds total_time{}; @@ -47,6 +54,8 @@ struct jit_compilation_stats{ // global cache statistics before and after? }; +TEST_F(RTCTest, CreateFragmentBasic) {} + TEST_F(RTCTest, CreateFragment) { // TODO: add configuration parameters necessary for testing @@ -55,8 +64,7 @@ TEST_F(RTCTest, CreateFragment) // they should use the provided LTO functions; might need a planner auto fn = []() { - auto lib = rtc::compile_and_link_udf("test_fragment", - R"***( + char const udf[] = R"***( #include "cudf/jit/lto/transform_params.cuh" #include "cudf/jit/lto/operators.cuh" #include "cudf/jit/lto/scope.cuh" @@ -68,26 +76,34 @@ TEST_F(RTCTest, CreateFragment) extern "C" __device__ void transform_operator(cudf::lto::transform_params p){ using namespace cudf::lto; + using ops = operators; /// <-- BEGIN OF INPUT UNPACKING: Defined by input planner // unpack inputs from scope using the appropriate getters based on the LTO context - using col_user_data = scope::user_data<0>; - using col0 = scope::column<1, column_view_core, int, false, false>; - using col1 = scope::column<2, column_view_core, int, false, false>; - using col2 = scope::column<3, column_view_core, double, false, false>; - using col3 = scope::column<4, column_view_core, float, false, false>; - using col4 = scope::column<5, span, float, false, false>; - using col5 = scope::column<6, optional_span, float, false, true>; - using out_col = scope::column<7, mutable_column_view_core, double, false, false>; - - auto user_data = col_user_data::element(p.scope, p.row_index); - auto in0 = col0::element(p.scope, p.row_index); - auto in1 = col1::element(p.scope, p.row_index); - auto in2 = col2::element(p.scope, p.row_index); - auto in3 = col3::element(p.scope, p.row_index); - auto in4 = col4::element(p.scope, p.row_index); - auto in5 = col5::element(p.scope, p.row_index); + using s0 = scope::user_data<0>; + using s1 = scope::column<1, column_view_core, int, false, false>; + using s2 = scope::column<2, column_view_core, int, false, false>; + using s3 = scope::column<3, column_view_core, double, false, false>; + using s4 = scope::column<4, column_view_core, float, false, false>; + using s4 = scope::column<4, string_view, float, false, false>; + using s4 = scope::column<4, decimal32, float, false, false>; + using s4 = scope::column<5, column_view_core, float, false, true>; + using s4 = scope::column<6, column_view_core, float, true, false>; + using s5 = scope::column<7, span, float, false, false>; + using s6 = scope::column<8, optional_span, float, false, true>; + using s7 = scope::column<9, mutable_column_view_core, double, false, false>; + + auto a0 = s0::element(p.scope, p.row_index); + auto a1 = s1::element(p.scope, p.row_index); + auto a2 = s2::element(p.scope, p.row_index); + auto a3 = s3::element(p.scope, p.row_index); + auto a4 = s4::element(p.scope, p.row_index); + auto a5 = s5::element(p.scope, p.row_index); + auto a6 = s6::element(p.scope, p.row_index); + auto a7 = s7::element(p.scope, p.row_index); + auto a7 = s8::element(p.scope, p.row_index); + auto a7 = s9::element(p.scope, p.row_index); auto result = 0.0; @@ -98,15 +114,15 @@ TEST_F(RTCTest, CreateFragment) // run operation using the LTO-compiled operators; these should be inlined into the final kernel and optimized together by NVJITLink - operators::add(&c, &a, &b); - operators::sub(&c, &a, &b); - operators::mul(&c, &a, &b); - operators::mul(&c, &a, &b); - operators::arctan(&c, &a, &b); - operators::sqrt(&c, &a, &b); - operators::cbrt(&c, &a, &b); - operators::arccos(&c, &a, &b); - operators::cast_to_float64(&d, &c); + ops::add(&c, &a, &b); + ops::sub(&c, &a, &b); + ops::mul(&c, &a, &b); + ops::mul(&c, &a, &b); + ops::arctan(&c, &a, &b); + ops::sqrt(&c, &a, &b); + ops::cbrt(&c, &a, &b); + ops::arccos(&c, &a, &b); + ops::cast_to_float64(&d, &c); /// <-- END OF USER-DEFINED OPERATOR @@ -120,9 +136,18 @@ TEST_F(RTCTest, CreateFragment) /// <-- END OF OUTPUT PACKING } - )***", - "test_udf_key", - "transform_kernel"); + )***"; + + // TODO: bincode dump arguments? + // TODO: add a bypass cache argument that forces recompilation for testing purposes + auto params = rtc::udf_compile_params{.name = "test_fragment", + .udf = udf, + .key = "test_udf_key", + .kernel_symbol = "transform_kernel", + .extra_compile_flags = {}, + .extra_link_flags = {}}; + + auto lib = rtc::compile_and_link_cuda_udf(params); auto kernel = lib->get_kernel("transform_kernel"); From 21a2610db84c4caa7294a53b9863de36cdec0a36 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 12 Feb 2026 14:16:49 +0000 Subject: [PATCH 052/254] refactoring --- cpp/include/cudf/transform.hpp | 27 -------------------- cpp/src/transform/transform2.cu | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 27 deletions(-) create mode 100644 cpp/src/transform/transform2.cu diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 0874bb5bace6..6a63589f9ce0 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -11,8 +11,6 @@ #include #include -#include -#include namespace CUDF_EXPORT cudf { @@ -65,31 +63,6 @@ std::unique_ptr transform( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); -enum class [[nodiscard]] udf_source_type : uint8_t { LTOIR_BINARY = 0, PTX_BINARY = 1, CUDA = 2 }; - -struct [[nodiscard]] transform_params { - using input_type = std::variant; - - std::vector inputs = {}; - - std::optional output_size = std::nullopt; - - std::vector output_types = {}; - - null_aware is_null_aware = null_aware::NO; - - std::span udf = {}; - - udf_source_type type = udf_source_type::LTOIR_BINARY; - - output_nullability null_policy = output_nullability::PRESERVE; - - rmm::cuda_stream_view stream = cudf::get_default_stream(); - - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref(); -}; - -std::unique_ptr
transform2(transform_params const& params); /** * @brief Creates a null_mask from `input` by converting `NaN` to null and diff --git a/cpp/src/transform/transform2.cu b/cpp/src/transform/transform2.cu new file mode 100644 index 000000000000..940cd5fdf20a --- /dev/null +++ b/cpp/src/transform/transform2.cu @@ -0,0 +1,44 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include +#include +#include + +namespace cudf { + +enum class [[nodiscard]] udf_source_type : uint8_t { LTOIR_BINARY = 0, PTX_BINARY = 1, CUDA = 2 }; + +struct [[nodiscard]] transform_operation_params { + using input = std::variant; + + std::vector inputs = {}; + + std::vector include_nullness = {}; + + std::vector output_sizes = {}; + + std::vector output_types = {}; + + std::vector output_nullability = {}; + + std::span udf = {}; + + udf_source_type type = udf_source_type::LTOIR_BINARY; + + rmm::cuda_stream_view stream = cudf::get_default_stream(); + + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref(); +}; + +std::unique_ptr
transform_operation(transform_operation_params const& params) +{ + return nullptr; +} + +} // namespace cudf \ No newline at end of file From c82867bbde8d9221438f93c1268dbe284f98eb3f Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 14 Feb 2026 01:06:51 +0000 Subject: [PATCH 053/254] Refactor JIT Compilation and Embed Functionality - Updated CMakeLists.txt to remove unnecessary comments and added src/jit/jit.cpp to the build. - Enhanced JitEmbed.cmake for better dependency management and code clarity. - Renamed structures in column_view.cuh for consistency and clarity. - Updated thunk.cuh to reflect new structure names and ensure proper mapping. - Modified types.cuh to align with new structure names and improve documentation. - Cleaned up transform.hpp by removing unnecessary whitespace. - Introduced jit.cpp and jit.hpp for improved JIT compilation handling. - Added README.md for documentation on JIT LTO functionality. - Updated cache.hpp and rtc.cpp to include necessary checks and improve error handling. - Refactored context.cpp and context.hpp to integrate new JIT bundle structure. - Adjusted transform2.cu to ensure proper namespace usage. - Updated rtc.cpp test cases to reflect changes in structure names and improve test clarity. --- cpp/CMakeLists.txt | 7 +- cpp/cmake/Modules/JitEmbed.cmake | 18 ++--- cpp/include/cudf/jit/lto/column_view.cuh | 22 +++--- cpp/include/cudf/jit/lto/scope.cuh | 4 +- cpp/include/cudf/jit/lto/thunk.cuh | 4 +- cpp/include/cudf/jit/lto/types.cuh | 7 +- cpp/include/cudf/transform.hpp | 1 - cpp/src/jit/{rtc/cudf.cpp => jit.cpp} | 90 ++++++++++++------------ cpp/src/jit/{rtc/cudf.hpp => jit.hpp} | 31 ++++---- cpp/src/jit/lto/README.md | 0 cpp/src/jit/rtc/cache.hpp | 14 ++-- cpp/src/jit/rtc/rtc.cpp | 9 +-- cpp/src/jit/rtc/rtc.hpp | 7 +- cpp/src/runtime/context.cpp | 6 +- cpp/src/runtime/context.hpp | 7 +- cpp/src/transform/transform2.cu | 2 +- cpp/tests/jit/rtc.cpp | 50 ++++++------- 17 files changed, 139 insertions(+), 140 deletions(-) rename cpp/src/jit/{rtc/cudf.cpp => jit.cpp} (83%) rename cpp/src/jit/{rtc/cudf.hpp => jit.hpp} (59%) create mode 100644 cpp/src/jit/lto/README.md diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8b1bb5816cb5..03a244aa6ceb 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -402,8 +402,7 @@ foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) INCLUDE_DIRECTORIES CCCL/libcudacxx/${INC_DIR_NAME} ) endforeach() -# TODO(lamarrr): inspect the generated cubin and make sure it doesn't contain any unneeded symbols. -# it should be as small as possible + jit_add_options(cudf_jit_embed OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) jit_add_blob(cudf_jit_embed FILE "$" DEST cudf_lto_library.fatbin) jit_embed(cudf_jit_embed COMPRESSION zstd) @@ -654,11 +653,11 @@ add_library( src/io/utilities/trie.cu src/jit/cache.cpp src/jit/helpers.cpp + src/jit/jit.cpp src/jit/parser.cpp src/jit/row_ir.cpp src/jit/util.cpp src/jit/rtc/cache.cpp - src/jit/rtc/cudf.cpp src/jit/rtc/rtc.cpp src/jit/rtc/sha256.cpp src/join/conditional_join.cu @@ -1073,7 +1072,7 @@ target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_A # Compile stringified JIT sources first add_dependencies(cudf jitify_preprocess_run) -# Specify the target module library dependencies +# Specify the target module library dependencies # TODO: use runtime nvrtc and nvjitlink and driver target_link_libraries( diff --git a/cpp/cmake/Modules/JitEmbed.cmake b/cpp/cmake/Modules/JitEmbed.cmake index d80aecb51ba1..d61f9c1b89d7 100644 --- a/cpp/cmake/Modules/JitEmbed.cmake +++ b/cpp/cmake/Modules/JitEmbed.cmake @@ -138,10 +138,9 @@ function(jit_add_blob) list(APPEND blob_files "${ARG_FILE}") list(APPEND blob_dests "${ARG_DEST}") - # If FILE is a $ generator expression, record the target - # name so jit_embed() can add an explicit ordering dependency via - # add_dependencies(). A plain file dependency on the evaluated paths is not - # sufficient to guarantee build ordering in parallel builds. + # If FILE is a $ generator expression, record the target name so jit_embed() + # can add an explicit ordering dependency via add_dependencies(). A plain file dependency on the + # evaluated paths is not sufficient to guarantee build ordering in parallel builds. if(ARG_FILE MATCHES "\\$]+)>") list(APPEND blob_target_deps "${CMAKE_MATCH_1}") set(jitembed_${TARGET}_blob__target_deps @@ -281,12 +280,13 @@ function(jit_embed) VERBATIM ) - add_custom_target(${TARGET} ALL DEPENDS ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s - ${OUTPUT_DIR}/embed.bin) + add_custom_target( + ${TARGET} ALL DEPENDS ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s ${OUTPUT_DIR}/embed.bin + ) - # Ensure any CMake targets whose objects are embedded are fully built before - # the custom command runs. File-level DEPENDS on $ paths - # does not guarantee target build ordering in parallel builds. + # Ensure any CMake targets whose objects are embedded are fully built before the custom command + # runs. File-level DEPENDS on $ paths does not guarantee target build ordering + # in parallel builds. if(DEFINED jitembed_${TARGET}_blob__target_deps) add_dependencies(${TARGET} ${jitembed_${TARGET}_blob__target_deps}) endif() diff --git a/cpp/include/cudf/jit/lto/column_view.cuh b/cpp/include/cudf/jit/lto/column_view.cuh index a225383500e7..d554a187c059 100644 --- a/cpp/include/cudf/jit/lto/column_view.cuh +++ b/cpp/include/cudf/jit/lto/column_view.cuh @@ -12,14 +12,14 @@ namespace CUDF_LTO_EXPORT cudf { namespace lto { -struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { +struct alignas(16) CUDF_LTO_ALIAS column_view { private: data_type _type = {}; size_type _size = 0; void const* _data = nullptr; bitmask_type const* _null_mask = nullptr; size_type _offset = 0; - column_device_view_core* _d_children = nullptr; + column_view* _d_children = nullptr; size_type _num_children = 0; public: @@ -53,14 +53,14 @@ struct alignas(16) CUDF_LTO_ALIAS column_device_view_core { __device__ size_type num_child_columns() const; }; -struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { +struct alignas(16) CUDF_LTO_ALIAS mutable_column_view { private: data_type _type = {}; size_type _size = 0; void const* _data = nullptr; bitmask_type const* _null_mask = nullptr; size_type _offset = 0; - mutable_column_device_view_core* _d_children = nullptr; + mutable_column_view* _d_children = nullptr; size_type _num_children = 0; public: @@ -183,42 +183,42 @@ struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view_core { DO_IT(duration_us) \ DO_IT(duration_ns) -#define DO_IT(T) extern template __device__ T const* column_device_view_core::head() const; +#define DO_IT(T) extern template __device__ T const* column_view::head() const; FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE #undef DO_IT #define DO_IT(T) \ - extern template __device__ T column_device_view_core::element(size_type idx) const; + extern template __device__ T column_view::element(size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT #define DO_IT(T) \ - extern template __device__ optional column_device_view_core::nullable_element( \ + extern template __device__ optional column_view::nullable_element( \ size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -#define DO_IT(T) extern template __device__ T* mutable_column_device_view_core::head() const; +#define DO_IT(T) extern template __device__ T* mutable_column_view::head() const; FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE #undef DO_IT #define DO_IT(T) \ - extern template __device__ T mutable_column_device_view_core::element(size_type idx) const; + extern template __device__ T mutable_column_view::element(size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT #define DO_IT(T) \ - extern template __device__ optional mutable_column_device_view_core::nullable_element( \ + extern template __device__ optional mutable_column_view::nullable_element( \ size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE @@ -226,7 +226,7 @@ FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT #define DO_IT(T) \ - extern template __device__ void mutable_column_device_view_core::assign(size_type idx, \ + extern template __device__ void mutable_column_view::assign(size_type idx, \ T value) const; FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE diff --git a/cpp/include/cudf/jit/lto/scope.cuh b/cpp/include/cudf/jit/lto/scope.cuh index 9876faa94da4..f910f760e7a1 100644 --- a/cpp/include/cudf/jit/lto/scope.cuh +++ b/cpp/include/cudf/jit/lto/scope.cuh @@ -21,7 +21,7 @@ template -struct column { +struct [[nodiscard]] column { static constexpr bool IS_SCALAR = IsScalar; static constexpr bool IS_NULLABLE = IsNullable; @@ -68,7 +68,7 @@ struct column { }; template -struct user_data { +struct [[nodiscard]] user_data { using Arg = void* __restrict__; static __device__ decltype(auto) element(args scope, [[maybe_unused]] size_type i) diff --git a/cpp/include/cudf/jit/lto/thunk.cuh b/cpp/include/cudf/jit/lto/thunk.cuh index 9d3ba4ab87ae..a9887d00b1d3 100644 --- a/cpp/include/cudf/jit/lto/thunk.cuh +++ b/cpp/include/cudf/jit/lto/thunk.cuh @@ -123,8 +123,8 @@ CUDF_LTO_MAP(duration_s, cudf::duration_s); CUDF_LTO_MAP(duration_ms, cudf::duration_ms); CUDF_LTO_MAP(duration_us, cudf::duration_us); CUDF_LTO_MAP(duration_ns, cudf::duration_ns); -CUDF_LTO_MAP(column_device_view_core, cudf::column_device_view_core); -CUDF_LTO_MAP(mutable_column_device_view_core, cudf::mutable_column_device_view_core); +CUDF_LTO_MAP(column_view, cudf::column_device_view_core); +CUDF_LTO_MAP(mutable_column_view, cudf::mutable_column_device_view_core); #undef CUDF_LTO_MAP diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index a8b1d0932bcf..56ab7ff44117 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -10,7 +10,8 @@ namespace CUDF_LTO_EXPORT cudf { namespace lto { - // TODO: update doc, and specify this is our ABI for JIT code and should be kept different from normal code +// TODO: update doc, and specify this is our ABI for JIT code and should be kept different from +// normal code /** * @brief LTO-JIT functions and thunk types * @@ -161,9 +162,9 @@ struct CUDF_LTO_ALIAS optional; struct CUDF_LTO_ALIAS string_view; -struct CUDF_LTO_ALIAS column_device_view_core; +struct CUDF_LTO_ALIAS column_view; -struct CUDF_LTO_ALIAS mutable_column_device_view_core; +struct CUDF_LTO_ALIAS mutable_column_view; } // namespace lto } // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 6a63589f9ce0..633de70a3f95 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -63,7 +63,6 @@ std::unique_ptr transform( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); - /** * @brief Creates a null_mask from `input` by converting `NaN` to null and * preserving existing null values and also returns new null_count. diff --git a/cpp/src/jit/rtc/cudf.cpp b/cpp/src/jit/jit.cpp similarity index 83% rename from cpp/src/jit/rtc/cudf.cpp rename to cpp/src/jit/jit.cpp index 79b0112242c7..4e9901707580 100644 --- a/cpp/src/jit/rtc/cudf.cpp +++ b/cpp/src/jit/jit.cpp @@ -12,8 +12,8 @@ #include #include +#include #include -#include #include #include #include @@ -42,13 +42,12 @@ } while (0) namespace CUDF_EXPORT cudf { -namespace rtc { namespace { -sha256_hash hash_string(std::span input) +rtc::sha256_hash hash_string(std::span input) { - sha256_context ctx; + rtc::sha256_context ctx; ctx.update(std::span{reinterpret_cast(input.data()), input.size()}); return ctx.finalize(); } @@ -239,24 +238,26 @@ bundle={})***", auto compile = [&] { auto path = std::format("{}/{}", get_directory(), "cudf_lto_library.fatbin"); - auto cubin = blob_t::from_file(path.c_str()); + auto cubin = rtc::blob_t::from_file(path.c_str()); CUDF_EXPECTS(cubin.has_value(), +std::format("Failed to load LTO library cubin from disk at ({})", path), std::runtime_error); - fragment_t::load_params load_params{.binary = std::make_shared(std::move(*cubin)), - .type = binary_type::FATBIN}; - return fragment_t::load(load_params); + rtc::fragment_t::load_params load_params{ + .binary = std::make_shared(std::move(*cubin)), .type = rtc::binary_type::FATBIN}; + return rtc::fragment_t::load(load_params); }; - auto fut = cache.query_or_insert_fragment( - cache_key_sha256, binary_type::FATBIN, fragment_compile_function_t::from_functor(compile)); + auto fut = + cache.query_or_insert_fragment(cache_key_sha256, + rtc::binary_type::FATBIN, + rtc::fragment_compile_function_t::from_functor(compile)); lto_library_ = fut.get(); } std::string jit_bundle_t::get_hash() const { - auto str = sha256_hex_string::make( + auto str = rtc::sha256_hex_string::make( std::span{cudf_jit_embed_hash.data, static_cast(cudf_jit_embed_hash.size)}); return std::string{str.view()}; } @@ -266,7 +267,7 @@ std::string jit_bundle_t::get_directory() const return std::format("{}/{}", install_dir_, get_hash()); } -fragment jit_bundle_t::get_lto_library() const { return lto_library_; } +rtc::fragment jit_bundle_t::get_lto_library() const { return lto_library_; } std::vector jit_bundle_t::get_include_directories() const { @@ -358,8 +359,6 @@ bundle={})***", auto cache_key_sha256 = hash_string(cache_key); - // TODO: add time function in cache - auto compile = [&] { auto begin = std::chrono::steady_clock::now(); @@ -396,13 +395,13 @@ bundle={})***", options_cstr.emplace_back(option.c_str()); } - auto params = fragment_t::compile_params{.name = name, - .source = source_code_cstr, - .headers = {}, - .options = options_cstr, - .target_type = binary_type::LTO_IR}; + auto params = rtc::fragment_t::compile_params{.name = name, + .source = source_code_cstr, + .headers = {}, + .options = options_cstr, + .target_type = rtc::binary_type::LTO_IR}; - auto frag = fragment_t::compile(params); + auto frag = rtc::fragment_t::compile(params); auto end = std::chrono::steady_clock::now(); @@ -416,16 +415,18 @@ bundle={})***", return frag; }; - auto fut = cache.query_or_insert_fragment( - cache_key_sha256, binary_type::LTO_IR, fragment_compile_function_t::from_functor(compile)); + auto fut = + cache.query_or_insert_fragment(cache_key_sha256, + rtc::binary_type::LTO_IR, + rtc::fragment_compile_function_t::from_functor(compile)); return fut.get(); } -library compile_and_link_udf(char const* name, - char const* udf_code, - char const* udf_key, - char const* kernel_symbol) +rtc::library compile_and_link_udf(char const* name, + char const* udf_code, + char const* udf_key, + char const* kernel_symbol) { CUDF_FUNC_RANGE(); @@ -443,7 +444,7 @@ library compile_and_link_udf(char const* name, auto fragment = get_or_compile_fragment(name, udf_code, udf_key); // TODO: sass dump - // TODO: time dump + // TODO: time trace dump // TODO: experiment with: // optimization flags @@ -469,25 +470,26 @@ library compile_and_link_udf(char const* name, options_cstr.emplace_back(option.c_str()); } - blob_view link_fragments[] = {library->get(binary_type::FATBIN)->view(), - fragment->get(binary_type::LTO_IR)->view()}; + rtc::blob_view link_fragments[] = {library->get(rtc::binary_type::FATBIN)->view(), + fragment->get(rtc::binary_type::LTO_IR)->view()}; - binary_type fragment_binary_types[] = {binary_type::FATBIN, binary_type::LTO_IR}; + rtc::binary_type fragment_binary_types[] = {rtc::binary_type::FATBIN, rtc::binary_type::LTO_IR}; char const* fragment_names[] = {"cudf_lto_library", name}; - auto params = library_t::link_params{.name = name, - .output_type = binary_type::CUBIN, - .fragments = link_fragments, - .fragment_binary_types = fragment_binary_types, - .fragment_names = fragment_names, - .link_options = options_cstr}; + auto params = rtc::library_t::link_params{.name = name, + .output_type = rtc::binary_type::CUBIN, + .fragments = link_fragments, + .fragment_binary_types = fragment_binary_types, + .fragment_names = fragment_names, + .link_options = options_cstr}; - auto blob = library_t::link_as_blob(params); + auto blob = rtc::library_t::link_as_blob(params); - auto load_params = library_t::load_params{.binary = blob->view(), .type = binary_type::CUBIN}; + auto load_params = + rtc::library_t::load_params{.binary = blob->view(), .type = rtc::binary_type::CUBIN}; - auto linked_library = library_t::load(load_params); + auto linked_library = rtc::library_t::load(load_params); auto end = std::chrono::steady_clock::now(); @@ -514,12 +516,12 @@ arch={})***", sm); auto library_cache_key_sha256 = hash_string(library_cache_key); - auto library = cache.query_or_insert_library(library_cache_key_sha256, - binary_type::CUBIN, - library_compile_function_t::from_functor(compile)); + auto library = + cache.query_or_insert_library(library_cache_key_sha256, + rtc::binary_type::CUBIN, + rtc::library_compile_function_t::from_functor(compile)); return library.get(); } -} // namespace rtc -} // namespace cudf +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/cudf.hpp b/cpp/src/jit/jit.hpp similarity index 59% rename from cpp/src/jit/rtc/cudf.hpp rename to cpp/src/jit/jit.hpp index acb33ac6fd97..758c0202d969 100644 --- a/cpp/src/jit/rtc/cudf.hpp +++ b/cpp/src/jit/jit.hpp @@ -1,5 +1,3 @@ - - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 @@ -12,30 +10,30 @@ #include namespace CUDF_EXPORT cudf { -namespace rtc { -class jit_bundle_t { +struct [[nodiscard]] jit_bundle_t { + private: std::string install_dir_; - fragment lto_library_; - cache_t* cache_; + rtc::fragment lto_library_; + rtc::cache_t* cache_; - private: void ensure_installed() const; void preload_lto_library(); public: - jit_bundle_t(std::string install_dir, cache_t& cache); + jit_bundle_t(std::string install_dir, rtc::cache_t& cache); - std::string get_hash() const; + [[nodiscard]] std::string get_hash() const; + [[nodiscard]] std::string get_directory() const; - fragment get_lto_library() const; + [[nodiscard]] rtc::fragment get_lto_library() const; - std::vector get_include_directories() const; + [[nodiscard]] std::vector get_include_directories() const; - std::vector get_compile_options() const; + [[nodiscard]] std::vector get_compile_options() const; }; struct [[nodiscard]] udf_compile_params { @@ -52,14 +50,14 @@ struct [[nodiscard]] udf_compile_params { std::span extra_link_flags = {}; }; -library compile_and_link_cuda_udf(udf_compile_params const& params); +[[nodiscard]] rtc::library compile_and_link_cuda_udf(udf_compile_params const& params); struct [[nodiscard]] udf_link_params { std::string_view name = {}; - std::span udf_binary = {}; + std::span udf_blob = {}; - binary_type type = binary_type::LTO_IR; + rtc::binary_type type = rtc::binary_type::LTO_IR; std::string_view key = {}; @@ -68,7 +66,6 @@ struct [[nodiscard]] udf_link_params { std::span extra_link_flags = {}; }; -library link_udf(udf_link_params const& params); +[[nodiscard]] rtc::library link_udf(udf_link_params const& params); -} // namespace rtc } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/lto/README.md b/cpp/src/jit/lto/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp index b164d2d1347c..80030997f4ec 100644 --- a/cpp/src/jit/rtc/cache.hpp +++ b/cpp/src/jit/rtc/cache.hpp @@ -237,16 +237,14 @@ struct cache_t { [[nodiscard]] std::string const& get_cache_dir(); - std::shared_future query_or_insert_blob(sha256_hash const& sha, - blob_compile_function_t compile); + [[nodiscard]] std::shared_future query_or_insert_blob(sha256_hash const& sha, + blob_compile_function_t compile); - std::shared_future query_or_insert_fragment(sha256_hash const& sha, - binary_type type, - fragment_compile_function_t compile); + [[nodiscard]] std::shared_future query_or_insert_fragment( + sha256_hash const& sha, binary_type type, fragment_compile_function_t compile); - std::shared_future query_or_insert_library(sha256_hash const& sha, - binary_type type, - library_compile_function_t compile); + [[nodiscard]] std::shared_future query_or_insert_library( + sha256_hash const& sha, binary_type type, library_compile_function_t compile); cache_statistics get_statistics(); diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index f3069f0154d5..7c2e993aa192 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -284,16 +284,17 @@ blob_t blob_t::from_static_data(std::span data) fragment fragment_t::load(load_params const& params) { CUDF_FUNC_RANGE(); - // TODO: validate parameters return std::make_shared(params.binary, params.type); } fragment fragment_t::compile(compile_params const& params) { - // TODO: check CUDF_FUNC_RANGE(); + CUDF_EXPECTS(params.name != nullptr, "Fragment name must not be null", std::logic_error); + CUDF_EXPECTS(params.source != nullptr, "Fragment source must not be null", std ::logic_error); + nvrtcProgram program = nullptr; CUDFRTC_CHECK_NVRTC(params, @@ -419,7 +420,6 @@ library_t::~library_t() library library_t::load(load_params const& params) { - // TODO: check CUDF_FUNC_RANGE(); CUlibrary handle; @@ -442,6 +442,7 @@ blob library_t::link_as_blob(link_params const& params) { CUDF_FUNC_RANGE(); + CUDF_EXPECTS(params.name != nullptr, "Link output name must not be null", std::logic_error); CUDF_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, "Only CUBIN and PTX output types are supported for linking modules", std::logic_error); @@ -566,4 +567,4 @@ std::string rtc::demangle_cuda_symbol(char const* mangled_name) return result; } -} // namespace cudf +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index abebabde036a..2b647cf64438 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -5,12 +5,13 @@ */ #pragma once +#include + #include #include #include #include #include -#include extern "C" { typedef struct CUlib_st* CUlibrary; @@ -98,7 +99,7 @@ struct function_ref; /// @brief Zero-copy, type-erased reference to a callable entity (e.g. lambda, function pointer) /// that can be invoked with the given signature. template -struct function_ref { +struct [[nodiscard]] function_ref { private: void* _user_data; R (*_thunk)(void*, Args...); @@ -246,4 +247,4 @@ struct [[nodiscard]] library_t { [[nodiscard]] std::string demangle_cuda_symbol(char const* mangled_name); } // namespace rtc -} // namespace cudf +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index c83770cc1b65..d9ce67ebf0fa 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -8,8 +8,8 @@ #include "io/comp/nvcomp_adapter.hpp" #include "io/utilities/getenv_or.hpp" #include "jit/cache.hpp" +#include "jit/jit.hpp" #include "jit/rtc/cache.hpp" -#include "jit/rtc/cudf.hpp" #include #include @@ -49,7 +49,7 @@ void context::ensure_jit_bundle_initialized() std::call_once(_jit_bundle_init_flag, [&]() { // make sure the jit bundle directory exists std::filesystem::create_directories(_config.jit_bundle_dir); - _jit_bundle = std::make_unique(_config.jit_bundle_dir, *_rtc_cache); + _jit_bundle = std::make_unique(_config.jit_bundle_dir, *_rtc_cache); }); } @@ -65,7 +65,7 @@ rtc::cache_t& context::rtc_cache() return *_rtc_cache; } -rtc::jit_bundle_t& context::jit_bundle() +jit_bundle_t& context::jit_bundle() { ensure_jit_bundle_initialized(); return *_jit_bundle; diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index a1c90c6e55cd..ae81039be059 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -19,9 +19,10 @@ class program_cache; namespace rtc { class cache_t; -class jit_bundle_t; } // namespace rtc +class jit_bundle_t; + struct [[nodiscard]] context_config { bool dump_codegen = false; bool use_jit = false; @@ -40,7 +41,7 @@ class context { std::once_flag _rtc_cache_init_flag; std::unique_ptr _rtc_cache; std::once_flag _jit_bundle_init_flag; - std::unique_ptr _jit_bundle; + std::unique_ptr _jit_bundle; private: void ensure_nvcomp_loaded(); @@ -63,7 +64,7 @@ class context { rtc::cache_t& rtc_cache(); - rtc::jit_bundle_t& jit_bundle(); + jit_bundle_t& jit_bundle(); [[nodiscard]] bool dump_codegen() const; diff --git a/cpp/src/transform/transform2.cu b/cpp/src/transform/transform2.cu index 940cd5fdf20a..c83f8dee53d2 100644 --- a/cpp/src/transform/transform2.cu +++ b/cpp/src/transform/transform2.cu @@ -41,4 +41,4 @@ std::unique_ptr
transform_operation(transform_operation_params const& par return nullptr; } -} // namespace cudf \ No newline at end of file +} // namespace cudf diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index 153f7ea2c025..796cf421fcf0 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include @@ -17,7 +17,6 @@ using namespace cudf; struct RTCTest : public ::testing::Test {}; -// TODO: use this to document how operators can use the accessors /* template ; - using s1 = scope::column<1, column_view_core, int, false, false>; - using s2 = scope::column<2, column_view_core, int, false, false>; - using s3 = scope::column<3, column_view_core, double, false, false>; - using s4 = scope::column<4, column_view_core, float, false, false>; + using s1 = scope::column<1, column_view, int, false, false>; + using s2 = scope::column<2, column_view, int, false, false>; + using s3 = scope::column<3, column_view, double, false, false>; + using s4 = scope::column<4, column_view, float, false, false>; using s4 = scope::column<4, string_view, float, false, false>; using s4 = scope::column<4, decimal32, float, false, false>; - using s4 = scope::column<5, column_view_core, float, false, true>; - using s4 = scope::column<6, column_view_core, float, true, false>; + using s4 = scope::column<5, column_view, float, false, true>; + using s4 = scope::column<6, column_view, float, true, false>; using s5 = scope::column<7, span, float, false, false>; using s6 = scope::column<8, optional_span, float, false, true>; - using s7 = scope::column<9, mutable_column_view_core, double, false, false>; + using s7 = scope::column<9, mutable_column_view, double, false, false>; auto a0 = s0::element(p.scope, p.row_index); auto a1 = s1::element(p.scope, p.row_index); @@ -138,16 +140,14 @@ TEST_F(RTCTest, CreateFragment) } )***"; - // TODO: bincode dump arguments? - // TODO: add a bypass cache argument that forces recompilation for testing purposes - auto params = rtc::udf_compile_params{.name = "test_fragment", - .udf = udf, - .key = "test_udf_key", - .kernel_symbol = "transform_kernel", - .extra_compile_flags = {}, - .extra_link_flags = {}}; + auto params = cudf::udf_compile_params{.name = "test_fragment", + .udf = udf, + .key = "test_udf_key", + .kernel_symbol = "transform_kernel", + .extra_compile_flags = {}, + .extra_link_flags = {}}; - auto lib = rtc::compile_and_link_cuda_udf(params); + auto lib = cudf::compile_and_link_cuda_udf(params); auto kernel = lib->get_kernel("transform_kernel"); From badacea8cf0b39db114c945ea03b8405ec36840d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 18 Feb 2026 10:37:14 +0000 Subject: [PATCH 054/254] checkpoint --- cpp/CMakeLists.txt | 2 +- cpp/include/cudf/jit/lto/column_view.cuh | 57 ++- cpp/include/cudf/jit/lto/export.cuh | 8 + cpp/include/cudf/jit/lto/scope.cuh | 8 +- cpp/include/cudf/jit/lto/thunk.cuh | 4 +- cpp/include/cudf/jit/lto/transform_params.cuh | 2 +- cpp/include/cudf/jit/lto/types.cuh | 6 +- cpp/src/jit/jit.cpp | 349 +++++++++++------- cpp/src/jit/jit.hpp | 44 +-- cpp/src/jit/lto/column_view.def.cuh | 74 ++-- cpp/src/jit/rtc/rtc.cpp | 1 - cpp/src/runtime/context.cpp | 10 + cpp/src/runtime/context.hpp | 3 + cpp/src/transform/jit/lto_kernel.inl.cuh | 2 +- cpp/tests/jit/rtc.cpp | 174 +++++++-- jit.md | 1 - lto-ir-notes.txt | 1 - todo.md | 2 - 18 files changed, 462 insertions(+), 286 deletions(-) delete mode 100644 jit.md delete mode 100644 lto-ir-notes.txt delete mode 100644 todo.md diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 03a244aa6ceb..8e8ed80b7826 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -360,7 +360,7 @@ list(APPEND CUDF_JIT_COMPILE_FLAGS -std=c++${CUDF_CUDA_STANDARD} --device-int128 set(CUDF_LTO_JIT_ARCHITECTURES "80-real;90-real;100-real;120-real") add_library(cudf_lto_library OBJECT src/jit/lto/library.cu) -target_compile_options(cudf_lto_library PRIVATE -Xfatbin=--compress-all --compress-mode=size) +target_compile_options(cudf_lto_library PRIVATE --compress-mode=none) set_target_properties( cudf_lto_library PROPERTIES CUDA_ARCHITECTURES "${CUDF_LTO_JIT_ARCHITECTURES}" diff --git a/cpp/include/cudf/jit/lto/column_view.cuh b/cpp/include/cudf/jit/lto/column_view.cuh index d554a187c059..50a03748ed2e 100644 --- a/cpp/include/cudf/jit/lto/column_view.cuh +++ b/cpp/include/cudf/jit/lto/column_view.cuh @@ -12,15 +12,15 @@ namespace CUDF_LTO_EXPORT cudf { namespace lto { -struct alignas(16) CUDF_LTO_ALIAS column_view { +struct alignas(16) CUDF_LTO_ALIAS column_device_view { private: - data_type _type = {}; - size_type _size = 0; - void const* _data = nullptr; - bitmask_type const* _null_mask = nullptr; - size_type _offset = 0; - column_view* _d_children = nullptr; - size_type _num_children = 0; + data_type _type = {}; + size_type _size = 0; + void const* _data = nullptr; + bitmask_type const* _null_mask = nullptr; + size_type _offset = 0; + column_device_view* _d_children = nullptr; + size_type _num_children = 0; public: template @@ -53,15 +53,15 @@ struct alignas(16) CUDF_LTO_ALIAS column_view { __device__ size_type num_child_columns() const; }; -struct alignas(16) CUDF_LTO_ALIAS mutable_column_view { +struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view { private: - data_type _type = {}; - size_type _size = 0; - void const* _data = nullptr; - bitmask_type const* _null_mask = nullptr; - size_type _offset = 0; - mutable_column_view* _d_children = nullptr; - size_type _num_children = 0; + data_type _type = {}; + size_type _size = 0; + void const* _data = nullptr; + bitmask_type const* _null_mask = nullptr; + size_type _offset = 0; + mutable_column_device_view* _d_children = nullptr; + size_type _num_children = 0; public: template @@ -183,51 +183,50 @@ struct alignas(16) CUDF_LTO_ALIAS mutable_column_view { DO_IT(duration_us) \ DO_IT(duration_ns) -#define DO_IT(T) extern template __device__ T const* column_view::head() const; +#define DO_IT(T) extern template __device__ T const* column_device_view::head() const; FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE #undef DO_IT -#define DO_IT(T) \ - extern template __device__ T column_view::element(size_type idx) const; +#define DO_IT(T) extern template __device__ T column_device_view::element(size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -#define DO_IT(T) \ - extern template __device__ optional column_view::nullable_element( \ - size_type idx) const; +#define DO_IT(T) \ + extern template __device__ optional column_device_view::nullable_element(size_type idx) \ + const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -#define DO_IT(T) extern template __device__ T* mutable_column_view::head() const; +#define DO_IT(T) extern template __device__ T* mutable_column_device_view::head() const; FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE #undef DO_IT #define DO_IT(T) \ - extern template __device__ T mutable_column_view::element(size_type idx) const; + extern template __device__ T mutable_column_device_view::element(size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -#define DO_IT(T) \ - extern template __device__ optional mutable_column_view::nullable_element( \ +#define DO_IT(T) \ + extern template __device__ optional mutable_column_device_view::nullable_element( \ size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -#define DO_IT(T) \ - extern template __device__ void mutable_column_view::assign(size_type idx, \ - T value) const; +#define DO_IT(T) \ + extern template __device__ void mutable_column_device_view::assign(size_type idx, T value) \ + const; FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE diff --git a/cpp/include/cudf/jit/lto/export.cuh b/cpp/include/cudf/jit/lto/export.cuh index b91aa226d625..39e58d993313 100644 --- a/cpp/include/cudf/jit/lto/export.cuh +++ b/cpp/include/cudf/jit/lto/export.cuh @@ -5,6 +5,14 @@ #pragma once +#if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) + #define CUDF_LTO_EXPORT __attribute__((visibility("default"))) +#else + +#define CUDF_LTO_EXPORT + +#endif + #define CUDF_LTO_ALIAS __attribute__((may_alias)) diff --git a/cpp/include/cudf/jit/lto/scope.cuh b/cpp/include/cudf/jit/lto/scope.cuh index f910f760e7a1..1705c8740a68 100644 --- a/cpp/include/cudf/jit/lto/scope.cuh +++ b/cpp/include/cudf/jit/lto/scope.cuh @@ -12,10 +12,10 @@ namespace CUDF_LTO_EXPORT cudf { namespace lto { namespace scope { -using args = void* const __restrict__* __restrict__; +using args = void* const*; template struct [[nodiscard]] user_data { - using Arg = void* __restrict__; + using Arg = void*; static __device__ decltype(auto) element(args scope, [[maybe_unused]] size_type i) { diff --git a/cpp/include/cudf/jit/lto/thunk.cuh b/cpp/include/cudf/jit/lto/thunk.cuh index a9887d00b1d3..ec844455db7b 100644 --- a/cpp/include/cudf/jit/lto/thunk.cuh +++ b/cpp/include/cudf/jit/lto/thunk.cuh @@ -123,8 +123,8 @@ CUDF_LTO_MAP(duration_s, cudf::duration_s); CUDF_LTO_MAP(duration_ms, cudf::duration_ms); CUDF_LTO_MAP(duration_us, cudf::duration_us); CUDF_LTO_MAP(duration_ns, cudf::duration_ns); -CUDF_LTO_MAP(column_view, cudf::column_device_view_core); -CUDF_LTO_MAP(mutable_column_view, cudf::mutable_column_device_view_core); +CUDF_LTO_MAP(column_device_view, cudf::column_device_view_core); +CUDF_LTO_MAP(mutable_column_device_view, cudf::mutable_column_device_view_core); #undef CUDF_LTO_MAP diff --git a/cpp/include/cudf/jit/lto/transform_params.cuh b/cpp/include/cudf/jit/lto/transform_params.cuh index 9e1789e05865..81651c5edcf6 100644 --- a/cpp/include/cudf/jit/lto/transform_params.cuh +++ b/cpp/include/cudf/jit/lto/transform_params.cuh @@ -13,7 +13,7 @@ namespace lto { /// @brief Type-erased parameters for LTO-JIT-compiled transform operations. struct [[nodiscard]] transform_params { /// @brief Pointer to scope data (e.g. column views, scalars, etc.). - void* __restrict__ const* __restrict__ scope = nullptr; + void* const* scope = nullptr; /// @brief Total number of rows to process. size_type num_rows = 0; diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index 56ab7ff44117..440ac29e2610 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -34,6 +34,8 @@ namespace lto { * */ +// TODO: rename directory to abi + using int8_t = signed char; using int16_t = signed short; using int32_t = signed int; @@ -162,9 +164,9 @@ struct CUDF_LTO_ALIAS optional; struct CUDF_LTO_ALIAS string_view; -struct CUDF_LTO_ALIAS column_view; +struct CUDF_LTO_ALIAS column_device_view; -struct CUDF_LTO_ALIAS mutable_column_view; +struct CUDF_LTO_ALIAS mutable_column_device_view; } // namespace lto } // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index 4e9901707580..d222ed69793f 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -188,7 +188,7 @@ void create_and_install_cudf_jit(char const* target_dir) } // namespace -jit_bundle_t::jit_bundle_t(std::string install_dir, cache_t& cache) +jit_bundle_t::jit_bundle_t(std::string install_dir, rtc::cache_t& cache) : install_dir_{std::move(install_dir)}, cache_{&cache} { ensure_installed(); @@ -331,14 +331,154 @@ int32_t get_current_device_physical_model() return props.major * 10 + props.minor; } -} // namespace +rtc::fragment compile_udf_uncached(char const* name, + char const* cuda_code, + bool use_pch, + bool log_pch) +{ + CUDF_FUNC_RANGE(); + + auto& bundle = cudf::get_context().jit_bundle(); + auto begin = std::chrono::steady_clock::now(); + auto sm = get_current_device_physical_model(); + + auto include_dirs = bundle.get_include_directories(); + auto compile_options = bundle.get_compile_options(); + auto pch_dir = cudf::get_context().get_jit_pch_dir(); + + std::vector options; + + for (auto const& include_dir : include_dirs) { + options.emplace_back(std::format("-I{}", include_dir)); + } + + for (auto const& compile_option : compile_options) { + options.emplace_back(compile_option); + } + + // TODO: experiment with: + // --fdevice-time-trace=jit_comp_trace.json + // --time=compile_trace.json + // -time + + options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); + options.emplace_back("--dlink-time-opt"); + options.emplace_back("--relocatable-device-code=true"); + options.emplace_back("--device-as-default-execution-space"); + options.emplace_back("--restrict"); + options.emplace_back("--minimal"); + options.emplace_back("--split-compile=0"); + + if (use_pch) { + options.emplace_back("--pch"); + options.emplace_back(std::format("--pch-dir={}", pch_dir)); + + if (log_pch) { + options.emplace_back("--pch-verbose=true"); + options.emplace_back("--pch-messages=true"); + } + } + + std::vector options_cstr; + for (auto const& option : options) { + options_cstr.emplace_back(option.c_str()); + } + + auto params = rtc::fragment_t::compile_params{.name = name, + .source = cuda_code, + .headers = {}, + .options = options_cstr, + .target_type = rtc::binary_type::LTO_IR}; + + auto frag = rtc::fragment_t::compile(params); + + auto end = std::chrono::steady_clock::now(); + + auto duration = end - begin; + + CUDF_LOG_WARN( + "Compiled fragment `%s` in %f ms", + name, + std::chrono::duration_cast>(duration).count()); + + return frag; +} -fragment get_or_compile_fragment(char const* name, char const* source_code_cstr, char const* key) +std::tuple link_udf_uncached(char const* name, + rtc::fragment const& fragment, + char const* kernel_symbol) { CUDF_FUNC_RANGE(); + auto sm = get_current_device_physical_model(); auto& bundle = cudf::get_context().jit_bundle(); + + auto begin = std::chrono::steady_clock::now(); + auto library = bundle.get_lto_library(); + + // TODO: sass dump + // TODO: time trace dump + // TODO: lineinfo and debug info options + // TODO: -nocache + + std::vector options; + + options.emplace_back("-O3"); + options.emplace_back("-lto"); + options.emplace_back(std::format("-arch=sm_{}", sm)); + options.emplace_back(std::format("-kernels-used={}", kernel_symbol)); + options.emplace_back("-optimize-unused-variables"); + options.emplace_back("-split-compile=0"); + + std::vector options_cstr; + for (auto const& option : options) { + options_cstr.emplace_back(option.c_str()); + } + + rtc::blob_view link_fragments[] = {library->get(rtc::binary_type::FATBIN)->view(), + fragment->get(rtc::binary_type::LTO_IR)->view()}; + + rtc::binary_type fragment_binary_types[] = {rtc::binary_type::FATBIN, rtc::binary_type::LTO_IR}; + + char const* fragment_names[] = {"cudf_lto_library", name}; + + auto params = rtc::library_t::link_params{.name = name, + .output_type = rtc::binary_type::CUBIN, + .fragments = link_fragments, + .fragment_binary_types = fragment_binary_types, + .fragment_names = fragment_names, + .link_options = options_cstr}; + + auto blob = rtc::library_t::link_as_blob(params); + + auto load_params = + rtc::library_t::load_params{.binary = blob->view(), .type = rtc::binary_type::CUBIN}; + + auto linked_library = rtc::library_t::load(load_params); + + auto end = std::chrono::steady_clock::now(); + + auto duration = end - begin; + + CUDF_LOG_WARN( + "Linked fragment `%s` in %f ms", + name, + std::chrono::duration_cast>(duration).count()); + + return std::make_tuple(linked_library, blob); +} + +rtc::fragment compile_udf(char const* name, + char const* key, + char const* cuda_udf, + bool use_cache, + bool use_pch, + bool log_pch) +{ + CUDF_FUNC_RANGE(); + auto& cache = cudf::get_context().rtc_cache(); + auto& bundle = cudf::get_context().jit_bundle(); auto runtime = get_runtime_version(); auto driver = get_driver_version(); @@ -359,61 +499,9 @@ bundle={})***", auto cache_key_sha256 = hash_string(cache_key); - auto compile = [&] { - auto begin = std::chrono::steady_clock::now(); - - auto include_dirs = bundle.get_include_directories(); - auto compile_options = bundle.get_compile_options(); - - std::vector options; - - for (auto const& include_dir : include_dirs) { - options.emplace_back(std::format("-I{}", include_dir)); - } - - for (auto const& compile_option : compile_options) { - options.emplace_back(compile_option); - } - - options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); - options.emplace_back("--dlink-time-opt"); - options.emplace_back("--relocatable-device-code=true"); - options.emplace_back("--device-as-default-execution-space"); - - // TODO: experiment with: - // --split-compile=0 - // --fdevice-time-trace=jit_comp_trace.json - // --minimal - // --time=compile_trace.json - // -time - // --fast-compile - // --pch - // --pch-dir=/tmp/cudf-rtc-pch - - std::vector options_cstr; - for (auto const& option : options) { - options_cstr.emplace_back(option.c_str()); - } - - auto params = rtc::fragment_t::compile_params{.name = name, - .source = source_code_cstr, - .headers = {}, - .options = options_cstr, - .target_type = rtc::binary_type::LTO_IR}; + auto compile = [&] { return compile_udf_uncached(name, cuda_udf, use_pch, log_pch); }; - auto frag = rtc::fragment_t::compile(params); - - auto end = std::chrono::steady_clock::now(); - - auto duration = end - begin; - - CUDF_LOG_INFO( - "Compiled fragment `{}` in {} ms", - name, - std::chrono::duration_cast>(duration).count()); - - return frag; - }; + if (!use_cache) { return compile(); } auto fut = cache.query_or_insert_fragment(cache_key_sha256, @@ -423,10 +511,11 @@ bundle={})***", return fut.get(); } -rtc::library compile_and_link_udf(char const* name, - char const* udf_code, - char const* udf_key, - char const* kernel_symbol) +rtc::library link_udf(char const* name, + char const* key, + rtc::fragment const& fragment, + char const* kernel_symbol, + bool use_cache) { CUDF_FUNC_RANGE(); @@ -438,90 +527,78 @@ rtc::library compile_and_link_udf(char const* name, auto sm = get_current_device_physical_model(); auto bundle_hash = bundle.get_hash(); - auto compile = [&] { - auto begin = std::chrono::steady_clock::now(); - auto library = bundle.get_lto_library(); - auto fragment = get_or_compile_fragment(name, udf_code, udf_key); - - // TODO: sass dump - // TODO: time trace dump - - // TODO: experiment with: - // optimization flags - // split-compile - // split-compile-extended - // lineinfo and debug info options - // -kernels-used= - // env variable to control options - // fma - // variables-used - // -optimize-unused-variables - // -nocache - // -device-stack-protector - - std::vector options; - - options.emplace_back("-lto"); - options.emplace_back(std::format("-arch=sm_{}", sm)); - options.emplace_back(std::format("-kernels-used={}", kernel_symbol)); - - std::vector options_cstr; - for (auto const& option : options) { - options_cstr.emplace_back(option.c_str()); - } - - rtc::blob_view link_fragments[] = {library->get(rtc::binary_type::FATBIN)->view(), - fragment->get(rtc::binary_type::LTO_IR)->view()}; - - rtc::binary_type fragment_binary_types[] = {rtc::binary_type::FATBIN, rtc::binary_type::LTO_IR}; - - char const* fragment_names[] = {"cudf_lto_library", name}; + auto cache_key = std::format(R"***(library_type=CUBIN +key={} +kernel={} +cuda_runtime={} +cuda_driver={} +arch={}, +bundle={})***", + key, + kernel_symbol, + runtime, + driver, + sm, + bundle_hash); - auto params = rtc::library_t::link_params{.name = name, - .output_type = rtc::binary_type::CUBIN, - .fragments = link_fragments, - .fragment_binary_types = fragment_binary_types, - .fragment_names = fragment_names, - .link_options = options_cstr}; + auto cache_key_sha256 = hash_string(cache_key); - auto blob = rtc::library_t::link_as_blob(params); + auto link = [&] { return link_udf_uncached(name, fragment, kernel_symbol); }; - auto load_params = - rtc::library_t::load_params{.binary = blob->view(), .type = rtc::binary_type::CUBIN}; + if (!use_cache) { + auto [lib, blob] = link(); + return lib; + } - auto linked_library = rtc::library_t::load(load_params); + auto fut = cache.query_or_insert_library( + cache_key_sha256, rtc::binary_type::CUBIN, rtc::library_compile_function_t::from_functor(link)); - auto end = std::chrono::steady_clock::now(); + return fut.get(); +} - auto duration = end - begin; +rtc::library compile_cuda_library(char const* name, + char const* key, + char const* cuda_udf, + char const* kernel_symbol, + bool use_cache, + bool use_pch, + bool log_pch) +{ + CUDF_FUNC_RANGE(); + auto fragment = compile_udf(name, key, cuda_udf, use_cache, use_pch, log_pch); + auto library = link_udf(name, key, fragment, kernel_symbol, use_cache); + return library; +} - CUDF_LOG_INFO( - "Compiled fragment `{}` in {} ms", - name, - std::chrono::duration_cast>(duration).count()); +} // namespace - return std::make_tuple(linked_library, blob); - }; +rtc::library compile_kernel(std::string const& name, + std::string const& key, + std::string const& cuda_udf, + std::string const& kernel_symbol, + bool use_cache, + bool use_pch, + bool log_pch) +{ + return compile_cuda_library(name.c_str(), + key.c_str(), + cuda_udf.c_str(), + kernel_symbol.c_str(), + use_cache, + use_pch, + log_pch); +} - auto library_cache_key = std::format(R"***(library_type=CUBIN -kernels={} -udf={} -cuda_runtime={} -cuda_driver={} -arch={})***", - kernel_symbol, - udf_key, - runtime, - driver, - sm); - auto library_cache_key_sha256 = hash_string(library_cache_key); - - auto library = - cache.query_or_insert_library(library_cache_key_sha256, - rtc::binary_type::CUBIN, - rtc::library_compile_function_t::from_functor(compile)); - - return library.get(); +rtc::library compile_lto_ir_kernel(std::string const& name, + std::string const& key, + std::span lto_ir_binary, + std::string const& kernel_symbol, + bool use_cache) +{ + auto blob = std::make_shared(rtc::blob_t::from_static_data(lto_ir_binary)); + auto fragment = rtc::fragment_t::load( + rtc::fragment_t::load_params{.binary = blob, .type = rtc::binary_type::LTO_IR}); + return link_udf(name.c_str(), key.c_str(), fragment, kernel_symbol.c_str(), use_cache); } } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index 758c0202d969..edbf1c454c7a 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -36,36 +36,18 @@ struct [[nodiscard]] jit_bundle_t { [[nodiscard]] std::vector get_compile_options() const; }; -struct [[nodiscard]] udf_compile_params { - std::string_view name = {}; - - std::span udf = {}; - - std::string_view key = {}; - - std::string_view kernel_symbol = {}; - - std::span extra_compile_flags = {}; - - std::span extra_link_flags = {}; -}; - -[[nodiscard]] rtc::library compile_and_link_cuda_udf(udf_compile_params const& params); - -struct [[nodiscard]] udf_link_params { - std::string_view name = {}; - - std::span udf_blob = {}; - - rtc::binary_type type = rtc::binary_type::LTO_IR; - - std::string_view key = {}; - - std::string_view kernel_symbol = {}; - - std::span extra_link_flags = {}; -}; - -[[nodiscard]] rtc::library link_udf(udf_link_params const& params); +[[nodiscard]] rtc::library compile_kernel(std::string const& name, + std::string const& key, + std::string const& cuda_udf, + std::string const& kernel_symbol, + bool use_cache = true, + bool use_pch = true, + bool log_pch = false); + +[[nodiscard]] rtc::library compile_lto_ir_kernel(std::string const& name, + std::string const& key, + std::span lto_ir_binary, + std::string const& kernel_symbol, + bool use_cache = true); } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/lto/column_view.def.cuh b/cpp/src/jit/lto/column_view.def.cuh index bd4df0b3b3a5..2c6bd57c5705 100644 --- a/cpp/src/jit/lto/column_view.def.cuh +++ b/cpp/src/jit/lto/column_view.def.cuh @@ -11,175 +11,171 @@ namespace CUDF_LTO_EXPORT cudf { namespace lto { template -__device__ T const* column_device_view_core::head() const +__device__ T const* column_device_view::head() const { return lower(lift(this)->head>()); } -__device__ size_type column_device_view_core::size() const { return lift(this)->size(); } +__device__ size_type column_device_view::size() const { return lift(this)->size(); } -__device__ bool column_device_view_core::nullable() const { return lift(this)->nullable(); } +__device__ bool column_device_view::nullable() const { return lift(this)->nullable(); } -__device__ bitmask_type const* column_device_view_core::null_mask() const +__device__ bitmask_type const* column_device_view::null_mask() const { return lift(this)->null_mask(); } -__device__ size_type column_device_view_core::offset() const { return lift(this)->offset(); } +__device__ size_type column_device_view::offset() const { return lift(this)->offset(); } -__device__ bool column_device_view_core::is_valid(size_type index) const +__device__ bool column_device_view::is_valid(size_type index) const { return lift(this)->is_valid(index); } -__device__ bool column_device_view_core::is_valid_nocheck(size_type index) const +__device__ bool column_device_view::is_valid_nocheck(size_type index) const { return lift(this)->is_valid_nocheck(index); } -__device__ bool column_device_view_core::is_null(size_type index) const +__device__ bool column_device_view::is_null(size_type index) const { return lift(this)->is_null(index); } -__device__ bool column_device_view_core::is_null_nocheck(size_type index) const +__device__ bool column_device_view::is_null_nocheck(size_type index) const { return lift(this)->is_null_nocheck(index); } -__device__ bitmask_type column_device_view_core::get_mask_word(size_type index) const +__device__ bitmask_type column_device_view::get_mask_word(size_type index) const { return lift(this)->get_mask_word(index); } template -__device__ T column_device_view_core::element(size_type index) const +__device__ T column_device_view::element(size_type index) const { auto ret = lift(this)->element>(index); return *lower(&ret); } template -__device__ optional column_device_view_core::nullable_element(size_type index) const +__device__ optional column_device_view::nullable_element(size_type index) const { auto ret = lift(this)->nullable_element>(index); return *lower(&ret); } -__device__ size_type column_device_view_core::num_child_columns() const +__device__ size_type column_device_view::num_child_columns() const { return lift(this)->num_child_columns(); } template -__device__ T* mutable_column_device_view_core::head() const +__device__ T* mutable_column_device_view::head() const { return lower(lift(this)->head>()); } -__device__ size_type mutable_column_device_view_core::size() const { return lift(this)->size(); } +__device__ size_type mutable_column_device_view::size() const { return lift(this)->size(); } -__device__ bool mutable_column_device_view_core::nullable() const { return lift(this)->nullable(); } +__device__ bool mutable_column_device_view::nullable() const { return lift(this)->nullable(); } -__device__ bitmask_type* mutable_column_device_view_core::null_mask() const +__device__ bitmask_type* mutable_column_device_view::null_mask() const { return lift(this)->null_mask(); } -__device__ size_type mutable_column_device_view_core::offset() const -{ - return lift(this)->offset(); -} +__device__ size_type mutable_column_device_view::offset() const { return lift(this)->offset(); } -__device__ bool mutable_column_device_view_core::is_valid(size_type index) const +__device__ bool mutable_column_device_view::is_valid(size_type index) const { return lift(this)->is_valid(index); } -__device__ bool mutable_column_device_view_core::is_valid_nocheck(size_type index) const +__device__ bool mutable_column_device_view::is_valid_nocheck(size_type index) const { return lift(this)->is_valid_nocheck(index); } -__device__ bool mutable_column_device_view_core::is_null(size_type index) const +__device__ bool mutable_column_device_view::is_null(size_type index) const { return lift(this)->is_null(index); } -__device__ bool mutable_column_device_view_core::is_null_nocheck(size_type index) const +__device__ bool mutable_column_device_view::is_null_nocheck(size_type index) const { return lift(this)->is_null_nocheck(index); } -__device__ bitmask_type mutable_column_device_view_core::get_mask_word(size_type index) const +__device__ bitmask_type mutable_column_device_view::get_mask_word(size_type index) const { return lift(this)->get_mask_word(index); } template -__device__ T mutable_column_device_view_core::element(size_type index) const +__device__ T mutable_column_device_view::element(size_type index) const { auto ret = lift(this)->element>(index); return *lower(&ret); } template -__device__ optional mutable_column_device_view_core::nullable_element(size_type index) const +__device__ optional mutable_column_device_view::nullable_element(size_type index) const { auto ret = lift(this)->nullable_element>(index); return *lower(&ret); } template -__device__ void mutable_column_device_view_core::assign(size_type index, T value) const +__device__ void mutable_column_device_view::assign(size_type index, T value) const { lift(this)->assign>(index, *lift(&value)); } /// Explicit template instantiations -#define DO_IT(T) template __device__ T const* column_device_view_core::head() const; +#define DO_IT(T) template __device__ T const* column_device_view::head() const; FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE #undef DO_IT -#define DO_IT(T) template __device__ T column_device_view_core::element(size_type idx) const; +#define DO_IT(T) template __device__ T column_device_view::element(size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT #define DO_IT(T) \ - template __device__ optional column_device_view_core::nullable_element(size_type idx) const; + template __device__ optional column_device_view::nullable_element(size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -#define DO_IT(T) template __device__ T* mutable_column_device_view_core::head() const; +#define DO_IT(T) template __device__ T* mutable_column_device_view::head() const; FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE #undef DO_IT -#define DO_IT(T) \ - template __device__ T mutable_column_device_view_core::element(size_type idx) const; +#define DO_IT(T) template __device__ T mutable_column_device_view::element(size_type idx) const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT -#define DO_IT(T) \ - template __device__ optional mutable_column_device_view_core::nullable_element( \ - size_type idx) const; +#define DO_IT(T) \ + template __device__ optional mutable_column_device_view::nullable_element(size_type idx) \ + const; FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE #undef DO_IT #define DO_IT(T) \ - template __device__ void mutable_column_device_view_core::assign(size_type idx, T value) const; + template __device__ void mutable_column_device_view::assign(size_type idx, T value) const; FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 7c2e993aa192..446957eb05f5 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -386,7 +386,6 @@ void kernel_ref::launch(uint32_t grid_dim_x, CUDF_EXPECTS(block_dim_x > 0 && block_dim_y > 0 && block_dim_z > 0, "Block dimensions must be greater than zero", std::logic_error); - CUDF_EXPECTS(stream != nullptr, "CUDA stream must not be null", std::logic_error); CUDF_EXPECTS( kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index d9ce67ebf0fa..7ba5c582b3ef 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -49,6 +49,7 @@ void context::ensure_jit_bundle_initialized() std::call_once(_jit_bundle_init_flag, [&]() { // make sure the jit bundle directory exists std::filesystem::create_directories(_config.jit_bundle_dir); + std::filesystem::create_directories(_config.jit_pch_dir); _jit_bundle = std::make_unique(_config.jit_bundle_dir, *_rtc_cache); }); } @@ -75,6 +76,8 @@ bool context::dump_codegen() const { return _config.dump_codegen; } bool context::use_jit() const { return _config.use_jit; } +std::string const& context::get_jit_pch_dir() const { return _config.jit_pch_dir; } + void context::initialize_components(init_flags flags) { if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { ensure_jit_cache_initialized(); } @@ -113,6 +116,11 @@ std::filesystem::path get_rtc_cache_dir() return getenv_or("LIBCUDF_RTC_CACHE_DIR", get_cudf_dir() / "rtc_cache"); } +std::filesystem::path get_jit_pch_dir() +{ + return getenv_or("LIBCUDF_JIT_PCH_DIR", get_cudf_dir() / "jit_pch"); +} + } // namespace cudf namespace CUDF_EXPORT cudf { @@ -127,12 +135,14 @@ void initialize(init_flags flags) auto jit_bundle_dir = get_jit_bundle_dir(); auto rtc_cache_dir = get_rtc_cache_dir(); + auto jit_pch_dir = get_jit_pch_dir(); context_config cfg{ .dump_codegen = dump_codegen, .use_jit = use_jit, .rtc_cache_dir = rtc_cache_dir, .jit_bundle_dir = jit_bundle_dir, + .jit_pch_dir = jit_pch_dir, }; _context.emplace(cfg, flags); diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index ae81039be059..f26fb8c08633 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -28,6 +28,7 @@ struct [[nodiscard]] context_config { bool use_jit = false; std::string rtc_cache_dir = {}; std::string jit_bundle_dir = {}; + std::string jit_pch_dir = {}; }; /// @brief The context object contains global state internal to CUDF. @@ -70,6 +71,8 @@ class context { [[nodiscard]] bool use_jit() const; + [[nodiscard]] std::string const& get_jit_pch_dir() const; + /// @brief Initialize additional components based on the provided flags /// @param flags The initialization flags to process void initialize_components(init_flags flags); diff --git a/cpp/src/transform/jit/lto_kernel.inl.cuh b/cpp/src/transform/jit/lto_kernel.inl.cuh index 3bc2a734d45a..da5a4c77161b 100644 --- a/cpp/src/transform/jit/lto_kernel.inl.cuh +++ b/cpp/src/transform/jit/lto_kernel.inl.cuh @@ -12,7 +12,7 @@ extern "C" { __device__ void transform_operator(cudf::lto::transform_params params); -__global__ void transform_kernel(void* __restrict__ const* __restrict__ scope, int32_t num_rows) +__global__ void transform_kernel(void* const* scope, int32_t num_rows) { auto start = cudf::detail::grid_1d::global_thread_id(); auto stride = cudf::detail::grid_1d::grid_stride(); diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index 796cf421fcf0..8e36546b24e5 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -37,32 +37,134 @@ struct element_operation { }; */ -// TODO: cache control policy: ignore cache -// TODO: statistics callback, in cudf layer? -// TODO: extra compile and link flags for LTO library and final kernel, use in CUDF layer, not -// here? -// -// TODO: ?? -// TODO: add time function in cache -// TODO: take optional pointer/callback to this for it to be written to -// TODO: take optional list of extra compile flags for linking and compiling -// TODO: cache hit statistics -// global cache statistics before and after? -// TODO: add configuration parameters necessary for testing -// and profiling cache behaviour -// TODO: declare getters and setters required for the specific LTO context of the operator -// they should use the provided LTO functions; might need a planner -// TODO: bincode dump arguments? -// TODO: add a bypass cache argument that forces recompilation for testing purposes - // TODO: include all headers and disable them based on a macro when not needed for compilation -struct jit_compile_stats { - std::chrono::nanoseconds cpp_compile_time{}; - std::chrono::nanoseconds fragment_link_time{}; - std::chrono::nanoseconds total_time{}; -}; +// TODO: write a planner +// TODO: flags to clear JIT at program startup +// TODO: nvrtc uses the program name to do PCH + +TEST_F(RTCTest, CompileKernelBasic) +{ + auto fn = []() { + char const* udf = R"***( + #include "cudf/jit/lto/column_view.cuh" + #include "cudf/jit/lto/operators.cuh" + #include "cudf/jit/lto/optional_span.cuh" + #include "cudf/jit/lto/optional.cuh" + #include "cudf/jit/lto/scope.cuh" + #include "cudf/jit/lto/span.cuh" + #include "cudf/jit/lto/string_view.cuh" + #include "cudf/jit/lto/transform_params.cuh" + #include "cudf/jit/lto/types.cuh" + + #pragma nv_hdrstop + + extern "C" __device__ void transform_operator(cudf::lto::transform_params p){ + using namespace cudf::lto; + + // unpack inputs from scope using the appropriate getters based on the LTO context + using s0 = scope::column<0, column_device_view, int, false, false>; + using s1 = scope::column<1, column_device_view, int, false, false>; + using s2 = scope::column<2, mutable_column_device_view, int, false, false>; + + auto a0 = s0::element(p.scope, p.row_index); + auto a1 = s1::element(p.scope, p.row_index); + int a2; + + operators::add(&a2, &a0, &a1); -TEST_F(RTCTest, CreateFragmentBasic) {} + s2::assign(p.scope, p.row_index, a2); + } + )***"; + static int i = 0; + + i++; + auto key = std::format("test_udf_key_{}", i); + auto lib = cudf::compile_kernel("test_fragment", + key, + udf, + "transform_kernel", + /*use_cache=*/true, + /*use_pch=*/true, + /*log_pch=*/true); + + auto kernel = lib->get_kernel("transform_kernel"); + + EXPECT_EQ("transform_kernel", kernel.get_name()); + + auto in0 = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9}.release(); + auto in1 = cudf::test::fixed_width_column_wrapper{9, 8, 7, 6, 5, 4, 3, 2, 1}.release(); + auto out = cudf::test::fixed_width_column_wrapper{0, 0, 0, 0, 0, 0, 0, 0, 0}.release(); + int32_t num_rows = 9; + + auto to_device_view = [](auto const& view) { + std::vector h_view{view}; + rmm::device_uvector device_view(1, rmm::cuda_stream_default); + cudf::detail::cuda_memcpy_async( + device_view, h_view, rmm::cuda_stream_default); + return device_view; + }; + + auto to_device_mutable_view = [](auto& view) { + std::vector h_view{view}; + rmm::device_uvector device_view(1, + rmm::cuda_stream_default); + cudf::detail::cuda_memcpy_async( + device_view, h_view, rmm::cuda_stream_default); + return device_view; + }; + + auto h_in0 = cudf::column_device_view::create(in0->view()); + auto h_in1 = cudf::column_device_view::create(in1->view()); + auto h_out = cudf::mutable_column_device_view::create(out->mutable_view()); + auto d_in0 = to_device_view(*h_in0); + auto d_in1 = to_device_view(*h_in1); + auto d_out = to_device_mutable_view(*h_out); + auto d_in0_ptr = d_in0.data(); + auto d_in1_ptr = d_in1.data(); + auto d_out_ptr = d_out.data(); + + rmm::device_buffer d_scope{sizeof(cudf::column_device_view*) + + sizeof(cudf::column_device_view*) + + sizeof(cudf::mutable_column_device_view*), + rmm::cuda_stream_default}; + + auto* p = static_cast(d_scope.data()); + + detail::cuda_memcpy_async_impl(p, + &d_in0_ptr, + sizeof(cudf::column_device_view*), + detail::host_memory_kind::PAGEABLE, + rmm::cuda_stream_default); + detail::cuda_memcpy_async_impl(p + 1, + &d_in1_ptr, + sizeof(cudf::column_device_view*), + detail::host_memory_kind::PAGEABLE, + rmm::cuda_stream_default); + detail::cuda_memcpy_async_impl(p + 2, + &d_out_ptr, + sizeof(cudf::mutable_column_device_view*), + detail::host_memory_kind::PAGEABLE, + rmm::cuda_stream_default); + + auto* scope_arg = d_scope.data(); + + void* args[] = {&scope_arg, &num_rows}; + + kernel.launch(1, 1, 1, 256, 1, 1, 0, cudaStreamDefault, args); + + auto expected = + cudf::test::fixed_width_column_wrapper{10, 10, 10, 10, 10, 10, 10, 10, 10}.release(); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(out->view(), expected->view()); + }; + + fn(); // warm up cache + fn(); + fn(); + fn(); +} + +/* TEST_F(RTCTest, CreateFragment) { auto fn = []() { @@ -70,7 +172,7 @@ TEST_F(RTCTest, CreateFragment) #include "cudf/jit/lto/transform_params.cuh" #include "cudf/jit/lto/operators.cuh" #include "cudf/jit/lto/scope.cuh" - #include "cudf/jit/lto/column_view.cuh" // for column_view_core, mutable_column_view_core + #include "cudf/jit/lto/column_view.cuh" // for column_device_view, mutable_column_device_view // if we detect that all types are simple types // we can exclude some of the getters, setters, and operators @@ -84,17 +186,17 @@ TEST_F(RTCTest, CreateFragment) // unpack inputs from scope using the appropriate getters based on the LTO context using s0 = scope::user_data<0>; - using s1 = scope::column<1, column_view, int, false, false>; - using s2 = scope::column<2, column_view, int, false, false>; - using s3 = scope::column<3, column_view, double, false, false>; - using s4 = scope::column<4, column_view, float, false, false>; - using s4 = scope::column<4, string_view, float, false, false>; - using s4 = scope::column<4, decimal32, float, false, false>; - using s4 = scope::column<5, column_view, float, false, true>; - using s4 = scope::column<6, column_view, float, true, false>; + using s1 = scope::column<1, column_device_view, int, false, false>; + using s2 = scope::column<2, column_device_view, int, false, false>; + using s3 = scope::column<3, column_device_view, double, false, false>; + using s4 = scope::column<4, column_device_view, float, false, false>; + using s4 = scope::column<4, column_device_view, string_view, false, false>; + using s4 = scope::column<4, column_device_view, decimal32, false, false>; + using s4 = scope::column<5, column_device_view, float, false, true>; + using s4 = scope::column<6, column_device_view, float, true, false>; using s5 = scope::column<7, span, float, false, false>; using s6 = scope::column<8, optional_span, float, false, true>; - using s7 = scope::column<9, mutable_column_view, double, false, false>; + using s7 = scope::column<9, mutable_column_device_view, double, false, false>; auto a0 = s0::element(p.scope, p.row_index); auto a1 = s1::element(p.scope, p.row_index); @@ -114,7 +216,8 @@ TEST_F(RTCTest, CreateFragment) /// <-- BEGIN OF OPERATOR: Derived from user - // run operation using the LTO-compiled operators; these should be inlined into the final kernel and optimized together by NVJITLink + // run operation using the LTO-compiled operators; these should be inlined into the final +kernel and optimized together by NVJITLink ops::add(&c, &a, &b); ops::sub(&c, &a, &b); @@ -162,5 +265,6 @@ TEST_F(RTCTest, CreateFragment) fn(); fn(); } +*/ CUDF_TEST_PROGRAM_MAIN() diff --git a/jit.md b/jit.md deleted file mode 100644 index 6e18b46d2386..000000000000 --- a/jit.md +++ /dev/null @@ -1 +0,0 @@ -# Explain JIT compilation flow as it is in CUDF right now diff --git a/lto-ir-notes.txt b/lto-ir-notes.txt deleted file mode 100644 index f5ab9944f873..000000000000 --- a/lto-ir-notes.txt +++ /dev/null @@ -1 +0,0 @@ -Not compatible across major versions diff --git a/todo.md b/todo.md deleted file mode 100644 index 29922bc7b073..000000000000 --- a/todo.md +++ /dev/null @@ -1,2 +0,0 @@ -[ ] Use PCH -[ ] Add time trace to JIT and AOT From 6f829acb9c732a7cab3dd3145a8fd41143d5c7b9 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 20 Feb 2026 13:53:01 +0000 Subject: [PATCH 055/254] started making operators AST-based --- cpp/include/cudf/jit/lto/operators.cuh | 931 +++++++++++++------------ cpp/include/cudf/jit/lto/types.cuh | 204 ++++++ 2 files changed, 686 insertions(+), 449 deletions(-) diff --git a/cpp/include/cudf/jit/lto/operators.cuh b/cpp/include/cudf/jit/lto/operators.cuh index 135d20f5c155..d103561f7791 100644 --- a/cpp/include/cudf/jit/lto/operators.cuh +++ b/cpp/include/cudf/jit/lto/operators.cuh @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ #pragma once +#include +#include #include namespace CUDF_LTO_EXPORT cudf { @@ -11,455 +13,486 @@ namespace lto { namespace operators { -#define CUDF_LTO_DECL(op, type) \ - __device__ extern void op(type* out, type const* a, type const* b); \ - \ - __device__ extern void op(optional* out, optional const* a, optional const* b) - -CUDF_LTO_DECL(add, int32_t); -CUDF_LTO_DECL(add, int64_t); -CUDF_LTO_DECL(add, uint32_t); -CUDF_LTO_DECL(add, uint64_t); -CUDF_LTO_DECL(add, float32_t); -CUDF_LTO_DECL(add, float64_t); -CUDF_LTO_DECL(add, decimal32); -CUDF_LTO_DECL(add, decimal64); -CUDF_LTO_DECL(add, decimal128); -CUDF_LTO_DECL(add, duration_D); -CUDF_LTO_DECL(add, duration_s); -CUDF_LTO_DECL(add, duration_ms); -CUDF_LTO_DECL(add, duration_ns); - -CUDF_LTO_DECL(sub, int32_t); -CUDF_LTO_DECL(sub, int64_t); -CUDF_LTO_DECL(sub, uint32_t); -CUDF_LTO_DECL(sub, uint64_t); -CUDF_LTO_DECL(sub, float32_t); -CUDF_LTO_DECL(sub, float64_t); -CUDF_LTO_DECL(sub, decimal32); -CUDF_LTO_DECL(sub, decimal64); -CUDF_LTO_DECL(sub, decimal128); -CUDF_LTO_DECL(sub, duration_D); -CUDF_LTO_DECL(sub, duration_s); -CUDF_LTO_DECL(sub, duration_ms); -CUDF_LTO_DECL(sub, duration_ns); - -CUDF_LTO_DECL(mul, int32_t); -CUDF_LTO_DECL(mul, int64_t); -CUDF_LTO_DECL(mul, uint32_t); -CUDF_LTO_DECL(mul, uint64_t); -CUDF_LTO_DECL(mul, float32_t); -CUDF_LTO_DECL(mul, float64_t); -CUDF_LTO_DECL(mul, decimal32); -CUDF_LTO_DECL(mul, decimal64); -CUDF_LTO_DECL(mul, decimal128); - -CUDF_LTO_DECL(div, int32_t); -CUDF_LTO_DECL(div, int64_t); -CUDF_LTO_DECL(div, uint32_t); -CUDF_LTO_DECL(div, uint64_t); -CUDF_LTO_DECL(div, float32_t); -CUDF_LTO_DECL(div, float64_t); -CUDF_LTO_DECL(div, decimal32); -CUDF_LTO_DECL(div, decimal64); -CUDF_LTO_DECL(div, decimal128); - -CUDF_LTO_DECL(mod, float32_t); -CUDF_LTO_DECL(mod, float64_t); - -CUDF_LTO_DECL(pymod, float32_t); -CUDF_LTO_DECL(pymod, float64_t); - -CUDF_LTO_DECL(pow, float32_t); -CUDF_LTO_DECL(pow, float64_t); - -#undef CUDF_LTO_DECL - -#define CUDF_LTO_DECL(op, type) \ - __device__ extern void op(bool* out, type const* a, type const* b); \ - \ - __device__ extern void op(bool* out, optional const* a, optional const* b); \ - \ - __device__ extern void op(optional* out, optional const* a, optional const* b) - -CUDF_LTO_DECL(equal, bool); -CUDF_LTO_DECL(equal, int8_t); -CUDF_LTO_DECL(equal, int16_t); -CUDF_LTO_DECL(equal, int32_t); -CUDF_LTO_DECL(equal, int64_t); -CUDF_LTO_DECL(equal, uint8_t); -CUDF_LTO_DECL(equal, uint16_t); -CUDF_LTO_DECL(equal, uint32_t); -CUDF_LTO_DECL(equal, uint64_t); -CUDF_LTO_DECL(equal, float32_t); -CUDF_LTO_DECL(equal, float64_t); -CUDF_LTO_DECL(equal, decimal32); -CUDF_LTO_DECL(equal, decimal64); -CUDF_LTO_DECL(equal, decimal128); -CUDF_LTO_DECL(equal, timestamp_D); -CUDF_LTO_DECL(equal, timestamp_s); -CUDF_LTO_DECL(equal, timestamp_ms); -CUDF_LTO_DECL(equal, timestamp_us); -CUDF_LTO_DECL(equal, timestamp_ns); -CUDF_LTO_DECL(equal, duration_D); -CUDF_LTO_DECL(equal, duration_s); -CUDF_LTO_DECL(equal, duration_ms); -CUDF_LTO_DECL(equal, duration_ns); -CUDF_LTO_DECL(equal, string_view); - -CUDF_LTO_DECL(null_equal, bool); -CUDF_LTO_DECL(null_equal, int8_t); -CUDF_LTO_DECL(null_equal, int16_t); -CUDF_LTO_DECL(null_equal, int32_t); -CUDF_LTO_DECL(null_equal, int64_t); -CUDF_LTO_DECL(null_equal, uint8_t); -CUDF_LTO_DECL(null_equal, uint16_t); -CUDF_LTO_DECL(null_equal, uint32_t); -CUDF_LTO_DECL(null_equal, uint64_t); -CUDF_LTO_DECL(null_equal, float32_t); -CUDF_LTO_DECL(null_equal, float64_t); -CUDF_LTO_DECL(null_equal, decimal32); -CUDF_LTO_DECL(null_equal, decimal64); -CUDF_LTO_DECL(null_equal, decimal128); -CUDF_LTO_DECL(null_equal, timestamp_D); -CUDF_LTO_DECL(null_equal, timestamp_s); -CUDF_LTO_DECL(null_equal, timestamp_ms); -CUDF_LTO_DECL(null_equal, timestamp_us); -CUDF_LTO_DECL(null_equal, timestamp_ns); -CUDF_LTO_DECL(null_equal, duration_D); -CUDF_LTO_DECL(null_equal, duration_s); -CUDF_LTO_DECL(null_equal, duration_ms); -CUDF_LTO_DECL(null_equal, duration_ns); -CUDF_LTO_DECL(null_equal, string_view); - -#undef CUDF_LTO_DECL - -#define CUDF_LTO_DECL(op, type) \ - __device__ extern void op(bool* out, type const* a, type const* b); \ - \ - __device__ extern void op(optional* out, optional const* a, optional const* b) - -CUDF_LTO_DECL(less, bool); -CUDF_LTO_DECL(less, int8_t); -CUDF_LTO_DECL(less, int16_t); -CUDF_LTO_DECL(less, int32_t); -CUDF_LTO_DECL(less, int64_t); -CUDF_LTO_DECL(less, uint8_t); -CUDF_LTO_DECL(less, uint16_t); -CUDF_LTO_DECL(less, uint32_t); -CUDF_LTO_DECL(less, uint64_t); -CUDF_LTO_DECL(less, float32_t); -CUDF_LTO_DECL(less, float64_t); -CUDF_LTO_DECL(less, decimal32); -CUDF_LTO_DECL(less, decimal64); -CUDF_LTO_DECL(less, decimal128); -CUDF_LTO_DECL(less, timestamp_D); -CUDF_LTO_DECL(less, timestamp_s); -CUDF_LTO_DECL(less, timestamp_ms); -CUDF_LTO_DECL(less, timestamp_us); -CUDF_LTO_DECL(less, timestamp_ns); -CUDF_LTO_DECL(less, duration_D); -CUDF_LTO_DECL(less, duration_s); -CUDF_LTO_DECL(less, duration_ms); -CUDF_LTO_DECL(less, duration_ns); -CUDF_LTO_DECL(less, string_view); - -CUDF_LTO_DECL(greater, bool); -CUDF_LTO_DECL(greater, int8_t); -CUDF_LTO_DECL(greater, int16_t); -CUDF_LTO_DECL(greater, int32_t); -CUDF_LTO_DECL(greater, int64_t); -CUDF_LTO_DECL(greater, uint8_t); -CUDF_LTO_DECL(greater, uint16_t); -CUDF_LTO_DECL(greater, uint32_t); -CUDF_LTO_DECL(greater, uint64_t); -CUDF_LTO_DECL(greater, float32_t); -CUDF_LTO_DECL(greater, float64_t); -CUDF_LTO_DECL(greater, decimal32); -CUDF_LTO_DECL(greater, decimal64); -CUDF_LTO_DECL(greater, decimal128); -CUDF_LTO_DECL(greater, timestamp_D); -CUDF_LTO_DECL(greater, timestamp_s); -CUDF_LTO_DECL(greater, timestamp_ms); -CUDF_LTO_DECL(greater, timestamp_us); -CUDF_LTO_DECL(greater, timestamp_ns); -CUDF_LTO_DECL(greater, duration_D); -CUDF_LTO_DECL(greater, duration_s); -CUDF_LTO_DECL(greater, duration_ms); -CUDF_LTO_DECL(greater, duration_ns); -CUDF_LTO_DECL(greater, string_view); - -CUDF_LTO_DECL(less_equal, bool); -CUDF_LTO_DECL(less_equal, int8_t); -CUDF_LTO_DECL(less_equal, int16_t); -CUDF_LTO_DECL(less_equal, int32_t); -CUDF_LTO_DECL(less_equal, int64_t); -CUDF_LTO_DECL(less_equal, uint8_t); -CUDF_LTO_DECL(less_equal, uint16_t); -CUDF_LTO_DECL(less_equal, uint32_t); -CUDF_LTO_DECL(less_equal, uint64_t); -CUDF_LTO_DECL(less_equal, float32_t); -CUDF_LTO_DECL(less_equal, float64_t); -CUDF_LTO_DECL(less_equal, decimal32); -CUDF_LTO_DECL(less_equal, decimal64); -CUDF_LTO_DECL(less_equal, decimal128); -CUDF_LTO_DECL(less_equal, timestamp_D); -CUDF_LTO_DECL(less_equal, timestamp_s); -CUDF_LTO_DECL(less_equal, timestamp_ms); -CUDF_LTO_DECL(less_equal, timestamp_us); -CUDF_LTO_DECL(less_equal, timestamp_ns); -CUDF_LTO_DECL(less_equal, duration_D); -CUDF_LTO_DECL(less_equal, duration_s); -CUDF_LTO_DECL(less_equal, duration_ms); -CUDF_LTO_DECL(less_equal, duration_ns); -CUDF_LTO_DECL(less_equal, string_view); - -CUDF_LTO_DECL(greater_equal, bool); -CUDF_LTO_DECL(greater_equal, int8_t); -CUDF_LTO_DECL(greater_equal, int16_t); -CUDF_LTO_DECL(greater_equal, int32_t); -CUDF_LTO_DECL(greater_equal, int64_t); -CUDF_LTO_DECL(greater_equal, uint8_t); -CUDF_LTO_DECL(greater_equal, uint16_t); -CUDF_LTO_DECL(greater_equal, uint32_t); -CUDF_LTO_DECL(greater_equal, uint64_t); -CUDF_LTO_DECL(greater_equal, float32_t); -CUDF_LTO_DECL(greater_equal, float64_t); -CUDF_LTO_DECL(greater_equal, decimal32); -CUDF_LTO_DECL(greater_equal, decimal64); -CUDF_LTO_DECL(greater_equal, decimal128); -CUDF_LTO_DECL(greater_equal, timestamp_D); -CUDF_LTO_DECL(greater_equal, timestamp_s); -CUDF_LTO_DECL(greater_equal, timestamp_ms); -CUDF_LTO_DECL(greater_equal, timestamp_us); -CUDF_LTO_DECL(greater_equal, timestamp_ns); -CUDF_LTO_DECL(greater_equal, duration_D); -CUDF_LTO_DECL(greater_equal, duration_s); -CUDF_LTO_DECL(greater_equal, duration_ms); -CUDF_LTO_DECL(greater_equal, duration_ns); -CUDF_LTO_DECL(greater_equal, string_view); - -#undef CUDF_LTO_DECL - -#define CUDF_LTO_DECL(op, type) \ - __device__ extern void op(type* out, type const* a, type const* b); \ - \ - __device__ extern void op(optional* out, optional const* a, optional const* b) - -CUDF_LTO_DECL(bitwise_and, int32_t); -CUDF_LTO_DECL(bitwise_and, int64_t); -CUDF_LTO_DECL(bitwise_and, uint32_t); -CUDF_LTO_DECL(bitwise_and, uint64_t); - -CUDF_LTO_DECL(bitwise_or, int32_t); -CUDF_LTO_DECL(bitwise_or, int64_t); -CUDF_LTO_DECL(bitwise_or, uint32_t); -CUDF_LTO_DECL(bitwise_or, uint64_t); - -CUDF_LTO_DECL(bitwise_xor, int32_t); -CUDF_LTO_DECL(bitwise_xor, int64_t); -CUDF_LTO_DECL(bitwise_xor, uint32_t); -CUDF_LTO_DECL(bitwise_xor, uint64_t); - -#undef CUDF_LTO_DECL - -#define CUDF_LTO_DECL(op, type) \ - __device__ extern void op(type* out, type const* a, type const* b); \ - \ - __device__ extern void op(optional* out, optional const* a, optional const* b); - -CUDF_LTO_DECL(logical_and, bool); - -CUDF_LTO_DECL(null_logical_and, bool); - -CUDF_LTO_DECL(logical_or, bool); - -CUDF_LTO_DECL(null_logical_or, bool); - -#undef CUDF_LTO_DECL - -#define CUDF_LTO_DECL(op, type) \ - __device__ extern void op(type* out, type const* a); \ - \ - __device__ extern void op(optional* out, optional const* a) - -CUDF_LTO_DECL(identity, bool); -CUDF_LTO_DECL(identity, int8_t); -CUDF_LTO_DECL(identity, int16_t); -CUDF_LTO_DECL(identity, int32_t); -CUDF_LTO_DECL(identity, int64_t); -CUDF_LTO_DECL(identity, uint8_t); -CUDF_LTO_DECL(identity, uint16_t); -CUDF_LTO_DECL(identity, uint32_t); -CUDF_LTO_DECL(identity, uint64_t); -CUDF_LTO_DECL(identity, float32_t); -CUDF_LTO_DECL(identity, float64_t); -CUDF_LTO_DECL(identity, decimal32); -CUDF_LTO_DECL(identity, decimal64); -CUDF_LTO_DECL(identity, decimal128); -CUDF_LTO_DECL(identity, timestamp_D); -CUDF_LTO_DECL(identity, timestamp_s); -CUDF_LTO_DECL(identity, timestamp_ms); -CUDF_LTO_DECL(identity, timestamp_us); -CUDF_LTO_DECL(identity, timestamp_ns); -CUDF_LTO_DECL(identity, duration_D); -CUDF_LTO_DECL(identity, duration_s); -CUDF_LTO_DECL(identity, duration_ms); -CUDF_LTO_DECL(identity, duration_ns); -CUDF_LTO_DECL(identity, string_view); - -CUDF_LTO_DECL(sin, float32_t); -CUDF_LTO_DECL(sin, float64_t); - -CUDF_LTO_DECL(cos, float32_t); -CUDF_LTO_DECL(cos, float64_t); - -CUDF_LTO_DECL(tan, float32_t); -CUDF_LTO_DECL(tan, float64_t); - -CUDF_LTO_DECL(arcsin, float32_t); -CUDF_LTO_DECL(arcsin, float64_t); - -CUDF_LTO_DECL(arccos, float32_t); -CUDF_LTO_DECL(arccos, float64_t); - -CUDF_LTO_DECL(arctan, float32_t); -CUDF_LTO_DECL(arctan, float64_t); - -CUDF_LTO_DECL(sinh, float32_t); -CUDF_LTO_DECL(sinh, float64_t); - -CUDF_LTO_DECL(cosh, float32_t); -CUDF_LTO_DECL(cosh, float64_t); - -CUDF_LTO_DECL(tanh, float32_t); -CUDF_LTO_DECL(tanh, float64_t); - -CUDF_LTO_DECL(arcsinh, float32_t); -CUDF_LTO_DECL(arcsinh, float64_t); - -CUDF_LTO_DECL(arccosh, float32_t); -CUDF_LTO_DECL(arccosh, float64_t); - -CUDF_LTO_DECL(arctanh, float32_t); -CUDF_LTO_DECL(arctanh, float64_t); - -CUDF_LTO_DECL(exp, float32_t); -CUDF_LTO_DECL(exp, float64_t); - -CUDF_LTO_DECL(log, float32_t); -CUDF_LTO_DECL(log, float64_t); - -CUDF_LTO_DECL(cbrt, float32_t); -CUDF_LTO_DECL(cbrt, float64_t); - -CUDF_LTO_DECL(ceil, float32_t); -CUDF_LTO_DECL(ceil, float64_t); - -CUDF_LTO_DECL(floor, float32_t); -CUDF_LTO_DECL(floor, float64_t); - -CUDF_LTO_DECL(abs, int32_t); -CUDF_LTO_DECL(abs, int64_t); -CUDF_LTO_DECL(abs, float32_t); -CUDF_LTO_DECL(abs, float64_t); - -CUDF_LTO_DECL(rint, float32_t); -CUDF_LTO_DECL(rint, float64_t); - -CUDF_LTO_DECL(bit_invert, uint32_t); -CUDF_LTO_DECL(bit_invert, uint64_t); -CUDF_LTO_DECL(bit_invert, int32_t); -CUDF_LTO_DECL(bit_invert, int64_t); - -#undef CUDF_LTO_DECL - -#define CUDF_LTO_DECL(op, ret_type, type) \ - __device__ extern void op(ret_type* out, type const* a); \ - \ - __device__ extern void op(optional* out, optional const* a) - -CUDF_LTO_DECL(cast_to_int64, int64_t, bool); -CUDF_LTO_DECL(cast_to_int64, int64_t, int8_t); -CUDF_LTO_DECL(cast_to_int64, int64_t, int16_t); -CUDF_LTO_DECL(cast_to_int64, int64_t, int32_t); -CUDF_LTO_DECL(cast_to_int64, int64_t, int64_t); -CUDF_LTO_DECL(cast_to_int64, int64_t, uint8_t); -CUDF_LTO_DECL(cast_to_int64, int64_t, uint16_t); -CUDF_LTO_DECL(cast_to_int64, int64_t, uint32_t); -CUDF_LTO_DECL(cast_to_int64, int64_t, uint64_t); -CUDF_LTO_DECL(cast_to_int64, int64_t, float32_t); -CUDF_LTO_DECL(cast_to_int64, int64_t, float64_t); - -CUDF_LTO_DECL(cast_to_uint64, uint64_t, bool); -CUDF_LTO_DECL(cast_to_uint64, uint64_t, int8_t); -CUDF_LTO_DECL(cast_to_uint64, uint64_t, int16_t); -CUDF_LTO_DECL(cast_to_uint64, uint64_t, int32_t); -CUDF_LTO_DECL(cast_to_uint64, uint64_t, int64_t); -CUDF_LTO_DECL(cast_to_uint64, uint64_t, uint8_t); -CUDF_LTO_DECL(cast_to_uint64, uint64_t, uint16_t); -CUDF_LTO_DECL(cast_to_uint64, uint64_t, uint32_t); -CUDF_LTO_DECL(cast_to_uint64, uint64_t, uint64_t); -CUDF_LTO_DECL(cast_to_uint64, uint64_t, float32_t); -CUDF_LTO_DECL(cast_to_uint64, uint64_t, float64_t); - -CUDF_LTO_DECL(cast_to_float64, float64_t, bool); -CUDF_LTO_DECL(cast_to_float64, float64_t, int8_t); -CUDF_LTO_DECL(cast_to_float64, float64_t, int16_t); -CUDF_LTO_DECL(cast_to_float64, float64_t, int32_t); -CUDF_LTO_DECL(cast_to_float64, float64_t, int64_t); -CUDF_LTO_DECL(cast_to_float64, float64_t, uint8_t); -CUDF_LTO_DECL(cast_to_float64, float64_t, uint16_t); -CUDF_LTO_DECL(cast_to_float64, float64_t, uint32_t); -CUDF_LTO_DECL(cast_to_float64, float64_t, uint64_t); -CUDF_LTO_DECL(cast_to_float64, float64_t, float32_t); -CUDF_LTO_DECL(cast_to_float64, float64_t, float64_t); - -#undef CUDF_LTO_DECL - -#define CUDF_LTO_DECL(op, type) \ - __device__ extern void op(bool* out, type const* a); \ - \ - __device__ extern void op(bool* out, optional const* a); \ - \ - __device__ extern void op(optional* out, optional const* a) - -CUDF_LTO_DECL(is_null, bool); -CUDF_LTO_DECL(is_null, int8_t); -CUDF_LTO_DECL(is_null, int16_t); -CUDF_LTO_DECL(is_null, int32_t); -CUDF_LTO_DECL(is_null, int64_t); -CUDF_LTO_DECL(is_null, uint8_t); -CUDF_LTO_DECL(is_null, uint16_t); -CUDF_LTO_DECL(is_null, uint32_t); -CUDF_LTO_DECL(is_null, uint64_t); -CUDF_LTO_DECL(is_null, float32_t); -CUDF_LTO_DECL(is_null, float64_t); -CUDF_LTO_DECL(is_null, decimal32); -CUDF_LTO_DECL(is_null, decimal64); -CUDF_LTO_DECL(is_null, decimal128); -CUDF_LTO_DECL(is_null, timestamp_D); -CUDF_LTO_DECL(is_null, timestamp_s); -CUDF_LTO_DECL(is_null, timestamp_ms); -CUDF_LTO_DECL(is_null, timestamp_us); -CUDF_LTO_DECL(is_null, timestamp_ns); -CUDF_LTO_DECL(is_null, duration_D); -CUDF_LTO_DECL(is_null, duration_s); -CUDF_LTO_DECL(is_null, duration_ms); -CUDF_LTO_DECL(is_null, duration_ns); -CUDF_LTO_DECL(is_null, string_view); - -CUDF_LTO_DECL(logical_not, bool); -CUDF_LTO_DECL(logical_not, int8_t); -CUDF_LTO_DECL(logical_not, int16_t); -CUDF_LTO_DECL(logical_not, int32_t); -CUDF_LTO_DECL(logical_not, int64_t); -CUDF_LTO_DECL(logical_not, uint8_t); -CUDF_LTO_DECL(logical_not, uint16_t); -CUDF_LTO_DECL(logical_not, uint32_t); -CUDF_LTO_DECL(logical_not, uint64_t); - -#undef CUDF_LTO_DECL +#define CUDF_UNOP(op, type, operand) \ + __device__ __forceinline__ void op(type* out, type const* a) { *out = operand(*a); } \ + __device__ __forceinline__ void op(optional* out, optional const* a) \ + { \ + if (a->has_value()) { \ + op(&(*out), &(*a)); \ + } else { \ + *out = {}; \ + } \ + } + +#define CUDF_UNOP_T(op, ret_type, type, operand) \ + __device__ __forceinline__ void op(ret_type* out, type const* a) { *out = operand(*a); } \ + __device__ __forceinline__ void op(optional* out, optional const* a) \ + { \ + if (a->has_value()) { \ + op(&(*out), &(*a)); \ + } else { \ + *out = {}; \ + } \ + } + +#define CUDF_BINOP(op, type, operand) \ + __device__ __forceinline__ void op(type* out, type const* a, type const* b) \ + { \ + *out = *a operand * b; \ + } \ + \ + __device__ __forceinline__ void op( \ + optional* out, optional const* a, optional const* b) \ + { \ + if (a->has_value() && b->has_value()) { \ + op(&(*out), &(*a), &(*b)); \ + } else { \ + *out = {}; \ + } \ + } + +#define CUDF_PRED_BINOP(op, type, operand) \ + __device__ __forceinline__ void op(bool* out, type const* a, type const* b) \ + { \ + *out = *a operand * b; \ + } \ + \ + __device__ __forceinline__ void op(bool* out, optional const* a, optional const* b) \ + { \ + if (a->has_value() && b->has_value()) { \ + op(out, &(*a), &(*b)); \ + } else if (!a->has_value() && !b->has_value()) { \ + *out = true; \ + } else { \ + *out = false; \ + } \ + } \ + \ + __device__ __forceinline__ void op( \ + optional* out, optional const* a, optional const* b) \ + { \ + bool r; \ + op(&r, a, b); \ + *out = r; \ + } + +#define CUDF_ID_OP(op, type) \ + __device__ __forceinline__ void op(type* out, type const* a) { *out = *a; } \ + __device__ __forceinline__ void op(optional* out, optional const* a) { *out = *a; } + +#define CUDF_IS_NULL_OP(op, type) \ + __device__ __forceinline__ void op(bool* out, type const* a) { *out = false; } \ + __device__ __forceinline__ void op(bool* out, optional const* a) \ + { \ + *out = !a->has_value(); \ + } \ + __device__ __forceinline__ void op(optional* out, optional const* a) \ + { \ + *out = !a->has_value(); \ + } + +#define CUDF_ABS_OP(op, type) \ + __device__ __forceinline__ void op(type* out, type const* a) { *out = (*a < 0) ? -(*a) : (*a); } \ + __device__ __forceinline__ void op(optional* out, optional const* a) \ + { \ + if (a->has_value()) { \ + op(&(*out), &(*a)); \ + } else { \ + *out = {}; \ + } \ + } + +#define CUDF_EXTERN_UNOP(op, type) \ + __device__ extern void op(type* out, type const* a); \ + __device__ extern void op(optional* out, optional const* a) \ + { \ + if (a->has_value()) { \ + op(&(*out), &(*a)); \ + } else { \ + *out = {}; \ + } \ + } + +CUDF_BINOP(add, int32_t, +); +CUDF_BINOP(add, int64_t, +); +CUDF_BINOP(add, uint32_t, +); +CUDF_BINOP(add, uint64_t, +); +CUDF_BINOP(add, float32_t, +); +CUDF_BINOP(add, float64_t, +); +CUDF_BINOP(add, decimal32, +); +CUDF_BINOP(add, decimal64, +); +CUDF_BINOP(add, decimal128, +); +CUDF_BINOP(add, duration_D, +); +CUDF_BINOP(add, duration_s, +); +CUDF_BINOP(add, duration_ms, +); +CUDF_BINOP(add, duration_ns, +); + +CUDF_BINOP(sub, int32_t, -); +CUDF_BINOP(sub, int64_t, -); +CUDF_BINOP(sub, uint32_t, -); +CUDF_BINOP(sub, uint64_t, -); +CUDF_BINOP(sub, float32_t, -); +CUDF_BINOP(sub, float64_t, -); +CUDF_BINOP(sub, decimal32, -); +CUDF_BINOP(sub, decimal64, -); +CUDF_BINOP(sub, decimal128, -); +CUDF_BINOP(sub, duration_D, -); +CUDF_BINOP(sub, duration_s, -); +CUDF_BINOP(sub, duration_ms, -); +CUDF_BINOP(sub, duration_ns, -); + +CUDF_BINOP(mul, int32_t, *); +CUDF_BINOP(mul, int64_t, *); +CUDF_BINOP(mul, uint32_t, *); +CUDF_BINOP(mul, uint64_t, *); +CUDF_BINOP(mul, float32_t, *); +CUDF_BINOP(mul, float64_t, *); +CUDF_BINOP(mul, decimal32, *); +CUDF_BINOP(mul, decimal64, *); +CUDF_BINOP(mul, decimal128, *); + +CUDF_BINOP(div, int32_t, /); +CUDF_BINOP(div, int64_t, /); +CUDF_BINOP(div, uint32_t, /); +CUDF_BINOP(div, uint64_t, /); +CUDF_BINOP(div, float32_t, /); +CUDF_BINOP(div, float64_t, /); +CUDF_BINOP(div, decimal32, /); +CUDF_BINOP(div, decimal64, /); +CUDF_BINOP(div, decimal128, /); + +// CUDF_OP(mod, float32_t); +// CUDF_OP(mod, float64_t); + +// CUDF_OP(pymod, float32_t); +// CUDF_OP(pymod, float64_t); + +// CUDF_OP(pow, float32_t); +// CUDF_OP(pow, float64_t); + +CUDF_PRED_BINOP(equal, bool, ==); +CUDF_PRED_BINOP(equal, int8_t, ==); +CUDF_PRED_BINOP(equal, int16_t, ==); +CUDF_PRED_BINOP(equal, int32_t, ==); +CUDF_PRED_BINOP(equal, int64_t, ==); +CUDF_PRED_BINOP(equal, uint8_t, ==); +CUDF_PRED_BINOP(equal, uint16_t, ==); +CUDF_PRED_BINOP(equal, uint32_t, ==); +CUDF_PRED_BINOP(equal, uint64_t, ==); +CUDF_PRED_BINOP(equal, float32_t, ==); +CUDF_PRED_BINOP(equal, float64_t, ==); +CUDF_PRED_BINOP(equal, decimal32, ==); +CUDF_PRED_BINOP(equal, decimal64, ==); +CUDF_PRED_BINOP(equal, decimal128, ==); +CUDF_PRED_BINOP(equal, timestamp_D, ==); +CUDF_PRED_BINOP(equal, timestamp_s, ==); +CUDF_PRED_BINOP(equal, timestamp_ms, ==); +CUDF_PRED_BINOP(equal, timestamp_us, ==); +CUDF_PRED_BINOP(equal, timestamp_ns, ==); +CUDF_PRED_BINOP(equal, duration_D, ==); +CUDF_PRED_BINOP(equal, duration_s, ==); +CUDF_PRED_BINOP(equal, duration_ms, ==); +CUDF_PRED_BINOP(equal, duration_ns, ==); +CUDF_PRED_BINOP(equal, string_view, ==); + +/* +CUDF_OP(null_equal, bool); +CUDF_OP(null_equal, int8_t); +CUDF_OP(null_equal, int16_t); +CUDF_OP(null_equal, int32_t); +CUDF_OP(null_equal, int64_t); +CUDF_OP(null_equal, uint8_t); +CUDF_OP(null_equal, uint16_t); +CUDF_OP(null_equal, uint32_t); +CUDF_OP(null_equal, uint64_t); +CUDF_OP(null_equal, float32_t); +CUDF_OP(null_equal, float64_t); +CUDF_OP(null_equal, decimal32); +CUDF_OP(null_equal, decimal64); +CUDF_OP(null_equal, decimal128); +CUDF_OP(null_equal, timestamp_D); +CUDF_OP(null_equal, timestamp_s); +CUDF_OP(null_equal, timestamp_ms); +CUDF_OP(null_equal, timestamp_us); +CUDF_OP(null_equal, timestamp_ns); +CUDF_OP(null_equal, duration_D); +CUDF_OP(null_equal, duration_s); +CUDF_OP(null_equal, duration_ms); +CUDF_OP(null_equal, duration_ns); +CUDF_OP(null_equal, string_view); +*/ + +CUDF_PRED_BINOP(less, bool, <); +CUDF_PRED_BINOP(less, int8_t, <); +CUDF_PRED_BINOP(less, int16_t, <); +CUDF_PRED_BINOP(less, int32_t, <); +CUDF_PRED_BINOP(less, int64_t, <); +CUDF_PRED_BINOP(less, uint8_t, <); +CUDF_PRED_BINOP(less, uint16_t, <); +CUDF_PRED_BINOP(less, uint32_t, <); +CUDF_PRED_BINOP(less, uint64_t, <); +CUDF_PRED_BINOP(less, float32_t, <); +CUDF_PRED_BINOP(less, float64_t, <); +CUDF_PRED_BINOP(less, decimal32, <); +CUDF_PRED_BINOP(less, decimal64, <); +CUDF_PRED_BINOP(less, decimal128, <); +CUDF_PRED_BINOP(less, timestamp_D, <); +CUDF_PRED_BINOP(less, timestamp_s, <); +CUDF_PRED_BINOP(less, timestamp_ms, <); +CUDF_PRED_BINOP(less, timestamp_us, <); +CUDF_PRED_BINOP(less, timestamp_ns, <); +CUDF_PRED_BINOP(less, duration_D, <); +CUDF_PRED_BINOP(less, duration_s, <); +CUDF_PRED_BINOP(less, duration_ms, <); +CUDF_PRED_BINOP(less, duration_ns, <); +CUDF_PRED_BINOP(less, string_view, <); + +CUDF_PRED_BINOP(greater, bool, >); +CUDF_PRED_BINOP(greater, int8_t, >); +CUDF_PRED_BINOP(greater, int16_t, >); +CUDF_PRED_BINOP(greater, int32_t, >); +CUDF_PRED_BINOP(greater, int64_t, >); +CUDF_PRED_BINOP(greater, uint8_t, >); +CUDF_PRED_BINOP(greater, uint16_t, >); +CUDF_PRED_BINOP(greater, uint32_t, >); +CUDF_PRED_BINOP(greater, uint64_t, >); +CUDF_PRED_BINOP(greater, float32_t, >); +CUDF_PRED_BINOP(greater, float64_t, >); +CUDF_PRED_BINOP(greater, decimal32, >); +CUDF_PRED_BINOP(greater, decimal64, >); +CUDF_PRED_BINOP(greater, decimal128, >); +CUDF_PRED_BINOP(greater, timestamp_D, >); +CUDF_PRED_BINOP(greater, timestamp_s, >); +CUDF_PRED_BINOP(greater, timestamp_ms, >); +CUDF_PRED_BINOP(greater, timestamp_us, >); +CUDF_PRED_BINOP(greater, timestamp_ns, >); +CUDF_PRED_BINOP(greater, duration_D, >); +CUDF_PRED_BINOP(greater, duration_s, >); +CUDF_PRED_BINOP(greater, duration_ms, >); +CUDF_PRED_BINOP(greater, duration_ns, >); +CUDF_PRED_BINOP(greater, string_view, >); + +CUDF_PRED_BINOP(less_equal, bool, <=); +CUDF_PRED_BINOP(less_equal, int8_t, <=); +CUDF_PRED_BINOP(less_equal, int16_t, <=); +CUDF_PRED_BINOP(less_equal, int32_t, <=); +CUDF_PRED_BINOP(less_equal, int64_t, <=); +CUDF_PRED_BINOP(less_equal, uint8_t, <=); +CUDF_PRED_BINOP(less_equal, uint16_t, <=); +CUDF_PRED_BINOP(less_equal, uint32_t, <=); +CUDF_PRED_BINOP(less_equal, uint64_t, <=); +CUDF_PRED_BINOP(less_equal, float32_t, <=); +CUDF_PRED_BINOP(less_equal, float64_t, <=); +CUDF_PRED_BINOP(less_equal, decimal32, <=); +CUDF_PRED_BINOP(less_equal, decimal64, <=); +CUDF_PRED_BINOP(less_equal, decimal128, <=); +CUDF_PRED_BINOP(less_equal, timestamp_D, <=); +CUDF_PRED_BINOP(less_equal, timestamp_s, <=); +CUDF_PRED_BINOP(less_equal, timestamp_ms, <=); +CUDF_PRED_BINOP(less_equal, timestamp_us, <=); +CUDF_PRED_BINOP(less_equal, timestamp_ns, <=); +CUDF_PRED_BINOP(less_equal, duration_D, <=); +CUDF_PRED_BINOP(less_equal, duration_s, <=); +CUDF_PRED_BINOP(less_equal, duration_ms, <=); +CUDF_PRED_BINOP(less_equal, duration_ns, <=); +CUDF_PRED_BINOP(less_equal, string_view, <=); + +CUDF_PRED_BINOP(greater_equal, bool, >=); +CUDF_PRED_BINOP(greater_equal, int8_t, >=); +CUDF_PRED_BINOP(greater_equal, int16_t, >=); +CUDF_PRED_BINOP(greater_equal, int32_t, >=); +CUDF_PRED_BINOP(greater_equal, int64_t, >=); +CUDF_PRED_BINOP(greater_equal, uint8_t, >=); +CUDF_PRED_BINOP(greater_equal, uint16_t, >=); +CUDF_PRED_BINOP(greater_equal, uint32_t, >=); +CUDF_PRED_BINOP(greater_equal, uint64_t, >=); +CUDF_PRED_BINOP(greater_equal, float32_t, >=); +CUDF_PRED_BINOP(greater_equal, float64_t, >=); +CUDF_PRED_BINOP(greater_equal, decimal32, >=); +CUDF_PRED_BINOP(greater_equal, decimal64, >=); +CUDF_PRED_BINOP(greater_equal, decimal128, >=); +CUDF_PRED_BINOP(greater_equal, timestamp_D, >=); +CUDF_PRED_BINOP(greater_equal, timestamp_s, >=); +CUDF_PRED_BINOP(greater_equal, timestamp_ms, >=); +CUDF_PRED_BINOP(greater_equal, timestamp_us, >=); +CUDF_PRED_BINOP(greater_equal, timestamp_ns, >=); +CUDF_PRED_BINOP(greater_equal, duration_D, >=); +CUDF_PRED_BINOP(greater_equal, duration_s, >=); +CUDF_PRED_BINOP(greater_equal, duration_ms, >=); +CUDF_PRED_BINOP(greater_equal, duration_ns, >=); +CUDF_PRED_BINOP(greater_equal, string_view, >=); + +CUDF_BINOP(bitwise_and, int32_t, &); +CUDF_BINOP(bitwise_and, int64_t, &); +CUDF_BINOP(bitwise_and, uint32_t, &); +CUDF_BINOP(bitwise_and, uint64_t, &); + +CUDF_BINOP(bitwise_or, int32_t, |); +CUDF_BINOP(bitwise_or, int64_t, |); +CUDF_BINOP(bitwise_or, uint32_t, |); +CUDF_BINOP(bitwise_or, uint64_t, |); + +CUDF_BINOP(bitwise_xor, int32_t, ^); +CUDF_BINOP(bitwise_xor, int64_t, ^); +CUDF_BINOP(bitwise_xor, uint32_t, ^); +CUDF_BINOP(bitwise_xor, uint64_t, ^); + +CUDF_BINOP(logical_and, bool, &&); +CUDF_BINOP(null_logical_and, bool, &&); +CUDF_BINOP(logical_or, bool, ||); +CUDF_BINOP(null_logical_or, bool, ||); + +CUDF_ID_OP(identity, bool); +CUDF_ID_OP(identity, int8_t); +CUDF_ID_OP(identity, int16_t); +CUDF_ID_OP(identity, int32_t); +CUDF_ID_OP(identity, int64_t); +CUDF_ID_OP(identity, uint8_t); +CUDF_ID_OP(identity, uint16_t); +CUDF_ID_OP(identity, uint32_t); +CUDF_ID_OP(identity, uint64_t); +CUDF_ID_OP(identity, float32_t); +CUDF_ID_OP(identity, float64_t); +CUDF_ID_OP(identity, decimal32); +CUDF_ID_OP(identity, decimal64); +CUDF_ID_OP(identity, decimal128); +CUDF_ID_OP(identity, timestamp_D); +CUDF_ID_OP(identity, timestamp_s); +CUDF_ID_OP(identity, timestamp_ms); +CUDF_ID_OP(identity, timestamp_us); +CUDF_ID_OP(identity, timestamp_ns); +CUDF_ID_OP(identity, duration_D); +CUDF_ID_OP(identity, duration_s); +CUDF_ID_OP(identity, duration_ms); +CUDF_ID_OP(identity, duration_ns); +CUDF_ID_OP(identity, string_view); + +CUDF_UNOP(bit_invert, uint32_t, ~); +CUDF_UNOP(bit_invert, uint64_t, ~); +CUDF_UNOP(bit_invert, int32_t, ~); +CUDF_UNOP(bit_invert, int64_t, ~); + +CUDF_UNOP_T(cast_to_int64, int64_t, bool, (int64_t)); +CUDF_UNOP_T(cast_to_int64, int64_t, int8_t, (int64_t)); +CUDF_UNOP_T(cast_to_int64, int64_t, int16_t, (int64_t)); +CUDF_UNOP_T(cast_to_int64, int64_t, int32_t, (int64_t)); +CUDF_UNOP_T(cast_to_int64, int64_t, int64_t, (int64_t)); +CUDF_UNOP_T(cast_to_int64, int64_t, uint8_t, (int64_t)); +CUDF_UNOP_T(cast_to_int64, int64_t, uint16_t, (int64_t)); +CUDF_UNOP_T(cast_to_int64, int64_t, uint32_t, (int64_t)); +CUDF_UNOP_T(cast_to_int64, int64_t, uint64_t, (int64_t)); +CUDF_UNOP_T(cast_to_int64, int64_t, float32_t, (int64_t)); +CUDF_UNOP_T(cast_to_int64, int64_t, float64_t, (int64_t)); + +CUDF_UNOP_T(cast_to_uint64, uint64_t, bool, (uint64_t)); +CUDF_UNOP_T(cast_to_uint64, uint64_t, int8_t, (uint64_t)); +CUDF_UNOP_T(cast_to_uint64, uint64_t, int16_t, (uint64_t)); +CUDF_UNOP_T(cast_to_uint64, uint64_t, int32_t, (uint64_t)); +CUDF_UNOP_T(cast_to_uint64, uint64_t, int64_t, (uint64_t)); +CUDF_UNOP_T(cast_to_uint64, uint64_t, uint8_t, (uint64_t)); +CUDF_UNOP_T(cast_to_uint64, uint64_t, uint16_t, (uint64_t)); +CUDF_UNOP_T(cast_to_uint64, uint64_t, uint32_t, (uint64_t)); +CUDF_UNOP_T(cast_to_uint64, uint64_t, uint64_t, (uint64_t)); +CUDF_UNOP_T(cast_to_uint64, uint64_t, float32_t, (uint64_t)); +CUDF_UNOP_T(cast_to_uint64, uint64_t, float64_t, (uint64_t)); + +CUDF_UNOP_T(cast_to_float64, float64_t, bool, (float64_t)); +CUDF_UNOP_T(cast_to_float64, float64_t, int8_t, (float64_t)); +CUDF_UNOP_T(cast_to_float64, float64_t, int16_t, (float64_t)); +CUDF_UNOP_T(cast_to_float64, float64_t, int32_t, (float64_t)); +CUDF_UNOP_T(cast_to_float64, float64_t, int64_t, (float64_t)); +CUDF_UNOP_T(cast_to_float64, float64_t, uint8_t, (float64_t)); +CUDF_UNOP_T(cast_to_float64, float64_t, uint16_t, (float64_t)); +CUDF_UNOP_T(cast_to_float64, float64_t, uint32_t, (float64_t)); +CUDF_UNOP_T(cast_to_float64, float64_t, uint64_t, (float64_t)); +CUDF_UNOP_T(cast_to_float64, float64_t, float32_t, (float64_t)); +CUDF_UNOP_T(cast_to_float64, float64_t, float64_t, (float64_t)); + +CUDF_IS_NULL_OP(is_null, bool); +CUDF_IS_NULL_OP(is_null, int8_t); +CUDF_IS_NULL_OP(is_null, int16_t); +CUDF_IS_NULL_OP(is_null, int32_t); +CUDF_IS_NULL_OP(is_null, int64_t); +CUDF_IS_NULL_OP(is_null, uint8_t); +CUDF_IS_NULL_OP(is_null, uint16_t); +CUDF_IS_NULL_OP(is_null, uint32_t); +CUDF_IS_NULL_OP(is_null, uint64_t); +CUDF_IS_NULL_OP(is_null, float32_t); +CUDF_IS_NULL_OP(is_null, float64_t); +CUDF_IS_NULL_OP(is_null, decimal32); +CUDF_IS_NULL_OP(is_null, decimal64); +CUDF_IS_NULL_OP(is_null, decimal128); +CUDF_IS_NULL_OP(is_null, timestamp_D); +CUDF_IS_NULL_OP(is_null, timestamp_s); +CUDF_IS_NULL_OP(is_null, timestamp_ms); +CUDF_IS_NULL_OP(is_null, timestamp_us); +CUDF_IS_NULL_OP(is_null, timestamp_ns); +CUDF_IS_NULL_OP(is_null, duration_D); +CUDF_IS_NULL_OP(is_null, duration_s); +CUDF_IS_NULL_OP(is_null, duration_ms); +CUDF_IS_NULL_OP(is_null, duration_ns); +CUDF_IS_NULL_OP(is_null, string_view); + +CUDF_ABS_OP(abs, int8_t); +CUDF_ABS_OP(abs, int16_t); +CUDF_ABS_OP(abs, int32_t); +CUDF_ABS_OP(abs, int64_t); +CUDF_ABS_OP(abs, float32_t); +CUDF_ABS_OP(abs, float64_t); + +CUDF_EXTERN_UNOP(sin, float32_t); +CUDF_EXTERN_UNOP(sin, float64_t); + +CUDF_EXTERN_UNOP(cos, float32_t); +CUDF_EXTERN_UNOP(cos, float64_t); + +CUDF_EXTERN_UNOP(tan, float32_t); +CUDF_EXTERN_UNOP(tan, float64_t); + +CUDF_EXTERN_UNOP(arcsin, float32_t); +CUDF_EXTERN_UNOP(arcsin, float64_t); + +CUDF_EXTERN_UNOP(arccos, float32_t); +CUDF_EXTERN_UNOP(arccos, float64_t); + +CUDF_EXTERN_UNOP(arctan, float32_t); +CUDF_EXTERN_UNOP(arctan, float64_t); + +CUDF_EXTERN_UNOP(sinh, float32_t); +CUDF_EXTERN_UNOP(sinh, float64_t); + +CUDF_EXTERN_UNOP(cosh, float32_t); +CUDF_EXTERN_UNOP(cosh, float64_t); + +CUDF_EXTERN_UNOP(tanh, float32_t); +CUDF_EXTERN_UNOP(tanh, float64_t); + +CUDF_EXTERN_UNOP(arcsinh, float32_t); +CUDF_EXTERN_UNOP(arcsinh, float64_t); + +CUDF_EXTERN_UNOP(arccosh, float32_t); +CUDF_EXTERN_UNOP(arccosh, float64_t); + +CUDF_EXTERN_UNOP(arctanh, float32_t); +CUDF_EXTERN_UNOP(arctanh, float64_t); + +CUDF_EXTERN_UNOP(exp, float32_t); +CUDF_EXTERN_UNOP(exp, float64_t); + +CUDF_EXTERN_UNOP(log, float32_t); +CUDF_EXTERN_UNOP(log, float64_t); + +CUDF_EXTERN_UNOP(cbrt, float32_t); +CUDF_EXTERN_UNOP(cbrt, float64_t); + +CUDF_EXTERN_UNOP(ceil, float32_t); +CUDF_EXTERN_UNOP(ceil, float64_t); + +CUDF_EXTERN_UNOP(floor, float32_t); +CUDF_EXTERN_UNOP(floor, float64_t); + +CUDF_EXTERN_UNOP(rint, float32_t); +CUDF_EXTERN_UNOP(rint, float64_t); } // namespace operators diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index 440ac29e2610..7babbf9ecd8d 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -75,88 +75,292 @@ struct CUDF_LTO_ALIAS decimal32 { private: int32_t _value = 0; scale_type _scale = scale_type{}; + + public: + constexpr decimal32 operator+(decimal32 const&) const; + constexpr decimal32 operator-(decimal32 const&) const; + constexpr decimal32 operator*(decimal32 const&) const; + constexpr decimal32 operator/(decimal32 const&) const; + constexpr bool operator==(decimal32 const&) const; + constexpr bool operator!=(decimal32 const&) const; + constexpr bool operator<(decimal32 const&) const; + constexpr bool operator>(decimal32 const&) const; + constexpr bool operator<=(decimal32 const&) const; + constexpr bool operator>=(decimal32 const&) const; }; struct CUDF_LTO_ALIAS decimal64 { private: int64_t _value = 0; scale_type _scale = scale_type{}; + + public: + constexpr decimal64 operator+(decimal64 const&) const; + constexpr decimal64 operator-(decimal64 const&) const; + constexpr decimal64 operator*(decimal64 const&) const; + constexpr decimal64 operator/(decimal64 const&) const; + constexpr bool operator==(decimal64 const&) const; + constexpr bool operator!=(decimal64 const&) const; + constexpr bool operator<(decimal64 const&) const; + constexpr bool operator>(decimal64 const&) const; + constexpr bool operator<=(decimal64 const&) const; + constexpr bool operator>=(decimal64 const&) const; }; struct CUDF_LTO_ALIAS decimal128 { private: __int128_t _value = 0; scale_type _scale = scale_type{}; + + public: + constexpr decimal128 operator+(decimal128 const&) const; + constexpr decimal128 operator-(decimal128 const&) const; + constexpr decimal128 operator*(decimal128 const&) const; + constexpr decimal128 operator/(decimal128 const&) const; + constexpr bool operator==(decimal128 const&) const; + constexpr bool operator!=(decimal128 const&) const; + constexpr bool operator<(decimal128 const&) const; + constexpr bool operator>(decimal128 const&) const; + constexpr bool operator<=(decimal128 const&) const; + constexpr bool operator>=(decimal128 const&) const; }; struct CUDF_LTO_ALIAS timestamp_D { private: int32_t _rep = 0; + + public: + constexpr timestamp_D operator+(timestamp_D const&) const; + constexpr timestamp_D operator-(timestamp_D const&) const; + constexpr timestamp_D operator*(timestamp_D const&) const; + constexpr timestamp_D operator/(timestamp_D const&) const; + constexpr bool operator==(timestamp_D const&) const; + constexpr bool operator!=(timestamp_D const&) const; + constexpr bool operator<(timestamp_D const&) const; + constexpr bool operator>(timestamp_D const&) const; + constexpr bool operator<=(timestamp_D const&) const; + constexpr bool operator>=(timestamp_D const&) const; }; struct CUDF_LTO_ALIAS timestamp_h { private: int32_t _rep = 0; + + public: + constexpr timestamp_h operator+(timestamp_h const&) const; + constexpr timestamp_h operator-(timestamp_h const&) const; + constexpr timestamp_h operator*(timestamp_h const&) const; + constexpr timestamp_h operator/(timestamp_h const&) const; + constexpr bool operator==(timestamp_h const&) const; + constexpr bool operator!=(timestamp_h const&) const; + constexpr bool operator<(timestamp_h const&) const; + constexpr bool operator>(timestamp_h const&) const; + constexpr bool operator<=(timestamp_h const&) const; + constexpr bool operator>=(timestamp_h const&) const; }; struct CUDF_LTO_ALIAS timestamp_m { private: int32_t _rep = 0; + + public: + constexpr timestamp_m operator+(timestamp_m const&) const; + constexpr timestamp_m operator-(timestamp_m const&) const; + constexpr timestamp_m operator*(timestamp_m const&) const; + constexpr timestamp_m operator/(timestamp_m const&) const; + constexpr bool operator==(timestamp_m const&) const; + constexpr bool operator!=(timestamp_m const&) const; + constexpr bool operator<(timestamp_m const&) const; + constexpr bool operator>(timestamp_m const&) const; + constexpr bool operator<=(timestamp_m const&) const; + constexpr bool operator>=(timestamp_m const&) const; }; struct CUDF_LTO_ALIAS timestamp_s { private: int64_t _rep = 0; + + public: + constexpr timestamp_s operator+(timestamp_s const&) const; + constexpr timestamp_s operator-(timestamp_s const&) const; + constexpr timestamp_s operator*(timestamp_s const&) const; + constexpr timestamp_s operator/(timestamp_s const&) const; + constexpr bool operator==(timestamp_s const&) const; + constexpr bool operator!=(timestamp_s const&) const; + constexpr bool operator<(timestamp_s const&) const; + constexpr bool operator>(timestamp_s const&) const; + constexpr bool operator<=(timestamp_s const&) const; + constexpr bool operator>=(timestamp_s const&) const; }; struct CUDF_LTO_ALIAS timestamp_ms { private: int64_t _rep = 0; + + public: + constexpr timestamp_ms operator+(timestamp_ms const&) const; + constexpr timestamp_ms operator-(timestamp_ms const&) const; + constexpr timestamp_ms operator*(timestamp_ms const&) const; + constexpr timestamp_ms operator/(timestamp_ms const&) const; + constexpr bool operator==(timestamp_ms const&) const; + constexpr bool operator!=(timestamp_ms const&) const; + constexpr bool operator<(timestamp_ms const&) const; + constexpr bool operator>(timestamp_ms const&) const; + constexpr bool operator<=(timestamp_ms const&) const; + constexpr bool operator>=(timestamp_ms const&) const; }; struct CUDF_LTO_ALIAS timestamp_us { private: int64_t _rep = 0; + + public: + constexpr timestamp_us operator+(timestamp_us const&) const; + constexpr timestamp_us operator-(timestamp_us const&) const; + constexpr timestamp_us operator*(timestamp_us const&) const; + constexpr timestamp_us operator/(timestamp_us const&) const; + constexpr bool operator==(timestamp_us const&) const; + constexpr bool operator!=(timestamp_us const&) const; + constexpr bool operator<(timestamp_us const&) const; + constexpr bool operator>(timestamp_us const&) const; + constexpr bool operator<=(timestamp_us const&) const; + constexpr bool operator>=(timestamp_us const&) const; }; struct CUDF_LTO_ALIAS timestamp_ns { private: int64_t _rep = 0; + + public: + constexpr timestamp_ns operator+(timestamp_ns const&) const; + constexpr timestamp_ns operator-(timestamp_ns const&) const; + constexpr timestamp_ns operator*(timestamp_ns const&) const; + constexpr timestamp_ns operator/(timestamp_ns const&) const; + constexpr bool operator==(timestamp_ns const&) const; + constexpr bool operator!=(timestamp_ns const&) const; + constexpr bool operator<(timestamp_ns const&) const; + constexpr bool operator>(timestamp_ns const&) const; + constexpr bool operator<=(timestamp_ns const&) const; + constexpr bool operator>=(timestamp_ns const&) const; }; struct CUDF_LTO_ALIAS duration_D { private: int32_t _rep = 0; + + public: + constexpr duration_D operator+(duration_D const&) const; + constexpr duration_D operator-(duration_D const&) const; + constexpr duration_D operator*(duration_D const&) const; + constexpr duration_D operator/(duration_D const&) const; + constexpr bool operator==(duration_D const&) const; + constexpr bool operator!=(duration_D const&) const; + constexpr bool operator<(duration_D const&) const; + constexpr bool operator>(duration_D const&) const; + constexpr bool operator<=(duration_D const&) const; + constexpr bool operator>=(duration_D const&) const; }; struct CUDF_LTO_ALIAS duration_h { private: int32_t _rep = 0; + + public: + constexpr duration_h operator+(duration_h const&) const; + constexpr duration_h operator-(duration_h const&) const; + constexpr duration_h operator*(duration_h const&) const; + constexpr duration_h operator/(duration_h const&) const; + constexpr bool operator==(duration_h const&) const; + constexpr bool operator!=(duration_h const&) const; + constexpr bool operator<(duration_h const&) const; + constexpr bool operator>(duration_h const&) const; + constexpr bool operator<=(duration_h const&) const; + constexpr bool operator>=(duration_h const&) const; }; struct CUDF_LTO_ALIAS duration_m { private: int32_t _rep = 0; + + public: + constexpr duration_m operator+(duration_m const&) const; + constexpr duration_m operator-(duration_m const&) const; + constexpr duration_m operator*(duration_m const&) const; + constexpr duration_m operator/(duration_m const&) const; + constexpr bool operator==(duration_m const&) const; + constexpr bool operator!=(duration_m const&) const; + constexpr bool operator<(duration_m const&) const; + constexpr bool operator>(duration_m const&) const; + constexpr bool operator<=(duration_m const&) const; + constexpr bool operator>=(duration_m const&) const; }; struct CUDF_LTO_ALIAS duration_s { private: int64_t _rep = 0; + + public: + constexpr duration_s operator+(duration_s const&) const; + constexpr duration_s operator-(duration_s const&) const; + constexpr duration_s operator*(duration_s const&) const; + constexpr duration_s operator/(duration_s const&) const; + constexpr bool operator==(duration_s const&) const; + constexpr bool operator!=(duration_s const&) const; + constexpr bool operator<(duration_s const&) const; + constexpr bool operator>(duration_s const&) const; + constexpr bool operator<=(duration_s const&) const; + constexpr bool operator>=(duration_s const&) const; }; struct CUDF_LTO_ALIAS duration_ms { private: int64_t _rep = 0; + + public: + constexpr duration_ms operator+(duration_ms const&) const; + constexpr duration_ms operator-(duration_ms const&) const; + constexpr duration_ms operator*(duration_ms const&) const; + constexpr duration_ms operator/(duration_ms const&) const; + constexpr bool operator==(duration_ms const&) const; + constexpr bool operator!=(duration_ms const&) const; + constexpr bool operator<(duration_ms const&) const; + constexpr bool operator>(duration_ms const&) const; + constexpr bool operator<=(duration_ms const&) const; + constexpr bool operator>=(duration_ms const&) const; }; struct CUDF_LTO_ALIAS duration_us { private: int64_t _rep = 0; + + public: + constexpr duration_us operator+(duration_us const&) const; + constexpr duration_us operator-(duration_us const&) const; + constexpr duration_us operator*(duration_us const&) const; + constexpr duration_us operator/(duration_us const&) const; + constexpr bool operator==(duration_us const&) const; + constexpr bool operator!=(duration_us const&) const; + constexpr bool operator<(duration_us const&) const; + constexpr bool operator>(duration_us const&) const; + constexpr bool operator<=(duration_us const&) const; + constexpr bool operator>=(duration_us const&) const; }; struct CUDF_LTO_ALIAS duration_ns { private: int64_t _rep = 0; + + public: + constexpr duration_ns operator+(duration_ns const&) const; + constexpr duration_ns operator-(duration_ns const&) const; + constexpr duration_ns operator*(duration_ns const&) const; + constexpr duration_ns operator/(duration_ns const&) const; + constexpr bool operator==(duration_ns const&) const; + constexpr bool operator!=(duration_ns const&) const; + constexpr bool operator<(duration_ns const&) const; + constexpr bool operator>(duration_ns const&) const; + constexpr bool operator<=(duration_ns const&) const; + constexpr bool operator>=(duration_ns const&) const; }; template From ec95217d1be493cb58ac08d7ae4c481cfefbe072 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 20 Feb 2026 23:09:06 +0000 Subject: [PATCH 056/254] update --- cpp/CMakeLists.txt | 3 +- cpp/cmake/thirdparty/get_lz4.cmake | 2 +- cpp/include/cudf/jit/lto/column_view.cuh | 236 ----- cpp/include/cudf/jit/lto/export.cuh | 18 - cpp/include/cudf/jit/lto/library.cuh | 11 +- cpp/include/cudf/jit/lto/operators.cuh | 982 +++++++++--------- cpp/include/cudf/jit/lto/optional.cuh | 74 -- cpp/include/cudf/jit/lto/optional_span.cuh | 69 -- cpp/include/cudf/jit/lto/scope.cuh | 82 -- cpp/include/cudf/jit/lto/span.cuh | 40 - cpp/include/cudf/jit/lto/string_view.cuh | 79 -- cpp/include/cudf/jit/lto/thunk.cuh | 133 --- cpp/include/cudf/jit/lto/transform_params.cuh | 26 - cpp/include/cudf/jit/lto/types.cuh | 975 +++++++++++------ cpp/include/cudf/utilities/export.hpp | 5 + cpp/src/jit/lto/column_view.def.cuh | 185 ---- cpp/src/jit/lto/operators.def.cuh | 580 +---------- 17 files changed, 1232 insertions(+), 2268 deletions(-) delete mode 100644 cpp/include/cudf/jit/lto/column_view.cuh delete mode 100644 cpp/include/cudf/jit/lto/export.cuh delete mode 100644 cpp/include/cudf/jit/lto/optional.cuh delete mode 100644 cpp/include/cudf/jit/lto/optional_span.cuh delete mode 100644 cpp/include/cudf/jit/lto/scope.cuh delete mode 100644 cpp/include/cudf/jit/lto/span.cuh delete mode 100644 cpp/include/cudf/jit/lto/string_view.cuh delete mode 100644 cpp/include/cudf/jit/lto/thunk.cuh delete mode 100644 cpp/include/cudf/jit/lto/transform_params.cuh delete mode 100644 cpp/src/jit/lto/column_view.def.cuh diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8e8ed80b7826..d7359a2ac5a6 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -360,7 +360,8 @@ list(APPEND CUDF_JIT_COMPILE_FLAGS -std=c++${CUDF_CUDA_STANDARD} --device-int128 set(CUDF_LTO_JIT_ARCHITECTURES "80-real;90-real;100-real;120-real") add_library(cudf_lto_library OBJECT src/jit/lto/library.cu) -target_compile_options(cudf_lto_library PRIVATE --compress-mode=none) +target_compile_options(cudf_lto_library PRIVATE --compress-mode=none --gen-opt-lto) +target_compile_definitions(cudf_lto_library PRIVATE CUDF_DISABLE_DEFAULT_EXPORT) set_target_properties( cudf_lto_library PROPERTIES CUDA_ARCHITECTURES "${CUDF_LTO_JIT_ARCHITECTURES}" diff --git a/cpp/cmake/thirdparty/get_lz4.cmake b/cpp/cmake/thirdparty/get_lz4.cmake index 42c79f9af425..57473030576b 100644 --- a/cpp/cmake/thirdparty/get_lz4.cmake +++ b/cpp/cmake/thirdparty/get_lz4.cmake @@ -14,7 +14,7 @@ function(find_and_configure_lz4) GLOBAL_TARGETS lz4 CPM_ARGS GIT_REPOSITORY https://github.com/lz4/lz4.git - GIT_TAG 446a35f + GIT_TAG 1519f46a3a11f312be8f5796e8fa4779140277f0 GIT_SHALLOW TRUE ) diff --git a/cpp/include/cudf/jit/lto/column_view.cuh b/cpp/include/cudf/jit/lto/column_view.cuh deleted file mode 100644 index 50a03748ed2e..000000000000 --- a/cpp/include/cudf/jit/lto/column_view.cuh +++ /dev/null @@ -1,236 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include -#include -#include -#include - -namespace CUDF_LTO_EXPORT cudf { -namespace lto { - -struct alignas(16) CUDF_LTO_ALIAS column_device_view { - private: - data_type _type = {}; - size_type _size = 0; - void const* _data = nullptr; - bitmask_type const* _null_mask = nullptr; - size_type _offset = 0; - column_device_view* _d_children = nullptr; - size_type _num_children = 0; - - public: - template - __device__ T const* head() const; - - __device__ size_type size() const; - - __device__ bool nullable() const; - - __device__ bitmask_type const* null_mask() const; - - __device__ size_type offset() const; - - __device__ bool is_valid(size_type idx) const; - - __device__ bool is_valid_nocheck(size_type idx) const; - - __device__ bool is_null(size_type idx) const; - - __device__ bool is_null_nocheck(size_type idx) const; - - __device__ bitmask_type get_mask_word(size_type word_index) const; - - template - __device__ T element(size_type idx) const; - - template - __device__ optional nullable_element(size_type idx) const; - - __device__ size_type num_child_columns() const; -}; - -struct alignas(16) CUDF_LTO_ALIAS mutable_column_device_view { - private: - data_type _type = {}; - size_type _size = 0; - void const* _data = nullptr; - bitmask_type const* _null_mask = nullptr; - size_type _offset = 0; - mutable_column_device_view* _d_children = nullptr; - size_type _num_children = 0; - - public: - template - __device__ T* head() const; - - __device__ size_type size() const; - - __device__ bool nullable() const; - - __device__ bitmask_type* null_mask() const; - - __device__ size_type offset() const; - - __device__ bool is_valid(size_type idx) const; - - __device__ bool is_valid_nocheck(size_type idx) const; - - __device__ bool is_null(size_type idx) const; - - __device__ bool is_null_nocheck(size_type idx) const; - - __device__ bitmask_type get_mask_word(size_type word_index) const; - - template - __device__ T element(size_type idx) const; - - template - __device__ optional nullable_element(size_type idx) const; - - template - __device__ void assign(size_type idx, T value) const; -}; - -#define FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE \ - DO_IT(bool) \ - DO_IT(int8_t) \ - DO_IT(int16_t) \ - DO_IT(int32_t) \ - DO_IT(int64_t) \ - DO_IT(uint8_t) \ - DO_IT(uint16_t) \ - DO_IT(uint32_t) \ - DO_IT(uint64_t) \ - DO_IT(float32_t) \ - DO_IT(float64_t) \ - DO_IT(timestamp_D) \ - DO_IT(timestamp_h) \ - DO_IT(timestamp_m) \ - DO_IT(timestamp_s) \ - DO_IT(timestamp_ms) \ - DO_IT(timestamp_us) \ - DO_IT(timestamp_ns) \ - DO_IT(duration_D) \ - DO_IT(duration_h) \ - DO_IT(duration_m) \ - DO_IT(duration_s) \ - DO_IT(duration_ms) \ - DO_IT(duration_us) \ - DO_IT(duration_ns) - -#define FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE \ - DO_IT(bool) \ - DO_IT(int8_t) \ - DO_IT(int16_t) \ - DO_IT(int32_t) \ - DO_IT(int64_t) \ - DO_IT(uint8_t) \ - DO_IT(uint16_t) \ - DO_IT(uint32_t) \ - DO_IT(uint64_t) \ - DO_IT(decimal32) \ - DO_IT(decimal64) \ - DO_IT(decimal128) \ - DO_IT(float32_t) \ - DO_IT(float64_t) \ - DO_IT(string_view) \ - DO_IT(timestamp_D) \ - DO_IT(timestamp_h) \ - DO_IT(timestamp_m) \ - DO_IT(timestamp_s) \ - DO_IT(timestamp_ms) \ - DO_IT(timestamp_us) \ - DO_IT(timestamp_ns) \ - DO_IT(duration_D) \ - DO_IT(duration_h) \ - DO_IT(duration_m) \ - DO_IT(duration_s) \ - DO_IT(duration_ms) \ - DO_IT(duration_us) \ - DO_IT(duration_ns) - -#define FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE \ - DO_IT(bool) \ - DO_IT(int8_t) \ - DO_IT(int16_t) \ - DO_IT(int32_t) \ - DO_IT(int64_t) \ - DO_IT(uint8_t) \ - DO_IT(uint16_t) \ - DO_IT(uint32_t) \ - DO_IT(uint64_t) \ - DO_IT(decimal32) \ - DO_IT(decimal64) \ - DO_IT(decimal128) \ - DO_IT(float32_t) \ - DO_IT(float64_t) \ - DO_IT(timestamp_D) \ - DO_IT(timestamp_h) \ - DO_IT(timestamp_m) \ - DO_IT(timestamp_s) \ - DO_IT(timestamp_ms) \ - DO_IT(timestamp_us) \ - DO_IT(timestamp_ns) \ - DO_IT(duration_D) \ - DO_IT(duration_h) \ - DO_IT(duration_m) \ - DO_IT(duration_s) \ - DO_IT(duration_ms) \ - DO_IT(duration_us) \ - DO_IT(duration_ns) - -#define DO_IT(T) extern template __device__ T const* column_device_view::head() const; - -FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE - -#undef DO_IT - -#define DO_IT(T) extern template __device__ T column_device_view::element(size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - extern template __device__ optional column_device_view::nullable_element(size_type idx) \ - const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) extern template __device__ T* mutable_column_device_view::head() const; - -FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - extern template __device__ T mutable_column_device_view::element(size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - extern template __device__ optional mutable_column_device_view::nullable_element( \ - size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - extern template __device__ void mutable_column_device_view::assign(size_type idx, T value) \ - const; - -FOREACH_CUDF_LTO_COLUMN_ASSIGN_TYPE - -#undef DO_IT - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/export.cuh b/cpp/include/cudf/jit/lto/export.cuh deleted file mode 100644 index 39e58d993313..000000000000 --- a/cpp/include/cudf/jit/lto/export.cuh +++ /dev/null @@ -1,18 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) - -#define CUDF_LTO_EXPORT __attribute__((visibility("default"))) - -#else - -#define CUDF_LTO_EXPORT - -#endif - -#define CUDF_LTO_ALIAS __attribute__((may_alias)) diff --git a/cpp/include/cudf/jit/lto/library.cuh b/cpp/include/cudf/jit/lto/library.cuh index 821048b09a46..4a4a3973ddfa 100644 --- a/cpp/include/cudf/jit/lto/library.cuh +++ b/cpp/include/cudf/jit/lto/library.cuh @@ -3,14 +3,5 @@ * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include diff --git a/cpp/include/cudf/jit/lto/operators.cuh b/cpp/include/cudf/jit/lto/operators.cuh index d103561f7791..c80b4b6fb163 100644 --- a/cpp/include/cudf/jit/lto/operators.cuh +++ b/cpp/include/cudf/jit/lto/operators.cuh @@ -3,8 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#include -#include #include namespace CUDF_LTO_EXPORT cudf { @@ -13,488 +11,534 @@ namespace lto { namespace operators { -#define CUDF_UNOP(op, type, operand) \ - __device__ __forceinline__ void op(type* out, type const* a) { *out = operand(*a); } \ - __device__ __forceinline__ void op(optional* out, optional const* a) \ - { \ - if (a->has_value()) { \ - op(&(*out), &(*a)); \ - } else { \ - *out = {}; \ - } \ +#define UNOP_T(op, Ret, T, expr) \ + __device__ __forceinline__ void op(R* out, T const& a) { *out = expr; } \ + \ + __device__ __forceinline__ void op(optional* out, optional const& a) \ + { \ + if (a.has_value()) { \ + R r; \ + op(&r, *a); \ + *out = r; \ + } else { \ + *out = nullopt; \ + } \ } -#define CUDF_UNOP_T(op, ret_type, type, operand) \ - __device__ __forceinline__ void op(ret_type* out, type const* a) { *out = operand(*a); } \ - __device__ __forceinline__ void op(optional* out, optional const* a) \ - { \ - if (a->has_value()) { \ - op(&(*out), &(*a)); \ - } else { \ - *out = {}; \ - } \ - } - -#define CUDF_BINOP(op, type, operand) \ - __device__ __forceinline__ void op(type* out, type const* a, type const* b) \ - { \ - *out = *a operand * b; \ - } \ - \ - __device__ __forceinline__ void op( \ - optional* out, optional const* a, optional const* b) \ - { \ - if (a->has_value() && b->has_value()) { \ - op(&(*out), &(*a), &(*b)); \ - } else { \ - *out = {}; \ - } \ +#define BINOP_T(op, R, T, expr) \ + __device__ __forceinline__ void op(R* out, T const& a, T const& b) { *out = expr; } \ + \ + __device__ __forceinline__ void op(optional* out, optional const& a, optional const& b) \ + { \ + if (a.has_value() && b.has_value()) { \ + R r; \ + op(&r, *a, *b); \ + *out = r; \ + } else { \ + *out = nullopt; \ + } \ } -#define CUDF_PRED_BINOP(op, type, operand) \ - __device__ __forceinline__ void op(bool* out, type const* a, type const* b) \ - { \ - *out = *a operand * b; \ - } \ - \ - __device__ __forceinline__ void op(bool* out, optional const* a, optional const* b) \ - { \ - if (a->has_value() && b->has_value()) { \ - op(out, &(*a), &(*b)); \ - } else if (!a->has_value() && !b->has_value()) { \ - *out = true; \ - } else { \ - *out = false; \ - } \ - } \ - \ - __device__ __forceinline__ void op( \ - optional* out, optional const* a, optional const* b) \ - { \ - bool r; \ - op(&r, a, b); \ - *out = r; \ +#define EXTERN_UNOP_T(op, R, T) \ + __device__ extern void op(R* out, T const& a); \ + \ + __device__ __forceinline__ void op(optional* out, optional const& a) \ + { \ + if (a.has_value()) { \ + R r; \ + op(&r, *a); \ + *out = r; \ + } else { \ + *out = nullopt; \ + } \ } -#define CUDF_ID_OP(op, type) \ - __device__ __forceinline__ void op(type* out, type const* a) { *out = *a; } \ - __device__ __forceinline__ void op(optional* out, optional const* a) { *out = *a; } - -#define CUDF_IS_NULL_OP(op, type) \ - __device__ __forceinline__ void op(bool* out, type const* a) { *out = false; } \ - __device__ __forceinline__ void op(bool* out, optional const* a) \ - { \ - *out = !a->has_value(); \ - } \ - __device__ __forceinline__ void op(optional* out, optional const* a) \ - { \ - *out = !a->has_value(); \ +#define EXTERN_BINOP_T(op, R, T) \ + __device__ extern void op(R* out, T const& a, T const& b); \ + \ + __device__ __forceinline__ void op(optional* out, optional const& a, optional const& b) \ + { \ + if (a.has_value() && b.has_value()) { \ + R r; \ + op(&r, *a, *b); \ + *out = r; \ + } else { \ + *out = nullopt; \ + } \ } -#define CUDF_ABS_OP(op, type) \ - __device__ __forceinline__ void op(type* out, type const* a) { *out = (*a < 0) ? -(*a) : (*a); } \ - __device__ __forceinline__ void op(optional* out, optional const* a) \ +#define EXTERN_UNOP(op, T) EXTERN_UNOP_T(op, T, T) +#define EXTERN_BINOP(op, T) EXTERN_BINOP_T(op, T, T) + +#define ADD_OP(T) BINOP_T(add, T, T, (a + b)) +#define SUB_OP(T) BINOP_T(sub, T, T, (a - b)) +#define MUL_OP(T) BINOP_T(mul, T, T, (a * b)) +#define DIV_OP(T) BINOP_T(div, T, T, (a / b)) +#define EQ_OP(T) BINOP_T(equal, bool, T, (a == b)) +#define LT_OP(T) BINOP_T(less, bool, T, (a < b)) +#define GT_OP(T) BINOP_T(greater, bool, T, (a > b)) +#define LE_OP(T) BINOP_T(less_equal, bool, T, (a <= b)) +#define GE_OP(T) BINOP_T(greater_equal, bool, T, (a >= b)) +#define BIT_AND_OP(T) BINOP_T(bitwise_and, T, T, (a & b)) +#define BIT_OR_OP(T) BINOP_T(bitwise_or, T, T, (a | b)) +#define BIT_XOR_OP(T) BINOP_T(bitwise_xor, T, T, (a ^ b)) +#define LOGICAL_AND_OP(T) BINOP_T(logical_and, bool, T, (a && b)) +#define LOGICAL_OR_OP(T) BINOP_T(logical_or, bool, T, (a || b)) +#define IDENTITY_OP(T) UNOP_T(identity, T, T, a) +#define BIT_INVERT_OP(T) UNOP_T(bit_invert, T, T, ~a) +#define CAST_OP(out_type, in_type) UNOP_T(cast_to_##out_type, out_type, in_type, (out_type)(a)) +#define IS_NULL_OP(T) UNOP_T(is_null, bool, T, !a.has_value()) +#define ABS_OP(T) UNOP_T(abs, T, T, ((a < 0) ? -a : a)) +#define MOD_OP(T) BINOP_T(mod, T, T, (a % b)) +#define PYMOD_OP(T) BINOP_T(pymod, T, T, ((a % b + b) % b)) + +#define NULL_EQ_OP(T) \ + __device__ __forceinline__ void null_equal(bool* out, T const& a, T const& b) { *out = a == b; } \ + \ + __device__ __forceinline__ void null_equal( \ + optional* out, optional const& a, optional const& b) \ { \ - if (a->has_value()) { \ - op(&(*out), &(*a)); \ + if (a.has_value() && b.has_value()) { \ + *out = (*a == *b); \ + } else if (!a.has_value() && !b.has_value()) { \ + *out = true; \ } else { \ - *out = {}; \ + *out = false; \ } \ } -#define CUDF_EXTERN_UNOP(op, type) \ - __device__ extern void op(type* out, type const* a); \ - __device__ extern void op(optional* out, optional const* a) \ - { \ - if (a->has_value()) { \ - op(&(*out), &(*a)); \ - } else { \ - *out = {}; \ - } \ +ADD_OP(i32) +ADD_OP(i64) +ADD_OP(u32) +ADD_OP(u64) +ADD_OP(f32) +ADD_OP(f64) +ADD_OP(decimal32) +ADD_OP(decimal64) +ADD_OP(decimal128) +ADD_OP(duration_D) +ADD_OP(duration_s) +ADD_OP(duration_ms) +ADD_OP(duration_ns) + +SUB_OP(i32) +SUB_OP(i64) +SUB_OP(u32) +SUB_OP(u64) +SUB_OP(f32) +SUB_OP(f64) +SUB_OP(decimal32) +SUB_OP(decimal64) +SUB_OP(decimal128) +SUB_OP(duration_D) +SUB_OP(duration_s) +SUB_OP(duration_ms) +SUB_OP(duration_ns) + +MUL_OP(i32) +MUL_OP(i64) +MUL_OP(u32) +MUL_OP(u64) +MUL_OP(f32) +MUL_OP(f64) +MUL_OP(decimal32) +MUL_OP(decimal64) +MUL_OP(decimal128) + +DIV_OP(i32) +DIV_OP(i64) +DIV_OP(u32) +DIV_OP(u64) +DIV_OP(f32) +DIV_OP(f64) +DIV_OP(decimal32) +DIV_OP(decimal64) +DIV_OP(decimal128) + +EQ_OP(bool) +EQ_OP(i8) +EQ_OP(i16) +EQ_OP(i32) +EQ_OP(i64) +EQ_OP(u8) +EQ_OP(u16) +EQ_OP(u32) +EQ_OP(u64) +EQ_OP(f32) +EQ_OP(f64) +EQ_OP(decimal32) +EQ_OP(decimal64) +EQ_OP(decimal128) +EQ_OP(timestamp_D) +EQ_OP(timestamp_s) +EQ_OP(timestamp_ms) +EQ_OP(timestamp_us) +EQ_OP(timestamp_ns) +EQ_OP(duration_D) +EQ_OP(duration_s) +EQ_OP(duration_ms) +EQ_OP(duration_ns) +EQ_OP(string_view) + +LT_OP(bool) +LT_OP(i8) +LT_OP(i16) +LT_OP(i32) +LT_OP(i64) +LT_OP(u8) +LT_OP(u16) +LT_OP(u32) +LT_OP(u64) +LT_OP(f32) +LT_OP(f64) +LT_OP(decimal32) +LT_OP(decimal64) +LT_OP(decimal128) +LT_OP(timestamp_D) +LT_OP(timestamp_s) +LT_OP(timestamp_ms) +LT_OP(timestamp_us) +LT_OP(timestamp_ns) +LT_OP(duration_D) +LT_OP(duration_s) +LT_OP(duration_ms) +LT_OP(duration_ns) +LT_OP(string_view) + +GT_OP(bool) +GT_OP(i8) +GT_OP(i16) +GT_OP(i32) +GT_OP(i64) +GT_OP(u8) +GT_OP(u16) +GT_OP(u32) +GT_OP(u64) +GT_OP(f32) +GT_OP(f64) +GT_OP(decimal32) +GT_OP(decimal64) +GT_OP(decimal128) +GT_OP(timestamp_D) +GT_OP(timestamp_s) +GT_OP(timestamp_ms) +GT_OP(timestamp_us) +GT_OP(timestamp_ns) +GT_OP(duration_D) +GT_OP(duration_s) +GT_OP(duration_ms) +GT_OP(duration_ns) +GT_OP(string_view) + +LE_OP(bool) +LE_OP(i8) +LE_OP(i16) +LE_OP(i32) +LE_OP(i64) +LE_OP(u8) +LE_OP(u16) +LE_OP(u32) +LE_OP(u64) +LE_OP(f32) +LE_OP(f64) +LE_OP(decimal32) +LE_OP(decimal64) +LE_OP(decimal128) +LE_OP(timestamp_D) +LE_OP(timestamp_s) +LE_OP(timestamp_ms) +LE_OP(timestamp_us) +LE_OP(timestamp_ns) +LE_OP(duration_D) +LE_OP(duration_s) +LE_OP(duration_ms) +LE_OP(duration_ns) +LE_OP(string_view) + +GE_OP(bool) +GE_OP(i8) +GE_OP(i16) +GE_OP(i32) +GE_OP(i64) +GE_OP(u8) +GE_OP(u16) +GE_OP(u32) +GE_OP(u64) +GE_OP(f32) +GE_OP(f64) +GE_OP(decimal32) +GE_OP(decimal64) +GE_OP(decimal128) +GE_OP(timestamp_D) +GE_OP(timestamp_s) +GE_OP(timestamp_ms) +GE_OP(timestamp_us) +GE_OP(timestamp_ns) +GE_OP(duration_D) +GE_OP(duration_s) +GE_OP(duration_ms) +GE_OP(duration_ns) +GE_OP(string_view) + +BIT_AND_OP(i32) +BIT_AND_OP(i64) +BIT_AND_OP(u32) +BIT_AND_OP(u64) + +BIT_OR_OP(i32) +BIT_OR_OP(i64) +BIT_OR_OP(u32) +BIT_OR_OP(u64) + +BIT_XOR_OP(i32) +BIT_XOR_OP(i64) +BIT_XOR_OP(u32) +BIT_XOR_OP(u64) + +LOGICAL_AND_OP(bool) + +LOGICAL_OR_OP(bool) + +IDENTITY_OP(bool) +IDENTITY_OP(i8) +IDENTITY_OP(i16) +IDENTITY_OP(i32) +IDENTITY_OP(i64) +IDENTITY_OP(u8) +IDENTITY_OP(u16) +IDENTITY_OP(u32) +IDENTITY_OP(u64) +IDENTITY_OP(f32) +IDENTITY_OP(f64) +IDENTITY_OP(decimal32) +IDENTITY_OP(decimal64) +IDENTITY_OP(decimal128) +IDENTITY_OP(timestamp_D) +IDENTITY_OP(timestamp_s) +IDENTITY_OP(timestamp_ms) +IDENTITY_OP(timestamp_us) +IDENTITY_OP(timestamp_ns) +IDENTITY_OP(duration_D) +IDENTITY_OP(duration_s) +IDENTITY_OP(duration_ms) +IDENTITY_OP(duration_ns) +IDENTITY_OP(string_view) + +BIT_INVERT_OP(u32) +BIT_INVERT_OP(u64) +BIT_INVERT_OP(i32) +BIT_INVERT_OP(i64) + +CAST_OP(i64, bool) +CAST_OP(i64, i8) +CAST_OP(i64, i16) +CAST_OP(i64, i32) +CAST_OP(i64, i64) +CAST_OP(i64, u8) +CAST_OP(i64, u16) +CAST_OP(i64, u32) +CAST_OP(i64, u64) +CAST_OP(i64, f32) +CAST_OP(i64, f64) + +CAST_OP(u64, bool) +CAST_OP(u64, i8) +CAST_OP(u64, i16) +CAST_OP(u64, i32) +CAST_OP(u64, i64) +CAST_OP(u64, u8) +CAST_OP(u64, u16) +CAST_OP(u64, u32) +CAST_OP(u64, u64) +CAST_OP(u64, f32) +CAST_OP(u64, f64) + +CAST_OP(f64, bool) +CAST_OP(f64, i8) +CAST_OP(f64, i16) +CAST_OP(f64, i32) +CAST_OP(f64, i64) +CAST_OP(f64, u8) +CAST_OP(f64, u16) +CAST_OP(f64, u32) +CAST_OP(f64, u64) +CAST_OP(f64, f32) +CAST_OP(f64, f64) + +IS_NULL_OP(bool) +IS_NULL_OP(i8) +IS_NULL_OP(i16) +IS_NULL_OP(i32) +IS_NULL_OP(i64) +IS_NULL_OP(u8) +IS_NULL_OP(u16) +IS_NULL_OP(u32) +IS_NULL_OP(u64) +IS_NULL_OP(f32) +IS_NULL_OP(f64) +IS_NULL_OP(decimal32) +IS_NULL_OP(decimal64) +IS_NULL_OP(decimal128) +IS_NULL_OP(timestamp_D) +IS_NULL_OP(timestamp_s) +IS_NULL_OP(timestamp_ms) +IS_NULL_OP(timestamp_us) +IS_NULL_OP(timestamp_ns) +IS_NULL_OP(duration_D) +IS_NULL_OP(duration_s) +IS_NULL_OP(duration_ms) +IS_NULL_OP(duration_ns) +IS_NULL_OP(string_view) + +ABS_OP(i8) +ABS_OP(i16) +ABS_OP(i32) +ABS_OP(i64) +ABS_OP(f32) +ABS_OP(f64) + +EXTERN_UNOP(sin, f32); +EXTERN_UNOP(sin, f64); + +EXTERN_UNOP(cos, f32); +EXTERN_UNOP(cos, f64); + +EXTERN_UNOP(tan, f32); +EXTERN_UNOP(tan, f64); + +EXTERN_UNOP(arcsin, f32); +EXTERN_UNOP(arcsin, f64); + +EXTERN_UNOP(arccos, f32); +EXTERN_UNOP(arccos, f64); + +EXTERN_UNOP(arctan, f32); +EXTERN_UNOP(arctan, f64); + +EXTERN_UNOP(sinh, f32); +EXTERN_UNOP(sinh, f64); + +EXTERN_UNOP(cosh, f32); +EXTERN_UNOP(cosh, f64); + +EXTERN_UNOP(tanh, f32); +EXTERN_UNOP(tanh, f64); + +EXTERN_UNOP(arcsinh, f32); +EXTERN_UNOP(arcsinh, f64); + +EXTERN_UNOP(arccosh, f32); +EXTERN_UNOP(arccosh, f64); + +EXTERN_UNOP(arctanh, f32); +EXTERN_UNOP(arctanh, f64); + +EXTERN_UNOP(exp, f32); +EXTERN_UNOP(exp, f64); + +EXTERN_UNOP(log, f32); +EXTERN_UNOP(log, f64); + +EXTERN_UNOP(cbrt, f32); +EXTERN_UNOP(cbrt, f64); + +EXTERN_UNOP(ceil, f32); +EXTERN_UNOP(ceil, f64); + +EXTERN_UNOP(floor, f32); +EXTERN_UNOP(floor, f64); + +EXTERN_UNOP(rint, f32); +EXTERN_UNOP(rint, f64); + +MOD_OP(i32) +MOD_OP(i64) +MOD_OP(u32) +MOD_OP(u64) +EXTERN_BINOP(mod, f32); +EXTERN_BINOP(mod, f64); + +PYMOD_OP(i32) +PYMOD_OP(i64) +PYMOD_OP(u32) +PYMOD_OP(u64) +EXTERN_BINOP(pymod, f32); +EXTERN_BINOP(pymod, f64); + +EXTERN_BINOP(pow, f32); +EXTERN_BINOP(pow, f64); + +NULL_EQ_OP(bool) +NULL_EQ_OP(i8) +NULL_EQ_OP(i16) +NULL_EQ_OP(i32) +NULL_EQ_OP(i64) +NULL_EQ_OP(u8) +NULL_EQ_OP(u16) +NULL_EQ_OP(u32) +NULL_EQ_OP(u64) +NULL_EQ_OP(f32) +NULL_EQ_OP(f64) +NULL_EQ_OP(decimal32) +NULL_EQ_OP(decimal64) +NULL_EQ_OP(decimal128) +NULL_EQ_OP(timestamp_D) +NULL_EQ_OP(timestamp_s) +NULL_EQ_OP(timestamp_ms) +NULL_EQ_OP(timestamp_us) +NULL_EQ_OP(timestamp_ns) +NULL_EQ_OP(duration_D) +NULL_EQ_OP(duration_s) +NULL_EQ_OP(duration_ms) +NULL_EQ_OP(duration_ns) +NULL_EQ_OP(string_view) + +__device__ __forceinline__ void null_logical_and(bool* out, bool const& a, bool const& b) +{ + *out = a && b; +} + +__device__ __forceinline__ void null_logical_and(optional* out, + optional const& a, + optional const& b) +{ + if (a.has_value() && b.has_value()) { + *out = (*a && *b); + } else if (!a.has_value() && !b.has_value()) { + *out = nullopt; + } else { + bool valid = a.has_value() ? *a : *b; + if (valid) { + *out = nullopt; + } else { + *out = false; + } } - -CUDF_BINOP(add, int32_t, +); -CUDF_BINOP(add, int64_t, +); -CUDF_BINOP(add, uint32_t, +); -CUDF_BINOP(add, uint64_t, +); -CUDF_BINOP(add, float32_t, +); -CUDF_BINOP(add, float64_t, +); -CUDF_BINOP(add, decimal32, +); -CUDF_BINOP(add, decimal64, +); -CUDF_BINOP(add, decimal128, +); -CUDF_BINOP(add, duration_D, +); -CUDF_BINOP(add, duration_s, +); -CUDF_BINOP(add, duration_ms, +); -CUDF_BINOP(add, duration_ns, +); - -CUDF_BINOP(sub, int32_t, -); -CUDF_BINOP(sub, int64_t, -); -CUDF_BINOP(sub, uint32_t, -); -CUDF_BINOP(sub, uint64_t, -); -CUDF_BINOP(sub, float32_t, -); -CUDF_BINOP(sub, float64_t, -); -CUDF_BINOP(sub, decimal32, -); -CUDF_BINOP(sub, decimal64, -); -CUDF_BINOP(sub, decimal128, -); -CUDF_BINOP(sub, duration_D, -); -CUDF_BINOP(sub, duration_s, -); -CUDF_BINOP(sub, duration_ms, -); -CUDF_BINOP(sub, duration_ns, -); - -CUDF_BINOP(mul, int32_t, *); -CUDF_BINOP(mul, int64_t, *); -CUDF_BINOP(mul, uint32_t, *); -CUDF_BINOP(mul, uint64_t, *); -CUDF_BINOP(mul, float32_t, *); -CUDF_BINOP(mul, float64_t, *); -CUDF_BINOP(mul, decimal32, *); -CUDF_BINOP(mul, decimal64, *); -CUDF_BINOP(mul, decimal128, *); - -CUDF_BINOP(div, int32_t, /); -CUDF_BINOP(div, int64_t, /); -CUDF_BINOP(div, uint32_t, /); -CUDF_BINOP(div, uint64_t, /); -CUDF_BINOP(div, float32_t, /); -CUDF_BINOP(div, float64_t, /); -CUDF_BINOP(div, decimal32, /); -CUDF_BINOP(div, decimal64, /); -CUDF_BINOP(div, decimal128, /); - -// CUDF_OP(mod, float32_t); -// CUDF_OP(mod, float64_t); - -// CUDF_OP(pymod, float32_t); -// CUDF_OP(pymod, float64_t); - -// CUDF_OP(pow, float32_t); -// CUDF_OP(pow, float64_t); - -CUDF_PRED_BINOP(equal, bool, ==); -CUDF_PRED_BINOP(equal, int8_t, ==); -CUDF_PRED_BINOP(equal, int16_t, ==); -CUDF_PRED_BINOP(equal, int32_t, ==); -CUDF_PRED_BINOP(equal, int64_t, ==); -CUDF_PRED_BINOP(equal, uint8_t, ==); -CUDF_PRED_BINOP(equal, uint16_t, ==); -CUDF_PRED_BINOP(equal, uint32_t, ==); -CUDF_PRED_BINOP(equal, uint64_t, ==); -CUDF_PRED_BINOP(equal, float32_t, ==); -CUDF_PRED_BINOP(equal, float64_t, ==); -CUDF_PRED_BINOP(equal, decimal32, ==); -CUDF_PRED_BINOP(equal, decimal64, ==); -CUDF_PRED_BINOP(equal, decimal128, ==); -CUDF_PRED_BINOP(equal, timestamp_D, ==); -CUDF_PRED_BINOP(equal, timestamp_s, ==); -CUDF_PRED_BINOP(equal, timestamp_ms, ==); -CUDF_PRED_BINOP(equal, timestamp_us, ==); -CUDF_PRED_BINOP(equal, timestamp_ns, ==); -CUDF_PRED_BINOP(equal, duration_D, ==); -CUDF_PRED_BINOP(equal, duration_s, ==); -CUDF_PRED_BINOP(equal, duration_ms, ==); -CUDF_PRED_BINOP(equal, duration_ns, ==); -CUDF_PRED_BINOP(equal, string_view, ==); - -/* -CUDF_OP(null_equal, bool); -CUDF_OP(null_equal, int8_t); -CUDF_OP(null_equal, int16_t); -CUDF_OP(null_equal, int32_t); -CUDF_OP(null_equal, int64_t); -CUDF_OP(null_equal, uint8_t); -CUDF_OP(null_equal, uint16_t); -CUDF_OP(null_equal, uint32_t); -CUDF_OP(null_equal, uint64_t); -CUDF_OP(null_equal, float32_t); -CUDF_OP(null_equal, float64_t); -CUDF_OP(null_equal, decimal32); -CUDF_OP(null_equal, decimal64); -CUDF_OP(null_equal, decimal128); -CUDF_OP(null_equal, timestamp_D); -CUDF_OP(null_equal, timestamp_s); -CUDF_OP(null_equal, timestamp_ms); -CUDF_OP(null_equal, timestamp_us); -CUDF_OP(null_equal, timestamp_ns); -CUDF_OP(null_equal, duration_D); -CUDF_OP(null_equal, duration_s); -CUDF_OP(null_equal, duration_ms); -CUDF_OP(null_equal, duration_ns); -CUDF_OP(null_equal, string_view); -*/ - -CUDF_PRED_BINOP(less, bool, <); -CUDF_PRED_BINOP(less, int8_t, <); -CUDF_PRED_BINOP(less, int16_t, <); -CUDF_PRED_BINOP(less, int32_t, <); -CUDF_PRED_BINOP(less, int64_t, <); -CUDF_PRED_BINOP(less, uint8_t, <); -CUDF_PRED_BINOP(less, uint16_t, <); -CUDF_PRED_BINOP(less, uint32_t, <); -CUDF_PRED_BINOP(less, uint64_t, <); -CUDF_PRED_BINOP(less, float32_t, <); -CUDF_PRED_BINOP(less, float64_t, <); -CUDF_PRED_BINOP(less, decimal32, <); -CUDF_PRED_BINOP(less, decimal64, <); -CUDF_PRED_BINOP(less, decimal128, <); -CUDF_PRED_BINOP(less, timestamp_D, <); -CUDF_PRED_BINOP(less, timestamp_s, <); -CUDF_PRED_BINOP(less, timestamp_ms, <); -CUDF_PRED_BINOP(less, timestamp_us, <); -CUDF_PRED_BINOP(less, timestamp_ns, <); -CUDF_PRED_BINOP(less, duration_D, <); -CUDF_PRED_BINOP(less, duration_s, <); -CUDF_PRED_BINOP(less, duration_ms, <); -CUDF_PRED_BINOP(less, duration_ns, <); -CUDF_PRED_BINOP(less, string_view, <); - -CUDF_PRED_BINOP(greater, bool, >); -CUDF_PRED_BINOP(greater, int8_t, >); -CUDF_PRED_BINOP(greater, int16_t, >); -CUDF_PRED_BINOP(greater, int32_t, >); -CUDF_PRED_BINOP(greater, int64_t, >); -CUDF_PRED_BINOP(greater, uint8_t, >); -CUDF_PRED_BINOP(greater, uint16_t, >); -CUDF_PRED_BINOP(greater, uint32_t, >); -CUDF_PRED_BINOP(greater, uint64_t, >); -CUDF_PRED_BINOP(greater, float32_t, >); -CUDF_PRED_BINOP(greater, float64_t, >); -CUDF_PRED_BINOP(greater, decimal32, >); -CUDF_PRED_BINOP(greater, decimal64, >); -CUDF_PRED_BINOP(greater, decimal128, >); -CUDF_PRED_BINOP(greater, timestamp_D, >); -CUDF_PRED_BINOP(greater, timestamp_s, >); -CUDF_PRED_BINOP(greater, timestamp_ms, >); -CUDF_PRED_BINOP(greater, timestamp_us, >); -CUDF_PRED_BINOP(greater, timestamp_ns, >); -CUDF_PRED_BINOP(greater, duration_D, >); -CUDF_PRED_BINOP(greater, duration_s, >); -CUDF_PRED_BINOP(greater, duration_ms, >); -CUDF_PRED_BINOP(greater, duration_ns, >); -CUDF_PRED_BINOP(greater, string_view, >); - -CUDF_PRED_BINOP(less_equal, bool, <=); -CUDF_PRED_BINOP(less_equal, int8_t, <=); -CUDF_PRED_BINOP(less_equal, int16_t, <=); -CUDF_PRED_BINOP(less_equal, int32_t, <=); -CUDF_PRED_BINOP(less_equal, int64_t, <=); -CUDF_PRED_BINOP(less_equal, uint8_t, <=); -CUDF_PRED_BINOP(less_equal, uint16_t, <=); -CUDF_PRED_BINOP(less_equal, uint32_t, <=); -CUDF_PRED_BINOP(less_equal, uint64_t, <=); -CUDF_PRED_BINOP(less_equal, float32_t, <=); -CUDF_PRED_BINOP(less_equal, float64_t, <=); -CUDF_PRED_BINOP(less_equal, decimal32, <=); -CUDF_PRED_BINOP(less_equal, decimal64, <=); -CUDF_PRED_BINOP(less_equal, decimal128, <=); -CUDF_PRED_BINOP(less_equal, timestamp_D, <=); -CUDF_PRED_BINOP(less_equal, timestamp_s, <=); -CUDF_PRED_BINOP(less_equal, timestamp_ms, <=); -CUDF_PRED_BINOP(less_equal, timestamp_us, <=); -CUDF_PRED_BINOP(less_equal, timestamp_ns, <=); -CUDF_PRED_BINOP(less_equal, duration_D, <=); -CUDF_PRED_BINOP(less_equal, duration_s, <=); -CUDF_PRED_BINOP(less_equal, duration_ms, <=); -CUDF_PRED_BINOP(less_equal, duration_ns, <=); -CUDF_PRED_BINOP(less_equal, string_view, <=); - -CUDF_PRED_BINOP(greater_equal, bool, >=); -CUDF_PRED_BINOP(greater_equal, int8_t, >=); -CUDF_PRED_BINOP(greater_equal, int16_t, >=); -CUDF_PRED_BINOP(greater_equal, int32_t, >=); -CUDF_PRED_BINOP(greater_equal, int64_t, >=); -CUDF_PRED_BINOP(greater_equal, uint8_t, >=); -CUDF_PRED_BINOP(greater_equal, uint16_t, >=); -CUDF_PRED_BINOP(greater_equal, uint32_t, >=); -CUDF_PRED_BINOP(greater_equal, uint64_t, >=); -CUDF_PRED_BINOP(greater_equal, float32_t, >=); -CUDF_PRED_BINOP(greater_equal, float64_t, >=); -CUDF_PRED_BINOP(greater_equal, decimal32, >=); -CUDF_PRED_BINOP(greater_equal, decimal64, >=); -CUDF_PRED_BINOP(greater_equal, decimal128, >=); -CUDF_PRED_BINOP(greater_equal, timestamp_D, >=); -CUDF_PRED_BINOP(greater_equal, timestamp_s, >=); -CUDF_PRED_BINOP(greater_equal, timestamp_ms, >=); -CUDF_PRED_BINOP(greater_equal, timestamp_us, >=); -CUDF_PRED_BINOP(greater_equal, timestamp_ns, >=); -CUDF_PRED_BINOP(greater_equal, duration_D, >=); -CUDF_PRED_BINOP(greater_equal, duration_s, >=); -CUDF_PRED_BINOP(greater_equal, duration_ms, >=); -CUDF_PRED_BINOP(greater_equal, duration_ns, >=); -CUDF_PRED_BINOP(greater_equal, string_view, >=); - -CUDF_BINOP(bitwise_and, int32_t, &); -CUDF_BINOP(bitwise_and, int64_t, &); -CUDF_BINOP(bitwise_and, uint32_t, &); -CUDF_BINOP(bitwise_and, uint64_t, &); - -CUDF_BINOP(bitwise_or, int32_t, |); -CUDF_BINOP(bitwise_or, int64_t, |); -CUDF_BINOP(bitwise_or, uint32_t, |); -CUDF_BINOP(bitwise_or, uint64_t, |); - -CUDF_BINOP(bitwise_xor, int32_t, ^); -CUDF_BINOP(bitwise_xor, int64_t, ^); -CUDF_BINOP(bitwise_xor, uint32_t, ^); -CUDF_BINOP(bitwise_xor, uint64_t, ^); - -CUDF_BINOP(logical_and, bool, &&); -CUDF_BINOP(null_logical_and, bool, &&); -CUDF_BINOP(logical_or, bool, ||); -CUDF_BINOP(null_logical_or, bool, ||); - -CUDF_ID_OP(identity, bool); -CUDF_ID_OP(identity, int8_t); -CUDF_ID_OP(identity, int16_t); -CUDF_ID_OP(identity, int32_t); -CUDF_ID_OP(identity, int64_t); -CUDF_ID_OP(identity, uint8_t); -CUDF_ID_OP(identity, uint16_t); -CUDF_ID_OP(identity, uint32_t); -CUDF_ID_OP(identity, uint64_t); -CUDF_ID_OP(identity, float32_t); -CUDF_ID_OP(identity, float64_t); -CUDF_ID_OP(identity, decimal32); -CUDF_ID_OP(identity, decimal64); -CUDF_ID_OP(identity, decimal128); -CUDF_ID_OP(identity, timestamp_D); -CUDF_ID_OP(identity, timestamp_s); -CUDF_ID_OP(identity, timestamp_ms); -CUDF_ID_OP(identity, timestamp_us); -CUDF_ID_OP(identity, timestamp_ns); -CUDF_ID_OP(identity, duration_D); -CUDF_ID_OP(identity, duration_s); -CUDF_ID_OP(identity, duration_ms); -CUDF_ID_OP(identity, duration_ns); -CUDF_ID_OP(identity, string_view); - -CUDF_UNOP(bit_invert, uint32_t, ~); -CUDF_UNOP(bit_invert, uint64_t, ~); -CUDF_UNOP(bit_invert, int32_t, ~); -CUDF_UNOP(bit_invert, int64_t, ~); - -CUDF_UNOP_T(cast_to_int64, int64_t, bool, (int64_t)); -CUDF_UNOP_T(cast_to_int64, int64_t, int8_t, (int64_t)); -CUDF_UNOP_T(cast_to_int64, int64_t, int16_t, (int64_t)); -CUDF_UNOP_T(cast_to_int64, int64_t, int32_t, (int64_t)); -CUDF_UNOP_T(cast_to_int64, int64_t, int64_t, (int64_t)); -CUDF_UNOP_T(cast_to_int64, int64_t, uint8_t, (int64_t)); -CUDF_UNOP_T(cast_to_int64, int64_t, uint16_t, (int64_t)); -CUDF_UNOP_T(cast_to_int64, int64_t, uint32_t, (int64_t)); -CUDF_UNOP_T(cast_to_int64, int64_t, uint64_t, (int64_t)); -CUDF_UNOP_T(cast_to_int64, int64_t, float32_t, (int64_t)); -CUDF_UNOP_T(cast_to_int64, int64_t, float64_t, (int64_t)); - -CUDF_UNOP_T(cast_to_uint64, uint64_t, bool, (uint64_t)); -CUDF_UNOP_T(cast_to_uint64, uint64_t, int8_t, (uint64_t)); -CUDF_UNOP_T(cast_to_uint64, uint64_t, int16_t, (uint64_t)); -CUDF_UNOP_T(cast_to_uint64, uint64_t, int32_t, (uint64_t)); -CUDF_UNOP_T(cast_to_uint64, uint64_t, int64_t, (uint64_t)); -CUDF_UNOP_T(cast_to_uint64, uint64_t, uint8_t, (uint64_t)); -CUDF_UNOP_T(cast_to_uint64, uint64_t, uint16_t, (uint64_t)); -CUDF_UNOP_T(cast_to_uint64, uint64_t, uint32_t, (uint64_t)); -CUDF_UNOP_T(cast_to_uint64, uint64_t, uint64_t, (uint64_t)); -CUDF_UNOP_T(cast_to_uint64, uint64_t, float32_t, (uint64_t)); -CUDF_UNOP_T(cast_to_uint64, uint64_t, float64_t, (uint64_t)); - -CUDF_UNOP_T(cast_to_float64, float64_t, bool, (float64_t)); -CUDF_UNOP_T(cast_to_float64, float64_t, int8_t, (float64_t)); -CUDF_UNOP_T(cast_to_float64, float64_t, int16_t, (float64_t)); -CUDF_UNOP_T(cast_to_float64, float64_t, int32_t, (float64_t)); -CUDF_UNOP_T(cast_to_float64, float64_t, int64_t, (float64_t)); -CUDF_UNOP_T(cast_to_float64, float64_t, uint8_t, (float64_t)); -CUDF_UNOP_T(cast_to_float64, float64_t, uint16_t, (float64_t)); -CUDF_UNOP_T(cast_to_float64, float64_t, uint32_t, (float64_t)); -CUDF_UNOP_T(cast_to_float64, float64_t, uint64_t, (float64_t)); -CUDF_UNOP_T(cast_to_float64, float64_t, float32_t, (float64_t)); -CUDF_UNOP_T(cast_to_float64, float64_t, float64_t, (float64_t)); - -CUDF_IS_NULL_OP(is_null, bool); -CUDF_IS_NULL_OP(is_null, int8_t); -CUDF_IS_NULL_OP(is_null, int16_t); -CUDF_IS_NULL_OP(is_null, int32_t); -CUDF_IS_NULL_OP(is_null, int64_t); -CUDF_IS_NULL_OP(is_null, uint8_t); -CUDF_IS_NULL_OP(is_null, uint16_t); -CUDF_IS_NULL_OP(is_null, uint32_t); -CUDF_IS_NULL_OP(is_null, uint64_t); -CUDF_IS_NULL_OP(is_null, float32_t); -CUDF_IS_NULL_OP(is_null, float64_t); -CUDF_IS_NULL_OP(is_null, decimal32); -CUDF_IS_NULL_OP(is_null, decimal64); -CUDF_IS_NULL_OP(is_null, decimal128); -CUDF_IS_NULL_OP(is_null, timestamp_D); -CUDF_IS_NULL_OP(is_null, timestamp_s); -CUDF_IS_NULL_OP(is_null, timestamp_ms); -CUDF_IS_NULL_OP(is_null, timestamp_us); -CUDF_IS_NULL_OP(is_null, timestamp_ns); -CUDF_IS_NULL_OP(is_null, duration_D); -CUDF_IS_NULL_OP(is_null, duration_s); -CUDF_IS_NULL_OP(is_null, duration_ms); -CUDF_IS_NULL_OP(is_null, duration_ns); -CUDF_IS_NULL_OP(is_null, string_view); - -CUDF_ABS_OP(abs, int8_t); -CUDF_ABS_OP(abs, int16_t); -CUDF_ABS_OP(abs, int32_t); -CUDF_ABS_OP(abs, int64_t); -CUDF_ABS_OP(abs, float32_t); -CUDF_ABS_OP(abs, float64_t); - -CUDF_EXTERN_UNOP(sin, float32_t); -CUDF_EXTERN_UNOP(sin, float64_t); - -CUDF_EXTERN_UNOP(cos, float32_t); -CUDF_EXTERN_UNOP(cos, float64_t); - -CUDF_EXTERN_UNOP(tan, float32_t); -CUDF_EXTERN_UNOP(tan, float64_t); - -CUDF_EXTERN_UNOP(arcsin, float32_t); -CUDF_EXTERN_UNOP(arcsin, float64_t); - -CUDF_EXTERN_UNOP(arccos, float32_t); -CUDF_EXTERN_UNOP(arccos, float64_t); - -CUDF_EXTERN_UNOP(arctan, float32_t); -CUDF_EXTERN_UNOP(arctan, float64_t); - -CUDF_EXTERN_UNOP(sinh, float32_t); -CUDF_EXTERN_UNOP(sinh, float64_t); - -CUDF_EXTERN_UNOP(cosh, float32_t); -CUDF_EXTERN_UNOP(cosh, float64_t); - -CUDF_EXTERN_UNOP(tanh, float32_t); -CUDF_EXTERN_UNOP(tanh, float64_t); - -CUDF_EXTERN_UNOP(arcsinh, float32_t); -CUDF_EXTERN_UNOP(arcsinh, float64_t); - -CUDF_EXTERN_UNOP(arccosh, float32_t); -CUDF_EXTERN_UNOP(arccosh, float64_t); - -CUDF_EXTERN_UNOP(arctanh, float32_t); -CUDF_EXTERN_UNOP(arctanh, float64_t); - -CUDF_EXTERN_UNOP(exp, float32_t); -CUDF_EXTERN_UNOP(exp, float64_t); - -CUDF_EXTERN_UNOP(log, float32_t); -CUDF_EXTERN_UNOP(log, float64_t); - -CUDF_EXTERN_UNOP(cbrt, float32_t); -CUDF_EXTERN_UNOP(cbrt, float64_t); - -CUDF_EXTERN_UNOP(ceil, float32_t); -CUDF_EXTERN_UNOP(ceil, float64_t); - -CUDF_EXTERN_UNOP(floor, float32_t); -CUDF_EXTERN_UNOP(floor, float64_t); - -CUDF_EXTERN_UNOP(rint, float32_t); -CUDF_EXTERN_UNOP(rint, float64_t); +} + +__device__ __forceinline__ void null_logical_or(bool* out, bool const& a, bool const& b) +{ + *out = a || b; +} + +__device__ __forceinline__ void null_logical_or(optional* out, + optional const& a, + optional const& b) +{ + if (a.has_value() && b.has_value()) { + *out = (*a || *b); + } else if (!a.has_value() && !b.has_value()) { + *out = nullopt; + } else { + bool valid = a.has_value() ? *a : *b; + if (valid) { + *out = true; + } else { + *out = nullopt; + } + } +} } // namespace operators - } // namespace lto } // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/optional.cuh b/cpp/include/cudf/jit/lto/optional.cuh deleted file mode 100644 index 33bcbc79a792..000000000000 --- a/cpp/include/cudf/jit/lto/optional.cuh +++ /dev/null @@ -1,74 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include - -namespace CUDF_LTO_EXPORT cudf { -namespace lto { - -struct inplace_t {}; - -inline constexpr inplace_t inplace{}; - -// TODO: assumes T is trivially copyable -template -struct CUDF_LTO_ALIAS optional { - private: - T __val; - bool __engaged; - - public: - __device__ constexpr optional() : __val{}, __engaged{false} {} - - template - __device__ constexpr optional(inplace_t, Args&&... args) - : __val{static_cast(args)...}, __engaged{true} - { - } - - __device__ constexpr optional(T val) : __val{val}, __engaged{true} {} - - constexpr optional(optional const&) = default; - - constexpr optional(optional&&) = default; - - constexpr optional& operator=(optional const&) = default; - - constexpr optional& operator=(optional&&) = default; - - constexpr ~optional() = default; - - __device__ constexpr bool has_value() const { return __engaged; } - - __device__ constexpr void reset() { __engaged = false; } - - __device__ constexpr T const& get() const { return __val; } - - __device__ constexpr T& get() { return __val; } - - __device__ constexpr T const* operator->() const { return &__val; } - - __device__ constexpr T* operator->() { return &__val; } - - __device__ constexpr T const& operator*() const { return __val; } - - __device__ constexpr T& operator*() { return __val; } - - __device__ constexpr T const& value() const { return __val; } - - __device__ constexpr T& value() { return __val; } - - __device__ constexpr explicit operator bool() const { return __engaged; } - - __device__ constexpr T value_or(T __v) const { return __engaged ? __val : __v; } -}; - -template -optional(T) -> optional; - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/optional_span.cuh b/cpp/include/cudf/jit/lto/optional_span.cuh deleted file mode 100644 index 1dad8ea25669..000000000000 --- a/cpp/include/cudf/jit/lto/optional_span.cuh +++ /dev/null @@ -1,69 +0,0 @@ - - -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include -#include -#include - -namespace CUDF_LTO_EXPORT cudf { -namespace lto { - -__device__ constexpr bool bit_is_set(bitmask_type const* bitmask, size_t bit_index) -{ - constexpr auto bits_per_word = sizeof(bitmask_type) * 8; - return bitmask[bit_index / bits_per_word] & (bitmask_type{1} << (bit_index % bits_per_word)); -} - -template -struct [[nodiscard]] optional_span { - private: - T* _data = nullptr; - size_t _size = 0; - bitmask_type* _null_mask = nullptr; - - public: - __device__ T* data() const { return _data; } - - __device__ size_t size() const { return _size; } - - __device__ bool empty() const { return _size == 0; } - - __device__ T& operator[](size_t pos) const { return _data[pos]; } - - __device__ T* begin() const { return _data; } - - __device__ T* end() const { return _data + _size; } - - __device__ optional_span as_const() const - { - return optional_span{_data, _size, _null_mask}; - } - - __device__ bool nullable() const { return _null_mask != nullptr; } - - [[nodiscard]] __device__ bool is_valid_nocheck(size_t element_index) const - { - return bit_is_set(_null_mask, element_index); - } - - __device__ bool is_valid(size_t element_index) const - { - return not nullable() or is_valid_nocheck(element_index); - } - - __device__ bool is_null(size_t element_index) const { return !is_valid(element_index); } - - __device__ T& element(size_t idx) const { return _data[idx]; } - - __device__ optional nullable_element(size_t idx) const; - - __device__ void assign(size_t idx, T value) const { _data[idx] = value; } -}; - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/scope.cuh b/cpp/include/cudf/jit/lto/scope.cuh deleted file mode 100644 index 1705c8740a68..000000000000 --- a/cpp/include/cudf/jit/lto/scope.cuh +++ /dev/null @@ -1,82 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include - -// TODO: scope variables should be aligned to avoid uncoalesced reads/writes - -namespace CUDF_LTO_EXPORT cudf { -namespace lto { -namespace scope { - -using args = void* const*; - -template -struct [[nodiscard]] column { - static constexpr bool IS_SCALAR = IsScalar; - static constexpr bool IS_NULLABLE = IsNullable; - - using Type = T; - using Arg = ColumnType const*; - - static __device__ decltype(auto) element(args scope, size_type i) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - if constexpr (!IsNullable) { - return p->template element(index); - } else { - return p->template nullable_element(index); - } - } - - static __device__ void assign(args scope, size_type i, T value) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - p->template assign(index, value); - } - - static __device__ auto* null_mask(args scope) - { - auto p = static_cast(scope[ScopeIndex]); - return p->null_mask(); - } - - static __device__ bool is_null(args scope, size_type i) - { - if constexpr (!IsNullable) { return false; } - - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - return p->is_null(index); - } - - static __device__ bool is_valid(args scope, size_type i) { return !is_null(scope, i); } -}; - -template -struct [[nodiscard]] user_data { - using Arg = void*; - - static __device__ decltype(auto) element(args scope, [[maybe_unused]] size_type i) - { - return static_cast(scope[ScopeIndex]); - } -}; - -} // namespace scope -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/span.cuh b/cpp/include/cudf/jit/lto/span.cuh deleted file mode 100644 index 75912e57981b..000000000000 --- a/cpp/include/cudf/jit/lto/span.cuh +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include -#include - -namespace CUDF_LTO_EXPORT cudf { -namespace lto { - -template -struct [[nodiscard]] span { - private: - T* _data = nullptr; - size_t _size = 0; - - public: - __device__ T* data() const { return _data; } - - __device__ size_t size() const { return _size; } - - __device__ bool empty() const { return _size == 0; } - - __device__ T& operator[](size_t pos) const { return _data[pos]; } - - __device__ T* begin() const { return _data; } - - __device__ T* end() const { return _data + _size; } - - __device__ span as_const() const { return span{_data, _size}; } - - __device__ T& element(size_t idx) const { return _data[idx]; } - - __device__ void assign(size_t idx, T value) const { _data[idx] = value; } -}; - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/string_view.cuh b/cpp/include/cudf/jit/lto/string_view.cuh deleted file mode 100644 index abaca66d741d..000000000000 --- a/cpp/include/cudf/jit/lto/string_view.cuh +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include - -namespace CUDF_LTO_EXPORT cudf { -namespace lto { - -struct CUDF_LTO_ALIAS string_view { - private: - char const* _data = nullptr; - size_type _bytes = 0; - mutable size_type _length = 0; - - public: - [[nodiscard]] __device__ size_type size_bytes() const; - - [[nodiscard]] __device__ size_type length() const; - - [[nodiscard]] __device__ char const* data() const; - - [[nodiscard]] __device__ bool empty() const; - - [[nodiscard]] __device__ char_utf8 operator[](size_type pos) const; - - [[nodiscard]] __device__ size_type byte_offset(size_type pos) const; - - [[nodiscard]] __device__ int compare(string_view const& str) const; - - [[nodiscard]] __device__ int compare(char const* str, size_type bytes) const; - - [[nodiscard]] __device__ bool operator==(string_view const& rhs) const; - - [[nodiscard]] __device__ bool operator!=(string_view const& rhs) const; - - [[nodiscard]] __device__ bool operator<(string_view const& rhs) const; - - [[nodiscard]] __device__ bool operator>(string_view const& rhs) const; - - [[nodiscard]] __device__ bool operator<=(string_view const& rhs) const; - - [[nodiscard]] __device__ bool operator>=(string_view const& rhs) const; - - [[nodiscard]] __device__ size_type find(string_view const& str, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ size_type find(char const* str, - size_type bytes, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ size_type find(char_utf8 character, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ size_type rfind(string_view const& str, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ size_type rfind(char const* str, - size_type bytes, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ size_type rfind(char_utf8 character, - size_type pos = 0, - size_type count = -1) const; - - [[nodiscard]] __device__ string_view substr(size_type start, size_type length) const; - - static inline size_type const npos{-1}; -}; - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/thunk.cuh b/cpp/include/cudf/jit/lto/thunk.cuh deleted file mode 100644 index ec844455db7b..000000000000 --- a/cpp/include/cudf/jit/lto/thunk.cuh +++ /dev/null @@ -1,133 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace CUDF_LTO_EXPORT cudf { - -namespace lto { - -template -struct lowered_type_of_t; - -template -using lowered_type_of = typename lowered_type_of_t::type; - -template -struct lifted_type_of_t; - -template -using lifted_type_of = typename lifted_type_of_t::type; - -#define CUDF_LTO_MAP(lowered_type, lifted_type) \ - static_assert(sizeof(lowered_type) == sizeof(lifted_type), \ - "(1: size). Lowered and Lifted types must be bitwise-equivalent"); \ - static_assert(alignof(lowered_type) == alignof(lifted_type), \ - "(2: alignment). Lowered and Lifted types must be bitwise-equivalent"); \ - static_assert( \ - sizeof(optional) == sizeof(cuda::std::optional), \ - "(1: size). Lowered and Lifted types must have bitwise-equivalent optional types"); \ - static_assert( \ - alignof(optional) == alignof(cuda::std::optional), \ - "(2: alignment). Lowered and Lifted types must have bitwise-equivalent optional types"); \ - \ - template <> \ - struct lifted_type_of_t { \ - using type = lifted_type; \ - }; \ - \ - template <> \ - struct lowered_type_of_t { \ - using type = lowered_type; \ - }; \ - \ - __device__ __forceinline__ lowered_type* lower(lifted_type* p) \ - { \ - return reinterpret_cast(p); \ - } \ - \ - __device__ __forceinline__ lowered_type const* lower(lifted_type const* p) \ - { \ - return reinterpret_cast(p); \ - } \ - \ - __device__ __forceinline__ optional* lower(cuda::std::optional* p) \ - { \ - return reinterpret_cast*>(p); \ - } \ - \ - __device__ __forceinline__ optional const* lower( \ - cuda::std::optional const* p) \ - { \ - return reinterpret_cast const*>(p); \ - } \ - \ - __device__ __forceinline__ lifted_type* lift(lowered_type* p) \ - { \ - return reinterpret_cast(p); \ - } \ - \ - __device__ __forceinline__ lifted_type const* lift(lowered_type const* p) \ - { \ - return reinterpret_cast(p); \ - } \ - \ - __device__ __forceinline__ cuda::std::optional* lift(optional* p) \ - { \ - return reinterpret_cast*>(p); \ - } \ - \ - __device__ __forceinline__ cuda::std::optional const* lift( \ - optional const* p) \ - { \ - return reinterpret_cast const*>(p); \ - } - -CUDF_LTO_MAP(bool, bool); -CUDF_LTO_MAP(data_type, cudf::data_type); -CUDF_LTO_MAP(int8_t, std::int8_t); -CUDF_LTO_MAP(int16_t, std::int16_t); -CUDF_LTO_MAP(int32_t, std::int32_t); -CUDF_LTO_MAP(int64_t, std::int64_t); -CUDF_LTO_MAP(uint8_t, std::uint8_t); -CUDF_LTO_MAP(uint16_t, std::uint16_t); -CUDF_LTO_MAP(uint32_t, std::uint32_t); -CUDF_LTO_MAP(uint64_t, std::uint64_t); -CUDF_LTO_MAP(float32_t, float); -CUDF_LTO_MAP(float64_t, double); -CUDF_LTO_MAP(decimal32, numeric::decimal32); -CUDF_LTO_MAP(decimal64, numeric::decimal64); -CUDF_LTO_MAP(decimal128, numeric::decimal128); -CUDF_LTO_MAP(string_view, cudf::string_view); -CUDF_LTO_MAP(timestamp_D, cudf::timestamp_D); -CUDF_LTO_MAP(timestamp_h, cudf::timestamp_h); -CUDF_LTO_MAP(timestamp_m, cudf::timestamp_m); -CUDF_LTO_MAP(timestamp_s, cudf::timestamp_s); -CUDF_LTO_MAP(timestamp_ms, cudf::timestamp_ms); -CUDF_LTO_MAP(timestamp_us, cudf::timestamp_us); -CUDF_LTO_MAP(timestamp_ns, cudf::timestamp_ns); -CUDF_LTO_MAP(duration_D, cudf::duration_D); -CUDF_LTO_MAP(duration_h, cudf::duration_h); -CUDF_LTO_MAP(duration_m, cudf::duration_m); -CUDF_LTO_MAP(duration_s, cudf::duration_s); -CUDF_LTO_MAP(duration_ms, cudf::duration_ms); -CUDF_LTO_MAP(duration_us, cudf::duration_us); -CUDF_LTO_MAP(duration_ns, cudf::duration_ns); -CUDF_LTO_MAP(column_device_view, cudf::column_device_view_core); -CUDF_LTO_MAP(mutable_column_device_view, cudf::mutable_column_device_view_core); - -#undef CUDF_LTO_MAP - -} // namespace lto - -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/transform_params.cuh b/cpp/include/cudf/jit/lto/transform_params.cuh deleted file mode 100644 index 81651c5edcf6..000000000000 --- a/cpp/include/cudf/jit/lto/transform_params.cuh +++ /dev/null @@ -1,26 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include - -namespace CUDF_LTO_EXPORT cudf { -namespace lto { - -/// @brief Type-erased parameters for LTO-JIT-compiled transform operations. -struct [[nodiscard]] transform_params { - /// @brief Pointer to scope data (e.g. column views, scalars, etc.). - void* const* scope = nullptr; - - /// @brief Total number of rows to process. - size_type num_rows = 0; - - /// @brief Current row index. - size_type row_index = 0; -}; - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh index 7babbf9ecd8d..671f869f85ad 100644 --- a/cpp/include/cudf/jit/lto/types.cuh +++ b/cpp/include/cudf/jit/lto/types.cuh @@ -4,373 +4,726 @@ */ #pragma once -#include +#if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) + +#define CUDF_LTO_EXPORT __attribute__((visibility("default"))) + +#else + +#define CUDF_LTO_EXPORT + +#endif namespace CUDF_LTO_EXPORT cudf { namespace lto { -// TODO: update doc, and specify this is our ABI for JIT code and should be kept different from -// normal code -/** - * @brief LTO-JIT functions and thunk types - * - * These are declarations for functions that will be used in LTO-JIT compiled code. - * They are pre-compiled into a device library that is linked at JIT compile time. - * This header should be minimal and only contain necessary types and function declarations as it - * will be included and compiled at JIT compile time. Including other headers will lead to longer - * JIT compile times which can be unbounded and cause slowdowns. - * - * This essentially serves as the ABI for LTO-JIT compiled code to interact with the rest of cuDF. - * Any changes to this header should be made with ABI stability in mind as it can break existing - * LTO-JIT compiled code and lead to undefined behavior. For example, adding new member variables to - * these structs will change their size and layout which can break existing code. Adding new - * functions is generally safe as long as they don't change the existing function signatures, but it - * can still lead to issues if the new functions are called from existing code that wasn't compiled - * with them. Removing or changing existing functions is not safe and will break existing code. - * Changing the types of existing member variables can also break existing code if it changes the - * size or layout of the structs. In general, any change to this header should be made with caution - * and thorough testing to ensure ABI compatibility. - * - */ +using i8 = signed char; +using i16 = signed short; +using i32 = signed int; +using i64 = signed long; +using i128 = __int128_t; +using u8 = unsigned char; +using u16 = unsigned short; +using u32 = unsigned int; +using u64 = unsigned long; + +using size_t = unsigned long; +using iptr = i64; +using uptr = u64; + +using intmax_t = i64; +using uintmax_t = u64; + +using f32 = float; +using f64 = double; + +using size_type = i32; + +using bitmask_type = u32; + +__device__ constexpr bool bit_is_set(bitmask_type const* bitmask, size_t bit_index) +{ + constexpr auto bits_per_word = sizeof(bitmask_type) * 8; + return bitmask[bit_index / bits_per_word] & (bitmask_type{1} << (bit_index % bits_per_word)); +} + +using char_utf8 = u32; + +enum class type_id : i32 { + EMPTY = 0, + INT8 = 1, + INT16 = 2, + INT32 = 3, + INT64 = 4, + UINT8 = 5, + UINT16 = 6, + UINT32 = 7, + UINT64 = 8, + FLOAT32 = 9, + FLOAT64 = 10, + BOOL8 = 11, + TIMESTAMP_DAYS = 12, + TIMESTAMP_SECONDS = 13, + TIMESTAMP_MILLISECONDS = 14, + TIMESTAMP_MICROSECONDS = 15, + TIMESTAMP_NANOSECONDS = 16, + DURATION_DAYS = 17, + DURATION_SECONDS = 18, + DURATION_MILLISECONDS = 19, + DURATION_MICROSECONDS = 20, + DURATION_NANOSECONDS = 21, + DICTIONARY32 = 22, + STRING = 23, + LIST = 24, + DECIMAL32 = 25, + DECIMAL64 = 26, + DECIMAL128 = 27, + STRUCT = 28, + NUM_TYPE_IDS = 29 +}; -// TODO: rename directory to abi +struct data_type { + type_id _id = {}; -using int8_t = signed char; -using int16_t = signed short; -using int32_t = signed int; -using int64_t = signed long long; -using uint8_t = unsigned char; -using uint16_t = unsigned short; -using uint32_t = unsigned int; -using uint64_t = unsigned long long; + i32 _scale = 0; -using size_t = unsigned long; -using intptr_t = int64_t; -using uintptr_t = uint64_t; + __device__ constexpr type_id id() const { return _id; } -using intmax_t = int64_t; -using uintmax_t = uint64_t; + __device__ constexpr i32 scale() const { return _scale; } +}; -using float32_t = float; -using float64_t = double; +struct scaled_t {}; -using size_type = int32_t; +inline constexpr scaled_t scaled{}; -using bitmask_type = uint32_t; +struct decimal32 { + i32 _value = 0; -using char_utf8 = uint32_t; + i32 _scale = 0; -enum class type_id : int32_t {}; + __device__ constexpr decimal32(scaled_t, i32 value, i32 scale) : _value{value}, _scale{scale} {} -enum scale_type : int32_t {}; + __device__ constexpr i32 value() const { return _value; } -struct CUDF_LTO_ALIAS data_type { - private: - type_id _id = {}; - int32_t _fixed_point_scale = 0; + __device__ constexpr i32 scale() const { return _scale; } }; -struct CUDF_LTO_ALIAS decimal32 { - private: - int32_t _value = 0; - scale_type _scale = scale_type{}; - - public: - constexpr decimal32 operator+(decimal32 const&) const; - constexpr decimal32 operator-(decimal32 const&) const; - constexpr decimal32 operator*(decimal32 const&) const; - constexpr decimal32 operator/(decimal32 const&) const; - constexpr bool operator==(decimal32 const&) const; - constexpr bool operator!=(decimal32 const&) const; - constexpr bool operator<(decimal32 const&) const; - constexpr bool operator>(decimal32 const&) const; - constexpr bool operator<=(decimal32 const&) const; - constexpr bool operator>=(decimal32 const&) const; +struct decimal64 { + i64 _value = 0; + + i32 _scale = 0; + + __device__ constexpr decimal64(scaled_t, i64 value, i32 scale) : _value{value}, _scale{scale} {} + + __device__ constexpr i64 value() const { return _value; } + + __device__ constexpr i32 scale() const { return _scale; } }; -struct CUDF_LTO_ALIAS decimal64 { - private: - int64_t _value = 0; - scale_type _scale = scale_type{}; - - public: - constexpr decimal64 operator+(decimal64 const&) const; - constexpr decimal64 operator-(decimal64 const&) const; - constexpr decimal64 operator*(decimal64 const&) const; - constexpr decimal64 operator/(decimal64 const&) const; - constexpr bool operator==(decimal64 const&) const; - constexpr bool operator!=(decimal64 const&) const; - constexpr bool operator<(decimal64 const&) const; - constexpr bool operator>(decimal64 const&) const; - constexpr bool operator<=(decimal64 const&) const; - constexpr bool operator>=(decimal64 const&) const; +struct decimal128 { + i128 _value = 0; + + i32 _scale = 0; + + __device__ constexpr decimal128(scaled_t, i128 value, i32 scale) : _value{value}, _scale{scale} {} + + __device__ constexpr i128 value() const { return _value; } + + __device__ constexpr i32 scale() const { return _scale; } }; -struct CUDF_LTO_ALIAS decimal128 { - private: - __int128_t _value = 0; - scale_type _scale = scale_type{}; - - public: - constexpr decimal128 operator+(decimal128 const&) const; - constexpr decimal128 operator-(decimal128 const&) const; - constexpr decimal128 operator*(decimal128 const&) const; - constexpr decimal128 operator/(decimal128 const&) const; - constexpr bool operator==(decimal128 const&) const; - constexpr bool operator!=(decimal128 const&) const; - constexpr bool operator<(decimal128 const&) const; - constexpr bool operator>(decimal128 const&) const; - constexpr bool operator<=(decimal128 const&) const; - constexpr bool operator>=(decimal128 const&) const; +// TODO: implement +#define DECIMAL_OPS(T) \ + __device__ extern T operator+(T const& lhs, T const& rhs); \ + __device__ extern T operator-(T const& lhs, T const& rhs); \ + __device__ extern T operator*(T const& lhs, T const& rhs); \ + __device__ extern T operator/(T const& lhs, T const& rhs); \ + __device__ extern T operator==(T const& lhs, T const& rhs); \ + __device__ extern T operator!=(T const& lhs, T const& rhs); \ + __device__ extern T operator>(T const& lhs, T const& rhs); \ + __device__ extern T operator<(T const& lhs, T const& rhs); \ + __device__ extern T operator>=(T const& lhs, T const& rhs); \ + __device__ extern T operator<=(T const& lhs, T const& rhs); + +DECIMAL_OPS(decimal32) +DECIMAL_OPS(decimal64) +DECIMAL_OPS(decimal128) + +struct timestamp_D { + i32 _rep = 0; }; -struct CUDF_LTO_ALIAS timestamp_D { - private: - int32_t _rep = 0; - - public: - constexpr timestamp_D operator+(timestamp_D const&) const; - constexpr timestamp_D operator-(timestamp_D const&) const; - constexpr timestamp_D operator*(timestamp_D const&) const; - constexpr timestamp_D operator/(timestamp_D const&) const; - constexpr bool operator==(timestamp_D const&) const; - constexpr bool operator!=(timestamp_D const&) const; - constexpr bool operator<(timestamp_D const&) const; - constexpr bool operator>(timestamp_D const&) const; - constexpr bool operator<=(timestamp_D const&) const; - constexpr bool operator>=(timestamp_D const&) const; +struct timestamp_h { + i32 _rep = 0; }; -struct CUDF_LTO_ALIAS timestamp_h { - private: - int32_t _rep = 0; - - public: - constexpr timestamp_h operator+(timestamp_h const&) const; - constexpr timestamp_h operator-(timestamp_h const&) const; - constexpr timestamp_h operator*(timestamp_h const&) const; - constexpr timestamp_h operator/(timestamp_h const&) const; - constexpr bool operator==(timestamp_h const&) const; - constexpr bool operator!=(timestamp_h const&) const; - constexpr bool operator<(timestamp_h const&) const; - constexpr bool operator>(timestamp_h const&) const; - constexpr bool operator<=(timestamp_h const&) const; - constexpr bool operator>=(timestamp_h const&) const; +struct timestamp_m { + i32 _rep = 0; }; -struct CUDF_LTO_ALIAS timestamp_m { - private: - int32_t _rep = 0; - - public: - constexpr timestamp_m operator+(timestamp_m const&) const; - constexpr timestamp_m operator-(timestamp_m const&) const; - constexpr timestamp_m operator*(timestamp_m const&) const; - constexpr timestamp_m operator/(timestamp_m const&) const; - constexpr bool operator==(timestamp_m const&) const; - constexpr bool operator!=(timestamp_m const&) const; - constexpr bool operator<(timestamp_m const&) const; - constexpr bool operator>(timestamp_m const&) const; - constexpr bool operator<=(timestamp_m const&) const; - constexpr bool operator>=(timestamp_m const&) const; +struct timestamp_s { + i64 _rep = 0; }; -struct CUDF_LTO_ALIAS timestamp_s { - private: - int64_t _rep = 0; - - public: - constexpr timestamp_s operator+(timestamp_s const&) const; - constexpr timestamp_s operator-(timestamp_s const&) const; - constexpr timestamp_s operator*(timestamp_s const&) const; - constexpr timestamp_s operator/(timestamp_s const&) const; - constexpr bool operator==(timestamp_s const&) const; - constexpr bool operator!=(timestamp_s const&) const; - constexpr bool operator<(timestamp_s const&) const; - constexpr bool operator>(timestamp_s const&) const; - constexpr bool operator<=(timestamp_s const&) const; - constexpr bool operator>=(timestamp_s const&) const; +struct timestamp_ms { + i64 _rep = 0; }; -struct CUDF_LTO_ALIAS timestamp_ms { - private: - int64_t _rep = 0; - - public: - constexpr timestamp_ms operator+(timestamp_ms const&) const; - constexpr timestamp_ms operator-(timestamp_ms const&) const; - constexpr timestamp_ms operator*(timestamp_ms const&) const; - constexpr timestamp_ms operator/(timestamp_ms const&) const; - constexpr bool operator==(timestamp_ms const&) const; - constexpr bool operator!=(timestamp_ms const&) const; - constexpr bool operator<(timestamp_ms const&) const; - constexpr bool operator>(timestamp_ms const&) const; - constexpr bool operator<=(timestamp_ms const&) const; - constexpr bool operator>=(timestamp_ms const&) const; +struct timestamp_us { + i64 _rep = 0; }; -struct CUDF_LTO_ALIAS timestamp_us { - private: - int64_t _rep = 0; - - public: - constexpr timestamp_us operator+(timestamp_us const&) const; - constexpr timestamp_us operator-(timestamp_us const&) const; - constexpr timestamp_us operator*(timestamp_us const&) const; - constexpr timestamp_us operator/(timestamp_us const&) const; - constexpr bool operator==(timestamp_us const&) const; - constexpr bool operator!=(timestamp_us const&) const; - constexpr bool operator<(timestamp_us const&) const; - constexpr bool operator>(timestamp_us const&) const; - constexpr bool operator<=(timestamp_us const&) const; - constexpr bool operator>=(timestamp_us const&) const; +struct timestamp_ns { + i64 _rep = 0; }; -struct CUDF_LTO_ALIAS timestamp_ns { - private: - int64_t _rep = 0; - - public: - constexpr timestamp_ns operator+(timestamp_ns const&) const; - constexpr timestamp_ns operator-(timestamp_ns const&) const; - constexpr timestamp_ns operator*(timestamp_ns const&) const; - constexpr timestamp_ns operator/(timestamp_ns const&) const; - constexpr bool operator==(timestamp_ns const&) const; - constexpr bool operator!=(timestamp_ns const&) const; - constexpr bool operator<(timestamp_ns const&) const; - constexpr bool operator>(timestamp_ns const&) const; - constexpr bool operator<=(timestamp_ns const&) const; - constexpr bool operator>=(timestamp_ns const&) const; +#define TIMESTAMP_OPS(T) \ + __device__ constexpr T operator==(T const& lhs, T const& rhs) { return lhs._rep == rhs._rep; } \ + __device__ constexpr T operator!=(T const& lhs, T const& rhs) { return lhs._rep != rhs._rep; } \ + __device__ constexpr T operator>(T const& lhs, T const& rhs) { return lhs._rep > rhs._rep; } \ + __device__ constexpr T operator<(T const& lhs, T const& rhs) { return lhs._rep < rhs._rep; } \ + __device__ constexpr T operator>=(T const& lhs, T const& rhs) { return lhs._rep >= rhs._rep; } \ + __device__ constexpr T operator<=(T const& lhs, T const& rhs) { return lhs._rep <= rhs._rep; } + +TIMESTAMP_OPS(timestamp_D) +TIMESTAMP_OPS(timestamp_h) +TIMESTAMP_OPS(timestamp_m) +TIMESTAMP_OPS(timestamp_s) +TIMESTAMP_OPS(timestamp_ms) +TIMESTAMP_OPS(timestamp_us) +TIMESTAMP_OPS(timestamp_ns) + +struct duration_D { + i32 _rep = 0; }; -struct CUDF_LTO_ALIAS duration_D { - private: - int32_t _rep = 0; - - public: - constexpr duration_D operator+(duration_D const&) const; - constexpr duration_D operator-(duration_D const&) const; - constexpr duration_D operator*(duration_D const&) const; - constexpr duration_D operator/(duration_D const&) const; - constexpr bool operator==(duration_D const&) const; - constexpr bool operator!=(duration_D const&) const; - constexpr bool operator<(duration_D const&) const; - constexpr bool operator>(duration_D const&) const; - constexpr bool operator<=(duration_D const&) const; - constexpr bool operator>=(duration_D const&) const; +struct duration_h { + i32 _rep = 0; }; -struct CUDF_LTO_ALIAS duration_h { - private: - int32_t _rep = 0; - - public: - constexpr duration_h operator+(duration_h const&) const; - constexpr duration_h operator-(duration_h const&) const; - constexpr duration_h operator*(duration_h const&) const; - constexpr duration_h operator/(duration_h const&) const; - constexpr bool operator==(duration_h const&) const; - constexpr bool operator!=(duration_h const&) const; - constexpr bool operator<(duration_h const&) const; - constexpr bool operator>(duration_h const&) const; - constexpr bool operator<=(duration_h const&) const; - constexpr bool operator>=(duration_h const&) const; +struct duration_m { + i32 _rep = 0; }; -struct CUDF_LTO_ALIAS duration_m { - private: - int32_t _rep = 0; - - public: - constexpr duration_m operator+(duration_m const&) const; - constexpr duration_m operator-(duration_m const&) const; - constexpr duration_m operator*(duration_m const&) const; - constexpr duration_m operator/(duration_m const&) const; - constexpr bool operator==(duration_m const&) const; - constexpr bool operator!=(duration_m const&) const; - constexpr bool operator<(duration_m const&) const; - constexpr bool operator>(duration_m const&) const; - constexpr bool operator<=(duration_m const&) const; - constexpr bool operator>=(duration_m const&) const; +struct duration_s { + i64 _rep = 0; }; -struct CUDF_LTO_ALIAS duration_s { - private: - int64_t _rep = 0; - - public: - constexpr duration_s operator+(duration_s const&) const; - constexpr duration_s operator-(duration_s const&) const; - constexpr duration_s operator*(duration_s const&) const; - constexpr duration_s operator/(duration_s const&) const; - constexpr bool operator==(duration_s const&) const; - constexpr bool operator!=(duration_s const&) const; - constexpr bool operator<(duration_s const&) const; - constexpr bool operator>(duration_s const&) const; - constexpr bool operator<=(duration_s const&) const; - constexpr bool operator>=(duration_s const&) const; +struct duration_ms { + i64 _rep = 0; }; -struct CUDF_LTO_ALIAS duration_ms { - private: - int64_t _rep = 0; - - public: - constexpr duration_ms operator+(duration_ms const&) const; - constexpr duration_ms operator-(duration_ms const&) const; - constexpr duration_ms operator*(duration_ms const&) const; - constexpr duration_ms operator/(duration_ms const&) const; - constexpr bool operator==(duration_ms const&) const; - constexpr bool operator!=(duration_ms const&) const; - constexpr bool operator<(duration_ms const&) const; - constexpr bool operator>(duration_ms const&) const; - constexpr bool operator<=(duration_ms const&) const; - constexpr bool operator>=(duration_ms const&) const; +struct duration_us { + i64 _rep = 0; }; -struct CUDF_LTO_ALIAS duration_us { - private: - int64_t _rep = 0; - - public: - constexpr duration_us operator+(duration_us const&) const; - constexpr duration_us operator-(duration_us const&) const; - constexpr duration_us operator*(duration_us const&) const; - constexpr duration_us operator/(duration_us const&) const; - constexpr bool operator==(duration_us const&) const; - constexpr bool operator!=(duration_us const&) const; - constexpr bool operator<(duration_us const&) const; - constexpr bool operator>(duration_us const&) const; - constexpr bool operator<=(duration_us const&) const; - constexpr bool operator>=(duration_us const&) const; +struct duration_ns { + i64 _rep = 0; }; -struct CUDF_LTO_ALIAS duration_ns { - private: - int64_t _rep = 0; - - public: - constexpr duration_ns operator+(duration_ns const&) const; - constexpr duration_ns operator-(duration_ns const&) const; - constexpr duration_ns operator*(duration_ns const&) const; - constexpr duration_ns operator/(duration_ns const&) const; - constexpr bool operator==(duration_ns const&) const; - constexpr bool operator!=(duration_ns const&) const; - constexpr bool operator<(duration_ns const&) const; - constexpr bool operator>(duration_ns const&) const; - constexpr bool operator<=(duration_ns const&) const; - constexpr bool operator>=(duration_ns const&) const; +#define DURATION_OPS(T) \ + __device__ constexpr T operator+(T const& lhs, T const& rhs) { return T{lhs._rep + rhs._rep}; } \ + __device__ constexpr T operator-(T const& lhs, T const& rhs) { return T{lhs._rep - rhs._rep}; } \ + __device__ constexpr T operator==(T const& lhs, T const& rhs) { return lhs._rep == rhs._rep; } \ + __device__ constexpr T operator!=(T const& lhs, T const& rhs) { return lhs._rep != rhs._rep; } \ + __device__ constexpr T operator>(T const& lhs, T const& rhs) { return lhs._rep > rhs._rep; } \ + __device__ constexpr T operator<(T const& lhs, T const& rhs) { return lhs._rep < rhs._rep; } \ + __device__ constexpr T operator>=(T const& lhs, T const& rhs) { return lhs._rep >= rhs._rep; } \ + __device__ constexpr T operator<=(T const& lhs, T const& rhs) { return lhs._rep <= rhs._rep; } + +DURATION_OPS(duration_D) +DURATION_OPS(duration_h) +DURATION_OPS(duration_m) +DURATION_OPS(duration_s) +DURATION_OPS(duration_ms) +DURATION_OPS(duration_us) +DURATION_OPS(duration_ns) + +struct string_view { + static constexpr size_type const UNKNOWN_STRING_LENGTH{-1}; + static constexpr size_type const npos{-1}; + + char const* _data = nullptr; + size_type _bytes = 0; + mutable size_type _length = UNKNOWN_STRING_LENGTH; + + __device__ size_type size_bytes() const { return _bytes; } + + __device__ char const* data() const { return _data; } + + __device__ bool empty() const { return _bytes == 0; } }; +struct inplace_t {}; + +inline constexpr inplace_t inplace{}; + +struct nullopt_t {}; + +inline constexpr nullopt_t nullopt{}; + +// TODO: assumes T is trivially copyable template -struct CUDF_LTO_ALIAS optional; +struct optional { + T _val; + bool _engaged; + + __device__ constexpr optional() : _val{}, _engaged{false} {} + + __device__ constexpr optional(nullopt_t) : _val{}, _engaged{false} {} -struct CUDF_LTO_ALIAS string_view; + template + __device__ constexpr optional(inplace_t, Args&&... args) + : _val{static_cast(args)...}, _engaged{true} + { + } + + __device__ constexpr optional(T val) : _val{val}, _engaged{true} {} + + constexpr optional(optional const&) = default; + + constexpr optional(optional&&) = default; + + constexpr optional& operator=(optional const&) = default; + + constexpr optional& operator=(optional&&) = default; + + constexpr ~optional() = default; + + __device__ constexpr bool has_value() const { return _engaged; } + + __device__ constexpr void reset() { _engaged = false; } + + __device__ constexpr T const& get() const { return _val; } + + __device__ constexpr T& get() { return _val; } + + __device__ constexpr T const* operator->() const { return &_val; } + + __device__ constexpr T* operator->() { return &_val; } + + __device__ constexpr T const& operator*() const { return _val; } + + __device__ constexpr T& operator*() { return _val; } + + __device__ constexpr T const& value() const { return _val; } + + __device__ constexpr T& value() { return _val; } + + __device__ constexpr explicit operator bool() const { return _engaged; } + + __device__ constexpr T value_or(T __v) const { return _engaged ? _val : __v; } +}; + +template +optional(T) -> optional; + +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; +template struct optional; -struct CUDF_LTO_ALIAS column_device_view; +template +struct span { + T* _data = nullptr; + + size_t _size = 0; + + __device__ constexpr T* data() const { return _data; } + + __device__ constexpr size_t size() const { return _size; } + + __device__ constexpr bool empty() const { return _size == 0; } + + __device__ constexpr T& operator[](size_t pos) const { return _data[pos]; } + + __device__ constexpr T* begin() const { return _data; } + + __device__ constexpr T* end() const { return _data + _size; } + + __device__ constexpr span as_const() const { return span{_data, _size}; } + + __device__ constexpr T& element(size_t idx) const { return _data[idx]; } + + __device__ constexpr void assign(size_t idx, T value) const { _data[idx] = value; } +}; + +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; +template struct span; + +template +struct optional_span { + T* _data = nullptr; + + size_t _size = 0; + + bitmask_type const* _null_mask = nullptr; + + __device__ constexpr T* data() const { return _data; } + + __device__ constexpr size_t size() const { return _size; } + + __device__ constexpr bool empty() const { return _size == 0; } + + __device__ constexpr T& operator[](size_t pos) const { return _data[pos]; } + + __device__ constexpr T* begin() const { return _data; } + + __device__ constexpr T* end() const { return _data + _size; } + + __device__ constexpr optional_span as_const() const + { + return optional_span{_data, _size, _null_mask}; + } + + __device__ constexpr bool nullable() const { return _null_mask != nullptr; } + + __device__ constexpr bool is_valid_nocheck(size_t element_index) const + { + return bit_is_set(_null_mask, element_index); + } + + __device__ constexpr bool is_valid(size_t element_index) const + { + return not nullable() or is_valid_nocheck(element_index); + } + + __device__ constexpr bool is_null(size_t element_index) const { return !is_valid(element_index); } + + __device__ constexpr T& element(size_t idx) const { return _data[idx]; } + + __device__ constexpr optional nullable_element(size_t idx) const; + + __device__ constexpr void assign(size_t idx, T value) const { _data[idx] = value; } +}; + +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; +template struct optional_span; + +struct alignas(16) column_device_view { + data_type _type = {}; + + size_type _size = 0; + + void const* _data = nullptr; + + bitmask_type const* _null_mask = nullptr; + + size_type _offset = 0; + + column_device_view* _d_children = nullptr; + + size_type _num_children = 0; + + __device__ constexpr size_type size() const { return _size; } + + __device__ constexpr bool nullable() const { return _null_mask != nullptr; } + + __device__ constexpr bitmask_type const* null_mask() const { return _null_mask; } + + __device__ constexpr size_type offset() const { return _offset; } + + __device__ constexpr bool is_valid(size_type idx) const + { + return !nullable() || is_valid_nocheck(idx); + } + + __device__ constexpr bool is_valid_nocheck(size_type idx) const + { + return bit_is_set(_null_mask, _offset + idx); + } + + __device__ constexpr bool is_null(size_type idx) const { return !is_valid(idx); } + + __device__ constexpr bool is_null_nocheck(size_type idx) const { return !is_valid_nocheck(idx); } + + __device__ constexpr bitmask_type get_mask_word(size_type word_index) const + { + return _null_mask[word_index]; + } + + __device__ constexpr size_type num_child_columns() const { return _num_children; } + + template + __device__ constexpr T element(size_type idx) const; + + template + __device__ constexpr optional nullable_element(size_type idx) const + { + if (!is_valid(idx)) return nullopt; + return element(idx); + } +}; + +#define CUDF_SPEC(T) \ + template <> \ + __device__ constexpr T column_device_view::element(size_type idx) const \ + { \ + return static_cast(_data)[_offset + idx]; \ + } + +CUDF_SPEC(bool) +CUDF_SPEC(i8) +CUDF_SPEC(i16) +CUDF_SPEC(i32) +CUDF_SPEC(i64) +CUDF_SPEC(u8) +CUDF_SPEC(u16) +CUDF_SPEC(u32) +CUDF_SPEC(u64) +CUDF_SPEC(f32) +CUDF_SPEC(f64) +CUDF_SPEC(timestamp_D) +CUDF_SPEC(timestamp_h) +CUDF_SPEC(timestamp_m) +CUDF_SPEC(timestamp_s) +CUDF_SPEC(timestamp_ms) +CUDF_SPEC(timestamp_us) +CUDF_SPEC(timestamp_ns) +CUDF_SPEC(duration_D) +CUDF_SPEC(duration_h) +CUDF_SPEC(duration_m) +CUDF_SPEC(duration_s) +CUDF_SPEC(duration_ms) +CUDF_SPEC(duration_us) +CUDF_SPEC(duration_ns) + +#undef CUDF_SPEC + +#define CUDF_SPEC(T, Repr) \ + template <> \ + __device__ constexpr T column_device_view::element(size_type idx) const \ + { \ + return T{static_cast(_data)[_offset + idx], _type.scale()}; \ + } + +CUDF_SPEC(decimal32, i32) +CUDF_SPEC(decimal64, i64) +CUDF_SPEC(decimal128, i128) + +#undef CUDF_SPEC + +template <> +__device__ constexpr string_view column_device_view::element(size_type idx) const +{ + inline constexpr i32 OFFSETS_CHILD = 0; + auto i = _offset + idx; + auto* str_data = static_cast(_data); + auto& offsets = _d_children[OFFSETS_CHILD]; + auto* i32_offsets = static_cast(offsets._data); + auto* i64_offsets = static_cast(offsets._data); + + i64 run_begin = 0; + i64 run_end = 0; + + if (offsets.type().id() == type_id::INT32) { + run_begin = i32_offsets[i]; + run_end = i32_offsets[i + 1]; + } else { + run_begin = i64_offsets[i]; + run_end = i64_offsets[i + 1]; + } + + return string_view{str_data + run_begin, static_cast(run_end - run_begin)}; +} + +#define CUDF_INST(T) +template __device__ constexpr optional column_device_view::nullable_element( + size_type idx) const; + +CUDF_INST(bool) +CUDF_INST(i8) +CUDF_INST(i16) +CUDF_INST(i32) +CUDF_INST(i64) +CUDF_INST(u8) +CUDF_INST(u16) +CUDF_INST(u32) +CUDF_INST(u64) +CUDF_INST(f32) +CUDF_INST(f64) +CUDF_INST(timestamp_D) +CUDF_INST(timestamp_h) +CUDF_INST(timestamp_m) +CUDF_INST(timestamp_s) +CUDF_INST(timestamp_ms) +CUDF_INST(timestamp_us) +CUDF_INST(timestamp_ns) +CUDF_INST(duration_D) +CUDF_INST(duration_h) +CUDF_INST(duration_m) +CUDF_INST(duration_s) +CUDF_INST(duration_ms) +CUDF_INST(duration_us) +CUDF_INST(duration_ns) +CUDF_INST(decimal32) +CUDF_INST(decimal64) +CUDF_INST(decimal128) +CUDF_INST(string_view) + +#undef CUDF_INST + +/// @brief Type-erased parameters for LTO-JIT-compiled transform operations. +struct transform_operator_params { + /// @brief Pointer to scope data (e.g. column views, scalars, etc.). + void* const* scope = nullptr; + + /// @brief Total number of rows to process. + size_type num_rows = 0; + + /// @brief Current row index. + size_type row_index = 0; +}; + +// TODO: scope variables should be aligned to avoid uncoalesced reads/writes +namespace scope { + +using args = void* const*; + +template +struct column { + static constexpr bool IS_SCALAR = IsScalar; + static constexpr bool IS_NULLABLE = IsNullable; + + using Type = T; + using Arg = ColumnType const*; + + static __device__ decltype(auto) element(args scope, size_type i) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + if constexpr (!IsNullable) { + return p->template element(index); + } else { + return p->template nullable_element(index); + } + } + + static __device__ void assign(args scope, size_type i, T value) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + p->template assign(index, value); + } + + static __device__ auto* null_mask(args scope) + { + auto p = static_cast(scope[ScopeIndex]); + return p->null_mask(); + } + + static __device__ bool is_null(args scope, size_type i) + { + if constexpr (!IsNullable) { return false; } + + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + return p->is_null(index); + } + + static __device__ bool is_valid(args scope, size_type i) { return !is_null(scope, i); } +}; + +template +struct user_data { + using Arg = void*; + + static __device__ decltype(auto) element(args scope, [[maybe_unused]] size_type i) + { + return static_cast(scope[ScopeIndex]); + } +}; -struct CUDF_LTO_ALIAS mutable_column_device_view; +} // namespace scope } // namespace lto } // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/include/cudf/utilities/export.hpp b/cpp/include/cudf/utilities/export.hpp index 815d3110f2cb..608a556d6c16 100644 --- a/cpp/include/cudf/utilities/export.hpp +++ b/cpp/include/cudf/utilities/export.hpp @@ -7,7 +7,12 @@ // Macros used for defining symbol visibility, only GLIBC is supported #if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) +#ifndef CUDF_DISABLE_DEFAULT_EXPORT #define CUDF_EXPORT __attribute__((visibility("default"))) +#else +#define CUDF_EXPORT __attribute__((visibility("hidden"))) +#endif + #define CUDF_HIDDEN __attribute__((visibility("hidden"))) #else #define CUDF_EXPORT diff --git a/cpp/src/jit/lto/column_view.def.cuh b/cpp/src/jit/lto/column_view.def.cuh deleted file mode 100644 index 2c6bd57c5705..000000000000 --- a/cpp/src/jit/lto/column_view.def.cuh +++ /dev/null @@ -1,185 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include -#include - -namespace CUDF_LTO_EXPORT cudf { -namespace lto { - -template -__device__ T const* column_device_view::head() const -{ - return lower(lift(this)->head>()); -} - -__device__ size_type column_device_view::size() const { return lift(this)->size(); } - -__device__ bool column_device_view::nullable() const { return lift(this)->nullable(); } - -__device__ bitmask_type const* column_device_view::null_mask() const -{ - return lift(this)->null_mask(); -} - -__device__ size_type column_device_view::offset() const { return lift(this)->offset(); } - -__device__ bool column_device_view::is_valid(size_type index) const -{ - return lift(this)->is_valid(index); -} - -__device__ bool column_device_view::is_valid_nocheck(size_type index) const -{ - return lift(this)->is_valid_nocheck(index); -} - -__device__ bool column_device_view::is_null(size_type index) const -{ - return lift(this)->is_null(index); -} - -__device__ bool column_device_view::is_null_nocheck(size_type index) const -{ - return lift(this)->is_null_nocheck(index); -} - -__device__ bitmask_type column_device_view::get_mask_word(size_type index) const -{ - return lift(this)->get_mask_word(index); -} - -template -__device__ T column_device_view::element(size_type index) const -{ - auto ret = lift(this)->element>(index); - return *lower(&ret); -} - -template -__device__ optional column_device_view::nullable_element(size_type index) const -{ - auto ret = lift(this)->nullable_element>(index); - return *lower(&ret); -} - -__device__ size_type column_device_view::num_child_columns() const -{ - return lift(this)->num_child_columns(); -} - -template -__device__ T* mutable_column_device_view::head() const -{ - return lower(lift(this)->head>()); -} - -__device__ size_type mutable_column_device_view::size() const { return lift(this)->size(); } - -__device__ bool mutable_column_device_view::nullable() const { return lift(this)->nullable(); } - -__device__ bitmask_type* mutable_column_device_view::null_mask() const -{ - return lift(this)->null_mask(); -} - -__device__ size_type mutable_column_device_view::offset() const { return lift(this)->offset(); } - -__device__ bool mutable_column_device_view::is_valid(size_type index) const -{ - return lift(this)->is_valid(index); -} - -__device__ bool mutable_column_device_view::is_valid_nocheck(size_type index) const -{ - return lift(this)->is_valid_nocheck(index); -} - -__device__ bool mutable_column_device_view::is_null(size_type index) const -{ - return lift(this)->is_null(index); -} - -__device__ bool mutable_column_device_view::is_null_nocheck(size_type index) const -{ - return lift(this)->is_null_nocheck(index); -} - -__device__ bitmask_type mutable_column_device_view::get_mask_word(size_type index) const -{ - return lift(this)->get_mask_word(index); -} - -template -__device__ T mutable_column_device_view::element(size_type index) const -{ - auto ret = lift(this)->element>(index); - return *lower(&ret); -} - -template -__device__ optional mutable_column_device_view::nullable_element(size_type index) const -{ - auto ret = lift(this)->nullable_element>(index); - return *lower(&ret); -} - -template -__device__ void mutable_column_device_view::assign(size_type index, T value) const -{ - lift(this)->assign>(index, *lift(&value)); -} - -/// Explicit template instantiations - -#define DO_IT(T) template __device__ T const* column_device_view::head() const; - -FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE - -#undef DO_IT - -#define DO_IT(T) template __device__ T column_device_view::element(size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - template __device__ optional column_device_view::nullable_element(size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) template __device__ T* mutable_column_device_view::head() const; - -FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE - -#undef DO_IT - -#define DO_IT(T) template __device__ T mutable_column_device_view::element(size_type idx) const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - template __device__ optional mutable_column_device_view::nullable_element(size_type idx) \ - const; - -FOREACH_CUDF_LTO_COLUMN_ELEMENT_TYPE - -#undef DO_IT - -#define DO_IT(T) \ - template __device__ void mutable_column_device_view::assign(size_type idx, T value) const; - -FOREACH_CUDF_LTO_COLUMN_HEAD_TYPE - -#undef DO_IT - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/jit/lto/operators.def.cuh b/cpp/src/jit/lto/operators.def.cuh index dbaf95ec5141..d847420bb598 100644 --- a/cpp/src/jit/lto/operators.def.cuh +++ b/cpp/src/jit/lto/operators.def.cuh @@ -4,567 +4,79 @@ */ #pragma once -#include #include -#include #include +#include + namespace CUDF_LTO_EXPORT cudf { namespace lto { -template -using func = ast::detail::operator_functor; - -template -using null_func = ast::detail::operator_functor; - -using opcode = ast::ast_operator; - -#define CUDF_LTO_DEF(op, OP, type) \ - __device__ void operators::op(type* out, type const* a, type const* b) \ - { \ - auto ret = func{}(*lto::lift(a), *lto::lift(b)); \ - *out = *lto::lower(&ret); \ - } \ - \ - __device__ void operators::op( \ - lto::optional* out, lto::optional const* a, lto::optional const* b) \ - { \ - auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ - *out = *lto::lower(&ret); \ - } - -CUDF_LTO_DEF(add, ADD, int32_t); -CUDF_LTO_DEF(add, ADD, int64_t); -CUDF_LTO_DEF(add, ADD, uint32_t); -CUDF_LTO_DEF(add, ADD, uint64_t); -CUDF_LTO_DEF(add, ADD, float32_t); -CUDF_LTO_DEF(add, ADD, float64_t); -CUDF_LTO_DEF(add, ADD, decimal32); -CUDF_LTO_DEF(add, ADD, decimal64); -CUDF_LTO_DEF(add, ADD, decimal128); -CUDF_LTO_DEF(add, ADD, duration_D); -CUDF_LTO_DEF(add, ADD, duration_s); -CUDF_LTO_DEF(add, ADD, duration_ms); -CUDF_LTO_DEF(add, ADD, duration_ns); - -CUDF_LTO_DEF(sub, SUB, int32_t); -CUDF_LTO_DEF(sub, SUB, int64_t); -CUDF_LTO_DEF(sub, SUB, uint32_t); -CUDF_LTO_DEF(sub, SUB, uint64_t); -CUDF_LTO_DEF(sub, SUB, float32_t); -CUDF_LTO_DEF(sub, SUB, float64_t); -CUDF_LTO_DEF(sub, SUB, decimal32); -CUDF_LTO_DEF(sub, SUB, decimal64); -CUDF_LTO_DEF(sub, SUB, decimal128); -CUDF_LTO_DEF(sub, SUB, duration_D); -CUDF_LTO_DEF(sub, SUB, duration_s); -CUDF_LTO_DEF(sub, SUB, duration_ms); -CUDF_LTO_DEF(sub, SUB, duration_ns); - -CUDF_LTO_DEF(mul, MUL, int32_t); -CUDF_LTO_DEF(mul, MUL, int64_t); -CUDF_LTO_DEF(mul, MUL, uint32_t); -CUDF_LTO_DEF(mul, MUL, uint64_t); -CUDF_LTO_DEF(mul, MUL, float32_t); -CUDF_LTO_DEF(mul, MUL, float64_t); -CUDF_LTO_DEF(mul, MUL, decimal32); -CUDF_LTO_DEF(mul, MUL, decimal64); -CUDF_LTO_DEF(mul, MUL, decimal128); - -CUDF_LTO_DEF(div, DIV, int32_t); -CUDF_LTO_DEF(div, DIV, int64_t); -CUDF_LTO_DEF(div, DIV, uint32_t); -CUDF_LTO_DEF(div, DIV, uint64_t); -CUDF_LTO_DEF(div, DIV, float32_t); -CUDF_LTO_DEF(div, DIV, float64_t); -CUDF_LTO_DEF(div, DIV, decimal32); -CUDF_LTO_DEF(div, DIV, decimal64); -CUDF_LTO_DEF(div, DIV, decimal128); - -CUDF_LTO_DEF(mod, MOD, float32_t); -CUDF_LTO_DEF(mod, MOD, float64_t); - -CUDF_LTO_DEF(pymod, PYMOD, float32_t); -CUDF_LTO_DEF(pymod, PYMOD, float64_t); - -CUDF_LTO_DEF(pow, POW, float32_t); -CUDF_LTO_DEF(pow, POW, float64_t); - -#undef CUDF_LTO_DEF - -#define CUDF_LTO_DEF(op, OP, type) \ - __device__ void operators::op(bool* out, type const* a, type const* b) \ - { \ - auto ret = func{}(*lto::lift(a), *lto::lift(b)); \ - *out = *lto::lower(&ret); \ - } \ - \ - __device__ void operators::op(bool* out, optional const* a, optional const* b) \ - { \ - auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ - *out = **lto::lower(&ret); \ - } \ - \ - __device__ void operators::op( \ - optional* out, optional const* a, optional const* b) \ - { \ - auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ - *out = *lto::lower(&ret); \ - } - -CUDF_LTO_DEF(equal, EQUAL, bool); -CUDF_LTO_DEF(equal, EQUAL, int8_t); -CUDF_LTO_DEF(equal, EQUAL, int16_t); -CUDF_LTO_DEF(equal, EQUAL, int32_t); -CUDF_LTO_DEF(equal, EQUAL, int64_t); -CUDF_LTO_DEF(equal, EQUAL, uint8_t); -CUDF_LTO_DEF(equal, EQUAL, uint16_t); -CUDF_LTO_DEF(equal, EQUAL, uint32_t); -CUDF_LTO_DEF(equal, EQUAL, uint64_t); -CUDF_LTO_DEF(equal, EQUAL, float32_t); -CUDF_LTO_DEF(equal, EQUAL, float64_t); -CUDF_LTO_DEF(equal, EQUAL, decimal32); -CUDF_LTO_DEF(equal, EQUAL, decimal64); -CUDF_LTO_DEF(equal, EQUAL, decimal128); -CUDF_LTO_DEF(equal, EQUAL, timestamp_D); -CUDF_LTO_DEF(equal, EQUAL, timestamp_s); -CUDF_LTO_DEF(equal, EQUAL, timestamp_ms); -CUDF_LTO_DEF(equal, EQUAL, timestamp_us); -CUDF_LTO_DEF(equal, EQUAL, timestamp_ns); -CUDF_LTO_DEF(equal, EQUAL, duration_D); -CUDF_LTO_DEF(equal, EQUAL, duration_s); -CUDF_LTO_DEF(equal, EQUAL, duration_ms); -CUDF_LTO_DEF(equal, EQUAL, duration_ns); -CUDF_LTO_DEF(equal, EQUAL, string_view); - -CUDF_LTO_DEF(null_equal, NULL_EQUAL, bool); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, int8_t); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, int16_t); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, int32_t); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, int64_t); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, uint8_t); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, uint16_t); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, uint32_t); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, uint64_t); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, float32_t); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, float64_t); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, decimal32); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, decimal64); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, decimal128); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, timestamp_D); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, timestamp_s); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, timestamp_ms); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, timestamp_us); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, timestamp_ns); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, duration_D); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, duration_s); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, duration_ms); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, duration_ns); -CUDF_LTO_DEF(null_equal, NULL_EQUAL, string_view); - -#undef CUDF_LTO_DEF - -#define CUDF_LTO_DEF(op, OP, type) \ - __device__ void operators::op(bool* out, type const* a, type const* b) \ - { \ - auto ret = func{}(*lto::lift(a), *lto::lift(b)); \ - *out = *lto::lower(&ret); \ - } \ - \ - __device__ void operators::op( \ - optional* out, optional const* a, optional const* b) \ - { \ - auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ - *out = *lto::lower(&ret); \ - } - -CUDF_LTO_DEF(less, LESS, bool); -CUDF_LTO_DEF(less, LESS, int8_t); -CUDF_LTO_DEF(less, LESS, int16_t); -CUDF_LTO_DEF(less, LESS, int32_t); -CUDF_LTO_DEF(less, LESS, int64_t); -CUDF_LTO_DEF(less, LESS, uint8_t); -CUDF_LTO_DEF(less, LESS, uint16_t); -CUDF_LTO_DEF(less, LESS, uint32_t); -CUDF_LTO_DEF(less, LESS, uint64_t); -CUDF_LTO_DEF(less, LESS, float32_t); -CUDF_LTO_DEF(less, LESS, float64_t); -CUDF_LTO_DEF(less, LESS, decimal32); -CUDF_LTO_DEF(less, LESS, decimal64); -CUDF_LTO_DEF(less, LESS, decimal128); -CUDF_LTO_DEF(less, LESS, timestamp_D); -CUDF_LTO_DEF(less, LESS, timestamp_s); -CUDF_LTO_DEF(less, LESS, timestamp_ms); -CUDF_LTO_DEF(less, LESS, timestamp_us); -CUDF_LTO_DEF(less, LESS, timestamp_ns); -CUDF_LTO_DEF(less, LESS, duration_D); -CUDF_LTO_DEF(less, LESS, duration_s); -CUDF_LTO_DEF(less, LESS, duration_ms); -CUDF_LTO_DEF(less, LESS, duration_ns); -CUDF_LTO_DEF(less, LESS, string_view); - -CUDF_LTO_DEF(greater, GREATER, bool); -CUDF_LTO_DEF(greater, GREATER, int8_t); -CUDF_LTO_DEF(greater, GREATER, int16_t); -CUDF_LTO_DEF(greater, GREATER, int32_t); -CUDF_LTO_DEF(greater, GREATER, int64_t); -CUDF_LTO_DEF(greater, GREATER, uint8_t); -CUDF_LTO_DEF(greater, GREATER, uint16_t); -CUDF_LTO_DEF(greater, GREATER, uint32_t); -CUDF_LTO_DEF(greater, GREATER, uint64_t); -CUDF_LTO_DEF(greater, GREATER, float32_t); -CUDF_LTO_DEF(greater, GREATER, float64_t); -CUDF_LTO_DEF(greater, GREATER, decimal32); -CUDF_LTO_DEF(greater, GREATER, decimal64); -CUDF_LTO_DEF(greater, GREATER, decimal128); -CUDF_LTO_DEF(greater, GREATER, timestamp_D); -CUDF_LTO_DEF(greater, GREATER, timestamp_s); -CUDF_LTO_DEF(greater, GREATER, timestamp_ms); -CUDF_LTO_DEF(greater, GREATER, timestamp_us); -CUDF_LTO_DEF(greater, GREATER, timestamp_ns); -CUDF_LTO_DEF(greater, GREATER, duration_D); -CUDF_LTO_DEF(greater, GREATER, duration_s); -CUDF_LTO_DEF(greater, GREATER, duration_ms); -CUDF_LTO_DEF(greater, GREATER, duration_ns); -CUDF_LTO_DEF(greater, GREATER, string_view); - -CUDF_LTO_DEF(less_equal, LESS_EQUAL, bool); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, int8_t); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, int16_t); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, int32_t); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, int64_t); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, uint8_t); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, uint16_t); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, uint32_t); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, uint64_t); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, float32_t); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, float64_t); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, decimal32); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, decimal64); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, decimal128); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, timestamp_D); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, timestamp_s); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, timestamp_ms); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, timestamp_us); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, timestamp_ns); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, duration_D); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, duration_s); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, duration_ms); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, duration_ns); -CUDF_LTO_DEF(less_equal, LESS_EQUAL, string_view); - -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, bool); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, int8_t); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, int16_t); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, int32_t); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, int64_t); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, uint8_t); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, uint16_t); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, uint32_t); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, uint64_t); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, float32_t); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, float64_t); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, decimal32); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, decimal64); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, decimal128); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, timestamp_D); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, timestamp_s); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, timestamp_ms); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, timestamp_us); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, timestamp_ns); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, duration_D); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, duration_s); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, duration_ms); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, duration_ns); -CUDF_LTO_DEF(greater_equal, GREATER_EQUAL, string_view); - -#undef CUDF_LTO_DEF - -#define CUDF_LTO_DEF(op, OP, type) \ - __device__ void operators::op(type* out, type const* a, type const* b) \ - { \ - auto ret = func{}(*lto::lift(a), *lto::lift(b)); \ - *out = *lto::lower(&ret); \ - } \ - \ - __device__ void operators::op( \ - optional* out, optional const* a, optional const* b) \ - { \ - auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ - *out = *lto::lower(&ret); \ - } - -CUDF_LTO_DEF(bitwise_and, BITWISE_AND, int32_t); -CUDF_LTO_DEF(bitwise_and, BITWISE_AND, int64_t); -CUDF_LTO_DEF(bitwise_and, BITWISE_AND, uint32_t); -CUDF_LTO_DEF(bitwise_and, BITWISE_AND, uint64_t); - -CUDF_LTO_DEF(bitwise_or, BITWISE_OR, int32_t); -CUDF_LTO_DEF(bitwise_or, BITWISE_OR, int64_t); -CUDF_LTO_DEF(bitwise_or, BITWISE_OR, uint32_t); -CUDF_LTO_DEF(bitwise_or, BITWISE_OR, uint64_t); - -CUDF_LTO_DEF(bitwise_xor, BITWISE_XOR, int32_t); -CUDF_LTO_DEF(bitwise_xor, BITWISE_XOR, int64_t); -CUDF_LTO_DEF(bitwise_xor, BITWISE_XOR, uint32_t); -CUDF_LTO_DEF(bitwise_xor, BITWISE_XOR, uint64_t); - -#undef CUDF_LTO_DEF - -#define CUDF_LTO_DEF(op, OP, type) \ - __device__ void operators::op(type* out, type const* a, type const* b) \ - { \ - auto ret = func{}(*lto::lift(a), *lto::lift(b)); \ - *out = *lto::lower(&ret); \ - } \ - \ - __device__ void operators::op( \ - optional* out, optional const* a, optional const* b) \ - { \ - auto ret = null_func{}(*lto::lift(a), *lto::lift(b)); \ - *out = *lto::lower(&ret); \ - } - -CUDF_LTO_DEF(logical_and, LOGICAL_AND, bool); - -CUDF_LTO_DEF(null_logical_and, NULL_LOGICAL_AND, bool); - -CUDF_LTO_DEF(logical_or, LOGICAL_OR, bool); - -CUDF_LTO_DEF(null_logical_or, NULL_LOGICAL_OR, bool); - -#undef CUDF_LTO_DEF - -#define CUDF_LTO_DEF(op, OP, type) \ - __device__ void operators::op(type* out, type const* a) \ - { \ - auto ret = func{}(*lto::lift(a)); \ - *out = *lto::lower(&ret); \ - } \ - \ - __device__ void operators::op(optional* out, optional const* a) \ - { \ - auto ret = null_func{}(*lto::lift(a)); \ - *out = *lto::lower(&ret); \ - } - -CUDF_LTO_DEF(identity, IDENTITY, bool); -CUDF_LTO_DEF(identity, IDENTITY, int8_t); -CUDF_LTO_DEF(identity, IDENTITY, int16_t); -CUDF_LTO_DEF(identity, IDENTITY, int32_t); -CUDF_LTO_DEF(identity, IDENTITY, int64_t); -CUDF_LTO_DEF(identity, IDENTITY, uint8_t); -CUDF_LTO_DEF(identity, IDENTITY, uint16_t); -CUDF_LTO_DEF(identity, IDENTITY, uint32_t); -CUDF_LTO_DEF(identity, IDENTITY, uint64_t); -CUDF_LTO_DEF(identity, IDENTITY, float32_t); -CUDF_LTO_DEF(identity, IDENTITY, float64_t); -CUDF_LTO_DEF(identity, IDENTITY, decimal32); -CUDF_LTO_DEF(identity, IDENTITY, decimal64); -CUDF_LTO_DEF(identity, IDENTITY, decimal128); -CUDF_LTO_DEF(identity, IDENTITY, timestamp_D); -CUDF_LTO_DEF(identity, IDENTITY, timestamp_s); -CUDF_LTO_DEF(identity, IDENTITY, timestamp_ms); -CUDF_LTO_DEF(identity, IDENTITY, timestamp_us); -CUDF_LTO_DEF(identity, IDENTITY, timestamp_ns); -CUDF_LTO_DEF(identity, IDENTITY, duration_D); -CUDF_LTO_DEF(identity, IDENTITY, duration_s); -CUDF_LTO_DEF(identity, IDENTITY, duration_ms); -CUDF_LTO_DEF(identity, IDENTITY, duration_ns); -CUDF_LTO_DEF(identity, IDENTITY, string_view); - -CUDF_LTO_DEF(sin, SIN, float32_t); -CUDF_LTO_DEF(sin, SIN, float64_t); - -CUDF_LTO_DEF(cos, COS, float32_t); -CUDF_LTO_DEF(cos, COS, float64_t); - -CUDF_LTO_DEF(tan, TAN, float32_t); -CUDF_LTO_DEF(tan, TAN, float64_t); - -CUDF_LTO_DEF(arcsin, ARCSIN, float32_t); -CUDF_LTO_DEF(arcsin, ARCSIN, float64_t); - -CUDF_LTO_DEF(arccos, ARCCOS, float32_t); -CUDF_LTO_DEF(arccos, ARCCOS, float64_t); - -CUDF_LTO_DEF(arctan, ARCTAN, float32_t); -CUDF_LTO_DEF(arctan, ARCTAN, float64_t); - -CUDF_LTO_DEF(sinh, SINH, float32_t); -CUDF_LTO_DEF(sinh, SINH, float64_t); - -CUDF_LTO_DEF(cosh, COSH, float32_t); -CUDF_LTO_DEF(cosh, COSH, float64_t); - -CUDF_LTO_DEF(tanh, TANH, float32_t); -CUDF_LTO_DEF(tanh, TANH, float64_t); - -CUDF_LTO_DEF(arcsinh, ARCSINH, float32_t); -CUDF_LTO_DEF(arcsinh, ARCSINH, float64_t); - -CUDF_LTO_DEF(arccosh, ARCCOSH, float32_t); -CUDF_LTO_DEF(arccosh, ARCCOSH, float64_t); - -CUDF_LTO_DEF(arctanh, ARCTANH, float32_t); -CUDF_LTO_DEF(arctanh, ARCTANH, float64_t); +#define DEF_UNOP(name, type, expr) \ + __device__ void name(type* out, type const& a) { *out = expr; } -CUDF_LTO_DEF(exp, EXP, float32_t); -CUDF_LTO_DEF(exp, EXP, float64_t); +#define DEF_BINOP(name, type, expr) \ + __device__ void name(type* out, type const& a, type const& b) { *out = expr; } -CUDF_LTO_DEF(log, LOG, float32_t); -CUDF_LTO_DEF(log, LOG, float64_t); +DEF_UNOP(sin, f32, (cuda::std::sin(a))) +DEF_UNOP(sin, f64, (cuda::std::sin(a))) -CUDF_LTO_DEF(cbrt, CBRT, float32_t); -CUDF_LTO_DEF(cbrt, CBRT, float64_t); +DEF_UNOP(cos, f32, (cuda::std::cos(a))) +DEF_UNOP(cos, f64, (cuda::std::cos(a))) -CUDF_LTO_DEF(ceil, CEIL, float32_t); -CUDF_LTO_DEF(ceil, CEIL, float64_t); +DEF_UNOP(tan, f32, (cuda::std::tan(a))) +DEF_UNOP(tan, f64, (cuda::std::tan(a))) -CUDF_LTO_DEF(floor, FLOOR, float32_t); -CUDF_LTO_DEF(floor, FLOOR, float64_t); +DEF_UNOP(arcsin, f32, (cuda::std::asin(a))) +DEF_UNOP(arcsin, f64, (cuda::std::asin(a))) -CUDF_LTO_DEF(abs, ABS, int32_t); -CUDF_LTO_DEF(abs, ABS, int64_t); -CUDF_LTO_DEF(abs, ABS, float32_t); -CUDF_LTO_DEF(abs, ABS, float64_t); +DEF_UNOP(arccos, f32, (cuda::std::acos(a))) +DEF_UNOP(arccos, f64, (cuda::std::acos(a))) -CUDF_LTO_DEF(rint, RINT, float32_t); -CUDF_LTO_DEF(rint, RINT, float64_t); +DEF_UNOP(arctan, f32, (cuda::std::atan(a))) +DEF_UNOP(arctan, f64, (cuda::std::atan(a))) -CUDF_LTO_DEF(bit_invert, BIT_INVERT, uint32_t); -CUDF_LTO_DEF(bit_invert, BIT_INVERT, uint64_t); -CUDF_LTO_DEF(bit_invert, BIT_INVERT, int32_t); -CUDF_LTO_DEF(bit_invert, BIT_INVERT, int64_t); +DEF_UNOP(sinh, f32, (cuda::std::sinh(a))) +DEF_UNOP(sinh, f64, (cuda::std::sinh(a))) -#undef CUDF_LTO_DEF +DEF_UNOP(cosh, f32, (cuda::std::cosh(a))) +DEF_UNOP(cosh, f64, (cuda::std::cosh(a))) -#define CUDF_LTO_DEF(op, OP, ret_type, type) \ - extern __device__ void operators::op(ret_type* out, type const* a) \ - { \ - auto ret = func{}(*lto::lift(a)); \ - *out = *lto::lower(&ret); \ - } \ - \ - extern __device__ void operators::op(optional* out, optional const* a) \ - { \ - auto ret = null_func{}(*lto::lift(a)); \ - *out = *lto::lower(&ret); \ - } +DEF_UNOP(tanh, f32, (cuda::std::tanh(a))) +DEF_UNOP(tanh, f64, (cuda::std::tanh(a))) -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, bool); -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, int8_t); -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, int16_t); -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, int32_t); -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, int64_t); -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, uint8_t); -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, uint16_t); -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, uint32_t); -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, uint64_t); -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, float32_t); -CUDF_LTO_DEF(cast_to_int64, CAST_TO_INT64, int64_t, float64_t); +DEF_UNOP(arcsinh, f32, (cuda::std::asinh(a))) +DEF_UNOP(arcsinh, f64, (cuda::std::asinh(a))) -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, bool); -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, int8_t); -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, int16_t); -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, int32_t); -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, int64_t); -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, uint8_t); -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, uint16_t); -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, uint32_t); -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, uint64_t); -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, float32_t); -CUDF_LTO_DEF(cast_to_uint64, CAST_TO_UINT64, uint64_t, float64_t); +DEF_UNOP(arccosh, f32, (cuda::std::acosh(a))) +DEF_UNOP(arccosh, f64, (cuda::std::acosh(a))) -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, bool); -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, int8_t); -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, int16_t); -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, int32_t); -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, int64_t); -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, uint8_t); -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, uint16_t); -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, uint32_t); -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, uint64_t); -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, float32_t); -CUDF_LTO_DEF(cast_to_float64, CAST_TO_FLOAT64, float64_t, float64_t); +DEF_UNOP(arctanh, f32, (cuda::std::atanh(a))) +DEF_UNOP(arctanh, f64, (cuda::std::atanh(a))) -#undef CUDF_LTO_DEF +DEF_UNOP(exp, f32, (cuda::std::exp(a))) +DEF_UNOP(exp, f64, (cuda::std::exp(a))) -#define CUDF_LTO_DEF(op, OP, type) \ - __device__ void operators::op(bool* out, type const* a) \ - { \ - auto ret = func{}(*lto::lift(a)); \ - *out = *lto::lower(&ret); \ - } \ - \ - __device__ void operators::op(bool* out, optional const* a) \ - { \ - auto ret = null_func{}(*lto::lift(a)); \ - *out = *lto::lower(&ret); \ - } \ - \ - __device__ void operators::op(optional* out, optional const* a) \ - { \ - auto ret = null_func{}(*lto::lift(a)); \ - *out = *lto::lower(&ret); \ - } +DEF_UNOP(log, f32, (cuda::std::log(a))) +DEF_UNOP(log, f64, (cuda::std::log(a))) -CUDF_LTO_DEF(is_null, IS_NULL, bool); -CUDF_LTO_DEF(is_null, IS_NULL, int8_t); -CUDF_LTO_DEF(is_null, IS_NULL, int16_t); -CUDF_LTO_DEF(is_null, IS_NULL, int32_t); -CUDF_LTO_DEF(is_null, IS_NULL, int64_t); -CUDF_LTO_DEF(is_null, IS_NULL, uint8_t); -CUDF_LTO_DEF(is_null, IS_NULL, uint16_t); -CUDF_LTO_DEF(is_null, IS_NULL, uint32_t); -CUDF_LTO_DEF(is_null, IS_NULL, uint64_t); -CUDF_LTO_DEF(is_null, IS_NULL, float32_t); -CUDF_LTO_DEF(is_null, IS_NULL, float64_t); -CUDF_LTO_DEF(is_null, IS_NULL, decimal32); -CUDF_LTO_DEF(is_null, IS_NULL, decimal64); -CUDF_LTO_DEF(is_null, IS_NULL, decimal128); -CUDF_LTO_DEF(is_null, IS_NULL, timestamp_D); -CUDF_LTO_DEF(is_null, IS_NULL, timestamp_s); -CUDF_LTO_DEF(is_null, IS_NULL, timestamp_ms); -CUDF_LTO_DEF(is_null, IS_NULL, timestamp_us); -CUDF_LTO_DEF(is_null, IS_NULL, timestamp_ns); -CUDF_LTO_DEF(is_null, IS_NULL, duration_D); -CUDF_LTO_DEF(is_null, IS_NULL, duration_s); -CUDF_LTO_DEF(is_null, IS_NULL, duration_ms); -CUDF_LTO_DEF(is_null, IS_NULL, duration_ns); -CUDF_LTO_DEF(is_null, IS_NULL, string_view); +DEF_UNOP(cbrt, f32, (cuda::std::cbrt(a))) +DEF_UNOP(cbrt, f64, (cuda::std::cbrt(a))) -#undef CUDF_LTO_DEF +DEF_UNOP(ceil, f32, (cuda::std::ceil(a))) +DEF_UNOP(ceil, f64, (cuda::std::ceil(a))) -// TODO: lamarrr, remove ABI-lock to cuda::std::optional and other types. -// CUDF should probably have its own set of types used in kernels for columns. -// JIT should have its own type set +DEF_UNOP(floor, f32, (cuda::std::floor(a))) +DEF_UNOP(floor, f64, (cuda::std::floor(a))) -#define CUDF_LTO_DEF(op, OP, type) \ - __device__ void operators::op(bool* out, type const* a) \ - { \ - auto ret = func{}(*lto::lift(a)); \ - *out = *lto::lower(&ret); \ - } \ - \ - __device__ void operators::op(optional* out, optional const* a) \ - { \ - auto ret = null_func{}(*lto::lift(a)); \ - *out = *lto::lower(&ret); \ - } +DEF_UNOP(rint, f32, (cuda::std::rint(a))) +DEF_UNOP(rint, f64, (cuda::std::rint(a))) -CUDF_LTO_DEF(logical_not, NOT, bool); -CUDF_LTO_DEF(logical_not, NOT, int8_t); -CUDF_LTO_DEF(logical_not, NOT, int16_t); -CUDF_LTO_DEF(logical_not, NOT, int32_t); -CUDF_LTO_DEF(logical_not, NOT, int64_t); -CUDF_LTO_DEF(logical_not, NOT, uint8_t); -CUDF_LTO_DEF(logical_not, NOT, uint16_t); -CUDF_LTO_DEF(logical_not, NOT, uint32_t); -CUDF_LTO_DEF(logical_not, NOT, uint64_t); +DEF_BINOP(mod, f32, (cuda::std::fmod(a, b))) +DEF_BINOP(mod, f64, (cuda::std::fmod(a, b))) -#undef CUDF_LTO_DEF +DEF_BINOP(pymod, f32, (cuda::std::fmod(cuda::std::fmod(a, b) + b, b))) +DEF_BINOP(pymod, f64, (cuda::std::fmod(cuda::std::fmod(a, b) + b, b))) } // namespace lto } // namespace CUDF_LTO_EXPORT cudf From 5e0f2e7b4a433a29aecdf1f5b11c36904993faee Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 21 Feb 2026 17:22:37 +0000 Subject: [PATCH 057/254] update --- cpp/CMakeLists.txt | 2 +- .../Modules/JitifyPreprocessKernels.cmake | 2 - cpp/include/cudf/jit/lto/library.cuh | 7 - cpp/include/cudf/jit/lto/types.cuh | 729 -------------- cpp/{src/jit/lto => include/jcudf}/README.md | 0 .../lto/operators.cuh => jcudf/functions.cuh} | 218 ++--- cpp/include/jcudf/functions.def.cuh | 11 + cpp/include/jcudf/library.cu | 6 + cpp/include/jcudf/types.cuh | 924 ++++++++++++++++++ cpp/src/jit/lto/library.cu | 9 - cpp/src/jit/lto/operators.def.cuh | 82 -- cpp/src/jit/lto/string_view.def.cuh | 125 --- cpp/src/transform/jit/lto_kernel.inl.cuh | 27 - cpp/src/transform/transform.cu | 1 - cpp/tests/jit/rtc.cpp | 42 +- 15 files changed, 1066 insertions(+), 1119 deletions(-) delete mode 100644 cpp/include/cudf/jit/lto/library.cuh delete mode 100644 cpp/include/cudf/jit/lto/types.cuh rename cpp/{src/jit/lto => include/jcudf}/README.md (100%) rename cpp/include/{cudf/jit/lto/operators.cuh => jcudf/functions.cuh} (70%) create mode 100644 cpp/include/jcudf/functions.def.cuh create mode 100644 cpp/include/jcudf/library.cu create mode 100644 cpp/include/jcudf/types.cuh delete mode 100644 cpp/src/jit/lto/library.cu delete mode 100644 cpp/src/jit/lto/operators.def.cuh delete mode 100644 cpp/src/jit/lto/string_view.def.cuh delete mode 100644 cpp/src/transform/jit/lto_kernel.inl.cuh diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d7359a2ac5a6..4421a802c271 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -359,7 +359,7 @@ list(APPEND CUDF_JIT_COMPILE_FLAGS -std=c++${CUDF_CUDA_STANDARD} --device-int128 ) set(CUDF_LTO_JIT_ARCHITECTURES "80-real;90-real;100-real;120-real") -add_library(cudf_lto_library OBJECT src/jit/lto/library.cu) +add_library(cudf_lto_library OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/include/jcudf/library.cu) target_compile_options(cudf_lto_library PRIVATE --compress-mode=none --gen-opt-lto) target_compile_definitions(cudf_lto_library PRIVATE CUDF_DISABLE_DEFAULT_EXPORT) set_target_properties( diff --git a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake index 8b09bf71243f..cdf9996c1eef 100644 --- a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake +++ b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake @@ -60,8 +60,6 @@ jit_preprocess_files( stream_compaction/filter/jit/kernel.cu transform/jit/kernel.cu ) -jit_preprocess_files(SOURCE_DIRECTORY ${CUDF_SOURCE_DIR}/include/cudf FILES jit/lto/library.cuh) - add_custom_target( jitify_preprocess_run DEPENDS ${JIT_PREPROCESSED_FILES} diff --git a/cpp/include/cudf/jit/lto/library.cuh b/cpp/include/cudf/jit/lto/library.cuh deleted file mode 100644 index 4a4a3973ddfa..000000000000 --- a/cpp/include/cudf/jit/lto/library.cuh +++ /dev/null @@ -1,7 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include -#include diff --git a/cpp/include/cudf/jit/lto/types.cuh b/cpp/include/cudf/jit/lto/types.cuh deleted file mode 100644 index 671f869f85ad..000000000000 --- a/cpp/include/cudf/jit/lto/types.cuh +++ /dev/null @@ -1,729 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) - -#define CUDF_LTO_EXPORT __attribute__((visibility("default"))) - -#else - -#define CUDF_LTO_EXPORT - -#endif - -namespace CUDF_LTO_EXPORT cudf { - -namespace lto { - -using i8 = signed char; -using i16 = signed short; -using i32 = signed int; -using i64 = signed long; -using i128 = __int128_t; -using u8 = unsigned char; -using u16 = unsigned short; -using u32 = unsigned int; -using u64 = unsigned long; - -using size_t = unsigned long; -using iptr = i64; -using uptr = u64; - -using intmax_t = i64; -using uintmax_t = u64; - -using f32 = float; -using f64 = double; - -using size_type = i32; - -using bitmask_type = u32; - -__device__ constexpr bool bit_is_set(bitmask_type const* bitmask, size_t bit_index) -{ - constexpr auto bits_per_word = sizeof(bitmask_type) * 8; - return bitmask[bit_index / bits_per_word] & (bitmask_type{1} << (bit_index % bits_per_word)); -} - -using char_utf8 = u32; - -enum class type_id : i32 { - EMPTY = 0, - INT8 = 1, - INT16 = 2, - INT32 = 3, - INT64 = 4, - UINT8 = 5, - UINT16 = 6, - UINT32 = 7, - UINT64 = 8, - FLOAT32 = 9, - FLOAT64 = 10, - BOOL8 = 11, - TIMESTAMP_DAYS = 12, - TIMESTAMP_SECONDS = 13, - TIMESTAMP_MILLISECONDS = 14, - TIMESTAMP_MICROSECONDS = 15, - TIMESTAMP_NANOSECONDS = 16, - DURATION_DAYS = 17, - DURATION_SECONDS = 18, - DURATION_MILLISECONDS = 19, - DURATION_MICROSECONDS = 20, - DURATION_NANOSECONDS = 21, - DICTIONARY32 = 22, - STRING = 23, - LIST = 24, - DECIMAL32 = 25, - DECIMAL64 = 26, - DECIMAL128 = 27, - STRUCT = 28, - NUM_TYPE_IDS = 29 -}; - -struct data_type { - type_id _id = {}; - - i32 _scale = 0; - - __device__ constexpr type_id id() const { return _id; } - - __device__ constexpr i32 scale() const { return _scale; } -}; - -struct scaled_t {}; - -inline constexpr scaled_t scaled{}; - -struct decimal32 { - i32 _value = 0; - - i32 _scale = 0; - - __device__ constexpr decimal32(scaled_t, i32 value, i32 scale) : _value{value}, _scale{scale} {} - - __device__ constexpr i32 value() const { return _value; } - - __device__ constexpr i32 scale() const { return _scale; } -}; - -struct decimal64 { - i64 _value = 0; - - i32 _scale = 0; - - __device__ constexpr decimal64(scaled_t, i64 value, i32 scale) : _value{value}, _scale{scale} {} - - __device__ constexpr i64 value() const { return _value; } - - __device__ constexpr i32 scale() const { return _scale; } -}; - -struct decimal128 { - i128 _value = 0; - - i32 _scale = 0; - - __device__ constexpr decimal128(scaled_t, i128 value, i32 scale) : _value{value}, _scale{scale} {} - - __device__ constexpr i128 value() const { return _value; } - - __device__ constexpr i32 scale() const { return _scale; } -}; - -// TODO: implement -#define DECIMAL_OPS(T) \ - __device__ extern T operator+(T const& lhs, T const& rhs); \ - __device__ extern T operator-(T const& lhs, T const& rhs); \ - __device__ extern T operator*(T const& lhs, T const& rhs); \ - __device__ extern T operator/(T const& lhs, T const& rhs); \ - __device__ extern T operator==(T const& lhs, T const& rhs); \ - __device__ extern T operator!=(T const& lhs, T const& rhs); \ - __device__ extern T operator>(T const& lhs, T const& rhs); \ - __device__ extern T operator<(T const& lhs, T const& rhs); \ - __device__ extern T operator>=(T const& lhs, T const& rhs); \ - __device__ extern T operator<=(T const& lhs, T const& rhs); - -DECIMAL_OPS(decimal32) -DECIMAL_OPS(decimal64) -DECIMAL_OPS(decimal128) - -struct timestamp_D { - i32 _rep = 0; -}; - -struct timestamp_h { - i32 _rep = 0; -}; - -struct timestamp_m { - i32 _rep = 0; -}; - -struct timestamp_s { - i64 _rep = 0; -}; - -struct timestamp_ms { - i64 _rep = 0; -}; - -struct timestamp_us { - i64 _rep = 0; -}; - -struct timestamp_ns { - i64 _rep = 0; -}; - -#define TIMESTAMP_OPS(T) \ - __device__ constexpr T operator==(T const& lhs, T const& rhs) { return lhs._rep == rhs._rep; } \ - __device__ constexpr T operator!=(T const& lhs, T const& rhs) { return lhs._rep != rhs._rep; } \ - __device__ constexpr T operator>(T const& lhs, T const& rhs) { return lhs._rep > rhs._rep; } \ - __device__ constexpr T operator<(T const& lhs, T const& rhs) { return lhs._rep < rhs._rep; } \ - __device__ constexpr T operator>=(T const& lhs, T const& rhs) { return lhs._rep >= rhs._rep; } \ - __device__ constexpr T operator<=(T const& lhs, T const& rhs) { return lhs._rep <= rhs._rep; } - -TIMESTAMP_OPS(timestamp_D) -TIMESTAMP_OPS(timestamp_h) -TIMESTAMP_OPS(timestamp_m) -TIMESTAMP_OPS(timestamp_s) -TIMESTAMP_OPS(timestamp_ms) -TIMESTAMP_OPS(timestamp_us) -TIMESTAMP_OPS(timestamp_ns) - -struct duration_D { - i32 _rep = 0; -}; - -struct duration_h { - i32 _rep = 0; -}; - -struct duration_m { - i32 _rep = 0; -}; - -struct duration_s { - i64 _rep = 0; -}; - -struct duration_ms { - i64 _rep = 0; -}; - -struct duration_us { - i64 _rep = 0; -}; - -struct duration_ns { - i64 _rep = 0; -}; - -#define DURATION_OPS(T) \ - __device__ constexpr T operator+(T const& lhs, T const& rhs) { return T{lhs._rep + rhs._rep}; } \ - __device__ constexpr T operator-(T const& lhs, T const& rhs) { return T{lhs._rep - rhs._rep}; } \ - __device__ constexpr T operator==(T const& lhs, T const& rhs) { return lhs._rep == rhs._rep; } \ - __device__ constexpr T operator!=(T const& lhs, T const& rhs) { return lhs._rep != rhs._rep; } \ - __device__ constexpr T operator>(T const& lhs, T const& rhs) { return lhs._rep > rhs._rep; } \ - __device__ constexpr T operator<(T const& lhs, T const& rhs) { return lhs._rep < rhs._rep; } \ - __device__ constexpr T operator>=(T const& lhs, T const& rhs) { return lhs._rep >= rhs._rep; } \ - __device__ constexpr T operator<=(T const& lhs, T const& rhs) { return lhs._rep <= rhs._rep; } - -DURATION_OPS(duration_D) -DURATION_OPS(duration_h) -DURATION_OPS(duration_m) -DURATION_OPS(duration_s) -DURATION_OPS(duration_ms) -DURATION_OPS(duration_us) -DURATION_OPS(duration_ns) - -struct string_view { - static constexpr size_type const UNKNOWN_STRING_LENGTH{-1}; - static constexpr size_type const npos{-1}; - - char const* _data = nullptr; - size_type _bytes = 0; - mutable size_type _length = UNKNOWN_STRING_LENGTH; - - __device__ size_type size_bytes() const { return _bytes; } - - __device__ char const* data() const { return _data; } - - __device__ bool empty() const { return _bytes == 0; } -}; - -struct inplace_t {}; - -inline constexpr inplace_t inplace{}; - -struct nullopt_t {}; - -inline constexpr nullopt_t nullopt{}; - -// TODO: assumes T is trivially copyable -template -struct optional { - T _val; - bool _engaged; - - __device__ constexpr optional() : _val{}, _engaged{false} {} - - __device__ constexpr optional(nullopt_t) : _val{}, _engaged{false} {} - - template - __device__ constexpr optional(inplace_t, Args&&... args) - : _val{static_cast(args)...}, _engaged{true} - { - } - - __device__ constexpr optional(T val) : _val{val}, _engaged{true} {} - - constexpr optional(optional const&) = default; - - constexpr optional(optional&&) = default; - - constexpr optional& operator=(optional const&) = default; - - constexpr optional& operator=(optional&&) = default; - - constexpr ~optional() = default; - - __device__ constexpr bool has_value() const { return _engaged; } - - __device__ constexpr void reset() { _engaged = false; } - - __device__ constexpr T const& get() const { return _val; } - - __device__ constexpr T& get() { return _val; } - - __device__ constexpr T const* operator->() const { return &_val; } - - __device__ constexpr T* operator->() { return &_val; } - - __device__ constexpr T const& operator*() const { return _val; } - - __device__ constexpr T& operator*() { return _val; } - - __device__ constexpr T const& value() const { return _val; } - - __device__ constexpr T& value() { return _val; } - - __device__ constexpr explicit operator bool() const { return _engaged; } - - __device__ constexpr T value_or(T __v) const { return _engaged ? _val : __v; } -}; - -template -optional(T) -> optional; - -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; -template struct optional; - -template -struct span { - T* _data = nullptr; - - size_t _size = 0; - - __device__ constexpr T* data() const { return _data; } - - __device__ constexpr size_t size() const { return _size; } - - __device__ constexpr bool empty() const { return _size == 0; } - - __device__ constexpr T& operator[](size_t pos) const { return _data[pos]; } - - __device__ constexpr T* begin() const { return _data; } - - __device__ constexpr T* end() const { return _data + _size; } - - __device__ constexpr span as_const() const { return span{_data, _size}; } - - __device__ constexpr T& element(size_t idx) const { return _data[idx]; } - - __device__ constexpr void assign(size_t idx, T value) const { _data[idx] = value; } -}; - -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; -template struct span; - -template -struct optional_span { - T* _data = nullptr; - - size_t _size = 0; - - bitmask_type const* _null_mask = nullptr; - - __device__ constexpr T* data() const { return _data; } - - __device__ constexpr size_t size() const { return _size; } - - __device__ constexpr bool empty() const { return _size == 0; } - - __device__ constexpr T& operator[](size_t pos) const { return _data[pos]; } - - __device__ constexpr T* begin() const { return _data; } - - __device__ constexpr T* end() const { return _data + _size; } - - __device__ constexpr optional_span as_const() const - { - return optional_span{_data, _size, _null_mask}; - } - - __device__ constexpr bool nullable() const { return _null_mask != nullptr; } - - __device__ constexpr bool is_valid_nocheck(size_t element_index) const - { - return bit_is_set(_null_mask, element_index); - } - - __device__ constexpr bool is_valid(size_t element_index) const - { - return not nullable() or is_valid_nocheck(element_index); - } - - __device__ constexpr bool is_null(size_t element_index) const { return !is_valid(element_index); } - - __device__ constexpr T& element(size_t idx) const { return _data[idx]; } - - __device__ constexpr optional nullable_element(size_t idx) const; - - __device__ constexpr void assign(size_t idx, T value) const { _data[idx] = value; } -}; - -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; -template struct optional_span; - -struct alignas(16) column_device_view { - data_type _type = {}; - - size_type _size = 0; - - void const* _data = nullptr; - - bitmask_type const* _null_mask = nullptr; - - size_type _offset = 0; - - column_device_view* _d_children = nullptr; - - size_type _num_children = 0; - - __device__ constexpr size_type size() const { return _size; } - - __device__ constexpr bool nullable() const { return _null_mask != nullptr; } - - __device__ constexpr bitmask_type const* null_mask() const { return _null_mask; } - - __device__ constexpr size_type offset() const { return _offset; } - - __device__ constexpr bool is_valid(size_type idx) const - { - return !nullable() || is_valid_nocheck(idx); - } - - __device__ constexpr bool is_valid_nocheck(size_type idx) const - { - return bit_is_set(_null_mask, _offset + idx); - } - - __device__ constexpr bool is_null(size_type idx) const { return !is_valid(idx); } - - __device__ constexpr bool is_null_nocheck(size_type idx) const { return !is_valid_nocheck(idx); } - - __device__ constexpr bitmask_type get_mask_word(size_type word_index) const - { - return _null_mask[word_index]; - } - - __device__ constexpr size_type num_child_columns() const { return _num_children; } - - template - __device__ constexpr T element(size_type idx) const; - - template - __device__ constexpr optional nullable_element(size_type idx) const - { - if (!is_valid(idx)) return nullopt; - return element(idx); - } -}; - -#define CUDF_SPEC(T) \ - template <> \ - __device__ constexpr T column_device_view::element(size_type idx) const \ - { \ - return static_cast(_data)[_offset + idx]; \ - } - -CUDF_SPEC(bool) -CUDF_SPEC(i8) -CUDF_SPEC(i16) -CUDF_SPEC(i32) -CUDF_SPEC(i64) -CUDF_SPEC(u8) -CUDF_SPEC(u16) -CUDF_SPEC(u32) -CUDF_SPEC(u64) -CUDF_SPEC(f32) -CUDF_SPEC(f64) -CUDF_SPEC(timestamp_D) -CUDF_SPEC(timestamp_h) -CUDF_SPEC(timestamp_m) -CUDF_SPEC(timestamp_s) -CUDF_SPEC(timestamp_ms) -CUDF_SPEC(timestamp_us) -CUDF_SPEC(timestamp_ns) -CUDF_SPEC(duration_D) -CUDF_SPEC(duration_h) -CUDF_SPEC(duration_m) -CUDF_SPEC(duration_s) -CUDF_SPEC(duration_ms) -CUDF_SPEC(duration_us) -CUDF_SPEC(duration_ns) - -#undef CUDF_SPEC - -#define CUDF_SPEC(T, Repr) \ - template <> \ - __device__ constexpr T column_device_view::element(size_type idx) const \ - { \ - return T{static_cast(_data)[_offset + idx], _type.scale()}; \ - } - -CUDF_SPEC(decimal32, i32) -CUDF_SPEC(decimal64, i64) -CUDF_SPEC(decimal128, i128) - -#undef CUDF_SPEC - -template <> -__device__ constexpr string_view column_device_view::element(size_type idx) const -{ - inline constexpr i32 OFFSETS_CHILD = 0; - auto i = _offset + idx; - auto* str_data = static_cast(_data); - auto& offsets = _d_children[OFFSETS_CHILD]; - auto* i32_offsets = static_cast(offsets._data); - auto* i64_offsets = static_cast(offsets._data); - - i64 run_begin = 0; - i64 run_end = 0; - - if (offsets.type().id() == type_id::INT32) { - run_begin = i32_offsets[i]; - run_end = i32_offsets[i + 1]; - } else { - run_begin = i64_offsets[i]; - run_end = i64_offsets[i + 1]; - } - - return string_view{str_data + run_begin, static_cast(run_end - run_begin)}; -} - -#define CUDF_INST(T) -template __device__ constexpr optional column_device_view::nullable_element( - size_type idx) const; - -CUDF_INST(bool) -CUDF_INST(i8) -CUDF_INST(i16) -CUDF_INST(i32) -CUDF_INST(i64) -CUDF_INST(u8) -CUDF_INST(u16) -CUDF_INST(u32) -CUDF_INST(u64) -CUDF_INST(f32) -CUDF_INST(f64) -CUDF_INST(timestamp_D) -CUDF_INST(timestamp_h) -CUDF_INST(timestamp_m) -CUDF_INST(timestamp_s) -CUDF_INST(timestamp_ms) -CUDF_INST(timestamp_us) -CUDF_INST(timestamp_ns) -CUDF_INST(duration_D) -CUDF_INST(duration_h) -CUDF_INST(duration_m) -CUDF_INST(duration_s) -CUDF_INST(duration_ms) -CUDF_INST(duration_us) -CUDF_INST(duration_ns) -CUDF_INST(decimal32) -CUDF_INST(decimal64) -CUDF_INST(decimal128) -CUDF_INST(string_view) - -#undef CUDF_INST - -/// @brief Type-erased parameters for LTO-JIT-compiled transform operations. -struct transform_operator_params { - /// @brief Pointer to scope data (e.g. column views, scalars, etc.). - void* const* scope = nullptr; - - /// @brief Total number of rows to process. - size_type num_rows = 0; - - /// @brief Current row index. - size_type row_index = 0; -}; - -// TODO: scope variables should be aligned to avoid uncoalesced reads/writes -namespace scope { - -using args = void* const*; - -template -struct column { - static constexpr bool IS_SCALAR = IsScalar; - static constexpr bool IS_NULLABLE = IsNullable; - - using Type = T; - using Arg = ColumnType const*; - - static __device__ decltype(auto) element(args scope, size_type i) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - if constexpr (!IsNullable) { - return p->template element(index); - } else { - return p->template nullable_element(index); - } - } - - static __device__ void assign(args scope, size_type i, T value) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - p->template assign(index, value); - } - - static __device__ auto* null_mask(args scope) - { - auto p = static_cast(scope[ScopeIndex]); - return p->null_mask(); - } - - static __device__ bool is_null(args scope, size_type i) - { - if constexpr (!IsNullable) { return false; } - - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - return p->is_null(index); - } - - static __device__ bool is_valid(args scope, size_type i) { return !is_null(scope, i); } -}; - -template -struct user_data { - using Arg = void*; - - static __device__ decltype(auto) element(args scope, [[maybe_unused]] size_type i) - { - return static_cast(scope[ScopeIndex]); - } -}; - -} // namespace scope - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/jit/lto/README.md b/cpp/include/jcudf/README.md similarity index 100% rename from cpp/src/jit/lto/README.md rename to cpp/include/jcudf/README.md diff --git a/cpp/include/cudf/jit/lto/operators.cuh b/cpp/include/jcudf/functions.cuh similarity index 70% rename from cpp/include/cudf/jit/lto/operators.cuh rename to cpp/include/jcudf/functions.cuh index c80b4b6fb163..61053d24ed40 100644 --- a/cpp/include/cudf/jit/lto/operators.cuh +++ b/cpp/include/jcudf/functions.cuh @@ -3,15 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#include +#include -namespace CUDF_LTO_EXPORT cudf { +namespace JCUDF_EXPORT jcudf { +namespace functions { -namespace lto { - -namespace operators { - -#define UNOP_T(op, Ret, T, expr) \ +#define UNOP_T(op, R, T, expr) \ __device__ __forceinline__ void op(R* out, T const& a) { *out = expr; } \ \ __device__ __forceinline__ void op(optional* out, optional const& a) \ @@ -39,36 +36,8 @@ namespace operators { } \ } -#define EXTERN_UNOP_T(op, R, T) \ - __device__ extern void op(R* out, T const& a); \ - \ - __device__ __forceinline__ void op(optional* out, optional const& a) \ - { \ - if (a.has_value()) { \ - R r; \ - op(&r, *a); \ - *out = r; \ - } else { \ - *out = nullopt; \ - } \ - } - -#define EXTERN_BINOP_T(op, R, T) \ - __device__ extern void op(R* out, T const& a, T const& b); \ - \ - __device__ __forceinline__ void op(optional* out, optional const& a, optional const& b) \ - { \ - if (a.has_value() && b.has_value()) { \ - R r; \ - op(&r, *a, *b); \ - *out = r; \ - } else { \ - *out = nullopt; \ - } \ - } - -#define EXTERN_UNOP(op, T) EXTERN_UNOP_T(op, T, T) -#define EXTERN_BINOP(op, T) EXTERN_BINOP_T(op, T, T) +#define UNOP(op, T, expr) UNOP_T(op, T, T, expr) +#define BINOP(op, T, expr) BINOP_T(op, T, T, expr) #define ADD_OP(T) BINOP_T(add, T, T, (a + b)) #define SUB_OP(T) BINOP_T(sub, T, T, (a - b)) @@ -87,7 +56,6 @@ namespace operators { #define IDENTITY_OP(T) UNOP_T(identity, T, T, a) #define BIT_INVERT_OP(T) UNOP_T(bit_invert, T, T, ~a) #define CAST_OP(out_type, in_type) UNOP_T(cast_to_##out_type, out_type, in_type, (out_type)(a)) -#define IS_NULL_OP(T) UNOP_T(is_null, bool, T, !a.has_value()) #define ABS_OP(T) UNOP_T(abs, T, T, ((a < 0) ? -a : a)) #define MOD_OP(T) BINOP_T(mod, T, T, (a % b)) #define PYMOD_OP(T) BINOP_T(pymod, T, T, ((a % b + b) % b)) @@ -100,22 +68,29 @@ namespace operators { { \ if (a.has_value() && b.has_value()) { \ *out = (*a == *b); \ - } else if (!a.has_value() && !b.has_value()) { \ + } else if (a.has_null() && b.has_null()) { \ *out = true; \ } else { \ *out = false; \ } \ } +#define IS_NULL_OP(T) \ + __device__ __forceinline__ void is_null(bool* out, T const& a) { *out = false; } \ + __device__ __forceinline__ void is_null(optional* out, optional const& a) \ + { \ + *out = a.has_null(); \ + } + ADD_OP(i32) ADD_OP(i64) ADD_OP(u32) ADD_OP(u64) ADD_OP(f32) ADD_OP(f64) -ADD_OP(decimal32) -ADD_OP(decimal64) -ADD_OP(decimal128) +ADD_OP(dec32) +ADD_OP(dec64) +ADD_OP(dec128) ADD_OP(duration_D) ADD_OP(duration_s) ADD_OP(duration_ms) @@ -127,9 +102,9 @@ SUB_OP(u32) SUB_OP(u64) SUB_OP(f32) SUB_OP(f64) -SUB_OP(decimal32) -SUB_OP(decimal64) -SUB_OP(decimal128) +SUB_OP(dec32) +SUB_OP(dec64) +SUB_OP(dec128) SUB_OP(duration_D) SUB_OP(duration_s) SUB_OP(duration_ms) @@ -141,9 +116,9 @@ MUL_OP(u32) MUL_OP(u64) MUL_OP(f32) MUL_OP(f64) -MUL_OP(decimal32) -MUL_OP(decimal64) -MUL_OP(decimal128) +MUL_OP(dec32) +MUL_OP(dec64) +MUL_OP(dec128) DIV_OP(i32) DIV_OP(i64) @@ -151,9 +126,9 @@ DIV_OP(u32) DIV_OP(u64) DIV_OP(f32) DIV_OP(f64) -DIV_OP(decimal32) -DIV_OP(decimal64) -DIV_OP(decimal128) +DIV_OP(dec32) +DIV_OP(dec64) +DIV_OP(dec128) EQ_OP(bool) EQ_OP(i8) @@ -166,9 +141,9 @@ EQ_OP(u32) EQ_OP(u64) EQ_OP(f32) EQ_OP(f64) -EQ_OP(decimal32) -EQ_OP(decimal64) -EQ_OP(decimal128) +EQ_OP(dec32) +EQ_OP(dec64) +EQ_OP(dec128) EQ_OP(timestamp_D) EQ_OP(timestamp_s) EQ_OP(timestamp_ms) @@ -191,9 +166,9 @@ LT_OP(u32) LT_OP(u64) LT_OP(f32) LT_OP(f64) -LT_OP(decimal32) -LT_OP(decimal64) -LT_OP(decimal128) +LT_OP(dec32) +LT_OP(dec64) +LT_OP(dec128) LT_OP(timestamp_D) LT_OP(timestamp_s) LT_OP(timestamp_ms) @@ -216,9 +191,9 @@ GT_OP(u32) GT_OP(u64) GT_OP(f32) GT_OP(f64) -GT_OP(decimal32) -GT_OP(decimal64) -GT_OP(decimal128) +GT_OP(dec32) +GT_OP(dec64) +GT_OP(dec128) GT_OP(timestamp_D) GT_OP(timestamp_s) GT_OP(timestamp_ms) @@ -241,9 +216,9 @@ LE_OP(u32) LE_OP(u64) LE_OP(f32) LE_OP(f64) -LE_OP(decimal32) -LE_OP(decimal64) -LE_OP(decimal128) +LE_OP(dec32) +LE_OP(dec64) +LE_OP(dec128) LE_OP(timestamp_D) LE_OP(timestamp_s) LE_OP(timestamp_ms) @@ -266,9 +241,9 @@ GE_OP(u32) GE_OP(u64) GE_OP(f32) GE_OP(f64) -GE_OP(decimal32) -GE_OP(decimal64) -GE_OP(decimal128) +GE_OP(dec32) +GE_OP(dec64) +GE_OP(dec128) GE_OP(timestamp_D) GE_OP(timestamp_s) GE_OP(timestamp_ms) @@ -310,9 +285,9 @@ IDENTITY_OP(u32) IDENTITY_OP(u64) IDENTITY_OP(f32) IDENTITY_OP(f64) -IDENTITY_OP(decimal32) -IDENTITY_OP(decimal64) -IDENTITY_OP(decimal128) +IDENTITY_OP(dec32) +IDENTITY_OP(dec64) +IDENTITY_OP(dec128) IDENTITY_OP(timestamp_D) IDENTITY_OP(timestamp_s) IDENTITY_OP(timestamp_ms) @@ -376,9 +351,9 @@ IS_NULL_OP(u32) IS_NULL_OP(u64) IS_NULL_OP(f32) IS_NULL_OP(f64) -IS_NULL_OP(decimal32) -IS_NULL_OP(decimal64) -IS_NULL_OP(decimal128) +IS_NULL_OP(dec32) +IS_NULL_OP(dec64) +IS_NULL_OP(dec128) IS_NULL_OP(timestamp_D) IS_NULL_OP(timestamp_s) IS_NULL_OP(timestamp_ms) @@ -397,76 +372,76 @@ ABS_OP(i64) ABS_OP(f32) ABS_OP(f64) -EXTERN_UNOP(sin, f32); -EXTERN_UNOP(sin, f64); +UNOP(sin, f32, (__builtin_sinf(a))) +UNOP(sin, f64, (__builtin_sin(a))) -EXTERN_UNOP(cos, f32); -EXTERN_UNOP(cos, f64); +UNOP(cos, f32, (__builtin_cosf(a))) +UNOP(cos, f64, (__builtin_cos(a))) -EXTERN_UNOP(tan, f32); -EXTERN_UNOP(tan, f64); +UNOP(tan, f32, (__builtin_tanf(a))) +UNOP(tan, f64, (__builtin_tan(a))) -EXTERN_UNOP(arcsin, f32); -EXTERN_UNOP(arcsin, f64); +UNOP(arcsin, f32, (__builtin_asinf(a))) +UNOP(arcsin, f64, (__builtin_asin(a))) -EXTERN_UNOP(arccos, f32); -EXTERN_UNOP(arccos, f64); +UNOP(arccos, f32, (__builtin_acosf(a))) +UNOP(arccos, f64, (__builtin_acos(a))) -EXTERN_UNOP(arctan, f32); -EXTERN_UNOP(arctan, f64); +UNOP(arctan, f32, (__builtin_atanf(a))) +UNOP(arctan, f64, (__builtin_atan(a))) -EXTERN_UNOP(sinh, f32); -EXTERN_UNOP(sinh, f64); +UNOP(sinh, f32, (__builtin_sinhf(a))) +UNOP(sinh, f64, (__builtin_sinh(a))) -EXTERN_UNOP(cosh, f32); -EXTERN_UNOP(cosh, f64); +UNOP(cosh, f32, (__builtin_coshf(a))) +UNOP(cosh, f64, (__builtin_cosh(a))) -EXTERN_UNOP(tanh, f32); -EXTERN_UNOP(tanh, f64); +UNOP(tanh, f32, (__builtin_tanhf(a))) +UNOP(tanh, f64, (__builtin_tanh(a))) -EXTERN_UNOP(arcsinh, f32); -EXTERN_UNOP(arcsinh, f64); +UNOP(arcsinh, f32, (__builtin_asinhf(a))) +UNOP(arcsinh, f64, (__builtin_asinh(a))) -EXTERN_UNOP(arccosh, f32); -EXTERN_UNOP(arccosh, f64); +UNOP(arccosh, f32, (__builtin_acoshf(a))) +UNOP(arccosh, f64, (__builtin_acosh(a))) -EXTERN_UNOP(arctanh, f32); -EXTERN_UNOP(arctanh, f64); +UNOP(arctanh, f32, (__builtin_atanhf(a))) +UNOP(arctanh, f64, (__builtin_atanh(a))) -EXTERN_UNOP(exp, f32); -EXTERN_UNOP(exp, f64); +UNOP(exp, f32, (__builtin_expf(a))) +UNOP(exp, f64, (__builtin_exp(a))) -EXTERN_UNOP(log, f32); -EXTERN_UNOP(log, f64); +UNOP(log, f32, (__builtin_logf(a))) +UNOP(log, f64, (__builtin_log(a))) -EXTERN_UNOP(cbrt, f32); -EXTERN_UNOP(cbrt, f64); +UNOP(cbrt, f32, (__builtin_cbrtf(a))) +UNOP(cbrt, f64, (__builtin_cbrt(a))) -EXTERN_UNOP(ceil, f32); -EXTERN_UNOP(ceil, f64); +UNOP(ceil, f32, (__builtin_ceilf(a))) +UNOP(ceil, f64, (__builtin_ceil(a))) -EXTERN_UNOP(floor, f32); -EXTERN_UNOP(floor, f64); +UNOP(floor, f32, (__builtin_floorf(a))) +UNOP(floor, f64, (__builtin_floor(a))) -EXTERN_UNOP(rint, f32); -EXTERN_UNOP(rint, f64); +UNOP(rint, f32, (__builtin_rintf(a))) +UNOP(rint, f64, (__builtin_rint(a))) MOD_OP(i32) MOD_OP(i64) MOD_OP(u32) MOD_OP(u64) -EXTERN_BINOP(mod, f32); -EXTERN_BINOP(mod, f64); +BINOP(mod, f32, (__builtin_fmodf(a, b))) +BINOP(mod, f64, (__builtin_fmod(a, b))) PYMOD_OP(i32) PYMOD_OP(i64) PYMOD_OP(u32) PYMOD_OP(u64) -EXTERN_BINOP(pymod, f32); -EXTERN_BINOP(pymod, f64); +BINOP(pymod, f32, (__builtin_fmodf(__builtin_fmodf(a, b) + b, b))) +BINOP(pymod, f64, (__builtin_fmod(__builtin_fmod(a, b) + b, b))) -EXTERN_BINOP(pow, f32); -EXTERN_BINOP(pow, f64); +BINOP(pow, f32, (__builtin_powf(a, b))) +BINOP(pow, f64, (__builtin_pow(a, b))) NULL_EQ_OP(bool) NULL_EQ_OP(i8) @@ -479,9 +454,9 @@ NULL_EQ_OP(u32) NULL_EQ_OP(u64) NULL_EQ_OP(f32) NULL_EQ_OP(f64) -NULL_EQ_OP(decimal32) -NULL_EQ_OP(decimal64) -NULL_EQ_OP(decimal128) +NULL_EQ_OP(dec32) +NULL_EQ_OP(dec64) +NULL_EQ_OP(dec128) NULL_EQ_OP(timestamp_D) NULL_EQ_OP(timestamp_s) NULL_EQ_OP(timestamp_ms) @@ -504,7 +479,7 @@ __device__ __forceinline__ void null_logical_and(optional* out, { if (a.has_value() && b.has_value()) { *out = (*a && *b); - } else if (!a.has_value() && !b.has_value()) { + } else if (a.has_null() && b.has_null()) { *out = nullopt; } else { bool valid = a.has_value() ? *a : *b; @@ -527,7 +502,7 @@ __device__ __forceinline__ void null_logical_or(optional* out, { if (a.has_value() && b.has_value()) { *out = (*a || *b); - } else if (!a.has_value() && !b.has_value()) { + } else if (a.has_null() && b.has_null()) { *out = nullopt; } else { bool valid = a.has_value() ? *a : *b; @@ -539,6 +514,5 @@ __device__ __forceinline__ void null_logical_or(optional* out, } } -} // namespace operators -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf +} // namespace functions +} // namespace JCUDF_EXPORT jcudf diff --git a/cpp/include/jcudf/functions.def.cuh b/cpp/include/jcudf/functions.def.cuh new file mode 100644 index 000000000000..b1fe86d98606 --- /dev/null +++ b/cpp/include/jcudf/functions.def.cuh @@ -0,0 +1,11 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace JCUDF_EXPORT jcudf { + +} // namespace JCUDF_EXPORT jcudf diff --git a/cpp/include/jcudf/library.cu b/cpp/include/jcudf/library.cu new file mode 100644 index 000000000000..42bbac350d09 --- /dev/null +++ b/cpp/include/jcudf/library.cu @@ -0,0 +1,6 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include diff --git a/cpp/include/jcudf/types.cuh b/cpp/include/jcudf/types.cuh new file mode 100644 index 000000000000..3e63c29739b9 --- /dev/null +++ b/cpp/include/jcudf/types.cuh @@ -0,0 +1,924 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) + +#define JCUDF_EXPORT __attribute__((visibility("default"))) + +#else + +#define JCUDF_EXPORT + +#endif + +namespace JCUDF_EXPORT jcudf { + +using i8 = signed char; +using i16 = signed short; +using i32 = signed int; +using i64 = signed long; +using i128 = __int128_t; +using u8 = unsigned char; +using u16 = unsigned short; +using u32 = unsigned int; +using u64 = unsigned long; + +using char_utf8 = u32; + +using usize = unsigned long; +using iptr = i64; +using uptr = u64; + +using intmax_t = i64; +using uintmax_t = u64; + +using f32 = float; +using f64 = double; + +using size_type = i32; + +using bitmask_t = u32; + +__device__ constexpr bool bit_is_set(bitmask_t const* bitmask, usize bit_index) +{ + constexpr auto bits_per_word = sizeof(bitmask_t) * 8; + return bitmask[bit_index / bits_per_word] & (bitmask_t{1} << (bit_index % bits_per_word)); +} + +enum class type_id : i32 { + EMPTY = 0, + INT8 = 1, + INT16 = 2, + INT32 = 3, + INT64 = 4, + UINT8 = 5, + UINT16 = 6, + UINT32 = 7, + UINT64 = 8, + FLOAT32 = 9, + FLOAT64 = 10, + BOOL8 = 11, + TIMESTAMP_DAYS = 12, + TIMESTAMP_SECONDS = 13, + TIMESTAMP_MILLISECONDS = 14, + TIMESTAMP_MICROSECONDS = 15, + TIMESTAMP_NANOSECONDS = 16, + DURATION_DAYS = 17, + DURATION_SECONDS = 18, + DURATION_MILLISECONDS = 19, + DURATION_MICROSECONDS = 20, + DURATION_NANOSECONDS = 21, + DICTIONARY32 = 22, + STRING = 23, + LIST = 24, + DECIMAL32 = 25, + DECIMAL64 = 26, + DECIMAL128 = 27, + STRUCT = 28, + NUM_TYPE_IDS = 29 +}; + +struct data_type { + type_id _id = {}; + + i32 _scale = 0; + + __device__ constexpr type_id id() const { return _id; } + + __device__ constexpr i32 scale() const { return _scale; } +}; + +template +__device__ constexpr T min(T a, T b) +{ + return a < b ? a : b; +} + +template +__device__ constexpr T max(T a, T b) +{ + return a > b ? a : b; +} + +template +__device__ constexpr T ipow10(T exponent) +{ + if (exponent == 0) { return 1; } + + T extra = 1; + T square = 10; + T n = exponent; + + while (n > 1) { + if ((n & 1) == 1) { extra *= square; } + n >>= 1; + square *= square; + } + + return square * extra; +} + +template +__device__ constexpr T dec_lshift(T v, i32 scale) +{ + return v * ipow10(-scale); +} + +template +__device__ constexpr T dec_rshift(T v, i32 scale) +{ + return v / ipow10(scale); +} + +template +__device__ constexpr T dec_shift(T v, i32 scale) +{ + if (scale == 0) { + return v; + } else if (scale < 0) { + return dec_lshift(v, scale); + } else { + return dec_rshift(v, scale); + } +} + +template +__device__ constexpr T dec_rescale(T v, i32 from_scale, i32 to_scale) +{ + return dec_shift(v, to_scale - from_scale); +} + +struct scaled_t {}; + +inline constexpr scaled_t scaled; + +struct dec32 { + i32 _value = 0; + + i32 _scale = 0; + + __device__ constexpr dec32(scaled_t, i32 value, i32 scale) : _value{value}, _scale{scale} {} + + constexpr dec32() = default; + + __device__ constexpr i32 value() const { return _value; } + + __device__ constexpr i32 scale() const { return _scale; } +}; + +struct dec64 { + i64 _value = 0; + + i32 _scale = 0; + + __device__ constexpr dec64(scaled_t, i64 value, i32 scale) : _value{value}, _scale{scale} {} + + constexpr dec64() = default; + + __device__ constexpr i64 value() const { return _value; } + + __device__ constexpr i32 scale() const { return _scale; } +}; + +struct dec128 { + i128 _value = 0; + + i32 _scale = 0; + + __device__ constexpr dec128(scaled_t, i128 value, i32 scale) : _value{value}, _scale{scale} {} + + constexpr dec128() = default; + + __device__ constexpr i128 value() const { return _value; } + + __device__ constexpr i32 scale() const { return _scale; } +}; + +#define DECIMAL_OPS(T) \ + __device__ constexpr T rescale(T a, i32 scale) \ + { \ + return T{scaled, dec_rescale(a._value, a._scale, scale), scale}; \ + } \ + \ + __device__ constexpr T operator+(T const& a, T const& b) \ + { \ + auto scale = min(a._scale, b._scale); \ + auto r = rescale(a, scale)._value + rescale(b, scale)._value; \ + return T{scaled, r, scale}; \ + } \ + \ + __device__ constexpr T operator-(T const& a, T const& b) \ + { \ + auto scale = min(a._scale, b._scale); \ + auto r = rescale(a, scale)._value - rescale(b, scale)._value; \ + return T{scaled, r, scale}; \ + } \ + \ + __device__ constexpr T operator*(T const& a, T const& b) \ + { \ + return T{scaled, a._value + b._value, a._scale + b._scale}; \ + } \ + \ + __device__ constexpr T operator/(T const& a, T const& b) \ + { \ + return T{scaled, a._value / b._value, a._scale - b._scale}; \ + } \ + \ + __device__ constexpr T operator%(T const& a, T const& b) \ + { \ + auto scale = min(a._scale, b._scale); \ + auto r = rescale(a, scale)._value % rescale(b, scale)._value; \ + return T{scaled, r, scale}; \ + } \ + \ + __device__ constexpr bool operator==(T const& a, T const& b) \ + { \ + auto scale = min(a._scale, b._scale); \ + return rescale(a, scale)._value == rescale(b, scale)._value; \ + } \ + \ + __device__ constexpr bool operator!=(T const& a, T const& b) { return !(a == b); } \ + \ + __device__ constexpr bool operator>(T const& a, T const& b) \ + { \ + auto scale = min(a._scale, b._scale); \ + return rescale(a, scale)._value > rescale(b, scale)._value; \ + } \ + \ + __device__ constexpr bool operator<(T const& a, T const& b) \ + { \ + auto scale = min(a._scale, b._scale); \ + return rescale(a, scale)._value < rescale(b, scale)._value; \ + } \ + \ + __device__ constexpr bool operator>=(T const& a, T const& b) \ + { \ + auto scale = min(a._scale, b._scale); \ + return rescale(a, scale)._value >= rescale(b, scale)._value; \ + } \ + \ + __device__ constexpr bool operator<=(T const& a, T const& b) \ + { \ + auto scale = min(a._scale, b._scale); \ + return rescale(a, scale)._value <= rescale(b, scale)._value; \ + } + +DECIMAL_OPS(dec32) +DECIMAL_OPS(dec64) +DECIMAL_OPS(dec128) + +struct timestamp_D { + i32 _rep = 0; +}; + +struct timestamp_h { + i32 _rep = 0; +}; + +struct timestamp_m { + i32 _rep = 0; +}; + +struct timestamp_s { + i64 _rep = 0; +}; + +struct timestamp_ms { + i64 _rep = 0; +}; + +struct timestamp_us { + i64 _rep = 0; +}; + +struct timestamp_ns { + i64 _rep = 0; +}; + +#define TIMESTAMP_OPS(T) \ + __device__ constexpr bool operator==(T a, T b) { return a._rep == b._rep; } \ + __device__ constexpr bool operator!=(T a, T b) { return a._rep != b._rep; } \ + __device__ constexpr bool operator>(T a, T b) { return a._rep > b._rep; } \ + __device__ constexpr bool operator<(T a, T b) { return a._rep < b._rep; } \ + __device__ constexpr bool operator>=(T a, T b) { return a._rep >= b._rep; } \ + __device__ constexpr bool operator<=(T a, T b) { return a._rep <= b._rep; } + +TIMESTAMP_OPS(timestamp_D) +TIMESTAMP_OPS(timestamp_h) +TIMESTAMP_OPS(timestamp_m) +TIMESTAMP_OPS(timestamp_s) +TIMESTAMP_OPS(timestamp_ms) +TIMESTAMP_OPS(timestamp_us) +TIMESTAMP_OPS(timestamp_ns) + +struct duration_D { + i32 _rep = 0; + + __device__ constexpr auto count() const { return _rep; } +}; + +struct duration_h { + i32 _rep = 0; + + __device__ constexpr auto count() const { return _rep; } +}; + +struct duration_m { + i32 _rep = 0; + + __device__ constexpr auto count() const { return _rep; } +}; + +struct duration_s { + i64 _rep = 0; + + __device__ constexpr auto count() const { return _rep; } +}; + +struct duration_ms { + i64 _rep = 0; + + __device__ constexpr auto count() const { return _rep; } +}; + +struct duration_us { + i64 _rep = 0; + + __device__ constexpr auto count() const { return _rep; } +}; + +struct duration_ns { + i64 _rep = 0; + + __device__ constexpr auto count() const { return _rep; } +}; + +#define DURATION_OPS(T) \ + __device__ constexpr T operator+(T a, T b) { return T{a._rep + b._rep}; } \ + __device__ constexpr T operator-(T a, T b) { return T{a._rep - b._rep}; } \ + __device__ constexpr bool operator==(T a, T b) { return a._rep == b._rep; } \ + __device__ constexpr bool operator!=(T a, T b) { return a._rep != b._rep; } \ + __device__ constexpr bool operator>(T a, T b) { return a._rep > b._rep; } \ + __device__ constexpr bool operator<(T a, T b) { return a._rep < b._rep; } \ + __device__ constexpr bool operator>=(T a, T b) { return a._rep >= b._rep; } \ + __device__ constexpr bool operator<=(T a, T b) { return a._rep <= b._rep; } + +DURATION_OPS(duration_D) +DURATION_OPS(duration_h) +DURATION_OPS(duration_m) +DURATION_OPS(duration_s) +DURATION_OPS(duration_ms) +DURATION_OPS(duration_us) +DURATION_OPS(duration_ns) + +struct string_view { + static constexpr size_type const UNKNOWN_STRING_LENGTH{-1}; + static constexpr size_type const npos{-1}; + + char const* _data = nullptr; + + size_type _bytes = 0; + + mutable size_type _length = UNKNOWN_STRING_LENGTH; + + __device__ constexpr size_type size_bytes() const { return _bytes; } + + __device__ constexpr char const* data() const { return _data; } + + __device__ constexpr bool empty() const { return _bytes == 0; } + + __device__ constexpr size_type compare(string_view const& other) const + { + auto* s0 = _data; + auto n0 = _bytes; + auto* s1 = other._data; + auto n1 = other._bytes; + auto max = n0 < n1 ? n0 : n1; + + if (s0 == s1 && n0 == n1) return 0; + + size_type idx = 0; + + while (idx < max) { + if (*s0 != *s1) return static_cast(*s0) - static_cast(*s1); + s0++; + s1++; + idx++; + } + + if (idx < n0) { return 1; } + if (idx < n1) { return -1; } + + return 0; + } +}; + +__device__ constexpr bool operator==(string_view const& a, string_view const& b) +{ + return a.compare(b) == 0; +} + +__device__ constexpr bool operator!=(string_view const& a, string_view const& b) +{ + return a.compare(b) != 0; +} + +__device__ constexpr bool operator<(string_view const& a, string_view const& b) +{ + return a.compare(b) < 0; +} + +__device__ constexpr bool operator>(string_view const& a, string_view const& b) +{ + return a.compare(b) > 0; +} + +__device__ constexpr bool operator<=(string_view const& a, string_view const& b) +{ + return a.compare(b) <= 0; +} + +__device__ constexpr bool operator>=(string_view const& a, string_view const& b) +{ + return a.compare(b) >= 0; +} + +struct inplace_t {}; + +inline constexpr inplace_t inplace; + +struct nullopt_t {}; + +inline constexpr nullopt_t nullopt; + +template +struct optional { + T _val = {}; + + bool _engaged = false; + + constexpr optional() = default; + + __device__ constexpr optional(nullopt_t) {} + + template + __device__ constexpr optional(inplace_t, Args&&... args) + : _val{static_cast(args)...}, _engaged{true} + { + } + + __device__ constexpr optional(T val) : _val{val}, _engaged{true} {} + + __device__ constexpr bool has_value() const { return _engaged; } + + __device__ constexpr bool has_null() const { return !_engaged; } + + __device__ constexpr void reset() { _engaged = false; } + + __device__ constexpr T const& get() const { return _val; } + + __device__ constexpr T& get() { return _val; } + + __device__ constexpr T const* operator->() const { return &_val; } + + __device__ constexpr T* operator->() { return &_val; } + + __device__ constexpr T const& operator*() const { return _val; } + + __device__ constexpr T& operator*() { return _val; } + + __device__ constexpr T const& value() const { return _val; } + + __device__ constexpr T& value() { return _val; } + + __device__ constexpr explicit operator bool() const { return _engaged; } + + __device__ constexpr T value_or(T __v) const { return _engaged ? _val : __v; } +}; + +template +optional(T) -> optional; + +#define INST(T) template struct optional; + +INST(bool); +INST(i8); +INST(i16); +INST(i32); +INST(i64); +INST(u8); +INST(u16); +INST(u32); +INST(u64); +INST(f32); +INST(f64); +INST(timestamp_D); +INST(timestamp_h); +INST(timestamp_m); +INST(timestamp_s); +INST(timestamp_ms); +INST(timestamp_us); +INST(timestamp_ns); +INST(duration_D); +INST(duration_h); +INST(duration_m); +INST(duration_s); +INST(duration_ms); +INST(duration_us); +INST(duration_ns); +INST(dec32); +INST(dec64); +INST(dec128); +INST(string_view); + +#undef INST + +template +struct span { + T* _data = nullptr; + + usize _size = 0; + + __device__ constexpr T* data() const { return _data; } + + __device__ constexpr usize size() const { return _size; } + + __device__ constexpr bool empty() const { return _size == 0; } + + __device__ constexpr T& operator[](usize pos) const { return _data[pos]; } + + __device__ constexpr T* begin() const { return _data; } + + __device__ constexpr T* end() const { return _data + _size; } + + __device__ constexpr span as_const() const { return span{_data, _size}; } + + __device__ constexpr T& element(usize idx) const { return _data[idx]; } + + __device__ constexpr void assign(usize idx, T value) const { _data[idx] = value; } +}; + +#define INST(T) template struct span; + +INST(bool); +INST(i8); +INST(i16); +INST(i32); +INST(i64); +INST(u8); +INST(u16); +INST(u32); +INST(u64); +INST(f32); +INST(f64); +INST(timestamp_D); +INST(timestamp_h); +INST(timestamp_m); +INST(timestamp_s); +INST(timestamp_ms); +INST(timestamp_us); +INST(timestamp_ns); +INST(duration_D); +INST(duration_h); +INST(duration_m); +INST(duration_s); +INST(duration_ms); +INST(duration_us); +INST(duration_ns); +INST(dec32); +INST(dec64); +INST(dec128); +INST(string_view); + +#undef INST + +template +struct optional_span { + T* _data = nullptr; + + usize _size = 0; + + bitmask_t const* _null_mask = nullptr; + + __device__ constexpr T* data() const { return _data; } + + __device__ constexpr usize size() const { return _size; } + + __device__ constexpr bool empty() const { return _size == 0; } + + __device__ constexpr T& operator[](usize pos) const { return _data[pos]; } + + __device__ constexpr T* begin() const { return _data; } + + __device__ constexpr T* end() const { return _data + _size; } + + __device__ constexpr optional_span as_const() const + { + return optional_span{_data, _size, _null_mask}; + } + + __device__ constexpr bool nullable() const { return _null_mask != nullptr; } + + __device__ constexpr bool is_valid_nocheck(usize element_index) const + { + return bit_is_set(_null_mask, element_index); + } + + __device__ constexpr bool is_valid(usize element_index) const + { + return not nullable() or is_valid_nocheck(element_index); + } + + __device__ constexpr bool is_null(usize element_index) const { return !is_valid(element_index); } + + __device__ constexpr T& element(usize idx) const { return _data[idx]; } + + __device__ constexpr optional nullable_element(usize idx) const; + + __device__ constexpr void assign(usize idx, T value) const { _data[idx] = value; } +}; + +#define INST(T) template struct optional_span; + +INST(bool); +INST(i8); +INST(i16); +INST(i32); +INST(i64); +INST(u8); +INST(u16); +INST(u32); +INST(u64); +INST(f32); +INST(f64); +INST(timestamp_D); +INST(timestamp_h); +INST(timestamp_m); +INST(timestamp_s); +INST(timestamp_ms); +INST(timestamp_us); +INST(timestamp_ns); +INST(duration_D); +INST(duration_h); +INST(duration_m); +INST(duration_s); +INST(duration_ms); +INST(duration_us); +INST(duration_ns); +INST(dec32); +INST(dec64); +INST(dec128); +INST(string_view); + +#undef INST + +struct alignas(16) column_device_view { + data_type _type = {}; + + size_type _size = 0; + + void const* _data = nullptr; + + bitmask_t const* _null_mask = nullptr; + + size_type _offset = 0; + + column_device_view* _d_children = nullptr; + + size_type _num_children = 0; + + __device__ constexpr data_type type() const { return _type; } + + __device__ constexpr size_type size() const { return _size; } + + __device__ constexpr bool nullable() const { return _null_mask != nullptr; } + + __device__ constexpr bitmask_t const* null_mask() const { return _null_mask; } + + __device__ constexpr size_type offset() const { return _offset; } + + __device__ constexpr bool is_valid(size_type idx) const + { + return !nullable() || is_valid_nocheck(idx); + } + + __device__ constexpr bool is_valid_nocheck(size_type idx) const + { + return bit_is_set(_null_mask, _offset + idx); + } + + __device__ constexpr bool is_null(size_type idx) const { return !is_valid(idx); } + + __device__ constexpr bool is_null_nocheck(size_type idx) const { return !is_valid_nocheck(idx); } + + __device__ constexpr size_type num_child_columns() const { return _num_children; } + + template + __device__ T element(size_type idx) const; + + template + __device__ optional nullable_element(size_type idx) const + { + if (!is_valid(idx)) return nullopt; + return element(idx); + } +}; + +#define SPEC(T) \ + template <> \ + __device__ T column_device_view::element(size_type idx) const \ + { \ + return static_cast(_data)[_offset + idx]; \ + } + +SPEC(bool) +SPEC(i8) +SPEC(i16) +SPEC(i32) +SPEC(i64) +SPEC(u8) +SPEC(u16) +SPEC(u32) +SPEC(u64) +SPEC(f32) +SPEC(f64) +SPEC(timestamp_D) +SPEC(timestamp_h) +SPEC(timestamp_m) +SPEC(timestamp_s) +SPEC(timestamp_ms) +SPEC(timestamp_us) +SPEC(timestamp_ns) +SPEC(duration_D) +SPEC(duration_h) +SPEC(duration_m) +SPEC(duration_s) +SPEC(duration_ms) +SPEC(duration_us) +SPEC(duration_ns) + +#undef SPEC + +#define SPEC(T, Repr) \ + template <> \ + __device__ T column_device_view::element(size_type idx) const \ + { \ + return T{scaled, static_cast(_data)[_offset + idx], _type.scale()}; \ + } + +SPEC(dec32, i32) +SPEC(dec64, i64) +SPEC(dec128, i128) + +#undef SPEC + +template <> +__device__ string_view column_device_view::element(size_type idx) const +{ + static constexpr i32 OFFSETS_CHILD = 0; + auto i = _offset + idx; + auto* str_data = static_cast(_data); + auto& offsets = _d_children[OFFSETS_CHILD]; + auto* i32_runs = static_cast(offsets._data); + auto* i64_runs = static_cast(offsets._data); + + i64 run_begin = 0; + i64 run_end = 0; + + switch (offsets.type().id()) { + case type_id::INT32: + run_begin = i32_runs[i]; + run_end = i32_runs[i + 1]; + break; + case type_id::INT64: + run_begin = i64_runs[i]; + run_end = i64_runs[i + 1]; + break; + default: __builtin_unreachable(); + } + + i64 run_size = run_end - run_begin; + + return string_view{str_data + run_begin, static_cast(run_size)}; +} + +#define INST(T) \ + template __device__ optional column_device_view::nullable_element(size_type idx) const; + +INST(bool) +INST(i8) +INST(i16) +INST(i32) +INST(i64) +INST(u8) +INST(u16) +INST(u32) +INST(u64) +INST(f32) +INST(f64) +INST(timestamp_D) +INST(timestamp_h) +INST(timestamp_m) +INST(timestamp_s) +INST(timestamp_ms) +INST(timestamp_us) +INST(timestamp_ns) +INST(duration_D) +INST(duration_h) +INST(duration_m) +INST(duration_s) +INST(duration_ms) +INST(duration_us) +INST(duration_ns) +INST(dec32) +INST(dec64) +INST(dec128) +INST(string_view) + +#undef INST + +/// @brief Type-erased parameters for LTO-JIT-compiled transform operations. +struct transform_operator_params { + /// @brief Pointer to scope data (e.g. column views, scalars, etc.). + void* const* scope = nullptr; + + /// @brief Total number of rows to process. + size_type num_rows = 0; + + /// @brief Current row index. + size_type row_index = 0; +}; + +// TODO: scope variables should be aligned to avoid uncoalesced reads/writes +namespace scope { + +using args = void* const*; + +template +struct column { + static constexpr bool IS_SCALAR = IsScalar; + static constexpr bool IS_NULLABLE = IsNullable; + + using Type = T; + using Arg = ColumnType const*; + + static __device__ decltype(auto) element(args scope, size_type i) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + if constexpr (!IsNullable) { + return p->template element(index); + } else { + return p->template nullable_element(index); + } + } + + static __device__ void assign(args scope, size_type i, T value) + { + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + p->template assign(index, value); + } + + static __device__ auto* null_mask(args scope) + { + auto p = static_cast(scope[ScopeIndex]); + return p->null_mask(); + } + + static __device__ bool is_null(args scope, size_type i) + { + if constexpr (!IsNullable) { return false; } + + auto p = static_cast(scope[ScopeIndex]); + auto index = IsScalar ? 0 : i; + + return p->is_null(index); + } + + static __device__ bool is_valid(args scope, size_type i) { return !is_null(scope, i); } +}; + +template +struct user_data { + using Arg = void*; + + static __device__ decltype(auto) element(args scope, [[maybe_unused]] size_type i) + { + return static_cast(scope[ScopeIndex]); + } +}; + +} // namespace scope +} // namespace JCUDF_EXPORT jcudf diff --git a/cpp/src/jit/lto/library.cu b/cpp/src/jit/lto/library.cu deleted file mode 100644 index 3776502185b6..000000000000 --- a/cpp/src/jit/lto/library.cu +++ /dev/null @@ -1,9 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include diff --git a/cpp/src/jit/lto/operators.def.cuh b/cpp/src/jit/lto/operators.def.cuh deleted file mode 100644 index d847420bb598..000000000000 --- a/cpp/src/jit/lto/operators.def.cuh +++ /dev/null @@ -1,82 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include -#include - -#include - -namespace CUDF_LTO_EXPORT cudf { -namespace lto { - -#define DEF_UNOP(name, type, expr) \ - __device__ void name(type* out, type const& a) { *out = expr; } - -#define DEF_BINOP(name, type, expr) \ - __device__ void name(type* out, type const& a, type const& b) { *out = expr; } - -DEF_UNOP(sin, f32, (cuda::std::sin(a))) -DEF_UNOP(sin, f64, (cuda::std::sin(a))) - -DEF_UNOP(cos, f32, (cuda::std::cos(a))) -DEF_UNOP(cos, f64, (cuda::std::cos(a))) - -DEF_UNOP(tan, f32, (cuda::std::tan(a))) -DEF_UNOP(tan, f64, (cuda::std::tan(a))) - -DEF_UNOP(arcsin, f32, (cuda::std::asin(a))) -DEF_UNOP(arcsin, f64, (cuda::std::asin(a))) - -DEF_UNOP(arccos, f32, (cuda::std::acos(a))) -DEF_UNOP(arccos, f64, (cuda::std::acos(a))) - -DEF_UNOP(arctan, f32, (cuda::std::atan(a))) -DEF_UNOP(arctan, f64, (cuda::std::atan(a))) - -DEF_UNOP(sinh, f32, (cuda::std::sinh(a))) -DEF_UNOP(sinh, f64, (cuda::std::sinh(a))) - -DEF_UNOP(cosh, f32, (cuda::std::cosh(a))) -DEF_UNOP(cosh, f64, (cuda::std::cosh(a))) - -DEF_UNOP(tanh, f32, (cuda::std::tanh(a))) -DEF_UNOP(tanh, f64, (cuda::std::tanh(a))) - -DEF_UNOP(arcsinh, f32, (cuda::std::asinh(a))) -DEF_UNOP(arcsinh, f64, (cuda::std::asinh(a))) - -DEF_UNOP(arccosh, f32, (cuda::std::acosh(a))) -DEF_UNOP(arccosh, f64, (cuda::std::acosh(a))) - -DEF_UNOP(arctanh, f32, (cuda::std::atanh(a))) -DEF_UNOP(arctanh, f64, (cuda::std::atanh(a))) - -DEF_UNOP(exp, f32, (cuda::std::exp(a))) -DEF_UNOP(exp, f64, (cuda::std::exp(a))) - -DEF_UNOP(log, f32, (cuda::std::log(a))) -DEF_UNOP(log, f64, (cuda::std::log(a))) - -DEF_UNOP(cbrt, f32, (cuda::std::cbrt(a))) -DEF_UNOP(cbrt, f64, (cuda::std::cbrt(a))) - -DEF_UNOP(ceil, f32, (cuda::std::ceil(a))) -DEF_UNOP(ceil, f64, (cuda::std::ceil(a))) - -DEF_UNOP(floor, f32, (cuda::std::floor(a))) -DEF_UNOP(floor, f64, (cuda::std::floor(a))) - -DEF_UNOP(rint, f32, (cuda::std::rint(a))) -DEF_UNOP(rint, f64, (cuda::std::rint(a))) - -DEF_BINOP(mod, f32, (cuda::std::fmod(a, b))) -DEF_BINOP(mod, f64, (cuda::std::fmod(a, b))) - -DEF_BINOP(pymod, f32, (cuda::std::fmod(cuda::std::fmod(a, b) + b, b))) -DEF_BINOP(pymod, f64, (cuda::std::fmod(cuda::std::fmod(a, b) + b, b))) - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/jit/lto/string_view.def.cuh b/cpp/src/jit/lto/string_view.def.cuh deleted file mode 100644 index 957eb6e07b96..000000000000 --- a/cpp/src/jit/lto/string_view.def.cuh +++ /dev/null @@ -1,125 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include -#include - -namespace CUDF_LTO_EXPORT cudf { -namespace lto { - -[[nodiscard]] __device__ size_type string_view::size_bytes() const -{ - return lift(this)->size_bytes(); -} - -[[nodiscard]] __device__ size_type string_view::length() const { return lift(this)->length(); } - -[[nodiscard]] __device__ char const* string_view::data() const { return lift(this)->data(); } - -[[nodiscard]] __device__ bool string_view::empty() const { return lift(this)->empty(); } - -__device__ char_utf8 string_view::operator[](size_type pos) const -{ - return lift(this)->operator[](pos); -} - -[[nodiscard]] __device__ size_type string_view::byte_offset(size_type pos) const -{ - return lift(this)->byte_offset(pos); -} - -[[nodiscard]] __device__ int string_view::compare(string_view const& str) const -{ - return lift(this)->compare(*lift(&str)); -} - -__device__ int string_view::compare(char const* str, size_type bytes) const -{ - return lift(this)->compare(str, bytes); -} - -__device__ bool string_view::operator==(string_view const& rhs) const -{ - return lift(this)->operator==(*lift(&rhs)); -} - -__device__ bool string_view::operator!=(string_view const& rhs) const -{ - return lift(this)->operator!=(*lift(&rhs)); -} - -__device__ bool string_view::operator<(string_view const& rhs) const -{ - return lift(this)->operator<(*lift(&rhs)); -} - -__device__ bool string_view::operator>(string_view const& rhs) const -{ - return lift(this)->operator>(*lift(&rhs)); -} - -__device__ bool string_view::operator<=(string_view const& rhs) const -{ - return lift(this)->operator<=(*lift(&rhs)); -} - -__device__ bool string_view::operator>=(string_view const& rhs) const -{ - return lift(this)->operator>=(*lift(&rhs)); -} - -[[nodiscard]] __device__ size_type string_view::find(string_view const& str, - size_type pos, - size_type count) const -{ - return lift(this)->find(*lift(&str), pos, count); -} - -__device__ size_type string_view::find(char const* str, - size_type bytes, - size_type pos, - size_type count) const -{ - return lift(this)->find(str, bytes, pos, count); -} - -[[nodiscard]] __device__ size_type string_view::find(char_utf8 character, - size_type pos, - size_type count) const -{ - return lift(this)->find(character, pos, count); -} - -[[nodiscard]] __device__ size_type string_view::rfind(string_view const& str, - size_type pos, - size_type count) const -{ - return lift(this)->rfind(*lift(&str), pos, count); -} - -__device__ size_type string_view::rfind(char const* str, - size_type bytes, - size_type pos, - size_type count) const -{ - return lift(this)->rfind(str, bytes, pos, count); -} - -[[nodiscard]] __device__ size_type string_view::rfind(char_utf8 character, - size_type pos, - size_type count) const -{ - return lift(this)->rfind(character, pos, count); -} - -[[nodiscard]] __device__ string_view string_view::substr(size_type start, size_type length) const -{ - auto ret = lift(this)->substr(start, length); - return *lower(&ret); -} - -} // namespace lto -} // namespace CUDF_LTO_EXPORT cudf diff --git a/cpp/src/transform/jit/lto_kernel.inl.cuh b/cpp/src/transform/jit/lto_kernel.inl.cuh deleted file mode 100644 index da5a4c77161b..000000000000 --- a/cpp/src/transform/jit/lto_kernel.inl.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include -#include -#include - -extern "C" { - -__device__ void transform_operator(cudf::lto::transform_params params); - -__global__ void transform_kernel(void* const* scope, int32_t num_rows) -{ - auto start = cudf::detail::grid_1d::global_thread_id(); - auto stride = cudf::detail::grid_1d::grid_stride(); - auto size = num_rows; - - for (auto i = start; i < size; i += stride) { - cudf::lto::transform_params params{.scope = scope, - .row_index = static_cast(i)}; - transform_operator(params); - } -} -} diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 6540de4f8f92..4cd12ac660d5 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -23,7 +23,6 @@ #include #include #include -#include namespace cudf { namespace transformation { diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index 8e36546b24e5..f47c90f53e7a 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -43,22 +43,24 @@ struct element_operation { TEST_F(RTCTest, CompileKernelBasic) { - auto fn = []() { + auto fn = [] { char const* udf = R"***( - #include "cudf/jit/lto/column_view.cuh" - #include "cudf/jit/lto/operators.cuh" - #include "cudf/jit/lto/optional_span.cuh" - #include "cudf/jit/lto/optional.cuh" - #include "cudf/jit/lto/scope.cuh" - #include "cudf/jit/lto/span.cuh" - #include "cudf/jit/lto/string_view.cuh" - #include "cudf/jit/lto/transform_params.cuh" - #include "cudf/jit/lto/types.cuh" + #include "jcudf/functions.cuh" #pragma nv_hdrstop - extern "C" __device__ void transform_operator(cudf::lto::transform_params p){ - using namespace cudf::lto; + struct operator_params{ + void* scope; + jcudf::size_type row_index; + }; + + struct kernel_params{ + void* scope; + jcudf::size_type num_rows; + }; + + extern "C" __device__ void transform_operator(operator_params const& p){ + using namespace jcudf; // unpack inputs from scope using the appropriate getters based on the LTO context using s0 = scope::column<0, column_device_view, int, false, false>; @@ -69,10 +71,21 @@ TEST_F(RTCTest, CompileKernelBasic) auto a1 = s1::element(p.scope, p.row_index); int a2; - operators::add(&a2, &a0, &a1); + functions::add(&a2, &a0, &a1); s2::assign(p.scope, p.row_index, a2); } + + extern "C" __global__ void transform_kernel(kernel_params params) + { + auto offset = static_cast(threadIdx.x) + static_cast(blockIdx.x) * static_cast(blockDim.x); + auto stride = static_cast(blockDim.x) * static_cast(gridDim.x); + + for(jcudf::i64 i = offset; i < params.num_rows; i += stride){ + operator_params p{params.scope, i}; + transform_operator(p); + } + } )***"; static int i = 0; @@ -91,6 +104,7 @@ TEST_F(RTCTest, CompileKernelBasic) EXPECT_EQ("transform_kernel", kernel.get_name()); + /* auto in0 = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9}.release(); auto in1 = cudf::test::fixed_width_column_wrapper{9, 8, 7, 6, 5, 4, 3, 2, 1}.release(); auto out = cudf::test::fixed_width_column_wrapper{0, 0, 0, 0, 0, 0, 0, 0, 0}.release(); @@ -155,7 +169,7 @@ TEST_F(RTCTest, CompileKernelBasic) auto expected = cudf::test::fixed_width_column_wrapper{10, 10, 10, 10, 10, 10, 10, 10, 10}.release(); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(out->view(), expected->view()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(out->view(), expected->view());*/ }; fn(); // warm up cache From 5ffddda594c2069801687fba2ed8a41939784ce5 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 22 Feb 2026 04:03:39 +0000 Subject: [PATCH 058/254] update --- cpp/include/jcudf/functions.cuh | 518 -------------- cpp/include/jcudf/functions.def.cuh | 11 - cpp/include/jcudf/functions/abs.cuh | 27 + cpp/include/jcudf/functions/add.cuh | 27 + cpp/include/jcudf/functions/arccos.cuh | 25 + cpp/include/jcudf/functions/arccosh.cuh | 25 + cpp/include/jcudf/functions/arcsin.cuh | 25 + cpp/include/jcudf/functions/arcsinh.cuh | 25 + cpp/include/jcudf/functions/arctan.cuh | 25 + cpp/include/jcudf/functions/arctanh.cuh | 25 + cpp/include/jcudf/functions/bit_and.cuh | 27 + cpp/include/jcudf/functions/bit_invert.cuh | 27 + cpp/include/jcudf/functions/bit_or.cuh | 27 + cpp/include/jcudf/functions/bit_xor.cuh | 27 + cpp/include/jcudf/functions/cast_to_f64.cuh | 27 + cpp/include/jcudf/functions/cast_to_i64.cuh | 27 + cpp/include/jcudf/functions/cast_to_u64.cuh | 27 + cpp/include/jcudf/functions/cbrt.cuh | 25 + cpp/include/jcudf/functions/ceil.cuh | 25 + cpp/include/jcudf/functions/cos.cuh | 25 + cpp/include/jcudf/functions/cosh.cuh | 25 + cpp/include/jcudf/functions/div.cuh | 27 + cpp/include/jcudf/functions/equal.cuh | 27 + cpp/include/jcudf/functions/exp.cuh | 25 + cpp/include/jcudf/functions/floor.cuh | 25 + cpp/include/jcudf/functions/greater.cuh | 25 + cpp/include/jcudf/functions/greater_equal.cuh | 25 + cpp/include/jcudf/functions/identity.cuh | 21 + cpp/include/jcudf/functions/is_null.cuh | 21 + cpp/include/jcudf/functions/less.cuh | 25 + cpp/include/jcudf/functions/less_equal.cuh | 25 + cpp/include/jcudf/functions/log.cuh | 25 + cpp/include/jcudf/functions/logical_and.cuh | 27 + cpp/include/jcudf/functions/logical_or.cuh | 27 + cpp/include/jcudf/functions/mod.cuh | 31 + cpp/include/jcudf/functions/mul.cuh | 27 + cpp/include/jcudf/functions/null_equal.cuh | 27 + .../jcudf/functions/null_logical_and.cuh | 27 + .../jcudf/functions/null_logical_or.cuh | 27 + cpp/include/jcudf/functions/pow.cuh | 25 + cpp/include/jcudf/functions/pymod.cuh | 37 + cpp/include/jcudf/functions/rint.cuh | 25 + cpp/include/jcudf/functions/sin.cuh | 25 + cpp/include/jcudf/functions/sinh.cuh | 25 + cpp/include/jcudf/functions/sub.cuh | 27 + cpp/include/jcudf/functions/tanh.cuh | 25 + cpp/include/jcudf/library.cu | 1 - cpp/include/jcudf/types.cuh | 670 ++++++------------ cpp/src/jit/jit.cpp | 242 +++++-- cpp/src/jit/jit.hpp | 33 +- cpp/src/jit/rtc/rtc.cpp | 20 +- cpp/src/jit/rtc/rtc.hpp | 2 + cpp/tests/jit/rtc.cpp | 43 +- 53 files changed, 1608 insertions(+), 1078 deletions(-) delete mode 100644 cpp/include/jcudf/functions.cuh delete mode 100644 cpp/include/jcudf/functions.def.cuh create mode 100644 cpp/include/jcudf/functions/abs.cuh create mode 100644 cpp/include/jcudf/functions/add.cuh create mode 100644 cpp/include/jcudf/functions/arccos.cuh create mode 100644 cpp/include/jcudf/functions/arccosh.cuh create mode 100644 cpp/include/jcudf/functions/arcsin.cuh create mode 100644 cpp/include/jcudf/functions/arcsinh.cuh create mode 100644 cpp/include/jcudf/functions/arctan.cuh create mode 100644 cpp/include/jcudf/functions/arctanh.cuh create mode 100644 cpp/include/jcudf/functions/bit_and.cuh create mode 100644 cpp/include/jcudf/functions/bit_invert.cuh create mode 100644 cpp/include/jcudf/functions/bit_or.cuh create mode 100644 cpp/include/jcudf/functions/bit_xor.cuh create mode 100644 cpp/include/jcudf/functions/cast_to_f64.cuh create mode 100644 cpp/include/jcudf/functions/cast_to_i64.cuh create mode 100644 cpp/include/jcudf/functions/cast_to_u64.cuh create mode 100644 cpp/include/jcudf/functions/cbrt.cuh create mode 100644 cpp/include/jcudf/functions/ceil.cuh create mode 100644 cpp/include/jcudf/functions/cos.cuh create mode 100644 cpp/include/jcudf/functions/cosh.cuh create mode 100644 cpp/include/jcudf/functions/div.cuh create mode 100644 cpp/include/jcudf/functions/equal.cuh create mode 100644 cpp/include/jcudf/functions/exp.cuh create mode 100644 cpp/include/jcudf/functions/floor.cuh create mode 100644 cpp/include/jcudf/functions/greater.cuh create mode 100644 cpp/include/jcudf/functions/greater_equal.cuh create mode 100644 cpp/include/jcudf/functions/identity.cuh create mode 100644 cpp/include/jcudf/functions/is_null.cuh create mode 100644 cpp/include/jcudf/functions/less.cuh create mode 100644 cpp/include/jcudf/functions/less_equal.cuh create mode 100644 cpp/include/jcudf/functions/log.cuh create mode 100644 cpp/include/jcudf/functions/logical_and.cuh create mode 100644 cpp/include/jcudf/functions/logical_or.cuh create mode 100644 cpp/include/jcudf/functions/mod.cuh create mode 100644 cpp/include/jcudf/functions/mul.cuh create mode 100644 cpp/include/jcudf/functions/null_equal.cuh create mode 100644 cpp/include/jcudf/functions/null_logical_and.cuh create mode 100644 cpp/include/jcudf/functions/null_logical_or.cuh create mode 100644 cpp/include/jcudf/functions/pow.cuh create mode 100644 cpp/include/jcudf/functions/pymod.cuh create mode 100644 cpp/include/jcudf/functions/rint.cuh create mode 100644 cpp/include/jcudf/functions/sin.cuh create mode 100644 cpp/include/jcudf/functions/sinh.cuh create mode 100644 cpp/include/jcudf/functions/sub.cuh create mode 100644 cpp/include/jcudf/functions/tanh.cuh diff --git a/cpp/include/jcudf/functions.cuh b/cpp/include/jcudf/functions.cuh deleted file mode 100644 index 61053d24ed40..000000000000 --- a/cpp/include/jcudf/functions.cuh +++ /dev/null @@ -1,518 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace JCUDF_EXPORT jcudf { -namespace functions { - -#define UNOP_T(op, R, T, expr) \ - __device__ __forceinline__ void op(R* out, T const& a) { *out = expr; } \ - \ - __device__ __forceinline__ void op(optional* out, optional const& a) \ - { \ - if (a.has_value()) { \ - R r; \ - op(&r, *a); \ - *out = r; \ - } else { \ - *out = nullopt; \ - } \ - } - -#define BINOP_T(op, R, T, expr) \ - __device__ __forceinline__ void op(R* out, T const& a, T const& b) { *out = expr; } \ - \ - __device__ __forceinline__ void op(optional* out, optional const& a, optional const& b) \ - { \ - if (a.has_value() && b.has_value()) { \ - R r; \ - op(&r, *a, *b); \ - *out = r; \ - } else { \ - *out = nullopt; \ - } \ - } - -#define UNOP(op, T, expr) UNOP_T(op, T, T, expr) -#define BINOP(op, T, expr) BINOP_T(op, T, T, expr) - -#define ADD_OP(T) BINOP_T(add, T, T, (a + b)) -#define SUB_OP(T) BINOP_T(sub, T, T, (a - b)) -#define MUL_OP(T) BINOP_T(mul, T, T, (a * b)) -#define DIV_OP(T) BINOP_T(div, T, T, (a / b)) -#define EQ_OP(T) BINOP_T(equal, bool, T, (a == b)) -#define LT_OP(T) BINOP_T(less, bool, T, (a < b)) -#define GT_OP(T) BINOP_T(greater, bool, T, (a > b)) -#define LE_OP(T) BINOP_T(less_equal, bool, T, (a <= b)) -#define GE_OP(T) BINOP_T(greater_equal, bool, T, (a >= b)) -#define BIT_AND_OP(T) BINOP_T(bitwise_and, T, T, (a & b)) -#define BIT_OR_OP(T) BINOP_T(bitwise_or, T, T, (a | b)) -#define BIT_XOR_OP(T) BINOP_T(bitwise_xor, T, T, (a ^ b)) -#define LOGICAL_AND_OP(T) BINOP_T(logical_and, bool, T, (a && b)) -#define LOGICAL_OR_OP(T) BINOP_T(logical_or, bool, T, (a || b)) -#define IDENTITY_OP(T) UNOP_T(identity, T, T, a) -#define BIT_INVERT_OP(T) UNOP_T(bit_invert, T, T, ~a) -#define CAST_OP(out_type, in_type) UNOP_T(cast_to_##out_type, out_type, in_type, (out_type)(a)) -#define ABS_OP(T) UNOP_T(abs, T, T, ((a < 0) ? -a : a)) -#define MOD_OP(T) BINOP_T(mod, T, T, (a % b)) -#define PYMOD_OP(T) BINOP_T(pymod, T, T, ((a % b + b) % b)) - -#define NULL_EQ_OP(T) \ - __device__ __forceinline__ void null_equal(bool* out, T const& a, T const& b) { *out = a == b; } \ - \ - __device__ __forceinline__ void null_equal( \ - optional* out, optional const& a, optional const& b) \ - { \ - if (a.has_value() && b.has_value()) { \ - *out = (*a == *b); \ - } else if (a.has_null() && b.has_null()) { \ - *out = true; \ - } else { \ - *out = false; \ - } \ - } - -#define IS_NULL_OP(T) \ - __device__ __forceinline__ void is_null(bool* out, T const& a) { *out = false; } \ - __device__ __forceinline__ void is_null(optional* out, optional const& a) \ - { \ - *out = a.has_null(); \ - } - -ADD_OP(i32) -ADD_OP(i64) -ADD_OP(u32) -ADD_OP(u64) -ADD_OP(f32) -ADD_OP(f64) -ADD_OP(dec32) -ADD_OP(dec64) -ADD_OP(dec128) -ADD_OP(duration_D) -ADD_OP(duration_s) -ADD_OP(duration_ms) -ADD_OP(duration_ns) - -SUB_OP(i32) -SUB_OP(i64) -SUB_OP(u32) -SUB_OP(u64) -SUB_OP(f32) -SUB_OP(f64) -SUB_OP(dec32) -SUB_OP(dec64) -SUB_OP(dec128) -SUB_OP(duration_D) -SUB_OP(duration_s) -SUB_OP(duration_ms) -SUB_OP(duration_ns) - -MUL_OP(i32) -MUL_OP(i64) -MUL_OP(u32) -MUL_OP(u64) -MUL_OP(f32) -MUL_OP(f64) -MUL_OP(dec32) -MUL_OP(dec64) -MUL_OP(dec128) - -DIV_OP(i32) -DIV_OP(i64) -DIV_OP(u32) -DIV_OP(u64) -DIV_OP(f32) -DIV_OP(f64) -DIV_OP(dec32) -DIV_OP(dec64) -DIV_OP(dec128) - -EQ_OP(bool) -EQ_OP(i8) -EQ_OP(i16) -EQ_OP(i32) -EQ_OP(i64) -EQ_OP(u8) -EQ_OP(u16) -EQ_OP(u32) -EQ_OP(u64) -EQ_OP(f32) -EQ_OP(f64) -EQ_OP(dec32) -EQ_OP(dec64) -EQ_OP(dec128) -EQ_OP(timestamp_D) -EQ_OP(timestamp_s) -EQ_OP(timestamp_ms) -EQ_OP(timestamp_us) -EQ_OP(timestamp_ns) -EQ_OP(duration_D) -EQ_OP(duration_s) -EQ_OP(duration_ms) -EQ_OP(duration_ns) -EQ_OP(string_view) - -LT_OP(bool) -LT_OP(i8) -LT_OP(i16) -LT_OP(i32) -LT_OP(i64) -LT_OP(u8) -LT_OP(u16) -LT_OP(u32) -LT_OP(u64) -LT_OP(f32) -LT_OP(f64) -LT_OP(dec32) -LT_OP(dec64) -LT_OP(dec128) -LT_OP(timestamp_D) -LT_OP(timestamp_s) -LT_OP(timestamp_ms) -LT_OP(timestamp_us) -LT_OP(timestamp_ns) -LT_OP(duration_D) -LT_OP(duration_s) -LT_OP(duration_ms) -LT_OP(duration_ns) -LT_OP(string_view) - -GT_OP(bool) -GT_OP(i8) -GT_OP(i16) -GT_OP(i32) -GT_OP(i64) -GT_OP(u8) -GT_OP(u16) -GT_OP(u32) -GT_OP(u64) -GT_OP(f32) -GT_OP(f64) -GT_OP(dec32) -GT_OP(dec64) -GT_OP(dec128) -GT_OP(timestamp_D) -GT_OP(timestamp_s) -GT_OP(timestamp_ms) -GT_OP(timestamp_us) -GT_OP(timestamp_ns) -GT_OP(duration_D) -GT_OP(duration_s) -GT_OP(duration_ms) -GT_OP(duration_ns) -GT_OP(string_view) - -LE_OP(bool) -LE_OP(i8) -LE_OP(i16) -LE_OP(i32) -LE_OP(i64) -LE_OP(u8) -LE_OP(u16) -LE_OP(u32) -LE_OP(u64) -LE_OP(f32) -LE_OP(f64) -LE_OP(dec32) -LE_OP(dec64) -LE_OP(dec128) -LE_OP(timestamp_D) -LE_OP(timestamp_s) -LE_OP(timestamp_ms) -LE_OP(timestamp_us) -LE_OP(timestamp_ns) -LE_OP(duration_D) -LE_OP(duration_s) -LE_OP(duration_ms) -LE_OP(duration_ns) -LE_OP(string_view) - -GE_OP(bool) -GE_OP(i8) -GE_OP(i16) -GE_OP(i32) -GE_OP(i64) -GE_OP(u8) -GE_OP(u16) -GE_OP(u32) -GE_OP(u64) -GE_OP(f32) -GE_OP(f64) -GE_OP(dec32) -GE_OP(dec64) -GE_OP(dec128) -GE_OP(timestamp_D) -GE_OP(timestamp_s) -GE_OP(timestamp_ms) -GE_OP(timestamp_us) -GE_OP(timestamp_ns) -GE_OP(duration_D) -GE_OP(duration_s) -GE_OP(duration_ms) -GE_OP(duration_ns) -GE_OP(string_view) - -BIT_AND_OP(i32) -BIT_AND_OP(i64) -BIT_AND_OP(u32) -BIT_AND_OP(u64) - -BIT_OR_OP(i32) -BIT_OR_OP(i64) -BIT_OR_OP(u32) -BIT_OR_OP(u64) - -BIT_XOR_OP(i32) -BIT_XOR_OP(i64) -BIT_XOR_OP(u32) -BIT_XOR_OP(u64) - -LOGICAL_AND_OP(bool) - -LOGICAL_OR_OP(bool) - -IDENTITY_OP(bool) -IDENTITY_OP(i8) -IDENTITY_OP(i16) -IDENTITY_OP(i32) -IDENTITY_OP(i64) -IDENTITY_OP(u8) -IDENTITY_OP(u16) -IDENTITY_OP(u32) -IDENTITY_OP(u64) -IDENTITY_OP(f32) -IDENTITY_OP(f64) -IDENTITY_OP(dec32) -IDENTITY_OP(dec64) -IDENTITY_OP(dec128) -IDENTITY_OP(timestamp_D) -IDENTITY_OP(timestamp_s) -IDENTITY_OP(timestamp_ms) -IDENTITY_OP(timestamp_us) -IDENTITY_OP(timestamp_ns) -IDENTITY_OP(duration_D) -IDENTITY_OP(duration_s) -IDENTITY_OP(duration_ms) -IDENTITY_OP(duration_ns) -IDENTITY_OP(string_view) - -BIT_INVERT_OP(u32) -BIT_INVERT_OP(u64) -BIT_INVERT_OP(i32) -BIT_INVERT_OP(i64) - -CAST_OP(i64, bool) -CAST_OP(i64, i8) -CAST_OP(i64, i16) -CAST_OP(i64, i32) -CAST_OP(i64, i64) -CAST_OP(i64, u8) -CAST_OP(i64, u16) -CAST_OP(i64, u32) -CAST_OP(i64, u64) -CAST_OP(i64, f32) -CAST_OP(i64, f64) - -CAST_OP(u64, bool) -CAST_OP(u64, i8) -CAST_OP(u64, i16) -CAST_OP(u64, i32) -CAST_OP(u64, i64) -CAST_OP(u64, u8) -CAST_OP(u64, u16) -CAST_OP(u64, u32) -CAST_OP(u64, u64) -CAST_OP(u64, f32) -CAST_OP(u64, f64) - -CAST_OP(f64, bool) -CAST_OP(f64, i8) -CAST_OP(f64, i16) -CAST_OP(f64, i32) -CAST_OP(f64, i64) -CAST_OP(f64, u8) -CAST_OP(f64, u16) -CAST_OP(f64, u32) -CAST_OP(f64, u64) -CAST_OP(f64, f32) -CAST_OP(f64, f64) - -IS_NULL_OP(bool) -IS_NULL_OP(i8) -IS_NULL_OP(i16) -IS_NULL_OP(i32) -IS_NULL_OP(i64) -IS_NULL_OP(u8) -IS_NULL_OP(u16) -IS_NULL_OP(u32) -IS_NULL_OP(u64) -IS_NULL_OP(f32) -IS_NULL_OP(f64) -IS_NULL_OP(dec32) -IS_NULL_OP(dec64) -IS_NULL_OP(dec128) -IS_NULL_OP(timestamp_D) -IS_NULL_OP(timestamp_s) -IS_NULL_OP(timestamp_ms) -IS_NULL_OP(timestamp_us) -IS_NULL_OP(timestamp_ns) -IS_NULL_OP(duration_D) -IS_NULL_OP(duration_s) -IS_NULL_OP(duration_ms) -IS_NULL_OP(duration_ns) -IS_NULL_OP(string_view) - -ABS_OP(i8) -ABS_OP(i16) -ABS_OP(i32) -ABS_OP(i64) -ABS_OP(f32) -ABS_OP(f64) - -UNOP(sin, f32, (__builtin_sinf(a))) -UNOP(sin, f64, (__builtin_sin(a))) - -UNOP(cos, f32, (__builtin_cosf(a))) -UNOP(cos, f64, (__builtin_cos(a))) - -UNOP(tan, f32, (__builtin_tanf(a))) -UNOP(tan, f64, (__builtin_tan(a))) - -UNOP(arcsin, f32, (__builtin_asinf(a))) -UNOP(arcsin, f64, (__builtin_asin(a))) - -UNOP(arccos, f32, (__builtin_acosf(a))) -UNOP(arccos, f64, (__builtin_acos(a))) - -UNOP(arctan, f32, (__builtin_atanf(a))) -UNOP(arctan, f64, (__builtin_atan(a))) - -UNOP(sinh, f32, (__builtin_sinhf(a))) -UNOP(sinh, f64, (__builtin_sinh(a))) - -UNOP(cosh, f32, (__builtin_coshf(a))) -UNOP(cosh, f64, (__builtin_cosh(a))) - -UNOP(tanh, f32, (__builtin_tanhf(a))) -UNOP(tanh, f64, (__builtin_tanh(a))) - -UNOP(arcsinh, f32, (__builtin_asinhf(a))) -UNOP(arcsinh, f64, (__builtin_asinh(a))) - -UNOP(arccosh, f32, (__builtin_acoshf(a))) -UNOP(arccosh, f64, (__builtin_acosh(a))) - -UNOP(arctanh, f32, (__builtin_atanhf(a))) -UNOP(arctanh, f64, (__builtin_atanh(a))) - -UNOP(exp, f32, (__builtin_expf(a))) -UNOP(exp, f64, (__builtin_exp(a))) - -UNOP(log, f32, (__builtin_logf(a))) -UNOP(log, f64, (__builtin_log(a))) - -UNOP(cbrt, f32, (__builtin_cbrtf(a))) -UNOP(cbrt, f64, (__builtin_cbrt(a))) - -UNOP(ceil, f32, (__builtin_ceilf(a))) -UNOP(ceil, f64, (__builtin_ceil(a))) - -UNOP(floor, f32, (__builtin_floorf(a))) -UNOP(floor, f64, (__builtin_floor(a))) - -UNOP(rint, f32, (__builtin_rintf(a))) -UNOP(rint, f64, (__builtin_rint(a))) - -MOD_OP(i32) -MOD_OP(i64) -MOD_OP(u32) -MOD_OP(u64) -BINOP(mod, f32, (__builtin_fmodf(a, b))) -BINOP(mod, f64, (__builtin_fmod(a, b))) - -PYMOD_OP(i32) -PYMOD_OP(i64) -PYMOD_OP(u32) -PYMOD_OP(u64) -BINOP(pymod, f32, (__builtin_fmodf(__builtin_fmodf(a, b) + b, b))) -BINOP(pymod, f64, (__builtin_fmod(__builtin_fmod(a, b) + b, b))) - -BINOP(pow, f32, (__builtin_powf(a, b))) -BINOP(pow, f64, (__builtin_pow(a, b))) - -NULL_EQ_OP(bool) -NULL_EQ_OP(i8) -NULL_EQ_OP(i16) -NULL_EQ_OP(i32) -NULL_EQ_OP(i64) -NULL_EQ_OP(u8) -NULL_EQ_OP(u16) -NULL_EQ_OP(u32) -NULL_EQ_OP(u64) -NULL_EQ_OP(f32) -NULL_EQ_OP(f64) -NULL_EQ_OP(dec32) -NULL_EQ_OP(dec64) -NULL_EQ_OP(dec128) -NULL_EQ_OP(timestamp_D) -NULL_EQ_OP(timestamp_s) -NULL_EQ_OP(timestamp_ms) -NULL_EQ_OP(timestamp_us) -NULL_EQ_OP(timestamp_ns) -NULL_EQ_OP(duration_D) -NULL_EQ_OP(duration_s) -NULL_EQ_OP(duration_ms) -NULL_EQ_OP(duration_ns) -NULL_EQ_OP(string_view) - -__device__ __forceinline__ void null_logical_and(bool* out, bool const& a, bool const& b) -{ - *out = a && b; -} - -__device__ __forceinline__ void null_logical_and(optional* out, - optional const& a, - optional const& b) -{ - if (a.has_value() && b.has_value()) { - *out = (*a && *b); - } else if (a.has_null() && b.has_null()) { - *out = nullopt; - } else { - bool valid = a.has_value() ? *a : *b; - if (valid) { - *out = nullopt; - } else { - *out = false; - } - } -} - -__device__ __forceinline__ void null_logical_or(bool* out, bool const& a, bool const& b) -{ - *out = a || b; -} - -__device__ __forceinline__ void null_logical_or(optional* out, - optional const& a, - optional const& b) -{ - if (a.has_value() && b.has_value()) { - *out = (*a || *b); - } else if (a.has_null() && b.has_null()) { - *out = nullopt; - } else { - bool valid = a.has_value() ? *a : *b; - if (valid) { - *out = true; - } else { - *out = nullopt; - } - } -} - -} // namespace functions -} // namespace JCUDF_EXPORT jcudf diff --git a/cpp/include/jcudf/functions.def.cuh b/cpp/include/jcudf/functions.def.cuh deleted file mode 100644 index b1fe86d98606..000000000000 --- a/cpp/include/jcudf/functions.def.cuh +++ /dev/null @@ -1,11 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include - -namespace JCUDF_EXPORT jcudf { - -} // namespace JCUDF_EXPORT jcudf diff --git a/cpp/include/jcudf/functions/abs.cuh b/cpp/include/jcudf/functions/abs.cuh new file mode 100644 index 000000000000..15bd02d8f865 --- /dev/null +++ b/cpp/include/jcudf/functions/abs.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void abs(T* out, T const& a) +{ + *out = (a < 0) ? -a : a; +} + +template +__device__ inline void abs(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + abs(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/add.cuh b/cpp/include/jcudf/functions/add.cuh new file mode 100644 index 000000000000..52a4f4c4dc76 --- /dev/null +++ b/cpp/include/jcudf/functions/add.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void add(T* out, T const& a, T const& b) +{ + *out = a + b; +} + +template +__device__ inline void add(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + add(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/arccos.cuh b/cpp/include/jcudf/functions/arccos.cuh new file mode 100644 index 000000000000..8738a5ae1821 --- /dev/null +++ b/cpp/include/jcudf/functions/arccos.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void arccos(f32* out, f32 const& a) { *out = __builtin_acosf(a); } + +__device__ inline void arccos(f64* out, f64 const& a) { *out = __builtin_acos(a); } + +template +__device__ inline void arccos(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + arccos(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/arccosh.cuh b/cpp/include/jcudf/functions/arccosh.cuh new file mode 100644 index 000000000000..5c7d8def25b8 --- /dev/null +++ b/cpp/include/jcudf/functions/arccosh.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void arccosh(f32* out, f32 const& a) { *out = __builtin_acoshf(a); } + +__device__ inline void arccosh(f64* out, f64 const& a) { *out = __builtin_acosh(a); } + +template +__device__ inline void arccosh(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + arccosh(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/arcsin.cuh b/cpp/include/jcudf/functions/arcsin.cuh new file mode 100644 index 000000000000..d37b98fc9589 --- /dev/null +++ b/cpp/include/jcudf/functions/arcsin.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void arcsin(f32* out, f32 const& a) { *out = __builtin_asinf(a); } + +__device__ inline void arcsin(f64* out, f64 const& a) { *out = __builtin_asin(a); } + +template +__device__ inline void arcsin(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + arcsin(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/arcsinh.cuh b/cpp/include/jcudf/functions/arcsinh.cuh new file mode 100644 index 000000000000..e4cb3d9f6be0 --- /dev/null +++ b/cpp/include/jcudf/functions/arcsinh.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void arcsinh(f32* out, f32 const& a) { *out = __builtin_asinhf(a); } + +__device__ inline void arcsinh(f64* out, f64 const& a) { *out = __builtin_asinh(a); } + +template +__device__ inline void arcsinh(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + arcsinh(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/arctan.cuh b/cpp/include/jcudf/functions/arctan.cuh new file mode 100644 index 000000000000..bf4740b18816 --- /dev/null +++ b/cpp/include/jcudf/functions/arctan.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void arctan(f32* out, f32 const& a) { *out = __builtin_atanf(a); } + +__device__ inline void arctan(f64* out, f64 const& a) { *out = __builtin_atan(a); } + +template +__device__ inline void arctan(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + arctan(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/arctanh.cuh b/cpp/include/jcudf/functions/arctanh.cuh new file mode 100644 index 000000000000..bdca05c9d252 --- /dev/null +++ b/cpp/include/jcudf/functions/arctanh.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void arctanh(f32* out, f32 const& a) { *out = __builtin_atanhf(a); } + +__device__ inline void arctanh(f64* out, f64 const& a) { *out = __builtin_atanh(a); } + +template +__device__ inline void arctanh(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + arctanh(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/bit_and.cuh b/cpp/include/jcudf/functions/bit_and.cuh new file mode 100644 index 000000000000..ecb1bff50a10 --- /dev/null +++ b/cpp/include/jcudf/functions/bit_and.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void bit_and(T* out, T const& a, T const& b) +{ + *out = a & b; +} + +template +__device__ inline void bit_and(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + bit_and(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/bit_invert.cuh b/cpp/include/jcudf/functions/bit_invert.cuh new file mode 100644 index 000000000000..6344c04a53ab --- /dev/null +++ b/cpp/include/jcudf/functions/bit_invert.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void bit_invert(T* out, T const& a) +{ + *out = ~a; +} + +template +__device__ inline void bit_invert(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + bit_invert(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/bit_or.cuh b/cpp/include/jcudf/functions/bit_or.cuh new file mode 100644 index 000000000000..4fcf7e90a620 --- /dev/null +++ b/cpp/include/jcudf/functions/bit_or.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void bit_or(T* out, T const& a, T const& b) +{ + *out = a | b; +} + +template +__device__ inline void bit_or(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + bit_or(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/bit_xor.cuh b/cpp/include/jcudf/functions/bit_xor.cuh new file mode 100644 index 000000000000..05040e029572 --- /dev/null +++ b/cpp/include/jcudf/functions/bit_xor.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void bit_xor(T* out, T const& a, T const& b) +{ + *out = a ^ b; +} + +template +__device__ inline void bit_xor(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + bit_xor(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/cast_to_f64.cuh b/cpp/include/jcudf/functions/cast_to_f64.cuh new file mode 100644 index 000000000000..9bdfa7945cf5 --- /dev/null +++ b/cpp/include/jcudf/functions/cast_to_f64.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void cast_to_f64(f64* out, T const& a) +{ + *out = static_cast(a); +} + +template +__device__ inline void cast_to_f64(optional* out, optional const& a) +{ + if (a.has_value()) { + f64 r; + cast_to_f64(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/cast_to_i64.cuh b/cpp/include/jcudf/functions/cast_to_i64.cuh new file mode 100644 index 000000000000..9b5611254f52 --- /dev/null +++ b/cpp/include/jcudf/functions/cast_to_i64.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void cast_to_i64(i64* out, T const& a) +{ + *out = static_cast(a); +} + +template +__device__ inline void cast_to_i64(optional* out, optional const& a) +{ + if (a.has_value()) { + i64 r; + cast_to_i64(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/cast_to_u64.cuh b/cpp/include/jcudf/functions/cast_to_u64.cuh new file mode 100644 index 000000000000..23d9e7b60fb7 --- /dev/null +++ b/cpp/include/jcudf/functions/cast_to_u64.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void cast_to_u64(u64* out, T const& a) +{ + *out = static_cast(a); +} + +template +__device__ inline void cast_to_u64(optional* out, optional const& a) +{ + if (a.has_value()) { + u64 r; + cast_to_u64(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/cbrt.cuh b/cpp/include/jcudf/functions/cbrt.cuh new file mode 100644 index 000000000000..ac0fac4b4183 --- /dev/null +++ b/cpp/include/jcudf/functions/cbrt.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void cbrt(f32* out, f32 const& a) { *out = __builtin_cbrtf(a); } + +__device__ inline void cbrt(f64* out, f64 const& a) { *out = __builtin_cbrt(a); } + +template +__device__ inline void cbrt(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + cbrt(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/ceil.cuh b/cpp/include/jcudf/functions/ceil.cuh new file mode 100644 index 000000000000..c6a98bda1787 --- /dev/null +++ b/cpp/include/jcudf/functions/ceil.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void ceil(f32* out, f32 const& a) { *out = __builtin_ceilf(a); } + +__device__ inline void ceil(f64* out, f64 const& a) { *out = __builtin_ceil(a); } + +template +__device__ inline void ceil(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + ceil(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/cos.cuh b/cpp/include/jcudf/functions/cos.cuh new file mode 100644 index 000000000000..6fb1ca08905c --- /dev/null +++ b/cpp/include/jcudf/functions/cos.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void cos(f32* out, f32 const& a) { *out = __builtin_cosf(a); } + +__device__ inline void cos(f64* out, f64 const& a) { *out = __builtin_cos(a); } + +template +__device__ inline void cos(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + cos(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/cosh.cuh b/cpp/include/jcudf/functions/cosh.cuh new file mode 100644 index 000000000000..c5144d21bcc3 --- /dev/null +++ b/cpp/include/jcudf/functions/cosh.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void cosh(f32* out, f32 const& a) { *out = __builtin_coshf(a); } + +__device__ inline void cosh(f64* out, f64 const& a) { *out = __builtin_cosh(a); } + +template +__device__ inline void cosh(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + cosh(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/div.cuh b/cpp/include/jcudf/functions/div.cuh new file mode 100644 index 000000000000..f8cd1396a615 --- /dev/null +++ b/cpp/include/jcudf/functions/div.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void div(T* out, T const& a, T const& b) +{ + *out = a / b; +} + +template +__device__ inline void div(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + div(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/equal.cuh b/cpp/include/jcudf/functions/equal.cuh new file mode 100644 index 000000000000..c40aa7874d36 --- /dev/null +++ b/cpp/include/jcudf/functions/equal.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void equal(bool* out, T const& a, T const& b) +{ + *out = a == b; +} + +template +__device__ inline void equal(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + *out = (*a == *b); + } else if (a.has_null() && b.has_null()) { + *out = true; + } else { + *out = false; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/exp.cuh b/cpp/include/jcudf/functions/exp.cuh new file mode 100644 index 000000000000..205f420fc84f --- /dev/null +++ b/cpp/include/jcudf/functions/exp.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void exp(f32* out, f32 const& a) { *out = __builtin_expf(a); } + +__device__ inline void exp(f64* out, f64 const& a) { *out = __builtin_exp(a); } + +template +__device__ inline void exp(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + exp(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/floor.cuh b/cpp/include/jcudf/functions/floor.cuh new file mode 100644 index 000000000000..490fbc53f2dd --- /dev/null +++ b/cpp/include/jcudf/functions/floor.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void floor(f32* out, f32 const& a) { *out = __builtin_floorf(a); } + +__device__ inline void floor(f64* out, f64 const& a) { *out = __builtin_floor(a); } + +template +__device__ inline void floor(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + floor(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/greater.cuh b/cpp/include/jcudf/functions/greater.cuh new file mode 100644 index 000000000000..9a9bd9ffad6b --- /dev/null +++ b/cpp/include/jcudf/functions/greater.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void greater(bool* out, T const& a, T const& b) +{ + *out = a > b; +} + +template +__device__ inline void greater(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + *out = (*a > *b); + } else { + *out = false; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/greater_equal.cuh b/cpp/include/jcudf/functions/greater_equal.cuh new file mode 100644 index 000000000000..d926a7829753 --- /dev/null +++ b/cpp/include/jcudf/functions/greater_equal.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void greater_equal(bool* out, T const& a, T const& b) +{ + *out = a >= b; +} + +template +__device__ inline void greater_equal(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + *out = (*a >= *b); + } else { + *out = false; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/identity.cuh b/cpp/include/jcudf/functions/identity.cuh new file mode 100644 index 000000000000..3e4b867a81ff --- /dev/null +++ b/cpp/include/jcudf/functions/identity.cuh @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void identity(T* out, T const& a) +{ + *out = a; +} + +template +__device__ inline void identity(optional* out, optional const& a) +{ + *out = a; +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/is_null.cuh b/cpp/include/jcudf/functions/is_null.cuh new file mode 100644 index 000000000000..374919c246c5 --- /dev/null +++ b/cpp/include/jcudf/functions/is_null.cuh @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void is_null(bool* out, T const& a) +{ + *out = false; +} + +template +__device__ inline void is_null(optional* out, optional const& a) +{ + *out = a.has_null(); +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/less.cuh b/cpp/include/jcudf/functions/less.cuh new file mode 100644 index 000000000000..0a2d0d7e063b --- /dev/null +++ b/cpp/include/jcudf/functions/less.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void less(bool* out, T const& a, T const& b) +{ + *out = a < b; +} + +template +__device__ inline void less(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + *out = (*a < *b); + } else { + *out = false; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/less_equal.cuh b/cpp/include/jcudf/functions/less_equal.cuh new file mode 100644 index 000000000000..701695c01b8b --- /dev/null +++ b/cpp/include/jcudf/functions/less_equal.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void less_equal(bool* out, T const& a, T const& b) +{ + *out = a <= b; +} + +template +__device__ inline void less_equal(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + *out = (*a <= *b); + } else { + *out = false; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/log.cuh b/cpp/include/jcudf/functions/log.cuh new file mode 100644 index 000000000000..549d7524c9cc --- /dev/null +++ b/cpp/include/jcudf/functions/log.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void log(f32* out, f32 const& a) { *out = __builtin_logf(a); } + +__device__ inline void log(f64* out, f64 const& a) { *out = __builtin_log(a); } + +template +__device__ inline void log(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + log(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/logical_and.cuh b/cpp/include/jcudf/functions/logical_and.cuh new file mode 100644 index 000000000000..185259fdf5ae --- /dev/null +++ b/cpp/include/jcudf/functions/logical_and.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void logical_and(T* out, T const& a, T const& b) +{ + *out = a && b; +} + +template +__device__ inline void logical_and(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + logical_and(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/logical_or.cuh b/cpp/include/jcudf/functions/logical_or.cuh new file mode 100644 index 000000000000..755d5a17ad50 --- /dev/null +++ b/cpp/include/jcudf/functions/logical_or.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void logical_or(T* out, T const& a, T const& b) +{ + *out = a || b; +} + +template +__device__ inline void logical_or(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + logical_or(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/mod.cuh b/cpp/include/jcudf/functions/mod.cuh new file mode 100644 index 000000000000..b85a17230681 --- /dev/null +++ b/cpp/include/jcudf/functions/mod.cuh @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void mod(T* out, T const& a, T const& b) +{ + *out = a % b; +} + +__device__ inline void mod(f32* out, f32 const& a, f32 const& b) { *out = __builtin_fmodf(a, b); } + +__device__ inline void mod(f64* out, f64 const& a, f64 const& b) { *out = __builtin_fmod(a, b); } + +template +__device__ inline void mod(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + mod(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/mul.cuh b/cpp/include/jcudf/functions/mul.cuh new file mode 100644 index 000000000000..65b27b44e9b9 --- /dev/null +++ b/cpp/include/jcudf/functions/mul.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void mul(T* out, T const& a, T const& b) +{ + *out = a * b; +} + +template +__device__ inline void mul(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + mul(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/null_equal.cuh b/cpp/include/jcudf/functions/null_equal.cuh new file mode 100644 index 000000000000..7bc83e40c493 --- /dev/null +++ b/cpp/include/jcudf/functions/null_equal.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void null_equal(bool* out, T const& a, T const& b) +{ + *out = a == b; +} + +template +__device__ inline void null_equal(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + *out = (*a == *b); + } else if (a.has_null() && b.has_null()) { + *out = true; + } else { + *out = false; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/null_logical_and.cuh b/cpp/include/jcudf/functions/null_logical_and.cuh new file mode 100644 index 000000000000..4032f7b44198 --- /dev/null +++ b/cpp/include/jcudf/functions/null_logical_and.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void null_logical_and(bool* out, bool const& a, bool const& b) { *out = a && b; } + +__device__ inline void null_logical_and(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + *out = (*a && *b); + } else if (a.has_null() && b.has_null()) { + *out = nullopt; + } else { + bool valid = a.has_value() ? *a : *b; + if (valid) { + *out = nullopt; + } else { + *out = false; + } + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/null_logical_or.cuh b/cpp/include/jcudf/functions/null_logical_or.cuh new file mode 100644 index 000000000000..74d5b4322889 --- /dev/null +++ b/cpp/include/jcudf/functions/null_logical_or.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void null_logical_or(bool* out, bool const& a, bool const& b) { *out = a || b; } + +__device__ inline void null_logical_or(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + *out = (*a || *b); + } else if (a.has_null() && b.has_null()) { + *out = nullopt; + } else { + bool valid = a.has_value() ? *a : *b; + if (valid) { + *out = true; + } else { + *out = nullopt; + } + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/pow.cuh b/cpp/include/jcudf/functions/pow.cuh new file mode 100644 index 000000000000..7dbf82f296a9 --- /dev/null +++ b/cpp/include/jcudf/functions/pow.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void pow(f32* out, f32 const& a, f32 const& b) { *out = __builtin_powf(a, b); } + +__device__ inline void pow(f64* out, f64 const& a, f64 const& b) { *out = __builtin_pow(a, b); } + +template +__device__ inline void pow(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + pow(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/pymod.cuh b/cpp/include/jcudf/functions/pymod.cuh new file mode 100644 index 000000000000..4ee1ff7a37f6 --- /dev/null +++ b/cpp/include/jcudf/functions/pymod.cuh @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void pymod(T* out, T const& a, T const& b) +{ + *out = (a % b + b) % b; +} + +__device__ inline void pymod(f32* out, f32 const& a, f32 const& b) +{ + *out = __builtin_fmodf(__builtin_fmodf(a, b) + b, b); +} + +__device__ inline void pymod(f64* out, f64 const& a, f64 const& b) +{ + *out = __builtin_fmod(__builtin_fmod(a, b) + b, b); +} + +template +__device__ inline void pymod(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + pymod(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/rint.cuh b/cpp/include/jcudf/functions/rint.cuh new file mode 100644 index 000000000000..dda95e2d81c6 --- /dev/null +++ b/cpp/include/jcudf/functions/rint.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void rint(f32* out, f32 const& a) { *out = __builtin_rintf(a); } + +__device__ inline void rint(f64* out, f64 const& a) { *out = __builtin_rint(a); } + +template +__device__ inline void rint(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + rint(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/sin.cuh b/cpp/include/jcudf/functions/sin.cuh new file mode 100644 index 000000000000..1c8312b8c04f --- /dev/null +++ b/cpp/include/jcudf/functions/sin.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void sin(f32* out, f32 const& a) { *out = __builtin_sinf(a); } + +__device__ inline void sin(f64* out, f64 const& a) { *out = __builtin_sin(a); } + +template +__device__ inline void sin(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + sin(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/sinh.cuh b/cpp/include/jcudf/functions/sinh.cuh new file mode 100644 index 000000000000..b9b5954635fc --- /dev/null +++ b/cpp/include/jcudf/functions/sinh.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void sinh(f32* out, f32 const& a) { *out = __builtin_sinhf(a); } + +__device__ inline void sinh(f64* out, f64 const& a) { *out = __builtin_sinh(a); } + +template +__device__ inline void sinh(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + sinh(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/sub.cuh b/cpp/include/jcudf/functions/sub.cuh new file mode 100644 index 000000000000..45c32c11ba95 --- /dev/null +++ b/cpp/include/jcudf/functions/sub.cuh @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +template +__device__ inline void sub(T* out, T const& a, T const& b) +{ + *out = a - b; +} + +template +__device__ inline void sub(optional* out, optional const& a, optional const& b) +{ + if (a.has_value() && b.has_value()) { + T r; + sub(&r, *a, *b); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/functions/tanh.cuh b/cpp/include/jcudf/functions/tanh.cuh new file mode 100644 index 000000000000..c88cd6089534 --- /dev/null +++ b/cpp/include/jcudf/functions/tanh.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +namespace jcudf { +namespace functions { + +__device__ inline void tanh(f32* out, f32 const& a) { *out = __builtin_tanhf(a); } + +__device__ inline void tanh(f64* out, f64 const& a) { *out = __builtin_tanh(a); } + +template +__device__ inline void tanh(optional* out, optional const& a) +{ + if (a.has_value()) { + T r; + tanh(&r, *a); + *out = r; + } else { + *out = nullopt; + } +} + +} // namespace functions +} // namespace jcudf diff --git a/cpp/include/jcudf/library.cu b/cpp/include/jcudf/library.cu index 42bbac350d09..55f6dc06ebf0 100644 --- a/cpp/include/jcudf/library.cu +++ b/cpp/include/jcudf/library.cu @@ -3,4 +3,3 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include diff --git a/cpp/include/jcudf/types.cuh b/cpp/include/jcudf/types.cuh index 3e63c29739b9..34f587a2b362 100644 --- a/cpp/include/jcudf/types.cuh +++ b/cpp/include/jcudf/types.cuh @@ -4,17 +4,7 @@ */ #pragma once -#if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) - -#define JCUDF_EXPORT __attribute__((visibility("default"))) - -#else - -#define JCUDF_EXPORT - -#endif - -namespace JCUDF_EXPORT jcudf { +namespace jcudf { using i8 = signed char; using i16 = signed short; @@ -42,12 +32,19 @@ using size_type = i32; using bitmask_t = u32; -__device__ constexpr bool bit_is_set(bitmask_t const* bitmask, usize bit_index) +template +__device__ constexpr bool bit_is_set(T const* bitmask, usize bit_index) { - constexpr auto bits_per_word = sizeof(bitmask_t) * 8; - return bitmask[bit_index / bits_per_word] & (bitmask_t{1} << (bit_index % bits_per_word)); + constexpr auto bits_per_word = sizeof(T) * 8; + return bitmask[bit_index / bits_per_word] & (T{1} << (bit_index % bits_per_word)); } +template +inline constexpr bool Same = false; + +template +inline constexpr bool Same = true; + enum class type_id : i32 { EMPTY = 0, INT8 = 1, @@ -155,224 +152,135 @@ struct scaled_t {}; inline constexpr scaled_t scaled; -struct dec32 { - i32 _value = 0; - - i32 _scale = 0; - - __device__ constexpr dec32(scaled_t, i32 value, i32 scale) : _value{value}, _scale{scale} {} - - constexpr dec32() = default; - - __device__ constexpr i32 value() const { return _value; } - - __device__ constexpr i32 scale() const { return _scale; } -}; - -struct dec64 { - i64 _value = 0; - - i32 _scale = 0; - - __device__ constexpr dec64(scaled_t, i64 value, i32 scale) : _value{value}, _scale{scale} {} - - constexpr dec64() = default; - - __device__ constexpr i64 value() const { return _value; } - - __device__ constexpr i32 scale() const { return _scale; } -}; +template +struct dec { + using Rep = R; -struct dec128 { - i128 _value = 0; + R _value = 0; i32 _scale = 0; - __device__ constexpr dec128(scaled_t, i128 value, i32 scale) : _value{value}, _scale{scale} {} + __device__ constexpr dec(scaled_t, R value, i32 scale) : _value{value}, _scale{scale} {} - constexpr dec128() = default; + constexpr dec() = default; - __device__ constexpr i128 value() const { return _value; } + __device__ constexpr R value() const { return _value; } __device__ constexpr i32 scale() const { return _scale; } }; -#define DECIMAL_OPS(T) \ - __device__ constexpr T rescale(T a, i32 scale) \ - { \ - return T{scaled, dec_rescale(a._value, a._scale, scale), scale}; \ - } \ - \ - __device__ constexpr T operator+(T const& a, T const& b) \ - { \ - auto scale = min(a._scale, b._scale); \ - auto r = rescale(a, scale)._value + rescale(b, scale)._value; \ - return T{scaled, r, scale}; \ - } \ - \ - __device__ constexpr T operator-(T const& a, T const& b) \ - { \ - auto scale = min(a._scale, b._scale); \ - auto r = rescale(a, scale)._value - rescale(b, scale)._value; \ - return T{scaled, r, scale}; \ - } \ - \ - __device__ constexpr T operator*(T const& a, T const& b) \ - { \ - return T{scaled, a._value + b._value, a._scale + b._scale}; \ - } \ - \ - __device__ constexpr T operator/(T const& a, T const& b) \ - { \ - return T{scaled, a._value / b._value, a._scale - b._scale}; \ - } \ - \ - __device__ constexpr T operator%(T const& a, T const& b) \ - { \ - auto scale = min(a._scale, b._scale); \ - auto r = rescale(a, scale)._value % rescale(b, scale)._value; \ - return T{scaled, r, scale}; \ - } \ - \ - __device__ constexpr bool operator==(T const& a, T const& b) \ - { \ - auto scale = min(a._scale, b._scale); \ - return rescale(a, scale)._value == rescale(b, scale)._value; \ - } \ - \ - __device__ constexpr bool operator!=(T const& a, T const& b) { return !(a == b); } \ - \ - __device__ constexpr bool operator>(T const& a, T const& b) \ - { \ - auto scale = min(a._scale, b._scale); \ - return rescale(a, scale)._value > rescale(b, scale)._value; \ - } \ - \ - __device__ constexpr bool operator<(T const& a, T const& b) \ - { \ - auto scale = min(a._scale, b._scale); \ - return rescale(a, scale)._value < rescale(b, scale)._value; \ - } \ - \ - __device__ constexpr bool operator>=(T const& a, T const& b) \ - { \ - auto scale = min(a._scale, b._scale); \ - return rescale(a, scale)._value >= rescale(b, scale)._value; \ - } \ - \ - __device__ constexpr bool operator<=(T const& a, T const& b) \ - { \ - auto scale = min(a._scale, b._scale); \ - return rescale(a, scale)._value <= rescale(b, scale)._value; \ - } +using dec32 = dec; +using dec64 = dec; +using dec128 = dec; -DECIMAL_OPS(dec32) -DECIMAL_OPS(dec64) -DECIMAL_OPS(dec128) +template +__device__ constexpr auto rescale(dec a, i32 scale) +{ + return dec{scaled, dec_rescale(a._value, a._scale, scale), scale}; +} -struct timestamp_D { - i32 _rep = 0; -}; +template +__device__ constexpr auto operator+(dec a, dec b) +{ + auto scale = min(a._scale, b._scale); + auto r = rescale(a, scale)._value + rescale(b, scale)._value; + return dec{scaled, r, scale}; +} -struct timestamp_h { - i32 _rep = 0; -}; +template +__device__ constexpr auto operator-(dec a, dec b) +{ + auto scale = min(a._scale, b._scale); + auto r = rescale(a, scale)._value - rescale(b, scale)._value; + return dec{scaled, r, scale}; +} -struct timestamp_m { - i32 _rep = 0; -}; +template +__device__ constexpr auto operator*(dec a, dec b) +{ + return dec{scaled, a._value * b._value, a._scale + b._scale}; +} -struct timestamp_s { - i64 _rep = 0; -}; +template +__device__ constexpr auto operator/(dec a, dec b) +{ + return dec{scaled, a._value / b._value, a._scale - b._scale}; +} -struct timestamp_ms { - i64 _rep = 0; -}; +template +__device__ constexpr auto operator%(dec a, dec b) +{ + auto scale = min(a._scale, b._scale); + auto r = rescale(a, scale)._value % rescale(b, scale)._value; + return dec{scaled, r, scale}; +} -struct timestamp_us { - i64 _rep = 0; -}; +template +__device__ constexpr int operator<=>(dec a, dec b) +{ + auto scale = min(a._scale, b._scale); + return rescale(a, scale)._value - rescale(b, scale)._value; +} -struct timestamp_ns { - i64 _rep = 0; -}; +enum class timestamp_unit : i32 { D, h, m, s, ms, us, ns }; -#define TIMESTAMP_OPS(T) \ - __device__ constexpr bool operator==(T a, T b) { return a._rep == b._rep; } \ - __device__ constexpr bool operator!=(T a, T b) { return a._rep != b._rep; } \ - __device__ constexpr bool operator>(T a, T b) { return a._rep > b._rep; } \ - __device__ constexpr bool operator<(T a, T b) { return a._rep < b._rep; } \ - __device__ constexpr bool operator>=(T a, T b) { return a._rep >= b._rep; } \ - __device__ constexpr bool operator<=(T a, T b) { return a._rep <= b._rep; } - -TIMESTAMP_OPS(timestamp_D) -TIMESTAMP_OPS(timestamp_h) -TIMESTAMP_OPS(timestamp_m) -TIMESTAMP_OPS(timestamp_s) -TIMESTAMP_OPS(timestamp_ms) -TIMESTAMP_OPS(timestamp_us) -TIMESTAMP_OPS(timestamp_ns) - -struct duration_D { - i32 _rep = 0; - - __device__ constexpr auto count() const { return _rep; } -}; +template +struct timestamp { + using Rep = R; -struct duration_h { - i32 _rep = 0; + R _rep = 0; - __device__ constexpr auto count() const { return _rep; } + __device__ constexpr R count() const { return _rep; } }; -struct duration_m { - i32 _rep = 0; - - __device__ constexpr auto count() const { return _rep; } -}; +using timestamp_D = timestamp; +using timestamp_h = timestamp; +using timestamp_m = timestamp; +using timestamp_s = timestamp; +using timestamp_ms = timestamp; +using timestamp_us = timestamp; +using timestamp_ns = timestamp; -struct duration_s { - i64 _rep = 0; +template +__device__ constexpr int operator<=>(timestamp a, timestamp b) +{ + return a._rep - b._rep; +} - __device__ constexpr auto count() const { return _rep; } -}; +template +struct duration { + using Rep = R; -struct duration_ms { - i64 _rep = 0; + R _rep = 0; - __device__ constexpr auto count() const { return _rep; } + __device__ constexpr R count() const { return _rep; } }; -struct duration_us { - i64 _rep = 0; - - __device__ constexpr auto count() const { return _rep; } -}; +using duration_D = duration; +using duration_h = duration; +using duration_m = duration; +using duration_s = duration; +using duration_ms = duration; +using duration_us = duration; +using duration_ns = duration; -struct duration_ns { - i64 _rep = 0; +template +__device__ constexpr duration operator+(duration a, duration b) +{ + return duration{a._rep + b._rep}; +} - __device__ constexpr auto count() const { return _rep; } -}; +template +__device__ constexpr duration operator-(duration a, duration b) +{ + return duration{a._rep - b._rep}; +} -#define DURATION_OPS(T) \ - __device__ constexpr T operator+(T a, T b) { return T{a._rep + b._rep}; } \ - __device__ constexpr T operator-(T a, T b) { return T{a._rep - b._rep}; } \ - __device__ constexpr bool operator==(T a, T b) { return a._rep == b._rep; } \ - __device__ constexpr bool operator!=(T a, T b) { return a._rep != b._rep; } \ - __device__ constexpr bool operator>(T a, T b) { return a._rep > b._rep; } \ - __device__ constexpr bool operator<(T a, T b) { return a._rep < b._rep; } \ - __device__ constexpr bool operator>=(T a, T b) { return a._rep >= b._rep; } \ - __device__ constexpr bool operator<=(T a, T b) { return a._rep <= b._rep; } - -DURATION_OPS(duration_D) -DURATION_OPS(duration_h) -DURATION_OPS(duration_m) -DURATION_OPS(duration_s) -DURATION_OPS(duration_ms) -DURATION_OPS(duration_us) -DURATION_OPS(duration_ns) +template +__device__ constexpr int operator<=>(duration a, duration b) +{ + return a._rep - b._rep; +} struct string_view { static constexpr size_type const UNKNOWN_STRING_LENGTH{-1}; @@ -416,34 +324,9 @@ struct string_view { } }; -__device__ constexpr bool operator==(string_view const& a, string_view const& b) +__device__ constexpr int operator<=>(string_view const& a, string_view const& b) { - return a.compare(b) == 0; -} - -__device__ constexpr bool operator!=(string_view const& a, string_view const& b) -{ - return a.compare(b) != 0; -} - -__device__ constexpr bool operator<(string_view const& a, string_view const& b) -{ - return a.compare(b) < 0; -} - -__device__ constexpr bool operator>(string_view const& a, string_view const& b) -{ - return a.compare(b) > 0; -} - -__device__ constexpr bool operator<=(string_view const& a, string_view const& b) -{ - return a.compare(b) <= 0; -} - -__device__ constexpr bool operator>=(string_view const& a, string_view const& b) -{ - return a.compare(b) >= 0; + return a.compare(b); } struct inplace_t {}; @@ -502,40 +385,6 @@ struct optional { template optional(T) -> optional; -#define INST(T) template struct optional; - -INST(bool); -INST(i8); -INST(i16); -INST(i32); -INST(i64); -INST(u8); -INST(u16); -INST(u32); -INST(u64); -INST(f32); -INST(f64); -INST(timestamp_D); -INST(timestamp_h); -INST(timestamp_m); -INST(timestamp_s); -INST(timestamp_ms); -INST(timestamp_us); -INST(timestamp_ns); -INST(duration_D); -INST(duration_h); -INST(duration_m); -INST(duration_s); -INST(duration_ms); -INST(duration_us); -INST(duration_ns); -INST(dec32); -INST(dec64); -INST(dec128); -INST(string_view); - -#undef INST - template struct span { T* _data = nullptr; @@ -556,44 +405,32 @@ struct span { __device__ constexpr span as_const() const { return span{_data, _size}; } - __device__ constexpr T& element(usize idx) const { return _data[idx]; } + template + __device__ constexpr T& element(usize idx) const + { + return _data[idx]; + } + + __device__ constexpr bool nullable() const { return false; } + + __device__ constexpr bool is_valid_nocheck(usize element_index) const { return true; } + + __device__ constexpr bool is_valid(usize element_index) const { return true; } + + __device__ constexpr bool is_null(usize element_index) const { return false; } + + template + __device__ constexpr optional nullable_element(usize idx) const + { + if (!is_valid(idx)) return nullopt; + return element(idx); + } __device__ constexpr void assign(usize idx, T value) const { _data[idx] = value; } }; -#define INST(T) template struct span; - -INST(bool); -INST(i8); -INST(i16); -INST(i32); -INST(i64); -INST(u8); -INST(u16); -INST(u32); -INST(u64); -INST(f32); -INST(f64); -INST(timestamp_D); -INST(timestamp_h); -INST(timestamp_m); -INST(timestamp_s); -INST(timestamp_ms); -INST(timestamp_us); -INST(timestamp_ns); -INST(duration_D); -INST(duration_h); -INST(duration_m); -INST(duration_s); -INST(duration_ms); -INST(duration_us); -INST(duration_ns); -INST(dec32); -INST(dec64); -INST(dec128); -INST(string_view); - -#undef INST +template +span(T*, usize) -> span; template struct optional_span { @@ -634,48 +471,42 @@ struct optional_span { __device__ constexpr bool is_null(usize element_index) const { return !is_valid(element_index); } - __device__ constexpr T& element(usize idx) const { return _data[idx]; } + template + __device__ constexpr T& element(usize idx) const + { + return _data[idx]; + } - __device__ constexpr optional nullable_element(usize idx) const; + template + __device__ constexpr optional nullable_element(usize idx) const + { + if (!is_valid(idx)) return nullopt; + return element(idx); + } - __device__ constexpr void assign(usize idx, T value) const { _data[idx] = value; } + template + __device__ constexpr void assign(usize idx, T value) const + { + _data[idx] = value; + } }; -#define INST(T) template struct optional_span; - -INST(bool); -INST(i8); -INST(i16); -INST(i32); -INST(i64); -INST(u8); -INST(u16); -INST(u32); -INST(u64); -INST(f32); -INST(f64); -INST(timestamp_D); -INST(timestamp_h); -INST(timestamp_m); -INST(timestamp_s); -INST(timestamp_ms); -INST(timestamp_us); -INST(timestamp_ns); -INST(duration_D); -INST(duration_h); -INST(duration_m); -INST(duration_s); -INST(duration_ms); -INST(duration_us); -INST(duration_ns); -INST(dec32); -INST(dec64); -INST(dec128); -INST(string_view); - -#undef INST - -struct alignas(16) column_device_view { +template +optional_span(T*, usize, bitmask_t const*) -> optional_span; + +struct alignas(16) column_view { + template + static constexpr bool HasSpanLayout = + Same || Same || Same || Same || Same || Same || + Same || Same || Same || Same || Same || + Same || Same || Same || Same || + Same || Same || Same || + Same || Same || Same || Same || + Same || Same || Same; + + template + static constexpr bool HasDecimalLayout = Same || Same || Same; + data_type _type = {}; size_type _size = 0; @@ -686,7 +517,7 @@ struct alignas(16) column_device_view { size_type _offset = 0; - column_device_view* _d_children = nullptr; + column_view* _d_children = nullptr; size_type _num_children = 0; @@ -717,139 +548,56 @@ struct alignas(16) column_device_view { __device__ constexpr size_type num_child_columns() const { return _num_children; } template - __device__ T element(size_type idx) const; - - template - __device__ optional nullable_element(size_type idx) const + __device__ auto& element(size_type idx) const + requires(HasSpanLayout) { - if (!is_valid(idx)) return nullopt; - return element(idx); - } -}; - -#define SPEC(T) \ - template <> \ - __device__ T column_device_view::element(size_type idx) const \ - { \ - return static_cast(_data)[_offset + idx]; \ + return static_cast(_data)[_offset + idx]; } -SPEC(bool) -SPEC(i8) -SPEC(i16) -SPEC(i32) -SPEC(i64) -SPEC(u8) -SPEC(u16) -SPEC(u32) -SPEC(u64) -SPEC(f32) -SPEC(f64) -SPEC(timestamp_D) -SPEC(timestamp_h) -SPEC(timestamp_m) -SPEC(timestamp_s) -SPEC(timestamp_ms) -SPEC(timestamp_us) -SPEC(timestamp_ns) -SPEC(duration_D) -SPEC(duration_h) -SPEC(duration_m) -SPEC(duration_s) -SPEC(duration_ms) -SPEC(duration_us) -SPEC(duration_ns) - -#undef SPEC - -#define SPEC(T, Repr) \ - template <> \ - __device__ T column_device_view::element(size_type idx) const \ - { \ - return T{scaled, static_cast(_data)[_offset + idx], _type.scale()}; \ + template + __device__ auto element(size_type idx) const + requires(HasDecimalLayout) + { + return T{scaled, static_cast(_data)[_offset + idx], _type.scale()}; } -SPEC(dec32, i32) -SPEC(dec64, i64) -SPEC(dec128, i128) + template + __device__ string_view element(size_type idx) const + requires(Same) + { + static constexpr i32 OFFSETS_CHILD = 0; + auto i = _offset + idx; + auto* str_data = static_cast(_data); + auto& offsets = _d_children[OFFSETS_CHILD]; + auto* i32_runs = static_cast(offsets._data); + auto* i64_runs = static_cast(offsets._data); + + i64 run_begin = 0; + i64 run_end = 0; + + switch (offsets.type().id()) { + case type_id::INT32: + run_begin = i32_runs[i]; + run_end = i32_runs[i + 1]; + break; + case type_id::INT64: + run_begin = i64_runs[i]; + run_end = i64_runs[i + 1]; + break; + default: __builtin_unreachable(); + } -#undef SPEC + i64 run_size = run_end - run_begin; -template <> -__device__ string_view column_device_view::element(size_type idx) const -{ - static constexpr i32 OFFSETS_CHILD = 0; - auto i = _offset + idx; - auto* str_data = static_cast(_data); - auto& offsets = _d_children[OFFSETS_CHILD]; - auto* i32_runs = static_cast(offsets._data); - auto* i64_runs = static_cast(offsets._data); - - i64 run_begin = 0; - i64 run_end = 0; - - switch (offsets.type().id()) { - case type_id::INT32: - run_begin = i32_runs[i]; - run_end = i32_runs[i + 1]; - break; - case type_id::INT64: - run_begin = i64_runs[i]; - run_end = i64_runs[i + 1]; - break; - default: __builtin_unreachable(); + return string_view{str_data + run_begin, static_cast(run_size)}; } - i64 run_size = run_end - run_begin; - - return string_view{str_data + run_begin, static_cast(run_size)}; -} - -#define INST(T) \ - template __device__ optional column_device_view::nullable_element(size_type idx) const; - -INST(bool) -INST(i8) -INST(i16) -INST(i32) -INST(i64) -INST(u8) -INST(u16) -INST(u32) -INST(u64) -INST(f32) -INST(f64) -INST(timestamp_D) -INST(timestamp_h) -INST(timestamp_m) -INST(timestamp_s) -INST(timestamp_ms) -INST(timestamp_us) -INST(timestamp_ns) -INST(duration_D) -INST(duration_h) -INST(duration_m) -INST(duration_s) -INST(duration_ms) -INST(duration_us) -INST(duration_ns) -INST(dec32) -INST(dec64) -INST(dec128) -INST(string_view) - -#undef INST - -/// @brief Type-erased parameters for LTO-JIT-compiled transform operations. -struct transform_operator_params { - /// @brief Pointer to scope data (e.g. column views, scalars, etc.). - void* const* scope = nullptr; - - /// @brief Total number of rows to process. - size_type num_rows = 0; - - /// @brief Current row index. - size_type row_index = 0; + template + __device__ optional nullable_element(size_type idx) const + { + if (!is_valid(idx)) return nullopt; + return element(idx); + } }; // TODO: scope variables should be aligned to avoid uncoalesced reads/writes @@ -858,9 +606,7 @@ namespace scope { using args = void* const*; template @@ -921,4 +667,4 @@ struct user_data { }; } // namespace scope -} // namespace JCUDF_EXPORT jcudf +} // namespace jcudf diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index d222ed69793f..e4c4bf121357 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -331,10 +331,10 @@ int32_t get_current_device_physical_model() return props.major * 10 + props.minor; } -rtc::fragment compile_udf_uncached(char const* name, - char const* cuda_code, - bool use_pch, - bool log_pch) +rtc::fragment compile_udf_to_lto_ir_uncached(char const* name, + char const* cuda_code, + bool use_pch, + bool log_pch) { CUDF_FUNC_RANGE(); @@ -404,9 +404,95 @@ rtc::fragment compile_udf_uncached(char const* name, return frag; } -std::tuple link_udf_uncached(char const* name, - rtc::fragment const& fragment, - char const* kernel_symbol) +std::tuple compile_kernel_to_cubin_uncached(char const* name, + char const* cuda_code, + bool use_pch, + bool log_pch) +{ + CUDF_FUNC_RANGE(); + + auto& bundle = cudf::get_context().jit_bundle(); + auto begin = std::chrono::steady_clock::now(); + auto sm = get_current_device_physical_model(); + + auto include_dirs = bundle.get_include_directories(); + auto compile_options = bundle.get_compile_options(); + auto pch_dir = cudf::get_context().get_jit_pch_dir(); + + std::vector options; + + for (auto const& include_dir : include_dirs) { + options.emplace_back(std::format("-I{}", include_dir)); + } + + for (auto const& compile_option : compile_options) { + options.emplace_back(compile_option); + } + + // TODO: experiment with: + // --fdevice-time-trace=jit_comp_trace.json + // --time=compile_trace.json + // -time + + options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); + options.emplace_back("--restrict"); + options.emplace_back("--minimal"); + options.emplace_back("--split-compile=0"); + + + auto pch_file = std::format("/home/coder/cudf/jit.pch"); + + static int can_use = 0; + + if (use_pch) { + // options.emplace_back("--pch"); + options.emplace_back(std::format("--pch-dir={}", pch_dir)); + if(can_use){ + options.emplace_back(std::format("--use-pch={}", pch_file)); + } else{ + options.emplace_back(std::format("--create-pch={}", pch_file)); + } + can_use = 1; + + if (log_pch) { + options.emplace_back("--pch-verbose=true"); + options.emplace_back("--pch-messages=true"); + } + } + + std::vector options_cstr; + for (auto const& option : options) { + options_cstr.emplace_back(option.c_str()); + } + + auto params = rtc::fragment_t::compile_params{.name = name, + .source = cuda_code, + .headers = {}, + .options = options_cstr, + .target_type = rtc::binary_type::CUBIN}; + + auto cubin = rtc::fragment_t::compile_raw(params); + + auto end = std::chrono::steady_clock::now(); + + auto duration = end - begin; + + CUDF_LOG_WARN( + "Compiled fragment `%s` in %f ms", + name, + std::chrono::duration_cast>(duration).count()); + + auto library = rtc::library_t::load( + rtc::library_t::load_params{.binary = cubin, .type = rtc::binary_type::CUBIN}); + + auto blob = rtc::blob_t::from_vector(std::move(cubin)); + + return std::make_tuple(library, std::make_shared(std::move(blob))); +} + +std::tuple link_udf_lto_ir_uncached(char const* name, + rtc::fragment const& fragment, + char const* kernel_symbol) { CUDF_FUNC_RANGE(); @@ -468,12 +554,12 @@ std::tuple link_udf_uncached(char const* name, return std::make_tuple(linked_library, blob); } -rtc::fragment compile_udf(char const* name, - char const* key, - char const* cuda_udf, - bool use_cache, - bool use_pch, - bool log_pch) +rtc::fragment compile_udf_to_lto_ir(char const* name, + char const* key, + char const* cuda_udf, + bool use_cache, + bool use_pch, + bool log_pch) { CUDF_FUNC_RANGE(); @@ -499,7 +585,7 @@ bundle={})***", auto cache_key_sha256 = hash_string(cache_key); - auto compile = [&] { return compile_udf_uncached(name, cuda_udf, use_pch, log_pch); }; + auto compile = [&] { return compile_udf_to_lto_ir_uncached(name, cuda_udf, use_pch, log_pch); }; if (!use_cache) { return compile(); } @@ -511,11 +597,11 @@ bundle={})***", return fut.get(); } -rtc::library link_udf(char const* name, - char const* key, - rtc::fragment const& fragment, - char const* kernel_symbol, - bool use_cache) +rtc::library link_lto_ir_udf(char const* name, + char const* key, + rtc::fragment const& fragment, + char const* kernel_symbol, + bool use_cache) { CUDF_FUNC_RANGE(); @@ -543,7 +629,7 @@ bundle={})***", auto cache_key_sha256 = hash_string(cache_key); - auto link = [&] { return link_udf_uncached(name, fragment, kernel_symbol); }; + auto link = [&] { return link_udf_lto_ir_uncached(name, fragment, kernel_symbol); }; if (!use_cache) { auto [lib, blob] = link(); @@ -556,49 +642,107 @@ bundle={})***", return fut.get(); } -rtc::library compile_cuda_library(char const* name, - char const* key, - char const* cuda_udf, - char const* kernel_symbol, - bool use_cache, - bool use_pch, - bool log_pch) +rtc::library compile_kernel_to_cubin(char const* name, + char const* key, + char const* cuda_source, + bool use_cache, + bool use_pch, + bool log_pch) { CUDF_FUNC_RANGE(); - auto fragment = compile_udf(name, key, cuda_udf, use_cache, use_pch, log_pch); - auto library = link_udf(name, key, fragment, kernel_symbol, use_cache); + + auto& cache = cudf::get_context().rtc_cache(); + auto& bundle = cudf::get_context().jit_bundle(); + + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_physical_model(); + auto bundle_hash = bundle.get_hash(); + + auto cache_key = std::format(R"***(binary_type=CUBIN +key={} +cuda_runtime={} +cuda_driver={} +arch={} +bundle={})***", + key, + runtime, + driver, + sm, + bundle_hash); + + auto cache_key_sha256 = hash_string(cache_key); + + auto compile = [&] { + return compile_kernel_to_cubin_uncached(name, cuda_source, use_pch, log_pch); + }; + + if (!use_cache) { + auto [lib, blob] = compile(); + return lib; + } + + auto fut = cache.query_or_insert_library(cache_key_sha256, + rtc::binary_type::CUBIN, + rtc::library_compile_function_t::from_functor(compile)); + + return fut.get(); +} + +rtc::library compile_and_link_kernel(char const* name, + char const* key, + char const* cuda_udf, + char const* kernel_symbol, + bool use_cache, + bool use_pch, + bool log_pch) +{ + CUDF_FUNC_RANGE(); + auto fragment = compile_udf_to_lto_ir(name, key, cuda_udf, use_cache, use_pch, log_pch); + auto library = link_lto_ir_udf(name, key, fragment, kernel_symbol, use_cache); return library; } } // namespace -rtc::library compile_kernel(std::string const& name, - std::string const& key, - std::string const& cuda_udf, - std::string const& kernel_symbol, - bool use_cache, - bool use_pch, - bool log_pch) +rtc::library compile_lto_kernel(std::string const& name, + std::string const& key, + std::string const& cuda_udf, + std::string const& kernel_symbol, + bool use_cache, + bool use_pch, + bool log_pch) +{ + return compile_and_link_kernel(name.c_str(), + key.c_str(), + cuda_udf.c_str(), + kernel_symbol.c_str(), + use_cache, + use_pch, + log_pch); +} + +[[nodiscard]] rtc::library compile_cuda_kernel(std::string const& name, + std::string const& key, + std::string const& cuda_udf, + bool use_cache, + bool use_pch, + bool log_pch) { - return compile_cuda_library(name.c_str(), - key.c_str(), - cuda_udf.c_str(), - kernel_symbol.c_str(), - use_cache, - use_pch, - log_pch); + return compile_kernel_to_cubin( + name.c_str(), key.c_str(), cuda_udf.c_str(), use_cache, use_pch, log_pch); } -rtc::library compile_lto_ir_kernel(std::string const& name, - std::string const& key, - std::span lto_ir_binary, - std::string const& kernel_symbol, - bool use_cache) +rtc::library compile_and_link_lto_ir_kernel(std::string const& name, + std::string const& key, + std::span lto_ir_binary, + std::string const& kernel_symbol, + bool use_cache) { auto blob = std::make_shared(rtc::blob_t::from_static_data(lto_ir_binary)); auto fragment = rtc::fragment_t::load( rtc::fragment_t::load_params{.binary = blob, .type = rtc::binary_type::LTO_IR}); - return link_udf(name.c_str(), key.c_str(), fragment, kernel_symbol.c_str(), use_cache); + return link_lto_ir_udf(name.c_str(), key.c_str(), fragment, kernel_symbol.c_str(), use_cache); } } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index edbf1c454c7a..04b07c2e51ca 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -36,18 +36,25 @@ struct [[nodiscard]] jit_bundle_t { [[nodiscard]] std::vector get_compile_options() const; }; -[[nodiscard]] rtc::library compile_kernel(std::string const& name, - std::string const& key, - std::string const& cuda_udf, - std::string const& kernel_symbol, - bool use_cache = true, - bool use_pch = true, - bool log_pch = false); - -[[nodiscard]] rtc::library compile_lto_ir_kernel(std::string const& name, - std::string const& key, - std::span lto_ir_binary, - std::string const& kernel_symbol, - bool use_cache = true); +[[nodiscard]] rtc::library compile_lto_kernel(std::string const& name, + std::string const& key, + std::string const& cuda_udf, + std::string const& kernel_symbol, + bool use_cache = true, + bool use_pch = true, + bool log_pch = false); + +[[nodiscard]] rtc::library compile_cuda_kernel(std::string const& name, + std::string const& key, + std::string const& cuda_udf, + bool use_cache = true, + bool use_pch = true, + bool log_pch = false); + +[[nodiscard]] rtc::library compile_and_link_lto_ir_kernel(std::string const& name, + std::string const& key, + std::span lto_ir_binary, + std::string const& kernel_symbol, + bool use_cache = true); } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp index 446957eb05f5..d53875b248f3 100644 --- a/cpp/src/jit/rtc/rtc.cpp +++ b/cpp/src/jit/rtc/rtc.cpp @@ -288,7 +288,7 @@ fragment fragment_t::load(load_params const& params) return std::make_shared(params.binary, params.type); } -fragment fragment_t::compile(compile_params const& params) +std::vector fragment_t::compile_raw(compile_params const& params) { CUDF_FUNC_RANGE(); @@ -325,9 +325,7 @@ fragment fragment_t::compile(compile_params const& params) CUDFRTC_CHECK_NVRTC( params, program, nvrtcGetLTOIR(program, reinterpret_cast(lto_ir.data()))); - auto shared_blob = std::make_shared(blob_t::from_vector(std::move(lto_ir))); - - return std::make_shared(std::move(shared_blob), binary_type::LTO_IR); + return lto_ir; } break; case binary_type::CUBIN: { @@ -339,15 +337,23 @@ fragment fragment_t::compile(compile_params const& params) CUDFRTC_CHECK_NVRTC( params, program, nvrtcGetCUBIN(program, reinterpret_cast(cubin.data()))); - auto shared_blob = std::make_shared(blob_t::from_vector(std::move(cubin))); - - return std::make_shared(std::move(shared_blob), binary_type::CUBIN); + return cubin; } break; default: CUDF_FAIL("Unsupported binary type for compiling fragment"); } } +fragment fragment_t::compile(compile_params const& params) +{ + CUDF_FUNC_RANGE(); + + auto result = compile_raw(params); + + return std::make_shared( + std::make_shared(rtc::blob_t::from_vector(std::move(result))), params.target_type); +} + blob const& fragment_t::get(binary_type type) const { CUDF_EXPECTS(type_ == type, "Fragment does not contain expected binary type"); diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp index 2b647cf64438..3d1108fb6aae 100644 --- a/cpp/src/jit/rtc/rtc.hpp +++ b/cpp/src/jit/rtc/rtc.hpp @@ -160,6 +160,8 @@ struct [[nodiscard]] fragment_t { binary_type target_type = binary_type::LTO_IR; }; + static std::vector compile_raw(compile_params const& params); + [[nodiscard]] static fragment load(load_params const& params); [[nodiscard]] static fragment compile(compile_params const& params); diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp index f47c90f53e7a..0983ef54419a 100644 --- a/cpp/tests/jit/rtc.cpp +++ b/cpp/tests/jit/rtc.cpp @@ -45,17 +45,17 @@ TEST_F(RTCTest, CompileKernelBasic) { auto fn = [] { char const* udf = R"***( - #include "jcudf/functions.cuh" + #include "jcudf/types.cuh" #pragma nv_hdrstop struct operator_params{ - void* scope; + void* const * scope; jcudf::size_type row_index; }; struct kernel_params{ - void* scope; + void* const * scope; jcudf::size_type num_rows; }; @@ -63,15 +63,14 @@ TEST_F(RTCTest, CompileKernelBasic) using namespace jcudf; // unpack inputs from scope using the appropriate getters based on the LTO context - using s0 = scope::column<0, column_device_view, int, false, false>; - using s1 = scope::column<1, column_device_view, int, false, false>; - using s2 = scope::column<2, mutable_column_device_view, int, false, false>; + using s0 = scope::column<0, column_view, int, false, false>; + using s1 = scope::column<1, column_view, int, false, false>; + using s2 = scope::column<2, optional_span, int, false, false>; auto a0 = s0::element(p.scope, p.row_index); auto a1 = s1::element(p.scope, p.row_index); - int a2; - functions::add(&a2, &a0, &a1); + int a2 = a0 + a1; s2::assign(p.scope, p.row_index, a2); } @@ -82,23 +81,33 @@ TEST_F(RTCTest, CompileKernelBasic) auto stride = static_cast(blockDim.x) * static_cast(gridDim.x); for(jcudf::i64 i = offset; i < params.num_rows; i += stride){ - operator_params p{params.scope, i}; + operator_params p{params.scope, static_cast(i)}; transform_operator(p); } } )***"; - static int i = 0; + + /*udf = R"***( + #include "jcudf/functions.cuh" + + #pragma nv_hdrstop + + extern "C" __global__ void transform_kernel(){ + // empty kernel for testing purposes + } + + )***";*/ + static int i = 0; i++; auto key = std::format("test_udf_key_{}", i); - auto lib = cudf::compile_kernel("test_fragment", - key, - udf, - "transform_kernel", - /*use_cache=*/true, - /*use_pch=*/true, - /*log_pch=*/true); + auto lib = cudf::compile_cuda_kernel("test_kernel", + key, + udf, + /*use_cache=*/true, + /*use_pch=*/true, + /*log_pch=*/true); auto kernel = lib->get_kernel("transform_kernel"); From dbaa1338997ab537b9fed756a18682b94bc37ee4 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 2 Mar 2026 20:30:34 +0000 Subject: [PATCH 059/254] Add system includes for CUDA standard library and refactor RTC cache - Introduced new header files in `librtcx/system_includes` for various standard library components, including algorithm, array, atomic, barrier, bit, cassert, complex, and more, all utilizing the CUDA standard library. - Updated the `context` class to replace the old RTC cache implementation with the new `rtcx` namespace, ensuring proper initialization and usage of the cache. - Refactored includes and namespaces to maintain consistency and improve code organization. --- cpp/CMakeLists.txt | 46 +- cpp/cmake/Modules/JitEmbed.cmake | 309 ----- cpp/cmake/Modules/jit_embed.py | 715 ---------- cpp/cmake/thirdparty/get_lz4.cmake | 2 +- cpp/include/cudf/utilities/defer.hpp | 42 - cpp/include/cudf/utilities/export.hpp | 2 +- cpp/include/jcudf/library.cu | 5 - cpp/include/{jcudf => jit_cudf}/README.md | 0 .../{jcudf => jit_cudf}/functions/abs.cuh | 6 +- .../{jcudf => jit_cudf}/functions/add.cuh | 6 +- cpp/include/jit_cudf/functions/all.cuh | 51 + .../{jcudf => jit_cudf}/functions/arccos.cuh | 6 +- .../{jcudf => jit_cudf}/functions/arccosh.cuh | 6 +- .../{jcudf => jit_cudf}/functions/arcsin.cuh | 6 +- .../{jcudf => jit_cudf}/functions/arcsinh.cuh | 6 +- .../{jcudf => jit_cudf}/functions/arctan.cuh | 6 +- .../{jcudf => jit_cudf}/functions/arctanh.cuh | 6 +- .../{jcudf => jit_cudf}/functions/bit_and.cuh | 6 +- .../functions/bit_invert.cuh | 6 +- .../{jcudf => jit_cudf}/functions/bit_or.cuh | 6 +- .../{jcudf => jit_cudf}/functions/bit_xor.cuh | 6 +- .../functions/cast_to_f64.cuh | 6 +- .../functions/cast_to_i64.cuh | 6 +- .../functions/cast_to_u64.cuh | 6 +- .../{jcudf => jit_cudf}/functions/cbrt.cuh | 6 +- .../{jcudf => jit_cudf}/functions/ceil.cuh | 6 +- .../{jcudf => jit_cudf}/functions/cos.cuh | 6 +- .../{jcudf => jit_cudf}/functions/cosh.cuh | 6 +- .../{jcudf => jit_cudf}/functions/div.cuh | 6 +- .../{jcudf => jit_cudf}/functions/equal.cuh | 6 +- .../{jcudf => jit_cudf}/functions/exp.cuh | 6 +- .../{jcudf => jit_cudf}/functions/floor.cuh | 6 +- .../{jcudf => jit_cudf}/functions/greater.cuh | 6 +- .../functions/greater_equal.cuh | 10 +- .../functions/identity.cuh | 6 +- .../{jcudf => jit_cudf}/functions/is_null.cuh | 6 +- .../{jcudf => jit_cudf}/functions/less.cuh | 6 +- .../functions/less_equal.cuh | 6 +- .../{jcudf => jit_cudf}/functions/log.cuh | 6 +- .../functions/logical_and.cuh | 6 +- .../functions/logical_or.cuh | 6 +- .../{jcudf => jit_cudf}/functions/mod.cuh | 6 +- .../{jcudf => jit_cudf}/functions/mul.cuh | 6 +- .../functions/null_equal.cuh | 6 +- .../functions/null_logical_and.cuh | 10 +- .../functions/null_logical_or.cuh | 10 +- .../{jcudf => jit_cudf}/functions/pow.cuh | 6 +- .../{jcudf => jit_cudf}/functions/pymod.cuh | 6 +- .../{jcudf => jit_cudf}/functions/rint.cuh | 6 +- .../{jcudf => jit_cudf}/functions/sin.cuh | 6 +- .../{jcudf => jit_cudf}/functions/sinh.cuh | 6 +- .../{jcudf => jit_cudf}/functions/sub.cuh | 6 +- .../{jcudf => jit_cudf}/functions/tanh.cuh | 6 +- cpp/include/{jcudf => jit_cudf}/types.cuh | 6 +- cpp/src/jit/jit.cpp | 479 +------ cpp/src/jit/jit.hpp | 43 +- cpp/src/jit/rtc/cache.cpp | 449 ------- cpp/src/jit/rtc/cache.hpp | 267 ---- cpp/src/jit/rtc/rtc.cpp | 575 -------- cpp/src/jit/rtc/rtc.hpp | 252 ---- cpp/src/jit/rtc/sha256.cpp | 47 - cpp/src/jit/rtc/sha256.hpp | 113 -- cpp/src/librtcx/CMake/embed.cmake | 161 +++ cpp/src/librtcx/CMake/embed.py.in | 197 +++ cpp/src/{jit/rtc => librtcx}/README.md | 0 cpp/src/librtcx/rtcx.cpp | 1180 +++++++++++++++++ cpp/src/librtcx/rtcx.hpp | 704 ++++++++++ .../system_includes}/README.md | 0 .../system_includes}/algorithm | 0 .../system => librtcx/system_includes}/array | 0 .../system => librtcx/system_includes}/atomic | 0 .../system_includes}/barrier | 0 .../system => librtcx/system_includes}/bit | 0 .../system_includes}/cassert | 0 .../system_includes}/cassert.h | 0 .../system_includes}/ccomplex | 0 .../system_includes}/ccomplex.h | 0 .../system => librtcx/system_includes}/cfloat | 0 .../system_includes}/cfloat.h | 0 .../system => librtcx/system_includes}/chrono | 0 .../system_includes}/climits | 0 .../system => librtcx/system_includes}/cmath | 0 .../system_includes}/cmath.h | 0 .../system_includes}/complex | 0 .../system_includes}/concepts | 0 .../system_includes}/cstddef | 0 .../system_includes}/cstdint | 0 .../system_includes}/cstdlib | 0 .../system_includes}/cstring | 0 .../system => librtcx/system_includes}/ctime | 0 .../system_includes}/expected | 0 .../system_includes}/functional | 0 .../system_includes}/initializer_list | 0 .../system_includes}/inplace_vector | 0 .../system_includes}/iterator | 0 .../system => librtcx/system_includes}/latch | 0 .../system => librtcx/system_includes}/limits | 0 .../system => librtcx/system_includes}/linalg | 0 .../system => librtcx/system_includes}/mdspan | 0 .../system => librtcx/system_includes}/memory | 0 .../system_includes}/numbers | 0 .../system_includes}/numeric | 0 .../system_includes}/optional | 0 .../system => librtcx/system_includes}/ranges | 0 .../system => librtcx/system_includes}/ratio | 0 .../system_includes}/semaphore | 0 .../system_includes}/source_location | 0 .../system => librtcx/system_includes}/span | 0 .../system_includes}/stddef.h | 0 .../system_includes}/stdint.h | 0 .../system_includes}/stdlib.h | 0 .../system_includes}/string.h | 0 .../system_includes}/string_view | 0 .../system => librtcx/system_includes}/time.h | 0 .../system => librtcx/system_includes}/tuple | 0 .../system_includes}/type_traits | 0 .../system_includes}/utility | 0 .../system_includes}/variant | 0 .../system_includes}/version | 0 cpp/src/runtime/context.cpp | 7 +- cpp/src/runtime/context.hpp | 12 +- 121 files changed, 2573 insertions(+), 3367 deletions(-) delete mode 100644 cpp/cmake/Modules/JitEmbed.cmake delete mode 100644 cpp/cmake/Modules/jit_embed.py delete mode 100644 cpp/include/cudf/utilities/defer.hpp delete mode 100644 cpp/include/jcudf/library.cu rename cpp/include/{jcudf => jit_cudf}/README.md (100%) rename cpp/include/{jcudf => jit_cudf}/functions/abs.cuh (86%) rename cpp/include/{jcudf => jit_cudf}/functions/add.cuh (87%) create mode 100644 cpp/include/jit_cudf/functions/all.cuh rename cpp/include/{jcudf => jit_cudf}/functions/arccos.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/arccosh.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/arcsin.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/arcsinh.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/arctan.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/arctanh.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/bit_and.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/bit_invert.cuh (86%) rename cpp/include/{jcudf => jit_cudf}/functions/bit_or.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/bit_xor.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/cast_to_f64.cuh (86%) rename cpp/include/{jcudf => jit_cudf}/functions/cast_to_i64.cuh (86%) rename cpp/include/{jcudf => jit_cudf}/functions/cast_to_u64.cuh (86%) rename cpp/include/{jcudf => jit_cudf}/functions/cbrt.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/ceil.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/cos.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/cosh.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/div.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/equal.cuh (88%) rename cpp/include/{jcudf => jit_cudf}/functions/exp.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/floor.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/greater.cuh (86%) rename cpp/include/{jcudf => jit_cudf}/functions/greater_equal.cuh (61%) rename cpp/include/{jcudf => jit_cudf}/functions/identity.cuh (83%) rename cpp/include/{jcudf => jit_cudf}/functions/is_null.cuh (83%) rename cpp/include/{jcudf => jit_cudf}/functions/less.cuh (86%) rename cpp/include/{jcudf => jit_cudf}/functions/less_equal.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/log.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/logical_and.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/logical_or.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/mod.cuh (90%) rename cpp/include/{jcudf => jit_cudf}/functions/mul.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/null_equal.cuh (88%) rename cpp/include/{jcudf => jit_cudf}/functions/null_logical_and.cuh (66%) rename cpp/include/{jcudf => jit_cudf}/functions/null_logical_or.cuh (66%) rename cpp/include/{jcudf => jit_cudf}/functions/pow.cuh (88%) rename cpp/include/{jcudf => jit_cudf}/functions/pymod.cuh (91%) rename cpp/include/{jcudf => jit_cudf}/functions/rint.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/sin.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/sinh.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/sub.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/functions/tanh.cuh (87%) rename cpp/include/{jcudf => jit_cudf}/types.cuh (99%) delete mode 100644 cpp/src/jit/rtc/cache.cpp delete mode 100644 cpp/src/jit/rtc/cache.hpp delete mode 100644 cpp/src/jit/rtc/rtc.cpp delete mode 100644 cpp/src/jit/rtc/rtc.hpp delete mode 100644 cpp/src/jit/rtc/sha256.cpp delete mode 100644 cpp/src/jit/rtc/sha256.hpp create mode 100644 cpp/src/librtcx/CMake/embed.cmake create mode 100644 cpp/src/librtcx/CMake/embed.py.in rename cpp/src/{jit/rtc => librtcx}/README.md (100%) create mode 100644 cpp/src/librtcx/rtcx.cpp create mode 100644 cpp/src/librtcx/rtcx.hpp rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/README.md (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/algorithm (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/array (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/atomic (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/barrier (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/bit (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/cassert (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/cassert.h (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/ccomplex (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/ccomplex.h (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/cfloat (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/cfloat.h (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/chrono (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/climits (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/cmath (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/cmath.h (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/complex (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/concepts (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/cstddef (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/cstdint (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/cstdlib (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/cstring (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/ctime (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/expected (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/functional (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/initializer_list (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/inplace_vector (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/iterator (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/latch (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/limits (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/linalg (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/mdspan (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/memory (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/numbers (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/numeric (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/optional (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/ranges (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/ratio (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/semaphore (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/source_location (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/span (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/stddef.h (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/stdint.h (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/stdlib.h (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/string.h (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/string_view (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/time.h (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/tuple (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/type_traits (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/utility (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/variant (100%) rename cpp/src/{jit/rtc/includes/system => librtcx/system_includes}/version (100%) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index db6517299f04..2c4706330df7 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -299,9 +299,6 @@ endif() # preprocess jitify-able kernels include(cmake/Modules/JitifyPreprocessKernels.cmake) -# JIT Embedding helper functions -include(cmake/Modules/JitEmbed.cmake) - # find KvikIO include(cmake/thirdparty/get_kvikio.cmake) @@ -311,6 +308,9 @@ include(cmake/thirdparty/get_nanoarrow.cmake) # find thread_pool include(cmake/thirdparty/get_thread_pool.cmake) +# JIT Embedding helper functions +include(src/librtcx/CMake/embed.cmake) + # find zstd include(cmake/thirdparty/get_zstd.cmake) @@ -358,31 +358,6 @@ list(APPEND CUDF_JIT_COMPILE_FLAGS -std=c++${CUDF_CUDA_STANDARD} --device-int128 ${CUDF_JIT_DEFINITIONS} ) -set(CUDF_LTO_JIT_ARCHITECTURES "80-real;90-real;100-real;120-real") -add_library(cudf_lto_library OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/include/jcudf/library.cu) -target_compile_options(cudf_lto_library PRIVATE --compress-mode=none --gen-opt-lto) -target_compile_definitions(cudf_lto_library PRIVATE CUDF_DISABLE_DEFAULT_EXPORT) -set_target_properties( - cudf_lto_library - PROPERTIES CUDA_ARCHITECTURES "${CUDF_LTO_JIT_ARCHITECTURES}" - CUDA_SEPARABLE_COMPILATION ON - CUDA_FATBIN_COMPILATION ON - POSITION_INDEPENDENT_CODE ON - INTERPROCEDURAL_OPTIMIZATION ON - CXX_STANDARD 20 - CXX_STANDARD_REQUIRED ON - CXX_EXTENSIONS ON - CXX_VISIBILITY_PRESET hidden - CUDA_STANDARD 20 - CUDA_STANDARD_REQUIRED ON - CUDA_VISIBILITY_PRESET hidden -) -target_link_libraries(cudf_lto_library PRIVATE CCCL::CCCL) -target_include_directories( - cudf_lto_library PRIVATE "$" - "$" -) - jit_add_include_directory( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include DEST_DIRECTORY cudf/include INCLUDE_DIRECTORIES cudf/include @@ -390,7 +365,12 @@ jit_add_include_directory( jit_add_include_directory( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src DEST_DIRECTORY cudf/src - INCLUDE_DIRECTORIES cudf/src cudf/src/jit/rtc/includes/system + INCLUDE_DIRECTORIES cudf/src +) + +jit_add_include_directory( + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/system_includes + DEST_DIRECTORY system_includes INCLUDE_DIRECTORIES system_includes ) get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) @@ -404,8 +384,6 @@ foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) ) endforeach() -jit_add_options(cudf_jit_embed OPTIONS ${CUDF_JIT_COMPILE_FLAGS}) -jit_add_blob(cudf_jit_embed FILE "$" DEST cudf_lto_library.fatbin) jit_embed(cudf_jit_embed COMPRESSION zstd) # ################################################################################################## @@ -658,9 +636,6 @@ add_library( src/jit/parser.cpp src/jit/row_ir.cpp src/jit/util.cpp - src/jit/rtc/cache.cpp - src/jit/rtc/rtc.cpp - src/jit/rtc/sha256.cpp src/join/conditional_join.cu src/join/cross_join.cu src/join/distinct_hash_join.cu @@ -927,7 +902,8 @@ add_library( src/utilities/traits.cpp src/utilities/type_checks.cpp src/utilities/type_dispatcher.cpp - ${cudf_jit_embed_SOURCE_DIR}/embed.s + src/librtcx/rtcx.cpp + ${cudf_jit_embed_SOURCE_DIR}/cudf_jit_embed.s ) add_dependencies(cudf cudf_jit_embed) diff --git a/cpp/cmake/Modules/JitEmbed.cmake b/cpp/cmake/Modules/JitEmbed.cmake deleted file mode 100644 index d61f9c1b89d7..000000000000 --- a/cpp/cmake/Modules/JitEmbed.cmake +++ /dev/null @@ -1,309 +0,0 @@ -# ============================================================================= -# cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# cmake-format: on -# ============================================================================= - -find_package(Python3 REQUIRED COMPONENTS Interpreter) - -function(jit_add_include_directory) - set(TARGET ${ARGV0}) - set(OPTIONS "") - set(ONE_VALUE_ARGS COPY_DIRECTORY # Source directory where files will be copied from - DEST_DIRECTORY # Destination directory where files will be copied to - ) - set(MULTI_VALUE_ARGS - FILES # Source files relative to COPY_DIRECTORY (optional, if not provided, all files under - # COPY_DIRECTORY will be used) - INCLUDE_DIRECTORIES # Include directories to be used when compiling with these files - ) - cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - - if(NOT DEFINED TARGET) - message(FATAL_ERROR "TARGET argument is required") - endif() - - if(NOT ARG_COPY_DIRECTORY) - message(FATAL_ERROR "COPY_DIRECTORY argument is required") - endif() - - if(NOT ARG_DEST_DIRECTORY) - message(FATAL_ERROR "DEST_DIRECTORY argument is required") - endif() - - if(NOT ARG_INCLUDE_DIRECTORIES) - message(FATAL_ERROR "INCLUDE_DIRECTORIES argument is required") - endif() - - if(NOT ARG_FILES) - # gather all include files under the specified directory - file(GLOB_RECURSE INCLUDE_FILES "${ARG_COPY_DIRECTORY}/*") - - # get their paths relative to the base include directory - set(INCLUDE_FILES_RELATIVE_PATHS "") - foreach(INCLUDE_FILE IN LISTS INCLUDE_FILES) - file(RELATIVE_PATH INCLUDE_FILE_REL_PATH "${ARG_COPY_DIRECTORY}" "${INCLUDE_FILE}") - list(APPEND INCLUDE_FILES_RELATIVE_PATHS "${INCLUDE_FILE_REL_PATH}") - endforeach() - - set(ARG_FILES ${INCLUDE_FILES_RELATIVE_PATHS}) - endif() - - # check that each source file exists - foreach(SOURCE_FILE IN LISTS ARG_FILES) - if(NOT EXISTS "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") - message(FATAL_ERROR "Source file '${ARG_COPY_DIRECTORY}/${SOURCE_FILE}' does not exist") - endif() - endforeach(SOURCE_FILE) - - # Set scope variables to accumulate results - - set(SOURCE_FILES ${jitembed_${TARGET}_incdir__source_files}) - set(SOURCE_FILE_DESTS ${jitembed_${TARGET}_incdir__source_file_dests}) - set(INCLUDE_DIRECTORIES ${jitembed_${TARGET}_incdir__include_directories}) - - foreach(SOURCE_FILE IN LISTS ARG_FILES) - list(APPEND SOURCE_FILES "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") - list(APPEND SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}") - endforeach() - - list(APPEND INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES}) - - set(jitembed_${TARGET}_incdir__source_files - ${SOURCE_FILES} - PARENT_SCOPE - ) - set(jitembed_${TARGET}_incdir__source_file_dests - ${SOURCE_FILE_DESTS} - PARENT_SCOPE - ) - set(jitembed_${TARGET}_incdir__include_directories - ${INCLUDE_DIRECTORIES} - PARENT_SCOPE - ) - -endfunction() - -function(jit_add_options) - set(TARGET ${ARGV0}) - set(OPTIONS "") - set(ONE_VALUE_ARGS "") - set(MULTI_VALUE_ARGS "OPTIONS") - cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - - if(NOT DEFINED TARGET) - message(FATAL_ERROR "TARGET argument is required") - endif() - - if(NOT ARG_OPTIONS) - message(FATAL_ERROR "OPTIONS argument is required") - endif() - - set(options ${jitembed_options_${TARGET}_options}) - foreach(OPTION IN LISTS ARG_OPTIONS) - list(APPEND options "${OPTION}") - endforeach() - - set(jitembed_options_${TARGET}_options - ${options} - PARENT_SCOPE - ) - -endfunction() - -function(jit_add_blob) - set(TARGET ${ARGV0}) - set(OPTIONS "") - set(ONE_VALUE_ARGS "FILE;DEST") - set(MULTI_VALUE_ARGS "") - cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - - if(NOT DEFINED TARGET) - message(FATAL_ERROR "TARGET argument is required") - endif() - - if(NOT ARG_FILE) - message(FATAL_ERROR "FILE argument is required") - endif() - - if(NOT ARG_DEST) - message(FATAL_ERROR "DEST argument is required") - endif() - - set(blob_files ${jitembed_${TARGET}_blob__files}) - set(blob_dests ${jitembed_${TARGET}_blob__dests}) - set(blob_target_deps ${jitembed_${TARGET}_blob__target_deps}) - - list(APPEND blob_files "${ARG_FILE}") - list(APPEND blob_dests "${ARG_DEST}") - - # If FILE is a $ generator expression, record the target name so jit_embed() - # can add an explicit ordering dependency via add_dependencies(). A plain file dependency on the - # evaluated paths is not sufficient to guarantee build ordering in parallel builds. - if(ARG_FILE MATCHES "\\$]+)>") - list(APPEND blob_target_deps "${CMAKE_MATCH_1}") - set(jitembed_${TARGET}_blob__target_deps - ${blob_target_deps} - PARENT_SCOPE - ) - endif() - - set(jitembed_${TARGET}_blob__files - ${blob_files} - PARENT_SCOPE - ) - - set(jitembed_${TARGET}_blob__dests - ${blob_dests} - PARENT_SCOPE - ) -endfunction() - -# pass the encoded args to the jit_embed.py script to generate the source and options maps -function(jit_embed) - set(TARGET ${ARGV0}) - set(OPTIONS "") - set(ONE_VALUE_ARGS "COMPRESSION") - set(MULTI_VALUE_ARGS "") - cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - - if(NOT DEFINED TARGET) - message(FATAL_ERROR "TARGET argument is required") - endif() - - if(NOT DEFINED ARG_COMPRESSION) - message(FATAL_ERROR "COMPRESSION argument is required") - endif() - - if(NOT ARG_COMPRESSION STREQUAL "none" - AND NOT ARG_COMPRESSION STREQUAL "lz4" - AND NOT ARG_COMPRESSION STREQUAL "zstd" - ) - message(FATAL_ERROR "COMPRESSION argument must be either none, lz4, or, zstd") - endif() - - string(APPEND TARGET_YAML "\"${TARGET}_sources\":\n") - string(APPEND TARGET_YAML " type: \"sources\"\n") - - if(DEFINED jitembed_${TARGET}_incdir__source_files) - - # gather source files - string(APPEND TARGET_YAML " sources:\n") - list(LENGTH jitembed_${TARGET}_incdir__source_files NUM_SOURCES) - - math(EXPR LAST_SOURCE_INDEX "${NUM_SOURCES} - 1") - foreach(i RANGE 0 ${LAST_SOURCE_INDEX}) - list(GET jitembed_${TARGET}_incdir__source_files ${i} SOURCE_FILE) - list(GET jitembed_${TARGET}_incdir__source_file_dests ${i} SOURCE_FILE_DEST) - string(APPEND TARGET_YAML " - file: \"${SOURCE_FILE}\"\n") - string(APPEND TARGET_YAML " dest: \"${SOURCE_FILE_DEST}\"\n") - endforeach() - - # gather include directories - - string(APPEND TARGET_YAML " include_directories:\n") - list(LENGTH jitembed_${TARGET}_incdir__include_directories NUM_INCLUDE_DIRS) - math(EXPR LAST_INCLUDE_DIR_INDEX "${NUM_INCLUDE_DIRS} - 1") - foreach(i RANGE 0 ${LAST_INCLUDE_DIR_INDEX}) - list(GET jitembed_${TARGET}_incdir__include_directories ${i} INCLUDE_DIR) - string(APPEND TARGET_YAML " - \"${INCLUDE_DIR}\"\n") - endforeach() - - string(APPEND TARGET_YAML " compression: ${ARG_COMPRESSION}\n") - - endif() - - string(APPEND TARGET_YAML "\n\n") - - if(DEFINED jitembed_options_${TARGET}_options) - - string(APPEND TARGET_YAML "\"${TARGET}_options\":\n") - string(APPEND TARGET_YAML " type: \"strings\"\n") - - # gather options - string(APPEND TARGET_YAML " strings:\n") - - list(LENGTH jitembed_options_${TARGET}_options NUM_OPTIONS) - math(EXPR LAST_OPTION_INDEX "${NUM_OPTIONS} - 1") - foreach(i RANGE 0 ${LAST_OPTION_INDEX}) - list(GET jitembed_options_${TARGET}_options ${i} OPTION) - string(APPEND TARGET_YAML " - \"${OPTION}\"\n") - endforeach() - - endif() - - string(APPEND TARGET_YAML "\n\n") - - if(DEFINED jitembed_${TARGET}_blob__files) - - string(APPEND TARGET_YAML "\"${TARGET}_blobs\":\n") - string(APPEND TARGET_YAML " type: \"blobs\"\n") - - # gather blobs - string(APPEND TARGET_YAML " blobs:\n") - - list(LENGTH jitembed_${TARGET}_blob__files NUM_BLOBS) - math(EXPR LAST_BLOB_INDEX "${NUM_BLOBS} - 1") - foreach(i RANGE 0 ${LAST_BLOB_INDEX}) - list(GET jitembed_${TARGET}_blob__files ${i} BLOB_FILE) - list(GET jitembed_${TARGET}_blob__dests ${i} BLOB_DEST) - string(APPEND TARGET_YAML " - file: \"${BLOB_FILE}\"\n") - string(APPEND TARGET_YAML " dest: \"${BLOB_DEST}\"\n") - endforeach() - - string(APPEND TARGET_YAML " compression: ${ARG_COMPRESSION}\n") - - endif() - - set(YAML_FILE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.yaml") - set(JIT_EMBED_DIR "${CUDF_GENERATED_INCLUDE_DIR}/jit_embed") - set(OUTPUT_DIR "${JIT_EMBED_DIR}/${TARGET}") - - # write CONFIG to temp file and pass file path to script - file( - GENERATE - OUTPUT "${YAML_FILE_PATH}" - CONTENT "${TARGET_YAML}" - ) - - add_custom_command( - OUTPUT ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s ${OUTPUT_DIR}/embed.bin - BYPRODUCTS ${OUTPUT_DIR}/* - COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" --id - "${TARGET}" --output-dir "${OUTPUT_DIR}" --input "${YAML_FILE_PATH}" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/jit_embed.py" "${YAML_FILE_PATH}" - ${jitembed_${TARGET}_incdir__source_files} ${jitembed_${TARGET}_blob__files} - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - COMMENT - "Generating JIT embed for ${TARGET} (YAML: ${YAML_FILE_PATH}) into ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s" - VERBATIM - ) - - add_custom_target( - ${TARGET} ALL DEPENDS ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s ${OUTPUT_DIR}/embed.bin - ) - - # Ensure any CMake targets whose objects are embedded are fully built before the custom command - # runs. File-level DEPENDS on $ paths does not guarantee target build ordering - # in parallel builds. - if(DEFINED jitembed_${TARGET}_blob__target_deps) - add_dependencies(${TARGET} ${jitembed_${TARGET}_blob__target_deps}) - endif() - - message( - STATUS - "JIT embed for target ${TARGET} (YAML: ${YAML_FILE_PATH}) will be generated into: ${OUTPUT_DIR}/embed.hpp ${OUTPUT_DIR}/embed.s" - ) - - set(${TARGET}_INCLUDE_DIRS - "${CUDF_GENERATED_INCLUDE_DIR};${OUTPUT_DIR}" - PARENT_SCOPE - ) - - set(${TARGET}_SOURCE_DIR - ${OUTPUT_DIR} - PARENT_SCOPE - ) - -endfunction() diff --git a/cpp/cmake/Modules/jit_embed.py b/cpp/cmake/Modules/jit_embed.py deleted file mode 100644 index b5a5f732e8cf..000000000000 --- a/cpp/cmake/Modules/jit_embed.py +++ /dev/null @@ -1,715 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -import argparse -import hashlib -import logging -import os -from typing import NamedTuple, Self - -import lz4.block -import yaml -import zstd - -LIST_LINE_WIDTH = 32 -NAMESPACE_PREFIX = "jit_" - - -# TODO: write a schema validator for the input YAML - - -### json schema - -"""entries -[ - $id: { - "type": "sources", - "sources": [ - { - "file": string, - "dest": string - } - ], - "include_directories": [string] - "compression": string - }, - $id: { - "type": "strings", - "strings": list[string] - }, - $id: { - "type": "blobs", - "blobs": [ - { - "file": string, - "dest": string - } - ], - "compression": string - } -] -""" - - -PREAMBLE = f""" -/// Auto-generated by jit_embed.py. Do not edit directly. -#pragma once - -extern "C" {{ - -typedef struct {NAMESPACE_PREFIX}bytes_t {{ - unsigned char const * data; - unsigned long size; -}} {NAMESPACE_PREFIX}bytes_t; - -typedef struct {NAMESPACE_PREFIX}byte_range_t {{ - unsigned long offset; - unsigned long size; -}} {NAMESPACE_PREFIX}byte_range_t; - -typedef struct {NAMESPACE_PREFIX}bytes_array_t {{ - {NAMESPACE_PREFIX}bytes_t bytes; - {NAMESPACE_PREFIX}byte_range_t const * ranges; - unsigned long num_ranges; -}} {NAMESPACE_PREFIX}bytes_array_t; - -}} - -""" - -logger = logging.getLogger(__name__) - - -def list_string(strings: list[str]) -> str: - lines = [] - for i in range(0, len(strings), LIST_LINE_WIDTH): - line = ", ".join(strings[i : i + LIST_LINE_WIDTH]) - lines.append(line) - return ",\n".join(lines) - - -def byte_hex_string(value: int) -> str: - return f"0x{value:02X}" - - -class CXXVarDecl(NamedTuple): - id: str - expr: str - - @staticmethod - def of_bytes(id: str, data: bytes, alignment: int) -> Self: - byte_array = list_string([byte_hex_string(b) for b in data]) - expr = f"""alignas({alignment}) static unsigned char const {id}[{len(data)}] = {{ -{byte_array} -}};""" - return CXXVarDecl(id=id, expr=expr) - - @staticmethod - def of_size(id: str, size: int) -> Self: - expr = f"static long const {id} = {size}L;" - return CXXVarDecl(id=id, expr=expr) - - def decl(self: Self) -> str: - return f"""{self.expr}""" - - -class CXXSizeDecl(NamedTuple): - id: str - size: int - - @staticmethod - def of_size(id: str, size: int) -> Self: - return CXXSizeDecl(id=id, size=size) - - def var(self: Self) -> CXXVarDecl: - expr = f"static unsigned long const {self.id} = {self.size}UL;" - return CXXVarDecl(id=self.id, expr=expr) - - -class CXXSizeArrayDecl(NamedTuple): - id: str - sizes: list[int] - - @staticmethod - def of_sizes(id: str, sizes: list[int]) -> Self: - return CXXSizeArrayDecl(id=id, sizes=sizes) - - def var(self: Self) -> CXXVarDecl: - size_array = list_string([f"{size}UL" for size in self.sizes]) - expr = f"""static unsigned long const {self.id}[{len(self.sizes)}] = {{ -{size_array} -}};""" - return CXXVarDecl(id=self.id, expr=expr) - - -class CXXBytesDecl(NamedTuple): - id: str - data: bytes - alignment: int - num_null_terminators: int - - @staticmethod - def of_bytes( - id: str, data: bytes, alignment: int, num_null_terminators: int - ) -> Self: - return CXXBytesDecl( - id=id, - data=data, - alignment=alignment, - num_null_terminators=num_null_terminators, - ) - - def var(self: Self) -> CXXVarDecl: - # exclude null terminator from length - size_decl = CXXVarDecl.of_size( - id=f"{self.id}_size", size=len(self.data) - ) - - data = self.data + b"\0" * self.num_null_terminators - - data_decl = CXXVarDecl.of_bytes( - id=f"{self.id}_data", data=data, alignment=self.alignment - ) - - return CXXVarDecl( - id=self.id, - expr=f""" -{data_decl.decl()} - -{size_decl.decl()} - - -static {NAMESPACE_PREFIX}bytes_t const {self.id} = {{ - .data = {data_decl.id}, - .size = {size_decl.id} -}}; -""", - ) - - -class CXXRangesDecl(NamedTuple): - id: str - ranges: list[tuple[int, int]] # list of (offset, size) - - @staticmethod - def of_ranges(id: str, ranges: list[tuple[int, int]]) -> Self: - return CXXRangesDecl(id=id, ranges=ranges) - - def var(self: Self) -> CXXVarDecl: - ranges_str = [ - f"{{{offset}UL, {size}UL}}" for offset, size in self.ranges - ] - - ranges_str_formatted = list_string(ranges_str) - - expr = f"""static {NAMESPACE_PREFIX}byte_range_t const {self.id}[{len(self.ranges)}] = {{ -{ranges_str_formatted} -}};""" - return CXXVarDecl(id=self.id, expr=expr) - - -class CXXAsciiStringDecl(NamedTuple): - id: str - string: str - - @staticmethod - def of_string(id: str, string: str) -> Self: - return CXXAsciiStringDecl(id=id, string=string) - - def var(self: Self) -> CXXVarDecl: - expr = f"""static char const {self.id}[] = "{self.string}";""" - return CXXVarDecl(id=self.id, expr=expr) - - -class CXXArrayOfBytesDecl(NamedTuple): - id: str - data: bytes - alignment: int - ranges: list[tuple[int, int]] # list of (offset, size) - - @staticmethod - def of_byte_ranges( - id: str, - data: bytes, - ranges: list[tuple[int, int]], - alignment: int, - ) -> Self: - return CXXArrayOfBytesDecl( - id=id, - data=data, - alignment=alignment, - ranges=ranges, - ) - - def var(self: Self) -> CXXVarDecl: - bytes_decl = CXXBytesDecl.of_bytes( - id=f"{self.id}_bytes", - data=self.data, - alignment=self.alignment, - num_null_terminators=0, - ) - - ranges_decl = CXXRangesDecl.of_ranges( - id=f"{self.id}_ranges", ranges=self.ranges - ) - - expr = f""" -{bytes_decl.var().decl()} - -{ranges_decl.var().decl()} - -static {NAMESPACE_PREFIX}bytes_array_t const {self.id} = {{ - .bytes = {bytes_decl.id}, - .ranges = {ranges_decl.id}, - .num_ranges = {len(self.ranges)} -}}; -""" - - return CXXVarDecl( - id=self.id, - expr=expr, - ) - - -class CXXBinEmbedDecl(NamedTuple): - id: str - - @staticmethod - def of(id: str) -> Self: - return CXXBinEmbedDecl(id=id) - - def var(self: Self) -> CXXVarDecl: - return CXXVarDecl( - id=self.id, - expr=f""" - extern unsigned char const {self.id}_begin[]; - extern unsigned char const {self.id}_end[]; - -static {NAMESPACE_PREFIX}bytes_t const {self.id} = {{ - .data = {self.id}_begin, - .size = (unsigned long)({self.id}_end - {self.id}_begin) -}}; - - """, - ) - - -class AsmEmbedDecl(NamedTuple): - id: str - file: str - - @staticmethod - def of_file(id: str, file: str) -> Self: - return AsmEmbedDecl(id=id, file=file) - - def decl(self: Self) -> str: - return f""" -.section .rodata -.global {self.id}_begin -.global {self.id}_end -{self.id}_begin: -.incbin "{self.file}" -{self.id}_end: -""" - - -def merge_bytes_with_null_terminators( - bytes_lists: list[bytes], -) -> tuple[bytes, list[tuple[int, int]]]: - merged: bytes = bytes() - ranges: list[tuple[int, int]] = [] - - for byte_data in bytes_lists: - ranges.append((len(merged), len(byte_data))) - merged += byte_data + b"\0" - - return merged, ranges - - -class EmbedOutput(NamedTuple): - cxx_header: str | None - cxx_source: str | None - asm_source: str | None - bin_file_name: str | None - bin_file_data: bytes | None - hash: bytes - - -def generate_cxx_strings_data(id: str, strings: list[str]) -> EmbedOutput: - data, ranges = merge_bytes_with_null_terminators( - [s.encode("utf-8") for s in strings] - ) - - sha = hashlib.sha256() - sha.update(data) - hash = sha.digest() - - arrays_decl = CXXArrayOfBytesDecl.of_byte_ranges( - id=f"{id}", - data=data, - alignment=1, - ranges=ranges, - ) - - cxx_header = f""" -{arrays_decl.var().decl()} -""" - - return EmbedOutput( - cxx_header=cxx_header, - cxx_source=None, - asm_source=None, - bin_file_name=None, - bin_file_data=None, - hash=hash, - ) - - -def load_file_bytes(file_path: str) -> bytes: - with open(file_path, "rb") as f: - return f.read() - - -def compress_bytes(data: bytes, compression: str) -> bytes: - assert compression in ("none", "lz4", "zstd"), "Invalid compression type" - - if compression == "none": - return data - elif compression == "lz4": - compressed_data = lz4.block.compress( - data, mode="high_compression", compression=12, store_size=False - ) - logger.info( - f"Uncompressed size is {len(data)} bytes, compressed size is {len(compressed_data)} bytes" - ) - return compressed_data - elif compression == "zstd": - compressed_data = zstd.compress(data, 22) - logger.info( - f"Uncompressed size is {len(data)} bytes, compressed size is {len(compressed_data)} bytes" - ) - return compressed_data - - -def generate_cxx_source_files_data( - id: str, - file_paths: list[str], - dests: list[str], - include_directories: list[str], - compression: str, -) -> EmbedOutput: - uncompressed_files_bytes, files_ranges = merge_bytes_with_null_terminators( - [load_file_bytes(p) for p in file_paths] - ) - - compress = compression != "none" - - compressed_files_bytes = ( - compress_bytes(uncompressed_files_bytes, compression) - if compress - else None - ) - - if compress: - logger.info( - f"{id}'s uncompressed size is {len(uncompressed_files_bytes)} bytes, compressed size is {len(compressed_files_bytes)} bytes" - ) - - merged_dests_bytes, merged_dests_ranges = ( - merge_bytes_with_null_terminators([d.encode("utf-8") for d in dests]) - ) - - merged_include_directories_bytes, merged_include_directories_ranges = ( - merge_bytes_with_null_terminators( - [d.encode("utf-8") for d in include_directories] - ) - ) - - # compute combined sha256 hash of all files - sha = hashlib.sha256() - sha.update(uncompressed_files_bytes) - sha.update(merged_dests_bytes) - sha.update(merged_include_directories_bytes) - sha.update(compression.encode("utf-8")) - - hash: bytes = sha.digest() - - file_destinations_decls: CXXArrayOfBytesDecl = ( - CXXArrayOfBytesDecl.of_byte_ranges( - id=f"{id}_file_destinations", - data=merged_dests_bytes, - ranges=merged_dests_ranges, - alignment=1, - ) - ) - - binary_file_name = f"{id}_binary.bin" - - binary_decl: CXXBinEmbedDecl = CXXBinEmbedDecl.of(id=f"{id}_binary") - - binary_size_decl: CXXSizeDecl = CXXSizeDecl.of_size( - id=f"{id}_uncompressed_size", size=len(uncompressed_files_bytes) - ) - - binary_embed_decl: AsmEmbedDecl = AsmEmbedDecl.of_file( - id=f"{id}_binary", file=binary_file_name - ) - - binary_ranges_decl = CXXRangesDecl.of_ranges( - id=f"{id}_ranges", ranges=files_ranges - ) - - binary_compression_decl = CXXAsciiStringDecl.of_string( - id=f"{id}_compression", string=compression - ) - - include_directories_decls: CXXArrayOfBytesDecl = ( - CXXArrayOfBytesDecl.of_byte_ranges( - id=f"{id}_include_directories", - data=merged_include_directories_bytes, - ranges=merged_include_directories_ranges, - alignment=1, - ) - ) - - cxx_header = f""" -{file_destinations_decls.var().decl()} - -{binary_decl.var().decl()} - -{binary_size_decl.var().decl()} - -{binary_compression_decl.var().decl()} - -{binary_ranges_decl.var().decl()} - -{include_directories_decls.var().decl()} - -""" - - asm_source = f""" -{binary_embed_decl.decl()} -""" - - return EmbedOutput( - cxx_header=cxx_header, - cxx_source=None, - asm_source=asm_source, - bin_file_name=binary_file_name, - bin_file_data=compressed_files_bytes - if compress - else uncompressed_files_bytes, - hash=hash, - ) - - -def generate_cxx_blobs_data( - id: str, blob_paths: list[str], dests: list[str], compression: str -) -> EmbedOutput: - uncompressed_blob_bytes, blob_ranges = merge_bytes_with_null_terminators( - [load_file_bytes(p) for p in blob_paths] - ) - - compress = compression != "none" - - compressed_blob_bytes = ( - compress_bytes(uncompressed_blob_bytes, compression) - if compress - else None - ) - - merged_dests_bytes, merged_dests_ranges = ( - merge_bytes_with_null_terminators([d.encode("utf-8") for d in dests]) - ) - - if compress: - logger.info( - f"{id}'s uncompressed size is {len(uncompressed_blob_bytes)} bytes, compressed size is {len(compressed_blob_bytes)} bytes" - ) - - # compute combined sha256 hash of all files - sha = hashlib.sha256() - sha.update(uncompressed_blob_bytes) - sha.update(merged_dests_bytes) - sha.update(compression.encode("utf-8")) - - hash: bytes = sha.digest() - - file_destinations_decls: CXXArrayOfBytesDecl = ( - CXXArrayOfBytesDecl.of_byte_ranges( - id=f"{id}_file_destinations", - data=merged_dests_bytes, - ranges=merged_dests_ranges, - alignment=1, - ) - ) - - binary_file_name = f"{id}_binary.bin" - - binary_decl: CXXBinEmbedDecl = CXXBinEmbedDecl.of(id=f"{id}_binary") - - binary_size_decl: CXXSizeDecl = CXXSizeDecl.of_size( - id=f"{id}_uncompressed_size", size=len(uncompressed_blob_bytes) - ) - - binary_compression_decl = CXXAsciiStringDecl.of_string( - id=f"{id}_compression", string=compression - ) - - binary_embed_decl: AsmEmbedDecl = AsmEmbedDecl.of_file( - id=f"{id}_binary", file=binary_file_name - ) - - binary_ranges_decl = CXXRangesDecl.of_ranges( - id=f"{id}_ranges", ranges=blob_ranges - ) - - cxx_header = f""" - -{file_destinations_decls.var().decl()} - -{binary_decl.var().decl()} - -{binary_compression_decl.var().decl()} - -{binary_size_decl.var().decl()} - -{binary_ranges_decl.var().decl()} -""" - - asm_source = f""" -{binary_embed_decl.decl()} -""" - - return EmbedOutput( - cxx_header=cxx_header, - cxx_source=None, - asm_source=asm_source, - bin_file_name=binary_file_name, - bin_file_data=compressed_blob_bytes - if compress - else uncompressed_blob_bytes, - hash=hash, - ) - - -def generate_embed( - id: str, entries: dict[str, dict[str, dict]], output_dir: str -): - outputs: list[EmbedOutput] = [] - sha = hashlib.sha256() - - for entry_id, entry_value in entries.items(): - entry_type = entry_value["type"] - - if entry_type == "sources": - sources: list[dict] = entry_value["sources"] - file_paths = [s["file"] for s in sources] - dests = [s["dest"] for s in sources] - include_directories: list[str] = entry_value["include_directories"] - compression = entry_value["compression"] - output = generate_cxx_source_files_data( - entry_id, file_paths, dests, include_directories, compression - ) - sha.update(output.hash) - outputs.append(output) - - elif entry_type == "strings": - options: list[str] = entry_value["strings"] - output = generate_cxx_strings_data(entry_id, options) - sha.update(output.hash) - outputs.append(output) - - elif entry_type == "blobs": - blobs: list[str] = entry_value["blobs"] - file_paths = [s["file"] for s in blobs] - dests = [s["dest"] for s in blobs] - compression = entry_value["compression"] - output = generate_cxx_blobs_data( - entry_id, file_paths, dests, compression - ) - sha.update(output.hash) - outputs.append(output) - - else: - raise ValueError(f"Unknown type: {entry_type}") - - hash = sha.digest() - - hash_decl: CXXBytesDecl = CXXBytesDecl.of_bytes( - id=f"{id}_hash", - data=hash, - alignment=1, - num_null_terminators=0, - ) - - cxx_header = f""" -{PREAMBLE} - -extern "C" {{ - - -{hash_decl.var().decl()} - -{"\n\n".join([output.cxx_header if output.cxx_header is not None else "" for output in outputs])} - -}} - -""" - - asm_source = f""" -{"\n\n".join([output.asm_source if output.asm_source is not None else "" for output in outputs])} - -.section .note.GNU-stack,"",@progbits -""" - - os.makedirs(output_dir, exist_ok=True) - - with open(f"{output_dir}/embed.hpp", "w") as f: - f.write(cxx_header) - - with open(f"{output_dir}/embed.s", "w") as f: - f.write(asm_source) - - for output in outputs: - if output.bin_file_name and output.bin_file_data: - with open(f"{output_dir}/{output.bin_file_name}", "wb") as f: - f.write(output.bin_file_data) - - -# Usage: embed.py --id --input --output-dir -def main(): - parser = argparse.ArgumentParser( - description="Embed headers, options, or binary blobs into C++ source code." - ) - - parser.add_argument( - "--id", - type=str, - required=True, - help="Identifier for the output", - ) - - parser.add_argument( - "--input", - type=str, - required=True, - help="YAML description of what to embed", - ) - - parser.add_argument( - "--output-dir", - type=str, - required=True, - help="Output directory for generated files", - ) - - args = parser.parse_args() - - with open(args.input, "rb") as f: - description = yaml.safe_load(f) - - generate_embed(args.id, description, args.output_dir) - - -if __name__ == "__main__": - main() diff --git a/cpp/cmake/thirdparty/get_lz4.cmake b/cpp/cmake/thirdparty/get_lz4.cmake index 57473030576b..ddf3c9a68994 100644 --- a/cpp/cmake/thirdparty/get_lz4.cmake +++ b/cpp/cmake/thirdparty/get_lz4.cmake @@ -14,7 +14,7 @@ function(find_and_configure_lz4) GLOBAL_TARGETS lz4 CPM_ARGS GIT_REPOSITORY https://github.com/lz4/lz4.git - GIT_TAG 1519f46a3a11f312be8f5796e8fa4779140277f0 + GIT_TAG 130f457aea830ab80e2e7b438408384fe8eefe07 GIT_SHALLOW TRUE ) diff --git a/cpp/include/cudf/utilities/defer.hpp b/cpp/include/cudf/utilities/defer.hpp deleted file mode 100644 index 075d848b4272..000000000000 --- a/cpp/include/cudf/utilities/defer.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include - -#define CUDF_DEFER__CONCATENATE_DETAIL(x, y) x##y -#define CUDF_DEFER__CONCATENATE(x, y) CUDF_DEFER__CONCATENATE_DETAIL(x, y) -#define CUDF_DEFER(...) ::cudf::defer CUDF_DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) - -namespace CUDF_EXPORT cudf { - -/// @brief RAII utility to execute a callable at the end of a scope. -/// This is useful for ensuring cleanup code is executed, even in the presence of exceptions. -/// And is intended for wrapping C APIs that require explicit resource management without having to -/// write custom wrapper types. -template -struct defer { - private: - T func_; - - public: - /// @brief Construct a `defer` object that will invoke the provided callable upon destruction. - /// @param args Arguments to forward to the callable's constructor. - template - defer(Args&&... args) : func_{static_cast(args)...} - { - } - defer(defer const&) = delete; - defer& operator=(defer const&) = delete; - defer(defer&&) = delete; - defer& operator=(defer&&) = delete; - ~defer() { func_(); } -}; - -template -defer(T) -> defer; ///< Class template argument deduction guide - -} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/utilities/export.hpp b/cpp/include/cudf/utilities/export.hpp index 608a556d6c16..9a8465efd6e5 100644 --- a/cpp/include/cudf/utilities/export.hpp +++ b/cpp/include/cudf/utilities/export.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/jcudf/library.cu b/cpp/include/jcudf/library.cu deleted file mode 100644 index 55f6dc06ebf0..000000000000 --- a/cpp/include/jcudf/library.cu +++ /dev/null @@ -1,5 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - diff --git a/cpp/include/jcudf/README.md b/cpp/include/jit_cudf/README.md similarity index 100% rename from cpp/include/jcudf/README.md rename to cpp/include/jit_cudf/README.md diff --git a/cpp/include/jcudf/functions/abs.cuh b/cpp/include/jit_cudf/functions/abs.cuh similarity index 86% rename from cpp/include/jcudf/functions/abs.cuh rename to cpp/include/jit_cudf/functions/abs.cuh index 15bd02d8f865..e148451d10ad 100644 --- a/cpp/include/jcudf/functions/abs.cuh +++ b/cpp/include/jit_cudf/functions/abs.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void abs(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/add.cuh b/cpp/include/jit_cudf/functions/add.cuh similarity index 87% rename from cpp/include/jcudf/functions/add.cuh rename to cpp/include/jit_cudf/functions/add.cuh index 52a4f4c4dc76..b690cb362a6d 100644 --- a/cpp/include/jcudf/functions/add.cuh +++ b/cpp/include/jit_cudf/functions/add.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void add(optional* out, optional const& a, optional c } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jit_cudf/functions/all.cuh b/cpp/include/jit_cudf/functions/all.cuh new file mode 100644 index 000000000000..90ce14cf5a16 --- /dev/null +++ b/cpp/include/jit_cudf/functions/all.cuh @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/cpp/include/jcudf/functions/arccos.cuh b/cpp/include/jit_cudf/functions/arccos.cuh similarity index 87% rename from cpp/include/jcudf/functions/arccos.cuh rename to cpp/include/jit_cudf/functions/arccos.cuh index 8738a5ae1821..eb729edadad1 100644 --- a/cpp/include/jcudf/functions/arccos.cuh +++ b/cpp/include/jit_cudf/functions/arccos.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void arccos(f32* out, f32 const& a) { *out = __builtin_acosf(a); } @@ -22,4 +23,5 @@ __device__ inline void arccos(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/arccosh.cuh b/cpp/include/jit_cudf/functions/arccosh.cuh similarity index 87% rename from cpp/include/jcudf/functions/arccosh.cuh rename to cpp/include/jit_cudf/functions/arccosh.cuh index 5c7d8def25b8..c121cc0be1d4 100644 --- a/cpp/include/jcudf/functions/arccosh.cuh +++ b/cpp/include/jit_cudf/functions/arccosh.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void arccosh(f32* out, f32 const& a) { *out = __builtin_acoshf(a); } @@ -22,4 +23,5 @@ __device__ inline void arccosh(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/arcsin.cuh b/cpp/include/jit_cudf/functions/arcsin.cuh similarity index 87% rename from cpp/include/jcudf/functions/arcsin.cuh rename to cpp/include/jit_cudf/functions/arcsin.cuh index d37b98fc9589..619951894068 100644 --- a/cpp/include/jcudf/functions/arcsin.cuh +++ b/cpp/include/jit_cudf/functions/arcsin.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void arcsin(f32* out, f32 const& a) { *out = __builtin_asinf(a); } @@ -22,4 +23,5 @@ __device__ inline void arcsin(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/arcsinh.cuh b/cpp/include/jit_cudf/functions/arcsinh.cuh similarity index 87% rename from cpp/include/jcudf/functions/arcsinh.cuh rename to cpp/include/jit_cudf/functions/arcsinh.cuh index e4cb3d9f6be0..a9e6f0ffd5be 100644 --- a/cpp/include/jcudf/functions/arcsinh.cuh +++ b/cpp/include/jit_cudf/functions/arcsinh.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void arcsinh(f32* out, f32 const& a) { *out = __builtin_asinhf(a); } @@ -22,4 +23,5 @@ __device__ inline void arcsinh(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/arctan.cuh b/cpp/include/jit_cudf/functions/arctan.cuh similarity index 87% rename from cpp/include/jcudf/functions/arctan.cuh rename to cpp/include/jit_cudf/functions/arctan.cuh index bf4740b18816..5c8a23e40097 100644 --- a/cpp/include/jcudf/functions/arctan.cuh +++ b/cpp/include/jit_cudf/functions/arctan.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void arctan(f32* out, f32 const& a) { *out = __builtin_atanf(a); } @@ -22,4 +23,5 @@ __device__ inline void arctan(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/arctanh.cuh b/cpp/include/jit_cudf/functions/arctanh.cuh similarity index 87% rename from cpp/include/jcudf/functions/arctanh.cuh rename to cpp/include/jit_cudf/functions/arctanh.cuh index bdca05c9d252..dcc25aef6f78 100644 --- a/cpp/include/jcudf/functions/arctanh.cuh +++ b/cpp/include/jit_cudf/functions/arctanh.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void arctanh(f32* out, f32 const& a) { *out = __builtin_atanhf(a); } @@ -22,4 +23,5 @@ __device__ inline void arctanh(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/bit_and.cuh b/cpp/include/jit_cudf/functions/bit_and.cuh similarity index 87% rename from cpp/include/jcudf/functions/bit_and.cuh rename to cpp/include/jit_cudf/functions/bit_and.cuh index ecb1bff50a10..7d9aea7d5e84 100644 --- a/cpp/include/jcudf/functions/bit_and.cuh +++ b/cpp/include/jit_cudf/functions/bit_and.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void bit_and(optional* out, optional const& a, optional< } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/bit_invert.cuh b/cpp/include/jit_cudf/functions/bit_invert.cuh similarity index 86% rename from cpp/include/jcudf/functions/bit_invert.cuh rename to cpp/include/jit_cudf/functions/bit_invert.cuh index 6344c04a53ab..6251641f4f76 100644 --- a/cpp/include/jcudf/functions/bit_invert.cuh +++ b/cpp/include/jit_cudf/functions/bit_invert.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void bit_invert(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/bit_or.cuh b/cpp/include/jit_cudf/functions/bit_or.cuh similarity index 87% rename from cpp/include/jcudf/functions/bit_or.cuh rename to cpp/include/jit_cudf/functions/bit_or.cuh index 4fcf7e90a620..019d864ab9c8 100644 --- a/cpp/include/jcudf/functions/bit_or.cuh +++ b/cpp/include/jit_cudf/functions/bit_or.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void bit_or(optional* out, optional const& a, optional @@ -24,4 +25,5 @@ __device__ inline void bit_xor(optional* out, optional const& a, optional< } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/cast_to_f64.cuh b/cpp/include/jit_cudf/functions/cast_to_f64.cuh similarity index 86% rename from cpp/include/jcudf/functions/cast_to_f64.cuh rename to cpp/include/jit_cudf/functions/cast_to_f64.cuh index 9bdfa7945cf5..9f9bbb01e55e 100644 --- a/cpp/include/jcudf/functions/cast_to_f64.cuh +++ b/cpp/include/jit_cudf/functions/cast_to_f64.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void cast_to_f64(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/cast_to_i64.cuh b/cpp/include/jit_cudf/functions/cast_to_i64.cuh similarity index 86% rename from cpp/include/jcudf/functions/cast_to_i64.cuh rename to cpp/include/jit_cudf/functions/cast_to_i64.cuh index 9b5611254f52..d9d2da81471f 100644 --- a/cpp/include/jcudf/functions/cast_to_i64.cuh +++ b/cpp/include/jit_cudf/functions/cast_to_i64.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void cast_to_i64(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/cast_to_u64.cuh b/cpp/include/jit_cudf/functions/cast_to_u64.cuh similarity index 86% rename from cpp/include/jcudf/functions/cast_to_u64.cuh rename to cpp/include/jit_cudf/functions/cast_to_u64.cuh index 23d9e7b60fb7..bf6e23f7a55d 100644 --- a/cpp/include/jcudf/functions/cast_to_u64.cuh +++ b/cpp/include/jit_cudf/functions/cast_to_u64.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void cast_to_u64(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/cbrt.cuh b/cpp/include/jit_cudf/functions/cbrt.cuh similarity index 87% rename from cpp/include/jcudf/functions/cbrt.cuh rename to cpp/include/jit_cudf/functions/cbrt.cuh index ac0fac4b4183..29f61f9fc9e9 100644 --- a/cpp/include/jcudf/functions/cbrt.cuh +++ b/cpp/include/jit_cudf/functions/cbrt.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void cbrt(f32* out, f32 const& a) { *out = __builtin_cbrtf(a); } @@ -22,4 +23,5 @@ __device__ inline void cbrt(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/ceil.cuh b/cpp/include/jit_cudf/functions/ceil.cuh similarity index 87% rename from cpp/include/jcudf/functions/ceil.cuh rename to cpp/include/jit_cudf/functions/ceil.cuh index c6a98bda1787..82ca69aca375 100644 --- a/cpp/include/jcudf/functions/ceil.cuh +++ b/cpp/include/jit_cudf/functions/ceil.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void ceil(f32* out, f32 const& a) { *out = __builtin_ceilf(a); } @@ -22,4 +23,5 @@ __device__ inline void ceil(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/cos.cuh b/cpp/include/jit_cudf/functions/cos.cuh similarity index 87% rename from cpp/include/jcudf/functions/cos.cuh rename to cpp/include/jit_cudf/functions/cos.cuh index 6fb1ca08905c..d5596b78cc2c 100644 --- a/cpp/include/jcudf/functions/cos.cuh +++ b/cpp/include/jit_cudf/functions/cos.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void cos(f32* out, f32 const& a) { *out = __builtin_cosf(a); } @@ -22,4 +23,5 @@ __device__ inline void cos(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/cosh.cuh b/cpp/include/jit_cudf/functions/cosh.cuh similarity index 87% rename from cpp/include/jcudf/functions/cosh.cuh rename to cpp/include/jit_cudf/functions/cosh.cuh index c5144d21bcc3..633c7f801cd7 100644 --- a/cpp/include/jcudf/functions/cosh.cuh +++ b/cpp/include/jit_cudf/functions/cosh.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void cosh(f32* out, f32 const& a) { *out = __builtin_coshf(a); } @@ -22,4 +23,5 @@ __device__ inline void cosh(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/div.cuh b/cpp/include/jit_cudf/functions/div.cuh similarity index 87% rename from cpp/include/jcudf/functions/div.cuh rename to cpp/include/jit_cudf/functions/div.cuh index f8cd1396a615..5a1883fcb1c9 100644 --- a/cpp/include/jcudf/functions/div.cuh +++ b/cpp/include/jit_cudf/functions/div.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void div(optional* out, optional const& a, optional c } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/equal.cuh b/cpp/include/jit_cudf/functions/equal.cuh similarity index 88% rename from cpp/include/jcudf/functions/equal.cuh rename to cpp/include/jit_cudf/functions/equal.cuh index c40aa7874d36..4e0c4a1a6477 100644 --- a/cpp/include/jcudf/functions/equal.cuh +++ b/cpp/include/jit_cudf/functions/equal.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void equal(optional* out, optional const& a, optional } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/exp.cuh b/cpp/include/jit_cudf/functions/exp.cuh similarity index 87% rename from cpp/include/jcudf/functions/exp.cuh rename to cpp/include/jit_cudf/functions/exp.cuh index 205f420fc84f..e8e1e44f3674 100644 --- a/cpp/include/jcudf/functions/exp.cuh +++ b/cpp/include/jit_cudf/functions/exp.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void exp(f32* out, f32 const& a) { *out = __builtin_expf(a); } @@ -22,4 +23,5 @@ __device__ inline void exp(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/floor.cuh b/cpp/include/jit_cudf/functions/floor.cuh similarity index 87% rename from cpp/include/jcudf/functions/floor.cuh rename to cpp/include/jit_cudf/functions/floor.cuh index 490fbc53f2dd..5065c2b90c6f 100644 --- a/cpp/include/jcudf/functions/floor.cuh +++ b/cpp/include/jit_cudf/functions/floor.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void floor(f32* out, f32 const& a) { *out = __builtin_floorf(a); } @@ -22,4 +23,5 @@ __device__ inline void floor(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/greater.cuh b/cpp/include/jit_cudf/functions/greater.cuh similarity index 86% rename from cpp/include/jcudf/functions/greater.cuh rename to cpp/include/jit_cudf/functions/greater.cuh index 9a9bd9ffad6b..0e76a98a321c 100644 --- a/cpp/include/jcudf/functions/greater.cuh +++ b/cpp/include/jit_cudf/functions/greater.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -22,4 +23,5 @@ __device__ inline void greater(optional* out, optional const& a, option } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/greater_equal.cuh b/cpp/include/jit_cudf/functions/greater_equal.cuh similarity index 61% rename from cpp/include/jcudf/functions/greater_equal.cuh rename to cpp/include/jit_cudf/functions/greater_equal.cuh index d926a7829753..bb38cb9a70ae 100644 --- a/cpp/include/jcudf/functions/greater_equal.cuh +++ b/cpp/include/jit_cudf/functions/greater_equal.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -12,7 +13,9 @@ __device__ inline void greater_equal(bool* out, T const& a, T const& b) } template -__device__ inline void greater_equal(optional* out, optional const& a, optional const& b) +__device__ inline void greater_equal(optional* out, + optional const& a, + optional const& b) { if (a.has_value() && b.has_value()) { *out = (*a >= *b); @@ -22,4 +25,5 @@ __device__ inline void greater_equal(optional* out, optional const& a, } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/identity.cuh b/cpp/include/jit_cudf/functions/identity.cuh similarity index 83% rename from cpp/include/jcudf/functions/identity.cuh rename to cpp/include/jit_cudf/functions/identity.cuh index 3e4b867a81ff..4ee467f6230d 100644 --- a/cpp/include/jcudf/functions/identity.cuh +++ b/cpp/include/jit_cudf/functions/identity.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -18,4 +19,5 @@ __device__ inline void identity(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/is_null.cuh b/cpp/include/jit_cudf/functions/is_null.cuh similarity index 83% rename from cpp/include/jcudf/functions/is_null.cuh rename to cpp/include/jit_cudf/functions/is_null.cuh index 374919c246c5..82db3952ab18 100644 --- a/cpp/include/jcudf/functions/is_null.cuh +++ b/cpp/include/jit_cudf/functions/is_null.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -18,4 +19,5 @@ __device__ inline void is_null(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/less.cuh b/cpp/include/jit_cudf/functions/less.cuh similarity index 86% rename from cpp/include/jcudf/functions/less.cuh rename to cpp/include/jit_cudf/functions/less.cuh index 0a2d0d7e063b..25c4aa32ffe1 100644 --- a/cpp/include/jcudf/functions/less.cuh +++ b/cpp/include/jit_cudf/functions/less.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -22,4 +23,5 @@ __device__ inline void less(optional* out, optional const& a, optional< } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/less_equal.cuh b/cpp/include/jit_cudf/functions/less_equal.cuh similarity index 87% rename from cpp/include/jcudf/functions/less_equal.cuh rename to cpp/include/jit_cudf/functions/less_equal.cuh index 701695c01b8b..2d881cd97b2f 100644 --- a/cpp/include/jcudf/functions/less_equal.cuh +++ b/cpp/include/jit_cudf/functions/less_equal.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -22,4 +23,5 @@ __device__ inline void less_equal(optional* out, optional const& a, opt } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/log.cuh b/cpp/include/jit_cudf/functions/log.cuh similarity index 87% rename from cpp/include/jcudf/functions/log.cuh rename to cpp/include/jit_cudf/functions/log.cuh index 549d7524c9cc..63b382cf82d9 100644 --- a/cpp/include/jcudf/functions/log.cuh +++ b/cpp/include/jit_cudf/functions/log.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void log(f32* out, f32 const& a) { *out = __builtin_logf(a); } @@ -22,4 +23,5 @@ __device__ inline void log(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/logical_and.cuh b/cpp/include/jit_cudf/functions/logical_and.cuh similarity index 87% rename from cpp/include/jcudf/functions/logical_and.cuh rename to cpp/include/jit_cudf/functions/logical_and.cuh index 185259fdf5ae..52d838d932de 100644 --- a/cpp/include/jcudf/functions/logical_and.cuh +++ b/cpp/include/jit_cudf/functions/logical_and.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void logical_and(optional* out, optional const& a, optio } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/logical_or.cuh b/cpp/include/jit_cudf/functions/logical_or.cuh similarity index 87% rename from cpp/include/jcudf/functions/logical_or.cuh rename to cpp/include/jit_cudf/functions/logical_or.cuh index 755d5a17ad50..e257f1feb0b4 100644 --- a/cpp/include/jcudf/functions/logical_or.cuh +++ b/cpp/include/jit_cudf/functions/logical_or.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void logical_or(optional* out, optional const& a, option } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/mod.cuh b/cpp/include/jit_cudf/functions/mod.cuh similarity index 90% rename from cpp/include/jcudf/functions/mod.cuh rename to cpp/include/jit_cudf/functions/mod.cuh index b85a17230681..3d603d92cff4 100644 --- a/cpp/include/jcudf/functions/mod.cuh +++ b/cpp/include/jit_cudf/functions/mod.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -28,4 +29,5 @@ __device__ inline void mod(optional* out, optional const& a, optional c } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/mul.cuh b/cpp/include/jit_cudf/functions/mul.cuh similarity index 87% rename from cpp/include/jcudf/functions/mul.cuh rename to cpp/include/jit_cudf/functions/mul.cuh index 65b27b44e9b9..89f3410be450 100644 --- a/cpp/include/jcudf/functions/mul.cuh +++ b/cpp/include/jit_cudf/functions/mul.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void mul(optional* out, optional const& a, optional c } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/null_equal.cuh b/cpp/include/jit_cudf/functions/null_equal.cuh similarity index 88% rename from cpp/include/jcudf/functions/null_equal.cuh rename to cpp/include/jit_cudf/functions/null_equal.cuh index 7bc83e40c493..20cdcf9b51a9 100644 --- a/cpp/include/jcudf/functions/null_equal.cuh +++ b/cpp/include/jit_cudf/functions/null_equal.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void null_equal(optional* out, optional const& a, opt } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/null_logical_and.cuh b/cpp/include/jit_cudf/functions/null_logical_and.cuh similarity index 66% rename from cpp/include/jcudf/functions/null_logical_and.cuh rename to cpp/include/jit_cudf/functions/null_logical_and.cuh index 4032f7b44198..1962ee57c874 100644 --- a/cpp/include/jcudf/functions/null_logical_and.cuh +++ b/cpp/include/jit_cudf/functions/null_logical_and.cuh @@ -2,12 +2,15 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void null_logical_and(bool* out, bool const& a, bool const& b) { *out = a && b; } -__device__ inline void null_logical_and(optional* out, optional const& a, optional const& b) +__device__ inline void null_logical_and(optional* out, + optional const& a, + optional const& b) { if (a.has_value() && b.has_value()) { *out = (*a && *b); @@ -24,4 +27,5 @@ __device__ inline void null_logical_and(optional* out, optional cons } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/null_logical_or.cuh b/cpp/include/jit_cudf/functions/null_logical_or.cuh similarity index 66% rename from cpp/include/jcudf/functions/null_logical_or.cuh rename to cpp/include/jit_cudf/functions/null_logical_or.cuh index 74d5b4322889..6d9d2efa77ae 100644 --- a/cpp/include/jcudf/functions/null_logical_or.cuh +++ b/cpp/include/jit_cudf/functions/null_logical_or.cuh @@ -2,12 +2,15 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void null_logical_or(bool* out, bool const& a, bool const& b) { *out = a || b; } -__device__ inline void null_logical_or(optional* out, optional const& a, optional const& b) +__device__ inline void null_logical_or(optional* out, + optional const& a, + optional const& b) { if (a.has_value() && b.has_value()) { *out = (*a || *b); @@ -24,4 +27,5 @@ __device__ inline void null_logical_or(optional* out, optional const } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/pow.cuh b/cpp/include/jit_cudf/functions/pow.cuh similarity index 88% rename from cpp/include/jcudf/functions/pow.cuh rename to cpp/include/jit_cudf/functions/pow.cuh index 7dbf82f296a9..245503b6ce39 100644 --- a/cpp/include/jcudf/functions/pow.cuh +++ b/cpp/include/jit_cudf/functions/pow.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void pow(f32* out, f32 const& a, f32 const& b) { *out = __builtin_powf(a, b); } @@ -22,4 +23,5 @@ __device__ inline void pow(optional* out, optional const& a, optional c } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/pymod.cuh b/cpp/include/jit_cudf/functions/pymod.cuh similarity index 91% rename from cpp/include/jcudf/functions/pymod.cuh rename to cpp/include/jit_cudf/functions/pymod.cuh index 4ee1ff7a37f6..a4c408f97bc2 100644 --- a/cpp/include/jcudf/functions/pymod.cuh +++ b/cpp/include/jit_cudf/functions/pymod.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -34,4 +35,5 @@ __device__ inline void pymod(optional* out, optional const& a, optional } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/rint.cuh b/cpp/include/jit_cudf/functions/rint.cuh similarity index 87% rename from cpp/include/jcudf/functions/rint.cuh rename to cpp/include/jit_cudf/functions/rint.cuh index dda95e2d81c6..1feef0f4daaf 100644 --- a/cpp/include/jcudf/functions/rint.cuh +++ b/cpp/include/jit_cudf/functions/rint.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void rint(f32* out, f32 const& a) { *out = __builtin_rintf(a); } @@ -22,4 +23,5 @@ __device__ inline void rint(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/sin.cuh b/cpp/include/jit_cudf/functions/sin.cuh similarity index 87% rename from cpp/include/jcudf/functions/sin.cuh rename to cpp/include/jit_cudf/functions/sin.cuh index 1c8312b8c04f..8291eaa22fd6 100644 --- a/cpp/include/jcudf/functions/sin.cuh +++ b/cpp/include/jit_cudf/functions/sin.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void sin(f32* out, f32 const& a) { *out = __builtin_sinf(a); } @@ -22,4 +23,5 @@ __device__ inline void sin(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/sinh.cuh b/cpp/include/jit_cudf/functions/sinh.cuh similarity index 87% rename from cpp/include/jcudf/functions/sinh.cuh rename to cpp/include/jit_cudf/functions/sinh.cuh index b9b5954635fc..293ae222e2d2 100644 --- a/cpp/include/jcudf/functions/sinh.cuh +++ b/cpp/include/jit_cudf/functions/sinh.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void sinh(f32* out, f32 const& a) { *out = __builtin_sinhf(a); } @@ -22,4 +23,5 @@ __device__ inline void sinh(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/sub.cuh b/cpp/include/jit_cudf/functions/sub.cuh similarity index 87% rename from cpp/include/jcudf/functions/sub.cuh rename to cpp/include/jit_cudf/functions/sub.cuh index 45c32c11ba95..4614ca3ffc74 100644 --- a/cpp/include/jcudf/functions/sub.cuh +++ b/cpp/include/jit_cudf/functions/sub.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { template @@ -24,4 +25,5 @@ __device__ inline void sub(optional* out, optional const& a, optional c } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/functions/tanh.cuh b/cpp/include/jit_cudf/functions/tanh.cuh similarity index 87% rename from cpp/include/jcudf/functions/tanh.cuh rename to cpp/include/jit_cudf/functions/tanh.cuh index c88cd6089534..664da0811e3c 100644 --- a/cpp/include/jcudf/functions/tanh.cuh +++ b/cpp/include/jit_cudf/functions/tanh.cuh @@ -2,7 +2,8 @@ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -namespace jcudf { +namespace jit { +namespace cudf { namespace functions { __device__ inline void tanh(f32* out, f32 const& a) { *out = __builtin_tanhf(a); } @@ -22,4 +23,5 @@ __device__ inline void tanh(optional* out, optional const& a) } } // namespace functions -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/include/jcudf/types.cuh b/cpp/include/jit_cudf/types.cuh similarity index 99% rename from cpp/include/jcudf/types.cuh rename to cpp/include/jit_cudf/types.cuh index 34f587a2b362..dad4f69d4c0c 100644 --- a/cpp/include/jcudf/types.cuh +++ b/cpp/include/jit_cudf/types.cuh @@ -4,7 +4,8 @@ */ #pragma once -namespace jcudf { +namespace jit { +namespace cudf { using i8 = signed char; using i16 = signed short; @@ -667,4 +668,5 @@ struct user_data { }; } // namespace scope -} // namespace jcudf +} // namespace cudf +} // namespace jit diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index e4c4bf121357..9fc1bb9fbf48 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -6,17 +6,14 @@ #include #include -#include #include #include +#include #include #include -#include -#include -#include -#include +#include #include #include #include @@ -45,9 +42,9 @@ namespace CUDF_EXPORT cudf { namespace { -rtc::sha256_hash hash_string(std::span input) +rtcx::sha256 hash_string(std::span input) { - rtc::sha256_context ctx; + rtcx::sha256_context ctx; ctx.update(std::span{reinterpret_cast(input.data()), input.size()}); return ctx.finalize(); } @@ -71,7 +68,7 @@ void install_file(char const* dst_path, std::span contents) throw_posix(std::format("Failed to create file ({})", dst_path), "open"); } - CUDF_DEFER([&] { + RTCX_DEFER([&] { if (close(dst_file) != 0) { throw_posix(std::format("Failed to close file ({})", dst_path), "close"); } @@ -82,7 +79,7 @@ void install_file(char const* dst_path, std::span contents) } } -std::vector decompress_blob(jit_bytes_t const& compressed_binary, +std::vector decompress_blob(std::span compressed_binary, size_t uncompressed_size, char const* compression) { @@ -90,9 +87,9 @@ std::vector decompress_blob(jit_bytes_t const& compressed_binary, decompressed.resize(uncompressed_size); if (std::string_view{compression} == "lz4") { - int errc = LZ4_decompress_safe(reinterpret_cast(compressed_binary.data), + int errc = LZ4_decompress_safe(reinterpret_cast(compressed_binary.data()), reinterpret_cast(decompressed.data()), - compressed_binary.size, + compressed_binary.size(), uncompressed_size); CUDF_EXPECTS( @@ -102,7 +99,7 @@ std::vector decompress_blob(jit_bytes_t const& compressed_binary, } else if (std::string_view{compression} == "zstd") { size_t const errc = ZSTD_decompress( - decompressed.data(), uncompressed_size, compressed_binary.data, compressed_binary.size); + decompressed.data(), uncompressed_size, compressed_binary.data(), compressed_binary.size()); CUDF_EXPECTS( !ZSTD_isError(errc) && errc == uncompressed_size, @@ -112,18 +109,19 @@ std::vector decompress_blob(jit_bytes_t const& compressed_binary, std::runtime_error); } else { // compression is "none", so just copy the data - std::copy( - compressed_binary.data, compressed_binary.data + compressed_binary.size, decompressed.data()); + std::copy(compressed_binary.data(), + compressed_binary.data() + compressed_binary.size(), + decompressed.data()); } return decompressed; } void install_file_set(char const* target_dir, - jit_bytes_t const& compressed_binary, + std::span compressed_binary, size_t uncompressed_size, - std::span file_ranges, - jit_bytes_array_t const& dst, + std::span file_ranges, + std::span destinations, char const* compression) { CUDF_EXPECTS(compression != nullptr, "Compression type must be specified", std::runtime_error); @@ -137,10 +135,8 @@ void install_file_set(char const* target_dir, for (size_t i = 0; i < file_ranges.size(); ++i) { auto file_data_range = file_ranges[i]; - auto dst_range = dst.ranges[i]; auto file_data = std::span{files_data + file_data_range.offset, file_data_range.size}; - auto dst_path = std::string_view{ - reinterpret_cast(dst.bytes.data) + dst_range.offset, dst_range.size}; + auto dst_path = destinations[i]; auto target_path = std::format("{}/{}", target_dir, dst_path); @@ -149,24 +145,7 @@ void install_file_set(char const* target_dir, } } -void install_cudf_jit(char const* target_dir) -{ - install_file_set(target_dir, - cudf_jit_embed_blobs_binary, - cudf_jit_embed_blobs_uncompressed_size, - cudf_jit_embed_blobs_ranges, - cudf_jit_embed_blobs_file_destinations, - cudf_jit_embed_blobs_compression); - - install_file_set(target_dir, - cudf_jit_embed_sources_binary, - cudf_jit_embed_sources_uncompressed_size, - cudf_jit_embed_sources_ranges, - cudf_jit_embed_sources_file_destinations, - cudf_jit_embed_sources_compression); -} - -void create_and_install_cudf_jit(char const* target_dir) +void install_cudf_jit_files(char const* target_dir) { // directory does not exist, so create it char tmp_dir_[] = "/tmp/cudf-jit-tmpdir_XXXXXX"; @@ -177,7 +156,12 @@ void create_and_install_cudf_jit(char const* target_dir) "mkdtemp"); } - install_cudf_jit(tmp_dir); + install_file_set(target_dir, + rtcx_embed::cudf_jit_embed_files, + rtcx_embed::cudf_jit_embed_files_uncompressed_size, + rtcx_embed::cudf_jit_embed_file_ranges, + rtcx_embed::cudf_jit_embed_file_destinations, + rtcx_embed::cudf_jit_embed_files_compression); // rename the temporary directory to the target install directory if (rename(tmp_dir, target_dir) == -1) { @@ -188,11 +172,10 @@ void create_and_install_cudf_jit(char const* target_dir) } // namespace -jit_bundle_t::jit_bundle_t(std::string install_dir, rtc::cache_t& cache) +jit_bundle_t::jit_bundle_t(std::string install_dir, rtcx::cache_t& cache) : install_dir_{std::move(install_dir)}, cache_{&cache} { ensure_installed(); - preload_lto_library(); // TODO: fix cmake tracking of the scripts and embedded files } @@ -212,7 +195,7 @@ void jit_bundle_t::ensure_installed() const // ensure base install directory exists CUDF_LOG_INFO("Creating JIT install directory at ({})", expected_path); std::filesystem::create_directories(install_dir_); - create_and_install_cudf_jit(expected_path.c_str()); + install_cudf_jit_files(expected_path.c_str()); } } else { // directory exists, perform minor sanity check @@ -222,43 +205,9 @@ void jit_bundle_t::ensure_installed() const } } -void jit_bundle_t::preload_lto_library() -{ - auto& cache = *cache_; - - auto bundle_hash = get_hash(); - - auto cache_key = std::format(R"***(fragment_type=FATBIN -key={} -bundle={})***", - "cudf_lto_library", - bundle_hash); - - auto cache_key_sha256 = hash_string(cache_key); - - auto compile = [&] { - auto path = std::format("{}/{}", get_directory(), "cudf_lto_library.fatbin"); - auto cubin = rtc::blob_t::from_file(path.c_str()); - CUDF_EXPECTS(cubin.has_value(), - +std::format("Failed to load LTO library cubin from disk at ({})", path), - std::runtime_error); - rtc::fragment_t::load_params load_params{ - .binary = std::make_shared(std::move(*cubin)), .type = rtc::binary_type::FATBIN}; - return rtc::fragment_t::load(load_params); - }; - - auto fut = - cache.query_or_insert_fragment(cache_key_sha256, - rtc::binary_type::FATBIN, - rtc::fragment_compile_function_t::from_functor(compile)); - - lto_library_ = fut.get(); -} - std::string jit_bundle_t::get_hash() const { - auto str = rtc::sha256_hex_string::make( - std::span{cudf_jit_embed_hash.data, static_cast(cudf_jit_embed_hash.size)}); + auto str = rtcx::sha256_hex_string::make(rtcx_embed::cudf_jit_embed_hash); return std::string{str.view()}; } @@ -267,40 +216,18 @@ std::string jit_bundle_t::get_directory() const return std::format("{}/{}", install_dir_, get_hash()); } -rtc::fragment jit_bundle_t::get_lto_library() const { return lto_library_; } - std::vector jit_bundle_t::get_include_directories() const { std::vector directories; auto base_dir = get_directory(); - auto include_directories_data = - reinterpret_cast(cudf_jit_embed_sources_include_directories.bytes.data); - - for (size_t i = 0; i < cudf_jit_embed_sources_include_directories.num_ranges; i++) { - auto range = cudf_jit_embed_sources_include_directories.ranges[i]; - auto dest_include_directory = include_directories_data + range.offset; - directories.emplace_back(std::format("{}/{}", base_dir, dest_include_directory)); + for (auto dir : rtcx_embed::cudf_jit_embed_include_directories) { + directories.emplace_back(std::format("{}/{}", base_dir, dir)); } return directories; } -std::vector jit_bundle_t::get_compile_options() const -{ - std::vector options; - - auto embed_options_data = reinterpret_cast(cudf_jit_embed_options.bytes.data); - - for (size_t i = 0; i < cudf_jit_embed_options.num_ranges; i++) { - auto range = cudf_jit_embed_options.ranges[i]; - auto option = embed_options_data + range.offset; - options.emplace_back(option); - } - - return options; -} - namespace { int32_t get_driver_version() @@ -331,10 +258,10 @@ int32_t get_current_device_physical_model() return props.major * 10 + props.minor; } -rtc::fragment compile_udf_to_lto_ir_uncached(char const* name, - char const* cuda_code, - bool use_pch, - bool log_pch) +std::tuple compile_library_uncached(char const* name, + char const* cuda_code, + bool use_pch, + bool log_pch) { CUDF_FUNC_RANGE(); @@ -342,9 +269,8 @@ rtc::fragment compile_udf_to_lto_ir_uncached(char const* name, auto begin = std::chrono::steady_clock::now(); auto sm = get_current_device_physical_model(); - auto include_dirs = bundle.get_include_directories(); - auto compile_options = bundle.get_compile_options(); - auto pch_dir = cudf::get_context().get_jit_pch_dir(); + auto include_dirs = bundle.get_include_directories(); + auto pch_dir = cudf::get_context().get_jit_pch_dir(); std::vector options; @@ -352,93 +278,14 @@ rtc::fragment compile_udf_to_lto_ir_uncached(char const* name, options.emplace_back(std::format("-I{}", include_dir)); } - for (auto const& compile_option : compile_options) { - options.emplace_back(compile_option); - } - // TODO: experiment with: // --fdevice-time-trace=jit_comp_trace.json // --time=compile_trace.json // -time + // --restrict options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); - options.emplace_back("--dlink-time-opt"); - options.emplace_back("--relocatable-device-code=true"); - options.emplace_back("--device-as-default-execution-space"); - options.emplace_back("--restrict"); options.emplace_back("--minimal"); - options.emplace_back("--split-compile=0"); - - if (use_pch) { - options.emplace_back("--pch"); - options.emplace_back(std::format("--pch-dir={}", pch_dir)); - - if (log_pch) { - options.emplace_back("--pch-verbose=true"); - options.emplace_back("--pch-messages=true"); - } - } - - std::vector options_cstr; - for (auto const& option : options) { - options_cstr.emplace_back(option.c_str()); - } - - auto params = rtc::fragment_t::compile_params{.name = name, - .source = cuda_code, - .headers = {}, - .options = options_cstr, - .target_type = rtc::binary_type::LTO_IR}; - - auto frag = rtc::fragment_t::compile(params); - - auto end = std::chrono::steady_clock::now(); - - auto duration = end - begin; - - CUDF_LOG_WARN( - "Compiled fragment `%s` in %f ms", - name, - std::chrono::duration_cast>(duration).count()); - - return frag; -} - -std::tuple compile_kernel_to_cubin_uncached(char const* name, - char const* cuda_code, - bool use_pch, - bool log_pch) -{ - CUDF_FUNC_RANGE(); - - auto& bundle = cudf::get_context().jit_bundle(); - auto begin = std::chrono::steady_clock::now(); - auto sm = get_current_device_physical_model(); - - auto include_dirs = bundle.get_include_directories(); - auto compile_options = bundle.get_compile_options(); - auto pch_dir = cudf::get_context().get_jit_pch_dir(); - - std::vector options; - - for (auto const& include_dir : include_dirs) { - options.emplace_back(std::format("-I{}", include_dir)); - } - - for (auto const& compile_option : compile_options) { - options.emplace_back(compile_option); - } - - // TODO: experiment with: - // --fdevice-time-trace=jit_comp_trace.json - // --time=compile_trace.json - // -time - - options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); - options.emplace_back("--restrict"); - options.emplace_back("--minimal"); - options.emplace_back("--split-compile=0"); - auto pch_file = std::format("/home/coder/cudf/jit.pch"); @@ -447,10 +294,10 @@ std::tuple compile_kernel_to_cubin_uncached(char const* if (use_pch) { // options.emplace_back("--pch"); options.emplace_back(std::format("--pch-dir={}", pch_dir)); - if(can_use){ - options.emplace_back(std::format("--use-pch={}", pch_file)); - } else{ - options.emplace_back(std::format("--create-pch={}", pch_file)); + if (can_use) { + options.emplace_back(std::format("--use-pch={}", pch_file)); + } else { + options.emplace_back(std::format("--create-pch={}", pch_file)); } can_use = 1; @@ -465,189 +312,38 @@ std::tuple compile_kernel_to_cubin_uncached(char const* options_cstr.emplace_back(option.c_str()); } - auto params = rtc::fragment_t::compile_params{.name = name, - .source = cuda_code, - .headers = {}, - .options = options_cstr, - .target_type = rtc::binary_type::CUBIN}; + auto params = rtcx::compile_params{.name = name, + .source = cuda_code, + .headers = {}, + .options = options_cstr, + .target_type = rtcx::binary_type::CUBIN}; - auto cubin = rtc::fragment_t::compile_raw(params); + auto cubin = rtcx::compile(params); auto end = std::chrono::steady_clock::now(); auto duration = end - begin; - CUDF_LOG_WARN( + CUDF_LOG_INFO( "Compiled fragment `%s` in %f ms", name, std::chrono::duration_cast>(duration).count()); - auto library = rtc::library_t::load( - rtc::library_t::load_params{.binary = cubin, .type = rtc::binary_type::CUBIN}); + auto library = rtcx::load_library(cubin, rtcx::binary_type::CUBIN); - auto blob = rtc::blob_t::from_vector(std::move(cubin)); + auto blob = rtcx::blob_t::from_vector(std::move(cubin)); - return std::make_tuple(library, std::make_shared(std::move(blob))); + return std::make_tuple(library, std::make_shared(std::move(blob))); } -std::tuple link_udf_lto_ir_uncached(char const* name, - rtc::fragment const& fragment, - char const* kernel_symbol) -{ - CUDF_FUNC_RANGE(); - - auto sm = get_current_device_physical_model(); - auto& bundle = cudf::get_context().jit_bundle(); - - auto begin = std::chrono::steady_clock::now(); - auto library = bundle.get_lto_library(); - - // TODO: sass dump - // TODO: time trace dump - // TODO: lineinfo and debug info options - // TODO: -nocache - - std::vector options; - - options.emplace_back("-O3"); - options.emplace_back("-lto"); - options.emplace_back(std::format("-arch=sm_{}", sm)); - options.emplace_back(std::format("-kernels-used={}", kernel_symbol)); - options.emplace_back("-optimize-unused-variables"); - options.emplace_back("-split-compile=0"); - - std::vector options_cstr; - for (auto const& option : options) { - options_cstr.emplace_back(option.c_str()); - } - - rtc::blob_view link_fragments[] = {library->get(rtc::binary_type::FATBIN)->view(), - fragment->get(rtc::binary_type::LTO_IR)->view()}; - - rtc::binary_type fragment_binary_types[] = {rtc::binary_type::FATBIN, rtc::binary_type::LTO_IR}; - - char const* fragment_names[] = {"cudf_lto_library", name}; - - auto params = rtc::library_t::link_params{.name = name, - .output_type = rtc::binary_type::CUBIN, - .fragments = link_fragments, - .fragment_binary_types = fragment_binary_types, - .fragment_names = fragment_names, - .link_options = options_cstr}; - - auto blob = rtc::library_t::link_as_blob(params); - - auto load_params = - rtc::library_t::load_params{.binary = blob->view(), .type = rtc::binary_type::CUBIN}; - - auto linked_library = rtc::library_t::load(load_params); - - auto end = std::chrono::steady_clock::now(); - - auto duration = end - begin; - - CUDF_LOG_WARN( - "Linked fragment `%s` in %f ms", - name, - std::chrono::duration_cast>(duration).count()); - - return std::make_tuple(linked_library, blob); -} - -rtc::fragment compile_udf_to_lto_ir(char const* name, - char const* key, - char const* cuda_udf, - bool use_cache, - bool use_pch, - bool log_pch) -{ - CUDF_FUNC_RANGE(); - - auto& cache = cudf::get_context().rtc_cache(); - auto& bundle = cudf::get_context().jit_bundle(); - - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_physical_model(); - auto bundle_hash = bundle.get_hash(); - - auto cache_key = std::format(R"***(fragment_type=LTO_IR -key={} -cuda_runtime={} -cuda_driver={} -arch={} -bundle={})***", - key, - runtime, - driver, - sm, - bundle_hash); - - auto cache_key_sha256 = hash_string(cache_key); - - auto compile = [&] { return compile_udf_to_lto_ir_uncached(name, cuda_udf, use_pch, log_pch); }; - - if (!use_cache) { return compile(); } - - auto fut = - cache.query_or_insert_fragment(cache_key_sha256, - rtc::binary_type::LTO_IR, - rtc::fragment_compile_function_t::from_functor(compile)); - - return fut.get(); -} - -rtc::library link_lto_ir_udf(char const* name, - char const* key, - rtc::fragment const& fragment, - char const* kernel_symbol, - bool use_cache) -{ - CUDF_FUNC_RANGE(); - - auto& cache = cudf::get_context().rtc_cache(); - auto& bundle = cudf::get_context().jit_bundle(); - - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_physical_model(); - auto bundle_hash = bundle.get_hash(); - - auto cache_key = std::format(R"***(library_type=CUBIN -key={} -kernel={} -cuda_runtime={} -cuda_driver={} -arch={}, -bundle={})***", - key, - kernel_symbol, - runtime, - driver, - sm, - bundle_hash); - - auto cache_key_sha256 = hash_string(cache_key); - - auto link = [&] { return link_udf_lto_ir_uncached(name, fragment, kernel_symbol); }; - - if (!use_cache) { - auto [lib, blob] = link(); - return lib; - } - - auto fut = cache.query_or_insert_library( - cache_key_sha256, rtc::binary_type::CUBIN, rtc::library_compile_function_t::from_functor(link)); - - return fut.get(); -} +} // namespace -rtc::library compile_kernel_to_cubin(char const* name, - char const* key, - char const* cuda_source, - bool use_cache, - bool use_pch, - bool log_pch) +[[nodiscard]] rtcx::library get_library(std::string const& name, + std::string const& key, + std::string const& cuda_udf, + bool use_cache, + bool use_pch, + bool log_pch) { CUDF_FUNC_RANGE(); @@ -674,7 +370,7 @@ bundle={})***", auto cache_key_sha256 = hash_string(cache_key); auto compile = [&] { - return compile_kernel_to_cubin_uncached(name, cuda_source, use_pch, log_pch); + return compile_library_uncached(name.c_str(), cuda_udf.c_str(), use_pch, log_pch); }; if (!use_cache) { @@ -682,67 +378,10 @@ bundle={})***", return lib; } - auto fut = cache.query_or_insert_library(cache_key_sha256, - rtc::binary_type::CUBIN, - rtc::library_compile_function_t::from_functor(compile)); + auto fut = cache.get_or_add_library( + cache_key_sha256, rtcx::binary_type::CUBIN, rtcx::library_compile_func::from_functor(compile)); return fut.get(); } -rtc::library compile_and_link_kernel(char const* name, - char const* key, - char const* cuda_udf, - char const* kernel_symbol, - bool use_cache, - bool use_pch, - bool log_pch) -{ - CUDF_FUNC_RANGE(); - auto fragment = compile_udf_to_lto_ir(name, key, cuda_udf, use_cache, use_pch, log_pch); - auto library = link_lto_ir_udf(name, key, fragment, kernel_symbol, use_cache); - return library; -} - -} // namespace - -rtc::library compile_lto_kernel(std::string const& name, - std::string const& key, - std::string const& cuda_udf, - std::string const& kernel_symbol, - bool use_cache, - bool use_pch, - bool log_pch) -{ - return compile_and_link_kernel(name.c_str(), - key.c_str(), - cuda_udf.c_str(), - kernel_symbol.c_str(), - use_cache, - use_pch, - log_pch); -} - -[[nodiscard]] rtc::library compile_cuda_kernel(std::string const& name, - std::string const& key, - std::string const& cuda_udf, - bool use_cache, - bool use_pch, - bool log_pch) -{ - return compile_kernel_to_cubin( - name.c_str(), key.c_str(), cuda_udf.c_str(), use_cache, use_pch, log_pch); -} - -rtc::library compile_and_link_lto_ir_kernel(std::string const& name, - std::string const& key, - std::span lto_ir_binary, - std::string const& kernel_symbol, - bool use_cache) -{ - auto blob = std::make_shared(rtc::blob_t::from_static_data(lto_ir_binary)); - auto fragment = rtc::fragment_t::load( - rtc::fragment_t::load_params{.binary = blob, .type = rtc::binary_type::LTO_IR}); - return link_lto_ir_udf(name.c_str(), key.c_str(), fragment, kernel_symbol.c_str(), use_cache); -} - } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index 04b07c2e51ca..02356b02675a 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -6,55 +6,32 @@ #pragma once #include -#include -#include +#include namespace CUDF_EXPORT cudf { struct [[nodiscard]] jit_bundle_t { private: std::string install_dir_; - rtc::fragment lto_library_; - rtc::cache_t* cache_; + rtcx::cache_t* cache_; void ensure_installed() const; - void preload_lto_library(); - public: - jit_bundle_t(std::string install_dir, rtc::cache_t& cache); + jit_bundle_t(std::string install_dir, rtcx::cache_t& cache); [[nodiscard]] std::string get_hash() const; - [[nodiscard]] - std::string get_directory() const; - - [[nodiscard]] rtc::fragment get_lto_library() const; + [[nodiscard]] std::string get_directory() const; [[nodiscard]] std::vector get_include_directories() const; - - [[nodiscard]] std::vector get_compile_options() const; }; -[[nodiscard]] rtc::library compile_lto_kernel(std::string const& name, - std::string const& key, - std::string const& cuda_udf, - std::string const& kernel_symbol, - bool use_cache = true, - bool use_pch = true, - bool log_pch = false); - -[[nodiscard]] rtc::library compile_cuda_kernel(std::string const& name, - std::string const& key, - std::string const& cuda_udf, - bool use_cache = true, - bool use_pch = true, - bool log_pch = false); - -[[nodiscard]] rtc::library compile_and_link_lto_ir_kernel(std::string const& name, - std::string const& key, - std::span lto_ir_binary, - std::string const& kernel_symbol, - bool use_cache = true); +[[nodiscard]] rtcx::library get_library(std::string const& name, + std::string const& key, + std::string const& cuda_udf, + bool use_cache = true, + bool use_pch = true, + bool log_pch = false); } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/cache.cpp b/cpp/src/jit/rtc/cache.cpp deleted file mode 100644 index 1d82292b1629..000000000000 --- a/cpp/src/jit/rtc/cache.cpp +++ /dev/null @@ -1,449 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace CUDF_EXPORT cudf { -namespace rtc { - -namespace { - -[[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) -{ - auto errc = errno; - auto err_str = - std::format("{}. `{}` failed with {} ({})", message, syscall_name, errc, std::strerror(errc)); - CUDF_FAIL(+err_str, std::runtime_error); -} - -} // namespace - -cache_t::cache_t(std::string cache_dir, cache_limits const& limits) - : cache_dir_{std::move(cache_dir)}, - limits_{limits}, - blobs_cache_{limits.num_blobs}, - fragments_cache_{limits.num_fragments}, - libraries_cache_{limits.num_libraries}, - tick_{0} -{ - CUDF_EXPECTS(limits.num_blobs >= 2, "Blob cache limit must be at least 2"); - CUDF_EXPECTS(limits.num_fragments >= 2, "Fragment cache limit must be at least 2"); - CUDF_EXPECTS(limits.num_libraries >= 2, "Library cache limit must be at least 2"); -} - -std::string const& cache_t::get_cache_dir() { return cache_dir_; } - -std::optional blob_t::from_file(char const* path) -{ - int fd = open(path, O_RDONLY); - - if (fd == -1) { - if (errno == ENOENT) { - return std::nullopt; - } else { - throw_posix("Failed to open RTC cache file from disk", "open"); - } - } - - auto file_size = lseek(fd, 0, SEEK_END); - if (file_size == -1) { throw_posix("Failed to determine size of RTC cache file", "lseek"); } - - void* map = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); - - if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTC cache file", "mmap"); } - - if (close(fd) == -1) { - throw_posix("Failed to close RTC cache file after memory-mapping", "close"); - } - - auto deleter = +[](void*, uint8_t const* buffer, size_t size) { - if (munmap(static_cast(const_cast(buffer)), size) == -1) { - throw_posix("Failed to unmap RTC cache file from memory", "munmap"); - } - }; - - return blob_t::from_parts(static_cast(map), file_size, nullptr, deleter); -} - -namespace { - -/// @brief retrieves a blob from disk based on the given sha256 hash and object type (e.g. "blob", -/// "fragment", "library"). Returns nullopt if the file doesn't exist on disk, and throws if any -/// other error occurs. -std::optional get_disk_blob(std::string const& cache_dir, - std::string const& object_type, - sha256_hash const& sha) -{ - auto hex = sha.to_hex_string(); - auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); - - auto blob = blob_t::from_file(path.c_str()); - - if (!blob.has_value()) { return std::nullopt; } - { - return std::make_shared(std::move(*blob)); - } -} - -/// @brief atomically writes a blob to disk by first writing to a temporary file and then renaming -/// it to the final path. -void add_blob_to_disk(std::string const& cache_dir, - std::string const& object_type, - sha256_hash const& sha, - blob_view binary) -{ - char temp_path[] = "/tmp/cudf-bin-XXXXXX"; - - { - int fd = mkstemp(temp_path); - if (fd == -1) { throw_posix("Failed to create temporary file for RTC cache", "mkstemp"); } - - CUDF_DEFER([&] { - if (close(fd) == -1) { throw_posix("Failed to close temporary RTC cache file", "close"); } - }); - - if (write(fd, binary.data(), binary.size()) == -1) { - throw_posix("Failed to write RTC cache to temporary file", "write"); - } - } - - auto hex = sha.to_hex_string(); - auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); - - std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); - - // rename is atomic, even if another process is performing the same operation - if (rename(temp_path, final_path.c_str()) == -1) { - auto errc = errno; - - if (errc == EEXIST) { - // another process has already created the file, so just remove our temp file - if (remove(temp_path) == -1) { - throw_posix("Failed to remove temporary RTC cache file", "remove"); - } - return; - } else { - throw_posix( - std::format("Failed to move temporary RTC cache file to final location ({})", final_path), - "rename"); - } - } -} - -} // namespace - -std::shared_future cache_t::query_or_insert_blob(sha256_hash const& sha, - blob_compile_function_t compile) -{ - CUDF_FUNC_RANGE(); - - std::atomic_ref tick{tick_}; - auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); - - bool unlocked = false; - lock_.lock(); - - CUDF_DEFER([&] { - if (!unlocked) { lock_.unlock(); } - }); - - // check memory cache - if (auto it = blobs_cache_.entries_.find(sha); it != blobs_cache_.entries_.end()) { - counter_.hit_memory_blob(); - - // update LRU tick - it->second.hit(current_tick); - - return it->second.value; - - } else { - counter_.miss_memory_blob(); - - // check disk cache - auto disk_blob = get_disk_blob(cache_dir_, "blob", sha); - - std::promise promise; - auto fut = promise.get_future().share(); - auto cache_fut = fut; - auto ret_fut = fut; - - if (disk_blob.has_value()) { - counter_.hit_disk_blob(); - - promise.set_value(std::move(*disk_blob)); - - // insert into cache - blobs_cache_.insert(sha, std::move(cache_fut), current_tick); - - return ret_fut; - - } else { - counter_.miss_disk_blob(); - - blobs_cache_.insert(sha, std::move(cache_fut), current_tick); - - // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this sha - lock_.unlock(); - unlocked = true; - - auto result = compile(); - promise.set_value(result); - - // store result to disk - add_blob_to_disk(cache_dir_, "blob", sha, result->view()); - - return ret_fut; - } - } -} - -std::shared_future cache_t::query_or_insert_fragment(sha256_hash const& sha, - binary_type type, - fragment_compile_function_t compile) -{ - CUDF_FUNC_RANGE(); - - std::atomic_ref tick{tick_}; - auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); - - bool unlocked = false; - lock_.lock(); - - CUDF_DEFER([&] { - if (!unlocked) { lock_.unlock(); } - }); - - // check memory cache - if (auto it = fragments_cache_.entries_.find(sha); it != fragments_cache_.entries_.end()) { - counter_.hit_memory_fragment(); - - // update LRU tick - - it->second.hit(current_tick); - - return it->second.value; - - } else { - counter_.miss_memory_fragment(); - - // check disk cache - auto disk_blob = get_disk_blob(cache_dir_, "fragment", sha); - - std::promise promise; - auto fut = promise.get_future().share(); - auto cache_fut = fut; - auto ret_fut = fut; - - if (disk_blob.has_value()) { - counter_.hit_disk_fragment(); - - fragments_cache_.insert(sha, std::move(cache_fut), current_tick); - - // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this sha - lock_.unlock(); - unlocked = true; - - fragment_t::load_params load_params{.binary = *disk_blob, .type = type}; - - auto frag = fragment_t::load(load_params); - promise.set_value(std::move(frag)); - - return ret_fut; - - } else { - counter_.miss_disk_fragment(); - - fragments_cache_.insert(sha, std::move(cache_fut), current_tick); - - // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this sha - lock_.unlock(); - unlocked = true; - - auto result = compile(); - promise.set_value(result); - - // store result to disk - add_blob_to_disk(cache_dir_, "fragment", sha, result->get(type)->view()); - - return ret_fut; - } - } -} - -std::shared_future cache_t::query_or_insert_library(sha256_hash const& sha, - binary_type type, - library_compile_function_t compile) -{ - CUDF_FUNC_RANGE(); - - std::atomic_ref tick{tick_}; - auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); - - bool unlocked = false; - lock_.lock(); - - CUDF_DEFER([&] { - if (!unlocked) { lock_.unlock(); } - }); - - // check memory cache - if (auto it = libraries_cache_.entries_.find(sha); it != libraries_cache_.entries_.end()) { - counter_.hit_memory_library(); - - // update LRU tick - it->second.hit(current_tick); - - return it->second.value; - - } else { - counter_.miss_memory_library(); - - // check disk cache - auto disk_blob = get_disk_blob(cache_dir_, "library", sha); - - std::promise promise; - auto fut = promise.get_future().share(); - auto cache_fut = fut; - auto ret_fut = fut; - - if (disk_blob.has_value()) { - counter_.hit_disk_library(); - - libraries_cache_.insert(sha, std::move(cache_fut), current_tick); - - // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this sha - lock_.unlock(); - unlocked = true; - - library_t::load_params load_params{.binary = (*disk_blob)->view(), .type = type}; - - auto lib = library_t::load(load_params); - promise.set_value(std::move(lib)); - - return ret_fut; - - } else { - counter_.miss_disk_library(); - - libraries_cache_.insert(sha, std::move(cache_fut), current_tick); - - // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this sha - lock_.unlock(); - unlocked = true; - - auto [library, blob] = compile(); - promise.set_value(library); - - // store result to disk - add_blob_to_disk(cache_dir_, "library", sha, blob->view()); - - return ret_fut; - } - } -} - -cache_statistics cache_t::get_statistics() { return counter_.get_statistics(); } - -void cache_t::clear_statistics() { counter_.clear(); } - -cache_limits cache_t::get_limits() { return limits_; } - -size_t cache_t::get_blob_count() -{ - std::lock_guard guard{lock_}; - return blobs_cache_.entries_.size(); -} - -size_t cache_t::get_fragment_count() -{ - std::lock_guard guard{lock_}; - return fragments_cache_.entries_.size(); -} - -size_t cache_t::get_library_count() -{ - std::lock_guard guard{lock_}; - return libraries_cache_.entries_.size(); -} - -void cache_t::clear_memory_store() -{ - CUDF_FUNC_RANGE(); - - std::lock_guard guard{lock_}; - - blobs_cache_.entries_.clear(); - fragments_cache_.entries_.clear(); - libraries_cache_.entries_.clear(); -} - -void cache_t::clear_disk_store() -{ - CUDF_FUNC_RANGE(); - - DIR* dir = opendir(cache_dir_.c_str()); - - if (dir == nullptr) { throw_posix("Failed to open RTC cache directory for clearing", "opendir"); } - - CUDF_DEFER([&] { closedir(dir); }); - - errno = 0; // reset errno before reading - - struct dirent* entry_iter = nullptr; - std::vector entry_path; - entry_path.resize(PATH_MAX + 1); - - while (true) { - entry_iter = readdir(dir); - - if (entry_iter == nullptr) { - if (errno != 0) { - throw_posix("Failed to read RTC cache directory for clearing", "readdir"); - } else { - break; - } - } - - struct stat entry_stat; - - if (lstat(entry_path.data(), &entry_stat) == -1) { - throw_posix("Failed to get file status for RTC cache clearing", "lstat"); - } - - if (S_ISREG(entry_stat.st_mode)) { - if (unlink(entry_path.data()) == -1) { - throw_posix("Failed to unlink RTC cache file during clearing", "unlink"); - } - } - - // reset errno for next iteration - errno = 0; - } - - return; -} - -} // namespace rtc -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/cache.hpp b/cpp/src/jit/rtc/cache.hpp deleted file mode 100644 index 80030997f4ec..000000000000 --- a/cpp/src/jit/rtc/cache.hpp +++ /dev/null @@ -1,267 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include - -#include -#include -#include -#include -#include - -namespace CUDF_EXPORT cudf { -namespace rtc { - -struct [[nodiscard]] cache_statistics { - uint64_t blob_memory_hits = 0; - uint64_t blob_memory_misses = 0; - uint64_t blob_disk_hits = 0; - uint64_t blob_disk_misses = 0; - uint64_t fragment_memory_hits = 0; - uint64_t fragment_memory_misses = 0; - uint64_t fragment_disk_hits = 0; - uint64_t fragment_disk_misses = 0; - uint64_t library_memory_hits = 0; - uint64_t library_memory_misses = 0; - uint64_t library_disk_hits = 0; - uint64_t library_disk_misses = 0; -}; - -struct [[nodiscard]] cache_limits { - uint32_t num_blobs = 1024; - uint32_t num_fragments = 1024; - uint32_t num_libraries = 1024; - - static constexpr cache_limits default_limits() { return cache_limits{}; } -}; - -namespace detail { - -inline constexpr size_t CACHELINE_ALIGNMENT = - 64; // = std::hardware_destructive_interference_size */ - -template -struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { - struct entry { - uint64_t last_touched_tick; - T value; - - void hit(uint64_t tick) { last_touched_tick = tick; } - }; - - std::unordered_map entries_; - size_t limit_; - - explicit lru_memory_cache(size_t limit) : entries_{}, limit_{limit} - { - // reserve space to avoid rehashing - CUDF_EXPECTS(limit > 0, "Cache limit must be greater than 0"); - entries_.reserve(limit * 2); - } - - void purge() - { - if (entries_.empty()) { return; } - - auto num_to_purge = std::max(entries_.size() / 2, static_cast(1)); - - std::vector> rankings; - rankings.reserve(entries_.size()); - - for (auto const& [key, entry] : entries_) { - rankings.emplace_back(key, entry.last_touched_tick); - } - - std::sort(rankings.begin(), rankings.end(), [](auto const& a, auto const& b) { - return a.second < b.second; - }); - - // purge least recently used half - rankings.resize(num_to_purge); - - for (auto [key, _] : rankings) { - entries_.erase(key); - } - } - - void insert(sha256_hash const& sha, T&& value, uint64_t tick) - { - if ((entries_.size() + 1) > limit_) { purge(); } - - entries_.emplace(sha, entry{tick, std::move(value)}); - } -}; - -struct alignas(CACHELINE_ALIGNMENT) counter { - uint64_t value_ = 0; - - void increment() - { - std::atomic_ref c{value_}; - c.fetch_add(1, std::memory_order_relaxed); - } - - [[nodiscard]] uint64_t get() const - { - std::atomic_ref c{value_}; - return c.load(std::memory_order_relaxed); - } - - void reset() - { - std::atomic_ref c{value_}; - c.store(0, std::memory_order_relaxed); - } -}; - -struct cache_statistics_counter { - counter blob_memory_hits; - counter blob_memory_misses; - counter blob_disk_hits; - counter blob_disk_misses; - counter fragment_memory_hits; - counter fragment_memory_misses; - counter fragment_disk_hits; - counter fragment_disk_misses; - counter library_memory_hits; - counter library_memory_misses; - counter library_disk_hits; - counter library_disk_misses; - - void clear() - { - blob_memory_hits.reset(); - blob_memory_misses.reset(); - blob_disk_hits.reset(); - blob_disk_misses.reset(); - fragment_memory_hits.reset(); - fragment_memory_misses.reset(); - fragment_disk_hits.reset(); - fragment_disk_misses.reset(); - library_memory_hits.reset(); - library_memory_misses.reset(); - library_disk_hits.reset(); - library_disk_misses.reset(); - } - - void hit_memory_blob() { blob_memory_hits.increment(); } - - void miss_memory_blob() { blob_memory_misses.increment(); } - - void hit_disk_blob() { blob_disk_hits.increment(); } - - void miss_disk_blob() { blob_disk_misses.increment(); } - - void hit_memory_fragment() { fragment_memory_hits.increment(); } - - void miss_memory_fragment() { fragment_memory_misses.increment(); } - - void hit_disk_fragment() { fragment_disk_hits.increment(); } - - void miss_disk_fragment() { fragment_disk_misses.increment(); } - - void hit_memory_library() { library_memory_hits.increment(); } - - void miss_memory_library() { library_memory_misses.increment(); } - - void hit_disk_library() { library_disk_hits.increment(); } - - void miss_disk_library() { library_disk_misses.increment(); } - - cache_statistics get_statistics() const - { - return cache_statistics{.blob_memory_hits = blob_memory_hits.get(), - .blob_memory_misses = blob_memory_misses.get(), - .blob_disk_hits = blob_disk_hits.get(), - .blob_disk_misses = blob_disk_misses.get(), - .fragment_memory_hits = fragment_memory_hits.get(), - .fragment_memory_misses = fragment_memory_misses.get(), - .fragment_disk_hits = fragment_disk_hits.get(), - .fragment_disk_misses = fragment_disk_misses.get(), - .library_memory_hits = library_memory_hits.get(), - .library_memory_misses = library_memory_misses.get(), - .library_disk_hits = library_disk_hits.get(), - .library_disk_misses = library_disk_misses.get()}; - } -}; - -} // namespace detail - -using blob_compile_function_t = function_ref; -using fragment_compile_function_t = function_ref; -using library_compile_function_t = function_ref()>; - -/// @brief Thread-safe compile cache for compiled blobs, fragments, and libraries -/// @details Provides in-memory and on-disk caching of compiled RTC artifacts. -/// The cache uses an LRU eviction policy when the number of cached items -/// exceeds user-defined limits. -/// In-memory cache is implemented using a thread-safe LRU cache that supports concurrent reads. -/// The on-disk cache also allows concurrent access and stores cached items in files -/// within a specified directory. Writing to disk is atomic to prevent corruption from -/// concurrent writes or process interruptions. -/// In addition, the cache maintains statistics on cache hits and misses for both -/// in-memory and on-disk caches to help monitor cache performance in benchmarking and debugging. -/// The interface is zero-copy throughout, using shared pointers and spans to avoid unnecessary data -/// copying across threads and disk. -struct cache_t { - private: - std::string cache_dir_; - - cache_limits limits_; - - std::mutex lock_; - - detail::lru_memory_cache> blobs_cache_; - - detail::lru_memory_cache> fragments_cache_; - - detail::lru_memory_cache> libraries_cache_; - - detail::cache_statistics_counter counter_; - - alignas(detail::CACHELINE_ALIGNMENT) uint64_t tick_; - - public: - cache_t(std::string cache_dir, cache_limits const& limits); - cache_t(cache_t const&) = delete; - cache_t& operator=(cache_t const&) = delete; - cache_t(cache_t&&) = delete; - cache_t& operator=(cache_t&&) = delete; - ~cache_t() = default; - - [[nodiscard]] std::string const& get_cache_dir(); - - [[nodiscard]] std::shared_future query_or_insert_blob(sha256_hash const& sha, - blob_compile_function_t compile); - - [[nodiscard]] std::shared_future query_or_insert_fragment( - sha256_hash const& sha, binary_type type, fragment_compile_function_t compile); - - [[nodiscard]] std::shared_future query_or_insert_library( - sha256_hash const& sha, binary_type type, library_compile_function_t compile); - - cache_statistics get_statistics(); - - void clear_statistics(); - - cache_limits get_limits(); - - [[nodiscard]] size_t get_blob_count(); - - [[nodiscard]] size_t get_fragment_count(); - - [[nodiscard]] size_t get_library_count(); - - void clear_memory_store(); - - void clear_disk_store(); -}; - -} // namespace rtc -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/rtc.cpp b/cpp/src/jit/rtc/rtc.cpp deleted file mode 100644 index d53875b248f3..000000000000 --- a/cpp/src/jit/rtc/rtc.cpp +++ /dev/null @@ -1,575 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include - -#include - -#define CUDFRTC_CONCATENATE_DETAIL(x, y) x##y -#define CUDFRTC_CONCATENATE(x, y) CUDFRTC_CONCATENATE_DETAIL(x, y) - -#define CUDFRTC_CHECK_CUDA(...) \ - do { \ - ::CUresult __result = (__VA_ARGS__); \ - if (__result != ::CUDA_SUCCESS) { \ - char const* __enum_str; \ - CUDF_EXPECTS(::cuGetErrorString(__result, &__enum_str) == ::CUDA_SUCCESS, \ - "Unable to get CUDA error string"); \ - auto __errstr = ::std::format("(cuda) Call {} failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast<::int64_t>(__result), \ - __enum_str); \ - CUDF_FAIL(+__errstr, ::std::runtime_error); \ - } \ - } while (0) - -#define CUDFRTC_CHECK_NVRTC(params, program, ...) \ - do { \ - ::nvrtcResult __result = (__VA_ARGS__); \ - ::cudf::rtc::log_nvrtc_result(params, program, __result); \ - if (__result != ::NVRTC_SUCCESS) { \ - auto __errstr = ::std::format("(nvrtc) Call {} failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast<::int64_t>(__result), \ - ::nvrtcGetErrorString(__result)); \ - CUDF_FAIL(+__errstr, ::std::runtime_error); \ - } \ - } while (0) - -#define CUDFRTC_CHECK_NVJITLINK(params, handle, ...) \ - do { \ - ::nvJitLinkResult __result = (__VA_ARGS__); \ - ::cudf::rtc::log_nvJitLink_result(params, handle, __result); \ - if (__result != ::NVJITLINK_SUCCESS) { \ - auto __errstr = ::std::format("(nvJitLink) Call {} failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast<::int64_t>(__result), \ - ::cudf::rtc::get_nvJitLinkResultString(__result)); \ - CUDF_FAIL(+__errstr, ::std::runtime_error); \ - } \ - } while (0) - -namespace CUDF_EXPORT cudf { -namespace rtc { - -namespace { -char const* get_nvJitLinkResultString(nvJitLinkResult result) -{ - switch (result) { - case NVJITLINK_SUCCESS: return "NVJITLINK_SUCCESS"; - case NVJITLINK_ERROR_UNRECOGNIZED_OPTION: return "NVJITLINK_ERROR_UNRECOGNIZED_OPTION"; - case NVJITLINK_ERROR_MISSING_ARCH: return "NVJITLINK_ERROR_MISSING_ARCH"; - case NVJITLINK_ERROR_INVALID_INPUT: return "NVJITLINK_ERROR_INVALID_INPUT"; - case NVJITLINK_ERROR_PTX_COMPILE: return "NVJITLINK_ERROR_PTX_COMPILE"; - case NVJITLINK_ERROR_NVVM_COMPILE: return "NVJITLINK_ERROR_NVVM_COMPILE"; - case NVJITLINK_ERROR_INTERNAL: return "NVJITLINK_ERROR_INTERNAL"; - case NVJITLINK_ERROR_THREADPOOL: return "NVJITLINK_ERROR_THREADPOOL"; - case NVJITLINK_ERROR_UNRECOGNIZED_INPUT: return "NVJITLINK_ERROR_UNRECOGNIZED_INPUT"; - case NVJITLINK_ERROR_FINALIZE: return "NVJITLINK_ERROR_FINALIZE"; - case NVJITLINK_ERROR_NULL_INPUT: return "NVJITLINK_ERROR_NULL_INPUT"; - case NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS: return "NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS"; - case NVJITLINK_ERROR_INCORRECT_INPUT_TYPE: return "NVJITLINK_ERROR_INCORRECT_INPUT_TYPE"; - case NVJITLINK_ERROR_ARCH_MISMATCH: return "NVJITLINK_ERROR_ARCH_MISMATCH"; - case NVJITLINK_ERROR_OUTDATED_LIBRARY: return "NVJITLINK_ERROR_OUTDATED_LIBRARY"; - case NVJITLINK_ERROR_MISSING_FATBIN: return "NVJITLINK_ERROR_MISSING_FATBIN"; - case NVJITLINK_ERROR_UNRECOGNIZED_ARCH: return "NVJITLINK_ERROR_UNRECOGNIZED_ARCH"; - case NVJITLINK_ERROR_UNSUPPORTED_ARCH: return "NVJITLINK_ERROR_UNSUPPORTED_ARCH"; - case NVJITLINK_ERROR_LTO_NOT_ENABLED: return "NVJITLINK_ERROR_LTO_NOT_ENABLED"; - default: - CUDF_FAIL( - +std::format("Unrecognized nvJitLinkResult type: ({})", static_cast(result)), - std::runtime_error); - } -} - -char const* binary_type_string(binary_type type) -{ - switch (type) { - case binary_type::LTO_IR: return "LTO_IR"; - case binary_type::CUBIN: return "CUBIN"; - case binary_type::FATBIN: return "FATBIN"; - case binary_type::PTX: return "PTX"; - default: - CUDF_FAIL(+std::format("Unrecognized binary_type: ({})", static_cast(type)), - std::runtime_error); - } -} - -nvJitLinkInputType to_nvjitlink_input_type(binary_type bin_type) -{ - switch (bin_type) { - case binary_type::LTO_IR: return NVJITLINK_INPUT_LTOIR; - case binary_type::CUBIN: return NVJITLINK_INPUT_CUBIN; - case binary_type::FATBIN: return NVJITLINK_INPUT_FATBIN; - case binary_type::PTX: return NVJITLINK_INPUT_PTX; - default: - CUDF_FAIL( - +std::format("Unrecognized binary type for linking: ({}) ", static_cast(bin_type)), - std::logic_error); - } -} - -void log_nvrtc_result(fragment_t::compile_params const& params, - nvrtcProgram program, - nvrtcResult compile_result) -{ - if (program == nullptr) { return; } - - size_t log_size; - if (auto errc = nvrtcGetProgramLogSize(program, &log_size); errc != NVRTC_SUCCESS) { - CUDF_FAIL(+std::format("Failed to get NVRTC program log size with error ({}): {}", - static_cast(errc), - nvrtcGetErrorString(errc)), - std::runtime_error); - } - - if (log_size <= 1) { return; } - - std::vector log; - log.resize(log_size); - - if (auto errc = nvrtcGetProgramLog(program, log.data()); errc != NVRTC_SUCCESS) { - CUDF_FAIL(+std::format("Failed to get NVRTC program log with error ({}): {}", - static_cast(errc), - nvrtcGetErrorString(errc)), - std::runtime_error); - } - - log.resize(log_size == 0 ? 0 : (log_size - 1)); - - auto status_str = (compile_result == NVRTC_SUCCESS && !log.empty()) ? "completed with warning" - : "failed with error"; - - std::string headers_str; - for (auto const& header : params.headers.include_names) { - headers_str = std::format("{}\t{}\n", headers_str, header); - } - - std::string options_str; - for (auto const& option : params.options) { - options_str = std::format("{}\t{}\n", options_str, option); - } - - auto msg = std::format( - "NVRTC Compilation for `{}` {} ({}): {}.\nHeaders:\n{}\n\nOptions:\n{}\n\nLog:\n\t{}", - params.name == nullptr ? "" : params.name, - status_str, - static_cast(compile_result), - nvrtcGetErrorString(compile_result), - headers_str, - options_str, - std::string_view{log.data(), log.size()}); - - if (compile_result != NVRTC_SUCCESS) { - CUDF_LOG_ERROR(msg); - } else { - CUDF_LOG_WARN(msg); - } -} - -void log_nvJitLink_result(library_t::link_params const& params, - nvJitLinkHandle handle, - nvJitLinkResult link_result) -{ - if (handle == nullptr) { return; } - - size_t info_log_size; - if (auto errc = nvJitLinkGetInfoLogSize(handle, &info_log_size); errc != NVJITLINK_SUCCESS) { - CUDF_FAIL(+std::format("Failed to get nvJitLink info log size with error ({}): {}", - static_cast(errc), - get_nvJitLinkResultString(errc)), - std::runtime_error); - } - - std::vector info_log; - if (info_log_size > 1) { - info_log.resize(info_log_size); - if (auto errc = nvJitLinkGetInfoLog(handle, info_log.data()); errc != NVJITLINK_SUCCESS) { - CUDF_FAIL(+std::format("Failed to get nvJitLink info log with error ({}): {}", - static_cast(errc), - get_nvJitLinkResultString(errc)), - std::runtime_error); - } - } - info_log.resize(info_log_size == 0 ? 0 : (info_log_size - 1)); - - size_t error_log_size; - if (auto errc = nvJitLinkGetErrorLogSize(handle, &error_log_size); errc != NVJITLINK_SUCCESS) { - CUDF_FAIL(+std::format("Failed to get nvJitLink error log size with error ({}): {}", - static_cast(errc), - get_nvJitLinkResultString(errc)), - std::runtime_error); - } - - std::vector error_log; - - if (error_log_size > 1) { - error_log.resize(error_log_size); - if (auto errc = nvJitLinkGetErrorLog(handle, error_log.data()); errc != NVJITLINK_SUCCESS) { - CUDF_FAIL(+std::format("Failed to get nvJitLink error log with error ({}): {}", - static_cast(errc), - get_nvJitLinkResultString(errc)), - std::runtime_error); - } - } - error_log.resize(error_log_size == 0 ? 0 : (error_log_size - 1)); - - if (info_log.empty() && error_log.empty()) { return; } - - std::string fragments_str; - for (auto const& fragment_name : params.fragment_names) { - fragments_str = std::format("{}\t{}\n", fragments_str, fragment_name); - } - - std::string link_options_str; - for (auto const& option : params.link_options) { - link_options_str = std::format("{}\t{}\n", link_options_str, option); - } - - char const* binary_type_str = binary_type_string(params.output_type); - - auto status_str = error_log.empty() ? "completed with warnings" : "failed with errors"; - - auto msg = std::format( - "(nvJitLink) Linking for `{}` ({}) {}, error code ({}): {}.\nFragments: \n{}\n" - "Link Options: \n{}\n\nInfo Log:\n\t{}\n\nError Log:\n\t{}\n\n", - params.name == nullptr ? "" : params.name, - binary_type_str, - status_str, - static_cast(link_result), - get_nvJitLinkResultString(link_result), - fragments_str, - link_options_str, - std::string_view{info_log.data(), info_log.size()}, - std::string_view{error_log.data(), error_log.size()}); - - if (!error_log.empty()) { - CUDF_LOG_ERROR(msg); - } else { - CUDF_LOG_WARN(msg); - } -} -} // namespace - -blob_t blob_t::from_vector(std::vector&& data) -{ - auto ptr = new std::vector(std::move(data)); - return blob_t::from_parts( - ptr->data(), ptr->size(), ptr, [](void* user_data, uint8_t const*, size_t) { - delete reinterpret_cast*>(user_data); - }); -} - -blob_t blob_t::from_static_data(std::span data) -{ - return blob_t::from_parts( - data.data(), data.size(), nullptr, [](void*, uint8_t const*, size_t) {}); -} - -fragment fragment_t::load(load_params const& params) -{ - CUDF_FUNC_RANGE(); - - return std::make_shared(params.binary, params.type); -} - -std::vector fragment_t::compile_raw(compile_params const& params) -{ - CUDF_FUNC_RANGE(); - - CUDF_EXPECTS(params.name != nullptr, "Fragment name must not be null", std::logic_error); - CUDF_EXPECTS(params.source != nullptr, "Fragment source must not be null", std ::logic_error); - - nvrtcProgram program = nullptr; - - CUDFRTC_CHECK_NVRTC(params, - program, - nvrtcCreateProgram(&program, - params.source, - params.name, - static_cast(params.headers.headers.size()), - params.headers.headers.data(), - params.headers.include_names.data())); - - CUDF_DEFER([&] { nvrtcDestroyProgram(&program); }); - - // TODO: log is printed twice when warnings are raised - CUDFRTC_CHECK_NVRTC( - params, - program, - nvrtcCompileProgram(program, static_cast(params.options.size()), params.options.data())); - - switch (params.target_type) { - case binary_type::LTO_IR: { - size_t lto_ir_size; - CUDFRTC_CHECK_NVRTC(params, program, nvrtcGetLTOIRSize(program, <o_ir_size)); - - std::vector lto_ir; - lto_ir.resize(lto_ir_size); - - CUDFRTC_CHECK_NVRTC( - params, program, nvrtcGetLTOIR(program, reinterpret_cast(lto_ir.data()))); - - return lto_ir; - - } break; - case binary_type::CUBIN: { - size_t cubin_size; - CUDFRTC_CHECK_NVRTC(params, program, nvrtcGetCUBINSize(program, &cubin_size)); - - std::vector cubin; - cubin.resize(cubin_size); - CUDFRTC_CHECK_NVRTC( - params, program, nvrtcGetCUBIN(program, reinterpret_cast(cubin.data()))); - - return cubin; - - } break; - default: CUDF_FAIL("Unsupported binary type for compiling fragment"); - } -} - -fragment fragment_t::compile(compile_params const& params) -{ - CUDF_FUNC_RANGE(); - - auto result = compile_raw(params); - - return std::make_shared( - std::make_shared(rtc::blob_t::from_vector(std::move(result))), params.target_type); -} - -blob const& fragment_t::get(binary_type type) const -{ - CUDF_EXPECTS(type_ == type, "Fragment does not contain expected binary type"); - return blob_; -} - -std::tuple kernel_ref::max_occupancy_config(size_t dynamic_shared_memory_bytes, - int32_t block_size_limit) const -{ - int32_t min_grid_size; - int32_t block_size; - CUDFRTC_CHECK_CUDA(cuOccupancyMaxPotentialBlockSize(&min_grid_size, - &block_size, - reinterpret_cast(handle_), - nullptr, - dynamic_shared_memory_bytes, - block_size_limit)); - return {min_grid_size, block_size}; -} - -void kernel_ref::launch(uint32_t grid_dim_x, - uint32_t grid_dim_y, - uint32_t grid_dim_z, - uint32_t block_dim_x, - uint32_t block_dim_y, - uint32_t block_dim_z, - uint32_t shared_mem_bytes, - CUstream stream, - void** kernel_params) const -{ - CUDF_FUNC_RANGE(); - - CUDF_EXPECTS(grid_dim_x > 0 && grid_dim_y > 0 && grid_dim_z > 0, - "Grid dimensions must be greater than zero", - std::logic_error); - CUDF_EXPECTS(block_dim_x > 0 && block_dim_y > 0 && block_dim_z > 0, - "Block dimensions must be greater than zero", - std::logic_error); - CUDF_EXPECTS( - kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); - - CUlaunchConfig cfg{.gridDimX = grid_dim_x, - .gridDimY = grid_dim_y, - .gridDimZ = grid_dim_z, - .blockDimX = block_dim_x, - .blockDimY = block_dim_y, - .blockDimZ = block_dim_z, - .sharedMemBytes = shared_mem_bytes, - .hStream = stream, - .attrs = nullptr, - .numAttrs = 0}; - CUDFRTC_CHECK_CUDA( - cuLaunchKernelEx(&cfg, reinterpret_cast(handle_), kernel_params, nullptr)); -} - -std::string_view kernel_ref::get_name() const -{ - char const* name; - CUDFRTC_CHECK_CUDA(cuKernelGetName(&name, handle_)); - return std::string_view{name == nullptr ? "" : name}; -} - -library_t::~library_t() -{ - if (handle_ != nullptr) { - if (cuLibraryUnload(handle_) != CUDA_SUCCESS) { std::terminate(); } - } -} - -library library_t::load(load_params const& params) -{ - CUDF_FUNC_RANGE(); - - CUlibrary handle; - - CUDFRTC_CHECK_CUDA( - cuLibraryLoadData(&handle, params.binary.data(), nullptr, nullptr, 0, nullptr, nullptr, 0)); - - CUDF_DEFER([&] { - if (handle != nullptr) { CUDFRTC_CHECK_CUDA(cuLibraryUnload(handle)); } - }); - - auto module = std::make_shared(handle); - - handle = nullptr; - - return module; -} - -blob library_t::link_as_blob(link_params const& params) -{ - CUDF_FUNC_RANGE(); - - CUDF_EXPECTS(params.name != nullptr, "Link output name must not be null", std::logic_error); - CUDF_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, - "Only CUBIN and PTX output types are supported for linking modules", - std::logic_error); - CUDF_EXPECTS(params.fragments.size() == params.fragment_binary_types.size(), - "Mismatched number of fragments and fragment binary types", - std::logic_error); - CUDF_EXPECTS(params.fragments.size() == params.fragment_names.size(), - "Mismatched number of fragments and fragment names", - std::logic_error); - CUDF_EXPECTS(params.fragments.size() > 0, "No fragments provided for linking", std::logic_error); - - for (auto& frag : params.fragments) { - CUDF_EXPECTS(frag.size_bytes() > 0, "Fragment binary data must be non-empty", std::logic_error); - } - - nvJitLinkHandle handle = nullptr; - - CUDFRTC_CHECK_NVJITLINK(params, - handle, - nvJitLinkCreate(&handle, - static_cast(params.link_options.size()), - const_cast(params.link_options.data()))); - - CUDF_DEFER([&] { nvJitLinkDestroy(&handle); }); - - for (size_t i = 0; i < params.fragments.size(); i++) { - auto name = params.fragment_names[i]; - auto fragment = params.fragments[i]; - auto bin_type = params.fragment_binary_types[i]; - nvJitLinkInputType nv_type = to_nvjitlink_input_type(bin_type); - - CUDFRTC_CHECK_NVJITLINK( - params, - handle, - nvJitLinkAddData(handle, nv_type, fragment.data(), fragment.size_bytes(), name)); - } - - CUDFRTC_CHECK_NVJITLINK(params, handle, nvJitLinkComplete(handle)); - - switch (params.output_type) { - case binary_type::CUBIN: { - size_t cubin_size; - CUDFRTC_CHECK_NVJITLINK(params, handle, nvJitLinkGetLinkedCubinSize(handle, &cubin_size)); - std::vector cubin; - cubin.resize(cubin_size); - CUDFRTC_CHECK_NVJITLINK(params, handle, nvJitLinkGetLinkedCubin(handle, cubin.data())); - - return std::make_shared(blob_t::from_vector(std::move(cubin))); - } break; - - case binary_type::PTX: { - size_t ptx_size; - - CUDFRTC_CHECK_NVJITLINK(params, handle, nvJitLinkGetLinkedPtxSize(handle, &ptx_size)); - std::vector ptx; - ptx.resize(ptx_size); - - CUDFRTC_CHECK_NVJITLINK( - params, handle, nvJitLinkGetLinkedPtx(handle, reinterpret_cast(ptx.data()))); - - return std::make_shared(blob_t::from_vector(std::move(ptx))); - } break; - - default: - CUDF_FAIL(+std::format("Unsupported output binary type for linking CUDA libraries: ({})", - static_cast(params.output_type)), - std::runtime_error); - } -} - -library library_t::link(link_params const& params) -{ - CUDF_FUNC_RANGE(); - - auto blob = link_as_blob(params); - return load(load_params{blob->view(), params.output_type}); -} - -kernel_ref library_t::get_kernel(char const* name) const -{ - CUkernel kernel; - CUDFRTC_CHECK_CUDA(cuLibraryGetKernel(&kernel, handle_, name)); - return kernel_ref{kernel}; -} - -std::vector library_t::enumerate_kernels() const -{ - uint32_t num_kernels; - CUDFRTC_CHECK_CUDA(cuLibraryGetKernelCount(&num_kernels, handle_)); - - std::vector kernels; - kernels.resize(num_kernels); - - CUDFRTC_CHECK_CUDA(cuLibraryEnumerateKernels(kernels.data(), num_kernels, handle_)); - - std::vector result; - for (CUkernel k : kernels) { - result.emplace_back(k); - } - - return result; -} - -} // namespace rtc - -std::string rtc::demangle_cuda_symbol(char const* mangled_name) -{ - int status; - size_t length; - - char* demangled_name = abi::__cxa_demangle(mangled_name, nullptr, &length, &status); - - CUDF_EXPECTS(status == 0, "Demangling CUDA symbol name failed"); - CUDF_EXPECTS(demangled_name != nullptr, "Demangling CUDA symbol name failed"); - - CUDF_DEFER([&] { - if (demangled_name != nullptr) free(demangled_name); - }); - - std::string result{demangled_name}; - - return result; -} - -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/rtc.hpp b/cpp/src/jit/rtc/rtc.hpp deleted file mode 100644 index 3d1108fb6aae..000000000000 --- a/cpp/src/jit/rtc/rtc.hpp +++ /dev/null @@ -1,252 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include - -#include -#include -#include -#include -#include - -extern "C" { -typedef struct CUlib_st* CUlibrary; -typedef struct CUkern_st* CUkernel; -typedef struct CUstream_st* CUstream; -} - -namespace CUDF_EXPORT cudf { -namespace rtc { - -enum class binary_type : int8_t { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; - -using blob_view = std::span; - -/// @brief Represents a binary blob -/// @details Manages the lifetime of the binary data via a user-provided -/// deallocator function. This enables zero-copy usage of binary data stored -/// in various forms (e.g., std::vector, mmap'd file, etc.). -struct [[nodiscard]] blob_t { - using dealloctor_fn = void (*)(void*, uint8_t const*, size_t); - - private: - static void noop_deallocator(void*, uint8_t const*, size_t) {} - - uint8_t const* data_; - size_t size_; - void* user_data_; - dealloctor_fn deallocator_; - - blob_t(uint8_t const* data, size_t size, void* user_data, dealloctor_fn deallocator) - : data_(data), size_(size), user_data_(user_data), deallocator_(deallocator) - { - } - - public: - blob_t() : data_(nullptr), size_(0), user_data_(nullptr), deallocator_(noop_deallocator) {} - - blob_t(blob_t const&) = delete; - blob_t& operator=(blob_t const&) = delete; - - blob_t(blob_t&& other) noexcept - : data_(other.data_), - size_(other.size_), - user_data_(other.user_data_), - deallocator_(other.deallocator_) - { - other.data_ = nullptr; - other.size_ = 0; - other.user_data_ = nullptr; - other.deallocator_ = noop_deallocator; - } - - blob_t& operator=(blob_t&& other) noexcept - { - if (this == &other) [[unlikely]] { return *this; } - this->~blob_t(); - new (this) blob_t(std::move(other)); - return *this; - } - - ~blob_t() { deallocator_(user_data_, data_, size_); } - - [[nodiscard]] blob_view view() const { return blob_view{data_, size_}; } - - static blob_t from_parts(uint8_t const* data, - size_t size, - void* user_data, - dealloctor_fn deallocator) - { - return blob_t{data, size, user_data, deallocator}; - } - - static blob_t from_vector(std::vector&& data); - - static blob_t from_static_data(std::span data); - - static std::optional from_file(char const* path); -}; - -using blob = std::shared_ptr; - -template -struct function_ref; - -/// @brief Zero-copy, type-erased reference to a callable entity (e.g. lambda, function pointer) -/// that can be invoked with the given signature. -template -struct [[nodiscard]] function_ref { - private: - void* _user_data; - R (*_thunk)(void*, Args...); - - public: - function_ref(void* user_data, R (*thunk)(void*, Args...)) : _user_data{user_data}, _thunk{thunk} - { - } - - R operator()(Args... args) const { return _thunk(_user_data, std::forward(args)...); } - - template - static function_ref from_functor(Lambda& func) - { - return function_ref{static_cast(std::addressof(func)), - +[](void* user_data, Args... args) -> R { - auto& lambda = *static_cast*>(user_data); - return lambda(std::forward(args)...); - }}; - } - - static function_ref from_function_pointer(R (*func)(Args...)) - { - return function_ref{func, +[](void* user_data, Args... args) -> R { - auto func = reinterpret_cast(user_data); - return func(std::forward(args)...); - }}; - } -}; - -struct [[nodiscard]] header_map { - std::span include_names = {}; // null-terminated header include names - std::span headers = {}; // null-terminated header contents - std::span header_sizes = {}; // sizes of each header (excluding null-terminator) -}; - -struct fragment_t; - -using fragment = std::shared_ptr; - -/// @brief Represents a partially compiled RTC kernel (i.e. fragment) in LTO-IR or PTX -struct [[nodiscard]] fragment_t { - private: - blob blob_; - binary_type type_; - - public: - struct [[nodiscard]] load_params { - blob binary = {}; - binary_type type = binary_type::LTO_IR; - }; - - struct [[nodiscard]] compile_params { - char const* name = nullptr; - char const* source = nullptr; - header_map headers = {}; - std::span options = {}; - binary_type target_type = binary_type::LTO_IR; - }; - - static std::vector compile_raw(compile_params const& params); - - [[nodiscard]] static fragment load(load_params const& params); - - [[nodiscard]] static fragment compile(compile_params const& params); - - fragment_t(blob blob, binary_type type) : blob_(std::move(blob)), type_(type) {} - fragment_t(fragment_t const&) = delete; - fragment_t(fragment_t&&) = delete; - fragment_t& operator=(fragment_t const&) = delete; - fragment_t& operator=(fragment_t&&) = delete; - ~fragment_t() = default; - - [[nodiscard]] binary_type get_type() const { return type_; } - - // TODO: lamarrr dumping of the byte code, LTO-IR: NVVM?, PTX: PTX Bytecode - [[nodiscard]] blob const& get(binary_type type) const; -}; - -struct library_t; - -using library = std::shared_ptr; - -struct [[nodiscard]] kernel_ref { - private: - CUkernel handle_; - - public: - explicit kernel_ref(CUkernel handle) : handle_(handle) {} - - std::tuple max_occupancy_config(size_t dynamic_shared_memory_bytes, - int32_t block_size_limit) const; - - void launch(uint32_t grid_dim_x, - uint32_t grid_dim_y, - uint32_t grid_dim_z, - uint32_t block_dim_x, - uint32_t block_dim_y, - uint32_t block_dim_z, - uint32_t shared_mem_bytes, - CUstream stream, - void** kernel_params) const; - - [[nodiscard]] std::string_view get_name() const; -}; - -/// @brief Represents a loaded RTC library containing compiled kernels -/// Input: CUBIN or PTX binary -/// Output: loaded library with launchable kernels -struct [[nodiscard]] library_t { - private: - CUlibrary handle_; - - public: - explicit library_t(CUlibrary handle) : handle_(handle) {} - library_t(library_t const&) = delete; - library_t(library_t&&) = delete; - library_t& operator=(library_t const&) = delete; - library_t& operator=(library_t&&) = delete; - ~library_t(); - - struct [[nodiscard]] load_params { - blob_view binary = {}; - binary_type type = binary_type::CUBIN; - }; - - struct [[nodiscard]] link_params { - char const* name = nullptr; - binary_type output_type = binary_type::CUBIN; - std::span fragments = {}; - std::span fragment_binary_types = {}; - std::span fragment_names = {}; - std::span link_options = {}; - }; - - [[nodiscard]] static library load(load_params const& params); - - [[nodiscard]] static blob link_as_blob(link_params const& params); - - [[nodiscard]] static library link(link_params const& params); - - [[nodiscard]] kernel_ref get_kernel(char const* name) const; - - [[nodiscard]] std::vector enumerate_kernels() const; -}; - -[[nodiscard]] std::string demangle_cuda_symbol(char const* mangled_name); - -} // namespace rtc -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/sha256.cpp b/cpp/src/jit/rtc/sha256.cpp deleted file mode 100644 index 1cd36d23187c..000000000000 --- a/cpp/src/jit/rtc/sha256.cpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -#include - -extern "C" { -#include -} - -namespace CUDF_EXPORT cudf { -namespace rtc { - -sha256_context::sha256_context() : ectx_(nullptr) -{ - const EVP_MD* type = EVP_sha256(); - ectx_ = EVP_MD_CTX_new(); - CUDF_EXPECTS(ectx_ != nullptr, "EVP_MD_CTX_new failed"); - CUDF_EXPECTS(EVP_DigestInit_ex(ectx_, type, nullptr) == 1, "EVP_DigestInit_ex failed"); -} - -sha256_context::~sha256_context() -{ - if (ectx_ != nullptr) { EVP_MD_CTX_free(ectx_); } -} - -void sha256_context::update(std::span data) -{ - CUDF_EXPECTS(EVP_DigestUpdate(ectx_, data.data(), data.size()) == 1, "EVP_DigestUpdate failed"); -} - -sha256_hash sha256_context::finalize() -{ - sha256_hash hash; - unsigned int length = 0; - CUDF_EXPECTS(EVP_DigestFinal_ex(ectx_, hash.data_, &length) == 1, "EVP_DigestFinal_ex failed"); - CUDF_EXPECTS(length == sizeof(sha256_hash::data_), "Unexpected SHA256 length"); - EVP_MD const* type = EVP_sha256(); - CUDF_EXPECTS(EVP_DigestInit_ex(ectx_, type, nullptr) == 1, "EVP_DigestInit_ex failed"); - return hash; -} - -} // namespace rtc -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/rtc/sha256.hpp b/cpp/src/jit/rtc/sha256.hpp deleted file mode 100644 index 51763b1bc9fc..000000000000 --- a/cpp/src/jit/rtc/sha256.hpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include -#include - -#include -#include -#include -#include - -extern "C" { -typedef struct evp_md_ctx_st EVP_MD_CTX; -} - -namespace CUDF_EXPORT cudf { -namespace rtc { - -struct [[nodiscard]] sha256_hex_string { - char data_[65]; - - constexpr std::string_view view() const { return std::string_view{data_, 64}; } - - constexpr operator std::string_view() const { return view(); } - - [[nodiscard]] char const* data() const { return data_; } - - [[nodiscard]] char const* c_str() const { return data_; } - - static constexpr size_t size() { return 64; } - - static sha256_hex_string make(std::span input) - { - CUDF_EXPECTS(input.size() == 32, "Input size must be 32 bytes for SHA256 hash"); - constexpr char const HEX_CHARS[] = "0123456789abcdef"; - sha256_hex_string hex; - for (size_t i = 0; i < 32; ++i) { - hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; - hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; - } - hex.data_[64] = '\0'; - return hex; - } -}; - -struct [[nodiscard]] sha256_hash { - alignas(16) uint8_t data_[32]; - - constexpr bool operator==(sha256_hash const& hash) const - { - return std::equal(std::begin(data_), std::end(data_), std::begin(hash.data_)); - } - - constexpr bool operator!=(sha256_hash const& hash) const { return !(*this == hash); } - - sha256_hex_string to_hex_string() const - { - return sha256_hex_string::make(std::span{data_, sizeof(data_)}); - } -}; - -struct [[nodiscard]] sha256_hash_hasher { - constexpr uint64_t operator()(sha256_hash const& obj) const - { - struct u64x4 { - alignas(16) uint64_t v[4]; - }; - - auto value = std::bit_cast(obj); - auto const h0 = value.v[0]; - auto const h1 = value.v[1]; - auto const h2 = value.v[2]; - auto const h3 = value.v[3]; - - auto mix = [](uint64_t seed, uint64_t v) { - seed ^= v + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); - return seed; - }; - - return mix(mix(mix(h0, h1), h2), h3); - } -}; - -struct [[nodiscard]] sha256_context { - private: - EVP_MD_CTX* ectx_; - - public: - sha256_context(); - sha256_context(sha256_context const& other) = delete; - sha256_context& operator=(sha256_context const& other) = delete; - sha256_context(sha256_context&& other) : ectx_(other.ectx_) { other.ectx_ = nullptr; } - - sha256_context& operator=(sha256_context&& other) - { - if (this == &other) [[unlikely]] { return *this; } - this->~sha256_context(); - new (this) sha256_context(std::move(other)); - return *this; - } - - ~sha256_context(); - - void update(std::span data); - - sha256_hash finalize(); -}; - -} // namespace rtc -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/librtcx/CMake/embed.cmake b/cpp/src/librtcx/CMake/embed.cmake new file mode 100644 index 000000000000..a14e19cb8ff9 --- /dev/null +++ b/cpp/src/librtcx/CMake/embed.cmake @@ -0,0 +1,161 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +find_package(Python3 REQUIRED COMPONENTS Interpreter) + +function(jit_add_include_directory) + set(TARGET ${ARGV0}) + set(OPTIONS "") + set(ONE_VALUE_ARGS COPY_DIRECTORY # Source directory where files will be copied from + DEST_DIRECTORY # Destination directory where files will be copied to + ) + set(MULTI_VALUE_ARGS + FILES # Source files relative to COPY_DIRECTORY (optional, if not provided, all files under + # COPY_DIRECTORY will be used) + INCLUDE_DIRECTORIES # Include directories to be used when compiling with these files + ) + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET argument is required") + endif() + + if(NOT ARG_COPY_DIRECTORY) + message(FATAL_ERROR "COPY_DIRECTORY argument is required") + endif() + + if(NOT ARG_DEST_DIRECTORY) + message(FATAL_ERROR "DEST_DIRECTORY argument is required") + endif() + + if(NOT ARG_INCLUDE_DIRECTORIES) + message(FATAL_ERROR "INCLUDE_DIRECTORIES argument is required") + endif() + + if(NOT ARG_FILES) + # gather all include files under the specified directory + file(GLOB_RECURSE INCLUDE_FILES "${ARG_COPY_DIRECTORY}/*") + + # get their paths relative to the base include directory + set(INCLUDE_FILES_RELATIVE_PATHS "") + foreach(INCLUDE_FILE IN LISTS INCLUDE_FILES) + file(RELATIVE_PATH INCLUDE_FILE_REL_PATH "${ARG_COPY_DIRECTORY}" "${INCLUDE_FILE}") + list(APPEND INCLUDE_FILES_RELATIVE_PATHS "${INCLUDE_FILE_REL_PATH}") + endforeach() + + set(ARG_FILES ${INCLUDE_FILES_RELATIVE_PATHS}) + endif() + + # check that each source file exists + foreach(SOURCE_FILE IN LISTS ARG_FILES) + if(NOT EXISTS "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") + message(FATAL_ERROR "Source file '${ARG_COPY_DIRECTORY}/${SOURCE_FILE}' does not exist") + endif() + endforeach(SOURCE_FILE) + + # Set scope variables to accumulate results + + set(SOURCE_FILES ${jitembed_${TARGET}_incdir__source_files}) + set(SOURCE_FILE_DESTS ${jitembed_${TARGET}_incdir__source_file_dests}) + set(INCLUDE_DIRECTORIES ${jitembed_${TARGET}_incdir__include_directories}) + + foreach(SOURCE_FILE IN LISTS ARG_FILES) + list(APPEND SOURCE_FILES "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") + list(APPEND SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}") + endforeach() + + list(APPEND INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES}) + + set(jitembed_${TARGET}_incdir__source_files + ${SOURCE_FILES} + PARENT_SCOPE + ) + set(jitembed_${TARGET}_incdir__source_file_dests + ${SOURCE_FILE_DESTS} + PARENT_SCOPE + ) + set(jitembed_${TARGET}_incdir__include_directories + ${INCLUDE_DIRECTORIES} + PARENT_SCOPE + ) + +endfunction() + +# pass the encoded args to the embed.py script to generate the embed +function(jit_embed) + set(TARGET ${ARGV0}) + set(OPTIONS "") + set(ONE_VALUE_ARGS "COMPRESSION") + set(MULTI_VALUE_ARGS "") + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET argument is required") + endif() + + if(NOT DEFINED ARG_COMPRESSION) + message(FATAL_ERROR "COMPRESSION argument is required") + endif() + + if(NOT ARG_COMPRESSION STREQUAL "none" + AND NOT ARG_COMPRESSION STREQUAL "lz4" + AND NOT ARG_COMPRESSION STREQUAL "zstd" + ) + message(FATAL_ERROR "COMPRESSION argument must be either none, lz4, or, zstd") + endif() + + if(NOT DEFINED jitembed_${TARGET}_incdir__source_files) + message( + FATAL_ERROR + "No source files registered for target '${TARGET}'. Call jit_add_include_directory() first" + ) + endif() + + set(OUTPUT_DIR "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") + set(CONFIGURED_EMBED_PY "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_embed.py") + set(EMBED_PY_IN "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.py.in") + + set(RTCX_EMBED_PY_ARG__ID "${TARGET}") + set(RTCX_EMBED_PY_ARG__FILE_PATHS "${jitembed_${TARGET}_incdir__source_files}") + set(RTCX_EMBED_PY_ARG__FILE_DESTS "${jitembed_${TARGET}_incdir__source_file_dests}") + set(RTCX_EMBED_PY_ARG__INCLUDE_DIRS "${jitembed_${TARGET}_incdir__include_directories}") + set(RTCX_EMBED_PY_ARG__COMPRESSION "${ARG_COMPRESSION}") + set(RTCX_EMBED_PY_ARG__OUTPUT_DIR "${OUTPUT_DIR}") + + configure_file("${EMBED_PY_IN}" "${CONFIGURED_EMBED_PY}" @ONLY) + + add_custom_command( + OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin + BYPRODUCTS ${OUTPUT_DIR}/* + COMMAND ${Python3_EXECUTABLE} "${CONFIGURED_EMBED_PY}" + DEPENDS "${EMBED_PY_IN}" "${CONFIGURED_EMBED_PY}" ${jitembed_${TARGET}_incdir__source_files} + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + COMMENT "Generating JIT embed for ${TARGET} into ${OUTPUT_DIR}" + VERBATIM + ) + + add_custom_target( + ${TARGET} ALL DEPENDS ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s + ${OUTPUT_DIR}/${TARGET}.bin + ) + + message( + STATUS + "JIT embed for target ${TARGET} will be generated into: ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin" + ) + + set(${TARGET}_INCLUDE_DIRS + "${OUTPUT_DIR}" + PARENT_SCOPE + ) + + set(${TARGET}_SOURCE_DIR + ${OUTPUT_DIR} + PARENT_SCOPE + ) + +endfunction() diff --git a/cpp/src/librtcx/CMake/embed.py.in b/cpp/src/librtcx/CMake/embed.py.in new file mode 100644 index 000000000000..716ace56186b --- /dev/null +++ b/cpp/src/librtcx/CMake/embed.py.in @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import hashlib,os, lz4.block, zstd +from typing import NamedTuple + +def merge_bytes_with_null_terminators( + bytes_lists: list[bytes], +) -> tuple[bytes, list[tuple[int, int]]]: + merged: bytes = bytes() + ranges: list[tuple[int, int]] = [] + + for byte_data in bytes_lists: + ranges.append((len(merged), len(byte_data))) + merged += byte_data + b"\0" + + return merged, ranges + +class EmbedOutput(NamedTuple): + cxx_header: str | None + cxx_source: str | None + asm_source: str | None + bin_file_name: str | None + bin_file_data: bytes | None + hash: bytes + +def load_file_bytes(file_path: str) -> bytes: + with open(file_path, "rb") as f: + return f.read() + +def compress_bytes(data: bytes, compression: str) -> bytes: + assert compression in ("none", "lz4", "zstd"), "Invalid compression type" + + if compression == "none": + return data + elif compression == "lz4": + return lz4.block.compress( + data, mode="high_compression", compression=12, store_size=False + ) + elif compression == "zstd": + return zstd.compress(data, 22) + +def generate_cxx_source_files_data( + id: str, + file_paths: list[str], + dests: list[str], + include_directories: list[str], + compression: str, +) -> EmbedOutput: + uncompressed_files_bytes, files_ranges = merge_bytes_with_null_terminators( + [load_file_bytes(p) for p in file_paths] + ) + + compress = compression != "none" + + compressed_files_bytes = ( + compress_bytes(uncompressed_files_bytes, compression) + if compress + else None + ) + + binary_size = len(compressed_files_bytes) if compress else len(uncompressed_files_bytes) + + if compress: + print( + f"-- Compressed {id}'s binary from {len(uncompressed_files_bytes)} bytes to {len(compressed_files_bytes)} bytes (compression ratio: {len(compressed_files_bytes)/len(uncompressed_files_bytes):.2f})" + ) + + merged_dests_bytes, _ = ( + merge_bytes_with_null_terminators([d.encode("utf-8") for d in dests]) + ) + + merged_include_directories_bytes, _ = ( + merge_bytes_with_null_terminators( + [d.encode("utf-8") for d in include_directories] + ) + ) + + # compute combined sha256 hash of all files + sha = hashlib.sha256() + sha.update(uncompressed_files_bytes) + sha.update(merged_dests_bytes) + sha.update(merged_include_directories_bytes) + sha.update(compression.encode("utf-8")) + + hash: bytes = sha.digest() + + binary_file_name = f"embed_{id}.bin" + + cxx_header = f""" + #pragma once +#include +#include +#include +#include + +namespace rtcx_embed {{ + +struct range {{ + std::size_t offset = 0; + std::size_t size = 0; +}}; + +constexpr char const * {id}_include_directories[{len(include_directories)}] = +{{ +{",\n".join([f'"{d}"' for d in include_directories])} +}}; + +constexpr char const * {id}_file_destinations[{len(dests)}] = +{{ +{",\n".join([f'"{d}"' for d in dests])} +}}; + +constexpr range {id}_file_ranges[{len(files_ranges)}] = +{{ +{",\n".join([f'{{{offset}, {size}}}' for offset, size in files_ranges])} +}}; + + +constexpr std::size_t {id}_files_uncompressed_size = {len(uncompressed_files_bytes)}; + +constexpr char const * {id}_files_compression = "{compression}"; + +extern "C" std::uint8_t const rtcx_embed_{id}_files_begin[]; + +static std::span const {id}_files = +{{ +rtcx_embed_{id}_files_begin, +{binary_size}L +}}; + +constexpr std::uint8_t {id}_hash[{len(hash)}] = +{{ +{", ".join([f'0x{b:02x}' for b in hash])} +}}; + +}} +""" + + asm_source = f""" +.section .rodata +.global rtcx_embed_{id}_files_begin +rtcx_embed_{id}_files_begin: +.incbin "{binary_file_name}" + +.section .note.GNU-stack,"",@progbits +""" + + return EmbedOutput( + cxx_header=cxx_header, + cxx_source=None, + asm_source=asm_source, + bin_file_name=binary_file_name, + bin_file_data=compressed_files_bytes + if compress + else uncompressed_files_bytes, + hash=hash, + ) + +def generate_embed( + id: str, + file_paths: list[str], + file_dests: list[str], + include_directories: list[str], + compression: str, + output_dir: str +): + output = generate_cxx_source_files_data( id, file_paths, file_dests, include_directories, compression) + + os.makedirs(output_dir, exist_ok=True) + + with open(f"{output_dir}/{id}.hpp", "w") as f: + f.write(output.cxx_header if output.cxx_header is not None else "") + + with open(f"{output_dir}/{id}.s", "w") as f: + f.write(output.asm_source if output.asm_source is not None else "") + + if output.bin_file_name and output.bin_file_data: + with open(f"{output_dir}/{output.bin_file_name}", "wb") as f: + f.write(output.bin_file_data) + +def main(): + id : str = "@RTCX_EMBED_PY_ARG__ID@" + file_paths_str :str = "@RTCX_EMBED_PY_ARG__FILE_PATHS@" + file_dests_str : str = "@RTCX_EMBED_PY_ARG__FILE_DESTS@" + include_directories_str : str = "@RTCX_EMBED_PY_ARG__INCLUDE_DIRS@" + compression : str = "@RTCX_EMBED_PY_ARG__COMPRESSION@" + output_dir : str = "@RTCX_EMBED_PY_ARG__OUTPUT_DIR@" + + file_paths = file_paths_str.split(";") if file_paths_str else [] + file_dests = file_dests_str.split(";") if file_dests_str else [] + include_directories = include_directories_str.split(";") if include_directories_str else [] + + generate_embed(id, file_paths, file_dests, include_directories, compression, output_dir) + + +if __name__ == "__main__": + main() diff --git a/cpp/src/jit/rtc/README.md b/cpp/src/librtcx/README.md similarity index 100% rename from cpp/src/jit/rtc/README.md rename to cpp/src/librtcx/README.md diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp new file mode 100644 index 000000000000..221b8d24d787 --- /dev/null +++ b/cpp/src/librtcx/rtcx.cpp @@ -0,0 +1,1180 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +extern "C" { +#include +} + +#define RTCX_CONCATENATE_DETAIL(x, y) x##y +#define RTCX_CONCATENATE(x, y) RTCX_CONCATENATE_DETAIL(x, y) + +#define RTCX_CHECK_CUDA(...) \ + do { \ + ::CUresult __result = (__VA_ARGS__); \ + if (__result != ::CUDA_SUCCESS) { \ + char const* __enum_str; \ + RTCX_EXPECTS(::rtcx::cuda->GetErrorString(__result, &__enum_str) == ::CUDA_SUCCESS, \ + "Unable to get CUDA error string", \ + std::runtime_error); \ + auto __errstr = ::std::format("(cuda) expression `{}` failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::rtcx::i64>(__result), \ + __enum_str); \ + RTCX_FAIL(__errstr, ::std::runtime_error); \ + } \ + } while (0) + +#define RTCX_CHECK_CUDART(...) \ + do { \ + ::cudaError_t __result = (__VA_ARGS__); \ + if (__result != ::cudaSuccess) { \ + char const* __enum_name = ::cudaGetErrorName(__result); \ + char const* __enum_msg = ::cudaGetErrorString(__result); \ + auto __errstr = ::std::format("(cudart) expression `{}` failed, with error ({}: {}): {}", \ + #__VA_ARGS__, \ + static_cast<::rtcx::i64>(__result), \ + __enum_name, \ + __enum_msg); \ + RTCX_FAIL(__errstr, ::std::runtime_error); \ + } \ + } while (0) + +#define RTCX_CHECK_NVRTC(params, program, ...) \ + do { \ + ::nvrtcResult __result = (__VA_ARGS__); \ + ::rtcx::log_nvrtc_result(params, program, __result); \ + if (__result != ::NVRTC_SUCCESS) { \ + auto __errstr = ::std::format("(nvrtc) expression `{}` failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::rtcx::i64>(__result), \ + ::rtcx::nvrtc->GetErrorString(__result)); \ + RTCX_FAIL(__errstr, ::std::runtime_error); \ + } \ + } while (0) + +#define RTCX_CHECK_NVJITLINK(params, handle, ...) \ + do { \ + ::nvJitLinkResult __result = (__VA_ARGS__); \ + ::rtcx::log_nvJitLink_result(params, handle, __result); \ + if (__result != ::NVJITLINK_SUCCESS) { \ + auto __errstr = ::std::format("(nvJitLink) expression `{}` failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::rtcx::i64>(__result), \ + ::rtcx::get_nvJitLinkResultString(__result)); \ + RTCX_FAIL(__errstr, ::std::runtime_error); \ + } \ + } while (0) + +namespace RTCX_EXPORT rtcx { + +void log_warning(std::string_view message) +{ + std::fprintf(stdout, "RTCX WARNING: %.*s\n", static_cast(message.size()), message.data()); +} + +void log_error(std::string_view message) +{ + std::fprintf(stderr, "RTCX ERROR: %.*s\n", static_cast(message.size()), message.data()); +} + +sha256_context::sha256_context() : ectx_(nullptr) +{ + const EVP_MD* type = EVP_sha256(); + ectx_ = EVP_MD_CTX_new(); + RTCX_EXPECTS(ectx_ != nullptr, "EVP_MD_CTX_new failed", std::runtime_error); + RTCX_EXPECTS( + EVP_DigestInit_ex(ectx_, type, nullptr) == 1, "EVP_DigestInit_ex failed", std::runtime_error); +} + +sha256_context::~sha256_context() +{ + if (ectx_ != nullptr) { EVP_MD_CTX_free(ectx_); } +} + +void sha256_context::update(std::span data) +{ + RTCX_EXPECTS(EVP_DigestUpdate(ectx_, data.data(), data.size()) == 1, + "EVP_DigestUpdate failed", + std::runtime_error); +} + +sha256 sha256_context::finalize() +{ + sha256 hash; + u32 length = 0; + RTCX_EXPECTS(EVP_DigestFinal_ex(ectx_, hash.data_, &length) == 1, + "EVP_DigestFinal_ex failed", + std::runtime_error); + RTCX_EXPECTS(length == sizeof(sha256::data_), "Unexpected SHA256 length", std::runtime_error); + EVP_MD const* type = EVP_sha256(); + RTCX_EXPECTS( + EVP_DigestInit_ex(ectx_, type, nullptr) == 1, "EVP_DigestInit_ex failed", std::runtime_error); + return hash; +} + +#define FOR_EACH_CUDA_FUNC(DO_IT) \ + DO_IT(GetErrorString) \ + DO_IT(GetErrorName) \ + DO_IT(Init) \ + DO_IT(DeviceGet) \ + DO_IT(DeviceGetCount) \ + DO_IT(DeviceGetName) \ + DO_IT(OccupancyMaxPotentialBlockSize) \ + DO_IT(LaunchKernel) \ + DO_IT(LaunchKernelEx) \ + DO_IT(KernelGetName) \ + DO_IT(LibraryLoadData) \ + DO_IT(LibraryGetKernel) \ + DO_IT(LibraryGetKernelCount) \ + DO_IT(LibraryEnumerateKernels) \ + DO_IT(LibraryUnload) + +#define FOR_EACH_NVRTC_FUNC(DO_IT) \ + DO_IT(GetErrorString) \ + DO_IT(CreateProgram) \ + DO_IT(DestroyProgram) \ + DO_IT(CompileProgram) \ + DO_IT(GetPTXSize) \ + DO_IT(GetPTX) \ + DO_IT(GetCUBINSize) \ + DO_IT(GetCUBIN) \ + DO_IT(GetLTOIRSize) \ + DO_IT(GetLTOIR) \ + DO_IT(GetProgramLogSize) \ + DO_IT(GetProgramLog) \ + DO_IT(AddNameExpression) \ + DO_IT(GetLoweredName) \ + DO_IT(GetPCHHeapSize) \ + DO_IT(SetPCHHeapSize) \ + DO_IT(GetPCHCreateStatus) \ + DO_IT(GetPCHHeapSizeRequired) \ + DO_IT(SetFlowCallback) + +#define FOR_EACH_NVJITLINK_FUNC(DO_IT) \ + DO_IT(Create) \ + DO_IT(Destroy) \ + DO_IT(AddData) \ + DO_IT(AddFile) \ + DO_IT(Complete) \ + DO_IT(GetLinkedCubinSize) \ + DO_IT(GetLinkedCubin) \ + DO_IT(GetLinkedPtxSize) \ + DO_IT(GetLinkedPtx) \ + DO_IT(GetErrorLogSize) \ + DO_IT(GetErrorLog) \ + DO_IT(GetInfoLog) \ + DO_IT(GetInfoLogSize) + +namespace { + +void* load_dll(std::string_view base_name, std::span names) +{ + for (auto const& name : names) { + void* handle = ::dlopen(name.c_str(), RTLD_NOW | RTLD_LOCAL); + if (handle != nullptr) { return handle; } + } + + std::string tried_names = std::accumulate( + names.begin(), names.end(), std::string{}, [&](std::string acc, std::string const& name) { + if (!acc.empty()) { acc += ", "; } + acc += name; + return acc; + }); + + RTCX_FAIL(std::format("Failed to load dynamic library `{}` (tried: {})", base_name, tried_names), + std::runtime_error); +} + +void* get_symbol(char const* lib_name, void* handle, char const* sym_name) +{ + void* sym = ::dlsym(handle, sym_name); + if (sym == nullptr) { + RTCX_FAIL(std::format( + "Failed to load symbol `{}` from `{}`, error: `{}`", sym_name, lib_name, dlerror()), + std::runtime_error); + } + return sym; +} + +inline constexpr i32 major_version(i32 version) { return version / 1000; } + +inline constexpr i32 minor_version(i32 version) { return (version % 1000) / 10; } + +struct LibCuda { + void* _handle = nullptr; + +#define DO_IT(func) decltype(::cu##func)* func = nullptr; + FOR_EACH_CUDA_FUNC(DO_IT) +#undef DO_IT + + explicit LibCuda(void* handle) : _handle(handle) { _load_symbols(); } + LibCuda(LibCuda const&) = delete; + LibCuda(LibCuda&&) = delete; + LibCuda& operator=(LibCuda const&) = delete; + LibCuda& operator=(LibCuda&&) = delete; + ~LibCuda() { dlclose(_handle); } + + static void* _load() + { + std::string lib_names[] = {"libcuda.so"}; + return load_dll("libcuda.so", lib_names); + } + + private: + void _load_symbols() + { +#define DO_IT(func) \ + this->func = reinterpret_cast(get_symbol("libcuda", _handle, "cu" #func)); + FOR_EACH_CUDA_FUNC(DO_IT) +#undef DO_IT + } +}; + +struct LibNVRTC { + void* _handle = nullptr; + +#define DO_IT(func) decltype(::nvrtc##func)* func = nullptr; + FOR_EACH_NVRTC_FUNC(DO_IT) +#undef DO_IT + + explicit LibNVRTC(void* handle) : _handle(handle) { _load_symbols(); } + LibNVRTC(LibNVRTC const&) = delete; + LibNVRTC(LibNVRTC&&) = delete; + LibNVRTC& operator=(LibNVRTC const&) = delete; + LibNVRTC& operator=(LibNVRTC&&) = delete; + ~LibNVRTC() { dlclose(_handle); } + + static void* _load() + { + i32 cuda_version; + RTCX_CHECK_CUDART(cudaRuntimeGetVersion(&cuda_version)); + + i32 major = major_version(cuda_version); + i32 minor = minor_version(cuda_version); + + std::string lib_names[] = {std::format("libnvrtc.so.{}.{}", major, minor), + std::format("libnvrtc.so.{}", major), + "libnvrtc.so"}; + + return load_dll("libnvrtc.so", lib_names); + } + + private: + void _load_symbols() + { +#define DO_IT(func) \ + this->func = \ + reinterpret_cast(get_symbol("libnvrtc", _handle, "nvrtc" #func)); + FOR_EACH_NVRTC_FUNC(DO_IT) +#undef DO_IT + } +}; + +struct LibNVJitLink { + void* _handle = nullptr; + +#define DO_IT(func) decltype(::nvJitLink##func)* func = nullptr; + FOR_EACH_NVJITLINK_FUNC(DO_IT) +#undef DO_IT + + explicit LibNVJitLink(void* handle) : _handle(handle) { _load_symbols(); } + LibNVJitLink(LibNVJitLink const&) = delete; + LibNVJitLink(LibNVJitLink&&) = delete; + LibNVJitLink& operator=(LibNVJitLink const&) = delete; + LibNVJitLink& operator=(LibNVJitLink&&) = delete; + ~LibNVJitLink() { dlclose(_handle); } + + static void* _load() + { + i32 cuda_version; + RTCX_CHECK_CUDART(cudaRuntimeGetVersion(&cuda_version)); + + i32 major = major_version(cuda_version); + i32 minor = minor_version(cuda_version); + + std::string lib_names[] = {std::format("libnvJitLink.so.{}.{}", major, minor), + std::format("libnvJitLink.so.{}", major), + "libnvJitLink.so"}; + + return load_dll("libnvJitLink.so", lib_names); + } + + private: + void _load_symbols() + { +#define DO_IT(func) \ + this->func = reinterpret_cast( \ + get_symbol("libnvJitLink", _handle, "nvJitLink" #func)); + FOR_EACH_NVJITLINK_FUNC(DO_IT) +#undef DO_IT + } +}; + +static std::optional cuda; +static std::optional nvrtc; +static std::optional nvjitlink; +static std::optional init_libraries_flag{std::in_place}; +static std::optional teardown_libraries_flag{std::in_place}; + +} // namespace + +void initialize() +{ + std::call_once(*init_libraries_flag, [] { + cuda.emplace(LibCuda::_load()); + nvrtc.emplace(LibNVRTC::_load()); + nvjitlink.emplace(LibNVJitLink::_load()); + }); +} + +void teardown() +{ + std::call_once(*teardown_libraries_flag, [] { + cuda.reset(); + nvrtc.reset(); + nvjitlink.reset(); + init_libraries_flag.reset(); + teardown_libraries_flag.reset(); + }); +} + +namespace { + +char const* get_nvJitLinkResultString(nvJitLinkResult result) +{ + switch (result) { + case NVJITLINK_SUCCESS: return "NVJITLINK_SUCCESS"; + case NVJITLINK_ERROR_UNRECOGNIZED_OPTION: return "NVJITLINK_ERROR_UNRECOGNIZED_OPTION"; + case NVJITLINK_ERROR_MISSING_ARCH: return "NVJITLINK_ERROR_MISSING_ARCH"; + case NVJITLINK_ERROR_INVALID_INPUT: return "NVJITLINK_ERROR_INVALID_INPUT"; + case NVJITLINK_ERROR_PTX_COMPILE: return "NVJITLINK_ERROR_PTX_COMPILE"; + case NVJITLINK_ERROR_NVVM_COMPILE: return "NVJITLINK_ERROR_NVVM_COMPILE"; + case NVJITLINK_ERROR_INTERNAL: return "NVJITLINK_ERROR_INTERNAL"; + case NVJITLINK_ERROR_THREADPOOL: return "NVJITLINK_ERROR_THREADPOOL"; + case NVJITLINK_ERROR_UNRECOGNIZED_INPUT: return "NVJITLINK_ERROR_UNRECOGNIZED_INPUT"; + case NVJITLINK_ERROR_FINALIZE: return "NVJITLINK_ERROR_FINALIZE"; +#if CUDA_VERSION >= 13000 + case NVJITLINK_ERROR_NULL_INPUT: return "NVJITLINK_ERROR_NULL_INPUT"; + case NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS: return "NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS"; + case NVJITLINK_ERROR_INCORRECT_INPUT_TYPE: return "NVJITLINK_ERROR_INCORRECT_INPUT_TYPE"; + case NVJITLINK_ERROR_ARCH_MISMATCH: return "NVJITLINK_ERROR_ARCH_MISMATCH"; + case NVJITLINK_ERROR_OUTDATED_LIBRARY: return "NVJITLINK_ERROR_OUTDATED_LIBRARY"; + case NVJITLINK_ERROR_MISSING_FATBIN: return "NVJITLINK_ERROR_MISSING_FATBIN"; + case NVJITLINK_ERROR_UNRECOGNIZED_ARCH: return "NVJITLINK_ERROR_UNRECOGNIZED_ARCH"; + case NVJITLINK_ERROR_UNSUPPORTED_ARCH: return "NVJITLINK_ERROR_UNSUPPORTED_ARCH"; + case NVJITLINK_ERROR_LTO_NOT_ENABLED: return "NVJITLINK_ERROR_LTO_NOT_ENABLED"; +#endif + default: + RTCX_FAIL(std::format("Unrecognized nvJitLinkResult type: ({})", static_cast(result)), + std::runtime_error); + } +} + +char const* binary_type_string(binary_type type) +{ + switch (type) { + case binary_type::LTO_IR: return "LTO_IR"; + case binary_type::CUBIN: return "CUBIN"; + case binary_type::FATBIN: return "FATBIN"; + case binary_type::PTX: return "PTX"; + default: + RTCX_FAIL(std::format("Unrecognized binary_type: ({})", static_cast(type)), + std::runtime_error); + } +} + +nvJitLinkInputType to_nvjitlink_input_type(binary_type bin_type) +{ + switch (bin_type) { + case binary_type::LTO_IR: return NVJITLINK_INPUT_LTOIR; + case binary_type::CUBIN: return NVJITLINK_INPUT_CUBIN; + case binary_type::FATBIN: return NVJITLINK_INPUT_FATBIN; + case binary_type::PTX: return NVJITLINK_INPUT_PTX; + default: + RTCX_FAIL( + std::format("Unrecognized binary type for linking: ({}) ", static_cast(bin_type)), + std::logic_error); + } +} + +void log_nvrtc_result(compile_params const& params, + nvrtcProgram program, + nvrtcResult compile_result) +{ + if (program == nullptr) { return; } + + usize log_size; + if (auto errc = nvrtc->GetProgramLogSize(program, &log_size); errc != NVRTC_SUCCESS) { + RTCX_FAIL(std::format("Failed to get NVRTC program log size with error ({}): {}", + static_cast(errc), + nvrtc->GetErrorString(errc)), + std::runtime_error); + } + + if (log_size <= 1) { return; } + + std::vector log; + log.resize(log_size); + + if (auto errc = nvrtc->GetProgramLog(program, log.data()); errc != NVRTC_SUCCESS) { + RTCX_FAIL(std::format("Failed to get NVRTC program log with error ({}): {}", + static_cast(errc), + nvrtc->GetErrorString(errc)), + std::runtime_error); + } + + log.resize(log_size == 0 ? 0 : (log_size - 1)); + + auto status_str = (compile_result == NVRTC_SUCCESS && !log.empty()) ? "completed with warning" + : "failed with error"; + + std::string headers_str; + for (auto const& header : params.header_include_names) { + headers_str = std::format("{}\t{}\n", headers_str, header); + } + + std::string options_str; + for (auto const& option : params.options) { + options_str = std::format("{}\t{}\n", options_str, option); + } + + auto msg = std::format( + "NVRTC Compilation for `{}` {} ({}): {}.\nHeaders:\n{}\n\nOptions:\n{}\n\nLog:\n\t{}", + params.name == nullptr ? "" : params.name, + status_str, + static_cast(compile_result), + nvrtc->GetErrorString(compile_result), + headers_str, + options_str, + std::string_view{log.data(), log.size()}); + + if (compile_result != NVRTC_SUCCESS) { + log_error(msg); + } else { + log_warning(msg); + } +} + +void log_nvJitLink_result(link_params const& params, + nvJitLinkHandle handle, + nvJitLinkResult link_result) +{ + if (handle == nullptr) { return; } + + usize info_log_size; + if (auto errc = nvjitlink->GetInfoLogSize(handle, &info_log_size); errc != NVJITLINK_SUCCESS) { + RTCX_FAIL(std::format("Failed to get nvJitLink info log size with error ({}): {}", + static_cast(errc), + get_nvJitLinkResultString(errc)), + std::runtime_error); + } + + std::vector info_log; + if (info_log_size > 1) { + info_log.resize(info_log_size); + if (auto errc = nvjitlink->GetInfoLog(handle, info_log.data()); errc != NVJITLINK_SUCCESS) { + RTCX_FAIL(std::format("Failed to get nvJitLink info log with error ({}): {}", + static_cast(errc), + get_nvJitLinkResultString(errc)), + std::runtime_error); + } + } + info_log.resize(info_log_size == 0 ? 0 : (info_log_size - 1)); + + usize error_log_size; + if (auto errc = nvjitlink->GetErrorLogSize(handle, &error_log_size); errc != NVJITLINK_SUCCESS) { + RTCX_FAIL(std::format("Failed to get nvJitLink error log size with error ({}): {}", + static_cast(errc), + get_nvJitLinkResultString(errc)), + std::runtime_error); + } + + std::vector error_log; + + if (error_log_size > 1) { + error_log.resize(error_log_size); + if (auto errc = nvjitlink->GetErrorLog(handle, error_log.data()); errc != NVJITLINK_SUCCESS) { + RTCX_FAIL(std::format("Failed to get nvJitLink error log with error ({}): {}", + static_cast(errc), + get_nvJitLinkResultString(errc)), + std::runtime_error); + } + } + error_log.resize(error_log_size == 0 ? 0 : (error_log_size - 1)); + + if (info_log.empty() && error_log.empty()) { return; } + + std::string fragments_str; + for (auto const& fragment_name : params.fragment_names) { + fragments_str = std::format("{}\t{}\n", fragments_str, fragment_name); + } + + std::string link_options_str; + for (auto const& option : params.link_options) { + link_options_str = std::format("{}\t{}\n", link_options_str, option); + } + + char const* binary_type_str = binary_type_string(params.output_type); + + auto status_str = error_log.empty() ? "completed with warnings" : "failed with errors"; + + auto msg = std::format( + "(nvJitLink) Linking for `{}` ({}) {}, error code ({}): {}.\nFragments: \n{}\n" + "Link Options: \n{}\n\nInfo Log:\n\t{}\n\nError Log:\n\t{}\n\n", + params.name == nullptr ? "" : params.name, + binary_type_str, + status_str, + static_cast(link_result), + get_nvJitLinkResultString(link_result), + fragments_str, + link_options_str, + std::string_view{info_log.data(), info_log.size()}, + std::string_view{error_log.data(), error_log.size()}); + + if (!error_log.empty()) { + log_error(msg); + } else { + log_warning(msg); + } +} + +} // namespace + +blob_t blob_t::from_vector(std::vector&& data) +{ + auto ptr = new std::vector(std::move(data)); + return blob_t::from_parts( + ptr->data(), ptr->size(), blob_t::deallocator{ptr, [](void* user_data, u8 const*, usize) { + delete reinterpret_cast*>( + user_data); + }}); +} + +blob_t blob_t::from_static_data(std::span data) +{ + return blob_t::from_parts(data.data(), data.size(), blob_t::noop_deallocator); +} + +std::vector compile(compile_params const& params) +{ + RTCX_EXPECTS(params.name != nullptr, "Fragment name must not be null", std::logic_error); + RTCX_EXPECTS(params.source != nullptr, "Fragment source must not be null", std::logic_error); + + nvrtcProgram program = nullptr; + + RTCX_CHECK_NVRTC(params, + program, + nvrtc->CreateProgram(&program, + params.source, + params.name, + static_cast(params.headers.size()), + params.headers.data(), + params.header_include_names.data())); + + RTCX_DEFER([&] { nvrtc->DestroyProgram(&program); }); + + // TODO: log is printed twice when warnings are raised + RTCX_CHECK_NVRTC( + params, + program, + nvrtc->CompileProgram(program, static_cast(params.options.size()), params.options.data())); + + switch (params.target_type) { + case binary_type::LTO_IR: { + usize lto_ir_size; + RTCX_CHECK_NVRTC(params, program, nvrtc->GetLTOIRSize(program, <o_ir_size)); + + std::vector lto_ir; + lto_ir.resize(lto_ir_size); + + RTCX_CHECK_NVRTC( + params, program, nvrtc->GetLTOIR(program, reinterpret_cast(lto_ir.data()))); + + return lto_ir; + + } break; + case binary_type::CUBIN: { + usize cubin_size; + RTCX_CHECK_NVRTC(params, program, nvrtc->GetCUBINSize(program, &cubin_size)); + + std::vector cubin; + cubin.resize(cubin_size); + RTCX_CHECK_NVRTC( + params, program, nvrtc->GetCUBIN(program, reinterpret_cast(cubin.data()))); + + return cubin; + + } break; + default: RTCX_FAIL("Unsupported binary type for compiling fragment", std::logic_error); + } +} + +kernel_occupancy_config kernel_ref::max_occupancy_config(usize dynamic_shared_memory_bytes, + i32 block_size_limit) const +{ + i32 min_grid_size; + i32 block_size; + RTCX_CHECK_CUDA(cuda->OccupancyMaxPotentialBlockSize(&min_grid_size, + &block_size, + reinterpret_cast(handle_), + nullptr, + dynamic_shared_memory_bytes, + block_size_limit)); + + return kernel_occupancy_config{.min_grid_size = min_grid_size, .block_size = block_size}; +} + +void kernel_ref::launch(u32 grid_dim_x, + u32 grid_dim_y, + u32 grid_dim_z, + u32 block_dim_x, + u32 block_dim_y, + u32 block_dim_z, + u32 shared_mem_bytes, + CUstream stream, + void** kernel_params) const +{ + RTCX_EXPECTS(grid_dim_x > 0 && grid_dim_y > 0 && grid_dim_z > 0, + "Grid dimensions must be greater than zero", + std::logic_error); + RTCX_EXPECTS(block_dim_x > 0 && block_dim_y > 0 && block_dim_z > 0, + "Block dimensions must be greater than zero", + std::logic_error); + RTCX_EXPECTS( + kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); + + CUlaunchConfig cfg{.gridDimX = grid_dim_x, + .gridDimY = grid_dim_y, + .gridDimZ = grid_dim_z, + .blockDimX = block_dim_x, + .blockDimY = block_dim_y, + .blockDimZ = block_dim_z, + .sharedMemBytes = shared_mem_bytes, + .hStream = stream, + .attrs = nullptr, + .numAttrs = 0}; + + RTCX_CHECK_CUDA( + cuda->LaunchKernelEx(&cfg, reinterpret_cast(handle_), kernel_params, nullptr)); +} + +std::string_view kernel_ref::get_name() const +{ + char const* name; + RTCX_CHECK_CUDA(cuda->KernelGetName(&name, handle_)); + return std::string_view{name == nullptr ? "" : name}; +} + +library_t::~library_t() +{ + if (handle_ != nullptr) { + if (cuda->LibraryUnload(handle_) != CUDA_SUCCESS) { std::terminate(); } + } +} + +library load_library(std::span binary, binary_type type) +{ + CUlibrary handle; + + RTCX_CHECK_CUDA( + cuda->LibraryLoadData(&handle, binary.data(), nullptr, nullptr, 0, nullptr, nullptr, 0)); + + RTCX_DEFER([&] { + if (handle != nullptr) { RTCX_CHECK_CUDA(cuda->LibraryUnload(handle)); } + }); + + auto library = std::make_shared(handle); + + handle = nullptr; + + return library; +} + +std::vector link_library(link_params const& params) +{ + RTCX_EXPECTS(params.name != nullptr, "Link output name must not be null", std::logic_error); + RTCX_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, + "Only CUBIN and PTX output types are supported for linking modules", + std::logic_error); + RTCX_EXPECTS(params.fragments.size() == params.fragment_binary_types.size(), + "Mismatched number of fragments and fragment binary types", + std::logic_error); + RTCX_EXPECTS(params.fragments.size() == params.fragment_names.size(), + "Mismatched number of fragments and fragment names", + std::logic_error); + RTCX_EXPECTS(params.fragments.size() > 0, "No fragments provided for linking", std::logic_error); + + for (auto& frag : params.fragments) { + RTCX_EXPECTS(frag.size_bytes() > 0, "Fragment binary data must be non-empty", std::logic_error); + } + + nvJitLinkHandle handle = nullptr; + + RTCX_CHECK_NVJITLINK(params, + handle, + nvjitlink->Create(&handle, + static_cast(params.link_options.size()), + const_cast(params.link_options.data()))); + + RTCX_DEFER([&] { nvjitlink->Destroy(&handle); }); + + for (usize i = 0; i < params.fragments.size(); i++) { + auto name = params.fragment_names[i]; + auto fragment = params.fragments[i]; + auto bin_type = params.fragment_binary_types[i]; + nvJitLinkInputType nv_type = to_nvjitlink_input_type(bin_type); + + RTCX_CHECK_NVJITLINK( + params, + handle, + nvjitlink->AddData(handle, nv_type, fragment.data(), fragment.size_bytes(), name)); + } + + RTCX_CHECK_NVJITLINK(params, handle, nvjitlink->Complete(handle)); + + switch (params.output_type) { + case binary_type::CUBIN: { + usize cubin_size; + RTCX_CHECK_NVJITLINK(params, handle, nvjitlink->GetLinkedCubinSize(handle, &cubin_size)); + std::vector cubin; + cubin.resize(cubin_size); + RTCX_CHECK_NVJITLINK(params, handle, nvjitlink->GetLinkedCubin(handle, cubin.data())); + return cubin; + } break; + + case binary_type::PTX: { + usize ptx_size; + + RTCX_CHECK_NVJITLINK(params, handle, nvjitlink->GetLinkedPtxSize(handle, &ptx_size)); + std::vector ptx; + ptx.resize(ptx_size); + + RTCX_CHECK_NVJITLINK( + params, handle, nvjitlink->GetLinkedPtx(handle, reinterpret_cast(ptx.data()))); + + return ptx; + } break; + + default: + RTCX_FAIL(std::format("Unsupported output binary type for linking CUDA libraries: ({})", + static_cast(params.output_type)), + std::runtime_error); + } +} + +kernel_ref library_t::get_kernel(char const* name) const +{ + CUkernel kernel; + RTCX_CHECK_CUDA(cuda->LibraryGetKernel(&kernel, handle_, name)); + return kernel_ref{kernel}; +} + +std::vector library_t::enumerate_kernels() const +{ + u32 num_kernels; + RTCX_CHECK_CUDA(cuda->LibraryGetKernelCount(&num_kernels, handle_)); + + std::vector kernels; + kernels.resize(num_kernels); + + RTCX_CHECK_CUDA(cuda->LibraryEnumerateKernels(kernels.data(), num_kernels, handle_)); + + std::vector result; + for (CUkernel k : kernels) { + result.emplace_back(k); + } + + return result; +} + +std::string demangle_cuda_symbol(char const* mangled_name) +{ + i32 status; + usize length; + + char* demangled_name = abi::__cxa_demangle(mangled_name, nullptr, &length, &status); + + RTCX_EXPECTS(status == 0, "Demangling CUDA symbol name failed", std::runtime_error); + RTCX_EXPECTS(demangled_name != nullptr, "Demangling CUDA symbol name failed", std::runtime_error); + + RTCX_DEFER([&] { + if (demangled_name != nullptr) free(demangled_name); + }); + + std::string result{demangled_name}; + + return result; +} + +namespace { + +[[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) +{ + auto errc = errno; + RTCX_FAIL( + std::format("{}. `{}` failed with {} ({})", message, syscall_name, errc, std::strerror(errc)), + std::runtime_error); +} + +} // namespace + +cache_t::cache_t(std::string cache_dir, cache_limits const& limits) + : cache_dir_{std::move(cache_dir)}, + limits_{limits}, + blobs_cache_{limits.num_blobs}, + libraries_cache_{limits.num_libraries}, + tick_{0} +{ + RTCX_EXPECTS(limits.num_blobs >= 2, "Blob cache limit must be at least 2", std::logic_error); + RTCX_EXPECTS( + limits.num_libraries >= 2, "Library cache limit must be at least 2", std::logic_error); +} + +std::string const& cache_t::get_cache_dir() { return cache_dir_; } + +std::optional blob_t::from_file(char const* path) +{ + i32 fd = open(path, O_RDONLY); + + if (fd == -1) { + if (errno == ENOENT) { + return std::nullopt; + } else { + throw_posix("Failed to open RTC cache file from disk", "open"); + } + } + + auto file_size = lseek(fd, 0, SEEK_END); + if (file_size == -1) { throw_posix("Failed to determine size of RTC cache file", "lseek"); } + + void* map = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); + + if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTC cache file", "mmap"); } + + if (close(fd) == -1) { + throw_posix("Failed to close RTC cache file after memory-mapping", "close"); + } + + auto deleter = +[](u8 const* buffer, usize size) { + if (munmap(static_cast(const_cast(buffer)), size) == -1) { + throw_posix("Failed to unmap RTC cache file from memory", "munmap"); + } + }; + + return blob_t::from_parts(static_cast(map), file_size, deleter); +} + +namespace { + +/// @brief retrieves a blob from disk based on the given sha256 hash and object type (e.g. "blob", +/// "fragment", "library"). Returns nullopt if the file doesn't exist on disk, and throws if any +/// other error occurs. +std::optional get_disk_blob(std::string const& cache_dir, + std::string const& object_type, + sha256 const& sha) +{ + auto hex = sha.to_hex_string(); + auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); + + auto blob = blob_t::from_file(path.c_str()); + + if (!blob.has_value()) { return std::nullopt; } + { + return std::make_shared(std::move(*blob)); + } +} + +/// @brief atomically writes a blob to disk by first writing to a temporary file and then renaming +/// it to the final path. +void add_blob_to_disk(std::string const& cache_dir, + std::string const& object_type, + sha256 const& sha, + std::span binary) +{ + char temp_path[] = "/tmp/rtcx-bin-XXXXXX"; + + { + i32 fd = mkstemp(temp_path); + if (fd == -1) { throw_posix("Failed to create temporary file for RTC cache", "mkstemp"); } + + RTCX_DEFER([&] { + if (close(fd) == -1) { throw_posix("Failed to close temporary RTC cache file", "close"); } + }); + + if (write(fd, binary.data(), binary.size()) == -1) { + throw_posix("Failed to write RTC cache to temporary file", "write"); + } + } + + auto hex = sha.to_hex_string(); + auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); + + std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); + + // rename is atomic, even if another process is performing the same operation + if (rename(temp_path, final_path.c_str()) == -1) { + auto errc = errno; + + if (errc == EEXIST) { + // another process has already created the file, so just remove our temp file + if (remove(temp_path) == -1) { + throw_posix("Failed to remove temporary RTC cache file", "remove"); + } + return; + } else { + throw_posix( + std::format("Failed to move temporary RTC cache file to final location ({})", final_path), + "rename"); + } + } +} + +} // namespace + +std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compile_func compile) +{ + std::atomic_ref tick{tick_}; + auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + + bool unlocked = false; + lock_.lock(); + + RTCX_DEFER([&] { + if (!unlocked) { lock_.unlock(); } + }); + + // check memory cache + if (auto it = blobs_cache_.entries_.find(sha); it != blobs_cache_.entries_.end()) { + counter_.blob_mem_hits.incr(); + + // update LRU tick + it->second.hit(current_tick); + + return it->second.value; + + } else { + counter_.blob_mem_misses.incr(); + + // check disk cache + auto disk_blob = get_disk_blob(cache_dir_, "blob", sha); + + std::promise promise; + auto fut = promise.get_future().share(); + auto cache_fut = fut; + auto ret_fut = fut; + + if (disk_blob.has_value()) { + counter_.blob_disk_hits.incr(); + + promise.set_value(std::move(*disk_blob)); + + // insert into cache + blobs_cache_.insert(sha, std::move(cache_fut), current_tick); + + return ret_fut; + + } else { + counter_.blob_disk_misses.incr(); + + blobs_cache_.insert(sha, std::move(cache_fut), current_tick); + + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this sha + lock_.unlock(); + unlocked = true; + + auto result = compile(); + promise.set_value(result); + + // store result to disk + add_blob_to_disk(cache_dir_, "blob", sha, result->view()); + + return ret_fut; + } + } +} + +std::shared_future cache_t::get_or_add_library(sha256 const& sha, + binary_type type, + library_compile_func compile) +{ + std::atomic_ref tick{tick_}; + auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + + bool unlocked = false; + lock_.lock(); + + RTCX_DEFER([&] { + if (!unlocked) { lock_.unlock(); } + }); + + // check memory cache + if (auto it = libraries_cache_.entries_.find(sha); it != libraries_cache_.entries_.end()) { + counter_.library_mem_hits.incr(); + + // update LRU tick + it->second.hit(current_tick); + + return it->second.value; + + } else { + counter_.library_mem_misses.incr(); + + // check disk cache + auto disk_blob = get_disk_blob(cache_dir_, "library", sha); + + std::promise promise; + auto fut = promise.get_future().share(); + auto cache_fut = fut; + auto ret_fut = fut; + + if (disk_blob.has_value()) { + counter_.library_disk_hits.incr(); + + libraries_cache_.insert(sha, std::move(cache_fut), current_tick); + + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this sha + lock_.unlock(); + unlocked = true; + + auto lib = load_library((*disk_blob)->view(), type); + promise.set_value(std::move(lib)); + + return ret_fut; + + } else { + counter_.library_disk_misses.incr(); + + libraries_cache_.insert(sha, std::move(cache_fut), current_tick); + + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this sha + lock_.unlock(); + unlocked = true; + + auto [library, blob] = compile(); + promise.set_value(library); + + // store result to disk + add_blob_to_disk(cache_dir_, "library", sha, blob->view()); + + return ret_fut; + } + } +} + +cache_stats cache_t::get_stats() +{ + return cache_stats{.blob_mem_hits = counter_.blob_mem_hits.get(), + .blob_mem_misses = counter_.blob_mem_misses.get(), + .blob_disk_hits = counter_.blob_disk_hits.get(), + .blob_disk_misses = counter_.blob_disk_misses.get(), + .library_mem_hits = counter_.library_mem_hits.get(), + .library_mem_misses = counter_.library_mem_misses.get(), + .library_disk_hits = counter_.library_disk_hits.get(), + .library_disk_misses = counter_.library_disk_misses.get()}; +} + +void cache_t::clear_stats() +{ + counter_.blob_mem_hits.reset(); + counter_.blob_mem_misses.reset(); + counter_.blob_disk_hits.reset(); + counter_.blob_disk_misses.reset(); + counter_.library_mem_hits.reset(); + counter_.library_mem_misses.reset(); + counter_.library_disk_hits.reset(); + counter_.library_disk_misses.reset(); +} + +cache_limits cache_t::get_limits() { return limits_; } + +usize cache_t::get_blob_count() +{ + std::lock_guard guard{lock_}; + return blobs_cache_.entries_.size(); +} + +usize cache_t::get_library_count() +{ + std::lock_guard guard{lock_}; + return libraries_cache_.entries_.size(); +} + +void cache_t::clear_memory_store() +{ + std::lock_guard guard{lock_}; + + blobs_cache_.entries_.clear(); + libraries_cache_.entries_.clear(); +} + +void cache_t::clear_disk_store() +{ + DIR* dir = opendir(cache_dir_.c_str()); + + if (dir == nullptr) { throw_posix("Failed to open RTC cache directory for clearing", "opendir"); } + + RTCX_DEFER([&] { closedir(dir); }); + + errno = 0; // reset errno before reading + + dirent* entry_iter = nullptr; + std::vector entry_path; + entry_path.resize(PATH_MAX + 1); + + while (true) { + entry_iter = readdir(dir); + + if (entry_iter == nullptr) { + if (errno != 0) { + throw_posix("Failed to read RTC cache directory for clearing", "readdir"); + } else { + break; + } + } + + struct stat entry_stat; + + if (lstat(entry_path.data(), &entry_stat) == -1) { + throw_posix("Failed to get file status for RTC cache clearing", "lstat"); + } + + if (S_ISREG(entry_stat.st_mode)) { + if (unlink(entry_path.data()) == -1) { + throw_posix("Failed to unlink RTC cache file during clearing", "unlink"); + } + } + + // reset errno for next iteration + errno = 0; + } + + return; +} +} // namespace RTCX_EXPORT rtcx diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp new file mode 100644 index 000000000000..2c0ca141075f --- /dev/null +++ b/cpp/src/librtcx/rtcx.hpp @@ -0,0 +1,704 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RTCX_EXPORT __attribute__((visibility("default"))) + +#define RTCX_DEFER__CONCATENATE_DETAIL(x, y) x##y +#define RTCX_DEFER__CONCATENATE(x, y) RTCX_DEFER__CONCATENATE_DETAIL(x, y) +#define RTCX_DEFER(...) ::rtcx::defer RTCX_DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) + +#define RTCX_EXPECTS(_condition, _reason, _exception_type) \ + do { \ + if (!(_condition)) { \ + throw _exception_type{::std::format("RTC failure at: {}:{}: {}", \ + ::std::source_location::current().file_name(), \ + ::std::source_location::current().line(), \ + (_reason))}; \ + } \ + } while (0) + +#define RTCX_FAIL(_reason, _exception_type) \ + do { \ + throw _exception_type{::std::format("RTC failure at: {}:{}: {}", \ + ::std::source_location::current().file_name(), \ + ::std::source_location::current().line(), \ + (_reason))}; \ + } while (0) + +extern "C" { +typedef struct evp_md_ctx_st EVP_MD_CTX; + +typedef struct CUlib_st* CUlibrary; +typedef struct CUkern_st* CUkernel; +typedef struct CUstream_st* CUstream; +} + +namespace RTCX_EXPORT rtcx { +using u8 = std::uint8_t; +using u16 = std::uint16_t; +using u32 = std::uint32_t; +using u64 = std::uint64_t; +using usize = std::size_t; +using i8 = std::int8_t; +using i16 = std::int16_t; +using i32 = std::int32_t; +using i64 = std::int64_t; +using iszie = std::ptrdiff_t; + +inline constexpr usize CACHELINE_ALIGNMENT = + 64; // = std::hardware_destructive_interference_size */ + +/** + * @brief RAII utility to execute a callable at the end of a scope. + */ +template +struct defer { + private: + T func_; + + public: + template + defer(Args&&... args) : func_{static_cast(args)...} + { + } + defer(defer const&) = delete; + defer& operator=(defer const&) = delete; + defer(defer&&) = delete; + defer& operator=(defer&&) = delete; + ~defer() { func_(); } +}; + +template +defer(T) -> defer; + +template +struct func; + +/** + * @brief Zero-copy, type-erased reference to a callable entity (e.g. lambda, function pointer) that + * can be invoked with the given signature. + */ +template +struct [[nodiscard]] func { + private: + void* _user_data; + R (*_thunk)(void*, Args...); + + public: + func(void* user_data, R (*thunk)(void*, Args...)) : _user_data{user_data}, _thunk{thunk} {} + + func(R (*func_ptr)(Args...)) + : _user_data{reinterpret_cast(func_ptr)}, + _thunk{+[](void* user_data, Args... args) -> R { + auto func = reinterpret_cast(user_data); + return func(std::forward(args)...); + }} + { + } + + R operator()(Args... args) const { return _thunk(_user_data, std::forward(args)...); } + + template + static func from_functor(Lambda& lambda) + { + return func{static_cast(std::addressof(lambda)), + +[](void* user_data, Args... args) -> R { + auto& lambda = *static_cast*>(user_data); + return lambda(std::forward(args)...); + }}; + } +}; + +template +func(void*, R (*)(void*, Args...)) -> func; + +template +func(R (*)(Args...)) -> func; + +struct [[nodiscard]] sha256_hex_string { + char data_[65]; + + constexpr std::string_view view() const { return std::string_view{data_, 64}; } + + constexpr operator std::string_view() const { return view(); } + + [[nodiscard]] char const* data() const { return data_; } + + [[nodiscard]] char const* c_str() const { return data_; } + + static constexpr usize size() { return 64; } + + static sha256_hex_string make(std::span input) + { + constexpr char const HEX_CHARS[] = "0123456789abcdef"; + sha256_hex_string hex; + for (usize i = 0; i < 32; ++i) { + hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; + hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; + } + hex.data_[64] = '\0'; + return hex; + } +}; + +struct [[nodiscard]] sha256 { + alignas(16) u8 data_[32]; + + constexpr bool operator==(sha256 const& hash) const + { + return std::equal(std::begin(data_), std::end(data_), std::begin(hash.data_)); + } + + constexpr bool operator!=(sha256 const& hash) const { return !(*this == hash); } + + sha256_hex_string to_hex_string() const { return sha256_hex_string::make(data_); } +}; + +struct [[nodiscard]] sha256_hasher { + constexpr u64 operator()(sha256 const& obj) const + { + struct u64x4 { + alignas(16) u64 v[4]; + }; + + auto value = std::bit_cast(obj); + auto const h0 = value.v[0]; + auto const h1 = value.v[1]; + auto const h2 = value.v[2]; + auto const h3 = value.v[3]; + + auto mix = [](u64 seed, u64 v) { + seed ^= v + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); + return seed; + }; + + return mix(mix(mix(h0, h1), h2), h3); + } +}; + +struct [[nodiscard]] sha256_context { + private: + EVP_MD_CTX* ectx_; + + public: + sha256_context(); + sha256_context(sha256_context const& other) = delete; + sha256_context& operator=(sha256_context const& other) = delete; + sha256_context(sha256_context&& other) : ectx_(other.ectx_) { other.ectx_ = nullptr; } + + sha256_context& operator=(sha256_context&& other) + { + if (this == &other) [[unlikely]] { return *this; } + this->~sha256_context(); + new (this) sha256_context(std::move(other)); + return *this; + } + + ~sha256_context(); + + void update(std::span data); + + sha256 finalize(); +}; + +enum class binary_type : i8 { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; + +/** + * @brief Represents a binary blob + * @details Manages the lifetime of the binary data via a user-provided deallocator function. This + * enables zero-copy usage of binary data stored in various forms (e.g., std::vector, mmap'd file, + * etc.). + */ +struct [[nodiscard]] blob_t { + private: + using deallocator = func; + + static void noop_deallocator(u8 const*, usize) {} + + u8 const* data_; + usize size_; + deallocator deallocator_; + + blob_t(u8 const* data, usize size, deallocator deallocator) + : data_(data), size_(size), deallocator_(deallocator) + { + } + + public: + blob_t() : data_(nullptr), size_(0), deallocator_(noop_deallocator) {} + + blob_t(blob_t const&) = delete; + blob_t& operator=(blob_t const&) = delete; + + blob_t(blob_t&& other) noexcept + : data_(other.data_), size_(other.size_), deallocator_(other.deallocator_) + { + other.data_ = nullptr; + other.size_ = 0; + other.deallocator_ = noop_deallocator; + } + + blob_t& operator=(blob_t&& other) noexcept + { + if (this == &other) [[unlikely]] { return *this; } + this->~blob_t(); + new (this) blob_t(std::move(other)); + return *this; + } + + ~blob_t() { deallocator_(data_, size_); } + + [[nodiscard]] std::span view() const { return {data_, size_}; } + + static blob_t from_parts(u8 const* data, usize size, deallocator deallocator) + { + return blob_t{data, size, deallocator}; + } + + static blob_t from_vector(std::vector&& data); + + static blob_t from_static_data(std::span data); + + static std::optional from_file(char const* path); +}; + +using blob = std::shared_ptr; + +struct [[nodiscard]] kernel_occupancy_config { + i32 min_grid_size = 0; + i32 block_size = 0; +}; + +/** + * @brief Represents a compiled kernel that can be launched on the GPU. + */ +struct [[nodiscard]] kernel_ref { + private: + CUkernel handle_; + + public: + explicit kernel_ref(CUkernel handle) : handle_(handle) {} + + /** + * @brief Computes the maximum occupancy configuration for the kernel, given the specified dynamic + * shared memory usage and block size limit. This function queries the CUDA driver for the optimal + * block size and minimum grid size to achieve maximum occupancy of the kernel on the GPU. + */ + kernel_occupancy_config max_occupancy_config(usize dynamic_shared_memory_bytes, + i32 block_size_limit) const; + + /** + * @brief Launches the kernel on the GPU with the specified grid and block dimensions, + * dynamic shared memory size, stream, and kernel parameters. This function wraps the CUDA driver + * kernel launch API, providing a convenient interface for executing the kernel with the desired + * configuration. + * @param grid_dim_x The number of blocks in the x-dimension of the grid + * @param grid_dim_y The number of blocks in the y-dimension of the grid + * @param grid_dim_z The number of blocks in the z-dimension of the grid + * @param block_dim_x The number of threads in the x-dimension of each block + * @param block_dim_y The number of threads in the y-dimension of each block + * @param block_dim_z The number of threads in the z-dimension of each block + * @param shared_mem_bytes The amount of dynamic shared memory (in bytes) to allocate for the + * kernel + * @param stream The CUDA stream on which to launch the kernel + * @param kernel_params A pointer to an array of pointers representing the kernel parameters to be + * passed to the kernel at launch time + */ + void launch(u32 grid_dim_x, + u32 grid_dim_y, + u32 grid_dim_z, + u32 block_dim_x, + u32 block_dim_y, + u32 block_dim_z, + u32 shared_mem_bytes, + CUstream stream, + void** kernel_params) const; + + /** + * @brief Retrieves the name of the kernel + */ + [[nodiscard]] std::string_view get_name() const; +}; + +/** + * @brief Represents a loaded RTC library containing compiled kernels + */ +struct [[nodiscard]] library_t { + private: + CUlibrary handle_; + + public: + explicit library_t(CUlibrary handle) : handle_(handle) {} + library_t(library_t const&) = delete; + library_t(library_t&&) = delete; + library_t& operator=(library_t const&) = delete; + library_t& operator=(library_t&&) = delete; + ~library_t(); + + /** + * @brief Retrieve a kernel from the library by name + */ + [[nodiscard]] kernel_ref get_kernel(char const* name) const; + + /** + * @brief Enumerate all kernels contained in the library, returning a vector of kernel references + * @return A vector of kernel_ref objects representing all kernels contained in the library + */ + [[nodiscard]] std::vector enumerate_kernels() const; +}; + +using library = std::shared_ptr; + +struct [[nodiscard]] compile_params { + /** + * @brief Name of the kernel or library being compiled + */ + char const* name = nullptr; + + /** + * @brief Source code to be compiled (e.g. PTX or LTO IR) + */ + char const* source = nullptr; + + /** + * @brief Include names of each header file provided in the `headers` field, used for resolving + * #include directives during compilation + */ + std::span header_include_names = {}; + + /** + * @brief Contents of header files required for compilation + */ + std::span headers = {}; + + /** + * @brief Sizes of each header file provided in the `headers` field + */ + std::span header_sizes = {}; + + /** + * @brief NVRTC compile options + */ + std::span options = {}; + + /** + * @brief Desired output binary type (e.g. PTX, CUBIN, etc.) + */ + binary_type target_type = binary_type::LTO_IR; +}; + +struct [[nodiscard]] link_params { + char const* name = nullptr; + binary_type output_type = binary_type::CUBIN; + std::span const> fragments = {}; + std::span fragment_binary_types = {}; + std::span fragment_names = {}; + std::span link_options = {}; +}; + +template +struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { + struct entry { + u64 last_touched_tick = 0; + T value; + + void hit(u64 tick) { last_touched_tick = tick; } + }; + + std::unordered_map entries_; + usize limit_; + + explicit lru_memory_cache(usize limit) : entries_{}, limit_{limit} + { + // reserve space to avoid rehashing + RTCX_EXPECTS(limit > 0, "Cache limit must be greater than 0", std::logic_error); + entries_.reserve(limit * 2); + } + + void purge() + { + if (entries_.empty()) { return; } + + auto num_to_purge = std::max(entries_.size() / 2, static_cast(1)); + + std::vector> rankings; + rankings.reserve(entries_.size()); + + for (auto const& [key, entry] : entries_) { + rankings.emplace_back(key, entry.last_touched_tick); + } + + std::sort(rankings.begin(), rankings.end(), [](auto const& a, auto const& b) { + return a.second < b.second; + }); + + // purge least recently used half + rankings.resize(num_to_purge); + + for (auto [key, _] : rankings) { + entries_.erase(key); + } + } + + void insert(sha256 const& sha, T&& value, u64 tick) + { + if ((entries_.size() + 1) > limit_) { purge(); } + + entries_.emplace(sha, entry{tick, std::move(value)}); + } +}; + +struct [[nodiscard]] cache_stats { + u64 blob_mem_hits = 0; + u64 blob_mem_misses = 0; + u64 blob_disk_hits = 0; + u64 blob_disk_misses = 0; + u64 library_mem_hits = 0; + u64 library_mem_misses = 0; + u64 library_disk_hits = 0; + u64 library_disk_misses = 0; +}; + +struct [[nodiscard]] cache_limits { + u32 num_blobs = 1024; + u32 num_libraries = 1024; +}; + +struct cache_stats_counter { + struct alignas(CACHELINE_ALIGNMENT) counter { + u64 value_ = 0; + + void incr() + { + std::atomic_ref c{value_}; + c.fetch_add(1, std::memory_order_relaxed); + } + + [[nodiscard]] u64 get() const + { + std::atomic_ref c{value_}; + return c.load(std::memory_order_relaxed); + } + + void reset() + { + std::atomic_ref c{value_}; + c.store(0, std::memory_order_relaxed); + } + }; + + counter blob_mem_hits; + counter blob_mem_misses; + counter blob_disk_hits; + counter blob_disk_misses; + counter library_mem_hits; + counter library_mem_misses; + counter library_disk_hits; + counter library_disk_misses; +}; + +using blob_compile_func = func; +using library_compile_func = func()>; + +/** + * @brief Thread-safe user-managed compile cache for compiled blobs and libraries + * + * @details Provides in-memory and on-disk caching of compiled RTC artifacts. + * The cache uses an LRU eviction policy when the number of cached items exceeds user-defined + * limits. In-memory cache is implemented using a thread-safe LRU cache that supports concurrent + * reads. The on-disk cache also allows concurrent access and stores cached items in files within a + * specified directory. Writing to disk is atomic to prevent corruption from concurrent writes or + * process interruptions. In addition, the cache maintains statistics on cache hits and misses for + * both in-memory and on-disk caches to help monitor cache performance in benchmarking and + * debugging. The interface is zero-copy, using shared pointers, mmap, and spans to avoid + * unnecessary data copying across threads and disk. + */ +struct cache_t { + private: + std::string cache_dir_; + + cache_limits limits_; + + std::mutex lock_; + + lru_memory_cache> blobs_cache_; + + lru_memory_cache> libraries_cache_; + + cache_stats_counter counter_; + + alignas(CACHELINE_ALIGNMENT) u64 tick_; + + public: + cache_t(std::string cache_dir, cache_limits const& limits); + cache_t(cache_t const&) = delete; + cache_t& operator=(cache_t const&) = delete; + cache_t(cache_t&&) = delete; + cache_t& operator=(cache_t&&) = delete; + ~cache_t() = default; + + /** + * @brief Get the directory path used for on-disk caching + * @return String reference to the cache directory path + */ + [[nodiscard]] std::string const& get_cache_dir(); + + /** + * @brief Query the cache for a compiled blob by its SHA-256 hash, or insert it if not present + * @param sha SHA-256 hash of the blob to query or insert + * @param compile Function to compile the blob if it's not found in the cache + * @return A shared future that will hold the compiled blob once it's available + */ + [[nodiscard]] std::shared_future get_or_add_blob(sha256 const& sha, + blob_compile_func compile); + + /** + * @brief Query the cache for a compiled library by its SHA-256 hash and binary type, or insert + * it if not present + * @param sha SHA-256 hash of the library to query or insert + * @param type Binary type of the library (e.g., CUBIN, PTX) + * @param compile Function to compile the library if it's not found in the cache + * @return A shared future that will hold the compiled library once it's available + */ + [[nodiscard]] std::shared_future get_or_add_library(sha256 const& sha, + binary_type type, + library_compile_func compile); + + /** + * @brief Retrieve current cache performance statistics, including hits and misses for both + * in-memory and on-disk caches + * + * @return A cache_statistics struct containing the current cache performance metrics + */ + cache_stats get_stats(); + + /** + * @brief Clear the current cache performance statistics, resetting all hit and miss counters to + * zero + */ + void clear_stats(); + + /** + * @brief Retrieve the current cache limits for blobs and libraries + * + * @return A cache_limits struct containing the maximum number of blobs and libraries that can be + * stored in the cache before eviction occurs + */ + cache_limits get_limits(); + + /** + * @brief Get the current number of blobs stored in the in-memory cache + * + * @return The number of blobs currently stored in the in-memory cache + */ + [[nodiscard]] usize get_blob_count(); + + /** + * @brief Get the current number of libraries stored in the in-memory cache + * + * @return The number of libraries currently stored in the in-memory cache + */ + [[nodiscard]] usize get_library_count(); + + /** + * @brief Clear all entries from the in-memory cache, removing all cached blobs and libraries + * without affecting the on-disk cache + * + * @details This function is useful for freeing up memory without losing the benefits of the + * on-disk cache, which can still be used to retrieve cached items in the future. + */ + void clear_memory_store(); + + /** + * @brief Clear all entries from the on-disk cache, removing all cached blobs and libraries + * stored on disk without affecting the in-memory cache + * @details This function is useful for freeing up disk space or resetting the on-disk cache + * without losing the benefits of the in-memory cache, which can still be used to retrieve cached + * items in the future. + */ + void clear_disk_store(); +}; + +/** + * @brief Compile source code into a binary blob + * + * @param params Compilation parameters including source code, headers, options, and target binary + * type + * @return A vector of bytes containing the compiled binary blob + */ +[[nodiscard]] std::vector compile(compile_params const& params); + +/** + * @brief Load a compiled library from binary data + * + * @param binary Span of bytes containing the compiled library binary data + * @param type Binary type of the library (e.g., CUBIN, PTX) + * @return A library object representing the loaded library with launchable kernels + */ +[[nodiscard]] library load_library(std::span binary, binary_type type); + +/** + * @brief Link multiple compiled binary fragments into a single binary blob containing the linked + * library + * + * @param params Linking parameters including the binary fragments to be linked and the target + * binary type + * @return A vector of bytes containing the linked library binary + */ +[[nodiscard]] std::vector link_library(link_params const& params); + +/** + * @brief Demangle a CUDA symbol name into a human-readable form + * @param mangled_name The mangled CUDA symbol name to be demangled + * @return A string containing the demangled, human-readable symbol name corresponding to the input + * mangled name + */ +[[nodiscard]] std::string demangle_cuda_symbol(char const* mangled_name); + +/** + * @brief Initialize the RTCX library, setting up necessary resources and state for subsequent + * operations + * @details This function must be called before using any other functions in the RTCX library. It + * performs necessary initialization tasks such as setting up CUDA contexts, initializing caches, + * and preparing any global state required for compilation, linking, and kernel management + * operations. Failure to call this function before using other RTCX functions may result in + * undefined behavior or runtime errors. + * This function is thread-safe. + * + */ +void initialize(); + +/** + * @brief Teardown the RTCX library, releasing any resources and cleaning up state used by the + * library + * @details This function should be called when RTCX functionality is no longer needed, such as at + * the end of the program or when cleaning up resources. It performs necessary cleanup tasks such as + * releasing CUDA contexts, clearing caches, and resetting any global state used by the library. + * After calling this function, other RTCX functions should not be used unless initialize() is + * called again to reinitialize the library. + * This function is not thread-safe. + */ +void teardown(); + +} // namespace RTCX_EXPORT rtcx diff --git a/cpp/src/jit/rtc/includes/system/README.md b/cpp/src/librtcx/system_includes/README.md similarity index 100% rename from cpp/src/jit/rtc/includes/system/README.md rename to cpp/src/librtcx/system_includes/README.md diff --git a/cpp/src/jit/rtc/includes/system/algorithm b/cpp/src/librtcx/system_includes/algorithm similarity index 100% rename from cpp/src/jit/rtc/includes/system/algorithm rename to cpp/src/librtcx/system_includes/algorithm diff --git a/cpp/src/jit/rtc/includes/system/array b/cpp/src/librtcx/system_includes/array similarity index 100% rename from cpp/src/jit/rtc/includes/system/array rename to cpp/src/librtcx/system_includes/array diff --git a/cpp/src/jit/rtc/includes/system/atomic b/cpp/src/librtcx/system_includes/atomic similarity index 100% rename from cpp/src/jit/rtc/includes/system/atomic rename to cpp/src/librtcx/system_includes/atomic diff --git a/cpp/src/jit/rtc/includes/system/barrier b/cpp/src/librtcx/system_includes/barrier similarity index 100% rename from cpp/src/jit/rtc/includes/system/barrier rename to cpp/src/librtcx/system_includes/barrier diff --git a/cpp/src/jit/rtc/includes/system/bit b/cpp/src/librtcx/system_includes/bit similarity index 100% rename from cpp/src/jit/rtc/includes/system/bit rename to cpp/src/librtcx/system_includes/bit diff --git a/cpp/src/jit/rtc/includes/system/cassert b/cpp/src/librtcx/system_includes/cassert similarity index 100% rename from cpp/src/jit/rtc/includes/system/cassert rename to cpp/src/librtcx/system_includes/cassert diff --git a/cpp/src/jit/rtc/includes/system/cassert.h b/cpp/src/librtcx/system_includes/cassert.h similarity index 100% rename from cpp/src/jit/rtc/includes/system/cassert.h rename to cpp/src/librtcx/system_includes/cassert.h diff --git a/cpp/src/jit/rtc/includes/system/ccomplex b/cpp/src/librtcx/system_includes/ccomplex similarity index 100% rename from cpp/src/jit/rtc/includes/system/ccomplex rename to cpp/src/librtcx/system_includes/ccomplex diff --git a/cpp/src/jit/rtc/includes/system/ccomplex.h b/cpp/src/librtcx/system_includes/ccomplex.h similarity index 100% rename from cpp/src/jit/rtc/includes/system/ccomplex.h rename to cpp/src/librtcx/system_includes/ccomplex.h diff --git a/cpp/src/jit/rtc/includes/system/cfloat b/cpp/src/librtcx/system_includes/cfloat similarity index 100% rename from cpp/src/jit/rtc/includes/system/cfloat rename to cpp/src/librtcx/system_includes/cfloat diff --git a/cpp/src/jit/rtc/includes/system/cfloat.h b/cpp/src/librtcx/system_includes/cfloat.h similarity index 100% rename from cpp/src/jit/rtc/includes/system/cfloat.h rename to cpp/src/librtcx/system_includes/cfloat.h diff --git a/cpp/src/jit/rtc/includes/system/chrono b/cpp/src/librtcx/system_includes/chrono similarity index 100% rename from cpp/src/jit/rtc/includes/system/chrono rename to cpp/src/librtcx/system_includes/chrono diff --git a/cpp/src/jit/rtc/includes/system/climits b/cpp/src/librtcx/system_includes/climits similarity index 100% rename from cpp/src/jit/rtc/includes/system/climits rename to cpp/src/librtcx/system_includes/climits diff --git a/cpp/src/jit/rtc/includes/system/cmath b/cpp/src/librtcx/system_includes/cmath similarity index 100% rename from cpp/src/jit/rtc/includes/system/cmath rename to cpp/src/librtcx/system_includes/cmath diff --git a/cpp/src/jit/rtc/includes/system/cmath.h b/cpp/src/librtcx/system_includes/cmath.h similarity index 100% rename from cpp/src/jit/rtc/includes/system/cmath.h rename to cpp/src/librtcx/system_includes/cmath.h diff --git a/cpp/src/jit/rtc/includes/system/complex b/cpp/src/librtcx/system_includes/complex similarity index 100% rename from cpp/src/jit/rtc/includes/system/complex rename to cpp/src/librtcx/system_includes/complex diff --git a/cpp/src/jit/rtc/includes/system/concepts b/cpp/src/librtcx/system_includes/concepts similarity index 100% rename from cpp/src/jit/rtc/includes/system/concepts rename to cpp/src/librtcx/system_includes/concepts diff --git a/cpp/src/jit/rtc/includes/system/cstddef b/cpp/src/librtcx/system_includes/cstddef similarity index 100% rename from cpp/src/jit/rtc/includes/system/cstddef rename to cpp/src/librtcx/system_includes/cstddef diff --git a/cpp/src/jit/rtc/includes/system/cstdint b/cpp/src/librtcx/system_includes/cstdint similarity index 100% rename from cpp/src/jit/rtc/includes/system/cstdint rename to cpp/src/librtcx/system_includes/cstdint diff --git a/cpp/src/jit/rtc/includes/system/cstdlib b/cpp/src/librtcx/system_includes/cstdlib similarity index 100% rename from cpp/src/jit/rtc/includes/system/cstdlib rename to cpp/src/librtcx/system_includes/cstdlib diff --git a/cpp/src/jit/rtc/includes/system/cstring b/cpp/src/librtcx/system_includes/cstring similarity index 100% rename from cpp/src/jit/rtc/includes/system/cstring rename to cpp/src/librtcx/system_includes/cstring diff --git a/cpp/src/jit/rtc/includes/system/ctime b/cpp/src/librtcx/system_includes/ctime similarity index 100% rename from cpp/src/jit/rtc/includes/system/ctime rename to cpp/src/librtcx/system_includes/ctime diff --git a/cpp/src/jit/rtc/includes/system/expected b/cpp/src/librtcx/system_includes/expected similarity index 100% rename from cpp/src/jit/rtc/includes/system/expected rename to cpp/src/librtcx/system_includes/expected diff --git a/cpp/src/jit/rtc/includes/system/functional b/cpp/src/librtcx/system_includes/functional similarity index 100% rename from cpp/src/jit/rtc/includes/system/functional rename to cpp/src/librtcx/system_includes/functional diff --git a/cpp/src/jit/rtc/includes/system/initializer_list b/cpp/src/librtcx/system_includes/initializer_list similarity index 100% rename from cpp/src/jit/rtc/includes/system/initializer_list rename to cpp/src/librtcx/system_includes/initializer_list diff --git a/cpp/src/jit/rtc/includes/system/inplace_vector b/cpp/src/librtcx/system_includes/inplace_vector similarity index 100% rename from cpp/src/jit/rtc/includes/system/inplace_vector rename to cpp/src/librtcx/system_includes/inplace_vector diff --git a/cpp/src/jit/rtc/includes/system/iterator b/cpp/src/librtcx/system_includes/iterator similarity index 100% rename from cpp/src/jit/rtc/includes/system/iterator rename to cpp/src/librtcx/system_includes/iterator diff --git a/cpp/src/jit/rtc/includes/system/latch b/cpp/src/librtcx/system_includes/latch similarity index 100% rename from cpp/src/jit/rtc/includes/system/latch rename to cpp/src/librtcx/system_includes/latch diff --git a/cpp/src/jit/rtc/includes/system/limits b/cpp/src/librtcx/system_includes/limits similarity index 100% rename from cpp/src/jit/rtc/includes/system/limits rename to cpp/src/librtcx/system_includes/limits diff --git a/cpp/src/jit/rtc/includes/system/linalg b/cpp/src/librtcx/system_includes/linalg similarity index 100% rename from cpp/src/jit/rtc/includes/system/linalg rename to cpp/src/librtcx/system_includes/linalg diff --git a/cpp/src/jit/rtc/includes/system/mdspan b/cpp/src/librtcx/system_includes/mdspan similarity index 100% rename from cpp/src/jit/rtc/includes/system/mdspan rename to cpp/src/librtcx/system_includes/mdspan diff --git a/cpp/src/jit/rtc/includes/system/memory b/cpp/src/librtcx/system_includes/memory similarity index 100% rename from cpp/src/jit/rtc/includes/system/memory rename to cpp/src/librtcx/system_includes/memory diff --git a/cpp/src/jit/rtc/includes/system/numbers b/cpp/src/librtcx/system_includes/numbers similarity index 100% rename from cpp/src/jit/rtc/includes/system/numbers rename to cpp/src/librtcx/system_includes/numbers diff --git a/cpp/src/jit/rtc/includes/system/numeric b/cpp/src/librtcx/system_includes/numeric similarity index 100% rename from cpp/src/jit/rtc/includes/system/numeric rename to cpp/src/librtcx/system_includes/numeric diff --git a/cpp/src/jit/rtc/includes/system/optional b/cpp/src/librtcx/system_includes/optional similarity index 100% rename from cpp/src/jit/rtc/includes/system/optional rename to cpp/src/librtcx/system_includes/optional diff --git a/cpp/src/jit/rtc/includes/system/ranges b/cpp/src/librtcx/system_includes/ranges similarity index 100% rename from cpp/src/jit/rtc/includes/system/ranges rename to cpp/src/librtcx/system_includes/ranges diff --git a/cpp/src/jit/rtc/includes/system/ratio b/cpp/src/librtcx/system_includes/ratio similarity index 100% rename from cpp/src/jit/rtc/includes/system/ratio rename to cpp/src/librtcx/system_includes/ratio diff --git a/cpp/src/jit/rtc/includes/system/semaphore b/cpp/src/librtcx/system_includes/semaphore similarity index 100% rename from cpp/src/jit/rtc/includes/system/semaphore rename to cpp/src/librtcx/system_includes/semaphore diff --git a/cpp/src/jit/rtc/includes/system/source_location b/cpp/src/librtcx/system_includes/source_location similarity index 100% rename from cpp/src/jit/rtc/includes/system/source_location rename to cpp/src/librtcx/system_includes/source_location diff --git a/cpp/src/jit/rtc/includes/system/span b/cpp/src/librtcx/system_includes/span similarity index 100% rename from cpp/src/jit/rtc/includes/system/span rename to cpp/src/librtcx/system_includes/span diff --git a/cpp/src/jit/rtc/includes/system/stddef.h b/cpp/src/librtcx/system_includes/stddef.h similarity index 100% rename from cpp/src/jit/rtc/includes/system/stddef.h rename to cpp/src/librtcx/system_includes/stddef.h diff --git a/cpp/src/jit/rtc/includes/system/stdint.h b/cpp/src/librtcx/system_includes/stdint.h similarity index 100% rename from cpp/src/jit/rtc/includes/system/stdint.h rename to cpp/src/librtcx/system_includes/stdint.h diff --git a/cpp/src/jit/rtc/includes/system/stdlib.h b/cpp/src/librtcx/system_includes/stdlib.h similarity index 100% rename from cpp/src/jit/rtc/includes/system/stdlib.h rename to cpp/src/librtcx/system_includes/stdlib.h diff --git a/cpp/src/jit/rtc/includes/system/string.h b/cpp/src/librtcx/system_includes/string.h similarity index 100% rename from cpp/src/jit/rtc/includes/system/string.h rename to cpp/src/librtcx/system_includes/string.h diff --git a/cpp/src/jit/rtc/includes/system/string_view b/cpp/src/librtcx/system_includes/string_view similarity index 100% rename from cpp/src/jit/rtc/includes/system/string_view rename to cpp/src/librtcx/system_includes/string_view diff --git a/cpp/src/jit/rtc/includes/system/time.h b/cpp/src/librtcx/system_includes/time.h similarity index 100% rename from cpp/src/jit/rtc/includes/system/time.h rename to cpp/src/librtcx/system_includes/time.h diff --git a/cpp/src/jit/rtc/includes/system/tuple b/cpp/src/librtcx/system_includes/tuple similarity index 100% rename from cpp/src/jit/rtc/includes/system/tuple rename to cpp/src/librtcx/system_includes/tuple diff --git a/cpp/src/jit/rtc/includes/system/type_traits b/cpp/src/librtcx/system_includes/type_traits similarity index 100% rename from cpp/src/jit/rtc/includes/system/type_traits rename to cpp/src/librtcx/system_includes/type_traits diff --git a/cpp/src/jit/rtc/includes/system/utility b/cpp/src/librtcx/system_includes/utility similarity index 100% rename from cpp/src/jit/rtc/includes/system/utility rename to cpp/src/librtcx/system_includes/utility diff --git a/cpp/src/jit/rtc/includes/system/variant b/cpp/src/librtcx/system_includes/variant similarity index 100% rename from cpp/src/jit/rtc/includes/system/variant rename to cpp/src/librtcx/system_includes/variant diff --git a/cpp/src/jit/rtc/includes/system/version b/cpp/src/librtcx/system_includes/version similarity index 100% rename from cpp/src/jit/rtc/includes/system/version rename to cpp/src/librtcx/system_includes/version diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 7ba5c582b3ef..3d13db1febad 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -9,7 +9,7 @@ #include "io/utilities/getenv_or.hpp" #include "jit/cache.hpp" #include "jit/jit.hpp" -#include "jit/rtc/cache.hpp" +#include "librtcx/rtcx.hpp" #include #include @@ -37,8 +37,7 @@ void context::ensure_rtc_cache_initialized() std::call_once(_rtc_cache_init_flag, [&]() { // make sure the rtc cache directory exists std::filesystem::create_directories(_config.rtc_cache_dir); - _rtc_cache = - std::make_unique(_config.rtc_cache_dir, rtc::cache_limits::default_limits()); + _rtc_cache = std::make_unique(_config.rtc_cache_dir, rtcx::cache_limits{}); }); } @@ -60,7 +59,7 @@ jit::program_cache& context::program_cache() return *_program_cache; } -rtc::cache_t& context::rtc_cache() +rtcx::cache_t& context::rtc_cache() { ensure_rtc_cache_initialized(); return *_rtc_cache; diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index f26fb8c08633..1bd754ebc9aa 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -11,16 +11,16 @@ #include #include +namespace rtcx { +class cache_t; +} // namespace rtcx + namespace cudf { namespace jit { class program_cache; } -namespace rtc { -class cache_t; -} // namespace rtc - class jit_bundle_t; struct [[nodiscard]] context_config { @@ -40,7 +40,7 @@ class context { std::once_flag _program_cache_init_flag; std::unique_ptr _program_cache; std::once_flag _rtc_cache_init_flag; - std::unique_ptr _rtc_cache; + std::unique_ptr _rtc_cache; std::once_flag _jit_bundle_init_flag; std::unique_ptr _jit_bundle; @@ -63,7 +63,7 @@ class context { jit::program_cache& program_cache(); - rtc::cache_t& rtc_cache(); + rtcx::cache_t& rtc_cache(); jit_bundle_t& jit_bundle(); From ca50cc9bca302527a3a0e76b6183da5ed6067b5a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 2 Mar 2026 20:45:12 +0000 Subject: [PATCH 060/254] remove faked headers --- cpp/include/jit_cudf/README.md | 0 cpp/include/jit_cudf/functions/abs.cuh | 29 - cpp/include/jit_cudf/functions/add.cuh | 29 - cpp/include/jit_cudf/functions/all.cuh | 51 -- cpp/include/jit_cudf/functions/arccos.cuh | 27 - cpp/include/jit_cudf/functions/arccosh.cuh | 27 - cpp/include/jit_cudf/functions/arcsin.cuh | 27 - cpp/include/jit_cudf/functions/arcsinh.cuh | 27 - cpp/include/jit_cudf/functions/arctan.cuh | 27 - cpp/include/jit_cudf/functions/arctanh.cuh | 27 - cpp/include/jit_cudf/functions/bit_and.cuh | 29 - cpp/include/jit_cudf/functions/bit_invert.cuh | 29 - cpp/include/jit_cudf/functions/bit_or.cuh | 29 - cpp/include/jit_cudf/functions/bit_xor.cuh | 29 - .../jit_cudf/functions/cast_to_f64.cuh | 29 - .../jit_cudf/functions/cast_to_i64.cuh | 29 - .../jit_cudf/functions/cast_to_u64.cuh | 29 - cpp/include/jit_cudf/functions/cbrt.cuh | 27 - cpp/include/jit_cudf/functions/ceil.cuh | 27 - cpp/include/jit_cudf/functions/cos.cuh | 27 - cpp/include/jit_cudf/functions/cosh.cuh | 27 - cpp/include/jit_cudf/functions/div.cuh | 29 - cpp/include/jit_cudf/functions/equal.cuh | 29 - cpp/include/jit_cudf/functions/exp.cuh | 27 - cpp/include/jit_cudf/functions/floor.cuh | 27 - cpp/include/jit_cudf/functions/greater.cuh | 27 - .../jit_cudf/functions/greater_equal.cuh | 29 - cpp/include/jit_cudf/functions/identity.cuh | 23 - cpp/include/jit_cudf/functions/is_null.cuh | 23 - cpp/include/jit_cudf/functions/less.cuh | 27 - cpp/include/jit_cudf/functions/less_equal.cuh | 27 - cpp/include/jit_cudf/functions/log.cuh | 27 - .../jit_cudf/functions/logical_and.cuh | 29 - cpp/include/jit_cudf/functions/logical_or.cuh | 29 - cpp/include/jit_cudf/functions/mod.cuh | 33 - cpp/include/jit_cudf/functions/mul.cuh | 29 - cpp/include/jit_cudf/functions/null_equal.cuh | 29 - .../jit_cudf/functions/null_logical_and.cuh | 31 - .../jit_cudf/functions/null_logical_or.cuh | 31 - cpp/include/jit_cudf/functions/pow.cuh | 27 - cpp/include/jit_cudf/functions/pymod.cuh | 39 - cpp/include/jit_cudf/functions/rint.cuh | 27 - cpp/include/jit_cudf/functions/sin.cuh | 27 - cpp/include/jit_cudf/functions/sinh.cuh | 27 - cpp/include/jit_cudf/functions/sub.cuh | 29 - cpp/include/jit_cudf/functions/tanh.cuh | 27 - cpp/include/jit_cudf/types.cuh | 672 ------------------ 47 files changed, 1963 deletions(-) delete mode 100644 cpp/include/jit_cudf/README.md delete mode 100644 cpp/include/jit_cudf/functions/abs.cuh delete mode 100644 cpp/include/jit_cudf/functions/add.cuh delete mode 100644 cpp/include/jit_cudf/functions/all.cuh delete mode 100644 cpp/include/jit_cudf/functions/arccos.cuh delete mode 100644 cpp/include/jit_cudf/functions/arccosh.cuh delete mode 100644 cpp/include/jit_cudf/functions/arcsin.cuh delete mode 100644 cpp/include/jit_cudf/functions/arcsinh.cuh delete mode 100644 cpp/include/jit_cudf/functions/arctan.cuh delete mode 100644 cpp/include/jit_cudf/functions/arctanh.cuh delete mode 100644 cpp/include/jit_cudf/functions/bit_and.cuh delete mode 100644 cpp/include/jit_cudf/functions/bit_invert.cuh delete mode 100644 cpp/include/jit_cudf/functions/bit_or.cuh delete mode 100644 cpp/include/jit_cudf/functions/bit_xor.cuh delete mode 100644 cpp/include/jit_cudf/functions/cast_to_f64.cuh delete mode 100644 cpp/include/jit_cudf/functions/cast_to_i64.cuh delete mode 100644 cpp/include/jit_cudf/functions/cast_to_u64.cuh delete mode 100644 cpp/include/jit_cudf/functions/cbrt.cuh delete mode 100644 cpp/include/jit_cudf/functions/ceil.cuh delete mode 100644 cpp/include/jit_cudf/functions/cos.cuh delete mode 100644 cpp/include/jit_cudf/functions/cosh.cuh delete mode 100644 cpp/include/jit_cudf/functions/div.cuh delete mode 100644 cpp/include/jit_cudf/functions/equal.cuh delete mode 100644 cpp/include/jit_cudf/functions/exp.cuh delete mode 100644 cpp/include/jit_cudf/functions/floor.cuh delete mode 100644 cpp/include/jit_cudf/functions/greater.cuh delete mode 100644 cpp/include/jit_cudf/functions/greater_equal.cuh delete mode 100644 cpp/include/jit_cudf/functions/identity.cuh delete mode 100644 cpp/include/jit_cudf/functions/is_null.cuh delete mode 100644 cpp/include/jit_cudf/functions/less.cuh delete mode 100644 cpp/include/jit_cudf/functions/less_equal.cuh delete mode 100644 cpp/include/jit_cudf/functions/log.cuh delete mode 100644 cpp/include/jit_cudf/functions/logical_and.cuh delete mode 100644 cpp/include/jit_cudf/functions/logical_or.cuh delete mode 100644 cpp/include/jit_cudf/functions/mod.cuh delete mode 100644 cpp/include/jit_cudf/functions/mul.cuh delete mode 100644 cpp/include/jit_cudf/functions/null_equal.cuh delete mode 100644 cpp/include/jit_cudf/functions/null_logical_and.cuh delete mode 100644 cpp/include/jit_cudf/functions/null_logical_or.cuh delete mode 100644 cpp/include/jit_cudf/functions/pow.cuh delete mode 100644 cpp/include/jit_cudf/functions/pymod.cuh delete mode 100644 cpp/include/jit_cudf/functions/rint.cuh delete mode 100644 cpp/include/jit_cudf/functions/sin.cuh delete mode 100644 cpp/include/jit_cudf/functions/sinh.cuh delete mode 100644 cpp/include/jit_cudf/functions/sub.cuh delete mode 100644 cpp/include/jit_cudf/functions/tanh.cuh delete mode 100644 cpp/include/jit_cudf/types.cuh diff --git a/cpp/include/jit_cudf/README.md b/cpp/include/jit_cudf/README.md deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/cpp/include/jit_cudf/functions/abs.cuh b/cpp/include/jit_cudf/functions/abs.cuh deleted file mode 100644 index e148451d10ad..000000000000 --- a/cpp/include/jit_cudf/functions/abs.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void abs(T* out, T const& a) -{ - *out = (a < 0) ? -a : a; -} - -template -__device__ inline void abs(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - abs(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/add.cuh b/cpp/include/jit_cudf/functions/add.cuh deleted file mode 100644 index b690cb362a6d..000000000000 --- a/cpp/include/jit_cudf/functions/add.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void add(T* out, T const& a, T const& b) -{ - *out = a + b; -} - -template -__device__ inline void add(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - add(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/all.cuh b/cpp/include/jit_cudf/functions/all.cuh deleted file mode 100644 index 90ce14cf5a16..000000000000 --- a/cpp/include/jit_cudf/functions/all.cuh +++ /dev/null @@ -1,51 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include diff --git a/cpp/include/jit_cudf/functions/arccos.cuh b/cpp/include/jit_cudf/functions/arccos.cuh deleted file mode 100644 index eb729edadad1..000000000000 --- a/cpp/include/jit_cudf/functions/arccos.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void arccos(f32* out, f32 const& a) { *out = __builtin_acosf(a); } - -__device__ inline void arccos(f64* out, f64 const& a) { *out = __builtin_acos(a); } - -template -__device__ inline void arccos(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - arccos(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/arccosh.cuh b/cpp/include/jit_cudf/functions/arccosh.cuh deleted file mode 100644 index c121cc0be1d4..000000000000 --- a/cpp/include/jit_cudf/functions/arccosh.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void arccosh(f32* out, f32 const& a) { *out = __builtin_acoshf(a); } - -__device__ inline void arccosh(f64* out, f64 const& a) { *out = __builtin_acosh(a); } - -template -__device__ inline void arccosh(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - arccosh(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/arcsin.cuh b/cpp/include/jit_cudf/functions/arcsin.cuh deleted file mode 100644 index 619951894068..000000000000 --- a/cpp/include/jit_cudf/functions/arcsin.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void arcsin(f32* out, f32 const& a) { *out = __builtin_asinf(a); } - -__device__ inline void arcsin(f64* out, f64 const& a) { *out = __builtin_asin(a); } - -template -__device__ inline void arcsin(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - arcsin(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/arcsinh.cuh b/cpp/include/jit_cudf/functions/arcsinh.cuh deleted file mode 100644 index a9e6f0ffd5be..000000000000 --- a/cpp/include/jit_cudf/functions/arcsinh.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void arcsinh(f32* out, f32 const& a) { *out = __builtin_asinhf(a); } - -__device__ inline void arcsinh(f64* out, f64 const& a) { *out = __builtin_asinh(a); } - -template -__device__ inline void arcsinh(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - arcsinh(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/arctan.cuh b/cpp/include/jit_cudf/functions/arctan.cuh deleted file mode 100644 index 5c8a23e40097..000000000000 --- a/cpp/include/jit_cudf/functions/arctan.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void arctan(f32* out, f32 const& a) { *out = __builtin_atanf(a); } - -__device__ inline void arctan(f64* out, f64 const& a) { *out = __builtin_atan(a); } - -template -__device__ inline void arctan(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - arctan(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/arctanh.cuh b/cpp/include/jit_cudf/functions/arctanh.cuh deleted file mode 100644 index dcc25aef6f78..000000000000 --- a/cpp/include/jit_cudf/functions/arctanh.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void arctanh(f32* out, f32 const& a) { *out = __builtin_atanhf(a); } - -__device__ inline void arctanh(f64* out, f64 const& a) { *out = __builtin_atanh(a); } - -template -__device__ inline void arctanh(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - arctanh(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/bit_and.cuh b/cpp/include/jit_cudf/functions/bit_and.cuh deleted file mode 100644 index 7d9aea7d5e84..000000000000 --- a/cpp/include/jit_cudf/functions/bit_and.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void bit_and(T* out, T const& a, T const& b) -{ - *out = a & b; -} - -template -__device__ inline void bit_and(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - bit_and(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/bit_invert.cuh b/cpp/include/jit_cudf/functions/bit_invert.cuh deleted file mode 100644 index 6251641f4f76..000000000000 --- a/cpp/include/jit_cudf/functions/bit_invert.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void bit_invert(T* out, T const& a) -{ - *out = ~a; -} - -template -__device__ inline void bit_invert(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - bit_invert(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/bit_or.cuh b/cpp/include/jit_cudf/functions/bit_or.cuh deleted file mode 100644 index 019d864ab9c8..000000000000 --- a/cpp/include/jit_cudf/functions/bit_or.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void bit_or(T* out, T const& a, T const& b) -{ - *out = a | b; -} - -template -__device__ inline void bit_or(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - bit_or(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/bit_xor.cuh b/cpp/include/jit_cudf/functions/bit_xor.cuh deleted file mode 100644 index 835e726c3d44..000000000000 --- a/cpp/include/jit_cudf/functions/bit_xor.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void bit_xor(T* out, T const& a, T const& b) -{ - *out = a ^ b; -} - -template -__device__ inline void bit_xor(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - bit_xor(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/cast_to_f64.cuh b/cpp/include/jit_cudf/functions/cast_to_f64.cuh deleted file mode 100644 index 9f9bbb01e55e..000000000000 --- a/cpp/include/jit_cudf/functions/cast_to_f64.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void cast_to_f64(f64* out, T const& a) -{ - *out = static_cast(a); -} - -template -__device__ inline void cast_to_f64(optional* out, optional const& a) -{ - if (a.has_value()) { - f64 r; - cast_to_f64(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/cast_to_i64.cuh b/cpp/include/jit_cudf/functions/cast_to_i64.cuh deleted file mode 100644 index d9d2da81471f..000000000000 --- a/cpp/include/jit_cudf/functions/cast_to_i64.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void cast_to_i64(i64* out, T const& a) -{ - *out = static_cast(a); -} - -template -__device__ inline void cast_to_i64(optional* out, optional const& a) -{ - if (a.has_value()) { - i64 r; - cast_to_i64(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/cast_to_u64.cuh b/cpp/include/jit_cudf/functions/cast_to_u64.cuh deleted file mode 100644 index bf6e23f7a55d..000000000000 --- a/cpp/include/jit_cudf/functions/cast_to_u64.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void cast_to_u64(u64* out, T const& a) -{ - *out = static_cast(a); -} - -template -__device__ inline void cast_to_u64(optional* out, optional const& a) -{ - if (a.has_value()) { - u64 r; - cast_to_u64(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/cbrt.cuh b/cpp/include/jit_cudf/functions/cbrt.cuh deleted file mode 100644 index 29f61f9fc9e9..000000000000 --- a/cpp/include/jit_cudf/functions/cbrt.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void cbrt(f32* out, f32 const& a) { *out = __builtin_cbrtf(a); } - -__device__ inline void cbrt(f64* out, f64 const& a) { *out = __builtin_cbrt(a); } - -template -__device__ inline void cbrt(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - cbrt(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/ceil.cuh b/cpp/include/jit_cudf/functions/ceil.cuh deleted file mode 100644 index 82ca69aca375..000000000000 --- a/cpp/include/jit_cudf/functions/ceil.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void ceil(f32* out, f32 const& a) { *out = __builtin_ceilf(a); } - -__device__ inline void ceil(f64* out, f64 const& a) { *out = __builtin_ceil(a); } - -template -__device__ inline void ceil(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - ceil(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/cos.cuh b/cpp/include/jit_cudf/functions/cos.cuh deleted file mode 100644 index d5596b78cc2c..000000000000 --- a/cpp/include/jit_cudf/functions/cos.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void cos(f32* out, f32 const& a) { *out = __builtin_cosf(a); } - -__device__ inline void cos(f64* out, f64 const& a) { *out = __builtin_cos(a); } - -template -__device__ inline void cos(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - cos(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/cosh.cuh b/cpp/include/jit_cudf/functions/cosh.cuh deleted file mode 100644 index 633c7f801cd7..000000000000 --- a/cpp/include/jit_cudf/functions/cosh.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void cosh(f32* out, f32 const& a) { *out = __builtin_coshf(a); } - -__device__ inline void cosh(f64* out, f64 const& a) { *out = __builtin_cosh(a); } - -template -__device__ inline void cosh(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - cosh(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/div.cuh b/cpp/include/jit_cudf/functions/div.cuh deleted file mode 100644 index 5a1883fcb1c9..000000000000 --- a/cpp/include/jit_cudf/functions/div.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void div(T* out, T const& a, T const& b) -{ - *out = a / b; -} - -template -__device__ inline void div(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - div(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/equal.cuh b/cpp/include/jit_cudf/functions/equal.cuh deleted file mode 100644 index 4e0c4a1a6477..000000000000 --- a/cpp/include/jit_cudf/functions/equal.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void equal(bool* out, T const& a, T const& b) -{ - *out = a == b; -} - -template -__device__ inline void equal(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - *out = (*a == *b); - } else if (a.has_null() && b.has_null()) { - *out = true; - } else { - *out = false; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/exp.cuh b/cpp/include/jit_cudf/functions/exp.cuh deleted file mode 100644 index e8e1e44f3674..000000000000 --- a/cpp/include/jit_cudf/functions/exp.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void exp(f32* out, f32 const& a) { *out = __builtin_expf(a); } - -__device__ inline void exp(f64* out, f64 const& a) { *out = __builtin_exp(a); } - -template -__device__ inline void exp(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - exp(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/floor.cuh b/cpp/include/jit_cudf/functions/floor.cuh deleted file mode 100644 index 5065c2b90c6f..000000000000 --- a/cpp/include/jit_cudf/functions/floor.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void floor(f32* out, f32 const& a) { *out = __builtin_floorf(a); } - -__device__ inline void floor(f64* out, f64 const& a) { *out = __builtin_floor(a); } - -template -__device__ inline void floor(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - floor(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/greater.cuh b/cpp/include/jit_cudf/functions/greater.cuh deleted file mode 100644 index 0e76a98a321c..000000000000 --- a/cpp/include/jit_cudf/functions/greater.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void greater(bool* out, T const& a, T const& b) -{ - *out = a > b; -} - -template -__device__ inline void greater(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - *out = (*a > *b); - } else { - *out = false; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/greater_equal.cuh b/cpp/include/jit_cudf/functions/greater_equal.cuh deleted file mode 100644 index bb38cb9a70ae..000000000000 --- a/cpp/include/jit_cudf/functions/greater_equal.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void greater_equal(bool* out, T const& a, T const& b) -{ - *out = a >= b; -} - -template -__device__ inline void greater_equal(optional* out, - optional const& a, - optional const& b) -{ - if (a.has_value() && b.has_value()) { - *out = (*a >= *b); - } else { - *out = false; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/identity.cuh b/cpp/include/jit_cudf/functions/identity.cuh deleted file mode 100644 index 4ee467f6230d..000000000000 --- a/cpp/include/jit_cudf/functions/identity.cuh +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void identity(T* out, T const& a) -{ - *out = a; -} - -template -__device__ inline void identity(optional* out, optional const& a) -{ - *out = a; -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/is_null.cuh b/cpp/include/jit_cudf/functions/is_null.cuh deleted file mode 100644 index 82db3952ab18..000000000000 --- a/cpp/include/jit_cudf/functions/is_null.cuh +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void is_null(bool* out, T const& a) -{ - *out = false; -} - -template -__device__ inline void is_null(optional* out, optional const& a) -{ - *out = a.has_null(); -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/less.cuh b/cpp/include/jit_cudf/functions/less.cuh deleted file mode 100644 index 25c4aa32ffe1..000000000000 --- a/cpp/include/jit_cudf/functions/less.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void less(bool* out, T const& a, T const& b) -{ - *out = a < b; -} - -template -__device__ inline void less(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - *out = (*a < *b); - } else { - *out = false; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/less_equal.cuh b/cpp/include/jit_cudf/functions/less_equal.cuh deleted file mode 100644 index 2d881cd97b2f..000000000000 --- a/cpp/include/jit_cudf/functions/less_equal.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void less_equal(bool* out, T const& a, T const& b) -{ - *out = a <= b; -} - -template -__device__ inline void less_equal(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - *out = (*a <= *b); - } else { - *out = false; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/log.cuh b/cpp/include/jit_cudf/functions/log.cuh deleted file mode 100644 index 63b382cf82d9..000000000000 --- a/cpp/include/jit_cudf/functions/log.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void log(f32* out, f32 const& a) { *out = __builtin_logf(a); } - -__device__ inline void log(f64* out, f64 const& a) { *out = __builtin_log(a); } - -template -__device__ inline void log(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - log(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/logical_and.cuh b/cpp/include/jit_cudf/functions/logical_and.cuh deleted file mode 100644 index 52d838d932de..000000000000 --- a/cpp/include/jit_cudf/functions/logical_and.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void logical_and(T* out, T const& a, T const& b) -{ - *out = a && b; -} - -template -__device__ inline void logical_and(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - logical_and(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/logical_or.cuh b/cpp/include/jit_cudf/functions/logical_or.cuh deleted file mode 100644 index e257f1feb0b4..000000000000 --- a/cpp/include/jit_cudf/functions/logical_or.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void logical_or(T* out, T const& a, T const& b) -{ - *out = a || b; -} - -template -__device__ inline void logical_or(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - logical_or(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/mod.cuh b/cpp/include/jit_cudf/functions/mod.cuh deleted file mode 100644 index 3d603d92cff4..000000000000 --- a/cpp/include/jit_cudf/functions/mod.cuh +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void mod(T* out, T const& a, T const& b) -{ - *out = a % b; -} - -__device__ inline void mod(f32* out, f32 const& a, f32 const& b) { *out = __builtin_fmodf(a, b); } - -__device__ inline void mod(f64* out, f64 const& a, f64 const& b) { *out = __builtin_fmod(a, b); } - -template -__device__ inline void mod(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - mod(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/mul.cuh b/cpp/include/jit_cudf/functions/mul.cuh deleted file mode 100644 index 89f3410be450..000000000000 --- a/cpp/include/jit_cudf/functions/mul.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void mul(T* out, T const& a, T const& b) -{ - *out = a * b; -} - -template -__device__ inline void mul(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - mul(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/null_equal.cuh b/cpp/include/jit_cudf/functions/null_equal.cuh deleted file mode 100644 index 20cdcf9b51a9..000000000000 --- a/cpp/include/jit_cudf/functions/null_equal.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void null_equal(bool* out, T const& a, T const& b) -{ - *out = a == b; -} - -template -__device__ inline void null_equal(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - *out = (*a == *b); - } else if (a.has_null() && b.has_null()) { - *out = true; - } else { - *out = false; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/null_logical_and.cuh b/cpp/include/jit_cudf/functions/null_logical_and.cuh deleted file mode 100644 index 1962ee57c874..000000000000 --- a/cpp/include/jit_cudf/functions/null_logical_and.cuh +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void null_logical_and(bool* out, bool const& a, bool const& b) { *out = a && b; } - -__device__ inline void null_logical_and(optional* out, - optional const& a, - optional const& b) -{ - if (a.has_value() && b.has_value()) { - *out = (*a && *b); - } else if (a.has_null() && b.has_null()) { - *out = nullopt; - } else { - bool valid = a.has_value() ? *a : *b; - if (valid) { - *out = nullopt; - } else { - *out = false; - } - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/null_logical_or.cuh b/cpp/include/jit_cudf/functions/null_logical_or.cuh deleted file mode 100644 index 6d9d2efa77ae..000000000000 --- a/cpp/include/jit_cudf/functions/null_logical_or.cuh +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void null_logical_or(bool* out, bool const& a, bool const& b) { *out = a || b; } - -__device__ inline void null_logical_or(optional* out, - optional const& a, - optional const& b) -{ - if (a.has_value() && b.has_value()) { - *out = (*a || *b); - } else if (a.has_null() && b.has_null()) { - *out = nullopt; - } else { - bool valid = a.has_value() ? *a : *b; - if (valid) { - *out = true; - } else { - *out = nullopt; - } - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/pow.cuh b/cpp/include/jit_cudf/functions/pow.cuh deleted file mode 100644 index 245503b6ce39..000000000000 --- a/cpp/include/jit_cudf/functions/pow.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void pow(f32* out, f32 const& a, f32 const& b) { *out = __builtin_powf(a, b); } - -__device__ inline void pow(f64* out, f64 const& a, f64 const& b) { *out = __builtin_pow(a, b); } - -template -__device__ inline void pow(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - pow(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/pymod.cuh b/cpp/include/jit_cudf/functions/pymod.cuh deleted file mode 100644 index a4c408f97bc2..000000000000 --- a/cpp/include/jit_cudf/functions/pymod.cuh +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void pymod(T* out, T const& a, T const& b) -{ - *out = (a % b + b) % b; -} - -__device__ inline void pymod(f32* out, f32 const& a, f32 const& b) -{ - *out = __builtin_fmodf(__builtin_fmodf(a, b) + b, b); -} - -__device__ inline void pymod(f64* out, f64 const& a, f64 const& b) -{ - *out = __builtin_fmod(__builtin_fmod(a, b) + b, b); -} - -template -__device__ inline void pymod(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - pymod(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/rint.cuh b/cpp/include/jit_cudf/functions/rint.cuh deleted file mode 100644 index 1feef0f4daaf..000000000000 --- a/cpp/include/jit_cudf/functions/rint.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void rint(f32* out, f32 const& a) { *out = __builtin_rintf(a); } - -__device__ inline void rint(f64* out, f64 const& a) { *out = __builtin_rint(a); } - -template -__device__ inline void rint(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - rint(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/sin.cuh b/cpp/include/jit_cudf/functions/sin.cuh deleted file mode 100644 index 8291eaa22fd6..000000000000 --- a/cpp/include/jit_cudf/functions/sin.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void sin(f32* out, f32 const& a) { *out = __builtin_sinf(a); } - -__device__ inline void sin(f64* out, f64 const& a) { *out = __builtin_sin(a); } - -template -__device__ inline void sin(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - sin(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/sinh.cuh b/cpp/include/jit_cudf/functions/sinh.cuh deleted file mode 100644 index 293ae222e2d2..000000000000 --- a/cpp/include/jit_cudf/functions/sinh.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void sinh(f32* out, f32 const& a) { *out = __builtin_sinhf(a); } - -__device__ inline void sinh(f64* out, f64 const& a) { *out = __builtin_sinh(a); } - -template -__device__ inline void sinh(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - sinh(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/sub.cuh b/cpp/include/jit_cudf/functions/sub.cuh deleted file mode 100644 index 4614ca3ffc74..000000000000 --- a/cpp/include/jit_cudf/functions/sub.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -template -__device__ inline void sub(T* out, T const& a, T const& b) -{ - *out = a - b; -} - -template -__device__ inline void sub(optional* out, optional const& a, optional const& b) -{ - if (a.has_value() && b.has_value()) { - T r; - sub(&r, *a, *b); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/functions/tanh.cuh b/cpp/include/jit_cudf/functions/tanh.cuh deleted file mode 100644 index 664da0811e3c..000000000000 --- a/cpp/include/jit_cudf/functions/tanh.cuh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -namespace jit { -namespace cudf { -namespace functions { - -__device__ inline void tanh(f32* out, f32 const& a) { *out = __builtin_tanhf(a); } - -__device__ inline void tanh(f64* out, f64 const& a) { *out = __builtin_tanh(a); } - -template -__device__ inline void tanh(optional* out, optional const& a) -{ - if (a.has_value()) { - T r; - tanh(&r, *a); - *out = r; - } else { - *out = nullopt; - } -} - -} // namespace functions -} // namespace cudf -} // namespace jit diff --git a/cpp/include/jit_cudf/types.cuh b/cpp/include/jit_cudf/types.cuh deleted file mode 100644 index dad4f69d4c0c..000000000000 --- a/cpp/include/jit_cudf/types.cuh +++ /dev/null @@ -1,672 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -namespace jit { -namespace cudf { - -using i8 = signed char; -using i16 = signed short; -using i32 = signed int; -using i64 = signed long; -using i128 = __int128_t; -using u8 = unsigned char; -using u16 = unsigned short; -using u32 = unsigned int; -using u64 = unsigned long; - -using char_utf8 = u32; - -using usize = unsigned long; -using iptr = i64; -using uptr = u64; - -using intmax_t = i64; -using uintmax_t = u64; - -using f32 = float; -using f64 = double; - -using size_type = i32; - -using bitmask_t = u32; - -template -__device__ constexpr bool bit_is_set(T const* bitmask, usize bit_index) -{ - constexpr auto bits_per_word = sizeof(T) * 8; - return bitmask[bit_index / bits_per_word] & (T{1} << (bit_index % bits_per_word)); -} - -template -inline constexpr bool Same = false; - -template -inline constexpr bool Same = true; - -enum class type_id : i32 { - EMPTY = 0, - INT8 = 1, - INT16 = 2, - INT32 = 3, - INT64 = 4, - UINT8 = 5, - UINT16 = 6, - UINT32 = 7, - UINT64 = 8, - FLOAT32 = 9, - FLOAT64 = 10, - BOOL8 = 11, - TIMESTAMP_DAYS = 12, - TIMESTAMP_SECONDS = 13, - TIMESTAMP_MILLISECONDS = 14, - TIMESTAMP_MICROSECONDS = 15, - TIMESTAMP_NANOSECONDS = 16, - DURATION_DAYS = 17, - DURATION_SECONDS = 18, - DURATION_MILLISECONDS = 19, - DURATION_MICROSECONDS = 20, - DURATION_NANOSECONDS = 21, - DICTIONARY32 = 22, - STRING = 23, - LIST = 24, - DECIMAL32 = 25, - DECIMAL64 = 26, - DECIMAL128 = 27, - STRUCT = 28, - NUM_TYPE_IDS = 29 -}; - -struct data_type { - type_id _id = {}; - - i32 _scale = 0; - - __device__ constexpr type_id id() const { return _id; } - - __device__ constexpr i32 scale() const { return _scale; } -}; - -template -__device__ constexpr T min(T a, T b) -{ - return a < b ? a : b; -} - -template -__device__ constexpr T max(T a, T b) -{ - return a > b ? a : b; -} - -template -__device__ constexpr T ipow10(T exponent) -{ - if (exponent == 0) { return 1; } - - T extra = 1; - T square = 10; - T n = exponent; - - while (n > 1) { - if ((n & 1) == 1) { extra *= square; } - n >>= 1; - square *= square; - } - - return square * extra; -} - -template -__device__ constexpr T dec_lshift(T v, i32 scale) -{ - return v * ipow10(-scale); -} - -template -__device__ constexpr T dec_rshift(T v, i32 scale) -{ - return v / ipow10(scale); -} - -template -__device__ constexpr T dec_shift(T v, i32 scale) -{ - if (scale == 0) { - return v; - } else if (scale < 0) { - return dec_lshift(v, scale); - } else { - return dec_rshift(v, scale); - } -} - -template -__device__ constexpr T dec_rescale(T v, i32 from_scale, i32 to_scale) -{ - return dec_shift(v, to_scale - from_scale); -} - -struct scaled_t {}; - -inline constexpr scaled_t scaled; - -template -struct dec { - using Rep = R; - - R _value = 0; - - i32 _scale = 0; - - __device__ constexpr dec(scaled_t, R value, i32 scale) : _value{value}, _scale{scale} {} - - constexpr dec() = default; - - __device__ constexpr R value() const { return _value; } - - __device__ constexpr i32 scale() const { return _scale; } -}; - -using dec32 = dec; -using dec64 = dec; -using dec128 = dec; - -template -__device__ constexpr auto rescale(dec a, i32 scale) -{ - return dec{scaled, dec_rescale(a._value, a._scale, scale), scale}; -} - -template -__device__ constexpr auto operator+(dec a, dec b) -{ - auto scale = min(a._scale, b._scale); - auto r = rescale(a, scale)._value + rescale(b, scale)._value; - return dec{scaled, r, scale}; -} - -template -__device__ constexpr auto operator-(dec a, dec b) -{ - auto scale = min(a._scale, b._scale); - auto r = rescale(a, scale)._value - rescale(b, scale)._value; - return dec{scaled, r, scale}; -} - -template -__device__ constexpr auto operator*(dec a, dec b) -{ - return dec{scaled, a._value * b._value, a._scale + b._scale}; -} - -template -__device__ constexpr auto operator/(dec a, dec b) -{ - return dec{scaled, a._value / b._value, a._scale - b._scale}; -} - -template -__device__ constexpr auto operator%(dec a, dec b) -{ - auto scale = min(a._scale, b._scale); - auto r = rescale(a, scale)._value % rescale(b, scale)._value; - return dec{scaled, r, scale}; -} - -template -__device__ constexpr int operator<=>(dec a, dec b) -{ - auto scale = min(a._scale, b._scale); - return rescale(a, scale)._value - rescale(b, scale)._value; -} - -enum class timestamp_unit : i32 { D, h, m, s, ms, us, ns }; - -template -struct timestamp { - using Rep = R; - - R _rep = 0; - - __device__ constexpr R count() const { return _rep; } -}; - -using timestamp_D = timestamp; -using timestamp_h = timestamp; -using timestamp_m = timestamp; -using timestamp_s = timestamp; -using timestamp_ms = timestamp; -using timestamp_us = timestamp; -using timestamp_ns = timestamp; - -template -__device__ constexpr int operator<=>(timestamp a, timestamp b) -{ - return a._rep - b._rep; -} - -template -struct duration { - using Rep = R; - - R _rep = 0; - - __device__ constexpr R count() const { return _rep; } -}; - -using duration_D = duration; -using duration_h = duration; -using duration_m = duration; -using duration_s = duration; -using duration_ms = duration; -using duration_us = duration; -using duration_ns = duration; - -template -__device__ constexpr duration operator+(duration a, duration b) -{ - return duration{a._rep + b._rep}; -} - -template -__device__ constexpr duration operator-(duration a, duration b) -{ - return duration{a._rep - b._rep}; -} - -template -__device__ constexpr int operator<=>(duration a, duration b) -{ - return a._rep - b._rep; -} - -struct string_view { - static constexpr size_type const UNKNOWN_STRING_LENGTH{-1}; - static constexpr size_type const npos{-1}; - - char const* _data = nullptr; - - size_type _bytes = 0; - - mutable size_type _length = UNKNOWN_STRING_LENGTH; - - __device__ constexpr size_type size_bytes() const { return _bytes; } - - __device__ constexpr char const* data() const { return _data; } - - __device__ constexpr bool empty() const { return _bytes == 0; } - - __device__ constexpr size_type compare(string_view const& other) const - { - auto* s0 = _data; - auto n0 = _bytes; - auto* s1 = other._data; - auto n1 = other._bytes; - auto max = n0 < n1 ? n0 : n1; - - if (s0 == s1 && n0 == n1) return 0; - - size_type idx = 0; - - while (idx < max) { - if (*s0 != *s1) return static_cast(*s0) - static_cast(*s1); - s0++; - s1++; - idx++; - } - - if (idx < n0) { return 1; } - if (idx < n1) { return -1; } - - return 0; - } -}; - -__device__ constexpr int operator<=>(string_view const& a, string_view const& b) -{ - return a.compare(b); -} - -struct inplace_t {}; - -inline constexpr inplace_t inplace; - -struct nullopt_t {}; - -inline constexpr nullopt_t nullopt; - -template -struct optional { - T _val = {}; - - bool _engaged = false; - - constexpr optional() = default; - - __device__ constexpr optional(nullopt_t) {} - - template - __device__ constexpr optional(inplace_t, Args&&... args) - : _val{static_cast(args)...}, _engaged{true} - { - } - - __device__ constexpr optional(T val) : _val{val}, _engaged{true} {} - - __device__ constexpr bool has_value() const { return _engaged; } - - __device__ constexpr bool has_null() const { return !_engaged; } - - __device__ constexpr void reset() { _engaged = false; } - - __device__ constexpr T const& get() const { return _val; } - - __device__ constexpr T& get() { return _val; } - - __device__ constexpr T const* operator->() const { return &_val; } - - __device__ constexpr T* operator->() { return &_val; } - - __device__ constexpr T const& operator*() const { return _val; } - - __device__ constexpr T& operator*() { return _val; } - - __device__ constexpr T const& value() const { return _val; } - - __device__ constexpr T& value() { return _val; } - - __device__ constexpr explicit operator bool() const { return _engaged; } - - __device__ constexpr T value_or(T __v) const { return _engaged ? _val : __v; } -}; - -template -optional(T) -> optional; - -template -struct span { - T* _data = nullptr; - - usize _size = 0; - - __device__ constexpr T* data() const { return _data; } - - __device__ constexpr usize size() const { return _size; } - - __device__ constexpr bool empty() const { return _size == 0; } - - __device__ constexpr T& operator[](usize pos) const { return _data[pos]; } - - __device__ constexpr T* begin() const { return _data; } - - __device__ constexpr T* end() const { return _data + _size; } - - __device__ constexpr span as_const() const { return span{_data, _size}; } - - template - __device__ constexpr T& element(usize idx) const - { - return _data[idx]; - } - - __device__ constexpr bool nullable() const { return false; } - - __device__ constexpr bool is_valid_nocheck(usize element_index) const { return true; } - - __device__ constexpr bool is_valid(usize element_index) const { return true; } - - __device__ constexpr bool is_null(usize element_index) const { return false; } - - template - __device__ constexpr optional nullable_element(usize idx) const - { - if (!is_valid(idx)) return nullopt; - return element(idx); - } - - __device__ constexpr void assign(usize idx, T value) const { _data[idx] = value; } -}; - -template -span(T*, usize) -> span; - -template -struct optional_span { - T* _data = nullptr; - - usize _size = 0; - - bitmask_t const* _null_mask = nullptr; - - __device__ constexpr T* data() const { return _data; } - - __device__ constexpr usize size() const { return _size; } - - __device__ constexpr bool empty() const { return _size == 0; } - - __device__ constexpr T& operator[](usize pos) const { return _data[pos]; } - - __device__ constexpr T* begin() const { return _data; } - - __device__ constexpr T* end() const { return _data + _size; } - - __device__ constexpr optional_span as_const() const - { - return optional_span{_data, _size, _null_mask}; - } - - __device__ constexpr bool nullable() const { return _null_mask != nullptr; } - - __device__ constexpr bool is_valid_nocheck(usize element_index) const - { - return bit_is_set(_null_mask, element_index); - } - - __device__ constexpr bool is_valid(usize element_index) const - { - return not nullable() or is_valid_nocheck(element_index); - } - - __device__ constexpr bool is_null(usize element_index) const { return !is_valid(element_index); } - - template - __device__ constexpr T& element(usize idx) const - { - return _data[idx]; - } - - template - __device__ constexpr optional nullable_element(usize idx) const - { - if (!is_valid(idx)) return nullopt; - return element(idx); - } - - template - __device__ constexpr void assign(usize idx, T value) const - { - _data[idx] = value; - } -}; - -template -optional_span(T*, usize, bitmask_t const*) -> optional_span; - -struct alignas(16) column_view { - template - static constexpr bool HasSpanLayout = - Same || Same || Same || Same || Same || Same || - Same || Same || Same || Same || Same || - Same || Same || Same || Same || - Same || Same || Same || - Same || Same || Same || Same || - Same || Same || Same; - - template - static constexpr bool HasDecimalLayout = Same || Same || Same; - - data_type _type = {}; - - size_type _size = 0; - - void const* _data = nullptr; - - bitmask_t const* _null_mask = nullptr; - - size_type _offset = 0; - - column_view* _d_children = nullptr; - - size_type _num_children = 0; - - __device__ constexpr data_type type() const { return _type; } - - __device__ constexpr size_type size() const { return _size; } - - __device__ constexpr bool nullable() const { return _null_mask != nullptr; } - - __device__ constexpr bitmask_t const* null_mask() const { return _null_mask; } - - __device__ constexpr size_type offset() const { return _offset; } - - __device__ constexpr bool is_valid(size_type idx) const - { - return !nullable() || is_valid_nocheck(idx); - } - - __device__ constexpr bool is_valid_nocheck(size_type idx) const - { - return bit_is_set(_null_mask, _offset + idx); - } - - __device__ constexpr bool is_null(size_type idx) const { return !is_valid(idx); } - - __device__ constexpr bool is_null_nocheck(size_type idx) const { return !is_valid_nocheck(idx); } - - __device__ constexpr size_type num_child_columns() const { return _num_children; } - - template - __device__ auto& element(size_type idx) const - requires(HasSpanLayout) - { - return static_cast(_data)[_offset + idx]; - } - - template - __device__ auto element(size_type idx) const - requires(HasDecimalLayout) - { - return T{scaled, static_cast(_data)[_offset + idx], _type.scale()}; - } - - template - __device__ string_view element(size_type idx) const - requires(Same) - { - static constexpr i32 OFFSETS_CHILD = 0; - auto i = _offset + idx; - auto* str_data = static_cast(_data); - auto& offsets = _d_children[OFFSETS_CHILD]; - auto* i32_runs = static_cast(offsets._data); - auto* i64_runs = static_cast(offsets._data); - - i64 run_begin = 0; - i64 run_end = 0; - - switch (offsets.type().id()) { - case type_id::INT32: - run_begin = i32_runs[i]; - run_end = i32_runs[i + 1]; - break; - case type_id::INT64: - run_begin = i64_runs[i]; - run_end = i64_runs[i + 1]; - break; - default: __builtin_unreachable(); - } - - i64 run_size = run_end - run_begin; - - return string_view{str_data + run_begin, static_cast(run_size)}; - } - - template - __device__ optional nullable_element(size_type idx) const - { - if (!is_valid(idx)) return nullopt; - return element(idx); - } -}; - -// TODO: scope variables should be aligned to avoid uncoalesced reads/writes -namespace scope { - -using args = void* const*; - -template -struct column { - static constexpr bool IS_SCALAR = IsScalar; - static constexpr bool IS_NULLABLE = IsNullable; - - using Type = T; - using Arg = ColumnType const*; - - static __device__ decltype(auto) element(args scope, size_type i) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - if constexpr (!IsNullable) { - return p->template element(index); - } else { - return p->template nullable_element(index); - } - } - - static __device__ void assign(args scope, size_type i, T value) - { - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - p->template assign(index, value); - } - - static __device__ auto* null_mask(args scope) - { - auto p = static_cast(scope[ScopeIndex]); - return p->null_mask(); - } - - static __device__ bool is_null(args scope, size_type i) - { - if constexpr (!IsNullable) { return false; } - - auto p = static_cast(scope[ScopeIndex]); - auto index = IsScalar ? 0 : i; - - return p->is_null(index); - } - - static __device__ bool is_valid(args scope, size_type i) { return !is_null(scope, i); } -}; - -template -struct user_data { - using Arg = void*; - - static __device__ decltype(auto) element(args scope, [[maybe_unused]] size_type i) - { - return static_cast(scope[ScopeIndex]); - } -}; - -} // namespace scope -} // namespace cudf -} // namespace jit From db2e2dd3fb8ec956d6415d6f27c673599d8fdca7 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 2 Mar 2026 21:17:30 +0000 Subject: [PATCH 061/254] Update conda environment files to require python-zstd version 1.5.7.2 and remove deprecated nullable_element functions --- .../all_cuda-129_arch-aarch64.yaml | 2 +- .../all_cuda-129_arch-x86_64.yaml | 2 +- .../all_cuda-131_arch-aarch64.yaml | 2 +- .../all_cuda-131_arch-x86_64.yaml | 2 +- cpp/CMakeLists.txt | 21 --- .../cudf/column/column_device_view_base.cuh | 169 ------------------ cpp/src/librtcx/README.md | 9 +- cpp/src/librtcx/rtcx.hpp | 5 +- 8 files changed, 12 insertions(+), 200 deletions(-) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 981c4ef50952..80acb102cbae 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -80,8 +80,8 @@ dependencies: - pytest-xdist - python-confluent-kafka - python-xxhash +- python-zstd>=1.5.7.2 - python>=3.11 -- python-zstd - pytorch>=2.4.0 - rapids-build-backend>=0.4.0,<0.5.0 - rapids-dask-dependency==26.4.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 9dfb18137e44..550ed73623e2 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -80,7 +80,7 @@ dependencies: - pytest-xdist - python-confluent-kafka - python-xxhash -- python-zstd +- python-zstd>=1.5.7.2 - python>=3.11 - pytorch>=2.4.0 - rapids-build-backend>=0.4.0,<0.5.0 diff --git a/conda/environments/all_cuda-131_arch-aarch64.yaml b/conda/environments/all_cuda-131_arch-aarch64.yaml index 4b50d2486161..17645b3d824c 100644 --- a/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -80,7 +80,7 @@ dependencies: - pytest-xdist - python-confluent-kafka - python-xxhash -- python-zstd +- python-zstd>=1.5.7.2 - python>=3.11 - rapids-build-backend>=0.4.0,<0.5.0 - rapids-dask-dependency==26.4.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-131_arch-x86_64.yaml b/conda/environments/all_cuda-131_arch-x86_64.yaml index 5d19f512ef28..42038217d27e 100644 --- a/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -80,7 +80,7 @@ dependencies: - pytest-xdist - python-confluent-kafka - python-xxhash -- python-zstd +- python-zstd>=1.5.7.2 - python>=3.11 - rapids-build-backend>=0.4.0,<0.5.0 - rapids-dask-dependency==26.4.*,>=0.0.0a0 diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2c4706330df7..e877d4c2df4d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -334,30 +334,9 @@ if(NOT BUILD_SHARED_LIBS) endif() endif() -if(CMAKE_BUILD_TYPE STREQUAL "Debug") - set(CUDF_JIT_CUDA_FLAGS "-G") -else() - if(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") - set(CUDF_JIT_CUDA_FLAGS "-O2;-DNDEBUG") - else() - if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(CUDF_JIT_CUDA_FLAGS "-O3;-DNDEBUG") - endif() - endif() -endif() - -if(CUDA_ENABLE_LINEINFO) - list(APPEND CUDF_JIT_CUDA_FLAGS "${CUDF_JIT_CUDA_FLAGS};-lineinfo") -endif() - set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) -list(APPEND CUDF_JIT_DEFINITIONS -DCUDF_RUNTIME_JIT -D__CUDACC_RTC__) -list(APPEND CUDF_JIT_COMPILE_FLAGS -std=c++${CUDF_CUDA_STANDARD} --device-int128 - ${CUDF_JIT_DEFINITIONS} -) - jit_add_include_directory( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include DEST_DIRECTORY cudf/include INCLUDE_DIRECTORIES cudf/include diff --git a/cpp/include/cudf/column/column_device_view_base.cuh b/cpp/include/cudf/column/column_device_view_base.cuh index c062f8029f6a..3e9130085c7e 100644 --- a/cpp/include/cudf/column/column_device_view_base.cuh +++ b/cpp/include/cudf/column/column_device_view_base.cuh @@ -405,30 +405,6 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba return data()[element_index]; } - /** - * @brief Returns an optional of the element at the specified index. - * - * This function accounts for the offset. - * - * This function does not participate in overload resolution if `is_rep_layout_compatible` is - * false. Specializations of this function may exist for types `T` where - *`is_rep_layout_compatible` is false. - * - * - * @tparam T The element type - * @param element_index Position of the desired element - * @return Optional containing the element at the specified index, or `nullopt` if the element is - *null - */ - template ())> - [[nodiscard]] __device__ cuda::std::optional nullable_element( - size_type element_index) const noexcept - { - if (is_null(element_index)) { return cuda::std::nullopt; } - - return element(element_index); - } - /** * @brief Returns `string_view` to the string element at the specified index. * @@ -451,27 +427,6 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba return string_view{d_strings + offset, static_cast(itr[index + 1] - offset)}; } - /** - * @brief Returns optional `string_view` to the string element at the specified index. - * - * If the element at the specified index is NULL, i.e., `is_null(element_index) - * == true`, then nullopt is returned. - * - * This function accounts for the offset. - * - * @param element_index Position of the desired string element - * @return Optional containing the string_view instance representing this element at this index, - * or `nullopt` if the element is null - */ - template )> - [[nodiscard]] __device__ cuda::std::optional nullable_element( - size_type element_index) const noexcept - { - if (is_null(element_index)) { return cuda::std::nullopt; } - - return element(element_index); - } - public: /** * @brief Returns a `numeric::fixed_point` element at the specified index for a `fixed_point` @@ -492,25 +447,6 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba return T{scaled_integer{data()[element_index], scale}}; } - /** - * @brief Returns a `numeric::fixed_point` element at the specified index for a `fixed_point` - * column. - * - * If the element at the specified index is NULL, i.e., `is_null(element_index) == true`, - * then any attempt to use the result will lead to undefined behavior. - * - * @param element_index Position of the desired element - * @return numeric::fixed_point representing the element at this index - */ - template ())> - [[nodiscard]] __device__ cuda::std::optional nullable_element( - size_type element_index) const noexcept - { - if (is_null(element_index)) { return cuda::std::nullopt; } - - return element(element_index); - } - /** * @brief Returns the specified child * @@ -665,111 +601,6 @@ class alignas(16) mutable_column_device_view_core : public detail::column_device return data()[element_index]; } - /** - * @brief Returns an optional of the element at the specified index. - * - * This function accounts for the offset. - * - * This function does not participate in overload resolution if `is_rep_layout_compatible` is - * false. Specializations of this function may exist for types `T` where - *`is_rep_layout_compatible` is false. - * - * - * @tparam T The element type - * @param element_index Position of the desired element - * @return Optional containing the element at the specified index, or `nullopt` if the element is - *null - */ - template ())> - [[nodiscard]] __device__ cuda::std::optional nullable_element( - size_type element_index) const noexcept - { - if (is_null(element_index)) { return cuda::std::nullopt; } - - return element(element_index); - } - - /** - * @brief Returns `string_view` to the string element at the specified index. - * - * If the element at the specified index is NULL, i.e., `is_null(element_index) - * == true`, then any attempt to use the result will lead to undefined behavior. - * - * This function accounts for the offset. - * - * @param element_index Position of the desired string element - * @return string_view instance representing this element at this index - */ - template )> - [[nodiscard]] __device__ T element(size_type element_index) const noexcept - { - size_type index = element_index + offset(); // account for this view's _offset - char const* d_strings = static_cast(_data); - auto const offsets = child(offsets_column_index); - auto const itr = cudf::detail::input_offsetalator(offsets.head(), offsets.type()); - auto const offset = itr[index]; - return string_view{d_strings + offset, static_cast(itr[index + 1] - offset)}; - } - - /** - * @brief Returns optional `string_view` to the string element at the specified index. - * - * If the element at the specified index is NULL, i.e., `is_null(element_index) - * == true`, then nullopt is returned. - * - * This function accounts for the offset. - * - * @param element_index Position of the desired string element - * @return Optional containing the string_view instance representing this element at this index, - * or `nullopt` if the element is null - */ - template )> - [[nodiscard]] __device__ cuda::std::optional nullable_element( - size_type element_index) const noexcept - { - if (is_null(element_index)) { return cuda::std::nullopt; } - - return element(element_index); - } - - /** - * @brief Returns a `numeric::fixed_point` element at the specified index for a `fixed_point` - * column. - * - * If the element at the specified index is NULL, i.e., `is_null(element_index) == true`, - * then any attempt to use the result will lead to undefined behavior. - * - * @param element_index Position of the desired element - * @return numeric::fixed_point representing the element at this index - */ - template ())> - [[nodiscard]] __device__ T element(size_type element_index) const noexcept - { - using namespace numeric; - using rep = typename T::rep; - auto const scale = scale_type{_type.scale()}; - return T{scaled_integer{data()[element_index], scale}}; - } - - /** - * @brief Returns a `numeric::fixed_point` element at the specified index for a `fixed_point` - * column. - * - * If the element at the specified index is NULL, i.e., `is_null(element_index) == true`, - * then any attempt to use the result will lead to undefined behavior. - * - * @param element_index Position of the desired element - * @return numeric::fixed_point representing the element at this index - */ - template ())> - [[nodiscard]] __device__ cuda::std::optional nullable_element( - size_type element_index) const noexcept - { - if (is_null(element_index)) { return cuda::std::nullopt; } - - return element(element_index); - } - /** * @brief Assigns `value` to the element at `element_index` * diff --git a/cpp/src/librtcx/README.md b/cpp/src/librtcx/README.md index 4f33fa6b665d..56d1c435bb6c 100644 --- a/cpp/src/librtcx/README.md +++ b/cpp/src/librtcx/README.md @@ -1,7 +1,8 @@ # Doc -# vs JITIFY +RTCX (runtime-compiler extended) is a wrapper around NVRTC and NVJitLink designed to provide: -# Design - -# How to Achieve Fast Compilation with LTO-IR +- User-controlled compilation, linking, caching, and pre-loading of CUDA kernels +- Zero-copy interfaces to manage JIT compilation and linking +- CMake script to embed **compressed** headers directly into an executable without incurring overhead at runtime on every compilation request +- Facilities to pre-load and teardown dynamic library dependencies (libcuda, libnvrtc, and libnvjitlink) diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 2c0ca141075f..ad5a61fc3c63 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -31,7 +31,7 @@ #define RTCX_EXPECTS(_condition, _reason, _exception_type) \ do { \ if (!(_condition)) { \ - throw _exception_type{::std::format("RTC failure at: {}:{}: {}", \ + throw _exception_type{::std::format("RTCX failure at: {}:{}: {}", \ ::std::source_location::current().file_name(), \ ::std::source_location::current().line(), \ (_reason))}; \ @@ -40,7 +40,7 @@ #define RTCX_FAIL(_reason, _exception_type) \ do { \ - throw _exception_type{::std::format("RTC failure at: {}:{}: {}", \ + throw _exception_type{::std::format("RTCX failure at: {}:{}: {}", \ ::std::source_location::current().file_name(), \ ::std::source_location::current().line(), \ (_reason))}; \ @@ -55,6 +55,7 @@ typedef struct CUstream_st* CUstream; } namespace RTCX_EXPORT rtcx { + using u8 = std::uint8_t; using u16 = std::uint16_t; using u32 = std::uint32_t; From 75916e464e9da33a4e3f75fd6e6a6f253257f03b Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 2 Mar 2026 21:24:55 +0000 Subject: [PATCH 062/254] Refactor init_flags and initialization logic to remove RTC and JIT bundle initialization --- cpp/include/cudf/context.hpp | 9 ++------- cpp/src/runtime/context.cpp | 11 ++++++----- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index 62665787b023..8912377a407c 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -20,14 +20,9 @@ enum class init_flags : std::uint32_t { LOAD_NVCOMP = 1 << 0, /// @brief Initialize the JIT program cache during initialization INIT_JIT_CACHE = 1 << 1, - /// @brief Initialize the RTC cache during initialization - INIT_RTC_CACHE = 1 << 2, - /// @brief Initialize the JIT bundle during initialization - INIT_JIT_BUNDLE = 1 << 3, - /// @brief The default initialization steps performed by `initialize()` if no flags are provided - DEFAULT = INIT_JIT_CACHE | INIT_RTC_CACHE | INIT_JIT_BUNDLE, + DEFAULT = INIT_JIT_CACHE, /// @brief All initialization steps - ALL = LOAD_NVCOMP | INIT_JIT_CACHE | INIT_RTC_CACHE | INIT_JIT_BUNDLE + ALL = LOAD_NVCOMP | INIT_JIT_CACHE }; /// @brief Bitwise OR operator for init_flags diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 3d13db1febad..a9999c3a1865 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -79,13 +79,14 @@ std::string const& context::get_jit_pch_dir() const { return _config.jit_pch_dir void context::initialize_components(init_flags flags) { - if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { ensure_jit_cache_initialized(); } + if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { + rtcx::initialize(); + ensure_jit_cache_initialized(); + ensure_rtc_cache_initialized(); + ensure_jit_bundle_initialized(); + } if (has_flag(flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } - - if (has_flag(flags, init_flags::INIT_RTC_CACHE)) { ensure_rtc_cache_initialized(); } - - if (has_flag(flags, init_flags::INIT_JIT_BUNDLE)) { ensure_jit_bundle_initialized(); } } static std::optional _context{std::nullopt}; From 8bb341d76ee7eb8df623768c6b8580d67c23e29d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 2 Mar 2026 21:32:02 +0000 Subject: [PATCH 063/254] Clean up JIT-related code: remove RTC test file, delete unused transform2.cu, and update export.hpp copyright year --- cpp/include/cudf/utilities/export.hpp | 7 +- cpp/src/librtcx/system_includes/README.md | 1 - cpp/src/transform/transform.cu | 1 + cpp/src/transform/transform2.cu | 44 ---- cpp/tests/CMakeLists.txt | 1 - cpp/tests/jit/rtc.cpp | 293 ---------------------- 6 files changed, 2 insertions(+), 345 deletions(-) delete mode 100644 cpp/src/librtcx/system_includes/README.md delete mode 100644 cpp/src/transform/transform2.cu delete mode 100644 cpp/tests/jit/rtc.cpp diff --git a/cpp/include/cudf/utilities/export.hpp b/cpp/include/cudf/utilities/export.hpp index 9a8465efd6e5..815d3110f2cb 100644 --- a/cpp/include/cudf/utilities/export.hpp +++ b/cpp/include/cudf/utilities/export.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,12 +7,7 @@ // Macros used for defining symbol visibility, only GLIBC is supported #if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) -#ifndef CUDF_DISABLE_DEFAULT_EXPORT #define CUDF_EXPORT __attribute__((visibility("default"))) -#else -#define CUDF_EXPORT __attribute__((visibility("hidden"))) -#endif - #define CUDF_HIDDEN __attribute__((visibility("hidden"))) #else #define CUDF_EXPORT diff --git a/cpp/src/librtcx/system_includes/README.md b/cpp/src/librtcx/system_includes/README.md deleted file mode 100644 index 14d14195d2c8..000000000000 --- a/cpp/src/librtcx/system_includes/README.md +++ /dev/null @@ -1 +0,0 @@ -# Doc diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 162c295e7507..c909b94efe32 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -23,6 +23,7 @@ #include #include #include +#include #include #include diff --git a/cpp/src/transform/transform2.cu b/cpp/src/transform/transform2.cu deleted file mode 100644 index c83f8dee53d2..000000000000 --- a/cpp/src/transform/transform2.cu +++ /dev/null @@ -1,44 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -#include -#include -#include - -namespace cudf { - -enum class [[nodiscard]] udf_source_type : uint8_t { LTOIR_BINARY = 0, PTX_BINARY = 1, CUDA = 2 }; - -struct [[nodiscard]] transform_operation_params { - using input = std::variant; - - std::vector inputs = {}; - - std::vector include_nullness = {}; - - std::vector output_sizes = {}; - - std::vector output_types = {}; - - std::vector output_nullability = {}; - - std::span udf = {}; - - udf_source_type type = udf_source_type::LTOIR_BINARY; - - rmm::cuda_stream_view stream = cudf::get_default_stream(); - - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref(); -}; - -std::unique_ptr
transform_operation(transform_operation_params const& params) -{ - return nullptr; -} - -} // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 0d940ee9a612..6d01bc80af83 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -713,7 +713,6 @@ ConfigureTest(JIT_PARSER_TEST jit/parse_ptx_function.cpp) target_include_directories(JIT_PARSER_TEST PRIVATE "$") ConfigureTest(ROW_IR_TEST jit/row_ir.cpp) -ConfigureTest(RTC_TEST jit/rtc.cpp) # ################################################################################################## # * stream testing --------------------------------------------------------------------------------- diff --git a/cpp/tests/jit/rtc.cpp b/cpp/tests/jit/rtc.cpp deleted file mode 100644 index 0983ef54419a..000000000000 --- a/cpp/tests/jit/rtc.cpp +++ /dev/null @@ -1,293 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "cudf_test/column_wrapper.hpp" -#include "jit/row_ir.hpp" - -#include -#include - -#include - -#include - -using namespace cudf; - -struct RTCTest : public ::testing::Test {}; - -/* -template -struct element_operation { - template - static __device__ void evaluate(args scope, cudf::size_type i, Operator&& op) - { - if constexpr (UserDataIndex >= 0) { - auto output_args; - GENERIC_TRANSFORM_OP(user_data, i, &res, In::element(inputs, i)...); - } else { - GENERIC_TRANSFORM_OP(&res, In::element(inputs, i)...); - } - } -}; -*/ - -// TODO: write a planner -// TODO: flags to clear JIT at program startup -// TODO: nvrtc uses the program name to do PCH - -TEST_F(RTCTest, CompileKernelBasic) -{ - auto fn = [] { - char const* udf = R"***( - #include "jcudf/types.cuh" - - #pragma nv_hdrstop - - struct operator_params{ - void* const * scope; - jcudf::size_type row_index; - }; - - struct kernel_params{ - void* const * scope; - jcudf::size_type num_rows; - }; - - extern "C" __device__ void transform_operator(operator_params const& p){ - using namespace jcudf; - - // unpack inputs from scope using the appropriate getters based on the LTO context - using s0 = scope::column<0, column_view, int, false, false>; - using s1 = scope::column<1, column_view, int, false, false>; - using s2 = scope::column<2, optional_span, int, false, false>; - - auto a0 = s0::element(p.scope, p.row_index); - auto a1 = s1::element(p.scope, p.row_index); - - int a2 = a0 + a1; - - s2::assign(p.scope, p.row_index, a2); - } - - extern "C" __global__ void transform_kernel(kernel_params params) - { - auto offset = static_cast(threadIdx.x) + static_cast(blockIdx.x) * static_cast(blockDim.x); - auto stride = static_cast(blockDim.x) * static_cast(gridDim.x); - - for(jcudf::i64 i = offset; i < params.num_rows; i += stride){ - operator_params p{params.scope, static_cast(i)}; - transform_operator(p); - } - } - )***"; - - /*udf = R"***( - #include "jcudf/functions.cuh" - - #pragma nv_hdrstop - - extern "C" __global__ void transform_kernel(){ - // empty kernel for testing purposes - } - - )***";*/ - static int i = 0; - - i++; - auto key = std::format("test_udf_key_{}", i); - - auto lib = cudf::compile_cuda_kernel("test_kernel", - key, - udf, - /*use_cache=*/true, - /*use_pch=*/true, - /*log_pch=*/true); - - auto kernel = lib->get_kernel("transform_kernel"); - - EXPECT_EQ("transform_kernel", kernel.get_name()); - - /* - auto in0 = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9}.release(); - auto in1 = cudf::test::fixed_width_column_wrapper{9, 8, 7, 6, 5, 4, 3, 2, 1}.release(); - auto out = cudf::test::fixed_width_column_wrapper{0, 0, 0, 0, 0, 0, 0, 0, 0}.release(); - int32_t num_rows = 9; - - auto to_device_view = [](auto const& view) { - std::vector h_view{view}; - rmm::device_uvector device_view(1, rmm::cuda_stream_default); - cudf::detail::cuda_memcpy_async( - device_view, h_view, rmm::cuda_stream_default); - return device_view; - }; - - auto to_device_mutable_view = [](auto& view) { - std::vector h_view{view}; - rmm::device_uvector device_view(1, - rmm::cuda_stream_default); - cudf::detail::cuda_memcpy_async( - device_view, h_view, rmm::cuda_stream_default); - return device_view; - }; - - auto h_in0 = cudf::column_device_view::create(in0->view()); - auto h_in1 = cudf::column_device_view::create(in1->view()); - auto h_out = cudf::mutable_column_device_view::create(out->mutable_view()); - auto d_in0 = to_device_view(*h_in0); - auto d_in1 = to_device_view(*h_in1); - auto d_out = to_device_mutable_view(*h_out); - auto d_in0_ptr = d_in0.data(); - auto d_in1_ptr = d_in1.data(); - auto d_out_ptr = d_out.data(); - - rmm::device_buffer d_scope{sizeof(cudf::column_device_view*) + - sizeof(cudf::column_device_view*) + - sizeof(cudf::mutable_column_device_view*), - rmm::cuda_stream_default}; - - auto* p = static_cast(d_scope.data()); - - detail::cuda_memcpy_async_impl(p, - &d_in0_ptr, - sizeof(cudf::column_device_view*), - detail::host_memory_kind::PAGEABLE, - rmm::cuda_stream_default); - detail::cuda_memcpy_async_impl(p + 1, - &d_in1_ptr, - sizeof(cudf::column_device_view*), - detail::host_memory_kind::PAGEABLE, - rmm::cuda_stream_default); - detail::cuda_memcpy_async_impl(p + 2, - &d_out_ptr, - sizeof(cudf::mutable_column_device_view*), - detail::host_memory_kind::PAGEABLE, - rmm::cuda_stream_default); - - auto* scope_arg = d_scope.data(); - - void* args[] = {&scope_arg, &num_rows}; - - kernel.launch(1, 1, 1, 256, 1, 1, 0, cudaStreamDefault, args); - - auto expected = - cudf::test::fixed_width_column_wrapper{10, 10, 10, 10, 10, 10, 10, 10, 10}.release(); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(out->view(), expected->view());*/ - }; - - fn(); // warm up cache - fn(); - fn(); - fn(); -} - -/* -TEST_F(RTCTest, CreateFragment) -{ - auto fn = []() { - char const udf[] = R"***( - #include "cudf/jit/lto/transform_params.cuh" - #include "cudf/jit/lto/operators.cuh" - #include "cudf/jit/lto/scope.cuh" - #include "cudf/jit/lto/column_view.cuh" // for column_device_view, mutable_column_device_view - - // if we detect that all types are simple types - // we can exclude some of the getters, setters, and operators - // for example, we have span, optional_span, and column_view_core - - extern "C" __device__ void transform_operator(cudf::lto::transform_params p){ - using namespace cudf::lto; - using ops = operators; - - /// <-- BEGIN OF INPUT UNPACKING: Defined by input planner - - // unpack inputs from scope using the appropriate getters based on the LTO context - using s0 = scope::user_data<0>; - using s1 = scope::column<1, column_device_view, int, false, false>; - using s2 = scope::column<2, column_device_view, int, false, false>; - using s3 = scope::column<3, column_device_view, double, false, false>; - using s4 = scope::column<4, column_device_view, float, false, false>; - using s4 = scope::column<4, column_device_view, string_view, false, false>; - using s4 = scope::column<4, column_device_view, decimal32, false, false>; - using s4 = scope::column<5, column_device_view, float, false, true>; - using s4 = scope::column<6, column_device_view, float, true, false>; - using s5 = scope::column<7, span, float, false, false>; - using s6 = scope::column<8, optional_span, float, false, true>; - using s7 = scope::column<9, mutable_column_device_view, double, false, false>; - - auto a0 = s0::element(p.scope, p.row_index); - auto a1 = s1::element(p.scope, p.row_index); - auto a2 = s2::element(p.scope, p.row_index); - auto a3 = s3::element(p.scope, p.row_index); - auto a4 = s4::element(p.scope, p.row_index); - auto a5 = s5::element(p.scope, p.row_index); - auto a6 = s6::element(p.scope, p.row_index); - auto a7 = s7::element(p.scope, p.row_index); - auto a7 = s8::element(p.scope, p.row_index); - auto a7 = s9::element(p.scope, p.row_index); - - auto result = 0.0; - - /// <-- END OF INPUT UNPACKING - - - /// <-- BEGIN OF OPERATOR: Derived from user - - // run operation using the LTO-compiled operators; these should be inlined into the final -kernel and optimized together by NVJITLink - - ops::add(&c, &a, &b); - ops::sub(&c, &a, &b); - ops::mul(&c, &a, &b); - ops::mul(&c, &a, &b); - ops::arctan(&c, &a, &b); - ops::sqrt(&c, &a, &b); - ops::cbrt(&c, &a, &b); - ops::arccos(&c, &a, &b); - ops::cast_to_float64(&d, &c); - - /// <-- END OF USER-DEFINED OPERATOR - - - /// <-- BEGIN OF OUTPUT PACKING: Defined by planner - - // write output to global memory - - out_col::assign(p.scope, p.row_index, result); - - - /// <-- END OF OUTPUT PACKING - } - )***"; - - auto params = cudf::udf_compile_params{.name = "test_fragment", - .udf = udf, - .key = "test_udf_key", - .kernel_symbol = "transform_kernel", - .extra_compile_flags = {}, - .extra_link_flags = {}}; - - auto lib = cudf::compile_and_link_cuda_udf(params); - - auto kernel = lib->get_kernel("transform_kernel"); - - EXPECT_EQ("transform_kernel", kernel.get_name()); - // void** scope = nullptr; - // int32_t num_rows = 0; - // void* args[] = {&scope, &num_rows}; - // kernel.launch(1, 1, 1, 1, 1, 1, 0, nullptr, args); - }; - - fn(); - fn(); - fn(); -} -*/ - -CUDF_TEST_PROGRAM_MAIN() From 4bb406a0290b48d9790462e9033e472ec5ecda53 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 2 Mar 2026 21:43:41 +0000 Subject: [PATCH 064/254] Refactor JIT compilation options: streamline precompiled header usage and add new compiler flags --- cpp/src/jit/jit.cpp | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index 9fc1bb9fbf48..40aab0f11240 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -286,20 +286,13 @@ std::tuple compile_library_uncached(char const* name, options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); options.emplace_back("--minimal"); - - auto pch_file = std::format("/home/coder/cudf/jit.pch"); - - static int can_use = 0; + options.emplace_back("-D__CUDACC_RTC__"); + options.emplace_back("-DCUDF_RUNTIME_JIT"); + options.emplace_back("--diag-suppress=47"); + options.emplace_back("--device-int128"); if (use_pch) { - // options.emplace_back("--pch"); - options.emplace_back(std::format("--pch-dir={}", pch_dir)); - if (can_use) { - options.emplace_back(std::format("--use-pch={}", pch_file)); - } else { - options.emplace_back(std::format("--create-pch={}", pch_file)); - } - can_use = 1; + options.emplace_back("--pch"); if (log_pch) { options.emplace_back("--pch-verbose=true"); From e53bf52c02032ecce576f66d24cbc32582d10f69 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 2 Mar 2026 21:44:10 +0000 Subject: [PATCH 065/254] Remove TODO comment regarding CMake tracking in jit_bundle_t constructor --- cpp/src/jit/jit.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index 40aab0f11240..b095de660407 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -176,7 +176,6 @@ jit_bundle_t::jit_bundle_t(std::string install_dir, rtcx::cache_t& cache) : install_dir_{std::move(install_dir)}, cache_{&cache} { ensure_installed(); - // TODO: fix cmake tracking of the scripts and embedded files } void jit_bundle_t::ensure_installed() const From bdc6597366f41ef1f85c4b2e8cbfa54521c54657 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 2 Mar 2026 21:47:00 +0000 Subject: [PATCH 066/254] Update log message to specify compiled CUDA library in compile_library_uncached function --- cpp/src/jit/jit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index b095de660407..b89db8e36875 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -317,7 +317,7 @@ std::tuple compile_library_uncached(char const* name, auto duration = end - begin; CUDF_LOG_INFO( - "Compiled fragment `%s` in %f ms", + "Compiled CUDA library `%s` in %f ms", name, std::chrono::duration_cast>(duration).count()); From 6c9be74ad8f442a1f6b212df9c78609a47892da2 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 7 Mar 2026 19:42:51 +0000 Subject: [PATCH 067/254] [FEA] Support Multi-Output JIT Transforms --- .../cudf/column/column_device_view.cuh | 12 + .../cudf/column/column_device_view_base.cuh | 61 ++ .../cudf/strings/detail/strings_children.cuh | 43 +- .../detail/strings_column_factories.cuh | 39 + cpp/include/cudf/transform.hpp | 52 +- cpp/src/jit/column_accessor.cuh | 109 +++ cpp/src/jit/helpers.cpp | 51 +- cpp/src/jit/helpers.hpp | 39 +- ...accessors.cuh => join_column_accessor.cuh} | 118 --- cpp/src/jit/span.cuh | 70 -- cpp/src/jit/transform_udf.cuh | 112 +++ cpp/src/jit/type_list.cuh | 36 + cpp/src/jit/wrappers.cuh | 120 +++ cpp/src/transform/jit/kernel.cu | 154 +--- cpp/src/transform/transform.cu | 709 ++++++++---------- 15 files changed, 955 insertions(+), 770 deletions(-) create mode 100644 cpp/src/jit/column_accessor.cuh rename cpp/src/jit/{accessors.cuh => join_column_accessor.cuh} (57%) create mode 100644 cpp/src/jit/transform_udf.cuh create mode 100644 cpp/src/jit/type_list.cuh create mode 100644 cpp/src/jit/wrappers.cuh diff --git a/cpp/include/cudf/column/column_device_view.cuh b/cpp/include/cudf/column/column_device_view.cuh index 1d218dc0a870..32225aaed0e2 100644 --- a/cpp/include/cudf/column/column_device_view.cuh +++ b/cpp/include/cudf/column/column_device_view.cuh @@ -749,6 +749,18 @@ class alignas(16) mutable_column_device_view : public mutable_column_device_view */ static std::size_t extent(mutable_column_view source_view); + static mutable_column_device_view from_parts(data_type type, + size_type size, + void const* data, + bitmask_type const* null_mask, + size_type offset, + mutable_column_device_view* children, + size_type num_children) + { + return mutable_column_device_view{ + mutable_column_device_view_core{type, size, data, null_mask, offset, children, num_children}}; + } + /** * @brief Destroy the `mutable_column_device_view` object. * diff --git a/cpp/include/cudf/column/column_device_view_base.cuh b/cpp/include/cudf/column/column_device_view_base.cuh index 3e9130085c7e..639602e778fb 100644 --- a/cpp/include/cudf/column/column_device_view_base.cuh +++ b/cpp/include/cudf/column/column_device_view_base.cuh @@ -325,6 +325,8 @@ struct mutable_value_accessor; */ class alignas(16) column_device_view_core : public detail::column_device_view_base { public: + static constexpr bool is_mutable = false; + column_device_view_core() = delete; ~column_device_view_core() = default; column_device_view_core(column_device_view_core const&) = default; ///< Copy constructor @@ -447,6 +449,14 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba return T{scaled_integer{data()[element_index], scale}}; } + template + [[nodiscard]] __device__ cuda::std::optional nullable_element( + size_type element_index) const noexcept + { + if (is_null(element_index)) { return cuda::std::nullopt; } + return element(element_index); + } + /** * @brief Returns the specified child * @@ -521,6 +531,8 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba */ class alignas(16) mutable_column_device_view_core : public detail::column_device_view_base { public: + static constexpr bool is_mutable = true; + mutable_column_device_view_core() = delete; ~mutable_column_device_view_core() = default; mutable_column_device_view_core(mutable_column_device_view_core const&) = @@ -601,6 +613,55 @@ class alignas(16) mutable_column_device_view_core : public detail::column_device return data()[element_index]; } + /** + * @brief Returns `string_view` to the string element at the specified index. + * + * If the element at the specified index is NULL, i.e., `is_null(element_index) + * == true`, then any attempt to use the result will lead to undefined behavior. + * + * This function accounts for the offset. + * + * @param element_index Position of the desired string element + * @return string_view instance representing this element at this index + */ + template )> + [[nodiscard]] __device__ T element(size_type element_index) const noexcept + { + size_type index = element_index + offset(); // account for this view's _offset + char const* d_strings = static_cast(_data); + auto const offsets = child(offsets_column_index); + auto const itr = cudf::detail::input_offsetalator(offsets.head(), offsets.type()); + auto const offset = itr[index]; + return string_view{d_strings + offset, static_cast(itr[index + 1] - offset)}; + } + + /** + * @brief Returns a `numeric::fixed_point` element at the specified index for a `fixed_point` + * column. + * + * If the element at the specified index is NULL, i.e., `is_null(element_index) == true`, + * then any attempt to use the result will lead to undefined behavior. + * + * @param element_index Position of the desired element + * @return numeric::fixed_point representing the element at this index + */ + template ())> + [[nodiscard]] __device__ T element(size_type element_index) const noexcept + { + using namespace numeric; + using rep = typename T::rep; + auto const scale = scale_type{_type.scale()}; + return T{scaled_integer{data()[element_index], scale}}; + } + + template + [[nodiscard]] __device__ cuda::std::optional nullable_element( + size_type element_index) const noexcept + { + if (is_null(element_index)) { return cuda::std::nullopt; } + return element(element_index); + } + /** * @brief Assigns `value` to the element at `element_index` * diff --git a/cpp/include/cudf/strings/detail/strings_children.cuh b/cpp/include/cudf/strings/detail/strings_children.cuh index 9369c43d3061..f691145f5445 100644 --- a/cpp/include/cudf/strings/detail/strings_children.cuh +++ b/cpp/include/cudf/strings/detail/strings_children.cuh @@ -72,12 +72,8 @@ rmm::device_uvector make_chars_buffer(column_view const& offsets, auto const src_ptrs = thrust::make_transform_iterator( thrust::make_counting_iterator(0), - cuda::proclaim_return_type([begin] __device__(uint32_t idx) { - // Due to a bug in cub (https://github.com/NVIDIA/cccl/issues/586), - // we have to use `const_cast` to remove `const` qualifier from the source pointer. - // This should be fine as long as we only read but not write anything to the source. - return reinterpret_cast(const_cast(begin[idx].first)); - })); + cuda::proclaim_return_type( + [begin] __device__(uint32_t idx) -> void const* { return begin[idx].first; })); auto const src_sizes = thrust::make_transform_iterator( thrust::make_counting_iterator(0), cuda::proclaim_return_type( @@ -102,6 +98,41 @@ rmm::device_uvector make_chars_buffer(column_view const& offsets, return chars_data; } +rmm::device_uvector make_chars_buffer(column_view const& offsets_view, + int64_t chars_size, + string_view const* begin, + bitmask_type const* stencil, + size_type size, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + auto offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets_view); + auto chars = rmm::device_uvector(chars_size, stream, mr); + + auto srcs = make_counting_transform_iterator( + size_type{0}, [begin] __device__(size_type idx) -> void const* { return begin[idx].data(); }); + + auto src_sizes = make_counting_transform_iterator( + size_type{0}, [begin, stencil] __device__(size_type idx) -> size_type { + if (stencil != nullptr && !bit_is_set(stencil, idx)) { return 0; } + return static_cast(begin[idx].size_bytes()); + }); + + auto dsts = make_counting_transform_iterator( + size_type{0}, [offsets, chars = chars.data()] __device__(size_type idx) -> void* { + return chars + offsets[idx]; + }); + + size_t temp_storage_bytes = 0; + CUDF_CUDA_TRY(cub::DeviceMemcpy::Batched( + nullptr, temp_storage_bytes, srcs, dsts, src_sizes, size, stream.value())); + rmm::device_buffer d_temp_storage(temp_storage_bytes, stream); + CUDF_CUDA_TRY(cub::DeviceMemcpy::Batched( + d_temp_storage.data(), temp_storage_bytes, srcs, dsts, src_sizes, size, stream.value())); + + return chars; +} + /** * @brief Create an offsets column to be a child of a compound column * diff --git a/cpp/include/cudf/strings/detail/strings_column_factories.cuh b/cpp/include/cudf/strings/detail/strings_column_factories.cuh index 6eafcbb636dd..f7e30d5901bb 100644 --- a/cpp/include/cudf/strings/detail/strings_column_factories.cuh +++ b/cpp/include/cudf/strings/detail/strings_column_factories.cuh @@ -84,6 +84,45 @@ std::unique_ptr make_strings_column(IndexPairIterator begin, std::move(null_mask)); } +std::unique_ptr make_strings_column(device_span strings, + rmm::device_buffer null_mask, + std::optional null_count, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto size = static_cast(strings.size()); + if (size == 0) return make_empty_column(type_id::STRING); + + // build offsets column from the strings sizes + auto sizes = thrust::make_counting_transform_iterator( + cudf::size_type{0}, + [stencil = null_mask.data(), + strings = strings.data()] __device__(cudf::size_type index) -> size_type { + if (stencil != nullptr && !bit_is_set(stencil, index)) { return size_type{0}; } + return static_cast(strings[index].size_bytes()); + }); + + auto [offsets, bytes] = + cudf::strings::detail::make_offsets_child_column(sizes, sizes + size, stream, mr); + + auto final_null_count = size_type{0}; + + if (!null_count.has_value()) { + final_null_count = + null_mask.empty() ? 0 + : cudf::detail::count_set_bits(null_mask.data(), 0, strings.size(), stream); + } else { + final_null_count = null_count.value(); + } + + auto chars = + make_chars_buffer(offsets->view(), bytes, strings.data(), null_mask.data(), size, stream, mr); + + return make_strings_column( + size, std::move(offsets), chars.release(), final_null_count, std::move(null_mask)); +} + /** * @brief Create a strings-type column from iterators to chars, offsets, and bitmask. * diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index e3830dd0a6cd..efa3f8e4b7e5 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -71,7 +71,12 @@ namespace CUDF_EXPORT cudf { * @brief Typedef for inputs to the transform function. Each input can be either a column or a * scalar column. */ -using transform_input = std::variant; +using transform_input = std::variant; + +struct transform_output { + data_type type = data_type{type_id::EMPTY}; + output_nullability nullability = output_nullability::PRESERVE; +}; /** * @brief Creates a new column by applying a transform function against every @@ -84,11 +89,9 @@ using transform_input = std::variant; * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string * types - * @throws std::invalid_argument if any of the input columns have nulls * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in * this case. - * @throws std::logic_error if JIT is not supported by the runtime * * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from * the input columns. @@ -119,6 +122,49 @@ std::unique_ptr transform_extended( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +/** + * @brief Creates a new column by applying a transform function against every + * element of the input columns. + * + * Computes: + * `UDF(&outputs[i]..., inputs[i]...)`. + * + * + * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) + * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string + * types + * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and + * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in + * this case. + * + * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from + * the input and pre-allocated output columns. + * + * @param udf The PTX/CUDA string of the transform function to apply + * @param source_type The source type of the UDF (CUDA or PTX) + * @param is_null_aware Signifies the UDF will receive row inputs as optional values + * @param row_size The row size of the transform operation. If not provided, it is inferred from the + * input columns. + * @param user_data User-defined device data to pass to the UDF. + * @param inputs Immutable views of the inputs to transform (columns and scalar columns) + * @param outputs Specification of the output columns to be created + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + * @return A vector of columns resulting from applying the transform function to + * every element of the input. These columns are returned in the same order as + * the `output_types`. + * + */ +std::vector> transform_extended2(std::string const& udf, + udf_source_type source_type, + null_aware is_null_aware, + std::optional row_size, + std::optional user_data, + std::span inputs, + std::span outputs, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); + /** * @brief Creates a null_mask from `input` by converting `NaN` to null and * preserving existing null values and also returns new null_count. diff --git a/cpp/src/jit/column_accessor.cuh b/cpp/src/jit/column_accessor.cuh new file mode 100644 index 000000000000..65876de4f6f6 --- /dev/null +++ b/cpp/src/jit/column_accessor.cuh @@ -0,0 +1,109 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include + +namespace cudf { +namespace jit { + +template +struct column_accessor { + static constexpr int32_t index = Index; + using column_type = Column; + using element_type = Element; + using optional_element_type = OptionalElement; + static constexpr bool as_scalar = AsScalar; + static constexpr bool may_be_nullable = MayBeNullable; + + static __device__ element_type element(column_type const* cols, size_type row) + { + auto& c = cols[index]; + + if constexpr (AsScalar) { + return c.template element(0); + } else { + return c.template element(row); + } + } + + static __device__ bool is_null(column_type const* cols, size_type row) + { + if constexpr (!MayBeNullable) { + return false; + } else { + auto& c = cols[index]; + + if constexpr (AsScalar) { + return c.is_null(0); + } else { + return c.is_null(row); + } + } + } + + static __device__ bool is_valid(column_type const* cols, size_type row) + { + if constexpr (!MayBeNullable) { + return true; + } else { + auto& c = cols[index]; + + if constexpr (AsScalar) { + return c.is_valid(0); + } else { + return c.is_valid(row); + } + } + } + + static __device__ optional_element_type nullable_element(column_type const* cols, size_type row) + { + auto& c = cols[index]; + + if constexpr (!MayBeNullable) { + return c.template element(row); + } else { + if constexpr (AsScalar) { + return c.template nullable_element(0); + } else { + return c.template nullable_element(row); + } + } + } + + static __device__ void set_null_word(column_type const* cols, size_type index, bitmask_type word) + { + auto& c = cols[index]; + + if constexpr (!MayBeNullable) { + return; + } else { + auto* mask = c.null_mask(); + + if (mask == nullptr) { return; } + + mask[index] = word; + } + } + + static __device__ void assign(column_type const* cols, size_type row, element_type value) + requires(!AsScalar) + { + cols[index].template assign(row, value); + } +}; + +} // namespace jit +} // namespace cudf diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 9541950ad785..4967ebfdc891 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -52,16 +52,6 @@ size_type get_projection_size(std::span build_ptx_params(std::span output_typenames, std::span input_typenames, bool has_user_data) @@ -85,51 +75,38 @@ std::map build_ptx_params(std::span ou return params; } -std::vector output_type_names(std::span views) -{ - std::vector names; - - std::transform(views.begin(), views.end(), std::back_inserter(names), [](auto const& view) { - return type_to_name(view.type()); - }); - - return names; -} - std::vector input_type_names( std::span const> views) { std::vector names; - auto get_type_name = [](auto const& var) { - return std::visit([](auto& a) { return type_to_name(a.type()); }, var); - }; std::transform(views.begin(), views.end(), std::back_inserter(names), [&](auto const& view) { - return get_type_name(view); + return std::visit([](auto& a) { return type_to_name(a.type()); }, view); }); return names; } -input_reflection reflect_input(std::variant const& input) +std::string reflect_input_accessor(std::variant const& input, + int32_t index) { - auto get_type_name = [](auto const& var) { - return std::visit([](auto& a) { return type_to_name(a.type()); }, var); - }; + auto element = std::visit([](auto& a) { return type_to_name(a.type()); }, input); + bool as_scalar = std::holds_alternative(input); - return input_reflection{get_type_name(input), std::holds_alternative(input)}; + return jitify2::Template("cudf::jit::column_accessor") + .instantiate("cudf::column_device_view_core", element, as_scalar); } -std::vector reflect_inputs( +std::vector reflect_input_accessors( std::span const> inputs) { - std::vector reflections; - std::transform( - inputs.begin(), inputs.end(), std::back_inserter(reflections), [&](auto const& view) { - return reflect_input(view); - }); + std::vector res; + std::transform(thrust::counting_iterator(0), + thrust::counting_iterator(inputs.size()), + std::back_inserter(res), + [&](auto i) { return reflect_input_accessor(inputs[i], i); }); - return reflections; + return res; } jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index 5580d6a5463c..b52a253ea32f 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -32,13 +32,6 @@ get_transform_base_column(std::vector const& inputs); size_type get_projection_size( std::span const> inputs); -struct input_reflection { - std::string type_name; - bool is_scalar = false; - - [[nodiscard]] std::string accessor(int32_t index) const; -}; - std::map build_ptx_params(std::span output_typenames, std::span input_typenames, bool has_user_data); @@ -78,15 +71,35 @@ column_views_to_device(std::span views, return std::make_tuple(std::move(handles), std::move(device_array)); } -std::vector output_type_names(std::span views); - std::vector input_type_names( std::span const> views); -input_reflection reflect_input(std::variant const& input); - -std::vector reflect_inputs( - std::span const> inputs); +// TODO: how do we support mutable column view?, the arguments need to be casted +/** + * @brief Reflects the input column or scalar into a string that can be used in the generated CUDA + * code. + * + * @param index The index of the input in the list of inputs. + * @param input The input column or scalar to reflect. + * @param may_be_nullable Whether to allow runtime checks of the nullability of the input + * @return A string representing the input in the generated CUDA code. + */ +std::string reflect_input( + int32_t index, + std::variant const& input, + bool may_be_nullable); + +/** + * @brief Reflects the input columns or scalars into strings that can be used in the generated CUDA + * code. + * + * @param inputs The input columns or scalars to reflect. + * @param may_be_nullable Whether to allow runtime checks of the nullability of the inputs + * @return A vector of strings representing the inputs in the generated CUDA code. + */ +std::vector reflect_inputs( + std::span const> inputs, + std::span may_be_nullable); jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, std::string const& kernel_name, diff --git a/cpp/src/jit/accessors.cuh b/cpp/src/jit/join_column_accessor.cuh similarity index 57% rename from cpp/src/jit/accessors.cuh rename to cpp/src/jit/join_column_accessor.cuh index 148bd8cb369e..2135bad952ba 100644 --- a/cpp/src/jit/accessors.cuh +++ b/cpp/src/jit/join_column_accessor.cuh @@ -10,127 +10,9 @@ #include #include -#include - -#include - namespace cudf { namespace jit { -template -struct column_accessor { - using type = T; - static constexpr int32_t index = Index; - - template - static __device__ decltype(auto) element(ColumnView const* columns, cudf::size_type row) - { - return columns[index].template element(row); - } - - static __device__ void assign(cudf::mutable_column_device_view_core const* outputs, - cudf::size_type row, - T value) - { - outputs[index].assign(row, value); - } - - template - static __device__ bool is_null(ColumnView const* inputs, cudf::size_type row) - { - return inputs[index].is_null(row); - } - - template - static __device__ bool is_valid(ColumnView const* inputs, cudf::size_type row) - { - return inputs[index].is_valid(row); - } - - template - static __device__ cuda::std::optional nullable_element(ColumnView const* columns, - cudf::size_type row) - { - if (is_null(columns, row)) { return cuda::std::nullopt; } - return columns[index].template element(row); - } -}; - -template -struct span_accessor { - using type = T; - static constexpr int32_t index = Index; - - static __device__ type& element(cudf::jit::device_optional_span const* spans, - cudf::size_type row) - { - return spans[index][row]; - } - - static __device__ void assign(cudf::jit::device_optional_span const* outputs, - cudf::size_type row, - T value) - { - outputs[index][row] = value; - } - - static __device__ bool is_null(cudf::jit::device_optional_span const* inputs, - cudf::size_type row) - { - return inputs[index].is_null(row); - } - - static __device__ bool is_valid(cudf::jit::device_optional_span const* inputs, - cudf::size_type row) - { - return inputs[index].is_valid(row); - } - - static __device__ cuda::std::optional nullable_element( - cudf::jit::device_optional_span const* outputs, cudf::size_type row) - { - if (is_null(outputs, row)) { return cuda::std::nullopt; } - return outputs[index].element(row); - } -}; - -template -struct scalar_accessor { - using type = typename Accessor::type; - static constexpr int32_t index = Accessor::index; - - template - static __device__ decltype(auto) element(ColumnView const* columns, cudf::size_type) - { - return Accessor::element(columns, 0); - } - - static __device__ void assign(cudf::mutable_column_device_view_core const* outputs, - cudf::size_type, - type value) - { - return Accessor::assign(outputs, 0, value); - } - - template - static __device__ bool is_null(ColumnView const* columns, cudf::size_type) - { - return Accessor::is_null(columns, 0); - } - - template - static __device__ bool is_valid(ColumnView const* columns, cudf::size_type) - { - return Accessor::is_valid(columns, 0); - } - - template - static __device__ decltype(auto) nullable_element(ColumnView const* columns, cudf::size_type) - { - return Accessor::nullable_element(columns, 0); - } -}; - // Join-specific accessor for indexed table access. // Receives both left and right table pointers plus both row indices, // and selects the appropriate table based on the Side template parameter. diff --git a/cpp/src/jit/span.cuh b/cpp/src/jit/span.cuh index 3171d575dd29..6d2fbe7411a1 100644 --- a/cpp/src/jit/span.cuh +++ b/cpp/src/jit/span.cuh @@ -92,76 +92,6 @@ struct device_span { } }; -/** - * @brief A span type with optional/nullable elements. - * - * Optional implies the span contains nullable elements. - * The nullability of the elements is internally represented by an optional bitmask which can be - * nullptr when all the elements are non-null. - */ -template -struct device_optional_span : device_span { - private: - using base = device_span; - bitmask_type* _null_mask = nullptr; - - public: - CUDF_HOST_DEVICE constexpr device_optional_span() {} - - /** - * @brief Constructs an optional span from a span and a null-mask. - * - * @param span Span containing the elements - * @param null_mask The null-mask determining the validity of the elements or nullptr if all - * valid. - */ - CUDF_HOST_DEVICE device_optional_span(device_span span, bitmask_type* null_mask) - : base{span}, _null_mask{null_mask} - { - } - - /// @copydoc column_device_view::nullable - [[nodiscard]] CUDF_HOST_DEVICE bool nullable() const { return _null_mask != nullptr; } - -#ifdef __CUDACC__ - - /// @copydoc column_device_view::is_valid_nocheck - [[nodiscard]] __device__ bool is_valid_nocheck(size_t element_index) const - { - return bit_is_set(_null_mask, element_index); - } - - /// @copydoc column_device_view::is_valid - [[nodiscard]] __device__ bool is_valid(size_t element_index) const - { - return not nullable() or is_valid_nocheck(element_index); - } - - /// @copydoc column_device_view::is_null - [[nodiscard]] __device__ bool is_null(size_t element_index) const - { - return !is_valid(element_index); - } - - CUDF_HOST_DEVICE constexpr T& element(size_t idx) const { return base::operator[](idx); } - - /// @copydoc column_device_view::element - __device__ void set_valid(size_type element_index) const noexcept - { - return set_bit(_null_mask, element_index); - } - - /// @copydoc column_device_view::set_null - __device__ void set_null(size_type element_index) const noexcept - { - return clear_bit(_null_mask, element_index); - } - - /// @brief converts the optional span to a regular non-nullable span. - [[nodiscard]] __device__ base to_span() const noexcept { return static_cast(*this); } - -#endif -}; } // namespace jit } // namespace cudf diff --git a/cpp/src/jit/transform_udf.cuh b/cpp/src/jit/transform_udf.cuh new file mode 100644 index 000000000000..af8f08d1e3cd --- /dev/null +++ b/cpp/src/jit/transform_udf.cuh @@ -0,0 +1,112 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include +#include + +#include +#include +#include + +#include + +namespace cudf { +namespace jit { + +/** + * @tparam is_null_aware = whether the UDF is null-aware or not. If `YES`, the UDF will receive + * nullable elements for each input column, and the UDF will determine the nullability of the output + * columns. If `NO`, the UDF will receive elements for each input column, and the UDF will not + * handle null masks for the output columns. + * @tparam has_stencil whether the operation has stencil values or not. If true, the kernel will + * receive a bitmask pointer for the stencil and will skip rows where the stencil bit is not set. + * @tparam has_user_data whether the UDF has user data or not. If true, the UDF will receive a + * pointer to the user data as the first argument, followed by the row index. + * @tparam Ins = type_list + * @tparam Outs = type_list + */ +template +struct transform_udf { + template + static __device__ void call(Fn&& udf, + size_type index, + void* user_data, + column_device_view_core const* in_cols, + bitmask_type const* stencil, + mutable_column_device_view_core const* out_cols, + [[maybe_unused]] bool* is_valid) + requires(is_null_aware == null_aware::NO) + { + if constexpr (has_stencil) { + if (stencil != nullptr && !bit_is_set(stencil, index)) { return; } + } + + auto outs = + Outs::map([]() { return cuda::std::tuple{typename A::element_type{}...}; }); + + auto out_ptrs = + cuda::std::apply([&](auto&&... args) { return cuda::std::tuple{&args...}; }, outs); + + auto inputs = + Ins::map([&]() { return cuda::std::tuple{A::element(in_cols, index)...}; }); + + if constexpr (has_user_data) { + auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, index}, out_ptrs, inputs); + cuda::std::apply(udf, args); + + } else { + auto args = cuda::std::tuple_cat(out_ptrs, inputs); + cuda::std::apply(udf, args); + } + + [&](cuda::std::integer_sequence) { + (Outs::at::assign(out_cols, index, cuda::std::get(outs)), ...); + }(Outs::indexed); + } + + template + static __device__ void call(Fn&& udf, + size_type index, + void* user_data, + column_device_view_core const* in_cols, + [[maybe_unused]] bitmask_type const* stencil, + mutable_column_device_view_core const* out_cols, + bool* is_valid) + requires(is_null_aware == null_aware::YES) + { + auto outs = Outs::map( + []() { return cuda::std::tuple{typename A::optional_element_type{}...}; }); + + auto out_ptrs = + cuda::std::apply([&](auto&&... args) { return cuda::std::tuple{&args...}; }, outs); + + auto inputs = Ins::map( + [&]() { return cuda::std::tuple{A::nullable_element(in_cols, index)...}; }); + + if constexpr (has_user_data) { + auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, index}, out_ptrs, inputs); + cuda::std::apply(udf, args); + + } else { + auto args = cuda::std::tuple_cat(out_ptrs, inputs); + cuda::std::apply(udf, args); + } + + [&](cuda::std::integer_sequence) { + (Outs::at::assign(out_cols, index, cuda::std::get(outs)), ...); + ((is_valid[I] = cuda::std::get(outs).has_value()), ...); + }(Ins::indexed); + } +}; + +} // namespace jit +} // namespace cudf diff --git a/cpp/src/jit/type_list.cuh b/cpp/src/jit/type_list.cuh new file mode 100644 index 000000000000..a435cb4fe72f --- /dev/null +++ b/cpp/src/jit/type_list.cuh @@ -0,0 +1,36 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include + +namespace cudf { +namespace jit { + +/** + * @brief A list of types with some helper functions to operate on them. + * + */ +template +struct type_list { + static constexpr int size = sizeof...(T); + + using tuple = cuda::std::tuple; + + template + using at = cuda::std::tuple_element_t; + + static constexpr cuda::std::make_integer_sequence indexed{}; + + template + static constexpr __device__ decltype(auto) map(Fn& fn) + { + return fn.template operator()(); + } +}; + +} // namespace jit +} // namespace cudf diff --git a/cpp/src/jit/wrappers.cuh b/cpp/src/jit/wrappers.cuh new file mode 100644 index 000000000000..ae6feb304d9b --- /dev/null +++ b/cpp/src/jit/wrappers.cuh @@ -0,0 +1,120 @@ + +#pragma once +#include +#include + +#include +#include + +namespace CUDF_EXPORT cudf { +namespace jit { + +/** + * @brief A column wrapper type that treats a column as a span of elements. This treats the element + * as a contiguous sequence of elements. + * + */ +template +struct span_column : private Column { + using base = Column; + + CUDF_HOST_DEVICE constexpr span_column(Column const& src) : base{src} {} + ~span_column() = default; + span_column(span_column const&) = default; + span_column(span_column&&) = default; + span_column& operator=(span_column const&) = default; + span_column& operator=(span_column&&) = default; + + using base::nullable; + using base::offset; + using base::size; + using base::type; + + template + CUDF_HOST_DEVICE T const* data() const noexcept + requires(!base::is_mut) + { + return static_cast(_data) + _offset; + } + + template + CUDF_HOST_DEVICE T* data() const noexcept + requires(base::is_mut) + { + return static_cast(_data) + _offset; + } + + using base::is_null; + using base::is_valid; + using base::null_mask; + + template + [[nodiscard]] __device__ decltype(auto) element(size_type element) const noexcept + { + return data()[element]; + } + + template + [[nodiscard]] __device__ cuda::std::optional nullable_element(size_type element) const noexcept + { + if (is_null(element)) { return cuda::std::nullopt; } + return element(element); + } + + CUDF_HOST_DEVICE Column& to_base() { return static_cast(*this); } + + CUDF_HOST_DEVICE Column const& to_base() const { return static_cast(*this); } +}; + +/** + * @brief A column wrapper type that treats a column as a column of mutable strings. + * + */ +template +struct mut_string_column : private Column { + using base = Column; + + CUDF_HOST_DEVICE constexpr mut_string_column(Column const& src) : base{src} {} + + ~mut_string_column() = default; + mut_string_column(mut_string_column const&) = default; + mut_string_column(mut_string_column&&) = default; + mut_string_column& operator=(mut_string_column const&) = default; + mut_string_column& operator=(mut_string_column&&) = default; + + using base::is_null; + using base::is_valid; + using base::nullable; + using base::offset; + using base::size; + using base::type; + + template > + [[nodiscard]] __device__ cuda::std::span element(size_type element) const noexcept + requires(base::is_mut && cuda::std::is_same_v>) + { + auto index = element + offset(); + auto chars = static_cast(_data); + auto offsets = child(offsets_column_index); + auto itr = cudf::detail::input_offsetalator(offsets.head(), offsets.type()); + auto offset = itr[index]; + return cuda::std::span{chars + offset, + static_cast(itr[index + 1] - offset)}; + } + + template > + [[nodiscard]] __device__ cuda::std::optional> nullable_element( + size_type element) const noexcept + requires(base::is_mut && cuda::std::is_same_v>) + { + if (is_null(element)) { return cuda::std::nullopt; } + return element(element); + } + + CUDF_HOST_DEVICE Column& to_base() { return static_cast(*this); } + + CUDF_HOST_DEVICE Column const& to_base() const { return static_cast(*this); } +}; + +} // namespace jit +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 4a7cb9c9af12..af4d8e8416a1 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -15,6 +15,8 @@ #include #include +#include +#include #pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before // it. Each UDF will have a different operation-udf.hpp generated for it, so we @@ -27,143 +29,47 @@ // clang-format on namespace cudf { -namespace transformation { namespace jit { - -template -CUDF_KERNEL void kernel(cudf::mutable_column_device_view_core const* outputs, - cudf::column_device_view_core const* inputs, - bool* intermediate_null_mask, - void* user_data) -{ - // inputs to JITIFY kernels have to be either sized-integral types or pointers. Structs or - // references can't be passed directly/correctly as they will be crossing an ABI boundary - - auto const start = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); - auto const size = outputs[0].size(); - - for (auto i = start; i < size; i += stride) { - if constexpr (is_null_aware == null_aware::NO) { - if constexpr (may_evaluate_null) { - if (Out::is_null(outputs, i)) { continue; } - } - - if constexpr (has_user_data) { - GENERIC_TRANSFORM_OP(user_data, i, &Out::element(outputs, i), In::element(inputs, i)...); - } else { - GENERIC_TRANSFORM_OP(&Out::element(outputs, i), In::element(inputs, i)...); - } - - } else { // is_null_aware == null_aware::YES - cuda::std::optional result; - - if constexpr (has_user_data) { - GENERIC_TRANSFORM_OP(user_data, i, &result, In::nullable_element(inputs, i)...); - } else { - GENERIC_TRANSFORM_OP(&result, In::nullable_element(inputs, i)...); - } - - Out::assign(outputs, i, *result); - - if constexpr (may_evaluate_null) { intermediate_null_mask[i] = result.has_value(); } - } - } -} - -template -CUDF_KERNEL void fixed_point_kernel(cudf::mutable_column_device_view_core const* outputs, - cudf::column_device_view_core const* inputs, - bool* intermediate_null_mask, - void* user_data) +namespace { +template +__device__ void warp_compact_validity(mutable_column_device_view_core const* outputs, + size_type row, + bool is_valid) { - auto const start = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); - auto const size = outputs[0].size(); - auto const output_scale = static_cast(outputs[0].type().scale()); - - for (auto i = start; i < size; i += stride) { - if constexpr (is_null_aware == null_aware::NO) { - if constexpr (may_evaluate_null) { - if (Out::is_null(outputs, i)) { continue; } - } - - typename Out::type result{numeric::scaled_integer{0, output_scale}}; - - if constexpr (has_user_data) { - GENERIC_TRANSFORM_OP(user_data, i, &result, In::element(inputs, i)...); - } else { - GENERIC_TRANSFORM_OP(&result, In::element(inputs, i)...); - } - - Out::assign(outputs, i, result); - - } else { // is_null_aware == null_aware::YES - cuda::std::optional result{ - typename Out::type{numeric::scaled_integer{0, output_scale}}}; - - if constexpr (has_user_data) { - GENERIC_TRANSFORM_OP(user_data, i, &result, In::nullable_element(inputs, i)...); - } else { - GENERIC_TRANSFORM_OP(&result, In::nullable_element(inputs, i)...); - } - - Out::assign(outputs, i, *result); - - if constexpr (may_evaluate_null) { intermediate_null_mask[i] = result.has_value(); } - } + if constexpr (!Out::may_be_nullable) { + return; + } else { + auto null_word = __ballot_sync(0xFFFF'FFFFU, is_valid); + if ((threadIdx.x % 32) == 0) { Out::set_null_word(outputs, row / 32, null_word); } } } +} // namespace template -CUDF_KERNEL void span_kernel(cudf::jit::device_optional_span const* outputs, - cudf::column_device_view_core const* inputs, - bool* intermediate_null_mask, - void* user_data) + typename Ins, + typename Outs> +CUDF_KERNEL void transform_kernel(size_type num_rows, + void* user_data, + column_device_view_core const* inputs, + bitmask_type const* stencil, + mutable_column_device_view_core const* outputs) { - auto const start = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); - auto const size = outputs[0].size(); - - for (auto i = start; i < size; i += stride) { - if constexpr (is_null_aware == null_aware::NO) { - if constexpr (may_evaluate_null) { - if (Out::is_null(outputs, i)) { continue; } - } - - if constexpr (has_user_data) { - GENERIC_TRANSFORM_OP(user_data, i, &Out::element(outputs, i), In::element(inputs, i)...); - } else { - GENERIC_TRANSFORM_OP(&Out::element(outputs, i), In::element(inputs, i)...); - } - } else { // is_null_aware == null_aware::YES - cuda::std::optional result; + auto const start = detail::grid_1d::global_thread_id(); + auto const stride = detail::grid_1d::grid_stride(); - if constexpr (has_user_data) { - GENERIC_TRANSFORM_OP(user_data, i, &result, In::nullable_element(inputs, i)...); - } else { - GENERIC_TRANSFORM_OP(&result, In::nullable_element(inputs, i)...); - } + for (auto row = start; row < num_rows; row += stride) { + bool is_valid[Outs::size]; - Out::assign(outputs, i, *result); + transform_udf::call( + GENERIC_TRANSFORM_OP, row, user_data, inputs, stencil, outputs, is_valid); - if constexpr (may_evaluate_null) { intermediate_null_mask[i] = result.has_value(); } - } + Outs::map([&] { + (warp_compact_validity(outputs, row, is_valid[Out::index]), ...); + }); } } } // namespace jit -} // namespace transformation } // namespace cudf diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index c909b94efe32..9bafc96aed34 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -29,186 +29,206 @@ #include namespace cudf { -using InputsView = std::span const>; - -namespace transformation { - -namespace jit { namespace { -jitify2::StringVec build_jit_template_params(null_aware is_null_aware, - bool may_evaluate_null, - bool has_user_data, - std::span span_outputs, - std::span column_outputs, - std::span inputs) -{ - jitify2::StringVec tparams; - - tparams.emplace_back(jitify2::reflection::reflect(is_null_aware)); - tparams.emplace_back(jitify2::reflection::reflect(may_evaluate_null)); - tparams.emplace_back(jitify2::reflection::reflect(has_user_data)); - - std::transform(thrust::counting_iterator(0), - thrust::counting_iterator(span_outputs.size()), - std::back_inserter(tparams), - [&](auto i) { - return jitify2::reflection::Template("cudf::jit::span_accessor") - .instantiate(span_outputs[i], i); - }); - - std::transform(thrust::counting_iterator(0), - thrust::counting_iterator(column_outputs.size()), - std::back_inserter(tparams), - [&](auto i) { - return jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate(column_outputs[i], i); - }); - - std::transform(thrust::counting_iterator(0), - thrust::counting_iterator(inputs.size()), - std::back_inserter(tparams), - [&](auto i) { return inputs[i].accessor(i); }); +// TODO: update parameter names and reflection +template +struct mutable_vector_column_view { + T* _data{nullptr}; + size_type _size{0}; + bitmask_type const* _null_mask{nullptr}; + size_type _offset{0}; + size_type _null_count{0}; + + auto to_device() const + { + auto deleter = +[](mutable_column_device_view* ptr) { delete ptr; }; + return std::unique_ptr>( + new mutable_column_device_view{mutable_column_device_view::from_parts( + data_type{type_id::EMPTY}, _size, _data, _null_mask, _offset, nullptr, 0)}, + deleter); + } +}; - return tparams; -} +template +struct vector_column { + using mutable_view_type = mutable_vector_column_view; + + rmm::device_uvector _data{}; + rmm::device_buffer _null_mask{}; + size_type _offset{0}; + size_type _null_count{0}; + + static auto make(size_type size, + null_mask_state null_state, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) + { + rmm::device_uvector data{size, stream, mr}; + auto null_mask = create_null_mask(size, null_state, stream, mr); + + return vector_column{std::move(data), std::move(null_mask), 0, size, 0}; + } -jitify2::ConfiguredKernel build_transform_kernel( - std::string_view kernel_name, - std::span output_columns, - InputsView inputs, - null_aware is_null_aware, - bool may_evaluate_null, - bool has_user_data, - std::string const& udf, - cudf::udf_source_type source_type, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + auto mutable_view() + { + return mutable_view_type{_data.data(), + _data.size(), + static_cast(_null_mask.data()), + _offset, + _null_count}; + } +}; + +using string_view_column = vector_column; +using mutable_string_view_column_view = typename string_view_column::mutable_view_type; + +using OutputColumn = std::variant, std::unique_ptr>; +using InputView = std::variant; +using OutputView = std::variant; +using InputViews = std::span; +using OutputViews = std::span; +using InHandle = std::unique_ptr>; +using OutHandle = + std::unique_ptr>; +using Handle = std::variant; + +std::string reflect_output_accessor(OutputView const& output) { - auto output_typenames = cudf::jit::output_type_names(output_columns); - auto input_typenames = cudf::jit::input_type_names(inputs); - auto input_reflections = cudf::jit::reflect_inputs(inputs); - - auto const cuda_source = - (source_type == cudf::udf_source_type::PTX) - ? cudf::jit::parse_single_function_ptx( - udf, - "GENERIC_TRANSFORM_OP", - cudf::jit::build_ptx_params(output_typenames, input_typenames, has_user_data)) - : cudf::jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); - - auto kernel_reflection = - jitify2::reflection::Template(kernel_name) - .instantiate(build_jit_template_params( - is_null_aware, may_evaluate_null, has_user_data, {}, output_typenames, input_reflections)); - - return cudf::jit::get_udf_kernel(*transform_jit_kernel_cu_jit, kernel_reflection, cuda_source) - ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); -} + auto element = std::visit([](auto& a) { return type_to_name(a.type()); }, output); + auto column = + std::holds_alternative(output) + ? "cudf::jit::column_device_view_span_wrapper" + : "cudf::mutable_column_device_view_core"; -jitify2::ConfiguredKernel build_span_kernel(std::string_view kernel_name, - std::span span_outputs, - InputsView inputs, - null_aware is_null_aware, - bool may_evaluate_null, - bool has_user_data, - std::string const& udf, - cudf::udf_source_type source_type, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - CUDF_FUNC_RANGE(); - auto output_typenames = span_outputs; - auto input_typenames = cudf::jit::input_type_names(inputs); - auto input_reflections = cudf::jit::reflect_inputs(inputs); - - auto const cuda_source = - (source_type == cudf::udf_source_type::PTX) - ? cudf::jit::parse_single_function_ptx( - udf, - "GENERIC_TRANSFORM_OP", - cudf::jit::build_ptx_params(output_typenames, input_typenames, has_user_data)) - : cudf::jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); - - auto kernel_reflection = - jitify2::reflection::Template(kernel_name) - .instantiate(build_jit_template_params( - is_null_aware, may_evaluate_null, has_user_data, span_outputs, {}, input_reflections)); - - return cudf::jit::get_udf_kernel(*transform_jit_kernel_cu_jit, kernel_reflection, cuda_source) - ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); + return jitify2::Template("cudf::jit::column_accessor").instantiate(column, element, false); } -column_view to_column_view(column_view const& col) { return col; } - -column_view to_column_view(scalar_column_view const& scalar) { return scalar.as_column_view(); } - -auto to_device_input_arg(InputsView inputs, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::vector reflect_output_accessors(OutputViews outputs) { - std::vector columns; - for (auto const& input : inputs) { - columns.emplace_back(std::visit([](auto const& col) { return to_column_view(col); }, input)); - } - - return cudf::jit::column_views_to_device(columns, stream, mr); -} + std::vector res; + std::transform(thrust::counting_iterator(0), + thrust::counting_iterator(outputs.size()), + std::back_inserter(res), + [&](auto i) { return reflect_output_accessor(outputs[i]); }); -auto to_device_output_arg(std::span outputs, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - return cudf::jit::column_views_to_device( - outputs, stream, mr); + return res; } -void launch_column_output_kernel(jitify2::ConfiguredKernel& kernel, - std::span output_columns, - InputsView inputs, - std::optional intermediate_null_mask, - std::optional user_data, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - auto [output_arg_handles, output_args] = to_device_output_arg(output_columns, stream, mr); - auto [input_arg_handles, input_args] = to_device_input_arg(inputs, stream, mr); - - mutable_column_device_view const* p_outputs = output_args.data(); - column_device_view const* p_inputs = input_args.data(); - bool* p_intermediate_null_mask = intermediate_null_mask.value_or(nullptr); - void* p_user_data = user_data.value_or(nullptr); +struct TransformKernel { + static jitify2::Kernel instantiate(null_aware is_null_aware, + bool may_evaluate_null, + bool has_user_data, + std::string const& ins, + std::string const& outs, + std::string const& udf, + udf_source_type source_type) + { + CUDF_FUNC_RANGE(); + auto cuda_source = + (source_type == udf_source_type::PTX) + ? jit::parse_single_function_ptx( + udf, + "GENERIC_TRANSFORM_OP", + jit::build_ptx_params(output_typenames, input_typenames, has_user_data)) + : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); + + auto kernel = jitify2::reflection::Template("cudf::jit::transform_kernel") + .instantiate(is_null_aware, may_evaluate_null, has_user_data, ins, outs); + + return jit::get_udf_kernel(*transform_jit_kernel_cu_jit, kernel, cuda_source); + } - std::array args{&p_outputs, &p_inputs, &p_intermediate_null_mask, &p_user_data}; + static void launch(jitify2::Kernel const& kernel, + size_type row_size, + void* user_data, + column_device_view_core const* inputs, + bitmask_type const* null_mask_and, + mutable_column_device_view_core const* outputs, + rmm::cuda_stream_view stream) + { + CUDF_FUNC_RANGE(); + void* args[] = {&row_size, &user_data, &inputs, &null_mask_and, &outputs}; + kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); + } - kernel->launch_raw(args.data()); -} + static auto to_device_args(InputViews inputs, + OutputViews outputs, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) + { + std::vector handles; + std::vector h_args; + + for (auto& in : inputs) { + if (auto* col = std::get_if(&in)) { + auto handle = column_device_view::create(*col, stream, mr); + h_args.push_back(*handle); + handles.push_back(std::move(handle)); + } else if (auto& scalar = std::get(in)) { + auto handle = column_device_view::create(scalar.as_column_view(), stream, mr); + h_args.push_back(*handle); + handles.push_back(std::move(handle)); + } + } -template -void launch_span_kernel(jitify2::ConfiguredKernel& kernel, - cudf::jit::device_optional_span const& output, - InputsView inputs, - std::optional intermediate_null_mask, - std::optional user_data, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - auto output_args = cudf::jit::to_device_vector(std::vector{output}, stream, mr); + for (auto& out : outputs) { + if (auto* col = std::get_if(&out)) { + auto handle = mutable_column_device_view::create(*col, stream, mr); + h_args.push_back(*handle); + handles.push_back(std::move(handle)); + } else if (auto& sv_column = std::get(out)) { + auto handle = sv_column.to_device(); + h_args.push_back(*handle); + handles.push_back(std::move(handle)); + } + } - auto [input_arg_handles, input_args] = to_device_input_arg(inputs, stream, mr); + rmm::device_uvector d_args{h_args.size(), stream, mr}; - cudf::jit::device_optional_span const* p_outputs = output_args.data(); - column_device_view const* p_inputs = input_args.data(); - bool* p_intermediate_null_mask = intermediate_null_mask.value_or(nullptr); - void* p_user_data = user_data.value_or(nullptr); + detail::cuda_memcpy_async_impl(d_args.data(), + h_args.data(), + h_args.size() * sizeof(column_device_view), + detail::host_memory_kind::PAGEABLE, + stream); - std::array args{&p_outputs, &p_inputs, &p_intermediate_null_mask, &p_user_data}; + return std::make_tuple(std::move(handles), std::move(d_args)); + } - kernel->launch_raw(args.data()); -} + static void run(null_aware is_null_aware, + bool may_evaluate_null, + bool has_user_data, + size_type row_size, + void* user_data, + InputViews inputs, + bitmask_type const* d_null_mask_and, + OutputsView outputs, + std::string const& udf, + udf_source_type source_type, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) + { + auto in_types = jitify2::reflection::Template("cudf::jit::type_list") + .instantiate(jit::reflect_input_accessors(inputs)); + auto out_types = jitify2::reflection::Template("cudf::jit::type_list") + .instantiate(jit::reflect_output_accessors(outputs)); + + auto kernel = instantiate( + is_null_aware, may_evaluate_null, has_user_data, in_types, out_types, udf, source_type); + + auto [args, handles] = to_device_args(inputs, outputs, stream, mr); + + auto* d_inputs = args.data(); + auto* d_outputs = + reinterpret_cast(args.data() + inputs.size()); + + launch(kernel, row_size, user_data, d_inputs, d_null_mask_and, d_outputs, stream); + } +}; std::tuple and_null_mask(size_type row_size, - InputsView inputs, + InputViews inputs, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { @@ -219,10 +239,9 @@ std::tuple and_null_mask(size_type row_size, // then all the rows of the transform output will be null. This helps us prevent creating // column-sized bitmasks for each scalar. for (auto const& in : inputs) { - if (std::holds_alternative(in)) { - auto& scalar = std::get(in); + if (auto* scalar = std::get_if(&in)) { // all nulls - if (scalar.has_nulls()) { + if (scalar->has_nulls()) { return std::make_tuple(create_null_mask(row_size, mask_state::ALL_NULL, stream, mr), row_size); } @@ -235,13 +254,13 @@ std::tuple and_null_mask(size_type row_size, if (bitmask_columns.empty()) { // if there are no non-scalar columns contributing to the null-mask, then the output is all // valid (scalar projection) given that the scalar is not null (checked above) - return std::make_tuple(create_null_mask(row_size, mask_state::ALL_VALID, stream, mr), 0); + return std::make_tuple(create_null_mask(row_size, mask_state::UNALLOCATED, stream, mr), 0); } return cudf::bitmask_and(table_view{bitmask_columns}, stream, mr); } -bool may_evaluate_null(InputsView inputs, null_aware is_null_aware, output_nullability null_out) +bool may_evaluate_null(InputViews inputs, null_aware is_null_aware, output_nullability null_out) { // null-aware UDFs will evaluate nulls unless explicitly marked as not producing nulls if (is_null_aware == null_aware::YES) { @@ -256,144 +275,107 @@ bool may_evaluate_null(InputsView inputs, null_aware is_null_aware, output_nulla } } -std::unique_ptr transform_operation(size_type row_size, - InputsView inputs, - std::string const& udf, - data_type output_type, - cudf::udf_source_type source_type, - std::optional user_data, - null_aware is_null_aware, - output_nullability null_policy, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +auto finalize(std::vector outputs, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { - auto output = - make_fixed_width_column(output_type, row_size, cudf::mask_state::UNALLOCATED, stream, mr); - - auto may_return_nulls = may_evaluate_null(inputs, is_null_aware, null_policy); - - std::optional> intermediate_null_mask = std::nullopt; - - if (is_null_aware == null_aware::NO) { - if (may_return_nulls) { - auto [and_mask_buffer, and_mask_null_count] = and_null_mask(row_size, inputs, stream, mr); - output->set_null_mask(std::move(and_mask_buffer), and_mask_null_count); + std::vector> results; + + for (auto& out : outputs) { + if (auto* col = std::get_if>(&out)) { + results.push_back(std::move(*col)); + } else if (auto& str = std::get>(out)) { + auto result = detail::make_strings_column( + str->_data, std::move(str->_null_mask), std::nullopt, stream, mr); + results.push_back(std::move(result)); } - } else if (is_null_aware == null_aware::YES) { - if (may_return_nulls) { intermediate_null_mask.emplace(row_size, stream, mr); } - } - - mutable_column_view outputs[] = {{*output}}; - auto kernel = build_transform_kernel(is_fixed_point(output_type) - ? "cudf::transformation::jit::fixed_point_kernel" - : "cudf::transformation::jit::kernel", - outputs, - inputs, - is_null_aware, - may_return_nulls, - user_data.has_value(), - udf, - source_type, - stream, - mr); - - launch_column_output_kernel(kernel, - outputs, - inputs, - intermediate_null_mask.has_value() - ? std::optional(intermediate_null_mask->data()) - : std::nullopt, - user_data, - stream, - mr); - - if (intermediate_null_mask.has_value()) { - auto [null_mask, null_count] = detail::valid_if( - intermediate_null_mask->begin(), - intermediate_null_mask->end(), - [] __device__(bool element) { return element; }, - stream, - mr); - - output->set_null_mask(std::move(null_mask), null_count); } - return output; + return results; } -std::unique_ptr string_view_operation(size_type row_size, - InputsView inputs, - std::string const& udf, - data_type output_type, - cudf::udf_source_type source_type, - std::optional user_data, - null_aware is_null_aware, - output_nullability null_policy, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::unique_ptr
transform_operation(size_type row_size, + InputViews inputs, + std::string const& udf, + std::span output_types, + udf_source_type source_type, + std::optional user_data, + null_aware is_null_aware, + std::span null_policies, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { - rmm::device_uvector string_views(row_size, stream, mr); + std::vector outputs; + + auto may_return_nulls = + std::any_of(null_policies.begin(), null_policies.end(), [](auto null_policy) { + return null_policy != output_nullability::ALL_VALID; + }) may_evaluate_null(inputs, is_null_aware, null_policy); + + for (size_t i = 0; i < output_types.size(); i++) { + auto type = output_types[i]; + auto null_policy = null_policies[i]; + + if (is_fixed_width(type)) { + auto col = make_fixed_width_column(type, row_size, stream, mr); + } else if (type == type_id::STRING) { + auto col = string_view_column::make( + row_size, + may_return_nulls ? null_mask_state::UNINITIALIZED : null_mask_state::UNALLOCATED, + stream, + mr); + + outputs.push_back(std::make_unique(std::move(col))); + } else { + CUDF_UNREACHABLE("Unsupported output type for transform"); + } + } - auto may_return_nulls = may_evaluate_null(inputs, is_null_aware, null_policy); + TransformKernel::run(is_null_aware, + may_return_nulls, + user_data.has_value(), + row_size, + user_data.value_or(nullptr), + inputs, + nullptr, // TODO: compute and pass the and of the null masks if needed + outputs, + udf, + source_type, + stream, + mr); + + return std::make_unique
(finalize(std::move(outputs), stream, mr)); +} - std::optional> intermediate_null_mask = std::nullopt; - std::optional> and_mask = std::nullopt; +void perform_checks(null_aware is_null_aware, + udf_source_type source_type, + std::optional in_row_size, + std::span output_types, + InputViews inputs) +{ + // TODO: what to do when mutable_column_view arguments violate expected flow? + // i.e. stencil generation, row-size determination, etc. + CUDF_EXPECTS( + !inputs.empty(), "Transform must have at least 1 input column", std::invalid_argument); + CUDF_EXPECTS(!(is_null_aware == null_aware::YES && source_type == udf_source_type::PTX), + "Optional types are not supported in PTX UDFs", + std::invalid_argument); - if (is_null_aware == null_aware::NO) { - if (may_return_nulls) { and_mask = and_null_mask(row_size, inputs, stream, mr); } - } else if (is_null_aware == null_aware::YES) { - if (may_return_nulls) { intermediate_null_mask.emplace(row_size, stream, mr); } - } + CUDF_EXPECTS(is_fixed_width(output_type) || output_type.id() == type_id::STRING, + "Transforms only support output of fixed-width or string types", + std::invalid_argument); - auto output_span = cudf::jit::device_optional_span{ - cudf::jit::device_span{string_views.data(), string_views.size()}, - and_mask.has_value() ? static_cast(std::get<0>(*and_mask).data()) : nullptr}; - - std::string output_typenames[] = {"cudf::string_view"}; - - auto kernel = build_span_kernel("cudf::transformation::jit::span_kernel", - output_typenames, - inputs, - is_null_aware, - may_return_nulls, - user_data.has_value(), - udf, - source_type, - stream, - mr); - - launch_span_kernel(kernel, - output_span, - inputs, - intermediate_null_mask.has_value() - ? std::optional(intermediate_null_mask->data()) - : std::nullopt, - user_data, - stream, - mr); - - auto output = make_strings_column(string_views, cudf::string_view{}, stream, mr); - - if (and_mask.has_value()) { - auto [and_mask_buffer, and_mask_null_count] = std::move(*and_mask); - output->set_null_mask(std::move(and_mask_buffer), and_mask_null_count); - } else if (intermediate_null_mask.has_value()) { - auto [null_mask, null_count] = detail::valid_if( - intermediate_null_mask->begin(), - intermediate_null_mask->end(), - [] __device__(bool element) { return element; }, - stream, - mr); - - output->set_null_mask(std::move(null_mask), null_count); - } + auto get_type = [](auto const& in) { + return std::visit([](auto const& col) { return col.type(); }, in); + }; - return output; -} + CUDF_EXPECTS( + std::all_of(thrust::make_transform_iterator(inputs.begin(), get_type), + thrust::make_transform_iterator(inputs.end(), get_type), + [](data_type t) { return is_fixed_width(t) || (t.id() == type_id::STRING); }), + "Transforms only support input of fixed-width or string types", + std::invalid_argument); -void check_row_size(std::optional in_row_size, InputsView inputs) -{ - auto row_size = in_row_size.value_or(cudf::jit::get_projection_size(inputs)); if (!in_row_size.has_value()) { CUDF_EXPECTS( @@ -404,96 +386,33 @@ void check_row_size(std::optional in_row_size, InputsView inputs) std::invalid_argument); } + auto row_size = in_row_size.value_or(jit::get_projection_size(inputs)); CUDF_EXPECTS(std::all_of(inputs.begin(), inputs.end(), - [&](auto const& input) { - if (std::holds_alternative(input)) { - return std::get(input).size() == row_size; + [&](auto & in) { + if (auto * col = std::get_if(&input)) { + return col->size() == row_size; } - return true; }), "All transform input columns must have the same size", std::invalid_argument); } -void perform_checks(std::optional in_row_size, data_type output_type, InputsView inputs) -{ - CUDF_EXPECTS(is_fixed_width(output_type) || output_type.id() == type_id::STRING, - "Transforms only support output of fixed-width or string types", - std::invalid_argument); - - auto get_type = [](auto const& in) { - return std::visit([](auto const& col) { return col.type(); }, in); - }; - - CUDF_EXPECTS( - std::all_of(thrust::make_transform_iterator(inputs.begin(), get_type), - thrust::make_transform_iterator(inputs.end(), get_type), - [](data_type t) { return is_fixed_width(t) || (t.id() == type_id::STRING); }), - "Transforms only support input of fixed-width or string types", - std::invalid_argument); - - check_row_size(in_row_size, inputs); -} - } // namespace -} // namespace jit -} // namespace transformation - -namespace detail { - -std::unique_ptr transform(InputsView inputs, - std::string const& udf, - data_type output_type, - cudf::udf_source_type source_type, - std::optional user_data, - null_aware is_null_aware, - std::optional in_row_size, - output_nullability null_policy, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::vector> transform_extended2(std::string const& udf, + udf_source_type source_type, + null_aware is_null_aware, + std::optional row_size, + std::optional user_data, + std::span inputs, + std::span outputs, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { - CUDF_EXPECTS( - !inputs.empty(), "Transform must have at least 1 input column", std::invalid_argument); - CUDF_EXPECTS(!(is_null_aware == null_aware::YES && source_type == cudf::udf_source_type::PTX), - "Optional types are not supported in PTX UDFs", - std::invalid_argument); - - transformation::jit::perform_checks(in_row_size, output_type, inputs); - - auto row_size = in_row_size.value_or(cudf::jit::get_projection_size(inputs)); - - if (is_fixed_width(output_type)) { - return transformation::jit::transform_operation(row_size, - inputs, - udf, - output_type, - source_type, - user_data, - is_null_aware, - null_policy, - stream, - mr); - } else if (output_type.id() == type_id::STRING) { - return transformation::jit::string_view_operation(row_size, - inputs, - udf, - output_type, - source_type, - user_data, - is_null_aware, - null_policy, - stream, - mr); - } else { - CUDF_FAIL("Unsupported output type for transform operation"); - } } -} // namespace detail - std::unique_ptr transform_extended( std::span const> inputs, std::string const& udf, @@ -507,16 +426,6 @@ std::unique_ptr transform_extended( rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::transform(inputs, - udf, - output_type, - source_type, - user_data, - is_null_aware, - row_size, - null_policy, - stream, - mr); } std::unique_ptr transform(std::vector const& columns, @@ -545,16 +454,19 @@ std::unique_ptr transform(std::vector const& columns, } } - return detail::transform(inputs, - transform_udf, - output_type, - is_ptx ? udf_source_type::PTX : udf_source_type::CUDA, - user_data, - is_null_aware, - base_column->size(), - null_policy, - stream, - mr); + // TODO: take inputs of mutable column view strings and pass their data to the kernel + // TODO: take sizer for the strings, needs to allow zero-sized outputs + + return transform_extended(inputs, + transform_udf, + output_type, + is_ptx ? udf_source_type::PTX : udf_source_type::CUDA, + user_data, + is_null_aware, + base_column->size(), + null_policy, + stream, + mr); } std::unique_ptr compute_column_jit(table_view const& table, @@ -562,20 +474,19 @@ std::unique_ptr compute_column_jit(table_view const& table, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - cudf::detail::row_ir::ast_args ast_args{.table = table}; - auto args = cudf::detail::row_ir::ast_converter::compute_column( - cudf::detail::row_ir::target::CUDA, expr, ast_args, stream, mr); - - return cudf::transform_extended(args.inputs, - args.udf, - args.output_type, - args.source_type, - args.user_data, - args.is_null_aware, - args.row_size, - args.null_policy, - stream, - mr); + detail::row_ir::ast_args ast_args{.table = table}; + auto args = detail::row_ir::ast_converter::compute_column( + detail::row_ir::target::CUDA, expr, ast_args, stream, mr); + return transform_extended(args.inputs, + args.udf, + args.output_type, + args.source_type, + args.user_data, + args.is_null_aware, + args.row_size, + args.null_policy, + stream, + mr); } } // namespace cudf From 8dba8b8724d6f5a29e072a4203f634a843e6b82b Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Mar 2026 16:14:27 +0000 Subject: [PATCH 068/254] Refactor transform functionality to support new output specifications and improve column handling - Updated `transform_input` to remove `mutable_column_view` and streamline input handling. - Introduced `transform_output` struct to specify output types and nullability policies. - Modified `transform_extended2` to return a `table` instead of a vector of columns, enhancing output management. - Refactored `column_accessor` to utilize `detail::column_device_view_base` for better abstraction. - Removed deprecated reflection functions for input accessors and adjusted related helper functions. - Enhanced `transform_udf` to work with the new column view structure, ensuring compatibility with the updated input/output specifications. - Updated kernel launch and argument handling to accommodate changes in input and output types. - Improved nullability checks and handling in the transform execution flow. - Added support for pre-allocated string offsets in output columns to optimize memory usage. --- .../cudf/column/column_device_view.cuh | 30 +- cpp/include/cudf/transform.hpp | 42 +- cpp/src/jit/column_accessor.cuh | 35 +- cpp/src/jit/helpers.cpp | 22 - cpp/src/jit/helpers.hpp | 27 - cpp/src/jit/transform_udf.cuh | 14 +- cpp/src/jit/wrappers.cuh | 40 +- cpp/src/transform/jit/kernel.cu | 15 +- cpp/src/transform/transform.cu | 537 +++++++++++------- 9 files changed, 422 insertions(+), 340 deletions(-) diff --git a/cpp/include/cudf/column/column_device_view.cuh b/cpp/include/cudf/column/column_device_view.cuh index 32225aaed0e2..8ead6e279882 100644 --- a/cpp/include/cudf/column/column_device_view.cuh +++ b/cpp/include/cudf/column/column_device_view.cuh @@ -749,16 +749,28 @@ class alignas(16) mutable_column_device_view : public mutable_column_device_view */ static std::size_t extent(mutable_column_view source_view); - static mutable_column_device_view from_parts(data_type type, - size_type size, - void const* data, - bitmask_type const* null_mask, - size_type offset, - mutable_column_device_view* children, - size_type num_children) + /** + * @brief Factory to construct a mutable column view that is usable in device memory from + * pre-existing device memory pointers to data, nullmask, and offset. + * + * @param type The type of the column + * @param size The number of elements in the column + * @param data Pointer to the device memory containing the data + * @param null_mask Pointer to the device memory containing the null bitmask + * @param offset The index of the first element in the column + * @param children Pointer to the device memory containing child data + * @param num_children The number of child columns + * + */ + static auto from_parts(data_type type, + size_type size, + void const* data, + bitmask_type const* null_mask, + size_type offset, + mutable_column_device_view* children, + size_type num_children) { - return mutable_column_device_view{ - mutable_column_device_view_core{type, size, data, null_mask, offset, children, num_children}}; + return mutable_column_device_view{type, size, data, null_mask, offset, children, num_children}; } /** diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index efa3f8e4b7e5..a3bc6cfcaaab 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -71,11 +71,19 @@ namespace CUDF_EXPORT cudf { * @brief Typedef for inputs to the transform function. Each input can be either a column or a * scalar column. */ -using transform_input = std::variant; +using transform_input = std::variant; +/** + * @brief Specification for the outputs of the transform function. This includes the output type and + * nullability policy for each output column. + * + */ struct transform_output { - data_type type = data_type{type_id::EMPTY}; - output_nullability nullability = output_nullability::PRESERVE; + data_type type = data_type{type_id::EMPTY}; ///< The output type of the column to be created + + output_nullability nullability = + output_nullability::PRESERVE; ///< Signifies if a null mask should be created for the output + ///< column }; /** @@ -127,7 +135,7 @@ std::unique_ptr transform_extended( * element of the input columns. * * Computes: - * `UDF(&outputs[i]..., inputs[i]...)`. + * `(outputs[i]...) = UDF(inputs[i]...)`. * * * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) @@ -148,22 +156,24 @@ std::unique_ptr transform_extended( * @param user_data User-defined device data to pass to the UDF. * @param inputs Immutable views of the inputs to transform (columns and scalar columns) * @param outputs Specification of the output columns to be created + * @param string_offsets For string output columns, the offsets can be pre-allocated and passed in + * to prevent overhead of compacting string views into run-end strings column. * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory - * @return A vector of columns resulting from applying the transform function to - * every element of the input. These columns are returned in the same order as - * the `output_types`. + * @return A table containing the columns resulting from applying the transform + * function to every element of the input according to the output specifications * */ -std::vector> transform_extended2(std::string const& udf, - udf_source_type source_type, - null_aware is_null_aware, - std::optional row_size, - std::optional user_data, - std::span inputs, - std::span outputs, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr); +std::unique_ptr
transform_extended2(std::string const& udf, + udf_source_type source_type, + null_aware is_null_aware, + std::optional row_size, + std::optional user_data, + std::span inputs, + std::span outputs, + std::vector> string_offsets, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); /** * @brief Creates a null_mask from `input` by converting `NaN` to null and diff --git a/cpp/src/jit/column_accessor.cuh b/cpp/src/jit/column_accessor.cuh index 65876de4f6f6..ce68c5245538 100644 --- a/cpp/src/jit/column_accessor.cuh +++ b/cpp/src/jit/column_accessor.cuh @@ -6,6 +6,7 @@ #pragma once +#include #include #include @@ -27,9 +28,14 @@ struct column_accessor { static constexpr bool as_scalar = AsScalar; static constexpr bool may_be_nullable = MayBeNullable; - static __device__ element_type element(column_type const* cols, size_type row) + static __device__ auto& get(detail::column_device_view_base const* cols) { - auto& c = cols[index]; + return reinterpret_cast(cols[index]); + } + + static __device__ element_type element(detail::column_device_view_base const* cols, size_type row) + { + auto& c = get(cols); if constexpr (AsScalar) { return c.template element(0); @@ -38,12 +44,12 @@ struct column_accessor { } } - static __device__ bool is_null(column_type const* cols, size_type row) + static __device__ bool is_null(detail::column_device_view_base const* cols, size_type row) { if constexpr (!MayBeNullable) { return false; } else { - auto& c = cols[index]; + auto& c = get(cols); if constexpr (AsScalar) { return c.is_null(0); @@ -53,12 +59,12 @@ struct column_accessor { } } - static __device__ bool is_valid(column_type const* cols, size_type row) + static __device__ bool is_valid(detail::column_device_view_base const* cols, size_type row) { if constexpr (!MayBeNullable) { return true; } else { - auto& c = cols[index]; + auto& c = get(cols); if constexpr (AsScalar) { return c.is_valid(0); @@ -68,9 +74,10 @@ struct column_accessor { } } - static __device__ optional_element_type nullable_element(column_type const* cols, size_type row) + static __device__ optional_element_type + nullable_element(detail::column_device_view_base const* cols, size_type row) { - auto& c = cols[index]; + auto& c = get(cols); if constexpr (!MayBeNullable) { return c.template element(row); @@ -83,9 +90,11 @@ struct column_accessor { } } - static __device__ void set_null_word(column_type const* cols, size_type index, bitmask_type word) + static __device__ void set_null_word(detail::column_device_view_base const* cols, + size_type index, + bitmask_type word) { - auto& c = cols[index]; + auto& c = get(cols); if constexpr (!MayBeNullable) { return; @@ -98,10 +107,12 @@ struct column_accessor { } } - static __device__ void assign(column_type const* cols, size_type row, element_type value) + static __device__ void assign(detail::column_device_view_base const* cols, + size_type row, + element_type value) requires(!AsScalar) { - cols[index].template assign(row, value); + get(cols).template assign(row, value); } }; diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 4967ebfdc891..a862e16024be 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -87,28 +87,6 @@ std::vector input_type_names( return names; } -std::string reflect_input_accessor(std::variant const& input, - int32_t index) -{ - auto element = std::visit([](auto& a) { return type_to_name(a.type()); }, input); - bool as_scalar = std::holds_alternative(input); - - return jitify2::Template("cudf::jit::column_accessor") - .instantiate("cudf::column_device_view_core", element, as_scalar); -} - -std::vector reflect_input_accessors( - std::span const> inputs) -{ - std::vector res; - std::transform(thrust::counting_iterator(0), - thrust::counting_iterator(inputs.size()), - std::back_inserter(res), - [&](auto i) { return reflect_input_accessor(inputs[i], i); }); - - return res; -} - jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, std::string const& kernel_name, std::string const& cuda_source) diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index b52a253ea32f..e2ea5095de7a 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -74,33 +74,6 @@ column_views_to_device(std::span views, std::vector input_type_names( std::span const> views); -// TODO: how do we support mutable column view?, the arguments need to be casted -/** - * @brief Reflects the input column or scalar into a string that can be used in the generated CUDA - * code. - * - * @param index The index of the input in the list of inputs. - * @param input The input column or scalar to reflect. - * @param may_be_nullable Whether to allow runtime checks of the nullability of the input - * @return A string representing the input in the generated CUDA code. - */ -std::string reflect_input( - int32_t index, - std::variant const& input, - bool may_be_nullable); - -/** - * @brief Reflects the input columns or scalars into strings that can be used in the generated CUDA - * code. - * - * @param inputs The input columns or scalars to reflect. - * @param may_be_nullable Whether to allow runtime checks of the nullability of the inputs - * @return A vector of strings representing the inputs in the generated CUDA code. - */ -std::vector reflect_inputs( - std::span const> inputs, - std::span may_be_nullable); - jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, std::string const& kernel_name, std::string const& cuda_source); diff --git a/cpp/src/jit/transform_udf.cuh b/cpp/src/jit/transform_udf.cuh index af8f08d1e3cd..19e1ca437386 100644 --- a/cpp/src/jit/transform_udf.cuh +++ b/cpp/src/jit/transform_udf.cuh @@ -40,9 +40,8 @@ struct transform_udf { static __device__ void call(Fn&& udf, size_type index, void* user_data, - column_device_view_core const* in_cols, bitmask_type const* stencil, - mutable_column_device_view_core const* out_cols, + detail::column_device_view_base const* cols, [[maybe_unused]] bool* is_valid) requires(is_null_aware == null_aware::NO) { @@ -57,7 +56,7 @@ struct transform_udf { cuda::std::apply([&](auto&&... args) { return cuda::std::tuple{&args...}; }, outs); auto inputs = - Ins::map([&]() { return cuda::std::tuple{A::element(in_cols, index)...}; }); + Ins::map([&]() { return cuda::std::tuple{A::element(cols, index)...}; }); if constexpr (has_user_data) { auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, index}, out_ptrs, inputs); @@ -69,7 +68,7 @@ struct transform_udf { } [&](cuda::std::integer_sequence) { - (Outs::at::assign(out_cols, index, cuda::std::get(outs)), ...); + (Outs::at::assign(cols, index, cuda::std::get(outs)), ...); }(Outs::indexed); } @@ -77,9 +76,8 @@ struct transform_udf { static __device__ void call(Fn&& udf, size_type index, void* user_data, - column_device_view_core const* in_cols, [[maybe_unused]] bitmask_type const* stencil, - mutable_column_device_view_core const* out_cols, + detail::column_device_view_base const* cols, bool* is_valid) requires(is_null_aware == null_aware::YES) { @@ -90,7 +88,7 @@ struct transform_udf { cuda::std::apply([&](auto&&... args) { return cuda::std::tuple{&args...}; }, outs); auto inputs = Ins::map( - [&]() { return cuda::std::tuple{A::nullable_element(in_cols, index)...}; }); + [&]() { return cuda::std::tuple{A::nullable_element(cols, index)...}; }); if constexpr (has_user_data) { auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, index}, out_ptrs, inputs); @@ -102,7 +100,7 @@ struct transform_udf { } [&](cuda::std::integer_sequence) { - (Outs::at::assign(out_cols, index, cuda::std::get(outs)), ...); + (Outs::at::assign(cols, index, cuda::std::get(outs)), ...); ((is_valid[I] = cuda::std::get(outs).has_value()), ...); }(Ins::indexed); } diff --git a/cpp/src/jit/wrappers.cuh b/cpp/src/jit/wrappers.cuh index ae6feb304d9b..af3fa2f16bd6 100644 --- a/cpp/src/jit/wrappers.cuh +++ b/cpp/src/jit/wrappers.cuh @@ -10,20 +10,19 @@ namespace CUDF_EXPORT cudf { namespace jit { /** - * @brief A column wrapper type that treats a column as a span of elements. This treats the element - * as a contiguous sequence of elements. + * @brief A column wrapper type that treats a column as a vector of elements. * */ template -struct span_column : private Column { +struct vector_column_device_view : private Column { using base = Column; - CUDF_HOST_DEVICE constexpr span_column(Column const& src) : base{src} {} - ~span_column() = default; - span_column(span_column const&) = default; - span_column(span_column&&) = default; - span_column& operator=(span_column const&) = default; - span_column& operator=(span_column&&) = default; + CUDF_HOST_DEVICE constexpr vector_column_device_view(Column const& src) : base{src} {} + ~vector_column_device_view() = default; + vector_column_device_view(vector_column_device_view const&) = default; + vector_column_device_view(vector_column_device_view&&) = default; + vector_column_device_view& operator=(vector_column_device_view const&) = default; + vector_column_device_view& operator=(vector_column_device_view&&) = default; using base::nullable; using base::offset; @@ -61,29 +60,27 @@ struct span_column : private Column { return element(element); } - CUDF_HOST_DEVICE Column& to_base() { return static_cast(*this); } - - CUDF_HOST_DEVICE Column const& to_base() const { return static_cast(*this); } }; /** * @brief A column wrapper type that treats a column as a column of mutable strings. - * + * The offsets will have been pre-initialized and the chars will have been pre-allocated. */ template -struct mut_string_column : private Column { +struct mut_strings_column_device_view : private Column { using base = Column; - CUDF_HOST_DEVICE constexpr mut_string_column(Column const& src) : base{src} {} + CUDF_HOST_DEVICE constexpr mut_strings_column_device_view(Column const& src) : base{src} {} - ~mut_string_column() = default; - mut_string_column(mut_string_column const&) = default; - mut_string_column(mut_string_column&&) = default; - mut_string_column& operator=(mut_string_column const&) = default; - mut_string_column& operator=(mut_string_column&&) = default; + ~mut_strings_column_device_view() = default; + mut_strings_column_device_view(mut_strings_column_device_view const&) = default; + mut_strings_column_device_view(mut_strings_column_device_view&&) = default; + mut_strings_column_device_view& operator=(mut_strings_column_device_view const&) = default; + mut_strings_column_device_view& operator=(mut_strings_column_device_view&&) = default; using base::is_null; using base::is_valid; + using base::null_mask; using base::nullable; using base::offset; using base::size; @@ -111,9 +108,6 @@ struct mut_string_column : private Column { return element(element); } - CUDF_HOST_DEVICE Column& to_base() { return static_cast(*this); } - - CUDF_HOST_DEVICE Column const& to_base() const { return static_cast(*this); } }; } // namespace jit diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index af4d8e8416a1..32097b942533 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -32,7 +32,7 @@ namespace cudf { namespace jit { namespace { template -__device__ void warp_compact_validity(mutable_column_device_view_core const* outputs, +__device__ void warp_compact_validity(detail::column_device_view_base const* columns, size_type row, bool is_valid) { @@ -40,7 +40,7 @@ __device__ void warp_compact_validity(mutable_column_device_view_core const* out return; } else { auto null_word = __ballot_sync(0xFFFF'FFFFU, is_valid); - if ((threadIdx.x % 32) == 0) { Out::set_null_word(outputs, row / 32, null_word); } + if ((threadIdx.x & 31) == 0) { Out::set_null_word(columns, row / 32, null_word); } } } } // namespace @@ -50,23 +50,22 @@ template -CUDF_KERNEL void transform_kernel(size_type num_rows, +CUDF_KERNEL void transform_kernel(size_type row_size, void* user_data, - column_device_view_core const* inputs, bitmask_type const* stencil, - mutable_column_device_view_core const* outputs) + detail::column_device_view_base const* columns) { auto const start = detail::grid_1d::global_thread_id(); auto const stride = detail::grid_1d::grid_stride(); - for (auto row = start; row < num_rows; row += stride) { + for (auto row = start; row < row_size; row += stride) { bool is_valid[Outs::size]; transform_udf::call( - GENERIC_TRANSFORM_OP, row, user_data, inputs, stencil, outputs, is_valid); + GENERIC_TRANSFORM_OP, row, user_data, stencil, columns, is_valid); Outs::map([&] { - (warp_compact_validity(outputs, row, is_valid[Out::index]), ...); + (warp_compact_validity(columns, row, is_valid[Out::index]), ...); }); } } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 9bafc96aed34..77ec9984f47f 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -33,7 +33,7 @@ namespace { // TODO: update parameter names and reflection template -struct mutable_vector_column_view { +struct mut_vector_column_view { T* _data{nullptr}; size_type _size{0}; bitmask_type const* _null_mask{nullptr}; @@ -53,7 +53,7 @@ struct mutable_vector_column_view { template struct vector_column { - using mutable_view_type = mutable_vector_column_view; + using mut_view_type = mut_vector_column_view; rmm::device_uvector _data{}; rmm::device_buffer _null_mask{}; @@ -73,165 +73,222 @@ struct vector_column { auto mutable_view() { - return mutable_view_type{_data.data(), - _data.size(), - static_cast(_null_mask.data()), - _offset, - _null_count}; + return mut_view_type{_data.data(), + _data.size(), + static_cast(_null_mask.data()), + _offset, + _null_count}; } }; -using string_view_column = vector_column; -using mutable_string_view_column_view = typename string_view_column::mutable_view_type; +struct mut_strings_column_view { + mutable_column_view view; +}; + +struct mut_strings_column { + using mut_view_type = mut_strings_column_view; + + std::unique_ptr strings = nullptr; + + static auto make(size_type size, + int64_t chars_size, + std::unique_ptr offsets, + null_mask_state null_state, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) + { + // TODO: null-count needs to be updated + auto null_mask = create_null_mask(size, null_state, stream, mr); + auto chars = rmm::device_buffer{chars_size, stream, mr}; + return mut_strings_column{.strings{detail::make_strings_column(size, + std::move(offsets), + std::move(chars), + /* null_count = */ size, + std::move(null_mask), + stream, + mr)}}; + } + + auto mutable_view() { return mut_view_type{.view{strings->mutable_view()}}; } +}; + +using string_views_column = vector_column; +using mut_string_views_column_view = typename string_views_column::mut_view_type; -using OutputColumn = std::variant, std::unique_ptr>; -using InputView = std::variant; -using OutputView = std::variant; -using InputViews = std::span; -using OutputViews = std::span; -using InHandle = std::unique_ptr>; +using OutputColumn = std::variant, string_views_column, mut_strings_column>; + +using InputView = std::variant; +using OutputView = + std::variant; + +using InputViews = std::span; +using OutputViews = std::span; + +using InHandle = std::unique_ptr>; using OutHandle = std::unique_ptr>; + using Handle = std::variant; -std::string reflect_output_accessor(OutputView const& output) +namespace jit { +namespace transform { + +jitify2::Kernel instantiate(null_aware is_null_aware, + bool has_stencil, + bool has_user_data, + std::string const& ins, + std::string const& outs, + std::string const& udf, + udf_source_type source_type) { - auto element = std::visit([](auto& a) { return type_to_name(a.type()); }, output); - auto column = - std::holds_alternative(output) - ? "cudf::jit::column_device_view_span_wrapper" - : "cudf::mutable_column_device_view_core"; + CUDF_FUNC_RANGE(); + auto cuda_source = (source_type == udf_source_type::PTX) + ? jit::parse_single_function_ptx( + udf, + "GENERIC_TRANSFORM_OP", + jit::build_ptx_params(output_typenames, input_typenames, has_user_data)) + : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); + + auto kernel = jitify2::reflection::Template("cudf::jit::transform_kernel") + .instantiate(is_null_aware, may_evaluate_null, has_user_data, ins, outs); - return jitify2::Template("cudf::jit::column_accessor").instantiate(column, element, false); + return jit::get_udf_kernel(*transform_jit_kernel_cu_jit, kernel, cuda_source); } -std::vector reflect_output_accessors(OutputViews outputs) +void launch(jitify2::Kernel const& kernel, + size_type row_size, + void* user_data, + bitmask_type const* stencil, + detail::column_device_view_base const* columns, + rmm::cuda_stream_view stream) { - std::vector res; - std::transform(thrust::counting_iterator(0), - thrust::counting_iterator(outputs.size()), - std::back_inserter(res), - [&](auto i) { return reflect_output_accessor(outputs[i]); }); - - return res; + CUDF_FUNC_RANGE(); + void* args[] = {&row_size, &user_data, &stencil, &columns}; + kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); } -struct TransformKernel { - static jitify2::Kernel instantiate(null_aware is_null_aware, - bool may_evaluate_null, - bool has_user_data, - std::string const& ins, - std::string const& outs, - std::string const& udf, - udf_source_type source_type) - { - CUDF_FUNC_RANGE(); - auto cuda_source = - (source_type == udf_source_type::PTX) - ? jit::parse_single_function_ptx( - udf, - "GENERIC_TRANSFORM_OP", - jit::build_ptx_params(output_typenames, input_typenames, has_user_data)) - : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); - - auto kernel = jitify2::reflection::Template("cudf::jit::transform_kernel") - .instantiate(is_null_aware, may_evaluate_null, has_user_data, ins, outs); - - return jit::get_udf_kernel(*transform_jit_kernel_cu_jit, kernel, cuda_source); +std::pair reflect(InputViews inputs, + OutputViews outputs, + std::span input_may_be_nullable, + std::span output_may_be_nullable) +{ + std::vector ins; + + for (size_t i = 0; i < inputs.size(); i++) { + auto& input = inputs[i]; + auto column = "cudf::column_device_view_core"; + auto element = std::visit([](auto& a) { return type_to_name(a.type()); }, input); + auto optional_element = std::format("cuda::std::optional<{}>", element); + bool as_scalar = std::holds_alternative(input); + bool may_be_nullable = input_may_be_nullable[i]; + auto accessor = + jitify2::Template("cudf::jit::column_accessor") + .instantiate(i, column, element, optional_element, as_scalar, may_be_nullable); + ins.push_back(accessor); } - static void launch(jitify2::Kernel const& kernel, - size_type row_size, - void* user_data, - column_device_view_core const* inputs, - bitmask_type const* null_mask_and, - mutable_column_device_view_core const* outputs, - rmm::cuda_stream_view stream) - { - CUDF_FUNC_RANGE(); - void* args[] = {&row_size, &user_data, &inputs, &null_mask_and, &outputs}; - kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); + std::vector outs; + + for (size_t i = 0; i < outputs.size(); i++) { + auto& output = outputs[i]; + auto column = + std::holds_alternative(output) + ? "cudf::jit::column_device_view_span_wrapper" + : "cudf::mutable_column_device_view_core"; + auto element = std::visit([](auto& a) { return type_to_name(a.type()); }, output); + auto optional_element = std::format("cuda::std::optional<{}>", element); + bool as_scalar = false; // output can never be a scalar + bool may_be_nullable = output_may_be_nullable[i]; + auto accessor = + jitify2::Template("cudf::jit::column_accessor") + .instantiate( + inputs.size() + i, column, element, optional_element, as_scalar, may_be_nullable); + + outs.push_back(accessor); } +} - static auto to_device_args(InputViews inputs, - OutputViews outputs, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) - { - std::vector handles; - std::vector h_args; - - for (auto& in : inputs) { - if (auto* col = std::get_if(&in)) { - auto handle = column_device_view::create(*col, stream, mr); - h_args.push_back(*handle); - handles.push_back(std::move(handle)); - } else if (auto& scalar = std::get(in)) { - auto handle = column_device_view::create(scalar.as_column_view(), stream, mr); - h_args.push_back(*handle); - handles.push_back(std::move(handle)); - } +auto to_args(InputViews inputs, + OutputViews outputs, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + std::vector handles; + std::vector h_args; + + for (auto& in : inputs) { + if (auto* col = std::get_if(&in)) { + auto handle = column_device_view::create(*col, stream, mr); + h_args.push_back(*handle); + handles.push_back(std::move(handle)); + } else { + auto& scalar = std::get(in); + auto handle = column_device_view::create(scalar.as_column_view(), stream, mr); + h_args.push_back(*handle); + handles.push_back(std::move(handle)); } + } - for (auto& out : outputs) { - if (auto* col = std::get_if(&out)) { - auto handle = mutable_column_device_view::create(*col, stream, mr); - h_args.push_back(*handle); - handles.push_back(std::move(handle)); - } else if (auto& sv_column = std::get(out)) { - auto handle = sv_column.to_device(); - h_args.push_back(*handle); - handles.push_back(std::move(handle)); - } + for (auto& out : outputs) { + if (auto* col = std::get_if(&out)) { + auto handle = mutable_column_device_view::create(*col, stream, mr); + h_args.push_back(*handle); + handles.push_back(std::move(handle)); + } else { + auto& sv_column = std::get(out); + auto handle = sv_column.to_device(); + h_args.push_back(*handle); + handles.push_back(std::move(handle)); } - - rmm::device_uvector d_args{h_args.size(), stream, mr}; - - detail::cuda_memcpy_async_impl(d_args.data(), - h_args.data(), - h_args.size() * sizeof(column_device_view), - detail::host_memory_kind::PAGEABLE, - stream); - - return std::make_tuple(std::move(handles), std::move(d_args)); } - static void run(null_aware is_null_aware, - bool may_evaluate_null, - bool has_user_data, - size_type row_size, - void* user_data, - InputViews inputs, - bitmask_type const* d_null_mask_and, - OutputsView outputs, - std::string const& udf, - udf_source_type source_type, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) - { - auto in_types = jitify2::reflection::Template("cudf::jit::type_list") - .instantiate(jit::reflect_input_accessors(inputs)); - auto out_types = jitify2::reflection::Template("cudf::jit::type_list") - .instantiate(jit::reflect_output_accessors(outputs)); + rmm::device_uvector d_args{h_args.size(), stream, mr}; - auto kernel = instantiate( - is_null_aware, may_evaluate_null, has_user_data, in_types, out_types, udf, source_type); + detail::cuda_memcpy_async_impl(d_args.data(), + h_args.data(), + h_args.size() * sizeof(detail::column_device_view_base), + detail::host_memory_kind::PAGEABLE, + stream); - auto [args, handles] = to_device_args(inputs, outputs, stream, mr); + return std::make_tuple(std::move(d_args), std::move(handles)); +} - auto* d_inputs = args.data(); - auto* d_outputs = - reinterpret_cast(args.data() + inputs.size()); +void run(null_aware is_null_aware, + bool has_stencil, + bool has_user_data, + size_type row_size, + void* user_data, + bitmask_type const* d_stencil, + InputViews inputs, + OutputsView outputs, + std::span input_may_be_nullable, + std::span output_may_be_nullable, + std::string const& udf, + udf_source_type source_type, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + auto [in_types, out_types] = + reflect(inputs, outputs, input_may_be_nullable, output_may_be_nullable); + auto kernel = + instantiate(is_null_aware, has_stencil, has_user_data, in_types, out_types, udf, source_type); + auto [cols, handles] = to_args(inputs, outputs, stream, mr); + return launch(kernel, row_size, user_data, d_stencil, cols.data(), stream); +} - launch(kernel, row_size, user_data, d_inputs, d_null_mask_and, d_outputs, stream); - } -}; +} // namespace transform +} // namespace jit -std::tuple and_null_mask(size_type row_size, +std::tuple null_mask_and(size_type row_size, InputViews inputs, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { + // TODO(lamarrr): handle non-nullable inputs? or is it handled? + // what if we only have scalars? + // what if some of the inputs are non-nullable + // what if none of the inputs are nullable? will an allocated null mask be created? + // collect the non-scalar elements that contribute to the resulting bitmask std::vector bitmask_columns; @@ -251,6 +308,7 @@ std::tuple and_null_mask(size_type row_size, } } + // TODO: do we depend on the nullness of the outputs? if (bitmask_columns.empty()) { // if there are no non-scalar columns contributing to the null-mask, then the output is all // valid (scalar projection) given that the scalar is not null (checked above) @@ -260,19 +318,35 @@ std::tuple and_null_mask(size_type row_size, return cudf::bitmask_and(table_view{bitmask_columns}, stream, mr); } -bool may_evaluate_null(InputViews inputs, null_aware is_null_aware, output_nullability null_out) +std::pair, std::vector> get_nullabilities( + null_aware is_null_aware, InputViews inputs, std::span outputs) { - // null-aware UDFs will evaluate nulls unless explicitly marked as not producing nulls - if (is_null_aware == null_aware::YES) { - return null_out != output_nullability::ALL_VALID; - } else { - /// null-unaware UDFs will evaluate nulls if any input is nullable unless explicitly marked - /// as not producing nulls - bool any_nullable = std::any_of(inputs.begin(), inputs.end(), [](auto const& input) { - return std::visit([](auto const& col) { return col.nullable(); }, input); - }); - return any_nullable && null_out == output_nullability::PRESERVE; + std::vector input_may_be_nullable; + + for (auto& in : inputs) { + input_may_be_nullable.push_back(true); + } + + std::vector output_may_be_nullable; + + bool any_input_nullable = std::any_of(inputs.begin(), inputs.end(), [](auto& input) { + return std::visit([](auto const& col) { return col.nullable(); }, input); + }); + + for (auto& out : outputs) { + bool may_eval_null = true; + if (is_null_aware == null_aware::YES) { + // null-aware UDFs may evaluate nulls unless the output is explicitly marked as all valid + may_eval_null = out.nullability != output_nullability::ALL_VALID; + } else { + // null-unaware UDFs may evaluate nulls if any input is nullable unless explicitly marked as not producing nulls + may_eval_null = any_input_nullable && (out.nullability == output_nullability::PRESERVE); + } + + output_may_be_nullable.push_back(may_eval_null); } + + return {input_may_be_nullable, output_may_be_nullable}; } auto finalize(std::vector outputs, @@ -281,10 +355,11 @@ auto finalize(std::vector outputs, { std::vector> results; + // TODO: finish for (auto& out : outputs) { if (auto* col = std::get_if>(&out)) { results.push_back(std::move(*col)); - } else if (auto& str = std::get>(out)) { + } else if (auto& str = std::get>(out)) { auto result = detail::make_strings_column( str->_data, std::move(str->_null_mask), std::nullopt, stream, mr); results.push_back(std::move(result)); @@ -294,67 +369,13 @@ auto finalize(std::vector outputs, return results; } -std::unique_ptr
transform_operation(size_type row_size, - InputViews inputs, - std::string const& udf, - std::span output_types, - udf_source_type source_type, - std::optional user_data, - null_aware is_null_aware, - std::span null_policies, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - std::vector outputs; - - auto may_return_nulls = - std::any_of(null_policies.begin(), null_policies.end(), [](auto null_policy) { - return null_policy != output_nullability::ALL_VALID; - }) may_evaluate_null(inputs, is_null_aware, null_policy); - - for (size_t i = 0; i < output_types.size(); i++) { - auto type = output_types[i]; - auto null_policy = null_policies[i]; - - if (is_fixed_width(type)) { - auto col = make_fixed_width_column(type, row_size, stream, mr); - } else if (type == type_id::STRING) { - auto col = string_view_column::make( - row_size, - may_return_nulls ? null_mask_state::UNINITIALIZED : null_mask_state::UNALLOCATED, - stream, - mr); - - outputs.push_back(std::make_unique(std::move(col))); - } else { - CUDF_UNREACHABLE("Unsupported output type for transform"); - } - } - - TransformKernel::run(is_null_aware, - may_return_nulls, - user_data.has_value(), - row_size, - user_data.value_or(nullptr), - inputs, - nullptr, // TODO: compute and pass the and of the null masks if needed - outputs, - udf, - source_type, - stream, - mr); - - return std::make_unique
(finalize(std::move(outputs), stream, mr)); -} - -void perform_checks(null_aware is_null_aware, - udf_source_type source_type, +void perform_checks(udf_source_type source_type, + null_aware is_null_aware, std::optional in_row_size, - std::span output_types, - InputViews inputs) + InputViews inputs, + std::span outputs, + std::span const> string_offsets) { - // TODO: what to do when mutable_column_view arguments violate expected flow? - // i.e. stencil generation, row-size determination, etc. CUDF_EXPECTS( !inputs.empty(), "Transform must have at least 1 input column", std::invalid_argument); CUDF_EXPECTS(!(is_null_aware == null_aware::YES && source_type == udf_source_type::PTX), @@ -376,7 +397,6 @@ void perform_checks(null_aware is_null_aware, "Transforms only support input of fixed-width or string types", std::invalid_argument); - if (!in_row_size.has_value()) { CUDF_EXPECTS( std::any_of(inputs.begin(), @@ -389,28 +409,116 @@ void perform_checks(null_aware is_null_aware, auto row_size = in_row_size.value_or(jit::get_projection_size(inputs)); CUDF_EXPECTS(std::all_of(inputs.begin(), inputs.end(), - [&](auto & in) { - if (auto * col = std::get_if(&input)) { + [&](auto& in) { + if (auto* col = std::get_if(&input)) { return col->size() == row_size; } return true; }), "All transform input columns must have the same size", std::invalid_argument); + + // TODO: if string offset is provided, make sure it is a string column +} + +std::unique_ptr
execute_transform(std::string const& udf, + udf_source_type source_type, + null_aware is_null_aware, + std::optional in_row_size, + std::optional user_data, + InputViews inputs, + std::span outputs, + std::vector> string_offsets, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + auto row_size = in_row_size.value_or(jit::get_projection_size(inputs)); + // TODO: account for MayBeNullable + + auto [input_may_be_nullable, output_may_be_nullable] = + get_nullabilities(is_null_aware, inputs, outputs); + + for (size_t i = 0; i < outputs.size(); i++) { + auto output = outputs[i]; + auto may_be_nullable = output_may_be_nullable[i]; + auto null_state = + may_be_nullable ? null_mask_state::UNINITIALIZED : null_mask_state::UNALLOCATED; + + if (is_fixed_width(type)) { + auto col = make_fixed_width_column(type, row_size, null_state, stream, mr); + } else if (type == type_id::STRING) { + if (string_offsets[i] == nullptr) { + auto col = string_views_column::make(row_size, null_state, stream, mr); + outputs.push_back(std::make_unique(std::move(col))); + } else { + auto col = mut_strings_column::make(row_size, + /*TODO: chars_size*/ 0, + std::move(string_offsets[i]), + null_state, + stream, + mr); + } + } else { + CUDF_UNREACHABLE("Unsupported output type for transform"); + } + } + + // stencil in-place + // inplace_bitmask_and + bool any_output_nullable = std::any_of( + output_may_be_nullable.begin(), output_may_be_nullable.end(), [](auto b) { return b; }); + bool can_use_stencil = (is_null_aware == null_aware::NO); + + std::vector outputs; + std::optional stencil = std::nullopt; + // TODO: create a null mask and copy to all outputs, don't create stencil if all the outputs are + // non-nullable + // TODO: how will this affect kernel and outputs? + // TODO: Null-mask-and might return nullptr + // TODO: if no output is nullable, the bitmask and should be nullptr + // TODO: when copying bitmasks stencil, only copy to the nullable outputs + + jit::transform::run(is_null_aware, + may_return_nulls, + user_data.has_value(), + row_size, + user_data.value_or(nullptr), + inputs, + nullptr, // TODO: compute and pass the and of the null masks if needed + outputs, + udf, + source_type, + stream, + mr); + + return std::make_unique
(finalize(std::move(outputs), stream, mr)); } } // namespace -std::vector> transform_extended2(std::string const& udf, - udf_source_type source_type, - null_aware is_null_aware, - std::optional row_size, - std::optional user_data, - std::span inputs, - std::span outputs, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::unique_ptr
transform_extended2(std::string const& udf, + udf_source_type source_type, + null_aware is_null_aware, + std::optional row_size, + std::optional user_data, + std::span inputs, + std::span outputs, + std::vector> string_offsets, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { + CUDF_FUNC_RANGE(); + perform_checks(source_type, is_null_aware, row_size, inputs, outputs, string_offsets); + return execute_transform(udf, + source_type, + is_null_aware, + row_size, + user_data, + inputs, + outputs, + std::move(string_offsets), + stream, + mr); } std::unique_ptr transform_extended( @@ -425,7 +533,9 @@ std::unique_ptr transform_extended( rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - CUDF_FUNC_RANGE(); + transform_output outputs[] = {{.type = output_type, .null_policy = null_policy}}; + auto table = transform_extended2( + udf, source_type, is_null_aware, row_size, user_data, inputs, outputs, {}, stream, mr); } std::unique_ptr transform(std::vector const& columns, @@ -454,9 +564,6 @@ std::unique_ptr transform(std::vector const& columns, } } - // TODO: take inputs of mutable column view strings and pass their data to the kernel - // TODO: take sizer for the strings, needs to allow zero-sized outputs - return transform_extended(inputs, transform_udf, output_type, From e8250e2bcf3755ff22d84e2c101ba0b920cd1bed Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 8 Mar 2026 17:20:23 +0000 Subject: [PATCH 069/254] update --- cpp/CMakeLists.txt | 3 - .../Modules/JitifyPreprocessKernels.cmake | 82 ----- cpp/cmake/thirdparty/get_jitify.cmake | 25 -- cpp/src/binaryop/binaryop.cpp | 17 +- cpp/src/binaryop/jit/kernel.cu | 26 -- cpp/src/jit/cache.cpp | 136 -------- cpp/src/jit/cache.hpp | 39 --- cpp/src/jit/helpers.cpp | 32 +- cpp/src/jit/helpers.hpp | 8 +- cpp/src/jit/jit.cpp | 55 +++- cpp/src/jit/jit.hpp | 60 +++- cpp/src/join/jit_filter_join_indices.cu | 49 +-- cpp/src/librtcx/CMake/embed.cmake | 2 +- .../CMake/{embed.py.in => embed.in.py} | 0 cpp/src/librtcx/README.md | 2 +- cpp/src/librtcx/rtcx.cpp | 240 +++++++++----- cpp/src/librtcx/rtcx.hpp | 305 ++++++++++++++++-- cpp/src/rolling/detail/rolling_udf.cuh | 35 +- cpp/src/runtime/context.cpp | 63 ++-- cpp/src/runtime/context.hpp | 25 +- cpp/src/transform/transform.cu | 111 +++---- 21 files changed, 715 insertions(+), 600 deletions(-) delete mode 100644 cpp/cmake/Modules/JitifyPreprocessKernels.cmake delete mode 100644 cpp/cmake/thirdparty/get_jitify.cmake delete mode 100644 cpp/src/jit/cache.cpp delete mode 100644 cpp/src/jit/cache.hpp rename cpp/src/librtcx/CMake/{embed.py.in => embed.in.py} (100%) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e877d4c2df4d..752e267ae682 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -296,9 +296,6 @@ if(CUDF_BUILD_TESTUTIL) include(cmake/thirdparty/get_gtest.cmake) endif() -# preprocess jitify-able kernels -include(cmake/Modules/JitifyPreprocessKernels.cmake) - # find KvikIO include(cmake/thirdparty/get_kvikio.cmake) diff --git a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake deleted file mode 100644 index 10ba33eb3974..000000000000 --- a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake +++ /dev/null @@ -1,82 +0,0 @@ -# ============================================================================= -# cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# cmake-format: on -# ============================================================================= - -# Create `jitify_preprocess` executable -add_executable(jitify_preprocess "${JITIFY_INCLUDE_DIR}/jitify2_preprocess.cpp") - -target_compile_definitions(jitify_preprocess PRIVATE "_FILE_OFFSET_BITS=64") -rapids_cuda_set_runtime(jitify_preprocess USE_STATIC ${CUDA_STATIC_RUNTIME}) -target_link_libraries(jitify_preprocess PUBLIC ${CMAKE_DL_LIBS}) - -# Take a list of files to JIT-compile and run them through jitify_preprocess. -function(jit_preprocess_files) - cmake_parse_arguments(ARG "" "SOURCE_DIRECTORY" "FILES" ${ARGN}) - - get_target_property(libcudacxx_raw_includes CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) - set(includes) - foreach(inc IN LISTS libcudacxx_raw_includes CUDAToolkit_INCLUDE_DIRS) - list(APPEND includes "-I${inc}") - endforeach() - foreach(ARG_FILE ${ARG_FILES}) - set(ARG_OUTPUT ${CUDF_GENERATED_INCLUDE_DIR}/include/jit_preprocessed_files/${ARG_FILE}.jit.hpp) - get_filename_component(jit_output_directory "${ARG_OUTPUT}" DIRECTORY) - list(APPEND JIT_PREPROCESSED_FILES "${ARG_OUTPUT}") - - get_filename_component(ARG_OUTPUT_DIR "${ARG_OUTPUT}" DIRECTORY) - - # Note: need to pass _FILE_OFFSET_BITS=64 in COMMAND due to a limitation in how conda builds - # glibc - add_custom_command( - OUTPUT ${ARG_OUTPUT} - DEPENDS jitify_preprocess "${ARG_SOURCE_DIRECTORY}/${ARG_FILE}" - WORKING_DIRECTORY ${ARG_SOURCE_DIRECTORY} - VERBATIM - COMMAND ${CMAKE_COMMAND} -E make_directory "${jit_output_directory}" - COMMAND - "${CMAKE_COMMAND}" -E env LD_LIBRARY_PATH=${CUDAToolkit_LIBRARY_DIR} - $ ${ARG_FILE} -o ${ARG_OUTPUT_DIR} -i -std=c++20 - -remove-unused-globals -D_FILE_OFFSET_BITS=64 -D__CUDACC_RTC__ -DCUDF_RUNTIME_JIT - -I${CUDF_SOURCE_DIR}/include -I${CUDF_SOURCE_DIR}/src ${includes} - --no-preinclude-workarounds --no-replace-pragma-once --diag-suppress=47 --device-int128 - COMMENT "Custom command to JIT-compile files." - ) - endforeach() - set(JIT_PREPROCESSED_FILES - "${JIT_PREPROCESSED_FILES}" - PARENT_SCOPE - ) -endfunction() - -if(NOT (EXISTS "${CUDF_GENERATED_INCLUDE_DIR}/include")) - make_directory("${CUDF_GENERATED_INCLUDE_DIR}/include") -endif() - -jit_preprocess_files( - SOURCE_DIRECTORY ${CUDF_SOURCE_DIR}/src FILES binaryop/jit/kernel.cu rolling/jit/kernel.cu - transform/jit/kernel.cu join/jit/filter_join_kernel.cu -) - -add_custom_target( - jitify_preprocess_run - DEPENDS ${JIT_PREPROCESSED_FILES} - COMMENT "Target representing jitified files." -) - -# when a user requests CMake to clean the build directory -# -# * `cmake --build --target clean` -# * `cmake --build --clean-first` -# * ninja clean -# -# We also remove the jitify2 program cache as well. This ensures that we don't keep older versions -# of the programs in cache -set(cache_path "$ENV{HOME}/.cudf") -if(ENV{LIBCUDF_KERNEL_CACHE_PATH}) - set(cache_path "$ENV{LIBCUDF_KERNEL_CACHE_PATH}") -endif() -cmake_path(APPEND cache_path "${CUDF_VERSION}/") -set_target_properties(jitify_preprocess_run PROPERTIES ADDITIONAL_CLEAN_FILES "${cache_path}") diff --git a/cpp/cmake/thirdparty/get_jitify.cmake b/cpp/cmake/thirdparty/get_jitify.cmake deleted file mode 100644 index 4595b33c5160..000000000000 --- a/cpp/cmake/thirdparty/get_jitify.cmake +++ /dev/null @@ -1,25 +0,0 @@ -# ============================================================================= -# cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# cmake-format: on -# ============================================================================= - -# Jitify doesn't have a version :/ - -# This function finds Jitify and sets any additional necessary environment variables. -function(find_and_configure_jitify) - rapids_cpm_find( - jitify 2.0.0 - GIT_REPOSITORY https://github.com/NVIDIA/jitify.git - GIT_TAG 44e978b21fc8bdb6b2d7d8d179523c8350db72e5 # jitify2 branch as of 23rd Aug 2025 - GIT_SHALLOW FALSE - DOWNLOAD_ONLY TRUE - ) - set(JITIFY_INCLUDE_DIR - "${jitify_SOURCE_DIR}" - PARENT_SCOPE - ) -endfunction() - -find_and_configure_jitify() diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index bb4a48629ec5..254513d80c9a 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -162,12 +162,17 @@ void binary_operation(mutable_column_view& out, cudf::type_to_name(rhs.type()), std::string("cudf::binops::jit::UserDefinedOp")); - cudf::jit::get_udf_kernel(*binaryop_jit_kernel_cu_jit, kernel_reflection, cuda_source) - ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()) - ->launch(out.size(), - cudf::jit::get_data_ptr(out), - cudf::jit::get_data_ptr(lhs), - cudf::jit::get_data_ptr(rhs)); + auto kernel = cudf::jit::get_udf_kernel( + "src/binaryop/jit/kernel.cu", "src/binaryop/jit/kernel.cu", kernel_reflection, cuda_source); + + auto out_arg = cudf::jit::get_data_ptr(out); + auto lhs_arg = cudf::jit::get_data_ptr(lhs); + auto rhs_arg = cudf::jit::get_data_ptr(rhs); + + void* args[] = {&out_arg, &lhs_arg, &rhs_arg}; + + auto cfg = kernel.max_occupancy_config(0, 0); + kernel.launch(cfg.min_grid_size, 1, 1, cfg.block_size, 1, 1, 0, stream, args); } } // namespace jit diff --git a/cpp/src/binaryop/jit/kernel.cu b/cpp/src/binaryop/jit/kernel.cu index 893c1866cb44..bcc5b9542fea 100644 --- a/cpp/src/binaryop/jit/kernel.cu +++ b/cpp/src/binaryop/jit/kernel.cu @@ -69,32 +69,6 @@ CUDF_KERNEL void kernel_v_v(cudf::size_type size, } } -template -CUDF_KERNEL void kernel_v_v_with_validity(cudf::size_type size, - TypeOut* out_data, - TypeLhs* lhs_data, - TypeRhs* rhs_data, - cudf::bitmask_type* output_mask, - cudf::bitmask_type const* lhs_mask, - cudf::size_type lhs_offset, - cudf::bitmask_type const* rhs_mask, - cudf::size_type rhs_offset) -{ - auto const start = cudf::detail::grid_1d::global_thread_id(); - auto const step = cudf::detail::grid_1d::grid_stride(); - - for (auto i = start; i < size; i += step) { - bool output_valid = false; - out_data[i] = TypeOpe::template operate( - lhs_data[i], - rhs_data[i], - lhs_mask ? cudf::bit_is_set(lhs_mask, lhs_offset + i) : true, - rhs_mask ? cudf::bit_is_set(rhs_mask, rhs_offset + i) : true, - output_valid); - if (output_mask && !output_valid) cudf::clear_bit(output_mask, i); - } -} - } // namespace jit } // namespace binops } // namespace cudf diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp deleted file mode 100644 index 7d03ed8fdbca..000000000000 --- a/cpp/src/jit/cache.cpp +++ /dev/null @@ -1,136 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "runtime/context.hpp" - -#include -#include -#include - -#include - -#include - -namespace cudf { -namespace { - -// Get the directory in home to use for storing the cache -std::filesystem::path get_user_home_cache_dir() -{ - auto home_dir = std::getenv("HOME"); - if (home_dir != nullptr) { - return std::filesystem::path(home_dir) / ".cudf"; - } else { - return {}; - } -} - -// Default `LIBCUDF_KERNEL_CACHE_PATH` to `$HOME/.cudf/$CUDF_VERSION`. -// This definition can be overridden at compile time by specifying a -// `-DLIBCUDF_KERNEL_CACHE_PATH=/kernel/cache/path` CMake argument. -// Use `std::filesystem` for cross-platform path resolution and dir -// creation. This path is used in the `getCacheDir()` function below. -#if !defined(LIBCUDF_KERNEL_CACHE_PATH) -#define LIBCUDF_KERNEL_CACHE_PATH get_user_home_cache_dir() -#endif - -/** - * @brief Get the string path to the JITIFY kernel cache directory. - * - * This path can be overridden at runtime by defining an environment variable - * named `LIBCUDF_KERNEL_CACHE_PATH`. The value of this variable must be a path - * under which the process' user has read/write privileges. - * - * This function returns a path to the cache directory, creating it if it - * doesn't exist. - * - * The default cache directory is `$HOME/.cudf/$CUDF_VERSION`. If no overrides - * are used and if $HOME is not defined, returns an empty path and file - * caching is not used. - */ -std::filesystem::path get_cache_dir() -{ - // The environment variable always overrides the - // default/compile-time value of `LIBCUDF_KERNEL_CACHE_PATH` - auto kernel_cache_path_env = std::getenv("LIBCUDF_KERNEL_CACHE_PATH"); - auto kernel_cache_path = std::filesystem::path( - kernel_cache_path_env != nullptr ? kernel_cache_path_env : LIBCUDF_KERNEL_CACHE_PATH); - - // Cache path could be empty when env HOME is unset or LIBCUDF_KERNEL_CACHE_PATH is defined to be - // empty, to disallow use of file cache at runtime. - if (not kernel_cache_path.empty()) { - kernel_cache_path /= std::string{CUDF_STRINGIFY(CUDF_VERSION)}; - - // Make per device cache based on compute capability. This is to avoid multiple devices of - // different compute capability to access the same kernel cache. - int device = 0; - int cc_major = 0; - int cc_minor = 0; - CUDF_CUDA_TRY(cudaGetDevice(&device)); - CUDF_CUDA_TRY(cudaDeviceGetAttribute(&cc_major, cudaDevAttrComputeCapabilityMajor, device)); - CUDF_CUDA_TRY(cudaDeviceGetAttribute(&cc_minor, cudaDevAttrComputeCapabilityMinor, device)); - int const cc = cc_major * 10 + cc_minor; - - kernel_cache_path /= std::to_string(cc); - - try { - // `mkdir -p` the kernel cache path if it doesn't exist - std::filesystem::create_directories(kernel_cache_path); - } catch (std::exception const& e) { - // if directory creation fails for any reason, return empty path - return {}; - } - } - return kernel_cache_path; -} - -std::string get_program_cache_dir() -{ -#if defined(JITIFY_USE_CACHE) - return get_cache_dir().string(); -#else - return {}; -#endif -} - -std::size_t try_parse_numeric_env_var(char const* const env_name, std::size_t default_val) -{ - auto const value = std::getenv(env_name); - return value != nullptr ? std::stoull(value) : default_val; -} -} // namespace - -jitify2::ProgramCache<>& jit::program_cache::get(jitify2::PreprocessedProgramData const& preprog) -{ - CUDF_FUNC_RANGE(); - std::lock_guard const caches_lock(_caches_mutex); - - auto existing_cache = _caches.find(preprog.name()); - - if (existing_cache == _caches.end()) { - auto const kernel_limit_proc = - try_parse_numeric_env_var("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 10'000); - auto const kernel_limit_disk = - try_parse_numeric_env_var("LIBCUDF_KERNEL_CACHE_LIMIT_DISK", 100'000); - - // if kernel_limit_disk is zero, jitify will assign it the value of kernel_limit_proc. - // to avoid this, we treat zero as "disable disk caching" by not providing the cache dir. - auto const cache_dir = kernel_limit_disk == 0 ? std::string{} : get_program_cache_dir(); - - auto const res = - _caches.insert({preprog.name(), - std::make_unique>( - kernel_limit_proc, preprog, nullptr, cache_dir, kernel_limit_disk)}); - existing_cache = res.first; - } - - return *(existing_cache->second); -} - -jitify2::ProgramCache<>& jit::get_program_cache(jitify2::PreprocessedProgramData const& preprog) -{ - return cudf::get_context().program_cache().get(preprog); -} -} // namespace cudf diff --git a/cpp/src/jit/cache.hpp b/cpp/src/jit/cache.hpp deleted file mode 100644 index c130b953fc9f..000000000000 --- a/cpp/src/jit/cache.hpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#pragma GCC diagnostic ignored "-Wignored-attributes" // Work-around for JITIFY2's false-positive - // warnings when compiled with GCC13 - -#include - -#include - -#include -#include -#include - -namespace cudf { -namespace jit { - -class program_cache { - std::mutex _caches_mutex; - std::unordered_map>> _caches; - - public: - program_cache() = default; - program_cache(program_cache const&) = delete; - program_cache(program_cache&&) = delete; - program_cache& operator=(program_cache const&) = delete; - program_cache& operator=(program_cache&&) = delete; - ~program_cache() = default; - - jitify2::ProgramCache<>& get(jitify2::PreprocessedProgramData const& preprog); -}; - -jitify2::ProgramCache<>& get_program_cache(jitify2::PreprocessedProgramData const& preprog); - -} // namespace jit -} // namespace cudf diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 9541950ad785..117ed3940f73 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -7,7 +7,7 @@ #include -#include +#include namespace cudf { namespace jit { @@ -132,23 +132,31 @@ std::vector reflect_inputs( return reflections; } -jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, - std::string const& kernel_name, - std::string const& cuda_source) +kernel get_udf_kernel(std::string const& name, + std::string const& source_file, + std::string const& kernel_name, + std::string const& udf_cuda_source) { CUDF_FUNC_RANGE(); - int runtime_version; - CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); - int constexpr min_pch_runtime_version = 12800; // CUDA 12.8 + char const* include_names[] = {"cudf/detail/operation-udf.hpp"}; - std::vector options; - options.emplace_back("-arch=sm_."); + char const* sources[] = {udf_cuda_source.c_str()}; - if (runtime_version >= min_pch_runtime_version) { options.emplace_back("-pch"); } + int constexpr min_pch_runtime_version = 12800; // CUDA 12.8 + + int runtime_version; + CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); - return cudf::jit::get_program_cache(preprocessed_program_data) - .get_kernel(kernel_name, {}, {{"cudf/detail/operation-udf.hpp", cuda_source}}, options); + return get_kernel(name.c_str(), + "", // TODO: key + std::format("#include <{}>\n", source_file), + include_names, + sources, + kernel_name.c_str(), + /*use_cache=*/true, // TODO: use context config + /*use_pch=*/runtime_version >= min_pch_runtime_version, + true); } } // namespace jit diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index 5580d6a5463c..39c5b9258a8f 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -88,9 +89,10 @@ input_reflection reflect_input(std::variant con std::vector reflect_inputs( std::span const> inputs); -jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, - std::string const& kernel_name, - std::string const& cuda_source); +kernel get_udf_kernel(std::string const& name, + std::string const& source_file, + std::string const& kernel_name, + std::string const& udf_cuda_source); } // namespace jit } // namespace cudf diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index b89db8e36875..74cc524c5f01 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -257,10 +257,14 @@ int32_t get_current_device_physical_model() return props.major * 10 + props.minor; } -std::tuple compile_library_uncached(char const* name, - char const* cuda_code, - bool use_pch, - bool log_pch) +std::tuple compile_library_uncached( + char const* name, + char const* cuda_code, + std::span extra_header_include_names, + std::span extra_headers, + std::span name_expressions, + bool use_pch, + bool log_pch) { CUDF_FUNC_RANGE(); @@ -304,11 +308,13 @@ std::tuple compile_library_uncached(char const* name, options_cstr.emplace_back(option.c_str()); } - auto params = rtcx::compile_params{.name = name, - .source = cuda_code, - .headers = {}, - .options = options_cstr, - .target_type = rtcx::binary_type::CUBIN}; + auto params = rtcx::compile_params{.name = name, + .source = cuda_code, + .header_include_names = extra_header_include_names, + .headers = extra_headers, + .options = options_cstr, + .name_expressions = name_expressions, + .target_type = rtcx::binary_type::CUBIN}; auto cubin = rtcx::compile(params); @@ -330,12 +336,27 @@ std::tuple compile_library_uncached(char const* name, } // namespace -[[nodiscard]] rtcx::library get_library(std::string const& name, - std::string const& key, - std::string const& cuda_udf, - bool use_cache, - bool use_pch, - bool log_pch) +static rtcx::kernel_ref get_kernel(rtcx::library const& lib) +{ + auto kernels = lib->enumerate_kernels(); + CUDF_EXPECTS( + kernels.size() == 1, + +std::format("Expected exactly one kernel in compiled library, but found {}", kernels.size()), + std::runtime_error); + return kernels[0]; +} + +kernel::kernel(rtcx::library lib) : _library(std::move(lib)), _kernel(get_kernel(_library)) {} + +kernel get_kernel(std::string const& name, + std::string const& key, + std::string const& cuda_udf, + std::span header_include_names, + std::span headers, + char const* name_expression, + bool use_cache, + bool use_pch, + bool log_pch) { CUDF_FUNC_RANGE(); @@ -362,7 +383,9 @@ bundle={})***", auto cache_key_sha256 = hash_string(cache_key); auto compile = [&] { - return compile_library_uncached(name.c_str(), cuda_udf.c_str(), use_pch, log_pch); + char const* name_exprs[] = {name_expression}; + return compile_library_uncached( + name.c_str(), cuda_udf.c_str(), header_include_names, headers, name_exprs, use_pch, log_pch); }; if (!use_cache) { diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index 02356b02675a..b3937d46f572 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -6,6 +6,8 @@ #pragma once #include +#include + #include namespace CUDF_EXPORT cudf { @@ -27,11 +29,57 @@ struct [[nodiscard]] jit_bundle_t { [[nodiscard]] std::vector get_include_directories() const; }; -[[nodiscard]] rtcx::library get_library(std::string const& name, - std::string const& key, - std::string const& cuda_udf, - bool use_cache = true, - bool use_pch = true, - bool log_pch = false); +struct [[nodiscard]] kernel { + private: + rtcx::library _library; + rtcx::kernel_ref _kernel; + + public: + kernel(rtcx::library lib); + kernel(kernel const&) = default; + kernel(kernel&&) = default; + kernel& operator=(kernel const&) = default; + kernel& operator=(kernel&&) = default; + ~kernel() = default; + + rtcx::kernel_ref get() const { return _kernel; } + + rtcx::kernel_occupancy_config max_occupancy_config(size_t dynamic_shared_memory_bytes, + int32_t block_size_limit) const + { + return _kernel.max_occupancy_config(dynamic_shared_memory_bytes, block_size_limit); + } + + void launch(uint32_t grid_dim_x, + uint32_t grid_dim_y, + uint32_t grid_dim_z, + uint32_t block_dim_x, + uint32_t block_dim_y, + uint32_t block_dim_z, + uint32_t shared_mem_bytes, + rmm::cuda_stream_view stream, + std::span kernel_params) const + { + return _kernel.launch(grid_dim_x, + grid_dim_y, + grid_dim_z, + block_dim_x, + block_dim_y, + block_dim_z, + shared_mem_bytes, + stream.value(), + kernel_params.data()); + } +}; + +kernel get_kernel(std::string const& name, + std::string const& key, + std::string const& cuda_udf, + std::span header_include_names, + std::span headers, + char const* name_expression, + bool use_cache = true, + bool use_pch = true, + bool log_pch = false); } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/join/jit_filter_join_indices.cu b/cpp/src/join/jit_filter_join_indices.cu index 6d5c982c5a16..dd05c64c5ae8 100644 --- a/cpp/src/join/jit_filter_join_indices.cu +++ b/cpp/src/join/jit_filter_join_indices.cu @@ -81,14 +81,14 @@ jitify2::StringVec build_join_filter_template_params(std::vector co } // Build the JIT kernel for join filtering -jitify2::ConfiguredKernel build_join_filter_kernel(std::string const& predicate_code, - std::vector const& left_columns, - std::vector const& right_columns, - bool is_ptx, - bool has_user_data, - null_aware is_null_aware, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +kernel build_join_filter_kernel(std::string const& predicate_code, + std::vector const& left_columns, + std::vector const& right_columns, + bool is_ptx, + bool has_user_data, + null_aware is_null_aware, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); @@ -114,14 +114,14 @@ jitify2::ConfiguredKernel build_join_filter_kernel(std::string const& predicate_ jitify2::reflection::Template("cudf::join::jit::filter_join_kernel").instantiate(template_args); // Get compiled kernel - auto kernel = - cudf::jit::get_udf_kernel(*join_jit_filter_join_kernel_cu_jit, kernel_name, cuda_source); - - return kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); + return cudf::jit::get_udf_kernel("src/join/jit/filter_join_kernel.cu", // TODO: use actual name + "src/join/jit/filter_join_kernel.cu", + kernel_name, + cuda_source); } // Launch the JIT kernel for join filtering -void launch_join_filter_kernel(jitify2::ConfiguredKernel& kernel, +void launch_join_filter_kernel(kernel const& kernel, cudf::table_view const& left, cudf::table_view const& right, cudf::device_span left_indices, @@ -152,14 +152,15 @@ void launch_join_filter_kernel(jitify2::ConfiguredKernel& kernel, cudf::column_device_view_core const* right_tables_ptr = right_device_views.data(); void* user_data_ptr = user_data.value_or(nullptr); - std::array args{&left_span, - &right_span, - &left_tables_ptr, - &right_tables_ptr, - &predicate_results, - &user_data_ptr}; + void* args[] = {&left_span, + &right_span, + &left_tables_ptr, + &right_tables_ptr, + &predicate_results, + &user_data_ptr}; - kernel->launch_raw(args.data()); + auto cfg = kernel.max_occupancy_config(0, 0); + kernel.launch(cfg.min_grid_size, 1, 1, cfg.block_size, 1, 1, stream, args); } // Same join semantics handling as the AST version @@ -510,8 +511,10 @@ jit_filter_join_indices(cudf::table_view const& left, auto kernel_name = jitify2::reflection::Template("cudf::join::jit::filter_join_kernel").instantiate(template_args); auto kernel = - cudf::jit::get_udf_kernel(*join_jit_filter_join_kernel_cu_jit, kernel_name, cuda_source); - auto configured_kernel = kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); + cudf::jit::get_udf_kernel("src/join/jit/filter_join_kernel.cu", // TODO: use actual name + "src/join/jit/filter_join_kernel.cu", + kernel_name, + cuda_source); // Collect scalar columns to append to left device views so join_scalar_accessor // can read them at indices >= left.num_columns(). @@ -522,7 +525,7 @@ jit_filter_join_indices(cudf::table_view const& left, // Allocate and compute predicate results auto predicate_results = rmm::device_uvector(left_indices.size(), stream); - launch_join_filter_kernel(configured_kernel, + launch_join_filter_kernel(kernel, left, right, left_indices, diff --git a/cpp/src/librtcx/CMake/embed.cmake b/cpp/src/librtcx/CMake/embed.cmake index a14e19cb8ff9..811c13a29040 100644 --- a/cpp/src/librtcx/CMake/embed.cmake +++ b/cpp/src/librtcx/CMake/embed.cmake @@ -117,7 +117,7 @@ function(jit_embed) set(OUTPUT_DIR "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") set(CONFIGURED_EMBED_PY "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_embed.py") - set(EMBED_PY_IN "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.py.in") + set(EMBED_PY_IN "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.py") set(RTCX_EMBED_PY_ARG__ID "${TARGET}") set(RTCX_EMBED_PY_ARG__FILE_PATHS "${jitembed_${TARGET}_incdir__source_files}") diff --git a/cpp/src/librtcx/CMake/embed.py.in b/cpp/src/librtcx/CMake/embed.in.py similarity index 100% rename from cpp/src/librtcx/CMake/embed.py.in rename to cpp/src/librtcx/CMake/embed.in.py diff --git a/cpp/src/librtcx/README.md b/cpp/src/librtcx/README.md index 56d1c435bb6c..a6f1a00e9635 100644 --- a/cpp/src/librtcx/README.md +++ b/cpp/src/librtcx/README.md @@ -5,4 +5,4 @@ RTCX (runtime-compiler extended) is a wrapper around NVRTC and NVJitLink designe - User-controlled compilation, linking, caching, and pre-loading of CUDA kernels - Zero-copy interfaces to manage JIT compilation and linking - CMake script to embed **compressed** headers directly into an executable without incurring overhead at runtime on every compilation request -- Facilities to pre-load and teardown dynamic library dependencies (libcuda, libnvrtc, and libnvjitlink) +- Facilities to pre-load and teardown dynamic library dependencies (`libcuda`, `libnvrtc`, and `libnvJitlink`) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 221b8d24d787..c902c1903be4 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -17,20 +17,38 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#include extern "C" { #include } -#define RTCX_CONCATENATE_DETAIL(x, y) x##y -#define RTCX_CONCATENATE(x, y) RTCX_CONCATENATE_DETAIL(x, y) +#define RTCX_EXPECTS(_condition, _reason, _exception_type) \ + do { \ + if (!(_condition)) { \ + throw _exception_type{::std::format("RTCX failure at: {}:{}: {}", \ + ::std::source_location::current().file_name(), \ + ::std::source_location::current().line(), \ + (_reason))}; \ + } \ + } while (0) + +#define RTCX_FAIL(_reason, _exception_type) \ + do { \ + throw _exception_type{::std::format("RTCX failure at: {}:{}: {}", \ + ::std::source_location::current().file_name(), \ + ::std::source_location::current().line(), \ + (_reason))}; \ + } while (0) #define RTCX_CHECK_CUDA(...) \ do { \ @@ -93,12 +111,12 @@ namespace RTCX_EXPORT rtcx { void log_warning(std::string_view message) { - std::fprintf(stdout, "RTCX WARNING: %.*s\n", static_cast(message.size()), message.data()); + std::fprintf(stdout, "RTCX WARNING: %.*s\n", static_cast(message.size()), message.data()); } void log_error(std::string_view message) { - std::fprintf(stderr, "RTCX ERROR: %.*s\n", static_cast(message.size()), message.data()); + std::fprintf(stderr, "RTCX ERROR: %.*s\n", static_cast(message.size()), message.data()); } sha256_context::sha256_context() : ectx_(nullptr) @@ -597,6 +615,10 @@ std::vector compile(compile_params const& params) RTCX_DEFER([&] { nvrtc->DestroyProgram(&program); }); + for (auto* name_expr : params.name_expressions) { + RTCX_CHECK_NVRTC(params, program, nvrtc->AddNameExpression(program, name_expr)); + } + // TODO: log is printed twice when warnings are raised RTCX_CHECK_NVRTC( params, @@ -845,13 +867,10 @@ namespace { cache_t::cache_t(std::string cache_dir, cache_limits const& limits) : cache_dir_{std::move(cache_dir)}, limits_{limits}, - blobs_cache_{limits.num_blobs}, - libraries_cache_{limits.num_libraries}, + blobs_cache_{limits.num_mem_blobs}, + libraries_cache_{limits.num_mem_libraries}, tick_{0} { - RTCX_EXPECTS(limits.num_blobs >= 2, "Blob cache limit must be at least 2", std::logic_error); - RTCX_EXPECTS( - limits.num_libraries >= 2, "Library cache limit must be at least 2", std::logic_error); } std::string const& cache_t::get_cache_dir() { return cache_dir_; } @@ -864,24 +883,24 @@ std::optional blob_t::from_file(char const* path) if (errno == ENOENT) { return std::nullopt; } else { - throw_posix("Failed to open RTC cache file from disk", "open"); + throw_posix("Failed to open RTCX cache file from disk", "open"); } } auto file_size = lseek(fd, 0, SEEK_END); - if (file_size == -1) { throw_posix("Failed to determine size of RTC cache file", "lseek"); } + if (file_size == -1) { throw_posix("Failed to determine size of RTCX cache file", "lseek"); } void* map = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); - if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTC cache file", "mmap"); } + if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTCX cache file", "mmap"); } if (close(fd) == -1) { - throw_posix("Failed to close RTC cache file after memory-mapping", "close"); + throw_posix("Failed to close RTCX cache file after memory-mapping", "close"); } auto deleter = +[](u8 const* buffer, usize size) { if (munmap(static_cast(const_cast(buffer)), size) == -1) { - throw_posix("Failed to unmap RTC cache file from memory", "munmap"); + throw_posix("Failed to unmap RTCX cache file from memory", "munmap"); } }; @@ -908,49 +927,123 @@ std::optional get_disk_blob(std::string const& cache_dir, } } -/// @brief atomically writes a blob to disk by first writing to a temporary file and then renaming -/// it to the final path. -void add_blob_to_disk(std::string const& cache_dir, - std::string const& object_type, - sha256 const& sha, - std::span binary) +void evict_disk_entries(std::string const& cache_dir, u32 limit) { - char temp_path[] = "/tmp/rtcx-bin-XXXXXX"; + i32 dir = open(cache_dir.c_str(), O_RDONLY | O_DIRECTORY); - { - i32 fd = mkstemp(temp_path); - if (fd == -1) { throw_posix("Failed to create temporary file for RTC cache", "mkstemp"); } + if (dir == -1) { throw_posix("Failed to open RTCX cache directory for evicting", "open"); } - RTCX_DEFER([&] { - if (close(fd) == -1) { throw_posix("Failed to close temporary RTC cache file", "close"); } - }); + RTCX_DEFER([&] { close(dir); }); + + std::vector buffer; + buffer.resize(8192); + + std::vector paths; + std::vector access_times; + + isize num_read = 0; + + while ((num_read = syscall(SYS_getdents64, dir, buffer.data(), buffer.size())) > 0) { + isize byte_pos = 0; - if (write(fd, binary.data(), binary.size()) == -1) { - throw_posix("Failed to write RTC cache to temporary file", "write"); + while (byte_pos < num_read) { + auto* ent = reinterpret_cast(buffer.data() + byte_pos); + + if (memcmp(ent->d_name, ".", 2) != 0 && memcmp(ent->d_name, "..", 3) != 0) { + RTCX_EXPECTS(ent->d_type != DT_UNKNOWN, + "Found unknown directory entry type in RTCX cache dir", + std::runtime_error); + + if (ent->d_type == DT_REG) { + auto path = std::format("{}/{}", cache_dir, ent->d_name); + struct stat st; + if (stat(path.c_str(), &st) == -1 && errno != ENOENT) { + throw_posix("Failed to get RTCX cache file stats", "stat"); + } + + auto access_time = + std::chrono::seconds{st.st_atim.tv_sec} + std::chrono::nanoseconds{st.st_atim.tv_nsec}; + + paths.emplace_back(std::move(path)); + access_times.emplace_back(access_time); + } + } + + byte_pos += ent->d_reclen; } } - auto hex = sha.to_hex_string(); - auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); + if (num_read == -1) { + throw_posix("Failed to read RTCX cache directory for clearing", "getdents64"); + } - std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); + if (paths.size() < limit) { return; } - // rename is atomic, even if another process is performing the same operation - if (rename(temp_path, final_path.c_str()) == -1) { - auto errc = errno; + std::vector ranking_indices; + ranking_indices.resize(paths.size()); - if (errc == EEXIST) { - // another process has already created the file, so just remove our temp file - if (remove(temp_path) == -1) { - throw_posix("Failed to remove temporary RTC cache file", "remove"); + std::iota(ranking_indices.begin(), ranking_indices.end(), 0); + + std::sort(ranking_indices.begin(), ranking_indices.end(), [&](i32 a, i32 b) { + return access_times[a] < access_times[b]; + }); + + // evict half of the least recently accessed + auto num_evict = (limit == 0) ? paths.size() : ((limit + 1) / 2); + + for (auto index : std::span{ranking_indices}.subspan(0, num_evict)) { + if (unlink(paths[index].c_str()) == -1 && errno != ENOENT) { + throw_posix("Failed to evict RTCX cache file", "unlink"); + } + } +} + +/// @brief atomically writes a blob to disk by first writing to a temporary file and then renaming +/// it to the final path. +void cache_blob_to_disk(std::string const& cache_dir, + std::string const& object_type, + sha256 const& sha, + std::span binary, + u32 limit) +{ + if (limit > 0) { + char temp_path[] = "/tmp/rtcx-bin-XXXXXX"; + + { + i32 fd = mkstemp(temp_path); + if (fd == -1) { throw_posix("Failed to create temporary file for RTCX cache", "mkstemp"); } + + RTCX_DEFER([&] { + if (close(fd) == -1) { throw_posix("Failed to close temporary RTCX cache file", "close"); } + }); + + if (write(fd, binary.data(), binary.size()) == -1) { + throw_posix("Failed to write RTCX cache to temporary file", "write"); + } + } + + auto hex = sha.to_hex_string(); + auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); + + std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); + + // rename is atomic, even if another process is performing the same operation + if (rename(temp_path, final_path.c_str()) == -1) { + if (errno == EEXIST) { + // another process has already created the file, so just remove our temp file + if (remove(temp_path) == -1) { + throw_posix("Failed to remove temporary RTCX cache file", "remove"); + } + return; + } else { + throw_posix(std::format("Failed to move temporary RTCX cache file to final location ({})", + final_path), + "rename"); } - return; - } else { - throw_posix( - std::format("Failed to move temporary RTC cache file to final location ({})", final_path), - "rename"); } } + + evict_disk_entries(cache_dir, limit); } } // namespace @@ -1011,7 +1104,7 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil promise.set_value(result); // store result to disk - add_blob_to_disk(cache_dir_, "blob", sha, result->view()); + cache_blob_to_disk(cache_dir_, "blob", sha, result->view(), limits_.num_disk_entries); return ret_fut; } @@ -1081,7 +1174,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, promise.set_value(library); // store result to disk - add_blob_to_disk(cache_dir_, "library", sha, blob->view()); + cache_blob_to_disk(cache_dir_, "library", sha, blob->view(), limits_.num_disk_entries); return ret_fut; } @@ -1136,45 +1229,46 @@ void cache_t::clear_memory_store() void cache_t::clear_disk_store() { - DIR* dir = opendir(cache_dir_.c_str()); - - if (dir == nullptr) { throw_posix("Failed to open RTC cache directory for clearing", "opendir"); } + i32 dir = open(cache_dir_.c_str(), O_RDONLY | O_DIRECTORY); - RTCX_DEFER([&] { closedir(dir); }); + if (dir == -1) { throw_posix("Failed to open RTCX cache directory for clearing", "opendir"); } - errno = 0; // reset errno before reading + RTCX_DEFER([&] { close(dir); }); - dirent* entry_iter = nullptr; + std::vector buffer; + buffer.resize(8192); std::vector entry_path; - entry_path.resize(PATH_MAX + 1); + entry_path.resize(4096); - while (true) { - entry_iter = readdir(dir); + isize num_read = 0; - if (entry_iter == nullptr) { - if (errno != 0) { - throw_posix("Failed to read RTC cache directory for clearing", "readdir"); - } else { - break; - } - } + while ((num_read = syscall(SYS_getdents64, dir, buffer.data(), buffer.size())) > 0) { + isize byte_pos = 0; - struct stat entry_stat; + while (byte_pos < num_read) { + auto* ent = reinterpret_cast(buffer.data() + byte_pos); - if (lstat(entry_path.data(), &entry_stat) == -1) { - throw_posix("Failed to get file status for RTC cache clearing", "lstat"); - } + if (memcmp(ent->d_name, ".", 2) != 0 && memcmp(ent->d_name, "..", 3) != 0) { + RTCX_EXPECTS(ent->d_type != DT_UNKNOWN, + "Found unknown directory entry type in RTCX cache dir", + std::runtime_error); + + if (ent->d_type == DT_REG) { + snprintf(entry_path.data(), entry_path.size(), "%s/%s", cache_dir_.c_str(), ent->d_name); - if (S_ISREG(entry_stat.st_mode)) { - if (unlink(entry_path.data()) == -1) { - throw_posix("Failed to unlink RTC cache file during clearing", "unlink"); + if (unlink(entry_path.data()) == -1 && errno != ENOENT) { + throw_posix("Failed to unlink RTCX cache file during clearing", "unlink"); + } + } } - } - // reset errno for next iteration - errno = 0; + byte_pos += ent->d_reclen; + } } - return; + if (num_read == -1) { + throw_posix("Failed to read RTCX cache directory for clearing", "getdents64"); + } } + } // namespace RTCX_EXPORT rtcx diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index ad5a61fc3c63..7a9c3e59c5a7 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -28,24 +27,6 @@ #define RTCX_DEFER__CONCATENATE(x, y) RTCX_DEFER__CONCATENATE_DETAIL(x, y) #define RTCX_DEFER(...) ::rtcx::defer RTCX_DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) -#define RTCX_EXPECTS(_condition, _reason, _exception_type) \ - do { \ - if (!(_condition)) { \ - throw _exception_type{::std::format("RTCX failure at: {}:{}: {}", \ - ::std::source_location::current().file_name(), \ - ::std::source_location::current().line(), \ - (_reason))}; \ - } \ - } while (0) - -#define RTCX_FAIL(_reason, _exception_type) \ - do { \ - throw _exception_type{::std::format("RTCX failure at: {}:{}: {}", \ - ::std::source_location::current().file_name(), \ - ::std::source_location::current().line(), \ - (_reason))}; \ - } while (0) - extern "C" { typedef struct evp_md_ctx_st EVP_MD_CTX; @@ -65,7 +46,9 @@ using i8 = std::int8_t; using i16 = std::int16_t; using i32 = std::int32_t; using i64 = std::int64_t; -using iszie = std::ptrdiff_t; +using isize = std::ptrdiff_t; +using f32 = float; +using f64 = double; inline constexpr usize CACHELINE_ALIGNMENT = 64; // = std::hardware_destructive_interference_size */ @@ -394,14 +377,15 @@ struct [[nodiscard]] compile_params { std::span headers = {}; /** - * @brief Sizes of each header file provided in the `headers` field + * @brief NVRTC compile options */ - std::span header_sizes = {}; + std::span options = {}; /** - * @brief NVRTC compile options + * @brief The name expressions of the kernel entry points to be compiled, used for retrieving + * kernel references after compilation. */ - std::span options = {}; + std::span name_expressions = {}; /** * @brief Desired output binary type (e.g. PTX, CUBIN, etc.) @@ -433,7 +417,6 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { explicit lru_memory_cache(usize limit) : entries_{}, limit_{limit} { // reserve space to avoid rehashing - RTCX_EXPECTS(limit > 0, "Cache limit must be greater than 0", std::logic_error); entries_.reserve(limit * 2); } @@ -441,7 +424,7 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { { if (entries_.empty()) { return; } - auto num_to_purge = std::max(entries_.size() / 2, static_cast(1)); + auto num_to_purge = (entries_.size() + 1) / 2; std::vector> rankings; rankings.reserve(entries_.size()); @@ -464,6 +447,8 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { void insert(sha256 const& sha, T&& value, u64 tick) { + if (limit_ == 0) { return; } + if ((entries_.size() + 1) > limit_) { purge(); } entries_.emplace(sha, entry{tick, std::move(value)}); @@ -482,8 +467,9 @@ struct [[nodiscard]] cache_stats { }; struct [[nodiscard]] cache_limits { - u32 num_blobs = 1024; - u32 num_libraries = 1024; + u32 num_mem_blobs = 16'384; + u32 num_mem_libraries = 16'384; + u32 num_disk_entries = 131'072; }; struct cache_stats_counter { @@ -702,4 +688,267 @@ void initialize(); */ void teardown(); +namespace reflection { + +template +struct NonType {}; + +// Forward declaration. +template +inline std::string reflect(const T& value); +template +inline std::string reflect(); + +namespace detail { + +template +struct ValueStringImpl { + static std::string value(const T& x) { return std::to_string(x); } +}; + +template +struct ValueStringImpl::value>::type> { + static std::string value(const T& x) { return x ? "true" : "false"; } +}; + +template +struct ValueStringImpl::value>::type> { + static std::string value(const T& x) + { + using UnderlyingT = typename std::underlying_type::type; + return ValueStringImpl::value(static_cast(x)); + } +}; + +template +inline std::string value_string(const T& x) +{ + return ValueStringImpl::value(x); +} + +// Returns the demangled name corresponding to the given typeinfo structure. +inline std::string get_type_name(const std::type_info& typeinfo) +{ + const char* mangled_name = typeinfo.name(); + size_t bufsize = 0; + char* buf = nullptr; + int status; + auto demangled_ptr = std::unique_ptr( + abi::__cxa_demangle(mangled_name, buf, &bufsize, &status), std::free); + // clang-format off + switch (status) { + case 0: return demangled_ptr.get(); // Demangled successfully + case -2: return mangled_name; // Interpret as plain unmangled name + case -1: // fall-through // Memory allocation failure + case -3: // fall-through // Invalid argument + default: return {}; + } + // clang-format on +} + +template +class JitifyTypeNameWrapper_ {}; + +// Returns the demangled name of the given type. +template +inline std::string get_type_name() +{ + // WAR for typeid discarding cv qualifiers on value-types. + // Wraps type in dummy template class to preserve cv-qualifiers, then strips + // off the wrapper from the resulting string. + std::string wrapped_name = get_type_name(typeid(JitifyTypeNameWrapper_)); + // Note: The reflected name of this class also has namespace prefixes. + const std::string wrapper_class_name = "JitifyTypeNameWrapper_<"; + size_t start = wrapped_name.find(wrapper_class_name); + if (start == std::string::npos) return {}; // Unexpected error + start += wrapper_class_name.size(); + return wrapped_name.substr(start, wrapped_name.size() - (start + 1)); +} + +template +struct ReflectType { + const std::string& operator()() const + { + // Storing this statically means it is cached after the first call. + static const std::string type_name = get_type_name(); + return type_name; + } +}; + +template +struct ReflectType> { + std::string operator()() const { return reflect(VALUE); } +}; + +} // namespace detail + +/*! A wrapper used for representing types as values. */ +template +struct Type {}; + +/*! Create an Instance object that contains a const reference to the + * value. We use this to wrap abstract objects from which we want to extract + * their type at runtime (e.g., derived type). This is used to facilitate + * templating on derived type when all we know at compile time is abstract + * type. + */ +template +struct Instance { + const T& value; + Instance(const T& value_arg) : value(value_arg) {} +}; + +/*! Create an Instance object from which we can extract the value's run-time + * type. + * \param value The const value to be captured. + */ +template +inline Instance instance_of(T const& value) +{ + return Instance(value); +} + +/*! Generate a code-string for a type. + * \code{.cpp}reflect() --> "float"\endcode + */ +template +inline std::string reflect() +{ + return detail::ReflectType()(); +} + +/*! Generate a code-string for a value. + * \code{.cpp}reflect(3.14f) --> "(float)3.14"\endcode + */ +template +inline std::string reflect(const T& value) +{ + return "(" + reflect() + ")" + detail::value_string(value); +} + +/*! Generate a code-string for an integer non-type template argument + * (via implicit conversion to int64_t). + * \code{.cpp}reflect<7>() --> "(int64_t)7"\endcode + */ +template +inline std::string reflect() +{ + return reflect>(); +} + +/*! Generate a code-string for a generic non-type template argument. + * \code{.cpp} reflect() --> "(int)7" \endcode + */ +template +inline std::string reflect() +{ + return reflect>(); +} + +/*! Generate a code-string for a type wrapped as a Type instance. + * \code{.cpp}reflect(Type()) --> "float"\endcode + */ +template +inline std::string reflect(Type) +{ + return reflect(); +} + +/*! Generate a code-string for a type wrapped as an Instance instance. + * \code{.cpp}reflect(Instance(3.1f)) --> "float"\endcode + * or more simply when passed to a instance_of helper + * \code{.cpp}reflect(instance_of(3.1f)) --> "float"\endcodei + * This is specifically for the case where we want to extract the run-time + * type, i.e., derived type, of an object pointer. + */ +template +inline std::string reflect(const Instance& value) +{ + return detail::get_type_name(typeid(value.value)); +} + +// TODO: Would there ever be a need to reflect a string literal? +/*! Use an existing code string as-is. */ +inline std::string reflect(const std::string& s) { return s; } +/*! Use an existing code string as-is. */ +inline const char* reflect(const char* s) { return s; } +#if JITIFY_CPLUSPLUS >= 201703L +/*! Use an existing code string as-is. */ +inline std::string_view reflect(std::string_view s) { return s; } +#endif + +/*! Create a Type object representing a value's type. + * \code{.cpp}type_of(3.14f) -> Type()\endcode + * \param [unnamed] The value whose type is to be captured. + */ +template +inline Type type_of(T&) +{ + return Type(); +} + +/*! Create a Type object representing a value's type. + * \param [unnamed] The const value whose type is to be captured. + */ +template +inline Type type_of(const T&) +{ + return Type(); +} + +/*! Generate a code-string for a template instantiation. */ +inline std::string reflect_template(const StringVec& args) +{ + // Note: The space in " >" is a WAR to avoid '>>' appearing + return jitify2::detail::string_join(args, ",", "<", " >"); +} + +/*! Generate a code-string for a template instantiation. */ +template +inline std::string reflect_template() +{ + return reflect_template({reflect()...}); +} + +/*! Generate a code-string for a template instantiation. */ +template +inline std::string reflect_template(const Args&... args) +{ + return reflect_template({reflect(args)...}); +} + +/*! Convenience class for generating code-strings for template instantiations. + */ +class Template { + std::string name_; + + public: + /*! Construct the class. + * \param name The name of the template. + */ + Template(StringRef name) : name_(name) {} + + /*! Generate a code-string for an instantiation of the template. */ + std::string instantiate(const StringVec& template_args = {}) const + { + return name_ + reflect_template(template_args); + } + + /*! Generate a code-string for an instantiation of the template. */ + template + std::string instantiate() const + { + return name_ + reflect_template(); + } + + /*! Generate a code-string for an instantiation of the template. */ + template + std::string instantiate(const TemplateArgs&... targs) const + { + return name_ + reflect_template(targs...); + } +}; + +} // namespace reflection + } // namespace RTCX_EXPORT rtcx diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index 22365d92972e..f8cbe6991abd 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -82,17 +82,30 @@ std::unique_ptr rolling_window_udf(column_view const& input, preceding_window_str.c_str(), following_window_str.c_str()); - cudf::jit::get_udf_kernel(*rolling_jit_kernel_cu_jit, kernel_reflection, cuda_source) - ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()) - ->launch(input.size(), - cudf::jit::get_data_ptr(input), - input.null_mask(), - cudf::jit::get_data_ptr(output_view), - output_view.null_mask(), - device_valid_count.data(), - preceding_window, - following_window, - min_periods); + cudf::size_type nrows_arg = input.size(); + auto incol_arg = cudf::jit::get_data_ptr(input); + auto incol_valid_arg = input.null_mask(); + auto outcol_arg = cudf::jit::get_data_ptr(output->mutable_view()); + auto outcol_valid_arg = output_view.null_mask(); + cudf::size_type* output_valid_count_arg = device_valid_count.data(); + auto preceding_window_arg = preceding_window; + auto following_window_arg = following_window; + auto min_periods_arg = min_periods; + + void* args[] = {&nrows_arg, + &incol_arg, + &incol_valid_arg, + &outcol_arg, + &outcol_valid_arg, + &output_valid_count_arg, + &preceding_window_arg, + &following_window_arg, + &min_periods_arg}; + + auto kernel = cudf::jit::get_udf_kernel( + "src/rolling/jit/kernel.cu", "src/rolling/jit/kernel.cu", kernel_reflection, cuda_source); + auto cfg = kernel.max_occupancy_config(0, 0); + kernel.launch(cfg.min_grid_size, 1, 1, cfg.block_size, 1, 1, 0, stream, args); output->set_null_count(output->size() - device_valid_count.value(stream)); diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index a9999c3a1865..8a3e0b61f2cf 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -7,19 +7,19 @@ #include "io/comp/nvcomp_adapter.hpp" #include "io/utilities/getenv_or.hpp" -#include "jit/cache.hpp" #include "jit/jit.hpp" #include "librtcx/rtcx.hpp" #include #include +#include #include namespace cudf { context::context(context_config cfg, init_flags flags) - : _config{std::move(cfg)}, _program_cache_init_flag{}, _program_cache{nullptr} + : _config{std::move(cfg)}, _jit_cache_init_flag{} { initialize_components(flags); } @@ -28,46 +28,33 @@ void context::ensure_nvcomp_loaded() { io::detail::nvcomp::load_nvcomp_library() void context::ensure_jit_cache_initialized() { - std::call_once(_program_cache_init_flag, - [&]() { _program_cache = std::make_unique(); }); -} - -void context::ensure_rtc_cache_initialized() -{ - std::call_once(_rtc_cache_init_flag, [&]() { - // make sure the rtc cache directory exists + std::call_once(_jit_cache_init_flag, [&]() { + // make sure the required directories exist std::filesystem::create_directories(_config.rtc_cache_dir); - _rtc_cache = std::make_unique(_config.rtc_cache_dir, rtcx::cache_limits{}); - }); -} - -void context::ensure_jit_bundle_initialized() -{ - // note that jit_bundle depends on rtc_cache, so we ensure rtc_cache is initialized first. - ensure_rtc_cache_initialized(); - std::call_once(_jit_bundle_init_flag, [&]() { - // make sure the jit bundle directory exists std::filesystem::create_directories(_config.jit_bundle_dir); std::filesystem::create_directories(_config.jit_pch_dir); + + _rtc_cache = std::make_unique( + _config.rtc_cache_dir, + rtcx::cache_limits{.num_mem_blobs = _config.kernel_cache_limit_process, + .num_mem_libraries = _config.kernel_cache_limit_process, + .num_disk_entries = _config.kernel_cache_limit_disk}); + // note that jit_bundle depends on rtc_cache, so we ensure rtc_cache is initialized first. _jit_bundle = std::make_unique(_config.jit_bundle_dir, *_rtc_cache); }); } -jit::program_cache& context::program_cache() -{ - ensure_jit_cache_initialized(); - return *_program_cache; -} +context::~context() { rtcx::teardown(); } rtcx::cache_t& context::rtc_cache() { - ensure_rtc_cache_initialized(); + ensure_jit_cache_initialized(); return *_rtc_cache; } jit_bundle_t& context::jit_bundle() { - ensure_jit_bundle_initialized(); + ensure_jit_cache_initialized(); return *_jit_bundle; } @@ -82,8 +69,6 @@ void context::initialize_components(init_flags flags) if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { rtcx::initialize(); ensure_jit_cache_initialized(); - ensure_rtc_cache_initialized(); - ensure_jit_bundle_initialized(); } if (has_flag(flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } @@ -128,8 +113,10 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { std::call_once(*_context_init_flag, [&]() { - bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); - bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); + bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); + bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); + auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16384U); + auto kernel_cache_limit_disk = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_DISK", 131'072U); flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); @@ -137,13 +124,13 @@ void initialize(init_flags flags) auto rtc_cache_dir = get_rtc_cache_dir(); auto jit_pch_dir = get_jit_pch_dir(); - context_config cfg{ - .dump_codegen = dump_codegen, - .use_jit = use_jit, - .rtc_cache_dir = rtc_cache_dir, - .jit_bundle_dir = jit_bundle_dir, - .jit_pch_dir = jit_pch_dir, - }; + context_config cfg{.dump_codegen = dump_codegen, + .use_jit = use_jit, + .rtc_cache_dir = rtc_cache_dir, + .jit_bundle_dir = jit_bundle_dir, + .jit_pch_dir = jit_pch_dir, + .kernel_cache_limit_process = kernel_cache_limit_process, + .kernel_cache_limit_disk = kernel_cache_limit_disk}; _context.emplace(cfg, flags); }); diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 1bd754ebc9aa..ae63bab44bb6 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -24,11 +24,13 @@ class program_cache; class jit_bundle_t; struct [[nodiscard]] context_config { - bool dump_codegen = false; - bool use_jit = false; - std::string rtc_cache_dir = {}; - std::string jit_bundle_dir = {}; - std::string jit_pch_dir = {}; + bool dump_codegen = false; + bool use_jit = false; + std::string rtc_cache_dir = {}; + std::string jit_bundle_dir = {}; + std::string jit_pch_dir = {}; + uint32_t kernel_cache_limit_process = 0; + uint32_t kernel_cache_limit_disk = 0; }; /// @brief The context object contains global state internal to CUDF. @@ -37,11 +39,8 @@ struct [[nodiscard]] context_config { class context { private: context_config _config; - std::once_flag _program_cache_init_flag; - std::unique_ptr _program_cache; - std::once_flag _rtc_cache_init_flag; + std::once_flag _jit_cache_init_flag; std::unique_ptr _rtc_cache; - std::once_flag _jit_bundle_init_flag; std::unique_ptr _jit_bundle; private: @@ -49,19 +48,13 @@ class context { void ensure_jit_cache_initialized(); - void ensure_rtc_cache_initialized(); - - void ensure_jit_bundle_initialized(); - public: context(context_config cfg = {}, init_flags flags = init_flags::DEFAULT); context(context const&) = delete; context& operator=(context const&) = delete; context(context&&) = delete; context& operator=(context&&) = delete; - ~context() = default; - - jit::program_cache& program_cache(); + ~context(); rtcx::cache_t& rtc_cache(); diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index c909b94efe32..8cbfccd783ed 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include @@ -36,33 +36,32 @@ namespace transformation { namespace jit { namespace { -jitify2::StringVec build_jit_template_params(null_aware is_null_aware, - bool may_evaluate_null, - bool has_user_data, - std::span span_outputs, - std::span column_outputs, - std::span inputs) +auto build_jit_template_params(null_aware is_null_aware, + bool may_evaluate_null, + bool has_user_data, + std::span span_outputs, + std::span column_outputs, + std::span inputs) { - jitify2::StringVec tparams; - - tparams.emplace_back(jitify2::reflection::reflect(is_null_aware)); - tparams.emplace_back(jitify2::reflection::reflect(may_evaluate_null)); - tparams.emplace_back(jitify2::reflection::reflect(has_user_data)); + std::vector tparams; + tparams.emplace_back(rtcx::reflect::value("cudf::null_aware", is_null_aware)); + tparams.emplace_back(rtcx::reflect::value(may_evaluate_null)); + tparams.emplace_back(rtcx::reflect::value(has_user_data)); std::transform(thrust::counting_iterator(0), thrust::counting_iterator(span_outputs.size()), std::back_inserter(tparams), [&](auto i) { - return jitify2::reflection::Template("cudf::jit::span_accessor") - .instantiate(span_outputs[i], i); + return rtcx::reflect::templated("cudf::jit::span_accessor", + {span_outputs[i], std::to_string(i)}); }); std::transform(thrust::counting_iterator(0), thrust::counting_iterator(column_outputs.size()), std::back_inserter(tparams), [&](auto i) { - return jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate(column_outputs[i], i); + return rtcx::reflect::templated("cudf::jit::column_accessor", + {column_outputs[i], std::to_string(i)}); }); std::transform(thrust::counting_iterator(0), @@ -73,17 +72,16 @@ jitify2::StringVec build_jit_template_params(null_aware is_null_aware, return tparams; } -jitify2::ConfiguredKernel build_transform_kernel( - std::string_view kernel_name, - std::span output_columns, - InputsView inputs, - null_aware is_null_aware, - bool may_evaluate_null, - bool has_user_data, - std::string const& udf, - cudf::udf_source_type source_type, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +kernel build_transform_kernel(std::string_view kernel_name, + std::span output_columns, + InputsView inputs, + null_aware is_null_aware, + bool may_evaluate_null, + bool has_user_data, + std::string const& udf, + cudf::udf_source_type source_type, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { auto output_typenames = cudf::jit::output_type_names(output_columns); auto input_typenames = cudf::jit::input_type_names(inputs); @@ -97,25 +95,25 @@ jitify2::ConfiguredKernel build_transform_kernel( cudf::jit::build_ptx_params(output_typenames, input_typenames, has_user_data)) : cudf::jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); - auto kernel_reflection = - jitify2::reflection::Template(kernel_name) - .instantiate(build_jit_template_params( - is_null_aware, may_evaluate_null, has_user_data, {}, output_typenames, input_reflections)); + auto kernel_reflection = rtcx::reflect::templated( + kernel_name, + build_jit_template_params( + is_null_aware, may_evaluate_null, has_user_data, {}, output_typenames, input_reflections)); - return cudf::jit::get_udf_kernel(*transform_jit_kernel_cu_jit, kernel_reflection, cuda_source) - ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); + return cudf::jit::get_udf_kernel( + "src/transform/jit/kernel.cu", "src/transform/jit/kernel.cu", kernel_reflection, cuda_source); } -jitify2::ConfiguredKernel build_span_kernel(std::string_view kernel_name, - std::span span_outputs, - InputsView inputs, - null_aware is_null_aware, - bool may_evaluate_null, - bool has_user_data, - std::string const& udf, - cudf::udf_source_type source_type, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +kernel build_span_kernel(std::string_view kernel_name, + std::span span_outputs, + InputsView inputs, + null_aware is_null_aware, + bool may_evaluate_null, + bool has_user_data, + std::string const& udf, + cudf::udf_source_type source_type, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); auto output_typenames = span_outputs; @@ -130,13 +128,13 @@ jitify2::ConfiguredKernel build_span_kernel(std::string_view kernel_name, cudf::jit::build_ptx_params(output_typenames, input_typenames, has_user_data)) : cudf::jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); - auto kernel_reflection = - jitify2::reflection::Template(kernel_name) - .instantiate(build_jit_template_params( - is_null_aware, may_evaluate_null, has_user_data, span_outputs, {}, input_reflections)); + auto kernel_reflection = rtcx::reflect::templated( + kernel_name, + build_jit_template_params( + is_null_aware, may_evaluate_null, has_user_data, span_outputs, {}, input_reflections)); - return cudf::jit::get_udf_kernel(*transform_jit_kernel_cu_jit, kernel_reflection, cuda_source) - ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); + return cudf::jit::get_udf_kernel( + "src/transform/jit/kernel.cu", "src/transform/jit/kernel.cu", kernel_reflection, cuda_source); } column_view to_column_view(column_view const& col) { return col; } @@ -163,7 +161,7 @@ auto to_device_output_arg(std::span outputs, outputs, stream, mr); } -void launch_column_output_kernel(jitify2::ConfiguredKernel& kernel, +void launch_column_output_kernel(kernel const& kernel, std::span output_columns, InputsView inputs, std::optional intermediate_null_mask, @@ -179,13 +177,14 @@ void launch_column_output_kernel(jitify2::ConfiguredKernel& kernel, bool* p_intermediate_null_mask = intermediate_null_mask.value_or(nullptr); void* p_user_data = user_data.value_or(nullptr); - std::array args{&p_outputs, &p_inputs, &p_intermediate_null_mask, &p_user_data}; + void* args[] = {&p_outputs, &p_inputs, &p_intermediate_null_mask, &p_user_data}; - kernel->launch_raw(args.data()); + auto cfg = kernel.max_occupancy_config(0, 0); + kernel.launch(cfg.min_grid_size, 1, 1, cfg.block_size, 1, 1, 0, stream, args); } template -void launch_span_kernel(jitify2::ConfiguredKernel& kernel, +void launch_span_kernel(kernel const& kernel, cudf::jit::device_optional_span const& output, InputsView inputs, std::optional intermediate_null_mask, @@ -202,9 +201,11 @@ void launch_span_kernel(jitify2::ConfiguredKernel& kernel, bool* p_intermediate_null_mask = intermediate_null_mask.value_or(nullptr); void* p_user_data = user_data.value_or(nullptr); - std::array args{&p_outputs, &p_inputs, &p_intermediate_null_mask, &p_user_data}; + void* args[] = {&p_outputs, &p_inputs, &p_intermediate_null_mask, &p_user_data}; - kernel->launch_raw(args.data()); + auto kernel_ref = kernel.get(); + auto cfg = kernel_ref.max_occupancy_config(0, 0); + kernel_ref.launch(cfg.min_grid_size, 1, 1, cfg.block_size, 1, 1, 0, stream, args); } std::tuple and_null_mask(size_type row_size, From f708ff9929d8add30648c31b92843319551a1364 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 12 Mar 2026 01:35:34 +0000 Subject: [PATCH 070/254] initial diff --- .../cudf/column/column_device_view.cuh | 40 +- .../cudf/column/column_device_view_base.cuh | 79 +- .../cudf/strings/detail/strings_children.cuh | 30 +- .../detail/strings_column_factories.cuh | 35 +- cpp/include/cudf/transform.hpp | 27 +- cpp/include/cudf/utilities/bit.hpp | 4 +- cpp/src/column/column_device_view.cu | 21 +- cpp/src/jit/column_accessor.cuh | 125 +-- ...rs.cuh => column_device_view_wrappers.cuh} | 71 +- cpp/src/jit/helpers.cpp | 8 +- cpp/src/jit/helpers.hpp | 3 +- cpp/src/jit/span.cuh | 3 +- cpp/src/jit/sync.cuh | 44 + cpp/src/jit/transform_udf.cuh | 62 +- cpp/src/jit/type_list.cuh | 4 +- cpp/src/join/jit/filter_join_kernel.cu | 3 +- cpp/src/transform/jit/kernel.cu | 45 +- cpp/src/transform/transform.cu | 866 ++++++++++++------ .../integration/unary_transform_test.cpp | 162 +++- 19 files changed, 1070 insertions(+), 562 deletions(-) rename cpp/src/jit/{wrappers.cuh => column_device_view_wrappers.cuh} (59%) create mode 100644 cpp/src/jit/sync.cuh diff --git a/cpp/include/cudf/column/column_device_view.cuh b/cpp/include/cudf/column/column_device_view.cuh index 8ead6e279882..a580c2541c91 100644 --- a/cpp/include/cudf/column/column_device_view.cuh +++ b/cpp/include/cudf/column/column_device_view.cuh @@ -92,7 +92,7 @@ class alignas(16) column_device_view : public column_device_view_core { this->null_count(), this->null_mask(), this->offset() + offset, - static_cast(d_children), + static_cast(_children), this->num_child_columns()}; } @@ -529,7 +529,7 @@ class alignas(16) column_device_view : public column_device_view_core { */ [[nodiscard]] __device__ column_device_view child(size_type child_index) const noexcept { - return static_cast(d_children)[child_index]; + return static_cast(_children)[child_index]; } /** @@ -539,7 +539,7 @@ class alignas(16) column_device_view : public column_device_view_core { */ [[nodiscard]] __device__ device_span children() const noexcept { - return {static_cast(d_children), static_cast(_num_children)}; + return {static_cast(_children), static_cast(_num_children)}; } /** @@ -736,7 +736,7 @@ class alignas(16) mutable_column_device_view : public mutable_column_device_view */ [[nodiscard]] __device__ mutable_column_device_view child(size_type child_index) const noexcept { - return static_cast(d_children)[child_index]; + return static_cast(_children)[child_index]; } /** @@ -760,6 +760,8 @@ class alignas(16) mutable_column_device_view : public mutable_column_device_view * @param offset The index of the first element in the column * @param children Pointer to the device memory containing child data * @param num_children The number of child columns + * @return A `mutable_column_device_view` instance representing the specified column data + * in device memory * */ static auto from_parts(data_type type, @@ -791,14 +793,30 @@ class alignas(16) mutable_column_device_view : public mutable_column_device_view *`mutable_column_device_view::create()` function should be used. */ mutable_column_device_view(mutable_column_view source); -}; - -static_assert(sizeof(column_device_view) == sizeof(column_device_view_core), - "column_device_view and raw_column_device_view must be bitwise-compatible"); -static_assert( - sizeof(mutable_column_device_view) == sizeof(mutable_column_device_view_core), - "mutable_column_device_view and raw_mutable_column_device_view must be bitwise-compatible"); + /** + * @brief Creates an instance of this class using pre-existing device memory pointers to data, + * nullmask, and offset. + * + * @param type The type of the column + * @param size The number of elements in the column + * @param data Pointer to the device memory containing the data + * @param null_mask Pointer to the device memory containing the null bitmask + * @param offset The index of the first element in the column + * @param children Pointer to the device memory containing child data + * @param num_children The number of child columns + */ + CUDF_HOST_DEVICE mutable_column_device_view(data_type type, + size_type size, + void const* data, + bitmask_type const* null_mask, + size_type offset, + mutable_column_device_view* children, + size_type num_children) + : mutable_column_device_view_core{type, size, data, null_mask, offset, children, num_children} + { + } +}; namespace detail { diff --git a/cpp/include/cudf/column/column_device_view_base.cuh b/cpp/include/cudf/column/column_device_view_base.cuh index 639602e778fb..39cd4878c258 100644 --- a/cpp/include/cudf/column/column_device_view_base.cuh +++ b/cpp/include/cudf/column/column_device_view_base.cuh @@ -269,10 +269,13 @@ class alignas(16) column_device_view_base { data_type _type{type_id::EMPTY}; ///< Element type cudf::size_type _size{}; ///< Number of elements void const* _data{}; ///< Pointer to device memory containing elements + size_type _null_count{}; ///< The number of nulls bitmask_type const* _null_mask{}; ///< Pointer to device memory containing ///< bitmask representing null elements. size_type _offset{}; ///< Index position of the first element. ///< Enables zero-copy slicing + void* _children{}; ///< Array of `column_device_view` objects in device memory. + size_type _num_children{}; ///< The number of child columns /** * @brief Constructs a column with the specified type, size, data, nullmask and offset. @@ -280,15 +283,28 @@ class alignas(16) column_device_view_base { * @param type The type of the column * @param size The number of elements in the column * @param data Pointer to device memory containing elements + * @param null_count The number of nulls in the column * @param null_mask Pointer to device memory containing bitmask representing valid elements * @param offset Index position of the first element + * @param children Pointer to device memory containing child `column_device_view` objects + * @param num_children The number of child columns */ CUDF_HOST_DEVICE column_device_view_base(data_type type, size_type size, void const* data, + size_type null_count, bitmask_type const* null_mask, - size_type offset) - : _type{type}, _size{size}, _data{data}, _null_mask{null_mask}, _offset{offset} + size_type offset, + void* children, + size_type num_children) + : _type{type}, + _size{size}, + _data{data}, + _null_count{null_count}, + _null_mask{null_mask}, + _offset{offset}, + _children{children}, + _num_children{num_children} { } @@ -325,7 +341,8 @@ struct mutable_value_accessor; */ class alignas(16) column_device_view_core : public detail::column_device_view_base { public: - static constexpr bool is_mutable = false; + static constexpr bool is_mutable = + false; ///< Indicates whether this view allows mutation of the underlying data column_device_view_core() = delete; ~column_device_view_core() = default; @@ -380,7 +397,7 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba this->null_count(), this->null_mask(), this->offset() + offset, - d_children, + static_cast(_children), this->num_child_columns()}; } @@ -449,6 +466,14 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba return T{scaled_integer{data()[element_index], scale}}; } + /** + * @brief Returns a nullable element at the specified index. If the element is null, returns + * `nullopt`. + * + * @param element_index Position of the desired element + * @return `optional` containing the element at the specified index, or `nullopt` if the element + * is null + */ template [[nodiscard]] __device__ cuda::std::optional nullable_element( size_type element_index) const noexcept @@ -465,7 +490,7 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba */ [[nodiscard]] __device__ column_device_view_core child(size_type child_index) const noexcept { - return d_children[child_index]; + return static_cast(_children)[child_index]; } /** @@ -507,20 +532,10 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba size_type offset, column_device_view_core* children, size_type num_children) - : column_device_view_base(type, size, data, null_mask, offset), - d_children(children), - _num_children(num_children), - _null_count{null_count} + : column_device_view_base( + type, size, data, null_count, null_mask, offset, children, num_children) { } - - protected: - column_device_view_core* d_children{}; ///< Array of `raw_column_device_view` - ///< objects in device memory. - ///< Based on element type, children - ///< may contain additional data - size_type _num_children{}; ///< The number of child columns - size_type _null_count{}; ///< The number of nulls }; /** @@ -531,7 +546,8 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba */ class alignas(16) mutable_column_device_view_core : public detail::column_device_view_base { public: - static constexpr bool is_mutable = true; + static constexpr bool is_mutable = + true; ///< Indicates whether this view allows mutation of the underlying data mutable_column_device_view_core() = delete; ~mutable_column_device_view_core() = default; @@ -654,6 +670,14 @@ class alignas(16) mutable_column_device_view_core : public detail::column_device return T{scaled_integer{data()[element_index], scale}}; } + /** + * @brief Returns a nullable element at the specified index. If the element is null, returns + * `nullopt`. + * + * @param element_index Position of the desired element + * @return `optional` containing the element at the specified index, or `nullopt` if the element + * is null + */ template [[nodiscard]] __device__ cuda::std::optional nullable_element( size_type element_index) const noexcept @@ -714,7 +738,7 @@ class alignas(16) mutable_column_device_view_core : public detail::column_device [[nodiscard]] __device__ mutable_column_device_view_core child(size_type child_index) const noexcept { - return d_children[child_index]; + return static_cast(_children)[child_index]; } #ifdef __CUDACC__ // because set_bit in bit.hpp is wrapped with __CUDACC__ @@ -792,17 +816,16 @@ class alignas(16) mutable_column_device_view_core : public detail::column_device size_type offset, mutable_column_device_view_core* children, size_type num_children) - : column_device_view_base(type, size, data, null_mask, offset), - d_children(children), - _num_children(num_children) + : column_device_view_base(type, + size, + data, + 0, // unused + null_mask, + offset, + children, + num_children) { } - - mutable_column_device_view_core* d_children{}; ///< Array of `raw_mutable_column_device_view` - ///< objects in device memory. - ///< Based on element type, children - ///< may contain additional data - size_type _num_children{}; ///< The number of child columns }; } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/strings/detail/strings_children.cuh b/cpp/include/cudf/strings/detail/strings_children.cuh index f691145f5445..c6aa82fca15d 100644 --- a/cpp/include/cudf/strings/detail/strings_children.cuh +++ b/cpp/include/cudf/strings/detail/strings_children.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -72,8 +72,12 @@ rmm::device_uvector make_chars_buffer(column_view const& offsets, auto const src_ptrs = thrust::make_transform_iterator( thrust::make_counting_iterator(0), - cuda::proclaim_return_type( - [begin] __device__(uint32_t idx) -> void const* { return begin[idx].first; })); + cuda::proclaim_return_type([begin] __device__(uint32_t idx) { + // Due to a bug in cub (https://github.com/NVIDIA/cccl/issues/586), + // we have to use `const_cast` to remove `const` qualifier from the source pointer. + // This should be fine as long as we only read but not write anything to the source. + return reinterpret_cast(const_cast(begin[idx].first)); + })); auto const src_sizes = thrust::make_transform_iterator( thrust::make_counting_iterator(0), cuda::proclaim_return_type( @@ -98,27 +102,27 @@ rmm::device_uvector make_chars_buffer(column_view const& offsets, return chars_data; } -rmm::device_uvector make_chars_buffer(column_view const& offsets_view, - int64_t chars_size, - string_view const* begin, - bitmask_type const* stencil, - size_type size, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +inline rmm::device_uvector make_chars_buffer(column_view const& offsets_view, + int64_t chars_size, + string_view const* begin, + bitmask_type const* stencil, + size_type size, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { auto offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets_view); auto chars = rmm::device_uvector(chars_size, stream, mr); - auto srcs = make_counting_transform_iterator( + auto srcs = cudf::detail::make_counting_transform_iterator( size_type{0}, [begin] __device__(size_type idx) -> void const* { return begin[idx].data(); }); - auto src_sizes = make_counting_transform_iterator( + auto src_sizes = cudf::detail::make_counting_transform_iterator( size_type{0}, [begin, stencil] __device__(size_type idx) -> size_type { if (stencil != nullptr && !bit_is_set(stencil, idx)) { return 0; } return static_cast(begin[idx].size_bytes()); }); - auto dsts = make_counting_transform_iterator( + auto dsts = cudf::detail::make_counting_transform_iterator( size_type{0}, [offsets, chars = chars.data()] __device__(size_type idx) -> void* { return chars + offsets[idx]; }); diff --git a/cpp/include/cudf/strings/detail/strings_column_factories.cuh b/cpp/include/cudf/strings/detail/strings_column_factories.cuh index f7e30d5901bb..39d26e9b5048 100644 --- a/cpp/include/cudf/strings/detail/strings_column_factories.cuh +++ b/cpp/include/cudf/strings/detail/strings_column_factories.cuh @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -84,43 +85,33 @@ std::unique_ptr make_strings_column(IndexPairIterator begin, std::move(null_mask)); } -std::unique_ptr make_strings_column(device_span strings, - rmm::device_buffer null_mask, - std::optional null_count, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +inline std::unique_ptr make_strings_column(device_span strings, + rmm::device_buffer null_mask, + size_type null_count, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); auto size = static_cast(strings.size()); if (size == 0) return make_empty_column(type_id::STRING); + auto stencil = static_cast(null_mask.data()); + // build offsets column from the strings sizes - auto sizes = thrust::make_counting_transform_iterator( + auto sizes = cudf::detail::make_counting_transform_iterator( cudf::size_type{0}, - [stencil = null_mask.data(), - strings = strings.data()] __device__(cudf::size_type index) -> size_type { - if (stencil != nullptr && !bit_is_set(stencil, index)) { return size_type{0}; } + [stencil, strings = strings.data()] __device__(cudf::size_type index) -> size_type { + if (stencil != nullptr && !bit_is_set(stencil, index)) { return 0; } return static_cast(strings[index].size_bytes()); }); auto [offsets, bytes] = cudf::strings::detail::make_offsets_child_column(sizes, sizes + size, stream, mr); - auto final_null_count = size_type{0}; - - if (!null_count.has_value()) { - final_null_count = - null_mask.empty() ? 0 - : cudf::detail::count_set_bits(null_mask.data(), 0, strings.size(), stream); - } else { - final_null_count = null_count.value(); - } - - auto chars = - make_chars_buffer(offsets->view(), bytes, strings.data(), null_mask.data(), size, stream, mr); + auto chars = make_chars_buffer(offsets->view(), bytes, strings.data(), stencil, size, stream, mr); return make_strings_column( - size, std::move(offsets), chars.release(), final_null_count, std::move(null_mask)); + size, std::move(offsets), chars.release(), null_count, std::move(null_mask)); } /** diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index a3bc6cfcaaab..d5159835a504 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -131,7 +131,7 @@ std::unique_ptr transform_extended( rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); /** - * @brief Creates a new column by applying a transform function against every + * @brief Creates a new table by applying a transform function against every * element of the input columns. * * Computes: @@ -151,29 +151,30 @@ std::unique_ptr transform_extended( * @param udf The PTX/CUDA string of the transform function to apply * @param source_type The source type of the UDF (CUDA or PTX) * @param is_null_aware Signifies the UDF will receive row inputs as optional values - * @param row_size The row size of the transform operation. If not provided, it is inferred from the - * input columns. * @param user_data User-defined device data to pass to the UDF. * @param inputs Immutable views of the inputs to transform (columns and scalar columns) * @param outputs Specification of the output columns to be created * @param string_offsets For string output columns, the offsets can be pre-allocated and passed in * to prevent overhead of compacting string views into run-end strings column. + * @param row_size The row size of the transform operation. If not provided, it is inferred from the + * input columns. * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory * @return A table containing the columns resulting from applying the transform * function to every element of the input according to the output specifications * */ -std::unique_ptr
transform_extended2(std::string const& udf, - udf_source_type source_type, - null_aware is_null_aware, - std::optional row_size, - std::optional user_data, - std::span inputs, - std::span outputs, - std::vector> string_offsets, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr); +std::unique_ptr
multi_transform( + std::string const& udf, + udf_source_type source_type, + null_aware is_null_aware, + std::optional user_data, + std::span inputs, + std::span outputs, + std::vector> string_offsets, + std::optional row_size, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); /** * @brief Creates a null_mask from `input` by converting `NaN` to null and diff --git a/cpp/include/cudf/utilities/bit.hpp b/cpp/include/cudf/utilities/bit.hpp index 545cd4c339c0..9b976704e610 100644 --- a/cpp/include/cudf/utilities/bit.hpp +++ b/cpp/include/cudf/utilities/bit.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -94,7 +94,7 @@ CUDF_HOST_DEVICE inline void clear_bit_unsafe(bitmask_type* bitmask, size_type b /** * @brief Indicates whether the specified bit is set to `1` * - * @param bitmask The bitmask containing the bit to clear + * @param bitmask The bitmask containing the bit to test * @param bit_index Index of the bit to test * @return true The specified bit is `1` * @return false The specified bit is `0` diff --git a/cpp/src/column/column_device_view.cu b/cpp/src/column/column_device_view.cu index 74b17a7b8236..e693a4d041c0 100644 --- a/cpp/src/column/column_device_view.cu +++ b/cpp/src/column/column_device_view.cu @@ -18,6 +18,23 @@ #include namespace cudf { + +template +inline constexpr bool bitwise_compatible = (sizeof(A) == sizeof(B)) && (alignof(A) == alignof(B)); + +static_assert( + bitwise_compatible, + "detail::column_device_view_base and column_device_view_core must be bitwise-compatible"); +static_assert(bitwise_compatible, + "column_device_view_core and column_device_view must be bitwise-compatible"); + +static_assert( + bitwise_compatible, + "detail::column_device_view_base and mutable_column_device_view_core must be bitwise-compatible"); +static_assert( + bitwise_compatible, + "mutable_column_device_view_core and mutable_column_device_view must be bitwise-compatible"); + // Trivially copy all members but the children column_device_view::column_device_view(column_view source) : column_device_view_core{source.type(), @@ -89,7 +106,7 @@ column_device_view::column_device_view(column_view source, void* h_ptr, void* d_ nullptr, source.num_children()} { - d_children = detail::child_columns_to_device_array( + _children = detail::child_columns_to_device_array( source.child_begin(), source.child_end(), h_ptr, d_ptr); } @@ -138,7 +155,7 @@ mutable_column_device_view::mutable_column_device_view(mutable_column_view sourc nullptr, source.num_children()} { - d_children = detail::child_columns_to_device_array( + _children = detail::child_columns_to_device_array( source.child_begin(), source.child_end(), h_ptr, d_ptr); } diff --git a/cpp/src/jit/column_accessor.cuh b/cpp/src/jit/column_accessor.cuh index ce68c5245538..eeeea10ed3fa 100644 --- a/cpp/src/jit/column_accessor.cuh +++ b/cpp/src/jit/column_accessor.cuh @@ -6,7 +6,7 @@ #pragma once -#include +#include #include #include @@ -19,100 +19,107 @@ template + bool MayBeNullable, + bool IsStringsOutput> struct column_accessor { - static constexpr int32_t index = Index; - using column_type = Column; - using element_type = Element; - using optional_element_type = OptionalElement; - static constexpr bool as_scalar = AsScalar; - static constexpr bool may_be_nullable = MayBeNullable; - - static __device__ auto& get(detail::column_device_view_base const* cols) + static constexpr int32_t index = Index; + using column_type = Column; + using element_type = Element; + using optional_element_type = OptionalElement; + static constexpr bool as_scalar = AsScalar; + static constexpr bool may_be_nullable = MayBeNullable; + static constexpr bool is_strings_output = IsStringsOutput; + + static __device__ constexpr size_type map_index(size_type row) { - return reinterpret_cast(cols[index]); + if constexpr (as_scalar) { + return 0; + } else { + return row; + } } - static __device__ element_type element(detail::column_device_view_base const* cols, size_type row) + template + static __device__ auto& column(T const* cols) + requires(sizeof(T) == sizeof(column_type)) { - auto& c = get(cols); + return reinterpret_cast(cols[index]); + } - if constexpr (AsScalar) { - return c.template element(0); - } else { - return c.template element(row); - } + static __device__ element_type element(auto const* cols, size_type row) + { + return column(cols).template element(map_index(row)); } - static __device__ bool is_null(detail::column_device_view_base const* cols, size_type row) + static __device__ bool is_null(auto const* cols, size_type row) { - if constexpr (!MayBeNullable) { + if constexpr (!may_be_nullable) { return false; } else { - auto& c = get(cols); - - if constexpr (AsScalar) { - return c.is_null(0); - } else { - return c.is_null(row); - } + return column(cols).is_null(map_index(row)); } } - static __device__ bool is_valid(detail::column_device_view_base const* cols, size_type row) + static __device__ bool is_valid(auto const* cols, size_type row) { - if constexpr (!MayBeNullable) { + if constexpr (!may_be_nullable) { return true; } else { - auto& c = get(cols); - - if constexpr (AsScalar) { - return c.is_valid(0); - } else { - return c.is_valid(row); - } + return column(cols).is_valid(map_index(row)); } } - static __device__ optional_element_type - nullable_element(detail::column_device_view_base const* cols, size_type row) + static __device__ optional_element_type nullable_element(auto const* cols, size_type row) { - auto& c = get(cols); + auto& c = column(cols); - if constexpr (!MayBeNullable) { - return c.template element(row); + if constexpr (!may_be_nullable) { + return c.template element(map_index(row)); } else { - if constexpr (AsScalar) { - return c.template nullable_element(0); - } else { - return c.template nullable_element(row); - } + return c.template nullable_element(map_index(row)); } } - static __device__ void set_null_word(detail::column_device_view_base const* cols, - size_type index, - bitmask_type word) + static __device__ void set_null_mask_word(auto const* cols, + size_type word_index, + bitmask_type word) + requires(!as_scalar) { - auto& c = get(cols); - - if constexpr (!MayBeNullable) { + if constexpr (!may_be_nullable) { return; } else { - auto* mask = c.null_mask(); + auto* mask = column(cols).null_mask(); if (mask == nullptr) { return; } - mask[index] = word; + mask[word_index] = word; } } - static __device__ void assign(detail::column_device_view_base const* cols, - size_type row, - element_type value) - requires(!AsScalar) + static __device__ void assign(auto const* cols, size_type row, element_type value) + requires(!as_scalar) { - get(cols).template assign(row, value); + column(cols).template assign(row, value); + } + + static __device__ element_type output_arg(auto const* cols, size_type row) + requires(!as_scalar) + { + if constexpr (is_strings_output) { + return element(cols, row); + } else { + return {}; + } + } + + static __device__ optional_element_type null_output_arg(auto const* cols, size_type row) + requires(!as_scalar) + { + if constexpr (is_strings_output) { + return element(cols, row); + } else { + return {}; + } } }; diff --git a/cpp/src/jit/wrappers.cuh b/cpp/src/jit/column_device_view_wrappers.cuh similarity index 59% rename from cpp/src/jit/wrappers.cuh rename to cpp/src/jit/column_device_view_wrappers.cuh index af3fa2f16bd6..6fed1f50e11c 100644 --- a/cpp/src/jit/wrappers.cuh +++ b/cpp/src/jit/column_device_view_wrappers.cuh @@ -1,5 +1,10 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ #pragma once +#include #include #include @@ -13,11 +18,10 @@ namespace jit { * @brief A column wrapper type that treats a column as a vector of elements. * */ -template -struct vector_column_device_view : private Column { - using base = Column; +struct vector_column_device_view : private mutable_column_device_view_core { + using base = mutable_column_device_view_core; - CUDF_HOST_DEVICE constexpr vector_column_device_view(Column const& src) : base{src} {} + CUDF_HOST_DEVICE constexpr vector_column_device_view(base const& src) : base{src} {} ~vector_column_device_view() = default; vector_column_device_view(vector_column_device_view const&) = default; vector_column_device_view(vector_column_device_view&&) = default; @@ -29,18 +33,10 @@ struct vector_column_device_view : private Column { using base::size; using base::type; - template - CUDF_HOST_DEVICE T const* data() const noexcept - requires(!base::is_mut) - { - return static_cast(_data) + _offset; - } - template CUDF_HOST_DEVICE T* data() const noexcept - requires(base::is_mut) { - return static_cast(_data) + _offset; + return static_cast(const_cast(_data)) + _offset; } using base::is_null; @@ -48,31 +44,36 @@ struct vector_column_device_view : private Column { using base::null_mask; template - [[nodiscard]] __device__ decltype(auto) element(size_type element) const noexcept + [[nodiscard]] __device__ decltype(auto) element(size_type element_index) const noexcept { - return data()[element]; + return data()[element_index]; } template - [[nodiscard]] __device__ cuda::std::optional nullable_element(size_type element) const noexcept + [[nodiscard]] __device__ cuda::std::optional nullable_element( + size_type element_index) const noexcept { - if (is_null(element)) { return cuda::std::nullopt; } - return element(element); + if (is_null(element_index)) { return cuda::std::nullopt; } + return element(element_index); } + template + __device__ void assign(size_type row, T value) const noexcept + { + data()[row] = value; + } }; /** * @brief A column wrapper type that treats a column as a column of mutable strings. * The offsets will have been pre-initialized and the chars will have been pre-allocated. */ -template -struct mut_strings_column_device_view : private Column { - using base = Column; +struct mut_strings_column_device_view : private mutable_column_device_view_core { + using base = mutable_column_device_view_core; - CUDF_HOST_DEVICE constexpr mut_strings_column_device_view(Column const& src) : base{src} {} + CUDF_HOST_DEVICE constexpr mut_strings_column_device_view(base const& src) : base{src} {} - ~mut_strings_column_device_view() = default; + ~mut_strings_column_device_view() = default; mut_strings_column_device_view(mut_strings_column_device_view const&) = default; mut_strings_column_device_view(mut_strings_column_device_view&&) = default; mut_strings_column_device_view& operator=(mut_strings_column_device_view const&) = default; @@ -87,11 +88,11 @@ struct mut_strings_column_device_view : private Column { using base::type; template > - [[nodiscard]] __device__ cuda::std::span element(size_type element) const noexcept - requires(base::is_mut && cuda::std::is_same_v>) + [[nodiscard]] __device__ cuda::std::span element(size_type element_index) const noexcept + requires(cuda::std::is_same_v>) { - auto index = element + offset(); - auto chars = static_cast(_data); + auto index = element_index + offset(); + auto chars = static_cast(const_cast(_data)); auto offsets = child(offsets_column_index); auto itr = cudf::detail::input_offsetalator(offsets.head(), offsets.type()); auto offset = itr[index]; @@ -101,13 +102,21 @@ struct mut_strings_column_device_view : private Column { template > [[nodiscard]] __device__ cuda::std::optional> nullable_element( - size_type element) const noexcept - requires(base::is_mut && cuda::std::is_same_v>) + size_type element_index) const noexcept + requires(cuda::std::is_same_v>) { - if (is_null(element)) { return cuda::std::nullopt; } - return element(element); + if (is_null(element_index)) { return cuda::std::nullopt; } + return element(element_index); } + template > + __device__ void assign(size_type row, cuda::std::span value) const noexcept + requires(cuda::std::is_same_v>) + { + // no-op for since we assume the chars have already been pre-allocated and they are mutated + // in-place + return; + } }; } // namespace jit diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index a862e16024be..88bee6796273 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -89,7 +89,8 @@ std::vector input_type_names( jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, std::string const& kernel_name, - std::string const& cuda_source) + std::string const& cuda_source, + std::vector const& extra_options) { CUDF_FUNC_RANGE(); @@ -99,9 +100,14 @@ jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preproces std::vector options; options.emplace_back("-arch=sm_."); + options.emplace_back("-minimal"); if (runtime_version >= min_pch_runtime_version) { options.emplace_back("-pch"); } + for (auto& opt : extra_options) { + options.push_back(opt); + } + return cudf::jit::get_program_cache(preprocessed_program_data) .get_kernel(kernel_name, {}, {{"cudf/detail/operation-udf.hpp", cuda_source}}, options); } diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index e2ea5095de7a..6c3277124a8c 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -76,7 +76,8 @@ std::vector input_type_names( jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, std::string const& kernel_name, - std::string const& cuda_source); + std::string const& cuda_source, + std::vector const& extra_options = {}); } // namespace jit } // namespace cudf diff --git a/cpp/src/jit/span.cuh b/cpp/src/jit/span.cuh index 6d2fbe7411a1..64b6f42314c3 100644 --- a/cpp/src/jit/span.cuh +++ b/cpp/src/jit/span.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -92,6 +92,5 @@ struct device_span { } }; - } // namespace jit } // namespace cudf diff --git a/cpp/src/jit/sync.cuh b/cpp/src/jit/sync.cuh new file mode 100644 index 000000000000..dc153317d145 --- /dev/null +++ b/cpp/src/jit/sync.cuh @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include + +#include + +namespace cudf { +namespace jit { + +__device__ inline bool warp_elect(unsigned int mask) +{ +#if __CUDA_ARCH__ >= 900 + // use elect.sync + return cuda::ptx::elect_sync(mask); +#else + // fallback: manually elect a leader (e.g., the first active thread) + int leader = __ffs(mask) - 1; + int lane = (threadIdx.x & 31); + return (lane == leader); +#endif +} + +template +__device__ void warp_compact_validity(mutable_column_device_view_core const* outcols, + size_type row, + bool is_valid) +{ + if constexpr (!Out::may_be_nullable) { + return; + } else { + auto active = __activemask(); + auto null_word = __ballot_sync(active, is_valid); + // use warp-elect to make sure we only issue one memory transaction per warp + if (warp_elect(active)) { Out::set_null_mask_word(outcols, row / 32, null_word); } + } +} + +} // namespace jit +} // namespace cudf diff --git a/cpp/src/jit/transform_udf.cuh b/cpp/src/jit/transform_udf.cuh index 19e1ca437386..e3788eacc776 100644 --- a/cpp/src/jit/transform_udf.cuh +++ b/cpp/src/jit/transform_udf.cuh @@ -38,28 +38,31 @@ template static __device__ void call(Fn&& udf, - size_type index, - void* user_data, bitmask_type const* stencil, - detail::column_device_view_base const* cols, - [[maybe_unused]] bool* is_valid) + void* user_data, + column_device_view_core const* incols, + mutable_column_device_view_core const* outcols, + [[maybe_unused]] bool* is_valid, + size_type element_idx) requires(is_null_aware == null_aware::NO) { if constexpr (has_stencil) { - if (stencil != nullptr && !bit_is_set(stencil, index)) { return; } + if (stencil != nullptr) { + if (!bit_is_set(stencil, element_idx)) { return; } + } } - auto outs = - Outs::map([]() { return cuda::std::tuple{typename A::element_type{}...}; }); + auto outs = Outs::map( + [&]() { return cuda::std::tuple{A::output_arg(outcols, element_idx)...}; }); auto out_ptrs = - cuda::std::apply([&](auto&&... args) { return cuda::std::tuple{&args...}; }, outs); + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - auto inputs = - Ins::map([&]() { return cuda::std::tuple{A::element(cols, index)...}; }); + auto inputs = Ins::map( + [&]() { return cuda::std::tuple{A::element(incols, element_idx)...}; }); if constexpr (has_user_data) { - auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, index}, out_ptrs, inputs); + auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); cuda::std::apply(udf, args); } else { @@ -67,31 +70,34 @@ struct transform_udf { cuda::std::apply(udf, args); } - [&](cuda::std::integer_sequence) { - (Outs::at::assign(cols, index, cuda::std::get(outs)), ...); - }(Outs::indexed); + Outs::map([&]() { + (A::assign(outcols, element_idx, cuda::std::get(outs)), ...); + }); } template static __device__ void call(Fn&& udf, - size_type index, - void* user_data, [[maybe_unused]] bitmask_type const* stencil, - detail::column_device_view_base const* cols, - bool* is_valid) + void* user_data, + column_device_view_core const* incols, + mutable_column_device_view_core const* outcols, + bool* is_valid, + size_type element_idx) requires(is_null_aware == null_aware::YES) { - auto outs = Outs::map( - []() { return cuda::std::tuple{typename A::optional_element_type{}...}; }); + auto outs = Outs::map([&]() { + return cuda::std::tuple{A::null_output_arg(outcols, element_idx)...}; + }); auto out_ptrs = - cuda::std::apply([&](auto&&... args) { return cuda::std::tuple{&args...}; }, outs); + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - auto inputs = Ins::map( - [&]() { return cuda::std::tuple{A::nullable_element(cols, index)...}; }); + auto inputs = Ins::map([&]() { + return cuda::std::tuple{A::nullable_element(incols, element_idx)...}; + }); if constexpr (has_user_data) { - auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, index}, out_ptrs, inputs); + auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); cuda::std::apply(udf, args); } else { @@ -99,10 +105,10 @@ struct transform_udf { cuda::std::apply(udf, args); } - [&](cuda::std::integer_sequence) { - (Outs::at::assign(cols, index, cuda::std::get(outs)), ...); - ((is_valid[I] = cuda::std::get(outs).has_value()), ...); - }(Ins::indexed); + Outs::map([&]() { + (A::assign(outcols, element_idx, *cuda::std::get(outs)), ...); + ((is_valid[A::index] = cuda::std::get(outs).has_value()), ...); + }); } }; diff --git a/cpp/src/jit/type_list.cuh b/cpp/src/jit/type_list.cuh index a435cb4fe72f..4007f9ed7e63 100644 --- a/cpp/src/jit/type_list.cuh +++ b/cpp/src/jit/type_list.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -26,7 +26,7 @@ struct type_list { static constexpr cuda::std::make_integer_sequence indexed{}; template - static constexpr __device__ decltype(auto) map(Fn& fn) + static constexpr __device__ decltype(auto) map(Fn&& fn) { return fn.template operator()(); } diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index 103cd92637da..c8b07bc7a625 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -11,8 +11,9 @@ #include #include -#include +#include #include +#include #pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before // it. Each UDF will have a different operation-udf.hpp generated for it, so we diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 32097b942533..6b597dd4c3f8 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -13,10 +13,11 @@ #include -#include -#include +#include +#include +#include +#include #include -#include #pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before // it. Each UDF will have a different operation-udf.hpp generated for it, so we @@ -30,20 +31,6 @@ namespace cudf { namespace jit { -namespace { -template -__device__ void warp_compact_validity(detail::column_device_view_base const* columns, - size_type row, - bool is_valid) -{ - if constexpr (!Out::may_be_nullable) { - return; - } else { - auto null_word = __ballot_sync(0xFFFF'FFFFU, is_valid); - if ((threadIdx.x & 31) == 0) { Out::set_null_word(columns, row / 32, null_word); } - } -} -} // namespace template CUDF_KERNEL void transform_kernel(size_type row_size, - void* user_data, bitmask_type const* stencil, - detail::column_device_view_base const* columns) + void* user_data, + column_device_view_core const* incols, + mutable_column_device_view_core const* outcols) { - auto const start = detail::grid_1d::global_thread_id(); - auto const stride = detail::grid_1d::grid_stride(); + // ensure block size is a multiple of warp size for correct warp-synchronous behavior + assert((blockDim.x & 31) == 0); + auto start = detail::grid_1d::global_thread_id(); + auto stride = detail::grid_1d::grid_stride(); - for (auto row = start; row < row_size; row += stride) { - bool is_valid[Outs::size]; + for (auto i = start; i < row_size; i += stride) { + bool is_valid[Outs::size] = {}; transform_udf::call( - GENERIC_TRANSFORM_OP, row, user_data, stencil, columns, is_valid); + GENERIC_TRANSFORM_OP, stencil, user_data, incols, outcols, is_valid, i); - Outs::map([&] { - (warp_compact_validity(columns, row, is_valid[Out::index]), ...); - }); + if constexpr (is_null_aware == null_aware::YES) { + Outs::map( + [&]() { (warp_compact_validity(outcols, i, is_valid[A::index]), ...); }); + } } } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 77ec9984f47f..1d6e939ac25d 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include @@ -31,113 +33,134 @@ namespace cudf { namespace { -// TODO: update parameter names and reflection -template -struct mut_vector_column_view { - T* _data{nullptr}; +struct mut_fixed_width_column_view { + mutable_column_view _view; + + auto to_device(rmm::cuda_stream_view stream) const + { + return mutable_column_device_view::create(_view, stream); + } +}; + +struct fixed_width_column { + std::unique_ptr _col = nullptr; + + static auto make(data_type type, + size_type size, + rmm::device_buffer null_mask, + size_type null_count, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) + { + return fixed_width_column{ + make_fixed_width_column(type, size, std::move(null_mask), null_count, stream, mr)}; + } + + auto mutable_view() const { return mut_fixed_width_column_view{_col->mutable_view()}; } + + void set_null_count(size_type count) { _col->set_null_count(count); } + + bool nullable() const { return _col->nullable(); } + + bitmask_type* null_mask() { return _col->mutable_view().null_mask(); } +}; + +struct mut_string_views_column_view { + void* _data{nullptr}; size_type _size{0}; bitmask_type const* _null_mask{nullptr}; size_type _offset{0}; size_type _null_count{0}; - auto to_device() const + auto to_device(rmm::cuda_stream_view stream) const { - auto deleter = +[](mutable_column_device_view* ptr) { delete ptr; }; - return std::unique_ptr>( - new mutable_column_device_view{mutable_column_device_view::from_parts( - data_type{type_id::EMPTY}, _size, _data, _null_mask, _offset, nullptr, 0)}, - deleter); + using view = mutable_column_device_view; + return std::unique_ptr>( + new view{ + view::from_parts(data_type{type_id::EMPTY}, _size, _data, _null_mask, _offset, nullptr, 0)}, + [](auto* p) { delete p; }); } }; -template -struct vector_column { - using mut_view_type = mut_vector_column_view; - - rmm::device_uvector _data{}; +struct string_views_column { + rmm::device_buffer _data; + size_type _size{0}; rmm::device_buffer _null_mask{}; - size_type _offset{0}; size_type _null_count{0}; static auto make(size_type size, - null_mask_state null_state, + rmm::device_buffer null_mask, + size_type null_count, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - rmm::device_uvector data{size, stream, mr}; - auto null_mask = create_null_mask(size, null_state, stream, mr); - - return vector_column{std::move(data), std::move(null_mask), 0, size, 0}; + rmm::device_buffer data{static_cast(size) * sizeof(string_view), stream, mr}; + return string_views_column{std::move(data), size, std::move(null_mask), null_count}; } - auto mutable_view() + auto mutable_view() const { - return mut_view_type{_data.data(), - _data.size(), - static_cast(_null_mask.data()), - _offset, - _null_count}; + return mut_string_views_column_view{ + const_cast(_data.data()), + static_cast(_data.size()), + static_cast(const_cast(_null_mask.data())), + 0, + _null_count}; } + + void set_null_count(size_type count) { _null_count = count; } + + bool nullable() const { return !_null_mask.is_empty(); } + + bitmask_type* null_mask() { return static_cast(_null_mask.data()); } }; struct mut_strings_column_view { - mutable_column_view view; + mutable_column_view _view; + + auto to_device(rmm::cuda_stream_view stream) const + { + return mutable_column_device_view::create(_view, stream); + } }; struct mut_strings_column { - using mut_view_type = mut_strings_column_view; - - std::unique_ptr strings = nullptr; + std::unique_ptr _col = nullptr; static auto make(size_type size, - int64_t chars_size, - std::unique_ptr offsets, - null_mask_state null_state, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + rmm::device_buffer chars, + std::unique_ptr offsets, + rmm::device_buffer null_mask, + size_type null_count) { - // TODO: null-count needs to be updated - auto null_mask = create_null_mask(size, null_state, stream, mr); - auto chars = rmm::device_buffer{chars_size, stream, mr}; - return mut_strings_column{.strings{detail::make_strings_column(size, - std::move(offsets), - std::move(chars), - /* null_count = */ size, - std::move(null_mask), - stream, - mr)}}; + return mut_strings_column{make_strings_column( + size, std::move(offsets), std::move(chars), null_count, std::move(null_mask))}; } - auto mutable_view() { return mut_view_type{.view{strings->mutable_view()}}; } -}; - -using string_views_column = vector_column; -using mut_string_views_column_view = typename string_views_column::mut_view_type; + auto mutable_view() const { return mut_strings_column_view{_col->mutable_view()}; } -using OutputColumn = std::variant, string_views_column, mut_strings_column>; + void set_null_count(size_type count) { _col->set_null_count(count); } -using InputView = std::variant; -using OutputView = - std::variant; + bool nullable() const { return _col->nullable(); } -using InputViews = std::span; -using OutputViews = std::span; - -using InHandle = std::unique_ptr>; -using OutHandle = - std::unique_ptr>; + bitmask_type* null_mask() { return _col->mutable_view().null_mask(); } +}; -using Handle = std::variant; +using input_column_view = transform_input; +using output_column = std::variant; +using handle = std::variant< + std::unique_ptr>, + std::unique_ptr>>; -namespace jit { -namespace transform { +namespace jit_transform { jitify2::Kernel instantiate(null_aware is_null_aware, bool has_stencil, bool has_user_data, std::string const& ins, std::string const& outs, + std::vector const& ptx_input_types, + std::vector const& ptx_output_types, std::string const& udf, udf_source_type source_type) { @@ -146,103 +169,156 @@ jitify2::Kernel instantiate(null_aware is_null_aware, ? jit::parse_single_function_ptx( udf, "GENERIC_TRANSFORM_OP", - jit::build_ptx_params(output_typenames, input_typenames, has_user_data)) + jit::build_ptx_params(ptx_output_types, ptx_input_types, has_user_data)) : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); auto kernel = jitify2::reflection::Template("cudf::jit::transform_kernel") - .instantiate(is_null_aware, may_evaluate_null, has_user_data, ins, outs); + .instantiate(is_null_aware, has_stencil, has_user_data, ins, outs); - return jit::get_udf_kernel(*transform_jit_kernel_cu_jit, kernel, cuda_source); + return jit::get_udf_kernel(*transform_jit_kernel_cu_jit, kernel, cuda_source, {"--restrict"}); } void launch(jitify2::Kernel const& kernel, size_type row_size, - void* user_data, bitmask_type const* stencil, - detail::column_device_view_base const* columns, + void* user_data, + column_device_view_core const* incols, + mutable_column_device_view_core const* outcols, rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - void* args[] = {&row_size, &user_data, &stencil, &columns}; + void* args[] = {&row_size, &stencil, &user_data, &incols, &outcols}; kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); } -std::pair reflect(InputViews inputs, - OutputViews outputs, - std::span input_may_be_nullable, - std::span output_may_be_nullable) +std::string reflect_input_element(column_view const& c) { return type_to_name(c.type()); } + +std::string reflect_input_element(scalar_column_view const& c) { return type_to_name(c.type()); } + +std::string reflect_output_element(fixed_width_column const& c) +{ + return type_to_name(c._col->type()); +} + +std::string reflect_output_element(string_views_column const&) { return "cudf::string_view"; } + +std::string reflect_output_element(mut_strings_column const&) { return "cuda::std::span"; } + +std::string reflect_input_column(column_view const&) { return "cudf::column_device_view_core"; } + +std::string reflect_input_column(scalar_column_view const&) +{ + return "cudf::column_device_view_core"; +} + +std::string reflect_output_column(fixed_width_column const&) +{ + return "cudf::mutable_column_device_view_core"; +} + +std::string reflect_output_column(string_views_column const&) +{ + return "cudf::jit::vector_column_device_view"; +} + +std::string reflect_output_column(mut_strings_column const&) +{ + return "cudf::jit::mut_strings_column_device_view"; +} + +auto reflect(udf_source_type source_type, + std::span inputs, + std::span outputs, + std::span input_may_be_nullable, + std::span output_may_be_nullable) { std::vector ins; for (size_t i = 0; i < inputs.size(); i++) { - auto& input = inputs[i]; - auto column = "cudf::column_device_view_core"; - auto element = std::visit([](auto& a) { return type_to_name(a.type()); }, input); - auto optional_element = std::format("cuda::std::optional<{}>", element); - bool as_scalar = std::holds_alternative(input); - bool may_be_nullable = input_may_be_nullable[i]; + auto& in = inputs[i]; + auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); + auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); + auto optional_element = std::format("cuda::std::optional<{}>", element); + bool as_scalar = std::holds_alternative(in); + bool may_be_nullable = input_may_be_nullable[i]; + auto is_strings_output = false; auto accessor = - jitify2::Template("cudf::jit::column_accessor") - .instantiate(i, column, element, optional_element, as_scalar, may_be_nullable); + jitify2::reflection::Template("cudf::jit::column_accessor") + .instantiate( + i, column, element, optional_element, as_scalar, may_be_nullable, is_strings_output); ins.push_back(accessor); } std::vector outs; for (size_t i = 0; i < outputs.size(); i++) { - auto& output = outputs[i]; - auto column = - std::holds_alternative(output) - ? "cudf::jit::column_device_view_span_wrapper" - : "cudf::mutable_column_device_view_core"; - auto element = std::visit([](auto& a) { return type_to_name(a.type()); }, output); - auto optional_element = std::format("cuda::std::optional<{}>", element); - bool as_scalar = false; // output can never be a scalar - bool may_be_nullable = output_may_be_nullable[i]; + auto& out = outputs[i]; + auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); + auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); + auto optional_element = std::format("cuda::std::optional<{}>", element); + bool as_scalar = false; // never scalar + bool may_be_nullable = output_may_be_nullable[i]; + auto is_strings_output = std::holds_alternative(out); auto accessor = - jitify2::Template("cudf::jit::column_accessor") + jitify2::reflection::Template("cudf::jit::column_accessor") .instantiate( - inputs.size() + i, column, element, optional_element, as_scalar, may_be_nullable); + i, column, element, optional_element, as_scalar, may_be_nullable, is_strings_output); outs.push_back(accessor); } + + auto in_list = jitify2::reflection::Template("cudf::jit::type_list").instantiate(ins); + auto out_list = jitify2::reflection::Template("cudf::jit::type_list").instantiate(outs); + + std::vector ptx_input_types; + std::vector ptx_output_types; + + if (source_type == udf_source_type::PTX) { + for (auto& in : inputs) { + ptx_input_types.push_back(std::visit([](auto& c) { return reflect_input_element(c); }, in)); + } + + for (auto& out : outputs) { + ptx_output_types.push_back( + std::visit([](auto& c) { return reflect_output_element(c); }, out)); + } + } + + return std::make_tuple(in_list, out_list, ptx_input_types, ptx_output_types); } -auto to_args(InputViews inputs, - OutputViews outputs, +auto to_args(std::span inputs, + std::span outputs, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - std::vector handles; + std::vector handles; std::vector h_args; for (auto& in : inputs) { if (auto* col = std::get_if(&in)) { - auto handle = column_device_view::create(*col, stream, mr); - h_args.push_back(*handle); - handles.push_back(std::move(handle)); + auto hnd = column_device_view::create(*col, stream); + h_args.push_back(*hnd); + handles.push_back(std::move(hnd)); } else { auto& scalar = std::get(in); - auto handle = column_device_view::create(scalar.as_column_view(), stream, mr); - h_args.push_back(*handle); - handles.push_back(std::move(handle)); + auto hnd = column_device_view::create(scalar.as_column_view(), stream); + h_args.push_back(*hnd); + handles.push_back(std::move(hnd)); } } for (auto& out : outputs) { - if (auto* col = std::get_if(&out)) { - auto handle = mutable_column_device_view::create(*col, stream, mr); - h_args.push_back(*handle); - handles.push_back(std::move(handle)); - } else { - auto& sv_column = std::get(out); - auto handle = sv_column.to_device(); - h_args.push_back(*handle); - handles.push_back(std::move(handle)); - } + std::visit( + [&](auto& col) { + auto hnd = col.mutable_view().to_device(stream); + h_args.push_back(*hnd); + handles.push_back(std::move(hnd)); + }, + out); } - rmm::device_uvector d_args{h_args.size(), stream, mr}; + rmm::device_buffer d_args{h_args.size() * sizeof(detail::column_device_view_base), stream, mr}; detail::cuda_memcpy_async_impl(d_args.data(), h_args.data(), @@ -257,10 +333,10 @@ void run(null_aware is_null_aware, bool has_stencil, bool has_user_data, size_type row_size, - void* user_data, bitmask_type const* d_stencil, - InputViews inputs, - OutputsView outputs, + void* user_data, + std::span inputs, + std::span outputs, std::span input_may_be_nullable, std::span output_may_be_nullable, std::string const& udf, @@ -268,58 +344,114 @@ void run(null_aware is_null_aware, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - auto [in_types, out_types] = - reflect(inputs, outputs, input_may_be_nullable, output_may_be_nullable); - auto kernel = - instantiate(is_null_aware, has_stencil, has_user_data, in_types, out_types, udf, source_type); + auto [in_types, out_types, ptx_in_types, ptx_out_types] = + reflect(source_type, inputs, outputs, input_may_be_nullable, output_may_be_nullable); + auto kernel = instantiate(is_null_aware, + has_stencil, + has_user_data, + in_types, + out_types, + ptx_in_types, + ptx_out_types, + udf, + source_type); auto [cols, handles] = to_args(inputs, outputs, stream, mr); - return launch(kernel, row_size, user_data, d_stencil, cols.data(), stream); + auto* incols = reinterpret_cast(cols.data()); + auto* outcols = reinterpret_cast(incols + inputs.size()); + return launch(kernel, row_size, d_stencil, user_data, incols, outcols, stream); } -} // namespace transform -} // namespace jit +} // namespace jit_transform -std::tuple null_mask_and(size_type row_size, - InputViews inputs, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +size_type inplace_null_mask_and(bitmask_type* null_mask, + size_type row_size, + std::span inputs, + rmm::cuda_stream_view stream) { - // TODO(lamarrr): handle non-nullable inputs? or is it handled? - // what if we only have scalars? - // what if some of the inputs are non-nullable - // what if none of the inputs are nullable? will an allocated null mask be created? - - // collect the non-scalar elements that contribute to the resulting bitmask - std::vector bitmask_columns; - - // to handle null masks for scalars, we just check if the scalar element is null. If it is null, - // then all the rows of the transform output will be null. This helps us prevent creating - // column-sized bitmasks for each scalar. - for (auto const& in : inputs) { - if (auto* scalar = std::get_if(&in)) { - // all nulls - if (scalar->has_nulls()) { - return std::make_tuple(create_null_mask(row_size, mask_state::ALL_NULL, stream, mr), - row_size); + auto is_nullable = null_mask != nullptr; + + if (!is_nullable) { return 0; } + + auto num_words = num_bitmask_words(row_size); + auto num_bytes = num_words * sizeof(bitmask_type); + + CUDF_EXPECTS(!inputs.empty(), "Inputs must not be empty", std::logic_error); + + auto has_scalars = std::any_of(inputs.begin(), inputs.end(), [](auto& in) { + return std::holds_alternative(in); + }); + + if (has_scalars) { + auto scalar_is_null = std::any_of(inputs.begin(), inputs.end(), [](auto& in) { + if (auto* scalar = std::get_if(&in)) { return scalar->has_nulls(); } + return false; + }); + + if (scalar_is_null) { + // scalar is null, all rows will be null + CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0x00, num_bytes, stream.value())); + + return row_size; + } + } + + auto has_cols = std::any_of( + inputs.begin(), inputs.end(), [](auto& in) { return std::holds_alternative(in); }); + + if (has_cols) { + // no non-scalar columns, so all rows are valid + CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0xFF, num_bytes, stream.value())); + + return 0; + } + + // collect the non-scalar nullable columns that contribute to the output nullmask + std::vector nullable_masks; + std::vector nullable_offsets; + std::vector nullable_null_counts; + + for (auto& in : inputs) { + if (auto* c = std::get_if(&in)) { + if (c->nullable()) { + nullable_masks.push_back(c->null_mask()); + nullable_offsets.push_back(c->offset()); + nullable_null_counts.push_back(c->null_count()); } - } else { - auto& col = std::get(in); - bitmask_columns.emplace_back(col); } } - // TODO: do we depend on the nullness of the outputs? - if (bitmask_columns.empty()) { - // if there are no non-scalar columns contributing to the null-mask, then the output is all - // valid (scalar projection) given that the scalar is not null (checked above) - return std::make_tuple(create_null_mask(row_size, mask_state::UNALLOCATED, stream, mr), 0); + if (nullable_masks.empty()) { + // we only have non-nullable columns, so all rows are valid + CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0xFF, num_bytes, stream.value())); + + return 0; } - return cudf::bitmask_and(table_view{bitmask_columns}, stream, mr); + if (nullable_masks.size() == 1) { + // only 1 mask provided, copy it directly to the output + + detail::cuda_memcpy_async_impl( + null_mask, nullable_masks[0], num_bytes, detail::host_memory_kind::PINNED, stream); + return nullable_null_counts[0]; + } + + return detail::inplace_bitmask_and( + device_span{null_mask, static_cast(num_words)}, + nullable_masks, + nullable_offsets, + row_size, + stream); } -std::pair, std::vector> get_nullabilities( - null_aware is_null_aware, InputViews inputs, std::span outputs) +/** + * @brief Get the null-mask transformation for the transform operation based on the UDF's parameters + * and inputs + * + * @return input and output null-policies for the UDF kernel + */ +auto get_null_transformation(null_aware is_null_aware, + std::span inputs, + std::span outputs) { std::vector input_may_be_nullable; @@ -327,52 +459,32 @@ std::pair, std::vector> get_nullabilities( input_may_be_nullable.push_back(true); } - std::vector output_may_be_nullable; - - bool any_input_nullable = std::any_of(inputs.begin(), inputs.end(), [](auto& input) { - return std::visit([](auto const& col) { return col.nullable(); }, input); + bool any_input_nullable = std::any_of(inputs.begin(), inputs.end(), [](auto& in) { + return std::visit([](auto& c) { return c.nullable(); }, in); }); + std::vector output_may_be_nullable; for (auto& out : outputs) { - bool may_eval_null = true; + bool may_eval_nulls = true; if (is_null_aware == null_aware::YES) { - // null-aware UDFs may evaluate nulls unless the output is explicitly marked as all valid - may_eval_null = out.nullability != output_nullability::ALL_VALID; + // null-aware UDFs may evaluate nulls unless the output is explicitly marked as all valid + may_eval_nulls = out.nullability != output_nullability::ALL_VALID; } else { - // null-unaware UDFs may evaluate nulls if any input is nullable unless explicitly marked as not producing nulls - may_eval_null = any_input_nullable && (out.nullability == output_nullability::PRESERVE); + // null-unaware UDFs may evaluate nulls if any input is nullable unless explicitly marked as + // not producing nulls + may_eval_nulls = any_input_nullable && (out.nullability == output_nullability::PRESERVE); } - output_may_be_nullable.push_back(may_eval_null); + output_may_be_nullable.push_back(may_eval_nulls); } - return {input_may_be_nullable, output_may_be_nullable}; -} - -auto finalize(std::vector outputs, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - std::vector> results; - - // TODO: finish - for (auto& out : outputs) { - if (auto* col = std::get_if>(&out)) { - results.push_back(std::move(*col)); - } else if (auto& str = std::get>(out)) { - auto result = detail::make_strings_column( - str->_data, std::move(str->_null_mask), std::nullopt, stream, mr); - results.push_back(std::move(result)); - } - } - - return results; + return std::make_tuple(std::move(input_may_be_nullable), std::move(output_may_be_nullable)); } void perform_checks(udf_source_type source_type, null_aware is_null_aware, std::optional in_row_size, - InputViews inputs, + std::span inputs, std::span outputs, std::span const> string_offsets) { @@ -382,26 +494,28 @@ void perform_checks(udf_source_type source_type, "Optional types are not supported in PTX UDFs", std::invalid_argument); - CUDF_EXPECTS(is_fixed_width(output_type) || output_type.id() == type_id::STRING, + CUDF_EXPECTS(std::all_of(outputs.begin(), + outputs.end(), + [](auto& out) { + return is_fixed_width(out.type) || (out.type.id() == type_id::STRING); + }), "Transforms only support output of fixed-width or string types", std::invalid_argument); - auto get_type = [](auto const& in) { - return std::visit([](auto const& col) { return col.type(); }, in); - }; - - CUDF_EXPECTS( - std::all_of(thrust::make_transform_iterator(inputs.begin(), get_type), - thrust::make_transform_iterator(inputs.end(), get_type), - [](data_type t) { return is_fixed_width(t) || (t.id() == type_id::STRING); }), - "Transforms only support input of fixed-width or string types", - std::invalid_argument); + CUDF_EXPECTS(std::all_of(inputs.begin(), + inputs.end(), + [&](auto& in) { + auto type = std::visit([](auto& c) { return c.type(); }, in); + return is_fixed_width(type) || (type.id() == type_id::STRING); + }), + "Transforms only support input of fixed-width or string types", + std::invalid_argument); if (!in_row_size.has_value()) { CUDF_EXPECTS( std::any_of(inputs.begin(), inputs.end(), - [](auto const& input) { return std::holds_alternative(input); }), + [](auto const& in) { return std::holds_alternative(in); }), "At least one input of a transform must be a non-scalar column if row size is not provided", std::invalid_argument); } @@ -410,7 +524,7 @@ void perform_checks(udf_source_type source_type, CUDF_EXPECTS(std::all_of(inputs.begin(), inputs.end(), [&](auto& in) { - if (auto* col = std::get_if(&input)) { + if (auto* col = std::get_if(&in)) { return col->size() == row_size; } return true; @@ -418,94 +532,273 @@ void perform_checks(udf_source_type source_type, "All transform input columns must have the same size", std::invalid_argument); - // TODO: if string offset is provided, make sure it is a string column + CUDF_EXPECTS(string_offsets.empty() || (string_offsets.size() == outputs.size()), + "Number of string offsets must be empty or match the number of outputs", + std::invalid_argument); + + CUDF_EXPECTS(std::all_of(thrust::make_counting_iterator(size_t{0}), + thrust::make_counting_iterator(string_offsets.size()), + [&](auto i) { + if (outputs[i].type.id() == type_id::STRING) { return true; } + return string_offsets.empty() || string_offsets[i] != nullptr; + }), + "String offsets must only be provided for string outputs", + std::invalid_argument); } -std::unique_ptr
execute_transform(std::string const& udf, - udf_source_type source_type, - null_aware is_null_aware, - std::optional in_row_size, - std::optional user_data, - InputViews inputs, - std::span outputs, - std::vector> string_offsets, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::optional prepare_stencil(null_aware is_null_aware, + bool null_strict, + size_type row_size, + std::span inputs, + std::span outputs, + std::span output_masks, + rmm::cuda_stream_view stream) { - auto row_size = in_row_size.value_or(jit::get_projection_size(inputs)); - // TODO: account for MayBeNullable + CUDF_FUNC_RANGE(); - auto [input_may_be_nullable, output_may_be_nullable] = - get_nullabilities(is_null_aware, inputs, outputs); + // null-aware, no stencil needed + if (is_null_aware != null_aware::NO) { return std::nullopt; } + + std::optional stencil = std::nullopt; + + for (auto* mask : output_masks) { + if (mask != nullptr) { + stencil = mask; + break; + } + } + + // no nullable outputs + if (!stencil.has_value()) { + if (!null_strict) { + // function is not null-strict, we don't need to use the stencil in the kernel + return std::nullopt; + } + + return nullptr; + } + + auto null_count = inplace_null_mask_and(*stencil, row_size, inputs, stream); + + for (size_t i = 0; i < outputs.size(); i++) { + auto* mask = output_masks[i]; + + if (mask != nullptr) { + if (mask != *stencil) { + detail::cuda_memcpy_async_impl(mask, + *stencil, + bitmask_allocation_size_bytes(row_size), + detail::host_memory_kind::PINNED, + stream); + } + + std::visit( + [&](auto& c) { + if (c.nullable()) { + c.set_null_count(null_count); + } else { + c.set_null_count(0); + } + }, + outputs[i]); + } + } + + if (!null_strict) { + // function is not null-strict, we don't need to use the stencil in the kernel + return std::nullopt; + } + + return stencil; +} + +auto allocate_outputs(size_type row_size, + std::span outputs, + std::span is_nullable, + std::vector> string_offsets, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + + std::vector cols; + std::vector masks; for (size_t i = 0; i < outputs.size(); i++) { - auto output = outputs[i]; - auto may_be_nullable = output_may_be_nullable[i]; - auto null_state = - may_be_nullable ? null_mask_state::UNINITIALIZED : null_mask_state::UNALLOCATED; - - if (is_fixed_width(type)) { - auto col = make_fixed_width_column(type, row_size, null_state, stream, mr); - } else if (type == type_id::STRING) { - if (string_offsets[i] == nullptr) { - auto col = string_views_column::make(row_size, null_state, stream, mr); - outputs.push_back(std::make_unique(std::move(col))); + auto output = outputs[i]; + auto nullable = is_nullable[i]; + auto null_state = nullable ? mask_state::UNINITIALIZED : mask_state::UNALLOCATED; + auto null_mask = create_null_mask(row_size, null_state, stream, mr); + masks.push_back(static_cast(null_mask.data())); + + if (is_fixed_width(output.type)) { + auto col = + fixed_width_column::make(output.type, row_size, std::move(null_mask), 0, stream, mr); + cols.push_back(std::move(col)); + } else if (output.type.id() == type_id::STRING) { + if (string_offsets.empty() || string_offsets[i] == nullptr) { + auto col = string_views_column::make(row_size, std::move(null_mask), 0, stream, mr); + cols.push_back(std::move(col)); } else { - auto col = mut_strings_column::make(row_size, - /*TODO: chars_size*/ 0, - std::move(string_offsets[i]), - null_state, - stream, - mr); + auto chars_size = + strings::detail::get_offset_value(string_offsets[i]->view(), row_size, stream); + auto chars = rmm::device_buffer{static_cast(chars_size), stream, mr}; + auto col = mut_strings_column::make( + row_size, std::move(chars), std::move(string_offsets[i]), std::move(null_mask), 0); + cols.push_back(std::move(col)); } } else { CUDF_UNREACHABLE("Unsupported output type for transform"); } } - // stencil in-place - // inplace_bitmask_and - bool any_output_nullable = std::any_of( - output_may_be_nullable.begin(), output_may_be_nullable.end(), [](auto b) { return b; }); - bool can_use_stencil = (is_null_aware == null_aware::NO); + return std::make_tuple(std::move(cols), std::move(masks)); +} - std::vector outputs; - std::optional stencil = std::nullopt; - // TODO: create a null mask and copy to all outputs, don't create stencil if all the outputs are - // non-nullable - // TODO: how will this affect kernel and outputs? - // TODO: Null-mask-and might return nullptr - // TODO: if no output is nullable, the bitmask and should be nullptr - // TODO: when copying bitmasks stencil, only copy to the nullable outputs - - jit::transform::run(is_null_aware, - may_return_nulls, - user_data.has_value(), - row_size, - user_data.value_or(nullptr), - inputs, - nullptr, // TODO: compute and pass the and of the null masks if needed - outputs, - udf, - source_type, - stream, - mr); - - return std::make_unique
(finalize(std::move(outputs), stream, mr)); +auto prepare_outputs(null_aware is_null_aware, + size_type row_size, + std::span inputs, + std::span outputs, + std::span output_is_nullable, + std::vector> string_offsets, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + + auto [output_columns, output_column_masks] = + allocate_outputs(row_size, outputs, output_is_nullable, std::move(string_offsets), stream, mr); + auto stencil = prepare_stencil( + is_null_aware, true, row_size, inputs, output_columns, output_column_masks, stream); + + return std::make_tuple(std::move(output_columns), stencil); +} + +void update_null_counts(std::span outputs, + null_aware is_null_aware, + size_type row_size, + rmm::cuda_stream_view stream) +{ + // update null counts if the function is not null-aware, since we haven't processed nullability + // ahead of time + if (is_null_aware == null_aware::YES) { + std::vector bitmasks; + std::vector indices; + + for (size_t i = 0; i < outputs.size(); i++) { + std::visit( + [&](auto& c) { + if (c.nullable()) { + indices.push_back(i); + bitmasks.push_back(c.null_mask()); + } + }, + outputs[i]); + } + + auto null_counts = batch_null_count(bitmasks, 0, row_size, stream); + + for (size_t i = 0; i < bitmasks.size(); i++) { + std::visit([&](auto& c) { c.set_null_count(null_counts[i]); }, outputs[indices[i]]); + } + } +} + +auto finalize_output(fixed_width_column&& c, rmm::cuda_stream_view, rmm::device_async_resource_ref) +{ + return std::move(c._col); +} + +auto finalize_output(mut_strings_column&& c, rmm::cuda_stream_view, rmm::device_async_resource_ref) +{ + return std::move(c._col); +} + +auto finalize_output(string_views_column&& c, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + return strings::detail::make_strings_column( + device_span{static_cast(c._data.data()), + static_cast(c._size)}, + std::move(c._null_mask), + c._null_count, + stream, + mr); +} + +auto finalize_outputs(null_aware is_null_aware, + size_type row_size, + std::vector outputs, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + + update_null_counts(outputs, is_null_aware, row_size, stream); + + std::vector> results; + + for (auto& out : outputs) { + std::visit([&](auto& c) { results.push_back(finalize_output(std::move(c), stream, mr)); }, out); + } + + return results; +} + +std::unique_ptr
execute_transform(std::string const& udf, + udf_source_type source_type, + null_aware is_null_aware, + std::optional in_row_size, + std::optional user_data, + std::span inputs, + std::span outputs, + std::vector> string_offsets, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + auto row_size = in_row_size.value_or(jit::get_projection_size(inputs)); + auto [input_may_be_nullable, output_may_be_nullable] = + get_null_transformation(is_null_aware, inputs, outputs); + auto [output_columns, stencil] = prepare_outputs(is_null_aware, + row_size, + inputs, + outputs, + output_may_be_nullable, + std::move(string_offsets), + stream, + mr); + jit_transform::run(is_null_aware, + stencil.has_value(), + user_data.has_value(), + row_size, + stencil.value_or(nullptr), + user_data.value_or(nullptr), + inputs, + output_columns, + input_may_be_nullable, + output_may_be_nullable, + udf, + source_type, + stream, + mr); + + auto finalized = finalize_outputs(is_null_aware, row_size, std::move(output_columns), stream, mr); + return std::make_unique
(std::move(finalized)); } } // namespace -std::unique_ptr
transform_extended2(std::string const& udf, - udf_source_type source_type, - null_aware is_null_aware, - std::optional row_size, - std::optional user_data, - std::span inputs, - std::span outputs, - std::vector> string_offsets, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::unique_ptr
multi_transform(std::string const& udf, + udf_source_type source_type, + null_aware is_null_aware, + std::optional user_data, + std::span inputs, + std::span outputs, + std::vector> string_offsets, + std::optional row_size, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); perform_checks(source_type, is_null_aware, row_size, inputs, outputs, string_offsets); @@ -521,21 +814,22 @@ std::unique_ptr
transform_extended2(std::string const& udf, mr); } -std::unique_ptr transform_extended( - std::span const> inputs, - std::string const& udf, - data_type output_type, - udf_source_type source_type, - std::optional user_data, - null_aware is_null_aware, - std::optional row_size, - output_nullability null_policy, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::unique_ptr transform_extended(std::span inputs, + std::string const& udf, + data_type output_type, + udf_source_type source_type, + std::optional user_data, + null_aware is_null_aware, + std::optional row_size, + output_nullability null_policy, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { - transform_output outputs[] = {{.type = output_type, .null_policy = null_policy}}; - auto table = transform_extended2( - udf, source_type, is_null_aware, row_size, user_data, inputs, outputs, {}, stream, mr); + transform_output outputs[] = {{.type = output_type, .nullability = null_policy}}; + auto table = multi_transform( + udf, source_type, is_null_aware, user_data, inputs, outputs, {}, row_size, stream, mr); + auto cols = table->release(); + return std::move(cols[0]); } std::unique_ptr transform(std::vector const& columns, @@ -550,7 +844,7 @@ std::unique_ptr transform(std::vector const& columns, { CUDF_FUNC_RANGE(); // legacy behavior was to detect which column were scalars based on their sizes - std::vector> inputs; + std::vector inputs; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index 025cacb518aa..cab51a313b57 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -204,24 +204,69 @@ TEST_F(UnaryOperationIntegrationTest, Transform_INT32_INT32) std::string const cuda = "__device__ inline void f(int* output,int input){*output = input*input - input;}"; + // Generated from NUMBA, using: + // + // ```py + // + // from numba import cuda, float32 + // from numba.cuda import compile_ptx_for_current_device + // + // # Define a CUDA device function + // + // @cuda.jit(device=True) + // def op(a): + // return a * a - a + // + // # Define argument types for the function + // arg_types = (float32, ) + // + // # Compile the device function as relocatable + // ptx, _ = cuda.compile_ptx_for_current_device(op, arg_types, device=True) + // + // + // # Print the PTX code + // print("Relocatable PTX Code:") + // print(ptx) + // + // + // ``` + // std::string const ptx = R"***( -.func _Z1fPii( - .param .b64 _Z1fPii_param_0, - .param .b32 _Z1fPii_param_1 +// +// Generated by NVIDIA NVVM Compiler +// +// Compiler Build ID: CL-37061995 +// Cuda compilation tools, release 13.1, V13.1.115 +// Based on NVVM 7.0.1 +// + +.version 9.1 +.target sm_86 +.address_size 64 + + // .globl _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd +.common .global .align 8 .u64 _ZN08NumbaEnv8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd; + +.visible .func (.param .b32 func_retval0) _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd( + .param .b64 _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_0, + .param .b64 _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_1 ) { - .reg .b32 %r<4>; - .reg .b64 %rd<3>; + .reg .b32 %r<2>; + .reg .f64 %fd<4>; + .reg .b64 %rd<2>; - ld.param.u64 %rd1, [_Z1fPii_param_0]; - ld.param.u32 %r1, [_Z1fPii_param_1]; - cvta.to.global.u64 %rd2, %rd1; - mul.lo.s32 %r2, %r1, %r1; - sub.s32 %r3, %r2, %r1; - st.global.u32 [%rd2], %r3; - ret; + ld.param.u64 %rd1, [_ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_0]; + ld.param.f64 %fd1, [_ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_1]; + mul.f64 %fd2, %fd1, %fd1; + sub.f64 %fd3, %fd2, %fd1; + st.f64 [%rd1], %fd3; + mov.u32 %r1, 0; + st.param.b32 [func_retval0+0], %r1; + ret; + } )***"; @@ -252,31 +297,82 @@ __device__ inline void f( } )***"; + // Generated from NUMBA, using: + // + // ```py + // + // from numba import cuda, float32 + // from numba.cuda import compile_ptx_for_current_device + // + // # Define a CUDA device function + // + // @cuda.jit(device=True) + // def op(a): + // return (a - 32) if (a > 96 and a < 123) else a + // + // # Define argument types for the function + // arg_types = (float32, float32, float32) + // + // # Compile the device function as relocatable + // ptx, _ = cuda.compile_ptx_for_current_device(op, arg_types, device=True) + // + // + // # Print the PTX code + // print("Relocatable PTX Code:") + // print(ptx) + // + // + // ``` + // std::string const ptx = R"***( -.func _Z1fPcc( - .param .b64 _Z1fPcc_param_0, - .param .b32 _Z1fPcc_param_1 +// +// Generated by NVIDIA NVVM Compiler +// +// Compiler Build ID: CL-37061995 +// Cuda compilation tools, release 13.1, V13.1.115 +// Based on NVVM 7.0.1 +// + +.version 9.1 +.target sm_86 +.address_size 64 + + // .globl _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd +.common .global .align 8 .u64 _ZN08NumbaEnv8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd; + +.visible .func (.param .b32 func_retval0) _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd( + .param .b64 _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_0, + .param .b64 _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_1 ) { - .reg .pred %p<2>; - .reg .b16 %rs<6>; - .reg .b32 %r<3>; - .reg .b64 %rd<3>; - - - ld.param.u64 %rd1, [_Z1fPcc_param_0]; - cvta.to.global.u64 %rd2, %rd1; - ld.param.s8 %rs1, [_Z1fPcc_param_1]; - add.s16 %rs2, %rs1, -97; - and.b16 %rs3, %rs2, 255; - setp.lt.u16 %p1, %rs3, 26; - cvt.u32.u16 %r1, %rs1; - add.s32 %r2, %r1, 224; - cvt.u16.u32 %rs4, %r2; - selp.b16 %rs5, %rs4, %rs1, %p1; - st.global.u8 [%rd2], %rs5; - ret; + .reg .pred %p<4>; + .reg .b32 %r<2>; + .reg .f64 %fd<3>; + .reg .b64 %rd<2>; + + + ld.param.u64 %rd1, [_ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_0]; + ld.param.f64 %fd1, [_ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_1]; + setp.gt.f64 %p1, %fd1, 0d4058000000000000; + setp.lt.f64 %p2, %fd1, 0d405EC00000000000; + and.pred %p3, %p1, %p2; + @%p3 bra $L__BB0_2; + bra.uni $L__BB0_1; + +$L__BB0_2: + add.f64 %fd2, %fd1, 0dC040000000000000; + st.f64 [%rd1], %fd2; + bra.uni $L__BB0_3; + +$L__BB0_1: + st.f64 [%rd1], %fd1; + +$L__BB0_3: + mov.u32 %r1, 0; + st.param.b32 [func_retval0+0], %r1; + ret; + } )***"; From 69ff5e196cd4528eb73287a1cd1965c546add966 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 12 Mar 2026 02:39:37 +0000 Subject: [PATCH 071/254] Add [[nodiscard]] attribute to head() and data() methods in mutable_column_device_view_core --- cpp/include/cudf/column/column_device_view_base.cuh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/cudf/column/column_device_view_base.cuh b/cpp/include/cudf/column/column_device_view_base.cuh index 39cd4878c258..6e59c48ae469 100644 --- a/cpp/include/cudf/column/column_device_view_base.cuh +++ b/cpp/include/cudf/column/column_device_view_base.cuh @@ -586,7 +586,7 @@ class alignas(16) mutable_column_device_view_core : public detail::column_device */ template or is_rep_layout_compatible())> - CUDF_HOST_DEVICE T* head() const noexcept + [[nodiscard]] CUDF_HOST_DEVICE T* head() const noexcept { return const_cast(detail::column_device_view_base::head()); } @@ -604,7 +604,7 @@ class alignas(16) mutable_column_device_view_core : public detail::column_device * @return Typed pointer to underlying data, including the offset */ template ())> - CUDF_HOST_DEVICE T* data() const noexcept + [[nodiscard]] CUDF_HOST_DEVICE T* data() const noexcept { return const_cast(detail::column_device_view_base::data()); } From 2ab3bc520589730855470c919e78515d17fcaf92 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 12 Mar 2026 07:41:08 +0000 Subject: [PATCH 072/254] Refactoring + add support for zero inputs --- cpp/src/transform/transform.cu | 251 ++++++++++++++++----------------- 1 file changed, 119 insertions(+), 132 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 1d6e939ac25d..12090fd6ca17 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -229,8 +229,8 @@ std::string reflect_output_column(mut_strings_column const&) auto reflect(udf_source_type source_type, std::span inputs, std::span outputs, - std::span input_may_be_nullable, - std::span output_may_be_nullable) + std::span input_may_be_nullable, + std::span output_may_be_nullable) { std::vector ins; @@ -267,8 +267,8 @@ auto reflect(udf_source_type source_type, outs.push_back(accessor); } - auto in_list = jitify2::reflection::Template("cudf::jit::type_list").instantiate(ins); - auto out_list = jitify2::reflection::Template("cudf::jit::type_list").instantiate(outs); + auto ins = jitify2::reflection::Template("cudf::jit::type_list").instantiate(ins); + auto outs = jitify2::reflection::Template("cudf::jit::type_list").instantiate(outs); std::vector ptx_input_types; std::vector ptx_output_types; @@ -284,7 +284,7 @@ auto reflect(udf_source_type source_type, } } - return std::make_tuple(in_list, out_list, ptx_input_types, ptx_output_types); + return std::make_tuple(ins, outs, ptx_input_types, ptx_output_types); } auto to_args(std::span inputs, @@ -337,8 +337,8 @@ void run(null_aware is_null_aware, void* user_data, std::span inputs, std::span outputs, - std::span input_may_be_nullable, - std::span output_may_be_nullable, + std::span input_may_be_nullable, + std::span output_may_be_nullable, std::string const& udf, udf_source_type source_type, rmm::cuda_stream_view stream, @@ -368,6 +368,8 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, std::span inputs, rmm::cuda_stream_view stream) { + CUDF_FUNC_RANGE(); + auto is_nullable = null_mask != nullptr; if (!is_nullable) { return 0; } @@ -375,7 +377,11 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, auto num_words = num_bitmask_words(row_size); auto num_bytes = num_words * sizeof(bitmask_type); - CUDF_EXPECTS(!inputs.empty(), "Inputs must not be empty", std::logic_error); + if (inputs.empty()) { + // no input, set all to valid + CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0xFF, num_bytes, stream.value())); + return 0; + } auto has_scalars = std::any_of(inputs.begin(), inputs.end(), [](auto& in) { return std::holds_alternative(in); @@ -390,7 +396,6 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, if (scalar_is_null) { // scalar is null, all rows will be null CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0x00, num_bytes, stream.value())); - return row_size; } } @@ -401,7 +406,6 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, if (has_cols) { // no non-scalar columns, so all rows are valid CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0xFF, num_bytes, stream.value())); - return 0; } @@ -423,24 +427,24 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, if (nullable_masks.empty()) { // we only have non-nullable columns, so all rows are valid CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0xFF, num_bytes, stream.value())); - return 0; } if (nullable_masks.size() == 1) { // only 1 mask provided, copy it directly to the output - detail::cuda_memcpy_async_impl( null_mask, nullable_masks[0], num_bytes, detail::host_memory_kind::PINNED, stream); return nullable_null_counts[0]; } - return detail::inplace_bitmask_and( + auto num_valid = detail::inplace_bitmask_and( device_span{null_mask, static_cast(num_words)}, nullable_masks, nullable_offsets, row_size, stream); + + return row_size - std::min(num_valid, row_size); } /** @@ -453,17 +457,17 @@ auto get_null_transformation(null_aware is_null_aware, std::span inputs, std::span outputs) { - std::vector input_may_be_nullable; + std::vector input_may_be_nullable; for (auto& in : inputs) { input_may_be_nullable.push_back(true); } - bool any_input_nullable = std::any_of(inputs.begin(), inputs.end(), [](auto& in) { + auto any_input_nullable = std::any_of(inputs.begin(), inputs.end(), [](auto& in) { return std::visit([](auto& c) { return c.nullable(); }, in); }); - std::vector output_may_be_nullable; + std::vector output_may_be_nullable; for (auto& out : outputs) { bool may_eval_nulls = true; if (is_null_aware == null_aware::YES) { @@ -488,26 +492,52 @@ void perform_checks(udf_source_type source_type, std::span outputs, std::span const> string_offsets) { - CUDF_EXPECTS( - !inputs.empty(), "Transform must have at least 1 input column", std::invalid_argument); - CUDF_EXPECTS(!(is_null_aware == null_aware::YES && source_type == udf_source_type::PTX), - "Optional types are not supported in PTX UDFs", - std::invalid_argument); + if (source_type == udf_source_type::PTX) { + CUDF_EXPECTS(std::none_of(inputs.begin(), + inputs.end(), + [](auto& in) { + return std::visit( + [](auto& c) { + auto type = c.type(); + return !is_integer(type) && !is_floating_point(type) && + type.id() != type_id::BOOL8; + }, + in); + }), + "Transforms with PTX UDFs only support integer, floating-point, and boolean", + std::invalid_argument); + CUDF_EXPECTS(std::none_of(outputs.begin(), + outputs.end(), + [](auto& out) { + return std::visit( + [](auto& c) { + auto type = c.type(); + return !is_integer(type) && !is_floating_point(type) && + type.id() != type_id::BOOL8; + }, + out); + }), + "Transforms with PTX UDFs only support integer, floating-point, and boolean types", + std::invalid_argument); + CUDF_EXPECTS(is_null_aware == null_aware::NO, + "PTX UDFs do not support null-aware transformations", + std::invalid_argument); + } - CUDF_EXPECTS(std::all_of(outputs.begin(), - outputs.end(), - [](auto& out) { - return is_fixed_width(out.type) || (out.type.id() == type_id::STRING); - }), + CUDF_EXPECTS(std::none_of(outputs.begin(), + outputs.end(), + [](auto& out) { + return !is_fixed_width(out.type) && out.type.id() != type_id::STRING; + }), "Transforms only support output of fixed-width or string types", std::invalid_argument); - CUDF_EXPECTS(std::all_of(inputs.begin(), - inputs.end(), - [&](auto& in) { - auto type = std::visit([](auto& c) { return c.type(); }, in); - return is_fixed_width(type) || (type.id() == type_id::STRING); - }), + CUDF_EXPECTS(std::none_of(inputs.begin(), + inputs.end(), + [&](auto& in) { + auto type = std::visit([](auto& c) { return c.type(); }, in); + return !is_fixed_width(type) && type.id() != type_id::STRING; + }), "Transforms only support input of fixed-width or string types", std::invalid_argument); @@ -520,39 +550,38 @@ void perform_checks(udf_source_type source_type, std::invalid_argument); } - auto row_size = in_row_size.value_or(jit::get_projection_size(inputs)); - CUDF_EXPECTS(std::all_of(inputs.begin(), - inputs.end(), - [&](auto& in) { - if (auto* col = std::get_if(&in)) { - return col->size() == row_size; - } - return true; - }), + auto row_size = in_row_size.has_value() ? *in_row_size : jit::get_projection_size(inputs); + CUDF_EXPECTS(std::none_of(inputs.begin(), + inputs.end(), + [&](auto& in) { + if (auto* col = std::get_if(&in)) { + return col->size() != row_size; + } + return false; + }), "All transform input columns must have the same size", std::invalid_argument); CUDF_EXPECTS(string_offsets.empty() || (string_offsets.size() == outputs.size()), - "Number of string offsets must be empty or match the number of outputs", + "Number of string offsets must be empty or match the number of outputs (with nulls " + "for each non-string column)", std::invalid_argument); - CUDF_EXPECTS(std::all_of(thrust::make_counting_iterator(size_t{0}), - thrust::make_counting_iterator(string_offsets.size()), - [&](auto i) { - if (outputs[i].type.id() == type_id::STRING) { return true; } - return string_offsets.empty() || string_offsets[i] != nullptr; - }), + CUDF_EXPECTS(std::none_of(thrust::make_counting_iterator(size_t{0}), + thrust::make_counting_iterator(string_offsets.size()), + [&](auto i) { + if (outputs[i].type.id() == type_id::STRING) { return false; } + return !string_offsets.empty() && string_offsets[i] == nullptr; + }), "String offsets must only be provided for string outputs", std::invalid_argument); } -std::optional prepare_stencil(null_aware is_null_aware, - bool null_strict, - size_type row_size, - std::span inputs, - std::span outputs, - std::span output_masks, - rmm::cuda_stream_view stream) +std::optional make_stencil(null_aware is_null_aware, + size_type row_size, + std::span inputs, + std::span outputs, + rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); @@ -561,75 +590,51 @@ std::optional prepare_stencil(null_aware is_null_aware, std::optional stencil = std::nullopt; - for (auto* mask : output_masks) { - if (mask != nullptr) { - stencil = mask; - break; - } + for (auto& out : outputs) { + if (auto* mask = std::visit([&](auto& c) { return c.null_mask(); }, out)) { stencil = mask; } } // no nullable outputs - if (!stencil.has_value()) { - if (!null_strict) { - // function is not null-strict, we don't need to use the stencil in the kernel - return std::nullopt; - } - - return nullptr; - } + if (!stencil.has_value()) { return nullptr; } - auto null_count = inplace_null_mask_and(*stencil, row_size, inputs, stream); - - for (size_t i = 0; i < outputs.size(); i++) { - auto* mask = output_masks[i]; - - if (mask != nullptr) { - if (mask != *stencil) { - detail::cuda_memcpy_async_impl(mask, - *stencil, - bitmask_allocation_size_bytes(row_size), - detail::host_memory_kind::PINNED, - stream); - } + auto stencil_null_count = inplace_null_mask_and(*stencil, row_size, inputs, stream); - std::visit( - [&](auto& c) { - if (c.nullable()) { - c.set_null_count(null_count); - } else { - c.set_null_count(0); - } - }, - outputs[i]); + for (auto& out : outputs) { + auto* mask = std::visit([&](auto& c) { return c.null_mask(); }, out); + + if (mask != nullptr && mask != *stencil) { + detail::cuda_memcpy_async_impl(mask, + *stencil, + bitmask_allocation_size_bytes(row_size), + detail::host_memory_kind::PINNED, + stream); } - } - if (!null_strict) { - // function is not null-strict, we don't need to use the stencil in the kernel - return std::nullopt; + auto null_count = mask == nullptr ? 0 : stencil_null_count; + + std::visit([&](auto& c) { c.set_null_count(null_count); }, out); } return stencil; } -auto allocate_outputs(size_type row_size, - std::span outputs, - std::span is_nullable, - std::vector> string_offsets, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +auto make_outputs(size_type row_size, + std::span inputs, + std::span outputs, + std::span is_output_nullable, + std::vector> string_offsets, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); std::vector cols; - std::vector masks; for (size_t i = 0; i < outputs.size(); i++) { auto output = outputs[i]; - auto nullable = is_nullable[i]; + auto nullable = is_output_nullable[i]; auto null_state = nullable ? mask_state::UNINITIALIZED : mask_state::UNALLOCATED; auto null_mask = create_null_mask(row_size, null_state, stream, mr); - masks.push_back(static_cast(null_mask.data())); if (is_fixed_width(output.type)) { auto col = @@ -652,24 +657,7 @@ auto allocate_outputs(size_type row_size, } } - return std::make_tuple(std::move(cols), std::move(masks)); -} - -auto prepare_outputs(null_aware is_null_aware, - size_type row_size, - std::span inputs, - std::span outputs, - std::span output_is_nullable, - std::vector> string_offsets, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - CUDF_FUNC_RANGE(); - - auto [output_columns, output_column_masks] = - allocate_outputs(row_size, outputs, output_is_nullable, std::move(string_offsets), stream, mr); - auto stencil = prepare_stencil( - is_null_aware, true, row_size, inputs, output_columns, output_column_masks, stream); + auto stencil = make_stencil(is_null_aware, row_size, inputs, output_columns, stream); return std::make_tuple(std::move(output_columns), stencil); } @@ -680,7 +668,7 @@ void update_null_counts(std::span outputs, rmm::cuda_stream_view stream) { // update null counts if the function is not null-aware, since we haven't processed nullability - // ahead of time + // ahead of time (as in the non-null-aware case) if (is_null_aware == null_aware::YES) { std::vector bitmasks; std::vector indices; @@ -757,17 +745,17 @@ std::unique_ptr
execute_transform(std::string const& udf, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - auto row_size = in_row_size.value_or(jit::get_projection_size(inputs)); + auto row_size = in_row_size.has_value() ? *in_row_size : jit::get_projection_size(inputs); auto [input_may_be_nullable, output_may_be_nullable] = get_null_transformation(is_null_aware, inputs, outputs); - auto [output_columns, stencil] = prepare_outputs(is_null_aware, - row_size, - inputs, - outputs, - output_may_be_nullable, - std::move(string_offsets), - stream, - mr); + auto [output_columns, stencil] = make_outputs(is_null_aware, + row_size, + inputs, + outputs, + output_may_be_nullable, + std::move(string_offsets), + stream, + mr); jit_transform::run(is_null_aware, stencil.has_value(), user_data.has_value(), @@ -842,7 +830,6 @@ std::unique_ptr transform(std::vector const& columns, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - CUDF_FUNC_RANGE(); // legacy behavior was to detect which column were scalars based on their sizes std::vector inputs; From 176d356a733b24c3705e38328765ad14f44ee3c9 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 12 Mar 2026 07:42:49 +0000 Subject: [PATCH 073/254] added test todos --- .../integration/unary_transform_test.cpp | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index cab51a313b57..3b9c3e78d18f 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -1033,4 +1033,34 @@ return l - t * l + t * h; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*cuda_result, *expected); } +// TODO: add offseted string output support tests +// TODO: add multi-output transform tests +// TODO: add zero-input transform test +// TODO: merge main +/** +* +* TODO: Regex extract with multiple capture groups, then the JIT implementation would just need to +handle multiple column outputs without handling AST common subexpression elimination. + +Example: +Example: Phone Number +Format: (123) 456-7890 + +Regex pattern: +\((\d{3})\)\s(\d{3})-(\d{4}) + +Example matches: +Input: (415) 555-1234 +Column 1: 415 +Column 2: 555 +Column 3: 1234 + +Input: (212) 867-5309 +Column 1: 212 +Column 2: 867 +Column 3: 5309 +* +* +*/ + } // namespace transformation From 8ff447da41edbf4f3037eae12aeb8e04fd5ed470 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 12 Mar 2026 09:59:04 +0000 Subject: [PATCH 074/254] Refactor variable names for clarity in transform functions --- cpp/src/column/column_device_view.cu | 18 +++++++++--------- cpp/src/transform/transform.cu | 23 +++++++++++------------ 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/cpp/src/column/column_device_view.cu b/cpp/src/column/column_device_view.cu index e693a4d041c0..f30f54ea93a8 100644 --- a/cpp/src/column/column_device_view.cu +++ b/cpp/src/column/column_device_view.cu @@ -20,20 +20,20 @@ namespace cudf { template -inline constexpr bool bitwise_compatible = (sizeof(A) == sizeof(B)) && (alignof(A) == alignof(B)); +inline constexpr bool layout_compatible = (sizeof(A) == sizeof(B)) && (alignof(A) == alignof(B)); static_assert( - bitwise_compatible, - "detail::column_device_view_base and column_device_view_core must be bitwise-compatible"); -static_assert(bitwise_compatible, - "column_device_view_core and column_device_view must be bitwise-compatible"); + layout_compatible, + "detail::column_device_view_base and column_device_view_core must be layout-compatible"); +static_assert(layout_compatible, + "column_device_view_core and column_device_view must be layout-compatible"); static_assert( - bitwise_compatible, - "detail::column_device_view_base and mutable_column_device_view_core must be bitwise-compatible"); + layout_compatible, + "detail::column_device_view_base and mutable_column_device_view_core must be layout-compatible"); static_assert( - bitwise_compatible, - "mutable_column_device_view_core and mutable_column_device_view must be bitwise-compatible"); + layout_compatible, + "mutable_column_device_view_core and mutable_column_device_view must be layout-compatible"); // Trivially copy all members but the children column_device_view::column_device_view(column_view source) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 12090fd6ca17..0a53fd38b32b 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -232,7 +232,7 @@ auto reflect(udf_source_type source_type, std::span input_may_be_nullable, std::span output_may_be_nullable) { - std::vector ins; + std::vector in_types; for (size_t i = 0; i < inputs.size(); i++) { auto& in = inputs[i]; @@ -246,10 +246,10 @@ auto reflect(udf_source_type source_type, jitify2::reflection::Template("cudf::jit::column_accessor") .instantiate( i, column, element, optional_element, as_scalar, may_be_nullable, is_strings_output); - ins.push_back(accessor); + in_types.push_back(accessor); } - std::vector outs; + std::vector out_types; for (size_t i = 0; i < outputs.size(); i++) { auto& out = outputs[i]; @@ -264,27 +264,26 @@ auto reflect(udf_source_type source_type, .instantiate( i, column, element, optional_element, as_scalar, may_be_nullable, is_strings_output); - outs.push_back(accessor); + out_types.push_back(accessor); } - auto ins = jitify2::reflection::Template("cudf::jit::type_list").instantiate(ins); - auto outs = jitify2::reflection::Template("cudf::jit::type_list").instantiate(outs); + auto ins = jitify2::reflection::Template("cudf::jit::type_list").instantiate(in_types); + auto outs = jitify2::reflection::Template("cudf::jit::type_list").instantiate(out_types); - std::vector ptx_input_types; - std::vector ptx_output_types; + std::vector ptx_in_types; + std::vector ptx_out_types; if (source_type == udf_source_type::PTX) { for (auto& in : inputs) { - ptx_input_types.push_back(std::visit([](auto& c) { return reflect_input_element(c); }, in)); + ptx_in_types.push_back(std::visit([](auto& c) { return reflect_input_element(c); }, in)); } for (auto& out : outputs) { - ptx_output_types.push_back( - std::visit([](auto& c) { return reflect_output_element(c); }, out)); + ptx_out_types.push_back(std::visit([](auto& c) { return reflect_output_element(c); }, out)); } } - return std::make_tuple(ins, outs, ptx_input_types, ptx_output_types); + return std::make_tuple(ins, outs, ptx_in_types, ptx_out_types); } auto to_args(std::span inputs, From 4bd0c72ec7dc1829ca919ca0d023882fe8990060 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 13 Mar 2026 05:02:42 +0000 Subject: [PATCH 075/254] Refactor JIT transform kernel and related components - Updated column_accessor to use __restrict__ qualifiers for improved performance. - Renamed vector_column_device_view to mut_vector_device_view for clarity. - Removed the transform_udf.cuh file and integrated its functionality directly into the transform kernel. - Modified the transform_kernel to handle stencil nullability and user data more efficiently. - Enhanced the launch function to accommodate stencil null checks. - Updated tests to include multi-output and offset string operations. - Cleaned up code and comments for better readability and maintainability. --- cpp/src/jit/column_accessor.cuh | 22 +- cpp/src/jit/column_device_view_wrappers.cuh | 31 +-- cpp/src/jit/transform_udf.cuh | 116 ---------- cpp/src/transform/jit/kernel.cu | 86 +++++-- cpp/src/transform/transform.cu | 63 +++--- .../integration/unary_transform_test.cpp | 209 ++++++++++++------ 6 files changed, 271 insertions(+), 256 deletions(-) delete mode 100644 cpp/src/jit/transform_udf.cuh diff --git a/cpp/src/jit/column_accessor.cuh b/cpp/src/jit/column_accessor.cuh index eeeea10ed3fa..484738f00926 100644 --- a/cpp/src/jit/column_accessor.cuh +++ b/cpp/src/jit/column_accessor.cuh @@ -40,18 +40,18 @@ struct column_accessor { } template - static __device__ auto& column(T const* cols) - requires(sizeof(T) == sizeof(column_type)) + static __device__ auto& column(T const* __restrict__ cols) + requires(sizeof(T) == sizeof(column_type) && alignof(T) == alignof(column_type)) { return reinterpret_cast(cols[index]); } - static __device__ element_type element(auto const* cols, size_type row) + static __device__ element_type element(auto const* __restrict__ cols, size_type row) { return column(cols).template element(map_index(row)); } - static __device__ bool is_null(auto const* cols, size_type row) + static __device__ bool is_null(auto const* __restrict__ cols, size_type row) { if constexpr (!may_be_nullable) { return false; @@ -60,7 +60,7 @@ struct column_accessor { } } - static __device__ bool is_valid(auto const* cols, size_type row) + static __device__ bool is_valid(auto const* __restrict__ cols, size_type row) { if constexpr (!may_be_nullable) { return true; @@ -69,7 +69,8 @@ struct column_accessor { } } - static __device__ optional_element_type nullable_element(auto const* cols, size_type row) + static __device__ optional_element_type nullable_element(auto const* __restrict__ cols, + size_type row) { auto& c = column(cols); @@ -80,7 +81,7 @@ struct column_accessor { } } - static __device__ void set_null_mask_word(auto const* cols, + static __device__ void set_null_mask_word(auto const* __restrict__ cols, size_type word_index, bitmask_type word) requires(!as_scalar) @@ -96,13 +97,13 @@ struct column_accessor { } } - static __device__ void assign(auto const* cols, size_type row, element_type value) + static __device__ void assign(auto const* __restrict__ cols, size_type row, element_type value) requires(!as_scalar) { column(cols).template assign(row, value); } - static __device__ element_type output_arg(auto const* cols, size_type row) + static __device__ element_type output_arg(auto const* __restrict__ cols, size_type row) requires(!as_scalar) { if constexpr (is_strings_output) { @@ -112,7 +113,8 @@ struct column_accessor { } } - static __device__ optional_element_type null_output_arg(auto const* cols, size_type row) + static __device__ optional_element_type null_output_arg(auto const* __restrict__ cols, + size_type row) requires(!as_scalar) { if constexpr (is_strings_output) { diff --git a/cpp/src/jit/column_device_view_wrappers.cuh b/cpp/src/jit/column_device_view_wrappers.cuh index 6fed1f50e11c..96b1ba902477 100644 --- a/cpp/src/jit/column_device_view_wrappers.cuh +++ b/cpp/src/jit/column_device_view_wrappers.cuh @@ -18,15 +18,15 @@ namespace jit { * @brief A column wrapper type that treats a column as a vector of elements. * */ -struct vector_column_device_view : private mutable_column_device_view_core { +struct mut_vector_device_view : private mutable_column_device_view_core { using base = mutable_column_device_view_core; - CUDF_HOST_DEVICE constexpr vector_column_device_view(base const& src) : base{src} {} - ~vector_column_device_view() = default; - vector_column_device_view(vector_column_device_view const&) = default; - vector_column_device_view(vector_column_device_view&&) = default; - vector_column_device_view& operator=(vector_column_device_view const&) = default; - vector_column_device_view& operator=(vector_column_device_view&&) = default; + CUDF_HOST_DEVICE constexpr mut_vector_device_view(base const& src) : base{src} {} + ~mut_vector_device_view() = default; + mut_vector_device_view(mut_vector_device_view const&) = default; + mut_vector_device_view(mut_vector_device_view&&) = default; + mut_vector_device_view& operator=(mut_vector_device_view const&) = default; + mut_vector_device_view& operator=(mut_vector_device_view&&) = default; using base::nullable; using base::offset; @@ -34,7 +34,7 @@ struct vector_column_device_view : private mutable_column_device_view_core { using base::type; template - CUDF_HOST_DEVICE T* data() const noexcept + CUDF_HOST_DEVICE T* __restrict__ data() const noexcept { return static_cast(const_cast(_data)) + _offset; } @@ -91,13 +91,14 @@ struct mut_strings_column_device_view : private mutable_column_device_view_core [[nodiscard]] __device__ cuda::std::span element(size_type element_index) const noexcept requires(cuda::std::is_same_v>) { - auto index = element_index + offset(); - auto chars = static_cast(const_cast(_data)); - auto offsets = child(offsets_column_index); - auto itr = cudf::detail::input_offsetalator(offsets.head(), offsets.type()); - auto offset = itr[index]; - return cuda::std::span{chars + offset, - static_cast(itr[index + 1] - offset)}; + auto index = element_index + offset(); + auto chars = static_cast(const_cast(_data)); + auto offsets = child(offsets_column_index); + auto itr = cudf::detail::input_offsetalator(offsets.head(), offsets.type()); + auto beg = itr[index]; + auto end = itr[index + 1]; + auto* __restrict__ str = chars + beg; + return cuda::std::span{str, static_cast(end - beg)}; } template > diff --git a/cpp/src/jit/transform_udf.cuh b/cpp/src/jit/transform_udf.cuh deleted file mode 100644 index e3788eacc776..000000000000 --- a/cpp/src/jit/transform_udf.cuh +++ /dev/null @@ -1,116 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include -#include -#include - -#include -#include -#include - -#include - -namespace cudf { -namespace jit { - -/** - * @tparam is_null_aware = whether the UDF is null-aware or not. If `YES`, the UDF will receive - * nullable elements for each input column, and the UDF will determine the nullability of the output - * columns. If `NO`, the UDF will receive elements for each input column, and the UDF will not - * handle null masks for the output columns. - * @tparam has_stencil whether the operation has stencil values or not. If true, the kernel will - * receive a bitmask pointer for the stencil and will skip rows where the stencil bit is not set. - * @tparam has_user_data whether the UDF has user data or not. If true, the UDF will receive a - * pointer to the user data as the first argument, followed by the row index. - * @tparam Ins = type_list - * @tparam Outs = type_list - */ -template -struct transform_udf { - template - static __device__ void call(Fn&& udf, - bitmask_type const* stencil, - void* user_data, - column_device_view_core const* incols, - mutable_column_device_view_core const* outcols, - [[maybe_unused]] bool* is_valid, - size_type element_idx) - requires(is_null_aware == null_aware::NO) - { - if constexpr (has_stencil) { - if (stencil != nullptr) { - if (!bit_is_set(stencil, element_idx)) { return; } - } - } - - auto outs = Outs::map( - [&]() { return cuda::std::tuple{A::output_arg(outcols, element_idx)...}; }); - - auto out_ptrs = - cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - - auto inputs = Ins::map( - [&]() { return cuda::std::tuple{A::element(incols, element_idx)...}; }); - - if constexpr (has_user_data) { - auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); - cuda::std::apply(udf, args); - - } else { - auto args = cuda::std::tuple_cat(out_ptrs, inputs); - cuda::std::apply(udf, args); - } - - Outs::map([&]() { - (A::assign(outcols, element_idx, cuda::std::get(outs)), ...); - }); - } - - template - static __device__ void call(Fn&& udf, - [[maybe_unused]] bitmask_type const* stencil, - void* user_data, - column_device_view_core const* incols, - mutable_column_device_view_core const* outcols, - bool* is_valid, - size_type element_idx) - requires(is_null_aware == null_aware::YES) - { - auto outs = Outs::map([&]() { - return cuda::std::tuple{A::null_output_arg(outcols, element_idx)...}; - }); - - auto out_ptrs = - cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - - auto inputs = Ins::map([&]() { - return cuda::std::tuple{A::nullable_element(incols, element_idx)...}; - }); - - if constexpr (has_user_data) { - auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); - cuda::std::apply(udf, args); - - } else { - auto args = cuda::std::tuple_cat(out_ptrs, inputs); - cuda::std::apply(udf, args); - } - - Outs::map([&]() { - (A::assign(outcols, element_idx, *cuda::std::get(outs)), ...); - ((is_valid[A::index] = cuda::std::get(outs).has_value()), ...); - }); - } -}; - -} // namespace jit -} // namespace cudf diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 6b597dd4c3f8..dd73b149ef9a 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -8,15 +8,17 @@ #include #include #include +#include #include #include #include +#include +#include #include #include #include -#include #include #pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before @@ -32,31 +34,79 @@ namespace cudf { namespace jit { -template + +/// @brief The generic transform kernel. Supports all types and nullability combinations. +template CUDF_KERNEL void transform_kernel(size_type row_size, - bitmask_type const* stencil, - void* user_data, - column_device_view_core const* incols, - mutable_column_device_view_core const* outcols) + bitmask_type const* __restrict__ stencil, + bool stencil_has_nulls, + void* __restrict__ user_data, + column_device_view_core const* __restrict__ incols, + mutable_column_device_view_core const* __restrict__ outcols) { // ensure block size is a multiple of warp size for correct warp-synchronous behavior - assert((blockDim.x & 31) == 0); auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); - for (auto i = start; i < row_size; i += stride) { - bool is_valid[Outs::size] = {}; + for (auto element_idx = start; element_idx < row_size; element_idx += stride) { + if constexpr (is_null_aware == null_aware::NO) { + if (stencil_has_nulls && !bit_is_set(stencil, element_idx)) { continue; } + + auto outs = Outs::map( + [&]() { return cuda::std::tuple{A::output_arg(outcols, element_idx)...}; }); + + auto out_ptrs = + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); + + auto inputs = Ins::map( + [&]() { return cuda::std::tuple{A::element(incols, element_idx)...}; }); + + if constexpr (has_user_data) { + auto args = + cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); + cuda::std::apply(udf, args); + + } else { + // TODO: static assert invocable + auto args = cuda::std::tuple_cat(out_ptrs, inputs); + cuda::std::apply(udf, args); + } + + Outs::map([&]() { + (A::assign(outcols, element_idx, cuda::std::get(outs)), ...); + }); + } else { + bool is_valid[Outs::size]; + + auto outs = Outs::map([&]() { + return cuda::std::tuple{A::null_output_arg(outcols, element_idx)...}; + }); + + auto out_ptrs = + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); + + auto inputs = Ins::map([&]() { + return cuda::std::tuple{A::nullable_element(incols, element_idx)...}; + }); + + if constexpr (has_user_data) { + auto args = + cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); + cuda::std::apply(udf, args); + + } else { + auto args = cuda::std::tuple_cat(out_ptrs, inputs); + cuda::std::apply(udf, args); + } - transform_udf::call( - GENERIC_TRANSFORM_OP, stencil, user_data, incols, outcols, is_valid, i); + Outs::map([&]() { + (A::assign(outcols, element_idx, *cuda::std::get(outs)), ...); + ((is_valid[A::index] = cuda::std::get(outs).has_value()), ...); + }); - if constexpr (is_null_aware == null_aware::YES) { - Outs::map( - [&]() { (warp_compact_validity(outcols, i, is_valid[A::index]), ...); }); + Outs::map([&]() { + (warp_compact_validity(outcols, element_idx, is_valid[A::index]), ...); + }); } } } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 0a53fd38b32b..2f6e57cf6add 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -155,7 +155,6 @@ using handle = std::variant< namespace jit_transform { jitify2::Kernel instantiate(null_aware is_null_aware, - bool has_stencil, bool has_user_data, std::string const& ins, std::string const& outs, @@ -173,21 +172,25 @@ jitify2::Kernel instantiate(null_aware is_null_aware, : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); auto kernel = jitify2::reflection::Template("cudf::jit::transform_kernel") - .instantiate(is_null_aware, has_stencil, has_user_data, ins, outs); + .instantiate(is_null_aware, has_user_data, ins, outs); - return jit::get_udf_kernel(*transform_jit_kernel_cu_jit, kernel, cuda_source, {"--restrict"}); + return jit::get_udf_kernel(*transform_jit_kernel_cu_jit, + kernel, + cuda_source, + {"-restrict", "--device-debug", "--dopt=on", "--generate-line-info"}); } void launch(jitify2::Kernel const& kernel, size_type row_size, bitmask_type const* stencil, + bool stencil_has_nulls, void* user_data, column_device_view_core const* incols, mutable_column_device_view_core const* outcols, rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - void* args[] = {&row_size, &stencil, &user_data, &incols, &outcols}; + void* args[] = {&row_size, &stencil, &stencil_has_nulls, &user_data, &incols, &outcols}; kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); } @@ -218,7 +221,7 @@ std::string reflect_output_column(fixed_width_column const&) std::string reflect_output_column(string_views_column const&) { - return "cudf::jit::vector_column_device_view"; + return "cudf::jit::mut_vector_device_view"; } std::string reflect_output_column(mut_strings_column const&) @@ -329,10 +332,10 @@ auto to_args(std::span inputs, } void run(null_aware is_null_aware, - bool has_stencil, bool has_user_data, size_type row_size, bitmask_type const* d_stencil, + bool stencil_has_nulls, void* user_data, std::span inputs, std::span outputs, @@ -346,7 +349,6 @@ void run(null_aware is_null_aware, auto [in_types, out_types, ptx_in_types, ptx_out_types] = reflect(source_type, inputs, outputs, input_may_be_nullable, output_may_be_nullable); auto kernel = instantiate(is_null_aware, - has_stencil, has_user_data, in_types, out_types, @@ -357,7 +359,7 @@ void run(null_aware is_null_aware, auto [cols, handles] = to_args(inputs, outputs, stream, mr); auto* incols = reinterpret_cast(cols.data()); auto* outcols = reinterpret_cast(incols + inputs.size()); - return launch(kernel, row_size, d_stencil, user_data, incols, outcols, stream); + return launch(kernel, row_size, d_stencil, stencil_has_nulls, user_data, incols, outcols, stream); } } // namespace jit_transform @@ -402,7 +404,7 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, auto has_cols = std::any_of( inputs.begin(), inputs.end(), [](auto& in) { return std::holds_alternative(in); }); - if (has_cols) { + if (!has_cols) { // no non-scalar columns, so all rows are valid CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0xFF, num_bytes, stream.value())); return 0; @@ -497,9 +499,7 @@ void perform_checks(udf_source_type source_type, [](auto& in) { return std::visit( [](auto& c) { - auto type = c.type(); - return !is_integer(type) && !is_floating_point(type) && - type.id() != type_id::BOOL8; + return !is_integral(c.type()) && !is_floating_point(c.type()); }, in); }), @@ -508,13 +508,7 @@ void perform_checks(udf_source_type source_type, CUDF_EXPECTS(std::none_of(outputs.begin(), outputs.end(), [](auto& out) { - return std::visit( - [](auto& c) { - auto type = c.type(); - return !is_integer(type) && !is_floating_point(type) && - type.id() != type_id::BOOL8; - }, - out); + return !is_integral(out.type) && !is_floating_point(out.type); }), "Transforms with PTX UDFs only support integer, floating-point, and boolean types", std::invalid_argument); @@ -576,11 +570,12 @@ void perform_checks(udf_source_type source_type, std::invalid_argument); } -std::optional make_stencil(null_aware is_null_aware, - size_type row_size, - std::span inputs, - std::span outputs, - rmm::cuda_stream_view stream) +std::optional> make_stencil( + null_aware is_null_aware, + size_type row_size, + std::span inputs, + std::span outputs, + rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); @@ -594,7 +589,7 @@ std::optional make_stencil(null_aware is_null_aware, } // no nullable outputs - if (!stencil.has_value()) { return nullptr; } + if (!stencil.has_value()) { return std::pair{nullptr, 0}; } auto stencil_null_count = inplace_null_mask_and(*stencil, row_size, inputs, stream); @@ -609,15 +604,16 @@ std::optional make_stencil(null_aware is_null_aware, stream); } - auto null_count = mask == nullptr ? 0 : stencil_null_count; + auto null_count = (mask == nullptr) ? 0 : stencil_null_count; std::visit([&](auto& c) { c.set_null_count(null_count); }, out); } - return stencil; + return std::pair{*stencil, stencil_null_count}; } -auto make_outputs(size_type row_size, +auto make_outputs(null_aware is_null_aware, + size_type row_size, std::span inputs, std::span outputs, std::span is_output_nullable, @@ -656,9 +652,9 @@ auto make_outputs(size_type row_size, } } - auto stencil = make_stencil(is_null_aware, row_size, inputs, output_columns, stream); + auto stencil = make_stencil(is_null_aware, row_size, inputs, cols, stream); - return std::make_tuple(std::move(output_columns), stencil); + return std::make_tuple(std::move(cols), stencil); } void update_null_counts(std::span outputs, @@ -755,11 +751,14 @@ std::unique_ptr
execute_transform(std::string const& udf, std::move(string_offsets), stream, mr); + + auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; + auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; jit_transform::run(is_null_aware, - stencil.has_value(), user_data.has_value(), row_size, - stencil.value_or(nullptr), + stencil_arg, + stencil_has_nulls, user_data.value_or(nullptr), inputs, output_columns, diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index 3b9c3e78d18f..5c4484136d09 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -208,7 +209,7 @@ TEST_F(UnaryOperationIntegrationTest, Transform_INT32_INT32) // // ```py // - // from numba import cuda, float32 + // from numba import cuda, int32 // from numba.cuda import compile_ptx_for_current_device // // # Define a CUDA device function @@ -218,7 +219,7 @@ TEST_F(UnaryOperationIntegrationTest, Transform_INT32_INT32) // return a * a - a // // # Define argument types for the function - // arg_types = (float32, ) + // arg_types = (int32, ) // // # Compile the device function as relocatable // ptx, _ = cuda.compile_ptx_for_current_device(op, arg_types, device=True) @@ -245,26 +246,26 @@ TEST_F(UnaryOperationIntegrationTest, Transform_INT32_INT32) .target sm_86 .address_size 64 - // .globl _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd -.common .global .align 8 .u64 _ZN08NumbaEnv8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd; + // .globl _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi +.common .global .align 8 .u64 _ZN08NumbaEnv8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi; -.visible .func (.param .b32 func_retval0) _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd( - .param .b64 _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_0, - .param .b64 _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_1 +.visible .func (.param .b32 func_retval0) _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi( + .param .b64 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0, + .param .b32 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1 ) { - .reg .b32 %r<2>; - .reg .f64 %fd<4>; - .reg .b64 %rd<2>; - - - ld.param.u64 %rd1, [_ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_0]; - ld.param.f64 %fd1, [_ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_1]; - mul.f64 %fd2, %fd1, %fd1; - sub.f64 %fd3, %fd2, %fd1; - st.f64 [%rd1], %fd3; - mov.u32 %r1, 0; - st.param.b32 [func_retval0+0], %r1; + .reg .b32 %r<3>; + .reg .b64 %rd<5>; + + + ld.param.u64 %rd1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0]; + ld.param.u32 %r1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1]; + cvt.s64.s32 %rd2, %r1; + mul.wide.s32 %rd3, %r1, %r1; + sub.s64 %rd4, %rd3, %rd2; + st.u64 [%rd1], %rd4; + mov.u32 %r2, 0; + st.param.b32 [func_retval0+0], %r2; ret; } @@ -301,7 +302,7 @@ __device__ inline void f( // // ```py // - // from numba import cuda, float32 + // from numba import cuda, int8 // from numba.cuda import compile_ptx_for_current_device // // # Define a CUDA device function @@ -311,7 +312,7 @@ __device__ inline void f( // return (a - 32) if (a > 96 and a < 123) else a // // # Define argument types for the function - // arg_types = (float32, float32, float32) + // arg_types = (int8, ) // // # Compile the device function as relocatable // ptx, _ = cuda.compile_ptx_for_current_device(op, arg_types, device=True) @@ -338,35 +339,37 @@ __device__ inline void f( .target sm_86 .address_size 64 - // .globl _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd -.common .global .align 8 .u64 _ZN08NumbaEnv8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd; + // .globl _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEa +.common .global .align 8 .u64 _ZN08NumbaEnv8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEa; -.visible .func (.param .b32 func_retval0) _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd( - .param .b64 _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_0, - .param .b64 _ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_1 +.visible .func (.param .b32 func_retval0) _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEa( + .param .b64 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEa_param_0, + .param .b32 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEa_param_1 ) { - .reg .pred %p<4>; + .reg .pred %p<2>; + .reg .b16 %rs<4>; .reg .b32 %r<2>; - .reg .f64 %fd<3>; - .reg .b64 %rd<2>; + .reg .b64 %rd<5>; - ld.param.u64 %rd1, [_ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_0]; - ld.param.f64 %fd1, [_ZN8__main__3opxB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEd_param_1]; - setp.gt.f64 %p1, %fd1, 0d4058000000000000; - setp.lt.f64 %p2, %fd1, 0d405EC00000000000; - and.pred %p3, %p1, %p2; - @%p3 bra $L__BB0_2; + ld.param.u64 %rd2, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEa_param_0]; + ld.param.u8 %rs1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEa_param_1]; + add.s16 %rs2, %rs1, -97; + and.b16 %rs3, %rs2, 255; + setp.lt.u16 %p1, %rs3, 26; + cvt.u64.u16 %rd3, %rs1; + cvt.s64.s8 %rd1, %rd3; + @%p1 bra $L__BB0_2; bra.uni $L__BB0_1; $L__BB0_2: - add.f64 %fd2, %fd1, 0dC040000000000000; - st.f64 [%rd1], %fd2; + add.s64 %rd4, %rd1, -32; + st.u64 [%rd2], %rd4; bra.uni $L__BB0_3; $L__BB0_1: - st.f64 [%rd1], %fd1; + st.u64 [%rd2], %rd1; $L__BB0_3: mov.u32 %r1, 0; @@ -687,6 +690,42 @@ TEST_F(StringOperationTest, Output) CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_empty, result_empty->view()); } +TEST_F(StringOperationTest, OutputOffseted) +{ + auto a = cudf::test::strings_column_wrapper{"x", "xx", "xxx", "xxxx", "xxxxx", "xxxxxx"}; + auto b = cudf::test::strings_column_wrapper{"aa", "bb", "cc", "dd", "ee", "ff"}; + auto c = cudf::test::strings_column_wrapper{"1", "2", "3", "4", "5", "6"}; + + std::string cuda = R"***( + __device__ void concat(cuda::std::span * out, cudf::string_view a, cudf::string_view b, cudf::string_view c){ + auto iter = out->data(); + memcpy(iter, a.data(), a.size_bytes()); + iter += a.size_bytes(); + memcpy(iter, b.data(), b.size_bytes()); + iter += b.size_bytes(); + memcpy(iter, c.data(), c.size_bytes()); + } + )***"; + + auto offsets = cudf::test::fixed_width_column_wrapper{0, 4, 9, 15, 22, 30, 39}.release(); + std::vector> strings_offsets; + strings_offsets.push_back(std::move(offsets)); + auto expected = cudf::test::strings_column_wrapper{ + "xaa1", "xxbb2", "xxxcc3", "xxxxdd4", "xxxxxee5", "xxxxxxff6"}; + cudf::transform_input inputs[] = {a, b, c}; + cudf::transform_output outputs[] = {cudf::data_type(cudf::type_id::STRING)}; + auto result = cudf::multi_transform(cuda, + cudf::udf_source_type::CUDA, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + std::move(strings_offsets), + std::nullopt); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->get_column(0)); +} + TEST_F(StringOperationTest, StringConcat) { auto first_name = cudf::test::strings_column_wrapper{ @@ -766,6 +805,69 @@ TEST_F(StringOperationTest, EmptyInput) EXPECT_EQ(0, result->size()); } +TEST_F(StringOperationTest, MultiOutput) +{ + auto a = cudf::test::strings_column_wrapper{ + "415 555 1234", "212 867 5309", "214 947 5689", "712 312 1863", "777 214 9063", "964 123 2934"}; + auto expected_area_code = + cudf::test::strings_column_wrapper{"415", "212", "214", "712", "777", "964"}; + auto expected_prefix = + cudf::test::strings_column_wrapper{"555", "867", "947", "312", "214", "123"}; + auto expected_line_number = + cudf::test::strings_column_wrapper{"1234", "5309", "5689", "1863", "9063", "2934"}; + + auto expected = cudf::table_view({expected_area_code, expected_prefix, expected_line_number}); + + std::string cuda = R"***( +__device__ void capture_group(cudf::string_view* area_code, + cudf::string_view* prefix, + cudf::string_view* line_number, + cudf::string_view input) { + auto seek = [] (char const*& iter, char const* end, char c) { + while (iter != end && *iter != c) { + iter++; + } + }; + auto* iter = input.data(); + auto* end = iter + input.size_bytes(); + auto* area_code_begin = iter; + seek(iter, end, ' '); + auto* area_code_end = iter; + iter++; + auto* prefix_begin = iter; + seek(iter, end, ' '); + auto* prefix_end = iter; + iter++; + auto* line_number_begin = iter; + seek(iter, end, ' '); + auto* line_number_end = iter; + *area_code = cudf::string_view{ + area_code_begin, + static_cast(area_code_end - area_code_begin)}; + *prefix = cudf::string_view{ + prefix_begin, static_cast(prefix_end - prefix_begin)}; + *line_number = cudf::string_view{ + line_number_begin, + static_cast(line_number_end - line_number_begin)}; +} + )***"; + + cudf::transform_input inputs[] = {a}; + cudf::transform_output outputs[] = {{.type = cudf::data_type(cudf::type_id::STRING)}, + {.type = cudf::data_type(cudf::type_id::STRING)}, + {.type = cudf::data_type(cudf::type_id::STRING)}}; + auto result = cudf::multi_transform(cuda, + cudf::udf_source_type::CUDA, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result->view()); +} + struct NullTest : public cudf::test::BaseFixture { protected: char const* const cuda = @@ -1033,34 +1135,11 @@ return l - t * l + t * h; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*cuda_result, *expected); } -// TODO: add offseted string output support tests -// TODO: add multi-output transform tests // TODO: add zero-input transform test -// TODO: merge main /** -* -* TODO: Regex extract with multiple capture groups, then the JIT implementation would just need to -handle multiple column outputs without handling AST common subexpression elimination. - -Example: -Example: Phone Number -Format: (123) 456-7890 - -Regex pattern: -\((\d{3})\)\s(\d{3})-(\d{4}) - -Example matches: -Input: (415) 555-1234 -Column 1: 415 -Column 2: 555 -Column 3: 1234 - -Input: (212) 867-5309 -Column 1: 212 -Column 2: 867 -Column 3: 5309 -* -* -*/ + * + * + * + */ } // namespace transformation From cfc1e2389c8214a215de3e0b45de2ce4de6e8e81 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 13 Mar 2026 05:06:25 +0000 Subject: [PATCH 076/254] Remove unnecessary blank line in transform kernel definition --- cpp/src/transform/jit/kernel.cu | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index dd73b149ef9a..2c40723f5f2c 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -34,7 +34,6 @@ namespace cudf { namespace jit { - /// @brief The generic transform kernel. Supports all types and nullability combinations. template CUDF_KERNEL void transform_kernel(size_type row_size, From b279d493ee1c7fc8f5893ef4297ed9432d94bcf3 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 13 Mar 2026 05:08:50 +0000 Subject: [PATCH 077/254] remove TODO --- cpp/tests/transform/integration/unary_transform_test.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index 5c4484136d09..9b9dcd91351e 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -1135,11 +1135,4 @@ return l - t * l + t * h; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*cuda_result, *expected); } -// TODO: add zero-input transform test -/** - * - * - * - */ - } // namespace transformation From 40f4129bc23f864d3a5e1680e25644f6aa48e5de Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 14 Mar 2026 06:30:38 +0000 Subject: [PATCH 078/254] code review refactoring --- .../cudf/column/column_device_view.cuh | 1 - .../cudf/column/column_device_view_base.cuh | 4 +- cpp/include/cudf/transform.hpp | 4 +- cpp/src/jit/column_accessor.cuh | 1 - cpp/src/jit/column_device_view_wrappers.cuh | 32 ++++---- cpp/src/transform/jit/kernel.cu | 31 ++++---- cpp/src/transform/transform.cu | 77 +++++++++---------- .../integration/unary_transform_test.cpp | 2 +- 8 files changed, 74 insertions(+), 78 deletions(-) diff --git a/cpp/include/cudf/column/column_device_view.cuh b/cpp/include/cudf/column/column_device_view.cuh index a580c2541c91..a7d806dc3642 100644 --- a/cpp/include/cudf/column/column_device_view.cuh +++ b/cpp/include/cudf/column/column_device_view.cuh @@ -762,7 +762,6 @@ class alignas(16) mutable_column_device_view : public mutable_column_device_view * @param num_children The number of child columns * @return A `mutable_column_device_view` instance representing the specified column data * in device memory - * */ static auto from_parts(data_type type, size_type size, diff --git a/cpp/include/cudf/column/column_device_view_base.cuh b/cpp/include/cudf/column/column_device_view_base.cuh index 6e59c48ae469..1d024d410087 100644 --- a/cpp/include/cudf/column/column_device_view_base.cuh +++ b/cpp/include/cudf/column/column_device_view_base.cuh @@ -633,7 +633,7 @@ class alignas(16) mutable_column_device_view_core : public detail::column_device * @brief Returns `string_view` to the string element at the specified index. * * If the element at the specified index is NULL, i.e., `is_null(element_index) - * == true`, then any attempt to use the result will lead to undefined behavior. + * == true`, then using the result is undefined behavior. * * This function accounts for the offset. * @@ -656,7 +656,7 @@ class alignas(16) mutable_column_device_view_core : public detail::column_device * column. * * If the element at the specified index is NULL, i.e., `is_null(element_index) == true`, - * then any attempt to use the result will lead to undefined behavior. + * then using the result is undefined behavior. * * @param element_index Position of the desired element * @return numeric::fixed_point representing the element at this index diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index d5159835a504..4c0a94fc9b04 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -144,6 +144,8 @@ std::unique_ptr transform_extended( * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in * this case. + * @throws std::invalid_argument if string offsets are provided for non-string output columns, or + * if the number of string offsets does not match the number of output columns. * * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from * the input and pre-allocated output columns. @@ -171,7 +173,7 @@ std::unique_ptr
multi_transform( std::optional user_data, std::span inputs, std::span outputs, - std::vector> string_offsets, + std::vector>&& string_offsets, std::optional row_size, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); diff --git a/cpp/src/jit/column_accessor.cuh b/cpp/src/jit/column_accessor.cuh index 484738f00926..0ea502a00c60 100644 --- a/cpp/src/jit/column_accessor.cuh +++ b/cpp/src/jit/column_accessor.cuh @@ -1,4 +1,3 @@ - /* * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 diff --git a/cpp/src/jit/column_device_view_wrappers.cuh b/cpp/src/jit/column_device_view_wrappers.cuh index 96b1ba902477..fb6343b2b4c7 100644 --- a/cpp/src/jit/column_device_view_wrappers.cuh +++ b/cpp/src/jit/column_device_view_wrappers.cuh @@ -4,6 +4,7 @@ */ #pragma once + #include #include #include @@ -18,15 +19,15 @@ namespace jit { * @brief A column wrapper type that treats a column as a vector of elements. * */ -struct mut_vector_device_view : private mutable_column_device_view_core { +struct mutable_vector_device_view : private mutable_column_device_view_core { using base = mutable_column_device_view_core; - CUDF_HOST_DEVICE constexpr mut_vector_device_view(base const& src) : base{src} {} - ~mut_vector_device_view() = default; - mut_vector_device_view(mut_vector_device_view const&) = default; - mut_vector_device_view(mut_vector_device_view&&) = default; - mut_vector_device_view& operator=(mut_vector_device_view const&) = default; - mut_vector_device_view& operator=(mut_vector_device_view&&) = default; + CUDF_HOST_DEVICE constexpr mutable_vector_device_view(base const& src) : base{src} {} + ~mutable_vector_device_view() = default; + mutable_vector_device_view(mutable_vector_device_view const&) = default; + mutable_vector_device_view(mutable_vector_device_view&&) = default; + mutable_vector_device_view& operator=(mutable_vector_device_view const&) = default; + mutable_vector_device_view& operator=(mutable_vector_device_view&&) = default; using base::nullable; using base::offset; @@ -68,16 +69,17 @@ struct mut_vector_device_view : private mutable_column_device_view_core { * @brief A column wrapper type that treats a column as a column of mutable strings. * The offsets will have been pre-initialized and the chars will have been pre-allocated. */ -struct mut_strings_column_device_view : private mutable_column_device_view_core { +struct mutable_strings_column_device_view : private mutable_column_device_view_core { using base = mutable_column_device_view_core; - CUDF_HOST_DEVICE constexpr mut_strings_column_device_view(base const& src) : base{src} {} + CUDF_HOST_DEVICE constexpr mutable_strings_column_device_view(base const& src) : base{src} {} - ~mut_strings_column_device_view() = default; - mut_strings_column_device_view(mut_strings_column_device_view const&) = default; - mut_strings_column_device_view(mut_strings_column_device_view&&) = default; - mut_strings_column_device_view& operator=(mut_strings_column_device_view const&) = default; - mut_strings_column_device_view& operator=(mut_strings_column_device_view&&) = default; + ~mutable_strings_column_device_view() = default; + mutable_strings_column_device_view(mutable_strings_column_device_view const&) = default; + mutable_strings_column_device_view(mutable_strings_column_device_view&&) = default; + mutable_strings_column_device_view& operator=(mutable_strings_column_device_view const&) = + default; + mutable_strings_column_device_view& operator=(mutable_strings_column_device_view&&) = default; using base::is_null; using base::is_valid; @@ -114,7 +116,7 @@ struct mut_strings_column_device_view : private mutable_column_device_view_core __device__ void assign(size_type row, cuda::std::span value) const noexcept requires(cuda::std::is_same_v>) { - // no-op for since we assume the chars have already been pre-allocated and they are mutated + // no-op since we assume the chars have already been pre-allocated and they are mutated // in-place return; } diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 2c40723f5f2c..f5248a44e816 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -40,10 +40,10 @@ CUDF_KERNEL void transform_kernel(size_type row_size, bitmask_type const* __restrict__ stencil, bool stencil_has_nulls, void* __restrict__ user_data, - column_device_view_core const* __restrict__ incols, - mutable_column_device_view_core const* __restrict__ outcols) + column_device_view_core const* __restrict__ input_cols, + mutable_column_device_view_core const* __restrict__ output_cols) { - // ensure block size is a multiple of warp size for correct warp-synchronous behavior + // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); @@ -51,60 +51,61 @@ CUDF_KERNEL void transform_kernel(size_type row_size, if constexpr (is_null_aware == null_aware::NO) { if (stencil_has_nulls && !bit_is_set(stencil, element_idx)) { continue; } - auto outs = Outs::map( - [&]() { return cuda::std::tuple{A::output_arg(outcols, element_idx)...}; }); + auto outs = Outs::map([&]() { + return cuda::std::tuple{A::output_arg(output_cols, element_idx)...}; + }); auto out_ptrs = cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); auto inputs = Ins::map( - [&]() { return cuda::std::tuple{A::element(incols, element_idx)...}; }); + [&]() { return cuda::std::tuple{A::element(input_cols, element_idx)...}; }); if constexpr (has_user_data) { auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); - cuda::std::apply(udf, args); + cuda::std::apply(GENERIC_TRANSFORM_OP, args); } else { // TODO: static assert invocable auto args = cuda::std::tuple_cat(out_ptrs, inputs); - cuda::std::apply(udf, args); + cuda::std::apply(GENERIC_TRANSFORM_OP, args); } Outs::map([&]() { - (A::assign(outcols, element_idx, cuda::std::get(outs)), ...); + (A::assign(output_cols, element_idx, cuda::std::get(outs)), ...); }); } else { bool is_valid[Outs::size]; auto outs = Outs::map([&]() { - return cuda::std::tuple{A::null_output_arg(outcols, element_idx)...}; + return cuda::std::tuple{A::null_output_arg(output_cols, element_idx)...}; }); auto out_ptrs = cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); auto inputs = Ins::map([&]() { - return cuda::std::tuple{A::nullable_element(incols, element_idx)...}; + return cuda::std::tuple{A::nullable_element(input_cols, element_idx)...}; }); if constexpr (has_user_data) { auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); - cuda::std::apply(udf, args); + cuda::std::apply(GENERIC_TRANSFORM_OP, args); } else { auto args = cuda::std::tuple_cat(out_ptrs, inputs); - cuda::std::apply(udf, args); + cuda::std::apply(GENERIC_TRANSFORM_OP, args); } Outs::map([&]() { - (A::assign(outcols, element_idx, *cuda::std::get(outs)), ...); + (A::assign(output_cols, element_idx, *cuda::std::get(outs)), ...); ((is_valid[A::index] = cuda::std::get(outs).has_value()), ...); }); Outs::map([&]() { - (warp_compact_validity(outcols, element_idx, is_valid[A::index]), ...); + (warp_compact_validity(output_cols, element_idx, is_valid[A::index]), ...); }); } } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 2f6e57cf6add..17f53cc2b575 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -174,10 +174,8 @@ jitify2::Kernel instantiate(null_aware is_null_aware, auto kernel = jitify2::reflection::Template("cudf::jit::transform_kernel") .instantiate(is_null_aware, has_user_data, ins, outs); - return jit::get_udf_kernel(*transform_jit_kernel_cu_jit, - kernel, - cuda_source, - {"-restrict", "--device-debug", "--dopt=on", "--generate-line-info"}); + return jit::get_udf_kernel( + *transform_jit_kernel_cu_jit, kernel, cuda_source, {"-restrict", "--dopt=on"}); } void launch(jitify2::Kernel const& kernel, @@ -185,12 +183,12 @@ void launch(jitify2::Kernel const& kernel, bitmask_type const* stencil, bool stencil_has_nulls, void* user_data, - column_device_view_core const* incols, - mutable_column_device_view_core const* outcols, + column_device_view_core const* input_cols, + mutable_column_device_view_core const* output_cols, rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - void* args[] = {&row_size, &stencil, &stencil_has_nulls, &user_data, &incols, &outcols}; + void* args[] = {&row_size, &stencil, &stencil_has_nulls, &user_data, &input_cols, &output_cols}; kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); } @@ -221,12 +219,12 @@ std::string reflect_output_column(fixed_width_column const&) std::string reflect_output_column(string_views_column const&) { - return "cudf::jit::mut_vector_device_view"; + return "cudf::jit::mutable_vector_device_view"; } std::string reflect_output_column(mut_strings_column const&) { - return "cudf::jit::mut_strings_column_device_view"; + return "cudf::jit::mutable_strings_column_device_view"; } auto reflect(udf_source_type source_type, @@ -299,34 +297,35 @@ auto to_args(std::span inputs, for (auto& in : inputs) { if (auto* col = std::get_if(&in)) { - auto hnd = column_device_view::create(*col, stream); - h_args.push_back(*hnd); - handles.push_back(std::move(hnd)); + auto handle = column_device_view::create(*col, stream); + h_args.push_back(*handle); + handles.emplace_back(std::move(handle)); } else { auto& scalar = std::get(in); - auto hnd = column_device_view::create(scalar.as_column_view(), stream); - h_args.push_back(*hnd); - handles.push_back(std::move(hnd)); + auto handle = column_device_view::create(scalar.as_column_view(), stream); + h_args.push_back(*handle); + handles.emplace_back(std::move(handle)); } } for (auto& out : outputs) { std::visit( [&](auto& col) { - auto hnd = col.mutable_view().to_device(stream); - h_args.push_back(*hnd); - handles.push_back(std::move(hnd)); + auto handle = col.mutable_view().to_device(stream); + h_args.push_back(*handle); + handles.push_back(std::move(handle)); }, out); } rmm::device_buffer d_args{h_args.size() * sizeof(detail::column_device_view_base), stream, mr}; - detail::cuda_memcpy_async_impl(d_args.data(), - h_args.data(), - h_args.size() * sizeof(detail::column_device_view_base), - detail::host_memory_kind::PAGEABLE, - stream); + CUDF_CUDA_TRY(detail::memcpy_async( + d_args.data(), h_args.data(), h_args.size() * sizeof(detail::column_device_view_base), stream)); + + // ensure the device buffer copy is complete before `h_args` goes out of scope and its destructors + // are called + CUDF_CUDA_TRY(cudaStreamSynchronize(stream.value())); return std::make_tuple(std::move(d_args), std::move(handles)); } @@ -357,9 +356,11 @@ void run(null_aware is_null_aware, udf, source_type); auto [cols, handles] = to_args(inputs, outputs, stream, mr); - auto* incols = reinterpret_cast(cols.data()); - auto* outcols = reinterpret_cast(incols + inputs.size()); - return launch(kernel, row_size, d_stencil, stencil_has_nulls, user_data, incols, outcols, stream); + auto* input_cols = reinterpret_cast(cols.data()); + auto* output_cols = + reinterpret_cast(input_cols + inputs.size()); + return launch( + kernel, row_size, d_stencil, stencil_has_nulls, user_data, input_cols, output_cols, stream); } } // namespace jit_transform @@ -433,8 +434,7 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, if (nullable_masks.size() == 1) { // only 1 mask provided, copy it directly to the output - detail::cuda_memcpy_async_impl( - null_mask, nullable_masks[0], num_bytes, detail::host_memory_kind::PINNED, stream); + CUDF_CUDA_TRY(detail::memcpy_async(null_mask, nullable_masks[0], num_bytes, stream)); return nullable_null_counts[0]; } @@ -458,11 +458,7 @@ auto get_null_transformation(null_aware is_null_aware, std::span inputs, std::span outputs) { - std::vector input_may_be_nullable; - - for (auto& in : inputs) { - input_may_be_nullable.push_back(true); - } + std::vector input_may_be_nullable(inputs.size(), true); auto any_input_nullable = std::any_of(inputs.begin(), inputs.end(), [](auto& in) { return std::visit([](auto& c) { return c.nullable(); }, in); @@ -597,11 +593,8 @@ std::optional> make_stencil( auto* mask = std::visit([&](auto& c) { return c.null_mask(); }, out); if (mask != nullptr && mask != *stencil) { - detail::cuda_memcpy_async_impl(mask, - *stencil, - bitmask_allocation_size_bytes(row_size), - detail::host_memory_kind::PINNED, - stream); + CUDF_CUDA_TRY( + detail::memcpy_async(mask, *stencil, bitmask_allocation_size_bytes(row_size), stream)); } auto null_count = (mask == nullptr) ? 0 : stencil_null_count; @@ -634,18 +627,18 @@ auto make_outputs(null_aware is_null_aware, if (is_fixed_width(output.type)) { auto col = fixed_width_column::make(output.type, row_size, std::move(null_mask), 0, stream, mr); - cols.push_back(std::move(col)); + cols.emplace_back(std::move(col)); } else if (output.type.id() == type_id::STRING) { if (string_offsets.empty() || string_offsets[i] == nullptr) { auto col = string_views_column::make(row_size, std::move(null_mask), 0, stream, mr); - cols.push_back(std::move(col)); + cols.emplace_back(std::move(col)); } else { auto chars_size = strings::detail::get_offset_value(string_offsets[i]->view(), row_size, stream); auto chars = rmm::device_buffer{static_cast(chars_size), stream, mr}; auto col = mut_strings_column::make( row_size, std::move(chars), std::move(string_offsets[i]), std::move(null_mask), 0); - cols.push_back(std::move(col)); + cols.emplace_back(std::move(col)); } } else { CUDF_UNREACHABLE("Unsupported output type for transform"); @@ -781,7 +774,7 @@ std::unique_ptr
multi_transform(std::string const& udf, std::optional user_data, std::span inputs, std::span outputs, - std::vector> string_offsets, + std::vector>&& string_offsets, std::optional row_size, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index 9b9dcd91351e..73853d5047e4 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -690,7 +690,7 @@ TEST_F(StringOperationTest, Output) CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_empty, result_empty->view()); } -TEST_F(StringOperationTest, OutputOffseted) +TEST_F(StringOperationTest, OutputOffsetted) { auto a = cudf::test::strings_column_wrapper{"x", "xx", "xxx", "xxxx", "xxxxx", "xxxxxx"}; auto b = cudf::test::strings_column_wrapper{"aa", "bb", "cc", "dd", "ee", "ff"}; From ddd251dcd4cf2a21addaf405e69861d4a7bf87ee Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 17 Mar 2026 00:44:20 +0000 Subject: [PATCH 079/254] more improvements --- cpp/CMakeLists.txt | 8 - cpp/cmake/thirdparty/get_lz4.cmake | 2 +- cpp/include/cudf/context.hpp | 3 +- cpp/src/binaryop/binaryop.cpp | 25 +- cpp/src/jit/helpers.cpp | 30 +- cpp/src/jit/helpers.hpp | 2 - cpp/src/jit/jit.cpp | 99 +-- cpp/src/jit/jit.hpp | 25 +- cpp/src/join/filter_join_indices_jit.cu | 57 +- cpp/src/librtcx/CMake/embed.cmake | 8 +- cpp/src/librtcx/CMake/embed.in.py | 72 +- cpp/src/librtcx/rtcx.cpp | 539 +++++++++------ cpp/src/librtcx/rtcx.hpp | 628 ++++++++---------- cpp/src/librtcx/system_includes/README.md | 0 cpp/src/librtcx/system_includes/algorithm | 12 - cpp/src/librtcx/system_includes/array | 12 - cpp/src/librtcx/system_includes/atomic | 12 - cpp/src/librtcx/system_includes/barrier | 12 - cpp/src/librtcx/system_includes/bit | 12 - cpp/src/librtcx/system_includes/cassert | 10 - cpp/src/librtcx/system_includes/cassert.h | 10 - cpp/src/librtcx/system_includes/ccomplex | 12 - cpp/src/librtcx/system_includes/ccomplex.h | 12 - cpp/src/librtcx/system_includes/cfloat | 10 - cpp/src/librtcx/system_includes/cfloat.h | 10 - cpp/src/librtcx/system_includes/chrono | 12 - cpp/src/librtcx/system_includes/climits | 10 - cpp/src/librtcx/system_includes/cmath | 12 - cpp/src/librtcx/system_includes/cmath.h | 12 - cpp/src/librtcx/system_includes/complex | 12 - cpp/src/librtcx/system_includes/concepts | 12 - cpp/src/librtcx/system_includes/cstddef | 12 - cpp/src/librtcx/system_includes/cstdint | 12 - cpp/src/librtcx/system_includes/cstdlib | 12 - cpp/src/librtcx/system_includes/cstring | 12 - cpp/src/librtcx/system_includes/ctime | 12 - cpp/src/librtcx/system_includes/expected | 12 - cpp/src/librtcx/system_includes/functional | 12 - .../librtcx/system_includes/initializer_list | 12 - .../librtcx/system_includes/inplace_vector | 12 - cpp/src/librtcx/system_includes/iterator | 12 - cpp/src/librtcx/system_includes/latch | 12 - cpp/src/librtcx/system_includes/limits | 12 - cpp/src/librtcx/system_includes/linalg | 12 - cpp/src/librtcx/system_includes/mdspan | 12 - cpp/src/librtcx/system_includes/memory | 12 - cpp/src/librtcx/system_includes/numbers | 12 - cpp/src/librtcx/system_includes/numeric | 12 - cpp/src/librtcx/system_includes/optional | 12 - cpp/src/librtcx/system_includes/ranges | 12 - cpp/src/librtcx/system_includes/ratio | 12 - cpp/src/librtcx/system_includes/semaphore | 12 - .../librtcx/system_includes/source_location | 12 - cpp/src/librtcx/system_includes/span | 12 - cpp/src/librtcx/system_includes/stddef.h | 12 - cpp/src/librtcx/system_includes/stdint.h | 12 - cpp/src/librtcx/system_includes/stdlib.h | 12 - cpp/src/librtcx/system_includes/string.h | 12 - cpp/src/librtcx/system_includes/string_view | 12 - cpp/src/librtcx/system_includes/time.h | 12 - cpp/src/librtcx/system_includes/tuple | 12 - cpp/src/librtcx/system_includes/type_traits | 12 - cpp/src/librtcx/system_includes/utility | 12 - cpp/src/librtcx/system_includes/variant | 12 - cpp/src/librtcx/system_includes/version | 6 - cpp/src/rolling/detail/rolling_udf.cuh | 20 +- cpp/src/runtime/context.cpp | 29 +- cpp/src/runtime/context.hpp | 6 +- cpp/src/transform/transform.cu | 24 +- 69 files changed, 842 insertions(+), 1331 deletions(-) create mode 100644 cpp/src/librtcx/system_includes/README.md delete mode 100644 cpp/src/librtcx/system_includes/algorithm delete mode 100644 cpp/src/librtcx/system_includes/array delete mode 100644 cpp/src/librtcx/system_includes/atomic delete mode 100644 cpp/src/librtcx/system_includes/barrier delete mode 100644 cpp/src/librtcx/system_includes/bit delete mode 100644 cpp/src/librtcx/system_includes/cassert delete mode 100644 cpp/src/librtcx/system_includes/cassert.h delete mode 100644 cpp/src/librtcx/system_includes/ccomplex delete mode 100644 cpp/src/librtcx/system_includes/ccomplex.h delete mode 100644 cpp/src/librtcx/system_includes/cfloat delete mode 100644 cpp/src/librtcx/system_includes/cfloat.h delete mode 100644 cpp/src/librtcx/system_includes/chrono delete mode 100644 cpp/src/librtcx/system_includes/climits delete mode 100644 cpp/src/librtcx/system_includes/cmath delete mode 100644 cpp/src/librtcx/system_includes/cmath.h delete mode 100644 cpp/src/librtcx/system_includes/complex delete mode 100644 cpp/src/librtcx/system_includes/concepts delete mode 100644 cpp/src/librtcx/system_includes/cstddef delete mode 100644 cpp/src/librtcx/system_includes/cstdint delete mode 100644 cpp/src/librtcx/system_includes/cstdlib delete mode 100644 cpp/src/librtcx/system_includes/cstring delete mode 100644 cpp/src/librtcx/system_includes/ctime delete mode 100644 cpp/src/librtcx/system_includes/expected delete mode 100644 cpp/src/librtcx/system_includes/functional delete mode 100644 cpp/src/librtcx/system_includes/initializer_list delete mode 100644 cpp/src/librtcx/system_includes/inplace_vector delete mode 100644 cpp/src/librtcx/system_includes/iterator delete mode 100644 cpp/src/librtcx/system_includes/latch delete mode 100644 cpp/src/librtcx/system_includes/limits delete mode 100644 cpp/src/librtcx/system_includes/linalg delete mode 100644 cpp/src/librtcx/system_includes/mdspan delete mode 100644 cpp/src/librtcx/system_includes/memory delete mode 100644 cpp/src/librtcx/system_includes/numbers delete mode 100644 cpp/src/librtcx/system_includes/numeric delete mode 100644 cpp/src/librtcx/system_includes/optional delete mode 100644 cpp/src/librtcx/system_includes/ranges delete mode 100644 cpp/src/librtcx/system_includes/ratio delete mode 100644 cpp/src/librtcx/system_includes/semaphore delete mode 100644 cpp/src/librtcx/system_includes/source_location delete mode 100644 cpp/src/librtcx/system_includes/span delete mode 100644 cpp/src/librtcx/system_includes/stddef.h delete mode 100644 cpp/src/librtcx/system_includes/stdint.h delete mode 100644 cpp/src/librtcx/system_includes/stdlib.h delete mode 100644 cpp/src/librtcx/system_includes/string.h delete mode 100644 cpp/src/librtcx/system_includes/string_view delete mode 100644 cpp/src/librtcx/system_includes/time.h delete mode 100644 cpp/src/librtcx/system_includes/tuple delete mode 100644 cpp/src/librtcx/system_includes/type_traits delete mode 100644 cpp/src/librtcx/system_includes/utility delete mode 100644 cpp/src/librtcx/system_includes/variant delete mode 100644 cpp/src/librtcx/system_includes/version diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bd5d2a245618..6744d64d5ca8 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -264,9 +264,6 @@ include(${rapids-cmake-dir}/cpm/rapids_logger.cmake) rapids_cpm_rapids_logger(BUILD_EXPORT_SET cudf-exports INSTALL_EXPORT_SET cudf-exports) create_logger_macros(CUDF "cudf::default_logger()" include/cudf) -# find jitify -include(cmake/thirdparty/get_jitify.cmake) - # find NVTX include(cmake/thirdparty/get_nvtx.cmake) @@ -606,7 +603,6 @@ add_library( src/io/utilities/row_selection.cpp src/io/utilities/type_inference.cu src/io/utilities/trie.cu - src/jit/cache.cpp src/jit/helpers.cpp src/jit/jit.cpp src/jit/parser.cpp @@ -1023,12 +1019,8 @@ target_compile_definitions(cudf PRIVATE $<$:CUDF_ # https://github.com/NVIDIA/cccl/pull/2844 target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_AVOID_SORT_UNROLL=1) -# Compile stringified JIT sources first -add_dependencies(cudf jitify_preprocess_run) - # Specify the target module library dependencies -# TODO: use runtime nvrtc and nvjitlink and driver target_link_libraries( cudf PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ diff --git a/cpp/cmake/thirdparty/get_lz4.cmake b/cpp/cmake/thirdparty/get_lz4.cmake index ddf3c9a68994..183fecef9adc 100644 --- a/cpp/cmake/thirdparty/get_lz4.cmake +++ b/cpp/cmake/thirdparty/get_lz4.cmake @@ -14,7 +14,7 @@ function(find_and_configure_lz4) GLOBAL_TARGETS lz4 CPM_ARGS GIT_REPOSITORY https://github.com/lz4/lz4.git - GIT_TAG 130f457aea830ab80e2e7b438408384fe8eefe07 + GIT_TAG 5c4c1fb2354133e1f3b087a341576985f8114bd5 GIT_SHALLOW TRUE ) diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index 8912377a407c..998d07cab144 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -20,7 +20,8 @@ enum class init_flags : std::uint32_t { LOAD_NVCOMP = 1 << 0, /// @brief Initialize the JIT program cache during initialization INIT_JIT_CACHE = 1 << 1, - DEFAULT = INIT_JIT_CACHE, + /// @brief Pre-load the JIT program cache from disk during initialization + DEFAULT = INIT_JIT_CACHE, /// @brief All initialization steps ALL = LOAD_NVCOMP | INIT_JIT_CACHE }; diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index 254513d80c9a..a555c5abff7a 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -22,8 +22,8 @@ */ #include "compiled/binary_ops.hpp" -#include "jit/cache.hpp" #include "jit/helpers.hpp" +#include "jit/jit.hpp" #include "jit/parser.hpp" #include "jit/util.hpp" @@ -43,8 +43,6 @@ #include -#include - #include namespace cudf { @@ -156,23 +154,24 @@ void binary_operation(mutable_column_view& out, {2, cudf::type_to_name(rhs.type())}, }); - std::string kernel_reflection = jitify2::reflection::Template("cudf::binops::jit::kernel_v_v") - .instantiate(output_type_name, // list of template arguments - cudf::type_to_name(lhs.type()), - cudf::type_to_name(rhs.type()), - std::string("cudf::binops::jit::UserDefinedOp")); + auto kernel_reflection = rtcx::reflect_template("cudf::binops::jit::kernel_v_v", + output_type_name, + cudf::type_to_name(lhs.type()), + cudf::type_to_name(rhs.type()), + "cudf::binops::jit::UserDefinedOp"); auto kernel = cudf::jit::get_udf_kernel( "src/binaryop/jit/kernel.cu", "src/binaryop/jit/kernel.cu", kernel_reflection, cuda_source); - auto out_arg = cudf::jit::get_data_ptr(out); - auto lhs_arg = cudf::jit::get_data_ptr(lhs); - auto rhs_arg = cudf::jit::get_data_ptr(rhs); + auto size_arg = static_cast(out.size()); + auto out_arg = cudf::jit::get_data_ptr(out); + auto lhs_arg = cudf::jit::get_data_ptr(lhs); + auto rhs_arg = cudf::jit::get_data_ptr(rhs); - void* args[] = {&out_arg, &lhs_arg, &rhs_arg}; + void* args[] = {&size_arg, &out_arg, &lhs_arg, &rhs_arg}; auto cfg = kernel.max_occupancy_config(0, 0); - kernel.launch(cfg.min_grid_size, 1, 1, cfg.block_size, 1, 1, 0, stream, args); + kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } } // namespace jit diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 117ed3940f73..6e42d01350a7 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -8,6 +8,8 @@ #include #include +#include +#include namespace cudf { namespace jit { @@ -55,10 +57,9 @@ size_type get_projection_size(std::span build_ptx_params(std::span ou if (has_user_data) { params.emplace(index++, "void *"); - params.emplace(index++, jitify2::reflection::reflect()); + params.emplace(index++, "cudf::size_type"); } for (auto& name : output_typenames) { @@ -139,24 +140,23 @@ kernel get_udf_kernel(std::string const& name, { CUDF_FUNC_RANGE(); - char const* include_names[] = {"cudf/detail/operation-udf.hpp"}; - - char const* sources[] = {udf_cuda_source.c_str()}; + char const* include_names[] = {"cudf/detail/operation-udf.hpp"}; + char const* include_headers[] = {udf_cuda_source.c_str()}; int constexpr min_pch_runtime_version = 12800; // CUDA 12.8 int runtime_version; CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); - return get_kernel(name.c_str(), - "", // TODO: key - std::format("#include <{}>\n", source_file), + return get_kernel(name, + source_file, include_names, - sources, - kernel_name.c_str(), - /*use_cache=*/true, // TODO: use context config - /*use_pch=*/runtime_version >= min_pch_runtime_version, - true); + include_headers, + kernel_name, + true, // TODO: use context config + runtime_version >= min_pch_runtime_version, + false // TODO: use context config + ); } } // namespace jit diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index 39c5b9258a8f..eea949f75837 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -11,10 +11,8 @@ #include -#include #include #include -#include #include #include diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index 74cc524c5f01..04213895c20c 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -26,18 +26,6 @@ #include #include -#define CUDFRTC_CHECK_CUDART(msg, ...) \ - do { \ - ::cudaError_t __result = (__VA_ARGS__); \ - if (__result != ::cudaSuccess) { \ - auto __errstr = ::std::format("(cudart) Call {} failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast<::int64_t>(__result), \ - ::cudaGetErrorString(__result)); \ - CUDF_FAIL(+std::format("{}. {}", msg, __errstr), ::std::runtime_error); \ - } \ - } while (0) - namespace CUDF_EXPORT cudf { namespace { @@ -49,6 +37,15 @@ rtcx::sha256 hash_string(std::span input) return ctx.finalize(); } +rtcx::sha256 hash_strings(std::span inputs) +{ + rtcx::sha256_context ctx; + for (auto const* input : inputs) { + ctx.update(std::span{reinterpret_cast(input), std::strlen(input)}); + } + return ctx.finalize(); +} + [[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) { auto error_code = errno; @@ -156,7 +153,7 @@ void install_cudf_jit_files(char const* target_dir) "mkdtemp"); } - install_file_set(target_dir, + install_file_set(tmp_dir, rtcx_embed::cudf_jit_embed_files, rtcx_embed::cudf_jit_embed_files_uncompressed_size, rtcx_embed::cudf_jit_embed_file_ranges, @@ -232,27 +229,24 @@ namespace { int32_t get_driver_version() { int32_t driver_version; - CUDFRTC_CHECK_CUDART("Failed to get CUDA driver version", cudaDriverGetVersion(&driver_version)); - + CUDF_CUDA_TRY(cudaDriverGetVersion(&driver_version)); return driver_version; } int32_t get_runtime_version() { int32_t runtime_version; - CUDFRTC_CHECK_CUDART("Failed to get CUDA runtime version", - cudaRuntimeGetVersion(&runtime_version)); - + CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); return runtime_version; } int32_t get_current_device_physical_model() { int32_t device; - CUDFRTC_CHECK_CUDART("Failed to get current CUDA device", cudaGetDevice(&device)); + CUDF_CUDA_TRY(cudaGetDevice(&device)); cudaDeviceProp props; - CUDFRTC_CHECK_CUDART("Failed to get device properties", cudaGetDeviceProperties(&props, device)); + CUDF_CUDA_TRY(cudaGetDeviceProperties(&props, device)); return props.major * 10 + props.minor; } @@ -293,6 +287,10 @@ std::tuple compile_library_uncached( options.emplace_back("-DCUDF_RUNTIME_JIT"); options.emplace_back("--diag-suppress=47"); options.emplace_back("--device-int128"); + options.emplace_back("-std=c++20"); + options.emplace_back("-default-device"); + options.emplace_back("--device-debug"); + options.emplace_back("--generate-line-info"); if (use_pch) { options.emplace_back("--pch"); @@ -327,7 +325,7 @@ std::tuple compile_library_uncached( name, std::chrono::duration_cast>(duration).count()); - auto library = rtcx::load_library(cubin, rtcx::binary_type::CUBIN); + auto library = rtcx::load_library(cubin); auto blob = rtcx::blob_t::from_vector(std::move(cubin)); @@ -349,11 +347,10 @@ static rtcx::kernel_ref get_kernel(rtcx::library const& lib) kernel::kernel(rtcx::library lib) : _library(std::move(lib)), _kernel(get_kernel(_library)) {} kernel get_kernel(std::string const& name, - std::string const& key, - std::string const& cuda_udf, + std::string const& source_file, std::span header_include_names, std::span headers, - char const* name_expression, + std::string const& name_expression, bool use_cache, bool use_pch, bool log_pch) @@ -363,29 +360,55 @@ kernel get_kernel(std::string const& name, auto& cache = cudf::get_context().rtc_cache(); auto& bundle = cudf::get_context().jit_bundle(); - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_physical_model(); - auto bundle_hash = bundle.get_hash(); + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_physical_model(); + auto header_include_names_hash = hash_strings(header_include_names).to_hex_string(); + auto headers_hash = hash_strings(headers).to_hex_string(); + auto bundle_hash = bundle.get_hash(); - auto cache_key = std::format(R"***(binary_type=CUBIN -key={} + auto cache_key = std::format(R"***(cuLibrary +binary_type=CUBIN cuda_runtime={} cuda_driver={} arch={} -bundle={})***", - key, +bundle={} +source_file={} +header_include_names={} +headers={} +name_expression={} +)***", runtime, driver, sm, - bundle_hash); + bundle_hash, + source_file, + header_include_names_hash.view(), + headers_hash.view(), + name_expression); auto cache_key_sha256 = hash_string(cache_key); auto compile = [&] { - char const* name_exprs[] = {name_expression}; - return compile_library_uncached( - name.c_str(), cuda_udf.c_str(), header_include_names, headers, name_exprs, use_pch, log_pch); + auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); + auto source_file_path = std::format("{}/cudf/{}", bundle_dir, source_file); + auto source = rtcx::blob_t::from_file(source_file_path.c_str()); + CUDF_EXPECTS( + source.has_value(), + +std::format( + "Failed to load UDF CUDA source file: `{}` for compilation, source file does not exist", + source_file_path), + std::runtime_error); + std::string source_str{source->view().begin(), source->view().end()}; + + char const* name_exprs[] = {name_expression.c_str()}; + return compile_library_uncached(name.c_str(), + source_str.c_str(), + header_include_names, + headers, + name_exprs, + use_pch, + log_pch); }; if (!use_cache) { @@ -393,8 +416,8 @@ bundle={})***", return lib; } - auto fut = cache.get_or_add_library( - cache_key_sha256, rtcx::binary_type::CUBIN, rtcx::library_compile_func::from_functor(compile)); + auto fut = + cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); return fut.get(); } diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index b3937d46f572..75f0c3dbaafc 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -50,34 +50,21 @@ struct [[nodiscard]] kernel { return _kernel.max_occupancy_config(dynamic_shared_memory_bytes, block_size_limit); } - void launch(uint32_t grid_dim_x, - uint32_t grid_dim_y, - uint32_t grid_dim_z, - uint32_t block_dim_x, - uint32_t block_dim_y, - uint32_t block_dim_z, + void launch(rtcx::cuda_dim3 grid_dim, + rtcx::cuda_dim3 block_dim, uint32_t shared_mem_bytes, rmm::cuda_stream_view stream, - std::span kernel_params) const + void** kernel_params) const { - return _kernel.launch(grid_dim_x, - grid_dim_y, - grid_dim_z, - block_dim_x, - block_dim_y, - block_dim_z, - shared_mem_bytes, - stream.value(), - kernel_params.data()); + return _kernel.launch(grid_dim, block_dim, shared_mem_bytes, stream.value(), kernel_params); } }; kernel get_kernel(std::string const& name, - std::string const& key, - std::string const& cuda_udf, + std::string const& source_file, std::span header_include_names, std::span headers, - char const* name_expression, + std::string const& name_expression, bool use_cache = true, bool use_pch = true, bool log_pch = false); diff --git a/cpp/src/join/filter_join_indices_jit.cu b/cpp/src/join/filter_join_indices_jit.cu index dfb58af6d40b..24627407efbb 100644 --- a/cpp/src/join/filter_join_indices_jit.cu +++ b/cpp/src/join/filter_join_indices_jit.cu @@ -33,12 +33,11 @@ #include #include -#include #include +#include #include #include #include -#include #include #include @@ -49,32 +48,35 @@ namespace detail { namespace { // Build template parameters for JIT kernel -jitify2::StringVec build_join_filter_template_params(std::vector const& left_columns, - std::vector const& right_columns, - bool has_user_data, - null_aware is_null_aware) +std::vector build_join_filter_template_params( + std::vector const& left_columns, + std::vector const& right_columns, + bool has_user_data, + null_aware is_null_aware) { - jitify2::StringVec template_params; + std::vector template_params; - template_params.emplace_back(jitify2::reflection::reflect(has_user_data)); - template_params.emplace_back(jitify2::reflection::reflect(is_null_aware)); + template_params.emplace_back(rtcx::reflect_bool(has_user_data)); + template_params.emplace_back(rtcx::reflect_enum("cudf::is_null_aware", is_null_aware)); // Add left column accessors for (size_t i = 0; i < left_columns.size(); ++i) { auto const& col = left_columns[i]; std::string type_name = cudf::type_to_name(col.type()); - template_params.emplace_back( - jitify2::reflection::Template("cudf::jit::join_column_accessor") - .instantiate(type_name, std::to_string(i), "cudf::jit::join_side::LEFT")); + template_params.emplace_back(rtcx::reflect_template("cudf::jit::join_column_accessor", + type_name, + std::to_string(i), + "cudf::jit::join_side::LEFT")); } // Add right column accessors for (size_t i = 0; i < right_columns.size(); ++i) { auto const& col = right_columns[i]; std::string type_name = cudf::type_to_name(col.type()); - template_params.emplace_back( - jitify2::reflection::Template("cudf::jit::join_column_accessor") - .instantiate(type_name, std::to_string(i), "cudf::jit::join_side::RIGHT")); + template_params.emplace_back(rtcx::reflect_template("cudf::jit::join_column_accessor", + type_name, + std::to_string(i), + "cudf::jit::join_side::RIGHT")); } return template_params; @@ -110,8 +112,7 @@ kernel build_join_filter_kernel(std::string const& predicate_code, // Build template parameters and kernel name auto template_args = build_join_filter_template_params(left_columns, right_columns, has_user_data, is_null_aware); - auto kernel_name = - jitify2::reflection::Template("cudf::join::jit::filter_join_kernel").instantiate(template_args); + auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); // Get compiled kernel return cudf::jit::get_udf_kernel("src/join/jit/filter_join_kernel.cu", // TODO: use actual name @@ -160,7 +161,7 @@ void launch_join_filter_kernel(kernel const& kernel, &user_data_ptr}; auto cfg = kernel.max_occupancy_config(0, 0); - kernel.launch(cfg.min_grid_size, 1, 1, cfg.block_size, 1, 1, stream, args); + kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } // Same join semantics handling as the AST version @@ -350,15 +351,15 @@ apply_join_semantics(cudf::table_view const& left, } // Build template parameters from AST input specs (preserves expression input order) -jitify2::StringVec build_join_filter_template_params_from_specs( +std::vector build_join_filter_template_params_from_specs( std::vector const& input_specs, cudf::table_view const& left, cudf::table_view const& right, null_aware is_null_aware) { - jitify2::StringVec template_params; - template_params.emplace_back(jitify2::reflection::reflect(false)); // has_user_data = false - template_params.emplace_back(jitify2::reflection::reflect(is_null_aware)); + std::vector template_params; + template_params.emplace_back(rtcx::reflect_bool(false)); // has_user_data = false + template_params.emplace_back(rtcx::reflect_enum("cudf::is_null_aware", is_null_aware)); // Scalar columns are appended to the left table's device views, // starting at index left.num_columns(). @@ -372,14 +373,13 @@ jitify2::StringVec build_join_filter_template_params_from_specs( ? "cudf::jit::join_side::LEFT" : "cudf::jit::join_side::RIGHT"; auto type_name = cudf::type_to_name(table.column(col_spec.column).type()); - template_params.emplace_back( - jitify2::reflection::Template("cudf::jit::join_column_accessor") - .instantiate(type_name, std::to_string(col_spec.column), side_str)); + template_params.emplace_back(rtcx::reflect_template( + "cudf::jit::join_column_accessor", type_name, std::to_string(col_spec.column), side_str)); } else if (std::holds_alternative(spec)) { auto const& scalar_spec = std::get(spec); auto type_name = cudf::type_to_name(scalar_spec.ref.get().type()); - template_params.emplace_back(jitify2::reflection::Template("cudf::jit::join_scalar_accessor") - .instantiate(type_name, std::to_string(scalar_index++))); + template_params.emplace_back(rtcx::reflect_template( + "cudf::jit::join_scalar_accessor", type_name, std::to_string(scalar_index++))); } } @@ -508,8 +508,7 @@ filter_join_indices_jit(cudf::table_view const& left, auto const cuda_source = cudf::jit::parse_single_function_cuda(filter_result.udf, "GENERIC_JOIN_FILTER_OP"); - auto kernel_name = - jitify2::reflection::Template("cudf::join::jit::filter_join_kernel").instantiate(template_args); + auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); auto kernel = cudf::jit::get_udf_kernel("src/join/jit/filter_join_kernel.cu", // TODO: use actual name "src/join/jit/filter_join_kernel.cu", diff --git a/cpp/src/librtcx/CMake/embed.cmake b/cpp/src/librtcx/CMake/embed.cmake index 811c13a29040..8f2227d75a3b 100644 --- a/cpp/src/librtcx/CMake/embed.cmake +++ b/cpp/src/librtcx/CMake/embed.cmake @@ -7,6 +7,9 @@ find_package(Python3 REQUIRED COMPONENTS Interpreter) +# This function registers a directory of include files to be embedded for JIT compilation. It +# gathers the specified files, their destinations, and include directories, and stores them in +# target-specific variables for later use when generating the embed. function(jit_add_include_directory) set(TARGET ${ARGV0}) set(OPTIONS "") @@ -139,8 +142,9 @@ function(jit_embed) ) add_custom_target( - ${TARGET} ALL DEPENDS ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s - ${OUTPUT_DIR}/${TARGET}.bin + ${TARGET} ALL + DEPENDS ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin + COMMENT "Custom target for JIT embed of ${TARGET}" ) message( diff --git a/cpp/src/librtcx/CMake/embed.in.py b/cpp/src/librtcx/CMake/embed.in.py index 716ace56186b..9c60ede643cf 100644 --- a/cpp/src/librtcx/CMake/embed.in.py +++ b/cpp/src/librtcx/CMake/embed.in.py @@ -1,8 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import hashlib,os, lz4.block, zstd +import hashlib +import os from typing import NamedTuple +import lz4.block +import zstd + + def merge_bytes_with_null_terminators( bytes_lists: list[bytes], ) -> tuple[bytes, list[tuple[int, int]]]: @@ -15,6 +20,7 @@ def merge_bytes_with_null_terminators( return merged, ranges + class EmbedOutput(NamedTuple): cxx_header: str | None cxx_source: str | None @@ -23,10 +29,12 @@ class EmbedOutput(NamedTuple): bin_file_data: bytes | None hash: bytes + def load_file_bytes(file_path: str) -> bytes: with open(file_path, "rb") as f: return f.read() + def compress_bytes(data: bytes, compression: str) -> bytes: assert compression in ("none", "lz4", "zstd"), "Invalid compression type" @@ -39,6 +47,7 @@ def compress_bytes(data: bytes, compression: str) -> bytes: elif compression == "zstd": return zstd.compress(data, 22) + def generate_cxx_source_files_data( id: str, file_paths: list[str], @@ -58,21 +67,23 @@ def generate_cxx_source_files_data( else None ) - binary_size = len(compressed_files_bytes) if compress else len(uncompressed_files_bytes) + binary_size = ( + len(compressed_files_bytes) + if compress + else len(uncompressed_files_bytes) + ) if compress: print( - f"-- Compressed {id}'s binary from {len(uncompressed_files_bytes)} bytes to {len(compressed_files_bytes)} bytes (compression ratio: {len(compressed_files_bytes)/len(uncompressed_files_bytes):.2f})" + f"-- Compressed {id}'s binary from {len(uncompressed_files_bytes)} bytes to {len(compressed_files_bytes)} bytes (compression ratio: {len(compressed_files_bytes) / len(uncompressed_files_bytes):.2f})" ) - merged_dests_bytes, _ = ( - merge_bytes_with_null_terminators([d.encode("utf-8") for d in dests]) + merged_dests_bytes, _ = merge_bytes_with_null_terminators( + [d.encode("utf-8") for d in dests] ) - merged_include_directories_bytes, _ = ( - merge_bytes_with_null_terminators( - [d.encode("utf-8") for d in include_directories] - ) + merged_include_directories_bytes, _ = merge_bytes_with_null_terminators( + [d.encode("utf-8") for d in include_directories] ) # compute combined sha256 hash of all files @@ -112,7 +123,7 @@ def generate_cxx_source_files_data( constexpr range {id}_file_ranges[{len(files_ranges)}] = {{ -{",\n".join([f'{{{offset}, {size}}}' for offset, size in files_ranges])} +{",\n".join([f"{{{offset}, {size}}}" for offset, size in files_ranges])} }}; @@ -130,7 +141,7 @@ def generate_cxx_source_files_data( constexpr std::uint8_t {id}_hash[{len(hash)}] = {{ -{", ".join([f'0x{b:02x}' for b in hash])} +{", ".join([f"0x{b:02x}" for b in hash])} }}; }} @@ -156,15 +167,18 @@ def generate_cxx_source_files_data( hash=hash, ) + def generate_embed( id: str, file_paths: list[str], file_dests: list[str], include_directories: list[str], compression: str, - output_dir: str + output_dir: str, ): - output = generate_cxx_source_files_data( id, file_paths, file_dests, include_directories, compression) + output = generate_cxx_source_files_data( + id, file_paths, file_dests, include_directories, compression + ) os.makedirs(output_dir, exist_ok=True) @@ -178,19 +192,27 @@ def generate_embed( with open(f"{output_dir}/{output.bin_file_name}", "wb") as f: f.write(output.bin_file_data) + def main(): - id : str = "@RTCX_EMBED_PY_ARG__ID@" - file_paths_str :str = "@RTCX_EMBED_PY_ARG__FILE_PATHS@" - file_dests_str : str = "@RTCX_EMBED_PY_ARG__FILE_DESTS@" - include_directories_str : str = "@RTCX_EMBED_PY_ARG__INCLUDE_DIRS@" - compression : str = "@RTCX_EMBED_PY_ARG__COMPRESSION@" - output_dir : str = "@RTCX_EMBED_PY_ARG__OUTPUT_DIR@" - - file_paths = file_paths_str.split(";") if file_paths_str else [] - file_dests = file_dests_str.split(";") if file_dests_str else [] - include_directories = include_directories_str.split(";") if include_directories_str else [] - - generate_embed(id, file_paths, file_dests, include_directories, compression, output_dir) + id: str = "@RTCX_EMBED_PY_ARG__ID@" + file_paths_str: str = "@RTCX_EMBED_PY_ARG__FILE_PATHS@" + file_dests_str: str = "@RTCX_EMBED_PY_ARG__FILE_DESTS@" + include_directories_str: str = "@RTCX_EMBED_PY_ARG__INCLUDE_DIRS@" + compression: str = "@RTCX_EMBED_PY_ARG__COMPRESSION@" + output_dir: str = "@RTCX_EMBED_PY_ARG__OUTPUT_DIR@" + + file_paths = file_paths_str.split(";") + file_dests = file_dests_str.split(";") + include_directories = include_directories_str.split(";") + + generate_embed( + id, + file_paths, + file_dests, + include_directories, + compression, + output_dir, + ) if __name__ == "__main__": diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index c902c1903be4..2b618ec7ebe4 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -20,7 +20,9 @@ #include #include +#include #include +#include #include #include #include @@ -60,7 +62,7 @@ extern "C" { std::runtime_error); \ auto __errstr = ::std::format("(cuda) expression `{}` failed, with error ({}): {}", \ #__VA_ARGS__, \ - static_cast<::rtcx::i64>(__result), \ + static_cast<::std::int64_t>(__result), \ __enum_str); \ RTCX_FAIL(__errstr, ::std::runtime_error); \ } \ @@ -74,34 +76,32 @@ extern "C" { char const* __enum_msg = ::cudaGetErrorString(__result); \ auto __errstr = ::std::format("(cudart) expression `{}` failed, with error ({}: {}): {}", \ #__VA_ARGS__, \ - static_cast<::rtcx::i64>(__result), \ + static_cast<::std::int64_t>(__result), \ __enum_name, \ __enum_msg); \ RTCX_FAIL(__errstr, ::std::runtime_error); \ } \ } while (0) -#define RTCX_CHECK_NVRTC(params, program, ...) \ +#define RTCX_CHECK_NVRTC(...) \ do { \ ::nvrtcResult __result = (__VA_ARGS__); \ - ::rtcx::log_nvrtc_result(params, program, __result); \ if (__result != ::NVRTC_SUCCESS) { \ auto __errstr = ::std::format("(nvrtc) expression `{}` failed, with error ({}): {}", \ #__VA_ARGS__, \ - static_cast<::rtcx::i64>(__result), \ + static_cast<::std::int64_t>(__result), \ ::rtcx::nvrtc->GetErrorString(__result)); \ RTCX_FAIL(__errstr, ::std::runtime_error); \ } \ } while (0) -#define RTCX_CHECK_NVJITLINK(params, handle, ...) \ +#define RTCX_CHECK_NVJITLINK(...) \ do { \ ::nvJitLinkResult __result = (__VA_ARGS__); \ - ::rtcx::log_nvJitLink_result(params, handle, __result); \ if (__result != ::NVJITLINK_SUCCESS) { \ auto __errstr = ::std::format("(nvJitLink) expression `{}` failed, with error ({}): {}", \ #__VA_ARGS__, \ - static_cast<::rtcx::i64>(__result), \ + static_cast<::std::int64_t>(__result), \ ::rtcx::get_nvJitLinkResultString(__result)); \ RTCX_FAIL(__errstr, ::std::runtime_error); \ } \ @@ -109,14 +109,70 @@ extern "C" { namespace RTCX_EXPORT rtcx { +namespace { + +template +std::string join_strings(std::span strings, std::string_view separator) +{ + if (strings.empty()) { return {}; } + + if (strings.size() == 1) { return std::string{strings[0].begin(), strings[0].end()}; } + + auto total_size = std::transform_reduce( + strings.begin(), + strings.end(), + size_t{0}, + [](size_t total, size_t str_size) { return total + str_size; }, + [](auto const& str) { return str.size(); }); + + auto separator_size = separator.size() * (strings.size() - 1); + + std::string result; + result.reserve(total_size + separator_size); + + for (size_t i = 0; i < strings.size(); ++i) { + result.append(strings[i].begin(), strings[i].end()); + if (i != (strings.size() - 1)) { result.append(separator); } + } + + return result; +} + +} // namespace + void log_warning(std::string_view message) { - std::fprintf(stdout, "RTCX WARNING: %.*s\n", static_cast(message.size()), message.data()); + std::fprintf( + stdout, "[RTCX WARNING] %.*s\n", static_cast(message.size()), message.data()); } void log_error(std::string_view message) { - std::fprintf(stderr, "RTCX ERROR: %.*s\n", static_cast(message.size()), message.data()); + std::fprintf( + stderr, "[RTCX ERROR] %.*s\n", static_cast(message.size()), message.data()); +} + +void log_trace(std::string_view message) +{ + std::fprintf( + stdout, "[RTCX TRACE] %.*s\n", static_cast(message.size()), message.data()); +} + +sha256 sha256::parse(std::string_view hex) +{ + RTCX_EXPECTS( + hex.size() == 64, + std::format( + "Invalid SHA256 hex string length, expected 64 got {} (sha: `{}`)", hex.size(), hex), + std::invalid_argument); + sha256 hash; + for (std::size_t i = 0; i < 32; ++i) { + auto hex_byte = hex.substr(i * 2, 2); + auto [ptr, ec] = std::from_chars(hex_byte.begin(), hex_byte.end(), hash.data_[i], 16); + RTCX_EXPECTS( + ec == std::errc(), "Invalid hex character in SHA256 string", std::invalid_argument); + } + return hash; } sha256_context::sha256_context() : ectx_(nullptr) @@ -133,7 +189,7 @@ sha256_context::~sha256_context() if (ectx_ != nullptr) { EVP_MD_CTX_free(ectx_); } } -void sha256_context::update(std::span data) +void sha256_context::update(std::span data) { RTCX_EXPECTS(EVP_DigestUpdate(ectx_, data.data(), data.size()) == 1, "EVP_DigestUpdate failed", @@ -143,7 +199,7 @@ void sha256_context::update(std::span data) sha256 sha256_context::finalize() { sha256 hash; - u32 length = 0; + std::uint32_t length = 0; RTCX_EXPECTS(EVP_DigestFinal_ex(ectx_, hash.data_, &length) == 1, "EVP_DigestFinal_ex failed", std::runtime_error); @@ -164,6 +220,7 @@ sha256 sha256_context::finalize() DO_IT(OccupancyMaxPotentialBlockSize) \ DO_IT(LaunchKernel) \ DO_IT(LaunchKernelEx) \ + DO_IT(LaunchCooperativeKernel) \ DO_IT(KernelGetName) \ DO_IT(LibraryLoadData) \ DO_IT(LibraryGetKernel) \ @@ -216,15 +273,10 @@ void* load_dll(std::string_view base_name, std::span names) if (handle != nullptr) { return handle; } } - std::string tried_names = std::accumulate( - names.begin(), names.end(), std::string{}, [&](std::string acc, std::string const& name) { - if (!acc.empty()) { acc += ", "; } - acc += name; - return acc; - }); - - RTCX_FAIL(std::format("Failed to load dynamic library `{}` (tried: {})", base_name, tried_names), - std::runtime_error); + RTCX_FAIL( + std::format( + "Failed to load dynamic library `{}` (tried: {})", base_name, join_strings(names, ", ")), + std::runtime_error); } void* get_symbol(char const* lib_name, void* handle, char const* sym_name) @@ -238,9 +290,9 @@ void* get_symbol(char const* lib_name, void* handle, char const* sym_name) return sym; } -inline constexpr i32 major_version(i32 version) { return version / 1000; } +inline constexpr std::int32_t major_version(std::int32_t version) { return version / 1000; } -inline constexpr i32 minor_version(i32 version) { return (version % 1000) / 10; } +inline constexpr std::int32_t minor_version(std::int32_t version) { return (version % 1000) / 10; } struct LibCuda { void* _handle = nullptr; @@ -288,11 +340,10 @@ struct LibNVRTC { static void* _load() { - i32 cuda_version; + std::int32_t cuda_version; RTCX_CHECK_CUDART(cudaRuntimeGetVersion(&cuda_version)); - - i32 major = major_version(cuda_version); - i32 minor = minor_version(cuda_version); + std::int32_t major = major_version(cuda_version); + std::int32_t minor = minor_version(cuda_version); std::string lib_names[] = {std::format("libnvrtc.so.{}.{}", major, minor), std::format("libnvrtc.so.{}", major), @@ -328,11 +379,10 @@ struct LibNVJitLink { static void* _load() { - i32 cuda_version; + std::int32_t cuda_version; RTCX_CHECK_CUDART(cudaRuntimeGetVersion(&cuda_version)); - - i32 major = major_version(cuda_version); - i32 minor = minor_version(cuda_version); + std::int32_t major = major_version(cuda_version); + std::int32_t minor = minor_version(cuda_version); std::string lib_names[] = {std::format("libnvJitLink.so.{}.{}", major, minor), std::format("libnvJitLink.so.{}", major), @@ -364,6 +414,8 @@ void initialize() { std::call_once(*init_libraries_flag, [] { cuda.emplace(LibCuda::_load()); + RTCX_EXPECTS( + cuda->Init(0) == CUDA_SUCCESS, "Failed to initialize CUDA driver API", std::runtime_error); nvrtc.emplace(LibNVRTC::_load()); nvjitlink.emplace(LibNVJitLink::_load()); }); @@ -407,8 +459,9 @@ char const* get_nvJitLinkResultString(nvJitLinkResult result) case NVJITLINK_ERROR_LTO_NOT_ENABLED: return "NVJITLINK_ERROR_LTO_NOT_ENABLED"; #endif default: - RTCX_FAIL(std::format("Unrecognized nvJitLinkResult type: ({})", static_cast(result)), - std::runtime_error); + RTCX_FAIL( + std::format("Unrecognized nvJitLinkResult type: ({})", static_cast(result)), + std::runtime_error); } } @@ -420,7 +473,7 @@ char const* binary_type_string(binary_type type) case binary_type::FATBIN: return "FATBIN"; case binary_type::PTX: return "PTX"; default: - RTCX_FAIL(std::format("Unrecognized binary_type: ({})", static_cast(type)), + RTCX_FAIL(std::format("Unrecognized binary_type: ({})", static_cast(type)), std::runtime_error); } } @@ -433,9 +486,9 @@ nvJitLinkInputType to_nvjitlink_input_type(binary_type bin_type) case binary_type::FATBIN: return NVJITLINK_INPUT_FATBIN; case binary_type::PTX: return NVJITLINK_INPUT_PTX; default: - RTCX_FAIL( - std::format("Unrecognized binary type for linking: ({}) ", static_cast(bin_type)), - std::logic_error); + RTCX_FAIL(std::format("Unrecognized binary type for linking: ({}) ", + static_cast(bin_type)), + std::logic_error); } } @@ -443,12 +496,12 @@ void log_nvrtc_result(compile_params const& params, nvrtcProgram program, nvrtcResult compile_result) { - if (program == nullptr) { return; } + if (program == nullptr || compile_result == NVRTC_SUCCESS) { return; } - usize log_size; + std::size_t log_size; if (auto errc = nvrtc->GetProgramLogSize(program, &log_size); errc != NVRTC_SUCCESS) { RTCX_FAIL(std::format("Failed to get NVRTC program log size with error ({}): {}", - static_cast(errc), + static_cast(errc), nvrtc->GetErrorString(errc)), std::runtime_error); } @@ -460,7 +513,7 @@ void log_nvrtc_result(compile_params const& params, if (auto errc = nvrtc->GetProgramLog(program, log.data()); errc != NVRTC_SUCCESS) { RTCX_FAIL(std::format("Failed to get NVRTC program log with error ({}): {}", - static_cast(errc), + static_cast(errc), nvrtc->GetErrorString(errc)), std::runtime_error); } @@ -484,7 +537,7 @@ void log_nvrtc_result(compile_params const& params, "NVRTC Compilation for `{}` {} ({}): {}.\nHeaders:\n{}\n\nOptions:\n{}\n\nLog:\n\t{}", params.name == nullptr ? "" : params.name, status_str, - static_cast(compile_result), + static_cast(compile_result), nvrtc->GetErrorString(compile_result), headers_str, options_str, @@ -501,12 +554,12 @@ void log_nvJitLink_result(link_params const& params, nvJitLinkHandle handle, nvJitLinkResult link_result) { - if (handle == nullptr) { return; } + if (handle == nullptr || link_result == NVJITLINK_SUCCESS) { return; } - usize info_log_size; + std::size_t info_log_size; if (auto errc = nvjitlink->GetInfoLogSize(handle, &info_log_size); errc != NVJITLINK_SUCCESS) { RTCX_FAIL(std::format("Failed to get nvJitLink info log size with error ({}): {}", - static_cast(errc), + static_cast(errc), get_nvJitLinkResultString(errc)), std::runtime_error); } @@ -516,17 +569,17 @@ void log_nvJitLink_result(link_params const& params, info_log.resize(info_log_size); if (auto errc = nvjitlink->GetInfoLog(handle, info_log.data()); errc != NVJITLINK_SUCCESS) { RTCX_FAIL(std::format("Failed to get nvJitLink info log with error ({}): {}", - static_cast(errc), + static_cast(errc), get_nvJitLinkResultString(errc)), std::runtime_error); } } info_log.resize(info_log_size == 0 ? 0 : (info_log_size - 1)); - usize error_log_size; + std::size_t error_log_size; if (auto errc = nvjitlink->GetErrorLogSize(handle, &error_log_size); errc != NVJITLINK_SUCCESS) { RTCX_FAIL(std::format("Failed to get nvJitLink error log size with error ({}): {}", - static_cast(errc), + static_cast(errc), get_nvJitLinkResultString(errc)), std::runtime_error); } @@ -537,7 +590,7 @@ void log_nvJitLink_result(link_params const& params, error_log.resize(error_log_size); if (auto errc = nvjitlink->GetErrorLog(handle, error_log.data()); errc != NVJITLINK_SUCCESS) { RTCX_FAIL(std::format("Failed to get nvJitLink error log with error ({}): {}", - static_cast(errc), + static_cast(errc), get_nvJitLinkResultString(errc)), std::runtime_error); } @@ -566,7 +619,7 @@ void log_nvJitLink_result(link_params const& params, params.name == nullptr ? "" : params.name, binary_type_str, status_str, - static_cast(link_result), + static_cast(link_result), get_nvJitLinkResultString(link_result), fragments_str, link_options_str, @@ -582,17 +635,18 @@ void log_nvJitLink_result(link_params const& params, } // namespace -blob_t blob_t::from_vector(std::vector&& data) +blob_t blob_t::from_vector(std::vector&& data) { - auto ptr = new std::vector(std::move(data)); + auto ptr = new std::vector(std::move(data)); return blob_t::from_parts( - ptr->data(), ptr->size(), blob_t::deallocator{ptr, [](void* user_data, u8 const*, usize) { - delete reinterpret_cast*>( - user_data); - }}); + ptr->data(), + ptr->size(), + blob_t::deallocator{ptr, [](void* user_data, std::uint8_t const*, std::size_t) { + delete reinterpret_cast*>(user_data); + }}); } -blob_t blob_t::from_static_data(std::span data) +blob_t blob_t::from_static_data(std::span data) { return blob_t::from_parts(data.data(), data.size(), blob_t::noop_deallocator); } @@ -603,63 +657,61 @@ std::vector compile(compile_params const& params) RTCX_EXPECTS(params.source != nullptr, "Fragment source must not be null", std::logic_error); nvrtcProgram program = nullptr; - - RTCX_CHECK_NVRTC(params, - program, - nvrtc->CreateProgram(&program, + RTCX_CHECK_NVRTC(nvrtc->CreateProgram(&program, params.source, params.name, - static_cast(params.headers.size()), + static_cast(params.headers.size()), params.headers.data(), params.header_include_names.data())); RTCX_DEFER([&] { nvrtc->DestroyProgram(&program); }); for (auto* name_expr : params.name_expressions) { - RTCX_CHECK_NVRTC(params, program, nvrtc->AddNameExpression(program, name_expr)); + RTCX_CHECK_NVRTC(nvrtc->AddNameExpression(program, name_expr)); } - // TODO: log is printed twice when warnings are raised - RTCX_CHECK_NVRTC( - params, - program, - nvrtc->CompileProgram(program, static_cast(params.options.size()), params.options.data())); + auto compile_result = nvrtc->CompileProgram( + program, static_cast(params.options.size()), params.options.data()); + log_nvrtc_result(params, program, compile_result); + RTCX_CHECK_NVRTC(compile_result); switch (params.target_type) { + case binary_type::CUBIN: { + std::size_t cubin_size; + RTCX_CHECK_NVRTC(nvrtc->GetCUBINSize(program, &cubin_size)); + std::vector cubin; + cubin.resize(cubin_size); + RTCX_CHECK_NVRTC(nvrtc->GetCUBIN(program, reinterpret_cast(cubin.data()))); + return cubin; + } break; case binary_type::LTO_IR: { - usize lto_ir_size; - RTCX_CHECK_NVRTC(params, program, nvrtc->GetLTOIRSize(program, <o_ir_size)); - + std::size_t lto_ir_size; + RTCX_CHECK_NVRTC(nvrtc->GetLTOIRSize(program, <o_ir_size)); std::vector lto_ir; lto_ir.resize(lto_ir_size); - - RTCX_CHECK_NVRTC( - params, program, nvrtc->GetLTOIR(program, reinterpret_cast(lto_ir.data()))); - + RTCX_CHECK_NVRTC(nvrtc->GetLTOIR(program, reinterpret_cast(lto_ir.data()))); return lto_ir; - } break; - case binary_type::CUBIN: { - usize cubin_size; - RTCX_CHECK_NVRTC(params, program, nvrtc->GetCUBINSize(program, &cubin_size)); - - std::vector cubin; - cubin.resize(cubin_size); - RTCX_CHECK_NVRTC( - params, program, nvrtc->GetCUBIN(program, reinterpret_cast(cubin.data()))); - - return cubin; - + case binary_type::PTX: { + std::size_t ptx_size; + RTCX_CHECK_NVRTC(nvrtc->GetPTXSize(program, &ptx_size)); + std::vector ptx; + ptx.resize(ptx_size); + RTCX_CHECK_NVRTC(nvrtc->GetPTX(program, reinterpret_cast(ptx.data()))); + return ptx; } break; - default: RTCX_FAIL("Unsupported binary type for compiling fragment", std::logic_error); + default: + RTCX_FAIL(std::format("Unsupported binary type for compiling fragment: {}", + binary_type_string(params.target_type)), + std::logic_error); } } -kernel_occupancy_config kernel_ref::max_occupancy_config(usize dynamic_shared_memory_bytes, - i32 block_size_limit) const +kernel_occupancy_config kernel_ref::max_occupancy_config(std::size_t dynamic_shared_memory_bytes, + std::int32_t block_size_limit) const { - i32 min_grid_size; - i32 block_size; + std::int32_t min_grid_size; + std::int32_t block_size; RTCX_CHECK_CUDA(cuda->OccupancyMaxPotentialBlockSize(&min_grid_size, &block_size, reinterpret_cast(handle_), @@ -667,34 +719,28 @@ kernel_occupancy_config kernel_ref::max_occupancy_config(usize dynamic_shared_me dynamic_shared_memory_bytes, block_size_limit)); - return kernel_occupancy_config{.min_grid_size = min_grid_size, .block_size = block_size}; + return kernel_occupancy_config{.min_grid_size = static_cast(min_grid_size), + .block_size = static_cast(block_size)}; } -void kernel_ref::launch(u32 grid_dim_x, - u32 grid_dim_y, - u32 grid_dim_z, - u32 block_dim_x, - u32 block_dim_y, - u32 block_dim_z, - u32 shared_mem_bytes, +void kernel_ref::launch(cuda_dim3 grid_dim, + cuda_dim3 block_dim, + std::uint32_t shared_mem_bytes, CUstream stream, void** kernel_params) const { - RTCX_EXPECTS(grid_dim_x > 0 && grid_dim_y > 0 && grid_dim_z > 0, - "Grid dimensions must be greater than zero", - std::logic_error); - RTCX_EXPECTS(block_dim_x > 0 && block_dim_y > 0 && block_dim_z > 0, - "Block dimensions must be greater than zero", - std::logic_error); + RTCX_EXPECTS(grid_dim.is_valid(), "Grid dimensions must be greater than zero", std::logic_error); + RTCX_EXPECTS( + block_dim.is_valid(), "Block dimensions must be greater than zero", std::logic_error); RTCX_EXPECTS( kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); - CUlaunchConfig cfg{.gridDimX = grid_dim_x, - .gridDimY = grid_dim_y, - .gridDimZ = grid_dim_z, - .blockDimX = block_dim_x, - .blockDimY = block_dim_y, - .blockDimZ = block_dim_z, + CUlaunchConfig cfg{.gridDimX = grid_dim.x, + .gridDimY = grid_dim.y, + .gridDimZ = grid_dim.z, + .blockDimX = block_dim.x, + .blockDimY = block_dim.y, + .blockDimZ = block_dim.z, .sharedMemBytes = shared_mem_bytes, .hStream = stream, .attrs = nullptr, @@ -704,6 +750,30 @@ void kernel_ref::launch(u32 grid_dim_x, cuda->LaunchKernelEx(&cfg, reinterpret_cast(handle_), kernel_params, nullptr)); } +void kernel_ref::launch_cooperative(cuda_dim3 grid_dim, + cuda_dim3 block_dim, + std::uint32_t shared_mem_bytes, + CUstream stream, + void** kernel_params) const +{ + RTCX_EXPECTS(grid_dim.is_valid(), "Grid dimensions must be greater than zero", std::logic_error); + RTCX_EXPECTS( + block_dim.is_valid(), "Block dimensions must be greater than zero", std::logic_error); + RTCX_EXPECTS( + kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); + + RTCX_CHECK_CUDA(cuda->LaunchCooperativeKernel(reinterpret_cast(handle_), + grid_dim.x, + grid_dim.y, + grid_dim.z, + block_dim.x, + block_dim.y, + block_dim.z, + shared_mem_bytes, + stream, + kernel_params)); +} + std::string_view kernel_ref::get_name() const { char const* name; @@ -718,7 +788,7 @@ library_t::~library_t() } } -library load_library(std::span binary, binary_type type) +library load_library(std::span binary) { CUlibrary handle; @@ -736,7 +806,7 @@ library load_library(std::span binary, binary_type type) return library; } -std::vector link_library(link_params const& params) +std::vector link_library(link_params const& params) { RTCX_EXPECTS(params.name != nullptr, "Link output name must not be null", std::logic_error); RTCX_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, @@ -755,55 +825,44 @@ std::vector link_library(link_params const& params) } nvJitLinkHandle handle = nullptr; - - RTCX_CHECK_NVJITLINK(params, - handle, - nvjitlink->Create(&handle, - static_cast(params.link_options.size()), + RTCX_CHECK_NVJITLINK(nvjitlink->Create(&handle, + static_cast(params.link_options.size()), const_cast(params.link_options.data()))); RTCX_DEFER([&] { nvjitlink->Destroy(&handle); }); - for (usize i = 0; i < params.fragments.size(); i++) { - auto name = params.fragment_names[i]; - auto fragment = params.fragments[i]; - auto bin_type = params.fragment_binary_types[i]; - nvJitLinkInputType nv_type = to_nvjitlink_input_type(bin_type); - + for (std::size_t i = 0; i < params.fragments.size(); i++) { + auto name = params.fragment_names[i]; + auto fragment = params.fragments[i]; + auto bin_type = to_nvjitlink_input_type(params.fragment_binary_types[i]); RTCX_CHECK_NVJITLINK( - params, - handle, - nvjitlink->AddData(handle, nv_type, fragment.data(), fragment.size_bytes(), name)); + nvjitlink->AddData(handle, bin_type, fragment.data(), fragment.size_bytes(), name)); } - RTCX_CHECK_NVJITLINK(params, handle, nvjitlink->Complete(handle)); + auto link_result = nvjitlink->Complete(handle); + log_nvJitLink_result(params, handle, link_result); + RTCX_CHECK_NVJITLINK(link_result); switch (params.output_type) { case binary_type::CUBIN: { - usize cubin_size; - RTCX_CHECK_NVJITLINK(params, handle, nvjitlink->GetLinkedCubinSize(handle, &cubin_size)); - std::vector cubin; + std::size_t cubin_size; + RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedCubinSize(handle, &cubin_size)); + std::vector cubin; cubin.resize(cubin_size); - RTCX_CHECK_NVJITLINK(params, handle, nvjitlink->GetLinkedCubin(handle, cubin.data())); + RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedCubin(handle, cubin.data())); return cubin; } break; - case binary_type::PTX: { - usize ptx_size; - - RTCX_CHECK_NVJITLINK(params, handle, nvjitlink->GetLinkedPtxSize(handle, &ptx_size)); - std::vector ptx; + std::size_t ptx_size; + RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedPtxSize(handle, &ptx_size)); + std::vector ptx; ptx.resize(ptx_size); - - RTCX_CHECK_NVJITLINK( - params, handle, nvjitlink->GetLinkedPtx(handle, reinterpret_cast(ptx.data()))); - + RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedPtx(handle, reinterpret_cast(ptx.data()))); return ptx; } break; - default: RTCX_FAIL(std::format("Unsupported output binary type for linking CUDA libraries: ({})", - static_cast(params.output_type)), + binary_type_string(params.output_type)), std::runtime_error); } } @@ -817,7 +876,7 @@ kernel_ref library_t::get_kernel(char const* name) const std::vector library_t::enumerate_kernels() const { - u32 num_kernels; + std::uint32_t num_kernels; RTCX_CHECK_CUDA(cuda->LibraryGetKernelCount(&num_kernels, handle_)); std::vector kernels; @@ -835,9 +894,8 @@ std::vector library_t::enumerate_kernels() const std::string demangle_cuda_symbol(char const* mangled_name) { - i32 status; - usize length; - + std::int32_t status; + std::size_t length; char* demangled_name = abi::__cxa_demangle(mangled_name, nullptr, &length, &status); RTCX_EXPECTS(status == 0, "Demangling CUDA symbol name failed", std::runtime_error); @@ -864,20 +922,23 @@ namespace { } // namespace -cache_t::cache_t(std::string cache_dir, cache_limits const& limits) - : cache_dir_{std::move(cache_dir)}, +cache_t::cache_t(std::string cache_dir, cache_limits const& limits, bool preload, bool disable) + : enabled_{!disable}, + cache_dir_{std::move(cache_dir)}, limits_{limits}, + lock_{}, blobs_cache_{limits.num_mem_blobs}, libraries_cache_{limits.num_mem_libraries}, tick_{0} { + if (preload) { preload_from_disk(); } } std::string const& cache_t::get_cache_dir() { return cache_dir_; } std::optional blob_t::from_file(char const* path) { - i32 fd = open(path, O_RDONLY); + std::int32_t fd = open(path, O_RDONLY); if (fd == -1) { if (errno == ENOENT) { @@ -898,20 +959,20 @@ std::optional blob_t::from_file(char const* path) throw_posix("Failed to close RTCX cache file after memory-mapping", "close"); } - auto deleter = +[](u8 const* buffer, usize size) { - if (munmap(static_cast(const_cast(buffer)), size) == -1) { + auto deleter = +[](std::uint8_t const* buffer, std::size_t size) { + if (munmap(static_cast(const_cast(buffer)), size) == -1) { throw_posix("Failed to unmap RTCX cache file from memory", "munmap"); } }; - return blob_t::from_parts(static_cast(map), file_size, deleter); + return blob_t::from_parts(static_cast(map), file_size, deleter); } namespace { /// @brief retrieves a blob from disk based on the given sha256 hash and object type (e.g. "blob", -/// "fragment", "library"). Returns nullopt if the file doesn't exist on disk, and throws if any -/// other error occurs. +/// "cuLibrary"). Returns nullopt if the file doesn't exist on disk, and throws if any other error +/// occurs. std::optional get_disk_blob(std::string const& cache_dir, std::string const& object_type, sha256 const& sha) @@ -922,14 +983,13 @@ std::optional get_disk_blob(std::string const& cache_dir, auto blob = blob_t::from_file(path.c_str()); if (!blob.has_value()) { return std::nullopt; } - { - return std::make_shared(std::move(*blob)); - } + return std::make_shared(std::move(*blob)); } -void evict_disk_entries(std::string const& cache_dir, u32 limit) +std::pair, std::vector> get_disk_entries( + std::string const& cache_dir) { - i32 dir = open(cache_dir.c_str(), O_RDONLY | O_DIRECTORY); + std::int32_t dir = open(cache_dir.c_str(), O_RDONLY | O_DIRECTORY); if (dir == -1) { throw_posix("Failed to open RTCX cache directory for evicting", "open"); } @@ -941,10 +1001,10 @@ void evict_disk_entries(std::string const& cache_dir, u32 limit) std::vector paths; std::vector access_times; - isize num_read = 0; + std::ptrdiff_t num_read = 0; while ((num_read = syscall(SYS_getdents64, dir, buffer.data(), buffer.size())) > 0) { - isize byte_pos = 0; + std::ptrdiff_t byte_pos = 0; while (byte_pos < num_read) { auto* ent = reinterpret_cast(buffer.data() + byte_pos); @@ -977,14 +1037,21 @@ void evict_disk_entries(std::string const& cache_dir, u32 limit) throw_posix("Failed to read RTCX cache directory for clearing", "getdents64"); } + return {std::move(paths), std::move(access_times)}; +} + +void evict_disk_entries(std::string const& cache_dir, std::uint32_t limit) +{ + auto [paths, access_times] = get_disk_entries(cache_dir); + if (paths.size() < limit) { return; } - std::vector ranking_indices; + std::vector ranking_indices; ranking_indices.resize(paths.size()); std::iota(ranking_indices.begin(), ranking_indices.end(), 0); - std::sort(ranking_indices.begin(), ranking_indices.end(), [&](i32 a, i32 b) { + std::sort(ranking_indices.begin(), ranking_indices.end(), [&](std::int32_t a, std::int32_t b) { return access_times[a] < access_times[b]; }); @@ -1003,14 +1070,14 @@ void evict_disk_entries(std::string const& cache_dir, u32 limit) void cache_blob_to_disk(std::string const& cache_dir, std::string const& object_type, sha256 const& sha, - std::span binary, - u32 limit) + std::span binary, + std::uint32_t limit) { if (limit > 0) { char temp_path[] = "/tmp/rtcx-bin-XXXXXX"; { - i32 fd = mkstemp(temp_path); + std::int32_t fd = mkstemp(temp_path); if (fd == -1) { throw_posix("Failed to create temporary file for RTCX cache", "mkstemp"); } RTCX_DEFER([&] { @@ -1061,7 +1128,8 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil }); // check memory cache - if (auto it = blobs_cache_.entries_.find(sha); it != blobs_cache_.entries_.end()) { + if (auto it = enabled_ ? blobs_cache_.entries_.find(sha) : blobs_cache_.entries_.end(); + it != blobs_cache_.entries_.end()) { counter_.blob_mem_hits.incr(); // update LRU tick @@ -1073,7 +1141,8 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil counter_.blob_mem_misses.incr(); // check disk cache - auto disk_blob = get_disk_blob(cache_dir_, "blob", sha); + std::optional disk_blob = std::nullopt; + if (enabled_) { disk_blob = get_disk_blob(cache_dir_, "blob", sha); } std::promise promise; auto fut = promise.get_future().share(); @@ -1112,7 +1181,6 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil } std::shared_future cache_t::get_or_add_library(sha256 const& sha, - binary_type type, library_compile_func compile) { std::atomic_ref tick{tick_}; @@ -1126,7 +1194,8 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, }); // check memory cache - if (auto it = libraries_cache_.entries_.find(sha); it != libraries_cache_.entries_.end()) { + if (auto it = enabled_ ? libraries_cache_.entries_.find(sha) : libraries_cache_.entries_.end(); + it != libraries_cache_.entries_.end()) { counter_.library_mem_hits.incr(); // update LRU tick @@ -1138,7 +1207,8 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, counter_.library_mem_misses.incr(); // check disk cache - auto disk_blob = get_disk_blob(cache_dir_, "library", sha); + std::optional disk_blob = std::nullopt; + if (enabled_) { disk_blob = get_disk_blob(cache_dir_, "cuLibrary", sha); } std::promise promise; auto fut = promise.get_future().share(); @@ -1155,7 +1225,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, lock_.unlock(); unlocked = true; - auto lib = load_library((*disk_blob)->view(), type); + auto lib = load_library((*disk_blob)->view()); promise.set_value(std::move(lib)); return ret_fut; @@ -1174,7 +1244,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, promise.set_value(library); // store result to disk - cache_blob_to_disk(cache_dir_, "library", sha, blob->view(), limits_.num_disk_entries); + cache_blob_to_disk(cache_dir_, "cuLibrary", sha, blob->view(), limits_.num_disk_entries); return ret_fut; } @@ -1207,13 +1277,13 @@ void cache_t::clear_stats() cache_limits cache_t::get_limits() { return limits_; } -usize cache_t::get_blob_count() +std::size_t cache_t::get_blob_count() { std::lock_guard guard{lock_}; return blobs_cache_.entries_.size(); } -usize cache_t::get_library_count() +std::size_t cache_t::get_library_count() { std::lock_guard guard{lock_}; return libraries_cache_.entries_.size(); @@ -1227,48 +1297,103 @@ void cache_t::clear_memory_store() libraries_cache_.entries_.clear(); } -void cache_t::clear_disk_store() +void cache_t::clear_disk_store() { evict_disk_entries(cache_dir_, 0); } + +void cache_t::preload_from_disk() { - i32 dir = open(cache_dir_.c_str(), O_RDONLY | O_DIRECTORY); + auto [paths, _] = get_disk_entries(cache_dir_); - if (dir == -1) { throw_posix("Failed to open RTCX cache directory for clearing", "opendir"); } + // TODO(lamarrr): we have promises that may not be fulfilled if errors occur, they need to be handled? + // i.e. if the exceptions is caught - RTCX_DEFER([&] { close(dir); }); + { + std::lock_guard guard{lock_}; + tick_++; + + for (auto& path : paths) { + try { + auto file_name = std::filesystem::path{path}.filename().string(); + auto sha_str_end = file_name.find('.'); + auto sha_str = file_name.substr(0, sha_str_end); + auto data = blob_t::from_file(path.c_str()); + if (!data.has_value()) { continue; } + + if (path.ends_with(".blob.bin")) { + std::promise promise; + auto fut = promise.get_future().share(); + promise.set_value(std::make_shared(std::move(*data))); + blobs_cache_.insert(sha256::parse(sha_str), std::move(fut), tick_); + } else if (path.ends_with(".cuLibrary.bin")) { + std::promise promise; + auto fut = promise.get_future().share(); + promise.set_value(load_library(data->view())); + libraries_cache_.insert(sha256::parse(sha_str), std::move(fut), tick_); + } + } catch (std::exception const& e) { + // ignore any errors during preload + log_trace(e.what()); + } catch (...) { + log_trace("Unknown error during preload"); + } + } + } +} - std::vector buffer; - buffer.resize(8192); - std::vector entry_path; - entry_path.resize(4096); +void cache_t::enable(bool enable) +{ + std::lock_guard guard{lock_}; + enabled_ = enable; +} - isize num_read = 0; +bool cache_t::is_enabled() +{ + std::lock_guard guard{lock_}; + return enabled_; +} - while ((num_read = syscall(SYS_getdents64, dir, buffer.data(), buffer.size())) > 0) { - isize byte_pos = 0; +std::string reflect_bool(bool value) { return std::format("(bool){}", value); } - while (byte_pos < num_read) { - auto* ent = reinterpret_cast(buffer.data() + byte_pos); +std::string reflect_int(std::uint8_t value) { return std::format("(unsigned char){}U", value); } - if (memcmp(ent->d_name, ".", 2) != 0 && memcmp(ent->d_name, "..", 3) != 0) { - RTCX_EXPECTS(ent->d_type != DT_UNKNOWN, - "Found unknown directory entry type in RTCX cache dir", - std::runtime_error); +std::string reflect_int(std::uint16_t value) { return std::format("(unsigned short){}U", value); } - if (ent->d_type == DT_REG) { - snprintf(entry_path.data(), entry_path.size(), "%s/%s", cache_dir_.c_str(), ent->d_name); +std::string reflect_int(std::uint32_t value) { return std::format("(unsigned int){}U", value); } - if (unlink(entry_path.data()) == -1 && errno != ENOENT) { - throw_posix("Failed to unlink RTCX cache file during clearing", "unlink"); - } - } - } +std::string reflect_int(std::uint64_t value) +{ + return std::format("(unsigned long long int){}ULL", value); +} - byte_pos += ent->d_reclen; - } - } +std::string reflect_int(std::int8_t value) { return std::format("(signed char){}", value); } - if (num_read == -1) { - throw_posix("Failed to read RTCX cache directory for clearing", "getdents64"); - } +std::string reflect_int(std::int16_t value) { return std::format("(signed short){}", value); } + +std::string reflect_int(std::int32_t value) { return std::format("(signed int){}", value); } + +std::string reflect_int(std::int64_t value) +{ + return std::format("(signed long long int){}LL", value); +} + +std::string reflect_float(float value) { return std::format("(float){}F", value); } + +std::string reflect_float(double value) { return std::format("(double){}", value); } + +std::string reflect_cast(std::string_view type, std::string_view value) +{ + return std::format("(({})({}))", type, value); +} + +std::string reflect_template(std::string_view template_name, + std::span template_args) +{ + return std::format("{}<{}>", template_name, join_strings(template_args, ", ")); +} + +std::string reflect_template(std::string_view template_name, + std::span template_args) +{ + return std::format("{}<{}>", template_name, join_strings(template_args, ", ")); } } // namespace RTCX_EXPORT rtcx diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 7a9c3e59c5a7..da84e9734c64 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -37,20 +36,7 @@ typedef struct CUstream_st* CUstream; namespace RTCX_EXPORT rtcx { -using u8 = std::uint8_t; -using u16 = std::uint16_t; -using u32 = std::uint32_t; -using u64 = std::uint64_t; -using usize = std::size_t; -using i8 = std::int8_t; -using i16 = std::int16_t; -using i32 = std::int32_t; -using i64 = std::int64_t; -using isize = std::ptrdiff_t; -using f32 = float; -using f64 = double; - -inline constexpr usize CACHELINE_ALIGNMENT = +inline constexpr std::size_t CACHELINE_ALIGNMENT = 64; // = std::hardware_destructive_interference_size */ /** @@ -131,13 +117,13 @@ struct [[nodiscard]] sha256_hex_string { [[nodiscard]] char const* c_str() const { return data_; } - static constexpr usize size() { return 64; } + static constexpr std::size_t size() { return 64; } - static sha256_hex_string make(std::span input) + static sha256_hex_string make(std::span input) { constexpr char const HEX_CHARS[] = "0123456789abcdef"; sha256_hex_string hex; - for (usize i = 0; i < 32; ++i) { + for (std::size_t i = 0; i < 32; ++i) { hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; } @@ -147,23 +133,20 @@ struct [[nodiscard]] sha256_hex_string { }; struct [[nodiscard]] sha256 { - alignas(16) u8 data_[32]; + alignas(16) std::uint8_t data_[32]; - constexpr bool operator==(sha256 const& hash) const - { - return std::equal(std::begin(data_), std::end(data_), std::begin(hash.data_)); - } - - constexpr bool operator!=(sha256 const& hash) const { return !(*this == hash); } + constexpr bool operator==(sha256 const&) const = default; sha256_hex_string to_hex_string() const { return sha256_hex_string::make(data_); } + + static sha256 parse(std::string_view hex); }; struct [[nodiscard]] sha256_hasher { - constexpr u64 operator()(sha256 const& obj) const + constexpr std::uint64_t operator()(sha256 const& obj) const { struct u64x4 { - alignas(16) u64 v[4]; + alignas(16) std::uint64_t v[4]; }; auto value = std::bit_cast(obj); @@ -172,7 +155,7 @@ struct [[nodiscard]] sha256_hasher { auto const h2 = value.v[2]; auto const h3 = value.v[3]; - auto mix = [](u64 seed, u64 v) { + auto mix = [](std::uint64_t seed, std::uint64_t v) { seed ^= v + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); return seed; }; @@ -201,12 +184,12 @@ struct [[nodiscard]] sha256_context { ~sha256_context(); - void update(std::span data); + void update(std::span data); sha256 finalize(); }; -enum class binary_type : i8 { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; +enum class binary_type : std::int8_t { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; /** * @brief Represents a binary blob @@ -216,15 +199,15 @@ enum class binary_type : i8 { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; */ struct [[nodiscard]] blob_t { private: - using deallocator = func; + using deallocator = func; - static void noop_deallocator(u8 const*, usize) {} + static void noop_deallocator(std::uint8_t const*, std::size_t) {} - u8 const* data_; - usize size_; + std::uint8_t const* data_; + std::size_t size_; deallocator deallocator_; - blob_t(u8 const* data, usize size, deallocator deallocator) + blob_t(std::uint8_t const* data, std::size_t size, deallocator deallocator) : data_(data), size_(size), deallocator_(deallocator) { } @@ -253,25 +236,43 @@ struct [[nodiscard]] blob_t { ~blob_t() { deallocator_(data_, size_); } - [[nodiscard]] std::span view() const { return {data_, size_}; } + [[nodiscard]] std::span view() const { return {data_, size_}; } - static blob_t from_parts(u8 const* data, usize size, deallocator deallocator) + static blob_t from_parts(std::uint8_t const* data, std::size_t size, deallocator deallocator) { return blob_t{data, size, deallocator}; } - static blob_t from_vector(std::vector&& data); + static blob_t from_vector(std::vector&& data); - static blob_t from_static_data(std::span data); + static blob_t from_static_data(std::span data); static std::optional from_file(char const* path); }; using blob = std::shared_ptr; +/** + * @brief Represents the occupancy configuration for a kernel, including the minimum grid size and + * the block size required to achieve that occupancy. This information can be used to optimize + * kernel launches for maximum performance on the GPU. + */ struct [[nodiscard]] kernel_occupancy_config { - i32 min_grid_size = 0; - i32 block_size = 0; + std::uint32_t min_grid_size = 0; //< Minimum grid size to achieve the maximum occupancy + std::uint32_t block_size = + 0; //< Number of threads per block to achieve the min_grid_size occupancy +}; + +/** + * @brief Represents the dimensions of a CUDA grid or block, with x, y, and z components. This + * struct is used to specify the configuration of kernel launches on the GPU. + */ +struct cuda_dim3 { + std::uint32_t x = 1; //< Value for the x dimension + std::uint32_t y = 1; //< Value for the y dimension + std::uint32_t z = 1; //< Value for the z dimension + + [[nodiscard]] constexpr bool is_valid() const { return x > 0 && y > 0 && z > 0; } }; /** @@ -289,36 +290,52 @@ struct [[nodiscard]] kernel_ref { * shared memory usage and block size limit. This function queries the CUDA driver for the optimal * block size and minimum grid size to achieve maximum occupancy of the kernel on the GPU. */ - kernel_occupancy_config max_occupancy_config(usize dynamic_shared_memory_bytes, - i32 block_size_limit) const; + kernel_occupancy_config max_occupancy_config(std::size_t dynamic_shared_memory_bytes, + std::int32_t block_size_limit) const; /** * @brief Launches the kernel on the GPU with the specified grid and block dimensions, * dynamic shared memory size, stream, and kernel parameters. This function wraps the CUDA driver * kernel launch API, providing a convenient interface for executing the kernel with the desired * configuration. - * @param grid_dim_x The number of blocks in the x-dimension of the grid - * @param grid_dim_y The number of blocks in the y-dimension of the grid - * @param grid_dim_z The number of blocks in the z-dimension of the grid - * @param block_dim_x The number of threads in the x-dimension of each block - * @param block_dim_y The number of threads in the y-dimension of each block - * @param block_dim_z The number of threads in the z-dimension of each block + * @param grid_dim The dimensions of the grid + * @param block_dim The dimensions of the block * @param shared_mem_bytes The amount of dynamic shared memory (in bytes) to allocate for the * kernel * @param stream The CUDA stream on which to launch the kernel * @param kernel_params A pointer to an array of pointers representing the kernel parameters to be * passed to the kernel at launch time */ - void launch(u32 grid_dim_x, - u32 grid_dim_y, - u32 grid_dim_z, - u32 block_dim_x, - u32 block_dim_y, - u32 block_dim_z, - u32 shared_mem_bytes, + void launch(cuda_dim3 grid_dim, + cuda_dim3 block_dim, + std::uint32_t shared_mem_bytes, CUstream stream, void** kernel_params) const; + /** + * @brief Launches the kernel on the GPU in cooperative mode with the specified grid and block + * dimensions, dynamic shared memory size, stream, and kernel parameters. This function wraps the + * CUDA driver cooperative kernel launch API, providing a convenient interface for executing the + * kernel with the desired configuration. + * @param grid_dim The dimensions of the grid + * @param block_dim The dimensions of the block + * @param shared_mem_bytes The amount of dynamic shared memory (in bytes) to allocate for the + * kernel + * @param stream The CUDA stream on which to launch the kernel + * @param kernel_params A pointer to an array of pointers representing the kernel parameters to be + * passed to the kernel at launch time + */ + void launch_cooperative(cuda_dim3 grid_dim, + cuda_dim3 block_dim, + std::uint32_t shared_mem_bytes, + CUstream stream, + void** kernel_params) const; + + /** + * @brief Retrieves the underlying CUDA kernel handle + */ + [[nodiscard]] CUkernel get() const { return handle_; } + /** * @brief Retrieves the name of the kernel */ @@ -340,6 +357,11 @@ struct [[nodiscard]] library_t { library_t& operator=(library_t&&) = delete; ~library_t(); + /** + * @brief Retrieves the underlying CUDA library handle + */ + [[nodiscard]] CUlibrary get() const { return handle_; } + /** * @brief Retrieve a kernel from the library by name */ @@ -394,27 +416,29 @@ struct [[nodiscard]] compile_params { }; struct [[nodiscard]] link_params { - char const* name = nullptr; - binary_type output_type = binary_type::CUBIN; - std::span const> fragments = {}; - std::span fragment_binary_types = {}; - std::span fragment_names = {}; - std::span link_options = {}; + char const* name = nullptr; + binary_type output_type = binary_type::CUBIN; + std::span const> fragments = {}; + std::span fragment_binary_types = {}; + std::span fragment_names = {}; + std::span link_options = {}; }; +namespace detail { + template struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { struct entry { - u64 last_touched_tick = 0; + std::uint64_t last_touched_tick = 0; T value; - void hit(u64 tick) { last_touched_tick = tick; } + void hit(std::uint64_t tick) { last_touched_tick = tick; } }; std::unordered_map entries_; - usize limit_; + std::size_t limit_; - explicit lru_memory_cache(usize limit) : entries_{}, limit_{limit} + explicit lru_memory_cache(std::size_t limit) : entries_{}, limit_{limit} { // reserve space to avoid rehashing entries_.reserve(limit * 2); @@ -426,7 +450,7 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { auto num_to_purge = (entries_.size() + 1) / 2; - std::vector> rankings; + std::vector> rankings; rankings.reserve(entries_.size()); for (auto const& [key, entry] : entries_) { @@ -445,7 +469,7 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { } } - void insert(sha256 const& sha, T&& value, u64 tick) + void insert(sha256 const& sha, T&& value, std::uint64_t tick) { if (limit_ == 0) { return; } @@ -455,26 +479,9 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { } }; -struct [[nodiscard]] cache_stats { - u64 blob_mem_hits = 0; - u64 blob_mem_misses = 0; - u64 blob_disk_hits = 0; - u64 blob_disk_misses = 0; - u64 library_mem_hits = 0; - u64 library_mem_misses = 0; - u64 library_disk_hits = 0; - u64 library_disk_misses = 0; -}; - -struct [[nodiscard]] cache_limits { - u32 num_mem_blobs = 16'384; - u32 num_mem_libraries = 16'384; - u32 num_disk_entries = 131'072; -}; - struct cache_stats_counter { - struct alignas(CACHELINE_ALIGNMENT) counter { - u64 value_ = 0; + struct alignas(CACHELINE_ALIGNMENT) entry { + std::uint64_t value_ = 0; void incr() { @@ -482,7 +489,7 @@ struct cache_stats_counter { c.fetch_add(1, std::memory_order_relaxed); } - [[nodiscard]] u64 get() const + [[nodiscard]] std::uint64_t get() const { std::atomic_ref c{value_}; return c.load(std::memory_order_relaxed); @@ -495,14 +502,33 @@ struct cache_stats_counter { } }; - counter blob_mem_hits; - counter blob_mem_misses; - counter blob_disk_hits; - counter blob_disk_misses; - counter library_mem_hits; - counter library_mem_misses; - counter library_disk_hits; - counter library_disk_misses; + entry blob_mem_hits; + entry blob_mem_misses; + entry blob_disk_hits; + entry blob_disk_misses; + entry library_mem_hits; + entry library_mem_misses; + entry library_disk_hits; + entry library_disk_misses; +}; + +}; // namespace detail + +struct [[nodiscard]] cache_stats { + std::uint64_t blob_mem_hits = 0; + std::uint64_t blob_mem_misses = 0; + std::uint64_t blob_disk_hits = 0; + std::uint64_t blob_disk_misses = 0; + std::uint64_t library_mem_hits = 0; + std::uint64_t library_mem_misses = 0; + std::uint64_t library_disk_hits = 0; + std::uint64_t library_disk_misses = 0; +}; + +struct [[nodiscard]] cache_limits { + std::uint32_t num_mem_blobs = 16'384; + std::uint32_t num_mem_libraries = 16'384; + std::uint32_t num_disk_entries = 131'072; }; using blob_compile_func = func; @@ -523,22 +549,36 @@ using library_compile_func = func()>; */ struct cache_t { private: + bool enabled_; + std::string cache_dir_; cache_limits limits_; std::mutex lock_; - lru_memory_cache> blobs_cache_; + detail::lru_memory_cache> blobs_cache_; - lru_memory_cache> libraries_cache_; + detail::lru_memory_cache> libraries_cache_; - cache_stats_counter counter_; + detail::cache_stats_counter counter_; - alignas(CACHELINE_ALIGNMENT) u64 tick_; + alignas(CACHELINE_ALIGNMENT) std::uint64_t tick_; public: - cache_t(std::string cache_dir, cache_limits const& limits); + /** + * @brief Construct a new cache_t object with the specified cache directory, limits, and options + * for preloading and enabling the cache. + * @param cache_dir The directory path to be used for on-disk caching of compiled blobs and + * libraries + * @param limits A cache_limits struct specifying the maximum number of blobs and libraries to + * store in the cache before eviction occurs + * @param preload A boolean flag indicating whether to preload the cache from disk during + * initialization, allowing for faster retrieval of previously compiled kernels at runtime + * @param disable A boolean flag indicating whether to disable the cache entirely, preventing any + * caching of compiled blobs and libraries in memory + */ + cache_t(std::string cache_dir, cache_limits const& limits, bool preload, bool disable); cache_t(cache_t const&) = delete; cache_t& operator=(cache_t const&) = delete; cache_t(cache_t&&) = delete; @@ -569,7 +609,6 @@ struct cache_t { * @return A shared future that will hold the compiled library once it's available */ [[nodiscard]] std::shared_future get_or_add_library(sha256 const& sha, - binary_type type, library_compile_func compile); /** @@ -599,14 +638,14 @@ struct cache_t { * * @return The number of blobs currently stored in the in-memory cache */ - [[nodiscard]] usize get_blob_count(); + [[nodiscard]] std::size_t get_blob_count(); /** * @brief Get the current number of libraries stored in the in-memory cache * * @return The number of libraries currently stored in the in-memory cache */ - [[nodiscard]] usize get_library_count(); + [[nodiscard]] std::size_t get_library_count(); /** * @brief Clear all entries from the in-memory cache, removing all cached blobs and libraries @@ -625,6 +664,25 @@ struct cache_t { * items in the future. */ void clear_disk_store(); + + /*** + * @brief Pre-load the JIT program cache from disk into memory during initialization, allowing for + * faster retrieval of previously compiled kernels at runtime. + */ + void preload_from_disk(); + + /*** + * @brief Enable the cache, allowing it to store and retrieve compiled blobs and libraries in + * memory. + * @param enabled A boolean flag indicating whether to enable (true) or disable (false) the cache. + */ + void enable(bool enabled); + + /** + * @brief Get whether the cache is currently enabled or disabled. + * @return A boolean value indicating whether the cache is currently enabled (true) or disabled + */ + [[nodiscard]] bool is_enabled(); }; /** @@ -634,7 +692,7 @@ struct cache_t { * type * @return A vector of bytes containing the compiled binary blob */ -[[nodiscard]] std::vector compile(compile_params const& params); +[[nodiscard]] std::vector compile(compile_params const& params); /** * @brief Load a compiled library from binary data @@ -643,7 +701,7 @@ struct cache_t { * @param type Binary type of the library (e.g., CUBIN, PTX) * @return A library object representing the loaded library with launchable kernels */ -[[nodiscard]] library load_library(std::span binary, binary_type type); +[[nodiscard]] library load_library(std::span binary); /** * @brief Link multiple compiled binary fragments into a single binary blob containing the linked @@ -653,7 +711,7 @@ struct cache_t { * binary type * @return A vector of bytes containing the linked library binary */ -[[nodiscard]] std::vector link_library(link_params const& params); +[[nodiscard]] std::vector link_library(link_params const& params); /** * @brief Demangle a CUDA symbol name into a human-readable form @@ -688,267 +746,159 @@ void initialize(); */ void teardown(); -namespace reflection { - -template -struct NonType {}; - -// Forward declaration. -template -inline std::string reflect(const T& value); -template -inline std::string reflect(); - -namespace detail { - -template -struct ValueStringImpl { - static std::string value(const T& x) { return std::to_string(x); } -}; - -template -struct ValueStringImpl::value>::type> { - static std::string value(const T& x) { return x ? "true" : "false"; } -}; - -template -struct ValueStringImpl::value>::type> { - static std::string value(const T& x) - { - using UnderlyingT = typename std::underlying_type::type; - return ValueStringImpl::value(static_cast(x)); - } -}; - -template -inline std::string value_string(const T& x) -{ - return ValueStringImpl::value(x); -} - -// Returns the demangled name corresponding to the given typeinfo structure. -inline std::string get_type_name(const std::type_info& typeinfo) -{ - const char* mangled_name = typeinfo.name(); - size_t bufsize = 0; - char* buf = nullptr; - int status; - auto demangled_ptr = std::unique_ptr( - abi::__cxa_demangle(mangled_name, buf, &bufsize, &status), std::free); - // clang-format off - switch (status) { - case 0: return demangled_ptr.get(); // Demangled successfully - case -2: return mangled_name; // Interpret as plain unmangled name - case -1: // fall-through // Memory allocation failure - case -3: // fall-through // Invalid argument - default: return {}; - } - // clang-format on -} - -template -class JitifyTypeNameWrapper_ {}; - -// Returns the demangled name of the given type. -template -inline std::string get_type_name() -{ - // WAR for typeid discarding cv qualifiers on value-types. - // Wraps type in dummy template class to preserve cv-qualifiers, then strips - // off the wrapper from the resulting string. - std::string wrapped_name = get_type_name(typeid(JitifyTypeNameWrapper_)); - // Note: The reflected name of this class also has namespace prefixes. - const std::string wrapper_class_name = "JitifyTypeNameWrapper_<"; - size_t start = wrapped_name.find(wrapper_class_name); - if (start == std::string::npos) return {}; // Unexpected error - start += wrapper_class_name.size(); - return wrapped_name.substr(start, wrapped_name.size() - (start + 1)); -} - -template -struct ReflectType { - const std::string& operator()() const - { - // Storing this statically means it is cached after the first call. - static const std::string type_name = get_type_name(); - return type_name; - } -}; - -template -struct ReflectType> { - std::string operator()() const { return reflect(VALUE); } -}; +/** + * @brief Reflect a boolean value into its CUDA string representation ("true" or "false") + * @param value The boolean value to be reflected + * @return A string containing the CUDA representation of the boolean value ("true" or "false") + */ +std::string reflect_bool(bool value); -} // namespace detail +/** + * @brief Reflect an integer value into its CUDA string representation + * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value + */ +std::string reflect_int(std::uint8_t value); -/*! A wrapper used for representing types as values. */ -template -struct Type {}; +/** + * @brief Reflect an integer value into its CUDA string representation + * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value + */ +std::string reflect_int(std::uint16_t value); -/*! Create an Instance object that contains a const reference to the - * value. We use this to wrap abstract objects from which we want to extract - * their type at runtime (e.g., derived type). This is used to facilitate - * templating on derived type when all we know at compile time is abstract - * type. +/** + * @brief Reflect an integer value into its CUDA string representation + * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value */ -template -struct Instance { - const T& value; - Instance(const T& value_arg) : value(value_arg) {} -}; +std::string reflect_int(std::uint32_t value); -/*! Create an Instance object from which we can extract the value's run-time - * type. - * \param value The const value to be captured. +/** + * @brief Reflect an integer value into its CUDA string representation + * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value */ -template -inline Instance instance_of(T const& value) -{ - return Instance(value); -} +std::string reflect_int(std::uint64_t value); -/*! Generate a code-string for a type. - * \code{.cpp}reflect() --> "float"\endcode +/** + * @brief Reflect an integer value into its CUDA string representation + * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value */ -template -inline std::string reflect() -{ - return detail::ReflectType()(); -} +std::string reflect_int(std::int8_t value); -/*! Generate a code-string for a value. - * \code{.cpp}reflect(3.14f) --> "(float)3.14"\endcode +/** + * @brief Reflect an integer value into its CUDA string representation + * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value */ -template -inline std::string reflect(const T& value) -{ - return "(" + reflect() + ")" + detail::value_string(value); -} +std::string reflect_int(std::int16_t value); -/*! Generate a code-string for an integer non-type template argument - * (via implicit conversion to int64_t). - * \code{.cpp}reflect<7>() --> "(int64_t)7"\endcode +/** + * @brief Reflect an integer value into its CUDA string representation + * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value */ -template -inline std::string reflect() -{ - return reflect>(); -} +std::string reflect_int(std::int32_t value); -/*! Generate a code-string for a generic non-type template argument. - * \code{.cpp} reflect() --> "(int)7" \endcode +/** + * @brief Reflect an integer value into its CUDA string representation + * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value */ -template -inline std::string reflect() -{ - return reflect>(); -} +std::string reflect_int(std::int64_t value); -/*! Generate a code-string for a type wrapped as a Type instance. - * \code{.cpp}reflect(Type()) --> "float"\endcode +/** + * @brief Reflect a floating-point value into its CUDA string representation + * @tparam T A floating-point type (e.g., float, double) + * @param value The floating-point value to be reflected + * @return A string containing the CUDA representation of the floating-point value */ -template -inline std::string reflect(Type) -{ - return reflect(); -} +std::string reflect_float(float value); -/*! Generate a code-string for a type wrapped as an Instance instance. - * \code{.cpp}reflect(Instance(3.1f)) --> "float"\endcode - * or more simply when passed to a instance_of helper - * \code{.cpp}reflect(instance_of(3.1f)) --> "float"\endcodei - * This is specifically for the case where we want to extract the run-time - * type, i.e., derived type, of an object pointer. +/** + * @brief Reflect a floating-point value into its CUDA string representation + * @tparam T A floating-point type (e.g., float, double) + * @param value The floating-point value to be reflected + * @return A string containing the CUDA representation of the floating-point value */ -template -inline std::string reflect(const Instance& value) -{ - return detail::get_type_name(typeid(value.value)); -} +std::string reflect_float(double value); -// TODO: Would there ever be a need to reflect a string literal? -/*! Use an existing code string as-is. */ -inline std::string reflect(const std::string& s) { return s; } -/*! Use an existing code string as-is. */ -inline const char* reflect(const char* s) { return s; } -#if JITIFY_CPLUSPLUS >= 201703L -/*! Use an existing code string as-is. */ -inline std::string_view reflect(std::string_view s) { return s; } -#endif - -/*! Create a Type object representing a value's type. - * \code{.cpp}type_of(3.14f) -> Type()\endcode - * \param [unnamed] The value whose type is to be captured. +/** + * @brief Reflect a value of any type into its CUDA string representation, given the type name as a + * string + * @param type The name of the type to be reflected (e.g., "int", "float", "MyStruct", etc.) + * @param value The string representation of the value to be reflected, which will be used in the + * resulting CUDA code + * @return A string containing the CUDA representation of the value with the specified type */ -template -inline Type type_of(T&) -{ - return Type(); -} +std::string reflect_cast(std::string_view type, std::string_view value); -/*! Create a Type object representing a value's type. - * \param [unnamed] The const value whose type is to be captured. +/** + * @brief Reflect an enumeration value into its CUDA string representation, given the type name as a + * string + * @tparam T An enumeration type + * @param type The name of the enumeration type to be reflected (e.g., "MyEnum") + * @param value The enumeration value to be reflected, which will be cast to its underlying integer + * type and represented as a string in the resulting CUDA code + * @return A string containing the CUDA representation of the enumeration value with the specified + * type */ template -inline Type type_of(const T&) + requires(std::is_enum_v) +std::string reflect_enum(std::string_view type, T value) { - return Type(); + return reflect_cast(type, reflect_int(static_cast>(value))); } -/*! Generate a code-string for a template instantiation. */ -inline std::string reflect_template(const StringVec& args) -{ - // Note: The space in " >" is a WAR to avoid '>>' appearing - return jitify2::detail::string_join(args, ",", "<", " >"); -} +/** + * @brief Reflect a template instantiation into its CUDA string representation, given the template + * name and its template arguments as strings + * @param template_name The name of the template to be reflected (e.g., "MyTemplate") + * @param template_args A span of strings representing the template arguments to be reflected, which + * will be used in the resulting CUDA code + * @return A string containing the CUDA representation of the template instantiation with the + * specified template name and arguments + */ +std::string reflect_template(std::string_view template_name, + std::span template_args); -/*! Generate a code-string for a template instantiation. */ -template -inline std::string reflect_template() -{ - return reflect_template({reflect()...}); -} +/** + * @brief Reflect a template instantiation into its CUDA string representation, given the template + * name and its template arguments as strings + * @param template_name The name of the template to be reflected (e.g., "MyTemplate") + * @param template_args A span of strings representing the template arguments to be reflected, which + * will be used in the resulting CUDA code + * @return A string containing the CUDA representation of the template instantiation with the + * specified template name and arguments + */ +std::string reflect_template(std::string_view template_name, + std::span template_args); -/*! Generate a code-string for a template instantiation. */ -template -inline std::string reflect_template(const Args&... args) +/** + * @brief Reflect a template instantiation into its CUDA string representation, given the template + * name and its template arguments as strings + * @param template_name The name of the template to be reflected (e.g., "MyTemplate") + * @param template_args A span of strings representing the template arguments to be reflected, which + * will be used in the resulting CUDA code + * @return A string containing the CUDA representation of the template instantiation with the + * specified template name and arguments + */ +template + requires((true && ... && std::is_constructible_v)) +std::string reflect_template(std::string_view template_name, TemplateArgs&&... template_args) { - return reflect_template({reflect(args)...}); + std::string_view const tparams[sizeof...(TemplateArgs)] = {std::string_view{template_args}...}; + return reflect_template(template_name, tparams); } -/*! Convenience class for generating code-strings for template instantiations. - */ -class Template { - std::string name_; - - public: - /*! Construct the class. - * \param name The name of the template. - */ - Template(StringRef name) : name_(name) {} - - /*! Generate a code-string for an instantiation of the template. */ - std::string instantiate(const StringVec& template_args = {}) const - { - return name_ + reflect_template(template_args); - } - - /*! Generate a code-string for an instantiation of the template. */ - template - std::string instantiate() const - { - return name_ + reflect_template(); - } - - /*! Generate a code-string for an instantiation of the template. */ - template - std::string instantiate(const TemplateArgs&... targs) const - { - return name_ + reflect_template(targs...); - } -}; - -} // namespace reflection - } // namespace RTCX_EXPORT rtcx diff --git a/cpp/src/librtcx/system_includes/README.md b/cpp/src/librtcx/system_includes/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cpp/src/librtcx/system_includes/algorithm b/cpp/src/librtcx/system_includes/algorithm deleted file mode 100644 index 8cf34e5afc0a..000000000000 --- a/cpp/src/librtcx/system_includes/algorithm +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/array b/cpp/src/librtcx/system_includes/array deleted file mode 100644 index fe2d963aba07..000000000000 --- a/cpp/src/librtcx/system_includes/array +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/atomic b/cpp/src/librtcx/system_includes/atomic deleted file mode 100644 index 9f32c5729abb..000000000000 --- a/cpp/src/librtcx/system_includes/atomic +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/barrier b/cpp/src/librtcx/system_includes/barrier deleted file mode 100644 index 54dca7534e0f..000000000000 --- a/cpp/src/librtcx/system_includes/barrier +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/bit b/cpp/src/librtcx/system_includes/bit deleted file mode 100644 index e6aeda22d2b1..000000000000 --- a/cpp/src/librtcx/system_includes/bit +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/cassert b/cpp/src/librtcx/system_includes/cassert deleted file mode 100644 index c7d847aafdc4..000000000000 --- a/cpp/src/librtcx/system_includes/cassert +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -} diff --git a/cpp/src/librtcx/system_includes/cassert.h b/cpp/src/librtcx/system_includes/cassert.h deleted file mode 100644 index c7d847aafdc4..000000000000 --- a/cpp/src/librtcx/system_includes/cassert.h +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -} diff --git a/cpp/src/librtcx/system_includes/ccomplex b/cpp/src/librtcx/system_includes/ccomplex deleted file mode 100644 index a2eded0c8289..000000000000 --- a/cpp/src/librtcx/system_includes/ccomplex +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/ccomplex.h b/cpp/src/librtcx/system_includes/ccomplex.h deleted file mode 100644 index a2eded0c8289..000000000000 --- a/cpp/src/librtcx/system_includes/ccomplex.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/cfloat b/cpp/src/librtcx/system_includes/cfloat deleted file mode 100644 index 0c23d7ca72f4..000000000000 --- a/cpp/src/librtcx/system_includes/cfloat +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -} diff --git a/cpp/src/librtcx/system_includes/cfloat.h b/cpp/src/librtcx/system_includes/cfloat.h deleted file mode 100644 index 0c23d7ca72f4..000000000000 --- a/cpp/src/librtcx/system_includes/cfloat.h +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -} diff --git a/cpp/src/librtcx/system_includes/chrono b/cpp/src/librtcx/system_includes/chrono deleted file mode 100644 index 4625526096eb..000000000000 --- a/cpp/src/librtcx/system_includes/chrono +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/climits b/cpp/src/librtcx/system_includes/climits deleted file mode 100644 index e7ddbe78a70e..000000000000 --- a/cpp/src/librtcx/system_includes/climits +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -} diff --git a/cpp/src/librtcx/system_includes/cmath b/cpp/src/librtcx/system_includes/cmath deleted file mode 100644 index 97ed927ef2a9..000000000000 --- a/cpp/src/librtcx/system_includes/cmath +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/cmath.h b/cpp/src/librtcx/system_includes/cmath.h deleted file mode 100644 index 97ed927ef2a9..000000000000 --- a/cpp/src/librtcx/system_includes/cmath.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/complex b/cpp/src/librtcx/system_includes/complex deleted file mode 100644 index 530e7601adf6..000000000000 --- a/cpp/src/librtcx/system_includes/complex +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/concepts b/cpp/src/librtcx/system_includes/concepts deleted file mode 100644 index 41357406d9ef..000000000000 --- a/cpp/src/librtcx/system_includes/concepts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/cstddef b/cpp/src/librtcx/system_includes/cstddef deleted file mode 100644 index 71352d667749..000000000000 --- a/cpp/src/librtcx/system_includes/cstddef +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/cstdint b/cpp/src/librtcx/system_includes/cstdint deleted file mode 100644 index 03c8eb0a9dd1..000000000000 --- a/cpp/src/librtcx/system_includes/cstdint +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/cstdlib b/cpp/src/librtcx/system_includes/cstdlib deleted file mode 100644 index c216518c9c29..000000000000 --- a/cpp/src/librtcx/system_includes/cstdlib +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/cstring b/cpp/src/librtcx/system_includes/cstring deleted file mode 100644 index 83275541eb0a..000000000000 --- a/cpp/src/librtcx/system_includes/cstring +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/ctime b/cpp/src/librtcx/system_includes/ctime deleted file mode 100644 index 05e4eab7bd9f..000000000000 --- a/cpp/src/librtcx/system_includes/ctime +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/expected b/cpp/src/librtcx/system_includes/expected deleted file mode 100644 index eea942704ceb..000000000000 --- a/cpp/src/librtcx/system_includes/expected +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/functional b/cpp/src/librtcx/system_includes/functional deleted file mode 100644 index b7b94f351aee..000000000000 --- a/cpp/src/librtcx/system_includes/functional +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/initializer_list b/cpp/src/librtcx/system_includes/initializer_list deleted file mode 100644 index 0ab9115d43c1..000000000000 --- a/cpp/src/librtcx/system_includes/initializer_list +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/inplace_vector b/cpp/src/librtcx/system_includes/inplace_vector deleted file mode 100644 index 0d60464a347f..000000000000 --- a/cpp/src/librtcx/system_includes/inplace_vector +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/iterator b/cpp/src/librtcx/system_includes/iterator deleted file mode 100644 index a5c45ae5f9d5..000000000000 --- a/cpp/src/librtcx/system_includes/iterator +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/latch b/cpp/src/librtcx/system_includes/latch deleted file mode 100644 index 64f1e2365895..000000000000 --- a/cpp/src/librtcx/system_includes/latch +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/limits b/cpp/src/librtcx/system_includes/limits deleted file mode 100644 index 95d9c27cf00d..000000000000 --- a/cpp/src/librtcx/system_includes/limits +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/linalg b/cpp/src/librtcx/system_includes/linalg deleted file mode 100644 index 59e85675b30a..000000000000 --- a/cpp/src/librtcx/system_includes/linalg +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/mdspan b/cpp/src/librtcx/system_includes/mdspan deleted file mode 100644 index 94e48d35185a..000000000000 --- a/cpp/src/librtcx/system_includes/mdspan +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/memory b/cpp/src/librtcx/system_includes/memory deleted file mode 100644 index 4706fe6f48d2..000000000000 --- a/cpp/src/librtcx/system_includes/memory +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/numbers b/cpp/src/librtcx/system_includes/numbers deleted file mode 100644 index a3df0d809d29..000000000000 --- a/cpp/src/librtcx/system_includes/numbers +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/numeric b/cpp/src/librtcx/system_includes/numeric deleted file mode 100644 index bc8958976325..000000000000 --- a/cpp/src/librtcx/system_includes/numeric +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/optional b/cpp/src/librtcx/system_includes/optional deleted file mode 100644 index 8f7e5862a0ea..000000000000 --- a/cpp/src/librtcx/system_includes/optional +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/ranges b/cpp/src/librtcx/system_includes/ranges deleted file mode 100644 index e45db1965fe5..000000000000 --- a/cpp/src/librtcx/system_includes/ranges +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/ratio b/cpp/src/librtcx/system_includes/ratio deleted file mode 100644 index f6ed2d1262ef..000000000000 --- a/cpp/src/librtcx/system_includes/ratio +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/semaphore b/cpp/src/librtcx/system_includes/semaphore deleted file mode 100644 index 60d7f0bca92b..000000000000 --- a/cpp/src/librtcx/system_includes/semaphore +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/source_location b/cpp/src/librtcx/system_includes/source_location deleted file mode 100644 index 0f9c293abf9e..000000000000 --- a/cpp/src/librtcx/system_includes/source_location +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/span b/cpp/src/librtcx/system_includes/span deleted file mode 100644 index f82a9a0e6e6d..000000000000 --- a/cpp/src/librtcx/system_includes/span +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/stddef.h b/cpp/src/librtcx/system_includes/stddef.h deleted file mode 100644 index 71352d667749..000000000000 --- a/cpp/src/librtcx/system_includes/stddef.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/stdint.h b/cpp/src/librtcx/system_includes/stdint.h deleted file mode 100644 index 03c8eb0a9dd1..000000000000 --- a/cpp/src/librtcx/system_includes/stdint.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/stdlib.h b/cpp/src/librtcx/system_includes/stdlib.h deleted file mode 100644 index c216518c9c29..000000000000 --- a/cpp/src/librtcx/system_includes/stdlib.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/string.h b/cpp/src/librtcx/system_includes/string.h deleted file mode 100644 index 83275541eb0a..000000000000 --- a/cpp/src/librtcx/system_includes/string.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/string_view b/cpp/src/librtcx/system_includes/string_view deleted file mode 100644 index 826afab8c607..000000000000 --- a/cpp/src/librtcx/system_includes/string_view +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/time.h b/cpp/src/librtcx/system_includes/time.h deleted file mode 100644 index 05e4eab7bd9f..000000000000 --- a/cpp/src/librtcx/system_includes/time.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/tuple b/cpp/src/librtcx/system_includes/tuple deleted file mode 100644 index 1b330a8db160..000000000000 --- a/cpp/src/librtcx/system_includes/tuple +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/type_traits b/cpp/src/librtcx/system_includes/type_traits deleted file mode 100644 index 2b2c1a21ba9b..000000000000 --- a/cpp/src/librtcx/system_includes/type_traits +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/utility b/cpp/src/librtcx/system_includes/utility deleted file mode 100644 index 78cf8d508129..000000000000 --- a/cpp/src/librtcx/system_includes/utility +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/variant b/cpp/src/librtcx/system_includes/variant deleted file mode 100644 index 142842ce87ad..000000000000 --- a/cpp/src/librtcx/system_includes/variant +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include - -namespace std { - -using namespace ::cuda::std; - -} diff --git a/cpp/src/librtcx/system_includes/version b/cpp/src/librtcx/system_includes/version deleted file mode 100644 index a24af52c843d..000000000000 --- a/cpp/src/librtcx/system_includes/version +++ /dev/null @@ -1,6 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once -#include diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index f8cbe6991abd..e26aba03fdd8 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -5,8 +5,8 @@ #pragma once -#include "jit/cache.hpp" #include "jit/helpers.hpp" +#include "jit/jit.hpp" #include "jit/parser.hpp" #include "jit/util.hpp" #include "rolling.hpp" @@ -21,8 +21,6 @@ #include -#include - #include namespace cudf { @@ -74,13 +72,13 @@ std::unique_ptr rolling_window_udf(column_view const& input, auto output_view = output->mutable_view(); cudf::detail::device_scalar device_valid_count{0, stream}; - std::string kernel_reflection = - jitify2::reflection::Template("cudf::rolling::jit::gpu_rolling_new") // - .instantiate(cudf::type_to_name(input.type()), // list of template arguments - cudf::type_to_name(output->type()), - udf_agg._operator_name, - preceding_window_str.c_str(), - following_window_str.c_str()); + auto kernel_reflection = + rtcx::reflect_template("cudf::rolling::jit::gpu_rolling_new", + cudf::type_to_name(input.type()), // list of template arguments + cudf::type_to_name(output->type()), + udf_agg._operator_name, + preceding_window_str, + following_window_str); cudf::size_type nrows_arg = input.size(); auto incol_arg = cudf::jit::get_data_ptr(input); @@ -105,7 +103,7 @@ std::unique_ptr rolling_window_udf(column_view const& input, auto kernel = cudf::jit::get_udf_kernel( "src/rolling/jit/kernel.cu", "src/rolling/jit/kernel.cu", kernel_reflection, cuda_source); auto cfg = kernel.max_occupancy_config(0, 0); - kernel.launch(cfg.min_grid_size, 1, 1, cfg.block_size, 1, 1, 0, stream, args); + kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); output->set_null_count(output->size() - device_valid_count.value(stream)); diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 8a3e0b61f2cf..90c52a64c0bb 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -34,11 +34,16 @@ void context::ensure_jit_cache_initialized() std::filesystem::create_directories(_config.jit_bundle_dir); std::filesystem::create_directories(_config.jit_pch_dir); - _rtc_cache = std::make_unique( - _config.rtc_cache_dir, - rtcx::cache_limits{.num_mem_blobs = _config.kernel_cache_limit_process, - .num_mem_libraries = _config.kernel_cache_limit_process, - .num_disk_entries = _config.kernel_cache_limit_disk}); + rtcx::initialize(); + + auto limits = rtcx::cache_limits{.num_mem_blobs = _config.kernel_cache_limit_process, + .num_mem_libraries = _config.kernel_cache_limit_process, + .num_disk_entries = _config.kernel_cache_limit_disk}; + + _rtc_cache = std::make_unique(_config.rtc_cache_dir, + limits, + bool{_config.preload_jit_cache}, + bool{_config.disable_jit_cache}); // note that jit_bundle depends on rtc_cache, so we ensure rtc_cache is initialized first. _jit_bundle = std::make_unique(_config.jit_bundle_dir, *_rtc_cache); }); @@ -66,10 +71,7 @@ std::string const& context::get_jit_pch_dir() const { return _config.jit_pch_dir void context::initialize_components(init_flags flags) { - if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { - rtcx::initialize(); - ensure_jit_cache_initialized(); - } + if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { ensure_jit_cache_initialized(); } if (has_flag(flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } } @@ -113,8 +115,11 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { std::call_once(*_context_init_flag, [&]() { - bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); - bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); + bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); + bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); + bool preload_jit_cache = get_bool_env_or("LIBCUDF_JIT_PRELOAD_CACHE", false); + bool disable_jit_cache = get_bool_env_or("LIBCUDF_JIT_CACHE_DISABLED", false); + auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16384U); auto kernel_cache_limit_disk = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_DISK", 131'072U); @@ -126,6 +131,8 @@ void initialize(init_flags flags) context_config cfg{.dump_codegen = dump_codegen, .use_jit = use_jit, + .preload_jit_cache = preload_jit_cache, + .disable_jit_cache = disable_jit_cache, .rtc_cache_dir = rtc_cache_dir, .jit_bundle_dir = jit_bundle_dir, .jit_pch_dir = jit_pch_dir, diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index ae63bab44bb6..590776be2636 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -24,8 +24,10 @@ class program_cache; class jit_bundle_t; struct [[nodiscard]] context_config { - bool dump_codegen = false; - bool use_jit = false; + bool dump_codegen : 1 = false; + bool use_jit : 1 = false; + bool preload_jit_cache : 1 = false; + bool disable_jit_cache : 1 = false; std::string rtc_cache_dir = {}; std::string jit_bundle_dir = {}; std::string jit_pch_dir = {}; diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 8cbfccd783ed..e7bb1a0e6651 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -17,8 +17,8 @@ #include -#include #include +#include #include #include #include @@ -44,24 +44,24 @@ auto build_jit_template_params(null_aware is_null_aware, std::span inputs) { std::vector tparams; - tparams.emplace_back(rtcx::reflect::value("cudf::null_aware", is_null_aware)); - tparams.emplace_back(rtcx::reflect::value(may_evaluate_null)); - tparams.emplace_back(rtcx::reflect::value(has_user_data)); + tparams.emplace_back(rtcx::reflect_enum("cudf::null_aware", is_null_aware)); + tparams.emplace_back(rtcx::reflect_bool(may_evaluate_null)); + tparams.emplace_back(rtcx::reflect_bool(has_user_data)); std::transform(thrust::counting_iterator(0), thrust::counting_iterator(span_outputs.size()), std::back_inserter(tparams), [&](auto i) { - return rtcx::reflect::templated("cudf::jit::span_accessor", - {span_outputs[i], std::to_string(i)}); + return rtcx::reflect_template( + "cudf::jit::span_accessor", span_outputs[i], std::to_string(i)); }); std::transform(thrust::counting_iterator(0), thrust::counting_iterator(column_outputs.size()), std::back_inserter(tparams), [&](auto i) { - return rtcx::reflect::templated("cudf::jit::column_accessor", - {column_outputs[i], std::to_string(i)}); + return rtcx::reflect_template( + "cudf::jit::column_accessor", column_outputs[i], std::to_string(i)); }); std::transform(thrust::counting_iterator(0), @@ -95,7 +95,7 @@ kernel build_transform_kernel(std::string_view kernel_name, cudf::jit::build_ptx_params(output_typenames, input_typenames, has_user_data)) : cudf::jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); - auto kernel_reflection = rtcx::reflect::templated( + auto kernel_reflection = rtcx::reflect_template( kernel_name, build_jit_template_params( is_null_aware, may_evaluate_null, has_user_data, {}, output_typenames, input_reflections)); @@ -128,7 +128,7 @@ kernel build_span_kernel(std::string_view kernel_name, cudf::jit::build_ptx_params(output_typenames, input_typenames, has_user_data)) : cudf::jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); - auto kernel_reflection = rtcx::reflect::templated( + auto kernel_reflection = rtcx::reflect_template( kernel_name, build_jit_template_params( is_null_aware, may_evaluate_null, has_user_data, span_outputs, {}, input_reflections)); @@ -180,7 +180,7 @@ void launch_column_output_kernel(kernel const& kernel, void* args[] = {&p_outputs, &p_inputs, &p_intermediate_null_mask, &p_user_data}; auto cfg = kernel.max_occupancy_config(0, 0); - kernel.launch(cfg.min_grid_size, 1, 1, cfg.block_size, 1, 1, 0, stream, args); + kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } template @@ -205,7 +205,7 @@ void launch_span_kernel(kernel const& kernel, auto kernel_ref = kernel.get(); auto cfg = kernel_ref.max_occupancy_config(0, 0); - kernel_ref.launch(cfg.min_grid_size, 1, 1, cfg.block_size, 1, 1, 0, stream, args); + kernel_ref.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } std::tuple and_null_mask(size_type row_size, From ce31d17ee6cbd0dc7de480864c5462988e210333 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 17 Mar 2026 10:39:26 +0000 Subject: [PATCH 080/254] refactor --- cpp/src/transform/transform.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 17f53cc2b575..c744002f741d 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -129,7 +129,7 @@ struct mut_strings_column { static auto make(size_type size, rmm::device_buffer chars, - std::unique_ptr offsets, + std::unique_ptr offsets, rmm::device_buffer null_mask, size_type null_count) { From ddfbba11824be33bb5f18b78c967c72aca480068 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 18 Mar 2026 13:35:35 +0000 Subject: [PATCH 081/254] Refactor JIT kernel paths, enhance cache management, and improve buffer handling - Updated JIT kernel paths to use the correct directory structure in multiple files. - Changed the handling of null-aware parameters in join filter templates. - Introduced a new `buffer` template class for better memory management of byte buffers. - Enhanced the `cache_t` class to support temporary directory management for atomic writes. - Updated context initialization to include new configuration options for cache management. - Improved error handling during cache preloading and disk operations. - Added support for new libcxx headers to ensure compatibility with CUDA standard library types. - Refactored various functions to utilize the new `byte_buffer` type for binary data handling. - Updated tests to reflect changes in the handling of CUDA standard library functions. --- .devcontainer/Dockerfile | 2 +- cpp/CMakeLists.txt | 12 +- .../cudf/column/column_device_view_base.cuh | 4 +- .../cudf/detail/normalizing_iterator.cuh | 12 +- .../detail/floating_conversion.hpp | 9 +- cpp/include/cudf/fixed_point/fixed_point.hpp | 9 +- cpp/include/cudf/fixed_point/temporary.hpp | 4 +- .../detail/convert/string_to_float.cuh | 21 +- cpp/include/cudf/strings/string_view.cuh | 6 +- cpp/include/cudf/strings/string_view.hpp | 20 +- cpp/include/cudf/types.hpp | 9 + cpp/include/cudf/wrappers/dictionary.hpp | 16 ++ cpp/src/binaryop/binaryop.cpp | 6 +- cpp/src/io/utilities/getenv_or.hpp | 13 + cpp/src/jit/jit.cpp | 121 ++++++---- cpp/src/join/filter_join_indices_jit.cu | 17 +- cpp/src/librtcx/CMake/embed.in.py | 4 +- cpp/src/librtcx/README.md | 9 + .../{system_includes => libcxx}/README.md | 0 cpp/src/librtcx/libcxx/cassert | 9 + cpp/src/librtcx/libcxx/climits | 9 + cpp/src/librtcx/libcxx/cstddef | 19 ++ cpp/src/librtcx/libcxx/cstdint | 46 ++++ cpp/src/librtcx/rtcx.cpp | 98 +++++--- cpp/src/librtcx/rtcx.hpp | 227 +++++++++++++----- cpp/src/rolling/detail/rolling_udf.cuh | 8 +- cpp/src/runtime/context.cpp | 99 ++++---- cpp/src/runtime/context.hpp | 17 +- cpp/src/transform/transform.cu | 12 +- .../integration/unary_transform_test.cpp | 2 +- 30 files changed, 582 insertions(+), 258 deletions(-) rename cpp/src/librtcx/{system_includes => libcxx}/README.md (100%) create mode 100644 cpp/src/librtcx/libcxx/cassert create mode 100644 cpp/src/librtcx/libcxx/climits create mode 100644 cpp/src/librtcx/libcxx/cstddef create mode 100644 cpp/src/librtcx/libcxx/cstdint diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4be36d4402cc..de4001d30e48 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -32,7 +32,7 @@ ENV PYTHONUNBUFFERED="1" ENV PYTHONDONTWRITEBYTECODE="1" ENV HISTFILE="/home/coder/.cache/._bash_history" -ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache" +ENV LIBCUDF_ROOT_DIR="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/libcudf" ### # sccache configuration diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a6fff3af2994..984c7c88aca9 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -332,18 +332,18 @@ set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include DEST_DIRECTORY cudf/include - INCLUDE_DIRECTORIES cudf/include + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include DEST_DIRECTORY cudf/cpp/include + INCLUDE_DIRECTORIES cudf/cpp/include ) jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src DEST_DIRECTORY cudf/src - INCLUDE_DIRECTORIES cudf/src + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src DEST_DIRECTORY cudf/cpp/src + INCLUDE_DIRECTORIES cudf/cpp/src ) jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/system_includes - DEST_DIRECTORY system_includes INCLUDE_DIRECTORIES system_includes + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/libcxx DEST_DIRECTORY + librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx ) get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) diff --git a/cpp/include/cudf/column/column_device_view_base.cuh b/cpp/include/cudf/column/column_device_view_base.cuh index 3e9130085c7e..0492f7be9ae8 100644 --- a/cpp/include/cudf/column/column_device_view_base.cuh +++ b/cpp/include/cudf/column/column_device_view_base.cuh @@ -11,12 +11,10 @@ #include #include +#include #include #include -#include -#include - /** * @file column_device_view_base.cuh * @brief Column device view class definitions diff --git a/cpp/include/cudf/detail/normalizing_iterator.cuh b/cpp/include/cudf/detail/normalizing_iterator.cuh index a4ac02b4b23a..3a0085f24260 100644 --- a/cpp/include/cudf/detail/normalizing_iterator.cuh +++ b/cpp/include/cudf/detail/normalizing_iterator.cuh @@ -1,11 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#ifndef CUDF_RUNTIME_JIT +#if !defined(__CUDACC_RTC__) #include @@ -13,7 +13,7 @@ #include -#include +#include namespace cudf { namespace detail { @@ -30,10 +30,10 @@ namespace detail { template struct alignas(16) base_normalator { static_assert(cudf::is_index_type()); - using difference_type = std::ptrdiff_t; + using difference_type = cuda::std::ptrdiff_t; using value_type = Integer; using pointer = Integer*; - using iterator_category = std::random_access_iterator_tag; + using iterator_category = cuda::std::random_access_iterator_tag; base_normalator() = default; base_normalator(base_normalator const&) = default; @@ -216,7 +216,7 @@ struct alignas(16) base_normalator { }; protected: -#ifndef CUDF_RUNTIME_JIT // TODO: refactor type_dispatcher to support NVRTC +#if !defined(__CUDACC_RTC__) /** * @brief Constructor assigns width and type member variables for base class. diff --git a/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp b/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp index 99e90afd7d0b..beb53e8df5da 100644 --- a/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp +++ b/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,12 +10,11 @@ #include #include +#include #include #include #include -#include - namespace CUDF_EXPORT numeric { namespace detail { @@ -120,7 +119,7 @@ struct floating_converter { { // Convert floating to integer IntegralType integer_rep; - memcpy(&integer_rep, &floating, sizeof(floating)); + cuda::std::memcpy(&integer_rep, &floating, sizeof(floating)); return integer_rep; } @@ -134,7 +133,7 @@ struct floating_converter { { // Convert back to float FloatingType floating; - memcpy(&floating, &integer, sizeof(floating)); + cuda::std::memcpy(&floating, &integer, sizeof(floating)); return floating; } diff --git a/cpp/include/cudf/fixed_point/fixed_point.hpp b/cpp/include/cudf/fixed_point/fixed_point.hpp index 5fde736dc798..d05c19110367 100644 --- a/cpp/include/cudf/fixed_point/fixed_point.hpp +++ b/cpp/include/cudf/fixed_point/fixed_point.hpp @@ -9,16 +9,15 @@ #include #include +#include +#include #include #include #include #include +#if !defined(__CUDACC_RTC__) #include -#include -#include - -#if (!defined(__CUDACC_RTC__)) && !defined(CUDF_RUNTIME_JIT) #include #endif @@ -573,7 +572,7 @@ class fixed_point { return fixed_point{scaled_integer{value, scale}}; } -#if (!defined(__CUDACC_RTC__)) && !defined(CUDF_RUNTIME_JIT) +#if !defined(__CUDACC_RTC__) /** * @brief Returns a string representation of the fixed_point value. diff --git a/cpp/include/cudf/fixed_point/temporary.hpp b/cpp/include/cudf/fixed_point/temporary.hpp index 7ef615c5192f..a912a8e0bcf2 100644 --- a/cpp/include/cudf/fixed_point/temporary.hpp +++ b/cpp/include/cudf/fixed_point/temporary.hpp @@ -10,7 +10,7 @@ #include #include -#if (!defined(__CUDACC_RTC__)) && !defined(CUDF_RUNTIME_JIT) +#if !defined(__CUDACC_RTC__) #include #include #endif @@ -18,7 +18,7 @@ namespace CUDF_EXPORT numeric { namespace detail { -#if (!defined(__CUDACC_RTC__)) && !defined(CUDF_RUNTIME_JIT) +#if !defined(__CUDACC_RTC__) template auto to_string(T value) -> std::string { diff --git a/cpp/include/cudf/strings/detail/convert/string_to_float.cuh b/cpp/include/cudf/strings/detail/convert/string_to_float.cuh index dac5b2ec3300..ea76cf106c84 100644 --- a/cpp/include/cudf/strings/detail/convert/string_to_float.cuh +++ b/cpp/include/cudf/strings/detail/convert/string_to_float.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -8,8 +8,8 @@ #include #include -#include -#include +#include +#include namespace cudf { namespace strings { @@ -35,13 +35,8 @@ __device__ inline double stod(string_view const& d_str) ++in_ptr; } -#ifndef CUDF_RUNTIME_JIT - constexpr double infinity = std::numeric_limits::infinity(); - constexpr uint64_t max_holding = (std::numeric_limits::max() - 9L) / 10L; -#else - constexpr double infinity = (1.0 / 0.0); - constexpr uint64_t max_holding = (18446744073709551615UL - 9UL) / 10UL; -#endif + constexpr double infinity = cuda::std::numeric_limits::infinity(); + constexpr uint64_t max_holding = (cuda::std::numeric_limits::max() - 9L) / 10L; // special strings: NaN, Inf if ((in_ptr < end) && *in_ptr > '9') { @@ -105,7 +100,7 @@ __device__ inline double stod(string_view const& d_str) exp_ten *= exp_sign; exp_ten += exp_off; exp_ten += num_digits - 1; - if (exp_ten > std::numeric_limits::max_exponent10) { + if (exp_ten > cuda::std::numeric_limits::max_exponent10) { return sign > 0 ? infinity : -infinity; } @@ -114,7 +109,7 @@ __device__ inline double stod(string_view const& d_str) exp_ten += 1 - num_digits; // If 10^exp_ten would result in a subnormal value, the base and // exponent should be adjusted so that 10^exp_ten is a normal value - auto const subnormal_shift = std::numeric_limits::min_exponent10 - exp_ten; + auto const subnormal_shift = cuda::std::numeric_limits::min_exponent10 - exp_ten; if (subnormal_shift > 0) { // Handle subnormal values. Ensure that both base and exponent are // normal values before computing their product. @@ -124,7 +119,7 @@ __device__ inline double stod(string_view const& d_str) return base * exponent; } - double const exponent = exp10(static_cast(std::abs(exp_ten))); + double const exponent = exp10(static_cast(cuda::std::abs(exp_ten))); return exp_ten < 0 ? base / exponent : base * exponent; } diff --git a/cpp/include/cudf/strings/string_view.cuh b/cpp/include/cudf/strings/string_view.cuh index 913ba0030999..67fb6e5f61e2 100644 --- a/cpp/include/cudf/strings/string_view.cuh +++ b/cpp/include/cudf/strings/string_view.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -15,7 +15,7 @@ // This is defined when including this header in a https://github.com/NVIDIA/jitify // or jitify2 source file. The jitify cannot include thrust headers at this time. -#ifndef CUDF_RUNTIME_JIT +#if !defined(__CUDACC_RTC__) #include #include #endif @@ -41,7 +41,7 @@ __device__ inline size_type characters_in_string(char const* str, size_type byte { if ((str == nullptr) || (bytes == 0)) return 0; auto ptr = reinterpret_cast(str); -#ifndef CUDF_RUNTIME_JIT +#if !defined(__CUDACC_RTC__) return thrust::count_if( thrust::seq, ptr, ptr + bytes, [](uint8_t chr) { return is_begin_utf8_char(chr); }); #else diff --git a/cpp/include/cudf/strings/string_view.hpp b/cpp/include/cudf/strings/string_view.hpp index dfaf54e4f415..ddb859bb72e0 100644 --- a/cpp/include/cudf/strings/string_view.hpp +++ b/cpp/include/cudf/strings/string_view.hpp @@ -6,11 +6,13 @@ #include -#if !defined(__CUDACC_RTC__) && !defined(CUDF_RUNTIME_JIT) +#if defined(__CUDACC_RTC__) +#include +#else #include -#endif #include +#endif /** * @file @@ -66,11 +68,17 @@ class string_view { class const_iterator { /// @cond public: - using difference_type = ptrdiff_t; - using value_type = char_utf8; - using reference = char_utf8&; - using pointer = char_utf8*; + using difference_type = ptrdiff_t; + using value_type = char_utf8; + using reference = char_utf8&; + using pointer = char_utf8*; + +#if defined(__CUDACC_RTC__) + using iterator_category = cuda::std::input_iterator_tag; +#else using iterator_category = std::input_iterator_tag; +#endif + __device__ inline const_iterator(string_view const& str, size_type pos); const_iterator(const_iterator const& mit) = default; const_iterator(const_iterator&& mit) = default; diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp index 659a77796012..7df7478f7096 100644 --- a/cpp/include/cudf/types.hpp +++ b/cpp/include/cudf/types.hpp @@ -30,7 +30,12 @@ #include #include #include + +#if defined(__CUDACC_RTC__) +#include +#else #include +#endif /** * @file @@ -98,7 +103,11 @@ using char_utf8 = uint32_t; ///< UTF-8 characters are 1-4 bytes template size_type distance(T f, T l) { +#if defined(__CUDACC_RTC__) + return static_cast(cuda::std::distance(f, l)); +#else return static_cast(std::distance(f, l)); +#endif } /** diff --git a/cpp/include/cudf/wrappers/dictionary.hpp b/cpp/include/cudf/wrappers/dictionary.hpp index 73a95ed7b249..492705af3425 100644 --- a/cpp/include/cudf/wrappers/dictionary.hpp +++ b/cpp/include/cudf/wrappers/dictionary.hpp @@ -7,7 +7,11 @@ #include +#if defined(__CUDACC_RTC__) +#include +#else #include +#endif /** * @file @@ -83,7 +87,11 @@ struct dictionary_wrapper { */ static CUDF_HOST_DEVICE inline constexpr value_type max_value() { +#if defined(__CUDACC_RTC__) + return cuda::std::numeric_limits::max(); +#else return std::numeric_limits::max(); +#endif } /** @@ -93,7 +101,11 @@ struct dictionary_wrapper { */ static CUDF_HOST_DEVICE inline constexpr value_type min_value() { +#if defined(__CUDACC_RTC__) + return cuda::std::numeric_limits::min(); +#else return std::numeric_limits::min(); +#endif } /** @@ -103,7 +115,11 @@ struct dictionary_wrapper { */ static CUDF_HOST_DEVICE inline constexpr value_type lowest_value() { +#if defined(__CUDACC_RTC__) + return cuda::std::numeric_limits::lowest(); +#else return std::numeric_limits::lowest(); +#endif } private: diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index a555c5abff7a..bd7eed5966b3 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -160,8 +160,10 @@ void binary_operation(mutable_column_view& out, cudf::type_to_name(rhs.type()), "cudf::binops::jit::UserDefinedOp"); - auto kernel = cudf::jit::get_udf_kernel( - "src/binaryop/jit/kernel.cu", "src/binaryop/jit/kernel.cu", kernel_reflection, cuda_source); + auto kernel = cudf::jit::get_udf_kernel("cudf/cpp/src/binaryop/jit/kernel.cu", + "cudf/cpp/src/binaryop/jit/kernel.cu", + kernel_reflection, + cuda_source); auto size_arg = static_cast(out.size()); auto out_arg = cudf::jit::get_data_ptr(out); diff --git a/cpp/src/io/utilities/getenv_or.hpp b/cpp/src/io/utilities/getenv_or.hpp index 67e81486cc30..d2ae4a347094 100644 --- a/cpp/src/io/utilities/getenv_or.hpp +++ b/cpp/src/io/utilities/getenv_or.hpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -42,6 +43,18 @@ T getenv_or(std::string_view env_var_name, T default_val) return converted_val; } +template +std::optional getenv_optional(std::string_view env_var_name) +{ + auto const env_val = std::getenv(env_var_name.data()); + if (env_val == nullptr) { return std::nullopt; } + + std::stringstream sstream(env_val); + T converted_val; + sstream >> converted_val; + return converted_val; +} + /** * @brief Specialization of getenv_or for bool, to allow common "ON"/"OFF" string values. */ diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index 04213895c20c..7e79fe2170db 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -54,7 +54,7 @@ rtcx::sha256 hash_strings(std::span inputs) CUDF_FAIL(+error_str, std::runtime_error); } -void install_file(char const* dst_path, std::span contents) +void install_file(char const* dst_path, std::span contents) { int dst_file = open(dst_path, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (dst_file == -1) { @@ -76,14 +76,50 @@ void install_file(char const* dst_path, std::span contents) } } -std::vector decompress_blob(std::span compressed_binary, - size_t uncompressed_size, - char const* compression) +/** + * @brief Reads the contents of a file into a byte buffer and null-terminates it to allow for safe + * usage as a C-string. + */ +rtcx::byte_buffer read_blob_cstring(char const* path) { - std::vector decompressed; - decompressed.resize(uncompressed_size); + int32_t fd = open(path, O_RDONLY); + if (fd == -1) { throw_posix(std::format("Failed to open file ({})", path), "open"); } + + RTCX_DEFER([&] { + if (close(fd) == -1) { throw_posix(std::format("Failed to close file ({})", path), "close"); } + }); + + auto file_size = lseek(fd, 0, SEEK_END); + if (file_size == -1) { + throw_posix(std::format("Failed to determine size of file ({})", path), "lseek"); + } + // TODO: make all read/write syscalls call read/write in a loop + + if (lseek(fd, 0, SEEK_SET) == -1) { + throw_posix(std::format("Failed to reset file offset for file ({})", path), "lseek"); + } + + auto contents = rtcx::byte_buffer::make(file_size + 1U); // +1 for null terminator + + if (read(fd, contents.data(), file_size) == -1) { + throw_posix(std::format("Failed to read file ({})", path), "read"); + } + + contents.data()[file_size] = '\0'; // null-terminate the buffer + + return contents; +} - if (std::string_view{compression} == "lz4") { +rtcx::byte_buffer decompress_blob(std::span compressed_binary, + size_t uncompressed_size, + std::string_view compression) +{ + CUDF_EXPECTS(compression == "none" || compression == "lz4" || compression == "zstd", + +std::format("Unsupported compression type specified: {}", compression), + std::runtime_error); + auto decompressed = rtcx::byte_buffer::make(uncompressed_size); + + if (compression == "lz4") { int errc = LZ4_decompress_safe(reinterpret_cast(compressed_binary.data()), reinterpret_cast(decompressed.data()), compressed_binary.size(), @@ -94,8 +130,8 @@ std::vector decompress_blob(std::span compressed_b +std::format("Failed to decompress embedded RTC source files with LZ4, error code {}", errc), std::runtime_error); - } else if (std::string_view{compression} == "zstd") { - size_t const errc = ZSTD_decompress( + } else if (compression == "zstd") { + size_t errc = ZSTD_decompress( decompressed.data(), uncompressed_size, compressed_binary.data(), compressed_binary.size()); CUDF_EXPECTS( @@ -114,19 +150,13 @@ std::vector decompress_blob(std::span compressed_b return decompressed; } -void install_file_set(char const* target_dir, +void install_file_set(std::string_view target_dir, std::span compressed_binary, size_t uncompressed_size, std::span file_ranges, std::span destinations, - char const* compression) + std::string_view compression) { - CUDF_EXPECTS(compression != nullptr, "Compression type must be specified", std::runtime_error); - CUDF_EXPECTS(compression == std::string_view{"none"} || compression == std::string_view{"lz4"} || - compression == std::string_view{"zstd"}, - +std::format("Unsupported compression type specified: {}", compression), - std::runtime_error); - auto decompressed = decompress_blob(compressed_binary, uncompressed_size, compression); auto files_data = decompressed.data(); @@ -134,26 +164,26 @@ void install_file_set(char const* target_dir, auto file_data_range = file_ranges[i]; auto file_data = std::span{files_data + file_data_range.offset, file_data_range.size}; auto dst_path = destinations[i]; - - auto target_path = std::format("{}/{}", target_dir, dst_path); + auto target_path = std::format("{}/{}", target_dir, dst_path); std::filesystem::create_directories(std::filesystem::path{target_path}.parent_path()); install_file(target_path.c_str(), file_data); } } -void install_cudf_jit_files(char const* target_dir) +void install_cudf_jit_files(std::string const& target_dir, std::string const& tmp_dir) { // directory does not exist, so create it - char tmp_dir_[] = "/tmp/cudf-jit-tmpdir_XXXXXX"; - char* tmp_dir = mkdtemp(tmp_dir_); - if (tmp_dir == nullptr) { + auto tmp_path_str = std::format("{}/cudf-jit-tmpdir_XXXXXX", tmp_dir); + (void)tmp_path_str.c_str(); // ensure null-terminated string for mkdtemp + char* tmp_path = mkdtemp(tmp_path_str.data()); + if (tmp_path == nullptr) { throw_posix( std::format("Failed to create temporary JIT install directory for ({})", target_dir), "mkdtemp"); } - install_file_set(tmp_dir, + install_file_set(tmp_path, rtcx_embed::cudf_jit_embed_files, rtcx_embed::cudf_jit_embed_files_uncompressed_size, rtcx_embed::cudf_jit_embed_file_ranges, @@ -161,7 +191,7 @@ void install_cudf_jit_files(char const* target_dir) rtcx_embed::cudf_jit_embed_files_compression); // rename the temporary directory to the target install directory - if (rename(tmp_dir, target_dir) == -1) { + if (rename(tmp_path, target_dir.c_str()) == -1) { throw_posix(std::format("Failed to rename temporary JIT install directory to ({})", target_dir), "rename"); } @@ -191,7 +221,7 @@ void jit_bundle_t::ensure_installed() const // ensure base install directory exists CUDF_LOG_INFO("Creating JIT install directory at ({})", expected_path); std::filesystem::create_directories(install_dir_); - install_cudf_jit_files(expected_path.c_str()); + install_cudf_jit_files(expected_path.c_str(), cache_->get_tmp_dir()); } } else { // directory exists, perform minor sanity check @@ -280,17 +310,28 @@ std::tuple compile_library_uncached( // --time=compile_trace.json // -time // --restrict + // --relocatable-device-code + // --extensible-whole-program + // --device-debug + // --use_fast_math + // --dlink-time-opt + // --gen-opt-lto + // --no-cache + // --create-pch + // --use-pch + // --pch-dir options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); options.emplace_back("--minimal"); - options.emplace_back("-D__CUDACC_RTC__"); - options.emplace_back("-DCUDF_RUNTIME_JIT"); options.emplace_back("--diag-suppress=47"); options.emplace_back("--device-int128"); + + if (sm >= 100) { options.emplace_back("--device-float128"); } + options.emplace_back("-std=c++20"); - options.emplace_back("-default-device"); - options.emplace_back("--device-debug"); + options.emplace_back("--device-as-default-execution-space"); options.emplace_back("--generate-line-info"); + options.emplace_back("--dopt=on"); if (use_pch) { options.emplace_back("--pch"); @@ -327,7 +368,7 @@ std::tuple compile_library_uncached( auto library = rtcx::load_library(cubin); - auto blob = rtcx::blob_t::from_vector(std::move(cubin)); + auto blob = rtcx::blob_t::from_buffer(std::move(cubin)); return std::make_tuple(library, std::make_shared(std::move(blob))); } @@ -357,7 +398,7 @@ kernel get_kernel(std::string const& name, { CUDF_FUNC_RANGE(); - auto& cache = cudf::get_context().rtc_cache(); + auto& cache = cudf::get_context().rtcx_cache(); auto& bundle = cudf::get_context().jit_bundle(); auto runtime = get_runtime_version(); @@ -390,20 +431,12 @@ name_expression={} auto cache_key_sha256 = hash_string(cache_key); auto compile = [&] { - auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); - auto source_file_path = std::format("{}/cudf/{}", bundle_dir, source_file); - auto source = rtcx::blob_t::from_file(source_file_path.c_str()); - CUDF_EXPECTS( - source.has_value(), - +std::format( - "Failed to load UDF CUDA source file: `{}` for compilation, source file does not exist", - source_file_path), - std::runtime_error); - std::string source_str{source->view().begin(), source->view().end()}; - + auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); + auto source_file_path = std::format("{}/{}", bundle_dir, source_file); + auto source = read_blob_cstring(source_file_path.c_str()); char const* name_exprs[] = {name_expression.c_str()}; return compile_library_uncached(name.c_str(), - source_str.c_str(), + reinterpret_cast(source.data()), header_include_names, headers, name_exprs, diff --git a/cpp/src/join/filter_join_indices_jit.cu b/cpp/src/join/filter_join_indices_jit.cu index 24627407efbb..90b8de84fbad 100644 --- a/cpp/src/join/filter_join_indices_jit.cu +++ b/cpp/src/join/filter_join_indices_jit.cu @@ -57,7 +57,7 @@ std::vector build_join_filter_template_params( std::vector template_params; template_params.emplace_back(rtcx::reflect_bool(has_user_data)); - template_params.emplace_back(rtcx::reflect_enum("cudf::is_null_aware", is_null_aware)); + template_params.emplace_back(rtcx::reflect_enum("cudf::null_aware", is_null_aware)); // Add left column accessors for (size_t i = 0; i < left_columns.size(); ++i) { @@ -115,8 +115,8 @@ kernel build_join_filter_kernel(std::string const& predicate_code, auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); // Get compiled kernel - return cudf::jit::get_udf_kernel("src/join/jit/filter_join_kernel.cu", // TODO: use actual name - "src/join/jit/filter_join_kernel.cu", + return cudf::jit::get_udf_kernel("cudf/cpp/src/join/jit/filter_join_kernel.cu", + "cudf/cpp/src/join/jit/filter_join_kernel.cu", kernel_name, cuda_source); } @@ -359,7 +359,7 @@ std::vector build_join_filter_template_params_from_specs( { std::vector template_params; template_params.emplace_back(rtcx::reflect_bool(false)); // has_user_data = false - template_params.emplace_back(rtcx::reflect_enum("cudf::is_null_aware", is_null_aware)); + template_params.emplace_back(rtcx::reflect_enum("cudf::null_aware", is_null_aware)); // Scalar columns are appended to the left table's device views, // starting at index left.num_columns(). @@ -509,11 +509,10 @@ filter_join_indices_jit(cudf::table_view const& left, cudf::jit::parse_single_function_cuda(filter_result.udf, "GENERIC_JOIN_FILTER_OP"); auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); - auto kernel = - cudf::jit::get_udf_kernel("src/join/jit/filter_join_kernel.cu", // TODO: use actual name - "src/join/jit/filter_join_kernel.cu", - kernel_name, - cuda_source); + auto kernel = cudf::jit::get_udf_kernel("cudf/cpp/src/join/jit/filter_join_kernel.cu", + "cudf/cpp/src/join/jit/filter_join_kernel.cu", + kernel_name, + cuda_source); // Collect scalar columns to append to left device views so join_scalar_accessor // can read them at indices >= left.num_columns(). diff --git a/cpp/src/librtcx/CMake/embed.in.py b/cpp/src/librtcx/CMake/embed.in.py index 9c60ede643cf..84934b457858 100644 --- a/cpp/src/librtcx/CMake/embed.in.py +++ b/cpp/src/librtcx/CMake/embed.in.py @@ -98,7 +98,9 @@ def generate_cxx_source_files_data( binary_file_name = f"embed_{id}.bin" cxx_header = f""" - #pragma once +// Auto-generated header for embedded files with ID: {id} +#pragma once + #include #include #include diff --git a/cpp/src/librtcx/README.md b/cpp/src/librtcx/README.md index a6f1a00e9635..aa258d6cc988 100644 --- a/cpp/src/librtcx/README.md +++ b/cpp/src/librtcx/README.md @@ -6,3 +6,12 @@ RTCX (runtime-compiler extended) is a wrapper around NVRTC and NVJitLink designe - Zero-copy interfaces to manage JIT compilation and linking - CMake script to embed **compressed** headers directly into an executable without incurring overhead at runtime on every compilation request - Facilities to pre-load and teardown dynamic library dependencies (`libcuda`, `libnvrtc`, and `libnvJitlink`) + +## Platforms Supported +- Linux + +## Build-Scripting Requirements +- CMake +- python 3 +- python-lz4 +- python-zstd diff --git a/cpp/src/librtcx/system_includes/README.md b/cpp/src/librtcx/libcxx/README.md similarity index 100% rename from cpp/src/librtcx/system_includes/README.md rename to cpp/src/librtcx/libcxx/README.md diff --git a/cpp/src/librtcx/libcxx/cassert b/cpp/src/librtcx/libcxx/cassert new file mode 100644 index 000000000000..199c5ee6cf9c --- /dev/null +++ b/cpp/src/librtcx/libcxx/cassert @@ -0,0 +1,9 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include diff --git a/cpp/src/librtcx/libcxx/climits b/cpp/src/librtcx/libcxx/climits new file mode 100644 index 000000000000..58f936c6739c --- /dev/null +++ b/cpp/src/librtcx/libcxx/climits @@ -0,0 +1,9 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include diff --git a/cpp/src/librtcx/libcxx/cstddef b/cpp/src/librtcx/libcxx/cstddef new file mode 100644 index 000000000000..ea13067bad5a --- /dev/null +++ b/cpp/src/librtcx/libcxx/cstddef @@ -0,0 +1,19 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace std { + +using max_align_t = cuda::std::max_align_t; +using nullptr_t = cuda::std::nullptr_t; +using ptrdiff_t = cuda::std::ptrdiff_t; +using size_t = cuda::std::size_t; +using byte = cuda::std::byte; + +} // namespace std diff --git a/cpp/src/librtcx/libcxx/cstdint b/cpp/src/librtcx/libcxx/cstdint new file mode 100644 index 000000000000..4c5f82cc79f8 --- /dev/null +++ b/cpp/src/librtcx/libcxx/cstdint @@ -0,0 +1,46 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace std { + +using int8_t = cuda::std::int8_t; +using int16_t = cuda::std::int16_t; +using int32_t = cuda::std::int32_t; +using int64_t = cuda::std::int64_t; +using uint8_t = cuda::std::uint8_t; +using uint16_t = cuda::std::uint16_t; +using uint32_t = cuda::std::uint32_t; +using uint64_t = cuda::std::uint64_t; + +using int_fast8_t = cuda::std::int_fast8_t; +using int_fast16_t = cuda::std::int_fast16_t; +using int_fast32_t = cuda::std::int_fast32_t; +using int_fast64_t = cuda::std::int_fast64_t; +using uint_fast8_t = cuda::std::uint_fast8_t; +using uint_fast16_t = cuda::std::uint_fast16_t; +using uint_fast32_t = cuda::std::uint_fast32_t; +using uint_fast64_t = cuda::std::uint_fast64_t; + +using int_least8_t = cuda::std::int_least8_t; +using int_least16_t = cuda::std::int_least16_t; +using int_least32_t = cuda::std::int_least32_t; +using int_least64_t = cuda::std::int_least64_t; +using uint_least8_t = cuda::std::uint_least8_t; +using uint_least16_t = cuda::std::uint_least16_t; +using uint_least32_t = cuda::std::uint_least32_t; +using uint_least64_t = cuda::std::uint_least64_t; + +using intptr_t = cuda::std::intptr_t; +using uintptr_t = cuda::std::uintptr_t; + +using intmax_t = cuda::std::intmax_t; +using uintmax_t = cuda::std::uintmax_t; + +} // namespace std diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 2b618ec7ebe4..6f66144eb157 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -222,6 +222,7 @@ sha256 sha256_context::finalize() DO_IT(LaunchKernelEx) \ DO_IT(LaunchCooperativeKernel) \ DO_IT(KernelGetName) \ + DO_IT(KernelGetFunction) \ DO_IT(LibraryLoadData) \ DO_IT(LibraryGetKernel) \ DO_IT(LibraryGetKernelCount) \ @@ -635,15 +636,14 @@ void log_nvJitLink_result(link_params const& params, } // namespace -blob_t blob_t::from_vector(std::vector&& data) +blob_t blob_t::from_buffer(byte_buffer&& buffer) { - auto ptr = new std::vector(std::move(data)); + auto size = buffer.size(); + auto data = buffer.release(); return blob_t::from_parts( - ptr->data(), - ptr->size(), - blob_t::deallocator{ptr, [](void* user_data, std::uint8_t const*, std::size_t) { - delete reinterpret_cast*>(user_data); - }}); + data, size, +[](std::uint8_t const* data, std::size_t) { + free(const_cast(data)); + }); } blob_t blob_t::from_static_data(std::span data) @@ -651,7 +651,7 @@ blob_t blob_t::from_static_data(std::span data) return blob_t::from_parts(data.data(), data.size(), blob_t::noop_deallocator); } -std::vector compile(compile_params const& params) +byte_buffer compile(compile_params const& params) { RTCX_EXPECTS(params.name != nullptr, "Fragment name must not be null", std::logic_error); RTCX_EXPECTS(params.source != nullptr, "Fragment source must not be null", std::logic_error); @@ -679,24 +679,21 @@ std::vector compile(compile_params const& params) case binary_type::CUBIN: { std::size_t cubin_size; RTCX_CHECK_NVRTC(nvrtc->GetCUBINSize(program, &cubin_size)); - std::vector cubin; - cubin.resize(cubin_size); + auto cubin = byte_buffer::make(cubin_size); RTCX_CHECK_NVRTC(nvrtc->GetCUBIN(program, reinterpret_cast(cubin.data()))); return cubin; } break; case binary_type::LTO_IR: { std::size_t lto_ir_size; RTCX_CHECK_NVRTC(nvrtc->GetLTOIRSize(program, <o_ir_size)); - std::vector lto_ir; - lto_ir.resize(lto_ir_size); + auto lto_ir = byte_buffer::make(lto_ir_size); RTCX_CHECK_NVRTC(nvrtc->GetLTOIR(program, reinterpret_cast(lto_ir.data()))); return lto_ir; } break; case binary_type::PTX: { std::size_t ptx_size; RTCX_CHECK_NVRTC(nvrtc->GetPTXSize(program, &ptx_size)); - std::vector ptx; - ptx.resize(ptx_size); + auto ptx = byte_buffer::make(ptx_size); RTCX_CHECK_NVRTC(nvrtc->GetPTX(program, reinterpret_cast(ptx.data()))); return ptx; } break; @@ -806,7 +803,7 @@ library load_library(std::span binary) return library; } -std::vector link_library(link_params const& params) +byte_buffer link_library(link_params const& params) { RTCX_EXPECTS(params.name != nullptr, "Link output name must not be null", std::logic_error); RTCX_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, @@ -847,16 +844,14 @@ std::vector link_library(link_params const& params) case binary_type::CUBIN: { std::size_t cubin_size; RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedCubinSize(handle, &cubin_size)); - std::vector cubin; - cubin.resize(cubin_size); + auto cubin = byte_buffer::make(cubin_size); RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedCubin(handle, cubin.data())); return cubin; } break; case binary_type::PTX: { std::size_t ptx_size; RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedPtxSize(handle, &ptx_size)); - std::vector ptx; - ptx.resize(ptx_size); + auto ptx = byte_buffer::make(ptx_size); RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedPtx(handle, reinterpret_cast(ptx.data()))); return ptx; } break; @@ -922,20 +917,28 @@ namespace { } // namespace -cache_t::cache_t(std::string cache_dir, cache_limits const& limits, bool preload, bool disable) +cache_t::cache_t(std::string cache_dir, + std::string tmp_dir, + cache_limits const& limits, + bool preload, + bool disable, + bool materialize_all) : enabled_{!disable}, cache_dir_{std::move(cache_dir)}, + tmp_dir_{std::move(tmp_dir)}, limits_{limits}, lock_{}, blobs_cache_{limits.num_mem_blobs}, libraries_cache_{limits.num_mem_libraries}, tick_{0} { - if (preload) { preload_from_disk(); } + if (preload) { preload_from_disk(materialize_all); } } std::string const& cache_t::get_cache_dir() { return cache_dir_; } +std::string const& cache_t::get_tmp_dir() { return tmp_dir_; } + std::optional blob_t::from_file(char const* path) { std::int32_t fd = open(path, O_RDONLY); @@ -1046,12 +1049,12 @@ void evict_disk_entries(std::string const& cache_dir, std::uint32_t limit) if (paths.size() < limit) { return; } - std::vector ranking_indices; + std::vector ranking_indices; ranking_indices.resize(paths.size()); std::iota(ranking_indices.begin(), ranking_indices.end(), 0); - std::sort(ranking_indices.begin(), ranking_indices.end(), [&](std::int32_t a, std::int32_t b) { + std::sort(ranking_indices.begin(), ranking_indices.end(), [&](auto a, auto b) { return access_times[a] < access_times[b]; }); @@ -1068,16 +1071,18 @@ void evict_disk_entries(std::string const& cache_dir, std::uint32_t limit) /// @brief atomically writes a blob to disk by first writing to a temporary file and then renaming /// it to the final path. void cache_blob_to_disk(std::string const& cache_dir, + std::string const& tmp_dir, std::string const& object_type, sha256 const& sha, std::span binary, std::uint32_t limit) { if (limit > 0) { - char temp_path[] = "/tmp/rtcx-bin-XXXXXX"; + auto tmp_path = std::format("{}/rtcx-bin-XXXXXX", tmp_dir); + (void)tmp_path.c_str(); // to ensure null-termination for mkstemp { - std::int32_t fd = mkstemp(temp_path); + std::int32_t fd = mkstemp(tmp_path.data()); if (fd == -1) { throw_posix("Failed to create temporary file for RTCX cache", "mkstemp"); } RTCX_DEFER([&] { @@ -1095,10 +1100,10 @@ void cache_blob_to_disk(std::string const& cache_dir, std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); // rename is atomic, even if another process is performing the same operation - if (rename(temp_path, final_path.c_str()) == -1) { + if (rename(tmp_path.c_str(), final_path.c_str()) == -1) { if (errno == EEXIST) { // another process has already created the file, so just remove our temp file - if (remove(temp_path) == -1) { + if (remove(tmp_path.c_str()) == -1) { throw_posix("Failed to remove temporary RTCX cache file", "remove"); } return; @@ -1173,7 +1178,8 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil promise.set_value(result); // store result to disk - cache_blob_to_disk(cache_dir_, "blob", sha, result->view(), limits_.num_disk_entries); + cache_blob_to_disk( + cache_dir_, tmp_dir_, "blob", sha, result->view(), limits_.num_disk_entries); return ret_fut; } @@ -1244,7 +1250,8 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, promise.set_value(library); // store result to disk - cache_blob_to_disk(cache_dir_, "cuLibrary", sha, blob->view(), limits_.num_disk_entries); + cache_blob_to_disk( + cache_dir_, tmp_dir_, "cuLibrary", sha, blob->view(), limits_.num_disk_entries); return ret_fut; } @@ -1299,18 +1306,28 @@ void cache_t::clear_memory_store() void cache_t::clear_disk_store() { evict_disk_entries(cache_dir_, 0); } -void cache_t::preload_from_disk() +void cache_t::preload_from_disk(bool materialize_all) { - auto [paths, _] = get_disk_entries(cache_dir_); + auto [paths, access_times] = get_disk_entries(cache_dir_); + + std::vector ranking_indices; + ranking_indices.resize(paths.size()); + std::iota(ranking_indices.begin(), ranking_indices.end(), 0); + std::sort(ranking_indices.begin(), ranking_indices.end(), [&](auto a, auto b) { + return access_times[a] > access_times[b]; + }); + + auto load_count = std::min(ranking_indices.size(), + limits_.num_mem_blobs + limits_.num_mem_libraries); - // TODO(lamarrr): we have promises that may not be fulfilled if errors occur, they need to be handled? - // i.e. if the exceptions is caught + ranking_indices.resize(load_count); { std::lock_guard guard{lock_}; tick_++; - for (auto& path : paths) { + for (auto index : ranking_indices) { + auto path = paths[index]; try { auto file_name = std::filesystem::path{path}.filename().string(); auto sha_str_end = file_name.find('.'); @@ -1319,21 +1336,24 @@ void cache_t::preload_from_disk() if (!data.has_value()) { continue; } if (path.ends_with(".blob.bin")) { - std::promise promise; + auto blob = std::make_shared(std::move(*data)); + std::promise promise; auto fut = promise.get_future().share(); - promise.set_value(std::make_shared(std::move(*data))); + promise.set_value(std::move(blob)); blobs_cache_.insert(sha256::parse(sha_str), std::move(fut), tick_); } else if (path.ends_with(".cuLibrary.bin")) { + auto lib = load_library(data->view()); + if (materialize_all) { [[maybe_unused]] auto kernels = lib->enumerate_kernels(); } std::promise promise; auto fut = promise.get_future().share(); - promise.set_value(load_library(data->view())); + promise.set_value(std::move(lib)); libraries_cache_.insert(sha256::parse(sha_str), std::move(fut), tick_); } } catch (std::exception const& e) { // ignore any errors during preload - log_trace(e.what()); + log_error(e.what()); } catch (...) { - log_trace("Unknown error during preload"); + log_error("Unknown error during preload"); } } } diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index da84e9734c64..1ab193b5d6bb 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -192,9 +192,129 @@ struct [[nodiscard]] sha256_context { enum class binary_type : std::int8_t { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; /** - * @brief Represents a binary blob + * @brief An heap-allocated statically-sized buffer. Its contents are not guaranteed to be + * initialized. + */ +template + requires(!std::is_const_v && std::is_trivially_copyable_v && + std::is_trivially_destructible_v) +struct buffer { + private: + T* _data; + std::size_t _size; + + buffer(T* data, std::size_t size) : _data(data), _size(size) {} + + public: + /** + * @brief Creates a new buffer of the given size, with uninitialized contents. + */ + static buffer make(std::size_t size) + { + T* data = static_cast(malloc(size * sizeof(T))); + if (data == nullptr) { throw std::bad_alloc(); } + return buffer{data, size}; + } + + buffer() : buffer{nullptr, 0} {} //< Default constructor. Creates an empty buffer + + buffer(buffer const&) = delete; + + buffer& operator=(buffer const&) = delete; + + /** + * @brief Move constructor. Transfers ownership of the buffer from the source to the new object. + * After the move, the source buffer is left in an empty state (data pointer is null and size is + * zero). + */ + buffer(buffer&& other) noexcept : _data(other._data), _size(other._size) + { + other._data = nullptr; + other._size = 0; + } + + /** + * @brief Move assignment operator. Transfers ownership of the buffer from the source to the + * current object. + */ + buffer& operator=(buffer&& other) noexcept + { + if (this == &other) [[unlikely]] { return *this; } + this->~buffer(); + new (this) buffer(std::move(other)); + return *this; + } + + ~buffer() noexcept { free(_data); } + + /** + * @brief Returns a pointer to the buffer's data + * @return A pointer to the buffer's data + */ + [[nodiscard]] T* data() const { return _data; } + + /** + * @brief Returns the size of the buffer + * @return The size of the buffer in number of elements + */ + [[nodiscard]] std::size_t size() const { return _size; } + + /** + * @brief Returns an iterator to the beginning of the buffer + * @return An iterator to the beginning of the buffer + */ + [[nodiscard]] T* begin() { return _data; } + + /** + * @brief Returns an iterator to the end of the buffer + * @return An iterator to the end of the buffer + */ + [[nodiscard]] T* end() { return _data + _size; } + + /** + * @brief Returns a const iterator to the beginning of the buffer + * @return A const iterator to the beginning of the buffer + */ + [[nodiscard]] T const* begin() const { return _data; } + + /** + * @brief Returns a const iterator to the end of the buffer + * @return A const iterator to the end of the buffer + */ + [[nodiscard]] T const* end() const { return _data + _size; } + + /** + * @brief Returns a const iterator to the beginning of the buffer + * @return A const iterator to the beginning of the buffer + */ + [[nodiscard]] T const* cbegin() const { return _data; } + + /** + * @brief Returns a const iterator to the end of the buffer + * @return A const iterator to the end of the buffer + */ + [[nodiscard]] T const* cend() const { return _data + _size; } + + /** + * @brief Releases ownership of the buffer's data and returns a pointer to it. After calling this + * function, the buffer is left in an empty state (data pointer is null and size is zero). + * @return A pointer to the buffer's data + */ + [[nodiscard]] T* release() + { + T* data = _data; + _data = nullptr; + _size = 0; + return data; + } +}; + +using byte_buffer = buffer; + +/** + * @brief Represents an immutable blob view * @details Manages the lifetime of the binary data via a user-provided deallocator function. This - * enables zero-copy usage of binary data stored in various forms (e.g., std::vector, mmap'd file, + * enables zero-copy view of binary data stored in various forms (e.g., std::vector, mmap'd file, * etc.). */ struct [[nodiscard]] blob_t { @@ -243,7 +363,7 @@ struct [[nodiscard]] blob_t { return blob_t{data, size, deallocator}; } - static blob_t from_vector(std::vector&& data); + static blob_t from_buffer(byte_buffer&& buffer); static blob_t from_static_data(std::span data); @@ -253,9 +373,8 @@ struct [[nodiscard]] blob_t { using blob = std::shared_ptr; /** - * @brief Represents the occupancy configuration for a kernel, including the minimum grid size and - * the block size required to achieve that occupancy. This information can be used to optimize - * kernel launches for maximum performance on the GPU. + * @brief Represents the occupancy configuration for a kernel. This information can be used to + * optimize kernel launches for maximum performance on the GPU. */ struct [[nodiscard]] kernel_occupancy_config { std::uint32_t min_grid_size = 0; //< Minimum grid size to achieve the maximum occupancy @@ -376,52 +495,29 @@ struct [[nodiscard]] library_t { using library = std::shared_ptr; +/** + * @brief Parameters for compiling source code into a binary blob using NVRTC + */ struct [[nodiscard]] compile_params { - /** - * @brief Name of the kernel or library being compiled - */ - char const* name = nullptr; - - /** - * @brief Source code to be compiled (e.g. PTX or LTO IR) - */ - char const* source = nullptr; - - /** - * @brief Include names of each header file provided in the `headers` field, used for resolving - * #include directives during compilation - */ - std::span header_include_names = {}; - - /** - * @brief Contents of header files required for compilation - */ - std::span headers = {}; - - /** - * @brief NVRTC compile options - */ - std::span options = {}; - - /** - * @brief The name expressions of the kernel entry points to be compiled, used for retrieving - * kernel references after compilation. - */ - std::span name_expressions = {}; - - /** - * @brief Desired output binary type (e.g. PTX, CUBIN, etc.) - */ - binary_type target_type = binary_type::LTO_IR; + char const* name = nullptr; //< Debug name for the compilation unit + char const* source = nullptr; //< Source code to be compiled + std::span header_include_names = {}; //< Header file names + std::span headers = {}; //< Header file contents + std::span options = {}; //< NVRTC compilation options + std::span name_expressions = {}; //< Name expressions to be instantiated + binary_type target_type = binary_type::LTO_IR; //< Output binary type }; +/** + * @brief Parameters for linking multiple compiled fragments into a single library + */ struct [[nodiscard]] link_params { - char const* name = nullptr; - binary_type output_type = binary_type::CUBIN; - std::span const> fragments = {}; - std::span fragment_binary_types = {}; - std::span fragment_names = {}; - std::span link_options = {}; + char const* name = nullptr; //< Debug name for the linked library + binary_type output_type = binary_type::CUBIN; //< Output binary type + std::span const> fragments = {}; //< Binary data for each fragment + std::span fragment_binary_types = {}; //< Binary type for each fragment + std::span fragment_names = {}; //< Debug name for each fragment + std::span link_options = {}; //< NVJITLink options }; namespace detail { @@ -553,6 +649,8 @@ struct cache_t { std::string cache_dir_; + std::string tmp_dir_; + cache_limits limits_; std::mutex lock_; @@ -571,14 +669,24 @@ struct cache_t { * for preloading and enabling the cache. * @param cache_dir The directory path to be used for on-disk caching of compiled blobs and * libraries + * @param tmp_dir The directory path to be used for temporary files during atomic writes to the + * on-disk cache * @param limits A cache_limits struct specifying the maximum number of blobs and libraries to * store in the cache before eviction occurs * @param preload A boolean flag indicating whether to preload the cache from disk during * initialization, allowing for faster retrieval of previously compiled kernels at runtime * @param disable A boolean flag indicating whether to disable the cache entirely, preventing any * caching of compiled blobs and libraries in memory + * @param materialize_all A boolean flag indicating whether to make the compiled kernels fully + * materialized in memory during preloading to improve runtime stability at the cost of increased + * memory usage */ - cache_t(std::string cache_dir, cache_limits const& limits, bool preload, bool disable); + cache_t(std::string cache_dir, + std::string tmp_dir, + cache_limits const& limits, + bool preload, + bool disable, + bool materialize_all); cache_t(cache_t const&) = delete; cache_t& operator=(cache_t const&) = delete; cache_t(cache_t&&) = delete; @@ -591,6 +699,13 @@ struct cache_t { */ [[nodiscard]] std::string const& get_cache_dir(); + /** + * @brief Get the directory path used for temporary files during atomic writes to the on-disk + * cache + * @return String reference to the temporary directory path + */ + [[nodiscard]] std::string const& get_tmp_dir(); + /** * @brief Query the cache for a compiled blob by its SHA-256 hash, or insert it if not present * @param sha SHA-256 hash of the blob to query or insert @@ -667,9 +782,11 @@ struct cache_t { /*** * @brief Pre-load the JIT program cache from disk into memory during initialization, allowing for - * faster retrieval of previously compiled kernels at runtime. + * faster retrieval and execution of previously compiled kernels at runtime. + * @param materialize_all Whether to make the compiled kernels fully materialized in memory to + * improve runtime stability. */ - void preload_from_disk(); + void preload_from_disk(bool materialize_all); /*** * @brief Enable the cache, allowing it to store and retrieve compiled blobs and libraries in @@ -690,9 +807,9 @@ struct cache_t { * * @param params Compilation parameters including source code, headers, options, and target binary * type - * @return A vector of bytes containing the compiled binary blob + * @return A buffer of bytes containing the compiled binary blob */ -[[nodiscard]] std::vector compile(compile_params const& params); +[[nodiscard]] byte_buffer compile(compile_params const& params); /** * @brief Load a compiled library from binary data @@ -709,9 +826,9 @@ struct cache_t { * * @param params Linking parameters including the binary fragments to be linked and the target * binary type - * @return A vector of bytes containing the linked library binary + * @return A buffer of bytes containing the linked library binary */ -[[nodiscard]] std::vector link_library(link_params const& params); +[[nodiscard]] byte_buffer link_library(link_params const& params); /** * @brief Demangle a CUDA symbol name into a human-readable form diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index e26aba03fdd8..96a95d3f7496 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -100,9 +100,11 @@ std::unique_ptr rolling_window_udf(column_view const& input, &following_window_arg, &min_periods_arg}; - auto kernel = cudf::jit::get_udf_kernel( - "src/rolling/jit/kernel.cu", "src/rolling/jit/kernel.cu", kernel_reflection, cuda_source); - auto cfg = kernel.max_occupancy_config(0, 0); + auto kernel = cudf::jit::get_udf_kernel("cudf/cpp/src/rolling/jit/kernel.cu", + "cudf/cpp/src/rolling/jit/kernel.cu", + kernel_reflection, + cuda_source); + auto cfg = kernel.max_occupancy_config(0, 0); kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); output->set_null_count(output->size() - device_valid_count.value(stream)); diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 94b8714ee8af..4f9dc4c820da 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -30,9 +30,10 @@ void context::ensure_jit_cache_initialized() { std::call_once(_jit_cache_init_flag, [&]() { // make sure the required directories exist - std::filesystem::create_directories(_config.rtc_cache_dir); + std::filesystem::create_directories(_config.rtcx_cache_dir); std::filesystem::create_directories(_config.jit_bundle_dir); std::filesystem::create_directories(_config.jit_pch_dir); + std::filesystem::create_directories(_config.jit_tmp_dir); rtcx::initialize(); @@ -40,21 +41,29 @@ void context::ensure_jit_cache_initialized() .num_mem_libraries = _config.kernel_cache_limit_process, .num_disk_entries = _config.kernel_cache_limit_disk}; - _rtc_cache = std::make_unique(_config.rtc_cache_dir, - limits, - bool{_config.preload_jit_cache}, - bool{_config.disable_jit_cache}); - // note that jit_bundle depends on rtc_cache, so we ensure rtc_cache is initialized first. - _jit_bundle = std::make_unique(_config.jit_bundle_dir, *_rtc_cache); + _rtcx_cache = std::make_unique(_config.rtcx_cache_dir, + _config.jit_tmp_dir, + limits, + bool{_config.preload_jit_cache}, + bool{_config.disable_jit_cache}, + bool{_config.prematerialize_jit_cache}); + + if (_config.clear_jit_cache) { + _rtcx_cache->clear_memory_store(); + _rtcx_cache->clear_disk_store(); + } + + // note that jit_bundle depends on rtcx_cache, so we ensure rtcx_cache is initialized first. + _jit_bundle = std::make_unique(_config.jit_bundle_dir, *_rtcx_cache); }); } context::~context() { rtcx::teardown(); } -rtcx::cache_t& context::rtc_cache() +rtcx::cache_t& context::rtcx_cache() { ensure_jit_cache_initialized(); - return *_rtc_cache; + return *_rtcx_cache; } jit_bundle_t& context::jit_bundle() @@ -80,32 +89,21 @@ static std::optional _context{std::nullopt}; static std::optional _context_init_flag{std::in_place}; static std::optional _context_deinit_flag{std::in_place}; -std::filesystem::path get_cudf_dir() +std::filesystem::path get_cudf_root_dir() { - auto cudf_dir_env = std::getenv("LIBCUDF_DIR"); - if (cudf_dir_env != nullptr) { - return std::filesystem::path(cudf_dir_env); - } else { - auto home_dir = std::getenv("HOME"); - CUDF_EXPECTS(home_dir != nullptr, "HOME environment variable is not set", std::runtime_error); - auto cudf_dir = std::filesystem::path(home_dir) / ".cudf"; - return cudf_dir; + if (auto cudf = getenv_optional("LIBCUDF_ROOT_DIR"); cudf.has_value()) { + return std::filesystem::path(*cudf); } -} - -std::filesystem::path get_jit_bundle_dir() -{ - return getenv_or("LIBCUDF_JIT_BUNDLE_DIR", get_cudf_dir() / "jit_bundle"); -} -std::filesystem::path get_rtc_cache_dir() -{ - return getenv_or("LIBCUDF_RTC_CACHE_DIR", get_cudf_dir() / "rtc_cache"); -} + if (auto home = getenv_optional("HOME"); home.has_value()) { + return std::filesystem::path(*home) / ".libcudf"; + } -std::filesystem::path get_jit_pch_dir() -{ - return getenv_or("LIBCUDF_JIT_PCH_DIR", get_cudf_dir() / "jit_pch"); + CUDF_FAIL( + "Unable to determine the CUDF root directory. Please set the `LIBCUDF_ROOT_DIR`, " + "`HOME` or `LIBCUDF_ROOT_DIR` environment variables to allow automatic resolution of the root " + "directory.", + std::runtime_error); } } // namespace cudf @@ -115,27 +113,33 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { std::call_once(*_context_init_flag, [&]() { - bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); - bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); - bool preload_jit_cache = get_bool_env_or("LIBCUDF_JIT_PRELOAD_CACHE", false); - bool disable_jit_cache = get_bool_env_or("LIBCUDF_JIT_CACHE_DISABLED", false); - - auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16384U); + bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); + bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); + bool preload_jit_cache = get_bool_env_or("LIBCUDF_JIT_PRELOAD_CACHE", false); + bool prematerialize_jit_cache = get_bool_env_or("LIBCUDF_JIT_PREMATERIALIZE_CACHE", false); + bool disable_jit_cache = get_bool_env_or("LIBCUDF_JIT_CACHE_DISABLED", false); + bool clear_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); + + auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16'384U); auto kernel_cache_limit_disk = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_DISK", 131'072U); flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); - auto jit_bundle_dir = get_jit_bundle_dir(); - auto rtc_cache_dir = get_rtc_cache_dir(); - auto jit_pch_dir = get_jit_pch_dir(); + auto jit_bundle_dir = get_cudf_root_dir() / "jit" / "bundle"; + auto rtcx_cache_dir = get_cudf_root_dir() / "jit" / "rtcx_cache"; + auto jit_pch_dir = get_cudf_root_dir() / "jit" / "pch"; + auto jit_tmp_dir = get_cudf_root_dir() / "jit" / "tmp"; context_config cfg{.dump_codegen = dump_codegen, .use_jit = use_jit, .preload_jit_cache = preload_jit_cache, + .prematerialize_jit_cache = prematerialize_jit_cache, .disable_jit_cache = disable_jit_cache, - .rtc_cache_dir = rtc_cache_dir, + .clear_jit_cache = clear_jit_cache, + .rtcx_cache_dir = rtcx_cache_dir, .jit_bundle_dir = jit_bundle_dir, .jit_pch_dir = jit_pch_dir, + .jit_tmp_dir = jit_tmp_dir, .kernel_cache_limit_process = kernel_cache_limit_process, .kernel_cache_limit_disk = kernel_cache_limit_disk}; @@ -156,9 +160,18 @@ void teardown() }); } -void enable_jit_cache(bool enable) { get_context().program_cache().enable(enable); } +void enable_jit_cache(bool enabled) +{ + auto& cache = get_context().rtcx_cache(); + cache.enable(enabled); +} -void clear_jit_cache() { get_context().program_cache().clear(); } +void clear_jit_cache() +{ + auto& cache = get_context().rtcx_cache(); + cache.clear_memory_store(); + cache.clear_disk_store(); +} context& get_context() { diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 590776be2636..e216f067855a 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -24,13 +24,16 @@ class program_cache; class jit_bundle_t; struct [[nodiscard]] context_config { - bool dump_codegen : 1 = false; - bool use_jit : 1 = false; - bool preload_jit_cache : 1 = false; - bool disable_jit_cache : 1 = false; - std::string rtc_cache_dir = {}; + bool dump_codegen : 1 = false; + bool use_jit : 1 = false; + bool preload_jit_cache : 1 = false; + bool prematerialize_jit_cache : 1 = false; + bool disable_jit_cache : 1 = false; + bool clear_jit_cache : 1 = false; + std::string rtcx_cache_dir = {}; std::string jit_bundle_dir = {}; std::string jit_pch_dir = {}; + std::string jit_tmp_dir = {}; uint32_t kernel_cache_limit_process = 0; uint32_t kernel_cache_limit_disk = 0; }; @@ -42,7 +45,7 @@ class context { private: context_config _config; std::once_flag _jit_cache_init_flag; - std::unique_ptr _rtc_cache; + std::unique_ptr _rtcx_cache; std::unique_ptr _jit_bundle; private: @@ -58,7 +61,7 @@ class context { context& operator=(context&&) = delete; ~context(); - rtcx::cache_t& rtc_cache(); + rtcx::cache_t& rtcx_cache(); jit_bundle_t& jit_bundle(); diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 78420c2eb0dc..94915c26a680 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -100,8 +100,10 @@ kernel build_transform_kernel(std::string_view kernel_name, build_jit_template_params( is_null_aware, may_evaluate_null, has_user_data, {}, output_typenames, input_reflections)); - return cudf::jit::get_udf_kernel( - "src/transform/jit/kernel.cu", "src/transform/jit/kernel.cu", kernel_reflection, cuda_source); + return cudf::jit::get_udf_kernel("cudf/cpp/src/transform/jit/kernel.cu", + "cudf/cpp/src/transform/jit/kernel.cu", + kernel_reflection, + cuda_source); } kernel build_span_kernel(std::string_view kernel_name, @@ -133,8 +135,10 @@ kernel build_span_kernel(std::string_view kernel_name, build_jit_template_params( is_null_aware, may_evaluate_null, has_user_data, span_outputs, {}, input_reflections)); - return cudf::jit::get_udf_kernel( - "src/transform/jit/kernel.cu", "src/transform/jit/kernel.cu", kernel_reflection, cuda_source); + return cudf::jit::get_udf_kernel("cudf/cpp/src/transform/jit/kernel.cu", + "cudf/cpp/src/transform/jit/kernel.cu", + kernel_reflection, + cuda_source); } column_view to_column_view(column_view const& col) { return col; } diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index 025cacb518aa..e4d18a43a462 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -571,7 +571,7 @@ TEST_F(StringOperationTest, Output) std::string cuda = R"***( __device__ void transform(cudf::string_view * out, cudf::string_view a, cudf::string_view b, cudf::string_view c, cudf::string_view d){ - *out = std::max(std::max(std::max(a, b), c), d); + *out = cuda::std::max(cuda::std::max(cuda::std::max(a, b), c), d); } )***"; From 5ea3317b8401bc01de70fe6c22fd28a02887b28d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 18 Mar 2026 13:59:17 +0000 Subject: [PATCH 082/254] Update copyright years in header files and enhance kernel launch parameters in rolling_udf --- cpp/include/cudf/utilities/error.hpp | 4 +-- cpp/include/cudf/wrappers/dictionary.hpp | 1 - cpp/src/jit/jit.hpp | 11 ++++++++ cpp/src/rolling/detail/rolling_udf.cuh | 34 +++++++++--------------- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/cpp/include/cudf/utilities/error.hpp b/cpp/include/cudf/utilities/error.hpp index f924af36943b..abf2f80330c8 100644 --- a/cpp/include/cudf/utilities/error.hpp +++ b/cpp/include/cudf/utilities/error.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -189,7 +189,7 @@ struct data_type_error : std::invalid_argument { #define CUDF_FAIL_2(_what, _exception_type) \ /*NOLINTNEXTLINE(bugprone-macro-parentheses)*/ \ - throw _exception_type { "CUDF failure at: " __FILE__ ":" CUDF_STRINGIFY(__LINE__) ": " _what } + throw _exception_type { "CUDF failure at:" __FILE__ ":" CUDF_STRINGIFY(__LINE__) ": " _what } #define CUDF_FAIL_1(_what) CUDF_FAIL_2(_what, cudf::logic_error) diff --git a/cpp/include/cudf/wrappers/dictionary.hpp b/cpp/include/cudf/wrappers/dictionary.hpp index 9d5eda00dcd3..77b2e80065b3 100644 --- a/cpp/include/cudf/wrappers/dictionary.hpp +++ b/cpp/include/cudf/wrappers/dictionary.hpp @@ -1,5 +1,4 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index 75f0c3dbaafc..8727bc507b0b 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -58,6 +58,17 @@ struct [[nodiscard]] kernel { { return _kernel.launch(grid_dim, block_dim, shared_mem_bytes, stream.value(), kernel_params); } + + template + void launch_with(rtcx::cuda_dim3 grid_dim, + rtcx::cuda_dim3 block_dim, + uint32_t shared_mem_bytes, + rmm::cuda_stream_view stream, + Args&&... args) + { + void const* params[] = {&args...}; + launch(grid_dim, block_dim, shared_mem_bytes, stream, const_cast(params)); + } }; kernel get_kernel(std::string const& name, diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index 96a95d3f7496..f8a2acbb4888 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -80,32 +80,24 @@ std::unique_ptr rolling_window_udf(column_view const& input, preceding_window_str, following_window_str); - cudf::size_type nrows_arg = input.size(); - auto incol_arg = cudf::jit::get_data_ptr(input); - auto incol_valid_arg = input.null_mask(); - auto outcol_arg = cudf::jit::get_data_ptr(output->mutable_view()); - auto outcol_valid_arg = output_view.null_mask(); - cudf::size_type* output_valid_count_arg = device_valid_count.data(); - auto preceding_window_arg = preceding_window; - auto following_window_arg = following_window; - auto min_periods_arg = min_periods; - - void* args[] = {&nrows_arg, - &incol_arg, - &incol_valid_arg, - &outcol_arg, - &outcol_valid_arg, - &output_valid_count_arg, - &preceding_window_arg, - &following_window_arg, - &min_periods_arg}; - auto kernel = cudf::jit::get_udf_kernel("cudf/cpp/src/rolling/jit/kernel.cu", "cudf/cpp/src/rolling/jit/kernel.cu", kernel_reflection, cuda_source); auto cfg = kernel.max_occupancy_config(0, 0); - kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); + kernel.launch_with({cfg.min_grid_size}, + {cfg.block_size}, + 0, + stream, + input.size(), + cudf::jit::get_data_ptr(input), + input.null_mask(), + cudf::jit::get_data_ptr(output->mutable_view()), + output_view.null_mask(), + device_valid_count.data(), + preceding_window, + following_window, + min_periods); output->set_null_count(output->size() - device_valid_count.value(stream)); From 78bdf40be09d0fdf334fa89f2cc253c19ef22d42 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 18 Mar 2026 14:28:13 +0000 Subject: [PATCH 083/254] update --- cpp/CMakeLists.txt | 2 +- cpp/cmake/thirdparty/get_lz4.cmake | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e6c8e4eeb339..3e51e28c3ba7 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1034,10 +1034,10 @@ target_link_libraries( kvikio::kvikio nanoarrow::nanoarrow zstd + liblz4_static CUDA::nvJitLink CUDA::nvrtc CUDA::cuda_driver - lz4_objects ) # Add Conda library, and include paths if specified diff --git a/cpp/cmake/thirdparty/get_lz4.cmake b/cpp/cmake/thirdparty/get_lz4.cmake index 183fecef9adc..a7e0e6f573f9 100644 --- a/cpp/cmake/thirdparty/get_lz4.cmake +++ b/cpp/cmake/thirdparty/get_lz4.cmake @@ -15,18 +15,26 @@ function(find_and_configure_lz4) CPM_ARGS GIT_REPOSITORY https://github.com/lz4/lz4.git GIT_TAG 5c4c1fb2354133e1f3b087a341576985f8114bd5 - GIT_SHALLOW TRUE + GIT_SHALLOW FALSE SOURCE_SUBDIR build/cmake ) if(lz4_ADDED) add_library( - lz4_objects OBJECT + liblz4_static OBJECT ${lz4_SOURCE_DIR}/lib/lz4.c ${lz4_SOURCE_DIR}/lib/lz4file.c ${lz4_SOURCE_DIR}/lib/lz4frame.c ${lz4_SOURCE_DIR}/lib/lz4hc.c ${lz4_SOURCE_DIR}/lib/xxhash.c ) - target_include_directories(lz4_objects PUBLIC $) + target_include_directories(liblz4_static PUBLIC $) endif() + if(DEFINED lz4_SOURCE_DIR) + set(LZ4_INCLUDE_DIR + "${lz4_SOURCE_DIR}/lib" + PARENT_SCOPE + ) + endif() + rapids_export_find_package_root(BUILD liblz4_static "${lz4_BINARY_DIR}" EXPORT_SET cudf-exports) + endfunction() find_and_configure_lz4() From 9fe78f33e0168c961c832c6c0d59389c20d54142 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 18 Mar 2026 23:17:27 +0000 Subject: [PATCH 084/254] CI errors fix + refactoring --- cpp/src/librtcx/CMake/embed.in.py | 28 ++++--- cpp/src/librtcx/rtcx.cpp | 132 +++++++++++++++++++----------- cpp/src/librtcx/rtcx.hpp | 16 ++-- cpp/src/runtime/context.cpp | 15 ++-- cpp/src/runtime/context.hpp | 11 ++- 5 files changed, 121 insertions(+), 81 deletions(-) diff --git a/cpp/src/librtcx/CMake/embed.in.py b/cpp/src/librtcx/CMake/embed.in.py index 84934b457858..ad043da0d894 100644 --- a/cpp/src/librtcx/CMake/embed.in.py +++ b/cpp/src/librtcx/CMake/embed.in.py @@ -97,6 +97,13 @@ def generate_cxx_source_files_data( binary_file_name = f"embed_{id}.bin" + include_dirs_list = ",\n".join([f'"{d}"' for d in include_directories]) + dests_list = ",\n".join([f'"{d}"' for d in dests]) + ranges_list = ",\n".join( + [f"{{{offset}, {size}}}" for offset, size in files_ranges] + ) + hash_list = ", ".join([f"0x{b:02x}" for b in hash]) + cxx_header = f""" // Auto-generated header for embedded files with ID: {id} #pragma once @@ -115,20 +122,19 @@ def generate_cxx_source_files_data( constexpr char const * {id}_include_directories[{len(include_directories)}] = {{ -{",\n".join([f'"{d}"' for d in include_directories])} +{include_dirs_list} }}; constexpr char const * {id}_file_destinations[{len(dests)}] = {{ -{",\n".join([f'"{d}"' for d in dests])} +{dests_list} }}; constexpr range {id}_file_ranges[{len(files_ranges)}] = {{ -{",\n".join([f"{{{offset}, {size}}}" for offset, size in files_ranges])} +{ranges_list} }}; - constexpr std::size_t {id}_files_uncompressed_size = {len(uncompressed_files_bytes)}; constexpr char const * {id}_files_compression = "{compression}"; @@ -143,7 +149,7 @@ def generate_cxx_source_files_data( constexpr std::uint8_t {id}_hash[{len(hash)}] = {{ -{", ".join([f"0x{b:02x}" for b in hash])} +{hash_list} }}; }} @@ -197,16 +203,14 @@ def generate_embed( def main(): id: str = "@RTCX_EMBED_PY_ARG__ID@" - file_paths_str: str = "@RTCX_EMBED_PY_ARG__FILE_PATHS@" - file_dests_str: str = "@RTCX_EMBED_PY_ARG__FILE_DESTS@" - include_directories_str: str = "@RTCX_EMBED_PY_ARG__INCLUDE_DIRS@" + file_paths: list[str] = "@RTCX_EMBED_PY_ARG__FILE_PATHS@".split(";") + file_dests: list[str] = "@RTCX_EMBED_PY_ARG__FILE_DESTS@".split(";") + include_directories: list[str] = "@RTCX_EMBED_PY_ARG__INCLUDE_DIRS@".split( + ";" + ) compression: str = "@RTCX_EMBED_PY_ARG__COMPRESSION@" output_dir: str = "@RTCX_EMBED_PY_ARG__OUTPUT_DIR@" - file_paths = file_paths_str.split(";") - file_dests = file_dests_str.split(";") - include_directories = include_directories_str.split(";") - generate_embed( id, file_paths, diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 6f66144eb157..88c892425689 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -57,7 +57,7 @@ extern "C" { ::CUresult __result = (__VA_ARGS__); \ if (__result != ::CUDA_SUCCESS) { \ char const* __enum_str; \ - RTCX_EXPECTS(::rtcx::cuda->GetErrorString(__result, &__enum_str) == ::CUDA_SUCCESS, \ + RTCX_EXPECTS(::rtcx::cu->GetErrorString(__result, &__enum_str) == ::CUDA_SUCCESS, \ "Unable to get CUDA error string", \ std::runtime_error); \ auto __errstr = ::std::format("(cuda) expression `{}` failed, with error ({}): {}", \ @@ -224,6 +224,7 @@ sha256 sha256_context::finalize() DO_IT(KernelGetName) \ DO_IT(KernelGetFunction) \ DO_IT(LibraryLoadData) \ + DO_IT(LibraryLoadFromFile) \ DO_IT(LibraryGetKernel) \ DO_IT(LibraryGetKernelCount) \ DO_IT(LibraryEnumerateKernels) \ @@ -403,7 +404,7 @@ struct LibNVJitLink { } }; -static std::optional cuda; +static std::optional cu; static std::optional nvrtc; static std::optional nvjitlink; static std::optional init_libraries_flag{std::in_place}; @@ -414,9 +415,9 @@ static std::optional teardown_libraries_flag{std::in_place}; void initialize() { std::call_once(*init_libraries_flag, [] { - cuda.emplace(LibCuda::_load()); + cu.emplace(LibCuda::_load()); RTCX_EXPECTS( - cuda->Init(0) == CUDA_SUCCESS, "Failed to initialize CUDA driver API", std::runtime_error); + cu->Init(0) == CUDA_SUCCESS, "Failed to initialize CUDA driver API", std::runtime_error); nvrtc.emplace(LibNVRTC::_load()); nvjitlink.emplace(LibNVJitLink::_load()); }); @@ -425,7 +426,7 @@ void initialize() void teardown() { std::call_once(*teardown_libraries_flag, [] { - cuda.reset(); + cu.reset(); nvrtc.reset(); nvjitlink.reset(); init_libraries_flag.reset(); @@ -709,12 +710,12 @@ kernel_occupancy_config kernel_ref::max_occupancy_config(std::size_t dynamic_sha { std::int32_t min_grid_size; std::int32_t block_size; - RTCX_CHECK_CUDA(cuda->OccupancyMaxPotentialBlockSize(&min_grid_size, - &block_size, - reinterpret_cast(handle_), - nullptr, - dynamic_shared_memory_bytes, - block_size_limit)); + RTCX_CHECK_CUDA(cu->OccupancyMaxPotentialBlockSize(&min_grid_size, + &block_size, + reinterpret_cast(handle_), + nullptr, + dynamic_shared_memory_bytes, + block_size_limit)); return kernel_occupancy_config{.min_grid_size = static_cast(min_grid_size), .block_size = static_cast(block_size)}; @@ -744,7 +745,7 @@ void kernel_ref::launch(cuda_dim3 grid_dim, .numAttrs = 0}; RTCX_CHECK_CUDA( - cuda->LaunchKernelEx(&cfg, reinterpret_cast(handle_), kernel_params, nullptr)); + cu->LaunchKernelEx(&cfg, reinterpret_cast(handle_), kernel_params, nullptr)); } void kernel_ref::launch_cooperative(cuda_dim3 grid_dim, @@ -759,29 +760,29 @@ void kernel_ref::launch_cooperative(cuda_dim3 grid_dim, RTCX_EXPECTS( kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); - RTCX_CHECK_CUDA(cuda->LaunchCooperativeKernel(reinterpret_cast(handle_), - grid_dim.x, - grid_dim.y, - grid_dim.z, - block_dim.x, - block_dim.y, - block_dim.z, - shared_mem_bytes, - stream, - kernel_params)); + RTCX_CHECK_CUDA(cu->LaunchCooperativeKernel(reinterpret_cast(handle_), + grid_dim.x, + grid_dim.y, + grid_dim.z, + block_dim.x, + block_dim.y, + block_dim.z, + shared_mem_bytes, + stream, + kernel_params)); } std::string_view kernel_ref::get_name() const { char const* name; - RTCX_CHECK_CUDA(cuda->KernelGetName(&name, handle_)); + RTCX_CHECK_CUDA(cu->KernelGetName(&name, handle_)); return std::string_view{name == nullptr ? "" : name}; } library_t::~library_t() { if (handle_ != nullptr) { - if (cuda->LibraryUnload(handle_) != CUDA_SUCCESS) { std::terminate(); } + if (cu->LibraryUnload(handle_) != CUDA_SUCCESS) { std::terminate(); } } } @@ -790,10 +791,29 @@ library load_library(std::span binary) CUlibrary handle; RTCX_CHECK_CUDA( - cuda->LibraryLoadData(&handle, binary.data(), nullptr, nullptr, 0, nullptr, nullptr, 0)); + cu->LibraryLoadData(&handle, binary.data(), nullptr, nullptr, 0, nullptr, nullptr, 0)); RTCX_DEFER([&] { - if (handle != nullptr) { RTCX_CHECK_CUDA(cuda->LibraryUnload(handle)); } + if (handle != nullptr) { RTCX_CHECK_CUDA(cu->LibraryUnload(handle)); } + }); + + auto library = std::make_shared(handle); + + handle = nullptr; + + return library; +} + +library load_library_from_file(char const* path) +{ + RTCX_EXPECTS(path != nullptr, "Library path must not be null", std::logic_error); + + CUlibrary handle; + + RTCX_CHECK_CUDA(cu->LibraryLoadFromFile(&handle, path, nullptr, nullptr, 0, nullptr, nullptr, 0)); + + RTCX_DEFER([&] { + if (handle != nullptr) { RTCX_CHECK_CUDA(cu->LibraryUnload(handle)); } }); auto library = std::make_shared(handle); @@ -865,19 +885,19 @@ byte_buffer link_library(link_params const& params) kernel_ref library_t::get_kernel(char const* name) const { CUkernel kernel; - RTCX_CHECK_CUDA(cuda->LibraryGetKernel(&kernel, handle_, name)); + RTCX_CHECK_CUDA(cu->LibraryGetKernel(&kernel, handle_, name)); return kernel_ref{kernel}; } std::vector library_t::enumerate_kernels() const { std::uint32_t num_kernels; - RTCX_CHECK_CUDA(cuda->LibraryGetKernelCount(&num_kernels, handle_)); + RTCX_CHECK_CUDA(cu->LibraryGetKernelCount(&num_kernels, handle_)); std::vector kernels; kernels.resize(num_kernels); - RTCX_CHECK_CUDA(cuda->LibraryEnumerateKernels(kernels.data(), num_kernels, handle_)); + RTCX_CHECK_CUDA(cu->LibraryEnumerateKernels(kernels.data(), num_kernels, handle_)); std::vector result; for (CUkernel k : kernels) { @@ -921,8 +941,7 @@ cache_t::cache_t(std::string cache_dir, std::string tmp_dir, cache_limits const& limits, bool preload, - bool disable, - bool materialize_all) + bool disable) : enabled_{!disable}, cache_dir_{std::move(cache_dir)}, tmp_dir_{std::move(tmp_dir)}, @@ -932,7 +951,7 @@ cache_t::cache_t(std::string cache_dir, libraries_cache_{limits.num_mem_libraries}, tick_{0} { - if (preload) { preload_from_disk(materialize_all); } + if (preload) { preload_from_disk(); } } std::string const& cache_t::get_cache_dir() { return cache_dir_; } @@ -989,6 +1008,24 @@ std::optional get_disk_blob(std::string const& cache_dir, return std::make_shared(std::move(*blob)); } +std::optional get_disk_library(std::string const& cache_dir, sha256 const& sha) +{ + auto hex = sha.to_hex_string(); + auto path = std::format("{}/{}.cuLibrary.bin", cache_dir, hex.view()); + + CUlibrary handle; + auto errc = + cu->LibraryLoadFromFile(&handle, path.c_str(), nullptr, nullptr, 0, nullptr, nullptr, 0); + + if (errc == CUDA_ERROR_FILE_NOT_FOUND) { return std::nullopt; } + + RTCX_EXPECTS(errc == CUDA_SUCCESS, + std::format("Failed to load library `{}` from RTCX cache file", path), + std::runtime_error); + + return std::make_shared(handle); +} + std::pair, std::vector> get_disk_entries( std::string const& cache_dir) { @@ -1213,15 +1250,15 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, counter_.library_mem_misses.incr(); // check disk cache - std::optional disk_blob = std::nullopt; - if (enabled_) { disk_blob = get_disk_blob(cache_dir_, "cuLibrary", sha); } + std::optional disk_library = std::nullopt; + if (enabled_) { disk_library = get_disk_library(cache_dir_, sha); } std::promise promise; auto fut = promise.get_future().share(); auto cache_fut = fut; auto ret_fut = fut; - if (disk_blob.has_value()) { + if (disk_library.has_value()) { counter_.library_disk_hits.incr(); libraries_cache_.insert(sha, std::move(cache_fut), current_tick); @@ -1231,8 +1268,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, lock_.unlock(); unlocked = true; - auto lib = load_library((*disk_blob)->view()); - promise.set_value(std::move(lib)); + promise.set_value(std::move(*disk_library)); return ret_fut; @@ -1306,7 +1342,7 @@ void cache_t::clear_memory_store() void cache_t::clear_disk_store() { evict_disk_entries(cache_dir_, 0); } -void cache_t::preload_from_disk(bool materialize_all) +void cache_t::preload_from_disk() { auto [paths, access_times] = get_disk_entries(cache_dir_); @@ -1329,25 +1365,25 @@ void cache_t::preload_from_disk(bool materialize_all) for (auto index : ranking_indices) { auto path = paths[index]; try { - auto file_name = std::filesystem::path{path}.filename().string(); - auto sha_str_end = file_name.find('.'); - auto sha_str = file_name.substr(0, sha_str_end); - auto data = blob_t::from_file(path.c_str()); - if (!data.has_value()) { continue; } + auto file_name = std::filesystem::path{path}.filename().string(); + auto sha_str = file_name.substr(0, file_name.find('.')); + auto sha = sha256::parse(sha_str); if (path.ends_with(".blob.bin")) { + auto data = blob_t::from_file(path.c_str()); + if (!data.has_value()) { continue; } auto blob = std::make_shared(std::move(*data)); std::promise promise; auto fut = promise.get_future().share(); promise.set_value(std::move(blob)); - blobs_cache_.insert(sha256::parse(sha_str), std::move(fut), tick_); + blobs_cache_.insert(sha, std::move(fut), tick_); } else if (path.ends_with(".cuLibrary.bin")) { - auto lib = load_library(data->view()); - if (materialize_all) { [[maybe_unused]] auto kernels = lib->enumerate_kernels(); } + auto lib = get_disk_library(cache_dir_, sha); + if (!lib.has_value()) { continue; } std::promise promise; auto fut = promise.get_future().share(); - promise.set_value(std::move(lib)); - libraries_cache_.insert(sha256::parse(sha_str), std::move(fut), tick_); + promise.set_value(std::move(*lib)); + libraries_cache_.insert(sha, std::move(fut), tick_); } } catch (std::exception const& e) { // ignore any errors during preload diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 1ab193b5d6bb..7f2f6bd665d3 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -685,8 +685,7 @@ struct cache_t { std::string tmp_dir, cache_limits const& limits, bool preload, - bool disable, - bool materialize_all); + bool disable); cache_t(cache_t const&) = delete; cache_t& operator=(cache_t const&) = delete; cache_t(cache_t&&) = delete; @@ -783,10 +782,8 @@ struct cache_t { /*** * @brief Pre-load the JIT program cache from disk into memory during initialization, allowing for * faster retrieval and execution of previously compiled kernels at runtime. - * @param materialize_all Whether to make the compiled kernels fully materialized in memory to - * improve runtime stability. */ - void preload_from_disk(bool materialize_all); + void preload_from_disk(); /*** * @brief Enable the cache, allowing it to store and retrieve compiled blobs and libraries in @@ -815,11 +812,18 @@ struct cache_t { * @brief Load a compiled library from binary data * * @param binary Span of bytes containing the compiled library binary data - * @param type Binary type of the library (e.g., CUBIN, PTX) * @return A library object representing the loaded library with launchable kernels */ [[nodiscard]] library load_library(std::span binary); +/** + * @brief Load a compiled library from binary data + * + * @param path Path to the file containing the library binary data + * @return A library object representing the loaded library with launchable kernels + */ +[[nodiscard]] library load_library_from_file(char const* path); + /** * @brief Link multiple compiled binary fragments into a single binary blob containing the linked * library diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 4f9dc4c820da..07368b5d32ca 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -45,8 +45,7 @@ void context::ensure_jit_cache_initialized() _config.jit_tmp_dir, limits, bool{_config.preload_jit_cache}, - bool{_config.disable_jit_cache}, - bool{_config.prematerialize_jit_cache}); + bool{_config.disable_jit_cache}); if (_config.clear_jit_cache) { _rtcx_cache->clear_memory_store(); @@ -113,12 +112,11 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { std::call_once(*_context_init_flag, [&]() { - bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); - bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); - bool preload_jit_cache = get_bool_env_or("LIBCUDF_JIT_PRELOAD_CACHE", false); - bool prematerialize_jit_cache = get_bool_env_or("LIBCUDF_JIT_PREMATERIALIZE_CACHE", false); - bool disable_jit_cache = get_bool_env_or("LIBCUDF_JIT_CACHE_DISABLED", false); - bool clear_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); + bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); + bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); + bool preload_jit_cache = get_bool_env_or("LIBCUDF_JIT_PRELOAD_CACHE", false); + bool disable_jit_cache = get_bool_env_or("LIBCUDF_JIT_CACHE_DISABLED", false); + bool clear_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16'384U); auto kernel_cache_limit_disk = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_DISK", 131'072U); @@ -133,7 +131,6 @@ void initialize(init_flags flags) context_config cfg{.dump_codegen = dump_codegen, .use_jit = use_jit, .preload_jit_cache = preload_jit_cache, - .prematerialize_jit_cache = prematerialize_jit_cache, .disable_jit_cache = disable_jit_cache, .clear_jit_cache = clear_jit_cache, .rtcx_cache_dir = rtcx_cache_dir, diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index e216f067855a..5e6485a26b89 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -24,12 +24,11 @@ class program_cache; class jit_bundle_t; struct [[nodiscard]] context_config { - bool dump_codegen : 1 = false; - bool use_jit : 1 = false; - bool preload_jit_cache : 1 = false; - bool prematerialize_jit_cache : 1 = false; - bool disable_jit_cache : 1 = false; - bool clear_jit_cache : 1 = false; + bool dump_codegen : 1 = false; + bool use_jit : 1 = false; + bool preload_jit_cache : 1 = false; + bool disable_jit_cache : 1 = false; + bool clear_jit_cache : 1 = false; std::string rtcx_cache_dir = {}; std::string jit_bundle_dir = {}; std::string jit_pch_dir = {}; From 68e7974e5a66af35efa7e1fa0ef8ea2b92029545 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 19 Mar 2026 02:45:10 +0000 Subject: [PATCH 085/254] added lz4 dep --- conda/environments/all_cuda-129_arch-aarch64.yaml | 1 + conda/environments/all_cuda-129_arch-x86_64.yaml | 1 + conda/environments/all_cuda-131_arch-aarch64.yaml | 1 + conda/environments/all_cuda-131_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + 5 files changed, 5 insertions(+) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 32c046e63016..dcf23dff59af 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -46,6 +46,7 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0 - librmm==26.6.*,>=0.0.0a0 +- lz4 - make - mmh3 - moto>=4.0.8 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 68b145973452..2d397fef6dd1 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -46,6 +46,7 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0 - librmm==26.6.*,>=0.0.0a0 +- lz4 - make - mmh3 - moto>=4.0.8 diff --git a/conda/environments/all_cuda-131_arch-aarch64.yaml b/conda/environments/all_cuda-131_arch-aarch64.yaml index 63884df5c3aa..1647aaf2c89b 100644 --- a/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -46,6 +46,7 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0 - librmm==26.6.*,>=0.0.0a0 +- lz4 - make - mmh3 - moto>=4.0.8 diff --git a/conda/environments/all_cuda-131_arch-x86_64.yaml b/conda/environments/all_cuda-131_arch-x86_64.yaml index 7e4ba003f40b..91e37b054d5b 100644 --- a/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -46,6 +46,7 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0 - librmm==26.6.*,>=0.0.0a0 +- lz4 - make - mmh3 - moto>=4.0.8 diff --git a/dependencies.yaml b/dependencies.yaml index 011abf8a56e6..39f11d88d8e7 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -485,6 +485,7 @@ dependencies: - dlpack>=0.8,<1.0 - zlib>=1.2.13 - python-zstd>=1.5.7.2 + - lz4 specific: - output_types: conda matrices: From 71e25ce37e4eaad384d6978ab572dfd2475d72e8 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 19 Mar 2026 03:12:59 +0000 Subject: [PATCH 086/254] update recipe --- conda/recipes/cudf/recipe.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index 4d8a9f2f241e..d91ba2d9ab5d 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -68,6 +68,7 @@ requirements: - ${{ compiler("cuda") }} - cuda-version =${{ cuda_version }} - ${{ stdlib("c") }} + - lz4 host: - cython >=3.2.2 - pip From cdb66c6700c9c26d3cec33925f108db7bd9d36ce Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 19 Mar 2026 03:57:34 +0000 Subject: [PATCH 087/254] update lz4 dependency to version 4.4.5 in environment YAML files --- conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-131_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-131_arch-x86_64.yaml | 2 +- dependencies.yaml | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index dcf23dff59af..e0c73643b82d 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -46,7 +46,7 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0 - librmm==26.6.*,>=0.0.0a0 -- lz4 +- lz4>=4.4.5 - make - mmh3 - moto>=4.0.8 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 2d397fef6dd1..a9488c9ab7c3 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -46,7 +46,7 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0 - librmm==26.6.*,>=0.0.0a0 -- lz4 +- lz4>=4.4.5 - make - mmh3 - moto>=4.0.8 diff --git a/conda/environments/all_cuda-131_arch-aarch64.yaml b/conda/environments/all_cuda-131_arch-aarch64.yaml index 1647aaf2c89b..8fa2438cfe7d 100644 --- a/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -46,7 +46,7 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0 - librmm==26.6.*,>=0.0.0a0 -- lz4 +- lz4>=4.4.5 - make - mmh3 - moto>=4.0.8 diff --git a/conda/environments/all_cuda-131_arch-x86_64.yaml b/conda/environments/all_cuda-131_arch-x86_64.yaml index 91e37b054d5b..b0eb200c1ef2 100644 --- a/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -46,7 +46,7 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0 - librmm==26.6.*,>=0.0.0a0 -- lz4 +- lz4>=4.4.5 - make - mmh3 - moto>=4.0.8 diff --git a/dependencies.yaml b/dependencies.yaml index a2929f2742b2..8456d74c5609 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -487,8 +487,6 @@ dependencies: - cxx-compiler - dlpack>=0.8,<1.0 - zlib>=1.2.13 - - python-zstd>=1.5.7.2 - - lz4 specific: - output_types: conda matrices: @@ -508,6 +506,8 @@ dependencies: packages: - flatbuffers==24.3.25 - librdkafka>=2.8.0,<2.9.0 + - lz4>=4.4.5 + - python-zstd>=1.5.7.2 depends_on_libnvcomp: common: - output_types: conda From b64eb2e1eec2d5ae4ed94693170e9fc65f83482f Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 20 Mar 2026 13:12:16 +0000 Subject: [PATCH 088/254] refactor: update linking parameters to use structured fragments for improved clarity and functionality --- cpp/src/librtcx/rtcx.cpp | 51 +++++++++++++++++++++++----------------- cpp/src/librtcx/rtcx.hpp | 29 ++++++++++++++++++----- 2 files changed, 53 insertions(+), 27 deletions(-) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 88c892425689..e07c0dbbb969 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -123,7 +123,7 @@ std::string join_strings(std::span strings, std::string_view separat strings.end(), size_t{0}, [](size_t total, size_t str_size) { return total + str_size; }, - [](auto const& str) { return str.size(); }); + [](auto& str) { return str.size(); }); auto separator_size = separator.size() * (strings.size() - 1); @@ -270,7 +270,7 @@ namespace { void* load_dll(std::string_view base_name, std::span names) { - for (auto const& name : names) { + for (auto& name : names) { void* handle = ::dlopen(name.c_str(), RTLD_NOW | RTLD_LOCAL); if (handle != nullptr) { return handle; } } @@ -526,12 +526,12 @@ void log_nvrtc_result(compile_params const& params, : "failed with error"; std::string headers_str; - for (auto const& header : params.header_include_names) { + for (auto& header : params.header_include_names) { headers_str = std::format("{}\t{}\n", headers_str, header); } std::string options_str; - for (auto const& option : params.options) { + for (auto& option : params.options) { options_str = std::format("{}\t{}\n", options_str, option); } @@ -602,12 +602,16 @@ void log_nvJitLink_result(link_params const& params, if (info_log.empty() && error_log.empty()) { return; } std::string fragments_str; - for (auto const& fragment_name : params.fragment_names) { - fragments_str = std::format("{}\t{}\n", fragments_str, fragment_name); + for (auto& frag : params.file_fragments) { + fragments_str = std::format("{}\t{}\n", fragments_str, frag.path); + } + + for (auto& frag : params.memory_fragments) { + fragments_str = std::format("{}\t{}\n", fragments_str, frag.name); } std::string link_options_str; - for (auto const& option : params.link_options) { + for (auto& option : params.link_options) { link_options_str = std::format("{}\t{}\n", link_options_str, option); } @@ -829,16 +833,17 @@ byte_buffer link_library(link_params const& params) RTCX_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, "Only CUBIN and PTX output types are supported for linking modules", std::logic_error); - RTCX_EXPECTS(params.fragments.size() == params.fragment_binary_types.size(), - "Mismatched number of fragments and fragment binary types", + RTCX_EXPECTS(params.file_fragments.size() != 0 || params.memory_fragments.size() != 0, + "At least one fragment must be provided for linking", std::logic_error); - RTCX_EXPECTS(params.fragments.size() == params.fragment_names.size(), - "Mismatched number of fragments and fragment names", - std::logic_error); - RTCX_EXPECTS(params.fragments.size() > 0, "No fragments provided for linking", std::logic_error); - for (auto& frag : params.fragments) { - RTCX_EXPECTS(frag.size_bytes() > 0, "Fragment binary data must be non-empty", std::logic_error); + for (auto& frag : params.file_fragments) { + RTCX_EXPECTS(frag.path != nullptr, "Fragment file path must not be empty", std::logic_error); + } + + for (auto& frag : params.memory_fragments) { + RTCX_EXPECTS( + frag.data.size_bytes() > 0, "Fragment binary data must be non-empty", std::logic_error); } nvJitLinkHandle handle = nullptr; @@ -848,12 +853,16 @@ byte_buffer link_library(link_params const& params) RTCX_DEFER([&] { nvjitlink->Destroy(&handle); }); - for (std::size_t i = 0; i < params.fragments.size(); i++) { - auto name = params.fragment_names[i]; - auto fragment = params.fragments[i]; - auto bin_type = to_nvjitlink_input_type(params.fragment_binary_types[i]); - RTCX_CHECK_NVJITLINK( - nvjitlink->AddData(handle, bin_type, fragment.data(), fragment.size_bytes(), name)); + for (auto& frag : params.file_fragments) { + RTCX_CHECK_NVJITLINK(nvjitlink->AddFile(handle, to_nvjitlink_input_type(frag.type), frag.path)); + } + + for (auto& frag : params.memory_fragments) { + RTCX_CHECK_NVJITLINK(nvjitlink->AddData(handle, + to_nvjitlink_input_type(frag.type), + frag.data.data(), + frag.data.size_bytes(), + frag.name)); } auto link_result = nvjitlink->Complete(handle); diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 7f2f6bd665d3..be0ac79d087b 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -508,16 +508,33 @@ struct [[nodiscard]] compile_params { binary_type target_type = binary_type::LTO_IR; //< Output binary type }; +/** + * @brief Represents a binary fragment in memory to be linked into a library + */ +struct memory_fragment { + std::span data = {}; //< Binary data for the fragment + binary_type type = binary_type::CUBIN; //< Binary type of the fragment data + char const* name = nullptr; //< Debug name for the fragment +}; + +/** + * @brief Represents a binary fragment to be linked into a library + */ +struct file_fragment { + char const* path = nullptr; //< Path to the binary fragment file + binary_type type = binary_type::CUBIN; //< Binary type of the fragment data +}; + /** * @brief Parameters for linking multiple compiled fragments into a single library */ struct [[nodiscard]] link_params { - char const* name = nullptr; //< Debug name for the linked library - binary_type output_type = binary_type::CUBIN; //< Output binary type - std::span const> fragments = {}; //< Binary data for each fragment - std::span fragment_binary_types = {}; //< Binary type for each fragment - std::span fragment_names = {}; //< Debug name for each fragment - std::span link_options = {}; //< NVJITLink options + char const* name = nullptr; //< Debug name for the linked library + binary_type output_type = binary_type::CUBIN; //< Output binary type + std::span file_fragments = {}; //< Binary data for each fragment + std::span memory_fragments = + {}; //< Memory-resident binary fragments to link + std::span link_options = {}; //< NVJITLink options }; namespace detail { From b9a21531e428f0e59fb9f77e0d156ea3a35db4a5 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 20 Mar 2026 22:10:25 -0500 Subject: [PATCH 089/254] Fix Python dependencies on lz4 and zstandard --- conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-131_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-131_arch-x86_64.yaml | 2 +- conda/recipes/libcudf/recipe.yaml | 2 ++ cpp/src/librtcx/CMake/embed.in.py | 4 ++-- dependencies.yaml | 4 +++- python/libcudf/pyproject.toml | 2 ++ 8 files changed, 13 insertions(+), 7 deletions(-) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index e0c73643b82d..61377fdb3e36 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -81,7 +81,6 @@ dependencies: - pytest-xdist - python-confluent-kafka - python-xxhash -- python-zstd>=1.5.7.2 - python>=3.11 - pytorch>=2.4.0 - rapids-build-backend>=0.4.0,<0.5.0 @@ -104,4 +103,5 @@ dependencies: - typing_extensions>=4.0.0 - zlib>=1.2.13 - zstandard +- zstandard>=0.25.0 name: all_cuda-129_arch-aarch64 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index a9488c9ab7c3..a4a396b4be82 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -81,7 +81,6 @@ dependencies: - pytest-xdist - python-confluent-kafka - python-xxhash -- python-zstd>=1.5.7.2 - python>=3.11 - pytorch>=2.4.0 - rapids-build-backend>=0.4.0,<0.5.0 @@ -104,4 +103,5 @@ dependencies: - typing_extensions>=4.0.0 - zlib>=1.2.13 - zstandard +- zstandard>=0.25.0 name: all_cuda-129_arch-x86_64 diff --git a/conda/environments/all_cuda-131_arch-aarch64.yaml b/conda/environments/all_cuda-131_arch-aarch64.yaml index 8fa2438cfe7d..fcc42b27286f 100644 --- a/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -81,7 +81,6 @@ dependencies: - pytest-xdist - python-confluent-kafka - python-xxhash -- python-zstd>=1.5.7.2 - python>=3.11 - pytorch>=2.10.0 - rapids-build-backend>=0.4.0,<0.5.0 @@ -104,4 +103,5 @@ dependencies: - typing_extensions>=4.0.0 - zlib>=1.2.13 - zstandard +- zstandard>=0.25.0 name: all_cuda-131_arch-aarch64 diff --git a/conda/environments/all_cuda-131_arch-x86_64.yaml b/conda/environments/all_cuda-131_arch-x86_64.yaml index b0eb200c1ef2..619ca86fd8ee 100644 --- a/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -81,7 +81,6 @@ dependencies: - pytest-xdist - python-confluent-kafka - python-xxhash -- python-zstd>=1.5.7.2 - python>=3.11 - pytorch>=2.10.0 - rapids-build-backend>=0.4.0,<0.5.0 @@ -104,4 +103,5 @@ dependencies: - typing_extensions>=4.0.0 - zlib>=1.2.13 - zstandard +- zstandard>=0.25.0 name: all_cuda-131_arch-x86_64 diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 67f921d9a679..d000a7c97f7c 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -93,6 +93,8 @@ cache: - flatbuffers =${{ flatbuffers_version }} - rapids-logger =0.2 - zlib ${{ zlib_version }} + - lz4 >=4.4.5 + - zstandard >=0.25.0 outputs: - package: diff --git a/cpp/src/librtcx/CMake/embed.in.py b/cpp/src/librtcx/CMake/embed.in.py index ad043da0d894..ecf300837f1b 100644 --- a/cpp/src/librtcx/CMake/embed.in.py +++ b/cpp/src/librtcx/CMake/embed.in.py @@ -5,7 +5,7 @@ from typing import NamedTuple import lz4.block -import zstd +import zstandard def merge_bytes_with_null_terminators( @@ -45,7 +45,7 @@ def compress_bytes(data: bytes, compression: str) -> bytes: data, mode="high_compression", compression=12, store_size=False ) elif compression == "zstd": - return zstd.compress(data, 22) + return zstandard.compress(data, 22) def generate_cxx_source_files_data( diff --git a/dependencies.yaml b/dependencies.yaml index 8456d74c5609..4d425e7adb69 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -506,8 +506,10 @@ dependencies: packages: - flatbuffers==24.3.25 - librdkafka>=2.8.0,<2.9.0 + - output_types: [conda, pyproject, requirements] + packages: - lz4>=4.4.5 - - python-zstd>=1.5.7.2 + - zstandard>=0.25.0 depends_on_libnvcomp: common: - output_types: conda diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index f581fd85f48b..0961415d1184 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -71,8 +71,10 @@ requires = [ "cmake>=3.30.4", "libkvikio==26.6.*,>=0.0.0a0", "librmm==26.6.*,>=0.0.0a0", + "lz4>=4.4.5", "ninja", "nvidia-libnvcomp==5.1.0.21", "nvidia-nvjitlink>=13.0,<14", "rapids-logger==0.2.*,>=0.0.0a0", + "zstandard>=0.25.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From ea11f0b2f5910f701547c71b2fc231a04ac2f137 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 21 Mar 2026 14:25:28 +0000 Subject: [PATCH 090/254] Refactor JIT installation error handling and remove unused kernel name retrieval --- cpp/src/jit/jit.cpp | 14 +++++++++----- cpp/src/librtcx/rtcx.cpp | 8 -------- cpp/src/librtcx/rtcx.hpp | 5 ----- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index 7e79fe2170db..c4b2bd9f1f27 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -192,8 +192,15 @@ void install_cudf_jit_files(std::string const& target_dir, std::string const& tm // rename the temporary directory to the target install directory if (rename(tmp_path, target_dir.c_str()) == -1) { - throw_posix(std::format("Failed to rename temporary JIT install directory to ({})", target_dir), - "rename"); + auto errc = errno; + // another process created it + if (errc == ENOTEMPTY || errc == EEXIST) { + std::filesystem::remove_all(tmp_path); + } else { + throw_posix( + std::format("Failed to rename temporary JIT install directory to ({})", target_dir), + "rename"); + } } } @@ -308,15 +315,12 @@ std::tuple compile_library_uncached( // TODO: experiment with: // --fdevice-time-trace=jit_comp_trace.json // --time=compile_trace.json - // -time // --restrict // --relocatable-device-code // --extensible-whole-program - // --device-debug // --use_fast_math // --dlink-time-opt // --gen-opt-lto - // --no-cache // --create-pch // --use-pch // --pch-dir diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index e07c0dbbb969..066f8ad9ee23 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -221,7 +221,6 @@ sha256 sha256_context::finalize() DO_IT(LaunchKernel) \ DO_IT(LaunchKernelEx) \ DO_IT(LaunchCooperativeKernel) \ - DO_IT(KernelGetName) \ DO_IT(KernelGetFunction) \ DO_IT(LibraryLoadData) \ DO_IT(LibraryLoadFromFile) \ @@ -776,13 +775,6 @@ void kernel_ref::launch_cooperative(cuda_dim3 grid_dim, kernel_params)); } -std::string_view kernel_ref::get_name() const -{ - char const* name; - RTCX_CHECK_CUDA(cu->KernelGetName(&name, handle_)); - return std::string_view{name == nullptr ? "" : name}; -} - library_t::~library_t() { if (handle_ != nullptr) { diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index be0ac79d087b..c1a83266f193 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -454,11 +454,6 @@ struct [[nodiscard]] kernel_ref { * @brief Retrieves the underlying CUDA kernel handle */ [[nodiscard]] CUkernel get() const { return handle_; } - - /** - * @brief Retrieves the name of the kernel - */ - [[nodiscard]] std::string_view get_name() const; }; /** From 422ea0a23879d534c500210908d63b34b0303219 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 21 Mar 2026 14:40:45 +0000 Subject: [PATCH 091/254] drop lz4 dependencies and support from rtcx --- conda/recipes/cudf/recipe.yaml | 1 - conda/recipes/libcudf/recipe.yaml | 1 - cpp/CMakeLists.txt | 16 ++-------------- cpp/src/jit/jit.cpp | 16 ++-------------- cpp/src/librtcx/CMake/embed.cmake | 7 ++----- cpp/src/librtcx/CMake/embed.in.py | 9 +++------ cpp/src/librtcx/README.md | 1 - 7 files changed, 9 insertions(+), 42 deletions(-) diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index d91ba2d9ab5d..4d8a9f2f241e 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -68,7 +68,6 @@ requirements: - ${{ compiler("cuda") }} - cuda-version =${{ cuda_version }} - ${{ stdlib("c") }} - - lz4 host: - cython >=3.2.2 - pip diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index d000a7c97f7c..15d04a56424f 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -93,7 +93,6 @@ cache: - flatbuffers =${{ flatbuffers_version }} - rapids-logger =0.2 - zlib ${{ zlib_version }} - - lz4 >=4.4.5 - zstandard >=0.25.0 outputs: diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3e51e28c3ba7..a03791239352 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -308,9 +308,6 @@ include(src/librtcx/CMake/embed.cmake) # find zstd include(cmake/thirdparty/get_zstd.cmake) -# find lz4 -include(cmake/thirdparty/get_lz4.cmake) - # Workaround until https://github.com/rapidsai/rapids-cmake/issues/176 is resolved if(NOT BUILD_SHARED_LIBS) include("${rapids-cmake-dir}/export/find_package_file.cmake") @@ -1027,17 +1024,8 @@ target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_A target_link_libraries( cudf PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ - PRIVATE $ - cuco::cuco - ZLIB::ZLIB - nvcomp::nvcomp - kvikio::kvikio - nanoarrow::nanoarrow - zstd - liblz4_static - CUDA::nvJitLink - CUDA::nvrtc - CUDA::cuda_driver + PRIVATE $ cuco::cuco ZLIB::ZLIB nvcomp::nvcomp + kvikio::kvikio nanoarrow::nanoarrow zstd CUDA::nvJitLink CUDA::nvrtc CUDA::cuda_driver ) # Add Conda library, and include paths if specified diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index c4b2bd9f1f27..21023d0efc14 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -114,23 +113,12 @@ rtcx::byte_buffer decompress_blob(std::span compressed_binary, size_t uncompressed_size, std::string_view compression) { - CUDF_EXPECTS(compression == "none" || compression == "lz4" || compression == "zstd", + CUDF_EXPECTS(compression == "none" || compression == "zstd", +std::format("Unsupported compression type specified: {}", compression), std::runtime_error); auto decompressed = rtcx::byte_buffer::make(uncompressed_size); - if (compression == "lz4") { - int errc = LZ4_decompress_safe(reinterpret_cast(compressed_binary.data()), - reinterpret_cast(decompressed.data()), - compressed_binary.size(), - uncompressed_size); - - CUDF_EXPECTS( - errc == static_cast(uncompressed_size), - +std::format("Failed to decompress embedded RTC source files with LZ4, error code {}", errc), - std::runtime_error); - - } else if (compression == "zstd") { + if (compression == "zstd") { size_t errc = ZSTD_decompress( decompressed.data(), uncompressed_size, compressed_binary.data(), compressed_binary.size()); diff --git a/cpp/src/librtcx/CMake/embed.cmake b/cpp/src/librtcx/CMake/embed.cmake index 8f2227d75a3b..f239f56eceb7 100644 --- a/cpp/src/librtcx/CMake/embed.cmake +++ b/cpp/src/librtcx/CMake/embed.cmake @@ -104,11 +104,8 @@ function(jit_embed) message(FATAL_ERROR "COMPRESSION argument is required") endif() - if(NOT ARG_COMPRESSION STREQUAL "none" - AND NOT ARG_COMPRESSION STREQUAL "lz4" - AND NOT ARG_COMPRESSION STREQUAL "zstd" - ) - message(FATAL_ERROR "COMPRESSION argument must be either none, lz4, or, zstd") + if(NOT ARG_COMPRESSION STREQUAL "none" AND NOT ARG_COMPRESSION STREQUAL "zstd") + message(FATAL_ERROR "COMPRESSION argument must be either none or zstd") endif() if(NOT DEFINED jitembed_${TARGET}_incdir__source_files) diff --git a/cpp/src/librtcx/CMake/embed.in.py b/cpp/src/librtcx/CMake/embed.in.py index ecf300837f1b..ca455989fb16 100644 --- a/cpp/src/librtcx/CMake/embed.in.py +++ b/cpp/src/librtcx/CMake/embed.in.py @@ -4,7 +4,6 @@ import os from typing import NamedTuple -import lz4.block import zstandard @@ -36,14 +35,12 @@ def load_file_bytes(file_path: str) -> bytes: def compress_bytes(data: bytes, compression: str) -> bytes: - assert compression in ("none", "lz4", "zstd"), "Invalid compression type" + assert compression in ("none", "zstd"), ( + f"Invalid compression type: {compression}, expected 'none' or 'zstd'" + ) if compression == "none": return data - elif compression == "lz4": - return lz4.block.compress( - data, mode="high_compression", compression=12, store_size=False - ) elif compression == "zstd": return zstandard.compress(data, 22) diff --git a/cpp/src/librtcx/README.md b/cpp/src/librtcx/README.md index aa258d6cc988..be59fa684960 100644 --- a/cpp/src/librtcx/README.md +++ b/cpp/src/librtcx/README.md @@ -13,5 +13,4 @@ RTCX (runtime-compiler extended) is a wrapper around NVRTC and NVJitLink designe ## Build-Scripting Requirements - CMake - python 3 -- python-lz4 - python-zstd From 96d254c22e1d5fccea02c8b39dad03c33af03cab Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 22 Mar 2026 12:31:51 +0000 Subject: [PATCH 092/254] update --- cpp/src/librtcx/rtcx.cpp | 9 +++++++++ cpp/src/librtcx/rtcx.hpp | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 066f8ad9ee23..69f9844c4752 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -222,6 +222,7 @@ sha256 sha256_context::finalize() DO_IT(LaunchKernelEx) \ DO_IT(LaunchCooperativeKernel) \ DO_IT(KernelGetFunction) \ + DO_IT(KernelGetName) \ DO_IT(LibraryLoadData) \ DO_IT(LibraryLoadFromFile) \ DO_IT(LibraryGetKernel) \ @@ -775,6 +776,13 @@ void kernel_ref::launch_cooperative(cuda_dim3 grid_dim, kernel_params)); } +std::string_view kernel_ref::get_name() const +{ + char const* name; + RTCX_CHECK_CUDA(cu->KernelGetName(&name, handle_)); + return std::string_view{name == nullptr ? "" : name}; +} + library_t::~library_t() { if (handle_ != nullptr) { @@ -1115,6 +1123,7 @@ void cache_blob_to_disk(std::string const& cache_dir, std::span binary, std::uint32_t limit) { + // TODO: add cuda driver and runtime version to log if (limit > 0) { auto tmp_path = std::format("{}/rtcx-bin-XXXXXX", tmp_dir); (void)tmp_path.c_str(); // to ensure null-termination for mkstemp diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index c1a83266f193..af646cdd87de 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -452,8 +452,15 @@ struct [[nodiscard]] kernel_ref { /** * @brief Retrieves the underlying CUDA kernel handle + * @return The CUDA kernel handle associated with this kernel reference */ [[nodiscard]] CUkernel get() const { return handle_; } + + /** + * @brief Retrieves the name of the kernel + * @return A string_view representing the name of the kernel + */ + [[nodiscard]] std::string_view get_name() const; }; /** From a0ee09d828bb2a37aee5ef08adf666bc00cea667 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 22 Mar 2026 12:49:58 +0000 Subject: [PATCH 093/254] Remove Jitify references and clean up CMake configuration --- cpp/CMakeLists.txt | 23 +---------------------- cpp/include/cudf/detail/operation-udf.hpp | 4 ++-- cpp/include/cudf/types.hpp | 2 +- cpp/src/librtcx/rtcx.cpp | 4 ---- 4 files changed, 4 insertions(+), 29 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a03791239352..ef10eec24bb3 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -38,7 +38,6 @@ option(USE_NVTX "Build with NVTX support" ON) option(BUILD_TESTS "Configure CMake to build tests" ON) option(BUILD_BENCHMARKS "Configure CMake to build benchmarks" OFF) option(BUILD_SHARED_LIBS "Build cuDF shared libraries" ON) -option(JITIFY_USE_CACHE "Use a file cache for JIT compiled kernels" ON) option(CUDF_BUILD_TESTUTIL "Whether to build the test utilities contained in libcudf" ON) mark_as_advanced(CUDF_BUILD_TESTUTIL) option(CUDF_LARGE_STRINGS_DISABLED "Build with large string support disabled" OFF) @@ -87,7 +86,6 @@ message(VERBOSE "CUDF: Build with NVTX support: ${USE_NVTX}") message(VERBOSE "CUDF: Configure CMake to build tests: ${BUILD_TESTS}") message(VERBOSE "CUDF: Configure CMake to build benchmarks: ${BUILD_BENCHMARKS}") message(VERBOSE "CUDF: Build cuDF shared libraries: ${BUILD_SHARED_LIBS}") -message(VERBOSE "CUDF: Use a file cache for JIT compiled kernels: ${JITIFY_USE_CACHE}") message(VERBOSE "CUDF: Build with per-thread default stream: ${CUDF_USE_PER_THREAD_DEFAULT_STREAM}") message( VERBOSE @@ -885,8 +883,6 @@ foreach(incbin_dir IN LISTS cudf_jit_embed_INCLUDE_DIRS) target_compile_options(cudf PRIVATE $<$:-Wa,-I${incbin_dir}>) endforeach() -# Anything that includes jitify needs to be compiled with _FILE_OFFSET_BITS=64 due to a limitation -# in how conda builds glibc set_source_files_properties( src/binaryop/binaryop.cpp src/jit/cache.cpp @@ -895,7 +891,6 @@ set_source_files_properties( src/rolling/grouped_rolling.cu src/rolling/rolling.cu src/transform/transform.cu - PROPERTIES COMPILE_DEFINITIONS "_FILE_OFFSET_BITS=64" ) set_property( @@ -960,9 +955,7 @@ target_compile_options( # Specify include paths for the current target and dependents target_include_directories( cudf - PUBLIC "$" - "$" - "$" + PUBLIC "$" "$" "$" PRIVATE "$" "$" @@ -976,20 +969,6 @@ target_compile_definitions( "$:${CUDF_CUDA_DEFINITIONS}>>" ) -# Disable Jitify log printing. See https://github.com/NVIDIA/jitify/issues/79 -target_compile_definitions(cudf PRIVATE "JITIFY_PRINT_LOG=0") - -if(JITIFY_USE_CACHE) - # Instruct src/jit/cache what version of cudf we are building so it can compute a cal-ver cache - # directory. We isolate this definition to the single source so it doesn't effect compiling - # caching for all of libcudf - set_property( - SOURCE src/jit/cache.cpp - APPEND - PROPERTY COMPILE_DEFINITIONS "JITIFY_USE_CACHE" "CUDF_VERSION=${PROJECT_VERSION}" - ) -endif() - # Per-thread default stream if(CUDF_USE_PER_THREAD_DEFAULT_STREAM) target_compile_definitions( diff --git a/cpp/include/cudf/detail/operation-udf.hpp b/cpp/include/cudf/detail/operation-udf.hpp index 47bb3e91170d..991405d5bd77 100644 --- a/cpp/include/cudf/detail/operation-udf.hpp +++ b/cpp/include/cudf/detail/operation-udf.hpp @@ -1,9 +1,9 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -// This file serves as a placeholder for user defined functions, so jitify can choose to override it +// This file serves as a placeholder for user defined functions, so NVRTC can choose to override it // at runtime. diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp index 7df7478f7096..42139898c573 100644 --- a/cpp/include/cudf/types.hpp +++ b/cpp/include/cudf/types.hpp @@ -361,7 +361,7 @@ class data_type { */ constexpr bool operator==(data_type const& lhs, data_type const& rhs) { - // use std::tie in the future, breaks JITIFY currently + // use std::tie in the future, breaks NVRTC currently return lhs.id() == rhs.id() && lhs.scale() == rhs.scale(); } diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 69f9844c4752..d05723389877 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -245,10 +245,6 @@ sha256 sha256_context::finalize() DO_IT(GetProgramLog) \ DO_IT(AddNameExpression) \ DO_IT(GetLoweredName) \ - DO_IT(GetPCHHeapSize) \ - DO_IT(SetPCHHeapSize) \ - DO_IT(GetPCHCreateStatus) \ - DO_IT(GetPCHHeapSizeRequired) \ DO_IT(SetFlowCallback) #define FOR_EACH_NVJITLINK_FUNC(DO_IT) \ From 11e01b325de7376dced4aec88a8bca824ff1bc0f Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 22 Mar 2026 12:59:54 +0000 Subject: [PATCH 094/254] Add TODO for glob restrictions in embed.cmake to filter source files --- cpp/src/librtcx/CMake/embed.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/src/librtcx/CMake/embed.cmake b/cpp/src/librtcx/CMake/embed.cmake index f239f56eceb7..cd5115fa1db6 100644 --- a/cpp/src/librtcx/CMake/embed.cmake +++ b/cpp/src/librtcx/CMake/embed.cmake @@ -7,6 +7,8 @@ find_package(Python3 REQUIRED COMPONENTS Interpreter) +# TODO: add glob restrictions:exclude .cpp .cc, .cmake and .cxx source files, only include .h .hpp .cuh .cu files + # This function registers a directory of include files to be embedded for JIT compilation. It # gathers the specified files, their destinations, and include directories, and stores them in # target-specific variables for later use when generating the embed. From 6bc9c0b76b96962f2e4f3782bde7f9ffb509deb2 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 22 Mar 2026 14:55:39 +0000 Subject: [PATCH 095/254] Update comments to clarify NVRTC and jit-compiling references in string_view.cuh and kernel.cu --- cpp/include/cudf/strings/string_view.cuh | 3 +-- cpp/src/rolling/jit/kernel.cu | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/include/cudf/strings/string_view.cuh b/cpp/include/cudf/strings/string_view.cuh index 67fb6e5f61e2..8275ebc6ff87 100644 --- a/cpp/include/cudf/strings/string_view.cuh +++ b/cpp/include/cudf/strings/string_view.cuh @@ -13,8 +13,7 @@ #include #endif -// This is defined when including this header in a https://github.com/NVIDIA/jitify -// or jitify2 source file. The jitify cannot include thrust headers at this time. +// This is defined when compiling with NVRTC. NVRTC cannot include thrust headers at this time. #if !defined(__CUDACC_RTC__) #include #include diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index 2a6e5ff7aa81..d701672f636d 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -112,7 +112,7 @@ CUDF_KERNEL void gpu_rolling_new(cudf::size_type nrows, } // TODO: likely faster to do a single_lane_block_reduce and a single - // atomic per block but that requires jitifying single_lane_block_reduce... + // atomic per block but that requires jit-compiling single_lane_block_reduce... if (0 == cudf::intra_word_index(threadIdx.x)) { atomicAdd(output_valid_count, warp_valid_count); } } From 060d369a40116b5aa3b87a6548cd5bf85cf65b30 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 30 Mar 2026 20:10:51 +0100 Subject: [PATCH 096/254] refactor null mask handling to use set_null_mask for consistency --- cpp/src/transform/transform.cu | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index c744002f741d..2b89bf2ff94a 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -381,7 +381,7 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, if (inputs.empty()) { // no input, set all to valid - CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0xFF, num_bytes, stream.value())); + set_null_mask(null_mask, 0, row_size, true, stream); return 0; } @@ -397,7 +397,7 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, if (scalar_is_null) { // scalar is null, all rows will be null - CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0x00, num_bytes, stream.value())); + set_null_mask(null_mask, 0, row_size, false, stream); return row_size; } } @@ -407,7 +407,7 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, if (!has_cols) { // no non-scalar columns, so all rows are valid - CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0xFF, num_bytes, stream.value())); + set_null_mask(null_mask, 0, row_size, true, stream); return 0; } @@ -428,7 +428,7 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, if (nullable_masks.empty()) { // we only have non-nullable columns, so all rows are valid - CUDF_CUDA_TRY(cudaMemsetAsync(null_mask, 0xFF, num_bytes, stream.value())); + set_null_mask(null_mask, 0, row_size, true, stream); return 0; } From 4ba506a8a0af41c4d072fc392d2ea4e43a5cdfd7 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 30 Mar 2026 20:11:24 +0100 Subject: [PATCH 097/254] properly handle cuda version for "-minimal" flag --- cpp/src/jit/helpers.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 88bee6796273..d035f43efe28 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -96,13 +96,16 @@ jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preproces int runtime_version; CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); - int constexpr min_pch_runtime_version = 12800; // CUDA 12.8 + + constexpr int min_pch_cuda_version = 12800; // CUDA 12.8 + constexpr int min_minimal_cuda_version = 12800; // CUDA 12.8 std::vector options; options.emplace_back("-arch=sm_."); - options.emplace_back("-minimal"); - if (runtime_version >= min_pch_runtime_version) { options.emplace_back("-pch"); } + if (runtime_version >= min_minimal_cuda_version) { options.emplace_back("-minimal"); } + + if (runtime_version >= min_pch_cuda_version) { options.emplace_back("-pch"); } for (auto& opt : extra_options) { options.push_back(opt); From 3aa3b20ded08d64f27eb3428ccc38568aa49c159 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 17:29:25 +0000 Subject: [PATCH 098/254] fix invocation error --- cpp/src/transform/jit/kernel.cu | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index f5248a44e816..ffade3ea4468 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -26,7 +26,7 @@ // need to put this pragma before including it to avoid PCH mismatch. // clang-format off -// This header is an inlined header that defines the GENERIC_FILTER_OP function. It is placed here +// This header is an inlined header that defines the GENERIC_TRANSFORM_OP function. It is placed here // so the symbols in the headers above can be used by it. #include // clang-format on @@ -64,12 +64,12 @@ CUDF_KERNEL void transform_kernel(size_type row_size, if constexpr (has_user_data) { auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); - cuda::std::apply(GENERIC_TRANSFORM_OP, args); + cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); } else { // TODO: static assert invocable auto args = cuda::std::tuple_cat(out_ptrs, inputs); - cuda::std::apply(GENERIC_TRANSFORM_OP, args); + cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); } Outs::map([&]() { @@ -92,11 +92,11 @@ CUDF_KERNEL void transform_kernel(size_type row_size, if constexpr (has_user_data) { auto args = cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); - cuda::std::apply(GENERIC_TRANSFORM_OP, args); + cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); } else { auto args = cuda::std::tuple_cat(out_ptrs, inputs); - cuda::std::apply(GENERIC_TRANSFORM_OP, args); + cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); } Outs::map([&]() { From e1a6f43bd7d314a0dc8a70d31064eacd473b4d66 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 17:50:41 +0000 Subject: [PATCH 099/254] Use global namespace for standard library functions in rtcx.cpp --- cpp/src/librtcx/rtcx.cpp | 44 +++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index d05723389877..506ada6d5698 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -142,19 +142,19 @@ std::string join_strings(std::span strings, std::string_view separat void log_warning(std::string_view message) { - std::fprintf( + ::fprintf( stdout, "[RTCX WARNING] %.*s\n", static_cast(message.size()), message.data()); } void log_error(std::string_view message) { - std::fprintf( + ::fprintf( stderr, "[RTCX ERROR] %.*s\n", static_cast(message.size()), message.data()); } void log_trace(std::string_view message) { - std::fprintf( + ::fprintf( stdout, "[RTCX TRACE] %.*s\n", static_cast(message.size()), message.data()); } @@ -373,7 +373,7 @@ struct LibNVJitLink { LibNVJitLink(LibNVJitLink&&) = delete; LibNVJitLink& operator=(LibNVJitLink const&) = delete; LibNVJitLink& operator=(LibNVJitLink&&) = delete; - ~LibNVJitLink() { dlclose(_handle); } + ~LibNVJitLink() { ::dlclose(_handle); } static void* _load() { @@ -922,7 +922,7 @@ std::string demangle_cuda_symbol(char const* mangled_name) RTCX_EXPECTS(demangled_name != nullptr, "Demangling CUDA symbol name failed", std::runtime_error); RTCX_DEFER([&] { - if (demangled_name != nullptr) free(demangled_name); + if (demangled_name != nullptr) ::free(demangled_name); }); std::string result{demangled_name}; @@ -965,7 +965,7 @@ std::string const& cache_t::get_tmp_dir() { return tmp_dir_; } std::optional blob_t::from_file(char const* path) { - std::int32_t fd = open(path, O_RDONLY); + std::int32_t fd = ::open(path, O_RDONLY); if (fd == -1) { if (errno == ENOENT) { @@ -975,19 +975,19 @@ std::optional blob_t::from_file(char const* path) } } - auto file_size = lseek(fd, 0, SEEK_END); + auto file_size = ::lseek(fd, 0, SEEK_END); if (file_size == -1) { throw_posix("Failed to determine size of RTCX cache file", "lseek"); } - void* map = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); + void* map = ::mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTCX cache file", "mmap"); } - if (close(fd) == -1) { + if (::close(fd) == -1) { throw_posix("Failed to close RTCX cache file after memory-mapping", "close"); } auto deleter = +[](std::uint8_t const* buffer, std::size_t size) { - if (munmap(static_cast(const_cast(buffer)), size) == -1) { + if (::munmap(static_cast(const_cast(buffer)), size) == -1) { throw_posix("Failed to unmap RTCX cache file from memory", "munmap"); } }; @@ -1034,11 +1034,11 @@ std::optional get_disk_library(std::string const& cache_dir, sha256 con std::pair, std::vector> get_disk_entries( std::string const& cache_dir) { - std::int32_t dir = open(cache_dir.c_str(), O_RDONLY | O_DIRECTORY); + std::int32_t dir = ::open(cache_dir.c_str(), O_RDONLY | O_DIRECTORY); if (dir == -1) { throw_posix("Failed to open RTCX cache directory for evicting", "open"); } - RTCX_DEFER([&] { close(dir); }); + RTCX_DEFER([&] { ::close(dir); }); std::vector buffer; buffer.resize(8192); @@ -1048,13 +1048,13 @@ std::pair, std::vector> get_d std::ptrdiff_t num_read = 0; - while ((num_read = syscall(SYS_getdents64, dir, buffer.data(), buffer.size())) > 0) { + while ((num_read = ::syscall(SYS_getdents64, dir, buffer.data(), buffer.size())) > 0) { std::ptrdiff_t byte_pos = 0; while (byte_pos < num_read) { auto* ent = reinterpret_cast(buffer.data() + byte_pos); - if (memcmp(ent->d_name, ".", 2) != 0 && memcmp(ent->d_name, "..", 3) != 0) { + if (::memcmp(ent->d_name, ".", 2) != 0 && ::memcmp(ent->d_name, "..", 3) != 0) { RTCX_EXPECTS(ent->d_type != DT_UNKNOWN, "Found unknown directory entry type in RTCX cache dir", std::runtime_error); @@ -1062,7 +1062,7 @@ std::pair, std::vector> get_d if (ent->d_type == DT_REG) { auto path = std::format("{}/{}", cache_dir, ent->d_name); struct stat st; - if (stat(path.c_str(), &st) == -1 && errno != ENOENT) { + if (::stat(path.c_str(), &st) == -1 && errno != ENOENT) { throw_posix("Failed to get RTCX cache file stats", "stat"); } @@ -1104,7 +1104,7 @@ void evict_disk_entries(std::string const& cache_dir, std::uint32_t limit) auto num_evict = (limit == 0) ? paths.size() : ((limit + 1) / 2); for (auto index : std::span{ranking_indices}.subspan(0, num_evict)) { - if (unlink(paths[index].c_str()) == -1 && errno != ENOENT) { + if (::unlink(paths[index].c_str()) == -1 && errno != ENOENT) { throw_posix("Failed to evict RTCX cache file", "unlink"); } } @@ -1125,14 +1125,16 @@ void cache_blob_to_disk(std::string const& cache_dir, (void)tmp_path.c_str(); // to ensure null-termination for mkstemp { - std::int32_t fd = mkstemp(tmp_path.data()); + std::int32_t fd = ::mkstemp(tmp_path.data()); if (fd == -1) { throw_posix("Failed to create temporary file for RTCX cache", "mkstemp"); } RTCX_DEFER([&] { - if (close(fd) == -1) { throw_posix("Failed to close temporary RTCX cache file", "close"); } + if (::close(fd) == -1) { + throw_posix("Failed to close temporary RTCX cache file", "close"); + } }); - if (write(fd, binary.data(), binary.size()) == -1) { + if (::write(fd, binary.data(), binary.size()) == -1) { throw_posix("Failed to write RTCX cache to temporary file", "write"); } } @@ -1143,10 +1145,10 @@ void cache_blob_to_disk(std::string const& cache_dir, std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); // rename is atomic, even if another process is performing the same operation - if (rename(tmp_path.c_str(), final_path.c_str()) == -1) { + if (::rename(tmp_path.c_str(), final_path.c_str()) == -1) { if (errno == EEXIST) { // another process has already created the file, so just remove our temp file - if (remove(tmp_path.c_str()) == -1) { + if (::remove(tmp_path.c_str()) == -1) { throw_posix("Failed to remove temporary RTCX cache file", "remove"); } return; From b382e49283f910513c25024b2cfb8ad69332826c Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 18:00:32 +0000 Subject: [PATCH 100/254] reorg --- cpp/src/librtcx/rtcx.cpp | 138 +++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 70 deletions(-) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 506ada6d5698..ee4c5efc1867 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -264,6 +264,64 @@ sha256 sha256_context::finalize() namespace { +char const* get_nvJitLinkResultString(nvJitLinkResult result) +{ + switch (result) { + case NVJITLINK_SUCCESS: return "NVJITLINK_SUCCESS"; + case NVJITLINK_ERROR_UNRECOGNIZED_OPTION: return "NVJITLINK_ERROR_UNRECOGNIZED_OPTION"; + case NVJITLINK_ERROR_MISSING_ARCH: return "NVJITLINK_ERROR_MISSING_ARCH"; + case NVJITLINK_ERROR_INVALID_INPUT: return "NVJITLINK_ERROR_INVALID_INPUT"; + case NVJITLINK_ERROR_PTX_COMPILE: return "NVJITLINK_ERROR_PTX_COMPILE"; + case NVJITLINK_ERROR_NVVM_COMPILE: return "NVJITLINK_ERROR_NVVM_COMPILE"; + case NVJITLINK_ERROR_INTERNAL: return "NVJITLINK_ERROR_INTERNAL"; + case NVJITLINK_ERROR_THREADPOOL: return "NVJITLINK_ERROR_THREADPOOL"; + case NVJITLINK_ERROR_UNRECOGNIZED_INPUT: return "NVJITLINK_ERROR_UNRECOGNIZED_INPUT"; + case NVJITLINK_ERROR_FINALIZE: return "NVJITLINK_ERROR_FINALIZE"; +#if CUDA_VERSION >= 13000 + case NVJITLINK_ERROR_NULL_INPUT: return "NVJITLINK_ERROR_NULL_INPUT"; + case NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS: return "NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS"; + case NVJITLINK_ERROR_INCORRECT_INPUT_TYPE: return "NVJITLINK_ERROR_INCORRECT_INPUT_TYPE"; + case NVJITLINK_ERROR_ARCH_MISMATCH: return "NVJITLINK_ERROR_ARCH_MISMATCH"; + case NVJITLINK_ERROR_OUTDATED_LIBRARY: return "NVJITLINK_ERROR_OUTDATED_LIBRARY"; + case NVJITLINK_ERROR_MISSING_FATBIN: return "NVJITLINK_ERROR_MISSING_FATBIN"; + case NVJITLINK_ERROR_UNRECOGNIZED_ARCH: return "NVJITLINK_ERROR_UNRECOGNIZED_ARCH"; + case NVJITLINK_ERROR_UNSUPPORTED_ARCH: return "NVJITLINK_ERROR_UNSUPPORTED_ARCH"; + case NVJITLINK_ERROR_LTO_NOT_ENABLED: return "NVJITLINK_ERROR_LTO_NOT_ENABLED"; +#endif + default: + RTCX_FAIL( + std::format("Unrecognized nvJitLinkResult type: ({})", static_cast(result)), + std::runtime_error); + } +} + +char const* binary_type_string(binary_type type) +{ + switch (type) { + case binary_type::LTO_IR: return "LTO_IR"; + case binary_type::CUBIN: return "CUBIN"; + case binary_type::FATBIN: return "FATBIN"; + case binary_type::PTX: return "PTX"; + default: + RTCX_FAIL(std::format("Unrecognized binary_type: ({})", static_cast(type)), + std::runtime_error); + } +} + +nvJitLinkInputType to_nvjitlink_input_type(binary_type bin_type) +{ + switch (bin_type) { + case binary_type::LTO_IR: return NVJITLINK_INPUT_LTOIR; + case binary_type::CUBIN: return NVJITLINK_INPUT_CUBIN; + case binary_type::FATBIN: return NVJITLINK_INPUT_FATBIN; + case binary_type::PTX: return NVJITLINK_INPUT_PTX; + default: + RTCX_FAIL(std::format("Unrecognized binary type for linking: ({}) ", + static_cast(bin_type)), + std::logic_error); + } +} + void* load_dll(std::string_view base_name, std::span names) { for (auto& name : names) { @@ -430,66 +488,22 @@ void teardown() }); } -namespace { - -char const* get_nvJitLinkResultString(nvJitLinkResult result) -{ - switch (result) { - case NVJITLINK_SUCCESS: return "NVJITLINK_SUCCESS"; - case NVJITLINK_ERROR_UNRECOGNIZED_OPTION: return "NVJITLINK_ERROR_UNRECOGNIZED_OPTION"; - case NVJITLINK_ERROR_MISSING_ARCH: return "NVJITLINK_ERROR_MISSING_ARCH"; - case NVJITLINK_ERROR_INVALID_INPUT: return "NVJITLINK_ERROR_INVALID_INPUT"; - case NVJITLINK_ERROR_PTX_COMPILE: return "NVJITLINK_ERROR_PTX_COMPILE"; - case NVJITLINK_ERROR_NVVM_COMPILE: return "NVJITLINK_ERROR_NVVM_COMPILE"; - case NVJITLINK_ERROR_INTERNAL: return "NVJITLINK_ERROR_INTERNAL"; - case NVJITLINK_ERROR_THREADPOOL: return "NVJITLINK_ERROR_THREADPOOL"; - case NVJITLINK_ERROR_UNRECOGNIZED_INPUT: return "NVJITLINK_ERROR_UNRECOGNIZED_INPUT"; - case NVJITLINK_ERROR_FINALIZE: return "NVJITLINK_ERROR_FINALIZE"; -#if CUDA_VERSION >= 13000 - case NVJITLINK_ERROR_NULL_INPUT: return "NVJITLINK_ERROR_NULL_INPUT"; - case NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS: return "NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS"; - case NVJITLINK_ERROR_INCORRECT_INPUT_TYPE: return "NVJITLINK_ERROR_INCORRECT_INPUT_TYPE"; - case NVJITLINK_ERROR_ARCH_MISMATCH: return "NVJITLINK_ERROR_ARCH_MISMATCH"; - case NVJITLINK_ERROR_OUTDATED_LIBRARY: return "NVJITLINK_ERROR_OUTDATED_LIBRARY"; - case NVJITLINK_ERROR_MISSING_FATBIN: return "NVJITLINK_ERROR_MISSING_FATBIN"; - case NVJITLINK_ERROR_UNRECOGNIZED_ARCH: return "NVJITLINK_ERROR_UNRECOGNIZED_ARCH"; - case NVJITLINK_ERROR_UNSUPPORTED_ARCH: return "NVJITLINK_ERROR_UNSUPPORTED_ARCH"; - case NVJITLINK_ERROR_LTO_NOT_ENABLED: return "NVJITLINK_ERROR_LTO_NOT_ENABLED"; -#endif - default: - RTCX_FAIL( - std::format("Unrecognized nvJitLinkResult type: ({})", static_cast(result)), - std::runtime_error); - } -} - -char const* binary_type_string(binary_type type) +blob_t blob_t::from_buffer(byte_buffer&& buffer) { - switch (type) { - case binary_type::LTO_IR: return "LTO_IR"; - case binary_type::CUBIN: return "CUBIN"; - case binary_type::FATBIN: return "FATBIN"; - case binary_type::PTX: return "PTX"; - default: - RTCX_FAIL(std::format("Unrecognized binary_type: ({})", static_cast(type)), - std::runtime_error); - } + auto size = buffer.size(); + auto data = buffer.release(); + return blob_t::from_parts( + data, size, +[](std::uint8_t const* data, std::size_t) { + free(const_cast(data)); + }); } -nvJitLinkInputType to_nvjitlink_input_type(binary_type bin_type) +blob_t blob_t::from_static_data(std::span data) { - switch (bin_type) { - case binary_type::LTO_IR: return NVJITLINK_INPUT_LTOIR; - case binary_type::CUBIN: return NVJITLINK_INPUT_CUBIN; - case binary_type::FATBIN: return NVJITLINK_INPUT_FATBIN; - case binary_type::PTX: return NVJITLINK_INPUT_PTX; - default: - RTCX_FAIL(std::format("Unrecognized binary type for linking: ({}) ", - static_cast(bin_type)), - std::logic_error); - } + return blob_t::from_parts(data.data(), data.size(), blob_t::noop_deallocator); } +namespace { void log_nvrtc_result(compile_params const& params, nvrtcProgram program, nvrtcResult compile_result) @@ -636,22 +650,6 @@ void log_nvJitLink_result(link_params const& params, } } // namespace - -blob_t blob_t::from_buffer(byte_buffer&& buffer) -{ - auto size = buffer.size(); - auto data = buffer.release(); - return blob_t::from_parts( - data, size, +[](std::uint8_t const* data, std::size_t) { - free(const_cast(data)); - }); -} - -blob_t blob_t::from_static_data(std::span data) -{ - return blob_t::from_parts(data.data(), data.size(), blob_t::noop_deallocator); -} - byte_buffer compile(compile_params const& params) { RTCX_EXPECTS(params.name != nullptr, "Fragment name must not be null", std::logic_error); From a8f8c3d17b429f6ed3e95d52ce6385957ae40f71 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 18:57:36 +0000 Subject: [PATCH 101/254] Remove unused kernel enumeration functions from library_t --- cpp/src/librtcx/rtcx.cpp | 23 ----------------------- cpp/src/librtcx/rtcx.hpp | 6 ------ 2 files changed, 29 deletions(-) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index ee4c5efc1867..363907f1d6e5 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -214,9 +214,6 @@ sha256 sha256_context::finalize() DO_IT(GetErrorString) \ DO_IT(GetErrorName) \ DO_IT(Init) \ - DO_IT(DeviceGet) \ - DO_IT(DeviceGetCount) \ - DO_IT(DeviceGetName) \ DO_IT(OccupancyMaxPotentialBlockSize) \ DO_IT(LaunchKernel) \ DO_IT(LaunchKernelEx) \ @@ -226,8 +223,6 @@ sha256 sha256_context::finalize() DO_IT(LibraryLoadData) \ DO_IT(LibraryLoadFromFile) \ DO_IT(LibraryGetKernel) \ - DO_IT(LibraryGetKernelCount) \ - DO_IT(LibraryEnumerateKernels) \ DO_IT(LibraryUnload) #define FOR_EACH_NVRTC_FUNC(DO_IT) \ @@ -892,24 +887,6 @@ kernel_ref library_t::get_kernel(char const* name) const return kernel_ref{kernel}; } -std::vector library_t::enumerate_kernels() const -{ - std::uint32_t num_kernels; - RTCX_CHECK_CUDA(cu->LibraryGetKernelCount(&num_kernels, handle_)); - - std::vector kernels; - kernels.resize(num_kernels); - - RTCX_CHECK_CUDA(cu->LibraryEnumerateKernels(kernels.data(), num_kernels, handle_)); - - std::vector result; - for (CUkernel k : kernels) { - result.emplace_back(k); - } - - return result; -} - std::string demangle_cuda_symbol(char const* mangled_name) { std::int32_t status; diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index af646cdd87de..46a4b7fb8607 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -487,12 +487,6 @@ struct [[nodiscard]] library_t { * @brief Retrieve a kernel from the library by name */ [[nodiscard]] kernel_ref get_kernel(char const* name) const; - - /** - * @brief Enumerate all kernels contained in the library, returning a vector of kernel references - * @return A vector of kernel_ref objects representing all kernels contained in the library - */ - [[nodiscard]] std::vector enumerate_kernels() const; }; using library = std::shared_ptr; From 3f927807d1509cde1a0fb43ba388787db8095309 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 18:59:12 +0000 Subject: [PATCH 102/254] Remove get_name method and its declaration from kernel_ref --- cpp/src/librtcx/rtcx.cpp | 8 -------- cpp/src/librtcx/rtcx.hpp | 6 ------ 2 files changed, 14 deletions(-) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 363907f1d6e5..12b2203722e8 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -219,7 +219,6 @@ sha256 sha256_context::finalize() DO_IT(LaunchKernelEx) \ DO_IT(LaunchCooperativeKernel) \ DO_IT(KernelGetFunction) \ - DO_IT(KernelGetName) \ DO_IT(LibraryLoadData) \ DO_IT(LibraryLoadFromFile) \ DO_IT(LibraryGetKernel) \ @@ -765,13 +764,6 @@ void kernel_ref::launch_cooperative(cuda_dim3 grid_dim, kernel_params)); } -std::string_view kernel_ref::get_name() const -{ - char const* name; - RTCX_CHECK_CUDA(cu->KernelGetName(&name, handle_)); - return std::string_view{name == nullptr ? "" : name}; -} - library_t::~library_t() { if (handle_ != nullptr) { diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 46a4b7fb8607..58d16d0e137c 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -455,12 +455,6 @@ struct [[nodiscard]] kernel_ref { * @return The CUDA kernel handle associated with this kernel reference */ [[nodiscard]] CUkernel get() const { return handle_; } - - /** - * @brief Retrieves the name of the kernel - * @return A string_view representing the name of the kernel - */ - [[nodiscard]] std::string_view get_name() const; }; /** From 49d81e63b3f9395e18f1a9593a47400238f17c8b Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 20:08:09 +0000 Subject: [PATCH 103/254] remove unused kernel header from helpers.hpp --- cpp/src/jit/helpers.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index 912e4eca5963..379364c8ef66 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -14,7 +14,6 @@ #include #include -#include #include #include From 70c558c6e7378149f9b3160295735ab0b04d38b0 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 20:08:46 +0000 Subject: [PATCH 104/254] Update PTX code in ColumnVectorTest with generated output from Numba --- .../java/ai/rapids/cudf/ColumnVectorTest.java | 81 +++++++++++++++---- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index efdb38f21229..e374fc3c12ea 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -51,23 +51,70 @@ public class ColumnVectorTest extends CudfTestBase { static final double NEGATIVE_DOUBLE_NAN_UPPER_RANGE = Double.longBitsToDouble(0xffffffffffffffffL); // c = a * a - a - static String ptx = "***(" + - " .func _Z1fPii(" + - " .param .b64 _Z1fPii_param_0," + - " .param .b32 _Z1fPii_param_1" + - " )" + - " {" + - " .reg .b32 %r<4>;" + - " .reg .b64 %rd<3>;" + - " ld.param.u64 %rd1, [_Z1fPii_param_0];" + - " ld.param.u32 %r1, [_Z1fPii_param_1];" + - " cvta.to.global.u64 %rd2, %rd1;" + - " mul.lo.s32 %r2, %r1, %r1;" + - " sub.s32 %r3, %r2, %r1;" + - " st.global.u32 [%rd2], %r3;" + - " ret;" + - " }" + - ")***"; + // Generated from NUMBA, using: + // + // ```py + // + // from numba import cuda, int32 + // from numba.cuda import compile_ptx_for_current_device + // + // # Define a CUDA device function + // + // @cuda.jit(device=True) + // def op(a): + // return a * a - a + // + // # Define argument types for the function + // arg_types = (int32, ) + // + // # Compile the device function as relocatable + // ptx, _ = cuda.compile_ptx_for_current_device(op, arg_types, device=True) + // + // + // # Print the PTX code + // print("Relocatable PTX Code:") + // print(ptx) + // + // + // ``` + // + static String ptx = "***( +// +// Generated by NVIDIA NVVM Compiler +// +// Compiler Build ID: CL-37061995 +// Cuda compilation tools, release 13.1, V13.1.115 +// Based on NVVM 7.0.1 +// + +.version 9.1 +.target sm_86 +.address_size 64 + + // .globl _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi +.common .global .align 8 .u64 _ZN08NumbaEnv8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi; + +.visible .func (.param .b32 func_retval0) _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi( + .param .b64 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0, + .param .b32 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1 +) +{ + .reg .b32 %r<3>; + .reg .b64 %rd<5>; + + + ld.param.u64 %rd1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0]; + ld.param.u32 %r1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1]; + cvt.s64.s32 %rd2, %r1; + mul.wide.s32 %rd3, %r1, %r1; + sub.s64 %rd4, %rd3, %rd2; + st.u64 [%rd1], %rd4; + mov.u32 %r2, 0; + st.param.b32 [func_retval0+0], %r2; + ret; + +} +***"; static String cuda = "__device__ inline void f(" + "int* output," + From 742b88a6c9a2bcbdbe0cbc21ed97ddb6307ba262 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 22:37:13 +0000 Subject: [PATCH 105/254] FIX JAVA string literal quote --- java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index e374fc3c12ea..9824650e1111 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -78,7 +78,7 @@ public class ColumnVectorTest extends CudfTestBase { // // ``` // - static String ptx = "***( + static String ptx = """ // // Generated by NVIDIA NVVM Compiler // @@ -114,7 +114,7 @@ public class ColumnVectorTest extends CudfTestBase { ret; } -***"; +"""; static String cuda = "__device__ inline void f(" + "int* output," + From cc503e2d437c7bae694216af4863dc6330b8be30 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 23:57:58 +0100 Subject: [PATCH 106/254] Update cpp/src/transform/transform.cu Co-authored-by: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> --- cpp/src/transform/transform.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 41e75e2f8f1f..1f2773d2dc70 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -327,7 +327,7 @@ auto to_args(std::span inputs, // ensure the device buffer copy is complete before `h_args` goes out of scope and its destructors // are called - CUDF_CUDA_TRY(cudaStreamSynchronize(stream.value())); + stream.synchronize(); return std::make_tuple(std::move(d_args), std::move(handles)); } From 24c450afc785b3b8e47632b886fb8775fa28dc89 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 23:58:43 +0100 Subject: [PATCH 107/254] Update cpp/src/transform/transform.cu Co-authored-by: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> --- cpp/src/transform/transform.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 1f2773d2dc70..c33074cf3c0d 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -378,15 +378,15 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, if (!is_nullable) { return 0; } - auto num_words = num_bitmask_words(row_size); - auto num_bytes = num_words * sizeof(bitmask_type); - if (inputs.empty()) { // no input, set all to valid set_null_mask(null_mask, 0, row_size, true, stream); return 0; } + auto num_words = num_bitmask_words(row_size); + auto num_bytes = num_words * sizeof(bitmask_type); + auto has_scalars = std::any_of(inputs.begin(), inputs.end(), [](auto& in) { return std::holds_alternative(in); }); From 29cfcc7a8545c9a7ffeebd8a4a38f8aa483c7971 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Apr 2026 23:16:27 +0000 Subject: [PATCH 108/254] Refactor inplace_null_mask_and to move bitmask calculations after input checks --- cpp/src/transform/transform.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index c33074cf3c0d..80102907d45f 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -384,9 +384,6 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, return 0; } - auto num_words = num_bitmask_words(row_size); - auto num_bytes = num_words * sizeof(bitmask_type); - auto has_scalars = std::any_of(inputs.begin(), inputs.end(), [](auto& in) { return std::holds_alternative(in); }); @@ -434,6 +431,9 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, return 0; } + auto num_words = num_bitmask_words(row_size); + auto num_bytes = num_words * sizeof(bitmask_type); + if (nullable_masks.size() == 1) { // only 1 mask provided, copy it directly to the output CUDF_CUDA_TRY(detail::memcpy_async(null_mask, nullable_masks[0], num_bytes, stream)); From 22b189b853393d561e60af262b79ec1adcf894f1 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 2 Apr 2026 02:02:53 +0000 Subject: [PATCH 109/254] reduce binary size + fix java string --- .../cudf/strings/detail/strings_children.cuh | 35 --------- .../detail/strings_column_factories.cuh | 30 -------- cpp/src/transform/transform.cu | 66 +++++++++++++++- .../java/ai/rapids/cudf/ColumnVectorTest.java | 76 ++++++++++--------- 4 files changed, 104 insertions(+), 103 deletions(-) diff --git a/cpp/include/cudf/strings/detail/strings_children.cuh b/cpp/include/cudf/strings/detail/strings_children.cuh index 803e8910f154..d15e5fa199f1 100644 --- a/cpp/include/cudf/strings/detail/strings_children.cuh +++ b/cpp/include/cudf/strings/detail/strings_children.cuh @@ -102,41 +102,6 @@ rmm::device_uvector make_chars_buffer(column_view const& offsets, return chars_data; } -inline rmm::device_uvector make_chars_buffer(column_view const& offsets_view, - int64_t chars_size, - string_view const* begin, - bitmask_type const* stencil, - size_type size, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - auto offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets_view); - auto chars = rmm::device_uvector(chars_size, stream, mr); - - auto srcs = cudf::detail::make_counting_transform_iterator( - size_type{0}, [begin] __device__(size_type idx) -> void const* { return begin[idx].data(); }); - - auto src_sizes = cudf::detail::make_counting_transform_iterator( - size_type{0}, [begin, stencil] __device__(size_type idx) -> size_type { - if (stencil != nullptr && !bit_is_set(stencil, idx)) { return 0; } - return static_cast(begin[idx].size_bytes()); - }); - - auto dsts = cudf::detail::make_counting_transform_iterator( - size_type{0}, [offsets, chars = chars.data()] __device__(size_type idx) -> void* { - return chars + offsets[idx]; - }); - - size_t temp_storage_bytes = 0; - CUDF_CUDA_TRY(cub::DeviceMemcpy::Batched( - nullptr, temp_storage_bytes, srcs, dsts, src_sizes, size, stream.value())); - rmm::device_buffer d_temp_storage(temp_storage_bytes, stream); - CUDF_CUDA_TRY(cub::DeviceMemcpy::Batched( - d_temp_storage.data(), temp_storage_bytes, srcs, dsts, src_sizes, size, stream.value())); - - return chars; -} - /** * @brief Create an offsets column to be a child of a compound column * diff --git a/cpp/include/cudf/strings/detail/strings_column_factories.cuh b/cpp/include/cudf/strings/detail/strings_column_factories.cuh index 50ab0366667c..d98cb994415d 100644 --- a/cpp/include/cudf/strings/detail/strings_column_factories.cuh +++ b/cpp/include/cudf/strings/detail/strings_column_factories.cuh @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -79,35 +78,6 @@ std::unique_ptr make_strings_column(IndexPairIterator begin, std::move(null_mask)); } -inline std::unique_ptr make_strings_column(device_span strings, - rmm::device_buffer null_mask, - size_type null_count, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - CUDF_FUNC_RANGE(); - auto size = static_cast(strings.size()); - if (size == 0) return make_empty_column(type_id::STRING); - - auto stencil = static_cast(null_mask.data()); - - // build offsets column from the strings sizes - auto sizes = cudf::detail::make_counting_transform_iterator( - cudf::size_type{0}, - [stencil, strings = strings.data()] __device__(cudf::size_type index) -> size_type { - if (stencil != nullptr && !bit_is_set(stencil, index)) { return 0; } - return static_cast(strings[index].size_bytes()); - }); - - auto [offsets, bytes] = - cudf::strings::detail::make_offsets_child_column(sizes, sizes + size, stream, mr); - - auto chars = make_chars_buffer(offsets->view(), bytes, strings.data(), stencil, size, stream, mr); - - return make_strings_column( - size, std::move(offsets), chars.release(), null_count, std::move(null_mask)); -} - } // namespace detail } // namespace strings } // namespace cudf diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 80102907d45f..9e673deaf5a5 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -607,6 +607,70 @@ std::optional> make_stencil( return std::pair{*stencil, stencil_null_count}; } +rmm::device_uvector make_chars_buffer(column_view const& offsets_view, + int64_t chars_size, + string_view const* begin, + bitmask_type const* stencil, + size_type size, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + auto offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets_view); + auto chars = rmm::device_uvector(chars_size, stream, mr); + + auto srcs = cudf::detail::make_counting_transform_iterator( + size_type{0}, [begin] __device__(size_type idx) -> void const* { return begin[idx].data(); }); + + auto src_sizes = cudf::detail::make_counting_transform_iterator( + size_type{0}, [begin, stencil] __device__(size_type idx) -> size_type { + if (stencil != nullptr && !bit_is_set(stencil, idx)) { return 0; } + return static_cast(begin[idx].size_bytes()); + }); + + auto dsts = cudf::detail::make_counting_transform_iterator( + size_type{0}, [offsets, chars = chars.data()] __device__(size_type idx) -> void* { + return chars + offsets[idx]; + }); + + size_t temp_storage_bytes = 0; + CUDF_CUDA_TRY(cub::DeviceMemcpy::Batched( + nullptr, temp_storage_bytes, srcs, dsts, src_sizes, size, stream.value())); + rmm::device_buffer d_temp_storage(temp_storage_bytes, stream); + CUDF_CUDA_TRY(cub::DeviceMemcpy::Batched( + d_temp_storage.data(), temp_storage_bytes, srcs, dsts, src_sizes, size, stream.value())); + + return chars; +} + +std::unique_ptr make_strings_column(device_span strings, + rmm::device_buffer null_mask, + size_type null_count, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto size = static_cast(strings.size()); + if (size == 0) return make_empty_column(type_id::STRING); + + auto stencil = static_cast(null_mask.data()); + + // build offsets column from the strings sizes + auto sizes = cudf::detail::make_counting_transform_iterator( + cudf::size_type{0}, + [stencil, strings = strings.data()] __device__(cudf::size_type index) -> size_type { + if (stencil != nullptr && !bit_is_set(stencil, index)) { return 0; } + return static_cast(strings[index].size_bytes()); + }); + + auto [offsets, bytes] = + cudf::strings::detail::make_offsets_child_column(sizes, sizes + size, stream, mr); + + auto chars = make_chars_buffer(offsets->view(), bytes, strings.data(), stencil, size, stream, mr); + + return make_strings_column( + size, std::move(offsets), chars.release(), null_count, std::move(null_mask)); +} + auto make_outputs(null_aware is_null_aware, size_type row_size, std::span inputs, @@ -696,7 +760,7 @@ auto finalize_output(string_views_column&& c, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - return strings::detail::make_strings_column( + return make_strings_column( device_span{static_cast(c._data.data()), static_cast(c._size)}, std::move(c._null_mask), diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index 9824650e1111..8b03564d3b41 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -78,43 +78,45 @@ public class ColumnVectorTest extends CudfTestBase { // // ``` // - static String ptx = """ -// -// Generated by NVIDIA NVVM Compiler -// -// Compiler Build ID: CL-37061995 -// Cuda compilation tools, release 13.1, V13.1.115 -// Based on NVVM 7.0.1 -// - -.version 9.1 -.target sm_86 -.address_size 64 - - // .globl _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi -.common .global .align 8 .u64 _ZN08NumbaEnv8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi; - -.visible .func (.param .b32 func_retval0) _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi( - .param .b64 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0, - .param .b32 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1 -) -{ - .reg .b32 %r<3>; - .reg .b64 %rd<5>; - - - ld.param.u64 %rd1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0]; - ld.param.u32 %r1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1]; - cvt.s64.s32 %rd2, %r1; - mul.wide.s32 %rd3, %r1, %r1; - sub.s64 %rd4, %rd3, %rd2; - st.u64 [%rd1], %rd4; - mov.u32 %r2, 0; - st.param.b32 [func_retval0+0], %r2; - ret; - -} -"""; + static String ptx = "//\n"+ +"// Generated by NVIDIA NVVM Compiler\n"+ +"// Compiler Build ID: CL-37061995\n"+ +"// Cuda compilation tools, release 13.1, V13.1.115\n"+ +"// Based on NVVM 7.0.1\n"+ +"//\n"+ +"// Compiler Build ID: CL-37061995\n" + +"// Cuda compilation tools, release 13.1, V13.1.115\n"+ +"// Based on NVVM 7.0.1\n"+ +"//\n"+ +"\n"+ +".version 9.1\n"+ +".target sm_86\n"+ +".address_size 64\n"+ +"\n"+ +" // .globl _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi\n"+ +".common .global .align 8 .u64 _ZN08NumbaEnv8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi;\n"+ +"\n"+ +"\n"+ +".visible .func (.param .b32 func_retval0) _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi(\n"+ +" .param .b64 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0,\n"+ +" .param .b32 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1\n"+ +")\n"+ +"{\n"+ +" .reg .b32 %r<3>;\n"+ +" .reg .b64 %rd<5>;\n"+ +"\n"+ +"\n"+ +" ld.param.u64 %rd1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0];\n"+ +" ld.param.u32 %r1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1];\n"+ +" cvt.s64.s32 %rd2, %r1;\n"+ +" mul.wide.s32 %rd3, %r1, %r1;\n"+ +" sub.s64 %rd4, %rd3, %rd2;\n"+ +" st.u64 [%rd1], %rd4;\n"+ +" mov.u32 %r2, 0;\n"+ +" st.param.b32 [func_retval0+0], %r2;\n"+ +" ret;\n"+ +"\n"+ +"}\n"; static String cuda = "__device__ inline void f(" + "int* output," + From c261b8334a28931b48fef791bd2d477a82962996 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 5 Apr 2026 19:57:42 +0000 Subject: [PATCH 110/254] update --- cpp/src/jit/jit.cpp | 57 +++++++++++++++++++-------------------- cpp/src/jit/jit.hpp | 4 +-- cpp/src/librtcx/README.md | 4 +++ cpp/src/librtcx/rtcx.cpp | 20 ++++++++++++++ cpp/src/librtcx/rtcx.hpp | 6 +++++ 5 files changed, 60 insertions(+), 31 deletions(-) diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index 21023d0efc14..874ef995be2f 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -50,7 +50,7 @@ rtcx::sha256 hash_strings(std::span inputs) auto error_code = errno; auto error_str = std::format( "{}. `{}` failed with {} ({})", message, syscall_name, error_code, std::strerror(error_code)); - CUDF_FAIL(+error_str, std::runtime_error); + CUDF_FAIL(error_str, std::runtime_error); } void install_file(char const* dst_path, std::span contents) @@ -114,7 +114,7 @@ rtcx::byte_buffer decompress_blob(std::span compressed_binary, std::string_view compression) { CUDF_EXPECTS(compression == "none" || compression == "zstd", - +std::format("Unsupported compression type specified: {}", compression), + std::format("Unsupported compression type specified: {}", compression), std::runtime_error); auto decompressed = rtcx::byte_buffer::make(uncompressed_size); @@ -124,9 +124,9 @@ rtcx::byte_buffer decompress_blob(std::span compressed_binary, CUDF_EXPECTS( !ZSTD_isError(errc) && errc == uncompressed_size, - +std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : ", - errc, - ZSTD_getErrorName(errc)), + std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : ", + errc, + ZSTD_getErrorName(errc)), std::runtime_error); } else { // compression is "none", so just copy the data @@ -214,14 +214,14 @@ void jit_bundle_t::ensure_installed() const throw_posix(std::format("Failed to get stat for directory ({})", expected_path), "lstat"); } else { // ensure base install directory exists - CUDF_LOG_INFO("Creating JIT install directory at ({})", expected_path); + CUDF_LOG_INFO("Creating JIT install directory at (%s)", expected_path.c_str()); std::filesystem::create_directories(install_dir_); install_cudf_jit_files(expected_path.c_str(), cache_->get_tmp_dir()); } } else { // directory exists, perform minor sanity check CUDF_EXPECTS(S_ISDIR(path_info.st_mode), - +std::format("JIT install path ({}) exists but is not a directory", expected_path), + std::format("JIT install path ({}) exists but is not a directory", expected_path), std::runtime_error); } } @@ -281,6 +281,7 @@ std::tuple compile_library_uncached( char const* cuda_code, std::span extra_header_include_names, std::span extra_headers, + std::span extra_options, std::span name_expressions, bool use_pch, bool log_pch) @@ -338,6 +339,9 @@ std::tuple compile_library_uncached( for (auto const& option : options) { options_cstr.emplace_back(option.c_str()); } + for (auto const& option : extra_options) { + options_cstr.emplace_back(option); + } auto params = rtcx::compile_params{.name = name, .source = cuda_code, @@ -367,23 +371,11 @@ std::tuple compile_library_uncached( } // namespace -static rtcx::kernel_ref get_kernel(rtcx::library const& lib) -{ - auto kernels = lib->enumerate_kernels(); - CUDF_EXPECTS( - kernels.size() == 1, - +std::format("Expected exactly one kernel in compiled library, but found {}", kernels.size()), - std::runtime_error); - return kernels[0]; -} - -kernel::kernel(rtcx::library lib) : _library(std::move(lib)), _kernel(get_kernel(_library)) {} - kernel get_kernel(std::string const& name, std::string const& source_file, std::span header_include_names, std::span headers, - std::string const& name_expression, + std::string const& kernel_instance, bool use_cache, bool use_pch, bool log_pch) @@ -409,7 +401,7 @@ bundle={} source_file={} header_include_names={} headers={} -name_expression={} +kernel_instance={} )***", runtime, driver, @@ -418,33 +410,40 @@ name_expression={} source_file, header_include_names_hash.view(), headers_hash.view(), - name_expression); + kernel_instance); auto cache_key_sha256 = hash_string(cache_key); auto compile = [&] { - auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); - auto source_file_path = std::format("{}/{}", bundle_dir, source_file); - auto source = read_blob_cstring(source_file_path.c_str()); - char const* name_exprs[] = {name_expression.c_str()}; + auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); + auto source_file_path = std::format("{}/{}", bundle_dir, source_file); + auto source = read_blob_cstring(source_file_path.c_str()); + char const* name_expressions[] = {kernel_instance.c_str()}; + return compile_library_uncached(name.c_str(), reinterpret_cast(source.data()), header_include_names, headers, - name_exprs, + {}, + name_expressions, use_pch, log_pch); }; if (!use_cache) { auto [lib, blob] = compile(); - return lib; + auto kernels = lib->enumerate_kernels(); + CUDF_EXPECTS(kernels.size() == 1, "Unexpected kernel count", std::logic_error); + return kernel{lib, kernels[0]}; } auto fut = cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); - return fut.get(); + auto lib = fut.get(); + auto kernels = lib->enumerate_kernels(); + CUDF_EXPECTS(kernels.size() == 1, "Unexpected kernel count", std::logic_error); + return kernel{lib, kernels[0]}; } } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index 8727bc507b0b..b6e91dd5a2b4 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -35,7 +35,7 @@ struct [[nodiscard]] kernel { rtcx::kernel_ref _kernel; public: - kernel(rtcx::library lib); + kernel(rtcx::library lib, rtcx::kernel_ref kernel) : _library(std::move(lib)), _kernel(kernel) {}; kernel(kernel const&) = default; kernel(kernel&&) = default; kernel& operator=(kernel const&) = default; @@ -75,7 +75,7 @@ kernel get_kernel(std::string const& name, std::string const& source_file, std::span header_include_names, std::span headers, - std::string const& name_expression, + std::string const& kernel_instance, bool use_cache = true, bool use_pch = true, bool log_pch = false); diff --git a/cpp/src/librtcx/README.md b/cpp/src/librtcx/README.md index be59fa684960..72d8ac38d047 100644 --- a/cpp/src/librtcx/README.md +++ b/cpp/src/librtcx/README.md @@ -14,3 +14,7 @@ RTCX (runtime-compiler extended) is a wrapper around NVRTC and NVJitLink designe - CMake - python 3 - python-zstd + +## Runtime Requirements +- CUDA >= 12.4.0 + diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 12b2203722e8..7a50237d59dc 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -222,6 +222,8 @@ sha256 sha256_context::finalize() DO_IT(LibraryLoadData) \ DO_IT(LibraryLoadFromFile) \ DO_IT(LibraryGetKernel) \ + DO_IT(LibraryGetKernelCount) \ + DO_IT(LibraryEnumerateKernels) \ DO_IT(LibraryUnload) #define FOR_EACH_NVRTC_FUNC(DO_IT) \ @@ -879,6 +881,24 @@ kernel_ref library_t::get_kernel(char const* name) const return kernel_ref{kernel}; } +std::vector library_t::enumerate_kernels() const +{ + std::uint32_t num_kernels; + RTCX_CHECK_CUDA(cu->LibraryGetKernelCount(&num_kernels, handle_)); + + std::vector kernels; + kernels.resize(num_kernels); + + RTCX_CHECK_CUDA(cu->LibraryEnumerateKernels(kernels.data(), num_kernels, handle_)); + + std::vector result; + for (CUkernel k : kernels) { + result.emplace_back(k); + } + + return result; +} + std::string demangle_cuda_symbol(char const* mangled_name) { std::int32_t status; diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 58d16d0e137c..c70e54c2fa41 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -481,6 +481,12 @@ struct [[nodiscard]] library_t { * @brief Retrieve a kernel from the library by name */ [[nodiscard]] kernel_ref get_kernel(char const* name) const; + + /** + * @brief Enumerate all kernels contained in the library, returning a vector of kernel references + * @return A vector of kernel_ref objects representing all kernels contained in the library + */ + [[nodiscard]] std::vector enumerate_kernels() const; }; using library = std::shared_ptr; From 25a35f07c9a2676a06a94a128686e6d569768478 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 5 Apr 2026 23:13:25 +0000 Subject: [PATCH 111/254] Remove lz4 and zstandard dependencies from environment and project files --- .../all_cuda-129_arch-aarch64.yaml | 2 - .../all_cuda-129_arch-x86_64.yaml | 2 - .../all_cuda-131_arch-aarch64.yaml | 2 - .../all_cuda-131_arch-x86_64.yaml | 2 - cpp/cmake/thirdparty/get_lz4.cmake | 40 ------------------- dependencies.yaml | 3 +- python/libcudf/pyproject.toml | 2 - 7 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 cpp/cmake/thirdparty/get_lz4.cmake diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index a4b5038e9c21..2b49cd41f298 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -46,7 +46,6 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.11.1,<2.12.0 - librmm==26.6.*,>=0.0.0a0 -- lz4>=4.4.5 - make - mmh3 - moto>=4.0.8 @@ -103,5 +102,4 @@ dependencies: - typing_extensions>=4.0.0 - zlib>=1.2.13 - zstandard -- zstandard>=0.25.0 name: all_cuda-129_arch-aarch64 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index ccc67c3d25cb..b3a4b7102146 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -46,7 +46,6 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.11.1,<2.12.0 - librmm==26.6.*,>=0.0.0a0 -- lz4>=4.4.5 - make - mmh3 - moto>=4.0.8 @@ -103,5 +102,4 @@ dependencies: - typing_extensions>=4.0.0 - zlib>=1.2.13 - zstandard -- zstandard>=0.25.0 name: all_cuda-129_arch-x86_64 diff --git a/conda/environments/all_cuda-131_arch-aarch64.yaml b/conda/environments/all_cuda-131_arch-aarch64.yaml index f1ba4879d84c..25ea42819e47 100644 --- a/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -46,7 +46,6 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.11.1,<2.12.0 - librmm==26.6.*,>=0.0.0a0 -- lz4>=4.4.5 - make - mmh3 - moto>=4.0.8 @@ -103,5 +102,4 @@ dependencies: - typing_extensions>=4.0.0 - zlib>=1.2.13 - zstandard -- zstandard>=0.25.0 name: all_cuda-131_arch-aarch64 diff --git a/conda/environments/all_cuda-131_arch-x86_64.yaml b/conda/environments/all_cuda-131_arch-x86_64.yaml index 33e79f1c4d99..2ccd1614d462 100644 --- a/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -46,7 +46,6 @@ dependencies: - libnvjitlink-dev - librdkafka>=2.11.1,<2.12.0 - librmm==26.6.*,>=0.0.0a0 -- lz4>=4.4.5 - make - mmh3 - moto>=4.0.8 @@ -103,5 +102,4 @@ dependencies: - typing_extensions>=4.0.0 - zlib>=1.2.13 - zstandard -- zstandard>=0.25.0 name: all_cuda-131_arch-x86_64 diff --git a/cpp/cmake/thirdparty/get_lz4.cmake b/cpp/cmake/thirdparty/get_lz4.cmake deleted file mode 100644 index a7e0e6f573f9..000000000000 --- a/cpp/cmake/thirdparty/get_lz4.cmake +++ /dev/null @@ -1,40 +0,0 @@ -# ============================================================================= -# cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# cmake-format: on -# ============================================================================= - -# Use CPM to find or clone lz4 -function(find_and_configure_lz4) - - set(CPM_DOWNLOAD_lz4 ON) - rapids_cpm_find( - lz4 dev - GLOBAL_TARGETS lz4 - CPM_ARGS - GIT_REPOSITORY https://github.com/lz4/lz4.git - GIT_TAG 5c4c1fb2354133e1f3b087a341576985f8114bd5 - GIT_SHALLOW FALSE SOURCE_SUBDIR build/cmake - ) - - if(lz4_ADDED) - add_library( - liblz4_static OBJECT - ${lz4_SOURCE_DIR}/lib/lz4.c ${lz4_SOURCE_DIR}/lib/lz4file.c ${lz4_SOURCE_DIR}/lib/lz4frame.c - ${lz4_SOURCE_DIR}/lib/lz4hc.c ${lz4_SOURCE_DIR}/lib/xxhash.c - ) - target_include_directories(liblz4_static PUBLIC $) - endif() - - if(DEFINED lz4_SOURCE_DIR) - set(LZ4_INCLUDE_DIR - "${lz4_SOURCE_DIR}/lib" - PARENT_SCOPE - ) - endif() - rapids_export_find_package_root(BUILD liblz4_static "${lz4_BINARY_DIR}" EXPORT_SET cudf-exports) - -endfunction() - -find_and_configure_lz4() diff --git a/dependencies.yaml b/dependencies.yaml index 09a08a92138d..4188f91b60b2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -509,8 +509,7 @@ dependencies: - librdkafka>=2.11.1,<2.12.0 - output_types: [conda, pyproject, requirements] packages: - - lz4>=4.4.5 - - zstandard>=0.25.0 + - zstandard depends_on_libnvcomp: common: - output_types: conda diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 0961415d1184..f581fd85f48b 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -71,10 +71,8 @@ requires = [ "cmake>=3.30.4", "libkvikio==26.6.*,>=0.0.0a0", "librmm==26.6.*,>=0.0.0a0", - "lz4>=4.4.5", "ninja", "nvidia-libnvcomp==5.1.0.21", "nvidia-nvjitlink>=13.0,<14", "rapids-logger==0.2.*,>=0.0.0a0", - "zstandard>=0.25.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From cedd9bdcdc1a0bf83e97e9ebab4ac48180b03531 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 5 Apr 2026 23:23:01 +0000 Subject: [PATCH 112/254] Add zstandard dependency to pyproject.toml --- cpp/src/librtcx/README.md | 1 - python/libcudf/pyproject.toml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/librtcx/README.md b/cpp/src/librtcx/README.md index 72d8ac38d047..1bf8a55599af 100644 --- a/cpp/src/librtcx/README.md +++ b/cpp/src/librtcx/README.md @@ -17,4 +17,3 @@ RTCX (runtime-compiler extended) is a wrapper around NVRTC and NVJitLink designe ## Runtime Requirements - CUDA >= 12.4.0 - diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index f581fd85f48b..81d89d876b2b 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -75,4 +75,5 @@ requires = [ "nvidia-libnvcomp==5.1.0.21", "nvidia-nvjitlink>=13.0,<14", "rapids-logger==0.2.*,>=0.0.0a0", + "zstandard", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 45e48f55000beb06150305a0eb0a8abb696064d4 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 5 Apr 2026 23:42:51 +0000 Subject: [PATCH 113/254] Update TODO comment to clarify glob restrictions for included source files --- cpp/src/librtcx/CMake/embed.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/librtcx/CMake/embed.cmake b/cpp/src/librtcx/CMake/embed.cmake index cd5115fa1db6..a719acb25842 100644 --- a/cpp/src/librtcx/CMake/embed.cmake +++ b/cpp/src/librtcx/CMake/embed.cmake @@ -7,7 +7,8 @@ find_package(Python3 REQUIRED COMPONENTS Interpreter) -# TODO: add glob restrictions:exclude .cpp .cc, .cmake and .cxx source files, only include .h .hpp .cuh .cu files +# TODO: add glob restrictions:exclude .cpp .cc, .cmake and .cxx source files, only include .h .hpp +# .cuh .cu files # This function registers a directory of include files to be embedded for JIT compilation. It # gathers the specified files, their destinations, and include directories, and stores them in From 6a1267f453e0d3cb4b3254539242a375e92feedc Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 6 Apr 2026 01:41:04 +0000 Subject: [PATCH 114/254] remove libraryunload handling --- cpp/src/librtcx/rtcx.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 7a50237d59dc..4595db6ab8b0 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -768,9 +768,7 @@ void kernel_ref::launch_cooperative(cuda_dim3 grid_dim, library_t::~library_t() { - if (handle_ != nullptr) { - if (cu->LibraryUnload(handle_) != CUDA_SUCCESS) { std::terminate(); } - } + if (handle_ != nullptr) { cu->LibraryUnload(handle_); } } library load_library(std::span binary) From 8f50bb3803e2e2093efdd05215d34db0e2bdf92f Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 6 Apr 2026 01:42:57 +0000 Subject: [PATCH 115/254] fix identation --- .../java/ai/rapids/cudf/ColumnVectorTest.java | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index 8b03564d3b41..1583c8179a34 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -78,45 +78,45 @@ public class ColumnVectorTest extends CudfTestBase { // // ``` // - static String ptx = "//\n"+ -"// Generated by NVIDIA NVVM Compiler\n"+ -"// Compiler Build ID: CL-37061995\n"+ -"// Cuda compilation tools, release 13.1, V13.1.115\n"+ -"// Based on NVVM 7.0.1\n"+ -"//\n"+ -"// Compiler Build ID: CL-37061995\n" + -"// Cuda compilation tools, release 13.1, V13.1.115\n"+ -"// Based on NVVM 7.0.1\n"+ -"//\n"+ -"\n"+ -".version 9.1\n"+ -".target sm_86\n"+ -".address_size 64\n"+ -"\n"+ -" // .globl _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi\n"+ -".common .global .align 8 .u64 _ZN08NumbaEnv8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi;\n"+ -"\n"+ -"\n"+ -".visible .func (.param .b32 func_retval0) _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi(\n"+ -" .param .b64 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0,\n"+ -" .param .b32 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1\n"+ -")\n"+ -"{\n"+ -" .reg .b32 %r<3>;\n"+ -" .reg .b64 %rd<5>;\n"+ -"\n"+ -"\n"+ -" ld.param.u64 %rd1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0];\n"+ -" ld.param.u32 %r1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1];\n"+ -" cvt.s64.s32 %rd2, %r1;\n"+ -" mul.wide.s32 %rd3, %r1, %r1;\n"+ -" sub.s64 %rd4, %rd3, %rd2;\n"+ -" st.u64 [%rd1], %rd4;\n"+ -" mov.u32 %r2, 0;\n"+ -" st.param.b32 [func_retval0+0], %r2;\n"+ -" ret;\n"+ -"\n"+ -"}\n"; + static String ptx = "//\n" + + "// Generated by NVIDIA NVVM Compiler\n" + + "// Compiler Build ID: CL-37061995\n" + + "// Cuda compilation tools, release 13.1, V13.1.115\n" + + "// Based on NVVM 7.0.1\n" + + "//\n" + + "// Compiler Build ID: CL-37061995\n" + + "// Cuda compilation tools, release 13.1, V13.1.115\n" + + "// Based on NVVM 7.0.1\n" + + "//\n" + + "\n" + + ".version 9.1\n" + + ".target sm_86\n" + + ".address_size 64\n" + + "\n" + + " // .globl _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi\n" + + ".common .global .align 8 .u64 _ZN08NumbaEnv8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi;\n" + + "\n" + + "\n" + + ".visible .func (.param .b32 func_retval0) _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi(\n" + + " .param .b64 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0,\n" + + " .param .b32 _ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1\n" + + ")\n" + + "{\n" + + " .reg .b32 %r<3>;\n" + + " .reg .b64 %rd<5>;\n" + + "\n" + + "\n" + + " ld.param.u64 %rd1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_0];\n" + + " ld.param.u32 %r1, [_ZN8__main__2opB2v1B96cw51cXTLSUwv1sCUt9Ww0FEw09RRQPKiLTj0gIGIFp_2b2oLQFEYYkHSQB1OQAk0Bynm21OizQ1K0UoIGvDpQE8oxrNQE_3dEi_param_1];\n" + + " cvt.s64.s32 %rd2, %r1;\n" + + " mul.wide.s32 %rd3, %r1, %r1;\n" + + " sub.s64 %rd4, %rd3, %rd2;\n" + + " st.u64 [%rd1], %rd4;\n" + + " mov.u32 %r2, 0;\n" + + " st.param.b32 [func_retval0+0], %r2;\n" + + " ret;\n" + + "\n" + + "}\n"; static String cuda = "__device__ inline void f(" + "int* output," + From f2282acf399a4004fd2fee4b91f220c74c23eed6 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 6 Apr 2026 02:21:48 +0000 Subject: [PATCH 116/254] Fix symbol loading error handling and improve lru_memory_cache initialization --- cpp/src/librtcx/rtcx.cpp | 11 ++++++----- cpp/src/librtcx/rtcx.hpp | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 4595db6ab8b0..a3d981c07e00 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -335,9 +335,10 @@ void* get_symbol(char const* lib_name, void* handle, char const* sym_name) { void* sym = ::dlsym(handle, sym_name); if (sym == nullptr) { - RTCX_FAIL(std::format( - "Failed to load symbol `{}` from `{}`, error: `{}`", sym_name, lib_name, dlerror()), - std::runtime_error); + RTCX_FAIL( + std::format( + "Failed to load symbol `{}` from `{}`, error: `{}`", sym_name, lib_name, ::dlerror()), + std::runtime_error); } return sym; } @@ -358,7 +359,7 @@ struct LibCuda { LibCuda(LibCuda&&) = delete; LibCuda& operator=(LibCuda const&) = delete; LibCuda& operator=(LibCuda&&) = delete; - ~LibCuda() { dlclose(_handle); } + ~LibCuda() { ::dlclose(_handle); } static void* _load() { @@ -490,7 +491,7 @@ blob_t blob_t::from_buffer(byte_buffer&& buffer) auto data = buffer.release(); return blob_t::from_parts( data, size, +[](std::uint8_t const* data, std::size_t) { - free(const_cast(data)); + ::free(const_cast(data)); }); } diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index c70e54c2fa41..1589fce1b9ed 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -544,10 +544,10 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { void hit(std::uint64_t tick) { last_touched_tick = tick; } }; - std::unordered_map entries_; + std::unordered_map entries_ = {}; std::size_t limit_; - explicit lru_memory_cache(std::size_t limit) : entries_{}, limit_{limit} + explicit lru_memory_cache(std::size_t limit) : limit_{limit} { // reserve space to avoid rehashing entries_.reserve(limit * 2); From 485107ef2362865cbc71dd20eb205ae236883202 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 11 Apr 2026 02:04:06 +0000 Subject: [PATCH 117/254] code review changes --- .../cudf/column/column_device_view.cuh | 14 ++-- cpp/include/cudf/transform.hpp | 5 +- cpp/src/jit/column_device_view_wrappers.cuh | 4 +- cpp/src/jit/sync.cuh | 11 +-- cpp/src/transform/jit/kernel.cu | 3 +- cpp/src/transform/transform.cu | 77 ++++++++++++++----- 6 files changed, 76 insertions(+), 38 deletions(-) diff --git a/cpp/include/cudf/column/column_device_view.cuh b/cpp/include/cudf/column/column_device_view.cuh index f3c8160661a1..027031ca246a 100644 --- a/cpp/include/cudf/column/column_device_view.cuh +++ b/cpp/include/cudf/column/column_device_view.cuh @@ -763,13 +763,13 @@ class alignas(16) mutable_column_device_view : public mutable_column_device_view * @return A `mutable_column_device_view` instance representing the specified column data * in device memory */ - static auto from_parts(data_type type, - size_type size, - void const* data, - bitmask_type const* null_mask, - size_type offset, - mutable_column_device_view* children, - size_type num_children) + [[nodiscard]] static auto create(data_type type, + size_type size, + void const* data, + bitmask_type const* null_mask, + size_type offset, + mutable_column_device_view* children, + size_type num_children) { return mutable_column_device_view{type, size, data, null_mask, offset, children, num_children}; } diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 4c0a94fc9b04..a2a99ed0c23b 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -95,8 +95,9 @@ struct transform_output { * * * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) - * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string - * types + * @throws std::invalid_argument if `output_type` or any of the input types are not supported. + * CUDA-supported types are fixed-width and string types, while PTX-supported types are integral and + * floating-point types. * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in * this case. diff --git a/cpp/src/jit/column_device_view_wrappers.cuh b/cpp/src/jit/column_device_view_wrappers.cuh index fb6343b2b4c7..2f170e6c2fd5 100644 --- a/cpp/src/jit/column_device_view_wrappers.cuh +++ b/cpp/src/jit/column_device_view_wrappers.cuh @@ -12,7 +12,7 @@ #include #include -namespace CUDF_EXPORT cudf { +namespace cudf { namespace jit { /** @@ -123,4 +123,4 @@ struct mutable_strings_column_device_view : private mutable_column_device_view_c }; } // namespace jit -} // namespace CUDF_EXPORT cudf +} // namespace cudf diff --git a/cpp/src/jit/sync.cuh b/cpp/src/jit/sync.cuh index dc153317d145..5d445433e27a 100644 --- a/cpp/src/jit/sync.cuh +++ b/cpp/src/jit/sync.cuh @@ -8,6 +8,7 @@ #include #include +#include namespace cudf { namespace jit { @@ -19,24 +20,24 @@ __device__ inline bool warp_elect(unsigned int mask) return cuda::ptx::elect_sync(mask); #else // fallback: manually elect a leader (e.g., the first active thread) - int leader = __ffs(mask) - 1; + int leader = mask == 0 ? 0 : cuda::std::countr_zero(mask); int lane = (threadIdx.x & 31); return (lane == leader); #endif } template -__device__ void warp_compact_validity(mutable_column_device_view_core const* outcols, +__device__ void warp_compact_validity(unsigned int active_mask, + mutable_column_device_view_core const* outcols, size_type row, bool is_valid) { if constexpr (!Out::may_be_nullable) { return; } else { - auto active = __activemask(); - auto null_word = __ballot_sync(active, is_valid); + auto null_word = __ballot_sync(active_mask, is_valid); // use warp-elect to make sure we only issue one memory transaction per warp - if (warp_elect(active)) { Out::set_null_mask_word(outcols, row / 32, null_word); } + if (warp_elect(active_mask)) { Out::set_null_mask_word(outcols, row / 32, null_word); } } } diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index ffade3ea4468..90b9ff34f110 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -105,7 +105,8 @@ CUDF_KERNEL void transform_kernel(size_type row_size, }); Outs::map([&]() { - (warp_compact_validity(output_cols, element_idx, is_valid[A::index]), ...); + auto active_mask = __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); + (warp_compact_validity(active_mask, output_cols, element_idx, is_valid[A::index]), ...); }); } } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 9e673deaf5a5..7edd7c35dc03 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -35,7 +36,7 @@ namespace cudf { namespace { -struct mut_fixed_width_column_view { +struct mutable_fixed_width_column_view { mutable_column_view _view; auto to_device(rmm::cuda_stream_view stream) const @@ -58,7 +59,7 @@ struct fixed_width_column { make_fixed_width_column(type, size, std::move(null_mask), null_count, stream, mr)}; } - auto mutable_view() const { return mut_fixed_width_column_view{_col->mutable_view()}; } + auto mutable_view() const { return mutable_fixed_width_column_view{_col->mutable_view()}; } void set_null_count(size_type count) { _col->set_null_count(count); } @@ -67,7 +68,7 @@ struct fixed_width_column { bitmask_type* null_mask() { return _col->mutable_view().null_mask(); } }; -struct mut_string_views_column_view { +struct mutable_string_views_column_view { void* _data{nullptr}; size_type _size{0}; bitmask_type const* _null_mask{nullptr}; @@ -79,7 +80,7 @@ struct mut_string_views_column_view { using view = mutable_column_device_view; return std::unique_ptr>( new view{ - view::from_parts(data_type{type_id::EMPTY}, _size, _data, _null_mask, _offset, nullptr, 0)}, + view::create(data_type{type_id::EMPTY}, _size, _data, _null_mask, _offset, nullptr, 0)}, [](auto* p) { delete p; }); } }; @@ -102,9 +103,9 @@ struct string_views_column { auto mutable_view() const { - return mut_string_views_column_view{ + return mutable_string_views_column_view{ const_cast(_data.data()), - static_cast(_data.size()), + _size, static_cast(const_cast(_null_mask.data())), 0, _null_count}; @@ -117,7 +118,7 @@ struct string_views_column { bitmask_type* null_mask() { return static_cast(_null_mask.data()); } }; -struct mut_strings_column_view { +struct mutable_strings_column_view { mutable_column_view _view; auto to_device(rmm::cuda_stream_view stream) const @@ -126,7 +127,7 @@ struct mut_strings_column_view { } }; -struct mut_strings_column { +struct mutable_strings_column { std::unique_ptr _col = nullptr; static auto make(size_type size, @@ -135,11 +136,11 @@ struct mut_strings_column { rmm::device_buffer null_mask, size_type null_count) { - return mut_strings_column{make_strings_column( + return mutable_strings_column{make_strings_column( size, std::move(offsets), std::move(chars), null_count, std::move(null_mask))}; } - auto mutable_view() const { return mut_strings_column_view{_col->mutable_view()}; } + auto mutable_view() const { return mutable_strings_column_view{_col->mutable_view()}; } void set_null_count(size_type count) { _col->set_null_count(count); } @@ -149,10 +150,10 @@ struct mut_strings_column { }; using input_column_view = transform_input; -using output_column = std::variant; -using handle = std::variant< - std::unique_ptr>, - std::unique_ptr>>; +using output_column = std::variant; +using handle = std::variant< + std::unique_ptr>, + std::unique_ptr>>; namespace jit_transform { @@ -205,7 +206,10 @@ std::string reflect_output_element(fixed_width_column const& c) std::string reflect_output_element(string_views_column const&) { return "cudf::string_view"; } -std::string reflect_output_element(mut_strings_column const&) { return "cuda::std::span"; } +std::string reflect_output_element(mutable_strings_column const&) +{ + return "cuda::std::span"; +} std::string reflect_input_column(column_view const&) { return "cudf::column_device_view_core"; } @@ -224,7 +228,7 @@ std::string reflect_output_column(string_views_column const&) return "cudf::jit::mutable_vector_device_view"; } -std::string reflect_output_column(mut_strings_column const&) +std::string reflect_output_column(mutable_strings_column const&) { return "cudf::jit::mutable_strings_column_device_view"; } @@ -261,7 +265,7 @@ auto reflect(udf_source_type source_type, auto optional_element = std::format("cuda::std::optional<{}>", element); bool as_scalar = false; // never scalar bool may_be_nullable = output_may_be_nullable[i]; - auto is_strings_output = std::holds_alternative(out); + auto is_strings_output = std::holds_alternative(out); auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") .instantiate( @@ -367,6 +371,21 @@ void run(null_aware is_null_aware, } // namespace jit_transform +CUDF_KERNEL void copy_offset_bitmask(bitmask_type* __restrict__ destination, + bitmask_type const* __restrict__ source, + size_type source_begin_bit, + size_type source_end_bit, + size_type number_of_mask_words) +{ + auto const stride = cudf::detail::grid_1d::grid_stride(); + for (thread_index_type destination_word_index = grid_1d::global_thread_id(); + destination_word_index < number_of_mask_words; + destination_word_index += stride) { + destination[destination_word_index] = detail::get_mask_offset_word( + source, destination_word_index, source_begin_bit, source_end_bit); + } +} + size_type inplace_null_mask_and(bitmask_type* null_mask, size_type row_size, std::span inputs, @@ -431,11 +450,25 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, return 0; } - auto num_words = num_bitmask_words(row_size); - auto num_bytes = num_words * sizeof(bitmask_type); + auto num_words = num_bitmask_words(row_size); + auto num_bytes = num_words * sizeof(bitmask_type); + constexpr auto bits_per_word = sizeof(bitmask_type) * 8; if (nullable_masks.size() == 1) { // only 1 mask provided, copy it directly to the output + if (nullable_offsets[0] % bits_per_word == 0) { + CUDF_CUDA_TRY(detail::memcpy_async( + null_mask, nullable_masks[0] + (nullable_offsets[0] / bits_per_word), num_bytes, stream)); + } else { + cudf::detail::grid_1d config(row_size, 256); + copy_offset_bitmask<<>>( + static_cast(null_mask), + nullable_masks[0], + nullable_offsets[0], + nullable_offsets[0] + row_size, + num_words); + CUDF_CHECK_CUDA(stream.value()); + } CUDF_CUDA_TRY(detail::memcpy_async(null_mask, nullable_masks[0], num_bytes, stream)); return nullable_null_counts[0]; } @@ -702,7 +735,7 @@ auto make_outputs(null_aware is_null_aware, auto chars_size = strings::detail::get_offset_value(string_offsets[i]->view(), row_size, stream); auto chars = rmm::device_buffer{static_cast(chars_size), stream, mr}; - auto col = mut_strings_column::make( + auto col = mutable_strings_column::make( row_size, std::move(chars), std::move(string_offsets[i]), std::move(null_mask), 0); cols.emplace_back(std::move(col)); } @@ -751,7 +784,9 @@ auto finalize_output(fixed_width_column&& c, rmm::cuda_stream_view, rmm::device_ return std::move(c._col); } -auto finalize_output(mut_strings_column&& c, rmm::cuda_stream_view, rmm::device_async_resource_ref) +auto finalize_output(mutable_strings_column&& c, + rmm::cuda_stream_view, + rmm::device_async_resource_ref) { return std::move(c._col); } From f6209b19a6bf1ac47e6a457c866a4c326842d33a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 11 Apr 2026 08:26:12 +0000 Subject: [PATCH 118/254] feature improvements + added workaround for missing cuda driver functions --- cpp/CMakeLists.txt | 30 +- cpp/src/binaryop/binaryop.cpp | 2 +- cpp/src/binaryop/jit/kernel.cu | 19 +- cpp/src/jit/jit.cpp | 79 +--- cpp/src/join/jit/filter_join_kernel.cu | 23 +- cpp/src/librtcx/CMake/embed.cmake | 53 +-- cpp/src/librtcx/CMake/embed.in.cpp | 373 ++++++++++++++++++ cpp/src/librtcx/CMake/embed.in.py | 222 ----------- cpp/src/librtcx/README.md | 6 +- cpp/src/librtcx/rtcx.cpp | 30 +- cpp/src/librtcx/rtcx.hpp | 23 +- .../rolling/detail/rolling_fixed_window.cu | 9 +- cpp/src/rolling/detail/rolling_jit.cuh | 91 +++++ cpp/src/rolling/detail/rolling_jit.hpp | 50 --- cpp/src/rolling/detail/rolling_udf.cuh | 60 ++- .../rolling/detail/rolling_variable_window.cu | 20 +- cpp/src/rolling/grouped_rolling.cu | 26 +- cpp/src/rolling/jit/kernel.cu | 63 +-- cpp/src/transform/jit/kernel.cu | 32 +- 19 files changed, 718 insertions(+), 493 deletions(-) create mode 100644 cpp/src/librtcx/CMake/embed.in.cpp delete mode 100644 cpp/src/librtcx/CMake/embed.in.py create mode 100644 cpp/src/rolling/detail/rolling_jit.cuh delete mode 100644 cpp/src/rolling/detail/rolling_jit.hpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ef10eec24bb3..2017a306bd5d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -326,19 +326,43 @@ endif() set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) + + +jit_add_include_directory( + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/libcxx DEST_DIRECTORY + librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx +) + jit_add_include_directory( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include DEST_DIRECTORY cudf/cpp/include INCLUDE_DIRECTORIES cudf/cpp/include ) + jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src DEST_DIRECTORY cudf/cpp/src + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/jit DEST_DIRECTORY cudf/cpp/src/jit INCLUDE_DIRECTORIES cudf/cpp/src ) jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/libcxx DEST_DIRECTORY - librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/binaryop/jit DEST_DIRECTORY cudf/cpp/src/binaryop/jit + INCLUDE_DIRECTORIES cudf/cpp/src +) + + +jit_add_include_directory( + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/join/jit DEST_DIRECTORY cudf/cpp/src/join/jit + INCLUDE_DIRECTORIES cudf/cpp/src +) + +jit_add_include_directory( + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling/jit DEST_DIRECTORY cudf/cpp/src/rolling/jit + INCLUDE_DIRECTORIES cudf/cpp/src +) + +jit_add_include_directory( + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/transform/jit DEST_DIRECTORY cudf/cpp/src/transform/jit + INCLUDE_DIRECTORIES cudf/cpp/src ) get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index bd7eed5966b3..6e0ee8fbaf12 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -154,7 +154,7 @@ void binary_operation(mutable_column_view& out, {2, cudf::type_to_name(rhs.type())}, }); - auto kernel_reflection = rtcx::reflect_template("cudf::binops::jit::kernel_v_v", + auto kernel_reflection = rtcx::reflect_template("cudf::binops::jit::binaryop_kernel", output_type_name, cudf::type_to_name(lhs.type()), cudf::type_to_name(rhs.type()), diff --git a/cpp/src/binaryop/jit/kernel.cu b/cpp/src/binaryop/jit/kernel.cu index bcc5b9542fea..6b200a0dce58 100644 --- a/cpp/src/binaryop/jit/kernel.cu +++ b/cpp/src/binaryop/jit/kernel.cu @@ -56,11 +56,14 @@ struct UserDefinedOp { }; template -CUDF_KERNEL void kernel_v_v(cudf::size_type size, - TypeOut* out_data, - TypeLhs* lhs_data, - TypeRhs* rhs_data) +__device__ void binaryop_kernel(cudf::size_type size, + void* p_out_data, + void* p_lhs_data, + void* p_rhs_data) { + auto out_data = static_cast(p_out_data); + auto lhs_data = static_cast(p_lhs_data); + auto rhs_data = static_cast(p_rhs_data); auto const start = cudf::detail::grid_1d::global_thread_id(); auto const step = cudf::detail::grid_1d::grid_stride(); @@ -72,3 +75,11 @@ CUDF_KERNEL void kernel_v_v(cudf::size_type size, } // namespace jit } // namespace binops } // namespace cudf + +extern "C" __global__ void kernel(cudf::size_type size, + void* out_data, + void* lhs_data, + void* rhs_data) +{ + KERNEL_INSTANCE(size, out_data, lhs_data, rhs_data); +} diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index 874ef995be2f..fad72ac7c88e 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -5,7 +5,6 @@ */ #include -#include #include #include @@ -75,40 +74,6 @@ void install_file(char const* dst_path, std::span contents) } } -/** - * @brief Reads the contents of a file into a byte buffer and null-terminates it to allow for safe - * usage as a C-string. - */ -rtcx::byte_buffer read_blob_cstring(char const* path) -{ - int32_t fd = open(path, O_RDONLY); - if (fd == -1) { throw_posix(std::format("Failed to open file ({})", path), "open"); } - - RTCX_DEFER([&] { - if (close(fd) == -1) { throw_posix(std::format("Failed to close file ({})", path), "close"); } - }); - - auto file_size = lseek(fd, 0, SEEK_END); - if (file_size == -1) { - throw_posix(std::format("Failed to determine size of file ({})", path), "lseek"); - } - // TODO: make all read/write syscalls call read/write in a loop - - if (lseek(fd, 0, SEEK_SET) == -1) { - throw_posix(std::format("Failed to reset file offset for file ({})", path), "lseek"); - } - - auto contents = rtcx::byte_buffer::make(file_size + 1U); // +1 for null terminator - - if (read(fd, contents.data(), file_size) == -1) { - throw_posix(std::format("Failed to read file ({})", path), "read"); - } - - contents.data()[file_size] = '\0'; // null-terminate the buffer - - return contents; -} - rtcx::byte_buffer decompress_blob(std::span compressed_binary, size_t uncompressed_size, std::string_view compression) @@ -214,7 +179,6 @@ void jit_bundle_t::ensure_installed() const throw_posix(std::format("Failed to get stat for directory ({})", expected_path), "lstat"); } else { // ensure base install directory exists - CUDF_LOG_INFO("Creating JIT install directory at (%s)", expected_path.c_str()); std::filesystem::create_directories(install_dir_); install_cudf_jit_files(expected_path.c_str(), cache_->get_tmp_dir()); } @@ -289,7 +253,6 @@ std::tuple compile_library_uncached( CUDF_FUNC_RANGE(); auto& bundle = cudf::get_context().jit_bundle(); - auto begin = std::chrono::steady_clock::now(); auto sm = get_current_device_physical_model(); auto include_dirs = bundle.get_include_directories(); @@ -307,7 +270,6 @@ std::tuple compile_library_uncached( // --restrict // --relocatable-device-code // --extensible-whole-program - // --use_fast_math // --dlink-time-opt // --gen-opt-lto // --create-pch @@ -339,7 +301,7 @@ std::tuple compile_library_uncached( for (auto const& option : options) { options_cstr.emplace_back(option.c_str()); } - for (auto const& option : extra_options) { + for (auto* option : extra_options) { options_cstr.emplace_back(option); } @@ -351,20 +313,9 @@ std::tuple compile_library_uncached( .name_expressions = name_expressions, .target_type = rtcx::binary_type::CUBIN}; - auto cubin = rtcx::compile(params); - - auto end = std::chrono::steady_clock::now(); - - auto duration = end - begin; - - CUDF_LOG_INFO( - "Compiled CUDA library `%s` in %f ms", - name, - std::chrono::duration_cast>(duration).count()); - + auto cubin = rtcx::compile(params); auto library = rtcx::load_library(cubin); - - auto blob = rtcx::blob_t::from_buffer(std::move(cubin)); + auto blob = rtcx::blob_t::from_buffer(std::move(cubin)); return std::make_tuple(library, std::make_shared(std::move(blob))); } @@ -415,35 +366,35 @@ kernel_instance={} auto cache_key_sha256 = hash_string(cache_key); auto compile = [&] { - auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); - auto source_file_path = std::format("{}/{}", bundle_dir, source_file); - auto source = read_blob_cstring(source_file_path.c_str()); - char const* name_expressions[] = {kernel_instance.c_str()}; + auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); + auto source = std::format(R"***( +#include "{}" + )***", + source_file); + auto kernel_instance_define = std::format("-DKERNEL_INSTANCE=\"{}\"", kernel_instance); + + char const* options[] = {kernel_instance_define.c_str()}; return compile_library_uncached(name.c_str(), reinterpret_cast(source.data()), header_include_names, headers, + options, {}, - name_expressions, use_pch, log_pch); }; if (!use_cache) { auto [lib, blob] = compile(); - auto kernels = lib->enumerate_kernels(); - CUDF_EXPECTS(kernels.size() == 1, "Unexpected kernel count", std::logic_error); - return kernel{lib, kernels[0]}; + return kernel{lib, lib->get_kernel("kernel")}; } auto fut = cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); - auto lib = fut.get(); - auto kernels = lib->enumerate_kernels(); - CUDF_EXPECTS(kernels.size() == 1, "Unexpected kernel count", std::logic_error); - return kernel{lib, kernels[0]}; + auto lib = fut.get(); + return kernel{lib, lib->get_kernel("kernel")}; } } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index 103cd92637da..eda46ebc53ee 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -31,12 +31,12 @@ namespace cudf::join::jit { constexpr cudf::size_type JoinNoMatch = cuda::std::numeric_limits::min(); template -CUDF_KERNEL void filter_join_kernel(cudf::jit::device_span left_indices, - cudf::jit::device_span right_indices, - cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - bool* predicate_results, - void* user_data) +__device__ void filter_join_kernel(cudf::jit::device_span left_indices, + cudf::jit::device_span right_indices, + cudf::column_device_view_core const* left_tables, + cudf::column_device_view_core const* right_tables, + bool* predicate_results, + void* user_data) { auto const start = cudf::detail::grid_1d::global_thread_id(); auto const stride = cudf::detail::grid_1d::grid_stride(); @@ -92,3 +92,14 @@ CUDF_KERNEL void filter_join_kernel(cudf::jit::device_span left_indices, + cudf::jit::device_span right_indices, + cudf::column_device_view_core const* left_tables, + cudf::column_device_view_core const* right_tables, + bool* predicate_results, + void* user_data) +{ + KERNEL_INSTANCE( + left_indices, right_indices, left_tables, right_tables, predicate_results, user_data); +} diff --git a/cpp/src/librtcx/CMake/embed.cmake b/cpp/src/librtcx/CMake/embed.cmake index a719acb25842..0df801f90579 100644 --- a/cpp/src/librtcx/CMake/embed.cmake +++ b/cpp/src/librtcx/CMake/embed.cmake @@ -5,10 +5,9 @@ # cmake-format: on # ============================================================================= -find_package(Python3 REQUIRED COMPONENTS Interpreter) +find_package(OpenSSL REQUIRED COMPONENTS Crypto) +find_package(zstd REQUIRED) -# TODO: add glob restrictions:exclude .cpp .cc, .cmake and .cxx source files, only include .h .hpp -# .cuh .cu files # This function registers a directory of include files to be embedded for JIT compilation. It # gathers the specified files, their destinations, and include directories, and stores them in @@ -65,9 +64,9 @@ function(jit_add_include_directory) # Set scope variables to accumulate results - set(SOURCE_FILES ${jitembed_${TARGET}_incdir__source_files}) - set(SOURCE_FILE_DESTS ${jitembed_${TARGET}_incdir__source_file_dests}) - set(INCLUDE_DIRECTORIES ${jitembed_${TARGET}_incdir__include_directories}) + set(SOURCE_FILES ${${TARGET}__jitembed_incdir__source_files}) + set(SOURCE_FILE_DESTS ${${TARGET}__jitembed_incdir__source_file_dests}) + set(INCLUDE_DIRECTORIES ${${TARGET}__jitembed_incdir__include_directories}) foreach(SOURCE_FILE IN LISTS ARG_FILES) list(APPEND SOURCE_FILES "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") @@ -76,15 +75,15 @@ function(jit_add_include_directory) list(APPEND INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES}) - set(jitembed_${TARGET}_incdir__source_files + set(${TARGET}__jitembed_incdir__source_files ${SOURCE_FILES} PARENT_SCOPE ) - set(jitembed_${TARGET}_incdir__source_file_dests + set(${TARGET}__jitembed_incdir__source_file_dests ${SOURCE_FILE_DESTS} PARENT_SCOPE ) - set(jitembed_${TARGET}_incdir__include_directories + set(${TARGET}__jitembed_incdir__include_directories ${INCLUDE_DIRECTORIES} PARENT_SCOPE ) @@ -111,7 +110,7 @@ function(jit_embed) message(FATAL_ERROR "COMPRESSION argument must be either none or zstd") endif() - if(NOT DEFINED jitembed_${TARGET}_incdir__source_files) + if(NOT DEFINED ${TARGET}__jitembed_incdir__source_files) message( FATAL_ERROR "No source files registered for target '${TARGET}'. Call jit_add_include_directory() first" @@ -119,23 +118,31 @@ function(jit_embed) endif() set(OUTPUT_DIR "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") - set(CONFIGURED_EMBED_PY "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_embed.py") - set(EMBED_PY_IN "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.py") - - set(RTCX_EMBED_PY_ARG__ID "${TARGET}") - set(RTCX_EMBED_PY_ARG__FILE_PATHS "${jitembed_${TARGET}_incdir__source_files}") - set(RTCX_EMBED_PY_ARG__FILE_DESTS "${jitembed_${TARGET}_incdir__source_file_dests}") - set(RTCX_EMBED_PY_ARG__INCLUDE_DIRS "${jitembed_${TARGET}_incdir__include_directories}") - set(RTCX_EMBED_PY_ARG__COMPRESSION "${ARG_COMPRESSION}") - set(RTCX_EMBED_PY_ARG__OUTPUT_DIR "${OUTPUT_DIR}") - - configure_file("${EMBED_PY_IN}" "${CONFIGURED_EMBED_PY}" @ONLY) + set(CONFIGURED_EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed.cpp") + set(EMBED_SCRIPT_IN "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.cpp") + + set(RTCX_EMBED_SCRIPT_ARG__ID "${TARGET}") + set(RTCX_EMBED_SCRIPT_ARG__FILE_PATHS "${${TARGET}__jitembed_incdir__source_files}") + set(RTCX_EMBED_SCRIPT_ARG__FILE_DESTS "${${TARGET}__jitembed_incdir__source_file_dests}") + set(RTCX_EMBED_SCRIPT_ARG__INCLUDE_DIRS "${${TARGET}__jitembed_incdir__include_directories}") + set(RTCX_EMBED_SCRIPT_ARG__COMPRESSION "${ARG_COMPRESSION}") + set(RTCX_EMBED_SCRIPT_ARG__OUTPUT_DIR "${OUTPUT_DIR}") + + configure_file("${EMBED_SCRIPT_IN}" "${CONFIGURED_EMBED_SCRIPT}" @ONLY) + + add_executable("${TARGET}__jit_embed_run" EXCLUDE_FROM_ALL "${CONFIGURED_EMBED_SCRIPT}") + target_include_directories("${TARGET}__jit_embed_run" PRIVATE ${ZSTD_INCLUDE_DIR}) + target_link_libraries("${TARGET}__jit_embed_run" PRIVATE ${CMAKE_DL_LIBS} zstd OpenSSL::Crypto) + set_target_properties( + "${TARGET}__jit_embed_run" PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES + ) add_custom_command( OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin BYPRODUCTS ${OUTPUT_DIR}/* - COMMAND ${Python3_EXECUTABLE} "${CONFIGURED_EMBED_PY}" - DEPENDS "${EMBED_PY_IN}" "${CONFIGURED_EMBED_PY}" ${jitembed_${TARGET}_incdir__source_files} + COMMAND "${CMAKE_COMMAND}" -E env $ + DEPENDS "${EMBED_SCRIPT_IN}" "${CONFIGURED_EMBED_SCRIPT}" + ${${TARGET}__jitembed_incdir__source_files} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" COMMENT "Generating JIT embed for ${TARGET} into ${OUTPUT_DIR}" VERBATIM diff --git a/cpp/src/librtcx/CMake/embed.in.cpp b/cpp/src/librtcx/CMake/embed.in.cpp new file mode 100644 index 000000000000..28dca324cd10 --- /dev/null +++ b/cpp/src/librtcx/CMake/embed.in.cpp @@ -0,0 +1,373 @@ + + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEFER__CONCATENATE_DETAIL(x, y) x##y +#define DEFER__CONCATENATE(x, y) DEFER__CONCATENATE_DETAIL(x, y) +#define DEFER(...) ::defer DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) + +template +struct defer { + private: + T func_; + + public: + template + defer(Args&&... args) : func_{static_cast(args)...} + { + } + defer(defer const&) = delete; + defer& operator=(defer const&) = delete; + defer(defer&&) = delete; + defer& operator=(defer&&) = delete; + ~defer() { func_(); } +}; + +template +defer(T) -> defer; + +struct size_range { + size_t offset = 0; + size_t size = 0; +}; + +std::pair, std::vector> merge_bytes_with_null_terminators( + std::span const> bytes_lists) +{ + std::vector merged; + std::vector ranges; + + for (auto& byte_data : bytes_lists) { + ranges.push_back({merged.size(), byte_data.size()}); + merged.insert(merged.end(), byte_data.begin(), byte_data.end()); + merged.push_back(0); + } + + return {std::move(merged), std::move(ranges)}; +} + +struct embed_output { + std::string cxx_header; + std::string cxx_source; + std::string asm_source; + std::string bin_file_name; + std::vector bin_file_data; + std::vector hash; +}; + +std::vector load_file_bytes(std::string_view file_path) +{ + std::string path_str(file_path); + std::ifstream file(path_str, std::ios::binary | std::ios::ate); + if (!file) { + throw std::runtime_error(std::format("Failed to open file at path: {}", file_path)); + } + auto file_size = file.tellg(); + if (file_size < 0) { + throw std::runtime_error( + std::format("Failed to determine size of file at path: {}", file_path)); + } + file.seekg(0, std::ios::beg); + std::vector bytes(file_size); + if (!file.read(reinterpret_cast(bytes.data()), file_size)) { + throw std::runtime_error(std::format("Failed to read file at path: {}", file_path)); + } + return bytes; +} + +std::vector compress_bytes(std::span bytes, std::string_view compression) +{ + if (compression != "none" && compression != "zstd") { + throw std::invalid_argument(std::format( + "Invalid compression type: {}. Supported values are 'none' and 'zstd'", compression)); + } + + if (compression == "none") { return std::vector(bytes.begin(), bytes.end()); } + + auto const max_compressed_size = ZSTD_compressBound(bytes.size()); + std::vector compressed(max_compressed_size); + auto const compressed_size = + ZSTD_compress(compressed.data(), compressed.size(), bytes.data(), bytes.size(), 22); + + if (ZSTD_isError(compressed_size)) { + throw std::runtime_error(std::string("ZSTD compression failed: ") + + ZSTD_getErrorName(compressed_size)); + } + + compressed.resize(compressed_size); + return compressed; +} + +std::vector compute_embed_hash(std::span uncompressed_files_bytes, + std::span merged_dests_bytes, + std::span merged_include_dirs_bytes, + std::string_view compression) +{ + std::vector hash(EVP_MD_size(EVP_sha256())); + EVP_MD_CTX* sha = EVP_MD_CTX_new(); + if (sha == nullptr) { throw std::runtime_error("Failed to allocate EVP_MD_CTX"); } + + DEFER([&] { EVP_MD_CTX_free(sha); }); + + if (EVP_DigestInit_ex(sha, EVP_sha256(), nullptr) != 1) { + throw std::runtime_error("EVP_DigestInit_ex failed"); + } + + if (EVP_DigestUpdate(sha, uncompressed_files_bytes.data(), uncompressed_files_bytes.size()) != + 1 || + EVP_DigestUpdate(sha, merged_dests_bytes.data(), merged_dests_bytes.size()) != 1 || + EVP_DigestUpdate(sha, merged_include_dirs_bytes.data(), merged_include_dirs_bytes.size()) != + 1 || + EVP_DigestUpdate(sha, compression.data(), compression.size()) != 1) { + throw std::runtime_error("EVP_DigestUpdate failed"); + } + + auto hash_size = static_cast(hash.size()); + if (EVP_DigestFinal_ex(sha, hash.data(), &hash_size) != 1) { + throw std::runtime_error("EVP_DigestFinal_ex failed"); + } + + hash.resize(hash_size); + + return hash; +} + +template +std::string join_formatted(Container& items, std::string_view delimiter, Formatter&& formatter) +{ + std::ostringstream result; + for (std::size_t i = 0; i < items.size(); ++i) { + if (i != 0) { result << delimiter; } + result << formatter(items[i]); + } + return result.str(); +} + +embed_output generate_cxx_source_files_data(std::string_view id, + std::span file_paths, + std::span file_dsts, + std::span include_dirs, + std::string_view compression) +{ + std::vector> file_bytes; + file_bytes.reserve(file_paths.size()); + for (auto const& path : file_paths) { + file_bytes.emplace_back(load_file_bytes(path)); + } + + auto [uncompressed_files_bytes, files_ranges] = merge_bytes_with_null_terminators(file_bytes); + + auto compress = compression != "none"; + std::vector compressed_files_bytes = + compress ? compress_bytes(uncompressed_files_bytes, compression) : uncompressed_files_bytes; + + auto binary_size = compress ? compressed_files_bytes.size() + : static_cast(uncompressed_files_bytes.size()); + + if (compress) { + std::cout << std::format( + "-- Compressed {}'s binary from {} bytes to {} bytes (compression ratio: {:.2f})\n", + id, + uncompressed_files_bytes.size(), + compressed_files_bytes.size(), + static_cast(compressed_files_bytes.size()) / + static_cast(uncompressed_files_bytes.size())); + } + + std::vector> destination_bytes; + destination_bytes.reserve(file_dsts.size()); + for (auto const& dest : file_dsts) { + destination_bytes.emplace_back(dest.begin(), dest.end()); + } + auto [merged_dests_bytes, _] = merge_bytes_with_null_terminators(destination_bytes); + + std::vector> include_directory_bytes; + include_directory_bytes.reserve(include_dirs.size()); + for (auto const& include_directory : include_dirs) { + include_directory_bytes.emplace_back(include_directory.begin(), include_directory.end()); + } + auto [merged_include_dirs_bytes, __] = merge_bytes_with_null_terminators(include_directory_bytes); + + auto hash = compute_embed_hash( + uncompressed_files_bytes, merged_dests_bytes, merged_include_dirs_bytes, compression); + + auto binary_file_name = std::format("embed_{}.bin", id); + + auto include_dirs_list = + join_formatted(include_dirs, ",\n", [](auto s) { return std::format("\"{}\"", s); }); + auto dests_list = + join_formatted(file_dsts, ",\n", [](auto s) { return std::format("\"{}\"", s); }); + auto ranges_list = join_formatted( + files_ranges, ",\n", [](auto r) { return std::format("{{{}, {}}}", r.offset, r.size); }); + + auto hash_list = join_formatted( + hash, ", ", [](uint8_t byte) { return std::format("0x{:02x}", static_cast(byte)); }); + + auto cxx_header = std::format( + R"***( +// Auto-generated header for embedded files with ID: {} +#pragma once + +#include +#include +#include +#include + +namespace rtcx_embed {{ + +struct range {{ + std::size_t offset = 0; + std::size_t size = 0; +}}; + +constexpr char const * {}_include_directories[{}] = +{{ +{} +}}; + +constexpr char const * {}_file_destinations[{}] = +{{ +{} +}}; + +constexpr range {}_file_ranges[{}] = +{{ +{} +}}; + +constexpr std::size_t {}_files_uncompressed_size = {}; + +constexpr char const * {}_files_compression = "{}"; + +extern "C" std::uint8_t const rtcx_embed_{}_files_begin[]; + +static std::span const {}_files = +{{ +rtcx_embed_{}_files_begin, +{}L +}}; + +constexpr std::uint8_t {}_hash[{}] = +{{ +{} +}}; + +}} +)***", + id, + id, + include_dirs.size(), + include_dirs_list, + id, + file_dsts.size(), + dests_list, + id, + files_ranges.size(), + ranges_list, + id, + uncompressed_files_bytes.size(), + id, + compression, + id, + id, + id, + binary_size, + id, + hash.size(), + hash_list); + + auto asm_source = std::format( + R"***( +.section .rodata +.global rtcx_embed_{}_files_begin +rtcx_embed_{}_files_begin: +.incbin "{}" + +.section .note.GNU-stack,"",@progbits +)***", + id, + id, + binary_file_name); + + return embed_output{ + .cxx_header = cxx_header, + .cxx_source = "", + .asm_source = asm_source, + .bin_file_name = binary_file_name, + .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes, + .hash = std::move(hash), + }; +} + +void generate_embed(std::string_view id, + std::span file_paths, + std::span file_dsts, + std::span include_dirs, + std::string_view compression, + std::string_view output_directory) +{ + auto output = + generate_cxx_source_files_data(id, file_paths, file_dsts, include_dirs, compression); + + std::filesystem::create_directories(std::filesystem::path(output_directory)); + + std::ofstream header_file(std::format("{}/{}.hpp", output_directory, id)); + header_file << output.cxx_header; + + std::ofstream asm_file(std::format("{}/{}.s", output_directory, id)); + asm_file << output.asm_source; + + std::ofstream bin_file(std::format("{}/{}", output_directory, output.bin_file_name), + std::ios::binary); + bin_file.write(reinterpret_cast(output.bin_file_data.data()), + static_cast(output.bin_file_data.size())); +} + +std::vector split_string(std::string_view str, char delimeter) +{ + std::vector tokens; + std::size_t start = 0; + + while (start <= str.size()) { + auto const pos = str.find(delimeter, start); + if (pos == std::string_view::npos) { + tokens.push_back(str.substr(start)); + break; + } + tokens.push_back(str.substr(start, pos - start)); + start = pos + 1; + } + + return tokens; +} + +int main() +{ + std::string_view id = "@RTCX_EMBED_SCRIPT_ARG__ID@"; + auto file_paths = split_string("@RTCX_EMBED_SCRIPT_ARG__FILE_PATHS@", ';'); + auto file_dests = split_string("@RTCX_EMBED_SCRIPT_ARG__FILE_DESTS@", ';'); + auto include_directories = split_string("@RTCX_EMBED_SCRIPT_ARG__INCLUDE_DIRS@", ';'); + std::string_view compression = "@RTCX_EMBED_SCRIPT_ARG__COMPRESSION@"; + std::string_view output_dir = "@RTCX_EMBED_SCRIPT_ARG__OUTPUT_DIR@"; + + generate_embed(id, file_paths, file_dests, include_directories, compression, output_dir); + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/cpp/src/librtcx/CMake/embed.in.py b/cpp/src/librtcx/CMake/embed.in.py deleted file mode 100644 index ca455989fb16..000000000000 --- a/cpp/src/librtcx/CMake/embed.in.py +++ /dev/null @@ -1,222 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -import hashlib -import os -from typing import NamedTuple - -import zstandard - - -def merge_bytes_with_null_terminators( - bytes_lists: list[bytes], -) -> tuple[bytes, list[tuple[int, int]]]: - merged: bytes = bytes() - ranges: list[tuple[int, int]] = [] - - for byte_data in bytes_lists: - ranges.append((len(merged), len(byte_data))) - merged += byte_data + b"\0" - - return merged, ranges - - -class EmbedOutput(NamedTuple): - cxx_header: str | None - cxx_source: str | None - asm_source: str | None - bin_file_name: str | None - bin_file_data: bytes | None - hash: bytes - - -def load_file_bytes(file_path: str) -> bytes: - with open(file_path, "rb") as f: - return f.read() - - -def compress_bytes(data: bytes, compression: str) -> bytes: - assert compression in ("none", "zstd"), ( - f"Invalid compression type: {compression}, expected 'none' or 'zstd'" - ) - - if compression == "none": - return data - elif compression == "zstd": - return zstandard.compress(data, 22) - - -def generate_cxx_source_files_data( - id: str, - file_paths: list[str], - dests: list[str], - include_directories: list[str], - compression: str, -) -> EmbedOutput: - uncompressed_files_bytes, files_ranges = merge_bytes_with_null_terminators( - [load_file_bytes(p) for p in file_paths] - ) - - compress = compression != "none" - - compressed_files_bytes = ( - compress_bytes(uncompressed_files_bytes, compression) - if compress - else None - ) - - binary_size = ( - len(compressed_files_bytes) - if compress - else len(uncompressed_files_bytes) - ) - - if compress: - print( - f"-- Compressed {id}'s binary from {len(uncompressed_files_bytes)} bytes to {len(compressed_files_bytes)} bytes (compression ratio: {len(compressed_files_bytes) / len(uncompressed_files_bytes):.2f})" - ) - - merged_dests_bytes, _ = merge_bytes_with_null_terminators( - [d.encode("utf-8") for d in dests] - ) - - merged_include_directories_bytes, _ = merge_bytes_with_null_terminators( - [d.encode("utf-8") for d in include_directories] - ) - - # compute combined sha256 hash of all files - sha = hashlib.sha256() - sha.update(uncompressed_files_bytes) - sha.update(merged_dests_bytes) - sha.update(merged_include_directories_bytes) - sha.update(compression.encode("utf-8")) - - hash: bytes = sha.digest() - - binary_file_name = f"embed_{id}.bin" - - include_dirs_list = ",\n".join([f'"{d}"' for d in include_directories]) - dests_list = ",\n".join([f'"{d}"' for d in dests]) - ranges_list = ",\n".join( - [f"{{{offset}, {size}}}" for offset, size in files_ranges] - ) - hash_list = ", ".join([f"0x{b:02x}" for b in hash]) - - cxx_header = f""" -// Auto-generated header for embedded files with ID: {id} -#pragma once - -#include -#include -#include -#include - -namespace rtcx_embed {{ - -struct range {{ - std::size_t offset = 0; - std::size_t size = 0; -}}; - -constexpr char const * {id}_include_directories[{len(include_directories)}] = -{{ -{include_dirs_list} -}}; - -constexpr char const * {id}_file_destinations[{len(dests)}] = -{{ -{dests_list} -}}; - -constexpr range {id}_file_ranges[{len(files_ranges)}] = -{{ -{ranges_list} -}}; - -constexpr std::size_t {id}_files_uncompressed_size = {len(uncompressed_files_bytes)}; - -constexpr char const * {id}_files_compression = "{compression}"; - -extern "C" std::uint8_t const rtcx_embed_{id}_files_begin[]; - -static std::span const {id}_files = -{{ -rtcx_embed_{id}_files_begin, -{binary_size}L -}}; - -constexpr std::uint8_t {id}_hash[{len(hash)}] = -{{ -{hash_list} -}}; - -}} -""" - - asm_source = f""" -.section .rodata -.global rtcx_embed_{id}_files_begin -rtcx_embed_{id}_files_begin: -.incbin "{binary_file_name}" - -.section .note.GNU-stack,"",@progbits -""" - - return EmbedOutput( - cxx_header=cxx_header, - cxx_source=None, - asm_source=asm_source, - bin_file_name=binary_file_name, - bin_file_data=compressed_files_bytes - if compress - else uncompressed_files_bytes, - hash=hash, - ) - - -def generate_embed( - id: str, - file_paths: list[str], - file_dests: list[str], - include_directories: list[str], - compression: str, - output_dir: str, -): - output = generate_cxx_source_files_data( - id, file_paths, file_dests, include_directories, compression - ) - - os.makedirs(output_dir, exist_ok=True) - - with open(f"{output_dir}/{id}.hpp", "w") as f: - f.write(output.cxx_header if output.cxx_header is not None else "") - - with open(f"{output_dir}/{id}.s", "w") as f: - f.write(output.asm_source if output.asm_source is not None else "") - - if output.bin_file_name and output.bin_file_data: - with open(f"{output_dir}/{output.bin_file_name}", "wb") as f: - f.write(output.bin_file_data) - - -def main(): - id: str = "@RTCX_EMBED_PY_ARG__ID@" - file_paths: list[str] = "@RTCX_EMBED_PY_ARG__FILE_PATHS@".split(";") - file_dests: list[str] = "@RTCX_EMBED_PY_ARG__FILE_DESTS@".split(";") - include_directories: list[str] = "@RTCX_EMBED_PY_ARG__INCLUDE_DIRS@".split( - ";" - ) - compression: str = "@RTCX_EMBED_PY_ARG__COMPRESSION@" - output_dir: str = "@RTCX_EMBED_PY_ARG__OUTPUT_DIR@" - - generate_embed( - id, - file_paths, - file_dests, - include_directories, - compression, - output_dir, - ) - - -if __name__ == "__main__": - main() diff --git a/cpp/src/librtcx/README.md b/cpp/src/librtcx/README.md index 1bf8a55599af..bad3b9acbd88 100644 --- a/cpp/src/librtcx/README.md +++ b/cpp/src/librtcx/README.md @@ -12,8 +12,8 @@ RTCX (runtime-compiler extended) is a wrapper around NVRTC and NVJitLink designe ## Build-Scripting Requirements - CMake -- python 3 -- python-zstd +- LibOpenSSL - for binary hashing +- LibZSTD - for binary compression ## Runtime Requirements -- CUDA >= 12.4.0 +- CUDA >= 11.8 diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index a3d981c07e00..73677e44ee8a 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -222,8 +222,6 @@ sha256 sha256_context::finalize() DO_IT(LibraryLoadData) \ DO_IT(LibraryLoadFromFile) \ DO_IT(LibraryGetKernel) \ - DO_IT(LibraryGetKernelCount) \ - DO_IT(LibraryEnumerateKernels) \ DO_IT(LibraryUnload) #define FOR_EACH_NVRTC_FUNC(DO_IT) \ @@ -880,24 +878,6 @@ kernel_ref library_t::get_kernel(char const* name) const return kernel_ref{kernel}; } -std::vector library_t::enumerate_kernels() const -{ - std::uint32_t num_kernels; - RTCX_CHECK_CUDA(cu->LibraryGetKernelCount(&num_kernels, handle_)); - - std::vector kernels; - kernels.resize(num_kernels); - - RTCX_CHECK_CUDA(cu->LibraryEnumerateKernels(kernels.data(), num_kernels, handle_)); - - std::vector result; - for (CUkernel k : kernels) { - result.emplace_back(k); - } - - return result; -} - std::string demangle_cuda_symbol(char const* mangled_name) { std::int32_t status; @@ -961,6 +941,12 @@ std::optional blob_t::from_file(char const* path) } } + RTCX_DEFER([&] { + if (::close(fd) == -1) { + throw_posix("Failed to close RTCX cache file after memory-mapping", "close"); + } + }); + auto file_size = ::lseek(fd, 0, SEEK_END); if (file_size == -1) { throw_posix("Failed to determine size of RTCX cache file", "lseek"); } @@ -968,10 +954,6 @@ std::optional blob_t::from_file(char const* path) if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTCX cache file", "mmap"); } - if (::close(fd) == -1) { - throw_posix("Failed to close RTCX cache file after memory-mapping", "close"); - } - auto deleter = +[](std::uint8_t const* buffer, std::size_t size) { if (::munmap(static_cast(const_cast(buffer)), size) == -1) { throw_posix("Failed to unmap RTCX cache file from memory", "munmap"); diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 1589fce1b9ed..5f6ffc73fc1a 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -149,11 +149,11 @@ struct [[nodiscard]] sha256_hasher { alignas(16) std::uint64_t v[4]; }; - auto value = std::bit_cast(obj); - auto const h0 = value.v[0]; - auto const h1 = value.v[1]; - auto const h2 = value.v[2]; - auto const h3 = value.v[3]; + auto value = std::bit_cast(obj); + auto h0 = value.v[0]; + auto h1 = value.v[1]; + auto h2 = value.v[2]; + auto h3 = value.v[3]; auto mix = [](std::uint64_t seed, std::uint64_t v) { seed ^= v + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); @@ -481,12 +481,6 @@ struct [[nodiscard]] library_t { * @brief Retrieve a kernel from the library by name */ [[nodiscard]] kernel_ref get_kernel(char const* name) const; - - /** - * @brief Enumerate all kernels contained in the library, returning a vector of kernel references - * @return A vector of kernel_ref objects representing all kernels contained in the library - */ - [[nodiscard]] std::vector enumerate_kernels() const; }; using library = std::shared_ptr; @@ -562,13 +556,12 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { std::vector> rankings; rankings.reserve(entries_.size()); - for (auto const& [key, entry] : entries_) { + for (auto& [key, entry] : entries_) { rankings.emplace_back(key, entry.last_touched_tick); } - std::sort(rankings.begin(), rankings.end(), [](auto const& a, auto const& b) { - return a.second < b.second; - }); + std::sort( + rankings.begin(), rankings.end(), [](auto& a, auto& b) { return a.second < b.second; }); // purge least recently used half rankings.resize(num_to_purge); diff --git a/cpp/src/rolling/detail/rolling_fixed_window.cu b/cpp/src/rolling/detail/rolling_fixed_window.cu index f91fccb35313..1a29aaae40c8 100644 --- a/cpp/src/rolling/detail/rolling_fixed_window.cu +++ b/cpp/src/rolling/detail/rolling_fixed_window.cu @@ -1,9 +1,10 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #include "rolling.cuh" +#include "rolling_jit.cuh" #include "rolling_udf.cuh" #include "rolling_utils.cuh" @@ -40,10 +41,8 @@ std::unique_ptr rolling_window(column_view const& input, if (agg.kind == aggregation::CUDA || agg.kind == aggregation::PTX) { // TODO: In future, might need to clamp preceding/following to column boundaries. return cudf::detail::rolling_window_udf(input, - preceding_window, - "cudf::size_type", - following_window, - "cudf::size_type", + cudf::detail::fixed_window_wrapper(preceding_window), + cudf::detail::fixed_window_wrapper(following_window), min_periods, agg, stream, diff --git a/cpp/src/rolling/detail/rolling_jit.cuh b/cpp/src/rolling/detail/rolling_jit.cuh new file mode 100644 index 000000000000..47fc30064cc8 --- /dev/null +++ b/cpp/src/rolling/detail/rolling_jit.cuh @@ -0,0 +1,91 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace cudf { + +namespace detail { + +template +T minimum(T a, T b) +{ + return b < a ? b : a; +} + +struct window_wrapper_base { + cudf::size_type const* group_offsets = nullptr; + cudf::size_type const* group_labels = nullptr; + cudf::size_type window = 0; +}; + +struct fixed_window_wrapper : public window_wrapper_base { + __device__ __host__ fixed_window_wrapper(cudf::size_type window) + : window_wrapper_base{nullptr, nullptr, window} + { + } + + __device__ __host__ cudf::size_type operator[](cudf::size_type) { return window; } +}; + +struct variable_window_wrapper : public window_wrapper_base { + __device__ __host__ variable_window_wrapper(cudf::size_type const* group_offsets) + : window_wrapper_base{group_offsets, nullptr, 0} + { + } + + __device__ __host__ cudf::size_type operator[](cudf::size_type idx) { return group_offsets[idx]; } +}; + +struct preceding_window_wrapper : public window_wrapper_base { + __device__ __host__ preceding_window_wrapper(cudf::size_type const* group_offsets, + cudf::size_type const* group_labels, + cudf::size_type window) + : window_wrapper_base{group_offsets, group_labels, window} + { + } + + __device__ cudf::size_type operator[](cudf::size_type idx) + { + auto group_label = group_labels[idx]; + auto group_start = group_offsets[group_label]; + return minimum(window, idx - group_start + 1); // Preceding includes current row. + } +}; + +struct following_window_wrapper : public window_wrapper_base { + __device__ __host__ following_window_wrapper(cudf::size_type const* group_offsets, + cudf::size_type const* group_labels, + cudf::size_type window) + : window_wrapper_base{group_offsets, group_labels, window} + { + } + + __device__ cudf::size_type operator[](cudf::size_type idx) + { + auto group_label = group_labels[idx]; + auto group_end = + group_offsets[group_label + + 1]; // Cannot fall off the end, since offsets is capped with `input.size()`. + return minimum(window, (group_end - 1) - idx); + } +}; + +static_assert(sizeof(fixed_window_wrapper) == sizeof(variable_window_wrapper)); +static_assert(alignof(fixed_window_wrapper) == alignof(variable_window_wrapper)); + +static_assert(sizeof(variable_window_wrapper) == sizeof(fixed_window_wrapper)); +static_assert(alignof(variable_window_wrapper) == alignof(fixed_window_wrapper)); + +static_assert(sizeof(fixed_window_wrapper) == sizeof(preceding_window_wrapper)); +static_assert(alignof(fixed_window_wrapper) == alignof(preceding_window_wrapper)); + +static_assert(sizeof(fixed_window_wrapper) == sizeof(following_window_wrapper)); +static_assert(alignof(fixed_window_wrapper) == alignof(following_window_wrapper)); +} // namespace detail + +} // namespace cudf diff --git a/cpp/src/rolling/detail/rolling_jit.hpp b/cpp/src/rolling/detail/rolling_jit.hpp deleted file mode 100644 index facc9a881837..000000000000 --- a/cpp/src/rolling/detail/rolling_jit.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2020-2023, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include - -namespace cudf { - -namespace detail { - -template -T minimum(T a, T b) -{ - return b < a ? b : a; -} - -struct preceding_window_wrapper { - cudf::size_type const* d_group_offsets; - cudf::size_type const* d_group_labels; - cudf::size_type preceding_window; - - cudf::size_type operator[](cudf::size_type idx) - { - auto group_label = d_group_labels[idx]; - auto group_start = d_group_offsets[group_label]; - return minimum(preceding_window, idx - group_start + 1); // Preceding includes current row. - } -}; - -struct following_window_wrapper { - cudf::size_type const* d_group_offsets; - cudf::size_type const* d_group_labels; - cudf::size_type following_window; - - cudf::size_type operator[](cudf::size_type idx) - { - auto group_label = d_group_labels[idx]; - auto group_end = - d_group_offsets[group_label + - 1]; // Cannot fall off the end, since offsets is capped with `input.size()`. - return minimum(following_window, (group_end - 1) - idx); - } -}; - -} // namespace detail - -} // namespace cudf diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index f8a2acbb4888..cf4010e4d929 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -10,7 +10,7 @@ #include "jit/parser.hpp" #include "jit/util.hpp" #include "rolling.hpp" -#include "rolling_jit.hpp" +#include "rolling_jit.cuh" #include #include @@ -26,17 +26,32 @@ namespace cudf { namespace detail { +template +std::string reflect_window_wrapper() +{ + if constexpr (std::is_same_v) { + return "cudf::detail::fixed_window_wrapper"; + } else if constexpr (std::is_same_v) { + return "cudf::detail::variable_window_wrapper"; + } else if constexpr (std::is_same_v) { + return "cudf::detail::preceding_window_wrapper"; + } else { + static_assert(std::is_same_v, "Unsupported window wrapper type"); + return "cudf::detail::following_window_wrapper"; + } +} + // Applies a user-defined rolling window function to the values in a column. -template -std::unique_ptr rolling_window_udf(column_view const& input, - PrecedingWindowIterator preceding_window, - std::string const& preceding_window_str, - FollowingWindowIterator following_window, - std::string const& following_window_str, - size_type min_periods, - rolling_aggregation const& agg, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::unique_ptr rolling_window_udf( + column_view const& input, + std::string const& preceding_window_str, + cudf::detail::window_wrapper_base const& preceding_window, + std::string const& following_window_str, + cudf::detail::window_wrapper_base const& following_window, + size_type min_periods, + rolling_aggregation const& agg, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { static_assert(warp_size == cudf::detail::size_in_bits(), "bitmask_type size does not match CUDA warp size"); @@ -73,7 +88,7 @@ std::unique_ptr rolling_window_udf(column_view const& input, cudf::detail::device_scalar device_valid_count{0, stream}; auto kernel_reflection = - rtcx::reflect_template("cudf::rolling::jit::gpu_rolling_new", + rtcx::reflect_template("cudf::rolling::jit::rolling_window_kernel", cudf::type_to_name(input.type()), // list of template arguments cudf::type_to_name(output->type()), udf_agg._operator_name, @@ -107,5 +122,26 @@ std::unique_ptr rolling_window_udf(column_view const& input, return output; } +// Applies a user-defined rolling window function to the values in a column. +template +std::unique_ptr rolling_window_udf(column_view const& input, + PrecedingWindowIterator preceding_window, + FollowingWindowIterator following_window, + size_type min_periods, + rolling_aggregation const& agg, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + return rolling_window_udf(input, + reflect_window_wrapper(), + preceding_window, + reflect_window_wrapper(), + following_window, + min_periods, + agg, + stream, + mr); +} + } // namespace detail } // namespace cudf diff --git a/cpp/src/rolling/detail/rolling_variable_window.cu b/cpp/src/rolling/detail/rolling_variable_window.cu index 155adf2d74a4..504d639f24de 100644 --- a/cpp/src/rolling/detail/rolling_variable_window.cu +++ b/cpp/src/rolling/detail/rolling_variable_window.cu @@ -1,9 +1,10 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #include "rolling.cuh" +#include "rolling_jit.cuh" #include "rolling_udf.cuh" #include @@ -37,15 +38,14 @@ std::unique_ptr rolling_window(column_view const& input, if (agg.kind == aggregation::CUDA || agg.kind == aggregation::PTX) { // TODO: In future, might need to clamp preceding/following to column boundaries. - return cudf::detail::rolling_window_udf(input, - preceding_window.begin(), - "cudf::size_type*", - following_window.begin(), - "cudf::size_type*", - min_periods, - agg, - stream, - mr); + return cudf::detail::rolling_window_udf( + input, + cudf::detail::variable_window_wrapper{preceding_window.begin()}, + cudf::detail::variable_window_wrapper{following_window.begin()}, + min_periods, + agg, + stream, + mr); } else { auto defaults_col = cudf::is_dictionary(input.type()) ? dictionary_column_view(input).indices() : input; diff --git a/cpp/src/rolling/grouped_rolling.cu b/cpp/src/rolling/grouped_rolling.cu index 0eee9c22b01f..70f0d78427bf 100644 --- a/cpp/src/rolling/grouped_rolling.cu +++ b/cpp/src/rolling/grouped_rolling.cu @@ -6,6 +6,7 @@ #include "detail/optimized_unbounded_window.hpp" #include "detail/range_window_bounds.hpp" #include "detail/rolling.cuh" +#include "detail/rolling_jit.cuh" #include "detail/rolling_udf.cuh" #include "detail/rolling_utils.cuh" @@ -92,21 +93,16 @@ std::unique_ptr grouped_rolling_window(table_view const& group_keys, // 3. [0, 500, 1000] indicates two equal-sized groups: [0,500), and [500,1000). if (aggr.kind == aggregation::CUDA || aggr.kind == aggregation::PTX) { - cudf::detail::preceding_window_wrapper grouped_preceding_window{ - group_offsets.data(), group_labels.data(), preceding_window}; - - cudf::detail::following_window_wrapper grouped_following_window{ - group_offsets.data(), group_labels.data(), following_window}; - - return cudf::detail::rolling_window_udf(input, - grouped_preceding_window, - "cudf::detail::preceding_window_wrapper", - grouped_following_window, - "cudf::detail::following_window_wrapper", - min_periods, - aggr, - stream, - mr); + return cudf::detail::rolling_window_udf( + input, + cudf::detail::preceding_window_wrapper{ + group_labels.data(), group_offsets.data(), preceding_window}, + cudf::detail::following_window_wrapper{ + group_labels.data(), group_offsets.data(), following_window}, + min_periods, + aggr, + stream, + mr); } else { namespace utils = cudf::detail::rolling; auto groups = utils::grouped{group_labels.data(), group_offsets.data()}; diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index d701672f636d..169e6db8bb82 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -7,7 +7,7 @@ #include #include -#include +#include #pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before // it. Each UDF will have a different operation-udf.hpp generated for it, so we @@ -39,42 +39,36 @@ namespace cudf { namespace rolling { namespace jit { -template -cudf::size_type __device__ get_window(WindowType window, cudf::thread_index_type index) -{ - return window[index]; -} - -template <> -cudf::size_type __device__ get_window(cudf::size_type window, cudf::thread_index_type index) -{ - return window; -} - template -CUDF_KERNEL void gpu_rolling_new(cudf::size_type nrows, - InType const* const __restrict__ in_col, - cudf::bitmask_type const* const __restrict__ in_col_valid, - OutType* __restrict__ out_col, - cudf::bitmask_type* __restrict__ out_col_valid, - cudf::size_type* __restrict__ output_valid_count, - PrecedingWindowType preceding_window_begin, - FollowingWindowType following_window_begin, - cudf::size_type min_periods) +__device__ void rolling_window_kernel(cudf::size_type nrows, + void const* __restrict__ p_in_col, + cudf::bitmask_type const* __restrict__ in_col_valid, + void* __restrict__ p_out_col, + cudf::bitmask_type* __restrict__ out_col_valid, + cudf::size_type* __restrict__ output_valid_count, + detail::window_wrapper_base b_preceding_window_begin, + detail::window_wrapper_base b_following_window_begin, + cudf::size_type min_periods) { auto i = cudf::detail::grid_1d::global_thread_id(); auto const stride = cudf::detail::grid_1d::grid_stride(); + auto const preceding_window_begin = + reinterpret_cast(b_preceding_window_begin); + auto const following_window_begin = + reinterpret_cast(b_following_window_begin); + auto const* __restrict__ in_col = static_cast(p_in_col); + auto* __restrict__ out_col = static_cast(p_in_col); cudf::size_type warp_valid_count{0}; auto active_threads = __ballot_sync(0xffff'ffffu, i < nrows); while (i < nrows) { - int64_t const preceding_window = get_window(preceding_window_begin, i); - int64_t const following_window = get_window(following_window_begin, i); + int64_t const preceding_window = preceding_window_begin[i]; + int64_t const following_window = following_window_begin[i]; // compute bounds auto const start = static_cast( @@ -119,3 +113,24 @@ CUDF_KERNEL void gpu_rolling_new(cudf::size_type nrows, } // namespace jit } // namespace rolling } // namespace cudf + +extern "C" __global__ void kernel(cudf::size_type nrows, + void const* const __restrict__ in_col, + cudf::bitmask_type const* const __restrict__ in_col_valid, + void* __restrict__ out_col, + cudf::bitmask_type* __restrict__ out_col_valid, + cudf::size_type* __restrict__ output_valid_count, + cudf::detail::window_wrapper_base preceding_window_begin, + cudf::detail::window_wrapper_base following_window_begin, + cudf::size_type min_periods) +{ + KERNEL_INSTANCE(nrows, + in_col, + in_col_valid, + out_col, + out_col_valid, + output_valid_count, + preceding_window_begin, + following_window_begin, + min_periods); +} diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 4a7cb9c9af12..46363323cc44 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -35,10 +35,10 @@ template -CUDF_KERNEL void kernel(cudf::mutable_column_device_view_core const* outputs, - cudf::column_device_view_core const* inputs, - bool* intermediate_null_mask, - void* user_data) +__device__ void kernel(cudf::mutable_column_device_view_core const* outputs, + cudf::column_device_view_core const* inputs, + bool* intermediate_null_mask, + void* user_data) { // inputs to JITIFY kernels have to be either sized-integral types or pointers. Structs or // references can't be passed directly/correctly as they will be crossing an ABI boundary @@ -80,10 +80,10 @@ template -CUDF_KERNEL void fixed_point_kernel(cudf::mutable_column_device_view_core const* outputs, - cudf::column_device_view_core const* inputs, - bool* intermediate_null_mask, - void* user_data) +__device__ void fixed_point_kernel(cudf::mutable_column_device_view_core const* outputs, + cudf::column_device_view_core const* inputs, + bool* intermediate_null_mask, + void* user_data) { auto const start = cudf::detail::grid_1d::global_thread_id(); auto const stride = cudf::detail::grid_1d::grid_stride(); @@ -128,10 +128,10 @@ template -CUDF_KERNEL void span_kernel(cudf::jit::device_optional_span const* outputs, - cudf::column_device_view_core const* inputs, - bool* intermediate_null_mask, - void* user_data) +__device__ void span_kernel(cudf::jit::device_optional_span const* outputs, + cudf::column_device_view_core const* inputs, + bool* intermediate_null_mask, + void* user_data) { auto const start = cudf::detail::grid_1d::global_thread_id(); auto const stride = cudf::detail::grid_1d::grid_stride(); @@ -167,3 +167,11 @@ CUDF_KERNEL void span_kernel(cudf::jit::device_optional_span } // namespace jit } // namespace transformation } // namespace cudf + +extern "C" __global__ void kernel(cudf::mutable_column_device_view_core const* outputs, + cudf::column_device_view_core const* inputs, + bool* intermediate_null_mask, + void* user_data) +{ + KERNEL_INSTANCE(outputs, inputs, intermediate_null_mask, user_data); +} From bbe8d5cdfe5c4b1b0868d7b82cc172038e928e8d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 11 Apr 2026 08:32:17 +0000 Subject: [PATCH 119/254] Refactor JIT include directories and fix delimiter typo in embed.in.cpp --- cpp/CMakeLists.txt | 24 ++++++++++-------------- cpp/src/librtcx/CMake/embed.cmake | 1 - cpp/src/librtcx/CMake/embed.in.cpp | 11 +++++++---- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2017a306bd5d..2a7f5c5d91a2 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -326,43 +326,39 @@ endif() set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) - - jit_add_include_directory( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/libcxx DEST_DIRECTORY librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx ) jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include DEST_DIRECTORY cudf/cpp/include - INCLUDE_DIRECTORIES cudf/cpp/include + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf DEST_DIRECTORY + cudf/cpp/include/cudf INCLUDE_DIRECTORIES cudf/cpp/include ) - jit_add_include_directory( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/jit DEST_DIRECTORY cudf/cpp/src/jit INCLUDE_DIRECTORIES cudf/cpp/src ) jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/binaryop/jit DEST_DIRECTORY cudf/cpp/src/binaryop/jit - INCLUDE_DIRECTORIES cudf/cpp/src + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/binaryop/jit DEST_DIRECTORY + cudf/cpp/src/binaryop/jit INCLUDE_DIRECTORIES cudf/cpp/src ) - jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/join/jit DEST_DIRECTORY cudf/cpp/src/join/jit - INCLUDE_DIRECTORIES cudf/cpp/src + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/join/jit DEST_DIRECTORY + cudf/cpp/src/join/jit INCLUDE_DIRECTORIES cudf/cpp/src ) jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling/jit DEST_DIRECTORY cudf/cpp/src/rolling/jit - INCLUDE_DIRECTORIES cudf/cpp/src + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling/jit DEST_DIRECTORY + cudf/cpp/src/rolling/jit INCLUDE_DIRECTORIES cudf/cpp/src ) jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/transform/jit DEST_DIRECTORY cudf/cpp/src/transform/jit - INCLUDE_DIRECTORIES cudf/cpp/src + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/transform/jit DEST_DIRECTORY + cudf/cpp/src/transform/jit INCLUDE_DIRECTORIES cudf/cpp/src ) get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) diff --git a/cpp/src/librtcx/CMake/embed.cmake b/cpp/src/librtcx/CMake/embed.cmake index 0df801f90579..9217365d7b93 100644 --- a/cpp/src/librtcx/CMake/embed.cmake +++ b/cpp/src/librtcx/CMake/embed.cmake @@ -8,7 +8,6 @@ find_package(OpenSSL REQUIRED COMPONENTS Crypto) find_package(zstd REQUIRED) - # This function registers a directory of include files to be embedded for JIT compilation. It # gathers the specified files, their destinations, and include directories, and stores them in # target-specific variables for later use when generating the embed. diff --git a/cpp/src/librtcx/CMake/embed.in.cpp b/cpp/src/librtcx/CMake/embed.in.cpp index 28dca324cd10..c888c04ba6d2 100644 --- a/cpp/src/librtcx/CMake/embed.in.cpp +++ b/cpp/src/librtcx/CMake/embed.in.cpp @@ -1,4 +1,7 @@ - +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ #include #include @@ -341,13 +344,13 @@ void generate_embed(std::string_view id, static_cast(output.bin_file_data.size())); } -std::vector split_string(std::string_view str, char delimeter) +std::vector split_string(std::string_view str, char delimiter) { std::vector tokens; std::size_t start = 0; while (start <= str.size()) { - auto const pos = str.find(delimeter, start); + auto const pos = str.find(delimiter, start); if (pos == std::string_view::npos) { tokens.push_back(str.substr(start)); break; @@ -370,4 +373,4 @@ int main() generate_embed(id, file_paths, file_dests, include_directories, compression, output_dir); return EXIT_SUCCESS; -} \ No newline at end of file +} From c46450e252f68d2ade1796d8ddb5f12b3e15754c Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 11 Apr 2026 12:24:18 +0000 Subject: [PATCH 120/254] Refactor JIT kernel paths and include directives for improved organization and clarity --- cpp/CMakeLists.txt | 4 +- cpp/src/binaryop/binaryop.cpp | 5 +-- cpp/src/binaryop/jit/kernel.cu | 1 + cpp/src/jit/helpers.cpp | 14 ++++--- cpp/src/jit/helpers.hpp | 3 +- cpp/src/jit/jit.cpp | 51 ++++++++++++++++++++----- cpp/src/join/filter_join_indices_jit.cu | 11 ++---- cpp/src/join/jit/filter_join_kernel.cu | 1 + cpp/src/rolling/detail/rolling_udf.cuh | 7 +--- cpp/src/rolling/jit/kernel.cu | 1 + cpp/src/transform/jit/kernel.cu | 1 + cpp/src/transform/transform.cu | 10 +---- 12 files changed, 65 insertions(+), 44 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2a7f5c5d91a2..aca31ca9e71a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -352,8 +352,8 @@ jit_add_include_directory( ) jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling/jit DEST_DIRECTORY - cudf/cpp/src/rolling/jit INCLUDE_DIRECTORIES cudf/cpp/src + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling DEST_DIRECTORY + cudf/cpp/src/rolling INCLUDE_DIRECTORIES cudf/cpp/src ) jit_add_include_directory( diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index 6e0ee8fbaf12..ba4647ea1595 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -160,10 +160,7 @@ void binary_operation(mutable_column_view& out, cudf::type_to_name(rhs.type()), "cudf::binops::jit::UserDefinedOp"); - auto kernel = cudf::jit::get_udf_kernel("cudf/cpp/src/binaryop/jit/kernel.cu", - "cudf/cpp/src/binaryop/jit/kernel.cu", - kernel_reflection, - cuda_source); + auto kernel = cudf::jit::get_udf_kernel("binaryop/jit/kernel.cu", kernel_reflection, cuda_source); auto size_arg = static_cast(out.size()); auto out_arg = cudf::jit::get_data_ptr(out); diff --git a/cpp/src/binaryop/jit/kernel.cu b/cpp/src/binaryop/jit/kernel.cu index 6b200a0dce58..a69c4d9d6416 100644 --- a/cpp/src/binaryop/jit/kernel.cu +++ b/cpp/src/binaryop/jit/kernel.cu @@ -37,6 +37,7 @@ // need to put this pragma before including it to avoid PCH mismatch. // clang-format off +#include #include // clang-format on diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 6e42d01350a7..2279c1f76687 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -133,22 +133,26 @@ std::vector reflect_inputs( return reflections; } -kernel get_udf_kernel(std::string const& name, - std::string const& source_file, +kernel get_udf_kernel(std::string const& source_file, std::string const& kernel_name, std::string const& udf_cuda_source) { CUDF_FUNC_RANGE(); - char const* include_names[] = {"cudf/detail/operation-udf.hpp"}; - char const* include_headers[] = {udf_cuda_source.c_str()}; + auto kernel_instance_source = std::format(R"***( +#define KERNEL_INSTANCE {} +)***", + kernel_name); + char const* include_names[] = {"cudf/detail/operation-udf.hpp", + "cudf/detail/kernel-instance.hpp"}; + char const* include_headers[] = {udf_cuda_source.c_str(), kernel_instance_source.c_str()}; int constexpr min_pch_runtime_version = 12800; // CUDA 12.8 int runtime_version; CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); - return get_kernel(name, + return get_kernel(std::format("{}.jit.cu", source_file), source_file, include_names, include_headers, diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index 361c6b71066d..2a47bd9e030a 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -89,8 +89,7 @@ input_reflection reflect_input(std::variant con std::vector reflect_inputs( std::span const> inputs); -kernel get_udf_kernel(std::string const& name, - std::string const& source_file, +kernel get_udf_kernel(std::string const& source_file, std::string const& kernel_name, std::string const& udf_cuda_source); diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index fad72ac7c88e..c3f2edb30933 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -74,6 +74,40 @@ void install_file(char const* dst_path, std::span contents) } } +/** + * @brief Reads the contents of a file into a byte buffer and null-terminates it to allow for safe + * usage as a C-string. + */ +rtcx::byte_buffer read_blob_cstring(char const* path) +{ + int32_t fd = open(path, O_RDONLY); + if (fd == -1) { throw_posix(std::format("Failed to open file ({})", path), "open"); } + + RTCX_DEFER([&] { + if (close(fd) == -1) { throw_posix(std::format("Failed to close file ({})", path), "close"); } + }); + + auto file_size = lseek(fd, 0, SEEK_END); + if (file_size == -1) { + throw_posix(std::format("Failed to determine size of file ({})", path), "lseek"); + } + // TODO: make all read/write syscalls call read/write in a loop + + if (lseek(fd, 0, SEEK_SET) == -1) { + throw_posix(std::format("Failed to reset file offset for file ({})", path), "lseek"); + } + + auto contents = rtcx::byte_buffer::make(file_size + 1U); // +1 for null terminator + + if (read(fd, contents.data(), file_size) == -1) { + throw_posix(std::format("Failed to read file ({})", path), "read"); + } + + contents.data()[file_size] = '\0'; // null-terminate the buffer + + return contents; +} + rtcx::byte_buffer decompress_blob(std::span compressed_binary, size_t uncompressed_size, std::string_view compression) @@ -323,7 +357,7 @@ std::tuple compile_library_uncached( } // namespace kernel get_kernel(std::string const& name, - std::string const& source_file, + std::string const& source_file_rel, std::span header_include_names, std::span headers, std::string const& kernel_instance, @@ -342,8 +376,10 @@ kernel get_kernel(std::string const& name, auto header_include_names_hash = hash_strings(header_include_names).to_hex_string(); auto headers_hash = hash_strings(headers).to_hex_string(); auto bundle_hash = bundle.get_hash(); + auto source_file = std::format("{}/cudf/cpp/src/{}", bundle.get_directory(), source_file_rel); auto cache_key = std::format(R"***(cuLibrary +name={} binary_type=CUBIN cuda_runtime={} cuda_driver={} @@ -354,6 +390,7 @@ header_include_names={} headers={} kernel_instance={} )***", + name, runtime, driver, sm, @@ -366,20 +403,14 @@ kernel_instance={} auto cache_key_sha256 = hash_string(cache_key); auto compile = [&] { - auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); - auto source = std::format(R"***( -#include "{}" - )***", - source_file); - auto kernel_instance_define = std::format("-DKERNEL_INSTANCE=\"{}\"", kernel_instance); - - char const* options[] = {kernel_instance_define.c_str()}; + auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); + auto source = read_blob_cstring(source_file.c_str()); return compile_library_uncached(name.c_str(), reinterpret_cast(source.data()), header_include_names, headers, - options, + {}, {}, use_pch, log_pch); diff --git a/cpp/src/join/filter_join_indices_jit.cu b/cpp/src/join/filter_join_indices_jit.cu index 702b2ad79dd5..85fb16df748b 100644 --- a/cpp/src/join/filter_join_indices_jit.cu +++ b/cpp/src/join/filter_join_indices_jit.cu @@ -115,10 +115,7 @@ kernel build_join_filter_kernel(std::string const& predicate_code, auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); // Get compiled kernel - return cudf::jit::get_udf_kernel("cudf/cpp/src/join/jit/filter_join_kernel.cu", - "cudf/cpp/src/join/jit/filter_join_kernel.cu", - kernel_name, - cuda_source); + return cudf::jit::get_udf_kernel("join/jit/filter_join_kernel.cu", kernel_name, cuda_source); } // Launch the JIT kernel for join filtering @@ -509,10 +506,8 @@ filter_join_indices_jit(cudf::table_view const& left, cudf::jit::parse_single_function_cuda(filter_result.udf, "GENERIC_JOIN_FILTER_OP"); auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); - auto kernel = cudf::jit::get_udf_kernel("cudf/cpp/src/join/jit/filter_join_kernel.cu", - "cudf/cpp/src/join/jit/filter_join_kernel.cu", - kernel_name, - cuda_source); + auto kernel = + cudf::jit::get_udf_kernel("join/jit/filter_join_kernel.cu", kernel_name, cuda_source); // Collect scalar columns to append to left device views so join_scalar_accessor // can read them at indices >= left.num_columns(). diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index eda46ebc53ee..f4cb37ec7fd8 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -21,6 +21,7 @@ // clang-format off // This header is an inlined header that defines the GENERIC_JOIN_FILTER_OP function. It is placed here // so the symbols in the headers above can be used by it. +#include #include // clang-format on diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index cf4010e4d929..caf4480c42b2 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -42,7 +42,7 @@ std::string reflect_window_wrapper() } // Applies a user-defined rolling window function to the values in a column. -std::unique_ptr rolling_window_udf( +static std::unique_ptr rolling_window_udf( column_view const& input, std::string const& preceding_window_str, cudf::detail::window_wrapper_base const& preceding_window, @@ -95,10 +95,7 @@ std::unique_ptr rolling_window_udf( preceding_window_str, following_window_str); - auto kernel = cudf::jit::get_udf_kernel("cudf/cpp/src/rolling/jit/kernel.cu", - "cudf/cpp/src/rolling/jit/kernel.cu", - kernel_reflection, - cuda_source); + auto kernel = cudf::jit::get_udf_kernel("rolling/jit/kernel.cu", kernel_reflection, cuda_source); auto cfg = kernel.max_occupancy_config(0, 0); kernel.launch_with({cfg.min_grid_size}, {cfg.block_size}, diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index 169e6db8bb82..d4a96caff551 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -13,6 +13,7 @@ // it. Each UDF will have a different operation-udf.hpp generated for it, so we // need to put this pragma before including it to avoid PCH mismatch. +#include #include struct rolling_udf_ptx { diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 46363323cc44..2f4cca14480c 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -23,6 +23,7 @@ // clang-format off // This header is an inlined header that defines the GENERIC_FILTER_OP function. It is placed here // so the symbols in the headers above can be used by it. +#include #include // clang-format on diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 775902c1c9fc..4909358ee1b8 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -102,10 +102,7 @@ kernel build_transform_kernel(std::string_view kernel_name, build_jit_template_params( is_null_aware, may_evaluate_null, has_user_data, {}, output_typenames, input_reflections)); - return cudf::jit::get_udf_kernel("cudf/cpp/src/transform/jit/kernel.cu", - "cudf/cpp/src/transform/jit/kernel.cu", - kernel_reflection, - cuda_source); + return cudf::jit::get_udf_kernel("transform/jit/kernel.cu", kernel_reflection, cuda_source); } kernel build_span_kernel(std::string_view kernel_name, @@ -137,10 +134,7 @@ kernel build_span_kernel(std::string_view kernel_name, build_jit_template_params( is_null_aware, may_evaluate_null, has_user_data, span_outputs, {}, input_reflections)); - return cudf::jit::get_udf_kernel("cudf/cpp/src/transform/jit/kernel.cu", - "cudf/cpp/src/transform/jit/kernel.cu", - kernel_reflection, - cuda_source); + return cudf::jit::get_udf_kernel("transform/jit/kernel.cu", kernel_reflection, cuda_source); } column_view to_column_view(column_view const& col) { return col; } From 4ba9950ee1c5e6ecd2c22a0c189cb6da74a89c8a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 11 Apr 2026 20:55:31 +0000 Subject: [PATCH 121/254] Enhance JIT kernel functions with additional parameters for minimal usage and extra options --- cpp/src/jit/helpers.cpp | 13 ++++--------- cpp/src/jit/jit.cpp | 20 +++++++++++++++----- cpp/src/jit/jit.hpp | 8 +++++--- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 722d4bf3c8be..7fd592815400 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -92,7 +92,7 @@ std::vector input_type_names( kernel get_udf_kernel(std::string const& source_file, std::string const& kernel_name, std::string const& udf_cuda_source, - std::vector const& extra_options) + std::vector const& extra_options) { CUDF_FUNC_RANGE(); @@ -104,7 +104,7 @@ kernel get_udf_kernel(std::string const& source_file, "cudf/detail/kernel-instance.hpp"}; char const* include_headers[] = {udf_cuda_source.c_str(), kernel_instance_source.c_str()}; - constexpr int min_pch_cuda_version = 12800; // CUDA 12.8 + constexpr int min_pch_cuda_version = 12800; // CUDA 12.8 constexpr int min_minimal_cuda_version = 12800; // CUDA 12.8 int runtime_version; @@ -113,11 +113,7 @@ kernel get_udf_kernel(std::string const& source_file, std::vector options; options.emplace_back("-arch=sm_."); - - - - - return get_kernel(std::format("{}.jit.cu", source_file), + return get_kernel(std::format("{}.jit.cu", source_file), source_file, include_names, include_headers, @@ -126,8 +122,7 @@ kernel get_udf_kernel(std::string const& source_file, runtime_version >= min_pch_cuda_version, runtime_version >= min_minimal_cuda_version, false, // TODO: use context config - extra_options - ); + extra_options); } } // namespace jit diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index c3f2edb30933..21b9626b16c2 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -282,6 +282,7 @@ std::tuple compile_library_uncached( std::span extra_options, std::span name_expressions, bool use_pch, + bool use_minimal, bool log_pch) { CUDF_FUNC_RANGE(); @@ -311,7 +312,7 @@ std::tuple compile_library_uncached( // --pch-dir options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); - options.emplace_back("--minimal"); + options.emplace_back("--diag-suppress=47"); options.emplace_back("--device-int128"); @@ -322,6 +323,8 @@ std::tuple compile_library_uncached( options.emplace_back("--generate-line-info"); options.emplace_back("--dopt=on"); + if (use_minimal) { options.emplace_back("--minimal"); } + if (use_pch) { options.emplace_back("--pch"); @@ -357,13 +360,15 @@ std::tuple compile_library_uncached( } // namespace kernel get_kernel(std::string const& name, - std::string const& source_file_rel, + std::string const& source_file_id, std::span header_include_names, std::span headers, std::string const& kernel_instance, bool use_cache, bool use_pch, - bool log_pch) + bool use_minimal, + bool log_pch, + std::span extra_options) { CUDF_FUNC_RANGE(); @@ -376,7 +381,7 @@ kernel get_kernel(std::string const& name, auto header_include_names_hash = hash_strings(header_include_names).to_hex_string(); auto headers_hash = hash_strings(headers).to_hex_string(); auto bundle_hash = bundle.get_hash(); - auto source_file = std::format("{}/cudf/cpp/src/{}", bundle.get_directory(), source_file_rel); + auto source_file = std::format("{}/cudf/cpp/src/{}", bundle.get_directory(), source_file_id); auto cache_key = std::format(R"***(cuLibrary name={} @@ -405,14 +410,19 @@ kernel_instance={} auto compile = [&] { auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); auto source = read_blob_cstring(source_file.c_str()); + std::vector extra_options_cstr; + for (auto const& option : extra_options) { + extra_options_cstr.emplace_back(option.c_str()); + } return compile_library_uncached(name.c_str(), reinterpret_cast(source.data()), header_include_names, headers, - {}, + extra_options_cstr, {}, use_pch, + use_minimal, log_pch); }; diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index b6e91dd5a2b4..6a0c6f877915 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -76,8 +76,10 @@ kernel get_kernel(std::string const& name, std::span header_include_names, std::span headers, std::string const& kernel_instance, - bool use_cache = true, - bool use_pch = true, - bool log_pch = false); + bool use_cache = true, + bool use_pch = true, + bool use_minimal = true, + bool log_pch = false, + std::span extra_options = {}); } // namespace CUDF_EXPORT cudf From 74286e02408ccac744023b079346a242b34c764c Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 11 Apr 2026 21:29:24 +0000 Subject: [PATCH 122/254] Fix formatting in compression output message for clarity --- cpp/src/librtcx/CMake/embed.in.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/librtcx/CMake/embed.in.cpp b/cpp/src/librtcx/CMake/embed.in.cpp index c888c04ba6d2..cbba911ae637 100644 --- a/cpp/src/librtcx/CMake/embed.in.cpp +++ b/cpp/src/librtcx/CMake/embed.in.cpp @@ -187,7 +187,7 @@ embed_output generate_cxx_source_files_data(std::string_view id, if (compress) { std::cout << std::format( - "-- Compressed {}'s binary from {} bytes to {} bytes (compression ratio: {:.2f})\n", + "-- Compressed {}'s binary from {} bytes to {} bytes (compression ratio: {:.2f})\n", id, uncompressed_files_bytes.size(), compressed_files_bytes.size(), From 25901ea290b519535ae5b63d10c9736ebe7083a6 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 11 Apr 2026 22:09:23 +0000 Subject: [PATCH 123/254] Refactor function parameter formatting for improved readability in JIT kernel instantiation --- cpp/src/jit/jit.cpp | 2 +- cpp/src/transform/transform.cu | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index 21b9626b16c2..f48cb601b21a 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -368,7 +368,7 @@ kernel get_kernel(std::string const& name, bool use_pch, bool use_minimal, bool log_pch, - std::span extra_options) + std::span extra_options) { CUDF_FUNC_RANGE(); diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 3f698dbd0389..4efae62d7778 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -158,13 +158,13 @@ using handle = std::variant< namespace jit_transform { cudf::kernel instantiate(null_aware is_null_aware, - bool has_user_data, - std::string const& ins, - std::string const& outs, - std::vector const& ptx_input_types, - std::vector const& ptx_output_types, - std::string const& udf, - udf_source_type source_type) + bool has_user_data, + std::string const& ins, + std::string const& outs, + std::vector const& ptx_input_types, + std::vector const& ptx_output_types, + std::string const& udf, + udf_source_type source_type) { CUDF_FUNC_RANGE(); auto cuda_source = (source_type == udf_source_type::PTX) @@ -175,10 +175,13 @@ cudf::kernel instantiate(null_aware is_null_aware, : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", - rtcx::reflect_enum(is_null_aware),rtcx::reflect_bool(has_user_data), ins, outs); + rtcx::reflect_enum(is_null_aware), + rtcx::reflect_bool(has_user_data), + ins, + outs); return jit::get_udf_kernel( - "transform/jit/kernel.cu", kernel, cuda_source, {"-restrict", "--dopt=on"}); + "transform/jit/kernel.cu", kernel, cuda_source, {"-restrict", "--dopt=on"}); } void launch(cudf::kernel const& kernel_obj, @@ -192,8 +195,8 @@ void launch(cudf::kernel const& kernel_obj, { CUDF_FUNC_RANGE(); void* args[] = {&row_size, &stencil, &stencil_has_nulls, &user_data, &input_cols, &output_cols}; - auto kernel = kernel_obj.get(); - auto cfg = kernel.max_occupancy_config(0, 0); + auto kernel = kernel_obj.get(); + auto cfg = kernel.max_occupancy_config(0, 0); kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } From 2b5dbc53532e942407f7538c24605f0dde872da7 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 06:04:47 +0000 Subject: [PATCH 124/254] fix up --- cpp/src/rolling/jit/kernel.cu | 4 ++-- cpp/src/transform/jit/kernel.cu | 13 +++++----- cpp/src/transform/transform.cu | 42 ++++++++++++++++++++------------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index d4a96caff551..e6caea44efea 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -68,8 +68,8 @@ __device__ void rolling_window_kernel(cudf::size_type nrows, auto active_threads = __ballot_sync(0xffff'ffffu, i < nrows); while (i < nrows) { - int64_t const preceding_window = preceding_window_begin[i]; - int64_t const following_window = following_window_begin[i]; + int64_t const preceding_window = preceding_window_begin[static_cast(i)]; + int64_t const following_window = following_window_begin[static_cast(i)]; // compute bounds auto const start = static_cast( diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 8449372aa23f..b09c90587952 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -116,12 +116,13 @@ CUDF_KERNEL void transform_kernel(size_type row_size, } // namespace jit } // namespace cudf -extern "C" __global__ void kernel(size_type row_size, - bitmask_type const* __restrict__ stencil, - bool stencil_has_nulls, - void* __restrict__ user_data, - column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols) +extern "C" __global__ void kernel( + cudf::size_type row_size, + cudf::bitmask_type const* __restrict__ stencil, + bool stencil_has_nulls, + void* __restrict__ user_data, + cudf::column_device_view_core const* __restrict__ input_cols, + cudf::mutable_column_device_view_core const* __restrict__ output_cols) { KERNEL_INSTANCE(row_size, stencil, stencil_has_nulls, user_data, input_cols, output_cols); } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 4efae62d7778..d1b4f035acbb 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -175,7 +175,7 @@ cudf::kernel instantiate(null_aware is_null_aware, : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", - rtcx::reflect_enum(is_null_aware), + rtcx::reflect_enum("cudf::null_aware", is_null_aware), rtcx::reflect_bool(has_user_data), ins, outs); @@ -184,7 +184,7 @@ cudf::kernel instantiate(null_aware is_null_aware, "transform/jit/kernel.cu", kernel, cuda_source, {"-restrict", "--dopt=on"}); } -void launch(cudf::kernel const& kernel_obj, +void launch(cudf::kernel const& kernel, size_type row_size, bitmask_type const* stencil, bool stencil_has_nulls, @@ -195,9 +195,9 @@ void launch(cudf::kernel const& kernel_obj, { CUDF_FUNC_RANGE(); void* args[] = {&row_size, &stencil, &stencil_has_nulls, &user_data, &input_cols, &output_cols}; - auto kernel = kernel_obj.get(); - auto cfg = kernel.max_occupancy_config(0, 0); - kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); + auto kernel_ref = kernel.get(); + auto cfg = kernel_ref.max_occupancy_config(0, 0); + kernel_ref.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } std::string reflect_input_element(column_view const& c) { return type_to_name(c.type()); } @@ -254,10 +254,14 @@ auto reflect(udf_source_type source_type, bool as_scalar = std::holds_alternative(in); bool may_be_nullable = input_may_be_nullable[i]; auto is_strings_output = false; - auto accessor = - jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate( - i, column, element, optional_element, as_scalar, may_be_nullable, is_strings_output); + auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", + rtcx::reflect_int(i), + column, + element, + optional_element, + rtcx::reflect_bool(as_scalar), + rtcx::reflect_bool(may_be_nullable), + rtcx::reflect_bool(is_strings_output)); in_types.push_back(accessor); } @@ -271,16 +275,20 @@ auto reflect(udf_source_type source_type, bool as_scalar = false; // never scalar bool may_be_nullable = output_may_be_nullable[i]; auto is_strings_output = std::holds_alternative(out); - auto accessor = - jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate( - i, column, element, optional_element, as_scalar, may_be_nullable, is_strings_output); + auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", + rtcx::reflect_int(i), + column, + element, + optional_element, + rtcx::reflect_bool(as_scalar), + rtcx::reflect_bool(may_be_nullable), + rtcx::reflect_bool(is_strings_output)); out_types.push_back(accessor); } - auto ins = jitify2::reflection::Template("cudf::jit::type_list").instantiate(in_types); - auto outs = jitify2::reflection::Template("cudf::jit::type_list").instantiate(out_types); + auto ins = rtcx::reflect_template("cudf::jit::type_list", in_types); + auto outs = rtcx::reflect_template("cudf::jit::type_list", out_types); std::vector ptx_in_types; std::vector ptx_out_types; @@ -382,8 +390,8 @@ CUDF_KERNEL void copy_offset_bitmask(bitmask_type* __restrict__ destination, size_type source_end_bit, size_type number_of_mask_words) { - auto const stride = cudf::detail::grid_1d::grid_stride(); - for (thread_index_type destination_word_index = grid_1d::global_thread_id(); + auto const stride = detail::grid_1d::grid_stride(); + for (thread_index_type destination_word_index = detail::grid_1d::global_thread_id(); destination_word_index < number_of_mask_words; destination_word_index += stride) { destination[destination_word_index] = detail::get_mask_offset_word( From a521f47ebf95dc1a7687bb67f44442b77b5a54f3 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 06:38:53 +0000 Subject: [PATCH 125/254] Refactor detail namespace usage in transform.cu for consistency --- cpp/src/transform/transform.cu | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 7edd7c35dc03..b639ae4d30c9 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -377,8 +377,8 @@ CUDF_KERNEL void copy_offset_bitmask(bitmask_type* __restrict__ destination, size_type source_end_bit, size_type number_of_mask_words) { - auto const stride = cudf::detail::grid_1d::grid_stride(); - for (thread_index_type destination_word_index = grid_1d::global_thread_id(); + auto const stride = detail::grid_1d::grid_stride(); + for (thread_index_type destination_word_index = detail::grid_1d::global_thread_id(); destination_word_index < number_of_mask_words; destination_word_index += stride) { destination[destination_word_index] = detail::get_mask_offset_word( @@ -456,20 +456,17 @@ size_type inplace_null_mask_and(bitmask_type* null_mask, if (nullable_masks.size() == 1) { // only 1 mask provided, copy it directly to the output - if (nullable_offsets[0] % bits_per_word == 0) { + auto src_begin = nullable_offsets[0]; + auto src_end = src_begin + row_size; + if (src_begin % bits_per_word == 0) { CUDF_CUDA_TRY(detail::memcpy_async( - null_mask, nullable_masks[0] + (nullable_offsets[0] / bits_per_word), num_bytes, stream)); + null_mask, nullable_masks[0] + (src_begin / bits_per_word), num_bytes, stream)); } else { - cudf::detail::grid_1d config(row_size, 256); + detail::grid_1d config(row_size, 256); copy_offset_bitmask<<>>( - static_cast(null_mask), - nullable_masks[0], - nullable_offsets[0], - nullable_offsets[0] + row_size, - num_words); + static_cast(null_mask), nullable_masks[0], src_begin, src_end, num_words); CUDF_CHECK_CUDA(stream.value()); } - CUDF_CUDA_TRY(detail::memcpy_async(null_mask, nullable_masks[0], num_bytes, stream)); return nullable_null_counts[0]; } @@ -648,19 +645,19 @@ rmm::device_uvector make_chars_buffer(column_view const& offsets_view, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - auto offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets_view); + auto offsets = detail::offsetalator_factory::make_input_iterator(offsets_view); auto chars = rmm::device_uvector(chars_size, stream, mr); - auto srcs = cudf::detail::make_counting_transform_iterator( + auto srcs = detail::make_counting_transform_iterator( size_type{0}, [begin] __device__(size_type idx) -> void const* { return begin[idx].data(); }); - auto src_sizes = cudf::detail::make_counting_transform_iterator( + auto src_sizes = detail::make_counting_transform_iterator( size_type{0}, [begin, stencil] __device__(size_type idx) -> size_type { if (stencil != nullptr && !bit_is_set(stencil, idx)) { return 0; } return static_cast(begin[idx].size_bytes()); }); - auto dsts = cudf::detail::make_counting_transform_iterator( + auto dsts = detail::make_counting_transform_iterator( size_type{0}, [offsets, chars = chars.data()] __device__(size_type idx) -> void* { return chars + offsets[idx]; }); @@ -688,15 +685,14 @@ std::unique_ptr make_strings_column(device_span strin auto stencil = static_cast(null_mask.data()); // build offsets column from the strings sizes - auto sizes = cudf::detail::make_counting_transform_iterator( - cudf::size_type{0}, - [stencil, strings = strings.data()] __device__(cudf::size_type index) -> size_type { + auto sizes = detail::make_counting_transform_iterator( + size_type{0}, [stencil, strings = strings.data()] __device__(size_type index) -> size_type { if (stencil != nullptr && !bit_is_set(stencil, index)) { return 0; } return static_cast(strings[index].size_bytes()); }); auto [offsets, bytes] = - cudf::strings::detail::make_offsets_child_column(sizes, sizes + size, stream, mr); + strings::detail::make_offsets_child_column(sizes, sizes + size, stream, mr); auto chars = make_chars_buffer(offsets->view(), bytes, strings.data(), stencil, size, stream, mr); From 843dbd50b99963b5916854bc4a10605c9b3742e9 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 07:08:03 +0000 Subject: [PATCH 126/254] Enhance argument handling in to_args function by utilizing pinned vector and device uvector for improved performance --- cpp/src/transform/transform.cu | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index b639ae4d30c9..b237493fdd76 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -299,7 +300,8 @@ auto to_args(std::span inputs, rmm::device_async_resource_ref mr) { std::vector handles; - std::vector h_args; + auto h_args = detail::make_pinned_vector_async( + inputs.size() + outputs.size(), stream); for (auto& in : inputs) { if (auto* col = std::get_if(&in)) { @@ -324,14 +326,7 @@ auto to_args(std::span inputs, out); } - rmm::device_buffer d_args{h_args.size() * sizeof(detail::column_device_view_base), stream, mr}; - - CUDF_CUDA_TRY(detail::memcpy_async( - d_args.data(), h_args.data(), h_args.size() * sizeof(detail::column_device_view_base), stream)); - - // ensure the device buffer copy is complete before `h_args` goes out of scope and its destructors - // are called - stream.synchronize(); + auto d_args = detail::make_device_uvector_async(h_args, stream, mr); return std::make_tuple(std::move(d_args), std::move(handles)); } From d44bbbbf5feec25a413281ceec0e5fb45fbe434b Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 07:19:03 +0000 Subject: [PATCH 127/254] Fix type in to_args function to use column_device_view_base for improved performance --- cpp/src/transform/transform.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index b237493fdd76..ab2a0284c815 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -300,7 +300,7 @@ auto to_args(std::span inputs, rmm::device_async_resource_ref mr) { std::vector handles; - auto h_args = detail::make_pinned_vector_async( + auto h_args = detail::make_pinned_vector_async( inputs.size() + outputs.size(), stream); for (auto& in : inputs) { From 58a63852c626dfb05207ea054ea3d3fdd53a4f76 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 07:46:10 +0000 Subject: [PATCH 128/254] Refactor to_args function to use host_vector for better memory management and performance --- cpp/src/transform/transform.cu | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index ab2a0284c815..19bc1716c5a1 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -300,8 +300,9 @@ auto to_args(std::span inputs, rmm::device_async_resource_ref mr) { std::vector handles; - auto h_args = detail::make_pinned_vector_async( - inputs.size() + outputs.size(), stream); + auto h_args = + detail::host_vector({get_pinned_memory_resource(), stream}); + h_args.reserve(inputs.size() + outputs.size()); for (auto& in : inputs) { if (auto* col = std::get_if(&in)) { From 7616d54b04857b94768fcc0965a07c027e00cf2e Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 12:24:17 +0000 Subject: [PATCH 129/254] Refactor rolling window wrappers to include constructors and improve const correctness; rename rolling_window_udf for clarity --- cpp/src/rolling/detail/rolling_jit.cuh | 31 ++++++++++++++++++++++---- cpp/src/rolling/detail/rolling_udf.cuh | 20 ++++++++--------- cpp/src/rolling/grouped_rolling.cu | 4 ++-- cpp/src/rolling/jit/kernel.cu | 18 +++++++-------- cpp/src/runtime/context.cpp | 4 ++-- cpp/src/transform/jit/kernel.cu | 1 - cpp/src/transform/transform.cu | 1 + 7 files changed, 50 insertions(+), 29 deletions(-) diff --git a/cpp/src/rolling/detail/rolling_jit.cuh b/cpp/src/rolling/detail/rolling_jit.cuh index 47fc30064cc8..38482a9f5580 100644 --- a/cpp/src/rolling/detail/rolling_jit.cuh +++ b/cpp/src/rolling/detail/rolling_jit.cuh @@ -29,7 +29,12 @@ struct fixed_window_wrapper : public window_wrapper_base { { } - __device__ __host__ cudf::size_type operator[](cudf::size_type) { return window; } + __device__ __host__ fixed_window_wrapper(window_wrapper_base const& base) + : window_wrapper_base(base) + { + } + + __device__ __host__ cudf::size_type operator[](cudf::size_type) const { return window; } }; struct variable_window_wrapper : public window_wrapper_base { @@ -38,7 +43,15 @@ struct variable_window_wrapper : public window_wrapper_base { { } - __device__ __host__ cudf::size_type operator[](cudf::size_type idx) { return group_offsets[idx]; } + __device__ __host__ variable_window_wrapper(window_wrapper_base const& base) + : window_wrapper_base(base) + { + } + + __device__ __host__ cudf::size_type operator[](cudf::size_type idx) const + { + return group_offsets[idx]; + } }; struct preceding_window_wrapper : public window_wrapper_base { @@ -49,7 +62,12 @@ struct preceding_window_wrapper : public window_wrapper_base { { } - __device__ cudf::size_type operator[](cudf::size_type idx) + __device__ __host__ preceding_window_wrapper(window_wrapper_base const& base) + : window_wrapper_base(base) + { + } + + __device__ cudf::size_type operator[](cudf::size_type idx) const { auto group_label = group_labels[idx]; auto group_start = group_offsets[group_label]; @@ -65,7 +83,12 @@ struct following_window_wrapper : public window_wrapper_base { { } - __device__ cudf::size_type operator[](cudf::size_type idx) + __device__ __host__ following_window_wrapper(window_wrapper_base const& base) + : window_wrapper_base(base) + { + } + + __device__ cudf::size_type operator[](cudf::size_type idx) const { auto group_label = group_labels[idx]; auto group_end = diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index f1d84fd80e42..9115e6642e7c 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -42,7 +42,7 @@ std::string reflect_window_wrapper() } // Applies a user-defined rolling window function to the values in a column. -static std::unique_ptr rolling_window_udf( +static std::unique_ptr rolling_window_udf_impl( column_view const& input, std::string const& preceding_window_str, cudf::detail::window_wrapper_base const& preceding_window, @@ -130,15 +130,15 @@ std::unique_ptr rolling_window_udf(column_view const& input, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - return rolling_window_udf(input, - reflect_window_wrapper(), - preceding_window, - reflect_window_wrapper(), - following_window, - min_periods, - agg, - stream, - mr); + return rolling_window_udf_impl(input, + reflect_window_wrapper(), + preceding_window, + reflect_window_wrapper(), + following_window, + min_periods, + agg, + stream, + mr); } } // namespace detail diff --git a/cpp/src/rolling/grouped_rolling.cu b/cpp/src/rolling/grouped_rolling.cu index 70f0d78427bf..658d814efb9e 100644 --- a/cpp/src/rolling/grouped_rolling.cu +++ b/cpp/src/rolling/grouped_rolling.cu @@ -96,9 +96,9 @@ std::unique_ptr grouped_rolling_window(table_view const& group_keys, return cudf::detail::rolling_window_udf( input, cudf::detail::preceding_window_wrapper{ - group_labels.data(), group_offsets.data(), preceding_window}, + group_offsets.data(), group_labels.data(), preceding_window}, cudf::detail::following_window_wrapper{ - group_labels.data(), group_offsets.data(), following_window}, + group_offsets.data(), group_labels.data(), following_window}, min_periods, aggr, stream, diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index e6caea44efea..501788731c46 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -55,21 +55,19 @@ __device__ void rolling_window_kernel(cudf::size_type nrows, detail::window_wrapper_base b_following_window_begin, cudf::size_type min_periods) { - auto i = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); - auto const preceding_window_begin = - reinterpret_cast(b_preceding_window_begin); - auto const following_window_begin = - reinterpret_cast(b_following_window_begin); - auto const* __restrict__ in_col = static_cast(p_in_col); - auto* __restrict__ out_col = static_cast(p_in_col); + auto i = cudf::detail::grid_1d::global_thread_id(); + auto const stride = cudf::detail::grid_1d::grid_stride(); + PrecedingWindowType const preceding_window_begin = b_preceding_window_begin; + FollowingWindowType const following_window_begin = b_following_window_begin; + auto const* __restrict__ in_col = static_cast(p_in_col); + auto* __restrict__ out_col = static_cast(p_out_col); cudf::size_type warp_valid_count{0}; auto active_threads = __ballot_sync(0xffff'ffffu, i < nrows); while (i < nrows) { - int64_t const preceding_window = preceding_window_begin[static_cast(i)]; - int64_t const following_window = following_window_begin[static_cast(i)]; + int64_t const preceding_window = preceding_window_begin[i]; + int64_t const following_window = following_window_begin[i]; // compute bounds auto const start = static_cast( diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 07368b5d32ca..e5a4b170a010 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights - * reserved. SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ #include "runtime/context.hpp" diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index b09c90587952..05dd65d627ef 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -44,7 +44,6 @@ CUDF_KERNEL void transform_kernel(size_type row_size, column_device_view_core const* __restrict__ input_cols, mutable_column_device_view_core const* __restrict__ output_cols) { - // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index aea3dc15096d..2dc5483dc1a5 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -198,6 +198,7 @@ void launch(cudf::kernel const& kernel, void* args[] = {&row_size, &stencil, &stencil_has_nulls, &user_data, &input_cols, &output_cols}; auto kernel_ref = kernel.get(); auto cfg = kernel_ref.max_occupancy_config(0, 0); + // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior kernel_ref.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } From bdbed4b557bc6458b416d86113affd37a6a1df0a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 13:35:23 +0000 Subject: [PATCH 130/254] Add embedded file generation and SHA256 hashing functionality - Introduced a new source file `embed.in.cpp` for generating C++ source files that embed binary data. - Implemented functions to load, compress, and hash files using SHA256. - Added a `sha256.hpp` header file containing the SHA256 implementation and related utilities. - Removed OpenSSL dependency for SHA256 hashing from `rtcx.cpp` and `rtcx.hpp`. - Updated `rtcx.cpp` and `rtcx.hpp` to reflect the new SHA256 implementation and removed unused code. - Enhanced error handling and logging for file operations and compression. --- cpp/CMakeLists.txt | 10 +- cpp/src/librtcx/README.md | 1 - cpp/src/librtcx/{CMake => }/embed.cmake | 10 +- cpp/src/librtcx/{CMake => }/embed.in.cpp | 49 +--- cpp/src/librtcx/rtcx.cpp | 61 +---- cpp/src/librtcx/rtcx.hpp | 69 +----- cpp/src/librtcx/sha256.hpp | 291 +++++++++++++++++++++++ 7 files changed, 323 insertions(+), 168 deletions(-) rename cpp/src/librtcx/{CMake => }/embed.cmake (95%) rename cpp/src/librtcx/{CMake => }/embed.in.cpp (87%) create mode 100644 cpp/src/librtcx/sha256.hpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e7bc2e79fd16..3ffc09c152db 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -300,12 +300,12 @@ include(cmake/thirdparty/get_nanoarrow.cmake) # find thread_pool include(cmake/thirdparty/get_thread_pool.cmake) -# JIT Embedding helper functions -include(src/librtcx/CMake/embed.cmake) - # find zstd include(cmake/thirdparty/get_zstd.cmake) +# JIT Embedding helper functions +include(src/librtcx/embed.cmake) + # Workaround until https://github.com/rapidsai/rapids-cmake/issues/176 is resolved if(NOT BUILD_SHARED_LIBS) include("${rapids-cmake-dir}/export/find_package_file.cmake") @@ -908,7 +908,9 @@ add_library( ) add_dependencies(cudf cudf_jit_embed) -target_include_directories(cudf PRIVATE ${cudf_jit_embed_INCLUDE_DIRS}) +target_include_directories( + cudf PRIVATE ${cudf_jit_embed_INCLUDE_DIRS} ${CMAKE_CURRENT_LIST_DIR}/src/librtcx +) foreach(incbin_dir IN LISTS cudf_jit_embed_INCLUDE_DIRS) target_compile_options(cudf PRIVATE $<$:-Wa,-I${incbin_dir}>) diff --git a/cpp/src/librtcx/README.md b/cpp/src/librtcx/README.md index bad3b9acbd88..006872f15f31 100644 --- a/cpp/src/librtcx/README.md +++ b/cpp/src/librtcx/README.md @@ -12,7 +12,6 @@ RTCX (runtime-compiler extended) is a wrapper around NVRTC and NVJitLink designe ## Build-Scripting Requirements - CMake -- LibOpenSSL - for binary hashing - LibZSTD - for binary compression ## Runtime Requirements diff --git a/cpp/src/librtcx/CMake/embed.cmake b/cpp/src/librtcx/embed.cmake similarity index 95% rename from cpp/src/librtcx/CMake/embed.cmake rename to cpp/src/librtcx/embed.cmake index 9217365d7b93..b8aa28704b72 100644 --- a/cpp/src/librtcx/CMake/embed.cmake +++ b/cpp/src/librtcx/embed.cmake @@ -5,8 +5,11 @@ # cmake-format: on # ============================================================================= -find_package(OpenSSL REQUIRED COMPONENTS Crypto) -find_package(zstd REQUIRED) +if(NOT TARGET zstd) + message( + FATAL_ERROR "zstd library is required for JIT embedding. Please ensure it is found by CMake." + ) +endif() # This function registers a directory of include files to be embedded for JIT compilation. It # gathers the specified files, their destinations, and include directories, and stores them in @@ -131,10 +134,11 @@ function(jit_embed) add_executable("${TARGET}__jit_embed_run" EXCLUDE_FROM_ALL "${CONFIGURED_EMBED_SCRIPT}") target_include_directories("${TARGET}__jit_embed_run" PRIVATE ${ZSTD_INCLUDE_DIR}) - target_link_libraries("${TARGET}__jit_embed_run" PRIVATE ${CMAKE_DL_LIBS} zstd OpenSSL::Crypto) + target_link_libraries("${TARGET}__jit_embed_run" PRIVATE ${CMAKE_DL_LIBS} zstd) set_target_properties( "${TARGET}__jit_embed_run" PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES ) + target_include_directories("${TARGET}__jit_embed_run" PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR}) add_custom_command( OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin diff --git a/cpp/src/librtcx/CMake/embed.in.cpp b/cpp/src/librtcx/embed.in.cpp similarity index 87% rename from cpp/src/librtcx/CMake/embed.in.cpp rename to cpp/src/librtcx/embed.in.cpp index cbba911ae637..e666d5bc1a41 100644 --- a/cpp/src/librtcx/CMake/embed.in.cpp +++ b/cpp/src/librtcx/embed.in.cpp @@ -3,8 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "sha256.hpp" + #include -#include #include #include #include @@ -73,7 +74,6 @@ struct embed_output { std::string asm_source; std::string bin_file_name; std::vector bin_file_data; - std::vector hash; }; std::vector load_file_bytes(std::string_view file_path) @@ -119,38 +119,17 @@ std::vector compress_bytes(std::span bytes, std::string_ return compressed; } -std::vector compute_embed_hash(std::span uncompressed_files_bytes, - std::span merged_dests_bytes, - std::span merged_include_dirs_bytes, - std::string_view compression) +rtcx::sha256 compute_embed_hash(std::span uncompressed_files_bytes, + std::span merged_dests_bytes, + std::span merged_include_dirs_bytes, + std::string_view compression) { - std::vector hash(EVP_MD_size(EVP_sha256())); - EVP_MD_CTX* sha = EVP_MD_CTX_new(); - if (sha == nullptr) { throw std::runtime_error("Failed to allocate EVP_MD_CTX"); } - - DEFER([&] { EVP_MD_CTX_free(sha); }); - - if (EVP_DigestInit_ex(sha, EVP_sha256(), nullptr) != 1) { - throw std::runtime_error("EVP_DigestInit_ex failed"); - } - - if (EVP_DigestUpdate(sha, uncompressed_files_bytes.data(), uncompressed_files_bytes.size()) != - 1 || - EVP_DigestUpdate(sha, merged_dests_bytes.data(), merged_dests_bytes.size()) != 1 || - EVP_DigestUpdate(sha, merged_include_dirs_bytes.data(), merged_include_dirs_bytes.size()) != - 1 || - EVP_DigestUpdate(sha, compression.data(), compression.size()) != 1) { - throw std::runtime_error("EVP_DigestUpdate failed"); - } - - auto hash_size = static_cast(hash.size()); - if (EVP_DigestFinal_ex(sha, hash.data(), &hash_size) != 1) { - throw std::runtime_error("EVP_DigestFinal_ex failed"); - } - - hash.resize(hash_size); - - return hash; + rtcx::sha256_context ctx; + ctx.update(uncompressed_files_bytes); + ctx.update(merged_dests_bytes); + ctx.update(merged_include_dirs_bytes); + ctx.update(std::span{reinterpret_cast(compression.data()), compression.size()}); + return ctx.finalize(); } template @@ -315,9 +294,7 @@ rtcx_embed_{}_files_begin: .cxx_source = "", .asm_source = asm_source, .bin_file_name = binary_file_name, - .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes, - .hash = std::move(hash), - }; + .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes}; } void generate_embed(std::string_view id, diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 73677e44ee8a..6c0976cd21c7 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -30,10 +29,6 @@ #include #include -extern "C" { -#include -} - #define RTCX_EXPECTS(_condition, _reason, _exception_type) \ do { \ if (!(_condition)) { \ @@ -107,7 +102,7 @@ extern "C" { } \ } while (0) -namespace RTCX_EXPORT rtcx { +namespace rtcx { namespace { @@ -158,58 +153,6 @@ void log_trace(std::string_view message) stdout, "[RTCX TRACE] %.*s\n", static_cast(message.size()), message.data()); } -sha256 sha256::parse(std::string_view hex) -{ - RTCX_EXPECTS( - hex.size() == 64, - std::format( - "Invalid SHA256 hex string length, expected 64 got {} (sha: `{}`)", hex.size(), hex), - std::invalid_argument); - sha256 hash; - for (std::size_t i = 0; i < 32; ++i) { - auto hex_byte = hex.substr(i * 2, 2); - auto [ptr, ec] = std::from_chars(hex_byte.begin(), hex_byte.end(), hash.data_[i], 16); - RTCX_EXPECTS( - ec == std::errc(), "Invalid hex character in SHA256 string", std::invalid_argument); - } - return hash; -} - -sha256_context::sha256_context() : ectx_(nullptr) -{ - const EVP_MD* type = EVP_sha256(); - ectx_ = EVP_MD_CTX_new(); - RTCX_EXPECTS(ectx_ != nullptr, "EVP_MD_CTX_new failed", std::runtime_error); - RTCX_EXPECTS( - EVP_DigestInit_ex(ectx_, type, nullptr) == 1, "EVP_DigestInit_ex failed", std::runtime_error); -} - -sha256_context::~sha256_context() -{ - if (ectx_ != nullptr) { EVP_MD_CTX_free(ectx_); } -} - -void sha256_context::update(std::span data) -{ - RTCX_EXPECTS(EVP_DigestUpdate(ectx_, data.data(), data.size()) == 1, - "EVP_DigestUpdate failed", - std::runtime_error); -} - -sha256 sha256_context::finalize() -{ - sha256 hash; - std::uint32_t length = 0; - RTCX_EXPECTS(EVP_DigestFinal_ex(ectx_, hash.data_, &length) == 1, - "EVP_DigestFinal_ex failed", - std::runtime_error); - RTCX_EXPECTS(length == sizeof(sha256::data_), "Unexpected SHA256 length", std::runtime_error); - EVP_MD const* type = EVP_sha256(); - RTCX_EXPECTS( - EVP_DigestInit_ex(ectx_, type, nullptr) == 1, "EVP_DigestInit_ex failed", std::runtime_error); - return hash; -} - #define FOR_EACH_CUDA_FUNC(DO_IT) \ DO_IT(GetErrorString) \ DO_IT(GetErrorName) \ @@ -1428,4 +1371,4 @@ std::string reflect_template(std::string_view template_name, return std::format("{}<{}>", template_name, join_strings(template_args, ", ")); } -} // namespace RTCX_EXPORT rtcx +} // namespace rtcx diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 5f6ffc73fc1a..581d5212af57 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -6,6 +6,8 @@ #pragma once +#include "sha256.hpp" + #include #include #include @@ -20,8 +22,6 @@ #include #include -#define RTCX_EXPORT __attribute__((visibility("default"))) - #define RTCX_DEFER__CONCATENATE_DETAIL(x, y) x##y #define RTCX_DEFER__CONCATENATE(x, y) RTCX_DEFER__CONCATENATE_DETAIL(x, y) #define RTCX_DEFER(...) ::rtcx::defer RTCX_DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) @@ -34,7 +34,7 @@ typedef struct CUkern_st* CUkernel; typedef struct CUstream_st* CUstream; } -namespace RTCX_EXPORT rtcx { +namespace rtcx { inline constexpr std::size_t CACHELINE_ALIGNMENT = 64; // = std::hardware_destructive_interference_size */ @@ -106,42 +106,6 @@ func(void*, R (*)(void*, Args...)) -> func; template func(R (*)(Args...)) -> func; -struct [[nodiscard]] sha256_hex_string { - char data_[65]; - - constexpr std::string_view view() const { return std::string_view{data_, 64}; } - - constexpr operator std::string_view() const { return view(); } - - [[nodiscard]] char const* data() const { return data_; } - - [[nodiscard]] char const* c_str() const { return data_; } - - static constexpr std::size_t size() { return 64; } - - static sha256_hex_string make(std::span input) - { - constexpr char const HEX_CHARS[] = "0123456789abcdef"; - sha256_hex_string hex; - for (std::size_t i = 0; i < 32; ++i) { - hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; - hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; - } - hex.data_[64] = '\0'; - return hex; - } -}; - -struct [[nodiscard]] sha256 { - alignas(16) std::uint8_t data_[32]; - - constexpr bool operator==(sha256 const&) const = default; - - sha256_hex_string to_hex_string() const { return sha256_hex_string::make(data_); } - - static sha256 parse(std::string_view hex); -}; - struct [[nodiscard]] sha256_hasher { constexpr std::uint64_t operator()(sha256 const& obj) const { @@ -164,31 +128,6 @@ struct [[nodiscard]] sha256_hasher { } }; -struct [[nodiscard]] sha256_context { - private: - EVP_MD_CTX* ectx_; - - public: - sha256_context(); - sha256_context(sha256_context const& other) = delete; - sha256_context& operator=(sha256_context const& other) = delete; - sha256_context(sha256_context&& other) : ectx_(other.ectx_) { other.ectx_ = nullptr; } - - sha256_context& operator=(sha256_context&& other) - { - if (this == &other) [[unlikely]] { return *this; } - this->~sha256_context(); - new (this) sha256_context(std::move(other)); - return *this; - } - - ~sha256_context(); - - void update(std::span data); - - sha256 finalize(); -}; - enum class binary_type : std::int8_t { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; /** @@ -1028,4 +967,4 @@ std::string reflect_template(std::string_view template_name, TemplateArgs&&... t return reflect_template(template_name, tparams); } -} // namespace RTCX_EXPORT rtcx +} // namespace rtcx diff --git a/cpp/src/librtcx/sha256.hpp b/cpp/src/librtcx/sha256.hpp new file mode 100644 index 000000000000..e1f9132619b8 --- /dev/null +++ b/cpp/src/librtcx/sha256.hpp @@ -0,0 +1,291 @@ + + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include + +namespace rtcx { + +namespace { + +std::uint32_t ror(std::uint32_t x, std::uint32_t n) { return (x >> n) | (x << (32 - n)); } + +std::uint32_t ch(std::uint32_t x, std::uint32_t y, std::uint32_t z) { return z ^ (x & (y ^ z)); } + +std::uint32_t maj(std::uint32_t x, std::uint32_t y, std::uint32_t z) +{ + return ((x | y) & z) | (x & y); +} + +std::uint32_t sigma0(std::uint32_t x) { return ror(x, 2) ^ ror(x, 13) ^ ror(x, 22); } + +std::uint32_t sigma1(std::uint32_t x) { return ror(x, 6) ^ ror(x, 11) ^ ror(x, 25); } + +std::uint32_t gamma0(std::uint32_t x) { return ror(x, 7) ^ ror(x, 18) ^ (x >> 3); } + +std::uint32_t gamma1(std::uint32_t x) { return ror(x, 17) ^ ror(x, 19) ^ (x >> 10); } + +void put_be32(void* ptr, std::uint32_t value) +{ + auto* p = (std::uint8_t*)ptr; + p[0] = (value >> 24) & 0xff; + p[1] = (value >> 16) & 0xff; + p[2] = (value >> 8) & 0xff; + p[3] = (value >> 0) & 0xff; +} + +std::uint32_t get_be32(void const* ptr) +{ + auto const* p = (std::uint8_t const*)ptr; + return (std::uint32_t)p[0] << 24 | (std::uint32_t)p[1] << 16 | (std::uint32_t)p[2] << 8 | + (std::uint32_t)p[3] << 0; +} + +} // namespace + +struct [[nodiscard]] sha256_hex_string { + char data_[65]; + + constexpr std::string_view view() const { return std::string_view{data_, 64}; } + + constexpr operator std::string_view() const { return view(); } + + [[nodiscard]] char const* data() const { return data_; } + + [[nodiscard]] char const* c_str() const { return data_; } + + static constexpr std::size_t size() { return 64; } + + static sha256_hex_string make(std::span input) + { + constexpr char const HEX_CHARS[] = "0123456789abcdef"; + sha256_hex_string hex; + for (std::size_t i = 0; i < 32; ++i) { + hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; + hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; + } + hex.data_[64] = '\0'; + return hex; + } +}; + +struct [[nodiscard]] sha256 { + alignas(16) std::uint8_t data_[32]; + + std::uint8_t operator[](std::size_t index) const { return data_[index]; } + + std::size_t size() const { return 32; } + + std::uint8_t const* data() const { return data_; } + + constexpr bool operator==(sha256 const&) const = default; + + sha256_hex_string to_hex_string() const { return sha256_hex_string::make(data_); } + + static sha256 parse(std::string_view hex) + { + if (hex.size() != 64) { + throw std::invalid_argument(std::format( + "Invalid SHA256 hex string length, expected 64 got {} (sha: `{}`)", hex.size(), hex)); + } + sha256 hash; + for (std::size_t i = 0; i < 32; ++i) { + auto hex_byte = hex.substr(i * 2, 2); + auto [ptr, ec] = std::from_chars(hex_byte.begin(), hex_byte.end(), hash.data_[i], 16); + if (ec != std::errc()) { + throw std::invalid_argument( + std::format("Invalid hex character in SHA256 string: `{}`", hex_byte)); + } + } + return hash; + } +}; + +struct sha256_context { + private: + static constexpr size_t BLOCK_SIZE = 64; + std::uint32_t state_[8]; + std::uint64_t size_; + std::uint32_t offset_; + std::uint8_t buf_[BLOCK_SIZE]; + + public: + sha256_context() + : state_{0x6a09'e667ul, + 0xbb67'ae85ul, + 0x3c6e'f372ul, + 0xa54f'f53aul, + 0x510e'527ful, + 0x9b05'688cul, + 0x1f83'd9abul, + 0x5be0'cd19ul}, + size_{0}, + offset_{0}, + buf_{} + { + } + sha256_context(sha256_context const&) = delete; + sha256_context& operator=(sha256_context const&) = delete; + sha256_context(sha256_context&&) = delete; + sha256_context& operator=(sha256_context&&) = delete; + ~sha256_context() = default; + + private: + void transform_(std::uint8_t const* buf) + { + std::uint32_t S[8], W[64], t0, t1; + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) + S[i] = state_[i]; + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++, buf += sizeof(std::uint32_t)) + W[i] = get_be32(buf); + + /* fill W[16..63] */ + for (i = 16; i < 64; i++) + W[i] = gamma1(W[i - 2]) + W[i - 7] + gamma0(W[i - 15]) + W[i - 16]; + +#define RND(a, b, c, d, e, f, g, h, i, ki) \ + t0 = h + sigma1(e) + ch(e, f, g) + ki + W[i]; \ + t1 = sigma0(a) + maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 0, 0x428a'2f98); + RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 1, 0x7137'4491); + RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 2, 0xb5c0'fbcf); + RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 3, 0xe9b5'dba5); + RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 4, 0x3956'c25b); + RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 5, 0x59f1'11f1); + RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 6, 0x923f'82a4); + RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 7, 0xab1c'5ed5); + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 8, 0xd807'aa98); + RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 9, 0x1283'5b01); + RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 10, 0x2431'85be); + RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 11, 0x550c'7dc3); + RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 12, 0x72be'5d74); + RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 13, 0x80de'b1fe); + RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 14, 0x9bdc'06a7); + RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 15, 0xc19b'f174); + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 16, 0xe49b'69c1); + RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 17, 0xefbe'4786); + RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 18, 0x0fc1'9dc6); + RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 19, 0x240c'a1cc); + RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 20, 0x2de9'2c6f); + RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 21, 0x4a74'84aa); + RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 22, 0x5cb0'a9dc); + RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 23, 0x76f9'88da); + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 24, 0x983e'5152); + RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 25, 0xa831'c66d); + RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 26, 0xb003'27c8); + RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 27, 0xbf59'7fc7); + RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 28, 0xc6e0'0bf3); + RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 29, 0xd5a7'9147); + RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 30, 0x06ca'6351); + RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 31, 0x1429'2967); + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 32, 0x27b7'0a85); + RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 33, 0x2e1b'2138); + RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 34, 0x4d2c'6dfc); + RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 35, 0x5338'0d13); + RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 36, 0x650a'7354); + RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 37, 0x766a'0abb); + RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 38, 0x81c2'c92e); + RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 39, 0x9272'2c85); + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 40, 0xa2bf'e8a1); + RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 41, 0xa81a'664b); + RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 42, 0xc24b'8b70); + RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 43, 0xc76c'51a3); + RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 44, 0xd192'e819); + RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 45, 0xd699'0624); + RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 46, 0xf40e'3585); + RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 47, 0x106a'a070); + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 48, 0x19a4'c116); + RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 49, 0x1e37'6c08); + RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 50, 0x2748'774c); + RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 51, 0x34b0'bcb5); + RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 52, 0x391c'0cb3); + RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 53, 0x4ed8'aa4a); + RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 54, 0x5b9c'ca4f); + RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 55, 0x682e'6ff3); + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 56, 0x748f'82ee); + RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 57, 0x78a5'636f); + RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 58, 0x84c8'7814); + RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 59, 0x8cc7'0208); + RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 60, 0x90be'fffa); + RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 61, 0xa450'6ceb); + RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 62, 0xbef9'a3f7); + RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 63, 0xc671'78f2); + + for (i = 0; i < 8; i++) { + state_[i] += S[i]; + } + } + + public: + void update(std::span span) + { + std::uint32_t len_buf = size_ & 63; + auto data = span.data(); + auto len = span.size(); + + size_ += len; + + /* Read the data into buf and process blocks as they get full */ + if (len_buf) { + std::uint32_t left = 64 - len_buf; + if (len < left) left = len; + memcpy(len_buf + buf_, data, left); + len_buf = (len_buf + left) & 63; + len -= left; + data = (data + left); + if (len_buf) return; + transform_(buf_); + } + + while (len >= 64) { + transform_(data); + data = data + 64; + len -= 64; + } + + if (len) memcpy(buf_, data, len); + } + + sha256 finalize() + { + static std::uint8_t const pad[64] = {0x80}; + std::uint32_t padlen[2]; + int i; + + /* Pad with a binary 1 (ie 0x80), then zeroes, then length */ + padlen[0] = __builtin_bswap32((std::uint32_t)(size_ >> 29)); + padlen[1] = __builtin_bswap32((std::uint32_t)(size_ << 3)); + + i = size_ & 63; + update(std::span{pad, (std::size_t)(1 + (63 & (55 - i)))}); + update(std::span{reinterpret_cast(padlen), sizeof(padlen)}); + + sha256 out; + std::uint8_t* digest = out.data_; + + /* copy output */ + for (i = 0; i < 8; i++, digest += sizeof(std::uint32_t)) { + put_be32(digest, state_[i]); + } + return out; + } +}; + +} // namespace rtcx From 689495feaa783b2536ef3fcf430635dde3860f02 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 14:36:09 +0000 Subject: [PATCH 131/254] Remove redundant SetFlowCallback from log_trace function --- cpp/src/librtcx/rtcx.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 6c0976cd21c7..62ac79725b26 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -181,8 +181,7 @@ void log_trace(std::string_view message) DO_IT(GetProgramLogSize) \ DO_IT(GetProgramLog) \ DO_IT(AddNameExpression) \ - DO_IT(GetLoweredName) \ - DO_IT(SetFlowCallback) + DO_IT(GetLoweredName) #define FOR_EACH_NVJITLINK_FUNC(DO_IT) \ DO_IT(Create) \ From ea2ef028dae584400b88ce420066fd041218ca84 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 19:53:19 +0000 Subject: [PATCH 132/254] Enhance const correctness by adding [[nodiscard]] to relevant methods in sha256_hex_string and sha256 structs; simplify sha256_context constructor to default. --- cpp/src/librtcx/sha256.hpp | 43 ++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/cpp/src/librtcx/sha256.hpp b/cpp/src/librtcx/sha256.hpp index e1f9132619b8..0e04d7d2189e 100644 --- a/cpp/src/librtcx/sha256.hpp +++ b/cpp/src/librtcx/sha256.hpp @@ -55,15 +55,15 @@ std::uint32_t get_be32(void const* ptr) struct [[nodiscard]] sha256_hex_string { char data_[65]; - constexpr std::string_view view() const { return std::string_view{data_, 64}; } + [[nodiscard]] constexpr std::string_view view() const { return std::string_view{data_, 64}; } - constexpr operator std::string_view() const { return view(); } + [[nodiscard]] constexpr operator std::string_view() const { return view(); } [[nodiscard]] char const* data() const { return data_; } [[nodiscard]] char const* c_str() const { return data_; } - static constexpr std::size_t size() { return 64; } + [[nodiscard]] static constexpr std::size_t size() { return 64; } static sha256_hex_string make(std::span input) { @@ -81,13 +81,13 @@ struct [[nodiscard]] sha256_hex_string { struct [[nodiscard]] sha256 { alignas(16) std::uint8_t data_[32]; - std::uint8_t operator[](std::size_t index) const { return data_[index]; } + [[nodiscard]] std::uint8_t operator[](std::size_t index) const { return data_[index]; } - std::size_t size() const { return 32; } + [[nodiscard]] std::size_t size() const { return 32; } - std::uint8_t const* data() const { return data_; } + [[nodiscard]] std::uint8_t const* data() const { return data_; } - constexpr bool operator==(sha256 const&) const = default; + [[nodiscard]] constexpr bool operator==(sha256 const&) const = default; sha256_hex_string to_hex_string() const { return sha256_hex_string::make(data_); } @@ -113,26 +113,19 @@ struct [[nodiscard]] sha256 { struct sha256_context { private: static constexpr size_t BLOCK_SIZE = 64; - std::uint32_t state_[8]; - std::uint64_t size_; - std::uint32_t offset_; - std::uint8_t buf_[BLOCK_SIZE]; + std::uint32_t state_[8] = {0x6a09'e667ul, + 0xbb67'ae85ul, + 0x3c6e'f372ul, + 0xa54f'f53aul, + 0x510e'527ful, + 0x9b05'688cul, + 0x1f83'd9abul, + 0x5be0'cd19ul}; + std::uint64_t size_ = 0; + std::uint8_t buf_[BLOCK_SIZE] = {}; public: - sha256_context() - : state_{0x6a09'e667ul, - 0xbb67'ae85ul, - 0x3c6e'f372ul, - 0xa54f'f53aul, - 0x510e'527ful, - 0x9b05'688cul, - 0x1f83'd9abul, - 0x5be0'cd19ul}, - size_{0}, - offset_{0}, - buf_{} - { - } + sha256_context() = default; sha256_context(sha256_context const&) = delete; sha256_context& operator=(sha256_context const&) = delete; sha256_context(sha256_context&&) = delete; From 0112b9845b735e539b1df3d973a9545bc7e6e3a8 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 22:56:50 +0000 Subject: [PATCH 133/254] Refactor transform_kernel template parameters for clarity and consistency in accessors --- cpp/src/transform/jit/kernel.cu | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 90b9ff34f110..eb1c6cf4a686 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -35,7 +35,10 @@ namespace cudf { namespace jit { /// @brief The generic transform kernel. Supports all types and nullability combinations. -template +template CUDF_KERNEL void transform_kernel(size_type row_size, bitmask_type const* __restrict__ stencil, bool stencil_has_nulls, @@ -51,14 +54,14 @@ CUDF_KERNEL void transform_kernel(size_type row_size, if constexpr (is_null_aware == null_aware::NO) { if (stencil_has_nulls && !bit_is_set(stencil, element_idx)) { continue; } - auto outs = Outs::map([&]() { + auto outs = OutputAccessors::map([&]() { return cuda::std::tuple{A::output_arg(output_cols, element_idx)...}; }); auto out_ptrs = cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - auto inputs = Ins::map( + auto inputs = InputAccessors::map( [&]() { return cuda::std::tuple{A::element(input_cols, element_idx)...}; }); if constexpr (has_user_data) { @@ -72,20 +75,20 @@ CUDF_KERNEL void transform_kernel(size_type row_size, cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); } - Outs::map([&]() { + OutputAccessors::map([&]() { (A::assign(output_cols, element_idx, cuda::std::get(outs)), ...); }); } else { - bool is_valid[Outs::size]; + bool is_valid[OutputAccessors::size]; - auto outs = Outs::map([&]() { + auto outs = OutputAccessors::map([&]() { return cuda::std::tuple{A::null_output_arg(output_cols, element_idx)...}; }); auto out_ptrs = cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - auto inputs = Ins::map([&]() { + auto inputs = InputAccessors::map([&]() { return cuda::std::tuple{A::nullable_element(input_cols, element_idx)...}; }); @@ -99,12 +102,12 @@ CUDF_KERNEL void transform_kernel(size_type row_size, cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); } - Outs::map([&]() { + OutputAccessors::map([&]() { (A::assign(output_cols, element_idx, *cuda::std::get(outs)), ...); ((is_valid[A::index] = cuda::std::get(outs).has_value()), ...); }); - Outs::map([&]() { + OutputAccessors::map([&]() { auto active_mask = __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); (warp_compact_validity(active_mask, output_cols, element_idx, is_valid[A::index]), ...); }); From 222618d4138527ea401778f530da672de4a5e196 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 12 Apr 2026 23:00:50 +0000 Subject: [PATCH 134/254] Add CUDF_KERNEL qualifier to kernel functions in binaryop, join, and rolling modules --- cpp/src/binaryop/jit/kernel.cu | 8 ++++---- cpp/src/join/jit/filter_join_kernel.cu | 12 ++++++------ cpp/src/rolling/jit/kernel.cu | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/cpp/src/binaryop/jit/kernel.cu b/cpp/src/binaryop/jit/kernel.cu index a69c4d9d6416..7e0c47d8d112 100644 --- a/cpp/src/binaryop/jit/kernel.cu +++ b/cpp/src/binaryop/jit/kernel.cu @@ -57,10 +57,10 @@ struct UserDefinedOp { }; template -__device__ void binaryop_kernel(cudf::size_type size, - void* p_out_data, - void* p_lhs_data, - void* p_rhs_data) +CUDF_KERNEL void binaryop_kernel(cudf::size_type size, + void* p_out_data, + void* p_lhs_data, + void* p_rhs_data) { auto out_data = static_cast(p_out_data); auto lhs_data = static_cast(p_lhs_data); diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index 61f97fd8f176..ccd02fc7cd9c 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -33,12 +33,12 @@ namespace cudf::join::jit { constexpr cudf::size_type JoinNoMatch = cuda::std::numeric_limits::min(); template -__device__ void filter_join_kernel(cudf::jit::device_span left_indices, - cudf::jit::device_span right_indices, - cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - bool* predicate_results, - void* user_data) +CUDF_KERNEL void filter_join_kernel(cudf::jit::device_span left_indices, + cudf::jit::device_span right_indices, + cudf::column_device_view_core const* left_tables, + cudf::column_device_view_core const* right_tables, + bool* predicate_results, + void* user_data) { auto const start = cudf::detail::grid_1d::global_thread_id(); auto const stride = cudf::detail::grid_1d::grid_stride(); diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index 501788731c46..037e64546122 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -45,15 +45,15 @@ template -__device__ void rolling_window_kernel(cudf::size_type nrows, - void const* __restrict__ p_in_col, - cudf::bitmask_type const* __restrict__ in_col_valid, - void* __restrict__ p_out_col, - cudf::bitmask_type* __restrict__ out_col_valid, - cudf::size_type* __restrict__ output_valid_count, - detail::window_wrapper_base b_preceding_window_begin, - detail::window_wrapper_base b_following_window_begin, - cudf::size_type min_periods) +CUDF_KERNEL void rolling_window_kernel(cudf::size_type nrows, + void const* __restrict__ p_in_col, + cudf::bitmask_type const* __restrict__ in_col_valid, + void* __restrict__ p_out_col, + cudf::bitmask_type* __restrict__ out_col_valid, + cudf::size_type* __restrict__ output_valid_count, + detail::window_wrapper_base b_preceding_window_begin, + detail::window_wrapper_base b_following_window_begin, + cudf::size_type min_periods) { auto i = cudf::detail::grid_1d::global_thread_id(); auto const stride = cudf::detail::grid_1d::grid_stride(); From b37b545712c7f19133cfcc7381283db29eacf592 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 13 Apr 2026 07:43:20 +0000 Subject: [PATCH 135/254] simplify column_accessor traits --- cpp/src/jit/column_accessor.cuh | 22 +++++++++++++--------- cpp/src/transform/transform.cu | 32 ++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/cpp/src/jit/column_accessor.cuh b/cpp/src/jit/column_accessor.cuh index 0ea502a00c60..afd1c55b01c3 100644 --- a/cpp/src/jit/column_accessor.cuh +++ b/cpp/src/jit/column_accessor.cuh @@ -9,6 +9,8 @@ #include #include +#include +#include namespace cudf { namespace jit { @@ -18,16 +20,18 @@ template + bool MayBeNullable> struct column_accessor { - static constexpr int32_t index = Index; - using column_type = Column; - using element_type = Element; - using optional_element_type = OptionalElement; - static constexpr bool as_scalar = AsScalar; - static constexpr bool may_be_nullable = MayBeNullable; - static constexpr bool is_strings_output = IsStringsOutput; + static constexpr int32_t index = Index; + using column_type = Column; + using element_type = Element; + using optional_element_type = OptionalElement; + + static constexpr bool as_scalar = AsScalar; + static constexpr bool may_be_nullable = MayBeNullable; + + static constexpr bool is_strings_output = + cuda::std::is_same_v>; static __device__ constexpr size_type map_index(size_type row) { diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 19bc1716c5a1..ff97ce9a4465 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -243,34 +243,30 @@ auto reflect(udf_source_type source_type, std::vector in_types; for (size_t i = 0; i < inputs.size(); i++) { - auto& in = inputs[i]; - auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); - auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); - auto optional_element = std::format("cuda::std::optional<{}>", element); - bool as_scalar = std::holds_alternative(in); - bool may_be_nullable = input_may_be_nullable[i]; - auto is_strings_output = false; + auto& in = inputs[i]; + auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); + auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); + auto optional_element = std::format("cuda::std::optional<{}>", element); + bool as_scalar = std::holds_alternative(in); + bool may_be_nullable = input_may_be_nullable[i]; auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate( - i, column, element, optional_element, as_scalar, may_be_nullable, is_strings_output); + .instantiate(i, column, element, optional_element, as_scalar, may_be_nullable); in_types.push_back(accessor); } std::vector out_types; for (size_t i = 0; i < outputs.size(); i++) { - auto& out = outputs[i]; - auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); - auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); - auto optional_element = std::format("cuda::std::optional<{}>", element); - bool as_scalar = false; // never scalar - bool may_be_nullable = output_may_be_nullable[i]; - auto is_strings_output = std::holds_alternative(out); + auto& out = outputs[i]; + auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); + auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); + auto optional_element = std::format("cuda::std::optional<{}>", element); + bool as_scalar = false; // never scalar + bool may_be_nullable = output_may_be_nullable[i]; auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate( - i, column, element, optional_element, as_scalar, may_be_nullable, is_strings_output); + .instantiate(i, column, element, optional_element, as_scalar, may_be_nullable); out_types.push_back(accessor); } From 225e59736d8d3f1dcbf87df8cb805ad56ee5bf7b Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 14 Apr 2026 04:06:10 +0000 Subject: [PATCH 136/254] Refactor CUDA utility includes and improve string offset validation logic in transform --- cpp/include/cudf/detail/utilities/cuda.cuh | 9 +++++++++ cpp/src/jit/join_column_accessor.cuh | 2 +- cpp/src/jit/sync.cuh | 9 ++++++--- cpp/src/jit/type_list.cuh | 2 +- cpp/src/transform/transform.cu | 12 ++++++------ 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/cpp/include/cudf/detail/utilities/cuda.cuh b/cpp/include/cudf/detail/utilities/cuda.cuh index 95dfe44bfb19..1e202f4e2229 100644 --- a/cpp/include/cudf/detail/utilities/cuda.cuh +++ b/cpp/include/cudf/detail/utilities/cuda.cuh @@ -6,6 +6,9 @@ #pragma once #include + +#ifndef __CUDACC_RTC__ + #include #include @@ -13,6 +16,8 @@ #include #include +#endif + namespace cudf { namespace detail { @@ -21,6 +26,8 @@ namespace detail { */ static constexpr size_type warp_size{32}; +#ifndef __CUDACC_RTC__ + /** * @brief Performs a sum reduction of values from the same lane across all * warps in a thread block and returns the result on thread 0 of the block. @@ -86,5 +93,7 @@ void device_single_thread(Functor functor, rmm::cuda_stream_view stream) single_thread_kernel<<<1, 1, 0, stream.value()>>>(functor); } +#endif + } // namespace detail } // namespace cudf diff --git a/cpp/src/jit/join_column_accessor.cuh b/cpp/src/jit/join_column_accessor.cuh index 2135bad952ba..e78a9ec1991c 100644 --- a/cpp/src/jit/join_column_accessor.cuh +++ b/cpp/src/jit/join_column_accessor.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/sync.cuh b/cpp/src/jit/sync.cuh index 5d445433e27a..bd4aa9df2c8a 100644 --- a/cpp/src/jit/sync.cuh +++ b/cpp/src/jit/sync.cuh @@ -1,10 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include +#include #include #include @@ -21,7 +22,7 @@ __device__ inline bool warp_elect(unsigned int mask) #else // fallback: manually elect a leader (e.g., the first active thread) int leader = mask == 0 ? 0 : cuda::std::countr_zero(mask); - int lane = (threadIdx.x & 31); + int lane = (threadIdx.x & (cudf::detail::warp_size - 1)); return (lane == leader); #endif } @@ -37,7 +38,9 @@ __device__ void warp_compact_validity(unsigned int active_mask, } else { auto null_word = __ballot_sync(active_mask, is_valid); // use warp-elect to make sure we only issue one memory transaction per warp - if (warp_elect(active_mask)) { Out::set_null_mask_word(outcols, row / 32, null_word); } + if (warp_elect(active_mask)) { + Out::set_null_mask_word(outcols, row / cudf::detail::warp_size, null_word); + } } } diff --git a/cpp/src/jit/type_list.cuh b/cpp/src/jit/type_list.cuh index 4007f9ed7e63..13c8bd8c95d3 100644 --- a/cpp/src/jit/type_list.cuh +++ b/cpp/src/jit/type_list.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index ff97ce9a4465..e1e0882cc838 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -580,12 +580,12 @@ void perform_checks(udf_source_type source_type, "for each non-string column)", std::invalid_argument); - CUDF_EXPECTS(std::none_of(cuda::counting_iterator(size_t{0}), - cuda::counting_iterator(string_offsets.size()), - [&](auto i) { - if (outputs[i].type.id() == type_id::STRING) { return false; } - return !string_offsets.empty() && string_offsets[i] == nullptr; - }), + CUDF_EXPECTS(std::all_of(cuda::counting_iterator(size_t{0}), + cuda::counting_iterator(string_offsets.size()), + [&](auto i) { + if (outputs[i].type.id() == type_id::STRING) { return true; } + return string_offsets.empty() || string_offsets[i] == nullptr; + }), "String offsets must only be provided for string outputs", std::invalid_argument); } From 80853232b0feb3ae0e53289cb4bca5937a7326bc Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 14 Apr 2026 08:22:04 +0000 Subject: [PATCH 137/254] Update copyright year to 2026 in cuda.cuh --- cpp/include/cudf/detail/utilities/cuda.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/cudf/detail/utilities/cuda.cuh b/cpp/include/cudf/detail/utilities/cuda.cuh index 1e202f4e2229..6dcf879017da 100644 --- a/cpp/include/cudf/detail/utilities/cuda.cuh +++ b/cpp/include/cudf/detail/utilities/cuda.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ From 6af15c04ffaf93523fba1b9ce4f46aebddee8a69 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 14 Apr 2026 10:56:09 +0000 Subject: [PATCH 138/254] remove nullable trait in column accessor --- cpp/src/jit/column_accessor.cuh | 49 ++++++++---------------------- cpp/src/jit/sync.cuh | 12 +++----- cpp/src/transform/transform.cu | 54 ++++++++++++--------------------- 3 files changed, 36 insertions(+), 79 deletions(-) diff --git a/cpp/src/jit/column_accessor.cuh b/cpp/src/jit/column_accessor.cuh index afd1c55b01c3..303e8cb029f3 100644 --- a/cpp/src/jit/column_accessor.cuh +++ b/cpp/src/jit/column_accessor.cuh @@ -9,28 +9,23 @@ #include #include +#include #include #include namespace cudf { namespace jit { -template +template struct column_accessor { static constexpr int32_t index = Index; using column_type = Column; using element_type = Element; - using optional_element_type = OptionalElement; + using optional_element_type = cuda::std::optional; - static constexpr bool as_scalar = AsScalar; - static constexpr bool may_be_nullable = MayBeNullable; + static constexpr bool as_scalar = AsScalar; - static constexpr bool is_strings_output = + static constexpr bool is_mutable_string = cuda::std::is_same_v>; static __device__ constexpr size_type map_index(size_type row) @@ -56,32 +51,18 @@ struct column_accessor { static __device__ bool is_null(auto const* __restrict__ cols, size_type row) { - if constexpr (!may_be_nullable) { - return false; - } else { - return column(cols).is_null(map_index(row)); - } + return column(cols).is_null(map_index(row)); } static __device__ bool is_valid(auto const* __restrict__ cols, size_type row) { - if constexpr (!may_be_nullable) { - return true; - } else { - return column(cols).is_valid(map_index(row)); - } + return column(cols).is_valid(map_index(row)); } static __device__ optional_element_type nullable_element(auto const* __restrict__ cols, size_type row) { - auto& c = column(cols); - - if constexpr (!may_be_nullable) { - return c.template element(map_index(row)); - } else { - return c.template nullable_element(map_index(row)); - } + return column(cols).template nullable_element(map_index(row)); } static __device__ void set_null_mask_word(auto const* __restrict__ cols, @@ -89,15 +70,11 @@ struct column_accessor { bitmask_type word) requires(!as_scalar) { - if constexpr (!may_be_nullable) { - return; - } else { - auto* mask = column(cols).null_mask(); + auto* mask = column(cols).null_mask(); - if (mask == nullptr) { return; } + if (mask == nullptr) { return; } - mask[word_index] = word; - } + mask[word_index] = word; } static __device__ void assign(auto const* __restrict__ cols, size_type row, element_type value) @@ -109,7 +86,7 @@ struct column_accessor { static __device__ element_type output_arg(auto const* __restrict__ cols, size_type row) requires(!as_scalar) { - if constexpr (is_strings_output) { + if constexpr (is_mutable_string) { return element(cols, row); } else { return {}; @@ -120,7 +97,7 @@ struct column_accessor { size_type row) requires(!as_scalar) { - if constexpr (is_strings_output) { + if constexpr (is_mutable_string) { return element(cols, row); } else { return {}; diff --git a/cpp/src/jit/sync.cuh b/cpp/src/jit/sync.cuh index bd4aa9df2c8a..8e9ff986dfbf 100644 --- a/cpp/src/jit/sync.cuh +++ b/cpp/src/jit/sync.cuh @@ -33,14 +33,10 @@ __device__ void warp_compact_validity(unsigned int active_mask, size_type row, bool is_valid) { - if constexpr (!Out::may_be_nullable) { - return; - } else { - auto null_word = __ballot_sync(active_mask, is_valid); - // use warp-elect to make sure we only issue one memory transaction per warp - if (warp_elect(active_mask)) { - Out::set_null_mask_word(outcols, row / cudf::detail::warp_size, null_word); - } + auto null_word = __ballot_sync(active_mask, is_valid); + // use warp-elect to make sure we only issue one memory transaction per warp + if (warp_elect(active_mask)) { + Out::set_null_mask_word(outcols, row / cudf::detail::warp_size, null_word); } } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index e1e0882cc838..291e917d19c9 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -236,37 +236,29 @@ std::string reflect_output_column(mutable_strings_column const&) auto reflect(udf_source_type source_type, std::span inputs, - std::span outputs, - std::span input_may_be_nullable, - std::span output_may_be_nullable) + std::span outputs) { std::vector in_types; for (size_t i = 0; i < inputs.size(); i++) { - auto& in = inputs[i]; - auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); - auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); - auto optional_element = std::format("cuda::std::optional<{}>", element); - bool as_scalar = std::holds_alternative(in); - bool may_be_nullable = input_may_be_nullable[i]; - auto accessor = - jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate(i, column, element, optional_element, as_scalar, may_be_nullable); + auto& in = inputs[i]; + auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); + auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); + bool as_scalar = std::holds_alternative(in); + auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") + .instantiate(i, column, element, as_scalar); in_types.push_back(accessor); } std::vector out_types; for (size_t i = 0; i < outputs.size(); i++) { - auto& out = outputs[i]; - auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); - auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); - auto optional_element = std::format("cuda::std::optional<{}>", element); - bool as_scalar = false; // never scalar - bool may_be_nullable = output_may_be_nullable[i]; - auto accessor = - jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate(i, column, element, optional_element, as_scalar, may_be_nullable); + auto& out = outputs[i]; + auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); + auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); + bool as_scalar = false; // never scalar + auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") + .instantiate(i, column, element, as_scalar); out_types.push_back(accessor); } @@ -336,16 +328,13 @@ void run(null_aware is_null_aware, void* user_data, std::span inputs, std::span outputs, - std::span input_may_be_nullable, - std::span output_may_be_nullable, std::string const& udf, udf_source_type source_type, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - auto [in_types, out_types, ptx_in_types, ptx_out_types] = - reflect(source_type, inputs, outputs, input_may_be_nullable, output_may_be_nullable); - auto kernel = instantiate(is_null_aware, + auto [in_types, out_types, ptx_in_types, ptx_out_types] = reflect(source_type, inputs, outputs); + auto kernel = instantiate(is_null_aware, has_user_data, in_types, out_types, @@ -353,8 +342,8 @@ void run(null_aware is_null_aware, ptx_out_types, udf, source_type); - auto [cols, handles] = to_args(inputs, outputs, stream, mr); - auto* input_cols = reinterpret_cast(cols.data()); + auto [cols, handles] = to_args(inputs, outputs, stream, mr); + auto* input_cols = reinterpret_cast(cols.data()); auto* output_cols = reinterpret_cast(input_cols + inputs.size()); return launch( @@ -482,8 +471,6 @@ auto get_null_transformation(null_aware is_null_aware, std::span inputs, std::span outputs) { - std::vector input_may_be_nullable(inputs.size(), true); - auto any_input_nullable = std::any_of(inputs.begin(), inputs.end(), [](auto& in) { return std::visit([](auto& c) { return c.nullable(); }, in); }); @@ -503,7 +490,7 @@ auto get_null_transformation(null_aware is_null_aware, output_may_be_nullable.push_back(may_eval_nulls); } - return std::make_tuple(std::move(input_may_be_nullable), std::move(output_may_be_nullable)); + return output_may_be_nullable; } void perform_checks(udf_source_type source_type, @@ -823,8 +810,7 @@ std::unique_ptr
execute_transform(std::string const& udf, rmm::device_async_resource_ref mr) { auto row_size = in_row_size.has_value() ? *in_row_size : jit::get_projection_size(inputs); - auto [input_may_be_nullable, output_may_be_nullable] = - get_null_transformation(is_null_aware, inputs, outputs); + auto output_may_be_nullable = get_null_transformation(is_null_aware, inputs, outputs); auto [output_columns, stencil] = make_outputs(is_null_aware, row_size, inputs, @@ -844,8 +830,6 @@ std::unique_ptr
execute_transform(std::string const& udf, user_data.value_or(nullptr), inputs, output_columns, - input_may_be_nullable, - output_may_be_nullable, udf, source_type, stream, From 3c5adc706493c3fa74918c98d7da8955e629959a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 14 Apr 2026 13:14:24 +0000 Subject: [PATCH 139/254] refactor: simplify variable declarations in reflect function --- cpp/src/transform/transform.cu | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 71012f9beb14..a6d5024715be 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -247,12 +247,11 @@ auto reflect(udf_source_type source_type, std::vector in_types; for (size_t i = 0; i < inputs.size(); i++) { - auto& in = inputs[i]; - auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); - auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); - bool as_scalar = std::holds_alternative(in); - bool may_be_nullable = input_may_be_nullable[i]; - auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", + auto& in = inputs[i]; + auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); + auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); + bool as_scalar = std::holds_alternative(in); + auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", rtcx::reflect_int(i), column, element, @@ -263,11 +262,11 @@ auto reflect(udf_source_type source_type, std::vector out_types; for (size_t i = 0; i < outputs.size(); i++) { - auto& out = outputs[i]; - auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); - auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); - bool as_scalar = false; // never scalar - auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", + auto& out = outputs[i]; + auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); + auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); + bool as_scalar = false; // never scalar + auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", rtcx::reflect_int(i), column, element, From d42466d4d4898be35dcc580f5365d7e442af49d6 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 14 Apr 2026 19:13:25 +0000 Subject: [PATCH 140/254] remove zstandard dependency from recipe and pyproject files --- conda/recipes/libcudf/recipe.yaml | 1 - dependencies.yaml | 3 --- python/libcudf/pyproject.toml | 1 - 3 files changed, 5 deletions(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 15d04a56424f..67f921d9a679 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -93,7 +93,6 @@ cache: - flatbuffers =${{ flatbuffers_version }} - rapids-logger =0.2 - zlib ${{ zlib_version }} - - zstandard >=0.25.0 outputs: - package: diff --git a/dependencies.yaml b/dependencies.yaml index 9d742e61b952..d350dc34e803 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -507,9 +507,6 @@ dependencies: packages: - flatbuffers==24.3.25 - librdkafka>=2.11.1,<2.12.0 - - output_types: [conda, pyproject, requirements] - packages: - - zstandard depends_on_libnvcomp: common: - output_types: conda diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 78f33c9bd91c..3ea79d371a1d 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -75,5 +75,4 @@ requires = [ "nvidia-libnvcomp==5.2.0.13", "nvidia-nvjitlink>=13.0,<14", "rapids-logger==0.2.*,>=0.0.0a0", - "zstandard", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 1e5cb73a0cefa7b8c9b3272a67d5cf6969f77979 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 18 Apr 2026 15:52:49 +0000 Subject: [PATCH 141/254] initial changes --- cpp/src/jit/element.cuh | 119 ++ cpp/src/jit/element_storage.cuh | 11 + cpp/src/jit/lite/lite.cuh | 1501 +++++++++++++++++ cpp/src/jit/lite/ops.cu | 34 + .../transform/jit/compute_columns_kernel.cu | 170 ++ todo.md | 17 + 6 files changed, 1852 insertions(+) create mode 100644 cpp/src/jit/element.cuh create mode 100644 cpp/src/jit/element_storage.cuh create mode 100644 cpp/src/jit/lite/lite.cuh create mode 100644 cpp/src/jit/lite/ops.cu create mode 100644 cpp/src/transform/jit/compute_columns_kernel.cu create mode 100644 todo.md diff --git a/cpp/src/jit/element.cuh b/cpp/src/jit/element.cuh new file mode 100644 index 000000000000..311184931f20 --- /dev/null +++ b/cpp/src/jit/element.cuh @@ -0,0 +1,119 @@ + +#pragma once +#include +#include +#include + +#include +#include + +namespace cudf { + +__device__ void load_element(bool is_scalar, + column_device_view const* column, + bitmask_type const* __restrict__ null_mask, + size_type element_index, + element_storage* storage) +{ + auto index = is_scalar ? 0 : element_index; + +#define CUDF_CASE(TYPE_ID, type_name) \ + case type_id::TYPE_ID: { \ + static_assert(sizeof(type_name) <= sizeof(element_storage)); \ + static_assert(alignof(type_name) <= alignof(element_storage)); \ + static_assert(sizeof(cuda::std::optional) <= sizeof(element_storage)); \ + static_assert(alignof(cuda::std::optional) <= alignof(element_storage)); \ + \ + if (!column->nullable()) { \ + auto v = column->template element(index); \ + *reinterpret_cast(storage->data) = v; \ + } else { \ + auto v = column->template nullable_element(index); \ + *reinterpret_cast*>(storage->data) = v; \ + } \ + } break; + + switch (column->type().id()) { + CUDF_CASE(INT8, int8_t) + CUDF_CASE(INT16, int16_t) + CUDF_CASE(INT32, int32_t) + CUDF_CASE(INT64, int64_t) + CUDF_CASE(UINT8, uint8_t) + CUDF_CASE(UINT16, uint16_t) + CUDF_CASE(UINT32, uint32_t) + CUDF_CASE(UINT64, uint64_t) + CUDF_CASE(FLOAT32, float) + CUDF_CASE(FLOAT64, double) + CUDF_CASE(BOOL8, bool) + CUDF_CASE(DECIMAL32, numeric::decimal32) + CUDF_CASE(DECIMAL64, numeric::decimal64) + CUDF_CASE(DECIMAL128, numeric::decimal128) + CUDF_CASE(TIMESTAMP_DAYS, cudf::timestamp_D) + CUDF_CASE(TIMESTAMP_SECONDS, cudf::timestamp_s) + CUDF_CASE(TIMESTAMP_MILLISECONDS, cudf::timestamp_ms) + CUDF_CASE(TIMESTAMP_MICROSECONDS, cudf::timestamp_us) + CUDF_CASE(TIMESTAMP_NANOSECONDS, cudf::timestamp_ns) + CUDF_CASE(DURATION_DAYS, cudf::duration_D) + CUDF_CASE(DURATION_SECONDS, cudf::duration_s) + CUDF_CASE(DURATION_MILLISECONDS, cudf::duration_ms) + CUDF_CASE(DURATION_MICROSECONDS, cudf::duration_us) + CUDF_CASE(DURATION_NANOSECONDS, cudf::duration_ns) + CUDF_CASE(STRING, cudf::string_view) + default: CUDF_UNREACHABLE(); + } + +#undef CUDF_CASE +} + +__device__ void store_element(mutable_column_device_view const* column, + element_storage const* storage, + size_type element_index, + unsigned int active_mask) +{ +// TODO: compact validity into a bitmask +// (warp_compact_validity(active_mask, output_cols, element_idx, is_valid[A::index]), ...); +#define CUDF_CASE(TYPE_ID, type_name) \ + case type_id::TYPE_ID: { \ + static_assert(sizeof(type_name) <= sizeof(element_storage)); \ + static_assert(alignof(type_name) <= alignof(element_storage)); \ + static_assert(sizeof(cuda::std::optional) <= sizeof(element_storage)); \ + static_assert(alignof(cuda::std::optional) <= alignof(element_storage)); \ + \ + if (!column->nullable()) { \ + auto v = *reinterpret_cast(storage->data); \ + column->template assign(element_index, v); \ + } else { \ + auto v = *reinterpret_cast const*>(storage->data); \ + column->template assign(element_index, *v); \ + } \ + } break; + + switch (column->type().id()) { + CUDF_CASE(INT16, int16_t) + CUDF_CASE(INT32, int32_t) + CUDF_CASE(INT64, int64_t) + CUDF_CASE(UINT8, uint8_t) + CUDF_CASE(UINT16, uint16_t) + CUDF_CASE(UINT32, uint32_t) + CUDF_CASE(UINT64, uint64_t) + CUDF_CASE(FLOAT32, float) + CUDF_CASE(FLOAT64, double) + CUDF_CASE(BOOL8, bool) + CUDF_CASE(DECIMAL32, numeric::decimal32) + CUDF_CASE(DECIMAL64, numeric::decimal64) + CUDF_CASE(DECIMAL128, numeric::decimal128) + CUDF_CASE(TIMESTAMP_DAYS, cudf::timestamp_D) + CUDF_CASE(TIMESTAMP_SECONDS, cudf::timestamp_s) + CUDF_CASE(TIMESTAMP_MILLISECONDS, cudf::timestamp_ms) + CUDF_CASE(TIMESTAMP_MICROSECONDS, cudf::timestamp_us) + CUDF_CASE(TIMESTAMP_NANOSECONDS, cudf::timestamp_ns) + CUDF_CASE(DURATION_DAYS, cudf::duration_D) + CUDF_CASE(DURATION_SECONDS, cudf::duration_s) + CUDF_CASE(DURATION_MILLISECONDS, cudf::duration_ms) + CUDF_CASE(DURATION_MICROSECONDS, cudf::duration_us) + CUDF_CASE(DURATION_NANOSECONDS, cudf::duration_ns) + default: CUDF_UNREACHABLE(); + } +} + +} // namespace cudf diff --git a/cpp/src/jit/element_storage.cuh b/cpp/src/jit/element_storage.cuh new file mode 100644 index 000000000000..9bbb76651e96 --- /dev/null +++ b/cpp/src/jit/element_storage.cuh @@ -0,0 +1,11 @@ + +#pragma once + +namespace cudf { + +// can store any nullable or non-nullable element type +struct element_storage { + alignas(32) unsigned char data[64]; +}; + +} // namespace cudf diff --git a/cpp/src/jit/lite/lite.cuh b/cpp/src/jit/lite/lite.cuh new file mode 100644 index 000000000000..67432136816d --- /dev/null +++ b/cpp/src/jit/lite/lite.cuh @@ -0,0 +1,1501 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +namespace __attribute__((visibility("hidden"))) cudf +{ + namespace lite { + + using int8_t = signed char; + using int16_t = signed short; + using int32_t = signed int; + using int64_t = signed long long; + using int128_t = __int128_t; + + using uint8_t = unsigned char; + using uint16_t = unsigned short; + using uint32_t = unsigned int; + using uint64_t = unsigned long long; + using uint128_t = __uint128_t; + + using size_t = unsigned long long; + using intptr_t = int64_t; + using uintptr_t = uint64_t; + + using intmax_t = int64_t; + using uintmax_t = uint64_t; + + using float32_t = float; + using float64_t = double; + + using size_type = int32_t; + + using char_utf8 = uint32_t; + + template + __device__ constexpr T min(T a, T b) + { + return a < b ? a : b; + } + + template + __device__ constexpr T max(T a, T b) + { + return a > b ? a : b; + } + + template + __device__ constexpr T ipow10(T exponent) + { + if (exponent == 0) { return 1; } + + T extra = 1; + T square = 10; + T n = exponent; + + while (n > 1) { + if ((n & 1) == 1) { extra *= square; } + n >>= 1; + square *= square; + } + + return square * extra; + } + + struct scaled_t {}; + + inline constexpr scaled_t scaled; + + template + struct decimal { + using Rep = R; + + R _value = 0; + + int32_t _scale = 0; + + constexpr decimal() = default; + + __device__ constexpr decimal(scaled_t, R value, int32_t scale) : _value{value}, _scale{scale} {} + + __device__ constexpr R value() const { return _value; } + + __device__ constexpr int32_t scale() const { return _scale; } + + private: + __device__ static constexpr R _lshift(R v, int32_t scale) { return v * ipow10(-scale); } + + __device__ static constexpr R _rshift(R v, int32_t scale) { return v / ipow10(scale); } + + __device__ static constexpr R _shift(R v, int32_t scale) + { + if (scale == 0) { + return v; + } else if (scale < 0) { + return _lshift(v, scale); + } else { + return _rshift(v, scale); + } + } + + __device__ static constexpr R _rescale(R v, int32_t from_scale, int32_t to_scale) + { + return _shift(v, to_scale - from_scale); + } + + public: + __device__ constexpr auto rescale(int32_t scale) const + { + return decimal{scaled, _rescale(_value, _scale, scale), scale}; + } + + __device__ constexpr auto operator+(decimal rhs) const + { + auto scale = min(_scale, rhs._scale); + auto r = rescale(scale)._value + rhs.rescale(scale)._value; + return decimal{scaled, r, scale}; + } + + __device__ constexpr auto operator-(decimal rhs) const + { + auto scale = min(_scale, rhs._scale); + auto r = rescale(scale)._value - rhs.rescale(scale)._value; + return decimal{scaled, r, scale}; + } + + __device__ constexpr auto operator*(decimal rhs) const + { + return decimal{scaled, _value * rhs._value, _scale + rhs._scale}; + } + + __device__ constexpr auto operator/(decimal rhs) const + { + return decimal{scaled, _value / rhs._value, _scale - rhs._scale}; + } + + __device__ constexpr auto operator%(decimal rhs) const + { + auto scale = min(_scale, rhs._scale); + auto r = rescale(scale)._value % rhs.rescale(scale)._value; + return decimal{scaled, r, scale}; + } + + __device__ constexpr int operator<=>(decimal rhs) const + { + auto scale = min(_scale, rhs._scale); + return rescale(scale)._value - rhs.rescale(scale)._value; + } + }; + + using decimal32 = decimal; + using decimal64 = decimal; + using decimal128 = decimal; + + enum class timestamp_unit : int32_t { D, h, m, s, ms, us, ns }; + + template + struct timestamp { + using Rep = R; + + R _rep = 0; + + __device__ constexpr R count() const { return _rep; } + + __device__ constexpr int operator<=>(timestamp rhs) const { return _rep - rhs._rep; } + }; + + using timestamp_D = timestamp; + using timestamp_h = timestamp; + using timestamp_m = timestamp; + using timestamp_s = timestamp; + using timestamp_ms = timestamp; + using timestamp_us = timestamp; + using timestamp_ns = timestamp; + + template + struct duration { + using Rep = R; + + R _rep = 0; + + __device__ constexpr R count() const { return _rep; } + + __device__ constexpr duration operator+(duration rhs) const + { + return duration{_rep + rhs._rep}; + } + + __device__ constexpr duration operator-(duration rhs) const + { + return duration{_rep - rhs._rep}; + } + + __device__ constexpr int operator<=>(duration rhs) const { return _rep - rhs._rep; } + }; + + using duration_D = duration; + using duration_h = duration; + using duration_m = duration; + using duration_s = duration; + using duration_ms = duration; + using duration_us = duration; + using duration_ns = duration; + + struct inplace_t {}; + + inline constexpr inplace_t inplace; + + struct nullopt_t {}; + + inline constexpr nullopt_t nullopt; + + template + struct optional { + T _value = {}; + + bool _is_valid = false; + + constexpr optional() = default; + + __device__ constexpr optional(nullopt_t) {} + + template + __device__ constexpr optional(inplace_t, Args&&... args) + : _value{static_cast(args)...}, _is_valid{true} + { + } + + __device__ constexpr optional(T value) : _value{value}, _is_valid{true} {} + + __device__ constexpr bool is_valid() const { return _is_valid; } + + __device__ constexpr bool is_null() const { return !_is_valid; } + + __device__ constexpr void reset() { _is_valid = false; } + + __device__ constexpr T const& get() const { return _value; } + + __device__ constexpr T& get() { return _value; } + + __device__ constexpr T const* operator->() const { return &_value; } + + __device__ constexpr T* operator->() { return &_value; } + + __device__ constexpr T const& operator*() const { return _value; } + + __device__ constexpr T& operator*() { return _value; } + + __device__ constexpr T const& value() const { return _value; } + + __device__ constexpr T& value() { return _value; } + + __device__ constexpr explicit operator bool() const { return _is_valid; } + + __device__ constexpr T value_or(T v) const { return _is_valid ? _value : v; } + }; + + template + optional(T) -> optional; + + template + struct span { + T* _data = nullptr; + + size_t _size = 0; + + constexpr span() = default; + + __device__ constexpr span(T* data, size_t size) : _data{data}, _size{size} {} + + __device__ constexpr T* data() const { return _data; } + + __device__ constexpr size_t size() const { return _size; } + + __device__ constexpr bool empty() const { return _size == 0; } + + __device__ constexpr T& operator[](size_t pos) const { return _data[pos]; } + + __device__ constexpr T* begin() const { return _data; } + + __device__ constexpr T* end() const { return _data + _size; } + + __device__ constexpr T const* cbegin() const { return _data; } + + __device__ constexpr T const* cend() const { return _data + _size; } + + __device__ constexpr span as_const() const { return span{_data, _size}; } + + __device__ constexpr T& element(size_t i) const { return _data[i]; } + }; + + template + span(T*, size_t) -> span; + + struct string_view { + static constexpr size_type const UNKNOWN_STRING_LENGTH{-1}; + static constexpr size_type const npos{-1}; + + char const* _data = ""; + + size_type _bytes = 0; + + mutable size_type _length = UNKNOWN_STRING_LENGTH; + + constexpr string_view() = default; + + __device__ constexpr string_view(char const* data, size_type bytes) : _data{data}, _bytes{bytes} + { + } + + __device__ constexpr string_view(char const* data, size_type bytes, size_type length) + : _data{data}, _bytes{bytes}, _length{length} + { + } + + __device__ constexpr size_type size_bytes() const { return _bytes; } + + __device__ constexpr auto* data() const { return _data; } + + __device__ constexpr auto* begin() const { return _data; } + + __device__ constexpr auto* end() const { return _data + _bytes; } + + __device__ constexpr auto const* cbegin() const { return _data; } + + __device__ constexpr auto const* cend() const { return _data + _bytes; } + + __device__ constexpr bool empty() const { return _bytes == 0; } + + __device__ constexpr size_type compare(string_view const& other) const + { + auto* s0 = _data; + auto n0 = _bytes; + auto* s1 = other._data; + auto n1 = other._bytes; + auto max = n0 < n1 ? n0 : n1; + + if (s0 == s1 && n0 == n1) return 0; + + size_type i = 0; + + while (i < max) { + if (*s0 != *s1) return static_cast(*s0) - static_cast(*s1); + s0++; + s1++; + i++; + } + + if (i < n0) { return 1; } + if (i < n1) { return -1; } + + return 0; + } + + __device__ constexpr int operator<=>(string_view const& rhs) const { return compare(rhs); } + }; + + struct mutable_string_view { + static constexpr size_type const UNKNOWN_STRING_LENGTH{-1}; + static constexpr size_type const npos{-1}; + + char* _data = nullptr; + + size_type _bytes = 0; + + mutable size_type _length = UNKNOWN_STRING_LENGTH; + + constexpr mutable_string_view() = default; + + __device__ constexpr mutable_string_view(char* data, size_type bytes) + : _data{data}, _bytes{bytes} + { + } + + __device__ constexpr size_type size_bytes() const { return _bytes; } + + __device__ constexpr auto* data() const { return _data; } + + __device__ constexpr auto* begin() const { return _data; } + + __device__ constexpr auto* end() const { return _data + _bytes; } + + __device__ constexpr auto const* cbegin() const { return _data; } + + __device__ constexpr auto const* cend() const { return _data + _bytes; } + + __device__ constexpr bool empty() const { return _bytes == 0; } + + __device__ explicit operator string_view() const { return string_view{_data, _bytes, _length}; } + }; + + namespace operators { + + template + __device__ inline int abs(T* out, T const* a) + { + *out = (*a < 0) ? -*a : *a; + return 0; + } + + template + __device__ inline int abs(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + abs(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int add(T* out, T const* a, T const* b) + { + *out = (*a + *b); + return 0; + } + + template + __device__ inline int add(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + add(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int arccos(T* out, T const* a); + + template <> + __device__ inline int arccos(float32_t* out, float32_t const* a) + { + *out = ::acosf(*a); + return 0; + } + + template <> + __device__ inline int arccos(float64_t* out, float64_t const* a) + { + *out = ::acos(*a); + return 0; + } + + template + __device__ inline int arccos(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + arccos(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int arccosh(T* out, T const* a); + + template <> + __device__ inline int arccosh(float32_t* out, float32_t const* a) + { + *out = ::acoshf(*a); + return 0; + } + + template <> + __device__ inline int arccosh(float64_t* out, float64_t const* a) + { + *out = ::acosh(*a); + return 0; + } + + template + __device__ inline int arccosh(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + arccosh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int arcsin(T* out, T const* a); + + template <> + __device__ inline int arcsin(float32_t* out, float32_t const* a) + { + *out = ::asinf(*a); + return 0; + } + + template <> + __device__ inline int arcsin(float64_t* out, float64_t const* a) + { + *out = ::asin(*a); + return 0; + } + + template + __device__ inline int arcsin(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + arcsin(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int arcsinh(T* out, T const* a); + + template <> + __device__ inline int arcsinh(float32_t* out, float32_t const* a) + { + *out = ::asinhf(*a); + return 0; + } + + template <> + __device__ inline int arcsinh(float64_t* out, float64_t const* a) + { + *out = ::asinh(*a); + return 0; + } + + template + __device__ inline int arcsinh(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + arcsinh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int arctan(T* out, T const* a); + + template <> + __device__ inline int arctan(float32_t* out, float32_t const* a) + { + *out = ::atanf(*a); + return 0; + } + + template <> + __device__ inline int arctan(float64_t* out, float64_t const* a) + { + *out = ::atan(*a); + return 0; + } + + template + __device__ inline int arctan(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + arctan(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int arctanh(T* out, T const* a); + + template <> + __device__ inline int arctanh(float32_t* out, float32_t const* a) + { + *out = ::atanhf(*a); + return 0; + } + + template <> + __device__ inline int arctanh(float64_t* out, float64_t const* a) + { + *out = ::atanh(*a); + return 0; + } + + template + __device__ inline int arctanh(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + arctanh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int bit_and(T* out, T const* a, T const* b) + { + *out = (*a & *b); + return 0; + } + + template + __device__ inline int bit_and(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + bit_and(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int bit_invert(T* out, T const* a) + { + *out = ~(*a); + return 0; + } + + template + __device__ inline int bit_invert(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + bit_invert(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int bit_or(T* out, T const* a, T const* b) + { + *out = (*a | *b); + return 0; + } + + template + __device__ inline int bit_or(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + bit_or(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int bit_xor(T* out, T const* a, T const* b) + { + *out = (*a ^ *b); + return 0; + } + + template + __device__ inline int bit_xor(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + bit_xor(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int cast_to_float64(float64_t* out, T const* a) + { + *out = static_cast(*a); + return 0; + } + + template + __device__ inline int cast_to_float64(optional* out, optional const* a) + { + if (a->is_valid()) { + float64_t r; + cast_to_float64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int cast_to_int64(int64_t* out, T const* a) + { + *out = static_cast(*a); + return 0; + } + + template + __device__ inline int cast_to_int64(optional* out, optional const* a) + { + if (a->is_valid()) { + int64_t r; + cast_to_int64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int cast_to_uint64(uint64_t* out, T const* a) + { + *out = static_cast(*a); + return 0; + } + + template + __device__ inline int cast_to_uint64(optional* out, optional const* a) + { + if (a->is_valid()) { + uint64_t r; + cast_to_uint64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int cbrt(T* out, T const* a); + + template <> + __device__ inline int cbrt(float32_t* out, float32_t const* a) + { + *out = ::cbrtf(*a); + return 0; + } + + template <> + __device__ inline int cbrt(float64_t* out, float64_t const* a) + { + *out = ::cbrt(*a); + return 0; + } + + template + __device__ inline int cbrt(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + cbrt(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int ceil(T* out, T const* a); + + template <> + __device__ inline int ceil(float32_t* out, float32_t const* a) + { + *out = ::ceilf(*a); + return 0; + } + + template <> + __device__ inline int ceil(float64_t* out, float64_t const* a) + { + *out = ::ceil(*a); + return 0; + } + + template + __device__ inline int ceil(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + ceil(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int cos(T* out, T const* a); + + template <> + __device__ inline int cos(float32_t* out, float32_t const* a) + { + *out = ::cosf(*a); + return 0; + } + + template <> + __device__ inline int cos(float64_t* out, float64_t const* a) + { + *out = ::cos(*a); + return 0; + } + + template + __device__ inline int cos(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + cos(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int cosh(T* out, T const* a); + + template <> + __device__ inline int cosh(float32_t* out, float32_t const* a) + { + *out = ::coshf(*a); + return 0; + } + + template <> + __device__ inline int cosh(float64_t* out, float64_t const* a) + { + *out = ::cosh(*a); + return 0; + } + + template + __device__ inline int cosh(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + cosh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int div(T* out, T const* a, T const* b) + { + *out = (*a / *b); + return 0; + } + + template + __device__ inline int div(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + div(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int equal(bool* out, T const* a, T const* b) + { + *out = (*a == *b); + return 0; + } + + template + __device__ inline int equal(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + *out = (*a == *b); + } else if (a->is_null() && b->is_null()) { + *out = true; + } else { + *out = false; + } + return 0; + } + + template + __device__ inline int exp(T* out, T const* a); + + template <> + __device__ inline int exp(float32_t* out, float32_t const* a) + { + *out = ::expf(*a); + return 0; + } + + template <> + __device__ inline int exp(float64_t* out, float64_t const* a) + { + *out = ::exp(*a); + return 0; + } + + template + __device__ inline int exp(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + exp(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int floor(T* out, T const* a); + + template <> + __device__ inline int floor(float32_t* out, float32_t const* a) + { + *out = ::floorf(*a); + return 0; + } + + template <> + __device__ inline int floor(float64_t* out, float64_t const* a) + { + *out = ::floor(*a); + return 0; + } + + template + __device__ inline int floor(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + floor(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int greater(bool* out, T const* a, T const* b) + { + *out = (*a > *b); + return 0; + } + + template + __device__ inline int greater(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + *out = (*a > *b); + } else { + *out = false; + } + return 0; + } + + template + __device__ inline int greater_equal(bool* out, T const* a, T const* b) + { + *out = (*a >= *b); + return 0; + } + + template + __device__ inline int greater_equal(optional* out, + optional const* a, + optional const* b) + { + if (a->is_valid() && b->is_valid()) { + *out = (*a >= *b); + } else { + *out = false; + } + return 0; + } + + template + __device__ inline int identity(T* out, T const* a) + { + *out = *a; + return 0; + } + + template + __device__ inline int identity(optional* out, optional const* a) + { + *out = *a; + return 0; + } + + template + __device__ inline int is_null(bool* out, T const* a) + { + *out = false; + return 0; + } + + template + __device__ inline int is_null(optional* out, optional const* a) + { + *out = a->is_null(); + return 0; + } + + template + __device__ inline int less(bool* out, T const* a, T const* b) + { + *out = (*a < *b); + return 0; + } + + template + __device__ inline int less(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + *out = (*a < *b); + } else { + *out = false; + } + return 0; + } + + template + __device__ inline int less_equal(bool* out, T const* a, T const* b) + { + *out = (*a <= *b); + return 0; + } + + template + __device__ inline int less_equal(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + *out = (*a <= *b); + } else { + *out = false; + } + return 0; + } + + template + __device__ inline int log(T* out, T const* a); + + template <> + __device__ inline int log(float32_t* out, float32_t const* a) + { + *out = ::logf(*a); + return 0; + } + + template <> + __device__ inline int log(float64_t* out, float64_t const* a) + { + *out = ::log(*a); + return 0; + } + + template + __device__ inline int log(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + log(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int logical_and(T* out, T const* a, T const* b) + { + *out = (*a && *b); + return 0; + } + + template + __device__ inline int logical_and(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + logical_and(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int logical_or(T* out, T const* a, T const* b) + { + *out = (*a || *b); + return 0; + } + + template + __device__ inline int logical_or(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + logical_or(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int mod(T* out, T const* a, T const* b) + { + *out = (*a % *b); + return 0; + } + + __device__ inline int mod(float32_t* out, float32_t const* a, float32_t const* b) + { + *out = ::fmodf(*a, *b); + return 0; + } + + __device__ inline int mod(float64_t* out, float64_t const* a, float64_t const* b) + { + *out = ::fmod(*a, *b); + return 0; + } + + template + __device__ inline int mod(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + mod(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int mul(T* out, T const* a, T const* b) + { + *out = (*a * *b); + return 0; + } + + template + __device__ inline int mul(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + mul(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int null_equal(bool* out, T const* a, T const* b) + { + *out = (*a == *b); + return 0; + } + + template + __device__ inline int null_equal(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + *out = (*a == *b); + } else if (a->is_null() && b->is_null()) { + *out = true; + } else { + *out = false; + } + return 0; + } + + template + __device__ inline int null_logical_and(T* out, T const* a, T const* b) + { + *out = (*a && *b); + return 0; + } + + template + __device__ inline int null_logical_and(optional* out, + optional const* a, + optional const* b) + { + if (a->is_valid() && b->is_valid()) { + *out = (*a && *b); + } else if (a->is_null() && b->is_null()) { + *out = nullopt; + } else { + if (a->is_valid() ? *a : *b) { + *out = nullopt; + } else { + *out = false; + } + } + return 0; + } + + template + __device__ inline int null_logical_or(T* out, T const* a, T const* b) + { + *out = (*a || *b); + return 0; + } + + template + __device__ inline int null_logical_or(optional* out, + optional const* a, + optional const* b) + { + if (a->is_valid() && b->is_valid()) { + *out = (*a || *b); + } else if (a->is_null() && b->is_null()) { + *out = nullopt; + } else { + if (a->is_valid() ? *a : *b) { + *out = true; + } else { + *out = nullopt; + } + } + return 0; + } + + template + __device__ inline int pow(T* out, T const* a, T const* b); + + template <> + __device__ inline int pow(float32_t* out, float32_t const* a, float32_t const* b) + { + *out = ::powf(*a, *b); + return 0; + } + + template <> + __device__ inline int pow(float64_t* out, float64_t const* a, float64_t const* b) + { + *out = ::pow(*a, *b); + return 0; + } + + template + __device__ inline int pow(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + pow(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int pymod(T* out, T const* a, T const* b) + { + *out = (*a % *b + *b) % *b; + return 0; + } + + template <> + __device__ inline int pymod(float32_t* out, float32_t const* a, float32_t const* b) + { + *out = ::fmodf(::fmodf(*a, *b) + *b, *b); + return 0; + } + + template <> + __device__ inline int pymod(float64_t* out, float64_t const* a, float64_t const* b) + { + *out = ::fmod(::fmod(*a, *b) + *b, *b); + return 0; + } + + template + __device__ inline int pymod(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + pymod(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int rint(T* out, T const* a); + + template <> + __device__ inline int rint(float32_t* out, float32_t const* a) + { + *out = ::rintf(*a); + return 0; + } + + template <> + __device__ inline int rint(float64_t* out, float64_t const* a) + { + *out = ::rint(*a); + return 0; + } + + template + __device__ inline int rint(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + rint(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int sin(T* out, T const* a); + + template <> + __device__ inline int sin(float32_t* out, float32_t const* a) + { + *out = ::sinf(*a); + return 0; + } + + template <> + __device__ inline int sin(float64_t* out, float64_t const* a) + { + *out = ::sin(*a); + return 0; + } + + template + __device__ inline int sin(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + sin(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int sinh(T* out, T const* a); + + template <> + __device__ inline int sinh(float32_t* out, float32_t const* a) + { + *out = ::sinhf(*a); + return 0; + } + + template <> + __device__ inline int sinh(float64_t* out, float64_t const* a) + { + *out = ::sinh(*a); + return 0; + } + + template + __device__ inline int sinh(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + sinh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int sub(T* out, T const* a, T const* b) + { + *out = *a - *b; + return 0; + } + + template + __device__ inline int sub(optional* out, optional const* a, optional const* b) + { + if (a->is_valid() && b->is_valid()) { + T r; + sub(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int tanh(T* out, T const* a); + + template <> + __device__ inline int tanh(float32_t* out, float32_t const* a) + { + *out = ::tanhf(*a); + return 0; + } + + template <> + __device__ inline int tanh(float64_t* out, float64_t const* a) + { + *out = ::tanh(*a); + return 0; + } + + template + __device__ inline int tanh(optional* out, optional const* a) + { + if (a->is_valid()) { + T r; + tanh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int if_else(T* out, + bool const* condition, + T const* true_value, + T const* false_value) + { + *out = *condition ? *true_value : *false_value; + return 0; + } + + template + __device__ inline int if_else(optional* out, + optional const* condition, + optional const* true_value, + optional const* false_value) + { + if (condition->is_valid() && true_value->is_valid() && false_value->is_valid()) { + if_else(&out->value(), &condition->value(), &true_value->value(), &false_value->value()); + } else { + *out = nullopt; + } + return 0; + } + + } // namespace operators + } // namespace lite +} // namespace cudf diff --git a/cpp/src/jit/lite/ops.cu b/cpp/src/jit/lite/ops.cu new file mode 100644 index 000000000000..813ca00215d7 --- /dev/null +++ b/cpp/src/jit/lite/ops.cu @@ -0,0 +1,34 @@ +#include + +namespace __attribute__((visibility("default"))) cudf +{ + namespace lite { + + // TODO: pre-instantiate some of the operators so they can be used in LTO + +#define DEFINE_UNARY_OP(NAME, RET_TYPE, TYPE, TYPE_TAG) \ + extern "C" __device__ int cudf_##NAME##__##TYPE_TAG(RET_TYPE* out, TYPE const* a) \ + { \ + return operators::NAME(out, a); \ + } + +#define DEFINE_BINARY_OP(NAME, RET_TYPE, TYPE, TYPE_TAG) \ + extern "C" __device__ int cudf_##NAME##__##TYPE_TAG(RET_TYPE* out, TYPE const* a, TYPE const* b) \ + { \ + return operators::NAME(out, a, b); \ + } + + DEFINE_UNARY_OP(abs, int8_t, int8_t, i8) + DEFINE_UNARY_OP(abs, optional, optional, i8_opt) + DEFINE_UNARY_OP(abs, int16_t, int16_t, i16) + DEFINE_UNARY_OP(abs, optional, optional, i16_opt) + DEFINE_UNARY_OP(abs, int32_t, int32_t, i32) + DEFINE_UNARY_OP(abs, optional, optional, i32_opt) + DEFINE_UNARY_OP(abs, int64_t, int64_t, i64) + DEFINE_UNARY_OP(abs, optional, optional, i64_opt) + + + + + } // namespace lite +} // namespace cudf diff --git a/cpp/src/transform/jit/compute_columns_kernel.cu b/cpp/src/transform/jit/compute_columns_kernel.cu new file mode 100644 index 000000000000..8fd0d4ab17ef --- /dev/null +++ b/cpp/src/transform/jit/compute_columns_kernel.cu @@ -0,0 +1,170 @@ + + +#include "jit/element.cuh" +#include "jit/element_storage.cuh" + +#include +#include +#include + +namespace cudf { + +extern "C" __device__ int operation(void* user_data, + int element_index, + void const* const* inputs, + void* const* outputs) +{ + // element_storage tmp0; + // add(&tmp0, inputs[0], inputs[1]); + // outputs[0] = tmp0; + + // knows the types + // cast to the target type + // load the element + // perform the operation + // use C++ AST to codegen + // store the element + // + // + // + // + // + // pre-compile some of these operators for common operations and types; compiling the operator + // becomes trivial as we only need to forward declare and then link to the pre-compiled fragments + // + // for example, we can pre-compile for all lexicographic comparisons cheaply + // + // + // + // WE ALSO NEED TO MAKE SURE THE ORDER OF INPUTS AND OUTPUTS doesnt affect the applicability of + // the generic operators + // + // + // + // TODO: functions to add fragments + // + // + return 0; +} + +// generic_column_operator: catch-all +// specific column_operator + +// unary and binary operators can be easily JIT-ed for all types and use JIT-ed operators +// +// the register pressure and compile-time of the generic operator is very high +// +// use PTX to specify the operator to reduce compile time +// +// +// +// should we pre-link some of the kernels to reduce work at JIT-time? + +/// @brief The generic transform kernel. Supports all types and nullability combinations. +extern "C" __global__ void compute_columns_kernel( + size_type row_size, + bitmask_type const* __restrict__ stencil, + bool stencil_has_nulls, + void* __restrict__ user_data, + column_device_view const* __restrict__ input_cols, + size_type num_inputs, + mutable_column_device_view const* __restrict__ output_cols, + size_type num_outputs, + bool const* is_scalar) +{ + static constexpr int MAX_INPUTS = 32; + static constexpr int MAX_OUTPUTS = 16; + + element_storage input_storage[MAX_INPUTS]; + void const* input_ptrs[MAX_INPUTS]; + element_storage output_storage[MAX_OUTPUTS]; + void* output_ptrs[MAX_OUTPUTS]; + + for (int i = 0; i < MAX_INPUTS; ++i) { + input_ptrs[i] = &input_storage[i]; + } + for (int i = 0; i < MAX_OUTPUTS; ++i) { + output_ptrs[i] = &output_storage[i]; + } + + auto start = detail::grid_1d::global_thread_id(); + auto stride = detail::grid_1d::grid_stride(); + + for (auto element_idx = start; element_idx < row_size; element_idx += stride) { + auto active_mask = __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); + + for (int i = 0; i < num_inputs; i++) { + load_element(is_scalar[i], input_cols + i, element_idx, input_storage + i); + } + + operation(user_data, element_idx, input_ptrs, output_ptrs); + + for (int i = 0; i < num_inputs; i++) { + store_element(output_cols, output_storage + i, element_idx, active_mask); + } + + if constexpr (is_null_aware == null_aware::NO) { + if (stencil_has_nulls && !bit_is_set(stencil, element_idx)) { continue; } + + auto outs = OutputAccessors::map([&]() { + return cuda::std::tuple{A::output_arg(output_cols, element_idx)...}; + }); + + auto out_ptrs = + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); + + auto inputs = InputAccessors::map( + [&]() { return cuda::std::tuple{A::element(input_cols, element_idx)...}; }); + + if constexpr (has_user_data) { + auto args = + cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); + cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); + + } else { + // TODO: static assert invocable + auto args = cuda::std::tuple_cat(out_ptrs, inputs); + cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); + } + + OutputAccessors::map([&]() { + (A::assign(output_cols, element_idx, cuda::std::get(outs)), ...); + }); + } else { + bool is_valid[OutputAccessors::size]; + + auto outs = OutputAccessors::map([&]() { + return cuda::std::tuple{A::null_output_arg(output_cols, element_idx)...}; + }); + + auto out_ptrs = + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); + + auto inputs = InputAccessors::map([&]() { + return cuda::std::tuple{A::nullable_element(input_cols, element_idx)...}; + }); + + if constexpr (has_user_data) { + auto args = + cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); + cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); + + } else { + auto args = cuda::std::tuple_cat(out_ptrs, inputs); + cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); + } + + OutputAccessors::map([&]() { + (A::assign(output_cols, element_idx, *cuda::std::get(outs)), ...); + ((is_valid[A::index] = cuda::std::get(outs).has_value()), ...); + }); + + OutputAccessors::map([&]() { + auto active_mask = __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); + (warp_compact_validity(active_mask, output_cols, element_idx, is_valid[A::index]), ...); + }); + } + } +} + +} // namespace cudf \ No newline at end of file diff --git a/todo.md b/todo.md new file mode 100644 index 000000000000..db578ce39d2b --- /dev/null +++ b/todo.md @@ -0,0 +1,17 @@ +Add POC: + - Successfully compile existing CUDF JIT kernels whilst reducing the binary size of the JIT kernels + - Precompiled and JIT-ed compute_columns_kernel; reduce register pressure and ...: + - Make AST codegen to use the pre-compiled operators and element_storage types + - Manage the AOT-compiled artifacts with keys for the system + - Functions to compile the artifacts and embed them + + + + + + + + + + + From 0ec68042d785f55880a00b8983c6ae3596e71b52 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 20 Apr 2026 12:35:56 +0000 Subject: [PATCH 142/254] Refactor JIT compilation and embedding process - Removed the `ops.cu` file as part of the JIT operator refactoring. - Updated `sync.cuh` to reintroduce the `jit` namespace for better organization. - Renamed and refactored CMake functions for embedding includes and blobs, improving clarity and functionality. - Introduced a new `embed.hpp` header for managing embedded file operations, including loading, compressing, and generating C++ source files. - Updated `embed.in.cpp` to utilize the new embedding functions and improved structure for better maintainability. - Refactored `compute_columns_kernel.cu` to streamline the compute operation and reduce register pressure, enhancing performance. - Updated `todo.md` to reflect progress on JIT kernel compilation and embedding tasks. --- cpp/CMakeLists.txt | 130 +++- cpp/include/cudf/utilities/export.hpp | 6 +- cpp/include/cudf/utilities/span.hpp | 5 +- cpp/src/jit/element.cuh | 185 +++--- cpp/src/jit/element_storage.cuh | 16 +- cpp/src/jit/lite/lite.cuh | 187 +++++- cpp/src/jit/lite/operators.cu | 602 ++++++++++++++++++ cpp/src/jit/lite/ops.cu | 34 - cpp/src/jit/sync.cuh | 3 +- cpp/src/librtcx/embed.cmake | 98 ++- cpp/src/librtcx/embed.hpp | 341 ++++++++++ cpp/src/librtcx/embed.in.cpp | 348 +--------- .../transform/jit/compute_columns_kernel.cu | 220 +++---- todo.md | 21 +- 14 files changed, 1508 insertions(+), 688 deletions(-) create mode 100644 cpp/src/jit/lite/operators.cu delete mode 100644 cpp/src/jit/lite/ops.cu create mode 100644 cpp/src/librtcx/embed.hpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3ffc09c152db..e3169f38b582 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -326,37 +326,149 @@ endif() set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) -jit_add_include_directory( +set(CUDF_LTO_JIT_ARCHITECTURES "80;90;100;120") + +macro(add_fragment) + set(TARGET ${ARGV0}) + set(ONE_VALUE_ARGS FRAGMENT SOURCE) + set(MULTI_VALUE_ARGS "DEFINITIONS") + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + foreach(ARCH IN LISTS CUDF_LTO_JIT_ARCHITECTURES) + set(TARGET_ID "sm${ARCH}_${TARGET}_${ARG_FRAGMENT}") + add_library(${TARGET_ID} OBJECT ${ARG_SOURCE}) + target_compile_options( + ${TARGET_ID} PRIVATE --compress-mode=none --gen-opt-lto --relaxed-constexpr --extended-lambda + ) + target_compile_definitions(${TARGET_ID} PRIVATE CUDF_DISABLE_EXPORTS ${ARG_DEFINITIONS}) + set_target_properties( + ${TARGET_ID} + PROPERTIES CUDA_ARCHITECTURES "${ARCH}-real" + CUDA_SEPARABLE_COMPILATION ON + CUDA_FATBIN_COMPILATION ON + POSITION_INDEPENDENT_CODE ON + INTERPROCEDURAL_OPTIMIZATION ON + CXX_STANDARD ${CUDF_CUDA_STANDARD} + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS ON + CXX_VISIBILITY_PRESET hidden + CUDA_STANDARD ${CUDF_CUDA_STANDARD} + CUDA_STANDARD_REQUIRED ON + CUDA_VISIBILITY_PRESET hidden + ) + target_link_libraries(${TARGET_ID} PRIVATE CCCL::CCCL) + target_include_directories( + ${TARGET_ID} PRIVATE "$" + "$" + ) + + embed_blob( + ${TARGET} FILE $ DEST + fragments/sm_${ARCH}_${ARG_FRAGMENT}.fatbin + ) + endforeach() + +endmacro() + +add_fragment( + cudf_jit_embed + FRAGMENT + compute_columns_kernel_hasnonulls_32 + SOURCE + src/transform/jit/compute_columns_kernel.cu + DEFINITIONS + COMPUTE_COLUMNS_HAS_NULLS=0 + COMPUTE_COLUMNS_MAX_ELEMENT_SIZE=4 +) + +add_fragment( + cudf_jit_embed + FRAGMENT + compute_columns_kernel_hasnulls_32 + SOURCE + src/transform/jit/compute_columns_kernel.cu + DEFINITIONS + COMPUTE_COLUMNS_HAS_NULLS=1 + COMPUTE_COLUMNS_MAX_ELEMENT_SIZE=4 +) + +add_fragment( + cudf_jit_embed + FRAGMENT + compute_columns_kernel_hasnonulls_64 + SOURCE + src/transform/jit/compute_columns_kernel.cu + DEFINITIONS + COMPUTE_COLUMNS_HAS_NULLS=0 + COMPUTE_COLUMNS_MAX_ELEMENT_SIZE=8 +) + +add_fragment( + cudf_jit_embed + FRAGMENT + compute_columns_kernel_hasnulls_64 + SOURCE + src/transform/jit/compute_columns_kernel.cu + DEFINITIONS + COMPUTE_COLUMNS_HAS_NULLS=1 + COMPUTE_COLUMNS_MAX_ELEMENT_SIZE=8 +) + +add_fragment( + cudf_jit_embed + FRAGMENT + compute_columns_kernel_hasnonulls_128 + SOURCE + src/transform/jit/compute_columns_kernel.cu + DEFINITIONS + COMPUTE_COLUMNS_HAS_NULLS=0 + COMPUTE_COLUMNS_MAX_ELEMENT_SIZE=16 +) + +add_fragment( + cudf_jit_embed + FRAGMENT + compute_columns_kernel_hasnulls_128 + SOURCE + src/transform/jit/compute_columns_kernel.cu + DEFINITIONS + COMPUTE_COLUMNS_HAS_NULLS=1 + COMPUTE_COLUMNS_MAX_ELEMENT_SIZE=16 +) + +add_fragment(cudf_jit_embed FRAGMENT operators SOURCE src/jit/lite/operators.cu) + +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/libcxx DEST_DIRECTORY librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf DEST_DIRECTORY cudf/cpp/include/cudf INCLUDE_DIRECTORIES cudf/cpp/include ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/jit DEST_DIRECTORY cudf/cpp/src/jit INCLUDE_DIRECTORIES cudf/cpp/src ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/binaryop/jit DEST_DIRECTORY cudf/cpp/src/binaryop/jit INCLUDE_DIRECTORIES cudf/cpp/src ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/join/jit DEST_DIRECTORY cudf/cpp/src/join/jit INCLUDE_DIRECTORIES cudf/cpp/src ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling DEST_DIRECTORY cudf/cpp/src/rolling INCLUDE_DIRECTORIES cudf/cpp/src ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/transform/jit DEST_DIRECTORY cudf/cpp/src/transform/jit INCLUDE_DIRECTORIES cudf/cpp/src ) @@ -366,13 +478,13 @@ get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLU foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) cmake_path(GET INC_DIR FILENAME INC_DIR_NAME) - jit_add_include_directory( + embed_includes( cudf_jit_embed COPY_DIRECTORY ${INC_DIR} DEST_DIRECTORY CCCL/libcudacxx/${INC_DIR_NAME} INCLUDE_DIRECTORIES CCCL/libcudacxx/${INC_DIR_NAME} ) endforeach() -jit_embed(cudf_jit_embed COMPRESSION zstd) +embed(cudf_jit_embed COMPRESSION zstd) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- diff --git a/cpp/include/cudf/utilities/export.hpp b/cpp/include/cudf/utilities/export.hpp index 815d3110f2cb..f3821a50dfc8 100644 --- a/cpp/include/cudf/utilities/export.hpp +++ b/cpp/include/cudf/utilities/export.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,7 +7,11 @@ // Macros used for defining symbol visibility, only GLIBC is supported #if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) +#ifndef CUDF_DISABLE_EXPORTS #define CUDF_EXPORT __attribute__((visibility("default"))) +#else +#define CUDF_EXPORT __attribute__((visibility("hidden"))) +#endif #define CUDF_HIDDEN __attribute__((visibility("hidden"))) #else #define CUDF_EXPORT diff --git a/cpp/include/cudf/utilities/span.hpp b/cpp/include/cudf/utilities/span.hpp index 1eecb1b51f9f..d79711e8f7e4 100644 --- a/cpp/include/cudf/utilities/span.hpp +++ b/cpp/include/cudf/utilities/span.hpp @@ -69,13 +69,14 @@ class span_base { */ CUDF_HOST_DEVICE constexpr span_base(pointer data, size_type size) : _data(data), _size(size) {} // constexpr span_base(pointer begin, pointer end) : _data(begin), _size(end - begin) {} - CUDF_HOST_DEVICE constexpr span_base(span_base const&) noexcept = default; ///< Copy constructor + constexpr span_base(span_base const&) noexcept = default; ///< Copy constructor + /** * @brief Copy assignment operator. * * @return Reference to this span. */ - CUDF_HOST_DEVICE constexpr span_base& operator=(span_base const&) noexcept = default; + constexpr span_base& operator=(span_base const&) noexcept = default; /** * @brief Returns an iterator to the first element of the span. diff --git a/cpp/src/jit/element.cuh b/cpp/src/jit/element.cuh index 311184931f20..361b3f89cc26 100644 --- a/cpp/src/jit/element.cuh +++ b/cpp/src/jit/element.cuh @@ -1,3 +1,7 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ #pragma once #include @@ -6,112 +10,115 @@ #include #include +#include namespace cudf { -__device__ void load_element(bool is_scalar, - column_device_view const* column, - bitmask_type const* __restrict__ null_mask, +template +inline constexpr bool storage_compatible = + sizeof(ElementType) <= sizeof(Storage) && alignof(ElementType) <= alignof(Storage); + +template +__device__ void load_element(column_device_view const* column, size_type element_index, - element_storage* storage) + element_storage_t* storage) { - auto index = is_scalar ? 0 : element_index; - -#define CUDF_CASE(TYPE_ID, type_name) \ - case type_id::TYPE_ID: { \ - static_assert(sizeof(type_name) <= sizeof(element_storage)); \ - static_assert(alignof(type_name) <= alignof(element_storage)); \ - static_assert(sizeof(cuda::std::optional) <= sizeof(element_storage)); \ - static_assert(alignof(cuda::std::optional) <= alignof(element_storage)); \ - \ - if (!column->nullable()) { \ - auto v = column->template element(index); \ - *reinterpret_cast(storage->data) = v; \ - } else { \ - auto v = column->template nullable_element(index); \ - *reinterpret_cast*>(storage->data) = v; \ - } \ - } break; + auto op = [&] __device__() { + if constexpr (!has_nulls) { + if constexpr (storage_compatible) { + auto v = column->template element(element_index); + *reinterpret_cast(storage->data) = v; + } + } else { + if constexpr (storage_compatible>) { + auto v = column->template nullable_element(element_index); + *reinterpret_cast*>(storage->data) = v; + } + } + }; switch (column->type().id()) { - CUDF_CASE(INT8, int8_t) - CUDF_CASE(INT16, int16_t) - CUDF_CASE(INT32, int32_t) - CUDF_CASE(INT64, int64_t) - CUDF_CASE(UINT8, uint8_t) - CUDF_CASE(UINT16, uint16_t) - CUDF_CASE(UINT32, uint32_t) - CUDF_CASE(UINT64, uint64_t) - CUDF_CASE(FLOAT32, float) - CUDF_CASE(FLOAT64, double) - CUDF_CASE(BOOL8, bool) - CUDF_CASE(DECIMAL32, numeric::decimal32) - CUDF_CASE(DECIMAL64, numeric::decimal64) - CUDF_CASE(DECIMAL128, numeric::decimal128) - CUDF_CASE(TIMESTAMP_DAYS, cudf::timestamp_D) - CUDF_CASE(TIMESTAMP_SECONDS, cudf::timestamp_s) - CUDF_CASE(TIMESTAMP_MILLISECONDS, cudf::timestamp_ms) - CUDF_CASE(TIMESTAMP_MICROSECONDS, cudf::timestamp_us) - CUDF_CASE(TIMESTAMP_NANOSECONDS, cudf::timestamp_ns) - CUDF_CASE(DURATION_DAYS, cudf::duration_D) - CUDF_CASE(DURATION_SECONDS, cudf::duration_s) - CUDF_CASE(DURATION_MILLISECONDS, cudf::duration_ms) - CUDF_CASE(DURATION_MICROSECONDS, cudf::duration_us) - CUDF_CASE(DURATION_NANOSECONDS, cudf::duration_ns) - CUDF_CASE(STRING, cudf::string_view) + case type_id::INT8: op.template operator()(); break; + case type_id::INT16: op.template operator()(); break; + case type_id::INT32: op.template operator()(); break; + case type_id::INT64: op.template operator()(); break; + case type_id::UINT8: op.template operator()(); break; + case type_id::UINT16: op.template operator()(); break; + case type_id::UINT32: op.template operator()(); break; + case type_id::UINT64: op.template operator()(); break; + case type_id::FLOAT32: op.template operator()(); break; + case type_id::FLOAT64: op.template operator()(); break; + case type_id::BOOL8: op.template operator()(); break; + case type_id::DECIMAL32: op.template operator()(); break; + case type_id::DECIMAL64: op.template operator()(); break; + case type_id::DECIMAL128: op.template operator()(); break; + case type_id::TIMESTAMP_DAYS: op.template operator()(); break; + case type_id::TIMESTAMP_SECONDS: op.template operator()(); break; + case type_id::TIMESTAMP_MILLISECONDS: op.template operator()(); break; + case type_id::TIMESTAMP_MICROSECONDS: op.template operator()(); break; + case type_id::TIMESTAMP_NANOSECONDS: op.template operator()(); break; + case type_id::DURATION_DAYS: op.template operator()(); break; + case type_id::DURATION_SECONDS: op.template operator()(); break; + case type_id::DURATION_MILLISECONDS: op.template operator()(); break; + case type_id::DURATION_MICROSECONDS: op.template operator()(); break; + case type_id::DURATION_NANOSECONDS: op.template operator()(); break; + case type_id::STRING: op.template operator()(); break; default: CUDF_UNREACHABLE(); } - -#undef CUDF_CASE } +template __device__ void store_element(mutable_column_device_view const* column, - element_storage const* storage, + element_storage_t const* storage, size_type element_index, unsigned int active_mask) { -// TODO: compact validity into a bitmask -// (warp_compact_validity(active_mask, output_cols, element_idx, is_valid[A::index]), ...); -#define CUDF_CASE(TYPE_ID, type_name) \ - case type_id::TYPE_ID: { \ - static_assert(sizeof(type_name) <= sizeof(element_storage)); \ - static_assert(alignof(type_name) <= alignof(element_storage)); \ - static_assert(sizeof(cuda::std::optional) <= sizeof(element_storage)); \ - static_assert(alignof(cuda::std::optional) <= alignof(element_storage)); \ - \ - if (!column->nullable()) { \ - auto v = *reinterpret_cast(storage->data); \ - column->template assign(element_index, v); \ - } else { \ - auto v = *reinterpret_cast const*>(storage->data); \ - column->template assign(element_index, *v); \ - } \ - } break; + auto op = [&] __device__() { + if constexpr (!has_nulls) { + if constexpr (storage_compatible) { + auto v = *reinterpret_cast(storage->data); + column->template assign(element_index, v); + } + } else { + if constexpr (storage_compatible>) { + auto v = *reinterpret_cast const*>(storage->data); + column->template assign(element_index, *v); + + auto null_word = __ballot_sync(active_mask, v.has_value()); + if (column->nullable()) { + if (warp_elect(active_mask)) { + column->null_mask()[element_index / detail::warp_size] = null_word; + } + } + } + } + }; switch (column->type().id()) { - CUDF_CASE(INT16, int16_t) - CUDF_CASE(INT32, int32_t) - CUDF_CASE(INT64, int64_t) - CUDF_CASE(UINT8, uint8_t) - CUDF_CASE(UINT16, uint16_t) - CUDF_CASE(UINT32, uint32_t) - CUDF_CASE(UINT64, uint64_t) - CUDF_CASE(FLOAT32, float) - CUDF_CASE(FLOAT64, double) - CUDF_CASE(BOOL8, bool) - CUDF_CASE(DECIMAL32, numeric::decimal32) - CUDF_CASE(DECIMAL64, numeric::decimal64) - CUDF_CASE(DECIMAL128, numeric::decimal128) - CUDF_CASE(TIMESTAMP_DAYS, cudf::timestamp_D) - CUDF_CASE(TIMESTAMP_SECONDS, cudf::timestamp_s) - CUDF_CASE(TIMESTAMP_MILLISECONDS, cudf::timestamp_ms) - CUDF_CASE(TIMESTAMP_MICROSECONDS, cudf::timestamp_us) - CUDF_CASE(TIMESTAMP_NANOSECONDS, cudf::timestamp_ns) - CUDF_CASE(DURATION_DAYS, cudf::duration_D) - CUDF_CASE(DURATION_SECONDS, cudf::duration_s) - CUDF_CASE(DURATION_MILLISECONDS, cudf::duration_ms) - CUDF_CASE(DURATION_MICROSECONDS, cudf::duration_us) - CUDF_CASE(DURATION_NANOSECONDS, cudf::duration_ns) + case type_id::INT8: op.template operator()(); break; + case type_id::INT16: op.template operator()(); break; + case type_id::INT32: op.template operator()(); break; + case type_id::INT64: op.template operator()(); break; + case type_id::UINT8: op.template operator()(); break; + case type_id::UINT16: op.template operator()(); break; + case type_id::UINT32: op.template operator()(); break; + case type_id::UINT64: op.template operator()(); break; + case type_id::FLOAT32: op.template operator()(); break; + case type_id::FLOAT64: op.template operator()(); break; + case type_id::BOOL8: op.template operator()(); break; + case type_id::DECIMAL32: op.template operator()(); break; + case type_id::DECIMAL64: op.template operator()(); break; + case type_id::DECIMAL128: op.template operator()(); break; + case type_id::TIMESTAMP_DAYS: op.template operator()(); break; + case type_id::TIMESTAMP_SECONDS: op.template operator()(); break; + case type_id::TIMESTAMP_MILLISECONDS: op.template operator()(); break; + case type_id::TIMESTAMP_MICROSECONDS: op.template operator()(); break; + case type_id::TIMESTAMP_NANOSECONDS: op.template operator()(); break; + case type_id::DURATION_DAYS: op.template operator()(); break; + case type_id::DURATION_SECONDS: op.template operator()(); break; + case type_id::DURATION_MILLISECONDS: op.template operator()(); break; + case type_id::DURATION_MICROSECONDS: op.template operator()(); break; + case type_id::DURATION_NANOSECONDS: op.template operator()(); break; default: CUDF_UNREACHABLE(); } } diff --git a/cpp/src/jit/element_storage.cuh b/cpp/src/jit/element_storage.cuh index 9bbb76651e96..12bd920fe522 100644 --- a/cpp/src/jit/element_storage.cuh +++ b/cpp/src/jit/element_storage.cuh @@ -1,11 +1,23 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ #pragma once namespace cudf { -// can store any nullable or non-nullable element type +// can store any nullable or non-nullable element type. +template struct element_storage { - alignas(32) unsigned char data[64]; + alignas(max_element_size) unsigned char data[max_element_size * 2]; +}; + +// can store any non-nullable element type. +// +template +struct element_storage { + alignas(max_element_size) unsigned char data[max_element_size]; }; } // namespace cudf diff --git a/cpp/src/jit/lite/lite.cuh b/cpp/src/jit/lite/lite.cuh index 67432136816d..e46b82d0d4e2 100644 --- a/cpp/src/jit/lite/lite.cuh +++ b/cpp/src/jit/lite/lite.cuh @@ -142,11 +142,18 @@ namespace __attribute__((visibility("hidden"))) cudf return decimal{scaled, r, scale}; } - __device__ constexpr int operator<=>(decimal rhs) const + __device__ constexpr int compare(decimal rhs) const { auto scale = min(_scale, rhs._scale); return rescale(scale)._value - rhs.rescale(scale)._value; } + + __device__ constexpr bool operator==(decimal rhs) const { return compare(rhs) == 0; } + __device__ constexpr bool operator!=(decimal rhs) const { return compare(rhs) != 0; } + __device__ constexpr bool operator<(decimal rhs) const { return compare(rhs) < 0; } + __device__ constexpr bool operator<=(decimal rhs) const { return compare(rhs) <= 0; } + __device__ constexpr bool operator>(decimal rhs) const { return compare(rhs) > 0; } + __device__ constexpr bool operator>=(decimal rhs) const { return compare(rhs) >= 0; } }; using decimal32 = decimal; @@ -163,7 +170,14 @@ namespace __attribute__((visibility("hidden"))) cudf __device__ constexpr R count() const { return _rep; } - __device__ constexpr int operator<=>(timestamp rhs) const { return _rep - rhs._rep; } + __device__ constexpr int compare(timestamp rhs) const { return _rep - rhs._rep; } + + __device__ constexpr bool operator==(timestamp rhs) const { return compare(rhs) == 0; } + __device__ constexpr bool operator!=(timestamp rhs) const { return compare(rhs) != 0; } + __device__ constexpr bool operator<(timestamp rhs) const { return compare(rhs) < 0; } + __device__ constexpr bool operator<=(timestamp rhs) const { return compare(rhs) <= 0; } + __device__ constexpr bool operator>(timestamp rhs) const { return compare(rhs) > 0; } + __device__ constexpr bool operator>=(timestamp rhs) const { return compare(rhs) >= 0; } }; using timestamp_D = timestamp; @@ -192,7 +206,14 @@ namespace __attribute__((visibility("hidden"))) cudf return duration{_rep - rhs._rep}; } - __device__ constexpr int operator<=>(duration rhs) const { return _rep - rhs._rep; } + __device__ constexpr int compare(duration rhs) const { return _rep - rhs._rep; } + + __device__ constexpr bool operator==(duration rhs) const { return compare(rhs) == 0; } + __device__ constexpr bool operator!=(duration rhs) const { return compare(rhs) != 0; } + __device__ constexpr bool operator<(duration rhs) const { return compare(rhs) < 0; } + __device__ constexpr bool operator<=(duration rhs) const { return compare(rhs) <= 0; } + __device__ constexpr bool operator>(duration rhs) const { return compare(rhs) > 0; } + __device__ constexpr bool operator>=(duration rhs) const { return compare(rhs) >= 0; } }; using duration_D = duration; @@ -353,7 +374,12 @@ namespace __attribute__((visibility("hidden"))) cudf return 0; } - __device__ constexpr int operator<=>(string_view const& rhs) const { return compare(rhs); } + __device__ constexpr bool operator==(string_view const& rhs) const { return compare(rhs) == 0; } + __device__ constexpr bool operator!=(string_view const& rhs) const { return compare(rhs) != 0; } + __device__ constexpr bool operator<(string_view const& rhs) const { return compare(rhs) < 0; } + __device__ constexpr bool operator<=(string_view const& rhs) const { return compare(rhs) <= 0; } + __device__ constexpr bool operator>(string_view const& rhs) const { return compare(rhs) > 0; } + __device__ constexpr bool operator>=(string_view const& rhs) const { return compare(rhs) >= 0; } }; struct mutable_string_view { @@ -390,6 +416,37 @@ namespace __attribute__((visibility("hidden"))) cudf __device__ explicit operator string_view() const { return string_view{_data, _bytes, _length}; } }; + // Aliases for codegen + using b8 = bool; + using i8 = int8_t; + using i16 = int16_t; + using i32 = int32_t; + using i64 = int64_t; + using u8 = uint8_t; + using u16 = uint16_t; + using u32 = uint32_t; + using u64 = uint64_t; + using f32 = float; + using f64 = double; + using dec32 = decimal32; + using dec64 = decimal64; + using dec128 = decimal128; + using ts_D = timestamp_D; + using ts_h = timestamp_h; + using ts_m = timestamp_m; + using ts_s = timestamp_s; + using ts_ms = timestamp_ms; + using ts_us = timestamp_us; + using ts_ns = timestamp_ns; + using dur_D = duration_D; + using dur_h = duration_h; + using dur_m = duration_m; + using dur_s = duration_s; + using dur_ms = duration_ms; + using dur_us = duration_us; + using dur_ns = duration_ns; + using str = string_view; + namespace operators { template @@ -693,18 +750,18 @@ namespace __attribute__((visibility("hidden"))) cudf } template - __device__ inline int cast_to_float64(float64_t* out, T const* a) + __device__ inline int to_i32(int32_t* out, T const* a) { - *out = static_cast(*a); + *out = static_cast(*a); return 0; } template - __device__ inline int cast_to_float64(optional* out, optional const* a) + __device__ inline int to_i32(optional* out, optional const* a) { if (a->is_valid()) { - float64_t r; - cast_to_float64(&r, &a->value()); + int32_t r; + to_i32(&r, &a->value()); *out = r; } else { *out = nullopt; @@ -713,18 +770,18 @@ namespace __attribute__((visibility("hidden"))) cudf } template - __device__ inline int cast_to_int64(int64_t* out, T const* a) + __device__ inline int to_i64(int64_t* out, T const* a) { *out = static_cast(*a); return 0; } template - __device__ inline int cast_to_int64(optional* out, optional const* a) + __device__ inline int to_i64(optional* out, optional const* a) { if (a->is_valid()) { int64_t r; - cast_to_int64(&r, &a->value()); + to_i64(&r, &a->value()); *out = r; } else { *out = nullopt; @@ -733,18 +790,78 @@ namespace __attribute__((visibility("hidden"))) cudf } template - __device__ inline int cast_to_uint64(uint64_t* out, T const* a) + __device__ inline int to_u32(uint32_t* out, T const* a) + { + *out = static_cast(*a); + return 0; + } + + template + __device__ inline int to_u32(optional* out, optional const* a) + { + if (a->is_valid()) { + uint32_t r; + to_u32(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int to_u64(uint64_t* out, T const* a) { *out = static_cast(*a); return 0; } template - __device__ inline int cast_to_uint64(optional* out, optional const* a) + __device__ inline int to_u64(optional* out, optional const* a) { if (a->is_valid()) { uint64_t r; - cast_to_uint64(&r, &a->value()); + to_u64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int to_f32(float32_t* out, T const* a) + { + *out = static_cast(*a); + return 0; + } + + template + __device__ inline int to_f32(optional* out, optional const* a) + { + if (a->is_valid()) { + float32_t r; + to_f32(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return 0; + } + + template + __device__ inline int to_f64(float64_t* out, T const* a) + { + *out = static_cast(*a); + return 0; + } + + template + __device__ inline int to_f64(optional* out, optional const* a) + { + if (a->is_valid()) { + float64_t r; + to_f64(&r, &a->value()); *out = r; } else { *out = nullopt; @@ -903,7 +1020,9 @@ namespace __attribute__((visibility("hidden"))) cudf __device__ inline int equal(optional* out, optional const* a, optional const* b) { if (a->is_valid() && b->is_valid()) { - *out = (*a == *b); + bool r; + equal(&r, &a->value(), &b->value()); + *out = r; } else if (a->is_null() && b->is_null()) { *out = true; } else { @@ -983,7 +1102,9 @@ namespace __attribute__((visibility("hidden"))) cudf __device__ inline int greater(optional* out, optional const* a, optional const* b) { if (a->is_valid() && b->is_valid()) { - *out = (*a > *b); + bool r; + greater(&r, &a->value(), &b->value()); + *out = r; } else { *out = false; } @@ -1003,7 +1124,9 @@ namespace __attribute__((visibility("hidden"))) cudf optional const* b) { if (a->is_valid() && b->is_valid()) { - *out = (*a >= *b); + bool r; + greater_equal(&r, &a->value(), &b->value()); + *out = r; } else { *out = false; } @@ -1049,7 +1172,9 @@ namespace __attribute__((visibility("hidden"))) cudf __device__ inline int less(optional* out, optional const* a, optional const* b) { if (a->is_valid() && b->is_valid()) { - *out = (*a < *b); + bool r; + less(&r, &a->value(), &b->value()); + *out = r; } else { *out = false; } @@ -1067,7 +1192,9 @@ namespace __attribute__((visibility("hidden"))) cudf __device__ inline int less_equal(optional* out, optional const* a, optional const* b) { if (a->is_valid() && b->is_valid()) { - *out = (*a <= *b); + bool r; + less_equal(&r, &a->value(), &b->value()); + *out = r; } else { *out = false; } @@ -1151,13 +1278,15 @@ namespace __attribute__((visibility("hidden"))) cudf return 0; } - __device__ inline int mod(float32_t* out, float32_t const* a, float32_t const* b) + template <> + __device__ inline int mod(float32_t* out, float32_t const* a, float32_t const* b) { *out = ::fmodf(*a, *b); return 0; } - __device__ inline int mod(float64_t* out, float64_t const* a, float64_t const* b) + template <> + __device__ inline int mod(float64_t* out, float64_t const* a, float64_t const* b) { *out = ::fmod(*a, *b); return 0; @@ -1207,7 +1336,7 @@ namespace __attribute__((visibility("hidden"))) cudf __device__ inline int null_equal(optional* out, optional const* a, optional const* b) { if (a->is_valid() && b->is_valid()) { - *out = (*a == *b); + *out = (*(*a) == *(*b)); } else if (a->is_null() && b->is_null()) { *out = true; } else { @@ -1229,11 +1358,13 @@ namespace __attribute__((visibility("hidden"))) cudf optional const* b) { if (a->is_valid() && b->is_valid()) { - *out = (*a && *b); + bool r; + null_logical_and(&r, &a->value(), &b->value()); + *out = r; } else if (a->is_null() && b->is_null()) { *out = nullopt; } else { - if (a->is_valid() ? *a : *b) { + if (a->is_valid() ? *(*a) : *(*b)) { *out = nullopt; } else { *out = false; @@ -1255,11 +1386,13 @@ namespace __attribute__((visibility("hidden"))) cudf optional const* b) { if (a->is_valid() && b->is_valid()) { - *out = (*a || *b); + bool r; + null_logical_or(&r, &a->value(), &b->value()); + *out = r; } else if (a->is_null() && b->is_null()) { *out = nullopt; } else { - if (a->is_valid() ? *a : *b) { + if (a->is_valid() ? *(*a) : *(*b)) { *out = true; } else { *out = nullopt; diff --git a/cpp/src/jit/lite/operators.cu b/cpp/src/jit/lite/operators.cu new file mode 100644 index 000000000000..37736a062477 --- /dev/null +++ b/cpp/src/jit/lite/operators.cu @@ -0,0 +1,602 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +namespace __attribute__((visibility("default"))) cudf +{ + namespace lite { + + // Pre-instantiate operator entry points so they can be linked by symbol name. + +#define UNOP_R(NAME, RET_TYPE, TYPE) \ + extern "C" __device__ int NAME##__##TYPE(RET_TYPE* out, TYPE const* a) \ + { \ + return operators::NAME(out, a); \ + } \ + \ + extern "C" __device__ int NAME##__opt##TYPE(optional* out, optional const* a) \ + { \ + return operators::NAME(out, a); \ + } + +#define UNOP(NAME, TYPE) UNOP_R(NAME, TYPE, TYPE) + +#define UNPRED(NAME, TYPE) UNOP_R(NAME, bool, TYPE) + +#define CASTOP(NAME, TARGET, TYPE) UNOP_R(NAME, TARGET, TYPE) + +#define BINOP_R(NAME, RET_TYPE, TYPE) \ + extern "C" __device__ int NAME##__##TYPE(RET_TYPE* out, TYPE const* a, TYPE const* b) \ + { \ + return operators::NAME(out, a, b); \ + } \ + \ + extern "C" __device__ int NAME##__opt##TYPE( \ + optional* out, optional const* a, optional const* b) \ + { \ + return operators::NAME(out, a, b); \ + } + +#define BINOP(NAME, TYPE) BINOP_R(NAME, TYPE, TYPE) + +#define BINCMP(NAME, TYPE) BINOP_R(NAME, bool, TYPE) + +#define COND_OP_R(NAME, RET_TYPE, TYPE) \ + extern "C" __device__ int NAME##__##TYPE( \ + RET_TYPE* out, bool const* a, TYPE const* b, TYPE const* c) \ + { \ + return operators::NAME(out, a, b, c); \ + } \ + \ + extern "C" __device__ int NAME##__opt##TYPE(optional* out, \ + optional const* a, \ + optional const* b, \ + optional const* c) \ + { \ + return operators::NAME(out, a, b, c); \ + } + +#define COND_OP(NAME, TYPE) COND_OP_R(NAME, TYPE, TYPE) + + // Unary numeric + UNOP(abs, i8) + UNOP(abs, i16) + UNOP(abs, i32) + UNOP(abs, i64) + UNOP(abs, f32) + UNOP(abs, f64) + + // Unary floating point math + UNOP(arccos, f32) + UNOP(arccos, f64) + UNOP(arccosh, f32) + UNOP(arccosh, f64) + UNOP(arcsin, f32) + UNOP(arcsin, f64) + UNOP(arcsinh, f32) + UNOP(arcsinh, f64) + UNOP(arctan, f32) + UNOP(arctan, f64) + UNOP(arctanh, f32) + UNOP(arctanh, f64) + UNOP(cbrt, f32) + UNOP(cbrt, f64) + UNOP(ceil, f32) + UNOP(ceil, f64) + UNOP(cos, f32) + UNOP(cos, f64) + UNOP(cosh, f32) + UNOP(cosh, f64) + UNOP(exp, f32) + UNOP(exp, f64) + UNOP(floor, f32) + UNOP(floor, f64) + UNOP(log, f32) + UNOP(log, f64) + UNOP(rint, f32) + UNOP(rint, f64) + UNOP(sin, f32) + UNOP(sin, f64) + UNOP(sinh, f32) + UNOP(sinh, f64) + UNOP(tanh, f32) + UNOP(tanh, f64) + + // Unary bitwise and identity-like + UNOP(bit_invert, i8) + UNOP(bit_invert, i16) + UNOP(bit_invert, i32) + UNOP(bit_invert, i64) + UNOP(bit_invert, u8) + UNOP(bit_invert, u16) + UNOP(bit_invert, u32) + UNOP(bit_invert, u64) + + UNOP(identity, b8) + UNOP(identity, i8) + UNOP(identity, i16) + UNOP(identity, i32) + UNOP(identity, i64) + UNOP(identity, u8) + UNOP(identity, u16) + UNOP(identity, u32) + UNOP(identity, u64) + UNOP(identity, f32) + UNOP(identity, f64) + UNOP(identity, dec32) + UNOP(identity, dec64) + UNOP(identity, dec128) + UNOP(identity, ts_D) + UNOP(identity, ts_h) + UNOP(identity, ts_m) + UNOP(identity, ts_s) + UNOP(identity, ts_ms) + UNOP(identity, ts_us) + UNOP(identity, ts_ns) + UNOP(identity, dur_D) + UNOP(identity, dur_h) + UNOP(identity, dur_m) + UNOP(identity, dur_s) + UNOP(identity, dur_ms) + UNOP(identity, dur_us) + UNOP(identity, dur_ns) + UNOP(identity, str) + + UNPRED(is_null, b8) + UNPRED(is_null, i8) + UNPRED(is_null, i16) + UNPRED(is_null, i32) + UNPRED(is_null, i64) + UNPRED(is_null, u8) + UNPRED(is_null, u16) + UNPRED(is_null, u32) + UNPRED(is_null, u64) + UNPRED(is_null, f32) + UNPRED(is_null, f64) + UNPRED(is_null, dec32) + UNPRED(is_null, dec64) + UNPRED(is_null, dec128) + UNPRED(is_null, ts_D) + UNPRED(is_null, ts_h) + UNPRED(is_null, ts_m) + UNPRED(is_null, ts_s) + UNPRED(is_null, ts_ms) + UNPRED(is_null, ts_us) + UNPRED(is_null, ts_ns) + UNPRED(is_null, dur_D) + UNPRED(is_null, dur_h) + UNPRED(is_null, dur_m) + UNPRED(is_null, dur_s) + UNPRED(is_null, dur_ms) + UNPRED(is_null, dur_us) + UNPRED(is_null, dur_ns) + UNPRED(is_null, str) + + // Binary arithmetic + BINOP(add, i8) + BINOP(add, i16) + BINOP(add, i32) + BINOP(add, i64) + BINOP(add, u8) + BINOP(add, u16) + BINOP(add, u32) + BINOP(add, u64) + BINOP(add, f32) + BINOP(add, f64) + BINOP(add, dec32) + BINOP(add, dec64) + BINOP(add, dec128) + BINOP(add, dur_D) + BINOP(add, dur_h) + BINOP(add, dur_m) + BINOP(add, dur_s) + BINOP(add, dur_ms) + BINOP(add, dur_us) + BINOP(add, dur_ns) + + BINOP(sub, i8) + BINOP(sub, i16) + BINOP(sub, i32) + BINOP(sub, i64) + BINOP(sub, u8) + BINOP(sub, u16) + BINOP(sub, u32) + BINOP(sub, u64) + BINOP(sub, f32) + BINOP(sub, f64) + BINOP(sub, dec32) + BINOP(sub, dec64) + BINOP(sub, dec128) + BINOP(sub, dur_D) + BINOP(sub, dur_h) + BINOP(sub, dur_m) + BINOP(sub, dur_s) + BINOP(sub, dur_ms) + BINOP(sub, dur_us) + BINOP(sub, dur_ns) + + BINOP(mul, i8) + BINOP(mul, i16) + BINOP(mul, i32) + BINOP(mul, i64) + BINOP(mul, u8) + BINOP(mul, u16) + BINOP(mul, u32) + BINOP(mul, u64) + BINOP(mul, f32) + BINOP(mul, f64) + BINOP(mul, dec32) + BINOP(mul, dec64) + BINOP(mul, dec128) + + BINOP(div, i8) + BINOP(div, i16) + BINOP(div, i32) + BINOP(div, i64) + BINOP(div, u8) + BINOP(div, u16) + BINOP(div, u32) + BINOP(div, u64) + BINOP(div, f32) + BINOP(div, f64) + BINOP(div, dec32) + BINOP(div, dec64) + BINOP(div, dec128) + + BINOP(mod, i8) + BINOP(mod, i16) + BINOP(mod, i32) + BINOP(mod, i64) + BINOP(mod, u8) + BINOP(mod, u16) + BINOP(mod, u32) + BINOP(mod, u64) + BINOP(mod, f32) + BINOP(mod, f64) + BINOP(mod, dec32) + BINOP(mod, dec64) + BINOP(mod, dec128) + + BINOP(pymod, i8) + BINOP(pymod, i16) + BINOP(pymod, i32) + BINOP(pymod, i64) + BINOP(pymod, u8) + BINOP(pymod, u16) + BINOP(pymod, u32) + BINOP(pymod, u64) + BINOP(pymod, f32) + BINOP(pymod, f64) + + BINOP(pow, f32) + BINOP(pow, f64) + + // Binary bitwise/logical + BINOP(bit_and, i8) + BINOP(bit_and, i16) + BINOP(bit_and, i32) + BINOP(bit_and, i64) + BINOP(bit_and, u8) + BINOP(bit_and, u16) + BINOP(bit_and, u32) + BINOP(bit_and, u64) + + BINOP(bit_or, i8) + BINOP(bit_or, i16) + BINOP(bit_or, i32) + BINOP(bit_or, i64) + BINOP(bit_or, u8) + BINOP(bit_or, u16) + BINOP(bit_or, u32) + BINOP(bit_or, u64) + + BINOP(bit_xor, i8) + BINOP(bit_xor, i16) + BINOP(bit_xor, i32) + BINOP(bit_xor, i64) + BINOP(bit_xor, u8) + BINOP(bit_xor, u16) + BINOP(bit_xor, u32) + BINOP(bit_xor, u64) + + BINOP(logical_and, b8) + BINOP(logical_and, i8) + BINOP(logical_and, i16) + BINOP(logical_and, i32) + BINOP(logical_and, i64) + BINOP(logical_and, u8) + BINOP(logical_and, u16) + BINOP(logical_and, u32) + BINOP(logical_and, u64) + + // Binary predicates + BINCMP(equal, b8) + BINCMP(equal, i8) + BINCMP(equal, i16) + BINCMP(equal, i32) + BINCMP(equal, i64) + BINCMP(equal, u8) + BINCMP(equal, u16) + BINCMP(equal, u32) + BINCMP(equal, u64) + BINCMP(equal, f32) + BINCMP(equal, f64) + BINCMP(equal, dec32) + BINCMP(equal, dec64) + BINCMP(equal, dec128) + BINCMP(equal, ts_D) + BINCMP(equal, ts_h) + BINCMP(equal, ts_m) + BINCMP(equal, ts_s) + BINCMP(equal, ts_ms) + BINCMP(equal, ts_us) + BINCMP(equal, ts_ns) + BINCMP(equal, dur_D) + BINCMP(equal, dur_h) + BINCMP(equal, dur_m) + BINCMP(equal, dur_s) + BINCMP(equal, dur_ms) + BINCMP(equal, dur_us) + BINCMP(equal, dur_ns) + BINCMP(equal, str) + + BINCMP(null_equal, b8) + BINCMP(null_equal, i8) + BINCMP(null_equal, i16) + BINCMP(null_equal, i32) + BINCMP(null_equal, i64) + BINCMP(null_equal, u8) + BINCMP(null_equal, u16) + BINCMP(null_equal, u32) + BINCMP(null_equal, u64) + BINCMP(null_equal, f32) + BINCMP(null_equal, f64) + BINCMP(null_equal, dec32) + BINCMP(null_equal, dec64) + BINCMP(null_equal, dec128) + BINCMP(null_equal, ts_D) + BINCMP(null_equal, ts_h) + BINCMP(null_equal, ts_m) + BINCMP(null_equal, ts_s) + BINCMP(null_equal, ts_ms) + BINCMP(null_equal, ts_us) + BINCMP(null_equal, ts_ns) + BINCMP(null_equal, dur_D) + BINCMP(null_equal, dur_h) + BINCMP(null_equal, dur_m) + BINCMP(null_equal, dur_s) + BINCMP(null_equal, dur_ms) + BINCMP(null_equal, dur_us) + BINCMP(null_equal, dur_ns) + BINCMP(null_equal, str) + + BINCMP(less, b8) + BINCMP(less, i8) + BINCMP(less, i16) + BINCMP(less, i32) + BINCMP(less, i64) + BINCMP(less, u8) + BINCMP(less, u16) + BINCMP(less, u32) + BINCMP(less, u64) + BINCMP(less, f32) + BINCMP(less, f64) + BINCMP(less, dec32) + BINCMP(less, dec64) + BINCMP(less, dec128) + BINCMP(less, ts_D) + BINCMP(less, ts_h) + BINCMP(less, ts_m) + BINCMP(less, ts_s) + BINCMP(less, ts_ms) + BINCMP(less, ts_us) + BINCMP(less, ts_ns) + BINCMP(less, dur_D) + BINCMP(less, dur_h) + BINCMP(less, dur_m) + BINCMP(less, dur_s) + BINCMP(less, dur_ms) + BINCMP(less, dur_us) + BINCMP(less, dur_ns) + BINCMP(less, str) + + BINCMP(less_equal, b8) + BINCMP(less_equal, i8) + BINCMP(less_equal, i16) + BINCMP(less_equal, i32) + BINCMP(less_equal, i64) + BINCMP(less_equal, u8) + BINCMP(less_equal, u16) + BINCMP(less_equal, u32) + BINCMP(less_equal, u64) + BINCMP(less_equal, f32) + BINCMP(less_equal, f64) + BINCMP(less_equal, dec32) + BINCMP(less_equal, dec64) + BINCMP(less_equal, dec128) + BINCMP(less_equal, ts_D) + BINCMP(less_equal, ts_h) + BINCMP(less_equal, ts_m) + BINCMP(less_equal, ts_s) + BINCMP(less_equal, ts_ms) + BINCMP(less_equal, ts_us) + BINCMP(less_equal, ts_ns) + BINCMP(less_equal, dur_D) + BINCMP(less_equal, dur_h) + BINCMP(less_equal, dur_m) + BINCMP(less_equal, dur_s) + BINCMP(less_equal, dur_ms) + BINCMP(less_equal, dur_us) + BINCMP(less_equal, dur_ns) + BINCMP(less_equal, str) + + BINCMP(greater, b8) + BINCMP(greater, i8) + BINCMP(greater, i16) + BINCMP(greater, i32) + BINCMP(greater, i64) + BINCMP(greater, u8) + BINCMP(greater, u16) + BINCMP(greater, u32) + BINCMP(greater, u64) + BINCMP(greater, f32) + BINCMP(greater, f64) + BINCMP(greater, dec32) + BINCMP(greater, dec64) + BINCMP(greater, dec128) + BINCMP(greater, ts_D) + BINCMP(greater, ts_h) + BINCMP(greater, ts_m) + BINCMP(greater, ts_s) + BINCMP(greater, ts_ms) + BINCMP(greater, ts_us) + BINCMP(greater, ts_ns) + BINCMP(greater, dur_D) + BINCMP(greater, dur_h) + BINCMP(greater, dur_m) + BINCMP(greater, dur_s) + BINCMP(greater, dur_ms) + BINCMP(greater, dur_us) + BINCMP(greater, dur_ns) + BINCMP(greater, str) + + BINCMP(greater_equal, b8) + BINCMP(greater_equal, i8) + BINCMP(greater_equal, i16) + BINCMP(greater_equal, i32) + BINCMP(greater_equal, i64) + BINCMP(greater_equal, u8) + BINCMP(greater_equal, u16) + BINCMP(greater_equal, u32) + BINCMP(greater_equal, u64) + BINCMP(greater_equal, f32) + BINCMP(greater_equal, f64) + BINCMP(greater_equal, dec32) + BINCMP(greater_equal, dec64) + BINCMP(greater_equal, dec128) + BINCMP(greater_equal, ts_D) + BINCMP(greater_equal, ts_h) + BINCMP(greater_equal, ts_m) + BINCMP(greater_equal, ts_s) + BINCMP(greater_equal, ts_ms) + BINCMP(greater_equal, ts_us) + BINCMP(greater_equal, ts_ns) + BINCMP(greater_equal, dur_D) + BINCMP(greater_equal, dur_h) + BINCMP(greater_equal, dur_m) + BINCMP(greater_equal, dur_s) + BINCMP(greater_equal, dur_ms) + BINCMP(greater_equal, dur_us) + BINCMP(greater_equal, dur_ns) + BINCMP(greater_equal, str) + + // Casts + + CASTOP(to_i32, i32, b8) + CASTOP(to_i32, i32, i8) + CASTOP(to_i32, i32, i16) + CASTOP(to_i32, i32, i32) + CASTOP(to_i32, i32, i64) + CASTOP(to_i32, i32, u8) + CASTOP(to_i32, i32, u16) + CASTOP(to_i32, i32, u32) + CASTOP(to_i32, i32, u64) + CASTOP(to_i32, i32, f32) + CASTOP(to_i32, i32, f64) + + CASTOP(to_i64, i64, b8) + CASTOP(to_i64, i64, i8) + CASTOP(to_i64, i64, i16) + CASTOP(to_i64, i64, i32) + CASTOP(to_i64, i64, i64) + CASTOP(to_i64, i64, u8) + CASTOP(to_i64, i64, u16) + CASTOP(to_i64, i64, u32) + CASTOP(to_i64, i64, u64) + CASTOP(to_i64, i64, f32) + CASTOP(to_i64, i64, f64) + + CASTOP(to_u32, u32, b8) + CASTOP(to_u32, u32, i8) + CASTOP(to_u32, u32, i16) + CASTOP(to_u32, u32, i32) + CASTOP(to_u32, u32, i64) + CASTOP(to_u32, u32, u8) + CASTOP(to_u32, u32, u16) + CASTOP(to_u32, u32, u32) + CASTOP(to_u32, u32, u64) + CASTOP(to_u32, u32, f32) + CASTOP(to_u32, u32, f64) + + CASTOP(to_u64, u64, b8) + CASTOP(to_u64, u64, i8) + CASTOP(to_u64, u64, i16) + CASTOP(to_u64, u64, i32) + CASTOP(to_u64, u64, i64) + CASTOP(to_u64, u64, u8) + CASTOP(to_u64, u64, u16) + CASTOP(to_u64, u64, u32) + CASTOP(to_u64, u64, u64) + CASTOP(to_u64, u64, f32) + CASTOP(to_u64, u64, f64) + + CASTOP(to_f32, f32, b8) + CASTOP(to_f32, f32, i8) + CASTOP(to_f32, f32, i16) + CASTOP(to_f32, f32, i32) + CASTOP(to_f32, f32, i64) + CASTOP(to_f32, f32, u8) + CASTOP(to_f32, f32, u16) + CASTOP(to_f32, f32, u32) + CASTOP(to_f32, f32, u64) + CASTOP(to_f32, f32, f32) + CASTOP(to_f32, f32, f64) + + CASTOP(to_f64, f64, b8) + CASTOP(to_f64, f64, i8) + CASTOP(to_f64, f64, i16) + CASTOP(to_f64, f64, i32) + CASTOP(to_f64, f64, i64) + CASTOP(to_f64, f64, u8) + CASTOP(to_f64, f64, u16) + CASTOP(to_f64, f64, u32) + CASTOP(to_f64, f64, u64) + CASTOP(to_f64, f64, f32) + CASTOP(to_f64, f64, f64) + + // Ternary + COND_OP(if_else, b8) + COND_OP(if_else, i8) + COND_OP(if_else, i16) + COND_OP(if_else, i32) + COND_OP(if_else, i64) + COND_OP(if_else, u8) + COND_OP(if_else, u16) + COND_OP(if_else, u32) + COND_OP(if_else, u64) + COND_OP(if_else, f32) + COND_OP(if_else, f64) + COND_OP(if_else, dec32) + COND_OP(if_else, dec64) + COND_OP(if_else, dec128) + COND_OP(if_else, ts_D) + COND_OP(if_else, ts_h) + COND_OP(if_else, ts_m) + COND_OP(if_else, ts_s) + COND_OP(if_else, ts_ms) + COND_OP(if_else, ts_us) + COND_OP(if_else, ts_ns) + COND_OP(if_else, dur_D) + COND_OP(if_else, dur_h) + COND_OP(if_else, dur_m) + COND_OP(if_else, dur_s) + COND_OP(if_else, dur_ms) + COND_OP(if_else, dur_us) + COND_OP(if_else, dur_ns) + COND_OP(if_else, str) + + } // namespace lite +} // namespace cudf diff --git a/cpp/src/jit/lite/ops.cu b/cpp/src/jit/lite/ops.cu deleted file mode 100644 index 813ca00215d7..000000000000 --- a/cpp/src/jit/lite/ops.cu +++ /dev/null @@ -1,34 +0,0 @@ -#include - -namespace __attribute__((visibility("default"))) cudf -{ - namespace lite { - - // TODO: pre-instantiate some of the operators so they can be used in LTO - -#define DEFINE_UNARY_OP(NAME, RET_TYPE, TYPE, TYPE_TAG) \ - extern "C" __device__ int cudf_##NAME##__##TYPE_TAG(RET_TYPE* out, TYPE const* a) \ - { \ - return operators::NAME(out, a); \ - } - -#define DEFINE_BINARY_OP(NAME, RET_TYPE, TYPE, TYPE_TAG) \ - extern "C" __device__ int cudf_##NAME##__##TYPE_TAG(RET_TYPE* out, TYPE const* a, TYPE const* b) \ - { \ - return operators::NAME(out, a, b); \ - } - - DEFINE_UNARY_OP(abs, int8_t, int8_t, i8) - DEFINE_UNARY_OP(abs, optional, optional, i8_opt) - DEFINE_UNARY_OP(abs, int16_t, int16_t, i16) - DEFINE_UNARY_OP(abs, optional, optional, i16_opt) - DEFINE_UNARY_OP(abs, int32_t, int32_t, i32) - DEFINE_UNARY_OP(abs, optional, optional, i32_opt) - DEFINE_UNARY_OP(abs, int64_t, int64_t, i64) - DEFINE_UNARY_OP(abs, optional, optional, i64_opt) - - - - - } // namespace lite -} // namespace cudf diff --git a/cpp/src/jit/sync.cuh b/cpp/src/jit/sync.cuh index 8e9ff986dfbf..8d0079927d23 100644 --- a/cpp/src/jit/sync.cuh +++ b/cpp/src/jit/sync.cuh @@ -12,7 +12,6 @@ #include namespace cudf { -namespace jit { __device__ inline bool warp_elect(unsigned int mask) { @@ -27,6 +26,8 @@ __device__ inline bool warp_elect(unsigned int mask) #endif } +namespace jit { + template __device__ void warp_compact_validity(unsigned int active_mask, mutable_column_device_view_core const* outcols, diff --git a/cpp/src/librtcx/embed.cmake b/cpp/src/librtcx/embed.cmake index b8aa28704b72..e58b2f7d49d7 100644 --- a/cpp/src/librtcx/embed.cmake +++ b/cpp/src/librtcx/embed.cmake @@ -14,7 +14,7 @@ endif() # This function registers a directory of include files to be embedded for JIT compilation. It # gathers the specified files, their destinations, and include directories, and stores them in # target-specific variables for later use when generating the embed. -function(jit_add_include_directory) +function(embed_includes) set(TARGET ${ARGV0}) set(OPTIONS "") set(ONE_VALUE_ARGS COPY_DIRECTORY # Source directory where files will be copied from @@ -66,9 +66,9 @@ function(jit_add_include_directory) # Set scope variables to accumulate results - set(SOURCE_FILES ${${TARGET}__jitembed_incdir__source_files}) - set(SOURCE_FILE_DESTS ${${TARGET}__jitembed_incdir__source_file_dests}) - set(INCLUDE_DIRECTORIES ${${TARGET}__jitembed_incdir__include_directories}) + set(SOURCE_FILES ${${TARGET}__embed__source_files}) + set(SOURCE_FILE_DESTS ${${TARGET}__embed__source_file_dests}) + set(INCLUDE_DIRECTORIES ${${TARGET}__embed__include_directories}) foreach(SOURCE_FILE IN LISTS ARG_FILES) list(APPEND SOURCE_FILES "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") @@ -77,23 +77,72 @@ function(jit_add_include_directory) list(APPEND INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES}) - set(${TARGET}__jitembed_incdir__source_files + set(${TARGET}__embed__source_files ${SOURCE_FILES} PARENT_SCOPE ) - set(${TARGET}__jitembed_incdir__source_file_dests + set(${TARGET}__embed__source_file_dests ${SOURCE_FILE_DESTS} PARENT_SCOPE ) - set(${TARGET}__jitembed_incdir__include_directories + set(${TARGET}__embed__include_directories ${INCLUDE_DIRECTORIES} PARENT_SCOPE ) endfunction() +function(embed_blob) + set(TARGET ${ARGV0}) + set(OPTIONS) + set(ONE_VALUE_ARGS FILE DEST) + set(MULTI_VALUE_ARGS "") + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET argument is required") + endif() + + if(NOT ARG_FILE) + message(FATAL_ERROR "FILE argument is required") + endif() + + if(NOT ARG_DEST) + message(FATAL_ERROR "DEST argument is required") + endif() + + set(SOURCE_FILES ${${TARGET}__embed__source_files}) + set(SOURCE_FILE_DESTS ${${TARGET}__embed__source_file_dests}) + set(TARGET_DEPS ${${TARGET}__embed__target_deps}) + + if(ARG_FILE MATCHES "\\$]+)>") + # If the file is a generator expression for target objects add as dependency + list(APPEND TARGET_DEPS $) + else() + if(NOT EXISTS "${ARG_FILE}") + message(FATAL_ERROR "Source file '${ARG_FILE}' does not exist") + endif() + endif() + list(APPEND SOURCE_FILES ${ARG_FILE}) + list(APPEND SOURCE_FILE_DESTS ${ARG_DEST}) + + set(${TARGET}__embed__source_files + ${SOURCE_FILES} + PARENT_SCOPE + ) + set(${TARGET}__embed__source_file_dests + ${SOURCE_FILE_DESTS} + PARENT_SCOPE + ) + set(${TARGET}__embed__target_deps + ${TARGET_DEPS} + PARENT_SCOPE + ) + +endfunction() + # pass the encoded args to the embed.py script to generate the embed -function(jit_embed) +function(embed) set(TARGET ${ARGV0}) set(OPTIONS "") set(ONE_VALUE_ARGS "COMPRESSION") @@ -112,27 +161,26 @@ function(jit_embed) message(FATAL_ERROR "COMPRESSION argument must be either none or zstd") endif() - if(NOT DEFINED ${TARGET}__jitembed_incdir__source_files) - message( - FATAL_ERROR - "No source files registered for target '${TARGET}'. Call jit_add_include_directory() first" - ) + if(NOT DEFINED ${TARGET}__embed__source_files) + message(FATAL_ERROR "No source files registered for target '${TARGET}'") endif() set(OUTPUT_DIR "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") - set(CONFIGURED_EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed.cpp") - set(EMBED_SCRIPT_IN "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.cpp") + set(EMBED_SCRIPT_TEMPLATE "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.cpp") + set(CONFIGURED_EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed_cfg.cpp") + set(EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed.cpp") - set(RTCX_EMBED_SCRIPT_ARG__ID "${TARGET}") - set(RTCX_EMBED_SCRIPT_ARG__FILE_PATHS "${${TARGET}__jitembed_incdir__source_files}") - set(RTCX_EMBED_SCRIPT_ARG__FILE_DESTS "${${TARGET}__jitembed_incdir__source_file_dests}") - set(RTCX_EMBED_SCRIPT_ARG__INCLUDE_DIRS "${${TARGET}__jitembed_incdir__include_directories}") - set(RTCX_EMBED_SCRIPT_ARG__COMPRESSION "${ARG_COMPRESSION}") - set(RTCX_EMBED_SCRIPT_ARG__OUTPUT_DIR "${OUTPUT_DIR}") + set(EMBED_SCRIPT__ID "${TARGET}") + set(EMBED_SCRIPT__FILE_PATHS "${${TARGET}__embed__source_files}") + set(EMBED_SCRIPT__FILE_DESTS "${${TARGET}__embed__source_file_dests}") + set(EMBED_SCRIPT__INCLUDE_DIRS "${${TARGET}__embed__include_directories}") + set(EMBED_SCRIPT__COMPRESSION "${ARG_COMPRESSION}") + set(EMBED_SCRIPT__OUTPUT_DIR "${OUTPUT_DIR}") - configure_file("${EMBED_SCRIPT_IN}" "${CONFIGURED_EMBED_SCRIPT}" @ONLY) + configure_file(${EMBED_SCRIPT_TEMPLATE} ${CONFIGURED_EMBED_SCRIPT} @ONLY) + file(GENERATE OUTPUT "${EMBED_SCRIPT}" INPUT "${CONFIGURED_EMBED_SCRIPT}") - add_executable("${TARGET}__jit_embed_run" EXCLUDE_FROM_ALL "${CONFIGURED_EMBED_SCRIPT}") + add_executable("${TARGET}__jit_embed_run" EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") target_include_directories("${TARGET}__jit_embed_run" PRIVATE ${ZSTD_INCLUDE_DIR}) target_link_libraries("${TARGET}__jit_embed_run" PRIVATE ${CMAKE_DL_LIBS} zstd) set_target_properties( @@ -144,8 +192,8 @@ function(jit_embed) OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin BYPRODUCTS ${OUTPUT_DIR}/* COMMAND "${CMAKE_COMMAND}" -E env $ - DEPENDS "${EMBED_SCRIPT_IN}" "${CONFIGURED_EMBED_SCRIPT}" - ${${TARGET}__jitembed_incdir__source_files} + DEPENDS "${CONFIGURED_EMBED_SCRIPT}" "${EMBED_SCRIPT}" ${${TARGET}__embed__source_files} + ${${TARGET}__embed__target_deps} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" COMMENT "Generating JIT embed for ${TARGET} into ${OUTPUT_DIR}" VERBATIM diff --git a/cpp/src/librtcx/embed.hpp b/cpp/src/librtcx/embed.hpp new file mode 100644 index 000000000000..82749b1740d5 --- /dev/null +++ b/cpp/src/librtcx/embed.hpp @@ -0,0 +1,341 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include "sha256.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEFER__CONCATENATE_DETAIL(x, y) x##y +#define DEFER__CONCATENATE(x, y) DEFER__CONCATENATE_DETAIL(x, y) +#define DEFER(...) ::defer DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) + +template +struct defer { + private: + T func_; + + public: + template + defer(Args&&... args) : func_{static_cast(args)...} + { + } + defer(defer const&) = delete; + defer& operator=(defer const&) = delete; + defer(defer&&) = delete; + defer& operator=(defer&&) = delete; + ~defer() { func_(); } +}; + +template +defer(T) -> defer; + +struct size_range { + size_t offset = 0; + size_t size = 0; +}; + +std::pair, std::vector> merge_bytes_with_null_terminators( + std::span const> bytes_lists) +{ + std::vector merged; + std::vector ranges; + + for (auto& byte_data : bytes_lists) { + ranges.push_back({merged.size(), byte_data.size()}); + merged.insert(merged.end(), byte_data.begin(), byte_data.end()); + merged.push_back(0); + } + + return {std::move(merged), std::move(ranges)}; +} + +struct embed_output { + std::string cxx_header; + std::string cxx_source; + std::string asm_source; + std::string bin_file_name; + std::vector bin_file_data; +}; + +std::vector load_file_bytes(std::string_view file_path) +{ + std::string path_str(file_path); + std::ifstream file(path_str, std::ios::binary | std::ios::ate); + if (!file) { + throw std::runtime_error(std::format("Failed to open file at path: {}", file_path)); + } + auto file_size = file.tellg(); + if (file_size < 0) { + throw std::runtime_error( + std::format("Failed to determine size of file at path: {}", file_path)); + } + file.seekg(0, std::ios::beg); + std::vector bytes(file_size); + if (!file.read(reinterpret_cast(bytes.data()), file_size)) { + throw std::runtime_error(std::format("Failed to read file at path: {}", file_path)); + } + return bytes; +} + +std::vector compress_bytes(std::span bytes, std::string_view compression) +{ + if (compression != "none" && compression != "zstd") { + throw std::invalid_argument(std::format( + "Invalid compression type: {}. Supported values are 'none' and 'zstd'", compression)); + } + + if (compression == "none") { return std::vector(bytes.begin(), bytes.end()); } + + auto const max_compressed_size = ZSTD_compressBound(bytes.size()); + std::vector compressed(max_compressed_size); + auto const compressed_size = + ZSTD_compress(compressed.data(), compressed.size(), bytes.data(), bytes.size(), 22); + + if (ZSTD_isError(compressed_size)) { + throw std::runtime_error(std::string("ZSTD compression failed: ") + + ZSTD_getErrorName(compressed_size)); + } + + compressed.resize(compressed_size); + return compressed; +} + +rtcx::sha256 compute_embed_hash(std::span uncompressed_files_bytes, + std::span merged_dests_bytes, + std::span merged_include_dirs_bytes, + std::string_view compression) +{ + rtcx::sha256_context ctx; + ctx.update(uncompressed_files_bytes); + ctx.update(merged_dests_bytes); + ctx.update(merged_include_dirs_bytes); + ctx.update(std::span{reinterpret_cast(compression.data()), compression.size()}); + return ctx.finalize(); +} + +template +std::string join_formatted(Container& items, std::string_view delimiter, Formatter&& formatter) +{ + std::ostringstream result; + for (std::size_t i = 0; i < items.size(); ++i) { + if (i != 0) { result << delimiter; } + result << formatter(items[i]); + } + return result.str(); +} + +embed_output generate_cxx_source_files_data(std::string_view id, + std::span file_paths, + std::span file_dsts, + std::span include_dirs, + std::string_view compression) +{ + std::vector> file_bytes; + file_bytes.reserve(file_paths.size()); + for (auto const& path : file_paths) { + file_bytes.emplace_back(load_file_bytes(path)); + } + + auto [uncompressed_files_bytes, files_ranges] = merge_bytes_with_null_terminators(file_bytes); + + auto compress = compression != "none"; + std::vector compressed_files_bytes = + compress ? compress_bytes(uncompressed_files_bytes, compression) : uncompressed_files_bytes; + + auto binary_size = compress ? compressed_files_bytes.size() + : static_cast(uncompressed_files_bytes.size()); + + if (compress) { + std::cout << std::format( + "-- Compressed {}'s binary from {} bytes to {} bytes (compression ratio: {:.2f})\n", + id, + uncompressed_files_bytes.size(), + compressed_files_bytes.size(), + static_cast(compressed_files_bytes.size()) / + static_cast(uncompressed_files_bytes.size())); + } + + std::vector> destination_bytes; + destination_bytes.reserve(file_dsts.size()); + for (auto const& dest : file_dsts) { + destination_bytes.emplace_back(dest.begin(), dest.end()); + } + auto [merged_dests_bytes, _] = merge_bytes_with_null_terminators(destination_bytes); + + std::vector> include_directory_bytes; + include_directory_bytes.reserve(include_dirs.size()); + for (auto const& include_directory : include_dirs) { + include_directory_bytes.emplace_back(include_directory.begin(), include_directory.end()); + } + auto [merged_include_dirs_bytes, __] = merge_bytes_with_null_terminators(include_directory_bytes); + + auto hash = compute_embed_hash( + uncompressed_files_bytes, merged_dests_bytes, merged_include_dirs_bytes, compression); + + auto binary_file_name = std::format("embed_{}.bin", id); + + auto include_dirs_list = + join_formatted(include_dirs, ",\n", [](auto s) { return std::format("\"{}\"", s); }); + auto dests_list = + join_formatted(file_dsts, ",\n", [](auto s) { return std::format("\"{}\"", s); }); + auto ranges_list = join_formatted( + files_ranges, ",\n", [](auto r) { return std::format("{{{}, {}}}", r.offset, r.size); }); + + auto hash_list = join_formatted( + hash, ", ", [](uint8_t byte) { return std::format("0x{:02x}", static_cast(byte)); }); + + auto cxx_header = std::format( + R"***( +// Auto-generated header for embedded files with ID: {} +#pragma once + +#include +#include +#include +#include + +namespace rtcx_embed {{ + +struct range {{ + std::size_t offset = 0; + std::size_t size = 0; +}}; + +constexpr char const * {}_include_directories[{}] = +{{ +{} +}}; + +constexpr char const * {}_file_destinations[{}] = +{{ +{} +}}; + +constexpr range {}_file_ranges[{}] = +{{ +{} +}}; + +constexpr std::size_t {}_files_uncompressed_size = {}; + +constexpr char const * {}_files_compression = "{}"; + +extern "C" std::uint8_t const rtcx_embed_{}_files_begin[]; + +static std::span const {}_files = +{{ +rtcx_embed_{}_files_begin, +{}L +}}; + +constexpr std::uint8_t {}_hash[{}] = +{{ +{} +}}; + +}} +)***", + id, + id, + include_dirs.size(), + include_dirs_list, + id, + file_dsts.size(), + dests_list, + id, + files_ranges.size(), + ranges_list, + id, + uncompressed_files_bytes.size(), + id, + compression, + id, + id, + id, + binary_size, + id, + hash.size(), + hash_list); + + auto asm_source = std::format( + R"***( +.section .rodata +.global rtcx_embed_{}_files_begin +rtcx_embed_{}_files_begin: +.incbin "{}" + +.section .note.GNU-stack,"",@progbits +)***", + id, + id, + binary_file_name); + + return embed_output{ + .cxx_header = cxx_header, + .cxx_source = "", + .asm_source = asm_source, + .bin_file_name = binary_file_name, + .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes}; +} + +void generate_embed(std::string_view id, + std::span file_paths, + std::span file_dsts, + std::span include_dirs, + std::string_view compression, + std::string_view output_directory) +{ + auto output = + generate_cxx_source_files_data(id, file_paths, file_dsts, include_dirs, compression); + + std::filesystem::create_directories(std::filesystem::path(output_directory)); + + std::ofstream header_file(std::format("{}/{}.hpp", output_directory, id)); + header_file << output.cxx_header; + + std::ofstream asm_file(std::format("{}/{}.s", output_directory, id)); + asm_file << output.asm_source; + + std::ofstream bin_file(std::format("{}/{}", output_directory, output.bin_file_name), + std::ios::binary); + bin_file.write(reinterpret_cast(output.bin_file_data.data()), + static_cast(output.bin_file_data.size())); +} + +std::vector split_string(std::string_view str, char delimiter) +{ + std::vector tokens; + std::size_t start = 0; + + while (start <= str.size()) { + auto const pos = str.find(delimiter, start); + if (pos == std::string_view::npos) { + tokens.push_back(str.substr(start)); + break; + } + tokens.push_back(str.substr(start, pos - start)); + start = pos + 1; + } + + return tokens; +} diff --git a/cpp/src/librtcx/embed.in.cpp b/cpp/src/librtcx/embed.in.cpp index e666d5bc1a41..a94af51bb680 100644 --- a/cpp/src/librtcx/embed.in.cpp +++ b/cpp/src/librtcx/embed.in.cpp @@ -3,350 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "sha256.hpp" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define DEFER__CONCATENATE_DETAIL(x, y) x##y -#define DEFER__CONCATENATE(x, y) DEFER__CONCATENATE_DETAIL(x, y) -#define DEFER(...) ::defer DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) - -template -struct defer { - private: - T func_; - - public: - template - defer(Args&&... args) : func_{static_cast(args)...} - { - } - defer(defer const&) = delete; - defer& operator=(defer const&) = delete; - defer(defer&&) = delete; - defer& operator=(defer&&) = delete; - ~defer() { func_(); } -}; - -template -defer(T) -> defer; - -struct size_range { - size_t offset = 0; - size_t size = 0; -}; - -std::pair, std::vector> merge_bytes_with_null_terminators( - std::span const> bytes_lists) -{ - std::vector merged; - std::vector ranges; - - for (auto& byte_data : bytes_lists) { - ranges.push_back({merged.size(), byte_data.size()}); - merged.insert(merged.end(), byte_data.begin(), byte_data.end()); - merged.push_back(0); - } - - return {std::move(merged), std::move(ranges)}; -} - -struct embed_output { - std::string cxx_header; - std::string cxx_source; - std::string asm_source; - std::string bin_file_name; - std::vector bin_file_data; -}; - -std::vector load_file_bytes(std::string_view file_path) -{ - std::string path_str(file_path); - std::ifstream file(path_str, std::ios::binary | std::ios::ate); - if (!file) { - throw std::runtime_error(std::format("Failed to open file at path: {}", file_path)); - } - auto file_size = file.tellg(); - if (file_size < 0) { - throw std::runtime_error( - std::format("Failed to determine size of file at path: {}", file_path)); - } - file.seekg(0, std::ios::beg); - std::vector bytes(file_size); - if (!file.read(reinterpret_cast(bytes.data()), file_size)) { - throw std::runtime_error(std::format("Failed to read file at path: {}", file_path)); - } - return bytes; -} - -std::vector compress_bytes(std::span bytes, std::string_view compression) -{ - if (compression != "none" && compression != "zstd") { - throw std::invalid_argument(std::format( - "Invalid compression type: {}. Supported values are 'none' and 'zstd'", compression)); - } - - if (compression == "none") { return std::vector(bytes.begin(), bytes.end()); } - - auto const max_compressed_size = ZSTD_compressBound(bytes.size()); - std::vector compressed(max_compressed_size); - auto const compressed_size = - ZSTD_compress(compressed.data(), compressed.size(), bytes.data(), bytes.size(), 22); - - if (ZSTD_isError(compressed_size)) { - throw std::runtime_error(std::string("ZSTD compression failed: ") + - ZSTD_getErrorName(compressed_size)); - } - - compressed.resize(compressed_size); - return compressed; -} - -rtcx::sha256 compute_embed_hash(std::span uncompressed_files_bytes, - std::span merged_dests_bytes, - std::span merged_include_dirs_bytes, - std::string_view compression) -{ - rtcx::sha256_context ctx; - ctx.update(uncompressed_files_bytes); - ctx.update(merged_dests_bytes); - ctx.update(merged_include_dirs_bytes); - ctx.update(std::span{reinterpret_cast(compression.data()), compression.size()}); - return ctx.finalize(); -} - -template -std::string join_formatted(Container& items, std::string_view delimiter, Formatter&& formatter) -{ - std::ostringstream result; - for (std::size_t i = 0; i < items.size(); ++i) { - if (i != 0) { result << delimiter; } - result << formatter(items[i]); - } - return result.str(); -} - -embed_output generate_cxx_source_files_data(std::string_view id, - std::span file_paths, - std::span file_dsts, - std::span include_dirs, - std::string_view compression) -{ - std::vector> file_bytes; - file_bytes.reserve(file_paths.size()); - for (auto const& path : file_paths) { - file_bytes.emplace_back(load_file_bytes(path)); - } - - auto [uncompressed_files_bytes, files_ranges] = merge_bytes_with_null_terminators(file_bytes); - - auto compress = compression != "none"; - std::vector compressed_files_bytes = - compress ? compress_bytes(uncompressed_files_bytes, compression) : uncompressed_files_bytes; - - auto binary_size = compress ? compressed_files_bytes.size() - : static_cast(uncompressed_files_bytes.size()); - - if (compress) { - std::cout << std::format( - "-- Compressed {}'s binary from {} bytes to {} bytes (compression ratio: {:.2f})\n", - id, - uncompressed_files_bytes.size(), - compressed_files_bytes.size(), - static_cast(compressed_files_bytes.size()) / - static_cast(uncompressed_files_bytes.size())); - } - - std::vector> destination_bytes; - destination_bytes.reserve(file_dsts.size()); - for (auto const& dest : file_dsts) { - destination_bytes.emplace_back(dest.begin(), dest.end()); - } - auto [merged_dests_bytes, _] = merge_bytes_with_null_terminators(destination_bytes); - - std::vector> include_directory_bytes; - include_directory_bytes.reserve(include_dirs.size()); - for (auto const& include_directory : include_dirs) { - include_directory_bytes.emplace_back(include_directory.begin(), include_directory.end()); - } - auto [merged_include_dirs_bytes, __] = merge_bytes_with_null_terminators(include_directory_bytes); - - auto hash = compute_embed_hash( - uncompressed_files_bytes, merged_dests_bytes, merged_include_dirs_bytes, compression); - - auto binary_file_name = std::format("embed_{}.bin", id); - - auto include_dirs_list = - join_formatted(include_dirs, ",\n", [](auto s) { return std::format("\"{}\"", s); }); - auto dests_list = - join_formatted(file_dsts, ",\n", [](auto s) { return std::format("\"{}\"", s); }); - auto ranges_list = join_formatted( - files_ranges, ",\n", [](auto r) { return std::format("{{{}, {}}}", r.offset, r.size); }); - - auto hash_list = join_formatted( - hash, ", ", [](uint8_t byte) { return std::format("0x{:02x}", static_cast(byte)); }); - - auto cxx_header = std::format( - R"***( -// Auto-generated header for embedded files with ID: {} -#pragma once - -#include -#include -#include -#include - -namespace rtcx_embed {{ - -struct range {{ - std::size_t offset = 0; - std::size_t size = 0; -}}; - -constexpr char const * {}_include_directories[{}] = -{{ -{} -}}; - -constexpr char const * {}_file_destinations[{}] = -{{ -{} -}}; - -constexpr range {}_file_ranges[{}] = -{{ -{} -}}; - -constexpr std::size_t {}_files_uncompressed_size = {}; - -constexpr char const * {}_files_compression = "{}"; - -extern "C" std::uint8_t const rtcx_embed_{}_files_begin[]; - -static std::span const {}_files = -{{ -rtcx_embed_{}_files_begin, -{}L -}}; - -constexpr std::uint8_t {}_hash[{}] = -{{ -{} -}}; - -}} -)***", - id, - id, - include_dirs.size(), - include_dirs_list, - id, - file_dsts.size(), - dests_list, - id, - files_ranges.size(), - ranges_list, - id, - uncompressed_files_bytes.size(), - id, - compression, - id, - id, - id, - binary_size, - id, - hash.size(), - hash_list); - - auto asm_source = std::format( - R"***( -.section .rodata -.global rtcx_embed_{}_files_begin -rtcx_embed_{}_files_begin: -.incbin "{}" - -.section .note.GNU-stack,"",@progbits -)***", - id, - id, - binary_file_name); - - return embed_output{ - .cxx_header = cxx_header, - .cxx_source = "", - .asm_source = asm_source, - .bin_file_name = binary_file_name, - .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes}; -} - -void generate_embed(std::string_view id, - std::span file_paths, - std::span file_dsts, - std::span include_dirs, - std::string_view compression, - std::string_view output_directory) -{ - auto output = - generate_cxx_source_files_data(id, file_paths, file_dsts, include_dirs, compression); - - std::filesystem::create_directories(std::filesystem::path(output_directory)); - - std::ofstream header_file(std::format("{}/{}.hpp", output_directory, id)); - header_file << output.cxx_header; - - std::ofstream asm_file(std::format("{}/{}.s", output_directory, id)); - asm_file << output.asm_source; - - std::ofstream bin_file(std::format("{}/{}", output_directory, output.bin_file_name), - std::ios::binary); - bin_file.write(reinterpret_cast(output.bin_file_data.data()), - static_cast(output.bin_file_data.size())); -} - -std::vector split_string(std::string_view str, char delimiter) -{ - std::vector tokens; - std::size_t start = 0; - - while (start <= str.size()) { - auto const pos = str.find(delimiter, start); - if (pos == std::string_view::npos) { - tokens.push_back(str.substr(start)); - break; - } - tokens.push_back(str.substr(start, pos - start)); - start = pos + 1; - } - - return tokens; -} +#include "embed.hpp" int main() { - std::string_view id = "@RTCX_EMBED_SCRIPT_ARG__ID@"; - auto file_paths = split_string("@RTCX_EMBED_SCRIPT_ARG__FILE_PATHS@", ';'); - auto file_dests = split_string("@RTCX_EMBED_SCRIPT_ARG__FILE_DESTS@", ';'); - auto include_directories = split_string("@RTCX_EMBED_SCRIPT_ARG__INCLUDE_DIRS@", ';'); - std::string_view compression = "@RTCX_EMBED_SCRIPT_ARG__COMPRESSION@"; - std::string_view output_dir = "@RTCX_EMBED_SCRIPT_ARG__OUTPUT_DIR@"; + std::string_view id = "@EMBED_SCRIPT__ID@"; + auto file_paths = split_string("@EMBED_SCRIPT__FILE_PATHS@", ';'); + auto file_dests = split_string("@EMBED_SCRIPT__FILE_DESTS@", ';'); + auto include_directories = split_string("@EMBED_SCRIPT__INCLUDE_DIRS@", ';'); + std::string_view compression = "@EMBED_SCRIPT__COMPRESSION@"; + std::string_view output_dir = "@EMBED_SCRIPT__OUTPUT_DIR@"; generate_embed(id, file_paths, file_dests, include_directories, compression, output_dir); return EXIT_SUCCESS; diff --git a/cpp/src/transform/jit/compute_columns_kernel.cu b/cpp/src/transform/jit/compute_columns_kernel.cu index 8fd0d4ab17ef..e2c31429aadf 100644 --- a/cpp/src/transform/jit/compute_columns_kernel.cu +++ b/cpp/src/transform/jit/compute_columns_kernel.cu @@ -1,170 +1,104 @@ - +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ #include "jit/element.cuh" #include "jit/element_storage.cuh" #include -#include -#include +#include +#include + +#include +#include +#include namespace cudf { +/// @brief The compute operation to perform on each element +/// @param user_data Pointer to user data passed to the kernel +/// @param element_index The index of the element to compute +/// @param inputs Pointer to the input elements for this operation; the caller guarantees the memory +/// layout and type of these elements based on the input column device views and input strides +/// @param input_stride The stride (in bytes) between consecutive input elements for a given input +/// column +/// @param outputs Pointer to the output elements for this operation; the caller guarantees the +/// memory layout and type of these elements based on the output column device views +/// @param output_stride The stride (in bytes) between consecutive output elements for a given +/// output column +/// @return An integer status code; the meaning of this code is defined by the caller and operator +/// implementation extern "C" __device__ int operation(void* user_data, - int element_index, - void const* const* inputs, - void* const* outputs) + long int element_index, + void const* inputs, + int input_stride, + void* outputs, + int output_stride); + +template +__device__ void compute_columns_kernel(size_type row_size, + void* __restrict__ user_data, + column_device_view const* __restrict__ input_cols, + size_type num_inputs, + mutable_column_device_view const* __restrict__ output_cols, + size_type num_outputs, + size_type const* __restrict__ input_strides) { - // element_storage tmp0; - // add(&tmp0, inputs[0], inputs[1]); - // outputs[0] = tmp0; - - // knows the types - // cast to the target type - // load the element - // perform the operation - // use C++ AST to codegen - // store the element - // - // - // - // - // - // pre-compile some of these operators for common operations and types; compiling the operator - // becomes trivial as we only need to forward declare and then link to the pre-compiled fragments - // - // for example, we can pre-compile for all lexicographic comparisons cheaply - // - // - // - // WE ALSO NEED TO MAKE SURE THE ORDER OF INPUTS AND OUTPUTS doesnt affect the applicability of - // the generic operators - // - // - // - // TODO: functions to add fragments - // - // - return 0; -} + // 255 i32 registers per thread for GH200 + // 227 SHMEM bytes per thread for GH200 -// generic_column_operator: catch-all -// specific column_operator - -// unary and binary operators can be easily JIT-ed for all types and use JIT-ed operators -// -// the register pressure and compile-time of the generic operator is very high -// -// use PTX to specify the operator to reduce compile time -// -// -// -// should we pre-link some of the kernels to reduce work at JIT-time? - -/// @brief The generic transform kernel. Supports all types and nullability combinations. -extern "C" __global__ void compute_columns_kernel( - size_type row_size, - bitmask_type const* __restrict__ stencil, - bool stencil_has_nulls, - void* __restrict__ user_data, - column_device_view const* __restrict__ input_cols, - size_type num_inputs, - mutable_column_device_view const* __restrict__ output_cols, - size_type num_outputs, - bool const* is_scalar) -{ - static constexpr int MAX_INPUTS = 32; - static constexpr int MAX_OUTPUTS = 16; + using input_storage_t = element_storage; + using output_storage_t = element_storage; - element_storage input_storage[MAX_INPUTS]; - void const* input_ptrs[MAX_INPUTS]; - element_storage output_storage[MAX_OUTPUTS]; - void* output_ptrs[MAX_OUTPUTS]; + extern __shared__ char shmem[]; - for (int i = 0; i < MAX_INPUTS; ++i) { - input_ptrs[i] = &input_storage[i]; - } - for (int i = 0; i < MAX_OUTPUTS; ++i) { - output_ptrs[i] = &output_storage[i]; - } + auto shmem_iter = shmem; + input_storage_t* input_storage = reinterpret_cast(shmem_iter); + shmem_iter += sizeof(input_storage_t) * num_inputs; + output_storage_t* output_storage = reinterpret_cast(shmem_iter); auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); for (auto element_idx = start; element_idx < row_size; element_idx += stride) { - auto active_mask = __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); - - for (int i = 0; i < num_inputs; i++) { - load_element(is_scalar[i], input_cols + i, element_idx, input_storage + i); - } - - operation(user_data, element_idx, input_ptrs, output_ptrs); - - for (int i = 0; i < num_inputs; i++) { - store_element(output_cols, output_storage + i, element_idx, active_mask); - } - - if constexpr (is_null_aware == null_aware::NO) { - if (stencil_has_nulls && !bit_is_set(stencil, element_idx)) { continue; } - - auto outs = OutputAccessors::map([&]() { - return cuda::std::tuple{A::output_arg(output_cols, element_idx)...}; - }); - - auto out_ptrs = - cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - - auto inputs = InputAccessors::map( - [&]() { return cuda::std::tuple{A::element(input_cols, element_idx)...}; }); - - if constexpr (has_user_data) { - auto args = - cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); - cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); - + auto active_mask = [&] __device__() { + if constexpr (has_nulls) { + return __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); } else { - // TODO: static assert invocable - auto args = cuda::std::tuple_cat(out_ptrs, inputs); - cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); + return 0xFFFF'FFFFU; } + }(); - OutputAccessors::map([&]() { - (A::assign(output_cols, element_idx, cuda::std::get(outs)), ...); - }); - } else { - bool is_valid[OutputAccessors::size]; - - auto outs = OutputAccessors::map([&]() { - return cuda::std::tuple{A::null_output_arg(output_cols, element_idx)...}; - }); - - auto out_ptrs = - cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - - auto inputs = InputAccessors::map([&]() { - return cuda::std::tuple{A::nullable_element(input_cols, element_idx)...}; - }); - - if constexpr (has_user_data) { - auto args = - cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, out_ptrs, inputs); - cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); - - } else { - auto args = cuda::std::tuple_cat(out_ptrs, inputs); - cuda::std::apply([](auto&&... a) { GENERIC_TRANSFORM_OP(a...); }, args); - } + for (int i = 0; i < num_inputs; i++) { + load_element( + input_cols + i, element_idx * input_strides[i], input_storage + i); + } - OutputAccessors::map([&]() { - (A::assign(output_cols, element_idx, *cuda::std::get(outs)), ...); - ((is_valid[A::index] = cuda::std::get(outs).has_value()), ...); - }); + operation(user_data, + element_idx, + input_storage, + sizeof(input_storage_t), + output_storage, + sizeof(output_storage_t)); - OutputAccessors::map([&]() { - auto active_mask = __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); - (warp_compact_validity(active_mask, output_cols, element_idx, is_valid[A::index]), ...); - }); + for (int i = 0; i < num_outputs; i++) { + store_element( + output_cols + i, output_storage + i, element_idx, active_mask); } } } -} // namespace cudf \ No newline at end of file +} // namespace cudf + +extern "C" __global__ void kernel(cudf::size_type row_size, + void* __restrict__ user_data, + cudf::column_device_view const* __restrict__ input_cols, + cudf::size_type num_inputs, + cudf::mutable_column_device_view const* __restrict__ output_cols, + cudf::size_type num_outputs, + cudf::size_type const* __restrict__ input_strides) +{ + compute_columns_kernel( + row_size, user_data, input_cols, num_inputs, output_cols, num_outputs, input_strides); +} diff --git a/todo.md b/todo.md index db578ce39d2b..0af985552e75 100644 --- a/todo.md +++ b/todo.md @@ -1,17 +1,10 @@ Add POC: - - Successfully compile existing CUDF JIT kernels whilst reducing the binary size of the JIT kernels - - Precompiled and JIT-ed compute_columns_kernel; reduce register pressure and ...: - - Make AST codegen to use the pre-compiled operators and element_storage types - - Manage the AOT-compiled artifacts with keys for the system - - Functions to compile the artifacts and embed them - - - - - - - - - + - [x] Successfully compile existing CUDF JIT kernels whilst reducing the binary size of the JIT kernels + - [x] Precompiled and JIT-ed compute_columns_kernel; reduce register pressure and ...: + - [ ] Make AST codegen to use the pre-compiled operators and element_storage types + - [ ] Manage the AOT-compiled artifacts with keys for the system + - [x] Functions to compile the artifacts and embed them + - [ ] Add sample pre-compiled UDF fragments + - [ ] Build matrix tables that can be used for lookup/testing to check existence of fragments - [ ] Store mangled identifier names of the fragments From aa3f3c3b96cea653bc32df7963dc2e49f1b7199f Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 20 Apr 2026 20:49:37 +0000 Subject: [PATCH 143/254] clean up diff --- cpp/src/transform/transform.cu | 46 +------------------ .../integration/unary_transform_test.cpp | 1 - 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 067bdfcceb7c..6cef7413fa6c 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -158,7 +158,6 @@ using handle = std::variant< namespace jit_transform { -<<<<<<< HEAD cudf::kernel instantiate(null_aware is_null_aware, bool has_user_data, std::string const& ins, @@ -167,16 +166,6 @@ cudf::kernel instantiate(null_aware is_null_aware, std::vector const& ptx_output_types, std::string const& udf, udf_source_type source_type) -======= -jitify2::Kernel instantiate(null_aware is_null_aware, - bool has_user_data, - std::string const& ins, - std::string const& outs, - std::vector const& ptx_input_types, - std::vector const& ptx_output_types, - std::string const& udf, - udf_source_type source_type) ->>>>>>> upstream/main { CUDF_FUNC_RANGE(); auto cuda_source = (source_type == udf_source_type::PTX) @@ -186,7 +175,6 @@ jitify2::Kernel instantiate(null_aware is_null_aware, jit::build_ptx_params(ptx_output_types, ptx_input_types, has_user_data)) : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); -<<<<<<< HEAD auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", rtcx::reflect_enum("cudf::null_aware", is_null_aware), rtcx::reflect_bool(has_user_data), @@ -200,35 +188,19 @@ jitify2::Kernel instantiate(null_aware is_null_aware, void launch(cudf::kernel const& kernel, size_type row_size, bitmask_type const* stencil, - bool stencil_has_nulls, -======= - auto kernel = jitify2::reflection::Template("cudf::jit::transform_kernel") - .instantiate(is_null_aware, has_user_data, ins, outs); - - return jit::get_udf_kernel( - *transform_jit_kernel_cu_jit, kernel, cuda_source, {"-restrict", "--dopt=on"}); -} - -void launch(jitify2::Kernel const& kernel, - size_type row_size, - bitmask_type const* stencil, ->>>>>>> upstream/main void* user_data, column_device_view_core const* input_cols, mutable_column_device_view_core const* output_cols, rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); -<<<<<<< HEAD void* args[] = {&row_size, &stencil, &stencil_has_nulls, &user_data, &input_cols, &output_cols}; auto kernel_ref = kernel.get(); auto cfg = kernel_ref.max_occupancy_config(0, 0); // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior kernel_ref.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); -======= void* args[] = {&row_size, &stencil, &user_data, &input_cols, &output_cols}; kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); ->>>>>>> upstream/main } std::string reflect_input_element(column_view const& c) { return type_to_name(c.type()); } @@ -280,16 +252,11 @@ auto reflect(udf_source_type source_type, auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); bool as_scalar = std::holds_alternative(in); -<<<<<<< HEAD auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", rtcx::reflect_int(i), column, element, rtcx::reflect_bool(as_scalar)); -======= - auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate(i, column, element, as_scalar); ->>>>>>> upstream/main in_types.push_back(accessor); } @@ -300,27 +267,17 @@ auto reflect(udf_source_type source_type, auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); bool as_scalar = false; // never scalar -<<<<<<< HEAD auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", rtcx::reflect_int(i), column, element, rtcx::reflect_bool(as_scalar)); -======= - auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate(i, column, element, as_scalar); ->>>>>>> upstream/main out_types.push_back(accessor); } -<<<<<<< HEAD auto ins = rtcx::reflect_template("cudf::jit::type_list", in_types); auto outs = rtcx::reflect_template("cudf::jit::type_list", out_types); -======= - auto ins = jitify2::reflection::Template("cudf::jit::type_list").instantiate(in_types); - auto outs = jitify2::reflection::Template("cudf::jit::type_list").instantiate(out_types); ->>>>>>> upstream/main std::vector ptx_in_types; std::vector ptx_out_types; @@ -401,8 +358,7 @@ void run(null_aware is_null_aware, auto* input_cols = reinterpret_cast(cols.data()); auto* output_cols = reinterpret_cast(input_cols + inputs.size()); - return launch( - kernel, row_size, d_stencil, user_data, input_cols, output_cols, stream); + return launch(kernel, row_size, d_stencil, user_data, input_cols, output_cols, stream); } } // namespace jit_transform diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index f2046aba651e..d7c476e35b97 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include From 6682abd27cb91547794bb256db580e5f75006d4c Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 20 Apr 2026 20:53:16 +0000 Subject: [PATCH 144/254] Remove unused kernel argument configuration in launch function --- cpp/src/transform/transform.cu | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 6cef7413fa6c..bc8a70d3a99e 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -199,8 +199,6 @@ void launch(cudf::kernel const& kernel, auto cfg = kernel_ref.max_occupancy_config(0, 0); // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior kernel_ref.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); - void* args[] = {&row_size, &stencil, &user_data, &input_cols, &output_cols}; - kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); } std::string reflect_input_element(column_view const& c) { return type_to_name(c.type()); } From d9f41852f0f497a0c57ccaead7a102737229dfc5 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 20 Apr 2026 22:29:44 +0000 Subject: [PATCH 145/254] update embed script --- cpp/CMakeLists.txt | 18 +- cpp/src/librtcx/embed.cmake | 106 +++++++--- cpp/src/librtcx/embed.hpp | 341 ++++++++++++++++++++++++++++++++ cpp/src/librtcx/embed.in.cpp | 348 +-------------------------------- cpp/src/transform/transform.cu | 6 +- 5 files changed, 440 insertions(+), 379 deletions(-) create mode 100644 cpp/src/librtcx/embed.hpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f87b99c0926a..bbc8b279f150 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -326,37 +326,37 @@ endif() set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/libcxx DEST_DIRECTORY librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf DEST_DIRECTORY cudf/cpp/include/cudf INCLUDE_DIRECTORIES cudf/cpp/include ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/jit DEST_DIRECTORY cudf/cpp/src/jit INCLUDE_DIRECTORIES cudf/cpp/src ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/binaryop/jit DEST_DIRECTORY cudf/cpp/src/binaryop/jit INCLUDE_DIRECTORIES cudf/cpp/src ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/join/jit DEST_DIRECTORY cudf/cpp/src/join/jit INCLUDE_DIRECTORIES cudf/cpp/src ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling DEST_DIRECTORY cudf/cpp/src/rolling INCLUDE_DIRECTORIES cudf/cpp/src ) -jit_add_include_directory( +embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/transform/jit DEST_DIRECTORY cudf/cpp/src/transform/jit INCLUDE_DIRECTORIES cudf/cpp/src ) @@ -366,13 +366,13 @@ get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLU foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) cmake_path(GET INC_DIR FILENAME INC_DIR_NAME) - jit_add_include_directory( + embed_includes( cudf_jit_embed COPY_DIRECTORY ${INC_DIR} DEST_DIRECTORY CCCL/libcudacxx/${INC_DIR_NAME} INCLUDE_DIRECTORIES CCCL/libcudacxx/${INC_DIR_NAME} ) endforeach() -jit_embed(cudf_jit_embed COMPRESSION zstd) +embed(cudf_jit_embed COMPRESSION zstd) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- diff --git a/cpp/src/librtcx/embed.cmake b/cpp/src/librtcx/embed.cmake index b8aa28704b72..4c6a6723d036 100644 --- a/cpp/src/librtcx/embed.cmake +++ b/cpp/src/librtcx/embed.cmake @@ -14,7 +14,7 @@ endif() # This function registers a directory of include files to be embedded for JIT compilation. It # gathers the specified files, their destinations, and include directories, and stores them in # target-specific variables for later use when generating the embed. -function(jit_add_include_directory) +function(embed_includes) set(TARGET ${ARGV0}) set(OPTIONS "") set(ONE_VALUE_ARGS COPY_DIRECTORY # Source directory where files will be copied from @@ -66,9 +66,9 @@ function(jit_add_include_directory) # Set scope variables to accumulate results - set(SOURCE_FILES ${${TARGET}__jitembed_incdir__source_files}) - set(SOURCE_FILE_DESTS ${${TARGET}__jitembed_incdir__source_file_dests}) - set(INCLUDE_DIRECTORIES ${${TARGET}__jitembed_incdir__include_directories}) + set(SOURCE_FILES ${${TARGET}__embed__source_files}) + set(SOURCE_FILE_DESTS ${${TARGET}__embed__source_file_dests}) + set(INCLUDE_DIRECTORIES ${${TARGET}__embed__include_directories}) foreach(SOURCE_FILE IN LISTS ARG_FILES) list(APPEND SOURCE_FILES "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") @@ -77,23 +77,72 @@ function(jit_add_include_directory) list(APPEND INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES}) - set(${TARGET}__jitembed_incdir__source_files + set(${TARGET}__embed__source_files ${SOURCE_FILES} PARENT_SCOPE ) - set(${TARGET}__jitembed_incdir__source_file_dests + set(${TARGET}__embed__source_file_dests ${SOURCE_FILE_DESTS} PARENT_SCOPE ) - set(${TARGET}__jitembed_incdir__include_directories + set(${TARGET}__embed__include_directories ${INCLUDE_DIRECTORIES} PARENT_SCOPE ) endfunction() +function(embed_blob) + set(TARGET ${ARGV0}) + set(OPTIONS) + set(ONE_VALUE_ARGS FILE DEST) + set(MULTI_VALUE_ARGS "") + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET argument is required") + endif() + + if(NOT ARG_FILE) + message(FATAL_ERROR "FILE argument is required") + endif() + + if(NOT ARG_DEST) + message(FATAL_ERROR "DEST argument is required") + endif() + + set(SOURCE_FILES ${${TARGET}__embed__source_files}) + set(SOURCE_FILE_DESTS ${${TARGET}__embed__source_file_dests}) + set(TARGET_DEPS ${${TARGET}__embed__target_deps}) + + if(ARG_FILE MATCHES "\\$]+)>") + # If the file is a generator expression for target objects add as dependency + list(APPEND TARGET_DEPS $) + else() + if(NOT EXISTS "${ARG_FILE}") + message(FATAL_ERROR "Source file '${ARG_FILE}' does not exist") + endif() + endif() + list(APPEND SOURCE_FILES ${ARG_FILE}) + list(APPEND SOURCE_FILE_DESTS ${ARG_DEST}) + + set(${TARGET}__embed__source_files + ${SOURCE_FILES} + PARENT_SCOPE + ) + set(${TARGET}__embed__source_file_dests + ${SOURCE_FILE_DESTS} + PARENT_SCOPE + ) + set(${TARGET}__embed__target_deps + ${TARGET_DEPS} + PARENT_SCOPE + ) + +endfunction() + # pass the encoded args to the embed.py script to generate the embed -function(jit_embed) +function(embed) set(TARGET ${ARGV0}) set(OPTIONS "") set(ONE_VALUE_ARGS "COMPRESSION") @@ -112,27 +161,30 @@ function(jit_embed) message(FATAL_ERROR "COMPRESSION argument must be either none or zstd") endif() - if(NOT DEFINED ${TARGET}__jitembed_incdir__source_files) - message( - FATAL_ERROR - "No source files registered for target '${TARGET}'. Call jit_add_include_directory() first" - ) + if(NOT DEFINED ${TARGET}__embed__source_files) + message(FATAL_ERROR "No source files registered for target '${TARGET}'") endif() set(OUTPUT_DIR "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") - set(CONFIGURED_EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed.cpp") - set(EMBED_SCRIPT_IN "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.cpp") - - set(RTCX_EMBED_SCRIPT_ARG__ID "${TARGET}") - set(RTCX_EMBED_SCRIPT_ARG__FILE_PATHS "${${TARGET}__jitembed_incdir__source_files}") - set(RTCX_EMBED_SCRIPT_ARG__FILE_DESTS "${${TARGET}__jitembed_incdir__source_file_dests}") - set(RTCX_EMBED_SCRIPT_ARG__INCLUDE_DIRS "${${TARGET}__jitembed_incdir__include_directories}") - set(RTCX_EMBED_SCRIPT_ARG__COMPRESSION "${ARG_COMPRESSION}") - set(RTCX_EMBED_SCRIPT_ARG__OUTPUT_DIR "${OUTPUT_DIR}") - - configure_file("${EMBED_SCRIPT_IN}" "${CONFIGURED_EMBED_SCRIPT}" @ONLY) + set(EMBED_SCRIPT_TEMPLATE "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.cpp") + set(CONFIGURED_EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed_cfg.cpp") + set(EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed.cpp") + + set(EMBED_SCRIPT__ID "${TARGET}") + set(EMBED_SCRIPT__FILE_PATHS "${${TARGET}__embed__source_files}") + set(EMBED_SCRIPT__FILE_DESTS "${${TARGET}__embed__source_file_dests}") + set(EMBED_SCRIPT__INCLUDE_DIRS "${${TARGET}__embed__include_directories}") + set(EMBED_SCRIPT__COMPRESSION "${ARG_COMPRESSION}") + set(EMBED_SCRIPT__OUTPUT_DIR "${OUTPUT_DIR}") + + configure_file(${EMBED_SCRIPT_TEMPLATE} ${CONFIGURED_EMBED_SCRIPT} @ONLY) + file( + GENERATE + OUTPUT "${EMBED_SCRIPT}" + INPUT "${CONFIGURED_EMBED_SCRIPT}" + ) - add_executable("${TARGET}__jit_embed_run" EXCLUDE_FROM_ALL "${CONFIGURED_EMBED_SCRIPT}") + add_executable("${TARGET}__jit_embed_run" EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") target_include_directories("${TARGET}__jit_embed_run" PRIVATE ${ZSTD_INCLUDE_DIR}) target_link_libraries("${TARGET}__jit_embed_run" PRIVATE ${CMAKE_DL_LIBS} zstd) set_target_properties( @@ -144,8 +196,8 @@ function(jit_embed) OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin BYPRODUCTS ${OUTPUT_DIR}/* COMMAND "${CMAKE_COMMAND}" -E env $ - DEPENDS "${EMBED_SCRIPT_IN}" "${CONFIGURED_EMBED_SCRIPT}" - ${${TARGET}__jitembed_incdir__source_files} + DEPENDS "${CONFIGURED_EMBED_SCRIPT}" "${EMBED_SCRIPT}" ${${TARGET}__embed__source_files} + ${${TARGET}__embed__target_deps} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" COMMENT "Generating JIT embed for ${TARGET} into ${OUTPUT_DIR}" VERBATIM diff --git a/cpp/src/librtcx/embed.hpp b/cpp/src/librtcx/embed.hpp new file mode 100644 index 000000000000..82749b1740d5 --- /dev/null +++ b/cpp/src/librtcx/embed.hpp @@ -0,0 +1,341 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include "sha256.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEFER__CONCATENATE_DETAIL(x, y) x##y +#define DEFER__CONCATENATE(x, y) DEFER__CONCATENATE_DETAIL(x, y) +#define DEFER(...) ::defer DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) + +template +struct defer { + private: + T func_; + + public: + template + defer(Args&&... args) : func_{static_cast(args)...} + { + } + defer(defer const&) = delete; + defer& operator=(defer const&) = delete; + defer(defer&&) = delete; + defer& operator=(defer&&) = delete; + ~defer() { func_(); } +}; + +template +defer(T) -> defer; + +struct size_range { + size_t offset = 0; + size_t size = 0; +}; + +std::pair, std::vector> merge_bytes_with_null_terminators( + std::span const> bytes_lists) +{ + std::vector merged; + std::vector ranges; + + for (auto& byte_data : bytes_lists) { + ranges.push_back({merged.size(), byte_data.size()}); + merged.insert(merged.end(), byte_data.begin(), byte_data.end()); + merged.push_back(0); + } + + return {std::move(merged), std::move(ranges)}; +} + +struct embed_output { + std::string cxx_header; + std::string cxx_source; + std::string asm_source; + std::string bin_file_name; + std::vector bin_file_data; +}; + +std::vector load_file_bytes(std::string_view file_path) +{ + std::string path_str(file_path); + std::ifstream file(path_str, std::ios::binary | std::ios::ate); + if (!file) { + throw std::runtime_error(std::format("Failed to open file at path: {}", file_path)); + } + auto file_size = file.tellg(); + if (file_size < 0) { + throw std::runtime_error( + std::format("Failed to determine size of file at path: {}", file_path)); + } + file.seekg(0, std::ios::beg); + std::vector bytes(file_size); + if (!file.read(reinterpret_cast(bytes.data()), file_size)) { + throw std::runtime_error(std::format("Failed to read file at path: {}", file_path)); + } + return bytes; +} + +std::vector compress_bytes(std::span bytes, std::string_view compression) +{ + if (compression != "none" && compression != "zstd") { + throw std::invalid_argument(std::format( + "Invalid compression type: {}. Supported values are 'none' and 'zstd'", compression)); + } + + if (compression == "none") { return std::vector(bytes.begin(), bytes.end()); } + + auto const max_compressed_size = ZSTD_compressBound(bytes.size()); + std::vector compressed(max_compressed_size); + auto const compressed_size = + ZSTD_compress(compressed.data(), compressed.size(), bytes.data(), bytes.size(), 22); + + if (ZSTD_isError(compressed_size)) { + throw std::runtime_error(std::string("ZSTD compression failed: ") + + ZSTD_getErrorName(compressed_size)); + } + + compressed.resize(compressed_size); + return compressed; +} + +rtcx::sha256 compute_embed_hash(std::span uncompressed_files_bytes, + std::span merged_dests_bytes, + std::span merged_include_dirs_bytes, + std::string_view compression) +{ + rtcx::sha256_context ctx; + ctx.update(uncompressed_files_bytes); + ctx.update(merged_dests_bytes); + ctx.update(merged_include_dirs_bytes); + ctx.update(std::span{reinterpret_cast(compression.data()), compression.size()}); + return ctx.finalize(); +} + +template +std::string join_formatted(Container& items, std::string_view delimiter, Formatter&& formatter) +{ + std::ostringstream result; + for (std::size_t i = 0; i < items.size(); ++i) { + if (i != 0) { result << delimiter; } + result << formatter(items[i]); + } + return result.str(); +} + +embed_output generate_cxx_source_files_data(std::string_view id, + std::span file_paths, + std::span file_dsts, + std::span include_dirs, + std::string_view compression) +{ + std::vector> file_bytes; + file_bytes.reserve(file_paths.size()); + for (auto const& path : file_paths) { + file_bytes.emplace_back(load_file_bytes(path)); + } + + auto [uncompressed_files_bytes, files_ranges] = merge_bytes_with_null_terminators(file_bytes); + + auto compress = compression != "none"; + std::vector compressed_files_bytes = + compress ? compress_bytes(uncompressed_files_bytes, compression) : uncompressed_files_bytes; + + auto binary_size = compress ? compressed_files_bytes.size() + : static_cast(uncompressed_files_bytes.size()); + + if (compress) { + std::cout << std::format( + "-- Compressed {}'s binary from {} bytes to {} bytes (compression ratio: {:.2f})\n", + id, + uncompressed_files_bytes.size(), + compressed_files_bytes.size(), + static_cast(compressed_files_bytes.size()) / + static_cast(uncompressed_files_bytes.size())); + } + + std::vector> destination_bytes; + destination_bytes.reserve(file_dsts.size()); + for (auto const& dest : file_dsts) { + destination_bytes.emplace_back(dest.begin(), dest.end()); + } + auto [merged_dests_bytes, _] = merge_bytes_with_null_terminators(destination_bytes); + + std::vector> include_directory_bytes; + include_directory_bytes.reserve(include_dirs.size()); + for (auto const& include_directory : include_dirs) { + include_directory_bytes.emplace_back(include_directory.begin(), include_directory.end()); + } + auto [merged_include_dirs_bytes, __] = merge_bytes_with_null_terminators(include_directory_bytes); + + auto hash = compute_embed_hash( + uncompressed_files_bytes, merged_dests_bytes, merged_include_dirs_bytes, compression); + + auto binary_file_name = std::format("embed_{}.bin", id); + + auto include_dirs_list = + join_formatted(include_dirs, ",\n", [](auto s) { return std::format("\"{}\"", s); }); + auto dests_list = + join_formatted(file_dsts, ",\n", [](auto s) { return std::format("\"{}\"", s); }); + auto ranges_list = join_formatted( + files_ranges, ",\n", [](auto r) { return std::format("{{{}, {}}}", r.offset, r.size); }); + + auto hash_list = join_formatted( + hash, ", ", [](uint8_t byte) { return std::format("0x{:02x}", static_cast(byte)); }); + + auto cxx_header = std::format( + R"***( +// Auto-generated header for embedded files with ID: {} +#pragma once + +#include +#include +#include +#include + +namespace rtcx_embed {{ + +struct range {{ + std::size_t offset = 0; + std::size_t size = 0; +}}; + +constexpr char const * {}_include_directories[{}] = +{{ +{} +}}; + +constexpr char const * {}_file_destinations[{}] = +{{ +{} +}}; + +constexpr range {}_file_ranges[{}] = +{{ +{} +}}; + +constexpr std::size_t {}_files_uncompressed_size = {}; + +constexpr char const * {}_files_compression = "{}"; + +extern "C" std::uint8_t const rtcx_embed_{}_files_begin[]; + +static std::span const {}_files = +{{ +rtcx_embed_{}_files_begin, +{}L +}}; + +constexpr std::uint8_t {}_hash[{}] = +{{ +{} +}}; + +}} +)***", + id, + id, + include_dirs.size(), + include_dirs_list, + id, + file_dsts.size(), + dests_list, + id, + files_ranges.size(), + ranges_list, + id, + uncompressed_files_bytes.size(), + id, + compression, + id, + id, + id, + binary_size, + id, + hash.size(), + hash_list); + + auto asm_source = std::format( + R"***( +.section .rodata +.global rtcx_embed_{}_files_begin +rtcx_embed_{}_files_begin: +.incbin "{}" + +.section .note.GNU-stack,"",@progbits +)***", + id, + id, + binary_file_name); + + return embed_output{ + .cxx_header = cxx_header, + .cxx_source = "", + .asm_source = asm_source, + .bin_file_name = binary_file_name, + .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes}; +} + +void generate_embed(std::string_view id, + std::span file_paths, + std::span file_dsts, + std::span include_dirs, + std::string_view compression, + std::string_view output_directory) +{ + auto output = + generate_cxx_source_files_data(id, file_paths, file_dsts, include_dirs, compression); + + std::filesystem::create_directories(std::filesystem::path(output_directory)); + + std::ofstream header_file(std::format("{}/{}.hpp", output_directory, id)); + header_file << output.cxx_header; + + std::ofstream asm_file(std::format("{}/{}.s", output_directory, id)); + asm_file << output.asm_source; + + std::ofstream bin_file(std::format("{}/{}", output_directory, output.bin_file_name), + std::ios::binary); + bin_file.write(reinterpret_cast(output.bin_file_data.data()), + static_cast(output.bin_file_data.size())); +} + +std::vector split_string(std::string_view str, char delimiter) +{ + std::vector tokens; + std::size_t start = 0; + + while (start <= str.size()) { + auto const pos = str.find(delimiter, start); + if (pos == std::string_view::npos) { + tokens.push_back(str.substr(start)); + break; + } + tokens.push_back(str.substr(start, pos - start)); + start = pos + 1; + } + + return tokens; +} diff --git a/cpp/src/librtcx/embed.in.cpp b/cpp/src/librtcx/embed.in.cpp index e666d5bc1a41..a94af51bb680 100644 --- a/cpp/src/librtcx/embed.in.cpp +++ b/cpp/src/librtcx/embed.in.cpp @@ -3,350 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "sha256.hpp" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define DEFER__CONCATENATE_DETAIL(x, y) x##y -#define DEFER__CONCATENATE(x, y) DEFER__CONCATENATE_DETAIL(x, y) -#define DEFER(...) ::defer DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) - -template -struct defer { - private: - T func_; - - public: - template - defer(Args&&... args) : func_{static_cast(args)...} - { - } - defer(defer const&) = delete; - defer& operator=(defer const&) = delete; - defer(defer&&) = delete; - defer& operator=(defer&&) = delete; - ~defer() { func_(); } -}; - -template -defer(T) -> defer; - -struct size_range { - size_t offset = 0; - size_t size = 0; -}; - -std::pair, std::vector> merge_bytes_with_null_terminators( - std::span const> bytes_lists) -{ - std::vector merged; - std::vector ranges; - - for (auto& byte_data : bytes_lists) { - ranges.push_back({merged.size(), byte_data.size()}); - merged.insert(merged.end(), byte_data.begin(), byte_data.end()); - merged.push_back(0); - } - - return {std::move(merged), std::move(ranges)}; -} - -struct embed_output { - std::string cxx_header; - std::string cxx_source; - std::string asm_source; - std::string bin_file_name; - std::vector bin_file_data; -}; - -std::vector load_file_bytes(std::string_view file_path) -{ - std::string path_str(file_path); - std::ifstream file(path_str, std::ios::binary | std::ios::ate); - if (!file) { - throw std::runtime_error(std::format("Failed to open file at path: {}", file_path)); - } - auto file_size = file.tellg(); - if (file_size < 0) { - throw std::runtime_error( - std::format("Failed to determine size of file at path: {}", file_path)); - } - file.seekg(0, std::ios::beg); - std::vector bytes(file_size); - if (!file.read(reinterpret_cast(bytes.data()), file_size)) { - throw std::runtime_error(std::format("Failed to read file at path: {}", file_path)); - } - return bytes; -} - -std::vector compress_bytes(std::span bytes, std::string_view compression) -{ - if (compression != "none" && compression != "zstd") { - throw std::invalid_argument(std::format( - "Invalid compression type: {}. Supported values are 'none' and 'zstd'", compression)); - } - - if (compression == "none") { return std::vector(bytes.begin(), bytes.end()); } - - auto const max_compressed_size = ZSTD_compressBound(bytes.size()); - std::vector compressed(max_compressed_size); - auto const compressed_size = - ZSTD_compress(compressed.data(), compressed.size(), bytes.data(), bytes.size(), 22); - - if (ZSTD_isError(compressed_size)) { - throw std::runtime_error(std::string("ZSTD compression failed: ") + - ZSTD_getErrorName(compressed_size)); - } - - compressed.resize(compressed_size); - return compressed; -} - -rtcx::sha256 compute_embed_hash(std::span uncompressed_files_bytes, - std::span merged_dests_bytes, - std::span merged_include_dirs_bytes, - std::string_view compression) -{ - rtcx::sha256_context ctx; - ctx.update(uncompressed_files_bytes); - ctx.update(merged_dests_bytes); - ctx.update(merged_include_dirs_bytes); - ctx.update(std::span{reinterpret_cast(compression.data()), compression.size()}); - return ctx.finalize(); -} - -template -std::string join_formatted(Container& items, std::string_view delimiter, Formatter&& formatter) -{ - std::ostringstream result; - for (std::size_t i = 0; i < items.size(); ++i) { - if (i != 0) { result << delimiter; } - result << formatter(items[i]); - } - return result.str(); -} - -embed_output generate_cxx_source_files_data(std::string_view id, - std::span file_paths, - std::span file_dsts, - std::span include_dirs, - std::string_view compression) -{ - std::vector> file_bytes; - file_bytes.reserve(file_paths.size()); - for (auto const& path : file_paths) { - file_bytes.emplace_back(load_file_bytes(path)); - } - - auto [uncompressed_files_bytes, files_ranges] = merge_bytes_with_null_terminators(file_bytes); - - auto compress = compression != "none"; - std::vector compressed_files_bytes = - compress ? compress_bytes(uncompressed_files_bytes, compression) : uncompressed_files_bytes; - - auto binary_size = compress ? compressed_files_bytes.size() - : static_cast(uncompressed_files_bytes.size()); - - if (compress) { - std::cout << std::format( - "-- Compressed {}'s binary from {} bytes to {} bytes (compression ratio: {:.2f})\n", - id, - uncompressed_files_bytes.size(), - compressed_files_bytes.size(), - static_cast(compressed_files_bytes.size()) / - static_cast(uncompressed_files_bytes.size())); - } - - std::vector> destination_bytes; - destination_bytes.reserve(file_dsts.size()); - for (auto const& dest : file_dsts) { - destination_bytes.emplace_back(dest.begin(), dest.end()); - } - auto [merged_dests_bytes, _] = merge_bytes_with_null_terminators(destination_bytes); - - std::vector> include_directory_bytes; - include_directory_bytes.reserve(include_dirs.size()); - for (auto const& include_directory : include_dirs) { - include_directory_bytes.emplace_back(include_directory.begin(), include_directory.end()); - } - auto [merged_include_dirs_bytes, __] = merge_bytes_with_null_terminators(include_directory_bytes); - - auto hash = compute_embed_hash( - uncompressed_files_bytes, merged_dests_bytes, merged_include_dirs_bytes, compression); - - auto binary_file_name = std::format("embed_{}.bin", id); - - auto include_dirs_list = - join_formatted(include_dirs, ",\n", [](auto s) { return std::format("\"{}\"", s); }); - auto dests_list = - join_formatted(file_dsts, ",\n", [](auto s) { return std::format("\"{}\"", s); }); - auto ranges_list = join_formatted( - files_ranges, ",\n", [](auto r) { return std::format("{{{}, {}}}", r.offset, r.size); }); - - auto hash_list = join_formatted( - hash, ", ", [](uint8_t byte) { return std::format("0x{:02x}", static_cast(byte)); }); - - auto cxx_header = std::format( - R"***( -// Auto-generated header for embedded files with ID: {} -#pragma once - -#include -#include -#include -#include - -namespace rtcx_embed {{ - -struct range {{ - std::size_t offset = 0; - std::size_t size = 0; -}}; - -constexpr char const * {}_include_directories[{}] = -{{ -{} -}}; - -constexpr char const * {}_file_destinations[{}] = -{{ -{} -}}; - -constexpr range {}_file_ranges[{}] = -{{ -{} -}}; - -constexpr std::size_t {}_files_uncompressed_size = {}; - -constexpr char const * {}_files_compression = "{}"; - -extern "C" std::uint8_t const rtcx_embed_{}_files_begin[]; - -static std::span const {}_files = -{{ -rtcx_embed_{}_files_begin, -{}L -}}; - -constexpr std::uint8_t {}_hash[{}] = -{{ -{} -}}; - -}} -)***", - id, - id, - include_dirs.size(), - include_dirs_list, - id, - file_dsts.size(), - dests_list, - id, - files_ranges.size(), - ranges_list, - id, - uncompressed_files_bytes.size(), - id, - compression, - id, - id, - id, - binary_size, - id, - hash.size(), - hash_list); - - auto asm_source = std::format( - R"***( -.section .rodata -.global rtcx_embed_{}_files_begin -rtcx_embed_{}_files_begin: -.incbin "{}" - -.section .note.GNU-stack,"",@progbits -)***", - id, - id, - binary_file_name); - - return embed_output{ - .cxx_header = cxx_header, - .cxx_source = "", - .asm_source = asm_source, - .bin_file_name = binary_file_name, - .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes}; -} - -void generate_embed(std::string_view id, - std::span file_paths, - std::span file_dsts, - std::span include_dirs, - std::string_view compression, - std::string_view output_directory) -{ - auto output = - generate_cxx_source_files_data(id, file_paths, file_dsts, include_dirs, compression); - - std::filesystem::create_directories(std::filesystem::path(output_directory)); - - std::ofstream header_file(std::format("{}/{}.hpp", output_directory, id)); - header_file << output.cxx_header; - - std::ofstream asm_file(std::format("{}/{}.s", output_directory, id)); - asm_file << output.asm_source; - - std::ofstream bin_file(std::format("{}/{}", output_directory, output.bin_file_name), - std::ios::binary); - bin_file.write(reinterpret_cast(output.bin_file_data.data()), - static_cast(output.bin_file_data.size())); -} - -std::vector split_string(std::string_view str, char delimiter) -{ - std::vector tokens; - std::size_t start = 0; - - while (start <= str.size()) { - auto const pos = str.find(delimiter, start); - if (pos == std::string_view::npos) { - tokens.push_back(str.substr(start)); - break; - } - tokens.push_back(str.substr(start, pos - start)); - start = pos + 1; - } - - return tokens; -} +#include "embed.hpp" int main() { - std::string_view id = "@RTCX_EMBED_SCRIPT_ARG__ID@"; - auto file_paths = split_string("@RTCX_EMBED_SCRIPT_ARG__FILE_PATHS@", ';'); - auto file_dests = split_string("@RTCX_EMBED_SCRIPT_ARG__FILE_DESTS@", ';'); - auto include_directories = split_string("@RTCX_EMBED_SCRIPT_ARG__INCLUDE_DIRS@", ';'); - std::string_view compression = "@RTCX_EMBED_SCRIPT_ARG__COMPRESSION@"; - std::string_view output_dir = "@RTCX_EMBED_SCRIPT_ARG__OUTPUT_DIR@"; + std::string_view id = "@EMBED_SCRIPT__ID@"; + auto file_paths = split_string("@EMBED_SCRIPT__FILE_PATHS@", ';'); + auto file_dests = split_string("@EMBED_SCRIPT__FILE_DESTS@", ';'); + auto include_directories = split_string("@EMBED_SCRIPT__INCLUDE_DIRS@", ';'); + std::string_view compression = "@EMBED_SCRIPT__COMPRESSION@"; + std::string_view output_dir = "@EMBED_SCRIPT__OUTPUT_DIR@"; generate_embed(id, file_paths, file_dests, include_directories, compression, output_dir); return EXIT_SUCCESS; diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index bc8a70d3a99e..c91a4bc86004 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -194,10 +194,12 @@ void launch(cudf::kernel const& kernel, rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - void* args[] = {&row_size, &stencil, &stencil_has_nulls, &user_data, &input_cols, &output_cols}; + void* args[] = {&row_size, &stencil, &user_data, &input_cols, &output_cols}; auto kernel_ref = kernel.get(); auto cfg = kernel_ref.max_occupancy_config(0, 0); - // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior + CUDF_EXPECTS(cfg.block_size % cudf::detail::warp_size == 0, + "Expected block size to be a multiple of warp size", + std::runtime_error); kernel_ref.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } From 8a9878c24c7768b5c99bb4bb08f7014a8eca7ad9 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 20 Apr 2026 22:58:35 +0000 Subject: [PATCH 146/254] update docs --- cpp/src/librtcx/embed.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/librtcx/embed.cmake b/cpp/src/librtcx/embed.cmake index 4c6a6723d036..f4128f61c6d5 100644 --- a/cpp/src/librtcx/embed.cmake +++ b/cpp/src/librtcx/embed.cmake @@ -11,9 +11,7 @@ if(NOT TARGET zstd) ) endif() -# This function registers a directory of include files to be embedded for JIT compilation. It -# gathers the specified files, their destinations, and include directories, and stores them in -# target-specific variables for later use when generating the embed. +# This function registers a directory of include files to be embedded for JIT compilation. function(embed_includes) set(TARGET ${ARGV0}) set(OPTIONS "") @@ -92,6 +90,7 @@ function(embed_includes) endfunction() +# This function registers a single file to be embedded for JIT compilation. function(embed_blob) set(TARGET ${ARGV0}) set(OPTIONS) @@ -141,7 +140,8 @@ function(embed_blob) endfunction() -# pass the encoded args to the embed.py script to generate the embed +# This function generates the necessary files and build targets to embed the registered source files +# for JIT compilation. function(embed) set(TARGET ${ARGV0}) set(OPTIONS "") From 43f9cd06edac51a8c50a25ffbabc4ab9930aa469 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 20 Apr 2026 23:24:26 +0000 Subject: [PATCH 147/254] disable excessive linter warnings --- cpp/src/binaryop/binaryop.cpp | 2 +- cpp/src/jit/jit.hpp | 2 +- cpp/src/librtcx/rtcx.hpp | 12 ++++++------ cpp/src/librtcx/sha256.hpp | 16 ++++++++-------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index ba4647ea1595..4b3928ffa011 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -167,7 +167,7 @@ void binary_operation(mutable_column_view& out, auto lhs_arg = cudf::jit::get_data_ptr(lhs); auto rhs_arg = cudf::jit::get_data_ptr(rhs); - void* args[] = {&size_arg, &out_arg, &lhs_arg, &rhs_arg}; + void* args[] = {&size_arg, &out_arg, &lhs_arg, &rhs_arg}; // NOLINT(modernize-avoid-c-arrays) auto cfg = kernel.max_occupancy_config(0, 0); kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index 6a0c6f877915..18e5e95157ef 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -66,7 +66,7 @@ struct [[nodiscard]] kernel { rmm::cuda_stream_view stream, Args&&... args) { - void const* params[] = {&args...}; + void const* params[] = {&args...}; // NOLINT(modernize-avoid-c-arrays) launch(grid_dim, block_dim, shared_mem_bytes, stream, const_cast(params)); } }; diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 581d5212af57..4b91de62a187 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -27,11 +27,11 @@ #define RTCX_DEFER(...) ::rtcx::defer RTCX_DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) extern "C" { -typedef struct evp_md_ctx_st EVP_MD_CTX; +typedef struct evp_md_ctx_st EVP_MD_CTX; // NOLINT(modernize-use-using) -typedef struct CUlib_st* CUlibrary; -typedef struct CUkern_st* CUkernel; -typedef struct CUstream_st* CUstream; +typedef struct CUlib_st* CUlibrary; // NOLINT(modernize-use-using) +typedef struct CUkern_st* CUkernel; // NOLINT(modernize-use-using) +typedef struct CUstream_st* CUstream; // NOLINT(modernize-use-using) } namespace rtcx { @@ -110,7 +110,7 @@ struct [[nodiscard]] sha256_hasher { constexpr std::uint64_t operator()(sha256 const& obj) const { struct u64x4 { - alignas(16) std::uint64_t v[4]; + alignas(16) std::uint64_t v[4]; // NOLINT(modernize-avoid-c-arrays) }; auto value = std::bit_cast(obj); @@ -588,7 +588,7 @@ using library_compile_func = func()>; * debugging. The interface is zero-copy, using shared pointers, mmap, and spans to avoid * unnecessary data copying across threads and disk. */ -struct cache_t { +struct cache_t { // NOLINT private: bool enabled_; diff --git a/cpp/src/librtcx/sha256.hpp b/cpp/src/librtcx/sha256.hpp index 0e04d7d2189e..1d78b1829dd7 100644 --- a/cpp/src/librtcx/sha256.hpp +++ b/cpp/src/librtcx/sha256.hpp @@ -53,7 +53,7 @@ std::uint32_t get_be32(void const* ptr) } // namespace struct [[nodiscard]] sha256_hex_string { - char data_[65]; + char data_[65]; // NOLINT(modernize-avoid-c-arrays) [[nodiscard]] constexpr std::string_view view() const { return std::string_view{data_, 64}; } @@ -67,7 +67,7 @@ struct [[nodiscard]] sha256_hex_string { static sha256_hex_string make(std::span input) { - constexpr char const HEX_CHARS[] = "0123456789abcdef"; + constexpr char const HEX_CHARS[] = "0123456789abcdef"; // NOLINT(modernize-avoid-c-arrays) sha256_hex_string hex; for (std::size_t i = 0; i < 32; ++i) { hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; @@ -79,7 +79,7 @@ struct [[nodiscard]] sha256_hex_string { }; struct [[nodiscard]] sha256 { - alignas(16) std::uint8_t data_[32]; + alignas(16) std::uint8_t data_[32]; // NOLINT(modernize-avoid-c-arrays) [[nodiscard]] std::uint8_t operator[](std::size_t index) const { return data_[index]; } @@ -120,9 +120,9 @@ struct sha256_context { 0x510e'527ful, 0x9b05'688cul, 0x1f83'd9abul, - 0x5be0'cd19ul}; + 0x5be0'cd19ul}; // NOLINT(modernize-avoid-c-arrays) std::uint64_t size_ = 0; - std::uint8_t buf_[BLOCK_SIZE] = {}; + std::uint8_t buf_[BLOCK_SIZE] = {}; // NOLINT(modernize-avoid-c-arrays) public: sha256_context() = default; @@ -135,7 +135,7 @@ struct sha256_context { private: void transform_(std::uint8_t const* buf) { - std::uint32_t S[8], W[64], t0, t1; + std::uint32_t S[8], W[64], t0, t1; // NOLINT(modernize-avoid-c-arrays) int i; /* copy state into S */ @@ -258,8 +258,8 @@ struct sha256_context { sha256 finalize() { - static std::uint8_t const pad[64] = {0x80}; - std::uint32_t padlen[2]; + static std::uint8_t const pad[64] = {0x80}; // NOLINT(modernize-avoid-c-arrays) + std::uint32_t padlen[2]; // NOLINT(modernize-avoid-c-arrays) int i; /* Pad with a binary 1 (ie 0x80), then zeroes, then length */ From b7c9f3c9ddb210afa616ea10c849c49552e75490 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 20 Apr 2026 23:45:05 +0000 Subject: [PATCH 148/254] fix lints --- cpp/src/librtcx/rtcx.hpp | 3 ++- cpp/src/librtcx/sha256.hpp | 21 +++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 4b91de62a187..462488ce9526 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -963,7 +963,8 @@ template requires((true && ... && std::is_constructible_v)) std::string reflect_template(std::string_view template_name, TemplateArgs&&... template_args) { - std::string_view const tparams[sizeof...(TemplateArgs)] = {std::string_view{template_args}...}; + std::string_view const tparams[sizeof...(TemplateArgs)] = // NOLINT(modernize-avoid-c-arrays) + {std::string_view{template_args}...}; return reflect_template(template_name, tparams); } diff --git a/cpp/src/librtcx/sha256.hpp b/cpp/src/librtcx/sha256.hpp index 1d78b1829dd7..70bf31fe5e13 100644 --- a/cpp/src/librtcx/sha256.hpp +++ b/cpp/src/librtcx/sha256.hpp @@ -113,16 +113,17 @@ struct [[nodiscard]] sha256 { struct sha256_context { private: static constexpr size_t BLOCK_SIZE = 64; - std::uint32_t state_[8] = {0x6a09'e667ul, - 0xbb67'ae85ul, - 0x3c6e'f372ul, - 0xa54f'f53aul, - 0x510e'527ful, - 0x9b05'688cul, - 0x1f83'd9abul, - 0x5be0'cd19ul}; // NOLINT(modernize-avoid-c-arrays) - std::uint64_t size_ = 0; - std::uint8_t buf_[BLOCK_SIZE] = {}; // NOLINT(modernize-avoid-c-arrays) + std::uint32_t state_[8] = // NOLINT(modernize-avoid-c-arrays) + {0x6a09'e667ul, + 0xbb67'ae85ul, + 0x3c6e'f372ul, + 0xa54f'f53aul, + 0x510e'527ful, + 0x9b05'688cul, + 0x1f83'd9abul, + 0x5be0'cd19ul}; + std::uint64_t size_ = 0; + std::uint8_t buf_[BLOCK_SIZE] = {}; // NOLINT(modernize-avoid-c-arrays) public: sha256_context() = default; From d0e86fd76619ba027d36b0edc8e1da1c8271faa1 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 21 Apr 2026 00:22:52 +0000 Subject: [PATCH 149/254] refactor: change class declarations to struct for cache_t and jit_bundle_t --- cpp/src/runtime/context.hpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 5e6485a26b89..44bc3701edc5 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -12,16 +12,12 @@ #include namespace rtcx { -class cache_t; +struct cache_t; } // namespace rtcx namespace cudf { -namespace jit { -class program_cache; -} - -class jit_bundle_t; +struct jit_bundle_t; struct [[nodiscard]] context_config { bool dump_codegen : 1 = false; From 067123e12f3e8b216b900fa99120a13134df5a28 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 21 Apr 2026 00:24:05 +0000 Subject: [PATCH 150/254] refactor: improve formatting and modernize array declarations in get_udf_kernel --- cpp/src/jit/helpers.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index f087ecfb1fa3..3f152da22531 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -97,13 +97,14 @@ kernel get_udf_kernel(std::string const& source_file, { CUDF_FUNC_RANGE(); - auto kernel_instance_source = std::format(R"***( + auto kernel_instance_source = std::format(R"***( #define KERNEL_INSTANCE {} )***", kernel_name); - char const* include_names[] = {"cudf/detail/operation-udf.hpp", - "cudf/detail/kernel-instance.hpp"}; - char const* include_headers[] = {udf_cuda_source.c_str(), kernel_instance_source.c_str()}; + char const* include_names[] = // NOLINT(modernize-avoid-c-arrays) + {"cudf/detail/operation-udf.hpp", "cudf/detail/kernel-instance.hpp"}; + char const* include_headers[] = // NOLINT(modernize-avoid-c-arrays) + {udf_cuda_source.c_str(), kernel_instance_source.c_str()}; constexpr int min_pch_cuda_version = 12800; // CUDA 12.8 constexpr int min_minimal_cuda_version = 12800; // CUDA 12.8 From 57d1192808a9f7f26dfbe8e229d713642b9b4cef Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 21 Apr 2026 10:29:17 +0000 Subject: [PATCH 151/254] removed static linking to CUDA libs and fixed lint errors --- cpp/CMakeLists.txt | 8 ++++---- cpp/src/librtcx/rtcx.cpp | 18 ++++++++++-------- cpp/src/librtcx/rtcx.hpp | 2 +- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bbc8b279f150..edc0228c1a12 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -909,9 +909,6 @@ add_library( ) add_dependencies(cudf cudf_jit_embed) -target_include_directories( - cudf PRIVATE ${cudf_jit_embed_INCLUDE_DIRS} ${CMAKE_CURRENT_LIST_DIR}/src/librtcx -) foreach(incbin_dir IN LISTS cudf_jit_embed_INCLUDE_DIRS) target_compile_options(cudf PRIVATE $<$:-Wa,-I${incbin_dir}>) @@ -995,6 +992,9 @@ target_include_directories( "$" "$" "$" + "$" + "$" + "$" INTERFACE "$" ) @@ -1038,7 +1038,7 @@ target_link_libraries( cudf PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ PRIVATE $ cuco::cuco ZLIB::ZLIB nvcomp::nvcomp - kvikio::kvikio nanoarrow::nanoarrow zstd CUDA::nvJitLink CUDA::nvrtc CUDA::cuda_driver + kvikio::kvikio nanoarrow::nanoarrow zstd ) # Add Conda library, and include paths if specified diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 62ac79725b26..796b094bd0aa 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -303,7 +303,7 @@ struct LibCuda { static void* _load() { - std::string lib_names[] = {"libcuda.so"}; + std::string lib_names[] = {"libcuda.so"}; // NOLINT(modernize-avoid-c-arrays) return load_dll("libcuda.so", lib_names); } @@ -338,9 +338,10 @@ struct LibNVRTC { std::int32_t major = major_version(cuda_version); std::int32_t minor = minor_version(cuda_version); - std::string lib_names[] = {std::format("libnvrtc.so.{}.{}", major, minor), - std::format("libnvrtc.so.{}", major), - "libnvrtc.so"}; + std::string lib_names[] = // NOLINT(modernize-avoid-c-arrays) + {std::format("libnvrtc.so.{}.{}", major, minor), + std::format("libnvrtc.so.{}", major), + "libnvrtc.so"}; return load_dll("libnvrtc.so", lib_names); } @@ -377,9 +378,10 @@ struct LibNVJitLink { std::int32_t major = major_version(cuda_version); std::int32_t minor = minor_version(cuda_version); - std::string lib_names[] = {std::format("libnvJitLink.so.{}.{}", major, minor), - std::format("libnvJitLink.so.{}", major), - "libnvJitLink.so"}; + std::string lib_names[] = // NOLINT(modernize-avoid-c-arrays) + {std::format("libnvJitLink.so.{}.{}", major, minor), + std::format("libnvJitLink.so.{}", major), + "libnvJitLink.so"}; return load_dll("libnvJitLink.so", lib_names); } @@ -972,7 +974,7 @@ std::pair, std::vector> get_d if (ent->d_type == DT_REG) { auto path = std::format("{}/{}", cache_dir, ent->d_name); struct stat st; - if (::stat(path.c_str(), &st) == -1 && errno != ENOENT) { + if (::stat(path.c_str(), &st) == -1) { throw_posix("Failed to get RTCX cache file stats", "stat"); } diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/src/librtcx/rtcx.hpp index 462488ce9526..6c9162f2a14a 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/src/librtcx/rtcx.hpp @@ -606,7 +606,7 @@ struct cache_t { // NOLINT detail::cache_stats_counter counter_; - alignas(CACHELINE_ALIGNMENT) std::uint64_t tick_; + alignas(CACHELINE_ALIGNMENT) std::uint64_t tick_; // NOLINT(modernize-use-default-member-init) public: /** From fae6e2c6fd8981be44f6c20f8ffc04ea5e593924 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 21 Apr 2026 11:02:49 +0000 Subject: [PATCH 152/254] removed invalid linter warnings --- cpp/src/librtcx/rtcx.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/src/librtcx/rtcx.cpp index 796b094bd0aa..4363e1961318 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/src/librtcx/rtcx.cpp @@ -1009,7 +1009,7 @@ void evict_disk_entries(std::string const& cache_dir, std::uint32_t limit) std::iota(ranking_indices.begin(), ranking_indices.end(), 0); std::sort(ranking_indices.begin(), ranking_indices.end(), [&](auto a, auto b) { - return access_times[a] < access_times[b]; + return access_times[a] < access_times[b]; // NOLINT(clang-analyzer-core.CallAndMessage) }); // evict half of the least recently accessed @@ -1270,7 +1270,7 @@ void cache_t::preload_from_disk() ranking_indices.resize(paths.size()); std::iota(ranking_indices.begin(), ranking_indices.end(), 0); std::sort(ranking_indices.begin(), ranking_indices.end(), [&](auto a, auto b) { - return access_times[a] > access_times[b]; + return access_times[a] > access_times[b]; // NOLINT(clang-analyzer-core.CallAndMessage) }); auto load_count = std::min(ranking_indices.size(), From 9ae509674d15acdece93b2314fa8d85ea4d38122 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 22 Apr 2026 14:00:17 +0000 Subject: [PATCH 153/254] refactoring + code review suggestions --- .devcontainer/Dockerfile | 2 +- cpp/CMakeLists.txt | 8 ++--- .../detail/floating_conversion.hpp | 9 ++--- cpp/include/cudf/strings/string_view.hpp | 19 ++++------ cpp/include/cudf/types.hpp | 12 ++----- cpp/{src => }/librtcx/README.md | 0 cpp/{src => }/librtcx/embed.cmake | 0 cpp/{src => }/librtcx/embed.hpp | 26 ++------------ cpp/librtcx/embed.in.cpp | 20 +++++++++++ cpp/{src => }/librtcx/libcxx/cassert | 0 cpp/{src => }/librtcx/libcxx/climits | 0 cpp/{src => }/librtcx/libcxx/cstddef | 0 cpp/{src => }/librtcx/libcxx/cstdint | 0 cpp/{src => }/librtcx/rtcx.cpp | 9 ++--- cpp/{src => }/librtcx/rtcx.hpp | 2 -- cpp/{src => }/librtcx/sha256.hpp | 8 +++-- cpp/src/binaryop/jit/kernel.cu | 10 +++--- cpp/src/jit/helpers.cpp | 2 +- cpp/src/jit/jit.cpp | 22 ++++-------- cpp/src/jit/jit.hpp | 2 +- cpp/src/join/jit/filter_join_kernel.cu | 14 ++++---- cpp/src/librtcx/embed.in.cpp | 19 ---------- cpp/src/librtcx/libcxx/README.md | 0 cpp/src/rolling/jit/kernel.cu | 36 +++++++++---------- cpp/src/runtime/context.cpp | 17 ++++----- cpp/src/transform/jit/kernel.cu | 4 +-- 26 files changed, 96 insertions(+), 145 deletions(-) rename cpp/{src => }/librtcx/README.md (100%) rename cpp/{src => }/librtcx/embed.cmake (100%) rename cpp/{src => }/librtcx/embed.hpp (93%) create mode 100644 cpp/librtcx/embed.in.cpp rename cpp/{src => }/librtcx/libcxx/cassert (100%) rename cpp/{src => }/librtcx/libcxx/climits (100%) rename cpp/{src => }/librtcx/libcxx/cstddef (100%) rename cpp/{src => }/librtcx/libcxx/cstdint (100%) rename cpp/{src => }/librtcx/rtcx.cpp (99%) rename cpp/{src => }/librtcx/rtcx.hpp (99%) rename cpp/{src => }/librtcx/sha256.hpp (99%) delete mode 100644 cpp/src/librtcx/embed.in.cpp delete mode 100644 cpp/src/librtcx/libcxx/README.md diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index de4001d30e48..ae32991823f6 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -32,7 +32,7 @@ ENV PYTHONUNBUFFERED="1" ENV PYTHONDONTWRITEBYTECODE="1" ENV HISTFILE="/home/coder/.cache/._bash_history" -ENV LIBCUDF_ROOT_DIR="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/libcudf" +ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/libcudf" ### # sccache configuration diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3da580141f0f..8a0c6012b8fc 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -304,7 +304,7 @@ include(cmake/thirdparty/get_thread_pool.cmake) include(cmake/thirdparty/get_zstd.cmake) # JIT Embedding helper functions -include(src/librtcx/embed.cmake) +include(librtcx/embed.cmake) # Workaround until https://github.com/rapidsai/rapids-cmake/issues/176 is resolved if(NOT BUILD_SHARED_LIBS) @@ -327,7 +327,7 @@ set(CUDF_CXX_STANDARD 20) set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) embed_includes( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/libcxx DEST_DIRECTORY + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/librtcx/libcxx DEST_DIRECTORY librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx ) @@ -905,7 +905,7 @@ add_library( src/utilities/traits.cpp src/utilities/type_checks.cpp src/utilities/type_dispatcher.cpp - src/librtcx/rtcx.cpp + librtcx/rtcx.cpp ${cudf_jit_embed_SOURCE_DIR}/cudf_jit_embed.s ) @@ -993,7 +993,7 @@ target_include_directories( "$" "$" "$" - "$" + "$" "$" "$" INTERFACE "$" diff --git a/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp b/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp index 45b198565940..fecc1b6b330d 100644 --- a/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp +++ b/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp @@ -10,7 +10,6 @@ #include #include -#include #include #include #include @@ -118,9 +117,7 @@ struct floating_converter { CUDF_HOST_DEVICE inline static IntegralType bit_cast_to_integer(FloatingType floating) { // Convert floating to integer - IntegralType integer_rep; - cuda::std::memcpy(&integer_rep, &floating, sizeof(floating)); - return integer_rep; + return cuda::std::bit_cast(floating); } /** @@ -132,9 +129,7 @@ struct floating_converter { CUDF_HOST_DEVICE inline static FloatingType bit_cast_to_floating(IntegralType integer) { // Convert back to float - FloatingType floating; - cuda::std::memcpy(&floating, &integer, sizeof(floating)); - return floating; + return cuda::std::bit_cast(integer); } /** diff --git a/cpp/include/cudf/strings/string_view.hpp b/cpp/include/cudf/strings/string_view.hpp index ddb859bb72e0..25dd26743a68 100644 --- a/cpp/include/cudf/strings/string_view.hpp +++ b/cpp/include/cudf/strings/string_view.hpp @@ -6,12 +6,10 @@ #include -#if defined(__CUDACC_RTC__) #include -#else -#include -#include +#if !defined(__CUDACC_RTC__) +#include #endif /** @@ -68,16 +66,11 @@ class string_view { class const_iterator { /// @cond public: - using difference_type = ptrdiff_t; - using value_type = char_utf8; - using reference = char_utf8&; - using pointer = char_utf8*; - -#if defined(__CUDACC_RTC__) + using difference_type = ptrdiff_t; + using value_type = char_utf8; + using reference = char_utf8&; + using pointer = char_utf8*; using iterator_category = cuda::std::input_iterator_tag; -#else - using iterator_category = std::input_iterator_tag; -#endif __device__ inline const_iterator(string_view const& str, size_type pos); const_iterator(const_iterator const& mit) = default; diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp index 42139898c573..63c8794f29f0 100644 --- a/cpp/include/cudf/types.hpp +++ b/cpp/include/cudf/types.hpp @@ -27,16 +27,12 @@ #include +#include + #include #include #include -#if defined(__CUDACC_RTC__) -#include -#else -#include -#endif - /** * @file * @brief Type declarations for libcudf. @@ -103,11 +99,7 @@ using char_utf8 = uint32_t; ///< UTF-8 characters are 1-4 bytes template size_type distance(T f, T l) { -#if defined(__CUDACC_RTC__) return static_cast(cuda::std::distance(f, l)); -#else - return static_cast(std::distance(f, l)); -#endif } /** diff --git a/cpp/src/librtcx/README.md b/cpp/librtcx/README.md similarity index 100% rename from cpp/src/librtcx/README.md rename to cpp/librtcx/README.md diff --git a/cpp/src/librtcx/embed.cmake b/cpp/librtcx/embed.cmake similarity index 100% rename from cpp/src/librtcx/embed.cmake rename to cpp/librtcx/embed.cmake diff --git a/cpp/src/librtcx/embed.hpp b/cpp/librtcx/embed.hpp similarity index 93% rename from cpp/src/librtcx/embed.hpp rename to cpp/librtcx/embed.hpp index 82749b1740d5..d8f9e6b3a2d8 100644 --- a/cpp/src/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -25,29 +25,7 @@ #include #include -#define DEFER__CONCATENATE_DETAIL(x, y) x##y -#define DEFER__CONCATENATE(x, y) DEFER__CONCATENATE_DETAIL(x, y) -#define DEFER(...) ::defer DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) - -template -struct defer { - private: - T func_; - - public: - template - defer(Args&&... args) : func_{static_cast(args)...} - { - } - defer(defer const&) = delete; - defer& operator=(defer const&) = delete; - defer(defer&&) = delete; - defer& operator=(defer&&) = delete; - ~defer() { func_(); } -}; - -template -defer(T) -> defer; +namespace rtcx_embed { struct size_range { size_t offset = 0; @@ -339,3 +317,5 @@ std::vector split_string(std::string_view str, char delimiter) return tokens; } + +} // namespace rtcx_embed diff --git a/cpp/librtcx/embed.in.cpp b/cpp/librtcx/embed.in.cpp new file mode 100644 index 000000000000..0a391e202508 --- /dev/null +++ b/cpp/librtcx/embed.in.cpp @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "embed.hpp" + +int main() +{ + std::string_view id = "@EMBED_SCRIPT__ID@"; + auto file_paths = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_PATHS@", ';'); + auto file_dests = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_DESTS@", ';'); + auto include_directories = rtcx_embed::split_string("@EMBED_SCRIPT__INCLUDE_DIRS@", ';'); + std::string_view compression = "@EMBED_SCRIPT__COMPRESSION@"; + std::string_view output_dir = "@EMBED_SCRIPT__OUTPUT_DIR@"; + + rtcx_embed::generate_embed( + id, file_paths, file_dests, include_directories, compression, output_dir); + return EXIT_SUCCESS; +} diff --git a/cpp/src/librtcx/libcxx/cassert b/cpp/librtcx/libcxx/cassert similarity index 100% rename from cpp/src/librtcx/libcxx/cassert rename to cpp/librtcx/libcxx/cassert diff --git a/cpp/src/librtcx/libcxx/climits b/cpp/librtcx/libcxx/climits similarity index 100% rename from cpp/src/librtcx/libcxx/climits rename to cpp/librtcx/libcxx/climits diff --git a/cpp/src/librtcx/libcxx/cstddef b/cpp/librtcx/libcxx/cstddef similarity index 100% rename from cpp/src/librtcx/libcxx/cstddef rename to cpp/librtcx/libcxx/cstddef diff --git a/cpp/src/librtcx/libcxx/cstdint b/cpp/librtcx/libcxx/cstdint similarity index 100% rename from cpp/src/librtcx/libcxx/cstdint rename to cpp/librtcx/libcxx/cstdint diff --git a/cpp/src/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp similarity index 99% rename from cpp/src/librtcx/rtcx.cpp rename to cpp/librtcx/rtcx.cpp index 4363e1961318..4b1b5de6f404 100644 --- a/cpp/src/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -419,9 +419,9 @@ void initialize() void teardown() { std::call_once(*teardown_libraries_flag, [] { - cu.reset(); - nvrtc.reset(); nvjitlink.reset(); + nvrtc.reset(); + cu.reset(); init_libraries_flag.reset(); teardown_libraries_flag.reset(); }); @@ -589,6 +589,7 @@ void log_nvJitLink_result(link_params const& params, } } // namespace + byte_buffer compile(compile_params const& params) { RTCX_EXPECTS(params.name != nullptr, "Fragment name must not be null", std::logic_error); @@ -831,10 +832,6 @@ std::string demangle_cuda_symbol(char const* mangled_name) RTCX_EXPECTS(status == 0, "Demangling CUDA symbol name failed", std::runtime_error); RTCX_EXPECTS(demangled_name != nullptr, "Demangling CUDA symbol name failed", std::runtime_error); - RTCX_DEFER([&] { - if (demangled_name != nullptr) ::free(demangled_name); - }); - std::string result{demangled_name}; return result; diff --git a/cpp/src/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp similarity index 99% rename from cpp/src/librtcx/rtcx.hpp rename to cpp/librtcx/rtcx.hpp index 6c9162f2a14a..0708a6fec2c2 100644 --- a/cpp/src/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -27,8 +27,6 @@ #define RTCX_DEFER(...) ::rtcx::defer RTCX_DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) extern "C" { -typedef struct evp_md_ctx_st EVP_MD_CTX; // NOLINT(modernize-use-using) - typedef struct CUlib_st* CUlibrary; // NOLINT(modernize-use-using) typedef struct CUkern_st* CUkernel; // NOLINT(modernize-use-using) typedef struct CUstream_st* CUstream; // NOLINT(modernize-use-using) diff --git a/cpp/src/librtcx/sha256.hpp b/cpp/librtcx/sha256.hpp similarity index 99% rename from cpp/src/librtcx/sha256.hpp rename to cpp/librtcx/sha256.hpp index 70bf31fe5e13..9d88bd57f3a6 100644 --- a/cpp/src/librtcx/sha256.hpp +++ b/cpp/librtcx/sha256.hpp @@ -134,7 +134,7 @@ struct sha256_context { ~sha256_context() = default; private: - void transform_(std::uint8_t const* buf) + void transform(std::uint8_t const* buf) { std::uint32_t S[8], W[64], t0, t1; // NOLINT(modernize-avoid-c-arrays) int i; @@ -222,6 +222,8 @@ struct sha256_context { RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 62, 0xbef9'a3f7); RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 63, 0xc671'78f2); +#undef RND + for (i = 0; i < 8; i++) { state_[i] += S[i]; } @@ -245,11 +247,11 @@ struct sha256_context { len -= left; data = (data + left); if (len_buf) return; - transform_(buf_); + transform(buf_); } while (len >= 64) { - transform_(data); + transform(data); data = data + 64; len -= 64; } diff --git a/cpp/src/binaryop/jit/kernel.cu b/cpp/src/binaryop/jit/kernel.cu index 7e0c47d8d112..85965c1187e9 100644 --- a/cpp/src/binaryop/jit/kernel.cu +++ b/cpp/src/binaryop/jit/kernel.cu @@ -77,10 +77,10 @@ CUDF_KERNEL void binaryop_kernel(cudf::size_type size, } // namespace binops } // namespace cudf -extern "C" __global__ void kernel(cudf::size_type size, - void* out_data, - void* lhs_data, - void* rhs_data) +extern "C" __global__ void cudf_kernel(cudf::size_type size, + void* out_data, + void* lhs_data, + void* rhs_data) { - KERNEL_INSTANCE(size, out_data, lhs_data, rhs_data); + CUDF_KERNEL_INSTANCE(size, out_data, lhs_data, rhs_data); } diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 3f152da22531..9c245b9dea82 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -98,7 +98,7 @@ kernel get_udf_kernel(std::string const& source_file, CUDF_FUNC_RANGE(); auto kernel_instance_source = std::format(R"***( -#define KERNEL_INSTANCE {} +#define CUDF_KERNEL_INSTANCE {} )***", kernel_name); char const* include_names[] = // NOLINT(modernize-avoid-c-arrays) diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index f48cb601b21a..6f5ac4098b98 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include @@ -97,6 +97,10 @@ rtcx::byte_buffer read_blob_cstring(char const* path) throw_posix(std::format("Failed to reset file offset for file ({})", path), "lseek"); } + CUDF_EXPECTS(file_size < std::numeric_limits::max() - 1, + std::format("File ({}) size is too large to read into memory", path), + std::runtime_error); + auto contents = rtcx::byte_buffer::make(file_size + 1U); // +1 for null terminator if (read(fd, contents.data(), file_size) == -1) { @@ -299,18 +303,6 @@ std::tuple compile_library_uncached( options.emplace_back(std::format("-I{}", include_dir)); } - // TODO: experiment with: - // --fdevice-time-trace=jit_comp_trace.json - // --time=compile_trace.json - // --restrict - // --relocatable-device-code - // --extensible-whole-program - // --dlink-time-opt - // --gen-opt-lto - // --create-pch - // --use-pch - // --pch-dir - options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); options.emplace_back("--diag-suppress=47"); @@ -428,14 +420,14 @@ kernel_instance={} if (!use_cache) { auto [lib, blob] = compile(); - return kernel{lib, lib->get_kernel("kernel")}; + return kernel{lib, lib->get_kernel("cudf_kernel")}; } auto fut = cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); auto lib = fut.get(); - return kernel{lib, lib->get_kernel("kernel")}; + return kernel{lib, lib->get_kernel("cudf_kernel")}; } } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index 18e5e95157ef..54b752256bb3 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -8,7 +8,7 @@ #include -#include +#include namespace CUDF_EXPORT cudf { diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index ccd02fc7cd9c..3cc8a72a14c5 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -95,13 +95,13 @@ CUDF_KERNEL void filter_join_kernel(cudf::jit::device_span left_indices, - cudf::jit::device_span right_indices, - cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - bool* predicate_results, - void* user_data) +extern "C" __global__ void cudf_kernel(cudf::jit::device_span left_indices, + cudf::jit::device_span right_indices, + cudf::column_device_view_core const* left_tables, + cudf::column_device_view_core const* right_tables, + bool* predicate_results, + void* user_data) { - KERNEL_INSTANCE( + CUDF_KERNEL_INSTANCE( left_indices, right_indices, left_tables, right_tables, predicate_results, user_data); } diff --git a/cpp/src/librtcx/embed.in.cpp b/cpp/src/librtcx/embed.in.cpp deleted file mode 100644 index a94af51bb680..000000000000 --- a/cpp/src/librtcx/embed.in.cpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "embed.hpp" - -int main() -{ - std::string_view id = "@EMBED_SCRIPT__ID@"; - auto file_paths = split_string("@EMBED_SCRIPT__FILE_PATHS@", ';'); - auto file_dests = split_string("@EMBED_SCRIPT__FILE_DESTS@", ';'); - auto include_directories = split_string("@EMBED_SCRIPT__INCLUDE_DIRS@", ';'); - std::string_view compression = "@EMBED_SCRIPT__COMPRESSION@"; - std::string_view output_dir = "@EMBED_SCRIPT__OUTPUT_DIR@"; - - generate_embed(id, file_paths, file_dests, include_directories, compression, output_dir); - return EXIT_SUCCESS; -} diff --git a/cpp/src/librtcx/libcxx/README.md b/cpp/src/librtcx/libcxx/README.md deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index 037e64546122..61b016b23213 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -113,23 +113,23 @@ CUDF_KERNEL void rolling_window_kernel(cudf::size_type nrows, } // namespace rolling } // namespace cudf -extern "C" __global__ void kernel(cudf::size_type nrows, - void const* const __restrict__ in_col, - cudf::bitmask_type const* const __restrict__ in_col_valid, - void* __restrict__ out_col, - cudf::bitmask_type* __restrict__ out_col_valid, - cudf::size_type* __restrict__ output_valid_count, - cudf::detail::window_wrapper_base preceding_window_begin, - cudf::detail::window_wrapper_base following_window_begin, - cudf::size_type min_periods) +extern "C" __global__ void cudf_kernel(cudf::size_type nrows, + void const* const __restrict__ in_col, + cudf::bitmask_type const* const __restrict__ in_col_valid, + void* __restrict__ out_col, + cudf::bitmask_type* __restrict__ out_col_valid, + cudf::size_type* __restrict__ output_valid_count, + cudf::detail::window_wrapper_base preceding_window_begin, + cudf::detail::window_wrapper_base following_window_begin, + cudf::size_type min_periods) { - KERNEL_INSTANCE(nrows, - in_col, - in_col_valid, - out_col, - out_col_valid, - output_valid_count, - preceding_window_begin, - following_window_begin, - min_periods); + CUDF_KERNEL_INSTANCE(nrows, + in_col, + in_col_valid, + out_col, + out_col_valid, + output_valid_count, + preceding_window_begin, + following_window_begin, + min_periods); } diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index e5a4b170a010..d79e2335ec17 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -88,9 +88,9 @@ static std::optional _context{std::nullopt}; static std::optional _context_init_flag{std::in_place}; static std::optional _context_deinit_flag{std::in_place}; -std::filesystem::path get_cudf_root_dir() +std::filesystem::path get_cudf_kernel_cache_dir() { - if (auto cudf = getenv_optional("LIBCUDF_ROOT_DIR"); cudf.has_value()) { + if (auto cudf = getenv_optional("LIBCUDF_KERNEL_CACHE_PATH"); cudf.has_value()) { return std::filesystem::path(*cudf); } @@ -99,8 +99,9 @@ std::filesystem::path get_cudf_root_dir() } CUDF_FAIL( - "Unable to determine the CUDF root directory. Please set the `LIBCUDF_ROOT_DIR`, " - "`HOME` or `LIBCUDF_ROOT_DIR` environment variables to allow automatic resolution of the root " + "Unable to determine the CUDF root directory. Please set the `LIBCUDF_KERNEL_CACHE_PATH` or " + "`HOME` " + "environment variables to allow automatic resolution of the root " "directory.", std::runtime_error); } @@ -123,10 +124,10 @@ void initialize(init_flags flags) flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); - auto jit_bundle_dir = get_cudf_root_dir() / "jit" / "bundle"; - auto rtcx_cache_dir = get_cudf_root_dir() / "jit" / "rtcx_cache"; - auto jit_pch_dir = get_cudf_root_dir() / "jit" / "pch"; - auto jit_tmp_dir = get_cudf_root_dir() / "jit" / "tmp"; + auto jit_bundle_dir = get_cudf_kernel_cache_dir() / "jit" / "bundle"; + auto rtcx_cache_dir = get_cudf_kernel_cache_dir() / "jit" / "rtcx_cache"; + auto jit_pch_dir = get_cudf_kernel_cache_dir() / "jit" / "pch"; + auto jit_tmp_dir = get_cudf_kernel_cache_dir() / "jit" / "tmp"; context_config cfg{.dump_codegen = dump_codegen, .use_jit = use_jit, diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 2441c3dfb9e7..4d85b7089432 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -113,12 +113,12 @@ CUDF_KERNEL void transform_kernel(size_type row_size, } // namespace jit } // namespace cudf -extern "C" __global__ void kernel( +extern "C" __global__ void cudf_kernel( cudf::size_type row_size, cudf::bitmask_type const* __restrict__ stencil, void* __restrict__ user_data, cudf::column_device_view_core const* __restrict__ input_cols, cudf::mutable_column_device_view_core const* __restrict__ output_cols) { - KERNEL_INSTANCE(row_size, stencil, user_data, input_cols, output_cols); + CUDF_KERNEL_INSTANCE(row_size, stencil, user_data, input_cols, output_cols); } From 610beab35b4f4eaa3cee45b632a25a598604ca3b Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 22 Apr 2026 19:44:21 +0000 Subject: [PATCH 154/254] refactoring Co-authored-by: Copilot --- cpp/librtcx/rtcx.cpp | 21 ++++----------------- cpp/src/jit/helpers.cpp | 1 - 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 4b1b5de6f404..30e872524b4f 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -1079,12 +1079,7 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil std::atomic_ref tick{tick_}; auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); - bool unlocked = false; - lock_.lock(); - - RTCX_DEFER([&] { - if (!unlocked) { lock_.unlock(); } - }); + std::unique_lock lock{lock_}; // check memory cache if (auto it = enabled_ ? blobs_cache_.entries_.find(sha) : blobs_cache_.entries_.end(); @@ -1125,8 +1120,7 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil // we can release the lock while calling the maker function since it may be expensive and we // have already reserved a spot in the cache for this sha - lock_.unlock(); - unlocked = true; + lock.unlock(); auto result = compile(); promise.set_value(result); @@ -1146,12 +1140,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, std::atomic_ref tick{tick_}; auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); - bool unlocked = false; - lock_.lock(); - - RTCX_DEFER([&] { - if (!unlocked) { lock_.unlock(); } - }); + std::unique_lock lock{lock_}; // check memory cache if (auto it = enabled_ ? libraries_cache_.entries_.find(sha) : libraries_cache_.entries_.end(); @@ -1183,7 +1172,6 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, // we can release the lock while calling the maker function since it may be expensive and we // have already reserved a spot in the cache for this sha lock_.unlock(); - unlocked = true; promise.set_value(std::move(*disk_library)); @@ -1196,8 +1184,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, // we can release the lock while calling the maker function since it may be expensive and we // have already reserved a spot in the cache for this sha - lock_.unlock(); - unlocked = true; + lock.unlock(); auto [library, blob] = compile(); promise.set_value(library); diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 9c245b9dea82..5eafd44f743e 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -84,7 +84,6 @@ std::vector input_type_names( std::transform(views.begin(), views.end(), std::back_inserter(names), [&](auto const& view) { return std::visit([](auto& a) { return type_to_name(a.type()); }, view); - return std::visit([](auto& a) { return type_to_name(a.type()); }, view); }); return names; From 457ee4048d19b274fcc3e88f4b24397bcf1e40c2 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 22 Apr 2026 21:10:43 +0000 Subject: [PATCH 155/254] Remove unnecessary blank lines and clean up SPDX license headers in multiple files Co-authored-by: Copilot --- cpp/librtcx/libcxx/cassert | 1 - cpp/librtcx/libcxx/climits | 1 - cpp/librtcx/libcxx/cstddef | 1 - cpp/librtcx/libcxx/cstdint | 1 - cpp/librtcx/rtcx.cpp | 3 +-- cpp/librtcx/rtcx.hpp | 1 - cpp/librtcx/sha256.hpp | 2 -- cpp/src/jit/helpers.cpp | 2 +- cpp/src/jit/jit.cpp | 3 +-- 9 files changed, 3 insertions(+), 12 deletions(-) diff --git a/cpp/librtcx/libcxx/cassert b/cpp/librtcx/libcxx/cassert index 199c5ee6cf9c..6bb3f0766ca8 100644 --- a/cpp/librtcx/libcxx/cassert +++ b/cpp/librtcx/libcxx/cassert @@ -1,4 +1,3 @@ - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 diff --git a/cpp/librtcx/libcxx/climits b/cpp/librtcx/libcxx/climits index 58f936c6739c..20653c8fc5d8 100644 --- a/cpp/librtcx/libcxx/climits +++ b/cpp/librtcx/libcxx/climits @@ -1,4 +1,3 @@ - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 diff --git a/cpp/librtcx/libcxx/cstddef b/cpp/librtcx/libcxx/cstddef index ea13067bad5a..d0c1fb2c31d6 100644 --- a/cpp/librtcx/libcxx/cstddef +++ b/cpp/librtcx/libcxx/cstddef @@ -1,4 +1,3 @@ - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 diff --git a/cpp/librtcx/libcxx/cstdint b/cpp/librtcx/libcxx/cstdint index 4c5f82cc79f8..eae43f7203db 100644 --- a/cpp/librtcx/libcxx/cstdint +++ b/cpp/librtcx/libcxx/cstdint @@ -1,4 +1,3 @@ - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 30e872524b4f..9fd92be9862d 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -1,4 +1,3 @@ - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 @@ -11,9 +10,9 @@ #include #include #include -#include #include #include +#include #include #include #include diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 0708a6fec2c2..5e6078b386d7 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -1,4 +1,3 @@ - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 diff --git a/cpp/librtcx/sha256.hpp b/cpp/librtcx/sha256.hpp index 9d88bd57f3a6..26f12a0b6216 100644 --- a/cpp/librtcx/sha256.hpp +++ b/cpp/librtcx/sha256.hpp @@ -1,5 +1,3 @@ - - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 5eafd44f743e..ff68e5a3a928 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include namespace cudf { diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index 6f5ac4098b98..ff2d5bb401dd 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -1,4 +1,3 @@ - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 @@ -97,7 +96,7 @@ rtcx::byte_buffer read_blob_cstring(char const* path) throw_posix(std::format("Failed to reset file offset for file ({})", path), "lseek"); } - CUDF_EXPECTS(file_size < std::numeric_limits::max() - 1, + CUDF_EXPECTS(file_size < std::numeric_limits::max() - 1, std::format("File ({}) size is too large to read into memory", path), std::runtime_error); From 855b246617e1aa936323721e75e78f8595c9e75a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 22 Apr 2026 23:19:19 +0000 Subject: [PATCH 156/254] Refactor embed functions to remove unused binary file name variable and streamline file output handling Co-authored-by: Copilot --- cpp/librtcx/embed.cmake | 1 - cpp/librtcx/embed.hpp | 11 +++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index f4128f61c6d5..79e53db4efb3 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -194,7 +194,6 @@ function(embed) add_custom_command( OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin - BYPRODUCTS ${OUTPUT_DIR}/* COMMAND "${CMAKE_COMMAND}" -E env $ DEPENDS "${CONFIGURED_EMBED_SCRIPT}" "${EMBED_SCRIPT}" ${${TARGET}__embed__source_files} ${${TARGET}__embed__target_deps} diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index d8f9e6b3a2d8..968307283587 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -51,7 +51,6 @@ struct embed_output { std::string cxx_header; std::string cxx_source; std::string asm_source; - std::string bin_file_name; std::vector bin_file_data; }; @@ -170,8 +169,6 @@ embed_output generate_cxx_source_files_data(std::string_view id, auto hash = compute_embed_hash( uncompressed_files_bytes, merged_dests_bytes, merged_include_dirs_bytes, compression); - auto binary_file_name = std::format("embed_{}.bin", id); - auto include_dirs_list = join_formatted(include_dirs, ",\n", [](auto s) { return std::format("\"{}\"", s); }); auto dests_list = @@ -260,19 +257,18 @@ constexpr std::uint8_t {}_hash[{}] = .section .rodata .global rtcx_embed_{}_files_begin rtcx_embed_{}_files_begin: -.incbin "{}" +.incbin "{}.bin" .section .note.GNU-stack,"",@progbits )***", id, id, - binary_file_name); + id); return embed_output{ .cxx_header = cxx_header, .cxx_source = "", .asm_source = asm_source, - .bin_file_name = binary_file_name, .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes}; } @@ -294,8 +290,7 @@ void generate_embed(std::string_view id, std::ofstream asm_file(std::format("{}/{}.s", output_directory, id)); asm_file << output.asm_source; - std::ofstream bin_file(std::format("{}/{}", output_directory, output.bin_file_name), - std::ios::binary); + std::ofstream bin_file(std::format("{}/{}.bin", output_directory, id), std::ios::binary); bin_file.write(reinterpret_cast(output.bin_file_data.data()), static_cast(output.bin_file_data.size())); } From 2c98a355785286cb1b1097d86c3c4849f5980da0 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 23 Apr 2026 00:11:19 +0000 Subject: [PATCH 157/254] Refactor CMakeLists.txt and source files to remove unused variables and streamline includes --- cpp/CMakeLists.txt | 4 ---- cpp/librtcx/rtcx.cpp | 8 +++----- cpp/librtcx/rtcx.hpp | 7 ++++--- cpp/src/runtime/context.cpp | 2 +- cpp/src/transform/transform.cu | 1 - 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8a0c6012b8fc..4e321871cdd8 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -323,9 +323,6 @@ if(NOT BUILD_SHARED_LIBS) endif() endif() -set(CUDF_CXX_STANDARD 20) -set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) - embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/librtcx/libcxx DEST_DIRECTORY librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx @@ -1034,7 +1031,6 @@ target_compile_definitions(cudf PRIVATE $<$:CUDF_ target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_AVOID_SORT_UNROLL=1) # Specify the target module library dependencies - target_link_libraries( cudf PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 9fd92be9862d..466b1fad48b9 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -302,7 +302,7 @@ struct LibCuda { static void* _load() { - std::string lib_names[] = {"libcuda.so"}; // NOLINT(modernize-avoid-c-arrays) + std::string lib_names[] = {"libcuda.so", "libcuda.so.1"}; // NOLINT(modernize-avoid-c-arrays) return load_dll("libcuda.so", lib_names); } @@ -426,7 +426,7 @@ void teardown() }); } -blob_t blob_t::from_buffer(byte_buffer&& buffer) +blob_t blob_t::from_buffer(byte_buffer buffer) { auto size = buffer.size(); auto data = buffer.release(); @@ -831,9 +831,7 @@ std::string demangle_cuda_symbol(char const* mangled_name) RTCX_EXPECTS(status == 0, "Demangling CUDA symbol name failed", std::runtime_error); RTCX_EXPECTS(demangled_name != nullptr, "Demangling CUDA symbol name failed", std::runtime_error); - std::string result{demangled_name}; - - return result; + return std::string{demangled_name}; } namespace { diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 5e6078b386d7..d9925c86da0e 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -128,7 +128,7 @@ struct [[nodiscard]] sha256_hasher { enum class binary_type : std::int8_t { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; /** - * @brief An heap-allocated statically-sized buffer. Its contents are not guaranteed to be + * @brief A heap-allocated statically-sized buffer. Its contents are not guaranteed to be * initialized. */ template @@ -233,7 +233,8 @@ struct buffer { /** * @brief Releases ownership of the buffer's data and returns a pointer to it. After calling this - * function, the buffer is left in an empty state (data pointer is null and size is zero). + * function, the buffer is left in an empty state. The caller is responsible for calling `free()` + * on the returned pointer when it is no longer needed. * @return A pointer to the buffer's data */ [[nodiscard]] T* release() @@ -299,7 +300,7 @@ struct [[nodiscard]] blob_t { return blob_t{data, size, deallocator}; } - static blob_t from_buffer(byte_buffer&& buffer); + static blob_t from_buffer(byte_buffer buffer); static blob_t from_static_data(std::span data); diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index d79e2335ec17..dac5a5dcb8bb 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -8,7 +8,7 @@ #include "io/comp/nvcomp_adapter.hpp" #include "io/utilities/getenv_or.hpp" #include "jit/jit.hpp" -#include "librtcx/rtcx.hpp" +#include "rtcx.hpp" #include #include diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index c91a4bc86004..87e7ce1a43a9 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -29,7 +29,6 @@ #include #include #include -#include #include #include From 37d6696a8b20da3cbc088e783f0b56b69280cae0 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 23 Apr 2026 00:32:33 +0000 Subject: [PATCH 158/254] Rename environment variable for clearing JIT cache to improve clarity --- cpp/src/runtime/context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index dac5a5dcb8bb..14474d427539 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -117,7 +117,7 @@ void initialize(init_flags flags) bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); bool preload_jit_cache = get_bool_env_or("LIBCUDF_JIT_PRELOAD_CACHE", false); bool disable_jit_cache = get_bool_env_or("LIBCUDF_JIT_CACHE_DISABLED", false); - bool clear_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); + bool clear_jit_cache = get_bool_env_or("LIBCUDF_JIT_CLEAR_CACHE", false); auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16'384U); auto kernel_cache_limit_disk = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_DISK", 131'072U); From 49636ccd380d4b2e6342586bb2fe106fc66dcb76 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 23 Apr 2026 19:23:28 +0000 Subject: [PATCH 159/254] Refactoring + removed disk LRU Co-authored-by: Copilot --- cpp/librtcx/rtcx.cpp | 300 ++++++++++++------------------------ cpp/librtcx/rtcx.hpp | 73 ++++++--- cpp/src/jit/jit.cpp | 191 +++++++---------------- cpp/src/runtime/context.cpp | 7 +- cpp/src/runtime/context.hpp | 1 - 5 files changed, 206 insertions(+), 366 deletions(-) diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 466b1fad48b9..de20d23d52a5 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -3,11 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include + #include #include -#include -#include #include #include #include @@ -16,12 +16,10 @@ #include #include #include -#include -#include +#include #include #include -#include #include #include #include @@ -96,13 +94,12 @@ auto __errstr = ::std::format("(nvJitLink) expression `{}` failed, with error ({}): {}", \ #__VA_ARGS__, \ static_cast<::std::int64_t>(__result), \ - ::rtcx::get_nvJitLinkResultString(__result)); \ + ::rtcx::nvJitLinkResult_string(__result)); \ RTCX_FAIL(__errstr, ::std::runtime_error); \ } \ } while (0) namespace rtcx { - namespace { template @@ -134,22 +131,14 @@ std::string join_strings(std::span strings, std::string_view separat } // namespace -void log_warning(std::string_view message) -{ - ::fprintf( - stdout, "[RTCX WARNING] %.*s\n", static_cast(message.size()), message.data()); -} - -void log_error(std::string_view message) +void log_warning(std::string_view msg) { - ::fprintf( - stderr, "[RTCX ERROR] %.*s\n", static_cast(message.size()), message.data()); + CUDF_LOG_WARN("%.*s", static_cast(msg.size()), msg.data()); } -void log_trace(std::string_view message) +void log_error(std::string_view msg) { - ::fprintf( - stdout, "[RTCX TRACE] %.*s\n", static_cast(message.size()), message.data()); + CUDF_LOG_ERROR("%.*s", static_cast(msg.size()), msg.data()); } #define FOR_EACH_CUDA_FUNC(DO_IT) \ @@ -199,7 +188,7 @@ void log_trace(std::string_view message) namespace { -char const* get_nvJitLinkResultString(nvJitLinkResult result) +std::string_view nvJitLinkResult_string(nvJitLinkResult result) { switch (result) { case NVJITLINK_SUCCESS: return "NVJITLINK_SUCCESS"; @@ -230,7 +219,7 @@ char const* get_nvJitLinkResultString(nvJitLinkResult result) } } -char const* binary_type_string(binary_type type) +std::string_view binary_type_string(binary_type type) { switch (type) { case binary_type::LTO_IR: return "LTO_IR"; @@ -510,7 +499,7 @@ void log_nvJitLink_result(link_params const& params, if (auto errc = nvjitlink->GetInfoLogSize(handle, &info_log_size); errc != NVJITLINK_SUCCESS) { RTCX_FAIL(std::format("Failed to get nvJitLink info log size with error ({}): {}", static_cast(errc), - get_nvJitLinkResultString(errc)), + nvJitLinkResult_string(errc)), std::runtime_error); } @@ -520,7 +509,7 @@ void log_nvJitLink_result(link_params const& params, if (auto errc = nvjitlink->GetInfoLog(handle, info_log.data()); errc != NVJITLINK_SUCCESS) { RTCX_FAIL(std::format("Failed to get nvJitLink info log with error ({}): {}", static_cast(errc), - get_nvJitLinkResultString(errc)), + nvJitLinkResult_string(errc)), std::runtime_error); } } @@ -530,7 +519,7 @@ void log_nvJitLink_result(link_params const& params, if (auto errc = nvjitlink->GetErrorLogSize(handle, &error_log_size); errc != NVJITLINK_SUCCESS) { RTCX_FAIL(std::format("Failed to get nvJitLink error log size with error ({}): {}", static_cast(errc), - get_nvJitLinkResultString(errc)), + nvJitLinkResult_string(errc)), std::runtime_error); } @@ -541,7 +530,7 @@ void log_nvJitLink_result(link_params const& params, if (auto errc = nvjitlink->GetErrorLog(handle, error_log.data()); errc != NVJITLINK_SUCCESS) { RTCX_FAIL(std::format("Failed to get nvJitLink error log with error ({}): {}", static_cast(errc), - get_nvJitLinkResultString(errc)), + nvJitLinkResult_string(errc)), std::runtime_error); } } @@ -563,18 +552,16 @@ void log_nvJitLink_result(link_params const& params, link_options_str = std::format("{}\t{}\n", link_options_str, option); } - char const* binary_type_str = binary_type_string(params.output_type); - auto status_str = error_log.empty() ? "completed with warnings" : "failed with errors"; auto msg = std::format( "(nvJitLink) Linking for `{}` ({}) {}, error code ({}): {}.\nFragments: \n{}\n" "Link Options: \n{}\n\nInfo Log:\n\t{}\n\nError Log:\n\t{}\n\n", params.name == nullptr ? "" : params.name, - binary_type_str, + binary_type_string(params.output_type), status_str, static_cast(link_result), - get_nvJitLinkResultString(link_result), + nvJitLinkResult_string(link_result), fragments_str, link_options_str, std::string_view{info_log.data(), info_log.size()}, @@ -822,18 +809,6 @@ kernel_ref library_t::get_kernel(char const* name) const return kernel_ref{kernel}; } -std::string demangle_cuda_symbol(char const* mangled_name) -{ - std::int32_t status; - std::size_t length; - char* demangled_name = abi::__cxa_demangle(mangled_name, nullptr, &length, &status); - - RTCX_EXPECTS(status == 0, "Demangling CUDA symbol name failed", std::runtime_error); - RTCX_EXPECTS(demangled_name != nullptr, "Demangling CUDA symbol name failed", std::runtime_error); - - return std::string{demangled_name}; -} - namespace { [[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) @@ -937,83 +912,15 @@ std::optional get_disk_library(std::string const& cache_dir, sha256 con return std::make_shared(handle); } -std::pair, std::vector> get_disk_entries( - std::string const& cache_dir) +std::vector get_disk_entries(std::string const& cache_dir) { - std::int32_t dir = ::open(cache_dir.c_str(), O_RDONLY | O_DIRECTORY); - - if (dir == -1) { throw_posix("Failed to open RTCX cache directory for evicting", "open"); } - - RTCX_DEFER([&] { ::close(dir); }); - - std::vector buffer; - buffer.resize(8192); - - std::vector paths; - std::vector access_times; - - std::ptrdiff_t num_read = 0; - - while ((num_read = ::syscall(SYS_getdents64, dir, buffer.data(), buffer.size())) > 0) { - std::ptrdiff_t byte_pos = 0; - - while (byte_pos < num_read) { - auto* ent = reinterpret_cast(buffer.data() + byte_pos); - - if (::memcmp(ent->d_name, ".", 2) != 0 && ::memcmp(ent->d_name, "..", 3) != 0) { - RTCX_EXPECTS(ent->d_type != DT_UNKNOWN, - "Found unknown directory entry type in RTCX cache dir", - std::runtime_error); - - if (ent->d_type == DT_REG) { - auto path = std::format("{}/{}", cache_dir, ent->d_name); - struct stat st; - if (::stat(path.c_str(), &st) == -1) { - throw_posix("Failed to get RTCX cache file stats", "stat"); - } - - auto access_time = - std::chrono::seconds{st.st_atim.tv_sec} + std::chrono::nanoseconds{st.st_atim.tv_nsec}; - - paths.emplace_back(std::move(path)); - access_times.emplace_back(access_time); - } - } - - byte_pos += ent->d_reclen; - } - } - - if (num_read == -1) { - throw_posix("Failed to read RTCX cache directory for clearing", "getdents64"); + std::vector entries; + for (auto& entry : std::filesystem::directory_iterator(cache_dir)) { + if (!entry.is_regular_file()) { continue; } + entries.push_back(entry.path().string()); } - return {std::move(paths), std::move(access_times)}; -} - -void evict_disk_entries(std::string const& cache_dir, std::uint32_t limit) -{ - auto [paths, access_times] = get_disk_entries(cache_dir); - - if (paths.size() < limit) { return; } - - std::vector ranking_indices; - ranking_indices.resize(paths.size()); - - std::iota(ranking_indices.begin(), ranking_indices.end(), 0); - - std::sort(ranking_indices.begin(), ranking_indices.end(), [&](auto a, auto b) { - return access_times[a] < access_times[b]; // NOLINT(clang-analyzer-core.CallAndMessage) - }); - - // evict half of the least recently accessed - auto num_evict = (limit == 0) ? paths.size() : ((limit + 1) / 2); - - for (auto index : std::span{ranking_indices}.subspan(0, num_evict)) { - if (::unlink(paths[index].c_str()) == -1 && errno != ENOENT) { - throw_posix("Failed to evict RTCX cache file", "unlink"); - } - } + return entries; } /// @brief atomically writes a blob to disk by first writing to a temporary file and then renaming @@ -1022,51 +929,43 @@ void cache_blob_to_disk(std::string const& cache_dir, std::string const& tmp_dir, std::string const& object_type, sha256 const& sha, - std::span binary, - std::uint32_t limit) + std::span binary) { - // TODO: add cuda driver and runtime version to log - if (limit > 0) { - auto tmp_path = std::format("{}/rtcx-bin-XXXXXX", tmp_dir); - (void)tmp_path.c_str(); // to ensure null-termination for mkstemp - - { - std::int32_t fd = ::mkstemp(tmp_path.data()); - if (fd == -1) { throw_posix("Failed to create temporary file for RTCX cache", "mkstemp"); } - - RTCX_DEFER([&] { - if (::close(fd) == -1) { - throw_posix("Failed to close temporary RTCX cache file", "close"); - } - }); + auto tmp_path = std::format("{}/rtcx-bin-XXXXXX", tmp_dir); + (void)tmp_path.c_str(); // to ensure null-termination for mkstemp - if (::write(fd, binary.data(), binary.size()) == -1) { - throw_posix("Failed to write RTCX cache to temporary file", "write"); - } + { + std::int32_t fd = ::mkstemp(tmp_path.data()); + if (fd == -1) { throw_posix("Failed to create temporary file for RTCX cache", "mkstemp"); } + + RTCX_DEFER([&] { + if (::close(fd) == -1) { throw_posix("Failed to close temporary RTCX cache file", "close"); } + }); + + if (::write(fd, binary.data(), binary.size()) == -1) { + throw_posix("Failed to write RTCX cache to temporary file", "write"); } + } - auto hex = sha.to_hex_string(); - auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); + auto hex = sha.to_hex_string(); + auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); - std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); + std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); - // rename is atomic, even if another process is performing the same operation - if (::rename(tmp_path.c_str(), final_path.c_str()) == -1) { - if (errno == EEXIST) { - // another process has already created the file, so just remove our temp file - if (::remove(tmp_path.c_str()) == -1) { - throw_posix("Failed to remove temporary RTCX cache file", "remove"); - } - return; - } else { - throw_posix(std::format("Failed to move temporary RTCX cache file to final location ({})", - final_path), - "rename"); + // rename is atomic, even if another process is performing the same operation + if (::rename(tmp_path.c_str(), final_path.c_str()) == -1) { + if (errno == EEXIST) { + // another process has already created the file, so just remove our temp file + if (::remove(tmp_path.c_str()) == -1) { + throw_posix("Failed to remove temporary RTCX cache file", "remove"); } + return; + } else { + throw_posix( + std::format("Failed to move temporary RTCX cache file to final location ({})", final_path), + "rename"); } } - - evict_disk_entries(cache_dir, limit); } } // namespace @@ -1123,8 +1022,7 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil promise.set_value(result); // store result to disk - cache_blob_to_disk( - cache_dir_, tmp_dir_, "blob", sha, result->view(), limits_.num_disk_entries); + cache_blob_to_disk(cache_dir_, tmp_dir_, "blob", sha, result->view()); return ret_fut; } @@ -1187,8 +1085,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, promise.set_value(library); // store result to disk - cache_blob_to_disk( - cache_dir_, tmp_dir_, "cuLibrary", sha, blob->view(), limits_.num_disk_entries); + cache_blob_to_disk(cache_dir_, tmp_dir_, "cuLibrary", sha, blob->view()); return ret_fut; } @@ -1241,30 +1138,33 @@ void cache_t::clear_memory_store() libraries_cache_.entries_.clear(); } -void cache_t::clear_disk_store() { evict_disk_entries(cache_dir_, 0); } +void cache_t::clear_disk_store() +{ + auto entries = get_disk_entries(cache_dir_); + + for (auto const& path : entries) { + try { + std::filesystem::remove(path); + } catch (...) { + log_error(std::format("Unknown error occurred while removing RTCX cache file `{}`", path)); + } + } +} void cache_t::preload_from_disk() { - auto [paths, access_times] = get_disk_entries(cache_dir_); - - std::vector ranking_indices; - ranking_indices.resize(paths.size()); - std::iota(ranking_indices.begin(), ranking_indices.end(), 0); - std::sort(ranking_indices.begin(), ranking_indices.end(), [&](auto a, auto b) { - return access_times[a] > access_times[b]; // NOLINT(clang-analyzer-core.CallAndMessage) - }); + auto entries = get_disk_entries(cache_dir_); - auto load_count = std::min(ranking_indices.size(), - limits_.num_mem_blobs + limits_.num_mem_libraries); + auto load_count = + std::min(entries.size(), limits_.num_mem_blobs + limits_.num_mem_libraries); - ranking_indices.resize(load_count); + entries.resize(load_count); { std::lock_guard guard{lock_}; tick_++; - for (auto index : ranking_indices) { - auto path = paths[index]; + for (auto const& path : entries) { try { auto file_name = std::filesystem::path{path}.filename().string(); auto sha_str = file_name.substr(0, file_name.find('.')); @@ -1308,39 +1208,6 @@ bool cache_t::is_enabled() return enabled_; } -std::string reflect_bool(bool value) { return std::format("(bool){}", value); } - -std::string reflect_int(std::uint8_t value) { return std::format("(unsigned char){}U", value); } - -std::string reflect_int(std::uint16_t value) { return std::format("(unsigned short){}U", value); } - -std::string reflect_int(std::uint32_t value) { return std::format("(unsigned int){}U", value); } - -std::string reflect_int(std::uint64_t value) -{ - return std::format("(unsigned long long int){}ULL", value); -} - -std::string reflect_int(std::int8_t value) { return std::format("(signed char){}", value); } - -std::string reflect_int(std::int16_t value) { return std::format("(signed short){}", value); } - -std::string reflect_int(std::int32_t value) { return std::format("(signed int){}", value); } - -std::string reflect_int(std::int64_t value) -{ - return std::format("(signed long long int){}LL", value); -} - -std::string reflect_float(float value) { return std::format("(float){}F", value); } - -std::string reflect_float(double value) { return std::format("(double){}", value); } - -std::string reflect_cast(std::string_view type, std::string_view value) -{ - return std::format("(({})({}))", type, value); -} - std::string reflect_template(std::string_view template_name, std::span template_args) { @@ -1353,4 +1220,33 @@ std::string reflect_template(std::string_view template_name, return std::format("{}<{}>", template_name, join_strings(template_args, ", ")); } +rtcx::byte_buffer decompress_blob(std::span compressed_binary, + size_t uncompressed_size, + std::string_view compression) +{ + RTCX_EXPECTS(compression == "none" || compression == "zstd", + std::format("Unsupported compression type specified: {}", compression), + std::runtime_error); + auto decompressed = rtcx::byte_buffer::make(uncompressed_size); + + if (compression == "zstd") { + size_t errc = ZSTD_decompress( + decompressed.data(), uncompressed_size, compressed_binary.data(), compressed_binary.size()); + + RTCX_EXPECTS( + !ZSTD_isError(errc) && errc == uncompressed_size, + std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : ", + errc, + ZSTD_getErrorName(errc)), + std::runtime_error); + } else { + // compression is "none", so just copy the data + std::copy(compressed_binary.data(), + compressed_binary.data() + compressed_binary.size(), + decompressed.data()); + } + + return decompressed; +} + } // namespace rtcx diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index d9925c86da0e..f74ae69fbc61 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -107,7 +108,8 @@ struct [[nodiscard]] sha256_hasher { constexpr std::uint64_t operator()(sha256 const& obj) const { struct u64x4 { - alignas(16) std::uint64_t v[4]; // NOLINT(modernize-avoid-c-arrays) + alignas(alignof(sha256)) std::uint64_t // NOLINT(modernize-avoid-c-arrays) + v[sizeof(sha256) / sizeof(std::uint64_t)]; }; auto value = std::bit_cast(obj); @@ -567,7 +569,6 @@ struct [[nodiscard]] cache_stats { struct [[nodiscard]] cache_limits { std::uint32_t num_mem_blobs = 16'384; std::uint32_t num_mem_libraries = 16'384; - std::uint32_t num_disk_entries = 131'072; }; using blob_compile_func = func; @@ -777,14 +778,6 @@ struct cache_t { // NOLINT */ [[nodiscard]] byte_buffer link_library(link_params const& params); -/** - * @brief Demangle a CUDA symbol name into a human-readable form - * @param mangled_name The mangled CUDA symbol name to be demangled - * @return A string containing the demangled, human-readable symbol name corresponding to the input - * mangled name - */ -[[nodiscard]] std::string demangle_cuda_symbol(char const* mangled_name); - /** * @brief Initialize the RTCX library, setting up necessary resources and state for subsequent * operations @@ -815,7 +808,7 @@ void teardown(); * @param value The boolean value to be reflected * @return A string containing the CUDA representation of the boolean value ("true" or "false") */ -std::string reflect_bool(bool value); +inline std::string reflect_bool(bool value) { return std::format("(bool){}", value); } /** * @brief Reflect an integer value into its CUDA string representation @@ -823,7 +816,10 @@ std::string reflect_bool(bool value); * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -std::string reflect_int(std::uint8_t value); +inline std::string reflect_int(std::uint8_t value) +{ + return std::format("(unsigned char){}U", value); +} /** * @brief Reflect an integer value into its CUDA string representation @@ -831,7 +827,10 @@ std::string reflect_int(std::uint8_t value); * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -std::string reflect_int(std::uint16_t value); +inline std::string reflect_int(std::uint16_t value) +{ + return std::format("(unsigned short){}U", value); +} /** * @brief Reflect an integer value into its CUDA string representation @@ -839,7 +838,10 @@ std::string reflect_int(std::uint16_t value); * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -std::string reflect_int(std::uint32_t value); +inline std::string reflect_int(std::uint32_t value) +{ + return std::format("(unsigned int){}U", value); +} /** * @brief Reflect an integer value into its CUDA string representation @@ -847,7 +849,10 @@ std::string reflect_int(std::uint32_t value); * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -std::string reflect_int(std::uint64_t value); +inline std::string reflect_int(std::uint64_t value) +{ + return std::format("(unsigned long long int){}ULL", value); +} /** * @brief Reflect an integer value into its CUDA string representation @@ -855,7 +860,7 @@ std::string reflect_int(std::uint64_t value); * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -std::string reflect_int(std::int8_t value); +inline std::string reflect_int(std::int8_t value) { return std::format("(signed char){}", value); } /** * @brief Reflect an integer value into its CUDA string representation @@ -863,7 +868,10 @@ std::string reflect_int(std::int8_t value); * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -std::string reflect_int(std::int16_t value); +inline std::string reflect_int(std::int16_t value) +{ + return std::format("(signed short){}", value); +} /** * @brief Reflect an integer value into its CUDA string representation @@ -871,7 +879,7 @@ std::string reflect_int(std::int16_t value); * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -std::string reflect_int(std::int32_t value); +inline std::string reflect_int(std::int32_t value) { return std::format("(signed int){}", value); } /** * @brief Reflect an integer value into its CUDA string representation @@ -879,7 +887,10 @@ std::string reflect_int(std::int32_t value); * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -std::string reflect_int(std::int64_t value); +inline std::string reflect_int(std::int64_t value) +{ + return std::format("(signed long long int){}LL", value); +} /** * @brief Reflect a floating-point value into its CUDA string representation @@ -887,7 +898,7 @@ std::string reflect_int(std::int64_t value); * @param value The floating-point value to be reflected * @return A string containing the CUDA representation of the floating-point value */ -std::string reflect_float(float value); +inline std::string reflect_float(float value) { return std::format("(float){}F", value); } /** * @brief Reflect a floating-point value into its CUDA string representation @@ -895,7 +906,7 @@ std::string reflect_float(float value); * @param value The floating-point value to be reflected * @return A string containing the CUDA representation of the floating-point value */ -std::string reflect_float(double value); +inline std::string reflect_float(double value) { return std::format("(double){}", value); } /** * @brief Reflect a value of any type into its CUDA string representation, given the type name as a @@ -905,7 +916,10 @@ std::string reflect_float(double value); * resulting CUDA code * @return A string containing the CUDA representation of the value with the specified type */ -std::string reflect_cast(std::string_view type, std::string_view value); +inline std::string reflect_cast(std::string_view type, std::string_view value) +{ + return std::format("(({})({}))", type, value); +} /** * @brief Reflect an enumeration value into its CUDA string representation, given the type name as a @@ -966,4 +980,19 @@ std::string reflect_template(std::string_view template_name, TemplateArgs&&... t return reflect_template(template_name, tparams); } +/** + * @brief Decompress a compressed binary blob using the specified compression algorithm + * @param compressed_binary A span of bytes containing the compressed binary data to be decompressed + * @param uncompressed_size The expected size of the uncompressed binary data in bytes + * @param compression A string view specifying the compression algorithm used to compress the binary + * data + * @return A byte buffer containing the decompressed binary data + * @throws std::runtime_error if decompression fails due to an unsupported compression algorithm, + * invalid compressed data, or if the decompressed data size does not match the expected + * uncompressed size + */ +rtcx::byte_buffer decompress_blob(std::span compressed_binary, + size_t uncompressed_size, + std::string_view compression); + } // namespace rtcx diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index ff2d5bb401dd..b88f6ac2f805 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -9,18 +9,13 @@ #include #include -#include #include #include #include -#include -#include -#include -#include -#include #include #include +#include #include namespace CUDF_EXPORT cudf { @@ -43,137 +38,64 @@ rtcx::sha256 hash_strings(std::span inputs) return ctx.finalize(); } -[[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) +void install_file_set(std::string_view target_dir, + std::span compressed_binary, + size_t uncompressed_size, + std::span file_ranges, + std::span destinations, + std::string_view compression) { - auto error_code = errno; - auto error_str = std::format( - "{}. `{}` failed with {} ({})", message, syscall_name, error_code, std::strerror(error_code)); - CUDF_FAIL(error_str, std::runtime_error); -} + auto decompressed = rtcx::decompress_blob(compressed_binary, uncompressed_size, compression); + for (size_t i = 0; i < file_ranges.size(); ++i) { + auto file_data_range = file_ranges[i]; + auto file_data = std::span{decompressed.data() + file_data_range.offset, file_data_range.size}; + auto dst_path = destinations[i]; + auto target_path = std::format("{}/{}", target_dir, dst_path); -void install_file(char const* dst_path, std::span contents) -{ - int dst_file = open(dst_path, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); - if (dst_file == -1) { - if (errno == EEXIST) { - // file already exists, so just return - return; - } - throw_posix(std::format("Failed to create file ({})", dst_path), "open"); - } + std::filesystem::create_directories(std::filesystem::path{target_path}.parent_path()); - RTCX_DEFER([&] { - if (close(dst_file) != 0) { - throw_posix(std::format("Failed to close file ({})", dst_path), "close"); + std::ofstream file(target_path, std::ios::binary); + if (!file) { + throw std::runtime_error( + std::format("Failed to open file for writing at path: {}", target_path)); } - }); - if (write(dst_file, contents.data(), contents.size()) == -1) { - throw_posix(std::format("Failed to write file ({})", dst_path), "write"); + file.write(reinterpret_cast(file_data.data()), file_data.size()); + if (!file) { + throw std::runtime_error(std::format("Failed to write file at path: {}", target_path)); + } } } -/** - * @brief Reads the contents of a file into a byte buffer and null-terminates it to allow for safe - * usage as a C-string. - */ -rtcx::byte_buffer read_blob_cstring(char const* path) +std::string read_file_string(char const* path) { - int32_t fd = open(path, O_RDONLY); - if (fd == -1) { throw_posix(std::format("Failed to open file ({})", path), "open"); } - - RTCX_DEFER([&] { - if (close(fd) == -1) { throw_posix(std::format("Failed to close file ({})", path), "close"); } - }); - - auto file_size = lseek(fd, 0, SEEK_END); - if (file_size == -1) { - throw_posix(std::format("Failed to determine size of file ({})", path), "lseek"); - } - // TODO: make all read/write syscalls call read/write in a loop + std::ifstream file(std::string{path}, std::ios::binary | std::ios::ate); + if (!file) { throw std::runtime_error(std::format("Failed to open file at path: {}", path)); } - if (lseek(fd, 0, SEEK_SET) == -1) { - throw_posix(std::format("Failed to reset file offset for file ({})", path), "lseek"); - } - - CUDF_EXPECTS(file_size < std::numeric_limits::max() - 1, - std::format("File ({}) size is too large to read into memory", path), - std::runtime_error); - - auto contents = rtcx::byte_buffer::make(file_size + 1U); // +1 for null terminator + auto size = file.tellg(); + file.seekg(0, std::ios::beg); - if (read(fd, contents.data(), file_size) == -1) { - throw_posix(std::format("Failed to read file ({})", path), "read"); + std::string contents(size, '\0'); + if (!file.read(contents.data(), size)) { + throw std::runtime_error(std::format("Failed to read file at path: {}", path)); } - contents.data()[file_size] = '\0'; // null-terminate the buffer - return contents; } -rtcx::byte_buffer decompress_blob(std::span compressed_binary, - size_t uncompressed_size, - std::string_view compression) -{ - CUDF_EXPECTS(compression == "none" || compression == "zstd", - std::format("Unsupported compression type specified: {}", compression), - std::runtime_error); - auto decompressed = rtcx::byte_buffer::make(uncompressed_size); - - if (compression == "zstd") { - size_t errc = ZSTD_decompress( - decompressed.data(), uncompressed_size, compressed_binary.data(), compressed_binary.size()); - - CUDF_EXPECTS( - !ZSTD_isError(errc) && errc == uncompressed_size, - std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : ", - errc, - ZSTD_getErrorName(errc)), - std::runtime_error); - } else { - // compression is "none", so just copy the data - std::copy(compressed_binary.data(), - compressed_binary.data() + compressed_binary.size(), - decompressed.data()); - } - - return decompressed; -} - -void install_file_set(std::string_view target_dir, - std::span compressed_binary, - size_t uncompressed_size, - std::span file_ranges, - std::span destinations, - std::string_view compression) -{ - auto decompressed = decompress_blob(compressed_binary, uncompressed_size, compression); - auto files_data = decompressed.data(); - - for (size_t i = 0; i < file_ranges.size(); ++i) { - auto file_data_range = file_ranges[i]; - auto file_data = std::span{files_data + file_data_range.offset, file_data_range.size}; - auto dst_path = destinations[i]; - auto target_path = std::format("{}/{}", target_dir, dst_path); - - std::filesystem::create_directories(std::filesystem::path{target_path}.parent_path()); - install_file(target_path.c_str(), file_data); - } -} - void install_cudf_jit_files(std::string const& target_dir, std::string const& tmp_dir) { // directory does not exist, so create it - auto tmp_path_str = std::format("{}/cudf-jit-tmpdir_XXXXXX", tmp_dir); - (void)tmp_path_str.c_str(); // ensure null-terminated string for mkdtemp - char* tmp_path = mkdtemp(tmp_path_str.data()); - if (tmp_path == nullptr) { - throw_posix( - std::format("Failed to create temporary JIT install directory for ({})", target_dir), - "mkdtemp"); - } - - install_file_set(tmp_path, + auto tmp_dir_path_str = std::format("{}/cudf-jit-tmpdir_XXXXXX", tmp_dir); + (void)tmp_dir_path_str.c_str(); // ensure null-terminated string for mkdtemp + char* tmp_dir_path = ::mkdtemp(tmp_dir_path_str.data()); + CUDF_EXPECTS( + tmp_dir_path != nullptr, + std::format("Failed to create temporary directory for JIT file installation in tmp dir: {}", + tmp_dir), + std::runtime_error); + + install_file_set(tmp_dir_path, rtcx_embed::cudf_jit_embed_files, rtcx_embed::cudf_jit_embed_files_uncompressed_size, rtcx_embed::cudf_jit_embed_file_ranges, @@ -181,15 +103,18 @@ void install_cudf_jit_files(std::string const& target_dir, std::string const& tm rtcx_embed::cudf_jit_embed_files_compression); // rename the temporary directory to the target install directory - if (rename(tmp_path, target_dir.c_str()) == -1) { + if (::rename(tmp_dir_path, target_dir.c_str()) == -1) { auto errc = errno; // another process created it if (errc == ENOTEMPTY || errc == EEXIST) { - std::filesystem::remove_all(tmp_path); + std::filesystem::remove_all(tmp_dir_path); } else { - throw_posix( - std::format("Failed to rename temporary JIT install directory to ({})", target_dir), - "rename"); + CUDF_FAIL( + std::format("Failed to install JIT files to target directory: {} with error ({}): {}", + target_dir, + errc, + std::strerror(errc)), + std::runtime_error); } } } @@ -209,19 +134,13 @@ void jit_bundle_t::ensure_installed() const auto expected_hash = get_hash(); auto expected_path = std::format("{}/{}", install_dir_, expected_hash); - struct stat path_info; - - if (lstat(expected_path.c_str(), &path_info) == -1) { - if (errno != ENOENT) { - throw_posix(std::format("Failed to get stat for directory ({})", expected_path), "lstat"); - } else { - // ensure base install directory exists - std::filesystem::create_directories(install_dir_); - install_cudf_jit_files(expected_path.c_str(), cache_->get_tmp_dir()); - } + if (!std::filesystem::exists(expected_path)) { + // ensure base install directory exists + std::filesystem::create_directories(install_dir_); + install_cudf_jit_files(expected_path.c_str(), cache_->get_tmp_dir()); } else { // directory exists, perform minor sanity check - CUDF_EXPECTS(S_ISDIR(path_info.st_mode), + CUDF_EXPECTS(std::filesystem::is_directory(expected_path), // throws if path does not exist std::format("JIT install path ({}) exists but is not a directory", expected_path), std::runtime_error); } @@ -400,14 +319,14 @@ kernel_instance={} auto compile = [&] { auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); - auto source = read_blob_cstring(source_file.c_str()); + auto source = read_file_string(source_file.c_str()); std::vector extra_options_cstr; for (auto const& option : extra_options) { extra_options_cstr.emplace_back(option.c_str()); } return compile_library_uncached(name.c_str(), - reinterpret_cast(source.data()), + source.c_str(), header_include_names, headers, extra_options_cstr, diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 14474d427539..05aa2080526d 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -38,8 +38,7 @@ void context::ensure_jit_cache_initialized() rtcx::initialize(); auto limits = rtcx::cache_limits{.num_mem_blobs = _config.kernel_cache_limit_process, - .num_mem_libraries = _config.kernel_cache_limit_process, - .num_disk_entries = _config.kernel_cache_limit_disk}; + .num_mem_libraries = _config.kernel_cache_limit_process}; _rtcx_cache = std::make_unique(_config.rtcx_cache_dir, _config.jit_tmp_dir, @@ -120,7 +119,6 @@ void initialize(init_flags flags) bool clear_jit_cache = get_bool_env_or("LIBCUDF_JIT_CLEAR_CACHE", false); auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16'384U); - auto kernel_cache_limit_disk = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_DISK", 131'072U); flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); @@ -138,8 +136,7 @@ void initialize(init_flags flags) .jit_bundle_dir = jit_bundle_dir, .jit_pch_dir = jit_pch_dir, .jit_tmp_dir = jit_tmp_dir, - .kernel_cache_limit_process = kernel_cache_limit_process, - .kernel_cache_limit_disk = kernel_cache_limit_disk}; + .kernel_cache_limit_process = kernel_cache_limit_process}; _context.emplace(cfg, flags); }); diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 44bc3701edc5..59e27e00f57f 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -30,7 +30,6 @@ struct [[nodiscard]] context_config { std::string jit_pch_dir = {}; std::string jit_tmp_dir = {}; uint32_t kernel_cache_limit_process = 0; - uint32_t kernel_cache_limit_disk = 0; }; /// @brief The context object contains global state internal to CUDF. From 169618d87edb3cd8b1c96c1e06b986e31a331ea4 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 23 Apr 2026 20:29:31 +0100 Subject: [PATCH 160/254] Apply suggestion from @wence- Co-authored-by: Lawrence Mitchell --- cpp/librtcx/rtcx.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index de20d23d52a5..2c05ca7343e7 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -1021,7 +1021,6 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil auto result = compile(); promise.set_value(result); - // store result to disk cache_blob_to_disk(cache_dir_, tmp_dir_, "blob", sha, result->view()); return ret_fut; From b5cd518e72c9e9e7f359718a61ad7b8cdeaddebb Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 24 Apr 2026 00:20:39 +0000 Subject: [PATCH 161/254] Refactor object type handling in disk cache functions for improved clarity and consistency Co-authored-by: Copilot --- cpp/librtcx/rtcx.cpp | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index de20d23d52a5..9641bce9a969 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -102,6 +102,19 @@ namespace rtcx { namespace { +enum class object_type : std::uint8_t { LIBRARY, BLOB }; + +std::string_view object_tag(object_type type) +{ + switch (type) { + case object_type::LIBRARY: return "cuLibrary"; + case object_type::BLOB: return "blob"; + default: + RTCX_FAIL(std::format("Unrecognized object type: ({})", static_cast(type)), + std::runtime_error); + } +} + template std::string join_strings(std::span strings, std::string_view separator) { @@ -881,13 +894,10 @@ namespace { /// @brief retrieves a blob from disk based on the given sha256 hash and object type (e.g. "blob", /// "cuLibrary"). Returns nullopt if the file doesn't exist on disk, and throws if any other error /// occurs. -std::optional get_disk_blob(std::string const& cache_dir, - std::string const& object_type, - sha256 const& sha) +std::optional get_disk_blob(std::string const& cache_dir, object_type type, sha256 const& sha) { auto hex = sha.to_hex_string(); - auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); - + auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(type)); auto blob = blob_t::from_file(path.c_str()); if (!blob.has_value()) { return std::nullopt; } @@ -897,7 +907,12 @@ std::optional get_disk_blob(std::string const& cache_dir, std::optional get_disk_library(std::string const& cache_dir, sha256 const& sha) { auto hex = sha.to_hex_string(); - auto path = std::format("{}/{}.cuLibrary.bin", cache_dir, hex.view()); + auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(object_type::LIBRARY)); + + // WAR: avoid a driver API call when the cache file is not present + // compute-sanitizer doesn't properly handle exceptions thrown from driver API calls, so we need + // to check for file existence first to avoid false positives in the sanitizer + if (!std::filesystem::exists(path)) { return std::nullopt; } CUlibrary handle; auto errc = @@ -927,7 +942,7 @@ std::vector get_disk_entries(std::string const& cache_dir) /// it to the final path. void cache_blob_to_disk(std::string const& cache_dir, std::string const& tmp_dir, - std::string const& object_type, + object_type type, sha256 const& sha, std::span binary) { @@ -948,7 +963,7 @@ void cache_blob_to_disk(std::string const& cache_dir, } auto hex = sha.to_hex_string(); - auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_type); + auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(type)); std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); @@ -992,7 +1007,7 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil // check disk cache std::optional disk_blob = std::nullopt; - if (enabled_) { disk_blob = get_disk_blob(cache_dir_, "blob", sha); } + if (enabled_) { disk_blob = get_disk_blob(cache_dir_, object_type::BLOB, sha); } std::promise promise; auto fut = promise.get_future().share(); @@ -1022,7 +1037,7 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil promise.set_value(result); // store result to disk - cache_blob_to_disk(cache_dir_, tmp_dir_, "blob", sha, result->view()); + cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::BLOB, sha, result->view()); return ret_fut; } @@ -1085,7 +1100,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, promise.set_value(library); // store result to disk - cache_blob_to_disk(cache_dir_, tmp_dir_, "cuLibrary", sha, blob->view()); + cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::LIBRARY, sha, blob->view()); return ret_fut; } From 8c3ac2f9a7b9cf814636bf722719f3c3242e1a7a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 24 Apr 2026 12:51:27 +0000 Subject: [PATCH 162/254] update --- cpp/src/jit/jit.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index b88f6ac2f805..68d423b8376b 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -87,8 +87,7 @@ void install_cudf_jit_files(std::string const& target_dir, std::string const& tm { // directory does not exist, so create it auto tmp_dir_path_str = std::format("{}/cudf-jit-tmpdir_XXXXXX", tmp_dir); - (void)tmp_dir_path_str.c_str(); // ensure null-terminated string for mkdtemp - char* tmp_dir_path = ::mkdtemp(tmp_dir_path_str.data()); + char* tmp_dir_path = ::mkdtemp(tmp_dir_path_str.data()); CUDF_EXPECTS( tmp_dir_path != nullptr, std::format("Failed to create temporary directory for JIT file installation in tmp dir: {}", From 29826d4301ae9c2b785c8347bfe49d455d10168d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 26 Apr 2026 14:03:55 +0000 Subject: [PATCH 163/254] fix(context): update environment variable names for JIT cache configuration --- cpp/src/runtime/context.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 05aa2080526d..24e66a2894a8 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -114,18 +114,18 @@ void initialize(init_flags flags) std::call_once(*_context_init_flag, [&]() { bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); - bool preload_jit_cache = get_bool_env_or("LIBCUDF_JIT_PRELOAD_CACHE", false); - bool disable_jit_cache = get_bool_env_or("LIBCUDF_JIT_CACHE_DISABLED", false); - bool clear_jit_cache = get_bool_env_or("LIBCUDF_JIT_CLEAR_CACHE", false); + bool preload_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_PRELOAD", false); + bool disable_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_DISABLED", false); + bool clear_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16'384U); flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); - auto jit_bundle_dir = get_cudf_kernel_cache_dir() / "jit" / "bundle"; - auto rtcx_cache_dir = get_cudf_kernel_cache_dir() / "jit" / "rtcx_cache"; - auto jit_pch_dir = get_cudf_kernel_cache_dir() / "jit" / "pch"; - auto jit_tmp_dir = get_cudf_kernel_cache_dir() / "jit" / "tmp"; + auto jit_bundle_dir = get_cudf_kernel_cache_dir() / "bundle"; + auto rtcx_cache_dir = get_cudf_kernel_cache_dir() / "rtcx_cache"; + auto jit_pch_dir = get_cudf_kernel_cache_dir() / "pch"; + auto jit_tmp_dir = get_cudf_kernel_cache_dir() / "tmp"; context_config cfg{.dump_codegen = dump_codegen, .use_jit = use_jit, From 8c7f36871112437ecf5eb765d2245781555d0f4a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 5 May 2026 13:26:59 +0000 Subject: [PATCH 164/254] Refactor JIT operators and add compute columns functionality - Removed the operators.cu file which contained JIT operator definitions. - Updated embed.cmake to improve formatting of file generation. - Introduced a new compute_columns_jit.cu file to handle column transformations. - Added a new test for compute columns in compute_columns_test.cpp. - Implemented a profit operator in compute_columns_profit_operator.cu to calculate net and gross values based on input parameters. --- cpp/CMakeLists.txt | 68 +- cpp/include/cudf/transform.hpp | 45 + cpp/src/jit/jit.cpp | 343 +++- cpp/src/jit/jit.hpp | 68 +- cpp/src/jit/lite/lite.cuh | 1634 ----------------- cpp/src/jit/lite/operators.cu | 602 ------ cpp/src/librtcx/embed.cmake | 6 +- cpp/src/transform/compute_columns_jit.cu | 18 + cpp/tests/CMakeLists.txt | 5 + .../compute_columns_profit_operator.cu | 70 + cpp/tests/transform/compute_columns_test.cpp | 0 11 files changed, 583 insertions(+), 2276 deletions(-) delete mode 100644 cpp/src/jit/lite/lite.cuh delete mode 100644 cpp/src/jit/lite/operators.cu create mode 100644 cpp/src/transform/compute_columns_jit.cu create mode 100644 cpp/tests/transform/compute_columns_profit_operator.cu create mode 100644 cpp/tests/transform/compute_columns_test.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e3169f38b582..4eccd6a091f1 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -334,40 +334,36 @@ macro(add_fragment) set(MULTI_VALUE_ARGS "DEFINITIONS") cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - foreach(ARCH IN LISTS CUDF_LTO_JIT_ARCHITECTURES) - set(TARGET_ID "sm${ARCH}_${TARGET}_${ARG_FRAGMENT}") - add_library(${TARGET_ID} OBJECT ${ARG_SOURCE}) - target_compile_options( - ${TARGET_ID} PRIVATE --compress-mode=none --gen-opt-lto --relaxed-constexpr --extended-lambda - ) - target_compile_definitions(${TARGET_ID} PRIVATE CUDF_DISABLE_EXPORTS ${ARG_DEFINITIONS}) - set_target_properties( - ${TARGET_ID} - PROPERTIES CUDA_ARCHITECTURES "${ARCH}-real" - CUDA_SEPARABLE_COMPILATION ON - CUDA_FATBIN_COMPILATION ON - POSITION_INDEPENDENT_CODE ON - INTERPROCEDURAL_OPTIMIZATION ON - CXX_STANDARD ${CUDF_CUDA_STANDARD} - CXX_STANDARD_REQUIRED ON - CXX_EXTENSIONS ON - CXX_VISIBILITY_PRESET hidden - CUDA_STANDARD ${CUDF_CUDA_STANDARD} - CUDA_STANDARD_REQUIRED ON - CUDA_VISIBILITY_PRESET hidden - ) - target_link_libraries(${TARGET_ID} PRIVATE CCCL::CCCL) - target_include_directories( - ${TARGET_ID} PRIVATE "$" - "$" - ) - - embed_blob( - ${TARGET} FILE $ DEST - fragments/sm_${ARCH}_${ARG_FRAGMENT}.fatbin - ) - endforeach() + set(TARGET_ID "${TARGET}_${ARG_FRAGMENT}") + add_library(${TARGET_ID} OBJECT ${ARG_SOURCE}) + target_compile_options( + ${TARGET_ID} PRIVATE --compress-mode=none --gen-opt-lto --relaxed-constexpr --extended-lambda + ) + target_compile_definitions(${TARGET_ID} PRIVATE CUDF_DISABLE_EXPORTS ${ARG_DEFINITIONS}) + set_target_properties( + ${TARGET_ID} + PROPERTIES CUDA_ARCHITECTURES "${CUDF_LTO_JIT_ARCHITECTURES}" + CUDA_SEPARABLE_COMPILATION ON + CUDA_FATBIN_COMPILATION ON + POSITION_INDEPENDENT_CODE ON + INTERPROCEDURAL_OPTIMIZATION ON + CXX_STANDARD ${CUDF_CUDA_STANDARD} + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS ON + CXX_VISIBILITY_PRESET hidden + CUDA_STANDARD ${CUDF_CUDA_STANDARD} + CUDA_STANDARD_REQUIRED ON + CUDA_VISIBILITY_PRESET hidden + ) + target_link_libraries(${TARGET_ID} PRIVATE CCCL::CCCL) + target_include_directories( + ${TARGET_ID} PRIVATE "$" + "$" + ) + embed_blob( + ${TARGET} FILE $ DEST fragments/sm_${ARCH}_${ARG_FRAGMENT}.fatbin + ) endmacro() add_fragment( @@ -438,6 +434,11 @@ add_fragment( add_fragment(cudf_jit_embed FRAGMENT operators SOURCE src/jit/lite/operators.cu) +add_fragment( + cudf_test_jit_embed FRAGMENT compute_columns_profit_operator SOURCE + tests/transform/compute_columns_profit_operator.cu +) + embed_includes( cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/libcxx DEST_DIRECTORY librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx @@ -987,6 +988,7 @@ add_library( src/text/wordpiece_tokenize.cu src/transform/bools_to_mask.cu src/transform/compute_column.cu + src/transform/compute_columns_jit.cu src/transform/compute_column_kernel_complex.cu src/transform/compute_column_kernel_null_complex.cu src/transform/compute_column_kernel_null_primitive.cu diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index a2a99ed0c23b..2146261fc570 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -179,6 +179,51 @@ std::unique_ptr
multi_transform( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +/** + * @brief Creates a new table by applying a transform function against every + * element of the input columns. + * + * Computes: + * `(outputs[i]...) = UDF(inputs[i]...)`. + * + * + * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) + * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string + * types + * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and + * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in + * this case. + * @throws std::invalid_argument if string offsets are provided for non-string output columns, or + * if the number of string offsets does not match the number of output columns. + * + * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from + * the input and pre-allocated output columns. + * + * @param udf The LTO-IR string of the transform function to apply + * @param is_null_aware Signifies the UDF will receive row inputs as optional values + * @param user_data User-defined device data to pass to the UDF. + * @param inputs Immutable views of the inputs to transform (columns and scalar columns) + * @param outputs Specification of the output columns to be created + * @param string_offsets For string output columns, the offsets can be pre-allocated and passed in + * to prevent overhead of compacting string views into run-end strings column. + * @param row_size The row size of the transform operation. If not provided, it is inferred from the + * input columns. + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + * @return A table containing the columns resulting from applying the transform + * function to every element of the input according to the output specifications + * + */ +std::unique_ptr
compute_columns_jit( + std::span inputs, + std::span udf, + std::span output_types, + void* user_data, + null_aware is_null_aware = null_aware::NO, + std::optional row_size = std::nullopt, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + /** * @brief Creates a null_mask from `input` by converting `NaN` to null and * preserving existing null values and also returns new null_count. diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index f48cb601b21a..15c7b294a01a 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -28,14 +28,21 @@ namespace CUDF_EXPORT cudf { namespace { -rtcx::sha256 hash_string(std::span input) +rtcx::sha256 hash(char const* input) +{ + rtcx::sha256_context ctx; + ctx.update(std::span{reinterpret_cast(input), std::strlen(input)}); + return ctx.finalize(); +} + +rtcx::sha256 hash(std::span input) { rtcx::sha256_context ctx; ctx.update(std::span{reinterpret_cast(input.data()), input.size()}); return ctx.finalize(); } -rtcx::sha256 hash_strings(std::span inputs) +rtcx::sha256 hash(std::span inputs) { rtcx::sha256_context ctx; for (auto const* input : inputs) { @@ -44,6 +51,34 @@ rtcx::sha256 hash_strings(std::span inputs) return ctx.finalize(); } +rtcx::sha256 hash(std::span inputs) +{ + rtcx::sha256_context ctx; + for (auto const& input : inputs) { + ctx.update(std::span{reinterpret_cast(input.data()), input.size()}); + } + return ctx.finalize(); +} + +rtcx::sha256 hash(std::span file_fragments) +{ + rtcx::sha256_context ctx; + for (auto const& fragment : file_fragments) { + ctx.update( + std::span{reinterpret_cast(fragment.path), std::strlen(fragment.path)}); + } + return ctx.finalize(); +} + +rtcx::sha256 hash(std::span memory_fragments) +{ + rtcx::sha256_context ctx; + for (auto const& fragment : memory_fragments) { + ctx.update(fragment.data); + } + return ctx.finalize(); +} + [[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) { auto error_code = errno; @@ -357,6 +392,178 @@ std::tuple compile_library_uncached( return std::make_tuple(library, std::make_shared(std::move(blob))); } +rtcx::byte_buffer compile_fragment_uncached(char const* name, + char const* cuda_code, + std::span extra_header_include_names, + std::span extra_headers, + std::span extra_options, + std::span name_expressions, + bool use_pch, + bool use_minimal, + bool log_pch) +{ + CUDF_FUNC_RANGE(); + + auto& bundle = cudf::get_context().jit_bundle(); + auto sm = get_current_device_physical_model(); + + auto include_dirs = bundle.get_include_directories(); + auto pch_dir = cudf::get_context().get_jit_pch_dir(); + + std::vector options; + + for (auto const& include_dir : include_dirs) { + options.emplace_back(std::format("-I{}", include_dir)); + } + + options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); + + options.emplace_back("--diag-suppress=47"); + options.emplace_back("--device-int128"); + + if (sm >= 100) { options.emplace_back("--device-float128"); } + + options.emplace_back("-std=c++20"); + options.emplace_back("--device-as-default-execution-space"); + options.emplace_back("--generate-line-info"); + options.emplace_back("--dopt=on"); + + if (use_minimal) { options.emplace_back("--minimal"); } + + if (use_pch) { + options.emplace_back("--pch"); + + if (log_pch) { + options.emplace_back("--pch-verbose=true"); + options.emplace_back("--pch-messages=true"); + } + } + + options.emplace_back("--relocatable-device-code=true"); + options.emplace_back("--gen-opt-lto"); + options.emplace_back("--dlink-time-opt"); + + std::vector options_cstr; + for (auto const& option : options) { + options_cstr.emplace_back(option.c_str()); + } + for (auto* option : extra_options) { + options_cstr.emplace_back(option); + } + + auto params = rtcx::compile_params{.name = name, + .source = cuda_code, + .header_include_names = extra_header_include_names, + .headers = extra_headers, + .options = options_cstr, + .name_expressions = name_expressions, + .target_type = rtcx::binary_type::LTO_IR}; + + return rtcx::compile(params); +} + +std::variant get_cuda_udf_fragment( + char const* name, + char const* cuda_code, + std::span extra_header_include_names, + std::span extra_headers, + std::span extra_options, + std::span name_expressions, + bool use_pch, + bool use_minimal, + bool log_pch, + bool use_cache) +{ + auto& cache = cudf::get_context().rtcx_cache(); + auto& bundle = cudf::get_context().jit_bundle(); + + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_physical_model(); + auto header_include_names_hash = hash(extra_header_include_names).to_hex_string(); + auto headers_hash = hash(extra_headers).to_hex_string(); + auto bundle_hash = bundle.get_hash(); + auto source_hash = hash(cuda_code).to_hex_string(); + + auto cache_key = std::format(R"***(cudaFragment +binary_type=LTO_IR +name={} +cuda_runtime={} +cuda_driver={} +arch={} +header_include_names={}, +headers={}, +bundle={}, +source_file={} +)***", + name, + runtime, + driver, + sm, + header_include_names_hash.view(), + headers_hash.view(), + bundle_hash, + source_hash.view()); + + auto cache_key_sha256 = hash(cache_key); + + auto compile = [&] { + return compile_fragment_uncached(name, + cuda_code, + extra_header_include_names, + extra_headers, + extra_options, + name_expressions, + use_pch, + use_minimal, + log_pch); + }; + + if (!use_cache) { return compile(); } + + auto fut = + cache.get_or_add_blob(cache_key_sha256, rtcx::blob_compile_func::from_functor(compile)); + return fut.get(); +} + +std::tuple link_library_uncached( + char const* name, + std::span file_fragments, + std::span memory_fragments, + std::span extra_options) +{ + CUDF_FUNC_RANGE(); + + auto sm = get_current_device_physical_model(); + + std::vector options; + + options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); + options.emplace_back("-lto"); + + std::vector options_cstr; + + for (auto const& option : options) { + options_cstr.emplace_back(option.c_str()); + } + + for (auto* option : extra_options) { + options_cstr.emplace_back(option); + } + + auto params = rtcx::link_params{.name = name, + .output_type = rtcx::binary_type::CUBIN, + .file_fragments = file_fragments, + .memory_fragments = memory_fragments, + .link_options = options_cstr}; + + auto cubin = rtcx::link_library(params); + auto library = rtcx::load_library(cubin); + auto blob = rtcx::blob_t::from_buffer(std::move(cubin)); + + return std::make_tuple(library, std::make_shared(std::move(blob))); +} + } // namespace kernel get_kernel(std::string const& name, @@ -378,8 +585,8 @@ kernel get_kernel(std::string const& name, auto runtime = get_runtime_version(); auto driver = get_driver_version(); auto sm = get_current_device_physical_model(); - auto header_include_names_hash = hash_strings(header_include_names).to_hex_string(); - auto headers_hash = hash_strings(headers).to_hex_string(); + auto header_include_names_hash = hash(header_include_names).to_hex_string(); + auto headers_hash = hash(headers).to_hex_string(); auto bundle_hash = bundle.get_hash(); auto source_file = std::format("{}/cudf/cpp/src/{}", bundle.get_directory(), source_file_id); @@ -405,7 +612,7 @@ kernel_instance={} headers_hash.view(), kernel_instance); - auto cache_key_sha256 = hash_string(cache_key); + auto cache_key_sha256 = hash(cache_key); auto compile = [&] { auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); @@ -438,4 +645,130 @@ kernel_instance={} return kernel{lib, lib->get_kernel("kernel")}; } +kernel get_linked_kernel(std::string const& name, + std::string const& fragment_file_id, + std::span file_fragments, + std::span memory_fragments, + bool use_cache, + std::span extra_options) +{ + CUDF_FUNC_RANGE(); + + auto& cache = cudf::get_context().rtcx_cache(); + auto& bundle = cudf::get_context().jit_bundle(); + + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_physical_model(); + auto file_fragments_hash = hash(file_fragments).to_hex_string(); + auto memory_fragments_hash = hash(memory_fragments).to_hex_string(); + auto bundle_hash = bundle.get_hash(); + auto fragment_file = std::format("{}/fragments/{}", bundle.get_directory(), fragment_file_id); + // TODO: resolve file fragment IDs + + auto cache_key = std::format(R"***(cuLibrary +name={} +binary_type=CUBIN +cuda_runtime={} +cuda_driver={} +arch={} +bundle={} +fragment_file={} +file_fragments={} +memory_fragments={} +)***", + name, + runtime, + driver, + sm, + bundle_hash, + fragment_file, + file_fragments_hash.view(), + memory_fragments_hash.view()); + + auto cache_key_sha256 = hash(cache_key); + + auto compile = [&] { + std::vector extra_options_cstr; + for (auto const& option : extra_options) { + extra_options_cstr.emplace_back(option.c_str()); + } + + return link_library_uncached( + name.c_str(), file_fragments, memory_fragments, extra_options_cstr); + }; + + if (!use_cache) { + auto [lib, blob] = compile(); + return kernel{lib, lib->get_kernel("kernel")}; + } + + auto fut = + cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); + + auto lib = fut.get(); + return kernel{lib, lib->get_kernel("kernel")}; +} + +kernel get_cuda_linked_kernel(std::string const& kernel_name, + std::string const& kernel_fragment_file_id, + std::string const& cuda_udf_name, + std::string const& cuda_udf_source, + std::span header_include_names, + std::span headers, + std::span file_fragments, + std::span memory_fragments, + bool use_cache, + bool use_pch, + bool use_minimal, + bool log_pch, + std::span extra_compile_options, + std::span extra_link_options) +{ + CUDF_FUNC_RANGE(); + + std::vector extra_compile_options_cstr; + for (auto const& option : extra_compile_options) { + extra_compile_options_cstr.emplace_back(option.c_str()); + } + + std::vector extra_link_options_cstr; + for (auto const& option : extra_link_options) { + extra_link_options_cstr.emplace_back(option.c_str()); + } + + auto udf_fragment = get_cuda_udf_fragment(cuda_udf_name.c_str(), + cuda_udf_source.c_str(), + header_include_names, + headers, + extra_compile_options_cstr, + {}, + use_pch, + use_minimal, + log_pch, + use_cache); + + auto fragment_name = std::format("{}_udf_fragment", cuda_udf_name); + + std::vector all_fragments; + all_fragments.insert(all_fragments.end(), memory_fragments.begin(), memory_fragments.end()); + + if (std::holds_alternative(udf_fragment)) { + auto& udf_blob = std::get(udf_fragment); + all_fragments.push_back(rtcx::memory_fragment{ + .data = udf_blob->view(), .type = rtcx::binary_type::LTO_IR, .name = fragment_name.c_str()}); + } else { + auto& udf_buffer = std::get(udf_fragment); + all_fragments.push_back(rtcx::memory_fragment{ + .data = udf_buffer, .type = rtcx::binary_type::LTO_IR, .name = fragment_name.c_str()}); + } + + return get_linked_kernel(kernel_name, + kernel_fragment_file_id, + file_fragments, + all_fragments, + use_cache, + extra_link_options); +} + } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index 6a0c6f877915..78a9710e1b25 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -71,8 +71,25 @@ struct [[nodiscard]] kernel { } }; +/** + * @brief Gets a kernel from an embedded CUDA source file + * @param name Debug name for the kernel (used for caching and logging) + * @param source_file_id Identifier for the embedded source file (used to locate the source and for + * caching) + * @param header_include_names Names of any additional embedded header files to include during + * compilation + * @param headers Contents of any additional embedded header files to include during compilation + * @param kernel_instance String identifier for the specific kernel instance being requested (used + * for caching) + * @param use_cache Whether to use the caching mechanism to avoid redundant compilations + * @param use_pch Whether to use precompiled headers for faster compilation (if supported) + * @param use_minimal Whether to use NVRTC's minimal compilation mode for faster compilation (if + * supported) + * @param log_pch Whether to log precompiled header usage + * @param extra_options Additional compilation options + */ kernel get_kernel(std::string const& name, - std::string const& source_file, + std::string const& source_file_id, std::span header_include_names, std::span headers, std::string const& kernel_instance, @@ -82,4 +99,53 @@ kernel get_kernel(std::string const& name, bool log_pch = false, std::span extra_options = {}); +/** + * @brief Gets a kernel by linking together embedded binary fragments + * @param name Debug name for the kernel (used for caching and logging) + * @param fragment_file_id Identifier for the embedded fragment file (used to locate the fragments + * and for caching) + * @param file_fragments Embedded fragments to link + * @param memory_fragments Memory fragments to link + * @param use_cache Whether to use the cache system + * @param extra_options Additional linking options + */ +kernel get_linked_kernel(std::string const& name, + std::string const& fragment_file_id, + std::span file_fragments, + std::span memory_fragments, + bool use_cache = true, + std::span extra_options = {}); + +/** + * @brief Gets a kernel by compiling a CUDA UDF and linking together binary fragments. + * @param kernel_name Debug name for the kernel (used for caching and logging) + * @param kernel_fragment_file_id Identifier for the embedded fragment file (used to locate the fragments + * and for caching) + * @param cuda_udf_name Name of the CUDA UDF + * @param cuda_udf_source CUDA source code for the UDF + * @param file_fragments Embedded fragments to link + * @param memory_fragments Memory fragments to link + * @param use_cache Whether to use the cache system + * @param use_pch Whether to use precompiled headers for faster compilation (if supported) + * @param use_minimal Whether to use NVRTC's minimal compilation mode for faster compilation (if + * supported) + * @param log_pch Whether to log precompiled header usage + * @param extra_compile_options Additional compilation options + * @param extra_link_options Additional linking options + */ +kernel get_cuda_linked_kernel(std::string const& kernel_name, + std::string const& kernel_fragment_file_id, + std::string const& cuda_udf_name, + std::string const& cuda_udf_source, + std::span header_include_names, + std::span headers, + std::span file_fragments, + std::span memory_fragments, + bool use_cache = true, + bool use_pch = true, + bool use_minimal = true, + bool log_pch = false, + std::span extra_compile_options = {}, + std::span extra_link_options = {}); + } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/lite/lite.cuh b/cpp/src/jit/lite/lite.cuh deleted file mode 100644 index e46b82d0d4e2..000000000000 --- a/cpp/src/jit/lite/lite.cuh +++ /dev/null @@ -1,1634 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -namespace __attribute__((visibility("hidden"))) cudf -{ - namespace lite { - - using int8_t = signed char; - using int16_t = signed short; - using int32_t = signed int; - using int64_t = signed long long; - using int128_t = __int128_t; - - using uint8_t = unsigned char; - using uint16_t = unsigned short; - using uint32_t = unsigned int; - using uint64_t = unsigned long long; - using uint128_t = __uint128_t; - - using size_t = unsigned long long; - using intptr_t = int64_t; - using uintptr_t = uint64_t; - - using intmax_t = int64_t; - using uintmax_t = uint64_t; - - using float32_t = float; - using float64_t = double; - - using size_type = int32_t; - - using char_utf8 = uint32_t; - - template - __device__ constexpr T min(T a, T b) - { - return a < b ? a : b; - } - - template - __device__ constexpr T max(T a, T b) - { - return a > b ? a : b; - } - - template - __device__ constexpr T ipow10(T exponent) - { - if (exponent == 0) { return 1; } - - T extra = 1; - T square = 10; - T n = exponent; - - while (n > 1) { - if ((n & 1) == 1) { extra *= square; } - n >>= 1; - square *= square; - } - - return square * extra; - } - - struct scaled_t {}; - - inline constexpr scaled_t scaled; - - template - struct decimal { - using Rep = R; - - R _value = 0; - - int32_t _scale = 0; - - constexpr decimal() = default; - - __device__ constexpr decimal(scaled_t, R value, int32_t scale) : _value{value}, _scale{scale} {} - - __device__ constexpr R value() const { return _value; } - - __device__ constexpr int32_t scale() const { return _scale; } - - private: - __device__ static constexpr R _lshift(R v, int32_t scale) { return v * ipow10(-scale); } - - __device__ static constexpr R _rshift(R v, int32_t scale) { return v / ipow10(scale); } - - __device__ static constexpr R _shift(R v, int32_t scale) - { - if (scale == 0) { - return v; - } else if (scale < 0) { - return _lshift(v, scale); - } else { - return _rshift(v, scale); - } - } - - __device__ static constexpr R _rescale(R v, int32_t from_scale, int32_t to_scale) - { - return _shift(v, to_scale - from_scale); - } - - public: - __device__ constexpr auto rescale(int32_t scale) const - { - return decimal{scaled, _rescale(_value, _scale, scale), scale}; - } - - __device__ constexpr auto operator+(decimal rhs) const - { - auto scale = min(_scale, rhs._scale); - auto r = rescale(scale)._value + rhs.rescale(scale)._value; - return decimal{scaled, r, scale}; - } - - __device__ constexpr auto operator-(decimal rhs) const - { - auto scale = min(_scale, rhs._scale); - auto r = rescale(scale)._value - rhs.rescale(scale)._value; - return decimal{scaled, r, scale}; - } - - __device__ constexpr auto operator*(decimal rhs) const - { - return decimal{scaled, _value * rhs._value, _scale + rhs._scale}; - } - - __device__ constexpr auto operator/(decimal rhs) const - { - return decimal{scaled, _value / rhs._value, _scale - rhs._scale}; - } - - __device__ constexpr auto operator%(decimal rhs) const - { - auto scale = min(_scale, rhs._scale); - auto r = rescale(scale)._value % rhs.rescale(scale)._value; - return decimal{scaled, r, scale}; - } - - __device__ constexpr int compare(decimal rhs) const - { - auto scale = min(_scale, rhs._scale); - return rescale(scale)._value - rhs.rescale(scale)._value; - } - - __device__ constexpr bool operator==(decimal rhs) const { return compare(rhs) == 0; } - __device__ constexpr bool operator!=(decimal rhs) const { return compare(rhs) != 0; } - __device__ constexpr bool operator<(decimal rhs) const { return compare(rhs) < 0; } - __device__ constexpr bool operator<=(decimal rhs) const { return compare(rhs) <= 0; } - __device__ constexpr bool operator>(decimal rhs) const { return compare(rhs) > 0; } - __device__ constexpr bool operator>=(decimal rhs) const { return compare(rhs) >= 0; } - }; - - using decimal32 = decimal; - using decimal64 = decimal; - using decimal128 = decimal; - - enum class timestamp_unit : int32_t { D, h, m, s, ms, us, ns }; - - template - struct timestamp { - using Rep = R; - - R _rep = 0; - - __device__ constexpr R count() const { return _rep; } - - __device__ constexpr int compare(timestamp rhs) const { return _rep - rhs._rep; } - - __device__ constexpr bool operator==(timestamp rhs) const { return compare(rhs) == 0; } - __device__ constexpr bool operator!=(timestamp rhs) const { return compare(rhs) != 0; } - __device__ constexpr bool operator<(timestamp rhs) const { return compare(rhs) < 0; } - __device__ constexpr bool operator<=(timestamp rhs) const { return compare(rhs) <= 0; } - __device__ constexpr bool operator>(timestamp rhs) const { return compare(rhs) > 0; } - __device__ constexpr bool operator>=(timestamp rhs) const { return compare(rhs) >= 0; } - }; - - using timestamp_D = timestamp; - using timestamp_h = timestamp; - using timestamp_m = timestamp; - using timestamp_s = timestamp; - using timestamp_ms = timestamp; - using timestamp_us = timestamp; - using timestamp_ns = timestamp; - - template - struct duration { - using Rep = R; - - R _rep = 0; - - __device__ constexpr R count() const { return _rep; } - - __device__ constexpr duration operator+(duration rhs) const - { - return duration{_rep + rhs._rep}; - } - - __device__ constexpr duration operator-(duration rhs) const - { - return duration{_rep - rhs._rep}; - } - - __device__ constexpr int compare(duration rhs) const { return _rep - rhs._rep; } - - __device__ constexpr bool operator==(duration rhs) const { return compare(rhs) == 0; } - __device__ constexpr bool operator!=(duration rhs) const { return compare(rhs) != 0; } - __device__ constexpr bool operator<(duration rhs) const { return compare(rhs) < 0; } - __device__ constexpr bool operator<=(duration rhs) const { return compare(rhs) <= 0; } - __device__ constexpr bool operator>(duration rhs) const { return compare(rhs) > 0; } - __device__ constexpr bool operator>=(duration rhs) const { return compare(rhs) >= 0; } - }; - - using duration_D = duration; - using duration_h = duration; - using duration_m = duration; - using duration_s = duration; - using duration_ms = duration; - using duration_us = duration; - using duration_ns = duration; - - struct inplace_t {}; - - inline constexpr inplace_t inplace; - - struct nullopt_t {}; - - inline constexpr nullopt_t nullopt; - - template - struct optional { - T _value = {}; - - bool _is_valid = false; - - constexpr optional() = default; - - __device__ constexpr optional(nullopt_t) {} - - template - __device__ constexpr optional(inplace_t, Args&&... args) - : _value{static_cast(args)...}, _is_valid{true} - { - } - - __device__ constexpr optional(T value) : _value{value}, _is_valid{true} {} - - __device__ constexpr bool is_valid() const { return _is_valid; } - - __device__ constexpr bool is_null() const { return !_is_valid; } - - __device__ constexpr void reset() { _is_valid = false; } - - __device__ constexpr T const& get() const { return _value; } - - __device__ constexpr T& get() { return _value; } - - __device__ constexpr T const* operator->() const { return &_value; } - - __device__ constexpr T* operator->() { return &_value; } - - __device__ constexpr T const& operator*() const { return _value; } - - __device__ constexpr T& operator*() { return _value; } - - __device__ constexpr T const& value() const { return _value; } - - __device__ constexpr T& value() { return _value; } - - __device__ constexpr explicit operator bool() const { return _is_valid; } - - __device__ constexpr T value_or(T v) const { return _is_valid ? _value : v; } - }; - - template - optional(T) -> optional; - - template - struct span { - T* _data = nullptr; - - size_t _size = 0; - - constexpr span() = default; - - __device__ constexpr span(T* data, size_t size) : _data{data}, _size{size} {} - - __device__ constexpr T* data() const { return _data; } - - __device__ constexpr size_t size() const { return _size; } - - __device__ constexpr bool empty() const { return _size == 0; } - - __device__ constexpr T& operator[](size_t pos) const { return _data[pos]; } - - __device__ constexpr T* begin() const { return _data; } - - __device__ constexpr T* end() const { return _data + _size; } - - __device__ constexpr T const* cbegin() const { return _data; } - - __device__ constexpr T const* cend() const { return _data + _size; } - - __device__ constexpr span as_const() const { return span{_data, _size}; } - - __device__ constexpr T& element(size_t i) const { return _data[i]; } - }; - - template - span(T*, size_t) -> span; - - struct string_view { - static constexpr size_type const UNKNOWN_STRING_LENGTH{-1}; - static constexpr size_type const npos{-1}; - - char const* _data = ""; - - size_type _bytes = 0; - - mutable size_type _length = UNKNOWN_STRING_LENGTH; - - constexpr string_view() = default; - - __device__ constexpr string_view(char const* data, size_type bytes) : _data{data}, _bytes{bytes} - { - } - - __device__ constexpr string_view(char const* data, size_type bytes, size_type length) - : _data{data}, _bytes{bytes}, _length{length} - { - } - - __device__ constexpr size_type size_bytes() const { return _bytes; } - - __device__ constexpr auto* data() const { return _data; } - - __device__ constexpr auto* begin() const { return _data; } - - __device__ constexpr auto* end() const { return _data + _bytes; } - - __device__ constexpr auto const* cbegin() const { return _data; } - - __device__ constexpr auto const* cend() const { return _data + _bytes; } - - __device__ constexpr bool empty() const { return _bytes == 0; } - - __device__ constexpr size_type compare(string_view const& other) const - { - auto* s0 = _data; - auto n0 = _bytes; - auto* s1 = other._data; - auto n1 = other._bytes; - auto max = n0 < n1 ? n0 : n1; - - if (s0 == s1 && n0 == n1) return 0; - - size_type i = 0; - - while (i < max) { - if (*s0 != *s1) return static_cast(*s0) - static_cast(*s1); - s0++; - s1++; - i++; - } - - if (i < n0) { return 1; } - if (i < n1) { return -1; } - - return 0; - } - - __device__ constexpr bool operator==(string_view const& rhs) const { return compare(rhs) == 0; } - __device__ constexpr bool operator!=(string_view const& rhs) const { return compare(rhs) != 0; } - __device__ constexpr bool operator<(string_view const& rhs) const { return compare(rhs) < 0; } - __device__ constexpr bool operator<=(string_view const& rhs) const { return compare(rhs) <= 0; } - __device__ constexpr bool operator>(string_view const& rhs) const { return compare(rhs) > 0; } - __device__ constexpr bool operator>=(string_view const& rhs) const { return compare(rhs) >= 0; } - }; - - struct mutable_string_view { - static constexpr size_type const UNKNOWN_STRING_LENGTH{-1}; - static constexpr size_type const npos{-1}; - - char* _data = nullptr; - - size_type _bytes = 0; - - mutable size_type _length = UNKNOWN_STRING_LENGTH; - - constexpr mutable_string_view() = default; - - __device__ constexpr mutable_string_view(char* data, size_type bytes) - : _data{data}, _bytes{bytes} - { - } - - __device__ constexpr size_type size_bytes() const { return _bytes; } - - __device__ constexpr auto* data() const { return _data; } - - __device__ constexpr auto* begin() const { return _data; } - - __device__ constexpr auto* end() const { return _data + _bytes; } - - __device__ constexpr auto const* cbegin() const { return _data; } - - __device__ constexpr auto const* cend() const { return _data + _bytes; } - - __device__ constexpr bool empty() const { return _bytes == 0; } - - __device__ explicit operator string_view() const { return string_view{_data, _bytes, _length}; } - }; - - // Aliases for codegen - using b8 = bool; - using i8 = int8_t; - using i16 = int16_t; - using i32 = int32_t; - using i64 = int64_t; - using u8 = uint8_t; - using u16 = uint16_t; - using u32 = uint32_t; - using u64 = uint64_t; - using f32 = float; - using f64 = double; - using dec32 = decimal32; - using dec64 = decimal64; - using dec128 = decimal128; - using ts_D = timestamp_D; - using ts_h = timestamp_h; - using ts_m = timestamp_m; - using ts_s = timestamp_s; - using ts_ms = timestamp_ms; - using ts_us = timestamp_us; - using ts_ns = timestamp_ns; - using dur_D = duration_D; - using dur_h = duration_h; - using dur_m = duration_m; - using dur_s = duration_s; - using dur_ms = duration_ms; - using dur_us = duration_us; - using dur_ns = duration_ns; - using str = string_view; - - namespace operators { - - template - __device__ inline int abs(T* out, T const* a) - { - *out = (*a < 0) ? -*a : *a; - return 0; - } - - template - __device__ inline int abs(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - abs(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int add(T* out, T const* a, T const* b) - { - *out = (*a + *b); - return 0; - } - - template - __device__ inline int add(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - add(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int arccos(T* out, T const* a); - - template <> - __device__ inline int arccos(float32_t* out, float32_t const* a) - { - *out = ::acosf(*a); - return 0; - } - - template <> - __device__ inline int arccos(float64_t* out, float64_t const* a) - { - *out = ::acos(*a); - return 0; - } - - template - __device__ inline int arccos(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - arccos(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int arccosh(T* out, T const* a); - - template <> - __device__ inline int arccosh(float32_t* out, float32_t const* a) - { - *out = ::acoshf(*a); - return 0; - } - - template <> - __device__ inline int arccosh(float64_t* out, float64_t const* a) - { - *out = ::acosh(*a); - return 0; - } - - template - __device__ inline int arccosh(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - arccosh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int arcsin(T* out, T const* a); - - template <> - __device__ inline int arcsin(float32_t* out, float32_t const* a) - { - *out = ::asinf(*a); - return 0; - } - - template <> - __device__ inline int arcsin(float64_t* out, float64_t const* a) - { - *out = ::asin(*a); - return 0; - } - - template - __device__ inline int arcsin(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - arcsin(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int arcsinh(T* out, T const* a); - - template <> - __device__ inline int arcsinh(float32_t* out, float32_t const* a) - { - *out = ::asinhf(*a); - return 0; - } - - template <> - __device__ inline int arcsinh(float64_t* out, float64_t const* a) - { - *out = ::asinh(*a); - return 0; - } - - template - __device__ inline int arcsinh(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - arcsinh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int arctan(T* out, T const* a); - - template <> - __device__ inline int arctan(float32_t* out, float32_t const* a) - { - *out = ::atanf(*a); - return 0; - } - - template <> - __device__ inline int arctan(float64_t* out, float64_t const* a) - { - *out = ::atan(*a); - return 0; - } - - template - __device__ inline int arctan(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - arctan(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int arctanh(T* out, T const* a); - - template <> - __device__ inline int arctanh(float32_t* out, float32_t const* a) - { - *out = ::atanhf(*a); - return 0; - } - - template <> - __device__ inline int arctanh(float64_t* out, float64_t const* a) - { - *out = ::atanh(*a); - return 0; - } - - template - __device__ inline int arctanh(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - arctanh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int bit_and(T* out, T const* a, T const* b) - { - *out = (*a & *b); - return 0; - } - - template - __device__ inline int bit_and(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - bit_and(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int bit_invert(T* out, T const* a) - { - *out = ~(*a); - return 0; - } - - template - __device__ inline int bit_invert(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - bit_invert(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int bit_or(T* out, T const* a, T const* b) - { - *out = (*a | *b); - return 0; - } - - template - __device__ inline int bit_or(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - bit_or(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int bit_xor(T* out, T const* a, T const* b) - { - *out = (*a ^ *b); - return 0; - } - - template - __device__ inline int bit_xor(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - bit_xor(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int to_i32(int32_t* out, T const* a) - { - *out = static_cast(*a); - return 0; - } - - template - __device__ inline int to_i32(optional* out, optional const* a) - { - if (a->is_valid()) { - int32_t r; - to_i32(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int to_i64(int64_t* out, T const* a) - { - *out = static_cast(*a); - return 0; - } - - template - __device__ inline int to_i64(optional* out, optional const* a) - { - if (a->is_valid()) { - int64_t r; - to_i64(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int to_u32(uint32_t* out, T const* a) - { - *out = static_cast(*a); - return 0; - } - - template - __device__ inline int to_u32(optional* out, optional const* a) - { - if (a->is_valid()) { - uint32_t r; - to_u32(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int to_u64(uint64_t* out, T const* a) - { - *out = static_cast(*a); - return 0; - } - - template - __device__ inline int to_u64(optional* out, optional const* a) - { - if (a->is_valid()) { - uint64_t r; - to_u64(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int to_f32(float32_t* out, T const* a) - { - *out = static_cast(*a); - return 0; - } - - template - __device__ inline int to_f32(optional* out, optional const* a) - { - if (a->is_valid()) { - float32_t r; - to_f32(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int to_f64(float64_t* out, T const* a) - { - *out = static_cast(*a); - return 0; - } - - template - __device__ inline int to_f64(optional* out, optional const* a) - { - if (a->is_valid()) { - float64_t r; - to_f64(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int cbrt(T* out, T const* a); - - template <> - __device__ inline int cbrt(float32_t* out, float32_t const* a) - { - *out = ::cbrtf(*a); - return 0; - } - - template <> - __device__ inline int cbrt(float64_t* out, float64_t const* a) - { - *out = ::cbrt(*a); - return 0; - } - - template - __device__ inline int cbrt(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - cbrt(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int ceil(T* out, T const* a); - - template <> - __device__ inline int ceil(float32_t* out, float32_t const* a) - { - *out = ::ceilf(*a); - return 0; - } - - template <> - __device__ inline int ceil(float64_t* out, float64_t const* a) - { - *out = ::ceil(*a); - return 0; - } - - template - __device__ inline int ceil(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - ceil(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int cos(T* out, T const* a); - - template <> - __device__ inline int cos(float32_t* out, float32_t const* a) - { - *out = ::cosf(*a); - return 0; - } - - template <> - __device__ inline int cos(float64_t* out, float64_t const* a) - { - *out = ::cos(*a); - return 0; - } - - template - __device__ inline int cos(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - cos(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int cosh(T* out, T const* a); - - template <> - __device__ inline int cosh(float32_t* out, float32_t const* a) - { - *out = ::coshf(*a); - return 0; - } - - template <> - __device__ inline int cosh(float64_t* out, float64_t const* a) - { - *out = ::cosh(*a); - return 0; - } - - template - __device__ inline int cosh(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - cosh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int div(T* out, T const* a, T const* b) - { - *out = (*a / *b); - return 0; - } - - template - __device__ inline int div(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - div(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int equal(bool* out, T const* a, T const* b) - { - *out = (*a == *b); - return 0; - } - - template - __device__ inline int equal(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - bool r; - equal(&r, &a->value(), &b->value()); - *out = r; - } else if (a->is_null() && b->is_null()) { - *out = true; - } else { - *out = false; - } - return 0; - } - - template - __device__ inline int exp(T* out, T const* a); - - template <> - __device__ inline int exp(float32_t* out, float32_t const* a) - { - *out = ::expf(*a); - return 0; - } - - template <> - __device__ inline int exp(float64_t* out, float64_t const* a) - { - *out = ::exp(*a); - return 0; - } - - template - __device__ inline int exp(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - exp(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int floor(T* out, T const* a); - - template <> - __device__ inline int floor(float32_t* out, float32_t const* a) - { - *out = ::floorf(*a); - return 0; - } - - template <> - __device__ inline int floor(float64_t* out, float64_t const* a) - { - *out = ::floor(*a); - return 0; - } - - template - __device__ inline int floor(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - floor(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int greater(bool* out, T const* a, T const* b) - { - *out = (*a > *b); - return 0; - } - - template - __device__ inline int greater(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - bool r; - greater(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = false; - } - return 0; - } - - template - __device__ inline int greater_equal(bool* out, T const* a, T const* b) - { - *out = (*a >= *b); - return 0; - } - - template - __device__ inline int greater_equal(optional* out, - optional const* a, - optional const* b) - { - if (a->is_valid() && b->is_valid()) { - bool r; - greater_equal(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = false; - } - return 0; - } - - template - __device__ inline int identity(T* out, T const* a) - { - *out = *a; - return 0; - } - - template - __device__ inline int identity(optional* out, optional const* a) - { - *out = *a; - return 0; - } - - template - __device__ inline int is_null(bool* out, T const* a) - { - *out = false; - return 0; - } - - template - __device__ inline int is_null(optional* out, optional const* a) - { - *out = a->is_null(); - return 0; - } - - template - __device__ inline int less(bool* out, T const* a, T const* b) - { - *out = (*a < *b); - return 0; - } - - template - __device__ inline int less(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - bool r; - less(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = false; - } - return 0; - } - - template - __device__ inline int less_equal(bool* out, T const* a, T const* b) - { - *out = (*a <= *b); - return 0; - } - - template - __device__ inline int less_equal(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - bool r; - less_equal(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = false; - } - return 0; - } - - template - __device__ inline int log(T* out, T const* a); - - template <> - __device__ inline int log(float32_t* out, float32_t const* a) - { - *out = ::logf(*a); - return 0; - } - - template <> - __device__ inline int log(float64_t* out, float64_t const* a) - { - *out = ::log(*a); - return 0; - } - - template - __device__ inline int log(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - log(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int logical_and(T* out, T const* a, T const* b) - { - *out = (*a && *b); - return 0; - } - - template - __device__ inline int logical_and(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - logical_and(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int logical_or(T* out, T const* a, T const* b) - { - *out = (*a || *b); - return 0; - } - - template - __device__ inline int logical_or(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - logical_or(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int mod(T* out, T const* a, T const* b) - { - *out = (*a % *b); - return 0; - } - - template <> - __device__ inline int mod(float32_t* out, float32_t const* a, float32_t const* b) - { - *out = ::fmodf(*a, *b); - return 0; - } - - template <> - __device__ inline int mod(float64_t* out, float64_t const* a, float64_t const* b) - { - *out = ::fmod(*a, *b); - return 0; - } - - template - __device__ inline int mod(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - mod(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int mul(T* out, T const* a, T const* b) - { - *out = (*a * *b); - return 0; - } - - template - __device__ inline int mul(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - mul(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int null_equal(bool* out, T const* a, T const* b) - { - *out = (*a == *b); - return 0; - } - - template - __device__ inline int null_equal(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - *out = (*(*a) == *(*b)); - } else if (a->is_null() && b->is_null()) { - *out = true; - } else { - *out = false; - } - return 0; - } - - template - __device__ inline int null_logical_and(T* out, T const* a, T const* b) - { - *out = (*a && *b); - return 0; - } - - template - __device__ inline int null_logical_and(optional* out, - optional const* a, - optional const* b) - { - if (a->is_valid() && b->is_valid()) { - bool r; - null_logical_and(&r, &a->value(), &b->value()); - *out = r; - } else if (a->is_null() && b->is_null()) { - *out = nullopt; - } else { - if (a->is_valid() ? *(*a) : *(*b)) { - *out = nullopt; - } else { - *out = false; - } - } - return 0; - } - - template - __device__ inline int null_logical_or(T* out, T const* a, T const* b) - { - *out = (*a || *b); - return 0; - } - - template - __device__ inline int null_logical_or(optional* out, - optional const* a, - optional const* b) - { - if (a->is_valid() && b->is_valid()) { - bool r; - null_logical_or(&r, &a->value(), &b->value()); - *out = r; - } else if (a->is_null() && b->is_null()) { - *out = nullopt; - } else { - if (a->is_valid() ? *(*a) : *(*b)) { - *out = true; - } else { - *out = nullopt; - } - } - return 0; - } - - template - __device__ inline int pow(T* out, T const* a, T const* b); - - template <> - __device__ inline int pow(float32_t* out, float32_t const* a, float32_t const* b) - { - *out = ::powf(*a, *b); - return 0; - } - - template <> - __device__ inline int pow(float64_t* out, float64_t const* a, float64_t const* b) - { - *out = ::pow(*a, *b); - return 0; - } - - template - __device__ inline int pow(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - pow(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int pymod(T* out, T const* a, T const* b) - { - *out = (*a % *b + *b) % *b; - return 0; - } - - template <> - __device__ inline int pymod(float32_t* out, float32_t const* a, float32_t const* b) - { - *out = ::fmodf(::fmodf(*a, *b) + *b, *b); - return 0; - } - - template <> - __device__ inline int pymod(float64_t* out, float64_t const* a, float64_t const* b) - { - *out = ::fmod(::fmod(*a, *b) + *b, *b); - return 0; - } - - template - __device__ inline int pymod(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - pymod(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int rint(T* out, T const* a); - - template <> - __device__ inline int rint(float32_t* out, float32_t const* a) - { - *out = ::rintf(*a); - return 0; - } - - template <> - __device__ inline int rint(float64_t* out, float64_t const* a) - { - *out = ::rint(*a); - return 0; - } - - template - __device__ inline int rint(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - rint(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int sin(T* out, T const* a); - - template <> - __device__ inline int sin(float32_t* out, float32_t const* a) - { - *out = ::sinf(*a); - return 0; - } - - template <> - __device__ inline int sin(float64_t* out, float64_t const* a) - { - *out = ::sin(*a); - return 0; - } - - template - __device__ inline int sin(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - sin(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int sinh(T* out, T const* a); - - template <> - __device__ inline int sinh(float32_t* out, float32_t const* a) - { - *out = ::sinhf(*a); - return 0; - } - - template <> - __device__ inline int sinh(float64_t* out, float64_t const* a) - { - *out = ::sinh(*a); - return 0; - } - - template - __device__ inline int sinh(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - sinh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int sub(T* out, T const* a, T const* b) - { - *out = *a - *b; - return 0; - } - - template - __device__ inline int sub(optional* out, optional const* a, optional const* b) - { - if (a->is_valid() && b->is_valid()) { - T r; - sub(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int tanh(T* out, T const* a); - - template <> - __device__ inline int tanh(float32_t* out, float32_t const* a) - { - *out = ::tanhf(*a); - return 0; - } - - template <> - __device__ inline int tanh(float64_t* out, float64_t const* a) - { - *out = ::tanh(*a); - return 0; - } - - template - __device__ inline int tanh(optional* out, optional const* a) - { - if (a->is_valid()) { - T r; - tanh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return 0; - } - - template - __device__ inline int if_else(T* out, - bool const* condition, - T const* true_value, - T const* false_value) - { - *out = *condition ? *true_value : *false_value; - return 0; - } - - template - __device__ inline int if_else(optional* out, - optional const* condition, - optional const* true_value, - optional const* false_value) - { - if (condition->is_valid() && true_value->is_valid() && false_value->is_valid()) { - if_else(&out->value(), &condition->value(), &true_value->value(), &false_value->value()); - } else { - *out = nullopt; - } - return 0; - } - - } // namespace operators - } // namespace lite -} // namespace cudf diff --git a/cpp/src/jit/lite/operators.cu b/cpp/src/jit/lite/operators.cu deleted file mode 100644 index 37736a062477..000000000000 --- a/cpp/src/jit/lite/operators.cu +++ /dev/null @@ -1,602 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -namespace __attribute__((visibility("default"))) cudf -{ - namespace lite { - - // Pre-instantiate operator entry points so they can be linked by symbol name. - -#define UNOP_R(NAME, RET_TYPE, TYPE) \ - extern "C" __device__ int NAME##__##TYPE(RET_TYPE* out, TYPE const* a) \ - { \ - return operators::NAME(out, a); \ - } \ - \ - extern "C" __device__ int NAME##__opt##TYPE(optional* out, optional const* a) \ - { \ - return operators::NAME(out, a); \ - } - -#define UNOP(NAME, TYPE) UNOP_R(NAME, TYPE, TYPE) - -#define UNPRED(NAME, TYPE) UNOP_R(NAME, bool, TYPE) - -#define CASTOP(NAME, TARGET, TYPE) UNOP_R(NAME, TARGET, TYPE) - -#define BINOP_R(NAME, RET_TYPE, TYPE) \ - extern "C" __device__ int NAME##__##TYPE(RET_TYPE* out, TYPE const* a, TYPE const* b) \ - { \ - return operators::NAME(out, a, b); \ - } \ - \ - extern "C" __device__ int NAME##__opt##TYPE( \ - optional* out, optional const* a, optional const* b) \ - { \ - return operators::NAME(out, a, b); \ - } - -#define BINOP(NAME, TYPE) BINOP_R(NAME, TYPE, TYPE) - -#define BINCMP(NAME, TYPE) BINOP_R(NAME, bool, TYPE) - -#define COND_OP_R(NAME, RET_TYPE, TYPE) \ - extern "C" __device__ int NAME##__##TYPE( \ - RET_TYPE* out, bool const* a, TYPE const* b, TYPE const* c) \ - { \ - return operators::NAME(out, a, b, c); \ - } \ - \ - extern "C" __device__ int NAME##__opt##TYPE(optional* out, \ - optional const* a, \ - optional const* b, \ - optional const* c) \ - { \ - return operators::NAME(out, a, b, c); \ - } - -#define COND_OP(NAME, TYPE) COND_OP_R(NAME, TYPE, TYPE) - - // Unary numeric - UNOP(abs, i8) - UNOP(abs, i16) - UNOP(abs, i32) - UNOP(abs, i64) - UNOP(abs, f32) - UNOP(abs, f64) - - // Unary floating point math - UNOP(arccos, f32) - UNOP(arccos, f64) - UNOP(arccosh, f32) - UNOP(arccosh, f64) - UNOP(arcsin, f32) - UNOP(arcsin, f64) - UNOP(arcsinh, f32) - UNOP(arcsinh, f64) - UNOP(arctan, f32) - UNOP(arctan, f64) - UNOP(arctanh, f32) - UNOP(arctanh, f64) - UNOP(cbrt, f32) - UNOP(cbrt, f64) - UNOP(ceil, f32) - UNOP(ceil, f64) - UNOP(cos, f32) - UNOP(cos, f64) - UNOP(cosh, f32) - UNOP(cosh, f64) - UNOP(exp, f32) - UNOP(exp, f64) - UNOP(floor, f32) - UNOP(floor, f64) - UNOP(log, f32) - UNOP(log, f64) - UNOP(rint, f32) - UNOP(rint, f64) - UNOP(sin, f32) - UNOP(sin, f64) - UNOP(sinh, f32) - UNOP(sinh, f64) - UNOP(tanh, f32) - UNOP(tanh, f64) - - // Unary bitwise and identity-like - UNOP(bit_invert, i8) - UNOP(bit_invert, i16) - UNOP(bit_invert, i32) - UNOP(bit_invert, i64) - UNOP(bit_invert, u8) - UNOP(bit_invert, u16) - UNOP(bit_invert, u32) - UNOP(bit_invert, u64) - - UNOP(identity, b8) - UNOP(identity, i8) - UNOP(identity, i16) - UNOP(identity, i32) - UNOP(identity, i64) - UNOP(identity, u8) - UNOP(identity, u16) - UNOP(identity, u32) - UNOP(identity, u64) - UNOP(identity, f32) - UNOP(identity, f64) - UNOP(identity, dec32) - UNOP(identity, dec64) - UNOP(identity, dec128) - UNOP(identity, ts_D) - UNOP(identity, ts_h) - UNOP(identity, ts_m) - UNOP(identity, ts_s) - UNOP(identity, ts_ms) - UNOP(identity, ts_us) - UNOP(identity, ts_ns) - UNOP(identity, dur_D) - UNOP(identity, dur_h) - UNOP(identity, dur_m) - UNOP(identity, dur_s) - UNOP(identity, dur_ms) - UNOP(identity, dur_us) - UNOP(identity, dur_ns) - UNOP(identity, str) - - UNPRED(is_null, b8) - UNPRED(is_null, i8) - UNPRED(is_null, i16) - UNPRED(is_null, i32) - UNPRED(is_null, i64) - UNPRED(is_null, u8) - UNPRED(is_null, u16) - UNPRED(is_null, u32) - UNPRED(is_null, u64) - UNPRED(is_null, f32) - UNPRED(is_null, f64) - UNPRED(is_null, dec32) - UNPRED(is_null, dec64) - UNPRED(is_null, dec128) - UNPRED(is_null, ts_D) - UNPRED(is_null, ts_h) - UNPRED(is_null, ts_m) - UNPRED(is_null, ts_s) - UNPRED(is_null, ts_ms) - UNPRED(is_null, ts_us) - UNPRED(is_null, ts_ns) - UNPRED(is_null, dur_D) - UNPRED(is_null, dur_h) - UNPRED(is_null, dur_m) - UNPRED(is_null, dur_s) - UNPRED(is_null, dur_ms) - UNPRED(is_null, dur_us) - UNPRED(is_null, dur_ns) - UNPRED(is_null, str) - - // Binary arithmetic - BINOP(add, i8) - BINOP(add, i16) - BINOP(add, i32) - BINOP(add, i64) - BINOP(add, u8) - BINOP(add, u16) - BINOP(add, u32) - BINOP(add, u64) - BINOP(add, f32) - BINOP(add, f64) - BINOP(add, dec32) - BINOP(add, dec64) - BINOP(add, dec128) - BINOP(add, dur_D) - BINOP(add, dur_h) - BINOP(add, dur_m) - BINOP(add, dur_s) - BINOP(add, dur_ms) - BINOP(add, dur_us) - BINOP(add, dur_ns) - - BINOP(sub, i8) - BINOP(sub, i16) - BINOP(sub, i32) - BINOP(sub, i64) - BINOP(sub, u8) - BINOP(sub, u16) - BINOP(sub, u32) - BINOP(sub, u64) - BINOP(sub, f32) - BINOP(sub, f64) - BINOP(sub, dec32) - BINOP(sub, dec64) - BINOP(sub, dec128) - BINOP(sub, dur_D) - BINOP(sub, dur_h) - BINOP(sub, dur_m) - BINOP(sub, dur_s) - BINOP(sub, dur_ms) - BINOP(sub, dur_us) - BINOP(sub, dur_ns) - - BINOP(mul, i8) - BINOP(mul, i16) - BINOP(mul, i32) - BINOP(mul, i64) - BINOP(mul, u8) - BINOP(mul, u16) - BINOP(mul, u32) - BINOP(mul, u64) - BINOP(mul, f32) - BINOP(mul, f64) - BINOP(mul, dec32) - BINOP(mul, dec64) - BINOP(mul, dec128) - - BINOP(div, i8) - BINOP(div, i16) - BINOP(div, i32) - BINOP(div, i64) - BINOP(div, u8) - BINOP(div, u16) - BINOP(div, u32) - BINOP(div, u64) - BINOP(div, f32) - BINOP(div, f64) - BINOP(div, dec32) - BINOP(div, dec64) - BINOP(div, dec128) - - BINOP(mod, i8) - BINOP(mod, i16) - BINOP(mod, i32) - BINOP(mod, i64) - BINOP(mod, u8) - BINOP(mod, u16) - BINOP(mod, u32) - BINOP(mod, u64) - BINOP(mod, f32) - BINOP(mod, f64) - BINOP(mod, dec32) - BINOP(mod, dec64) - BINOP(mod, dec128) - - BINOP(pymod, i8) - BINOP(pymod, i16) - BINOP(pymod, i32) - BINOP(pymod, i64) - BINOP(pymod, u8) - BINOP(pymod, u16) - BINOP(pymod, u32) - BINOP(pymod, u64) - BINOP(pymod, f32) - BINOP(pymod, f64) - - BINOP(pow, f32) - BINOP(pow, f64) - - // Binary bitwise/logical - BINOP(bit_and, i8) - BINOP(bit_and, i16) - BINOP(bit_and, i32) - BINOP(bit_and, i64) - BINOP(bit_and, u8) - BINOP(bit_and, u16) - BINOP(bit_and, u32) - BINOP(bit_and, u64) - - BINOP(bit_or, i8) - BINOP(bit_or, i16) - BINOP(bit_or, i32) - BINOP(bit_or, i64) - BINOP(bit_or, u8) - BINOP(bit_or, u16) - BINOP(bit_or, u32) - BINOP(bit_or, u64) - - BINOP(bit_xor, i8) - BINOP(bit_xor, i16) - BINOP(bit_xor, i32) - BINOP(bit_xor, i64) - BINOP(bit_xor, u8) - BINOP(bit_xor, u16) - BINOP(bit_xor, u32) - BINOP(bit_xor, u64) - - BINOP(logical_and, b8) - BINOP(logical_and, i8) - BINOP(logical_and, i16) - BINOP(logical_and, i32) - BINOP(logical_and, i64) - BINOP(logical_and, u8) - BINOP(logical_and, u16) - BINOP(logical_and, u32) - BINOP(logical_and, u64) - - // Binary predicates - BINCMP(equal, b8) - BINCMP(equal, i8) - BINCMP(equal, i16) - BINCMP(equal, i32) - BINCMP(equal, i64) - BINCMP(equal, u8) - BINCMP(equal, u16) - BINCMP(equal, u32) - BINCMP(equal, u64) - BINCMP(equal, f32) - BINCMP(equal, f64) - BINCMP(equal, dec32) - BINCMP(equal, dec64) - BINCMP(equal, dec128) - BINCMP(equal, ts_D) - BINCMP(equal, ts_h) - BINCMP(equal, ts_m) - BINCMP(equal, ts_s) - BINCMP(equal, ts_ms) - BINCMP(equal, ts_us) - BINCMP(equal, ts_ns) - BINCMP(equal, dur_D) - BINCMP(equal, dur_h) - BINCMP(equal, dur_m) - BINCMP(equal, dur_s) - BINCMP(equal, dur_ms) - BINCMP(equal, dur_us) - BINCMP(equal, dur_ns) - BINCMP(equal, str) - - BINCMP(null_equal, b8) - BINCMP(null_equal, i8) - BINCMP(null_equal, i16) - BINCMP(null_equal, i32) - BINCMP(null_equal, i64) - BINCMP(null_equal, u8) - BINCMP(null_equal, u16) - BINCMP(null_equal, u32) - BINCMP(null_equal, u64) - BINCMP(null_equal, f32) - BINCMP(null_equal, f64) - BINCMP(null_equal, dec32) - BINCMP(null_equal, dec64) - BINCMP(null_equal, dec128) - BINCMP(null_equal, ts_D) - BINCMP(null_equal, ts_h) - BINCMP(null_equal, ts_m) - BINCMP(null_equal, ts_s) - BINCMP(null_equal, ts_ms) - BINCMP(null_equal, ts_us) - BINCMP(null_equal, ts_ns) - BINCMP(null_equal, dur_D) - BINCMP(null_equal, dur_h) - BINCMP(null_equal, dur_m) - BINCMP(null_equal, dur_s) - BINCMP(null_equal, dur_ms) - BINCMP(null_equal, dur_us) - BINCMP(null_equal, dur_ns) - BINCMP(null_equal, str) - - BINCMP(less, b8) - BINCMP(less, i8) - BINCMP(less, i16) - BINCMP(less, i32) - BINCMP(less, i64) - BINCMP(less, u8) - BINCMP(less, u16) - BINCMP(less, u32) - BINCMP(less, u64) - BINCMP(less, f32) - BINCMP(less, f64) - BINCMP(less, dec32) - BINCMP(less, dec64) - BINCMP(less, dec128) - BINCMP(less, ts_D) - BINCMP(less, ts_h) - BINCMP(less, ts_m) - BINCMP(less, ts_s) - BINCMP(less, ts_ms) - BINCMP(less, ts_us) - BINCMP(less, ts_ns) - BINCMP(less, dur_D) - BINCMP(less, dur_h) - BINCMP(less, dur_m) - BINCMP(less, dur_s) - BINCMP(less, dur_ms) - BINCMP(less, dur_us) - BINCMP(less, dur_ns) - BINCMP(less, str) - - BINCMP(less_equal, b8) - BINCMP(less_equal, i8) - BINCMP(less_equal, i16) - BINCMP(less_equal, i32) - BINCMP(less_equal, i64) - BINCMP(less_equal, u8) - BINCMP(less_equal, u16) - BINCMP(less_equal, u32) - BINCMP(less_equal, u64) - BINCMP(less_equal, f32) - BINCMP(less_equal, f64) - BINCMP(less_equal, dec32) - BINCMP(less_equal, dec64) - BINCMP(less_equal, dec128) - BINCMP(less_equal, ts_D) - BINCMP(less_equal, ts_h) - BINCMP(less_equal, ts_m) - BINCMP(less_equal, ts_s) - BINCMP(less_equal, ts_ms) - BINCMP(less_equal, ts_us) - BINCMP(less_equal, ts_ns) - BINCMP(less_equal, dur_D) - BINCMP(less_equal, dur_h) - BINCMP(less_equal, dur_m) - BINCMP(less_equal, dur_s) - BINCMP(less_equal, dur_ms) - BINCMP(less_equal, dur_us) - BINCMP(less_equal, dur_ns) - BINCMP(less_equal, str) - - BINCMP(greater, b8) - BINCMP(greater, i8) - BINCMP(greater, i16) - BINCMP(greater, i32) - BINCMP(greater, i64) - BINCMP(greater, u8) - BINCMP(greater, u16) - BINCMP(greater, u32) - BINCMP(greater, u64) - BINCMP(greater, f32) - BINCMP(greater, f64) - BINCMP(greater, dec32) - BINCMP(greater, dec64) - BINCMP(greater, dec128) - BINCMP(greater, ts_D) - BINCMP(greater, ts_h) - BINCMP(greater, ts_m) - BINCMP(greater, ts_s) - BINCMP(greater, ts_ms) - BINCMP(greater, ts_us) - BINCMP(greater, ts_ns) - BINCMP(greater, dur_D) - BINCMP(greater, dur_h) - BINCMP(greater, dur_m) - BINCMP(greater, dur_s) - BINCMP(greater, dur_ms) - BINCMP(greater, dur_us) - BINCMP(greater, dur_ns) - BINCMP(greater, str) - - BINCMP(greater_equal, b8) - BINCMP(greater_equal, i8) - BINCMP(greater_equal, i16) - BINCMP(greater_equal, i32) - BINCMP(greater_equal, i64) - BINCMP(greater_equal, u8) - BINCMP(greater_equal, u16) - BINCMP(greater_equal, u32) - BINCMP(greater_equal, u64) - BINCMP(greater_equal, f32) - BINCMP(greater_equal, f64) - BINCMP(greater_equal, dec32) - BINCMP(greater_equal, dec64) - BINCMP(greater_equal, dec128) - BINCMP(greater_equal, ts_D) - BINCMP(greater_equal, ts_h) - BINCMP(greater_equal, ts_m) - BINCMP(greater_equal, ts_s) - BINCMP(greater_equal, ts_ms) - BINCMP(greater_equal, ts_us) - BINCMP(greater_equal, ts_ns) - BINCMP(greater_equal, dur_D) - BINCMP(greater_equal, dur_h) - BINCMP(greater_equal, dur_m) - BINCMP(greater_equal, dur_s) - BINCMP(greater_equal, dur_ms) - BINCMP(greater_equal, dur_us) - BINCMP(greater_equal, dur_ns) - BINCMP(greater_equal, str) - - // Casts - - CASTOP(to_i32, i32, b8) - CASTOP(to_i32, i32, i8) - CASTOP(to_i32, i32, i16) - CASTOP(to_i32, i32, i32) - CASTOP(to_i32, i32, i64) - CASTOP(to_i32, i32, u8) - CASTOP(to_i32, i32, u16) - CASTOP(to_i32, i32, u32) - CASTOP(to_i32, i32, u64) - CASTOP(to_i32, i32, f32) - CASTOP(to_i32, i32, f64) - - CASTOP(to_i64, i64, b8) - CASTOP(to_i64, i64, i8) - CASTOP(to_i64, i64, i16) - CASTOP(to_i64, i64, i32) - CASTOP(to_i64, i64, i64) - CASTOP(to_i64, i64, u8) - CASTOP(to_i64, i64, u16) - CASTOP(to_i64, i64, u32) - CASTOP(to_i64, i64, u64) - CASTOP(to_i64, i64, f32) - CASTOP(to_i64, i64, f64) - - CASTOP(to_u32, u32, b8) - CASTOP(to_u32, u32, i8) - CASTOP(to_u32, u32, i16) - CASTOP(to_u32, u32, i32) - CASTOP(to_u32, u32, i64) - CASTOP(to_u32, u32, u8) - CASTOP(to_u32, u32, u16) - CASTOP(to_u32, u32, u32) - CASTOP(to_u32, u32, u64) - CASTOP(to_u32, u32, f32) - CASTOP(to_u32, u32, f64) - - CASTOP(to_u64, u64, b8) - CASTOP(to_u64, u64, i8) - CASTOP(to_u64, u64, i16) - CASTOP(to_u64, u64, i32) - CASTOP(to_u64, u64, i64) - CASTOP(to_u64, u64, u8) - CASTOP(to_u64, u64, u16) - CASTOP(to_u64, u64, u32) - CASTOP(to_u64, u64, u64) - CASTOP(to_u64, u64, f32) - CASTOP(to_u64, u64, f64) - - CASTOP(to_f32, f32, b8) - CASTOP(to_f32, f32, i8) - CASTOP(to_f32, f32, i16) - CASTOP(to_f32, f32, i32) - CASTOP(to_f32, f32, i64) - CASTOP(to_f32, f32, u8) - CASTOP(to_f32, f32, u16) - CASTOP(to_f32, f32, u32) - CASTOP(to_f32, f32, u64) - CASTOP(to_f32, f32, f32) - CASTOP(to_f32, f32, f64) - - CASTOP(to_f64, f64, b8) - CASTOP(to_f64, f64, i8) - CASTOP(to_f64, f64, i16) - CASTOP(to_f64, f64, i32) - CASTOP(to_f64, f64, i64) - CASTOP(to_f64, f64, u8) - CASTOP(to_f64, f64, u16) - CASTOP(to_f64, f64, u32) - CASTOP(to_f64, f64, u64) - CASTOP(to_f64, f64, f32) - CASTOP(to_f64, f64, f64) - - // Ternary - COND_OP(if_else, b8) - COND_OP(if_else, i8) - COND_OP(if_else, i16) - COND_OP(if_else, i32) - COND_OP(if_else, i64) - COND_OP(if_else, u8) - COND_OP(if_else, u16) - COND_OP(if_else, u32) - COND_OP(if_else, u64) - COND_OP(if_else, f32) - COND_OP(if_else, f64) - COND_OP(if_else, dec32) - COND_OP(if_else, dec64) - COND_OP(if_else, dec128) - COND_OP(if_else, ts_D) - COND_OP(if_else, ts_h) - COND_OP(if_else, ts_m) - COND_OP(if_else, ts_s) - COND_OP(if_else, ts_ms) - COND_OP(if_else, ts_us) - COND_OP(if_else, ts_ns) - COND_OP(if_else, dur_D) - COND_OP(if_else, dur_h) - COND_OP(if_else, dur_m) - COND_OP(if_else, dur_s) - COND_OP(if_else, dur_ms) - COND_OP(if_else, dur_us) - COND_OP(if_else, dur_ns) - COND_OP(if_else, str) - - } // namespace lite -} // namespace cudf diff --git a/cpp/src/librtcx/embed.cmake b/cpp/src/librtcx/embed.cmake index e58b2f7d49d7..4c6a6723d036 100644 --- a/cpp/src/librtcx/embed.cmake +++ b/cpp/src/librtcx/embed.cmake @@ -178,7 +178,11 @@ function(embed) set(EMBED_SCRIPT__OUTPUT_DIR "${OUTPUT_DIR}") configure_file(${EMBED_SCRIPT_TEMPLATE} ${CONFIGURED_EMBED_SCRIPT} @ONLY) - file(GENERATE OUTPUT "${EMBED_SCRIPT}" INPUT "${CONFIGURED_EMBED_SCRIPT}") + file( + GENERATE + OUTPUT "${EMBED_SCRIPT}" + INPUT "${CONFIGURED_EMBED_SCRIPT}" + ) add_executable("${TARGET}__jit_embed_run" EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") target_include_directories("${TARGET}__jit_embed_run" PRIVATE ${ZSTD_INCLUDE_DIR}) diff --git a/cpp/src/transform/compute_columns_jit.cu b/cpp/src/transform/compute_columns_jit.cu new file mode 100644 index 000000000000..3060c26af7d0 --- /dev/null +++ b/cpp/src/transform/compute_columns_jit.cu @@ -0,0 +1,18 @@ + + + +#include + +namespace CUDF_EXPORT cudf { + +std::unique_ptr compute_columns( + std::span inputs, + std::span udf, + std::span output_types, + void* user_data, + null_aware is_null_aware = null_aware::NO, + std::optional row_size = std::nullopt, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + +} \ No newline at end of file diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index f13a3af71b69..88fc9080ac15 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -679,6 +679,11 @@ ConfigureTest(ENCODE_TEST encode/encode_tests.cpp) # * ast tests ------------------------------------------------------------------------------------- ConfigureTest(AST_TEST ast/transform_tests.cpp ast/ast_tree_tests.cpp) +# ################################################################################################## +# * compute columns tests +# ------------------------------------------------------------------------------------- +ConfigureTest(COMPUTE_COLUMNS_TEST transform/compute_columns_test.cpp) + # ################################################################################################## # * lists tests ---------------------------------------------------------------------------------- ConfigureTest( diff --git a/cpp/tests/transform/compute_columns_profit_operator.cu b/cpp/tests/transform/compute_columns_profit_operator.cu new file mode 100644 index 000000000000..9bf51de2754d --- /dev/null +++ b/cpp/tests/transform/compute_columns_profit_operator.cu @@ -0,0 +1,70 @@ + +#include + +#include + +#include + +__device__ inline bool strcontains(cudf::string_view text, cudf::string_view token) +{ + return text.find(token) != cudf::string_view::npos; +} + +template +__device__ inline T load(void const* inputs, int input_stride, int arg) +{ + auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); + return *p; +} + +template +__device__ inline void store(void* outputs, int output_stride, int arg, T value) +{ + auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); + *p = value; +} + +extern "C" __device__ int operation( + void*, long int, void const* inputs, int input_stride, void* outputs, int output_stride) +{ + // Input schema: + // 0: price(double), + // 1: qty(int32), + // 2: discount(float), + // 3: tax_rate(float), + // 4: ship_mode(string_view), + // 5: comment(string_view) + auto price = load(inputs, input_stride, 0); + auto qty = load(inputs, input_stride, 1); + auto discount = load(inputs, input_stride, 2); + auto tax_rate = load(inputs, input_stride, 3); + auto ship = load(inputs, input_stride, 4); + auto comment = load(inputs, input_stride, 5); + + // Parameters: + // 6: priority_threshold(double), + // 7: require_expedited(bool), + // 8: promo_discount_threshold(float) + auto priority_threshold = load(inputs, input_stride, 6); + auto require_expedited = load(inputs, input_stride, 7); + auto promo_discount_threshold = load(inputs, input_stride, 8); + + auto net = price * qty * (1.0 - discount); + auto gross = net * (1.0 + tax_rate); + + auto expedited = strcontains(ship, cudf::string_view{"expedited", 9}); + auto priority = (gross > priority_threshold) && (!require_expedited || expedited); + + auto has_promo_keyword = strcontains(comment, cudf::string_view{"promo", 5}) || + strcontains(comment, cudf::string_view{"sale", 4}) || + strcontains(comment, cudf::string_view{"coupon", 6}); + + auto is_promo = has_promo_keyword || (discount >= promo_discount_threshold); + + store(outputs, output_stride, 0, net); + store(outputs, output_stride, 1, gross); + store(outputs, output_stride, 2, priority); + store(outputs, output_stride, 3, is_promo); + + return 0; +} diff --git a/cpp/tests/transform/compute_columns_test.cpp b/cpp/tests/transform/compute_columns_test.cpp new file mode 100644 index 000000000000..e69de29bb2d1 From 6fe31084e3c0a245786d98e3fe057bb7b0dff890 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 5 May 2026 20:29:21 +0000 Subject: [PATCH 165/254] code review changes --- cpp/librtcx/rtcx.cpp | 137 +++++++++++++++++++++++-------------------- cpp/librtcx/rtcx.hpp | 12 +--- 2 files changed, 73 insertions(+), 76 deletions(-) diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 68dca71aaf40..47da7273f93b 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -26,77 +26,77 @@ #include #include -#define RTCX_EXPECTS(_condition, _reason, _exception_type) \ +#define RTCX_EXPECTS(condition_, reason_, exception_type_) \ do { \ - if (!(_condition)) { \ - throw _exception_type{::std::format("RTCX failure at: {}:{}: {}", \ + if (!(condition_)) { \ + throw exception_type_{::std::format("RTCX failure at: {}:{}: {}", \ ::std::source_location::current().file_name(), \ ::std::source_location::current().line(), \ - (_reason))}; \ + (reason_))}; \ } \ } while (0) -#define RTCX_FAIL(_reason, _exception_type) \ +#define RTCX_FAIL(reason_, exception_type_) \ do { \ - throw _exception_type{::std::format("RTCX failure at: {}:{}: {}", \ + throw exception_type_{::std::format("RTCX failure at: {}:{}: {}", \ ::std::source_location::current().file_name(), \ ::std::source_location::current().line(), \ - (_reason))}; \ + (reason_))}; \ } while (0) -#define RTCX_CHECK_CUDA(...) \ - do { \ - ::CUresult __result = (__VA_ARGS__); \ - if (__result != ::CUDA_SUCCESS) { \ - char const* __enum_str; \ - RTCX_EXPECTS(::rtcx::cu->GetErrorString(__result, &__enum_str) == ::CUDA_SUCCESS, \ - "Unable to get CUDA error string", \ - std::runtime_error); \ - auto __errstr = ::std::format("(cuda) expression `{}` failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast<::std::int64_t>(__result), \ - __enum_str); \ - RTCX_FAIL(__errstr, ::std::runtime_error); \ - } \ +#define RTCX_CHECK_CUDA(...) \ + do { \ + ::CUresult result_ = (__VA_ARGS__); \ + if (result_ != ::CUDA_SUCCESS) { \ + char const* enum_str_; \ + RTCX_EXPECTS(::rtcx::cu->GetErrorString(result_, &enum_str_) == ::CUDA_SUCCESS, \ + "Unable to get CUDA error string", \ + std::runtime_error); \ + auto errstr_ = ::std::format("(cuda) expression `{}` failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::std::int64_t>(result_), \ + enum_str_); \ + RTCX_FAIL(errstr_, ::std::runtime_error); \ + } \ } while (0) -#define RTCX_CHECK_CUDART(...) \ - do { \ - ::cudaError_t __result = (__VA_ARGS__); \ - if (__result != ::cudaSuccess) { \ - char const* __enum_name = ::cudaGetErrorName(__result); \ - char const* __enum_msg = ::cudaGetErrorString(__result); \ - auto __errstr = ::std::format("(cudart) expression `{}` failed, with error ({}: {}): {}", \ - #__VA_ARGS__, \ - static_cast<::std::int64_t>(__result), \ - __enum_name, \ - __enum_msg); \ - RTCX_FAIL(__errstr, ::std::runtime_error); \ - } \ +#define RTCX_CHECK_CUDART(...) \ + do { \ + ::cudaError_t result_ = (__VA_ARGS__); \ + if (result_ != ::cudaSuccess) { \ + char const* enum_name_ = ::cudaGetErrorName(result_); \ + char const* enum_msg_ = ::cudaGetErrorString(result_); \ + auto errstr_ = ::std::format("(cudart) expression `{}` failed, with error ({}: {}): {}", \ + #__VA_ARGS__, \ + static_cast<::std::int64_t>(result_), \ + enum_name_, \ + enum_msg_); \ + RTCX_FAIL(errstr_, ::std::runtime_error); \ + } \ } while (0) -#define RTCX_CHECK_NVRTC(...) \ - do { \ - ::nvrtcResult __result = (__VA_ARGS__); \ - if (__result != ::NVRTC_SUCCESS) { \ - auto __errstr = ::std::format("(nvrtc) expression `{}` failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast<::std::int64_t>(__result), \ - ::rtcx::nvrtc->GetErrorString(__result)); \ - RTCX_FAIL(__errstr, ::std::runtime_error); \ - } \ +#define RTCX_CHECK_NVRTC(...) \ + do { \ + ::nvrtcResult result_ = (__VA_ARGS__); \ + if (result_ != ::NVRTC_SUCCESS) { \ + auto errstr_ = ::std::format("(nvrtc) expression `{}` failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::std::int64_t>(result_), \ + ::rtcx::nvrtc->GetErrorString(result_)); \ + RTCX_FAIL(errstr_, ::std::runtime_error); \ + } \ } while (0) -#define RTCX_CHECK_NVJITLINK(...) \ - do { \ - ::nvJitLinkResult __result = (__VA_ARGS__); \ - if (__result != ::NVJITLINK_SUCCESS) { \ - auto __errstr = ::std::format("(nvJitLink) expression `{}` failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast<::std::int64_t>(__result), \ - ::rtcx::nvJitLinkResult_string(__result)); \ - RTCX_FAIL(__errstr, ::std::runtime_error); \ - } \ +#define RTCX_CHECK_NVJITLINK(...) \ + do { \ + ::nvJitLinkResult result_ = (__VA_ARGS__); \ + if (result_ != ::NVJITLINK_SUCCESS) { \ + auto errstr_ = ::std::format("(nvJitLink) expression `{}` failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::std::int64_t>(result_), \ + ::rtcx::nvJitLinkResult_string(result_)); \ + RTCX_FAIL(errstr_, ::std::runtime_error); \ + } \ } while (0) namespace rtcx { @@ -286,8 +286,6 @@ void* get_symbol(char const* lib_name, void* handle, char const* sym_name) inline constexpr std::int32_t major_version(std::int32_t version) { return version / 1000; } -inline constexpr std::int32_t minor_version(std::int32_t version) { return (version % 1000) / 10; } - struct LibCuda { void* _handle = nullptr; @@ -304,7 +302,7 @@ struct LibCuda { static void* _load() { - std::string lib_names[] = {"libcuda.so", "libcuda.so.1"}; // NOLINT(modernize-avoid-c-arrays) + std::string lib_names[] = {"libcuda.so.1"}; // NOLINT(modernize-avoid-c-arrays) return load_dll("libcuda.so", lib_names); } @@ -334,15 +332,19 @@ struct LibNVRTC { static void* _load() { + auto expected_major_version = major_version(CUDA_VERSION); std::int32_t cuda_version; RTCX_CHECK_CUDART(cudaRuntimeGetVersion(&cuda_version)); std::int32_t major = major_version(cuda_version); - std::int32_t minor = minor_version(cuda_version); + RTCX_EXPECTS(expected_major_version == major, + std::format("LibNVRTC Compatibility Error: CUDA major version mismatch. Expected " + "major runtime version: {}, got major runtime version: {})", + expected_major_version, + major), + std::runtime_error); std::string lib_names[] = // NOLINT(modernize-avoid-c-arrays) - {std::format("libnvrtc.so.{}.{}", major, minor), - std::format("libnvrtc.so.{}", major), - "libnvrtc.so"}; + {std::format("libnvrtc.so.{}", major)}; return load_dll("libnvrtc.so", lib_names); } @@ -374,15 +376,20 @@ struct LibNVJitLink { static void* _load() { + auto expected_major_version = major_version(CUDA_VERSION); std::int32_t cuda_version; RTCX_CHECK_CUDART(cudaRuntimeGetVersion(&cuda_version)); std::int32_t major = major_version(cuda_version); - std::int32_t minor = minor_version(cuda_version); + RTCX_EXPECTS( + expected_major_version == major, + std::format("LibNVJitLink Compatibility Error: CUDA major version mismatch. Expected " + "major runtime version: {}, got major runtime version: {})", + expected_major_version, + major), + std::runtime_error); std::string lib_names[] = // NOLINT(modernize-avoid-c-arrays) - {std::format("libnvJitLink.so.{}.{}", major, minor), - std::format("libnvJitLink.so.{}", major), - "libnvJitLink.so"}; + {std::format("libnvJitLink.so.{}", major), "libnvJitLink.so"}; return load_dll("libnvJitLink.so", lib_names); } diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index f74ae69fbc61..11cdd173d408 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -521,7 +521,7 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { }; struct cache_stats_counter { - struct alignas(CACHELINE_ALIGNMENT) entry { + struct entry { std::uint64_t value_ = 0; void incr() @@ -812,7 +812,6 @@ inline std::string reflect_bool(bool value) { return std::format("(bool){}", val /** * @brief Reflect an integer value into its CUDA string representation - * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ @@ -823,7 +822,6 @@ inline std::string reflect_int(std::uint8_t value) /** * @brief Reflect an integer value into its CUDA string representation - * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ @@ -834,7 +832,6 @@ inline std::string reflect_int(std::uint16_t value) /** * @brief Reflect an integer value into its CUDA string representation - * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ @@ -845,7 +842,6 @@ inline std::string reflect_int(std::uint32_t value) /** * @brief Reflect an integer value into its CUDA string representation - * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ @@ -856,7 +852,6 @@ inline std::string reflect_int(std::uint64_t value) /** * @brief Reflect an integer value into its CUDA string representation - * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ @@ -864,7 +859,6 @@ inline std::string reflect_int(std::int8_t value) { return std::format("(signed /** * @brief Reflect an integer value into its CUDA string representation - * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ @@ -875,7 +869,6 @@ inline std::string reflect_int(std::int16_t value) /** * @brief Reflect an integer value into its CUDA string representation - * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ @@ -883,7 +876,6 @@ inline std::string reflect_int(std::int32_t value) { return std::format("(signed /** * @brief Reflect an integer value into its CUDA string representation - * @tparam T An integral type (e.g., std::uint8_t, std::int32_t, etc.) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ @@ -894,7 +886,6 @@ inline std::string reflect_int(std::int64_t value) /** * @brief Reflect a floating-point value into its CUDA string representation - * @tparam T A floating-point type (e.g., float, double) * @param value The floating-point value to be reflected * @return A string containing the CUDA representation of the floating-point value */ @@ -902,7 +893,6 @@ inline std::string reflect_float(float value) { return std::format("(float){}F", /** * @brief Reflect a floating-point value into its CUDA string representation - * @tparam T A floating-point type (e.g., float, double) * @param value The floating-point value to be reflected * @return A string containing the CUDA representation of the floating-point value */ From e9ec212916a7739b57163a3ee3246eef4211ff11 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 5 May 2026 20:31:17 +0000 Subject: [PATCH 166/254] Refactor decompress_blob function signature to use std::uint8_t and std::size_t for improved type clarity --- cpp/librtcx/rtcx.cpp | 10 +++++----- cpp/librtcx/rtcx.hpp | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 47da7273f93b..50be25bb6111 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -1241,8 +1241,8 @@ std::string reflect_template(std::string_view template_name, return std::format("{}<{}>", template_name, join_strings(template_args, ", ")); } -rtcx::byte_buffer decompress_blob(std::span compressed_binary, - size_t uncompressed_size, +rtcx::byte_buffer decompress_blob(std::span compressed_binary, + std::size_t uncompressed_size, std::string_view compression) { RTCX_EXPECTS(compression == "none" || compression == "zstd", @@ -1251,14 +1251,14 @@ rtcx::byte_buffer decompress_blob(std::span compressed_binary, auto decompressed = rtcx::byte_buffer::make(uncompressed_size); if (compression == "zstd") { - size_t errc = ZSTD_decompress( + std::size_t errc = ::ZSTD_decompress( decompressed.data(), uncompressed_size, compressed_binary.data(), compressed_binary.size()); RTCX_EXPECTS( - !ZSTD_isError(errc) && errc == uncompressed_size, + !::ZSTD_isError(errc) && errc == uncompressed_size, std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : ", errc, - ZSTD_getErrorName(errc)), + ::ZSTD_getErrorName(errc)), std::runtime_error); } else { // compression is "none", so just copy the data diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 11cdd173d408..dce681d72304 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -981,8 +981,8 @@ std::string reflect_template(std::string_view template_name, TemplateArgs&&... t * invalid compressed data, or if the decompressed data size does not match the expected * uncompressed size */ -rtcx::byte_buffer decompress_blob(std::span compressed_binary, - size_t uncompressed_size, +rtcx::byte_buffer decompress_blob(std::span compressed_binary, + std::size_t uncompressed_size, std::string_view compression); } // namespace rtcx From 5188ecafea0a05f76ebc4a9b4182ee01e7abffa1 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 5 May 2026 23:27:56 +0000 Subject: [PATCH 167/254] Remove inline specifier from load and store functions in compute_columns_profit_operator.cu --- cpp/tests/transform/compute_columns_profit_operator.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/tests/transform/compute_columns_profit_operator.cu b/cpp/tests/transform/compute_columns_profit_operator.cu index 9bf51de2754d..f05b7b2cb091 100644 --- a/cpp/tests/transform/compute_columns_profit_operator.cu +++ b/cpp/tests/transform/compute_columns_profit_operator.cu @@ -11,14 +11,14 @@ __device__ inline bool strcontains(cudf::string_view text, cudf::string_view tok } template -__device__ inline T load(void const* inputs, int input_stride, int arg) +__device__ T load(void const* inputs, int input_stride, int arg) { auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); return *p; } template -__device__ inline void store(void* outputs, int output_stride, int arg, T value) +__device__ void store(void* outputs, int output_stride, int arg, T value) { auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); *p = value; From 0b0ba2fee98b48cb7992666a77a094b411d3e842 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 12 May 2026 16:45:32 +0000 Subject: [PATCH 168/254] Implement LTO (Link Time Optimization) for transform operations - Added an untyped LTO kernel for generic transform operations in `untyped_lto_kernel.cu`. - Modified the transform implementation in `transform.cu` to support LTO, including changes to kernel instantiation and argument handling. - Introduced new test cases for LTO functionality in `transform_lto_test.cpp`, covering various scenarios including TPC-DS Q7 and inverse square root operations. - Removed obsolete compute columns tests and fragments, consolidating the codebase for better maintainability. - Added new fragments for profit calculation and inverse square root operations to support LTO. - Updated CMake configuration to include new test fragments and dependencies. --- cpp/CMakeLists.txt | 162 ++++++- cpp/doxygen/developer_guide/JIT.md | 89 ++++ cpp/include/cudf/jit/transform_operation.cuh | 25 + cpp/include/cudf/jit/type_tags.cuh | 56 +++ cpp/include/cudf/transform.hpp | 20 +- cpp/librtcx/embed.cmake | 95 +++- cpp/librtcx/embed.hpp | 183 +++++-- cpp/librtcx/embed.in.cpp | 14 +- cpp/librtcx/rtcx.cpp | 92 +++- cpp/src/binaryop/jit/kernel.cu | 8 +- cpp/src/jit/element.cuh | 93 ++-- cpp/src/jit/element_storage.cuh | 53 +- cpp/src/jit/helpers.cpp | 19 +- cpp/src/jit/helpers.hpp | 3 +- cpp/src/jit/jit.cpp | 453 +++--------------- cpp/src/jit/jit.hpp | 59 +-- cpp/src/join/jit/filter_join_kernel.cu | 13 +- cpp/src/rolling/jit/kernel.cu | 19 +- cpp/src/runtime/context.cpp | 18 +- cpp/src/runtime/context.hpp | 17 +- cpp/src/transform/compute_columns_jit.cu | 18 - .../transform/jit/compute_columns_kernel.cu | 104 ---- cpp/src/transform/jit/kernel.cu | 75 ++- cpp/src/transform/jit/lto_kernel.cu | 145 ++++++ cpp/src/transform/jit/untyped_lto_kernel.cu | 139 ++++++ cpp/src/transform/transform.cu | 340 ++++++++++++- cpp/tests/CMakeLists.txt | 26 +- .../compute_columns_profit_operator.cu | 70 --- cpp/tests/transform/compute_columns_test.cpp | 0 cpp/tests/transform/fragments/invsqrt.cu | 29 ++ cpp/tests/transform/fragments/profit.cu | 49 ++ cpp/tests/transform/fragments/tpcds_q7.cu | 56 +++ cpp/tests/transform/transform_lto_test.cpp | 144 ++++++ todo.md | 10 - 34 files changed, 1807 insertions(+), 889 deletions(-) create mode 100644 cpp/doxygen/developer_guide/JIT.md create mode 100644 cpp/include/cudf/jit/transform_operation.cuh create mode 100644 cpp/include/cudf/jit/type_tags.cuh delete mode 100644 cpp/src/transform/compute_columns_jit.cu delete mode 100644 cpp/src/transform/jit/compute_columns_kernel.cu create mode 100644 cpp/src/transform/jit/lto_kernel.cu create mode 100644 cpp/src/transform/jit/untyped_lto_kernel.cu delete mode 100644 cpp/tests/transform/compute_columns_profit_operator.cu delete mode 100644 cpp/tests/transform/compute_columns_test.cpp create mode 100644 cpp/tests/transform/fragments/invsqrt.cu create mode 100644 cpp/tests/transform/fragments/profit.cu create mode 100644 cpp/tests/transform/fragments/tpcds_q7.cu create mode 100644 cpp/tests/transform/transform_lto_test.cpp delete mode 100644 todo.md diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index eec15b8489b1..e7fb147f8134 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -323,11 +323,156 @@ if(NOT BUILD_SHARED_LIBS) endif() endif() -set(CUDF_CXX_STANDARD 20) -set(CUDF_CUDA_STANDARD ${CUDF_CXX_STANDARD}) +macro(add_fragment) + set(TARGET ${ARGV0}) + set(ONE_VALUE_ARGS FRAGMENT SOURCE KERNEL_ONLY KERNEL_INSTANCE) + set(MULTI_VALUE_ARGS DEFINITIONS ARRAY_IDS ARRAY_VALUES) + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT ARG_FRAGMENT) + message(FATAL_ERROR "add_fragment requires FRAGMENT argument") + endif() + + if(NOT ARG_SOURCE) + message(FATAL_ERROR "add_fragment requires SOURCE argument") + endif() + + set(OBJECT_ID ${TARGET}_${ARG_FRAGMENT}) + add_library(${OBJECT_ID} OBJECT ${ARG_SOURCE}) + target_compile_options( + ${OBJECT_ID} PRIVATE --compress-mode=size --expt-relaxed-constexpr --extended-lambda + ) + + if(DEFINED ARG_KERNEL_ONLY AND ARG_KERNEL_ONLY) + # ensure that the FATBIN symbols only contain the specified kernel + target_compile_options(${OBJECT_ID} PRIVATE -Xnvlink=--kernels-used=cudf_kernel_entry) + endif() + + if(ARG_KERNEL_INSTANCE) + set(SPECIALIZATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/specializations/${ARG_FRAGMENT}") + file( + GENERATE + OUTPUT "${SPECIALIZATION_DIR}/cudf/detail/kernel-instance.hpp" + CONTENT "#pragma once\n#define CUDF_KERNEL_INSTANCE ${ARG_KERNEL_INSTANCE}" + ) + target_include_directories(${OBJECT_ID} PRIVATE ${SPECIALIZATION_DIR}) + endif() + + target_compile_definitions(${OBJECT_ID} PRIVATE CUDF_DISABLE_EXPORTS ${ARG_DEFINITIONS}) + set_target_properties( + ${OBJECT_ID} + PROPERTIES CUDA_SEPARABLE_COMPILATION ON + CUDA_FATBIN_COMPILATION ON + POSITION_INDEPENDENT_CODE ON + INTERPROCEDURAL_OPTIMIZATION ON + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS ON + CXX_VISIBILITY_PRESET hidden + CUDA_STANDARD 20 + CUDA_STANDARD_REQUIRED ON + CUDA_VISIBILITY_PRESET hidden + ) + target_link_libraries(${OBJECT_ID} PRIVATE CCCL::CCCL) + target_include_directories( + ${OBJECT_ID} PRIVATE "$" + "$" + ) -jit_add_include_directory( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/librtcx/libcxx DEST_DIRECTORY + embed_blob( + ${TARGET} FILE $ DEST fragments/${ARG_FRAGMENT}.fatbin ID + ${ARG_FRAGMENT} ARRAY_IDS ${ARG_ARRAY_IDS} ARRAY_VALUES ${ARG_ARRAY_VALUES} + ) +endmacro() + +add_embed(cudf_fragments) + +foreach(TYPE IN ITEMS uint8 uint16 uint32 uint64 decimal32 decimal64 decimal128) + foreach(NULL_AWARE IN ITEMS 0 1) + set(FRAGMENT_NAME unop_lto_kernel) + set(VARIANT_NAME ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__element_${TYPE}) + add_fragment( + cudf_fragments + FRAGMENT + ${VARIANT_NAME} + SOURCE + src/transform/jit/lto_kernel.cu + KERNEL_INSTANCE + "cudf::lto_transform_kernel<${NULL_AWARE}, cudf::jit::type_list>, cudf::jit::type_list>>" + ARRAY_IDS + ${FRAGMENT_NAME}_FILE_INDEX + ${FRAGMENT_NAME}_NULL_AWARE + ${FRAGMENT_NAME}_TYPE + ARRAY_VALUES + ${cudf_fragments_FILE_INDEX} + ${NULL_AWARE} + ${TYPE} + ) + endforeach() +endforeach() + +foreach(NULL_AWARE IN ITEMS 0 1) + foreach(MAX_ELEMENT_SIZE IN ITEMS 4 8) + foreach(MAX_ELEMENTS IN ITEMS 2 4) + set(FRAGMENT_NAME untyped_lto_transform_kernel_stack) + set(VARIANT_NAME + ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__max_element_size_${MAX_ELEMENT_SIZE}__max_elements_${MAX_ELEMENTS} + ) + add_fragment( + cudf_fragments + FRAGMENT + ${VARIANT_NAME} + SOURCE + src/transform/jit/untyped_lto_kernel.cu + KERNEL_INSTANCE + "cudf::untyped_lto_transform_kernel_stack<${NULL_AWARE}, ${MAX_ELEMENT_SIZE}, ${MAX_ELEMENTS}>" + ARRAY_IDS + ${FRAGMENT_NAME}_FILE_INDEX + ${FRAGMENT_NAME}_NULL_AWARE + ${FRAGMENT_NAME}_MAX_ELEMENT_SIZE + ${FRAGMENT_NAME}_MAX_ELEMENTS + ARRAY_VALUES + ${cudf_fragments_FILE_INDEX} + ${NULL_AWARE} + ${MAX_ELEMENT_SIZE} + ${MAX_ELEMENTS} + ) + endforeach() + endforeach() +endforeach() + +foreach(MAX_ELEMENT_SIZE IN ITEMS 4 8 16 32) + foreach(NULL_AWARE IN ITEMS 0 1) + set(FRAGMENT_NAME untyped_lto_transform_kernel_shmem) + set(VARIANT_NAME + ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__max_element_size_${MAX_ELEMENT_SIZE} + ) + add_fragment( + cudf_fragments + FRAGMENT + ${VARIANT_NAME} + SOURCE + src/transform/jit/untyped_lto_kernel.cu + KERNEL_INSTANCE + "cudf::untyped_lto_transform_kernel_shmem<${NULL_AWARE}, ${MAX_ELEMENT_SIZE}>" + ARRAY_IDS + ${FRAGMENT_NAME}_FILE_INDEX + ${FRAGMENT_NAME}_NULL_AWARE + ${FRAGMENT_NAME}_MAX_ELEMENT_SIZE + ARRAY_VALUES + ${cudf_fragments_FILE_INDEX} + ${NULL_AWARE} + ${MAX_ELEMENT_SIZE} + ) + endforeach() +endforeach() + +embed(cudf_fragments COMPRESSION none) + +add_embed(cudf_jit_embed) + +embed_includes( + cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/librtcx/libcxx DEST_DIRECTORY librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx ) @@ -877,7 +1022,6 @@ add_library( src/text/wordpiece_tokenize.cu src/transform/bools_to_mask.cu src/transform/compute_column.cu - src/transform/compute_columns_jit.cu src/transform/compute_column_kernel_complex.cu src/transform/compute_column_kernel_null_complex.cu src/transform/compute_column_kernel_null_primitive.cu @@ -909,13 +1053,10 @@ add_library( src/utilities/type_dispatcher.cpp librtcx/rtcx.cpp ${cudf_jit_embed_SOURCE_DIR}/cudf_jit_embed.s + ${cudf_fragments_SOURCE_DIR}/cudf_fragments.s ) -add_dependencies(cudf cudf_jit_embed) - -foreach(incbin_dir IN LISTS cudf_jit_embed_INCLUDE_DIRS) - target_compile_options(cudf PRIVATE $<$:-Wa,-I${incbin_dir}>) -endforeach() +add_dependencies(cudf cudf_jit_embed cudf_fragments) set_source_files_properties( src/binaryop/binaryop.cpp @@ -997,6 +1138,7 @@ target_include_directories( "$" "$" "$" + "$" "$" INTERFACE "$" ) diff --git a/cpp/doxygen/developer_guide/JIT.md b/cpp/doxygen/developer_guide/JIT.md new file mode 100644 index 000000000000..b35afe1c7a0a --- /dev/null +++ b/cpp/doxygen/developer_guide/JIT.md @@ -0,0 +1,89 @@ + + + +# Terminologies + +FATBIN +CUBIN +LTO-IR +PTX +OBJ +SASS + +# Normal CUDA Kernel + +# What is LTO? + +# LTO vs AOT vs Source + +# Example + +# Environment Variables + + +# Benchmarking +NVTX ranges +Statistics + + +# Why is My JIT Slow + + +# Jitify + +## How JITIFY Works + +## JITIFY's Limitations + +### Binary Bloat + +### String Source Embedding + +### + + +# PCH & The C++/CUDA AST + +# JIT-LTO +# Comparison to CPU Linking + + +# Adding a CUDF JIT Kernel + +# Adding a CUDF LTO Kernel + + +# LIBRTCX + + +# Kernel Combinatorics in CUDF + +# Progressive Reduction + +# Pre-warming techniques + + + +# Why AST is Slow + + + +# How can NVRTC/C++/NVJITLink Help Us? + + + +# Perf Numbers + +# CUDA +# CUDA + PCH +# LTO-Linked Kernel +# + +# LTO & Optimization Barriers + +# Kernel Variant Management with RTCX + +# What makes this challenging +# Knobs for Each Kernel + +## Adding an LTO-Fragment diff --git a/cpp/include/cudf/jit/transform_operation.cuh b/cpp/include/cudf/jit/transform_operation.cuh new file mode 100644 index 000000000000..92f43cfd10ab --- /dev/null +++ b/cpp/include/cudf/jit/transform_operation.cuh @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +/// @brief The compute operation to perform on each element. This is a generic entry point for n-ary +/// transform operations. +/// @param user_data Pointer to user data passed to the kernel +/// @param element_index The index of the element to compute +/// @param inputs Pointer to the input elements for this operation; the caller guarantees the memory +/// layout and type of these elements based on the input column device views and input strides +/// @param input_stride The stride (in bytes) between consecutive input elements for a given input +/// column +/// @param outputs Pointer to the output elements for this operation; the caller guarantees the +/// memory layout and type of these elements based on the output column device views +/// @param output_stride The stride (in bytes) between consecutive output elements for a given +/// output column +/// @return An integer status code +extern "C" __device__ int cudf_transform_operation(void* __restrict__ user_data, + long int element_index, + void const* __restrict__ inputs, + int input_stride, + void* __restrict__ outputs, + int output_stride); diff --git a/cpp/include/cudf/jit/type_tags.cuh b/cpp/include/cudf/jit/type_tags.cuh new file mode 100644 index 000000000000..15a4e7caef9f --- /dev/null +++ b/cpp/include/cudf/jit/type_tags.cuh @@ -0,0 +1,56 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include +#include +#include + +namespace CUDF_EXPORT cudf { +namespace tags { + +template +inline constexpr char const* tag_of = "void"; + +#define CUDF_TYPE_TAG(type, tag) \ + using tag = type; \ + template <> \ + inline constexpr char const* tag_of = #tag; + +CUDF_TYPE_TAG(bool, bool8); +CUDF_TYPE_TAG(int8_t, int8); +CUDF_TYPE_TAG(int16_t, int16); +CUDF_TYPE_TAG(int32_t, int32); +CUDF_TYPE_TAG(int64_t, int64); +CUDF_TYPE_TAG(__int128_t, int128); +CUDF_TYPE_TAG(uint8_t, uint8); +CUDF_TYPE_TAG(uint16_t, uint16); +CUDF_TYPE_TAG(uint32_t, uint32); +CUDF_TYPE_TAG(uint64_t, uint64); +CUDF_TYPE_TAG(__uint128_t, uint128); +CUDF_TYPE_TAG(float, float32); +CUDF_TYPE_TAG(double, float64); +CUDF_TYPE_TAG(cudf::string_view, string_view); +CUDF_TYPE_TAG(numeric::decimal32, decimal32); +CUDF_TYPE_TAG(numeric::decimal64, decimal64); +CUDF_TYPE_TAG(numeric::decimal128, decimal128); +CUDF_TYPE_TAG(cudf::duration_D, duration_D); +CUDF_TYPE_TAG(cudf::duration_h, duration_h); +CUDF_TYPE_TAG(cudf::duration_m, duration_m); +CUDF_TYPE_TAG(cudf::duration_s, duration_s); +CUDF_TYPE_TAG(cudf::duration_ms, duration_ms); +CUDF_TYPE_TAG(cudf::duration_us, duration_us); +CUDF_TYPE_TAG(cudf::duration_ns, duration_ns); +CUDF_TYPE_TAG(cudf::timestamp_D, timestamp_D); +CUDF_TYPE_TAG(cudf::timestamp_h, timestamp_h); +CUDF_TYPE_TAG(cudf::timestamp_m, timestamp_m); +CUDF_TYPE_TAG(cudf::timestamp_s, timestamp_s); +CUDF_TYPE_TAG(cudf::timestamp_ms, timestamp_ms); +CUDF_TYPE_TAG(cudf::timestamp_us, timestamp_us); +CUDF_TYPE_TAG(cudf::timestamp_ns, timestamp_ns); + +} // namespace tags +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 2146261fc570..f8007deacb37 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -179,6 +179,14 @@ std::unique_ptr
multi_transform( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +/** + * @brief The type of LTO Binary + */ +enum class lto_binary_type : uint8_t { + LTO_IR, //< LTO-IR binary + FATBIN //< FATBIN binary +}; + /** * @brief Creates a new table by applying a transform function against every * element of the input columns. @@ -199,13 +207,12 @@ std::unique_ptr
multi_transform( * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from * the input and pre-allocated output columns. * + * @param inputs Immutable views of the inputs to transform (columns and scalar columns) * @param udf The LTO-IR string of the transform function to apply - * @param is_null_aware Signifies the UDF will receive row inputs as optional values + * @param binary_type The type of the LTO binary provided in `udf` * @param user_data User-defined device data to pass to the UDF. - * @param inputs Immutable views of the inputs to transform (columns and scalar columns) + * @param is_null_aware Signifies the UDF will receive row inputs as optional values * @param outputs Specification of the output columns to be created - * @param string_offsets For string output columns, the offsets can be pre-allocated and passed in - * to prevent overhead of compacting string views into run-end strings column. * @param row_size The row size of the transform operation. If not provided, it is inferred from the * input columns. * @param stream CUDA stream used for device memory operations and kernel launches @@ -214,10 +221,11 @@ std::unique_ptr
multi_transform( * function to every element of the input according to the output specifications * */ -std::unique_ptr
compute_columns_jit( +std::unique_ptr
transform_lto( std::span inputs, std::span udf, - std::span output_types, + lto_binary_type binary_type, + std::span outputs, void* user_data, null_aware is_null_aware = null_aware::NO, std::optional row_size = std::nullopt, diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index 79e53db4efb3..a55c01cfd107 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -11,6 +11,27 @@ if(NOT TARGET zstd) ) endif() +function(add_embed) + set(TARGET ${ARGV0}) + set(OPTIONS "") + set(ONE_VALUE_ARGS) + set(MULTI_VALUE_ARGS) + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET argument is required") + endif() + + set(${TARGET}_FILE_INDEX + 0 + PARENT_SCOPE + ) + set(${TARGET}_INITIALIZED + TRUE + PARENT_SCOPE + ) +endfunction() + # This function registers a directory of include files to be embedded for JIT compilation. function(embed_includes) set(TARGET ${ARGV0}) @@ -29,6 +50,10 @@ function(embed_includes) message(FATAL_ERROR "TARGET argument is required") endif() + if(NOT ${TARGET}_INITIALIZED) + message(FATAL_ERROR "Target '${TARGET}' has not been initialized with add_embed()") + endif() + if(NOT ARG_COPY_DIRECTORY) message(FATAL_ERROR "COPY_DIRECTORY argument is required") endif() @@ -64,17 +89,30 @@ function(embed_includes) # Set scope variables to accumulate results + set(SOURCE_FILE_IDS ${${TARGET}__embed__source_file_ids}) set(SOURCE_FILES ${${TARGET}__embed__source_files}) set(SOURCE_FILE_DESTS ${${TARGET}__embed__source_file_dests}) set(INCLUDE_DIRECTORIES ${${TARGET}__embed__include_directories}) foreach(SOURCE_FILE IN LISTS ARG_FILES) + list(LENGTH SOURCE_FILE_IDS SOURCE_FILE_IDS_LENGTH) + list(APPEND SOURCE_FILE_IDS "include_${SOURCE_FILE_IDS_LENGTH}") list(APPEND SOURCE_FILES "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") list(APPEND SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}") endforeach() list(APPEND INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES}) + list(LENGTH SOURCE_FILE_IDS SOURCE_FILE_IDS_LENGTH) + + set(${TARGET}_FILE_INDEX + ${SOURCE_FILE_IDS_LENGTH} + PARENT_SCOPE + ) + set(${TARGET}__embed__source_file_ids + ${SOURCE_FILE_IDS} + PARENT_SCOPE + ) set(${TARGET}__embed__source_files ${SOURCE_FILES} PARENT_SCOPE @@ -94,14 +132,22 @@ endfunction() function(embed_blob) set(TARGET ${ARGV0}) set(OPTIONS) - set(ONE_VALUE_ARGS FILE DEST) - set(MULTI_VALUE_ARGS "") + set(ONE_VALUE_ARGS ID FILE DEST) + set(MULTI_VALUE_ARGS ARRAY_IDS ARRAY_VALUES) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) if(NOT DEFINED TARGET) message(FATAL_ERROR "TARGET argument is required") endif() + if(NOT ${TARGET}_INITIALIZED) + message(FATAL_ERROR "Target '${TARGET}' has not been initialized with add_embed()") + endif() + + if(NOT ARG_ID) + message(FATAL_ERROR "ID argument is required") + endif() + if(NOT ARG_FILE) message(FATAL_ERROR "FILE argument is required") endif() @@ -110,9 +156,28 @@ function(embed_blob) message(FATAL_ERROR "DEST argument is required") endif() + set(SOURCE_FILE_IDS ${${TARGET}__embed__source_file_ids}) set(SOURCE_FILES ${${TARGET}__embed__source_files}) set(SOURCE_FILE_DESTS ${${TARGET}__embed__source_file_dests}) set(TARGET_DEPS ${${TARGET}__embed__target_deps}) + set(ARRAY_IDS ${${TARGET}__embed__array_ids}) + set(ARRAY_VALUES ${${TARGET}__embed__array_values}) + + if(ARG_ARRAY_IDS) + if(NOT ARG_ARRAY_VALUES) + message(FATAL_ERROR "ARRAY_VALUES argument is required when ARRAY_IDS is provided") + endif() + + list(LENGTH ARG_ARRAY_IDS ARG_ARRAY_IDS_LENGTH) + list(LENGTH ARG_ARRAY_VALUES ARG_ARRAY_VALUES_LENGTH) + + if(NOT ARG_ARRAY_IDS_LENGTH EQUAL ARG_ARRAY_VALUES_LENGTH) + message(FATAL_ERROR "ARRAY_IDS and ARRAY_VALUES must have the same length") + endif() + + list(APPEND ARRAY_IDS ${ARG_ARRAY_IDS}) + list(APPEND ARRAY_VALUES ${ARG_ARRAY_VALUES}) + endif() if(ARG_FILE MATCHES "\\$]+)>") # If the file is a generator expression for target objects add as dependency @@ -122,9 +187,20 @@ function(embed_blob) message(FATAL_ERROR "Source file '${ARG_FILE}' does not exist") endif() endif() + list(APPEND SOURCE_FILE_IDS ${ARG_ID}) list(APPEND SOURCE_FILES ${ARG_FILE}) list(APPEND SOURCE_FILE_DESTS ${ARG_DEST}) + list(LENGTH SOURCE_FILE_IDS SOURCE_FILE_IDS_LENGTH) + + set(${TARGET}_FILE_INDEX + ${SOURCE_FILE_IDS_LENGTH} + PARENT_SCOPE + ) + set(${TARGET}__embed__source_file_ids + ${SOURCE_FILE_IDS} + PARENT_SCOPE + ) set(${TARGET}__embed__source_files ${SOURCE_FILES} PARENT_SCOPE @@ -137,6 +213,14 @@ function(embed_blob) ${TARGET_DEPS} PARENT_SCOPE ) + set(${TARGET}__embed__array_ids + ${ARRAY_IDS} + PARENT_SCOPE + ) + set(${TARGET}__embed__array_values + ${ARRAY_VALUES} + PARENT_SCOPE + ) endfunction() @@ -153,6 +237,10 @@ function(embed) message(FATAL_ERROR "TARGET argument is required") endif() + if(NOT ${TARGET}_INITIALIZED) + message(FATAL_ERROR "Target '${TARGET}' has not been initialized with add_embed()") + endif() + if(NOT DEFINED ARG_COMPRESSION) message(FATAL_ERROR "COMPRESSION argument is required") endif() @@ -171,6 +259,9 @@ function(embed) set(EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed.cpp") set(EMBED_SCRIPT__ID "${TARGET}") + set(EMBED_SCRIPT__ARRAY_IDS "${${TARGET}__embed__array_ids}") + set(EMBED_SCRIPT__ARRAY_VALUES "${${TARGET}__embed__array_values}") + set(EMBED_SCRIPT__FILE_IDS "${${TARGET}__embed__source_file_ids}") set(EMBED_SCRIPT__FILE_PATHS "${${TARGET}__embed__source_files}") set(EMBED_SCRIPT__FILE_DESTS "${${TARGET}__embed__source_file_dests}") set(EMBED_SCRIPT__INCLUDE_DIRS "${${TARGET}__embed__include_directories}") diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index 968307283587..a29b8eaca805 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -32,6 +34,14 @@ struct size_range { size_t size = 0; }; +struct embed_output { + std::string cxx_header; + std::string asm_source; + std::vector bin_file_data; +}; + +enum class value_type : int8_t { INT, STRING }; + std::pair, std::vector> merge_bytes_with_null_terminators( std::span const> bytes_lists) { @@ -47,13 +57,6 @@ std::pair, std::vector> merge_bytes_with_null_t return {std::move(merged), std::move(ranges)}; } -struct embed_output { - std::string cxx_header; - std::string cxx_source; - std::string asm_source; - std::vector bin_file_data; -}; - std::vector load_file_bytes(std::string_view file_path) { std::string path_str(file_path); @@ -121,7 +124,94 @@ std::string join_formatted(Container& items, std::string_view delimiter, Formatt return result.str(); } +std::string generate_arrays(std::span array_ids, + std::span array_values) +{ + auto get_type = [](std::string_view value) -> value_type { + return std::isdigit(value[0]) ? value_type::INT : value_type::STRING; + }; + + using strings_t = std::vector; + using ints_t = std::vector; + using values_t = std::variant; + + struct array_entry { + value_type type = value_type::INT; + values_t values; + }; + + std::map arrays; + + for (size_t i = 0; i < array_ids.size(); ++i) { + auto id = array_ids[i]; + auto value = array_values[i]; + if (auto array_it = arrays.find(id); array_it == arrays.end()) { + switch (get_type(value)) { + { + case value_type::INT: + arrays.emplace(id, array_entry(value_type::INT, std::vector{})); + } + break; + case value_type::STRING: { + arrays.emplace(id, array_entry(value_type::STRING, std::vector{})); + } break; + default: throw std::logic_error("Unexpected constant type"); + } + } + + auto& array = arrays[id]; + + switch (array.type) { + case value_type::INT: { + std::int64_t int_value; + if (std::from_chars(value.data(), value.data() + value.size(), int_value).ec != + std::errc()) { + throw std::invalid_argument(std::format("Invalid integer constant value: {}", value)); + } + std::get>(array.values).push_back(int_value); + } break; + + case value_type::STRING: { + std::get>(array.values).push_back(value); + } break; + + default: break; + } + } + + std::string result; + + for (auto& [id, array] : arrays) { + switch (array.type) { + case value_type::INT: { + auto& values = std::get>(array.values); + result += std::format( + "constexpr std::int64_t {}[{}] = {{ {} }};\n\n", + id, + values.size(), + join_formatted(values, ", ", [](std::int64_t v) { return std::to_string(v); })); + } break; + + case value_type::STRING: { + auto& values = std::get>(array.values); + result += std::format( + "constexpr char const* {}[{}] = {{ {} }};\n\n", + id, + values.size(), + join_formatted(values, ", ", [](auto s) { return std::format("\"{}\"", s); })); + } break; + + default: break; + } + } + + return result; +} + embed_output generate_cxx_source_files_data(std::string_view id, + std::span array_ids, + std::span array_values, + std::span file_ids, std::span file_paths, std::span file_dsts, std::span include_dirs, @@ -133,7 +223,7 @@ embed_output generate_cxx_source_files_data(std::string_view id, file_bytes.emplace_back(load_file_bytes(path)); } - auto [uncompressed_files_bytes, files_ranges] = merge_bytes_with_null_terminators(file_bytes); + auto [uncompressed_files_bytes, file_ranges] = merge_bytes_with_null_terminators(file_bytes); auto compress = compression != "none"; std::vector compressed_files_bytes = @@ -171,13 +261,20 @@ embed_output generate_cxx_source_files_data(std::string_view id, auto include_dirs_list = join_formatted(include_dirs, ",\n", [](auto s) { return std::format("\"{}\"", s); }); - auto dests_list = + auto file_dests_list = join_formatted(file_dsts, ",\n", [](auto s) { return std::format("\"{}\"", s); }); - auto ranges_list = join_formatted( - files_ranges, ",\n", [](auto r) { return std::format("{{{}, {}}}", r.offset, r.size); }); - + auto file_ids_list = + join_formatted(file_ids, ",\n", [](auto s) { return std::format("\"{}\"", s); }); + std::vector file_indices(file_ids.size()); + std::iota(file_indices.begin(), file_indices.end(), 0ULL); + auto file_indices_list = join_formatted(file_indices, "\n", [&](auto i) { + return std::format("constexpr std::size_t {} = {}ULL;", file_ids[i], i); + }); + auto file_ranges_list = join_formatted( + file_ranges, ",\n", [](auto r) { return std::format("{{{}, {}}}", r.offset, r.size); }); auto hash_list = join_formatted( hash, ", ", [](uint8_t byte) { return std::format("0x{:02x}", static_cast(byte)); }); + auto arrays_list = generate_arrays(array_ids, array_values); auto cxx_header = std::format( R"***( @@ -189,74 +286,77 @@ embed_output generate_cxx_source_files_data(std::string_view id, #include #include -namespace rtcx_embed {{ +namespace {} {{ + -struct range {{ - std::size_t offset = 0; - std::size_t size = 0; +constexpr char const * include_directories[{}] = +{{ +{} }}; -constexpr char const * {}_include_directories[{}] = +{} + +constexpr char const * file_ids[{}] = {{ {} }}; -constexpr char const * {}_file_destinations[{}] = +constexpr char const * file_destinations[{}] = {{ {} }}; -constexpr range {}_file_ranges[{}] = +constexpr std::size_t file_ranges[{}][2] = {{ {} }}; -constexpr std::size_t {}_files_uncompressed_size = {}; +constexpr std::size_t files_uncompressed_size = {}; -constexpr char const * {}_files_compression = "{}"; +constexpr char const * files_compression = "{}"; -extern "C" std::uint8_t const rtcx_embed_{}_files_begin[]; +extern "C" std::uint8_t const {}_files_begin[]; -static std::span const {}_files = +static std::span const files = {{ -rtcx_embed_{}_files_begin, +{}_files_begin, {}L }}; -constexpr std::uint8_t {}_hash[{}] = +constexpr std::uint8_t hash[{}] = {{ {} }}; +{} + }} )***", id, id, include_dirs.size(), include_dirs_list, - id, + file_indices_list, + file_ids.size(), + file_ids_list, file_dsts.size(), - dests_list, - id, - files_ranges.size(), - ranges_list, - id, + file_dests_list, + file_ranges.size(), + file_ranges_list, uncompressed_files_bytes.size(), - id, compression, id, id, - id, binary_size, - id, hash.size(), - hash_list); + hash_list, + arrays_list); auto asm_source = std::format( R"***( .section .rodata -.global rtcx_embed_{}_files_begin -rtcx_embed_{}_files_begin: +.global {}_files_begin +{}_files_begin: .incbin "{}.bin" .section .note.GNU-stack,"",@progbits @@ -267,20 +367,22 @@ rtcx_embed_{}_files_begin: return embed_output{ .cxx_header = cxx_header, - .cxx_source = "", .asm_source = asm_source, .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes}; } void generate_embed(std::string_view id, + std::span array_ids, + std::span array_values, + std::span file_ids, std::span file_paths, std::span file_dsts, std::span include_dirs, std::string_view compression, std::string_view output_directory) { - auto output = - generate_cxx_source_files_data(id, file_paths, file_dsts, include_dirs, compression); + auto output = generate_cxx_source_files_data( + id, array_ids, array_values, file_ids, file_paths, file_dsts, include_dirs, compression); std::filesystem::create_directories(std::filesystem::path(output_directory)); @@ -297,6 +399,7 @@ void generate_embed(std::string_view id, std::vector split_string(std::string_view str, char delimiter) { + if (str.empty()) { return {}; } std::vector tokens; std::size_t start = 0; diff --git a/cpp/librtcx/embed.in.cpp b/cpp/librtcx/embed.in.cpp index 0a391e202508..c53a12d1d0f4 100644 --- a/cpp/librtcx/embed.in.cpp +++ b/cpp/librtcx/embed.in.cpp @@ -8,13 +8,23 @@ int main() { std::string_view id = "@EMBED_SCRIPT__ID@"; + auto array_ids = rtcx_embed::split_string("@EMBED_SCRIPT__ARRAY_IDS@", ';'); + auto array_values = rtcx_embed::split_string("@EMBED_SCRIPT__ARRAY_VALUES@", ';'); + auto file_ids = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_IDS@", ';'); auto file_paths = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_PATHS@", ';'); auto file_dests = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_DESTS@", ';'); auto include_directories = rtcx_embed::split_string("@EMBED_SCRIPT__INCLUDE_DIRS@", ';'); std::string_view compression = "@EMBED_SCRIPT__COMPRESSION@"; std::string_view output_dir = "@EMBED_SCRIPT__OUTPUT_DIR@"; - rtcx_embed::generate_embed( - id, file_paths, file_dests, include_directories, compression, output_dir); + rtcx_embed::generate_embed(id, + array_ids, + array_values, + file_ids, + file_paths, + file_dests, + include_directories, + compression, + output_dir); return EXIT_SUCCESS; } diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 50be25bb6111..d7f5d3386c90 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -99,9 +100,16 @@ } \ } while (0) +#define RTCX_FUNC_RANGE() \ + ::nvtx3::scoped_range_in<::rtcx::nvtx_domain> rtcx_func_range__ { __func__ } + namespace rtcx { namespace { +struct nvtx_domain { + static constexpr char const* name = "rtcx"; +}; + enum class object_type : std::uint8_t { LIBRARY, BLOB }; std::string_view object_tag(object_type type) @@ -328,13 +336,13 @@ struct LibNVRTC { LibNVRTC(LibNVRTC&&) = delete; LibNVRTC& operator=(LibNVRTC const&) = delete; LibNVRTC& operator=(LibNVRTC&&) = delete; - ~LibNVRTC() { dlclose(_handle); } + ~LibNVRTC() { ::dlclose(_handle); } static void* _load() { auto expected_major_version = major_version(CUDA_VERSION); std::int32_t cuda_version; - RTCX_CHECK_CUDART(cudaRuntimeGetVersion(&cuda_version)); + RTCX_CHECK_CUDART(::cudaRuntimeGetVersion(&cuda_version)); std::int32_t major = major_version(cuda_version); RTCX_EXPECTS(expected_major_version == major, std::format("LibNVRTC Compatibility Error: CUDA major version mismatch. Expected " @@ -378,7 +386,7 @@ struct LibNVJitLink { { auto expected_major_version = major_version(CUDA_VERSION); std::int32_t cuda_version; - RTCX_CHECK_CUDART(cudaRuntimeGetVersion(&cuda_version)); + RTCX_CHECK_CUDART(::cudaRuntimeGetVersion(&cuda_version)); std::int32_t major = major_version(cuda_version); RTCX_EXPECTS( expected_major_version == major, @@ -415,6 +423,8 @@ static std::optional teardown_libraries_flag{std::in_place}; void initialize() { + RTCX_FUNC_RANGE(); + std::call_once(*init_libraries_flag, [] { cu.emplace(LibCuda::_load()); RTCX_EXPECTS( @@ -426,6 +436,8 @@ void initialize() void teardown() { + RTCX_FUNC_RANGE(); + std::call_once(*teardown_libraries_flag, [] { nvjitlink.reset(); nvrtc.reset(); @@ -455,7 +467,7 @@ void log_nvrtc_result(compile_params const& params, nvrtcProgram program, nvrtcResult compile_result) { - if (program == nullptr || compile_result == NVRTC_SUCCESS) { return; } + if (program == nullptr) { return; } std::size_t log_size; if (auto errc = nvrtc->GetProgramLogSize(program, &log_size); errc != NVRTC_SUCCESS) { @@ -465,22 +477,22 @@ void log_nvrtc_result(compile_params const& params, std::runtime_error); } - if (log_size <= 1) { return; } - std::vector log; - log.resize(log_size); - if (auto errc = nvrtc->GetProgramLog(program, log.data()); errc != NVRTC_SUCCESS) { - RTCX_FAIL(std::format("Failed to get NVRTC program log with error ({}): {}", - static_cast(errc), - nvrtc->GetErrorString(errc)), - std::runtime_error); + if (log_size > 1) { + log.resize(log_size); + if (auto errc = nvrtc->GetProgramLog(program, log.data()); errc != NVRTC_SUCCESS) { + RTCX_FAIL(std::format("Failed to get NVRTC program log with error ({}): {}", + static_cast(errc), + nvrtc->GetErrorString(errc)), + std::runtime_error); + } } log.resize(log_size == 0 ? 0 : (log_size - 1)); - auto status_str = (compile_result == NVRTC_SUCCESS && !log.empty()) ? "completed with warning" - : "failed with error"; + auto status_str = + (compile_result == NVRTC_SUCCESS && !log.empty()) ? "completed with" : "failed with"; std::string headers_str; for (auto& header : params.header_include_names) { @@ -492,6 +504,8 @@ void log_nvrtc_result(compile_params const& params, options_str = std::format("{}\t{}\n", options_str, option); } + if (log.empty()) { return; } + auto msg = std::format( "NVRTC Compilation for `{}` {} ({}): {}.\nHeaders:\n{}\n\nOptions:\n{}\n\nLog:\n\t{}", params.name == nullptr ? "" : params.name, @@ -513,7 +527,7 @@ void log_nvJitLink_result(link_params const& params, nvJitLinkHandle handle, nvJitLinkResult link_result) { - if (handle == nullptr || link_result == NVJITLINK_SUCCESS) { return; } + if (handle == nullptr) { return; } std::size_t info_log_size; if (auto errc = nvjitlink->GetInfoLogSize(handle, &info_log_size); errc != NVJITLINK_SUCCESS) { @@ -533,6 +547,7 @@ void log_nvJitLink_result(link_params const& params, std::runtime_error); } } + info_log.resize(info_log_size == 0 ? 0 : (info_log_size - 1)); std::size_t error_log_size; @@ -554,6 +569,7 @@ void log_nvJitLink_result(link_params const& params, std::runtime_error); } } + error_log.resize(error_log_size == 0 ? 0 : (error_log_size - 1)); if (info_log.empty() && error_log.empty()) { return; } @@ -572,10 +588,10 @@ void log_nvJitLink_result(link_params const& params, link_options_str = std::format("{}\t{}\n", link_options_str, option); } - auto status_str = error_log.empty() ? "completed with warnings" : "failed with errors"; + auto status_str = link_result == NVJITLINK_SUCCESS ? "completed with" : "failed with"; auto msg = std::format( - "(nvJitLink) Linking for `{}` ({}) {}, error code ({}): {}.\nFragments: \n{}\n" + "(nvJitLink) Linking for `{}` ({}) {} error code ({}): {}.\nFragments: \n{}\n" "Link Options: \n{}\n\nInfo Log:\n\t{}\n\nError Log:\n\t{}\n\n", params.name == nullptr ? "" : params.name, binary_type_string(params.output_type), @@ -587,10 +603,16 @@ void log_nvJitLink_result(link_params const& params, std::string_view{info_log.data(), info_log.size()}, std::string_view{error_log.data(), error_log.size()}); - if (!error_log.empty()) { - log_error(msg); + bool needs_info_log = + std::find_if( + params.link_options.begin(), params.link_options.end(), [](std::string_view option) { + return option == "--verbose" || option == "-time"; + }) != params.link_options.end(); + + if (link_result == NVJITLINK_SUCCESS) { + if (needs_info_log) { log_warning(msg); } } else { - log_warning(msg); + log_error(msg); } } @@ -598,6 +620,8 @@ void log_nvJitLink_result(link_params const& params, byte_buffer compile(compile_params const& params) { + RTCX_FUNC_RANGE(); + RTCX_EXPECTS(params.name != nullptr, "Fragment name must not be null", std::logic_error); RTCX_EXPECTS(params.source != nullptr, "Fragment source must not be null", std::logic_error); @@ -671,6 +695,8 @@ void kernel_ref::launch(cuda_dim3 grid_dim, CUstream stream, void** kernel_params) const { + RTCX_FUNC_RANGE(); + RTCX_EXPECTS(grid_dim.is_valid(), "Grid dimensions must be greater than zero", std::logic_error); RTCX_EXPECTS( block_dim.is_valid(), "Block dimensions must be greater than zero", std::logic_error); @@ -698,6 +724,8 @@ void kernel_ref::launch_cooperative(cuda_dim3 grid_dim, CUstream stream, void** kernel_params) const { + RTCX_FUNC_RANGE(); + RTCX_EXPECTS(grid_dim.is_valid(), "Grid dimensions must be greater than zero", std::logic_error); RTCX_EXPECTS( block_dim.is_valid(), "Block dimensions must be greater than zero", std::logic_error); @@ -723,6 +751,8 @@ library_t::~library_t() library load_library(std::span binary) { + RTCX_FUNC_RANGE(); + CUlibrary handle; RTCX_CHECK_CUDA( @@ -741,6 +771,8 @@ library load_library(std::span binary) library load_library_from_file(char const* path) { + RTCX_FUNC_RANGE(); + RTCX_EXPECTS(path != nullptr, "Library path must not be null", std::logic_error); CUlibrary handle; @@ -760,6 +792,8 @@ library load_library_from_file(char const* path) byte_buffer link_library(link_params const& params) { + RTCX_FUNC_RANGE(); + RTCX_EXPECTS(params.name != nullptr, "Link output name must not be null", std::logic_error); RTCX_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, "Only CUBIN and PTX output types are supported for linking modules", @@ -824,6 +858,8 @@ byte_buffer link_library(link_params const& params) kernel_ref library_t::get_kernel(char const* name) const { + RTCX_FUNC_RANGE(); + CUkernel kernel; RTCX_CHECK_CUDA(cu->LibraryGetKernel(&kernel, handle_, name)); return kernel_ref{kernel}; @@ -913,6 +949,8 @@ std::optional get_disk_blob(std::string const& cache_dir, object_type type std::optional get_disk_library(std::string const& cache_dir, sha256 const& sha) { + RTCX_FUNC_RANGE(); + auto hex = sha.to_hex_string(); auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(object_type::LIBRARY)); @@ -953,6 +991,8 @@ void cache_blob_to_disk(std::string const& cache_dir, sha256 const& sha, std::span binary) { + RTCX_FUNC_RANGE(); + auto tmp_path = std::format("{}/rtcx-bin-XXXXXX", tmp_dir); (void)tmp_path.c_str(); // to ensure null-termination for mkstemp @@ -994,6 +1034,8 @@ void cache_blob_to_disk(std::string const& cache_dir, std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compile_func compile) { + RTCX_FUNC_RANGE(); + std::atomic_ref tick{tick_}; auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); @@ -1053,6 +1095,8 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil std::shared_future cache_t::get_or_add_library(sha256 const& sha, library_compile_func compile) { + RTCX_FUNC_RANGE(); + std::atomic_ref tick{tick_}; auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); @@ -1153,6 +1197,8 @@ std::size_t cache_t::get_library_count() void cache_t::clear_memory_store() { + RTCX_FUNC_RANGE(); + std::lock_guard guard{lock_}; blobs_cache_.entries_.clear(); @@ -1161,6 +1207,8 @@ void cache_t::clear_memory_store() void cache_t::clear_disk_store() { + RTCX_FUNC_RANGE(); + auto entries = get_disk_entries(cache_dir_); for (auto const& path : entries) { @@ -1174,6 +1222,8 @@ void cache_t::clear_disk_store() void cache_t::preload_from_disk() { + RTCX_FUNC_RANGE(); + auto entries = get_disk_entries(cache_dir_); auto load_count = @@ -1245,6 +1295,8 @@ rtcx::byte_buffer decompress_blob(std::span compressed_binar std::size_t uncompressed_size, std::string_view compression) { + RTCX_FUNC_RANGE(); + RTCX_EXPECTS(compression == "none" || compression == "zstd", std::format("Unsupported compression type specified: {}", compression), std::runtime_error); diff --git a/cpp/src/binaryop/jit/kernel.cu b/cpp/src/binaryop/jit/kernel.cu index 85965c1187e9..3d8f1d7cb5bd 100644 --- a/cpp/src/binaryop/jit/kernel.cu +++ b/cpp/src/binaryop/jit/kernel.cu @@ -77,10 +77,10 @@ CUDF_KERNEL void binaryop_kernel(cudf::size_type size, } // namespace binops } // namespace cudf -extern "C" __global__ void cudf_kernel(cudf::size_type size, - void* out_data, - void* lhs_data, - void* rhs_data) +extern "C" __global__ void cudf_kernel_entry(cudf::size_type size, + void* out_data, + void* lhs_data, + void* rhs_data) { CUDF_KERNEL_INSTANCE(size, out_data, lhs_data, rhs_data); } diff --git a/cpp/src/jit/element.cuh b/cpp/src/jit/element.cuh index 361b3f89cc26..c1adb4379f9a 100644 --- a/cpp/src/jit/element.cuh +++ b/cpp/src/jit/element.cuh @@ -9,17 +9,12 @@ #include #include -#include #include -namespace cudf { - -template -inline constexpr bool storage_compatible = - sizeof(ElementType) <= sizeof(Storage) && alignof(ElementType) <= alignof(Storage); +namespace CUDF_EXPORT cudf { template -__device__ void load_element(column_device_view const* column, +__device__ void load_element(column_device_view_core const* column, size_type element_index, element_storage_t* storage) { @@ -38,37 +33,37 @@ __device__ void load_element(column_device_view const* column, }; switch (column->type().id()) { - case type_id::INT8: op.template operator()(); break; - case type_id::INT16: op.template operator()(); break; - case type_id::INT32: op.template operator()(); break; - case type_id::INT64: op.template operator()(); break; - case type_id::UINT8: op.template operator()(); break; + case type_id::INT8: + case type_id::UINT8: + case type_id::BOOL8: op.template operator()(); break; + case type_id::INT16: case type_id::UINT16: op.template operator()(); break; - case type_id::UINT32: op.template operator()(); break; - case type_id::UINT64: op.template operator()(); break; - case type_id::FLOAT32: op.template operator()(); break; - case type_id::FLOAT64: op.template operator()(); break; - case type_id::BOOL8: op.template operator()(); break; + case type_id::INT32: + case type_id::UINT32: + case type_id::FLOAT32: + case type_id::TIMESTAMP_DAYS: + case type_id::DURATION_DAYS: op.template operator()(); break; + case type_id::INT64: + case type_id::UINT64: + case type_id::FLOAT64: + case type_id::TIMESTAMP_SECONDS: + case type_id::TIMESTAMP_MILLISECONDS: + case type_id::TIMESTAMP_MICROSECONDS: + case type_id::TIMESTAMP_NANOSECONDS: + case type_id::DURATION_SECONDS: + case type_id::DURATION_MILLISECONDS: + case type_id::DURATION_MICROSECONDS: + case type_id::DURATION_NANOSECONDS: op.template operator()(); break; case type_id::DECIMAL32: op.template operator()(); break; case type_id::DECIMAL64: op.template operator()(); break; case type_id::DECIMAL128: op.template operator()(); break; - case type_id::TIMESTAMP_DAYS: op.template operator()(); break; - case type_id::TIMESTAMP_SECONDS: op.template operator()(); break; - case type_id::TIMESTAMP_MILLISECONDS: op.template operator()(); break; - case type_id::TIMESTAMP_MICROSECONDS: op.template operator()(); break; - case type_id::TIMESTAMP_NANOSECONDS: op.template operator()(); break; - case type_id::DURATION_DAYS: op.template operator()(); break; - case type_id::DURATION_SECONDS: op.template operator()(); break; - case type_id::DURATION_MILLISECONDS: op.template operator()(); break; - case type_id::DURATION_MICROSECONDS: op.template operator()(); break; - case type_id::DURATION_NANOSECONDS: op.template operator()(); break; case type_id::STRING: op.template operator()(); break; default: CUDF_UNREACHABLE(); } } template -__device__ void store_element(mutable_column_device_view const* column, +__device__ void store_element(mutable_column_device_view_core const* column, element_storage_t const* storage, size_type element_index, unsigned int active_mask) @@ -95,32 +90,32 @@ __device__ void store_element(mutable_column_device_view const* column, }; switch (column->type().id()) { - case type_id::INT8: op.template operator()(); break; - case type_id::INT16: op.template operator()(); break; - case type_id::INT32: op.template operator()(); break; - case type_id::INT64: op.template operator()(); break; - case type_id::UINT8: op.template operator()(); break; + case type_id::INT8: + case type_id::UINT8: + case type_id::BOOL8: op.template operator()(); break; + case type_id::INT16: case type_id::UINT16: op.template operator()(); break; - case type_id::UINT32: op.template operator()(); break; - case type_id::UINT64: op.template operator()(); break; - case type_id::FLOAT32: op.template operator()(); break; - case type_id::FLOAT64: op.template operator()(); break; - case type_id::BOOL8: op.template operator()(); break; + case type_id::INT32: + case type_id::UINT32: + case type_id::FLOAT32: + case type_id::TIMESTAMP_DAYS: + case type_id::DURATION_DAYS: op.template operator()(); break; + case type_id::INT64: + case type_id::UINT64: + case type_id::FLOAT64: + case type_id::TIMESTAMP_SECONDS: + case type_id::TIMESTAMP_MILLISECONDS: + case type_id::TIMESTAMP_MICROSECONDS: + case type_id::TIMESTAMP_NANOSECONDS: + case type_id::DURATION_SECONDS: + case type_id::DURATION_MILLISECONDS: + case type_id::DURATION_MICROSECONDS: + case type_id::DURATION_NANOSECONDS: op.template operator()(); break; case type_id::DECIMAL32: op.template operator()(); break; case type_id::DECIMAL64: op.template operator()(); break; case type_id::DECIMAL128: op.template operator()(); break; - case type_id::TIMESTAMP_DAYS: op.template operator()(); break; - case type_id::TIMESTAMP_SECONDS: op.template operator()(); break; - case type_id::TIMESTAMP_MILLISECONDS: op.template operator()(); break; - case type_id::TIMESTAMP_MICROSECONDS: op.template operator()(); break; - case type_id::TIMESTAMP_NANOSECONDS: op.template operator()(); break; - case type_id::DURATION_DAYS: op.template operator()(); break; - case type_id::DURATION_SECONDS: op.template operator()(); break; - case type_id::DURATION_MILLISECONDS: op.template operator()(); break; - case type_id::DURATION_MICROSECONDS: op.template operator()(); break; - case type_id::DURATION_NANOSECONDS: op.template operator()(); break; default: CUDF_UNREACHABLE(); } } -} // namespace cudf +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/element_storage.cuh b/cpp/src/jit/element_storage.cuh index 12bd920fe522..406797dcc2df 100644 --- a/cpp/src/jit/element_storage.cuh +++ b/cpp/src/jit/element_storage.cuh @@ -4,20 +4,51 @@ */ #pragma once +#include -namespace cudf { +#include +#include +#include -// can store any nullable or non-nullable element type. -template -struct element_storage { - alignas(max_element_size) unsigned char data[max_element_size * 2]; +namespace CUDF_EXPORT cudf { + +struct [[nodiscard]] layout { + std::size_t size = 0; //< Size in bytes of the layout + std::size_t alignment = 1; //< Non-zero power of 2 alignment + + constexpr layout unioned(layout const& other) const + { + return layout{.size = cuda::std::max(size, other.size), + .alignment = cuda::std::max(alignment, other.alignment)}; + } }; -// can store any non-nullable element type. -// -template -struct element_storage { - alignas(max_element_size) unsigned char data[max_element_size]; +template +inline constexpr layout layout_of = layout{.size = sizeof(T), .alignment = alignof(T)}; + +template +struct storage { + alignas(layout.alignment) unsigned char data[layout.size]; }; -} // namespace cudf +template +inline constexpr bool storage_compatible = + sizeof(ElementType) <= sizeof(Storage) && alignof(ElementType) <= alignof(Storage); + +template +using maybe_nullable = cuda::std::conditional_t, T>; + +template +using nonnull_element_storage = + storage; + +template +using nullable_element_storage = + storage>>>; + +template +using element_storage = cuda::std::conditional_t, + nonnull_element_storage>; + +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index ff68e5a3a928..2cf8d647dec7 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -91,8 +91,7 @@ std::vector input_type_names( kernel get_udf_kernel(std::string const& source_file, std::string const& kernel_name, - std::string const& udf_cuda_source, - std::vector const& extra_options) + std::string const& udf_cuda_source) { CUDF_FUNC_RANGE(); @@ -105,25 +104,11 @@ kernel get_udf_kernel(std::string const& source_file, char const* include_headers[] = // NOLINT(modernize-avoid-c-arrays) {udf_cuda_source.c_str(), kernel_instance_source.c_str()}; - constexpr int min_pch_cuda_version = 12800; // CUDA 12.8 - constexpr int min_minimal_cuda_version = 12800; // CUDA 12.8 - - int runtime_version; - CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); - - std::vector options; - options.emplace_back("-arch=sm_."); - return get_kernel(std::format("{}.jit.cu", source_file), source_file, include_names, include_headers, - kernel_name, - true, // TODO: use context config - runtime_version >= min_pch_cuda_version, - runtime_version >= min_minimal_cuda_version, - false, // TODO: use context config - extra_options); + kernel_name); } } // namespace jit diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index 2504566e8df0..1831db7edcba 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -77,8 +77,7 @@ std::vector input_type_names( kernel get_udf_kernel(std::string const& source_file, std::string const& kernel_name, - std::string const& udf_cuda_source, - std::vector const& extra_options = {}); + std::string const& udf_cuda_source); } // namespace jit } // namespace cudf diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp index e267e9429afd..d7aaa314bc55 100644 --- a/cpp/src/jit/jit.cpp +++ b/cpp/src/jit/jit.cpp @@ -22,13 +22,6 @@ namespace CUDF_EXPORT cudf { namespace { -rtcx::sha256 hash(char const* input) -{ - rtcx::sha256_context ctx; - ctx.update(std::span{reinterpret_cast(input), std::strlen(input)}); - return ctx.finalize(); -} - rtcx::sha256 hash(std::span input) { rtcx::sha256_context ctx; @@ -45,112 +38,35 @@ rtcx::sha256 hash(std::span inputs) return ctx.finalize(); } -[[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) -{ - auto error_code = errno; - auto error_str = std::format( - "{}. `{}` failed with {} ({})", message, syscall_name, error_code, std::strerror(error_code)); - CUDF_FAIL(error_str, std::runtime_error); -} - -void install_file(char const* dst_path, std::span contents) -{ - int dst_file = open(dst_path, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); - if (dst_file == -1) { - if (errno == EEXIST) { - // file already exists, so just return - return; - } - throw_posix(std::format("Failed to create file ({})", dst_path), "open"); - } - - RTCX_DEFER([&] { - if (close(dst_file) != 0) { - throw_posix(std::format("Failed to close file ({})", dst_path), "close"); - } - }); - - if (write(dst_file, contents.data(), contents.size()) == -1) { - throw_posix(std::format("Failed to write file ({})", dst_path), "write"); - } -} - -/** - * @brief Reads the contents of a file into a byte buffer and null-terminates it to allow for safe - * usage as a C-string. - */ -rtcx::byte_buffer read_blob_cstring(char const* path) +rtcx::sha256 hash(std::span file_fragments, + std::span memory_fragments) { - int32_t fd = open(path, O_RDONLY); - if (fd == -1) { throw_posix(std::format("Failed to open file ({})", path), "open"); } - - RTCX_DEFER([&] { - if (close(fd) == -1) { throw_posix(std::format("Failed to close file ({})", path), "close"); } - }); - - auto file_size = lseek(fd, 0, SEEK_END); - if (file_size == -1) { - throw_posix(std::format("Failed to determine size of file ({})", path), "lseek"); - } - // TODO: make all read/write syscalls call read/write in a loop - - if (lseek(fd, 0, SEEK_SET) == -1) { - throw_posix(std::format("Failed to reset file offset for file ({})", path), "lseek"); - } - - auto contents = rtcx::byte_buffer::make(file_size + 1U); // +1 for null terminator - - if (read(fd, contents.data(), file_size) == -1) { - throw_posix(std::format("Failed to read file ({})", path), "read"); + rtcx::sha256_context ctx; + for (auto const& fragment : file_fragments) { + ctx.update( + std::span{reinterpret_cast(fragment.path), std::strlen(fragment.path)}); } - contents.data()[file_size] = '\0'; // null-terminate the buffer - - return contents; -} - -rtcx::byte_buffer decompress_blob(std::span compressed_binary, - size_t uncompressed_size, - std::string_view compression) -{ - CUDF_EXPECTS(compression == "none" || compression == "zstd", - std::format("Unsupported compression type specified: {}", compression), - std::runtime_error); - auto decompressed = rtcx::byte_buffer::make(uncompressed_size); - - if (compression == "zstd") { - size_t errc = ZSTD_decompress( - decompressed.data(), uncompressed_size, compressed_binary.data(), compressed_binary.size()); - - CUDF_EXPECTS( - !ZSTD_isError(errc) && errc == uncompressed_size, - std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : ", - errc, - ZSTD_getErrorName(errc)), - std::runtime_error); - } else { - // compression is "none", so just copy the data - std::copy(compressed_binary.data(), - compressed_binary.data() + compressed_binary.size(), - decompressed.data()); + for (auto const& fragment : memory_fragments) { + ctx.update(fragment.data); } - return decompressed; + return ctx.finalize(); } void install_file_set(std::string_view target_dir, std::span compressed_binary, size_t uncompressed_size, - std::span file_ranges, + std::span file_ranges, std::span destinations, std::string_view compression) { auto decompressed = rtcx::decompress_blob(compressed_binary, uncompressed_size, compression); for (size_t i = 0; i < file_ranges.size(); ++i) { auto file_data_range = file_ranges[i]; - auto file_data = std::span{decompressed.data() + file_data_range.offset, file_data_range.size}; - auto dst_path = destinations[i]; - auto target_path = std::format("{}/{}", target_dir, dst_path); + auto file_data = std::span{decompressed.data() + file_data_range[0], file_data_range[1]}; + auto dst_path = destinations[i]; + auto target_path = std::format("{}/{}", target_dir, dst_path); std::filesystem::create_directories(std::filesystem::path{target_path}.parent_path()); @@ -195,11 +111,11 @@ void install_cudf_jit_files(std::string const& target_dir, std::string const& tm std::runtime_error); install_file_set(tmp_dir_path, - rtcx_embed::cudf_jit_embed_files, - rtcx_embed::cudf_jit_embed_files_uncompressed_size, - rtcx_embed::cudf_jit_embed_file_ranges, - rtcx_embed::cudf_jit_embed_file_destinations, - rtcx_embed::cudf_jit_embed_files_compression); + cudf_jit_embed::files, + cudf_jit_embed::files_uncompressed_size, + cudf_jit_embed::file_ranges, + cudf_jit_embed::file_destinations, + cudf_jit_embed::files_compression); // rename the temporary directory to the target install directory if (::rename(tmp_dir_path, target_dir.c_str()) == -1) { @@ -247,7 +163,7 @@ void jit_bundle_t::ensure_installed() const std::string jit_bundle_t::get_hash() const { - auto str = rtcx::sha256_hex_string::make(rtcx_embed::cudf_jit_embed_hash); + auto str = rtcx::sha256_hex_string::make(cudf_jit_embed::hash); return std::string{str.view()}; } @@ -261,7 +177,7 @@ std::vector jit_bundle_t::get_include_directories() const std::vector directories; auto base_dir = get_directory(); - for (auto dir : rtcx_embed::cudf_jit_embed_include_directories) { + for (auto dir : cudf_jit_embed::include_directories) { directories.emplace_back(std::format("{}/{}", base_dir, dir)); } @@ -270,6 +186,9 @@ std::vector jit_bundle_t::get_include_directories() const namespace { +constexpr int MIN_CUDA_VERSION_PCH = 12800; // CUDA 12.8 +constexpr int MIN_CUDA_VERSION_MINIMAL = 12800; // CUDA 12.8 + int32_t get_driver_version() { int32_t driver_version; @@ -300,19 +219,21 @@ std::tuple compile_library_uncached( char const* cuda_code, std::span extra_header_include_names, std::span extra_headers, - std::span extra_options, - std::span name_expressions, - bool use_pch, - bool use_minimal, - bool log_pch) + std::span name_expressions) { CUDF_FUNC_RANGE(); - auto& bundle = cudf::get_context().jit_bundle(); + auto& ctx = cudf::get_context(); + auto& cfg = ctx.config(); + auto& bundle = ctx.jit_bundle(); auto sm = get_current_device_physical_model(); + auto runtime = get_runtime_version(); auto include_dirs = bundle.get_include_directories(); - auto pch_dir = cudf::get_context().get_jit_pch_dir(); + auto pch_dir = ctx.get_jit_pch_dir(); + + auto use_pch = runtime >= MIN_CUDA_VERSION_PCH; + auto use_minimal = runtime >= MIN_CUDA_VERSION_MINIMAL; std::vector options; @@ -337,19 +258,27 @@ std::tuple compile_library_uncached( if (use_pch) { options.emplace_back("--pch"); - if (log_pch) { + if (cfg.jit_verbose) { options.emplace_back("--pch-verbose=true"); options.emplace_back("--pch-messages=true"); + } else { + options.emplace_back("--pch-verbose=false"); + options.emplace_back("--pch-messages=false"); } } + if (cfg.disable_cuda_cache) { options.emplace_back("--no-cache"); } + + if (cfg.dump_jit_trace) { options.emplace_back("--time=-"); } + + if (cfg.dump_jit_time_profile) { + options.emplace_back(std::format("--fdevice-time-trace=cudf_kernel_{}_trace", name)); + } + std::vector options_cstr; for (auto const& option : options) { options_cstr.emplace_back(option.c_str()); } - for (auto* option : extra_options) { - options_cstr.emplace_back(option); - } auto params = rtcx::compile_params{.name = name, .source = cuda_code, @@ -366,154 +295,27 @@ std::tuple compile_library_uncached( return std::make_tuple(library, std::make_shared(std::move(blob))); } -rtcx::byte_buffer compile_fragment_uncached(char const* name, - char const* cuda_code, - std::span extra_header_include_names, - std::span extra_headers, - std::span extra_options, - std::span name_expressions, - bool use_pch, - bool use_minimal, - bool log_pch) -{ - CUDF_FUNC_RANGE(); - - auto& bundle = cudf::get_context().jit_bundle(); - auto sm = get_current_device_physical_model(); - - auto include_dirs = bundle.get_include_directories(); - auto pch_dir = cudf::get_context().get_jit_pch_dir(); - - std::vector options; - - for (auto const& include_dir : include_dirs) { - options.emplace_back(std::format("-I{}", include_dir)); - } - - options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); - - options.emplace_back("--diag-suppress=47"); - options.emplace_back("--device-int128"); - - if (sm >= 100) { options.emplace_back("--device-float128"); } - - options.emplace_back("-std=c++20"); - options.emplace_back("--device-as-default-execution-space"); - options.emplace_back("--generate-line-info"); - options.emplace_back("--dopt=on"); - - if (use_minimal) { options.emplace_back("--minimal"); } - - if (use_pch) { - options.emplace_back("--pch"); - - if (log_pch) { - options.emplace_back("--pch-verbose=true"); - options.emplace_back("--pch-messages=true"); - } - } - - options.emplace_back("--relocatable-device-code=true"); - options.emplace_back("--gen-opt-lto"); - options.emplace_back("--dlink-time-opt"); - - std::vector options_cstr; - for (auto const& option : options) { - options_cstr.emplace_back(option.c_str()); - } - for (auto* option : extra_options) { - options_cstr.emplace_back(option); - } - - auto params = rtcx::compile_params{.name = name, - .source = cuda_code, - .header_include_names = extra_header_include_names, - .headers = extra_headers, - .options = options_cstr, - .name_expressions = name_expressions, - .target_type = rtcx::binary_type::LTO_IR}; - - return rtcx::compile(params); -} - -std::variant get_cuda_udf_fragment( - char const* name, - char const* cuda_code, - std::span extra_header_include_names, - std::span extra_headers, - std::span extra_options, - std::span name_expressions, - bool use_pch, - bool use_minimal, - bool log_pch, - bool use_cache) -{ - auto& cache = cudf::get_context().rtcx_cache(); - auto& bundle = cudf::get_context().jit_bundle(); - - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_physical_model(); - auto header_include_names_hash = hash(extra_header_include_names).to_hex_string(); - auto headers_hash = hash(extra_headers).to_hex_string(); - auto bundle_hash = bundle.get_hash(); - auto source_hash = hash(cuda_code).to_hex_string(); - - auto cache_key = std::format(R"***(cudaFragment -binary_type=LTO_IR -name={} -cuda_runtime={} -cuda_driver={} -arch={} -header_include_names={}, -headers={}, -bundle={}, -source_file={} -)***", - name, - runtime, - driver, - sm, - header_include_names_hash.view(), - headers_hash.view(), - bundle_hash, - source_hash.view()); - - auto cache_key_sha256 = hash(cache_key); - - auto compile = [&] { - return compile_fragment_uncached(name, - cuda_code, - extra_header_include_names, - extra_headers, - extra_options, - name_expressions, - use_pch, - use_minimal, - log_pch); - }; - - if (!use_cache) { return compile(); } - - auto fut = - cache.get_or_add_blob(cache_key_sha256, rtcx::blob_compile_func::from_functor(compile)); - return fut.get(); -} - std::tuple link_library_uncached( char const* name, std::span file_fragments, - std::span memory_fragments, - std::span extra_options) + std::span memory_fragments) { CUDF_FUNC_RANGE(); - auto sm = get_current_device_physical_model(); + auto sm = get_current_device_physical_model(); + auto& ctx = cudf::get_context(); + auto& cfg = ctx.config(); std::vector options; - options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); options.emplace_back("-lto"); + options.emplace_back(std::format("-arch=sm_{}", sm)); + + if (cfg.disable_cuda_cache) { options.emplace_back("--no-cache"); } + + if (cfg.jit_verbose) { options.emplace_back("-verbose"); } + + if (cfg.dump_jit_trace) { options.emplace_back("-time"); } std::vector options_cstr; @@ -521,10 +323,6 @@ std::tuple link_library_uncached( options_cstr.emplace_back(option.c_str()); } - for (auto* option : extra_options) { - options_cstr.emplace_back(option); - } - auto params = rtcx::link_params{.name = name, .output_type = rtcx::binary_type::CUBIN, .file_fragments = file_fragments, @@ -544,12 +342,7 @@ kernel get_kernel(std::string const& name, std::string const& source_file_id, std::span header_include_names, std::span headers, - std::string const& kernel_instance, - bool use_cache, - bool use_pch, - bool use_minimal, - bool log_pch, - std::span extra_options) + std::string const& kernel_instance) { CUDF_FUNC_RANGE(); @@ -591,54 +384,30 @@ kernel_instance={} auto compile = [&] { auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); auto source = read_file_string(source_file.c_str()); - std::vector extra_options_cstr; - for (auto const& option : extra_options) { - extra_options_cstr.emplace_back(option.c_str()); - } - - return compile_library_uncached(name.c_str(), - source.c_str(), - header_include_names, - headers, - extra_options_cstr, - {}, - use_pch, - use_minimal, - log_pch); + return compile_library_uncached( + name.c_str(), source.c_str(), header_include_names, headers, {}); }; - if (!use_cache) { - auto [lib, blob] = compile(); - return kernel{lib, lib->get_kernel("cudf_kernel")}; - } - auto fut = cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); auto lib = fut.get(); - return kernel{lib, lib->get_kernel("cudf_kernel")}; + return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; } -kernel get_linked_kernel(std::string const& name, - std::string const& fragment_file_id, - std::span file_fragments, - std::span memory_fragments, - bool use_cache, - std::span extra_options) +kernel get_lto_linked_kernel(std::string const& name, + std::span file_fragments, + std::span memory_fragments) { CUDF_FUNC_RANGE(); - auto& cache = cudf::get_context().rtcx_cache(); - auto& bundle = cudf::get_context().jit_bundle(); - - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_physical_model(); - auto file_fragments_hash = hash(file_fragments).to_hex_string(); - auto memory_fragments_hash = hash(memory_fragments).to_hex_string(); - auto bundle_hash = bundle.get_hash(); - auto fragment_file = std::format("{}/fragments/{}", bundle.get_directory(), fragment_file_id); - // TODO: resolve file fragment IDs + auto& cache = cudf::get_context().rtcx_cache(); + auto& bundle = cudf::get_context().jit_bundle(); + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_physical_model(); + auto bundle_hash = bundle.get_hash(); + auto fragments_hash = hash(file_fragments, memory_fragments).to_hex_string(); auto cache_key = std::format(R"***(cuLibrary name={} @@ -647,102 +416,26 @@ cuda_runtime={} cuda_driver={} arch={} bundle={} -fragment_file={} -file_fragments={} -memory_fragments={} +fragments={} )***", name, runtime, driver, sm, bundle_hash, - fragment_file, - file_fragments_hash.view(), - memory_fragments_hash.view()); + fragments_hash.view()); auto cache_key_sha256 = hash(cache_key); auto compile = [&] { - std::vector extra_options_cstr; - for (auto const& option : extra_options) { - extra_options_cstr.emplace_back(option.c_str()); - } - - return link_library_uncached( - name.c_str(), file_fragments, memory_fragments, extra_options_cstr); + return link_library_uncached(name.c_str(), file_fragments, memory_fragments); }; - if (!use_cache) { - auto [lib, blob] = compile(); - return kernel{lib, lib->get_kernel("kernel")}; - } - auto fut = cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); auto lib = fut.get(); - return kernel{lib, lib->get_kernel("kernel")}; -} - -kernel get_cuda_linked_kernel(std::string const& kernel_name, - std::string const& kernel_fragment_file_id, - std::string const& cuda_udf_name, - std::string const& cuda_udf_source, - std::span header_include_names, - std::span headers, - std::span file_fragments, - std::span memory_fragments, - bool use_cache, - bool use_pch, - bool use_minimal, - bool log_pch, - std::span extra_compile_options, - std::span extra_link_options) -{ - CUDF_FUNC_RANGE(); - - std::vector extra_compile_options_cstr; - for (auto const& option : extra_compile_options) { - extra_compile_options_cstr.emplace_back(option.c_str()); - } - - std::vector extra_link_options_cstr; - for (auto const& option : extra_link_options) { - extra_link_options_cstr.emplace_back(option.c_str()); - } - - auto udf_fragment = get_cuda_udf_fragment(cuda_udf_name.c_str(), - cuda_udf_source.c_str(), - header_include_names, - headers, - extra_compile_options_cstr, - {}, - use_pch, - use_minimal, - log_pch, - use_cache); - - auto fragment_name = std::format("{}_udf_fragment", cuda_udf_name); - - std::vector all_fragments; - all_fragments.insert(all_fragments.end(), memory_fragments.begin(), memory_fragments.end()); - - if (std::holds_alternative(udf_fragment)) { - auto& udf_blob = std::get(udf_fragment); - all_fragments.push_back(rtcx::memory_fragment{ - .data = udf_blob->view(), .type = rtcx::binary_type::LTO_IR, .name = fragment_name.c_str()}); - } else { - auto& udf_buffer = std::get(udf_fragment); - all_fragments.push_back(rtcx::memory_fragment{ - .data = udf_buffer, .type = rtcx::binary_type::LTO_IR, .name = fragment_name.c_str()}); - } - - return get_linked_kernel(kernel_name, - kernel_fragment_file_id, - file_fragments, - all_fragments, - use_cache, - extra_link_options); + return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; } } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp index 256b944606d5..8525c16dde0e 100644 --- a/cpp/src/jit/jit.hpp +++ b/cpp/src/jit/jit.hpp @@ -81,71 +81,22 @@ struct [[nodiscard]] kernel { * @param headers Contents of any additional embedded header files to include during compilation * @param kernel_instance String identifier for the specific kernel instance being requested (used * for caching) - * @param use_cache Whether to use the caching mechanism to avoid redundant compilations - * @param use_pch Whether to use precompiled headers for faster compilation (if supported) - * @param use_minimal Whether to use NVRTC's minimal compilation mode for faster compilation (if - * supported) - * @param log_pch Whether to log precompiled header usage - * @param extra_options Additional compilation options */ kernel get_kernel(std::string const& name, std::string const& source_file_id, std::span header_include_names, std::span headers, - std::string const& kernel_instance, - bool use_cache = true, - bool use_pch = true, - bool use_minimal = true, - bool log_pch = false, - std::span extra_options = {}); + std::string const& kernel_instance); /** * @brief Gets a kernel by linking together embedded binary fragments * @param name Debug name for the kernel (used for caching and logging) - * @param fragment_file_id Identifier for the embedded fragment file (used to locate the fragments - * and for caching) - * @param file_fragments Embedded fragments to link + * @param file_fragments Paths of the fragments to link together to form the kernel * @param memory_fragments Memory fragments to link - * @param use_cache Whether to use the cache system * @param extra_options Additional linking options */ -kernel get_linked_kernel(std::string const& name, - std::string const& fragment_file_id, - std::span file_fragments, - std::span memory_fragments, - bool use_cache = true, - std::span extra_options = {}); - -/** - * @brief Gets a kernel by compiling a CUDA UDF and linking together binary fragments. - * @param kernel_name Debug name for the kernel (used for caching and logging) - * @param kernel_fragment_file_id Identifier for the embedded fragment file (used to locate the fragments - * and for caching) - * @param cuda_udf_name Name of the CUDA UDF - * @param cuda_udf_source CUDA source code for the UDF - * @param file_fragments Embedded fragments to link - * @param memory_fragments Memory fragments to link - * @param use_cache Whether to use the cache system - * @param use_pch Whether to use precompiled headers for faster compilation (if supported) - * @param use_minimal Whether to use NVRTC's minimal compilation mode for faster compilation (if - * supported) - * @param log_pch Whether to log precompiled header usage - * @param extra_compile_options Additional compilation options - * @param extra_link_options Additional linking options - */ -kernel get_cuda_linked_kernel(std::string const& kernel_name, - std::string const& kernel_fragment_file_id, - std::string const& cuda_udf_name, - std::string const& cuda_udf_source, - std::span header_include_names, - std::span headers, - std::span file_fragments, - std::span memory_fragments, - bool use_cache = true, - bool use_pch = true, - bool use_minimal = true, - bool log_pch = false, - std::span extra_compile_options = {}, - std::span extra_link_options = {}); +kernel get_lto_linked_kernel(std::string const& name, + std::span file_fragments, + std::span memory_fragments); } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index 3cc8a72a14c5..10a94ed01bf6 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -95,12 +95,13 @@ CUDF_KERNEL void filter_join_kernel(cudf::jit::device_span left_indices, - cudf::jit::device_span right_indices, - cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - bool* predicate_results, - void* user_data) +extern "C" __global__ void cudf_kernel_entry( + cudf::jit::device_span left_indices, + cudf::jit::device_span right_indices, + cudf::column_device_view_core const* left_tables, + cudf::column_device_view_core const* right_tables, + bool* predicate_results, + void* user_data) { CUDF_KERNEL_INSTANCE( left_indices, right_indices, left_tables, right_tables, predicate_results, user_data); diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index 61b016b23213..7aa18382c361 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -113,15 +113,16 @@ CUDF_KERNEL void rolling_window_kernel(cudf::size_type nrows, } // namespace rolling } // namespace cudf -extern "C" __global__ void cudf_kernel(cudf::size_type nrows, - void const* const __restrict__ in_col, - cudf::bitmask_type const* const __restrict__ in_col_valid, - void* __restrict__ out_col, - cudf::bitmask_type* __restrict__ out_col_valid, - cudf::size_type* __restrict__ output_valid_count, - cudf::detail::window_wrapper_base preceding_window_begin, - cudf::detail::window_wrapper_base following_window_begin, - cudf::size_type min_periods) +extern "C" __global__ void cudf_kernel_entry( + cudf::size_type nrows, + void const* const __restrict__ in_col, + cudf::bitmask_type const* const __restrict__ in_col_valid, + void* __restrict__ out_col, + cudf::bitmask_type* __restrict__ out_col_valid, + cudf::size_type* __restrict__ output_valid_count, + cudf::detail::window_wrapper_base preceding_window_begin, + cudf::detail::window_wrapper_base following_window_begin, + cudf::size_type min_periods) { CUDF_KERNEL_INSTANCE(nrows, in_col, diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 24e66a2894a8..118ab8cc6770 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -112,11 +112,15 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { std::call_once(*_context_init_flag, [&]() { - bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); - bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); - bool preload_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_PRELOAD", false); - bool disable_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_DISABLED", false); - bool clear_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); + bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); + bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); + bool preload_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_PRELOAD", false); + bool disable_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_DISABLED", false); + bool clear_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); + bool disable_cuda_cache = get_bool_env_or("LIBCUDF_JIT_DISABLE_CUDA_CACHE", false); + bool jit_verbose = get_bool_env_or("LIBCUDF_JIT_VERBOSE", false); + bool dump_jit_trace = get_bool_env_or("LIBCUDF_JIT_DUMP_TRACE", false); + bool dump_jit_time_profile = get_bool_env_or("LIBCUDF_JIT_DUMP_TIME_PROFILE", false); auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16'384U); @@ -132,6 +136,10 @@ void initialize(init_flags flags) .preload_jit_cache = preload_jit_cache, .disable_jit_cache = disable_jit_cache, .clear_jit_cache = clear_jit_cache, + .disable_cuda_cache = disable_cuda_cache, + .jit_verbose = jit_verbose, + .dump_jit_trace = dump_jit_trace, + .dump_jit_time_profile = dump_jit_time_profile, .rtcx_cache_dir = rtcx_cache_dir, .jit_bundle_dir = jit_bundle_dir, .jit_pch_dir = jit_pch_dir, diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 59e27e00f57f..559adb0abd8c 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -20,11 +20,16 @@ namespace cudf { struct jit_bundle_t; struct [[nodiscard]] context_config { - bool dump_codegen : 1 = false; - bool use_jit : 1 = false; - bool preload_jit_cache : 1 = false; - bool disable_jit_cache : 1 = false; - bool clear_jit_cache : 1 = false; + bool dump_codegen : 1 = false; + bool use_jit : 1 = false; + bool preload_jit_cache : 1 = false; + bool disable_jit_cache : 1 = false; + bool clear_jit_cache : 1 = false; + bool disable_codegen_cache : 1 = false; + bool disable_cuda_cache : 1 = false; + bool jit_verbose : 1 = false; + bool dump_jit_trace : 1 = false; + bool dump_jit_time_profile : 1 = false; std::string rtcx_cache_dir = {}; std::string jit_bundle_dir = {}; std::string jit_pch_dir = {}; @@ -63,6 +68,8 @@ class context { [[nodiscard]] bool use_jit() const; + [[nodiscard]] context_config const& config() const { return _config; } + [[nodiscard]] std::string const& get_jit_pch_dir() const; /// @brief Initialize additional components based on the provided flags diff --git a/cpp/src/transform/compute_columns_jit.cu b/cpp/src/transform/compute_columns_jit.cu deleted file mode 100644 index 3060c26af7d0..000000000000 --- a/cpp/src/transform/compute_columns_jit.cu +++ /dev/null @@ -1,18 +0,0 @@ - - - -#include - -namespace CUDF_EXPORT cudf { - -std::unique_ptr compute_columns( - std::span inputs, - std::span udf, - std::span output_types, - void* user_data, - null_aware is_null_aware = null_aware::NO, - std::optional row_size = std::nullopt, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); - -} \ No newline at end of file diff --git a/cpp/src/transform/jit/compute_columns_kernel.cu b/cpp/src/transform/jit/compute_columns_kernel.cu deleted file mode 100644 index e2c31429aadf..000000000000 --- a/cpp/src/transform/jit/compute_columns_kernel.cu +++ /dev/null @@ -1,104 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "jit/element.cuh" -#include "jit/element_storage.cuh" - -#include -#include -#include - -#include -#include -#include - -namespace cudf { - -/// @brief The compute operation to perform on each element -/// @param user_data Pointer to user data passed to the kernel -/// @param element_index The index of the element to compute -/// @param inputs Pointer to the input elements for this operation; the caller guarantees the memory -/// layout and type of these elements based on the input column device views and input strides -/// @param input_stride The stride (in bytes) between consecutive input elements for a given input -/// column -/// @param outputs Pointer to the output elements for this operation; the caller guarantees the -/// memory layout and type of these elements based on the output column device views -/// @param output_stride The stride (in bytes) between consecutive output elements for a given -/// output column -/// @return An integer status code; the meaning of this code is defined by the caller and operator -/// implementation -extern "C" __device__ int operation(void* user_data, - long int element_index, - void const* inputs, - int input_stride, - void* outputs, - int output_stride); - -template -__device__ void compute_columns_kernel(size_type row_size, - void* __restrict__ user_data, - column_device_view const* __restrict__ input_cols, - size_type num_inputs, - mutable_column_device_view const* __restrict__ output_cols, - size_type num_outputs, - size_type const* __restrict__ input_strides) -{ - // 255 i32 registers per thread for GH200 - // 227 SHMEM bytes per thread for GH200 - - using input_storage_t = element_storage; - using output_storage_t = element_storage; - - extern __shared__ char shmem[]; - - auto shmem_iter = shmem; - input_storage_t* input_storage = reinterpret_cast(shmem_iter); - shmem_iter += sizeof(input_storage_t) * num_inputs; - output_storage_t* output_storage = reinterpret_cast(shmem_iter); - - auto start = detail::grid_1d::global_thread_id(); - auto stride = detail::grid_1d::grid_stride(); - - for (auto element_idx = start; element_idx < row_size; element_idx += stride) { - auto active_mask = [&] __device__() { - if constexpr (has_nulls) { - return __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); - } else { - return 0xFFFF'FFFFU; - } - }(); - - for (int i = 0; i < num_inputs; i++) { - load_element( - input_cols + i, element_idx * input_strides[i], input_storage + i); - } - - operation(user_data, - element_idx, - input_storage, - sizeof(input_storage_t), - output_storage, - sizeof(output_storage_t)); - - for (int i = 0; i < num_outputs; i++) { - store_element( - output_cols + i, output_storage + i, element_idx, active_mask); - } - } -} - -} // namespace cudf - -extern "C" __global__ void kernel(cudf::size_type row_size, - void* __restrict__ user_data, - cudf::column_device_view const* __restrict__ input_cols, - cudf::size_type num_inputs, - cudf::mutable_column_device_view const* __restrict__ output_cols, - cudf::size_type num_outputs, - cudf::size_type const* __restrict__ input_strides) -{ - compute_columns_kernel( - row_size, user_data, input_cols, num_inputs, output_cols, num_outputs, input_strides); -} diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 4d85b7089432..0f78264966ef 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -35,75 +36,59 @@ namespace cudf { namespace jit { -template -__device__ void execute_transform_op(void* user_data, size_type element_idx, Args args) -{ - // TODO: static assert invocable - if constexpr (has_user_data) { - cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, - cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, args)); - } else { - cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, args); - } -} - /// @brief The generic transform kernel. Supports all types and nullability combinations. -template +template CUDF_KERNEL void transform_kernel(size_type row_size, bitmask_type const* __restrict__ stencil, void* __restrict__ user_data, column_device_view_core const* __restrict__ input_cols, mutable_column_device_view_core const* __restrict__ output_cols) { - // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); - for (auto element_idx = start; element_idx < row_size; element_idx += stride) { - if constexpr (is_null_aware == null_aware::NO) { - if (stencil != nullptr && !bit_is_set(stencil, element_idx)) { continue; } + for (auto row = start; row < row_size; row += stride) { + auto operation = [&](Args const& args) { + if constexpr (has_user_data) { + cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, + cuda::std::tuple_cat(cuda::std::tuple{user_data, row}, args)); + } else { + cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, args); + } + }; - auto ins = InputAccessors::map( - [&]() { return cuda::std::tuple{A::element(input_cols, element_idx)...}; }); + if constexpr (!is_null_aware) { + if (stencil != nullptr && !bit_is_set(stencil, row)) { continue; } - auto outs = OutputAccessors::map([&]() { - return cuda::std::tuple{A::output_arg(output_cols, element_idx)...}; - }); + auto ins = InputAccessors::map( + [&]() { return cuda::std::tuple{A::element(input_cols, row)...}; }); - auto out_ptrs = - cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); + auto outs = OutputAccessors::map( + [&]() { return cuda::std::tuple{A::output_arg(output_cols, row)...}; }); - execute_transform_op( - user_data, element_idx, cuda::std::tuple_cat(out_ptrs, ins)); + operation(cuda::std::tuple_cat( + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs), ins)); OutputAccessors::map([&]() { - (A::assign(output_cols, element_idx, cuda::std::get(outs)), ...); + (A::assign(output_cols, row, cuda::std::get(outs)), ...); }); } else { - auto active_mask = __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); - - auto ins = InputAccessors::map([&]() { - return cuda::std::tuple{A::nullable_element(input_cols, element_idx)...}; - }); + auto ins = InputAccessors::map( + [&]() { return cuda::std::tuple{A::nullable_element(input_cols, row)...}; }); - auto outs = OutputAccessors::map([&]() { - return cuda::std::tuple{A::null_output_arg(output_cols, element_idx)...}; - }); + auto outs = OutputAccessors::map( + [&]() { return cuda::std::tuple{A::null_output_arg(output_cols, row)...}; }); - auto out_ptrs = - cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); + operation(cuda::std::tuple_cat( + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs), ins)); - execute_transform_op( - user_data, element_idx, cuda::std::tuple_cat(out_ptrs, ins)); + auto active_mask = __ballot_sync(0xFFFF'FFFFU, row < row_size); OutputAccessors::map([&]() { - (A::assign(output_cols, element_idx, *cuda::std::get(outs)), ...); + (A::assign(output_cols, row, *cuda::std::get(outs)), ...); (warp_compact_validity( - active_mask, output_cols, element_idx, cuda::std::get(outs).has_value()), + active_mask, output_cols, row, cuda::std::get(outs).has_value()), ...); }); } @@ -113,7 +98,7 @@ CUDF_KERNEL void transform_kernel(size_type row_size, } // namespace jit } // namespace cudf -extern "C" __global__ void cudf_kernel( +extern "C" __global__ void cudf_kernel_entry( cudf::size_type row_size, cudf::bitmask_type const* __restrict__ stencil, void* __restrict__ user_data, diff --git a/cpp/src/transform/jit/lto_kernel.cu b/cpp/src/transform/jit/lto_kernel.cu new file mode 100644 index 000000000000..17450ce9065b --- /dev/null +++ b/cpp/src/transform/jit/lto_kernel.cu @@ -0,0 +1,145 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before + // it. Each UDF will have a different operation-udf.hpp generated for it, so we + // need to put this pragma before including it to avoid PCH mismatch. + +// clang-format off +#include +// clang-format on + +namespace cudf { + +/// @brief The generic LTO transform kernel. Supports all types and nullability combinations. +/// This kernel is intended to be used with LTO, it has uses registers and has clear memory +/// boundaries. This is intended to be used for simple n-ary operators. +template +__device__ void lto_transform_kernel( + size_type row_size, + bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + column_device_view_core const* __restrict__ input_cols, + mutable_column_device_view_core const* __restrict__ output_cols) +{ + auto start = detail::grid_1d::global_thread_id(); + auto stride = detail::grid_1d::grid_stride(); + + static constexpr auto input_layout = + null_aware ? InputAccessors::map([&]() { + layout result{}; + ((result = result.unioned(layout_of)), ...); + return result; + }) + : InputAccessors::map([&]() { + layout result{}; + ((result = result.unioned(layout_of)), ...); + return result; + }); + + static constexpr auto output_layout = + null_aware ? OutputAccessors::map([&]() { + layout result{}; + ((result = result.unioned(layout_of)), ...); + return result; + }) + : OutputAccessors::map([&]() { + layout result{}; + ((result = result.unioned(layout_of)), ...); + return result; + }); + + for (auto row = start; row < row_size; row += stride) { + if constexpr (null_aware) { + if (stencil != nullptr && !bit_is_set(stencil, row)) { continue; } + } + + using input_storage_t = storage; + using output_storage_t = storage; + + output_storage_t outputs_storage[OutputAccessors::size]; + input_storage_t inputs_storage[InputAccessors::size]; + + InputAccessors::map([&]() { + if constexpr (null_aware) { + ((*reinterpret_cast(inputs_storage[A::index].data) = + A::nullable_element(input_cols, row)), + ...); + } else { + ((*reinterpret_cast(inputs_storage[A::index].data) = + A::element(input_cols, row)), + ...); + } + }); + + OutputAccessors::map([&]() { + if constexpr (null_aware) { + ((*reinterpret_cast(outputs_storage[A::index].data) = + A::null_output_arg(output_cols, row)), + ...); + } else { + ((*reinterpret_cast(outputs_storage[A::index].data) = + A::output_arg(output_cols, row)), + ...); + } + }); + + [[maybe_unused]] auto errc = cudf_transform_operation(user_data, + row, + &inputs_storage, + sizeof(inputs_storage), + &outputs_storage, + sizeof(outputs_storage)); + + // used only for null-aware + auto active_mask = null_aware ? __ballot_sync(0xFFFF'FFFFU, row < row_size) : 0xFFFF'FFFFU; + + auto assign = [&]() { + auto* src = outputs_storage[A::index].data; + if constexpr (null_aware) { + auto& ret = *reinterpret_cast(src); + A::assign(output_cols, row, *ret); + jit::warp_compact_validity(active_mask, output_cols, row, ret.has_value()); + } else { + A::assign(output_cols, row, *reinterpret_cast(src)); + } + }; + + OutputAccessors::map([&]() { (assign.template operator()(), ...); }); + } +} + +} // namespace cudf + +extern "C" __global__ void cudf_kernel_entry( + cudf::size_type row_size, + cudf::bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + cudf::column_device_view_core const* __restrict__ input_cols, + cudf::mutable_column_device_view_core const* __restrict__ output_cols) +{ + CUDF_KERNEL_INSTANCE(row_size, stencil, user_data, input_cols, output_cols); +} diff --git a/cpp/src/transform/jit/untyped_lto_kernel.cu b/cpp/src/transform/jit/untyped_lto_kernel.cu new file mode 100644 index 000000000000..60fe72758289 --- /dev/null +++ b/cpp/src/transform/jit/untyped_lto_kernel.cu @@ -0,0 +1,139 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "jit/element.cuh" +#include "jit/element_storage.cuh" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before + // it. Each UDF will have a different operation-udf.hpp generated for it, so we + // need to put this pragma before including it to avoid PCH mismatch. + +// clang-format off +#include +// clang-format on + +namespace cudf { + +/** + * @brief An untyped Generic Transform kernel. This is a catch-all kernel that can be used for any + * transform operation, but is not expected to be the most performant. The intent is that this + * kernel can be used for any operator and type combination with minimal specialization, but that + * more specialized kernels can be generated for common cases (e.g. binary operations on + * fixed-width types) that will be more performant. + * + */ +template +__device__ void untyped_lto_transform_kernel_shmem( + size_type row_size, + bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + column_device_view_core const* __restrict__ input_cols, + mutable_column_device_view_core const* __restrict__ output_cols, + size_type num_inputs, + size_type num_outputs, + size_type const* __restrict__ input_strides) +{ + using storage_t = element_storage; + + extern __shared__ char shmem[]; + + auto per_thread_bytes = sizeof(storage_t) * num_inputs + sizeof(storage_t) * num_outputs; + auto shmem_iter = shmem + per_thread_bytes * threadIdx.x; + + auto* __restrict__ input_storage = reinterpret_cast(shmem_iter); + shmem_iter += sizeof(storage_t) * num_inputs; + auto* __restrict__ output_storage = reinterpret_cast(shmem_iter); + + auto start = detail::grid_1d::global_thread_id(); + auto stride = detail::grid_1d::grid_stride(); + + for (auto i = start; i < row_size; i += stride) { + if constexpr (!null_aware) { + if (stencil != nullptr && !bit_is_set(stencil, i)) { continue; } + } + + // used only for null-aware + auto active_mask = null_aware ? __ballot_sync(0xFFFF'FFFFU, i < row_size) : 0xFFFF'FFFFU; + + for (size_type c = 0; c < num_inputs; c++) { + load_element(input_cols + c, i * input_strides[c], input_storage + c); + } + + cudf_transform_operation( + user_data, i, input_storage, sizeof(storage_t), output_storage, sizeof(storage_t)); + + for (size_type c = 0; c < num_outputs; c++) { + store_element(output_cols + c, output_storage + c, i, active_mask); + } + } +} + +template +__device__ void untyped_lto_transform_kernel_stack( + size_type row_size, + bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + column_device_view_core const* __restrict__ input_cols, + mutable_column_device_view_core const* __restrict__ output_cols, + size_type num_inputs, + size_type num_outputs, + size_type const* __restrict__ input_strides) +{ + using storage_t = element_storage; + + storage_t storage[max_elements]; + auto* __restrict__ input_storage = storage; + auto* __restrict__ output_storage = storage + num_inputs; + + auto start = detail::grid_1d::global_thread_id(); + auto stride = detail::grid_1d::grid_stride(); + + for (auto i = start; i < row_size; i += stride) { + if constexpr (!null_aware) { + if (stencil != nullptr && !bit_is_set(stencil, i)) { continue; } + } + + // used only for null-aware + auto active_mask = null_aware ? __ballot_sync(0xFFFF'FFFFU, i < row_size) : 0xFFFF'FFFFU; + + for (size_type c = 0; c < num_inputs; c++) { + load_element(input_cols + c, i * input_strides[c], input_storage + c); + } + + cudf_transform_operation( + user_data, i, input_storage, sizeof(storage_t), output_storage, sizeof(storage_t)); + + for (size_type c = 0; c < num_outputs; c++) { + store_element(output_cols + c, output_storage + c, i, active_mask); + } + } +} + +} // namespace cudf + +extern "C" __global__ void cudf_kernel_entry( + cudf::size_type row_size, + cudf::bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + cudf::column_device_view_core const* __restrict__ input_cols, + cudf::mutable_column_device_view_core const* __restrict__ output_cols, + cudf::size_type num_inputs, + cudf::size_type num_outputs, + cudf::size_type const* __restrict__ input_strides) +{ + CUDF_KERNEL_INSTANCE( + row_size, stencil, user_data, input_cols, output_cols, num_inputs, num_outputs, input_strides); +} diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 004b319225ff..5bd76ccdb31a 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -23,6 +23,7 @@ #include +#include #include #include #include @@ -157,7 +158,7 @@ using handle = std::variant< namespace jit_transform { -cudf::kernel instantiate(null_aware is_null_aware, +cudf::kernel instantiate(bool null_aware, bool has_user_data, std::string const& ins, std::string const& outs, @@ -175,13 +176,12 @@ cudf::kernel instantiate(null_aware is_null_aware, : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", - rtcx::reflect_enum("cudf::null_aware", is_null_aware), + rtcx::reflect_bool(null_aware), rtcx::reflect_bool(has_user_data), ins, outs); - return jit::get_udf_kernel( - "transform/jit/kernel.cu", kernel, cuda_source, {"-restrict", "--dopt=on"}); + return jit::get_udf_kernel("transform/jit/kernel.cu", kernel, cuda_source); } void launch(cudf::kernel const& kernel, @@ -332,7 +332,7 @@ auto to_args(std::span inputs, return std::make_tuple(std::move(d_args), std::move(handles)); } -void run(null_aware is_null_aware, +void run(bool null_aware, bool has_user_data, size_type row_size, bitmask_type const* d_stencil, @@ -345,16 +345,10 @@ void run(null_aware is_null_aware, rmm::device_async_resource_ref mr) { auto [in_types, out_types, ptx_in_types, ptx_out_types] = reflect(source_type, inputs, outputs); - auto kernel = instantiate(is_null_aware, - has_user_data, - in_types, - out_types, - ptx_in_types, - ptx_out_types, - udf, - source_type); - auto [cols, handles] = to_args(inputs, outputs, stream, mr); - auto* input_cols = reinterpret_cast(cols.data()); + auto kernel = instantiate( + null_aware, has_user_data, in_types, out_types, ptx_in_types, ptx_out_types, udf, source_type); + auto [cols, handles] = to_args(inputs, outputs, stream, mr); + auto* input_cols = reinterpret_cast(cols.data()); auto* output_cols = reinterpret_cast(input_cols + inputs.size()); return launch(kernel, row_size, d_stencil, user_data, input_cols, output_cols, stream); @@ -503,14 +497,15 @@ auto get_null_transformation(null_aware is_null_aware, return output_may_be_nullable; } -void perform_checks(udf_source_type source_type, +void perform_checks(std::variant source_type, null_aware is_null_aware, std::optional in_row_size, std::span inputs, std::span outputs, std::span const> string_offsets) { - if (source_type == udf_source_type::PTX) { + if (auto* udf_source = std::get_if(&source_type); + udf_source != nullptr && *udf_source == udf_source_type::PTX) { CUDF_EXPECTS(std::none_of(inputs.begin(), inputs.end(), [](auto& in) { @@ -532,6 +527,25 @@ void perform_checks(udf_source_type source_type, CUDF_EXPECTS(is_null_aware == null_aware::NO, "PTX UDFs do not support null-aware transformations", std::invalid_argument); + } else if (std::holds_alternative(source_type)) { + [[maybe_unused]] auto binary_type = std::get(source_type); + CUDF_EXPECTS( + std::none_of(inputs.begin(), + inputs.end(), + [](auto& in) { + return std::visit( + [](auto& c) { + return !is_fixed_width(c.type()) && c.type().id() != type_id::STRING; + }, + in); + }), + "Transforms with LTO binaries only support fixed-width types and strings as inputs", + std::invalid_argument); + CUDF_EXPECTS( + std::none_of( + outputs.begin(), outputs.end(), [](auto& out) { return !is_fixed_width(out.type); }), + "Transforms with LTO binaries only support output of fixed-width types", + std::invalid_argument); } CUDF_EXPECTS(std::none_of(outputs.begin(), @@ -832,7 +846,7 @@ std::unique_ptr
execute_transform(std::string const& udf, auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; - jit_transform::run(is_null_aware, + jit_transform::run(is_null_aware == null_aware::YES, user_data.has_value(), row_size, stencil_has_nulls ? stencil_arg : nullptr, @@ -950,4 +964,294 @@ std::unique_ptr compute_column_jit(table_view const& table, mr); } +type_id as_storage_type(type_id id) +{ + switch (id) { + case type_id::INT8: + case type_id::UINT8: return type_id::UINT8; + case type_id::INT16: + case type_id::UINT16: return type_id::UINT16; + case type_id::INT32: + case type_id::UINT32: + case type_id::FLOAT32: + case type_id::TIMESTAMP_DAYS: + case type_id::DURATION_DAYS: return type_id::UINT32; + case type_id::INT64: + case type_id::UINT64: + case type_id::FLOAT64: + case type_id::TIMESTAMP_SECONDS: + case type_id::TIMESTAMP_MILLISECONDS: + case type_id::TIMESTAMP_MICROSECONDS: + case type_id::TIMESTAMP_NANOSECONDS: + case type_id::DURATION_SECONDS: + case type_id::DURATION_MILLISECONDS: + case type_id::DURATION_MICROSECONDS: + case type_id::DURATION_NANOSECONDS: return type_id::UINT64; + default: return id; + } +} + +std::string_view as_tag(type_id id) +{ + switch (id) { + case type_id::BOOL8: return "bool8"; + case type_id::INT8: return "int8"; + case type_id::INT16: return "int16"; + case type_id::INT32: return "int32"; + case type_id::INT64: return "int64"; + case type_id::UINT8: return "uint8"; + case type_id::UINT16: return "uint16"; + case type_id::UINT32: return "uint32"; + case type_id::UINT64: return "uint64"; + case type_id::FLOAT32: return "float32"; + case type_id::FLOAT64: return "float64"; + case type_id::STRING: return "string_view"; + case type_id::DECIMAL32: return "decimal32"; + case type_id::DECIMAL64: return "decimal64"; + case type_id::DECIMAL128: return "decimal128"; + case type_id::DURATION_DAYS: return "duration_D"; + case type_id::DURATION_SECONDS: return "duration_s"; + case type_id::DURATION_MILLISECONDS: return "duration_ms"; + case type_id::DURATION_MICROSECONDS: return "duration_us"; + case type_id::DURATION_NANOSECONDS: return "duration_ns"; + case type_id::TIMESTAMP_DAYS: return "timestamp_D"; + case type_id::TIMESTAMP_SECONDS: return "timestamp_s"; + case type_id::TIMESTAMP_MILLISECONDS: return "timestamp_ms"; + case type_id::TIMESTAMP_MICROSECONDS: return "timestamp_us"; + case type_id::TIMESTAMP_NANOSECONDS: return "timestamp_ns"; + default: CUDF_FAIL("Unsupported type for JIT dispatch", std::invalid_argument); + } +} + +struct element_size_fn { + template + int32_t operator()() + { + return sizeof(T); + } +}; + +int32_t element_size(type_id id) { return type_dispatcher(data_type{id}, element_size_fn{}); } + +std::optional> dispatch_unop_lto_kernel( + bool null_aware, + std::span inputs, + std::span outputs) +{ + auto input_type = as_storage_type(std::visit([](auto& c) { return c.type().id(); }, inputs[0])); + + for (size_t i = 0; i < std::size(cudf_fragments::unop_lto_kernel_FILE_INDEX); i++) { + auto FILE_INDEX = cudf_fragments::unop_lto_kernel_FILE_INDEX[i]; + auto NULL_AWARE = cudf_fragments::unop_lto_kernel_NULL_AWARE[i]; + auto TYPE = cudf_fragments::unop_lto_kernel_TYPE[i]; + if (as_tag(input_type) == TYPE && null_aware == NULL_AWARE) { + auto range = cudf_fragments::file_ranges[FILE_INDEX]; + return cudf_fragments::files.subspan(range[0], range[1]); + } + } + + return std::nullopt; +} + +std::optional> dispatch_untyped_lto_transform_kernel_stack( + bool null_aware, + std::span inputs, + std::span outputs) +{ + int32_t max_element_size = 0; + for (auto& input : inputs) { + auto size = element_size(std::visit([](auto& c) { return c.type().id(); }, input)); + max_element_size = std::max(max_element_size, size); + } + int32_t elements = inputs.size() + outputs.size(); + + for (size_t i = 0; i < std::size(cudf_fragments::untyped_lto_transform_kernel_stack_FILE_INDEX); + i++) { + auto FILE_INDEX = cudf_fragments::untyped_lto_transform_kernel_stack_FILE_INDEX[i]; + auto NULL_AWARE = cudf_fragments::untyped_lto_transform_kernel_stack_NULL_AWARE[i]; + auto MAX_ELEMENT_SIZE = cudf_fragments::untyped_lto_transform_kernel_stack_MAX_ELEMENT_SIZE[i]; + auto MAX_ELEMENTS = cudf_fragments::untyped_lto_transform_kernel_stack_MAX_ELEMENTS[i]; + if (null_aware == NULL_AWARE && max_element_size <= MAX_ELEMENT_SIZE && + elements <= MAX_ELEMENTS) { + auto range = cudf_fragments::file_ranges[FILE_INDEX]; + return cudf_fragments::files.subspan(range[0], range[1]); + } + } + + return std::nullopt; +} + +std::optional, size_t>> +dispatch_untyped_lto_transform_kernel_shmem(bool null_aware, + std::span inputs, + std::span outputs) +{ + int32_t max_element_size = 0; + for (auto& input : inputs) { + auto size = element_size(std::visit([](auto& c) { return c.type().id(); }, input)); + max_element_size = std::max(max_element_size, size); + } + + auto nullable_max_element_size = max_element_size * 2; // type + boolean is_null struct + + auto per_thread_shmem = + (null_aware ? nullable_max_element_size : max_element_size) * (inputs.size() + outputs.size()); + + for (size_t i = 0; i < std::size(cudf_fragments::untyped_lto_transform_kernel_shmem_FILE_INDEX); + i++) { + auto FILE_INDEX = cudf_fragments::untyped_lto_transform_kernel_shmem_FILE_INDEX[i]; + auto NULL_AWARE = cudf_fragments::untyped_lto_transform_kernel_shmem_NULL_AWARE[i]; + auto MAX_ELEMENT_SIZE = cudf_fragments::untyped_lto_transform_kernel_shmem_MAX_ELEMENT_SIZE[i]; + if (null_aware == NULL_AWARE && max_element_size <= MAX_ELEMENT_SIZE) { + auto range = cudf_fragments::file_ranges[FILE_INDEX]; + return std::make_tuple(cudf_fragments::files.subspan(range[0], range[1]), + static_cast(per_thread_shmem)); + } + } + + return std::nullopt; +} + +rtcx::kernel_occupancy_config configure_shmem(cudf::kernel const& transform_kernel, + size_t dynamic_smem_per_thread) +{ + size_t static_shmem_per_thread = 0; + + auto shmem_for_block = [&](int block_size) -> size_t { + return (static_shmem_per_thread + dynamic_smem_per_thread) * block_size; + }; + + int min_grid_size; + int block_size; + + CUDF_CUDA_TRY(cudaOccupancyMaxPotentialBlockSizeVariableSMem( + &min_grid_size, &block_size, transform_kernel.get().get(), shmem_for_block, 0)); + + return {.min_grid_size = static_cast(min_grid_size), + .block_size = static_cast(block_size)}; +} + +// Dispatches to the appropriate LTO kernel based on the number of inputs and outputs, their types, +// and nullability. +std::tuple, size_t> dispatch_lto_kernel( + bool null_aware, + std::span inputs, + std::span outputs) +{ + if (inputs.size() == 1 && outputs.size() == 1) { + auto input0_type = + as_storage_type(std::visit([](auto& c) { return c.type().id(); }, inputs[0])); + auto output_type = as_storage_type(outputs[0].type.id()); + + if (input0_type == output_type && is_fixed_width(data_type{input0_type})) { + if (auto kernel = dispatch_unop_lto_kernel(null_aware, inputs, outputs)) { + return {*kernel, 0}; + } + } + } + + if (auto kernel = dispatch_untyped_lto_transform_kernel_stack(null_aware, inputs, outputs)) { + return {*kernel, 0}; + } + + if (auto kernel = dispatch_untyped_lto_transform_kernel_shmem(null_aware, inputs, outputs)) { + return *kernel; + } + + CUDF_FAIL("No suitable LTO kernel found for the given transform parameters", + std::invalid_argument); +} + +rtcx::binary_type as_rtcx_binary_type(lto_binary_type type) +{ + switch (type) { + case lto_binary_type::LTO_IR: return rtcx::binary_type::LTO_IR; + case lto_binary_type::FATBIN: return rtcx::binary_type::FATBIN; + default: + CUDF_FAIL( + std::format("Unrecognized LTO binary type {} for LTO transform", static_cast(type)), + std::invalid_argument); + } +} + +std::unique_ptr
transform_lto(std::span inputs, + std::span udf, + lto_binary_type binary_type, + std::span outputs, + void* user_data, + null_aware is_null_aware, + std::optional in_row_size, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + perform_checks(binary_type, is_null_aware, in_row_size, inputs, outputs, {}); + auto row_size = in_row_size.has_value() ? *in_row_size : jit::get_projection_size(inputs); + auto output_may_be_nullable = get_null_transformation(is_null_aware, inputs, outputs); + + auto [output_columns, stencil] = + make_outputs(is_null_aware, row_size, inputs, outputs, output_may_be_nullable, {}, stream, mr); + auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; + auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; + + std::vector input_strides; + + for (auto& input : inputs) { + if (std::holds_alternative(input)) { + input_strides.push_back(0); + } else { + input_strides.push_back(1); + } + } + + rmm::device_uvector d_input_strides(input_strides.size(), stream, mr); + CUDF_CUDA_TRY(cudaMemcpyAsync(d_input_strides.data(), + input_strides.data(), + input_strides.size() * sizeof(cudf::size_type), + cudaMemcpyHostToDevice, + stream.value())); + + auto [kernel_fatbin, shmem_requirement_per_thread] = + dispatch_lto_kernel(is_null_aware == null_aware::YES, inputs, outputs); + + rtcx::memory_fragment fragments[] = { + {.data = kernel_fatbin, .type = rtcx::binary_type::FATBIN, .name = "kernel"}, + {.data = udf, .type = as_rtcx_binary_type(binary_type), .name = "udf"}}; + + auto kernel = get_lto_linked_kernel("transform_lto_kernel", {}, fragments); + auto [cols, handles] = jit_transform::to_args(inputs, output_columns, stream, mr); + cudf::size_type num_inputs = static_cast(inputs.size()); + cudf::size_type num_outputs = static_cast(outputs.size()); + cudf::size_type num_rows = row_size; + cudf::bitmask_type* p_stencil = stencil_has_nulls ? stencil_arg : nullptr; + auto* input_cols = reinterpret_cast(cols.data()); + auto* output_cols = + reinterpret_cast(input_cols + inputs.size()); + cudf::size_type* p_input_strides = d_input_strides.data(); + + rtcx::kernel_occupancy_config cfg; + + if (shmem_requirement_per_thread > 0) { + cfg = configure_shmem(kernel, shmem_requirement_per_thread); + } else { + cfg = kernel.max_occupancy_config(0, 0); + } + + void* args[] = {&num_rows, + &user_data, + &p_stencil, + &input_cols, + &output_cols, + &num_inputs, + &num_outputs, + &p_input_strides}; + + auto per_block_shmem = shmem_requirement_per_thread * cfg.block_size; + + kernel.launch({cfg.min_grid_size}, {cfg.block_size}, per_block_shmem, stream, args); + + auto finalized = finalize_outputs(is_null_aware, row_size, std::move(output_columns), stream, mr); + return std::make_unique
(std::move(finalized)); +} + } // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 82290dbe5892..79205e5ad75e 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -683,8 +683,30 @@ ConfigureTest(AST_TEST ast/transform_tests.cpp ast/ast_tree_tests.cpp) # ################################################################################################## # * compute columns tests -# ------------------------------------------------------------------------------------- -ConfigureTest(COMPUTE_COLUMNS_TEST transform/compute_columns_test.cpp) +# ------------------------------------------------------------------------------------- +add_embed(cudf_test_fragments) + +add_fragment( + cudf_test_fragments FRAGMENT transform_profit_operator SOURCE transform/fragments/profit.cu +) + +add_fragment( + cudf_test_fragments FRAGMENT transform_invsqrt_operator SOURCE + transform/fragments/invsqrt.cu +) + +add_fragment( + cudf_test_fragments FRAGMENT transform_tpcds_q7 SOURCE transform/fragments/tpcds_q7.cu +) + +embed(cudf_test_fragments COMPRESSION none) + +ConfigureTest( + TRANSFORM_LTO_TEST transform/transform_lto_test.cpp + ${cudf_test_fragments_SOURCE_DIR}/cudf_test_fragments.s +) +add_dependencies(TRANSFORM_LTO_TEST cudf_test_fragments) +target_include_directories(TRANSFORM_LTO_TEST PRIVATE ${cudf_test_fragments_SOURCE_DIR}) # ################################################################################################## # * lists tests ---------------------------------------------------------------------------------- diff --git a/cpp/tests/transform/compute_columns_profit_operator.cu b/cpp/tests/transform/compute_columns_profit_operator.cu deleted file mode 100644 index f05b7b2cb091..000000000000 --- a/cpp/tests/transform/compute_columns_profit_operator.cu +++ /dev/null @@ -1,70 +0,0 @@ - -#include - -#include - -#include - -__device__ inline bool strcontains(cudf::string_view text, cudf::string_view token) -{ - return text.find(token) != cudf::string_view::npos; -} - -template -__device__ T load(void const* inputs, int input_stride, int arg) -{ - auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); - return *p; -} - -template -__device__ void store(void* outputs, int output_stride, int arg, T value) -{ - auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); - *p = value; -} - -extern "C" __device__ int operation( - void*, long int, void const* inputs, int input_stride, void* outputs, int output_stride) -{ - // Input schema: - // 0: price(double), - // 1: qty(int32), - // 2: discount(float), - // 3: tax_rate(float), - // 4: ship_mode(string_view), - // 5: comment(string_view) - auto price = load(inputs, input_stride, 0); - auto qty = load(inputs, input_stride, 1); - auto discount = load(inputs, input_stride, 2); - auto tax_rate = load(inputs, input_stride, 3); - auto ship = load(inputs, input_stride, 4); - auto comment = load(inputs, input_stride, 5); - - // Parameters: - // 6: priority_threshold(double), - // 7: require_expedited(bool), - // 8: promo_discount_threshold(float) - auto priority_threshold = load(inputs, input_stride, 6); - auto require_expedited = load(inputs, input_stride, 7); - auto promo_discount_threshold = load(inputs, input_stride, 8); - - auto net = price * qty * (1.0 - discount); - auto gross = net * (1.0 + tax_rate); - - auto expedited = strcontains(ship, cudf::string_view{"expedited", 9}); - auto priority = (gross > priority_threshold) && (!require_expedited || expedited); - - auto has_promo_keyword = strcontains(comment, cudf::string_view{"promo", 5}) || - strcontains(comment, cudf::string_view{"sale", 4}) || - strcontains(comment, cudf::string_view{"coupon", 6}); - - auto is_promo = has_promo_keyword || (discount >= promo_discount_threshold); - - store(outputs, output_stride, 0, net); - store(outputs, output_stride, 1, gross); - store(outputs, output_stride, 2, priority); - store(outputs, output_stride, 3, is_promo); - - return 0; -} diff --git a/cpp/tests/transform/compute_columns_test.cpp b/cpp/tests/transform/compute_columns_test.cpp deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/cpp/tests/transform/fragments/invsqrt.cu b/cpp/tests/transform/fragments/invsqrt.cu new file mode 100644 index 000000000000..63b06f7920ea --- /dev/null +++ b/cpp/tests/transform/fragments/invsqrt.cu @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +template +__device__ T load(void const* inputs, int input_stride, int arg) +{ + auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); + return *p; +} + +template +__device__ void store(void* outputs, int output_stride, int arg, T value) +{ + auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); + *p = value; +} + +extern "C" __device__ int cudf_transform_operation( + void*, long int, void const* inputs, int input_stride, void* outputs, int output_stride) +{ + auto input = load(inputs, input_stride, 0); + auto result = 1.0f / sqrtf(input); + store(outputs, output_stride, 0, result); + return 0; +} diff --git a/cpp/tests/transform/fragments/profit.cu b/cpp/tests/transform/fragments/profit.cu new file mode 100644 index 000000000000..4dc0a8ff3051 --- /dev/null +++ b/cpp/tests/transform/fragments/profit.cu @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +template +__device__ T load(void const* inputs, int input_stride, int arg) +{ + auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); + return *p; +} + +template +__device__ void store(void* outputs, int output_stride, int arg, T value) +{ + auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); + *p = value; +} + +extern "C" __device__ int cudf_transform_operation( + void*, long int, void const* inputs, int input_stride, void* outputs, int output_stride) +{ + // Input schema: + // 0: extended_price(double), + // 1: discount(float), + // 2: tax(float), + // 3: ship_date(int32 YYYYMMDD) + auto extended_price = load(inputs, input_stride, 0); + auto discount = load(inputs, input_stride, 1); + auto tax = load(inputs, input_stride, 2); + auto ship_date = load(inputs, input_stride, 3); + + // Parameters: + // 4: ship_date_cutoff(int32 YYYYMMDD) + auto ship_date_cutoff = load(inputs, input_stride, 4); + + auto base_price = extended_price; + auto disc_price = extended_price * (1.0 - discount); + auto charge = disc_price * (1.0 + tax); + + auto before_cutoff = ship_date <= ship_date_cutoff; + + store(outputs, output_stride, 0, base_price); + store(outputs, output_stride, 1, charge); + store(outputs, output_stride, 2, disc_price); + store(outputs, output_stride, 3, before_cutoff); + + return 0; +} diff --git a/cpp/tests/transform/fragments/tpcds_q7.cu b/cpp/tests/transform/fragments/tpcds_q7.cu new file mode 100644 index 000000000000..7ef60f98d3d9 --- /dev/null +++ b/cpp/tests/transform/fragments/tpcds_q7.cu @@ -0,0 +1,56 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + + +template +__device__ T load(void const* inputs, int input_stride, int arg) +{ + auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); + return *p; +} + +template +__device__ void store(void* outputs, int output_stride, int arg, T value) +{ + auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); + *p = value; +} + +/** + * TPC-DS Q7: "Report the profit of each returned item, and the total profit for all returned items, + * for a given date range." + * + */ +extern "C" __device__ int cudf_transform_operation( + void*, long int, void const* inputs, int input_stride, void* outputs, int output_stride) +{ + // Input schema: + // 0: store_sales(double), + // 1: catalog_sales(double), + // 2: web_sales(double), + // 3: store_returns(double), + // 4: catalog_returns(double), + // 5: web_returns(double), + // 6: profit(double), + // 7: profit_loss(double) + auto store_sales = load(inputs, input_stride, 0); + auto catalog_sales = load(inputs, input_stride, 1); + auto web_sales = load(inputs, input_stride, 2); + auto store_returns = load(inputs, input_stride, 3); + auto catalog_returns = load(inputs, input_stride, 4); + auto web_returns = load(inputs, input_stride, 5); + auto profit = load(inputs, input_stride, 6); + auto profit_loss = load(inputs, input_stride, 7); + + double sales = store_sales + catalog_sales + web_sales; + double returns = store_returns + catalog_returns + web_returns; + double net_sales = sales - returns; + double net_profit = profit - profit_loss; + + store(outputs, output_stride, 0, net_profit); + store(outputs, output_stride, 1, net_sales); + return 0; +} diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp new file mode 100644 index 000000000000..f857e05ffe93 --- /dev/null +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +struct TransformLTOTest : public cudf::test::BaseFixture {}; + +template +using column_wrapper = cudf::test::fixed_width_column_wrapper; + +TEST_F(TransformLTOTest, ComputeTpchLineitem) +{ + column_wrapper extended_price{{21168.23, 45983.16, 13309.60, 28955.64}}; + column_wrapper discount{{0.04f, 0.06f, 0.07f, 0.02f}}; + column_wrapper tax{{0.02f, 0.08f, 0.04f, 0.06f}}; + column_wrapper ship_date{{19980901, 19980902, 19980903, 19960115}}; + + column_wrapper ship_date_cutoff{{19980902}}; + + std::vector inputs{ + extended_price, discount, tax, ship_date, cudf::scalar_column_view{ship_date_cutoff}}; + + std::vector outputs{ + {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::ALL_VALID}, + {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::ALL_VALID}, + {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::ALL_VALID}, + {cudf::data_type{cudf::type_id::BOOL8}, cudf::output_nullability::ALL_VALID}}; + + auto const range = + cudf_test_fragments::file_ranges[cudf_test_fragments::transform_profit_operator]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::transform_lto(inputs, + udf, + cudf::lto_binary_type::FATBIN, + outputs, + nullptr, + cudf::null_aware::NO, + std::nullopt, + cudf::test::get_default_stream()); + + column_wrapper expected_base_price{{21168.23, 45983.16, 13309.60, 28955.64}}; + column_wrapper expected_disc_price{{21168.23 * (1.0 - static_cast(0.04f)), + 45983.16 * (1.0 - static_cast(0.06f)), + 13309.60 * (1.0 - static_cast(0.07f)), + 28955.64 * (1.0 - static_cast(0.02f))}}; + column_wrapper expected_charge{ + {(21168.23 * (1.0 - static_cast(0.04f))) * (1.0 + static_cast(0.02f)), + (45983.16 * (1.0 - static_cast(0.06f))) * (1.0 + static_cast(0.08f)), + (13309.60 * (1.0 - static_cast(0.07f))) * (1.0 + static_cast(0.04f)), + (28955.64 * (1.0 - static_cast(0.02f))) * (1.0 + static_cast(0.06f))}}; + column_wrapper expected_before_cutoff{{true, true, false, true}}; + + cudf::test::detail::expect_columns_equivalent( + result->view().column(1), expected_charge, cudf::test::debug_output_level::FIRST_ERROR, 64); + cudf::test::detail::expect_columns_equivalent( + result->view().column(2), expected_disc_price, cudf::test::debug_output_level::FIRST_ERROR, 64); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view().column(3), expected_before_cutoff); +} + +TEST_F(TransformLTOTest, InvSqrtOperator) +{ + column_wrapper input{{1.0f, 4.0f, 9.0f, 16.0f}}; + + std::vector inputs{input}; + + std::vector outputs{ + {cudf::data_type{cudf::type_id::FLOAT32}, cudf::output_nullability::ALL_VALID}}; + + auto const range = + cudf_test_fragments::file_ranges[cudf_test_fragments::transform_invsqrt_operator]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::transform_lto(inputs, + udf, + cudf::lto_binary_type::FATBIN, + outputs, + nullptr, + cudf::null_aware::NO, + std::nullopt, + cudf::test::get_default_stream()); + + column_wrapper expected{{1.0f, 0.5f, 0.33333334f, 0.25f}}; + + cudf::test::detail::expect_columns_equivalent( + result->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, 64); +} + +TEST_F(TransformLTOTest, TPCDS_Q7) +{ + column_wrapper store_sales{{100.0, 20.0, 0.0, 45.5}}; + column_wrapper catalog_sales{{10.0, 5.0, 3.0, 4.5}}; + column_wrapper web_sales{{1.0, 0.0, 7.0, 0.5}}; + column_wrapper store_returns{{4.0, 1.0, 0.0, 5.0}}; + column_wrapper catalog_returns{{0.5, 0.0, 1.0, 0.5}}; + column_wrapper web_returns{{0.5, 0.0, 2.0, 0.0}}; + column_wrapper profit{{30.0, 5.0, -2.0, 8.0}}; + column_wrapper profit_loss{{3.0, 1.0, 4.0, 0.5}}; + + std::vector inputs{store_sales, + catalog_sales, + web_sales, + store_returns, + catalog_returns, + web_returns, + profit, + profit_loss}; + + std::vector outputs{ + {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::ALL_VALID}, + {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::ALL_VALID}}; + + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::transform_tpcds_q7]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::transform_lto(inputs, + udf, + cudf::lto_binary_type::FATBIN, + outputs, + nullptr, + cudf::null_aware::NO, + std::nullopt, + cudf::test::get_default_stream()); + + column_wrapper expected_net_profit{{27.0, 4.0, -6.0, 7.5}}; + column_wrapper expected_net_sales{{106.0, 24.0, 7.0, 45.0}}; + + cudf::test::detail::expect_columns_equivalent( + result->view().column(0), expected_net_profit, cudf::test::debug_output_level::FIRST_ERROR, 64); + cudf::test::detail::expect_columns_equivalent( + result->view().column(1), expected_net_sales, cudf::test::debug_output_level::FIRST_ERROR, 64); +} diff --git a/todo.md b/todo.md deleted file mode 100644 index 0af985552e75..000000000000 --- a/todo.md +++ /dev/null @@ -1,10 +0,0 @@ -Add POC: - - - [x] Successfully compile existing CUDF JIT kernels whilst reducing the binary size of the JIT kernels - - [x] Precompiled and JIT-ed compute_columns_kernel; reduce register pressure and ...: - - [ ] Make AST codegen to use the pre-compiled operators and element_storage types - - [ ] Manage the AOT-compiled artifacts with keys for the system - - [x] Functions to compile the artifacts and embed them - - [ ] Add sample pre-compiled UDF fragments - - [ ] Build matrix tables that can be used for lookup/testing to check existence of fragments - [ ] Store mangled identifier names of the fragments - From 0ea56562b51db66e4e02b0473ebfaaeaf1cc2db6 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 12 May 2026 21:26:10 +0000 Subject: [PATCH 169/254] Implement LTO (Link Time Optimization) for transform operations and update related benchmarks --- cpp/CMakeLists.txt | 6 +- cpp/benchmarks/CMakeLists.txt | 9 +- cpp/benchmarks/ndsh/fragments/q09.cu | 30 ++++++ cpp/benchmarks/ndsh/q09.cpp | 46 ++++++++- cpp/include/cudf/transform.hpp | 4 +- cpp/src/transform/jit/untyped_lto_kernel.cu | 2 +- .../transform/jit/untyped_lto_kernel_shmem.cu | 98 +++++++++++++++++++ .../transform/jit/untyped_lto_kernel_stack.cu | 85 ++++++++++++++++ cpp/src/transform/transform.cu | 26 +++-- cpp/tests/CMakeLists.txt | 9 +- cpp/tests/transform/fragments/invsqrt.cu | 2 - cpp/tests/transform/fragments/tpcds_q7.cu | 1 - 12 files changed, 292 insertions(+), 26 deletions(-) create mode 100644 cpp/benchmarks/ndsh/fragments/q09.cu create mode 100644 cpp/src/transform/jit/untyped_lto_kernel_shmem.cu create mode 100644 cpp/src/transform/jit/untyped_lto_kernel_stack.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e7fb147f8134..39d9ee003598 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -349,13 +349,13 @@ macro(add_fragment) endif() if(ARG_KERNEL_INSTANCE) - set(SPECIALIZATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/specializations/${ARG_FRAGMENT}") + set(INSTANTIATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/${TARGET}/instantiations/${ARG_FRAGMENT}") file( GENERATE - OUTPUT "${SPECIALIZATION_DIR}/cudf/detail/kernel-instance.hpp" + OUTPUT "${INSTANTIATION_DIR}/cudf/detail/kernel-instance.hpp" CONTENT "#pragma once\n#define CUDF_KERNEL_INSTANCE ${ARG_KERNEL_INSTANCE}" ) - target_include_directories(${OBJECT_ID} PRIVATE ${SPECIALIZATION_DIR}) + target_include_directories(${OBJECT_ID} PRIVATE ${INSTANTIATION_DIR}) endif() target_compile_definitions(${OBJECT_ID} PRIVATE CUDF_DISABLE_EXPORTS ${ARG_DEFINITIONS}) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 066716e85f25..a120f33ccfa9 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -121,10 +121,17 @@ ConfigureNVBench(TRANSPOSE_NVBENCH transpose/transpose.cpp) # ################################################################################################## # * nds-h benchmark -------------------------------------------------------------------------------- + +add_embed(cudf_benchmark_fragments) +add_fragment(cudf_benchmark_fragments FRAGMENT ndsh_q09 SOURCE ndsh/fragments/q09.cu) +embed(cudf_benchmark_fragments COMPRESSION none) + ConfigureNVBench(NDSH_Q01_NVBENCH ndsh/q01.cpp ndsh/utilities.cpp) ConfigureNVBench(NDSH_Q05_NVBENCH ndsh/q05.cpp ndsh/utilities.cpp) ConfigureNVBench(NDSH_Q06_NVBENCH ndsh/q06.cpp ndsh/utilities.cpp) -ConfigureNVBench(NDSH_Q09_NVBENCH ndsh/q09.cpp ndsh/utilities.cpp) +ConfigureNVBench(NDSH_Q09_NVBENCH ndsh/q09.cpp ndsh/utilities.cpp ${cudf_benchmark_fragments_SOURCE_DIR}/cudf_benchmark_fragments.s) +add_dependencies(NDSH_Q09_NVBENCH cudf_benchmark_fragments) +target_include_directories(NDSH_Q09_NVBENCH PRIVATE ${cudf_benchmark_fragments_SOURCE_DIR}) ConfigureNVBench(NDSH_Q10_NVBENCH ndsh/q10.cpp ndsh/utilities.cpp) # ################################################################################################## diff --git a/cpp/benchmarks/ndsh/fragments/q09.cu b/cpp/benchmarks/ndsh/fragments/q09.cu new file mode 100644 index 000000000000..e2afad822e99 --- /dev/null +++ b/cpp/benchmarks/ndsh/fragments/q09.cu @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +template +__device__ T load(void const* inputs, int input_stride, int arg) +{ + auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); + return *p; +} + +template +__device__ void store(void* outputs, int output_stride, int arg, T value) +{ + auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); + *p = value; +} + +extern "C" __device__ int cudf_transform_operation( + void*, long int, void const* inputs, int input_stride, void* outputs, int output_stride) +{ + auto discount = load(inputs, input_stride, 0); + auto extended_price = load(inputs, input_stride, 1); + auto supply_cost = load(inputs, input_stride, 2); + auto quantity = load(inputs, input_stride, 3); + auto amount = extended_price * (1 - discount) - supply_cost * quantity; + store(outputs, output_stride, 0, amount); + return 0; +} diff --git a/cpp/benchmarks/ndsh/q09.cpp b/cpp/benchmarks/ndsh/q09.cpp index f94718ec5fa4..182fd2b300b5 100644 --- a/cpp/benchmarks/ndsh/q09.cpp +++ b/cpp/benchmarks/ndsh/q09.cpp @@ -17,9 +17,10 @@ #include #include +#include #include -enum class engine_type : int32_t { BINARYOP = 0, AST = 1, TRANSFORM = 2 }; +enum class engine_type : int32_t { BINARYOP = 0, AST = 1, TRANSFORM = 2, TRANSFORM_LTO = 3 }; engine_type engine_from_string(std::string const& str) { @@ -29,6 +30,8 @@ engine_type engine_from_string(std::string const& str) return engine_type::AST; } else if (str == "transform") { return engine_type::TRANSFORM; + } else if (str == "transform_lto") { + return engine_type::TRANSFORM_LTO; } else { CUDF_FAIL("unrecognized engine enum: " + str); } @@ -197,6 +200,38 @@ struct q9_data { return cudf::compute_column(table, result, stream, mr); } +[[nodiscard]] std::unique_ptr compute_amount_transform_lto( + cudf::column_view const& discount, + cudf::column_view const& extendedprice, + cudf::column_view const& supplycost, + cudf::column_view const& quantity, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) +{ + CUDF_BENCHMARK_RANGE(); + + cudf::transform_input inputs[] = {discount, extendedprice, supplycost, quantity}; + + cudf::transform_output outputs[] = { + {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::PRESERVE}}; + + auto const range = cudf_benchmark_fragments::file_ranges[cudf_benchmark_fragments::ndsh_q09]; + std::span udf{cudf_benchmark_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::transform_lto(inputs, + udf, + cudf::lto_binary_type::FATBIN, + outputs, + nullptr, + cudf::null_aware::NO, + std::nullopt, + stream, + mr); + + auto table = result->release(); + return std::move(table.front()); +} + [[nodiscard]] std::unique_ptr compute_amount( cudf::column_view const& discount, cudf::column_view const& extendedprice, @@ -213,6 +248,9 @@ struct q9_data { return compute_amount_ast(discount, extendedprice, supplycost, quantity, stream, mr); case engine_type::TRANSFORM: return compute_amount_transform(discount, extendedprice, supplycost, quantity, stream, mr); + case engine_type::TRANSFORM_LTO: + return compute_amount_transform_lto( + discount, extendedprice, supplycost, quantity, stream, mr); default: CUDF_UNREACHABLE("invalid engine_type enum"); } } @@ -376,14 +414,14 @@ void ndsh_q9_amount(nvbench::state& state) NVBENCH_BENCH(ndsh_q9) .set_name("ndsh_q9") .add_float64_axis("scale_factor", {0.01, 0.1, 1}) - .add_string_axis("engine", {"binaryop", "ast", "transform"}); + .add_string_axis("engine", {"binaryop", "ast", "transform", "transform_lto"}); NVBENCH_BENCH(ndsh_q9_noio) .set_name("ndsh_q9_noio") .add_float64_axis("scale_factor", {0.01, 0.1, 1}) - .add_string_axis("engine", {"binaryop", "ast", "transform"}); + .add_string_axis("engine", {"binaryop", "ast", "transform", "transform_lto"}); NVBENCH_BENCH(ndsh_q9_amount) .set_name("ndsh_q9_amount") .add_float64_axis("scale_factor", {0.01, 0.1, 1}) - .add_string_axis("engine", {"binaryop", "ast", "transform"}); + .add_string_axis("engine", {"binaryop", "ast", "transform", "transform_lto"}); diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index f8007deacb37..6657644dec0d 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -227,8 +227,8 @@ std::unique_ptr
transform_lto( lto_binary_type binary_type, std::span outputs, void* user_data, - null_aware is_null_aware = null_aware::NO, - std::optional row_size = std::nullopt, + null_aware is_null_aware, + std::optional row_size, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); diff --git a/cpp/src/transform/jit/untyped_lto_kernel.cu b/cpp/src/transform/jit/untyped_lto_kernel.cu index 60fe72758289..739dd88c6562 100644 --- a/cpp/src/transform/jit/untyped_lto_kernel.cu +++ b/cpp/src/transform/jit/untyped_lto_kernel.cu @@ -50,7 +50,7 @@ __device__ void untyped_lto_transform_kernel_shmem( extern __shared__ char shmem[]; - auto per_thread_bytes = sizeof(storage_t) * num_inputs + sizeof(storage_t) * num_outputs; + auto per_thread_bytes = sizeof(storage_t) * (num_inputs + num_outputs); auto shmem_iter = shmem + per_thread_bytes * threadIdx.x; auto* __restrict__ input_storage = reinterpret_cast(shmem_iter); diff --git a/cpp/src/transform/jit/untyped_lto_kernel_shmem.cu b/cpp/src/transform/jit/untyped_lto_kernel_shmem.cu new file mode 100644 index 000000000000..c14528159d49 --- /dev/null +++ b/cpp/src/transform/jit/untyped_lto_kernel_shmem.cu @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "jit/element.cuh" +#include "jit/element_storage.cuh" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before + // it. Each UDF will have a different operation-udf.hpp generated for it, so we + // need to put this pragma before including it to avoid PCH mismatch. + +// clang-format off +#include +// clang-format on + +namespace cudf { + +/** + * @brief An untyped Generic Transform kernel. This is a catch-all kernel that can be used for any + * transform operation, but is not expected to be the most performant. The intent is that this + * kernel can be used for any operator and type combination with minimal specialization, but that + * more specialized kernels can be generated for common cases (e.g. binary operations on + * fixed-width types) that will be more performant. + * + */ +template +__device__ void untyped_lto_transform_kernel_shmem( + size_type row_size, + bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + column_device_view_core const* __restrict__ input_cols, + mutable_column_device_view_core const* __restrict__ output_cols, + size_type num_inputs, + size_type num_outputs, + size_type const* __restrict__ input_strides) +{ + using storage_t = element_storage; + + extern __shared__ char shmem[]; + + auto per_thread_bytes = sizeof(storage_t) * (num_inputs + num_outputs); + auto shmem_iter = shmem + per_thread_bytes * threadIdx.x; + + auto* __restrict__ input_storage = reinterpret_cast(shmem_iter); + shmem_iter += sizeof(storage_t) * num_inputs; + auto* __restrict__ output_storage = reinterpret_cast(shmem_iter); + + auto start = detail::grid_1d::global_thread_id(); + auto stride = detail::grid_1d::grid_stride(); + + for (auto i = start; i < row_size; i += stride) { + if constexpr (!null_aware) { + if (stencil != nullptr && !bit_is_set(stencil, i)) { continue; } + } + + // used only for null-aware + auto active_mask = null_aware ? __ballot_sync(0xFFFF'FFFFU, i < row_size) : 0xFFFF'FFFFU; + + for (size_type c = 0; c < num_inputs; c++) { + load_element(input_cols + c, i * input_strides[c], input_storage + c); + } + + cudf_transform_operation( + user_data, i, input_storage, sizeof(storage_t), output_storage, sizeof(storage_t)); + + for (size_type c = 0; c < num_outputs; c++) { + store_element(output_cols + c, output_storage + c, i, active_mask); + } + } +} + +} // namespace cudf + +extern "C" __launch_bounds__(256) __global__ + void cudf_kernel_entry(cudf::size_type row_size, + cudf::bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + cudf::column_device_view_core const* __restrict__ input_cols, + cudf::mutable_column_device_view_core const* __restrict__ output_cols, + cudf::size_type num_inputs, + cudf::size_type num_outputs, + cudf::size_type const* __restrict__ input_strides) +{ + CUDF_KERNEL_INSTANCE( + row_size, stencil, user_data, input_cols, output_cols, num_inputs, num_outputs, input_strides); +} diff --git a/cpp/src/transform/jit/untyped_lto_kernel_stack.cu b/cpp/src/transform/jit/untyped_lto_kernel_stack.cu new file mode 100644 index 000000000000..245af24c1e57 --- /dev/null +++ b/cpp/src/transform/jit/untyped_lto_kernel_stack.cu @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "jit/element.cuh" +#include "jit/element_storage.cuh" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before + // it. Each UDF will have a different operation-udf.hpp generated for it, so we + // need to put this pragma before including it to avoid PCH mismatch. + +// clang-format off +#include +// clang-format on + +namespace cudf { + +template +__device__ void untyped_lto_transform_kernel_stack( + size_type row_size, + bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + column_device_view_core const* __restrict__ input_cols, + mutable_column_device_view_core const* __restrict__ output_cols, + size_type num_inputs, + size_type num_outputs, + size_type const* __restrict__ input_strides) +{ + using storage_t = element_storage; + + storage_t storage[max_elements]; + auto* __restrict__ input_storage = storage; + auto* __restrict__ output_storage = storage + num_inputs; + + auto start = detail::grid_1d::global_thread_id(); + auto stride = detail::grid_1d::grid_stride(); + + for (auto i = start; i < row_size; i += stride) { + if constexpr (!null_aware) { + if (stencil != nullptr && !bit_is_set(stencil, i)) { continue; } + } + + // used only for null-aware + auto active_mask = null_aware ? __ballot_sync(0xFFFF'FFFFU, i < row_size) : 0xFFFF'FFFFU; + + for (size_type c = 0; c < num_inputs; c++) { + load_element(input_cols + c, i * input_strides[c], input_storage + c); + } + + cudf_transform_operation( + user_data, i, input_storage, sizeof(storage_t), output_storage, sizeof(storage_t)); + + for (size_type c = 0; c < num_outputs; c++) { + store_element(output_cols + c, output_storage + c, i, active_mask); + } + } +} + +} // namespace cudf + +extern "C" __global__ void cudf_kernel_entry( + cudf::size_type row_size, + cudf::bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + cudf::column_device_view_core const* __restrict__ input_cols, + cudf::mutable_column_device_view_core const* __restrict__ output_cols, + cudf::size_type num_inputs, + cudf::size_type num_outputs, + cudf::size_type const* __restrict__ input_strides) +{ + CUDF_KERNEL_INSTANCE( + row_size, stencil, user_data, input_cols, output_cols, num_inputs, num_outputs, input_strides); +} diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 5bd76ccdb31a..3c3521581bb8 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -1092,10 +1093,22 @@ dispatch_untyped_lto_transform_kernel_shmem(bool null_aware, max_element_size = std::max(max_element_size, size); } - auto nullable_max_element_size = max_element_size * 2; // type + boolean is_null struct - - auto per_thread_shmem = - (null_aware ? nullable_max_element_size : max_element_size) * (inputs.size() + outputs.size()); + auto kernel_shmem = [&](int max_storage_size) -> size_t { + switch (max_storage_size) { + case 4: + return null_aware ? sizeof(element_storage) : sizeof(element_storage); + case 8: + return null_aware ? sizeof(element_storage) : sizeof(element_storage); + case 16: + return null_aware ? sizeof(element_storage) : sizeof(element_storage); + case 32: + return null_aware ? sizeof(element_storage) : sizeof(element_storage); + default: + CUDF_FAIL( + std::format("Unsupported untyped LTO shmem max element size {}", max_storage_size), + std::invalid_argument); + } + }; for (size_t i = 0; i < std::size(cudf_fragments::untyped_lto_transform_kernel_shmem_FILE_INDEX); i++) { @@ -1103,7 +1116,8 @@ dispatch_untyped_lto_transform_kernel_shmem(bool null_aware, auto NULL_AWARE = cudf_fragments::untyped_lto_transform_kernel_shmem_NULL_AWARE[i]; auto MAX_ELEMENT_SIZE = cudf_fragments::untyped_lto_transform_kernel_shmem_MAX_ELEMENT_SIZE[i]; if (null_aware == NULL_AWARE && max_element_size <= MAX_ELEMENT_SIZE) { - auto range = cudf_fragments::file_ranges[FILE_INDEX]; + auto per_thread_shmem = kernel_shmem(MAX_ELEMENT_SIZE) * (inputs.size() + outputs.size()); + auto range = cudf_fragments::file_ranges[FILE_INDEX]; return std::make_tuple(cudf_fragments::files.subspan(range[0], range[1]), static_cast(per_thread_shmem)); } @@ -1238,8 +1252,8 @@ std::unique_ptr
transform_lto(std::span inputs, } void* args[] = {&num_rows, - &user_data, &p_stencil, + &user_data, &input_cols, &output_cols, &num_inputs, diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 79205e5ad75e..e302be19f211 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -682,7 +682,7 @@ ConfigureTest(ENCODE_TEST encode/encode_tests.cpp) ConfigureTest(AST_TEST ast/transform_tests.cpp ast/ast_tree_tests.cpp) # ################################################################################################## -# * compute columns tests +# * transform LTO tests # ------------------------------------------------------------------------------------- add_embed(cudf_test_fragments) @@ -691,13 +691,10 @@ add_fragment( ) add_fragment( - cudf_test_fragments FRAGMENT transform_invsqrt_operator SOURCE - transform/fragments/invsqrt.cu + cudf_test_fragments FRAGMENT transform_invsqrt_operator SOURCE transform/fragments/invsqrt.cu ) -add_fragment( - cudf_test_fragments FRAGMENT transform_tpcds_q7 SOURCE transform/fragments/tpcds_q7.cu -) +add_fragment(cudf_test_fragments FRAGMENT transform_tpcds_q7 SOURCE transform/fragments/tpcds_q7.cu) embed(cudf_test_fragments COMPRESSION none) diff --git a/cpp/tests/transform/fragments/invsqrt.cu b/cpp/tests/transform/fragments/invsqrt.cu index 63b06f7920ea..cf7bbebd2f98 100644 --- a/cpp/tests/transform/fragments/invsqrt.cu +++ b/cpp/tests/transform/fragments/invsqrt.cu @@ -3,8 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include - template __device__ T load(void const* inputs, int input_stride, int arg) { diff --git a/cpp/tests/transform/fragments/tpcds_q7.cu b/cpp/tests/transform/fragments/tpcds_q7.cu index 7ef60f98d3d9..fb667a5f8b6e 100644 --- a/cpp/tests/transform/fragments/tpcds_q7.cu +++ b/cpp/tests/transform/fragments/tpcds_q7.cu @@ -4,7 +4,6 @@ * SPDX-License-Identifier: Apache-2.0 */ - template __device__ T load(void const* inputs, int input_stride, int arg) { From 6b2e0e746603a12e894233f7596570044f63c529 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 13 May 2026 17:28:25 +0000 Subject: [PATCH 170/254] Remove untyped LTO kernel implementations and related code - Deleted `untyped_lto_kernel_shmem.cu` and `untyped_lto_kernel_stack.cu` files as they are no longer needed. - Refactored `transform.cu` to remove unused functions and streamline the dispatching of LTO kernels. - Introduced new unary and binary operator implementations for decimal types in `decimal_square.cu` and `sum_of_squares.cu`. - Updated tests to reflect changes in the LTO kernel dispatching and added new tests for decimal operations. - Removed obsolete profit calculation logic and related test cases. --- cpp/CMakeLists.txt | 62 ++--- cpp/benchmarks/CMakeLists.txt | 19 +- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 81 ++++++ cpp/benchmarks/binaryop/fragments/add.cu | 25 ++ cpp/benchmarks/binaryop/fragments/null_max.cu | 26 ++ cpp/benchmarks/ndsh/fragments/q09.cu | 30 --- cpp/benchmarks/ndsh/q09.cpp | 46 +--- cpp/include/cudf/jit/transform_operation.cuh | 25 -- cpp/include/cudf/jit/transform_operator.cuh | 29 +++ cpp/include/cudf/transform.hpp | 66 ++++- cpp/src/jit/element.cuh | 121 --------- cpp/src/jit/element_storage.cuh | 54 ---- cpp/src/transform/jit/kernel.cu | 34 ++- cpp/src/transform/jit/lto_kernel.cu | 145 ----------- cpp/src/transform/jit/untyped_lto_kernel.cu | 139 ---------- .../transform/jit/untyped_lto_kernel_shmem.cu | 98 ------- .../transform/jit/untyped_lto_kernel_stack.cu | 85 ------ cpp/src/transform/transform.cu | 241 +++++------------- cpp/tests/CMakeLists.txt | 8 +- .../transform/fragments/decimal_square.cu | 28 ++ cpp/tests/transform/fragments/invsqrt.cu | 23 +- cpp/tests/transform/fragments/profit.cu | 49 ---- .../transform/fragments/sum_of_squares.cu | 14 + cpp/tests/transform/fragments/tpcds_q7.cu | 55 ---- cpp/tests/transform/transform_lto_test.cpp | 146 ++++------- 25 files changed, 444 insertions(+), 1205 deletions(-) create mode 100644 cpp/benchmarks/binaryop/fragments/add.cu create mode 100644 cpp/benchmarks/binaryop/fragments/null_max.cu delete mode 100644 cpp/benchmarks/ndsh/fragments/q09.cu delete mode 100644 cpp/include/cudf/jit/transform_operation.cuh create mode 100644 cpp/include/cudf/jit/transform_operator.cuh delete mode 100644 cpp/src/jit/element.cuh delete mode 100644 cpp/src/jit/element_storage.cuh delete mode 100644 cpp/src/transform/jit/lto_kernel.cu delete mode 100644 cpp/src/transform/jit/untyped_lto_kernel.cu delete mode 100644 cpp/src/transform/jit/untyped_lto_kernel_shmem.cu delete mode 100644 cpp/src/transform/jit/untyped_lto_kernel_stack.cu create mode 100644 cpp/tests/transform/fragments/decimal_square.cu delete mode 100644 cpp/tests/transform/fragments/profit.cu create mode 100644 cpp/tests/transform/fragments/sum_of_squares.cu delete mode 100644 cpp/tests/transform/fragments/tpcds_q7.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 39d9ee003598..bafc5b8c3ada 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -387,7 +387,9 @@ endmacro() add_embed(cudf_fragments) -foreach(TYPE IN ITEMS uint8 uint16 uint32 uint64 decimal32 decimal64 decimal128) +foreach(TYPE IN ITEMS uint8 uint16 uint32 int8 int16 int32 int64 float32 float64 uint64 decimal32 + decimal64 decimal128 +) foreach(NULL_AWARE IN ITEMS 0 1) set(FRAGMENT_NAME unop_lto_kernel) set(VARIANT_NAME ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__element_${TYPE}) @@ -396,9 +398,11 @@ foreach(TYPE IN ITEMS uint8 uint16 uint32 uint64 decimal32 decimal64 decimal128) FRAGMENT ${VARIANT_NAME} SOURCE - src/transform/jit/lto_kernel.cu + src/transform/jit/kernel.cu KERNEL_INSTANCE - "cudf::lto_transform_kernel<${NULL_AWARE}, cudf::jit::type_list>, cudf::jit::type_list>>" + "cudf::jit::transform_kernel<${NULL_AWARE}, false, cudf::jit::type_list>, cudf::jit::type_list>>" + DEFINITIONS + CUDF_LTO_MODE ARRAY_IDS ${FRAGMENT_NAME}_FILE_INDEX ${FRAGMENT_NAME}_NULL_AWARE @@ -411,62 +415,40 @@ foreach(TYPE IN ITEMS uint8 uint16 uint32 uint64 decimal32 decimal64 decimal128) endforeach() endforeach() -foreach(NULL_AWARE IN ITEMS 0 1) - foreach(MAX_ELEMENT_SIZE IN ITEMS 4 8) - foreach(MAX_ELEMENTS IN ITEMS 2 4) - set(FRAGMENT_NAME untyped_lto_transform_kernel_stack) +foreach(TYPE IN ITEMS uint8 uint16 uint32 uint64 int8 int16 int32 int64 float32 float64 decimal32 + decimal64 decimal128 +) + foreach(NULL_AWARE IN ITEMS 0 1) + foreach(RHS_IS_SCALAR IN ITEMS 0 1) + set(FRAGMENT_NAME binop_lto_kernel) set(VARIANT_NAME - ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__max_element_size_${MAX_ELEMENT_SIZE}__max_elements_${MAX_ELEMENTS} + ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__element_${TYPE}__rhs_is_scalar_${RHS_IS_SCALAR} ) add_fragment( cudf_fragments FRAGMENT ${VARIANT_NAME} SOURCE - src/transform/jit/untyped_lto_kernel.cu + src/transform/jit/kernel.cu KERNEL_INSTANCE - "cudf::untyped_lto_transform_kernel_stack<${NULL_AWARE}, ${MAX_ELEMENT_SIZE}, ${MAX_ELEMENTS}>" + "cudf::jit::transform_kernel<${NULL_AWARE}, false, cudf::jit::type_list, cudf::jit::column_accessor<1, cudf::column_device_view_core, cudf::tags::${TYPE}, ${RHS_IS_SCALAR}>>, cudf::jit::type_list>>" + DEFINITIONS + CUDF_LTO_MODE ARRAY_IDS ${FRAGMENT_NAME}_FILE_INDEX ${FRAGMENT_NAME}_NULL_AWARE - ${FRAGMENT_NAME}_MAX_ELEMENT_SIZE - ${FRAGMENT_NAME}_MAX_ELEMENTS + ${FRAGMENT_NAME}_TYPE + ${FRAGMENT_NAME}_RHS_IS_SCALAR ARRAY_VALUES ${cudf_fragments_FILE_INDEX} ${NULL_AWARE} - ${MAX_ELEMENT_SIZE} - ${MAX_ELEMENTS} + ${TYPE} + ${RHS_IS_SCALAR} ) endforeach() endforeach() endforeach() -foreach(MAX_ELEMENT_SIZE IN ITEMS 4 8 16 32) - foreach(NULL_AWARE IN ITEMS 0 1) - set(FRAGMENT_NAME untyped_lto_transform_kernel_shmem) - set(VARIANT_NAME - ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__max_element_size_${MAX_ELEMENT_SIZE} - ) - add_fragment( - cudf_fragments - FRAGMENT - ${VARIANT_NAME} - SOURCE - src/transform/jit/untyped_lto_kernel.cu - KERNEL_INSTANCE - "cudf::untyped_lto_transform_kernel_shmem<${NULL_AWARE}, ${MAX_ELEMENT_SIZE}>" - ARRAY_IDS - ${FRAGMENT_NAME}_FILE_INDEX - ${FRAGMENT_NAME}_NULL_AWARE - ${FRAGMENT_NAME}_MAX_ELEMENT_SIZE - ARRAY_VALUES - ${cudf_fragments_FILE_INDEX} - ${NULL_AWARE} - ${MAX_ELEMENT_SIZE} - ) - endforeach() -endforeach() - embed(cudf_fragments COMPRESSION none) add_embed(cudf_jit_embed) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index a120f33ccfa9..242523d78972 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -96,6 +96,13 @@ function(ConfigureNVBench CMAKE_BENCH_NAME) ) endfunction() +# ################################################################################################## +# * LTO Fragments ---------------------------------------------------------------------------- +add_embed(cudf_benchmark_fragments) +add_fragment(cudf_benchmark_fragments FRAGMENT add SOURCE binaryop/fragments/add.cu) +add_fragment(cudf_benchmark_fragments FRAGMENT null_max SOURCE binaryop/fragments/null_max.cu) +embed(cudf_benchmark_fragments COMPRESSION none) + # ################################################################################################## # * copying benchmarks ---------------------------------------------------------------------------- ConfigureNVBench( @@ -121,17 +128,10 @@ ConfigureNVBench(TRANSPOSE_NVBENCH transpose/transpose.cpp) # ################################################################################################## # * nds-h benchmark -------------------------------------------------------------------------------- - -add_embed(cudf_benchmark_fragments) -add_fragment(cudf_benchmark_fragments FRAGMENT ndsh_q09 SOURCE ndsh/fragments/q09.cu) -embed(cudf_benchmark_fragments COMPRESSION none) - ConfigureNVBench(NDSH_Q01_NVBENCH ndsh/q01.cpp ndsh/utilities.cpp) ConfigureNVBench(NDSH_Q05_NVBENCH ndsh/q05.cpp ndsh/utilities.cpp) ConfigureNVBench(NDSH_Q06_NVBENCH ndsh/q06.cpp ndsh/utilities.cpp) -ConfigureNVBench(NDSH_Q09_NVBENCH ndsh/q09.cpp ndsh/utilities.cpp ${cudf_benchmark_fragments_SOURCE_DIR}/cudf_benchmark_fragments.s) -add_dependencies(NDSH_Q09_NVBENCH cudf_benchmark_fragments) -target_include_directories(NDSH_Q09_NVBENCH PRIVATE ${cudf_benchmark_fragments_SOURCE_DIR}) +ConfigureNVBench(NDSH_Q09_NVBENCH ndsh/q09.cpp ndsh/utilities.cpp) ConfigureNVBench(NDSH_Q10_NVBENCH ndsh/q10.cpp ndsh/utilities.cpp) # ################################################################################################## @@ -396,7 +396,10 @@ ConfigureNVBench(AST_NVBENCH ast/polynomials.cpp ast/transform.cpp) # * binaryop benchmark ---------------------------------------------------------------------------- ConfigureNVBench( BINARYOP_NVBENCH binaryop/binaryop.cpp binaryop/compiled_binaryop.cpp binaryop/polynomials.cpp + ${cudf_benchmark_fragments_SOURCE_DIR}/cudf_benchmark_fragments.s ) +target_include_directories(BINARYOP_NVBENCH PRIVATE ${cudf_benchmark_fragments_SOURCE_DIR}) +add_dependencies(BINARYOP_NVBENCH cudf_benchmark_fragments) # ################################################################################################## # * transform benchmark diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 9befc76d2b45..7d458f95d556 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -6,7 +6,9 @@ #include #include +#include +#include #include template @@ -96,3 +98,82 @@ BINARYOP_BENCHMARK_DEFINE(duration_ms, duration_ns, NULL_EQUALS, bool BINARYOP_BENCHMARK_DEFINE(duration_ms, duration_ns, NULL_NOT_EQUALS, bool); BINARYOP_BENCHMARK_DEFINE(decimal32, decimal32, NULL_MAX, decimal32); BINARYOP_BENCHMARK_DEFINE(timestamp_D, timestamp_s, NULL_MIN, timestamp_s); + + + +template +void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) +{ + auto const num_rows = static_cast(state.get_int64("num_rows")); + + auto const source_table = create_random_table( + {cudf::type_to_id(), cudf::type_to_id()}, row_count{num_rows}); + + auto lhs = cudf::column_view(source_table->get_column(0)); + auto rhs = cudf::column_view(source_table->get_column(1)); + + size_t fragment_id = 0; + bool null_aware = false; + + switch (binop) { + case cudf::binary_operator::ADD: { + fragment_id = cudf_benchmark_fragments::add; + null_aware = false; + } break; + case cudf::binary_operator::NULL_MAX: { + fragment_id = cudf_benchmark_fragments::null_max; + null_aware = true; + } break; + default: throw std::runtime_error("Unsupported binary operator for LTO benchmark"); + } + + // Call once for hot cache. + cudf::transform_output output{cudf::data_type{cudf::type_to_id()}, + cudf::output_nullability::ALL_VALID}; + + auto const range = cudf_benchmark_fragments::file_ranges[fragment_id]; + std::span udf{cudf_benchmark_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::binary_op_lto(source_table->get_column(0), + source_table->get_column(1), + output, + udf, + cudf::lto_binary_type::FATBIN, + null_aware ? cudf::null_aware::YES : cudf::null_aware::NO + ); + + // use number of bytes read and written to global memory + state.add_global_memory_reads(num_rows); + state.add_global_memory_reads(num_rows); + state.add_global_memory_writes(num_rows); + + state.exec(nvbench::exec_tag::sync, [&](nvbench::launch&) { + cudf::binary_op_lto(source_table->get_column(0), + source_table->get_column(1), + output, + udf, + cudf::lto_binary_type::FATBIN, + null_aware ? cudf::null_aware::YES : cudf::null_aware::NO); + }); +} + + +#define BM_LTO_BINARYOP_BENCHMARK_DEFINE(name, lhs, rhs, bop, tout) \ + static void name(::nvbench::state& st) \ + { \ + ::BM_lto_binaryop(st, ::cudf::binary_operator::bop); \ + } \ + NVBENCH_BENCH(name) \ + .set_name("lto_binary_op_" BM_STRINGIFY(name)) \ + .add_int64_axis("num_rows", {10'000, 100'000, 1'000'000, 10'000'000, 100'000'000}) + + +#define build_name_lto(a, b, c, d) a##_##b##_##c##_##d##_lto + + +#define LTO_BINARYOP_BENCHMARK_DEFINE(lhs, rhs, bop, tout) \ + BM_LTO_BINARYOP_BENCHMARK_DEFINE(build_name_lto(bop, lhs, rhs, tout), lhs, rhs, bop, tout) + + +LTO_BINARYOP_BENCHMARK_DEFINE(float, float, ADD, float); +LTO_BINARYOP_BENCHMARK_DEFINE(decimal32, decimal32, NULL_MAX, decimal32); diff --git a/cpp/benchmarks/binaryop/fragments/add.cu b/cpp/benchmarks/binaryop/fragments/add.cu new file mode 100644 index 000000000000..e5879e950ffe --- /dev/null +++ b/cpp/benchmarks/binaryop/fragments/add.cu @@ -0,0 +1,25 @@ + + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +template <> +__device__ void cudf::lto::binary_operator(int32_t* __restrict__ out, + int32_t a, + int32_t b) +{ + *out = a + b; +} + +template <> +__device__ void cudf::lto::binary_operator(float* __restrict__ out, + float a, + float b) +{ + *out = a + b; +} diff --git a/cpp/benchmarks/binaryop/fragments/null_max.cu b/cpp/benchmarks/binaryop/fragments/null_max.cu new file mode 100644 index 000000000000..2283a17aa98a --- /dev/null +++ b/cpp/benchmarks/binaryop/fragments/null_max.cu @@ -0,0 +1,26 @@ + + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include + +template <> +__device__ void cudf::lto::binary_operator, + cuda::std::optional, + cuda::std::optional>( + cuda::std::optional* __restrict__ out, + cuda::std::optional a, + cuda::std::optional b) +{ + if (a.has_value() || b.has_value()) { + *out = (a.has_value() && (!b.has_value() || (*a > *b))) ? *a : *b; + } else { + *out = cuda::std::nullopt; + } +} diff --git a/cpp/benchmarks/ndsh/fragments/q09.cu b/cpp/benchmarks/ndsh/fragments/q09.cu deleted file mode 100644 index e2afad822e99..000000000000 --- a/cpp/benchmarks/ndsh/fragments/q09.cu +++ /dev/null @@ -1,30 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -template -__device__ T load(void const* inputs, int input_stride, int arg) -{ - auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); - return *p; -} - -template -__device__ void store(void* outputs, int output_stride, int arg, T value) -{ - auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); - *p = value; -} - -extern "C" __device__ int cudf_transform_operation( - void*, long int, void const* inputs, int input_stride, void* outputs, int output_stride) -{ - auto discount = load(inputs, input_stride, 0); - auto extended_price = load(inputs, input_stride, 1); - auto supply_cost = load(inputs, input_stride, 2); - auto quantity = load(inputs, input_stride, 3); - auto amount = extended_price * (1 - discount) - supply_cost * quantity; - store(outputs, output_stride, 0, amount); - return 0; -} diff --git a/cpp/benchmarks/ndsh/q09.cpp b/cpp/benchmarks/ndsh/q09.cpp index 182fd2b300b5..f94718ec5fa4 100644 --- a/cpp/benchmarks/ndsh/q09.cpp +++ b/cpp/benchmarks/ndsh/q09.cpp @@ -17,10 +17,9 @@ #include #include -#include #include -enum class engine_type : int32_t { BINARYOP = 0, AST = 1, TRANSFORM = 2, TRANSFORM_LTO = 3 }; +enum class engine_type : int32_t { BINARYOP = 0, AST = 1, TRANSFORM = 2 }; engine_type engine_from_string(std::string const& str) { @@ -30,8 +29,6 @@ engine_type engine_from_string(std::string const& str) return engine_type::AST; } else if (str == "transform") { return engine_type::TRANSFORM; - } else if (str == "transform_lto") { - return engine_type::TRANSFORM_LTO; } else { CUDF_FAIL("unrecognized engine enum: " + str); } @@ -200,38 +197,6 @@ struct q9_data { return cudf::compute_column(table, result, stream, mr); } -[[nodiscard]] std::unique_ptr compute_amount_transform_lto( - cudf::column_view const& discount, - cudf::column_view const& extendedprice, - cudf::column_view const& supplycost, - cudf::column_view const& quantity, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) -{ - CUDF_BENCHMARK_RANGE(); - - cudf::transform_input inputs[] = {discount, extendedprice, supplycost, quantity}; - - cudf::transform_output outputs[] = { - {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::PRESERVE}}; - - auto const range = cudf_benchmark_fragments::file_ranges[cudf_benchmark_fragments::ndsh_q09]; - std::span udf{cudf_benchmark_fragments::files.subspan(range[0], range[1])}; - - auto result = cudf::transform_lto(inputs, - udf, - cudf::lto_binary_type::FATBIN, - outputs, - nullptr, - cudf::null_aware::NO, - std::nullopt, - stream, - mr); - - auto table = result->release(); - return std::move(table.front()); -} - [[nodiscard]] std::unique_ptr compute_amount( cudf::column_view const& discount, cudf::column_view const& extendedprice, @@ -248,9 +213,6 @@ struct q9_data { return compute_amount_ast(discount, extendedprice, supplycost, quantity, stream, mr); case engine_type::TRANSFORM: return compute_amount_transform(discount, extendedprice, supplycost, quantity, stream, mr); - case engine_type::TRANSFORM_LTO: - return compute_amount_transform_lto( - discount, extendedprice, supplycost, quantity, stream, mr); default: CUDF_UNREACHABLE("invalid engine_type enum"); } } @@ -414,14 +376,14 @@ void ndsh_q9_amount(nvbench::state& state) NVBENCH_BENCH(ndsh_q9) .set_name("ndsh_q9") .add_float64_axis("scale_factor", {0.01, 0.1, 1}) - .add_string_axis("engine", {"binaryop", "ast", "transform", "transform_lto"}); + .add_string_axis("engine", {"binaryop", "ast", "transform"}); NVBENCH_BENCH(ndsh_q9_noio) .set_name("ndsh_q9_noio") .add_float64_axis("scale_factor", {0.01, 0.1, 1}) - .add_string_axis("engine", {"binaryop", "ast", "transform", "transform_lto"}); + .add_string_axis("engine", {"binaryop", "ast", "transform"}); NVBENCH_BENCH(ndsh_q9_amount) .set_name("ndsh_q9_amount") .add_float64_axis("scale_factor", {0.01, 0.1, 1}) - .add_string_axis("engine", {"binaryop", "ast", "transform", "transform_lto"}); + .add_string_axis("engine", {"binaryop", "ast", "transform"}); diff --git a/cpp/include/cudf/jit/transform_operation.cuh b/cpp/include/cudf/jit/transform_operation.cuh deleted file mode 100644 index 92f43cfd10ab..000000000000 --- a/cpp/include/cudf/jit/transform_operation.cuh +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -/// @brief The compute operation to perform on each element. This is a generic entry point for n-ary -/// transform operations. -/// @param user_data Pointer to user data passed to the kernel -/// @param element_index The index of the element to compute -/// @param inputs Pointer to the input elements for this operation; the caller guarantees the memory -/// layout and type of these elements based on the input column device views and input strides -/// @param input_stride The stride (in bytes) between consecutive input elements for a given input -/// column -/// @param outputs Pointer to the output elements for this operation; the caller guarantees the -/// memory layout and type of these elements based on the output column device views -/// @param output_stride The stride (in bytes) between consecutive output elements for a given -/// output column -/// @return An integer status code -extern "C" __device__ int cudf_transform_operation(void* __restrict__ user_data, - long int element_index, - void const* __restrict__ inputs, - int input_stride, - void* __restrict__ outputs, - int output_stride); diff --git a/cpp/include/cudf/jit/transform_operator.cuh b/cpp/include/cudf/jit/transform_operator.cuh new file mode 100644 index 000000000000..49d15a2d4974 --- /dev/null +++ b/cpp/include/cudf/jit/transform_operator.cuh @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +namespace cudf { +namespace lto { + +/// @brief The unary operator for the transform operation. +/// @tparam Out The output type of the operator. +/// @tparam In0 The input type of the operator. +/// @param out The output destination for the operator result. +/// @param a The input value for the operator. +template +__device__ void unary_operator(Out* __restrict__ out, In0 a); + +/// @brief The binary operator for the transform operation. +/// @tparam Out The output type of the operator. +/// @tparam In0 The first input type of the operator. +/// @tparam In1 The second input type of the operator. +/// @param out The output destination for the operator result. +/// @param a The first input value for the operator. +/// @param b The second input value for the operator. +template +__device__ void binary_operator(Out* __restrict__ out, In0 a, In1 b); + +} // namespace lto +} // namespace cudf diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 6657644dec0d..1885e214281c 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -188,11 +188,11 @@ enum class lto_binary_type : uint8_t { }; /** - * @brief Creates a new table by applying a transform function against every + * @brief Creates a new column by applying a transform function against every * element of the input columns. * * Computes: - * `(outputs[i]...) = UDF(inputs[i]...)`. + * `(output[i]) = UDF(input[i])`. * * * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) @@ -207,28 +207,66 @@ enum class lto_binary_type : uint8_t { * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from * the input and pre-allocated output columns. * - * @param inputs Immutable views of the inputs to transform (columns and scalar columns) + * @param input Immutable view of the input to transform + * @param output Specification of the output column to be created * @param udf The LTO-IR string of the transform function to apply * @param binary_type The type of the LTO binary provided in `udf` - * @param user_data User-defined device data to pass to the UDF. * @param is_null_aware Signifies the UDF will receive row inputs as optional values - * @param outputs Specification of the output columns to be created - * @param row_size The row size of the transform operation. If not provided, it is inferred from the - * input columns. * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory - * @return A table containing the columns resulting from applying the transform - * function to every element of the input according to the output specifications + * @return A column resulting from applying the transform function to every element of + * the input according to the output specifications * */ -std::unique_ptr
transform_lto( - std::span inputs, +std::unique_ptr unary_op_lto( + column_view input, + transform_output output, + std::span udf, + lto_binary_type binary_type, + null_aware is_null_aware, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + +/** + * @brief Creates a new column by applying a transform function against every + * element of the input columns. + * + * Computes: + * `(output[i]...) = UDF(inputs[i]...)`. + * + * + * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) + * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string + * types + * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and + * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in + * this case. + * @throws std::invalid_argument if string offsets are provided for non-string output columns, or + * if the number of string offsets does not match the number of output columns. + * + * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from + * the input and pre-allocated output columns. + * + * @param lhs Immutable view of the left-hand side input to transform + * @param rhs Immutable view of the right-hand side input to transform (can be a column or + * scalar) + * @param output Specification of the output column to be created + * @param udf The LTO-IR string of the transform function to apply + * @param binary_type The type of the LTO binary provided in `udf` + * @param is_null_aware Signifies the UDF will receive row inputs as optional values + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + * @return A column resulting from applying the transform function to every element of + * the input according to the output specifications + * + */ +std::unique_ptr binary_op_lto( + column_view lhs, + transform_input rhs, + transform_output output, std::span udf, lto_binary_type binary_type, - std::span outputs, - void* user_data, null_aware is_null_aware, - std::optional row_size, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); diff --git a/cpp/src/jit/element.cuh b/cpp/src/jit/element.cuh deleted file mode 100644 index c1adb4379f9a..000000000000 --- a/cpp/src/jit/element.cuh +++ /dev/null @@ -1,121 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include -#include -#include - -#include -#include - -namespace CUDF_EXPORT cudf { - -template -__device__ void load_element(column_device_view_core const* column, - size_type element_index, - element_storage_t* storage) -{ - auto op = [&] __device__() { - if constexpr (!has_nulls) { - if constexpr (storage_compatible) { - auto v = column->template element(element_index); - *reinterpret_cast(storage->data) = v; - } - } else { - if constexpr (storage_compatible>) { - auto v = column->template nullable_element(element_index); - *reinterpret_cast*>(storage->data) = v; - } - } - }; - - switch (column->type().id()) { - case type_id::INT8: - case type_id::UINT8: - case type_id::BOOL8: op.template operator()(); break; - case type_id::INT16: - case type_id::UINT16: op.template operator()(); break; - case type_id::INT32: - case type_id::UINT32: - case type_id::FLOAT32: - case type_id::TIMESTAMP_DAYS: - case type_id::DURATION_DAYS: op.template operator()(); break; - case type_id::INT64: - case type_id::UINT64: - case type_id::FLOAT64: - case type_id::TIMESTAMP_SECONDS: - case type_id::TIMESTAMP_MILLISECONDS: - case type_id::TIMESTAMP_MICROSECONDS: - case type_id::TIMESTAMP_NANOSECONDS: - case type_id::DURATION_SECONDS: - case type_id::DURATION_MILLISECONDS: - case type_id::DURATION_MICROSECONDS: - case type_id::DURATION_NANOSECONDS: op.template operator()(); break; - case type_id::DECIMAL32: op.template operator()(); break; - case type_id::DECIMAL64: op.template operator()(); break; - case type_id::DECIMAL128: op.template operator()(); break; - case type_id::STRING: op.template operator()(); break; - default: CUDF_UNREACHABLE(); - } -} - -template -__device__ void store_element(mutable_column_device_view_core const* column, - element_storage_t const* storage, - size_type element_index, - unsigned int active_mask) -{ - auto op = [&] __device__() { - if constexpr (!has_nulls) { - if constexpr (storage_compatible) { - auto v = *reinterpret_cast(storage->data); - column->template assign(element_index, v); - } - } else { - if constexpr (storage_compatible>) { - auto v = *reinterpret_cast const*>(storage->data); - column->template assign(element_index, *v); - - auto null_word = __ballot_sync(active_mask, v.has_value()); - if (column->nullable()) { - if (warp_elect(active_mask)) { - column->null_mask()[element_index / detail::warp_size] = null_word; - } - } - } - } - }; - - switch (column->type().id()) { - case type_id::INT8: - case type_id::UINT8: - case type_id::BOOL8: op.template operator()(); break; - case type_id::INT16: - case type_id::UINT16: op.template operator()(); break; - case type_id::INT32: - case type_id::UINT32: - case type_id::FLOAT32: - case type_id::TIMESTAMP_DAYS: - case type_id::DURATION_DAYS: op.template operator()(); break; - case type_id::INT64: - case type_id::UINT64: - case type_id::FLOAT64: - case type_id::TIMESTAMP_SECONDS: - case type_id::TIMESTAMP_MILLISECONDS: - case type_id::TIMESTAMP_MICROSECONDS: - case type_id::TIMESTAMP_NANOSECONDS: - case type_id::DURATION_SECONDS: - case type_id::DURATION_MILLISECONDS: - case type_id::DURATION_MICROSECONDS: - case type_id::DURATION_NANOSECONDS: op.template operator()(); break; - case type_id::DECIMAL32: op.template operator()(); break; - case type_id::DECIMAL64: op.template operator()(); break; - case type_id::DECIMAL128: op.template operator()(); break; - default: CUDF_UNREACHABLE(); - } -} - -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/element_storage.cuh b/cpp/src/jit/element_storage.cuh deleted file mode 100644 index 406797dcc2df..000000000000 --- a/cpp/src/jit/element_storage.cuh +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include - -#include -#include -#include - -namespace CUDF_EXPORT cudf { - -struct [[nodiscard]] layout { - std::size_t size = 0; //< Size in bytes of the layout - std::size_t alignment = 1; //< Non-zero power of 2 alignment - - constexpr layout unioned(layout const& other) const - { - return layout{.size = cuda::std::max(size, other.size), - .alignment = cuda::std::max(alignment, other.alignment)}; - } -}; - -template -inline constexpr layout layout_of = layout{.size = sizeof(T), .alignment = alignof(T)}; - -template -struct storage { - alignas(layout.alignment) unsigned char data[layout.size]; -}; - -template -inline constexpr bool storage_compatible = - sizeof(ElementType) <= sizeof(Storage) && alignof(ElementType) <= alignof(Storage); - -template -using maybe_nullable = cuda::std::conditional_t, T>; - -template -using nonnull_element_storage = - storage; - -template -using nullable_element_storage = - storage>>>; - -template -using element_storage = cuda::std::conditional_t, - nonnull_element_storage>; - -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 0f78264966ef..badfdd7463b5 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -18,7 +20,6 @@ #include #include -#include #include #include @@ -38,16 +39,18 @@ namespace jit { /// @brief The generic transform kernel. Supports all types and nullability combinations. template -CUDF_KERNEL void transform_kernel(size_type row_size, - bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols) +__device__ void transform_kernel(size_type row_size, + bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + column_device_view_core const* __restrict__ input_cols, + mutable_column_device_view_core const* __restrict__ output_cols) { auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); for (auto row = start; row < row_size; row += stride) { +#ifndef CUDF_LTO_MODE + auto operation = [&](Args const& args) { if constexpr (has_user_data) { cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, @@ -57,6 +60,25 @@ CUDF_KERNEL void transform_kernel(size_type row_size, } }; +#else + + auto operation = [&](Args const& args) { + static_assert(!has_user_data); + static_assert(OutputAccessors::size == 1); + static_assert(InputAccessors::size == 2 || InputAccessors::size == 1); + cuda::std::apply( + [&](auto... a) { + if constexpr (InputAccessors::size == 1) { + cudf::lto::unary_operator(a...); + } else if constexpr (InputAccessors::size == 2) { + cudf::lto::binary_operator(a...); + } + }, + args); + }; + +#endif + if constexpr (!is_null_aware) { if (stencil != nullptr && !bit_is_set(stencil, row)) { continue; } diff --git a/cpp/src/transform/jit/lto_kernel.cu b/cpp/src/transform/jit/lto_kernel.cu deleted file mode 100644 index 17450ce9065b..000000000000 --- a/cpp/src/transform/jit/lto_kernel.cu +++ /dev/null @@ -1,145 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include - -#pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before - // it. Each UDF will have a different operation-udf.hpp generated for it, so we - // need to put this pragma before including it to avoid PCH mismatch. - -// clang-format off -#include -// clang-format on - -namespace cudf { - -/// @brief The generic LTO transform kernel. Supports all types and nullability combinations. -/// This kernel is intended to be used with LTO, it has uses registers and has clear memory -/// boundaries. This is intended to be used for simple n-ary operators. -template -__device__ void lto_transform_kernel( - size_type row_size, - bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols) -{ - auto start = detail::grid_1d::global_thread_id(); - auto stride = detail::grid_1d::grid_stride(); - - static constexpr auto input_layout = - null_aware ? InputAccessors::map([&]() { - layout result{}; - ((result = result.unioned(layout_of)), ...); - return result; - }) - : InputAccessors::map([&]() { - layout result{}; - ((result = result.unioned(layout_of)), ...); - return result; - }); - - static constexpr auto output_layout = - null_aware ? OutputAccessors::map([&]() { - layout result{}; - ((result = result.unioned(layout_of)), ...); - return result; - }) - : OutputAccessors::map([&]() { - layout result{}; - ((result = result.unioned(layout_of)), ...); - return result; - }); - - for (auto row = start; row < row_size; row += stride) { - if constexpr (null_aware) { - if (stencil != nullptr && !bit_is_set(stencil, row)) { continue; } - } - - using input_storage_t = storage; - using output_storage_t = storage; - - output_storage_t outputs_storage[OutputAccessors::size]; - input_storage_t inputs_storage[InputAccessors::size]; - - InputAccessors::map([&]() { - if constexpr (null_aware) { - ((*reinterpret_cast(inputs_storage[A::index].data) = - A::nullable_element(input_cols, row)), - ...); - } else { - ((*reinterpret_cast(inputs_storage[A::index].data) = - A::element(input_cols, row)), - ...); - } - }); - - OutputAccessors::map([&]() { - if constexpr (null_aware) { - ((*reinterpret_cast(outputs_storage[A::index].data) = - A::null_output_arg(output_cols, row)), - ...); - } else { - ((*reinterpret_cast(outputs_storage[A::index].data) = - A::output_arg(output_cols, row)), - ...); - } - }); - - [[maybe_unused]] auto errc = cudf_transform_operation(user_data, - row, - &inputs_storage, - sizeof(inputs_storage), - &outputs_storage, - sizeof(outputs_storage)); - - // used only for null-aware - auto active_mask = null_aware ? __ballot_sync(0xFFFF'FFFFU, row < row_size) : 0xFFFF'FFFFU; - - auto assign = [&]() { - auto* src = outputs_storage[A::index].data; - if constexpr (null_aware) { - auto& ret = *reinterpret_cast(src); - A::assign(output_cols, row, *ret); - jit::warp_compact_validity(active_mask, output_cols, row, ret.has_value()); - } else { - A::assign(output_cols, row, *reinterpret_cast(src)); - } - }; - - OutputAccessors::map([&]() { (assign.template operator()(), ...); }); - } -} - -} // namespace cudf - -extern "C" __global__ void cudf_kernel_entry( - cudf::size_type row_size, - cudf::bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - cudf::column_device_view_core const* __restrict__ input_cols, - cudf::mutable_column_device_view_core const* __restrict__ output_cols) -{ - CUDF_KERNEL_INSTANCE(row_size, stencil, user_data, input_cols, output_cols); -} diff --git a/cpp/src/transform/jit/untyped_lto_kernel.cu b/cpp/src/transform/jit/untyped_lto_kernel.cu deleted file mode 100644 index 739dd88c6562..000000000000 --- a/cpp/src/transform/jit/untyped_lto_kernel.cu +++ /dev/null @@ -1,139 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "jit/element.cuh" -#include "jit/element_storage.cuh" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before - // it. Each UDF will have a different operation-udf.hpp generated for it, so we - // need to put this pragma before including it to avoid PCH mismatch. - -// clang-format off -#include -// clang-format on - -namespace cudf { - -/** - * @brief An untyped Generic Transform kernel. This is a catch-all kernel that can be used for any - * transform operation, but is not expected to be the most performant. The intent is that this - * kernel can be used for any operator and type combination with minimal specialization, but that - * more specialized kernels can be generated for common cases (e.g. binary operations on - * fixed-width types) that will be more performant. - * - */ -template -__device__ void untyped_lto_transform_kernel_shmem( - size_type row_size, - bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols, - size_type num_inputs, - size_type num_outputs, - size_type const* __restrict__ input_strides) -{ - using storage_t = element_storage; - - extern __shared__ char shmem[]; - - auto per_thread_bytes = sizeof(storage_t) * (num_inputs + num_outputs); - auto shmem_iter = shmem + per_thread_bytes * threadIdx.x; - - auto* __restrict__ input_storage = reinterpret_cast(shmem_iter); - shmem_iter += sizeof(storage_t) * num_inputs; - auto* __restrict__ output_storage = reinterpret_cast(shmem_iter); - - auto start = detail::grid_1d::global_thread_id(); - auto stride = detail::grid_1d::grid_stride(); - - for (auto i = start; i < row_size; i += stride) { - if constexpr (!null_aware) { - if (stencil != nullptr && !bit_is_set(stencil, i)) { continue; } - } - - // used only for null-aware - auto active_mask = null_aware ? __ballot_sync(0xFFFF'FFFFU, i < row_size) : 0xFFFF'FFFFU; - - for (size_type c = 0; c < num_inputs; c++) { - load_element(input_cols + c, i * input_strides[c], input_storage + c); - } - - cudf_transform_operation( - user_data, i, input_storage, sizeof(storage_t), output_storage, sizeof(storage_t)); - - for (size_type c = 0; c < num_outputs; c++) { - store_element(output_cols + c, output_storage + c, i, active_mask); - } - } -} - -template -__device__ void untyped_lto_transform_kernel_stack( - size_type row_size, - bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols, - size_type num_inputs, - size_type num_outputs, - size_type const* __restrict__ input_strides) -{ - using storage_t = element_storage; - - storage_t storage[max_elements]; - auto* __restrict__ input_storage = storage; - auto* __restrict__ output_storage = storage + num_inputs; - - auto start = detail::grid_1d::global_thread_id(); - auto stride = detail::grid_1d::grid_stride(); - - for (auto i = start; i < row_size; i += stride) { - if constexpr (!null_aware) { - if (stencil != nullptr && !bit_is_set(stencil, i)) { continue; } - } - - // used only for null-aware - auto active_mask = null_aware ? __ballot_sync(0xFFFF'FFFFU, i < row_size) : 0xFFFF'FFFFU; - - for (size_type c = 0; c < num_inputs; c++) { - load_element(input_cols + c, i * input_strides[c], input_storage + c); - } - - cudf_transform_operation( - user_data, i, input_storage, sizeof(storage_t), output_storage, sizeof(storage_t)); - - for (size_type c = 0; c < num_outputs; c++) { - store_element(output_cols + c, output_storage + c, i, active_mask); - } - } -} - -} // namespace cudf - -extern "C" __global__ void cudf_kernel_entry( - cudf::size_type row_size, - cudf::bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - cudf::column_device_view_core const* __restrict__ input_cols, - cudf::mutable_column_device_view_core const* __restrict__ output_cols, - cudf::size_type num_inputs, - cudf::size_type num_outputs, - cudf::size_type const* __restrict__ input_strides) -{ - CUDF_KERNEL_INSTANCE( - row_size, stencil, user_data, input_cols, output_cols, num_inputs, num_outputs, input_strides); -} diff --git a/cpp/src/transform/jit/untyped_lto_kernel_shmem.cu b/cpp/src/transform/jit/untyped_lto_kernel_shmem.cu deleted file mode 100644 index c14528159d49..000000000000 --- a/cpp/src/transform/jit/untyped_lto_kernel_shmem.cu +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "jit/element.cuh" -#include "jit/element_storage.cuh" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before - // it. Each UDF will have a different operation-udf.hpp generated for it, so we - // need to put this pragma before including it to avoid PCH mismatch. - -// clang-format off -#include -// clang-format on - -namespace cudf { - -/** - * @brief An untyped Generic Transform kernel. This is a catch-all kernel that can be used for any - * transform operation, but is not expected to be the most performant. The intent is that this - * kernel can be used for any operator and type combination with minimal specialization, but that - * more specialized kernels can be generated for common cases (e.g. binary operations on - * fixed-width types) that will be more performant. - * - */ -template -__device__ void untyped_lto_transform_kernel_shmem( - size_type row_size, - bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols, - size_type num_inputs, - size_type num_outputs, - size_type const* __restrict__ input_strides) -{ - using storage_t = element_storage; - - extern __shared__ char shmem[]; - - auto per_thread_bytes = sizeof(storage_t) * (num_inputs + num_outputs); - auto shmem_iter = shmem + per_thread_bytes * threadIdx.x; - - auto* __restrict__ input_storage = reinterpret_cast(shmem_iter); - shmem_iter += sizeof(storage_t) * num_inputs; - auto* __restrict__ output_storage = reinterpret_cast(shmem_iter); - - auto start = detail::grid_1d::global_thread_id(); - auto stride = detail::grid_1d::grid_stride(); - - for (auto i = start; i < row_size; i += stride) { - if constexpr (!null_aware) { - if (stencil != nullptr && !bit_is_set(stencil, i)) { continue; } - } - - // used only for null-aware - auto active_mask = null_aware ? __ballot_sync(0xFFFF'FFFFU, i < row_size) : 0xFFFF'FFFFU; - - for (size_type c = 0; c < num_inputs; c++) { - load_element(input_cols + c, i * input_strides[c], input_storage + c); - } - - cudf_transform_operation( - user_data, i, input_storage, sizeof(storage_t), output_storage, sizeof(storage_t)); - - for (size_type c = 0; c < num_outputs; c++) { - store_element(output_cols + c, output_storage + c, i, active_mask); - } - } -} - -} // namespace cudf - -extern "C" __launch_bounds__(256) __global__ - void cudf_kernel_entry(cudf::size_type row_size, - cudf::bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - cudf::column_device_view_core const* __restrict__ input_cols, - cudf::mutable_column_device_view_core const* __restrict__ output_cols, - cudf::size_type num_inputs, - cudf::size_type num_outputs, - cudf::size_type const* __restrict__ input_strides) -{ - CUDF_KERNEL_INSTANCE( - row_size, stencil, user_data, input_cols, output_cols, num_inputs, num_outputs, input_strides); -} diff --git a/cpp/src/transform/jit/untyped_lto_kernel_stack.cu b/cpp/src/transform/jit/untyped_lto_kernel_stack.cu deleted file mode 100644 index 245af24c1e57..000000000000 --- a/cpp/src/transform/jit/untyped_lto_kernel_stack.cu +++ /dev/null @@ -1,85 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "jit/element.cuh" -#include "jit/element_storage.cuh" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before - // it. Each UDF will have a different operation-udf.hpp generated for it, so we - // need to put this pragma before including it to avoid PCH mismatch. - -// clang-format off -#include -// clang-format on - -namespace cudf { - -template -__device__ void untyped_lto_transform_kernel_stack( - size_type row_size, - bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols, - size_type num_inputs, - size_type num_outputs, - size_type const* __restrict__ input_strides) -{ - using storage_t = element_storage; - - storage_t storage[max_elements]; - auto* __restrict__ input_storage = storage; - auto* __restrict__ output_storage = storage + num_inputs; - - auto start = detail::grid_1d::global_thread_id(); - auto stride = detail::grid_1d::grid_stride(); - - for (auto i = start; i < row_size; i += stride) { - if constexpr (!null_aware) { - if (stencil != nullptr && !bit_is_set(stencil, i)) { continue; } - } - - // used only for null-aware - auto active_mask = null_aware ? __ballot_sync(0xFFFF'FFFFU, i < row_size) : 0xFFFF'FFFFU; - - for (size_type c = 0; c < num_inputs; c++) { - load_element(input_cols + c, i * input_strides[c], input_storage + c); - } - - cudf_transform_operation( - user_data, i, input_storage, sizeof(storage_t), output_storage, sizeof(storage_t)); - - for (size_type c = 0; c < num_outputs; c++) { - store_element(output_cols + c, output_storage + c, i, active_mask); - } - } -} - -} // namespace cudf - -extern "C" __global__ void cudf_kernel_entry( - cudf::size_type row_size, - cudf::bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - cudf::column_device_view_core const* __restrict__ input_cols, - cudf::mutable_column_device_view_core const* __restrict__ output_cols, - cudf::size_type num_inputs, - cudf::size_type num_outputs, - cudf::size_type const* __restrict__ input_strides) -{ - CUDF_KERNEL_INSTANCE( - row_size, stencil, user_data, input_cols, output_cols, num_inputs, num_outputs, input_strides); -} diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 3c3521581bb8..b7a21747af81 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -24,7 +24,6 @@ #include #include -#include #include #include #include @@ -965,33 +964,6 @@ std::unique_ptr compute_column_jit(table_view const& table, mr); } -type_id as_storage_type(type_id id) -{ - switch (id) { - case type_id::INT8: - case type_id::UINT8: return type_id::UINT8; - case type_id::INT16: - case type_id::UINT16: return type_id::UINT16; - case type_id::INT32: - case type_id::UINT32: - case type_id::FLOAT32: - case type_id::TIMESTAMP_DAYS: - case type_id::DURATION_DAYS: return type_id::UINT32; - case type_id::INT64: - case type_id::UINT64: - case type_id::FLOAT64: - case type_id::TIMESTAMP_SECONDS: - case type_id::TIMESTAMP_MILLISECONDS: - case type_id::TIMESTAMP_MICROSECONDS: - case type_id::TIMESTAMP_NANOSECONDS: - case type_id::DURATION_SECONDS: - case type_id::DURATION_MILLISECONDS: - case type_id::DURATION_MICROSECONDS: - case type_id::DURATION_NANOSECONDS: return type_id::UINT64; - default: return id; - } -} - std::string_view as_tag(type_id id) { switch (id) { @@ -1024,22 +996,12 @@ std::string_view as_tag(type_id id) } } -struct element_size_fn { - template - int32_t operator()() - { - return sizeof(T); - } -}; - -int32_t element_size(type_id id) { return type_dispatcher(data_type{id}, element_size_fn{}); } - std::optional> dispatch_unop_lto_kernel( bool null_aware, std::span inputs, std::span outputs) { - auto input_type = as_storage_type(std::visit([](auto& c) { return c.type().id(); }, inputs[0])); + auto input_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); for (size_t i = 0; i < std::size(cudf_fragments::unop_lto_kernel_FILE_INDEX); i++) { auto FILE_INDEX = cudf_fragments::unop_lto_kernel_FILE_INDEX[i]; @@ -1054,26 +1016,23 @@ std::optional> dispatch_unop_lto_kernel( return std::nullopt; } -std::optional> dispatch_untyped_lto_transform_kernel_stack( +std::optional> dispatch_binop_lto_kernel( bool null_aware, std::span inputs, std::span outputs) { - int32_t max_element_size = 0; - for (auto& input : inputs) { - auto size = element_size(std::visit([](auto& c) { return c.type().id(); }, input)); - max_element_size = std::max(max_element_size, size); - } - int32_t elements = inputs.size() + outputs.size(); - - for (size_t i = 0; i < std::size(cudf_fragments::untyped_lto_transform_kernel_stack_FILE_INDEX); - i++) { - auto FILE_INDEX = cudf_fragments::untyped_lto_transform_kernel_stack_FILE_INDEX[i]; - auto NULL_AWARE = cudf_fragments::untyped_lto_transform_kernel_stack_NULL_AWARE[i]; - auto MAX_ELEMENT_SIZE = cudf_fragments::untyped_lto_transform_kernel_stack_MAX_ELEMENT_SIZE[i]; - auto MAX_ELEMENTS = cudf_fragments::untyped_lto_transform_kernel_stack_MAX_ELEMENTS[i]; - if (null_aware == NULL_AWARE && max_element_size <= MAX_ELEMENT_SIZE && - elements <= MAX_ELEMENTS) { + auto input_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); + auto lhs_is_scalar = std::holds_alternative(inputs[0]); + auto rhs_is_scalar = std::holds_alternative(inputs[1]); + + if (lhs_is_scalar) { return std::nullopt; } + + for (size_t i = 0; i < std::size(cudf_fragments::binop_lto_kernel_FILE_INDEX); i++) { + auto FILE_INDEX = cudf_fragments::binop_lto_kernel_FILE_INDEX[i]; + auto NULL_AWARE = cudf_fragments::binop_lto_kernel_NULL_AWARE[i]; + auto TYPE = cudf_fragments::binop_lto_kernel_TYPE[i]; + auto RHS_IS_SCALAR = cudf_fragments::binop_lto_kernel_RHS_IS_SCALAR[i]; + if (as_tag(input_type) == TYPE && null_aware == NULL_AWARE && rhs_is_scalar == RHS_IS_SCALAR) { auto range = cudf_fragments::file_ranges[FILE_INDEX]; return cudf_fragments::files.subspan(range[0], range[1]); } @@ -1082,94 +1041,28 @@ std::optional> dispatch_untyped_lto_transform_kernel_st return std::nullopt; } -std::optional, size_t>> -dispatch_untyped_lto_transform_kernel_shmem(bool null_aware, - std::span inputs, - std::span outputs) -{ - int32_t max_element_size = 0; - for (auto& input : inputs) { - auto size = element_size(std::visit([](auto& c) { return c.type().id(); }, input)); - max_element_size = std::max(max_element_size, size); - } - - auto kernel_shmem = [&](int max_storage_size) -> size_t { - switch (max_storage_size) { - case 4: - return null_aware ? sizeof(element_storage) : sizeof(element_storage); - case 8: - return null_aware ? sizeof(element_storage) : sizeof(element_storage); - case 16: - return null_aware ? sizeof(element_storage) : sizeof(element_storage); - case 32: - return null_aware ? sizeof(element_storage) : sizeof(element_storage); - default: - CUDF_FAIL( - std::format("Unsupported untyped LTO shmem max element size {}", max_storage_size), - std::invalid_argument); - } - }; - - for (size_t i = 0; i < std::size(cudf_fragments::untyped_lto_transform_kernel_shmem_FILE_INDEX); - i++) { - auto FILE_INDEX = cudf_fragments::untyped_lto_transform_kernel_shmem_FILE_INDEX[i]; - auto NULL_AWARE = cudf_fragments::untyped_lto_transform_kernel_shmem_NULL_AWARE[i]; - auto MAX_ELEMENT_SIZE = cudf_fragments::untyped_lto_transform_kernel_shmem_MAX_ELEMENT_SIZE[i]; - if (null_aware == NULL_AWARE && max_element_size <= MAX_ELEMENT_SIZE) { - auto per_thread_shmem = kernel_shmem(MAX_ELEMENT_SIZE) * (inputs.size() + outputs.size()); - auto range = cudf_fragments::file_ranges[FILE_INDEX]; - return std::make_tuple(cudf_fragments::files.subspan(range[0], range[1]), - static_cast(per_thread_shmem)); - } - } - - return std::nullopt; -} - -rtcx::kernel_occupancy_config configure_shmem(cudf::kernel const& transform_kernel, - size_t dynamic_smem_per_thread) -{ - size_t static_shmem_per_thread = 0; - - auto shmem_for_block = [&](int block_size) -> size_t { - return (static_shmem_per_thread + dynamic_smem_per_thread) * block_size; - }; - - int min_grid_size; - int block_size; - - CUDF_CUDA_TRY(cudaOccupancyMaxPotentialBlockSizeVariableSMem( - &min_grid_size, &block_size, transform_kernel.get().get(), shmem_for_block, 0)); - - return {.min_grid_size = static_cast(min_grid_size), - .block_size = static_cast(block_size)}; -} - // Dispatches to the appropriate LTO kernel based on the number of inputs and outputs, their types, // and nullability. -std::tuple, size_t> dispatch_lto_kernel( - bool null_aware, - std::span inputs, - std::span outputs) +std::span dispatch_lto_kernel(bool null_aware, + std::span inputs, + std::span outputs) { if (inputs.size() == 1 && outputs.size() == 1) { - auto input0_type = - as_storage_type(std::visit([](auto& c) { return c.type().id(); }, inputs[0])); - auto output_type = as_storage_type(outputs[0].type.id()); - + auto input0_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); + auto output_type = outputs[0].type.id(); if (input0_type == output_type && is_fixed_width(data_type{input0_type})) { - if (auto kernel = dispatch_unop_lto_kernel(null_aware, inputs, outputs)) { - return {*kernel, 0}; - } + if (auto kernel = dispatch_unop_lto_kernel(null_aware, inputs, outputs)) { return *kernel; } } } - if (auto kernel = dispatch_untyped_lto_transform_kernel_stack(null_aware, inputs, outputs)) { - return {*kernel, 0}; - } - - if (auto kernel = dispatch_untyped_lto_transform_kernel_shmem(null_aware, inputs, outputs)) { - return *kernel; + if (inputs.size() == 2 && outputs.size() == 1) { + auto input0_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); + auto input1_type = std::visit([](auto& c) { return c.type().id(); }, inputs[1]); + auto output_type = outputs[0].type.id(); + if (input0_type == output_type && input1_type == output_type && + is_fixed_width(data_type{input0_type})) { + if (auto kernel = dispatch_binop_lto_kernel(null_aware, inputs, outputs)) { return *kernel; } + } } CUDF_FAIL("No suitable LTO kernel found for the given transform parameters", @@ -1192,7 +1085,6 @@ std::unique_ptr
transform_lto(std::span inputs, std::span udf, lto_binary_type binary_type, std::span outputs, - void* user_data, null_aware is_null_aware, std::optional in_row_size, rmm::cuda_stream_view stream, @@ -1207,26 +1099,7 @@ std::unique_ptr
transform_lto(std::span inputs, make_outputs(is_null_aware, row_size, inputs, outputs, output_may_be_nullable, {}, stream, mr); auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; - - std::vector input_strides; - - for (auto& input : inputs) { - if (std::holds_alternative(input)) { - input_strides.push_back(0); - } else { - input_strides.push_back(1); - } - } - - rmm::device_uvector d_input_strides(input_strides.size(), stream, mr); - CUDF_CUDA_TRY(cudaMemcpyAsync(d_input_strides.data(), - input_strides.data(), - input_strides.size() * sizeof(cudf::size_type), - cudaMemcpyHostToDevice, - stream.value())); - - auto [kernel_fatbin, shmem_requirement_per_thread] = - dispatch_lto_kernel(is_null_aware == null_aware::YES, inputs, outputs); + auto kernel_fatbin = dispatch_lto_kernel(is_null_aware == null_aware::YES, inputs, outputs); rtcx::memory_fragment fragments[] = { {.data = kernel_fatbin, .type = rtcx::binary_type::FATBIN, .name = "kernel"}, @@ -1241,31 +1114,49 @@ std::unique_ptr
transform_lto(std::span inputs, auto* input_cols = reinterpret_cast(cols.data()); auto* output_cols = reinterpret_cast(input_cols + inputs.size()); - cudf::size_type* p_input_strides = d_input_strides.data(); - - rtcx::kernel_occupancy_config cfg; - - if (shmem_requirement_per_thread > 0) { - cfg = configure_shmem(kernel, shmem_requirement_per_thread); - } else { - cfg = kernel.max_occupancy_config(0, 0); - } - void* args[] = {&num_rows, - &p_stencil, - &user_data, - &input_cols, - &output_cols, - &num_inputs, - &num_outputs, - &p_input_strides}; + auto cfg = kernel.max_occupancy_config(0, 0); + void* user_data = nullptr; - auto per_block_shmem = shmem_requirement_per_thread * cfg.block_size; + void* args[] = {&num_rows, &p_stencil, &user_data, &input_cols, &output_cols}; - kernel.launch({cfg.min_grid_size}, {cfg.block_size}, per_block_shmem, stream, args); + kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); auto finalized = finalize_outputs(is_null_aware, row_size, std::move(output_columns), stream, mr); return std::make_unique
(std::move(finalized)); } +std::unique_ptr unary_op_lto(column_view input, + transform_output output, + std::span udf, + lto_binary_type binary_type, + null_aware is_null_aware, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + transform_input inputs[] = {input}; + transform_output outputs[] = {output}; + auto table = + transform_lto(inputs, udf, binary_type, outputs, is_null_aware, std::nullopt, stream, mr); + auto cols = table->release(); + return std::move(cols[0]); +} + +std::unique_ptr binary_op_lto(column_view lhs, + transform_input rhs, + transform_output output, + std::span udf, + lto_binary_type binary_type, + null_aware is_null_aware, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + transform_input inputs[] = {lhs, rhs}; + transform_output outputs[] = {output}; + auto table = + transform_lto(inputs, udf, binary_type, outputs, is_null_aware, std::nullopt, stream, mr); + auto cols = table->release(); + return std::move(cols[0]); +} + } // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index e302be19f211..43f137d1a6ec 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -686,16 +686,16 @@ ConfigureTest(AST_TEST ast/transform_tests.cpp ast/ast_tree_tests.cpp) # ------------------------------------------------------------------------------------- add_embed(cudf_test_fragments) +add_fragment(cudf_test_fragments FRAGMENT invsqrt SOURCE transform/fragments/invsqrt.cu) + add_fragment( - cudf_test_fragments FRAGMENT transform_profit_operator SOURCE transform/fragments/profit.cu + cudf_test_fragments FRAGMENT sum_of_squares SOURCE transform/fragments/sum_of_squares.cu ) add_fragment( - cudf_test_fragments FRAGMENT transform_invsqrt_operator SOURCE transform/fragments/invsqrt.cu + cudf_test_fragments FRAGMENT decimal_square SOURCE transform/fragments/decimal_square.cu ) -add_fragment(cudf_test_fragments FRAGMENT transform_tpcds_q7 SOURCE transform/fragments/tpcds_q7.cu) - embed(cudf_test_fragments COMPRESSION none) ConfigureTest( diff --git a/cpp/tests/transform/fragments/decimal_square.cu b/cpp/tests/transform/fragments/decimal_square.cu new file mode 100644 index 000000000000..19535ec5d11e --- /dev/null +++ b/cpp/tests/transform/fragments/decimal_square.cu @@ -0,0 +1,28 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +template <> +__device__ void cudf::lto::unary_operator( + numeric::decimal32* __restrict__ out, numeric::decimal32 a) +{ + *out = a * a; +} + +template <> +__device__ void cudf::lto::unary_operator( + numeric::decimal64* __restrict__ out, numeric::decimal64 a) +{ + *out = a * a; +} + +template <> +__device__ void cudf::lto::unary_operator( + numeric::decimal128* __restrict__ out, numeric::decimal128 a) +{ + *out = a * a; +} diff --git a/cpp/tests/transform/fragments/invsqrt.cu b/cpp/tests/transform/fragments/invsqrt.cu index cf7bbebd2f98..3d7a6750b00e 100644 --- a/cpp/tests/transform/fragments/invsqrt.cu +++ b/cpp/tests/transform/fragments/invsqrt.cu @@ -3,25 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -template -__device__ T load(void const* inputs, int input_stride, int arg) -{ - auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); - return *p; -} - -template -__device__ void store(void* outputs, int output_stride, int arg, T value) -{ - auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); - *p = value; -} +#include -extern "C" __device__ int cudf_transform_operation( - void*, long int, void const* inputs, int input_stride, void* outputs, int output_stride) +template <> +__device__ void cudf::lto::unary_operator(float* __restrict__ out, float a) { - auto input = load(inputs, input_stride, 0); - auto result = 1.0f / sqrtf(input); - store(outputs, output_stride, 0, result); - return 0; + *out = 1.0F / sqrtf(a); } diff --git a/cpp/tests/transform/fragments/profit.cu b/cpp/tests/transform/fragments/profit.cu deleted file mode 100644 index 4dc0a8ff3051..000000000000 --- a/cpp/tests/transform/fragments/profit.cu +++ /dev/null @@ -1,49 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -template -__device__ T load(void const* inputs, int input_stride, int arg) -{ - auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); - return *p; -} - -template -__device__ void store(void* outputs, int output_stride, int arg, T value) -{ - auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); - *p = value; -} - -extern "C" __device__ int cudf_transform_operation( - void*, long int, void const* inputs, int input_stride, void* outputs, int output_stride) -{ - // Input schema: - // 0: extended_price(double), - // 1: discount(float), - // 2: tax(float), - // 3: ship_date(int32 YYYYMMDD) - auto extended_price = load(inputs, input_stride, 0); - auto discount = load(inputs, input_stride, 1); - auto tax = load(inputs, input_stride, 2); - auto ship_date = load(inputs, input_stride, 3); - - // Parameters: - // 4: ship_date_cutoff(int32 YYYYMMDD) - auto ship_date_cutoff = load(inputs, input_stride, 4); - - auto base_price = extended_price; - auto disc_price = extended_price * (1.0 - discount); - auto charge = disc_price * (1.0 + tax); - - auto before_cutoff = ship_date <= ship_date_cutoff; - - store(outputs, output_stride, 0, base_price); - store(outputs, output_stride, 1, charge); - store(outputs, output_stride, 2, disc_price); - store(outputs, output_stride, 3, before_cutoff); - - return 0; -} diff --git a/cpp/tests/transform/fragments/sum_of_squares.cu b/cpp/tests/transform/fragments/sum_of_squares.cu new file mode 100644 index 000000000000..6c1d233939f6 --- /dev/null +++ b/cpp/tests/transform/fragments/sum_of_squares.cu @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +template <> +__device__ void cudf::lto::binary_operator(float* __restrict__ out, + float a, + float b) +{ + *out = a * a + b * b; +} diff --git a/cpp/tests/transform/fragments/tpcds_q7.cu b/cpp/tests/transform/fragments/tpcds_q7.cu deleted file mode 100644 index fb667a5f8b6e..000000000000 --- a/cpp/tests/transform/fragments/tpcds_q7.cu +++ /dev/null @@ -1,55 +0,0 @@ - -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -template -__device__ T load(void const* inputs, int input_stride, int arg) -{ - auto p = reinterpret_cast(static_cast(inputs) + arg * input_stride); - return *p; -} - -template -__device__ void store(void* outputs, int output_stride, int arg, T value) -{ - auto p = reinterpret_cast(static_cast(outputs) + arg * output_stride); - *p = value; -} - -/** - * TPC-DS Q7: "Report the profit of each returned item, and the total profit for all returned items, - * for a given date range." - * - */ -extern "C" __device__ int cudf_transform_operation( - void*, long int, void const* inputs, int input_stride, void* outputs, int output_stride) -{ - // Input schema: - // 0: store_sales(double), - // 1: catalog_sales(double), - // 2: web_sales(double), - // 3: store_returns(double), - // 4: catalog_returns(double), - // 5: web_returns(double), - // 6: profit(double), - // 7: profit_loss(double) - auto store_sales = load(inputs, input_stride, 0); - auto catalog_sales = load(inputs, input_stride, 1); - auto web_sales = load(inputs, input_stride, 2); - auto store_returns = load(inputs, input_stride, 3); - auto catalog_returns = load(inputs, input_stride, 4); - auto web_returns = load(inputs, input_stride, 5); - auto profit = load(inputs, input_stride, 6); - auto profit_loss = load(inputs, input_stride, 7); - - double sales = store_sales + catalog_sales + web_sales; - double returns = store_returns + catalog_returns + web_returns; - double net_sales = sales - returns; - double net_profit = profit - profit_loss; - - store(outputs, output_stride, 0, net_profit); - store(outputs, output_stride, 1, net_sales); - return 0; -} diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp index f857e05ffe93..4dec31865b5a 100644 --- a/cpp/tests/transform/transform_lto_test.cpp +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -20,125 +20,79 @@ struct TransformLTOTest : public cudf::test::BaseFixture {}; template using column_wrapper = cudf::test::fixed_width_column_wrapper; -TEST_F(TransformLTOTest, ComputeTpchLineitem) -{ - column_wrapper extended_price{{21168.23, 45983.16, 13309.60, 28955.64}}; - column_wrapper discount{{0.04f, 0.06f, 0.07f, 0.02f}}; - column_wrapper tax{{0.02f, 0.08f, 0.04f, 0.06f}}; - column_wrapper ship_date{{19980901, 19980902, 19980903, 19960115}}; - - column_wrapper ship_date_cutoff{{19980902}}; +template +using decimal_wrapper = cudf::test::fixed_point_column_wrapper; - std::vector inputs{ - extended_price, discount, tax, ship_date, cudf::scalar_column_view{ship_date_cutoff}}; +TEST_F(TransformLTOTest, InvSqrt) +{ + column_wrapper input{{1.0f, 4.0f, 9.0f, 16.0f}}; - std::vector outputs{ - {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::ALL_VALID}, - {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::ALL_VALID}, - {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::ALL_VALID}, - {cudf::data_type{cudf::type_id::BOOL8}, cudf::output_nullability::ALL_VALID}}; + cudf::transform_output output{cudf::data_type{cudf::type_id::FLOAT32}, + cudf::output_nullability::ALL_VALID}; - auto const range = - cudf_test_fragments::file_ranges[cudf_test_fragments::transform_profit_operator]; + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::invsqrt]; std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; - auto result = cudf::transform_lto(inputs, - udf, - cudf::lto_binary_type::FATBIN, - outputs, - nullptr, - cudf::null_aware::NO, - std::nullopt, - cudf::test::get_default_stream()); + auto result = cudf::unary_op_lto(input, + output, + udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + cudf::test::get_default_stream()); + + column_wrapper expected{{1.0f, 0.5f, 0.33333334f, 0.25f}}; - column_wrapper expected_base_price{{21168.23, 45983.16, 13309.60, 28955.64}}; - column_wrapper expected_disc_price{{21168.23 * (1.0 - static_cast(0.04f)), - 45983.16 * (1.0 - static_cast(0.06f)), - 13309.60 * (1.0 - static_cast(0.07f)), - 28955.64 * (1.0 - static_cast(0.02f))}}; - column_wrapper expected_charge{ - {(21168.23 * (1.0 - static_cast(0.04f))) * (1.0 + static_cast(0.02f)), - (45983.16 * (1.0 - static_cast(0.06f))) * (1.0 + static_cast(0.08f)), - (13309.60 * (1.0 - static_cast(0.07f))) * (1.0 + static_cast(0.04f)), - (28955.64 * (1.0 - static_cast(0.02f))) * (1.0 + static_cast(0.06f))}}; - column_wrapper expected_before_cutoff{{true, true, false, true}}; - - cudf::test::detail::expect_columns_equivalent( - result->view().column(1), expected_charge, cudf::test::debug_output_level::FIRST_ERROR, 64); - cudf::test::detail::expect_columns_equivalent( - result->view().column(2), expected_disc_price, cudf::test::debug_output_level::FIRST_ERROR, 64); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view().column(3), expected_before_cutoff); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); } -TEST_F(TransformLTOTest, InvSqrtOperator) +TEST_F(TransformLTOTest, SumOfSquares) { - column_wrapper input{{1.0f, 4.0f, 9.0f, 16.0f}}; - - std::vector inputs{input}; + column_wrapper lhs{{1.0f, 4.0f, 9.0f, 16.0f}}; + column_wrapper rhs{{1.0f, 2.0f, 2.0f, 10.0f}}; - std::vector outputs{ - {cudf::data_type{cudf::type_id::FLOAT32}, cudf::output_nullability::ALL_VALID}}; + cudf::transform_output output{cudf::data_type{cudf::type_id::FLOAT32}, + cudf::output_nullability::ALL_VALID}; - auto const range = - cudf_test_fragments::file_ranges[cudf_test_fragments::transform_invsqrt_operator]; + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::sum_of_squares]; std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; - auto result = cudf::transform_lto(inputs, + auto result = cudf::binary_op_lto(lhs, + rhs, + output, udf, cudf::lto_binary_type::FATBIN, - outputs, - nullptr, cudf::null_aware::NO, - std::nullopt, cudf::test::get_default_stream()); - column_wrapper expected{{1.0f, 0.5f, 0.33333334f, 0.25f}}; + column_wrapper expected{{2.0f, 20.0f, 85.0f, 356.0f}}; - cudf::test::detail::expect_columns_equivalent( - result->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, 64); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); } -TEST_F(TransformLTOTest, TPCDS_Q7) +TEST_F(TransformLTOTest, Decimal32Square) { - column_wrapper store_sales{{100.0, 20.0, 0.0, 45.5}}; - column_wrapper catalog_sales{{10.0, 5.0, 3.0, 4.5}}; - column_wrapper web_sales{{1.0, 0.0, 7.0, 0.5}}; - column_wrapper store_returns{{4.0, 1.0, 0.0, 5.0}}; - column_wrapper catalog_returns{{0.5, 0.0, 1.0, 0.5}}; - column_wrapper web_returns{{0.5, 0.0, 2.0, 0.0}}; - column_wrapper profit{{30.0, 5.0, -2.0, 8.0}}; - column_wrapper profit_loss{{3.0, 1.0, 4.0, 0.5}}; - - std::vector inputs{store_sales, - catalog_sales, - web_sales, - store_returns, - catalog_returns, - web_returns, - profit, - profit_loss}; - - std::vector outputs{ - {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::ALL_VALID}, - {cudf::data_type{cudf::type_id::FLOAT64}, cudf::output_nullability::ALL_VALID}}; - - auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::transform_tpcds_q7]; - std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + auto test_type = []() { + decimal_wrapper input{{1, 2, 3}, numeric::scale_type{-2}}; - auto result = cudf::transform_lto(inputs, - udf, - cudf::lto_binary_type::FATBIN, - outputs, - nullptr, - cudf::null_aware::NO, - std::nullopt, - cudf::test::get_default_stream()); + cudf::transform_output output{cudf::data_type{cudf::type_to_id(), numeric::scale_type{-4}}, + cudf::output_nullability::ALL_VALID}; + + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::decimal_square]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::unary_op_lto(input, + output, + udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + cudf::test::get_default_stream()); + + decimal_wrapper expected{{1, 4, 9}, numeric::scale_type{-4}}; - column_wrapper expected_net_profit{{27.0, 4.0, -6.0, 7.5}}; - column_wrapper expected_net_sales{{106.0, 24.0, 7.0, 45.0}}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + }; - cudf::test::detail::expect_columns_equivalent( - result->view().column(0), expected_net_profit, cudf::test::debug_output_level::FIRST_ERROR, 64); - cudf::test::detail::expect_columns_equivalent( - result->view().column(1), expected_net_sales, cudf::test::debug_output_level::FIRST_ERROR, 64); + test_type.operator()(); + test_type.operator()(); + test_type.operator()(); } From dc95bba9818deaf9cb4ae8937bd720e95a83c7a2 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 13 May 2026 18:46:50 +0000 Subject: [PATCH 171/254] Update CMakeLists.txt to enhance library linking and update copyright year in compiled_binaryop.cpp --- cpp/CMakeLists.txt | 7 ++++++- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bafc5b8c3ada..d5054ca79652 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -373,7 +373,12 @@ macro(add_fragment) CUDA_STANDARD_REQUIRED ON CUDA_VISIBILITY_PRESET hidden ) - target_link_libraries(${OBJECT_ID} PRIVATE CCCL::CCCL) + target_link_libraries( + ${OBJECT_ID} + PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ + PRIVATE $ $ + ZLIB::ZLIB nvcomp::nvcomp kvikio::kvikio nanoarrow::nanoarrow zstd + ) target_include_directories( ${OBJECT_ID} PRIVATE "$" "$" diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 7d458f95d556..8b7372ce3a15 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -141,7 +141,7 @@ void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) cudf::lto_binary_type::FATBIN, null_aware ? cudf::null_aware::YES : cudf::null_aware::NO ); - + // use number of bytes read and written to global memory state.add_global_memory_reads(num_rows); state.add_global_memory_reads(num_rows); @@ -167,7 +167,7 @@ void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) .set_name("lto_binary_op_" BM_STRINGIFY(name)) \ .add_int64_axis("num_rows", {10'000, 100'000, 1'000'000, 10'000'000, 100'000'000}) - + #define build_name_lto(a, b, c, d) a##_##b##_##c##_##d##_lto From 40fd3ebe747eabb7f255e64f9f7bf470d9fed163 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 13 May 2026 19:03:32 +0000 Subject: [PATCH 172/254] Add documentation for JIT embedding functions in CMake files --- cpp/CMakeLists.txt | 4 +++ cpp/librtcx/embed.cmake | 60 +++++++++++++++-------------------------- 2 files changed, 26 insertions(+), 38 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d5054ca79652..b405f49fbc2d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -323,6 +323,10 @@ if(NOT BUILD_SHARED_LIBS) endif() endif() +# This macro is used to create object libraries for JIT compilation fragments, and embed them as +# fatbins in the final library. It compiles the specified source file with the appropriate flags to +# generate a fatbin containing the specified kernel instance, and then embeds that fatbin in the +# final library with metadata that allows it to be looked up at runtime. macro(add_fragment) set(TARGET ${ARGV0}) set(ONE_VALUE_ARGS FRAGMENT SOURCE KERNEL_ONLY KERNEL_INSTANCE) diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index a55c01cfd107..706f2214185e 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -11,6 +11,10 @@ if(NOT TARGET zstd) ) endif() +# This function initializes a target for JIT embedding. It must be called before any calls to +# embed_includes() or embed_blob() for the target. It sets up necessary variables and state to track +# the registered files and dependencies for the target. The TARGET argument specifies the name of +# the target being initialized. function(add_embed) set(TARGET ${ARGV0}) set(OPTIONS "") @@ -46,24 +50,17 @@ function(embed_includes) ) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - if(NOT DEFINED TARGET) - message(FATAL_ERROR "TARGET argument is required") - endif() - - if(NOT ${TARGET}_INITIALIZED) - message(FATAL_ERROR "Target '${TARGET}' has not been initialized with add_embed()") - endif() - - if(NOT ARG_COPY_DIRECTORY) - message(FATAL_ERROR "COPY_DIRECTORY argument is required") + if(NOT DEFINED TARGET OR NOT ${TARGET}_INITIALIZED) + message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") endif() - if(NOT ARG_DEST_DIRECTORY) - message(FATAL_ERROR "DEST_DIRECTORY argument is required") - endif() - - if(NOT ARG_INCLUDE_DIRECTORIES) - message(FATAL_ERROR "INCLUDE_DIRECTORIES argument is required") + if(NOT ARG_COPY_DIRECTORY + OR NOT ARG_DEST_DIRECTORY + OR NOT ARG_INCLUDE_DIRECTORIES + ) + message( + FATAL_ERROR "COPY_DIRECTORY, DEST_DIRECTORY, and INCLUDE_DIRECTORIES arguments are required" + ) endif() if(NOT ARG_FILES) @@ -136,24 +133,15 @@ function(embed_blob) set(MULTI_VALUE_ARGS ARRAY_IDS ARRAY_VALUES) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - if(NOT DEFINED TARGET) - message(FATAL_ERROR "TARGET argument is required") - endif() - - if(NOT ${TARGET}_INITIALIZED) - message(FATAL_ERROR "Target '${TARGET}' has not been initialized with add_embed()") - endif() - - if(NOT ARG_ID) - message(FATAL_ERROR "ID argument is required") - endif() - - if(NOT ARG_FILE) - message(FATAL_ERROR "FILE argument is required") + if(NOT DEFINED TARGET OR NOT ${TARGET}_INITIALIZED) + message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") endif() - if(NOT ARG_DEST) - message(FATAL_ERROR "DEST argument is required") + if(NOT ARG_ID + OR NOT ARG_FILE + OR NOT ARG_DEST + ) + message(FATAL_ERROR "ID, FILE, and DEST arguments are required") endif() set(SOURCE_FILE_IDS ${${TARGET}__embed__source_file_ids}) @@ -233,12 +221,8 @@ function(embed) set(MULTI_VALUE_ARGS "") cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - if(NOT DEFINED TARGET) - message(FATAL_ERROR "TARGET argument is required") - endif() - - if(NOT ${TARGET}_INITIALIZED) - message(FATAL_ERROR "Target '${TARGET}' has not been initialized with add_embed()") + if(NOT DEFINED TARGET OR NOT ${TARGET}_INITIALIZED) + message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") endif() if(NOT DEFINED ARG_COMPRESSION) From 6e3840e5a7f313d4291dceddb0475cb6d9295500 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 13 May 2026 19:27:22 +0000 Subject: [PATCH 173/254] update --- cpp/doxygen/developer_guide/JIT.md | 39 +++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/cpp/doxygen/developer_guide/JIT.md b/cpp/doxygen/developer_guide/JIT.md index b35afe1c7a0a..0d4eebe00857 100644 --- a/cpp/doxygen/developer_guide/JIT.md +++ b/cpp/doxygen/developer_guide/JIT.md @@ -2,13 +2,40 @@ # Terminologies +## IR -FATBIN -CUBIN -LTO-IR -PTX -OBJ -SASS + +## CUDF ROW-IR + + +## LTO-IR + + +## NVVM-IR + + +## PTX + + +## .so + + +## SASS + + +## FATBIN + + +## CUBIN + + +## JIT-Compilation + + +## JIT-Linking + + +## PCH # Normal CUDA Kernel From 630beab230d974466c1fee4153401bdab7f94083 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 13 May 2026 20:00:04 +0000 Subject: [PATCH 174/254] revert non-librtcx changes --- cpp/CMakeLists.txt | 235 +---- cpp/benchmarks/CMakeLists.txt | 10 - cpp/benchmarks/binaryop/compiled_binaryop.cpp | 83 +- cpp/benchmarks/binaryop/fragments/add.cu | 25 - cpp/benchmarks/binaryop/fragments/null_max.cu | 26 - .../Modules/JitifyPreprocessKernels.cmake | 82 ++ cpp/cmake/thirdparty/get_jitify.cmake | 25 + cpp/doxygen/developer_guide/JIT.md | 116 -- cpp/doxygen/developer_guide/JIT_LTO.md | 989 ++++++++++++++++++ .../cudf/column/column_device_view_base.cuh | 4 +- cpp/include/cudf/context.hpp | 6 +- .../cudf/detail/normalizing_iterator.cuh | 12 +- cpp/include/cudf/detail/operation-udf.hpp | 4 +- cpp/include/cudf/detail/utilities/assert.cuh | 4 +- .../detail/floating_conversion.hpp | 10 +- cpp/include/cudf/fixed_point/fixed_point.hpp | 12 +- cpp/include/cudf/fixed_point/temporary.hpp | 6 +- cpp/include/cudf/jit/transform_operator.cuh | 29 - cpp/include/cudf/jit/type_tags.cuh | 56 - .../detail/convert/string_to_float.cuh | 5 + cpp/include/cudf/strings/string_view.cuh | 9 +- cpp/include/cudf/strings/string_view.hpp | 11 +- cpp/include/cudf/transform.hpp | 91 -- cpp/include/cudf/types.hpp | 7 +- cpp/include/cudf/utilities/export.hpp | 6 +- cpp/include/cudf/utilities/span.hpp | 5 +- cpp/include/cudf/wrappers/dictionary.hpp | 1 + cpp/src/binaryop/binaryop.cpp | 33 +- cpp/src/binaryop/jit/kernel.cu | 46 +- cpp/src/io/utilities/getenv_or.hpp | 13 - cpp/src/jit/cache.cpp | 160 +++ cpp/src/jit/cache.hpp | 63 ++ cpp/src/jit/helpers.cpp | 46 +- cpp/src/jit/helpers.hpp | 9 +- cpp/src/jit/jit.cpp | 441 -------- cpp/src/jit/jit.hpp | 102 -- cpp/src/jit/row_ir.cpp | 1 + cpp/src/join/filter_join_indices_jit.cu | 98 +- cpp/src/join/jit/filter_join_kernel.cu | 13 - .../rolling/detail/rolling_fixed_window.cu | 9 +- cpp/src/rolling/detail/rolling_jit.cuh | 114 -- cpp/src/rolling/detail/rolling_jit.hpp | 50 + cpp/src/rolling/detail/rolling_udf.cuh | 104 +- .../rolling/detail/rolling_variable_window.cu | 20 +- cpp/src/rolling/grouped_rolling.cu | 26 +- cpp/src/rolling/jit/kernel.cu | 69 +- cpp/src/runtime/context.cpp | 117 +-- cpp/src/runtime/context.hpp | 43 +- cpp/src/transform/jit/kernel.cu | 116 +- cpp/src/transform/transform.cu | 296 +----- cpp/tests/CMakeLists.txt | 24 - .../transform/fragments/decimal_square.cu | 28 - cpp/tests/transform/fragments/invsqrt.cu | 12 - .../transform/fragments/sum_of_squares.cu | 14 - .../integration/unary_transform_test.cpp | 2 +- cpp/tests/transform/transform_lto_test.cpp | 98 -- 56 files changed, 1781 insertions(+), 2255 deletions(-) delete mode 100644 cpp/benchmarks/binaryop/fragments/add.cu delete mode 100644 cpp/benchmarks/binaryop/fragments/null_max.cu create mode 100644 cpp/cmake/Modules/JitifyPreprocessKernels.cmake create mode 100644 cpp/cmake/thirdparty/get_jitify.cmake delete mode 100644 cpp/doxygen/developer_guide/JIT.md create mode 100644 cpp/doxygen/developer_guide/JIT_LTO.md delete mode 100644 cpp/include/cudf/jit/transform_operator.cuh delete mode 100644 cpp/include/cudf/jit/type_tags.cuh create mode 100644 cpp/src/jit/cache.cpp create mode 100644 cpp/src/jit/cache.hpp delete mode 100644 cpp/src/jit/jit.cpp delete mode 100644 cpp/src/jit/jit.hpp delete mode 100644 cpp/src/rolling/detail/rolling_jit.cuh create mode 100644 cpp/src/rolling/detail/rolling_jit.hpp delete mode 100644 cpp/tests/transform/fragments/decimal_square.cu delete mode 100644 cpp/tests/transform/fragments/invsqrt.cu delete mode 100644 cpp/tests/transform/fragments/sum_of_squares.cu delete mode 100644 cpp/tests/transform/transform_lto_test.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e1995ddd90ff..030023801a5b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -20,7 +20,7 @@ rapids_cuda_init_architectures(CUDF) project( CUDF VERSION "${RAPIDS_VERSION}" - LANGUAGES C CXX CUDA ASM + LANGUAGES C CXX CUDA ) if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 12.2) @@ -38,6 +38,7 @@ option(USE_NVTX "Build with NVTX support" ON) option(BUILD_TESTS "Configure CMake to build tests" ON) option(BUILD_BENCHMARKS "Configure CMake to build benchmarks" OFF) option(BUILD_SHARED_LIBS "Build cuDF shared libraries" ON) +option(JITIFY_USE_CACHE "Use a file cache for JIT compiled kernels" ON) option(CUDF_BUILD_TESTUTIL "Whether to build the test utilities contained in libcudf" ON) mark_as_advanced(CUDF_BUILD_TESTUTIL) option(CUDF_LARGE_STRINGS_DISABLED "Build with large string support disabled" OFF) @@ -109,6 +110,7 @@ message(VERBOSE "CUDF: Build with NVTX support: ${USE_NVTX}") message(VERBOSE "CUDF: Configure CMake to build tests: ${BUILD_TESTS}") message(VERBOSE "CUDF: Configure CMake to build benchmarks: ${BUILD_BENCHMARKS}") message(VERBOSE "CUDF: Build cuDF shared libraries: ${BUILD_SHARED_LIBS}") +message(VERBOSE "CUDF: Use a file cache for JIT compiled kernels: ${JITIFY_USE_CACHE}") message(VERBOSE "CUDF: Build with per-thread default stream: ${CUDF_USE_PER_THREAD_DEFAULT_STREAM}") message( VERBOSE @@ -343,6 +345,9 @@ endif() create_logger_macros(CUDF "cudf::default_logger()" include/cudf) +# find jitify +include(cmake/thirdparty/get_jitify.cmake) + # find NVTX include(cmake/thirdparty/get_nvtx.cmake) @@ -372,6 +377,9 @@ if(CUDF_BUILD_TESTUTIL) include(cmake/thirdparty/get_gtest.cmake) endif() +# preprocess jitify-able kernels +include(cmake/Modules/JitifyPreprocessKernels.cmake) + # find KvikIO include(cmake/thirdparty/get_kvikio.cmake) @@ -384,9 +392,6 @@ include(cmake/thirdparty/get_thread_pool.cmake) # find zstd include(cmake/thirdparty/get_zstd.cmake) -# JIT Embedding helper functions -include(librtcx/embed.cmake) - # Workaround until https://github.com/rapidsai/rapids-cmake/issues/176 is resolved if(NOT BUILD_SHARED_LIBS) include("${rapids-cmake-dir}/export/find_package_file.cmake") @@ -404,193 +409,6 @@ if(NOT BUILD_SHARED_LIBS) endif() endif() -# This macro is used to create object libraries for JIT compilation fragments, and embed them as -# fatbins in the final library. It compiles the specified source file with the appropriate flags to -# generate a fatbin containing the specified kernel instance, and then embeds that fatbin in the -# final library with metadata that allows it to be looked up at runtime. -macro(add_fragment) - set(TARGET ${ARGV0}) - set(ONE_VALUE_ARGS FRAGMENT SOURCE KERNEL_ONLY KERNEL_INSTANCE) - set(MULTI_VALUE_ARGS DEFINITIONS ARRAY_IDS ARRAY_VALUES) - cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - - if(NOT ARG_FRAGMENT) - message(FATAL_ERROR "add_fragment requires FRAGMENT argument") - endif() - - if(NOT ARG_SOURCE) - message(FATAL_ERROR "add_fragment requires SOURCE argument") - endif() - - set(OBJECT_ID ${TARGET}_${ARG_FRAGMENT}) - add_library(${OBJECT_ID} OBJECT ${ARG_SOURCE}) - target_compile_options( - ${OBJECT_ID} PRIVATE --compress-mode=size --expt-relaxed-constexpr --extended-lambda - ) - - if(DEFINED ARG_KERNEL_ONLY AND ARG_KERNEL_ONLY) - # ensure that the FATBIN symbols only contain the specified kernel - target_compile_options(${OBJECT_ID} PRIVATE -Xnvlink=--kernels-used=cudf_kernel_entry) - endif() - - if(ARG_KERNEL_INSTANCE) - set(INSTANTIATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/${TARGET}/instantiations/${ARG_FRAGMENT}") - file( - GENERATE - OUTPUT "${INSTANTIATION_DIR}/cudf/detail/kernel-instance.hpp" - CONTENT "#pragma once\n#define CUDF_KERNEL_INSTANCE ${ARG_KERNEL_INSTANCE}" - ) - target_include_directories(${OBJECT_ID} PRIVATE ${INSTANTIATION_DIR}) - endif() - - target_compile_definitions(${OBJECT_ID} PRIVATE CUDF_DISABLE_EXPORTS ${ARG_DEFINITIONS}) - set_target_properties( - ${OBJECT_ID} - PROPERTIES CUDA_SEPARABLE_COMPILATION ON - CUDA_FATBIN_COMPILATION ON - POSITION_INDEPENDENT_CODE ON - INTERPROCEDURAL_OPTIMIZATION ON - CXX_STANDARD 20 - CXX_STANDARD_REQUIRED ON - CXX_EXTENSIONS ON - CXX_VISIBILITY_PRESET hidden - CUDA_STANDARD 20 - CUDA_STANDARD_REQUIRED ON - CUDA_VISIBILITY_PRESET hidden - ) - target_link_libraries( - ${OBJECT_ID} - PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ - PRIVATE $ $ - ZLIB::ZLIB nvcomp::nvcomp kvikio::kvikio nanoarrow::nanoarrow zstd - ) - target_include_directories( - ${OBJECT_ID} PRIVATE "$" - "$" - ) - - embed_blob( - ${TARGET} FILE $ DEST fragments/${ARG_FRAGMENT}.fatbin ID - ${ARG_FRAGMENT} ARRAY_IDS ${ARG_ARRAY_IDS} ARRAY_VALUES ${ARG_ARRAY_VALUES} - ) -endmacro() - -add_embed(cudf_fragments) - -foreach(TYPE IN ITEMS uint8 uint16 uint32 int8 int16 int32 int64 float32 float64 uint64 decimal32 - decimal64 decimal128 -) - foreach(NULL_AWARE IN ITEMS 0 1) - set(FRAGMENT_NAME unop_lto_kernel) - set(VARIANT_NAME ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__element_${TYPE}) - add_fragment( - cudf_fragments - FRAGMENT - ${VARIANT_NAME} - SOURCE - src/transform/jit/kernel.cu - KERNEL_INSTANCE - "cudf::jit::transform_kernel<${NULL_AWARE}, false, cudf::jit::type_list>, cudf::jit::type_list>>" - DEFINITIONS - CUDF_LTO_MODE - ARRAY_IDS - ${FRAGMENT_NAME}_FILE_INDEX - ${FRAGMENT_NAME}_NULL_AWARE - ${FRAGMENT_NAME}_TYPE - ARRAY_VALUES - ${cudf_fragments_FILE_INDEX} - ${NULL_AWARE} - ${TYPE} - ) - endforeach() -endforeach() - -foreach(TYPE IN ITEMS uint8 uint16 uint32 uint64 int8 int16 int32 int64 float32 float64 decimal32 - decimal64 decimal128 -) - foreach(NULL_AWARE IN ITEMS 0 1) - foreach(RHS_IS_SCALAR IN ITEMS 0 1) - set(FRAGMENT_NAME binop_lto_kernel) - set(VARIANT_NAME - ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__element_${TYPE}__rhs_is_scalar_${RHS_IS_SCALAR} - ) - add_fragment( - cudf_fragments - FRAGMENT - ${VARIANT_NAME} - SOURCE - src/transform/jit/kernel.cu - KERNEL_INSTANCE - "cudf::jit::transform_kernel<${NULL_AWARE}, false, cudf::jit::type_list, cudf::jit::column_accessor<1, cudf::column_device_view_core, cudf::tags::${TYPE}, ${RHS_IS_SCALAR}>>, cudf::jit::type_list>>" - DEFINITIONS - CUDF_LTO_MODE - ARRAY_IDS - ${FRAGMENT_NAME}_FILE_INDEX - ${FRAGMENT_NAME}_NULL_AWARE - ${FRAGMENT_NAME}_TYPE - ${FRAGMENT_NAME}_RHS_IS_SCALAR - ARRAY_VALUES - ${cudf_fragments_FILE_INDEX} - ${NULL_AWARE} - ${TYPE} - ${RHS_IS_SCALAR} - ) - endforeach() - endforeach() -endforeach() - -embed(cudf_fragments COMPRESSION none) - -add_embed(cudf_jit_embed) - -embed_includes( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/librtcx/libcxx DEST_DIRECTORY - librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx -) - -embed_includes( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf DEST_DIRECTORY - cudf/cpp/include/cudf INCLUDE_DIRECTORIES cudf/cpp/include -) - -embed_includes( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/jit DEST_DIRECTORY cudf/cpp/src/jit - INCLUDE_DIRECTORIES cudf/cpp/src -) - -embed_includes( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/binaryop/jit DEST_DIRECTORY - cudf/cpp/src/binaryop/jit INCLUDE_DIRECTORIES cudf/cpp/src -) - -embed_includes( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/join/jit DEST_DIRECTORY - cudf/cpp/src/join/jit INCLUDE_DIRECTORIES cudf/cpp/src -) - -embed_includes( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling DEST_DIRECTORY - cudf/cpp/src/rolling INCLUDE_DIRECTORIES cudf/cpp/src -) - -embed_includes( - cudf_jit_embed COPY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/transform/jit DEST_DIRECTORY - cudf/cpp/src/transform/jit INCLUDE_DIRECTORIES cudf/cpp/src -) - -get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) - -foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) - cmake_path(GET INC_DIR FILENAME INC_DIR_NAME) - - embed_includes( - cudf_jit_embed COPY_DIRECTORY ${INC_DIR} DEST_DIRECTORY CCCL/libcudacxx/${INC_DIR_NAME} - INCLUDE_DIRECTORIES CCCL/libcudacxx/${INC_DIR_NAME} - ) -endforeach() - -embed(cudf_jit_embed COMPRESSION zstd) - # ################################################################################################## # * library targets ------------------------------------------------------------------------------- add_library( @@ -838,8 +656,8 @@ add_library( src/io/utilities/row_selection.cpp src/io/utilities/type_inference.cu src/io/utilities/trie.cu + src/jit/cache.cpp src/jit/helpers.cpp - src/jit/jit.cpp src/jit/parser.cpp src/jit/row_ir.cpp src/jit/util.cpp @@ -1124,13 +942,10 @@ add_library( src/utilities/traits.cpp src/utilities/type_checks.cpp src/utilities/type_dispatcher.cpp - librtcx/rtcx.cpp - ${cudf_jit_embed_SOURCE_DIR}/cudf_jit_embed.s - ${cudf_fragments_SOURCE_DIR}/cudf_fragments.s ) -add_dependencies(cudf cudf_jit_embed cudf_fragments) - +# Anything that includes jitify needs to be compiled with _FILE_OFFSET_BITS=64 due to a limitation +# in how conda builds glibc set_source_files_properties( src/binaryop/binaryop.cpp src/jit/cache.cpp @@ -1139,6 +954,7 @@ set_source_files_properties( src/rolling/grouped_rolling.cu src/rolling/rolling.cu src/transform/transform.cu + PROPERTIES COMPILE_DEFINITIONS "_FILE_OFFSET_BITS=64" ) set_property( @@ -1203,16 +1019,14 @@ target_compile_options( # Specify include paths for the current target and dependents target_include_directories( cudf - PUBLIC "$" "$" + PUBLIC "$" + "$" + "$" "$" PRIVATE "$" "$" "$" "$" - "$" - "$" - "$" - "$" INTERFACE "$" ) @@ -1221,6 +1035,20 @@ target_compile_definitions( "$:${CUDF_CUDA_DEFINITIONS}>>" ) +# Disable Jitify log printing. See https://github.com/NVIDIA/jitify/issues/79 +target_compile_definitions(cudf PRIVATE "JITIFY_PRINT_LOG=0") + +if(JITIFY_USE_CACHE) + # Instruct src/jit/cache what version of cudf we are building so it can compute a cal-ver cache + # directory. We isolate this definition to the single source so it doesn't effect compiling + # caching for all of libcudf + set_property( + SOURCE src/jit/cache.cpp + APPEND + PROPERTY COMPILE_DEFINITIONS "JITIFY_USE_CACHE" "CUDF_VERSION=${PROJECT_VERSION}" + ) +endif() + # Per-thread default stream if(CUDF_USE_PER_THREAD_DEFAULT_STREAM) target_compile_definitions( @@ -1250,6 +1078,9 @@ target_compile_definitions(cudf PRIVATE $<$:CUDF_ # https://github.com/NVIDIA/cccl/pull/2844 target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_AVOID_SORT_UNROLL=1) +# Compile stringified JIT sources first +add_dependencies(cudf jitify_preprocess_run) + # Specify the target module library dependencies target_link_libraries( cudf diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 242523d78972..066716e85f25 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -96,13 +96,6 @@ function(ConfigureNVBench CMAKE_BENCH_NAME) ) endfunction() -# ################################################################################################## -# * LTO Fragments ---------------------------------------------------------------------------- -add_embed(cudf_benchmark_fragments) -add_fragment(cudf_benchmark_fragments FRAGMENT add SOURCE binaryop/fragments/add.cu) -add_fragment(cudf_benchmark_fragments FRAGMENT null_max SOURCE binaryop/fragments/null_max.cu) -embed(cudf_benchmark_fragments COMPRESSION none) - # ################################################################################################## # * copying benchmarks ---------------------------------------------------------------------------- ConfigureNVBench( @@ -396,10 +389,7 @@ ConfigureNVBench(AST_NVBENCH ast/polynomials.cpp ast/transform.cpp) # * binaryop benchmark ---------------------------------------------------------------------------- ConfigureNVBench( BINARYOP_NVBENCH binaryop/binaryop.cpp binaryop/compiled_binaryop.cpp binaryop/polynomials.cpp - ${cudf_benchmark_fragments_SOURCE_DIR}/cudf_benchmark_fragments.s ) -target_include_directories(BINARYOP_NVBENCH PRIVATE ${cudf_benchmark_fragments_SOURCE_DIR}) -add_dependencies(BINARYOP_NVBENCH cudf_benchmark_fragments) # ################################################################################################## # * transform benchmark diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 8b7372ce3a15..9befc76d2b45 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -1,14 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #include #include -#include -#include #include template @@ -98,82 +96,3 @@ BINARYOP_BENCHMARK_DEFINE(duration_ms, duration_ns, NULL_EQUALS, bool BINARYOP_BENCHMARK_DEFINE(duration_ms, duration_ns, NULL_NOT_EQUALS, bool); BINARYOP_BENCHMARK_DEFINE(decimal32, decimal32, NULL_MAX, decimal32); BINARYOP_BENCHMARK_DEFINE(timestamp_D, timestamp_s, NULL_MIN, timestamp_s); - - - -template -void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) -{ - auto const num_rows = static_cast(state.get_int64("num_rows")); - - auto const source_table = create_random_table( - {cudf::type_to_id(), cudf::type_to_id()}, row_count{num_rows}); - - auto lhs = cudf::column_view(source_table->get_column(0)); - auto rhs = cudf::column_view(source_table->get_column(1)); - - size_t fragment_id = 0; - bool null_aware = false; - - switch (binop) { - case cudf::binary_operator::ADD: { - fragment_id = cudf_benchmark_fragments::add; - null_aware = false; - } break; - case cudf::binary_operator::NULL_MAX: { - fragment_id = cudf_benchmark_fragments::null_max; - null_aware = true; - } break; - default: throw std::runtime_error("Unsupported binary operator for LTO benchmark"); - } - - // Call once for hot cache. - cudf::transform_output output{cudf::data_type{cudf::type_to_id()}, - cudf::output_nullability::ALL_VALID}; - - auto const range = cudf_benchmark_fragments::file_ranges[fragment_id]; - std::span udf{cudf_benchmark_fragments::files.subspan(range[0], range[1])}; - - auto result = cudf::binary_op_lto(source_table->get_column(0), - source_table->get_column(1), - output, - udf, - cudf::lto_binary_type::FATBIN, - null_aware ? cudf::null_aware::YES : cudf::null_aware::NO - ); - - // use number of bytes read and written to global memory - state.add_global_memory_reads(num_rows); - state.add_global_memory_reads(num_rows); - state.add_global_memory_writes(num_rows); - - state.exec(nvbench::exec_tag::sync, [&](nvbench::launch&) { - cudf::binary_op_lto(source_table->get_column(0), - source_table->get_column(1), - output, - udf, - cudf::lto_binary_type::FATBIN, - null_aware ? cudf::null_aware::YES : cudf::null_aware::NO); - }); -} - - -#define BM_LTO_BINARYOP_BENCHMARK_DEFINE(name, lhs, rhs, bop, tout) \ - static void name(::nvbench::state& st) \ - { \ - ::BM_lto_binaryop(st, ::cudf::binary_operator::bop); \ - } \ - NVBENCH_BENCH(name) \ - .set_name("lto_binary_op_" BM_STRINGIFY(name)) \ - .add_int64_axis("num_rows", {10'000, 100'000, 1'000'000, 10'000'000, 100'000'000}) - - -#define build_name_lto(a, b, c, d) a##_##b##_##c##_##d##_lto - - -#define LTO_BINARYOP_BENCHMARK_DEFINE(lhs, rhs, bop, tout) \ - BM_LTO_BINARYOP_BENCHMARK_DEFINE(build_name_lto(bop, lhs, rhs, tout), lhs, rhs, bop, tout) - - -LTO_BINARYOP_BENCHMARK_DEFINE(float, float, ADD, float); -LTO_BINARYOP_BENCHMARK_DEFINE(decimal32, decimal32, NULL_MAX, decimal32); diff --git a/cpp/benchmarks/binaryop/fragments/add.cu b/cpp/benchmarks/binaryop/fragments/add.cu deleted file mode 100644 index e5879e950ffe..000000000000 --- a/cpp/benchmarks/binaryop/fragments/add.cu +++ /dev/null @@ -1,25 +0,0 @@ - - -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -template <> -__device__ void cudf::lto::binary_operator(int32_t* __restrict__ out, - int32_t a, - int32_t b) -{ - *out = a + b; -} - -template <> -__device__ void cudf::lto::binary_operator(float* __restrict__ out, - float a, - float b) -{ - *out = a + b; -} diff --git a/cpp/benchmarks/binaryop/fragments/null_max.cu b/cpp/benchmarks/binaryop/fragments/null_max.cu deleted file mode 100644 index 2283a17aa98a..000000000000 --- a/cpp/benchmarks/binaryop/fragments/null_max.cu +++ /dev/null @@ -1,26 +0,0 @@ - - -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -#include - -template <> -__device__ void cudf::lto::binary_operator, - cuda::std::optional, - cuda::std::optional>( - cuda::std::optional* __restrict__ out, - cuda::std::optional a, - cuda::std::optional b) -{ - if (a.has_value() || b.has_value()) { - *out = (a.has_value() && (!b.has_value() || (*a > *b))) ? *a : *b; - } else { - *out = cuda::std::nullopt; - } -} diff --git a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake new file mode 100644 index 000000000000..d035e1ea6ab4 --- /dev/null +++ b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake @@ -0,0 +1,82 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +# Create `jitify_preprocess` executable +add_executable(jitify_preprocess "${JITIFY_INCLUDE_DIR}/jitify2_preprocess.cpp") + +target_compile_definitions(jitify_preprocess PRIVATE "_FILE_OFFSET_BITS=64") +rapids_cuda_set_runtime(jitify_preprocess USE_STATIC ON) +target_link_libraries(jitify_preprocess PUBLIC ${CMAKE_DL_LIBS}) + +# Take a list of files to JIT-compile and run them through jitify_preprocess. +function(jit_preprocess_files) + cmake_parse_arguments(ARG "" "SOURCE_DIRECTORY" "FILES" ${ARGN}) + + get_target_property(libcudacxx_raw_includes CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) + set(includes) + foreach(inc IN LISTS libcudacxx_raw_includes CUDAToolkit_INCLUDE_DIRS) + list(APPEND includes "-I${inc}") + endforeach() + foreach(ARG_FILE ${ARG_FILES}) + set(ARG_OUTPUT ${CUDF_GENERATED_INCLUDE_DIR}/include/jit_preprocessed_files/${ARG_FILE}.jit.hpp) + get_filename_component(jit_output_directory "${ARG_OUTPUT}" DIRECTORY) + list(APPEND JIT_PREPROCESSED_FILES "${ARG_OUTPUT}") + + get_filename_component(ARG_OUTPUT_DIR "${ARG_OUTPUT}" DIRECTORY) + + # Note: need to pass _FILE_OFFSET_BITS=64 in COMMAND due to a limitation in how conda builds + # glibc + add_custom_command( + OUTPUT ${ARG_OUTPUT} + DEPENDS jitify_preprocess "${ARG_SOURCE_DIRECTORY}/${ARG_FILE}" + WORKING_DIRECTORY ${ARG_SOURCE_DIRECTORY} + VERBATIM + COMMAND ${CMAKE_COMMAND} -E make_directory "${jit_output_directory}" + COMMAND + "${CMAKE_COMMAND}" -E env LD_LIBRARY_PATH=${CUDAToolkit_LIBRARY_DIR} + $ ${ARG_FILE} -o ${ARG_OUTPUT_DIR} -i -std=c++20 + -remove-unused-globals -D_FILE_OFFSET_BITS=64 -D__CUDACC_RTC__ -DCUDF_RUNTIME_JIT + -I${CUDF_SOURCE_DIR}/include -I${CUDF_SOURCE_DIR}/src ${includes} + --no-preinclude-workarounds --no-replace-pragma-once --diag-suppress=47 --device-int128 + COMMENT "Custom command to JIT-compile files." + ) + endforeach() + set(JIT_PREPROCESSED_FILES + "${JIT_PREPROCESSED_FILES}" + PARENT_SCOPE + ) +endfunction() + +if(NOT (EXISTS "${CUDF_GENERATED_INCLUDE_DIR}/include")) + make_directory("${CUDF_GENERATED_INCLUDE_DIR}/include") +endif() + +jit_preprocess_files( + SOURCE_DIRECTORY ${CUDF_SOURCE_DIR}/src FILES binaryop/jit/kernel.cu rolling/jit/kernel.cu + transform/jit/kernel.cu join/jit/filter_join_kernel.cu +) + +add_custom_target( + jitify_preprocess_run + DEPENDS ${JIT_PREPROCESSED_FILES} + COMMENT "Target representing jitified files." +) + +# when a user requests CMake to clean the build directory +# +# * `cmake --build --target clean` +# * `cmake --build --clean-first` +# * ninja clean +# +# We also remove the jitify2 program cache as well. This ensures that we don't keep older versions +# of the programs in cache +set(cache_path "$ENV{HOME}/.cudf") +if(ENV{LIBCUDF_KERNEL_CACHE_PATH}) + set(cache_path "$ENV{LIBCUDF_KERNEL_CACHE_PATH}") +endif() +cmake_path(APPEND cache_path "${CUDF_VERSION}/") +set_target_properties(jitify_preprocess_run PROPERTIES ADDITIONAL_CLEAN_FILES "${cache_path}") diff --git a/cpp/cmake/thirdparty/get_jitify.cmake b/cpp/cmake/thirdparty/get_jitify.cmake new file mode 100644 index 000000000000..4595b33c5160 --- /dev/null +++ b/cpp/cmake/thirdparty/get_jitify.cmake @@ -0,0 +1,25 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +# Jitify doesn't have a version :/ + +# This function finds Jitify and sets any additional necessary environment variables. +function(find_and_configure_jitify) + rapids_cpm_find( + jitify 2.0.0 + GIT_REPOSITORY https://github.com/NVIDIA/jitify.git + GIT_TAG 44e978b21fc8bdb6b2d7d8d179523c8350db72e5 # jitify2 branch as of 23rd Aug 2025 + GIT_SHALLOW FALSE + DOWNLOAD_ONLY TRUE + ) + set(JITIFY_INCLUDE_DIR + "${jitify_SOURCE_DIR}" + PARENT_SCOPE + ) +endfunction() + +find_and_configure_jitify() diff --git a/cpp/doxygen/developer_guide/JIT.md b/cpp/doxygen/developer_guide/JIT.md deleted file mode 100644 index 0d4eebe00857..000000000000 --- a/cpp/doxygen/developer_guide/JIT.md +++ /dev/null @@ -1,116 +0,0 @@ - - - -# Terminologies -## IR - - -## CUDF ROW-IR - - -## LTO-IR - - -## NVVM-IR - - -## PTX - - -## .so - - -## SASS - - -## FATBIN - - -## CUBIN - - -## JIT-Compilation - - -## JIT-Linking - - -## PCH - -# Normal CUDA Kernel - -# What is LTO? - -# LTO vs AOT vs Source - -# Example - -# Environment Variables - - -# Benchmarking -NVTX ranges -Statistics - - -# Why is My JIT Slow - - -# Jitify - -## How JITIFY Works - -## JITIFY's Limitations - -### Binary Bloat - -### String Source Embedding - -### - - -# PCH & The C++/CUDA AST - -# JIT-LTO -# Comparison to CPU Linking - - -# Adding a CUDF JIT Kernel - -# Adding a CUDF LTO Kernel - - -# LIBRTCX - - -# Kernel Combinatorics in CUDF - -# Progressive Reduction - -# Pre-warming techniques - - - -# Why AST is Slow - - - -# How can NVRTC/C++/NVJITLink Help Us? - - - -# Perf Numbers - -# CUDA -# CUDA + PCH -# LTO-Linked Kernel -# - -# LTO & Optimization Barriers - -# Kernel Variant Management with RTCX - -# What makes this challenging -# Knobs for Each Kernel - -## Adding an LTO-Fragment diff --git a/cpp/doxygen/developer_guide/JIT_LTO.md b/cpp/doxygen/developer_guide/JIT_LTO.md new file mode 100644 index 000000000000..a495c4da2bb4 --- /dev/null +++ b/cpp/doxygen/developer_guide/JIT_LTO.md @@ -0,0 +1,989 @@ +# JIT LTO (Just-In-Time Link-Time Optimization) Guide + +## Background + +### What is JIT LTO? + +[JIT LTO (Just-In-Time Link-Time Optimization)](https://developer.nvidia.com/blog/cuda-12-0-compiler-support-for-runtime-lto-using-nvjitlink-library/) is a CUDA compilation strategy that enables dynamic kernel compilation and linking at runtime. Instead of pre-compiling all possible kernel variants (which would result in an explosion of binary size), JIT LTO compiles kernel **fragments** separately and links them together on-demand when a specific kernel configuration is needed. + +### Fragment Terminology + +A **fragment** is a self-contained, compilable unit of CUDA code that can be linked with other fragments to form a complete kernel. In the JIT LTO system: + +- **Entrypoint Fragment**: The main kernel function that serves as the entry point. This is always the `__global__` kernel function. +- **Device Function Fragments**: Separate fragments containing device functions (e.g., distance computations, filters, post-processing) that are called by the entrypoint kernel. +- **Fragment Key**: A unique identifier for a fragment, typically constructed from template parameters and configuration values. +- **Fatbin**: The compiled binary representation of a fragment, embedded in the executable. + +The key advantage is that device functions can be compiled independently and reused across multiple kernel entrypoints, reducing compilation time and binary size. + +### How It Works + +1. **Build Time**: Fragments are compiled into fatbins and embedded in the executable. +2. **Runtime**: When a kernel needs to be launched: + - The planner identifies which fragments are needed based on the configuration + - Fragments are loaded from the embedded fatbins + - Nvjitlink (Link-Time Optimization) links the fragments together + - The linked kernel is cached and launched + +## Walkthrough Example + +Let's walk through creating a JIT LTO kernel system for a search kernel with templated device functions. + +### Step 1: Define the Kernel and Device Functions + +We start with a kernel that has templated device functions that we want to separate into fragments: + +**`search_kernel.cuh`**: + +```cpp +#pragma once + +#include + +namespace example::detail { + +// Device function for distance computation +template +__device__ float compute_distance_euclidean(T a, T b) { + T diff = a - b; + return diff * diff; +} + +template +__device__ float compute_distance_inner_product(T a, T b) { + return -a * b; // Negative for max inner product search +} + +// Device function for filtering +template +__device__ bool apply_filter_none(uint32_t query_id, IdxT node_id, void* filter_data) { + return true; +} + +template +__device__ bool apply_filter_bitset(uint32_t query_id, IdxT node_id, void* filter_data) { + // Simplified - actual implementation would check bitset + return true; +} + +// Main kernel - will use generic extern device functions +template +__device__ void search_kernel_impl( + const T* dataset, + const T* queries, + IdxT* results, + OutT* distances, // Output distance type + uint32_t num_queries, + uint32_t dataset_size, + void* filter_data) { + + uint32_t query_id = blockIdx.x * blockDim.x + threadIdx.x; + if (query_id >= num_queries) return; + + OutT best_dist = std::numeric_limits::max(); + IdxT best_idx = 0; + + for (IdxT i = 0; i < dataset_size; ++i) { + // Call generic extern device functions (implementations linked from fragments) + if (!apply_filter(query_id, i, filter_data)) continue; + + OutT dist = static_cast(compute_distance(queries[query_id], dataset[i])); + + // Use optimized path if enabled + if constexpr (UseOptimizedPath) { + // Optimized implementation + if (dist < best_dist) { + best_dist = dist; + best_idx = i; + } + } else { + // Standard implementation + if (dist < best_dist) { + best_dist = dist; + best_idx = i; + } + } + } + + results[query_id] = best_idx; + distances[query_id] = best_dist; +} + +} // namespace example::detail +``` + +### Step 2: Create Device Function Fragments + +We'll create separate header files for each device function variant. Each implements the generic function signature that the kernel expects: + +**`compute_distance_euclidean.cuh`**: + +```cpp +#pragma once + +namespace example::detail { + +// Implements the generic compute_distance function for euclidean distance +template +__device__ float compute_distance(T a, T b) { + T diff = a - b; + return diff * diff; +} + +} // namespace example::detail +``` + +**`compute_distance_inner_product.cuh`**: + +```cpp +#pragma once + +namespace example::detail { + +// Implements the generic compute_distance function for inner product +template +__device__ float compute_distance(T a, T b) { + return -a * b; // Negative for max inner product search +} + +} // namespace example::detail +``` + +**`filter_none.cuh`**: + +```cpp +#pragma once + +namespace example::detail { + +// Implements the generic apply_filter function for no filtering +template +__device__ bool apply_filter(uint32_t query_id, IdxT node_id, void* filter_data) { + return true; +} + +} // namespace example::detail +``` + +**`filter_bitset.cuh`**: + +```cpp +#pragma once + +namespace example::detail { + +// Implements the generic apply_filter function for bitset filtering +template +__device__ bool apply_filter(uint32_t query_id, IdxT node_id, void* filter_data) { + // Actual bitset implementation + return true; +} + +} // namespace example::detail +``` + +### Step 3: Create JSON Matrix Files + +JSON matrix files define all the parameter combinations that need to be compiled. The build system uses these to generate `.cu` files from `.cu.in` templates. + +**How JSON Cross-Product Works**: +- The build system computes a modified **Cartesian product** (cross-product) of all parameter combinations. +- **Leaf nodes** are the actual values. These can be strings, numbers, booleans, or `null`, but only strings should be used, even for numbers, for example ``"1"``. +- Related values can be grouped together in a dictionary consisting of single values. Any dictionary key in such a dictionary's ancestry will not be used in the final product, and should be prefixed with `_` to indicate that it is used only for grouping. +- Keys containing only leaf nodes will be used in the final product, and should not be prefixed with `_`. +- The matrix product algorithm will automatically warn if the proper naming convention (`_` prefix or not) is not followed. +- Each group expands to create multiple combinations, and all groups are cross-multiplied. + +For example, if you have: +```json +{ + "_data_type": [{"data_type": "float"}, {"data_type": "half"}], + "_index": [{"idx_type": "uint32_t"}, {"idx_type": "int64_t"}], + "capacity": ["1", "2"] +} +``` + +This generates 2 × 2 × 2 = 8 combinations: +- `{data_type: "float", idx_type: "uint32_t", capacity: "1"}` +- `{data_type: "float", idx_type: "uint32_t", capacity: "2"}` +- `{data_type: "float", idx_type: "int64_t", capacity: "1"}` +- ... and so on + +When a group contains nested arrays (like `veclen: ["1", "4"]`), those are also expanded within that group before the cross-product is computed. + +#### `compute_distance_matrix.json` + +```json +{ + "_distance_type": [ + { + "distance_name": "euclidean", + "header_file": "example/jit_lto_kernels/compute_distance_euclidean.cuh" + }, + { + "distance_name": "inner_product", + "header_file": "example/jit_lto_kernels/compute_distance_inner_product.cuh" + } + ], + "_data_type": [ + { + "data_type": "float", + "type_abbrev": "f" + }, + { + "data_type": "__half", + "type_abbrev": "h" + } + ] +} +``` + +#### `filter_matrix.json` + +```json +{ + "filter_name": [ + "filter_none", + "filter_bitset" + ], + "_index": [ + { + "idx_type": "uint32_t", + "idx_abbrev": "ui" + }, + { + "idx_type": "int64_t", + "idx_abbrev": "l" + } + ] +} +``` + +#### `search_kernel_matrix.json` + +This example demonstrates conditional combinations: `OutT` can be `float` or `double` when `T` is `float`, but only `float` when `T` is `__half`. + +```json +{ + "_data_type": [ + { + "data_type": "float", + "type_abbrev": "f", + "_output_type": [ + { + "out_type": "float", + "out_abbrev": "f" + }, + { + "out_type": "double", + "out_abbrev": "d" + } + ] + }, + { + "data_type": "__half", + "type_abbrev": "h", + "_output_type": [ + { + "out_type": "float", + "out_abbrev": "f" + } + ] + } + ], + "_index": [ + { + "idx_type": "uint32_t", + "idx_abbrev": "ui" + }, + { + "idx_type": "int64_t", + "idx_abbrev": "l" + } + ], + "_optimized": [ + { + "optimized_name": "optimized", + "optimized_value": "true", + "veclen": ["1", "4"] + }, + { + "optimized_name": "standard", + "optimized_value": "false", + "veclen": ["8", "16"] + } + ] +} +``` + +This generates 24 combinations (3 data/output type combinations × 2 index types × 4 optimized/veclen combinations): +- `float` + `float` + `uint32_t` + `optimized` + `veclen=1` +- `float` + `float` + `uint32_t` + `optimized` + `veclen=4` +- `float` + `float` + `uint32_t` + `standard` + `veclen=8` +- `float` + `float` + `uint32_t` + `standard` + `veclen=16` +- `float` + `double` + `uint32_t` + `optimized` + `veclen=1` +- `float` + `double` + `uint32_t` + `optimized` + `veclen=4` +- `float` + `double` + `uint32_t` + `standard` + `veclen=8` +- `float` + `double` + `uint32_t` + `standard` + `veclen=16` +- `__half` + `float` + `uint32_t` + `optimized` + `veclen=1` +- `__half` + `float` + `uint32_t` + `optimized` + `veclen=4` +- `__half` + `float` + `uint32_t` + `standard` + `veclen=8` +- `__half` + `float` + `uint32_t` + `standard` + `veclen=16` +- ... and the same with `int64_t` (total: 24 combinations) + +### Step 4: Create `.cu.in` Template Files + +The `.cu.in` files are templates that get instantiated for each combination in the JSON matrix. They contain explicit template instantiations. + +#### `compute_distance_kernel.cu.in` + +```text +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "@header_file@" + +namespace example::detail { + +// Instantiate the generic compute_distance device function template +// The specific implementation (euclidean or inner_product) comes from the header +template __device__ float compute_distance<@data_type@>(@data_type@, @data_type@); + +} // namespace example::detail +``` + +#### `filter_kernel.cu.in` + +```text +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "example/jit_lto_kernels/@filter_name@.cuh" + +namespace example::detail { + +// Instantiate the generic apply_filter device function template +// The specific implementation (filter_none or filter_bitset) comes from the header +template __device__ bool apply_filter<@idx_type@>(uint32_t, @idx_type@, void*); + +} // namespace example::detail +``` + +#### Update `search_kernel.cuh` with Extern Declarations + +The kernel header needs to declare generic extern device functions so the kernel code can call them. The specific implementations will be linked from fragments at runtime: + +**`search_kernel.cuh`**: + +```cpp +#pragma once + +#include + +namespace example::detail { + +// Forward declare generic extern device functions that will be linked from fragments +// The specific implementations (euclidean, inner_product, etc.) are resolved at link time +template +extern __device__ float compute_distance(T, T); + +template +extern __device__ bool apply_filter(uint32_t, IdxT, void*); + +// Main kernel - uses generic extern device functions +template +__device__ void search_kernel_impl( + const T* dataset, + const T* queries, + IdxT* results, + OutT* distances, // Output distance type + uint32_t num_queries, + uint32_t dataset_size, + void* filter_data) { + + uint32_t query_id = blockIdx.x * blockDim.x + threadIdx.x; + if (query_id >= num_queries) return; + + OutT best_dist = std::numeric_limits::max(); + IdxT best_idx = 0; + + for (IdxT i = 0; i < dataset_size; ++i) { + // Call generic extern device functions (specific implementations linked from fragments) + if (!apply_filter(query_id, i, filter_data)) continue; + + OutT dist = static_cast(compute_distance(queries[query_id], dataset[i])); + + // Use optimized path if enabled + if constexpr (UseOptimizedPath) { + // Optimized implementation + if (dist < best_dist) { + best_dist = dist; + best_idx = i; + } + } else { + // Standard implementation + if (dist < best_dist) { + best_dist = dist; + best_idx = i; + } + } + } + + results[query_id] = best_idx; + distances[query_id] = best_dist; +} + +} // namespace example::detail +``` + +#### `search_kernel.cu.in` + +The `.cu.in` file only contains the explicit template instantiation: + +```text +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "example/jit_lto_kernels/search_kernel.cuh" + +namespace example::detail { + +// Instantiate the kernel template +extern "C" __global__ void search_kernel( + const @data_type@* dataset, const @data_type@* queries, @idx_type@* results, @out_type@* distances, + uint32_t num_queries, uint32_t dataset_size, void* filter_data) +{ + search_kernel_impl<@data_type@, @out_type@, @idx_type@, @optimized_value@, @veclen@>( + dataset, + queries, + results, + distances, + num_queries, + dataset_size, + filter_data); +} + +} // namespace example::detail +``` + +**Note**: The kernel uses generic function templates (`compute_distance` and `apply_filter`) that are resolved at link time. The specific implementations (euclidean vs inner_product, filter_none vs filter_bitset) are provided by the fragments that get linked together. + +### Step 5: Create Fragment Tags for Embedding + +Fragment tags register the compiled fatbins so they can be loaded at runtime. They are used to help the linker find and include the relevant fatbins at build time. When calling `generate_jit_lto_kernels()`, we pass a `FRAGMENT_TAG_FORMAT` argument, which constructs the tag type from the given placeholders, and a `FRAGMENT_TAG_HEADER_FILES` argument, which specifies one or more header files that the fragment tags come from. The JIT+LTO system will then automatically generate and compile a `.cpp` file that registers the fragment using the provided tag. + +**Important**: When requesting fragments from the `AlgorithmPlanner`, we use **tags** (like `tag_f`, `tag_h`) instead of real types (like `float`, `__half`) in the `add_static_fragment` template parameters. This avoids including heavy headers that define the actual types, significantly improving compilation times. The tags are lightweight empty structs that serve only as compile-time identifiers. + +**`registration_tags.hpp`** + +```cpp +#pragma once + +struct tag_h{}; +struct tag_f{}; +struct tag_d{}; +struct tag_ui{}; +struct tag_l{}; + +struct tag_metric_euclidean {}; +struct tag_metric_inner_product {}; + +struct tag_filter_none {}; +struct tag_filter_bitset {}; + +template +struct fragment_tag_search {}; + +template +struct fragment_tag_compute_distance {}; + +template +struct fragment_tag_filter {}; +``` + +### Step 6: Create the Planner + +The planner is responsible for: +1. Identifying which fragments are needed for a given configuration +2. Building a unique key for the fragment combination +3. Requesting the fragments from the fragment database +4. Linking them together to create a launchable kernel + +**CRITICAL**: The fragment keys constructed in the planner methods must match **EXACTLY** with the keys used in the corresponding `FRAGMENT_TAG_FORMAT` argument. Any mismatch will result in runtime linking failures. + +**`search_planner.hpp`**: + +```cpp +#pragma once + +#include +#include +#include +#include + +struct SearchPlanner : AlgorithmPlanner { + inline static LauncherJitCache launcher_jit_cache{}; + + SearchPlanner() + : AlgorithmPlanner("search_kernel", launcher_jit_cache) + { + } + + template + void add_search_function() + { + add_static_fragment>(); + } + + template + void add_compute_distance_function() + { + add_static_fragment>(); + } + + template + void add_filter_function() + { + add_static_fragment>(); + } +}; +``` + +### Step 7: Integrate with Code Path + +Now we integrate the planner into the actual search function: + +**`search_jit.cuh`**: + +```cpp +#pragma once + +#include "search_planner.hpp" +#include +#include + +namespace example::detail { + +// Type tag helpers +template +constexpr auto get_data_type_tag() { + if constexpr (std::is_same_v) return tag_f{}; + if constexpr (std::is_same_v) return tag_h{}; +} + +template +constexpr auto get_idx_type_tag() { + if constexpr (std::is_same_v) return tag_ui{}; + if constexpr (std::is_same_v) return tag_l{}; +} + +template +constexpr auto get_out_type_tag() { + if constexpr (std::is_same_v) return tag_f{}; + if constexpr (std::is_same_v) return tag_d{}; +} + +template +constexpr auto get_metric_tag() { + if constexpr (Metric == DistanceType::Euclidean) return tag_metric_euclidean{}; + if constexpr (Metric == DistanceType::InnerProduct) return tag_metric_inner_product{}; +} + +template +constexpr auto get_filter_tag() { + if constexpr (Filter == FilterType::None) return tag_filter_none{}; + if constexpr (Filter == FilterType::Bitset) return tag_filter_bitset{}; +} + +template +void search_jit( + raft::device_resources const& handle, + const T* dataset, + const T* queries, + IdxT* results, + OutT* distances, + uint32_t num_queries, + uint32_t dataset_size, + void* filter_data = nullptr) { + + using data_tag = decltype(get_data_type_tag()); + using idx_tag = decltype(get_idx_type_tag()); + using out_tag = decltype(get_out_type_tag()); + using metric_tag = decltype(get_metric_tag()); + using filter_tag = decltype(get_filter_tag()); + + // Create planner with type tags and boolean parameter + // Note: The boolean is appended to the fragment key since make_fragment_key + // cannot handle non-type template parameters + SearchPlanner planner; + + // Add required device function fragments + planner.add_search_function(); + planner.add_compute_distance_device_function(); + planner.add_filter_device_function(); + + // Get the launcher (this will build/link fragments if needed) + auto launcher = planner.get_launcher(); + + // Launch configuration + dim3 block(256); + dim3 grid((num_queries + block.x - 1) / block.x); + + // Launch the kernel - arguments are passed directly + launcher->dispatch( + raft::resource::get_cuda_stream(handle), + grid, + block, + 0, // shared memory size + dataset, + queries, + results, + distances, + num_queries, + dataset_size, + filter_data); +} + +} // namespace example::detail +``` + +## Key Concepts + +### Fragment Tags + +Fragment tags uniquely identify fragments. They're simple lightweight types that are passed as the +sole template parameter to `StaticFatbinFragmentEntry`: + +```cpp +template +struct fragment_tag_get_score {}; +``` + +Fragment tags may themselves take template parameters in order to uniquely identify them. Typically, one fragment tag template will correspond to a single function, and a fragment tag template specialization will correspond to a function specialization. + +When a fatbin is compiled and embedded in C++ code, a translation unit specializes `StaticFatbinFragmentEntry` +to specify its `data` and `length` static fields: + +```cpp +using _FragmentEntry = StaticFatbinFragmentEntry>; + +template <> +const uint8_t* const _FragmentEntry::data = embedded_fatbin; + +template <> +const size_t _FragmentEntry::length = sizeof(embedded_fatbin); +``` + +Then, an `AlgorithmPlanner` can call `add_static_fragment()` with the fragment tag (NOT the `StaticFatbinFragmentEntry` +specialization) as the sole template parameter: + +```cpp +template +void add_get_score_function() +{ + add_static_fragment>(); +} +``` + +At build time, the linker takes care of finding and including the static fragments that have been specified by the +algorithm planner. + +### Registration Tags + +Registration tags are type-safe identifiers used to organize fragments. They're typically empty structs: + +```cpp +struct tag_f {}; // float +struct tag_h {}; // half +struct tag_ui {}; // uint32_t +struct tag_l {}; // int64_t +``` + +These tags are used in `registerAlgorithm<>()` to create a hierarchical organization of fragments. + +### AlgorithmLauncher + +The `AlgorithmLauncher` is the runtime handle for a linked kernel. It: +- Holds a `cudaKernel_t` handle to the linked kernel +- Provides `call()` and `call_cooperative()` methods to launch the kernel +- Manages the lifetime of the `cudaLibrary_t` that contains the kernel + +## Best Practices + +1. **Minimize Includes**: JIT LTO fragments should have minimal includes, especially avoiding host-side headers. Extract device-only code into separate headers. + +2. **Fragment Granularity**: Balance between too many small fragments (overhead) and too few large fragments (less reuse). Device functions that are reused across multiple kernels are good candidates for separate fragments. + +3. **Naming Consistency**: Ensure fragment tags match exactly between registration and lookup. Use helper functions to construct tags consistently. + +4. **Type Safety**: Use registration tags to provide compile-time type safety and avoid runtime string mismatches. + +5. **Caching**: Each planner type should hold a static `LauncherJitCache` and pass it to `AlgorithmPlanner`; `get_launcher()` then reuses linked kernels for the same fragment key within that cache. + +## Example: IVF Flat + +IVF Flat uses JIT LTO with: +- **Metric fragments**: Euclidean and inner product distance computations (16 fatbins) +- **Post-lambda fragments**: Identity, sqrt, and compose post-processing (3 fatbins) +- **Interleaved scan fragments**: Main search kernel with various configurations (320 fatbins) +- **Filter fragments**: None and bitset filters (2 fatbins) + +**Total: 341 fatbins** that can be combined into many more kernel variants at runtime. + +### Step 8: Integrate with CMake Build System + +To integrate JIT LTO kernels into the CMake build system, add calls to `generate_jit_lto_kernels()` in your main `CMakeLists.txt` file (typically in `cpp/CMakeLists.txt`). + +The `generate_jit_lto_kernels()` function (defined in `cmake/modules/generate_jit_lto_kernels.cmake`) takes: +- `NAME_FORMAT`: Format string for generated kernel names (using `@variable@` syntax) +- `MATRIX_JSON_FILE`: Path to the JSON matrix file +- `KERNEL_INPUT_FILE`: Path to the `.cu.in` template +- `FRAGMENT_TAG_FORMAT`: Format string for fragment tag type (using `@variable@` syntax) +- `FRAGMENT_TAG_HEADER_FILES`: List of header files that provide the fragment tag types (can be enclosed in `<`/`>` or `"`/`"`, automatically enclosed in quotes if quotes and brackets are not provided) +- `OUTPUT_DIRECTORY`: Where generated files are placed +- `KERNEL_LINK_LIBRARIES`: Interface library with compilation settings + +Call `generate_jit_lto_kernels()` once for each fragment type (compute_distance, filter, search_kernel, etc.). The function reads the JSON matrix, computes the cross-product of all combinations, generates `.cu` and `.cpp` files from the templates, compiles them into fatbins, and returns a list of generated source files that should be added to your JIT LTO library target. + +See the CUVS `cpp/CMakeLists.txt` file for a complete example of how to set up the interface library, call `generate_jit_lto_kernels()` for each fragment type, and create the final library target. + +## Summary + +JIT LTO enables: +- **Reduced binary size**: Compile fragments once, combine many ways +- **Faster compilation**: Fragments compile independently +- **Runtime flexibility**: Link fragments on-demand based on configuration +- **Code reuse**: Device function fragments shared across kernels + +The process involves: +1. Separating device functions into fragment headers +2. Creating JSON matrices defining parameter combinations +3. Creating `.cu.in` templates for explicit instantiations +4. Creating fragment tag types for fatbin registration +5. Creating a planner to manage fragment dependencies +6. Integrating the planner into the code path to launch kernels +7. **Adding CMake integration** to generate and compile all fragment variants + +## Fragment Architecture + +JIT LTO kernels are split into _fragments_, which are fatbins containing individual pieces of code that can be strung together +rather than instantiating the whole kernel at once. Each fragment only needs to be multiplied out over the dimensions (template +parameters) that the fragment itself contains rather than the kernel as a whole. At runtime, these fragments are combined together +by nvjitlink into the final program. + +In JIT LTO, there are two kinds of code: _algorithms_ and _adapters_. Algorithms are, roughly speaking, code that actually "does +stuff" - searching, sorting, even as simple as initializing variables. Adapters don't do anything by themselves, but are merely +thin wrappers around algorithms that exist only for reducing the number of template parameters that the caller needs to know about. +It should generally be assumed that algorithm code is expensive to multiply over a matrix, and thus such multiplication should be +minimized, while adapter code is cheap to multiply. + +An algorithm function is a function that contains real code for the algorithm, and an adapter function merely calls an algorithm +function with more template parameters than the adapter function itself has. An algorithm file contains algorithm code, and an +adapter file contains adapter code. + +Here is an example of an algorithm file that contains an algorithm function: + +```c++ +template +__device__ bool is_divisible_impl(T value) +{ + return value % Divisor == 0; +} +``` + +Here is an example of an adapter file that contains an adapter function: + +``` +#include "device_functions.cuh" // is_divisible +#include "is_divisible_impl.cuh" // is_divisible_impl + +namespace { + +using data_t = @data_type@; +constexpr data_t divisor = @divisor@; + +} // namespace + +template <> +__device__ bool is_divisible(data_t value) +{ + return is_divisible_impl(value); +} +``` + +This is the most common pattern that you will see in cuVS's JIT LTO code. Note that any code that calls `is_divisible()` does not +need to know the value of `Divisor`, which allows the caller to be multiplied over fewer dimensions, thus reducing the amount of code +generated. + +Note that in the above adapter file, `@data_type@` and `@divisor@` are build-time substitutions performed by CMake. These +substitutions will be filled in with values from the matrix product. Note that they are all grouped together in a single `namespace`, +making it easy to find all substitutions. This should be preferred to sprinkling the substitutions throughout the code. + +Here is an example with two algorithm files: + +```c++ +// greater_than_impl.cuh +#include "device_impl_functions.cuh" // filter + +template +__device__ bool filter(T value) +{ + return value > Comparand; +} +``` + +```c++ +// less_than_impl.cuh +#include "device_impl_functions.cuh" // filter + +template +__device__ bool filter(T value) +{ + return value < Comparand; +} +``` + +And here is the accompanying adapter file: + +``` +#include "@op_name@_impl.cuh" // filter + +namespace { + +using data_t = @data_type@; + +} + +template __device__ bool filter(data_t value); +``` + +This is another common pattern that you will see in cuVS JIT LTO. Note that the adapter file does not contain any adapter functions, +but merely instantiates a different algorithm function based on which algorithm file is included based on the CMake substitution. + +When a piece of algorithm code is used in multiple kernels, it should be split into its own shared fragment. At this point, it +becomes important to also distinguish algorithm fragments and adapter fragments. An algorithm fragment contains an algorithm function +that exposes all of the relevant template parameters, and this fragment is shared between multiple kernels. An adapter fragment +is specific to a kernel. If a kernel wishes to invoke the same shared algorithm multiple times in the same run with +different template parameters, it can employ multiple adapter fragments to accomplish this. Consider the following header file: + +```c++ +// filter.cuh + +template +__device__ bool filter_less_than(T value); + +template +__device__ bool filter_greater_than(T value); +``` + +And the following adapter files: + +``` +#include "device_functions.cuh" // filter_first_pass +#include "filter.cuh" // filter + +namespace { + +using data_t = @data_type@; +constexpr data_t comparand = @comparand@; + +} + +template <> +__device__ bool filter_first_pass(data_t value) +{ + return filter_@op_name@(value); +} +``` + +``` +#include "device_functions.cuh" // filter_second_pass +#include "filter.cuh" // filter + +namespace { + +using data_t = @data_type@; +constexpr data_t comparand = @comparand@; +} + +template <> +__device__ bool filter_second_pass(data_t value) +{ + return filter_@op_name@(value); +} +``` + +And the following algorithm file: + +```c++ +#include "device_functions.cuh" // filter_first_pass, filter_second_pass + +template +__device__ bool filter_all_passes(T value) +{ + return filter_first_pass(value) && filter_second_pass(value); +} +``` + +Note that `filter_first_pass` and `filter_second_pass` both invoke one of the `filter` functions, but which one they invoke is +decided independently for each. Also note that neither of the adapter fragments contains the underlying algorithm code, but rather +links against the corresponding shared algorithm fragments. + +The key to minimizing code generation is to minimize the number of dimensions that any given fragment needs to be multiplied out +over. If a section of algorithm code uses lots of template parameters, try to separate out sections that use only a subset of +these parameters, put them into their own fragment, and remove the corresponding template parameters from the caller. Make judicious +use of adapter code to accomplish this. An adapter function should only have the template parameters that appear in its signature, +whereas an algorithm function should have all of the template parameters that appear in its signature or its implementation. + +Unoptimized algorithm: + +```c++ +#include "filter_less_than.cuh" + +template +__device__ size_t find_first(T* values, size_t count) +{ + for (size_t i = 0; i < count; i++) { + if (filter_less_than_impl(values[i])) { + return i; + } + } + + // Could not find any + return count; +} +``` + +Note that the algorithm includes the `Comparand` template parameter, which means the entire algorithm has to be multiplied out over +all the possible values of this parameter. + +Optimized algorithm: + +```c++ +#include "device_functions.cuh" + +template +__device__ size_t find_first(T* values, size_t count) +{ + for (size_t i = 0; i < count; i++) { + if (filter_less_than(values[i])) { + return i; + } + } + + // Could not find any + return count; +} +``` + +We are now using an adapter function (possibly inside an adapter fragment) called `filter_less_than` to invoke +`filter_less_than_impl` (which may be inside a shared algorithm fragment). This allows us to hide the `Comparand` parameter +from `find_first`, which means we no longer need to multiply the entire algorithm over all possible values of `Comparand`, only the +`filter_less_than` adapter and algorithm. \ No newline at end of file diff --git a/cpp/include/cudf/column/column_device_view_base.cuh b/cpp/include/cudf/column/column_device_view_base.cuh index 3c1dce082b50..1d024d410087 100644 --- a/cpp/include/cudf/column/column_device_view_base.cuh +++ b/cpp/include/cudf/column/column_device_view_base.cuh @@ -11,10 +11,12 @@ #include #include -#include #include #include +#include +#include + /** * @file column_device_view_base.cuh * @brief Column device view class definitions diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index ea98323320f3..7eb6f5d273fc 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -20,9 +20,7 @@ enum class init_flags : std::uint32_t { LOAD_NVCOMP = 1 << 0, /// @brief Initialize the JIT program cache during initialization INIT_JIT_CACHE = 1 << 1, - /// @brief Pre-load the JIT program cache from disk during initialization - DEFAULT = INIT_JIT_CACHE, - /// @brief All initialization steps + /// @brief All initialization steps (default behavior) ALL = LOAD_NVCOMP | INIT_JIT_CACHE }; @@ -70,7 +68,7 @@ constexpr bool has_flag(init_flags flags, init_flags flag) noexcept /// @param flags Optional flags to control which initialization steps to perform. /// Can be called multiple times to initialize additional components. If all selected /// steps are already performed, the call has no effect. -void initialize(init_flags flags = init_flags::DEFAULT); +void initialize(init_flags flags = init_flags::INIT_JIT_CACHE); /// @brief Destroy the cudf global context, resetting it to an uninitialized state. This is /// primarily intended for testing purposes, allowing for re-initialization of the context after diff --git a/cpp/include/cudf/detail/normalizing_iterator.cuh b/cpp/include/cudf/detail/normalizing_iterator.cuh index 3a0085f24260..a4ac02b4b23a 100644 --- a/cpp/include/cudf/detail/normalizing_iterator.cuh +++ b/cpp/include/cudf/detail/normalizing_iterator.cuh @@ -1,11 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#if !defined(__CUDACC_RTC__) +#ifndef CUDF_RUNTIME_JIT #include @@ -13,7 +13,7 @@ #include -#include +#include namespace cudf { namespace detail { @@ -30,10 +30,10 @@ namespace detail { template struct alignas(16) base_normalator { static_assert(cudf::is_index_type()); - using difference_type = cuda::std::ptrdiff_t; + using difference_type = std::ptrdiff_t; using value_type = Integer; using pointer = Integer*; - using iterator_category = cuda::std::random_access_iterator_tag; + using iterator_category = std::random_access_iterator_tag; base_normalator() = default; base_normalator(base_normalator const&) = default; @@ -216,7 +216,7 @@ struct alignas(16) base_normalator { }; protected: -#if !defined(__CUDACC_RTC__) +#ifndef CUDF_RUNTIME_JIT // TODO: refactor type_dispatcher to support NVRTC /** * @brief Constructor assigns width and type member variables for base class. diff --git a/cpp/include/cudf/detail/operation-udf.hpp b/cpp/include/cudf/detail/operation-udf.hpp index 991405d5bd77..47bb3e91170d 100644 --- a/cpp/include/cudf/detail/operation-udf.hpp +++ b/cpp/include/cudf/detail/operation-udf.hpp @@ -1,9 +1,9 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -// This file serves as a placeholder for user defined functions, so NVRTC can choose to override it +// This file serves as a placeholder for user defined functions, so jitify can choose to override it // at runtime. diff --git a/cpp/include/cudf/detail/utilities/assert.cuh b/cpp/include/cudf/detail/utilities/assert.cuh index 077fb7b4fc5b..5ecd00db4fba 100644 --- a/cpp/include/cudf/detail/utilities/assert.cuh +++ b/cpp/include/cudf/detail/utilities/assert.cuh @@ -1,11 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2023, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#include +#include /** * @brief `assert`-like macro for device code diff --git a/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp b/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp index fecc1b6b330d..6bf74613da70 100644 --- a/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp +++ b/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp @@ -14,6 +14,8 @@ #include #include +#include + namespace numeric { namespace detail { @@ -117,7 +119,9 @@ struct floating_converter { CUDF_HOST_DEVICE inline static IntegralType bit_cast_to_integer(FloatingType floating) { // Convert floating to integer - return cuda::std::bit_cast(floating); + IntegralType integer_rep; + memcpy(&integer_rep, &floating, sizeof(floating)); + return integer_rep; } /** @@ -129,7 +133,9 @@ struct floating_converter { CUDF_HOST_DEVICE inline static FloatingType bit_cast_to_floating(IntegralType integer) { // Convert back to float - return cuda::std::bit_cast(integer); + FloatingType floating; + memcpy(&floating, &integer, sizeof(floating)); + return floating; } /** diff --git a/cpp/include/cudf/fixed_point/fixed_point.hpp b/cpp/include/cudf/fixed_point/fixed_point.hpp index d05c19110367..6dcc4aed20a1 100644 --- a/cpp/include/cudf/fixed_point/fixed_point.hpp +++ b/cpp/include/cudf/fixed_point/fixed_point.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,17 +9,15 @@ #include #include -#include -#include #include #include #include #include -#if !defined(__CUDACC_RTC__) #include +#include +#include #include -#endif /// `fixed_point` and supporting types namespace CUDF_EXPORT numeric { @@ -572,8 +570,6 @@ class fixed_point { return fixed_point{scaled_integer{value, scale}}; } -#if !defined(__CUDACC_RTC__) - /** * @brief Returns a string representation of the fixed_point value. */ @@ -593,8 +589,6 @@ class fixed_point { auto const zeros = std::string(_scale, '0'); return detail::to_string(_value) + zeros; } - -#endif }; /** diff --git a/cpp/include/cudf/fixed_point/temporary.hpp b/cpp/include/cudf/fixed_point/temporary.hpp index a912a8e0bcf2..097f0cb061d8 100644 --- a/cpp/include/cudf/fixed_point/temporary.hpp +++ b/cpp/include/cudf/fixed_point/temporary.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,15 +10,12 @@ #include #include -#if !defined(__CUDACC_RTC__) #include #include -#endif namespace CUDF_EXPORT numeric { namespace detail { -#if !defined(__CUDACC_RTC__) template auto to_string(T value) -> std::string { @@ -44,7 +41,6 @@ auto to_string(T value) -> std::string } return std::string{}; // won't ever hit here, need to suppress warning though } -#endif template CUDF_HOST_DEVICE constexpr auto abs(T value) diff --git a/cpp/include/cudf/jit/transform_operator.cuh b/cpp/include/cudf/jit/transform_operator.cuh deleted file mode 100644 index 49d15a2d4974..000000000000 --- a/cpp/include/cudf/jit/transform_operator.cuh +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -namespace cudf { -namespace lto { - -/// @brief The unary operator for the transform operation. -/// @tparam Out The output type of the operator. -/// @tparam In0 The input type of the operator. -/// @param out The output destination for the operator result. -/// @param a The input value for the operator. -template -__device__ void unary_operator(Out* __restrict__ out, In0 a); - -/// @brief The binary operator for the transform operation. -/// @tparam Out The output type of the operator. -/// @tparam In0 The first input type of the operator. -/// @tparam In1 The second input type of the operator. -/// @param out The output destination for the operator result. -/// @param a The first input value for the operator. -/// @param b The second input value for the operator. -template -__device__ void binary_operator(Out* __restrict__ out, In0 a, In1 b); - -} // namespace lto -} // namespace cudf diff --git a/cpp/include/cudf/jit/type_tags.cuh b/cpp/include/cudf/jit/type_tags.cuh deleted file mode 100644 index 15a4e7caef9f..000000000000 --- a/cpp/include/cudf/jit/type_tags.cuh +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include -#include -#include -#include - -namespace CUDF_EXPORT cudf { -namespace tags { - -template -inline constexpr char const* tag_of = "void"; - -#define CUDF_TYPE_TAG(type, tag) \ - using tag = type; \ - template <> \ - inline constexpr char const* tag_of = #tag; - -CUDF_TYPE_TAG(bool, bool8); -CUDF_TYPE_TAG(int8_t, int8); -CUDF_TYPE_TAG(int16_t, int16); -CUDF_TYPE_TAG(int32_t, int32); -CUDF_TYPE_TAG(int64_t, int64); -CUDF_TYPE_TAG(__int128_t, int128); -CUDF_TYPE_TAG(uint8_t, uint8); -CUDF_TYPE_TAG(uint16_t, uint16); -CUDF_TYPE_TAG(uint32_t, uint32); -CUDF_TYPE_TAG(uint64_t, uint64); -CUDF_TYPE_TAG(__uint128_t, uint128); -CUDF_TYPE_TAG(float, float32); -CUDF_TYPE_TAG(double, float64); -CUDF_TYPE_TAG(cudf::string_view, string_view); -CUDF_TYPE_TAG(numeric::decimal32, decimal32); -CUDF_TYPE_TAG(numeric::decimal64, decimal64); -CUDF_TYPE_TAG(numeric::decimal128, decimal128); -CUDF_TYPE_TAG(cudf::duration_D, duration_D); -CUDF_TYPE_TAG(cudf::duration_h, duration_h); -CUDF_TYPE_TAG(cudf::duration_m, duration_m); -CUDF_TYPE_TAG(cudf::duration_s, duration_s); -CUDF_TYPE_TAG(cudf::duration_ms, duration_ms); -CUDF_TYPE_TAG(cudf::duration_us, duration_us); -CUDF_TYPE_TAG(cudf::duration_ns, duration_ns); -CUDF_TYPE_TAG(cudf::timestamp_D, timestamp_D); -CUDF_TYPE_TAG(cudf::timestamp_h, timestamp_h); -CUDF_TYPE_TAG(cudf::timestamp_m, timestamp_m); -CUDF_TYPE_TAG(cudf::timestamp_s, timestamp_s); -CUDF_TYPE_TAG(cudf::timestamp_ms, timestamp_ms); -CUDF_TYPE_TAG(cudf::timestamp_us, timestamp_us); -CUDF_TYPE_TAG(cudf::timestamp_ns, timestamp_ns); - -} // namespace tags -} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/strings/detail/convert/string_to_float.cuh b/cpp/include/cudf/strings/detail/convert/string_to_float.cuh index ea76cf106c84..564406db6c1b 100644 --- a/cpp/include/cudf/strings/detail/convert/string_to_float.cuh +++ b/cpp/include/cudf/strings/detail/convert/string_to_float.cuh @@ -35,8 +35,13 @@ __device__ inline double stod(string_view const& d_str) ++in_ptr; } +#ifndef CUDF_RUNTIME_JIT constexpr double infinity = cuda::std::numeric_limits::infinity(); constexpr uint64_t max_holding = (cuda::std::numeric_limits::max() - 9L) / 10L; +#else + constexpr double infinity = (1.0 / 0.0); + constexpr uint64_t max_holding = (18446744073709551615UL - 9UL) / 10UL; +#endif // special strings: NaN, Inf if ((in_ptr < end) && *in_ptr > '9') { diff --git a/cpp/include/cudf/strings/string_view.cuh b/cpp/include/cudf/strings/string_view.cuh index 8275ebc6ff87..913ba0030999 100644 --- a/cpp/include/cudf/strings/string_view.cuh +++ b/cpp/include/cudf/strings/string_view.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -13,8 +13,9 @@ #include #endif -// This is defined when compiling with NVRTC. NVRTC cannot include thrust headers at this time. -#if !defined(__CUDACC_RTC__) +// This is defined when including this header in a https://github.com/NVIDIA/jitify +// or jitify2 source file. The jitify cannot include thrust headers at this time. +#ifndef CUDF_RUNTIME_JIT #include #include #endif @@ -40,7 +41,7 @@ __device__ inline size_type characters_in_string(char const* str, size_type byte { if ((str == nullptr) || (bytes == 0)) return 0; auto ptr = reinterpret_cast(str); -#if !defined(__CUDACC_RTC__) +#ifndef CUDF_RUNTIME_JIT return thrust::count_if( thrust::seq, ptr, ptr + bytes, [](uint8_t chr) { return is_begin_utf8_char(chr); }); #else diff --git a/cpp/include/cudf/strings/string_view.hpp b/cpp/include/cudf/strings/string_view.hpp index 25dd26743a68..2a1db81ec7a3 100644 --- a/cpp/include/cudf/strings/string_view.hpp +++ b/cpp/include/cudf/strings/string_view.hpp @@ -1,16 +1,14 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -#include - -#if !defined(__CUDACC_RTC__) #include -#endif + +#include /** * @file @@ -70,8 +68,7 @@ class string_view { using value_type = char_utf8; using reference = char_utf8&; using pointer = char_utf8*; - using iterator_category = cuda::std::input_iterator_tag; - + using iterator_category = std::input_iterator_tag; __device__ inline const_iterator(string_view const& str, size_type pos); const_iterator(const_iterator const& mit) = default; const_iterator(const_iterator&& mit) = default; diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 1885e214281c..a2a99ed0c23b 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -179,97 +179,6 @@ std::unique_ptr
multi_transform( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); -/** - * @brief The type of LTO Binary - */ -enum class lto_binary_type : uint8_t { - LTO_IR, //< LTO-IR binary - FATBIN //< FATBIN binary -}; - -/** - * @brief Creates a new column by applying a transform function against every - * element of the input columns. - * - * Computes: - * `(output[i]) = UDF(input[i])`. - * - * - * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) - * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string - * types - * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and - * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in - * this case. - * @throws std::invalid_argument if string offsets are provided for non-string output columns, or - * if the number of string offsets does not match the number of output columns. - * - * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from - * the input and pre-allocated output columns. - * - * @param input Immutable view of the input to transform - * @param output Specification of the output column to be created - * @param udf The LTO-IR string of the transform function to apply - * @param binary_type The type of the LTO binary provided in `udf` - * @param is_null_aware Signifies the UDF will receive row inputs as optional values - * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned column's device memory - * @return A column resulting from applying the transform function to every element of - * the input according to the output specifications - * - */ -std::unique_ptr unary_op_lto( - column_view input, - transform_output output, - std::span udf, - lto_binary_type binary_type, - null_aware is_null_aware, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); - -/** - * @brief Creates a new column by applying a transform function against every - * element of the input columns. - * - * Computes: - * `(output[i]...) = UDF(inputs[i]...)`. - * - * - * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) - * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string - * types - * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and - * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in - * this case. - * @throws std::invalid_argument if string offsets are provided for non-string output columns, or - * if the number of string offsets does not match the number of output columns. - * - * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from - * the input and pre-allocated output columns. - * - * @param lhs Immutable view of the left-hand side input to transform - * @param rhs Immutable view of the right-hand side input to transform (can be a column or - * scalar) - * @param output Specification of the output column to be created - * @param udf The LTO-IR string of the transform function to apply - * @param binary_type The type of the LTO binary provided in `udf` - * @param is_null_aware Signifies the UDF will receive row inputs as optional values - * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned column's device memory - * @return A column resulting from applying the transform function to every element of - * the input according to the output specifications - * - */ -std::unique_ptr binary_op_lto( - column_view lhs, - transform_input rhs, - transform_output output, - std::span udf, - lto_binary_type binary_type, - null_aware is_null_aware, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); - /** * @brief Creates a null_mask from `input` by converting `NaN` to null and * preserving existing null values and also returns new null_count. diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp index 63c8794f29f0..659a77796012 100644 --- a/cpp/include/cudf/types.hpp +++ b/cpp/include/cudf/types.hpp @@ -27,11 +27,10 @@ #include -#include - #include #include #include +#include /** * @file @@ -99,7 +98,7 @@ using char_utf8 = uint32_t; ///< UTF-8 characters are 1-4 bytes template size_type distance(T f, T l) { - return static_cast(cuda::std::distance(f, l)); + return static_cast(std::distance(f, l)); } /** @@ -353,7 +352,7 @@ class data_type { */ constexpr bool operator==(data_type const& lhs, data_type const& rhs) { - // use std::tie in the future, breaks NVRTC currently + // use std::tie in the future, breaks JITIFY currently return lhs.id() == rhs.id() && lhs.scale() == rhs.scale(); } diff --git a/cpp/include/cudf/utilities/export.hpp b/cpp/include/cudf/utilities/export.hpp index f3821a50dfc8..815d3110f2cb 100644 --- a/cpp/include/cudf/utilities/export.hpp +++ b/cpp/include/cudf/utilities/export.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,11 +7,7 @@ // Macros used for defining symbol visibility, only GLIBC is supported #if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__MINGW64__)) -#ifndef CUDF_DISABLE_EXPORTS #define CUDF_EXPORT __attribute__((visibility("default"))) -#else -#define CUDF_EXPORT __attribute__((visibility("hidden"))) -#endif #define CUDF_HIDDEN __attribute__((visibility("hidden"))) #else #define CUDF_EXPORT diff --git a/cpp/include/cudf/utilities/span.hpp b/cpp/include/cudf/utilities/span.hpp index d79711e8f7e4..1eecb1b51f9f 100644 --- a/cpp/include/cudf/utilities/span.hpp +++ b/cpp/include/cudf/utilities/span.hpp @@ -69,14 +69,13 @@ class span_base { */ CUDF_HOST_DEVICE constexpr span_base(pointer data, size_type size) : _data(data), _size(size) {} // constexpr span_base(pointer begin, pointer end) : _data(begin), _size(end - begin) {} - constexpr span_base(span_base const&) noexcept = default; ///< Copy constructor - + CUDF_HOST_DEVICE constexpr span_base(span_base const&) noexcept = default; ///< Copy constructor /** * @brief Copy assignment operator. * * @return Reference to this span. */ - constexpr span_base& operator=(span_base const&) noexcept = default; + CUDF_HOST_DEVICE constexpr span_base& operator=(span_base const&) noexcept = default; /** * @brief Returns an iterator to the first element of the span. diff --git a/cpp/include/cudf/wrappers/dictionary.hpp b/cpp/include/cudf/wrappers/dictionary.hpp index 77b2e80065b3..025c899be144 100644 --- a/cpp/include/cudf/wrappers/dictionary.hpp +++ b/cpp/include/cudf/wrappers/dictionary.hpp @@ -8,6 +8,7 @@ #include #include +#include /** * @file diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index 4b3928ffa011..bb4a48629ec5 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -22,8 +22,8 @@ */ #include "compiled/binary_ops.hpp" +#include "jit/cache.hpp" #include "jit/helpers.hpp" -#include "jit/jit.hpp" #include "jit/parser.hpp" #include "jit/util.hpp" @@ -43,6 +43,8 @@ #include +#include + #include namespace cudf { @@ -154,23 +156,18 @@ void binary_operation(mutable_column_view& out, {2, cudf::type_to_name(rhs.type())}, }); - auto kernel_reflection = rtcx::reflect_template("cudf::binops::jit::binaryop_kernel", - output_type_name, - cudf::type_to_name(lhs.type()), - cudf::type_to_name(rhs.type()), - "cudf::binops::jit::UserDefinedOp"); - - auto kernel = cudf::jit::get_udf_kernel("binaryop/jit/kernel.cu", kernel_reflection, cuda_source); - - auto size_arg = static_cast(out.size()); - auto out_arg = cudf::jit::get_data_ptr(out); - auto lhs_arg = cudf::jit::get_data_ptr(lhs); - auto rhs_arg = cudf::jit::get_data_ptr(rhs); - - void* args[] = {&size_arg, &out_arg, &lhs_arg, &rhs_arg}; // NOLINT(modernize-avoid-c-arrays) - - auto cfg = kernel.max_occupancy_config(0, 0); - kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); + std::string kernel_reflection = jitify2::reflection::Template("cudf::binops::jit::kernel_v_v") + .instantiate(output_type_name, // list of template arguments + cudf::type_to_name(lhs.type()), + cudf::type_to_name(rhs.type()), + std::string("cudf::binops::jit::UserDefinedOp")); + + cudf::jit::get_udf_kernel(*binaryop_jit_kernel_cu_jit, kernel_reflection, cuda_source) + ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()) + ->launch(out.size(), + cudf::jit::get_data_ptr(out), + cudf::jit::get_data_ptr(lhs), + cudf::jit::get_data_ptr(rhs)); } } // namespace jit diff --git a/cpp/src/binaryop/jit/kernel.cu b/cpp/src/binaryop/jit/kernel.cu index 3d8f1d7cb5bd..893c1866cb44 100644 --- a/cpp/src/binaryop/jit/kernel.cu +++ b/cpp/src/binaryop/jit/kernel.cu @@ -37,7 +37,6 @@ // need to put this pragma before including it to avoid PCH mismatch. // clang-format off -#include #include // clang-format on @@ -57,14 +56,11 @@ struct UserDefinedOp { }; template -CUDF_KERNEL void binaryop_kernel(cudf::size_type size, - void* p_out_data, - void* p_lhs_data, - void* p_rhs_data) +CUDF_KERNEL void kernel_v_v(cudf::size_type size, + TypeOut* out_data, + TypeLhs* lhs_data, + TypeRhs* rhs_data) { - auto out_data = static_cast(p_out_data); - auto lhs_data = static_cast(p_lhs_data); - auto rhs_data = static_cast(p_rhs_data); auto const start = cudf::detail::grid_1d::global_thread_id(); auto const step = cudf::detail::grid_1d::grid_stride(); @@ -73,14 +69,32 @@ CUDF_KERNEL void binaryop_kernel(cudf::size_type size, } } +template +CUDF_KERNEL void kernel_v_v_with_validity(cudf::size_type size, + TypeOut* out_data, + TypeLhs* lhs_data, + TypeRhs* rhs_data, + cudf::bitmask_type* output_mask, + cudf::bitmask_type const* lhs_mask, + cudf::size_type lhs_offset, + cudf::bitmask_type const* rhs_mask, + cudf::size_type rhs_offset) +{ + auto const start = cudf::detail::grid_1d::global_thread_id(); + auto const step = cudf::detail::grid_1d::grid_stride(); + + for (auto i = start; i < size; i += step) { + bool output_valid = false; + out_data[i] = TypeOpe::template operate( + lhs_data[i], + rhs_data[i], + lhs_mask ? cudf::bit_is_set(lhs_mask, lhs_offset + i) : true, + rhs_mask ? cudf::bit_is_set(rhs_mask, rhs_offset + i) : true, + output_valid); + if (output_mask && !output_valid) cudf::clear_bit(output_mask, i); + } +} + } // namespace jit } // namespace binops } // namespace cudf - -extern "C" __global__ void cudf_kernel_entry(cudf::size_type size, - void* out_data, - void* lhs_data, - void* rhs_data) -{ - CUDF_KERNEL_INSTANCE(size, out_data, lhs_data, rhs_data); -} diff --git a/cpp/src/io/utilities/getenv_or.hpp b/cpp/src/io/utilities/getenv_or.hpp index d2ae4a347094..67e81486cc30 100644 --- a/cpp/src/io/utilities/getenv_or.hpp +++ b/cpp/src/io/utilities/getenv_or.hpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -43,18 +42,6 @@ T getenv_or(std::string_view env_var_name, T default_val) return converted_val; } -template -std::optional getenv_optional(std::string_view env_var_name) -{ - auto const env_val = std::getenv(env_var_name.data()); - if (env_val == nullptr) { return std::nullopt; } - - std::stringstream sstream(env_val); - T converted_val; - sstream >> converted_val; - return converted_val; -} - /** * @brief Specialization of getenv_or for bool, to allow common "ON"/"OFF" string values. */ diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp new file mode 100644 index 000000000000..b988296b0b7e --- /dev/null +++ b/cpp/src/jit/cache.cpp @@ -0,0 +1,160 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "io/utilities/getenv_or.hpp" +#include "runtime/context.hpp" + +#include +#include +#include + +#include + +#include + +namespace cudf { +namespace { + +// Get the directory in home to use for storing the cache +std::filesystem::path get_user_home_cache_dir() +{ + auto home_dir = std::getenv("HOME"); + if (home_dir != nullptr) { + return std::filesystem::path(home_dir) / ".cudf"; + } else { + return {}; + } +} + +// Default `LIBCUDF_KERNEL_CACHE_PATH` to `$HOME/.cudf/$CUDF_VERSION`. +// This definition can be overridden at compile time by specifying a +// `-DLIBCUDF_KERNEL_CACHE_PATH=/kernel/cache/path` CMake argument. +// Use `std::filesystem` for cross-platform path resolution and dir +// creation. This path is used in the `getCacheDir()` function below. +#if !defined(LIBCUDF_KERNEL_CACHE_PATH) +#define LIBCUDF_KERNEL_CACHE_PATH get_user_home_cache_dir() +#endif + +/** + * @brief Get the string path to the JITIFY kernel cache directory. + * + * This path can be overridden at runtime by defining an environment variable + * named `LIBCUDF_KERNEL_CACHE_PATH`. The value of this variable must be a path + * under which the process' user has read/write privileges. + * + * This function returns a path to the cache directory, creating it if it + * doesn't exist. + * + * The default cache directory is `$HOME/.cudf/$CUDF_VERSION`. If no overrides + * are used and if $HOME is not defined, returns an empty path and file + * caching is not used. + */ +std::filesystem::path get_cache_dir() +{ + // The environment variable always overrides the + // default/compile-time value of `LIBCUDF_KERNEL_CACHE_PATH` + auto kernel_cache_path_env = std::getenv("LIBCUDF_KERNEL_CACHE_PATH"); + auto kernel_cache_path = std::filesystem::path( + kernel_cache_path_env != nullptr ? kernel_cache_path_env : LIBCUDF_KERNEL_CACHE_PATH); + + // Cache path could be empty when env HOME is unset or LIBCUDF_KERNEL_CACHE_PATH is defined to be + // empty, to disallow use of file cache at runtime. + if (not kernel_cache_path.empty()) { + kernel_cache_path /= std::string{CUDF_STRINGIFY(CUDF_VERSION)}; + + // Make per device cache based on compute capability. This is to avoid multiple devices of + // different compute capability to access the same kernel cache. + int device = 0; + int cc_major = 0; + int cc_minor = 0; + CUDF_CUDA_TRY(cudaGetDevice(&device)); + CUDF_CUDA_TRY(cudaDeviceGetAttribute(&cc_major, cudaDevAttrComputeCapabilityMajor, device)); + CUDF_CUDA_TRY(cudaDeviceGetAttribute(&cc_minor, cudaDevAttrComputeCapabilityMinor, device)); + int const cc = cc_major * 10 + cc_minor; + + kernel_cache_path /= std::to_string(cc); + + try { + // `mkdir -p` the kernel cache path if it doesn't exist + std::filesystem::create_directories(kernel_cache_path); + } catch (std::exception const& e) { + // if directory creation fails for any reason, return empty path + return {}; + } + } + return kernel_cache_path; +} + +std::string get_program_cache_dir() +{ +#if defined(JITIFY_USE_CACHE) + return get_cache_dir().string(); +#else + return {}; +#endif +} + +} // namespace + +jitify2::ProgramCache<>& jit::program_cache::get(jitify2::PreprocessedProgramData const& preprog) +{ + CUDF_FUNC_RANGE(); + std::lock_guard caches_lock(_caches_mutex); + + auto existing_cache = _caches.find(preprog.name()); + + if (existing_cache == _caches.end() || _disabled.load(std::memory_order_seq_cst)) { + auto res = + _caches.emplace(preprog.name(), + std::make_unique>( + _kernel_limit_proc, preprog, nullptr, _cache_dir, _kernel_limit_disk)); + existing_cache = res.first; + } + + return *(existing_cache->second); +} + +void jit::program_cache::clear() +{ + CUDF_FUNC_RANGE(); + std::lock_guard caches_lock(_caches_mutex); + + _caches.clear(); + + // non-atomic + std::filesystem::remove_all(_cache_dir); +} + +void jit::program_cache::enable(bool enable) +{ + _disabled.store(!enable, std::memory_order_seq_cst); +} + +bool jit::program_cache::is_enabled() const { return !_disabled.load(std::memory_order_seq_cst); } + +std::unique_ptr jit::program_cache::create() +{ + auto const kernel_limit_proc = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 10'000); + auto const kernel_limit_disk = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_DISK", 100'000); + auto const disabled = get_bool_env_or("LIBCUDF_KERNEL_CACHE_DISABLED", false); + auto const clear_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); + + // if kernel_limit_disk is zero, jitify will assign it the value of kernel_limit_proc. + // to avoid this, we treat zero as "disable disk caching" by not providing the cache dir. + auto cache_dir = kernel_limit_disk == 0 ? std::string{} : get_program_cache_dir(); + + auto cache = + std::make_unique(kernel_limit_proc, kernel_limit_disk, cache_dir, disabled); + + if (clear_cache) { cache->clear(); } + + return cache; +} + +jitify2::ProgramCache<>& jit::get_program_cache(jitify2::PreprocessedProgramData const& preprog) +{ + return cudf::get_context().program_cache().get(preprog); +} +} // namespace cudf diff --git a/cpp/src/jit/cache.hpp b/cpp/src/jit/cache.hpp new file mode 100644 index 000000000000..0e6738fed176 --- /dev/null +++ b/cpp/src/jit/cache.hpp @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#pragma GCC diagnostic ignored "-Wignored-attributes" // Work-around for JITIFY2's false-positive + // warnings when compiled with GCC13 + +#include + +#include + +#include +#include +#include +#include +#include + +namespace cudf { +namespace jit { + +class program_cache { + std::mutex _caches_mutex; + std::unordered_map>> _caches; + int32_t _kernel_limit_proc; + int32_t _kernel_limit_disk; + std::filesystem::path _cache_dir; + std::atomic _disabled; + + public: + program_cache(int32_t kernel_limit_proc, + int32_t kernel_limit_disk, + std::filesystem::path cache_dir, + bool disabled) + : _kernel_limit_proc{kernel_limit_proc}, + _kernel_limit_disk{kernel_limit_disk}, + _cache_dir{std::move(cache_dir)}, + _disabled{disabled} + { + } + + program_cache(program_cache const&) = delete; + program_cache(program_cache&&) = delete; + program_cache& operator=(program_cache const&) = delete; + program_cache& operator=(program_cache&&) = delete; + ~program_cache() = default; + + jitify2::ProgramCache<>& get(jitify2::PreprocessedProgramData const& preprog); + + void clear(); + + void enable(bool enable); + + bool is_enabled() const; + + static std::unique_ptr create(); +}; + +jitify2::ProgramCache<>& get_program_cache(jitify2::PreprocessedProgramData const& preprog); + +} // namespace jit +} // namespace cudf diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 2cf8d647dec7..d035f43efe28 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -7,9 +7,7 @@ #include -#include -#include -#include +#include namespace cudf { namespace jit { @@ -63,7 +61,7 @@ std::map build_ptx_params(std::span ou if (has_user_data) { params.emplace(index++, "void *"); - params.emplace(index++, "cudf::size_type"); + params.emplace(index++, jitify2::reflection::reflect()); } for (auto& name : output_typenames) { @@ -89,26 +87,32 @@ std::vector input_type_names( return names; } -kernel get_udf_kernel(std::string const& source_file, - std::string const& kernel_name, - std::string const& udf_cuda_source) +jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, + std::string const& kernel_name, + std::string const& cuda_source, + std::vector const& extra_options) { CUDF_FUNC_RANGE(); - auto kernel_instance_source = std::format(R"***( -#define CUDF_KERNEL_INSTANCE {} -)***", - kernel_name); - char const* include_names[] = // NOLINT(modernize-avoid-c-arrays) - {"cudf/detail/operation-udf.hpp", "cudf/detail/kernel-instance.hpp"}; - char const* include_headers[] = // NOLINT(modernize-avoid-c-arrays) - {udf_cuda_source.c_str(), kernel_instance_source.c_str()}; - - return get_kernel(std::format("{}.jit.cu", source_file), - source_file, - include_names, - include_headers, - kernel_name); + int runtime_version; + CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); + + constexpr int min_pch_cuda_version = 12800; // CUDA 12.8 + constexpr int min_minimal_cuda_version = 12800; // CUDA 12.8 + + std::vector options; + options.emplace_back("-arch=sm_."); + + if (runtime_version >= min_minimal_cuda_version) { options.emplace_back("-minimal"); } + + if (runtime_version >= min_pch_cuda_version) { options.emplace_back("-pch"); } + + for (auto& opt : extra_options) { + options.push_back(opt); + } + + return cudf::jit::get_program_cache(preprocessed_program_data) + .get_kernel(kernel_name, {}, {{"cudf/detail/operation-udf.hpp", cuda_source}}, options); } } // namespace jit diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index 1831db7edcba..4a3f27b048c1 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -12,7 +12,7 @@ #include -#include +#include #include #include @@ -75,9 +75,10 @@ column_views_to_device(std::span views, std::vector input_type_names( std::span const> views); -kernel get_udf_kernel(std::string const& source_file, - std::string const& kernel_name, - std::string const& udf_cuda_source); +jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, + std::string const& kernel_name, + std::string const& cuda_source, + std::vector const& extra_options = {}); } // namespace jit } // namespace cudf diff --git a/cpp/src/jit/jit.cpp b/cpp/src/jit/jit.cpp deleted file mode 100644 index d7aaa314bc55..000000000000 --- a/cpp/src/jit/jit.cpp +++ /dev/null @@ -1,441 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace CUDF_EXPORT cudf { - -namespace { - -rtcx::sha256 hash(std::span input) -{ - rtcx::sha256_context ctx; - ctx.update(std::span{reinterpret_cast(input.data()), input.size()}); - return ctx.finalize(); -} - -rtcx::sha256 hash(std::span inputs) -{ - rtcx::sha256_context ctx; - for (auto const* input : inputs) { - ctx.update(std::span{reinterpret_cast(input), std::strlen(input)}); - } - return ctx.finalize(); -} - -rtcx::sha256 hash(std::span file_fragments, - std::span memory_fragments) -{ - rtcx::sha256_context ctx; - for (auto const& fragment : file_fragments) { - ctx.update( - std::span{reinterpret_cast(fragment.path), std::strlen(fragment.path)}); - } - - for (auto const& fragment : memory_fragments) { - ctx.update(fragment.data); - } - - return ctx.finalize(); -} - -void install_file_set(std::string_view target_dir, - std::span compressed_binary, - size_t uncompressed_size, - std::span file_ranges, - std::span destinations, - std::string_view compression) -{ - auto decompressed = rtcx::decompress_blob(compressed_binary, uncompressed_size, compression); - for (size_t i = 0; i < file_ranges.size(); ++i) { - auto file_data_range = file_ranges[i]; - auto file_data = std::span{decompressed.data() + file_data_range[0], file_data_range[1]}; - auto dst_path = destinations[i]; - auto target_path = std::format("{}/{}", target_dir, dst_path); - - std::filesystem::create_directories(std::filesystem::path{target_path}.parent_path()); - - std::ofstream file(target_path, std::ios::binary); - if (!file) { - throw std::runtime_error( - std::format("Failed to open file for writing at path: {}", target_path)); - } - - file.write(reinterpret_cast(file_data.data()), file_data.size()); - if (!file) { - throw std::runtime_error(std::format("Failed to write file at path: {}", target_path)); - } - } -} - -std::string read_file_string(char const* path) -{ - std::ifstream file(std::string{path}, std::ios::binary | std::ios::ate); - if (!file) { throw std::runtime_error(std::format("Failed to open file at path: {}", path)); } - - auto size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::string contents(size, '\0'); - if (!file.read(contents.data(), size)) { - throw std::runtime_error(std::format("Failed to read file at path: {}", path)); - } - - return contents; -} - -void install_cudf_jit_files(std::string const& target_dir, std::string const& tmp_dir) -{ - // directory does not exist, so create it - auto tmp_dir_path_str = std::format("{}/cudf-jit-tmpdir_XXXXXX", tmp_dir); - char* tmp_dir_path = ::mkdtemp(tmp_dir_path_str.data()); - CUDF_EXPECTS( - tmp_dir_path != nullptr, - std::format("Failed to create temporary directory for JIT file installation in tmp dir: {}", - tmp_dir), - std::runtime_error); - - install_file_set(tmp_dir_path, - cudf_jit_embed::files, - cudf_jit_embed::files_uncompressed_size, - cudf_jit_embed::file_ranges, - cudf_jit_embed::file_destinations, - cudf_jit_embed::files_compression); - - // rename the temporary directory to the target install directory - if (::rename(tmp_dir_path, target_dir.c_str()) == -1) { - auto errc = errno; - // another process created it - if (errc == ENOTEMPTY || errc == EEXIST) { - std::filesystem::remove_all(tmp_dir_path); - } else { - CUDF_FAIL( - std::format("Failed to install JIT files to target directory: {} with error ({}): {}", - target_dir, - errc, - std::strerror(errc)), - std::runtime_error); - } - } -} - -} // namespace - -jit_bundle_t::jit_bundle_t(std::string install_dir, rtcx::cache_t& cache) - : install_dir_{std::move(install_dir)}, cache_{&cache} -{ - ensure_installed(); -} - -void jit_bundle_t::ensure_installed() const -{ - CUDF_FUNC_RANGE(); - - auto expected_hash = get_hash(); - auto expected_path = std::format("{}/{}", install_dir_, expected_hash); - - if (!std::filesystem::exists(expected_path)) { - // ensure base install directory exists - std::filesystem::create_directories(install_dir_); - install_cudf_jit_files(expected_path.c_str(), cache_->get_tmp_dir()); - } else { - // directory exists, perform minor sanity check - CUDF_EXPECTS(std::filesystem::is_directory(expected_path), // throws if path does not exist - std::format("JIT install path ({}) exists but is not a directory", expected_path), - std::runtime_error); - } -} - -std::string jit_bundle_t::get_hash() const -{ - auto str = rtcx::sha256_hex_string::make(cudf_jit_embed::hash); - return std::string{str.view()}; -} - -std::string jit_bundle_t::get_directory() const -{ - return std::format("{}/{}", install_dir_, get_hash()); -} - -std::vector jit_bundle_t::get_include_directories() const -{ - std::vector directories; - auto base_dir = get_directory(); - - for (auto dir : cudf_jit_embed::include_directories) { - directories.emplace_back(std::format("{}/{}", base_dir, dir)); - } - - return directories; -} - -namespace { - -constexpr int MIN_CUDA_VERSION_PCH = 12800; // CUDA 12.8 -constexpr int MIN_CUDA_VERSION_MINIMAL = 12800; // CUDA 12.8 - -int32_t get_driver_version() -{ - int32_t driver_version; - CUDF_CUDA_TRY(cudaDriverGetVersion(&driver_version)); - return driver_version; -} - -int32_t get_runtime_version() -{ - int32_t runtime_version; - CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); - return runtime_version; -} - -int32_t get_current_device_physical_model() -{ - int32_t device; - CUDF_CUDA_TRY(cudaGetDevice(&device)); - - cudaDeviceProp props; - CUDF_CUDA_TRY(cudaGetDeviceProperties(&props, device)); - - return props.major * 10 + props.minor; -} - -std::tuple compile_library_uncached( - char const* name, - char const* cuda_code, - std::span extra_header_include_names, - std::span extra_headers, - std::span name_expressions) -{ - CUDF_FUNC_RANGE(); - - auto& ctx = cudf::get_context(); - auto& cfg = ctx.config(); - auto& bundle = ctx.jit_bundle(); - auto sm = get_current_device_physical_model(); - auto runtime = get_runtime_version(); - - auto include_dirs = bundle.get_include_directories(); - auto pch_dir = ctx.get_jit_pch_dir(); - - auto use_pch = runtime >= MIN_CUDA_VERSION_PCH; - auto use_minimal = runtime >= MIN_CUDA_VERSION_MINIMAL; - - std::vector options; - - for (auto const& include_dir : include_dirs) { - options.emplace_back(std::format("-I{}", include_dir)); - } - - options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); - - options.emplace_back("--diag-suppress=47"); - options.emplace_back("--device-int128"); - - if (sm >= 100) { options.emplace_back("--device-float128"); } - - options.emplace_back("-std=c++20"); - options.emplace_back("--device-as-default-execution-space"); - options.emplace_back("--generate-line-info"); - options.emplace_back("--dopt=on"); - - if (use_minimal) { options.emplace_back("--minimal"); } - - if (use_pch) { - options.emplace_back("--pch"); - - if (cfg.jit_verbose) { - options.emplace_back("--pch-verbose=true"); - options.emplace_back("--pch-messages=true"); - } else { - options.emplace_back("--pch-verbose=false"); - options.emplace_back("--pch-messages=false"); - } - } - - if (cfg.disable_cuda_cache) { options.emplace_back("--no-cache"); } - - if (cfg.dump_jit_trace) { options.emplace_back("--time=-"); } - - if (cfg.dump_jit_time_profile) { - options.emplace_back(std::format("--fdevice-time-trace=cudf_kernel_{}_trace", name)); - } - - std::vector options_cstr; - for (auto const& option : options) { - options_cstr.emplace_back(option.c_str()); - } - - auto params = rtcx::compile_params{.name = name, - .source = cuda_code, - .header_include_names = extra_header_include_names, - .headers = extra_headers, - .options = options_cstr, - .name_expressions = name_expressions, - .target_type = rtcx::binary_type::CUBIN}; - - auto cubin = rtcx::compile(params); - auto library = rtcx::load_library(cubin); - auto blob = rtcx::blob_t::from_buffer(std::move(cubin)); - - return std::make_tuple(library, std::make_shared(std::move(blob))); -} - -std::tuple link_library_uncached( - char const* name, - std::span file_fragments, - std::span memory_fragments) -{ - CUDF_FUNC_RANGE(); - - auto sm = get_current_device_physical_model(); - auto& ctx = cudf::get_context(); - auto& cfg = ctx.config(); - - std::vector options; - - options.emplace_back("-lto"); - options.emplace_back(std::format("-arch=sm_{}", sm)); - - if (cfg.disable_cuda_cache) { options.emplace_back("--no-cache"); } - - if (cfg.jit_verbose) { options.emplace_back("-verbose"); } - - if (cfg.dump_jit_trace) { options.emplace_back("-time"); } - - std::vector options_cstr; - - for (auto const& option : options) { - options_cstr.emplace_back(option.c_str()); - } - - auto params = rtcx::link_params{.name = name, - .output_type = rtcx::binary_type::CUBIN, - .file_fragments = file_fragments, - .memory_fragments = memory_fragments, - .link_options = options_cstr}; - - auto cubin = rtcx::link_library(params); - auto library = rtcx::load_library(cubin); - auto blob = rtcx::blob_t::from_buffer(std::move(cubin)); - - return std::make_tuple(library, std::make_shared(std::move(blob))); -} - -} // namespace - -kernel get_kernel(std::string const& name, - std::string const& source_file_id, - std::span header_include_names, - std::span headers, - std::string const& kernel_instance) -{ - CUDF_FUNC_RANGE(); - - auto& cache = cudf::get_context().rtcx_cache(); - auto& bundle = cudf::get_context().jit_bundle(); - - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_physical_model(); - auto header_include_names_hash = hash(header_include_names).to_hex_string(); - auto headers_hash = hash(headers).to_hex_string(); - auto bundle_hash = bundle.get_hash(); - auto source_file = std::format("{}/cudf/cpp/src/{}", bundle.get_directory(), source_file_id); - - auto cache_key = std::format(R"***(cuLibrary -name={} -binary_type=CUBIN -cuda_runtime={} -cuda_driver={} -arch={} -bundle={} -source_file={} -header_include_names={} -headers={} -kernel_instance={} -)***", - name, - runtime, - driver, - sm, - bundle_hash, - source_file, - header_include_names_hash.view(), - headers_hash.view(), - kernel_instance); - - auto cache_key_sha256 = hash(cache_key); - - auto compile = [&] { - auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); - auto source = read_file_string(source_file.c_str()); - return compile_library_uncached( - name.c_str(), source.c_str(), header_include_names, headers, {}); - }; - - auto fut = - cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); - - auto lib = fut.get(); - return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; -} - -kernel get_lto_linked_kernel(std::string const& name, - std::span file_fragments, - std::span memory_fragments) -{ - CUDF_FUNC_RANGE(); - - auto& cache = cudf::get_context().rtcx_cache(); - auto& bundle = cudf::get_context().jit_bundle(); - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_physical_model(); - auto bundle_hash = bundle.get_hash(); - auto fragments_hash = hash(file_fragments, memory_fragments).to_hex_string(); - - auto cache_key = std::format(R"***(cuLibrary -name={} -binary_type=CUBIN -cuda_runtime={} -cuda_driver={} -arch={} -bundle={} -fragments={} -)***", - name, - runtime, - driver, - sm, - bundle_hash, - fragments_hash.view()); - - auto cache_key_sha256 = hash(cache_key); - - auto compile = [&] { - return link_library_uncached(name.c_str(), file_fragments, memory_fragments); - }; - - auto fut = - cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); - - auto lib = fut.get(); - return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; -} - -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/jit.hpp b/cpp/src/jit/jit.hpp deleted file mode 100644 index 8525c16dde0e..000000000000 --- a/cpp/src/jit/jit.hpp +++ /dev/null @@ -1,102 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include - -#include - -#include - -namespace CUDF_EXPORT cudf { - -struct [[nodiscard]] jit_bundle_t { - private: - std::string install_dir_; - rtcx::cache_t* cache_; - - void ensure_installed() const; - - public: - jit_bundle_t(std::string install_dir, rtcx::cache_t& cache); - - [[nodiscard]] std::string get_hash() const; - - [[nodiscard]] std::string get_directory() const; - - [[nodiscard]] std::vector get_include_directories() const; -}; - -struct [[nodiscard]] kernel { - private: - rtcx::library _library; - rtcx::kernel_ref _kernel; - - public: - kernel(rtcx::library lib, rtcx::kernel_ref kernel) : _library(std::move(lib)), _kernel(kernel) {}; - kernel(kernel const&) = default; - kernel(kernel&&) = default; - kernel& operator=(kernel const&) = default; - kernel& operator=(kernel&&) = default; - ~kernel() = default; - - rtcx::kernel_ref get() const { return _kernel; } - - rtcx::kernel_occupancy_config max_occupancy_config(size_t dynamic_shared_memory_bytes, - int32_t block_size_limit) const - { - return _kernel.max_occupancy_config(dynamic_shared_memory_bytes, block_size_limit); - } - - void launch(rtcx::cuda_dim3 grid_dim, - rtcx::cuda_dim3 block_dim, - uint32_t shared_mem_bytes, - rmm::cuda_stream_view stream, - void** kernel_params) const - { - return _kernel.launch(grid_dim, block_dim, shared_mem_bytes, stream.value(), kernel_params); - } - - template - void launch_with(rtcx::cuda_dim3 grid_dim, - rtcx::cuda_dim3 block_dim, - uint32_t shared_mem_bytes, - rmm::cuda_stream_view stream, - Args&&... args) - { - void const* params[] = {&args...}; // NOLINT(modernize-avoid-c-arrays) - launch(grid_dim, block_dim, shared_mem_bytes, stream, const_cast(params)); - } -}; - -/** - * @brief Gets a kernel from an embedded CUDA source file - * @param name Debug name for the kernel (used for caching and logging) - * @param source_file_id Identifier for the embedded source file (used to locate the source and for - * caching) - * @param header_include_names Names of any additional embedded header files to include during - * compilation - * @param headers Contents of any additional embedded header files to include during compilation - * @param kernel_instance String identifier for the specific kernel instance being requested (used - * for caching) - */ -kernel get_kernel(std::string const& name, - std::string const& source_file_id, - std::span header_include_names, - std::span headers, - std::string const& kernel_instance); - -/** - * @brief Gets a kernel by linking together embedded binary fragments - * @param name Debug name for the kernel (used for caching and logging) - * @param file_fragments Paths of the fragments to link together to form the kernel - * @param memory_fragments Memory fragments to link - * @param extra_options Additional linking options - */ -kernel get_lto_linked_kernel(std::string const& name, - std::span file_fragments, - std::span memory_fragments); - -} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index 40091ac7aa4d..82cd308c2f5d 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include diff --git a/cpp/src/join/filter_join_indices_jit.cu b/cpp/src/join/filter_join_indices_jit.cu index 1484ef6829c6..d7a2565df341 100644 --- a/cpp/src/join/filter_join_indices_jit.cu +++ b/cpp/src/join/filter_join_indices_jit.cu @@ -33,11 +33,12 @@ #include #include +#include #include -#include #include #include #include +#include #include #include @@ -48,49 +49,46 @@ namespace detail { namespace { // Build template parameters for JIT kernel -std::vector build_join_filter_template_params( - std::vector const& left_columns, - std::vector const& right_columns, - bool has_user_data, - null_aware is_null_aware) +jitify2::StringVec build_join_filter_template_params(std::vector const& left_columns, + std::vector const& right_columns, + bool has_user_data, + null_aware is_null_aware) { - std::vector template_params; + jitify2::StringVec template_params; - template_params.emplace_back(rtcx::reflect_bool(has_user_data)); - template_params.emplace_back(rtcx::reflect_enum("cudf::null_aware", is_null_aware)); + template_params.emplace_back(jitify2::reflection::reflect(has_user_data)); + template_params.emplace_back(jitify2::reflection::reflect(is_null_aware)); // Add left column accessors for (std::size_t i = 0; i < left_columns.size(); ++i) { auto const& col = left_columns[i]; std::string type_name = cudf::type_to_name(col.type()); - template_params.emplace_back(rtcx::reflect_template("cudf::jit::join_column_accessor", - type_name, - std::to_string(i), - "cudf::jit::join_side::LEFT")); + template_params.emplace_back( + jitify2::reflection::Template("cudf::jit::join_column_accessor") + .instantiate(type_name, std::to_string(i), "cudf::jit::join_side::LEFT")); } // Add right column accessors for (std::size_t i = 0; i < right_columns.size(); ++i) { auto const& col = right_columns[i]; std::string type_name = cudf::type_to_name(col.type()); - template_params.emplace_back(rtcx::reflect_template("cudf::jit::join_column_accessor", - type_name, - std::to_string(i), - "cudf::jit::join_side::RIGHT")); + template_params.emplace_back( + jitify2::reflection::Template("cudf::jit::join_column_accessor") + .instantiate(type_name, std::to_string(i), "cudf::jit::join_side::RIGHT")); } return template_params; } // Build the JIT kernel for join filtering -kernel build_join_filter_kernel(std::string const& predicate_code, - std::vector const& left_columns, - std::vector const& right_columns, - bool is_ptx, - bool has_user_data, - null_aware is_null_aware, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +jitify2::ConfiguredKernel build_join_filter_kernel(std::string const& predicate_code, + std::vector const& left_columns, + std::vector const& right_columns, + bool is_ptx, + bool has_user_data, + null_aware is_null_aware, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); @@ -112,14 +110,18 @@ kernel build_join_filter_kernel(std::string const& predicate_code, // Build template parameters and kernel name auto template_args = build_join_filter_template_params(left_columns, right_columns, has_user_data, is_null_aware); - auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); + auto kernel_name = + jitify2::reflection::Template("cudf::join::jit::filter_join_kernel").instantiate(template_args); // Get compiled kernel - return cudf::jit::get_udf_kernel("join/jit/filter_join_kernel.cu", kernel_name, cuda_source); + auto kernel = + cudf::jit::get_udf_kernel(*join_jit_filter_join_kernel_cu_jit, kernel_name, cuda_source); + + return kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); } // Launch the JIT kernel for join filtering -void launch_join_filter_kernel(kernel const& kernel, +void launch_join_filter_kernel(jitify2::ConfiguredKernel& kernel, cudf::table_view const& left, cudf::table_view const& right, cudf::device_span left_indices, @@ -150,15 +152,14 @@ void launch_join_filter_kernel(kernel const& kernel, cudf::column_device_view_core const* right_tables_ptr = right_device_views.data(); void* user_data_ptr = user_data.value_or(nullptr); - void* args[] = {&left_span, - &right_span, - &left_tables_ptr, - &right_tables_ptr, - &predicate_results, - &user_data_ptr}; + std::array args{&left_span, + &right_span, + &left_tables_ptr, + &right_tables_ptr, + &predicate_results, + &user_data_ptr}; - auto cfg = kernel.max_occupancy_config(0, 0); - kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); + kernel->launch_raw(args.data()); } // Same join semantics handling as the AST version @@ -348,15 +349,15 @@ apply_join_semantics(cudf::table_view const& left, } // Build template parameters from AST input specs (preserves expression input order) -std::vector build_join_filter_template_params_from_specs( +jitify2::StringVec build_join_filter_template_params_from_specs( std::vector const& input_specs, cudf::table_view const& left, cudf::table_view const& right, null_aware is_null_aware) { - std::vector template_params; - template_params.emplace_back(rtcx::reflect_bool(false)); // has_user_data = false - template_params.emplace_back(rtcx::reflect_enum("cudf::null_aware", is_null_aware)); + jitify2::StringVec template_params; + template_params.emplace_back(jitify2::reflection::reflect(false)); // has_user_data = false + template_params.emplace_back(jitify2::reflection::reflect(is_null_aware)); // Scalar columns are appended to the left table's device views, // starting at index left.num_columns(). @@ -370,13 +371,14 @@ std::vector build_join_filter_template_params_from_specs( ? "cudf::jit::join_side::LEFT" : "cudf::jit::join_side::RIGHT"; auto type_name = cudf::type_to_name(table.column(col_spec.column).type()); - template_params.emplace_back(rtcx::reflect_template( - "cudf::jit::join_column_accessor", type_name, std::to_string(col_spec.column), side_str)); + template_params.emplace_back( + jitify2::reflection::Template("cudf::jit::join_column_accessor") + .instantiate(type_name, std::to_string(col_spec.column), side_str)); } else if (std::holds_alternative(spec)) { auto const& scalar_spec = std::get(spec); auto type_name = cudf::type_to_name(scalar_spec.ref.get().type()); - template_params.emplace_back(rtcx::reflect_template( - "cudf::jit::join_scalar_accessor", type_name, std::to_string(scalar_index++))); + template_params.emplace_back(jitify2::reflection::Template("cudf::jit::join_scalar_accessor") + .instantiate(type_name, std::to_string(scalar_index++))); } } @@ -505,9 +507,11 @@ filter_join_indices_jit(cudf::table_view const& left, auto const cuda_source = cudf::jit::parse_single_function_cuda(filter_result.udf, "GENERIC_JOIN_FILTER_OP"); - auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); + auto kernel_name = + jitify2::reflection::Template("cudf::join::jit::filter_join_kernel").instantiate(template_args); auto kernel = - cudf::jit::get_udf_kernel("join/jit/filter_join_kernel.cu", kernel_name, cuda_source); + cudf::jit::get_udf_kernel(*join_jit_filter_join_kernel_cu_jit, kernel_name, cuda_source); + auto configured_kernel = kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); // Collect scalar columns to append to left device views so join_scalar_accessor // can read them at indices >= left.num_columns(). @@ -518,7 +522,7 @@ filter_join_indices_jit(cudf::table_view const& left, // Allocate and compute predicate results auto predicate_results = rmm::device_uvector(left_indices.size(), stream); - launch_join_filter_kernel(kernel, + launch_join_filter_kernel(configured_kernel, left, right, left_indices, diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index 10a94ed01bf6..c8b07bc7a625 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -22,7 +22,6 @@ // clang-format off // This header is an inlined header that defines the GENERIC_JOIN_FILTER_OP function. It is placed here // so the symbols in the headers above can be used by it. -#include #include // clang-format on @@ -94,15 +93,3 @@ CUDF_KERNEL void filter_join_kernel(cudf::jit::device_span left_indices, - cudf::jit::device_span right_indices, - cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - bool* predicate_results, - void* user_data) -{ - CUDF_KERNEL_INSTANCE( - left_indices, right_indices, left_tables, right_tables, predicate_results, user_data); -} diff --git a/cpp/src/rolling/detail/rolling_fixed_window.cu b/cpp/src/rolling/detail/rolling_fixed_window.cu index 1a29aaae40c8..f91fccb35313 100644 --- a/cpp/src/rolling/detail/rolling_fixed_window.cu +++ b/cpp/src/rolling/detail/rolling_fixed_window.cu @@ -1,10 +1,9 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #include "rolling.cuh" -#include "rolling_jit.cuh" #include "rolling_udf.cuh" #include "rolling_utils.cuh" @@ -41,8 +40,10 @@ std::unique_ptr rolling_window(column_view const& input, if (agg.kind == aggregation::CUDA || agg.kind == aggregation::PTX) { // TODO: In future, might need to clamp preceding/following to column boundaries. return cudf::detail::rolling_window_udf(input, - cudf::detail::fixed_window_wrapper(preceding_window), - cudf::detail::fixed_window_wrapper(following_window), + preceding_window, + "cudf::size_type", + following_window, + "cudf::size_type", min_periods, agg, stream, diff --git a/cpp/src/rolling/detail/rolling_jit.cuh b/cpp/src/rolling/detail/rolling_jit.cuh deleted file mode 100644 index 38482a9f5580..000000000000 --- a/cpp/src/rolling/detail/rolling_jit.cuh +++ /dev/null @@ -1,114 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include - -namespace cudf { - -namespace detail { - -template -T minimum(T a, T b) -{ - return b < a ? b : a; -} - -struct window_wrapper_base { - cudf::size_type const* group_offsets = nullptr; - cudf::size_type const* group_labels = nullptr; - cudf::size_type window = 0; -}; - -struct fixed_window_wrapper : public window_wrapper_base { - __device__ __host__ fixed_window_wrapper(cudf::size_type window) - : window_wrapper_base{nullptr, nullptr, window} - { - } - - __device__ __host__ fixed_window_wrapper(window_wrapper_base const& base) - : window_wrapper_base(base) - { - } - - __device__ __host__ cudf::size_type operator[](cudf::size_type) const { return window; } -}; - -struct variable_window_wrapper : public window_wrapper_base { - __device__ __host__ variable_window_wrapper(cudf::size_type const* group_offsets) - : window_wrapper_base{group_offsets, nullptr, 0} - { - } - - __device__ __host__ variable_window_wrapper(window_wrapper_base const& base) - : window_wrapper_base(base) - { - } - - __device__ __host__ cudf::size_type operator[](cudf::size_type idx) const - { - return group_offsets[idx]; - } -}; - -struct preceding_window_wrapper : public window_wrapper_base { - __device__ __host__ preceding_window_wrapper(cudf::size_type const* group_offsets, - cudf::size_type const* group_labels, - cudf::size_type window) - : window_wrapper_base{group_offsets, group_labels, window} - { - } - - __device__ __host__ preceding_window_wrapper(window_wrapper_base const& base) - : window_wrapper_base(base) - { - } - - __device__ cudf::size_type operator[](cudf::size_type idx) const - { - auto group_label = group_labels[idx]; - auto group_start = group_offsets[group_label]; - return minimum(window, idx - group_start + 1); // Preceding includes current row. - } -}; - -struct following_window_wrapper : public window_wrapper_base { - __device__ __host__ following_window_wrapper(cudf::size_type const* group_offsets, - cudf::size_type const* group_labels, - cudf::size_type window) - : window_wrapper_base{group_offsets, group_labels, window} - { - } - - __device__ __host__ following_window_wrapper(window_wrapper_base const& base) - : window_wrapper_base(base) - { - } - - __device__ cudf::size_type operator[](cudf::size_type idx) const - { - auto group_label = group_labels[idx]; - auto group_end = - group_offsets[group_label + - 1]; // Cannot fall off the end, since offsets is capped with `input.size()`. - return minimum(window, (group_end - 1) - idx); - } -}; - -static_assert(sizeof(fixed_window_wrapper) == sizeof(variable_window_wrapper)); -static_assert(alignof(fixed_window_wrapper) == alignof(variable_window_wrapper)); - -static_assert(sizeof(variable_window_wrapper) == sizeof(fixed_window_wrapper)); -static_assert(alignof(variable_window_wrapper) == alignof(fixed_window_wrapper)); - -static_assert(sizeof(fixed_window_wrapper) == sizeof(preceding_window_wrapper)); -static_assert(alignof(fixed_window_wrapper) == alignof(preceding_window_wrapper)); - -static_assert(sizeof(fixed_window_wrapper) == sizeof(following_window_wrapper)); -static_assert(alignof(fixed_window_wrapper) == alignof(following_window_wrapper)); -} // namespace detail - -} // namespace cudf diff --git a/cpp/src/rolling/detail/rolling_jit.hpp b/cpp/src/rolling/detail/rolling_jit.hpp new file mode 100644 index 000000000000..facc9a881837 --- /dev/null +++ b/cpp/src/rolling/detail/rolling_jit.hpp @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2023, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace cudf { + +namespace detail { + +template +T minimum(T a, T b) +{ + return b < a ? b : a; +} + +struct preceding_window_wrapper { + cudf::size_type const* d_group_offsets; + cudf::size_type const* d_group_labels; + cudf::size_type preceding_window; + + cudf::size_type operator[](cudf::size_type idx) + { + auto group_label = d_group_labels[idx]; + auto group_start = d_group_offsets[group_label]; + return minimum(preceding_window, idx - group_start + 1); // Preceding includes current row. + } +}; + +struct following_window_wrapper { + cudf::size_type const* d_group_offsets; + cudf::size_type const* d_group_labels; + cudf::size_type following_window; + + cudf::size_type operator[](cudf::size_type idx) + { + auto group_label = d_group_labels[idx]; + auto group_end = + d_group_offsets[group_label + + 1]; // Cannot fall off the end, since offsets is capped with `input.size()`. + return minimum(following_window, (group_end - 1) - idx); + } +}; + +} // namespace detail + +} // namespace cudf diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index 9115e6642e7c..1fb6d5eabfd0 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -5,12 +5,12 @@ #pragma once +#include "jit/cache.hpp" #include "jit/helpers.hpp" -#include "jit/jit.hpp" #include "jit/parser.hpp" #include "jit/util.hpp" #include "rolling.hpp" -#include "rolling_jit.cuh" +#include "rolling_jit.hpp" #include #include @@ -21,37 +21,24 @@ #include +#include + #include namespace cudf { namespace detail { -template -std::string reflect_window_wrapper() -{ - if constexpr (std::is_same_v) { - return "cudf::detail::fixed_window_wrapper"; - } else if constexpr (std::is_same_v) { - return "cudf::detail::variable_window_wrapper"; - } else if constexpr (std::is_same_v) { - return "cudf::detail::preceding_window_wrapper"; - } else { - static_assert(std::is_same_v, "Unsupported window wrapper type"); - return "cudf::detail::following_window_wrapper"; - } -} - // Applies a user-defined rolling window function to the values in a column. -static std::unique_ptr rolling_window_udf_impl( - column_view const& input, - std::string const& preceding_window_str, - cudf::detail::window_wrapper_base const& preceding_window, - std::string const& following_window_str, - cudf::detail::window_wrapper_base const& following_window, - size_type min_periods, - rolling_aggregation const& agg, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +template +std::unique_ptr rolling_window_udf(column_view const& input, + PrecedingWindowIterator preceding_window, + std::string const& preceding_window_str, + FollowingWindowIterator following_window, + std::string const& following_window_str, + size_type min_periods, + rolling_aggregation const& agg, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { static_assert(warp_size == cudf::detail::size_in_bits(), "bitmask_type size does not match CUDA warp size"); @@ -88,29 +75,25 @@ static std::unique_ptr rolling_window_udf_impl( cudf::detail::device_scalar device_valid_count{ 0, stream, cudf::get_current_device_resource_ref()}; - auto kernel_reflection = - rtcx::reflect_template("cudf::rolling::jit::rolling_window_kernel", - cudf::type_to_name(input.type()), // list of template arguments - cudf::type_to_name(output->type()), - udf_agg._operator_name, - preceding_window_str, - following_window_str); - - auto kernel = cudf::jit::get_udf_kernel("rolling/jit/kernel.cu", kernel_reflection, cuda_source); - auto cfg = kernel.max_occupancy_config(0, 0); - kernel.launch_with({cfg.min_grid_size}, - {cfg.block_size}, - 0, - stream, - input.size(), - cudf::jit::get_data_ptr(input), - input.null_mask(), - cudf::jit::get_data_ptr(output->mutable_view()), - output_view.null_mask(), - device_valid_count.data(), - preceding_window, - following_window, - min_periods); + std::string kernel_reflection = + jitify2::reflection::Template("cudf::rolling::jit::gpu_rolling_new") // + .instantiate(cudf::type_to_name(input.type()), // list of template arguments + cudf::type_to_name(output->type()), + udf_agg._operator_name, + preceding_window_str.c_str(), + following_window_str.c_str()); + + cudf::jit::get_udf_kernel(*rolling_jit_kernel_cu_jit, kernel_reflection, cuda_source) + ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()) + ->launch(input.size(), + cudf::jit::get_data_ptr(input), + input.null_mask(), + cudf::jit::get_data_ptr(output_view), + output_view.null_mask(), + device_valid_count.data(), + preceding_window, + following_window, + min_periods); output->set_null_count(output->size() - device_valid_count.value(stream)); @@ -120,26 +103,5 @@ static std::unique_ptr rolling_window_udf_impl( return output; } -// Applies a user-defined rolling window function to the values in a column. -template -std::unique_ptr rolling_window_udf(column_view const& input, - PrecedingWindowIterator preceding_window, - FollowingWindowIterator following_window, - size_type min_periods, - rolling_aggregation const& agg, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - return rolling_window_udf_impl(input, - reflect_window_wrapper(), - preceding_window, - reflect_window_wrapper(), - following_window, - min_periods, - agg, - stream, - mr); -} - } // namespace detail } // namespace cudf diff --git a/cpp/src/rolling/detail/rolling_variable_window.cu b/cpp/src/rolling/detail/rolling_variable_window.cu index 504d639f24de..155adf2d74a4 100644 --- a/cpp/src/rolling/detail/rolling_variable_window.cu +++ b/cpp/src/rolling/detail/rolling_variable_window.cu @@ -1,10 +1,9 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #include "rolling.cuh" -#include "rolling_jit.cuh" #include "rolling_udf.cuh" #include @@ -38,14 +37,15 @@ std::unique_ptr rolling_window(column_view const& input, if (agg.kind == aggregation::CUDA || agg.kind == aggregation::PTX) { // TODO: In future, might need to clamp preceding/following to column boundaries. - return cudf::detail::rolling_window_udf( - input, - cudf::detail::variable_window_wrapper{preceding_window.begin()}, - cudf::detail::variable_window_wrapper{following_window.begin()}, - min_periods, - agg, - stream, - mr); + return cudf::detail::rolling_window_udf(input, + preceding_window.begin(), + "cudf::size_type*", + following_window.begin(), + "cudf::size_type*", + min_periods, + agg, + stream, + mr); } else { auto defaults_col = cudf::is_dictionary(input.type()) ? dictionary_column_view(input).indices() : input; diff --git a/cpp/src/rolling/grouped_rolling.cu b/cpp/src/rolling/grouped_rolling.cu index e39b1591dbc7..1b45758e7002 100644 --- a/cpp/src/rolling/grouped_rolling.cu +++ b/cpp/src/rolling/grouped_rolling.cu @@ -6,7 +6,6 @@ #include "detail/optimized_unbounded_window.hpp" #include "detail/range_window_bounds.hpp" #include "detail/rolling.cuh" -#include "detail/rolling_jit.cuh" #include "detail/rolling_udf.cuh" #include "detail/rolling_utils.cuh" @@ -93,16 +92,21 @@ std::unique_ptr grouped_rolling_window(table_view const& group_keys, // 3. [0, 500, 1000] indicates two equal-sized groups: [0,500), and [500,1000). if (aggr.kind == aggregation::CUDA || aggr.kind == aggregation::PTX) { - return cudf::detail::rolling_window_udf( - input, - cudf::detail::preceding_window_wrapper{ - group_offsets.data(), group_labels.data(), preceding_window}, - cudf::detail::following_window_wrapper{ - group_offsets.data(), group_labels.data(), following_window}, - min_periods, - aggr, - stream, - mr); + cudf::detail::preceding_window_wrapper grouped_preceding_window{ + group_offsets.data(), group_labels.data(), preceding_window}; + + cudf::detail::following_window_wrapper grouped_following_window{ + group_offsets.data(), group_labels.data(), following_window}; + + return cudf::detail::rolling_window_udf(input, + grouped_preceding_window, + "cudf::detail::preceding_window_wrapper", + grouped_following_window, + "cudf::detail::following_window_wrapper", + min_periods, + aggr, + stream, + mr); } else { namespace utils = cudf::detail::rolling; auto groups = utils::grouped{group_labels.data(), group_offsets.data()}; diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index 7aa18382c361..2a6e5ff7aa81 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -7,13 +7,12 @@ #include #include -#include +#include #pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before // it. Each UDF will have a different operation-udf.hpp generated for it, so we // need to put this pragma before including it to avoid PCH mismatch. -#include #include struct rolling_udf_ptx { @@ -40,34 +39,42 @@ namespace cudf { namespace rolling { namespace jit { +template +cudf::size_type __device__ get_window(WindowType window, cudf::thread_index_type index) +{ + return window[index]; +} + +template <> +cudf::size_type __device__ get_window(cudf::size_type window, cudf::thread_index_type index) +{ + return window; +} + template -CUDF_KERNEL void rolling_window_kernel(cudf::size_type nrows, - void const* __restrict__ p_in_col, - cudf::bitmask_type const* __restrict__ in_col_valid, - void* __restrict__ p_out_col, - cudf::bitmask_type* __restrict__ out_col_valid, - cudf::size_type* __restrict__ output_valid_count, - detail::window_wrapper_base b_preceding_window_begin, - detail::window_wrapper_base b_following_window_begin, - cudf::size_type min_periods) +CUDF_KERNEL void gpu_rolling_new(cudf::size_type nrows, + InType const* const __restrict__ in_col, + cudf::bitmask_type const* const __restrict__ in_col_valid, + OutType* __restrict__ out_col, + cudf::bitmask_type* __restrict__ out_col_valid, + cudf::size_type* __restrict__ output_valid_count, + PrecedingWindowType preceding_window_begin, + FollowingWindowType following_window_begin, + cudf::size_type min_periods) { - auto i = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); - PrecedingWindowType const preceding_window_begin = b_preceding_window_begin; - FollowingWindowType const following_window_begin = b_following_window_begin; - auto const* __restrict__ in_col = static_cast(p_in_col); - auto* __restrict__ out_col = static_cast(p_out_col); + auto i = cudf::detail::grid_1d::global_thread_id(); + auto const stride = cudf::detail::grid_1d::grid_stride(); cudf::size_type warp_valid_count{0}; auto active_threads = __ballot_sync(0xffff'ffffu, i < nrows); while (i < nrows) { - int64_t const preceding_window = preceding_window_begin[i]; - int64_t const following_window = following_window_begin[i]; + int64_t const preceding_window = get_window(preceding_window_begin, i); + int64_t const following_window = get_window(following_window_begin, i); // compute bounds auto const start = static_cast( @@ -105,32 +112,10 @@ CUDF_KERNEL void rolling_window_kernel(cudf::size_type nrows, } // TODO: likely faster to do a single_lane_block_reduce and a single - // atomic per block but that requires jit-compiling single_lane_block_reduce... + // atomic per block but that requires jitifying single_lane_block_reduce... if (0 == cudf::intra_word_index(threadIdx.x)) { atomicAdd(output_valid_count, warp_valid_count); } } } // namespace jit } // namespace rolling } // namespace cudf - -extern "C" __global__ void cudf_kernel_entry( - cudf::size_type nrows, - void const* const __restrict__ in_col, - cudf::bitmask_type const* const __restrict__ in_col_valid, - void* __restrict__ out_col, - cudf::bitmask_type* __restrict__ out_col_valid, - cudf::size_type* __restrict__ output_valid_count, - cudf::detail::window_wrapper_base preceding_window_begin, - cudf::detail::window_wrapper_base following_window_begin, - cudf::size_type min_periods) -{ - CUDF_KERNEL_INSTANCE(nrows, - in_col, - in_col_valid, - out_col, - out_col_valid, - output_valid_count, - preceding_window_begin, - following_window_begin, - min_periods); -} diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 118ab8cc6770..6e78060e8792 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -7,19 +7,17 @@ #include "io/comp/nvcomp_adapter.hpp" #include "io/utilities/getenv_or.hpp" -#include "jit/jit.hpp" -#include "rtcx.hpp" +#include "jit/cache.hpp" #include #include -#include #include namespace cudf { -context::context(context_config cfg, init_flags flags) - : _config{std::move(cfg)}, _jit_cache_init_flag{} +context::context(context_config const& cfg, init_flags flags) + : _config{cfg}, _program_cache_init_flag{}, _program_cache{nullptr} { initialize_components(flags); } @@ -28,54 +26,20 @@ void context::ensure_nvcomp_loaded() { io::detail::nvcomp::load_nvcomp_library() void context::ensure_jit_cache_initialized() { - std::call_once(_jit_cache_init_flag, [&]() { - // make sure the required directories exist - std::filesystem::create_directories(_config.rtcx_cache_dir); - std::filesystem::create_directories(_config.jit_bundle_dir); - std::filesystem::create_directories(_config.jit_pch_dir); - std::filesystem::create_directories(_config.jit_tmp_dir); - - rtcx::initialize(); - - auto limits = rtcx::cache_limits{.num_mem_blobs = _config.kernel_cache_limit_process, - .num_mem_libraries = _config.kernel_cache_limit_process}; - - _rtcx_cache = std::make_unique(_config.rtcx_cache_dir, - _config.jit_tmp_dir, - limits, - bool{_config.preload_jit_cache}, - bool{_config.disable_jit_cache}); - - if (_config.clear_jit_cache) { - _rtcx_cache->clear_memory_store(); - _rtcx_cache->clear_disk_store(); - } - - // note that jit_bundle depends on rtcx_cache, so we ensure rtcx_cache is initialized first. - _jit_bundle = std::make_unique(_config.jit_bundle_dir, *_rtcx_cache); - }); + std::call_once(_program_cache_init_flag, + [&]() { _program_cache = jit::program_cache::create(); }); } -context::~context() { rtcx::teardown(); } - -rtcx::cache_t& context::rtcx_cache() +jit::program_cache& context::program_cache() { ensure_jit_cache_initialized(); - return *_rtcx_cache; -} - -jit_bundle_t& context::jit_bundle() -{ - ensure_jit_cache_initialized(); - return *_jit_bundle; + return *_program_cache; } bool context::dump_codegen() const { return _config.dump_codegen; } bool context::use_jit() const { return _config.use_jit; } -std::string const& context::get_jit_pch_dir() const { return _config.jit_pch_dir; } - void context::initialize_components(init_flags flags) { if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { ensure_jit_cache_initialized(); } @@ -87,24 +51,6 @@ static std::optional _context{std::nullopt}; static std::optional _context_init_flag{std::in_place}; static std::optional _context_deinit_flag{std::in_place}; -std::filesystem::path get_cudf_kernel_cache_dir() -{ - if (auto cudf = getenv_optional("LIBCUDF_KERNEL_CACHE_PATH"); cudf.has_value()) { - return std::filesystem::path(*cudf); - } - - if (auto home = getenv_optional("HOME"); home.has_value()) { - return std::filesystem::path(*home) / ".libcudf"; - } - - CUDF_FAIL( - "Unable to determine the CUDF root directory. Please set the `LIBCUDF_KERNEL_CACHE_PATH` or " - "`HOME` " - "environment variables to allow automatic resolution of the root " - "directory.", - std::runtime_error); -} - } // namespace cudf namespace CUDF_EXPORT cudf { @@ -112,39 +58,15 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { std::call_once(*_context_init_flag, [&]() { - bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); - bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); - bool preload_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_PRELOAD", false); - bool disable_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_DISABLED", false); - bool clear_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); - bool disable_cuda_cache = get_bool_env_or("LIBCUDF_JIT_DISABLE_CUDA_CACHE", false); - bool jit_verbose = get_bool_env_or("LIBCUDF_JIT_VERBOSE", false); - bool dump_jit_trace = get_bool_env_or("LIBCUDF_JIT_DUMP_TRACE", false); - bool dump_jit_time_profile = get_bool_env_or("LIBCUDF_JIT_DUMP_TIME_PROFILE", false); - - auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16'384U); + bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); + bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); - auto jit_bundle_dir = get_cudf_kernel_cache_dir() / "bundle"; - auto rtcx_cache_dir = get_cudf_kernel_cache_dir() / "rtcx_cache"; - auto jit_pch_dir = get_cudf_kernel_cache_dir() / "pch"; - auto jit_tmp_dir = get_cudf_kernel_cache_dir() / "tmp"; - - context_config cfg{.dump_codegen = dump_codegen, - .use_jit = use_jit, - .preload_jit_cache = preload_jit_cache, - .disable_jit_cache = disable_jit_cache, - .clear_jit_cache = clear_jit_cache, - .disable_cuda_cache = disable_cuda_cache, - .jit_verbose = jit_verbose, - .dump_jit_trace = dump_jit_trace, - .dump_jit_time_profile = dump_jit_time_profile, - .rtcx_cache_dir = rtcx_cache_dir, - .jit_bundle_dir = jit_bundle_dir, - .jit_pch_dir = jit_pch_dir, - .jit_tmp_dir = jit_tmp_dir, - .kernel_cache_limit_process = kernel_cache_limit_process}; + context_config cfg{ + .dump_codegen = dump_codegen, + .use_jit = use_jit, + }; _context.emplace(cfg, flags); }); @@ -163,18 +85,9 @@ void teardown() }); } -void enable_jit_cache(bool enabled) -{ - auto& cache = get_context().rtcx_cache(); - cache.enable(enabled); -} +void enable_jit_cache(bool enable) { get_context().program_cache().enable(enable); } -void clear_jit_cache() -{ - auto& cache = get_context().rtcx_cache(); - cache.clear_memory_store(); - cache.clear_disk_store(); -} +void clear_jit_cache() { get_context().program_cache().clear(); } context& get_context() { diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 559adb0abd8c..f95b9684034e 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -11,41 +11,26 @@ #include #include -namespace rtcx { -struct cache_t; -} // namespace rtcx - namespace cudf { -struct jit_bundle_t; +namespace jit { +class program_cache; +} struct [[nodiscard]] context_config { - bool dump_codegen : 1 = false; - bool use_jit : 1 = false; - bool preload_jit_cache : 1 = false; - bool disable_jit_cache : 1 = false; - bool clear_jit_cache : 1 = false; - bool disable_codegen_cache : 1 = false; - bool disable_cuda_cache : 1 = false; - bool jit_verbose : 1 = false; - bool dump_jit_trace : 1 = false; - bool dump_jit_time_profile : 1 = false; - std::string rtcx_cache_dir = {}; - std::string jit_bundle_dir = {}; - std::string jit_pch_dir = {}; - std::string jit_tmp_dir = {}; - uint32_t kernel_cache_limit_process = 0; + bool dump_codegen = false; + bool use_jit = false; }; /// @brief The context object contains global state internal to CUDF. /// It helps to ensure structured and well-defined construction and destruction of global /// objects/state across translation units. class context { + public: private: context_config _config; - std::once_flag _jit_cache_init_flag; - std::unique_ptr _rtcx_cache; - std::unique_ptr _jit_bundle; + std::once_flag _program_cache_init_flag; + std::unique_ptr _program_cache; private: void ensure_nvcomp_loaded(); @@ -53,25 +38,19 @@ class context { void ensure_jit_cache_initialized(); public: - context(context_config cfg = {}, init_flags flags = init_flags::DEFAULT); + context(context_config const& cfg = {}, init_flags flags = init_flags::INIT_JIT_CACHE); context(context const&) = delete; context& operator=(context const&) = delete; context(context&&) = delete; context& operator=(context&&) = delete; - ~context(); - - rtcx::cache_t& rtcx_cache(); + ~context() = default; - jit_bundle_t& jit_bundle(); + jit::program_cache& program_cache(); [[nodiscard]] bool dump_codegen() const; [[nodiscard]] bool use_jit() const; - [[nodiscard]] context_config const& config() const { return _config; } - - [[nodiscard]] std::string const& get_jit_pch_dir() const; - /// @brief Initialize additional components based on the provided flags /// @param flags The initialization flags to process void initialize_components(init_flags flags); diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index badfdd7463b5..2456433d0bc6 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -6,8 +6,6 @@ #include #include #include -#include -#include #include #include #include @@ -30,87 +28,81 @@ // clang-format off // This header is an inlined header that defines the GENERIC_TRANSFORM_OP function. It is placed here // so the symbols in the headers above can be used by it. -#include #include // clang-format on namespace cudf { namespace jit { +template +__device__ void execute_transform_op(void* user_data, size_type element_idx, Args args) +{ + // TODO: static assert invocable + if constexpr (has_user_data) { + cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, + cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, args)); + } else { + cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, args); + } +} + /// @brief The generic transform kernel. Supports all types and nullability combinations. -template -__device__ void transform_kernel(size_type row_size, - bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols) +template +CUDF_KERNEL void transform_kernel(size_type row_size, + bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + column_device_view_core const* __restrict__ input_cols, + mutable_column_device_view_core const* __restrict__ output_cols) { + // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); - for (auto row = start; row < row_size; row += stride) { -#ifndef CUDF_LTO_MODE - - auto operation = [&](Args const& args) { - if constexpr (has_user_data) { - cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, - cuda::std::tuple_cat(cuda::std::tuple{user_data, row}, args)); - } else { - cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, args); - } - }; - -#else - - auto operation = [&](Args const& args) { - static_assert(!has_user_data); - static_assert(OutputAccessors::size == 1); - static_assert(InputAccessors::size == 2 || InputAccessors::size == 1); - cuda::std::apply( - [&](auto... a) { - if constexpr (InputAccessors::size == 1) { - cudf::lto::unary_operator(a...); - } else if constexpr (InputAccessors::size == 2) { - cudf::lto::binary_operator(a...); - } - }, - args); - }; - -#endif - - if constexpr (!is_null_aware) { - if (stencil != nullptr && !bit_is_set(stencil, row)) { continue; } + for (auto element_idx = start; element_idx < row_size; element_idx += stride) { + if constexpr (is_null_aware == null_aware::NO) { + if (stencil != nullptr && !bit_is_set(stencil, element_idx)) { continue; } auto ins = InputAccessors::map( - [&]() { return cuda::std::tuple{A::element(input_cols, row)...}; }); + [&]() { return cuda::std::tuple{A::element(input_cols, element_idx)...}; }); - auto outs = OutputAccessors::map( - [&]() { return cuda::std::tuple{A::output_arg(output_cols, row)...}; }); + auto outs = OutputAccessors::map([&]() { + return cuda::std::tuple{A::output_arg(output_cols, element_idx)...}; + }); + + auto out_ptrs = + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - operation(cuda::std::tuple_cat( - cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs), ins)); + execute_transform_op( + user_data, element_idx, cuda::std::tuple_cat(out_ptrs, ins)); OutputAccessors::map([&]() { - (A::assign(output_cols, row, cuda::std::get(outs)), ...); + (A::assign(output_cols, element_idx, cuda::std::get(outs)), ...); }); } else { - auto ins = InputAccessors::map( - [&]() { return cuda::std::tuple{A::nullable_element(input_cols, row)...}; }); + auto active_mask = __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); - auto outs = OutputAccessors::map( - [&]() { return cuda::std::tuple{A::null_output_arg(output_cols, row)...}; }); + auto ins = InputAccessors::map([&]() { + return cuda::std::tuple{A::nullable_element(input_cols, element_idx)...}; + }); + + auto outs = OutputAccessors::map([&]() { + return cuda::std::tuple{A::null_output_arg(output_cols, element_idx)...}; + }); - operation(cuda::std::tuple_cat( - cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs), ins)); + auto out_ptrs = + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - auto active_mask = __ballot_sync(0xFFFF'FFFFU, row < row_size); + execute_transform_op( + user_data, element_idx, cuda::std::tuple_cat(out_ptrs, ins)); OutputAccessors::map([&]() { - (A::assign(output_cols, row, *cuda::std::get(outs)), ...); + (A::assign(output_cols, element_idx, *cuda::std::get(outs)), ...); (warp_compact_validity( - active_mask, output_cols, row, cuda::std::get(outs).has_value()), + active_mask, output_cols, element_idx, cuda::std::get(outs).has_value()), ...); }); } @@ -119,13 +111,3 @@ __device__ void transform_kernel(size_type row_size, } // namespace jit } // namespace cudf - -extern "C" __global__ void cudf_kernel_entry( - cudf::size_type row_size, - cudf::bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - cudf::column_device_view_core const* __restrict__ input_cols, - cudf::mutable_column_device_view_core const* __restrict__ output_cols) -{ - CUDF_KERNEL_INSTANCE(row_size, stencil, user_data, input_cols, output_cols); -} diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index b7a21747af81..dd905f59770c 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -23,13 +23,13 @@ #include -#include +#include #include -#include #include #include #include #include +#include #include #include @@ -158,14 +158,14 @@ using handle = std::variant< namespace jit_transform { -cudf::kernel instantiate(bool null_aware, - bool has_user_data, - std::string const& ins, - std::string const& outs, - std::vector const& ptx_input_types, - std::vector const& ptx_output_types, - std::string const& udf, - udf_source_type source_type) +jitify2::Kernel instantiate(null_aware is_null_aware, + bool has_user_data, + std::string const& ins, + std::string const& outs, + std::vector const& ptx_input_types, + std::vector const& ptx_output_types, + std::string const& udf, + udf_source_type source_type) { CUDF_FUNC_RANGE(); auto cuda_source = (source_type == udf_source_type::PTX) @@ -175,16 +175,14 @@ cudf::kernel instantiate(bool null_aware, jit::build_ptx_params(ptx_output_types, ptx_input_types, has_user_data)) : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); - auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", - rtcx::reflect_bool(null_aware), - rtcx::reflect_bool(has_user_data), - ins, - outs); + auto kernel = jitify2::reflection::Template("cudf::jit::transform_kernel") + .instantiate(is_null_aware, has_user_data, ins, outs); - return jit::get_udf_kernel("transform/jit/kernel.cu", kernel, cuda_source); + return jit::get_udf_kernel( + *transform_jit_kernel_cu_jit, kernel, cuda_source, {"-restrict", "--dopt=on"}); } -void launch(cudf::kernel const& kernel, +void launch(jitify2::Kernel const& kernel, size_type row_size, bitmask_type const* stencil, void* user_data, @@ -193,13 +191,8 @@ void launch(cudf::kernel const& kernel, rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - void* args[] = {&row_size, &stencil, &user_data, &input_cols, &output_cols}; - auto kernel_ref = kernel.get(); - auto cfg = kernel_ref.max_occupancy_config(0, 0); - CUDF_EXPECTS(cfg.block_size % cudf::detail::warp_size == 0, - "Expected block size to be a multiple of warp size", - std::runtime_error); - kernel_ref.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); + void* args[] = {&row_size, &stencil, &user_data, &input_cols, &output_cols}; + kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); } std::string reflect_input_element(column_view const& c) { return type_to_name(c.type()); } @@ -251,11 +244,8 @@ auto reflect(udf_source_type source_type, auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); bool as_scalar = std::holds_alternative(in); - auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", - rtcx::reflect_int(i), - column, - element, - rtcx::reflect_bool(as_scalar)); + auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") + .instantiate(i, column, element, as_scalar); in_types.push_back(accessor); } @@ -266,17 +256,14 @@ auto reflect(udf_source_type source_type, auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); bool as_scalar = false; // never scalar - auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", - rtcx::reflect_int(i), - column, - element, - rtcx::reflect_bool(as_scalar)); + auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") + .instantiate(i, column, element, as_scalar); out_types.push_back(accessor); } - auto ins = rtcx::reflect_template("cudf::jit::type_list", in_types); - auto outs = rtcx::reflect_template("cudf::jit::type_list", out_types); + auto ins = jitify2::reflection::Template("cudf::jit::type_list").instantiate(in_types); + auto outs = jitify2::reflection::Template("cudf::jit::type_list").instantiate(out_types); std::vector ptx_in_types; std::vector ptx_out_types; @@ -332,7 +319,7 @@ auto to_args(std::span inputs, return std::make_tuple(std::move(d_args), std::move(handles)); } -void run(bool null_aware, +void run(null_aware is_null_aware, bool has_user_data, size_type row_size, bitmask_type const* d_stencil, @@ -345,10 +332,16 @@ void run(bool null_aware, rmm::device_async_resource_ref mr) { auto [in_types, out_types, ptx_in_types, ptx_out_types] = reflect(source_type, inputs, outputs); - auto kernel = instantiate( - null_aware, has_user_data, in_types, out_types, ptx_in_types, ptx_out_types, udf, source_type); - auto [cols, handles] = to_args(inputs, outputs, stream, mr); - auto* input_cols = reinterpret_cast(cols.data()); + auto kernel = instantiate(is_null_aware, + has_user_data, + in_types, + out_types, + ptx_in_types, + ptx_out_types, + udf, + source_type); + auto [cols, handles] = to_args(inputs, outputs, stream, mr); + auto* input_cols = reinterpret_cast(cols.data()); auto* output_cols = reinterpret_cast(input_cols + inputs.size()); return launch(kernel, row_size, d_stencil, user_data, input_cols, output_cols, stream); @@ -497,15 +490,14 @@ auto get_null_transformation(null_aware is_null_aware, return output_may_be_nullable; } -void perform_checks(std::variant source_type, +void perform_checks(udf_source_type source_type, null_aware is_null_aware, std::optional in_row_size, std::span inputs, std::span outputs, std::span const> string_offsets) { - if (auto* udf_source = std::get_if(&source_type); - udf_source != nullptr && *udf_source == udf_source_type::PTX) { + if (source_type == udf_source_type::PTX) { CUDF_EXPECTS(std::none_of(inputs.begin(), inputs.end(), [](auto& in) { @@ -527,25 +519,6 @@ void perform_checks(std::variant source_type, CUDF_EXPECTS(is_null_aware == null_aware::NO, "PTX UDFs do not support null-aware transformations", std::invalid_argument); - } else if (std::holds_alternative(source_type)) { - [[maybe_unused]] auto binary_type = std::get(source_type); - CUDF_EXPECTS( - std::none_of(inputs.begin(), - inputs.end(), - [](auto& in) { - return std::visit( - [](auto& c) { - return !is_fixed_width(c.type()) && c.type().id() != type_id::STRING; - }, - in); - }), - "Transforms with LTO binaries only support fixed-width types and strings as inputs", - std::invalid_argument); - CUDF_EXPECTS( - std::none_of( - outputs.begin(), outputs.end(), [](auto& out) { return !is_fixed_width(out.type); }), - "Transforms with LTO binaries only support output of fixed-width types", - std::invalid_argument); } CUDF_EXPECTS(std::none_of(outputs.begin(), @@ -846,7 +819,7 @@ std::unique_ptr
execute_transform(std::string const& udf, auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; - jit_transform::run(is_null_aware == null_aware::YES, + jit_transform::run(is_null_aware, user_data.has_value(), row_size, stencil_has_nulls ? stencil_arg : nullptr, @@ -964,199 +937,4 @@ std::unique_ptr compute_column_jit(table_view const& table, mr); } -std::string_view as_tag(type_id id) -{ - switch (id) { - case type_id::BOOL8: return "bool8"; - case type_id::INT8: return "int8"; - case type_id::INT16: return "int16"; - case type_id::INT32: return "int32"; - case type_id::INT64: return "int64"; - case type_id::UINT8: return "uint8"; - case type_id::UINT16: return "uint16"; - case type_id::UINT32: return "uint32"; - case type_id::UINT64: return "uint64"; - case type_id::FLOAT32: return "float32"; - case type_id::FLOAT64: return "float64"; - case type_id::STRING: return "string_view"; - case type_id::DECIMAL32: return "decimal32"; - case type_id::DECIMAL64: return "decimal64"; - case type_id::DECIMAL128: return "decimal128"; - case type_id::DURATION_DAYS: return "duration_D"; - case type_id::DURATION_SECONDS: return "duration_s"; - case type_id::DURATION_MILLISECONDS: return "duration_ms"; - case type_id::DURATION_MICROSECONDS: return "duration_us"; - case type_id::DURATION_NANOSECONDS: return "duration_ns"; - case type_id::TIMESTAMP_DAYS: return "timestamp_D"; - case type_id::TIMESTAMP_SECONDS: return "timestamp_s"; - case type_id::TIMESTAMP_MILLISECONDS: return "timestamp_ms"; - case type_id::TIMESTAMP_MICROSECONDS: return "timestamp_us"; - case type_id::TIMESTAMP_NANOSECONDS: return "timestamp_ns"; - default: CUDF_FAIL("Unsupported type for JIT dispatch", std::invalid_argument); - } -} - -std::optional> dispatch_unop_lto_kernel( - bool null_aware, - std::span inputs, - std::span outputs) -{ - auto input_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); - - for (size_t i = 0; i < std::size(cudf_fragments::unop_lto_kernel_FILE_INDEX); i++) { - auto FILE_INDEX = cudf_fragments::unop_lto_kernel_FILE_INDEX[i]; - auto NULL_AWARE = cudf_fragments::unop_lto_kernel_NULL_AWARE[i]; - auto TYPE = cudf_fragments::unop_lto_kernel_TYPE[i]; - if (as_tag(input_type) == TYPE && null_aware == NULL_AWARE) { - auto range = cudf_fragments::file_ranges[FILE_INDEX]; - return cudf_fragments::files.subspan(range[0], range[1]); - } - } - - return std::nullopt; -} - -std::optional> dispatch_binop_lto_kernel( - bool null_aware, - std::span inputs, - std::span outputs) -{ - auto input_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); - auto lhs_is_scalar = std::holds_alternative(inputs[0]); - auto rhs_is_scalar = std::holds_alternative(inputs[1]); - - if (lhs_is_scalar) { return std::nullopt; } - - for (size_t i = 0; i < std::size(cudf_fragments::binop_lto_kernel_FILE_INDEX); i++) { - auto FILE_INDEX = cudf_fragments::binop_lto_kernel_FILE_INDEX[i]; - auto NULL_AWARE = cudf_fragments::binop_lto_kernel_NULL_AWARE[i]; - auto TYPE = cudf_fragments::binop_lto_kernel_TYPE[i]; - auto RHS_IS_SCALAR = cudf_fragments::binop_lto_kernel_RHS_IS_SCALAR[i]; - if (as_tag(input_type) == TYPE && null_aware == NULL_AWARE && rhs_is_scalar == RHS_IS_SCALAR) { - auto range = cudf_fragments::file_ranges[FILE_INDEX]; - return cudf_fragments::files.subspan(range[0], range[1]); - } - } - - return std::nullopt; -} - -// Dispatches to the appropriate LTO kernel based on the number of inputs and outputs, their types, -// and nullability. -std::span dispatch_lto_kernel(bool null_aware, - std::span inputs, - std::span outputs) -{ - if (inputs.size() == 1 && outputs.size() == 1) { - auto input0_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); - auto output_type = outputs[0].type.id(); - if (input0_type == output_type && is_fixed_width(data_type{input0_type})) { - if (auto kernel = dispatch_unop_lto_kernel(null_aware, inputs, outputs)) { return *kernel; } - } - } - - if (inputs.size() == 2 && outputs.size() == 1) { - auto input0_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); - auto input1_type = std::visit([](auto& c) { return c.type().id(); }, inputs[1]); - auto output_type = outputs[0].type.id(); - if (input0_type == output_type && input1_type == output_type && - is_fixed_width(data_type{input0_type})) { - if (auto kernel = dispatch_binop_lto_kernel(null_aware, inputs, outputs)) { return *kernel; } - } - } - - CUDF_FAIL("No suitable LTO kernel found for the given transform parameters", - std::invalid_argument); -} - -rtcx::binary_type as_rtcx_binary_type(lto_binary_type type) -{ - switch (type) { - case lto_binary_type::LTO_IR: return rtcx::binary_type::LTO_IR; - case lto_binary_type::FATBIN: return rtcx::binary_type::FATBIN; - default: - CUDF_FAIL( - std::format("Unrecognized LTO binary type {} for LTO transform", static_cast(type)), - std::invalid_argument); - } -} - -std::unique_ptr
transform_lto(std::span inputs, - std::span udf, - lto_binary_type binary_type, - std::span outputs, - null_aware is_null_aware, - std::optional in_row_size, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - CUDF_FUNC_RANGE(); - perform_checks(binary_type, is_null_aware, in_row_size, inputs, outputs, {}); - auto row_size = in_row_size.has_value() ? *in_row_size : jit::get_projection_size(inputs); - auto output_may_be_nullable = get_null_transformation(is_null_aware, inputs, outputs); - - auto [output_columns, stencil] = - make_outputs(is_null_aware, row_size, inputs, outputs, output_may_be_nullable, {}, stream, mr); - auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; - auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; - auto kernel_fatbin = dispatch_lto_kernel(is_null_aware == null_aware::YES, inputs, outputs); - - rtcx::memory_fragment fragments[] = { - {.data = kernel_fatbin, .type = rtcx::binary_type::FATBIN, .name = "kernel"}, - {.data = udf, .type = as_rtcx_binary_type(binary_type), .name = "udf"}}; - - auto kernel = get_lto_linked_kernel("transform_lto_kernel", {}, fragments); - auto [cols, handles] = jit_transform::to_args(inputs, output_columns, stream, mr); - cudf::size_type num_inputs = static_cast(inputs.size()); - cudf::size_type num_outputs = static_cast(outputs.size()); - cudf::size_type num_rows = row_size; - cudf::bitmask_type* p_stencil = stencil_has_nulls ? stencil_arg : nullptr; - auto* input_cols = reinterpret_cast(cols.data()); - auto* output_cols = - reinterpret_cast(input_cols + inputs.size()); - - auto cfg = kernel.max_occupancy_config(0, 0); - void* user_data = nullptr; - - void* args[] = {&num_rows, &p_stencil, &user_data, &input_cols, &output_cols}; - - kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); - - auto finalized = finalize_outputs(is_null_aware, row_size, std::move(output_columns), stream, mr); - return std::make_unique
(std::move(finalized)); -} - -std::unique_ptr unary_op_lto(column_view input, - transform_output output, - std::span udf, - lto_binary_type binary_type, - null_aware is_null_aware, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - transform_input inputs[] = {input}; - transform_output outputs[] = {output}; - auto table = - transform_lto(inputs, udf, binary_type, outputs, is_null_aware, std::nullopt, stream, mr); - auto cols = table->release(); - return std::move(cols[0]); -} - -std::unique_ptr binary_op_lto(column_view lhs, - transform_input rhs, - transform_output output, - std::span udf, - lto_binary_type binary_type, - null_aware is_null_aware, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - transform_input inputs[] = {lhs, rhs}; - transform_output outputs[] = {output}; - auto table = - transform_lto(inputs, udf, binary_type, outputs, is_null_aware, std::nullopt, stream, mr); - auto cols = table->release(); - return std::move(cols[0]); -} - } // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index de41f68cbe69..bae781f36bda 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -683,30 +683,6 @@ ConfigureTest(ENCODE_TEST encode/encode_tests.cpp) # * ast tests ------------------------------------------------------------------------------------- ConfigureTest(AST_TEST ast/transform_tests.cpp ast/ast_tree_tests.cpp) -# ################################################################################################## -# * transform LTO tests -# ------------------------------------------------------------------------------------- -add_embed(cudf_test_fragments) - -add_fragment(cudf_test_fragments FRAGMENT invsqrt SOURCE transform/fragments/invsqrt.cu) - -add_fragment( - cudf_test_fragments FRAGMENT sum_of_squares SOURCE transform/fragments/sum_of_squares.cu -) - -add_fragment( - cudf_test_fragments FRAGMENT decimal_square SOURCE transform/fragments/decimal_square.cu -) - -embed(cudf_test_fragments COMPRESSION none) - -ConfigureTest( - TRANSFORM_LTO_TEST transform/transform_lto_test.cpp - ${cudf_test_fragments_SOURCE_DIR}/cudf_test_fragments.s -) -add_dependencies(TRANSFORM_LTO_TEST cudf_test_fragments) -target_include_directories(TRANSFORM_LTO_TEST PRIVATE ${cudf_test_fragments_SOURCE_DIR}) - # ################################################################################################## # * lists tests ---------------------------------------------------------------------------------- ConfigureTest( diff --git a/cpp/tests/transform/fragments/decimal_square.cu b/cpp/tests/transform/fragments/decimal_square.cu deleted file mode 100644 index 19535ec5d11e..000000000000 --- a/cpp/tests/transform/fragments/decimal_square.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -template <> -__device__ void cudf::lto::unary_operator( - numeric::decimal32* __restrict__ out, numeric::decimal32 a) -{ - *out = a * a; -} - -template <> -__device__ void cudf::lto::unary_operator( - numeric::decimal64* __restrict__ out, numeric::decimal64 a) -{ - *out = a * a; -} - -template <> -__device__ void cudf::lto::unary_operator( - numeric::decimal128* __restrict__ out, numeric::decimal128 a) -{ - *out = a * a; -} diff --git a/cpp/tests/transform/fragments/invsqrt.cu b/cpp/tests/transform/fragments/invsqrt.cu deleted file mode 100644 index 3d7a6750b00e..000000000000 --- a/cpp/tests/transform/fragments/invsqrt.cu +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -template <> -__device__ void cudf::lto::unary_operator(float* __restrict__ out, float a) -{ - *out = 1.0F / sqrtf(a); -} diff --git a/cpp/tests/transform/fragments/sum_of_squares.cu b/cpp/tests/transform/fragments/sum_of_squares.cu deleted file mode 100644 index 6c1d233939f6..000000000000 --- a/cpp/tests/transform/fragments/sum_of_squares.cu +++ /dev/null @@ -1,14 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -template <> -__device__ void cudf::lto::binary_operator(float* __restrict__ out, - float a, - float b) -{ - *out = a * a + b * b; -} diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index d7c476e35b97..9c342a8f06c7 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -671,7 +671,7 @@ TEST_F(StringOperationTest, Output) std::string cuda = R"***( __device__ void transform(cudf::string_view * out, cudf::string_view a, cudf::string_view b, cudf::string_view c, cudf::string_view d){ - *out = cuda::std::max(cuda::std::max(cuda::std::max(a, b), c), d); + *out = std::max(std::max(std::max(a, b), c), d); } )***"; diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp deleted file mode 100644 index 4dec31865b5a..000000000000 --- a/cpp/tests/transform/transform_lto_test.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -struct TransformLTOTest : public cudf::test::BaseFixture {}; - -template -using column_wrapper = cudf::test::fixed_width_column_wrapper; - -template -using decimal_wrapper = cudf::test::fixed_point_column_wrapper; - -TEST_F(TransformLTOTest, InvSqrt) -{ - column_wrapper input{{1.0f, 4.0f, 9.0f, 16.0f}}; - - cudf::transform_output output{cudf::data_type{cudf::type_id::FLOAT32}, - cudf::output_nullability::ALL_VALID}; - - auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::invsqrt]; - std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; - - auto result = cudf::unary_op_lto(input, - output, - udf, - cudf::lto_binary_type::FATBIN, - cudf::null_aware::NO, - cudf::test::get_default_stream()); - - column_wrapper expected{{1.0f, 0.5f, 0.33333334f, 0.25f}}; - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); -} - -TEST_F(TransformLTOTest, SumOfSquares) -{ - column_wrapper lhs{{1.0f, 4.0f, 9.0f, 16.0f}}; - column_wrapper rhs{{1.0f, 2.0f, 2.0f, 10.0f}}; - - cudf::transform_output output{cudf::data_type{cudf::type_id::FLOAT32}, - cudf::output_nullability::ALL_VALID}; - - auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::sum_of_squares]; - std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; - - auto result = cudf::binary_op_lto(lhs, - rhs, - output, - udf, - cudf::lto_binary_type::FATBIN, - cudf::null_aware::NO, - cudf::test::get_default_stream()); - - column_wrapper expected{{2.0f, 20.0f, 85.0f, 356.0f}}; - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); -} - -TEST_F(TransformLTOTest, Decimal32Square) -{ - auto test_type = []() { - decimal_wrapper input{{1, 2, 3}, numeric::scale_type{-2}}; - - cudf::transform_output output{cudf::data_type{cudf::type_to_id(), numeric::scale_type{-4}}, - cudf::output_nullability::ALL_VALID}; - - auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::decimal_square]; - std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; - - auto result = cudf::unary_op_lto(input, - output, - udf, - cudf::lto_binary_type::FATBIN, - cudf::null_aware::NO, - cudf::test::get_default_stream()); - - decimal_wrapper expected{{1, 4, 9}, numeric::scale_type{-4}}; - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); - }; - - test_type.operator()(); - test_type.operator()(); - test_type.operator()(); -} From 3f5fc79afe369b325c9b211e866ee0dd320cde33 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 13 May 2026 20:02:19 +0000 Subject: [PATCH 175/254] revert LTO doc --- cpp/doxygen/developer_guide/JIT_LTO.md | 989 ------------------------- 1 file changed, 989 deletions(-) delete mode 100644 cpp/doxygen/developer_guide/JIT_LTO.md diff --git a/cpp/doxygen/developer_guide/JIT_LTO.md b/cpp/doxygen/developer_guide/JIT_LTO.md deleted file mode 100644 index a495c4da2bb4..000000000000 --- a/cpp/doxygen/developer_guide/JIT_LTO.md +++ /dev/null @@ -1,989 +0,0 @@ -# JIT LTO (Just-In-Time Link-Time Optimization) Guide - -## Background - -### What is JIT LTO? - -[JIT LTO (Just-In-Time Link-Time Optimization)](https://developer.nvidia.com/blog/cuda-12-0-compiler-support-for-runtime-lto-using-nvjitlink-library/) is a CUDA compilation strategy that enables dynamic kernel compilation and linking at runtime. Instead of pre-compiling all possible kernel variants (which would result in an explosion of binary size), JIT LTO compiles kernel **fragments** separately and links them together on-demand when a specific kernel configuration is needed. - -### Fragment Terminology - -A **fragment** is a self-contained, compilable unit of CUDA code that can be linked with other fragments to form a complete kernel. In the JIT LTO system: - -- **Entrypoint Fragment**: The main kernel function that serves as the entry point. This is always the `__global__` kernel function. -- **Device Function Fragments**: Separate fragments containing device functions (e.g., distance computations, filters, post-processing) that are called by the entrypoint kernel. -- **Fragment Key**: A unique identifier for a fragment, typically constructed from template parameters and configuration values. -- **Fatbin**: The compiled binary representation of a fragment, embedded in the executable. - -The key advantage is that device functions can be compiled independently and reused across multiple kernel entrypoints, reducing compilation time and binary size. - -### How It Works - -1. **Build Time**: Fragments are compiled into fatbins and embedded in the executable. -2. **Runtime**: When a kernel needs to be launched: - - The planner identifies which fragments are needed based on the configuration - - Fragments are loaded from the embedded fatbins - - Nvjitlink (Link-Time Optimization) links the fragments together - - The linked kernel is cached and launched - -## Walkthrough Example - -Let's walk through creating a JIT LTO kernel system for a search kernel with templated device functions. - -### Step 1: Define the Kernel and Device Functions - -We start with a kernel that has templated device functions that we want to separate into fragments: - -**`search_kernel.cuh`**: - -```cpp -#pragma once - -#include - -namespace example::detail { - -// Device function for distance computation -template -__device__ float compute_distance_euclidean(T a, T b) { - T diff = a - b; - return diff * diff; -} - -template -__device__ float compute_distance_inner_product(T a, T b) { - return -a * b; // Negative for max inner product search -} - -// Device function for filtering -template -__device__ bool apply_filter_none(uint32_t query_id, IdxT node_id, void* filter_data) { - return true; -} - -template -__device__ bool apply_filter_bitset(uint32_t query_id, IdxT node_id, void* filter_data) { - // Simplified - actual implementation would check bitset - return true; -} - -// Main kernel - will use generic extern device functions -template -__device__ void search_kernel_impl( - const T* dataset, - const T* queries, - IdxT* results, - OutT* distances, // Output distance type - uint32_t num_queries, - uint32_t dataset_size, - void* filter_data) { - - uint32_t query_id = blockIdx.x * blockDim.x + threadIdx.x; - if (query_id >= num_queries) return; - - OutT best_dist = std::numeric_limits::max(); - IdxT best_idx = 0; - - for (IdxT i = 0; i < dataset_size; ++i) { - // Call generic extern device functions (implementations linked from fragments) - if (!apply_filter(query_id, i, filter_data)) continue; - - OutT dist = static_cast(compute_distance(queries[query_id], dataset[i])); - - // Use optimized path if enabled - if constexpr (UseOptimizedPath) { - // Optimized implementation - if (dist < best_dist) { - best_dist = dist; - best_idx = i; - } - } else { - // Standard implementation - if (dist < best_dist) { - best_dist = dist; - best_idx = i; - } - } - } - - results[query_id] = best_idx; - distances[query_id] = best_dist; -} - -} // namespace example::detail -``` - -### Step 2: Create Device Function Fragments - -We'll create separate header files for each device function variant. Each implements the generic function signature that the kernel expects: - -**`compute_distance_euclidean.cuh`**: - -```cpp -#pragma once - -namespace example::detail { - -// Implements the generic compute_distance function for euclidean distance -template -__device__ float compute_distance(T a, T b) { - T diff = a - b; - return diff * diff; -} - -} // namespace example::detail -``` - -**`compute_distance_inner_product.cuh`**: - -```cpp -#pragma once - -namespace example::detail { - -// Implements the generic compute_distance function for inner product -template -__device__ float compute_distance(T a, T b) { - return -a * b; // Negative for max inner product search -} - -} // namespace example::detail -``` - -**`filter_none.cuh`**: - -```cpp -#pragma once - -namespace example::detail { - -// Implements the generic apply_filter function for no filtering -template -__device__ bool apply_filter(uint32_t query_id, IdxT node_id, void* filter_data) { - return true; -} - -} // namespace example::detail -``` - -**`filter_bitset.cuh`**: - -```cpp -#pragma once - -namespace example::detail { - -// Implements the generic apply_filter function for bitset filtering -template -__device__ bool apply_filter(uint32_t query_id, IdxT node_id, void* filter_data) { - // Actual bitset implementation - return true; -} - -} // namespace example::detail -``` - -### Step 3: Create JSON Matrix Files - -JSON matrix files define all the parameter combinations that need to be compiled. The build system uses these to generate `.cu` files from `.cu.in` templates. - -**How JSON Cross-Product Works**: -- The build system computes a modified **Cartesian product** (cross-product) of all parameter combinations. -- **Leaf nodes** are the actual values. These can be strings, numbers, booleans, or `null`, but only strings should be used, even for numbers, for example ``"1"``. -- Related values can be grouped together in a dictionary consisting of single values. Any dictionary key in such a dictionary's ancestry will not be used in the final product, and should be prefixed with `_` to indicate that it is used only for grouping. -- Keys containing only leaf nodes will be used in the final product, and should not be prefixed with `_`. -- The matrix product algorithm will automatically warn if the proper naming convention (`_` prefix or not) is not followed. -- Each group expands to create multiple combinations, and all groups are cross-multiplied. - -For example, if you have: -```json -{ - "_data_type": [{"data_type": "float"}, {"data_type": "half"}], - "_index": [{"idx_type": "uint32_t"}, {"idx_type": "int64_t"}], - "capacity": ["1", "2"] -} -``` - -This generates 2 × 2 × 2 = 8 combinations: -- `{data_type: "float", idx_type: "uint32_t", capacity: "1"}` -- `{data_type: "float", idx_type: "uint32_t", capacity: "2"}` -- `{data_type: "float", idx_type: "int64_t", capacity: "1"}` -- ... and so on - -When a group contains nested arrays (like `veclen: ["1", "4"]`), those are also expanded within that group before the cross-product is computed. - -#### `compute_distance_matrix.json` - -```json -{ - "_distance_type": [ - { - "distance_name": "euclidean", - "header_file": "example/jit_lto_kernels/compute_distance_euclidean.cuh" - }, - { - "distance_name": "inner_product", - "header_file": "example/jit_lto_kernels/compute_distance_inner_product.cuh" - } - ], - "_data_type": [ - { - "data_type": "float", - "type_abbrev": "f" - }, - { - "data_type": "__half", - "type_abbrev": "h" - } - ] -} -``` - -#### `filter_matrix.json` - -```json -{ - "filter_name": [ - "filter_none", - "filter_bitset" - ], - "_index": [ - { - "idx_type": "uint32_t", - "idx_abbrev": "ui" - }, - { - "idx_type": "int64_t", - "idx_abbrev": "l" - } - ] -} -``` - -#### `search_kernel_matrix.json` - -This example demonstrates conditional combinations: `OutT` can be `float` or `double` when `T` is `float`, but only `float` when `T` is `__half`. - -```json -{ - "_data_type": [ - { - "data_type": "float", - "type_abbrev": "f", - "_output_type": [ - { - "out_type": "float", - "out_abbrev": "f" - }, - { - "out_type": "double", - "out_abbrev": "d" - } - ] - }, - { - "data_type": "__half", - "type_abbrev": "h", - "_output_type": [ - { - "out_type": "float", - "out_abbrev": "f" - } - ] - } - ], - "_index": [ - { - "idx_type": "uint32_t", - "idx_abbrev": "ui" - }, - { - "idx_type": "int64_t", - "idx_abbrev": "l" - } - ], - "_optimized": [ - { - "optimized_name": "optimized", - "optimized_value": "true", - "veclen": ["1", "4"] - }, - { - "optimized_name": "standard", - "optimized_value": "false", - "veclen": ["8", "16"] - } - ] -} -``` - -This generates 24 combinations (3 data/output type combinations × 2 index types × 4 optimized/veclen combinations): -- `float` + `float` + `uint32_t` + `optimized` + `veclen=1` -- `float` + `float` + `uint32_t` + `optimized` + `veclen=4` -- `float` + `float` + `uint32_t` + `standard` + `veclen=8` -- `float` + `float` + `uint32_t` + `standard` + `veclen=16` -- `float` + `double` + `uint32_t` + `optimized` + `veclen=1` -- `float` + `double` + `uint32_t` + `optimized` + `veclen=4` -- `float` + `double` + `uint32_t` + `standard` + `veclen=8` -- `float` + `double` + `uint32_t` + `standard` + `veclen=16` -- `__half` + `float` + `uint32_t` + `optimized` + `veclen=1` -- `__half` + `float` + `uint32_t` + `optimized` + `veclen=4` -- `__half` + `float` + `uint32_t` + `standard` + `veclen=8` -- `__half` + `float` + `uint32_t` + `standard` + `veclen=16` -- ... and the same with `int64_t` (total: 24 combinations) - -### Step 4: Create `.cu.in` Template Files - -The `.cu.in` files are templates that get instantiated for each combination in the JSON matrix. They contain explicit template instantiations. - -#### `compute_distance_kernel.cu.in` - -```text -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "@header_file@" - -namespace example::detail { - -// Instantiate the generic compute_distance device function template -// The specific implementation (euclidean or inner_product) comes from the header -template __device__ float compute_distance<@data_type@>(@data_type@, @data_type@); - -} // namespace example::detail -``` - -#### `filter_kernel.cu.in` - -```text -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "example/jit_lto_kernels/@filter_name@.cuh" - -namespace example::detail { - -// Instantiate the generic apply_filter device function template -// The specific implementation (filter_none or filter_bitset) comes from the header -template __device__ bool apply_filter<@idx_type@>(uint32_t, @idx_type@, void*); - -} // namespace example::detail -``` - -#### Update `search_kernel.cuh` with Extern Declarations - -The kernel header needs to declare generic extern device functions so the kernel code can call them. The specific implementations will be linked from fragments at runtime: - -**`search_kernel.cuh`**: - -```cpp -#pragma once - -#include - -namespace example::detail { - -// Forward declare generic extern device functions that will be linked from fragments -// The specific implementations (euclidean, inner_product, etc.) are resolved at link time -template -extern __device__ float compute_distance(T, T); - -template -extern __device__ bool apply_filter(uint32_t, IdxT, void*); - -// Main kernel - uses generic extern device functions -template -__device__ void search_kernel_impl( - const T* dataset, - const T* queries, - IdxT* results, - OutT* distances, // Output distance type - uint32_t num_queries, - uint32_t dataset_size, - void* filter_data) { - - uint32_t query_id = blockIdx.x * blockDim.x + threadIdx.x; - if (query_id >= num_queries) return; - - OutT best_dist = std::numeric_limits::max(); - IdxT best_idx = 0; - - for (IdxT i = 0; i < dataset_size; ++i) { - // Call generic extern device functions (specific implementations linked from fragments) - if (!apply_filter(query_id, i, filter_data)) continue; - - OutT dist = static_cast(compute_distance(queries[query_id], dataset[i])); - - // Use optimized path if enabled - if constexpr (UseOptimizedPath) { - // Optimized implementation - if (dist < best_dist) { - best_dist = dist; - best_idx = i; - } - } else { - // Standard implementation - if (dist < best_dist) { - best_dist = dist; - best_idx = i; - } - } - } - - results[query_id] = best_idx; - distances[query_id] = best_dist; -} - -} // namespace example::detail -``` - -#### `search_kernel.cu.in` - -The `.cu.in` file only contains the explicit template instantiation: - -```text -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "example/jit_lto_kernels/search_kernel.cuh" - -namespace example::detail { - -// Instantiate the kernel template -extern "C" __global__ void search_kernel( - const @data_type@* dataset, const @data_type@* queries, @idx_type@* results, @out_type@* distances, - uint32_t num_queries, uint32_t dataset_size, void* filter_data) -{ - search_kernel_impl<@data_type@, @out_type@, @idx_type@, @optimized_value@, @veclen@>( - dataset, - queries, - results, - distances, - num_queries, - dataset_size, - filter_data); -} - -} // namespace example::detail -``` - -**Note**: The kernel uses generic function templates (`compute_distance` and `apply_filter`) that are resolved at link time. The specific implementations (euclidean vs inner_product, filter_none vs filter_bitset) are provided by the fragments that get linked together. - -### Step 5: Create Fragment Tags for Embedding - -Fragment tags register the compiled fatbins so they can be loaded at runtime. They are used to help the linker find and include the relevant fatbins at build time. When calling `generate_jit_lto_kernels()`, we pass a `FRAGMENT_TAG_FORMAT` argument, which constructs the tag type from the given placeholders, and a `FRAGMENT_TAG_HEADER_FILES` argument, which specifies one or more header files that the fragment tags come from. The JIT+LTO system will then automatically generate and compile a `.cpp` file that registers the fragment using the provided tag. - -**Important**: When requesting fragments from the `AlgorithmPlanner`, we use **tags** (like `tag_f`, `tag_h`) instead of real types (like `float`, `__half`) in the `add_static_fragment` template parameters. This avoids including heavy headers that define the actual types, significantly improving compilation times. The tags are lightweight empty structs that serve only as compile-time identifiers. - -**`registration_tags.hpp`** - -```cpp -#pragma once - -struct tag_h{}; -struct tag_f{}; -struct tag_d{}; -struct tag_ui{}; -struct tag_l{}; - -struct tag_metric_euclidean {}; -struct tag_metric_inner_product {}; - -struct tag_filter_none {}; -struct tag_filter_bitset {}; - -template -struct fragment_tag_search {}; - -template -struct fragment_tag_compute_distance {}; - -template -struct fragment_tag_filter {}; -``` - -### Step 6: Create the Planner - -The planner is responsible for: -1. Identifying which fragments are needed for a given configuration -2. Building a unique key for the fragment combination -3. Requesting the fragments from the fragment database -4. Linking them together to create a launchable kernel - -**CRITICAL**: The fragment keys constructed in the planner methods must match **EXACTLY** with the keys used in the corresponding `FRAGMENT_TAG_FORMAT` argument. Any mismatch will result in runtime linking failures. - -**`search_planner.hpp`**: - -```cpp -#pragma once - -#include -#include -#include -#include - -struct SearchPlanner : AlgorithmPlanner { - inline static LauncherJitCache launcher_jit_cache{}; - - SearchPlanner() - : AlgorithmPlanner("search_kernel", launcher_jit_cache) - { - } - - template - void add_search_function() - { - add_static_fragment>(); - } - - template - void add_compute_distance_function() - { - add_static_fragment>(); - } - - template - void add_filter_function() - { - add_static_fragment>(); - } -}; -``` - -### Step 7: Integrate with Code Path - -Now we integrate the planner into the actual search function: - -**`search_jit.cuh`**: - -```cpp -#pragma once - -#include "search_planner.hpp" -#include -#include - -namespace example::detail { - -// Type tag helpers -template -constexpr auto get_data_type_tag() { - if constexpr (std::is_same_v) return tag_f{}; - if constexpr (std::is_same_v) return tag_h{}; -} - -template -constexpr auto get_idx_type_tag() { - if constexpr (std::is_same_v) return tag_ui{}; - if constexpr (std::is_same_v) return tag_l{}; -} - -template -constexpr auto get_out_type_tag() { - if constexpr (std::is_same_v) return tag_f{}; - if constexpr (std::is_same_v) return tag_d{}; -} - -template -constexpr auto get_metric_tag() { - if constexpr (Metric == DistanceType::Euclidean) return tag_metric_euclidean{}; - if constexpr (Metric == DistanceType::InnerProduct) return tag_metric_inner_product{}; -} - -template -constexpr auto get_filter_tag() { - if constexpr (Filter == FilterType::None) return tag_filter_none{}; - if constexpr (Filter == FilterType::Bitset) return tag_filter_bitset{}; -} - -template -void search_jit( - raft::device_resources const& handle, - const T* dataset, - const T* queries, - IdxT* results, - OutT* distances, - uint32_t num_queries, - uint32_t dataset_size, - void* filter_data = nullptr) { - - using data_tag = decltype(get_data_type_tag()); - using idx_tag = decltype(get_idx_type_tag()); - using out_tag = decltype(get_out_type_tag()); - using metric_tag = decltype(get_metric_tag()); - using filter_tag = decltype(get_filter_tag()); - - // Create planner with type tags and boolean parameter - // Note: The boolean is appended to the fragment key since make_fragment_key - // cannot handle non-type template parameters - SearchPlanner planner; - - // Add required device function fragments - planner.add_search_function(); - planner.add_compute_distance_device_function(); - planner.add_filter_device_function(); - - // Get the launcher (this will build/link fragments if needed) - auto launcher = planner.get_launcher(); - - // Launch configuration - dim3 block(256); - dim3 grid((num_queries + block.x - 1) / block.x); - - // Launch the kernel - arguments are passed directly - launcher->dispatch( - raft::resource::get_cuda_stream(handle), - grid, - block, - 0, // shared memory size - dataset, - queries, - results, - distances, - num_queries, - dataset_size, - filter_data); -} - -} // namespace example::detail -``` - -## Key Concepts - -### Fragment Tags - -Fragment tags uniquely identify fragments. They're simple lightweight types that are passed as the -sole template parameter to `StaticFatbinFragmentEntry`: - -```cpp -template -struct fragment_tag_get_score {}; -``` - -Fragment tags may themselves take template parameters in order to uniquely identify them. Typically, one fragment tag template will correspond to a single function, and a fragment tag template specialization will correspond to a function specialization. - -When a fatbin is compiled and embedded in C++ code, a translation unit specializes `StaticFatbinFragmentEntry` -to specify its `data` and `length` static fields: - -```cpp -using _FragmentEntry = StaticFatbinFragmentEntry>; - -template <> -const uint8_t* const _FragmentEntry::data = embedded_fatbin; - -template <> -const size_t _FragmentEntry::length = sizeof(embedded_fatbin); -``` - -Then, an `AlgorithmPlanner` can call `add_static_fragment()` with the fragment tag (NOT the `StaticFatbinFragmentEntry` -specialization) as the sole template parameter: - -```cpp -template -void add_get_score_function() -{ - add_static_fragment>(); -} -``` - -At build time, the linker takes care of finding and including the static fragments that have been specified by the -algorithm planner. - -### Registration Tags - -Registration tags are type-safe identifiers used to organize fragments. They're typically empty structs: - -```cpp -struct tag_f {}; // float -struct tag_h {}; // half -struct tag_ui {}; // uint32_t -struct tag_l {}; // int64_t -``` - -These tags are used in `registerAlgorithm<>()` to create a hierarchical organization of fragments. - -### AlgorithmLauncher - -The `AlgorithmLauncher` is the runtime handle for a linked kernel. It: -- Holds a `cudaKernel_t` handle to the linked kernel -- Provides `call()` and `call_cooperative()` methods to launch the kernel -- Manages the lifetime of the `cudaLibrary_t` that contains the kernel - -## Best Practices - -1. **Minimize Includes**: JIT LTO fragments should have minimal includes, especially avoiding host-side headers. Extract device-only code into separate headers. - -2. **Fragment Granularity**: Balance between too many small fragments (overhead) and too few large fragments (less reuse). Device functions that are reused across multiple kernels are good candidates for separate fragments. - -3. **Naming Consistency**: Ensure fragment tags match exactly between registration and lookup. Use helper functions to construct tags consistently. - -4. **Type Safety**: Use registration tags to provide compile-time type safety and avoid runtime string mismatches. - -5. **Caching**: Each planner type should hold a static `LauncherJitCache` and pass it to `AlgorithmPlanner`; `get_launcher()` then reuses linked kernels for the same fragment key within that cache. - -## Example: IVF Flat - -IVF Flat uses JIT LTO with: -- **Metric fragments**: Euclidean and inner product distance computations (16 fatbins) -- **Post-lambda fragments**: Identity, sqrt, and compose post-processing (3 fatbins) -- **Interleaved scan fragments**: Main search kernel with various configurations (320 fatbins) -- **Filter fragments**: None and bitset filters (2 fatbins) - -**Total: 341 fatbins** that can be combined into many more kernel variants at runtime. - -### Step 8: Integrate with CMake Build System - -To integrate JIT LTO kernels into the CMake build system, add calls to `generate_jit_lto_kernels()` in your main `CMakeLists.txt` file (typically in `cpp/CMakeLists.txt`). - -The `generate_jit_lto_kernels()` function (defined in `cmake/modules/generate_jit_lto_kernels.cmake`) takes: -- `NAME_FORMAT`: Format string for generated kernel names (using `@variable@` syntax) -- `MATRIX_JSON_FILE`: Path to the JSON matrix file -- `KERNEL_INPUT_FILE`: Path to the `.cu.in` template -- `FRAGMENT_TAG_FORMAT`: Format string for fragment tag type (using `@variable@` syntax) -- `FRAGMENT_TAG_HEADER_FILES`: List of header files that provide the fragment tag types (can be enclosed in `<`/`>` or `"`/`"`, automatically enclosed in quotes if quotes and brackets are not provided) -- `OUTPUT_DIRECTORY`: Where generated files are placed -- `KERNEL_LINK_LIBRARIES`: Interface library with compilation settings - -Call `generate_jit_lto_kernels()` once for each fragment type (compute_distance, filter, search_kernel, etc.). The function reads the JSON matrix, computes the cross-product of all combinations, generates `.cu` and `.cpp` files from the templates, compiles them into fatbins, and returns a list of generated source files that should be added to your JIT LTO library target. - -See the CUVS `cpp/CMakeLists.txt` file for a complete example of how to set up the interface library, call `generate_jit_lto_kernels()` for each fragment type, and create the final library target. - -## Summary - -JIT LTO enables: -- **Reduced binary size**: Compile fragments once, combine many ways -- **Faster compilation**: Fragments compile independently -- **Runtime flexibility**: Link fragments on-demand based on configuration -- **Code reuse**: Device function fragments shared across kernels - -The process involves: -1. Separating device functions into fragment headers -2. Creating JSON matrices defining parameter combinations -3. Creating `.cu.in` templates for explicit instantiations -4. Creating fragment tag types for fatbin registration -5. Creating a planner to manage fragment dependencies -6. Integrating the planner into the code path to launch kernels -7. **Adding CMake integration** to generate and compile all fragment variants - -## Fragment Architecture - -JIT LTO kernels are split into _fragments_, which are fatbins containing individual pieces of code that can be strung together -rather than instantiating the whole kernel at once. Each fragment only needs to be multiplied out over the dimensions (template -parameters) that the fragment itself contains rather than the kernel as a whole. At runtime, these fragments are combined together -by nvjitlink into the final program. - -In JIT LTO, there are two kinds of code: _algorithms_ and _adapters_. Algorithms are, roughly speaking, code that actually "does -stuff" - searching, sorting, even as simple as initializing variables. Adapters don't do anything by themselves, but are merely -thin wrappers around algorithms that exist only for reducing the number of template parameters that the caller needs to know about. -It should generally be assumed that algorithm code is expensive to multiply over a matrix, and thus such multiplication should be -minimized, while adapter code is cheap to multiply. - -An algorithm function is a function that contains real code for the algorithm, and an adapter function merely calls an algorithm -function with more template parameters than the adapter function itself has. An algorithm file contains algorithm code, and an -adapter file contains adapter code. - -Here is an example of an algorithm file that contains an algorithm function: - -```c++ -template -__device__ bool is_divisible_impl(T value) -{ - return value % Divisor == 0; -} -``` - -Here is an example of an adapter file that contains an adapter function: - -``` -#include "device_functions.cuh" // is_divisible -#include "is_divisible_impl.cuh" // is_divisible_impl - -namespace { - -using data_t = @data_type@; -constexpr data_t divisor = @divisor@; - -} // namespace - -template <> -__device__ bool is_divisible(data_t value) -{ - return is_divisible_impl(value); -} -``` - -This is the most common pattern that you will see in cuVS's JIT LTO code. Note that any code that calls `is_divisible()` does not -need to know the value of `Divisor`, which allows the caller to be multiplied over fewer dimensions, thus reducing the amount of code -generated. - -Note that in the above adapter file, `@data_type@` and `@divisor@` are build-time substitutions performed by CMake. These -substitutions will be filled in with values from the matrix product. Note that they are all grouped together in a single `namespace`, -making it easy to find all substitutions. This should be preferred to sprinkling the substitutions throughout the code. - -Here is an example with two algorithm files: - -```c++ -// greater_than_impl.cuh -#include "device_impl_functions.cuh" // filter - -template -__device__ bool filter(T value) -{ - return value > Comparand; -} -``` - -```c++ -// less_than_impl.cuh -#include "device_impl_functions.cuh" // filter - -template -__device__ bool filter(T value) -{ - return value < Comparand; -} -``` - -And here is the accompanying adapter file: - -``` -#include "@op_name@_impl.cuh" // filter - -namespace { - -using data_t = @data_type@; - -} - -template __device__ bool filter(data_t value); -``` - -This is another common pattern that you will see in cuVS JIT LTO. Note that the adapter file does not contain any adapter functions, -but merely instantiates a different algorithm function based on which algorithm file is included based on the CMake substitution. - -When a piece of algorithm code is used in multiple kernels, it should be split into its own shared fragment. At this point, it -becomes important to also distinguish algorithm fragments and adapter fragments. An algorithm fragment contains an algorithm function -that exposes all of the relevant template parameters, and this fragment is shared between multiple kernels. An adapter fragment -is specific to a kernel. If a kernel wishes to invoke the same shared algorithm multiple times in the same run with -different template parameters, it can employ multiple adapter fragments to accomplish this. Consider the following header file: - -```c++ -// filter.cuh - -template -__device__ bool filter_less_than(T value); - -template -__device__ bool filter_greater_than(T value); -``` - -And the following adapter files: - -``` -#include "device_functions.cuh" // filter_first_pass -#include "filter.cuh" // filter - -namespace { - -using data_t = @data_type@; -constexpr data_t comparand = @comparand@; - -} - -template <> -__device__ bool filter_first_pass(data_t value) -{ - return filter_@op_name@(value); -} -``` - -``` -#include "device_functions.cuh" // filter_second_pass -#include "filter.cuh" // filter - -namespace { - -using data_t = @data_type@; -constexpr data_t comparand = @comparand@; -} - -template <> -__device__ bool filter_second_pass(data_t value) -{ - return filter_@op_name@(value); -} -``` - -And the following algorithm file: - -```c++ -#include "device_functions.cuh" // filter_first_pass, filter_second_pass - -template -__device__ bool filter_all_passes(T value) -{ - return filter_first_pass(value) && filter_second_pass(value); -} -``` - -Note that `filter_first_pass` and `filter_second_pass` both invoke one of the `filter` functions, but which one they invoke is -decided independently for each. Also note that neither of the adapter fragments contains the underlying algorithm code, but rather -links against the corresponding shared algorithm fragments. - -The key to minimizing code generation is to minimize the number of dimensions that any given fragment needs to be multiplied out -over. If a section of algorithm code uses lots of template parameters, try to separate out sections that use only a subset of -these parameters, put them into their own fragment, and remove the corresponding template parameters from the caller. Make judicious -use of adapter code to accomplish this. An adapter function should only have the template parameters that appear in its signature, -whereas an algorithm function should have all of the template parameters that appear in its signature or its implementation. - -Unoptimized algorithm: - -```c++ -#include "filter_less_than.cuh" - -template -__device__ size_t find_first(T* values, size_t count) -{ - for (size_t i = 0; i < count; i++) { - if (filter_less_than_impl(values[i])) { - return i; - } - } - - // Could not find any - return count; -} -``` - -Note that the algorithm includes the `Comparand` template parameter, which means the entire algorithm has to be multiplied out over -all the possible values of this parameter. - -Optimized algorithm: - -```c++ -#include "device_functions.cuh" - -template -__device__ size_t find_first(T* values, size_t count) -{ - for (size_t i = 0; i < count; i++) { - if (filter_less_than(values[i])) { - return i; - } - } - - // Could not find any - return count; -} -``` - -We are now using an adapter function (possibly inside an adapter fragment) called `filter_less_than` to invoke -`filter_less_than_impl` (which may be inside a shared algorithm fragment). This allows us to hide the `Comparand` parameter -from `find_first`, which means we no longer need to multiply the entire algorithm over all possible values of `Comparand`, only the -`filter_less_than` adapter and algorithm. \ No newline at end of file From 8aad23658ccf4b89ac411befc8848a2839e529fc Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 13 May 2026 20:03:19 +0000 Subject: [PATCH 176/254] revert docker change --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 0886ae63fb75..57ccf6302c5b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -34,7 +34,7 @@ ENV PYTHONUNBUFFERED="1" ENV PYTHONDONTWRITEBYTECODE="1" ENV HISTFILE="/home/coder/.cache/._bash_history" -ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/libcudf" +ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache" ### # sccache configuration From eb3ef7916059f11eee9c75b5140d3b265457a264 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 13 May 2026 21:11:45 +0000 Subject: [PATCH 177/254] fix coderabbit nits --- cpp/librtcx/embed.hpp | 45 ++++++++++++++++++++++--------------------- cpp/librtcx/rtcx.cpp | 25 ++++++++++++++++++------ cpp/librtcx/rtcx.hpp | 3 --- 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index a29b8eaca805..3a027faaa73f 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -27,6 +27,13 @@ #include #include +#define RTCX_EMBED_EXPECTS(condition, message) \ + do { \ + if (!(condition)) { \ + throw std::runtime_error(std::format("{}:{}: {}", __FILE__, __LINE__, message)); \ + } \ + } while (false) + namespace rtcx_embed { struct size_range { @@ -61,28 +68,23 @@ std::vector load_file_bytes(std::string_view file_path) { std::string path_str(file_path); std::ifstream file(path_str, std::ios::binary | std::ios::ate); - if (!file) { - throw std::runtime_error(std::format("Failed to open file at path: {}", file_path)); - } + RTCX_EMBED_EXPECTS(file.is_open(), std::format("Failed to open file at path: {}", file_path)); auto file_size = file.tellg(); - if (file_size < 0) { - throw std::runtime_error( - std::format("Failed to determine size of file at path: {}", file_path)); - } + RTCX_EMBED_EXPECTS(file_size >= 0, + std::format("Failed to determine size of file at path: {}", file_path)); file.seekg(0, std::ios::beg); std::vector bytes(file_size); - if (!file.read(reinterpret_cast(bytes.data()), file_size)) { - throw std::runtime_error(std::format("Failed to read file at path: {}", file_path)); - } + RTCX_EMBED_EXPECTS(file.read(reinterpret_cast(bytes.data()), file_size), + std::format("Failed to read file at path: {}", file_path)); return bytes; } std::vector compress_bytes(std::span bytes, std::string_view compression) { - if (compression != "none" && compression != "zstd") { - throw std::invalid_argument(std::format( - "Invalid compression type: {}. Supported values are 'none' and 'zstd'", compression)); - } + RTCX_EMBED_EXPECTS( + compression == "none" || compression == "zstd", + std::format("Invalid compression type: {}. Supported values are 'none' and 'zstd'", + compression)); if (compression == "none") { return std::vector(bytes.begin(), bytes.end()); } @@ -91,10 +93,9 @@ std::vector compress_bytes(std::span bytes, std::string_ auto const compressed_size = ZSTD_compress(compressed.data(), compressed.size(), bytes.data(), bytes.size(), 22); - if (ZSTD_isError(compressed_size)) { - throw std::runtime_error(std::string("ZSTD compression failed: ") + - ZSTD_getErrorName(compressed_size)); - } + RTCX_EMBED_EXPECTS( + !ZSTD_isError(compressed_size), + std::format("Compression failed with error: {}", ZSTD_getErrorName(compressed_size))); compressed.resize(compressed_size); return compressed; @@ -128,6 +129,7 @@ std::string generate_arrays(std::span array_ids, std::span array_values) { auto get_type = [](std::string_view value) -> value_type { + RTCX_EMBED_EXPECTS(!value.empty(), "Value cannot be empty"); return std::isdigit(value[0]) ? value_type::INT : value_type::STRING; }; @@ -164,10 +166,9 @@ std::string generate_arrays(std::span array_ids, switch (array.type) { case value_type::INT: { std::int64_t int_value; - if (std::from_chars(value.data(), value.data() + value.size(), int_value).ec != - std::errc()) { - throw std::invalid_argument(std::format("Invalid integer constant value: {}", value)); - } + RTCX_EMBED_EXPECTS( + std::from_chars(value.data(), value.data() + value.size(), int_value).ec == std::errc(), + std::format("Invalid integer constant value: {}", value)); std::get>(array.values).push_back(int_value); } break; diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index d7f5d3386c90..0772bfb20231 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -1082,8 +1082,14 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil // have already reserved a spot in the cache for this sha lock.unlock(); - auto result = compile(); - promise.set_value(result); + std::shared_ptr result = nullptr; + + try { + result = compile(); + promise.set_value(result); + } catch (...) { + promise.set_exception(std::current_exception()); + } cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::BLOB, sha, result->view()); @@ -1131,7 +1137,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, // we can release the lock while calling the maker function since it may be expensive and we // have already reserved a spot in the cache for this sha - lock_.unlock(); + lock.unlock(); promise.set_value(std::move(*disk_library)); @@ -1146,8 +1152,15 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, // have already reserved a spot in the cache for this sha lock.unlock(); - auto [library, blob] = compile(); - promise.set_value(library); + std::shared_ptr library = nullptr; + std::shared_ptr blob = nullptr; + + try { + std::tie(library, blob) = compile(); + promise.set_value(library); + } catch (...) { + promise.set_exception(std::current_exception()); + } // store result to disk cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::LIBRARY, sha, blob->view()); @@ -1308,7 +1321,7 @@ rtcx::byte_buffer decompress_blob(std::span compressed_binar RTCX_EXPECTS( !::ZSTD_isError(errc) && errc == uncompressed_size, - std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : ", + std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : {}", errc, ::ZSTD_getErrorName(errc)), std::runtime_error); diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index dce681d72304..a293aeb920f6 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -621,9 +621,6 @@ struct cache_t { // NOLINT * initialization, allowing for faster retrieval of previously compiled kernels at runtime * @param disable A boolean flag indicating whether to disable the cache entirely, preventing any * caching of compiled blobs and libraries in memory - * @param materialize_all A boolean flag indicating whether to make the compiled kernels fully - * materialized in memory during preloading to improve runtime stability at the cost of increased - * memory usage */ cache_t(std::string cache_dir, std::string tmp_dir, From 03ec682aa3065fb26b82c287289256ca4d59bb6a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 15 May 2026 00:55:50 +0100 Subject: [PATCH 178/254] revert: code rabbit suggestion --- cpp/librtcx/rtcx.cpp | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 0772bfb20231..88760e02c919 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -1082,14 +1082,8 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil // have already reserved a spot in the cache for this sha lock.unlock(); - std::shared_ptr result = nullptr; - - try { - result = compile(); - promise.set_value(result); - } catch (...) { - promise.set_exception(std::current_exception()); - } + auto result = compile(); + promise.set_value(result); cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::BLOB, sha, result->view()); @@ -1152,15 +1146,8 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, // have already reserved a spot in the cache for this sha lock.unlock(); - std::shared_ptr library = nullptr; - std::shared_ptr blob = nullptr; - - try { - std::tie(library, blob) = compile(); - promise.set_value(library); - } catch (...) { - promise.set_exception(std::current_exception()); - } + auto [library, blob] = compile(); + promise.set_value(library); // store result to disk cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::LIBRARY, sha, blob->view()); From 00b32b39d178482939bfe75e1baa1e84f5d1d383 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 15 May 2026 01:54:31 +0100 Subject: [PATCH 179/254] Update documentation and improve function signatures in librtcx - Corrected NVJitLink reference in README.md - Enhanced CMake functions to accept TARGET as an argument - Refactored embed functions to streamline embedding logic - Improved type handling in embed.hpp for better clarity - Renamed load_dll to load_dso for consistency - Added checks for zero-length files in blob_t::from_file - Enhanced cache_t documentation for clarity on directory requirements --- cpp/librtcx/README.md | 11 ++++--- cpp/librtcx/embed.cmake | 27 +++++++--------- cpp/librtcx/embed.hpp | 72 ++++++++++++++++------------------------- cpp/librtcx/rtcx.cpp | 26 +++++++++++---- cpp/librtcx/rtcx.hpp | 5 +-- 5 files changed, 68 insertions(+), 73 deletions(-) diff --git a/cpp/librtcx/README.md b/cpp/librtcx/README.md index 006872f15f31..8d0ab0499160 100644 --- a/cpp/librtcx/README.md +++ b/cpp/librtcx/README.md @@ -1,18 +1,19 @@ -# Doc +# librtcx -RTCX (runtime-compiler extended) is a wrapper around NVRTC and NVJitLink designed to provide: +RTCX (runtime-compiler extended) is a wrapper around NVRTC and nvJitLink designed to provide: - User-controlled compilation, linking, caching, and pre-loading of CUDA kernels - Zero-copy interfaces to manage JIT compilation and linking - CMake script to embed **compressed** headers directly into an executable without incurring overhead at runtime on every compilation request -- Facilities to pre-load and teardown dynamic library dependencies (`libcuda`, `libnvrtc`, and `libnvJitlink`) +- Facilities to pre-load and teardown dynamic library dependencies (`libcuda`, `libnvrtc`, and `libnvJitLink`) ## Platforms Supported - Linux ## Build-Scripting Requirements -- CMake +- CMake >= 4.0 - LibZSTD - for binary compression +- CUDA >= 12.2 ## Runtime Requirements -- CUDA >= 11.8 +- CUDA >= 12.2 diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index 706f2214185e..2b1ff3bff2b5 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -15,8 +15,7 @@ endif() # embed_includes() or embed_blob() for the target. It sets up necessary variables and state to track # the registered files and dependencies for the target. The TARGET argument specifies the name of # the target being initialized. -function(add_embed) - set(TARGET ${ARGV0}) +function(add_embed TARGET) set(OPTIONS "") set(ONE_VALUE_ARGS) set(MULTI_VALUE_ARGS) @@ -37,8 +36,7 @@ function(add_embed) endfunction() # This function registers a directory of include files to be embedded for JIT compilation. -function(embed_includes) - set(TARGET ${ARGV0}) +function(embed_includes TARGET) set(OPTIONS "") set(ONE_VALUE_ARGS COPY_DIRECTORY # Source directory where files will be copied from DEST_DIRECTORY # Destination directory where files will be copied to @@ -126,8 +124,7 @@ function(embed_includes) endfunction() # This function registers a single file to be embedded for JIT compilation. -function(embed_blob) - set(TARGET ${ARGV0}) +function(embed_blob TARGET) set(OPTIONS) set(ONE_VALUE_ARGS ID FILE DEST) set(MULTI_VALUE_ARGS ARRAY_IDS ARRAY_VALUES) @@ -214,8 +211,7 @@ endfunction() # This function generates the necessary files and build targets to embed the registered source files # for JIT compilation. -function(embed) - set(TARGET ${ARGV0}) +function(embed TARGET) set(OPTIONS "") set(ONE_VALUE_ARGS "COMPRESSION") set(MULTI_VALUE_ARGS "") @@ -259,18 +255,19 @@ function(embed) INPUT "${CONFIGURED_EMBED_SCRIPT}" ) - add_executable("${TARGET}__jit_embed_run" EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") - target_include_directories("${TARGET}__jit_embed_run" PRIVATE ${ZSTD_INCLUDE_DIR}) - target_link_libraries("${TARGET}__jit_embed_run" PRIVATE ${CMAKE_DL_LIBS} zstd) + set(RUNNER "${TARGET}__jit_embed_run") + add_executable(${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") + target_include_directories(${RUNNER} PRIVATE ${ZSTD_INCLUDE_DIR}) + target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} zstd) set_target_properties( - "${TARGET}__jit_embed_run" PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES + ${RUNNER} PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES ) - target_include_directories("${TARGET}__jit_embed_run" PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR}) + target_include_directories(${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR}) add_custom_command( OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin - COMMAND "${CMAKE_COMMAND}" -E env $ - DEPENDS "${CONFIGURED_EMBED_SCRIPT}" "${EMBED_SCRIPT}" ${${TARGET}__embed__source_files} + COMMAND "${CMAKE_COMMAND}" -E env $ + DEPENDS "${EMBED_SCRIPT}" ${${TARGET}__embed__source_files} ${${TARGET}__embed__target_deps} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" COMMENT "Generating JIT embed for ${TARGET} into ${OUTPUT_DIR}" diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index 3a027faaa73f..e7c15beeee31 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -47,8 +47,6 @@ struct embed_output { std::vector bin_file_data; }; -enum class value_type : int8_t { INT, STRING }; - std::pair, std::vector> merge_bytes_with_null_terminators( std::span const> bytes_lists) { @@ -125,6 +123,8 @@ std::string join_formatted(Container& items, std::string_view delimiter, Formatt return result.str(); } +enum class value_type : int8_t { INT, STRING }; + std::string generate_arrays(std::span array_ids, std::span array_values) { @@ -137,25 +137,19 @@ std::string generate_arrays(std::span array_ids, using ints_t = std::vector; using values_t = std::variant; - struct array_entry { - value_type type = value_type::INT; - values_t values; - }; - - std::map arrays; + std::map arrays; for (size_t i = 0; i < array_ids.size(); ++i) { auto id = array_ids[i]; auto value = array_values[i]; + auto type = get_type(value); if (auto array_it = arrays.find(id); array_it == arrays.end()) { - switch (get_type(value)) { - { - case value_type::INT: - arrays.emplace(id, array_entry(value_type::INT, std::vector{})); - } - break; + switch (type) { + case value_type::INT: { + arrays.emplace(id, ints_t{}); + } break; case value_type::STRING: { - arrays.emplace(id, array_entry(value_type::STRING, std::vector{})); + arrays.emplace(id, strings_t{}); } break; default: throw std::logic_error("Unexpected constant type"); } @@ -163,17 +157,16 @@ std::string generate_arrays(std::span array_ids, auto& array = arrays[id]; - switch (array.type) { + switch (type) { case value_type::INT: { std::int64_t int_value; RTCX_EMBED_EXPECTS( std::from_chars(value.data(), value.data() + value.size(), int_value).ec == std::errc(), std::format("Invalid integer constant value: {}", value)); - std::get>(array.values).push_back(int_value); + std::get(array).push_back(int_value); } break; - case value_type::STRING: { - std::get>(array.values).push_back(value); + std::get(array).push_back(value); } break; default: break; @@ -183,26 +176,19 @@ std::string generate_arrays(std::span array_ids, std::string result; for (auto& [id, array] : arrays) { - switch (array.type) { - case value_type::INT: { - auto& values = std::get>(array.values); - result += std::format( - "constexpr std::int64_t {}[{}] = {{ {} }};\n\n", - id, - values.size(), - join_formatted(values, ", ", [](std::int64_t v) { return std::to_string(v); })); - } break; - - case value_type::STRING: { - auto& values = std::get>(array.values); - result += std::format( - "constexpr char const* {}[{}] = {{ {} }};\n\n", - id, - values.size(), - join_formatted(values, ", ", [](auto s) { return std::format("\"{}\"", s); })); - } break; - - default: break; + if (auto* ints = std::get_if(&array); ints != nullptr) { + result += + std::format("constexpr std::int64_t {}[{}] = {{ {} }};\n\n", + id, + ints->size(), + join_formatted(*ints, ", ", [](std::int64_t v) { return std::to_string(v); })); + } else { + auto& strings = std::get(array); + result += + std::format("constexpr char const* {}[{}] = {{ {} }};\n\n", + id, + strings.size(), + join_formatted(strings, ", ", [](auto s) { return std::format("\"{}\"", s); })); } } @@ -356,14 +342,12 @@ constexpr std::uint8_t hash[{}] = auto asm_source = std::format( R"***( .section .rodata -.global {}_files_begin -{}_files_begin: -.incbin "{}.bin" +.global {0}_files_begin +{0}_files_begin: +.incbin "{0}.bin" .section .note.GNU-stack,"",@progbits )***", - id, - id, id); return embed_output{ diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 88760e02c919..bc9ac16fc1bf 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -267,7 +267,7 @@ nvJitLinkInputType to_nvjitlink_input_type(binary_type bin_type) } } -void* load_dll(std::string_view base_name, std::span names) +void* load_dso(std::string_view base_name, std::span names) { for (auto& name : names) { void* handle = ::dlopen(name.c_str(), RTLD_NOW | RTLD_LOCAL); @@ -311,7 +311,7 @@ struct LibCuda { static void* _load() { std::string lib_names[] = {"libcuda.so.1"}; // NOLINT(modernize-avoid-c-arrays) - return load_dll("libcuda.so", lib_names); + return load_dso("libcuda.so", lib_names); } private: @@ -354,7 +354,7 @@ struct LibNVRTC { std::string lib_names[] = // NOLINT(modernize-avoid-c-arrays) {std::format("libnvrtc.so.{}", major)}; - return load_dll("libnvrtc.so", lib_names); + return load_dso("libnvrtc.so", lib_names); } private: @@ -397,9 +397,9 @@ struct LibNVJitLink { std::runtime_error); std::string lib_names[] = // NOLINT(modernize-avoid-c-arrays) - {std::format("libnvJitLink.so.{}", major), "libnvJitLink.so"}; + {std::format("libnvJitLink.so.{}", major)}; - return load_dll("libnvJitLink.so", lib_names); + return load_dso("libnvJitLink.so", lib_names); } private: @@ -919,6 +919,11 @@ std::optional blob_t::from_file(char const* path) auto file_size = ::lseek(fd, 0, SEEK_END); if (file_size == -1) { throw_posix("Failed to determine size of RTCX cache file", "lseek"); } + if (file_size == 0) { + // mmap does not support mapping zero-length files, so we return an empty blob in this case + return blob_t::from_static_data({}); + } + void* map = ::mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTCX cache file", "mmap"); } @@ -1004,8 +1009,15 @@ void cache_blob_to_disk(std::string const& cache_dir, if (::close(fd) == -1) { throw_posix("Failed to close temporary RTCX cache file", "close"); } }); - if (::write(fd, binary.data(), binary.size()) == -1) { - throw_posix("Failed to write RTCX cache to temporary file", "write"); + { + auto ptr = binary.data(); + auto remaining = binary.size(); + while (remaining > 0) { + auto written = ::write(fd, ptr, remaining); + if (written == -1) { throw_posix("Failed to write RTCX cache to temporary file", "write"); } + ptr += static_cast(written); + remaining -= static_cast(written); + } } } diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index a293aeb920f6..94a4b8a7d6db 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -612,9 +612,10 @@ struct cache_t { // NOLINT * @brief Construct a new cache_t object with the specified cache directory, limits, and options * for preloading and enabling the cache. * @param cache_dir The directory path to be used for on-disk caching of compiled blobs and - * libraries + * libraries (this directory must exist and be writable by the process) * @param tmp_dir The directory path to be used for temporary files during atomic writes to the - * on-disk cache + * on-disk cache (this directory must exist, writable by the process, and be on the same + * filesystem as cache_dir to ensure atomic renames work correctly) * @param limits A cache_limits struct specifying the maximum number of blobs and libraries to * store in the cache before eviction occurs * @param preload A boolean flag indicating whether to preload the cache from disk during From 8bde405032535831d4368d7f15a47d33ea58e90d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 15 May 2026 01:10:33 +0000 Subject: [PATCH 180/254] Refactor embed_includes function to use SOURCE_DIRECTORY instead of COPY_DIRECTORY for clarity Remove unnecessary comment in initialize function documentation Change anonymous namespace to inline namespace detail in sha256.hpp for consistency --- cpp/librtcx/embed.cmake | 20 ++++++++++---------- cpp/librtcx/rtcx.hpp | 1 - cpp/librtcx/sha256.hpp | 4 ++-- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index 2b1ff3bff2b5..0399ac58088f 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -38,12 +38,12 @@ endfunction() # This function registers a directory of include files to be embedded for JIT compilation. function(embed_includes TARGET) set(OPTIONS "") - set(ONE_VALUE_ARGS COPY_DIRECTORY # Source directory where files will be copied from + set(ONE_VALUE_ARGS SOURCE_DIRECTORY # Source directory where files will be copied from DEST_DIRECTORY # Destination directory where files will be copied to ) set(MULTI_VALUE_ARGS - FILES # Source files relative to COPY_DIRECTORY (optional, if not provided, all files under - # COPY_DIRECTORY will be used) + FILES # Source files relative to SOURCE_DIRECTORY (optional, if not provided, all files under + # SOURCE_DIRECTORY will be used) INCLUDE_DIRECTORIES # Include directories to be used when compiling with these files ) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) @@ -52,23 +52,23 @@ function(embed_includes TARGET) message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") endif() - if(NOT ARG_COPY_DIRECTORY + if(NOT ARG_SOURCE_DIRECTORY OR NOT ARG_DEST_DIRECTORY OR NOT ARG_INCLUDE_DIRECTORIES ) message( - FATAL_ERROR "COPY_DIRECTORY, DEST_DIRECTORY, and INCLUDE_DIRECTORIES arguments are required" + FATAL_ERROR "SOURCE_DIRECTORY, DEST_DIRECTORY, and INCLUDE_DIRECTORIES arguments are required" ) endif() if(NOT ARG_FILES) # gather all include files under the specified directory - file(GLOB_RECURSE INCLUDE_FILES "${ARG_COPY_DIRECTORY}/*") + file(GLOB_RECURSE INCLUDE_FILES "${ARG_SOURCE_DIRECTORY}/*") # get their paths relative to the base include directory set(INCLUDE_FILES_RELATIVE_PATHS "") foreach(INCLUDE_FILE IN LISTS INCLUDE_FILES) - file(RELATIVE_PATH INCLUDE_FILE_REL_PATH "${ARG_COPY_DIRECTORY}" "${INCLUDE_FILE}") + file(RELATIVE_PATH INCLUDE_FILE_REL_PATH "${ARG_SOURCE_DIRECTORY}" "${INCLUDE_FILE}") list(APPEND INCLUDE_FILES_RELATIVE_PATHS "${INCLUDE_FILE_REL_PATH}") endforeach() @@ -77,8 +77,8 @@ function(embed_includes TARGET) # check that each source file exists foreach(SOURCE_FILE IN LISTS ARG_FILES) - if(NOT EXISTS "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") - message(FATAL_ERROR "Source file '${ARG_COPY_DIRECTORY}/${SOURCE_FILE}' does not exist") + if(NOT EXISTS "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}") + message(FATAL_ERROR "Source file '${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}' does not exist") endif() endforeach(SOURCE_FILE) @@ -92,7 +92,7 @@ function(embed_includes TARGET) foreach(SOURCE_FILE IN LISTS ARG_FILES) list(LENGTH SOURCE_FILE_IDS SOURCE_FILE_IDS_LENGTH) list(APPEND SOURCE_FILE_IDS "include_${SOURCE_FILE_IDS_LENGTH}") - list(APPEND SOURCE_FILES "${ARG_COPY_DIRECTORY}/${SOURCE_FILE}") + list(APPEND SOURCE_FILES "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}") list(APPEND SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}") endforeach() diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 94a4b8a7d6db..0dc3574e2efc 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -785,7 +785,6 @@ struct cache_t { // NOLINT * operations. Failure to call this function before using other RTCX functions may result in * undefined behavior or runtime errors. * This function is thread-safe. - * */ void initialize(); diff --git a/cpp/librtcx/sha256.hpp b/cpp/librtcx/sha256.hpp index 26f12a0b6216..14543fa684d8 100644 --- a/cpp/librtcx/sha256.hpp +++ b/cpp/librtcx/sha256.hpp @@ -13,7 +13,7 @@ namespace rtcx { -namespace { +inline namespace detail { std::uint32_t ror(std::uint32_t x, std::uint32_t n) { return (x >> n) | (x << (32 - n)); } @@ -48,7 +48,7 @@ std::uint32_t get_be32(void const* ptr) (std::uint32_t)p[3] << 0; } -} // namespace +} // namespace detail struct [[nodiscard]] sha256_hex_string { char data_[65]; // NOLINT(modernize-avoid-c-arrays) From 463d0084be2f8624afe6f146ccc9cab841dea184 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 15 May 2026 01:21:44 +0000 Subject: [PATCH 181/254] Add template specialization for reflect function to handle various types --- cpp/librtcx/rtcx.hpp | 59 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 0dc3574e2efc..329da4ea33f5 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -800,19 +800,35 @@ void initialize(); */ void teardown(); +/** + * @brief Reflect a value of any type into its CUDA string representation + * @tparam T The type of the value to be reflected + * @param value The value to be reflected + * @return A string containing the CUDA representation of the value + * @details This is a template function that can be specialized for different types to provide + * appropriate CUDA string representations. + */ +template +std::string reflect(T value); + /** * @brief Reflect a boolean value into its CUDA string representation ("true" or "false") * @param value The boolean value to be reflected * @return A string containing the CUDA representation of the boolean value ("true" or "false") */ -inline std::string reflect_bool(bool value) { return std::format("(bool){}", value); } +template <> +inline std::string reflect(bool value) +{ + return std::format("(bool){}", value); +} /** * @brief Reflect an integer value into its CUDA string representation * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -inline std::string reflect_int(std::uint8_t value) +template <> +inline std::string reflect(std::uint8_t value) { return std::format("(unsigned char){}U", value); } @@ -822,7 +838,8 @@ inline std::string reflect_int(std::uint8_t value) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -inline std::string reflect_int(std::uint16_t value) +template <> +inline std::string reflect(std::uint16_t value) { return std::format("(unsigned short){}U", value); } @@ -832,7 +849,8 @@ inline std::string reflect_int(std::uint16_t value) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -inline std::string reflect_int(std::uint32_t value) +template <> +inline std::string reflect(std::uint32_t value) { return std::format("(unsigned int){}U", value); } @@ -842,7 +860,8 @@ inline std::string reflect_int(std::uint32_t value) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -inline std::string reflect_int(std::uint64_t value) +template <> +inline std::string reflect(std::uint64_t value) { return std::format("(unsigned long long int){}ULL", value); } @@ -852,14 +871,19 @@ inline std::string reflect_int(std::uint64_t value) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -inline std::string reflect_int(std::int8_t value) { return std::format("(signed char){}", value); } +template <> +inline std::string reflect(std::int8_t value) +{ + return std::format("(signed char){}", value); +} /** * @brief Reflect an integer value into its CUDA string representation * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -inline std::string reflect_int(std::int16_t value) +template <> +inline std::string reflect(std::int16_t value) { return std::format("(signed short){}", value); } @@ -869,14 +893,19 @@ inline std::string reflect_int(std::int16_t value) * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -inline std::string reflect_int(std::int32_t value) { return std::format("(signed int){}", value); } +template <> +inline std::string reflect(std::int32_t value) +{ + return std::format("(signed int){}", value); +} /** * @brief Reflect an integer value into its CUDA string representation * @param value The integer value to be reflected * @return A string containing the CUDA representation of the integer value */ -inline std::string reflect_int(std::int64_t value) +template <> +inline std::string reflect(std::int64_t value) { return std::format("(signed long long int){}LL", value); } @@ -886,14 +915,22 @@ inline std::string reflect_int(std::int64_t value) * @param value The floating-point value to be reflected * @return A string containing the CUDA representation of the floating-point value */ -inline std::string reflect_float(float value) { return std::format("(float){}F", value); } +template <> +inline std::string reflect(float value) +{ + return std::format("(float){}F", value); +} /** * @brief Reflect a floating-point value into its CUDA string representation * @param value The floating-point value to be reflected * @return A string containing the CUDA representation of the floating-point value */ -inline std::string reflect_float(double value) { return std::format("(double){}", value); } +template <> +inline std::string reflect(double value) +{ + return std::format("(double){}", value); +} /** * @brief Reflect a value of any type into its CUDA string representation, given the type name as a From 2a2642876e2ca2fb0412208541801ab1b5e78dd5 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 15 May 2026 01:34:09 +0000 Subject: [PATCH 182/254] WAR: pre-commit namespace decl --- cpp/librtcx/embed.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index e7c15beeee31..0440c102bf1b 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -262,6 +262,7 @@ embed_output generate_cxx_source_files_data(std::string_view id, auto hash_list = join_formatted( hash, ", ", [](uint8_t byte) { return std::format("0x{:02x}", static_cast(byte)); }); auto arrays_list = generate_arrays(array_ids, array_values); + auto namespace_decl = "namespace " + std::string(id); auto cxx_header = std::format( R"***( @@ -273,7 +274,7 @@ embed_output generate_cxx_source_files_data(std::string_view id, #include #include -namespace {} {{ +{} {{ constexpr char const * include_directories[{}] = @@ -319,7 +320,7 @@ constexpr std::uint8_t hash[{}] = }} )***", - id, +namespace_decl, id, include_dirs.size(), include_dirs_list, From df6c97d94a1d17ddd2a7dc20c2fc791abd600cbd Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 15 May 2026 01:37:45 +0000 Subject: [PATCH 183/254] Fix formatting in generate_cxx_source_files_data function for improved readability --- cpp/librtcx/embed.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index 0440c102bf1b..30b45a19fd5b 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -261,7 +261,7 @@ embed_output generate_cxx_source_files_data(std::string_view id, file_ranges, ",\n", [](auto r) { return std::format("{{{}, {}}}", r.offset, r.size); }); auto hash_list = join_formatted( hash, ", ", [](uint8_t byte) { return std::format("0x{:02x}", static_cast(byte)); }); - auto arrays_list = generate_arrays(array_ids, array_values); + auto arrays_list = generate_arrays(array_ids, array_values); auto namespace_decl = "namespace " + std::string(id); auto cxx_header = std::format( @@ -320,7 +320,7 @@ constexpr std::uint8_t hash[{}] = }} )***", -namespace_decl, + namespace_decl, id, include_dirs.size(), include_dirs_list, From ac0caaacfc5c1e42209cf94323ec2d8aca048bfc Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 18 May 2026 13:47:19 +0000 Subject: [PATCH 184/254] replace error-prone parent scope variable assignment with target properties --- cpp/librtcx/embed.cmake | 138 +++++++++++----------------------------- 1 file changed, 38 insertions(+), 100 deletions(-) diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index 0399ac58088f..7ba8e76fc9a2 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -12,9 +12,9 @@ if(NOT TARGET zstd) endif() # This function initializes a target for JIT embedding. It must be called before any calls to -# embed_includes() or embed_blob() for the target. It sets up necessary variables and state to track -# the registered files and dependencies for the target. The TARGET argument specifies the name of -# the target being initialized. +# embed_includes() or embed_blob() for the target. It creates a dedicated INTERFACE library target +# that is used to track registered files and dependencies via target properties. The TARGET argument +# specifies the name of the target being initialized. function(add_embed TARGET) set(OPTIONS "") set(ONE_VALUE_ARGS) @@ -25,14 +25,7 @@ function(add_embed TARGET) message(FATAL_ERROR "TARGET argument is required") endif() - set(${TARGET}_FILE_INDEX - 0 - PARENT_SCOPE - ) - set(${TARGET}_INITIALIZED - TRUE - PARENT_SCOPE - ) + add_library(${TARGET}__embed__props INTERFACE) endfunction() # This function registers a directory of include files to be embedded for JIT compilation. @@ -48,7 +41,7 @@ function(embed_includes TARGET) ) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - if(NOT DEFINED TARGET OR NOT ${TARGET}_INITIALIZED) + if(NOT TARGET ${TARGET}__embed__props) message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") endif() @@ -82,44 +75,18 @@ function(embed_includes TARGET) endif() endforeach(SOURCE_FILE) - # Set scope variables to accumulate results - - set(SOURCE_FILE_IDS ${${TARGET}__embed__source_file_ids}) - set(SOURCE_FILES ${${TARGET}__embed__source_files}) - set(SOURCE_FILE_DESTS ${${TARGET}__embed__source_file_dests}) - set(INCLUDE_DIRECTORIES ${${TARGET}__embed__include_directories}) + # Determine the starting index for new IDs from the current list length + get_property(SOURCE_FILE_IDS TARGET ${TARGET}__embed__props PROPERTY EMBED_SOURCE_FILE_IDS) + list(LENGTH SOURCE_FILE_IDS IDX) foreach(SOURCE_FILE IN LISTS ARG_FILES) - list(LENGTH SOURCE_FILE_IDS SOURCE_FILE_IDS_LENGTH) - list(APPEND SOURCE_FILE_IDS "include_${SOURCE_FILE_IDS_LENGTH}") - list(APPEND SOURCE_FILES "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}") - list(APPEND SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}") + set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILE_IDS "include_${IDX}") + set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILES "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}") + set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}") + math(EXPR IDX "${IDX} + 1") endforeach() - list(APPEND INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES}) - - list(LENGTH SOURCE_FILE_IDS SOURCE_FILE_IDS_LENGTH) - - set(${TARGET}_FILE_INDEX - ${SOURCE_FILE_IDS_LENGTH} - PARENT_SCOPE - ) - set(${TARGET}__embed__source_file_ids - ${SOURCE_FILE_IDS} - PARENT_SCOPE - ) - set(${TARGET}__embed__source_files - ${SOURCE_FILES} - PARENT_SCOPE - ) - set(${TARGET}__embed__source_file_dests - ${SOURCE_FILE_DESTS} - PARENT_SCOPE - ) - set(${TARGET}__embed__include_directories - ${INCLUDE_DIRECTORIES} - PARENT_SCOPE - ) + set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES}) endfunction() @@ -130,7 +97,7 @@ function(embed_blob TARGET) set(MULTI_VALUE_ARGS ARRAY_IDS ARRAY_VALUES) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - if(NOT DEFINED TARGET OR NOT ${TARGET}_INITIALIZED) + if(NOT TARGET ${TARGET}__embed__props) message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") endif() @@ -141,13 +108,6 @@ function(embed_blob TARGET) message(FATAL_ERROR "ID, FILE, and DEST arguments are required") endif() - set(SOURCE_FILE_IDS ${${TARGET}__embed__source_file_ids}) - set(SOURCE_FILES ${${TARGET}__embed__source_files}) - set(SOURCE_FILE_DESTS ${${TARGET}__embed__source_file_dests}) - set(TARGET_DEPS ${${TARGET}__embed__target_deps}) - set(ARRAY_IDS ${${TARGET}__embed__array_ids}) - set(ARRAY_VALUES ${${TARGET}__embed__array_values}) - if(ARG_ARRAY_IDS) if(NOT ARG_ARRAY_VALUES) message(FATAL_ERROR "ARRAY_VALUES argument is required when ARRAY_IDS is provided") @@ -160,52 +120,22 @@ function(embed_blob TARGET) message(FATAL_ERROR "ARRAY_IDS and ARRAY_VALUES must have the same length") endif() - list(APPEND ARRAY_IDS ${ARG_ARRAY_IDS}) - list(APPEND ARRAY_VALUES ${ARG_ARRAY_VALUES}) + set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_ARRAY_IDS ${ARG_ARRAY_IDS}) + set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_ARRAY_VALUES ${ARG_ARRAY_VALUES}) endif() if(ARG_FILE MATCHES "\\$]+)>") # If the file is a generator expression for target objects add as dependency - list(APPEND TARGET_DEPS $) + set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_TARGET_DEPS $) else() if(NOT EXISTS "${ARG_FILE}") message(FATAL_ERROR "Source file '${ARG_FILE}' does not exist") endif() endif() - list(APPEND SOURCE_FILE_IDS ${ARG_ID}) - list(APPEND SOURCE_FILES ${ARG_FILE}) - list(APPEND SOURCE_FILE_DESTS ${ARG_DEST}) - list(LENGTH SOURCE_FILE_IDS SOURCE_FILE_IDS_LENGTH) - - set(${TARGET}_FILE_INDEX - ${SOURCE_FILE_IDS_LENGTH} - PARENT_SCOPE - ) - set(${TARGET}__embed__source_file_ids - ${SOURCE_FILE_IDS} - PARENT_SCOPE - ) - set(${TARGET}__embed__source_files - ${SOURCE_FILES} - PARENT_SCOPE - ) - set(${TARGET}__embed__source_file_dests - ${SOURCE_FILE_DESTS} - PARENT_SCOPE - ) - set(${TARGET}__embed__target_deps - ${TARGET_DEPS} - PARENT_SCOPE - ) - set(${TARGET}__embed__array_ids - ${ARRAY_IDS} - PARENT_SCOPE - ) - set(${TARGET}__embed__array_values - ${ARRAY_VALUES} - PARENT_SCOPE - ) + set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILE_IDS ${ARG_ID}) + set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILES ${ARG_FILE}) + set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILE_DESTS ${ARG_DEST}) endfunction() @@ -217,7 +147,7 @@ function(embed TARGET) set(MULTI_VALUE_ARGS "") cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - if(NOT DEFINED TARGET OR NOT ${TARGET}_INITIALIZED) + if(NOT TARGET ${TARGET}__embed__props) message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") endif() @@ -229,22 +159,30 @@ function(embed TARGET) message(FATAL_ERROR "COMPRESSION argument must be either none or zstd") endif() - if(NOT DEFINED ${TARGET}__embed__source_files) + get_property(EMBED_SOURCE_FILES TARGET ${TARGET}__embed__props PROPERTY EMBED_SOURCE_FILES) + if(NOT EMBED_SOURCE_FILES) message(FATAL_ERROR "No source files registered for target '${TARGET}'") endif() + get_property(EMBED_SOURCE_FILE_IDS TARGET ${TARGET}__embed__props PROPERTY EMBED_SOURCE_FILE_IDS) + get_property(EMBED_SOURCE_FILE_DESTS TARGET ${TARGET}__embed__props PROPERTY EMBED_SOURCE_FILE_DESTS) + get_property(EMBED_TARGET_DEPS TARGET ${TARGET}__embed__props PROPERTY EMBED_TARGET_DEPS) + get_property(EMBED_ARRAY_IDS TARGET ${TARGET}__embed__props PROPERTY EMBED_ARRAY_IDS) + get_property(EMBED_ARRAY_VALUES TARGET ${TARGET}__embed__props PROPERTY EMBED_ARRAY_VALUES) + get_property(EMBED_INCLUDE_DIRS TARGET ${TARGET}__embed__props PROPERTY EMBED_INCLUDE_DIRECTORIES) + set(OUTPUT_DIR "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") set(EMBED_SCRIPT_TEMPLATE "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.cpp") set(CONFIGURED_EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed_cfg.cpp") set(EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed.cpp") set(EMBED_SCRIPT__ID "${TARGET}") - set(EMBED_SCRIPT__ARRAY_IDS "${${TARGET}__embed__array_ids}") - set(EMBED_SCRIPT__ARRAY_VALUES "${${TARGET}__embed__array_values}") - set(EMBED_SCRIPT__FILE_IDS "${${TARGET}__embed__source_file_ids}") - set(EMBED_SCRIPT__FILE_PATHS "${${TARGET}__embed__source_files}") - set(EMBED_SCRIPT__FILE_DESTS "${${TARGET}__embed__source_file_dests}") - set(EMBED_SCRIPT__INCLUDE_DIRS "${${TARGET}__embed__include_directories}") + set(EMBED_SCRIPT__ARRAY_IDS "${EMBED_ARRAY_IDS}") + set(EMBED_SCRIPT__ARRAY_VALUES "${EMBED_ARRAY_VALUES}") + set(EMBED_SCRIPT__FILE_IDS "${EMBED_SOURCE_FILE_IDS}") + set(EMBED_SCRIPT__FILE_PATHS "${EMBED_SOURCE_FILES}") + set(EMBED_SCRIPT__FILE_DESTS "${EMBED_SOURCE_FILE_DESTS}") + set(EMBED_SCRIPT__INCLUDE_DIRS "${EMBED_INCLUDE_DIRS}") set(EMBED_SCRIPT__COMPRESSION "${ARG_COMPRESSION}") set(EMBED_SCRIPT__OUTPUT_DIR "${OUTPUT_DIR}") @@ -267,8 +205,8 @@ function(embed TARGET) add_custom_command( OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin COMMAND "${CMAKE_COMMAND}" -E env $ - DEPENDS "${EMBED_SCRIPT}" ${${TARGET}__embed__source_files} - ${${TARGET}__embed__target_deps} + DEPENDS "${EMBED_SCRIPT}" ${EMBED_SOURCE_FILES} + ${EMBED_TARGET_DEPS} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" COMMENT "Generating JIT embed for ${TARGET} into ${OUTPUT_DIR}" VERBATIM From 908cf50d8b4715a2ee318ef32ecbf8ba83f5a157 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 18 May 2026 13:50:54 +0000 Subject: [PATCH 185/254] pre-commit: formatting --- cpp/librtcx/embed.cmake | 115 ++++++++++++++++++++++++++++++++-------- 1 file changed, 92 insertions(+), 23 deletions(-) diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index 7ba8e76fc9a2..7930f95ccbe7 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -76,17 +76,37 @@ function(embed_includes TARGET) endforeach(SOURCE_FILE) # Determine the starting index for new IDs from the current list length - get_property(SOURCE_FILE_IDS TARGET ${TARGET}__embed__props PROPERTY EMBED_SOURCE_FILE_IDS) + get_property( + SOURCE_FILE_IDS + TARGET ${TARGET}__embed__props + PROPERTY EMBED_SOURCE_FILE_IDS + ) list(LENGTH SOURCE_FILE_IDS IDX) foreach(SOURCE_FILE IN LISTS ARG_FILES) - set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILE_IDS "include_${IDX}") - set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILES "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}") - set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}") + set_property( + TARGET ${TARGET}__embed__props + APPEND + PROPERTY EMBED_SOURCE_FILE_IDS "include_${IDX}" + ) + set_property( + TARGET ${TARGET}__embed__props + APPEND + PROPERTY EMBED_SOURCE_FILES "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}" + ) + set_property( + TARGET ${TARGET}__embed__props + APPEND + PROPERTY EMBED_SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}" + ) math(EXPR IDX "${IDX} + 1") endforeach() - set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES}) + set_property( + TARGET ${TARGET}__embed__props + APPEND + PROPERTY EMBED_INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES} + ) endfunction() @@ -120,22 +140,46 @@ function(embed_blob TARGET) message(FATAL_ERROR "ARRAY_IDS and ARRAY_VALUES must have the same length") endif() - set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_ARRAY_IDS ${ARG_ARRAY_IDS}) - set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_ARRAY_VALUES ${ARG_ARRAY_VALUES}) + set_property( + TARGET ${TARGET}__embed__props + APPEND + PROPERTY EMBED_ARRAY_IDS ${ARG_ARRAY_IDS} + ) + set_property( + TARGET ${TARGET}__embed__props + APPEND + PROPERTY EMBED_ARRAY_VALUES ${ARG_ARRAY_VALUES} + ) endif() if(ARG_FILE MATCHES "\\$]+)>") # If the file is a generator expression for target objects add as dependency - set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_TARGET_DEPS $) + set_property( + TARGET ${TARGET}__embed__props + APPEND + PROPERTY EMBED_TARGET_DEPS $ + ) else() if(NOT EXISTS "${ARG_FILE}") message(FATAL_ERROR "Source file '${ARG_FILE}' does not exist") endif() endif() - set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILE_IDS ${ARG_ID}) - set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILES ${ARG_FILE}) - set_property(TARGET ${TARGET}__embed__props APPEND PROPERTY EMBED_SOURCE_FILE_DESTS ${ARG_DEST}) + set_property( + TARGET ${TARGET}__embed__props + APPEND + PROPERTY EMBED_SOURCE_FILE_IDS ${ARG_ID} + ) + set_property( + TARGET ${TARGET}__embed__props + APPEND + PROPERTY EMBED_SOURCE_FILES ${ARG_FILE} + ) + set_property( + TARGET ${TARGET}__embed__props + APPEND + PROPERTY EMBED_SOURCE_FILE_DESTS ${ARG_DEST} + ) endfunction() @@ -159,17 +203,45 @@ function(embed TARGET) message(FATAL_ERROR "COMPRESSION argument must be either none or zstd") endif() - get_property(EMBED_SOURCE_FILES TARGET ${TARGET}__embed__props PROPERTY EMBED_SOURCE_FILES) + get_property( + EMBED_SOURCE_FILES + TARGET ${TARGET}__embed__props + PROPERTY EMBED_SOURCE_FILES + ) if(NOT EMBED_SOURCE_FILES) message(FATAL_ERROR "No source files registered for target '${TARGET}'") endif() - get_property(EMBED_SOURCE_FILE_IDS TARGET ${TARGET}__embed__props PROPERTY EMBED_SOURCE_FILE_IDS) - get_property(EMBED_SOURCE_FILE_DESTS TARGET ${TARGET}__embed__props PROPERTY EMBED_SOURCE_FILE_DESTS) - get_property(EMBED_TARGET_DEPS TARGET ${TARGET}__embed__props PROPERTY EMBED_TARGET_DEPS) - get_property(EMBED_ARRAY_IDS TARGET ${TARGET}__embed__props PROPERTY EMBED_ARRAY_IDS) - get_property(EMBED_ARRAY_VALUES TARGET ${TARGET}__embed__props PROPERTY EMBED_ARRAY_VALUES) - get_property(EMBED_INCLUDE_DIRS TARGET ${TARGET}__embed__props PROPERTY EMBED_INCLUDE_DIRECTORIES) + get_property( + EMBED_SOURCE_FILE_IDS + TARGET ${TARGET}__embed__props + PROPERTY EMBED_SOURCE_FILE_IDS + ) + get_property( + EMBED_SOURCE_FILE_DESTS + TARGET ${TARGET}__embed__props + PROPERTY EMBED_SOURCE_FILE_DESTS + ) + get_property( + EMBED_TARGET_DEPS + TARGET ${TARGET}__embed__props + PROPERTY EMBED_TARGET_DEPS + ) + get_property( + EMBED_ARRAY_IDS + TARGET ${TARGET}__embed__props + PROPERTY EMBED_ARRAY_IDS + ) + get_property( + EMBED_ARRAY_VALUES + TARGET ${TARGET}__embed__props + PROPERTY EMBED_ARRAY_VALUES + ) + get_property( + EMBED_INCLUDE_DIRS + TARGET ${TARGET}__embed__props + PROPERTY EMBED_INCLUDE_DIRECTORIES + ) set(OUTPUT_DIR "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") set(EMBED_SCRIPT_TEMPLATE "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.cpp") @@ -197,16 +269,13 @@ function(embed TARGET) add_executable(${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") target_include_directories(${RUNNER} PRIVATE ${ZSTD_INCLUDE_DIR}) target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} zstd) - set_target_properties( - ${RUNNER} PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES - ) + set_target_properties(${RUNNER} PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES) target_include_directories(${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR}) add_custom_command( OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin COMMAND "${CMAKE_COMMAND}" -E env $ - DEPENDS "${EMBED_SCRIPT}" ${EMBED_SOURCE_FILES} - ${EMBED_TARGET_DEPS} + DEPENDS "${EMBED_SCRIPT}" ${EMBED_SOURCE_FILES} ${EMBED_TARGET_DEPS} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" COMMENT "Generating JIT embed for ${TARGET} into ${OUTPUT_DIR}" VERBATIM From 48d1ec06832c74b67ac33f5e625cc49c5b401154 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 18 May 2026 14:30:55 +0000 Subject: [PATCH 186/254] fix: correct variable name for embed properties in JIT functions --- cpp/librtcx/embed.cmake | 44 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index 7930f95ccbe7..a2b60635fa4b 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -25,7 +25,7 @@ function(add_embed TARGET) message(FATAL_ERROR "TARGET argument is required") endif() - add_library(${TARGET}__embed__props INTERFACE) + add_library(${TARGET}__embed_props INTERFACE) endfunction() # This function registers a directory of include files to be embedded for JIT compilation. @@ -41,7 +41,7 @@ function(embed_includes TARGET) ) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - if(NOT TARGET ${TARGET}__embed__props) + if(NOT TARGET ${TARGET}__embed_props) message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") endif() @@ -78,24 +78,24 @@ function(embed_includes TARGET) # Determine the starting index for new IDs from the current list length get_property( SOURCE_FILE_IDS - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props PROPERTY EMBED_SOURCE_FILE_IDS ) list(LENGTH SOURCE_FILE_IDS IDX) foreach(SOURCE_FILE IN LISTS ARG_FILES) set_property( - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props APPEND PROPERTY EMBED_SOURCE_FILE_IDS "include_${IDX}" ) set_property( - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props APPEND PROPERTY EMBED_SOURCE_FILES "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}" ) set_property( - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props APPEND PROPERTY EMBED_SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}" ) @@ -103,7 +103,7 @@ function(embed_includes TARGET) endforeach() set_property( - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props APPEND PROPERTY EMBED_INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES} ) @@ -117,7 +117,7 @@ function(embed_blob TARGET) set(MULTI_VALUE_ARGS ARRAY_IDS ARRAY_VALUES) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - if(NOT TARGET ${TARGET}__embed__props) + if(NOT TARGET ${TARGET}__embed_props) message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") endif() @@ -141,12 +141,12 @@ function(embed_blob TARGET) endif() set_property( - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props APPEND PROPERTY EMBED_ARRAY_IDS ${ARG_ARRAY_IDS} ) set_property( - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props APPEND PROPERTY EMBED_ARRAY_VALUES ${ARG_ARRAY_VALUES} ) @@ -155,7 +155,7 @@ function(embed_blob TARGET) if(ARG_FILE MATCHES "\\$]+)>") # If the file is a generator expression for target objects add as dependency set_property( - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props APPEND PROPERTY EMBED_TARGET_DEPS $ ) @@ -166,17 +166,17 @@ function(embed_blob TARGET) endif() set_property( - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props APPEND PROPERTY EMBED_SOURCE_FILE_IDS ${ARG_ID} ) set_property( - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props APPEND PROPERTY EMBED_SOURCE_FILES ${ARG_FILE} ) set_property( - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props APPEND PROPERTY EMBED_SOURCE_FILE_DESTS ${ARG_DEST} ) @@ -191,7 +191,7 @@ function(embed TARGET) set(MULTI_VALUE_ARGS "") cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - if(NOT TARGET ${TARGET}__embed__props) + if(NOT TARGET ${TARGET}__embed_props) message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") endif() @@ -205,7 +205,7 @@ function(embed TARGET) get_property( EMBED_SOURCE_FILES - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props PROPERTY EMBED_SOURCE_FILES ) if(NOT EMBED_SOURCE_FILES) @@ -214,32 +214,32 @@ function(embed TARGET) get_property( EMBED_SOURCE_FILE_IDS - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props PROPERTY EMBED_SOURCE_FILE_IDS ) get_property( EMBED_SOURCE_FILE_DESTS - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props PROPERTY EMBED_SOURCE_FILE_DESTS ) get_property( EMBED_TARGET_DEPS - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props PROPERTY EMBED_TARGET_DEPS ) get_property( EMBED_ARRAY_IDS - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props PROPERTY EMBED_ARRAY_IDS ) get_property( EMBED_ARRAY_VALUES - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props PROPERTY EMBED_ARRAY_VALUES ) get_property( EMBED_INCLUDE_DIRS - TARGET ${TARGET}__embed__props + TARGET ${TARGET}__embed_props PROPERTY EMBED_INCLUDE_DIRECTORIES ) From cbfc0863f5ca26584fc8f3aed16906b243f65d80 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 20 May 2026 12:40:27 +0000 Subject: [PATCH 187/254] fix: update std::call_once to use value() for initialization flags --- cpp/librtcx/rtcx.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index bc9ac16fc1bf..ae2a970c5ec4 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -425,7 +425,7 @@ void initialize() { RTCX_FUNC_RANGE(); - std::call_once(*init_libraries_flag, [] { + std::call_once(init_libraries_flag.value(), [] { cu.emplace(LibCuda::_load()); RTCX_EXPECTS( cu->Init(0) == CUDA_SUCCESS, "Failed to initialize CUDA driver API", std::runtime_error); @@ -438,12 +438,14 @@ void teardown() { RTCX_FUNC_RANGE(); - std::call_once(*teardown_libraries_flag, [] { + std::call_once(teardown_libraries_flag.value(), [] { nvjitlink.reset(); nvrtc.reset(); cu.reset(); init_libraries_flag.reset(); teardown_libraries_flag.reset(); + init_libraries_flag.emplace(); + teardown_libraries_flag.emplace(); }); } From 56ac2491f9479de8f68abdae9d8536c2da40c6c9 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 20 May 2026 12:50:46 +0000 Subject: [PATCH 188/254] fix: change namespace declaration to inline for detail --- cpp/librtcx/rtcx.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 329da4ea33f5..58ae03a1fed5 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -466,7 +466,7 @@ struct [[nodiscard]] link_params { std::span link_options = {}; //< NVJITLink options }; -namespace detail { +inline namespace detail { template struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { From a138cb7386dd4730a5608f97970824e0970164cf Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 20 May 2026 13:15:24 +0000 Subject: [PATCH 189/254] fix: update reflect_enum to use reflect instead of reflect_int --- cpp/librtcx/rtcx.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 58ae03a1fed5..d0ea4bff235c 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -959,7 +959,7 @@ template requires(std::is_enum_v) std::string reflect_enum(std::string_view type, T value) { - return reflect_cast(type, reflect_int(static_cast>(value))); + return reflect_cast(type, reflect(static_cast>(value))); } /** From f554c66a166cb498d6a3e2960e5195dec6aa8c33 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 22 May 2026 15:57:23 +0000 Subject: [PATCH 190/254] code refactoring --- .../cudf/column/column_device_view_base.cuh | 4 +- cpp/include/cudf/context.hpp | 4 +- .../cudf/detail/normalizing_iterator.cuh | 10 +- cpp/include/cudf/detail/operation-udf.hpp | 5 +- cpp/include/cudf/detail/utilities/assert.cuh | 3 +- .../detail/floating_conversion.hpp | 10 +- cpp/include/cudf/fixed_point/fixed_point.hpp | 11 +- cpp/include/cudf/fixed_point/temporary.hpp | 8 +- .../detail/convert/string_to_float.cuh | 5 - cpp/include/cudf/strings/string_view.cuh | 9 +- cpp/include/cudf/strings/string_view.hpp | 10 +- cpp/include/cudf/types.hpp | 7 +- cpp/include/cudf/utilities/span.hpp | 4 +- .../rolling/detail/rolling_fixed_window.cu | 6 +- cpp/src/rolling/detail/rolling_jit.cuh | 113 ++++++++++++++++++ cpp/src/rolling/detail/rolling_jit.hpp | 50 -------- cpp/src/rolling/detail/rolling_udf.cuh | 65 +++++++--- .../rolling/detail/rolling_variable_window.cu | 17 ++- cpp/src/rolling/grouped_rolling.cu | 19 +-- cpp/src/rolling/jit/kernel.cu | 44 +++---- cpp/src/runtime/context.hpp | 2 +- .../integration/unary_transform_test.cpp | 2 +- 22 files changed, 248 insertions(+), 160 deletions(-) create mode 100644 cpp/src/rolling/detail/rolling_jit.cuh delete mode 100644 cpp/src/rolling/detail/rolling_jit.hpp diff --git a/cpp/include/cudf/column/column_device_view_base.cuh b/cpp/include/cudf/column/column_device_view_base.cuh index 1d024d410087..3c1dce082b50 100644 --- a/cpp/include/cudf/column/column_device_view_base.cuh +++ b/cpp/include/cudf/column/column_device_view_base.cuh @@ -11,12 +11,10 @@ #include #include +#include #include #include -#include -#include - /** * @file column_device_view_base.cuh * @brief Column device view class definitions diff --git a/cpp/include/cudf/context.hpp b/cpp/include/cudf/context.hpp index 7eb6f5d273fc..fa91d1eafd3c 100644 --- a/cpp/include/cudf/context.hpp +++ b/cpp/include/cudf/context.hpp @@ -20,6 +20,8 @@ enum class init_flags : std::uint32_t { LOAD_NVCOMP = 1 << 0, /// @brief Initialize the JIT program cache during initialization INIT_JIT_CACHE = 1 << 1, + /// @brief Pre-load the JIT program cache from disk during initialization + DEFAULT = INIT_JIT_CACHE, /// @brief All initialization steps (default behavior) ALL = LOAD_NVCOMP | INIT_JIT_CACHE }; @@ -68,7 +70,7 @@ constexpr bool has_flag(init_flags flags, init_flags flag) noexcept /// @param flags Optional flags to control which initialization steps to perform. /// Can be called multiple times to initialize additional components. If all selected /// steps are already performed, the call has no effect. -void initialize(init_flags flags = init_flags::INIT_JIT_CACHE); +void initialize(init_flags flags = init_flags::DEFAULT); /// @brief Destroy the cudf global context, resetting it to an uninitialized state. This is /// primarily intended for testing purposes, allowing for re-initialization of the context after diff --git a/cpp/include/cudf/detail/normalizing_iterator.cuh b/cpp/include/cudf/detail/normalizing_iterator.cuh index a4ac02b4b23a..3d777dfb87ec 100644 --- a/cpp/include/cudf/detail/normalizing_iterator.cuh +++ b/cpp/include/cudf/detail/normalizing_iterator.cuh @@ -1,11 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#ifndef CUDF_RUNTIME_JIT +#ifndef __CUDACC_RTC__ #include @@ -13,7 +13,7 @@ #include -#include +#include namespace cudf { namespace detail { @@ -33,7 +33,7 @@ struct alignas(16) base_normalator { using difference_type = std::ptrdiff_t; using value_type = Integer; using pointer = Integer*; - using iterator_category = std::random_access_iterator_tag; + using iterator_category = cuda::std::random_access_iterator_tag; base_normalator() = default; base_normalator(base_normalator const&) = default; @@ -216,7 +216,7 @@ struct alignas(16) base_normalator { }; protected: -#ifndef CUDF_RUNTIME_JIT // TODO: refactor type_dispatcher to support NVRTC +#ifndef __CUDACC_RTC__ /** * @brief Constructor assigns width and type member variables for base class. diff --git a/cpp/include/cudf/detail/operation-udf.hpp b/cpp/include/cudf/detail/operation-udf.hpp index 47bb3e91170d..3ffc72e5df37 100644 --- a/cpp/include/cudf/detail/operation-udf.hpp +++ b/cpp/include/cudf/detail/operation-udf.hpp @@ -1,9 +1,8 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -// This file serves as a placeholder for user defined functions, so jitify can choose to override it -// at runtime. +// This file serves as a placeholder for user defined functions, so NVRTC/NVCC can override it diff --git a/cpp/include/cudf/detail/utilities/assert.cuh b/cpp/include/cudf/detail/utilities/assert.cuh index 5ecd00db4fba..c1fb8afc3b67 100644 --- a/cpp/include/cudf/detail/utilities/assert.cuh +++ b/cpp/include/cudf/detail/utilities/assert.cuh @@ -1,10 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2023, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once +#include #include /** diff --git a/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp b/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp index 6bf74613da70..fecc1b6b330d 100644 --- a/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp +++ b/cpp/include/cudf/fixed_point/detail/floating_conversion.hpp @@ -14,8 +14,6 @@ #include #include -#include - namespace numeric { namespace detail { @@ -119,9 +117,7 @@ struct floating_converter { CUDF_HOST_DEVICE inline static IntegralType bit_cast_to_integer(FloatingType floating) { // Convert floating to integer - IntegralType integer_rep; - memcpy(&integer_rep, &floating, sizeof(floating)); - return integer_rep; + return cuda::std::bit_cast(floating); } /** @@ -133,9 +129,7 @@ struct floating_converter { CUDF_HOST_DEVICE inline static FloatingType bit_cast_to_floating(IntegralType integer) { // Convert back to float - FloatingType floating; - memcpy(&floating, &integer, sizeof(floating)); - return floating; + return cuda::std::bit_cast(integer); } /** diff --git a/cpp/include/cudf/fixed_point/fixed_point.hpp b/cpp/include/cudf/fixed_point/fixed_point.hpp index 0b7c4428a563..da2632ea4095 100644 --- a/cpp/include/cudf/fixed_point/fixed_point.hpp +++ b/cpp/include/cudf/fixed_point/fixed_point.hpp @@ -10,15 +10,16 @@ #include #include +#include +#include #include #include #include #include +#ifndef __CUDACC_RTC__ #include -#include -#include -#include +#endif /// `fixed_point` and supporting types namespace CUDF_EXPORT numeric { @@ -571,6 +572,8 @@ class fixed_point { return fixed_point{scaled_integer{value, scale}}; } +#ifndef __CUDACC_RTC__ + /** * @brief Returns a string representation of the fixed_point value. */ @@ -590,6 +593,8 @@ class fixed_point { auto const zeros = std::string(_scale, '0'); return detail::to_string(_value) + zeros; } + +#endif }; /** diff --git a/cpp/include/cudf/fixed_point/temporary.hpp b/cpp/include/cudf/fixed_point/temporary.hpp index 097f0cb061d8..da768c328a0b 100644 --- a/cpp/include/cudf/fixed_point/temporary.hpp +++ b/cpp/include/cudf/fixed_point/temporary.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,12 +10,16 @@ #include #include +#ifndef __CUDACC_RTC__ #include #include +#endif namespace CUDF_EXPORT numeric { namespace detail { +#ifndef __CUDACC_RTC__ + template auto to_string(T value) -> std::string { @@ -42,6 +46,8 @@ auto to_string(T value) -> std::string return std::string{}; // won't ever hit here, need to suppress warning though } +#endif + template CUDF_HOST_DEVICE constexpr auto abs(T value) { diff --git a/cpp/include/cudf/strings/detail/convert/string_to_float.cuh b/cpp/include/cudf/strings/detail/convert/string_to_float.cuh index 564406db6c1b..ea76cf106c84 100644 --- a/cpp/include/cudf/strings/detail/convert/string_to_float.cuh +++ b/cpp/include/cudf/strings/detail/convert/string_to_float.cuh @@ -35,13 +35,8 @@ __device__ inline double stod(string_view const& d_str) ++in_ptr; } -#ifndef CUDF_RUNTIME_JIT constexpr double infinity = cuda::std::numeric_limits::infinity(); constexpr uint64_t max_holding = (cuda::std::numeric_limits::max() - 9L) / 10L; -#else - constexpr double infinity = (1.0 / 0.0); - constexpr uint64_t max_holding = (18446744073709551615UL - 9UL) / 10UL; -#endif // special strings: NaN, Inf if ((in_ptr < end) && *in_ptr > '9') { diff --git a/cpp/include/cudf/strings/string_view.cuh b/cpp/include/cudf/strings/string_view.cuh index 913ba0030999..64907da35905 100644 --- a/cpp/include/cudf/strings/string_view.cuh +++ b/cpp/include/cudf/strings/string_view.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -13,9 +13,8 @@ #include #endif -// This is defined when including this header in a https://github.com/NVIDIA/jitify -// or jitify2 source file. The jitify cannot include thrust headers at this time. -#ifndef CUDF_RUNTIME_JIT +// This is defined when compiling with NVRTC. NVRTC cannot include thrust headers at this time. +#ifndef __CUDACC_RTC__ #include #include #endif @@ -41,7 +40,7 @@ __device__ inline size_type characters_in_string(char const* str, size_type byte { if ((str == nullptr) || (bytes == 0)) return 0; auto ptr = reinterpret_cast(str); -#ifndef CUDF_RUNTIME_JIT +#ifndef __CUDACC_RTC__ return thrust::count_if( thrust::seq, ptr, ptr + bytes, [](uint8_t chr) { return is_begin_utf8_char(chr); }); #else diff --git a/cpp/include/cudf/strings/string_view.hpp b/cpp/include/cudf/strings/string_view.hpp index 2a1db81ec7a3..2167743c2c30 100644 --- a/cpp/include/cudf/strings/string_view.hpp +++ b/cpp/include/cudf/strings/string_view.hpp @@ -1,14 +1,16 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include -#include +#include -#include +#ifndef __CUDACC_RTC__ +#include +#endif /** * @file @@ -68,7 +70,7 @@ class string_view { using value_type = char_utf8; using reference = char_utf8&; using pointer = char_utf8*; - using iterator_category = std::input_iterator_tag; + using iterator_category = cuda::std::input_iterator_tag; __device__ inline const_iterator(string_view const& str, size_type pos); const_iterator(const_iterator const& mit) = default; const_iterator(const_iterator&& mit) = default; diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp index 659a77796012..63c8794f29f0 100644 --- a/cpp/include/cudf/types.hpp +++ b/cpp/include/cudf/types.hpp @@ -27,10 +27,11 @@ #include +#include + #include #include #include -#include /** * @file @@ -98,7 +99,7 @@ using char_utf8 = uint32_t; ///< UTF-8 characters are 1-4 bytes template size_type distance(T f, T l) { - return static_cast(std::distance(f, l)); + return static_cast(cuda::std::distance(f, l)); } /** @@ -352,7 +353,7 @@ class data_type { */ constexpr bool operator==(data_type const& lhs, data_type const& rhs) { - // use std::tie in the future, breaks JITIFY currently + // use std::tie in the future, breaks NVRTC currently return lhs.id() == rhs.id() && lhs.scale() == rhs.scale(); } diff --git a/cpp/include/cudf/utilities/span.hpp b/cpp/include/cudf/utilities/span.hpp index 1eecb1b51f9f..aace1adff144 100644 --- a/cpp/include/cudf/utilities/span.hpp +++ b/cpp/include/cudf/utilities/span.hpp @@ -69,13 +69,13 @@ class span_base { */ CUDF_HOST_DEVICE constexpr span_base(pointer data, size_type size) : _data(data), _size(size) {} // constexpr span_base(pointer begin, pointer end) : _data(begin), _size(end - begin) {} - CUDF_HOST_DEVICE constexpr span_base(span_base const&) noexcept = default; ///< Copy constructor + constexpr span_base(span_base const&) noexcept = default; ///< Copy constructor /** * @brief Copy assignment operator. * * @return Reference to this span. */ - CUDF_HOST_DEVICE constexpr span_base& operator=(span_base const&) noexcept = default; + constexpr span_base& operator=(span_base const&) noexcept = default; /** * @brief Returns an iterator to the first element of the span. diff --git a/cpp/src/rolling/detail/rolling_fixed_window.cu b/cpp/src/rolling/detail/rolling_fixed_window.cu index f91fccb35313..8c87a9ca0a60 100644 --- a/cpp/src/rolling/detail/rolling_fixed_window.cu +++ b/cpp/src/rolling/detail/rolling_fixed_window.cu @@ -40,10 +40,8 @@ std::unique_ptr rolling_window(column_view const& input, if (agg.kind == aggregation::CUDA || agg.kind == aggregation::PTX) { // TODO: In future, might need to clamp preceding/following to column boundaries. return cudf::detail::rolling_window_udf(input, - preceding_window, - "cudf::size_type", - following_window, - "cudf::size_type", + cudf::detail::fixed_window_wrapper(preceding_window), + cudf::detail::fixed_window_wrapper(following_window), min_periods, agg, stream, diff --git a/cpp/src/rolling/detail/rolling_jit.cuh b/cpp/src/rolling/detail/rolling_jit.cuh new file mode 100644 index 000000000000..dec40de9a921 --- /dev/null +++ b/cpp/src/rolling/detail/rolling_jit.cuh @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace cudf { +namespace detail { + +template +T minimum(T a, T b) +{ + return b < a ? b : a; +} + +struct window_wrapper_base { + cudf::size_type const* group_offsets = nullptr; + cudf::size_type const* group_labels = nullptr; + cudf::size_type window = 0; +}; + +struct fixed_window_wrapper : public window_wrapper_base { + __device__ __host__ fixed_window_wrapper(cudf::size_type window) + : window_wrapper_base{nullptr, nullptr, window} + { + } + + __device__ __host__ fixed_window_wrapper(window_wrapper_base const& base) + : window_wrapper_base(base) + { + } + + __device__ __host__ cudf::size_type operator[](cudf::size_type) const { return window; } +}; + +struct variable_window_wrapper : public window_wrapper_base { + __device__ __host__ variable_window_wrapper(cudf::size_type const* group_offsets) + : window_wrapper_base{group_offsets, nullptr, 0} + { + } + + __device__ __host__ variable_window_wrapper(window_wrapper_base const& base) + : window_wrapper_base(base) + { + } + + __device__ __host__ cudf::size_type operator[](cudf::size_type idx) const + { + return group_offsets[idx]; + } +}; + +struct preceding_window_wrapper : public window_wrapper_base { + __device__ __host__ preceding_window_wrapper(cudf::size_type const* group_offsets, + cudf::size_type const* group_labels, + cudf::size_type window) + : window_wrapper_base{group_offsets, group_labels, window} + { + } + + __device__ __host__ preceding_window_wrapper(window_wrapper_base const& base) + : window_wrapper_base(base) + { + } + + __device__ cudf::size_type operator[](cudf::size_type idx) const + { + auto group_label = group_labels[idx]; + auto group_start = group_offsets[group_label]; + return minimum(window, idx - group_start + 1); // Preceding includes current row. + } +}; + +struct following_window_wrapper : public window_wrapper_base { + __device__ __host__ following_window_wrapper(cudf::size_type const* group_offsets, + cudf::size_type const* group_labels, + cudf::size_type window) + : window_wrapper_base{group_offsets, group_labels, window} + { + } + + __device__ __host__ following_window_wrapper(window_wrapper_base const& base) + : window_wrapper_base(base) + { + } + + __device__ cudf::size_type operator[](cudf::size_type idx) const + { + auto group_label = group_labels[idx]; + auto group_end = + group_offsets[group_label + + 1]; // Cannot fall off the end, since offsets is capped with `input.size()`. + return minimum(window, (group_end - 1) - idx); + } +}; + +static_assert(sizeof(fixed_window_wrapper) == sizeof(variable_window_wrapper)); +static_assert(alignof(fixed_window_wrapper) == alignof(variable_window_wrapper)); + +static_assert(sizeof(variable_window_wrapper) == sizeof(fixed_window_wrapper)); +static_assert(alignof(variable_window_wrapper) == alignof(fixed_window_wrapper)); + +static_assert(sizeof(fixed_window_wrapper) == sizeof(preceding_window_wrapper)); +static_assert(alignof(fixed_window_wrapper) == alignof(preceding_window_wrapper)); + +static_assert(sizeof(fixed_window_wrapper) == sizeof(following_window_wrapper)); +static_assert(alignof(fixed_window_wrapper) == alignof(following_window_wrapper)); + +} // namespace detail +} // namespace cudf diff --git a/cpp/src/rolling/detail/rolling_jit.hpp b/cpp/src/rolling/detail/rolling_jit.hpp deleted file mode 100644 index facc9a881837..000000000000 --- a/cpp/src/rolling/detail/rolling_jit.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2020-2023, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include - -namespace cudf { - -namespace detail { - -template -T minimum(T a, T b) -{ - return b < a ? b : a; -} - -struct preceding_window_wrapper { - cudf::size_type const* d_group_offsets; - cudf::size_type const* d_group_labels; - cudf::size_type preceding_window; - - cudf::size_type operator[](cudf::size_type idx) - { - auto group_label = d_group_labels[idx]; - auto group_start = d_group_offsets[group_label]; - return minimum(preceding_window, idx - group_start + 1); // Preceding includes current row. - } -}; - -struct following_window_wrapper { - cudf::size_type const* d_group_offsets; - cudf::size_type const* d_group_labels; - cudf::size_type following_window; - - cudf::size_type operator[](cudf::size_type idx) - { - auto group_label = d_group_labels[idx]; - auto group_end = - d_group_offsets[group_label + - 1]; // Cannot fall off the end, since offsets is capped with `input.size()`. - return minimum(following_window, (group_end - 1) - idx); - } -}; - -} // namespace detail - -} // namespace cudf diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index 1fb6d5eabfd0..2981a14e64d8 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -10,7 +10,7 @@ #include "jit/parser.hpp" #include "jit/util.hpp" #include "rolling.hpp" -#include "rolling_jit.hpp" +#include "rolling_jit.cuh" #include #include @@ -28,17 +28,31 @@ namespace cudf { namespace detail { -// Applies a user-defined rolling window function to the values in a column. -template -std::unique_ptr rolling_window_udf(column_view const& input, - PrecedingWindowIterator preceding_window, - std::string const& preceding_window_str, - FollowingWindowIterator following_window, - std::string const& following_window_str, - size_type min_periods, - rolling_aggregation const& agg, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +template +std::string reflect_window_wrapper() +{ + if constexpr (std::is_same_v) { + return "cudf::detail::fixed_window_wrapper"; + } else if constexpr (std::is_same_v) { + return "cudf::detail::variable_window_wrapper"; + } else if constexpr (std::is_same_v) { + return "cudf::detail::preceding_window_wrapper"; + } else { + static_assert(std::is_same_v, "Unsupported window wrapper type"); + return "cudf::detail::following_window_wrapper"; + } +} + +inline std::unique_ptr rolling_window_udf_impl( + column_view const& input, + std::string const& preceding_window_str, + cudf::detail::window_wrapper_base const& preceding_window, + std::string const& following_window_str, + cudf::detail::window_wrapper_base const& following_window, + size_type min_periods, + rolling_aggregation const& agg, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { static_assert(warp_size == cudf::detail::size_in_bits(), "bitmask_type size does not match CUDA warp size"); @@ -76,12 +90,12 @@ std::unique_ptr rolling_window_udf(column_view const& input, 0, stream, cudf::get_current_device_resource_ref()}; std::string kernel_reflection = - jitify2::reflection::Template("cudf::rolling::jit::gpu_rolling_new") // + jitify2::reflection::Template("cudf::rolling::jit::rolling_window_kernel") // .instantiate(cudf::type_to_name(input.type()), // list of template arguments cudf::type_to_name(output->type()), udf_agg._operator_name, - preceding_window_str.c_str(), - following_window_str.c_str()); + preceding_window_str, + following_window_str); cudf::jit::get_udf_kernel(*rolling_jit_kernel_cu_jit, kernel_reflection, cuda_source) ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()) @@ -103,5 +117,26 @@ std::unique_ptr rolling_window_udf(column_view const& input, return output; } +// Applies a user-defined rolling window function to the values in a column. +template +std::unique_ptr rolling_window_udf(column_view const& input, + PrecedingWindowIterator preceding_window, + FollowingWindowIterator following_window, + size_type min_periods, + rolling_aggregation const& agg, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + return rolling_window_udf_impl(input, + reflect_window_wrapper(), + preceding_window, + reflect_window_wrapper(), + following_window, + min_periods, + agg, + stream, + mr); +} + } // namespace detail } // namespace cudf diff --git a/cpp/src/rolling/detail/rolling_variable_window.cu b/cpp/src/rolling/detail/rolling_variable_window.cu index 155adf2d74a4..28a1b56abb3e 100644 --- a/cpp/src/rolling/detail/rolling_variable_window.cu +++ b/cpp/src/rolling/detail/rolling_variable_window.cu @@ -37,15 +37,14 @@ std::unique_ptr rolling_window(column_view const& input, if (agg.kind == aggregation::CUDA || agg.kind == aggregation::PTX) { // TODO: In future, might need to clamp preceding/following to column boundaries. - return cudf::detail::rolling_window_udf(input, - preceding_window.begin(), - "cudf::size_type*", - following_window.begin(), - "cudf::size_type*", - min_periods, - agg, - stream, - mr); + return cudf::detail::rolling_window_udf( + input, + cudf::detail::variable_window_wrapper{preceding_window.begin()}, + cudf::detail::variable_window_wrapper{following_window.begin()}, + min_periods, + agg, + stream, + mr); } else { auto defaults_col = cudf::is_dictionary(input.type()) ? dictionary_column_view(input).indices() : input; diff --git a/cpp/src/rolling/grouped_rolling.cu b/cpp/src/rolling/grouped_rolling.cu index 1b45758e7002..0676614cc385 100644 --- a/cpp/src/rolling/grouped_rolling.cu +++ b/cpp/src/rolling/grouped_rolling.cu @@ -98,15 +98,16 @@ std::unique_ptr grouped_rolling_window(table_view const& group_keys, cudf::detail::following_window_wrapper grouped_following_window{ group_offsets.data(), group_labels.data(), following_window}; - return cudf::detail::rolling_window_udf(input, - grouped_preceding_window, - "cudf::detail::preceding_window_wrapper", - grouped_following_window, - "cudf::detail::following_window_wrapper", - min_periods, - aggr, - stream, - mr); + return cudf::detail::rolling_window_udf( + input, + cudf::detail::preceding_window_wrapper{ + group_offsets.data(), group_labels.data(), preceding_window}, + cudf::detail::following_window_wrapper{ + group_offsets.data(), group_labels.data(), following_window}, + min_periods, + aggr, + stream, + mr); } else { namespace utils = cudf::detail::rolling; auto groups = utils::grouped{group_labels.data(), group_offsets.data()}; diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index 2a6e5ff7aa81..e13add4e326a 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -7,7 +7,7 @@ #include #include -#include +#include #pragma nv_hdrstop // The above headers are used by the kernel below and need to be included before // it. Each UDF will have a different operation-udf.hpp generated for it, so we @@ -39,42 +39,32 @@ namespace cudf { namespace rolling { namespace jit { -template -cudf::size_type __device__ get_window(WindowType window, cudf::thread_index_type index) -{ - return window[index]; -} - -template <> -cudf::size_type __device__ get_window(cudf::size_type window, cudf::thread_index_type index) -{ - return window; -} - template -CUDF_KERNEL void gpu_rolling_new(cudf::size_type nrows, - InType const* const __restrict__ in_col, - cudf::bitmask_type const* const __restrict__ in_col_valid, - OutType* __restrict__ out_col, - cudf::bitmask_type* __restrict__ out_col_valid, - cudf::size_type* __restrict__ output_valid_count, - PrecedingWindowType preceding_window_begin, - FollowingWindowType following_window_begin, - cudf::size_type min_periods) +CUDF_KERNEL void rolling_window_kernel(cudf::size_type nrows, + InType const* const __restrict__ in_col, + cudf::bitmask_type const* const __restrict__ in_col_valid, + OutType* __restrict__ out_col, + cudf::bitmask_type* __restrict__ out_col_valid, + cudf::size_type* __restrict__ output_valid_count, + detail::window_wrapper_base b_preceding_window_begin, + detail::window_wrapper_base b_following_window_begin, + cudf::size_type min_periods) { - auto i = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); + auto i = cudf::detail::grid_1d::global_thread_id(); + auto const stride = cudf::detail::grid_1d::grid_stride(); + PrecedingWindowType const preceding_window_begin = b_preceding_window_begin; + FollowingWindowType const following_window_begin = b_following_window_begin; cudf::size_type warp_valid_count{0}; auto active_threads = __ballot_sync(0xffff'ffffu, i < nrows); while (i < nrows) { - int64_t const preceding_window = get_window(preceding_window_begin, i); - int64_t const following_window = get_window(following_window_begin, i); + int64_t const preceding_window = preceding_window_begin[i]; + int64_t const following_window = following_window_begin[i]; // compute bounds auto const start = static_cast( @@ -112,7 +102,7 @@ CUDF_KERNEL void gpu_rolling_new(cudf::size_type nrows, } // TODO: likely faster to do a single_lane_block_reduce and a single - // atomic per block but that requires jitifying single_lane_block_reduce... + // atomic per block but that requires jit-compiling single_lane_block_reduce... if (0 == cudf::intra_word_index(threadIdx.x)) { atomicAdd(output_valid_count, warp_valid_count); } } diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index f95b9684034e..39d259dd2fb2 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -38,7 +38,7 @@ class context { void ensure_jit_cache_initialized(); public: - context(context_config const& cfg = {}, init_flags flags = init_flags::INIT_JIT_CACHE); + context(context_config const& cfg = {}, init_flags flags = init_flags::DEFAULT); context(context const&) = delete; context& operator=(context const&) = delete; context(context&&) = delete; diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index 9c342a8f06c7..d7c476e35b97 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -671,7 +671,7 @@ TEST_F(StringOperationTest, Output) std::string cuda = R"***( __device__ void transform(cudf::string_view * out, cudf::string_view a, cudf::string_view b, cudf::string_view c, cudf::string_view d){ - *out = std::max(std::max(std::max(a, b), c), d); + *out = cuda::std::max(cuda::std::max(cuda::std::max(a, b), c), d); } )***"; From 91944104453eea8a2b48ec48b55a06086318e8f9 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 22 May 2026 17:16:56 +0000 Subject: [PATCH 191/254] initial jitify replacement diff --- cpp/CMakeLists.txt | 53 +-- .../Modules/JitifyPreprocessKernels.cmake | 82 ---- cpp/cmake/thirdparty/get_jitify.cmake | 25 -- .../cudf/detail/utilities/getenv_or.hpp | 12 + cpp/src/binaryop/binaryop.cpp | 32 +- cpp/src/binaryop/jit/kernel.cu | 47 +- cpp/src/jit/cache.cpp | 416 +++++++++++++----- cpp/src/jit/cache.hpp | 112 +++-- cpp/src/jit/helpers.cpp | 44 +- cpp/src/jit/helpers.hpp | 7 +- cpp/src/join/filter_join_indices_jit.cu | 84 ++-- cpp/src/join/jit/filter_join_kernel.cu | 24 +- .../rolling/detail/rolling_fixed_window.cu | 2 +- cpp/src/rolling/detail/rolling_udf.cuh | 42 +- .../rolling/detail/rolling_variable_window.cu | 2 +- cpp/src/rolling/jit/kernel.cu | 41 +- cpp/src/runtime/context.cpp | 103 ++++- cpp/src/runtime/context.hpp | 38 +- cpp/src/transform/jit/kernel.cu | 27 +- cpp/src/transform/transform.cu | 60 ++- 20 files changed, 744 insertions(+), 509 deletions(-) delete mode 100644 cpp/cmake/Modules/JitifyPreprocessKernels.cmake delete mode 100644 cpp/cmake/thirdparty/get_jitify.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 63c292646773..ac905e35117f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -20,7 +20,7 @@ rapids_cuda_init_architectures(CUDF) project( CUDF VERSION "${RAPIDS_VERSION}" - LANGUAGES C CXX CUDA + LANGUAGES C CXX CUDA ASM ) if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 12.2) @@ -38,7 +38,6 @@ option(USE_NVTX "Build with NVTX support" ON) option(BUILD_TESTS "Configure CMake to build tests" ON) option(BUILD_BENCHMARKS "Configure CMake to build benchmarks" OFF) option(BUILD_SHARED_LIBS "Build cuDF shared libraries" ON) -option(JITIFY_USE_CACHE "Use a file cache for JIT compiled kernels" ON) option(CUDF_BUILD_TESTUTIL "Whether to build the test utilities contained in libcudf" ON) mark_as_advanced(CUDF_BUILD_TESTUTIL) option(CUDF_LARGE_STRINGS_DISABLED "Build with large string support disabled" OFF) @@ -110,7 +109,6 @@ message(VERBOSE "CUDF: Build with NVTX support: ${USE_NVTX}") message(VERBOSE "CUDF: Configure CMake to build tests: ${BUILD_TESTS}") message(VERBOSE "CUDF: Configure CMake to build benchmarks: ${BUILD_BENCHMARKS}") message(VERBOSE "CUDF: Build cuDF shared libraries: ${BUILD_SHARED_LIBS}") -message(VERBOSE "CUDF: Use a file cache for JIT compiled kernels: ${JITIFY_USE_CACHE}") message(VERBOSE "CUDF: Build with per-thread default stream: ${CUDF_USE_PER_THREAD_DEFAULT_STREAM}") message( VERBOSE @@ -345,9 +343,6 @@ endif() create_logger_macros(CUDF "cudf::default_logger()" include/cudf) -# find jitify -include(cmake/thirdparty/get_jitify.cmake) - # find NVTX include(cmake/thirdparty/get_nvtx.cmake) @@ -377,9 +372,6 @@ if(CUDF_BUILD_TESTUTIL) include(cmake/thirdparty/get_gtest.cmake) endif() -# preprocess jitify-able kernels -include(cmake/Modules/JitifyPreprocessKernels.cmake) - # find KvikIO include(cmake/thirdparty/get_kvikio.cmake) @@ -392,6 +384,9 @@ include(cmake/thirdparty/get_thread_pool.cmake) # find zstd include(cmake/thirdparty/get_zstd.cmake) +# JIT Embedding helper functions +include(librtcx/embed.cmake) + # Workaround until https://github.com/rapidsai/rapids-cmake/issues/176 is resolved if(NOT BUILD_SHARED_LIBS) include("${rapids-cmake-dir}/export/find_package_file.cmake") @@ -667,11 +662,11 @@ add_library( src/io/utilities/row_selection.cpp src/io/utilities/type_inference.cu src/io/utilities/trie.cu - src/jit/cache.cpp src/jit/helpers.cpp src/jit/parser.cpp src/jit/row_ir.cpp src/jit/util.cpp + src/jit/jit.cpp src/join/conditional_join.cu src/join/cross_join.cu src/join/distinct_hash_join.cu @@ -963,20 +958,11 @@ add_library( src/utilities/traits.cpp src/utilities/type_checks.cpp src/utilities/type_dispatcher.cpp + librtcx/rtcx.cpp + ${cudf_jit_embed_SOURCE_DIR}/cudf_jit_embed.s ) -# Anything that includes jitify needs to be compiled with _FILE_OFFSET_BITS=64 due to a limitation -# in how conda builds glibc -set_source_files_properties( - src/binaryop/binaryop.cpp - src/jit/cache.cpp - src/rolling/detail/rolling_fixed_window.cu - src/rolling/detail/rolling_variable_window.cu - src/rolling/grouped_rolling.cu - src/rolling/rolling.cu - src/transform/transform.cu - PROPERTIES COMPILE_DEFINITIONS "_FILE_OFFSET_BITS=64" -) +add_dependencies(cudf cudf_jit_embed cudf_fragments) set_property( SOURCE src/io/parquet/writer_impl.cu @@ -1040,14 +1026,14 @@ target_compile_options( # Specify include paths for the current target and dependents target_include_directories( cudf - PUBLIC "$" - "$" - "$" + PUBLIC "$" "$" "$" PRIVATE "$" "$" "$" "$" + "$" + "$" INTERFACE "$" ) @@ -1056,20 +1042,6 @@ target_compile_definitions( "$:${CUDF_CUDA_DEFINITIONS}>>" ) -# Disable Jitify log printing. See https://github.com/NVIDIA/jitify/issues/79 -target_compile_definitions(cudf PRIVATE "JITIFY_PRINT_LOG=0") - -if(JITIFY_USE_CACHE) - # Instruct src/jit/cache what version of cudf we are building so it can compute a cal-ver cache - # directory. We isolate this definition to the single source so it doesn't effect compiling - # caching for all of libcudf - set_property( - SOURCE src/jit/cache.cpp - APPEND - PROPERTY COMPILE_DEFINITIONS "JITIFY_USE_CACHE" "CUDF_VERSION=${PROJECT_VERSION}" - ) -endif() - # Per-thread default stream if(CUDF_USE_PER_THREAD_DEFAULT_STREAM) target_compile_definitions( @@ -1099,9 +1071,6 @@ target_compile_definitions(cudf PRIVATE $<$:CUDF_ # https://github.com/NVIDIA/cccl/pull/2844 target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_AVOID_SORT_UNROLL=1) -# Compile stringified JIT sources first -add_dependencies(cudf jitify_preprocess_run) - # Specify the target module library dependencies target_link_libraries( cudf diff --git a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake deleted file mode 100644 index d035e1ea6ab4..000000000000 --- a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake +++ /dev/null @@ -1,82 +0,0 @@ -# ============================================================================= -# cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# cmake-format: on -# ============================================================================= - -# Create `jitify_preprocess` executable -add_executable(jitify_preprocess "${JITIFY_INCLUDE_DIR}/jitify2_preprocess.cpp") - -target_compile_definitions(jitify_preprocess PRIVATE "_FILE_OFFSET_BITS=64") -rapids_cuda_set_runtime(jitify_preprocess USE_STATIC ON) -target_link_libraries(jitify_preprocess PUBLIC ${CMAKE_DL_LIBS}) - -# Take a list of files to JIT-compile and run them through jitify_preprocess. -function(jit_preprocess_files) - cmake_parse_arguments(ARG "" "SOURCE_DIRECTORY" "FILES" ${ARGN}) - - get_target_property(libcudacxx_raw_includes CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) - set(includes) - foreach(inc IN LISTS libcudacxx_raw_includes CUDAToolkit_INCLUDE_DIRS) - list(APPEND includes "-I${inc}") - endforeach() - foreach(ARG_FILE ${ARG_FILES}) - set(ARG_OUTPUT ${CUDF_GENERATED_INCLUDE_DIR}/include/jit_preprocessed_files/${ARG_FILE}.jit.hpp) - get_filename_component(jit_output_directory "${ARG_OUTPUT}" DIRECTORY) - list(APPEND JIT_PREPROCESSED_FILES "${ARG_OUTPUT}") - - get_filename_component(ARG_OUTPUT_DIR "${ARG_OUTPUT}" DIRECTORY) - - # Note: need to pass _FILE_OFFSET_BITS=64 in COMMAND due to a limitation in how conda builds - # glibc - add_custom_command( - OUTPUT ${ARG_OUTPUT} - DEPENDS jitify_preprocess "${ARG_SOURCE_DIRECTORY}/${ARG_FILE}" - WORKING_DIRECTORY ${ARG_SOURCE_DIRECTORY} - VERBATIM - COMMAND ${CMAKE_COMMAND} -E make_directory "${jit_output_directory}" - COMMAND - "${CMAKE_COMMAND}" -E env LD_LIBRARY_PATH=${CUDAToolkit_LIBRARY_DIR} - $ ${ARG_FILE} -o ${ARG_OUTPUT_DIR} -i -std=c++20 - -remove-unused-globals -D_FILE_OFFSET_BITS=64 -D__CUDACC_RTC__ -DCUDF_RUNTIME_JIT - -I${CUDF_SOURCE_DIR}/include -I${CUDF_SOURCE_DIR}/src ${includes} - --no-preinclude-workarounds --no-replace-pragma-once --diag-suppress=47 --device-int128 - COMMENT "Custom command to JIT-compile files." - ) - endforeach() - set(JIT_PREPROCESSED_FILES - "${JIT_PREPROCESSED_FILES}" - PARENT_SCOPE - ) -endfunction() - -if(NOT (EXISTS "${CUDF_GENERATED_INCLUDE_DIR}/include")) - make_directory("${CUDF_GENERATED_INCLUDE_DIR}/include") -endif() - -jit_preprocess_files( - SOURCE_DIRECTORY ${CUDF_SOURCE_DIR}/src FILES binaryop/jit/kernel.cu rolling/jit/kernel.cu - transform/jit/kernel.cu join/jit/filter_join_kernel.cu -) - -add_custom_target( - jitify_preprocess_run - DEPENDS ${JIT_PREPROCESSED_FILES} - COMMENT "Target representing jitified files." -) - -# when a user requests CMake to clean the build directory -# -# * `cmake --build --target clean` -# * `cmake --build --clean-first` -# * ninja clean -# -# We also remove the jitify2 program cache as well. This ensures that we don't keep older versions -# of the programs in cache -set(cache_path "$ENV{HOME}/.cudf") -if(ENV{LIBCUDF_KERNEL_CACHE_PATH}) - set(cache_path "$ENV{LIBCUDF_KERNEL_CACHE_PATH}") -endif() -cmake_path(APPEND cache_path "${CUDF_VERSION}/") -set_target_properties(jitify_preprocess_run PROPERTIES ADDITIONAL_CLEAN_FILES "${cache_path}") diff --git a/cpp/cmake/thirdparty/get_jitify.cmake b/cpp/cmake/thirdparty/get_jitify.cmake deleted file mode 100644 index 4595b33c5160..000000000000 --- a/cpp/cmake/thirdparty/get_jitify.cmake +++ /dev/null @@ -1,25 +0,0 @@ -# ============================================================================= -# cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# cmake-format: on -# ============================================================================= - -# Jitify doesn't have a version :/ - -# This function finds Jitify and sets any additional necessary environment variables. -function(find_and_configure_jitify) - rapids_cpm_find( - jitify 2.0.0 - GIT_REPOSITORY https://github.com/NVIDIA/jitify.git - GIT_TAG 44e978b21fc8bdb6b2d7d8d179523c8350db72e5 # jitify2 branch as of 23rd Aug 2025 - GIT_SHALLOW FALSE - DOWNLOAD_ONLY TRUE - ) - set(JITIFY_INCLUDE_DIR - "${jitify_SOURCE_DIR}" - PARENT_SCOPE - ) -endfunction() - -find_and_configure_jitify() diff --git a/cpp/include/cudf/detail/utilities/getenv_or.hpp b/cpp/include/cudf/detail/utilities/getenv_or.hpp index 5003a11df084..ae57ec482c54 100644 --- a/cpp/include/cudf/detail/utilities/getenv_or.hpp +++ b/cpp/include/cudf/detail/utilities/getenv_or.hpp @@ -43,6 +43,18 @@ T getenv_or(std::string_view env_var_name, T default_val) return converted_val; } +template +std::optional getenv_optional(std::string_view env_var_name) +{ + auto const env_val = std::getenv(env_var_name.data()); + if (env_val == nullptr) { return std::nullopt; } + + std::stringstream sstream(env_val); + T converted_val; + sstream >> converted_val; + return converted_val; +} + /** * @brief Specialization of getenv_or for bool, to allow common "ON"/"OFF" string values. */ diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index bb4a48629ec5..167621d3f7fc 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -43,8 +43,6 @@ #include -#include - #include namespace cudf { @@ -156,18 +154,24 @@ void binary_operation(mutable_column_view& out, {2, cudf::type_to_name(rhs.type())}, }); - std::string kernel_reflection = jitify2::reflection::Template("cudf::binops::jit::kernel_v_v") - .instantiate(output_type_name, // list of template arguments - cudf::type_to_name(lhs.type()), - cudf::type_to_name(rhs.type()), - std::string("cudf::binops::jit::UserDefinedOp")); - - cudf::jit::get_udf_kernel(*binaryop_jit_kernel_cu_jit, kernel_reflection, cuda_source) - ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()) - ->launch(out.size(), - cudf::jit::get_data_ptr(out), - cudf::jit::get_data_ptr(lhs), - cudf::jit::get_data_ptr(rhs)); + std::string kernel_reflection = rtcx::reflect_template("cudf::binops::jit::kernel_v_v", + output_type_name, + cudf::type_to_name(lhs.type()), + cudf::type_to_name(rhs.type()), + "cudf::binops::jit::UserDefinedOp"); + + auto kernel = + cudf::jit::get_udf_kernel(*binaryop_jit_kernel_cu_jit, kernel_reflection, cuda_source); + auto cfg = kernel.max_occupancy_config(0, 0); + + kernel.launch_with({cfg.min_grid_size}, + {cfg.block_size}, + 0, + stream, + static_cast(out.size()), + cudf::jit::get_data_ptr(out), + cudf::jit::get_data_ptr(lhs), + cudf::jit::get_data_ptr(rhs)); } } // namespace jit diff --git a/cpp/src/binaryop/jit/kernel.cu b/cpp/src/binaryop/jit/kernel.cu index 893c1866cb44..9baadd31d8cb 100644 --- a/cpp/src/binaryop/jit/kernel.cu +++ b/cpp/src/binaryop/jit/kernel.cu @@ -37,6 +37,7 @@ // need to put this pragma before including it to avoid PCH mismatch. // clang-format off +#include #include // clang-format on @@ -56,45 +57,31 @@ struct UserDefinedOp { }; template -CUDF_KERNEL void kernel_v_v(cudf::size_type size, - TypeOut* out_data, - TypeLhs* lhs_data, - TypeRhs* rhs_data) +__device__ void binary_op_kernel(cudf::size_type size, + void* p_out_data, + void* p_lhs_data, + void* p_rhs_data) { auto const start = cudf::detail::grid_1d::global_thread_id(); auto const step = cudf::detail::grid_1d::grid_stride(); - for (auto i = start; i < size; i += step) { - out_data[i] = TypeOpe::template operate(lhs_data[i], rhs_data[i]); - } -} - -template -CUDF_KERNEL void kernel_v_v_with_validity(cudf::size_type size, - TypeOut* out_data, - TypeLhs* lhs_data, - TypeRhs* rhs_data, - cudf::bitmask_type* output_mask, - cudf::bitmask_type const* lhs_mask, - cudf::size_type lhs_offset, - cudf::bitmask_type const* rhs_mask, - cudf::size_type rhs_offset) -{ - auto const start = cudf::detail::grid_1d::global_thread_id(); - auto const step = cudf::detail::grid_1d::grid_stride(); + auto* out_data = static_cast(p_out_data); + auto* lhs_data = static_cast(p_lhs_data); + auto* rhs_data = static_cast(p_rhs_data); for (auto i = start; i < size; i += step) { - bool output_valid = false; - out_data[i] = TypeOpe::template operate( - lhs_data[i], - rhs_data[i], - lhs_mask ? cudf::bit_is_set(lhs_mask, lhs_offset + i) : true, - rhs_mask ? cudf::bit_is_set(rhs_mask, rhs_offset + i) : true, - output_valid); - if (output_mask && !output_valid) cudf::clear_bit(output_mask, i); + out_data[i] = TypeOpe::template operate(lhs_data[i], rhs_data[i]); } } } // namespace jit } // namespace binops } // namespace cudf + +extern "C" __global__ void cudf_kernel_entry(cudf::size_type size, + void* out_data, + void* lhs_data, + void* rhs_data) +{ + CUDF_KERNEL_INSTANCE(size, out_data, lhs_data, rhs_data); +} diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index a2fb3d10fd51..2c8a2cd86bdf 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -1,162 +1,358 @@ + /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ -#include "runtime/context.hpp" - -#include #include -#include #include +#include + +#include #include +#include +#include #include +#include +#include +#include + +namespace CUDF_EXPORT cudf { -namespace cudf { namespace { -// Get the directory in home to use for storing the cache -std::filesystem::path get_user_home_cache_dir() +rtcx::sha256 hash(std::span input) { - auto home_dir = std::getenv("HOME"); - if (home_dir != nullptr) { - return std::filesystem::path(home_dir) / ".cudf"; - } else { - return {}; + rtcx::sha256_context ctx; + ctx.update(std::span{reinterpret_cast(input.data()), input.size()}); + return ctx.finalize(); +} + +rtcx::sha256 hash(std::span inputs) +{ + rtcx::sha256_context ctx; + for (auto const* input : inputs) { + ctx.update(std::span{reinterpret_cast(input), std::strlen(input)}); } + return ctx.finalize(); } -// Default `LIBCUDF_KERNEL_CACHE_PATH` to `$HOME/.cudf/$CUDF_VERSION`. -// This definition can be overridden at compile time by specifying a -// `-DLIBCUDF_KERNEL_CACHE_PATH=/kernel/cache/path` CMake argument. -// Use `std::filesystem` for cross-platform path resolution and dir -// creation. This path is used in the `getCacheDir()` function below. -#if !defined(LIBCUDF_KERNEL_CACHE_PATH) -#define LIBCUDF_KERNEL_CACHE_PATH get_user_home_cache_dir() -#endif - -/** - * @brief Get the string path to the JITIFY kernel cache directory. - * - * This path can be overridden at runtime by defining an environment variable - * named `LIBCUDF_KERNEL_CACHE_PATH`. The value of this variable must be a path - * under which the process' user has read/write privileges. - * - * This function returns a path to the cache directory, creating it if it - * doesn't exist. - * - * The default cache directory is `$HOME/.cudf/$CUDF_VERSION`. If no overrides - * are used and if $HOME is not defined, returns an empty path and file - * caching is not used. - */ -std::filesystem::path get_cache_dir() -{ - // The environment variable always overrides the - // default/compile-time value of `LIBCUDF_KERNEL_CACHE_PATH` - auto kernel_cache_path_env = std::getenv("LIBCUDF_KERNEL_CACHE_PATH"); - auto kernel_cache_path = std::filesystem::path( - kernel_cache_path_env != nullptr ? kernel_cache_path_env : LIBCUDF_KERNEL_CACHE_PATH); - - // Cache path could be empty when env HOME is unset or LIBCUDF_KERNEL_CACHE_PATH is defined to be - // empty, to disallow use of file cache at runtime. - if (not kernel_cache_path.empty()) { - kernel_cache_path /= std::string{CUDF_STRINGIFY(CUDF_VERSION)}; - - // Make per device cache based on compute capability. This is to avoid multiple devices of - // different compute capability to access the same kernel cache. - int device = 0; - int cc_major = 0; - int cc_minor = 0; - CUDF_CUDA_TRY(cudaGetDevice(&device)); - CUDF_CUDA_TRY(cudaDeviceGetAttribute(&cc_major, cudaDevAttrComputeCapabilityMajor, device)); - CUDF_CUDA_TRY(cudaDeviceGetAttribute(&cc_minor, cudaDevAttrComputeCapabilityMinor, device)); - int const cc = cc_major * 10 + cc_minor; - - kernel_cache_path /= std::to_string(cc); - - try { - // `mkdir -p` the kernel cache path if it doesn't exist - std::filesystem::create_directories(kernel_cache_path); - } catch (std::exception const& e) { - // if directory creation fails for any reason, return empty path - return {}; +rtcx::sha256 hash(std::span file_fragments, + std::span memory_fragments) +{ + rtcx::sha256_context ctx; + for (auto const& fragment : file_fragments) { + ctx.update( + std::span{reinterpret_cast(fragment.path), std::strlen(fragment.path)}); + } + + for (auto const& fragment : memory_fragments) { + ctx.update(fragment.data); + } + + return ctx.finalize(); +} + +void install_file_set(std::string_view target_dir, + std::span compressed_binary, + size_t uncompressed_size, + std::span file_ranges, + std::span destinations, + std::string_view compression) +{ + auto decompressed = rtcx::decompress_blob(compressed_binary, uncompressed_size, compression); + for (size_t i = 0; i < file_ranges.size(); ++i) { + auto file_data_range = file_ranges[i]; + auto file_data = std::span{decompressed.data() + file_data_range[0], file_data_range[1]}; + auto dst_path = destinations[i]; + auto target_path = std::format("{}/{}", target_dir, dst_path); + + std::filesystem::create_directories(std::filesystem::path{target_path}.parent_path()); + + std::ofstream file(target_path, std::ios::binary); + if (!file) { + throw std::runtime_error( + std::format("Failed to open file for writing at path: {}", target_path)); + } + + file.write(reinterpret_cast(file_data.data()), file_data.size()); + if (!file) { + throw std::runtime_error(std::format("Failed to write file at path: {}", target_path)); } } - return kernel_cache_path; } -std::string get_program_cache_dir() +std::string read_file_string(char const* path) { -#if defined(JITIFY_USE_CACHE) - return get_cache_dir().string(); -#else - return {}; -#endif + std::ifstream file(std::string{path}, std::ios::binary | std::ios::ate); + if (!file) { throw std::runtime_error(std::format("Failed to open file at path: {}", path)); } + + auto size = file.tellg(); + file.seekg(0, std::ios::beg); + + std::string contents(size, '\0'); + if (!file.read(contents.data(), size)) { + throw std::runtime_error(std::format("Failed to read file at path: {}", path)); + } + + return contents; +} + +void install_cudf_jit_files(std::string const& target_dir, std::string const& tmp_dir) +{ + // directory does not exist, so create it + auto tmp_dir_path_str = std::format("{}/cudf-jit-tmpdir_XXXXXX", tmp_dir); + char* tmp_dir_path = ::mkdtemp(tmp_dir_path_str.data()); + CUDF_EXPECTS( + tmp_dir_path != nullptr, + std::format("Failed to create temporary directory for JIT file installation in tmp dir: {}", + tmp_dir), + std::runtime_error); + + install_file_set(tmp_dir_path, + cudf_jit_embed::files, + cudf_jit_embed::files_uncompressed_size, + cudf_jit_embed::file_ranges, + cudf_jit_embed::file_destinations, + cudf_jit_embed::files_compression); + + // rename the temporary directory to the target install directory + if (::rename(tmp_dir_path, target_dir.c_str()) == -1) { + auto errc = errno; + // another process created it + if (errc == ENOTEMPTY || errc == EEXIST) { + std::filesystem::remove_all(tmp_dir_path); + } else { + CUDF_FAIL( + std::format("Failed to install JIT files to target directory: {} with error ({}): {}", + target_dir, + errc, + std::strerror(errc)), + std::runtime_error); + } + } } } // namespace -jitify2::ProgramCache<>& jit::program_cache::get(jitify2::PreprocessedProgramData const& preprog) +jit_bundle_t::jit_bundle_t(std::string install_dir, rtcx::cache_t& cache) + : install_dir_{std::move(install_dir)}, cache_{&cache} +{ + ensure_installed(); +} + +void jit_bundle_t::ensure_installed() const { CUDF_FUNC_RANGE(); - std::lock_guard caches_lock(_caches_mutex); - auto existing_cache = _caches.find(preprog.name()); + auto expected_hash = get_hash(); + auto expected_path = std::format("{}/{}", install_dir_, expected_hash); - if (existing_cache == _caches.end() || _disabled.load(std::memory_order_seq_cst)) { - auto res = - _caches.emplace(preprog.name(), - std::make_unique>( - _kernel_limit_proc, preprog, nullptr, _cache_dir, _kernel_limit_disk)); - existing_cache = res.first; + if (!std::filesystem::exists(expected_path)) { + // ensure base install directory exists + std::filesystem::create_directories(install_dir_); + install_cudf_jit_files(expected_path.c_str(), cache_->get_tmp_dir()); + } else { + // directory exists, perform minor sanity check + CUDF_EXPECTS(std::filesystem::is_directory(expected_path), // throws if path does not exist + std::format("JIT install path ({}) exists but is not a directory", expected_path), + std::runtime_error); } +} + +std::string jit_bundle_t::get_hash() const +{ + auto str = rtcx::sha256_hex_string::make(cudf_jit_embed::hash); + return std::string{str.view()}; +} - return *(existing_cache->second); +std::string jit_bundle_t::get_directory() const +{ + return std::format("{}/{}", install_dir_, get_hash()); } -void jit::program_cache::clear() +std::vector jit_bundle_t::get_include_directories() const { - CUDF_FUNC_RANGE(); - std::lock_guard caches_lock(_caches_mutex); + std::vector directories; + auto base_dir = get_directory(); - _caches.clear(); + for (auto dir : cudf_jit_embed::include_directories) { + directories.emplace_back(std::format("{}/{}", base_dir, dir)); + } - // non-atomic - std::filesystem::remove_all(_cache_dir); + return directories; } -void jit::program_cache::enable(bool enable) +namespace { + +constexpr int MIN_CUDA_VERSION_PCH = 12800; // CUDA 12.8 +constexpr int MIN_CUDA_VERSION_MINIMAL = 12800; // CUDA 12.8 + +int32_t get_driver_version() { - _disabled.store(!enable, std::memory_order_seq_cst); + int32_t driver_version; + CUDF_CUDA_TRY(cudaDriverGetVersion(&driver_version)); + return driver_version; } -bool jit::program_cache::is_enabled() const { return !_disabled.load(std::memory_order_seq_cst); } +int32_t get_runtime_version() +{ + int32_t runtime_version; + CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); + return runtime_version; +} -std::unique_ptr jit::program_cache::create() +int32_t get_current_device_physical_model() { - auto const kernel_limit_proc = - cudf::detail::getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 10'000); - auto const kernel_limit_disk = - cudf::detail::getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_DISK", 100'000); - auto const disabled = cudf::detail::get_bool_env_or("LIBCUDF_KERNEL_CACHE_DISABLED", false); - auto const clear_cache = cudf::detail::get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); + int32_t device; + CUDF_CUDA_TRY(cudaGetDevice(&device)); + + cudaDeviceProp props; + CUDF_CUDA_TRY(cudaGetDeviceProperties(&props, device)); + + return props.major * 10 + props.minor; +} + +std::tuple compile_library_uncached( + char const* name, + char const* cuda_code, + std::span extra_header_include_names, + std::span extra_headers, + std::span name_expressions) +{ + CUDF_FUNC_RANGE(); + + auto& ctx = cudf::get_context(); + auto& cfg = ctx.config(); + auto& bundle = ctx.jit_bundle(); + auto sm = get_current_device_physical_model(); + auto runtime = get_runtime_version(); + + auto include_dirs = bundle.get_include_directories(); + auto pch_dir = ctx.get_jit_pch_dir(); + + auto use_pch = runtime >= MIN_CUDA_VERSION_PCH; + auto use_minimal = runtime >= MIN_CUDA_VERSION_MINIMAL; + + std::vector options; + + for (auto const& include_dir : include_dirs) { + options.emplace_back(std::format("-I{}", include_dir)); + } + + options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); + + options.emplace_back("--diag-suppress=47"); + options.emplace_back("--device-int128"); + + if (sm >= 100) { options.emplace_back("--device-float128"); } + + options.emplace_back("-std=c++20"); + options.emplace_back("--device-as-default-execution-space"); + options.emplace_back("--generate-line-info"); + options.emplace_back("--dopt=on"); - // if kernel_limit_disk is zero, jitify will assign it the value of kernel_limit_proc. - // to avoid this, we treat zero as "disable disk caching" by not providing the cache dir. - auto cache_dir = kernel_limit_disk == 0 ? std::string{} : get_program_cache_dir(); + if (use_minimal) { options.emplace_back("--minimal"); } - auto cache = - std::make_unique(kernel_limit_proc, kernel_limit_disk, cache_dir, disabled); + if (use_pch) { + options.emplace_back("--pch"); - if (clear_cache) { cache->clear(); } + if (cfg.jit_verbose) { + options.emplace_back("--pch-verbose=true"); + options.emplace_back("--pch-messages=true"); + } else { + options.emplace_back("--pch-verbose=false"); + options.emplace_back("--pch-messages=false"); + } + } + + if (cfg.disable_cuda_cache) { options.emplace_back("--no-cache"); } + + if (cfg.dump_jit_trace) { options.emplace_back("--time=-"); } + + if (cfg.dump_jit_time_profile) { + options.emplace_back(std::format("--fdevice-time-trace=cudf_kernel_{}_trace", name)); + } - return cache; + std::vector options_cstr; + for (auto const& option : options) { + options_cstr.emplace_back(option.c_str()); + } + + auto params = rtcx::compile_params{.name = name, + .source = cuda_code, + .header_include_names = extra_header_include_names, + .headers = extra_headers, + .options = options_cstr, + .name_expressions = name_expressions, + .target_type = rtcx::binary_type::CUBIN}; + + auto cubin = rtcx::compile(params); + auto library = rtcx::load_library(cubin); + auto blob = rtcx::blob_t::from_buffer(std::move(cubin)); + + return std::make_tuple(library, std::make_shared(std::move(blob))); } -jitify2::ProgramCache<>& jit::get_program_cache(jitify2::PreprocessedProgramData const& preprog) +} // namespace + +kernel get_kernel(std::string const& name, + std::string const& source_file_id, + std::span header_include_names, + std::span headers, + std::string const& kernel_instance) { - return cudf::get_context().program_cache().get(preprog); + CUDF_FUNC_RANGE(); + + auto& cache = cudf::get_context().rtcx_cache(); + auto& bundle = cudf::get_context().jit_bundle(); + + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_physical_model(); + auto header_include_names_hash = hash(header_include_names).to_hex_string(); + auto headers_hash = hash(headers).to_hex_string(); + auto bundle_hash = bundle.get_hash(); + auto source_file = std::format("{}/cudf/cpp/src/{}", bundle.get_directory(), source_file_id); + + auto cache_key = std::format(R"***(cuLibrary +name={} +binary_type=CUBIN +cuda_runtime={} +cuda_driver={} +arch={} +bundle={} +source_file={} +header_include_names={} +headers={} +kernel_instance={} +)***", + name, + runtime, + driver, + sm, + bundle_hash, + source_file, + header_include_names_hash.view(), + headers_hash.view(), + kernel_instance); + + auto cache_key_sha256 = hash(cache_key); + + auto compile = [&] { + auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); + auto source = read_file_string(source_file.c_str()); + return compile_library_uncached( + name.c_str(), source.c_str(), header_include_names, headers, {}); + }; + + auto fut = + cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); + + auto lib = fut.get(); + return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; } -} // namespace cudf + +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/cache.hpp b/cpp/src/jit/cache.hpp index 0e6738fed176..679f8011fa3e 100644 --- a/cpp/src/jit/cache.hpp +++ b/cpp/src/jit/cache.hpp @@ -1,63 +1,91 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#pragma GCC diagnostic ignored "-Wignored-attributes" // Work-around for JITIFY2's false-positive - // warnings when compiled with GCC13 - #include -#include +#include + +#include -#include -#include -#include -#include -#include +namespace CUDF_EXPORT cudf { -namespace cudf { -namespace jit { +struct [[nodiscard]] jit_bundle_t { + private: + std::string install_dir_; + rtcx::cache_t* cache_; -class program_cache { - std::mutex _caches_mutex; - std::unordered_map>> _caches; - int32_t _kernel_limit_proc; - int32_t _kernel_limit_disk; - std::filesystem::path _cache_dir; - std::atomic _disabled; + void ensure_installed() const; public: - program_cache(int32_t kernel_limit_proc, - int32_t kernel_limit_disk, - std::filesystem::path cache_dir, - bool disabled) - : _kernel_limit_proc{kernel_limit_proc}, - _kernel_limit_disk{kernel_limit_disk}, - _cache_dir{std::move(cache_dir)}, - _disabled{disabled} - { - } + jit_bundle_t(std::string install_dir, rtcx::cache_t& cache); + + [[nodiscard]] std::string get_hash() const; + + [[nodiscard]] std::string get_directory() const; + + [[nodiscard]] std::vector get_include_directories() const; +}; - program_cache(program_cache const&) = delete; - program_cache(program_cache&&) = delete; - program_cache& operator=(program_cache const&) = delete; - program_cache& operator=(program_cache&&) = delete; - ~program_cache() = default; +struct [[nodiscard]] kernel { + private: + rtcx::library _library; + rtcx::kernel_ref _kernel; - jitify2::ProgramCache<>& get(jitify2::PreprocessedProgramData const& preprog); + public: + kernel(rtcx::library lib, rtcx::kernel_ref kernel) : _library(std::move(lib)), _kernel(kernel) {}; + kernel(kernel const&) = default; + kernel(kernel&&) = default; + kernel& operator=(kernel const&) = default; + kernel& operator=(kernel&&) = default; + ~kernel() = default; - void clear(); + rtcx::kernel_ref get() const { return _kernel; } - void enable(bool enable); + rtcx::kernel_occupancy_config max_occupancy_config(size_t dynamic_shared_memory_bytes, + int32_t block_size_limit) const + { + return _kernel.max_occupancy_config(dynamic_shared_memory_bytes, block_size_limit); + } - bool is_enabled() const; + void launch(rtcx::cuda_dim3 grid_dim, + rtcx::cuda_dim3 block_dim, + uint32_t shared_mem_bytes, + rmm::cuda_stream_view stream, + void** kernel_params) const + { + return _kernel.launch(grid_dim, block_dim, shared_mem_bytes, stream.value(), kernel_params); + } - static std::unique_ptr create(); + template + void launch_with(rtcx::cuda_dim3 grid_dim, + rtcx::cuda_dim3 block_dim, + uint32_t shared_mem_bytes, + rmm::cuda_stream_view stream, + Args&&... args) + { + void const* params[] = {&args...}; // NOLINT(modernize-avoid-c-arrays) + launch(grid_dim, block_dim, shared_mem_bytes, stream, const_cast(params)); + } }; -jitify2::ProgramCache<>& get_program_cache(jitify2::PreprocessedProgramData const& preprog); +/** + * @brief Gets a kernel from an embedded CUDA source file + * @param name Debug name for the kernel (used for caching and logging) + * @param source_file_id Identifier for the embedded source file (used to locate the source and for + * caching) + * @param header_include_names Names of any additional embedded header files to include during + * compilation + * @param headers Contents of any additional embedded header files to include during compilation + * @param kernel_instance String identifier for the specific kernel instance being requested (used + * for caching) + */ +kernel get_kernel(std::string const& name, + std::string const& source_file_id, + std::span header_include_names, + std::span headers, + std::string const& kernel_instance); -} // namespace jit -} // namespace cudf +} // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index d035f43efe28..bf83745d7df0 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -8,6 +8,8 @@ #include #include +#include +#include namespace cudf { namespace jit { @@ -61,7 +63,7 @@ std::map build_ptx_params(std::span ou if (has_user_data) { params.emplace(index++, "void *"); - params.emplace(index++, jitify2::reflection::reflect()); + params.emplace(index++, "cudf::size_type"); } for (auto& name : output_typenames) { @@ -87,32 +89,26 @@ std::vector input_type_names( return names; } -jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, - std::string const& kernel_name, - std::string const& cuda_source, - std::vector const& extra_options) +kernel get_udf_kernel(std::string const& source_file, + std::string const& kernel_name, + std::string const& cuda_source) { CUDF_FUNC_RANGE(); - int runtime_version; - CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); - - constexpr int min_pch_cuda_version = 12800; // CUDA 12.8 - constexpr int min_minimal_cuda_version = 12800; // CUDA 12.8 - - std::vector options; - options.emplace_back("-arch=sm_."); - - if (runtime_version >= min_minimal_cuda_version) { options.emplace_back("-minimal"); } - - if (runtime_version >= min_pch_cuda_version) { options.emplace_back("-pch"); } - - for (auto& opt : extra_options) { - options.push_back(opt); - } - - return cudf::jit::get_program_cache(preprocessed_program_data) - .get_kernel(kernel_name, {}, {{"cudf/detail/operation-udf.hpp", cuda_source}}, options); + auto kernel_instance_source = std::format(R"***( + #define CUDF_KERNEL_INSTANCE {} + )***", + kernel_name); + char const* include_names[] = // NOLINT(modernize-avoid-c-arrays) + {"cudf/detail/operation-udf.hpp", "cudf/detail/kernel-instance.hpp"}; + char const* include_headers[] = // NOLINT(modernize-avoid-c-arrays) + {udf_cuda_source.c_str(), kernel_instance_source.c_str()}; + + return get_kernel(std::format("{}.jit.cu", source_file), + source_file, + include_names, + include_headers, + kernel_name); } } // namespace jit diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index 4a3f27b048c1..d17ed93924d9 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -75,10 +75,9 @@ column_views_to_device(std::span views, std::vector input_type_names( std::span const> views); -jitify2::Kernel get_udf_kernel(jitify2::PreprocessedProgramData const& preprocessed_program_data, - std::string const& kernel_name, - std::string const& cuda_source, - std::vector const& extra_options = {}); +kernel get_udf_kernel(std::string const& kernel_name, + std::string const& cuda_source, + std::vector const& extra_options = {}); } // namespace jit } // namespace cudf diff --git a/cpp/src/join/filter_join_indices_jit.cu b/cpp/src/join/filter_join_indices_jit.cu index 15b532cac030..6c066032bf83 100644 --- a/cpp/src/join/filter_join_indices_jit.cu +++ b/cpp/src/join/filter_join_indices_jit.cu @@ -38,7 +38,6 @@ #include #include #include -#include #include #include @@ -48,55 +47,54 @@ namespace detail { namespace { -jitify2::StringVec build_join_filter_template_params( - std::span inputs, - std::span> table_sources, +std::vector build_join_filter_template_params( + std::vector const& left_columns, + std::vector const& right_columns, bool has_user_data, null_aware is_null_aware) { - jitify2::StringVec template_params; - template_params.emplace_back(jitify2::reflection::reflect(has_user_data)); - template_params.emplace_back(jitify2::reflection::reflect(is_null_aware)); + std::vector template_params; + template_params.emplace_back(rtcx::reflect(has_user_data)); + template_params.emplace_back(rtcx::reflect(is_null_aware)); - jitify2::StringVec accessors; + std::vector accessors; for (size_t i = 0; i < inputs.size(); ++i) { auto const& input = inputs[i]; if (auto* col = std::get_if(&input)) { - auto element = cudf::type_to_name(col->type()); - accessors.emplace_back( - jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate( - i, "cudf::column_device_view_core", element, false, table_sources[i].value())); + accessors.emplace_back(rtcx::reflect_template("cudf::jit::column_accessor", + rtcx::reflect(i), + "cudf::column_device_view_core", + cudf::type_to_name(col->type()), + rtcx::reflect(false), + rtcx::reflect(table_sources[i].value()))); } else { auto& scalar = std::get(input); - auto element = cudf::type_to_name(scalar.as_column_view().type()); - accessors.emplace_back(jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate( - i, - "cudf::column_device_view_core", - element, - true, - 0 // scalars don't belong to a table, so just use 0 as placeholder - )); + accessors.emplace_back(rtcx::reflect_template( + "cudf::jit::column_accessor", + rtcx::reflect(i), + "cudf::column_device_view_core", + cudf::type_to_name(scalar.as_column_view().type()), + rtcx::reflect(true), + rtxc::reflect(0) // scalars don't belong to a table, so just use 0 as placeholder + )); } } - template_params.push_back( - jitify2::reflection::Template("cudf::jit::type_list").instantiate(accessors)); + template_params.push_back(rtcx::reflect_template("cudf::jit::type_list", accessors)); return template_params; } // Build the JIT kernel for join filtering -jitify2::ConfiguredKernel build_join_filter_kernel(std::string const& predicate_code, - std::span inputs, - std::span> table_sources, - bool is_ptx, - bool has_user_data, - null_aware is_null_aware, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +kernel build_join_filter_kernel(std::string const& predicate_code, + std::span inputs, + std::span> table_sources, + bool is_ptx, + bool has_user_data, + null_aware is_null_aware, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); @@ -123,18 +121,14 @@ jitify2::ConfiguredKernel build_join_filter_kernel(std::string const& predicate_ // Build template parameters and kernel name auto template_args = build_join_filter_template_params(inputs, table_sources, has_user_data, is_null_aware); - auto kernel_name = - jitify2::reflection::Template("cudf::join::jit::filter_join_kernel").instantiate(template_args); + auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); // Get compiled kernel - auto kernel = - cudf::jit::get_udf_kernel(*join_jit_filter_join_kernel_cu_jit, kernel_name, cuda_source); - - return kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); + return cudf::jit::get_udf_kernel("join/jit/filter_join_kernel.cu", kernel_name, cuda_source); } // Launch the JIT kernel for join filtering -void launch_join_filter_kernel(jitify2::ConfiguredKernel& kernel, +void launch_join_filter_kernel(kernel const& kernel, cudf::device_span left_indices, cudf::device_span right_indices, std::span inputs, @@ -173,7 +167,9 @@ void launch_join_filter_kernel(jitify2::ConfiguredKernel& kernel, &predicate_results, &user_data_ptr}; - kernel->launch_raw(args); + auto cfg = kernel.max_occupancy_config(0, 0); + + kernel->launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } // Same join semantics handling as the AST version @@ -490,15 +486,13 @@ filter_join_indices_jit(cudf::table_view const& left, auto const cuda_source = cudf::jit::parse_single_function_cuda(filter_result.udf, "GENERIC_JOIN_FILTER_OP"); - auto kernel_name = - jitify2::reflection::Template("cudf::join::jit::filter_join_kernel").instantiate(template_args); + auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); auto kernel = - cudf::jit::get_udf_kernel(*join_jit_filter_join_kernel_cu_jit, kernel_name, cuda_source); - auto configured_kernel = kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); + cudf::jit::get_udf_kernel("join/jit/filter_join_kernel.cu", kernel_name, cuda_source); // Allocate and compute predicate results auto predicate_results = rmm::device_uvector(left_indices.size(), stream); - launch_join_filter_kernel(configured_kernel, + launch_join_filter_kernel(kernel, left_indices, right_indices, filter_result.inputs, diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index 61f26d261067..d61788c9b08f 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -23,6 +23,7 @@ // clang-format off // This header is an inlined header that defines the GENERIC_JOIN_FILTER_OP function. It is placed here // so the symbols in the headers above can be used by it. +#include #include // clang-format on @@ -46,12 +47,12 @@ __device__ void execute_predicate_op(void* user_data, } template -CUDF_KERNEL void filter_join_kernel(cudf::size_type num_rows, - cudf::size_type const* __restrict__ left_indices, - cudf::size_type const* __restrict__ right_indices, - cudf::column_device_view_core const* __restrict__ columns, - bool* __restrict__ predicate_results, - void* __restrict__ user_data) +__device__ void filter_join_kernel(cudf::size_type num_rows, + cudf::size_type const* __restrict__ left_indices, + cudf::size_type const* __restrict__ right_indices, + cudf::column_device_view_core const* __restrict__ columns, + bool* __restrict__ predicate_results, + void* __restrict__ user_data) { auto const start = cudf::detail::grid_1d::global_thread_id(); auto const stride = cudf::detail::grid_1d::grid_stride(); @@ -96,3 +97,14 @@ CUDF_KERNEL void filter_join_kernel(cudf::size_type num_rows, } } // namespace cudf::join::jit + +extern "C" __global__ void cudf_kernel_entry(cudf::size_type const* __restrict__ left_indices, + cudf::size_type const* __restrict__ right_indices, + cudf::column_device_view_core const* left_tables, + cudf::column_device_view_core const* right_tables, + bool* predicate_results, + void* user_data) +{ + CUDF_KERNEL_INSTANCE( + left_indices, right_indices, left_tables, right_tables, predicate_results, user_data); +} diff --git a/cpp/src/rolling/detail/rolling_fixed_window.cu b/cpp/src/rolling/detail/rolling_fixed_window.cu index 8c87a9ca0a60..07050fb7c0e0 100644 --- a/cpp/src/rolling/detail/rolling_fixed_window.cu +++ b/cpp/src/rolling/detail/rolling_fixed_window.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index 2981a14e64d8..f1da37415a17 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -21,8 +21,6 @@ #include -#include - #include namespace cudf { @@ -90,24 +88,28 @@ inline std::unique_ptr rolling_window_udf_impl( 0, stream, cudf::get_current_device_resource_ref()}; std::string kernel_reflection = - jitify2::reflection::Template("cudf::rolling::jit::rolling_window_kernel") // - .instantiate(cudf::type_to_name(input.type()), // list of template arguments - cudf::type_to_name(output->type()), - udf_agg._operator_name, - preceding_window_str, - following_window_str); - - cudf::jit::get_udf_kernel(*rolling_jit_kernel_cu_jit, kernel_reflection, cuda_source) - ->configure_1d_max_occupancy(0, 0, nullptr, stream.value()) - ->launch(input.size(), - cudf::jit::get_data_ptr(input), - input.null_mask(), - cudf::jit::get_data_ptr(output_view), - output_view.null_mask(), - device_valid_count.data(), - preceding_window, - following_window, - min_periods); + rtcx::reflect_template("cudf::rolling::jit::rolling_window_kernel", + cudf::type_to_name(input.type()), + cudf::type_to_name(output->type()), + udf_agg._operator_name, + preceding_window_str, + following_window_str); + + auto kernel = cudf::jit::get_udf_kernel("rolling/jit/kernel.cu", kernel_reflection, cuda_source); + auto cfg = kernel.max_occupancy_config(0, 0); + kernel.launch_with({cfg.min_grid_size}, + {cfg.block_size}, + 0, + stream, + input.size(), + cudf::jit::get_data_ptr(input), + input.null_mask(), + cudf::jit::get_data_ptr(output->mutable_view()), + output_view.null_mask(), + device_valid_count.data(), + preceding_window, + following_window, + min_periods); output->set_null_count(output->size() - device_valid_count.value(stream)); diff --git a/cpp/src/rolling/detail/rolling_variable_window.cu b/cpp/src/rolling/detail/rolling_variable_window.cu index 28a1b56abb3e..5eba5ab31b7b 100644 --- a/cpp/src/rolling/detail/rolling_variable_window.cu +++ b/cpp/src/rolling/detail/rolling_variable_window.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index e13add4e326a..6573c10890cd 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -13,6 +13,7 @@ // it. Each UDF will have a different operation-udf.hpp generated for it, so we // need to put this pragma before including it to avoid PCH mismatch. +#include #include struct rolling_udf_ptx { @@ -44,15 +45,15 @@ template -CUDF_KERNEL void rolling_window_kernel(cudf::size_type nrows, - InType const* const __restrict__ in_col, - cudf::bitmask_type const* const __restrict__ in_col_valid, - OutType* __restrict__ out_col, - cudf::bitmask_type* __restrict__ out_col_valid, - cudf::size_type* __restrict__ output_valid_count, - detail::window_wrapper_base b_preceding_window_begin, - detail::window_wrapper_base b_following_window_begin, - cudf::size_type min_periods) +__device__ void rolling_window_kernel(cudf::size_type nrows, + InType const* const __restrict__ in_col, + cudf::bitmask_type const* const __restrict__ in_col_valid, + OutType* __restrict__ out_col, + cudf::bitmask_type* __restrict__ out_col_valid, + cudf::size_type* __restrict__ output_valid_count, + detail::window_wrapper_base b_preceding_window_begin, + detail::window_wrapper_base b_following_window_begin, + cudf::size_type min_periods) { auto i = cudf::detail::grid_1d::global_thread_id(); auto const stride = cudf::detail::grid_1d::grid_stride(); @@ -109,3 +110,25 @@ CUDF_KERNEL void rolling_window_kernel(cudf::size_type nrows, } // namespace jit } // namespace rolling } // namespace cudf + +extern "C" __global__ void cudf_kernel_entry( + cudf::size_type nrows, + void const* const __restrict__ in_col, + cudf::bitmask_type const* const __restrict__ in_col_valid, + void* __restrict__ out_col, + cudf::bitmask_type* __restrict__ out_col_valid, + cudf::size_type* __restrict__ output_valid_count, + cudf::detail::window_wrapper_base preceding_window_begin, + cudf::detail::window_wrapper_base following_window_begin, + cudf::size_type min_periods) +{ + CUDF_KERNEL_INSTANCE(nrows, + in_col, + in_col_valid, + out_col, + out_col_valid, + output_valid_count, + preceding_window_begin, + following_window_begin, + min_periods); +} diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 9e945c6a2446..ed15bda69610 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -26,20 +26,42 @@ void context::ensure_nvcomp_loaded() { io::detail::nvcomp::load_nvcomp_library() void context::ensure_jit_cache_initialized() { - std::call_once(_program_cache_init_flag, - [&]() { _program_cache = jit::program_cache::create(); }); + std::call_once(_jit_cache_init_flag, [&]() { + // make sure the required directories exist + std::filesystem::create_directories(_config.rtcx_cache_dir); + std::filesystem::create_directories(_config.jit_bundle_dir); + std::filesystem::create_directories(_config.jit_pch_dir); + std::filesystem::create_directories(_config.jit_tmp_dir); + + rtcx::initialize(); + + auto limits = rtcx::cache_limits{.num_mem_blobs = _config.kernel_cache_limit_process, + .num_mem_libraries = _config.kernel_cache_limit_process}; + + _rtcx_cache = std::make_unique(_config.rtcx_cache_dir, + _config.jit_tmp_dir, + limits, + bool{_config.preload_jit_cache}, + bool{_config.disable_jit_cache}); + + if (_config.clear_jit_cache) { + _rtcx_cache->clear_memory_store(); + _rtcx_cache->clear_disk_store(); + } + + // note that jit_bundle depends on rtcx_cache, so we ensure rtcx_cache is initialized first. + _jit_bundle = std::make_unique(_config.jit_bundle_dir, *_rtcx_cache); + }); } -jit::program_cache& context::program_cache() -{ - ensure_jit_cache_initialized(); - return *_program_cache; -} +context::~context() { rtcx::teardown(); } bool context::dump_codegen() const { return _config.dump_codegen; } bool context::use_jit() const { return _config.use_jit; } +std::string const& context::get_jit_pch_dir() const { return _config.jit_pch_dir; } + void context::initialize_components(init_flags flags) { if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { ensure_jit_cache_initialized(); } @@ -47,6 +69,24 @@ void context::initialize_components(init_flags flags) if (has_flag(flags, init_flags::LOAD_NVCOMP)) { io::detail::nvcomp::load_nvcomp_library(); } } +std::filesystem::path get_cudf_kernel_cache_dir() +{ + if (auto cudf = getenv_optional("LIBCUDF_KERNEL_CACHE_PATH"); cudf.has_value()) { + return std::filesystem::path(*cudf); + } + + if (auto home = getenv_optional("HOME"); home.has_value()) { + return std::filesystem::path(*home) / ".libcudf"; + } + + CUDF_FAIL( + "Unable to determine the CUDF root directory. Please set the `LIBCUDF_KERNEL_CACHE_PATH` or " + "`HOME` " + "environment variables to allow automatic resolution of the root " + "directory.", + std::runtime_error); +} + static std::optional _context{std::nullopt}; static std::optional _context_init_flag{std::in_place}; static std::optional _context_deinit_flag{std::in_place}; @@ -58,15 +98,39 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { std::call_once(*_context_init_flag, [&]() { - bool dump_codegen = cudf::detail::get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); - bool use_jit = cudf::detail::get_bool_env_or("LIBCUDF_JIT_ENABLED", false); + bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); + bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); + bool preload_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_PRELOAD", false); + bool disable_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_DISABLED", false); + bool clear_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); + bool disable_cuda_cache = get_bool_env_or("LIBCUDF_JIT_DISABLE_CUDA_CACHE", false); + bool jit_verbose = get_bool_env_or("LIBCUDF_JIT_VERBOSE", false); + bool dump_jit_trace = get_bool_env_or("LIBCUDF_JIT_DUMP_TRACE", false); + bool dump_jit_time_profile = get_bool_env_or("LIBCUDF_JIT_DUMP_TIME_PROFILE", false); + + auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16'384U); flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); - context_config cfg{ - .dump_codegen = dump_codegen, - .use_jit = use_jit, - }; + auto jit_bundle_dir = get_cudf_kernel_cache_dir() / "bundle"; + auto rtcx_cache_dir = get_cudf_kernel_cache_dir() / "rtcx_cache"; + auto jit_pch_dir = get_cudf_kernel_cache_dir() / "pch"; + auto jit_tmp_dir = get_cudf_kernel_cache_dir() / "tmp"; + + context_config cfg{.dump_codegen = dump_codegen, + .use_jit = use_jit, + .preload_jit_cache = preload_jit_cache, + .disable_jit_cache = disable_jit_cache, + .clear_jit_cache = clear_jit_cache, + .disable_cuda_cache = disable_cuda_cache, + .jit_verbose = jit_verbose, + .dump_jit_trace = dump_jit_trace, + .dump_jit_time_profile = dump_jit_time_profile, + .rtcx_cache_dir = rtcx_cache_dir, + .jit_bundle_dir = jit_bundle_dir, + .jit_pch_dir = jit_pch_dir, + .jit_tmp_dir = jit_tmp_dir, + .kernel_cache_limit_process = kernel_cache_limit_process}; _context.emplace(cfg, flags); }); @@ -85,9 +149,18 @@ void teardown() }); } -void enable_jit_cache(bool enable) { get_context().program_cache().enable(enable); } +void enable_jit_cache(bool enabled) +{ + auto& cache = get_context().rtcx_cache(); + cache.enable(enabled); +} -void clear_jit_cache() { get_context().program_cache().clear(); } +void clear_jit_cache() +{ + auto& cache = get_context().rtcx_cache(); + cache.clear_memory_store(); + cache.clear_disk_store(); +} context& get_context() { diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 39d259dd2fb2..fcf76dd9e88e 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -11,15 +11,34 @@ #include #include +namespace rtcx { +struct cache_t; +} // namespace rtcx + namespace cudf { namespace jit { class program_cache; } +struct jit_bundle_t; + struct [[nodiscard]] context_config { - bool dump_codegen = false; - bool use_jit = false; + bool dump_codegen : 1 = false; + bool use_jit : 1 = false; + bool preload_jit_cache : 1 = false; + bool disable_jit_cache : 1 = false; + bool clear_jit_cache : 1 = false; + bool disable_codegen_cache : 1 = false; + bool disable_cuda_cache : 1 = false; + bool jit_verbose : 1 = false; + bool dump_jit_trace : 1 = false; + bool dump_jit_time_profile : 1 = false; + std::string rtcx_cache_dir = {}; + std::string jit_bundle_dir = {}; + std::string jit_pch_dir = {}; + std::string jit_tmp_dir = {}; + uint32_t kernel_cache_limit_process = 0; }; /// @brief The context object contains global state internal to CUDF. @@ -29,8 +48,9 @@ class context { public: private: context_config _config; - std::once_flag _program_cache_init_flag; - std::unique_ptr _program_cache; + std::once_flag _jit_cache_init_flag; + std::unique_ptr _rtcx_cache; + std::unique_ptr _jit_bundle; private: void ensure_nvcomp_loaded(); @@ -43,14 +63,20 @@ class context { context& operator=(context const&) = delete; context(context&&) = delete; context& operator=(context&&) = delete; - ~context() = default; + ~context(); + + rtcx::cache_t& rtcx_cache(); - jit::program_cache& program_cache(); + jit_bundle_t& jit_bundle(); [[nodiscard]] bool dump_codegen() const; [[nodiscard]] bool use_jit() const; + [[nodiscard]] context_config const& config() const { return _config; } + + [[nodiscard]] std::string const& get_jit_pch_dir() const; + /// @brief Initialize additional components based on the provided flags /// @param flags The initialization flags to process void initialize_components(init_flags flags); diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 2456433d0bc6..0613f4bd03f0 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -47,22 +47,19 @@ __device__ void execute_transform_op(void* user_data, size_type element_idx, Arg } /// @brief The generic transform kernel. Supports all types and nullability combinations. -template -CUDF_KERNEL void transform_kernel(size_type row_size, - bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols) +template +__device__ void transform_kernel(size_type row_size, + bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + column_device_view_core const* __restrict__ input_cols, + mutable_column_device_view_core const* __restrict__ output_cols) { // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); for (auto element_idx = start; element_idx < row_size; element_idx += stride) { - if constexpr (is_null_aware == null_aware::NO) { + if constexpr (is_null_aware) { if (stencil != nullptr && !bit_is_set(stencil, element_idx)) { continue; } auto ins = InputAccessors::map( @@ -111,3 +108,13 @@ CUDF_KERNEL void transform_kernel(size_type row_size, } // namespace jit } // namespace cudf + +extern "C" __global__ void cudf_kernel_entry( + cudf::size_type row_size, + cudf::bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + cudf::column_device_view_core const* __restrict__ input_cols, + cudf::mutable_column_device_view_core const* __restrict__ output_cols) +{ + CUDF_KERNEL_INSTANCE(row_size, stencil, user_data, input_cols, output_cols); +} diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index b635c1c42d50..e13a2d60f6a9 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -29,7 +29,6 @@ #include #include #include -#include #include #include @@ -158,14 +157,14 @@ using handle = std::variant< namespace jit_transform { -jitify2::Kernel instantiate(null_aware is_null_aware, - bool has_user_data, - std::string const& ins, - std::string const& outs, - std::vector const& ptx_input_types, - std::vector const& ptx_output_types, - std::string const& udf, - udf_source_type source_type) +kernel instantiate(bool is_null_aware, + bool has_user_data, + std::string const& ins, + std::string const& outs, + std::vector const& ptx_input_types, + std::vector const& ptx_output_types, + std::string const& udf, + udf_source_type source_type) { CUDF_FUNC_RANGE(); auto cuda_source = (source_type == udf_source_type::PTX) @@ -175,14 +174,16 @@ jitify2::Kernel instantiate(null_aware is_null_aware, jit::build_ptx_params(ptx_output_types, ptx_input_types, has_user_data)) : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); - auto kernel = jitify2::reflection::Template("cudf::jit::transform_kernel") - .instantiate(is_null_aware, has_user_data, ins, outs); + auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", + rtcx::reflect(is_null_aware), + rtcx::reflect(has_user_data), + ins, + outs); - return jit::get_udf_kernel( - *transform_jit_kernel_cu_jit, kernel, cuda_source, {"-restrict", "--dopt=on"}); + return jit::get_udf_kernel("transform/jit/kernel.cu", kernel, cuda_source); } -void launch(jitify2::Kernel const& kernel, +void launch(cudf::kernel const& kernel, size_type row_size, bitmask_type const* stencil, void* user_data, @@ -191,8 +192,13 @@ void launch(jitify2::Kernel const& kernel, rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - void* args[] = {&row_size, &stencil, &user_data, &input_cols, &output_cols}; - kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); + void* args[] = {&row_size, &stencil, &user_data, &input_cols, &output_cols}; + auto kernel_ref = kernel.get(); + auto cfg = kernel_ref.max_occupancy_config(0, 0); + CUDF_EXPECTS(cfg.block_size % cudf::detail::warp_size == 0, + "Expected block size to be a multiple of warp size", + std::runtime_error); + kernel_ref.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } std::string reflect_input_element(column_view const& c) { return type_to_name(c.type()); } @@ -244,8 +250,12 @@ auto reflect(udf_source_type source_type, auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); bool as_scalar = std::holds_alternative(in); - auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate(i, column, element, as_scalar, 0); + auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", + rtcx::reflect(i), + column, + element, + rtcx::reflect(as_scalar), + rtcx::reflect(0)); in_types.push_back(accessor); } @@ -256,14 +266,18 @@ auto reflect(udf_source_type source_type, auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); bool as_scalar = false; // never scalar - auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate(i, column, element, as_scalar, 0); + auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", + rtcx::reflect(i), + column, + element, + rtcx::reflect(as_scalar), + rtcx::reflect(0)); out_types.push_back(accessor); } - auto ins = jitify2::reflection::Template("cudf::jit::type_list").instantiate(in_types); - auto outs = jitify2::reflection::Template("cudf::jit::type_list").instantiate(out_types); + auto ins = rtcx::reflect_template("cudf::jit::type_list", in_types); + auto outs = rtcx::reflect_template("cudf::jit::type_list", out_types); std::vector ptx_in_types; std::vector ptx_out_types; @@ -319,7 +333,7 @@ auto to_args(std::span inputs, return std::make_tuple(std::move(d_args), std::move(handles)); } -void run(null_aware is_null_aware, +void run(bool is_null_aware, bool has_user_data, size_type row_size, bitmask_type const* d_stencil, From 01f40f7fb94f1f32d71f4ff73b91102b8aa70d94 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 22 May 2026 17:33:00 +0000 Subject: [PATCH 192/254] update copyright year to 2026 in rolling window files --- cpp/src/rolling/detail/rolling_fixed_window.cu | 2 +- cpp/src/rolling/detail/rolling_jit.cuh | 12 ++++-------- cpp/src/rolling/detail/rolling_variable_window.cu | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/cpp/src/rolling/detail/rolling_fixed_window.cu b/cpp/src/rolling/detail/rolling_fixed_window.cu index 8c87a9ca0a60..07050fb7c0e0 100644 --- a/cpp/src/rolling/detail/rolling_fixed_window.cu +++ b/cpp/src/rolling/detail/rolling_fixed_window.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/rolling/detail/rolling_jit.cuh b/cpp/src/rolling/detail/rolling_jit.cuh index dec40de9a921..4c32695134dc 100644 --- a/cpp/src/rolling/detail/rolling_jit.cuh +++ b/cpp/src/rolling/detail/rolling_jit.cuh @@ -7,15 +7,11 @@ #include +#include + namespace cudf { namespace detail { -template -T minimum(T a, T b) -{ - return b < a ? b : a; -} - struct window_wrapper_base { cudf::size_type const* group_offsets = nullptr; cudf::size_type const* group_labels = nullptr; @@ -70,7 +66,7 @@ struct preceding_window_wrapper : public window_wrapper_base { { auto group_label = group_labels[idx]; auto group_start = group_offsets[group_label]; - return minimum(window, idx - group_start + 1); // Preceding includes current row. + return cuda::std::min(window, idx - group_start + 1); // Preceding includes current row. } }; @@ -93,7 +89,7 @@ struct following_window_wrapper : public window_wrapper_base { auto group_end = group_offsets[group_label + 1]; // Cannot fall off the end, since offsets is capped with `input.size()`. - return minimum(window, (group_end - 1) - idx); + return cuda::std::min(window, (group_end - 1) - idx); } }; diff --git a/cpp/src/rolling/detail/rolling_variable_window.cu b/cpp/src/rolling/detail/rolling_variable_window.cu index 28a1b56abb3e..5eba5ab31b7b 100644 --- a/cpp/src/rolling/detail/rolling_variable_window.cu +++ b/cpp/src/rolling/detail/rolling_variable_window.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ From 67ee5f6a02ac6952e7d6ab749f501756d8aee779 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 22 May 2026 18:46:22 +0000 Subject: [PATCH 193/254] Update LIBCUDF_KERNEL_CACHE_PATH in Dockerfile to use libcudf_kernel_cache --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 57ccf6302c5b..1b89799aa330 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -34,7 +34,7 @@ ENV PYTHONUNBUFFERED="1" ENV PYTHONDONTWRITEBYTECODE="1" ENV HISTFILE="/home/coder/.cache/._bash_history" -ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache" +ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/libcudf_kernel_cache" ### # sccache configuration From ef0fc46a3b206a9e87859c0cf354d75cd62a86e6 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 22 May 2026 20:46:05 +0000 Subject: [PATCH 194/254] Implement JIT kernel embedding and update related includes for improved performance --- cpp/CMakeLists.txt | 54 ++++++++++++++++++- .../cudf/detail/utilities/getenv_or.hpp | 1 + cpp/librtcx/embed.hpp | 2 +- cpp/librtcx/sha256.hpp | 47 ++++++++-------- cpp/src/binaryop/binaryop.cpp | 4 +- cpp/src/jit/cache.cpp | 18 +------ cpp/src/jit/helpers.cpp | 8 +-- cpp/src/jit/helpers.hpp | 6 +-- cpp/src/join/filter_join_indices_jit.cu | 30 ++++++----- cpp/src/join/jit/filter_join_kernel.cu | 4 +- cpp/src/rolling/detail/rolling_udf.cuh | 5 +- cpp/src/runtime/context.cpp | 44 +++++++++------ cpp/src/transform/transform.cu | 4 +- 13 files changed, 139 insertions(+), 88 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ac905e35117f..bca705a56787 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -404,6 +404,56 @@ if(NOT BUILD_SHARED_LIBS) endif() endif() +add_embed(cudf_jit_embed) + +embed_includes( + cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/librtcx/libcxx DEST_DIRECTORY + librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx +) + +embed_includes( + cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf DEST_DIRECTORY + cudf/cpp/include/cudf INCLUDE_DIRECTORIES cudf/cpp/include +) + +embed_includes( + cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/jit DEST_DIRECTORY + cudf/cpp/src/jit INCLUDE_DIRECTORIES cudf/cpp/src +) + +embed_includes( + cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/binaryop/jit DEST_DIRECTORY + cudf/cpp/src/binaryop/jit INCLUDE_DIRECTORIES cudf/cpp/src +) + +embed_includes( + cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/join/jit DEST_DIRECTORY + cudf/cpp/src/join/jit INCLUDE_DIRECTORIES cudf/cpp/src +) + +embed_includes( + cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling DEST_DIRECTORY + cudf/cpp/src/rolling INCLUDE_DIRECTORIES cudf/cpp/src +) + +embed_includes( + cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/transform/jit DEST_DIRECTORY + cudf/cpp/src/transform/jit INCLUDE_DIRECTORIES cudf/cpp/src +) + +get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLUDE_DIRECTORIES) + +foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) + cmake_path(GET INC_DIR FILENAME INC_DIR_NAME) + + embed_includes( + cudf_jit_embed SOURCE_DIRECTORY ${INC_DIR} DEST_DIRECTORY CCCL/libcudacxx/${INC_DIR_NAME} + INCLUDE_DIRECTORIES CCCL/libcudacxx/${INC_DIR_NAME} + ) +endforeach() + +embed(cudf_jit_embed COMPRESSION zstd) + # ################################################################################################## # * library targets ------------------------------------------------------------------------------- add_library( @@ -662,11 +712,11 @@ add_library( src/io/utilities/row_selection.cpp src/io/utilities/type_inference.cu src/io/utilities/trie.cu + src/jit/cache.cpp src/jit/helpers.cpp src/jit/parser.cpp src/jit/row_ir.cpp src/jit/util.cpp - src/jit/jit.cpp src/join/conditional_join.cu src/join/cross_join.cu src/join/distinct_hash_join.cu @@ -962,7 +1012,7 @@ add_library( ${cudf_jit_embed_SOURCE_DIR}/cudf_jit_embed.s ) -add_dependencies(cudf cudf_jit_embed cudf_fragments) +add_dependencies(cudf cudf_jit_embed) set_property( SOURCE src/io/parquet/writer_impl.cu diff --git a/cpp/include/cudf/detail/utilities/getenv_or.hpp b/cpp/include/cudf/detail/utilities/getenv_or.hpp index ae57ec482c54..7076d470250e 100644 --- a/cpp/include/cudf/detail/utilities/getenv_or.hpp +++ b/cpp/include/cudf/detail/utilities/getenv_or.hpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace cudf::detail { diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index 30b45a19fd5b..4a0a15932cf2 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -320,8 +320,8 @@ constexpr std::uint8_t hash[{}] = }} )***", - namespace_decl, id, + namespace_decl, include_dirs.size(), include_dirs_list, file_indices_list, diff --git a/cpp/librtcx/sha256.hpp b/cpp/librtcx/sha256.hpp index 14543fa684d8..1149477f4b38 100644 --- a/cpp/librtcx/sha256.hpp +++ b/cpp/librtcx/sha256.hpp @@ -13,26 +13,29 @@ namespace rtcx { -inline namespace detail { +namespace functions { -std::uint32_t ror(std::uint32_t x, std::uint32_t n) { return (x >> n) | (x << (32 - n)); } +inline std::uint32_t ror(std::uint32_t x, std::uint32_t n) { return (x >> n) | (x << (32 - n)); } -std::uint32_t ch(std::uint32_t x, std::uint32_t y, std::uint32_t z) { return z ^ (x & (y ^ z)); } +inline std::uint32_t ch(std::uint32_t x, std::uint32_t y, std::uint32_t z) +{ + return z ^ (x & (y ^ z)); +} -std::uint32_t maj(std::uint32_t x, std::uint32_t y, std::uint32_t z) +inline std::uint32_t maj(std::uint32_t x, std::uint32_t y, std::uint32_t z) { return ((x | y) & z) | (x & y); } -std::uint32_t sigma0(std::uint32_t x) { return ror(x, 2) ^ ror(x, 13) ^ ror(x, 22); } +inline std::uint32_t sigma0(std::uint32_t x) { return ror(x, 2) ^ ror(x, 13) ^ ror(x, 22); } -std::uint32_t sigma1(std::uint32_t x) { return ror(x, 6) ^ ror(x, 11) ^ ror(x, 25); } +inline std::uint32_t sigma1(std::uint32_t x) { return ror(x, 6) ^ ror(x, 11) ^ ror(x, 25); } -std::uint32_t gamma0(std::uint32_t x) { return ror(x, 7) ^ ror(x, 18) ^ (x >> 3); } +inline std::uint32_t gamma0(std::uint32_t x) { return ror(x, 7) ^ ror(x, 18) ^ (x >> 3); } -std::uint32_t gamma1(std::uint32_t x) { return ror(x, 17) ^ ror(x, 19) ^ (x >> 10); } +inline std::uint32_t gamma1(std::uint32_t x) { return ror(x, 17) ^ ror(x, 19) ^ (x >> 10); } -void put_be32(void* ptr, std::uint32_t value) +inline void put_be32(void* ptr, std::uint32_t value) { auto* p = (std::uint8_t*)ptr; p[0] = (value >> 24) & 0xff; @@ -41,14 +44,14 @@ void put_be32(void* ptr, std::uint32_t value) p[3] = (value >> 0) & 0xff; } -std::uint32_t get_be32(void const* ptr) +inline std::uint32_t get_be32(void const* ptr) { auto const* p = (std::uint8_t const*)ptr; return (std::uint32_t)p[0] << 24 | (std::uint32_t)p[1] << 16 | (std::uint32_t)p[2] << 8 | (std::uint32_t)p[3] << 0; } -} // namespace detail +} // namespace functions struct [[nodiscard]] sha256_hex_string { char data_[65]; // NOLINT(modernize-avoid-c-arrays) @@ -110,8 +113,8 @@ struct [[nodiscard]] sha256 { struct sha256_context { private: - static constexpr size_t BLOCK_SIZE = 64; - std::uint32_t state_[8] = // NOLINT(modernize-avoid-c-arrays) + static constexpr std::size_t BLOCK_SIZE = 64; + std::uint32_t state_[8] = // NOLINT(modernize-avoid-c-arrays) {0x6a09'e667ul, 0xbb67'ae85ul, 0x3c6e'f372ul, @@ -143,16 +146,16 @@ struct sha256_context { /* copy the state into 512-bits into W[0..15] */ for (i = 0; i < 16; i++, buf += sizeof(std::uint32_t)) - W[i] = get_be32(buf); + W[i] = functions::get_be32(buf); /* fill W[16..63] */ for (i = 16; i < 64; i++) - W[i] = gamma1(W[i - 2]) + W[i - 7] + gamma0(W[i - 15]) + W[i - 16]; + W[i] = functions::gamma1(W[i - 2]) + W[i - 7] + functions::gamma0(W[i - 15]) + W[i - 16]; -#define RND(a, b, c, d, e, f, g, h, i, ki) \ - t0 = h + sigma1(e) + ch(e, f, g) + ki + W[i]; \ - t1 = sigma0(a) + maj(a, b, c); \ - d += t0; \ +#define RND(a, b, c, d, e, f, g, h, i, ki) \ + t0 = h + functions::sigma1(e) + functions::ch(e, f, g) + ki + W[i]; \ + t1 = functions::sigma0(a) + functions::maj(a, b, c); \ + d += t0; \ h = t0 + t1; RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 0, 0x428a'2f98); @@ -240,7 +243,7 @@ struct sha256_context { if (len_buf) { std::uint32_t left = 64 - len_buf; if (len < left) left = len; - memcpy(len_buf + buf_, data, left); + std::memcpy(len_buf + buf_, data, left); len_buf = (len_buf + left) & 63; len -= left; data = (data + left); @@ -254,7 +257,7 @@ struct sha256_context { len -= 64; } - if (len) memcpy(buf_, data, len); + if (len) std::memcpy(buf_, data, len); } sha256 finalize() @@ -276,7 +279,7 @@ struct sha256_context { /* copy output */ for (i = 0; i < 8; i++, digest += sizeof(std::uint32_t)) { - put_be32(digest, state_[i]); + functions::put_be32(digest, state_[i]); } return out; } diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index 167621d3f7fc..baa741f6095c 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -160,8 +160,8 @@ void binary_operation(mutable_column_view& out, cudf::type_to_name(rhs.type()), "cudf::binops::jit::UserDefinedOp"); - auto kernel = - cudf::jit::get_udf_kernel(*binaryop_jit_kernel_cu_jit, kernel_reflection, cuda_source); + auto kernel = cudf::jit::get_udf_kernel( + "cudf/cpp/src/binaryop/jit/kernel.cu", kernel_reflection, cuda_source); auto cfg = kernel.max_occupancy_config(0, 0); kernel.launch_with({cfg.min_grid_size}, diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 2c8a2cd86bdf..1c89d5872aed 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -39,22 +39,6 @@ rtcx::sha256 hash(std::span inputs) return ctx.finalize(); } -rtcx::sha256 hash(std::span file_fragments, - std::span memory_fragments) -{ - rtcx::sha256_context ctx; - for (auto const& fragment : file_fragments) { - ctx.update( - std::span{reinterpret_cast(fragment.path), std::strlen(fragment.path)}); - } - - for (auto const& fragment : memory_fragments) { - ctx.update(fragment.data); - } - - return ctx.finalize(); -} - void install_file_set(std::string_view target_dir, std::span compressed_binary, size_t uncompressed_size, @@ -315,7 +299,7 @@ kernel get_kernel(std::string const& name, auto header_include_names_hash = hash(header_include_names).to_hex_string(); auto headers_hash = hash(headers).to_hex_string(); auto bundle_hash = bundle.get_hash(); - auto source_file = std::format("{}/cudf/cpp/src/{}", bundle.get_directory(), source_file_id); + auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); auto cache_key = std::format(R"***(cuLibrary name={} diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index bf83745d7df0..30c7882e4ecf 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -102,13 +102,9 @@ kernel get_udf_kernel(std::string const& source_file, char const* include_names[] = // NOLINT(modernize-avoid-c-arrays) {"cudf/detail/operation-udf.hpp", "cudf/detail/kernel-instance.hpp"}; char const* include_headers[] = // NOLINT(modernize-avoid-c-arrays) - {udf_cuda_source.c_str(), kernel_instance_source.c_str()}; + {cuda_source.c_str(), kernel_instance_source.c_str()}; - return get_kernel(std::format("{}.jit.cu", source_file), - source_file, - include_names, - include_headers, - kernel_name); + return get_kernel(source_file, source_file, include_names, include_headers, kernel_name); } } // namespace jit diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index d17ed93924d9..8a7685a63ebd 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -75,9 +75,9 @@ column_views_to_device(std::span views, std::vector input_type_names( std::span const> views); -kernel get_udf_kernel(std::string const& kernel_name, - std::string const& cuda_source, - std::vector const& extra_options = {}); +kernel get_udf_kernel(std::string const& source_file, + std::string const& kernel_name, + std::string const& cuda_source); } // namespace jit } // namespace cudf diff --git a/cpp/src/join/filter_join_indices_jit.cu b/cpp/src/join/filter_join_indices_jit.cu index 6c066032bf83..ae57a4263058 100644 --- a/cpp/src/join/filter_join_indices_jit.cu +++ b/cpp/src/join/filter_join_indices_jit.cu @@ -48,10 +48,10 @@ namespace detail { namespace { std::vector build_join_filter_template_params( - std::vector const& left_columns, - std::vector const& right_columns, + std::span inputs, + std::span> table_sources, bool has_user_data, - null_aware is_null_aware) + bool is_null_aware) { std::vector template_params; template_params.emplace_back(rtcx::reflect(has_user_data)); @@ -76,7 +76,7 @@ std::vector build_join_filter_template_params( "cudf::column_device_view_core", cudf::type_to_name(scalar.as_column_view().type()), rtcx::reflect(true), - rtxc::reflect(0) // scalars don't belong to a table, so just use 0 as placeholder + rtcx::reflect(0) // scalars don't belong to a table, so just use 0 as placeholder )); } } @@ -92,7 +92,7 @@ kernel build_join_filter_kernel(std::string const& predicate_code, std::span> table_sources, bool is_ptx, bool has_user_data, - null_aware is_null_aware, + bool is_null_aware, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { @@ -124,7 +124,8 @@ kernel build_join_filter_kernel(std::string const& predicate_code, auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); // Get compiled kernel - return cudf::jit::get_udf_kernel("join/jit/filter_join_kernel.cu", kernel_name, cuda_source); + return cudf::jit::get_udf_kernel( + "cudf/cpp/src/join/jit/filter_join_kernel.cu", kernel_name, cuda_source); } // Launch the JIT kernel for join filtering @@ -169,7 +170,7 @@ void launch_join_filter_kernel(kernel const& kernel, auto cfg = kernel.max_occupancy_config(0, 0); - kernel->launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); + kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } // Same join semantics handling as the AST version @@ -422,7 +423,7 @@ filter_join_indices_jit(cudf::table_view const& left, table_sources, is_ptx, false, // has_user_data = false for now - null_aware::NO, + false, stream, mr); @@ -478,17 +479,18 @@ filter_join_indices_jit(cudf::table_view const& left, auto filter_result = row_ir::ast_converter::filter( row_ir::target::CUDA, predicate, left, right, "filter_operation", stream, mr); - auto template_args = build_join_filter_template_params(filter_result.inputs, - filter_result.input_table_sources, - filter_result.user_data.has_value(), - filter_result.is_null_aware); + auto template_args = + build_join_filter_template_params(filter_result.inputs, + filter_result.input_table_sources, + filter_result.user_data.has_value(), + filter_result.is_null_aware == null_aware::YES); auto const cuda_source = cudf::jit::parse_single_function_cuda(filter_result.udf, "GENERIC_JOIN_FILTER_OP"); auto kernel_name = rtcx::reflect_template("cudf::join::jit::filter_join_kernel", template_args); - auto kernel = - cudf::jit::get_udf_kernel("join/jit/filter_join_kernel.cu", kernel_name, cuda_source); + auto kernel = cudf::jit::get_udf_kernel( + "cudf/cpp/src/join/jit/filter_join_kernel.cu", kernel_name, cuda_source); // Allocate and compute predicate results auto predicate_results = rmm::device_uvector(left_indices.size(), stream); diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index d61788c9b08f..283b3d7492dc 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -46,7 +46,7 @@ __device__ void execute_predicate_op(void* user_data, } } -template +template __device__ void filter_join_kernel(cudf::size_type num_rows, cudf::size_type const* __restrict__ left_indices, cudf::size_type const* __restrict__ right_indices, @@ -68,7 +68,7 @@ __device__ void filter_join_kernel(cudf::size_type num_rows, // Each accessor receives both tables and both indices, and internally selects // the appropriate table based on whether it's a left or right accessor. - if constexpr (is_null_aware == null_aware::YES) { + if constexpr (is_null_aware) { // Null-aware path: pass optional inputs, get optional result cuda::std::optional result{false}; auto inputs = Accessors::map([&]() { diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index f1da37415a17..82d497c796ea 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -95,8 +95,9 @@ inline std::unique_ptr rolling_window_udf_impl( preceding_window_str, following_window_str); - auto kernel = cudf::jit::get_udf_kernel("rolling/jit/kernel.cu", kernel_reflection, cuda_source); - auto cfg = kernel.max_occupancy_config(0, 0); + auto kernel = + cudf::jit::get_udf_kernel("cudf/cpp/src/rolling/jit/kernel.cu", kernel_reflection, cuda_source); + auto cfg = kernel.max_occupancy_config(0, 0); kernel.launch_with({cfg.min_grid_size}, {cfg.block_size}, 0, diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index ed15bda69610..837d241d8ef8 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -12,12 +12,12 @@ #include #include +#include #include namespace cudf { -context::context(context_config const& cfg, init_flags flags) - : _config{cfg}, _program_cache_init_flag{}, _program_cache{nullptr} +context::context(context_config const& cfg, init_flags flags) : _config{cfg}, _jit_cache_init_flag{} { initialize_components(flags); } @@ -56,6 +56,18 @@ void context::ensure_jit_cache_initialized() context::~context() { rtcx::teardown(); } +rtcx::cache_t& context::rtcx_cache() +{ + ensure_jit_cache_initialized(); + return *_rtcx_cache; +} + +jit_bundle_t& context::jit_bundle() +{ + ensure_jit_cache_initialized(); + return *_jit_bundle; +} + bool context::dump_codegen() const { return _config.dump_codegen; } bool context::use_jit() const { return _config.use_jit; } @@ -71,11 +83,12 @@ void context::initialize_components(init_flags flags) std::filesystem::path get_cudf_kernel_cache_dir() { - if (auto cudf = getenv_optional("LIBCUDF_KERNEL_CACHE_PATH"); cudf.has_value()) { + if (auto cudf = detail::getenv_optional("LIBCUDF_KERNEL_CACHE_PATH"); + cudf.has_value()) { return std::filesystem::path(*cudf); } - if (auto home = getenv_optional("HOME"); home.has_value()) { + if (auto home = detail::getenv_optional("HOME"); home.has_value()) { return std::filesystem::path(*home) / ".libcudf"; } @@ -98,17 +111,18 @@ namespace CUDF_EXPORT cudf { void initialize(init_flags flags) { std::call_once(*_context_init_flag, [&]() { - bool dump_codegen = get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); - bool use_jit = get_bool_env_or("LIBCUDF_JIT_ENABLED", false); - bool preload_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_PRELOAD", false); - bool disable_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_DISABLED", false); - bool clear_jit_cache = get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); - bool disable_cuda_cache = get_bool_env_or("LIBCUDF_JIT_DISABLE_CUDA_CACHE", false); - bool jit_verbose = get_bool_env_or("LIBCUDF_JIT_VERBOSE", false); - bool dump_jit_trace = get_bool_env_or("LIBCUDF_JIT_DUMP_TRACE", false); - bool dump_jit_time_profile = get_bool_env_or("LIBCUDF_JIT_DUMP_TIME_PROFILE", false); - - auto kernel_cache_limit_process = getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16'384U); + bool dump_codegen = detail::get_bool_env_or("LIBCUDF_JIT_DUMP_CODEGEN", false); + bool use_jit = detail::get_bool_env_or("LIBCUDF_JIT_ENABLED", false); + bool preload_jit_cache = detail::get_bool_env_or("LIBCUDF_KERNEL_CACHE_PRELOAD", false); + bool disable_jit_cache = detail::get_bool_env_or("LIBCUDF_KERNEL_CACHE_DISABLED", false); + bool clear_jit_cache = detail::get_bool_env_or("LIBCUDF_KERNEL_CACHE_CLEAR", false); + bool disable_cuda_cache = detail::get_bool_env_or("LIBCUDF_JIT_DISABLE_CUDA_CACHE", false); + bool jit_verbose = detail::get_bool_env_or("LIBCUDF_JIT_VERBOSE", false); + bool dump_jit_trace = detail::get_bool_env_or("LIBCUDF_JIT_DUMP_TRACE", false); + bool dump_jit_time_profile = detail::get_bool_env_or("LIBCUDF_JIT_DUMP_TIME_PROFILE", false); + + auto kernel_cache_limit_process = + detail::getenv_or("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 16'384U); flags = flags | (use_jit ? init_flags::INIT_JIT_CACHE : init_flags::NONE); diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index e13a2d60f6a9..7a91c3ca7945 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -180,7 +180,7 @@ kernel instantiate(bool is_null_aware, ins, outs); - return jit::get_udf_kernel("transform/jit/kernel.cu", kernel, cuda_source); + return jit::get_udf_kernel("cudf/cpp/src/transform/jit/kernel.cu", kernel, cuda_source); } void launch(cudf::kernel const& kernel, @@ -833,7 +833,7 @@ std::unique_ptr
execute_transform(std::string const& udf, auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; - jit_transform::run(is_null_aware, + jit_transform::run(is_null_aware == null_aware::YES, user_data.has_value(), row_size, stencil_has_nulls ? stencil_arg : nullptr, From 560901ed4276c83c0c187844f4f6eae90df984e6 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 22 May 2026 20:47:21 +0000 Subject: [PATCH 195/254] Fix include guard placement in assert.cuh --- cpp/include/cudf/detail/utilities/assert.cuh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/include/cudf/detail/utilities/assert.cuh b/cpp/include/cudf/detail/utilities/assert.cuh index c1fb8afc3b67..a3f334fdc5e0 100644 --- a/cpp/include/cudf/detail/utilities/assert.cuh +++ b/cpp/include/cudf/detail/utilities/assert.cuh @@ -6,7 +6,10 @@ #pragma once #include + +#ifndef __CUDACC_RTC__ #include +#endif /** * @brief `assert`-like macro for device code From ea73d7d550f62ebaf091bff31983cc9efcf6d47e Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 23 May 2026 01:20:38 +0000 Subject: [PATCH 196/254] update --- cpp/include/cudf/wrappers/dictionary.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/include/cudf/wrappers/dictionary.hpp b/cpp/include/cudf/wrappers/dictionary.hpp index 025c899be144..1f5fd2818089 100644 --- a/cpp/include/cudf/wrappers/dictionary.hpp +++ b/cpp/include/cudf/wrappers/dictionary.hpp @@ -8,7 +8,10 @@ #include #include + +#ifndef __CUDACC_RTC__ #include +#endif /** * @file From e44478cf8dfe70ac9880b2ae2b34e25c9ef72241 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sat, 23 May 2026 13:24:46 +0000 Subject: [PATCH 197/254] Refactor embedded file header comment and delete unused reflect function --- cpp/librtcx/embed.hpp | 3 +-- cpp/librtcx/rtcx.hpp | 2 +- cpp/src/transform/jit/kernel.cu | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index 4a0a15932cf2..d29c75bc6f88 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -266,7 +266,7 @@ embed_output generate_cxx_source_files_data(std::string_view id, auto cxx_header = std::format( R"***( -// Auto-generated header for embedded files with ID: {} +// Auto-generated header for embedded files #pragma once #include @@ -320,7 +320,6 @@ constexpr std::uint8_t hash[{}] = }} )***", - id, namespace_decl, include_dirs.size(), include_dirs_list, diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index d0ea4bff235c..bc3ff8fd5569 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -809,7 +809,7 @@ void teardown(); * appropriate CUDA string representations. */ template -std::string reflect(T value); +std::string reflect(T value) = delete; /** * @brief Reflect a boolean value into its CUDA string representation ("true" or "false") diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 0613f4bd03f0..b9d57dd909ed 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -28,6 +28,7 @@ // clang-format off // This header is an inlined header that defines the GENERIC_TRANSFORM_OP function. It is placed here // so the symbols in the headers above can be used by it. +#include #include // clang-format on @@ -59,7 +60,7 @@ __device__ void transform_kernel(size_type row_size, auto stride = detail::grid_1d::grid_stride(); for (auto element_idx = start; element_idx < row_size; element_idx += stride) { - if constexpr (is_null_aware) { + if constexpr (!is_null_aware) { if (stencil != nullptr && !bit_is_set(stencil, element_idx)) { continue; } auto ins = InputAccessors::map( From ffea1b63e3e14d49ab4208e29f115e1bcb5f5a07 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 26 May 2026 12:30:07 +0000 Subject: [PATCH 198/254] update --- cpp/src/jit/cache.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index a2fb3d10fd51..b0694556ee29 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -87,6 +87,7 @@ std::filesystem::path get_cache_dir() return kernel_cache_path; } + std::string get_program_cache_dir() { #if defined(JITIFY_USE_CACHE) From 522866b5641606b951e4b9df38535aedb6b058ca Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 26 May 2026 12:30:19 +0000 Subject: [PATCH 199/254] update --- cpp/src/jit/cache.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index b0694556ee29..a2fb3d10fd51 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -87,7 +87,6 @@ std::filesystem::path get_cache_dir() return kernel_cache_path; } - std::string get_program_cache_dir() { #if defined(JITIFY_USE_CACHE) From 812b6b7c77c0074b0c2420291e4b8c4a4a359715 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 26 May 2026 17:09:40 +0000 Subject: [PATCH 200/254] update --- cpp/src/rolling/detail/rolling_udf.cuh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/src/rolling/detail/rolling_udf.cuh b/cpp/src/rolling/detail/rolling_udf.cuh index 57f9214e6fed..d0f2641e3d78 100644 --- a/cpp/src/rolling/detail/rolling_udf.cuh +++ b/cpp/src/rolling/detail/rolling_udf.cuh @@ -88,12 +88,12 @@ inline std::unique_ptr rolling_window_udf_impl( 0, stream, cudf::get_current_device_resource_ref()}; std::string kernel_reflection = - jitify2::reflection::Template("cudf::rolling::jit::rolling_window_kernel") // - .instantiate(cudf::type_to_name(input.type()), // list of template arguments - cudf::type_to_name(output->type()), - udf_agg._operator_name, - preceding_window_str, - following_window_str); + rtcx::reflect_template("cudf::rolling::jit::rolling_window_kernel", + cudf::type_to_name(input.type()), // list of template arguments + cudf::type_to_name(output->type()), + udf_agg._operator_name, + preceding_window_str, + following_window_str); auto kernel = cudf::jit::get_udf_kernel("cudf/cpp/src/rolling/jit/kernel.cu", kernel_reflection, cuda_source); From cd162a11142dc707e5a58db1a1476e0b9cb7bbb9 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 26 May 2026 17:35:26 +0000 Subject: [PATCH 201/254] Refactor includes and format source file assignment in JIT cache --- cpp/include/cudf/detail/utilities/getenv_or.hpp | 2 +- cpp/src/jit/cache.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/cudf/detail/utilities/getenv_or.hpp b/cpp/include/cudf/detail/utilities/getenv_or.hpp index 7076d470250e..1c282525b2dd 100644 --- a/cpp/include/cudf/detail/utilities/getenv_or.hpp +++ b/cpp/include/cudf/detail/utilities/getenv_or.hpp @@ -8,9 +8,9 @@ #include #include +#include #include #include -#include namespace cudf::detail { diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 1c89d5872aed..fb36fe4816a3 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -299,7 +299,7 @@ kernel get_kernel(std::string const& name, auto header_include_names_hash = hash(header_include_names).to_hex_string(); auto headers_hash = hash(headers).to_hex_string(); auto bundle_hash = bundle.get_hash(); - auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); + auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); auto cache_key = std::format(R"***(cuLibrary name={} From ac00b4a0e32754723efb221d932160298b58676c Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 26 May 2026 18:31:21 +0000 Subject: [PATCH 202/254] Refactor kernel entry parameters in filter_join_kernel and remove TODO in transform_kernel --- cpp/src/join/jit/filter_join_kernel.cu | 15 ++++++++------- cpp/src/transform/jit/kernel.cu | 1 - 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index 283b3d7492dc..5c83b6ad3e49 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -98,13 +98,14 @@ __device__ void filter_join_kernel(cudf::size_type num_rows, } // namespace cudf::join::jit -extern "C" __global__ void cudf_kernel_entry(cudf::size_type const* __restrict__ left_indices, - cudf::size_type const* __restrict__ right_indices, - cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - bool* predicate_results, - void* user_data) +extern "C" __global__ void cudf_kernel_entry( + cudf::size_type num_rows, + cudf::size_type const* __restrict__ left_indices, + cudf::size_type const* __restrict__ right_indices, + cudf::column_device_view_core const* __restrict__ columns, + bool* __restrict__ predicate_results, + void* __restrict__ user_data) { CUDF_KERNEL_INSTANCE( - left_indices, right_indices, left_tables, right_tables, predicate_results, user_data); + num_rows, left_indices, right_indices, columns, predicate_results, user_data); } diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index b9d57dd909ed..17e410ceafc4 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -55,7 +55,6 @@ __device__ void transform_kernel(size_type row_size, column_device_view_core const* __restrict__ input_cols, mutable_column_device_view_core const* __restrict__ output_cols) { - // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); From d190ac05f768a8cbdf3036548a5cf99dddcfdbc1 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 27 May 2026 18:49:58 +0000 Subject: [PATCH 203/254] Add LTO (Link Time Optimization) support for transform operations - Implemented unary and binary operators for LTO in `transform_operator.cuh`. - Added new CUDA files for LTO operations: `add.cu`, `null_max.cu`, and fragments for various operations. - Updated `transform.hpp` to include new LTO functions: `unary_op_lto` and `binary_op_lto`. - Enhanced JIT compilation cache to support LTO linked kernels. - Created test cases for LTO transformations including invsqrt, sum of squares, and decimal square operations. - Added necessary type tags and utility functions for LTO operations. - Updated CMake configuration to embed LTO fragments for testing. --- cpp/CMakeLists.txt | 151 +++++++++++- cpp/benchmarks/CMakeLists.txt | 10 + cpp/benchmarks/binaryop/compiled_binaryop.cpp | 82 ++++++- cpp/benchmarks/binaryop/fragments/add.cu | 23 ++ cpp/benchmarks/binaryop/fragments/null_max.cu | 24 ++ cpp/include/cudf/jit/transform_operator.cuh | 29 +++ cpp/include/cudf/jit/type_tags.cuh | 56 +++++ cpp/include/cudf/transform.hpp | 91 +++++++ cpp/librtcx/embed.cmake | 19 ++ cpp/src/jit/cache.cpp | 100 ++++++++ cpp/src/jit/cache.hpp | 11 + cpp/src/transform/jit/kernel.cu | 87 ++++--- cpp/src/transform/transform.cu | 223 +++++++++++++++++- cpp/tests/CMakeLists.txt | 24 ++ .../transform/fragments/decimal_square.cu | 28 +++ cpp/tests/transform/fragments/invsqrt.cu | 12 + .../transform/fragments/sum_of_squares.cu | 14 ++ cpp/tests/transform/transform_lto_test.cpp | 98 ++++++++ 18 files changed, 1040 insertions(+), 42 deletions(-) create mode 100644 cpp/benchmarks/binaryop/fragments/add.cu create mode 100644 cpp/benchmarks/binaryop/fragments/null_max.cu create mode 100644 cpp/include/cudf/jit/transform_operator.cuh create mode 100644 cpp/include/cudf/jit/type_tags.cuh create mode 100644 cpp/tests/transform/fragments/decimal_square.cu create mode 100644 cpp/tests/transform/fragments/invsqrt.cu create mode 100644 cpp/tests/transform/fragments/sum_of_squares.cu create mode 100644 cpp/tests/transform/transform_lto_test.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bca705a56787..147a3daab81b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -454,6 +454,153 @@ endforeach() embed(cudf_jit_embed COMPRESSION zstd) +# This macro is used to create object libraries for JIT compilation fragments, and embed them as +# fatbins in the final library. It compiles the specified source file with the appropriate flags to +# generate a fatbin containing the specified kernel instance, and then embeds that fatbin in the +# final library with metadata that allows it to be looked up at runtime. +macro(add_fragment) + set(TARGET ${ARGV0}) + set(ONE_VALUE_ARGS FRAGMENT SOURCE KERNEL_ONLY KERNEL_INSTANCE) + set(MULTI_VALUE_ARGS DEFINITIONS ARRAY_IDS ARRAY_VALUES) + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT ARG_FRAGMENT) + message(FATAL_ERROR "add_fragment requires FRAGMENT argument") + endif() + + if(NOT ARG_SOURCE) + message(FATAL_ERROR "add_fragment requires SOURCE argument") + endif() + + set(OBJECT_ID ${TARGET}_${ARG_FRAGMENT}) + add_library(${OBJECT_ID} OBJECT ${ARG_SOURCE}) + target_compile_options( + ${OBJECT_ID} PRIVATE --compress-mode=size --expt-relaxed-constexpr --extended-lambda + ) + + if(DEFINED ARG_KERNEL_ONLY AND ARG_KERNEL_ONLY) + # ensure that the FATBIN symbols only contain the specified kernel + target_compile_options(${OBJECT_ID} PRIVATE -Xnvlink=--kernels-used=cudf_kernel_entry) + endif() + + if(ARG_KERNEL_INSTANCE) + set(INSTANTIATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/${TARGET}/instantiations/${ARG_FRAGMENT}") + file( + GENERATE + OUTPUT "${INSTANTIATION_DIR}/cudf/detail/kernel-instance.hpp" + CONTENT "#pragma once\n#define CUDF_KERNEL_INSTANCE ${ARG_KERNEL_INSTANCE}" + ) + target_include_directories(${OBJECT_ID} PRIVATE ${INSTANTIATION_DIR}) + endif() + + target_compile_definitions(${OBJECT_ID} PRIVATE CUDF_DISABLE_EXPORTS ${ARG_DEFINITIONS}) + set_target_properties( + ${OBJECT_ID} + PROPERTIES CUDA_SEPARABLE_COMPILATION ON + CUDA_FATBIN_COMPILATION ON + POSITION_INDEPENDENT_CODE ON + INTERPROCEDURAL_OPTIMIZATION ON + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS ON + CXX_VISIBILITY_PRESET hidden + CUDA_STANDARD 20 + CUDA_STANDARD_REQUIRED ON + CUDA_VISIBILITY_PRESET hidden + ) + target_link_libraries( + ${OBJECT_ID} + PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ + PRIVATE $ $ + ZLIB::ZLIB nvcomp::nvcomp kvikio::kvikio nanoarrow::nanoarrow zstd + ) + target_include_directories( + ${OBJECT_ID} PRIVATE "$" + "$" + ) + + embed_blob( + ${TARGET} FILE $ DEST fragments/${ARG_FRAGMENT}.fatbin ID + ${ARG_FRAGMENT} ARRAY_IDS ${ARG_ARRAY_IDS} ARRAY_VALUES ${ARG_ARRAY_VALUES} + ) +endmacro() + +add_embed(cudf_fragments) + +foreach(TYPE IN ITEMS uint8 uint16 uint32 int8 int16 int32 int64 float32 float64 uint64 decimal32 + decimal64 decimal128 +) + foreach(NULL_AWARE IN ITEMS 0 1) + set(FRAGMENT_NAME unop_lto_kernel) + set(VARIANT_NAME ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__element_${TYPE}) + get_property( + FILE_INDEX + TARGET cudf_fragments__embed_props + PROPERTY EMBED_FILE_INDEX + ) + add_fragment( + cudf_fragments + FRAGMENT + ${VARIANT_NAME} + SOURCE + src/transform/jit/kernel.cu + KERNEL_INSTANCE + "cudf::jit::transform_kernel<${NULL_AWARE}, false, cudf::jit::type_list>, cudf::jit::type_list>>" + DEFINITIONS + CUDF_LTO_MODE + ARRAY_IDS + ${FRAGMENT_NAME}_FILE_INDEX + ${FRAGMENT_NAME}_NULL_AWARE + ${FRAGMENT_NAME}_TYPE + ARRAY_VALUES + ${FILE_INDEX} + ${NULL_AWARE} + ${TYPE} + ) + endforeach() +endforeach() + +foreach(TYPE IN ITEMS uint8 uint16 uint32 uint64 int8 int16 int32 int64 float32 float64 decimal32 + decimal64 decimal128 +) + foreach(NULL_AWARE IN ITEMS 0 1) + foreach(RHS_IS_SCALAR IN ITEMS 0 1) + set(FRAGMENT_NAME binop_lto_kernel) + set(VARIANT_NAME + ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__element_${TYPE}__rhs_is_scalar_${RHS_IS_SCALAR} + ) + get_property( + FILE_INDEX + TARGET cudf_fragments__embed_props + PROPERTY EMBED_FILE_INDEX + ) + add_fragment( + cudf_fragments + FRAGMENT + ${VARIANT_NAME} + SOURCE + src/transform/jit/kernel.cu + KERNEL_INSTANCE + "cudf::jit::transform_kernel<${NULL_AWARE}, false, cudf::jit::type_list, cudf::jit::column_accessor<1, cudf::column_device_view_core, cudf::tags::${TYPE}, ${RHS_IS_SCALAR}, 0>>, cudf::jit::type_list>>" + DEFINITIONS + CUDF_LTO_MODE + ARRAY_IDS + ${FRAGMENT_NAME}_FILE_INDEX + ${FRAGMENT_NAME}_NULL_AWARE + ${FRAGMENT_NAME}_TYPE + ${FRAGMENT_NAME}_RHS_IS_SCALAR + ARRAY_VALUES + ${FILE_INDEX} + ${NULL_AWARE} + ${TYPE} + ${RHS_IS_SCALAR} + ) + endforeach() + endforeach() +endforeach() + +embed(cudf_fragments COMPRESSION none) + # ################################################################################################## # * library targets ------------------------------------------------------------------------------- add_library( @@ -1010,9 +1157,10 @@ add_library( src/utilities/type_dispatcher.cpp librtcx/rtcx.cpp ${cudf_jit_embed_SOURCE_DIR}/cudf_jit_embed.s + ${cudf_fragments_SOURCE_DIR}/cudf_fragments.s ) -add_dependencies(cudf cudf_jit_embed) +add_dependencies(cudf cudf_jit_embed cudf_fragments) set_property( SOURCE src/io/parquet/writer_impl.cu @@ -1084,6 +1232,7 @@ target_include_directories( "$" "$" "$" + "$" INTERFACE "$" ) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index e8bc5ae97a00..842541794569 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -386,11 +386,21 @@ ConfigureNVBench(CSV_WRITER_NVBENCH io/csv/csv_writer.cpp) # * ast benchmark --------------------------------------------------------------------------------- ConfigureNVBench(AST_NVBENCH ast/polynomials.cpp ast/transform.cpp) +# ################################################################################################## +# * LTO Fragments ---------------------------------------------------------------------------- +add_embed(cudf_benchmark_fragments) +add_fragment(cudf_benchmark_fragments FRAGMENT add SOURCE binaryop/fragments/add.cu) +add_fragment(cudf_benchmark_fragments FRAGMENT null_max SOURCE binaryop/fragments/null_max.cu) +embed(cudf_benchmark_fragments COMPRESSION none) + # ################################################################################################## # * binaryop benchmark ---------------------------------------------------------------------------- ConfigureNVBench( BINARYOP_NVBENCH binaryop/binaryop.cpp binaryop/compiled_binaryop.cpp binaryop/polynomials.cpp + ${cudf_benchmark_fragments_SOURCE_DIR}/cudf_benchmark_fragments.s ) +target_include_directories(BINARYOP_NVBENCH PRIVATE ${cudf_benchmark_fragments_SOURCE_DIR}) +add_dependencies(BINARYOP_NVBENCH cudf_benchmark_fragments) # ################################################################################################## # * transform benchmark diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 9befc76d2b45..4cb88ba9a0fc 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -1,12 +1,14 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #include #include +#include +#include #include template @@ -96,3 +98,81 @@ BINARYOP_BENCHMARK_DEFINE(duration_ms, duration_ns, NULL_EQUALS, bool BINARYOP_BENCHMARK_DEFINE(duration_ms, duration_ns, NULL_NOT_EQUALS, bool); BINARYOP_BENCHMARK_DEFINE(decimal32, decimal32, NULL_MAX, decimal32); BINARYOP_BENCHMARK_DEFINE(timestamp_D, timestamp_s, NULL_MIN, timestamp_s); + + +template +void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) +{ + auto const num_rows = static_cast(state.get_int64("num_rows")); + + auto const source_table = create_random_table( + {cudf::type_to_id(), cudf::type_to_id()}, row_count{num_rows}); + + auto lhs = cudf::column_view(source_table->get_column(0)); + auto rhs = cudf::column_view(source_table->get_column(1)); + + size_t fragment_id = 0; + bool null_aware = false; + + switch (binop) { + case cudf::binary_operator::ADD: { + fragment_id = cudf_benchmark_fragments::add; + null_aware = false; + } break; + case cudf::binary_operator::NULL_MAX: { + fragment_id = cudf_benchmark_fragments::null_max; + null_aware = true; + } break; + default: throw std::runtime_error("Unsupported binary operator for LTO benchmark"); + } + + // Call once for hot cache. + cudf::transform_output output{cudf::data_type{cudf::type_to_id()}, + cudf::output_nullability::ALL_VALID}; + + auto const range = cudf_benchmark_fragments::file_ranges[fragment_id]; + std::span udf{cudf_benchmark_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::binary_op_lto(source_table->get_column(0), + source_table->get_column(1), + output, + udf, + cudf::lto_binary_type::FATBIN, + null_aware ? cudf::null_aware::YES : cudf::null_aware::NO + ); + + // use number of bytes read and written to global memory + state.add_global_memory_reads(num_rows); + state.add_global_memory_reads(num_rows); + state.add_global_memory_writes(num_rows); + + state.exec(nvbench::exec_tag::sync, [&](nvbench::launch&) { + cudf::binary_op_lto(source_table->get_column(0), + source_table->get_column(1), + output, + udf, + cudf::lto_binary_type::FATBIN, + null_aware ? cudf::null_aware::YES : cudf::null_aware::NO); + }); +} + + +#define BM_LTO_BINARYOP_BENCHMARK_DEFINE(name, lhs, rhs, bop, tout) \ + static void name(::nvbench::state& st) \ + { \ + ::BM_lto_binaryop(st, ::cudf::binary_operator::bop); \ + } \ + NVBENCH_BENCH(name) \ + .set_name("lto_binary_op_" BM_STRINGIFY(name)) \ + .add_int64_axis("num_rows", {10'000, 100'000, 1'000'000, 10'000'000, 100'000'000}) + + +#define build_name_lto(a, b, c, d) a##_##b##_##c##_##d##_lto + + +#define LTO_BINARYOP_BENCHMARK_DEFINE(lhs, rhs, bop, tout) \ + BM_LTO_BINARYOP_BENCHMARK_DEFINE(build_name_lto(bop, lhs, rhs, tout), lhs, rhs, bop, tout) + + +LTO_BINARYOP_BENCHMARK_DEFINE(float, float, ADD, float); +LTO_BINARYOP_BENCHMARK_DEFINE(decimal32, decimal32, NULL_MAX, decimal32); diff --git a/cpp/benchmarks/binaryop/fragments/add.cu b/cpp/benchmarks/binaryop/fragments/add.cu new file mode 100644 index 000000000000..1d009811d4a9 --- /dev/null +++ b/cpp/benchmarks/binaryop/fragments/add.cu @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +template <> +__device__ void cudf::lto::binary_operator(int32_t* __restrict__ out, + int32_t a, + int32_t b) +{ + *out = a + b; +} + +template <> +__device__ void cudf::lto::binary_operator(float* __restrict__ out, + float a, + float b) +{ + *out = a + b; +} diff --git a/cpp/benchmarks/binaryop/fragments/null_max.cu b/cpp/benchmarks/binaryop/fragments/null_max.cu new file mode 100644 index 000000000000..c29d09984023 --- /dev/null +++ b/cpp/benchmarks/binaryop/fragments/null_max.cu @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include + +template <> +__device__ void cudf::lto::binary_operator, + cuda::std::optional, + cuda::std::optional>( + cuda::std::optional* __restrict__ out, + cuda::std::optional a, + cuda::std::optional b) +{ + if (a.has_value() || b.has_value()) { + *out = (a.has_value() && (!b.has_value() || (*a > *b))) ? *a : *b; + } else { + *out = cuda::std::nullopt; + } +} diff --git a/cpp/include/cudf/jit/transform_operator.cuh b/cpp/include/cudf/jit/transform_operator.cuh new file mode 100644 index 000000000000..49d15a2d4974 --- /dev/null +++ b/cpp/include/cudf/jit/transform_operator.cuh @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +namespace cudf { +namespace lto { + +/// @brief The unary operator for the transform operation. +/// @tparam Out The output type of the operator. +/// @tparam In0 The input type of the operator. +/// @param out The output destination for the operator result. +/// @param a The input value for the operator. +template +__device__ void unary_operator(Out* __restrict__ out, In0 a); + +/// @brief The binary operator for the transform operation. +/// @tparam Out The output type of the operator. +/// @tparam In0 The first input type of the operator. +/// @tparam In1 The second input type of the operator. +/// @param out The output destination for the operator result. +/// @param a The first input value for the operator. +/// @param b The second input value for the operator. +template +__device__ void binary_operator(Out* __restrict__ out, In0 a, In1 b); + +} // namespace lto +} // namespace cudf diff --git a/cpp/include/cudf/jit/type_tags.cuh b/cpp/include/cudf/jit/type_tags.cuh new file mode 100644 index 000000000000..15a4e7caef9f --- /dev/null +++ b/cpp/include/cudf/jit/type_tags.cuh @@ -0,0 +1,56 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include +#include +#include + +namespace CUDF_EXPORT cudf { +namespace tags { + +template +inline constexpr char const* tag_of = "void"; + +#define CUDF_TYPE_TAG(type, tag) \ + using tag = type; \ + template <> \ + inline constexpr char const* tag_of = #tag; + +CUDF_TYPE_TAG(bool, bool8); +CUDF_TYPE_TAG(int8_t, int8); +CUDF_TYPE_TAG(int16_t, int16); +CUDF_TYPE_TAG(int32_t, int32); +CUDF_TYPE_TAG(int64_t, int64); +CUDF_TYPE_TAG(__int128_t, int128); +CUDF_TYPE_TAG(uint8_t, uint8); +CUDF_TYPE_TAG(uint16_t, uint16); +CUDF_TYPE_TAG(uint32_t, uint32); +CUDF_TYPE_TAG(uint64_t, uint64); +CUDF_TYPE_TAG(__uint128_t, uint128); +CUDF_TYPE_TAG(float, float32); +CUDF_TYPE_TAG(double, float64); +CUDF_TYPE_TAG(cudf::string_view, string_view); +CUDF_TYPE_TAG(numeric::decimal32, decimal32); +CUDF_TYPE_TAG(numeric::decimal64, decimal64); +CUDF_TYPE_TAG(numeric::decimal128, decimal128); +CUDF_TYPE_TAG(cudf::duration_D, duration_D); +CUDF_TYPE_TAG(cudf::duration_h, duration_h); +CUDF_TYPE_TAG(cudf::duration_m, duration_m); +CUDF_TYPE_TAG(cudf::duration_s, duration_s); +CUDF_TYPE_TAG(cudf::duration_ms, duration_ms); +CUDF_TYPE_TAG(cudf::duration_us, duration_us); +CUDF_TYPE_TAG(cudf::duration_ns, duration_ns); +CUDF_TYPE_TAG(cudf::timestamp_D, timestamp_D); +CUDF_TYPE_TAG(cudf::timestamp_h, timestamp_h); +CUDF_TYPE_TAG(cudf::timestamp_m, timestamp_m); +CUDF_TYPE_TAG(cudf::timestamp_s, timestamp_s); +CUDF_TYPE_TAG(cudf::timestamp_ms, timestamp_ms); +CUDF_TYPE_TAG(cudf::timestamp_us, timestamp_us); +CUDF_TYPE_TAG(cudf::timestamp_ns, timestamp_ns); + +} // namespace tags +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index a2a99ed0c23b..1885e214281c 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -179,6 +179,97 @@ std::unique_ptr
multi_transform( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +/** + * @brief The type of LTO Binary + */ +enum class lto_binary_type : uint8_t { + LTO_IR, //< LTO-IR binary + FATBIN //< FATBIN binary +}; + +/** + * @brief Creates a new column by applying a transform function against every + * element of the input columns. + * + * Computes: + * `(output[i]) = UDF(input[i])`. + * + * + * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) + * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string + * types + * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and + * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in + * this case. + * @throws std::invalid_argument if string offsets are provided for non-string output columns, or + * if the number of string offsets does not match the number of output columns. + * + * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from + * the input and pre-allocated output columns. + * + * @param input Immutable view of the input to transform + * @param output Specification of the output column to be created + * @param udf The LTO-IR string of the transform function to apply + * @param binary_type The type of the LTO binary provided in `udf` + * @param is_null_aware Signifies the UDF will receive row inputs as optional values + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + * @return A column resulting from applying the transform function to every element of + * the input according to the output specifications + * + */ +std::unique_ptr unary_op_lto( + column_view input, + transform_output output, + std::span udf, + lto_binary_type binary_type, + null_aware is_null_aware, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + +/** + * @brief Creates a new column by applying a transform function against every + * element of the input columns. + * + * Computes: + * `(output[i]...) = UDF(inputs[i]...)`. + * + * + * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) + * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string + * types + * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and + * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in + * this case. + * @throws std::invalid_argument if string offsets are provided for non-string output columns, or + * if the number of string offsets does not match the number of output columns. + * + * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from + * the input and pre-allocated output columns. + * + * @param lhs Immutable view of the left-hand side input to transform + * @param rhs Immutable view of the right-hand side input to transform (can be a column or + * scalar) + * @param output Specification of the output column to be created + * @param udf The LTO-IR string of the transform function to apply + * @param binary_type The type of the LTO binary provided in `udf` + * @param is_null_aware Signifies the UDF will receive row inputs as optional values + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + * @return A column resulting from applying the transform function to every element of + * the input according to the output specifications + * + */ +std::unique_ptr binary_op_lto( + column_view lhs, + transform_input rhs, + transform_output output, + std::span udf, + lto_binary_type binary_type, + null_aware is_null_aware, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + /** * @brief Creates a null_mask from `input` by converting `NaN` to null and * preserving existing null values and also returns new null_count. diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index a2b60635fa4b..c44726efbacb 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -26,6 +26,7 @@ function(add_embed TARGET) endif() add_library(${TARGET}__embed_props INTERFACE) + set_property(TARGET ${TARGET}__embed_props PROPERTY EMBED_FILE_INDEX 0) endfunction() # This function registers a directory of include files to be embedded for JIT compilation. @@ -108,6 +109,15 @@ function(embed_includes TARGET) PROPERTY EMBED_INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES} ) + get_property( + SOURCE_FILE_IDS + TARGET ${TARGET}__embed_props + PROPERTY EMBED_SOURCE_FILE_IDS + ) + list(LENGTH SOURCE_FILE_IDS IDX) + + set_property(TARGET ${TARGET}__embed_props PROPERTY EMBED_FILE_INDEX ${IDX}) + endfunction() # This function registers a single file to be embedded for JIT compilation. @@ -181,6 +191,15 @@ function(embed_blob TARGET) PROPERTY EMBED_SOURCE_FILE_DESTS ${ARG_DEST} ) + get_property( + SOURCE_FILE_IDS + TARGET ${TARGET}__embed_props + PROPERTY EMBED_SOURCE_FILE_IDS + ) + list(LENGTH SOURCE_FILE_IDS IDX) + + set_property(TARGET ${TARGET}__embed_props PROPERTY EMBED_FILE_INDEX ${IDX}) + endfunction() # This function generates the necessary files and build targets to embed the registered source files diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index fb36fe4816a3..0f4d0afc8353 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -39,6 +39,22 @@ rtcx::sha256 hash(std::span inputs) return ctx.finalize(); } +rtcx::sha256 hash(std::span file_fragments, + std::span memory_fragments) +{ + rtcx::sha256_context ctx; + for (auto const& fragment : file_fragments) { + ctx.update( + std::span{reinterpret_cast(fragment.path), std::strlen(fragment.path)}); + } + + for (auto const& fragment : memory_fragments) { + ctx.update(fragment.data); + } + + return ctx.finalize(); +} + void install_file_set(std::string_view target_dir, std::span compressed_binary, size_t uncompressed_size, @@ -339,4 +355,88 @@ kernel_instance={} return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; } +std::tuple link_library_uncached( + char const* name, + std::span file_fragments, + std::span memory_fragments) +{ + CUDF_FUNC_RANGE(); + + auto sm = get_current_device_physical_model(); + auto& ctx = cudf::get_context(); + auto& cfg = ctx.config(); + + std::vector options; + + options.emplace_back("-lto"); + options.emplace_back(std::format("-arch=sm_{}", sm)); + + if (cfg.disable_cuda_cache) { options.emplace_back("--no-cache"); } + + if (cfg.jit_verbose) { options.emplace_back("-verbose"); } + + if (cfg.dump_jit_trace) { options.emplace_back("-time"); } + + std::vector options_cstr; + + for (auto const& option : options) { + options_cstr.emplace_back(option.c_str()); + } + + auto params = rtcx::link_params{.name = name, + .output_type = rtcx::binary_type::CUBIN, + .file_fragments = file_fragments, + .memory_fragments = memory_fragments, + .link_options = options_cstr}; + + auto cubin = rtcx::link_library(params); + auto library = rtcx::load_library(cubin); + auto blob = rtcx::blob_t::from_buffer(std::move(cubin)); + + return std::make_tuple(library, std::make_shared(std::move(blob))); +} + +kernel get_lto_linked_kernel(std::string const& name, + std::span file_fragments, + std::span memory_fragments) +{ + CUDF_FUNC_RANGE(); + + auto& cache = cudf::get_context().rtcx_cache(); + auto& bundle = cudf::get_context().jit_bundle(); + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_physical_model(); + auto bundle_hash = bundle.get_hash(); + auto fragments_hash = hash(file_fragments, memory_fragments).to_hex_string(); + + auto cache_key = std::format(R"***(cuLibrary +name={} +binary_type=CUBIN +cuda_runtime={} +cuda_driver={} +arch={} +bundle={} +fragments={} +)***", + name, + runtime, + driver, + sm, + bundle_hash, + fragments_hash.view()); + + auto cache_key_sha256 = hash(cache_key); + + auto compile = [&] { + return link_library_uncached(name.c_str(), file_fragments, memory_fragments); + }; + + auto fut = + cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); + + auto lib = fut.get(); + return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; +} + } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/jit/cache.hpp b/cpp/src/jit/cache.hpp index 679f8011fa3e..8525c16dde0e 100644 --- a/cpp/src/jit/cache.hpp +++ b/cpp/src/jit/cache.hpp @@ -88,4 +88,15 @@ kernel get_kernel(std::string const& name, std::span headers, std::string const& kernel_instance); +/** + * @brief Gets a kernel by linking together embedded binary fragments + * @param name Debug name for the kernel (used for caching and logging) + * @param file_fragments Paths of the fragments to link together to form the kernel + * @param memory_fragments Memory fragments to link + * @param extra_options Additional linking options + */ +kernel get_lto_linked_kernel(std::string const& name, + std::span file_fragments, + std::span memory_fragments); + } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 17e410ceafc4..badfdd7463b5 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -35,18 +37,6 @@ namespace cudf { namespace jit { -template -__device__ void execute_transform_op(void* user_data, size_type element_idx, Args args) -{ - // TODO: static assert invocable - if constexpr (has_user_data) { - cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, - cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, args)); - } else { - cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, args); - } -} - /// @brief The generic transform kernel. Supports all types and nullability combinations. template __device__ void transform_kernel(size_type row_size, @@ -58,48 +48,69 @@ __device__ void transform_kernel(size_type row_size, auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); - for (auto element_idx = start; element_idx < row_size; element_idx += stride) { + for (auto row = start; row < row_size; row += stride) { +#ifndef CUDF_LTO_MODE + + auto operation = [&](Args const& args) { + if constexpr (has_user_data) { + cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, + cuda::std::tuple_cat(cuda::std::tuple{user_data, row}, args)); + } else { + cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, args); + } + }; + +#else + + auto operation = [&](Args const& args) { + static_assert(!has_user_data); + static_assert(OutputAccessors::size == 1); + static_assert(InputAccessors::size == 2 || InputAccessors::size == 1); + cuda::std::apply( + [&](auto... a) { + if constexpr (InputAccessors::size == 1) { + cudf::lto::unary_operator(a...); + } else if constexpr (InputAccessors::size == 2) { + cudf::lto::binary_operator(a...); + } + }, + args); + }; + +#endif + if constexpr (!is_null_aware) { - if (stencil != nullptr && !bit_is_set(stencil, element_idx)) { continue; } + if (stencil != nullptr && !bit_is_set(stencil, row)) { continue; } auto ins = InputAccessors::map( - [&]() { return cuda::std::tuple{A::element(input_cols, element_idx)...}; }); + [&]() { return cuda::std::tuple{A::element(input_cols, row)...}; }); - auto outs = OutputAccessors::map([&]() { - return cuda::std::tuple{A::output_arg(output_cols, element_idx)...}; - }); + auto outs = OutputAccessors::map( + [&]() { return cuda::std::tuple{A::output_arg(output_cols, row)...}; }); - auto out_ptrs = - cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - - execute_transform_op( - user_data, element_idx, cuda::std::tuple_cat(out_ptrs, ins)); + operation(cuda::std::tuple_cat( + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs), ins)); OutputAccessors::map([&]() { - (A::assign(output_cols, element_idx, cuda::std::get(outs)), ...); + (A::assign(output_cols, row, cuda::std::get(outs)), ...); }); } else { - auto active_mask = __ballot_sync(0xFFFF'FFFFU, element_idx < row_size); - - auto ins = InputAccessors::map([&]() { - return cuda::std::tuple{A::nullable_element(input_cols, element_idx)...}; - }); + auto ins = InputAccessors::map( + [&]() { return cuda::std::tuple{A::nullable_element(input_cols, row)...}; }); - auto outs = OutputAccessors::map([&]() { - return cuda::std::tuple{A::null_output_arg(output_cols, element_idx)...}; - }); + auto outs = OutputAccessors::map( + [&]() { return cuda::std::tuple{A::null_output_arg(output_cols, row)...}; }); - auto out_ptrs = - cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); + operation(cuda::std::tuple_cat( + cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs), ins)); - execute_transform_op( - user_data, element_idx, cuda::std::tuple_cat(out_ptrs, ins)); + auto active_mask = __ballot_sync(0xFFFF'FFFFU, row < row_size); OutputAccessors::map([&]() { - (A::assign(output_cols, element_idx, *cuda::std::get(outs)), ...); + (A::assign(output_cols, row, *cuda::std::get(outs)), ...); (warp_compact_validity( - active_mask, output_cols, element_idx, cuda::std::get(outs).has_value()), + active_mask, output_cols, row, cuda::std::get(outs).has_value()), ...); }); } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 7a91c3ca7945..ac6a060dfbdf 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -23,6 +23,7 @@ #include +#include #include #include #include @@ -504,14 +505,15 @@ auto get_null_transformation(null_aware is_null_aware, return output_may_be_nullable; } -void perform_checks(udf_source_type source_type, +void perform_checks(std::variant source_type, null_aware is_null_aware, std::optional in_row_size, std::span inputs, std::span outputs, std::span const> string_offsets) { - if (source_type == udf_source_type::PTX) { + if (auto* udf_source = std::get_if(&source_type); + udf_source != nullptr && *udf_source == udf_source_type::PTX) { CUDF_EXPECTS(std::none_of(inputs.begin(), inputs.end(), [](auto& in) { @@ -533,6 +535,28 @@ void perform_checks(udf_source_type source_type, CUDF_EXPECTS(is_null_aware == null_aware::NO, "PTX UDFs do not support null-aware transformations", std::invalid_argument); + } else if (std::holds_alternative(source_type)) { + [[maybe_unused]] auto binary_type = std::get(source_type); + CUDF_EXPECTS( + std::none_of(inputs.begin(), + inputs.end(), + [](auto& in) { + return std::visit( + [](auto& c) { + return !is_fixed_width(c.type()) && c.type().id() != type_id::STRING; + }, + in); + }), + "Transforms with LTO binaries only support fixed-width types and strings as inputs", + std::invalid_argument); + CUDF_EXPECTS( + std::none_of( + outputs.begin(), outputs.end(), [](auto& out) { return !is_fixed_width(out.type); }), + "Transforms with LTO binaries only support output of fixed-width types", + std::invalid_argument); + CUDF_EXPECTS(inputs.size() == 2 || inputs.size() == 1, + "LTO binary transforms only support 1 or 2 inputs", + std::invalid_argument); } CUDF_EXPECTS(std::none_of(outputs.begin(), @@ -952,4 +976,199 @@ std::unique_ptr compute_column_jit(table_view const& table, return std::move(cols[0]); } +std::string_view as_tag(type_id id) +{ + switch (id) { + case type_id::BOOL8: return "bool8"; + case type_id::INT8: return "int8"; + case type_id::INT16: return "int16"; + case type_id::INT32: return "int32"; + case type_id::INT64: return "int64"; + case type_id::UINT8: return "uint8"; + case type_id::UINT16: return "uint16"; + case type_id::UINT32: return "uint32"; + case type_id::UINT64: return "uint64"; + case type_id::FLOAT32: return "float32"; + case type_id::FLOAT64: return "float64"; + case type_id::STRING: return "string_view"; + case type_id::DECIMAL32: return "decimal32"; + case type_id::DECIMAL64: return "decimal64"; + case type_id::DECIMAL128: return "decimal128"; + case type_id::DURATION_DAYS: return "duration_D"; + case type_id::DURATION_SECONDS: return "duration_s"; + case type_id::DURATION_MILLISECONDS: return "duration_ms"; + case type_id::DURATION_MICROSECONDS: return "duration_us"; + case type_id::DURATION_NANOSECONDS: return "duration_ns"; + case type_id::TIMESTAMP_DAYS: return "timestamp_D"; + case type_id::TIMESTAMP_SECONDS: return "timestamp_s"; + case type_id::TIMESTAMP_MILLISECONDS: return "timestamp_ms"; + case type_id::TIMESTAMP_MICROSECONDS: return "timestamp_us"; + case type_id::TIMESTAMP_NANOSECONDS: return "timestamp_ns"; + default: CUDF_FAIL("Unsupported type for JIT dispatch", std::invalid_argument); + } +} + +std::optional> dispatch_unop_lto_kernel( + bool null_aware, + std::span inputs, + std::span outputs) +{ + auto input_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); + + for (size_t i = 0; i < std::size(cudf_fragments::unop_lto_kernel_FILE_INDEX); i++) { + auto FILE_INDEX = cudf_fragments::unop_lto_kernel_FILE_INDEX[i]; + auto NULL_AWARE = cudf_fragments::unop_lto_kernel_NULL_AWARE[i]; + auto TYPE = cudf_fragments::unop_lto_kernel_TYPE[i]; + if (as_tag(input_type) == TYPE && null_aware == NULL_AWARE) { + auto range = cudf_fragments::file_ranges[FILE_INDEX]; + return cudf_fragments::files.subspan(range[0], range[1]); + } + } + + return std::nullopt; +} + +std::optional> dispatch_binop_lto_kernel( + bool null_aware, + std::span inputs, + std::span outputs) +{ + auto input_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); + auto lhs_is_scalar = std::holds_alternative(inputs[0]); + auto rhs_is_scalar = std::holds_alternative(inputs[1]); + + if (lhs_is_scalar) { return std::nullopt; } + + for (size_t i = 0; i < std::size(cudf_fragments::binop_lto_kernel_FILE_INDEX); i++) { + auto FILE_INDEX = cudf_fragments::binop_lto_kernel_FILE_INDEX[i]; + auto NULL_AWARE = cudf_fragments::binop_lto_kernel_NULL_AWARE[i]; + auto TYPE = cudf_fragments::binop_lto_kernel_TYPE[i]; + auto RHS_IS_SCALAR = cudf_fragments::binop_lto_kernel_RHS_IS_SCALAR[i]; + if (as_tag(input_type) == TYPE && null_aware == NULL_AWARE && rhs_is_scalar == RHS_IS_SCALAR) { + auto range = cudf_fragments::file_ranges[FILE_INDEX]; + return cudf_fragments::files.subspan(range[0], range[1]); + } + } + + return std::nullopt; +} + +// Dispatches to the appropriate LTO kernel based on the number of inputs and outputs, their types, +// and nullability. +std::span dispatch_lto_kernel(bool null_aware, + std::span inputs, + std::span outputs) +{ + if (inputs.size() == 1 && outputs.size() == 1) { + auto input0_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); + auto output_type = outputs[0].type.id(); + if (input0_type == output_type && is_fixed_width(data_type{input0_type})) { + if (auto kernel = dispatch_unop_lto_kernel(null_aware, inputs, outputs)) { return *kernel; } + } + } + + if (inputs.size() == 2 && outputs.size() == 1) { + auto input0_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); + auto input1_type = std::visit([](auto& c) { return c.type().id(); }, inputs[1]); + auto output_type = outputs[0].type.id(); + if (input0_type == output_type && input1_type == output_type && + is_fixed_width(data_type{input0_type})) { + if (auto kernel = dispatch_binop_lto_kernel(null_aware, inputs, outputs)) { return *kernel; } + } + } + + CUDF_FAIL("No suitable LTO kernel found for the given transform parameters", + std::invalid_argument); +} + +rtcx::binary_type as_rtcx_binary_type(lto_binary_type type) +{ + switch (type) { + case lto_binary_type::LTO_IR: return rtcx::binary_type::LTO_IR; + case lto_binary_type::FATBIN: return rtcx::binary_type::FATBIN; + default: + CUDF_FAIL( + std::format("Unrecognized LTO binary type {} for LTO transform", static_cast(type)), + std::invalid_argument); + } +} + +std::unique_ptr
transform_lto(std::span inputs, + std::span udf, + lto_binary_type binary_type, + std::span outputs, + null_aware is_null_aware, + std::optional in_row_size, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + perform_checks(binary_type, is_null_aware, in_row_size, inputs, outputs, {}); + auto row_size = in_row_size.has_value() ? *in_row_size : jit::get_projection_size(inputs); + auto output_may_be_nullable = get_null_transformation(is_null_aware, inputs, outputs); + + auto [output_columns, stencil] = + make_outputs(is_null_aware, row_size, inputs, outputs, output_may_be_nullable, {}, stream, mr); + auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; + auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; + auto kernel_fatbin = dispatch_lto_kernel(is_null_aware == null_aware::YES, inputs, outputs); + + rtcx::memory_fragment fragments[] = { + {.data = kernel_fatbin, .type = rtcx::binary_type::FATBIN, .name = "kernel"}, + {.data = udf, .type = as_rtcx_binary_type(binary_type), .name = "udf"}}; + + auto kernel = get_lto_linked_kernel("transform_lto_kernel", {}, fragments); + auto [cols, handles] = jit_transform::to_args(inputs, output_columns, stream, mr); + cudf::size_type num_inputs = static_cast(inputs.size()); + cudf::size_type num_outputs = static_cast(outputs.size()); + cudf::size_type num_rows = row_size; + cudf::bitmask_type* p_stencil = stencil_has_nulls ? stencil_arg : nullptr; + auto* input_cols = reinterpret_cast(cols.data()); + auto* output_cols = + reinterpret_cast(input_cols + inputs.size()); + + auto cfg = kernel.max_occupancy_config(0, 0); + void* user_data = nullptr; + + void* args[] = {&num_rows, &p_stencil, &user_data, &input_cols, &output_cols}; + + kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); + + auto finalized = finalize_outputs(is_null_aware, row_size, std::move(output_columns), stream, mr); + return std::make_unique
(std::move(finalized)); +} + +std::unique_ptr unary_op_lto(column_view input, + transform_output output, + std::span udf, + lto_binary_type binary_type, + null_aware is_null_aware, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + transform_input inputs[] = {input}; + transform_output outputs[] = {output}; + auto table = + transform_lto(inputs, udf, binary_type, outputs, is_null_aware, std::nullopt, stream, mr); + auto cols = table->release(); + return std::move(cols[0]); +} + +std::unique_ptr binary_op_lto(column_view lhs, + transform_input rhs, + transform_output output, + std::span udf, + lto_binary_type binary_type, + null_aware is_null_aware, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + transform_input inputs[] = {lhs, rhs}; + transform_output outputs[] = {output}; + auto table = + transform_lto(inputs, udf, binary_type, outputs, is_null_aware, std::nullopt, stream, mr); + auto cols = table->release(); + return std::move(cols[0]); +} + } // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index ce7bfdbed0f0..df194efcdc94 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -684,6 +684,30 @@ ConfigureTest(ENCODE_TEST encode/encode_tests.cpp) # * ast tests ------------------------------------------------------------------------------------- ConfigureTest(AST_TEST ast/transform_tests.cpp ast/ast_tree_tests.cpp) +# ################################################################################################## +# * transform LTO tests +# ------------------------------------------------------------------------------------- +add_embed(cudf_test_fragments) + +add_fragment(cudf_test_fragments FRAGMENT invsqrt SOURCE transform/fragments/invsqrt.cu) + +add_fragment( + cudf_test_fragments FRAGMENT sum_of_squares SOURCE transform/fragments/sum_of_squares.cu +) + +add_fragment( + cudf_test_fragments FRAGMENT decimal_square SOURCE transform/fragments/decimal_square.cu +) + +embed(cudf_test_fragments COMPRESSION none) + +ConfigureTest( + TRANSFORM_LTO_TEST transform/transform_lto_test.cpp + ${cudf_test_fragments_SOURCE_DIR}/cudf_test_fragments.s +) +add_dependencies(TRANSFORM_LTO_TEST cudf_test_fragments) +target_include_directories(TRANSFORM_LTO_TEST PRIVATE ${cudf_test_fragments_SOURCE_DIR}) + # ################################################################################################## # * lists tests ---------------------------------------------------------------------------------- ConfigureTest( diff --git a/cpp/tests/transform/fragments/decimal_square.cu b/cpp/tests/transform/fragments/decimal_square.cu new file mode 100644 index 000000000000..19535ec5d11e --- /dev/null +++ b/cpp/tests/transform/fragments/decimal_square.cu @@ -0,0 +1,28 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +template <> +__device__ void cudf::lto::unary_operator( + numeric::decimal32* __restrict__ out, numeric::decimal32 a) +{ + *out = a * a; +} + +template <> +__device__ void cudf::lto::unary_operator( + numeric::decimal64* __restrict__ out, numeric::decimal64 a) +{ + *out = a * a; +} + +template <> +__device__ void cudf::lto::unary_operator( + numeric::decimal128* __restrict__ out, numeric::decimal128 a) +{ + *out = a * a; +} diff --git a/cpp/tests/transform/fragments/invsqrt.cu b/cpp/tests/transform/fragments/invsqrt.cu new file mode 100644 index 000000000000..3d7a6750b00e --- /dev/null +++ b/cpp/tests/transform/fragments/invsqrt.cu @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +template <> +__device__ void cudf::lto::unary_operator(float* __restrict__ out, float a) +{ + *out = 1.0F / sqrtf(a); +} diff --git a/cpp/tests/transform/fragments/sum_of_squares.cu b/cpp/tests/transform/fragments/sum_of_squares.cu new file mode 100644 index 000000000000..6c1d233939f6 --- /dev/null +++ b/cpp/tests/transform/fragments/sum_of_squares.cu @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +template <> +__device__ void cudf::lto::binary_operator(float* __restrict__ out, + float a, + float b) +{ + *out = a * a + b * b; +} diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp new file mode 100644 index 000000000000..4dec31865b5a --- /dev/null +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +struct TransformLTOTest : public cudf::test::BaseFixture {}; + +template +using column_wrapper = cudf::test::fixed_width_column_wrapper; + +template +using decimal_wrapper = cudf::test::fixed_point_column_wrapper; + +TEST_F(TransformLTOTest, InvSqrt) +{ + column_wrapper input{{1.0f, 4.0f, 9.0f, 16.0f}}; + + cudf::transform_output output{cudf::data_type{cudf::type_id::FLOAT32}, + cudf::output_nullability::ALL_VALID}; + + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::invsqrt]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::unary_op_lto(input, + output, + udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + cudf::test::get_default_stream()); + + column_wrapper expected{{1.0f, 0.5f, 0.33333334f, 0.25f}}; + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); +} + +TEST_F(TransformLTOTest, SumOfSquares) +{ + column_wrapper lhs{{1.0f, 4.0f, 9.0f, 16.0f}}; + column_wrapper rhs{{1.0f, 2.0f, 2.0f, 10.0f}}; + + cudf::transform_output output{cudf::data_type{cudf::type_id::FLOAT32}, + cudf::output_nullability::ALL_VALID}; + + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::sum_of_squares]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::binary_op_lto(lhs, + rhs, + output, + udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + cudf::test::get_default_stream()); + + column_wrapper expected{{2.0f, 20.0f, 85.0f, 356.0f}}; + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); +} + +TEST_F(TransformLTOTest, Decimal32Square) +{ + auto test_type = []() { + decimal_wrapper input{{1, 2, 3}, numeric::scale_type{-2}}; + + cudf::transform_output output{cudf::data_type{cudf::type_to_id(), numeric::scale_type{-4}}, + cudf::output_nullability::ALL_VALID}; + + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::decimal_square]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::unary_op_lto(input, + output, + udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + cudf::test::get_default_stream()); + + decimal_wrapper expected{{1, 4, 9}, numeric::scale_type{-4}}; + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + }; + + test_type.operator()(); + test_type.operator()(); + test_type.operator()(); +} From b462e22f841b6eb73e027163d3d77ba1f6429879 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 27 May 2026 19:02:34 +0000 Subject: [PATCH 204/254] fix compile error + refactoring --- cpp/CMakeLists.txt | 26 +++++++++++++------------- cpp/src/jit/cache.cpp | 16 ++++++++-------- cpp/src/rolling/jit/kernel.cu | 24 +++++++++++++----------- 3 files changed, 34 insertions(+), 32 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bca705a56787..ca95937ff1d4 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -404,40 +404,40 @@ if(NOT BUILD_SHARED_LIBS) endif() endif() -add_embed(cudf_jit_embed) +add_embed(cudf_cuda_embed) embed_includes( - cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/librtcx/libcxx DEST_DIRECTORY + cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/librtcx/libcxx DEST_DIRECTORY librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx ) embed_includes( - cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf DEST_DIRECTORY + cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf DEST_DIRECTORY cudf/cpp/include/cudf INCLUDE_DIRECTORIES cudf/cpp/include ) embed_includes( - cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/jit DEST_DIRECTORY + cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/jit DEST_DIRECTORY cudf/cpp/src/jit INCLUDE_DIRECTORIES cudf/cpp/src ) embed_includes( - cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/binaryop/jit DEST_DIRECTORY + cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/binaryop/jit DEST_DIRECTORY cudf/cpp/src/binaryop/jit INCLUDE_DIRECTORIES cudf/cpp/src ) embed_includes( - cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/join/jit DEST_DIRECTORY + cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/join/jit DEST_DIRECTORY cudf/cpp/src/join/jit INCLUDE_DIRECTORIES cudf/cpp/src ) embed_includes( - cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling DEST_DIRECTORY + cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling DEST_DIRECTORY cudf/cpp/src/rolling INCLUDE_DIRECTORIES cudf/cpp/src ) embed_includes( - cudf_jit_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/transform/jit DEST_DIRECTORY + cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/transform/jit DEST_DIRECTORY cudf/cpp/src/transform/jit INCLUDE_DIRECTORIES cudf/cpp/src ) @@ -447,12 +447,12 @@ foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) cmake_path(GET INC_DIR FILENAME INC_DIR_NAME) embed_includes( - cudf_jit_embed SOURCE_DIRECTORY ${INC_DIR} DEST_DIRECTORY CCCL/libcudacxx/${INC_DIR_NAME} + cudf_cuda_embed SOURCE_DIRECTORY ${INC_DIR} DEST_DIRECTORY CCCL/libcudacxx/${INC_DIR_NAME} INCLUDE_DIRECTORIES CCCL/libcudacxx/${INC_DIR_NAME} ) endforeach() -embed(cudf_jit_embed COMPRESSION zstd) +embed(cudf_cuda_embed COMPRESSION zstd) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- @@ -1009,10 +1009,10 @@ add_library( src/utilities/type_checks.cpp src/utilities/type_dispatcher.cpp librtcx/rtcx.cpp - ${cudf_jit_embed_SOURCE_DIR}/cudf_jit_embed.s + ${cudf_cuda_embed_SOURCE_DIR}/cudf_cuda_embed.s ) -add_dependencies(cudf cudf_jit_embed) +add_dependencies(cudf cudf_cuda_embed) set_property( SOURCE src/io/parquet/writer_impl.cu @@ -1083,7 +1083,7 @@ target_include_directories( "$" "$" "$" - "$" + "$" INTERFACE "$" ) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index fb36fe4816a3..d7d01806c87a 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -9,7 +9,7 @@ #include -#include +#include #include #include #include @@ -96,11 +96,11 @@ void install_cudf_jit_files(std::string const& target_dir, std::string const& tm std::runtime_error); install_file_set(tmp_dir_path, - cudf_jit_embed::files, - cudf_jit_embed::files_uncompressed_size, - cudf_jit_embed::file_ranges, - cudf_jit_embed::file_destinations, - cudf_jit_embed::files_compression); + cudf_cuda_embed::files, + cudf_cuda_embed::files_uncompressed_size, + cudf_cuda_embed::file_ranges, + cudf_cuda_embed::file_destinations, + cudf_cuda_embed::files_compression); // rename the temporary directory to the target install directory if (::rename(tmp_dir_path, target_dir.c_str()) == -1) { @@ -148,7 +148,7 @@ void jit_bundle_t::ensure_installed() const std::string jit_bundle_t::get_hash() const { - auto str = rtcx::sha256_hex_string::make(cudf_jit_embed::hash); + auto str = rtcx::sha256_hex_string::make(cudf_cuda_embed::hash); return std::string{str.view()}; } @@ -162,7 +162,7 @@ std::vector jit_bundle_t::get_include_directories() const std::vector directories; auto base_dir = get_directory(); - for (auto dir : cudf_jit_embed::include_directories) { + for (auto dir : cudf_cuda_embed::include_directories) { directories.emplace_back(std::format("{}/{}", base_dir, dir)); } diff --git a/cpp/src/rolling/jit/kernel.cu b/cpp/src/rolling/jit/kernel.cu index ff6931a6f8fd..1b36099568bd 100644 --- a/cpp/src/rolling/jit/kernel.cu +++ b/cpp/src/rolling/jit/kernel.cu @@ -45,20 +45,22 @@ template -CUDF_KERNEL void rolling_window_kernel(cudf::size_type nrows, - InType const* const __restrict__ in_col, - cudf::bitmask_type const* const __restrict__ in_col_valid, - OutType* __restrict__ out_col, - cudf::bitmask_type* __restrict__ out_col_valid, - cudf::size_type* __restrict__ output_valid_count, - detail::window_wrapper_base b_preceding_window_begin, - detail::window_wrapper_base b_following_window_begin, - cudf::size_type min_periods) +__device__ void rolling_window_kernel(cudf::size_type nrows, + void const* __restrict__ p_in_col, + cudf::bitmask_type const* const __restrict__ in_col_valid, + void* __restrict__ p_out_col, + cudf::bitmask_type* __restrict__ out_col_valid, + cudf::size_type* __restrict__ output_valid_count, + detail::window_wrapper_base b_preceding_window_begin, + detail::window_wrapper_base b_following_window_begin, + cudf::size_type min_periods) { auto i = cudf::detail::grid_1d::global_thread_id(); auto const stride = cudf::detail::grid_1d::grid_stride(); PrecedingWindowType const preceding_window_begin = b_preceding_window_begin; FollowingWindowType const following_window_begin = b_following_window_begin; + auto const* const in_col = static_cast(p_in_col); + auto* const out_col = static_cast(p_out_col); cudf::size_type warp_valid_count{0}; @@ -113,8 +115,8 @@ CUDF_KERNEL void rolling_window_kernel(cudf::size_type nrows, extern "C" __global__ void cudf_kernel_entry( cudf::size_type nrows, - void const* const __restrict__ in_col, - cudf::bitmask_type const* const __restrict__ in_col_valid, + void const* __restrict__ in_col, + cudf::bitmask_type const* __restrict__ in_col_valid, void* __restrict__ out_col, cudf::bitmask_type* __restrict__ out_col_valid, cudf::size_type* __restrict__ output_valid_count, From e7c314f304e0020124e167749cc62cf1138c321e Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 31 May 2026 10:03:23 +0000 Subject: [PATCH 205/254] Refactor binary operator functions to use a unified transform operator for improved consistency and maintainability --- cpp/benchmarks/binaryop/fragments/add.cu | 8 ++----- cpp/benchmarks/binaryop/fragments/null_max.cu | 8 +++---- cpp/include/cudf/jit/transform_operator.cuh | 23 +++++-------------- cpp/src/transform/jit/kernel.cu | 12 +--------- .../transform/fragments/decimal_square.cu | 12 +++++----- cpp/tests/transform/fragments/invsqrt.cu | 2 +- .../transform/fragments/sum_of_squares.cu | 4 +--- 7 files changed, 21 insertions(+), 48 deletions(-) diff --git a/cpp/benchmarks/binaryop/fragments/add.cu b/cpp/benchmarks/binaryop/fragments/add.cu index 1d009811d4a9..c2dcfaa286e0 100644 --- a/cpp/benchmarks/binaryop/fragments/add.cu +++ b/cpp/benchmarks/binaryop/fragments/add.cu @@ -7,17 +7,13 @@ #include template <> -__device__ void cudf::lto::binary_operator(int32_t* __restrict__ out, - int32_t a, - int32_t b) +__device__ void cudf::lto::transform(int32_t* out, int32_t a, int32_t b) { *out = a + b; } template <> -__device__ void cudf::lto::binary_operator(float* __restrict__ out, - float a, - float b) +__device__ void cudf::lto::transform(float* out, float a, float b) { *out = a + b; } diff --git a/cpp/benchmarks/binaryop/fragments/null_max.cu b/cpp/benchmarks/binaryop/fragments/null_max.cu index c29d09984023..99de343811fb 100644 --- a/cpp/benchmarks/binaryop/fragments/null_max.cu +++ b/cpp/benchmarks/binaryop/fragments/null_max.cu @@ -9,10 +9,10 @@ #include template <> -__device__ void cudf::lto::binary_operator, - cuda::std::optional, - cuda::std::optional>( - cuda::std::optional* __restrict__ out, +__device__ void cudf::lto::transform*, + cuda::std::optional, + cuda::std::optional>( + cuda::std::optional* out, cuda::std::optional a, cuda::std::optional b) { diff --git a/cpp/include/cudf/jit/transform_operator.cuh b/cpp/include/cudf/jit/transform_operator.cuh index 49d15a2d4974..6cccc5f91af4 100644 --- a/cpp/include/cudf/jit/transform_operator.cuh +++ b/cpp/include/cudf/jit/transform_operator.cuh @@ -7,23 +7,12 @@ namespace cudf { namespace lto { -/// @brief The unary operator for the transform operation. -/// @tparam Out The output type of the operator. -/// @tparam In0 The input type of the operator. -/// @param out The output destination for the operator result. -/// @param a The input value for the operator. -template -__device__ void unary_operator(Out* __restrict__ out, In0 a); - -/// @brief The binary operator for the transform operation. -/// @tparam Out The output type of the operator. -/// @tparam In0 The first input type of the operator. -/// @tparam In1 The second input type of the operator. -/// @param out The output destination for the operator result. -/// @param a The first input value for the operator. -/// @param b The second input value for the operator. -template -__device__ void binary_operator(Out* __restrict__ out, In0 a, In1 b); +/// @brief The operator for the transform operation. +/// @tparam T the types of the operator arguments, which can be either input or output arguments. +/// The output argument must be the first argument if there are multiple arguments. +/// @param args The arguments for the operator. +template +__device__ void transform(T... args); } // namespace lto } // namespace cudf diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index badfdd7463b5..29d479ac771f 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -64,17 +64,7 @@ __device__ void transform_kernel(size_type row_size, auto operation = [&](Args const& args) { static_assert(!has_user_data); - static_assert(OutputAccessors::size == 1); - static_assert(InputAccessors::size == 2 || InputAccessors::size == 1); - cuda::std::apply( - [&](auto... a) { - if constexpr (InputAccessors::size == 1) { - cudf::lto::unary_operator(a...); - } else if constexpr (InputAccessors::size == 2) { - cudf::lto::binary_operator(a...); - } - }, - args); + cuda::std::apply([&](auto... a) { cudf::lto::transform(a...); }, args); }; #endif diff --git a/cpp/tests/transform/fragments/decimal_square.cu b/cpp/tests/transform/fragments/decimal_square.cu index 19535ec5d11e..6c9979964369 100644 --- a/cpp/tests/transform/fragments/decimal_square.cu +++ b/cpp/tests/transform/fragments/decimal_square.cu @@ -7,22 +7,22 @@ #include template <> -__device__ void cudf::lto::unary_operator( - numeric::decimal32* __restrict__ out, numeric::decimal32 a) +__device__ void cudf::lto::transform( + numeric::decimal32* out, numeric::decimal32 a) { *out = a * a; } template <> -__device__ void cudf::lto::unary_operator( - numeric::decimal64* __restrict__ out, numeric::decimal64 a) +__device__ void cudf::lto::transform( + numeric::decimal64* out, numeric::decimal64 a) { *out = a * a; } template <> -__device__ void cudf::lto::unary_operator( - numeric::decimal128* __restrict__ out, numeric::decimal128 a) +__device__ void cudf::lto::transform( + numeric::decimal128* out, numeric::decimal128 a) { *out = a * a; } diff --git a/cpp/tests/transform/fragments/invsqrt.cu b/cpp/tests/transform/fragments/invsqrt.cu index 3d7a6750b00e..30d763208339 100644 --- a/cpp/tests/transform/fragments/invsqrt.cu +++ b/cpp/tests/transform/fragments/invsqrt.cu @@ -6,7 +6,7 @@ #include template <> -__device__ void cudf::lto::unary_operator(float* __restrict__ out, float a) +__device__ void cudf::lto::transform(float* out, float a) { *out = 1.0F / sqrtf(a); } diff --git a/cpp/tests/transform/fragments/sum_of_squares.cu b/cpp/tests/transform/fragments/sum_of_squares.cu index 6c1d233939f6..b96dc3b3b3d8 100644 --- a/cpp/tests/transform/fragments/sum_of_squares.cu +++ b/cpp/tests/transform/fragments/sum_of_squares.cu @@ -6,9 +6,7 @@ #include template <> -__device__ void cudf::lto::binary_operator(float* __restrict__ out, - float a, - float b) +__device__ void cudf::lto::transform(float* out, float a, float b) { *out = a * a + b * b; } From 4440bd0f96a17de3dfeac2bd5d79722af078f377 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 11 Jun 2026 06:30:34 +0000 Subject: [PATCH 206/254] Refactor JIT Transform Operations and Introduce Bankers Rounding - Removed the decimal square fragment and replaced it with a new bankers rounding fragment. - Updated the JIT cache and helper files to streamline kernel retrieval and fragment instantiation. - Enhanced the transform kernel to support LTO (Link Time Optimization) for better performance. - Modified the transform tests to accommodate the new bankers rounding functionality. - Improved the reflection of UDF signatures and added support for null-aware and user data configurations. - Cleaned up redundant includes and code segments across various files. --- cpp/CMakeLists.txt | 77 ++-- cpp/benchmarks/CMakeLists.txt | 4 +- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 33 +- cpp/include/cudf/jit/transform_operator.cuh | 18 - cpp/include/cudf/jit/type_tags.cuh | 56 --- cpp/include/cudf/transform.hpp | 63 +--- cpp/librtcx/rtcx.hpp | 37 +- cpp/src/jit/cache.cpp | 153 +++++++- cpp/src/jit/cache.hpp | 47 +-- cpp/src/jit/helpers.cpp | 23 +- cpp/src/jit/helpers.hpp | 4 + cpp/src/runtime/context.hpp | 17 +- cpp/src/transform/jit/kernel.cu | 31 +- cpp/src/transform/transform.cu | 334 +++++++++--------- cpp/tests/CMakeLists.txt | 6 +- .../transform/fragments/bankers_rounding.cu | 47 +++ .../transform/fragments/decimal_square.cu | 28 -- cpp/tests/transform/fragments/invsqrt.cu | 4 +- .../transform/fragments/sum_of_squares.cu | 8 +- cpp/tests/transform/transform_lto_test.cpp | 80 +++-- 20 files changed, 571 insertions(+), 499 deletions(-) delete mode 100644 cpp/include/cudf/jit/transform_operator.cuh delete mode 100644 cpp/include/cudf/jit/type_tags.cuh create mode 100644 cpp/tests/transform/fragments/bankers_rounding.cu delete mode 100644 cpp/tests/transform/fragments/decimal_square.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 683983860859..926161f097ea 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -494,7 +494,7 @@ embed(cudf_cuda_embed COMPRESSION zstd OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUD # final library with metadata that allows it to be looked up at runtime. macro(add_fragment) set(TARGET ${ARGV0}) - set(ONE_VALUE_ARGS FRAGMENT SOURCE KERNEL_ONLY KERNEL_INSTANCE) + set(ONE_VALUE_ARGS FRAGMENT SOURCE KERNEL_ONLY KERNEL_INSTANCE UDF_TYPE) set(MULTI_VALUE_ARGS DEFINITIONS ARRAY_IDS ARRAY_VALUES) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) @@ -517,14 +517,24 @@ macro(add_fragment) target_compile_options(${OBJECT_ID} PRIVATE -Xnvlink=--kernels-used=cudf_kernel_entry) endif() + set(INSTANTIATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/${TARGET}/instantiations/${ARG_FRAGMENT}") + target_include_directories(${OBJECT_ID} PRIVATE ${INSTANTIATION_DIR}) + if(ARG_KERNEL_INSTANCE) - set(INSTANTIATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/${TARGET}/instantiations/${ARG_FRAGMENT}") file( GENERATE - OUTPUT "${INSTANTIATION_DIR}/cudf/detail/kernel-instance.hpp" + OUTPUT "${INSTANTIATION_DIR}/cudf/detail/kernel_instance.cuh" CONTENT "#pragma once\n#define CUDF_KERNEL_INSTANCE ${ARG_KERNEL_INSTANCE}" ) - target_include_directories(${OBJECT_ID} PRIVATE ${INSTANTIATION_DIR}) + endif() + + if(ARG_UDF_TYPE) + set(INSTANTIATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/${TARGET}/instantiations/${ARG_FRAGMENT}") + file( + GENERATE + OUTPUT "${INSTANTIATION_DIR}/cudf/detail/operation_udf.cuh" + CONTENT "#pragma once\n#define CUDF_UDF_TYPE ${ARG_UDF_TYPE}" + ) endif() target_compile_definitions(${OBJECT_ID} PRIVATE CUDF_DISABLE_EXPORTS ${ARG_DEFINITIONS}) @@ -552,6 +562,11 @@ macro(add_fragment) ${OBJECT_ID} PRIVATE "$" "$" ) + target_compile_options( + ${OBJECT_ID} PRIVATE + "$<$:${CUDF_CUDA_FLAGS}>" + "$<$:--gen-opt-lto>" + ) embed_blob( ${TARGET} FILE $ DEST fragments/${ARG_FRAGMENT}.fatbin ID @@ -561,17 +576,19 @@ endmacro() add_embed(cudf_fragments) -foreach(TYPE IN ITEMS uint8 uint16 uint32 int8 int16 int32 int64 float32 float64 uint64 decimal32 - decimal64 decimal128 +foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t int64_t float + double numeric::decimal32 numeric::decimal64 numeric::decimal128 ) - foreach(NULL_AWARE IN ITEMS 0 1) - set(FRAGMENT_NAME unop_lto_kernel) - set(VARIANT_NAME ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__element_${TYPE}) + set(FRAGMENT_NAME transform_kernel) get_property( FILE_INDEX TARGET cudf_fragments__embed_props PROPERTY EMBED_FILE_INDEX ) + set(VARIANT_NAME transform_kernel_${FILE_INDEX}) + set(INSTANCE + "cudf::jit::transform_kernel>, cudf::jit::type_list>>" + ) add_fragment( cudf_fragments FRAGMENT @@ -579,35 +596,34 @@ foreach(TYPE IN ITEMS uint8 uint16 uint32 int8 int16 int32 int64 float32 float64 SOURCE src/transform/jit/kernel.cu KERNEL_INSTANCE - "cudf::jit::transform_kernel<${NULL_AWARE}, false, cudf::jit::type_list>, cudf::jit::type_list>>" + ${INSTANCE} + UDF_TYPE + "void(${TYPE} *, ${TYPE})" DEFINITIONS CUDF_LTO_MODE ARRAY_IDS ${FRAGMENT_NAME}_FILE_INDEX - ${FRAGMENT_NAME}_NULL_AWARE - ${FRAGMENT_NAME}_TYPE + ${FRAGMENT_NAME}_INSTANCE ARRAY_VALUES ${FILE_INDEX} - ${NULL_AWARE} - ${TYPE} + "${INSTANCE}" ) endforeach() -endforeach() -foreach(TYPE IN ITEMS uint8 uint16 uint32 uint64 int8 int16 int32 int64 float32 float64 decimal32 - decimal64 decimal128 +foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t int64_t float + double numeric::decimal32 numeric::decimal64 numeric::decimal128 ) - foreach(NULL_AWARE IN ITEMS 0 1) - foreach(RHS_IS_SCALAR IN ITEMS 0 1) - set(FRAGMENT_NAME binop_lto_kernel) - set(VARIANT_NAME - ${FRAGMENT_NAME}__null_aware_${NULL_AWARE}__element_${TYPE}__rhs_is_scalar_${RHS_IS_SCALAR} - ) + foreach(RHS_IS_SCALAR IN ITEMS "false" "true") + set(FRAGMENT_NAME transform_kernel) get_property( FILE_INDEX TARGET cudf_fragments__embed_props PROPERTY EMBED_FILE_INDEX ) + set(VARIANT_NAME transform_kernel_${FILE_INDEX}) + set(INSTANCE + "cudf::jit::transform_kernel, cudf::jit::column_accessor<1ULL, cudf::column_device_view_core, ${TYPE}, ${RHS_IS_SCALAR}, 0>>, cudf::jit::type_list>>" + ) add_fragment( cudf_fragments FRAGMENT @@ -615,25 +631,22 @@ foreach(TYPE IN ITEMS uint8 uint16 uint32 uint64 int8 int16 int32 int64 float32 SOURCE src/transform/jit/kernel.cu KERNEL_INSTANCE - "cudf::jit::transform_kernel<${NULL_AWARE}, false, cudf::jit::type_list, cudf::jit::column_accessor<1, cudf::column_device_view_core, cudf::tags::${TYPE}, ${RHS_IS_SCALAR}, 0>>, cudf::jit::type_list>>" + ${INSTANCE} + UDF_TYPE + "void(${TYPE} *, ${TYPE}, ${TYPE})" DEFINITIONS CUDF_LTO_MODE ARRAY_IDS ${FRAGMENT_NAME}_FILE_INDEX - ${FRAGMENT_NAME}_NULL_AWARE - ${FRAGMENT_NAME}_TYPE - ${FRAGMENT_NAME}_RHS_IS_SCALAR + ${FRAGMENT_NAME}_INSTANCE ARRAY_VALUES ${FILE_INDEX} - ${NULL_AWARE} - ${TYPE} - ${RHS_IS_SCALAR} + "${INSTANCE}" ) endforeach() endforeach() -endforeach() -embed(cudf_fragments COMPRESSION none) +embed(cudf_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") # ################################################################################################## # * library targets ------------------------------------------------------------------------------- diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 842541794569..d9015e10f268 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -391,7 +391,9 @@ ConfigureNVBench(AST_NVBENCH ast/polynomials.cpp ast/transform.cpp) add_embed(cudf_benchmark_fragments) add_fragment(cudf_benchmark_fragments FRAGMENT add SOURCE binaryop/fragments/add.cu) add_fragment(cudf_benchmark_fragments FRAGMENT null_max SOURCE binaryop/fragments/null_max.cu) -embed(cudf_benchmark_fragments COMPRESSION none) +embed(cudf_benchmark_fragments COMPRESSION none OUTPUT_DIRECTORY + "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" +) # ################################################################################################## # * binaryop benchmark ---------------------------------------------------------------------------- diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 864ecb7fd1d4..b048c6e60408 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -132,19 +132,22 @@ void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) } // Call once for hot cache. - cudf::transform_output output{cudf::data_type{cudf::type_to_id()}, - cudf::output_nullability::ALL_VALID}; + cudf::transform_input inputs[] = {lhs, rhs}; + cudf::transform_output outputs[] = {{cudf::data_type{cudf::type_to_id()}, + cudf::output_nullability::ALL_VALID}}; auto const range = cudf_benchmark_fragments::file_ranges[fragment_id]; std::span udf{cudf_benchmark_fragments::files.subspan(range[0], range[1])}; - auto result = cudf::binary_op_lto(source_table->get_column(0), - source_table->get_column(1), - output, - udf, + auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, - null_aware ? cudf::null_aware::YES : cudf::null_aware::NO - ); + null_aware ? cudf::null_aware::YES : cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt); + // use number of bytes read and written to global memory state.add_global_memory_reads(num_rows); @@ -152,12 +155,14 @@ void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) state.add_global_memory_writes(num_rows); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch&) { - cudf::binary_op_lto(source_table->get_column(0), - source_table->get_column(1), - output, - udf, - cudf::lto_binary_type::FATBIN, - null_aware ? cudf::null_aware::YES : cudf::null_aware::NO); + cudf::transform_lto(udf, + cudf::lto_binary_type::FATBIN, + null_aware ? cudf::null_aware::YES : cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt); }); } diff --git a/cpp/include/cudf/jit/transform_operator.cuh b/cpp/include/cudf/jit/transform_operator.cuh deleted file mode 100644 index 6cccc5f91af4..000000000000 --- a/cpp/include/cudf/jit/transform_operator.cuh +++ /dev/null @@ -1,18 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -namespace cudf { -namespace lto { - -/// @brief The operator for the transform operation. -/// @tparam T the types of the operator arguments, which can be either input or output arguments. -/// The output argument must be the first argument if there are multiple arguments. -/// @param args The arguments for the operator. -template -__device__ void transform(T... args); - -} // namespace lto -} // namespace cudf diff --git a/cpp/include/cudf/jit/type_tags.cuh b/cpp/include/cudf/jit/type_tags.cuh deleted file mode 100644 index 15a4e7caef9f..000000000000 --- a/cpp/include/cudf/jit/type_tags.cuh +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include -#include -#include -#include - -namespace CUDF_EXPORT cudf { -namespace tags { - -template -inline constexpr char const* tag_of = "void"; - -#define CUDF_TYPE_TAG(type, tag) \ - using tag = type; \ - template <> \ - inline constexpr char const* tag_of = #tag; - -CUDF_TYPE_TAG(bool, bool8); -CUDF_TYPE_TAG(int8_t, int8); -CUDF_TYPE_TAG(int16_t, int16); -CUDF_TYPE_TAG(int32_t, int32); -CUDF_TYPE_TAG(int64_t, int64); -CUDF_TYPE_TAG(__int128_t, int128); -CUDF_TYPE_TAG(uint8_t, uint8); -CUDF_TYPE_TAG(uint16_t, uint16); -CUDF_TYPE_TAG(uint32_t, uint32); -CUDF_TYPE_TAG(uint64_t, uint64); -CUDF_TYPE_TAG(__uint128_t, uint128); -CUDF_TYPE_TAG(float, float32); -CUDF_TYPE_TAG(double, float64); -CUDF_TYPE_TAG(cudf::string_view, string_view); -CUDF_TYPE_TAG(numeric::decimal32, decimal32); -CUDF_TYPE_TAG(numeric::decimal64, decimal64); -CUDF_TYPE_TAG(numeric::decimal128, decimal128); -CUDF_TYPE_TAG(cudf::duration_D, duration_D); -CUDF_TYPE_TAG(cudf::duration_h, duration_h); -CUDF_TYPE_TAG(cudf::duration_m, duration_m); -CUDF_TYPE_TAG(cudf::duration_s, duration_s); -CUDF_TYPE_TAG(cudf::duration_ms, duration_ms); -CUDF_TYPE_TAG(cudf::duration_us, duration_us); -CUDF_TYPE_TAG(cudf::duration_ns, duration_ns); -CUDF_TYPE_TAG(cudf::timestamp_D, timestamp_D); -CUDF_TYPE_TAG(cudf::timestamp_h, timestamp_h); -CUDF_TYPE_TAG(cudf::timestamp_m, timestamp_m); -CUDF_TYPE_TAG(cudf::timestamp_s, timestamp_s); -CUDF_TYPE_TAG(cudf::timestamp_ms, timestamp_ms); -CUDF_TYPE_TAG(cudf::timestamp_us, timestamp_us); -CUDF_TYPE_TAG(cudf::timestamp_ns, timestamp_ns); - -} // namespace tags -} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 1885e214281c..2829a1b9fbf5 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -207,66 +207,31 @@ enum class lto_binary_type : uint8_t { * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from * the input and pre-allocated output columns. * - * @param input Immutable view of the input to transform - * @param output Specification of the output column to be created - * @param udf The LTO-IR string of the transform function to apply - * @param binary_type The type of the LTO binary provided in `udf` - * @param is_null_aware Signifies the UDF will receive row inputs as optional values - * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned column's device memory - * @return A column resulting from applying the transform function to every element of - * the input according to the output specifications - * - */ -std::unique_ptr unary_op_lto( - column_view input, - transform_output output, - std::span udf, - lto_binary_type binary_type, - null_aware is_null_aware, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); - -/** - * @brief Creates a new column by applying a transform function against every - * element of the input columns. - * - * Computes: - * `(output[i]...) = UDF(inputs[i]...)`. - * - * - * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) - * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string - * types - * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and - * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in - * this case. - * @throws std::invalid_argument if string offsets are provided for non-string output columns, or - * if the number of string offsets does not match the number of output columns. - * - * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from - * the input and pre-allocated output columns. - * - * @param lhs Immutable view of the left-hand side input to transform - * @param rhs Immutable view of the right-hand side input to transform (can be a column or - * scalar) - * @param output Specification of the output column to be created * @param udf The LTO-IR string of the transform function to apply * @param binary_type The type of the LTO binary provided in `udf` * @param is_null_aware Signifies the UDF will receive row inputs as optional values + * @param user_data User-defined device data to pass to the UDF. + * @param inputs Immutable view of the input to transform + * @param outputs Specification of the output columns to be created + * @param string_offsets For string output columns, the offsets can be pre-allocated and passed in + * to prevent overhead of compacting string views into run-end strings column. + * @param row_size The row size of the transform operation. If not provided, it is inferred + * from the input columns. * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory - * @return A column resulting from applying the transform function to every element of + * @return A table resulting from applying the transform function to every element of * the input according to the output specifications * */ -std::unique_ptr binary_op_lto( - column_view lhs, - transform_input rhs, - transform_output output, +std::unique_ptr
transform_lto( std::span udf, lto_binary_type binary_type, null_aware is_null_aware, + std::optional user_data, + std::span inputs, + std::span outputs, + std::vector>&& string_offsets, + std::optional row_size, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index bc3ff8fd5569..87d55436db1c 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -819,7 +819,7 @@ std::string reflect(T value) = delete; template <> inline std::string reflect(bool value) { - return std::format("(bool){}", value); + return std::format("{}", value); } /** @@ -830,7 +830,7 @@ inline std::string reflect(bool value) template <> inline std::string reflect(std::uint8_t value) { - return std::format("(unsigned char){}U", value); + return std::format("{}U", value); } /** @@ -841,7 +841,7 @@ inline std::string reflect(std::uint8_t value) template <> inline std::string reflect(std::uint16_t value) { - return std::format("(unsigned short){}U", value); + return std::format("{}U", value); } /** @@ -852,7 +852,7 @@ inline std::string reflect(std::uint16_t value) template <> inline std::string reflect(std::uint32_t value) { - return std::format("(unsigned int){}U", value); + return std::format("{}U", value); } /** @@ -863,7 +863,7 @@ inline std::string reflect(std::uint32_t value) template <> inline std::string reflect(std::uint64_t value) { - return std::format("(unsigned long long int){}ULL", value); + return std::format("{}ULL", value); } /** @@ -874,7 +874,7 @@ inline std::string reflect(std::uint64_t value) template <> inline std::string reflect(std::int8_t value) { - return std::format("(signed char){}", value); + return std::format("{}", value); } /** @@ -885,7 +885,7 @@ inline std::string reflect(std::int8_t value) template <> inline std::string reflect(std::int16_t value) { - return std::format("(signed short){}", value); + return std::format("{}", value); } /** @@ -896,7 +896,7 @@ inline std::string reflect(std::int16_t value) template <> inline std::string reflect(std::int32_t value) { - return std::format("(signed int){}", value); + return std::format("{}", value); } /** @@ -907,7 +907,7 @@ inline std::string reflect(std::int32_t value) template <> inline std::string reflect(std::int64_t value) { - return std::format("(signed long long int){}LL", value); + return std::format("{}LL", value); } /** @@ -918,7 +918,7 @@ inline std::string reflect(std::int64_t value) template <> inline std::string reflect(float value) { - return std::format("(float){}F", value); + return std::format("{}F", value); } /** @@ -929,20 +929,7 @@ inline std::string reflect(float value) template <> inline std::string reflect(double value) { - return std::format("(double){}", value); -} - -/** - * @brief Reflect a value of any type into its CUDA string representation, given the type name as a - * string - * @param type The name of the type to be reflected (e.g., "int", "float", "MyStruct", etc.) - * @param value The string representation of the value to be reflected, which will be used in the - * resulting CUDA code - * @return A string containing the CUDA representation of the value with the specified type - */ -inline std::string reflect_cast(std::string_view type, std::string_view value) -{ - return std::format("(({})({}))", type, value); + return std::format("{}", value); } /** @@ -959,7 +946,7 @@ template requires(std::is_enum_v) std::string reflect_enum(std::string_view type, T value) { - return reflect_cast(type, reflect(static_cast>(value))); + return std::format("{}{}{}{}", type, "{", static_cast>(value), "}"); } /** diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 057c3d4721f0..852e62e943c2 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -39,6 +39,20 @@ rtcx::sha256 hash(std::span inputs) return ctx.finalize(); } +rtcx::sha256 hash(std::span file_fragments, + std::span memory_fragments) +{ + rtcx::sha256_context ctx; + for (auto const& fragment : file_fragments) { + ctx.update( + std::span{reinterpret_cast(fragment.path), std::strlen(fragment.path)}); + } + for (auto const& fragment : memory_fragments) { + ctx.update(fragment.data); + } + return ctx.finalize(); +} + void install_file_set( std::string_view target_dir, std::span compressed_binary, @@ -283,6 +297,86 @@ std::tuple compile_library( return std::make_tuple(library, std::make_shared(std::move(blob))); } +rtcx::blob compile_fragment(char const* name, + char const* cuda_code, + std::span extra_header_include_names, + std::span extra_headers, + std::span name_expressions) +{ + CUDF_FUNC_RANGE(); + + auto& ctx = cudf::get_context(); + auto& cfg = ctx.config(); + auto& bundle = ctx.jit_bundle(); + auto sm = get_current_device_compute_capability(); + auto runtime = get_runtime_version(); + + auto include_dirs = bundle.get_include_directories(); + auto pch_dir = ctx.get_jit_pch_dir(); + + auto use_pch = runtime >= MIN_CUDA_VERSION_PCH; + auto use_minimal = runtime >= MIN_CUDA_VERSION_MINIMAL; + + std::vector options; + + for (auto const& include_dir : include_dirs) { + options.emplace_back(std::format("-I{}", include_dir)); + } + + options.emplace_back(std::format("--gpu-architecture=sm_{}", sm)); + + options.emplace_back("--diag-suppress=47"); + options.emplace_back("--device-int128"); + + if (sm >= 100) { options.emplace_back("--device-float128"); } + + options.emplace_back("-std=c++20"); + options.emplace_back("--device-as-default-execution-space"); + options.emplace_back("--generate-line-info"); + options.emplace_back("--dopt=on"); + options.emplace_back("--dlink-time-opt"); + options.emplace_back("--gen-opt-lto"); + + if (use_minimal) { options.emplace_back("--minimal"); } + + if (use_pch) { + options.emplace_back("--pch"); + options.emplace_back(std::format("--pch-dir={}", pch_dir)); + + if (cfg.jit_verbose) { + options.emplace_back("--pch-verbose=true"); + options.emplace_back("--pch-messages=true"); + } else { + options.emplace_back("--pch-verbose=false"); + options.emplace_back("--pch-messages=false"); + } + } + + if (cfg.disable_cuda_cache) { options.emplace_back("--no-cache"); } + + if (cfg.dump_jit_trace) { options.emplace_back("--time=-"); } + + if (cfg.dump_jit_time_profile) { + options.emplace_back(std::format("--fdevice-time-trace=cudf_kernel_{}_trace", name)); + } + + std::vector options_cstr; + for (auto const& option : options) { + options_cstr.emplace_back(option.c_str()); + } + + auto params = rtcx::compile_params{.name = name, + .source = cuda_code, + .header_include_names = extra_header_include_names, + .headers = extra_headers, + .options = options_cstr, + .name_expressions = name_expressions, + .target_type = rtcx::binary_type::LTO_IR}; + + auto cubin = rtcx::compile(params); + return std::make_shared(rtcx::blob_t::from_buffer(std::move(cubin))); +} + } // namespace kernel get_kernel(std::string const& name, @@ -341,6 +435,61 @@ kernel_instance={} return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; } +rtcx::blob get_kernel_fragment(std::string const& name, + std::string const& source_file_id, + std::span header_include_names, + std::span headers, + std::string const& kernel_instance) +{ + CUDF_FUNC_RANGE(); + + auto& cache = cudf::get_context().rtcx_cache(); + auto& bundle = cudf::get_context().jit_bundle(); + + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_compute_capability(); + auto header_include_names_hash = hash(header_include_names).to_hex_string(); + auto headers_hash = hash(headers).to_hex_string(); + auto bundle_hash = bundle.get_hash(); + auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); + + auto cache_key = std::format(R"***(objectFile +name={} +binary_type=LTO_IR +cuda_runtime={} +cuda_driver={} +arch={} +bundle={} +source_file={} +header_include_names={} +headers={} +kernel_instance={} +)***", + name, + runtime, + driver, + sm, + bundle_hash, + source_file, + header_include_names_hash.view(), + headers_hash.view(), + kernel_instance); + + auto cache_key_sha256 = hash(cache_key); + + auto compile = [&] { + auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); + auto source = read_file_string(source_file.c_str()); + return compile_fragment(name.c_str(), source.c_str(), header_include_names, headers, {}); + }; + + auto fut = + cache.get_or_add_blob(cache_key_sha256, rtcx::blob_compile_func::from_functor(compile)); + + return fut.get(); +} + std::tuple link_library_uncached( char const* name, std::span file_fragments, @@ -348,7 +497,7 @@ std::tuple link_library_uncached( { CUDF_FUNC_RANGE(); - auto sm = get_current_device_physical_model(); + auto sm = get_current_device_compute_capability(); auto& ctx = cudf::get_context(); auto& cfg = ctx.config(); @@ -392,7 +541,7 @@ kernel get_lto_linked_kernel(std::string const& name, auto& bundle = cudf::get_context().jit_bundle(); auto runtime = get_runtime_version(); auto driver = get_driver_version(); - auto sm = get_current_device_physical_model(); + auto sm = get_current_device_compute_capability(); auto bundle_hash = bundle.get_hash(); auto fragments_hash = hash(file_fragments, memory_fragments).to_hex_string(); diff --git a/cpp/src/jit/cache.hpp b/cpp/src/jit/cache.hpp index c8986fd6f92a..95327aeb8dfe 100644 --- a/cpp/src/jit/cache.hpp +++ b/cpp/src/jit/cache.hpp @@ -1,5 +1,4 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,9 +8,6 @@ #include -#include -#include - #include namespace CUDF_EXPORT cudf { @@ -37,32 +33,9 @@ struct [[nodiscard]] kernel { private: rtcx::library _library; rtcx::kernel_ref _kernel; -namespace CUDF_EXPORT cudf { - -struct [[nodiscard]] jit_bundle_t { - private: - std::string install_dir_; - rtcx::cache_t* cache_; - - void ensure_installed() const; public: - jit_bundle_t(std::string install_dir, rtcx::cache_t& cache); - - [[nodiscard]] std::string get_hash() const; - - [[nodiscard]] std::string get_directory() const; - - [[nodiscard]] std::vector get_include_directories() const; -}; - -struct [[nodiscard]] kernel { - private: - rtcx::library _library; - rtcx::kernel_ref _kernel; - - public: - kernel(rtcx::library lib, rtcx::kernel_ref kernel) : _library(std::move(lib)), _kernel(kernel) {}; + kernel(rtcx::library lib, rtcx::kernel_ref kernel) : _library(std::move(lib)), _kernel(kernel) {} kernel(kernel const&) = default; kernel(kernel&&) = default; kernel& operator=(kernel const&) = default; @@ -92,6 +65,7 @@ struct [[nodiscard]] kernel { uint32_t shared_mem_bytes, rmm::cuda_stream_view stream, Args&&... args) + requires(sizeof...(Args) > 0) { void const* params[] = {&args...}; // NOLINT(modernize-avoid-c-arrays) launch(grid_dim, block_dim, shared_mem_bytes, stream, const_cast(params)); @@ -115,6 +89,23 @@ kernel get_kernel(std::string const& name, std::span headers, std::string const& kernel_instance); +/** + * @brief Gets a kernel fragment from an embedded CUDA source file + * @param name Debug name for the kernel fragment (used for caching and logging) + * @param source_file_id Identifier for the embedded source file (used to locate the source and for + * caching) + * @param header_include_names Names of any additional embedded header files to include during + * compilation + * @param headers Contents of any additional embedded header files to include during compilation + * @param kernel_instance String identifier for the specific kernel instance being requested (used + * for caching) + */ +rtcx::blob get_kernel_fragment(std::string const& name, + std::string const& source_file_id, + std::span header_include_names, + std::span headers, + std::string const& kernel_instance); + /** * @brief Gets a kernel by linking together embedded binary fragments * @param name Debug name for the kernel (used for caching and logging) diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 5ec9094f7d35..0ea8f9dc283e 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -10,8 +10,6 @@ #include #include #include -#include -#include namespace cudf { namespace jit { @@ -66,7 +64,6 @@ std::map build_ptx_params(std::span ou if (has_user_data) { params.emplace(index++, "void *"); params.emplace(index++, "cudf::size_type"); - params.emplace(index++, "cudf::size_type"); } for (auto& name : output_typenames) { @@ -92,9 +89,6 @@ std::vector input_type_names( return names; } -kernel get_udf_kernel(std::string const& source_file, - std::string const& kernel_name, - std::string const& cuda_source) kernel get_udf_kernel(std::string const& source_file, std::string const& kernel_name, std::string const& cuda_source) @@ -113,5 +107,22 @@ kernel get_udf_kernel(std::string const& source_file, return get_kernel(source_file, source_file, include_names, include_headers, kernel_name); } +rtcx::blob get_udf_kernel_fragment(std::string const& source_file, + std::string const& kernel_name, + std::string const& udf_type) +{ + auto kernel_instance_source = std::format(R"***(#define CUDF_KERNEL_INSTANCE {} + #define CUDF_LTO_MODE)***", + kernel_name); + auto kernel_udf_source = std::format(R"***(#define CUDF_UDF_TYPE {})***", + udf_type); + char const* include_names[] = // NOLINT(modernize-avoid-c-arrays) + {"cudf/detail/kernel_instance.cuh", "cudf/detail/operation_udf.cuh"}; + char const* include_headers[] = // NOLINT(modernize-avoid-c-arrays) + {kernel_instance_source.c_str(), kernel_udf_source.c_str()}; + + return get_kernel_fragment(source_file, source_file, include_names, include_headers, kernel_name); +} + } // namespace jit } // namespace cudf diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index 8a7685a63ebd..42827b011177 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -79,5 +79,9 @@ kernel get_udf_kernel(std::string const& source_file, std::string const& kernel_name, std::string const& cuda_source); +rtcx::blob get_udf_kernel_fragment(std::string const& source_file, + std::string const& kernel_name, + std::string const& udf_type); + } // namespace jit } // namespace cudf diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 3026ffcc8b20..8477fcc89f7b 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -24,21 +24,8 @@ class program_cache; struct jit_bundle_t; struct [[nodiscard]] context_config { - bool dump_codegen : 1 : 1 = false; - bool use_jit : 1 : 1 = false; - bool preload_jit_cache : 1 = false; - bool disable_jit_cache : 1 = false; - bool clear_jit_cache : 1 = false; - bool disable_codegen_cache : 1 = false; - bool disable_cuda_cache : 1 = false; - bool jit_verbose : 1 = false; - bool dump_jit_trace : 1 = false; - bool dump_jit_time_profile : 1 = false; - std::string rtcx_cache_dir = {}; - std::string jit_bundle_dir = {}; - std::string jit_pch_dir = {}; - std::string jit_tmp_dir = {}; - uint32_t kernel_cache_limit_process = 0; + bool dump_codegen : 1 = false; + bool use_jit : 1 = false; bool preload_jit_cache : 1 = false; bool disable_jit_cache : 1 = false; bool clear_jit_cache : 1 = false; diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 960de1099a9e..21c675bc9955 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -6,8 +6,6 @@ #include #include #include -#include -#include #include #include #include @@ -34,8 +32,26 @@ #include // clang-format on +#ifndef CUDF_LTO_MODE +#define CUDF_UDF_TYPE void() +#endif + +// Use LTO-dispatch for transform operators if we're in LTO mode. This allows the operator to be +// defined in a separate translation unit and compiled with LTO, which can result in better +// performance due to more optimization opportunities +#ifdef CUDF_LTO_MODE +#define GENERIC_TRANSFORM_OP(...) ::cudf::jit::lto::transform(__VA_ARGS__) +#endif + namespace cudf { namespace jit { +namespace lto { + +using transform_type = CUDF_UDF_TYPE; + +extern "C" __device__ transform_type transform; + +} // namespace lto /// @brief The generic transform kernel. Supports all types and nullability combinations. template @@ -49,8 +65,6 @@ __device__ void transform_kernel(size_type row_size, auto stride = detail::grid_1d::grid_stride(); for (auto row = start; row < row_size; row += stride) { -#ifndef CUDF_LTO_MODE - auto operation = [&](Args const& args) { if constexpr (has_user_data) { cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, @@ -60,15 +74,6 @@ __device__ void transform_kernel(size_type row_size, } }; -#else - - auto operation = [&](Args const& args) { - static_assert(!has_user_data); - cuda::std::apply([&](auto... a) { cudf::lto::transform(a...); }, args); - }; - -#endif - if constexpr (!is_null_aware) { if (stencil != nullptr && !bit_is_set(stencil, row)) { continue; } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 4ca80f6c8762..196a0edc8bc4 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -31,6 +31,8 @@ #include #include +#include +#include #include #include @@ -239,7 +241,7 @@ std::string reflect_output_column(mutable_strings_column const&) return "cudf::jit::mutable_strings_column_device_view"; } -auto reflect(udf_source_type source_type, +auto reflect(std::variant source_type, std::span inputs, std::span outputs) { @@ -282,7 +284,8 @@ auto reflect(udf_source_type source_type, std::vector ptx_in_types; std::vector ptx_out_types; - if (source_type == udf_source_type::PTX) { + if (std::holds_alternative(source_type) && + std::get(source_type) == udf_source_type::PTX) { for (auto& in : inputs) { ptx_in_types.push_back(std::visit([](auto& c) { return reflect_input_element(c); }, in)); } @@ -295,6 +298,60 @@ auto reflect(udf_source_type source_type, return std::make_tuple(ins, outs, ptx_in_types, ptx_out_types); } +std::string reflect_udf_signature(bool is_null_aware, + bool has_user_data, + std::span inputs, + std::span outputs) +{ + std::vector in_types; + + for (size_t i = 0; i < inputs.size(); i++) { + auto& in = inputs[i]; + auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); + in_types.push_back(is_null_aware ? std::format("cuda::std::optional<{}>", element) : element); + } + + std::vector out_types; + + for (size_t i = 0; i < outputs.size(); i++) { + auto& out = outputs[i]; + auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); + out_types.push_back(is_null_aware ? std::format("cuda::std::optional<{}> *", element) + : std::format("{} *", element)); + } + + std::vector params; + if (has_user_data) { params.push_back("void*"); } + params.insert(params.end(), out_types.begin(), out_types.end()); + params.insert(params.end(), in_types.begin(), in_types.end()); + + auto joined = + params.empty() + ? "" + : std::accumulate(std::next(params.begin()), params.end(), params[0], [](auto a, auto b) { + return std::format("{}, {}", a, b); + }); + + return std::format("void({})", joined); +} + +rtcx::blob instantiate_fragment(bool is_null_aware, + bool has_user_data, + std::string const& ins, + std::string const& outs, + std::span inputs, + std::span outputs) +{ + CUDF_FUNC_RANGE(); + auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", + rtcx::reflect(is_null_aware), + rtcx::reflect(has_user_data), + ins, + outs); + auto signature = reflect_udf_signature(is_null_aware, has_user_data, inputs, outputs); + return jit::get_udf_kernel_fragment("cudf/cpp/src/transform/jit/kernel.cu", kernel, signature); +} + auto to_args(std::span inputs, std::span outputs, rmm::cuda_stream_view stream, @@ -361,6 +418,68 @@ void run(bool is_null_aware, return launch(kernel, row_size, d_stencil, user_data, input_cols, output_cols, stream); } +rtcx::binary_type as_rtcx_binary_type(lto_binary_type type) +{ + switch (type) { + case lto_binary_type::LTO_IR: return rtcx::binary_type::LTO_IR; + case lto_binary_type::FATBIN: return rtcx::binary_type::FATBIN; + default: + CUDF_FAIL( + std::format("Unrecognized LTO binary type {} for LTO transform", static_cast(type)), + std::invalid_argument); + } +} + +void run_lto( + std::optional, lto_binary_type>> precompiled_kernel_fragment, + bool is_null_aware, + bool has_user_data, + size_type row_size, + bitmask_type const* d_stencil, + void* user_data, + std::span inputs, + std::span outputs, + std::span udf_binary, + lto_binary_type source_type, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + auto [in_types, out_types, ptx_in_types, ptx_out_types] = reflect(source_type, inputs, outputs); + + std::span kernel_fragment; + lto_binary_type kernel_fragment_binary_type = lto_binary_type::FATBIN; + rtcx::blob fragment_blob = nullptr; + + if (precompiled_kernel_fragment.has_value()) { + std::tie(kernel_fragment, kernel_fragment_binary_type) = *precompiled_kernel_fragment; + } else { + fragment_blob = + instantiate_fragment(is_null_aware, has_user_data, in_types, out_types, inputs, outputs); + kernel_fragment = fragment_blob->view(); + kernel_fragment_binary_type = lto_binary_type::LTO_IR; + } + + rtcx::memory_fragment memory_fragments[] = { + { + .data = kernel_fragment, + .type = as_rtcx_binary_type(kernel_fragment_binary_type), + .name = "transform_kernel_fragment", + }, + { + .data = udf_binary, + .type = as_rtcx_binary_type(source_type), + .name = "transform_udf_binary", + }}; + + auto kernel = get_lto_linked_kernel("cudf/cpp/src/transform/jit/kernel.cu", {}, memory_fragments); + + auto [cols, handles] = to_args(inputs, outputs, stream, mr); + auto* input_cols = reinterpret_cast(cols.data()); + auto* output_cols = + reinterpret_cast(input_cols + inputs.size()); + return launch(kernel, row_size, d_stencil, user_data, input_cols, output_cols, stream); +} + } // namespace jit_transform CUDF_KERNEL void copy_offset_bitmask(bitmask_type* __restrict__ destination, @@ -975,128 +1094,54 @@ std::unique_ptr compute_column_jit(table_view const& table, return std::move(cols[0]); } -std::string_view as_tag(type_id id) +std::string strip_whitespace(std::string_view str) { - switch (id) { - case type_id::BOOL8: return "bool8"; - case type_id::INT8: return "int8"; - case type_id::INT16: return "int16"; - case type_id::INT32: return "int32"; - case type_id::INT64: return "int64"; - case type_id::UINT8: return "uint8"; - case type_id::UINT16: return "uint16"; - case type_id::UINT32: return "uint32"; - case type_id::UINT64: return "uint64"; - case type_id::FLOAT32: return "float32"; - case type_id::FLOAT64: return "float64"; - case type_id::STRING: return "string_view"; - case type_id::DECIMAL32: return "decimal32"; - case type_id::DECIMAL64: return "decimal64"; - case type_id::DECIMAL128: return "decimal128"; - case type_id::DURATION_DAYS: return "duration_D"; - case type_id::DURATION_SECONDS: return "duration_s"; - case type_id::DURATION_MILLISECONDS: return "duration_ms"; - case type_id::DURATION_MICROSECONDS: return "duration_us"; - case type_id::DURATION_NANOSECONDS: return "duration_ns"; - case type_id::TIMESTAMP_DAYS: return "timestamp_D"; - case type_id::TIMESTAMP_SECONDS: return "timestamp_s"; - case type_id::TIMESTAMP_MILLISECONDS: return "timestamp_ms"; - case type_id::TIMESTAMP_MICROSECONDS: return "timestamp_us"; - case type_id::TIMESTAMP_NANOSECONDS: return "timestamp_ns"; - default: CUDF_FAIL("Unsupported type for JIT dispatch", std::invalid_argument); + std::string result; + result.reserve(str.size()); + for (char c : str) { + if (!std::isspace(static_cast(c))) { result.push_back(c); } } + return result; } -std::optional> dispatch_unop_lto_kernel( - bool null_aware, +// if we have a matching pre-compiled kernel fragment for the given transform configuration, return +// it to use for LTO linking instead of compiling a new one +std::optional, lto_binary_type>> dispatch_lto_kernel_fragment( + bool is_null_aware, + bool has_user_data, std::span inputs, - std::span outputs) + std::span outputs) { - auto input_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); - - for (size_t i = 0; i < std::size(cudf_fragments::unop_lto_kernel_FILE_INDEX); i++) { - auto FILE_INDEX = cudf_fragments::unop_lto_kernel_FILE_INDEX[i]; - auto NULL_AWARE = cudf_fragments::unop_lto_kernel_NULL_AWARE[i]; - auto TYPE = cudf_fragments::unop_lto_kernel_TYPE[i]; - if (as_tag(input_type) == TYPE && null_aware == NULL_AWARE) { - auto range = cudf_fragments::file_ranges[FILE_INDEX]; - return cudf_fragments::files.subspan(range[0], range[1]); + // TODO: better symbol mangling, this is sufficient for now + // the contract here is that CMake and this function agree on symbol mangling of the reflected + // kernel name. + auto [in_types, out_types, ptx_in_types, ptx_out_types] = + jit_transform::reflect(lto_binary_type::FATBIN, inputs, outputs); + auto target = strip_whitespace(rtcx::reflect_template("cudf::jit::transform_kernel", + rtcx::reflect(is_null_aware), + rtcx::reflect(has_user_data), + in_types, + out_types)); + for (size_t i = 0; i < std::size(cudf_fragments::transform_kernel_FILE_INDEX); i++) { + auto file_index = cudf_fragments::transform_kernel_FILE_INDEX[i]; + auto instance = strip_whitespace(cudf_fragments::transform_kernel_INSTANCE[i]); + if (target == instance) { + auto range = cudf_fragments::file_ranges[file_index]; + return std::make_tuple(cudf_fragments::files.subspan(range[0], range[1]), + lto_binary_type::FATBIN); } } return std::nullopt; } -std::optional> dispatch_binop_lto_kernel( - bool null_aware, - std::span inputs, - std::span outputs) -{ - auto input_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); - auto lhs_is_scalar = std::holds_alternative(inputs[0]); - auto rhs_is_scalar = std::holds_alternative(inputs[1]); - - if (lhs_is_scalar) { return std::nullopt; } - - for (size_t i = 0; i < std::size(cudf_fragments::binop_lto_kernel_FILE_INDEX); i++) { - auto FILE_INDEX = cudf_fragments::binop_lto_kernel_FILE_INDEX[i]; - auto NULL_AWARE = cudf_fragments::binop_lto_kernel_NULL_AWARE[i]; - auto TYPE = cudf_fragments::binop_lto_kernel_TYPE[i]; - auto RHS_IS_SCALAR = cudf_fragments::binop_lto_kernel_RHS_IS_SCALAR[i]; - if (as_tag(input_type) == TYPE && null_aware == NULL_AWARE && rhs_is_scalar == RHS_IS_SCALAR) { - auto range = cudf_fragments::file_ranges[FILE_INDEX]; - return cudf_fragments::files.subspan(range[0], range[1]); - } - } - - return std::nullopt; -} - -// Dispatches to the appropriate LTO kernel based on the number of inputs and outputs, their types, -// and nullability. -std::span dispatch_lto_kernel(bool null_aware, - std::span inputs, - std::span outputs) -{ - if (inputs.size() == 1 && outputs.size() == 1) { - auto input0_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); - auto output_type = outputs[0].type.id(); - if (input0_type == output_type && is_fixed_width(data_type{input0_type})) { - if (auto kernel = dispatch_unop_lto_kernel(null_aware, inputs, outputs)) { return *kernel; } - } - } - - if (inputs.size() == 2 && outputs.size() == 1) { - auto input0_type = std::visit([](auto& c) { return c.type().id(); }, inputs[0]); - auto input1_type = std::visit([](auto& c) { return c.type().id(); }, inputs[1]); - auto output_type = outputs[0].type.id(); - if (input0_type == output_type && input1_type == output_type && - is_fixed_width(data_type{input0_type})) { - if (auto kernel = dispatch_binop_lto_kernel(null_aware, inputs, outputs)) { return *kernel; } - } - } - - CUDF_FAIL("No suitable LTO kernel found for the given transform parameters", - std::invalid_argument); -} - -rtcx::binary_type as_rtcx_binary_type(lto_binary_type type) -{ - switch (type) { - case lto_binary_type::LTO_IR: return rtcx::binary_type::LTO_IR; - case lto_binary_type::FATBIN: return rtcx::binary_type::FATBIN; - default: - CUDF_FAIL( - std::format("Unrecognized LTO binary type {} for LTO transform", static_cast(type)), - std::invalid_argument); - } -} - -std::unique_ptr
transform_lto(std::span inputs, - std::span udf, +std::unique_ptr
transform_lto(std::span udf, lto_binary_type binary_type, - std::span outputs, null_aware is_null_aware, + std::optional user_data, + std::span inputs, + std::span outputs, + std::vector>&& string_offsets, std::optional in_row_size, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) @@ -1110,64 +1155,23 @@ std::unique_ptr
transform_lto(std::span inputs, make_outputs(is_null_aware, row_size, inputs, outputs, output_may_be_nullable, {}, stream, mr); auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; - auto kernel_fatbin = dispatch_lto_kernel(is_null_aware == null_aware::YES, inputs, outputs); - - rtcx::memory_fragment fragments[] = { - {.data = kernel_fatbin, .type = rtcx::binary_type::FATBIN, .name = "kernel"}, - {.data = udf, .type = as_rtcx_binary_type(binary_type), .name = "udf"}}; - - auto kernel = get_lto_linked_kernel("transform_lto_kernel", {}, fragments); - auto [cols, handles] = jit_transform::to_args(inputs, output_columns, stream, mr); - cudf::size_type num_inputs = static_cast(inputs.size()); - cudf::size_type num_outputs = static_cast(outputs.size()); - cudf::size_type num_rows = row_size; - cudf::bitmask_type* p_stencil = stencil_has_nulls ? stencil_arg : nullptr; - auto* input_cols = reinterpret_cast(cols.data()); - auto* output_cols = - reinterpret_cast(input_cols + inputs.size()); - - auto cfg = kernel.max_occupancy_config(0, 0); - void* user_data = nullptr; - - void* args[] = {&num_rows, &p_stencil, &user_data, &input_cols, &output_cols}; - - kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); + auto precompiled_kernel_fragment = dispatch_lto_kernel_fragment( + is_null_aware == null_aware::YES, user_data.has_value(), inputs, output_columns); + jit_transform::run_lto(precompiled_kernel_fragment, + is_null_aware == null_aware::YES, + user_data.has_value(), + row_size, + stencil_has_nulls ? stencil_arg : nullptr, + user_data.value_or(nullptr), + inputs, + output_columns, + udf, + binary_type, + stream, + mr); auto finalized = finalize_outputs(is_null_aware, row_size, std::move(output_columns), stream, mr); return std::make_unique
(std::move(finalized)); } -std::unique_ptr unary_op_lto(column_view input, - transform_output output, - std::span udf, - lto_binary_type binary_type, - null_aware is_null_aware, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - transform_input inputs[] = {input}; - transform_output outputs[] = {output}; - auto table = - transform_lto(inputs, udf, binary_type, outputs, is_null_aware, std::nullopt, stream, mr); - auto cols = table->release(); - return std::move(cols[0]); -} - -std::unique_ptr binary_op_lto(column_view lhs, - transform_input rhs, - transform_output output, - std::span udf, - lto_binary_type binary_type, - null_aware is_null_aware, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - transform_input inputs[] = {lhs, rhs}; - transform_output outputs[] = {output}; - auto table = - transform_lto(inputs, udf, binary_type, outputs, is_null_aware, std::nullopt, stream, mr); - auto cols = table->release(); - return std::move(cols[0]); -} - } // namespace cudf diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 6d62626e5df4..78d3fda7d8be 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -697,10 +697,12 @@ add_fragment( ) add_fragment( - cudf_test_fragments FRAGMENT decimal_square SOURCE transform/fragments/decimal_square.cu + cudf_test_fragments FRAGMENT bankers_rounding SOURCE transform/fragments/bankers_rounding.cu ) -embed(cudf_test_fragments COMPRESSION none) +embed(cudf_test_fragments COMPRESSION none OUTPUT_DIRECTORY + "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" +) ConfigureTest( TRANSFORM_LTO_TEST transform/transform_lto_test.cpp diff --git a/cpp/tests/transform/fragments/bankers_rounding.cu b/cpp/tests/transform/fragments/bankers_rounding.cu new file mode 100644 index 000000000000..8f98b3e133b3 --- /dev/null +++ b/cpp/tests/transform/fragments/bankers_rounding.cu @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +template +__device__ Decimal bankers_round(Decimal x) +{ + using Rep = typename Decimal::rep; + + auto const scale = x.scale(); + auto const value = x.value(); + + if (scale >= numeric::scale_type{0}) { return x; } + + Rep factor = 1; + for (int32_t i = 0; i < -static_cast(scale); ++i) { + factor *= 10; + } + + Rep q = value / factor; + Rep r = value % factor; + + if (r < 0) r = -r; + + auto const half = factor / 2; + + bool round_up = false; + + if (r > half) { + round_up = true; + } else if (r == half) { + // tie: round to even + round_up = (q % 2) != 0; + } + + if (round_up) { q += value >= 0 ? Rep{1} : Rep{-1}; } + + return Decimal{q, numeric::scale_type{0}}; +} + +extern "C" __device__ void transform(numeric::decimal128* out, numeric::decimal128 in) +{ + *out = bankers_round(in); +} diff --git a/cpp/tests/transform/fragments/decimal_square.cu b/cpp/tests/transform/fragments/decimal_square.cu deleted file mode 100644 index 6c9979964369..000000000000 --- a/cpp/tests/transform/fragments/decimal_square.cu +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -template <> -__device__ void cudf::lto::transform( - numeric::decimal32* out, numeric::decimal32 a) -{ - *out = a * a; -} - -template <> -__device__ void cudf::lto::transform( - numeric::decimal64* out, numeric::decimal64 a) -{ - *out = a * a; -} - -template <> -__device__ void cudf::lto::transform( - numeric::decimal128* out, numeric::decimal128 a) -{ - *out = a * a; -} diff --git a/cpp/tests/transform/fragments/invsqrt.cu b/cpp/tests/transform/fragments/invsqrt.cu index 30d763208339..932179dcfe29 100644 --- a/cpp/tests/transform/fragments/invsqrt.cu +++ b/cpp/tests/transform/fragments/invsqrt.cu @@ -3,10 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include -template <> -__device__ void cudf::lto::transform(float* out, float a) +extern "C" __device__ void transform(float* out, float a) { *out = 1.0F / sqrtf(a); } diff --git a/cpp/tests/transform/fragments/sum_of_squares.cu b/cpp/tests/transform/fragments/sum_of_squares.cu index b96dc3b3b3d8..4cd3ae4cc7f2 100644 --- a/cpp/tests/transform/fragments/sum_of_squares.cu +++ b/cpp/tests/transform/fragments/sum_of_squares.cu @@ -3,10 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include - -template <> -__device__ void cudf::lto::transform(float* out, float a, float b) -{ - *out = a * a + b * b; -} +extern "C" __device__ void transform(float* out, float a, float b) { *out = a * a + b * b; } diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp index 4dec31865b5a..d8360b97ac98 100644 --- a/cpp/tests/transform/transform_lto_test.cpp +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -27,22 +27,26 @@ TEST_F(TransformLTOTest, InvSqrt) { column_wrapper input{{1.0f, 4.0f, 9.0f, 16.0f}}; - cudf::transform_output output{cudf::data_type{cudf::type_id::FLOAT32}, - cudf::output_nullability::ALL_VALID}; + cudf::transform_input inputs[] = {input}; + cudf::transform_output outputs[] = { + {cudf::data_type{cudf::type_id::FLOAT32}, cudf::output_nullability::ALL_VALID}}; auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::invsqrt]; std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; - auto result = cudf::unary_op_lto(input, - output, - udf, - cudf::lto_binary_type::FATBIN, - cudf::null_aware::NO, - cudf::test::get_default_stream()); + auto result = cudf::transform_lto(udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt, + cudf::test::get_default_stream()); column_wrapper expected{{1.0f, 0.5f, 0.33333334f, 0.25f}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); } TEST_F(TransformLTOTest, SumOfSquares) @@ -50,49 +54,55 @@ TEST_F(TransformLTOTest, SumOfSquares) column_wrapper lhs{{1.0f, 4.0f, 9.0f, 16.0f}}; column_wrapper rhs{{1.0f, 2.0f, 2.0f, 10.0f}}; - cudf::transform_output output{cudf::data_type{cudf::type_id::FLOAT32}, - cudf::output_nullability::ALL_VALID}; + cudf::transform_input inputs[] = {lhs, rhs}; + cudf::transform_output outputs[] = { + {cudf::data_type{cudf::type_id::FLOAT32}, cudf::output_nullability::ALL_VALID}}; auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::sum_of_squares]; std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; - auto result = cudf::binary_op_lto(lhs, - rhs, - output, - udf, + auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt, cudf::test::get_default_stream()); column_wrapper expected{{2.0f, 20.0f, 85.0f, 356.0f}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); } -TEST_F(TransformLTOTest, Decimal32Square) +TEST_F(TransformLTOTest, BankersRounding) { - auto test_type = []() { - decimal_wrapper input{{1, 2, 3}, numeric::scale_type{-2}}; + using T = numeric::decimal128; - cudf::transform_output output{cudf::data_type{cudf::type_to_id(), numeric::scale_type{-4}}, - cudf::output_nullability::ALL_VALID}; + decimal_wrapper input{{12450, 12550, 12650, 12750, -12450, -12550, -12650, -12750}, + numeric::scale_type{-2}}; - auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::decimal_square]; - std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + cudf::transform_output output{cudf::data_type{cudf::type_to_id(), numeric::scale_type{0}}, + cudf::output_nullability::ALL_VALID}; - auto result = cudf::unary_op_lto(input, - output, - udf, - cudf::lto_binary_type::FATBIN, - cudf::null_aware::NO, - cudf::test::get_default_stream()); + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::bankers_rounding]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + cudf::transform_input inputs[] = {input}; + cudf::transform_output outputs[] = {output}; - decimal_wrapper expected{{1, 4, 9}, numeric::scale_type{-4}}; + auto result = cudf::transform_lto(udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt, + cudf::test::get_default_stream()); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); - }; + decimal_wrapper expected{{124, 126, 126, 128, -124, -126, -126, -128}, numeric::scale_type{0}}; - test_type.operator()(); - test_type.operator()(); - test_type.operator()(); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); } From 183be8fd1dd2de00afecc91dd827468e12f2948a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 11 Jun 2026 16:18:37 +0000 Subject: [PATCH 207/254] Remove LTO optimization flag from CUDA compile options in CMakeLists and cache.cpp --- cpp/CMakeLists.txt | 1 - cpp/src/jit/cache.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 926161f097ea..177f27d6ea3f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -565,7 +565,6 @@ macro(add_fragment) target_compile_options( ${OBJECT_ID} PRIVATE "$<$:${CUDF_CUDA_FLAGS}>" - "$<$:--gen-opt-lto>" ) embed_blob( diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 852e62e943c2..764ef4ec608a 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -335,7 +335,6 @@ rtcx::blob compile_fragment(char const* name, options.emplace_back("--generate-line-info"); options.emplace_back("--dopt=on"); options.emplace_back("--dlink-time-opt"); - options.emplace_back("--gen-opt-lto"); if (use_minimal) { options.emplace_back("--minimal"); } From 111f023d79f3770d41eca17cde870020eb1575f9 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 11 Jun 2026 16:32:26 +0000 Subject: [PATCH 208/254] Refactor transform functions to return int instead of void and update benchmark fragments --- cpp/CMakeLists.txt | 4 ++-- cpp/benchmarks/CMakeLists.txt | 4 ++-- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 2 +- cpp/benchmarks/binaryop/fragments/add.cu | 19 --------------- cpp/benchmarks/binaryop/fragments/add_f32.cu | 6 +++++ cpp/benchmarks/binaryop/fragments/mul_f32.cu | 6 +++++ cpp/benchmarks/binaryop/fragments/null_max.cu | 24 ------------------- cpp/src/transform/jit/kernel.cu | 2 +- cpp/src/transform/transform.cu | 2 +- .../transform/fragments/bankers_rounding.cu | 3 ++- cpp/tests/transform/fragments/invsqrt.cu | 3 ++- .../transform/fragments/sum_of_squares.cu | 6 ++++- 12 files changed, 28 insertions(+), 53 deletions(-) delete mode 100644 cpp/benchmarks/binaryop/fragments/add.cu create mode 100644 cpp/benchmarks/binaryop/fragments/add_f32.cu create mode 100644 cpp/benchmarks/binaryop/fragments/mul_f32.cu delete mode 100644 cpp/benchmarks/binaryop/fragments/null_max.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 177f27d6ea3f..a683ea954d95 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -597,7 +597,7 @@ foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t KERNEL_INSTANCE ${INSTANCE} UDF_TYPE - "void(${TYPE} *, ${TYPE})" + "int(${TYPE} *, ${TYPE})" DEFINITIONS CUDF_LTO_MODE ARRAY_IDS @@ -632,7 +632,7 @@ foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t KERNEL_INSTANCE ${INSTANCE} UDF_TYPE - "void(${TYPE} *, ${TYPE}, ${TYPE})" + "int(${TYPE} *, ${TYPE}, ${TYPE})" DEFINITIONS CUDF_LTO_MODE ARRAY_IDS diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index d9015e10f268..11256ce9c7f0 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -389,8 +389,8 @@ ConfigureNVBench(AST_NVBENCH ast/polynomials.cpp ast/transform.cpp) # ################################################################################################## # * LTO Fragments ---------------------------------------------------------------------------- add_embed(cudf_benchmark_fragments) -add_fragment(cudf_benchmark_fragments FRAGMENT add SOURCE binaryop/fragments/add.cu) -add_fragment(cudf_benchmark_fragments FRAGMENT null_max SOURCE binaryop/fragments/null_max.cu) +add_fragment(cudf_benchmark_fragments FRAGMENT add_f32 SOURCE binaryop/fragments/add_f32.cu) +add_fragment(cudf_benchmark_fragments FRAGMENT mul_f32 SOURCE binaryop/fragments/mul_f32.cu) embed(cudf_benchmark_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" ) diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index b048c6e60408..bdedfe44da4e 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -185,4 +185,4 @@ void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) LTO_BINARYOP_BENCHMARK_DEFINE(float, float, ADD, float); -LTO_BINARYOP_BENCHMARK_DEFINE(decimal32, decimal32, NULL_MAX, decimal32); +LTO_BINARYOP_BENCHMARK_DEFINE(float, float, MUL, float); diff --git a/cpp/benchmarks/binaryop/fragments/add.cu b/cpp/benchmarks/binaryop/fragments/add.cu deleted file mode 100644 index c2dcfaa286e0..000000000000 --- a/cpp/benchmarks/binaryop/fragments/add.cu +++ /dev/null @@ -1,19 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -template <> -__device__ void cudf::lto::transform(int32_t* out, int32_t a, int32_t b) -{ - *out = a + b; -} - -template <> -__device__ void cudf::lto::transform(float* out, float a, float b) -{ - *out = a + b; -} diff --git a/cpp/benchmarks/binaryop/fragments/add_f32.cu b/cpp/benchmarks/binaryop/fragments/add_f32.cu new file mode 100644 index 000000000000..b9318e158d66 --- /dev/null +++ b/cpp/benchmarks/binaryop/fragments/add_f32.cu @@ -0,0 +1,6 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +extern "C" __device__ int transform(float* out, float a, float b) { *out = a + b; return 0; } diff --git a/cpp/benchmarks/binaryop/fragments/mul_f32.cu b/cpp/benchmarks/binaryop/fragments/mul_f32.cu new file mode 100644 index 000000000000..618e7f104e74 --- /dev/null +++ b/cpp/benchmarks/binaryop/fragments/mul_f32.cu @@ -0,0 +1,6 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +extern "C" __device__ int transform(int32_t* out, int32_t a, int32_t b) { *out = a * b; return 0; } diff --git a/cpp/benchmarks/binaryop/fragments/null_max.cu b/cpp/benchmarks/binaryop/fragments/null_max.cu deleted file mode 100644 index 99de343811fb..000000000000 --- a/cpp/benchmarks/binaryop/fragments/null_max.cu +++ /dev/null @@ -1,24 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -#include - -template <> -__device__ void cudf::lto::transform*, - cuda::std::optional, - cuda::std::optional>( - cuda::std::optional* out, - cuda::std::optional a, - cuda::std::optional b) -{ - if (a.has_value() || b.has_value()) { - *out = (a.has_value() && (!b.has_value() || (*a > *b))) ? *a : *b; - } else { - *out = cuda::std::nullopt; - } -} diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 21c675bc9955..333c0c22ddd1 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -33,7 +33,7 @@ // clang-format on #ifndef CUDF_LTO_MODE -#define CUDF_UDF_TYPE void() +#define CUDF_UDF_TYPE int() #endif // Use LTO-dispatch for transform operators if we're in LTO mode. This allows the operator to be diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 196a0edc8bc4..3703e990462c 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -332,7 +332,7 @@ std::string reflect_udf_signature(bool is_null_aware, return std::format("{}, {}", a, b); }); - return std::format("void({})", joined); + return std::format("int({})", joined); } rtcx::blob instantiate_fragment(bool is_null_aware, diff --git a/cpp/tests/transform/fragments/bankers_rounding.cu b/cpp/tests/transform/fragments/bankers_rounding.cu index 8f98b3e133b3..85494bf93abc 100644 --- a/cpp/tests/transform/fragments/bankers_rounding.cu +++ b/cpp/tests/transform/fragments/bankers_rounding.cu @@ -41,7 +41,8 @@ __device__ Decimal bankers_round(Decimal x) return Decimal{q, numeric::scale_type{0}}; } -extern "C" __device__ void transform(numeric::decimal128* out, numeric::decimal128 in) +extern "C" __device__ int transform(numeric::decimal128* out, numeric::decimal128 in) { *out = bankers_round(in); + return 0; } diff --git a/cpp/tests/transform/fragments/invsqrt.cu b/cpp/tests/transform/fragments/invsqrt.cu index 932179dcfe29..6ae3950631bc 100644 --- a/cpp/tests/transform/fragments/invsqrt.cu +++ b/cpp/tests/transform/fragments/invsqrt.cu @@ -4,7 +4,8 @@ */ -extern "C" __device__ void transform(float* out, float a) +extern "C" __device__ int transform(float* out, float a) { *out = 1.0F / sqrtf(a); + return 0; } diff --git a/cpp/tests/transform/fragments/sum_of_squares.cu b/cpp/tests/transform/fragments/sum_of_squares.cu index 4cd3ae4cc7f2..b3bb5e289662 100644 --- a/cpp/tests/transform/fragments/sum_of_squares.cu +++ b/cpp/tests/transform/fragments/sum_of_squares.cu @@ -3,4 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -extern "C" __device__ void transform(float* out, float a, float b) { *out = a * a + b * b; } +extern "C" __device__ int transform(float* out, float a, float b) +{ + *out = a * a + b * b; + return 0; +} From c6760a0d69d714dd0dcf39376e53f224495503af Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 11 Jun 2026 17:23:02 +0000 Subject: [PATCH 209/254] Add static assertions for type consistency and update fragment IDs in LTO binaryop benchmark --- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index bdedfe44da4e..45ed7806afa8 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -109,6 +109,8 @@ template void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) { auto const num_rows = static_cast(state.get_int64("num_rows")); + static_assert(std::is_same_v && std::is_same_v); + static_assert(std::is_same_v); auto const source_table = create_random_table( {cudf::type_to_id(), cudf::type_to_id()}, row_count{num_rows}); @@ -121,12 +123,12 @@ void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) switch (binop) { case cudf::binary_operator::ADD: { - fragment_id = cudf_benchmark_fragments::add; + fragment_id = cudf_benchmark_fragments::add_f32; null_aware = false; } break; - case cudf::binary_operator::NULL_MAX: { - fragment_id = cudf_benchmark_fragments::null_max; - null_aware = true; + case cudf::binary_operator::MUL: { + fragment_id = cudf_benchmark_fragments::mul_f32; + null_aware = false; } break; default: throw std::runtime_error("Unsupported binary operator for LTO benchmark"); } From 104e8e90d04cac1702bd67e639ee1572f81c0e4d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 11 Jun 2026 17:33:04 +0000 Subject: [PATCH 210/254] Refactor LTO checks and move whitespace stripping logic to a lambda function --- cpp/src/transform/transform.cu | 48 +++++++++------------------------- 1 file changed, 13 insertions(+), 35 deletions(-) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 3703e990462c..c1abd953f6d2 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -653,28 +653,6 @@ void perform_checks(std::variant source_type, CUDF_EXPECTS(is_null_aware == null_aware::NO, "PTX UDFs do not support null-aware transformations", std::invalid_argument); - } else if (std::holds_alternative(source_type)) { - [[maybe_unused]] auto binary_type = std::get(source_type); - CUDF_EXPECTS( - std::none_of(inputs.begin(), - inputs.end(), - [](auto& in) { - return std::visit( - [](auto& c) { - return !is_fixed_width(c.type()) && c.type().id() != type_id::STRING; - }, - in); - }), - "Transforms with LTO binaries only support fixed-width types and strings as inputs", - std::invalid_argument); - CUDF_EXPECTS( - std::none_of( - outputs.begin(), outputs.end(), [](auto& out) { return !is_fixed_width(out.type); }), - "Transforms with LTO binaries only support output of fixed-width types", - std::invalid_argument); - CUDF_EXPECTS(inputs.size() == 2 || inputs.size() == 1, - "LTO binary transforms only support 1 or 2 inputs", - std::invalid_argument); } CUDF_EXPECTS(std::none_of(outputs.begin(), @@ -1094,16 +1072,6 @@ std::unique_ptr compute_column_jit(table_view const& table, return std::move(cols[0]); } -std::string strip_whitespace(std::string_view str) -{ - std::string result; - result.reserve(str.size()); - for (char c : str) { - if (!std::isspace(static_cast(c))) { result.push_back(c); } - } - return result; -} - // if we have a matching pre-compiled kernel fragment for the given transform configuration, return // it to use for LTO linking instead of compiling a new one std::optional, lto_binary_type>> dispatch_lto_kernel_fragment( @@ -1112,9 +1080,19 @@ std::optional, lto_binary_type>> dispatch_lt std::span inputs, std::span outputs) { - // TODO: better symbol mangling, this is sufficient for now - // the contract here is that CMake and this function agree on symbol mangling of the reflected - // kernel name. + auto strip_whitespace = [](std::string_view str) { + std::string result; + result.reserve(str.size()); + for (char c : str) { + if (!std::isspace(static_cast(c))) { result.push_back(c); } + } + return result; + }; + + // TODO: better and less error-prone symbol mangling, but this is sufficient for now. + + // the contract here is that CMake and this dispatch function agree on symbol mangling of the + // reflected kernel name. auto [in_types, out_types, ptx_in_types, ptx_out_types] = jit_transform::reflect(lto_binary_type::FATBIN, inputs, outputs); auto target = strip_whitespace(rtcx::reflect_template("cudf::jit::transform_kernel", From 7bb29e24a39867363af37f75301b5f47434323d6 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 11 Jun 2026 20:40:35 +0000 Subject: [PATCH 211/254] Update transform functions to use float types and enhance output handling in LTO --- cpp/benchmarks/binaryop/fragments/mul_f32.cu | 2 +- cpp/src/transform/transform.cu | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cpp/benchmarks/binaryop/fragments/mul_f32.cu b/cpp/benchmarks/binaryop/fragments/mul_f32.cu index 618e7f104e74..047240aeb805 100644 --- a/cpp/benchmarks/binaryop/fragments/mul_f32.cu +++ b/cpp/benchmarks/binaryop/fragments/mul_f32.cu @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -extern "C" __device__ int transform(int32_t* out, int32_t a, int32_t b) { *out = a * b; return 0; } +extern "C" __device__ int transform(float* out, float a, float b) { *out = a * b; return 0; } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index c1abd953f6d2..0b0e9c99cd09 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -1125,14 +1125,20 @@ std::unique_ptr
transform_lto(std::span udf, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - perform_checks(binary_type, is_null_aware, in_row_size, inputs, outputs, {}); + perform_checks(binary_type, is_null_aware, in_row_size, inputs, outputs, string_offsets); auto row_size = in_row_size.has_value() ? *in_row_size : jit::get_projection_size(inputs); auto output_may_be_nullable = get_null_transformation(is_null_aware, inputs, outputs); - auto [output_columns, stencil] = - make_outputs(is_null_aware, row_size, inputs, outputs, output_may_be_nullable, {}, stream, mr); - auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; - auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; + auto [output_columns, stencil] = make_outputs(is_null_aware, + row_size, + inputs, + outputs, + output_may_be_nullable, + std::move(string_offsets), + stream, + mr); + auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; + auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; auto precompiled_kernel_fragment = dispatch_lto_kernel_fragment( is_null_aware == null_aware::YES, user_data.has_value(), inputs, output_columns); From 5f2147e23aef319fcfe91f0ad28e379d7b5fcc9a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 11 Jun 2026 20:58:33 +0000 Subject: [PATCH 212/254] refactoring + cleanup --- cpp/CMakeLists.txt | 86 +++++++++----------- cpp/benchmarks/binaryop/fragments/add_f32.cu | 6 +- cpp/benchmarks/binaryop/fragments/mul_f32.cu | 6 +- cpp/src/jit/helpers.cpp | 3 +- cpp/tests/transform/fragments/invsqrt.cu | 1 - 5 files changed, 51 insertions(+), 51 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a683ea954d95..c83a33a542d0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -508,9 +508,7 @@ macro(add_fragment) set(OBJECT_ID ${TARGET}_${ARG_FRAGMENT}) add_library(${OBJECT_ID} OBJECT ${ARG_SOURCE}) - target_compile_options( - ${OBJECT_ID} PRIVATE --compress-mode=size --expt-relaxed-constexpr --extended-lambda - ) + target_compile_options(${OBJECT_ID} PRIVATE --compress-mode=size) if(DEFINED ARG_KERNEL_ONLY AND ARG_KERNEL_ONLY) # ensure that the FATBIN symbols only contain the specified kernel @@ -529,7 +527,6 @@ macro(add_fragment) endif() if(ARG_UDF_TYPE) - set(INSTANTIATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/${TARGET}/instantiations/${ARG_FRAGMENT}") file( GENERATE OUTPUT "${INSTANTIATION_DIR}/cudf/detail/operation_udf.cuh" @@ -562,10 +559,7 @@ macro(add_fragment) ${OBJECT_ID} PRIVATE "$" "$" ) - target_compile_options( - ${OBJECT_ID} PRIVATE - "$<$:${CUDF_CUDA_FLAGS}>" - ) + target_compile_options(${OBJECT_ID} PRIVATE "$<$:${CUDF_CUDA_FLAGS}>") embed_blob( ${TARGET} FILE $ DEST fragments/${ARG_FRAGMENT}.fatbin ID @@ -578,6 +572,41 @@ add_embed(cudf_fragments) foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t int64_t float double numeric::decimal32 numeric::decimal64 numeric::decimal128 ) + set(FRAGMENT_NAME transform_kernel) + get_property( + FILE_INDEX + TARGET cudf_fragments__embed_props + PROPERTY EMBED_FILE_INDEX + ) + set(VARIANT_NAME transform_kernel_${FILE_INDEX}) + set(INSTANCE + "cudf::jit::transform_kernel>, cudf::jit::type_list>>" + ) + add_fragment( + cudf_fragments + FRAGMENT + ${VARIANT_NAME} + SOURCE + src/transform/jit/kernel.cu + KERNEL_INSTANCE + ${INSTANCE} + UDF_TYPE + "int(${TYPE} *, ${TYPE})" + DEFINITIONS + CUDF_LTO_MODE + ARRAY_IDS + ${FRAGMENT_NAME}_FILE_INDEX + ${FRAGMENT_NAME}_INSTANCE + ARRAY_VALUES + ${FILE_INDEX} + "${INSTANCE}" + ) +endforeach() + +foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t int64_t float + double numeric::decimal32 numeric::decimal64 numeric::decimal128 +) + foreach(RHS_IS_SCALAR IN ITEMS "false" "true") set(FRAGMENT_NAME transform_kernel) get_property( FILE_INDEX @@ -586,7 +615,7 @@ foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t ) set(VARIANT_NAME transform_kernel_${FILE_INDEX}) set(INSTANCE - "cudf::jit::transform_kernel>, cudf::jit::type_list>>" + "cudf::jit::transform_kernel, cudf::jit::column_accessor<1ULL, cudf::column_device_view_core, ${TYPE}, ${RHS_IS_SCALAR}, 0>>, cudf::jit::type_list>>" ) add_fragment( cudf_fragments @@ -597,7 +626,7 @@ foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t KERNEL_INSTANCE ${INSTANCE} UDF_TYPE - "int(${TYPE} *, ${TYPE})" + "int(${TYPE} *, ${TYPE}, ${TYPE})" DEFINITIONS CUDF_LTO_MODE ARRAY_IDS @@ -608,42 +637,7 @@ foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t "${INSTANCE}" ) endforeach() - -foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t int64_t float - double numeric::decimal32 numeric::decimal64 numeric::decimal128 -) - foreach(RHS_IS_SCALAR IN ITEMS "false" "true") - set(FRAGMENT_NAME transform_kernel) - get_property( - FILE_INDEX - TARGET cudf_fragments__embed_props - PROPERTY EMBED_FILE_INDEX - ) - set(VARIANT_NAME transform_kernel_${FILE_INDEX}) - set(INSTANCE - "cudf::jit::transform_kernel, cudf::jit::column_accessor<1ULL, cudf::column_device_view_core, ${TYPE}, ${RHS_IS_SCALAR}, 0>>, cudf::jit::type_list>>" - ) - add_fragment( - cudf_fragments - FRAGMENT - ${VARIANT_NAME} - SOURCE - src/transform/jit/kernel.cu - KERNEL_INSTANCE - ${INSTANCE} - UDF_TYPE - "int(${TYPE} *, ${TYPE}, ${TYPE})" - DEFINITIONS - CUDF_LTO_MODE - ARRAY_IDS - ${FRAGMENT_NAME}_FILE_INDEX - ${FRAGMENT_NAME}_INSTANCE - ARRAY_VALUES - ${FILE_INDEX} - "${INSTANCE}" - ) - endforeach() - endforeach() +endforeach() embed(cudf_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") diff --git a/cpp/benchmarks/binaryop/fragments/add_f32.cu b/cpp/benchmarks/binaryop/fragments/add_f32.cu index b9318e158d66..b2bee38e7df2 100644 --- a/cpp/benchmarks/binaryop/fragments/add_f32.cu +++ b/cpp/benchmarks/binaryop/fragments/add_f32.cu @@ -3,4 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -extern "C" __device__ int transform(float* out, float a, float b) { *out = a + b; return 0; } +extern "C" __device__ int transform(float* out, float a, float b) +{ + *out = a + b; + return 0; +} diff --git a/cpp/benchmarks/binaryop/fragments/mul_f32.cu b/cpp/benchmarks/binaryop/fragments/mul_f32.cu index 047240aeb805..a6f4edc7c303 100644 --- a/cpp/benchmarks/binaryop/fragments/mul_f32.cu +++ b/cpp/benchmarks/binaryop/fragments/mul_f32.cu @@ -3,4 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -extern "C" __device__ int transform(float* out, float a, float b) { *out = a * b; return 0; } +extern "C" __device__ int transform(float* out, float a, float b) +{ + *out = a * b; + return 0; +} diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index 0ea8f9dc283e..a0ad474f73fb 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -114,8 +114,7 @@ rtcx::blob get_udf_kernel_fragment(std::string const& source_file, auto kernel_instance_source = std::format(R"***(#define CUDF_KERNEL_INSTANCE {} #define CUDF_LTO_MODE)***", kernel_name); - auto kernel_udf_source = std::format(R"***(#define CUDF_UDF_TYPE {})***", - udf_type); + auto kernel_udf_source = std::format(R"***(#define CUDF_UDF_TYPE {})***", udf_type); char const* include_names[] = // NOLINT(modernize-avoid-c-arrays) {"cudf/detail/kernel_instance.cuh", "cudf/detail/operation_udf.cuh"}; char const* include_headers[] = // NOLINT(modernize-avoid-c-arrays) diff --git a/cpp/tests/transform/fragments/invsqrt.cu b/cpp/tests/transform/fragments/invsqrt.cu index 6ae3950631bc..689fc5ac4692 100644 --- a/cpp/tests/transform/fragments/invsqrt.cu +++ b/cpp/tests/transform/fragments/invsqrt.cu @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ - extern "C" __device__ int transform(float* out, float a) { *out = 1.0F / sqrtf(a); From 4fe25e60a00b49994a52fbd26f0970364d61e486 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 15 Jun 2026 21:22:16 +0000 Subject: [PATCH 213/254] Add JIT support for binary operations and new 'to_lower' transform function - Refactor binary operation benchmarks to use JIT compilation. - Introduce a new 'to_lower' transform function for character case conversion. - Update documentation to clarify UDF requirements for LTO. - Enhance CMake configuration to include the new transform fragment. --- cpp/CMakeLists.txt | 34 ++++++++-- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 66 ++++++++++++++----- cpp/include/cudf/transform.hpp | 3 +- cpp/tests/CMakeLists.txt | 2 + cpp/tests/transform/fragments/to_lower.cu | 18 +++++ cpp/tests/transform/transform_lto_test.cpp | 26 ++++++++ 6 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 cpp/tests/transform/fragments/to_lower.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c83a33a542d0..c0e7082b95fb 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -569,9 +569,35 @@ endmacro() add_embed(cudf_fragments) -foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t int64_t float - double numeric::decimal32 numeric::decimal64 numeric::decimal128 +list( + APPEND + CUDF_PRECOMPILE_TYPES + uint8_t + uint16_t + uint32_t + uint64_t + int8_t + int16_t + int32_t + int64_t + float + double + numeric::decimal32 + numeric::decimal64 + numeric::decimal128 + cudf::timestamp_D + cudf::timestamp_s + cudf::timestamp_ms + cudf::timestamp_us + cudf::timestamp_ns + cudf::duration_D + cudf::duration_s + cudf::duration_ms + cudf::duration_us + cudf::duration_ns ) + +foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) set(FRAGMENT_NAME transform_kernel) get_property( FILE_INDEX @@ -603,9 +629,7 @@ foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t ) endforeach() -foreach(TYPE IN ITEMS uint8_t uint16_t uint32_t uint64_t int8_t int16_t int32_t int64_t float - double numeric::decimal32 numeric::decimal64 numeric::decimal128 -) +foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) foreach(RHS_IS_SCALAR IN ITEMS "false" "true") set(FRAGMENT_NAME transform_kernel) get_property( diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 45ed7806afa8..625560d3e15e 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -106,9 +106,22 @@ BINARYOP_BENCHMARK_DEFINE(timestamp_D, timestamp_s, NULL_MIN, time template -void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) +void BM_jit_binaryop(nvbench::state& state, cudf::binary_operator binop) { + constexpr auto const jit_mul_cuda = R"***( +__device__ void transform(float* out, float a, float b) { + *out = a * b; +} +)***"; + +constexpr auto const jit_add_cuda = R"***( +__device__ void transform(float* out, float a, float b) { + *out = a + b; +} +)***"; + auto const num_rows = static_cast(state.get_int64("num_rows")); + auto const use_lto = state.get_string("use_lto") == "true"; static_assert(std::is_same_v && std::is_same_v); static_assert(std::is_same_v); @@ -119,18 +132,18 @@ void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) auto rhs = cudf::column_view(source_table->get_column(1)); size_t fragment_id = 0; - bool null_aware = false; + char const * cuda = nullptr; switch (binop) { case cudf::binary_operator::ADD: { fragment_id = cudf_benchmark_fragments::add_f32; - null_aware = false; + cuda = jit_add_cuda; } break; case cudf::binary_operator::MUL: { fragment_id = cudf_benchmark_fragments::mul_f32; - null_aware = false; + cuda = jit_mul_cuda; } break; - default: throw std::runtime_error("Unsupported binary operator for LTO benchmark"); + default: throw std::runtime_error("Unsupported binary operator for JIT benchmark"); } // Call once for hot cache. @@ -141,9 +154,18 @@ void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) auto const range = cudf_benchmark_fragments::file_ranges[fragment_id]; std::span udf{cudf_benchmark_fragments::files.subspan(range[0], range[1])}; - auto result = cudf::transform_lto(udf, + auto result = use_lto ? + cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, - null_aware ? cudf::null_aware::YES : cudf::null_aware::NO, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt): + cudf::multi_transform(cuda, + cudf::udf_source_type::CUDA, + cudf::null_aware::NO, std::nullopt, inputs, outputs, @@ -157,9 +179,16 @@ void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) state.add_global_memory_writes(num_rows); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch&) { - cudf::transform_lto(udf, + [[maybe_unused]] auto result = use_lto ? cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, - null_aware ? cudf::null_aware::YES : cudf::null_aware::NO, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt): cudf::multi_transform(cuda, + cudf::udf_source_type::CUDA, + cudf::null_aware::NO, std::nullopt, inputs, outputs, @@ -169,22 +198,23 @@ void BM_lto_binaryop(nvbench::state& state, cudf::binary_operator binop) } -#define BM_LTO_BINARYOP_BENCHMARK_DEFINE(name, lhs, rhs, bop, tout) \ +#define BM_JIT_BINARYOP_BENCHMARK_DEFINE(name, lhs, rhs, bop, tout) \ static void name(::nvbench::state& st) \ { \ - ::BM_lto_binaryop(st, ::cudf::binary_operator::bop); \ + ::BM_jit_binaryop(st, ::cudf::binary_operator::bop); \ } \ NVBENCH_BENCH(name) \ - .set_name("lto_binary_op_" BM_STRINGIFY(name)) \ - .add_int64_axis("num_rows", {10'000, 100'000, 1'000'000, 10'000'000, 100'000'000}) + .set_name("jit_binary_op_" BM_STRINGIFY(name)) \ + .add_int64_axis("num_rows", {10'000, 100'000, 1'000'000, 10'000'000, 100'000'000})\ + .add_string_axis("use_lto", {"true", "false"}) -#define build_name_lto(a, b, c, d) a##_##b##_##c##_##d##_lto +#define build_name_jit(a, b, c, d) a##_##b##_##c##_##d##_jit -#define LTO_BINARYOP_BENCHMARK_DEFINE(lhs, rhs, bop, tout) \ - BM_LTO_BINARYOP_BENCHMARK_DEFINE(build_name_lto(bop, lhs, rhs, tout), lhs, rhs, bop, tout) +#define JIT_BINARYOP_BENCHMARK_DEFINE(lhs, rhs, bop, tout) \ + BM_JIT_BINARYOP_BENCHMARK_DEFINE(build_name_jit(bop, lhs, rhs, tout), lhs, rhs, bop, tout) -LTO_BINARYOP_BENCHMARK_DEFINE(float, float, ADD, float); -LTO_BINARYOP_BENCHMARK_DEFINE(float, float, MUL, float); +JIT_BINARYOP_BENCHMARK_DEFINE(float, float, ADD, float); +JIT_BINARYOP_BENCHMARK_DEFINE(float, float, MUL, float); diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 2829a1b9fbf5..3b7c465260a8 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -207,7 +207,8 @@ enum class lto_binary_type : uint8_t { * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from * the input and pre-allocated output columns. * - * @param udf The LTO-IR string of the transform function to apply + * @param udf The LTO-IR string of the transform function to apply. The UDF must be named + * `transform` and follow the CUDF UDF ABI. * @param binary_type The type of the LTO binary provided in `udf` * @param is_null_aware Signifies the UDF will receive row inputs as optional values * @param user_data User-defined device data to pass to the UDF. diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 78d3fda7d8be..92fc3946003c 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -692,6 +692,8 @@ add_embed(cudf_test_fragments) add_fragment(cudf_test_fragments FRAGMENT invsqrt SOURCE transform/fragments/invsqrt.cu) +add_fragment(cudf_test_fragments FRAGMENT to_lower SOURCE transform/fragments/to_lower.cu) + add_fragment( cudf_test_fragments FRAGMENT sum_of_squares SOURCE transform/fragments/sum_of_squares.cu ) diff --git a/cpp/tests/transform/fragments/to_lower.cu b/cpp/tests/transform/fragments/to_lower.cu new file mode 100644 index 000000000000..0ad86b562200 --- /dev/null +++ b/cpp/tests/transform/fragments/to_lower.cu @@ -0,0 +1,18 @@ + + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +extern "C" __device__ int transform(uint8_t* output, uint8_t input) +{ + if (input > 96 && input < 123) { + *output = input - 32; + } else { + *output = input; + } + return 0; +} diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp index d8360b97ac98..b0b46385970c 100644 --- a/cpp/tests/transform/transform_lto_test.cpp +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -49,6 +49,32 @@ TEST_F(TransformLTOTest, InvSqrt) CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); } +TEST_F(TransformLTOTest, ToLower) +{ + column_wrapper input{{65, 66, 97, 98, 48, 49, 32, 33, 127, 255}}; + + cudf::transform_input inputs[] = {input}; + cudf::transform_output outputs[] = { + {cudf::data_type{cudf::type_id::UINT8}, cudf::output_nullability::ALL_VALID}}; + + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::to_lower]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::transform_lto(udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt, + cudf::test::get_default_stream()); + + column_wrapper expected{{65, 66, 65, 66, 48, 49, 32, 33, 127, 255}}; + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); +} + TEST_F(TransformLTOTest, SumOfSquares) { column_wrapper lhs{{1.0f, 4.0f, 9.0f, 16.0f}}; From 3d4897572ef231f9d3da633018fd4eb6f99c92e8 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 15 Jun 2026 21:35:16 +0000 Subject: [PATCH 214/254] Rename 'ToLower' test to 'ToUpper' and update associated fragment reference --- cpp/tests/CMakeLists.txt | 2 +- cpp/tests/transform/transform_lto_test.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index c6d7c81b2a70..aa38e5cad194 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -694,7 +694,7 @@ add_embed(cudf_test_fragments) add_fragment(cudf_test_fragments FRAGMENT invsqrt SOURCE transform/fragments/invsqrt.cu) -add_fragment(cudf_test_fragments FRAGMENT to_lower SOURCE transform/fragments/to_lower.cu) +add_fragment(cudf_test_fragments FRAGMENT to_upper SOURCE transform/fragments/to_upper.cu) add_fragment( cudf_test_fragments FRAGMENT sum_of_squares SOURCE transform/fragments/sum_of_squares.cu diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp index b0b46385970c..2d6a2cbd326f 100644 --- a/cpp/tests/transform/transform_lto_test.cpp +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -49,7 +49,7 @@ TEST_F(TransformLTOTest, InvSqrt) CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); } -TEST_F(TransformLTOTest, ToLower) +TEST_F(TransformLTOTest, ToUpper) { column_wrapper input{{65, 66, 97, 98, 48, 49, 32, 33, 127, 255}}; @@ -57,7 +57,7 @@ TEST_F(TransformLTOTest, ToLower) cudf::transform_output outputs[] = { {cudf::data_type{cudf::type_id::UINT8}, cudf::output_nullability::ALL_VALID}}; - auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::to_lower]; + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::to_upper]; std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; auto result = cudf::transform_lto(udf, From 881603649e9a414b0af46d343395bf95e2d534ac Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 16 Jun 2026 02:31:38 +0000 Subject: [PATCH 215/254] use kernel id as cache, and require file/memory fragment names to be unique --- cpp/src/jit/cache.cpp | 7 +- cpp/src/transform/transform.cu | 69 ++++++++++--------- .../fragments/{to_lower.cu => to_upper.cu} | 0 3 files changed, 43 insertions(+), 33 deletions(-) rename cpp/tests/transform/fragments/{to_lower.cu => to_upper.cu} (100%) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 1d9c67327b3e..9691596cc693 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -48,7 +48,12 @@ rtcx::sha256 hash(std::span file_fragments, std::span{reinterpret_cast(fragment.path), std::strlen(fragment.path)}); } for (auto const& fragment : memory_fragments) { - ctx.update(fragment.data); + if (fragment.name != nullptr) { + ctx.update( + std::span{reinterpret_cast(fragment.name), std::strlen(fragment.name)}); + } else { + ctx.update(fragment.data); + } } return ctx.finalize(); } diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 0b0e9c99cd09..7b0c1cb48a51 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -335,12 +335,13 @@ std::string reflect_udf_signature(bool is_null_aware, return std::format("int({})", joined); } -rtcx::blob instantiate_fragment(bool is_null_aware, - bool has_user_data, - std::string const& ins, - std::string const& outs, - std::span inputs, - std::span outputs) +std::tuple instantiate_fragment( + bool is_null_aware, + bool has_user_data, + std::string const& ins, + std::string const& outs, + std::span inputs, + std::span outputs) { CUDF_FUNC_RANGE(); auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", @@ -349,7 +350,9 @@ rtcx::blob instantiate_fragment(bool is_null_aware, ins, outs); auto signature = reflect_udf_signature(is_null_aware, has_user_data, inputs, outputs); - return jit::get_udf_kernel_fragment("cudf/cpp/src/transform/jit/kernel.cu", kernel, signature); + return {jit::get_udf_kernel_fragment("cudf/cpp/src/transform/jit/kernel.cu", kernel, signature), + lto_binary_type::LTO_IR, + kernel}; } auto to_args(std::span inputs, @@ -430,45 +433,46 @@ rtcx::binary_type as_rtcx_binary_type(lto_binary_type type) } } -void run_lto( - std::optional, lto_binary_type>> precompiled_kernel_fragment, - bool is_null_aware, - bool has_user_data, - size_type row_size, - bitmask_type const* d_stencil, - void* user_data, - std::span inputs, - std::span outputs, - std::span udf_binary, - lto_binary_type source_type, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +void run_lto(std::optional, lto_binary_type, char const*>> + precompiled_kernel_fragment, + bool is_null_aware, + bool has_user_data, + size_type row_size, + bitmask_type const* d_stencil, + void* user_data, + std::span inputs, + std::span outputs, + std::span udf_binary, + lto_binary_type source_type, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { auto [in_types, out_types, ptx_in_types, ptx_out_types] = reflect(source_type, inputs, outputs); std::span kernel_fragment; lto_binary_type kernel_fragment_binary_type = lto_binary_type::FATBIN; rtcx::blob fragment_blob = nullptr; + std::string kernel_fragment_id; if (precompiled_kernel_fragment.has_value()) { - std::tie(kernel_fragment, kernel_fragment_binary_type) = *precompiled_kernel_fragment; + std::tie(kernel_fragment, kernel_fragment_binary_type, kernel_fragment_id) = + *precompiled_kernel_fragment; } else { - fragment_blob = + std::tie(fragment_blob, kernel_fragment_binary_type, kernel_fragment_id) = instantiate_fragment(is_null_aware, has_user_data, in_types, out_types, inputs, outputs); - kernel_fragment = fragment_blob->view(); - kernel_fragment_binary_type = lto_binary_type::LTO_IR; + kernel_fragment = fragment_blob->view(); } rtcx::memory_fragment memory_fragments[] = { { .data = kernel_fragment, .type = as_rtcx_binary_type(kernel_fragment_binary_type), - .name = "transform_kernel_fragment", + .name = kernel_fragment_id.c_str(), }, { .data = udf_binary, .type = as_rtcx_binary_type(source_type), - .name = "transform_udf_binary", + .name = nullptr // unnamed fragment: hash of the binary will be used as the fragment name }}; auto kernel = get_lto_linked_kernel("cudf/cpp/src/transform/jit/kernel.cu", {}, memory_fragments); @@ -1074,11 +1078,11 @@ std::unique_ptr compute_column_jit(table_view const& table, // if we have a matching pre-compiled kernel fragment for the given transform configuration, return // it to use for LTO linking instead of compiling a new one -std::optional, lto_binary_type>> dispatch_lto_kernel_fragment( - bool is_null_aware, - bool has_user_data, - std::span inputs, - std::span outputs) +std::optional, lto_binary_type, char const*>> +dispatch_lto_kernel_fragment(bool is_null_aware, + bool has_user_data, + std::span inputs, + std::span outputs) { auto strip_whitespace = [](std::string_view str) { std::string result; @@ -1106,7 +1110,8 @@ std::optional, lto_binary_type>> dispatch_lt if (target == instance) { auto range = cudf_fragments::file_ranges[file_index]; return std::make_tuple(cudf_fragments::files.subspan(range[0], range[1]), - lto_binary_type::FATBIN); + lto_binary_type::FATBIN, + cudf_fragments::transform_kernel_INSTANCE[i]); } } diff --git a/cpp/tests/transform/fragments/to_lower.cu b/cpp/tests/transform/fragments/to_upper.cu similarity index 100% rename from cpp/tests/transform/fragments/to_lower.cu rename to cpp/tests/transform/fragments/to_upper.cu From 376f836e0e1c921834b22458a18041ab3c195af5 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 16 Jun 2026 03:11:55 +0000 Subject: [PATCH 216/254] use kernel id as cache, and require file/memory fragment names to be unique --- cpp/librtcx/embed.cmake | 15 ++++++--------- cpp/librtcx/rtcx.cpp | 3 ++- cpp/src/transform/transform.cu | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index fb32990ddfdc..d7f49e6852f4 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -5,6 +5,10 @@ # cmake-format: on # ============================================================================= +if(NOT TARGET zstd) + message(FATAL_ERROR "embed(): zstd target is required when COMPRESSION is not none.") +endif() + # This function initializes a target for JIT embedding. It must be called before any calls to # embed_includes() or embed_blob() for the target. It creates a dedicated INTERFACE library target # that is used to track registered files and dependencies via target properties. The TARGET argument @@ -287,16 +291,9 @@ function(embed TARGET) set(RUNNER "${TARGET}__jit_embed_run") add_executable(${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") - target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS}) - if(NOT ARG_COMPRESSION STREQUAL "none") - if(NOT TARGET zstd) - message(FATAL_ERROR "embed(): zstd target is required when COMPRESSION is not none.") - endif() - target_include_directories(${RUNNER} PRIVATE ${ZSTD_INCLUDE_DIR}) - target_link_libraries(${RUNNER} PRIVATE zstd) - endif() + target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} zstd) + target_include_directories(${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR} ${ZSTD_INCLUDE_DIR}) set_target_properties(${RUNNER} PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES) - target_include_directories(${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR}) add_custom_command( OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 1ec3338d19b5..199f2b563eab 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -619,7 +619,8 @@ void log_nvJitLink_result(link_params const& params, } for (auto& frag : params.memory_fragments) { - fragments_str = std::format("{}\t{}\n", fragments_str, frag.name); + fragments_str = + std::format("{}\t{}\n", fragments_str, frag.name == nullptr ? "" : frag.name); } std::string link_options_str; diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 7b0c1cb48a51..b49338f6fd4b 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -472,7 +472,7 @@ void run_lto(std::optional, lto_binary_type, { .data = udf_binary, .type = as_rtcx_binary_type(source_type), - .name = nullptr // unnamed fragment: hash of the binary will be used as the fragment name + .name = nullptr // unnamed fragment: the binary will be used to hash the UDF }}; auto kernel = get_lto_linked_kernel("cudf/cpp/src/transform/jit/kernel.cu", {}, memory_fragments); From 16052fcbd43a35d125a191134e3778f604b75dd0 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 16 Jun 2026 05:47:47 +0000 Subject: [PATCH 217/254] Add optional UDF identifier for caching in LTO transform functions --- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 5 +++++ cpp/include/cudf/transform.hpp | 3 +++ cpp/librtcx/embed.cmake | 8 +++++--- cpp/src/transform/transform.cu | 7 ++++++- cpp/tests/transform/transform_lto_test.cpp | 4 ++++ 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 625560d3e15e..2591bca21a44 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -119,6 +119,7 @@ __device__ void transform(float* out, float a, float b) { *out = a + b; } )***"; +std::string udf_hash; auto const num_rows = static_cast(state.get_int64("num_rows")); auto const use_lto = state.get_string("use_lto") == "true"; @@ -138,10 +139,12 @@ __device__ void transform(float* out, float a, float b) { case cudf::binary_operator::ADD: { fragment_id = cudf_benchmark_fragments::add_f32; cuda = jit_add_cuda; + udf_hash = "add_f32"; } break; case cudf::binary_operator::MUL: { fragment_id = cudf_benchmark_fragments::mul_f32; cuda = jit_mul_cuda; + udf_hash = "mul_f32"; } break; default: throw std::runtime_error("Unsupported binary operator for JIT benchmark"); } @@ -156,6 +159,7 @@ __device__ void transform(float* out, float a, float b) { auto result = use_lto ? cudf::transform_lto(udf, + udf_hash, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, @@ -180,6 +184,7 @@ __device__ void transform(float* out, float a, float b) { state.exec(nvbench::exec_tag::sync, [&](nvbench::launch&) { [[maybe_unused]] auto result = use_lto ? cudf::transform_lto(udf, + udf_hash, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 3b7c465260a8..ce06208febbe 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -209,6 +209,8 @@ enum class lto_binary_type : uint8_t { * * @param udf The LTO-IR string of the transform function to apply. The UDF must be named * `transform` and follow the CUDF UDF ABI. + * @param udf_id An optional identifier of the UDF to use for caching. If not provided, the UDF will + * be hashed and cached based on its binary contents. * @param binary_type The type of the LTO binary provided in `udf` * @param is_null_aware Signifies the UDF will receive row inputs as optional values * @param user_data User-defined device data to pass to the UDF. @@ -226,6 +228,7 @@ enum class lto_binary_type : uint8_t { */ std::unique_ptr
transform_lto( std::span udf, + std::optional udf_id, lto_binary_type binary_type, null_aware is_null_aware, std::optional user_data, diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index d7f49e6852f4..8c313b9b6ded 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -8,7 +8,7 @@ if(NOT TARGET zstd) message(FATAL_ERROR "embed(): zstd target is required when COMPRESSION is not none.") endif() - + # This function initializes a target for JIT embedding. It must be called before any calls to # embed_includes() or embed_blob() for the target. It creates a dedicated INTERFACE library target # that is used to track registered files and dependencies via target properties. The TARGET argument @@ -291,8 +291,10 @@ function(embed TARGET) set(RUNNER "${TARGET}__jit_embed_run") add_executable(${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") - target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} zstd) - target_include_directories(${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR} ${ZSTD_INCLUDE_DIR}) + target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} zstd) + target_include_directories( + ${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR} ${ZSTD_INCLUDE_DIR} + ) set_target_properties(${RUNNER} PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES) add_custom_command( diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index b49338f6fd4b..dc8e9fe53a07 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -443,6 +443,7 @@ void run_lto(std::optional, lto_binary_type, std::span inputs, std::span outputs, std::span udf_binary, + std::optional udf_id, lto_binary_type source_type, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) @@ -472,7 +473,9 @@ void run_lto(std::optional, lto_binary_type, { .data = udf_binary, .type = as_rtcx_binary_type(source_type), - .name = nullptr // unnamed fragment: the binary will be used to hash the UDF + .name = udf_id.has_value() + ? udf_id->c_str() + : nullptr // nullptr = unnamed fragment: the binary will be used to hash the UDF }}; auto kernel = get_lto_linked_kernel("cudf/cpp/src/transform/jit/kernel.cu", {}, memory_fragments); @@ -1119,6 +1122,7 @@ dispatch_lto_kernel_fragment(bool is_null_aware, } std::unique_ptr
transform_lto(std::span udf, + std::optional udf_id, lto_binary_type binary_type, null_aware is_null_aware, std::optional user_data, @@ -1156,6 +1160,7 @@ std::unique_ptr
transform_lto(std::span udf, inputs, output_columns, udf, + udf_id, binary_type, stream, mr); diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp index 2d6a2cbd326f..bc23dc4057d4 100644 --- a/cpp/tests/transform/transform_lto_test.cpp +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -35,6 +35,7 @@ TEST_F(TransformLTOTest, InvSqrt) std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; auto result = cudf::transform_lto(udf, + std::nullopt, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, @@ -61,6 +62,7 @@ TEST_F(TransformLTOTest, ToUpper) std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; auto result = cudf::transform_lto(udf, + std::nullopt, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, @@ -88,6 +90,7 @@ TEST_F(TransformLTOTest, SumOfSquares) std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; auto result = cudf::transform_lto(udf, + std::nullopt, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, @@ -119,6 +122,7 @@ TEST_F(TransformLTOTest, BankersRounding) cudf::transform_output outputs[] = {output}; auto result = cudf::transform_lto(udf, + std::nullopt, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, From 16b950952ed86e919378cc874d5572112902aa4a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 16 Jun 2026 05:53:39 +0000 Subject: [PATCH 218/254] Rename udf_hash to udf_id for clarity in binary operator transformations --- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 2591bca21a44..5a84b1f9f181 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -119,7 +119,7 @@ __device__ void transform(float* out, float a, float b) { *out = a + b; } )***"; -std::string udf_hash; +std::string udf_id; auto const num_rows = static_cast(state.get_int64("num_rows")); auto const use_lto = state.get_string("use_lto") == "true"; @@ -139,12 +139,12 @@ std::string udf_hash; case cudf::binary_operator::ADD: { fragment_id = cudf_benchmark_fragments::add_f32; cuda = jit_add_cuda; - udf_hash = "add_f32"; + udf_id = "add_f32"; } break; case cudf::binary_operator::MUL: { fragment_id = cudf_benchmark_fragments::mul_f32; cuda = jit_mul_cuda; - udf_hash = "mul_f32"; + udf_id = "mul_f32"; } break; default: throw std::runtime_error("Unsupported binary operator for JIT benchmark"); } @@ -159,7 +159,7 @@ std::string udf_hash; auto result = use_lto ? cudf::transform_lto(udf, - udf_hash, + udf_id, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, @@ -184,7 +184,7 @@ std::string udf_hash; state.exec(nvbench::exec_tag::sync, [&](nvbench::launch&) { [[maybe_unused]] auto result = use_lto ? cudf::transform_lto(udf, - udf_hash, + udf_id, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, From 69690ee670b7e69885ce21051d46dbf1a7f8a7d3 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 17 Jun 2026 14:55:58 +0000 Subject: [PATCH 219/254] Refactor hash handling: Replace sha256 with hash128 - Introduced a new hash128 structure to replace the previous sha256 implementation. - Added hash128_hex_string for hexadecimal representation of hash128. - Updated cache management functions to utilize hash128 instead of sha256. - Removed sha256.hpp and its associated logic, consolidating hashing functionality. - Adjusted file handling and caching mechanisms to accommodate the new hash type. - Ensured compatibility with existing code by updating relevant function signatures and implementations. --- cpp/librtcx/README.md | 3 +- cpp/librtcx/embed.hpp | 28 ++-- cpp/librtcx/hash.hpp | 106 +++++++++++++++ cpp/librtcx/rtcx.cpp | 61 +++++---- cpp/librtcx/rtcx.hpp | 39 +++--- cpp/librtcx/sha256.hpp | 288 ----------------------------------------- cpp/src/jit/cache.cpp | 64 +++++---- 7 files changed, 207 insertions(+), 382 deletions(-) create mode 100644 cpp/librtcx/hash.hpp delete mode 100644 cpp/librtcx/sha256.hpp diff --git a/cpp/librtcx/README.md b/cpp/librtcx/README.md index 06ec01b01f82..8704c12167c2 100644 --- a/cpp/librtcx/README.md +++ b/cpp/librtcx/README.md @@ -12,7 +12,8 @@ RTCX (runtime-compiler extended) is a wrapper around NVRTC and nvJitLink designe ## Build-time Requirements - CMake >= 4.0 -- LibZSTD - for binary compression +- LibZSTD +- xxHash - CUDA >= 12.2 # Dependencies diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index d29c75bc6f88..905e676e94f1 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -4,11 +4,14 @@ */ #pragma once -#include "sha256.hpp" +#include "hash.hpp" #include #include #include + +#define XXH_INLINE_ALL +#include #include #include @@ -99,17 +102,20 @@ std::vector compress_bytes(std::span bytes, std::string_ return compressed; } -rtcx::sha256 compute_embed_hash(std::span uncompressed_files_bytes, - std::span merged_dests_bytes, - std::span merged_include_dirs_bytes, - std::string_view compression) +rtcx::hash128 compute_embed_hash(std::span uncompressed_files_bytes, + std::span merged_dests_bytes, + std::span merged_include_dirs_bytes, + std::string_view compression) { - rtcx::sha256_context ctx; - ctx.update(uncompressed_files_bytes); - ctx.update(merged_dests_bytes); - ctx.update(merged_include_dirs_bytes); - ctx.update(std::span{reinterpret_cast(compression.data()), compression.size()}); - return ctx.finalize(); + XXH3_state_t* state = XXH3_createState(); + XXH3_128bits_reset(state); + XXH3_128bits_update(state, uncompressed_files_bytes.data(), uncompressed_files_bytes.size()); + XXH3_128bits_update(state, merged_dests_bytes.data(), merged_dests_bytes.size()); + XXH3_128bits_update(state, merged_include_dirs_bytes.data(), merged_include_dirs_bytes.size()); + XXH3_128bits_update(state, compression.data(), compression.size()); + XXH128_hash_t hash = XXH3_128bits_digest(state); + XXH3_freeState(state); + return rtcx::hash128{hash.high64, hash.low64}; } template diff --git a/cpp/librtcx/hash.hpp b/cpp/librtcx/hash.hpp new file mode 100644 index 000000000000..16c8fb7c52a0 --- /dev/null +++ b/cpp/librtcx/hash.hpp @@ -0,0 +1,106 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rtcx { + +struct [[nodiscard]] hash128_hex_string { + static constexpr std::size_t NUM_HEX_DIGITS = 32; + static constexpr std::size_t NUM_HEX_BYTES = NUM_HEX_DIGITS / 2; + + char data_[NUM_HEX_DIGITS + 1]; // NOLINT(modernize-avoid-c-arrays) + + [[nodiscard]] constexpr std::string_view view() const + { + return std::string_view{data_, NUM_HEX_DIGITS}; + } + + [[nodiscard]] constexpr operator std::string_view() const { return view(); } + + [[nodiscard]] char const* data() const { return data_; } + + [[nodiscard]] char const* c_str() const { return data_; } + + [[nodiscard]] static constexpr std::size_t size() { return NUM_HEX_DIGITS; } + + static hash128_hex_string make(std::span input) + { + constexpr char const HEX_CHARS[] = "0123456789abcdef"; // NOLINT(modernize-avoid-c-arrays) + hash128_hex_string hex; + for (std::size_t i = 0; i < NUM_HEX_BYTES; ++i) { + hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; + hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; + } + hex.data_[NUM_HEX_DIGITS] = '\0'; + return hex; + } + + static hash128_hex_string make(__uint128_t hash) + { + auto array = std::bit_cast>(hash); + return make(array); + } +}; + +struct hash128 { + __uint128_t value; + + constexpr hash128(__uint128_t v = 0) : value(v) {} + + constexpr hash128(std::uint64_t high, std::uint64_t low) + : value((static_cast<__uint128_t>(high) << 64) | low) + { + } + + [[nodiscard]] constexpr bool operator==(hash128 const&) const = default; + + [[nodiscard]] std::uint8_t operator[](std::size_t index) const + { + return reinterpret_cast(&value)[16 - index]; + } + + [[nodiscard]] std::size_t size() const { return 16; } + + [[nodiscard]] std::uint8_t const* data() const + { + return reinterpret_cast(&value); + } + + hash128_hex_string to_hex_string() const { return hash128_hex_string::make(value); } + + static hash128 parse(std::string_view hex) + { + if (hex.size() != hash128_hex_string::NUM_HEX_DIGITS) { + throw std::invalid_argument( + std::format("Invalid hash128 hex string length, expected {} got {} (hash: `{}`)", + hash128_hex_string::NUM_HEX_DIGITS, + hex.size(), + hex)); + } + std::array data{}; + for (std::size_t i = 0; i < hash128_hex_string::NUM_HEX_BYTES; ++i) { + auto hex_byte = hex.substr(i * 2, 2); + auto [ptr, ec] = std::from_chars(hex_byte.begin(), hex_byte.end(), data[i], 16); + if (ec != std::errc()) { + throw std::invalid_argument( + std::format("Invalid hex character {} in HEX string: `{}`", hex_byte, hex)); + } + } + return hash128{std::bit_cast<__uint128_t>(data)}; + } +}; + +} // namespace rtcx diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 199f2b563eab..43d3ee95f36c 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,9 @@ #include #include #include + +#define XXH_INLINE_ALL +#include #include #include @@ -979,12 +982,14 @@ std::optional blob_t::from_file(char const* path) namespace { -/// @brief retrieves a blob from disk based on the given sha256 hash and object type (e.g. "blob", +/// @brief retrieves a blob from disk based on the given hash and object type (e.g. "blob", /// "cuLibrary"). Returns nullopt if the file doesn't exist on disk, and throws if any other error /// occurs. -std::optional get_disk_blob(std::string const& cache_dir, object_type type, sha256 const& sha) +std::optional get_disk_blob(std::string const& cache_dir, + object_type type, + hash128 const& hash) { - auto hex = sha.to_hex_string(); + auto hex = hash.to_hex_string(); auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(type)); auto blob = blob_t::from_file(path.c_str()); @@ -992,11 +997,11 @@ std::optional get_disk_blob(std::string const& cache_dir, object_type type return std::make_shared(std::move(*blob)); } -std::optional get_disk_library(std::string const& cache_dir, sha256 const& sha) +std::optional get_disk_library(std::string const& cache_dir, hash128 const& hash) { RTCX_FUNC_RANGE(); - auto hex = sha.to_hex_string(); + auto hex = hash.to_hex_string(); auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(object_type::LIBRARY)); // WAR: avoid a driver API call when the cache file is not present @@ -1033,7 +1038,7 @@ std::vector get_disk_entries(std::string const& cache_dir) void cache_blob_to_disk(std::string const& cache_dir, std::string const& tmp_dir, object_type type, - sha256 const& sha, + hash128 const& hash, std::span binary) { RTCX_FUNC_RANGE(); @@ -1061,7 +1066,7 @@ void cache_blob_to_disk(std::string const& cache_dir, } } - auto hex = sha.to_hex_string(); + auto hex = hash.to_hex_string(); auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(type)); std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); @@ -1084,7 +1089,7 @@ void cache_blob_to_disk(std::string const& cache_dir, } // namespace -std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compile_func compile) +std::shared_future cache_t::get_or_add_blob(hash128 const& hash, blob_compile_func compile) { RTCX_FUNC_RANGE(); @@ -1094,7 +1099,7 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil std::unique_lock lock{lock_}; // check memory cache - if (auto it = enabled_ ? blobs_cache_.entries_.find(sha) : blobs_cache_.entries_.end(); + if (auto it = enabled_ ? blobs_cache_.entries_.find(hash) : blobs_cache_.entries_.end(); it != blobs_cache_.entries_.end()) { counter_.blob_mem_hits.incr(); @@ -1108,7 +1113,7 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil // check disk cache std::optional disk_blob = std::nullopt; - if (enabled_) { disk_blob = get_disk_blob(cache_dir_, object_type::BLOB, sha); } + if (enabled_) { disk_blob = get_disk_blob(cache_dir_, object_type::BLOB, hash); } std::promise promise; auto fut = promise.get_future().share(); @@ -1121,30 +1126,30 @@ std::shared_future cache_t::get_or_add_blob(sha256 const& sha, blob_compil promise.set_value(std::move(*disk_blob)); // insert into cache - blobs_cache_.insert(sha, std::move(cache_fut), current_tick); + blobs_cache_.insert(hash, std::move(cache_fut), current_tick); return ret_fut; } else { counter_.blob_disk_misses.incr(); - blobs_cache_.insert(sha, std::move(cache_fut), current_tick); + blobs_cache_.insert(hash, std::move(cache_fut), current_tick); // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this sha + // have already reserved a spot in the cache for this hash lock.unlock(); auto result = compile(); promise.set_value(result); - cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::BLOB, sha, result->view()); + cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::BLOB, hash, result->view()); return ret_fut; } } } -std::shared_future cache_t::get_or_add_library(sha256 const& sha, +std::shared_future cache_t::get_or_add_library(hash128 const& hash, library_compile_func compile) { RTCX_FUNC_RANGE(); @@ -1155,7 +1160,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, std::unique_lock lock{lock_}; // check memory cache - if (auto it = enabled_ ? libraries_cache_.entries_.find(sha) : libraries_cache_.entries_.end(); + if (auto it = enabled_ ? libraries_cache_.entries_.find(hash) : libraries_cache_.entries_.end(); it != libraries_cache_.entries_.end()) { counter_.library_mem_hits.incr(); @@ -1169,7 +1174,7 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, // check disk cache std::optional disk_library = std::nullopt; - if (enabled_) { disk_library = get_disk_library(cache_dir_, sha); } + if (enabled_) { disk_library = get_disk_library(cache_dir_, hash); } std::promise promise; auto fut = promise.get_future().share(); @@ -1179,10 +1184,10 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, if (disk_library.has_value()) { counter_.library_disk_hits.incr(); - libraries_cache_.insert(sha, std::move(cache_fut), current_tick); + libraries_cache_.insert(hash, std::move(cache_fut), current_tick); // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this sha + // have already reserved a spot in the cache for this hash lock.unlock(); promise.set_value(std::move(*disk_library)); @@ -1192,17 +1197,17 @@ std::shared_future cache_t::get_or_add_library(sha256 const& sha, } else { counter_.library_disk_misses.incr(); - libraries_cache_.insert(sha, std::move(cache_fut), current_tick); + libraries_cache_.insert(hash, std::move(cache_fut), current_tick); // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this sha + // have already reserved a spot in the cache for this hash lock.unlock(); auto [library, blob] = compile(); promise.set_value(library); // store result to disk - cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::LIBRARY, sha, blob->view()); + cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::LIBRARY, hash, blob->view()); return ret_fut; } @@ -1290,8 +1295,8 @@ void cache_t::preload_from_disk() for (auto const& path : entries) { try { auto file_name = std::filesystem::path{path}.filename().string(); - auto sha_str = file_name.substr(0, file_name.find('.')); - auto sha = sha256::parse(sha_str); + auto hash_str = file_name.substr(0, file_name.find('.')); + auto hash = hash128::parse(hash_str); if (path.ends_with(".blob.bin")) { auto data = blob_t::from_file(path.c_str()); @@ -1300,14 +1305,14 @@ void cache_t::preload_from_disk() std::promise promise; auto fut = promise.get_future().share(); promise.set_value(std::move(blob)); - blobs_cache_.insert(sha, std::move(fut), tick_); + blobs_cache_.insert(hash, std::move(fut), tick_); } else if (path.ends_with(".cuLibrary.bin")) { - auto lib = get_disk_library(cache_dir_, sha); + auto lib = get_disk_library(cache_dir_, hash); if (!lib.has_value()) { continue; } std::promise promise; auto fut = promise.get_future().share(); promise.set_value(std::move(*lib)); - libraries_cache_.insert(sha, std::move(fut), tick_); + libraries_cache_.insert(hash, std::move(fut), tick_); } } catch (std::exception const& e) { // ignore any errors during preload diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index bc3ff8fd5569..3f1a53112c8f 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -1,11 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#include "sha256.hpp" +#include #include #include @@ -104,26 +104,24 @@ func(void*, R (*)(void*, Args...)) -> func; template func(R (*)(Args...)) -> func; -struct [[nodiscard]] sha256_hasher { - constexpr std::uint64_t operator()(sha256 const& obj) const +struct [[nodiscard]] hash128_hasher { + constexpr std::uint64_t operator()(hash128 const& obj) const { - struct u64x4 { - alignas(alignof(sha256)) std::uint64_t // NOLINT(modernize-avoid-c-arrays) - v[sizeof(sha256) / sizeof(std::uint64_t)]; + struct u64x2 { + alignas(alignof(hash128)) std::uint64_t // NOLINT(modernize-avoid-c-arrays) + v[sizeof(hash128) / sizeof(std::uint64_t)]; }; - auto value = std::bit_cast(obj); + auto value = std::bit_cast(obj); auto h0 = value.v[0]; auto h1 = value.v[1]; - auto h2 = value.v[2]; - auto h3 = value.v[3]; auto mix = [](std::uint64_t seed, std::uint64_t v) { seed ^= v + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); return seed; }; - return mix(mix(mix(h0, h1), h2), h3); + return mix(h0, h1); } }; @@ -477,7 +475,7 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { void hit(std::uint64_t tick) { last_touched_tick = tick; } }; - std::unordered_map entries_ = {}; + std::unordered_map entries_ = {}; std::size_t limit_; explicit lru_memory_cache(std::size_t limit) : limit_{limit} @@ -492,7 +490,7 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { auto num_to_purge = (entries_.size() + 1) / 2; - std::vector> rankings; + std::vector> rankings; rankings.reserve(entries_.size()); for (auto& [key, entry] : entries_) { @@ -510,7 +508,7 @@ struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { } } - void insert(sha256 const& sha, T&& value, std::uint64_t tick) + void insert(hash128 const& sha, T&& value, std::uint64_t tick) { if (limit_ == 0) { return; } @@ -648,23 +646,22 @@ struct cache_t { // NOLINT [[nodiscard]] std::string const& get_tmp_dir(); /** - * @brief Query the cache for a compiled blob by its SHA-256 hash, or insert it if not present - * @param sha SHA-256 hash of the blob to query or insert + * @brief Query the cache for a compiled blob by its hash, or insert it if not present + * @param hash hash of the blob to query or insert * @param compile Function to compile the blob if it's not found in the cache * @return A shared future that will hold the compiled blob once it's available */ - [[nodiscard]] std::shared_future get_or_add_blob(sha256 const& sha, + [[nodiscard]] std::shared_future get_or_add_blob(hash128 const& hash, blob_compile_func compile); /** - * @brief Query the cache for a compiled library by its SHA-256 hash and binary type, or insert + * @brief Query the cache for a compiled library by its hash and binary type, or insert * it if not present - * @param sha SHA-256 hash of the library to query or insert - * @param type Binary type of the library (e.g., CUBIN, PTX) + * @param hash hash of the library to query or insert * @param compile Function to compile the library if it's not found in the cache * @return A shared future that will hold the compiled library once it's available */ - [[nodiscard]] std::shared_future get_or_add_library(sha256 const& sha, + [[nodiscard]] std::shared_future get_or_add_library(hash128 const& hash, library_compile_func compile); /** diff --git a/cpp/librtcx/sha256.hpp b/cpp/librtcx/sha256.hpp deleted file mode 100644 index 1149477f4b38..000000000000 --- a/cpp/librtcx/sha256.hpp +++ /dev/null @@ -1,288 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include -#include -#include - -namespace rtcx { - -namespace functions { - -inline std::uint32_t ror(std::uint32_t x, std::uint32_t n) { return (x >> n) | (x << (32 - n)); } - -inline std::uint32_t ch(std::uint32_t x, std::uint32_t y, std::uint32_t z) -{ - return z ^ (x & (y ^ z)); -} - -inline std::uint32_t maj(std::uint32_t x, std::uint32_t y, std::uint32_t z) -{ - return ((x | y) & z) | (x & y); -} - -inline std::uint32_t sigma0(std::uint32_t x) { return ror(x, 2) ^ ror(x, 13) ^ ror(x, 22); } - -inline std::uint32_t sigma1(std::uint32_t x) { return ror(x, 6) ^ ror(x, 11) ^ ror(x, 25); } - -inline std::uint32_t gamma0(std::uint32_t x) { return ror(x, 7) ^ ror(x, 18) ^ (x >> 3); } - -inline std::uint32_t gamma1(std::uint32_t x) { return ror(x, 17) ^ ror(x, 19) ^ (x >> 10); } - -inline void put_be32(void* ptr, std::uint32_t value) -{ - auto* p = (std::uint8_t*)ptr; - p[0] = (value >> 24) & 0xff; - p[1] = (value >> 16) & 0xff; - p[2] = (value >> 8) & 0xff; - p[3] = (value >> 0) & 0xff; -} - -inline std::uint32_t get_be32(void const* ptr) -{ - auto const* p = (std::uint8_t const*)ptr; - return (std::uint32_t)p[0] << 24 | (std::uint32_t)p[1] << 16 | (std::uint32_t)p[2] << 8 | - (std::uint32_t)p[3] << 0; -} - -} // namespace functions - -struct [[nodiscard]] sha256_hex_string { - char data_[65]; // NOLINT(modernize-avoid-c-arrays) - - [[nodiscard]] constexpr std::string_view view() const { return std::string_view{data_, 64}; } - - [[nodiscard]] constexpr operator std::string_view() const { return view(); } - - [[nodiscard]] char const* data() const { return data_; } - - [[nodiscard]] char const* c_str() const { return data_; } - - [[nodiscard]] static constexpr std::size_t size() { return 64; } - - static sha256_hex_string make(std::span input) - { - constexpr char const HEX_CHARS[] = "0123456789abcdef"; // NOLINT(modernize-avoid-c-arrays) - sha256_hex_string hex; - for (std::size_t i = 0; i < 32; ++i) { - hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; - hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; - } - hex.data_[64] = '\0'; - return hex; - } -}; - -struct [[nodiscard]] sha256 { - alignas(16) std::uint8_t data_[32]; // NOLINT(modernize-avoid-c-arrays) - - [[nodiscard]] std::uint8_t operator[](std::size_t index) const { return data_[index]; } - - [[nodiscard]] std::size_t size() const { return 32; } - - [[nodiscard]] std::uint8_t const* data() const { return data_; } - - [[nodiscard]] constexpr bool operator==(sha256 const&) const = default; - - sha256_hex_string to_hex_string() const { return sha256_hex_string::make(data_); } - - static sha256 parse(std::string_view hex) - { - if (hex.size() != 64) { - throw std::invalid_argument(std::format( - "Invalid SHA256 hex string length, expected 64 got {} (sha: `{}`)", hex.size(), hex)); - } - sha256 hash; - for (std::size_t i = 0; i < 32; ++i) { - auto hex_byte = hex.substr(i * 2, 2); - auto [ptr, ec] = std::from_chars(hex_byte.begin(), hex_byte.end(), hash.data_[i], 16); - if (ec != std::errc()) { - throw std::invalid_argument( - std::format("Invalid hex character in SHA256 string: `{}`", hex_byte)); - } - } - return hash; - } -}; - -struct sha256_context { - private: - static constexpr std::size_t BLOCK_SIZE = 64; - std::uint32_t state_[8] = // NOLINT(modernize-avoid-c-arrays) - {0x6a09'e667ul, - 0xbb67'ae85ul, - 0x3c6e'f372ul, - 0xa54f'f53aul, - 0x510e'527ful, - 0x9b05'688cul, - 0x1f83'd9abul, - 0x5be0'cd19ul}; - std::uint64_t size_ = 0; - std::uint8_t buf_[BLOCK_SIZE] = {}; // NOLINT(modernize-avoid-c-arrays) - - public: - sha256_context() = default; - sha256_context(sha256_context const&) = delete; - sha256_context& operator=(sha256_context const&) = delete; - sha256_context(sha256_context&&) = delete; - sha256_context& operator=(sha256_context&&) = delete; - ~sha256_context() = default; - - private: - void transform(std::uint8_t const* buf) - { - std::uint32_t S[8], W[64], t0, t1; // NOLINT(modernize-avoid-c-arrays) - int i; - - /* copy state into S */ - for (i = 0; i < 8; i++) - S[i] = state_[i]; - - /* copy the state into 512-bits into W[0..15] */ - for (i = 0; i < 16; i++, buf += sizeof(std::uint32_t)) - W[i] = functions::get_be32(buf); - - /* fill W[16..63] */ - for (i = 16; i < 64; i++) - W[i] = functions::gamma1(W[i - 2]) + W[i - 7] + functions::gamma0(W[i - 15]) + W[i - 16]; - -#define RND(a, b, c, d, e, f, g, h, i, ki) \ - t0 = h + functions::sigma1(e) + functions::ch(e, f, g) + ki + W[i]; \ - t1 = functions::sigma0(a) + functions::maj(a, b, c); \ - d += t0; \ - h = t0 + t1; - - RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 0, 0x428a'2f98); - RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 1, 0x7137'4491); - RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 2, 0xb5c0'fbcf); - RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 3, 0xe9b5'dba5); - RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 4, 0x3956'c25b); - RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 5, 0x59f1'11f1); - RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 6, 0x923f'82a4); - RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 7, 0xab1c'5ed5); - RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 8, 0xd807'aa98); - RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 9, 0x1283'5b01); - RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 10, 0x2431'85be); - RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 11, 0x550c'7dc3); - RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 12, 0x72be'5d74); - RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 13, 0x80de'b1fe); - RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 14, 0x9bdc'06a7); - RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 15, 0xc19b'f174); - RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 16, 0xe49b'69c1); - RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 17, 0xefbe'4786); - RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 18, 0x0fc1'9dc6); - RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 19, 0x240c'a1cc); - RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 20, 0x2de9'2c6f); - RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 21, 0x4a74'84aa); - RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 22, 0x5cb0'a9dc); - RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 23, 0x76f9'88da); - RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 24, 0x983e'5152); - RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 25, 0xa831'c66d); - RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 26, 0xb003'27c8); - RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 27, 0xbf59'7fc7); - RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 28, 0xc6e0'0bf3); - RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 29, 0xd5a7'9147); - RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 30, 0x06ca'6351); - RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 31, 0x1429'2967); - RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 32, 0x27b7'0a85); - RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 33, 0x2e1b'2138); - RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 34, 0x4d2c'6dfc); - RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 35, 0x5338'0d13); - RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 36, 0x650a'7354); - RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 37, 0x766a'0abb); - RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 38, 0x81c2'c92e); - RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 39, 0x9272'2c85); - RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 40, 0xa2bf'e8a1); - RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 41, 0xa81a'664b); - RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 42, 0xc24b'8b70); - RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 43, 0xc76c'51a3); - RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 44, 0xd192'e819); - RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 45, 0xd699'0624); - RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 46, 0xf40e'3585); - RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 47, 0x106a'a070); - RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 48, 0x19a4'c116); - RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 49, 0x1e37'6c08); - RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 50, 0x2748'774c); - RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 51, 0x34b0'bcb5); - RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 52, 0x391c'0cb3); - RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 53, 0x4ed8'aa4a); - RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 54, 0x5b9c'ca4f); - RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 55, 0x682e'6ff3); - RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], 56, 0x748f'82ee); - RND(S[7], S[0], S[1], S[2], S[3], S[4], S[5], S[6], 57, 0x78a5'636f); - RND(S[6], S[7], S[0], S[1], S[2], S[3], S[4], S[5], 58, 0x84c8'7814); - RND(S[5], S[6], S[7], S[0], S[1], S[2], S[3], S[4], 59, 0x8cc7'0208); - RND(S[4], S[5], S[6], S[7], S[0], S[1], S[2], S[3], 60, 0x90be'fffa); - RND(S[3], S[4], S[5], S[6], S[7], S[0], S[1], S[2], 61, 0xa450'6ceb); - RND(S[2], S[3], S[4], S[5], S[6], S[7], S[0], S[1], 62, 0xbef9'a3f7); - RND(S[1], S[2], S[3], S[4], S[5], S[6], S[7], S[0], 63, 0xc671'78f2); - -#undef RND - - for (i = 0; i < 8; i++) { - state_[i] += S[i]; - } - } - - public: - void update(std::span span) - { - std::uint32_t len_buf = size_ & 63; - auto data = span.data(); - auto len = span.size(); - - size_ += len; - - /* Read the data into buf and process blocks as they get full */ - if (len_buf) { - std::uint32_t left = 64 - len_buf; - if (len < left) left = len; - std::memcpy(len_buf + buf_, data, left); - len_buf = (len_buf + left) & 63; - len -= left; - data = (data + left); - if (len_buf) return; - transform(buf_); - } - - while (len >= 64) { - transform(data); - data = data + 64; - len -= 64; - } - - if (len) std::memcpy(buf_, data, len); - } - - sha256 finalize() - { - static std::uint8_t const pad[64] = {0x80}; // NOLINT(modernize-avoid-c-arrays) - std::uint32_t padlen[2]; // NOLINT(modernize-avoid-c-arrays) - int i; - - /* Pad with a binary 1 (ie 0x80), then zeroes, then length */ - padlen[0] = __builtin_bswap32((std::uint32_t)(size_ >> 29)); - padlen[1] = __builtin_bswap32((std::uint32_t)(size_ << 3)); - - i = size_ & 63; - update(std::span{pad, (std::size_t)(1 + (63 & (55 - i)))}); - update(std::span{reinterpret_cast(padlen), sizeof(padlen)}); - - sha256 out; - std::uint8_t* digest = out.data_; - - /* copy output */ - for (i = 0; i < 8; i++, digest += sizeof(std::uint32_t)) { - functions::put_be32(digest, state_[i]); - } - return out; - } -}; - -} // namespace rtcx diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 10a2e5558737..74f26fa95e21 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -23,20 +24,16 @@ namespace CUDF_EXPORT cudf { namespace { -rtcx::sha256 hash(std::span input) +void hash(XXH3_state_t* ctx, std::span input) { - rtcx::sha256_context ctx; - ctx.update(std::span{reinterpret_cast(input.data()), input.size()}); - return ctx.finalize(); + XXH3_128bits_update(ctx, input.data(), input.size()); } -rtcx::sha256 hash(std::span inputs) +void hash(XXH3_state_t* ctx, std::span inputs) { - rtcx::sha256_context ctx; for (auto const* input : inputs) { - ctx.update(std::span{reinterpret_cast(input), std::strlen(input)}); + XXH3_128bits_update(ctx, input, std::strlen(input)); } - return ctx.finalize(); } void install_file_set( @@ -149,7 +146,7 @@ void jit_bundle_t::ensure_installed() const std::string jit_bundle_t::get_hash() const { - auto str = rtcx::sha256_hex_string::make(cudf_cuda_embed::hash); + auto str = rtcx::hash128_hex_string::make(cudf_cuda_embed::hash); return std::string{str.view()}; } @@ -293,15 +290,16 @@ kernel get_kernel(std::string const& name, auto& cache = cudf::get_context().rtcx_cache(); auto& bundle = cudf::get_context().jit_bundle(); - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_compute_capability(); - auto header_include_names_hash = hash(header_include_names).to_hex_string(); - auto headers_hash = hash(headers).to_hex_string(); - auto bundle_hash = bundle.get_hash(); - auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_compute_capability(); + auto bundle_hash = bundle.get_hash(); + auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); - auto cache_key = std::format(R"***(cuLibrary + XXH3_state_t* state = XXH3_createState(); + XXH3_128bits_reset(state); + RTCX_DEFER([state] { XXH3_freeState(state); }); + auto spec = std::format(R"***(cuLibrary name={} binary_type=CUBIN cuda_runtime={} @@ -309,21 +307,22 @@ cuda_driver={} arch={} bundle={} source_file={} -header_include_names={} -headers={} kernel_instance={} )***", - name, - runtime, - driver, - sm, - bundle_hash, - source_file, - header_include_names_hash.view(), - headers_hash.view(), - kernel_instance); - - auto cache_key_sha256 = hash(cache_key); + name, + runtime, + driver, + sm, + bundle_hash, + source_file, + kernel_instance); + + hash(state, spec); + hash(state, header_include_names); + hash(state, headers); + auto digest = XXH3_128bits_digest(state); + + rtcx::hash128 cache_key{digest.high64, digest.low64}; auto compile = [&] { auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); @@ -331,8 +330,7 @@ kernel_instance={} return compile_library(name.c_str(), source.c_str(), header_include_names, headers, {}); }; - auto fut = - cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); + auto fut = cache.get_or_add_library(cache_key, rtcx::library_compile_func::from_functor(compile)); auto lib = fut.get(); return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; From c479bad897a0cb126c8fea92f3a9124911f14cf1 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 17 Jun 2026 16:20:19 +0000 Subject: [PATCH 220/254] Fix index calculation in hash128 operator[] for correct byte access --- cpp/librtcx/hash.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/librtcx/hash.hpp b/cpp/librtcx/hash.hpp index 16c8fb7c52a0..ce573c24d58b 100644 --- a/cpp/librtcx/hash.hpp +++ b/cpp/librtcx/hash.hpp @@ -69,7 +69,7 @@ struct hash128 { [[nodiscard]] std::uint8_t operator[](std::size_t index) const { - return reinterpret_cast(&value)[16 - index]; + return reinterpret_cast(&value)[15 - index]; } [[nodiscard]] std::size_t size() const { return 16; } From 454a03674680f130017726ae7692ceb4d3452076 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 17 Jun 2026 16:22:07 +0000 Subject: [PATCH 221/254] Add state creation checks for XXH3 in hash computation functions --- cpp/librtcx/embed.hpp | 1 + cpp/src/jit/cache.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index 905e676e94f1..015e2c301ec1 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -108,6 +108,7 @@ rtcx::hash128 compute_embed_hash(std::span uncompressed_files_byt std::string_view compression) { XXH3_state_t* state = XXH3_createState(); + RTCX_EMBED_EXPECTS(state != nullptr, "Failed to create XXH3 state"); XXH3_128bits_reset(state); XXH3_128bits_update(state, uncompressed_files_bytes.data(), uncompressed_files_bytes.size()); XXH3_128bits_update(state, merged_dests_bytes.data(), merged_dests_bytes.size()); diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 74f26fa95e21..38dceec432ee 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -297,6 +297,7 @@ kernel get_kernel(std::string const& name, auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); XXH3_state_t* state = XXH3_createState(); + CUDF_EXPECTS(state != nullptr, "Failed to create XXH3 state", std::runtime_error); XXH3_128bits_reset(state); RTCX_DEFER([state] { XXH3_freeState(state); }); auto spec = std::format(R"***(cuLibrary From b8bf0f26552f95188ef7a2570c78103cbbe8a655 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 17 Jun 2026 16:39:50 +0000 Subject: [PATCH 222/254] Add null terminator update in hash function for XXH3 --- cpp/src/jit/cache.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 38dceec432ee..58e8ba6dd071 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -33,6 +33,7 @@ void hash(XXH3_state_t* ctx, std::span inputs) { for (auto const* input : inputs) { XXH3_128bits_update(ctx, input, std::strlen(input)); + XXH3_128bits_update(ctx, "\0", 1); // null terminator } } From 99f5d918600b607335ae4e00b88cf132b8c3a512 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 17 Jun 2026 16:49:52 +0000 Subject: [PATCH 223/254] remove heap allocation in xxhash state --- cpp/librtcx/embed.hpp | 17 ++++++++--------- cpp/src/jit/cache.cpp | 15 +++++++-------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index 015e2c301ec1..87281bfc1496 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -107,15 +107,14 @@ rtcx::hash128 compute_embed_hash(std::span uncompressed_files_byt std::span merged_include_dirs_bytes, std::string_view compression) { - XXH3_state_t* state = XXH3_createState(); - RTCX_EMBED_EXPECTS(state != nullptr, "Failed to create XXH3 state"); - XXH3_128bits_reset(state); - XXH3_128bits_update(state, uncompressed_files_bytes.data(), uncompressed_files_bytes.size()); - XXH3_128bits_update(state, merged_dests_bytes.data(), merged_dests_bytes.size()); - XXH3_128bits_update(state, merged_include_dirs_bytes.data(), merged_include_dirs_bytes.size()); - XXH3_128bits_update(state, compression.data(), compression.size()); - XXH128_hash_t hash = XXH3_128bits_digest(state); - XXH3_freeState(state); + XXH3_state_t state; + XXH3_INITSTATE(&state); + XXH3_128bits_reset(&state); + XXH3_128bits_update(&state, uncompressed_files_bytes.data(), uncompressed_files_bytes.size()); + XXH3_128bits_update(&state, merged_dests_bytes.data(), merged_dests_bytes.size()); + XXH3_128bits_update(&state, merged_include_dirs_bytes.data(), merged_include_dirs_bytes.size()); + XXH3_128bits_update(&state, compression.data(), compression.size()); + XXH128_hash_t hash = XXH3_128bits_digest(&state); return rtcx::hash128{hash.high64, hash.low64}; } diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 58e8ba6dd071..a766dde4863b 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -297,10 +297,9 @@ kernel get_kernel(std::string const& name, auto bundle_hash = bundle.get_hash(); auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); - XXH3_state_t* state = XXH3_createState(); - CUDF_EXPECTS(state != nullptr, "Failed to create XXH3 state", std::runtime_error); - XXH3_128bits_reset(state); - RTCX_DEFER([state] { XXH3_freeState(state); }); + XXH3_state_t state; + XXH3_INITSTATE(&state); + XXH3_128bits_reset(&state); auto spec = std::format(R"***(cuLibrary name={} binary_type=CUBIN @@ -319,10 +318,10 @@ kernel_instance={} source_file, kernel_instance); - hash(state, spec); - hash(state, header_include_names); - hash(state, headers); - auto digest = XXH3_128bits_digest(state); + hash(&state, spec); + hash(&state, header_include_names); + hash(&state, headers); + auto digest = XXH3_128bits_digest(&state); rtcx::hash128 cache_key{digest.high64, digest.low64}; From 9f5e8877d61aff23d977572a6c24c79e77199b2b Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 17 Jun 2026 16:52:44 +0000 Subject: [PATCH 224/254] Refactor hash state initialization in get_kernel function --- cpp/src/jit/cache.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index a766dde4863b..df955ca3085c 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -297,9 +297,6 @@ kernel get_kernel(std::string const& name, auto bundle_hash = bundle.get_hash(); auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); - XXH3_state_t state; - XXH3_INITSTATE(&state); - XXH3_128bits_reset(&state); auto spec = std::format(R"***(cuLibrary name={} binary_type=CUBIN @@ -318,12 +315,15 @@ kernel_instance={} source_file, kernel_instance); + XXH3_state_t state; + XXH3_INITSTATE(&state); + XXH3_128bits_reset(&state); hash(&state, spec); hash(&state, header_include_names); hash(&state, headers); - auto digest = XXH3_128bits_digest(&state); - rtcx::hash128 cache_key{digest.high64, digest.low64}; + auto digest = XXH3_128bits_digest(&state); + auto cache_key = rtcx::hash128{digest.high64, digest.low64}; auto compile = [&] { auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); From 510ecdc9544b73db69ff2f83b7604771069b8170 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 18 Jun 2026 01:42:03 +0000 Subject: [PATCH 225/254] Add xxHash integration and update dependencies in CMake configuration --- cpp/CMakeLists.txt | 5 ++- cpp/cmake/thirdparty/get_xxhash.cmake | 48 +++++++++++++++++++++++++++ cpp/librtcx/embed.cmake | 8 +++-- cpp/librtcx/embed.hpp | 2 +- cpp/librtcx/rtcx.cpp | 3 -- cpp/src/jit/cache.cpp | 4 ++- 6 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 cpp/cmake/thirdparty/get_xxhash.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 88e12f801caf..3e325e8448c2 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -407,6 +407,9 @@ include(cmake/thirdparty/get_nanoarrow.cmake) # find thread_pool include(cmake/thirdparty/get_thread_pool.cmake) +# find xxhash +include(cmake/thirdparty/get_xxhash.cmake) + # find zstd include(cmake/thirdparty/get_zstd.cmake) @@ -1165,7 +1168,7 @@ target_link_libraries( cudf PUBLIC CCCL::CCCL $ PRIVATE $ $ ZLIB::ZLIB - ${CUDF_nvcomp_TARGET} kvikio::kvikio ${CUDF_nanoarrow_TARGET} zstd rtcx::rtcx + ${CUDF_nvcomp_TARGET} kvikio::kvikio ${CUDF_nanoarrow_TARGET} zstd xxhash rtcx::rtcx ) # When building a shared libcudf with static deps, we absorb them via whole-archive linking so that diff --git a/cpp/cmake/thirdparty/get_xxhash.cmake b/cpp/cmake/thirdparty/get_xxhash.cmake new file mode 100644 index 000000000000..152ebaff330d --- /dev/null +++ b/cpp/cmake/thirdparty/get_xxhash.cmake @@ -0,0 +1,48 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +# Use CPM to find or clone xxHash +function(find_and_configure_xxhash) + + include(${rapids-cmake-dir}/find/generate_module.cmake) + rapids_find_generate_module(xxhash HEADER_NAMES xxhash.h xxh3.h) + + set(CPM_DOWNLOAD_xxhash ON) + rapids_cpm_find( + xxhash 0.8.3 + GLOBAL_TARGETS xxhash + CPM_ARGS + GIT_REPOSITORY https://github.com/Cyan4973/xxHash.git + GIT_TAG v0.8.3 + GIT_SHALLOW TRUE + DOWNLOAD_ONLY TRUE + EXCLUDE_FROM_ALL ${CUDF_EXCLUDE_DEPS_FROM_ALL} + ) + + if(xxhash_ADDED AND NOT TARGET xxhash) + add_library(xxhash INTERFACE) + target_include_directories(xxhash INTERFACE "${xxhash_SOURCE_DIR}") + endif() + + if(DEFINED xxhash_SOURCE_DIR) + set(XXHASH_INCLUDE_DIR + "${xxhash_SOURCE_DIR}" + PARENT_SCOPE + ) + set(xxhash_SOURCE_DIR + "${xxhash_SOURCE_DIR}" + PARENT_SCOPE + ) + endif() + + if(DEFINED xxhash_SOURCE_DIR) + include("${rapids-cmake-dir}/export/find_package_root.cmake") + rapids_export_find_package_root(BUILD xxhash "${xxhash_SOURCE_DIR}" EXPORT_SET cudf-exports) + endif() +endfunction() + +find_and_configure_xxhash() \ No newline at end of file diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index b4510b6d1c64..5d9fb906c7e0 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -6,7 +6,11 @@ # ============================================================================= if(NOT TARGET zstd) - message(FATAL_ERROR "embed(): zstd target is required for LIBRTCX embedding.") + message(FATAL_ERROR "zstd target is required for LIBRTCX embedding.") +endif() + +if(NOT TARGET xxhash) + message(FATAL_ERROR "xxhash target is required for LIBRTCX embedding.") endif() # This function initializes a target for JIT embedding. It must be called before any calls to @@ -272,7 +276,7 @@ function(embed TARGET) set(RUNNER "${TARGET}__jit_embed_run") add_executable(${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") - target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} zstd) + target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} zstd xxhash) target_include_directories( ${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR} ${ZSTD_INCLUDE_DIR} ) diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp index 87281bfc1496..5df1aa4db40b 100644 --- a/cpp/librtcx/embed.hpp +++ b/cpp/librtcx/embed.hpp @@ -114,7 +114,7 @@ rtcx::hash128 compute_embed_hash(std::span uncompressed_files_byt XXH3_128bits_update(&state, merged_dests_bytes.data(), merged_dests_bytes.size()); XXH3_128bits_update(&state, merged_include_dirs_bytes.data(), merged_include_dirs_bytes.size()); XXH3_128bits_update(&state, compression.data(), compression.size()); - XXH128_hash_t hash = XXH3_128bits_digest(&state); + auto hash = XXH3_128bits_digest(&state); return rtcx::hash128{hash.high64, hash.low64}; } diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index 43d3ee95f36c..a0d057ed4c90 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -15,9 +15,6 @@ #include #include #include - -#define XXH_INLINE_ALL -#include #include #include diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index df955ca3085c..f3d708f4bee0 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -13,7 +13,9 @@ #include #include #include -#include + +#define XXH_INLINE_ALL +#include #include #include From 1dafe3924d516330fba56df1ad407e17f30a23ca Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 18 Jun 2026 01:44:40 +0000 Subject: [PATCH 226/254] Refactor hash functions to include null terminators for file and memory fragments --- cpp/src/jit/cache.cpp | 122 ++++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 53 deletions(-) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 36e6d626ff11..96a9a50d067e 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -37,23 +37,25 @@ void hash(XXH3_state_t* ctx, std::span inputs) } } -rtcx::sha256 hash(std::span file_fragments, - std::span memory_fragments) +void hash(XXH3_state_t* ctx, std::span file_fragments) { - rtcx::sha256_context ctx; for (auto const& fragment : file_fragments) { - ctx.update( - std::span{reinterpret_cast(fragment.path), std::strlen(fragment.path)}); + XXH3_128bits_update(ctx, fragment.path, std::strlen(fragment.path)); + XXH3_128bits_update(ctx, "\0", 1); // null terminator } +} + +void hash(XXH3_state_t* ctx, std::span memory_fragments) +{ for (auto const& fragment : memory_fragments) { if (fragment.name != nullptr) { - ctx.update( - std::span{reinterpret_cast(fragment.name), std::strlen(fragment.name)}); + XXH3_128bits_update(ctx, fragment.name, std::strlen(fragment.name)); + XXH3_128bits_update(ctx, "\0", 1); // null terminator } else { - ctx.update(fragment.data); + XXH3_128bits_update(ctx, fragment.data.data(), fragment.data.size()); + XXH3_128bits_update(ctx, "\0", 1); // null terminator } } - return ctx.finalize(); } void install_file_set( @@ -398,6 +400,7 @@ kernel get_kernel(std::string const& name, XXH3_state_t state; XXH3_INITSTATE(&state); XXH3_128bits_reset(&state); + auto spec = std::format(R"***(cuLibrary name={} binary_type=CUBIN @@ -417,11 +420,13 @@ kernel_instance={} kernel_instance); hash(&state, spec); + hash(&state, "header_include_names: "); hash(&state, header_include_names); + hash(&state, "headers: "); hash(&state, headers); - auto digest = XXH3_128bits_digest(&state); - rtcx::hash128 cache_key{digest.high64, digest.low64}; + auto digest = XXH3_128bits_digest(&state); + auto key = rtcx::hash128{digest.high64, digest.low64}; auto compile = [&] { auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); @@ -429,7 +434,7 @@ kernel_instance={} return compile_library(name.c_str(), source.c_str(), header_include_names, headers, {}); }; - auto fut = cache.get_or_add_library(cache_key, rtcx::library_compile_func::from_functor(compile)); + auto fut = cache.get_or_add_library(key, rtcx::library_compile_func::from_functor(compile)); auto lib = fut.get(); return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; @@ -446,15 +451,13 @@ rtcx::blob get_kernel_fragment(std::string const& name, auto& cache = cudf::get_context().rtcx_cache(); auto& bundle = cudf::get_context().jit_bundle(); - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_compute_capability(); - auto header_include_names_hash = hash(header_include_names).to_hex_string(); - auto headers_hash = hash(headers).to_hex_string(); - auto bundle_hash = bundle.get_hash(); - auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_compute_capability(); + auto bundle_hash = bundle.get_hash(); + auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); - auto cache_key = std::format(R"***(objectFile + auto spec = std::format(R"***(objectFile name={} binary_type=LTO_IR cuda_runtime={} @@ -462,21 +465,28 @@ cuda_driver={} arch={} bundle={} source_file={} -header_include_names={} -headers={} kernel_instance={} )***", - name, - runtime, - driver, - sm, - bundle_hash, - source_file, - header_include_names_hash.view(), - headers_hash.view(), - kernel_instance); - - auto cache_key_sha256 = hash(cache_key); + name, + runtime, + driver, + sm, + bundle_hash, + source_file, + kernel_instance); + + XXH3_state_t state; + XXH3_INITSTATE(&state); + XXH3_128bits_reset(&state); + + hash(&state, spec); + hash(&state, "header_include_names: "); + hash(&state, header_include_names); + hash(&state, "headers: "); + hash(&state, headers); + + auto digest = XXH3_128bits_digest(&state); + auto key = rtcx::hash128{digest.high64, digest.low64}; auto compile = [&] { auto bundle_dir = cudf::get_context().jit_bundle().get_directory(); @@ -484,8 +494,7 @@ kernel_instance={} return compile_fragment(name.c_str(), source.c_str(), header_include_names, headers, {}); }; - auto fut = - cache.get_or_add_blob(cache_key_sha256, rtcx::blob_compile_func::from_functor(compile)); + auto fut = cache.get_or_add_blob(key, rtcx::blob_compile_func::from_functor(compile)); return fut.get(); } @@ -537,38 +546,45 @@ kernel get_lto_linked_kernel(std::string const& name, { CUDF_FUNC_RANGE(); - auto& cache = cudf::get_context().rtcx_cache(); - auto& bundle = cudf::get_context().jit_bundle(); - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_compute_capability(); - auto bundle_hash = bundle.get_hash(); - auto fragments_hash = hash(file_fragments, memory_fragments).to_hex_string(); + auto& cache = cudf::get_context().rtcx_cache(); + auto& bundle = cudf::get_context().jit_bundle(); + auto runtime = get_runtime_version(); + auto driver = get_driver_version(); + auto sm = get_current_device_compute_capability(); + auto bundle_hash = bundle.get_hash(); - auto cache_key = std::format(R"***(cuLibrary + auto spec = std::format(R"***(cuLibrary name={} binary_type=CUBIN cuda_runtime={} cuda_driver={} arch={} bundle={} -fragments={} )***", - name, - runtime, - driver, - sm, - bundle_hash, - fragments_hash.view()); + name, + runtime, + driver, + sm, + bundle_hash); - auto cache_key_sha256 = hash(cache_key); + XXH3_state_t state; + XXH3_INITSTATE(&state); + XXH3_128bits_reset(&state); + + hash(&state, spec); + hash(&state, "file_fragments: "); + hash(&state, file_fragments); + hash(&state, "memory_fragments: "); + hash(&state, memory_fragments); + + auto digest = XXH3_128bits_digest(&state); + auto key = rtcx::hash128{digest.high64, digest.low64}; auto compile = [&] { return link_library_uncached(name.c_str(), file_fragments, memory_fragments); }; - auto fut = - cache.get_or_add_library(cache_key_sha256, rtcx::library_compile_func::from_functor(compile)); + auto fut = cache.get_or_add_library(key, rtcx::library_compile_func::from_functor(compile)); auto lib = fut.get(); return kernel{lib, lib->get_kernel("cudf_kernel_entry")}; From f5120e1548d33252dc54c89329645dab80e5c174 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 18 Jun 2026 01:48:09 +0000 Subject: [PATCH 227/254] Remove unnecessary blank lines in hash initialization for kernel and bundle --- cpp/src/jit/cache.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 007cfb2a4f2b..21a8b2c77f9d 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -479,7 +479,6 @@ kernel_instance={} XXH3_state_t state; XXH3_INITSTATE(&state); XXH3_128bits_reset(&state); - hash(&state, spec); hash(&state, "header_include_names: "); hash(&state, header_include_names); @@ -571,7 +570,6 @@ bundle={} XXH3_state_t state; XXH3_INITSTATE(&state); XXH3_128bits_reset(&state); - hash(&state, spec); hash(&state, "file_fragments: "); hash(&state, file_fragments); From a5d330ff70848dd39afe1ad9316968e8836df185 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 18 Jun 2026 02:06:09 +0000 Subject: [PATCH 228/254] Update copyright statements in CMake files to include "AFFILIATES" for consistency --- cpp/CMakeLists.txt | 2 +- cpp/cmake/thirdparty/get_xxhash.cmake | 4 ++-- cpp/librtcx/embed.cmake | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3e325e8448c2..a92439ed9963 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= diff --git a/cpp/cmake/thirdparty/get_xxhash.cmake b/cpp/cmake/thirdparty/get_xxhash.cmake index 152ebaff330d..558b1ee5dda3 100644 --- a/cpp/cmake/thirdparty/get_xxhash.cmake +++ b/cpp/cmake/thirdparty/get_xxhash.cmake @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -45,4 +45,4 @@ function(find_and_configure_xxhash) endif() endfunction() -find_and_configure_xxhash() \ No newline at end of file +find_and_configure_xxhash() diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index 5d9fb906c7e0..2d82c396d380 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= From 1bef8cbeefb1ebfae5dd1446cba255cc473e100e Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 18 Jun 2026 11:12:02 +0000 Subject: [PATCH 229/254] Remove udf_id parameter from transform functions and tests --- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 5 ----- cpp/include/cudf/transform.hpp | 3 --- cpp/src/transform/transform.cu | 7 +------ cpp/tests/transform/transform_lto_test.cpp | 4 ---- 4 files changed, 1 insertion(+), 18 deletions(-) diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 5a84b1f9f181..625560d3e15e 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -119,7 +119,6 @@ __device__ void transform(float* out, float a, float b) { *out = a + b; } )***"; -std::string udf_id; auto const num_rows = static_cast(state.get_int64("num_rows")); auto const use_lto = state.get_string("use_lto") == "true"; @@ -139,12 +138,10 @@ std::string udf_id; case cudf::binary_operator::ADD: { fragment_id = cudf_benchmark_fragments::add_f32; cuda = jit_add_cuda; - udf_id = "add_f32"; } break; case cudf::binary_operator::MUL: { fragment_id = cudf_benchmark_fragments::mul_f32; cuda = jit_mul_cuda; - udf_id = "mul_f32"; } break; default: throw std::runtime_error("Unsupported binary operator for JIT benchmark"); } @@ -159,7 +156,6 @@ std::string udf_id; auto result = use_lto ? cudf::transform_lto(udf, - udf_id, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, @@ -184,7 +180,6 @@ std::string udf_id; state.exec(nvbench::exec_tag::sync, [&](nvbench::launch&) { [[maybe_unused]] auto result = use_lto ? cudf::transform_lto(udf, - udf_id, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index ce06208febbe..3b7c465260a8 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -209,8 +209,6 @@ enum class lto_binary_type : uint8_t { * * @param udf The LTO-IR string of the transform function to apply. The UDF must be named * `transform` and follow the CUDF UDF ABI. - * @param udf_id An optional identifier of the UDF to use for caching. If not provided, the UDF will - * be hashed and cached based on its binary contents. * @param binary_type The type of the LTO binary provided in `udf` * @param is_null_aware Signifies the UDF will receive row inputs as optional values * @param user_data User-defined device data to pass to the UDF. @@ -228,7 +226,6 @@ enum class lto_binary_type : uint8_t { */ std::unique_ptr
transform_lto( std::span udf, - std::optional udf_id, lto_binary_type binary_type, null_aware is_null_aware, std::optional user_data, diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index dc8e9fe53a07..9b954376fe4e 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -443,7 +443,6 @@ void run_lto(std::optional, lto_binary_type, std::span inputs, std::span outputs, std::span udf_binary, - std::optional udf_id, lto_binary_type source_type, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) @@ -473,9 +472,7 @@ void run_lto(std::optional, lto_binary_type, { .data = udf_binary, .type = as_rtcx_binary_type(source_type), - .name = udf_id.has_value() - ? udf_id->c_str() - : nullptr // nullptr = unnamed fragment: the binary will be used to hash the UDF + .name = nullptr // nullptr = unnamed fragment: the binary will be used to hash the UDF }}; auto kernel = get_lto_linked_kernel("cudf/cpp/src/transform/jit/kernel.cu", {}, memory_fragments); @@ -1122,7 +1119,6 @@ dispatch_lto_kernel_fragment(bool is_null_aware, } std::unique_ptr
transform_lto(std::span udf, - std::optional udf_id, lto_binary_type binary_type, null_aware is_null_aware, std::optional user_data, @@ -1160,7 +1156,6 @@ std::unique_ptr
transform_lto(std::span udf, inputs, output_columns, udf, - udf_id, binary_type, stream, mr); diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp index bc23dc4057d4..2d6a2cbd326f 100644 --- a/cpp/tests/transform/transform_lto_test.cpp +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -35,7 +35,6 @@ TEST_F(TransformLTOTest, InvSqrt) std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; auto result = cudf::transform_lto(udf, - std::nullopt, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, @@ -62,7 +61,6 @@ TEST_F(TransformLTOTest, ToUpper) std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; auto result = cudf::transform_lto(udf, - std::nullopt, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, @@ -90,7 +88,6 @@ TEST_F(TransformLTOTest, SumOfSquares) std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; auto result = cudf::transform_lto(udf, - std::nullopt, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, @@ -122,7 +119,6 @@ TEST_F(TransformLTOTest, BankersRounding) cudf::transform_output outputs[] = {output}; auto result = cudf::transform_lto(udf, - std::nullopt, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, std::nullopt, From 8208ecb9bb59f5872676bd63fa2480f2348c5f16 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Thu, 18 Jun 2026 11:40:57 +0000 Subject: [PATCH 230/254] Add benchmark for float multiplication in binary operations --- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 625560d3e15e..77116b54f874 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -74,6 +74,7 @@ BINARYOP_BENCHMARK_DEFINE(timestamp_s, duration_s, ADD, time BINARYOP_BENCHMARK_DEFINE(duration_s, duration_D, SUB, duration_ms); BINARYOP_BENCHMARK_DEFINE(int64_t, int64_t, SUB, int64_t); BINARYOP_BENCHMARK_DEFINE(float, float, MUL, int64_t); +BINARYOP_BENCHMARK_DEFINE(float, float, MUL, float); BINARYOP_BENCHMARK_DEFINE(duration_s, int64_t, MUL, duration_s); BINARYOP_BENCHMARK_DEFINE(int64_t, int64_t, DIV, int64_t); BINARYOP_BENCHMARK_DEFINE(duration_ms, int32_t, DIV, duration_ms); From c9e0a1fc0c48bf7b995f92fc29a609cfa7466796 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 22 Jun 2026 15:54:44 +0000 Subject: [PATCH 231/254] Update copyright statements to include "AFFILIATES" for consistency across files --- cpp/benchmarks/CMakeLists.txt | 2 +- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 2 +- cpp/include/cudf/transform.hpp | 2 +- cpp/src/jit/cache.hpp | 2 +- cpp/src/jit/helpers.cpp | 2 +- cpp/src/jit/helpers.hpp | 2 +- cpp/src/transform/jit/kernel.cu | 2 +- cpp/src/transform/transform.cu | 2 +- cpp/tests/CMakeLists.txt | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index f215f7b3813f..dc1f3e0a0c2c 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 77116b54f874..ece28a515243 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 3b7c465260a8..3052b98299e1 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/cache.hpp b/cpp/src/jit/cache.hpp index 95327aeb8dfe..16696f3d0a71 100644 --- a/cpp/src/jit/cache.hpp +++ b/cpp/src/jit/cache.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/helpers.cpp b/cpp/src/jit/helpers.cpp index a0ad474f73fb..1a2f0fc59cdd 100644 --- a/cpp/src/jit/helpers.cpp +++ b/cpp/src/jit/helpers.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/jit/helpers.hpp b/cpp/src/jit/helpers.hpp index 42827b011177..33f746805e6e 100644 --- a/cpp/src/jit/helpers.hpp +++ b/cpp/src/jit/helpers.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 333c0c22ddd1..de08c496f5bb 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 9b954376fe4e..15304459164b 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index aa38e5cad194..2dcc054792bb 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= From 7cc1bc589bafd09df227efb2901c7849c84025e3 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 22 Jun 2026 15:57:55 +0000 Subject: [PATCH 232/254] Refactor hash128_hasher to simplify operator() implementation --- cpp/librtcx/rtcx.hpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 8ed1c625f182..7b350e00bf43 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -107,14 +107,8 @@ func(R (*)(Args...)) -> func; struct [[nodiscard]] hash128_hasher { constexpr std::uint64_t operator()(hash128 const& obj) const { - struct u64x2 { - alignas(alignof(hash128)) std::uint64_t // NOLINT(modernize-avoid-c-arrays) - v[sizeof(hash128) / sizeof(std::uint64_t)]; - }; - - auto value = std::bit_cast(obj); - auto h0 = value.v[0]; - auto h1 = value.v[1]; + auto h1 = (obj.value >> 64); + auto h0 = static_cast(obj.value); auto mix = [](std::uint64_t seed, std::uint64_t v) { seed ^= v + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); From dbbe3847a830cc09ef094e3891ba8769893f5a53 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 22 Jun 2026 16:03:26 +0000 Subject: [PATCH 233/254] Format code and improve readability in binary operation benchmarks --- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 101 +++++++++--------- 1 file changed, 49 insertions(+), 52 deletions(-) diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index ece28a515243..bc359f825730 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -104,7 +104,7 @@ BINARYOP_BENCHMARK_DEFINE(duration_ms, duration_ns, NULL_EQUALS, bool BINARYOP_BENCHMARK_DEFINE(duration_ms, duration_ns, NULL_NOT_EQUALS, bool); BINARYOP_BENCHMARK_DEFINE(decimal32, decimal32, NULL_MAX, decimal32); BINARYOP_BENCHMARK_DEFINE(timestamp_D, timestamp_s, NULL_MIN, timestamp_s); - +// clang-format on template void BM_jit_binaryop(nvbench::state& state, cudf::binary_operator binop) @@ -115,7 +115,7 @@ __device__ void transform(float* out, float a, float b) { } )***"; -constexpr auto const jit_add_cuda = R"***( + constexpr auto const jit_add_cuda = R"***( __device__ void transform(float* out, float a, float b) { *out = a + b; } @@ -133,46 +133,44 @@ __device__ void transform(float* out, float a, float b) { auto rhs = cudf::column_view(source_table->get_column(1)); size_t fragment_id = 0; - char const * cuda = nullptr; + char const* cuda = nullptr; switch (binop) { case cudf::binary_operator::ADD: { fragment_id = cudf_benchmark_fragments::add_f32; - cuda = jit_add_cuda; + cuda = jit_add_cuda; } break; case cudf::binary_operator::MUL: { fragment_id = cudf_benchmark_fragments::mul_f32; - cuda = jit_mul_cuda; + cuda = jit_mul_cuda; } break; default: throw std::runtime_error("Unsupported binary operator for JIT benchmark"); } // Call once for hot cache. cudf::transform_input inputs[] = {lhs, rhs}; - cudf::transform_output outputs[] = {{cudf::data_type{cudf::type_to_id()}, - cudf::output_nullability::ALL_VALID}}; + cudf::transform_output outputs[] = { + {cudf::data_type{cudf::type_to_id()}, cudf::output_nullability::ALL_VALID}}; auto const range = cudf_benchmark_fragments::file_ranges[fragment_id]; std::span udf{cudf_benchmark_fragments::files.subspan(range[0], range[1])}; - auto result = use_lto ? - cudf::transform_lto(udf, - cudf::lto_binary_type::FATBIN, - cudf::null_aware::NO, - std::nullopt, - inputs, - outputs, - {}, - std::nullopt): - cudf::multi_transform(cuda, - cudf::udf_source_type::CUDA, - cudf::null_aware::NO, - std::nullopt, - inputs, - outputs, - {}, - std::nullopt); - + auto result = use_lto ? cudf::transform_lto(udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt) + : cudf::multi_transform(cuda, + cudf::udf_source_type::CUDA, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt); // use number of bytes read and written to global memory state.add_global_memory_reads(num_rows); @@ -180,42 +178,41 @@ __device__ void transform(float* out, float a, float b) { state.add_global_memory_writes(num_rows); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch&) { - [[maybe_unused]] auto result = use_lto ? cudf::transform_lto(udf, - cudf::lto_binary_type::FATBIN, - cudf::null_aware::NO, - std::nullopt, - inputs, - outputs, - {}, - std::nullopt): cudf::multi_transform(cuda, - cudf::udf_source_type::CUDA, - cudf::null_aware::NO, - std::nullopt, - inputs, - outputs, - {}, - std::nullopt); + [[maybe_unused]] auto result = use_lto ? cudf::transform_lto(udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt) + : cudf::multi_transform(cuda, + cudf::udf_source_type::CUDA, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt); }); } - -#define BM_JIT_BINARYOP_BENCHMARK_DEFINE(name, lhs, rhs, bop, tout) \ - static void name(::nvbench::state& st) \ - { \ - ::BM_jit_binaryop(st, ::cudf::binary_operator::bop); \ - } \ - NVBENCH_BENCH(name) \ - .set_name("jit_binary_op_" BM_STRINGIFY(name)) \ - .add_int64_axis("num_rows", {10'000, 100'000, 1'000'000, 10'000'000, 100'000'000})\ +#define BM_JIT_BINARYOP_BENCHMARK_DEFINE(name, lhs, rhs, bop, tout) \ + static void name(::nvbench::state& st) \ + { \ + ::BM_jit_binaryop(st, ::cudf::binary_operator::bop); \ + } \ + NVBENCH_BENCH(name) \ + .set_name("jit_binary_op_" BM_STRINGIFY(name)) \ + .add_int64_axis("num_rows", {10'000, 100'000, 1'000'000, 10'000'000, 100'000'000}) \ .add_string_axis("use_lto", {"true", "false"}) - #define build_name_jit(a, b, c, d) a##_##b##_##c##_##d##_jit - #define JIT_BINARYOP_BENCHMARK_DEFINE(lhs, rhs, bop, tout) \ BM_JIT_BINARYOP_BENCHMARK_DEFINE(build_name_jit(bop, lhs, rhs, tout), lhs, rhs, bop, tout) - +// clang-format off JIT_BINARYOP_BENCHMARK_DEFINE(float, float, ADD, float); JIT_BINARYOP_BENCHMARK_DEFINE(float, float, MUL, float); +// clang-format on From 700b83dd9ecb3746abc9610a99a1007f6f551a19 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 23 Jun 2026 22:15:02 +0000 Subject: [PATCH 234/254] fix(transform_kernel): remove redundant stencil check and clean up lambda syntax --- cpp/src/transform/jit/kernel.cu | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index f97438fb57dd..56b6c8de4825 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -78,14 +78,10 @@ __device__ void transform_kernel(size_type row_size, if constexpr (!is_null_aware) { if (stencil != nullptr && !bit_is_set(stencil, row)) { continue; } - if (stencil != nullptr && !bit_is_set(stencil, row)) { continue; } auto ins = InputAccessors::map( [&]() { return cuda::std::tuple{A::element(input_cols, row)...}; }); - [&]() { return cuda::std::tuple{A::element(input_cols, row)...}; }); - auto outs = OutputAccessors::map( - [&]() { return cuda::std::tuple{A::output_arg(output_cols, row)...}; }); auto outs = OutputAccessors::map( [&]() { return cuda::std::tuple{A::output_arg(output_cols, row)...}; }); From 06d1194d6d0134406d80e55ccb2c84abff33beff Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 23 Jun 2026 23:56:42 +0000 Subject: [PATCH 235/254] fix(CMakeLists): update target_link_libraries to use BUILD_LOCAL_INTERFACE for xxhash --- cpp/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 09cde3cd8968..4f3d9ec4357a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1353,7 +1353,8 @@ target_link_libraries( cudf PUBLIC CCCL::CCCL $ PRIVATE $ $ ZLIB::ZLIB - ${CUDF_nvcomp_TARGET} kvikio::kvikio ${CUDF_nanoarrow_TARGET} zstd xxhash rtcx::rtcx + ${CUDF_nvcomp_TARGET} kvikio::kvikio ${CUDF_nanoarrow_TARGET} zstd + $ rtcx::rtcx ) # When rmm is a static library being absorbed via whole-archive, strip nvtx3 from its public From f3b9908e2f1546dcda1270d70c7dc2845010fcea Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 24 Jun 2026 00:05:11 +0000 Subject: [PATCH 236/254] fix(CMakeLists): format target_link_libraries for better readability --- cpp/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4f3d9ec4357a..96f3e7ff9bad 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1352,9 +1352,15 @@ target_compile_definitions(cudf PRIVATE THRUST_FORCE_32_BIT_OFFSET_TYPE=1 CCCL_A target_link_libraries( cudf PUBLIC CCCL::CCCL $ - PRIVATE $ $ ZLIB::ZLIB - ${CUDF_nvcomp_TARGET} kvikio::kvikio ${CUDF_nanoarrow_TARGET} zstd - $ rtcx::rtcx + PRIVATE $ + $ + ZLIB::ZLIB + ${CUDF_nvcomp_TARGET} + kvikio::kvikio + ${CUDF_nanoarrow_TARGET} + zstd + $ + rtcx::rtcx ) # When rmm is a static library being absorbed via whole-archive, strip nvtx3 from its public From e19544b52b706659ff96a3b1baff45f2382fac30 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 29 Jun 2026 15:46:33 +0000 Subject: [PATCH 237/254] fixed cuda version check for compile options --- cpp/src/jit/cache.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 21a8b2c77f9d..0585c906c691 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -193,9 +193,15 @@ std::vector jit_bundle_t::get_include_directories() const namespace { -constexpr int MIN_CUDA_VERSION_PCH = 12800; // minimum CUDA version for the "--pch" NVRTC flag +constexpr int make_cuda_version(int major, int minor, int patch) +{ + return major * 1000 + minor * 10 + patch; +} + +constexpr int MIN_CUDA_VERSION_PCH = + make_cuda_version(12, 8, 0); // minimum CUDA version for the "--pch" NVRTC flag constexpr int MIN_CUDA_VERSION_MINIMAL = - 12800; // minimum CUDA version for the "--minimal" NVRTC flag + make_cuda_version(12, 8, 0); // minimum CUDA version for the "--minimal" NVRTC flag int32_t get_driver_version() { From 943cde3bafa440e025a37e018567751c6e96c46a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 29 Jun 2026 19:08:55 +0100 Subject: [PATCH 238/254] Update cpp/cmake/thirdparty/get_xxhash.cmake Co-authored-by: Bradley Dice --- cpp/cmake/thirdparty/get_xxhash.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_xxhash.cmake b/cpp/cmake/thirdparty/get_xxhash.cmake index 558b1ee5dda3..046d95b23f2a 100644 --- a/cpp/cmake/thirdparty/get_xxhash.cmake +++ b/cpp/cmake/thirdparty/get_xxhash.cmake @@ -17,7 +17,7 @@ function(find_and_configure_xxhash) GLOBAL_TARGETS xxhash CPM_ARGS GIT_REPOSITORY https://github.com/Cyan4973/xxHash.git - GIT_TAG v0.8.3 + GIT_TAG e626a72bc2321cd320e953a0ccf1584cad60f363 GIT_SHALLOW TRUE DOWNLOAD_ONLY TRUE EXCLUDE_FROM_ALL ${CUDF_EXCLUDE_DEPS_FROM_ALL} From e37513988ac0fe21e0225866c19aedb53940a79c Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 29 Jun 2026 19:09:13 +0100 Subject: [PATCH 239/254] Update cpp/librtcx/README.md Co-authored-by: Bradley Dice --- cpp/librtcx/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/librtcx/README.md b/cpp/librtcx/README.md index 8704c12167c2..b8f9898c0d13 100644 --- a/cpp/librtcx/README.md +++ b/cpp/librtcx/README.md @@ -12,7 +12,7 @@ RTCX (runtime-compiler extended) is a wrapper around NVRTC and nvJitLink designe ## Build-time Requirements - CMake >= 4.0 -- LibZSTD +- libzstd - xxHash - CUDA >= 12.2 From 78fe49fcbe08d7a8213a9ba682bfb3323c85ce9f Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 29 Jun 2026 19:11:33 +0100 Subject: [PATCH 240/254] Update cpp/librtcx/embed.cmake Co-authored-by: Bradley Dice --- cpp/librtcx/embed.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index 7b1682b546bf..56869f4f6e01 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -295,7 +295,7 @@ function(embed TARGET) set(RUNNER "${TARGET}__jit_embed_run") add_executable(${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") - target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} zstd xxhash) + target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} xxhash zstd) target_include_directories( ${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR} ${ZSTD_INCLUDE_DIR} ) From 9ce7ecf131f7d86635c4669ca66be2c3b453eb8a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 29 Jun 2026 19:44:02 +0000 Subject: [PATCH 241/254] code review changes --- cpp/CMakeLists.txt | 27 +++++---- cpp/benchmarks/CMakeLists.txt | 4 +- cpp/librtcx/CMakeLists.txt | 4 +- cpp/librtcx/embed.cmake | 14 ++--- cpp/librtcx/hash.cpp | 70 +++++++++++++++++++++++ cpp/librtcx/hash.hpp | 64 ++++----------------- cpp/librtcx/rtcx.hpp | 5 ++ cpp/src/jit/cache.cpp | 1 - cpp/tests/CMakeLists.txt | 4 +- cpp/tests/transform/fragments/to_upper.cu | 2 - 10 files changed, 111 insertions(+), 84 deletions(-) create mode 100644 cpp/librtcx/hash.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7cb1da53fed1..55c755c5ba16 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -447,39 +447,39 @@ if(NOT BUILD_SHARED_LIBS) ) endif() -add_embed(cudf_cuda_embed) +rtcx_add_embed(cudf_cuda_embed) -embed_includes( +rtcx_embed_includes( cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/librtcx/libcxx DEST_DIRECTORY librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx ) -embed_includes( +rtcx_embed_includes( cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/cudf DEST_DIRECTORY cudf/cpp/include/cudf INCLUDE_DIRECTORIES cudf/cpp/include ) -embed_includes( +rtcx_embed_includes( cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/jit DEST_DIRECTORY cudf/cpp/src/jit INCLUDE_DIRECTORIES cudf/cpp/src ) -embed_includes( +rtcx_embed_includes( cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/binaryop/jit DEST_DIRECTORY cudf/cpp/src/binaryop/jit INCLUDE_DIRECTORIES cudf/cpp/src ) -embed_includes( +rtcx_embed_includes( cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/join/jit DEST_DIRECTORY cudf/cpp/src/join/jit INCLUDE_DIRECTORIES cudf/cpp/src ) -embed_includes( +rtcx_embed_includes( cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/rolling DEST_DIRECTORY cudf/cpp/src/rolling INCLUDE_DIRECTORIES cudf/cpp/src ) -embed_includes( +rtcx_embed_includes( cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/transform/jit DEST_DIRECTORY cudf/cpp/src/transform/jit INCLUDE_DIRECTORIES cudf/cpp/src ) @@ -489,13 +489,13 @@ get_target_property(LIBCUDACXX_RAW_INCLUDE_DIRS CCCL::libcudacxx INTERFACE_INCLU foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) cmake_path(GET INC_DIR FILENAME INC_DIR_NAME) - embed_includes( + rtcx_embed_includes( cudf_cuda_embed SOURCE_DIRECTORY ${INC_DIR} DEST_DIRECTORY CCCL/libcudacxx/${INC_DIR_NAME} INCLUDE_DIRECTORIES CCCL/libcudacxx/${INC_DIR_NAME} ) endforeach() -embed(cudf_cuda_embed COMPRESSION zstd OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") +rtcx_embed(cudf_cuda_embed COMPRESSION zstd OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") # This macro is used to create object libraries for JIT compilation fragments, and embed them as # fatbins in the final library. It compiles the specified source file with the appropriate flags to @@ -570,13 +570,13 @@ macro(add_fragment) ) target_compile_options(${OBJECT_ID} PRIVATE "$<$:${CUDF_CUDA_FLAGS}>") - embed_blob( + rtcx_embed_blob( ${TARGET} FILE $ DEST fragments/${ARG_FRAGMENT}.fatbin ID ${ARG_FRAGMENT} ARRAY_IDS ${ARG_ARRAY_IDS} ARRAY_VALUES ${ARG_ARRAY_VALUES} ) endmacro() -add_embed(cudf_fragments) +rtcx_add_embed(cudf_fragments) list( APPEND @@ -672,7 +672,7 @@ foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) endforeach() endforeach() -embed(cudf_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") +rtcx_embed(cudf_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") # ################################################################################################## # * library targets ------------------------------------------------------------------------------- @@ -1235,7 +1235,6 @@ add_library( src/utilities/traits.cpp src/utilities/type_checks.cpp src/utilities/type_dispatcher.cpp - librtcx/rtcx.cpp ${cudf_cuda_embed_SOURCE_DIR}/cudf_cuda_embed.s ${cudf_fragments_SOURCE_DIR}/cudf_fragments.s ) diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index ebcfde4b1fb0..7cb440c1829b 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -388,10 +388,10 @@ ConfigureNVBench(AST_NVBENCH ast/polynomials.cpp ast/transform.cpp) # ################################################################################################## # * LTO Fragments ---------------------------------------------------------------------------- -add_embed(cudf_benchmark_fragments) +rtcx_add_embed(cudf_benchmark_fragments) add_fragment(cudf_benchmark_fragments FRAGMENT add_f32 SOURCE binaryop/fragments/add_f32.cu) add_fragment(cudf_benchmark_fragments FRAGMENT mul_f32 SOURCE binaryop/fragments/mul_f32.cu) -embed(cudf_benchmark_fragments COMPRESSION none OUTPUT_DIRECTORY +rtcx_embed(cudf_benchmark_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" ) diff --git a/cpp/librtcx/CMakeLists.txt b/cpp/librtcx/CMakeLists.txt index 47662051f7f1..a40bb36d2914 100644 --- a/cpp/librtcx/CMakeLists.txt +++ b/cpp/librtcx/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -60,7 +60,7 @@ if(NOT TARGET nvtx3::nvtx3-cpp) rapids_cpm_nvtx3() endif() -add_library(rtcx STATIC rtcx.cpp) +add_library(rtcx STATIC hash.cpp rtcx.cpp) add_library(rtcx::rtcx ALIAS rtcx) set_target_properties( diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index 7b1682b546bf..26cb54397ae2 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -17,7 +17,7 @@ endif() # embed_includes() or embed_blob() for the target. It creates a dedicated INTERFACE library target # that is used to track registered files and dependencies via target properties. The TARGET argument # specifies the name of the target being initialized. -function(add_embed TARGET) +function(rtcx_add_embed TARGET) set(OPTIONS "") set(ONE_VALUE_ARGS) set(MULTI_VALUE_ARGS) @@ -32,7 +32,7 @@ function(add_embed TARGET) endfunction() # This function registers a directory of include files to be embedded for JIT compilation. -function(embed_includes TARGET) +function(rtcx_embed_includes TARGET) set(OPTIONS "") set(ONE_VALUE_ARGS SOURCE_DIRECTORY # Source directory where files will be copied from DEST_DIRECTORY # Destination directory where files will be copied to @@ -123,14 +123,14 @@ function(embed_includes TARGET) endfunction() # This function registers a single file to be embedded for JIT compilation. -function(embed_blob TARGET) +function(rtcx_embed_blob TARGET) set(OPTIONS) set(ONE_VALUE_ARGS ID FILE DEST) set(MULTI_VALUE_ARGS ARRAY_IDS ARRAY_VALUES) cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) if(NOT TARGET ${TARGET}__embed_props) - message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") + message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with rtcx_add_embed()") endif() if(NOT ARG_ID @@ -209,14 +209,14 @@ endfunction() # for JIT compilation. #]==] # cmake-lint: disable=R0915 -function(embed TARGET) +function(rtcx_embed TARGET) set(OPTIONS "") set(ONE_VALUE_ARGS "COMPRESSION" "OUTPUT_DIRECTORY") set(MULTI_VALUE_ARGS "") cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) if(NOT TARGET ${TARGET}__embed_props) - message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") + message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with rtcx_add_embed()") endif() if(NOT DEFINED ARG_COMPRESSION) @@ -294,7 +294,7 @@ function(embed TARGET) ) set(RUNNER "${TARGET}__jit_embed_run") - add_executable(${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") + add_executable(${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}" ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/hash.cpp) target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} zstd xxhash) target_include_directories( ${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR} ${ZSTD_INCLUDE_DIR} diff --git a/cpp/librtcx/hash.cpp b/cpp/librtcx/hash.cpp new file mode 100644 index 000000000000..fd32d55b2269 --- /dev/null +++ b/cpp/librtcx/hash.cpp @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "hash.hpp" + +#include +#include +#include +#include +#include + +namespace rtcx { + +char const* hash128_hex_string::data() const { return data_; } + +char const* hash128_hex_string::c_str() const { return data_; } + +hash128_hex_string hash128_hex_string::make(std::span input) +{ + constexpr char const HEX_CHARS[] = "0123456789abcdef"; // NOLINT(modernize-avoid-c-arrays) + hash128_hex_string hex; + for (std::size_t i = 0; i < NUM_HEX_BYTES; ++i) { + hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; + hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; + } + hex.data_[NUM_HEX_DIGITS] = '\0'; + return hex; +} + +hash128_hex_string hash128_hex_string::make(__uint128_t hash) +{ + auto array = std::bit_cast>(hash); + return make(array); +} + +std::uint8_t hash128::operator[](std::size_t index) const +{ + return reinterpret_cast(&value)[15 - index]; +} + +std::size_t hash128::size() const { return 16; } + +std::uint8_t const* hash128::data() const { return reinterpret_cast(&value); } + +hash128_hex_string hash128::to_hex_string() const { return hash128_hex_string::make(value); } + +hash128 hash128::parse(std::string_view hex) +{ + if (hex.size() != hash128_hex_string::NUM_HEX_DIGITS) { + throw std::invalid_argument( + std::format("Invalid hash128 hex string length, expected {} got {} (hash: `{}`)", + hash128_hex_string::NUM_HEX_DIGITS, + hex.size(), + hex)); + } + std::array data{}; + for (std::size_t i = 0; i < hash128_hex_string::NUM_HEX_BYTES; ++i) { + auto hex_byte = hex.substr(i * 2, 2); + auto [ptr, ec] = std::from_chars(hex_byte.begin(), hex_byte.end(), data[i], 16); + if (ec != std::errc()) { + throw std::invalid_argument( + std::format("Invalid hex character {} in HEX string: `{}`", hex_byte, hex)); + } + } + return hash128{std::bit_cast<__uint128_t>(data)}; +} + +} // namespace rtcx diff --git a/cpp/librtcx/hash.hpp b/cpp/librtcx/hash.hpp index ce573c24d58b..d1aa12c85c61 100644 --- a/cpp/librtcx/hash.hpp +++ b/cpp/librtcx/hash.hpp @@ -5,14 +5,9 @@ #pragma once -#include -#include -#include +#include #include -#include -#include #include -#include #include namespace rtcx { @@ -30,29 +25,15 @@ struct [[nodiscard]] hash128_hex_string { [[nodiscard]] constexpr operator std::string_view() const { return view(); } - [[nodiscard]] char const* data() const { return data_; } + [[nodiscard]] char const* data() const; - [[nodiscard]] char const* c_str() const { return data_; } + [[nodiscard]] char const* c_str() const; [[nodiscard]] static constexpr std::size_t size() { return NUM_HEX_DIGITS; } - static hash128_hex_string make(std::span input) - { - constexpr char const HEX_CHARS[] = "0123456789abcdef"; // NOLINT(modernize-avoid-c-arrays) - hash128_hex_string hex; - for (std::size_t i = 0; i < NUM_HEX_BYTES; ++i) { - hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; - hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; - } - hex.data_[NUM_HEX_DIGITS] = '\0'; - return hex; - } + static hash128_hex_string make(std::span input); - static hash128_hex_string make(__uint128_t hash) - { - auto array = std::bit_cast>(hash); - return make(array); - } + static hash128_hex_string make(__uint128_t hash); }; struct hash128 { @@ -67,40 +48,15 @@ struct hash128 { [[nodiscard]] constexpr bool operator==(hash128 const&) const = default; - [[nodiscard]] std::uint8_t operator[](std::size_t index) const - { - return reinterpret_cast(&value)[15 - index]; - } + [[nodiscard]] std::uint8_t operator[](std::size_t index) const; - [[nodiscard]] std::size_t size() const { return 16; } + [[nodiscard]] std::size_t size() const; - [[nodiscard]] std::uint8_t const* data() const - { - return reinterpret_cast(&value); - } + [[nodiscard]] std::uint8_t const* data() const; - hash128_hex_string to_hex_string() const { return hash128_hex_string::make(value); } + hash128_hex_string to_hex_string() const; - static hash128 parse(std::string_view hex) - { - if (hex.size() != hash128_hex_string::NUM_HEX_DIGITS) { - throw std::invalid_argument( - std::format("Invalid hash128 hex string length, expected {} got {} (hash: `{}`)", - hash128_hex_string::NUM_HEX_DIGITS, - hex.size(), - hex)); - } - std::array data{}; - for (std::size_t i = 0; i < hash128_hex_string::NUM_HEX_BYTES; ++i) { - auto hex_byte = hex.substr(i * 2, 2); - auto [ptr, ec] = std::from_chars(hex_byte.begin(), hex_byte.end(), data[i], 16); - if (ec != std::errc()) { - throw std::invalid_argument( - std::format("Invalid hex character {} in HEX string: `{}`", hex_byte, hex)); - } - } - return hash128{std::bit_cast<__uint128_t>(data)}; - } + static hash128 parse(std::string_view hex); }; } // namespace rtcx diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 7b350e00bf43..45b5c593b2a4 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -110,7 +110,12 @@ struct [[nodiscard]] hash128_hasher { auto h1 = (obj.value >> 64); auto h0 = static_cast(obj.value); + // map two 64-bit hash values into a single 64-bit hash value to use their combined entropy. + // std::hash requires a single 64-bit hash value, but we want to use the combined entropy of + // both 64-bit values. auto mix = [](std::uint64_t seed, std::uint64_t v) { + // based on `cccl::hash_combine` + // (https://github.com/NVIDIA/cccl/blob/e1cb6965571312e43c6a519901f5403b0a0df4a6/cudax/include/cuda/experimental/__stf/utility/hash.cuh#L93-L94) seed ^= v + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); return seed; }; diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 0585c906c691..a0cd387b8342 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -1,4 +1,3 @@ - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 2dcc054792bb..d85cf7e5fff6 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -690,7 +690,7 @@ ConfigureTest(AST_TEST ast/transform_tests.cpp ast/ast_tree_tests.cpp) # ################################################################################################## # * transform LTO tests # ------------------------------------------------------------------------------------- -add_embed(cudf_test_fragments) +rtcx_add_embed(cudf_test_fragments) add_fragment(cudf_test_fragments FRAGMENT invsqrt SOURCE transform/fragments/invsqrt.cu) @@ -704,7 +704,7 @@ add_fragment( cudf_test_fragments FRAGMENT bankers_rounding SOURCE transform/fragments/bankers_rounding.cu ) -embed(cudf_test_fragments COMPRESSION none OUTPUT_DIRECTORY +rtcx_embed(cudf_test_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" ) diff --git a/cpp/tests/transform/fragments/to_upper.cu b/cpp/tests/transform/fragments/to_upper.cu index 0ad86b562200..953d4992c823 100644 --- a/cpp/tests/transform/fragments/to_upper.cu +++ b/cpp/tests/transform/fragments/to_upper.cu @@ -1,5 +1,3 @@ - - /* * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 From de90c1017cb6939f6c9cc4d0aa1b2c9c729f5391 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 30 Jun 2026 09:45:29 +0000 Subject: [PATCH 242/254] comment update --- cpp/librtcx/rtcx.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 45b5c593b2a4..d3339df14e6b 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -114,7 +114,7 @@ struct [[nodiscard]] hash128_hasher { // std::hash requires a single 64-bit hash value, but we want to use the combined entropy of // both 64-bit values. auto mix = [](std::uint64_t seed, std::uint64_t v) { - // based on `cccl::hash_combine` + // based on `cuda::experimental::stf::hash_combine` // (https://github.com/NVIDIA/cccl/blob/e1cb6965571312e43c6a519901f5403b0a0df4a6/cudax/include/cuda/experimental/__stf/utility/hash.cuh#L93-L94) seed ^= v + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); return seed; From 693c3bb4c35d120833ef50d22da84fe81d6cb2bb Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 30 Jun 2026 17:46:47 +0000 Subject: [PATCH 243/254] update --- cpp/CMakeLists.txt | 8 ++++++-- cpp/benchmarks/CMakeLists.txt | 5 +++-- cpp/librtcx/embed.cmake | 4 +++- cpp/tests/CMakeLists.txt | 4 ++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 55c755c5ba16..0de4d9d839f5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -495,7 +495,9 @@ foreach(INC_DIR IN LISTS LIBCUDACXX_RAW_INCLUDE_DIRS) ) endforeach() -rtcx_embed(cudf_cuda_embed COMPRESSION zstd OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") +rtcx_embed( + cudf_cuda_embed COMPRESSION zstd OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" +) # This macro is used to create object libraries for JIT compilation fragments, and embed them as # fatbins in the final library. It compiles the specified source file with the appropriate flags to @@ -672,7 +674,9 @@ foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) endforeach() endforeach() -rtcx_embed(cudf_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed") +rtcx_embed( + cudf_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" +) # ################################################################################################## # * library targets ------------------------------------------------------------------------------- diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 7cb440c1829b..cfef95bc914c 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -391,8 +391,9 @@ ConfigureNVBench(AST_NVBENCH ast/polynomials.cpp ast/transform.cpp) rtcx_add_embed(cudf_benchmark_fragments) add_fragment(cudf_benchmark_fragments FRAGMENT add_f32 SOURCE binaryop/fragments/add_f32.cu) add_fragment(cudf_benchmark_fragments FRAGMENT mul_f32 SOURCE binaryop/fragments/mul_f32.cu) -rtcx_embed(cudf_benchmark_fragments COMPRESSION none OUTPUT_DIRECTORY - "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" +rtcx_embed( + cudf_benchmark_fragments COMPRESSION none OUTPUT_DIRECTORY + "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" ) # ################################################################################################## diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index b0e8a4fda408..dd1e83190835 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -294,7 +294,9 @@ function(rtcx_embed TARGET) ) set(RUNNER "${TARGET}__jit_embed_run") - add_executable(${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}" ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/hash.cpp) + add_executable( + ${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}" ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/hash.cpp + ) target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} xxhash zstd) target_include_directories( ${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR} ${ZSTD_INCLUDE_DIR} diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index d85cf7e5fff6..c6d2ef388543 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -704,8 +704,8 @@ add_fragment( cudf_test_fragments FRAGMENT bankers_rounding SOURCE transform/fragments/bankers_rounding.cu ) -rtcx_embed(cudf_test_fragments COMPRESSION none OUTPUT_DIRECTORY - "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" +rtcx_embed( + cudf_test_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" ) ConfigureTest( From 641732d431f7db59432637fa9783bda3e8945ec7 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 30 Jun 2026 17:53:04 +0000 Subject: [PATCH 244/254] only use lower 64 bits of hash128 for memory hash table --- cpp/librtcx/rtcx.hpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index d3339df14e6b..256f52fbe26d 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -107,20 +107,8 @@ func(R (*)(Args...)) -> func; struct [[nodiscard]] hash128_hasher { constexpr std::uint64_t operator()(hash128 const& obj) const { - auto h1 = (obj.value >> 64); - auto h0 = static_cast(obj.value); - - // map two 64-bit hash values into a single 64-bit hash value to use their combined entropy. - // std::hash requires a single 64-bit hash value, but we want to use the combined entropy of - // both 64-bit values. - auto mix = [](std::uint64_t seed, std::uint64_t v) { - // based on `cuda::experimental::stf::hash_combine` - // (https://github.com/NVIDIA/cccl/blob/e1cb6965571312e43c6a519901f5403b0a0df4a6/cudax/include/cuda/experimental/__stf/utility/hash.cuh#L93-L94) - seed ^= v + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); - return seed; - }; - - return mix(h0, h1); + // use only the lower 64 bits of the hash for the hash table + return static_cast(obj.value); } }; From 00fe96cacbcde65688677e38ed6b3edb2dc4a093 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Jul 2026 19:10:34 +0000 Subject: [PATCH 245/254] add fragment management helper functions to CMake --- cpp/CMakeLists.txt | 82 +-------------------------- cpp/cmake/Modules/AddFragment.cmake | 87 +++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 79 deletions(-) create mode 100644 cpp/cmake/Modules/AddFragment.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0de4d9d839f5..cc55f4efef03 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -425,6 +425,9 @@ add_subdirectory(librtcx) # JIT Embedding helper functions include(librtcx/embed.cmake) +# Pre-compiled fragment management helper functions +include(cmake/Modules/AddFragment.cmake) + # Workaround until https://github.com/rapidsai/rapids-cmake/issues/176 is resolved if(NOT BUILD_SHARED_LIBS) include("${rapids-cmake-dir}/export/find_package_file.cmake") @@ -499,85 +502,6 @@ rtcx_embed( cudf_cuda_embed COMPRESSION zstd OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" ) -# This macro is used to create object libraries for JIT compilation fragments, and embed them as -# fatbins in the final library. It compiles the specified source file with the appropriate flags to -# generate a fatbin containing the specified kernel instance, and then embeds that fatbin in the -# final library with metadata that allows it to be looked up at runtime. -macro(add_fragment) - set(TARGET ${ARGV0}) - set(ONE_VALUE_ARGS FRAGMENT SOURCE KERNEL_ONLY KERNEL_INSTANCE UDF_TYPE) - set(MULTI_VALUE_ARGS DEFINITIONS ARRAY_IDS ARRAY_VALUES) - cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - - if(NOT ARG_FRAGMENT) - message(FATAL_ERROR "add_fragment requires FRAGMENT argument") - endif() - - if(NOT ARG_SOURCE) - message(FATAL_ERROR "add_fragment requires SOURCE argument") - endif() - - set(OBJECT_ID ${TARGET}_${ARG_FRAGMENT}) - add_library(${OBJECT_ID} OBJECT ${ARG_SOURCE}) - target_compile_options(${OBJECT_ID} PRIVATE --compress-mode=size) - - if(DEFINED ARG_KERNEL_ONLY AND ARG_KERNEL_ONLY) - # ensure that the FATBIN symbols only contain the specified kernel - target_compile_options(${OBJECT_ID} PRIVATE -Xnvlink=--kernels-used=cudf_kernel_entry) - endif() - - set(INSTANTIATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/${TARGET}/instantiations/${ARG_FRAGMENT}") - target_include_directories(${OBJECT_ID} PRIVATE ${INSTANTIATION_DIR}) - - if(ARG_KERNEL_INSTANCE) - file( - GENERATE - OUTPUT "${INSTANTIATION_DIR}/cudf/detail/kernel_instance.cuh" - CONTENT "#pragma once\n#define CUDF_KERNEL_INSTANCE ${ARG_KERNEL_INSTANCE}" - ) - endif() - - if(ARG_UDF_TYPE) - file( - GENERATE - OUTPUT "${INSTANTIATION_DIR}/cudf/detail/operation_udf.cuh" - CONTENT "#pragma once\n#define CUDF_UDF_TYPE ${ARG_UDF_TYPE}" - ) - endif() - - target_compile_definitions(${OBJECT_ID} PRIVATE CUDF_DISABLE_EXPORTS ${ARG_DEFINITIONS}) - set_target_properties( - ${OBJECT_ID} - PROPERTIES CUDA_SEPARABLE_COMPILATION ON - CUDA_FATBIN_COMPILATION ON - POSITION_INDEPENDENT_CODE ON - INTERPROCEDURAL_OPTIMIZATION ON - CXX_STANDARD 20 - CXX_STANDARD_REQUIRED ON - CXX_EXTENSIONS ON - CXX_VISIBILITY_PRESET hidden - CUDA_STANDARD 20 - CUDA_STANDARD_REQUIRED ON - CUDA_VISIBILITY_PRESET hidden - ) - target_link_libraries( - ${OBJECT_ID} - PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ - PRIVATE $ $ - ZLIB::ZLIB nvcomp::nvcomp kvikio::kvikio nanoarrow::nanoarrow zstd - ) - target_include_directories( - ${OBJECT_ID} PRIVATE "$" - "$" - ) - target_compile_options(${OBJECT_ID} PRIVATE "$<$:${CUDF_CUDA_FLAGS}>") - - rtcx_embed_blob( - ${TARGET} FILE $ DEST fragments/${ARG_FRAGMENT}.fatbin ID - ${ARG_FRAGMENT} ARRAY_IDS ${ARG_ARRAY_IDS} ARRAY_VALUES ${ARG_ARRAY_VALUES} - ) -endmacro() - rtcx_add_embed(cudf_fragments) list( diff --git a/cpp/cmake/Modules/AddFragment.cmake b/cpp/cmake/Modules/AddFragment.cmake new file mode 100644 index 000000000000..4c9a7bd66b7a --- /dev/null +++ b/cpp/cmake/Modules/AddFragment.cmake @@ -0,0 +1,87 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +include_guard(GLOBAL) + +# This macro is used to create object libraries for JIT compilation fragments, and embed them as +# fatbins in the final library. It compiles the specified source file with the appropriate flags to +# generate a fatbin containing the specified kernel instance, and then embeds that fatbin in the +# final library with metadata that allows it to be looked up at runtime. +macro(add_fragment) + set(TARGET ${ARGV0}) + set(ONE_VALUE_ARGS FRAGMENT SOURCE KERNEL_ONLY KERNEL_INSTANCE UDF_TYPE) + set(MULTI_VALUE_ARGS DEFINITIONS ARRAY_IDS ARRAY_VALUES) + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT ARG_FRAGMENT) + message(FATAL_ERROR "add_fragment requires FRAGMENT argument") + endif() + + if(NOT ARG_SOURCE) + message(FATAL_ERROR "add_fragment requires SOURCE argument") + endif() + + set(OBJECT_ID ${TARGET}_${ARG_FRAGMENT}) + add_library(${OBJECT_ID} OBJECT ${ARG_SOURCE}) + target_compile_options(${OBJECT_ID} PRIVATE --compress-mode=size) + + if(DEFINED ARG_KERNEL_ONLY AND ARG_KERNEL_ONLY) + # ensure that the FATBIN symbols only contain the specified kernel + target_compile_options(${OBJECT_ID} PRIVATE -Xnvlink=--kernels-used=cudf_kernel_entry) + endif() + + set(INSTANTIATION_DIR "${CUDF_GENERATED_INCLUDE_DIR}/${TARGET}/instantiations/${ARG_FRAGMENT}") + target_include_directories(${OBJECT_ID} PRIVATE ${INSTANTIATION_DIR}) + + if(ARG_KERNEL_INSTANCE) + file( + GENERATE + OUTPUT "${INSTANTIATION_DIR}/cudf/detail/kernel_instance.cuh" + CONTENT "#pragma once\n#define CUDF_KERNEL_INSTANCE ${ARG_KERNEL_INSTANCE}" + ) + endif() + + if(ARG_UDF_TYPE) + file( + GENERATE + OUTPUT "${INSTANTIATION_DIR}/cudf/detail/operation_udf.cuh" + CONTENT "#pragma once\n#define CUDF_UDF_TYPE ${ARG_UDF_TYPE}" + ) + endif() + + target_compile_definitions(${OBJECT_ID} PRIVATE CUDF_DISABLE_EXPORTS ${ARG_DEFINITIONS}) + set_target_properties( + ${OBJECT_ID} + PROPERTIES CUDA_SEPARABLE_COMPILATION ON + CUDA_FATBIN_COMPILATION ON + POSITION_INDEPENDENT_CODE ON + INTERPROCEDURAL_OPTIMIZATION ON + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS ON + CXX_VISIBILITY_PRESET hidden + CUDA_STANDARD 20 + CUDA_STANDARD_REQUIRED ON + CUDA_VISIBILITY_PRESET hidden + ) + target_link_libraries( + ${OBJECT_ID} + PUBLIC CCCL::CCCL rapids_logger::rapids_logger rmm::rmm $ + PRIVATE $ $ + ZLIB::ZLIB nvcomp::nvcomp kvikio::kvikio nanoarrow::nanoarrow zstd + ) + target_include_directories( + ${OBJECT_ID} PRIVATE "$" + "$" + ) + target_compile_options(${OBJECT_ID} PRIVATE "$<$:${CUDF_CUDA_FLAGS}>") + + rtcx_embed_blob( + ${TARGET} FILE $ DEST fragments/${ARG_FRAGMENT}.fatbin ID + ${ARG_FRAGMENT} ARRAY_IDS ${ARG_ARRAY_IDS} ARRAY_VALUES ${ARG_ARRAY_VALUES} + ) +endmacro() From bcd5c5ca90fa3746a2b86486d1ababf4177fffbe Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Jul 2026 20:26:24 +0000 Subject: [PATCH 246/254] refactor: move device version retrieval to context class and update cache implementation --- cpp/src/jit/cache.cpp | 96 ++++++++++++++++--------------------- cpp/src/runtime/context.cpp | 41 +++++++++++++++- cpp/src/runtime/context.hpp | 11 ++++- 3 files changed, 90 insertions(+), 58 deletions(-) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index a0cd387b8342..52c3d13e624d 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -202,31 +202,6 @@ constexpr int MIN_CUDA_VERSION_PCH = constexpr int MIN_CUDA_VERSION_MINIMAL = make_cuda_version(12, 8, 0); // minimum CUDA version for the "--minimal" NVRTC flag -int32_t get_driver_version() -{ - int32_t driver_version; - CUDF_CUDA_TRY(cudaDriverGetVersion(&driver_version)); - return driver_version; -} - -int32_t get_runtime_version() -{ - int32_t runtime_version; - CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); - return runtime_version; -} - -int32_t get_current_device_compute_capability() -{ - int32_t device; - CUDF_CUDA_TRY(cudaGetDevice(&device)); - - cudaDeviceProp props; - CUDF_CUDA_TRY(cudaGetDeviceProperties(&props, device)); - - return props.major * 10 + props.minor; -} - std::tuple compile_library( char const* name, char const* cuda_code, @@ -236,11 +211,12 @@ std::tuple compile_library( { CUDF_FUNC_RANGE(); - auto& ctx = cudf::get_context(); - auto& cfg = ctx.config(); - auto& bundle = ctx.jit_bundle(); - auto sm = get_current_device_compute_capability(); - auto runtime = get_runtime_version(); + auto& ctx = cudf::get_context(); + auto& cfg = ctx.config(); + auto& bundle = ctx.jit_bundle(); + auto& device_properties = ctx.get_device_properties(); + auto sm = device_properties.compute_capability; + auto runtime = device_properties.runtime_version; auto include_dirs = bundle.get_include_directories(); auto use_pch = runtime >= MIN_CUDA_VERSION_PCH; @@ -314,11 +290,12 @@ rtcx::blob compile_fragment(char const* name, { CUDF_FUNC_RANGE(); - auto& ctx = cudf::get_context(); - auto& cfg = ctx.config(); - auto& bundle = ctx.jit_bundle(); - auto sm = get_current_device_compute_capability(); - auto runtime = get_runtime_version(); + auto& ctx = cudf::get_context(); + auto& cfg = ctx.config(); + auto& bundle = ctx.jit_bundle(); + auto& device_properties = ctx.get_device_properties(); + auto sm = device_properties.compute_capability; + auto runtime = device_properties.runtime_version; auto include_dirs = bundle.get_include_directories(); auto pch_dir = ctx.get_jit_pch_dir(); @@ -395,13 +372,15 @@ kernel get_kernel(std::string const& name, { CUDF_FUNC_RANGE(); - auto& cache = cudf::get_context().rtcx_cache(); - auto& bundle = cudf::get_context().jit_bundle(); + auto& ctx = cudf::get_context(); + auto& cache = ctx.rtcx_cache(); + auto& bundle = ctx.jit_bundle(); + auto& device_properties = ctx.get_device_properties(); + auto sm = device_properties.compute_capability; + auto runtime = device_properties.runtime_version; + auto driver = device_properties.driver_version; + auto bundle_hash = bundle.get_hash(); - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_compute_capability(); - auto bundle_hash = bundle.get_hash(); auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); auto spec = std::format(R"***(cuLibrary @@ -454,13 +433,15 @@ rtcx::blob get_kernel_fragment(std::string const& name, { CUDF_FUNC_RANGE(); - auto& cache = cudf::get_context().rtcx_cache(); - auto& bundle = cudf::get_context().jit_bundle(); + auto& ctx = cudf::get_context(); + auto& cache = ctx.rtcx_cache(); + auto& bundle = ctx.jit_bundle(); + auto& device_properties = ctx.get_device_properties(); + auto runtime = device_properties.runtime_version; + auto driver = device_properties.driver_version; + auto sm = device_properties.compute_capability; + auto bundle_hash = bundle.get_hash(); - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_compute_capability(); - auto bundle_hash = bundle.get_hash(); auto source_file = std::format("{}/{}", bundle.get_directory(), source_file_id); auto spec = std::format(R"***(objectFile @@ -511,9 +492,10 @@ std::tuple link_library_uncached( { CUDF_FUNC_RANGE(); - auto sm = get_current_device_compute_capability(); - auto& ctx = cudf::get_context(); - auto& cfg = ctx.config(); + auto& ctx = cudf::get_context(); + auto& device_properties = ctx.get_device_properties(); + auto sm = device_properties.compute_capability; + auto& cfg = ctx.config(); std::vector options; @@ -551,12 +533,14 @@ kernel get_lto_linked_kernel(std::string const& name, { CUDF_FUNC_RANGE(); - auto& cache = cudf::get_context().rtcx_cache(); - auto& bundle = cudf::get_context().jit_bundle(); - auto runtime = get_runtime_version(); - auto driver = get_driver_version(); - auto sm = get_current_device_compute_capability(); - auto bundle_hash = bundle.get_hash(); + auto& ctx = cudf::get_context(); + auto& cache = ctx.rtcx_cache(); + auto& bundle = ctx.jit_bundle(); + auto& device_properties = ctx.get_device_properties(); + auto runtime = device_properties.runtime_version; + auto driver = device_properties.driver_version; + auto sm = device_properties.compute_capability; + auto bundle_hash = bundle.get_hash(); auto spec = std::format(R"***(cuLibrary name={} diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index be7be0cb446c..8a2d15cf80e6 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -17,8 +17,40 @@ namespace cudf { +namespace { + +int32_t get_driver_version() +{ + int32_t driver_version; + CUDF_CUDA_TRY(cudaDriverGetVersion(&driver_version)); + return driver_version; +} + +int32_t get_runtime_version() +{ + int32_t runtime_version; + CUDF_CUDA_TRY(cudaRuntimeGetVersion(&runtime_version)); + return runtime_version; +} + +int32_t get_current_device_compute_capability() +{ + int32_t device; + CUDF_CUDA_TRY(cudaGetDevice(&device)); + + cudaDeviceProp props; + CUDF_CUDA_TRY(cudaGetDeviceProperties(&props, device)); + + return props.major * 10 + props.minor; +} + +} // namespace + context::context(context_config cfg, init_flags flags) - : _config{std::move(cfg)}, _jit_cache_init_flag{} + : _config{std::move(cfg)}, + _jit_cache_init_flag{}, + _device_properties{ + get_driver_version(), get_runtime_version(), get_current_device_compute_capability()} { initialize_components(flags); } @@ -78,8 +110,15 @@ bool context::dump_codegen() const { return _config.dump_codegen; } bool context::use_jit() const { return _config.use_jit; } +context_config const& context::config() const { return _config; } + std::string const& context::get_jit_pch_dir() const { return _config.jit_pch_dir; } +context::device_properties const& context::get_device_properties() const +{ + return _device_properties; +} + void context::initialize_components(init_flags flags) { if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { ensure_jit_cache_initialized(); } diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 8477fcc89f7b..7fdb553dee3e 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -45,11 +45,18 @@ struct [[nodiscard]] context_config { /// objects/state across translation units. class context { public: + struct device_properties { + int32_t driver_version = 0; + int32_t runtime_version = 0; + int32_t compute_capability = 0; + }; + private: context_config _config; std::once_flag _jit_cache_init_flag; std::unique_ptr _rtcx_cache; std::unique_ptr _jit_bundle; + device_properties _device_properties; private: void ensure_nvcomp_loaded(); @@ -72,10 +79,12 @@ class context { [[nodiscard]] bool use_jit() const; - [[nodiscard]] context_config const& config() const { return _config; } + [[nodiscard]] context_config const& config() const; [[nodiscard]] std::string const& get_jit_pch_dir() const; + [[nodiscard]] device_properties const& get_device_properties() const; + /// @brief Initialize additional components based on the provided flags /// @param flags The initialization flags to process void initialize_components(init_flags flags); From 5d83a530cd5966f8522d5ea29f2eb707e67565be Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Jul 2026 21:44:37 +0000 Subject: [PATCH 247/254] update copyright notice in AddFragment.cmake --- cpp/cmake/Modules/AddFragment.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/Modules/AddFragment.cmake b/cpp/cmake/Modules/AddFragment.cmake index 4c9a7bd66b7a..187d9c36d594 100644 --- a/cpp/cmake/Modules/AddFragment.cmake +++ b/cpp/cmake/Modules/AddFragment.cmake @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= From 24b929cd124034a8e380105a6979111109952a2e Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 1 Jul 2026 23:50:24 +0000 Subject: [PATCH 248/254] Suppress nv_hdrstop directive warnings for later CUDA versions in compile_library and compile_fragment --- cpp/src/jit/cache.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index a0cd387b8342..d67b8fb1493c 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -276,6 +276,9 @@ std::tuple compile_library( options.emplace_back("--pch-verbose=false"); options.emplace_back("--pch-messages=false"); } + } else { + // suppress warning about nv_hdrstop directive on later CUDA versions + options.emplace_back("--diag-suppress=161"); } if (cfg.disable_cuda_cache) { options.emplace_back("--no-cache"); } @@ -358,6 +361,9 @@ rtcx::blob compile_fragment(char const* name, options.emplace_back("--pch-verbose=false"); options.emplace_back("--pch-messages=false"); } + } else { + // suppress warning about nv_hdrstop directive on later CUDA versions + options.emplace_back("--diag-suppress=161"); } if (cfg.disable_cuda_cache) { options.emplace_back("--no-cache"); } From f730f369609c98a921d9fe1dad66796f54508e06 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 3 Jul 2026 04:33:33 +0000 Subject: [PATCH 249/254] rebase on main with fallibility support --- cpp/CMakeLists.txt | 74 ++++++++++--------- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 4 + cpp/include/cudf/transform.hpp | 2 + cpp/src/transform/jit/kernel.cu | 16 ++-- cpp/src/transform/transform.cu | 36 +++++++-- cpp/tests/CMakeLists.txt | 10 ++- cpp/tests/transform/fragments/invsqrt.cu | 4 +- cpp/tests/transform/fragments/lehmer_mean.cu | 43 +++++++++++ .../transform/fragments/sum_of_squares.cu | 4 +- cpp/tests/transform/fragments/to_upper.cu | 11 ++- cpp/tests/transform/transform_lto_test.cpp | 56 ++++++++++++++ 11 files changed, 203 insertions(+), 57 deletions(-) create mode 100644 cpp/tests/transform/fragments/lehmer_mean.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1ca85b800881..17430ea1135f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -533,39 +533,7 @@ list( ) foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) - set(FRAGMENT_NAME transform_kernel) - get_property( - FILE_INDEX - TARGET cudf_fragments__embed_props - PROPERTY EMBED_FILE_INDEX - ) - set(VARIANT_NAME transform_kernel_${FILE_INDEX}) - set(INSTANCE - "cudf::jit::transform_kernel>, cudf::jit::type_list>>" - ) - add_fragment( - cudf_fragments - FRAGMENT - ${VARIANT_NAME} - SOURCE - src/transform/jit/kernel.cu - KERNEL_INSTANCE - ${INSTANCE} - UDF_TYPE - "int(${TYPE} *, ${TYPE})" - DEFINITIONS - CUDF_LTO_MODE - ARRAY_IDS - ${FRAGMENT_NAME}_FILE_INDEX - ${FRAGMENT_NAME}_INSTANCE - ARRAY_VALUES - ${FILE_INDEX} - "${INSTANCE}" - ) -endforeach() - -foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) - foreach(RHS_IS_SCALAR IN ITEMS "false" "true") + foreach(DISCARD_ERRORS IN ITEMS "false" "true") set(FRAGMENT_NAME transform_kernel) get_property( FILE_INDEX @@ -574,7 +542,7 @@ foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) ) set(VARIANT_NAME transform_kernel_${FILE_INDEX}) set(INSTANCE - "cudf::jit::transform_kernel, cudf::jit::column_accessor<1ULL, cudf::column_device_view_core, ${TYPE}, ${RHS_IS_SCALAR}, 0>>, cudf::jit::type_list>>" + "cudf::jit::transform_kernel>, cudf::jit::type_list>>" ) add_fragment( cudf_fragments @@ -585,7 +553,7 @@ foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) KERNEL_INSTANCE ${INSTANCE} UDF_TYPE - "int(${TYPE} *, ${TYPE}, ${TYPE})" + "int(${TYPE} *, ${TYPE})" DEFINITIONS CUDF_LTO_MODE ARRAY_IDS @@ -598,6 +566,42 @@ foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) endforeach() endforeach() +foreach(DISCARD_ERRORS IN ITEMS "false" "true") + foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) + foreach(RHS_IS_SCALAR IN ITEMS "false" "true") + set(FRAGMENT_NAME transform_kernel) + get_property( + FILE_INDEX + TARGET cudf_fragments__embed_props + PROPERTY EMBED_FILE_INDEX + ) + set(VARIANT_NAME transform_kernel_${FILE_INDEX}) + set(INSTANCE + "cudf::jit::transform_kernel, cudf::jit::column_accessor<1ULL, cudf::column_device_view_core, ${TYPE}, ${RHS_IS_SCALAR}, 0>>, cudf::jit::type_list>>" + ) + add_fragment( + cudf_fragments + FRAGMENT + ${VARIANT_NAME} + SOURCE + src/transform/jit/kernel.cu + KERNEL_INSTANCE + ${INSTANCE} + UDF_TYPE + "int(${TYPE} *, ${TYPE}, ${TYPE})" + DEFINITIONS + CUDF_LTO_MODE + ARRAY_IDS + ${FRAGMENT_NAME}_FILE_INDEX + ${FRAGMENT_NAME}_INSTANCE + ARRAY_VALUES + ${FILE_INDEX} + "${INSTANCE}" + ) + endforeach() + endforeach() +endforeach() + rtcx_embed( cudf_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" ) diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index bc359f825730..3bb831d295ec 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -158,6 +158,7 @@ __device__ void transform(float* out, float a, float b) { auto result = use_lto ? cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, + cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -166,6 +167,7 @@ __device__ void transform(float* out, float a, float b) { : cudf::multi_transform(cuda, cudf::udf_source_type::CUDA, cudf::null_aware::NO, + cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -181,6 +183,7 @@ __device__ void transform(float* out, float a, float b) { [[maybe_unused]] auto result = use_lto ? cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, + cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -189,6 +192,7 @@ __device__ void transform(float* out, float a, float b) { : cudf::multi_transform(cuda, cudf::udf_source_type::CUDA, cudf::null_aware::NO, + cudf::fallible::NO, std::nullopt, inputs, outputs, diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 01796426b158..1346788787c7 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -263,6 +263,7 @@ enum class lto_binary_type : uint8_t { * `transform` and follow the CUDF UDF ABI. * @param binary_type The type of the LTO binary provided in `udf` * @param is_null_aware Signifies the UDF will receive row inputs as optional values + * @param is_fallible Signifies the UDF may produce errors during execution * @param user_data User-defined device data to pass to the UDF. * @param inputs Immutable view of the input to transform * @param outputs Specification of the output columns to be created @@ -280,6 +281,7 @@ std::unique_ptr
transform_lto( std::span udf, lto_binary_type binary_type, null_aware is_null_aware, + fallible is_fallible, std::optional user_data, std::span inputs, std::span outputs, diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 48c6bac78ee1..019dc9757395 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -61,12 +61,12 @@ template -CUDF_KERNEL void transform_kernel(size_type row_size, - bitmask_type const* __restrict__ stencil, - void* __restrict__ user_data, - column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols, - int32_t* __restrict__ max_error) +__device__ void transform_kernel(size_type row_size, + bitmask_type const* __restrict__ stencil, + void* __restrict__ user_data, + column_device_view_core const* __restrict__ input_cols, + mutable_column_device_view_core const* __restrict__ output_cols, + int32_t* __restrict__ max_error) { auto start = detail::grid_1d::global_thread_id(); auto stride = detail::grid_1d::grid_stride(); @@ -103,7 +103,7 @@ CUDF_KERNEL void transform_kernel(size_type row_size, auto out_ptrs = cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - auto row_error = operation(cuda::std::tuple_cat(out_ptrs, ins)); + auto row_error = static_cast(operation(cuda::std::tuple_cat(out_ptrs, ins))); OutputAccessors::map([&]() { (A::assign(output_cols, row, cuda::std::get(outs)), ...); @@ -123,7 +123,7 @@ CUDF_KERNEL void transform_kernel(size_type row_size, auto out_ptrs = cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - auto row_error = operation(cuda::std::tuple_cat(out_ptrs, ins)); + auto row_error = static_cast(operation(cuda::std::tuple_cat(out_ptrs, ins))); OutputAccessors::map([&]() { (A::assign(output_cols, row, *cuda::std::get(outs)), ...); diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index bea1be5e67e5..cd67701cb046 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -339,8 +339,10 @@ std::string reflect_udf_signature(bool is_null_aware, return std::format("int({})", joined); } +// determine CUDF_KERNEL_INSTANCE and CUDF_UDF_TYPE std::tuple instantiate_fragment( bool is_null_aware, + bool discard_errors, bool has_user_data, std::string const& ins, std::string const& outs, @@ -350,6 +352,7 @@ std::tuple instantiate_fragment( CUDF_FUNC_RANGE(); auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", rtcx::reflect(is_null_aware), + rtcx::reflect(discard_errors), rtcx::reflect(has_user_data), ins, outs); @@ -444,12 +447,14 @@ rtcx::binary_type as_rtcx_binary_type(lto_binary_type type) void run_lto(std::optional, lto_binary_type, char const*>> precompiled_kernel_fragment, bool is_null_aware, + bool discard_errors, bool has_user_data, size_type row_size, bitmask_type const* d_stencil, void* user_data, std::span inputs, std::span outputs, + int32_t* d_max_error, std::span udf_binary, lto_binary_type source_type, rmm::cuda_stream_view stream, @@ -466,8 +471,8 @@ void run_lto(std::optional, lto_binary_type, std::tie(kernel_fragment, kernel_fragment_binary_type, kernel_fragment_id) = *precompiled_kernel_fragment; } else { - std::tie(fragment_blob, kernel_fragment_binary_type, kernel_fragment_id) = - instantiate_fragment(is_null_aware, has_user_data, in_types, out_types, inputs, outputs); + std::tie(fragment_blob, kernel_fragment_binary_type, kernel_fragment_id) = instantiate_fragment( + is_null_aware, discard_errors, has_user_data, in_types, out_types, inputs, outputs); kernel_fragment = fragment_blob->view(); } @@ -489,7 +494,8 @@ void run_lto(std::optional, lto_binary_type, auto* input_cols = reinterpret_cast(cols.data()); auto* output_cols = reinterpret_cast(input_cols + inputs.size()); - return launch(kernel, row_size, d_stencil, user_data, input_cols, output_cols, stream); + return launch( + kernel, row_size, d_stencil, user_data, input_cols, output_cols, d_max_error, stream); } } // namespace jit_transform @@ -1141,6 +1147,7 @@ std::unique_ptr compute_column_jit(table_view const& table, // it to use for LTO linking instead of compiling a new one std::optional, lto_binary_type, char const*>> dispatch_lto_kernel_fragment(bool is_null_aware, + bool discard_errors, bool has_user_data, std::span inputs, std::span outputs) @@ -1162,6 +1169,7 @@ dispatch_lto_kernel_fragment(bool is_null_aware, jit_transform::reflect(lto_binary_type::FATBIN, inputs, outputs); auto target = strip_whitespace(rtcx::reflect_template("cudf::jit::transform_kernel", rtcx::reflect(is_null_aware), + rtcx::reflect(discard_errors), rtcx::reflect(has_user_data), in_types, out_types)); @@ -1182,6 +1190,7 @@ dispatch_lto_kernel_fragment(bool is_null_aware, std::unique_ptr
transform_lto(std::span udf, lto_binary_type binary_type, null_aware is_null_aware, + fallible is_fallible, std::optional user_data, std::span inputs, std::span outputs, @@ -1206,20 +1215,37 @@ std::unique_ptr
transform_lto(std::span udf, auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; - auto precompiled_kernel_fragment = dispatch_lto_kernel_fragment( - is_null_aware == null_aware::YES, user_data.has_value(), inputs, output_columns); + auto precompiled_kernel_fragment = dispatch_lto_kernel_fragment(is_null_aware == null_aware::YES, + is_fallible == fallible::NO, + user_data.has_value(), + inputs, + output_columns); + + rmm::device_scalar d_max_error(static_cast(errc::SUCCESS), stream, mr); + jit_transform::run_lto(precompiled_kernel_fragment, is_null_aware == null_aware::YES, + is_fallible == fallible::NO, user_data.has_value(), row_size, stencil_has_nulls ? stencil_arg : nullptr, user_data.value_or(nullptr), inputs, output_columns, + d_max_error.data(), udf, binary_type, stream, mr); + + auto error = static_cast(d_max_error.value(stream)); + switch (error) { + case errc::SUCCESS: break; + default: + throw evaluation_error( + error, std::format("Transform UDF evaluation failed with error `{}`", to_string(error))); + } + auto finalized = finalize_outputs(is_null_aware, row_size, std::move(output_columns), stream, mr); return std::make_unique
(std::move(finalized)); } diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 59ef89328a28..c18bedc8de53 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -695,17 +695,19 @@ ConfigureTest(AST_TEST ast/transform_tests.cpp ast/ast_tree_tests.cpp ast/jit_ex # ------------------------------------------------------------------------------------- rtcx_add_embed(cudf_test_fragments) +add_fragment( + cudf_test_fragments FRAGMENT bankers_rounding SOURCE transform/fragments/bankers_rounding.cu +) + add_fragment(cudf_test_fragments FRAGMENT invsqrt SOURCE transform/fragments/invsqrt.cu) -add_fragment(cudf_test_fragments FRAGMENT to_upper SOURCE transform/fragments/to_upper.cu) +add_fragment(cudf_test_fragments FRAGMENT lehmer_mean SOURCE transform/fragments/lehmer_mean.cu) add_fragment( cudf_test_fragments FRAGMENT sum_of_squares SOURCE transform/fragments/sum_of_squares.cu ) -add_fragment( - cudf_test_fragments FRAGMENT bankers_rounding SOURCE transform/fragments/bankers_rounding.cu -) +add_fragment(cudf_test_fragments FRAGMENT to_upper SOURCE transform/fragments/to_upper.cu) rtcx_embed( cudf_test_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" diff --git a/cpp/tests/transform/fragments/invsqrt.cu b/cpp/tests/transform/fragments/invsqrt.cu index 689fc5ac4692..7c652b0ce27e 100644 --- a/cpp/tests/transform/fragments/invsqrt.cu +++ b/cpp/tests/transform/fragments/invsqrt.cu @@ -3,8 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ +__device__ float invsqrt(float a) { return 1.0F / sqrtf(a); } + extern "C" __device__ int transform(float* out, float a) { - *out = 1.0F / sqrtf(a); + *out = invsqrt(a); return 0; } diff --git a/cpp/tests/transform/fragments/lehmer_mean.cu b/cpp/tests/transform/fragments/lehmer_mean.cu new file mode 100644 index 000000000000..e1747d69adc5 --- /dev/null +++ b/cpp/tests/transform/fragments/lehmer_mean.cu @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include + +/** + * @brief Calculates the integer Lehmer mean of two integers using checked arithmetic. + * The Lehmer mean is defined as `(a^2 + b^2) / (a + b)`. + * @param a The first integer. + * @param b The second integer. + * @return The Lehmer mean of the two integers, or an error code if an overflow occurs during the + * calculation. + * + */ +__device__ cuda::std::expected lehmer_mean(int32_t a, int32_t b) +{ + auto a2 = cudf::detail::ops::mul_overflow(a, a); + if (!a2) return cuda::std::unexpected(a2.error()); + + auto b2 = cudf::detail::ops::mul_overflow(b, b); + if (!b2) return cuda::std::unexpected(b2.error()); + + auto a_b_sum = cudf::detail::ops::add_overflow(a, b); + if (!a_b_sum) return cuda::std::unexpected(a_b_sum.error()); + + auto a2_b2_sum = cudf::detail::ops::add_overflow(a2.value(), b2.value()); + if (!a2_b2_sum) return cuda::std::unexpected(a2_b2_sum.error()); + + return cudf::detail::ops::div_overflow(a2_b2_sum.value(), a_b_sum.value()); +} + +extern "C" __device__ int transform(int32_t* out, int32_t a, int32_t b) +{ + auto result = lehmer_mean(a, b); + if (!result) return static_cast(result.error()); + *out = result.value(); + return 0; +} diff --git a/cpp/tests/transform/fragments/sum_of_squares.cu b/cpp/tests/transform/fragments/sum_of_squares.cu index b3bb5e289662..e3e7a149f715 100644 --- a/cpp/tests/transform/fragments/sum_of_squares.cu +++ b/cpp/tests/transform/fragments/sum_of_squares.cu @@ -3,8 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ +__device__ float sum_of_squares(float a, float b) { return a * a + b * b; } + extern "C" __device__ int transform(float* out, float a, float b) { - *out = a * a + b * b; + *out = sum_of_squares(a, b); return 0; } diff --git a/cpp/tests/transform/fragments/to_upper.cu b/cpp/tests/transform/fragments/to_upper.cu index 953d4992c823..f809acc63133 100644 --- a/cpp/tests/transform/fragments/to_upper.cu +++ b/cpp/tests/transform/fragments/to_upper.cu @@ -5,12 +5,17 @@ #include -extern "C" __device__ int transform(uint8_t* output, uint8_t input) +__device__ uint8_t to_upper(uint8_t input) { if (input > 96 && input < 123) { - *output = input - 32; + return input - 32; } else { - *output = input; + return input; } +} + +extern "C" __device__ int transform(uint8_t* output, uint8_t input) +{ + *output = to_upper(input); return 0; } diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp index 2d6a2cbd326f..aea0175852e0 100644 --- a/cpp/tests/transform/transform_lto_test.cpp +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -37,6 +37,7 @@ TEST_F(TransformLTOTest, InvSqrt) auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, + cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -63,6 +64,7 @@ TEST_F(TransformLTOTest, ToUpper) auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, + cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -90,6 +92,7 @@ TEST_F(TransformLTOTest, SumOfSquares) auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, + cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -102,6 +105,58 @@ TEST_F(TransformLTOTest, SumOfSquares) CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); } +TEST_F(TransformLTOTest, LehmerMean) +{ + // computes integer lehmer mean: `(a^2 + b^2) / (a + b)` for each row using checked arithmetic and + // throws if an overflow occurs + + column_wrapper a{{2, 3, 4, 6, 8}}; + column_wrapper b{{1, 1, 2, 3, 4}}; + column_wrapper a_fail{{2, 3, -2, 6, 8}}; + + cudf::transform_output outputs[] = { + {cudf::data_type{cudf::type_id::INT32}, cudf::output_nullability::ALL_VALID}}; + + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::lehmer_mean]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + cudf::transform_input inputs[] = {a, b}; + + auto result = cudf::transform_lto(udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + cudf::fallible::YES, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt, + cudf::test::get_default_stream()); + auto lehmer_mean = [](int32_t a, int32_t b) { return (a * a + b * b) / (a + b); }; + + column_wrapper expected{{lehmer_mean(2, 1), + lehmer_mean(3, 1), + lehmer_mean(4, 2), + lehmer_mean(6, 3), + lehmer_mean(8, 4)}}; + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); + + cudf::transform_input inputs_fail[] = {a_fail, b}; + + EXPECT_THROW(cudf::transform_lto(udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + cudf::fallible::YES, + std::nullopt, + inputs_fail, + outputs, + {}, + std::nullopt, + cudf::test::get_default_stream()), + cudf::evaluation_error); +} + TEST_F(TransformLTOTest, BankersRounding) { using T = numeric::decimal128; @@ -121,6 +176,7 @@ TEST_F(TransformLTOTest, BankersRounding) auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, + cudf::fallible::NO, std::nullopt, inputs, outputs, From 69bac2ea0066ba077178cbe38444f548fc94086d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 3 Jul 2026 09:16:28 +0000 Subject: [PATCH 250/254] Refactor transform kernel and related functions to remove fallibility checks - Removed the `fallible` enum and related checks from the transform functions and kernels. - Updated the `multi_transform` and `transform_lto` functions to no longer accept a fallibility parameter. - Adjusted the kernel implementation to handle errors without the fallibility flag. - Modified the CMake configuration to reflect changes in the transform kernel instantiation. - Updated tests to remove fallibility assertions and ensure compatibility with the new implementation. --- cpp/CMakeLists.txt | 98 ++++++------- cpp/benchmarks/binaryop/compiled_binaryop.cpp | 4 - cpp/include/cudf/transform.hpp | 53 ------- cpp/include/cudf/types.hpp | 8 -- cpp/src/jit/row_ir.cpp | 27 +--- cpp/src/jit/row_ir.hpp | 10 +- cpp/src/jit/util.cpp | 41 +++++- cpp/src/jit/util.hpp | 34 ++++- cpp/src/transform/jit/kernel.cu | 27 ++-- cpp/src/transform/transform.cu | 130 +++++++----------- .../integration/unary_transform_test.cpp | 2 - cpp/tests/transform/transform_lto_test.cpp | 8 +- 12 files changed, 181 insertions(+), 261 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 17430ea1135f..9f58f79e9e80 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -506,34 +506,54 @@ rtcx_add_embed(cudf_fragments) list( APPEND - CUDF_PRECOMPILE_TYPES + CUDF_PRECOMPILE_PHYSICAL_TYPES uint8_t uint16_t uint32_t uint64_t - int8_t - int16_t - int32_t - int64_t - float - double numeric::decimal32 numeric::decimal64 numeric::decimal128 - cudf::timestamp_D - cudf::timestamp_s - cudf::timestamp_ms - cudf::timestamp_us - cudf::timestamp_ns - cudf::duration_D - cudf::duration_s - cudf::duration_ms - cudf::duration_us - cudf::duration_ns + # cudf::dictionary_element cudf::dictionary_element + # cudf::dictionary_element cudf::dictionary_element + # cudf::dictionary_element cudf::dictionary_element cudf::dictionary_element ) -foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) - foreach(DISCARD_ERRORS IN ITEMS "false" "true") +foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_PHYSICAL_TYPES}) + set(FRAGMENT_NAME transform_kernel) + get_property( + FILE_INDEX + TARGET cudf_fragments__embed_props + PROPERTY EMBED_FILE_INDEX + ) + set(VARIANT_NAME transform_kernel_${FILE_INDEX}) + set(INSTANCE + "cudf::jit::transform_kernel>, cudf::jit::type_list>>" + ) + add_fragment( + cudf_fragments + FRAGMENT + ${VARIANT_NAME} + SOURCE + src/transform/jit/kernel.cu + KERNEL_INSTANCE + ${INSTANCE} + UDF_TYPE + "int(${TYPE} *, ${TYPE})" + DEFINITIONS + CUDF_LTO_MODE + ARRAY_IDS + ${FRAGMENT_NAME}_FILE_INDEX + ${FRAGMENT_NAME}_INSTANCE + ARRAY_VALUES + ${FILE_INDEX} + "${INSTANCE}" + ) +endforeach() + +foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_PHYSICAL_TYPES}) + foreach(RHS_IS_SCALAR IN ITEMS "false" "true") set(FRAGMENT_NAME transform_kernel) get_property( FILE_INDEX @@ -542,7 +562,7 @@ foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) ) set(VARIANT_NAME transform_kernel_${FILE_INDEX}) set(INSTANCE - "cudf::jit::transform_kernel>, cudf::jit::type_list>>" + "cudf::jit::transform_kernel, cudf::jit::column_accessor<1ULL, cudf::column_device_view_core, ${TYPE}, ${RHS_IS_SCALAR}, 0>>, cudf::jit::type_list>>" ) add_fragment( cudf_fragments @@ -553,7 +573,7 @@ foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) KERNEL_INSTANCE ${INSTANCE} UDF_TYPE - "int(${TYPE} *, ${TYPE})" + "int(${TYPE} *, ${TYPE}, ${TYPE})" DEFINITIONS CUDF_LTO_MODE ARRAY_IDS @@ -566,42 +586,6 @@ foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) endforeach() endforeach() -foreach(DISCARD_ERRORS IN ITEMS "false" "true") - foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_TYPES}) - foreach(RHS_IS_SCALAR IN ITEMS "false" "true") - set(FRAGMENT_NAME transform_kernel) - get_property( - FILE_INDEX - TARGET cudf_fragments__embed_props - PROPERTY EMBED_FILE_INDEX - ) - set(VARIANT_NAME transform_kernel_${FILE_INDEX}) - set(INSTANCE - "cudf::jit::transform_kernel, cudf::jit::column_accessor<1ULL, cudf::column_device_view_core, ${TYPE}, ${RHS_IS_SCALAR}, 0>>, cudf::jit::type_list>>" - ) - add_fragment( - cudf_fragments - FRAGMENT - ${VARIANT_NAME} - SOURCE - src/transform/jit/kernel.cu - KERNEL_INSTANCE - ${INSTANCE} - UDF_TYPE - "int(${TYPE} *, ${TYPE}, ${TYPE})" - DEFINITIONS - CUDF_LTO_MODE - ARRAY_IDS - ${FRAGMENT_NAME}_FILE_INDEX - ${FRAGMENT_NAME}_INSTANCE - ARRAY_VALUES - ${FILE_INDEX} - "${INSTANCE}" - ) - endforeach() - endforeach() -endforeach() - rtcx_embed( cudf_fragments COMPRESSION none OUTPUT_DIRECTORY "${CUDF_GENERATED_INCLUDE_DIR}/rtcx_embed" ) diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index 3bb831d295ec..bc359f825730 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -158,7 +158,6 @@ __device__ void transform(float* out, float a, float b) { auto result = use_lto ? cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, - cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -167,7 +166,6 @@ __device__ void transform(float* out, float a, float b) { : cudf::multi_transform(cuda, cudf::udf_source_type::CUDA, cudf::null_aware::NO, - cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -183,7 +181,6 @@ __device__ void transform(float* out, float a, float b) { [[maybe_unused]] auto result = use_lto ? cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, - cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -192,7 +189,6 @@ __device__ void transform(float* out, float a, float b) { : cudf::multi_transform(cuda, cudf::udf_source_type::CUDA, cudf::null_aware::NO, - cudf::fallible::NO, std::nullopt, inputs, outputs, diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 1346788787c7..024a26792724 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -180,57 +180,6 @@ std::unique_ptr
multi_transform( rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); -/** - * @brief Creates a new table by applying a transform function against every - * element of the input columns. - * - * Computes: - * `(outputs[i]...) = UDF(inputs[i]...)`. - * - * - * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) - * @throws std::invalid_argument if `output_type` or any of the inputs are not fixed-width or string - * types - * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and - * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in - * this case - * @throws std::invalid_argument if string offsets are provided for non-string output columns, or - * if the number of string offsets does not match the number of output columns - * @throws cudf::evaluation_error if the UDF produces an error during execution - * - * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from - * the input and pre-allocated output columns. - * - * @param udf The PTX/CUDA string of the transform function to apply - * @param source_type The source type of the UDF (CUDA or PTX) - * @param is_null_aware Signifies the UDF will receive row inputs as optional values - * @param is_fallible Signifies the UDF may produce errors during execution - * @param user_data User-defined device data to pass to the UDF - * @param inputs Immutable views of the inputs to transform (columns and scalar columns) - * @param outputs Specification of the output columns to be created - * @param string_offsets For string output columns, the offsets can be pre-allocated and passed in - * to prevent overhead of compacting string views into run-end strings column. - * @param row_size The row size of the transform operation. If not provided, it is inferred from the - * input columns - * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned column's device memory - * @return A table containing the columns resulting from applying the transform - * function to every element of the input according to the output specifications - * - */ -std::unique_ptr
multi_transform( - std::string const& udf, - udf_source_type source_type, - null_aware is_null_aware, - fallible is_fallible, - std::optional user_data, - std::span inputs, - std::span outputs, - std::vector>&& string_offsets, - std::optional row_size, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); - /** * @brief The type of LTO Binary */ @@ -263,7 +212,6 @@ enum class lto_binary_type : uint8_t { * `transform` and follow the CUDF UDF ABI. * @param binary_type The type of the LTO binary provided in `udf` * @param is_null_aware Signifies the UDF will receive row inputs as optional values - * @param is_fallible Signifies the UDF may produce errors during execution * @param user_data User-defined device data to pass to the UDF. * @param inputs Immutable view of the input to transform * @param outputs Specification of the output columns to be created @@ -281,7 +229,6 @@ std::unique_ptr
transform_lto( std::span udf, lto_binary_type binary_type, null_aware is_null_aware, - fallible is_fallible, std::optional user_data, std::span inputs, std::span outputs, diff --git a/cpp/include/cudf/types.hpp b/cpp/include/cudf/types.hpp index 188424299a64..414f4c554473 100644 --- a/cpp/include/cudf/types.hpp +++ b/cpp/include/cudf/types.hpp @@ -261,14 +261,6 @@ enum class output_nullability : uint8_t { ///< null values are produced }; -/** - * @brief Indicates whether a function can produce an error during execution. - */ -enum class fallible : bool { - NO = 0, ///< The function is not fallible - YES = 1 ///< The function is fallible -}; - /** * @brief Indicates whether a function nullifies its output on error. * diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index cb801b30e857..a41bde6bdc0b 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -551,20 +551,6 @@ bool node::is_null_aware() const return std::any_of(args_.begin(), args_.end(), [](auto& a) { return a->is_null_aware(); }); } -bool node::is_fallible() const -{ - if (op_ == opcode::GET_INPUT) { return false; } - - if (get_op_info(op_, error_policy_).is_fallible) { return true; } - - CUDF_EXPECTS(!args_.empty(), - "Unexpectedly found an operator node with no arguments. All operator nodes should " - "have at least one argument.", - std::runtime_error); - - return std::any_of(args_.begin(), args_.end(), [](auto& a) { return a->is_fallible(); }); -} - bool node::is_always_valid() const { if (op_ == opcode::GET_INPUT) { return false; } @@ -789,7 +775,7 @@ bool is_nullable(scalar_input const& in) { return in.scalar_column->view().nulla bool is_nullable(column_input const& in) { return in.column.nullable(); } -std::tuple ast_converter::generate_code( +std::tuple ast_converter::generate_code( target target_id, ast::expression const& expr, std::string_view function_name) { // add 1 auto-deduced output variable @@ -805,9 +791,6 @@ std::tuple ast_converter: bool is_null_aware = std::any_of( output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_null_aware(); }); - bool is_fallible = std::any_of( - output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_fallible(); }); - bool output_is_always_valid = std::all_of( output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_always_valid(); }); @@ -870,10 +853,7 @@ std::tuple ast_converter: ir->emit_code(instance_, target, sink); } sink.emit("return cudf::errc::SUCCESS;\n}"); - return {sink.get_code(), - is_null_aware ? null_aware::YES : null_aware::NO, - is_fallible ? fallible::YES : fallible::NO, - null_policy}; + return {sink.get_code(), is_null_aware ? null_aware::YES : null_aware::NO, null_policy}; } std::variant get_column_view(scalar_input const& in) @@ -901,7 +881,7 @@ transform_args ast_converter::compute_column(target target_id, // TODO(lamarrr): consider deduplicating ast expression's input column references. See // TransformTest/1.DeeplyNestedArithmeticLogicalExpression for reference - auto [code, is_null_aware, is_fallible, output_nullability] = + auto [code, is_null_aware, output_nullability] = converter.generate_code(target_id, expr, function_name); std::vector> inputs; std::vector> scalar_columns; @@ -937,7 +917,6 @@ transform_args ast_converter::compute_column(target target_id, .udf = std::move(code), .source_type = cudf::udf_source_type::CUDA, .is_null_aware = is_null_aware, - .is_fallible = is_fallible, .user_data = std::nullopt, .inputs = inputs, .outputs{output}, diff --git a/cpp/src/jit/row_ir.hpp b/cpp/src/jit/row_ir.hpp index 3fd10037a437..d54ac08b42cd 100644 --- a/cpp/src/jit/row_ir.hpp +++ b/cpp/src/jit/row_ir.hpp @@ -91,7 +91,6 @@ struct [[nodiscard]] transform_args { std::string udf = {}; udf_source_type source_type = cudf::udf_source_type::CUDA; null_aware is_null_aware = null_aware::NO; - fallible is_fallible = fallible::NO; std::optional user_data = std::nullopt; std::vector inputs = {}; std::vector outputs = {}; @@ -363,13 +362,6 @@ struct [[nodiscard]] node { */ [[nodiscard]] bool is_null_aware() const; - /** - * @brief Returns `true` if this node can produce an error during execution. - * e.g. `ADD_OVERFLOW` operator can produce an error if the result of the addition overflows the - * range of the data type. - */ - [[nodiscard]] bool is_fallible() const; - /** * @brief Returns `true` if this node always produces a valid output even if its inputs are * nullable, e.g., `IS_NULL` operator produces a valid boolean output regardless of the @@ -447,7 +439,7 @@ struct [[nodiscard]] ast_converter { [[nodiscard]] std::unique_ptr add_ir_node(ast::jit::detail::operation const& expr); - [[nodiscard]] std::tuple generate_code( + [[nodiscard]] std::tuple generate_code( target target, ast::expression const& expr, std::string_view function_name); /** diff --git a/cpp/src/jit/util.cpp b/cpp/src/jit/util.cpp index b5747226ec47..b19967be8270 100644 --- a/cpp/src/jit/util.cpp +++ b/cpp/src/jit/util.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,8 @@ #include #include #include +#include +#include namespace cudf { namespace jit { @@ -54,5 +56,42 @@ void const* get_data_ptr(scalar const& s) return type_dispatcher(s.type(), get_data_ptr_functor{}, s); } +data_type physical_type_of(data_type type) +{ + using enum type_id; + switch (type.id()) { + case EMPTY: return data_type{EMPTY}; + case BOOL8: + case INT8: + case UINT8: return data_type{UINT8}; + case INT16: + case UINT16: return data_type{UINT16}; + case INT32: + case UINT32: + case FLOAT32: return data_type{UINT32}; + case INT64: + case UINT64: + case FLOAT64: return data_type{UINT64}; + case TIMESTAMP_DAYS: return data_type{type_to_id()}; + case TIMESTAMP_SECONDS: return data_type{type_to_id()}; + case TIMESTAMP_MILLISECONDS: return data_type{type_to_id()}; + case TIMESTAMP_MICROSECONDS: return data_type{type_to_id()}; + case TIMESTAMP_NANOSECONDS: return data_type{type_to_id()}; + case DURATION_DAYS: return data_type{type_to_id()}; + case DURATION_SECONDS: return data_type{type_to_id()}; + case DURATION_MILLISECONDS: return data_type{type_to_id()}; + case DURATION_MICROSECONDS: return data_type{type_to_id()}; + case DURATION_NANOSECONDS: return data_type{type_to_id()}; + case DICTIONARY32: + case STRING: + case LIST: + case DECIMAL32: + case DECIMAL64: + case DECIMAL128: + case STRUCT: + default: return type; + } +} + } // namespace jit } // namespace cudf diff --git a/cpp/src/jit/util.hpp b/cpp/src/jit/util.hpp index 1e189e115cba..d2abcd48bac9 100644 --- a/cpp/src/jit/util.hpp +++ b/cpp/src/jit/util.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2023, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -22,5 +22,37 @@ void const* get_data_ptr(column_view const& view); */ void const* get_data_ptr(scalar const& s); +/** + * @brief Determine the physical type of a given element type. + * The Physical type is the type that can be used to alias the element type through a + * `cudf::column_device_view::element`. For example, the physical type of `int32`, `uint32`, + * `float`, `duration_D` is `uint32`. This is because `uint32` can be used to alias all 5 types + * through a `cudf::column_device_view::element`. + * + * It also means these types can safely alias each other across an ABI boundary as they have the + * same register storage type (PTX `b32`). + * + * e.g. + * + * ```cpp + * // PTX: .extern .func (.param .b32 func_retval0) my_udf_uint32(.param .b32 a); + * + * extern void my_udf_uint32(uint32_t a); + * extern void my_udf_int32(int32_t a); + * extern void my_udf_float(float a); + * extern void my_udf_duration_D(duration_D a); + * ``` + * + * `my_udf_int32` and `my_udf_float`, `my_udf_duration_D` can safely alias `my_udf_uint32` because + * they have the same register storage type (PTX `b32`). + * + * + * This means that some CUDA functions/kernels that are template-specialized on physical types can + * be re-used for other types that have the same physical type, thus reducing the cost of code + * specialization. + * + */ +data_type physical_type_of(data_type type); + } // namespace jit } // namespace cudf diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 019dc9757395..1ec0c3c3752b 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -56,11 +56,7 @@ extern "C" __device__ transform_type transform; } // namespace lto /// @brief The generic transform kernel. Supports all types and nullability combinations. -template +template __device__ void transform_kernel(size_type row_size, bitmask_type const* __restrict__ stencil, void* __restrict__ user_data, @@ -76,8 +72,8 @@ __device__ void transform_kernel(size_type row_size, auto operation = [&](Args args) { // TODO: static assert invocable auto func = [&](auto... a) { - if constexpr (!discard_errors) { - return GENERIC_TRANSFORM_OP(a...); + if constexpr (!cuda::std::is_void_v) { + return static_cast(GENERIC_TRANSFORM_OP(a...)); } else { (void)GENERIC_TRANSFORM_OP(a...); return errc::SUCCESS; @@ -103,13 +99,13 @@ __device__ void transform_kernel(size_type row_size, auto out_ptrs = cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - auto row_error = static_cast(operation(cuda::std::tuple_cat(out_ptrs, ins))); + auto row_error = operation(cuda::std::tuple_cat(out_ptrs, ins)); OutputAccessors::map([&]() { (A::assign(output_cols, row, cuda::std::get(outs)), ...); }); - if constexpr (!discard_errors) { thread_error = cuda::std::max(thread_error, row_error); } + thread_error = cuda::std::max(thread_error, row_error); } else { auto active_mask = __ballot_sync(__activemask(), row < row_size); @@ -123,7 +119,7 @@ __device__ void transform_kernel(size_type row_size, auto out_ptrs = cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - auto row_error = static_cast(operation(cuda::std::tuple_cat(out_ptrs, ins))); + auto row_error = operation(cuda::std::tuple_cat(out_ptrs, ins)); OutputAccessors::map([&]() { (A::assign(output_cols, row, *cuda::std::get(outs)), ...); @@ -132,14 +128,15 @@ __device__ void transform_kernel(size_type row_size, ...); }); - if constexpr (!discard_errors) { thread_error = cuda::std::max(thread_error, row_error); } + thread_error = cuda::std::max(thread_error, row_error); } } - if constexpr (!discard_errors) { - cuda::atomic_ref ref(*max_error); - ref.fetch_max(static_cast(thread_error), cuda::std::memory_order_relaxed); - } + // early exit if no error occurred + if (thread_error == errc::SUCCESS) { return; } + + cuda::atomic_ref ref(*max_error); + ref.fetch_max(static_cast(thread_error), cuda::std::memory_order_relaxed); } } // namespace jit diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index cd67701cb046..b4bfda6b8686 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -162,7 +162,6 @@ using handle = std::variant< namespace jit_transform { kernel instantiate(bool is_null_aware, - bool discard_errors, bool has_user_data, std::string const& ins, std::string const& outs, @@ -181,7 +180,6 @@ kernel instantiate(bool is_null_aware, auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", rtcx::reflect(is_null_aware), - rtcx::reflect(discard_errors), rtcx::reflect(has_user_data), ins, outs); @@ -207,18 +205,29 @@ void launch(cudf::kernel const& kernel, kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } -std::string reflect_input_element(column_view const& c) { return type_to_name(c.type()); } +std::string reflect_input_element(column_view const& c, bool use_physical_type) +{ + return type_to_name(use_physical_type ? jit::physical_type_of(c.type()) : c.type()); +} -std::string reflect_input_element(scalar_column_view const& c) { return type_to_name(c.type()); } +std::string reflect_input_element(scalar_column_view const& c, bool use_physical_type) +{ + return type_to_name(use_physical_type ? jit::physical_type_of(c.type()) : c.type()); +} -std::string reflect_output_element(fixed_width_column const& c) +std::string reflect_output_element(fixed_width_column const& c, bool use_physical_type) { - return type_to_name(c._col->type()); + return type_to_name(use_physical_type ? jit::physical_type_of(c._col->type()) : c._col->type()); } -std::string reflect_output_element(string_views_column const&) { return "cudf::string_view"; } +std::string reflect_output_element(string_views_column const&, + [[maybe_unused]] bool use_physical_type) +{ + return "cudf::string_view"; +} -std::string reflect_output_element(mutable_strings_column const&) +std::string reflect_output_element(mutable_strings_column const&, + [[maybe_unused]] bool use_physical_type) { return "cuda::std::span"; } @@ -250,11 +259,13 @@ auto reflect(std::variant source_type, std::span outputs) { std::vector in_types; + bool use_physical_types = std::holds_alternative(source_type); for (size_t i = 0; i < inputs.size(); i++) { - auto& in = inputs[i]; - auto column = std::visit([](auto& c) { return reflect_input_column(c); }, in); - auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); + auto& in = inputs[i]; + auto column = std::visit([&](auto& c) { return reflect_input_column(c); }, in); + auto element = + std::visit([&](auto& c) { return reflect_input_element(c, use_physical_types); }, in); bool as_scalar = std::holds_alternative(in); auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", rtcx::reflect(i), @@ -268,9 +279,10 @@ auto reflect(std::variant source_type, std::vector out_types; for (size_t i = 0; i < outputs.size(); i++) { - auto& out = outputs[i]; - auto column = std::visit([](auto& c) { return reflect_output_column(c); }, out); - auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); + auto& out = outputs[i]; + auto column = std::visit([&](auto& c) { return reflect_output_column(c); }, out); + auto element = + std::visit([&](auto& c) { return reflect_output_element(c, use_physical_types); }, out); bool as_scalar = false; // never scalar auto accessor = rtcx::reflect_template("cudf::jit::column_accessor", rtcx::reflect(i), @@ -291,11 +303,13 @@ auto reflect(std::variant source_type, if (std::holds_alternative(source_type) && std::get(source_type) == udf_source_type::PTX) { for (auto& in : inputs) { - ptx_in_types.push_back(std::visit([](auto& c) { return reflect_input_element(c); }, in)); + ptx_in_types.push_back( + std::visit([&](auto& c) { return reflect_input_element(c, use_physical_types); }, in)); } for (auto& out : outputs) { - ptx_out_types.push_back(std::visit([](auto& c) { return reflect_output_element(c); }, out)); + ptx_out_types.push_back( + std::visit([&](auto& c) { return reflect_output_element(c, use_physical_types); }, out)); } } @@ -305,21 +319,24 @@ auto reflect(std::variant source_type, std::string reflect_udf_signature(bool is_null_aware, bool has_user_data, std::span inputs, - std::span outputs) + std::span outputs, + bool use_physical_types) { std::vector in_types; for (size_t i = 0; i < inputs.size(); i++) { - auto& in = inputs[i]; - auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); + auto& in = inputs[i]; + auto element = + std::visit([&](auto& c) { return reflect_input_element(c, use_physical_types); }, in); in_types.push_back(is_null_aware ? std::format("cuda::std::optional<{}>", element) : element); } std::vector out_types; for (size_t i = 0; i < outputs.size(); i++) { - auto& out = outputs[i]; - auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); + auto& out = outputs[i]; + auto element = + std::visit([&](auto& c) { return reflect_output_element(c, use_physical_types); }, out); out_types.push_back(is_null_aware ? std::format("cuda::std::optional<{}> *", element) : std::format("{} *", element)); } @@ -339,10 +356,8 @@ std::string reflect_udf_signature(bool is_null_aware, return std::format("int({})", joined); } -// determine CUDF_KERNEL_INSTANCE and CUDF_UDF_TYPE std::tuple instantiate_fragment( bool is_null_aware, - bool discard_errors, bool has_user_data, std::string const& ins, std::string const& outs, @@ -350,13 +365,16 @@ std::tuple instantiate_fragment( std::span outputs) { CUDF_FUNC_RANGE(); - auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", + // substitutes the CUDF_KERNEL_INSTANCE macro + auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", rtcx::reflect(is_null_aware), - rtcx::reflect(discard_errors), rtcx::reflect(has_user_data), ins, outs); - auto signature = reflect_udf_signature(is_null_aware, has_user_data, inputs, outputs); + + // substitutes the CUDF_UDF_TYPE macro + auto signature = reflect_udf_signature( + is_null_aware, has_user_data, inputs, outputs, /*use_physical_types=*/true); return {jit::get_udf_kernel_fragment("cudf/cpp/src/transform/jit/kernel.cu", kernel, signature), lto_binary_type::LTO_IR, kernel}; @@ -401,7 +419,6 @@ auto to_args(std::span inputs, } void run(bool is_null_aware, - bool discard_errors, bool has_user_data, size_type row_size, bitmask_type const* d_stencil, @@ -416,7 +433,6 @@ void run(bool is_null_aware, { auto [in_types, out_types, ptx_in_types, ptx_out_types] = reflect(source_type, inputs, outputs); auto kernel = instantiate(is_null_aware, - discard_errors, has_user_data, in_types, out_types, @@ -447,7 +463,6 @@ rtcx::binary_type as_rtcx_binary_type(lto_binary_type type) void run_lto(std::optional, lto_binary_type, char const*>> precompiled_kernel_fragment, bool is_null_aware, - bool discard_errors, bool has_user_data, size_type row_size, bitmask_type const* d_stencil, @@ -471,8 +486,8 @@ void run_lto(std::optional, lto_binary_type, std::tie(kernel_fragment, kernel_fragment_binary_type, kernel_fragment_id) = *precompiled_kernel_fragment; } else { - std::tie(fragment_blob, kernel_fragment_binary_type, kernel_fragment_id) = instantiate_fragment( - is_null_aware, discard_errors, has_user_data, in_types, out_types, inputs, outputs); + std::tie(fragment_blob, kernel_fragment_binary_type, kernel_fragment_id) = + instantiate_fragment(is_null_aware, has_user_data, in_types, out_types, inputs, outputs); kernel_fragment = fragment_blob->view(); } @@ -950,7 +965,6 @@ auto finalize_outputs(null_aware is_null_aware, std::unique_ptr
execute_transform(std::string const& udf, udf_source_type source_type, null_aware is_null_aware, - fallible is_fallible, std::optional in_row_size, std::optional user_data, std::span inputs, @@ -976,7 +990,6 @@ std::unique_ptr
execute_transform(std::string const& udf, rmm::device_scalar d_max_error(static_cast(errc::SUCCESS), stream, mr); jit_transform::run(is_null_aware == null_aware::YES, - is_fallible == fallible::NO, user_data.has_value(), row_size, stencil_has_nulls ? stencil_arg : nullptr, @@ -1007,7 +1020,6 @@ std::unique_ptr
execute_transform(std::string const& udf, std::unique_ptr
multi_transform(std::string const& udf, udf_source_type source_type, null_aware is_null_aware, - fallible is_fallible, std::optional user_data, std::span inputs, std::span outputs, @@ -1021,7 +1033,6 @@ std::unique_ptr
multi_transform(std::string const& udf, return execute_transform(udf, source_type, is_null_aware, - is_fallible, row_size, user_data, inputs, @@ -1031,30 +1042,6 @@ std::unique_ptr
multi_transform(std::string const& udf, mr); } -std::unique_ptr
multi_transform(std::string const& udf, - udf_source_type source_type, - null_aware is_null_aware, - std::optional user_data, - std::span inputs, - std::span outputs, - std::vector>&& string_offsets, - std::optional row_size, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) -{ - return multi_transform(udf, - source_type, - is_null_aware, - fallible::NO, - user_data, - inputs, - outputs, - std::move(string_offsets), - row_size, - stream, - mr); -} - std::unique_ptr transform_extended(std::span inputs, std::string const& udf, data_type output_type, @@ -1068,17 +1055,8 @@ std::unique_ptr transform_extended(std::span inpu { transform_output outputs[] = {{.type = output_type, .nullability = null_policy}}; - auto table = multi_transform(udf, - source_type, - is_null_aware, - fallible::NO, - user_data, - inputs, - outputs, - {}, - row_size, - stream, - mr); + auto table = multi_transform( + udf, source_type, is_null_aware, user_data, inputs, outputs, {}, row_size, stream, mr); auto cols = table->release(); return std::move(cols[0]); @@ -1131,7 +1109,6 @@ std::unique_ptr compute_column_jit(table_view const& table, auto result = multi_transform(args.udf, args.source_type, args.is_null_aware, - args.is_fallible, args.user_data, args.inputs, args.outputs, @@ -1147,7 +1124,6 @@ std::unique_ptr compute_column_jit(table_view const& table, // it to use for LTO linking instead of compiling a new one std::optional, lto_binary_type, char const*>> dispatch_lto_kernel_fragment(bool is_null_aware, - bool discard_errors, bool has_user_data, std::span inputs, std::span outputs) @@ -1169,7 +1145,6 @@ dispatch_lto_kernel_fragment(bool is_null_aware, jit_transform::reflect(lto_binary_type::FATBIN, inputs, outputs); auto target = strip_whitespace(rtcx::reflect_template("cudf::jit::transform_kernel", rtcx::reflect(is_null_aware), - rtcx::reflect(discard_errors), rtcx::reflect(has_user_data), in_types, out_types)); @@ -1190,7 +1165,6 @@ dispatch_lto_kernel_fragment(bool is_null_aware, std::unique_ptr
transform_lto(std::span udf, lto_binary_type binary_type, null_aware is_null_aware, - fallible is_fallible, std::optional user_data, std::span inputs, std::span outputs, @@ -1215,17 +1189,13 @@ std::unique_ptr
transform_lto(std::span udf, auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; - auto precompiled_kernel_fragment = dispatch_lto_kernel_fragment(is_null_aware == null_aware::YES, - is_fallible == fallible::NO, - user_data.has_value(), - inputs, - output_columns); + auto precompiled_kernel_fragment = dispatch_lto_kernel_fragment( + is_null_aware == null_aware::YES, user_data.has_value(), inputs, output_columns); rmm::device_scalar d_max_error(static_cast(errc::SUCCESS), stream, mr); jit_transform::run_lto(precompiled_kernel_fragment, is_null_aware == null_aware::YES, - is_fallible == fallible::NO, user_data.has_value(), row_size, stencil_has_nulls ? stencil_arg : nullptr, diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index 0a4d6bf501bd..639edbef226d 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -1213,7 +1213,6 @@ __device__ cudf::errc expression ( EXPECT_NO_THROW(result = cudf::multi_transform(cuda, cudf::udf_source_type::CUDA, cudf::null_aware::NO, - cudf::fallible::YES, std::nullopt, inputs, outputs, @@ -1231,7 +1230,6 @@ __device__ cudf::errc expression ( EXPECT_THROW(result = cudf::multi_transform(cuda, cudf::udf_source_type::CUDA, cudf::null_aware::NO, - cudf::fallible::YES, std::nullopt, inputs, outputs, diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp index aea0175852e0..6a9e6d8f5ef3 100644 --- a/cpp/tests/transform/transform_lto_test.cpp +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -37,7 +37,6 @@ TEST_F(TransformLTOTest, InvSqrt) auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, - cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -64,7 +63,6 @@ TEST_F(TransformLTOTest, ToUpper) auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, - cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -92,7 +90,6 @@ TEST_F(TransformLTOTest, SumOfSquares) auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, - cudf::fallible::NO, std::nullopt, inputs, outputs, @@ -105,7 +102,7 @@ TEST_F(TransformLTOTest, SumOfSquares) CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); } -TEST_F(TransformLTOTest, LehmerMean) +TEST_F(TransformLTOTest, FallibleIntegerLehmerMean) { // computes integer lehmer mean: `(a^2 + b^2) / (a + b)` for each row using checked arithmetic and // throws if an overflow occurs @@ -125,7 +122,6 @@ TEST_F(TransformLTOTest, LehmerMean) auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, - cudf::fallible::YES, std::nullopt, inputs, outputs, @@ -147,7 +143,6 @@ TEST_F(TransformLTOTest, LehmerMean) EXPECT_THROW(cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, - cudf::fallible::YES, std::nullopt, inputs_fail, outputs, @@ -176,7 +171,6 @@ TEST_F(TransformLTOTest, BankersRounding) auto result = cudf::transform_lto(udf, cudf::lto_binary_type::FATBIN, cudf::null_aware::NO, - cudf::fallible::NO, std::nullopt, inputs, outputs, From 7aa8dc099502874586edba6a29a2e5c5190eb06f Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 3 Jul 2026 11:36:06 +0000 Subject: [PATCH 251/254] Add distance transform fragment and corresponding test case --- cpp/CMakeLists.txt | 16 ++------- cpp/src/transform/transform.cu | 37 +++++++++++---------- cpp/tests/CMakeLists.txt | 2 ++ cpp/tests/transform/fragments/distance.cu | 16 +++++++++ cpp/tests/transform/transform_lto_test.cpp | 38 ++++++++++++++++++++++ 5 files changed, 77 insertions(+), 32 deletions(-) create mode 100644 cpp/tests/transform/fragments/distance.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9f58f79e9e80..b715d2203a1a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -504,20 +504,8 @@ rtcx_embed( rtcx_add_embed(cudf_fragments) -list( - APPEND - CUDF_PRECOMPILE_PHYSICAL_TYPES - uint8_t - uint16_t - uint32_t - uint64_t - numeric::decimal32 - numeric::decimal64 - numeric::decimal128 - # cudf::dictionary_element cudf::dictionary_element - # cudf::dictionary_element cudf::dictionary_element - # cudf::dictionary_element cudf::dictionary_element cudf::dictionary_element +list(APPEND CUDF_PRECOMPILE_PHYSICAL_TYPES uint8_t uint16_t uint32_t uint64_t numeric::decimal32 + numeric::decimal64 numeric::decimal128 ) foreach(TYPE IN ITEMS ${CUDF_PRECOMPILE_PHYSICAL_TYPES}) diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 15bdcbdf6a5e..a34b8389a774 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -209,7 +209,6 @@ void launch(cudf::kernel const& kernel, kernel.launch({cfg.min_grid_size}, {cfg.block_size}, 0, stream, args); } - std::string get_element_type_name(column_view const& view, bool use_physical_type); struct element_type_name_fn { @@ -224,9 +223,10 @@ struct element_type_name_fn { std::string operator()(column_view const& view, bool use_physical_type) const requires(std::same_as) { - return std::format("cudf::dictionary_element<{}, {}>", - get_element_type_name(view.child(cudf::dictionary_indices_column_index), use_physical_type), - get_element_type_name(view.child(cudf::dictionary_keys_column_index), use_physical_type)); + return std::format( + "cudf::dictionary_element<{}, {}>", + get_element_type_name(view.child(cudf::dictionary_indices_column_index), use_physical_type), + get_element_type_name(view.child(cudf::dictionary_keys_column_index), use_physical_type)); } template @@ -243,11 +243,10 @@ std::string get_element_type_name(column_view const& view, bool use_physical_typ return cudf::type_dispatcher(view.type(), element_type_name_fn{}, view, use_physical_type); } - - - - -std::string reflect_input_element(column_view const& c, bool use_physical_type) { return get_element_type_name(c, use_physical_type); } +std::string reflect_input_element(column_view const& c, bool use_physical_type) +{ + return get_element_type_name(c, use_physical_type); +} std::string reflect_input_element(scalar_column_view const& c, bool use_physical_type) { @@ -256,10 +255,9 @@ std::string reflect_input_element(scalar_column_view const& c, bool use_physical std::string reflect_output_element(fixed_width_column const& c, bool use_physical_type) { - return get_element_type_name(c._col->type(), use_physical_type); + return get_element_type_name(c._col->view(), use_physical_type); } - std::string reflect_output_element(string_views_column const&, [[maybe_unused]] bool use_physical_type) { @@ -274,9 +272,9 @@ std::string reflect_output_element(mutable_strings_column const&, std::string reflect_input_value_type(column_view const& c, bool use_physical_type) { - return is_dictionary(c.type()) - ? reflect_input_value_type(c.child(cudf::dictionary_keys_column_index), use_physical_type) - : reflect_input_element(c, use_physical_type); + return is_dictionary(c.type()) ? reflect_input_value_type( + c.child(cudf::dictionary_keys_column_index), use_physical_type) + : reflect_input_element(c, use_physical_type); } std::string reflect_input_value_type(scalar_column_view const& c, bool use_physical_type) @@ -370,7 +368,8 @@ auto reflect(std::variant source_type, if (std::holds_alternative(source_type) && std::get(source_type) == udf_source_type::PTX) { for (auto& in : inputs) { - ptx_in_types.push_back(std::visit([&](auto& c) { return reflect_input_value_type(c, use_physical_types); }, in)); + ptx_in_types.push_back( + std::visit([&](auto& c) { return reflect_input_value_type(c, use_physical_types); }, in)); } for (auto& out : outputs) { @@ -431,16 +430,17 @@ std::tuple instantiate_fragment( std::span outputs) { CUDF_FUNC_RANGE(); - // substitutes the CUDF_KERNEL_INSTANCE macro + // substitutes the `CUDF_KERNEL_INSTANCE` macro auto kernel = rtcx::reflect_template("cudf::jit::transform_kernel", rtcx::reflect(is_null_aware), rtcx::reflect(has_user_data), ins, outs); - // substitutes the CUDF_UDF_TYPE macro + // substitutes the `CUDF_UDF_TYPE` macro auto signature = reflect_udf_signature( is_null_aware, has_user_data, inputs, outputs, /*use_physical_types=*/true); + return {jit::get_udf_kernel_fragment("cudf/cpp/src/transform/jit/kernel.cu", kernel, signature), lto_binary_type::LTO_IR, kernel}; @@ -729,7 +729,7 @@ void perform_checks(std::variant source_type, std::span outputs, std::span const> string_offsets) { - if (uto* udf_source = std::get_if(&source_type); + if (auto* udf_source = std::get_if(&source_type); udf_source != nullptr && *udf_source == udf_source_type::PTX) { static constexpr auto is_input_value_supported = [](auto const& c) { return is_integral(c.type()) || is_floating_point(c.type()); @@ -1223,6 +1223,7 @@ dispatch_lto_kernel_fragment(bool is_null_aware, rtcx::reflect(has_user_data), in_types, out_types)); + for (size_t i = 0; i < std::size(cudf_fragments::transform_kernel_FILE_INDEX); i++) { auto file_index = cudf_fragments::transform_kernel_FILE_INDEX[i]; auto instance = strip_whitespace(cudf_fragments::transform_kernel_INSTANCE[i]); diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index c18bedc8de53..ceb308590ee6 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -699,6 +699,8 @@ add_fragment( cudf_test_fragments FRAGMENT bankers_rounding SOURCE transform/fragments/bankers_rounding.cu ) +add_fragment(cudf_test_fragments FRAGMENT distance SOURCE transform/fragments/distance.cu) + add_fragment(cudf_test_fragments FRAGMENT invsqrt SOURCE transform/fragments/invsqrt.cu) add_fragment(cudf_test_fragments FRAGMENT lehmer_mean SOURCE transform/fragments/lehmer_mean.cu) diff --git a/cpp/tests/transform/fragments/distance.cu b/cpp/tests/transform/fragments/distance.cu new file mode 100644 index 000000000000..fee1bded29d9 --- /dev/null +++ b/cpp/tests/transform/fragments/distance.cu @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include + +__device__ float distance(float x1, float y1, float x2, float y2) +{ + return cuda::std::sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1)); +} + +extern "C" __device__ int transform(float* out, float x1, float y1, float x2, float y2) +{ + *out = distance(x1, y1, x2, y2); + return 0; +} diff --git a/cpp/tests/transform/transform_lto_test.cpp b/cpp/tests/transform/transform_lto_test.cpp index 6a9e6d8f5ef3..194778804cb2 100644 --- a/cpp/tests/transform/transform_lto_test.cpp +++ b/cpp/tests/transform/transform_lto_test.cpp @@ -49,6 +49,44 @@ TEST_F(TransformLTOTest, InvSqrt) CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); } +TEST_F(TransformLTOTest, Distance) +{ + column_wrapper x1{{0, 0, 0, 0}}; + column_wrapper y1{{0.0F, 1.2F, 2.5F, 3.7F}}; + column_wrapper x2{{1.6F, 2.1F, 3.2f, 4.5f}}; + column_wrapper y2{{0, 0, 0, 0}}; + + cudf::transform_input inputs[] = {x1, y1, x2, y2}; + cudf::transform_output outputs[] = { + {cudf::data_type{cudf::type_id::FLOAT32}, cudf::output_nullability::ALL_VALID}}; + + auto const range = cudf_test_fragments::file_ranges[cudf_test_fragments::distance]; + std::span udf{cudf_test_fragments::files.subspan(range[0], range[1])}; + + auto result = cudf::transform_lto(udf, + cudf::lto_binary_type::FATBIN, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt, + cudf::test::get_default_stream()); + + auto distance = [](float x1, float y1, float x2, float y2) { + return std::sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1)); + }; + + column_wrapper expected{{ + distance(0, 0, 1.6F, 0), + distance(0, 1.2F, 2.1F, 0), + distance(0, 2.5F, 3.2F, 0), + distance(0, 3.7F, 4.5F, 0), + }}; + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); +} + TEST_F(TransformLTOTest, ToUpper) { column_wrapper input{{65, 66, 97, 98, 48, 49, 32, 33, 127, 255}}; From a2034e2c21c4762d0e70e79dee988a11d48b7fb7 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 3 Jul 2026 11:47:36 +0000 Subject: [PATCH 252/254] Update transform function documentation for clarity and accuracy --- cpp/include/cudf/transform.hpp | 25 +++++++++++++------------ cpp/src/jit/util.hpp | 8 ++++---- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index 7803c01da32f..b5519246a6ad 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -196,11 +196,11 @@ enum class lto_binary_type : uint8_t { }; /** - * @brief Creates a new column by applying a transform function against every + * @brief Creates a new table by applying a transform function against every * element of the input columns. * * Computes: - * `(output[i]) = UDF(input[i])`. + * `(outputs[i]...) = UDF(inputs[i]...)`. * * * @throws std::invalid_argument if any of the input columns have different sizes (except scalars) @@ -208,28 +208,29 @@ enum class lto_binary_type : uint8_t { * types * @throws std::invalid_argument if the inputs only have a scalar with no column inputs and * `row_size` is not provided. This is because the row size cannot be inferred from the inputs in - * this case. + * this case * @throws std::invalid_argument if string offsets are provided for non-string output columns, or - * if the number of string offsets does not match the number of output columns. + * if the number of string offsets does not match the number of output columns + * @throws cudf::evaluation_error if the UDF produces an error during execution * * The size of the resulting column is the `row_size` if provided, otherwise it is inferred from * the input and pre-allocated output columns. * - * @param udf The LTO-IR string of the transform function to apply. The UDF must be named - * `transform` and follow the CUDF UDF ABI. + * @param udf The LTO-IR fragment containing the transform function to apply. The UDF must + * be named `transform` and follow the CUDF UDF ABI * @param binary_type The type of the LTO binary provided in `udf` * @param is_null_aware Signifies the UDF will receive row inputs as optional values - * @param user_data User-defined device data to pass to the UDF. - * @param inputs Immutable view of the input to transform + * @param user_data User-defined device data to pass to the UDF + * @param inputs Immutable views of the inputs to transform * @param outputs Specification of the output columns to be created * @param string_offsets For string output columns, the offsets can be pre-allocated and passed in * to prevent overhead of compacting string views into run-end strings column. - * @param row_size The row size of the transform operation. If not provided, it is inferred - * from the input columns. + * @param row_size The row size of the transform operation. If not provided, it is inferred from the + * input columns * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory - * @return A table resulting from applying the transform function to every element of - * the input according to the output specifications + * @return A table containing the columns resulting from applying the transform + * function to every element of the input according to the output specifications * */ std::unique_ptr
transform_lto( diff --git a/cpp/src/jit/util.hpp b/cpp/src/jit/util.hpp index d2abcd48bac9..3220a2286a48 100644 --- a/cpp/src/jit/util.hpp +++ b/cpp/src/jit/util.hpp @@ -35,7 +35,7 @@ void const* get_data_ptr(scalar const& s); * e.g. * * ```cpp - * // PTX: .extern .func (.param .b32 func_retval0) my_udf_uint32(.param .b32 a); + * // PTX: .extern .func (.param .b32 func_retval0) my_udf(.param .b32 a); * * extern void my_udf_uint32(uint32_t a); * extern void my_udf_int32(int32_t a); @@ -43,13 +43,13 @@ void const* get_data_ptr(scalar const& s); * extern void my_udf_duration_D(duration_D a); * ``` * - * `my_udf_int32` and `my_udf_float`, `my_udf_duration_D` can safely alias `my_udf_uint32` because + * `my_udf_int32` and `my_udf_float`, `my_udf_duration_D` can safely alias `my_udf` because * they have the same register storage type (PTX `b32`). * * * This means that some CUDA functions/kernels that are template-specialized on physical types can - * be re-used for other types that have the same physical type, thus reducing the cost of code - * specialization. + * be re-used for other types that have the same physical type, thus reducing the compilation cost + * of code specialization. * */ data_type physical_type_of(data_type type); From 5a8580253564ef01327d4c8fc25f30881b6331a5 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 3 Jul 2026 17:30:52 +0000 Subject: [PATCH 253/254] Add NVRTC and NVJITLINK version retrieval functions and update context to store versions --- cpp/librtcx/rtcx.cpp | 20 ++++++++++++++++++++ cpp/librtcx/rtcx.hpp | 14 ++++++++++++++ cpp/src/jit/cache.cpp | 19 ++++++++++--------- cpp/src/runtime/context.cpp | 11 ++++++++++- cpp/src/runtime/context.hpp | 9 ++++++++- 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp index a0d057ed4c90..1c3144869a8b 100644 --- a/cpp/librtcx/rtcx.cpp +++ b/cpp/librtcx/rtcx.cpp @@ -183,6 +183,7 @@ void log_error(std::string_view msg) DO_IT(LibraryUnload) #define FOR_EACH_NVRTC_FUNC(DO_IT) \ + DO_IT(Version) \ DO_IT(GetErrorString) \ DO_IT(CreateProgram) \ DO_IT(DestroyProgram) \ @@ -199,6 +200,7 @@ void log_error(std::string_view msg) DO_IT(GetLoweredName) #define FOR_EACH_NVJITLINK_FUNC(DO_IT) \ + DO_IT(Version) \ DO_IT(Create) \ DO_IT(Destroy) \ DO_IT(AddData) \ @@ -658,6 +660,24 @@ void log_nvJitLink_result(link_params const& params, } // namespace +std::int32_t nvrtc_version() +{ + RTCX_FUNC_RANGE(); + + std::int32_t major, minor; + RTCX_CHECK_NVRTC(nvrtc->Version(&major, &minor)); + return major * 1000 + minor * 10; +} + +std::int32_t nvjitlink_version() +{ + RTCX_FUNC_RANGE(); + + std::uint32_t major, minor; + RTCX_CHECK_NVJITLINK(nvjitlink->Version(&major, &minor)); + return static_cast(major * 1000 + minor * 10); +} + byte_buffer compile(compile_params const& params) { RTCX_FUNC_RANGE(); diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp index 256f52fbe26d..3a9929c20301 100644 --- a/cpp/librtcx/rtcx.hpp +++ b/cpp/librtcx/rtcx.hpp @@ -725,6 +725,20 @@ struct cache_t { // NOLINT [[nodiscard]] bool is_enabled(); }; +/** + * @brief Get the version of the NVRTC library + * @return An integer representing the NVRTC version. With the encoding major * 1000 + minor * 10 + + * patch + */ +[[nodiscard]] std::int32_t nvrtc_version(); + +/** + * @brief Get the version of the NVJITLINK library + * @return An integer representing the NVJITLINK version. With the encoding major * 1000 + minor * + * 10 + patch + */ +[[nodiscard]] std::int32_t nvjitlink_version(); + /** * @brief Compile source code into a binary blob * diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index a0e2775c2548..186e9b3ad945 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -192,14 +192,14 @@ std::vector jit_bundle_t::get_include_directories() const namespace { -constexpr int make_cuda_version(int major, int minor, int patch) +constexpr int32_t make_cuda_version(int32_t major, int32_t minor, int32_t patch) { return major * 1000 + minor * 10 + patch; } -constexpr int MIN_CUDA_VERSION_PCH = +constexpr int32_t MIN_NVRTC_VERSION_PCH = make_cuda_version(12, 8, 0); // minimum CUDA version for the "--pch" NVRTC flag -constexpr int MIN_CUDA_VERSION_MINIMAL = +constexpr int32_t MIN_NVRTC_VERSION_MINIMAL = make_cuda_version(12, 8, 0); // minimum CUDA version for the "--minimal" NVRTC flag std::tuple compile_library( @@ -216,11 +216,11 @@ std::tuple compile_library( auto& bundle = ctx.jit_bundle(); auto& device_properties = ctx.get_device_properties(); auto sm = device_properties.compute_capability; - auto runtime = device_properties.runtime_version; + auto nvrtc_version = ctx.nvrtc_version(); auto include_dirs = bundle.get_include_directories(); - auto use_pch = runtime >= MIN_CUDA_VERSION_PCH; - auto use_minimal = runtime >= MIN_CUDA_VERSION_MINIMAL; + auto use_pch = nvrtc_version >= MIN_NVRTC_VERSION_PCH; + auto use_minimal = nvrtc_version >= MIN_NVRTC_VERSION_MINIMAL; std::vector options; @@ -298,13 +298,14 @@ rtcx::blob compile_fragment(char const* name, auto& bundle = ctx.jit_bundle(); auto& device_properties = ctx.get_device_properties(); auto sm = device_properties.compute_capability; - auto runtime = device_properties.runtime_version; auto include_dirs = bundle.get_include_directories(); auto pch_dir = ctx.get_jit_pch_dir(); - auto use_pch = runtime >= MIN_CUDA_VERSION_PCH; - auto use_minimal = runtime >= MIN_CUDA_VERSION_MINIMAL; + auto nvrtc_version = ctx.nvrtc_version(); + + auto use_pch = nvrtc_version >= MIN_NVRTC_VERSION_PCH; + auto use_minimal = nvrtc_version >= MIN_NVRTC_VERSION_MINIMAL; std::vector options; diff --git a/cpp/src/runtime/context.cpp b/cpp/src/runtime/context.cpp index 74e3ca6af31b..7c4ffaf192ec 100644 --- a/cpp/src/runtime/context.cpp +++ b/cpp/src/runtime/context.cpp @@ -53,7 +53,9 @@ context::context(context_config cfg, init_flags flags) : _config{std::move(cfg)}, _jit_cache_init_flag{}, _device_properties{ - get_driver_version(), get_runtime_version(), get_current_device_compute_capability()} + get_driver_version(), get_runtime_version(), get_current_device_compute_capability()}, + _nvrtc_version{0}, + _nvjitlink_version{0} { initialize_components(flags); } @@ -71,6 +73,9 @@ void context::ensure_jit_cache_initialized() rtcx::initialize(); + _nvrtc_version = rtcx::nvrtc_version(); + _nvjitlink_version = rtcx::nvjitlink_version(); + auto limits = rtcx::cache_limits{.num_mem_blobs = _config.kernel_cache_limit_process, .num_mem_libraries = _config.kernel_cache_limit_process}; @@ -122,6 +127,10 @@ context::device_properties const& context::get_device_properties() const return _device_properties; } +std::optional context::nvrtc_version() const { return _nvrtc_version; } + +std::optional context::nvjitlink_version() const { return _nvjitlink_version; } + void context::initialize_components(init_flags flags) { if (has_flag(flags, init_flags::INIT_JIT_CACHE)) { ensure_jit_cache_initialized(); } diff --git a/cpp/src/runtime/context.hpp b/cpp/src/runtime/context.hpp index 7fdb553dee3e..98baeb586385 100644 --- a/cpp/src/runtime/context.hpp +++ b/cpp/src/runtime/context.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include +#include namespace rtcx { struct cache_t; @@ -57,6 +58,8 @@ class context { std::unique_ptr _rtcx_cache; std::unique_ptr _jit_bundle; device_properties _device_properties; + std::optional _nvrtc_version; + std::optional _nvjitlink_version; private: void ensure_nvcomp_loaded(); @@ -85,6 +88,10 @@ class context { [[nodiscard]] device_properties const& get_device_properties() const; + [[nodiscard]] std::optional nvrtc_version() const; + + [[nodiscard]] std::optional nvjitlink_version() const; + /// @brief Initialize additional components based on the provided flags /// @param flags The initialization flags to process void initialize_components(init_flags flags); From f7d2883b34c487d5efc4b8e4150f3adbc401dee8 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 3 Jul 2026 17:43:48 +0000 Subject: [PATCH 254/254] Fix nvrtc_version retrieval to ensure proper value extraction in compile_library and compile_fragment functions --- cpp/src/jit/cache.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 186e9b3ad945..8a33e807be2c 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -216,7 +216,7 @@ std::tuple compile_library( auto& bundle = ctx.jit_bundle(); auto& device_properties = ctx.get_device_properties(); auto sm = device_properties.compute_capability; - auto nvrtc_version = ctx.nvrtc_version(); + auto nvrtc_version = ctx.nvrtc_version().value(); auto include_dirs = bundle.get_include_directories(); auto use_pch = nvrtc_version >= MIN_NVRTC_VERSION_PCH; @@ -302,7 +302,7 @@ rtcx::blob compile_fragment(char const* name, auto include_dirs = bundle.get_include_directories(); auto pch_dir = ctx.get_jit_pch_dir(); - auto nvrtc_version = ctx.nvrtc_version(); + auto nvrtc_version = ctx.nvrtc_version().value(); auto use_pch = nvrtc_version >= MIN_NVRTC_VERSION_PCH; auto use_minimal = nvrtc_version >= MIN_NVRTC_VERSION_MINIMAL;