From 0764ae6085851a715a038e10b50f8e1879417614 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 Jul 2026 12:07:07 -0700 Subject: [PATCH 01/11] feat: fetch librtcx via CPM instead of add_subdirectory --- cpp/CMakeLists.txt | 16 ++++------------ cpp/cmake/thirdparty/get_rtcx.cmake | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 cpp/cmake/thirdparty/get_rtcx.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b715d2203a1a..eacf0eb2d204 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -419,11 +419,8 @@ include(cmake/thirdparty/get_xxhash.cmake) # find zstd include(cmake/thirdparty/get_zstd.cmake) -# Build librtcx as a subdirectory (zstd and nvtx3 are already available at this point) -add_subdirectory(librtcx) - -# JIT Embedding helper functions -include(librtcx/embed.cmake) +# Fetch librtcx via CPM (zstd and nvtx3 are already available at this point) +include(cmake/thirdparty/get_rtcx.cmake) # Pre-compiled fragment management helper functions include(cmake/Modules/AddFragment.cmake) @@ -443,18 +440,13 @@ if(NOT BUILD_SHARED_LIBS) if(TARGET conda_env) install(TARGETS conda_env EXPORT cudf-exports) endif() - install( - TARGETS rtcx - EXPORT cudf-exports - DESTINATION ${lib_dir} - ) endif() rtcx_add_embed(cudf_cuda_embed) rtcx_embed_includes( - cudf_cuda_embed SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/librtcx/libcxx DEST_DIRECTORY - librtcx/libcxx INCLUDE_DIRECTORIES librtcx/libcxx + cudf_cuda_embed SOURCE_DIRECTORY ${rtcx_SOURCE_DIR}/libcxx DEST_DIRECTORY librtcx/libcxx + INCLUDE_DIRECTORIES librtcx/libcxx ) rtcx_embed_includes( diff --git a/cpp/cmake/thirdparty/get_rtcx.cmake b/cpp/cmake/thirdparty/get_rtcx.cmake new file mode 100644 index 000000000000..4e091531eac3 --- /dev/null +++ b/cpp/cmake/thirdparty/get_rtcx.cmake @@ -0,0 +1,28 @@ +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on + +# This function finds rtcx +function(find_and_configure_rtcx VERSION) + + rapids_cpm_find( + rtcx ${VERSION} + GLOBAL_TARGETS rtcx::rtcx + CPM_ARGS + GIT_REPOSITORY https://github.com/rapidsai/librtcx.git + GIT_TAG "${RAPIDS_BRANCH}" + GIT_SHALLOW TRUE + EXCLUDE_FROM_ALL ${CUDF_EXCLUDE_DEPS_FROM_ALL} + ) + + # When CPM fetches from source (add_subdirectory), embed.cmake is not auto-included. Include it + # explicitly so add_embed/embed_includes/embed functions are available. + if(rtcx_ADDED OR DEFINED CPM_rtcx_SOURCE) + include("${rtcx_SOURCE_DIR}/embed.cmake") + endif() + +endfunction() + +set(RTCX_MIN_VERSION_cudf "0.1") +find_and_configure_rtcx(${RTCX_MIN_VERSION_cudf}) From 185d02f3051d06cb8c1ebf255f1c58961752a723 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 Jul 2026 17:22:12 -0700 Subject: [PATCH 02/11] chore: remove bundled librtcx sources --- cpp/librtcx/CMakeLists.txt | 166 --- cpp/librtcx/CONTRIBUTING.md | 58 - cpp/librtcx/LICENSE | 201 ---- cpp/librtcx/RAPIDS_BRANCH | 1 - cpp/librtcx/README.md | 22 - cpp/librtcx/VERSION | 1 - cpp/librtcx/cmake/RAPIDS.cmake | 83 -- cpp/librtcx/cmake/rapids_config.cmake | 38 - cpp/librtcx/embed.cmake | 349 ------ cpp/librtcx/embed.hpp | 410 -------- cpp/librtcx/embed.in.cpp | 30 - cpp/librtcx/hash.cpp | 70 -- cpp/librtcx/hash.hpp | 62 -- cpp/librtcx/libcxx/cassert | 8 - cpp/librtcx/libcxx/climits | 8 - cpp/librtcx/libcxx/cstddef | 18 - cpp/librtcx/libcxx/cstdint | 45 - cpp/librtcx/rtcx.cpp | 1399 ------------------------- cpp/librtcx/rtcx.hpp | 1007 ------------------ 19 files changed, 3976 deletions(-) delete mode 100644 cpp/librtcx/CMakeLists.txt delete mode 100644 cpp/librtcx/CONTRIBUTING.md delete mode 100644 cpp/librtcx/LICENSE delete mode 120000 cpp/librtcx/RAPIDS_BRANCH delete mode 100644 cpp/librtcx/README.md delete mode 120000 cpp/librtcx/VERSION delete mode 100644 cpp/librtcx/cmake/RAPIDS.cmake delete mode 100644 cpp/librtcx/cmake/rapids_config.cmake delete mode 100644 cpp/librtcx/embed.cmake delete mode 100644 cpp/librtcx/embed.hpp delete mode 100644 cpp/librtcx/embed.in.cpp delete mode 100644 cpp/librtcx/hash.cpp delete mode 100644 cpp/librtcx/hash.hpp delete mode 100644 cpp/librtcx/libcxx/cassert delete mode 100644 cpp/librtcx/libcxx/climits delete mode 100644 cpp/librtcx/libcxx/cstddef delete mode 100644 cpp/librtcx/libcxx/cstdint delete mode 100644 cpp/librtcx/rtcx.cpp delete mode 100644 cpp/librtcx/rtcx.hpp diff --git a/cpp/librtcx/CMakeLists.txt b/cpp/librtcx/CMakeLists.txt deleted file mode 100644 index a40bb36d2914..000000000000 --- a/cpp/librtcx/CMakeLists.txt +++ /dev/null @@ -1,166 +0,0 @@ -# ============================================================================= -# cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# cmake-format: on -# ============================================================================= - -cmake_minimum_required(VERSION 4.0 FATAL_ERROR) - -if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) - include(cmake/rapids_config.cmake) - include(rapids-cmake) - include(rapids-cpm) - include(rapids-export) - include(rapids-find) - rapids_cpm_init() -endif() - -project( - rtcx - VERSION 0.1.0 - LANGUAGES CXX -) - -option(RTCX_STATIC_LINK_NVRTC "Use static linking for NVRTC" OFF) -option(RTCX_STATIC_LINK_NVJITLINK "Use static linking for nvJitLink" OFF) - -rapids_find_package( - CUDAToolkit REQUIRED - BUILD_EXPORT_SET rtcx-exports - INSTALL_EXPORT_SET rtcx-exports -) - -if(NOT TARGET zstd) - set(CPM_DOWNLOAD_zstd ON) - rapids_cpm_find( - zstd 1.5.7 - GLOBAL_TARGETS zstd - CPM_ARGS - GIT_REPOSITORY https://github.com/facebook/zstd.git - GIT_TAG v1.5.7 - GIT_SHALLOW FALSE SOURCE_SUBDIR build/cmake - OPTIONS "ZSTD_BUILD_STATIC ON" "ZSTD_BUILD_SHARED OFF" "ZSTD_BUILD_TESTS OFF" - "ZSTD_BUILD_PROGRAMS OFF" "BUILD_SHARED_LIBS OFF" - ) - - if(zstd_ADDED) - # disable weak symbols support to hide tracing APIs as well - target_compile_definitions(libzstd_static PRIVATE ZSTD_HAVE_WEAK_SYMBOLS=0) - # expose experimental API - target_compile_definitions(libzstd_static PUBLIC ZSTD_STATIC_LINKING_ONLY=0N) - # suppress warnings from uninitialized variables and redefining ZSTD_STATIC_LINKING_ONLY - target_compile_options(libzstd_static PRIVATE -w) - add_library(zstd ALIAS libzstd_static) - endif() -endif() - -if(NOT TARGET nvtx3::nvtx3-cpp) - include(${rapids-cmake-dir}/cpm/nvtx3.cmake) - rapids_cpm_nvtx3() -endif() - -add_library(rtcx STATIC hash.cpp rtcx.cpp) -add_library(rtcx::rtcx ALIAS rtcx) - -set_target_properties( - rtcx - PROPERTIES CXX_STANDARD 20 - CXX_STANDARD_REQUIRED YES - CXX_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN YES - POSITION_INDEPENDENT_CODE ON -) - -target_include_directories( - rtcx - PUBLIC $ - PRIVATE ${CUDAToolkit_INCLUDE_DIRS} -) - -target_link_libraries(rtcx PRIVATE zstd ${CMAKE_DL_LIBS} nvtx3::nvtx3-cpp) - -if(RTCX_STATIC_LINK_NVRTC) - target_link_libraries(rtcx PRIVATE CUDA::nvrtc_static CUDA::nvrtc_builtins_static) - target_compile_definitions(rtcx PRIVATE RTCX_STATIC_LINK_LIBNVRTC=1) -else() - target_compile_definitions(rtcx PRIVATE RTCX_STATIC_LINK_LIBNVRTC=0) -endif() - -if(RTCX_STATIC_LINK_NVJITLINK) - target_link_libraries(rtcx PRIVATE CUDA::nvJitLink_static) - target_compile_definitions(rtcx PRIVATE RTCX_STATIC_LINK_LIBNVJITLINK=1) -else() - target_compile_definitions(rtcx PRIVATE RTCX_STATIC_LINK_LIBNVJITLINK=0) -endif() - -# ============================================================================= -# Install / Export -# ============================================================================= -include(GNUInstallDirs) -include(${rapids-cmake-dir}/cmake/install_lib_dir.cmake) -rapids_cmake_install_lib_dir(lib_dir) - -# Option to control install (default ON when built standalone, OFF when used as subdirectory) -option(RTCX_INSTALL "Enable installation of rtcx targets" ${PROJECT_IS_TOP_LEVEL}) - -set(rtcx_install_code_string - [=[ -# Embed functions (add_embed, embed_includes, embed_blob, embed) -# are included automatically so consumers don't need explicit include(). -include("${CMAKE_CURRENT_LIST_DIR}/embed.cmake") - -# Set rtcx_LIBCXX_DIR for consumers using embed_includes with libcxx headers. -set(rtcx_LIBCXX_DIR "${PACKAGE_PREFIX_DIR}/share/rtcx/libcxx") -]=] -) - -string( - CONFIGURE - [=[ -# Embed functions (add_embed, embed_includes, embed_blob, embed) -# are included automatically so consumers don't need explicit include(). -include("@CMAKE_CURRENT_SOURCE_DIR@/embed.cmake") - -# Set rtcx_LIBCXX_DIR for consumers using embed_includes with libcxx headers. -set(rtcx_LIBCXX_DIR "@CMAKE_CURRENT_SOURCE_DIR@/libcxx") -]=] - rtcx_build_code_string - @ONLY -) - -if(NOT RTCX_INSTALL) - set(rtcx_exclude_from_install EXCLUDE_FROM_ALL) -endif() - -install( - TARGETS rtcx - EXPORT rtcx-exports - ARCHIVE DESTINATION ${lib_dir} - INCLUDES - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - ${rtcx_exclude_from_install} -) - -if(RTCX_INSTALL) - install(FILES rtcx.hpp sha256.hpp embed.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rtcx) - install(FILES embed.cmake embed.in.cpp DESTINATION ${lib_dir}/cmake/rtcx) - install(DIRECTORY libcxx/ DESTINATION ${CMAKE_INSTALL_DATADIR}/rtcx/libcxx) - - rapids_export( - INSTALL rtcx - EXPORT_SET rtcx-exports - GLOBAL_TARGETS rtcx - NAMESPACE rtcx:: - FINAL_CODE_BLOCK rtcx_install_code_string - ) -endif() - -# Build-tree export (always, so CPM consumers can find_package from the build tree) -rapids_export( - BUILD rtcx - EXPORT_SET rtcx-exports - GLOBAL_TARGETS rtcx - NAMESPACE rtcx:: - FINAL_CODE_BLOCK rtcx_build_code_string -) diff --git a/cpp/librtcx/CONTRIBUTING.md b/cpp/librtcx/CONTRIBUTING.md deleted file mode 100644 index fae540321764..000000000000 --- a/cpp/librtcx/CONTRIBUTING.md +++ /dev/null @@ -1,58 +0,0 @@ -# Contributing to librtcx - -Contributions to librtcx fall into the following categories: - -1. To report a bug, request a new feature, or report a problem with documentation, please file an - [issue](https://github.com/rapidsai/cudf/issues/new/choose) describing the problem or new feature - in detail. -2. To propose and implement a new feature, please file a new feature request - [issue](https://github.com/rapidsai/cudf/issues/new/choose). Describe the intended feature and - discuss the design and implementation with the team and community. Once the team agrees that the - plan looks good, go ahead and implement it, using the [code contributions](#code-contributions) - guide below. -3. To implement a feature or bug fix for an existing issue, please follow the [code - contributions](#code-contributions) guide below. If you need more context on a particular issue, - please ask in a comment. - -As contributors and maintainers to this project, you are expected to abide by the -[Contributor Code of Conduct](https://docs.rapids.ai/resources/conduct/). - -## Code contributions - -1. Create a fork of the [cudf repository](https://github.com/rapidsai/cudf) and check out a branch - with a name that describes your planned work. -2. Write code to address the issue or implement the feature. -3. Add unit tests. -4. [Create your pull request](https://github.com/rapidsai/cudf/compare). -5. Verify that CI passes all status checks. Fix if needed. -6. Wait for other developers to review your code and update code as needed. -7. Once reviewed and approved, a maintainer will merge your pull request. - -If you are unsure about anything, don't hesitate to comment on issues and ask for clarification! - -## Code Formatting - -librtcx uses [pre-commit](https://pre-commit.com/) to execute code linters and formatters. -These tools ensure a consistent code format throughout the project. - -To use `pre-commit`, install via `conda` or `pip`: - -```bash -conda install -c conda-forge pre-commit -``` - -```bash -pip install pre-commit -``` - -Then run pre-commit hooks before committing code: - -```bash -pre-commit run -``` - -Optionally, you may set up the pre-commit hooks to run automatically when you make a git commit: - -```bash -pre-commit install -``` diff --git a/cpp/librtcx/LICENSE b/cpp/librtcx/LICENSE deleted file mode 100644 index 18bcb4316e6d..000000000000 --- a/cpp/librtcx/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2018 NVIDIA Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/cpp/librtcx/RAPIDS_BRANCH b/cpp/librtcx/RAPIDS_BRANCH deleted file mode 120000 index 15b9ac7c238b..000000000000 --- a/cpp/librtcx/RAPIDS_BRANCH +++ /dev/null @@ -1 +0,0 @@ -../../RAPIDS_BRANCH \ No newline at end of file diff --git a/cpp/librtcx/README.md b/cpp/librtcx/README.md deleted file mode 100644 index b8f9898c0d13..000000000000 --- a/cpp/librtcx/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# librtcx - -RTCX (runtime-compiler extended) is a wrapper around NVRTC and nvJitLink designed to provide: - -- User-controlled compilation, linking, caching, and pre-loading of CUDA kernels -- Zero-copy interfaces to manage JIT compilation and linking -- CMake script to embed **compressed** headers directly into an executable without incurring overhead at runtime on every compilation request -- Facilities to pre-load and teardown dynamic library dependencies (`libcuda`, `libnvrtc`, and `libnvJitLink`) - -## Platforms Supported -- Linux - -## Build-time Requirements -- CMake >= 4.0 -- libzstd -- xxHash -- CUDA >= 12.2 - -# Dependencies -- nvJitlink >= 12.2 -- NVRTC >= 12.2 -- LibCUDA >= 12.2 diff --git a/cpp/librtcx/VERSION b/cpp/librtcx/VERSION deleted file mode 120000 index 558194c5a5a5..000000000000 --- a/cpp/librtcx/VERSION +++ /dev/null @@ -1 +0,0 @@ -../../VERSION \ No newline at end of file diff --git a/cpp/librtcx/cmake/RAPIDS.cmake b/cpp/librtcx/cmake/RAPIDS.cmake deleted file mode 100644 index a3ad5e3f481b..000000000000 --- a/cpp/librtcx/cmake/RAPIDS.cmake +++ /dev/null @@ -1,83 +0,0 @@ -# ============================================================================= -# cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# cmake-format: on -# ============================================================================= -# -# This is the preferred entry point for projects using rapids-cmake -# -# Enforce the minimum required CMake version for all users -cmake_minimum_required(VERSION 4.0 FATAL_ERROR) - -# Allow users to control which version is used -if(NOT (rapids-cmake-branch OR rapids-cmake-version)) - message( - FATAL_ERROR "The CMake variable `rapids-cmake-branch` or `rapids-cmake-version` must be defined" - ) -endif() - -# Allow users to control which GitHub repo is fetched -if(NOT rapids-cmake-repo) - # Define a default repo if the user doesn't set one - set(rapids-cmake-repo rapidsai/rapids-cmake) -endif() - -# Allow users to control which branch is fetched -if(NOT rapids-cmake-branch) - # Define a default branch if the user doesn't set one - set(rapids-cmake-branch "release/${rapids-cmake-version}") -endif() - -# Allow users to control the exact URL passed to FetchContent -if(NOT rapids-cmake-url) - # Construct a default URL if the user doesn't set one - set(rapids-cmake-url "https://github.com/${rapids-cmake-repo}/") - - # In order of specificity - if(rapids-cmake-fetch-via-git) - if(rapids-cmake-sha) - # An exact git SHA takes precedence over anything - set(rapids-cmake-value-to-clone "${rapids-cmake-sha}") - elseif(rapids-cmake-tag) - # Followed by a git tag name - set(rapids-cmake-value-to-clone "${rapids-cmake-tag}") - else() - # Or if neither of the above two were defined, use a branch - set(rapids-cmake-value-to-clone "${rapids-cmake-branch}") - endif() - else() - if(rapids-cmake-sha) - # An exact git SHA takes precedence over anything - set(rapids-cmake-value-to-clone "archive/${rapids-cmake-sha}.zip") - elseif(rapids-cmake-tag) - # Followed by a git tag name - set(rapids-cmake-value-to-clone "archive/refs/tags/${rapids-cmake-tag}.zip") - else() - # Or if neither of the above two were defined, use a branch - set(rapids-cmake-value-to-clone "archive/refs/heads/${rapids-cmake-branch}.zip") - endif() - endif() -endif() - -include(FetchContent) -if(rapids-cmake-fetch-via-git) - FetchContent_Declare( - rapids-cmake - GIT_REPOSITORY "${rapids-cmake-url}" - GIT_TAG "${rapids-cmake-value-to-clone}" - ) -else() - string(APPEND rapids-cmake-url "${rapids-cmake-value-to-clone}") - FetchContent_Declare(rapids-cmake URL "${rapids-cmake-url}") -endif() -FetchContent_GetProperties(rapids-cmake) -if(rapids-cmake_POPULATED) - # Something else has already populated rapids-cmake, only thing we need to do is setup the - # CMAKE_MODULE_PATH - if(NOT "${rapids-cmake-dir}" IN_LIST CMAKE_MODULE_PATH) - list(APPEND CMAKE_MODULE_PATH "${rapids-cmake-dir}") - endif() -else() - FetchContent_MakeAvailable(rapids-cmake) -endif() diff --git a/cpp/librtcx/cmake/rapids_config.cmake b/cpp/librtcx/cmake/rapids_config.cmake deleted file mode 100644 index fe59436c012d..000000000000 --- a/cpp/librtcx/cmake/rapids_config.cmake +++ /dev/null @@ -1,38 +0,0 @@ -# ============================================================================= -# cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# cmake-format: on -# ============================================================================= -file(READ "${CMAKE_CURRENT_LIST_DIR}/../VERSION" _rapids_version) -if(_rapids_version MATCHES [[^([0-9][0-9])\.([0-9][0-9])\.([0-9][0-9])]]) - set(RAPIDS_VERSION_MAJOR "${CMAKE_MATCH_1}") - set(RAPIDS_VERSION_MINOR "${CMAKE_MATCH_2}") - set(RAPIDS_VERSION_PATCH "${CMAKE_MATCH_3}") - set(RAPIDS_VERSION_MAJOR_MINOR "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}") - set(RAPIDS_VERSION "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}.${RAPIDS_VERSION_PATCH}") -else() - string(REPLACE "\n" "\n " _rapids_version_formatted " ${_rapids_version}") - message( - FATAL_ERROR - "Could not determine RAPIDS version. Contents of VERSION file:\n${_rapids_version_formatted}" - ) -endif() - -file(STRINGS "${CMAKE_CURRENT_LIST_DIR}/../RAPIDS_BRANCH" RAPIDS_BRANCH) -if(NOT RAPIDS_BRANCH) - message( - FATAL_ERROR - "Could not determine branch name to use for checking out rapids-cmake. The file \"${CMAKE_CURRENT_LIST_DIR}/../RAPIDS_BRANCH\" is missing." - ) -endif() - -if(NOT rapids-cmake-version) - set(rapids-cmake-version "${RAPIDS_VERSION_MAJOR_MINOR}") -endif() - -if(NOT rapids-cmake-branch) - set(rapids-cmake-branch "${RAPIDS_BRANCH}") -endif() - -include("${CMAKE_CURRENT_LIST_DIR}/RAPIDS.cmake") diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake deleted file mode 100644 index e0002b8ae370..000000000000 --- a/cpp/librtcx/embed.cmake +++ /dev/null @@ -1,349 +0,0 @@ -# ============================================================================= -# cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# cmake-format: on -# ============================================================================= - -if(NOT TARGET zstd) - message(FATAL_ERROR "zstd target is required for LIBRTCX embedding.") -endif() - -if(NOT TARGET xxhash) - message(FATAL_ERROR "xxhash target is required for LIBRTCX embedding.") -endif() - -# This function initializes a target for JIT embedding. It must be called before any calls to -# embed_includes() or embed_blob() for the target. It creates a dedicated INTERFACE library target -# that is used to track registered files and dependencies via target properties. The TARGET argument -# specifies the name of the target being initialized. -function(rtcx_add_embed TARGET) - set(OPTIONS "") - set(ONE_VALUE_ARGS) - set(MULTI_VALUE_ARGS) - cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - - if(NOT DEFINED TARGET) - message(FATAL_ERROR "TARGET argument is required") - endif() - - add_library(${TARGET}__embed_props INTERFACE) - set_property(TARGET ${TARGET}__embed_props PROPERTY EMBED_FILE_INDEX 0) -endfunction() - -# This function registers a directory of include files to be embedded for JIT compilation. -function(rtcx_embed_includes TARGET) - set(OPTIONS "") - set(ONE_VALUE_ARGS SOURCE_DIRECTORY # Source directory where files will be copied from - DEST_DIRECTORY # Destination directory where files will be copied to - ) - set(MULTI_VALUE_ARGS - FILES # Source files relative to SOURCE_DIRECTORY (optional, if not provided, all files under - # SOURCE_DIRECTORY will be used) - INCLUDE_DIRECTORIES # Include directories to be used when compiling with these files - ) - cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - - if(NOT TARGET ${TARGET}__embed_props) - message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") - endif() - - if(NOT ARG_SOURCE_DIRECTORY - OR NOT ARG_DEST_DIRECTORY - OR NOT ARG_INCLUDE_DIRECTORIES - ) - message( - FATAL_ERROR "SOURCE_DIRECTORY, DEST_DIRECTORY, and INCLUDE_DIRECTORIES arguments are required" - ) - endif() - - if(NOT ARG_FILES) - # gather all include files under the specified directory - file(GLOB_RECURSE INCLUDE_FILES "${ARG_SOURCE_DIRECTORY}/*") - - # get their paths relative to the base include directory - set(INCLUDE_FILES_RELATIVE_PATHS "") - foreach(INCLUDE_FILE IN LISTS INCLUDE_FILES) - file(RELATIVE_PATH INCLUDE_FILE_REL_PATH "${ARG_SOURCE_DIRECTORY}" "${INCLUDE_FILE}") - list(APPEND INCLUDE_FILES_RELATIVE_PATHS "${INCLUDE_FILE_REL_PATH}") - endforeach() - - set(ARG_FILES ${INCLUDE_FILES_RELATIVE_PATHS}) - endif() - - # check that each source file exists - foreach(SOURCE_FILE IN LISTS ARG_FILES) - if(NOT EXISTS "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}") - message(FATAL_ERROR "Source file '${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}' does not exist") - endif() - endforeach(SOURCE_FILE) - - # Determine the starting index for new IDs from the current list length - get_property( - SOURCE_FILE_IDS - TARGET ${TARGET}__embed_props - PROPERTY EMBED_SOURCE_FILE_IDS - ) - list(LENGTH SOURCE_FILE_IDS IDX) - - foreach(SOURCE_FILE IN LISTS ARG_FILES) - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_SOURCE_FILE_IDS "include_${IDX}" - ) - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_SOURCE_FILES "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}" - ) - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}" - ) - math(EXPR IDX "${IDX} + 1") - endforeach() - - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES} - ) - - get_property( - SOURCE_FILE_IDS - TARGET ${TARGET}__embed_props - PROPERTY EMBED_SOURCE_FILE_IDS - ) - list(LENGTH SOURCE_FILE_IDS IDX) - - set_property(TARGET ${TARGET}__embed_props PROPERTY EMBED_FILE_INDEX ${IDX}) - -endfunction() - -# This function registers a single file to be embedded for JIT compilation. -function(rtcx_embed_blob TARGET) - set(OPTIONS) - set(ONE_VALUE_ARGS ID FILE DEST) - set(MULTI_VALUE_ARGS ARRAY_IDS ARRAY_VALUES) - cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - - if(NOT TARGET ${TARGET}__embed_props) - message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with rtcx_add_embed()") - endif() - - if(NOT ARG_ID - OR NOT ARG_FILE - OR NOT ARG_DEST - ) - message(FATAL_ERROR "ID, FILE, and DEST arguments are required") - endif() - - if(ARG_ARRAY_IDS) - if(NOT ARG_ARRAY_VALUES) - message(FATAL_ERROR "ARRAY_VALUES argument is required when ARRAY_IDS is provided") - endif() - - list(LENGTH ARG_ARRAY_IDS ARG_ARRAY_IDS_LENGTH) - list(LENGTH ARG_ARRAY_VALUES ARG_ARRAY_VALUES_LENGTH) - - if(NOT ARG_ARRAY_IDS_LENGTH EQUAL ARG_ARRAY_VALUES_LENGTH) - message(FATAL_ERROR "ARRAY_IDS and ARRAY_VALUES must have the same length") - endif() - - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_ARRAY_IDS ${ARG_ARRAY_IDS} - ) - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_ARRAY_VALUES ${ARG_ARRAY_VALUES} - ) - endif() - - if(ARG_FILE MATCHES "\\$]+)>") - # If the file is a generator expression for target objects add as dependency - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_TARGET_DEPS $ - ) - # Also record the target name itself. Depending only on $ creates file-level - # dependencies without a target-level ordering, which breaks the Makefiles generator (Ninja - # resolves it via its global build graph). - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_TARGET_DEP_NAMES ${CMAKE_MATCH_1} - ) - else() - if(NOT EXISTS "${ARG_FILE}") - message(FATAL_ERROR "Source file '${ARG_FILE}' does not exist") - endif() - endif() - - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_SOURCE_FILE_IDS ${ARG_ID} - ) - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_SOURCE_FILES ${ARG_FILE} - ) - set_property( - TARGET ${TARGET}__embed_props - APPEND - PROPERTY EMBED_SOURCE_FILE_DESTS ${ARG_DEST} - ) - - get_property( - SOURCE_FILE_IDS - TARGET ${TARGET}__embed_props - PROPERTY EMBED_SOURCE_FILE_IDS - ) - list(LENGTH SOURCE_FILE_IDS IDX) - - set_property(TARGET ${TARGET}__embed_props PROPERTY EMBED_FILE_INDEX ${IDX}) - -endfunction() - -#[==[ -# This function generates the necessary files and build targets to embed the registered source files -# for JIT compilation. -#]==] -# cmake-lint: disable=R0915 -function(rtcx_embed TARGET) - set(OPTIONS "") - set(ONE_VALUE_ARGS "COMPRESSION" "OUTPUT_DIRECTORY") - set(MULTI_VALUE_ARGS "") - cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) - - if(NOT TARGET ${TARGET}__embed_props) - message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with rtcx_add_embed()") - endif() - - if(NOT DEFINED ARG_COMPRESSION) - message(FATAL_ERROR "COMPRESSION argument is required") - endif() - - if(NOT ARG_COMPRESSION STREQUAL "none" AND NOT ARG_COMPRESSION STREQUAL "zstd") - message(FATAL_ERROR "COMPRESSION argument must be either none or zstd") - endif() - - if(NOT DEFINED ARG_OUTPUT_DIRECTORY) - message(FATAL_ERROR "OUTPUT_DIRECTORY argument is required") - endif() - - get_property( - EMBED_SOURCE_FILES - TARGET ${TARGET}__embed_props - PROPERTY EMBED_SOURCE_FILES - ) - if(NOT EMBED_SOURCE_FILES) - message(FATAL_ERROR "No source files registered for target '${TARGET}'") - endif() - - get_property( - EMBED_SOURCE_FILE_IDS - TARGET ${TARGET}__embed_props - PROPERTY EMBED_SOURCE_FILE_IDS - ) - get_property( - EMBED_SOURCE_FILE_DESTS - TARGET ${TARGET}__embed_props - PROPERTY EMBED_SOURCE_FILE_DESTS - ) - get_property( - EMBED_TARGET_DEPS - TARGET ${TARGET}__embed_props - PROPERTY EMBED_TARGET_DEPS - ) - get_property( - EMBED_TARGET_DEP_NAMES - TARGET ${TARGET}__embed_props - PROPERTY EMBED_TARGET_DEP_NAMES - ) - get_property( - EMBED_ARRAY_IDS - TARGET ${TARGET}__embed_props - PROPERTY EMBED_ARRAY_IDS - ) - get_property( - EMBED_ARRAY_VALUES - TARGET ${TARGET}__embed_props - PROPERTY EMBED_ARRAY_VALUES - ) - get_property( - EMBED_INCLUDE_DIRS - TARGET ${TARGET}__embed_props - PROPERTY EMBED_INCLUDE_DIRECTORIES - ) - - set(OUTPUT_DIR "${ARG_OUTPUT_DIRECTORY}") - set(EMBED_SCRIPT_TEMPLATE "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.cpp") - set(CONFIGURED_EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed_cfg.cpp") - set(EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed.cpp") - - set(EMBED_SCRIPT__ID "${TARGET}") - set(EMBED_SCRIPT__ARRAY_IDS "${EMBED_ARRAY_IDS}") - set(EMBED_SCRIPT__ARRAY_VALUES "${EMBED_ARRAY_VALUES}") - set(EMBED_SCRIPT__FILE_IDS "${EMBED_SOURCE_FILE_IDS}") - set(EMBED_SCRIPT__FILE_PATHS "${EMBED_SOURCE_FILES}") - set(EMBED_SCRIPT__FILE_DESTS "${EMBED_SOURCE_FILE_DESTS}") - set(EMBED_SCRIPT__INCLUDE_DIRS "${EMBED_INCLUDE_DIRS}") - set(EMBED_SCRIPT__COMPRESSION "${ARG_COMPRESSION}") - set(EMBED_SCRIPT__OUTPUT_DIR "${OUTPUT_DIR}") - - configure_file(${EMBED_SCRIPT_TEMPLATE} ${CONFIGURED_EMBED_SCRIPT} @ONLY) - file( - GENERATE - OUTPUT "${EMBED_SCRIPT}" - INPUT "${CONFIGURED_EMBED_SCRIPT}" - ) - - set(RUNNER "${TARGET}__jit_embed_run") - add_executable( - ${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}" ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/hash.cpp - ) - target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} xxhash zstd) - target_include_directories( - ${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR} ${ZSTD_INCLUDE_DIR} - ) - set_target_properties(${RUNNER} PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES) - - add_custom_command( - OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin - COMMAND "${CMAKE_COMMAND}" -E env $ - DEPENDS "${EMBED_SCRIPT}" ${EMBED_SOURCE_FILES} ${EMBED_TARGET_DEPS} ${EMBED_TARGET_DEP_NAMES} - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - COMMENT "Generating JIT embed for ${TARGET} into ${OUTPUT_DIR}" - VERBATIM - ) - - add_custom_target( - ${TARGET} ALL - DEPENDS ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin - COMMENT "Custom target for JIT embed of ${TARGET}" - ) - - message( - STATUS - "JIT embed for target ${TARGET} will be generated into: ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin" - ) - - set(${TARGET}_INCLUDE_DIRS - "${OUTPUT_DIR}" - PARENT_SCOPE - ) - - set(${TARGET}_SOURCE_DIR - ${OUTPUT_DIR} - PARENT_SCOPE - ) - -endfunction() diff --git a/cpp/librtcx/embed.hpp b/cpp/librtcx/embed.hpp deleted file mode 100644 index 5df1aa4db40b..000000000000 --- a/cpp/librtcx/embed.hpp +++ /dev/null @@ -1,410 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include "hash.hpp" - -#include -#include -#include - -#define XXH_INLINE_ALL -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define RTCX_EMBED_EXPECTS(condition, message) \ - do { \ - if (!(condition)) { \ - throw std::runtime_error(std::format("{}:{}: {}", __FILE__, __LINE__, message)); \ - } \ - } while (false) - -namespace rtcx_embed { - -struct size_range { - size_t offset = 0; - size_t size = 0; -}; - -struct embed_output { - std::string cxx_header; - std::string asm_source; - std::vector bin_file_data; -}; - -std::pair, std::vector> merge_bytes_with_null_terminators( - std::span const> bytes_lists) -{ - std::vector merged; - std::vector ranges; - - for (auto& byte_data : bytes_lists) { - ranges.push_back({merged.size(), byte_data.size()}); - merged.insert(merged.end(), byte_data.begin(), byte_data.end()); - merged.push_back(0); - } - - return {std::move(merged), std::move(ranges)}; -} - -std::vector load_file_bytes(std::string_view file_path) -{ - std::string path_str(file_path); - std::ifstream file(path_str, std::ios::binary | std::ios::ate); - RTCX_EMBED_EXPECTS(file.is_open(), std::format("Failed to open file at path: {}", file_path)); - auto file_size = file.tellg(); - RTCX_EMBED_EXPECTS(file_size >= 0, - std::format("Failed to determine size of file at path: {}", file_path)); - file.seekg(0, std::ios::beg); - std::vector bytes(file_size); - RTCX_EMBED_EXPECTS(file.read(reinterpret_cast(bytes.data()), file_size), - std::format("Failed to read file at path: {}", file_path)); - return bytes; -} - -std::vector compress_bytes(std::span bytes, std::string_view compression) -{ - RTCX_EMBED_EXPECTS( - compression == "none" || compression == "zstd", - std::format("Invalid compression type: {}. Supported values are 'none' and 'zstd'", - compression)); - - if (compression == "none") { return std::vector(bytes.begin(), bytes.end()); } - - auto const max_compressed_size = ZSTD_compressBound(bytes.size()); - std::vector compressed(max_compressed_size); - auto const compressed_size = - ZSTD_compress(compressed.data(), compressed.size(), bytes.data(), bytes.size(), 22); - - RTCX_EMBED_EXPECTS( - !ZSTD_isError(compressed_size), - std::format("Compression failed with error: {}", ZSTD_getErrorName(compressed_size))); - - compressed.resize(compressed_size); - return compressed; -} - -rtcx::hash128 compute_embed_hash(std::span uncompressed_files_bytes, - std::span merged_dests_bytes, - std::span merged_include_dirs_bytes, - std::string_view compression) -{ - XXH3_state_t state; - XXH3_INITSTATE(&state); - XXH3_128bits_reset(&state); - XXH3_128bits_update(&state, uncompressed_files_bytes.data(), uncompressed_files_bytes.size()); - XXH3_128bits_update(&state, merged_dests_bytes.data(), merged_dests_bytes.size()); - XXH3_128bits_update(&state, merged_include_dirs_bytes.data(), merged_include_dirs_bytes.size()); - XXH3_128bits_update(&state, compression.data(), compression.size()); - auto hash = XXH3_128bits_digest(&state); - return rtcx::hash128{hash.high64, hash.low64}; -} - -template -std::string join_formatted(Container& items, std::string_view delimiter, Formatter&& formatter) -{ - std::ostringstream result; - for (std::size_t i = 0; i < items.size(); ++i) { - if (i != 0) { result << delimiter; } - result << formatter(items[i]); - } - return result.str(); -} - -enum class value_type : int8_t { INT, STRING }; - -std::string generate_arrays(std::span array_ids, - std::span array_values) -{ - auto get_type = [](std::string_view value) -> value_type { - RTCX_EMBED_EXPECTS(!value.empty(), "Value cannot be empty"); - return std::isdigit(value[0]) ? value_type::INT : value_type::STRING; - }; - - using strings_t = std::vector; - using ints_t = std::vector; - using values_t = std::variant; - - std::map arrays; - - for (size_t i = 0; i < array_ids.size(); ++i) { - auto id = array_ids[i]; - auto value = array_values[i]; - auto type = get_type(value); - if (auto array_it = arrays.find(id); array_it == arrays.end()) { - switch (type) { - case value_type::INT: { - arrays.emplace(id, ints_t{}); - } break; - case value_type::STRING: { - arrays.emplace(id, strings_t{}); - } break; - default: throw std::logic_error("Unexpected constant type"); - } - } - - auto& array = arrays[id]; - - switch (type) { - case value_type::INT: { - std::int64_t int_value; - RTCX_EMBED_EXPECTS( - std::from_chars(value.data(), value.data() + value.size(), int_value).ec == std::errc(), - std::format("Invalid integer constant value: {}", value)); - std::get(array).push_back(int_value); - } break; - case value_type::STRING: { - std::get(array).push_back(value); - } break; - - default: break; - } - } - - std::string result; - - for (auto& [id, array] : arrays) { - if (auto* ints = std::get_if(&array); ints != nullptr) { - result += - std::format("constexpr std::int64_t {}[{}] = {{ {} }};\n\n", - id, - ints->size(), - join_formatted(*ints, ", ", [](std::int64_t v) { return std::to_string(v); })); - } else { - auto& strings = std::get(array); - result += - std::format("constexpr char const* {}[{}] = {{ {} }};\n\n", - id, - strings.size(), - join_formatted(strings, ", ", [](auto s) { return std::format("\"{}\"", s); })); - } - } - - return result; -} - -embed_output generate_cxx_source_files_data(std::string_view id, - std::span array_ids, - std::span array_values, - std::span file_ids, - std::span file_paths, - std::span file_dsts, - std::span include_dirs, - std::string_view compression) -{ - std::vector> file_bytes; - file_bytes.reserve(file_paths.size()); - for (auto const& path : file_paths) { - file_bytes.emplace_back(load_file_bytes(path)); - } - - auto [uncompressed_files_bytes, file_ranges] = merge_bytes_with_null_terminators(file_bytes); - - auto compress = compression != "none"; - std::vector compressed_files_bytes = - compress ? compress_bytes(uncompressed_files_bytes, compression) : uncompressed_files_bytes; - - auto binary_size = compress ? compressed_files_bytes.size() - : static_cast(uncompressed_files_bytes.size()); - - if (compress) { - std::cout << std::format( - "-- Compressed {}'s binary from {} bytes to {} bytes (compression ratio: {:.2f})\n", - id, - uncompressed_files_bytes.size(), - compressed_files_bytes.size(), - static_cast(compressed_files_bytes.size()) / - static_cast(uncompressed_files_bytes.size())); - } - - std::vector> destination_bytes; - destination_bytes.reserve(file_dsts.size()); - for (auto const& dest : file_dsts) { - destination_bytes.emplace_back(dest.begin(), dest.end()); - } - auto [merged_dests_bytes, _] = merge_bytes_with_null_terminators(destination_bytes); - - std::vector> include_directory_bytes; - include_directory_bytes.reserve(include_dirs.size()); - for (auto const& include_directory : include_dirs) { - include_directory_bytes.emplace_back(include_directory.begin(), include_directory.end()); - } - auto [merged_include_dirs_bytes, __] = merge_bytes_with_null_terminators(include_directory_bytes); - - auto hash = compute_embed_hash( - uncompressed_files_bytes, merged_dests_bytes, merged_include_dirs_bytes, compression); - - auto include_dirs_list = - join_formatted(include_dirs, ",\n", [](auto s) { return std::format("\"{}\"", s); }); - auto file_dests_list = - join_formatted(file_dsts, ",\n", [](auto s) { return std::format("\"{}\"", s); }); - auto file_ids_list = - join_formatted(file_ids, ",\n", [](auto s) { return std::format("\"{}\"", s); }); - std::vector file_indices(file_ids.size()); - std::iota(file_indices.begin(), file_indices.end(), 0ULL); - auto file_indices_list = join_formatted(file_indices, "\n", [&](auto i) { - return std::format("constexpr std::size_t {} = {}ULL;", file_ids[i], i); - }); - auto file_ranges_list = join_formatted( - file_ranges, ",\n", [](auto r) { return std::format("{{{}, {}}}", r.offset, r.size); }); - auto hash_list = join_formatted( - hash, ", ", [](uint8_t byte) { return std::format("0x{:02x}", static_cast(byte)); }); - auto arrays_list = generate_arrays(array_ids, array_values); - auto namespace_decl = "namespace " + std::string(id); - - auto cxx_header = std::format( - R"***( -// Auto-generated header for embedded files -#pragma once - -#include -#include -#include -#include - -{} {{ - - -constexpr char const * include_directories[{}] = -{{ -{} -}}; - -{} - -constexpr char const * file_ids[{}] = -{{ -{} -}}; - -constexpr char const * file_destinations[{}] = -{{ -{} -}}; - -constexpr std::size_t file_ranges[{}][2] = -{{ -{} -}}; - -constexpr std::size_t files_uncompressed_size = {}; - -constexpr char const * files_compression = "{}"; - -extern "C" std::uint8_t const {}_files_begin[]; - -static std::span const files = -{{ -{}_files_begin, -{}L -}}; - -constexpr std::uint8_t hash[{}] = -{{ -{} -}}; - -{} - -}} -)***", - namespace_decl, - include_dirs.size(), - include_dirs_list, - file_indices_list, - file_ids.size(), - file_ids_list, - file_dsts.size(), - file_dests_list, - file_ranges.size(), - file_ranges_list, - uncompressed_files_bytes.size(), - compression, - id, - id, - binary_size, - hash.size(), - hash_list, - arrays_list); - - auto asm_source = std::format( - R"***( -.section .rodata -.global {0}_files_begin -{0}_files_begin: -.incbin "{0}.bin" - -.section .note.GNU-stack,"",@progbits -)***", - id); - - return embed_output{ - .cxx_header = cxx_header, - .asm_source = asm_source, - .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes}; -} - -void generate_embed(std::string_view id, - std::span array_ids, - std::span array_values, - std::span file_ids, - std::span file_paths, - std::span file_dsts, - std::span include_dirs, - std::string_view compression, - std::string_view output_directory) -{ - auto output = generate_cxx_source_files_data( - id, array_ids, array_values, file_ids, file_paths, file_dsts, include_dirs, compression); - - std::filesystem::create_directories(std::filesystem::path(output_directory)); - - std::ofstream header_file(std::format("{}/{}.hpp", output_directory, id)); - header_file << output.cxx_header; - - std::ofstream asm_file(std::format("{}/{}.s", output_directory, id)); - asm_file << output.asm_source; - - std::ofstream bin_file(std::format("{}/{}.bin", output_directory, id), std::ios::binary); - bin_file.write(reinterpret_cast(output.bin_file_data.data()), - static_cast(output.bin_file_data.size())); -} - -std::vector split_string(std::string_view str, char delimiter) -{ - if (str.empty()) { return {}; } - std::vector tokens; - std::size_t start = 0; - - while (start <= str.size()) { - auto const pos = str.find(delimiter, start); - if (pos == std::string_view::npos) { - tokens.push_back(str.substr(start)); - break; - } - tokens.push_back(str.substr(start, pos - start)); - start = pos + 1; - } - - return tokens; -} - -} // namespace rtcx_embed diff --git a/cpp/librtcx/embed.in.cpp b/cpp/librtcx/embed.in.cpp deleted file mode 100644 index c53a12d1d0f4..000000000000 --- a/cpp/librtcx/embed.in.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "embed.hpp" - -int main() -{ - std::string_view id = "@EMBED_SCRIPT__ID@"; - auto array_ids = rtcx_embed::split_string("@EMBED_SCRIPT__ARRAY_IDS@", ';'); - auto array_values = rtcx_embed::split_string("@EMBED_SCRIPT__ARRAY_VALUES@", ';'); - auto file_ids = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_IDS@", ';'); - auto file_paths = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_PATHS@", ';'); - auto file_dests = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_DESTS@", ';'); - auto include_directories = rtcx_embed::split_string("@EMBED_SCRIPT__INCLUDE_DIRS@", ';'); - std::string_view compression = "@EMBED_SCRIPT__COMPRESSION@"; - std::string_view output_dir = "@EMBED_SCRIPT__OUTPUT_DIR@"; - - rtcx_embed::generate_embed(id, - array_ids, - array_values, - file_ids, - file_paths, - file_dests, - include_directories, - compression, - output_dir); - return EXIT_SUCCESS; -} diff --git a/cpp/librtcx/hash.cpp b/cpp/librtcx/hash.cpp deleted file mode 100644 index fd32d55b2269..000000000000 --- a/cpp/librtcx/hash.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "hash.hpp" - -#include -#include -#include -#include -#include - -namespace rtcx { - -char const* hash128_hex_string::data() const { return data_; } - -char const* hash128_hex_string::c_str() const { return data_; } - -hash128_hex_string hash128_hex_string::make(std::span input) -{ - constexpr char const HEX_CHARS[] = "0123456789abcdef"; // NOLINT(modernize-avoid-c-arrays) - hash128_hex_string hex; - for (std::size_t i = 0; i < NUM_HEX_BYTES; ++i) { - hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; - hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; - } - hex.data_[NUM_HEX_DIGITS] = '\0'; - return hex; -} - -hash128_hex_string hash128_hex_string::make(__uint128_t hash) -{ - auto array = std::bit_cast>(hash); - return make(array); -} - -std::uint8_t hash128::operator[](std::size_t index) const -{ - return reinterpret_cast(&value)[15 - index]; -} - -std::size_t hash128::size() const { return 16; } - -std::uint8_t const* hash128::data() const { return reinterpret_cast(&value); } - -hash128_hex_string hash128::to_hex_string() const { return hash128_hex_string::make(value); } - -hash128 hash128::parse(std::string_view hex) -{ - if (hex.size() != hash128_hex_string::NUM_HEX_DIGITS) { - throw std::invalid_argument( - std::format("Invalid hash128 hex string length, expected {} got {} (hash: `{}`)", - hash128_hex_string::NUM_HEX_DIGITS, - hex.size(), - hex)); - } - std::array data{}; - for (std::size_t i = 0; i < hash128_hex_string::NUM_HEX_BYTES; ++i) { - auto hex_byte = hex.substr(i * 2, 2); - auto [ptr, ec] = std::from_chars(hex_byte.begin(), hex_byte.end(), data[i], 16); - if (ec != std::errc()) { - throw std::invalid_argument( - std::format("Invalid hex character {} in HEX string: `{}`", hex_byte, hex)); - } - } - return hash128{std::bit_cast<__uint128_t>(data)}; -} - -} // namespace rtcx diff --git a/cpp/librtcx/hash.hpp b/cpp/librtcx/hash.hpp deleted file mode 100644 index d1aa12c85c61..000000000000 --- a/cpp/librtcx/hash.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include -#include -#include - -namespace rtcx { - -struct [[nodiscard]] hash128_hex_string { - static constexpr std::size_t NUM_HEX_DIGITS = 32; - static constexpr std::size_t NUM_HEX_BYTES = NUM_HEX_DIGITS / 2; - - char data_[NUM_HEX_DIGITS + 1]; // NOLINT(modernize-avoid-c-arrays) - - [[nodiscard]] constexpr std::string_view view() const - { - return std::string_view{data_, NUM_HEX_DIGITS}; - } - - [[nodiscard]] constexpr operator std::string_view() const { return view(); } - - [[nodiscard]] char const* data() const; - - [[nodiscard]] char const* c_str() const; - - [[nodiscard]] static constexpr std::size_t size() { return NUM_HEX_DIGITS; } - - static hash128_hex_string make(std::span input); - - static hash128_hex_string make(__uint128_t hash); -}; - -struct hash128 { - __uint128_t value; - - constexpr hash128(__uint128_t v = 0) : value(v) {} - - constexpr hash128(std::uint64_t high, std::uint64_t low) - : value((static_cast<__uint128_t>(high) << 64) | low) - { - } - - [[nodiscard]] constexpr bool operator==(hash128 const&) const = default; - - [[nodiscard]] std::uint8_t operator[](std::size_t index) const; - - [[nodiscard]] std::size_t size() const; - - [[nodiscard]] std::uint8_t const* data() const; - - hash128_hex_string to_hex_string() const; - - static hash128 parse(std::string_view hex); -}; - -} // namespace rtcx diff --git a/cpp/librtcx/libcxx/cassert b/cpp/librtcx/libcxx/cassert deleted file mode 100644 index 6bb3f0766ca8..000000000000 --- a/cpp/librtcx/libcxx/cassert +++ /dev/null @@ -1,8 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include diff --git a/cpp/librtcx/libcxx/climits b/cpp/librtcx/libcxx/climits deleted file mode 100644 index 20653c8fc5d8..000000000000 --- a/cpp/librtcx/libcxx/climits +++ /dev/null @@ -1,8 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include diff --git a/cpp/librtcx/libcxx/cstddef b/cpp/librtcx/libcxx/cstddef deleted file mode 100644 index d0c1fb2c31d6..000000000000 --- a/cpp/librtcx/libcxx/cstddef +++ /dev/null @@ -1,18 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include - -namespace std { - -using max_align_t = cuda::std::max_align_t; -using nullptr_t = cuda::std::nullptr_t; -using ptrdiff_t = cuda::std::ptrdiff_t; -using size_t = cuda::std::size_t; -using byte = cuda::std::byte; - -} // namespace std diff --git a/cpp/librtcx/libcxx/cstdint b/cpp/librtcx/libcxx/cstdint deleted file mode 100644 index eae43f7203db..000000000000 --- a/cpp/librtcx/libcxx/cstdint +++ /dev/null @@ -1,45 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include - -namespace std { - -using int8_t = cuda::std::int8_t; -using int16_t = cuda::std::int16_t; -using int32_t = cuda::std::int32_t; -using int64_t = cuda::std::int64_t; -using uint8_t = cuda::std::uint8_t; -using uint16_t = cuda::std::uint16_t; -using uint32_t = cuda::std::uint32_t; -using uint64_t = cuda::std::uint64_t; - -using int_fast8_t = cuda::std::int_fast8_t; -using int_fast16_t = cuda::std::int_fast16_t; -using int_fast32_t = cuda::std::int_fast32_t; -using int_fast64_t = cuda::std::int_fast64_t; -using uint_fast8_t = cuda::std::uint_fast8_t; -using uint_fast16_t = cuda::std::uint_fast16_t; -using uint_fast32_t = cuda::std::uint_fast32_t; -using uint_fast64_t = cuda::std::uint_fast64_t; - -using int_least8_t = cuda::std::int_least8_t; -using int_least16_t = cuda::std::int_least16_t; -using int_least32_t = cuda::std::int_least32_t; -using int_least64_t = cuda::std::int_least64_t; -using uint_least8_t = cuda::std::uint_least8_t; -using uint_least16_t = cuda::std::uint_least16_t; -using uint_least32_t = cuda::std::uint_least32_t; -using uint_least64_t = cuda::std::uint_least64_t; - -using intptr_t = cuda::std::intptr_t; -using uintptr_t = cuda::std::uintptr_t; - -using intmax_t = cuda::std::intmax_t; -using uintmax_t = cuda::std::uintmax_t; - -} // namespace std diff --git a/cpp/librtcx/rtcx.cpp b/cpp/librtcx/rtcx.cpp deleted file mode 100644 index 1c3144869a8b..000000000000 --- a/cpp/librtcx/rtcx.cpp +++ /dev/null @@ -1,1399 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#ifndef RTCX_STATIC_LINK_LIBNVRTC -#define RTCX_STATIC_LINK_LIBNVRTC 0 -#endif - -#ifndef RTCX_STATIC_LINK_LIBNVJITLINK -#define RTCX_STATIC_LINK_LIBNVJITLINK 0 -#endif - -#define RTCX_EXPECTS(condition_, reason_, exception_type_) \ - do { \ - if (!(condition_)) { \ - throw exception_type_{::std::format("RTCX failure at: {}:{}: {}", \ - ::std::source_location::current().file_name(), \ - ::std::source_location::current().line(), \ - (reason_))}; \ - } \ - } while (0) - -#define RTCX_FAIL(reason_, exception_type_) \ - do { \ - throw exception_type_{::std::format("RTCX failure at: {}:{}: {}", \ - ::std::source_location::current().file_name(), \ - ::std::source_location::current().line(), \ - (reason_))}; \ - } while (0) - -#define RTCX_CHECK_CUDA(...) \ - do { \ - ::CUresult result_ = (__VA_ARGS__); \ - if (result_ != ::CUDA_SUCCESS) { \ - char const* enum_str_; \ - RTCX_EXPECTS(::rtcx::cu->GetErrorString(result_, &enum_str_) == ::CUDA_SUCCESS, \ - "Unable to get CUDA error string", \ - std::runtime_error); \ - auto errstr_ = ::std::format("(cuda) expression `{}` failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast<::std::int64_t>(result_), \ - enum_str_); \ - RTCX_FAIL(errstr_, ::std::runtime_error); \ - } \ - } while (0) - -#define RTCX_CHECK_CUDART(...) \ - do { \ - ::cudaError_t result_ = (__VA_ARGS__); \ - if (result_ != ::cudaSuccess) { \ - char const* enum_name_ = ::cudaGetErrorName(result_); \ - char const* enum_msg_ = ::cudaGetErrorString(result_); \ - auto errstr_ = ::std::format("(cudart) expression `{}` failed, with error ({}: {}): {}", \ - #__VA_ARGS__, \ - static_cast<::std::int64_t>(result_), \ - enum_name_, \ - enum_msg_); \ - RTCX_FAIL(errstr_, ::std::runtime_error); \ - } \ - } while (0) - -#define RTCX_CHECK_NVRTC(...) \ - do { \ - ::nvrtcResult result_ = (__VA_ARGS__); \ - if (result_ != ::NVRTC_SUCCESS) { \ - auto errstr_ = ::std::format("(nvrtc) expression `{}` failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast<::std::int64_t>(result_), \ - ::rtcx::nvrtc->GetErrorString(result_)); \ - RTCX_FAIL(errstr_, ::std::runtime_error); \ - } \ - } while (0) - -#define RTCX_CHECK_NVJITLINK(...) \ - do { \ - ::nvJitLinkResult result_ = (__VA_ARGS__); \ - if (result_ != ::NVJITLINK_SUCCESS) { \ - auto errstr_ = ::std::format("(nvJitLink) expression `{}` failed, with error ({}): {}", \ - #__VA_ARGS__, \ - static_cast<::std::int64_t>(result_), \ - ::rtcx::nvJitLinkResult_string(result_)); \ - RTCX_FAIL(errstr_, ::std::runtime_error); \ - } \ - } while (0) - -#define RTCX_FUNC_RANGE() \ - ::nvtx3::scoped_range_in<::rtcx::nvtx_domain> rtcx_func_range__ { __func__ } - -namespace rtcx { -namespace { - -struct nvtx_domain { - static constexpr char const* name [[maybe_unused]] = "rtcx"; -}; - -enum class object_type : std::uint8_t { LIBRARY, BLOB }; - -std::string_view object_tag(object_type type) -{ - switch (type) { - case object_type::LIBRARY: return "cuLibrary"; - case object_type::BLOB: return "blob"; - default: - RTCX_FAIL(std::format("Unrecognized object type: ({})", static_cast(type)), - std::runtime_error); - } -} - -template -std::string join_strings(std::span strings, std::string_view separator) -{ - if (strings.empty()) { return {}; } - - if (strings.size() == 1) { return std::string{strings[0].begin(), strings[0].end()}; } - - auto total_size = std::transform_reduce( - strings.begin(), - strings.end(), - size_t{0}, - [](size_t total, size_t str_size) { return total + str_size; }, - [](auto& str) { return str.size(); }); - - auto separator_size = separator.size() * (strings.size() - 1); - - std::string result; - result.reserve(total_size + separator_size); - - for (size_t i = 0; i < strings.size(); ++i) { - result.append(strings[i].begin(), strings[i].end()); - if (i != (strings.size() - 1)) { result.append(separator); } - } - - return result; -} - -} // namespace - -void log_warning(std::string_view msg) -{ - std::fprintf(stderr, "[rtcx] warn: %.*s\n", static_cast(msg.size()), msg.data()); -} - -void log_error(std::string_view msg) -{ - std::fprintf(stderr, "[rtcx] error: %.*s\n", static_cast(msg.size()), msg.data()); -} - -#define FOR_EACH_CUDA_FUNC(DO_IT) \ - DO_IT(GetErrorString) \ - DO_IT(GetErrorName) \ - DO_IT(Init) \ - DO_IT(OccupancyMaxPotentialBlockSize) \ - DO_IT(LaunchKernel) \ - DO_IT(LaunchKernelEx) \ - DO_IT(LaunchCooperativeKernel) \ - DO_IT(KernelGetFunction) \ - DO_IT(LibraryLoadData) \ - DO_IT(LibraryLoadFromFile) \ - DO_IT(LibraryGetKernel) \ - DO_IT(LibraryUnload) - -#define FOR_EACH_NVRTC_FUNC(DO_IT) \ - DO_IT(Version) \ - DO_IT(GetErrorString) \ - DO_IT(CreateProgram) \ - DO_IT(DestroyProgram) \ - DO_IT(CompileProgram) \ - DO_IT(GetPTXSize) \ - DO_IT(GetPTX) \ - DO_IT(GetCUBINSize) \ - DO_IT(GetCUBIN) \ - DO_IT(GetLTOIRSize) \ - DO_IT(GetLTOIR) \ - DO_IT(GetProgramLogSize) \ - DO_IT(GetProgramLog) \ - DO_IT(AddNameExpression) \ - DO_IT(GetLoweredName) - -#define FOR_EACH_NVJITLINK_FUNC(DO_IT) \ - DO_IT(Version) \ - DO_IT(Create) \ - DO_IT(Destroy) \ - DO_IT(AddData) \ - DO_IT(AddFile) \ - DO_IT(Complete) \ - DO_IT(GetLinkedCubinSize) \ - DO_IT(GetLinkedCubin) \ - DO_IT(GetLinkedPtxSize) \ - DO_IT(GetLinkedPtx) \ - DO_IT(GetErrorLogSize) \ - DO_IT(GetErrorLog) \ - DO_IT(GetInfoLog) \ - DO_IT(GetInfoLogSize) - -namespace { - -std::string_view nvJitLinkResult_string(nvJitLinkResult result) -{ - switch (result) { - case NVJITLINK_SUCCESS: return "NVJITLINK_SUCCESS"; - case NVJITLINK_ERROR_UNRECOGNIZED_OPTION: return "NVJITLINK_ERROR_UNRECOGNIZED_OPTION"; - case NVJITLINK_ERROR_MISSING_ARCH: return "NVJITLINK_ERROR_MISSING_ARCH"; - case NVJITLINK_ERROR_INVALID_INPUT: return "NVJITLINK_ERROR_INVALID_INPUT"; - case NVJITLINK_ERROR_PTX_COMPILE: return "NVJITLINK_ERROR_PTX_COMPILE"; - case NVJITLINK_ERROR_NVVM_COMPILE: return "NVJITLINK_ERROR_NVVM_COMPILE"; - case NVJITLINK_ERROR_INTERNAL: return "NVJITLINK_ERROR_INTERNAL"; - case NVJITLINK_ERROR_THREADPOOL: return "NVJITLINK_ERROR_THREADPOOL"; - case NVJITLINK_ERROR_UNRECOGNIZED_INPUT: return "NVJITLINK_ERROR_UNRECOGNIZED_INPUT"; - case NVJITLINK_ERROR_FINALIZE: return "NVJITLINK_ERROR_FINALIZE"; -#if CUDA_VERSION >= 13000 - case NVJITLINK_ERROR_NULL_INPUT: return "NVJITLINK_ERROR_NULL_INPUT"; - case NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS: return "NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS"; - case NVJITLINK_ERROR_INCORRECT_INPUT_TYPE: return "NVJITLINK_ERROR_INCORRECT_INPUT_TYPE"; - case NVJITLINK_ERROR_ARCH_MISMATCH: return "NVJITLINK_ERROR_ARCH_MISMATCH"; - case NVJITLINK_ERROR_OUTDATED_LIBRARY: return "NVJITLINK_ERROR_OUTDATED_LIBRARY"; - case NVJITLINK_ERROR_MISSING_FATBIN: return "NVJITLINK_ERROR_MISSING_FATBIN"; - case NVJITLINK_ERROR_UNRECOGNIZED_ARCH: return "NVJITLINK_ERROR_UNRECOGNIZED_ARCH"; - case NVJITLINK_ERROR_UNSUPPORTED_ARCH: return "NVJITLINK_ERROR_UNSUPPORTED_ARCH"; - case NVJITLINK_ERROR_LTO_NOT_ENABLED: return "NVJITLINK_ERROR_LTO_NOT_ENABLED"; -#endif - default: - RTCX_FAIL( - std::format("Unrecognized nvJitLinkResult type: ({})", static_cast(result)), - std::runtime_error); - } -} - -std::string_view binary_type_string(binary_type type) -{ - switch (type) { - case binary_type::LTO_IR: return "LTO_IR"; - case binary_type::CUBIN: return "CUBIN"; - case binary_type::FATBIN: return "FATBIN"; - case binary_type::PTX: return "PTX"; - default: - RTCX_FAIL(std::format("Unrecognized binary_type: ({})", static_cast(type)), - std::runtime_error); - } -} - -nvJitLinkInputType to_nvjitlink_input_type(binary_type bin_type) -{ - switch (bin_type) { - case binary_type::LTO_IR: return NVJITLINK_INPUT_LTOIR; - case binary_type::CUBIN: return NVJITLINK_INPUT_CUBIN; - case binary_type::FATBIN: return NVJITLINK_INPUT_FATBIN; - case binary_type::PTX: return NVJITLINK_INPUT_PTX; - default: - RTCX_FAIL(std::format("Unrecognized binary type for linking: ({}) ", - static_cast(bin_type)), - std::logic_error); - } -} - -[[maybe_unused]] void* load_dso(std::string_view base_name, std::span names) -{ - for (auto& name : names) { - void* handle = ::dlopen(name.c_str(), RTLD_NOW | RTLD_LOCAL); - if (handle != nullptr) { return handle; } - } - - RTCX_FAIL( - std::format( - "Failed to load dynamic library `{}` (tried: {})", base_name, join_strings(names, ", ")), - std::runtime_error); -} - -[[maybe_unused]] void* get_dso_symbol(char const* lib_name, void* handle, char const* sym_name) -{ - void* sym = ::dlsym(handle, sym_name); - if (sym == nullptr) { - RTCX_FAIL( - std::format( - "Failed to load symbol `{}` from `{}`, error: `{}`", sym_name, lib_name, ::dlerror()), - std::runtime_error); - } - return sym; -} - -inline constexpr std::int32_t major_version(std::int32_t version) { return version / 1000; } - -struct LibCuda { - void* _handle = nullptr; - -#define DO_IT(func) decltype(::cu##func)* func = nullptr; - FOR_EACH_CUDA_FUNC(DO_IT) -#undef DO_IT - - explicit LibCuda(void* handle) : _handle(handle) { _load_symbols(); } - LibCuda(LibCuda const&) = delete; - LibCuda(LibCuda&&) = delete; - LibCuda& operator=(LibCuda const&) = delete; - LibCuda& operator=(LibCuda&&) = delete; - ~LibCuda() { ::dlclose(_handle); } - - static void* _load() - { - std::string lib_names[] = {"libcuda.so.1"}; // NOLINT(modernize-avoid-c-arrays) - return load_dso("libcuda.so", lib_names); - } - - private: - void _load_symbols() - { -#define DO_IT(func) \ - this->func = \ - reinterpret_cast(get_dso_symbol("libcuda", _handle, "cu" #func)); - - FOR_EACH_CUDA_FUNC(DO_IT) - -#undef DO_IT - } -}; - -struct LibNVRTC { - void* _handle = nullptr; - -#define DO_IT(func) decltype(::nvrtc##func)* func = nullptr; - FOR_EACH_NVRTC_FUNC(DO_IT) -#undef DO_IT - - explicit LibNVRTC(void* handle) : _handle(handle) { _load_symbols(); } - LibNVRTC(LibNVRTC const&) = delete; - LibNVRTC(LibNVRTC&&) = delete; - LibNVRTC& operator=(LibNVRTC const&) = delete; - LibNVRTC& operator=(LibNVRTC&&) = delete; - ~LibNVRTC() - { -#if !RTCX_STATIC_LINK_LIBNVRTC - ::dlclose(_handle); -#endif - } - - static void* _load() - { -#if !RTCX_STATIC_LINK_LIBNVRTC - auto expected_major_version = major_version(CUDA_VERSION); - std::int32_t cuda_version; - RTCX_CHECK_CUDART(::cudaRuntimeGetVersion(&cuda_version)); - std::int32_t major = major_version(cuda_version); - RTCX_EXPECTS(expected_major_version == major, - std::format("LibNVRTC Compatibility Error: CUDA major version mismatch. Expected " - "major runtime version: {}, got major runtime version: {})", - expected_major_version, - major), - std::runtime_error); - - std::string lib_names[] = // NOLINT(modernize-avoid-c-arrays) - {std::format("libnvrtc.so.{}", major)}; - - return load_dso("libnvrtc.so", lib_names); -#else - return nullptr; -#endif - } - - private: - void _load_symbols() - { -#if !RTCX_STATIC_LINK_LIBNVRTC -#define DO_IT(func) \ - this->func = \ - reinterpret_cast(get_dso_symbol("libnvrtc", _handle, "nvrtc" #func)); -#else -#define DO_IT(func) this->func = ::nvrtc##func; -#endif - - FOR_EACH_NVRTC_FUNC(DO_IT) -#undef DO_IT - } -}; - -struct LibNVJitLink { - void* _handle = nullptr; - -#define DO_IT(func) decltype(::nvJitLink##func)* func = nullptr; - FOR_EACH_NVJITLINK_FUNC(DO_IT) -#undef DO_IT - - explicit LibNVJitLink(void* handle) : _handle(handle) { _load_symbols(); } - LibNVJitLink(LibNVJitLink const&) = delete; - LibNVJitLink(LibNVJitLink&&) = delete; - LibNVJitLink& operator=(LibNVJitLink const&) = delete; - LibNVJitLink& operator=(LibNVJitLink&&) = delete; - ~LibNVJitLink() - { -#if !RTCX_STATIC_LINK_LIBNVJITLINK - ::dlclose(_handle); -#endif - } - - static void* _load() - { -#if !RTCX_STATIC_LINK_LIBNVJITLINK - auto expected_major_version = major_version(CUDA_VERSION); - std::int32_t cuda_version; - RTCX_CHECK_CUDART(::cudaRuntimeGetVersion(&cuda_version)); - std::int32_t major = major_version(cuda_version); - RTCX_EXPECTS( - expected_major_version == major, - std::format("LibNVJitLink Compatibility Error: CUDA major version mismatch. Expected " - "major runtime version: {}, got major runtime version: {})", - expected_major_version, - major), - std::runtime_error); - - std::string lib_names[] = // NOLINT(modernize-avoid-c-arrays) - {std::format("libnvJitLink.so.{}", major)}; - - return load_dso("libnvJitLink.so", lib_names); -#else - return nullptr; -#endif - } - - private: - void _load_symbols() - { -#if !RTCX_STATIC_LINK_LIBNVJITLINK -#define DO_IT(func) \ - this->func = reinterpret_cast( \ - get_dso_symbol("libnvJitLink", _handle, "nvJitLink" #func)); -#else -#define DO_IT(func) this->func = ::nvJitLink##func; -#endif - - FOR_EACH_NVJITLINK_FUNC(DO_IT) -#undef DO_IT - } -}; - -static std::optional cu; -static std::optional nvrtc; -static std::optional nvjitlink; -static std::optional init_libraries_flag{std::in_place}; -static std::optional teardown_libraries_flag{std::in_place}; - -} // namespace - -void initialize() -{ - RTCX_FUNC_RANGE(); - - std::call_once(init_libraries_flag.value(), [] { - cu.emplace(LibCuda::_load()); - RTCX_EXPECTS( - cu->Init(0) == CUDA_SUCCESS, "Failed to initialize CUDA driver API", std::runtime_error); - nvrtc.emplace(LibNVRTC::_load()); - nvjitlink.emplace(LibNVJitLink::_load()); - }); -} - -void teardown() -{ - RTCX_FUNC_RANGE(); - - std::call_once(teardown_libraries_flag.value(), [] { - nvjitlink.reset(); - nvrtc.reset(); - cu.reset(); - init_libraries_flag.reset(); - teardown_libraries_flag.reset(); - init_libraries_flag.emplace(); - teardown_libraries_flag.emplace(); - }); -} - -blob_t blob_t::from_buffer(byte_buffer buffer) -{ - auto size = buffer.size(); - auto data = buffer.release(); - return blob_t::from_parts( - data, size, +[](std::uint8_t const* data, std::size_t) { - ::free(const_cast(data)); - }); -} - -blob_t blob_t::from_static_data(std::span data) -{ - return blob_t::from_parts(data.data(), data.size(), blob_t::noop_deallocator); -} - -namespace { -void log_nvrtc_result(compile_params const& params, - nvrtcProgram program, - nvrtcResult compile_result) -{ - if (program == nullptr) { return; } - - std::size_t log_size; - if (auto errc = nvrtc->GetProgramLogSize(program, &log_size); errc != NVRTC_SUCCESS) { - RTCX_FAIL(std::format("Failed to get NVRTC program log size with error ({}): {}", - static_cast(errc), - nvrtc->GetErrorString(errc)), - std::runtime_error); - } - - std::vector log; - - if (log_size > 1) { - log.resize(log_size); - if (auto errc = nvrtc->GetProgramLog(program, log.data()); errc != NVRTC_SUCCESS) { - RTCX_FAIL(std::format("Failed to get NVRTC program log with error ({}): {}", - static_cast(errc), - nvrtc->GetErrorString(errc)), - std::runtime_error); - } - } - - log.resize(log_size == 0 ? 0 : (log_size - 1)); - - auto status_str = - (compile_result == NVRTC_SUCCESS && !log.empty()) ? "completed with" : "failed with"; - - std::string headers_str; - for (auto& header : params.header_include_names) { - headers_str = std::format("{}\t{}\n", headers_str, header); - } - - std::string options_str; - for (auto& option : params.options) { - options_str = std::format("{}\t{}\n", options_str, option); - } - - if (log.empty()) { return; } - - auto msg = std::format( - "NVRTC Compilation for `{}` {} ({}): {}.\nHeaders:\n{}\n\nOptions:\n{}\n\nLog:\n\t{}", - params.name == nullptr ? "" : params.name, - status_str, - static_cast(compile_result), - nvrtc->GetErrorString(compile_result), - headers_str, - options_str, - std::string_view{log.data(), log.size()}); - - if (compile_result != NVRTC_SUCCESS) { - log_error(msg); - } else { - log_warning(msg); - } -} - -void log_nvJitLink_result(link_params const& params, - nvJitLinkHandle handle, - nvJitLinkResult link_result) -{ - if (handle == nullptr) { return; } - - std::size_t info_log_size; - if (auto errc = nvjitlink->GetInfoLogSize(handle, &info_log_size); errc != NVJITLINK_SUCCESS) { - RTCX_FAIL(std::format("Failed to get nvJitLink info log size with error ({}): {}", - static_cast(errc), - nvJitLinkResult_string(errc)), - std::runtime_error); - } - - std::vector info_log; - if (info_log_size > 1) { - info_log.resize(info_log_size); - if (auto errc = nvjitlink->GetInfoLog(handle, info_log.data()); errc != NVJITLINK_SUCCESS) { - RTCX_FAIL(std::format("Failed to get nvJitLink info log with error ({}): {}", - static_cast(errc), - nvJitLinkResult_string(errc)), - std::runtime_error); - } - } - - info_log.resize(info_log_size == 0 ? 0 : (info_log_size - 1)); - - std::size_t error_log_size; - if (auto errc = nvjitlink->GetErrorLogSize(handle, &error_log_size); errc != NVJITLINK_SUCCESS) { - RTCX_FAIL(std::format("Failed to get nvJitLink error log size with error ({}): {}", - static_cast(errc), - nvJitLinkResult_string(errc)), - std::runtime_error); - } - - std::vector error_log; - - if (error_log_size > 1) { - error_log.resize(error_log_size); - if (auto errc = nvjitlink->GetErrorLog(handle, error_log.data()); errc != NVJITLINK_SUCCESS) { - RTCX_FAIL(std::format("Failed to get nvJitLink error log with error ({}): {}", - static_cast(errc), - nvJitLinkResult_string(errc)), - std::runtime_error); - } - } - - error_log.resize(error_log_size == 0 ? 0 : (error_log_size - 1)); - - if (info_log.empty() && error_log.empty()) { return; } - - std::string fragments_str; - for (auto& frag : params.file_fragments) { - fragments_str = std::format("{}\t{}\n", fragments_str, frag.path); - } - - for (auto& frag : params.memory_fragments) { - fragments_str = - std::format("{}\t{}\n", fragments_str, frag.name == nullptr ? "" : frag.name); - } - - std::string link_options_str; - for (auto& option : params.link_options) { - link_options_str = std::format("{}\t{}\n", link_options_str, option); - } - - auto status_str = link_result == NVJITLINK_SUCCESS ? "completed with" : "failed with"; - - auto msg = std::format( - "(nvJitLink) Linking for `{}` ({}) {} error code ({}): {}.\nFragments: \n{}\n" - "Link Options: \n{}\n\nInfo Log:\n\t{}\n\nError Log:\n\t{}\n\n", - params.name == nullptr ? "" : params.name, - binary_type_string(params.output_type), - status_str, - static_cast(link_result), - nvJitLinkResult_string(link_result), - fragments_str, - link_options_str, - std::string_view{info_log.data(), info_log.size()}, - std::string_view{error_log.data(), error_log.size()}); - - bool needs_info_log = - std::find_if( - params.link_options.begin(), params.link_options.end(), [](std::string_view option) { - return option == "--verbose" || option == "-time"; - }) != params.link_options.end(); - - if (link_result == NVJITLINK_SUCCESS) { - if (needs_info_log) { log_warning(msg); } - } else { - log_error(msg); - } -} - -} // namespace - -std::int32_t nvrtc_version() -{ - RTCX_FUNC_RANGE(); - - std::int32_t major, minor; - RTCX_CHECK_NVRTC(nvrtc->Version(&major, &minor)); - return major * 1000 + minor * 10; -} - -std::int32_t nvjitlink_version() -{ - RTCX_FUNC_RANGE(); - - std::uint32_t major, minor; - RTCX_CHECK_NVJITLINK(nvjitlink->Version(&major, &minor)); - return static_cast(major * 1000 + minor * 10); -} - -byte_buffer compile(compile_params const& params) -{ - RTCX_FUNC_RANGE(); - - RTCX_EXPECTS(params.name != nullptr, "Fragment name must not be null", std::logic_error); - RTCX_EXPECTS(params.source != nullptr, "Fragment source must not be null", std::logic_error); - - nvrtcProgram program = nullptr; - RTCX_CHECK_NVRTC(nvrtc->CreateProgram(&program, - params.source, - params.name, - static_cast(params.headers.size()), - params.headers.data(), - params.header_include_names.data())); - - RTCX_DEFER([&] { nvrtc->DestroyProgram(&program); }); - - for (auto* name_expr : params.name_expressions) { - RTCX_CHECK_NVRTC(nvrtc->AddNameExpression(program, name_expr)); - } - - auto compile_result = nvrtc->CompileProgram( - program, static_cast(params.options.size()), params.options.data()); - log_nvrtc_result(params, program, compile_result); - RTCX_CHECK_NVRTC(compile_result); - - switch (params.target_type) { - case binary_type::CUBIN: { - std::size_t cubin_size; - RTCX_CHECK_NVRTC(nvrtc->GetCUBINSize(program, &cubin_size)); - auto cubin = byte_buffer::make(cubin_size); - RTCX_CHECK_NVRTC(nvrtc->GetCUBIN(program, reinterpret_cast(cubin.data()))); - return cubin; - } break; - case binary_type::LTO_IR: { - std::size_t lto_ir_size; - RTCX_CHECK_NVRTC(nvrtc->GetLTOIRSize(program, <o_ir_size)); - auto lto_ir = byte_buffer::make(lto_ir_size); - RTCX_CHECK_NVRTC(nvrtc->GetLTOIR(program, reinterpret_cast(lto_ir.data()))); - return lto_ir; - } break; - case binary_type::PTX: { - std::size_t ptx_size; - RTCX_CHECK_NVRTC(nvrtc->GetPTXSize(program, &ptx_size)); - auto ptx = byte_buffer::make(ptx_size); - RTCX_CHECK_NVRTC(nvrtc->GetPTX(program, reinterpret_cast(ptx.data()))); - return ptx; - } break; - default: - RTCX_FAIL(std::format("Unsupported binary type for compiling fragment: {}", - binary_type_string(params.target_type)), - std::logic_error); - } -} - -kernel_occupancy_config kernel_ref::max_occupancy_config(std::size_t dynamic_shared_memory_bytes, - std::int32_t block_size_limit) const -{ - std::int32_t min_grid_size; - std::int32_t block_size; - RTCX_CHECK_CUDA(cu->OccupancyMaxPotentialBlockSize(&min_grid_size, - &block_size, - reinterpret_cast(handle_), - nullptr, - dynamic_shared_memory_bytes, - block_size_limit)); - - return kernel_occupancy_config{.min_grid_size = static_cast(min_grid_size), - .block_size = static_cast(block_size)}; -} - -void kernel_ref::launch(cuda_dim3 grid_dim, - cuda_dim3 block_dim, - std::uint32_t shared_mem_bytes, - CUstream stream, - void** kernel_params) const -{ - RTCX_FUNC_RANGE(); - - RTCX_EXPECTS(grid_dim.is_valid(), "Grid dimensions must be greater than zero", std::logic_error); - RTCX_EXPECTS( - block_dim.is_valid(), "Block dimensions must be greater than zero", std::logic_error); - RTCX_EXPECTS( - kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); - - CUlaunchConfig cfg{.gridDimX = grid_dim.x, - .gridDimY = grid_dim.y, - .gridDimZ = grid_dim.z, - .blockDimX = block_dim.x, - .blockDimY = block_dim.y, - .blockDimZ = block_dim.z, - .sharedMemBytes = shared_mem_bytes, - .hStream = stream, - .attrs = nullptr, - .numAttrs = 0}; - - RTCX_CHECK_CUDA( - cu->LaunchKernelEx(&cfg, reinterpret_cast(handle_), kernel_params, nullptr)); -} - -void kernel_ref::launch_cooperative(cuda_dim3 grid_dim, - cuda_dim3 block_dim, - std::uint32_t shared_mem_bytes, - CUstream stream, - void** kernel_params) const -{ - RTCX_FUNC_RANGE(); - - RTCX_EXPECTS(grid_dim.is_valid(), "Grid dimensions must be greater than zero", std::logic_error); - RTCX_EXPECTS( - block_dim.is_valid(), "Block dimensions must be greater than zero", std::logic_error); - RTCX_EXPECTS( - kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); - - RTCX_CHECK_CUDA(cu->LaunchCooperativeKernel(reinterpret_cast(handle_), - grid_dim.x, - grid_dim.y, - grid_dim.z, - block_dim.x, - block_dim.y, - block_dim.z, - shared_mem_bytes, - stream, - kernel_params)); -} - -library_t::~library_t() -{ - if (handle_ != nullptr) { cu->LibraryUnload(handle_); } -} - -library load_library(std::span binary) -{ - RTCX_FUNC_RANGE(); - - CUlibrary handle; - - RTCX_CHECK_CUDA( - cu->LibraryLoadData(&handle, binary.data(), nullptr, nullptr, 0, nullptr, nullptr, 0)); - - RTCX_DEFER([&] { - if (handle != nullptr) { RTCX_CHECK_CUDA(cu->LibraryUnload(handle)); } - }); - - auto library = std::make_shared(handle); - - handle = nullptr; - - return library; -} - -library load_library_from_file(char const* path) -{ - RTCX_FUNC_RANGE(); - - RTCX_EXPECTS(path != nullptr, "Library path must not be null", std::logic_error); - - CUlibrary handle; - - RTCX_CHECK_CUDA(cu->LibraryLoadFromFile(&handle, path, nullptr, nullptr, 0, nullptr, nullptr, 0)); - - RTCX_DEFER([&] { - if (handle != nullptr) { RTCX_CHECK_CUDA(cu->LibraryUnload(handle)); } - }); - - auto library = std::make_shared(handle); - - handle = nullptr; - - return library; -} - -byte_buffer link_library(link_params const& params) -{ - RTCX_FUNC_RANGE(); - - RTCX_EXPECTS(params.name != nullptr, "Link output name must not be null", std::logic_error); - RTCX_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, - "Only CUBIN and PTX output types are supported for linking modules", - std::logic_error); - RTCX_EXPECTS(params.file_fragments.size() != 0 || params.memory_fragments.size() != 0, - "At least one fragment must be provided for linking", - std::logic_error); - - for (auto& frag : params.file_fragments) { - RTCX_EXPECTS(frag.path != nullptr, "Fragment file path must not be empty", std::logic_error); - } - - for (auto& frag : params.memory_fragments) { - RTCX_EXPECTS( - frag.data.size_bytes() > 0, "Fragment binary data must be non-empty", std::logic_error); - } - - nvJitLinkHandle handle = nullptr; - RTCX_CHECK_NVJITLINK(nvjitlink->Create(&handle, - static_cast(params.link_options.size()), - const_cast(params.link_options.data()))); - - RTCX_DEFER([&] { nvjitlink->Destroy(&handle); }); - - for (auto& frag : params.file_fragments) { - RTCX_CHECK_NVJITLINK(nvjitlink->AddFile(handle, to_nvjitlink_input_type(frag.type), frag.path)); - } - - for (auto& frag : params.memory_fragments) { - RTCX_CHECK_NVJITLINK(nvjitlink->AddData(handle, - to_nvjitlink_input_type(frag.type), - frag.data.data(), - frag.data.size_bytes(), - frag.name)); - } - - auto link_result = nvjitlink->Complete(handle); - log_nvJitLink_result(params, handle, link_result); - RTCX_CHECK_NVJITLINK(link_result); - - switch (params.output_type) { - case binary_type::CUBIN: { - std::size_t cubin_size; - RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedCubinSize(handle, &cubin_size)); - auto cubin = byte_buffer::make(cubin_size); - RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedCubin(handle, cubin.data())); - return cubin; - } break; - case binary_type::PTX: { - std::size_t ptx_size; - RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedPtxSize(handle, &ptx_size)); - auto ptx = byte_buffer::make(ptx_size); - RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedPtx(handle, reinterpret_cast(ptx.data()))); - return ptx; - } break; - default: - RTCX_FAIL(std::format("Unsupported output binary type for linking CUDA libraries: ({})", - binary_type_string(params.output_type)), - std::runtime_error); - } -} - -kernel_ref library_t::get_kernel(char const* name) const -{ - RTCX_FUNC_RANGE(); - - CUkernel kernel; - RTCX_CHECK_CUDA(cu->LibraryGetKernel(&kernel, handle_, name)); - return kernel_ref{kernel}; -} - -namespace { - -[[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) -{ - auto errc = errno; - RTCX_FAIL( - std::format("{}. `{}` failed with {} ({})", message, syscall_name, errc, std::strerror(errc)), - std::runtime_error); -} - -} // namespace - -cache_t::cache_t(std::string cache_dir, - std::string tmp_dir, - cache_limits const& limits, - bool preload, - bool disable) - : enabled_{!disable}, - cache_dir_{std::move(cache_dir)}, - tmp_dir_{std::move(tmp_dir)}, - limits_{limits}, - lock_{}, - blobs_cache_{limits.num_mem_blobs}, - libraries_cache_{limits.num_mem_libraries}, - tick_{0} -{ - if (preload) { preload_from_disk(); } -} - -std::string const& cache_t::get_cache_dir() { return cache_dir_; } - -std::string const& cache_t::get_tmp_dir() { return tmp_dir_; } - -std::optional blob_t::from_file(char const* path) -{ - std::int32_t fd = ::open(path, O_RDONLY); - - if (fd == -1) { - if (errno == ENOENT) { - return std::nullopt; - } else { - throw_posix("Failed to open RTCX cache file from disk", "open"); - } - } - - RTCX_DEFER([&] { - if (::close(fd) == -1) { - throw_posix("Failed to close RTCX cache file after memory-mapping", "close"); - } - }); - - auto file_size = ::lseek(fd, 0, SEEK_END); - if (file_size == -1) { throw_posix("Failed to determine size of RTCX cache file", "lseek"); } - - if (file_size == 0) { - // mmap does not support mapping zero-length files, so we return an empty blob in this case - return blob_t::from_static_data({}); - } - - void* map = ::mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); - - if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTCX cache file", "mmap"); } - - auto deleter = +[](std::uint8_t const* buffer, std::size_t size) { - if (::munmap(static_cast(const_cast(buffer)), size) == -1) { - throw_posix("Failed to unmap RTCX cache file from memory", "munmap"); - } - }; - - return blob_t::from_parts(static_cast(map), file_size, deleter); -} - -namespace { - -/// @brief retrieves a blob from disk based on the given hash and object type (e.g. "blob", -/// "cuLibrary"). Returns nullopt if the file doesn't exist on disk, and throws if any other error -/// occurs. -std::optional get_disk_blob(std::string const& cache_dir, - object_type type, - hash128 const& hash) -{ - auto hex = hash.to_hex_string(); - auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(type)); - auto blob = blob_t::from_file(path.c_str()); - - if (!blob.has_value()) { return std::nullopt; } - return std::make_shared(std::move(*blob)); -} - -std::optional get_disk_library(std::string const& cache_dir, hash128 const& hash) -{ - RTCX_FUNC_RANGE(); - - auto hex = hash.to_hex_string(); - auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(object_type::LIBRARY)); - - // WAR: avoid a driver API call when the cache file is not present - // compute-sanitizer doesn't properly handle exceptions thrown from driver API calls, so we need - // to check for file existence first to avoid false positives in the sanitizer - if (!std::filesystem::exists(path)) { return std::nullopt; } - - CUlibrary handle; - auto errc = - cu->LibraryLoadFromFile(&handle, path.c_str(), nullptr, nullptr, 0, nullptr, nullptr, 0); - - if (errc == CUDA_ERROR_FILE_NOT_FOUND) { return std::nullopt; } - - RTCX_EXPECTS(errc == CUDA_SUCCESS, - std::format("Failed to load library `{}` from RTCX cache file", path), - std::runtime_error); - - return std::make_shared(handle); -} - -std::vector get_disk_entries(std::string const& cache_dir) -{ - std::vector entries; - for (auto& entry : std::filesystem::directory_iterator(cache_dir)) { - if (!entry.is_regular_file()) { continue; } - entries.push_back(entry.path().string()); - } - - return entries; -} - -/// @brief atomically writes a blob to disk by first writing to a temporary file and then renaming -/// it to the final path. -void cache_blob_to_disk(std::string const& cache_dir, - std::string const& tmp_dir, - object_type type, - hash128 const& hash, - std::span binary) -{ - RTCX_FUNC_RANGE(); - - auto tmp_path = std::format("{}/rtcx-bin-XXXXXX", tmp_dir); - (void)tmp_path.c_str(); // to ensure null-termination for mkstemp - - { - std::int32_t fd = ::mkstemp(tmp_path.data()); - if (fd == -1) { throw_posix("Failed to create temporary file for RTCX cache", "mkstemp"); } - - RTCX_DEFER([&] { - if (::close(fd) == -1) { throw_posix("Failed to close temporary RTCX cache file", "close"); } - }); - - { - auto ptr = binary.data(); - auto remaining = binary.size(); - while (remaining > 0) { - auto written = ::write(fd, ptr, remaining); - if (written == -1) { throw_posix("Failed to write RTCX cache to temporary file", "write"); } - ptr += static_cast(written); - remaining -= static_cast(written); - } - } - } - - auto hex = hash.to_hex_string(); - auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(type)); - - std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); - - // rename is atomic, even if another process is performing the same operation - if (::rename(tmp_path.c_str(), final_path.c_str()) == -1) { - if (errno == EEXIST) { - // another process has already created the file, so just remove our temp file - if (::remove(tmp_path.c_str()) == -1) { - throw_posix("Failed to remove temporary RTCX cache file", "remove"); - } - return; - } else { - throw_posix( - std::format("Failed to move temporary RTCX cache file to final location ({})", final_path), - "rename"); - } - } -} - -} // namespace - -std::shared_future cache_t::get_or_add_blob(hash128 const& hash, blob_compile_func compile) -{ - RTCX_FUNC_RANGE(); - - std::atomic_ref tick{tick_}; - auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); - - std::unique_lock lock{lock_}; - - // check memory cache - if (auto it = enabled_ ? blobs_cache_.entries_.find(hash) : blobs_cache_.entries_.end(); - it != blobs_cache_.entries_.end()) { - counter_.blob_mem_hits.incr(); - - // update LRU tick - it->second.hit(current_tick); - - return it->second.value; - - } else { - counter_.blob_mem_misses.incr(); - - // check disk cache - std::optional disk_blob = std::nullopt; - if (enabled_) { disk_blob = get_disk_blob(cache_dir_, object_type::BLOB, hash); } - - std::promise promise; - auto fut = promise.get_future().share(); - auto cache_fut = fut; - auto ret_fut = fut; - - if (disk_blob.has_value()) { - counter_.blob_disk_hits.incr(); - - promise.set_value(std::move(*disk_blob)); - - // insert into cache - blobs_cache_.insert(hash, std::move(cache_fut), current_tick); - - return ret_fut; - - } else { - counter_.blob_disk_misses.incr(); - - blobs_cache_.insert(hash, std::move(cache_fut), current_tick); - - // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this hash - lock.unlock(); - - auto result = compile(); - promise.set_value(result); - - cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::BLOB, hash, result->view()); - - return ret_fut; - } - } -} - -std::shared_future cache_t::get_or_add_library(hash128 const& hash, - library_compile_func compile) -{ - RTCX_FUNC_RANGE(); - - std::atomic_ref tick{tick_}; - auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); - - std::unique_lock lock{lock_}; - - // check memory cache - if (auto it = enabled_ ? libraries_cache_.entries_.find(hash) : libraries_cache_.entries_.end(); - it != libraries_cache_.entries_.end()) { - counter_.library_mem_hits.incr(); - - // update LRU tick - it->second.hit(current_tick); - - return it->second.value; - - } else { - counter_.library_mem_misses.incr(); - - // check disk cache - std::optional disk_library = std::nullopt; - if (enabled_) { disk_library = get_disk_library(cache_dir_, hash); } - - std::promise promise; - auto fut = promise.get_future().share(); - auto cache_fut = fut; - auto ret_fut = fut; - - if (disk_library.has_value()) { - counter_.library_disk_hits.incr(); - - libraries_cache_.insert(hash, std::move(cache_fut), current_tick); - - // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this hash - lock.unlock(); - - promise.set_value(std::move(*disk_library)); - - return ret_fut; - - } else { - counter_.library_disk_misses.incr(); - - libraries_cache_.insert(hash, std::move(cache_fut), current_tick); - - // we can release the lock while calling the maker function since it may be expensive and we - // have already reserved a spot in the cache for this hash - lock.unlock(); - - auto [library, blob] = compile(); - promise.set_value(library); - - // store result to disk - cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::LIBRARY, hash, blob->view()); - - return ret_fut; - } - } -} - -cache_stats cache_t::get_stats() -{ - return cache_stats{.blob_mem_hits = counter_.blob_mem_hits.get(), - .blob_mem_misses = counter_.blob_mem_misses.get(), - .blob_disk_hits = counter_.blob_disk_hits.get(), - .blob_disk_misses = counter_.blob_disk_misses.get(), - .library_mem_hits = counter_.library_mem_hits.get(), - .library_mem_misses = counter_.library_mem_misses.get(), - .library_disk_hits = counter_.library_disk_hits.get(), - .library_disk_misses = counter_.library_disk_misses.get()}; -} - -void cache_t::clear_stats() -{ - counter_.blob_mem_hits.reset(); - counter_.blob_mem_misses.reset(); - counter_.blob_disk_hits.reset(); - counter_.blob_disk_misses.reset(); - counter_.library_mem_hits.reset(); - counter_.library_mem_misses.reset(); - counter_.library_disk_hits.reset(); - counter_.library_disk_misses.reset(); -} - -cache_limits cache_t::get_limits() { return limits_; } - -std::size_t cache_t::get_blob_count() -{ - std::lock_guard guard{lock_}; - return blobs_cache_.entries_.size(); -} - -std::size_t cache_t::get_library_count() -{ - std::lock_guard guard{lock_}; - return libraries_cache_.entries_.size(); -} - -void cache_t::clear_memory_store() -{ - RTCX_FUNC_RANGE(); - - std::lock_guard guard{lock_}; - - blobs_cache_.entries_.clear(); - libraries_cache_.entries_.clear(); -} - -void cache_t::clear_disk_store() -{ - RTCX_FUNC_RANGE(); - - auto entries = get_disk_entries(cache_dir_); - - for (auto const& path : entries) { - try { - std::filesystem::remove(path); - } catch (...) { - log_error(std::format("Unknown error occurred while removing RTCX cache file `{}`", path)); - } - } -} - -void cache_t::preload_from_disk() -{ - RTCX_FUNC_RANGE(); - - auto entries = get_disk_entries(cache_dir_); - - auto load_count = - std::min(entries.size(), limits_.num_mem_blobs + limits_.num_mem_libraries); - - entries.resize(load_count); - - { - std::lock_guard guard{lock_}; - tick_++; - - for (auto const& path : entries) { - try { - auto file_name = std::filesystem::path{path}.filename().string(); - auto hash_str = file_name.substr(0, file_name.find('.')); - auto hash = hash128::parse(hash_str); - - if (path.ends_with(".blob.bin")) { - auto data = blob_t::from_file(path.c_str()); - if (!data.has_value()) { continue; } - auto blob = std::make_shared(std::move(*data)); - std::promise promise; - auto fut = promise.get_future().share(); - promise.set_value(std::move(blob)); - blobs_cache_.insert(hash, std::move(fut), tick_); - } else if (path.ends_with(".cuLibrary.bin")) { - auto lib = get_disk_library(cache_dir_, hash); - if (!lib.has_value()) { continue; } - std::promise promise; - auto fut = promise.get_future().share(); - promise.set_value(std::move(*lib)); - libraries_cache_.insert(hash, std::move(fut), tick_); - } - } catch (std::exception const& e) { - // ignore any errors during preload - log_error(e.what()); - } catch (...) { - log_error("Unknown error during preload"); - } - } - } -} - -void cache_t::enable(bool enable) -{ - std::lock_guard guard{lock_}; - enabled_ = enable; -} - -bool cache_t::is_enabled() -{ - std::lock_guard guard{lock_}; - return enabled_; -} - -std::string reflect_template(std::string_view template_name, - std::span template_args) -{ - return std::format("{}<{}>", template_name, join_strings(template_args, ", ")); -} - -std::string reflect_template(std::string_view template_name, - std::span template_args) -{ - return std::format("{}<{}>", template_name, join_strings(template_args, ", ")); -} - -rtcx::byte_buffer decompress_blob(std::span compressed_binary, - std::size_t uncompressed_size, - std::string_view compression) -{ - RTCX_FUNC_RANGE(); - - RTCX_EXPECTS(compression == "none" || compression == "zstd", - std::format("Unsupported compression type specified: {}", compression), - std::runtime_error); - auto decompressed = rtcx::byte_buffer::make(uncompressed_size); - - if (compression == "zstd") { - std::size_t errc = ::ZSTD_decompress( - decompressed.data(), uncompressed_size, compressed_binary.data(), compressed_binary.size()); - - RTCX_EXPECTS( - !::ZSTD_isError(errc) && errc == uncompressed_size, - std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : {}", - errc, - ::ZSTD_getErrorName(errc)), - std::runtime_error); - } else { - // compression is "none", so just copy the data - std::copy(compressed_binary.data(), - compressed_binary.data() + compressed_binary.size(), - decompressed.data()); - } - - return decompressed; -} - -} // namespace rtcx diff --git a/cpp/librtcx/rtcx.hpp b/cpp/librtcx/rtcx.hpp deleted file mode 100644 index 3a9929c20301..000000000000 --- a/cpp/librtcx/rtcx.hpp +++ /dev/null @@ -1,1007 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define RTCX_DEFER__CONCATENATE_DETAIL(x, y) x##y -#define RTCX_DEFER__CONCATENATE(x, y) RTCX_DEFER__CONCATENATE_DETAIL(x, y) -#define RTCX_DEFER(...) ::rtcx::defer RTCX_DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) - -extern "C" { -typedef struct CUlib_st* CUlibrary; // NOLINT(modernize-use-using) -typedef struct CUkern_st* CUkernel; // NOLINT(modernize-use-using) -typedef struct CUstream_st* CUstream; // NOLINT(modernize-use-using) -} - -namespace rtcx { - -inline constexpr std::size_t CACHELINE_ALIGNMENT = - 64; // = std::hardware_destructive_interference_size */ - -/** - * @brief RAII utility to execute a callable at the end of a scope. - */ -template -struct defer { - private: - T func_; - - public: - template - defer(Args&&... args) : func_{static_cast(args)...} - { - } - defer(defer const&) = delete; - defer& operator=(defer const&) = delete; - defer(defer&&) = delete; - defer& operator=(defer&&) = delete; - ~defer() { func_(); } -}; - -template -defer(T) -> defer; - -template -struct func; - -/** - * @brief Zero-copy, type-erased reference to a callable entity (e.g. lambda, function pointer) that - * can be invoked with the given signature. - */ -template -struct [[nodiscard]] func { - private: - void* _user_data; - R (*_thunk)(void*, Args...); - - public: - func(void* user_data, R (*thunk)(void*, Args...)) : _user_data{user_data}, _thunk{thunk} {} - - func(R (*func_ptr)(Args...)) - : _user_data{reinterpret_cast(func_ptr)}, - _thunk{+[](void* user_data, Args... args) -> R { - auto func = reinterpret_cast(user_data); - return func(std::forward(args)...); - }} - { - } - - R operator()(Args... args) const { return _thunk(_user_data, std::forward(args)...); } - - template - static func from_functor(Lambda& lambda) - { - return func{static_cast(std::addressof(lambda)), - +[](void* user_data, Args... args) -> R { - auto& lambda = *static_cast*>(user_data); - return lambda(std::forward(args)...); - }}; - } -}; - -template -func(void*, R (*)(void*, Args...)) -> func; - -template -func(R (*)(Args...)) -> func; - -struct [[nodiscard]] hash128_hasher { - constexpr std::uint64_t operator()(hash128 const& obj) const - { - // use only the lower 64 bits of the hash for the hash table - return static_cast(obj.value); - } -}; - -enum class binary_type : std::int8_t { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; - -/** - * @brief A heap-allocated statically-sized buffer. Its contents are not guaranteed to be - * initialized. - */ -template - requires(!std::is_const_v && std::is_trivially_copyable_v && - std::is_trivially_destructible_v) -struct buffer { - private: - T* _data; - std::size_t _size; - - buffer(T* data, std::size_t size) : _data(data), _size(size) {} - - public: - /** - * @brief Creates a new buffer of the given size, with uninitialized contents. - */ - static buffer make(std::size_t size) - { - T* data = static_cast(malloc(size * sizeof(T))); - if (data == nullptr) { throw std::bad_alloc(); } - return buffer{data, size}; - } - - buffer() : buffer{nullptr, 0} {} //< Default constructor. Creates an empty buffer - - buffer(buffer const&) = delete; - - buffer& operator=(buffer const&) = delete; - - /** - * @brief Move constructor. Transfers ownership of the buffer from the source to the new object. - * After the move, the source buffer is left in an empty state (data pointer is null and size is - * zero). - */ - buffer(buffer&& other) noexcept : _data(other._data), _size(other._size) - { - other._data = nullptr; - other._size = 0; - } - - /** - * @brief Move assignment operator. Transfers ownership of the buffer from the source to the - * current object. - */ - buffer& operator=(buffer&& other) noexcept - { - if (this == &other) [[unlikely]] { return *this; } - this->~buffer(); - new (this) buffer(std::move(other)); - return *this; - } - - ~buffer() noexcept { free(_data); } - - /** - * @brief Returns a pointer to the buffer's data - * @return A pointer to the buffer's data - */ - [[nodiscard]] T* data() const { return _data; } - - /** - * @brief Returns the size of the buffer - * @return The size of the buffer in number of elements - */ - [[nodiscard]] std::size_t size() const { return _size; } - - /** - * @brief Returns an iterator to the beginning of the buffer - * @return An iterator to the beginning of the buffer - */ - [[nodiscard]] T* begin() { return _data; } - - /** - * @brief Returns an iterator to the end of the buffer - * @return An iterator to the end of the buffer - */ - [[nodiscard]] T* end() { return _data + _size; } - - /** - * @brief Returns a const iterator to the beginning of the buffer - * @return A const iterator to the beginning of the buffer - */ - [[nodiscard]] T const* begin() const { return _data; } - - /** - * @brief Returns a const iterator to the end of the buffer - * @return A const iterator to the end of the buffer - */ - [[nodiscard]] T const* end() const { return _data + _size; } - - /** - * @brief Returns a const iterator to the beginning of the buffer - * @return A const iterator to the beginning of the buffer - */ - [[nodiscard]] T const* cbegin() const { return _data; } - - /** - * @brief Returns a const iterator to the end of the buffer - * @return A const iterator to the end of the buffer - */ - [[nodiscard]] T const* cend() const { return _data + _size; } - - /** - * @brief Releases ownership of the buffer's data and returns a pointer to it. After calling this - * function, the buffer is left in an empty state. The caller is responsible for calling `free()` - * on the returned pointer when it is no longer needed. - * @return A pointer to the buffer's data - */ - [[nodiscard]] T* release() - { - T* data = _data; - _data = nullptr; - _size = 0; - return data; - } -}; - -using byte_buffer = buffer; - -/** - * @brief Represents an immutable blob view - * @details Manages the lifetime of the binary data via a user-provided deallocator function. This - * enables zero-copy view of binary data stored in various forms (e.g., std::vector, mmap'd file, - * etc.). - */ -struct [[nodiscard]] blob_t { - private: - using deallocator = func; - - static void noop_deallocator(std::uint8_t const*, std::size_t) {} - - std::uint8_t const* data_; - std::size_t size_; - deallocator deallocator_; - - blob_t(std::uint8_t const* data, std::size_t size, deallocator deallocator) - : data_(data), size_(size), deallocator_(deallocator) - { - } - - public: - blob_t() : data_(nullptr), size_(0), deallocator_(noop_deallocator) {} - - blob_t(blob_t const&) = delete; - blob_t& operator=(blob_t const&) = delete; - - blob_t(blob_t&& other) noexcept - : data_(other.data_), size_(other.size_), deallocator_(other.deallocator_) - { - other.data_ = nullptr; - other.size_ = 0; - other.deallocator_ = noop_deallocator; - } - - blob_t& operator=(blob_t&& other) noexcept - { - if (this == &other) [[unlikely]] { return *this; } - this->~blob_t(); - new (this) blob_t(std::move(other)); - return *this; - } - - ~blob_t() { deallocator_(data_, size_); } - - [[nodiscard]] std::span view() const { return {data_, size_}; } - - static blob_t from_parts(std::uint8_t const* data, std::size_t size, deallocator deallocator) - { - return blob_t{data, size, deallocator}; - } - - static blob_t from_buffer(byte_buffer buffer); - - static blob_t from_static_data(std::span data); - - static std::optional from_file(char const* path); -}; - -using blob = std::shared_ptr; - -/** - * @brief Represents the occupancy configuration for a kernel. This information can be used to - * optimize kernel launches for maximum performance on the GPU. - */ -struct [[nodiscard]] kernel_occupancy_config { - std::uint32_t min_grid_size = 0; //< Minimum grid size to achieve the maximum occupancy - std::uint32_t block_size = - 0; //< Number of threads per block to achieve the min_grid_size occupancy -}; - -/** - * @brief Represents the dimensions of a CUDA grid or block, with x, y, and z components. This - * struct is used to specify the configuration of kernel launches on the GPU. - */ -struct cuda_dim3 { - std::uint32_t x = 1; //< Value for the x dimension - std::uint32_t y = 1; //< Value for the y dimension - std::uint32_t z = 1; //< Value for the z dimension - - [[nodiscard]] constexpr bool is_valid() const { return x > 0 && y > 0 && z > 0; } -}; - -/** - * @brief Represents a compiled kernel that can be launched on the GPU. - */ -struct [[nodiscard]] kernel_ref { - private: - CUkernel handle_; - - public: - explicit kernel_ref(CUkernel handle) : handle_(handle) {} - - /** - * @brief Computes the maximum occupancy configuration for the kernel, given the specified dynamic - * shared memory usage and block size limit. This function queries the CUDA driver for the optimal - * block size and minimum grid size to achieve maximum occupancy of the kernel on the GPU. - */ - kernel_occupancy_config max_occupancy_config(std::size_t dynamic_shared_memory_bytes, - std::int32_t block_size_limit) const; - - /** - * @brief Launches the kernel on the GPU with the specified grid and block dimensions, - * dynamic shared memory size, stream, and kernel parameters. This function wraps the CUDA driver - * kernel launch API, providing a convenient interface for executing the kernel with the desired - * configuration. - * @param grid_dim The dimensions of the grid - * @param block_dim The dimensions of the block - * @param shared_mem_bytes The amount of dynamic shared memory (in bytes) to allocate for the - * kernel - * @param stream The CUDA stream on which to launch the kernel - * @param kernel_params A pointer to an array of pointers representing the kernel parameters to be - * passed to the kernel at launch time - */ - void launch(cuda_dim3 grid_dim, - cuda_dim3 block_dim, - std::uint32_t shared_mem_bytes, - CUstream stream, - void** kernel_params) const; - - /** - * @brief Launches the kernel on the GPU in cooperative mode with the specified grid and block - * dimensions, dynamic shared memory size, stream, and kernel parameters. This function wraps the - * CUDA driver cooperative kernel launch API, providing a convenient interface for executing the - * kernel with the desired configuration. - * @param grid_dim The dimensions of the grid - * @param block_dim The dimensions of the block - * @param shared_mem_bytes The amount of dynamic shared memory (in bytes) to allocate for the - * kernel - * @param stream The CUDA stream on which to launch the kernel - * @param kernel_params A pointer to an array of pointers representing the kernel parameters to be - * passed to the kernel at launch time - */ - void launch_cooperative(cuda_dim3 grid_dim, - cuda_dim3 block_dim, - std::uint32_t shared_mem_bytes, - CUstream stream, - void** kernel_params) const; - - /** - * @brief Retrieves the underlying CUDA kernel handle - * @return The CUDA kernel handle associated with this kernel reference - */ - [[nodiscard]] CUkernel get() const { return handle_; } -}; - -/** - * @brief Represents a loaded RTC library containing compiled kernels - */ -struct [[nodiscard]] library_t { - private: - CUlibrary handle_; - - public: - explicit library_t(CUlibrary handle) : handle_(handle) {} - library_t(library_t const&) = delete; - library_t(library_t&&) = delete; - library_t& operator=(library_t const&) = delete; - library_t& operator=(library_t&&) = delete; - ~library_t(); - - /** - * @brief Retrieves the underlying CUDA library handle - */ - [[nodiscard]] CUlibrary get() const { return handle_; } - - /** - * @brief Retrieve a kernel from the library by name - */ - [[nodiscard]] kernel_ref get_kernel(char const* name) const; -}; - -using library = std::shared_ptr; - -/** - * @brief Parameters for compiling source code into a binary blob using NVRTC - */ -struct [[nodiscard]] compile_params { - char const* name = nullptr; //< Debug name for the compilation unit - char const* source = nullptr; //< Source code to be compiled - std::span header_include_names = {}; //< Header file names - std::span headers = {}; //< Header file contents - std::span options = {}; //< NVRTC compilation options - std::span name_expressions = {}; //< Name expressions to be instantiated - binary_type target_type = binary_type::LTO_IR; //< Output binary type -}; - -/** - * @brief Represents a binary fragment in memory to be linked into a library - */ -struct memory_fragment { - std::span data = {}; //< Binary data for the fragment - binary_type type = binary_type::CUBIN; //< Binary type of the fragment data - char const* name = nullptr; //< Debug name for the fragment -}; - -/** - * @brief Represents a binary fragment to be linked into a library - */ -struct file_fragment { - char const* path = nullptr; //< Path to the binary fragment file - binary_type type = binary_type::CUBIN; //< Binary type of the fragment data -}; - -/** - * @brief Parameters for linking multiple compiled fragments into a single library - */ -struct [[nodiscard]] link_params { - char const* name = nullptr; //< Debug name for the linked library - binary_type output_type = binary_type::CUBIN; //< Output binary type - std::span file_fragments = {}; //< Binary data for each fragment - std::span memory_fragments = - {}; //< Memory-resident binary fragments to link - std::span link_options = {}; //< NVJITLink options -}; - -inline namespace detail { - -template -struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { - struct entry { - std::uint64_t last_touched_tick = 0; - T value; - - void hit(std::uint64_t tick) { last_touched_tick = tick; } - }; - - std::unordered_map entries_ = {}; - std::size_t limit_; - - explicit lru_memory_cache(std::size_t limit) : limit_{limit} - { - // reserve space to avoid rehashing - entries_.reserve(limit * 2); - } - - void purge() - { - if (entries_.empty()) { return; } - - auto num_to_purge = (entries_.size() + 1) / 2; - - std::vector> rankings; - rankings.reserve(entries_.size()); - - for (auto& [key, entry] : entries_) { - rankings.emplace_back(key, entry.last_touched_tick); - } - - std::sort( - rankings.begin(), rankings.end(), [](auto& a, auto& b) { return a.second < b.second; }); - - // purge least recently used half - rankings.resize(num_to_purge); - - for (auto [key, _] : rankings) { - entries_.erase(key); - } - } - - void insert(hash128 const& sha, T&& value, std::uint64_t tick) - { - if (limit_ == 0) { return; } - - if ((entries_.size() + 1) > limit_) { purge(); } - - entries_.emplace(sha, entry{tick, std::move(value)}); - } -}; - -struct cache_stats_counter { - struct entry { - std::uint64_t value_ = 0; - - void incr() - { - std::atomic_ref c{value_}; - c.fetch_add(1, std::memory_order_relaxed); - } - - [[nodiscard]] std::uint64_t get() const - { - std::atomic_ref c{value_}; - return c.load(std::memory_order_relaxed); - } - - void reset() - { - std::atomic_ref c{value_}; - c.store(0, std::memory_order_relaxed); - } - }; - - entry blob_mem_hits; - entry blob_mem_misses; - entry blob_disk_hits; - entry blob_disk_misses; - entry library_mem_hits; - entry library_mem_misses; - entry library_disk_hits; - entry library_disk_misses; -}; - -}; // namespace detail - -struct [[nodiscard]] cache_stats { - std::uint64_t blob_mem_hits = 0; - std::uint64_t blob_mem_misses = 0; - std::uint64_t blob_disk_hits = 0; - std::uint64_t blob_disk_misses = 0; - std::uint64_t library_mem_hits = 0; - std::uint64_t library_mem_misses = 0; - std::uint64_t library_disk_hits = 0; - std::uint64_t library_disk_misses = 0; -}; - -struct [[nodiscard]] cache_limits { - std::uint32_t num_mem_blobs = 16'384; - std::uint32_t num_mem_libraries = 16'384; -}; - -using blob_compile_func = func; -using library_compile_func = func()>; - -/** - * @brief Thread-safe user-managed compile cache for compiled blobs and libraries - * - * @details Provides in-memory and on-disk caching of compiled RTC artifacts. - * The cache uses an LRU eviction policy when the number of cached items exceeds user-defined - * limits. In-memory cache is implemented using a thread-safe LRU cache that supports concurrent - * reads. The on-disk cache also allows concurrent access and stores cached items in files within a - * specified directory. Writing to disk is atomic to prevent corruption from concurrent writes or - * process interruptions. In addition, the cache maintains statistics on cache hits and misses for - * both in-memory and on-disk caches to help monitor cache performance in benchmarking and - * debugging. The interface is zero-copy, using shared pointers, mmap, and spans to avoid - * unnecessary data copying across threads and disk. - */ -struct cache_t { // NOLINT - private: - bool enabled_; - - std::string cache_dir_; - - std::string tmp_dir_; - - cache_limits limits_; - - std::mutex lock_; - - detail::lru_memory_cache> blobs_cache_; - - detail::lru_memory_cache> libraries_cache_; - - detail::cache_stats_counter counter_; - - alignas(CACHELINE_ALIGNMENT) std::uint64_t tick_; // NOLINT(modernize-use-default-member-init) - - public: - /** - * @brief Construct a new cache_t object with the specified cache directory, limits, and options - * for preloading and enabling the cache. - * @param cache_dir The directory path to be used for on-disk caching of compiled blobs and - * libraries (this directory must exist and be writable by the process) - * @param tmp_dir The directory path to be used for temporary files during atomic writes to the - * on-disk cache (this directory must exist, writable by the process, and be on the same - * filesystem as cache_dir to ensure atomic renames work correctly) - * @param limits A cache_limits struct specifying the maximum number of blobs and libraries to - * store in the cache before eviction occurs - * @param preload A boolean flag indicating whether to preload the cache from disk during - * initialization, allowing for faster retrieval of previously compiled kernels at runtime - * @param disable A boolean flag indicating whether to disable the cache entirely, preventing any - * caching of compiled blobs and libraries in memory - */ - cache_t(std::string cache_dir, - std::string tmp_dir, - cache_limits const& limits, - bool preload, - bool disable); - cache_t(cache_t const&) = delete; - cache_t& operator=(cache_t const&) = delete; - cache_t(cache_t&&) = delete; - cache_t& operator=(cache_t&&) = delete; - ~cache_t() = default; - - /** - * @brief Get the directory path used for on-disk caching - * @return String reference to the cache directory path - */ - [[nodiscard]] std::string const& get_cache_dir(); - - /** - * @brief Get the directory path used for temporary files during atomic writes to the on-disk - * cache - * @return String reference to the temporary directory path - */ - [[nodiscard]] std::string const& get_tmp_dir(); - - /** - * @brief Query the cache for a compiled blob by its hash, or insert it if not present - * @param hash hash of the blob to query or insert - * @param compile Function to compile the blob if it's not found in the cache - * @return A shared future that will hold the compiled blob once it's available - */ - [[nodiscard]] std::shared_future get_or_add_blob(hash128 const& hash, - blob_compile_func compile); - - /** - * @brief Query the cache for a compiled library by its hash and binary type, or insert - * it if not present - * @param hash hash of the library to query or insert - * @param compile Function to compile the library if it's not found in the cache - * @return A shared future that will hold the compiled library once it's available - */ - [[nodiscard]] std::shared_future get_or_add_library(hash128 const& hash, - library_compile_func compile); - - /** - * @brief Retrieve current cache performance statistics, including hits and misses for both - * in-memory and on-disk caches - * - * @return A cache_statistics struct containing the current cache performance metrics - */ - cache_stats get_stats(); - - /** - * @brief Clear the current cache performance statistics, resetting all hit and miss counters to - * zero - */ - void clear_stats(); - - /** - * @brief Retrieve the current cache limits for blobs and libraries - * - * @return A cache_limits struct containing the maximum number of blobs and libraries that can be - * stored in the cache before eviction occurs - */ - cache_limits get_limits(); - - /** - * @brief Get the current number of blobs stored in the in-memory cache - * - * @return The number of blobs currently stored in the in-memory cache - */ - [[nodiscard]] std::size_t get_blob_count(); - - /** - * @brief Get the current number of libraries stored in the in-memory cache - * - * @return The number of libraries currently stored in the in-memory cache - */ - [[nodiscard]] std::size_t get_library_count(); - - /** - * @brief Clear all entries from the in-memory cache, removing all cached blobs and libraries - * without affecting the on-disk cache - * - * @details This function is useful for freeing up memory without losing the benefits of the - * on-disk cache, which can still be used to retrieve cached items in the future. - */ - void clear_memory_store(); - - /** - * @brief Clear all entries from the on-disk cache, removing all cached blobs and libraries - * stored on disk without affecting the in-memory cache - * @details This function is useful for freeing up disk space or resetting the on-disk cache - * without losing the benefits of the in-memory cache, which can still be used to retrieve cached - * items in the future. - */ - void clear_disk_store(); - - /*** - * @brief Pre-load the JIT program cache from disk into memory during initialization, allowing for - * faster retrieval and execution of previously compiled kernels at runtime. - */ - void preload_from_disk(); - - /*** - * @brief Enable the cache, allowing it to store and retrieve compiled blobs and libraries in - * memory. - * @param enabled A boolean flag indicating whether to enable (true) or disable (false) the cache. - */ - void enable(bool enabled); - - /** - * @brief Get whether the cache is currently enabled or disabled. - * @return A boolean value indicating whether the cache is currently enabled (true) or disabled - */ - [[nodiscard]] bool is_enabled(); -}; - -/** - * @brief Get the version of the NVRTC library - * @return An integer representing the NVRTC version. With the encoding major * 1000 + minor * 10 + - * patch - */ -[[nodiscard]] std::int32_t nvrtc_version(); - -/** - * @brief Get the version of the NVJITLINK library - * @return An integer representing the NVJITLINK version. With the encoding major * 1000 + minor * - * 10 + patch - */ -[[nodiscard]] std::int32_t nvjitlink_version(); - -/** - * @brief Compile source code into a binary blob - * - * @param params Compilation parameters including source code, headers, options, and target binary - * type - * @return A buffer of bytes containing the compiled binary blob - */ -[[nodiscard]] byte_buffer compile(compile_params const& params); - -/** - * @brief Load a compiled library from binary data - * - * @param binary Span of bytes containing the compiled library binary data - * @return A library object representing the loaded library with launchable kernels - */ -[[nodiscard]] library load_library(std::span binary); - -/** - * @brief Load a compiled library from binary data - * - * @param path Path to the file containing the library binary data - * @return A library object representing the loaded library with launchable kernels - */ -[[nodiscard]] library load_library_from_file(char const* path); - -/** - * @brief Link multiple compiled binary fragments into a single binary blob containing the linked - * library - * - * @param params Linking parameters including the binary fragments to be linked and the target - * binary type - * @return A buffer of bytes containing the linked library binary - */ -[[nodiscard]] byte_buffer link_library(link_params const& params); - -/** - * @brief Initialize the RTCX library, setting up necessary resources and state for subsequent - * operations - * @details This function must be called before using any other functions in the RTCX library. It - * performs necessary initialization tasks such as setting up CUDA contexts, initializing caches, - * and preparing any global state required for compilation, linking, and kernel management - * operations. Failure to call this function before using other RTCX functions may result in - * undefined behavior or runtime errors. - * This function is thread-safe. - */ -void initialize(); - -/** - * @brief Teardown the RTCX library, releasing any resources and cleaning up state used by the - * library - * @details This function should be called when RTCX functionality is no longer needed, such as at - * the end of the program or when cleaning up resources. It performs necessary cleanup tasks such as - * releasing CUDA contexts, clearing caches, and resetting any global state used by the library. - * After calling this function, other RTCX functions should not be used unless initialize() is - * called again to reinitialize the library. - * This function is not thread-safe. - */ -void teardown(); - -/** - * @brief Reflect a value of any type into its CUDA string representation - * @tparam T The type of the value to be reflected - * @param value The value to be reflected - * @return A string containing the CUDA representation of the value - * @details This is a template function that can be specialized for different types to provide - * appropriate CUDA string representations. - */ -template -std::string reflect(T value) = delete; - -/** - * @brief Reflect a boolean value into its CUDA string representation ("true" or "false") - * @param value The boolean value to be reflected - * @return A string containing the CUDA representation of the boolean value ("true" or "false") - */ -template <> -inline std::string reflect(bool value) -{ - return std::format("{}", value); -} - -/** - * @brief Reflect an integer value into its CUDA string representation - * @param value The integer value to be reflected - * @return A string containing the CUDA representation of the integer value - */ -template <> -inline std::string reflect(std::uint8_t value) -{ - return std::format("{}U", value); -} - -/** - * @brief Reflect an integer value into its CUDA string representation - * @param value The integer value to be reflected - * @return A string containing the CUDA representation of the integer value - */ -template <> -inline std::string reflect(std::uint16_t value) -{ - return std::format("{}U", value); -} - -/** - * @brief Reflect an integer value into its CUDA string representation - * @param value The integer value to be reflected - * @return A string containing the CUDA representation of the integer value - */ -template <> -inline std::string reflect(std::uint32_t value) -{ - return std::format("{}U", value); -} - -/** - * @brief Reflect an integer value into its CUDA string representation - * @param value The integer value to be reflected - * @return A string containing the CUDA representation of the integer value - */ -template <> -inline std::string reflect(std::uint64_t value) -{ - return std::format("{}ULL", value); -} - -/** - * @brief Reflect an integer value into its CUDA string representation - * @param value The integer value to be reflected - * @return A string containing the CUDA representation of the integer value - */ -template <> -inline std::string reflect(std::int8_t value) -{ - return std::format("{}", value); -} - -/** - * @brief Reflect an integer value into its CUDA string representation - * @param value The integer value to be reflected - * @return A string containing the CUDA representation of the integer value - */ -template <> -inline std::string reflect(std::int16_t value) -{ - return std::format("{}", value); -} - -/** - * @brief Reflect an integer value into its CUDA string representation - * @param value The integer value to be reflected - * @return A string containing the CUDA representation of the integer value - */ -template <> -inline std::string reflect(std::int32_t value) -{ - return std::format("{}", value); -} - -/** - * @brief Reflect an integer value into its CUDA string representation - * @param value The integer value to be reflected - * @return A string containing the CUDA representation of the integer value - */ -template <> -inline std::string reflect(std::int64_t value) -{ - return std::format("{}LL", value); -} - -/** - * @brief Reflect a floating-point value into its CUDA string representation - * @param value The floating-point value to be reflected - * @return A string containing the CUDA representation of the floating-point value - */ -template <> -inline std::string reflect(float value) -{ - return std::format("{}F", value); -} - -/** - * @brief Reflect a floating-point value into its CUDA string representation - * @param value The floating-point value to be reflected - * @return A string containing the CUDA representation of the floating-point value - */ -template <> -inline std::string reflect(double value) -{ - return std::format("{}", value); -} - -/** - * @brief Reflect an enumeration value into its CUDA string representation, given the type name as a - * string - * @tparam T An enumeration type - * @param type The name of the enumeration type to be reflected (e.g., "MyEnum") - * @param value The enumeration value to be reflected, which will be cast to its underlying integer - * type and represented as a string in the resulting CUDA code - * @return A string containing the CUDA representation of the enumeration value with the specified - * type - */ -template - requires(std::is_enum_v) -std::string reflect_enum(std::string_view type, T value) -{ - return std::format("{}{}{}{}", type, "{", static_cast>(value), "}"); -} - -/** - * @brief Reflect a template instantiation into its CUDA string representation, given the template - * name and its template arguments as strings - * @param template_name The name of the template to be reflected (e.g., "MyTemplate") - * @param template_args A span of strings representing the template arguments to be reflected, which - * will be used in the resulting CUDA code - * @return A string containing the CUDA representation of the template instantiation with the - * specified template name and arguments - */ -std::string reflect_template(std::string_view template_name, - std::span template_args); - -/** - * @brief Reflect a template instantiation into its CUDA string representation, given the template - * name and its template arguments as strings - * @param template_name The name of the template to be reflected (e.g., "MyTemplate") - * @param template_args A span of strings representing the template arguments to be reflected, which - * will be used in the resulting CUDA code - * @return A string containing the CUDA representation of the template instantiation with the - * specified template name and arguments - */ -std::string reflect_template(std::string_view template_name, - std::span template_args); - -/** - * @brief Reflect a template instantiation into its CUDA string representation, given the template - * name and its template arguments as strings - * @param template_name The name of the template to be reflected (e.g., "MyTemplate") - * @param template_args A span of strings representing the template arguments to be reflected, which - * will be used in the resulting CUDA code - * @return A string containing the CUDA representation of the template instantiation with the - * specified template name and arguments - */ -template - requires((true && ... && std::is_constructible_v)) -std::string reflect_template(std::string_view template_name, TemplateArgs&&... template_args) -{ - std::string_view const tparams[sizeof...(TemplateArgs)] = // NOLINT(modernize-avoid-c-arrays) - {std::string_view{template_args}...}; - return reflect_template(template_name, tparams); -} - -/** - * @brief Decompress a compressed binary blob using the specified compression algorithm - * @param compressed_binary A span of bytes containing the compressed binary data to be decompressed - * @param uncompressed_size The expected size of the uncompressed binary data in bytes - * @param compression A string view specifying the compression algorithm used to compress the binary - * data - * @return A byte buffer containing the decompressed binary data - * @throws std::runtime_error if decompression fails due to an unsupported compression algorithm, - * invalid compressed data, or if the decompressed data size does not match the expected - * uncompressed size - */ -rtcx::byte_buffer decompress_blob(std::span compressed_binary, - std::size_t uncompressed_size, - std::string_view compression); - -} // namespace rtcx From 6c57281bae162c3125a0a6df8df8425005410359 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 Jul 2026 20:58:22 -0700 Subject: [PATCH 03/11] ci: temporarily point rtcx CPM fetch at vyasr/librtcx@31a7295 --- cpp/cmake/thirdparty/get_rtcx.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/cmake/thirdparty/get_rtcx.cmake b/cpp/cmake/thirdparty/get_rtcx.cmake index 4e091531eac3..bcea249a3648 100644 --- a/cpp/cmake/thirdparty/get_rtcx.cmake +++ b/cpp/cmake/thirdparty/get_rtcx.cmake @@ -10,9 +10,9 @@ function(find_and_configure_rtcx VERSION) rtcx ${VERSION} GLOBAL_TARGETS rtcx::rtcx CPM_ARGS - GIT_REPOSITORY https://github.com/rapidsai/librtcx.git - GIT_TAG "${RAPIDS_BRANCH}" - GIT_SHALLOW TRUE + GIT_REPOSITORY https://github.com/vyasr/librtcx.git + GIT_TAG 31a72956cca60fa6039e6af746ed4e9771cca740 + GIT_SHALLOW FALSE EXCLUDE_FROM_ALL ${CUDF_EXCLUDE_DEPS_FROM_ALL} ) From 0c77b71ef7c31878749f119e36b47ce9bcaa46d3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 7 Jul 2026 12:19:26 -0700 Subject: [PATCH 04/11] fix: exclude rtcx from install export set rtcx is statically linked into libcudf and should not appear in the installed CMake export set. Wrap it with BUILD_LOCAL_INTERFACE (matching the xxhash pattern) so downstream consumers do not need to find rtcx separately. Also remove the stale include path to the now-deleted bundled cpp/librtcx/ directory; headers are provided by the rtcx::rtcx target. --- cpp/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index eacf0eb2d204..43eac90615e6 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1207,7 +1207,6 @@ target_include_directories( "$" "$" "$" - "$" "$" "$" INTERFACE "$" @@ -1259,7 +1258,7 @@ target_link_libraries( ${CUDF_nanoarrow_TARGET} zstd $ - rtcx::rtcx + $ ) # When rmm is a static library being absorbed via whole-archive, strip nvtx3 from its public From 2720c1b81e9ba752debf5878e69e99b2d5af0609 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 7 Jul 2026 13:51:28 -0700 Subject: [PATCH 05/11] fix: properly export rtcx for static builds and install validation Three changes: 1. Set RTCX_INSTALL=ON before fetching rtcx so that librtcx's install targets are unconditionally registered (no EXCLUDE_FROM_ALL). This satisfies CMake's install(EXPORT) validation which requires all link dependencies to be in some export set. 2. Add rapids_export_find_package_root(BUILD rtcx ...) so the build-tree export set can locate the rtcx package. 3. Add rtcx to the static-deps export list so that static libcudf consumers (e.g. spark-rapids-jni) get find_dependency(rtcx) in the installed cudf config and can link librtcx.a. 4. Revert BUILD_LOCAL_INTERFACE on rtcx::rtcx since the target must remain visible to the install export set. --- cpp/CMakeLists.txt | 4 ++-- cpp/cmake/thirdparty/get_rtcx.cmake | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 43eac90615e6..0ecf4e182abb 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -429,7 +429,7 @@ include(cmake/Modules/AddFragment.cmake) if(NOT BUILD_SHARED_LIBS) include("${rapids-cmake-dir}/export/find_package_file.cmake") list(APPEND METADATA_KINDS BUILD INSTALL) - list(APPEND dependencies KvikIO ZLIB nvcomp nanoarrow zstd) + list(APPEND dependencies KvikIO ZLIB nvcomp nanoarrow zstd rtcx) foreach(METADATA_KIND IN LISTS METADATA_KINDS) foreach(dep IN LISTS dependencies) @@ -1258,7 +1258,7 @@ target_link_libraries( ${CUDF_nanoarrow_TARGET} zstd $ - $ + rtcx::rtcx ) # When rmm is a static library being absorbed via whole-archive, strip nvtx3 from its public diff --git a/cpp/cmake/thirdparty/get_rtcx.cmake b/cpp/cmake/thirdparty/get_rtcx.cmake index bcea249a3648..a3bbb742dff6 100644 --- a/cpp/cmake/thirdparty/get_rtcx.cmake +++ b/cpp/cmake/thirdparty/get_rtcx.cmake @@ -6,6 +6,11 @@ # This function finds rtcx function(find_and_configure_rtcx VERSION) + # Ensure rtcx installs its targets unconditionally. In shared builds the static library is + # absorbed into libcudf.so, but CMake still requires the target to be in an export set for + # install(EXPORT) validation. In static builds consumers need to link librtcx.a directly. + set(RTCX_INSTALL ON) + rapids_cpm_find( rtcx ${VERSION} GLOBAL_TARGETS rtcx::rtcx @@ -21,7 +26,6 @@ function(find_and_configure_rtcx VERSION) if(rtcx_ADDED OR DEFINED CPM_rtcx_SOURCE) include("${rtcx_SOURCE_DIR}/embed.cmake") endif() - endfunction() set(RTCX_MIN_VERSION_cudf "0.1") From f797da48dd627d7e50c7ec4a85b239eef80dab9c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 7 Jul 2026 14:30:42 -0700 Subject: [PATCH 06/11] ci: update rtcx pin to include embed.cmake lazy-check fix The new commit defers zstd/xxhash target checks from embed.cmake include time to rtcx_embed() call time, so consumers that only link librtcx (like spark-rapids-jni) don't need these targets available. --- cpp/cmake/thirdparty/get_rtcx.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_rtcx.cmake b/cpp/cmake/thirdparty/get_rtcx.cmake index a3bbb742dff6..49c009281e45 100644 --- a/cpp/cmake/thirdparty/get_rtcx.cmake +++ b/cpp/cmake/thirdparty/get_rtcx.cmake @@ -16,7 +16,7 @@ function(find_and_configure_rtcx VERSION) GLOBAL_TARGETS rtcx::rtcx CPM_ARGS GIT_REPOSITORY https://github.com/vyasr/librtcx.git - GIT_TAG 31a72956cca60fa6039e6af746ed4e9771cca740 + GIT_TAG 1087bdfecb48138bee3b8c73d86a284aa5ae827e GIT_SHALLOW FALSE EXCLUDE_FROM_ALL ${CUDF_EXCLUDE_DEPS_FROM_ALL} ) From cb3700519e520641fbf317d8d147e5d7311787df Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 8 Jul 2026 09:56:23 -0700 Subject: [PATCH 07/11] ci: update rtcx pin to split-embed-cmake branch head --- cpp/cmake/thirdparty/get_rtcx.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_rtcx.cmake b/cpp/cmake/thirdparty/get_rtcx.cmake index 49c009281e45..4018890c6b3e 100644 --- a/cpp/cmake/thirdparty/get_rtcx.cmake +++ b/cpp/cmake/thirdparty/get_rtcx.cmake @@ -16,7 +16,7 @@ function(find_and_configure_rtcx VERSION) GLOBAL_TARGETS rtcx::rtcx CPM_ARGS GIT_REPOSITORY https://github.com/vyasr/librtcx.git - GIT_TAG 1087bdfecb48138bee3b8c73d86a284aa5ae827e + GIT_TAG be8d096c91885141d45f2e054262e1df3e7ce0c9 GIT_SHALLOW FALSE EXCLUDE_FROM_ALL ${CUDF_EXCLUDE_DEPS_FROM_ALL} ) From 4d975018e0cd7b1145f907d00124ade71701db5d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 10 Jul 2026 10:24:42 -0700 Subject: [PATCH 08/11] Update to latest commit --- cpp/cmake/thirdparty/get_rtcx.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_rtcx.cmake b/cpp/cmake/thirdparty/get_rtcx.cmake index 4018890c6b3e..28ad4d094883 100644 --- a/cpp/cmake/thirdparty/get_rtcx.cmake +++ b/cpp/cmake/thirdparty/get_rtcx.cmake @@ -16,7 +16,7 @@ function(find_and_configure_rtcx VERSION) GLOBAL_TARGETS rtcx::rtcx CPM_ARGS GIT_REPOSITORY https://github.com/vyasr/librtcx.git - GIT_TAG be8d096c91885141d45f2e054262e1df3e7ce0c9 + GIT_TAG 73f700ebddcdc2c98b6c7587dfa09bb18f1237ff GIT_SHALLOW FALSE EXCLUDE_FROM_ALL ${CUDF_EXCLUDE_DEPS_FROM_ALL} ) From bf43e3a93e8c5a0f1eb00c5313b2b82cbe72e2e2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 10 Jul 2026 10:28:43 -0700 Subject: [PATCH 09/11] Fix org --- cpp/cmake/thirdparty/get_rtcx.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_rtcx.cmake b/cpp/cmake/thirdparty/get_rtcx.cmake index 28ad4d094883..f753c2b575c4 100644 --- a/cpp/cmake/thirdparty/get_rtcx.cmake +++ b/cpp/cmake/thirdparty/get_rtcx.cmake @@ -15,7 +15,7 @@ function(find_and_configure_rtcx VERSION) rtcx ${VERSION} GLOBAL_TARGETS rtcx::rtcx CPM_ARGS - GIT_REPOSITORY https://github.com/vyasr/librtcx.git + GIT_REPOSITORY https://github.com/rapidsai/librtcx.git GIT_TAG 73f700ebddcdc2c98b6c7587dfa09bb18f1237ff GIT_SHALLOW FALSE EXCLUDE_FROM_ALL ${CUDF_EXCLUDE_DEPS_FROM_ALL} From 2dec5d8a098cd49f3f57562f00aee92c84b5e00c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 10 Jul 2026 14:17:01 -0700 Subject: [PATCH 10/11] Pin librtcx to pre-cuVS commit The cuVS upstreaming (rapidsai/librtcx#8) added a raft dependency that breaks the cmake-tests export consumer. Pin to efad266 which is the last commit before that addition. --- cpp/cmake/thirdparty/get_rtcx.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_rtcx.cmake b/cpp/cmake/thirdparty/get_rtcx.cmake index f753c2b575c4..dc203d27ca34 100644 --- a/cpp/cmake/thirdparty/get_rtcx.cmake +++ b/cpp/cmake/thirdparty/get_rtcx.cmake @@ -16,7 +16,7 @@ function(find_and_configure_rtcx VERSION) GLOBAL_TARGETS rtcx::rtcx CPM_ARGS GIT_REPOSITORY https://github.com/rapidsai/librtcx.git - GIT_TAG 73f700ebddcdc2c98b6c7587dfa09bb18f1237ff + GIT_TAG efad266a397a6b1e04e2d1be0e62e78a458f0e30 GIT_SHALLOW FALSE EXCLUDE_FROM_ALL ${CUDF_EXCLUDE_DEPS_FROM_ALL} ) From 7849e42edd4227c7ac340c0ff0c78e6e96b287bb Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 10 Jul 2026 14:25:07 -0700 Subject: [PATCH 11/11] Fix librtcx commit hash The previous commit used an incorrect hash. Use the actual full SHA for the 'Add and apply pre-commit hooks (#6)' commit. --- cpp/cmake/thirdparty/get_rtcx.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_rtcx.cmake b/cpp/cmake/thirdparty/get_rtcx.cmake index dc203d27ca34..5659162d9718 100644 --- a/cpp/cmake/thirdparty/get_rtcx.cmake +++ b/cpp/cmake/thirdparty/get_rtcx.cmake @@ -16,7 +16,7 @@ function(find_and_configure_rtcx VERSION) GLOBAL_TARGETS rtcx::rtcx CPM_ARGS GIT_REPOSITORY https://github.com/rapidsai/librtcx.git - GIT_TAG efad266a397a6b1e04e2d1be0e62e78a458f0e30 + GIT_TAG efad266c1fd9de6d8486c6ba71bfa74df063eb1f GIT_SHALLOW FALSE EXCLUDE_FROM_ALL ${CUDF_EXCLUDE_DEPS_FROM_ALL} )