diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..762c6fa --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,166 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +cmake_minimum_required(VERSION 4.0 FATAL_ERROR) + +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + include(cmake/rapids_config.cmake) + include(rapids-cmake) + include(rapids-cpm) + include(rapids-export) + include(rapids-find) + rapids_cpm_init() +endif() + +project( + rtcx + VERSION 0.1.0 + LANGUAGES CXX +) + +option(RTCX_STATIC_LINK_NVRTC "Use static linking for NVRTC" OFF) +option(RTCX_STATIC_LINK_NVJITLINK "Use static linking for nvJitLink" OFF) + +rapids_find_package( + CUDAToolkit REQUIRED + BUILD_EXPORT_SET rtcx-exports + INSTALL_EXPORT_SET rtcx-exports +) + +if(NOT TARGET zstd) + set(CPM_DOWNLOAD_zstd ON) + rapids_cpm_find( + zstd 1.5.7 + GLOBAL_TARGETS zstd + CPM_ARGS + GIT_REPOSITORY https://github.com/facebook/zstd.git + GIT_TAG v1.5.7 + GIT_SHALLOW FALSE SOURCE_SUBDIR build/cmake + OPTIONS "ZSTD_BUILD_STATIC ON" "ZSTD_BUILD_SHARED OFF" "ZSTD_BUILD_TESTS OFF" + "ZSTD_BUILD_PROGRAMS OFF" "BUILD_SHARED_LIBS OFF" + ) + + if(zstd_ADDED) + # disable weak symbols support to hide tracing APIs as well + target_compile_definitions(libzstd_static PRIVATE ZSTD_HAVE_WEAK_SYMBOLS=0) + # expose experimental API + target_compile_definitions(libzstd_static PUBLIC ZSTD_STATIC_LINKING_ONLY=0N) + # suppress warnings from uninitialized variables and redefining ZSTD_STATIC_LINKING_ONLY + target_compile_options(libzstd_static PRIVATE -w) + add_library(zstd ALIAS libzstd_static) + endif() +endif() + +if(NOT TARGET nvtx3::nvtx3-cpp) + include(${rapids-cmake-dir}/cpm/nvtx3.cmake) + rapids_cpm_nvtx3() +endif() + +add_library(rtcx STATIC hash.cpp rtcx.cpp) +add_library(rtcx::rtcx ALIAS rtcx) + +set_target_properties( + rtcx + PROPERTIES CXX_STANDARD 20 + CXX_STANDARD_REQUIRED YES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN YES + POSITION_INDEPENDENT_CODE ON +) + +target_include_directories( + rtcx + PUBLIC $ + PRIVATE ${CUDAToolkit_INCLUDE_DIRS} +) + +target_link_libraries(rtcx PRIVATE zstd ${CMAKE_DL_LIBS} nvtx3::nvtx3-cpp) + +if(RTCX_STATIC_LINK_NVRTC) + target_link_libraries(rtcx PRIVATE CUDA::nvrtc_static CUDA::nvrtc_builtins_static) + target_compile_definitions(rtcx PRIVATE RTCX_STATIC_LINK_LIBNVRTC=1) +else() + target_compile_definitions(rtcx PRIVATE RTCX_STATIC_LINK_LIBNVRTC=0) +endif() + +if(RTCX_STATIC_LINK_NVJITLINK) + target_link_libraries(rtcx PRIVATE CUDA::nvJitLink_static) + target_compile_definitions(rtcx PRIVATE RTCX_STATIC_LINK_LIBNVJITLINK=1) +else() + target_compile_definitions(rtcx PRIVATE RTCX_STATIC_LINK_LIBNVJITLINK=0) +endif() + +# ============================================================================= +# Install / Export +# ============================================================================= +include(GNUInstallDirs) +include(${rapids-cmake-dir}/cmake/install_lib_dir.cmake) +rapids_cmake_install_lib_dir(lib_dir) + +# Option to control install (default ON when built standalone, OFF when used as subdirectory) +option(RTCX_INSTALL "Enable installation of rtcx targets" ${PROJECT_IS_TOP_LEVEL}) + +set(rtcx_install_code_string + [=[ +# Embed functions (add_embed, embed_includes, embed_blob, embed) +# are included automatically so consumers don't need explicit include(). +include("${CMAKE_CURRENT_LIST_DIR}/embed.cmake") + +# Set rtcx_LIBCXX_DIR for consumers using embed_includes with libcxx headers. +set(rtcx_LIBCXX_DIR "${PACKAGE_PREFIX_DIR}/share/rtcx/libcxx") +]=] +) + +string( + CONFIGURE + [=[ +# Embed functions (add_embed, embed_includes, embed_blob, embed) +# are included automatically so consumers don't need explicit include(). +include("@CMAKE_CURRENT_SOURCE_DIR@/embed.cmake") + +# Set rtcx_LIBCXX_DIR for consumers using embed_includes with libcxx headers. +set(rtcx_LIBCXX_DIR "@CMAKE_CURRENT_SOURCE_DIR@/libcxx") +]=] + rtcx_build_code_string + @ONLY +) + +if(NOT RTCX_INSTALL) + set(rtcx_exclude_from_install EXCLUDE_FROM_ALL) +endif() + +install( + TARGETS rtcx + EXPORT rtcx-exports + ARCHIVE DESTINATION ${lib_dir} + INCLUDES + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ${rtcx_exclude_from_install} +) + +if(RTCX_INSTALL) + install(FILES rtcx.hpp hash.hpp embed.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rtcx) + install(FILES embed.cmake embed.in.cpp DESTINATION ${lib_dir}/cmake/rtcx) + install(DIRECTORY libcxx/ DESTINATION ${CMAKE_INSTALL_DATADIR}/rtcx/libcxx) + + rapids_export( + INSTALL rtcx + EXPORT_SET rtcx-exports + GLOBAL_TARGETS rtcx + NAMESPACE rtcx:: + FINAL_CODE_BLOCK rtcx_install_code_string + ) +endif() + +# Build-tree export (always, so CPM consumers can find_package from the build tree) +rapids_export( + BUILD rtcx + EXPORT_SET rtcx-exports + GLOBAL_TARGETS rtcx + NAMESPACE rtcx:: + FINAL_CODE_BLOCK rtcx_build_code_string +) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5c26bde --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,58 @@ +# Contributing to librtcx + +Contributions to librtcx fall into the following categories: + +1. To report a bug, request a new feature, or report a problem with documentation, please file an + [issue](https://github.com/rapidsai/librtcx/issues/new/choose) describing the problem or new feature + in detail. +2. To propose and implement a new feature, please file a new feature request + [issue](https://github.com/rapidsai/librtcx/issues/new/choose). Describe the intended feature and + discuss the design and implementation with the team and community. Once the team agrees that the + plan looks good, go ahead and implement it, using the [code contributions](#code-contributions) + guide below. +3. To implement a feature or bug fix for an existing issue, please follow the [code + contributions](#code-contributions) guide below. If you need more context on a particular issue, + please ask in a comment. + +As contributors and maintainers to this project, you are expected to abide by the +[Contributor Code of Conduct](https://docs.rapids.ai/resources/conduct/). + +## Code contributions + +1. Create a fork of the [librtcx repository](https://github.com/rapidsai/librtcx) and check out a branch + with a name that describes your planned work. +2. Write code to address the issue or implement the feature. +3. Add unit tests. +4. [Create your pull request](https://github.com/rapidsai/librtcx/compare). +5. Verify that CI passes all status checks. Fix if needed. +6. Wait for other developers to review your code and update code as needed. +7. Once reviewed and approved, a maintainer will merge your pull request. + +If you are unsure about anything, don't hesitate to comment on issues and ask for clarification! + +## Code Formatting + +librtcx uses [pre-commit](https://pre-commit.com/) to execute code linters and formatters. +These tools ensure a consistent code format throughout the project. + +To use `pre-commit`, install via `conda` or `pip`: + +```bash +conda install -c conda-forge pre-commit +``` + +```bash +pip install pre-commit +``` + +Then run pre-commit hooks before committing code: + +```bash +pre-commit run +``` + +Optionally, you may set up the pre-commit hooks to run automatically when you make a git commit: + +```bash +pre-commit install +``` diff --git a/LICENSE b/LICENSE index 1a89b90..c191723 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2020 NVIDIA Corporation + Copyright 2026 NVIDIA Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/RAPIDS_BRANCH b/RAPIDS_BRANCH new file mode 100644 index 0000000..ba2906d --- /dev/null +++ b/RAPIDS_BRANCH @@ -0,0 +1 @@ +main diff --git a/README.md b/README.md new file mode 100644 index 0000000..b8f9898 --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# librtcx + +RTCX (runtime-compiler extended) is a wrapper around NVRTC and nvJitLink designed to provide: + +- User-controlled compilation, linking, caching, and pre-loading of CUDA kernels +- Zero-copy interfaces to manage JIT compilation and linking +- CMake script to embed **compressed** headers directly into an executable without incurring overhead at runtime on every compilation request +- Facilities to pre-load and teardown dynamic library dependencies (`libcuda`, `libnvrtc`, and `libnvJitLink`) + +## Platforms Supported +- Linux + +## Build-time Requirements +- CMake >= 4.0 +- libzstd +- xxHash +- CUDA >= 12.2 + +# Dependencies +- nvJitlink >= 12.2 +- NVRTC >= 12.2 +- LibCUDA >= 12.2 diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/cmake/RAPIDS.cmake b/cmake/RAPIDS.cmake new file mode 100644 index 0000000..a3ad5e3 --- /dev/null +++ b/cmake/RAPIDS.cmake @@ -0,0 +1,83 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= +# +# This is the preferred entry point for projects using rapids-cmake +# +# Enforce the minimum required CMake version for all users +cmake_minimum_required(VERSION 4.0 FATAL_ERROR) + +# Allow users to control which version is used +if(NOT (rapids-cmake-branch OR rapids-cmake-version)) + message( + FATAL_ERROR "The CMake variable `rapids-cmake-branch` or `rapids-cmake-version` must be defined" + ) +endif() + +# Allow users to control which GitHub repo is fetched +if(NOT rapids-cmake-repo) + # Define a default repo if the user doesn't set one + set(rapids-cmake-repo rapidsai/rapids-cmake) +endif() + +# Allow users to control which branch is fetched +if(NOT rapids-cmake-branch) + # Define a default branch if the user doesn't set one + set(rapids-cmake-branch "release/${rapids-cmake-version}") +endif() + +# Allow users to control the exact URL passed to FetchContent +if(NOT rapids-cmake-url) + # Construct a default URL if the user doesn't set one + set(rapids-cmake-url "https://github.com/${rapids-cmake-repo}/") + + # In order of specificity + if(rapids-cmake-fetch-via-git) + if(rapids-cmake-sha) + # An exact git SHA takes precedence over anything + set(rapids-cmake-value-to-clone "${rapids-cmake-sha}") + elseif(rapids-cmake-tag) + # Followed by a git tag name + set(rapids-cmake-value-to-clone "${rapids-cmake-tag}") + else() + # Or if neither of the above two were defined, use a branch + set(rapids-cmake-value-to-clone "${rapids-cmake-branch}") + endif() + else() + if(rapids-cmake-sha) + # An exact git SHA takes precedence over anything + set(rapids-cmake-value-to-clone "archive/${rapids-cmake-sha}.zip") + elseif(rapids-cmake-tag) + # Followed by a git tag name + set(rapids-cmake-value-to-clone "archive/refs/tags/${rapids-cmake-tag}.zip") + else() + # Or if neither of the above two were defined, use a branch + set(rapids-cmake-value-to-clone "archive/refs/heads/${rapids-cmake-branch}.zip") + endif() + endif() +endif() + +include(FetchContent) +if(rapids-cmake-fetch-via-git) + FetchContent_Declare( + rapids-cmake + GIT_REPOSITORY "${rapids-cmake-url}" + GIT_TAG "${rapids-cmake-value-to-clone}" + ) +else() + string(APPEND rapids-cmake-url "${rapids-cmake-value-to-clone}") + FetchContent_Declare(rapids-cmake URL "${rapids-cmake-url}") +endif() +FetchContent_GetProperties(rapids-cmake) +if(rapids-cmake_POPULATED) + # Something else has already populated rapids-cmake, only thing we need to do is setup the + # CMAKE_MODULE_PATH + if(NOT "${rapids-cmake-dir}" IN_LIST CMAKE_MODULE_PATH) + list(APPEND CMAKE_MODULE_PATH "${rapids-cmake-dir}") + endif() +else() + FetchContent_MakeAvailable(rapids-cmake) +endif() diff --git a/cmake/rapids_config.cmake b/cmake/rapids_config.cmake new file mode 100644 index 0000000..52b3452 --- /dev/null +++ b/cmake/rapids_config.cmake @@ -0,0 +1,38 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= +file(READ "${CMAKE_CURRENT_LIST_DIR}/../VERSION" _rapids_version) +if(_rapids_version MATCHES [[^([0-9]+)\.([0-9]+)\.([0-9]+)]]) + set(RAPIDS_VERSION_MAJOR "${CMAKE_MATCH_1}") + set(RAPIDS_VERSION_MINOR "${CMAKE_MATCH_2}") + set(RAPIDS_VERSION_PATCH "${CMAKE_MATCH_3}") + set(RAPIDS_VERSION_MAJOR_MINOR "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}") + set(RAPIDS_VERSION "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}.${RAPIDS_VERSION_PATCH}") +else() + string(REPLACE "\n" "\n " _rapids_version_formatted " ${_rapids_version}") + message( + FATAL_ERROR + "Could not determine RAPIDS version. Contents of VERSION file:\n${_rapids_version_formatted}" + ) +endif() + +file(STRINGS "${CMAKE_CURRENT_LIST_DIR}/../RAPIDS_BRANCH" RAPIDS_BRANCH) +if(NOT RAPIDS_BRANCH) + message( + FATAL_ERROR + "Could not determine branch name to use for checking out rapids-cmake. The file \"${CMAKE_CURRENT_LIST_DIR}/../RAPIDS_BRANCH\" is missing." + ) +endif() + +if(NOT rapids-cmake-version) + set(rapids-cmake-version "${RAPIDS_VERSION_MAJOR_MINOR}") +endif() + +if(NOT rapids-cmake-branch) + set(rapids-cmake-branch "${RAPIDS_BRANCH}") +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/RAPIDS.cmake") diff --git a/embed.cmake b/embed.cmake new file mode 100644 index 0000000..ff4ee3e --- /dev/null +++ b/embed.cmake @@ -0,0 +1,351 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +# Embedding requires zstd and xxhash targets. Check lazily when embedding functions are called +# rather than at include time, so consumers that only link librtcx don't need these targets. + +# This function initializes a target for JIT embedding. It must be called before any calls to +# embed_includes() or embed_blob() for the target. It creates a dedicated INTERFACE library target +# that is used to track registered files and dependencies via target properties. The TARGET argument +# specifies the name of the target being initialized. +function(rtcx_add_embed TARGET) + set(OPTIONS "") + set(ONE_VALUE_ARGS) + set(MULTI_VALUE_ARGS) + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT DEFINED TARGET) + message(FATAL_ERROR "TARGET argument is required") + endif() + + add_library(${TARGET}__embed_props INTERFACE) + set_property(TARGET ${TARGET}__embed_props PROPERTY EMBED_FILE_INDEX 0) +endfunction() + +# This function registers a directory of include files to be embedded for JIT compilation. +function(rtcx_embed_includes TARGET) + set(OPTIONS "") + set(ONE_VALUE_ARGS SOURCE_DIRECTORY # Source directory where files will be copied from + DEST_DIRECTORY # Destination directory where files will be copied to + ) + set(MULTI_VALUE_ARGS + FILES # Source files relative to SOURCE_DIRECTORY (optional, if not provided, all files under + # SOURCE_DIRECTORY will be used) + INCLUDE_DIRECTORIES # Include directories to be used when compiling with these files + ) + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT TARGET ${TARGET}__embed_props) + message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with add_embed()") + endif() + + if(NOT ARG_SOURCE_DIRECTORY + OR NOT ARG_DEST_DIRECTORY + OR NOT ARG_INCLUDE_DIRECTORIES + ) + message( + FATAL_ERROR "SOURCE_DIRECTORY, DEST_DIRECTORY, and INCLUDE_DIRECTORIES arguments are required" + ) + endif() + + if(NOT ARG_FILES) + # gather all include files under the specified directory + file(GLOB_RECURSE INCLUDE_FILES "${ARG_SOURCE_DIRECTORY}/*") + + # get their paths relative to the base include directory + set(INCLUDE_FILES_RELATIVE_PATHS "") + foreach(INCLUDE_FILE IN LISTS INCLUDE_FILES) + file(RELATIVE_PATH INCLUDE_FILE_REL_PATH "${ARG_SOURCE_DIRECTORY}" "${INCLUDE_FILE}") + list(APPEND INCLUDE_FILES_RELATIVE_PATHS "${INCLUDE_FILE_REL_PATH}") + endforeach() + + set(ARG_FILES ${INCLUDE_FILES_RELATIVE_PATHS}) + endif() + + # check that each source file exists + foreach(SOURCE_FILE IN LISTS ARG_FILES) + if(NOT EXISTS "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}") + message(FATAL_ERROR "Source file '${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}' does not exist") + endif() + endforeach(SOURCE_FILE) + + # Determine the starting index for new IDs from the current list length + get_property( + SOURCE_FILE_IDS + TARGET ${TARGET}__embed_props + PROPERTY EMBED_SOURCE_FILE_IDS + ) + list(LENGTH SOURCE_FILE_IDS IDX) + + foreach(SOURCE_FILE IN LISTS ARG_FILES) + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_SOURCE_FILE_IDS "include_${IDX}" + ) + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_SOURCE_FILES "${ARG_SOURCE_DIRECTORY}/${SOURCE_FILE}" + ) + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_SOURCE_FILE_DESTS "${ARG_DEST_DIRECTORY}/${SOURCE_FILE}" + ) + math(EXPR IDX "${IDX} + 1") + endforeach() + + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_INCLUDE_DIRECTORIES ${ARG_INCLUDE_DIRECTORIES} + ) + + get_property( + SOURCE_FILE_IDS + TARGET ${TARGET}__embed_props + PROPERTY EMBED_SOURCE_FILE_IDS + ) + list(LENGTH SOURCE_FILE_IDS IDX) + + set_property(TARGET ${TARGET}__embed_props PROPERTY EMBED_FILE_INDEX ${IDX}) + +endfunction() + +# This function registers a single file to be embedded for JIT compilation. +function(rtcx_embed_blob TARGET) + set(OPTIONS) + set(ONE_VALUE_ARGS ID FILE DEST) + set(MULTI_VALUE_ARGS ARRAY_IDS ARRAY_VALUES) + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT TARGET ${TARGET}__embed_props) + message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with rtcx_add_embed()") + endif() + + if(NOT ARG_ID + OR NOT ARG_FILE + OR NOT ARG_DEST + ) + message(FATAL_ERROR "ID, FILE, and DEST arguments are required") + endif() + + if(ARG_ARRAY_IDS) + if(NOT ARG_ARRAY_VALUES) + message(FATAL_ERROR "ARRAY_VALUES argument is required when ARRAY_IDS is provided") + endif() + + list(LENGTH ARG_ARRAY_IDS ARG_ARRAY_IDS_LENGTH) + list(LENGTH ARG_ARRAY_VALUES ARG_ARRAY_VALUES_LENGTH) + + if(NOT ARG_ARRAY_IDS_LENGTH EQUAL ARG_ARRAY_VALUES_LENGTH) + message(FATAL_ERROR "ARRAY_IDS and ARRAY_VALUES must have the same length") + endif() + + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_ARRAY_IDS ${ARG_ARRAY_IDS} + ) + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_ARRAY_VALUES ${ARG_ARRAY_VALUES} + ) + endif() + + if(ARG_FILE MATCHES "\\$]+)>") + # If the file is a generator expression for target objects add as dependency + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_TARGET_DEPS $ + ) + # Also record the target name itself. Depending only on $ creates file-level + # dependencies without a target-level ordering, which breaks the Makefiles generator (Ninja + # resolves it via its global build graph). + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_TARGET_DEP_NAMES ${CMAKE_MATCH_1} + ) + else() + if(NOT EXISTS "${ARG_FILE}") + message(FATAL_ERROR "Source file '${ARG_FILE}' does not exist") + endif() + endif() + + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_SOURCE_FILE_IDS ${ARG_ID} + ) + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_SOURCE_FILES ${ARG_FILE} + ) + set_property( + TARGET ${TARGET}__embed_props + APPEND + PROPERTY EMBED_SOURCE_FILE_DESTS ${ARG_DEST} + ) + + get_property( + SOURCE_FILE_IDS + TARGET ${TARGET}__embed_props + PROPERTY EMBED_SOURCE_FILE_IDS + ) + list(LENGTH SOURCE_FILE_IDS IDX) + + set_property(TARGET ${TARGET}__embed_props PROPERTY EMBED_FILE_INDEX ${IDX}) + +endfunction() + +#[==[ +# This function generates the necessary files and build targets to embed the registered source files +# for JIT compilation. +#]==] +# cmake-lint: disable=R0915 +function(rtcx_embed TARGET) + set(OPTIONS "") + set(ONE_VALUE_ARGS "COMPRESSION" "OUTPUT_DIRECTORY") + set(MULTI_VALUE_ARGS "") + cmake_parse_arguments(ARG "${OPTIONS}" "${ONE_VALUE_ARGS}" "${MULTI_VALUE_ARGS}" ${ARGN}) + + if(NOT TARGET zstd) + message(FATAL_ERROR "zstd target is required for rtcx_embed().") + endif() + if(NOT TARGET xxhash) + message(FATAL_ERROR "xxhash target is required for rtcx_embed().") + endif() + + if(NOT TARGET ${TARGET}__embed_props) + message(FATAL_ERROR "embed target '${TARGET}' has not been initialized with rtcx_add_embed()") + endif() + + if(NOT DEFINED ARG_COMPRESSION) + message(FATAL_ERROR "COMPRESSION argument is required") + endif() + + if(NOT ARG_COMPRESSION STREQUAL "none" AND NOT ARG_COMPRESSION STREQUAL "zstd") + message(FATAL_ERROR "COMPRESSION argument must be either none or zstd") + endif() + + if(NOT DEFINED ARG_OUTPUT_DIRECTORY) + message(FATAL_ERROR "OUTPUT_DIRECTORY argument is required") + endif() + + get_property( + EMBED_SOURCE_FILES + TARGET ${TARGET}__embed_props + PROPERTY EMBED_SOURCE_FILES + ) + if(NOT EMBED_SOURCE_FILES) + message(FATAL_ERROR "No source files registered for target '${TARGET}'") + endif() + + get_property( + EMBED_SOURCE_FILE_IDS + TARGET ${TARGET}__embed_props + PROPERTY EMBED_SOURCE_FILE_IDS + ) + get_property( + EMBED_SOURCE_FILE_DESTS + TARGET ${TARGET}__embed_props + PROPERTY EMBED_SOURCE_FILE_DESTS + ) + get_property( + EMBED_TARGET_DEPS + TARGET ${TARGET}__embed_props + PROPERTY EMBED_TARGET_DEPS + ) + get_property( + EMBED_TARGET_DEP_NAMES + TARGET ${TARGET}__embed_props + PROPERTY EMBED_TARGET_DEP_NAMES + ) + get_property( + EMBED_ARRAY_IDS + TARGET ${TARGET}__embed_props + PROPERTY EMBED_ARRAY_IDS + ) + get_property( + EMBED_ARRAY_VALUES + TARGET ${TARGET}__embed_props + PROPERTY EMBED_ARRAY_VALUES + ) + get_property( + EMBED_INCLUDE_DIRS + TARGET ${TARGET}__embed_props + PROPERTY EMBED_INCLUDE_DIRECTORIES + ) + + set(OUTPUT_DIR "${ARG_OUTPUT_DIRECTORY}") + set(EMBED_SCRIPT_TEMPLATE "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/embed.in.cpp") + set(CONFIGURED_EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed_cfg.cpp") + set(EMBED_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}__embed.cpp") + + set(EMBED_SCRIPT__ID "${TARGET}") + set(EMBED_SCRIPT__ARRAY_IDS "${EMBED_ARRAY_IDS}") + set(EMBED_SCRIPT__ARRAY_VALUES "${EMBED_ARRAY_VALUES}") + set(EMBED_SCRIPT__FILE_IDS "${EMBED_SOURCE_FILE_IDS}") + set(EMBED_SCRIPT__FILE_PATHS "${EMBED_SOURCE_FILES}") + set(EMBED_SCRIPT__FILE_DESTS "${EMBED_SOURCE_FILE_DESTS}") + set(EMBED_SCRIPT__INCLUDE_DIRS "${EMBED_INCLUDE_DIRS}") + set(EMBED_SCRIPT__COMPRESSION "${ARG_COMPRESSION}") + set(EMBED_SCRIPT__OUTPUT_DIR "${OUTPUT_DIR}") + + configure_file(${EMBED_SCRIPT_TEMPLATE} ${CONFIGURED_EMBED_SCRIPT} @ONLY) + file( + GENERATE + OUTPUT "${EMBED_SCRIPT}" + INPUT "${CONFIGURED_EMBED_SCRIPT}" + ) + + set(RUNNER "${TARGET}__jit_embed_run") + add_executable( + ${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}" ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/hash.cpp + ) + target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} xxhash zstd) + target_include_directories( + ${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR} ${ZSTD_INCLUDE_DIR} + ) + set_target_properties(${RUNNER} PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES) + + add_custom_command( + OUTPUT ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin + COMMAND "${CMAKE_COMMAND}" -E env $ + DEPENDS "${EMBED_SCRIPT}" ${EMBED_SOURCE_FILES} ${EMBED_TARGET_DEPS} ${EMBED_TARGET_DEP_NAMES} + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + COMMENT "Generating JIT embed for ${TARGET} into ${OUTPUT_DIR}" + VERBATIM + ) + + add_custom_target( + ${TARGET} ALL + DEPENDS ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin + COMMENT "Custom target for JIT embed of ${TARGET}" + ) + + message( + STATUS + "JIT embed for target ${TARGET} will be generated into: ${OUTPUT_DIR}/${TARGET}.hpp ${OUTPUT_DIR}/${TARGET}.s ${OUTPUT_DIR}/${TARGET}.bin" + ) + + set(${TARGET}_INCLUDE_DIRS + "${OUTPUT_DIR}" + PARENT_SCOPE + ) + + set(${TARGET}_SOURCE_DIR + ${OUTPUT_DIR} + PARENT_SCOPE + ) + +endfunction() diff --git a/embed.hpp b/embed.hpp new file mode 100644 index 0000000..5df1aa4 --- /dev/null +++ b/embed.hpp @@ -0,0 +1,410 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include "hash.hpp" + +#include +#include +#include + +#define XXH_INLINE_ALL +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RTCX_EMBED_EXPECTS(condition, message) \ + do { \ + if (!(condition)) { \ + throw std::runtime_error(std::format("{}:{}: {}", __FILE__, __LINE__, message)); \ + } \ + } while (false) + +namespace rtcx_embed { + +struct size_range { + size_t offset = 0; + size_t size = 0; +}; + +struct embed_output { + std::string cxx_header; + std::string asm_source; + std::vector bin_file_data; +}; + +std::pair, std::vector> merge_bytes_with_null_terminators( + std::span const> bytes_lists) +{ + std::vector merged; + std::vector ranges; + + for (auto& byte_data : bytes_lists) { + ranges.push_back({merged.size(), byte_data.size()}); + merged.insert(merged.end(), byte_data.begin(), byte_data.end()); + merged.push_back(0); + } + + return {std::move(merged), std::move(ranges)}; +} + +std::vector load_file_bytes(std::string_view file_path) +{ + std::string path_str(file_path); + std::ifstream file(path_str, std::ios::binary | std::ios::ate); + RTCX_EMBED_EXPECTS(file.is_open(), std::format("Failed to open file at path: {}", file_path)); + auto file_size = file.tellg(); + RTCX_EMBED_EXPECTS(file_size >= 0, + std::format("Failed to determine size of file at path: {}", file_path)); + file.seekg(0, std::ios::beg); + std::vector bytes(file_size); + RTCX_EMBED_EXPECTS(file.read(reinterpret_cast(bytes.data()), file_size), + std::format("Failed to read file at path: {}", file_path)); + return bytes; +} + +std::vector compress_bytes(std::span bytes, std::string_view compression) +{ + RTCX_EMBED_EXPECTS( + compression == "none" || compression == "zstd", + std::format("Invalid compression type: {}. Supported values are 'none' and 'zstd'", + compression)); + + if (compression == "none") { return std::vector(bytes.begin(), bytes.end()); } + + auto const max_compressed_size = ZSTD_compressBound(bytes.size()); + std::vector compressed(max_compressed_size); + auto const compressed_size = + ZSTD_compress(compressed.data(), compressed.size(), bytes.data(), bytes.size(), 22); + + RTCX_EMBED_EXPECTS( + !ZSTD_isError(compressed_size), + std::format("Compression failed with error: {}", ZSTD_getErrorName(compressed_size))); + + compressed.resize(compressed_size); + return compressed; +} + +rtcx::hash128 compute_embed_hash(std::span uncompressed_files_bytes, + std::span merged_dests_bytes, + std::span merged_include_dirs_bytes, + std::string_view compression) +{ + XXH3_state_t state; + XXH3_INITSTATE(&state); + XXH3_128bits_reset(&state); + XXH3_128bits_update(&state, uncompressed_files_bytes.data(), uncompressed_files_bytes.size()); + XXH3_128bits_update(&state, merged_dests_bytes.data(), merged_dests_bytes.size()); + XXH3_128bits_update(&state, merged_include_dirs_bytes.data(), merged_include_dirs_bytes.size()); + XXH3_128bits_update(&state, compression.data(), compression.size()); + auto hash = XXH3_128bits_digest(&state); + return rtcx::hash128{hash.high64, hash.low64}; +} + +template +std::string join_formatted(Container& items, std::string_view delimiter, Formatter&& formatter) +{ + std::ostringstream result; + for (std::size_t i = 0; i < items.size(); ++i) { + if (i != 0) { result << delimiter; } + result << formatter(items[i]); + } + return result.str(); +} + +enum class value_type : int8_t { INT, STRING }; + +std::string generate_arrays(std::span array_ids, + std::span array_values) +{ + auto get_type = [](std::string_view value) -> value_type { + RTCX_EMBED_EXPECTS(!value.empty(), "Value cannot be empty"); + return std::isdigit(value[0]) ? value_type::INT : value_type::STRING; + }; + + using strings_t = std::vector; + using ints_t = std::vector; + using values_t = std::variant; + + std::map arrays; + + for (size_t i = 0; i < array_ids.size(); ++i) { + auto id = array_ids[i]; + auto value = array_values[i]; + auto type = get_type(value); + if (auto array_it = arrays.find(id); array_it == arrays.end()) { + switch (type) { + case value_type::INT: { + arrays.emplace(id, ints_t{}); + } break; + case value_type::STRING: { + arrays.emplace(id, strings_t{}); + } break; + default: throw std::logic_error("Unexpected constant type"); + } + } + + auto& array = arrays[id]; + + switch (type) { + case value_type::INT: { + std::int64_t int_value; + RTCX_EMBED_EXPECTS( + std::from_chars(value.data(), value.data() + value.size(), int_value).ec == std::errc(), + std::format("Invalid integer constant value: {}", value)); + std::get(array).push_back(int_value); + } break; + case value_type::STRING: { + std::get(array).push_back(value); + } break; + + default: break; + } + } + + std::string result; + + for (auto& [id, array] : arrays) { + if (auto* ints = std::get_if(&array); ints != nullptr) { + result += + std::format("constexpr std::int64_t {}[{}] = {{ {} }};\n\n", + id, + ints->size(), + join_formatted(*ints, ", ", [](std::int64_t v) { return std::to_string(v); })); + } else { + auto& strings = std::get(array); + result += + std::format("constexpr char const* {}[{}] = {{ {} }};\n\n", + id, + strings.size(), + join_formatted(strings, ", ", [](auto s) { return std::format("\"{}\"", s); })); + } + } + + return result; +} + +embed_output generate_cxx_source_files_data(std::string_view id, + std::span array_ids, + std::span array_values, + std::span file_ids, + std::span file_paths, + std::span file_dsts, + std::span include_dirs, + std::string_view compression) +{ + std::vector> file_bytes; + file_bytes.reserve(file_paths.size()); + for (auto const& path : file_paths) { + file_bytes.emplace_back(load_file_bytes(path)); + } + + auto [uncompressed_files_bytes, file_ranges] = merge_bytes_with_null_terminators(file_bytes); + + auto compress = compression != "none"; + std::vector compressed_files_bytes = + compress ? compress_bytes(uncompressed_files_bytes, compression) : uncompressed_files_bytes; + + auto binary_size = compress ? compressed_files_bytes.size() + : static_cast(uncompressed_files_bytes.size()); + + if (compress) { + std::cout << std::format( + "-- Compressed {}'s binary from {} bytes to {} bytes (compression ratio: {:.2f})\n", + id, + uncompressed_files_bytes.size(), + compressed_files_bytes.size(), + static_cast(compressed_files_bytes.size()) / + static_cast(uncompressed_files_bytes.size())); + } + + std::vector> destination_bytes; + destination_bytes.reserve(file_dsts.size()); + for (auto const& dest : file_dsts) { + destination_bytes.emplace_back(dest.begin(), dest.end()); + } + auto [merged_dests_bytes, _] = merge_bytes_with_null_terminators(destination_bytes); + + std::vector> include_directory_bytes; + include_directory_bytes.reserve(include_dirs.size()); + for (auto const& include_directory : include_dirs) { + include_directory_bytes.emplace_back(include_directory.begin(), include_directory.end()); + } + auto [merged_include_dirs_bytes, __] = merge_bytes_with_null_terminators(include_directory_bytes); + + auto hash = compute_embed_hash( + uncompressed_files_bytes, merged_dests_bytes, merged_include_dirs_bytes, compression); + + auto include_dirs_list = + join_formatted(include_dirs, ",\n", [](auto s) { return std::format("\"{}\"", s); }); + auto file_dests_list = + join_formatted(file_dsts, ",\n", [](auto s) { return std::format("\"{}\"", s); }); + auto file_ids_list = + join_formatted(file_ids, ",\n", [](auto s) { return std::format("\"{}\"", s); }); + std::vector file_indices(file_ids.size()); + std::iota(file_indices.begin(), file_indices.end(), 0ULL); + auto file_indices_list = join_formatted(file_indices, "\n", [&](auto i) { + return std::format("constexpr std::size_t {} = {}ULL;", file_ids[i], i); + }); + auto file_ranges_list = join_formatted( + file_ranges, ",\n", [](auto r) { return std::format("{{{}, {}}}", r.offset, r.size); }); + auto hash_list = join_formatted( + hash, ", ", [](uint8_t byte) { return std::format("0x{:02x}", static_cast(byte)); }); + auto arrays_list = generate_arrays(array_ids, array_values); + auto namespace_decl = "namespace " + std::string(id); + + auto cxx_header = std::format( + R"***( +// Auto-generated header for embedded files +#pragma once + +#include +#include +#include +#include + +{} {{ + + +constexpr char const * include_directories[{}] = +{{ +{} +}}; + +{} + +constexpr char const * file_ids[{}] = +{{ +{} +}}; + +constexpr char const * file_destinations[{}] = +{{ +{} +}}; + +constexpr std::size_t file_ranges[{}][2] = +{{ +{} +}}; + +constexpr std::size_t files_uncompressed_size = {}; + +constexpr char const * files_compression = "{}"; + +extern "C" std::uint8_t const {}_files_begin[]; + +static std::span const files = +{{ +{}_files_begin, +{}L +}}; + +constexpr std::uint8_t hash[{}] = +{{ +{} +}}; + +{} + +}} +)***", + namespace_decl, + include_dirs.size(), + include_dirs_list, + file_indices_list, + file_ids.size(), + file_ids_list, + file_dsts.size(), + file_dests_list, + file_ranges.size(), + file_ranges_list, + uncompressed_files_bytes.size(), + compression, + id, + id, + binary_size, + hash.size(), + hash_list, + arrays_list); + + auto asm_source = std::format( + R"***( +.section .rodata +.global {0}_files_begin +{0}_files_begin: +.incbin "{0}.bin" + +.section .note.GNU-stack,"",@progbits +)***", + id); + + return embed_output{ + .cxx_header = cxx_header, + .asm_source = asm_source, + .bin_file_data = compress ? compressed_files_bytes : uncompressed_files_bytes}; +} + +void generate_embed(std::string_view id, + std::span array_ids, + std::span array_values, + std::span file_ids, + std::span file_paths, + std::span file_dsts, + std::span include_dirs, + std::string_view compression, + std::string_view output_directory) +{ + auto output = generate_cxx_source_files_data( + id, array_ids, array_values, file_ids, file_paths, file_dsts, include_dirs, compression); + + std::filesystem::create_directories(std::filesystem::path(output_directory)); + + std::ofstream header_file(std::format("{}/{}.hpp", output_directory, id)); + header_file << output.cxx_header; + + std::ofstream asm_file(std::format("{}/{}.s", output_directory, id)); + asm_file << output.asm_source; + + std::ofstream bin_file(std::format("{}/{}.bin", output_directory, id), std::ios::binary); + bin_file.write(reinterpret_cast(output.bin_file_data.data()), + static_cast(output.bin_file_data.size())); +} + +std::vector split_string(std::string_view str, char delimiter) +{ + if (str.empty()) { return {}; } + std::vector tokens; + std::size_t start = 0; + + while (start <= str.size()) { + auto const pos = str.find(delimiter, start); + if (pos == std::string_view::npos) { + tokens.push_back(str.substr(start)); + break; + } + tokens.push_back(str.substr(start, pos - start)); + start = pos + 1; + } + + return tokens; +} + +} // namespace rtcx_embed diff --git a/embed.in.cpp b/embed.in.cpp new file mode 100644 index 0000000..c53a12d --- /dev/null +++ b/embed.in.cpp @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "embed.hpp" + +int main() +{ + std::string_view id = "@EMBED_SCRIPT__ID@"; + auto array_ids = rtcx_embed::split_string("@EMBED_SCRIPT__ARRAY_IDS@", ';'); + auto array_values = rtcx_embed::split_string("@EMBED_SCRIPT__ARRAY_VALUES@", ';'); + auto file_ids = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_IDS@", ';'); + auto file_paths = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_PATHS@", ';'); + auto file_dests = rtcx_embed::split_string("@EMBED_SCRIPT__FILE_DESTS@", ';'); + auto include_directories = rtcx_embed::split_string("@EMBED_SCRIPT__INCLUDE_DIRS@", ';'); + std::string_view compression = "@EMBED_SCRIPT__COMPRESSION@"; + std::string_view output_dir = "@EMBED_SCRIPT__OUTPUT_DIR@"; + + rtcx_embed::generate_embed(id, + array_ids, + array_values, + file_ids, + file_paths, + file_dests, + include_directories, + compression, + output_dir); + return EXIT_SUCCESS; +} diff --git a/hash.cpp b/hash.cpp new file mode 100644 index 0000000..fd32d55 --- /dev/null +++ b/hash.cpp @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "hash.hpp" + +#include +#include +#include +#include +#include + +namespace rtcx { + +char const* hash128_hex_string::data() const { return data_; } + +char const* hash128_hex_string::c_str() const { return data_; } + +hash128_hex_string hash128_hex_string::make(std::span input) +{ + constexpr char const HEX_CHARS[] = "0123456789abcdef"; // NOLINT(modernize-avoid-c-arrays) + hash128_hex_string hex; + for (std::size_t i = 0; i < NUM_HEX_BYTES; ++i) { + hex.data_[i * 2] = HEX_CHARS[(input[i] >> 4) & 0x0F]; + hex.data_[i * 2 + 1] = HEX_CHARS[input[i] & 0x0F]; + } + hex.data_[NUM_HEX_DIGITS] = '\0'; + return hex; +} + +hash128_hex_string hash128_hex_string::make(__uint128_t hash) +{ + auto array = std::bit_cast>(hash); + return make(array); +} + +std::uint8_t hash128::operator[](std::size_t index) const +{ + return reinterpret_cast(&value)[15 - index]; +} + +std::size_t hash128::size() const { return 16; } + +std::uint8_t const* hash128::data() const { return reinterpret_cast(&value); } + +hash128_hex_string hash128::to_hex_string() const { return hash128_hex_string::make(value); } + +hash128 hash128::parse(std::string_view hex) +{ + if (hex.size() != hash128_hex_string::NUM_HEX_DIGITS) { + throw std::invalid_argument( + std::format("Invalid hash128 hex string length, expected {} got {} (hash: `{}`)", + hash128_hex_string::NUM_HEX_DIGITS, + hex.size(), + hex)); + } + std::array data{}; + for (std::size_t i = 0; i < hash128_hex_string::NUM_HEX_BYTES; ++i) { + auto hex_byte = hex.substr(i * 2, 2); + auto [ptr, ec] = std::from_chars(hex_byte.begin(), hex_byte.end(), data[i], 16); + if (ec != std::errc()) { + throw std::invalid_argument( + std::format("Invalid hex character {} in HEX string: `{}`", hex_byte, hex)); + } + } + return hash128{std::bit_cast<__uint128_t>(data)}; +} + +} // namespace rtcx diff --git a/hash.hpp b/hash.hpp new file mode 100644 index 0000000..d1aa12c --- /dev/null +++ b/hash.hpp @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include + +namespace rtcx { + +struct [[nodiscard]] hash128_hex_string { + static constexpr std::size_t NUM_HEX_DIGITS = 32; + static constexpr std::size_t NUM_HEX_BYTES = NUM_HEX_DIGITS / 2; + + char data_[NUM_HEX_DIGITS + 1]; // NOLINT(modernize-avoid-c-arrays) + + [[nodiscard]] constexpr std::string_view view() const + { + return std::string_view{data_, NUM_HEX_DIGITS}; + } + + [[nodiscard]] constexpr operator std::string_view() const { return view(); } + + [[nodiscard]] char const* data() const; + + [[nodiscard]] char const* c_str() const; + + [[nodiscard]] static constexpr std::size_t size() { return NUM_HEX_DIGITS; } + + static hash128_hex_string make(std::span input); + + static hash128_hex_string make(__uint128_t hash); +}; + +struct hash128 { + __uint128_t value; + + constexpr hash128(__uint128_t v = 0) : value(v) {} + + constexpr hash128(std::uint64_t high, std::uint64_t low) + : value((static_cast<__uint128_t>(high) << 64) | low) + { + } + + [[nodiscard]] constexpr bool operator==(hash128 const&) const = default; + + [[nodiscard]] std::uint8_t operator[](std::size_t index) const; + + [[nodiscard]] std::size_t size() const; + + [[nodiscard]] std::uint8_t const* data() const; + + hash128_hex_string to_hex_string() const; + + static hash128 parse(std::string_view hex); +}; + +} // namespace rtcx diff --git a/libcxx/cassert b/libcxx/cassert new file mode 100644 index 0000000..6bb3f07 --- /dev/null +++ b/libcxx/cassert @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include diff --git a/libcxx/climits b/libcxx/climits new file mode 100644 index 0000000..20653c8 --- /dev/null +++ b/libcxx/climits @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include diff --git a/libcxx/cstddef b/libcxx/cstddef new file mode 100644 index 0000000..d0c1fb2 --- /dev/null +++ b/libcxx/cstddef @@ -0,0 +1,18 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace std { + +using max_align_t = cuda::std::max_align_t; +using nullptr_t = cuda::std::nullptr_t; +using ptrdiff_t = cuda::std::ptrdiff_t; +using size_t = cuda::std::size_t; +using byte = cuda::std::byte; + +} // namespace std diff --git a/libcxx/cstdint b/libcxx/cstdint new file mode 100644 index 0000000..eae43f7 --- /dev/null +++ b/libcxx/cstdint @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace std { + +using int8_t = cuda::std::int8_t; +using int16_t = cuda::std::int16_t; +using int32_t = cuda::std::int32_t; +using int64_t = cuda::std::int64_t; +using uint8_t = cuda::std::uint8_t; +using uint16_t = cuda::std::uint16_t; +using uint32_t = cuda::std::uint32_t; +using uint64_t = cuda::std::uint64_t; + +using int_fast8_t = cuda::std::int_fast8_t; +using int_fast16_t = cuda::std::int_fast16_t; +using int_fast32_t = cuda::std::int_fast32_t; +using int_fast64_t = cuda::std::int_fast64_t; +using uint_fast8_t = cuda::std::uint_fast8_t; +using uint_fast16_t = cuda::std::uint_fast16_t; +using uint_fast32_t = cuda::std::uint_fast32_t; +using uint_fast64_t = cuda::std::uint_fast64_t; + +using int_least8_t = cuda::std::int_least8_t; +using int_least16_t = cuda::std::int_least16_t; +using int_least32_t = cuda::std::int_least32_t; +using int_least64_t = cuda::std::int_least64_t; +using uint_least8_t = cuda::std::uint_least8_t; +using uint_least16_t = cuda::std::uint_least16_t; +using uint_least32_t = cuda::std::uint_least32_t; +using uint_least64_t = cuda::std::uint_least64_t; + +using intptr_t = cuda::std::intptr_t; +using uintptr_t = cuda::std::uintptr_t; + +using intmax_t = cuda::std::intmax_t; +using uintmax_t = cuda::std::uintmax_t; + +} // namespace std diff --git a/rtcx.cpp b/rtcx.cpp new file mode 100644 index 0000000..1c31448 --- /dev/null +++ b/rtcx.cpp @@ -0,0 +1,1399 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef RTCX_STATIC_LINK_LIBNVRTC +#define RTCX_STATIC_LINK_LIBNVRTC 0 +#endif + +#ifndef RTCX_STATIC_LINK_LIBNVJITLINK +#define RTCX_STATIC_LINK_LIBNVJITLINK 0 +#endif + +#define RTCX_EXPECTS(condition_, reason_, exception_type_) \ + do { \ + if (!(condition_)) { \ + throw exception_type_{::std::format("RTCX failure at: {}:{}: {}", \ + ::std::source_location::current().file_name(), \ + ::std::source_location::current().line(), \ + (reason_))}; \ + } \ + } while (0) + +#define RTCX_FAIL(reason_, exception_type_) \ + do { \ + throw exception_type_{::std::format("RTCX failure at: {}:{}: {}", \ + ::std::source_location::current().file_name(), \ + ::std::source_location::current().line(), \ + (reason_))}; \ + } while (0) + +#define RTCX_CHECK_CUDA(...) \ + do { \ + ::CUresult result_ = (__VA_ARGS__); \ + if (result_ != ::CUDA_SUCCESS) { \ + char const* enum_str_; \ + RTCX_EXPECTS(::rtcx::cu->GetErrorString(result_, &enum_str_) == ::CUDA_SUCCESS, \ + "Unable to get CUDA error string", \ + std::runtime_error); \ + auto errstr_ = ::std::format("(cuda) expression `{}` failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::std::int64_t>(result_), \ + enum_str_); \ + RTCX_FAIL(errstr_, ::std::runtime_error); \ + } \ + } while (0) + +#define RTCX_CHECK_CUDART(...) \ + do { \ + ::cudaError_t result_ = (__VA_ARGS__); \ + if (result_ != ::cudaSuccess) { \ + char const* enum_name_ = ::cudaGetErrorName(result_); \ + char const* enum_msg_ = ::cudaGetErrorString(result_); \ + auto errstr_ = ::std::format("(cudart) expression `{}` failed, with error ({}: {}): {}", \ + #__VA_ARGS__, \ + static_cast<::std::int64_t>(result_), \ + enum_name_, \ + enum_msg_); \ + RTCX_FAIL(errstr_, ::std::runtime_error); \ + } \ + } while (0) + +#define RTCX_CHECK_NVRTC(...) \ + do { \ + ::nvrtcResult result_ = (__VA_ARGS__); \ + if (result_ != ::NVRTC_SUCCESS) { \ + auto errstr_ = ::std::format("(nvrtc) expression `{}` failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::std::int64_t>(result_), \ + ::rtcx::nvrtc->GetErrorString(result_)); \ + RTCX_FAIL(errstr_, ::std::runtime_error); \ + } \ + } while (0) + +#define RTCX_CHECK_NVJITLINK(...) \ + do { \ + ::nvJitLinkResult result_ = (__VA_ARGS__); \ + if (result_ != ::NVJITLINK_SUCCESS) { \ + auto errstr_ = ::std::format("(nvJitLink) expression `{}` failed, with error ({}): {}", \ + #__VA_ARGS__, \ + static_cast<::std::int64_t>(result_), \ + ::rtcx::nvJitLinkResult_string(result_)); \ + RTCX_FAIL(errstr_, ::std::runtime_error); \ + } \ + } while (0) + +#define RTCX_FUNC_RANGE() \ + ::nvtx3::scoped_range_in<::rtcx::nvtx_domain> rtcx_func_range__ { __func__ } + +namespace rtcx { +namespace { + +struct nvtx_domain { + static constexpr char const* name [[maybe_unused]] = "rtcx"; +}; + +enum class object_type : std::uint8_t { LIBRARY, BLOB }; + +std::string_view object_tag(object_type type) +{ + switch (type) { + case object_type::LIBRARY: return "cuLibrary"; + case object_type::BLOB: return "blob"; + default: + RTCX_FAIL(std::format("Unrecognized object type: ({})", static_cast(type)), + std::runtime_error); + } +} + +template +std::string join_strings(std::span strings, std::string_view separator) +{ + if (strings.empty()) { return {}; } + + if (strings.size() == 1) { return std::string{strings[0].begin(), strings[0].end()}; } + + auto total_size = std::transform_reduce( + strings.begin(), + strings.end(), + size_t{0}, + [](size_t total, size_t str_size) { return total + str_size; }, + [](auto& str) { return str.size(); }); + + auto separator_size = separator.size() * (strings.size() - 1); + + std::string result; + result.reserve(total_size + separator_size); + + for (size_t i = 0; i < strings.size(); ++i) { + result.append(strings[i].begin(), strings[i].end()); + if (i != (strings.size() - 1)) { result.append(separator); } + } + + return result; +} + +} // namespace + +void log_warning(std::string_view msg) +{ + std::fprintf(stderr, "[rtcx] warn: %.*s\n", static_cast(msg.size()), msg.data()); +} + +void log_error(std::string_view msg) +{ + std::fprintf(stderr, "[rtcx] error: %.*s\n", static_cast(msg.size()), msg.data()); +} + +#define FOR_EACH_CUDA_FUNC(DO_IT) \ + DO_IT(GetErrorString) \ + DO_IT(GetErrorName) \ + DO_IT(Init) \ + DO_IT(OccupancyMaxPotentialBlockSize) \ + DO_IT(LaunchKernel) \ + DO_IT(LaunchKernelEx) \ + DO_IT(LaunchCooperativeKernel) \ + DO_IT(KernelGetFunction) \ + DO_IT(LibraryLoadData) \ + DO_IT(LibraryLoadFromFile) \ + DO_IT(LibraryGetKernel) \ + DO_IT(LibraryUnload) + +#define FOR_EACH_NVRTC_FUNC(DO_IT) \ + DO_IT(Version) \ + DO_IT(GetErrorString) \ + DO_IT(CreateProgram) \ + DO_IT(DestroyProgram) \ + DO_IT(CompileProgram) \ + DO_IT(GetPTXSize) \ + DO_IT(GetPTX) \ + DO_IT(GetCUBINSize) \ + DO_IT(GetCUBIN) \ + DO_IT(GetLTOIRSize) \ + DO_IT(GetLTOIR) \ + DO_IT(GetProgramLogSize) \ + DO_IT(GetProgramLog) \ + DO_IT(AddNameExpression) \ + DO_IT(GetLoweredName) + +#define FOR_EACH_NVJITLINK_FUNC(DO_IT) \ + DO_IT(Version) \ + DO_IT(Create) \ + DO_IT(Destroy) \ + DO_IT(AddData) \ + DO_IT(AddFile) \ + DO_IT(Complete) \ + DO_IT(GetLinkedCubinSize) \ + DO_IT(GetLinkedCubin) \ + DO_IT(GetLinkedPtxSize) \ + DO_IT(GetLinkedPtx) \ + DO_IT(GetErrorLogSize) \ + DO_IT(GetErrorLog) \ + DO_IT(GetInfoLog) \ + DO_IT(GetInfoLogSize) + +namespace { + +std::string_view nvJitLinkResult_string(nvJitLinkResult result) +{ + switch (result) { + case NVJITLINK_SUCCESS: return "NVJITLINK_SUCCESS"; + case NVJITLINK_ERROR_UNRECOGNIZED_OPTION: return "NVJITLINK_ERROR_UNRECOGNIZED_OPTION"; + case NVJITLINK_ERROR_MISSING_ARCH: return "NVJITLINK_ERROR_MISSING_ARCH"; + case NVJITLINK_ERROR_INVALID_INPUT: return "NVJITLINK_ERROR_INVALID_INPUT"; + case NVJITLINK_ERROR_PTX_COMPILE: return "NVJITLINK_ERROR_PTX_COMPILE"; + case NVJITLINK_ERROR_NVVM_COMPILE: return "NVJITLINK_ERROR_NVVM_COMPILE"; + case NVJITLINK_ERROR_INTERNAL: return "NVJITLINK_ERROR_INTERNAL"; + case NVJITLINK_ERROR_THREADPOOL: return "NVJITLINK_ERROR_THREADPOOL"; + case NVJITLINK_ERROR_UNRECOGNIZED_INPUT: return "NVJITLINK_ERROR_UNRECOGNIZED_INPUT"; + case NVJITLINK_ERROR_FINALIZE: return "NVJITLINK_ERROR_FINALIZE"; +#if CUDA_VERSION >= 13000 + case NVJITLINK_ERROR_NULL_INPUT: return "NVJITLINK_ERROR_NULL_INPUT"; + case NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS: return "NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS"; + case NVJITLINK_ERROR_INCORRECT_INPUT_TYPE: return "NVJITLINK_ERROR_INCORRECT_INPUT_TYPE"; + case NVJITLINK_ERROR_ARCH_MISMATCH: return "NVJITLINK_ERROR_ARCH_MISMATCH"; + case NVJITLINK_ERROR_OUTDATED_LIBRARY: return "NVJITLINK_ERROR_OUTDATED_LIBRARY"; + case NVJITLINK_ERROR_MISSING_FATBIN: return "NVJITLINK_ERROR_MISSING_FATBIN"; + case NVJITLINK_ERROR_UNRECOGNIZED_ARCH: return "NVJITLINK_ERROR_UNRECOGNIZED_ARCH"; + case NVJITLINK_ERROR_UNSUPPORTED_ARCH: return "NVJITLINK_ERROR_UNSUPPORTED_ARCH"; + case NVJITLINK_ERROR_LTO_NOT_ENABLED: return "NVJITLINK_ERROR_LTO_NOT_ENABLED"; +#endif + default: + RTCX_FAIL( + std::format("Unrecognized nvJitLinkResult type: ({})", static_cast(result)), + std::runtime_error); + } +} + +std::string_view binary_type_string(binary_type type) +{ + switch (type) { + case binary_type::LTO_IR: return "LTO_IR"; + case binary_type::CUBIN: return "CUBIN"; + case binary_type::FATBIN: return "FATBIN"; + case binary_type::PTX: return "PTX"; + default: + RTCX_FAIL(std::format("Unrecognized binary_type: ({})", static_cast(type)), + std::runtime_error); + } +} + +nvJitLinkInputType to_nvjitlink_input_type(binary_type bin_type) +{ + switch (bin_type) { + case binary_type::LTO_IR: return NVJITLINK_INPUT_LTOIR; + case binary_type::CUBIN: return NVJITLINK_INPUT_CUBIN; + case binary_type::FATBIN: return NVJITLINK_INPUT_FATBIN; + case binary_type::PTX: return NVJITLINK_INPUT_PTX; + default: + RTCX_FAIL(std::format("Unrecognized binary type for linking: ({}) ", + static_cast(bin_type)), + std::logic_error); + } +} + +[[maybe_unused]] void* load_dso(std::string_view base_name, std::span names) +{ + for (auto& name : names) { + void* handle = ::dlopen(name.c_str(), RTLD_NOW | RTLD_LOCAL); + if (handle != nullptr) { return handle; } + } + + RTCX_FAIL( + std::format( + "Failed to load dynamic library `{}` (tried: {})", base_name, join_strings(names, ", ")), + std::runtime_error); +} + +[[maybe_unused]] void* get_dso_symbol(char const* lib_name, void* handle, char const* sym_name) +{ + void* sym = ::dlsym(handle, sym_name); + if (sym == nullptr) { + RTCX_FAIL( + std::format( + "Failed to load symbol `{}` from `{}`, error: `{}`", sym_name, lib_name, ::dlerror()), + std::runtime_error); + } + return sym; +} + +inline constexpr std::int32_t major_version(std::int32_t version) { return version / 1000; } + +struct LibCuda { + void* _handle = nullptr; + +#define DO_IT(func) decltype(::cu##func)* func = nullptr; + FOR_EACH_CUDA_FUNC(DO_IT) +#undef DO_IT + + explicit LibCuda(void* handle) : _handle(handle) { _load_symbols(); } + LibCuda(LibCuda const&) = delete; + LibCuda(LibCuda&&) = delete; + LibCuda& operator=(LibCuda const&) = delete; + LibCuda& operator=(LibCuda&&) = delete; + ~LibCuda() { ::dlclose(_handle); } + + static void* _load() + { + std::string lib_names[] = {"libcuda.so.1"}; // NOLINT(modernize-avoid-c-arrays) + return load_dso("libcuda.so", lib_names); + } + + private: + void _load_symbols() + { +#define DO_IT(func) \ + this->func = \ + reinterpret_cast(get_dso_symbol("libcuda", _handle, "cu" #func)); + + FOR_EACH_CUDA_FUNC(DO_IT) + +#undef DO_IT + } +}; + +struct LibNVRTC { + void* _handle = nullptr; + +#define DO_IT(func) decltype(::nvrtc##func)* func = nullptr; + FOR_EACH_NVRTC_FUNC(DO_IT) +#undef DO_IT + + explicit LibNVRTC(void* handle) : _handle(handle) { _load_symbols(); } + LibNVRTC(LibNVRTC const&) = delete; + LibNVRTC(LibNVRTC&&) = delete; + LibNVRTC& operator=(LibNVRTC const&) = delete; + LibNVRTC& operator=(LibNVRTC&&) = delete; + ~LibNVRTC() + { +#if !RTCX_STATIC_LINK_LIBNVRTC + ::dlclose(_handle); +#endif + } + + static void* _load() + { +#if !RTCX_STATIC_LINK_LIBNVRTC + auto expected_major_version = major_version(CUDA_VERSION); + std::int32_t cuda_version; + RTCX_CHECK_CUDART(::cudaRuntimeGetVersion(&cuda_version)); + std::int32_t major = major_version(cuda_version); + RTCX_EXPECTS(expected_major_version == major, + std::format("LibNVRTC Compatibility Error: CUDA major version mismatch. Expected " + "major runtime version: {}, got major runtime version: {})", + expected_major_version, + major), + std::runtime_error); + + std::string lib_names[] = // NOLINT(modernize-avoid-c-arrays) + {std::format("libnvrtc.so.{}", major)}; + + return load_dso("libnvrtc.so", lib_names); +#else + return nullptr; +#endif + } + + private: + void _load_symbols() + { +#if !RTCX_STATIC_LINK_LIBNVRTC +#define DO_IT(func) \ + this->func = \ + reinterpret_cast(get_dso_symbol("libnvrtc", _handle, "nvrtc" #func)); +#else +#define DO_IT(func) this->func = ::nvrtc##func; +#endif + + FOR_EACH_NVRTC_FUNC(DO_IT) +#undef DO_IT + } +}; + +struct LibNVJitLink { + void* _handle = nullptr; + +#define DO_IT(func) decltype(::nvJitLink##func)* func = nullptr; + FOR_EACH_NVJITLINK_FUNC(DO_IT) +#undef DO_IT + + explicit LibNVJitLink(void* handle) : _handle(handle) { _load_symbols(); } + LibNVJitLink(LibNVJitLink const&) = delete; + LibNVJitLink(LibNVJitLink&&) = delete; + LibNVJitLink& operator=(LibNVJitLink const&) = delete; + LibNVJitLink& operator=(LibNVJitLink&&) = delete; + ~LibNVJitLink() + { +#if !RTCX_STATIC_LINK_LIBNVJITLINK + ::dlclose(_handle); +#endif + } + + static void* _load() + { +#if !RTCX_STATIC_LINK_LIBNVJITLINK + auto expected_major_version = major_version(CUDA_VERSION); + std::int32_t cuda_version; + RTCX_CHECK_CUDART(::cudaRuntimeGetVersion(&cuda_version)); + std::int32_t major = major_version(cuda_version); + RTCX_EXPECTS( + expected_major_version == major, + std::format("LibNVJitLink Compatibility Error: CUDA major version mismatch. Expected " + "major runtime version: {}, got major runtime version: {})", + expected_major_version, + major), + std::runtime_error); + + std::string lib_names[] = // NOLINT(modernize-avoid-c-arrays) + {std::format("libnvJitLink.so.{}", major)}; + + return load_dso("libnvJitLink.so", lib_names); +#else + return nullptr; +#endif + } + + private: + void _load_symbols() + { +#if !RTCX_STATIC_LINK_LIBNVJITLINK +#define DO_IT(func) \ + this->func = reinterpret_cast( \ + get_dso_symbol("libnvJitLink", _handle, "nvJitLink" #func)); +#else +#define DO_IT(func) this->func = ::nvJitLink##func; +#endif + + FOR_EACH_NVJITLINK_FUNC(DO_IT) +#undef DO_IT + } +}; + +static std::optional cu; +static std::optional nvrtc; +static std::optional nvjitlink; +static std::optional init_libraries_flag{std::in_place}; +static std::optional teardown_libraries_flag{std::in_place}; + +} // namespace + +void initialize() +{ + RTCX_FUNC_RANGE(); + + std::call_once(init_libraries_flag.value(), [] { + cu.emplace(LibCuda::_load()); + RTCX_EXPECTS( + cu->Init(0) == CUDA_SUCCESS, "Failed to initialize CUDA driver API", std::runtime_error); + nvrtc.emplace(LibNVRTC::_load()); + nvjitlink.emplace(LibNVJitLink::_load()); + }); +} + +void teardown() +{ + RTCX_FUNC_RANGE(); + + std::call_once(teardown_libraries_flag.value(), [] { + nvjitlink.reset(); + nvrtc.reset(); + cu.reset(); + init_libraries_flag.reset(); + teardown_libraries_flag.reset(); + init_libraries_flag.emplace(); + teardown_libraries_flag.emplace(); + }); +} + +blob_t blob_t::from_buffer(byte_buffer buffer) +{ + auto size = buffer.size(); + auto data = buffer.release(); + return blob_t::from_parts( + data, size, +[](std::uint8_t const* data, std::size_t) { + ::free(const_cast(data)); + }); +} + +blob_t blob_t::from_static_data(std::span data) +{ + return blob_t::from_parts(data.data(), data.size(), blob_t::noop_deallocator); +} + +namespace { +void log_nvrtc_result(compile_params const& params, + nvrtcProgram program, + nvrtcResult compile_result) +{ + if (program == nullptr) { return; } + + std::size_t log_size; + if (auto errc = nvrtc->GetProgramLogSize(program, &log_size); errc != NVRTC_SUCCESS) { + RTCX_FAIL(std::format("Failed to get NVRTC program log size with error ({}): {}", + static_cast(errc), + nvrtc->GetErrorString(errc)), + std::runtime_error); + } + + std::vector log; + + if (log_size > 1) { + log.resize(log_size); + if (auto errc = nvrtc->GetProgramLog(program, log.data()); errc != NVRTC_SUCCESS) { + RTCX_FAIL(std::format("Failed to get NVRTC program log with error ({}): {}", + static_cast(errc), + nvrtc->GetErrorString(errc)), + std::runtime_error); + } + } + + log.resize(log_size == 0 ? 0 : (log_size - 1)); + + auto status_str = + (compile_result == NVRTC_SUCCESS && !log.empty()) ? "completed with" : "failed with"; + + std::string headers_str; + for (auto& header : params.header_include_names) { + headers_str = std::format("{}\t{}\n", headers_str, header); + } + + std::string options_str; + for (auto& option : params.options) { + options_str = std::format("{}\t{}\n", options_str, option); + } + + if (log.empty()) { return; } + + auto msg = std::format( + "NVRTC Compilation for `{}` {} ({}): {}.\nHeaders:\n{}\n\nOptions:\n{}\n\nLog:\n\t{}", + params.name == nullptr ? "" : params.name, + status_str, + static_cast(compile_result), + nvrtc->GetErrorString(compile_result), + headers_str, + options_str, + std::string_view{log.data(), log.size()}); + + if (compile_result != NVRTC_SUCCESS) { + log_error(msg); + } else { + log_warning(msg); + } +} + +void log_nvJitLink_result(link_params const& params, + nvJitLinkHandle handle, + nvJitLinkResult link_result) +{ + if (handle == nullptr) { return; } + + std::size_t info_log_size; + if (auto errc = nvjitlink->GetInfoLogSize(handle, &info_log_size); errc != NVJITLINK_SUCCESS) { + RTCX_FAIL(std::format("Failed to get nvJitLink info log size with error ({}): {}", + static_cast(errc), + nvJitLinkResult_string(errc)), + std::runtime_error); + } + + std::vector info_log; + if (info_log_size > 1) { + info_log.resize(info_log_size); + if (auto errc = nvjitlink->GetInfoLog(handle, info_log.data()); errc != NVJITLINK_SUCCESS) { + RTCX_FAIL(std::format("Failed to get nvJitLink info log with error ({}): {}", + static_cast(errc), + nvJitLinkResult_string(errc)), + std::runtime_error); + } + } + + info_log.resize(info_log_size == 0 ? 0 : (info_log_size - 1)); + + std::size_t error_log_size; + if (auto errc = nvjitlink->GetErrorLogSize(handle, &error_log_size); errc != NVJITLINK_SUCCESS) { + RTCX_FAIL(std::format("Failed to get nvJitLink error log size with error ({}): {}", + static_cast(errc), + nvJitLinkResult_string(errc)), + std::runtime_error); + } + + std::vector error_log; + + if (error_log_size > 1) { + error_log.resize(error_log_size); + if (auto errc = nvjitlink->GetErrorLog(handle, error_log.data()); errc != NVJITLINK_SUCCESS) { + RTCX_FAIL(std::format("Failed to get nvJitLink error log with error ({}): {}", + static_cast(errc), + nvJitLinkResult_string(errc)), + std::runtime_error); + } + } + + error_log.resize(error_log_size == 0 ? 0 : (error_log_size - 1)); + + if (info_log.empty() && error_log.empty()) { return; } + + std::string fragments_str; + for (auto& frag : params.file_fragments) { + fragments_str = std::format("{}\t{}\n", fragments_str, frag.path); + } + + for (auto& frag : params.memory_fragments) { + fragments_str = + std::format("{}\t{}\n", fragments_str, frag.name == nullptr ? "" : frag.name); + } + + std::string link_options_str; + for (auto& option : params.link_options) { + link_options_str = std::format("{}\t{}\n", link_options_str, option); + } + + auto status_str = link_result == NVJITLINK_SUCCESS ? "completed with" : "failed with"; + + auto msg = std::format( + "(nvJitLink) Linking for `{}` ({}) {} error code ({}): {}.\nFragments: \n{}\n" + "Link Options: \n{}\n\nInfo Log:\n\t{}\n\nError Log:\n\t{}\n\n", + params.name == nullptr ? "" : params.name, + binary_type_string(params.output_type), + status_str, + static_cast(link_result), + nvJitLinkResult_string(link_result), + fragments_str, + link_options_str, + std::string_view{info_log.data(), info_log.size()}, + std::string_view{error_log.data(), error_log.size()}); + + bool needs_info_log = + std::find_if( + params.link_options.begin(), params.link_options.end(), [](std::string_view option) { + return option == "--verbose" || option == "-time"; + }) != params.link_options.end(); + + if (link_result == NVJITLINK_SUCCESS) { + if (needs_info_log) { log_warning(msg); } + } else { + log_error(msg); + } +} + +} // namespace + +std::int32_t nvrtc_version() +{ + RTCX_FUNC_RANGE(); + + std::int32_t major, minor; + RTCX_CHECK_NVRTC(nvrtc->Version(&major, &minor)); + return major * 1000 + minor * 10; +} + +std::int32_t nvjitlink_version() +{ + RTCX_FUNC_RANGE(); + + std::uint32_t major, minor; + RTCX_CHECK_NVJITLINK(nvjitlink->Version(&major, &minor)); + return static_cast(major * 1000 + minor * 10); +} + +byte_buffer compile(compile_params const& params) +{ + RTCX_FUNC_RANGE(); + + RTCX_EXPECTS(params.name != nullptr, "Fragment name must not be null", std::logic_error); + RTCX_EXPECTS(params.source != nullptr, "Fragment source must not be null", std::logic_error); + + nvrtcProgram program = nullptr; + RTCX_CHECK_NVRTC(nvrtc->CreateProgram(&program, + params.source, + params.name, + static_cast(params.headers.size()), + params.headers.data(), + params.header_include_names.data())); + + RTCX_DEFER([&] { nvrtc->DestroyProgram(&program); }); + + for (auto* name_expr : params.name_expressions) { + RTCX_CHECK_NVRTC(nvrtc->AddNameExpression(program, name_expr)); + } + + auto compile_result = nvrtc->CompileProgram( + program, static_cast(params.options.size()), params.options.data()); + log_nvrtc_result(params, program, compile_result); + RTCX_CHECK_NVRTC(compile_result); + + switch (params.target_type) { + case binary_type::CUBIN: { + std::size_t cubin_size; + RTCX_CHECK_NVRTC(nvrtc->GetCUBINSize(program, &cubin_size)); + auto cubin = byte_buffer::make(cubin_size); + RTCX_CHECK_NVRTC(nvrtc->GetCUBIN(program, reinterpret_cast(cubin.data()))); + return cubin; + } break; + case binary_type::LTO_IR: { + std::size_t lto_ir_size; + RTCX_CHECK_NVRTC(nvrtc->GetLTOIRSize(program, <o_ir_size)); + auto lto_ir = byte_buffer::make(lto_ir_size); + RTCX_CHECK_NVRTC(nvrtc->GetLTOIR(program, reinterpret_cast(lto_ir.data()))); + return lto_ir; + } break; + case binary_type::PTX: { + std::size_t ptx_size; + RTCX_CHECK_NVRTC(nvrtc->GetPTXSize(program, &ptx_size)); + auto ptx = byte_buffer::make(ptx_size); + RTCX_CHECK_NVRTC(nvrtc->GetPTX(program, reinterpret_cast(ptx.data()))); + return ptx; + } break; + default: + RTCX_FAIL(std::format("Unsupported binary type for compiling fragment: {}", + binary_type_string(params.target_type)), + std::logic_error); + } +} + +kernel_occupancy_config kernel_ref::max_occupancy_config(std::size_t dynamic_shared_memory_bytes, + std::int32_t block_size_limit) const +{ + std::int32_t min_grid_size; + std::int32_t block_size; + RTCX_CHECK_CUDA(cu->OccupancyMaxPotentialBlockSize(&min_grid_size, + &block_size, + reinterpret_cast(handle_), + nullptr, + dynamic_shared_memory_bytes, + block_size_limit)); + + return kernel_occupancy_config{.min_grid_size = static_cast(min_grid_size), + .block_size = static_cast(block_size)}; +} + +void kernel_ref::launch(cuda_dim3 grid_dim, + cuda_dim3 block_dim, + std::uint32_t shared_mem_bytes, + CUstream stream, + void** kernel_params) const +{ + RTCX_FUNC_RANGE(); + + RTCX_EXPECTS(grid_dim.is_valid(), "Grid dimensions must be greater than zero", std::logic_error); + RTCX_EXPECTS( + block_dim.is_valid(), "Block dimensions must be greater than zero", std::logic_error); + RTCX_EXPECTS( + kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); + + CUlaunchConfig cfg{.gridDimX = grid_dim.x, + .gridDimY = grid_dim.y, + .gridDimZ = grid_dim.z, + .blockDimX = block_dim.x, + .blockDimY = block_dim.y, + .blockDimZ = block_dim.z, + .sharedMemBytes = shared_mem_bytes, + .hStream = stream, + .attrs = nullptr, + .numAttrs = 0}; + + RTCX_CHECK_CUDA( + cu->LaunchKernelEx(&cfg, reinterpret_cast(handle_), kernel_params, nullptr)); +} + +void kernel_ref::launch_cooperative(cuda_dim3 grid_dim, + cuda_dim3 block_dim, + std::uint32_t shared_mem_bytes, + CUstream stream, + void** kernel_params) const +{ + RTCX_FUNC_RANGE(); + + RTCX_EXPECTS(grid_dim.is_valid(), "Grid dimensions must be greater than zero", std::logic_error); + RTCX_EXPECTS( + block_dim.is_valid(), "Block dimensions must be greater than zero", std::logic_error); + RTCX_EXPECTS( + kernel_params != nullptr, "Kernel parameters pointer must not be null", std::logic_error); + + RTCX_CHECK_CUDA(cu->LaunchCooperativeKernel(reinterpret_cast(handle_), + grid_dim.x, + grid_dim.y, + grid_dim.z, + block_dim.x, + block_dim.y, + block_dim.z, + shared_mem_bytes, + stream, + kernel_params)); +} + +library_t::~library_t() +{ + if (handle_ != nullptr) { cu->LibraryUnload(handle_); } +} + +library load_library(std::span binary) +{ + RTCX_FUNC_RANGE(); + + CUlibrary handle; + + RTCX_CHECK_CUDA( + cu->LibraryLoadData(&handle, binary.data(), nullptr, nullptr, 0, nullptr, nullptr, 0)); + + RTCX_DEFER([&] { + if (handle != nullptr) { RTCX_CHECK_CUDA(cu->LibraryUnload(handle)); } + }); + + auto library = std::make_shared(handle); + + handle = nullptr; + + return library; +} + +library load_library_from_file(char const* path) +{ + RTCX_FUNC_RANGE(); + + RTCX_EXPECTS(path != nullptr, "Library path must not be null", std::logic_error); + + CUlibrary handle; + + RTCX_CHECK_CUDA(cu->LibraryLoadFromFile(&handle, path, nullptr, nullptr, 0, nullptr, nullptr, 0)); + + RTCX_DEFER([&] { + if (handle != nullptr) { RTCX_CHECK_CUDA(cu->LibraryUnload(handle)); } + }); + + auto library = std::make_shared(handle); + + handle = nullptr; + + return library; +} + +byte_buffer link_library(link_params const& params) +{ + RTCX_FUNC_RANGE(); + + RTCX_EXPECTS(params.name != nullptr, "Link output name must not be null", std::logic_error); + RTCX_EXPECTS(params.output_type == binary_type::CUBIN || params.output_type == binary_type::PTX, + "Only CUBIN and PTX output types are supported for linking modules", + std::logic_error); + RTCX_EXPECTS(params.file_fragments.size() != 0 || params.memory_fragments.size() != 0, + "At least one fragment must be provided for linking", + std::logic_error); + + for (auto& frag : params.file_fragments) { + RTCX_EXPECTS(frag.path != nullptr, "Fragment file path must not be empty", std::logic_error); + } + + for (auto& frag : params.memory_fragments) { + RTCX_EXPECTS( + frag.data.size_bytes() > 0, "Fragment binary data must be non-empty", std::logic_error); + } + + nvJitLinkHandle handle = nullptr; + RTCX_CHECK_NVJITLINK(nvjitlink->Create(&handle, + static_cast(params.link_options.size()), + const_cast(params.link_options.data()))); + + RTCX_DEFER([&] { nvjitlink->Destroy(&handle); }); + + for (auto& frag : params.file_fragments) { + RTCX_CHECK_NVJITLINK(nvjitlink->AddFile(handle, to_nvjitlink_input_type(frag.type), frag.path)); + } + + for (auto& frag : params.memory_fragments) { + RTCX_CHECK_NVJITLINK(nvjitlink->AddData(handle, + to_nvjitlink_input_type(frag.type), + frag.data.data(), + frag.data.size_bytes(), + frag.name)); + } + + auto link_result = nvjitlink->Complete(handle); + log_nvJitLink_result(params, handle, link_result); + RTCX_CHECK_NVJITLINK(link_result); + + switch (params.output_type) { + case binary_type::CUBIN: { + std::size_t cubin_size; + RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedCubinSize(handle, &cubin_size)); + auto cubin = byte_buffer::make(cubin_size); + RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedCubin(handle, cubin.data())); + return cubin; + } break; + case binary_type::PTX: { + std::size_t ptx_size; + RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedPtxSize(handle, &ptx_size)); + auto ptx = byte_buffer::make(ptx_size); + RTCX_CHECK_NVJITLINK(nvjitlink->GetLinkedPtx(handle, reinterpret_cast(ptx.data()))); + return ptx; + } break; + default: + RTCX_FAIL(std::format("Unsupported output binary type for linking CUDA libraries: ({})", + binary_type_string(params.output_type)), + std::runtime_error); + } +} + +kernel_ref library_t::get_kernel(char const* name) const +{ + RTCX_FUNC_RANGE(); + + CUkernel kernel; + RTCX_CHECK_CUDA(cu->LibraryGetKernel(&kernel, handle_, name)); + return kernel_ref{kernel}; +} + +namespace { + +[[noreturn]] void throw_posix(std::string_view message, std::string_view syscall_name) +{ + auto errc = errno; + RTCX_FAIL( + std::format("{}. `{}` failed with {} ({})", message, syscall_name, errc, std::strerror(errc)), + std::runtime_error); +} + +} // namespace + +cache_t::cache_t(std::string cache_dir, + std::string tmp_dir, + cache_limits const& limits, + bool preload, + bool disable) + : enabled_{!disable}, + cache_dir_{std::move(cache_dir)}, + tmp_dir_{std::move(tmp_dir)}, + limits_{limits}, + lock_{}, + blobs_cache_{limits.num_mem_blobs}, + libraries_cache_{limits.num_mem_libraries}, + tick_{0} +{ + if (preload) { preload_from_disk(); } +} + +std::string const& cache_t::get_cache_dir() { return cache_dir_; } + +std::string const& cache_t::get_tmp_dir() { return tmp_dir_; } + +std::optional blob_t::from_file(char const* path) +{ + std::int32_t fd = ::open(path, O_RDONLY); + + if (fd == -1) { + if (errno == ENOENT) { + return std::nullopt; + } else { + throw_posix("Failed to open RTCX cache file from disk", "open"); + } + } + + RTCX_DEFER([&] { + if (::close(fd) == -1) { + throw_posix("Failed to close RTCX cache file after memory-mapping", "close"); + } + }); + + auto file_size = ::lseek(fd, 0, SEEK_END); + if (file_size == -1) { throw_posix("Failed to determine size of RTCX cache file", "lseek"); } + + if (file_size == 0) { + // mmap does not support mapping zero-length files, so we return an empty blob in this case + return blob_t::from_static_data({}); + } + + void* map = ::mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0); + + if (map == MAP_FAILED) { throw_posix("Failed to memory-map RTCX cache file", "mmap"); } + + auto deleter = +[](std::uint8_t const* buffer, std::size_t size) { + if (::munmap(static_cast(const_cast(buffer)), size) == -1) { + throw_posix("Failed to unmap RTCX cache file from memory", "munmap"); + } + }; + + return blob_t::from_parts(static_cast(map), file_size, deleter); +} + +namespace { + +/// @brief retrieves a blob from disk based on the given hash and object type (e.g. "blob", +/// "cuLibrary"). Returns nullopt if the file doesn't exist on disk, and throws if any other error +/// occurs. +std::optional get_disk_blob(std::string const& cache_dir, + object_type type, + hash128 const& hash) +{ + auto hex = hash.to_hex_string(); + auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(type)); + auto blob = blob_t::from_file(path.c_str()); + + if (!blob.has_value()) { return std::nullopt; } + return std::make_shared(std::move(*blob)); +} + +std::optional get_disk_library(std::string const& cache_dir, hash128 const& hash) +{ + RTCX_FUNC_RANGE(); + + auto hex = hash.to_hex_string(); + auto path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(object_type::LIBRARY)); + + // WAR: avoid a driver API call when the cache file is not present + // compute-sanitizer doesn't properly handle exceptions thrown from driver API calls, so we need + // to check for file existence first to avoid false positives in the sanitizer + if (!std::filesystem::exists(path)) { return std::nullopt; } + + CUlibrary handle; + auto errc = + cu->LibraryLoadFromFile(&handle, path.c_str(), nullptr, nullptr, 0, nullptr, nullptr, 0); + + if (errc == CUDA_ERROR_FILE_NOT_FOUND) { return std::nullopt; } + + RTCX_EXPECTS(errc == CUDA_SUCCESS, + std::format("Failed to load library `{}` from RTCX cache file", path), + std::runtime_error); + + return std::make_shared(handle); +} + +std::vector get_disk_entries(std::string const& cache_dir) +{ + std::vector entries; + for (auto& entry : std::filesystem::directory_iterator(cache_dir)) { + if (!entry.is_regular_file()) { continue; } + entries.push_back(entry.path().string()); + } + + return entries; +} + +/// @brief atomically writes a blob to disk by first writing to a temporary file and then renaming +/// it to the final path. +void cache_blob_to_disk(std::string const& cache_dir, + std::string const& tmp_dir, + object_type type, + hash128 const& hash, + std::span binary) +{ + RTCX_FUNC_RANGE(); + + auto tmp_path = std::format("{}/rtcx-bin-XXXXXX", tmp_dir); + (void)tmp_path.c_str(); // to ensure null-termination for mkstemp + + { + std::int32_t fd = ::mkstemp(tmp_path.data()); + if (fd == -1) { throw_posix("Failed to create temporary file for RTCX cache", "mkstemp"); } + + RTCX_DEFER([&] { + if (::close(fd) == -1) { throw_posix("Failed to close temporary RTCX cache file", "close"); } + }); + + { + auto ptr = binary.data(); + auto remaining = binary.size(); + while (remaining > 0) { + auto written = ::write(fd, ptr, remaining); + if (written == -1) { throw_posix("Failed to write RTCX cache to temporary file", "write"); } + ptr += static_cast(written); + remaining -= static_cast(written); + } + } + } + + auto hex = hash.to_hex_string(); + auto final_path = std::format("{}/{}.{}.bin", cache_dir, hex.view(), object_tag(type)); + + std::filesystem::create_directories(std::filesystem::path{final_path}.parent_path()); + + // rename is atomic, even if another process is performing the same operation + if (::rename(tmp_path.c_str(), final_path.c_str()) == -1) { + if (errno == EEXIST) { + // another process has already created the file, so just remove our temp file + if (::remove(tmp_path.c_str()) == -1) { + throw_posix("Failed to remove temporary RTCX cache file", "remove"); + } + return; + } else { + throw_posix( + std::format("Failed to move temporary RTCX cache file to final location ({})", final_path), + "rename"); + } + } +} + +} // namespace + +std::shared_future cache_t::get_or_add_blob(hash128 const& hash, blob_compile_func compile) +{ + RTCX_FUNC_RANGE(); + + std::atomic_ref tick{tick_}; + auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + + std::unique_lock lock{lock_}; + + // check memory cache + if (auto it = enabled_ ? blobs_cache_.entries_.find(hash) : blobs_cache_.entries_.end(); + it != blobs_cache_.entries_.end()) { + counter_.blob_mem_hits.incr(); + + // update LRU tick + it->second.hit(current_tick); + + return it->second.value; + + } else { + counter_.blob_mem_misses.incr(); + + // check disk cache + std::optional disk_blob = std::nullopt; + if (enabled_) { disk_blob = get_disk_blob(cache_dir_, object_type::BLOB, hash); } + + std::promise promise; + auto fut = promise.get_future().share(); + auto cache_fut = fut; + auto ret_fut = fut; + + if (disk_blob.has_value()) { + counter_.blob_disk_hits.incr(); + + promise.set_value(std::move(*disk_blob)); + + // insert into cache + blobs_cache_.insert(hash, std::move(cache_fut), current_tick); + + return ret_fut; + + } else { + counter_.blob_disk_misses.incr(); + + blobs_cache_.insert(hash, std::move(cache_fut), current_tick); + + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this hash + lock.unlock(); + + auto result = compile(); + promise.set_value(result); + + cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::BLOB, hash, result->view()); + + return ret_fut; + } + } +} + +std::shared_future cache_t::get_or_add_library(hash128 const& hash, + library_compile_func compile) +{ + RTCX_FUNC_RANGE(); + + std::atomic_ref tick{tick_}; + auto current_tick = tick.fetch_add(1, std::memory_order_relaxed); + + std::unique_lock lock{lock_}; + + // check memory cache + if (auto it = enabled_ ? libraries_cache_.entries_.find(hash) : libraries_cache_.entries_.end(); + it != libraries_cache_.entries_.end()) { + counter_.library_mem_hits.incr(); + + // update LRU tick + it->second.hit(current_tick); + + return it->second.value; + + } else { + counter_.library_mem_misses.incr(); + + // check disk cache + std::optional disk_library = std::nullopt; + if (enabled_) { disk_library = get_disk_library(cache_dir_, hash); } + + std::promise promise; + auto fut = promise.get_future().share(); + auto cache_fut = fut; + auto ret_fut = fut; + + if (disk_library.has_value()) { + counter_.library_disk_hits.incr(); + + libraries_cache_.insert(hash, std::move(cache_fut), current_tick); + + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this hash + lock.unlock(); + + promise.set_value(std::move(*disk_library)); + + return ret_fut; + + } else { + counter_.library_disk_misses.incr(); + + libraries_cache_.insert(hash, std::move(cache_fut), current_tick); + + // we can release the lock while calling the maker function since it may be expensive and we + // have already reserved a spot in the cache for this hash + lock.unlock(); + + auto [library, blob] = compile(); + promise.set_value(library); + + // store result to disk + cache_blob_to_disk(cache_dir_, tmp_dir_, object_type::LIBRARY, hash, blob->view()); + + return ret_fut; + } + } +} + +cache_stats cache_t::get_stats() +{ + return cache_stats{.blob_mem_hits = counter_.blob_mem_hits.get(), + .blob_mem_misses = counter_.blob_mem_misses.get(), + .blob_disk_hits = counter_.blob_disk_hits.get(), + .blob_disk_misses = counter_.blob_disk_misses.get(), + .library_mem_hits = counter_.library_mem_hits.get(), + .library_mem_misses = counter_.library_mem_misses.get(), + .library_disk_hits = counter_.library_disk_hits.get(), + .library_disk_misses = counter_.library_disk_misses.get()}; +} + +void cache_t::clear_stats() +{ + counter_.blob_mem_hits.reset(); + counter_.blob_mem_misses.reset(); + counter_.blob_disk_hits.reset(); + counter_.blob_disk_misses.reset(); + counter_.library_mem_hits.reset(); + counter_.library_mem_misses.reset(); + counter_.library_disk_hits.reset(); + counter_.library_disk_misses.reset(); +} + +cache_limits cache_t::get_limits() { return limits_; } + +std::size_t cache_t::get_blob_count() +{ + std::lock_guard guard{lock_}; + return blobs_cache_.entries_.size(); +} + +std::size_t cache_t::get_library_count() +{ + std::lock_guard guard{lock_}; + return libraries_cache_.entries_.size(); +} + +void cache_t::clear_memory_store() +{ + RTCX_FUNC_RANGE(); + + std::lock_guard guard{lock_}; + + blobs_cache_.entries_.clear(); + libraries_cache_.entries_.clear(); +} + +void cache_t::clear_disk_store() +{ + RTCX_FUNC_RANGE(); + + auto entries = get_disk_entries(cache_dir_); + + for (auto const& path : entries) { + try { + std::filesystem::remove(path); + } catch (...) { + log_error(std::format("Unknown error occurred while removing RTCX cache file `{}`", path)); + } + } +} + +void cache_t::preload_from_disk() +{ + RTCX_FUNC_RANGE(); + + auto entries = get_disk_entries(cache_dir_); + + auto load_count = + std::min(entries.size(), limits_.num_mem_blobs + limits_.num_mem_libraries); + + entries.resize(load_count); + + { + std::lock_guard guard{lock_}; + tick_++; + + for (auto const& path : entries) { + try { + auto file_name = std::filesystem::path{path}.filename().string(); + auto hash_str = file_name.substr(0, file_name.find('.')); + auto hash = hash128::parse(hash_str); + + if (path.ends_with(".blob.bin")) { + auto data = blob_t::from_file(path.c_str()); + if (!data.has_value()) { continue; } + auto blob = std::make_shared(std::move(*data)); + std::promise promise; + auto fut = promise.get_future().share(); + promise.set_value(std::move(blob)); + blobs_cache_.insert(hash, std::move(fut), tick_); + } else if (path.ends_with(".cuLibrary.bin")) { + auto lib = get_disk_library(cache_dir_, hash); + if (!lib.has_value()) { continue; } + std::promise promise; + auto fut = promise.get_future().share(); + promise.set_value(std::move(*lib)); + libraries_cache_.insert(hash, std::move(fut), tick_); + } + } catch (std::exception const& e) { + // ignore any errors during preload + log_error(e.what()); + } catch (...) { + log_error("Unknown error during preload"); + } + } + } +} + +void cache_t::enable(bool enable) +{ + std::lock_guard guard{lock_}; + enabled_ = enable; +} + +bool cache_t::is_enabled() +{ + std::lock_guard guard{lock_}; + return enabled_; +} + +std::string reflect_template(std::string_view template_name, + std::span template_args) +{ + return std::format("{}<{}>", template_name, join_strings(template_args, ", ")); +} + +std::string reflect_template(std::string_view template_name, + std::span template_args) +{ + return std::format("{}<{}>", template_name, join_strings(template_args, ", ")); +} + +rtcx::byte_buffer decompress_blob(std::span compressed_binary, + std::size_t uncompressed_size, + std::string_view compression) +{ + RTCX_FUNC_RANGE(); + + RTCX_EXPECTS(compression == "none" || compression == "zstd", + std::format("Unsupported compression type specified: {}", compression), + std::runtime_error); + auto decompressed = rtcx::byte_buffer::make(uncompressed_size); + + if (compression == "zstd") { + std::size_t errc = ::ZSTD_decompress( + decompressed.data(), uncompressed_size, compressed_binary.data(), compressed_binary.size()); + + RTCX_EXPECTS( + !::ZSTD_isError(errc) && errc == uncompressed_size, + std::format("Failed to decompress embedded RTC source files with ZSTD, error code {} : {}", + errc, + ::ZSTD_getErrorName(errc)), + std::runtime_error); + } else { + // compression is "none", so just copy the data + std::copy(compressed_binary.data(), + compressed_binary.data() + compressed_binary.size(), + decompressed.data()); + } + + return decompressed; +} + +} // namespace rtcx diff --git a/rtcx.hpp b/rtcx.hpp new file mode 100644 index 0000000..3a9929c --- /dev/null +++ b/rtcx.hpp @@ -0,0 +1,1007 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RTCX_DEFER__CONCATENATE_DETAIL(x, y) x##y +#define RTCX_DEFER__CONCATENATE(x, y) RTCX_DEFER__CONCATENATE_DETAIL(x, y) +#define RTCX_DEFER(...) ::rtcx::defer RTCX_DEFER__CONCATENATE(defer_, __COUNTER__)(__VA_ARGS__) + +extern "C" { +typedef struct CUlib_st* CUlibrary; // NOLINT(modernize-use-using) +typedef struct CUkern_st* CUkernel; // NOLINT(modernize-use-using) +typedef struct CUstream_st* CUstream; // NOLINT(modernize-use-using) +} + +namespace rtcx { + +inline constexpr std::size_t CACHELINE_ALIGNMENT = + 64; // = std::hardware_destructive_interference_size */ + +/** + * @brief RAII utility to execute a callable at the end of a scope. + */ +template +struct defer { + private: + T func_; + + public: + template + defer(Args&&... args) : func_{static_cast(args)...} + { + } + defer(defer const&) = delete; + defer& operator=(defer const&) = delete; + defer(defer&&) = delete; + defer& operator=(defer&&) = delete; + ~defer() { func_(); } +}; + +template +defer(T) -> defer; + +template +struct func; + +/** + * @brief Zero-copy, type-erased reference to a callable entity (e.g. lambda, function pointer) that + * can be invoked with the given signature. + */ +template +struct [[nodiscard]] func { + private: + void* _user_data; + R (*_thunk)(void*, Args...); + + public: + func(void* user_data, R (*thunk)(void*, Args...)) : _user_data{user_data}, _thunk{thunk} {} + + func(R (*func_ptr)(Args...)) + : _user_data{reinterpret_cast(func_ptr)}, + _thunk{+[](void* user_data, Args... args) -> R { + auto func = reinterpret_cast(user_data); + return func(std::forward(args)...); + }} + { + } + + R operator()(Args... args) const { return _thunk(_user_data, std::forward(args)...); } + + template + static func from_functor(Lambda& lambda) + { + return func{static_cast(std::addressof(lambda)), + +[](void* user_data, Args... args) -> R { + auto& lambda = *static_cast*>(user_data); + return lambda(std::forward(args)...); + }}; + } +}; + +template +func(void*, R (*)(void*, Args...)) -> func; + +template +func(R (*)(Args...)) -> func; + +struct [[nodiscard]] hash128_hasher { + constexpr std::uint64_t operator()(hash128 const& obj) const + { + // use only the lower 64 bits of the hash for the hash table + return static_cast(obj.value); + } +}; + +enum class binary_type : std::int8_t { LTO_IR = 0, CUBIN = 2, FATBIN = 3, PTX = 4 }; + +/** + * @brief A heap-allocated statically-sized buffer. Its contents are not guaranteed to be + * initialized. + */ +template + requires(!std::is_const_v && std::is_trivially_copyable_v && + std::is_trivially_destructible_v) +struct buffer { + private: + T* _data; + std::size_t _size; + + buffer(T* data, std::size_t size) : _data(data), _size(size) {} + + public: + /** + * @brief Creates a new buffer of the given size, with uninitialized contents. + */ + static buffer make(std::size_t size) + { + T* data = static_cast(malloc(size * sizeof(T))); + if (data == nullptr) { throw std::bad_alloc(); } + return buffer{data, size}; + } + + buffer() : buffer{nullptr, 0} {} //< Default constructor. Creates an empty buffer + + buffer(buffer const&) = delete; + + buffer& operator=(buffer const&) = delete; + + /** + * @brief Move constructor. Transfers ownership of the buffer from the source to the new object. + * After the move, the source buffer is left in an empty state (data pointer is null and size is + * zero). + */ + buffer(buffer&& other) noexcept : _data(other._data), _size(other._size) + { + other._data = nullptr; + other._size = 0; + } + + /** + * @brief Move assignment operator. Transfers ownership of the buffer from the source to the + * current object. + */ + buffer& operator=(buffer&& other) noexcept + { + if (this == &other) [[unlikely]] { return *this; } + this->~buffer(); + new (this) buffer(std::move(other)); + return *this; + } + + ~buffer() noexcept { free(_data); } + + /** + * @brief Returns a pointer to the buffer's data + * @return A pointer to the buffer's data + */ + [[nodiscard]] T* data() const { return _data; } + + /** + * @brief Returns the size of the buffer + * @return The size of the buffer in number of elements + */ + [[nodiscard]] std::size_t size() const { return _size; } + + /** + * @brief Returns an iterator to the beginning of the buffer + * @return An iterator to the beginning of the buffer + */ + [[nodiscard]] T* begin() { return _data; } + + /** + * @brief Returns an iterator to the end of the buffer + * @return An iterator to the end of the buffer + */ + [[nodiscard]] T* end() { return _data + _size; } + + /** + * @brief Returns a const iterator to the beginning of the buffer + * @return A const iterator to the beginning of the buffer + */ + [[nodiscard]] T const* begin() const { return _data; } + + /** + * @brief Returns a const iterator to the end of the buffer + * @return A const iterator to the end of the buffer + */ + [[nodiscard]] T const* end() const { return _data + _size; } + + /** + * @brief Returns a const iterator to the beginning of the buffer + * @return A const iterator to the beginning of the buffer + */ + [[nodiscard]] T const* cbegin() const { return _data; } + + /** + * @brief Returns a const iterator to the end of the buffer + * @return A const iterator to the end of the buffer + */ + [[nodiscard]] T const* cend() const { return _data + _size; } + + /** + * @brief Releases ownership of the buffer's data and returns a pointer to it. After calling this + * function, the buffer is left in an empty state. The caller is responsible for calling `free()` + * on the returned pointer when it is no longer needed. + * @return A pointer to the buffer's data + */ + [[nodiscard]] T* release() + { + T* data = _data; + _data = nullptr; + _size = 0; + return data; + } +}; + +using byte_buffer = buffer; + +/** + * @brief Represents an immutable blob view + * @details Manages the lifetime of the binary data via a user-provided deallocator function. This + * enables zero-copy view of binary data stored in various forms (e.g., std::vector, mmap'd file, + * etc.). + */ +struct [[nodiscard]] blob_t { + private: + using deallocator = func; + + static void noop_deallocator(std::uint8_t const*, std::size_t) {} + + std::uint8_t const* data_; + std::size_t size_; + deallocator deallocator_; + + blob_t(std::uint8_t const* data, std::size_t size, deallocator deallocator) + : data_(data), size_(size), deallocator_(deallocator) + { + } + + public: + blob_t() : data_(nullptr), size_(0), deallocator_(noop_deallocator) {} + + blob_t(blob_t const&) = delete; + blob_t& operator=(blob_t const&) = delete; + + blob_t(blob_t&& other) noexcept + : data_(other.data_), size_(other.size_), deallocator_(other.deallocator_) + { + other.data_ = nullptr; + other.size_ = 0; + other.deallocator_ = noop_deallocator; + } + + blob_t& operator=(blob_t&& other) noexcept + { + if (this == &other) [[unlikely]] { return *this; } + this->~blob_t(); + new (this) blob_t(std::move(other)); + return *this; + } + + ~blob_t() { deallocator_(data_, size_); } + + [[nodiscard]] std::span view() const { return {data_, size_}; } + + static blob_t from_parts(std::uint8_t const* data, std::size_t size, deallocator deallocator) + { + return blob_t{data, size, deallocator}; + } + + static blob_t from_buffer(byte_buffer buffer); + + static blob_t from_static_data(std::span data); + + static std::optional from_file(char const* path); +}; + +using blob = std::shared_ptr; + +/** + * @brief Represents the occupancy configuration for a kernel. This information can be used to + * optimize kernel launches for maximum performance on the GPU. + */ +struct [[nodiscard]] kernel_occupancy_config { + std::uint32_t min_grid_size = 0; //< Minimum grid size to achieve the maximum occupancy + std::uint32_t block_size = + 0; //< Number of threads per block to achieve the min_grid_size occupancy +}; + +/** + * @brief Represents the dimensions of a CUDA grid or block, with x, y, and z components. This + * struct is used to specify the configuration of kernel launches on the GPU. + */ +struct cuda_dim3 { + std::uint32_t x = 1; //< Value for the x dimension + std::uint32_t y = 1; //< Value for the y dimension + std::uint32_t z = 1; //< Value for the z dimension + + [[nodiscard]] constexpr bool is_valid() const { return x > 0 && y > 0 && z > 0; } +}; + +/** + * @brief Represents a compiled kernel that can be launched on the GPU. + */ +struct [[nodiscard]] kernel_ref { + private: + CUkernel handle_; + + public: + explicit kernel_ref(CUkernel handle) : handle_(handle) {} + + /** + * @brief Computes the maximum occupancy configuration for the kernel, given the specified dynamic + * shared memory usage and block size limit. This function queries the CUDA driver for the optimal + * block size and minimum grid size to achieve maximum occupancy of the kernel on the GPU. + */ + kernel_occupancy_config max_occupancy_config(std::size_t dynamic_shared_memory_bytes, + std::int32_t block_size_limit) const; + + /** + * @brief Launches the kernel on the GPU with the specified grid and block dimensions, + * dynamic shared memory size, stream, and kernel parameters. This function wraps the CUDA driver + * kernel launch API, providing a convenient interface for executing the kernel with the desired + * configuration. + * @param grid_dim The dimensions of the grid + * @param block_dim The dimensions of the block + * @param shared_mem_bytes The amount of dynamic shared memory (in bytes) to allocate for the + * kernel + * @param stream The CUDA stream on which to launch the kernel + * @param kernel_params A pointer to an array of pointers representing the kernel parameters to be + * passed to the kernel at launch time + */ + void launch(cuda_dim3 grid_dim, + cuda_dim3 block_dim, + std::uint32_t shared_mem_bytes, + CUstream stream, + void** kernel_params) const; + + /** + * @brief Launches the kernel on the GPU in cooperative mode with the specified grid and block + * dimensions, dynamic shared memory size, stream, and kernel parameters. This function wraps the + * CUDA driver cooperative kernel launch API, providing a convenient interface for executing the + * kernel with the desired configuration. + * @param grid_dim The dimensions of the grid + * @param block_dim The dimensions of the block + * @param shared_mem_bytes The amount of dynamic shared memory (in bytes) to allocate for the + * kernel + * @param stream The CUDA stream on which to launch the kernel + * @param kernel_params A pointer to an array of pointers representing the kernel parameters to be + * passed to the kernel at launch time + */ + void launch_cooperative(cuda_dim3 grid_dim, + cuda_dim3 block_dim, + std::uint32_t shared_mem_bytes, + CUstream stream, + void** kernel_params) const; + + /** + * @brief Retrieves the underlying CUDA kernel handle + * @return The CUDA kernel handle associated with this kernel reference + */ + [[nodiscard]] CUkernel get() const { return handle_; } +}; + +/** + * @brief Represents a loaded RTC library containing compiled kernels + */ +struct [[nodiscard]] library_t { + private: + CUlibrary handle_; + + public: + explicit library_t(CUlibrary handle) : handle_(handle) {} + library_t(library_t const&) = delete; + library_t(library_t&&) = delete; + library_t& operator=(library_t const&) = delete; + library_t& operator=(library_t&&) = delete; + ~library_t(); + + /** + * @brief Retrieves the underlying CUDA library handle + */ + [[nodiscard]] CUlibrary get() const { return handle_; } + + /** + * @brief Retrieve a kernel from the library by name + */ + [[nodiscard]] kernel_ref get_kernel(char const* name) const; +}; + +using library = std::shared_ptr; + +/** + * @brief Parameters for compiling source code into a binary blob using NVRTC + */ +struct [[nodiscard]] compile_params { + char const* name = nullptr; //< Debug name for the compilation unit + char const* source = nullptr; //< Source code to be compiled + std::span header_include_names = {}; //< Header file names + std::span headers = {}; //< Header file contents + std::span options = {}; //< NVRTC compilation options + std::span name_expressions = {}; //< Name expressions to be instantiated + binary_type target_type = binary_type::LTO_IR; //< Output binary type +}; + +/** + * @brief Represents a binary fragment in memory to be linked into a library + */ +struct memory_fragment { + std::span data = {}; //< Binary data for the fragment + binary_type type = binary_type::CUBIN; //< Binary type of the fragment data + char const* name = nullptr; //< Debug name for the fragment +}; + +/** + * @brief Represents a binary fragment to be linked into a library + */ +struct file_fragment { + char const* path = nullptr; //< Path to the binary fragment file + binary_type type = binary_type::CUBIN; //< Binary type of the fragment data +}; + +/** + * @brief Parameters for linking multiple compiled fragments into a single library + */ +struct [[nodiscard]] link_params { + char const* name = nullptr; //< Debug name for the linked library + binary_type output_type = binary_type::CUBIN; //< Output binary type + std::span file_fragments = {}; //< Binary data for each fragment + std::span memory_fragments = + {}; //< Memory-resident binary fragments to link + std::span link_options = {}; //< NVJITLink options +}; + +inline namespace detail { + +template +struct alignas(CACHELINE_ALIGNMENT) lru_memory_cache { + struct entry { + std::uint64_t last_touched_tick = 0; + T value; + + void hit(std::uint64_t tick) { last_touched_tick = tick; } + }; + + std::unordered_map entries_ = {}; + std::size_t limit_; + + explicit lru_memory_cache(std::size_t limit) : limit_{limit} + { + // reserve space to avoid rehashing + entries_.reserve(limit * 2); + } + + void purge() + { + if (entries_.empty()) { return; } + + auto num_to_purge = (entries_.size() + 1) / 2; + + std::vector> rankings; + rankings.reserve(entries_.size()); + + for (auto& [key, entry] : entries_) { + rankings.emplace_back(key, entry.last_touched_tick); + } + + std::sort( + rankings.begin(), rankings.end(), [](auto& a, auto& b) { return a.second < b.second; }); + + // purge least recently used half + rankings.resize(num_to_purge); + + for (auto [key, _] : rankings) { + entries_.erase(key); + } + } + + void insert(hash128 const& sha, T&& value, std::uint64_t tick) + { + if (limit_ == 0) { return; } + + if ((entries_.size() + 1) > limit_) { purge(); } + + entries_.emplace(sha, entry{tick, std::move(value)}); + } +}; + +struct cache_stats_counter { + struct entry { + std::uint64_t value_ = 0; + + void incr() + { + std::atomic_ref c{value_}; + c.fetch_add(1, std::memory_order_relaxed); + } + + [[nodiscard]] std::uint64_t get() const + { + std::atomic_ref c{value_}; + return c.load(std::memory_order_relaxed); + } + + void reset() + { + std::atomic_ref c{value_}; + c.store(0, std::memory_order_relaxed); + } + }; + + entry blob_mem_hits; + entry blob_mem_misses; + entry blob_disk_hits; + entry blob_disk_misses; + entry library_mem_hits; + entry library_mem_misses; + entry library_disk_hits; + entry library_disk_misses; +}; + +}; // namespace detail + +struct [[nodiscard]] cache_stats { + std::uint64_t blob_mem_hits = 0; + std::uint64_t blob_mem_misses = 0; + std::uint64_t blob_disk_hits = 0; + std::uint64_t blob_disk_misses = 0; + std::uint64_t library_mem_hits = 0; + std::uint64_t library_mem_misses = 0; + std::uint64_t library_disk_hits = 0; + std::uint64_t library_disk_misses = 0; +}; + +struct [[nodiscard]] cache_limits { + std::uint32_t num_mem_blobs = 16'384; + std::uint32_t num_mem_libraries = 16'384; +}; + +using blob_compile_func = func; +using library_compile_func = func()>; + +/** + * @brief Thread-safe user-managed compile cache for compiled blobs and libraries + * + * @details Provides in-memory and on-disk caching of compiled RTC artifacts. + * The cache uses an LRU eviction policy when the number of cached items exceeds user-defined + * limits. In-memory cache is implemented using a thread-safe LRU cache that supports concurrent + * reads. The on-disk cache also allows concurrent access and stores cached items in files within a + * specified directory. Writing to disk is atomic to prevent corruption from concurrent writes or + * process interruptions. In addition, the cache maintains statistics on cache hits and misses for + * both in-memory and on-disk caches to help monitor cache performance in benchmarking and + * debugging. The interface is zero-copy, using shared pointers, mmap, and spans to avoid + * unnecessary data copying across threads and disk. + */ +struct cache_t { // NOLINT + private: + bool enabled_; + + std::string cache_dir_; + + std::string tmp_dir_; + + cache_limits limits_; + + std::mutex lock_; + + detail::lru_memory_cache> blobs_cache_; + + detail::lru_memory_cache> libraries_cache_; + + detail::cache_stats_counter counter_; + + alignas(CACHELINE_ALIGNMENT) std::uint64_t tick_; // NOLINT(modernize-use-default-member-init) + + public: + /** + * @brief Construct a new cache_t object with the specified cache directory, limits, and options + * for preloading and enabling the cache. + * @param cache_dir The directory path to be used for on-disk caching of compiled blobs and + * libraries (this directory must exist and be writable by the process) + * @param tmp_dir The directory path to be used for temporary files during atomic writes to the + * on-disk cache (this directory must exist, writable by the process, and be on the same + * filesystem as cache_dir to ensure atomic renames work correctly) + * @param limits A cache_limits struct specifying the maximum number of blobs and libraries to + * store in the cache before eviction occurs + * @param preload A boolean flag indicating whether to preload the cache from disk during + * initialization, allowing for faster retrieval of previously compiled kernels at runtime + * @param disable A boolean flag indicating whether to disable the cache entirely, preventing any + * caching of compiled blobs and libraries in memory + */ + cache_t(std::string cache_dir, + std::string tmp_dir, + cache_limits const& limits, + bool preload, + bool disable); + cache_t(cache_t const&) = delete; + cache_t& operator=(cache_t const&) = delete; + cache_t(cache_t&&) = delete; + cache_t& operator=(cache_t&&) = delete; + ~cache_t() = default; + + /** + * @brief Get the directory path used for on-disk caching + * @return String reference to the cache directory path + */ + [[nodiscard]] std::string const& get_cache_dir(); + + /** + * @brief Get the directory path used for temporary files during atomic writes to the on-disk + * cache + * @return String reference to the temporary directory path + */ + [[nodiscard]] std::string const& get_tmp_dir(); + + /** + * @brief Query the cache for a compiled blob by its hash, or insert it if not present + * @param hash hash of the blob to query or insert + * @param compile Function to compile the blob if it's not found in the cache + * @return A shared future that will hold the compiled blob once it's available + */ + [[nodiscard]] std::shared_future get_or_add_blob(hash128 const& hash, + blob_compile_func compile); + + /** + * @brief Query the cache for a compiled library by its hash and binary type, or insert + * it if not present + * @param hash hash of the library to query or insert + * @param compile Function to compile the library if it's not found in the cache + * @return A shared future that will hold the compiled library once it's available + */ + [[nodiscard]] std::shared_future get_or_add_library(hash128 const& hash, + library_compile_func compile); + + /** + * @brief Retrieve current cache performance statistics, including hits and misses for both + * in-memory and on-disk caches + * + * @return A cache_statistics struct containing the current cache performance metrics + */ + cache_stats get_stats(); + + /** + * @brief Clear the current cache performance statistics, resetting all hit and miss counters to + * zero + */ + void clear_stats(); + + /** + * @brief Retrieve the current cache limits for blobs and libraries + * + * @return A cache_limits struct containing the maximum number of blobs and libraries that can be + * stored in the cache before eviction occurs + */ + cache_limits get_limits(); + + /** + * @brief Get the current number of blobs stored in the in-memory cache + * + * @return The number of blobs currently stored in the in-memory cache + */ + [[nodiscard]] std::size_t get_blob_count(); + + /** + * @brief Get the current number of libraries stored in the in-memory cache + * + * @return The number of libraries currently stored in the in-memory cache + */ + [[nodiscard]] std::size_t get_library_count(); + + /** + * @brief Clear all entries from the in-memory cache, removing all cached blobs and libraries + * without affecting the on-disk cache + * + * @details This function is useful for freeing up memory without losing the benefits of the + * on-disk cache, which can still be used to retrieve cached items in the future. + */ + void clear_memory_store(); + + /** + * @brief Clear all entries from the on-disk cache, removing all cached blobs and libraries + * stored on disk without affecting the in-memory cache + * @details This function is useful for freeing up disk space or resetting the on-disk cache + * without losing the benefits of the in-memory cache, which can still be used to retrieve cached + * items in the future. + */ + void clear_disk_store(); + + /*** + * @brief Pre-load the JIT program cache from disk into memory during initialization, allowing for + * faster retrieval and execution of previously compiled kernels at runtime. + */ + void preload_from_disk(); + + /*** + * @brief Enable the cache, allowing it to store and retrieve compiled blobs and libraries in + * memory. + * @param enabled A boolean flag indicating whether to enable (true) or disable (false) the cache. + */ + void enable(bool enabled); + + /** + * @brief Get whether the cache is currently enabled or disabled. + * @return A boolean value indicating whether the cache is currently enabled (true) or disabled + */ + [[nodiscard]] bool is_enabled(); +}; + +/** + * @brief Get the version of the NVRTC library + * @return An integer representing the NVRTC version. With the encoding major * 1000 + minor * 10 + + * patch + */ +[[nodiscard]] std::int32_t nvrtc_version(); + +/** + * @brief Get the version of the NVJITLINK library + * @return An integer representing the NVJITLINK version. With the encoding major * 1000 + minor * + * 10 + patch + */ +[[nodiscard]] std::int32_t nvjitlink_version(); + +/** + * @brief Compile source code into a binary blob + * + * @param params Compilation parameters including source code, headers, options, and target binary + * type + * @return A buffer of bytes containing the compiled binary blob + */ +[[nodiscard]] byte_buffer compile(compile_params const& params); + +/** + * @brief Load a compiled library from binary data + * + * @param binary Span of bytes containing the compiled library binary data + * @return A library object representing the loaded library with launchable kernels + */ +[[nodiscard]] library load_library(std::span binary); + +/** + * @brief Load a compiled library from binary data + * + * @param path Path to the file containing the library binary data + * @return A library object representing the loaded library with launchable kernels + */ +[[nodiscard]] library load_library_from_file(char const* path); + +/** + * @brief Link multiple compiled binary fragments into a single binary blob containing the linked + * library + * + * @param params Linking parameters including the binary fragments to be linked and the target + * binary type + * @return A buffer of bytes containing the linked library binary + */ +[[nodiscard]] byte_buffer link_library(link_params const& params); + +/** + * @brief Initialize the RTCX library, setting up necessary resources and state for subsequent + * operations + * @details This function must be called before using any other functions in the RTCX library. It + * performs necessary initialization tasks such as setting up CUDA contexts, initializing caches, + * and preparing any global state required for compilation, linking, and kernel management + * operations. Failure to call this function before using other RTCX functions may result in + * undefined behavior or runtime errors. + * This function is thread-safe. + */ +void initialize(); + +/** + * @brief Teardown the RTCX library, releasing any resources and cleaning up state used by the + * library + * @details This function should be called when RTCX functionality is no longer needed, such as at + * the end of the program or when cleaning up resources. It performs necessary cleanup tasks such as + * releasing CUDA contexts, clearing caches, and resetting any global state used by the library. + * After calling this function, other RTCX functions should not be used unless initialize() is + * called again to reinitialize the library. + * This function is not thread-safe. + */ +void teardown(); + +/** + * @brief Reflect a value of any type into its CUDA string representation + * @tparam T The type of the value to be reflected + * @param value The value to be reflected + * @return A string containing the CUDA representation of the value + * @details This is a template function that can be specialized for different types to provide + * appropriate CUDA string representations. + */ +template +std::string reflect(T value) = delete; + +/** + * @brief Reflect a boolean value into its CUDA string representation ("true" or "false") + * @param value The boolean value to be reflected + * @return A string containing the CUDA representation of the boolean value ("true" or "false") + */ +template <> +inline std::string reflect(bool value) +{ + return std::format("{}", value); +} + +/** + * @brief Reflect an integer value into its CUDA string representation + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value + */ +template <> +inline std::string reflect(std::uint8_t value) +{ + return std::format("{}U", value); +} + +/** + * @brief Reflect an integer value into its CUDA string representation + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value + */ +template <> +inline std::string reflect(std::uint16_t value) +{ + return std::format("{}U", value); +} + +/** + * @brief Reflect an integer value into its CUDA string representation + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value + */ +template <> +inline std::string reflect(std::uint32_t value) +{ + return std::format("{}U", value); +} + +/** + * @brief Reflect an integer value into its CUDA string representation + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value + */ +template <> +inline std::string reflect(std::uint64_t value) +{ + return std::format("{}ULL", value); +} + +/** + * @brief Reflect an integer value into its CUDA string representation + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value + */ +template <> +inline std::string reflect(std::int8_t value) +{ + return std::format("{}", value); +} + +/** + * @brief Reflect an integer value into its CUDA string representation + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value + */ +template <> +inline std::string reflect(std::int16_t value) +{ + return std::format("{}", value); +} + +/** + * @brief Reflect an integer value into its CUDA string representation + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value + */ +template <> +inline std::string reflect(std::int32_t value) +{ + return std::format("{}", value); +} + +/** + * @brief Reflect an integer value into its CUDA string representation + * @param value The integer value to be reflected + * @return A string containing the CUDA representation of the integer value + */ +template <> +inline std::string reflect(std::int64_t value) +{ + return std::format("{}LL", value); +} + +/** + * @brief Reflect a floating-point value into its CUDA string representation + * @param value The floating-point value to be reflected + * @return A string containing the CUDA representation of the floating-point value + */ +template <> +inline std::string reflect(float value) +{ + return std::format("{}F", value); +} + +/** + * @brief Reflect a floating-point value into its CUDA string representation + * @param value The floating-point value to be reflected + * @return A string containing the CUDA representation of the floating-point value + */ +template <> +inline std::string reflect(double value) +{ + return std::format("{}", value); +} + +/** + * @brief Reflect an enumeration value into its CUDA string representation, given the type name as a + * string + * @tparam T An enumeration type + * @param type The name of the enumeration type to be reflected (e.g., "MyEnum") + * @param value The enumeration value to be reflected, which will be cast to its underlying integer + * type and represented as a string in the resulting CUDA code + * @return A string containing the CUDA representation of the enumeration value with the specified + * type + */ +template + requires(std::is_enum_v) +std::string reflect_enum(std::string_view type, T value) +{ + return std::format("{}{}{}{}", type, "{", static_cast>(value), "}"); +} + +/** + * @brief Reflect a template instantiation into its CUDA string representation, given the template + * name and its template arguments as strings + * @param template_name The name of the template to be reflected (e.g., "MyTemplate") + * @param template_args A span of strings representing the template arguments to be reflected, which + * will be used in the resulting CUDA code + * @return A string containing the CUDA representation of the template instantiation with the + * specified template name and arguments + */ +std::string reflect_template(std::string_view template_name, + std::span template_args); + +/** + * @brief Reflect a template instantiation into its CUDA string representation, given the template + * name and its template arguments as strings + * @param template_name The name of the template to be reflected (e.g., "MyTemplate") + * @param template_args A span of strings representing the template arguments to be reflected, which + * will be used in the resulting CUDA code + * @return A string containing the CUDA representation of the template instantiation with the + * specified template name and arguments + */ +std::string reflect_template(std::string_view template_name, + std::span template_args); + +/** + * @brief Reflect a template instantiation into its CUDA string representation, given the template + * name and its template arguments as strings + * @param template_name The name of the template to be reflected (e.g., "MyTemplate") + * @param template_args A span of strings representing the template arguments to be reflected, which + * will be used in the resulting CUDA code + * @return A string containing the CUDA representation of the template instantiation with the + * specified template name and arguments + */ +template + requires((true && ... && std::is_constructible_v)) +std::string reflect_template(std::string_view template_name, TemplateArgs&&... template_args) +{ + std::string_view const tparams[sizeof...(TemplateArgs)] = // NOLINT(modernize-avoid-c-arrays) + {std::string_view{template_args}...}; + return reflect_template(template_name, tparams); +} + +/** + * @brief Decompress a compressed binary blob using the specified compression algorithm + * @param compressed_binary A span of bytes containing the compressed binary data to be decompressed + * @param uncompressed_size The expected size of the uncompressed binary data in bytes + * @param compression A string view specifying the compression algorithm used to compress the binary + * data + * @return A byte buffer containing the decompressed binary data + * @throws std::runtime_error if decompression fails due to an unsupported compression algorithm, + * invalid compressed data, or if the decompressed data size does not match the expected + * uncompressed size + */ +rtcx::byte_buffer decompress_blob(std::span compressed_binary, + std::size_t uncompressed_size, + std::string_view compression); + +} // namespace rtcx