diff --git a/cpp/librtcx/CMakeLists.txt b/cpp/librtcx/CMakeLists.txt index 354d36150fce..47662051f7f1 100644 --- a/cpp/librtcx/CMakeLists.txt +++ b/cpp/librtcx/CMakeLists.txt @@ -11,6 +11,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) include(cmake/rapids_config.cmake) include(rapids-cmake) include(rapids-cpm) + include(rapids-export) + include(rapids-find) rapids_cpm_init() endif() @@ -23,7 +25,11 @@ project( option(RTCX_STATIC_LINK_NVRTC "Use static linking for NVRTC" OFF) option(RTCX_STATIC_LINK_NVJITLINK "Use static linking for nvJitLink" OFF) -find_package(CUDAToolkit REQUIRED) +rapids_find_package( + CUDAToolkit REQUIRED + BUILD_EXPORT_SET rtcx-exports + INSTALL_EXPORT_SET rtcx-exports +) if(NOT TARGET zstd) set(CPM_DOWNLOAD_zstd ON) @@ -87,3 +93,74 @@ if(RTCX_STATIC_LINK_NVJITLINK) else() target_compile_definitions(rtcx PRIVATE RTCX_STATIC_LINK_LIBNVJITLINK=0) endif() + +# ============================================================================= +# Install / Export +# ============================================================================= +include(GNUInstallDirs) +include(${rapids-cmake-dir}/cmake/install_lib_dir.cmake) +rapids_cmake_install_lib_dir(lib_dir) + +# Option to control install (default ON when built standalone, OFF when used as subdirectory) +option(RTCX_INSTALL "Enable installation of rtcx targets" ${PROJECT_IS_TOP_LEVEL}) + +set(rtcx_install_code_string + [=[ +# Embed functions (add_embed, embed_includes, embed_blob, embed) +# are included automatically so consumers don't need explicit include(). +include("${CMAKE_CURRENT_LIST_DIR}/embed.cmake") + +# Set rtcx_LIBCXX_DIR for consumers using embed_includes with libcxx headers. +set(rtcx_LIBCXX_DIR "${PACKAGE_PREFIX_DIR}/share/rtcx/libcxx") +]=] +) + +string( + CONFIGURE + [=[ +# Embed functions (add_embed, embed_includes, embed_blob, embed) +# are included automatically so consumers don't need explicit include(). +include("@CMAKE_CURRENT_SOURCE_DIR@/embed.cmake") + +# Set rtcx_LIBCXX_DIR for consumers using embed_includes with libcxx headers. +set(rtcx_LIBCXX_DIR "@CMAKE_CURRENT_SOURCE_DIR@/libcxx") +]=] + rtcx_build_code_string + @ONLY +) + +if(NOT RTCX_INSTALL) + set(rtcx_exclude_from_install EXCLUDE_FROM_ALL) +endif() + +install( + TARGETS rtcx + EXPORT rtcx-exports + ARCHIVE DESTINATION ${lib_dir} + INCLUDES + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ${rtcx_exclude_from_install} +) + +if(RTCX_INSTALL) + install(FILES rtcx.hpp sha256.hpp embed.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rtcx) + install(FILES embed.cmake embed.in.cpp DESTINATION ${lib_dir}/cmake/rtcx) + install(DIRECTORY libcxx/ DESTINATION ${CMAKE_INSTALL_DATADIR}/rtcx/libcxx) + + rapids_export( + INSTALL rtcx + EXPORT_SET rtcx-exports + GLOBAL_TARGETS rtcx + NAMESPACE rtcx:: + FINAL_CODE_BLOCK rtcx_install_code_string + ) +endif() + +# Build-tree export (always, so CPM consumers can find_package from the build tree) +rapids_export( + BUILD rtcx + EXPORT_SET rtcx-exports + GLOBAL_TARGETS rtcx + NAMESPACE rtcx:: + FINAL_CODE_BLOCK rtcx_build_code_string +) diff --git a/cpp/librtcx/RAPIDS_BRANCH b/cpp/librtcx/RAPIDS_BRANCH new file mode 120000 index 000000000000..15b9ac7c238b --- /dev/null +++ b/cpp/librtcx/RAPIDS_BRANCH @@ -0,0 +1 @@ +../../RAPIDS_BRANCH \ No newline at end of file diff --git a/cpp/librtcx/VERSION b/cpp/librtcx/VERSION new file mode 120000 index 000000000000..558194c5a5a5 --- /dev/null +++ b/cpp/librtcx/VERSION @@ -0,0 +1 @@ +../../VERSION \ No newline at end of file diff --git a/cpp/librtcx/cmake/rapids_config.cmake b/cpp/librtcx/cmake/rapids_config.cmake index 60d6971cca33..fe59436c012d 100644 --- a/cpp/librtcx/cmake/rapids_config.cmake +++ b/cpp/librtcx/cmake/rapids_config.cmake @@ -4,7 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= -file(READ "${CMAKE_CURRENT_LIST_DIR}/../../../VERSION" _rapids_version) +file(READ "${CMAKE_CURRENT_LIST_DIR}/../VERSION" _rapids_version) if(_rapids_version MATCHES [[^([0-9][0-9])\.([0-9][0-9])\.([0-9][0-9])]]) set(RAPIDS_VERSION_MAJOR "${CMAKE_MATCH_1}") set(RAPIDS_VERSION_MINOR "${CMAKE_MATCH_2}") @@ -19,11 +19,11 @@ else() ) endif() -file(STRINGS "${CMAKE_CURRENT_LIST_DIR}/../../../RAPIDS_BRANCH" RAPIDS_BRANCH) +file(STRINGS "${CMAKE_CURRENT_LIST_DIR}/../RAPIDS_BRANCH" RAPIDS_BRANCH) if(NOT RAPIDS_BRANCH) message( FATAL_ERROR - "Could not determine branch name to use for checking out rapids-cmake. The file \"${CMAKE_CURRENT_LIST_DIR}/../../../RAPIDS_BRANCH\" is missing." + "Could not determine branch name to use for checking out rapids-cmake. The file \"${CMAKE_CURRENT_LIST_DIR}/../RAPIDS_BRANCH\" is missing." ) endif() diff --git a/cpp/librtcx/embed.cmake b/cpp/librtcx/embed.cmake index 5713f2d72b24..12cb32a26bde 100644 --- a/cpp/librtcx/embed.cmake +++ b/cpp/librtcx/embed.cmake @@ -5,12 +5,6 @@ # cmake-format: on # ============================================================================= -if(NOT TARGET zstd) - message( - FATAL_ERROR "zstd library is required for JIT embedding. Please ensure it is found by CMake." - ) -endif() - # This function initializes a target for JIT embedding. It must be called before any calls to # embed_includes() or embed_blob() for the target. It creates a dedicated INTERFACE library target # that is used to track registered files and dependencies via target properties. The TARGET argument @@ -274,8 +268,14 @@ function(embed TARGET) set(RUNNER "${TARGET}__jit_embed_run") add_executable(${RUNNER} EXCLUDE_FROM_ALL "${EMBED_SCRIPT}") - target_include_directories(${RUNNER} PRIVATE ${ZSTD_INCLUDE_DIR}) - target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS} zstd) + target_link_libraries(${RUNNER} PRIVATE ${CMAKE_DL_LIBS}) + if(NOT ARG_COMPRESSION STREQUAL "none") + if(NOT TARGET zstd) + message(FATAL_ERROR "embed(): zstd target is required when COMPRESSION is not none.") + endif() + target_include_directories(${RUNNER} PRIVATE ${ZSTD_INCLUDE_DIR}) + target_link_libraries(${RUNNER} PRIVATE zstd) + endif() set_target_properties(${RUNNER} PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED YES) target_include_directories(${RUNNER} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR}) diff --git a/python/cudf_streaming/cudf_streaming/examples/ray_shuffle_example.py b/python/cudf_streaming/cudf_streaming/examples/ray_shuffle_example.py index 6ba7f5b1dbc2..fcac7b9f8311 100644 --- a/python/cudf_streaming/cudf_streaming/examples/ray_shuffle_example.py +++ b/python/cudf_streaming/cudf_streaming/examples/ray_shuffle_example.py @@ -16,11 +16,11 @@ partition_and_pack, unpack_and_concat, ) +from cudf_streaming.testing import assert_eq from rapidsmpf.integrations.ray import RapidsMPFActor, setup_ray_ucxx_cluster from rapidsmpf.memory.buffer_resource import BufferResource from rapidsmpf.memory.spill import unspill_partitions from rapidsmpf.shuffler import Shuffler -from rapidsmpf.testing import assert_eq class ShufflingActor(RapidsMPFActor): diff --git a/python/cudf_streaming/cudf_streaming/testing.py b/python/cudf_streaming/cudf_streaming/testing.py new file mode 100644 index 000000000000..427d79298473 --- /dev/null +++ b/python/cudf_streaming/cudf_streaming/testing.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +"""Testing utilities for cudf_streaming.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pylibcudf + +from rmm.pylibrmm.stream import DEFAULT_STREAM + +if TYPE_CHECKING: + from rmm.pylibrmm.stream import Stream + + +def assert_eq( + left: pylibcudf.Table, + right: pylibcudf.Table, + *, + sort_rows: int | None = None, + stream: Stream | None = None, +) -> None: + """ + Assert that two tables are equivalent using pylibcudf. + + Parameters + ---------- + left + plc.Table to compare. + right + plc.Table to compare. + sort_rows + If not None, sort both tables by this column before comparing. + An ``int`` is treated as a column index. + stream + CUDA stream to use for the comparison. + + Raises + ------ + AssertionError + If the two tables do not compare equal. + """ + if stream is None: + stream = DEFAULT_STREAM + + if sort_rows is not None: + column_order = [pylibcudf.types.Order.ASCENDING] + null_precedence = [pylibcudf.types.NullOrder.BEFORE] + left = pylibcudf.sorting.stable_sort_by_key( + left, + pylibcudf.Table([left.columns()[sort_rows]]), + column_order, + null_precedence, + stream=stream, + ) + right = pylibcudf.sorting.stable_sort_by_key( + right, + pylibcudf.Table([right.columns()[sort_rows]]), + column_order, + null_precedence, + stream=stream, + ) + if not pylibcudf.table_equality.tables_equal(left, right, stream=stream): + raise AssertionError(f"Table are not equal with {sort_rows=}") diff --git a/python/cudf_streaming/cudf_streaming/tests/test_allgather.py b/python/cudf_streaming/cudf_streaming/tests/test_allgather.py index 010121e5d609..f53298ec5983 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_allgather.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_allgather.py @@ -16,9 +16,9 @@ packed_data_from_cudf_packed_columns, unpack_and_concat, ) +from cudf_streaming.testing import assert_eq from rapidsmpf.coll import AllGather from rapidsmpf.memory.buffer_resource import BufferResource -from rapidsmpf.testing import assert_eq if TYPE_CHECKING: import rmm.mr diff --git a/python/cudf_streaming/cudf_streaming/tests/test_bloom_filter.py b/python/cudf_streaming/cudf_streaming/tests/test_bloom_filter.py index cdf1d36a3a51..538e33e3867c 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_bloom_filter.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_bloom_filter.py @@ -13,13 +13,13 @@ from cudf_streaming.streaming import ChannelMetadata from cudf_streaming.streaming.bloom_filter import BloomFilter from cudf_streaming.streaming.table_chunk import TableChunk +from cudf_streaming.testing import assert_eq from rapidsmpf.streaming.core.actor import define_actor, run_actor_network from rapidsmpf.streaming.core.leaf_actor import ( pull_from_channel, push_to_channel, ) from rapidsmpf.streaming.core.message import Message -from rapidsmpf.testing import assert_eq if TYPE_CHECKING: from collections.abc import Awaitable diff --git a/python/cudf_streaming/cudf_streaming/tests/test_integration_partition.py b/python/cudf_streaming/cudf_streaming/tests/test_integration_partition.py index ba779970b945..ed932d7e1c3e 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_integration_partition.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_integration_partition.py @@ -13,9 +13,9 @@ split_and_pack, unpack_and_concat, ) +from cudf_streaming.testing import assert_eq from rapidsmpf.memory.buffer_resource import BufferResource from rapidsmpf.memory.spill import spill_partitions, unspill_partitions -from rapidsmpf.testing import assert_eq from rmm.pylibrmm.stream import DEFAULT_STREAM if TYPE_CHECKING: diff --git a/python/cudf_streaming/cudf_streaming/tests/test_partition.py b/python/cudf_streaming/cudf_streaming/tests/test_partition.py index f0948a7666c6..165e6e435ef8 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_partition.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_partition.py @@ -13,13 +13,13 @@ unpack_and_concat, ) from cudf_streaming.streaming.table_chunk import TableChunk +from cudf_streaming.testing import assert_eq from rapidsmpf.streaming.core.actor import run_actor_network from rapidsmpf.streaming.core.leaf_actor import ( pull_from_channel, push_to_channel, ) from rapidsmpf.streaming.core.message import Message -from rapidsmpf.testing import assert_eq if TYPE_CHECKING: from rapidsmpf.streaming.chunks.partition import PartitionMapChunk diff --git a/python/cudf_streaming/cudf_streaming/tests/test_shuffler.py b/python/cudf_streaming/cudf_streaming/tests/test_shuffler.py index 7e7c4037fce3..4deb5bdfc9fe 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_shuffler.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_shuffler.py @@ -13,12 +13,12 @@ partition_and_pack, unpack_and_concat, ) +from cudf_streaming.testing import assert_eq from rapidsmpf.memory.buffer_resource import BufferResource from rapidsmpf.memory.spill import unspill_partitions from rapidsmpf.shuffler import ( Shuffler, ) -from rapidsmpf.testing import assert_eq if TYPE_CHECKING: import rmm.mr diff --git a/python/cudf_streaming/cudf_streaming/tests/test_sparse_alltoall.py b/python/cudf_streaming/cudf_streaming/tests/test_sparse_alltoall.py index 7853a13b025f..abe0f8fd6c15 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_sparse_alltoall.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_sparse_alltoall.py @@ -14,9 +14,9 @@ packed_data_from_cudf_packed_columns, unpack_and_concat, ) +from cudf_streaming.testing import assert_eq from rapidsmpf.coll.sparse_alltoall import SparseAlltoall from rapidsmpf.memory.buffer_resource import BufferResource -from rapidsmpf.testing import assert_eq if TYPE_CHECKING: import rmm.mr diff --git a/python/cudf_streaming/cudf_streaming/tests/test_streaming_allgather.py b/python/cudf_streaming/cudf_streaming/tests/test_streaming_allgather.py index 5206bf0beb1c..71ca535d05b9 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_streaming_allgather.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_streaming_allgather.py @@ -15,6 +15,7 @@ unpack_and_concat, ) from cudf_streaming.streaming.table_chunk import TableChunk +from cudf_streaming.testing import assert_eq from rapidsmpf.streaming.chunks.packed_data import PackedDataChunk from rapidsmpf.streaming.coll.allgather import AllGather, allgather from rapidsmpf.streaming.core.actor import define_actor, run_actor_network @@ -23,7 +24,6 @@ push_to_channel, ) from rapidsmpf.streaming.core.message import Message -from rapidsmpf.testing import assert_eq if TYPE_CHECKING: from collections.abc import Awaitable diff --git a/python/cudf_streaming/cudf_streaming/tests/test_streaming_define_actor.py b/python/cudf_streaming/cudf_streaming/tests/test_streaming_define_actor.py index fca157f09c0a..596fc624309e 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_streaming_define_actor.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_streaming_define_actor.py @@ -9,6 +9,7 @@ import pytest from cudf_streaming.streaming.table_chunk import TableChunk +from cudf_streaming.testing import assert_eq from rapidsmpf.streaming.chunks.arbitrary import ArbitraryChunk from rapidsmpf.streaming.core.actor import define_actor, run_actor_network from rapidsmpf.streaming.core.leaf_actor import ( @@ -16,7 +17,6 @@ push_to_channel, ) from rapidsmpf.streaming.core.message import Message -from rapidsmpf.testing import assert_eq @pytest.fixture diff --git a/python/cudf_streaming/cudf_streaming/tests/test_streaming_fanout.py b/python/cudf_streaming/cudf_streaming/tests/test_streaming_fanout.py index 1ac22589ca1e..58dbdf32750a 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_streaming_fanout.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_streaming_fanout.py @@ -11,6 +11,7 @@ import pytest from cudf_streaming.streaming.table_chunk import TableChunk +from cudf_streaming.testing import assert_eq from rapidsmpf.streaming.core.actor import run_actor_network from rapidsmpf.streaming.core.fanout import FanoutPolicy, fanout from rapidsmpf.streaming.core.leaf_actor import ( @@ -18,7 +19,6 @@ push_to_channel, ) from rapidsmpf.streaming.core.message import Message -from rapidsmpf.testing import assert_eq _INT64 = plc.DataType(plc.TypeId.INT64) diff --git a/python/cudf_streaming/cudf_streaming/tests/test_streaming_leaf_actor.py b/python/cudf_streaming/cudf_streaming/tests/test_streaming_leaf_actor.py index d60b252c4cf0..00a5be995104 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_streaming_leaf_actor.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_streaming_leaf_actor.py @@ -8,13 +8,13 @@ import pylibcudf as plc from cudf_streaming.streaming.table_chunk import TableChunk +from cudf_streaming.testing import assert_eq from rapidsmpf.streaming.core.actor import run_actor_network from rapidsmpf.streaming.core.leaf_actor import ( pull_from_channel, push_to_channel, ) from rapidsmpf.streaming.core.message import Message -from rapidsmpf.testing import assert_eq if TYPE_CHECKING: from rapidsmpf.streaming.core.channel import Channel diff --git a/python/cudf_streaming/cudf_streaming/tests/test_streaming_shuffler.py b/python/cudf_streaming/cudf_streaming/tests/test_streaming_shuffler.py index 43fb6751f691..f056ff95adc0 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_streaming_shuffler.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_streaming_shuffler.py @@ -19,6 +19,7 @@ unpack_and_concat as streaming_unpack_and_concat, ) from cudf_streaming.streaming.table_chunk import TableChunk +from cudf_streaming.testing import assert_eq from rapidsmpf.shuffler import PartitionAssignment from rapidsmpf.streaming.coll.shuffler import ( ShufflerAsync, @@ -30,7 +31,6 @@ push_to_channel, ) from rapidsmpf.streaming.core.message import Message -from rapidsmpf.testing import assert_eq if TYPE_CHECKING: from collections.abc import Awaitable diff --git a/python/cudf_streaming/cudf_streaming/tests/test_streaming_sparse_alltoall.py b/python/cudf_streaming/cudf_streaming/tests/test_streaming_sparse_alltoall.py index 78df8b3adc78..8b7544a42240 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_streaming_sparse_alltoall.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_streaming_sparse_alltoall.py @@ -14,8 +14,8 @@ packed_data_from_cudf_packed_columns, unpack_and_concat, ) +from cudf_streaming.testing import assert_eq from rapidsmpf.streaming.coll.sparse_alltoall import SparseAlltoall -from rapidsmpf.testing import assert_eq if TYPE_CHECKING: from rapidsmpf.communicator.communicator import Communicator diff --git a/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py b/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py index 6b55abd01f3c..4829dfca4721 100644 --- a/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py +++ b/python/cudf_streaming/cudf_streaming/tests/test_table_chunk.py @@ -16,6 +16,7 @@ TableChunk, make_table_chunks_available_or_wait, ) +from cudf_streaming.testing import assert_eq from rapidsmpf.cuda_stream import is_equal_streams from rapidsmpf.memory.buffer import MemoryType from rapidsmpf.memory.content_description import ContentDescription @@ -23,7 +24,6 @@ from rapidsmpf.streaming.core.actor import define_actor, run_actor_network from rapidsmpf.streaming.core.message import Message from rapidsmpf.streaming.core.spillable_messages import SpillableMessages -from rapidsmpf.testing import assert_eq if TYPE_CHECKING: from rapidsmpf.streaming.core.context import Context