Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
603 changes: 602 additions & 1 deletion .github/workflows/rdc-ci.yml

Large diffs are not rendered by default.

16 changes: 16 additions & 0 deletions projects/rdc/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,11 @@ option(BUILD_TESTS "Build test suite" OFF)
# When cmake -DBUILD_EXAMPLES=off, it will not build RDC examples
option(BUILD_EXAMPLES "Build examples" OFF)

# ThreadSanitizer race tests (tests/race); need BUILD_TESTS and a compiler with
# -fsanitize=thread. CI builds them with AMD clang. On kernels with 32 bits of
# mmap entropy, GCC's ThreadSanitizer needs `setarch -R` to run them.
option(RDC_BUILD_RACE_TESTS "Build the ThreadSanitizer race tests" OFF)

# Enable shared libraries for gtest
option(BUILD_SHARED_LIBS "Build shared library (.so) or not." ON)

Expand Down Expand Up @@ -395,6 +400,17 @@ if(BUILD_TESTS)
enable_testing()
add_subdirectory("tests/unit")
add_subdirectory("tests/fake_amdsmi")
if(RDC_BUILD_RACE_TESTS)
if(ADDRESS_SANITIZER OR CMAKE_CXX_FLAGS MATCHES "fsanitize=[a-z,]*(address|memory)")
message(
FATAL_ERROR
"RDC_BUILD_RACE_TESTS cannot be combined with AddressSanitizer or MemorySanitizer"
)
endif()
add_subdirectory("tests/race")
endif()
elseif(RDC_BUILD_RACE_TESTS)
message(FATAL_ERROR "RDC_BUILD_RACE_TESTS needs BUILD_TESTS=ON")
endif()

if(BUILD_EXAMPLES)
Expand Down
6 changes: 6 additions & 0 deletions projects/rdc/tests/ci/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@
import time

import pytest
from fake_gpu_stress import ( # noqa: F401
pytest_configure,
pytest_runtest_logreport,
pytest_runtest_setup,
pytest_sessionfinish,
)


def _find_binary(name):
Expand Down
7 changes: 7 additions & 0 deletions projects/rdc/tests/ci/fake_gpu_conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Installed as conftest.py next to the fake-GPU suite; only the stress plumbing lives here.
from fake_gpu_stress import ( # noqa: F401
pytest_configure,
pytest_runtest_logreport,
pytest_runtest_setup,
pytest_sessionfinish,
)
40 changes: 40 additions & 0 deletions projects/rdc/tests/ci/fake_gpu_stress.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
"""The `stress` marker, for tests/ci/conftest.py and the fake-GPU suite's installed conftest.py.

A stress test (a long loop) runs only with RDC_TEST_STRESS=1. With RDC_CI_REQUIRE_STRESS=1, a
session that ran no stress test fails, so a CI job cannot lose its stress tests without noticing.
"""

import os
import sys

import pytest

_ran = set() # stress tests whose body ran


def pytest_configure(config):
config.addinivalue_line("markers", "stress: long loop, runs only with RDC_TEST_STRESS=1")


def pytest_runtest_setup(item):
if item.get_closest_marker("stress") and os.environ.get("RDC_TEST_STRESS") != "1":
pytest.skip("stress test: set RDC_TEST_STRESS=1 to run it")


def pytest_runtest_logreport(report):
# The body ran if the call passed, failed or xfailed; a test skipped in setup or by itself did
# not run. (A failed setup or teardown fails the session anyway.)
if (
"stress" in report.keywords
and report.when == "call"
and (not report.skipped or hasattr(report, "wasxfail"))
):
_ran.add(report.nodeid)


def pytest_sessionfinish(session, exitstatus):
if os.environ.get("RDC_CI_REQUIRE_STRESS") != "1" or session.exitstatus != pytest.ExitCode.OK:
return
if not _ran:
print("RDC_CI_REQUIRE_STRESS=1, but no stress test ran", file=sys.stderr)
session.exitstatus = pytest.ExitCode.TESTS_FAILED
1 change: 1 addition & 0 deletions projects/rdc/tests/ci/pytest.ini
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@ python_classes = Test*
python_functions = test_*
markers =
performance: marks tests as performance/scalability (may be slow)
stress: long loop, runs only with RDC_TEST_STRESS=1
addopts = -v --tb=short
17 changes: 17 additions & 0 deletions projects/rdc/tests/ci/test_fake_gpu.py
Original file line number Diff line number Diff line change
Expand Up @@ -724,3 +724,20 @@ def test_runtime_files_do_not_contain_the_fake():
for path in binaries:
with open(path, "rb") as f:
assert b"RDC_FAKE_" not in f.read(), f"{path} references the fake"


# ---------------------------------------------------------------------------
# Stress: long loops, run only with RDC_TEST_STRESS=1
# ---------------------------------------------------------------------------
@pytest.mark.stress
def test_rdcd_start_watch_stop_loop(tools, tmp_path):
"""rdcd starts, serves a watch and stops cleanly 50 times in a row."""
for i in range(50):
run = tmp_path / f"run{i}"
run.mkdir()
d = Rdcd(tools, run).start()
try:
d.ok("dmon", "-e", ",".join(str(f) for f in DMON_FIELDS), "-i", "0", "-c", "1")
finally:
code = d.stop()
assert code == 0, f"run {i}: rdcd exited with {code}\n{d.output()}"
14 changes: 13 additions & 1 deletion projects/rdc/tests/fake_amdsmi/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,19 @@ install(
# tests/ci skips them.
file(GLOB FAKE_GPU_TESTS CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/tests/ci/test_fake_gpu*.py)
install(
FILES fake_values.inc ${FAKE_GPU_TESTS} ${PROJECT_SOURCE_DIR}/tests/ci/fake_gpu_golden.py
FILES
fake_values.inc
${FAKE_GPU_TESTS}
${PROJECT_SOURCE_DIR}/tests/ci/fake_gpu_golden.py
${PROJECT_SOURCE_DIR}/tests/ci/fake_gpu_stress.py
DESTINATION ${FAKE_GPU_TEST_DIR}
COMPONENT ${TESTS_COMPONENT}
)
# The installed suite's conftest.py carries only the stress plumbing; helpers and fixtures come
# from test_fake_gpu.py.
install(
FILES ${PROJECT_SOURCE_DIR}/tests/ci/fake_gpu_conftest.py
DESTINATION ${FAKE_GPU_TEST_DIR}
COMPONENT ${TESTS_COMPONENT}
RENAME conftest.py
)
4 changes: 3 additions & 1 deletion projects/rdc/tests/fake_amdsmi/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,6 @@ RDC_TEST_PREFIX=<install prefix> LD_LIBRARY_PATH=<prefix>/lib:<prefix>/lib/rdc \
```

`rdcd` allows one instance per host, so stop any other `rdcd` first. Set
`RDC_TEST_TIME_FACTOR` to stretch the timeouts for sanitizer builds.
`RDC_TEST_TIME_FACTOR` to stretch the timeouts for sanitizer builds, and
`RDC_TEST_STRESS=1` to run the stress tests (long loops, marked `stress`).
With `RDC_CI_REQUIRE_STRESS=1` a run in which no stress test ran fails.
43 changes: 43 additions & 0 deletions projects/rdc/tests/race/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Copyright (c) 2026 - present Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# rdc_race_tests: tests that drive RDC sources from several threads under
# ThreadSanitizer. Each race test compiles the RDC sources it drives into this
# target with -fsanitize=thread (the sanity test drives no RDC source);
# GoogleTest is not instrumented, and no RDC library, gRPC or protobuf is
# linked. Built with RDC_BUILD_RACE_TESTS=ON, run by CI, never installed.

# GoogleTest comes from tests/unit, which the top-level CMakeLists.txt adds
# first.
if(NOT TARGET GTest::gtest)
message(FATAL_ERROR "tests/race needs GoogleTest from tests/unit")
endif()
find_package(Threads REQUIRED)

add_executable(rdc_race_tests ${CMAKE_CURRENT_SOURCE_DIR}/test_race_sanity.cc)
# RDC's headers, for the sources the race tests compile in.
target_include_directories(rdc_race_tests PRIVATE ${PROJECT_SOURCE_DIR}/include)
target_compile_options(rdc_race_tests PRIVATE -fsanitize=thread -fno-omit-frame-pointer)
target_link_options(rdc_race_tests PRIVATE -fsanitize=thread)
target_link_libraries(rdc_race_tests PRIVATE GTest::gtest Threads::Threads)
add_test(NAME rdc_race_tests COMMAND rdc_race_tests)

# Each race test adds its test file and the RDC sources it drives below, in a
# target_sources() block of its own.
60 changes: 60 additions & 0 deletions projects/rdc/tests/race/test_race_sanity.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
Copyright (c) 2026 - present Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

// The race-test binary checks itself: ThreadSanitizer must be active, or every
// race test in it would pass without looking.

#include <gtest/gtest.h>

#include <cstdlib>
#include <thread>

namespace {

int g_unsynchronized = 0;

__attribute__((noinline)) void write_unsynchronized() { g_unsynchronized = g_unsynchronized + 1; }

// Two threads write one int without synchronization, then the process exits.
// ThreadSanitizer reports the race and exits with 66, by default and with
// halt_on_error=1 alike; an uninstrumented binary exits with 0.
void plant_a_race_and_exit() {
std::thread a(write_unsynchronized);
std::thread b(write_unsynchronized);
a.join();
b.join();
std::exit(0);
}

} // namespace

TEST(RaceSanity, ThreadSanitizerReportsAPlantedRace) {
EXPECT_EXIT(plant_a_race_and_exit(), testing::ExitedWithCode(66), "ThreadSanitizer: data race");
}

int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
// Death tests re-execute the binary ("threadsafe") instead of forking a
// threaded process.
GTEST_FLAG_SET(death_test_style, "threadsafe");
return RUN_ALL_TESTS();
}
Loading