From db0b967b3e49ec0e1c653f7b7e1988c19e65e0ea Mon Sep 17 00:00:00 2001 From: Adam360x Date: Wed, 14 Oct 2026 17:57:43 -0500 Subject: [PATCH 1/2] ci(rdc): run the no-GPU tests under ASan and under UBSan A new job, rdc_sanitize, has two legs. One builds RDC with GCC's -fsanitize=address and the other with -fsanitize=undefined, and each runs the installed gtests that need no GPU (all but rdc_diag_kernel_tests, which has no install RPATH), the fake-GPU suite and tests/ci against its build. GCC links the sanitizer runtime as a shared library, so rdcd, rdci, the gtests, the fake amd-smi and Python, which loads RDC's libraries through ctypes and gets the runtime through LD_PRELOAD, share one runtime. Every process writes its reports to one directory. A later step prints them, lists the distinct errors in the job summary and fails the job if there are any, if a report shows up instead in output the tests keep (such as rdcd's logs), or if the directory is missing because no test ran. The legs report what the other jobs cannot see, such as a read past a stack array or an invalid enum value that does not crash. The sanitizers get separate builds because in a process with both of GCC's runtimes, UBSan's reports go to stderr instead of to log_path (both runtimes export __sanitizer_set_report_path, and ASan's copy wins). The tests capture that stderr, so with one combined build the log directory had one report while rdcd's captured output held 270 UBSan errors. Before the tests, a step checks that rdcd, rdci, librdc, librdc_bootstrap and the fake amd-smi load the leg's runtime, that librdc calls the leg's checks, and that a program with a known bug leaves a report at log_path, so a build without the sanitizer fails instead of passing quietly. ASan's leak detection is off: Python under the runtime does not free everything at exit (151 allocations for "import ctypes, json"), and rdcd and rdci inherit its options. verify_asan_link_order is off because the tests preload the fake amd-smi, which links the runtime, ahead of it into rdcd; with the check on, rdcd exits at once. The legs build the same gRPC 1.67.1 as rdc_build and cache the install; each leg's ccache has its own key. Both are restored and saved in separate steps, since actions/cache saves nothing when a job fails. The legs name gcc and g++, build amd-smi with E-SMI and without a sanitizer, and build RDC without the runtime, profiler and RVS modules, which need ROCm's GPU runtime; rdcd runs without them. Each gtest has a time limit and writes a JUnit report, and the JUnit results go to the job summary as in the other test jobs. The legs do not gate until the repo variable RDC_SANITIZE_GATING is 'true'. On develop with the fake-GPU suite, in a GPU-less Ubuntu 22.04 container rather than the job's image, both legs' test runs pass every test: the 8 gtest programs the job runs (43 cases), 106 fake-GPU tests (24 skipped), and tests/ci but for test_prometheus_help, which needs prometheus_client (the job installs it). The ASan leg reports nothing. The UBSan leg reports, in 89 processes: a reference bound to a null pointer when RdcTelemetryModule passes &v[0] of an empty vector; an invalid rdc_field_type_t in fetch_smi_field() and the cache, because RDC_HEALTH_EEPROM_CONFIG_VALID reports success without setting a type or value, which then come from RdcSmiLib's uninitialized values array; the link type and P2P flag that rdc_device_topology_get() reads after a failed amd-smi call; and an invalid rdc_field_t where rdci dmon, the client library and rdcd load the field ID 99999 that a tests/ci test passes on purpose. GCC rather than clang for now: in local builds, ROCm's clang 22 with -shared-libsan reported ODR violations in the gtests (each string literal registered twice), and with clang's static runtime every process that loads RDC's instrumented libraries without being an instrumented executable (Python, setpriv, llvm-symbolizer) needed the runtime preloaded. A clang build like TheRock's is a follow-up. Signed-off-by: Adam360x --- .github/workflows/rdc-ci.yml | 324 +++++++++++++++++++++++++++++++++++ 1 file changed, 324 insertions(+) diff --git a/.github/workflows/rdc-ci.yml b/.github/workflows/rdc-ci.yml index dc7924889924..29e35849ecc0 100644 --- a/.github/workflows/rdc-ci.yml +++ b/.github/workflows/rdc-ci.yml @@ -332,6 +332,330 @@ jobs: path: projects/rdc/build/test-results-*.xml retention-days: 30 + # ================================================================= + # Sanitizer jobs — the no-GPU tests under ASan, and under UBSan + # Report problems that do not crash, such as reads past a buffer. + # Not gating until repo variable RDC_SANITIZE_GATING is 'true'. + # ================================================================= + rdc_sanitize: + name: rdc_sanitize (${{ matrix.sanitizer }}) + if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} + runs-on: lstt_ci_runners + timeout-minutes: 120 + continue-on-error: ${{ vars.RDC_SANITIZE_GATING != 'true' }} + container: rocm/dev-ubuntu-22.04:7.2 + strategy: + fail-fast: false + # One build per sanitizer: in a process with both of GCC's runtimes, + # UBSan reports go to stderr instead of to log_path. + matrix: + include: + - sanitizer: address + runtime: libasan.so + symbol: __asan_report_ + - sanitizer: undefined + runtime: libubsan.so + symbol: __ubsan_handle_ + env: + # Absolute paths inside the job container. + SANITIZE_PREFIX: /opt/rdc-sanitize + SANITIZER_LOGS: /tmp/rdc-sanitizer-logs + SANITIZER: ${{ matrix.sanitizer }} + SANITIZER_RUNTIME: ${{ matrix.runtime }} + SANITIZER_SYMBOL: ${{ matrix.symbol }} + + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + projects/amdsmi + projects/rdc + + - name: Install build and test tools + working-directory: . + run: | + apt update -y + apt install -y \ + build-essential \ + ccache \ + cmake \ + git \ + libcap-dev \ + libdrm-dev \ + libgflags-dev \ + libmnl-dev \ + libnl-3-dev \ + libnl-genl-3-dev \ + pkg-config \ + python3 \ + python3-pip \ + rocm-core + pip3 install pytest pytest-timeout prometheus_client + + - name: Set the cache date + run: echo "TODAY=$(date +%Y_%m_%d)" >> "$GITHUB_ENV" + + # actions/cache saves only when the job succeeds, and this job fails + # while there are sanitizer reports, so restore and save separately. + # The key names the container image; change it with the image or with + # the gRPC build. + - name: Restore gRPC + id: grpc-cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.GRPC_ROOT }} + key: grpc-${{ env.GRPC_VERSION }}-rocm-dev-ubuntu-22.04-7.2 + + - name: Build and install gRPC + if: ${{ steps.grpc-cache.outputs.cache-hit != 'true' }} + working-directory: . + shell: bash + run: | + set -euo pipefail + # The same gRPC as rdc_build: version, submodules and options. + git clone --branch "v${GRPC_VERSION}" --depth 1 \ + https://github.com/grpc/grpc.git /tmp/grpc + git -C /tmp/grpc submodule update --init --depth 1 \ + third_party/abseil-cpp \ + third_party/boringssl-with-bazel \ + third_party/cares/cares \ + third_party/protobuf \ + third_party/re2 \ + third_party/zlib + cmake -S /tmp/grpc -B /tmp/grpc/build \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX="${GRPC_ROOT}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release + cmake --build /tmp/grpc/build --parallel "$(nproc)" + cmake --install /tmp/grpc/build + + - name: Save gRPC + if: ${{ steps.grpc-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ env.GRPC_ROOT }} + key: grpc-${{ env.GRPC_VERSION }}-rocm-dev-ubuntu-22.04-7.2 + + - name: Register gRPC libraries + working-directory: . + run: | + echo "${GRPC_ROOT}/lib" > /etc/ld.so.conf.d/grpc.conf + ldconfig + + - name: Restore .ccache + id: ccache + uses: actions/cache/restore@v4 + with: + path: ~/.cache/ccache + key: ${{ runner.os }}-rdc-sanitize-${{ matrix.sanitizer }}-ccache-${{ github.ref_name }}-${{ env.TODAY }} + restore-keys: | + ${{ runner.os }}-rdc-sanitize-${{ matrix.sanitizer }}-ccache-${{ github.ref_name }}- + ${{ runner.os }}-rdc-sanitize-${{ matrix.sanitizer }}-ccache- + + - name: Build and install amd-smi + working-directory: projects/amdsmi + run: | + # Without the sanitizers: these tests check RDC, and amd-smi either has + # no GPU here or is replaced by the fake amd-smi. + cmake -B build \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${ROCM_DIR}" \ + -DENABLE_ESMI_LIB=ON + make -C build -j "$(nproc)" + make -C build install + + - name: Build RDC with the sanitizer + run: | + # GCC links the sanitizer runtime as a shared library, so rdcd, rdci, + # the gtests, the fake amd-smi and Python (with LD_PRELOAD below) share + # one runtime. + flags="-fsanitize=${SANITIZER} -fno-omit-frame-pointer" + link="-fsanitize=${SANITIZER}" + cmake \ + -B build-sanitize \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DCMAKE_C_FLAGS="${flags}" \ + -DCMAKE_CXX_FLAGS="${flags}" \ + -DCMAKE_EXE_LINKER_FLAGS="${link}" \ + -DCMAKE_SHARED_LINKER_FLAGS="${link}" \ + -DCMAKE_MODULE_LINKER_FLAGS="${link}" \ + -DGRPC_DESIRED_VERSION="${GRPC_VERSION}" \ + -DGRPC_ROOT="${GRPC_ROOT}" \ + -DBUILD_RUNTIME=OFF \ + -DBUILD_PROFILER=OFF \ + -DBUILD_RVS=OFF \ + -DBUILD_TESTS=ON \ + -DCMAKE_INSTALL_PREFIX="${SANITIZE_PREFIX}" + make -C build-sanitize -j "$(nproc)" + make -C build-sanitize install + ccache --show-stats + + - name: Save .ccache + if: ${{ !cancelled() && steps.ccache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: ~/.cache/ccache + key: ${{ runner.os }}-rdc-sanitize-${{ matrix.sanitizer }}-ccache-${{ github.ref_name }}-${{ env.TODAY }} + + - name: Check that the sanitizers are active + shell: bash + run: | + set -euo pipefail + # A build or environment that drops the sanitizers would pass every + # test and write no reports. + tmp="$(mktemp -d)" + for f in bin/rdcd bin/rdci lib/librdc.so lib/librdc_bootstrap.so \ + share/rdc/rdctst_tests/fake_gpu/librdc_fake_amdsmi.so; do + # The runtime must be needed and found ("=> /path", not "not found"). + if ! ldd "${SANITIZE_PREFIX}/${f}" > "${tmp}/ldd" \ + || ! awk -v r="${SANITIZER_RUNTIME}" \ + 'index($1, r) == 1 && $3 ~ /^\// { ok = 1 } END { exit !ok }' "${tmp}/ldd"; then + echo "${f} is missing or does not load ${SANITIZER_RUNTIME}" + exit 1 + fi + done + # Linking is not enough: the code must call the checks. + nm -D --undefined-only "${SANITIZE_PREFIX}/lib/librdc.so" > "${tmp}/syms" + if ! grep -qF "${SANITIZER_SYMBOL}" "${tmp}/syms"; then + echo "librdc.so calls no ${SANITIZER_SYMBOL}* checks" + exit 1 + fi + runtime="$(gcc -print-file-name="${SANITIZER_RUNTIME}")" + test -e "${runtime}" || { echo "no GCC ${SANITIZER_RUNTIME}"; exit 1; } + echo "SANITIZER_RUNTIME_PATH=${runtime}" >> "$GITHUB_ENV" + # Each leg's runtime must report its own error at log_path: ASan the + # read past the array, UBSan the signed overflow. The path is + # temporary because a report in SANITIZER_LOGS would count. + printf 'int main(int c, char** v) { int s = c + 2147483647; char a[4]; return a[c + 3] + s; }\n' > "${tmp}/c.c" + gcc -fsanitize="${SANITIZER}" -O0 -o "${tmp}/c" "${tmp}/c.c" + ASAN_OPTIONS="log_path=${tmp}/report" UBSAN_OPTIONS="log_path=${tmp}/report" "${tmp}/c" || true + grep -qE 'ERROR: AddressSanitizer|runtime error:' "${tmp}"/report.* \ + || { echo "the ${SANITIZER} runtime wrote no report to log_path"; exit 1; } + + - name: Run the no-GPU tests under the sanitizers + shell: bash + run: | + set -euo pipefail + mkdir -p build + # Reports from every process, whichever user a test runs it as. + install -d -m 1777 "${SANITIZER_LOGS}" + export LD_LIBRARY_PATH="${SANITIZE_PREFIX}/lib:${SANITIZE_PREFIX}/lib/rdc:${GRPC_ROOT}/lib:${ROCM_DIR}/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + # Each runtime reads only its own options. + # Leak detection is off: Python, which runs the tests with the runtime + # preloaded, does not free everything at exit, and rdcd and rdci + # inherit its options. + # The fake amd-smi, which links the runtime, is preloaded ahead of it + # into rdcd, so the runtime cannot insist on loading first. + export ASAN_OPTIONS="detect_leaks=0:verify_asan_link_order=0:log_path=${SANITIZER_LOGS}/asan" + export UBSAN_OPTIONS="print_stacktrace=1:log_path=${SANITIZER_LOGS}/ubsan" + # Python loads RDC's instrumented libraries through ctypes. + runtime="${SANITIZER_RUNTIME_PATH}" + failed=0 + ran=0 + shopt -s nullglob + for t in "${SANITIZE_PREFIX}"/share/rdc/rdctst_tests/rdc_*_tests; do + name="$(basename "$t")" + case "${name}" in + rdc_unit_tests) continue ;; # needs a GPU + rdc_diag_kernel_tests) continue ;; # has no install RPATH + esac + echo "::group::${name}" + timeout -k 30 600 "$t" --gtest_output="xml:build/test-results-sanitize-${name}.xml" || failed=1 + echo "::endgroup::" + ran=$((ran + 1)) + done + # rdc_codec_tests and rdc_client_tests at least. + test "${ran}" -ge 2 || { echo "ran ${ran} gtests"; failed=1; } + LD_PRELOAD="${runtime}" python3 -m pytest \ + "${SANITIZE_PREFIX}/share/rdc/rdctst_tests/fake_gpu" \ + -v --tb=short -p no:cacheprovider --timeout=600 \ + --junitxml=build/test-results-sanitize-fake-gpu.xml || failed=1 + ROCM_DIR="${SANITIZE_PREFIX}" LD_PRELOAD="${runtime}" python3 -m pytest \ + tests/ci --ignore-glob='tests/ci/test_fake_gpu*.py' \ + -v --tb=short -p no:cacheprovider --timeout=600 \ + --junitxml=build/test-results-sanitize-nogpu.xml || failed=1 + exit "${failed}" + env: + PYTHONDONTWRITEBYTECODE: "1" + RDC_TEST_PREFIX: ${{ env.SANITIZE_PREFIX }} + RDC_CI_REQUIRE_FAKE: "1" + RDC_TEST_TIME_FACTOR: "3" + + - name: Publish test summary + if: always() + run: python3 tests/ci/junit_summary.py >> "$GITHUB_STEP_SUMMARY" + + - name: Check sanitizer reports + if: ${{ !cancelled() }} + shell: bash + run: | + set -euo pipefail + # No directory means that no test ran, which is not a clean result. + test -d "${SANITIZER_LOGS}" \ + || { echo "no ${SANITIZER_LOGS}: the tests did not run"; exit 1; } + shopt -s nullglob + # The runtimes also log warnings; only these lines are errors. + errors='ERROR: [A-Za-z]*Sanitizer|CHECK failed:|runtime error:' + findings=() + for f in "${SANITIZER_LOGS}"/*; do + [ -f "$f" ] || continue + echo "::group::$(basename "$f")" + cat "$f" + echo "::endgroup::" + if grep -qaE "${errors}" "$f"; then + findings+=("$f") + fi + done + # A report that misses log_path lands in output the tests keep, such + # as the fake-GPU suite's rdcd.log files. + stray=() + for d in /tmp/pytest-of-*; do + while IFS= read -r f; do + stray+=("$f") + done < <(grep -rlaE "${errors}" "$d" || true) + done + { + echo "### ${SANITIZER} sanitizer: ${#findings[@]} report files with errors" + if [ "${#stray[@]}" -gt 0 ]; then + echo "Reports outside ${SANITIZER_LOGS}: ${stray[*]}" + fi + if [ "${#findings[@]}" -gt 0 ]; then + # Each distinct error once, the most frequent first, at most 50. + # A report cut short has no SUMMARY line, so its ERROR line + # stands in. Numbers after "ERROR:" or "runtime error:" vary, so + # they become N; a "==pid==" prefix goes. + for f in "${findings[@]}"; do + grep -haE 'SUMMARY: AddressSanitizer|CHECK failed:|runtime error:' "$f" \ + || grep -haE -m1 'ERROR: [A-Za-z]*Sanitizer' "$f" || true + done \ + | sed -E 's/^==[0-9]+==//' \ + | awk '{ i = index($0, "runtime error:"); if (!i && index($0, "ERROR:") == 1) i = 1; if (i) { t = substr($0, i); gsub(/0x[0-9a-fA-F]+|[0-9]+/, "N", t); $0 = substr($0, 1, i - 1) t } print }' \ + | sort | uniq -c | sort -rn \ + | awk 'NR <= 50 { n = $1; $1 = ""; printf "- %d x `%s`\n", n, substr($0, 2) } + END { if (NR > 50) printf "- and %d more\n", NR - 50 }' + fi + } | tee -a "$GITHUB_STEP_SUMMARY" + test "${#findings[@]}" -eq 0 && test "${#stray[@]}" -eq 0 + + - name: Upload sanitizer reports and test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: rdc-sanitize-${{ matrix.sanitizer }}-results + path: | + ${{ env.SANITIZER_LOGS }} + projects/rdc/build/test-results-sanitize-*.xml + retention-days: 30 + # ================================================================= # GPU test job — requires AMD GPU hardware # Enable by setting repo variable: RDC_GPU_TESTS_ENABLED=true From c8b88c05bce47ead3a7c4c800c79242c26121d2c Mon Sep 17 00:00:00 2001 From: Adam360x Date: Thu, 15 Oct 2026 04:27:42 -0500 Subject: [PATCH 2/2] ci(rdc): build and test RDC with AMD clang and static gRPC Add the rdc_clang job: RDC built with the toolchain TheRock uses, AMD clang with gRPC 1.78.1 linked statically. It checks that RDC was compiled by AMD clang and that no RDC binary or library needs a gRPC, protobuf or Abseil library; runs the gtests, the fake-GPU suite and the other no-GPU tests; and checks that the gtests ran and that the pytest suites still collect their tests. It also runs: - rdc_race_tests: a ThreadSanitizer test binary into which each race test compiles the RDC sources it drives, built with RDC_BUILD_RACE_TESTS (off by default) and never installed. Its only test so far checks that ThreadSanitizer reports a planted race. - stress tests: a test marked `stress` runs only with RDC_TEST_STRESS=1, which this job sets, and with RDC_CI_REQUIRE_STRESS=1 a run in which no stress test ran fails. The first one starts rdcd, reads one sample of eight fields and stops rdcd, 50 times. The installed fake-GPU suite gains a conftest.py and fake_gpu_stress.py for this. The job's container runs without Docker's default seccomp profile: on hosts with 32 bits of mmap entropy ThreadSanitizer turns address randomization off for itself, which that profile blocks. The job does not gate until the repository variable RDC_CLANG_GATING is 'true'. Signed-off-by: Adam360x --- .github/workflows/rdc-ci.yml | 279 +++++++++++++++++- projects/rdc/CMakeLists.txt | 16 + projects/rdc/tests/ci/conftest.py | 6 + projects/rdc/tests/ci/fake_gpu_conftest.py | 7 + projects/rdc/tests/ci/fake_gpu_stress.py | 40 +++ projects/rdc/tests/ci/pytest.ini | 1 + projects/rdc/tests/ci/test_fake_gpu.py | 17 ++ projects/rdc/tests/fake_amdsmi/CMakeLists.txt | 14 +- projects/rdc/tests/fake_amdsmi/README.md | 4 +- projects/rdc/tests/race/CMakeLists.txt | 43 +++ projects/rdc/tests/race/test_race_sanity.cc | 60 ++++ 11 files changed, 484 insertions(+), 3 deletions(-) create mode 100644 projects/rdc/tests/ci/fake_gpu_conftest.py create mode 100644 projects/rdc/tests/ci/fake_gpu_stress.py create mode 100644 projects/rdc/tests/race/CMakeLists.txt create mode 100644 projects/rdc/tests/race/test_race_sanity.cc diff --git a/.github/workflows/rdc-ci.yml b/.github/workflows/rdc-ci.yml index 29e35849ecc0..48fdac6aa61d 100644 --- a/.github/workflows/rdc-ci.yml +++ b/.github/workflows/rdc-ci.yml @@ -288,7 +288,7 @@ jobs: shell: bash run: | # TheRock runs this copy; it must import on its own and hold only the - # fake library, the suite and its value table. + # fake library, the suite, its value table and its stress plumbing. suite="$ROCM_DIR/share/rdc/rdctst_tests/fake_gpu" python3 -m pytest --collect-only -q -p no:cacheprovider "$suite" extra=$(find "$suite" -type f ! -name '*.py' ! -name '*.so' ! -name '*.inc') @@ -656,6 +656,283 @@ jobs: projects/rdc/build/test-results-sanitize-*.xml retention-days: 30 + # ================================================================= + # Clang job — RDC built with the toolchain TheRock uses: AMD clang, + # gRPC 1.78.1 linked statically. Runs the no-GPU tests, the stress + # tests and the ThreadSanitizer race tests. Not gating until repo + # variable RDC_CLANG_GATING is 'true'. + # ================================================================= + rdc_clang: + if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} + needs: rdc_build + runs-on: lstt_ci_runners + timeout-minutes: 60 + continue-on-error: ${{ vars.RDC_CLANG_GATING != 'true' }} + container: + image: rocm/dev-ubuntu-22.04:7.2 + # ThreadSanitizer turns address randomization off for itself on hosts + # with 32 bits of mmap entropy (vm.mmap_rnd_bits=32, as Ubuntu's 6.8 + # kernels set it), and Docker's default seccomp profile blocks that + # (setarch -R fails there too). Only seccomp is relaxed: no --privileged, + # no added capabilities, and like the other jobs this one runs only for + # branches of this repository. Drop the option if the race tests leave + # this job. + options: --security-opt seccomp=unconfined + env: + CC: /opt/rocm/llvm/bin/clang + CXX: /opt/rocm/llvm/bin/clang++ + GRPC_VERSION: "1.78.1" + GRPC_ROOT: /opt/grpc-static + CLANG_PREFIX: /opt/rdc-clang + + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: projects/rdc + + - name: Install build and test tools + working-directory: . + run: | + apt update -y + apt install -y \ + build-essential \ + ccache \ + cmake \ + git \ + libcap-dev \ + libdrm-dev \ + libgflags-dev \ + libmnl-dev \ + libnl-3-dev \ + libnl-genl-3-dev \ + pkg-config \ + python3 \ + python3-pip \ + rocm-core + pip3 install pytest pytest-timeout prometheus_client + + - name: Check the compiler + shell: bash + run: | + set -euo pipefail + version="$("${CXX}" --version | head -1)" + echo "${version}" + grep -q '^AMD clang version' <<< "${version}" \ + || { echo "${CXX} is not AMD clang"; exit 1; } + + - name: Set the cache date and compiler + shell: bash + run: | + set -euo pipefail + echo "TODAY=$(date +%Y_%m_%d)" >> "$GITHUB_ENV" + # A static gRPC is linked into RDC, so its cache is per compiler. + echo "CLANG_ID=$("${CXX}" --version | head -1 | sha256sum | cut -c1-12)" >> "$GITHUB_ENV" + + - name: Download the amd-smi package + uses: actions/download-artifact@v4 + with: + name: rdc-packages + path: /tmp/rdc-packages + + - name: Install amd-smi + shell: bash + working-directory: . + run: | + set -euo pipefail + mapfile -t debs < <(find /tmp/rdc-packages -iname 'amd-smi-lib*.deb' ! -iname '*tests*') + test "${#debs[@]}" -eq 1 + apt-get install -y "${debs[@]}" + + # actions/cache saves only when the job succeeds, so restore and save + # separately. The key names the image, the compiler and the gRPC version; + # change it when the gRPC build options below change. + - name: Restore gRPC + id: grpc-cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.GRPC_ROOT }} + key: grpc-${{ env.GRPC_VERSION }}-static-clang-${{ env.CLANG_ID }}-rocm-dev-ubuntu-22.04-7.2 + + - name: Build and install gRPC + if: ${{ steps.grpc-cache.outputs.cache-hit != 'true' }} + working-directory: . + shell: bash + run: | + set -euo pipefail + git clone --branch "v${GRPC_VERSION}" --depth 1 \ + https://github.com/grpc/grpc.git /tmp/grpc + git -C /tmp/grpc submodule update --init --depth 1 \ + third_party/abseil-cpp \ + third_party/boringssl-with-bazel \ + third_party/cares/cares \ + third_party/protobuf \ + third_party/re2 \ + third_party/zlib + # Static and position-independent, as TheRock links it into RDC. + cmake -S /tmp/grpc -B /tmp/grpc/build \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DCMAKE_INSTALL_PREFIX="${GRPC_ROOT}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release + cmake --build /tmp/grpc/build --parallel "$(nproc)" + cmake --install /tmp/grpc/build + + - name: Save gRPC + if: ${{ steps.grpc-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ env.GRPC_ROOT }} + key: grpc-${{ env.GRPC_VERSION }}-static-clang-${{ env.CLANG_ID }}-rocm-dev-ubuntu-22.04-7.2 + + - name: Restore .ccache + id: ccache + uses: actions/cache/restore@v4 + with: + path: ~/.cache/ccache + key: ${{ runner.os }}-rdc-clang-ccache-${{ github.ref_name }}-${{ env.TODAY }} + restore-keys: | + ${{ runner.os }}-rdc-clang-ccache-${{ github.ref_name }}- + ${{ runner.os }}-rdc-clang-ccache- + + - name: Build RDC with clang + shell: bash + run: | + set -euo pipefail + cmake \ + -B build-clang \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DGRPC_DESIRED_VERSION="${GRPC_VERSION}" \ + -DGRPC_ROOT="${GRPC_ROOT}" \ + -DBUILD_RUNTIME=OFF \ + -DBUILD_PROFILER=OFF \ + -DBUILD_RVS=OFF \ + -DBUILD_TESTS=ON \ + -DRDC_BUILD_RACE_TESTS=ON \ + -DCMAKE_INSTALL_PREFIX="${CLANG_PREFIX}" + make -C build-clang -j "$(nproc)" + make -C build-clang install + ccache --show-stats + + - name: Save .ccache + if: ${{ !cancelled() && steps.ccache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: ~/.cache/ccache + key: ${{ runner.os }}-rdc-clang-ccache-${{ github.ref_name }}-${{ env.TODAY }} + + - name: Check the build + shell: bash + run: | + set -euo pipefail + # RDC itself compiled by AMD clang: the configured compiler, and the + # producer recorded in RDC's own objects (a linked binary also holds + # the producers of the static gRPC it contains). + grep -qxF "CMAKE_CXX_COMPILER:FILEPATH=${CXX}" build-clang/CMakeCache.txt \ + || { echo "RDC was not configured with ${CXX}"; exit 1; } + readelf="$(dirname "${CXX}")/llvm-readelf" + test -x "${readelf}" || { echo "no ${readelf}"; exit 1; } + for src in RdcEmbeddedHandler.cc rdc_server_main.cc; do + obj="$(find build-clang -name "${src}.o" -print -quit)" + test -n "${obj}" || { echo "no object file for ${src}"; exit 1; } + grep -q 'AMD clang' <<< "$("${readelf}" -p .comment "${obj}")" \ + || { echo "${obj} was not compiled by AMD clang"; exit 1; } + done + # gRPC linked statically: no RDC binary or library needs a gRPC, + # protobuf or Abseil library. + for f in bin/rdcd bin/rdci lib/librdc.so lib/librdc_bootstrap.so lib/librdc_client.so; do + test -f "${CLANG_PREFIX}/${f}" || { echo "no ${f}"; exit 1; } + dynamic="$("${readelf}" -d "${CLANG_PREFIX}/${f}")" + if grep -E '\(NEEDED\).*\[lib(grpc|gpr|protobuf|absl)' <<< "${dynamic}"; then + echo "${f} needs a gRPC, protobuf or Abseil library"; exit 1 + fi + done + # The installed fake-GPU suite carries its stress plumbing. + for f in conftest.py fake_gpu_stress.py; do + test -f "${CLANG_PREFIX}/share/rdc/rdctst_tests/fake_gpu/${f}" \ + || { echo "the installed fake-GPU suite has no ${f}"; exit 1; } + done + + - name: Run the tests + shell: bash + run: | + set -euo pipefail + mkdir -p build + # Time limits scale like the tests' own waits (an integer factor). + limit=$((300 * RDC_TEST_TIME_FACTOR)) + export ROCM_DIR="${CLANG_PREFIX}" RDC_TEST_PREFIX="${CLANG_PREFIX}" + # gRPC is linked in: no gRPC directory on the path. + export LD_LIBRARY_PATH="${CLANG_PREFIX}/lib:${CLANG_PREFIX}/lib/rdc:/opt/rocm/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + failed=0 + ran=0 + shopt -s nullglob + for t in "${CLANG_PREFIX}"/share/rdc/rdctst_tests/rdc_*_tests; do + name="$(basename "$t")" + case "${name}" in + rdc_unit_tests) continue ;; # needs a GPU + rdc_diag_kernel_tests) continue ;; # has no install RPATH + esac + echo "::group::${name}" + timeout -k 30 "${limit}" "$t" --gtest_output="xml:build/test-results-clang-${name}.xml" || failed=1 + echo "::endgroup::" + ran=$((ran + 1)) + done + test "${ran}" -ge 2 || { echo "ran ${ran} gtests"; failed=1; } + # Race tests: built, never installed. ThreadSanitizer reports every + # race it sees, then exits with 66; the test results still pass. + echo "::group::rdc_race_tests" + timeout -k 30 "${limit}" build-clang/tests/race/rdc_race_tests \ + --gtest_output="xml:build/test-results-clang-rdc_race_tests.xml" || { + rc=$? + failed=1 + echo "::error title=rdc_race_tests::exit ${rc} (66: ThreadSanitizer found a race or could not start; 124: timed out; 137: killed)" + } + echo "::endgroup::" + # Each pytest suite must still collect at least this many tests: a + # broken import must not shrink it to a pass. + collected() { + local out + out="$(timeout -k 30 "${limit}" python3 -m pytest --collect-only -q -p no:cacheprovider "$@" 2>&1)" \ + || { echo 0; return; } + grep -Eo '[0-9]+ tests? collected' <<< "${out}" | tail -1 | grep -Eo '^[0-9]+' || echo 0 + } + suite="${CLANG_PREFIX}/share/rdc/rdctst_tests/fake_gpu" + fake="$(collected "${suite}")" + nogpu="$(collected tests/ci --ignore-glob='tests/ci/test_fake_gpu*.py')" + echo "collected: ${fake} fake-GPU tests, ${nogpu} other no-GPU tests" + test "${fake}" -ge 100 || { echo "too few fake-GPU tests"; failed=1; } + test "${nogpu}" -ge 50 || { echo "too few no-GPU tests"; failed=1; } + RDC_CI_REQUIRE_FAKE=1 RDC_CI_REQUIRE_STRESS=1 python3 -m pytest "${suite}" \ + -v --tb=short -p no:cacheprovider --timeout="${limit}" \ + --junitxml=build/test-results-clang-fake-gpu.xml || failed=1 + python3 -m pytest tests/ci --ignore-glob='tests/ci/test_fake_gpu*.py' \ + -v --tb=short -p no:cacheprovider --timeout="${limit}" \ + --junitxml=build/test-results-clang-nogpu.xml || failed=1 + exit "${failed}" + env: + PYTHONDONTWRITEBYTECODE: "1" + RDC_TEST_STRESS: "1" + # The runners are shared: room for a loaded host. + RDC_TEST_TIME_FACTOR: "2" + RDC_SRC_DIR: ${{ github.workspace }}/projects/rdc/python_binding + + - name: Publish test summary + if: always() + run: python3 tests/ci/junit_summary.py >> "$GITHUB_STEP_SUMMARY" + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: rdc-clang-results + path: projects/rdc/build/test-results-clang-*.xml + retention-days: 30 + # ================================================================= # GPU test job — requires AMD GPU hardware # Enable by setting repo variable: RDC_GPU_TESTS_ENABLED=true diff --git a/projects/rdc/CMakeLists.txt b/projects/rdc/CMakeLists.txt index 195b3ae8a2cf..1fdece4fafcd 100755 --- a/projects/rdc/CMakeLists.txt +++ b/projects/rdc/CMakeLists.txt @@ -122,6 +122,11 @@ option(BUILD_TESTS "Build test suite" OFF) # When cmake -DBUILD_EXAMPLES=off, it will not build RDC examples option(BUILD_EXAMPLES "Build examples" OFF) +# ThreadSanitizer race tests (tests/race); need BUILD_TESTS and a compiler with +# -fsanitize=thread. CI builds them with AMD clang. On kernels with 32 bits of +# mmap entropy, GCC's ThreadSanitizer needs `setarch -R` to run them. +option(RDC_BUILD_RACE_TESTS "Build the ThreadSanitizer race tests" OFF) + # Enable shared libraries for gtest option(BUILD_SHARED_LIBS "Build shared library (.so) or not." ON) @@ -395,6 +400,17 @@ if(BUILD_TESTS) enable_testing() add_subdirectory("tests/unit") add_subdirectory("tests/fake_amdsmi") + if(RDC_BUILD_RACE_TESTS) + if(ADDRESS_SANITIZER OR CMAKE_CXX_FLAGS MATCHES "fsanitize=[a-z,]*(address|memory)") + message( + FATAL_ERROR + "RDC_BUILD_RACE_TESTS cannot be combined with AddressSanitizer or MemorySanitizer" + ) + endif() + add_subdirectory("tests/race") + endif() +elseif(RDC_BUILD_RACE_TESTS) + message(FATAL_ERROR "RDC_BUILD_RACE_TESTS needs BUILD_TESTS=ON") endif() if(BUILD_EXAMPLES) diff --git a/projects/rdc/tests/ci/conftest.py b/projects/rdc/tests/ci/conftest.py index baf616c90add..b17c4ea6da42 100644 --- a/projects/rdc/tests/ci/conftest.py +++ b/projects/rdc/tests/ci/conftest.py @@ -8,6 +8,12 @@ import time import pytest +from fake_gpu_stress import ( # noqa: F401 + pytest_configure, + pytest_runtest_logreport, + pytest_runtest_setup, + pytest_sessionfinish, +) def _find_binary(name): diff --git a/projects/rdc/tests/ci/fake_gpu_conftest.py b/projects/rdc/tests/ci/fake_gpu_conftest.py new file mode 100644 index 000000000000..f0b3958a4b74 --- /dev/null +++ b/projects/rdc/tests/ci/fake_gpu_conftest.py @@ -0,0 +1,7 @@ +# Installed as conftest.py next to the fake-GPU suite; only the stress plumbing lives here. +from fake_gpu_stress import ( # noqa: F401 + pytest_configure, + pytest_runtest_logreport, + pytest_runtest_setup, + pytest_sessionfinish, +) diff --git a/projects/rdc/tests/ci/fake_gpu_stress.py b/projects/rdc/tests/ci/fake_gpu_stress.py new file mode 100644 index 000000000000..e502d3eca826 --- /dev/null +++ b/projects/rdc/tests/ci/fake_gpu_stress.py @@ -0,0 +1,40 @@ +"""The `stress` marker, for tests/ci/conftest.py and the fake-GPU suite's installed conftest.py. + +A stress test (a long loop) runs only with RDC_TEST_STRESS=1. With RDC_CI_REQUIRE_STRESS=1, a +session that ran no stress test fails, so a CI job cannot lose its stress tests without noticing. +""" + +import os +import sys + +import pytest + +_ran = set() # stress tests whose body ran + + +def pytest_configure(config): + config.addinivalue_line("markers", "stress: long loop, runs only with RDC_TEST_STRESS=1") + + +def pytest_runtest_setup(item): + if item.get_closest_marker("stress") and os.environ.get("RDC_TEST_STRESS") != "1": + pytest.skip("stress test: set RDC_TEST_STRESS=1 to run it") + + +def pytest_runtest_logreport(report): + # The body ran if the call passed, failed or xfailed; a test skipped in setup or by itself did + # not run. (A failed setup or teardown fails the session anyway.) + if ( + "stress" in report.keywords + and report.when == "call" + and (not report.skipped or hasattr(report, "wasxfail")) + ): + _ran.add(report.nodeid) + + +def pytest_sessionfinish(session, exitstatus): + if os.environ.get("RDC_CI_REQUIRE_STRESS") != "1" or session.exitstatus != pytest.ExitCode.OK: + return + if not _ran: + print("RDC_CI_REQUIRE_STRESS=1, but no stress test ran", file=sys.stderr) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/projects/rdc/tests/ci/pytest.ini b/projects/rdc/tests/ci/pytest.ini index cc4935f23e6b..bb9048cf2284 100644 --- a/projects/rdc/tests/ci/pytest.ini +++ b/projects/rdc/tests/ci/pytest.ini @@ -5,4 +5,5 @@ python_classes = Test* python_functions = test_* markers = performance: marks tests as performance/scalability (may be slow) + stress: long loop, runs only with RDC_TEST_STRESS=1 addopts = -v --tb=short diff --git a/projects/rdc/tests/ci/test_fake_gpu.py b/projects/rdc/tests/ci/test_fake_gpu.py index 632bde9468b8..c636bd333664 100644 --- a/projects/rdc/tests/ci/test_fake_gpu.py +++ b/projects/rdc/tests/ci/test_fake_gpu.py @@ -724,3 +724,20 @@ def test_runtime_files_do_not_contain_the_fake(): for path in binaries: with open(path, "rb") as f: assert b"RDC_FAKE_" not in f.read(), f"{path} references the fake" + + +# --------------------------------------------------------------------------- +# Stress: long loops, run only with RDC_TEST_STRESS=1 +# --------------------------------------------------------------------------- +@pytest.mark.stress +def test_rdcd_start_watch_stop_loop(tools, tmp_path): + """rdcd starts, serves a watch and stops cleanly 50 times in a row.""" + for i in range(50): + run = tmp_path / f"run{i}" + run.mkdir() + d = Rdcd(tools, run).start() + try: + d.ok("dmon", "-e", ",".join(str(f) for f in DMON_FIELDS), "-i", "0", "-c", "1") + finally: + code = d.stop() + assert code == 0, f"run {i}: rdcd exited with {code}\n{d.output()}" diff --git a/projects/rdc/tests/fake_amdsmi/CMakeLists.txt b/projects/rdc/tests/fake_amdsmi/CMakeLists.txt index dd9b6113bfc6..e33901dd42e6 100644 --- a/projects/rdc/tests/fake_amdsmi/CMakeLists.txt +++ b/projects/rdc/tests/fake_amdsmi/CMakeLists.txt @@ -37,7 +37,19 @@ install( # tests/ci skips them. file(GLOB FAKE_GPU_TESTS CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/tests/ci/test_fake_gpu*.py) install( - FILES fake_values.inc ${FAKE_GPU_TESTS} ${PROJECT_SOURCE_DIR}/tests/ci/fake_gpu_golden.py + FILES + fake_values.inc + ${FAKE_GPU_TESTS} + ${PROJECT_SOURCE_DIR}/tests/ci/fake_gpu_golden.py + ${PROJECT_SOURCE_DIR}/tests/ci/fake_gpu_stress.py DESTINATION ${FAKE_GPU_TEST_DIR} COMPONENT ${TESTS_COMPONENT} ) +# The installed suite's conftest.py carries only the stress plumbing; helpers and fixtures come +# from test_fake_gpu.py. +install( + FILES ${PROJECT_SOURCE_DIR}/tests/ci/fake_gpu_conftest.py + DESTINATION ${FAKE_GPU_TEST_DIR} + COMPONENT ${TESTS_COMPONENT} + RENAME conftest.py +) diff --git a/projects/rdc/tests/fake_amdsmi/README.md b/projects/rdc/tests/fake_amdsmi/README.md index 6ea1d04d5828..9b07bd10741b 100644 --- a/projects/rdc/tests/fake_amdsmi/README.md +++ b/projects/rdc/tests/fake_amdsmi/README.md @@ -43,4 +43,6 @@ RDC_TEST_PREFIX= LD_LIBRARY_PATH=/lib:/lib/rdc \ ``` `rdcd` allows one instance per host, so stop any other `rdcd` first. Set -`RDC_TEST_TIME_FACTOR` to stretch the timeouts for sanitizer builds. +`RDC_TEST_TIME_FACTOR` to stretch the timeouts for sanitizer builds, and +`RDC_TEST_STRESS=1` to run the stress tests (long loops, marked `stress`). +With `RDC_CI_REQUIRE_STRESS=1` a run in which no stress test ran fails. diff --git a/projects/rdc/tests/race/CMakeLists.txt b/projects/rdc/tests/race/CMakeLists.txt new file mode 100644 index 000000000000..3012d6f0293e --- /dev/null +++ b/projects/rdc/tests/race/CMakeLists.txt @@ -0,0 +1,43 @@ +# Copyright (c) 2026 - present Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# rdc_race_tests: tests that drive RDC sources from several threads under +# ThreadSanitizer. Each race test compiles the RDC sources it drives into this +# target with -fsanitize=thread (the sanity test drives no RDC source); +# GoogleTest is not instrumented, and no RDC library, gRPC or protobuf is +# linked. Built with RDC_BUILD_RACE_TESTS=ON, run by CI, never installed. + +# GoogleTest comes from tests/unit, which the top-level CMakeLists.txt adds +# first. +if(NOT TARGET GTest::gtest) + message(FATAL_ERROR "tests/race needs GoogleTest from tests/unit") +endif() +find_package(Threads REQUIRED) + +add_executable(rdc_race_tests ${CMAKE_CURRENT_SOURCE_DIR}/test_race_sanity.cc) +# RDC's headers, for the sources the race tests compile in. +target_include_directories(rdc_race_tests PRIVATE ${PROJECT_SOURCE_DIR}/include) +target_compile_options(rdc_race_tests PRIVATE -fsanitize=thread -fno-omit-frame-pointer) +target_link_options(rdc_race_tests PRIVATE -fsanitize=thread) +target_link_libraries(rdc_race_tests PRIVATE GTest::gtest Threads::Threads) +add_test(NAME rdc_race_tests COMMAND rdc_race_tests) + +# Each race test adds its test file and the RDC sources it drives below, in a +# target_sources() block of its own. diff --git a/projects/rdc/tests/race/test_race_sanity.cc b/projects/rdc/tests/race/test_race_sanity.cc new file mode 100644 index 000000000000..0c9ce0bce21e --- /dev/null +++ b/projects/rdc/tests/race/test_race_sanity.cc @@ -0,0 +1,60 @@ +/* +Copyright (c) 2026 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +// The race-test binary checks itself: ThreadSanitizer must be active, or every +// race test in it would pass without looking. + +#include + +#include +#include + +namespace { + +int g_unsynchronized = 0; + +__attribute__((noinline)) void write_unsynchronized() { g_unsynchronized = g_unsynchronized + 1; } + +// Two threads write one int without synchronization, then the process exits. +// ThreadSanitizer reports the race and exits with 66, by default and with +// halt_on_error=1 alike; an uninstrumented binary exits with 0. +void plant_a_race_and_exit() { + std::thread a(write_unsynchronized); + std::thread b(write_unsynchronized); + a.join(); + b.join(); + std::exit(0); +} + +} // namespace + +TEST(RaceSanity, ThreadSanitizerReportsAPlantedRace) { + EXPECT_EXIT(plant_a_race_and_exit(), testing::ExitedWithCode(66), "ThreadSanitizer: data race"); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + // Death tests re-execute the binary ("threadsafe") instead of forking a + // threaded process. + GTEST_FLAG_SET(death_test_style, "threadsafe"); + return RUN_ALL_TESTS(); +}