diff --git a/projects/rccl-tests/src/CMakeLists.txt b/projects/rccl-tests/src/CMakeLists.txt index 3f671325cce2..12056a84b4fb 100644 --- a/projects/rccl-tests/src/CMakeLists.txt +++ b/projects/rccl-tests/src/CMakeLists.txt @@ -145,27 +145,6 @@ endif() add_rccl_test(all_gather) add_rccl_test(all_reduce) add_rccl_test(alltoall) -if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem) - target_compile_definitions(alltoall_perf PRIVATE ENABLE_ROCSHMEM_GIN) - target_compile_options(alltoall_perf PRIVATE -fgpu-rdc $<$:-ggdb -O0>) - # --whole-archive: librccl.so (ENABLE_ROCSHMEM_GIN) leaves rocshmem host - # symbols unresolved (--allow-shlib-undefined). The executable must pull - # in ALL rocshmem symbols (not just those it references directly) so that - # -rdynamic exports them for librccl.so to resolve at runtime. - target_link_libraries(alltoall_perf PRIVATE - -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive) - target_link_options(alltoall_perf PRIVATE -fgpu-rdc --hip-link -rdynamic) - - # GIN-SDMA hybrid AllGather (all_gather_perf -D 3): same device-RDC link as - # alltoall_perf so the GinHybridAllGatherKernel resolves the GIN device backend. - if (ENABLE_DEVICE_API) - target_compile_definitions(all_gather_perf PRIVATE ENABLE_DEVICE_API) - endif() - target_compile_options(all_gather_perf PRIVATE -fgpu-rdc $<$:-ggdb -O0>) - target_link_libraries(all_gather_perf PRIVATE - -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive) - target_link_options(all_gather_perf PRIVATE -fgpu-rdc --hip-link -rdynamic) -endif() if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem AND RCCL_SOURCE_DIR) add_executable(gin_rocshmem_gda_factory_test ${CMAKE_CURRENT_SOURCE_DIR}/gin_rocshmem_gda_factory_test.cu) set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/gin_rocshmem_gda_factory_test.cu PROPERTIES LANGUAGE CXX) @@ -185,16 +164,6 @@ if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem AND RCCL_SOURCE_DIR) endif() add_rccl_test(alltoallv) add_rccl_test(broadcast) -if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem) - # GIN-SDMA hybrid Broadcast (broadcast_perf -D 3): same device-RDC link as - # alltoall_perf/all_gather_perf so the Gin*BroadcastKernel variants resolve - # the GIN device backend (SDMA puts + LSA/LL peer stores). - target_compile_definitions(broadcast_perf PRIVATE ENABLE_ROCSHMEM_GIN) - target_compile_options(broadcast_perf PRIVATE -fgpu-rdc $<$:-ggdb -O0>) - target_link_libraries(broadcast_perf PRIVATE - -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive) - target_link_options(broadcast_perf PRIVATE -fgpu-rdc --hip-link -rdynamic) -endif() add_rccl_test(gather) add_rccl_test(hypercube) add_rccl_test(reduce_scatter) @@ -202,3 +171,32 @@ add_rccl_test(reduce) add_rccl_test(scatter) add_rccl_test(sendrecv) add_rccl_test(all_reduce_bias) +if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem) + function(_rccl_tests_link_rocshmem_gin_rdc target) + target_compile_definitions(${target} PRIVATE ENABLE_ROCSHMEM_GIN NCCL_GIN_ANVIL_SDMA_ENABLE=1) + target_compile_options(${target} PRIVATE -fgpu-rdc $<$:-ggdb -O0>) + # --whole-archive: librccl.so (ENABLE_ROCSHMEM_GIN) leaves rocshmem host + # symbols unresolved (--allow-shlib-undefined). The executable must pull + # in ALL rocshmem symbols (not just those it references directly) so that + # -rdynamic exports them for librccl.so to resolve at runtime. + target_link_libraries(${target} PRIVATE + -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive) + target_link_options(${target} PRIVATE -fgpu-rdc --hip-link -rdynamic) + endfunction() + + # Device-api CI builds one --rocshmem-gin librccl.so and runs the matrix + # against it. Every launched binary must RDC-link rocSHMEM so GIN host + # symbols resolve from the executable. all_reduce_perf is omitted: it + # still does not build under ENABLE_ROCSHMEM_GIN (gin.sbatch uses + # `make -k` for the same reason). + foreach(_gin_rdc_bin IN ITEMS + all_gather_perf + alltoall_perf + broadcast_perf + gather_perf + reduce_perf + scatter_perf + sendrecv_perf) + _rccl_tests_link_rocshmem_gin_rdc(${_gin_rdc_bin}) + endforeach() +endif() diff --git a/projects/rccl/tools/ci/lib/device-api-nongin-tests.json b/projects/rccl/tools/ci/lib/device-api-nongin-tests.json new file mode 100644 index 000000000000..bae550ca682f --- /dev/null +++ b/projects/rccl/tools/ci/lib/device-api-nongin-tests.json @@ -0,0 +1,29 @@ +{ + "_comment": [ + "Non-GIN device-API matrix for run-device-api-ci.sh.", + "Built with ENABLE_DEVICE_API=1 ./install.sh (no ENABLE_ROCSHMEM_GIN).", + "LSA all_reduce_perf (-D 1/-D 2) is not part of this job.", + "device-api.sbatch runs this matrix first, then device-api-tests.json against", + "the cmake ENABLE_ROCSHMEM_GIN build. Launch path is ${RCCL_TESTS_BIN_DIR}/", + "(absolute; default rccl-tests/build for this matrix).", + "bench_args: common bench flags | base_env: -x on every run", + "debug_env: -x added only when RCCL_CI_DEBUG=1", + " {LOGDIR} in debug_env is replaced by the runner with the per-job debug dir,", + " so NCCL_DEBUG_FILE writes per-rank logs there instead of flooding stdout.", + "suites: [ { name, bins:[...], args, env:[...] (optional) } ]" + ], + + "bench_args": "-b 8 -e 1G -f 2 -g 1", + + "base_env": ["NCCL_CUMEM_ENABLE=1", "HSA_FORCE_FINE_GRAIN_PCIE=1"], + + "debug_env": ["NCCL_DEBUG=INFO", "NCCL_DEBUG_SUBSYS=INIT,NET", "NCCL_DEBUG_FILE={LOGDIR}/nccl-debug.%h.%p.log"], + + "suites": [ + { + "name": "symmetric", + "bins": ["broadcast_perf", "reduce_perf", "sendrecv_perf", "scatter_perf", "gather_perf"], + "args": "-R 2" + } + ] +} diff --git a/projects/rccl/tools/ci/lib/device-api-tests.json b/projects/rccl/tools/ci/lib/device-api-tests.json index 721fcc14b9d3..da3916a747a6 100644 --- a/projects/rccl/tools/ci/lib/device-api-tests.json +++ b/projects/rccl/tools/ci/lib/device-api-tests.json @@ -1,8 +1,11 @@ { "_comment": [ - "Device-API benchmark matrix for run-device-api-ci.sh.", + "GIN device-API benchmark matrix for run-device-api-ci.sh.", "Add a mode by appending a suites entry. Each suite runs every bin as:", - " mpirun -np $NP -x LD_LIBRARY_PATH ./build/ $bench_args ", + " mpirun -np $NP -x LD_LIBRARY_PATH ${RCCL_TESTS_BIN_DIR}/ $bench_args ", + "RCCL_TESTS_BIN_DIR is the absolute cmake build dir (.ci-out/rccl-tests-build", + "for this job). The non-GIN symmetric matrix is device-api-nongin-tests.json,", + "run against the ENABLE_DEVICE_API=1 ./install.sh build.", "After the matrix, run-device-api-ci.sh runs pytest test/test_AllToAllDevtime.py", "(GIN -D 3 device-timing smoke: -B 1/2 and -B 2 -H 1; ENABLE_DEVICE_API build).", "bench_args: common bench flags | base_env: -x on every run", @@ -20,27 +23,18 @@ "debug_env": ["NCCL_DEBUG=INFO", "NCCL_DEBUG_SUBSYS=INIT,NET", "NCCL_DEBUG_FILE={LOGDIR}/nccl-debug.%h.%p.log"], "suites": [ - { - "name": "symmetric", - "bins": ["broadcast_perf", "reduce_perf", "sendrecv_perf", "scatter_perf", "gather_perf"], - "args": "-R 2" - }, - { - "name": "lsa-d1", - "bins": ["all_reduce_perf"], - "args": "-R 2 -D 1" - }, - { - "name": "lsa-d2", - "bins": ["all_reduce_perf"], - "args": "-R 2 -D 2" - }, { "name": "gin-d3", "bins": ["alltoall_perf"], "args": "-R 2 -D 3", "env": ["NCCL_DMABUF_ENABLE=1", "NCCL_GIN_TYPE=2", "HSA_NO_SCRATCH_RECLAIM=1", "NCCL_ENV_PLUGIN=none", "RCCL_ENABLE_INTRANET=1"] }, + { + "name": "gin-d3-allgather", + "bins": ["all_gather_perf"], + "args": "-R 2 -D 3", + "env": ["NCCL_DMABUF_ENABLE=1", "NCCL_GIN_ENABLE=1", "NCCL_GIN_TYPE=7", "HSA_NO_SCRATCH_RECLAIM=1", "NCCL_ENV_PLUGIN=none", "RCCL_ENABLE_INTRANET=1"] + }, { "name": "gin-d4", "bins": ["alltoall_perf"], diff --git a/projects/rccl/tools/ci/lib/device-api.sbatch b/projects/rccl/tools/ci/lib/device-api.sbatch index 81549ecd1883..516955c42dad 100755 --- a/projects/rccl/tools/ci/lib/device-api.sbatch +++ b/projects/rccl/tools/ci/lib/device-api.sbatch @@ -4,7 +4,7 @@ #SBATCH --ntasks=8 #SBATCH --partition=interactive #SBATCH --exclusive -#SBATCH --time=02:00:00 +#SBATCH --time=04:00:00 #SBATCH --output=%x-%j.out #SBATCH --error=%x-%j.err #SBATCH --propagate=NONE @@ -15,9 +15,13 @@ # On one 8-GPU node: # 1. consume ROCm (provisioned on the head node) via .ci-out/rocm.env # 2. lib/build-ompi.sh build OpenMPI (--with-rocm) -> ompi.env -# 3. projects/rccl/install.sh build RCCL (in-tree, release) -# 4. rccl-tests install.sh build rccl-tests (ENABLE_DEVICE_API=1) -# 5. run-device-api-ci.sh run the device-API benchmark suite +# 3. lib/build-rocshmem.sh build rocSHMEM -> rocshmem.env +# 4. projects/rccl/install.sh build RCCL (in-tree, release, --rocshmem-gin) +# 5. rccl-tests (install.sh) ENABLE_DEVICE_API=1, no ENABLE_ROCSHMEM_GIN +# (symmetric matrix; LSA all_reduce is not run) +# 6. rccl-tests (cmake) ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN +# (GIN -D 3/-D 4 + allgather Anvil SDMA) +# 7. run-device-api-ci.sh non-GIN matrix, then GIN matrix # # Required env (from the workflow via `sbatch --export=ALL,...`): # RCCL_DEVICE_API_WORKDIR GHA workspace (rocm-systems root) @@ -58,11 +62,13 @@ fi export RCCL_BUILD_JOBS build_jobs="${RCCL_BUILD_JOBS}" -# --- build phase -> build-%j.{out,err} --- -{ +# --- build phase -> build-%j.{out,err}. Runs in a subshell so any failure (a +# hard `exit` or a set -e abort) is caught below and skips the test phase. --- +build_rc=0 +( # 1) ROCm: provisioned on the head node from TheRock CI artifacts; consume the # workspace tree via rocm.env. -echo "==> [1/5] use ROCm (provisioned from TheRock artifacts on head node)" +echo "==> [1/7] use ROCm (provisioned from TheRock artifacts on head node)" rocm_env="${workdir}/.ci-out/rocm.env" if [[ ! -f "${rocm_env}" ]]; then echo "ERROR: ${rocm_env} not found. The workflow must run the 'Provision ROCm" >&2 @@ -83,7 +89,7 @@ echo " hipcc: $("${ROCM_PATH}/bin/hipcc" --version 2>/dev/null | head -n1)" # 2) OpenMPI: built here against ROCM_PATH into the per-job workspace cache # (no shared cache), then ompi.env (MPI_HOME) is sourced below. -echo "==> [2/5] build OpenMPI" +echo "==> [2/7] build OpenMPI" "${ci_dir}/lib/build-ompi.sh" set -a # shellcheck source=/dev/null # runtime fragment written by build-ompi.sh @@ -108,31 +114,168 @@ echo "==> verified hipcc = ${resolved_hipcc}" # shellcheck source=/dev/null source "${ci_dir}/lib/ensure-python-yaml.sh" -# 3) RCCL via its own install.sh (-l = local gpu arch only). -echo "==> [3/5] build RCCL (targets=${amdgpu_targets})" -( cd "${workdir}/projects/rccl" \ +# 3) rocSHMEM (in-tree, IPC+SDMA) -> rocshmem.env (ROCSHMEM_INSTALL_DIR) +echo "==> [3/7] build rocSHMEM" +GPU_TARGETS="${amdgpu_targets}" "${ci_dir}/lib/build-rocshmem.sh" +set -a +# shellcheck source=/dev/null +source "${workdir}/.ci-out/rocshmem.env" +set +a + +# 4) RCCL with GIN / rocSHMEM (device-API + GIN-SDMA -D 3 paths need rocshmem in the executable). +# --prefix is required: without an install tree, lib/cmake/rccl/*config.cmake is +# never written and the rccl_cfg check below fails the job before any test. +echo "==> [4/7] build RCCL (GIN, targets=${amdgpu_targets})" +rccl_prefix="${workdir}/.ci-out/rccl-install" +rm -rf "${rccl_prefix}" +( + cd "${workdir}/projects/rccl" \ && MPI_PATH="${MPI_HOME}" ./install.sh -lj "${build_jobs}" \ - --amdgpu_targets="${amdgpu_targets}" ) -rccl_prefix="${workdir}/projects/rccl/build/release" - -# 4) rccl-tests: built in-tree from projects/rccl-tests (must match -# RCCL_TESTS_DIR in run-device-api-ci.sh). install.sh wipes its own build/. -echo "==> [4/5] build rccl-tests (ENABLE_DEVICE_API=1)" -rccl_tests_dir="${workdir}/projects/rccl-tests" -( cd "${rccl_tests_dir}" \ - && ENABLE_DEVICE_API=1 ./install.sh \ - --mpi --mpi_home="${MPI_HOME}" --rccl_home="${rccl_prefix}" \ - --gpu_targets="${amdgpu_targets}" ) -} > "${build_out}" 2> "${build_err}" + --amdgpu_targets="${amdgpu_targets}" \ + --prefix="${rccl_prefix}" \ + --rocshmem-gin --no_clean \ + --cmake-options "-DROCSHMEM_INSTALL_DIR=${ROCSHMEM_INSTALL_DIR}" +) +if [[ ! -f "${rccl_prefix}/lib/librccl.so" && ! -f "${rccl_prefix}/lib/librccl.so.1" ]]; then + echo "ERROR: no librccl.so under ${rccl_prefix}/lib" >&2 + ls -la "${rccl_prefix}/lib" 2>/dev/null >&2 || true + exit 1 +fi +rccl_cfg="$(ls "${rccl_prefix}"/lib*/cmake/rccl/*[Cc]onfig.cmake 2>/dev/null | head -n1 || true)" +if [[ -z "${rccl_cfg}" ]]; then + echo "ERROR: no RCCL cmake config under ${rccl_prefix}/lib*/cmake/rccl" >&2 + ls -laR "${rccl_prefix}"/lib*/cmake 2>/dev/null >&2 || true + exit 1 +fi +rccl_cmake_dir="$(dirname "${rccl_cfg}")" +echo " in-tree RCCL cmake config: ${rccl_cmake_dir}" + +# 5) rccl-tests via install.sh with ENABLE_DEVICE_API only. This is the +# device-API-without-rocSHMEM build. LSA all_reduce_perf (-D 1/-D 2) is not +# part of this job. install.sh always writes into projects/rccl-tests/build. +echo "==> [5/7] build rccl-tests (ENABLE_DEVICE_API via install.sh)" +rccl_tests_src="${workdir}/projects/rccl-tests" +rccl_tests_nongin_build="${rccl_tests_src}/build" +( + cd "${rccl_tests_src}" \ + && ./install.sh --mpi --enable-device-api \ + --mpi_home="${MPI_HOME}" --rccl_home="${rccl_prefix}" \ + --rocm_home="${ROCM_PATH}" \ + --gpu_targets="${amdgpu_targets}" +) +# Symmetric suites in device-api-nongin-tests.json. all_reduce_perf is not required. +required_nongin_bins=( + broadcast_perf reduce_perf sendrecv_perf scatter_perf gather_perf +) +for perf_bin in "${required_nongin_bins[@]}"; do + if [[ ! -x "${rccl_tests_nongin_build}/${perf_bin}" ]]; then + echo "ERROR: ${perf_bin} not built under ${rccl_tests_nongin_build}" >&2 + ls -la "${rccl_tests_nongin_build}" 2>/dev/null >&2 || true + exit 1 + fi +done + +# 6) rccl-tests via cmake so every GIN matrix binary RDC-links rocshmem +# (install.sh/Makefile path does not enable ENABLE_ROCSHMEM_GIN). Needed because +# this job's librccl.so is --rocshmem-gin. RCCL_TESTS_BIN_DIR is consumed by +# run-device-api-ci.sh for the GIN matrix. +echo "==> [6/7] build rccl-tests (ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN)" +rccl_tests_build="${workdir}/.ci-out/rccl-tests-build" +rm -rf "${rccl_tests_build}"; mkdir -p "${rccl_tests_build}" +( + cd "${rccl_tests_build}" + cmake -DCMAKE_BUILD_TYPE=Release -DUSE_MPI=ON \ + -DENABLE_DEVICE_API=ON -DENABLE_ROCSHMEM_GIN=ON \ + -DROCSHMEM_INSTALL_DIR="${ROCSHMEM_INSTALL_DIR}" \ + -DRCCL_DIR="${rccl_cmake_dir}" \ + -DCMAKE_PREFIX_PATH="${rccl_prefix};${MPI_HOME}" \ + -DGPU_TARGETS="${amdgpu_targets}" \ + "${rccl_tests_src}" + # -k: all_reduce_perf still fails under ENABLE_ROCSHMEM_GIN (see gin.sbatch). + # This configure also builds gin_rocshmem_gda_factory_test and the test/ + # host unit target. required_perf_bins below does not check those, so a + # green job only proves the seven matrix binaries exist. + make -j"${build_jobs}" -k || true +) +# Same seven binaries as foreach(_gin_rdc_bin) in +# projects/rccl-tests/src/CMakeLists.txt and the union of bins in +# lib/device-api-tests.json. A suite bin missing from that foreach still +# builds and passes this -x check, then fails at load on an unresolved +# rocSHMEM host symbol. +required_perf_bins=( + all_gather_perf alltoall_perf broadcast_perf + gather_perf reduce_perf scatter_perf sendrecv_perf +) +for perf_bin in "${required_perf_bins[@]}"; do + if [[ ! -x "${rccl_tests_build}/${perf_bin}" ]]; then + echo "ERROR: ${perf_bin} not built under ${rccl_tests_build}" >&2 + ls -la "${rccl_tests_build}" 2>/dev/null >&2 || true + exit 1 + fi +done +# Persist for a standalone run-device-api-ci.sh (the in-process export above +# does not survive a new shell). Mirrors gin.sbatch's .ci-out/rccl.env. +{ + printf 'export RCCL_INSTALL_PREFIX=%q\n' "${rccl_prefix}" + printf 'export RCCL_TESTS_BIN_DIR=%q\n' "${rccl_tests_build}" + printf 'export RCCL_TESTS_NONGIN_BIN_DIR=%q\n' "${rccl_tests_nongin_build}" + printf 'export RCCL_TESTS_DIR=%q\n' "${rccl_tests_src}" +} > "${workdir}/.ci-out/rccl.env" +) > "${build_out}" 2> "${build_err}" || build_rc=$? echo "==> build phase logs: ${build_out} / ${build_err}" +if [[ "${build_rc}" -ne 0 ]]; then + echo "ERROR: build phase failed (rc=${build_rc}); skipping device-api tests." >&2 + echo "----- last 60 lines of ${build_err} -----" >&2 + tail -n 60 "${build_err}" >&2 || true + exit "${build_rc}" +fi + # --- test phase -> test-%j.{out,err} --- +# Source build fragments so ROCM_PATH / MPI_HOME / ROCSHMEM / RCCL paths are set +# even though the build ran in a subshell. +set -a +# shellcheck source=/dev/null +source "${workdir}/.ci-out/rocm.env" +# shellcheck source=/dev/null +source "${workdir}/.ci-out/ompi.env" +# shellcheck source=/dev/null +source "${workdir}/.ci-out/rocshmem.env" +# shellcheck source=/dev/null +source "${workdir}/.ci-out/rccl.env" +set +a + +nongin_rc=0 +gin_rc=0 { -# 5) Benchmarks. -echo "==> [5/5] run benchmarks" -# GIN/RoCE QP creation pins memory; a low memlock limit makes ibv_create_qp fail with ENOMEM. Raise it for the mpirun ranks and log it. +# 7) Benchmarks: symmetric against the install.sh build, then GIN against +# the cmake ENABLE_ROCSHMEM_GIN build. Skip the GIN AllToAllDevtime smoke on the +# non-GIN pass; the GIN pass runs it. LSA all_reduce (-D 1/-D 2) is not run. +# Capture each matrix with its own || rc=$?: set -e is ignored on the left of || +# inside a compound command, so a single trailing || on the brace group would +# keep only the second matrix's status. +echo "==> [7/7] run benchmarks" ulimit -l unlimited 2>/dev/null || echo "WARN: could not raise RLIMIT_MEMLOCK (ulimit -l)" echo "==> RLIMIT_MEMLOCK (ulimit -l) = $(ulimit -l)" -"${ci_dir}/run-device-api-ci.sh" + +echo "==> non-GIN symmetric matrix (install.sh ENABLE_DEVICE_API build)" +CONFIG="${ci_dir}/lib/device-api-nongin-tests.json" \ +RCCL_TESTS_BIN_DIR="${RCCL_TESTS_NONGIN_BIN_DIR}" \ +RCCL_CI_SKIP_DEVTIME=1 \ + "${ci_dir}/run-device-api-ci.sh" || nongin_rc=$? + +echo "==> GIN matrix (cmake ENABLE_ROCSHMEM_GIN build)" +CONFIG="${ci_dir}/lib/device-api-tests.json" \ +RCCL_TESTS_BIN_DIR="${RCCL_TESTS_BIN_DIR}" \ + "${ci_dir}/run-device-api-ci.sh" || gin_rc=$? } > "${test_out}" 2> "${test_err}" echo "==> test phase logs: ${test_out} / ${test_err}" +if [[ "${nongin_rc}" -ne 0 || "${gin_rc}" -ne 0 ]]; then + echo "ERROR: device-api tests failed (nongin_rc=${nongin_rc} gin_rc=${gin_rc})" >&2 + echo "----- last 60 lines of ${test_err} -----" >&2 + tail -n 60 "${test_err}" >&2 || true + if [[ "${nongin_rc}" -ne 0 ]]; then + exit "${nongin_rc}" + fi + exit "${gin_rc}" +fi diff --git a/projects/rccl/tools/ci/run-device-api-ci.sh b/projects/rccl/tools/ci/run-device-api-ci.sh index aae310f913d8..361af8bddf75 100755 --- a/projects/rccl/tools/ci/run-device-api-ci.sh +++ b/projects/rccl/tools/ci/run-device-api-ci.sh @@ -1,23 +1,36 @@ #!/usr/bin/env bash -# Run the RCCL device-API benchmark suite (symmetric memory / LSA device kernels -# / GIN proxy) against the freshly built rccl-tests tree. Mirrors the legacy -# rocJenkins "device-api" testCommand. +# Run the RCCL device-API benchmark suite (symmetric memory / GIN proxy) +# against a freshly built rccl-tests tree. Mirrors the legacy rocJenkins +# "device-api" testCommand. LSA all_reduce_perf (-D 1/-D 2) is not launched. # -# Consumes ROCM_PATH (rocm.env), MPI_HOME (ompi.env), the in-tree RCCL build at -# projects/rccl/build/release, and rccl-tests/build. ROCM_PATH / MPI_HOME come -# from the environment (device-api.sbatch exports them) or, when run standalone, -# from $WORKDIR/.ci-out/{rocm,ompi}.env. +# Consumes ROCM_PATH (rocm.env), MPI_HOME (ompi.env), ROCSHMEM_INSTALL_DIR +# (rocshmem.env), and rccl.env (RCCL_INSTALL_PREFIX, RCCL_TESTS_BIN_DIR, +# RCCL_TESTS_DIR). Build paths come from the environment (device-api.sbatch +# exports them) or, when run standalone, from +# $WORKDIR/.ci-out/{rocm,ompi,rocshmem,rccl}.env. +# librccl.so comes from RCCL_INSTALL_PREFIX/lib when that is set, else +# projects/rccl/build/release. RCCL_TESTS_BIN_DIR defaults to rccl-tests/build. +# +# device-api.sbatch invokes this twice: +# 1. CONFIG=device-api-nongin-tests.json against the install.sh ENABLE_DEVICE_API +# build (symmetric only; no LSA), with RCCL_CI_SKIP_DEVTIME=1 +# 2. CONFIG=device-api-tests.json against the cmake ENABLE_ROCSHMEM_GIN build +# (GIN suites + AllToAllDevtime smoke) # # Each bench is wrapped in `timeout` so a hung mpirun/driver can't wedge the job; # failures are collected and surfaced at the end (exit non-zero iff any failed). # After the JSON matrix, runs pytest test/test_AllToAllDevtime.py (GIN AllToAll -# in-kernel device-timing smoke: modes -B 1, -B 2, -B 2 -H 1). +# in-kernel device-timing smoke: modes -B 1, -B 2, -B 2 -H 1) unless +# RCCL_CI_SKIP_DEVTIME=1. # -# Test matrix lives in lib/device-api-tests.json; RCCL_CI_DEBUG=1 adds its -# debug_env to every run. +# Test matrix lives in lib/device-api-tests.json by default; RCCL_CI_DEBUG=1 +# adds its debug_env to every run. # # Environment overrides: -# ROCM_PATH / MPI_HOME Else read from .ci-out/{rocm,ompi}.env +# ROCM_PATH / MPI_HOME / ROCSHMEM_INSTALL_DIR / RCCL_INSTALL_PREFIX +# Else read from matching .ci-out/*.env fragments +# RCCL_TESTS_BIN_DIR rccl-tests build directory (default: rccl-tests/build; +# device-api.sbatch writes it to .ci-out/rccl.env) # NP MPI ranks per run (default: 8) # BENCH_ARGS Common bench args (default: from JSON bench_args) # BENCH_TIMEOUT Per-bench wall-clock cap (default: 600s) @@ -26,6 +39,7 @@ # RCCL_CI_DEBUG Set to 1 to add debug_env to every run # RCCL_CI_DEBUG_DIR Dir for NCCL_DEBUG_FILE output when RCCL_CI_DEBUG=1 # (default: ${SLURM_SUBMIT_DIR:-$PWD}/nccl-debug) +# RCCL_CI_SKIP_DEVTIME Set to 1 to skip test_AllToAllDevtime.py set -euo pipefail @@ -37,15 +51,29 @@ script_dir="$(cd "$(dirname "$0")" && pwd)" RCCL_DIR="$(cd "${script_dir}/../.." && pwd)" WORKDIR="$(cd "${RCCL_DIR}/../.." && pwd)" -RCCL_LIB_DIR="${WORKDIR}/projects/rccl/build/release" RCCL_TESTS_DIR="${WORKDIR}/projects/rccl-tests" CONFIG="${CONFIG:-${script_dir}/lib/device-api-tests.json}" -# Prefer the build stages' env fragments over any ambient ROCM_PATH / MPI_HOME. -# shellcheck source=/dev/null # runtime fragment written by the provision step -[[ -f "${WORKDIR}/.ci-out/rocm.env" ]] && source "${WORKDIR}/.ci-out/rocm.env" -# shellcheck source=/dev/null # runtime fragment written by build-ompi.sh -[[ -f "${WORKDIR}/.ci-out/ompi.env" ]] && source "${WORKDIR}/.ci-out/ompi.env" +# Prefer the build stages' env fragments over ambient build paths. +# rccl.env is written by device-api.sbatch (RCCL_INSTALL_PREFIX, RCCL_TESTS_BIN_DIR). +# Keep an already-exported RCCL_TESTS_BIN_DIR: device-api.sbatch sets it per +# matrix (install.sh non-GIN build vs cmake GIN build) and rccl.env would clobber it. +_saved_tests_bin_dir="${RCCL_TESTS_BIN_DIR-}" +for frag in rocm ompi rocshmem rccl; do + env_file="${WORKDIR}/.ci-out/${frag}.env" + # shellcheck source=/dev/null # runtime fragment written by the build stages + [[ -f "${env_file}" ]] && source "${env_file}" +done +if [[ -n "${_saved_tests_bin_dir}" ]]; then + RCCL_TESTS_BIN_DIR="${_saved_tests_bin_dir}" +fi +unset _saved_tests_bin_dir + +if [[ -n "${RCCL_INSTALL_PREFIX:-}" ]]; then + RCCL_LIB_DIR="${RCCL_INSTALL_PREFIX}/lib" +else + RCCL_LIB_DIR="${WORKDIR}/projects/rccl/build/release" +fi : "${ROCM_PATH:?run-device-api-ci.sh: ROCM_PATH unset (provisioned via rocm.env / sbatch)}" : "${MPI_HOME:?run-device-api-ci.sh: MPI_HOME unset (run build-ompi.sh / via sbatch)}" @@ -72,14 +100,22 @@ fi cd "${RCCL_TESTS_DIR}" export PATH="${MPI_HOME}/bin:${ROCM_PATH}/bin:${PATH}" -export LD_LIBRARY_PATH="${RCCL_LIB_DIR}:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}" +export LD_LIBRARY_PATH="${RCCL_LIB_DIR}${ROCSHMEM_INSTALL_DIR:+:${ROCSHMEM_INSTALL_DIR}/lib}:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}" -PERF_DIR=build -if [[ ! -d "${PERF_DIR}" || ! -f "${PERF_DIR}/all_reduce_perf" ]]; then +PERF_DIR="${RCCL_TESTS_BIN_DIR:-${RCCL_TESTS_DIR}/build}" +# alltoall_perf is built for both matrices. Fall back to broadcast_perf, which +# the non-GIN symmetric matrix always launches. +if [[ ! -d "${PERF_DIR}" ]]; then echo "rccl-tests perf binaries not found under ${PERF_DIR}" - ls -la + ls -la "${RCCL_TESTS_DIR}" 2>/dev/null || true exit 1 fi +if [[ ! -x "${PERF_DIR}/alltoall_perf" && ! -x "${PERF_DIR}/broadcast_perf" ]]; then + echo "rccl-tests perf binaries not found under ${PERF_DIR}" + ls -la "${PERF_DIR}" 2>/dev/null || true + exit 1 +fi +echo "==> rccl-tests perf dir = ${PERF_DIR}" PARSER="${script_dir}/lib/parse_device_api_config.py" [[ -f "${CONFIG}" ]] || { echo "ERROR: test-matrix config not found: ${CONFIG}" >&2; exit 1; } @@ -136,7 +172,7 @@ run_bench() { set +e timeout --kill-after="${BENCH_KILL_AFTER}" "${BENCH_TIMEOUT}" \ mpirun -np "${NP}" ${env_flags} -x LD_LIBRARY_PATH \ - "./${PERF_DIR}/${bin}" ${BENCH_ARGS} ${extra_args} + "${PERF_DIR}/${bin}" ${BENCH_ARGS} ${extra_args} local rc=$? set -e if [[ ${rc} -ne 0 ]]; then @@ -171,11 +207,16 @@ fi echo "All device-api benchmark runs succeeded." +if [[ -n "${RCCL_CI_SKIP_DEVTIME:-}" ]]; then + echo "==> skip AllToAllDevtime smoke (RCCL_CI_SKIP_DEVTIME=${RCCL_CI_SKIP_DEVTIME})" + exit 0 +fi + # GIN AllToAll in-kernel device-timing smoke (wall_clock64 / *TimedKernel). # Requires ENABLE_DEVICE_API=ON rccl-tests build (alltoall_perf with -B flags). run_devtime_smoke() { local pytest_dir="${RCCL_TESTS_DIR}/test" - local perf_bin="${RCCL_TESTS_DIR}/${PERF_DIR}/alltoall_perf" + local perf_bin="${PERF_DIR}/alltoall_perf" if [[ ! -x "${perf_bin}" ]]; then echo "WARN: skip devtime smoke: ${perf_bin} not found" return 0