Repository navigation
ci(rccl): build device-api with rocSHMEM GIN #11895
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| { | ||
| "_comment": [ | ||
| "Non-GIN device-API matrix for run-device-api-ci.sh.", | ||
| "Built with ENABLE_DEVICE_API=1 ./install.sh (no ENABLE_ROCSHMEM_GIN).", | ||
| "LSA all_reduce_perf (-D 1/-D 2) is not part of this job.", | ||
| "device-api.sbatch runs this matrix first, then device-api-tests.json against", | ||
| "the cmake ENABLE_ROCSHMEM_GIN build. Launch path is ${RCCL_TESTS_BIN_DIR}/<bin>", | ||
| "(absolute; default rccl-tests/build for this matrix).", | ||
| "bench_args: common bench flags | base_env: -x on every run", | ||
| "debug_env: -x added only when RCCL_CI_DEBUG=1", | ||
| " {LOGDIR} in debug_env is replaced by the runner with the per-job debug dir,", | ||
| " so NCCL_DEBUG_FILE writes per-rank logs there instead of flooding stdout.", | ||
| "suites: [ { name, bins:[...], args, env:[...] (optional) } ]" | ||
| ], | ||
|
|
||
| "bench_args": "-b 8 -e 1G -f 2 -g 1", | ||
|
|
||
| "base_env": ["NCCL_CUMEM_ENABLE=1", "HSA_FORCE_FINE_GRAIN_PCIE=1"], | ||
|
|
||
| "debug_env": ["NCCL_DEBUG=INFO", "NCCL_DEBUG_SUBSYS=INIT,NET", "NCCL_DEBUG_FILE={LOGDIR}/nccl-debug.%h.%p.log"], | ||
|
|
||
| "suites": [ | ||
| { | ||
| "name": "symmetric", | ||
| "bins": ["broadcast_perf", "reduce_perf", "sendrecv_perf", "scatter_perf", "gather_perf"], | ||
| "args": "-R 2" | ||
| } | ||
| ] | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,7 +4,7 @@ | |
| #SBATCH --ntasks=8 | ||
| #SBATCH --partition=interactive | ||
| #SBATCH --exclusive | ||
| #SBATCH --time=04:00:00 | ||
| #SBATCH --output=%x-%j.out | ||
| #SBATCH --error=%x-%j.err | ||
| #SBATCH --propagate=NONE | ||
|
|
@@ -15,9 +15,13 @@ | |
| # On one 8-GPU node: | ||
| # 1. consume ROCm (provisioned on the head node) via .ci-out/rocm.env | ||
| # 2. lib/build-ompi.sh build OpenMPI (--with-rocm) -> ompi.env | ||
| # 3. lib/build-rocshmem.sh build rocSHMEM -> rocshmem.env | ||
| # 4. projects/rccl/install.sh build RCCL (in-tree, release, --rocshmem-gin) | ||
| # 5. rccl-tests (install.sh) ENABLE_DEVICE_API=1, no ENABLE_ROCSHMEM_GIN | ||
| # (symmetric matrix; LSA all_reduce is not run) | ||
| # 6. rccl-tests (cmake) ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN | ||
| # (GIN -D 3/-D 4 + allgather Anvil SDMA) | ||
| # 7. run-device-api-ci.sh non-GIN matrix, then GIN matrix | ||
| # | ||
| # Required env (from the workflow via `sbatch --export=ALL,...`): | ||
| # RCCL_DEVICE_API_WORKDIR GHA workspace (rocm-systems root) | ||
|
|
@@ -58,11 +62,13 @@ fi | |
| export RCCL_BUILD_JOBS | ||
| build_jobs="${RCCL_BUILD_JOBS}" | ||
|
|
||
| # --- build phase -> build-%j.{out,err}. Runs in a subshell so any failure (a | ||
| # hard `exit` or a set -e abort) is caught below and skips the test phase. --- | ||
| build_rc=0 | ||
| ( | ||
| # 1) ROCm: provisioned on the head node from TheRock CI artifacts; consume the | ||
| # workspace tree via rocm.env. | ||
| echo "==> [1/7] use ROCm (provisioned from TheRock artifacts on head node)" | ||
| rocm_env="${workdir}/.ci-out/rocm.env" | ||
| if [[ ! -f "${rocm_env}" ]]; then | ||
| echo "ERROR: ${rocm_env} not found. The workflow must run the 'Provision ROCm" >&2 | ||
|
|
@@ -83,7 +89,7 @@ echo " hipcc: $("${ROCM_PATH}/bin/hipcc" --version 2>/dev/null | head -n1)" | |
|
|
||
| # 2) OpenMPI: built here against ROCM_PATH into the per-job workspace cache | ||
| # (no shared cache), then ompi.env (MPI_HOME) is sourced below. | ||
| echo "==> [2/7] build OpenMPI" | ||
| "${ci_dir}/lib/build-ompi.sh" | ||
| set -a | ||
| # shellcheck source=/dev/null # runtime fragment written by build-ompi.sh | ||
|
|
@@ -108,31 +114,161 @@ echo "==> verified hipcc = ${resolved_hipcc}" | |
| # shellcheck source=/dev/null | ||
| source "${ci_dir}/lib/ensure-python-yaml.sh" | ||
|
|
||
| # 3) rocSHMEM (in-tree, IPC+SDMA) -> rocshmem.env (ROCSHMEM_INSTALL_DIR) | ||
| echo "==> [3/7] build rocSHMEM" | ||
| GPU_TARGETS="${amdgpu_targets}" "${ci_dir}/lib/build-rocshmem.sh" | ||
| set -a | ||
| # shellcheck source=/dev/null | ||
| source "${workdir}/.ci-out/rocshmem.env" | ||
| set +a | ||
|
|
||
| # 4) RCCL with GIN / rocSHMEM (device-API + GIN-SDMA -D 3 paths need rocshmem in the executable). | ||
| # --prefix is required: without an install tree, lib/cmake/rccl/*config.cmake is | ||
| # never written and the rccl_cfg check below fails the job before any test. | ||
| echo "==> [4/7] build RCCL (GIN, targets=${amdgpu_targets})" | ||
| rccl_prefix="${workdir}/.ci-out/rccl-install" | ||
| rm -rf "${rccl_prefix}" | ||
| ( | ||
| cd "${workdir}/projects/rccl" \ | ||
| && MPI_PATH="${MPI_HOME}" ./install.sh -lj "${build_jobs}" \ | ||
| --amdgpu_targets="${amdgpu_targets}" \ | ||
| --prefix="${rccl_prefix}" \ | ||
| --rocshmem-gin --no_clean \ | ||
| --cmake-options "-DROCSHMEM_INSTALL_DIR=${ROCSHMEM_INSTALL_DIR}" | ||
| ) | ||
| if [[ ! -f "${rccl_prefix}/lib/librccl.so" && ! -f "${rccl_prefix}/lib/librccl.so.1" ]]; then | ||
| echo "ERROR: no librccl.so under ${rccl_prefix}/lib" >&2 | ||
| ls -la "${rccl_prefix}/lib" 2>/dev/null >&2 || true | ||
| exit 1 | ||
| fi | ||
| rccl_cfg="$(ls "${rccl_prefix}"/lib*/cmake/rccl/*[Cc]onfig.cmake 2>/dev/null | head -n1 || true)" | ||
| if [[ -z "${rccl_cfg}" ]]; then | ||
| echo "ERROR: no RCCL cmake config under ${rccl_prefix}/lib*/cmake/rccl" >&2 | ||
| ls -laR "${rccl_prefix}"/lib*/cmake 2>/dev/null >&2 || true | ||
| exit 1 | ||
| fi | ||
| rccl_cmake_dir="$(dirname "${rccl_cfg}")" | ||
| echo " in-tree RCCL cmake config: ${rccl_cmake_dir}" | ||
|
|
||
| # 5) rccl-tests via install.sh with ENABLE_DEVICE_API only. This is the | ||
| # device-API-without-rocSHMEM build. LSA all_reduce_perf (-D 1/-D 2) is not | ||
| # part of this job. install.sh always writes into projects/rccl-tests/build. | ||
| echo "==> [5/7] build rccl-tests (ENABLE_DEVICE_API via install.sh)" | ||
| rccl_tests_src="${workdir}/projects/rccl-tests" | ||
| rccl_tests_nongin_build="${rccl_tests_src}/build" | ||
| ( | ||
| cd "${rccl_tests_src}" \ | ||
| && ./install.sh --mpi --enable-device-api \ | ||
| --mpi_home="${MPI_HOME}" --rccl_home="${rccl_prefix}" \ | ||
|
Contributor
There was a problem hiding this comment. High The step 5 build takes
Contributor
Author
There was a problem hiding this comment. Thanks for the flag — checked against the green device-API job on 7c6cc22: https://github.com/ROCm/rocm-systems/actions/runs/36286627742/job/108528513362 Step 5's install.sh binaries load the |
||
| --rocm_home="${ROCM_PATH}" \ | ||
| --gpu_targets="${amdgpu_targets}" | ||
| ) | ||
| # Symmetric suites in device-api-nongin-tests.json. all_reduce_perf is not required. | ||
| required_nongin_bins=( | ||
| broadcast_perf reduce_perf sendrecv_perf scatter_perf gather_perf | ||
| ) | ||
| for perf_bin in "${required_nongin_bins[@]}"; do | ||
| if [[ ! -x "${rccl_tests_nongin_build}/${perf_bin}" ]]; then | ||
| echo "ERROR: ${perf_bin} not built under ${rccl_tests_nongin_build}" >&2 | ||
| ls -la "${rccl_tests_nongin_build}" 2>/dev/null >&2 || true | ||
| exit 1 | ||
| fi | ||
| done | ||
|
|
||
| # 6) rccl-tests via cmake so every GIN matrix binary RDC-links rocshmem | ||
| # (install.sh/Makefile path does not enable ENABLE_ROCSHMEM_GIN). Needed because | ||
|
dlamd1dai marked this conversation as resolved.
Resolved
|
||
| # this job's librccl.so is --rocshmem-gin. RCCL_TESTS_BIN_DIR is consumed by | ||
| # run-device-api-ci.sh for the GIN matrix. | ||
| echo "==> [6/7] build rccl-tests (ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN)" | ||
| rccl_tests_build="${workdir}/.ci-out/rccl-tests-build" | ||
| rm -rf "${rccl_tests_build}"; mkdir -p "${rccl_tests_build}" | ||
| ( | ||
| cd "${rccl_tests_build}" | ||
| cmake -DCMAKE_BUILD_TYPE=Release -DUSE_MPI=ON \ | ||
| -DENABLE_DEVICE_API=ON -DENABLE_ROCSHMEM_GIN=ON \ | ||
| -DROCSHMEM_INSTALL_DIR="${ROCSHMEM_INSTALL_DIR}" \ | ||
| -DRCCL_DIR="${rccl_cmake_dir}" \ | ||
| -DCMAKE_PREFIX_PATH="${rccl_prefix};${MPI_HOME}" \ | ||
| -DGPU_TARGETS="${amdgpu_targets}" \ | ||
| "${rccl_tests_src}" | ||
| # -k: all_reduce_perf still fails under ENABLE_ROCSHMEM_GIN (see gin.sbatch). | ||
| # This configure also builds gin_rocshmem_gda_factory_test and the test/ | ||
| # host unit target. required_perf_bins below does not check those, so a | ||
| # green job only proves the seven matrix binaries exist. | ||
| make -j"${build_jobs}" -k || true | ||
|
dlamd1dai marked this conversation as resolved.
Resolved
|
||
| ) | ||
| # Same seven binaries as foreach(_gin_rdc_bin) in | ||
| # projects/rccl-tests/src/CMakeLists.txt and the union of bins in | ||
| # lib/device-api-tests.json. A suite bin missing from that foreach still | ||
| # builds and passes this -x check, then fails at load on an unresolved | ||
| # rocSHMEM host symbol. | ||
| required_perf_bins=( | ||
|
dlamd1dai marked this conversation as resolved.
Resolved
|
||
| all_gather_perf alltoall_perf broadcast_perf | ||
| gather_perf reduce_perf scatter_perf sendrecv_perf | ||
| ) | ||
| for perf_bin in "${required_perf_bins[@]}"; do | ||
| if [[ ! -x "${rccl_tests_build}/${perf_bin}" ]]; then | ||
| echo "ERROR: ${perf_bin} not built under ${rccl_tests_build}" >&2 | ||
| ls -la "${rccl_tests_build}" 2>/dev/null >&2 || true | ||
| exit 1 | ||
| fi | ||
| done | ||
| # Persist for a standalone run-device-api-ci.sh (the in-process export above | ||
| # does not survive a new shell). Mirrors gin.sbatch's .ci-out/rccl.env. | ||
| { | ||
| printf 'export RCCL_INSTALL_PREFIX=%q\n' "${rccl_prefix}" | ||
| printf 'export RCCL_TESTS_BIN_DIR=%q\n' "${rccl_tests_build}" | ||
| printf 'export RCCL_TESTS_NONGIN_BIN_DIR=%q\n' "${rccl_tests_nongin_build}" | ||
| printf 'export RCCL_TESTS_DIR=%q\n' "${rccl_tests_src}" | ||
| } > "${workdir}/.ci-out/rccl.env" | ||
| ) > "${build_out}" 2> "${build_err}" || build_rc=$? | ||
| echo "==> build phase logs: ${build_out} / ${build_err}" | ||
|
|
||
| if [[ "${build_rc}" -ne 0 ]]; then | ||
| echo "ERROR: build phase failed (rc=${build_rc}); skipping device-api tests." >&2 | ||
| echo "----- last 60 lines of ${build_err} -----" >&2 | ||
| tail -n 60 "${build_err}" >&2 || true | ||
| exit "${build_rc}" | ||
| fi | ||
|
|
||
| # --- test phase -> test-%j.{out,err} --- | ||
| # Source build fragments so ROCM_PATH / MPI_HOME / ROCSHMEM / RCCL paths are set | ||
| # even though the build ran in a subshell. | ||
| set -a | ||
| # shellcheck source=/dev/null | ||
| source "${workdir}/.ci-out/rocm.env" | ||
| # shellcheck source=/dev/null | ||
| source "${workdir}/.ci-out/ompi.env" | ||
| # shellcheck source=/dev/null | ||
| source "${workdir}/.ci-out/rocshmem.env" | ||
| # shellcheck source=/dev/null | ||
| source "${workdir}/.ci-out/rccl.env" | ||
| set +a | ||
|
|
||
| test_rc=0 | ||
| { | ||
| # 7) Benchmarks: symmetric against the install.sh build, then GIN against | ||
| # the cmake ENABLE_ROCSHMEM_GIN build. Skip the GIN AllToAllDevtime smoke on the | ||
| # non-GIN pass; the GIN pass runs it. LSA all_reduce (-D 1/-D 2) is not run. | ||
| echo "==> [7/7] run benchmarks" | ||
| ulimit -l unlimited 2>/dev/null || echo "WARN: could not raise RLIMIT_MEMLOCK (ulimit -l)" | ||
| echo "==> RLIMIT_MEMLOCK (ulimit -l) = $(ulimit -l)" | ||
|
|
||
| echo "==> non-GIN symmetric matrix (install.sh ENABLE_DEVICE_API build)" | ||
| CONFIG="${ci_dir}/lib/device-api-nongin-tests.json" \ | ||
| RCCL_TESTS_BIN_DIR="${RCCL_TESTS_NONGIN_BIN_DIR}" \ | ||
| RCCL_CI_SKIP_DEVTIME=1 \ | ||
| "${ci_dir}/run-device-api-ci.sh" | ||
|
|
||
| echo "==> GIN matrix (cmake ENABLE_ROCSHMEM_GIN build)" | ||
| CONFIG="${ci_dir}/lib/device-api-tests.json" \ | ||
| RCCL_TESTS_BIN_DIR="${RCCL_TESTS_BIN_DIR}" \ | ||
| "${ci_dir}/run-device-api-ci.sh" | ||
| } > "${test_out}" 2> "${test_err}" || test_rc=$? | ||
|
Contributor
There was a problem hiding this comment. High
Contributor
Author
There was a problem hiding this comment. Addressed in 2b6f1bf. Each
Contributor
There was a problem hiding this comment. Resolved by |
||
| echo "==> test phase logs: ${test_out} / ${test_err}" | ||
| if [[ "${test_rc}" -ne 0 ]]; then | ||
| echo "ERROR: device-api tests failed (rc=${test_rc})" >&2 | ||
| echo "----- last 60 lines of ${test_err} -----" >&2 | ||
| tail -n 60 "${test_err}" >&2 || true | ||
| exit "${test_rc}" | ||
| fi | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.