Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,7 @@ jobs:
secrets: inherit

integration-test:
needs: [build, build-wheel-cu13, check-paths]
needs: [build, build-wheel-cu13, build-wheel-rocm, check-paths]
if: needs.check-paths.outputs.should-run-downstream == 'true'
uses: ./.github/workflows/integration-test.yml
secrets: inherit
Expand Down
12 changes: 9 additions & 3 deletions .github/workflows/ci_rocm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,16 @@ name: 'Build Wheel (ROCm)'
# ROCm/HIP CI parity with the standard CUDA wheel CI: build the AMD ROCm wheel
# on PRs so packaging regressions are caught. Runs inside the ROCm dev image so
# hipcc / HIP headers / hsa-runtime are available; no GPU is needed to compile.
# On-device transfer tests (test_transfer_on_hip.py) require AMD hardware and
# run outside GitHub-hosted runners.
# On-device external-PD tests consume these artifacts later on the trusted
# self-hosted ROCm runner in integration-test.yml.

on:
workflow_call: {}
workflow_call:
inputs:
checkout_ref:
description: 'Git revision to build (used by trusted E2E dispatches)'
required: false
type: string

jobs:
build-wheel-rocm:
Expand Down Expand Up @@ -39,6 +44,7 @@ jobs:

- uses: actions/checkout@v4
with:
ref: ${{ inputs.checkout_ref || github.sha }}
persist-credentials: false
submodules: recursive

Expand Down
11 changes: 11 additions & 0 deletions .github/workflows/e2e-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,20 @@ jobs:
checkout_ref: ${{ inputs.pr_sha || github.event.pull_request.head.sha }}
secrets: inherit

build-wheel-rocm:
if: >
github.event_name == 'workflow_dispatch' ||
github.event.label.name == 'run-e2e-ci'
uses: ./.github/workflows/ci_rocm.yml
with:
checkout_ref: ${{ inputs.pr_sha || github.event.pull_request.head.sha }}

integration-test:
if: >
github.event_name == 'workflow_dispatch' ||
github.event.label.name == 'run-e2e-ci'
needs:
- build-wheel-rocm
uses: ./.github/workflows/integration-test.yml
with:
pr_sha: ${{ inputs.pr_sha || github.event.pull_request.head.sha }}
Expand All @@ -55,6 +65,7 @@ jobs:
github.event.label.name == 'run-e2e-ci')
needs:
- ascend-test
- build-wheel-rocm
- integration-test
runs-on: ubuntu-latest
steps:
Expand Down
115 changes: 115 additions & 0 deletions .github/workflows/integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -149,3 +149,118 @@ jobs:
sleep 30
done
shell: bash

test-rocm-integration:
name: ROCm MI350X external PD
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main') ||
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'pull_request_target' &&
github.event.action == 'labeled' &&
github.event.label.name == 'run-e2e-ci')
runs-on: [self-hosted, Linux, X64, amd, rocm, gfx950, mooncake-pd]
timeout-minutes: 390
concurrency:
group: mooncake-rocm-mi350x
cancel-in-progress: false
steps:
# pull_request_target keeps this workflow definition on the trusted base
# branch. A maintainer must apply run-e2e-ci before PR code can reach the
# persistent GPU hosts.
- name: Check out tested revision
uses: actions/checkout@v4
with:
ref: ${{ inputs.pr_sha || github.sha }}
persist-credentials: false
clean: true

- name: Download Python 3.10 ROCm wheel
uses: actions/download-artifact@v4
with:
name: mooncake-wheel-rocm-ubuntu-py310
path: ${{ runner.temp }}/mooncake-wheel-rocm-py310

- name: Download Python 3.12 ROCm wheel
uses: actions/download-artifact@v4
with:
name: mooncake-wheel-rocm-ubuntu-py312
path: ${{ runner.temp }}/mooncake-wheel-rocm-py312

- name: Run two-node ROCm external PD suite
shell: bash
env:
WHEEL_DIR_SGLANG: ${{ runner.temp }}/mooncake-wheel-rocm-py310
WHEEL_DIR_VLLM: ${{ runner.temp }}/mooncake-wheel-rocm-py312
run: |
set -euo pipefail
[ "$(id -u)" -ne 0 ] || {
echo "The self-hosted runner must not run as root" >&2
exit 1
}
[ -r /etc/mooncake-ci/runner.env ] || {
echo "Missing controller profile: /etc/mooncake-ci/runner.env" >&2
exit 1
}
set -a
# shellcheck disable=SC1091
source /etc/mooncake-ci/runner.env
set +a
export WHEEL_DIR_SGLANG WHEEL_DIR_VLLM

required_vars=(
CI_ACCELERATOR LOCAL_IP REMOTE_IP REMOTE_SSH_TARGET
MODEL_CACHE MOONCAKE_CI_TIER MOONCAKE_GPU_INDICES
MOONCAKE_RENDER_DEVICES MOONCAKE_CPUSET_CPUS
MOONCAKE_CPUSET_MEMS MOONCAKE_SSH_CONFIG
MOONCAKE_LOCK_FILE REMOTE_TEST_DIR MOONCAKE_RDMA_DEVICES
MOONCAKE_RDMA_NETDEVS MOONCAKE_GID_INDEX
)
for variable in "${required_vars[@]}"; do
[ -n "${!variable:-}" ] || {
echo "Missing controller setting: $variable" >&2
exit 1
}
done
[ "$CI_ACCELERATOR" = rocm ]
export SSH_CMD="ssh -F ${MOONCAKE_SSH_CONFIG}"

for command in docker flock rocm-smi rsync ssh; do
command -v "$command" >/dev/null || {
echo "Missing required command: $command" >&2
exit 1
}
done
test -c /dev/kfd
ssh -F "${MOONCAKE_SSH_CONFIG}" "${REMOTE_SSH_TARGET}" true
[ -w "$(dirname "$MOONCAKE_LOCK_FILE")" ] || {
echo "Runner cannot write the Mooncake lock directory" >&2
exit 1
}

exec 9>"${MOONCAKE_LOCK_FILE}"
flock -n 9 || {
echo "The MI350X Mooncake allocation is already in use" >&2
exit 1
}

cleanup() {
if [ -f scripts/tone_tests/run/.shrc ]; then
set +e
# shellcheck disable=SC1091
source scripts/tone_tests/run/.shrc
# shellcheck disable=SC1091
source scripts/tone_tests/scripts/common.sh
cleanup_test_env double
fi
}
trap cleanup EXIT INT TERM
bash scripts/tone_tests/scripts/run_test.sh run-all

- name: Upload ROCm external PD logs
if: always()
uses: actions/upload-artifact@v4
with:
name: rocm-external-pd-logs-${{ github.run_id }}-${{ github.run_attempt }}
path: scripts/tone_tests/run/logs
if-no-files-found: warn
retention-days: 14
19 changes: 19 additions & 0 deletions scripts/tone_tests/rocm_runner.env.example
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Host-owned profile for the controller-only ROCm self-hosted runner.
# Install as /etc/mooncake-ci/runner.env; do not store credentials here.
CI_ACCELERATOR=rocm
MOONCAKE_CI_TIER=core-4gpu
LOCAL_IP=192.168.210.1
REMOTE_IP=192.168.210.2
REMOTE_SSH_TARGET=mooncake-worker
REMOTE_TEST_DIR=/var/lib/mooncake-ci/work
MODEL_CACHE=/var/lib/mooncake-ci/cache
MOONCAKE_RENDER_DEVICES="/dev/dri/renderD129 /dev/dri/renderD137 /dev/dri/renderD145 /dev/dri/renderD153"
MOONCAKE_GPU_INDICES=0,1,2,3
MOONCAKE_CPUSET_CPUS=0-95
MOONCAKE_CPUSET_MEMS=0
MOONCAKE_RDMA_DEVICES=ionic_0,ionic_1,ionic_2,ionic_3
MOONCAKE_RDMA_NETDEVS=eth2,eth3,eth4,eth5
MOONCAKE_GID_INDEX=1
MOONCAKE_SSH_CONFIG=/etc/mooncake-ci/runner_ssh_config
MOONCAKE_LOCK_FILE=/var/lib/mooncake-ci/run/mooncake.lock
CONTAINER_NAME=mooncake-rocm-ci
Loading
Loading