diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6aaf90ab6..5117c325bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -399,6 +399,20 @@ jobs: - name: Test CI service and smoke scripts run: python -m unittest discover -s scripts/ci/tests -v + tone-script-tests: + name: Test external CI lifecycle helpers + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Run focused shell regression tests + run: | + bash scripts/tone_tests/tests/test_common_cleanup.sh + python3 scripts/tone_tests/tests/test_hicache_model_selection.py + shell: bash + clang-format: name: Check code format runs-on: ubuntu-22.04 @@ -750,6 +764,7 @@ jobs: - docs-check - reshard-type-check - ci-script-tests + - tone-script-tests - build-wheel - unit-tests - build-flags @@ -771,7 +786,8 @@ jobs: needs.clang-format.result == 'success' && needs.python-lint.result == 'success' && needs.reshard-type-check.result == 'success' && - needs.ci-script-tests.result == 'success' }} + needs.ci-script-tests.result == 'success' && + needs.tone-script-tests.result == 'success' }} run: | echo "$NEEDS_JSON" test "$PASSED" = true diff --git a/.github/workflows/ci_rocm.yml b/.github/workflows/ci_rocm.yml index c445be1230..0a6bb137f4 100644 --- a/.github/workflows/ci_rocm.yml +++ b/.github/workflows/ci_rocm.yml @@ -3,11 +3,21 @@ name: 'Build Wheel (ROCm)' # ROCm/HIP CI parity with the standard CUDA wheel CI: build the AMD ROCm wheel # on PRs so packaging regressions are caught. Runs inside the ROCm dev image so # hipcc / HIP headers / hsa-runtime are available; no GPU is needed to compile. -# On-device transfer tests (test_transfer_on_hip.py) require AMD hardware and -# run outside GitHub-hosted runners. +# On-device external-PD tests consume these artifacts later on the trusted +# self-hosted ROCm runner in integration-test-rocm.yml. on: - workflow_call: {} + workflow_call: + inputs: + checkout_ref: + description: 'Git revision to build (used by trusted E2E dispatches)' + required: false + type: string + python_versions: + description: 'JSON array of Python versions to build' + required: false + default: '["3.10", "3.12"]' + type: string jobs: build-wheel-rocm: @@ -17,9 +27,10 @@ jobs: # so it also loads on the ROCm 7.0 image variants. container: rocm/dev-ubuntu-22.04:7.2.3-complete strategy: + fail-fast: false matrix: # 3.10 covers the SGLang ROCm image; 3.12 covers vLLM ROCm. - python-version: ['3.10', '3.12'] + python-version: ${{ fromJSON(inputs.python_versions) }} env: HIP_BUILD: "1" SCCACHE_GHA_ENABLED: "true" @@ -37,15 +48,35 @@ jobs: apt-get update -y apt-get install -y --no-install-recommends git ca-certificates - - uses: actions/checkout@v4 + - name: Check out source + uses: actions/checkout@v4 with: + ref: ${{ inputs.checkout_ref || github.sha }} persist-credentials: false - submodules: recursive + submodules: false - name: Mark repository as safe run: git config --global --add safe.directory "$GITHUB_WORKSPACE" || true shell: bash + - name: Check out submodules + shell: bash + run: | + set -eo pipefail + git submodule sync --recursive + for attempt in 1 2 3; do + if git -c http.version=HTTP/1.1 submodule update \ + --init --recursive --depth=1 --jobs=2; then + exit 0 + fi + if [ "$attempt" -eq 3 ]; then + echo "Submodule checkout failed after $attempt attempts" >&2 + exit 1 + fi + echo "Submodule checkout attempt $attempt failed; retrying..." >&2 + sleep $((attempt * 10)) + done + - name: Install toolchain and Python ${{ matrix.python-version }} shell: bash run: | @@ -96,6 +127,7 @@ jobs: rm -rf build && mkdir build && cd build cmake -G Ninja .. \ -DUSE_HIP=ON -DUSE_CUDA=OFF -DWITH_EP=OFF \ + -DENABLE_MULTI_PROTOCOL=ON \ -DUSE_HTTP=ON -DUSE_ETCD=ON -DSTORE_USE_ETCD=ON \ -DBUILD_UNIT_TESTS=OFF -DENABLE_SCCACHE=ON \ -DCMAKE_BUILD_TYPE=Release \ @@ -147,7 +179,8 @@ jobs: mooncake-wheel/dist-rocm-py${{ steps.generate_tag.outputs.python_version_tag }}/*.whl # ROCm runtime is excluded from the wheel and bound at load time. export LD_LIBRARY_PATH="/opt/rocm/lib:/usr/local/lib:${LD_LIBRARY_PATH:-}" - site="$("$smoke_venv/bin/python" -c 'import mooncake,os;print(os.path.dirname(mooncake.__file__))')" + "$smoke_venv/bin/python" scripts/tone_tests/python/verify_rocm_wheel.py + site="$("$smoke_venv/bin/python" -c 'import mooncake, os; print(os.path.dirname(mooncake.__file__))')" "$site/mooncake_master" --version - name: Upload Python wheel artifact diff --git a/.github/workflows/e2e-ci.yml b/.github/workflows/e2e-ci.yml index b6abfee2e8..d3ec8215e3 100644 --- a/.github/workflows/e2e-ci.yml +++ b/.github/workflows/e2e-ci.yml @@ -37,6 +37,14 @@ jobs: checkout_ref: ${{ inputs.pr_sha || github.event.pull_request.head.sha }} secrets: inherit + build-wheel-rocm: + if: > + github.event_name == 'workflow_dispatch' || + github.event.label.name == 'run-e2e-ci' + uses: ./.github/workflows/ci_rocm.yml + with: + checkout_ref: ${{ inputs.pr_sha || github.event.pull_request.head.sha }} + integration-test: if: > github.event_name == 'workflow_dispatch' || @@ -47,6 +55,17 @@ jobs: pr_number: ${{ inputs.pr_number || github.event.pull_request.number }} secrets: inherit + rocm-integration-test: + if: > + github.event_name == 'workflow_dispatch' || + github.event.label.name == 'run-e2e-ci' + needs: + - build-wheel-rocm + uses: ./.github/workflows/integration-test-rocm.yml + with: + checkout_ref: ${{ inputs.pr_sha || github.event.pull_request.head.sha }} + secrets: inherit + e2e-gate: name: E2E Gate if: > @@ -55,7 +74,9 @@ jobs: github.event.label.name == 'run-e2e-ci') needs: - ascend-test + - build-wheel-rocm - integration-test + - rocm-integration-test runs-on: ubuntu-latest steps: - name: Check E2E results diff --git a/.github/workflows/integration-test-rocm.yml b/.github/workflows/integration-test-rocm.yml new file mode 100644 index 0000000000..2dea302e85 --- /dev/null +++ b/.github/workflows/integration-test-rocm.yml @@ -0,0 +1,167 @@ +name: 'Integration test (ROCm)' + +on: + workflow_call: + inputs: + checkout_ref: + description: 'Git revision whose ROCm wheel and integration tests are exercised' + required: false + type: string + run_sglang: + description: 'Run SGLang cases in addition to vLLM' + required: false + default: true + type: boolean + +jobs: + test-rocm-integration: + name: ROCm MI350X external PD + runs-on: [self-hosted, Linux, X64, amd, rocm, gfx950, mooncake-pd] + timeout-minutes: 390 + concurrency: + group: mooncake-rocm-mi350x + cancel-in-progress: false + env: + REGISTRY_ADDR_SGLANG: 'lmsysorg/sglang:v0.5.18-rocm720-mi35x@sha256:6d68cd19206716cb3f1e31e2ad89cd0852d7ae614a792773c30a4277f8955c72' + REGISTRY_ADDR_VLLM: 'vllm/vllm-openai-rocm:v0.21.0@sha256:98a77b20df03adeb1cfc0ced009b4df6dd52b0a994ab99a32421f30876a9ae0c' + steps: + # Pytest previously ran as root in a bind-mounted container and left + # caches that the non-root runner could not remove on the next checkout. + - name: Remove root-owned Python test caches + shell: bash + run: | + set -euo pipefail + docker run --rm --network=none \ + -v "${GITHUB_WORKSPACE}:/workspace" \ + --entrypoint /bin/rm \ + "${REGISTRY_ADDR_VLLM}" \ + -rf \ + /workspace/scripts/tone_tests/python/.pytest_cache \ + /workspace/scripts/tone_tests/python/__pycache__ + + - name: Check out tested revision + uses: actions/checkout@v4 + with: + ref: ${{ inputs.checkout_ref || github.sha }} + persist-credentials: false + clean: true + + - name: Download Python 3.10 ROCm wheel + if: inputs.run_sglang + uses: actions/download-artifact@v4 + with: + name: mooncake-wheel-rocm-ubuntu-py310 + path: ${{ runner.temp }}/mooncake-wheel-rocm-py310 + + - name: Download Python 3.12 ROCm wheel + uses: actions/download-artifact@v4 + with: + name: mooncake-wheel-rocm-ubuntu-py312 + path: ${{ runner.temp }}/mooncake-wheel-rocm-py312 + + - name: Run two-node ROCm external PD suite + shell: bash + env: + WHEEL_DIR_SGLANG: ${{ runner.temp }}/mooncake-wheel-rocm-py310 + WHEEL_DIR_VLLM: ${{ runner.temp }}/mooncake-wheel-rocm-py312 + run: | + set -euo pipefail + [ "$(id -u)" -ne 0 ] || { + echo "The self-hosted runner must not run as root" >&2 + exit 1 + } + [ -r /etc/mooncake-ci/runner.env ] || { + echo "Missing controller profile: /etc/mooncake-ci/runner.env" >&2 + exit 1 + } + set -a + # shellcheck disable=SC1091 + source /etc/mooncake-ci/runner.env + set +a + export REGISTRY_ADDR_SGLANG REGISTRY_ADDR_VLLM + export WHEEL_DIR_SGLANG WHEEL_DIR_VLLM + + required_vars=( + CI_ACCELERATOR LOCAL_IP REMOTE_IP REMOTE_SSH_TARGET + MODEL_CACHE MOONCAKE_RUNTIME_CACHE MOONCAKE_CI_TIER + MOONCAKE_GPU_INDICES + MOONCAKE_RENDER_DEVICES MOONCAKE_CPUSET_CPUS + MOONCAKE_CPUSET_MEMS MOONCAKE_SSH_CONFIG + MOONCAKE_LOCK_FILE REMOTE_TEST_DIR MOONCAKE_RDMA_DEVICES + MOONCAKE_RDMA_NETDEVS MOONCAKE_TRANSFER_DEVICE + MOONCAKE_GID_INDEX MOONCAKE_SGLANG_BASE_GPU_ID + MOONCAKE_EPD_ENCODER_GPU_ID MOONCAKE_EPD_PREFILL_GPU_ID + MOONCAKE_EPD_DECODE_GPU_ID MOONCAKE_VLLM_VISIBLE_DEVICES + MOONCAKE_SGLANG_MEM_FRACTION_STATIC AINIC_VERSION + ) + for variable in "${required_vars[@]}"; do + [ -n "${!variable:-}" ] || { + echo "Missing controller setting: $variable" >&2 + exit 1 + } + done + [ "$CI_ACCELERATOR" = rocm ] + export SSH_CMD="ssh -F ${MOONCAKE_SSH_CONFIG}" + + for command in docker flock python3 realpath rocm-smi rsync ssh; do + command -v "$command" >/dev/null || { + echo "Missing required command: $command" >&2 + exit 1 + } + done + test -c /dev/kfd + test -d /sys/class/kfd/kfd/topology/nodes + test -d /sys/class/kfd/kfd/proc + ssh -F "${MOONCAKE_SSH_CONFIG}" "${REMOTE_SSH_TARGET}" ' + set -eu + for command in docker python3 rocm-smi; do + command -v "$command" >/dev/null || { + echo "Missing required remote command: $command" >&2 + exit 1 + } + done + test -c /dev/kfd + test -d /sys/class/kfd/kfd/topology/nodes + test -d /sys/class/kfd/kfd/proc + ' + [ -w "$(dirname "$MOONCAKE_LOCK_FILE")" ] || { + echo "Runner cannot write the Mooncake lock directory" >&2 + exit 1 + } + + exec 9>"${MOONCAKE_LOCK_FILE}" + flock -n 9 || { + echo "The MI350X Mooncake allocation is already in use" >&2 + exit 1 + } + + cleanup_best_effort() { + local exit_code=$? + trap - EXIT + if [ -f scripts/tone_tests/run/.shrc ]; then + set +e + # shellcheck disable=SC1091 + source scripts/tone_tests/run/.shrc + # shellcheck disable=SC1091 + source scripts/tone_tests/scripts/common.sh + cleanup_test_env double || true + fi + exit "$exit_code" + } + trap cleanup_best_effort EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + if ${{ inputs.run_sglang }}; then + bash scripts/tone_tests/scripts/run_test.sh run-all + else + bash scripts/tone_tests/scripts/run_test.sh run-all VLLM + fi + + - name: Upload ROCm external PD logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: rocm-external-pd-logs-${{ github.run_id }}-${{ github.run_attempt }} + path: scripts/tone_tests/run/logs + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7026a57668..6c410012b5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -93,6 +93,17 @@ jobs: artifact_name: nightly-cuda13-x86-py312 secrets: inherit + build-wheel-rocm: + needs: version-stamp + uses: ./.github/workflows/ci_rocm.yml + + rocm-integration-test: + needs: build-wheel-rocm + uses: ./.github/workflows/integration-test-rocm.yml + with: + checkout_ref: ${{ github.sha }} + secrets: inherit + build-musa: runs-on: ubuntu-22.04 container: registry.mthreads.com/mcconline/inference/pytorch:2.9.1.post1-py3.10-musa5.2.0-mp31-devel-ubuntu22.04-amd64 @@ -575,6 +586,8 @@ jobs: - version-stamp - build-wheels - tone-sglang-integration + - build-wheel-rocm + - rocm-integration-test - build-musa - build-docker - ascend-test diff --git a/.github/workflows/release-rocm.yaml b/.github/workflows/release-rocm.yaml index 060285da36..c7b9329415 100644 --- a/.github/workflows/release-rocm.yaml +++ b/.github/workflows/release-rocm.yaml @@ -56,6 +56,18 @@ jobs: fetch-depth: 0 submodules: recursive + # workflow_dispatch may build a tag older than the verification helper. + # Keep release policy tied to the trusted workflow revision instead of + # relying on files from the historical source tree. + - name: Check out release verification helper + uses: actions/checkout@v4 + with: + ref: ${{ github.workflow_sha }} + path: .release-ci + persist-credentials: false + sparse-checkout: scripts/tone_tests/python/verify_rocm_wheel.py + sparse-checkout-cone-mode: false + - name: Mark repository as safe shell: bash run: git config --global --add safe.directory "$GITHUB_WORKSPACE" || true @@ -131,6 +143,7 @@ jobs: rm -rf build && mkdir build && cd build cmake -G Ninja .. \ -DUSE_HIP=ON -DUSE_CUDA=OFF -DWITH_EP=OFF \ + -DENABLE_MULTI_PROTOCOL=ON \ -DUSE_HTTP=ON -DUSE_ETCD=ON -DSTORE_USE_ETCD=ON \ -DBUILD_UNIT_TESTS=OFF -DENABLE_DEBUG_SYMBOLS=OFF \ -DCMAKE_BUILD_TYPE=Release \ @@ -187,13 +200,16 @@ jobs: # Install the actual per-matrix release artifact (cp310-cp313) into a # fresh venv and exercise the packaged binary, so packaging/ELF-layout # failures in any version are caught before this wheel reaches PyPI. - # No GPU is needed for `mooncake_master --version`. + # Import both native bindings as well as exercising the packaged + # binary. No GPU is needed for these load-time checks. smoke_venv=$(mktemp -d) "${PYTHON_BIN}" -m venv "$smoke_venv" "$smoke_venv/bin/python" -m pip install --no-deps \ mooncake-wheel/dist-rocm-py${{ steps.generate_tag_release.outputs.python_version_tag }}/*.whl export LD_LIBRARY_PATH="/opt/rocm/lib:/usr/local/lib:${LD_LIBRARY_PATH:-}" - site="$("$smoke_venv/bin/python" -c 'import mooncake,os;print(os.path.dirname(mooncake.__file__))')" + "$smoke_venv/bin/python" \ + .release-ci/scripts/tone_tests/python/verify_rocm_wheel.py + site="$("$smoke_venv/bin/python" -c 'import mooncake, os; print(os.path.dirname(mooncake.__file__))')" "$site/mooncake_master" --version - name: Upload Python wheel artifact diff --git a/scripts/tone_tests/README_en.md b/scripts/tone_tests/README_en.md index c2cc8f4535..e8b01da39b 100644 --- a/scripts/tone_tests/README_en.md +++ b/scripts/tone_tests/README_en.md @@ -23,6 +23,15 @@ This directory contains end-to-end (E2E) test cases for the Mooncake project. Th **Description**: Prefill-Decode disaggregation test with different TP configurations +### 4. test_vllm_1p1d_erdma.sh + +**Description**: vLLM Prefill-Decode disaggregation test with the Mooncake connector + +The ROCm core tier runs this as serialized smoke coverage with +`num_workers=1`. It validates basic connector and transfer behavior, but it +does not validate concurrent sender workers while +[vLLM issue #44238](https://github.com/vllm-project/vllm/issues/44238) remains unresolved. + ## T-One/tone-cli Support [tone-cli](https://gitee.com/anolis/tone-cli/tree/master/tests/mooncake-ci-test) provides the following automation features: @@ -184,7 +193,7 @@ run_test() ${docker_exec} "\ cd /test_workspace && \ python3 -m pytest test_demo.py -v -s --tb=long" | tee "$log_file" - + return ${PIPESTATUS[0]} } @@ -210,7 +219,7 @@ if [ "${BASH_SOURCE[0]}" == "${0}" ]; then if ! run_test; then exit_code=1 fi - + parse $exit_code exit $? fi diff --git a/scripts/tone_tests/README_zh.md b/scripts/tone_tests/README_zh.md index b29ceedc99..96baffe1e6 100644 --- a/scripts/tone_tests/README_zh.md +++ b/scripts/tone_tests/README_zh.md @@ -23,6 +23,15 @@ **用例说明**:不同 TP 配置的 Prefill-Decode 分离架构测试 +### 4. test_vllm_1p1d_erdma.sh + +**用例说明**:使用 Mooncake connector 的 vLLM Prefill-Decode 分离测试 + +ROCm core tier 使用 `num_workers=1` 运行串行 smoke test。该用例验证基础的 +connector 和传输行为,但在 +[vLLM issue #44238](https://github.com/vllm-project/vllm/issues/44238) +解决前,不覆盖并发 sender worker 场景。 + ## T-One/tone-cli 支持 @@ -185,7 +194,7 @@ run_test() ${docker_exec} "\ cd /test_workspace && \ python3 -m pytest test_demo.py -v -s --tb=long" | tee "$log_file" - + return ${PIPESTATUS[0]} } @@ -211,7 +220,7 @@ if [ "${BASH_SOURCE[0]}" == "${0}" ]; then if ! run_test; then exit_code=1 fi - + parse $exit_code exit $? fi diff --git a/scripts/tone_tests/python/test_hicache_storage_mooncake_backend.py b/scripts/tone_tests/python/test_hicache_storage_mooncake_backend.py index e3d751c356..f736b60c45 100644 --- a/scripts/tone_tests/python/test_hicache_storage_mooncake_backend.py +++ b/scripts/tone_tests/python/test_hicache_storage_mooncake_backend.py @@ -22,6 +22,8 @@ is_in_ci, ) +ROCM_MHA_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-8B" + class HiCacheStorageMooncakeBackendBaseMixin(HiCacheStorageBaseMixin): """Base mixin class with common setup and utilities""" @@ -30,6 +32,12 @@ class HiCacheStorageMooncakeBackendBaseMixin(HiCacheStorageBaseMixin): mooncake_master_port_base = 50051 mooncake_metadata_port_base = 8080 + @classmethod + def _get_model_name(cls): + if os.getenv("CI_ACCELERATOR") == "rocm": + return ROCM_MHA_MODEL_NAME_FOR_TEST + return super()._get_model_name() + @classmethod def setUpClass(cls): """Set up test environment and launch Mooncake services before server setup""" @@ -285,9 +293,45 @@ def _get_additional_server_args_and_env(cls): def test_eval_accuracy(self): """Test eval accuracy with cache persistence across cache flushes""" - from test_hicache_storage_file_backend import run_eval_accuracy_test - - run_eval_accuracy_test(self) + if os.getenv("CI_ACCELERATOR") != "rocm": + from test_hicache_storage_file_backend import run_eval_accuracy_test + + run_eval_accuracy_test(self) + return + + # Qwen3 can spend the upstream 512-token budget entirely in its + # thinking trace, leaving no final GSM8K answer. Give ROCm CI enough + # room to finish the answer so this remains a cache-accuracy test rather + # than a reasoning-length lottery. + from types import SimpleNamespace + + from sglang.test.run_eval import run_eval + + args = SimpleNamespace( + base_url=f"http://{self.base_host}:{self.base_port}", + eval_name="gsm8k", + api="completion", + max_tokens=2048, + num_examples=200, + num_threads=64, + ) + metrics_initial = run_eval(args) + self.flush_cache() + metrics_cached = run_eval(args) + + accuracy_diff = abs(metrics_initial["score"] - metrics_cached["score"]) + print(f"Accuracy difference: {accuracy_diff:.4f}") + self.assertGreater( + metrics_initial["score"], 0.6, "Initial accuracy should be reasonable" + ) + self.assertGreater( + metrics_cached["score"], 0.6, "Cached accuracy should be reasonable" + ) + self.assertLess( + accuracy_diff, + 0.03, + "Accuracy should be consistent between cache states", + ) if __name__ == "__main__": diff --git a/scripts/tone_tests/python/test_moe_mooncake.py b/scripts/tone_tests/python/test_moe_mooncake.py index cce365dfeb..c6a87e4493 100644 --- a/scripts/tone_tests/python/test_moe_mooncake.py +++ b/scripts/tone_tests/python/test_moe_mooncake.py @@ -1,8 +1,9 @@ -''' +""" this test case is from https://github.com/HanHan009527/sglang/blob/a100-ci/test/manual/ep/test_moe_mooncake.py End-to-End Integration Test for SGLang with Mooncake Elastic EP Backend. -''' +""" +import os import unittest from types import SimpleNamespace @@ -17,8 +18,7 @@ popen_launch_server, ) -ib_devices = get_rdma_devices_args() - +ib_devices = os.getenv("MOONCAKE_DEVICE") or get_rdma_devices_args() class TestMooncakeBackend(CustomTestCase): diff --git a/scripts/tone_tests/python/verify_rocm_wheel.py b/scripts/tone_tests/python/verify_rocm_wheel.py new file mode 100644 index 0000000000..f247611844 --- /dev/null +++ b/scripts/tone_tests/python/verify_rocm_wheel.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +"""Verify that the active Mooncake native modules belong to the ROCm wheel.""" + +import base64 +import hashlib +import importlib +import importlib.metadata as metadata +from pathlib import Path + + +def record_digest(path: Path, algorithm: str) -> str: + digest = hashlib.new(algorithm, path.read_bytes()).digest() + return base64.urlsafe_b64encode(digest).rstrip(b"=").decode() + + +def main() -> None: + distribution = metadata.distribution("mooncake-transfer-engine-rocm") + package = importlib.import_module("mooncake") + package_dir = Path(package.__file__).resolve().parent + records = {str(item): item for item in distribution.files or ()} + expected_files = { + "mooncake/engine.so": "mooncake.engine", + "mooncake/store.so": "mooncake.store", + "mooncake/mooncake_master": None, + } + + print("Mooncake package:", package.__file__) + print("Mooncake ROCm distribution:", distribution.version) + for relative_path, module_name in expected_files.items(): + record = records.get(relative_path) + if record is None or record.hash is None: + raise RuntimeError(f"Missing hashed wheel record: {relative_path}") + + installed_path = Path(distribution.locate_file(record)).resolve() + package_path = (package_dir / Path(relative_path).name).resolve() + if installed_path != package_path: + raise RuntimeError( + f"{relative_path} resolves outside the active Mooncake package: " + f"{installed_path} != {package_path}" + ) + + digest = record_digest(installed_path, record.hash.mode) + if digest != record.hash.value: + raise RuntimeError(f"Installed file does not match wheel: {installed_path}") + + if module_name is not None: + module = importlib.import_module(module_name) + if Path(module.__file__).resolve() != installed_path: + raise RuntimeError( + f"{module_name} loaded from an unexpected path: {module.__file__}" + ) + + print(relative_path, installed_path, f"{record.hash.mode}={digest}") + + +if __name__ == "__main__": + main() diff --git a/scripts/tone_tests/rocm_runner.env.example b/scripts/tone_tests/rocm_runner.env.example new file mode 100644 index 0000000000..c72725d515 --- /dev/null +++ b/scripts/tone_tests/rocm_runner.env.example @@ -0,0 +1,32 @@ +# Host-owned profile for the controller-only ROCm self-hosted runner. +# Install as /etc/mooncake-ci/runner.env; do not store credentials here. +CI_ACCELERATOR=rocm +MOONCAKE_CI_TIER=core-4gpu +LOCAL_IP=192.168.210.1 +REMOTE_IP=192.168.210.2 +REMOTE_SSH_TARGET=mooncake-worker +REMOTE_TEST_DIR=/var/lib/mooncake-ci/work +MODEL_CACHE=/var/lib/mooncake-ci/cache +# Keep compiler/JIT output off the runner root filesystem. The harness creates +# a separate subdirectory per pinned container image. +MOONCAKE_RUNTIME_CACHE=/data/mooncake-ci/runtime-cache +MOONCAKE_RENDER_DEVICES="/dev/dri/renderD129 /dev/dri/renderD137 /dev/dri/renderD145 /dev/dri/renderD153" +MOONCAKE_GPU_INDICES=0,1,2,3 +MOONCAKE_CPUSET_CPUS=0-95 +MOONCAKE_CPUSET_MEMS=0 +MOONCAKE_SGLANG_BASE_GPU_ID=0 +MOONCAKE_EPD_ENCODER_GPU_ID=0 +MOONCAKE_EPD_PREFILL_GPU_ID=2 +MOONCAKE_EPD_DECODE_GPU_ID=0 +MOONCAKE_VLLM_VISIBLE_DEVICES=0,1 +MOONCAKE_SGLANG_MEM_FRACTION_STATIC=0.5 +MOONCAKE_RDMA_DEVICES=ionic_0,ionic_1,ionic_2,ionic_3 +MOONCAKE_RDMA_NETDEVS=eth2,eth3,eth4,eth5 +# Keep Mooncake traffic on one verified local/remote rail. NCCL may still use +# every device listed in MOONCAKE_RDMA_DEVICES. +MOONCAKE_TRANSFER_DEVICE=ionic_0 +MOONCAKE_GID_INDEX=1 +AINIC_VERSION=1.117.5 +MOONCAKE_SSH_CONFIG=/etc/mooncake-ci/runner_ssh_config +MOONCAKE_LOCK_FILE=/var/lib/mooncake-ci/run/mooncake.lock +CONTAINER_NAME=mooncake-rocm-ci diff --git a/scripts/tone_tests/scripts/common.sh b/scripts/tone_tests/scripts/common.sh index 27f21ad7bf..c26da80d03 100755 --- a/scripts/tone_tests/scripts/common.sh +++ b/scripts/tone_tests/scripts/common.sh @@ -1,16 +1,16 @@ #!/bin/bash -TEST_CASE_RESULT_PATH="run/logs/$test_case_name" +TEST_CASE_RESULT_PATH="run/logs/${test_case_name:-}" docker_exec="docker exec ${CONTAINER_NAME} bash -c" setup_directory(){ local dir_path=$1 - + if [ -z "$dir_path" ]; then echo "ERROR: Directory path not provided" >&2 return 1 fi - + if [ -d "$dir_path" ]; then echo "Directory already exists: $dir_path" return 0 @@ -27,7 +27,7 @@ setup_directory(){ setup_log_directory(){ local log_dir="$1" - + if [ -d "$log_dir" ]; then echo "Removing existing log directory: $log_dir" rm -rf "$log_dir" @@ -36,24 +36,218 @@ setup_log_directory(){ echo "Log directory set up at: $log_dir" } +prepare_rocm_runtime_cache_args() { + local registry_addr=$1 + local cache_root=${MOONCAKE_RUNTIME_CACHE:-} + local cache_key image_cache_dir cache_dir + + if [ -z "$cache_root" ]; then + echo "ERROR: MOONCAKE_RUNTIME_CACHE is required for ROCm" >&2 + return 1 + fi + if ! [[ "$cache_root" =~ ^/[A-Za-z0-9._/-]+$ ]]; then + echo "ERROR: MOONCAKE_RUNTIME_CACHE must be an absolute path without spaces: $cache_root" >&2 + return 1 + fi + mkdir -p -- "$cache_root" || return 1 + cache_root=$(cd -P -- "$cache_root" && pwd) || return 1 + if [ "$cache_root" = "/" ]; then + echo "ERROR: MOONCAKE_RUNTIME_CACHE must not resolve to /" >&2 + return 1 + fi + + cache_key=${registry_addr##*@sha256:} + if [ "$cache_key" = "$registry_addr" ]; then + cache_key=$(printf '%s' "$registry_addr" | cksum | awk '{print $1}') + fi + cache_key=${cache_key:0:16} + image_cache_dir="${cache_root}/${cache_key}" + for cache_dir in \ + aiter-jit pip tmp torch-extensions torchinductor triton xdg; do + mkdir -p -- "${image_cache_dir}/${cache_dir}" || return 1 + [ -w "${image_cache_dir}/${cache_dir}" ] || { + echo "ERROR: ROCm runtime cache is not writable: ${image_cache_dir}/${cache_dir}" >&2 + return 1 + } + done + + ROCM_RUNTIME_CACHE_ARGS=( + -v "${image_cache_dir}:/runtime-cache" + -e AITER_JIT_DIR=/runtime-cache/aiter-jit + -e PIP_CACHE_DIR=/runtime-cache/pip + -e TMPDIR=/runtime-cache/tmp + -e TORCH_EXTENSIONS_DIR=/runtime-cache/torch-extensions + -e TORCHINDUCTOR_CACHE_DIR=/runtime-cache/torchinductor + -e TRITON_CACHE_DIR=/runtime-cache/triton + -e XDG_CACHE_HOME=/runtime-cache/xdg + ) + echo "Using image-scoped ROCm runtime cache: $image_cache_dir" +} + docker_launch(){ local registry_addr=$1 local extra_args=$2 - docker_run_cmd="docker run --init --name ${CONTAINER_NAME} \ - -d --ipc=host --cap-add=SYS_PTRACE --network=host --gpus all \ - --ulimit memlock=-1 --ulimit stack=67108864 --shm-size=128g \ - -v ${MODEL_CACHE}:/root/.cache $extra_args --privileged \ - -v $BASE_DIR:/test_run \ - -v /root/test.jsonl:/tmp/test.jsonl \ - --entrypoint bash \ - ${registry_addr} -c \"hostname;sleep 360000\"" - - echo "Executing Docker run command:" - echo "$docker_run_cmd" - if ! eval "$docker_run_cmd"; then - echo "ERROR: Failed to launch docker container" >&2 - return 1 + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + local -a docker_args=( + run --init --name "${CONTAINER_NAME}" -d + --network=host + --device=/dev/kfd + --cpuset-cpus="${MOONCAKE_CPUSET_CPUS}" + --cpuset-mems="${MOONCAKE_CPUSET_MEMS}" + --cap-drop=ALL + # apt/dpkg drops privileges to _apt while installing the standard + # verbs userspace. Retain only the filesystem/identity capabilities + # needed for that setup; serving never receives the default Docker + # capability set. + --cap-add=CHOWN + --cap-add=DAC_OVERRIDE + --cap-add=FOWNER + --cap-add=IPC_LOCK + --cap-add=SETGID + --cap-add=SETUID + # Mooncake queries page placement with move_pages(2) to select the + # nearest RoCE rail. Docker's default seccomp profile rejects that + # syscall with EPERM even for the container's own pages. Keep the + # capability and device allowlists above as the security boundary. + --security-opt=seccomp=unconfined + --security-opt=no-new-privileges:true + --pids-limit=32768 + --ulimit memlock=-1:-1 + --ulimit stack=67108864:67108864 + --shm-size=128g + --stop-timeout=120 + -e CI_ACCELERATOR=rocm + -e CI=true + -e PYTHONDONTWRITEBYTECODE=1 + -e PYTHONFAULTHANDLER=1 + -e PYTHONUNBUFFERED=1 + -e "PYTEST_ADDOPTS=-p no:cacheprovider" + -e NCCL_GIN_TYPE=0 + -e "NCCL_IB_HCA=${MOONCAKE_RDMA_DEVICES}" + -e "NCCL_SOCKET_IFNAME=${MOONCAKE_RDMA_NETDEVS}" + -e "MOONCAKE_DEVICE=${MOONCAKE_TRANSFER_DEVICE}" + -e "MC_GID_INDEX=${MOONCAKE_GID_INDEX}" + -e MC_FORCE_HCA=1 + # The MI35x image enables a host-wide SGLang affinity heuristic. + # It ignores Docker's cpuset and assigns TP rank 1 to CPU96+, which + # is outside this NUMA0 allocation. Docker already enforces the + # correct affinity, so disable the conflicting inner policy. + -e SGLANG_SET_CPU_AFFINITY=0 + -v "${MODEL_CACHE}:/root/.cache" + -v "${BASE_DIR}:/test_run" + --entrypoint bash + ) + prepare_rocm_runtime_cache_args "$registry_addr" || return 1 + docker_args+=("${ROCM_RUNTIME_CACHE_ARGS[@]}") + local host_libionic="" + if command -v ldconfig >/dev/null 2>&1; then + host_libionic=$(ldconfig -p 2>/dev/null | awk '/libionic[.]so[.]1/{print $NF; exit}') + fi + if [ -z "$host_libionic" ]; then + local ionic_candidate + for ionic_candidate in \ + /usr/lib/x86_64-linux-gnu/libionic.so.1 \ + /lib/x86_64-linux-gnu/libionic.so.1; do + if [ -r "$ionic_candidate" ]; then + host_libionic=$ionic_candidate + break + fi + done + fi + if [ -n "$host_libionic" ]; then + host_libionic=$(readlink -f "$host_libionic") + fi + if [ -n "$host_libionic" ] && [ -r "$host_libionic" ]; then + echo "Using host-matched Ionic provider library: $host_libionic" + docker_args+=(-v "${host_libionic}:/opt/mooncake-host-rdma/libionic.so.1:ro") + else + echo "WARNING: Host libionic.so.1 is unavailable; ROCm images must provide a compatible Ionic provider" >&2 + fi + local -a render_nodes + read -r -a render_nodes <<<"${MOONCAKE_RENDER_DEVICES:-}" + if [ "${#render_nodes[@]}" -eq 0 ]; then + echo "ERROR: ROCm profile must expose at least one render node" >&2 + return 1 + fi + local device + for device in "${render_nodes[@]}"; do + if [[ ! "$device" =~ ^/dev/dri/render[D][0-9]+$ ]] || [ ! -c "$device" ]; then + echo "ERROR: Invalid or missing ROCm render node: $device" >&2 + return 1 + fi + docker_args+=(--device="$device") + done + if [ -z "${MOONCAKE_RDMA_DEVICES:-}" ]; then + echo "ERROR: MOONCAKE_RDMA_DEVICES is required for ROCm" >&2 + return 1 + fi + local rdma_device verbs_path uverbs_node uverbs_found + local -A mounted_uverbs=() + for rdma_device in ${MOONCAKE_RDMA_DEVICES//,/ }; do + verbs_path="/sys/class/infiniband/${rdma_device}/device/infiniband_verbs" + [ -d "$verbs_path" ] || { + echo "ERROR: Missing verbs mapping for RDMA device $rdma_device" >&2 + return 1 + } + uverbs_found=0 + for uverbs_node in "$verbs_path"/uverbs*; do + [ -e "$uverbs_node" ] || continue + uverbs_found=1 + device="/dev/infiniband/$(basename "$uverbs_node")" + [ -c "$device" ] || { + echo "ERROR: Missing RDMA character device $device" >&2 + return 1 + } + if [ -z "${mounted_uverbs[$device]:-}" ]; then + docker_args+=(--device="$device") + mounted_uverbs[$device]=1 + fi + done + if [ "$uverbs_found" -eq 0 ]; then + echo "ERROR: No userspace verbs device found for RDMA device $rdma_device" >&2 + return 1 + fi + done + if [ -c /dev/infiniband/rdma_cm ]; then + docker_args+=(--device=/dev/infiniband/rdma_cm) + fi + if [ "${USE_HUGGINGFACE_MIRROR}" = "true" ]; then + docker_args+=(-e "HF_ENDPOINT=${HUGGINGFACE_MIRROR}" -e HF_HUB_ENABLE_HF_TRANSFER=1) + fi + if [ "${USE_MODELSCOPE}" = "true" ]; then + docker_args+=(-e SGLANG_USE_MODELSCOPE=true) + fi + if [ -n "${HF_TOKEN_FILE:-}" ] && [ -r "$HF_TOKEN_FILE" ]; then + local hf_token + hf_token=$(<"$HF_TOKEN_FILE") + [ -n "$hf_token" ] || { echo "ERROR: $HF_TOKEN_FILE is empty" >&2; return 1; } + export HF_TOKEN="$hf_token" + docker_args+=(-e HF_TOKEN) + fi + printf 'Executing Docker run command:' + printf ' %q' docker "${docker_args[@]}" "$registry_addr" -c 'hostname; sleep 360000' + printf '\n' + if ! docker "${docker_args[@]}" "$registry_addr" -c 'hostname; sleep 360000'; then + echo "ERROR: Failed to launch ROCm container" >&2 + return 1 + fi + else + docker_run_cmd="docker run --init --name ${CONTAINER_NAME} \ + -d --ipc=host --cap-add=SYS_PTRACE --network=host --gpus all \ + --ulimit memlock=-1 --ulimit stack=67108864 --shm-size=128g \ + -v ${MODEL_CACHE}:/root/.cache $extra_args --privileged \ + -v $BASE_DIR:/test_run \ + -v /root/test.jsonl:/tmp/test.jsonl \ + --entrypoint bash \ + ${registry_addr} -c \"hostname;sleep 360000\"" + + echo "Executing Docker run command:" + echo "$docker_run_cmd" + if ! eval "$docker_run_cmd"; then + echo "ERROR: Failed to launch docker container" >&2 + return 1 + fi fi pip_cmd="" @@ -86,7 +280,17 @@ docker_launch(){ fi local relative_path=${TEST_RUN_DIR#$BASE_DIR} local cleaned_path=${relative_path#/} - pip_cmd=$(append_str "${pip_cmd}" "pip install --force-reinstall /test_run/$cleaned_path/whls/$mooncake_whl_file") + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + # SGLang and vLLM images may already contain the CUDA distribution. + # The CUDA and ROCm distributions share the same `mooncake` package, + # so installing the ROCm wheel on top leaves a mixture of old and new + # Python modules/native libraries and breaks the Store RPC ABI. + pip_cmd=$(append_str "${pip_cmd}" \ + "python3 -m pip uninstall -y mooncake-transfer-engine mooncake-transfer-engine-rocm") + pip_cmd=$(append_str "${pip_cmd}" \ + "python3 -c 'import shutil, site, sysconfig; from pathlib import Path; roots={Path(path).resolve() for path in (*site.getsitepackages(), site.getusersitepackages(), sysconfig.get_path(\"purelib\"), sysconfig.get_path(\"platlib\")) if path}; packages=sorted({root / \"mooncake\" for root in roots}); [(print(\"Removing orphaned Mooncake package:\", package), shutil.rmtree(package)) for package in packages if package.is_dir()]'") + fi + pip_cmd=$(append_str "${pip_cmd}" "python3 -m pip install --force-reinstall /test_run/$cleaned_path/whls/$mooncake_whl_file") # Check if sglang-router is needed and missing if [[ "$registry_addr" == *"sglang"* ]]; then @@ -100,28 +304,110 @@ docker_launch(){ else echo "sglang-router already installed, skipping" fi - # Reuse SGLang CI's single source of truth for the git-only evaluator # pin instead of duplicating the commit here. pip_cmd=$(append_str "${pip_cmd}" \ 'source /sgl-workspace/sglang/scripts/ci/utils/sgl_eval_ref.sh && pip install "$SGL_EVAL_SPEC"') fi - echo "Installing ERDMA drivers" - echo "Executing ERDMA driver installation command:" - echo "${erdma_driver_cmd}" - if ! ${docker_exec} "${erdma_driver_cmd}"; then - echo "ERROR: Failed to install ERDMA drivers" >&2 - return 1 + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + local rocm_rdma_cmd="set -euo pipefail +rdma_ready=false +if command -v ibv_devinfo >/dev/null 2>&1 && ibv_devinfo >/tmp/mooncake-ibv-devinfo.log 2>&1; then + rdma_ready=true + echo 'ROCm RoCE userspace is functional' +else + echo 'Initial ibv_devinfo failure:' >&2 + cat /tmp/mooncake-ibv-devinfo.log >&2 2>/dev/null || true + echo 'Installing AMD Pensando AINIC userspace ${AINIC_VERSION}' + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + apt-transport-https ca-certificates curl gnupg + install -d -m 0755 /etc/apt/keyrings + curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key \ + | gpg --dearmor --yes --output /etc/apt/keyrings/amdainic.gpg + echo 'deb [arch=amd64 signed-by=/etc/apt/keyrings/amdainic.gpg] https://repo.radeon.com/amdainic/pensando/ubuntu/${AINIC_VERSION} ${ubuntu_codename} main' \ + > /etc/apt/sources.list.d/amdainic.list + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ibverbs-utils ionic-common libionic-dev librdmacm1 + rm -rf /var/lib/apt/lists/* +fi + +if [ \"\$rdma_ready\" != true ] && [ -r /opt/mooncake-host-rdma/libionic.so.1 ]; then + echo 'Overlaying host-matched Ionic provider library' + # Let the package installation update the loader cache before replacing + # its ABI-incompatible provider. Running ldconfig afterwards would restore + # libionic.so.1 to the newer container library. + ldconfig + install -m 0644 /opt/mooncake-host-rdma/libionic.so.1 \ + /usr/lib/x86_64-linux-gnu/libionic-host.so.1 + ln -sfn libionic-host.so.1 /usr/lib/x86_64-linux-gnu/libionic.so.1 + ln -sfn libionic-host.so.1 /usr/lib/x86_64-linux-gnu/libionic.so + + verbs_abi=\$(find /usr/lib/x86_64-linux-gnu/libibverbs -maxdepth 1 \ + \( -type f -o -type l \) 2>/dev/null \ + | sed -n 's/.*-rdmav\([0-9][0-9]*\)[.]so$/\1/p' | head -n 1) + if [ -n \"\$verbs_abi\" ]; then + ln -sfn ../libionic-host.so.1 \ + /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav\${verbs_abi}.so + echo 'Ionic provider path:' \ + \"\$(readlink -f /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav\${verbs_abi}.so)\" + else + echo 'WARNING: Unable to determine the container libibverbs provider ABI' >&2 + fi + echo 'Ionic provider checksums:' + sha256sum /opt/mooncake-host-rdma/libionic.so.1 \ + /usr/lib/x86_64-linux-gnu/libionic-host.so.1 +fi" + echo "Checking ROCm RoCE userspace" + if ! ${docker_exec} "${rocm_rdma_cmd}"; then + echo "ERROR: Failed to install ROCm RoCE userspace" >&2 + return 1 + fi + else + echo "Installing ERDMA drivers" + echo "Executing ERDMA driver installation command:" + echo "${erdma_driver_cmd}" + if ! ${docker_exec} "${erdma_driver_cmd}"; then + echo "ERROR: Failed to install ERDMA drivers" >&2 + return 1 + fi fi - echo "Checking RDMA devices" - echo "Executing ibv_devinfo check command:" - echo "ibv_devinfo" - if ! ${docker_exec} "ibv_devinfo" >/dev/null 2>&1; then - echo "ibv_devinfo execution failed" >&2 - return 1 + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + local rdma_device=${MOONCAKE_TRANSFER_DEVICE:-} + if [ -z "$rdma_device" ]; then + echo "ERROR: MOONCAKE_TRANSFER_DEVICE is required for ROCm" >&2 + return 1 + fi + if ! [[ "$rdma_device" =~ ^[a-zA-Z0-9_.-]+$ ]]; then + echo "ERROR: Invalid MOONCAKE_TRANSFER_DEVICE: $rdma_device" >&2 + return 1 + fi + echo "Checking ROCm RDMA device ${rdma_device}" + local rdma_preflight_cmd="set -e +echo '=== ibv_devinfo ===' +ibv_devinfo -d '${rdma_device}' +echo '=== RDMA link state ===' +if command -v rdma >/dev/null 2>&1; then rdma link show; fi +state=\$(cat '/sys/class/infiniband/${rdma_device}/ports/1/state') +echo '${rdma_device} port 1 state:' \"\$state\" +case \"\$state\" in *ACTIVE*) ;; *) echo 'RDMA port is not active' >&2; exit 1;; esac + gid=\$(cat '/sys/class/infiniband/${rdma_device}/ports/1/gids/${MOONCAKE_GID_INDEX}') + echo '${rdma_device} GID index ${MOONCAKE_GID_INDEX}:' \"\$gid\" +case \"\$gid\" in ''|'::'|'0:0:0:0:0:0:0:0') echo 'RDMA GID is empty' >&2; exit 1;; esac" + if ! ${docker_exec} "${rdma_preflight_cmd}"; then + echo "RDMA preflight failed for $rdma_device" >&2 + return 1 + fi + echo "RDMA preflight successful" else + echo "Checking RDMA devices" + if ! ${docker_exec} "ibv_devinfo" >/dev/null 2>&1; then + echo "ibv_devinfo execution failed" >&2 + return 1 + fi echo "ibv_devinfo execution successful" fi @@ -136,7 +422,15 @@ docker_launch(){ echo "ERROR: Failed to install Mooncake dependencies" >&2 return 1 fi - + + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + local mooncake_install_check="python3 /test_run/python/verify_rocm_wheel.py && ! python3 -m pip show mooncake-transfer-engine >/dev/null 2>&1" + if ! ${docker_exec} "${mooncake_install_check}"; then + echo "ERROR: The ROCm wheel did not replace all image-provided Mooncake files" >&2 + return 1 + fi + fi + return 0 } @@ -146,7 +440,7 @@ clean_container(){ echo "No container name provided" return 1 fi - + # check if container exists if docker ps -a --format '{{.Names}}' | grep -q "^${container_name}$"; then echo "Stopping and removing existing container: ${container_name}" @@ -170,7 +464,7 @@ clean_container(){ append_str() { local original_str="$1" local append_value="$2" - + if [ -z "$original_str" ]; then echo "$append_value" else @@ -178,7 +472,7 @@ append_str() { fi } -check_server_ready() { +check_server_ready() { local server_log_path=$1 local max_attempts=${2:-120} @@ -198,7 +492,7 @@ check_server_ready() { sleep 2 fi done - + echo "ERROR: Server failed to start within timeout" return 1 } @@ -220,11 +514,17 @@ check_server_ready_with_pattern() { echo "Server is ready!" return 0 fi + if grep -qE 'Fatal Python error|Segfault encountered|Subprocess .* crashed with exit code' \ + "$server_log_path" 2>/dev/null; then + echo "ERROR: Server process crashed during startup; see $server_log_path" >&2 + tail -n 80 "$server_log_path" >&2 + return 1 + fi echo "Waiting... ($i/$max_attempts)" sleep 2 fi done - + echo "ERROR: Server did not become ready in time" >&2 return 1 } @@ -234,10 +534,26 @@ get_whl(){ echo "whls_path: $whls_path and mkdir..." mkdir -p "$whls_path" - echo "get whl file from github action" + echo "get whl file from CI artifacts" rm -f "$whls_path/mooncake.zip" rm -f "$whls_path"/*.whl + if [ -n "${WHEEL_DIR:-}" ]; then + local local_wheel="" + local local_wheel_count=0 + while IFS= read -r wheel; do + local_wheel=$wheel + local_wheel_count=$((local_wheel_count + 1)) + done < <(find "$WHEEL_DIR" -type f -name '*.whl' -print) + if [ "$local_wheel_count" -ne 1 ]; then + echo "ERROR: Expected exactly one wheel in $WHEEL_DIR, found $local_wheel_count" >&2 + return 1 + fi + cp -L "$local_wheel" "$whls_path/" + echo "Copied self-hosted Actions wheel: $(basename "$local_wheel")" + return 0 + fi + local max_retries=5 local base_delay=5 # seconds local success=false @@ -278,7 +594,7 @@ get_whl(){ return 1 fi echo "Found wheel file: $mooncake_whl_file" - + echo "Successfully downloaded and extracted wheel file to $whls_path" return 0 } @@ -298,7 +614,7 @@ get_image(){ return 0 } -check_proxy_ready() { +check_proxy_ready() { local proxy_log_path=$1 local max_attempts=${2:-60} local expected_workers=2 @@ -310,7 +626,7 @@ check_proxy_ready() { echo "Waiting for SGLang Router to be ready and $expected_workers workers to be activated..." echo "Checking log file: $proxy_log_path" - + for i in $(seq 1 $max_attempts); do activated_count=0 tokenizer_ready=0 @@ -318,13 +634,13 @@ check_proxy_ready() { if [ -f "$proxy_log_path" ]; then # "Activated 1 worker(s) (marked as healthy)" activated_count=$(grep -cF "Activated 1 worker(s) (marked as healthy)" "$proxy_log_path" 2>/dev/null) || activated_count=0 - + # "Successfully loaded tokenizer" tokenizer_ready=$(grep -cE "Successfully (loaded|registered) tokenizer" "$proxy_log_path" 2>/dev/null) || tokenizer_ready=0 # "Starting server on 0.0.0.0:8000" server_started=$(grep -cF "Starting server on 0.0.0.0" "$proxy_log_path" 2>/dev/null) || server_started=0 - + if [ "$activated_count" -ge "$expected_workers" ] && [ "$tokenizer_ready" -gt 0 ]; then echo "Router is ready!" echo " - Workers activated: $activated_count/$expected_workers" @@ -335,7 +651,7 @@ check_proxy_ready() { return 0 fi fi - + if [ "$activated_count" -gt 0 ]; then echo "Waiting... ($i/$max_attempts) [Workers: $activated_count/$expected_workers, Tokenizer: $tokenizer_ready]" else @@ -343,7 +659,7 @@ check_proxy_ready() { fi sleep 2 done - + echo "ERROR: Router failed to start or workers failed to register within timeout" return 1 } @@ -351,26 +667,31 @@ check_proxy_ready() { stop_container(){ local container_name=${1:-$CONTAINER_NAME} - local remote_host=$2 + local remote_host=${2:-} local location="local" - + if [ -z "$container_name" ]; then echo "ERROR: No container name provided" >&2 return 1 fi - + if [ -n "$remote_host" ]; then location="remote" fi - + echo "Stopping ${location} Docker container: ${container_name}" - + if [ "$location" == "remote" ]; then - ssh -o StrictHostKeyChecking=no $remote_host "docker stop ${container_name} >/dev/null 2>&1" + local ssh_target=$remote_host + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + ssh_target=${REMOTE_SSH_TARGET:-$remote_host} + fi + ${SSH_CMD:-ssh -o StrictHostKeyChecking=no} "$ssh_target" \ + "docker stop ${container_name} >/dev/null 2>&1" else docker stop ${container_name} >/dev/null 2>&1 fi - + if [ $? -eq 0 ]; then echo "Successfully stopped ${location} container: ${container_name}" return 0 @@ -384,9 +705,9 @@ save_test_result() { local test_case_name=$1 local status=$2 local result_dir=$3 - + local result_json="${result_dir}/test_results.json" - + echo "{\"test_case\": \"$test_case_name\", \"status\": \"$status\", \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" > "$result_json" echo "Test results saved to: $result_json" echo "$test_case_name: $status" @@ -394,36 +715,93 @@ save_test_result() { cleanup_test_env() { local test_type=$1 - + local cleanup_failed=false + echo "===== Cleaning up $test_type machine environment =====" - - stop_container "${CONTAINER_NAME}" - - if [ "$test_type" = "double" ] && [ -n "$REMOTE_IP" ]; then - stop_container "${CONTAINER_NAME}" "$REMOTE_IP" - fi - - echo "Cleanup completed" + + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + if ! cleanup_rocm_node "local"; then + cleanup_failed=true + fi + + if [ "$test_type" = "double" ] && [ -n "${REMOTE_IP:-}" ]; then + echo "===== Running ROCm postflight on remote node ${REMOTE_IP} =====" + if ! ${SSH_CMD} "${REMOTE_SSH_TARGET:-$REMOTE_IP}" " + source ${REMOTE_TEST_DIR}/run/.shrc && \ + source ${REMOTE_TEST_DIR}/scripts/common.sh && \ + cleanup_rocm_node remote + "; then + echo "ERROR: Remote ROCm cleanup/postflight failed on ${REMOTE_IP}" >&2 + cleanup_failed=true + fi + fi + else + if ! stop_container "${CONTAINER_NAME}"; then + cleanup_failed=true + fi + + if [ "$test_type" = "double" ] && [ -n "${REMOTE_IP:-}" ]; then + if ! stop_container "${CONTAINER_NAME}" "$REMOTE_IP"; then + cleanup_failed=true + fi + fi + fi + + if $cleanup_failed; then + echo "ERROR: Cleanup did not complete successfully" >&2 + return 1 + fi + + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + echo "Cleanup and postflight completed" + else + echo "Cleanup completed" + fi + return 0 } -# Wait until GPU memory on the local host drains below a threshold. +# Return the maximum used memory, in MiB, for only this CI allocation. +gpu_max_used_mb() { + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + command -v rocm-smi >/dev/null 2>&1 || return 1 + rocm-smi --showmeminfo vram --json 2>/dev/null | python3 -c ' +import json, sys +indices = {f"card{i}" for i in sys.argv[1].split(",")} +data = json.load(sys.stdin) +used = {} +for card, values in data.items(): + if card not in indices: + continue + for key, value in values.items(): + if "VRAM Total Used Memory" in key: + used[card] = int(value) // (1024 * 1024) +missing = sorted(indices - used.keys()) +if missing: + print("Missing ROCm memory data for: " + ", ".join(missing), file=sys.stderr) + print(-1) +else: + print(max(used.values())) +' "${MOONCAKE_GPU_INDICES:-0,1,2,3}" + else + command -v nvidia-smi >/dev/null 2>&1 || return 1 + nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null \ + | sort -n | tail -n 1 + fi +} + +# Wait until GPU memory on the allocated devices drains below a threshold. # Returns 0 once drained, 1 if it times out. wait_gpu_idle() { local max_seconds=${1:-90} local threshold_mb=${2:-1024} - if ! command -v nvidia-smi >/dev/null 2>&1; then - echo "ERROR: nvidia-smi not available; cannot verify GPU drain" >&2 - return 1 - fi - echo "Waiting for GPU memory to drain (threshold ${threshold_mb}MB, timeout ${max_seconds}s)..." local elapsed=0 local max_used=0 while [ $elapsed -lt $max_seconds ]; do - max_used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -n | tail -n 1) + max_used=$(gpu_max_used_mb) if ! [[ "$max_used" =~ ^[0-9]+$ ]]; then - echo "ERROR: nvidia-smi query failed; cannot verify GPU drain" >&2 + echo "ERROR: ${CI_ACCELERATOR:-CUDA} GPU memory query failed; cannot verify GPU drain" >&2 return 1 fi if [ "$max_used" -le "$threshold_mb" ]; then @@ -437,6 +815,135 @@ wait_gpu_idle() { return 1 } +# Fail closed if KFD still reports a process with a queue on one of this +# allocation's GPUs. Mapping the configured render nodes through KFD topology +# keeps this check scoped to the shared host's ROCm CI partition. +verify_no_allocated_gpu_processes() { + if [ "${CI_ACCELERATOR:-cuda}" != "rocm" ]; then + return 0 + fi + + local topology_root=${MOONCAKE_KFD_TOPOLOGY_ROOT:-/sys/class/kfd/kfd/topology/nodes} + local process_root=${MOONCAKE_KFD_PROCESS_ROOT:-/sys/class/kfd/kfd/proc} + if [ ! -d "$topology_root" ] || [ ! -d "$process_root" ]; then + echo "ERROR: KFD topology or process sysfs is unavailable" >&2 + return 1 + fi + + local -a render_nodes + read -r -a render_nodes <<<"${MOONCAKE_RENDER_DEVICES:-}" + if [ "${#render_nodes[@]}" -eq 0 ]; then + echo "ERROR: MOONCAKE_RENDER_DEVICES is empty; cannot check ROCm processes" >&2 + return 1 + fi + + local device render_minor node_dir node_render_minor gpu_id + local allocation_gpu_ids="" + for device in "${render_nodes[@]}"; do + if [ ! -e "$device" ] || [[ ! "$device" =~ /render[D]([0-9]+)$ ]]; then + echo "ERROR: Allocated ROCm render device is missing: $device" >&2 + return 1 + fi + render_minor=${BASH_REMATCH[1]} + + gpu_id="" + for node_dir in "$topology_root"/*; do + [ -r "$node_dir/properties" ] && [ -r "$node_dir/gpu_id" ] || continue + node_render_minor=$(awk '$1 == "drm_render_minor" { print $2; exit }' \ + "$node_dir/properties") + if [ "$node_render_minor" = "$render_minor" ]; then + gpu_id=$(tr -d '[:space:]' < "$node_dir/gpu_id") + break + fi + done + if ! [[ "$gpu_id" =~ ^[0-9]+$ ]] || [ "$gpu_id" = 0 ]; then + echo "ERROR: Could not map $device to a KFD GPU ID" >&2 + return 1 + fi + allocation_gpu_ids="${allocation_gpu_ids}${gpu_id}"$'\n' + done + + local proc_dir queue_gpu_file queue_gpu_id pid + local remaining_pids="" + for proc_dir in "$process_root"/[0-9]*; do + [ -d "$proc_dir" ] || continue + pid=${proc_dir##*/} + for queue_gpu_file in "$proc_dir"/queues/*/gpuid; do + [ -r "$queue_gpu_file" ] || continue + queue_gpu_id=$(tr -d '[:space:]' < "$queue_gpu_file") + if printf '%s' "$allocation_gpu_ids" | grep -Fxq -- "$queue_gpu_id"; then + remaining_pids="${remaining_pids}${pid}"$'\n' + break + fi + done + done + + remaining_pids=$(printf '%s' "$remaining_pids" | sort -nu) + if [ -n "$remaining_pids" ]; then + echo "ERROR: KFD processes remain on allocated ROCm GPUs:" >&2 + ps -o pid,ppid,stat,args -p \ + "$(printf '%s\n' "$remaining_pids" | paste -sd, -)" \ + >&2 2>/dev/null || true + return 1 + fi + + echo "No KFD processes remain on the allocated ROCm GPUs" + return 0 +} + +# Final ROCm teardown is intentionally stricter than the reusable between-test +# reset: remove the named container, then prove the allocated GPUs and device +# handles are clean before releasing the cluster lock. +cleanup_rocm_node() { + local location=${1:-local} + local cleanup_failed=false + local container_names + + echo "Stopping and removing ${location} ROCm container: ${CONTAINER_NAME}" + if ! container_names=$(docker ps -a --format '{{.Names}}'); then + echo "ERROR: Failed to list ${location} Docker containers" >&2 + cleanup_failed=true + container_names="" + fi + + if printf '%s\n' "$container_names" | grep -Fxq -- "${CONTAINER_NAME}"; then + if ! docker stop "${CONTAINER_NAME}" >/dev/null 2>&1; then + echo "ERROR: Failed to stop ${location} container: ${CONTAINER_NAME}" >&2 + cleanup_failed=true + fi + if ! docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1; then + echo "ERROR: Failed to remove ${location} container: ${CONTAINER_NAME}" >&2 + cleanup_failed=true + fi + else + echo "No ${location} container named ${CONTAINER_NAME}" + fi + + if ! container_names=$(docker ps -a --format '{{.Names}}'); then + echo "ERROR: Failed to verify ${location} Docker container removal" >&2 + cleanup_failed=true + elif printf '%s\n' "$container_names" | grep -Fxq -- "${CONTAINER_NAME}"; then + echo "ERROR: ${location} container still exists: ${CONTAINER_NAME}" >&2 + cleanup_failed=true + else + echo "Verified ${location} container removal: ${CONTAINER_NAME}" + fi + + if ! wait_gpu_idle "${MOONCAKE_CLEANUP_TIMEOUT_SECONDS:-90}" \ + "${MOONCAKE_GPU_IDLE_THRESHOLD_MB:-1024}"; then + echo "ERROR: ${location} ROCm GPU allocation did not drain" >&2 + cleanup_failed=true + fi + if ! verify_no_allocated_gpu_processes; then + echo "ERROR: ${location} ROCm process postflight failed" >&2 + cleanup_failed=true + fi + + $cleanup_failed && return 1 + echo "ROCm cleanup/postflight passed on ${location} node" + return 0 +} + # Kill only GPU processes whose cgroup still identifies them as belonging to # the reused test container. Unknown or unrelated processes must be left alone; # the caller will quarantine the environment if GPU memory remains occupied. @@ -447,6 +954,10 @@ gpu_pid_belongs_to_container() { } force_kill_container_gpu_procs() { + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + echo "ROCm cleanup refuses host PID killing; quarantine the allocation if container restart does not drain it" >&2 + return 1 + fi command -v nvidia-smi >/dev/null 2>&1 || return 1 local container_id @@ -516,6 +1027,13 @@ drain_gpu_local() { # state on both the local and (for double-machine runs) remote nodes via a # lightweight container restart (no wheel / ERDMA driver reinstall). drain_gpu_between_tests() { + # The reset protocol is currently defined only for the dedicated ROCm + # allocation. Preserve the existing CUDA/T-one lifecycle until an + # accelerator-neutral reset contract is introduced and validated there. + if [ "${CI_ACCELERATOR:-cuda}" != "rocm" ]; then + return 0 + fi + echo "===== Resetting environment between test cases =====" local reset_failed=false if ! drain_gpu_local; then @@ -525,7 +1043,7 @@ drain_gpu_between_tests() { if [ -n "$REMOTE_IP" ]; then echo "Resetting environment on remote node $REMOTE_IP..." - if ! ${SSH_CMD} "$REMOTE_IP" " + if ! ${SSH_CMD} "${REMOTE_SSH_TARGET:-$REMOTE_IP}" " source ${REMOTE_TEST_DIR}/run/.shrc && \ source ${REMOTE_TEST_DIR}/scripts/common.sh && \ drain_gpu_local @@ -542,7 +1060,7 @@ drain_gpu_between_tests() { setup_node_env() { local registry_addr=$1 echo "===== Setting up docker environment =====" - + if ! get_image "$registry_addr"; then echo "ERROR: Failed to get the required image" return 1 @@ -555,7 +1073,9 @@ setup_node_env() { local extra_args="" extra_args="$extra_args -e NCCL_GIN_TYPE=0 " - extra_args="$extra_args --device=/dev/infiniband/uverbs0 --device=/dev/infiniband/uverbs1 --device=/dev/infiniband/rdma_cm " + if [ "${CI_ACCELERATOR:-cuda}" != "rocm" ]; then + extra_args="$extra_args --device=/dev/infiniband/uverbs0 --device=/dev/infiniband/uverbs1 --device=/dev/infiniband/rdma_cm " + fi if [ "${USE_HUGGINGFACE_MIRROR}" = "true" ]; then extra_args="$extra_args -e HF_ENDPOINT=${HUGGINGFACE_MIRROR} -e HF_HUB_ENABLE_HF_TRANSFER=1" fi @@ -573,40 +1093,93 @@ setup_node_env() { } launch_and_track_process() { - local full_cmd="$1" - local grep_pattern="$2" - local pid_file="$3" + local process_cmd=$1 + local log_path=$2 + local pid_file=$3 + local grep_pattern=${4:-} + + if [ "${CI_ACCELERATOR:-cuda}" != "rocm" ]; then + if [ -z "$grep_pattern" ]; then + echo "ERROR: CUDA process tracking requires a grep pattern" >&2 + return 1 + fi - echo "Executing command..." - echo "$full_cmd" - eval "$full_cmd" + local escaped_log launch_cmd + printf -v escaped_log '%q' "$log_path" + launch_cmd="${process_cmd} > ${escaped_log} 2>&1 &" + echo "Executing command..." + printf 'docker exec %q bash -c %q\n' "${CONTAINER_NAME}" "$launch_cmd" + if ! docker exec "${CONTAINER_NAME}" bash -c "$launch_cmd"; then + echo "ERROR: Failed to launch process in ${CONTAINER_NAME}" >&2 + return 1 + fi - echo "Waiting for process to initialize..." - for i in {1..15}; do - local container_main_pid=$(docker inspect --format '{{.State.Pid}}' "${CONTAINER_NAME}" 2>/dev/null) - if [ -n "$container_main_pid" ] && [ "$container_main_pid" != "0" ]; then - pid=$(ps -eo pid,ppid,cmd | awk -v root="$container_main_pid" -v pattern="$grep_pattern" ' - BEGIN { pids[root] = 1 } - { - if ($2 in pids && $0 ~ pattern) { - print $1 - exit + echo "Waiting for process to initialize..." + local container_main_pid pid + for i in {1..15}; do + container_main_pid=$(docker inspect --format '{{.State.Pid}}' \ + "${CONTAINER_NAME}" 2>/dev/null) + if [ -n "$container_main_pid" ] && [ "$container_main_pid" != "0" ]; then + pid=$(ps -eo pid,ppid,cmd | awk \ + -v root="$container_main_pid" -v pattern="$grep_pattern" ' + BEGIN { pids[root] = 1 } + { + if ($2 in pids && $0 ~ pattern) { + print $1 + exit + } } - } - ') - fi + ') + fi - if [ -n "$pid" ]; then - echo "$pid" > "$pid_file" - echo "PID $pid (on host) saved to $pid_file" + if [ -n "${pid:-}" ]; then + mkdir -p "$(dirname "$pid_file")" + echo "$pid" > "$pid_file" + echo "PID $pid (on host) saved to $pid_file" + return 0 + fi + + echo " Attempt $i/15..." + sleep 2 + done + + echo "Process not found after 30 seconds" + return 1 + fi + + local escaped_cmd escaped_log launch_cmd container_pid process_group + + printf -v escaped_cmd '%q' "$process_cmd" + printf -v escaped_log '%q' "$log_path" + launch_cmd="setsid bash -c ${escaped_cmd} > ${escaped_log} 2>&1 < /dev/null & echo \$!" + + echo "Executing command in a dedicated container process group..." + echo "$process_cmd" + container_pid=$(docker exec "${CONTAINER_NAME}" bash -c "$launch_cmd") || { + echo "ERROR: Failed to launch process in ${CONTAINER_NAME}" >&2 + return 1 + } + container_pid=$(printf '%s\n' "$container_pid" | tail -n 1 | tr -d '[:space:]') + if ! [[ "$container_pid" =~ ^[0-9]+$ ]]; then + echo "ERROR: Invalid container PID returned by launcher: $container_pid" >&2 + return 1 + fi + + for i in {1..15}; do + process_group=$(docker exec "${CONTAINER_NAME}" \ + ps -o pgid= -p "$container_pid" 2>/dev/null | tr -d '[:space:]') + if [[ "$process_group" =~ ^[0-9]+$ ]]; then + mkdir -p "$(dirname "$pid_file")" + echo "$process_group" > "$pid_file" + echo "Container process group $process_group saved to $pid_file" return 0 fi - - echo " Attempt $i/15..." + + echo " Waiting for process group... ($i/15)" sleep 2 done - echo "Process not found after 30 seconds" + echo "ERROR: Container process group not found after 30 seconds" >&2 return 1 } @@ -619,25 +1192,86 @@ kill_process() { return 0 fi - local pid=$(cat "$pid_file") - if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then + if [ "${CI_ACCELERATOR:-cuda}" != "rocm" ]; then + local pid + pid=$(cat "$pid_file") + if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then + rm -f "$pid_file" + return 0 + fi + + echo "Stopping $service_name (PID: $pid)..." + kill -TERM "$pid" 2>/dev/null + sleep 2 + if kill -0 "$pid" 2>/dev/null; then + kill -KILL "$pid" 2>/dev/null + fi + + rm -f "$pid_file" + echo "✓ $service_name stopped" + return 0 + fi + + local process_group + process_group=$(tr -d '[:space:]' < "$pid_file") + if ! [[ "$process_group" =~ ^[0-9]+$ ]]; then + echo "ERROR: Invalid process group in $pid_file" >&2 + rm -f "$pid_file" + return 1 + fi + + if ! docker exec "${CONTAINER_NAME}" bash -c \ + "kill -0 -- -${process_group} 2>/dev/null"; then rm -f "$pid_file" return 0 fi - echo "Stopping $service_name (PID: $pid)..." - - kill -TERM "$pid" 2>/dev/null + echo "Stopping $service_name (container process group: $process_group)..." + docker exec "${CONTAINER_NAME}" bash -c \ + "kill -TERM -- -${process_group} 2>/dev/null || true" + local attempt + for attempt in {1..15}; do + if ! docker exec "${CONTAINER_NAME}" bash -c \ + "kill -0 -- -${process_group} 2>/dev/null"; then + rm -f "$pid_file" + echo "✓ $service_name stopped" + return 0 + fi + sleep 2 + done + + echo "Process group $process_group did not stop after SIGTERM; sending SIGKILL" >&2 + docker exec "${CONTAINER_NAME}" bash -c \ + "kill -KILL -- -${process_group} 2>/dev/null || true" sleep 2 - if kill -0 "$pid" 2>/dev/null; then - kill -KILL "$pid" 2>/dev/null + if docker exec "${CONTAINER_NAME}" bash -c \ + "kill -0 -- -${process_group} 2>/dev/null"; then + echo "ERROR: $service_name process group $process_group survived SIGKILL" >&2 + return 1 fi - + rm -f "$pid_file" echo "✓ $service_name stopped" return 0 } +verify_model_processes_stopped() { + if [ "${CI_ACCELERATOR:-cuda}" != "rocm" ]; then + return 0 + fi + + local process_pattern='sglang[.]launch_server|sglang_router[.]launch_router|sglang::router|vllm[.]entrypoints[.]openai[.]api_server|mooncake_connector_proxy[.]py|toy_proxy_server[.]py' + local remaining + remaining=$(docker exec "${CONTAINER_NAME}" bash -c \ + "ps -eo pid,ppid,pgid,stat,args | grep -E '${process_pattern}' | grep -v grep" 2>/dev/null || true) + if [ -n "$remaining" ]; then + echo "ERROR: Model processes remain in ${CONTAINER_NAME}:" >&2 + echo "$remaining" >&2 + return 1 + fi + return 0 +} + check_vllm_server_ready(){ local server_log_path=$1 local max_attempts=${2:-120} @@ -658,7 +1292,7 @@ check_vllm_server_ready(){ sleep 2 fi done - + echo "ERROR: Server failed to start within timeout" return 1 } @@ -685,7 +1319,7 @@ check_vllm_proxy_ready(){ sleep 2 fi done - + echo "ERROR: Proxy failed to start within timeout" return 1 } @@ -706,7 +1340,7 @@ wait_for_server_ready() { for i in $(seq 1 $max_attempts); do local response_code response_code=$(curl -o /dev/null -s -w "%{http_code}" "http://$host:$port$endpoint" 2>/dev/null) - + if [ "$response_code" = "200" ]; then echo "Server is ready! Health check returned 200." return 0 @@ -716,10 +1350,10 @@ wait_for_server_ready() { else echo "Waiting... ($i/$max_attempts) - Server not ready yet (response: $response_code)" fi - + sleep 2 done - + echo "ERROR: Server failed to become ready within timeout (last response: $response_code)" return 1 } @@ -747,50 +1381,60 @@ convert_container_path_to_host() { setup_log_directory_dual() { local test_case_name=$1 local model_name_clean=$2 - + setup_log_directory "$TEST_RUN_DIR/logs/$test_case_name/$model_name_clean" - + if [ -n "$REMOTE_IP" ]; then - ${SSH_CMD} $REMOTE_IP "source $REMOTE_TEST_DIR/run/.shrc; cd \$BASE_DIR/scripts && source ./common.sh && setup_log_directory \"\$TEST_RUN_DIR/logs/$test_case_name/$model_name_clean\"" + ${SSH_CMD} "${REMOTE_SSH_TARGET:-$REMOTE_IP}" "source $REMOTE_TEST_DIR/run/.shrc; cd \$BASE_DIR/scripts && source ./common.sh && setup_log_directory \"\$TEST_RUN_DIR/logs/$test_case_name/$model_name_clean\"" fi } cleanup_model_processes() { local pid_dir=$1 local test_case_name=$2 - + echo "===== Killing model processes =====" - + local cleanup_failed=false + if [ -d "$pid_dir" ]; then echo "Cleaning up by PID files in $pid_dir..." for pid_file in "${pid_dir}"/*.pid; do if [ -f "$pid_file" ]; then local service_name=$(basename "$pid_file" .pid) - kill_process "$pid_file" "$service_name" + kill_process "$pid_file" "$service_name" || cleanup_failed=true fi done fi - + + verify_model_processes_stopped || cleanup_failed=true + if [ "$ISREMOTE" == "0" ] && [ -n "$REMOTE_IP" ]; then echo "===== Killing model processes (remote: $REMOTE_IP) =====" - ${SSH_CMD} "$REMOTE_IP" "source $REMOTE_TEST_DIR/run/.shrc; cd \$BASE_DIR/scripts && ./$test_case_name.sh stop_server" 2>/dev/null || true + if ! ${SSH_CMD} "${REMOTE_SSH_TARGET:-$REMOTE_IP}" \ + "source $REMOTE_TEST_DIR/run/.shrc; cd \$BASE_DIR/scripts && ./$test_case_name.sh stop_server"; then + echo "ERROR: Remote model-process cleanup failed" >&2 + cleanup_failed=true + fi fi - + echo "Process cleanup completed." + $cleanup_failed && return 1 + return 0 } collect_remote_log_file() { local model_name_clean=$1 local remote_log_filename=$2 local test_case_name=$3 - + local remote_log_dir="${REMOTE_TEST_DIR}/${TEST_CASE_RESULT_PATH}/${model_name_clean}" local local_log_dir="${BASE_DIR}/${TEST_CASE_RESULT_PATH}/${model_name_clean}" - + echo " Copying remote ${remote_log_filename}..." - scp ${REMOTE_IP}:${remote_log_dir}/${remote_log_filename} \ - ${local_log_dir}/ 2>/dev/null - + ${SCP_CMD:-scp} \ + "${REMOTE_SSH_TARGET:-$REMOTE_IP}:${remote_log_dir}/${remote_log_filename}" \ + "${local_log_dir}/" 2>/dev/null + if [ $? -eq 0 ]; then echo " ✓ Successfully copied ${remote_log_filename} for $model_name_clean" return 0 @@ -804,14 +1448,14 @@ collect_remote_log_file() { validate_json_response_error() { local response=$1 local model_name=${2:-"unknown"} - + if echo "$response" | grep -q "\"object\":\"error\""; then local error_message=$(echo "$response" | grep -o '"message":"[^"]*"' | sed 's/"message":"//' | sed 's/"$//') echo " ERROR: $error_message" >&2 echo " $model_name: Fail" return 1 fi - + return 0 } @@ -819,17 +1463,17 @@ validate_json_response_error() { validate_http_status() { local status_code=$1 local expected_code=${2:-200} - + if [ -z "$status_code" ]; then echo "ERROR: HTTP status code is empty" >&2 return 1 fi - + if ! [[ "$status_code" =~ ^[0-9]+$ ]]; then echo "ERROR: HTTP status code is not a valid number: '$status_code'" >&2 return 1 fi - + if [ "$status_code" -eq "$expected_code" ]; then return 0 else @@ -847,13 +1491,13 @@ validate_response_content() { if [ -z "$json_query" ]; then return 0 fi - + local content=$(echo "$response" | jq -r "$json_query" 2>/dev/null) if [ -z "$content" ] || [ "$content" = "null" ]; then echo "ERROR: Failed to extract content from JSON with query: $json_query" >&2 return 1 fi - + if [ -n "$expected_pattern" ]; then if [[ "${content,,}" =~ ${expected_pattern,,} ]]; then echo "Content validation passed: found '$expected_pattern'" @@ -865,7 +1509,7 @@ validate_response_content() { return 1 fi fi - + echo "Content extracted successfully: $content" return 0 } @@ -875,15 +1519,15 @@ validate_api_response() { local status_code=$2 local json_query=${3:-""} local expected_pattern=${4:-""} - + if ! validate_http_status "$status_code" 200; then return 1 fi - + if ! validate_json_response_error "$response_body"; then return 1 fi - + if [ -n "$json_query" ]; then if ! validate_response_content "$response_body" "$json_query" "$expected_pattern"; then return 1 @@ -891,7 +1535,7 @@ validate_api_response() { else echo "Basic validation passed" fi - + return 0 } @@ -899,24 +1543,24 @@ validate_curl_response_from_log() { local log_file=$1 local model_name=$2 local expected_pattern=${3:-""} - + if [ ! -f "$log_file" ]; then echo " ERROR: Curl response log not found at $log_file" >&2 echo " $model_name: Fail" return 1 fi - + local curl_response=$(cat "$log_file") if [ -z "$curl_response" ]; then echo " ERROR: Curl response log is empty" >&2 echo " $model_name: Fail" return 1 fi - + if ! validate_json_response_error "$curl_response" "$model_name"; then return 1 fi - + if [ -n "$expected_pattern" ]; then if echo "$curl_response" | grep -qEi "$expected_pattern"; then echo " $model_name: Pass (pattern matched)" @@ -928,7 +1572,7 @@ validate_curl_response_from_log() { else echo " $model_name: Pass" fi - + return 0 } @@ -938,40 +1582,40 @@ collect_and_validate_model_results() { local remote_log_filename=$2 local test_case_name=$3 local expected_pattern=${4:-""} - + local all_passed=true - + if [ -z "$REMOTE_IP" ]; then echo "ERROR: No REMOTE_IP specified, skipping result parsing" >&2 return 1 fi - + echo "Getting remote results from remote server..." - + for model in "${models[@]}"; do local model_name_clean=$(sanitize_model_name "$model") - + local remote_log_dir="${REMOTE_TEST_DIR}/${TEST_CASE_RESULT_PATH}/${model_name_clean}" local local_log_dir="${BASE_DIR}/${TEST_CASE_RESULT_PATH}/${model_name_clean}" - + echo "Processing model: $model_name_clean" echo " Remote log dir: $remote_log_dir" echo " Local log dir: $local_log_dir" - + collect_remote_log_file "$model_name_clean" "$remote_log_filename" "$test_case_name" - + local log_file="${local_log_dir}/curl_response.log" echo " Checking results for model: $model" - + if ! validate_curl_response_from_log "$log_file" "$model" "$expected_pattern"; then all_passed=false fi - + echo "" done - + echo "Remote log collection completed" - + if [ "$all_passed" = true ]; then return 0 else @@ -979,15 +1623,61 @@ collect_and_validate_model_results() { fi } -# Echo an offline env prefix when the given model already exists in the -# container's HuggingFace cache, so servers use the local snapshot instead of -# querying the hub (skips downloads and avoids hf-mirror 429 rate limiting). -# Models are pre-cached under MODEL_CACHE (mounted at /root/.cache) on both nodes. +# Echo an offline env prefix only when a complete set of model weights exists. +# A config-only or interrupted snapshot must stay online so Hugging Face can +# resume it instead of failing later with "Cannot find any model weights". hf_offline_prefix() { local model_name=$1 [ -z "$model_name" ] && return 0 local cache_dir="models--$(echo "$model_name" | sed 's#/#--#g')" - if ${docker_exec} "ls /root/.cache/huggingface/hub/${cache_dir}/snapshots/*/config.json >/dev/null 2>&1"; then + if docker exec -i "${CONTAINER_NAME}" python3 - \ + "/root/.cache/huggingface/hub/${cache_dir}" <<'PY' +import glob +import json +import os +import re +import sys + + +def snapshot_complete(snapshot): + if not os.path.isfile(os.path.join(snapshot, "config.json")): + return False + for index_name in ("model.safetensors.index.json", "pytorch_model.bin.index.json"): + index_path = os.path.join(snapshot, index_name) + if not os.path.isfile(index_path): + continue + try: + with open(index_path, encoding="utf-8") as index_file: + weights = set(json.load(index_file).get("weight_map", {}).values()) + except (OSError, ValueError): + return False + return bool(weights) and all( + os.path.isfile(os.path.join(snapshot, weight)) + and os.path.getsize(os.path.join(snapshot, weight)) > 0 + for weight in weights + ) + return any( + os.path.isfile(path) and os.path.getsize(path) > 0 + for pattern in ("*.safetensors", "pytorch_model*.bin", "*.pt") + for path in glob.glob(os.path.join(snapshot, pattern)) + ) + + +repository = sys.argv[1] +ref_path = os.path.join(repository, "refs", "main") +try: + with open(ref_path, encoding="utf-8") as ref_file: + revision = ref_file.read().strip() +except OSError: + sys.exit(1) + +if re.fullmatch(r"[0-9a-fA-F]{40,64}", revision) is None: + sys.exit(1) + +snapshot = os.path.join(repository, "snapshots", revision) +sys.exit(0 if snapshot_complete(snapshot) else 1) +PY + then echo "HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 " fi } @@ -1000,30 +1690,32 @@ launch_sglang_server() { local pid_suffix=$5 local extra_args=${6:-""} local ready_pattern=${7:-"The server is fired up and ready to roll!"} - + if [ -z "$model_path" ] || [ -z "$host" ] || [ -z "$port" ] || [ -z "$log_path" ] || [ -z "$pid_suffix" ]; then echo "ERROR: Missing required parameters for launch_sglang_server" >&2 echo "Usage: launch_sglang_server [extra_args] [ready_pattern]" >&2 return 1 fi - + local offline_prefix=$(hf_offline_prefix "$model_path") - local sglang_cmd="${docker_exec} \"${offline_prefix}python -m sglang.launch_server --model-path ${model_path} --host ${host} --port ${port}" + local sglang_cmd="${offline_prefix}python -m sglang.launch_server --model-path ${model_path} --host ${host} --port ${port}" if [ -n "$extra_args" ]; then sglang_cmd="${sglang_cmd} ${extra_args}" fi - + if [ -n "${MOONCAKE_SGLANG_MEM_FRACTION_STATIC:-}" ] && \ + [[ " $extra_args " != *" --mem-fraction-static "* ]]; then + sglang_cmd="${sglang_cmd} --mem-fraction-static ${MOONCAKE_SGLANG_MEM_FRACTION_STATIC}" + fi + - sglang_cmd="${sglang_cmd} > ${log_path} 2>&1 &\"" - local pid_file="${PID_DIR}/server_${pid_suffix}.pid" local grep_pattern="python -m sglang.launch_server.*${model_path}" - + echo "Starting SGLang Server..." - if ! launch_and_track_process "$sglang_cmd" "$grep_pattern" "$pid_file"; then + if ! launch_and_track_process "$sglang_cmd" "$log_path" "$pid_file" "$grep_pattern"; then return 1 fi - + local host_log_path=$(convert_container_path_to_host "$log_path") if ! check_server_ready_with_pattern "$host_log_path" "$ready_pattern"; then return 1 @@ -1035,7 +1727,7 @@ launch_sglang_server() { return 1 fi echo "${pid_suffix} health check passed" - + return 0 } @@ -1047,45 +1739,43 @@ launch_vllm_server() { local pid_suffix=$5 local extra_args=${6:-""} local env_vars=${7:-""} - + if [ -z "$model_path" ] || [ -z "$host" ] || [ -z "$port" ] || [ -z "$log_path" ] || [ -z "$pid_suffix" ]; then echo "ERROR: Missing required parameters for launch_vllm_server" >&2 echo "Usage: launch_vllm_server [extra_args] [env_vars]" >&2 return 1 fi - + local env_prefix="" if [ -n "$env_vars" ]; then env_prefix="${env_vars} " fi env_prefix="${env_prefix}$(hf_offline_prefix "$model_path")" - - local vllm_cmd="${docker_exec} \"${env_prefix}python3 -m vllm.entrypoints.openai.api_server --model '${model_path}' --host '${host}' --port ${port}" - + + local vllm_cmd="${env_prefix}python3 -m vllm.entrypoints.openai.api_server --model '${model_path}' --host '${host}' --port ${port}" + if [ -n "$extra_args" ]; then vllm_cmd="${vllm_cmd} ${extra_args}" fi - - vllm_cmd="${vllm_cmd} > '${log_path}' 2>&1 &\"" - + local pid_file="${PID_DIR}/server_${pid_suffix}.pid" local grep_pattern="python3 -m vllm.entrypoints.openai.api_server.*${model_path}" - + echo "Starting vLLM Server..." echo "Command: $vllm_cmd" - if ! launch_and_track_process "$vllm_cmd" "$grep_pattern" "$pid_file"; then + if ! launch_and_track_process "$vllm_cmd" "$log_path" "$pid_file" "$grep_pattern"; then return 1 fi - + local host_log_path=$(convert_container_path_to_host "$log_path") if ! check_vllm_server_ready "$host_log_path"; then return 1 fi - + if ! wait_for_server_ready "$host" "$port" "/health"; then return 1 fi - + return 0 } @@ -1096,35 +1786,33 @@ launch_sglang_router() { local port=$4 local log_path=$5 local extra_args=${6:-""} - + if [ -z "$prefill_url" ] || [ -z "$decode_url" ] || [ -z "$host" ] || [ -z "$port" ] || [ -z "$log_path" ]; then echo "ERROR: Missing required parameters for launch_sglang_router" >&2 echo "Usage: launch_sglang_router [extra_args]" >&2 return 1 fi - + echo "===== Starting SGLang Router =====" - - local router_cmd="${docker_exec} \"python3 -m sglang_router.launch_router --pd-disaggregation --prefill ${prefill_url} --decode ${decode_url} --host ${host} --port ${port}" + + local router_cmd="python3 -m sglang_router.launch_router --pd-disaggregation --prefill ${prefill_url} --decode ${decode_url} --host ${host} --port ${port}" if [ -n "$extra_args" ]; then router_cmd="${router_cmd} ${extra_args}" fi - - router_cmd="${router_cmd} > ${log_path} 2>&1 &\"" - + local pid_file="${PID_DIR}/proxy.pid" local grep_pattern="sglang::router" - + echo "Load balancer starting..." echo "Command: $router_cmd" - if ! launch_and_track_process "$router_cmd" "$grep_pattern" "$pid_file"; then + if ! launch_and_track_process "$router_cmd" "$log_path" "$pid_file" "$grep_pattern"; then return 1 fi - + local host_log_path=$(convert_container_path_to_host "$log_path") if ! check_proxy_ready "$host_log_path"; then return 1 fi - + return 0 } diff --git a/scripts/tone_tests/scripts/run_test.sh b/scripts/tone_tests/scripts/run_test.sh index 30ff521bbe..1a7553f867 100755 --- a/scripts/tone_tests/scripts/run_test.sh +++ b/scripts/tone_tests/scripts/run_test.sh @@ -2,30 +2,106 @@ CONTAINER_NAME=${CONTAINER_NAME:-"mooncake-ci-test"} MODEL_CACHE=${MODEL_CACHE:-"/root/.cache"} -REGISTRY_ADDR_SGLANG=${REGISTRY_ADDR_SGLANG:-"lmsysorg/sglang:latest"} -REGISTRY_ADDR_VLLM=${REGISTRY_ADDR_VLLM:-"vllm/vllm-openai:latest"} -USE_HUGGINGFACE_MIRROR=${USE_HUGGINGFACE_MIRROR:-true} +MOONCAKE_RUNTIME_CACHE=${MOONCAKE_RUNTIME_CACHE:-} +HF_TOKEN_FILE=${HF_TOKEN_FILE:-"/etc/mooncake-ci/huggingface.token"} +CI_ACCELERATOR=${CI_ACCELERATOR:-"cuda"} +if [ "$CI_ACCELERATOR" = "rocm" ]; then + : "${REGISTRY_ADDR_SGLANG:?REGISTRY_ADDR_SGLANG is required for ROCm}" + : "${REGISTRY_ADDR_VLLM:?REGISTRY_ADDR_VLLM is required for ROCm}" + : "${MOONCAKE_CI_TIER:?MOONCAKE_CI_TIER is required for ROCm}" + : "${MOONCAKE_RENDER_DEVICES:?MOONCAKE_RENDER_DEVICES is required for ROCm}" + : "${MOONCAKE_GPU_INDICES:?MOONCAKE_GPU_INDICES is required for ROCm}" + : "${MOONCAKE_CPUSET_CPUS:?MOONCAKE_CPUSET_CPUS is required for ROCm}" + : "${MOONCAKE_CPUSET_MEMS:?MOONCAKE_CPUSET_MEMS is required for ROCm}" + : "${MOONCAKE_RDMA_DEVICES:?MOONCAKE_RDMA_DEVICES is required for ROCm}" + : "${MOONCAKE_RDMA_NETDEVS:?MOONCAKE_RDMA_NETDEVS is required for ROCm}" + : "${MOONCAKE_TRANSFER_DEVICE:?MOONCAKE_TRANSFER_DEVICE is required for ROCm}" + : "${MOONCAKE_GID_INDEX:?MOONCAKE_GID_INDEX is required for ROCm}" + : "${MOONCAKE_SGLANG_BASE_GPU_ID:?MOONCAKE_SGLANG_BASE_GPU_ID is required for ROCm}" + : "${MOONCAKE_EPD_ENCODER_GPU_ID:?MOONCAKE_EPD_ENCODER_GPU_ID is required for ROCm}" + : "${MOONCAKE_EPD_PREFILL_GPU_ID:?MOONCAKE_EPD_PREFILL_GPU_ID is required for ROCm}" + : "${MOONCAKE_EPD_DECODE_GPU_ID:?MOONCAKE_EPD_DECODE_GPU_ID is required for ROCm}" + : "${MOONCAKE_VLLM_VISIBLE_DEVICES:?MOONCAKE_VLLM_VISIBLE_DEVICES is required for ROCm}" + : "${MOONCAKE_SGLANG_MEM_FRACTION_STATIC:?MOONCAKE_SGLANG_MEM_FRACTION_STATIC is required for ROCm}" + : "${MOONCAKE_RUNTIME_CACHE:?MOONCAKE_RUNTIME_CACHE is required for ROCm}" + : "${AINIC_VERSION:?AINIC_VERSION is required for ROCm}" + USE_HUGGINGFACE_MIRROR=${USE_HUGGINGFACE_MIRROR:-false} +else + REGISTRY_ADDR_SGLANG=${REGISTRY_ADDR_SGLANG:-"lmsysorg/sglang:latest"} + REGISTRY_ADDR_VLLM=${REGISTRY_ADDR_VLLM:-"vllm/vllm-openai:latest"} + MOONCAKE_SGLANG_BASE_GPU_ID=${MOONCAKE_SGLANG_BASE_GPU_ID:-6} + MOONCAKE_EPD_ENCODER_GPU_ID=${MOONCAKE_EPD_ENCODER_GPU_ID:-0} + MOONCAKE_EPD_PREFILL_GPU_ID=${MOONCAKE_EPD_PREFILL_GPU_ID:-4} + MOONCAKE_EPD_DECODE_GPU_ID=${MOONCAKE_EPD_DECODE_GPU_ID:-6} + MOONCAKE_VLLM_VISIBLE_DEVICES=${MOONCAKE_VLLM_VISIBLE_DEVICES:-6,7} + MOONCAKE_SGLANG_MEM_FRACTION_STATIC=${MOONCAKE_SGLANG_MEM_FRACTION_STATIC:-} + MOONCAKE_CI_TIER=${MOONCAKE_CI_TIER:-"full"} + MOONCAKE_RDMA_DEVICES=${MOONCAKE_RDMA_DEVICES:-} + MOONCAKE_RDMA_NETDEVS=${MOONCAKE_RDMA_NETDEVS:-} + MOONCAKE_TRANSFER_DEVICE=${MOONCAKE_TRANSFER_DEVICE:-} + MOONCAKE_GID_INDEX=${MOONCAKE_GID_INDEX:-} + USE_HUGGINGFACE_MIRROR=${USE_HUGGINGFACE_MIRROR:-true} +fi +MOONCAKE_RENDER_DEVICES=${MOONCAKE_RENDER_DEVICES:-} +MOONCAKE_GPU_INDICES=${MOONCAKE_GPU_INDICES:-} +MOONCAKE_CPUSET_CPUS=${MOONCAKE_CPUSET_CPUS:-} +MOONCAKE_CPUSET_MEMS=${MOONCAKE_CPUSET_MEMS:-} +AINIC_VERSION=${AINIC_VERSION:-} HUGGINGFACE_MIRROR=${HUGGINGFACE_MIRROR:-"https://hf-mirror.com"} USE_MODELSCOPE=${USE_MODELSCOPE:-false} REMOTE_TEST_DIR=${REMOTE_TEST_DIR:-"/tmp/Mooncake_tone/mooncake_ci_test"} LOCAL_IP=${LOCAL_IP} REMOTE_IP=${REMOTE_IP} -ARTIFACT_ID=${ARTIFACT_ID} -GIT_REPO=${GIT_REPO} - -All_TEST_SCRIPTS_SGLANG=( - "test_hicache_storage_mooncake_backend.sh" - "test_disaggregation_different_tp.sh" - "test_1p1d_erdma.sh" - "test_epd_sglang.sh" - "test_moe_mooncake.sh" -) +ARTIFACT_ID=${ARTIFACT_ID:-} +ARTIFACT_ID_SGLANG=${ARTIFACT_ID_SGLANG:-$ARTIFACT_ID} +ARTIFACT_ID_VLLM=${ARTIFACT_ID_VLLM:-$ARTIFACT_ID} +WHEEL_DIR=${WHEEL_DIR:-} +WHEEL_DIR_SGLANG=${WHEEL_DIR_SGLANG:-$WHEEL_DIR} +WHEEL_DIR_VLLM=${WHEEL_DIR_VLLM:-$WHEEL_DIR} +GIT_REPO=${GIT_REPO:-} +MOONCAKE_ENV_UNHEALTHY=false + +if [ "$MOONCAKE_CI_TIER" = "core-4gpu" ]; then + # The upstream heterogeneous-TP test starts TP4 and TP2 workers on the + # same host and therefore needs eight GPUs. Mooncake Elastic EP is also + # CUDA-only, so neither test belongs in the permanent 4+4 ROCm tier. + All_TEST_SCRIPTS_SGLANG=( + "test_hicache_storage_mooncake_backend.sh" + "test_1p1d_erdma.sh" + "test_epd_sglang.sh" + ) +elif [ "$MOONCAKE_CI_TIER" = "full" ]; then + All_TEST_SCRIPTS_SGLANG=( + "test_hicache_storage_mooncake_backend.sh" + "test_disaggregation_different_tp.sh" + "test_1p1d_erdma.sh" + "test_epd_sglang.sh" + "test_moe_mooncake.sh" + ) +else + echo "ERROR: unsupported MOONCAKE_CI_TIER: $MOONCAKE_CI_TIER" >&2 + exit 2 +fi All_TEST_SCRIPTS_VLLM=( "test_vllm_1p1d_erdma.sh" ) -readonly SSH_CMD="ssh -o StrictHostKeyChecking=no" +if [ "$CI_ACCELERATOR" = "rocm" ]; then + # The ROCm cluster uses a dedicated CI identity and pinned host key. Keep + # the serving/RDMA address separate from the SSH management endpoint. + : "${REMOTE_SSH_TARGET:?REMOTE_SSH_TARGET is required for ROCm}" + : "${MOONCAKE_SSH_CONFIG:?MOONCAKE_SSH_CONFIG is required for ROCm}" + SSH_CMD=${SSH_CMD:-"ssh -F ${MOONCAKE_SSH_CONFIG}"} + RSYNC_RSH=${RSYNC_RSH:-"ssh -F ${MOONCAKE_SSH_CONFIG}"} + SCP_CMD=${SCP_CMD:-"scp -F ${MOONCAKE_SSH_CONFIG}"} +else + REMOTE_SSH_TARGET=${REMOTE_SSH_TARGET:-"$REMOTE_IP"} + SSH_CMD=${SSH_CMD:-"ssh -o StrictHostKeyChecking=no"} + RSYNC_RSH=${RSYNC_RSH:-"ssh -o StrictHostKeyChecking=no"} + SCP_CMD=${SCP_CMD:-"scp -o StrictHostKeyChecking=no"} +fi +readonly REMOTE_SSH_TARGET SSH_CMD RSYNC_RSH SCP_CMD TONE_TESTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. && pwd) RUN_DIR="$TONE_TESTS_DIR/run" @@ -34,26 +110,26 @@ RUN_DIR="$TONE_TESTS_DIR/run" get_test_type() { local test_name=$1 - + if [ ! -f "$TONE_TESTS_DIR/scripts/$test_name" ]; then echo "unknown" return 1 fi - + local test_type=$(grep "^TEST_TYPE=" "$TONE_TESTS_DIR/scripts/$test_name" | head -n 1 | cut -d'"' -f2) - + if [ -z "$test_type" ]; then echo "unknown" return 1 fi - + echo "$test_type" return 0 } get_framework_from_test_array() { local test_array_name=$1 - + case $test_array_name in "All_TEST_SCRIPTS_SGLANG") echo "SGLANG" @@ -69,14 +145,14 @@ get_framework_from_test_array() { get_framework_from_test_name() { local test_name=$1 - + for vllm_test in "${All_TEST_SCRIPTS_VLLM[@]}"; do if [ "$test_name" = "$vllm_test" ]; then echo "VLLM" return 0 fi done - + # default SGLANG echo "SGLANG" } @@ -101,11 +177,15 @@ is_base_env_prepared() { return 1 # config file doesn't exist fi - # Check if ARTIFACT_ID matches current environment + # Check if the wheel source matches the current environment. local current_artifact_id=$(grep "^export ARTIFACT_ID=" "$RUN_DIR/.shrc" | cut -d'=' -f2-) + local current_wheel_dir=$(grep "^export WHEEL_DIR=" "$RUN_DIR/.shrc" | cut -d'=' -f2-) if [ "$current_artifact_id" != "$ARTIFACT_ID" ]; then return 1 # ARTIFACT_ID mismatch fi + if [ "$current_wheel_dir" != "$WHEEL_DIR" ]; then + return 1 # self-hosted artifact path mismatch + fi # Check if whl package exists local whl_count=$(find "$RUN_DIR/whls/" -name "*.whl" -type f 2>/dev/null | wc -l) @@ -113,7 +193,7 @@ is_base_env_prepared() { } prepare_single_env(){ - local registry_addr=$1 + local registry_addr=$1 local framework_type=$2 if is_base_env_prepared; then @@ -122,20 +202,25 @@ prepare_single_env(){ fi echo "===== Preparing environment for $framework_type (registry: $registry_addr) =====" - + setup_directory $RUN_DIR setup_directory $RUN_DIR/logs cat > $RUN_DIR/.shrc << EOF # Mooncake CI Test Environment Variables - Main Controller export CONTAINER_NAME=${CONTAINER_NAME} +export CI_ACCELERATOR=${CI_ACCELERATOR} +export MOONCAKE_CI_TIER=${MOONCAKE_CI_TIER} export MODEL_CACHE=${MODEL_CACHE} +export MOONCAKE_RUNTIME_CACHE=${MOONCAKE_RUNTIME_CACHE} +export HF_TOKEN_FILE=${HF_TOKEN_FILE} export REGISTRY_ADDR_SGLANG=${REGISTRY_ADDR_SGLANG} export REGISTRY_ADDR_VLLM=${REGISTRY_ADDR_VLLM} export USE_HUGGINGFACE_MIRROR=${USE_HUGGINGFACE_MIRROR} export HUGGINGFACE_MIRROR=${HUGGINGFACE_MIRROR} export USE_MODELSCOPE=${USE_MODELSCOPE} export ARTIFACT_ID=${ARTIFACT_ID} +export WHEEL_DIR=${WHEEL_DIR} export GIT_REPO=${GIT_REPO} export LOCAL_IP=${LOCAL_IP} export REMOTE_IP=${REMOTE_IP} @@ -143,6 +228,21 @@ export BASE_DIR=${TONE_TESTS_DIR} export TEST_RUN_DIR=${RUN_DIR} export TEST_RESULT_DIR=${RUN_DIR}/logs export REMOTE_TEST_DIR=${REMOTE_TEST_DIR} +export MOONCAKE_RENDER_DEVICES="${MOONCAKE_RENDER_DEVICES}" +export MOONCAKE_GPU_INDICES=${MOONCAKE_GPU_INDICES} +export MOONCAKE_CPUSET_CPUS=${MOONCAKE_CPUSET_CPUS} +export MOONCAKE_CPUSET_MEMS=${MOONCAKE_CPUSET_MEMS} +export MOONCAKE_SGLANG_BASE_GPU_ID=${MOONCAKE_SGLANG_BASE_GPU_ID} +export MOONCAKE_EPD_ENCODER_GPU_ID=${MOONCAKE_EPD_ENCODER_GPU_ID} +export MOONCAKE_EPD_PREFILL_GPU_ID=${MOONCAKE_EPD_PREFILL_GPU_ID} +export MOONCAKE_EPD_DECODE_GPU_ID=${MOONCAKE_EPD_DECODE_GPU_ID} +export MOONCAKE_VLLM_VISIBLE_DEVICES=${MOONCAKE_VLLM_VISIBLE_DEVICES} +export MOONCAKE_SGLANG_MEM_FRACTION_STATIC=${MOONCAKE_SGLANG_MEM_FRACTION_STATIC} +export MOONCAKE_RDMA_DEVICES=${MOONCAKE_RDMA_DEVICES} +export MOONCAKE_RDMA_NETDEVS=${MOONCAKE_RDMA_NETDEVS} +export MOONCAKE_TRANSFER_DEVICE=${MOONCAKE_TRANSFER_DEVICE} +export MOONCAKE_GID_INDEX=${MOONCAKE_GID_INDEX} +export AINIC_VERSION=${AINIC_VERSION} EOF echo "===== Preparing local machine =====" @@ -157,12 +257,43 @@ EOF return 0 } +validated_remote_test_dir() { + local remote_dir=$1 + local allowed_root + if [ "$CI_ACCELERATOR" = "rocm" ]; then + allowed_root=/var/lib/mooncake-ci/work + else + allowed_root=/tmp/Mooncake_tone/mooncake_ci_test + fi + + if ! [[ "$remote_dir" =~ ^/[A-Za-z0-9._/-]+$ ]]; then + echo "ERROR: REMOTE_TEST_DIR contains unsupported characters: $remote_dir" >&2 + return 1 + fi + + local normalized_root normalized_dir + normalized_root=$(realpath -m -- "$allowed_root") || return 1 + normalized_dir=$(realpath -m -- "$remote_dir") || return 1 + case "$normalized_dir" in + "$normalized_root"|"$normalized_root"/*) + printf '%s\n' "$normalized_dir" + ;; + *) + echo "ERROR: REMOTE_TEST_DIR must remain under $normalized_root: $normalized_dir" >&2 + return 1 + ;; + esac +} + prepare_double_env(){ local registry_addr=$1 local framework_type=$2 echo "===== Preparing Double-Machine Environment (local + remote) =====" + REMOTE_TEST_DIR=$(validated_remote_test_dir "$REMOTE_TEST_DIR") || return 1 + export REMOTE_TEST_DIR + if ! prepare_single_env "$registry_addr" "$framework_type"; then echo "ERROR: prepare_single_env failed" return 1 @@ -174,18 +305,26 @@ prepare_double_env(){ fi echo "Preparing remote machine $REMOTE_IP..." - ${SSH_CMD} $REMOTE_IP "rm -rf ${REMOTE_TEST_DIR} && mkdir -p ${REMOTE_TEST_DIR}" - - rsync -av ${TONE_TESTS_DIR}/ $REMOTE_IP:${REMOTE_TEST_DIR}/ - if [ $? -ne 0 ]; then - echo "Failed to sync files to remote server" + if ! ${SSH_CMD} "$REMOTE_SSH_TARGET" \ + "set -eu; remote_test_dir='${REMOTE_TEST_DIR}'; rm -rf -- \"\$remote_test_dir\"; mkdir -p -- \"\$remote_test_dir\""; then + echo "ERROR: Failed to prepare remote test directory on $REMOTE_IP" >&2 + return 1 + fi + + if ! rsync -av -e "$RSYNC_RSH" "${TONE_TESTS_DIR}/" \ + "$REMOTE_SSH_TARGET:${REMOTE_TEST_DIR}/"; then + echo "ERROR: Failed to sync files to remote server" >&2 return 1 fi - - ${SSH_CMD} $REMOTE_IP "sed -i 's|^export BASE_DIR=.*$|export BASE_DIR=${REMOTE_TEST_DIR}|' ${REMOTE_TEST_DIR}/run/.shrc && \ - sed -i 's|^export TEST_RUN_DIR=.*$|export TEST_RUN_DIR=${REMOTE_TEST_DIR}/run|' ${REMOTE_TEST_DIR}/run/.shrc && \ - sed -i 's|^export TEST_RESULT_DIR=.*$|export TEST_RESULT_DIR=${REMOTE_TEST_DIR}/logs|' ${REMOTE_TEST_DIR}/run/.shrc" - + + if ! ${SSH_CMD} "$REMOTE_SSH_TARGET" \ + "sed -i 's|^export BASE_DIR=.*$|export BASE_DIR=${REMOTE_TEST_DIR}|' ${REMOTE_TEST_DIR}/run/.shrc && \ + sed -i 's|^export TEST_RUN_DIR=.*$|export TEST_RUN_DIR=${REMOTE_TEST_DIR}/run|' ${REMOTE_TEST_DIR}/run/.shrc && \ + sed -i 's|^export TEST_RESULT_DIR=.*$|export TEST_RESULT_DIR=${REMOTE_TEST_DIR}/logs|' ${REMOTE_TEST_DIR}/run/.shrc"; then + echo "ERROR: Failed to configure the remote test environment on $REMOTE_IP" >&2 + return 1 + fi + echo "Remote preparation completed successfully" return 0 @@ -198,6 +337,22 @@ setup_env_for_test() { [ "$target" = "all" ] && type="double" local registry_addr=$(get_registry_addr_for_framework "$framework_type") + case "$framework_type" in + SGLANG) + ARTIFACT_ID=$ARTIFACT_ID_SGLANG + WHEEL_DIR=$WHEEL_DIR_SGLANG + ;; + VLLM) + ARTIFACT_ID=$ARTIFACT_ID_VLLM + WHEEL_DIR=$WHEEL_DIR_VLLM + ;; + *) echo "ERROR: unknown framework type: $framework_type" >&2; return 1 ;; + esac + [ -n "$ARTIFACT_ID" ] || [ -n "$WHEEL_DIR" ] || { + echo "ERROR: no wheel source configured for $framework_type" >&2 + return 1 + } + export ARTIFACT_ID WHEEL_DIR if [ "$type" = "double" ]; then prepare_double_env "$registry_addr" "$framework_type" || return 1 @@ -209,13 +364,13 @@ setup_env_for_test() { if [ "$type" = "double" ]; then echo "Initializing remote node $REMOTE_IP..." - ${SSH_CMD} "$REMOTE_IP" " + ${SSH_CMD} "$REMOTE_SSH_TARGET" " source ${REMOTE_TEST_DIR}/run/.shrc && \ source ${REMOTE_TEST_DIR}/scripts/common.sh && \ setup_node_env '${registry_addr}' " || { echo "ERROR: Remote setup failed"; return 1; } fi - + echo "All environments are ready." } @@ -226,9 +381,14 @@ run_single_test(){ local framework_type=$(get_framework_from_test_name "$test_name") echo "Test $test_name will use framework: $framework_type" - - setup_env_for_test "$test_name" "$framework_type" || return 1 - + + local type + type=$(get_test_type "$test_name") || return 1 + if ! setup_env_for_test "$test_name" "$framework_type"; then + cleanup_test_env "$type" || true + return 1 + fi + source "$RUN_DIR/.shrc" cd "$TONE_TESTS_DIR/scripts" source "./$test_name" @@ -238,19 +398,21 @@ run_single_test(){ local exit_code=0 run_test "$@" || exit_code=1 - + if declare -f parse >/dev/null 2>&1; then parse "$exit_code" || exit_code=1 fi - local type=$(get_test_type "$test_name") - cleanup_test_env "$type" + if ! cleanup_test_env "$type"; then + echo "ERROR: Test cleanup/postflight failed" >&2 + exit_code=1 + fi return $exit_code } run_all_tests(){ local input_tests=$1 - if ! [[ -v "$input_tests" ]]; then + if ! declare -p "$input_tests" >/dev/null 2>&1; then echo "ERROR: Variable '$input_tests' does not exist" return 1 fi @@ -259,12 +421,16 @@ run_all_tests(){ local framework_type=$(get_framework_from_test_array "$input_tests") echo "===== Running All Tests for $framework_type Framework (Double Machine Mode) =====" - - setup_env_for_test "all" "$framework_type" || return 1 - + + if ! setup_env_for_test "all" "$framework_type"; then + MOONCAKE_ENV_UNHEALTHY=true + cleanup_test_env "double" || true + return 1 + fi + source "$RUN_DIR/.shrc" cd "$TONE_TESTS_DIR/scripts" - + local all_passed=true local test_index=0 local test_count=${#tests[@]} @@ -282,11 +448,11 @@ run_all_tests(){ source "./$test_name" local exit_code=0 run_test || exit_code=1 - + if declare -f parse >/dev/null 2>&1; then parse "$exit_code" || exit_code=1 fi - + [ $exit_code -ne 0 ] && all_passed=false # Container is shared across cases in run-all. Do not schedule another @@ -296,11 +462,16 @@ run_all_tests(){ echo "ERROR: Shared test environment is unhealthy after ${test_name}." >&2 echo "ERROR: Skipping ${remaining_count} remaining test case(s); node intervention is required." >&2 all_passed=false + MOONCAKE_ENV_UNHEALTHY=true break fi done - - cleanup_test_env "double" + + if ! cleanup_test_env "double"; then + echo "ERROR: Test cleanup/postflight failed" >&2 + all_passed=false + MOONCAKE_ENV_UNHEALTHY=true + fi $all_passed && return 0 || return 1 } @@ -316,34 +487,46 @@ show_help(){ echo " run-all VLLM - Run all VLLM tests (using VLLM image)" } -case "$1" in - "run-single") - shift - run_single_test "$@" - ;; - "run-all") - shift - if [ -z "$1" ]; then - # No parameter specified, run both SGLANG and VLLM tests - echo "No framework specified, running all SGLANG tests..." - run_all_tests "All_TEST_SCRIPTS_SGLANG" - - echo "Running all VLLM tests..." - run_all_tests "All_TEST_SCRIPTS_VLLM" - else - FRAMEWORK=$1 - if [ "$FRAMEWORK" = "VLLM" ]; then - run_all_tests "All_TEST_SCRIPTS_VLLM" - elif [ "$FRAMEWORK" = "SGLANG" ]; then - run_all_tests "All_TEST_SCRIPTS_SGLANG" - else - echo "ERROR: Unknown framework '$FRAMEWORK'. Use SGLANG or VLLM." +main() { + case "${1:-}" in + "run-single") + shift + run_single_test "$@" + ;; + "run-all") + shift + if [ -z "${1:-}" ]; then + # No parameter specified, run both SGLANG and VLLM tests + all_frameworks_passed=true + echo "No framework specified, running all SGLANG tests..." + run_all_tests "All_TEST_SCRIPTS_SGLANG" || all_frameworks_passed=false + + if $MOONCAKE_ENV_UNHEALTHY; then + echo "ERROR: Skipping VLLM because the shared environment is unhealthy" >&2 + else + echo "Running all VLLM tests..." + run_all_tests "All_TEST_SCRIPTS_VLLM" || all_frameworks_passed=false + fi + $all_frameworks_passed + else + FRAMEWORK=$1 + if [ "$FRAMEWORK" = "VLLM" ]; then + run_all_tests "All_TEST_SCRIPTS_VLLM" + elif [ "$FRAMEWORK" = "SGLANG" ]; then + run_all_tests "All_TEST_SCRIPTS_SGLANG" + else + echo "ERROR: Unknown framework '$FRAMEWORK'. Use SGLANG or VLLM." + show_help + return 1 + fi + fi + ;; + *) show_help - return 1 - fi - fi - ;; - *) - show_help - ;; -esac + ;; + esac +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + main "$@" +fi diff --git a/scripts/tone_tests/scripts/test_1p1d_erdma.sh b/scripts/tone_tests/scripts/test_1p1d_erdma.sh index 4fb6c2b47f..c565a1dc68 100755 --- a/scripts/tone_tests/scripts/test_1p1d_erdma.sh +++ b/scripts/tone_tests/scripts/test_1p1d_erdma.sh @@ -12,13 +12,13 @@ detect_remote_mode mkdir -p "$PID_DIR" start_server() -{ +{ local host local model_name=$1 local model_name_clean=$2 local sglang_server_log_path local mode_name - if [ "$ISREMOTE" == "0" ]; then + if [ "$ISREMOTE" == "0" ]; then host=$LOCAL_IP sglang_server_log_path=/test_run/run/logs/$test_case_name/$model_name_clean/sglang_server_local.log mode_name=prefill @@ -28,7 +28,10 @@ start_server() mode_name=decode fi - local extra_args="--disaggregation-mode $mode_name --tp-size 2 --base-gpu-id=6" + local extra_args="--disaggregation-mode $mode_name --tp-size 2 --base-gpu-id=${MOONCAKE_SGLANG_BASE_GPU_ID:-6}" + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + extra_args="${extra_args} --disaggregation-ib-device=${MOONCAKE_TRANSFER_DEVICE:-ionic_0}" + fi if ! launch_sglang_server "$model_name" "$host" "30001" "$sglang_server_log_path" "$mode_name" "$extra_args"; then return 1 fi @@ -44,7 +47,7 @@ run_proxy(){ echo "ERROR: Failed to start SGLang Router" return 1 fi - + return 0 } @@ -84,16 +87,16 @@ run_single_model() local model_name_clean=$(sanitize_model_name "$model_name") local status=0 - setup_log_directory_dual "$test_case_name" "$model_name_clean" + setup_log_directory_dual "$test_case_name" "$model_name_clean" - echo "===== Run MODEL NAME: $model_name =====" + echo "===== Run MODEL NAME: $model_name =====" # Local start server if ! start_server $model_name $model_name_clean; then echo "ERROR: Failed to start local server for model $model_name" status=1 else # Remote start server - if ! ${SSH_CMD} $REMOTE_IP "source $REMOTE_TEST_DIR/run/.shrc; cd \$BASE_DIR/scripts && ./$test_case_name.sh start_server $model_name $model_name_clean"; then + if ! ${SSH_CMD} "${REMOTE_SSH_TARGET:-$REMOTE_IP}" "source $REMOTE_TEST_DIR/run/.shrc; cd \$BASE_DIR/scripts && ./$test_case_name.sh start_server $model_name $model_name_clean"; then echo "ERROR: Failed to start remote server for model $model_name" status=1 else @@ -113,8 +116,10 @@ run_single_model() fi echo "===== Cleaning up model processes for $model_name =====" - kill_model_processes - sleep 2 + if ! kill_model_processes; then + echo "ERROR: Model process cleanup failed for $model_name" >&2 + status=1 + fi return $status } @@ -127,25 +132,36 @@ run_test() fi echo "===== Running test case: $test_case_name for all supported models =====" + if [ "$#" -gt 0 ]; then + SUPPORT_MODELS=("$@") + fi + local test_failed=false + local model_index=0 + local model_count=${#SUPPORT_MODELS[@]} for model in "${SUPPORT_MODELS[@]}"; do + model_index=$((model_index + 1)) if ! run_single_model "$model"; then echo "ERROR: Test case $test_case_name failed for model $model" test_failed=true fi + if [ "$model_index" -lt "$model_count" ] && ! drain_gpu_between_tests; then + echo "ERROR: Failed to isolate the next model from $model" >&2 + return 1 + fi done if [ "$test_failed" = true ]; then return 1 fi - + return 0 } parse() { echo "===== Parsing test results =====" - + if collect_and_validate_model_results "SUPPORT_MODELS" "sglang_server_remote.log" "$test_case_name"; then save_test_result "$test_case_name" "Pass" "${BASE_DIR}/${TEST_CASE_RESULT_PATH}" echo "✓ Test PASSED" @@ -165,7 +181,7 @@ case "$1" in ;; "stop_server") kill_model_processes - exit 0 + exit $? ;; *) if [ "${BASH_SOURCE[0]}" == "${0}" ]; then diff --git a/scripts/tone_tests/scripts/test_epd_sglang.sh b/scripts/tone_tests/scripts/test_epd_sglang.sh index f345048479..fffbb62caf 100755 --- a/scripts/tone_tests/scripts/test_epd_sglang.sh +++ b/scripts/tone_tests/scripts/test_epd_sglang.sh @@ -17,21 +17,23 @@ start_epd_component() local component_type=$1 # encoder,prefill, decode local model_name=$2 local model_name_clean=$3 - + local host local port local log_path local pid_suffix local extra_args local ready_pattern - case $component_type in "encoder") host=$LOCAL_IP port=30000 log_path="/test_run/run/logs/$test_case_name/$model_name_clean/encoder.log" pid_suffix="encoder" - extra_args="--encoder-only --encoder-transfer-backend mooncake --tp-size 2" + extra_args="--encoder-only --encoder-transfer-backend mooncake --tp-size 2 --base-gpu-id=${MOONCAKE_EPD_ENCODER_GPU_ID:-0}" + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + extra_args="${extra_args} --mooncake-ib-device=${MOONCAKE_TRANSFER_DEVICE}" + fi ready_pattern="Application startup complete." echo "Starting Encoder..." ;; @@ -40,7 +42,10 @@ start_epd_component() port=30002 log_path="/test_run/run/logs/$test_case_name/$model_name_clean/sglang_server_prefill.log" pid_suffix="prefill" - extra_args="--disaggregation-mode prefill --language-only --encoder-urls http://${LOCAL_IP}:30000 --tp-size 2 --encoder-transfer-backend mooncake --base-gpu-id=4" + extra_args="--disaggregation-mode prefill --language-only --encoder-urls http://${LOCAL_IP}:30000 --tp-size 2 --encoder-transfer-backend mooncake --base-gpu-id=${MOONCAKE_EPD_PREFILL_GPU_ID:-4}" + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + extra_args="${extra_args} --disaggregation-ib-device=${MOONCAKE_TRANSFER_DEVICE} --mooncake-ib-device=${MOONCAKE_TRANSFER_DEVICE}" + fi ready_pattern="The server is fired up and ready to roll!" echo "Starting Prefill Server..." ;; @@ -49,7 +54,10 @@ start_epd_component() port=30003 log_path="/test_run/run/logs/$test_case_name/$model_name_clean/sglang_server_decode.log" pid_suffix="decode" - extra_args="--disaggregation-mode decode --tp-size 2 --base-gpu-id=6" + extra_args="--disaggregation-mode decode --tp-size 2 --base-gpu-id=${MOONCAKE_EPD_DECODE_GPU_ID:-6}" + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + extra_args="${extra_args} --disaggregation-ib-device=${MOONCAKE_TRANSFER_DEVICE}" + fi ready_pattern="The server is fired up and ready to roll!" echo "Starting Decode Server..." ;; @@ -58,12 +66,12 @@ start_epd_component() return 1 ;; esac - + if ! launch_sglang_server "$model_name" "$host" "$port" "$log_path" "$pid_suffix" "$extra_args" "$ready_pattern"; then echo "ERROR: Failed to start $component_type" return 1 fi - + return 0 } @@ -76,7 +84,7 @@ run_proxy() echo "ERROR: Failed to start SGLang Router" return 1 fi - + return 0 } @@ -84,8 +92,12 @@ run_request() { local model_name=$1 local image_file_path=${2:-"${BASE_DIR}/assets/test_cat.jpg"} + local request_timeout=60 + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + request_timeout=180 + fi echo "===== Sending Test Request =====" - + if [ ! -f "$image_file_path" ]; then echo "ERROR: Image file not found: $image_file_path" return 1 @@ -125,20 +137,20 @@ with open('$temp_json_file', 'w') as f: print("✓ JSON file generated successfully") EOF - + if [ $? -ne 0 ]; then echo "ERROR: Failed to generate JSON request file" rm -f "$temp_json_file" return 1 fi - + curl_response=$(curl -s -w "\n%{http_code}" -X POST http://127.0.0.1:8000/v1/chat/completions \ -H "Content-Type: application/json" \ -d @$temp_json_file \ - --max-time 60) - + --max-time "$request_timeout") + rm -f "$temp_json_file" - + response_body=$(echo "$curl_response" | head -n -1) status_code=$(echo "$curl_response" | tail -n 1) echo "Curl Response:" @@ -166,7 +178,7 @@ start_local_components() local model_name=$1 local model_name_clean=$2 local components=("encoder" "prefill") - + for component in "${components[@]}"; do start_epd_component "$component" "$model_name" "$model_name_clean" || return 1 done @@ -177,12 +189,12 @@ start_remote_decode() { local model_name=$1 local model_name_clean=$2 - - if ! ${SSH_CMD} $REMOTE_IP "source $REMOTE_TEST_DIR/run/.shrc; cd \$BASE_DIR/scripts && ./$test_case_name.sh start_component decode $model_name $model_name_clean"; then + + if ! ${SSH_CMD} "${REMOTE_SSH_TARGET:-$REMOTE_IP}" "source $REMOTE_TEST_DIR/run/.shrc; cd \$BASE_DIR/scripts && ./$test_case_name.sh start_component decode $model_name $model_name_clean"; then echo "ERROR: Failed to start remote decode component" return 1 fi - + return 0 } @@ -192,9 +204,9 @@ run_single_model() local model_name_clean=$(sanitize_model_name "$model_name") setup_log_directory_dual "$test_case_name" "$model_name_clean" - + echo "===== Run MODEL NAME: $model_name =====" - + # Encoders → Local Prefill → Remote Decode → Router → Test if start_local_components "$model_name" "$model_name_clean" && \ start_remote_decode "$model_name" "$model_name_clean" && \ @@ -207,8 +219,10 @@ run_single_model() fi echo "===== Cleaning up model processes for $model_name =====" - kill_model_processes - sleep 2 + if ! kill_model_processes; then + echo "ERROR: Model process cleanup failed for $model_name" >&2 + status=1 + fi return $status } @@ -219,7 +233,7 @@ run_test() echo "ERROR: Please specify LOCAL_IP and REMOTE_IP" return 1 fi - + echo "===== Running EPD test case: $test_case_name for all supported models =====" local test_failed=false @@ -233,14 +247,14 @@ run_test() if [ "$test_failed" = true ]; then return 1 fi - + return 0 } parse() { echo "===== Parsing test results =====" - + if collect_and_validate_model_results "SUPPORT_MODELS" "sglang_server_decode.log" "$test_case_name" "cat|kitten|feline"; then save_test_result "$test_case_name" "Pass" "${BASE_DIR}/${TEST_CASE_RESULT_PATH}" echo "✓ Test PASSED" @@ -260,7 +274,7 @@ case "$1" in ;; "stop_server") kill_model_processes - exit 0 + exit $? ;; *) if [ "${BASH_SOURCE[0]}" == "${0}" ]; then diff --git a/scripts/tone_tests/scripts/test_hicache_storage_mooncake_backend.sh b/scripts/tone_tests/scripts/test_hicache_storage_mooncake_backend.sh index 8206ea88de..a72097e004 100644 --- a/scripts/tone_tests/scripts/test_hicache_storage_mooncake_backend.sh +++ b/scripts/tone_tests/scripts/test_hicache_storage_mooncake_backend.sh @@ -10,13 +10,18 @@ run_test() { echo "===== Running pytest tests =====" local log_file="${BASE_DIR}/${TEST_CASE_RESULT_PATH}/${test_case_name}.log" + local pytest_env="" + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + pytest_env="SGLANG_IS_IN_CI=1 " + fi echo "Running tests in container and saving output to: $log_file" ${docker_exec} "\ export PYTHONPATH=/sgl-workspace/sglang/test/registered/hicache:\$PYTHONPATH && \ cd /test_run/python && \ - python3 -m pytest test_hicache_storage_mooncake_backend.py -v -s --tb=long" | tee "$log_file" - + ${pytest_env}python3 -m pytest test_hicache_storage_mooncake_backend.py -v -s --tb=long" \ + 2>&1 | tee "$log_file" + return ${PIPESTATUS[0]} } @@ -41,7 +46,7 @@ if [ "${BASH_SOURCE[0]}" == "${0}" ]; then if ! run_test; then exit_code=1 fi - + parse $exit_code exit $? fi diff --git a/scripts/tone_tests/scripts/test_moe_mooncake.sh b/scripts/tone_tests/scripts/test_moe_mooncake.sh index d70cdb3bc2..fb457fe68d 100644 --- a/scripts/tone_tests/scripts/test_moe_mooncake.sh +++ b/scripts/tone_tests/scripts/test_moe_mooncake.sh @@ -8,7 +8,7 @@ BASE_DIR=${BASE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. && pwd)} . ${BASE_DIR}/scripts/common.sh run_test() -{ +{ echo "===== Running pytest tests =====" local log_file="${BASE_DIR}/${TEST_CASE_RESULT_PATH}/${test_case_name}.log" @@ -26,7 +26,8 @@ run_test() ${docker_exec} "\ export PYTHONPATH=/sgl-workspace/sglang:\$PYTHONPATH && \ cd /test_run/python && \ - ${offline_prefix}python3 -m pytest test_moe_mooncake.py -v -s --tb=long" | tee "$log_file" + ${offline_prefix}python3 -m pytest test_moe_mooncake.py -v -s --tb=long" \ + 2>&1 | tee "$log_file" return ${PIPESTATUS[0]} } @@ -52,7 +53,7 @@ if [ "${BASH_SOURCE[0]}" == "${0}" ]; then if ! run_test; then exit_code=1 fi - + parse $exit_code exit $? fi diff --git a/scripts/tone_tests/scripts/test_vllm_1p1d_erdma.sh b/scripts/tone_tests/scripts/test_vllm_1p1d_erdma.sh index 150aa98ecb..580faa0bae 100755 --- a/scripts/tone_tests/scripts/test_vllm_1p1d_erdma.sh +++ b/scripts/tone_tests/scripts/test_vllm_1p1d_erdma.sh @@ -2,7 +2,11 @@ test_case_name="test_vllm_1p1d_erdma" TEST_TYPE="double" -SUPPORT_MODELS=("Qwen/Qwen3-8B" "deepseek-ai/DeepSeek-V2-Lite") +if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + SUPPORT_MODELS=("Qwen/Qwen3-8B") +else + SUPPORT_MODELS=("Qwen/Qwen3-8B" "deepseek-ai/DeepSeek-V2-Lite") +fi PID_DIR=${BASE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. && pwd)}/run/pids/${test_case_name} BASE_DIR=${BASE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. && pwd)} @@ -20,7 +24,7 @@ start_server() local port local kv_role - if [ "$ISREMOTE" == "0" ]; then + if [ "$ISREMOTE" == "0" ]; then host=$LOCAL_IP vllm_server_log_path=/test_run/run/logs/$test_case_name/$model_name_clean/vllm_server_local.log port=8020 @@ -32,12 +36,32 @@ start_server() kv_role="kv_producer" fi - local kv_config_json="{\\\"kv_connector\\\":\\\"MooncakeConnector\\\",\\\"kv_role\\\":\\\"$kv_role\\\"}" + local kv_config_json + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + # This lane is serialized smoke coverage for basic connector + # correctness. It does not exercise concurrent sender workers; that + # remains separate coverage while vllm-project/vllm#44238 is unresolved. + echo "ROCm vLLM coverage: serialized Mooncake connector smoke (num_workers=1)" + kv_config_json="{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"$kv_role\",\"kv_connector_extra_config\":{\"num_workers\":1}}" + else + kv_config_json="{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"$kv_role\"}" + fi + + local env_vars + if [ "${CI_ACCELERATOR:-cuda}" = "rocm" ]; then + # The MI350X default allocates roughly 233 GiB of KV cache per rank, + # producing 72 multi-GiB dma-buf registrations against every visible + # RNIC. The connector smoke test needs only a small cache and the RoCE + # rail selected by the runner profile. + local gpu_memory_utilization=0.3 + env_vars="ROCR_VISIBLE_DEVICES=${MOONCAKE_VLLM_VISIBLE_DEVICES:-0,1} HIP_VISIBLE_DEVICES=${MOONCAKE_VLLM_VISIBLE_DEVICES:-0,1} MC_MAX_CONCURRENT_REG_MR=1 MC_TE_FILTERS=${MOONCAKE_TRANSFER_DEVICE:-ionic_0}" + else + local gpu_memory_utilization=0.85 + env_vars="CUDA_VISIBLE_DEVICES=${MOONCAKE_VLLM_VISIBLE_DEVICES:-6,7}" + fi + + local extra_args="--tensor-parallel-size 2 --max-model-len 32768 --gpu-memory-utilization ${gpu_memory_utilization} --no-enable-prefix-caching --kv-transfer-config '$kv_config_json'" - local extra_args="--tensor-parallel-size 2 --max-model-len 32768 --gpu-memory-utilization 0.85 --no-enable-prefix-caching --kv-transfer-config '$kv_config_json'" - - local env_vars="CUDA_VISIBLE_DEVICES=6,7" - if ! launch_vllm_server "$model_name" "$host" "$port" "$vllm_server_log_path" "$kv_role" "$extra_args" "$env_vars"; then return 1 fi @@ -51,18 +75,28 @@ run_proxy() local proxy_log_path="/test_run/run/logs/$test_case_name/$model_name/proxy.log" echo "===== Proxy Run =====" - + # Get vLLM version and decide which proxy to use local vllm_version=$(${docker_exec} "python3 -c 'import vllm; print(vllm.__version__)' 2>/dev/null" || echo "0.15.0") echo "Detected vLLM version: $vllm_version" - + # Determine proxy script and ready check strategy local proxy_script local ready_pattern local use_health_check=0 if python3 -c "from packaging import version; import sys; sys.exit(0 if version.parse('$vllm_version') >= version.parse('0.16.0') else 1)" 2>/dev/null; then echo "Using Mooncake Connector Proxy (vLLM >= 0.16.0)" - proxy_script="python3 -u /vllm-workspace/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py --prefill http://$REMOTE_IP:8010 --decode http://$LOCAL_IP:8020 --host 0.0.0.0 --port 8000" + local proxy_path + proxy_path=$(${docker_exec} "for path in \ + /app/vllm/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py \ + /vllm-workspace/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py; do \ + if [ -f \"\$path\" ]; then printf '%s' \"\$path\"; break; fi; \ + done") + if [ -z "$proxy_path" ]; then + echo "ERROR: vLLM Mooncake connector proxy not found in a supported image layout" >&2 + return 1 + fi + proxy_script="python3 -u $proxy_path --prefill http://$REMOTE_IP:8010 --decode http://$LOCAL_IP:8020 --host 0.0.0.0 --port 8000" ready_pattern="All prefiller instances are ready." else echo "Using NIXL Proxy (vLLM < 0.16.0)" @@ -72,13 +106,12 @@ run_proxy() fi # Launch proxy - local lb_cmd="${docker_exec} \"$proxy_script > $proxy_log_path 2>&1 &\"" local pid_file="${PID_DIR}/proxy.pid" - # Extract Python script path (second word) and get filename - local grep_pattern=$(echo "$proxy_script" | grep -oE '[^/]+\.py') - + local grep_pattern + grep_pattern=$(echo "$proxy_script" | grep -oE '[^/]+\.py') + echo "Starting proxy server..." - if ! launch_and_track_process "$lb_cmd" "$grep_pattern" "$pid_file"; then + if ! launch_and_track_process "$proxy_script" "$proxy_log_path" "$pid_file" "$grep_pattern"; then return 1 fi @@ -90,7 +123,7 @@ run_proxy() # Additional health check for toy_proxy_server if [ "$use_health_check" -eq 1 ]; then - if ! wait_for_server_ready "$LOCAL_IP" "8000" "/health"; then + if ! wait_for_server_ready "$LOCAL_IP" "8000" "/healthcheck"; then return 1 fi fi @@ -141,14 +174,14 @@ run_single_model() setup_log_directory_dual "$test_case_name" "$model_name_clean" - echo "===== Run MODEL NAME: $model_name =====" + echo "===== Run MODEL NAME: $model_name =====" # Local start server if ! start_server $model_name $model_name_clean; then echo "ERROR: Failed to start local server for model $model_name" status=1 else # Remote start server - if ! ${SSH_CMD} $REMOTE_IP "source $REMOTE_TEST_DIR/run/.shrc; cd \$BASE_DIR/scripts && ./$test_case_name.sh start_server $model_name $model_name_clean"; then + if ! ${SSH_CMD} "${REMOTE_SSH_TARGET:-$REMOTE_IP}" "source $REMOTE_TEST_DIR/run/.shrc; cd \$BASE_DIR/scripts && ./$test_case_name.sh start_server $model_name $model_name_clean"; then echo "ERROR: Failed to start remote server for model $model_name" status=1 else @@ -168,8 +201,10 @@ run_single_model() fi echo "===== Cleaning up model processes for $model_name =====" - kill_model_processes - sleep 2 + if ! kill_model_processes; then + echo "ERROR: Model process cleanup failed for $model_name" >&2 + status=1 + fi return $status } @@ -183,17 +218,24 @@ run_test() echo "===== Running test case: $test_case_name for all supported models =====" local test_failed=false + local model_index=0 + local model_count=${#SUPPORT_MODELS[@]} for model in "${SUPPORT_MODELS[@]}"; do + model_index=$((model_index + 1)) if ! run_single_model "$model"; then echo "ERROR: Test case $test_case_name failed for model $model" test_failed=true fi + if [ "$model_index" -lt "$model_count" ] && ! drain_gpu_between_tests; then + echo "ERROR: Failed to isolate the next model from $model" >&2 + return 1 + fi done if [ "$test_failed" = true ]; then return 1 fi - + return 0 } @@ -220,11 +262,11 @@ case "$1" in ;; "stop_server") kill_model_processes - exit 0 + exit $? ;; *) if [ "${BASH_SOURCE[0]}" == "${0}" ]; then run_test && parse || parse fi ;; -esac \ No newline at end of file +esac diff --git a/scripts/tone_tests/tests/test_common_cleanup.sh b/scripts/tone_tests/tests/test_common_cleanup.sh new file mode 100755 index 0000000000..cc6b1e0d34 --- /dev/null +++ b/scripts/tone_tests/tests/test_common_cleanup.sh @@ -0,0 +1,328 @@ +#!/usr/bin/env bash +set -euo pipefail + +TEST_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +COMMON_SH="${TEST_DIR}/../scripts/common.sh" + +fail() { + echo "FAIL: $*" >&2 + return 1 +} + +test_rocm_stop_failure_is_not_masked() ( + export CI_ACCELERATOR=rocm + export CONTAINER_NAME=mooncake-cleanup-test + export REMOTE_IP= + export MOONCAKE_RENDER_DEVICES=/dev/dri/renderD129 + test_case_name=cleanup_unit + # shellcheck disable=SC1090 + source "$COMMON_SH" + + call_log=$(mktemp) + trap 'rm -f "$call_log"' EXIT + container_present=true + + docker() { + printf '%s\n' "$*" >> "$call_log" + case "$1" in + ps) + if $container_present; then + printf '%s\n' "$CONTAINER_NAME" + fi + ;; + stop) + return 42 + ;; + rm) + container_present=false + return 0 + ;; + *) + fail "unexpected docker command: $*" + ;; + esac + } + wait_gpu_idle() { + echo wait_gpu_idle >> "$call_log" + return 0 + } + verify_no_allocated_gpu_processes() { + echo verify_no_allocated_gpu_processes >> "$call_log" + return 0 + } + + if cleanup_test_env single; then + fail "cleanup succeeded after docker stop failed" + fi + grep -Fq "stop $CONTAINER_NAME" "$call_log" \ + || fail "docker stop was not attempted" + grep -Fq "rm -f $CONTAINER_NAME" "$call_log" \ + || fail "docker rm was not attempted after stop failed" + grep -Fq wait_gpu_idle "$call_log" \ + || fail "GPU drain was skipped after stop failed" + grep -Fq verify_no_allocated_gpu_processes "$call_log" \ + || fail "process postflight was skipped after stop failed" +) + +test_rocm_cleanup_is_idempotent() ( + export CI_ACCELERATOR=rocm + export CONTAINER_NAME=mooncake-cleanup-test + export REMOTE_IP= + export MOONCAKE_RENDER_DEVICES=/dev/dri/renderD129 + test_case_name=cleanup_unit + # shellcheck disable=SC1090 + source "$COMMON_SH" + + call_log=$(mktemp) + trap 'rm -f "$call_log"' EXIT + container_present=true + + docker() { + printf '%s\n' "$*" >> "$call_log" + case "$1" in + ps) + if $container_present; then + printf '%s\n' "$CONTAINER_NAME" + fi + ;; + stop) + return 0 + ;; + rm) + container_present=false + return 0 + ;; + *) + fail "unexpected docker command: $*" + ;; + esac + } + wait_gpu_idle() { return 0; } + verify_no_allocated_gpu_processes() { return 0; } + + cleanup_test_env single || fail "first cleanup failed" + cleanup_test_env single || fail "second cleanup should accept an absent container" + [ "$(grep -Fc "stop $CONTAINER_NAME" "$call_log")" -eq 1 ] \ + || fail "idempotent cleanup stopped the container more than once" +) + +test_remote_rocm_failure_is_aggregated() ( + export CI_ACCELERATOR=rocm + export CONTAINER_NAME=mooncake-cleanup-test + export REMOTE_IP=192.0.2.2 + export REMOTE_SSH_TARGET=mooncake-worker + export REMOTE_TEST_DIR=/var/lib/mooncake-ci/work + export SSH_CMD=mock_ssh + test_case_name=cleanup_unit + # shellcheck disable=SC1090 + source "$COMMON_SH" + + docker() { + case "$1" in + ps) return 0 ;; + *) fail "unexpected local docker command: $*" ;; + esac + } + wait_gpu_idle() { return 0; } + verify_no_allocated_gpu_processes() { return 0; } + mock_ssh() { return 17; } + + if cleanup_test_env double; then + fail "cleanup succeeded after remote postflight failed" + fi +) + +test_rocm_process_postflight_detects_allocated_device_holder() ( + export CI_ACCELERATOR=rocm + export CONTAINER_NAME=mooncake-cleanup-test + temp_dir=$(mktemp -d) + trap 'rm -rf "$temp_dir"' EXIT + export MOONCAKE_RENDER_DEVICES="$temp_dir/dev/dri/renderD129" + export MOONCAKE_KFD_TOPOLOGY_ROOT="$temp_dir/kfd/topology/nodes" + export MOONCAKE_KFD_PROCESS_ROOT="$temp_dir/kfd/proc" + test_case_name=cleanup_unit + # shellcheck disable=SC1090 + source "$COMMON_SH" + + mkdir -p "$temp_dir/dev/dri" \ + "$MOONCAKE_KFD_TOPOLOGY_ROOT/7" \ + "$MOONCAKE_KFD_PROCESS_ROOT/321/queues/9" + : > "$MOONCAKE_RENDER_DEVICES" + printf 'drm_render_minor 129\n' \ + > "$MOONCAKE_KFD_TOPOLOGY_ROOT/7/properties" + printf '42\n' > "$MOONCAKE_KFD_TOPOLOGY_ROOT/7/gpu_id" + printf '42\n' > "$MOONCAKE_KFD_PROCESS_ROOT/321/queues/9/gpuid" + ps() { return 0; } + + if verify_no_allocated_gpu_processes; then + fail "ROCm postflight ignored a process holding an allocated device" + fi +) + +test_rocm_memory_query_requires_every_allocated_gpu() ( + export CI_ACCELERATOR=rocm + export CONTAINER_NAME=mooncake-cleanup-test + export MOONCAKE_GPU_INDICES=0,1 + test_case_name=cleanup_unit + # shellcheck disable=SC1090 + source "$COMMON_SH" + + rocm-smi() { + printf '%s\n' '{"card0":{"VRAM Total Used Memory (B)":0}}' + } + + [ "$(gpu_max_used_mb)" = -1 ] \ + || fail "ROCm memory query accepted an incomplete allocation" +) + +test_rocm_runtime_cache_is_image_scoped() ( + export CONTAINER_NAME=mooncake-cache-test + temp_dir=$(mktemp -d) + trap 'rm -rf "$temp_dir"' EXIT + export MOONCAKE_RUNTIME_CACHE="$temp_dir/runtime-cache" + test_case_name=cleanup_unit + # shellcheck disable=SC1090 + source "$COMMON_SH" + + registry_addr='example/sglang@sha256:0123456789abcdef0123456789abcdef' + prepare_rocm_runtime_cache_args "$registry_addr" \ + || { fail "ROCm runtime cache preparation failed"; return 1; } + + expected_dir="$(cd -P "$MOONCAKE_RUNTIME_CACHE" && pwd)/0123456789abcdef" + for cache_dir in \ + aiter-jit pip tmp torch-extensions torchinductor triton xdg; do + [ -d "$expected_dir/$cache_dir" ] \ + || { fail "missing runtime cache directory: $cache_dir"; return 1; } + done + + args=$(printf '%s\n' "${ROCM_RUNTIME_CACHE_ARGS[@]}") + printf '%s\n' "$args" | grep -Fxq "$expected_dir:/runtime-cache" \ + || { fail "runtime cache bind mount is missing"; return 1; } + printf '%s\n' "$args" | grep -Fxq 'AITER_JIT_DIR=/runtime-cache/aiter-jit' \ + || { fail "AITER JIT cache is not redirected"; return 1; } + printf '%s\n' "$args" | grep -Fxq 'TRITON_CACHE_DIR=/runtime-cache/triton' \ + || { fail "Triton cache is not redirected"; return 1; } +) + +test_rocm_runtime_cache_rejects_unsafe_root() ( + export CONTAINER_NAME=mooncake-cache-test + export MOONCAKE_RUNTIME_CACHE=/ + test_case_name=cleanup_unit + # shellcheck disable=SC1090 + source "$COMMON_SH" + + if prepare_rocm_runtime_cache_args example/sglang:latest; then + fail "ROCm runtime cache accepted the filesystem root" + fi +) + +test_cuda_cleanup_aggregates_both_nodes() ( + export CI_ACCELERATOR=cuda + export CONTAINER_NAME=mooncake-cleanup-test + export REMOTE_IP=192.0.2.2 + test_case_name=cleanup_unit + # shellcheck disable=SC1090 + source "$COMMON_SH" + + call_log=$(mktemp) + trap 'rm -f "$call_log"' EXIT + stop_container() { + printf '%s\n' "$*" >> "$call_log" + if [ "$#" -eq 1 ]; then + return 9 + fi + return 0 + } + + if cleanup_test_env double; then + fail "CUDA cleanup succeeded after the local stop failed" + fi + [ "$(wc -l < "$call_log" | tr -d ' ')" -eq 2 ] \ + || fail "CUDA cleanup did not attempt both nodes" +) + +test_cuda_process_launch_keeps_legacy_pid_tracking() ( + export CI_ACCELERATOR=cuda + export CONTAINER_NAME=mooncake-cleanup-test + test_case_name=cleanup_unit + # shellcheck disable=SC1090 + source "$COMMON_SH" + + temp_dir=$(mktemp -d) + trap 'rm -rf "$temp_dir"' EXIT + call_log="$temp_dir/docker.log" + pid_file="$temp_dir/server.pid" + process_cmd='CUDA_VISIBLE_DEVICES=6,7 python3 -m server --config '\''{"workers":2}'\''' + + docker() { + printf '%s\n' "$*" >> "$call_log" + if [ "$1" = inspect ]; then + printf '100\n' + fi + return 0 + } + ps() { + printf '200 100 python3 -m server --config {"workers":2}\n' + } + sleep() { return 0; } + + launch_and_track_process "$process_cmd" /tmp/server.log "$pid_file" \ + 'python3 -m server' || fail "CUDA legacy process launch failed" + grep -Fq -- "--config '{\"workers\":2}'" "$call_log" \ + || fail "CUDA launch command lost its quoted JSON" + [ "$(cat "$pid_file")" = 200 ] \ + || fail "CUDA launch did not record the legacy host PID" +) + +test_run_all_propagates_cleanup_failure() ( + export CI_ACCELERATOR=cuda + export LOCAL_IP=127.0.0.1 + export REMOTE_IP= + export ARTIFACT_ID=unit-test + export GIT_REPO=example/Mooncake + # shellcheck disable=SC1090 + source "${TEST_DIR}/../scripts/run_test.sh" + + temp_dir=$(mktemp -d) + trap 'rm -rf "$temp_dir"' EXIT + TONE_TESTS_DIR=$temp_dir + RUN_DIR="$temp_dir/run" + mkdir -p "$RUN_DIR" "$TONE_TESTS_DIR/scripts" + printf 'test_case_name="cleanup_caller"\nTEST_TYPE="double"\nrun_test() { return 0; }\nparse() { return 0; }\n' \ + > "$TONE_TESTS_DIR/scripts/fake_test.sh" + UNIT_TESTS=(fake_test.sh) + + setup_env_for_test() { + printf 'export BASE_DIR=%s\nexport TEST_RUN_DIR=%s\n' \ + "$TONE_TESTS_DIR" "$RUN_DIR" > "$RUN_DIR/.shrc" + return 0 + } + cleanup_test_env() { return 1; } + + if run_all_tests UNIT_TESTS; then + fail "run_all_tests masked a cleanup failure" + fi + [ "$MOONCAKE_ENV_UNHEALTHY" = true ] \ + || fail "run_all_tests did not quarantine the shared environment" +) + +run_test_case() { + local name=$1 + if "$name"; then + echo "PASS: $name" + else + echo "FAIL: $name" >&2 + return 1 + fi +} + +run_test_case test_rocm_stop_failure_is_not_masked +run_test_case test_rocm_cleanup_is_idempotent +run_test_case test_remote_rocm_failure_is_aggregated +run_test_case test_rocm_process_postflight_detects_allocated_device_holder +run_test_case test_rocm_memory_query_requires_every_allocated_gpu +run_test_case test_rocm_runtime_cache_is_image_scoped +run_test_case test_rocm_runtime_cache_rejects_unsafe_root +run_test_case test_cuda_cleanup_aggregates_both_nodes +run_test_case test_cuda_process_launch_keeps_legacy_pid_tracking +run_test_case test_run_all_propagates_cleanup_failure diff --git a/scripts/tone_tests/tests/test_hicache_model_selection.py b/scripts/tone_tests/tests/test_hicache_model_selection.py new file mode 100644 index 0000000000..559442fc48 --- /dev/null +++ b/scripts/tone_tests/tests/test_hicache_model_selection.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +import importlib.util +import os +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + + +class FakeHiCacheStorageBaseMixin: + @classmethod + def _get_model_name(cls): + return "upstream-default-model" + + +def load_test_module(): + requests = types.ModuleType("requests") + requests.RequestException = Exception + sys.modules["requests"] = requests + + file_backend = types.ModuleType("test_hicache_storage_file_backend") + file_backend.HiCacheStorageBaseMixin = FakeHiCacheStorageBaseMixin + sys.modules["test_hicache_storage_file_backend"] = file_backend + + sglang = types.ModuleType("sglang") + sglang.__path__ = [] + sglang_test = types.ModuleType("sglang.test") + sglang_test.__path__ = [] + test_utils = types.ModuleType("sglang.test.test_utils") + test_utils.DEFAULT_MLA_MODEL_NAME_FOR_TEST = "upstream-mla-model" + test_utils.CustomTestCase = unittest.TestCase + test_utils.find_available_port = lambda base: base + test_utils.is_in_ci = lambda: False + sys.modules.update( + { + "sglang": sglang, + "sglang.test": sglang_test, + "sglang.test.test_utils": test_utils, + } + ) + + module_path = ( + Path(__file__).resolve().parents[1] + / "python" + / "test_hicache_storage_mooncake_backend.py" + ) + spec = importlib.util.spec_from_file_location("mooncake_hicache_test", module_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {module_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TestModelSelection(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.module = load_test_module() + + def test_cuda_inherits_upstream_default(self): + with mock.patch.dict(os.environ, {"CI_ACCELERATOR": "cuda"}): + self.assertEqual( + self.module.HiCacheStorageMooncakeBackendBaseMixin._get_model_name(), + "upstream-default-model", + ) + + def test_rocm_uses_ci_model(self): + with mock.patch.dict(os.environ, {"CI_ACCELERATOR": "rocm"}): + self.assertEqual( + self.module.HiCacheStorageMooncakeBackendBaseMixin._get_model_name(), + "Qwen/Qwen3-8B", + ) + + +if __name__ == "__main__": + unittest.main()