From 51a0d262fd44b63dd178121eed9920582e9a5e9f Mon Sep 17 00:00:00 2001 From: haic0 <149741444+haic0@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:39:44 -0400 Subject: [PATCH] [CI/Build][ROCm] Move PersonaPlex temporal stress to nightly Assisted-by: GPT-5.6 Sol Signed-off-by: haic0 <149741444+haic0@users.noreply.github.com> Co-authored-by: Cursor Signed-off-by: andyluo7 --- .buildkite/amd/test-amd-merge.yml | 8 +- .buildkite/amd/test-amd-nightly.yml | 33 ++++++ .buildkite/amd/test-amd-ready.yml | 8 +- tests/buildkite/test_amd_pipeline.py | 151 ++++++++++++++++++++++++++- 4 files changed, 197 insertions(+), 3 deletions(-) diff --git a/.buildkite/amd/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml index b9653f64cae..8444cf0d90e 100644 --- a/.buildkite/amd/test-amd-merge.yml +++ b/.buildkite/amd/test-amd-merge.yml @@ -18,7 +18,13 @@ steps: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" + - >- + pytest -sv tests/model_executor + -m 'core_model and cpu and not omni' + --deselect=tests/model_executor/models/personaplex/test_temporal_streaming_hoist.py::test_temporal_streaming_step_matches_legacy_end_to_end + --deselect=tests/model_executor/models/personaplex/test_temporal_streaming_hoist.py::test_mimi_streaming_step_matches_legacy_end_to_end + --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + --shard-id=$$BUILDKITE_PARALLEL_JOB - label: "Simple · Omni Processor Snapshot Smoke" agent_pool: mi300_1 diff --git a/.buildkite/amd/test-amd-nightly.yml b/.buildkite/amd/test-amd-nightly.yml index d0ddb392a6a..e5d34cc039f 100644 --- a/.buildkite/amd/test-amd-nightly.yml +++ b/.buildkite/amd/test-amd-nightly.yml @@ -144,6 +144,39 @@ steps: - export VLLM_ROCM_USE_AITER=0 - bash .buildkite/amd/scripts/run-omni-processor-test.sh + # AMD builds #13509 and #13511 measured the four long-running oracle cases + # at about 57-84 minutes total. Keep enough budget for the slower observation, + # collection, and artifact finalization while running the stress cases serially. + - group: ":performing_arts: PersonaPlex CPU Nightly" + steps: + - label: "Simple · PersonaPlex Temporal Streaming Test" + agent_pool: mi300_1 + mirror_hardwares: [amdproduction] + grade: NonBlocking + timeout_in_minutes: 120 + artifact_paths: + - "artifacts/personaplex-temporal-streaming/**/*" + commands: + - export VLLM_ROCM_USE_AITER=0 + - export PERSONAPLEX_TEMPORAL_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/personaplex-temporal-streaming" + - mkdir -p "$$PERSONAPLEX_TEMPORAL_ARTIFACT_DIR" + - >- + pytest --collect-only -q + tests/model_executor/models/personaplex/test_temporal_streaming_hoist.py::test_temporal_streaming_step_matches_legacy_end_to_end + tests/model_executor/models/personaplex/test_temporal_streaming_hoist.py::test_mimi_streaming_step_matches_legacy_end_to_end + 2>&1 | tee "$$PERSONAPLEX_TEMPORAL_ARTIFACT_DIR/collection.txt" + - >- + timeout --signal=TERM --kill-after=2m 110m + pytest -s -v -ra + tests/model_executor/models/personaplex/test_temporal_streaming_hoist.py::test_temporal_streaming_step_matches_legacy_end_to_end + tests/model_executor/models/personaplex/test_temporal_streaming_hoist.py::test_mimi_streaming_step_matches_legacy_end_to_end + --durations=50 + --junitxml="$$PERSONAPLEX_TEMPORAL_ARTIFACT_DIR/pytest.xml" + 2>&1 | tee "$$PERSONAPLEX_TEMPORAL_ARTIFACT_DIR/pytest.log" + - >- + tail -n 100 "$$PERSONAPLEX_TEMPORAL_ARTIFACT_DIR/pytest.log" > + "$$PERSONAPLEX_TEMPORAL_ARTIFACT_DIR/pytest-summary.txt" + - group: ":full_moon: LingBot-Video MI300 Nightly" steps: - label: "LingBot-Video Dense CFG-off Smoke" diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index bcb16866aab..7bfdfe74fc0 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -18,7 +18,13 @@ steps: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" + - >- + pytest -sv tests/model_executor + -m 'core_model and cpu and not omni' + --deselect=tests/model_executor/models/personaplex/test_temporal_streaming_hoist.py::test_temporal_streaming_step_matches_legacy_end_to_end + --deselect=tests/model_executor/models/personaplex/test_temporal_streaming_hoist.py::test_mimi_streaming_step_matches_legacy_end_to_end + --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + --shard-id=$$BUILDKITE_PARALLEL_JOB - label: "Simple · Omni Processor Snapshot Smoke" agent_pool: mi300_1 diff --git a/tests/buildkite/test_amd_pipeline.py b/tests/buildkite/test_amd_pipeline.py index 69eb5d4ca5c..71c613dda64 100644 --- a/tests/buildkite/test_amd_pipeline.py +++ b/tests/buildkite/test_amd_pipeline.py @@ -13,6 +13,7 @@ AMD_MERGE_PIPELINE = Path(".buildkite/amd/test-amd-merge.yml") AMD_NIGHTLY_PIPELINE = Path(".buildkite/amd/test-amd-nightly.yml") AMD_READY_PIPELINE = Path(".buildkite/amd/test-amd-ready.yml") +AMD_RUNNER = Path(".buildkite/amd/scripts/run-amd-test.sh") AMD_TEMPLATE = Path(".buildkite/amd/test-template-amd-omni.j2") AMD_OMNI_PROCESSOR_RUNNER = Path(".buildkite/amd/scripts/run-omni-processor-test.sh") CUDA_MERGE_PIPELINE = Path(".buildkite/cuda/test-merge.yml") @@ -22,6 +23,14 @@ AR_PAGED_ATTENTION_MARKERS = "core_model and rocm and MI325 and cards_1" JOY_GPU_LABEL = "Diffusion · JoyImage GPU Inference Test" JOY_GPU_PATH = "tests/diffusion/models/joy_image/test_joy_image_gpu.py" +MODEL_EXECUTOR_CPU_MARKERS = "core_model and cpu and not omni" +PERSONAPLEX_TEMPORAL_LABEL = "Simple · PersonaPlex Temporal Streaming Test" +PERSONAPLEX_TEMPORAL_PATH = "tests/model_executor/models/personaplex/test_temporal_streaming_hoist.py" +PERSONAPLEX_TEMPORAL_STRESS_NODES = [ + f"{PERSONAPLEX_TEMPORAL_PATH}::test_temporal_streaming_step_matches_legacy_end_to_end", + f"{PERSONAPLEX_TEMPORAL_PATH}::test_mimi_streaming_step_matches_legacy_end_to_end", +] +PERSONAPLEX_TEMPORAL_ARTIFACTS = "artifacts/personaplex-temporal-streaming/**/*" DIFFUSION_CPU_MARKERS = ( "core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)" ) @@ -61,6 +70,44 @@ def _walk_steps(steps: list[dict]): yield from _walk_steps(step.get("steps", [])) +def _find_steps_selecting_path(pipeline_path: Path, test_path: str) -> list[dict]: + pipeline = yaml.safe_load(pipeline_path.read_text(encoding="utf-8")) + matches = [] + + def walk(steps: list[dict]) -> None: + for step in steps: + if any( + arg == test_path or arg.startswith(f"{test_path}::") + for command in step.get("commands", []) + for arg in split(command) + ): + matches.append(step) + walk(step.get("steps", [])) + + walk(pipeline.get("steps", [])) + return matches + + +def _find_model_executor_cpu_shard_step(pipeline_path: Path) -> dict: + matches = [] + pipeline = yaml.safe_load(pipeline_path.read_text(encoding="utf-8")) + + def walk(steps: list[dict]) -> None: + for step in steps: + commands = step.get("commands", []) + if any( + split(command)[:3] == ["pytest", "-sv", "tests/model_executor"] + and "--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT" in split(command) + for command in commands + ): + matches.append(step) + walk(step.get("steps", [])) + + walk(pipeline.get("steps", [])) + assert len(matches) == 1, f"expected one model executor CPU shard step in {pipeline_path}" + return matches[0] + + def _find_diffusion_cpu_shard_step(pipeline_path: Path) -> dict: pipeline = yaml.safe_load(pipeline_path.read_text(encoding="utf-8")) matches = [] @@ -149,12 +196,95 @@ def test_omni_processor_full_matrix_moves_to_nightly_with_blocking_rocm_smoke() assert 'exit "${pytest_status}"' in runner +@pytest.mark.parametrize( + "pipeline_path", + [AMD_READY_PIPELINE, AMD_MERGE_PIPELINE], + ids=["ready", "merge"], +) +def test_model_executor_cpu_shards_keep_fast_personaplex_and_exclude_slow_stress( + pipeline_path: Path, +) -> None: + step = _find_model_executor_cpu_shard_step(pipeline_path) + pytest_command = next(command for command in step["commands"] if "tests/model_executor" in command) + argv = split(pytest_command) + + assert step["parallelism"] == 3 + assert argv[:3] == ["pytest", "-sv", "tests/model_executor"] + assert argv[argv.index("-m") + 1] == MODEL_EXECUTOR_CPU_MARKERS + assert all(not arg.startswith("--ignore") for arg in argv) + assert [arg.removeprefix("--deselect=") for arg in argv if arg.startswith("--deselect=")] == ( + PERSONAPLEX_TEMPORAL_STRESS_NODES + ) + assert "--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT" in argv + assert "--shard-id=$$BUILDKITE_PARALLEL_JOB" in argv + assert "-k" not in argv + + +def test_personaplex_temporal_stress_has_one_nonblocking_nightly_owner() -> None: + steps = _find_steps_selecting_path( + AMD_NIGHTLY_PIPELINE, + PERSONAPLEX_TEMPORAL_PATH, + ) + assert len(steps) == 1 + step = steps[0] + + assert step["label"] == PERSONAPLEX_TEMPORAL_LABEL + assert step["agent_pool"] == "mi300_1" + assert step["mirror_hardwares"] == ["amdproduction"] + assert step["grade"] == "NonBlocking" + assert step["timeout_in_minutes"] == 120 + assert step["artifact_paths"] == [PERSONAPLEX_TEMPORAL_ARTIFACTS] + assert "depends_on" not in step + assert "retry" not in step + + commands = step["commands"] + all_commands = "\n".join(commands) + pytest_commands = [command for command in commands if "pytest" in split(command)] + collect_commands = [command for command in pytest_commands if "--collect-only" in split(command)] + run_commands = [command for command in pytest_commands if "--collect-only" not in split(command)] + assert len(collect_commands) == 1 + assert len(run_commands) == 1 + + collect_argv = split(collect_commands[0]) + assert collect_argv[:3] == [ + "pytest", + "--collect-only", + "-q", + ] + assert collect_argv[3:5] == PERSONAPLEX_TEMPORAL_STRESS_NODES + assert "$$PERSONAPLEX_TEMPORAL_ARTIFACT_DIR/collection.txt" in collect_commands[0] + + run_argv = split(run_commands[0]) + assert run_argv[:5] == [ + "timeout", + "--signal=TERM", + "--kill-after=2m", + "110m", + "pytest", + ] + assert run_argv[5:8] == [ + "-s", + "-v", + "-ra", + ] + assert run_argv[8:10] == PERSONAPLEX_TEMPORAL_STRESS_NODES + assert "--durations=50" in run_argv + assert "--junitxml=$$PERSONAPLEX_TEMPORAL_ARTIFACT_DIR/pytest.xml" in run_argv + assert "$$PERSONAPLEX_TEMPORAL_ARTIFACT_DIR/pytest.log" in run_commands[0] + assert "-n" not in run_argv + assert "-k" not in run_argv + assert all(not arg.startswith("--ignore") for arg in run_argv) + assert "|| true" not in all_commands + assert "VLLM_CI_ALLOW_NO_TESTS" not in all_commands + assert "$$PERSONAPLEX_TEMPORAL_ARTIFACT_DIR/pytest-summary.txt" in all_commands + + @pytest.mark.parametrize( "pipeline_path", [CUDA_READY_PIPELINE, CUDA_MERGE_PIPELINE], ids=["ready", "merge"], ) -def test_cuda_pr_gates_retain_full_omni_processor_coverage(pipeline_path: Path) -> None: +def test_cuda_pr_gates_retain_full_model_executor_coverage(pipeline_path: Path) -> None: step = _find_step("Simple · Model Executor Test", pipeline_path) assert "source_file_dependencies" not in step @@ -165,6 +295,12 @@ def test_cuda_pr_gates_retain_full_omni_processor_coverage(pipeline_path: Path) assert "tests/model_executor" in argv assert marker_expression == "core_model and cpu" assert "not omni" not in marker_expression + assert all(not arg.startswith("--ignore") for arg in argv) + assert all(not arg.startswith("--deselect") for arg in argv) + + test_source = Path(PERSONAPLEX_TEMPORAL_PATH).read_text(encoding="utf-8") + assert "pytestmark = [pytest.mark.core_model, pytest.mark.cpu]" in test_source + assert "@pytest.mark.slow" not in test_source def test_ar_paged_attention_gpu_lane_is_blocking_and_pinned() -> None: @@ -374,3 +510,16 @@ def test_amd_template_preserves_step_retry_policy() -> None: # policy when the source suite is rendered into the uploaded pipeline. assert template.count("{% if step.retry %}") == 2 assert template.count("{% for retry_rule in step.retry.automatic %}") == 2 + + +def test_amd_runner_preserves_piped_failures_and_empty_collection() -> None: + runner = AMD_RUNNER.read_text(encoding="utf-8") + template = AMD_TEMPLATE.read_text(encoding="utf-8") + + assert "set -euo pipefail" in runner + assert "if /bin/bash -o pipefail -c" in runner + assert 'exit "${test_status}"' in runner + assert 'exit "${exit_code}"' in runner + assert 'VLLM_CI_ALLOW_NO_TESTS:-0}" == "1"' in runner + assert template.count('{% if step.grade and step.grade == "Blocking" %}') == 2 + assert template.count("soft_fail: true") == 2