diff --git a/.buildkite/common/ci_source_file_dependencies.yml b/.buildkite/common/ci_source_file_dependencies.yml index 99bbffa7548..edf0a480216 100644 --- a/.buildkite/common/ci_source_file_dependencies.yml +++ b/.buildkite/common/ci_source_file_dependencies.yml @@ -317,7 +317,7 @@ source_file_dependencies: diffusion_wan22_perf: - *wan22 - - tests/dfx/perf/scripts/run_diffusion_benchmark.py + - tests/dfx/perf/scripts/run_benchmark.py - tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json diffusion_wan22_reliability: @@ -343,7 +343,7 @@ source_file_dependencies: diffusion_cosmos3_perf: - *cosmos3 - - tests/dfx/perf/scripts/run_diffusion_benchmark.py + - tests/dfx/perf/scripts/run_benchmark.py - tests/dfx/perf/tests/test_cosmos3_vllm_omni.json diffusion_qwen_image_function: @@ -416,7 +416,7 @@ source_file_dependencies: diffusion_minimax_h3_perf: - *minimax_h3 - - tests/dfx/perf/scripts/run_diffusion_benchmark.py + - tests/dfx/perf/scripts/run_benchmark.py - tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json diffusion_hunyuan_video_function: @@ -430,7 +430,7 @@ source_file_dependencies: diffusion_hunyuan_video_perf: - *hunyuan_video - - tests/dfx/perf/scripts/run_diffusion_benchmark.py + - tests/dfx/perf/scripts/run_benchmark.py - tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json - tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json @@ -457,6 +457,11 @@ source_file_dependencies: - tests/e2e/online_serving/test_lingbot_video_moe.py - tests/e2e/offline_inference/test_lingbot_world_v2.py + diffusion_lingbot_perf: + - *lingbot_video + - tests/dfx/perf/scripts/run_benchmark.py + - tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json + # Invalid-param jobs: model business code only, plus the reliability scripts. reliability_invalid_param_h100: diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 843b8d9a339..c69dcb14662 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -53,80 +53,62 @@ steps: - label: ":full_moon: Omni · Accuracy Test" source_file_dependencies: omni_qwen3_omni_accuracy timeout_in_minutes: 180 + artifact_paths: + - tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json commands: - export SEED_TTS_WER_EVAL=1 - export SEED_TTS_EVAL_DEVICE=cuda:1 - - | - set +e - pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and H100 and B200 and cards_2" --run-level full_model - EXIT=$$? - buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json" - exit $$EXIT + - pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and H100 and B200 and cards_2" --run-level full_model - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test" source_file_dependencies: omni_minicpmo_4_5_accuracy timeout_in_minutes: 180 + artifact_paths: + - tests/e2e/accuracy/minicpmo_4_5/results/*.json commands: - export SEED_TTS_WER_EVAL=1 - export SEED_TTS_EVAL_DEVICE=cuda:1 - - | - set +e - pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level full_model - EXIT=$$? - buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json" - exit $$EXIT + - pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level full_model - label: ":full_moon: Omni · Perf Test · No Async Chunk" source_file_dependencies: omni_qwen3_omni_perf key: nightly-omni-performance-no-async-chunk timeout_in_minutes: 300 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json -m "H100 and B200 and cards_2" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json -m "H100 and B200 and cards_2" - label: ":full_moon: Omni · Perf Test · Async Chunk" source_file_dependencies: omni_qwen3_omni_perf key: nightly-omni-performance-async-chunk timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and B200 and full_model and cards_2" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and B200 and full_model and cards_2" - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test" source_file_dependencies: omni_minicpmo_4_5_perf key: nightly-omni-performance-minicpmo-4-5 timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json -m "H100 and B200 and cards_1" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json -m "H100 and B200 and cards_1" - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test" source_file_dependencies: omni_minicpmo_4_5_duplex_perf key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json -m "H100 and B200 and cards_1" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json -m "H100 and B200 and cards_1" - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100" source_file_dependencies: omni_qwen3_omni_function @@ -153,29 +135,23 @@ steps: source_file_dependencies: tts_qwen3_tts_perf key: nightly-tts-performance-single timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - export BENCHMARK_DIR=tests/dfx/perf/results - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_1" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_1" - label: ":full_moon: TTS · Perf Test · 2-GPU" source_file_dependencies: tts_qwen3_tts_perf key: nightly-tts-performance-2gpu timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - export BENCHMARK_DIR=tests/dfx/perf/results - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_2" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_2" - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]" key: nightly-tiny-model-test-group @@ -277,33 +253,27 @@ steps: source_file_dependencies: diffusion_qwen_image_perf key: nightly-diffusion-x2iat-performance-qwen-image-single timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/diffusion_result_*.json + - tests/dfx/perf/results/logs/*.log commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_1" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_1" - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU" source_file_dependencies: diffusion_qwen_image_perf key: nightly-diffusion-x2iat-performance-qwen-image-4gpu timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/diffusion_result_*.json + - tests/dfx/perf/results/logs/*.log commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - export CACHE_DIT_VERSION=1.5.0 # Do not pin DIFFUSION_ATTENTION_BACKEND: H100 auto-selects FLASH_ATTN; # B200 rejects explicit FLASH_ATTN without FA4 and uses CUDNN/TRTLLM. - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_4" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_4" - group: ":card_index_dividers: Diffusion X2V Model Test" key: nightly-diffusion-x2v-group @@ -399,15 +369,11 @@ steps: source_file_dependencies: diffusion_wan22_perf key: nightly-diffusion-x2v-performance-single timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_1" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_1" - label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU" source_file_dependencies: @@ -415,16 +381,16 @@ steps: - diffusion_cosmos3_perf key: nightly-diffusion-x2v-performance-2gpu timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_2" + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_2" EXIT1=$$? - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and B200 and cards_2" + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and B200 and cards_2" EXIT2=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi exit $$EXIT2 @@ -432,16 +398,16 @@ steps: source_file_dependencies: diffusion_hunyuan_video_perf key: nightly-diffusion-x2v-performance-hunyuanvideo15 timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)" + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)" EXIT1=$$? - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)" + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)" EXIT2=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi exit $$EXIT2 @@ -449,15 +415,21 @@ steps: source_file_dependencies: diffusion_minimax_h3_perf key: nightly-diffusion-x2v-performance-minimax-h3 timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json -m "H100 and B200 and cards_4" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json -m "H100 and B200 and cards_4" + + - label: ":full_moon: Diffusion X2V · Perf Test · LingBot" + source_file_dependencies: diffusion_lingbot_perf + key: nightly-diffusion-x2v-performance-lingbot + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json -m "H100 and B200 and cards_1" - group: ":card_index_dividers: Diffusion Test" key: nightly-diffusion-test-group @@ -533,6 +505,9 @@ steps: - nightly-diffusion-x2v-performance-single - nightly-diffusion-x2v-performance-2gpu if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1" + artifact_paths: + - tests/dfx/perf/results/*.xlsx + - tests/dfx/perf/results/*.html commands: - pip install openpyxl - export DEFAULT_INPUT_DIR=tests/dfx/perf/results @@ -548,7 +523,5 @@ steps: - python tools/nightly/generate_nightly_perf_excel.py - python tools/nightly/generate_nightly_perf_html.py - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html" - - buildkite-agent artifact upload "tests/dfx/perf/results/*.xlsx" - - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 70ab5935673..5c193945e34 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -12,6 +12,8 @@ steps: depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" timeout_in_minutes: 60 + artifact_paths: + - coverage-core-model-cpu-*.xml.gz commands: - | set +e @@ -19,7 +21,6 @@ steps: pytest -sv tests/ -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:$${REPORT} EXIT=$$? gzip -9 -f "$${REPORT}" - buildkite-agent artifact upload "$${REPORT}.gz" exit $$EXIT mirror_hardwares: l4_1 @@ -79,41 +80,32 @@ steps: source_file_dependencies: omni_qwen3_omni_perf key: weekly-omni-performance-vllm-text timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json mirror_hardwares: h100_2 - label: ":full_moon: Omni · Perf Test · Async Chunk · Random" key: weekly-omni-performance-async-chunk-random timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and slow" - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and slow" mirror_hardwares: h100_2 - label: ":full_moon: Omni · Perf Test · Multi-Replica" source_file_dependencies: omni_qwen3_omni_perf key: weekly-omni-performance-multi-replicas timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json commands: - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json mirror_hardwares: h100_3 - group: ":card_index_dividers: E2E Tests" diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 0c39a3f93a1..601a7bf4bd4 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -286,11 +286,10 @@ steps: HF_TOKEN: "${HF_TOKEN}" DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT diff --git a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md index feb4f38ede7..981e6089e3c 100644 --- a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md @@ -7,16 +7,17 @@ When you want to add L4-level ***performance test*** cases, add entries to JSON | Omni (nightly) | `run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json` (`full_model` without `slow` in `mark`) | | Omni (weekly) | `run_benchmark.py` | `test_qwen3_omni_async_chunk.json` (CUDA only), `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` (`slow` in `mark`; **Perf Test** in `test-weekly.yml`) | | TTS | `run_benchmark.py` | `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json` | -| Diffusion | `run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`, `test_wan22_i2v_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, … | +| Diffusion (`/v1/chat/completions`) | `run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`, … | +| Diffusion (`/v1/images/generations`, `/v1/images/edits`, `/v1/videos`) | `run_benchmark.py` | `test_wan22_i2v_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, `test_lingbot_video_vllm_omni.json`, … | #### How runners pick cases Without **`--test-config-file`**, each runner scans all `*.json` under `tests/dfx/perf/tests/` but only keeps its own model type: -- **`run_benchmark.py`**: omni and TTS cases only (skips diffusion JSON). -- **`run_diffusion_benchmark.py`**: diffusion cases only (skips omni / TTS JSON). +- **`run_benchmark.py`**: cases whose `benchmark_params` use ``dataset_name`` (omni, TTS, and image/video OpenAI generation via `vllm bench serve --omni`). +- **`run_diffusion_benchmark.py`**: cases whose `benchmark_params` use ``dataset`` (diffusion client schema, including custom jsonl). -Diffusion cases are detected when the JSON has `server_type` (typically `"vllm-omni"`) or `"diffusion"` in the `mark` array. Omni / TTS JSON has neither. +Diffusion-script cases are detected by `is_diffusion_perf_config()`: presence of ``dataset`` (without ``dataset_name``) in `benchmark_params`. Marks / `server_type` / endpoint values are not used for this split. #### Running perf cases @@ -60,16 +61,16 @@ Pass **`--test-config-file`** to load one JSON file, or omit it for the bulk sca ##### Overview -| Field | Required | Description | -| ------------------ | -------------- | -------------------------------------------- | -| test_name | Yes | Unique identifier for the test case | -| mark | No | Pytest marks; see **`mark` field** below | -| server_params | Yes | Server-side configuration parameters | -| benchmark_params | Yes | Benchmark running parameters | -| server_type | Diffusion only | Routes case to run_diffusion_benchmark.py | -| benchmark_endpoint | Diffusion only | Benchmark API path | +| Field | Required | Description | +| ------------------ | --------- | ---------------------------------------------------------------------------------------- | +| test_name | Yes | Unique identifier for the test case | +| mark | No | Pytest marks; see **`mark` field** below | +| server_params | Yes | Server-side configuration parameters | +| benchmark_params | Yes | Benchmark running parameters | +| server_type | Diffusion | Only for diffusion-script JSON; omit on omni-bench generation cases | +| benchmark_endpoint | Optional | Legacy diffusion custom-jsonl alias; prefer `benchmark_params[].endpoint` | -Omit `mark` only for configs not meant to be filtered by `-m`. `server_type` is typically `"vllm-omni"`. `benchmark_endpoint` examples: `/v1/videos`, `/v1/images/generations`. +Omit `mark` only for configs not meant to be filtered by `-m`. Cases that call `/v1/images/edits`, `/v1/images/generations`, or `/v1/videos` use the same `benchmark_params` schema as Omni/TTS (`dataset_name`, `endpoint`, `extra_body`) and are executed by `run_benchmark.py` — do not set `server_type` or `task` on those cases. Remaining diffusion cases (usually `/v1/chat/completions`, or custom jsonl) stay on `run_diffusion_benchmark.py` and may keep `server_type`. #### `mark` field @@ -100,7 +101,7 @@ Recommended for L4 perf cases: } ``` -Multi-GPU diffusion (example: Cosmos3 with `cfg-parallel-size=2`): +Multi-GPU generation via omni bench (example: Cosmos3 with `cfg-parallel-size=2`): ```JSON { @@ -110,10 +111,17 @@ Multi-GPU diffusion (example: Cosmos3 with `cfg-parallel-size=2`): "full_model", "diffusion" ], - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/images/generations", "server_params": { "...": "..." }, - "benchmark_params": [ { "name": "1024x1024_steps4", "...": "..." } ] + "benchmark_params": [ + { + "name": "1024x1024_steps4", + "dataset_name": "random", + "endpoint": "/v1/images/generations", + "num_prompts": 3, + "max_concurrency": 1, + "extra_body": { "width": 1024, "height": 1024, "num_inference_steps": 4 } + } + ] } ``` @@ -132,8 +140,8 @@ Result files use the **runtime** hardware label from `get_runtime_resource_label Examples: -- Omni/TTS: `result_{test_name}_{optional_hw}_{dataset}_....json` under `BENCHMARK_DIR` -- Diffusion: one aggregate `diffusion_result_{config_stem}_{optional_hw}_{timestamp}.json` per source JSON file (array of all runs from that file) +- Omni/TTS and OpenAI generation endpoints (`/v1/images/*`, `/v1/videos`): `result_{test_name}_{optional_hw}_{dataset}_....json` under `BENCHMARK_DIR` +- Remaining diffusion (`run_diffusion_benchmark.py`): one aggregate `diffusion_result_{config_stem}_{optional_hw}_{timestamp}.json` per source JSON file #### Local commands @@ -145,6 +153,9 @@ pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "full_model and omni and # Single file (same selectors as the CI Perf steps) pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \ --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json +pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \ + --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json \ + -m "H100 and B200 and cards_2" pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \ --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json \ -m "H100 and full_model and not slow" @@ -203,7 +214,8 @@ You can add any benchmark running parameters you need here. For all optional par 2. For boolean variables in the running parameters, modify them to forms such as ignore_eos: true/false and fill them into the JSON file. 3. Optionally add a `baseline` object (see **Baseline thresholds** below). If you omit `baseline` or leave it empty, the performance test still runs but does not assert metric thresholds from this field. 4. Set `"name"` on each `benchmark_params` entry for stable pytest ids and readable result keys. -5. The qps and concurrency modes are recommended to be mutually exclusive. For detailed explanations, see the table below: +5. Image/video generation cases (`/v1/images/generations`, `/v1/images/edits`, `/v1/videos`) use this same schema: set `endpoint` to the API path, put width/height/steps/frames in `extra_body`, use `dataset_name: random` for text-only inputs or `random-mm` when the request needs a synthetic image/video. Do not set `server_type` or `task`, and do not use `random-request-config` or kebab-case diffusion client fields. +6. The qps and concurrency modes are recommended to be mutually exclusive. For detailed explanations, see the table below: | Parameter | Type | Required | Example/Values | Description | | --------------- | ------------- | -------- | -------------------- | ------------------------------------ | diff --git a/docs/contributing/ci/test_execution_guide.md b/docs/contributing/ci/test_execution_guide.md index d3b02d27f73..4daf9ef8909 100644 --- a/docs/contributing/ci/test_execution_guide.md +++ b/docs/contributing/ci/test_execution_guide.md @@ -165,7 +165,7 @@ Failed jobs: 1/2 pytest -sv tests/dfx/perf/scripts/run_benchmark.py -m "full_model and tts and H100" pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100" pytest -sv tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json - pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json + pytest -sv tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json ``` Nightly **Perf Test** jobs in [``test-nightly.yml``](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-nightly.yml) use ``--test-config-file`` only (no ``-m``). Weekly **Perf Test** in [``test-weekly.yml``](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml) runs ``test_qwen3_omni_vllm_text.json`` and ``test_qwen3_omni_multi_replicas.json`` (JSON ``mark`` includes ``slow``). E2e L4 function tests use ``full_model`` + ``--run-level full_model``. Example: diff --git a/docs/contributing/ci/test_writing_guide.md b/docs/contributing/ci/test_writing_guide.md index 47a62911280..da282608c45 100644 --- a/docs/contributing/ci/test_writing_guide.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -159,7 +159,8 @@ When `mark` is present, it must be an **array** with exactly one ``hardware_mark } ``` -- Local bulk load: `pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"` +- Local bulk load: `pytest -sv tests/dfx/perf/scripts/run_benchmark.py -m "full_model and H100"` (omni/TTS and `/v1/images/*` + `/v1/videos` diffusion) +- Diffusion chat-completions remaining cases: `pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"` - Nightly CI perf steps: `--test-config-file tests/dfx/perf/tests/test__vllm_omni.json` (file selects cases; no `-m`) - Result filenames use **runtime** GPU detection (`get_runtime_resource_label`); `H100` is omitted on the default CI pool diff --git a/recipes/LTX/LTX-2.md b/recipes/LTX/LTX-2.md index 5a6a049eeed..b84bebd6680 100644 --- a/recipes/LTX/LTX-2.md +++ b/recipes/LTX/LTX-2.md @@ -5,7 +5,7 @@ ## Pipelines | `--model-class-name` | Task | Required checkpoint repositories | -|---|---|---| +| --- | --- | --- | | `LTX2Pipeline` | LTX-2 one-stage T2V/I2V | `Lightricks/LTX-2` | | `LTX2TwoStagePipeline` | LTX-2 ordinary two-stage T2V/I2V | `Lightricks/LTX-2` | | `LTX2DistilledOneStagePipeline` | LTX-2 merged-distilled one-stage T2V/I2V | `rootonchair/LTX-2-19b-distilled` | @@ -53,7 +53,7 @@ pipe(req, image=image, prompt=prompt) The consolidation also removes these registry names without aliases: | Removed name | Replacement | -|---|---| +| --- | --- | | `LTX23Pipeline` | `LTX2Pipeline`; checkpoint metadata selects LTX-2.3 | | `LTX2ImageToVideoPipeline` | `LTX2Pipeline` with `image=` | | `LTX23ImageToVideoPipeline` | `LTX2Pipeline` with `image=`; checkpoint metadata selects LTX-2.3 | @@ -67,7 +67,7 @@ offline and serving entrypoints already use named fields and are unaffected. ## One-Stage Defaults | Parameter | LTX-2 | LTX-2.3 | -|---|---:|---:| +| --- | ---: | ---: | | Width × height | 768 × 512 | 768 × 512 | | Frames / frame rate | 121 / 24 | 121 / 24 | | Denoise steps | 40 | 30 | @@ -85,7 +85,7 @@ default to `121`. ## Two-Stage Defaults | Parameter | Ordinary | Full-distilled | -|---|---:|---:| +| --- | ---: | ---: | | Final width × height | 1536 × 1024 | 1536 × 1024 | | Stage 1 width × height | 768 × 512 | 768 × 512 | | Frames / frame rate | 121 / 24 | 121 / 24 | @@ -172,7 +172,7 @@ spatio-temporal guidance (STG), cross-modality guidance, and rescaling. Distilled stages and ordinary Stage 2 are fixed positive-only. | Parameter | Default | Effect | Alias | -|---|---:|---|---| +| --- | ---: | --- | --- | | `video_cfg_scale` | 3.0 | Video text CFG; `1.0` disables it | `video_cfg_guidance_scale` | | `audio_cfg_scale` | 7.0 | Audio text CFG; `1.0` disables it | `audio_cfg_guidance_scale` | | `video_stg_scale` | 1.0 | Video STG; `0.0` disables it | `video_stg_guidance_scale` | @@ -210,7 +210,7 @@ per denoise step: `cond`, `uncond`, `ptb` (STG), and `mod` (cross-modality). The useful balanced configurations are therefore: | `--cfg-parallel-size` | Passes per rank | Guidance-slot utilization | Notes | -|---:|---:|---:|---| +| ---: | ---: | ---: | --- | | `1` | 4 | 100% | Single-rank fused guidance batch | | `2` | 2 | 100% | Recommended two-rank configuration | | `4` | 1 | 100% | One guidance pass per rank | @@ -269,7 +269,7 @@ noted below. ### Complete `forward` Surface | Argument | Type/default | Meaning and constraints | -|---|---|---| +| --- | --- | --- | | `req` | `DiffusionRequestBatch`, required | Only positional argument; contains prompts and per-request sampling parameters. | | `image` | image or batch, `None` | Direct value wins over request images; no image selects T2V. I2V accepts one image per prompt, and a batch cannot mix T2V/I2V. | | `prompt` | string or list, `None` | Positive-text fallback; request prompts win. Mutually exclusive with `prompt_embeds`. | @@ -306,7 +306,7 @@ request prompt payload; LTX guidance fields live in sampling `extra_args`. ### Recipe-Specific Request Capabilities | Override | One-stage | Ordinary two-stage | Distilled two-stage | -|---|---|---|---| +| --- | --- | --- | --- | | Guidance | Supported | Stage 1 only; Stage 2 is positive-only | Fixed positive-only | | Negative prompt/embeddings | Supported | Supported by Stage 1 | Rejected | | `num_inference_steps` | Supported | Controls Stage 1; Stage 2 uses 3 | Fixed at 8 for Stage 1; Stage 2 uses 3 | @@ -357,7 +357,7 @@ bundled offline CLI do not currently expose `sigmas`. - The output audio sample rate comes from the loaded components and is not a request parameter. - For benchmarks, use `tests/dfx/perf/tests/test_ltx2_vllm_omni.json` with - `tests/dfx/perf/scripts/run_diffusion_benchmark.py`. + `tests/dfx/perf/scripts/run_benchmark.py`. - Ordinary and full-distilled two-stage T2V/I2V are supported; HQ execution remains out of scope. diff --git a/tests/benchmarks/metrics/test_metrics.py b/tests/benchmarks/metrics/test_metrics.py index 0aae841afa4..e7d782f9e02 100644 --- a/tests/benchmarks/metrics/test_metrics.py +++ b/tests/benchmarks/metrics/test_metrics.py @@ -6,11 +6,16 @@ """ import math +from types import SimpleNamespace import pytest from vllm.benchmarks.serve import TaskType -from vllm_omni.benchmarks.metrics.metrics import calculate_metrics +from vllm_omni.benchmarks.metrics.metrics import ( + aggregate_stage_durations, + calculate_metrics, + print_stage_durations_metrics, +) from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu] @@ -379,12 +384,7 @@ class _EmptyAwareTokenizer: """ def __call__(self, text, add_special_tokens=False): - class _R: - pass - - r = _R() - r.input_ids = [0] * len(text) - return r + return SimpleNamespace(input_ids=[0] * len(text)) def _make_tts_output(prompt_len: int) -> MixRequestFuncOutput: @@ -588,5 +588,80 @@ def test_image_with_generated_text_still_reports_text_result(capsys): assert " Image Result " in out +def test_aggregate_stage_durations_mean_p50_p99() -> None: + ok_a = MixRequestFuncOutput() + ok_a.success = True + ok_a.stage_durations = {"diffuse": 1.0, "vae.decode": 0.2, "stage_0_gen_ms": 50.0} + ok_b = MixRequestFuncOutput() + ok_b.success = True + ok_b.stage_durations = {"diffuse": 3.0, "vae.decode": 0.4, "stage_1_gen_ms": 80.0} + failed = MixRequestFuncOutput() + failed.success = False + failed.stage_durations = {"diffuse": 99.0} + + summaries = aggregate_stage_durations([ok_a, ok_b, failed]) + assert summaries["stage_durations_mean"]["diffuse"] == pytest.approx(2.0) + assert summaries["stage_durations_p50"]["diffuse"] == pytest.approx(2.0) + assert summaries["stage_durations_mean"]["vae.decode"] == pytest.approx(0.3) + assert "stage_durations_p99" in summaries + assert "stage_0_gen_ms" not in summaries["stage_durations_mean"] + assert "stage_1_gen_ms" not in summaries["stage_durations_mean"] + + +def test_print_stage_durations_metrics(capsys) -> None: + output = MixRequestFuncOutput() + output.success = True + output.stage_durations = { + "Wan22I2VPipeline.diffuse": 1.25, + "Wan22I2VPipeline.text_encoder.forward": 0.4, + "queue_wait_ms": 0.5, + "stage_0_gen_ms": 1000.0, + } + summaries = aggregate_stage_durations([output]) + metrics = SimpleNamespace(**summaries) + print_stage_durations_metrics([99.0], metrics) + out = capsys.readouterr().out + assert "Wan22I2VPipeline" not in out + assert "Mean Diffuse (s):" in out + assert "Median Diffuse (s):" in out + assert "P99 Diffuse (s):" in out + assert "Mean Text Encoder Forward (s):" in out + assert "Mean Queue Wait (ms):" in out + assert "Stage 0 Gen" not in out + assert "stage_0_gen" not in out + + +def test_profiler_stage_durations_print_without_print_stage(capsys) -> None: + output = MixRequestFuncOutput() + output.success = True + output.prompt_len = 8 + output.latency = 1.0 + output.stage_durations = { + "Wan22I2VPipeline.diffuse": 1.25, + "queue_wait_ms": 0.5, + "stage_0_gen_ms": 1000.0, + } + common = dict( + input_requests=[], + outputs=[output], + dur_s=1.0, + tokenizer=None, + selected_percentiles=[50.0, 99.0], + goodput_config_dict={}, + task_type=TaskType.GENERATION, + selected_percentile_metrics=["e2el"], + max_concurrency=None, + request_rate=float("inf"), + benchmark_duration=1.0, + ) + metrics, _ = calculate_metrics(**common, print_stage=False) + shown = capsys.readouterr().out + assert "Mean Diffuse (s):" in shown + assert "Mean Queue Wait (ms):" in shown + assert "Stage 0 Gen" not in shown + assert metrics.stage_durations_mean["Wan22I2VPipeline.diffuse"] == pytest.approx(1.25) + assert "stage_0_gen_ms" not in metrics.stage_durations_mean + + if __name__ == "__main__": pytest.main([__file__, "-v", "-s"]) diff --git a/tests/benchmarks/patch/test_patch.py b/tests/benchmarks/patch/test_patch.py index d2552bbff53..b203fce59d2 100644 --- a/tests/benchmarks/patch/test_patch.py +++ b/tests/benchmarks/patch/test_patch.py @@ -22,12 +22,17 @@ ) from vllm_omni.benchmarks.patch.patch import ( MixRequestFuncOutput, + _add_combined_video_form_references, _add_video_extra_body_to_form, _add_video_reference_to_form, + _apply_image_metrics_from_payload, _apply_stage0_token_timings, _apply_video_metrics_from_payload, _attach_seed_tts_to_request_func_input, _build_benchmark_session, + _extract_stage_durations_from_payload, + _iter_image_reference_inputs, + _iter_video_reference_inputs, _omni_request_timeout_s, async_request_openai_chat_omni_completions, async_request_openai_image_edits_omni, @@ -1359,16 +1364,15 @@ async def __aexit__(self, *_args): return None captured_stream: list[str] = [] - real_add_field = None + import aiohttp + + real_add_field = aiohttp.FormData.add_field def tracking_add_field(self, name, value=None, **kwargs): if name == "stream": captured_stream.append(str(value)) return real_add_field(self, name, value, **kwargs) - import aiohttp - - real_add_field = aiohttp.FormData.add_field mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field) mock_session = mocker.AsyncMock() @@ -1512,8 +1516,7 @@ def tracking_add_field(self, name, value=None, **kwargs): "reference", [ {"image_url": "https://example.com/ref.png"}, - {"file_id": "file-abc"}, - [{"image_url": "https://example.com/a.png"}, {"file_id": "file-xyz"}], + [{"image_url": "https://example.com/a.png"}, {"image_url": "https://example.com/b.png"}], ], ) def test_video_structured_image_reference_serialized_to_form(reference: object, mocker: MockerFixture) -> None: @@ -1541,18 +1544,395 @@ def tracking_add_field(self, name, value=None, **kwargs): assert field_names.count("image_reference") == 1 assert "input_reference" not in field_names payload = next(value for name, value in captured if name == "image_reference") + assert isinstance(payload, (str, bytes, bytearray)) assert json.loads(payload) == reference +def test_video_file_id_reference_is_rejected() -> None: + """file_id is unsupported on the server and must not be sent as image_reference.""" + import aiohttp + + form = aiohttp.FormData() + with pytest.raises(ValueError, match="file_id is not supported yet"): + _add_video_reference_to_form(form, {"file_id": "file-abc"}) + with pytest.raises(ValueError, match="file_id is not supported yet"): + _add_video_reference_to_form(form, [{"image_url": "https://example.com/a.png"}, {"file_id": "file-xyz"}]) + with pytest.raises(ValueError, match="file_id is not supported yet"): + _add_combined_video_form_references(form, None, {"video_reference": {"file_id": "file-vid"}}) + + +def test_image_reference_urls_from_random_mm_content(mocker: MockerFixture) -> None: + """random-mm image_url parts keep an explicit image_reference type.""" + import aiohttp + + content = [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/ref.png"}, + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{_MIN_PNG_B64}"}, + }, + ] + refs = list(_iter_image_reference_inputs(content)) + assert refs == [ + {"image_url": "https://example.com/ref.png"}, + {"image_url": f"data:image/png;base64,{_MIN_PNG_B64}"}, + ] + + captured: list[tuple[str, object]] = [] + real_add_field = aiohttp.FormData.add_field + + def tracking_add_field(self, name, value=None, **kwargs): + captured.append((str(name), value)) + return real_add_field(self, name, value, **kwargs) + + mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field) + form = aiohttp.FormData() + assert _add_video_reference_to_form(form, refs[0]) is True + assert _add_video_reference_to_form(form, refs[1]) is True + + field_names = [name for name, _ in captured] + assert field_names.count("image_reference") == 2 + assert "video_reference" not in field_names + payloads = [] + for name, value in captured: + if name != "image_reference": + continue + assert isinstance(value, (str, bytes, bytearray)) + payloads.append(json.loads(value)) + assert payloads == refs + + +def test_video_reference_urls_from_random_mm_content(mocker: MockerFixture) -> None: + """random-mm data:video_url parts stay on video_reference.""" + import aiohttp + + content = [ + { + "type": "video_url", + "video_url": {"url": "data:video/mp4;base64,AAAA"}, + } + ] + refs = list(_iter_video_reference_inputs(content)) + assert refs == [{"video_url": "data:video/mp4;base64,AAAA"}] + + captured: list[tuple[str, object]] = [] + real_add_field = aiohttp.FormData.add_field + + def tracking_add_field(self, name, value=None, **kwargs): + captured.append((str(name), value)) + return real_add_field(self, name, value, **kwargs) + + mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field) + form = aiohttp.FormData() + assert _add_video_reference_to_form(form, refs[0]) is True + + field_names = [name for name, _ in captured] + assert field_names.count("video_reference") == 1 + assert "input_references" not in field_names + assert "input_reference" not in field_names + assert "image_reference" not in field_names + payload = next(value for name, value in captured if name == "video_reference") + assert isinstance(payload, (str, bytes, bytearray)) + assert json.loads(payload) == {"video_url": "data:video/mp4;base64,AAAA"} + + +def test_video_reference_https_url_from_random_mm_content(mocker: MockerFixture) -> None: + """HTTP(S) video_url parts must stay on video_reference, not image_reference.""" + import aiohttp + + content = [ + { + "type": "video_url", + "video_url": {"url": "https://example.com/ref.mp4"}, + } + ] + refs = list(_iter_video_reference_inputs(content)) + assert refs == [{"video_url": "https://example.com/ref.mp4"}] + + captured: list[tuple[str, object]] = [] + real_add_field = aiohttp.FormData.add_field + + def tracking_add_field(self, name, value=None, **kwargs): + captured.append((str(name), value)) + return real_add_field(self, name, value, **kwargs) + + mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field) + form = aiohttp.FormData() + assert _add_video_reference_to_form(form, refs[0]) is True + + field_names = [name for name, _ in captured] + assert field_names.count("video_reference") == 1 + assert "image_reference" not in field_names + assert "input_references" not in field_names + payload = next(value for name, value in captured if name == "video_reference") + assert isinstance(payload, (str, bytes, bytearray)) + assert json.loads(payload) == {"video_url": "https://example.com/ref.mp4"} + + +@pytest.mark.parametrize( + ("reference", "field_name", "expected"), + [ + ("data:image/png;base64,AAAA", "image_reference", {"image_url": "data:image/png;base64,AAAA"}), + ("data:video/mp4;base64,AAAA", "video_reference", {"video_url": "data:video/mp4;base64,AAAA"}), + ("https://example.com/ref.png", "image_reference", {"image_url": "https://example.com/ref.png"}), + ("https://example.com/ref.mp4", "video_reference", {"video_url": "https://example.com/ref.mp4"}), + ], +) +def test_bare_reference_string_uses_matching_json_field( + reference: str, + field_name: str, + expected: dict[str, str], + mocker: MockerFixture, +) -> None: + """Image and video URLs use matching JSON fields; neither is a file upload.""" + import aiohttp + + captured: list[tuple[str, object]] = [] + real_add_field = aiohttp.FormData.add_field + + def tracking_add_field(self, name, value=None, **kwargs): + captured.append((str(name), value)) + return real_add_field(self, name, value, **kwargs) + + mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field) + assert _add_video_reference_to_form(aiohttp.FormData(), reference) is True + assert [name for name, _ in captured] == [field_name] + payload = captured[0][1] + assert isinstance(payload, (str, bytes, bytearray)) + assert json.loads(payload) == expected + + +def test_bare_video_string_is_not_collected_as_image() -> None: + content = ["https://example.com/ref.mp4", "data:video/mp4;base64,AAAA"] + assert list(_iter_image_reference_inputs(content)) == [] + assert list(_iter_video_reference_inputs(content)) == [ + {"video_url": "https://example.com/ref.mp4"}, + {"video_url": "data:video/mp4;base64,AAAA"}, + ] + + +def test_bare_http_without_media_extension_is_rejected() -> None: + import aiohttp + + with pytest.raises(ValueError, match="image or video extension"): + _add_video_reference_to_form(aiohttp.FormData(), "https://example.com/ref") + + +def test_image_and_inline_video_use_combined_reference_fields(mocker: MockerFixture) -> None: + """Image plus data:video must be image_reference + video_reference, not input_references.""" + import aiohttp + + content = [ + {"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}}, + {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,AAAA"}}, + ] + captured: list[tuple[str, object]] = [] + real_add_field = aiohttp.FormData.add_field + + def tracking_add_field(self, name, value=None, **kwargs): + captured.append((str(name), value)) + return real_add_field(self, name, value, **kwargs) + + mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field) + form = aiohttp.FormData() + _add_combined_video_form_references(form, content) + + field_names = [name for name, _ in captured] + assert field_names.count("image_reference") == 1 + assert field_names.count("video_reference") == 1 + assert "input_references" not in field_names + assert "input_reference" not in field_names + image_payload = next(value for name, value in captured if name == "image_reference") + video_payload = next(value for name, value in captured if name == "video_reference") + assert isinstance(image_payload, (str, bytes, bytearray)) + assert isinstance(video_payload, (str, bytes, bytearray)) + assert json.loads(image_payload) == {"image_url": "https://example.com/ref.png"} + assert json.loads(video_payload) == {"video_url": "data:video/mp4;base64,AAAA"} + + +def test_image_url_and_bare_data_video_string_use_json_fields(mocker: MockerFixture) -> None: + """Image URL plus a bare data:video string must not upload input_references.""" + import aiohttp + + content = [{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}}] + extra_body = {"video_reference": "data:video/mp4;base64,AAAA"} + captured: list[tuple[str, object]] = [] + real_add_field = aiohttp.FormData.add_field + + def tracking_add_field(self, name, value=None, **kwargs): + captured.append((str(name), value)) + return real_add_field(self, name, value, **kwargs) + + mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field) + form = aiohttp.FormData() + _add_combined_video_form_references(form, content, extra_body) + + field_names = [name for name, _ in captured] + assert field_names.count("image_reference") == 1 + assert field_names.count("video_reference") == 1 + assert "input_references" not in field_names + assert "input_reference" not in field_names + image_payload = next(value for name, value in captured if name == "image_reference") + video_payload = next(value for name, value in captured if name == "video_reference") + assert isinstance(image_payload, (str, bytes, bytearray)) + assert isinstance(video_payload, (str, bytes, bytearray)) + assert json.loads(image_payload) == {"image_url": "https://example.com/ref.png"} + assert json.loads(video_payload) == {"video_url": "data:video/mp4;base64,AAAA"} + + captured.clear() + assert _add_video_reference_to_form(form, "data:video/mp4;base64,AAAA") is True + assert [name for name, _ in captured] == ["video_reference"] + + +def _oversized_data_video_url() -> str: + """data:video whose JSON text part is larger than 1MB.""" + return "data:video/mp4;base64," + ("A" * (1024 * 1024)) + + +def test_oversized_data_video_uploads_as_input_references(mocker: MockerFixture) -> None: + """A lone data:video over the 1MB text limit is uploaded, not sent as JSON.""" + import aiohttp + + video_url = _oversized_data_video_url() + captured: list[tuple[str, object]] = [] + real_add_field = aiohttp.FormData.add_field + + def tracking_add_field(self, name, value=None, **kwargs): + captured.append((str(name), value)) + return real_add_field(self, name, value, **kwargs) + + mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field) + assert _add_video_reference_to_form(aiohttp.FormData(), {"video_url": video_url}) is True + uploaded = next(value for name, value in captured if name == "input_references") + assert uploaded == base64.b64decode("A" * (1024 * 1024)) + field_names = [name for name, _ in captured] + assert "video_reference" not in field_names + assert "image_reference" not in field_names + + +def test_oversized_data_video_with_image_stays_json(mocker: MockerFixture) -> None: + """input_references cannot be combined with image_reference, even for a large video.""" + import aiohttp + + content = [{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}}] + extra_body = {"video_reference": _oversized_data_video_url()} + captured: list[tuple[str, object]] = [] + real_add_field = aiohttp.FormData.add_field + + def tracking_add_field(self, name, value=None, **kwargs): + captured.append((str(name), value)) + return real_add_field(self, name, value, **kwargs) + + mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field) + _add_combined_video_form_references(aiohttp.FormData(), content, extra_body) + field_names = [name for name, _ in captured] + assert "image_reference" in field_names + assert "video_reference" in field_names + assert "input_references" not in field_names + + +def test_image_upload_bytes_and_structured_video_use_json_fields(mocker: MockerFixture) -> None: + """Image upload bytes plus a video URL must not use singular input_reference.""" + import aiohttp + + image_bytes = base64.b64decode(_MIN_PNG_B64) + extra_body = { + "image_reference": {"bytes": image_bytes, "content_type": "image/png"}, + "video_reference": {"video_url": "https://example.com/ref.mp4"}, + } + captured: list[tuple[str, object]] = [] + real_add_field = aiohttp.FormData.add_field + + def tracking_add_field(self, name, value=None, **kwargs): + captured.append((str(name), value)) + return real_add_field(self, name, value, **kwargs) + + mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field) + form = aiohttp.FormData() + _add_combined_video_form_references(form, None, extra_body) + + field_names = [name for name, _ in captured] + assert field_names.count("image_reference") == 1 + assert field_names.count("video_reference") == 1 + assert "input_reference" not in field_names + assert "input_references" not in field_names + image_payload = next(value for name, value in captured if name == "image_reference") + video_payload = next(value for name, value in captured if name == "video_reference") + assert isinstance(image_payload, (str, bytes, bytearray)) + assert isinstance(video_payload, (str, bytes, bytearray)) + assert json.loads(image_payload) == {"image_url": f"data:image/png;base64,{_MIN_PNG_B64}"} + assert json.loads(video_payload) == {"video_url": "https://example.com/ref.mp4"} + + def test_video_unsupported_image_reference_raises() -> None: import aiohttp form = aiohttp.FormData() - with pytest.raises(ValueError, match="Unsupported image_reference"): + with pytest.raises(ValueError, match="Unsupported reference"): _add_video_reference_to_form(form, {"not_a_supported_key": "x"}) - with pytest.raises(ValueError, match="Unsupported image_reference"): + with pytest.raises(ValueError, match="Unsupported reference"): _add_video_reference_to_form(form, "/tmp/does-not-exist-ref.png") +def test_extract_stage_durations_from_video_and_image_shapes() -> None: + video_payload = { + "stage_durations": { + "diffuse": 1.5, + "text_encoder.forward": 0.2, + "vae.decode": 0.1, + "stage_0_gen_ms": 4000.0, + } + } + assert _extract_stage_durations_from_payload(video_payload) == video_payload["stage_durations"] + + image_stage_durations = { + "diffuse": 2.0, + "vae.decode": 0.3, + } + image_payload = { + "metrics": {"stage_durations": image_stage_durations}, + "data": [{"b64_json": "x"}], + } + assert _extract_stage_durations_from_payload(image_payload) == image_stage_durations + + +def test_video_metrics_persist_full_stage_durations() -> None: + output = MixRequestFuncOutput() + output.latency = 4.2 + stage_durations = { + "diffuse": 1.5, + "vae.decode": 0.25, + "stage_0_gen_ms": 4000.0, + } + payload = { + "duration_s": 2.0, + "num_frames": 48, + "fps": 24.0, + "stage_durations": stage_durations, + } + _apply_video_metrics_from_payload(output, payload, {}) + assert output.stage_durations == stage_durations + assert output.video_generation_time_ms == pytest.approx(4000.0) + + +def test_image_metrics_persist_stage_durations_from_metrics() -> None: + output = MixRequestFuncOutput() + stage_durations = { + "diffuse": 1.1, + "text_encoder.forward": 0.4, + "vae.decode": 0.2, + } + payload = { + "created": 1, + "data": [{"b64_json": _MIN_PNG_B64}], + "metrics": {"stage_durations": stage_durations}, + } + assert _apply_image_metrics_from_payload(output, payload) == 1 + assert output.stage_durations == stage_durations + + if __name__ == "__main__": pytest.main([__file__, "-v", "-s"]) diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index 864fb7ab382..2015f6389b7 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + import json import os import re @@ -84,17 +87,20 @@ def resolve_pytest_marks(mark_field: Any) -> list[pytest.MarkDecorator]: raise ValueError(f"mark must be a list; got {type(mark_field).__name__}") -def _mark_names(mark_field: Any) -> set[str]: - if isinstance(mark_field, list): - return {str(item) for item in mark_field if isinstance(item, str)} - return set() +def is_diffusion_perf_config(cfg: dict[str, Any]) -> bool: + """True for perf JSON cases intended for ``run_diffusion_benchmark.py``. + Schema split (not marks / endpoints): -def is_diffusion_perf_config(cfg: dict[str, Any]) -> bool: - """True for perf JSON cases intended for ``run_diffusion_benchmark.py``.""" - if cfg.get("server_type") is not None: - return True - return "diffusion" in _mark_names(cfg.get("mark")) + - ``benchmark_params[].dataset`` → diffusion client (``run_diffusion_benchmark.py``) + - ``benchmark_params[].dataset_name`` → ``vllm bench serve --omni`` (``run_benchmark.py``) + """ + for params in cfg.get("benchmark_params") or []: + if not isinstance(params, dict): + continue + if "dataset" in params and "dataset_name" not in params: + return True + return False def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]: @@ -654,6 +660,7 @@ def run_benchmark( random_output_len: Any | None = None, resource_label: str | None = None, num_warmups: int = 2, + benchmark_params_name: str | None = None, ) -> dict[str, Any]: """Run one ``vllm bench serve --omni`` iteration and return parsed metrics. @@ -662,15 +669,35 @@ def run_benchmark( maps keep every hardware bucket, but each metric is reduced to the value for this concurrency / request-rate step. If the benchmark exits without writing a result file, ``result_omni_template.json`` is used as a fallback. + + ``benchmark_params_name`` (the ``name`` field from ``benchmark_params``) is + persisted in the result JSON and filename so multiple generation configs + under one ``test_name`` stay distinguishable (e.g. Wan USP2 832x480 vs + 1280x720). """ current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") ri = _safe_filename_token(random_input_len) ro = _safe_filename_token(random_output_len) hw = resource_label_for_filename(resource_label) + name_token = _safe_filename_token(benchmark_params_name) if benchmark_params_name else "" + if name_token == "na": + name_token = "" + name_parts = [f"result_{test_name}"] if hw: - result_filename = f"result_{test_name}_{hw}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json" - else: - result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json" + name_parts.append(hw) + if name_token: + name_parts.append(name_token) + name_parts.extend( + [ + str(dataset_name), + str(flow), + str(num_prompt), + f"in{ri}", + f"out{ro}", + current_dt, + ] + ) + result_filename = "_".join(name_parts) + ".json" if "--result-filename" in args: print(f"The result file will be overwritten by {result_filename}") command = ( @@ -740,6 +767,9 @@ def _forward_stream(stream) -> None: if random_output_len is not None: result["random_output_len"] = random_output_len result["Hardware"] = hardware_json_value(resource_label) + result["test_name"] = test_name + if benchmark_params_name: + result["name"] = benchmark_params_name with open(result_path, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) return result diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 94df2edc55e..5648ffff5f2 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -50,12 +50,19 @@ def _get_config_file_from_argv() -> str | None: _all_configs = load_benchmark_configs(config_dir=_PERF_TESTS_DIR) BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if not is_diffusion_perf_config(cfg)] print( - f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts case(s) from " + f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts/generation case(s) from " f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} diffusion; " f"use -m to filter, e.g. -m tts)" ) else: - BENCHMARK_CONFIGS = load_benchmark_configs(CONFIG_FILE_PATH) + _loaded = load_benchmark_configs(CONFIG_FILE_PATH) + BENCHMARK_CONFIGS = [cfg for cfg in _loaded if not is_diffusion_perf_config(cfg)] + skipped = len(_loaded) - len(BENCHMARK_CONFIGS) + if skipped: + print( + f"--test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts/generation case(s); " + f"skipped {skipped} remaining diffusion case(s) (chat completions / custom jsonl)" + ) DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy" server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) @@ -93,24 +100,107 @@ def close(self) -> None: stack.close() +# OmniServer defaults for flags not already present in JSON ``serve_args`` / +# ``extra_cli_args``. Add new (flag, value) pairs here rather than special-casing. +_OMNI_DEFAULT_SERVER_ARGS: tuple[tuple[str, str], ...] = ( + ("--stage-init-timeout", "600"), + ("--init-timeout", "900"), +) + + +def _cli_flag_names(cli_args: tuple[str, ...] | list[str]) -> set[str]: + """Return long-option names present in a flat CLI argv list.""" + names: set[str] = set() + for item in cli_args: + token = str(item) + if not token.startswith("--"): + continue + names.add(token.split("=", 1)[0]) + return names + + +def _merge_omni_default_server_args( + extra_cli_args: tuple[str, ...] | list[str], + *, + use_omni: bool, + defaults: tuple[tuple[str, str], ...] = _OMNI_DEFAULT_SERVER_ARGS, +) -> list[str]: + """Fill Omni defaults for flags not already set in JSON-derived CLI args. + + JSON ``serve_args`` / ``extra_cli_args`` win; only missing flags are appended. + """ + if not use_omni: + return [] + present = _cli_flag_names(extra_cli_args) + args: list[str] = [] + for flag, value in defaults: + if flag not in present: + args += [flag, value] + return args + + +def _omni_server_env() -> dict[str, str]: + """Writable video/image storage for ``/v1/videos`` and related generation APIs.""" + result_dir = Path(os.environ.get("BENCHMARK_DIR", "tests/dfx/perf/results")) + storage_path = Path(os.environ.get("VLLM_OMNI_STORAGE_PATH", str(result_dir / "storage"))) + storage_path.mkdir(parents=True, exist_ok=True) + return {"VLLM_OMNI_STORAGE_PATH": str(storage_path)} + + +def _resolve_offline_model(model: str) -> str: + """Resolve HF ids / MiniMax env overrides the same way as the diffusion runner.""" + import huggingface_hub + + from vllm_omni.transformers_utils.repo_utils import hf_api + + if not model or os.path.isdir(model): + return model + + model_env_overrides = { + "MiniMaxAI/MiniMax-H3": "VLLM_TEST_MINIMAX_H3_MODEL", + "MiniMaxAI/MiniMax-H3/FL2VA": "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL", + "MiniMaxAI/MiniMax-H3/Ref2VA": "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL", + } + env_name = model_env_overrides.get(model) + if env_name: + env_model = os.environ.get(env_name) + if env_model: + return env_model + + parts = model.split("/") + if len(parts) >= 3: + repo_id = "/".join(parts[:2]) + subfolder = "/".join(parts[2:]) + snapshot_root = hf_api().snapshot_download( + repo_id, + allow_patterns=[f"{subfolder}/**"], + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ) + return str(Path(snapshot_root) / subfolder) + + if not huggingface_hub.constants.HF_HUB_OFFLINE: + return model + return hf_api().snapshot_download(model, local_files_only=True) + + @contextmanager def _start_omni_server(server_param): test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = server_param + extra = tuple(extra_cli_args or ()) + model = _resolve_offline_model(model) print(f"Starting OmniServer with test: {test_name}, model: {model}") - server_args: list[str] = [] - if use_omni: - server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"] + server_args: list[str] = _merge_omni_default_server_args(extra, use_omni=use_omni) # --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py): # deploy-config sets the base; stage-overrides are applied on top. Both can be set. if stage_config_path: server_args = ["--deploy-config", stage_config_path] + server_args if stage_overrides: server_args = ["--stage-overrides", stage_overrides] + server_args - if extra_cli_args: - server_args = list(extra_cli_args) + server_args - with OmniServer(model, server_args, use_omni=use_omni) as server: + if extra: + server_args = list(extra) + server_args + with OmniServer(model, server_args, use_omni=use_omni, env_dict=_omni_server_env()) as server: server.test_name = test_name print("OmniServer started successfully") yield server @@ -259,11 +349,24 @@ def to_list(value, default=None): "eval_phase", "trust_remote_code", "expected_duplex_audio_turns_per_session", + "name", + "enable_negative_prompt", + "random_request_config", + "num_input_images", + "warmup_requests", + "warmup_concurrency", + "warmup_num_inference_steps", } + param_keys = {str(key).replace("-", "_") for key in params} + if "model" not in param_keys: + args.extend(["--model", str(model)]) + for key, value in params.items(): if key in exclude_keys or value is None: continue + if key in {"extra_body", "extra-body"} and value == {}: + continue arg_name = f"--{key.replace('_', '-')}" @@ -298,6 +401,7 @@ def to_list(value, default=None): random_output_len=params.get("random_output_len"), resource_label=resource_label, num_warmups=_resolve_num_warmups(params, default=2), + benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None, ) assert_result(result, params, num_prompt) @@ -316,5 +420,6 @@ def to_list(value, default=None): random_output_len=params.get("random_output_len"), resource_label=resource_label, num_warmups=_resolve_num_warmups(params, default=max(2, int(concurrency))), + benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None, ) assert_result(result, params, num_prompt) diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index ffcfc45b721..0523b9b3a1a 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -204,11 +204,11 @@ def load_diffusion_benchmark_configs( ) -> list[dict[str, Any]]: """Load one diffusion benchmark JSON, or merge all ``*.json`` under *config_dir*.""" if config_path is not None: - configs = load_configs(config_path) + loaded = load_configs(config_path) source = str(Path(config_path).resolve()) - for cfg in configs: + for cfg in loaded: cfg.setdefault(_DIFFUSION_SOURCE_CONFIG_KEY, source) - return configs + return loaded if config_dir is None: raise ValueError("load_diffusion_benchmark_configs requires config_path or config_dir") configs: list[dict[str, Any]] = [] @@ -231,7 +231,15 @@ def load_diffusion_benchmark_configs( f"use -m to filter, e.g. -m diffusion)" ) else: - BENCHMARK_CONFIGS = load_diffusion_benchmark_configs(CONFIG_FILE_PATH) + _loaded = load_diffusion_benchmark_configs(CONFIG_FILE_PATH) + BENCHMARK_CONFIGS = [cfg for cfg in _loaded if is_diffusion_perf_config(cfg)] + skipped = len(_loaded) - len(BENCHMARK_CONFIGS) + if skipped: + print( + f"--test-config-file: loaded {len(BENCHMARK_CONFIGS)} diffusion case(s); " + f"skipped {skipped} omni-bench generation case(s) " + f"(/v1/images/edits, /v1/images/generations, /v1/videos → run_benchmark.py)" + ) _AGGREGATED_RESULT_FILES_BY_SOURCE: dict[str, Path] = {} diff --git a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json index 5e2b9d3dc7f..662f845f648 100644 --- a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json +++ b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json @@ -5,7 +5,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 2 } @@ -14,8 +17,6 @@ "diffusion" ], "description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/images/generations", "server_params": { "model": "nvidia/Cosmos3-Nano", "serve_args": { @@ -34,24 +35,28 @@ "benchmark_params": [ { "name": "1024x1024_steps4", - "dataset": "random", - "task": "t2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 4, - "num-prompts": 3, - "max-concurrency": 1, - "seed": 42, - "extra-body": { + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/images/generations", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 1024, + "height": 1024, + "num_inference_steps": 4, + "seed": 42, "guidance_scale": 7.0, "flow_shift": 3.0, "guardrails": false }, "baseline": { "H100": { - "throughput_qps": 1.1958, - "latency_mean": 0.843, - "peak_memory_mb_mean": 80000.0 + "request_throughput": 1.1958, + "mean_e2el_ms": 843.0, + "mean_peak_memory_mb": 80000.0 } } } @@ -63,7 +68,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 2 } @@ -72,8 +80,6 @@ "diffusion" ], "description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", "server_params": { "model": "nvidia/Cosmos3-Nano", "serve_args": { @@ -92,23 +98,21 @@ "benchmark_params": [ { "name": "1280x720_frames189_steps4", - "dataset": "random", - "task": "t2v", - "num-prompts": 3, - "max-concurrency": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "prompt": "A robot arm is cleaning a plate in the kitchen.", - "num_inference_steps": 4, - "num_frames": 189, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "seed": 42, "guidance_scale": 6.0, "flow_shift": 10.0, "max_sequence_length": 4096, @@ -118,9 +122,9 @@ }, "baseline": { "H100": { - "throughput_qps": 0.0307, - "latency_mean": 32.7665, - "peak_memory_mb_mean": 28762.4762 + "request_throughput": 0.0307, + "mean_e2el_ms": 32766.5, + "mean_peak_memory_mb": 28762.4762 } } } @@ -132,7 +136,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 2 } @@ -141,8 +148,6 @@ "diffusion" ], "description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", "server_params": { "model": "nvidia/Cosmos3-Nano", "serve_args": { @@ -161,24 +166,30 @@ "benchmark_params": [ { "name": "1280x720_frames189_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 3, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "prompt": "The scene comes to life with smooth, natural motion.", - "num_inference_steps": 4, - "num_frames": 189, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(720, 1280, 1)": 1.0 + }, + "extra_body": { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "seed": 42, "guidance_scale": 6.0, "flow_shift": 10.0, "max_sequence_length": 4096, @@ -186,9 +197,9 @@ }, "baseline": { "H100": { - "throughput_qps": 0.0305, - "latency_mean": 33.1009, - "peak_memory_mb_mean": 29441.1429 + "request_throughput": 0.0305, + "mean_e2el_ms": 33100.9, + "mean_peak_memory_mb": 29441.1429 } } } @@ -200,7 +211,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 2 } @@ -209,8 +223,6 @@ "diffusion" ], "description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", "server_params": { "model": "nvidia/Cosmos3-Nano", "serve_args": { @@ -229,23 +241,30 @@ "benchmark_params": [ { "name": "1280x720_frames189_steps4", - "dataset": "random", - "task": "v2v", - "num-prompts": 3, - "max-concurrency": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "prompt": "Continue the same scene with smooth natural motion.", - "num_inference_steps": 4, - "num_frames": 189, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "video": 1 + }, + "random_mm_bucket_config": { + "(720, 1280, 189)": 1.0 + }, + "extra_body": { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "seed": 42, "guidance_scale": 6.0, "flow_shift": 10.0, "max_sequence_length": 4096, @@ -258,9 +277,9 @@ }, "baseline": { "H100": { - "throughput_qps": 0.0299, - "latency_mean": 33.925, - "peak_memory_mb_mean": 29443.1429 + "request_throughput": 0.0299, + "mean_e2el_ms": 33925.0, + "mean_peak_memory_mb": 29443.1429 } } } diff --git a/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json index 25d91c44fda..f14e77bcb92 100644 --- a/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json @@ -5,7 +5,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 1 } @@ -14,7 +17,6 @@ "diffusion" ], "description": "Single-device baseline", - "server_type": "vllm-omni", "server_params": { "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v", "serve_args": { @@ -25,26 +27,33 @@ "benchmark_params": [ { "name": "832x480_frames33_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 832, - "height": 480, - "num_inference_steps": 4, - "num_frames": 33, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(480, 832, 1)": 1.0 + }, + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "seed": 42, "guidance_scale": 6.0, - "flow_shift": 5.0 + "flow_shift": 5.0, + "negative_prompt": "Negative prompt for benchmarking diffusion models" } } ] @@ -55,7 +64,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 2 } @@ -64,7 +76,6 @@ "diffusion" ], "description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling", - "server_type": "vllm-omni", "server_params": { "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v", "serve_args": { @@ -79,26 +90,33 @@ "benchmark_params": [ { "name": "832x480_frames33_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 832, - "height": 480, - "num_inference_steps": 4, - "num_frames": 33, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(480, 832, 1)": 1.0 + }, + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "seed": 42, "guidance_scale": 6.0, - "flow_shift": 5.0 + "flow_shift": 5.0, + "negative_prompt": "Negative prompt for benchmarking diffusion models" } } ] diff --git a/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json b/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json index 6a3433dbe2c..3a2f66d82c9 100644 --- a/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json @@ -5,7 +5,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 1 } @@ -14,7 +17,6 @@ "diffusion" ], "description": "Single-device baseline", - "server_type": "vllm-omni", "server_params": { "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", "serve_args": { @@ -25,25 +27,24 @@ "benchmark_params": [ { "name": "832x480_frames33_steps4", - "dataset": "random", - "task": "t2v", - "num-prompts": 10, - "max-concurrency": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 832, - "height": 480, - "num_inference_steps": 4, - "num_frames": 33, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "seed": 42, "guidance_scale": 6.0, - "flow_shift": 5.0 + "flow_shift": 5.0, + "negative_prompt": "Negative prompt for benchmarking diffusion models" } } ] @@ -64,7 +65,6 @@ "diffusion" ], "description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling", - "server_type": "vllm-omni", "server_params": { "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", "serve_args": { @@ -79,25 +79,24 @@ "benchmark_params": [ { "name": "832x480_frames33_steps4", - "dataset": "random", - "task": "t2v", - "num-prompts": 10, - "max-concurrency": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 832, - "height": 480, - "num_inference_steps": 4, - "num_frames": 33, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "seed": 42, "guidance_scale": 6.0, - "flow_shift": 5.0 + "flow_shift": 5.0, + "negative_prompt": "Negative prompt for benchmarking diffusion models" } } ] diff --git a/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json b/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json index 07403e78f9a..9f85a14a2d5 100644 --- a/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json +++ b/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json @@ -1,33 +1,50 @@ [ - { - "test_name": "test_lingbot_video_single_device", - "description": "Single-device LingBot dense T2V baseline at 320x192, 9 frames, 2 steps.", - "server_type": "vllm-omni", - "server_params": { - "model": "robbyant/lingbot-video-dense-1.3b", - "serve_args": { - "model-class-name": "LingBotVideoPipeline", - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "320x192_frames9_steps2", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 320, - "height": 192, - "num-frames": 9, - "fps": 24, - "num-inference-steps": 2, - "num-prompts": 3, - "max-concurrency": 1, - "warmup-requests": 1, - "warmup-concurrency": 1, - "warmup-num-inference-steps": 2, - "enable-negative-prompt": true - } - ] - } + { + "test_name": "test_lingbot_video_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device LingBot dense T2V baseline at 320x192, 9 frames, 2 steps.", + "server_params": { + "model": "robbyant/lingbot-video-dense-1.3b", + "serve_args": { + "model-class-name": "LingBotVideoPipeline", + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "320x192_frames9_steps2", + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "num_warmups": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 320, + "height": 192, + "num_frames": 9, + "fps": 24, + "num_inference_steps": 2, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_ltx2_vllm_omni.json b/tests/dfx/perf/tests/test_ltx2_vllm_omni.json index 9b0634a202d..6b4c9eb702b 100644 --- a/tests/dfx/perf/tests/test_ltx2_vllm_omni.json +++ b/tests/dfx/perf/tests/test_ltx2_vllm_omni.json @@ -14,7 +14,6 @@ "diffusion" ], "description": "Single-device baseline with enforce-eager (no torch.compile)", - "server_type": "vllm-omni", "server_params": { "model": "Lightricks/LTX-2", "serve_args": { @@ -25,31 +24,41 @@ "benchmark_params": [ { "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 256, + "height": 256, + "num_frames": 145, + "fps": 24, + "num_inference_steps": 6, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } }, { "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 768, + "height": 480, + "num_frames": 41, + "fps": 24, + "num_inference_steps": 20, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } } ] }, @@ -68,7 +77,6 @@ "diffusion" ], "description": "Single-device with torch.compile (default, no enforce-eager)", - "server_type": "vllm-omni", "server_params": { "model": "Lightricks/LTX-2", "serve_args": { @@ -78,31 +86,41 @@ "benchmark_params": [ { "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 256, + "height": 256, + "num_frames": 145, + "fps": 24, + "num_inference_steps": 6, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } }, { "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 768, + "height": 480, + "num_frames": 41, + "fps": 24, + "num_inference_steps": 20, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } } ] }, @@ -121,7 +139,6 @@ "diffusion" ], "description": "CFG-parallel=2 with enforce-eager", - "server_type": "vllm-omni", "server_params": { "model": "Lightricks/LTX-2", "serve_args": { @@ -133,31 +150,41 @@ "benchmark_params": [ { "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 256, + "height": 256, + "num_frames": 145, + "fps": 24, + "num_inference_steps": 6, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } }, { "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 768, + "height": 480, + "num_frames": 41, + "fps": 24, + "num_inference_steps": 20, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } } ] }, @@ -176,7 +203,6 @@ "diffusion" ], "description": "CFG-parallel=2 with torch.compile", - "server_type": "vllm-omni", "server_params": { "model": "Lightricks/LTX-2", "serve_args": { @@ -187,31 +213,41 @@ "benchmark_params": [ { "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 256, + "height": 256, + "num_frames": 145, + "fps": 24, + "num_inference_steps": 6, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } }, { "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 768, + "height": 480, + "num_frames": 41, + "fps": 24, + "num_inference_steps": 20, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } } ] }, @@ -230,7 +266,6 @@ "diffusion" ], "description": "CacheDiT with enforce-eager", - "server_type": "vllm-omni", "server_params": { "model": "Lightricks/LTX-2", "serve_args": { @@ -242,31 +277,41 @@ "benchmark_params": [ { "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 256, + "height": 256, + "num_frames": 145, + "fps": 24, + "num_inference_steps": 6, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } }, { "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 768, + "height": 480, + "num_frames": 41, + "fps": 24, + "num_inference_steps": 20, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } } ] } diff --git a/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json b/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json index 7dfbe6f3de9..3bf68064eee 100644 --- a/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json +++ b/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json @@ -5,7 +5,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 4 } @@ -14,8 +17,6 @@ "diffusion" ], "description": "MiniMax-H3 T2VA text-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", "server_params": { "model": "MiniMaxAI/MiniMax-H3", "serve_args": { @@ -38,33 +39,30 @@ "benchmark_params": [ { "name": "1344x768_frames209_steps8", - "dataset": "random", - "task": "t2v", - "num-prompts": 3, - "max-concurrency": 1, - "warmup-requests": 1, - "warmup-concurrency": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1344, - "height": 768, - "prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.", - "num_inference_steps": 8, - "num_frames": 209, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "num_warmups": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 1344, + "height": 768, + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "seed": 42, "flow_shift": 12.0, "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}" }, "baseline": { "H100": { - "throughput_qps": 0.0211, - "latency_mean": 38.3252, - "peak_memory_mb_mean": 54219.6 + "request_throughput": 0.0211, + "mean_e2el_ms": 38325.2, + "mean_peak_memory_mb": 54219.6 } } } @@ -76,7 +74,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 4 } @@ -85,8 +86,6 @@ "diffusion" ], "description": "MiniMax-H3 FL2VA first-frame image-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", "server_params": { "model": "MiniMaxAI/MiniMax-H3", "serve_args": { @@ -109,34 +108,39 @@ "benchmark_params": [ { "name": "1344x768_frames209_steps8", - "dataset": "random", - "task": "ti2v", - "num-prompts": 3, - "max-concurrency": 1, - "warmup-requests": 1, - "warmup-concurrency": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1344, - "height": 768, - "prompt": "Continue the scene: the camera slowly pushes in on the subject as the background city lights drift and blur, with the actor singing softly and emotionally.", - "num_inference_steps": 8, - "num_frames": 209, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "num_warmups": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(768, 1344, 1)": 1.0 + }, + "extra_body": { + "width": 1344, + "height": 768, + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "seed": 42, "flow_shift": 12.0, "extra_params": "{\"task\":\"fl2va\",\"duration\":8.7,\"audio_flow_shift\":3.0}" }, - "num-input-images": 1, "baseline": { "H100": { - "throughput_qps": 0.0212, - "latency_mean": 38.1385, - "peak_memory_mb_mean": 54622.8 + "request_throughput": 0.0212, + "mean_e2el_ms": 38138.5, + "mean_peak_memory_mb": 54622.8 } } } @@ -148,7 +152,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 4 } @@ -157,8 +164,6 @@ "diffusion" ], "description": "MiniMax-H3 Ref2VA video-reference-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", "server_params": { "model": "MiniMaxAI/MiniMax-H3", "serve_args": { @@ -181,33 +186,39 @@ "benchmark_params": [ { "name": "1344x768_frames209_steps8", - "dataset": "random", - "task": "v2v", - "num-prompts": 3, - "max-concurrency": 1, - "warmup-requests": 1, - "warmup-concurrency": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1344, - "height": 768, - "prompt": "Restyle the reference footage into a cinematic night scene with neon reflections, keeping the same camera motion and subject.", - "num_inference_steps": 8, - "num_frames": 209, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "num_warmups": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "video": 1 + }, + "random_mm_bucket_config": { + "(480, 832, 209)": 1.0 + }, + "extra_body": { + "width": 1344, + "height": 768, + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "seed": 42, "flow_shift": 12.0, "extra_params": "{\"task\":\"ref2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}" }, "baseline": { "H100": { - "throughput_qps": 0.0077, - "latency_mean": 105.7508, - "peak_memory_mb_mean": 55180.8 + "request_throughput": 0.0077, + "mean_e2el_ms": 105750.8, + "mean_peak_memory_mb": 55180.8 } } } @@ -219,7 +230,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 4 } @@ -228,8 +242,6 @@ "diffusion" ], "description": "MiniMax-H3 T2VA with 4-card distributed layerwise offload (DLO), 8 denoise steps (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", "server_params": { "model": "MiniMaxAI/MiniMax-H3", "serve_args": { @@ -256,33 +268,30 @@ "benchmark_params": [ { "name": "1344x768_frames209_steps8", - "dataset": "random", - "task": "t2v", - "num-prompts": 3, - "max-concurrency": 1, - "warmup-requests": 1, - "warmup-concurrency": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1344, - "height": 768, - "prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.", - "num_inference_steps": 8, - "num_frames": 209, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "num_warmups": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 1344, + "height": 768, + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "seed": 42, "flow_shift": 12.0, "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}" }, "baseline": { "H100": { - "throughput_qps": 0.0212, - "latency_mean": 38.2089, - "peak_memory_mb_mean": 36218.4 + "request_throughput": 0.0212, + "mean_e2el_ms": 38208.9, + "mean_peak_memory_mb": 36218.4 } } } diff --git a/tests/dfx/perf/tests/test_runner_metadata.py b/tests/dfx/perf/tests/test_runner_metadata.py index a0f966121a1..bdbd83df0a3 100644 --- a/tests/dfx/perf/tests/test_runner_metadata.py +++ b/tests/dfx/perf/tests/test_runner_metadata.py @@ -205,15 +205,53 @@ def test_is_diffusion_perf_config(): from tests.dfx.conftest import is_diffusion_perf_config assert not is_diffusion_perf_config( - {"test_name": "omni_a", "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"]} + { + "test_name": "omni_a", + "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"], + "benchmark_params": [{"dataset_name": "random", "endpoint": "/v1/chat/completions"}], + } ) assert is_diffusion_perf_config( { "test_name": "diff_a", "server_type": "vllm-omni", "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"], + "benchmark_params": [{"task": "t2i", "dataset": "random"}], } ) + videos_cfg = { + "test_name": "diff_videos", + "server_type": "vllm-omni", + "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"], + "benchmark_params": [{"task": "t2v", "dataset_name": "random", "endpoint": "/v1/videos"}], + } + assert not is_diffusion_perf_config(videos_cfg) + custom_edits_cfg = { + "test_name": "diff_custom_edits", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/images/edits", + "benchmark_params": [{"dataset": "custom", "task": "ti2i"}], + } + assert is_diffusion_perf_config(custom_edits_cfg) + + +def test_merge_omni_default_server_args_respects_json(): + from tests.dfx.perf.scripts.run_benchmark import _merge_omni_default_server_args + + extra = ("--stage-init-timeout", "1800", "--init-timeout", "1800", "--usp", "4") + assert _merge_omni_default_server_args(extra, use_omni=True) == [] + assert _merge_omni_default_server_args((), use_omni=True) == [ + "--stage-init-timeout", + "600", + "--init-timeout", + "900", + ] + assert _merge_omni_default_server_args((), use_omni=False) == [] + # Only fill the missing default; keep JSON's other timeouts. + assert _merge_omni_default_server_args(("--stage-init-timeout=1800",), use_omni=True) == [ + "--init-timeout", + "900", + ] def test_benchmark_param_id_suffix_from_task_eval_phase(): @@ -303,12 +341,84 @@ def fake_start(server_param): finally: active_context.close() - assert events == [ - ("start", omni_p0_server), - ("stop", omni_p0_server), - ("start", tts_server), - ("stop", tts_server), - ] + +def test_run_benchmark_persists_distinct_benchmark_params_name(tmp_path, monkeypatch): + """Two benchmark_params under one test_name must keep distinct saved identity.""" + import io + from pathlib import Path + + from tests.dfx import conftest as dfx_conftest + + result_dir = tmp_path / "results" + result_dir.mkdir() + monkeypatch.setenv("BENCHMARK_DIR", str(result_dir)) + + class _FakePopen: + def __init__(self, command, **kwargs): + self.stdout = io.StringIO("") + self.stderr = io.StringIO("") + result_dir_idx = command.index("--result-dir") + filename_idx = command.index("--result-filename") + out = Path(command[result_dir_idx + 1]) / command[filename_idx + 1] + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps({"completed": 3, "request_throughput": 0.01}), encoding="utf-8") + + def wait(self): + return 0 + + monkeypatch.setattr(dfx_conftest.subprocess, "Popen", _FakePopen) + + shared_test = "test_wan22_i2v_usp2" + name_a = "832x480_frames81_steps4" + name_b = "1280x720_frames121_steps4" + + result_a = dfx_conftest.run_benchmark( + args=["--host", "127.0.0.1", "--port", "8000"], + test_name=shared_test, + flow=1, + dataset_name="random-mm", + num_prompt=10, + random_input_len=8, + random_output_len=1, + resource_label="H800", + benchmark_params_name=name_a, + ) + result_b = dfx_conftest.run_benchmark( + args=["--host", "127.0.0.1", "--port", "8000"], + test_name=shared_test, + flow=1, + dataset_name="random-mm", + num_prompt=10, + random_input_len=8, + random_output_len=1, + resource_label="H800", + benchmark_params_name=name_b, + ) + + assert result_a["test_name"] == shared_test + assert result_b["test_name"] == shared_test + assert result_a["name"] == name_a + assert result_b["name"] == name_b + assert "benchmark_params" not in result_a + assert "benchmark_params" not in result_b + + files = sorted(p.name for p in result_dir.glob("result_*.json")) + assert len(files) == 2 + assert any(name_a in name for name in files) + assert any(name_b in name for name in files) + assert files[0] != files[1] + + def omni_group_key(record: dict) -> tuple: + return ( + record.get("model_id") or "", + record.get("test_name") or "", + record.get("name") or "", + record.get("dataset_name") or "", + record.get("max_concurrency") if record.get("max_concurrency") is not None else 0, + record.get("num_prompts") if record.get("num_prompts") is not None else 0, + ) + + assert omni_group_key(result_a) != omni_group_key(result_b) def test_is_hardware_nested_baseline(): diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 3244d1c4155..0aa060b4cbc 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -5,7 +5,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": [ + "H100", + "B200" + ] }, "num_cards": 1 } @@ -14,7 +17,6 @@ "diffusion" ], "description": "Single-device baseline", - "server_type": "vllm-omni", "server_params": { "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", "serve_args": { @@ -24,28 +26,37 @@ "benchmark_params": [ { "name": "832x480_frames81_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 832, - "height": 480, - "num_inference_steps": 4, - "num_frames": 81, - "fps": 16, - "weight": 1 - } - ], + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(480, 832, 1)": 1.0 + }, + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 81, + "fps": 16, + "seed": 42, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + }, "baseline": { "H100": { - "throughput_qps": 0.0361, - "latency_mean": 27.6206, - "peak_memory_mb_mean": 80548.0 + "request_throughput": 0.0361, + "mean_e2el_ms": 27620.6, + "mean_peak_memory_mb": 80548.0 } } } @@ -57,7 +68,10 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"], + "cuda": [ + "H100", + "B200" + ], "npu": "A3" }, "num_cards": 2 @@ -67,7 +81,6 @@ "diffusion" ], "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing", - "server_type": "vllm-omni", "server_params": { "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", "serve_args": { @@ -81,55 +94,73 @@ "benchmark_params": [ { "name": "832x480_frames81_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 832, - "height": 480, - "num_inference_steps": 4, - "num_frames": 81, - "fps": 16, - "weight": 1 - } - ], + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(480, 832, 1)": 1.0 + }, + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 81, + "fps": 16, + "seed": 42, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + }, "baseline": { "H100": { - "throughput_qps": 0.05263, - "latency_mean": 18.9415, - "peak_memory_mb_mean": 50053.7 + "request_throughput": 0.05263, + "mean_e2el_ms": 18941.5, + "mean_peak_memory_mb": 50053.7 } } }, { "name": "1280x720_frames121_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "num_inference_steps": 4, - "num_frames": 121, - "fps": 16, - "weight": 1 - } - ], + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(720, 1280, 1)": 1.0 + }, + "extra_body": { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 121, + "fps": 16, + "seed": 42, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + }, "baseline": { "H100": { - "throughput_qps": 0.009595, - "latency_mean": 104.1058, - "peak_memory_mb_mean": 59718.0 + "request_throughput": 0.009595, + "mean_e2el_ms": 104105.8, + "mean_peak_memory_mb": 59718.0 } } } diff --git a/tools/nightly/generate_nightly_perf_excel.py b/tools/nightly/generate_nightly_perf_excel.py index 54c4b3f3406..eae463173a0 100644 --- a/tools/nightly/generate_nightly_perf_excel.py +++ b/tools/nightly/generate_nightly_perf_excel.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Generate a nightly Excel performance report from JSON results. @@ -137,6 +140,7 @@ def _omni_group_key(record: dict[str, Any]) -> tuple[Any, ...]: return ( record.get("model_id") or "", record.get("test_name") or "", + record.get("name") or "", record.get("dataset_name") or "", record.get("max_concurrency") if record.get("max_concurrency") is not None else 0, record.get("num_prompts") if record.get("num_prompts") is not None else 0, @@ -171,6 +175,7 @@ def _load_summary_columns(script_dir: str) -> list[str]: "model_id", "tokenizer_id", "test_name", + "name", "dataset_name", "num_prompts", "request_rate", @@ -218,7 +223,7 @@ def _load_summary_columns(script_dir: str) -> list[str]: def _ensure_omni_summary_columns(summary_columns: list[str]) -> list[str]: """Ensure omni summary contains required columns, even when a custom columns file exists.""" - required = ("test_name", "dataset_name", "source_file") + required = ("test_name", "name", "dataset_name", "source_file") existing = set(summary_columns) if all(c in existing for c in required): return summary_columns @@ -232,8 +237,18 @@ def _ensure_omni_summary_columns(summary_columns: list[str]) -> list[str]: else: out.append("test_name") existing = set(out) - if "dataset_name" not in existing: + if "name" not in existing: if "test_name" in out: + idx = out.index("test_name") + 1 + out.insert(idx, "name") + else: + out.append("name") + existing = set(out) + if "dataset_name" not in existing: + if "name" in out: + idx = out.index("name") + 1 + out.insert(idx, "dataset_name") + elif "test_name" in out: idx = out.index("test_name") + 1 out.insert(idx, "dataset_name") else: diff --git a/tools/nightly/generate_nightly_perf_html.py b/tools/nightly/generate_nightly_perf_html.py index 11e9b0961ee..cb29f627c57 100644 --- a/tools/nightly/generate_nightly_perf_html.py +++ b/tools/nightly/generate_nightly_perf_html.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Generate an improved nightly HTML performance dashboard without modifying the existing generator. @@ -256,6 +259,7 @@ def _sort_omni_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]: key=lambda r: ( r.get("model_id") or "", r.get("test_name") or "", + r.get("name") or "", r.get("dataset_name") or "", r.get("max_concurrency") or 0, r.get("num_prompts") or 0, @@ -1236,7 +1240,7 @@ def _build_html_document( const extraKey = prefix === "diff" ? "" : "tokenizer_id"; const metaKeys = prefix === "diff" ? ["test_name"] - : ["test_name", "max_concurrency", "num_prompts"]; + : ["test_name", "name", "max_concurrency", "num_prompts"]; const configFields = prefix === "diff" ? ["test_name", "model", "endpoint", "dataset"] : [ @@ -1245,13 +1249,14 @@ def _build_html_document( "model_id", "tokenizer_id", "test_name", + "name", "dataset_name", "max_concurrency", "num_prompts", ]; const labelFields = prefix === "diff" ? ["test_name", "dataset"] - : ["test_name", "dataset_name"]; + : ["test_name", "name", "dataset_name"]; const latestMetric = prefix === "diff" ? "throughput_qps" : "output_throughput"; const secondaryMetric = prefix === "diff" ? "latency_mean" : "mean_e2el_ms"; const modelList = document.getElementById(`${{prefix}}-model-list`); @@ -1650,7 +1655,7 @@ def _build_html_document( '
Chat/audio benchmark history with ' "trend charts grouped by throughput, latency, and audio metrics." '
' - "Grouping test_name + dataset + concurrency + prompts
" + "Grouping test_name + name + dataset + concurrency + prompts
" '
Snapshots single-point series ' "stay readable
" ), diff --git a/tools/nightly/run_nightly_jobs.sh b/tools/nightly/run_nightly_jobs.sh index 8a08097260b..1b37740ce01 100644 --- a/tools/nightly/run_nightly_jobs.sh +++ b/tools/nightly/run_nightly_jobs.sh @@ -41,7 +41,7 @@ # From repo root: pytest -sv -m " and local_model" (markers from MODEL_TYPE: omni, tts, # diffusion; all → "(omni or tts or diffusion) and local_model"). Not filtered by nightly YAML. # LABEL_SUBSTR: if set, restrict to tests/**/test_*.py basenames and tests/dfx/perf/tests/*.json -# whose filename contains the substring (benchmark runner chosen by JSON family). +# whose filename contains the substring (benchmark runner via is_diffusion_perf_config). # # stability (when included in TEST_TYPE): # From repo root: pytest -s -v --run-level full_model -m "" tests/dfx/stability/scripts/... @@ -711,8 +711,19 @@ def perf_json_model_family(json_basename: str) -> str: def perf_json_runner(json_basename: str) -> Path: - if perf_json_model_family(json_basename) in ("omni", "tts"): + """Pick benchmark runner from JSON schema (``dataset`` vs ``dataset_name``).""" + json_path = REPO_ROOT / PERF_TESTS_REL / json_basename + try: + if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + from tests.dfx.conftest import is_diffusion_perf_config, load_configs + + configs = load_configs(str(json_path)) + if configs and any(is_diffusion_perf_config(cfg) for cfg in configs): + return RUN_DIFFUSION_BENCHMARK_REL return RUN_BENCHMARK_REL + except Exception as exc: + print(f"# warn: could not classify {json_basename}: {exc}", file=sys.stderr) return RUN_DIFFUSION_BENCHMARK_REL diff --git a/vllm_omni/benchmarks/metrics/metrics.py b/vllm_omni/benchmarks/metrics/metrics.py index 759e606671f..6cfe473d5b1 100644 --- a/vllm_omni/benchmarks/metrics/metrics.py +++ b/vllm_omni/benchmarks/metrics/metrics.py @@ -5,6 +5,7 @@ from collections import defaultdict from collections.abc import Sequence from dataclasses import field, make_dataclass +from typing import Any import numpy as np from vllm.benchmarks.datasets import SampleRequest @@ -15,6 +16,7 @@ from vllm_omni.metrics import definitions as defs _PERCENTILE_ROWS_TYPE = list[tuple[float, float]] | None +_STAGE_DURATION_MAP_TYPE = dict[str, float] | None _FLOAT_LIST_TYPE = list[float] _INT_LIST_TYPE = list[int] @@ -67,9 +69,14 @@ (defs.MEDIAN_PEAK_MEMORY_MB, float, field(default=0.0)), (defs.STD_PEAK_MEMORY_MB, float, field(default=0.0)), (defs.PERCENTILES_PEAK_MEMORY_MB, _PERCENTILE_ROWS_TYPE, field(default=None)), + (defs.STAGE_DURATIONS_MEAN, _STAGE_DURATION_MAP_TYPE, field(default=None)), + (defs.STAGE_DURATIONS_P50, _STAGE_DURATION_MAP_TYPE, field(default=None)), + (defs.STAGE_DURATIONS_P99, _STAGE_DURATION_MAP_TYPE, field(default=None)), ] -MultiModalsBenchmarkMetrics = make_dataclass( +# ``make_dataclass`` returns a runtime class that mypy treats as a variable, not +# a type. Annotate as Any so call sites can use these names in annotations. +MultiModalsBenchmarkMetrics: Any = make_dataclass( "MultiModalsBenchmarkMetrics", _MULTIMODAL_BENCHMARK_FIELDS, bases=(BenchmarkMetrics,), @@ -99,7 +106,7 @@ (defs.INTER_OUTPUT_LATENCIES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)), ] -StageBenchmarkMetrics = make_dataclass( +StageBenchmarkMetrics: Any = make_dataclass( "StageBenchmarkMetrics", _STAGE_BENCHMARK_FIELDS, namespace={"__doc__": "Aggregated metrics for one pipeline stage (for printing only)."}, @@ -240,6 +247,7 @@ def print_metrics( if isinstance(metrics, MultiModalsBenchmarkMetrics): print("{:<40} {:<10.2f}".format("Peak concurrent requests:", metrics.max_concurrent_requests)) print_peak_memory_metrics(metrics) + print_stage_durations_metrics(selected_percentiles or [], metrics) if task_type != TaskType.GENERATION or "e2el" in selected_percentile_metrics: process_one_metric("e2el", metrics) print_text_metrics(task_type, selected_percentile_metrics, metrics) @@ -338,6 +346,92 @@ def print_peak_memory_metrics(metrics: MultiModalsBenchmarkMetrics): print("{:<40} {:<10.2f}".format(f"P{_p_label(p)} PEAK_MEMORY_MB (MB):", value)) +def aggregate_stage_durations(outputs: Sequence[RequestFuncOutput]) -> dict[str, dict[str, float]]: + """Aggregate pipeline profiler timings into mean/p50/p99 maps. + + ``stage_N_gen_ms`` is omitted. That clock already belongs to + ``stage_gen_time`` / image / video generation, not to this profiler block. + The maps are stored on ``MultiModalsBenchmarkMetrics`` and printed with the + other run-level metrics. ``--print-stage`` does not gate them. + """ + stage_duration_lists: dict[str, list[float]] = {} + for output in outputs: + if not getattr(output, "success", False): + continue + stage_durations = getattr(output, "stage_durations", None) + if not isinstance(stage_durations, dict): + continue + for stage, duration in stage_durations.items(): + if _is_printed_stage_gen_key(str(stage)): + continue + if isinstance(duration, bool) or not isinstance(duration, (int, float)) or not np.isfinite(duration): + continue + stage_duration_lists.setdefault(str(stage), []).append(float(duration)) + if not stage_duration_lists: + return {} + return { + defs.STAGE_DURATIONS_MEAN: {stage: float(np.mean(values)) for stage, values in stage_duration_lists.items()}, + defs.STAGE_DURATIONS_P50: { + stage: float(np.percentile(values, 50)) for stage, values in stage_duration_lists.items() + }, + defs.STAGE_DURATIONS_P99: { + stage: float(np.percentile(values, 99)) for stage, values in stage_duration_lists.items() + }, + } + + +def _stage_duration_display_name(stage: str) -> str: + """Strip pipeline-class prefix and prettify profiler keys for printing. + + ``Wan22I2VPipeline.diffuse`` -> ``Diffuse`` + ``Wan22I2VPipeline.text_encoder.forward`` -> ``Text Encoder Forward`` + ``queue_wait_ms`` -> ``Queue Wait`` + """ + name = str(stage) + head, sep, tail = name.partition(".") + if sep and head.endswith("Pipeline"): + name = tail + if name.endswith("_ms"): + name = name[: -len("_ms")] + return name.replace("_", " ").replace(".", " ").title() + + +def _is_printed_stage_gen_key(stage: str) -> bool: + """True for ``stage_0_gen_ms``, which duplicates ``--print-stage`` stage timing.""" + name = str(stage) + prefix, suffix = "stage_", "_gen_ms" + if not name.startswith(prefix) or not name.endswith(suffix): + return False + return name[len(prefix) : -len(suffix)].isdigit() + + +def print_stage_durations_metrics( + selected_percentiles: list[float], + metrics: MultiModalsBenchmarkMetrics, +) -> None: + """Print profiler timings already stored on ``metrics``. + + Mean and median always print, matching :func:`print_image_metrics`. P99 prints + only when 99 is in ``selected_percentiles``. ``stage_N_gen_ms`` is not here; + that clock stays on stage_gen_time / image / video generation. + """ + mean = getattr(metrics, defs.STAGE_DURATIONS_MEAN, None) or {} + if not mean: + return + p50 = getattr(metrics, defs.STAGE_DURATIONS_P50, None) or {} + p99 = getattr(metrics, defs.STAGE_DURATIONS_P99, None) or {} + show_p99 = any(float(p) == 99.0 for p in selected_percentiles) + print("{s:{c}^{n}}".format(s=" Stage Durations ", n=50, c="-")) + for stage in mean: + label = _stage_duration_display_name(stage) + unit = " (ms)" if str(stage).endswith("_ms") else " (s)" + print("{:<40} {:<10.2f}".format(f"Mean {label}{unit}:", mean[stage])) + if stage in p50: + print("{:<40} {:<10.2f}".format(f"Median {label}{unit}:", p50[stage])) + if show_p99 and stage in p99: + print("{:<40} {:<10.2f}".format(f"P99 {label}{unit}:", p99[stage])) + + def print_image_metrics(selected_percentiles: list[float], metrics: MultiModalsBenchmarkMetrics): print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="=")) print("{:<40} {:<10}".format("Total images generated:", getattr(metrics, defs.TOTAL_IMAGES))) @@ -818,7 +912,7 @@ def calculate_metrics( request_rate, benchmark_duration, print_stage: bool = False, -) -> tuple[BenchmarkMetrics, list[int]]: +) -> tuple[MultiModalsBenchmarkMetrics, list[int]]: """Calculate the metrics for the benchmark. Args: @@ -928,7 +1022,7 @@ def calculate_metrics( goodput_ttfts.append(outputs[i].ttft if ttft_measured else None) goodput_audio_ttfps.append(getattr(outputs[i], defs.AUDIO_TTFP, 0.0) if audio_ttfp_measured else None) audio_duration.append(getattr(outputs[i], defs.AUDIO_DURATION, 0.0)) - audio_frames.append(getattr(outputs[i], defs.AUDIO_FRAMES, 0.0)) + audio_frames.append(getattr(outputs[i], defs.AUDIO_FRAMES, 0)) image_count = int(getattr(outputs[i], defs.IMAGE_COUNT, 0) or 0) total_images += image_count image_generation_time_ms = float(getattr(outputs[i], defs.IMAGE_GENERATION_TIME_MS, 0.0) or 0.0) @@ -982,9 +1076,17 @@ def calculate_metrics( good_completed += 1 if completed == 0: - warnings.formatwarning = lambda msg, category, filename, lineno, line=None: ( - f"{filename}:{lineno}: {category.__name__}: {msg}\n" - ) + + def _formatwarning( + message: Warning | str, + category: type[Warning], + filename: str, + lineno: int, + line: str | None = None, + ) -> str: + return f"{filename}:{lineno}: {category.__name__}: {message}\n" + + warnings.formatwarning = _formatwarning warnings.warn( "All requests failed. This is likely due to a misconfiguration on the benchmark arguments.", stacklevel=2, @@ -1074,6 +1176,7 @@ def calculate_metrics( isinstance(getattr(output, "duplex_session_metrics", None), dict) for output in outputs ) missing_duplex_value = float("nan") if duplex_metrics_present else 0 + stage_duration_summaries = aggregate_stage_durations(outputs) metrics = MultiModalsBenchmarkMetrics( completed=completed, failed=len(failed_outputs), @@ -1167,6 +1270,9 @@ def calculate_metrics( defs.AUDIO_CONTINUITY_OK_RATE: ( (sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0 ), + defs.STAGE_DURATIONS_MEAN: stage_duration_summaries.get(defs.STAGE_DURATIONS_MEAN) or {}, + defs.STAGE_DURATIONS_P50: stage_duration_summaries.get(defs.STAGE_DURATIONS_P50) or {}, + defs.STAGE_DURATIONS_P99: stage_duration_summaries.get(defs.STAGE_DURATIONS_P99) or {}, }, ) print_metrics( diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index 87fdb2daa1e..cfc609c3e34 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -19,6 +19,7 @@ from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any, Literal +from urllib.parse import urlparse import aiohttp import numpy as np @@ -873,6 +874,9 @@ class MixRequestFuncOutput(RequestFuncOutput): tts_turn_pcm_bytes: list[bytes] | None = None #: Per-stage snapshot from orchestrator ``metrics["stage_metrics"]`` (merged across SSE chunks). stage_metrics: dict[str, dict] | None = None + #: Diffusion pipeline profiler timings from response ``stage_durations`` + #: (e.g. diffuse / text_encoder.forward / vae.decode), when present. + stage_durations: dict[str, float] | None = None stage_id: int | None = None final_output_type: str | None = None duplex_request_metrics: list[dict[str, object]] | None = None @@ -883,6 +887,7 @@ class MixRequestFuncOutput(RequestFuncOutput): "negative_prompt", "num_inference_steps", "guidance_scale", + "guidance_scale_2", "strength", "true_cfg_scale", "seed", @@ -902,15 +907,51 @@ def _guess_mime_type(path: str) -> str: return mime or "application/octet-stream" -def _iter_image_edit_inputs(value: Any) -> Iterable[Any]: - """Yield image references from benchmark multimodal content.""" +_IMAGE_REFERENCE_SUFFIXES = frozenset({".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".heic", ".heif"}) +_VIDEO_REFERENCE_SUFFIXES = frozenset({".mp4", ".mov", ".webm", ".mkv", ".m4v"}) + + +def _string_reference_kind(reference: str) -> str | None: + """Classify a bare reference string as image, video, or file. + + ``data:image`` / ``data:video`` carry their type in the URL. Bare http(s) + URLs use the path extension. An existing local path is a file upload, not + an image or a video URL. + """ + if reference.startswith("data:image"): + return "image" + if reference.startswith("data:video"): + return "video" + if reference.startswith(("http://", "https://")): + suffix = Path(urlparse(reference).path).suffix.lower() + if suffix in _VIDEO_REFERENCE_SUFFIXES: + return "video" + if suffix in _IMAGE_REFERENCE_SUFFIXES: + return "image" + return None + local_path = reference.removeprefix("file://") + if local_path and os.path.exists(local_path): + return "file" + return None + + +def _iter_image_reference_inputs(value: Any) -> Iterable[Any]: + """Yield image references from benchmark multimodal content. + + ``random-mm`` image buckets arrive as OpenAI chat parts + ``{"type": "image_url", "image_url": {"url": ...}}``. Yield + ``{"image_url": url}`` so the form helper keeps an explicit image type. + Bare video strings are left for ``_iter_video_reference_inputs``. + """ if value is None: return if isinstance(value, list): for item in value: - yield from _iter_image_edit_inputs(item) + yield from _iter_image_reference_inputs(item) return if not isinstance(value, dict): + if isinstance(value, str) and _string_reference_kind(value) == "video": + return yield value return @@ -919,15 +960,49 @@ def _iter_image_edit_inputs(value: Any) -> Iterable[Any]: image_url = value.get("image_url") if isinstance(image_url, dict): url = image_url.get("url") - if url: - yield url - elif image_url: - yield image_url + if isinstance(url, str) and url: + yield {"image_url": url} + elif isinstance(image_url, str) and image_url: + yield {"image_url": image_url} return for key in ("image", "images"): if key in value: - yield from _iter_image_edit_inputs(value[key]) + yield from _iter_image_reference_inputs(value[key]) + + +def _iter_video_reference_inputs(value: Any) -> Iterable[dict[str, str]]: + """Yield structured video references from benchmark multimodal content. + + ``random-mm`` video buckets arrive as OpenAI chat parts + ``{"type": "video_url", "video_url": {"url": ...}}``, or as a bare + ``data:video`` / video http(s) string. Yield ``{"video_url": url}`` so the + form helper keeps the video branch. + """ + if value is None: + return + if isinstance(value, list): + for item in value: + yield from _iter_video_reference_inputs(item) + return + if not isinstance(value, dict): + if isinstance(value, str) and _string_reference_kind(value) == "video": + yield {"video_url": value} + return + + if value.get("type") == "video_url": + video_url = value.get("video_url") + if isinstance(video_url, dict): + url = video_url.get("url") + if isinstance(url, str) and url: + yield {"video_url": url} + elif isinstance(video_url, str) and video_url: + yield {"video_url": video_url} + return + + for key in ("video", "videos"): + if key in value: + yield from _iter_video_reference_inputs(value[key]) def _add_image_edit_input_to_form(form: aiohttp.FormData, image_input: Any) -> None: @@ -940,6 +1015,12 @@ def _add_image_edit_input_to_form(form: aiohttp.FormData, image_input: Any) -> N ) return + if isinstance(image_input, Mapping) and _is_structured_image_reference(image_input): + image_url = image_input.get("image_url") + if isinstance(image_url, str) and image_url: + _add_image_edit_input_to_form(form, image_url) + return + if isinstance(image_input, str): if image_input.startswith(("data:image", "http://", "https://")): form.add_field("url", image_input) @@ -1137,6 +1218,68 @@ def _update_output_peak_memory_from_payload(output: MixRequestFuncOutput, data: output.peak_memory_mb = peak_memory_mb +def _coerce_stage_durations_dict(raw: object) -> dict[str, float] | None: + """Normalize a stage_durations mapping to ``dict[str, float]``.""" + if not isinstance(raw, dict) or not raw: + return None + coerced: dict[str, float] = {} + for key, value in raw.items(): + if isinstance(value, bool) or not isinstance(value, (int, float)) or not np.isfinite(value): + continue + coerced[str(key)] = float(value) + return coerced or None + + +def _extract_stage_durations_from_payload(data: Mapping[str, object]) -> dict[str, float] | None: + """Pull pipeline profiler timings from video/image/chat response shapes.""" + found = _coerce_stage_durations_dict(data.get("stage_durations")) + if found: + return found + + metrics = data.get("metrics") + if isinstance(metrics, dict): + found = _coerce_stage_durations_dict(metrics.get("stage_durations")) + if found: + return found + + response_data = data.get("data") + if isinstance(response_data, list): + for item in response_data: + if isinstance(item, dict): + found = _coerce_stage_durations_dict(item.get("stage_durations")) + if found: + return found + + choices = data.get("choices") + if isinstance(choices, list): + for choice in choices: + if not isinstance(choice, dict): + continue + for message_key in ("message", "delta"): + message = choice.get(message_key) + if not isinstance(message, dict): + continue + content = message.get("content") + if isinstance(content, list): + for item in content: + if isinstance(item, dict): + found = _coerce_stage_durations_dict(item.get("stage_durations")) + if found: + return found + elif isinstance(content, dict): + found = _coerce_stage_durations_dict(content.get("stage_durations")) + if found: + return found + return None + + +def _update_output_stage_durations_from_payload(output: MixRequestFuncOutput, data: Mapping[str, object]) -> None: + """Persist the full profiler ``stage_durations`` map when the response has one.""" + found = _extract_stage_durations_from_payload(data) + if found: + output.stage_durations = found + + def _image_metrics_from_stage_metrics(metrics: object) -> tuple[int, float, int, float]: if not isinstance(metrics, dict): return 0, 0.0, 0, 0.0 @@ -1209,6 +1352,7 @@ def _apply_image_metrics_from_payload(output: MixRequestFuncOutput, data: Mappin """Populate image benchmark fields from an OpenAI-compatible image payload.""" _update_output_stage_metrics_from_payload(output, data, update_output_tokens=False) _update_output_peak_memory_from_payload(output, data) + _update_output_stage_durations_from_payload(output, data) payload_image_count = 0 response_data = data.get("data") @@ -1321,15 +1465,129 @@ def _video_frames_from_payload(data: Mapping[str, object], request_body: Mapping def _is_structured_image_reference(reference: Mapping[str, object]) -> bool: - """True for API image_reference objects ({image_url}/{file_id}).""" + """True for API image_reference objects ({"image_url": "..."}).""" image_url = reference.get("image_url") - file_id = reference.get("file_id") - has_url = isinstance(image_url, str) and bool(image_url) - has_file_id = isinstance(file_id, str) and bool(file_id) - return has_url or has_file_id + return isinstance(image_url, str) and bool(image_url) + + +def _is_structured_video_reference(reference: Mapping[str, object]) -> bool: + """True for API video_reference objects ({"video_url": "..."}).""" + video_url = reference.get("video_url") + return isinstance(video_url, str) and bool(video_url) + + +_VIDEO_REFERENCE_JSON_MAX_BYTES = 1024 * 1024 + + +def _data_video_json_exceeds_text_limit(video_url: str) -> bool: + """True when a data:video URL would exceed the ~1MB multipart text-part limit.""" + if not video_url.startswith("data:video"): + return False + encoded = json.dumps({"video_url": video_url}).encode("utf-8") + return len(encoded) > _VIDEO_REFERENCE_JSON_MAX_BYTES + + +def _add_data_video_upload(form: aiohttp.FormData, video_url: str) -> bool: + """Upload one inline video as ``input_references`` instead of a JSON text part.""" + header, _, payload = video_url.partition(",") + if not payload: + raise ValueError(f"Unsupported video data URL: {video_url[:64]!r}") + try: + video_bytes = base64.b64decode(payload) + except (ValueError, TypeError) as exc: + raise ValueError("video data URL is not valid base64") from exc + mime = header[len("data:") :].split(";", 1)[0] or "video/mp4" + suffix = ".mp4" if mime.endswith("mp4") else ".bin" + form.add_field( + "input_references", + video_bytes, + filename=f"benchmark-reference{suffix}", + content_type=mime, + ) + return True -def _add_video_reference_to_form(form: aiohttp.FormData, reference: object) -> bool: +def _file_bytes_as_data_url(raw: bytes, mime: str) -> str: + encoded = base64.b64encode(raw).decode("ascii") + return f"data:{mime};base64,{encoded}" + + +def _image_reference_json_value(reference: object) -> object: + """Turn an image file into a JSON ``image_reference`` when it must share a form. + + ``input_reference`` cannot be combined with ``video_reference``. Image URLs + stay URLs. Upload bytes and local image files become ``data:image`` URLs. + """ + if isinstance(reference, Mapping) and "bytes" in reference and not _is_structured_image_reference(reference): + raw = reference["bytes"] + if not isinstance(raw, (bytes, bytearray)): + raise ValueError(f"image reference bytes must be bytes (got {type(raw).__name__}).") + content_type = reference.get("content_type", "image/png") + if not isinstance(content_type, str) or not content_type.startswith("image/"): + content_type = "image/png" + return {"image_url": _file_bytes_as_data_url(bytes(raw), content_type.split(";", 1)[0])} + if isinstance(reference, str): + kind = _string_reference_kind(reference) + if kind == "image": + return {"image_url": reference} + if kind == "file": + local_path = reference.removeprefix("file://") + mime = _guess_mime_type(local_path) + if not mime.startswith("image/"): + mime = "image/png" + with open(local_path, "rb") as handle: + return {"image_url": _file_bytes_as_data_url(handle.read(), mime)} + return reference + + +def _video_reference_json_value(reference: object) -> object: + """Turn a video file into a JSON ``video_reference`` when it must share a form. + + ``input_reference`` cannot be combined with ``image_reference``. Video URLs + stay URLs. Local video files become ``data:video`` URLs. + """ + if isinstance(reference, Mapping) and "bytes" in reference and not _is_structured_video_reference(reference): + raw = reference["bytes"] + if not isinstance(raw, (bytes, bytearray)): + raise ValueError(f"video reference bytes must be bytes (got {type(raw).__name__}).") + content_type = reference.get("content_type", "video/mp4") + if not isinstance(content_type, str) or not content_type.startswith("video/"): + content_type = "video/mp4" + return {"video_url": _file_bytes_as_data_url(bytes(raw), content_type.split(";", 1)[0])} + if isinstance(reference, str): + kind = _string_reference_kind(reference) + if kind == "video": + return {"video_url": reference} + if kind == "file": + local_path = reference.removeprefix("file://") + mime = _guess_mime_type(local_path) + if not mime.startswith("video/"): + mime = "video/mp4" + with open(local_path, "rb") as handle: + return {"video_url": _file_bytes_as_data_url(handle.read(), mime)} + return reference + + +def _add_video_reference_to_form( + form: aiohttp.FormData, + reference: object, + *, + upload_inline_video: bool = True, +) -> bool: + """Encode one reference: image URL, video URL, or file upload. + + Image URLs use ``image_reference``. Video URLs use ``video_reference``. + A lone ``data:video`` whose JSON text exceeds 1MB is uploaded as + ``input_references``. Local paths and raw bytes use ``input_reference``. + ``upload_inline_video`` must be false when an image is on the same form: + ``input_references`` cannot be combined with ``image_reference``. + """ + candidates = reference if isinstance(reference, list) else [reference] + for item in candidates: + if isinstance(item, Mapping): + file_id = item.get("file_id") + if isinstance(file_id, str) and file_id: + raise ValueError("file_id is not supported yet") if isinstance(reference, dict) and "bytes" in reference: form.add_field( "input_reference", @@ -1343,23 +1601,39 @@ def _add_video_reference_to_form(form: aiohttp.FormData, reference: object) -> b form.add_field("image_reference", json.dumps(dict(reference))) return True + if isinstance(reference, Mapping) and _is_structured_video_reference(reference): + video_url = reference.get("video_url") + if upload_inline_video and isinstance(video_url, str) and _data_video_json_exceeds_text_limit(video_url): + return _add_data_video_upload(form, video_url) + form.add_field("video_reference", json.dumps(dict(reference))) + return True + if isinstance(reference, list): if reference and all(isinstance(item, Mapping) and _is_structured_image_reference(item) for item in reference): form.add_field("image_reference", json.dumps([dict(item) for item in reference])) return True + if reference and all(isinstance(item, Mapping) and _is_structured_video_reference(item) for item in reference): + form.add_field("video_reference", json.dumps([dict(item) for item in reference])) + return True raise ValueError( - "Unsupported image_reference list; expected non-empty list of " - '{"image_url": "..."} and/or {"file_id": "..."} objects.' + "Unsupported reference list; expected non-empty list of " + '{"image_url": "..."} or {"video_url": "..."} objects.' ) if isinstance(reference, str): - if reference.startswith(("data:image", "http://", "https://")): + kind = _string_reference_kind(reference) + if kind == "image": form.add_field("image_reference", json.dumps({"image_url": reference})) return True - local_path = reference.removeprefix("file://") - if os.path.exists(local_path): - with open(local_path, "rb") as f: - reference_bytes = f.read() + if kind == "video": + if upload_inline_video and _data_video_json_exceeds_text_limit(reference): + return _add_data_video_upload(form, reference) + form.add_field("video_reference", json.dumps({"video_url": reference})) + return True + if kind == "file": + local_path = reference.removeprefix("file://") + with open(local_path, "rb") as handle: + reference_bytes = handle.read() form.add_field( "input_reference", reference_bytes, @@ -1367,15 +1641,64 @@ def _add_video_reference_to_form(form: aiohttp.FormData, reference: object) -> b content_type=_guess_mime_type(local_path), ) return True - raise ValueError(f"Unsupported image_reference path or URL: {reference!r}") + if reference.startswith(("http://", "https://")): + raise ValueError( + "Bare http(s) reference needs an image or video extension " + f"({', '.join(sorted(_IMAGE_REFERENCE_SUFFIXES | _VIDEO_REFERENCE_SUFFIXES))}); " + f"got {reference!r}." + ) + raise ValueError(f"Unsupported reference path or URL: {reference!r}") raise ValueError( - "Unsupported image_reference; expected upload bytes, local path/URL string, " - 'or {"image_url": "..."} / {"file_id": "..."} object ' + "Unsupported reference; expected image URL, video URL, upload bytes, or a local file " f"(got {type(reference).__name__})." ) +def _add_combined_video_form_references( + form: aiohttp.FormData, + multi_modal_content: Any, + extra_body: Mapping[str, Any] | None = None, +) -> None: + """Serialize image and video refs using a server-accepted field pair. + + Alone, each reference uses its own field: image URL → ``image_reference``, + video URL → ``video_reference``, file → ``input_reference``. A lone inline + video whose JSON text exceeds 1MB is uploaded as ``input_references``. + Together, that upload cannot be combined with ``image_reference``, so both + sides stay on the JSON fields. A file paired with the other media is rewritten + as a data URL of the matching type. + """ + extra_body = extra_body or {} + image_refs = list(_iter_image_reference_inputs(multi_modal_content)) + video_refs = list(_iter_video_reference_inputs(multi_modal_content)) + if not image_refs and extra_body.get("image_reference") is not None: + image_refs = [extra_body["image_reference"]] + if not video_refs and extra_body.get("video_reference") is not None: + video_refs = [extra_body["video_reference"]] + + if image_refs and video_refs: + for raw in (image_refs[0], video_refs[0]): + candidates = raw if isinstance(raw, list) else [raw] + for item in candidates: + if isinstance(item, Mapping): + file_id = item.get("file_id") + if isinstance(file_id, str) and file_id: + raise ValueError("file_id is not supported yet") + _add_video_reference_to_form(form, _image_reference_json_value(image_refs[0])) + _add_video_reference_to_form( + form, + _video_reference_json_value(video_refs[0]), + upload_inline_video=False, + ) + return + + if image_refs: + _add_video_reference_to_form(form, image_refs[0]) + if video_refs: + _add_video_reference_to_form(form, video_refs[0]) + + def _add_video_extra_body_to_form( form: aiohttp.FormData, extra_body: Mapping[str, object], @@ -1398,9 +1721,11 @@ def _add_video_extra_body_to_form( "height", "poll_interval_s", "poll_timeout_s", - # Handled only by _add_video_reference_to_form (upload / JSON image_url). + # Handled only by _add_video_reference_to_form (upload / JSON image_url / video_url). "image_reference", + "video_reference", "input_reference", + "input_references", *_VIDEO_FORM_FIELDS, } for key, value in extra_body.items(): @@ -1421,8 +1746,9 @@ def _apply_video_metrics_from_payload( output.video_frames = _video_frames_from_payload(data, request_body) _update_output_stage_metrics_from_payload(output, data, update_output_tokens=False) _update_output_peak_memory_from_payload(output, data) + _update_output_stage_durations_from_payload(output, data) - stage_durations = data.get("stage_durations") + stage_durations = output.stage_durations if output.stage_durations is not None else data.get("stage_durations") stage_gen_ms = _video_generation_ms_from_stage_durations(stage_durations) if stage_gen_ms <= 0: inference_time_s = coerce_positive_float_scalar(data.get("inference_time_s")) @@ -1534,6 +1860,7 @@ async def async_request_openai_chat_omni_completions( output.image_pixels = 0 output.denoise_step_latency_ms = 0.0 output.peak_memory_mb = 0.0 + output.stage_durations = None completion_tokens_seen = 0 try: async with session.post(url=api_url, json=payload, headers=headers) as response: @@ -1564,6 +1891,7 @@ async def async_request_openai_chat_omni_completions( data = json.loads(chunk) _update_output_stage_metrics_from_payload(output, data) _update_output_peak_memory_from_payload(output, data) + _update_output_stage_durations_from_payload(output, data) usage = data.get("usage") completion_tokens = None if isinstance(usage, dict): @@ -1926,15 +2254,7 @@ async def async_request_openai_videos_omni( form.add_field("size", str(request_body["size"])) _add_video_extra_body_to_form(form, extra_body, request_body) - reference_added = False - for reference in _iter_image_edit_inputs(request_func_input.multi_modal_content): - if _add_video_reference_to_form(form, reference): - reference_added = True - break - if not reference_added: - image_reference = extra_body.get("image_reference") - if image_reference is not None: - _add_video_reference_to_form(form, image_reference) + _add_combined_video_form_references(form, request_func_input.multi_modal_content, extra_body) headers = { "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}", @@ -2044,7 +2364,7 @@ async def async_request_openai_image_edits_omni( _add_image_edit_extra_body_to_form(form, extra_body) try: - image_inputs = list(_iter_image_edit_inputs(request_func_input.multi_modal_content)) + image_inputs = list(_iter_image_reference_inputs(request_func_input.multi_modal_content)) if not image_inputs: raise ValueError( "openai-image-edits-omni requires image multimodal content. " @@ -2102,6 +2422,7 @@ async def async_request_openai_image_edits_omni( update_output_tokens=(data.get("type") == "ar_delta"), ) _update_output_peak_memory_from_payload(output, data) + _update_output_stage_durations_from_payload(output, data) chunk_type = data.get("type") if chunk_type == "ar_delta": @@ -3201,6 +3522,9 @@ def measured_ttft(output: RequestFuncOutput) -> float | None: defs.MEAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEAN_PEAK_MEMORY_MB), defs.MEDIAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEDIAN_PEAK_MEMORY_MB), defs.PERCENTILES_PEAK_MEMORY_MB: getattr(mm_metrics, defs.PERCENTILES_PEAK_MEMORY_MB), + defs.STAGE_DURATIONS_MEAN: getattr(mm_metrics, defs.STAGE_DURATIONS_MEAN) or {}, + defs.STAGE_DURATIONS_P50: getattr(mm_metrics, defs.STAGE_DURATIONS_P50) or {}, + defs.STAGE_DURATIONS_P99: getattr(mm_metrics, defs.STAGE_DURATIONS_P99) or {}, "input_lens": [output.prompt_len for output in outputs], "start_times": [output.start_time for output in outputs], "output_lens": actual_output_lens, diff --git a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py index 04f25ac58d9..95ba4767fe9 100644 --- a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py @@ -1730,7 +1730,10 @@ def _forward_robolab_policy( }, }, } - return DiffusionOutput(output=action_output) + return DiffusionOutput( + output=action_output, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) @staticmethod def _truthy(value) -> bool: @@ -3610,6 +3613,7 @@ def _forward_transfer( "payload": {"video": output_video}, "metadata": {"video": {"fps": frame_rate}}, }, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, ) full_output = torch.cat(output_chunks, dim=2)[:, :, :total_frames] @@ -3636,6 +3640,7 @@ def _forward_transfer( "video": {"fps": frame_rate}, }, }, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, ) # -- Forward (main generation entry point) ------------------------------- @@ -4072,7 +4077,10 @@ def _run_diffusion(start_latents): if _is_rank_zero(): logger.info("Sound tokenizer decoded in %.2fs", time.time() - sound_decode_start) logger.info("Total pipeline time: %.2fs", time.time() - pipeline_start) - return DiffusionOutput(output={"video": video, "audio": audio, "audio_sample_rate": sound_sample_rate}) + return DiffusionOutput( + output={"video": video, "audio": audio, "audio_sample_rate": sound_sample_rate}, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) if action_enabled: if action_latents is None or raw_action_dim is None or domain_id is None: @@ -4092,6 +4100,10 @@ def _run_diffusion(start_latents): }, }, }, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, ) - return DiffusionOutput(output={"image": video} if is_t2i else {"video": video}) + return DiffusionOutput( + output={"image": video} if is_t2i else {"video": video}, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) diff --git a/vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5.py b/vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5.py index 942e0b28435..5f0ecb452e1 100644 --- a/vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5.py +++ b/vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -548,7 +548,10 @@ def forward( latents = latents.to(self.vae.dtype) / self.vae.config.scaling_factor output = self.vae.decode(latents, return_dict=False)[0] - return DiffusionOutput(output=output) + return DiffusionOutput( + output=output, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) diff --git a/vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5_i2v.py b/vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5_i2v.py index 1281a9e39d6..f01d45a4ec9 100644 --- a/vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5_i2v.py +++ b/vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5_i2v.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -643,7 +643,10 @@ def forward( latents = latents.to(self.vae.dtype) / self.vae.config.scaling_factor output = self.vae.decode(latents, return_dict=False)[0] - return DiffusionOutput(output=output) + return DiffusionOutput( + output=output, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) diff --git a/vllm_omni/metrics/definitions.py b/vllm_omni/metrics/definitions.py index a09fbcc2ff2..59b575e50fc 100644 --- a/vllm_omni/metrics/definitions.py +++ b/vllm_omni/metrics/definitions.py @@ -99,6 +99,10 @@ STD_PEAK_MEMORY_MB = f"std_{PEAK_MEMORY_MB}" PERCENTILES_PEAK_MEMORY_MB = f"percentiles_{PEAK_MEMORY_MB}" +STAGE_DURATIONS_MEAN = "stage_durations_mean" +STAGE_DURATIONS_P50 = "stage_durations_p50" +STAGE_DURATIONS_P99 = "stage_durations_p99" + # Stage snapshot / StageBenchmarkMetrics field names. TOTAL_OUTPUT = "total_output" TTFTS = "ttfts"