From ff262e2f45ed66c06c4e4a7b280e1c2f03b8b0e7 Mon Sep 17 00:00:00 2001 From: John Liu BUAA Date: Wed, 15 Apr 2026 11:28:33 +0800 Subject: [PATCH 1/3] Add perf tests for Qwen-Image-Layered Add a new perf test JSON for the Qwen/Qwen-Image-Layered model using the vllm-omni server. Defines a single-device baseline with diffusion pipeline profiler enabled and two benchmark scenarios (640x640, 20 steps; 1024x1024, 35 steps) including baseline throughput, latency, and peak memory targets. Signed-off-by: John Liu BUAA --- .../test_qwen_image_layered_vllm_omni.json | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json new file mode 100644 index 00000000000..406cf3ad881 --- /dev/null +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -0,0 +1,49 @@ +[ + { + "test_name": "test_qwen_image_layered_single_device", + "description": "Single-device baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Layered", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "640x640_steps20_i2i", + "dataset": "random", + "task": "i2i", + "width": 640, + "height": 640, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.02, + "latency_mean": 40.0, + "peak_memory_mb_max": 82000, + "peak_memory_mb_mean": 82000 + } + }, + { + "name": "1024x1024_steps35_i2i", + "dataset": "random", + "task": "i2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.005, + "latency_mean": 150.0, + "peak_memory_mb_max": 90000, + "peak_memory_mb_mean": 90000 + } + } + ] + } +] From 2bdfe117aa88033b3eb177e76a4e936415128bfe Mon Sep 17 00:00:00 2001 From: John Liu BUAA Date: Wed, 15 Apr 2026 15:53:39 +0800 Subject: [PATCH 2/3] Add layered image test to nightly diffusion Run an additional pytest using tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json in the nightly diffusion pipeline. Capture its exit code as EXIT4 and include it in the success conditional and final exit bitwise OR so the step considers this new test when uploading artifacts and determining overall status. Signed-off-by: John Liu BUAA --- .../perf/tests/test_qwen_image_layered_vllm_omni.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index 406cf3ad881..3cf13509c8d 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -23,8 +23,8 @@ "baseline": { "throughput_qps": 0.02, "latency_mean": 40.0, - "peak_memory_mb_max": 82000, - "peak_memory_mb_mean": 82000 + "peak_memory_mb_max": 70000, + "peak_memory_mb_mean": 70000 } }, { @@ -39,9 +39,9 @@ "enable-negative-prompt": true, "baseline": { "throughput_qps": 0.005, - "latency_mean": 150.0, - "peak_memory_mb_max": 90000, - "peak_memory_mb_mean": 90000 + "latency_mean": 80.0, + "peak_memory_mb_max": 70000, + "peak_memory_mb_mean": 70000 } } ] From fd13f9e59b41fc21b493a44ec6551dbf9b8a39be Mon Sep 17 00:00:00 2001 From: John Liu BUAA Date: Thu, 16 Apr 2026 11:44:13 +0800 Subject: [PATCH 3/3] Add layered image benchmark to nightly tests Run an additional diffusion benchmark (tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json) in the nightly pipeline, capture its exit code (EXIT4), and include it in the success condition so artifacts (results and logs) are uploaded if any benchmark succeeds. Signed-off-by: John Liu BUAA --- .buildkite/test-nightly.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 58e1e55af7f..ac43b597d15 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -415,7 +415,9 @@ steps: EXIT2=$$? pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json EXIT3=$$? - if [ $$EXIT1 -eq 0 ] || [ $$EXIT2 -eq 0 ] || [ $$EXIT3 -eq 0 ]; then + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json + EXIT4=$$? + if [ $$EXIT1 -eq 0 ] || [ $$EXIT2 -eq 0 ] || [ $$EXIT3 -eq 0 ] || [ $$EXIT4 -eq 0 ]; then buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" fi