From 5eafd329639969fc56c326ab8df44b7a5c55cb0c Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Tue, 9 Jun 2026 17:02:44 +0800 Subject: [PATCH 01/19] Update baseline Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index bc954cee59f..f391e6e15c4 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -43,7 +43,7 @@ 43000, 52000, 72000, - 109000 + 84625 ], "mean_audio_rtf": [ 0.15, From 992611697820c3ba238a69e647fc0c69a813f797 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Thu, 11 Jun 2026 15:30:50 +0800 Subject: [PATCH 02/19] update baseline (#8) Signed-off-by: amy-why-3459 --- .../tests/test_qwen3_omni_async_chunk.json | 78 +++++++++---------- .../tests/test_qwen3_omni_no_async_chunk.json | 48 ++++++------ .../perf/tests/test_qwen3_omni_vllm_text.json | 32 ++++---- 3 files changed, 79 insertions(+), 79 deletions(-) diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index f391e6e15c4..9a51dbee756 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -32,25 +32,25 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 400, - 300, - 400, - 600, - 900 + 146, + 222, + 350, + 462, + 807 ], "mean_e2el_ms": [ - 26000, - 43000, - 52000, - 72000, - 84625 + 21872, + 39929, + 48687, + 62009, + 93860 ], "mean_audio_rtf": [ 0.15, - 0.30, - 0.35, - 0.50, - 0.75 + 0.26, + 0.33, + 0.43, + 0.65 ] } }, @@ -79,13 +79,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 600 + 406 ], "mean_e2el_ms": [ - 6000 + 5245 ], "mean_audio_rtf": [ - 0.20 + 0.16 ] } }, @@ -116,13 +116,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 300 + 271 ], "mean_e2el_ms": [ - 7000 + 6440 ], "mean_audio_rtf": [ - 0.25 + 0.22 ] } }, @@ -155,13 +155,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 700 + 495 ], "mean_e2el_ms": [ - 11000 + 9229 ], "mean_audio_rtf": [ - 0.35 + 0.3 ] } }, @@ -194,18 +194,18 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 100, - 300, - 500, - 800, - 1500 + 86, + 228, + 390, + 706, + 1281 ], "mean_e2el_ms": [ - 5000, - 8000, - 10000, - 14000, - 20000 + 4933, + 8532, + 11005, + 14591, + 21004 ] } }, @@ -239,10 +239,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 500 + 319 ], "mean_e2el_ms": [ - 1000 + 841 ] } }, @@ -278,10 +278,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 200 + 214 ], "mean_e2el_ms": [ - 800 + 802 ] } }, @@ -319,10 +319,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 600 + 388 ], "mean_e2el_ms": [ - 1300 + 1071 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index e57acecba4a..af94f607c63 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -32,25 +32,25 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 400, - 300, - 400, - 600, - 900 + 136, + 283, + 461, + 455, + 829 ], "mean_e2el_ms": [ - 32000, - 34000, - 45000, - 61000, - 90000 + 27765, + 34669, + 43197, + 53022, + 76128 ], "mean_audio_rtf": [ - 0.20, - 0.25, - 0.30, - 0.45, - 0.60 + 0.2, + 0.23, + 0.27, + 0.38, + 0.54 ] } }, @@ -79,13 +79,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 500 + 363 ], "mean_e2el_ms": [ - 6000 + 5364 ], "mean_audio_rtf": [ - 0.20 + 0.17 ] } }, @@ -116,13 +116,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 300 + 257 ], "mean_e2el_ms": [ - 7000 + 5719 ], "mean_audio_rtf": [ - 0.20 + 0.2 ] } }, @@ -155,13 +155,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 700 + 1744 ], "mean_e2el_ms": [ - 9000 + 9817 ], "mean_audio_rtf": [ - 0.30 + 0.32 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index a973bb0e64b..6d8e084a311 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -40,18 +40,18 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 300, - 300, - 1000, - 600, - 800 + 90, + 355, + 348, + 514, + 703 ], "mean_e2el_ms": [ - 6000, - 8000, - 11000, - 13000, - 17000 + 5212, + 8875, + 11085, + 15008, + 20113 ] } }, @@ -86,10 +86,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 30 + 31 ], "mean_e2el_ms": [ - 540 + 579 ] } }, @@ -126,10 +126,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 30 + 32 ], "mean_e2el_ms": [ - 600 + 625 ] } }, @@ -168,10 +168,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 40 + 35 ], "mean_e2el_ms": [ - 700 + 681 ] } } From 683da45a7f0f77f5ba3bde5ccce46e8479d596cc Mon Sep 17 00:00:00 2001 From: herotai214 Date: Thu, 11 Jun 2026 08:26:09 +0000 Subject: [PATCH 03/19] Updated Qwen-image and Qwen-image-layered baseline Signed-off-by: herotai214 --- .../test_qwen_image_layered_vllm_omni.json | 16 ++++---- .../perf/tests/test_qwen_image_vllm_omni.json | 41 +++++++++++-------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index c0cd3b7c015..7b449bb56b9 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -21,10 +21,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, - "latency_mean": 20, - "peak_memory_mb_max": 63000, - "peak_memory_mb_mean": 63000 + "throughput_qps": 0.06, + "latency_mean": 16.24, + "peak_memory_mb_max": 70000, + "peak_memory_mb_mean": 70000 } }, { @@ -38,10 +38,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.03, - "latency_mean": 30, - "peak_memory_mb_max": 63000, - "peak_memory_mb_mean": 63000 + "throughput_qps": 0.04, + "latency_mean": 27.60, + "peak_memory_mb_max": 70000, + "peak_memory_mb_mean": 70000 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index a6eacf88e59..3f0771c14a7 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -22,8 +22,9 @@ "enable-negative-prompt": true, "baseline": { "throughput_qps": 0.35, - "latency_mean": 3, - "peak_memory_mb_mean": 56000 + "latency_mean": 2.85, + "peak_memory_mb_mean": 61000, + "peak_memory_mb_max": 61000 } }, { @@ -37,9 +38,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.03, - "latency_mean": 26, - "peak_memory_mb_mean": 65000 + "throughput_qps": 0.04, + "latency_mean": 27.32, + "peak_memory_mb_mean": 71000, + "peak_memory_mb_max": 71000 } } ] @@ -68,7 +70,7 @@ "enable-negative-prompt": true, "baseline": { "throughput_qps": 0.35, - "latency_mean": 3, + "latency_mean": 2.82, "peak_memory_mb_mean": 56000 } }, @@ -83,8 +85,8 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.03, - "latency_mean": 26, + "throughput_qps": 0.04, + "latency_mean": 27.29, "peak_memory_mb_mean": 65000 } } @@ -116,9 +118,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.11, - "latency_mean": 9, - "peak_memory_mb_mean": 55000 + "throughput_qps": 0.1, + "latency_mean": 9.6, + "peak_memory_mb_mean": 60000, + "peak_memory_mb_max": 60000 } } ] @@ -160,8 +163,9 @@ "enable-negative-prompt": true, "baseline": { "throughput_qps": 0.46, - "latency_mean": 3, - "peak_memory_mb_mean": 56000 + "latency_mean": 2.17, + "peak_memory_mb_mean": 61000, + "peak_memory_mb_max": 61000 } }, { @@ -175,9 +179,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.18, - "latency_mean": 6, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.16, + "latency_mean": 6.07, + "peak_memory_mb_mean": 71000, + "peak_memory_mb_max": 71000 } } ] @@ -209,8 +214,8 @@ "warmup-num-inference-steps": 20, "enable-negative-prompt": true, "baseline": { - "throughput_qps": [0.30, 0.50, 0.60, 0.70], - "latency_mean": [3.50, 4.00, 7.00, 12.00], + "throughput_qps": [0.36, 0.58, 0.57, 0.71], + "latency_mean": [2.76, 3.47, 6.86, 11.07], "peak_memory_mb_mean": [67000, 67000, 67000, 67000] } } From 8ca992f72a8cc3e7d57ca7440f6c84a70a302ec2 Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Thu, 11 Jun 2026 08:29:13 +0000 Subject: [PATCH 04/19] Update Wan2.2 perf baselines Signed-off-by: bjf-frz --- .../perf/tests/test_wan22_i2v_vllm_omni.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index d70a28efd80..1934b6138e2 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -30,9 +30,9 @@ } ], "baseline": { - "throughput_qps": 0.04, - "latency_mean": 30, - "peak_memory_mb_mean": 75000 + "throughput_qps": 0.0421, + "latency_mean": 23.6918, + "peak_memory_mb_mean": 80195.22 } } ] @@ -72,9 +72,9 @@ } ], "baseline": { - "throughput_qps": 0.06, - "latency_mean": 20, - "peak_memory_mb_mean": 48000 + "throughput_qps": 0.0581, + "latency_mean": 17.1966, + "peak_memory_mb_mean": 50557.92 } }, { @@ -97,9 +97,9 @@ } ], "baseline": { - "throughput_qps": 0.01, - "latency_mean": 90, - "peak_memory_mb_mean": 56000 + "throughput_qps": 0.0116, + "latency_mean": 86.0519, + "peak_memory_mb_mean": 59626.56 } } ] From 2edd501dcbcfbf200760bf0dab825d8d7845317e Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Thu, 11 Jun 2026 16:36:38 +0800 Subject: [PATCH 05/19] hunyuan baseline Signed-off-by: dengyunyang <584797741@qq.com> --- tests/dfx/perf/tests/test_hunyuan_image3_it2i.json | 11 +++++------ tests/dfx/perf/tests/test_hunyuan_image_tp4.json | 9 +++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index b95aacd054e..60216d0093b 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -119,12 +119,11 @@ "num-prompts": 96, "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.1, - "latency_p99": 44.46, - "latency_mean": 35.57, - "latency_std_dev": 0.5, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + "throughput_qps": 0.18741696812379248, + "latency_p99": 101.85202961772058, + "latency_mean": 79.03846078318437, + "peak_memory_mb_max": 0, + "peak_memory_mb_mean": 0 } } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json index 8f4c0ec5000..adfb830992e 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json @@ -23,10 +23,11 @@ "max-concurrency": 1, "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.21, - "latency_p99": 4.9235, - "peak_memory_mb_max": 57576, - "peak_memory_mb_mean": 57576 + "throughput_qps": 0.21298869417358122, + "latency_mean": 4.695256627351046, + "latency_p99": 4.860056182866295, + "peak_memory_mb_max": 56912.0, + "peak_memory_mb_mean": 56912.0 } } ] From b539266ce206707b65026546c1cf82bda86ae83f Mon Sep 17 00:00:00 2001 From: "Huang, Zeyu" <11222265+fhfuih@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:37:44 +0800 Subject: [PATCH 06/19] adjust qwen image edit perf baseline Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> --- .../test_qwen_image_edit_2511_vllm_omni.json | 20 +++++++++---------- .../tests/test_qwen_image_edit_vllm_omni.json | 20 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 1903bc6708f..72111344f06 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -22,8 +22,8 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.06, - "latency_mean": 20, + "throughput_qps": 0.0651, + "latency_mean": 15.3204, "peak_memory_mb_max": 57000, "peak_memory_mb_mean": 57000 } @@ -40,8 +40,8 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.01, - "latency_mean": 60, + "throughput_qps": 0.0168, + "latency_mean": 59.3302, "peak_memory_mb_max": 67000, "peak_memory_mb_mean": 67000 } @@ -75,8 +75,8 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, - "latency_mean": 20, + "throughput_qps": 0.0488, + "latency_mean": 20.4401, "peak_memory_mb_max": 55000, "peak_memory_mb_mean": 55000 } @@ -120,8 +120,8 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.21, - "latency_mean": 10, + "throughput_qps": 0.2491, + "latency_mean": 4.0062, "peak_memory_mb_max": 57000, "peak_memory_mb_mean": 57000 } @@ -138,8 +138,8 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.08, - "latency_mean": 20, + "throughput_qps": 0.0951, + "latency_mean": 10.4885, "peak_memory_mb_max": 66000, "peak_memory_mb_mean": 66000 } diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index 21695b7528c..ac45f3d5a64 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -21,8 +21,8 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 10, + "throughput_qps": 0.1020, + "latency_mean": 9.7810, "peak_memory_mb_max": 57000, "peak_memory_mb_mean": 57000 } @@ -38,8 +38,8 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.02, - "latency_mean": 50, + "throughput_qps": 0.0217, + "latency_mean": 45.9861, "peak_memory_mb_max": 65000, "peak_memory_mb_mean": 65000 } @@ -72,8 +72,8 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.06, - "latency_mean": 20, + "throughput_qps": 0.0606, + "latency_mean": 16.4505, "peak_memory_mb_max": 56000, "peak_memory_mb_mean": 56000 } @@ -116,8 +116,8 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.26, - "latency_mean": 10, + "throughput_qps": 0.2526, + "latency_mean": 3.9584, "peak_memory_mb_max": 58000, "peak_memory_mb_mean": 58000 } @@ -133,8 +133,8 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 10, + "throughput_qps": 0.1001, + "latency_mean": 9.9661, "peak_memory_mb_max": 67000, "peak_memory_mb_mean": 67000 } From e3f9e15374e86e5dee1c6da0c4862acc809a0d82 Mon Sep 17 00:00:00 2001 From: CHEN <116010019@link.cuhk.edu.cn> Date: Thu, 11 Jun 2026 16:40:09 +0800 Subject: [PATCH 07/19] voxcpm2 baseline update Signed-off-by: CHEN <116010019@link.cuhk.edu.cn> (cherry picked from commit 5ec51d8dd404e3ee8f23106ca0465116d5cacbc2) --- tests/dfx/perf/tests/test_voxcpm2.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json index f19f95960c7..4f24582815b 100644 --- a/tests/dfx/perf/tests/test_voxcpm2.json +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -23,10 +23,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 600 + 145.22 ], "median_audio_rtf": [ - 0.2 + 0.0527 ] } }, @@ -47,13 +47,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 3500 + 323.33 ], "median_audio_rtf": [ - 0.85 + 0.156 ], "audio_throughput": [ - 8 + 24.88 ] } }, @@ -100,10 +100,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 350 + 85.56 ], "median_audio_rtf": [ - 0.15 + 0.0448 ] } }, @@ -127,13 +127,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 3000 + 228.46 ], "median_audio_rtf": [ - 0.5 + 0.1086 ], "audio_throughput": [ - 10 + 72.68 ] } }, @@ -157,13 +157,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 500 + 88.97 ], "median_audio_rtf": [ - 0.3 + 0.0488 ], "audio_throughput": [ - 5 + 16.69 ] } }, From ec46d87f04a91cf1dca08e5a70b9ea3acb8c050a Mon Sep 17 00:00:00 2001 From: rein yang <73573651+R2-Y@users.noreply.github.com> Date: Thu, 11 Jun 2026 18:02:49 +0800 Subject: [PATCH 08/19] update tts baseline (#13) Signed-off-by: rein yang <73573651+R2-Y@users.noreply.github.com> Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- tests/dfx/perf/tests/test_tts.json | 144 ++++++++++++++++++----------- 1 file changed, 89 insertions(+), 55 deletions(-) diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 9246b502f2f..250e64c2a70 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -27,10 +27,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 3000 + 147 ], "median_audio_rtf": [ - 0.6 + 0.18 ] } }, @@ -55,19 +55,19 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 1000, - 3000, - 14000 + 247, + 697, + 4709 ], "median_audio_rtf": [ - 0.70, - 1.15, - 3.90 + 0.35, + 0.38, + 1.26 ], "audio_throughput": [ - 12, - 14, - 14 + 21, + 39, + 41 ] } }, @@ -129,10 +129,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 2000 + 53 ], "median_audio_rtf": [ - 0.40 + 0.15 ] } }, @@ -162,19 +162,19 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 300, - 1000, - 5000 + 79, + 743, + 6517 ], "median_audio_rtf": [ - 0.25, - 0.35, - 1.10 + 0.2, + 0.34, + 1.27 ], "audio_throughput": [ - 30, - 40, - 40 + 35, + 47, + 43 ] } }, @@ -228,13 +228,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 100 + 61 ], "median_audio_rtf": [ - 0.20 + 0.17 ], "audio_throughput": [ - 10 + 11 ] } }, @@ -259,10 +259,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 100 + 55 ], "median_audio_rtf": [ - 0.15 + 0.16 ] } }, @@ -291,19 +291,19 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 100, - 1000, - 5000 + 78, + 616, + 5061 ], "median_audio_rtf": [ - 0.25, - 0.40, - 1.20 + 0.2, + 0.34, + 1.17 ], "audio_throughput": [ - 30, - 40, - 40 + 39, + 47, + 47 ] } }, @@ -328,10 +328,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 100 + 62 ], "median_audio_rtf": [ - 0.20 + 0.17 ], "audio_throughput": [ 9 @@ -353,14 +353,24 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [20], - "max_concurrency": [1], + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "casual_male"}, + "extra_body": { + "voice": "casual_male" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [150], - "median_audio_rtf": [0.15] + "median_audio_ttfp_ms": [ + 150 + ], + "median_audio_rtf": [ + 0.15 + ] } }, { @@ -370,15 +380,27 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [80], - "max_concurrency": [8], + "num_prompts": [ + 80 + ], + "max_concurrency": [ + 8 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "casual_male"}, + "extra_body": { + "voice": "casual_male" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [1500], - "median_audio_rtf": [0.30], - "audio_throughput": [30.0] + "median_audio_ttfp_ms": [ + 1500 + ], + "median_audio_rtf": [ + 0.3 + ], + "audio_throughput": [ + 30.0 + ] } }, { @@ -388,15 +410,27 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [100], - "request_rate": [2.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 2.0 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "casual_male"}, + "extra_body": { + "voice": "casual_male" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [3000], - "median_audio_rtf": [0.25], - "audio_throughput": [4.0] + "median_audio_ttfp_ms": [ + 3000 + ], + "median_audio_rtf": [ + 0.25 + ], + "audio_throughput": [ + 4.0 + ] } } ] From 486a901d39dcdf3ac173a0541016df91606838b5 Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Fri, 12 Jun 2026 15:06:57 +0800 Subject: [PATCH 09/19] Update bagel baseline Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- .../dfx/perf/tests/test_bagel_vllm_omni.json | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json index 09fc96030b8..cc52d3a5cb9 100644 --- a/tests/dfx/perf/tests/test_bagel_vllm_omni.json +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -28,10 +28,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.3678, - "latency_mean": 2.691, - "peak_memory_mb_max": 33083.6, - "peak_memory_mb_mean": 33082.5 + "throughput_qps": 0.3193636894676252, + "latency_mean": 3.1331455306547077, + "latency_p99": 3.445326166654205, + "peak_memory_mb_max": 30074.0, + "peak_memory_mb_mean": 30073.0 } } ] @@ -66,10 +67,11 @@ "enable-negative-prompt": true, "num-input-images": 1, "baseline": { - "throughput_qps": 0.3124, - "latency_mean": 3.1686, - "peak_memory_mb_max": 33448.8, - "peak_memory_mb_mean": 33447.7 + "throughput_qps": 0.2776741950959088, + "latency_mean": 3.60918410273537, + "latency_p99": 4.398247424278628, + "peak_memory_mb_max": 30070.0, + "peak_memory_mb_mean": 30070.0 } } ] @@ -103,10 +105,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.3746, - "latency_mean": 2.6425, - "peak_memory_mb_max": 33077.0, - "peak_memory_mb_mean": 33077.0 + "throughput_qps": 0.3131853155139403, + "latency_mean": 3.195182329009549, + "latency_p99": 3.7151144233344944, + "peak_memory_mb_max": 0.0, + "peak_memory_mb_mean": 0.0 } } ] @@ -141,10 +144,11 @@ "enable-negative-prompt": true, "num-input-images": 1, "baseline": { - "throughput_qps": 0.1349, - "latency_mean": 7.339, - "peak_memory_mb_max": 33321.2, - "peak_memory_mb_mean": 33321.2 + "throughput_qps": 0.13625870107838878, + "latency_mean": 7.345074247307223, + "latency_p99": 8.528473965078425, + "peak_memory_mb_max": 0.0, + "peak_memory_mb_mean": 0.0 } } ] From cdfdaccd5c137bd35e1e65a6ca43d3c0a264e318 Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Tue, 16 Jun 2026 15:54:29 +0800 Subject: [PATCH 10/19] Update qwen image edit Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- .../tests/test_qwen_image_edit_vllm_omni.json | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index ac45f3d5a64..da32881820d 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -21,10 +21,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1020, - "latency_mean": 9.7810, - "peak_memory_mb_max": 57000, - "peak_memory_mb_mean": 57000 + "throughput_qps": 0.1129, + "latency_mean": 8.8352, + "peak_memory_mb_max": 55356.5, + "peak_memory_mb_mean": 55356.5 } }, { @@ -38,10 +38,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0217, - "latency_mean": 45.9861, - "peak_memory_mb_max": 65000, - "peak_memory_mb_mean": 65000 + "throughput_qps": 0.0240, + "latency_mean": 41.5686, + "peak_memory_mb_max": 62762.7, + "peak_memory_mb_mean": 62762.7 } } ] @@ -72,10 +72,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0606, - "latency_mean": 16.4505, - "peak_memory_mb_max": 56000, - "peak_memory_mb_mean": 56000 + "throughput_qps": 0.0672, + "latency_mean": 14.8517, + "peak_memory_mb_max": 53931.5, + "peak_memory_mb_mean": 53931.5 } } ] @@ -116,10 +116,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.2526, - "latency_mean": 3.9584, - "peak_memory_mb_max": 58000, - "peak_memory_mb_mean": 58000 + "throughput_qps": 0.2879, + "latency_mean": 3.4701, + "peak_memory_mb_max": 55525.6, + "peak_memory_mb_mean": 55525.6 } }, { @@ -133,10 +133,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1001, - "latency_mean": 9.9661, - "peak_memory_mb_max": 67000, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.1113, + "latency_mean": 8.9656, + "peak_memory_mb_max": 63758.3, + "peak_memory_mb_mean": 63758.3 } } ] From dbacac7787df3972852d85255ed5ad8196007299 Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Tue, 16 Jun 2026 16:38:00 +0800 Subject: [PATCH 11/19] Revert bagel baseline due to regression. Remove hunyuan memory updation. Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- .../dfx/perf/tests/test_bagel_vllm_omni.json | 36 +++++++++---------- .../perf/tests/test_hunyuan_image3_it2i.json | 18 +++++++--- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json index cc52d3a5cb9..09fc96030b8 100644 --- a/tests/dfx/perf/tests/test_bagel_vllm_omni.json +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -28,11 +28,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.3193636894676252, - "latency_mean": 3.1331455306547077, - "latency_p99": 3.445326166654205, - "peak_memory_mb_max": 30074.0, - "peak_memory_mb_mean": 30073.0 + "throughput_qps": 0.3678, + "latency_mean": 2.691, + "peak_memory_mb_max": 33083.6, + "peak_memory_mb_mean": 33082.5 } } ] @@ -67,11 +66,10 @@ "enable-negative-prompt": true, "num-input-images": 1, "baseline": { - "throughput_qps": 0.2776741950959088, - "latency_mean": 3.60918410273537, - "latency_p99": 4.398247424278628, - "peak_memory_mb_max": 30070.0, - "peak_memory_mb_mean": 30070.0 + "throughput_qps": 0.3124, + "latency_mean": 3.1686, + "peak_memory_mb_max": 33448.8, + "peak_memory_mb_mean": 33447.7 } } ] @@ -105,11 +103,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.3131853155139403, - "latency_mean": 3.195182329009549, - "latency_p99": 3.7151144233344944, - "peak_memory_mb_max": 0.0, - "peak_memory_mb_mean": 0.0 + "throughput_qps": 0.3746, + "latency_mean": 2.6425, + "peak_memory_mb_max": 33077.0, + "peak_memory_mb_mean": 33077.0 } } ] @@ -144,11 +141,10 @@ "enable-negative-prompt": true, "num-input-images": 1, "baseline": { - "throughput_qps": 0.13625870107838878, - "latency_mean": 7.345074247307223, - "latency_p99": 8.528473965078425, - "peak_memory_mb_max": 0.0, - "peak_memory_mb_mean": 0.0 + "throughput_qps": 0.1349, + "latency_mean": 7.339, + "peak_memory_mb_max": 33321.2, + "peak_memory_mb_mean": 33321.2 } } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index 60216d0093b..c8ca576e739 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -41,7 +41,11 @@ "tensor_parallel_size": 2, "hf_overrides": { "rope_parameters": { - "mrope_section": [0, 32, 32], + "mrope_section": [ + 0, + 32, + 32 + ], "rope_type": "default" } }, @@ -58,7 +62,9 @@ "max_tokens": 8192, "detokenize": true, "skip_special_tokens": false, - "stop_token_ids": [128025] + "stop_token_ids": [ + 128025 + ] } }, { @@ -106,7 +112,9 @@ "content": [ { "prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。", - "image_urls": ["https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png"], + "image_urls": [ + "https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png" + ], "num_inference_steps": 8, "seed": 42, "guidance_scale": 1.0 @@ -122,8 +130,8 @@ "throughput_qps": 0.18741696812379248, "latency_p99": 101.85202961772058, "latency_mean": 79.03846078318437, - "peak_memory_mb_max": 0, - "peak_memory_mb_mean": 0 + "peak_memory_mb_max": 70000, + "peak_memory_mb_mean": 70000 } } ] From 71cf59d7d6abf14b733b2669f3300cbe56baaaab Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Tue, 16 Jun 2026 16:39:06 +0800 Subject: [PATCH 12/19] Update qwen image 2511 baseline Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- .../test_qwen_image_edit_2511_vllm_omni.json | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 72111344f06..118fa6733a0 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -22,10 +22,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0651, - "latency_mean": 15.3204, - "peak_memory_mb_max": 57000, - "peak_memory_mb_mean": 57000 + "throughput_qps": 0.0721, + "latency_mean": 13.8398, + "peak_memory_mb_max": 54731.4, + "peak_memory_mb_mean": 54731.4 } }, { @@ -40,10 +40,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0168, - "latency_mean": 59.3302, - "peak_memory_mb_max": 67000, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.0186, + "latency_mean": 53.6942, + "peak_memory_mb_max": 64402.4, + "peak_memory_mb_mean": 64402.4 } } ] @@ -75,10 +75,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0488, - "latency_mean": 20.4401, - "peak_memory_mb_max": 55000, - "peak_memory_mb_mean": 55000 + "throughput_qps": 0.0538, + "latency_mean": 18.5591, + "peak_memory_mb_max": 53485.0, + "peak_memory_mb_mean": 53485.0 } } ] @@ -120,10 +120,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.2491, - "latency_mean": 4.0062, - "peak_memory_mb_max": 57000, - "peak_memory_mb_mean": 57000 + "throughput_qps": 0.2776, + "latency_mean": 3.5938, + "peak_memory_mb_max": 55214.0, + "peak_memory_mb_mean": 55214.0 } }, { @@ -138,10 +138,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0951, - "latency_mean": 10.4885, - "peak_memory_mb_max": 66000, - "peak_memory_mb_mean": 66000 + "throughput_qps": 0.1051, + "latency_mean": 9.4918, + "peak_memory_mb_max": 64265.6, + "peak_memory_mb_mean": 64265.6 } } ] From 37502ca1faf07e778ac347c7656cfedc40c82eca Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:21:34 +0800 Subject: [PATCH 13/19] Hunyuan metirc round to 4 decimals. Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- tests/dfx/perf/tests/test_hunyuan_image3_it2i.json | 6 +++--- tests/dfx/perf/tests/test_hunyuan_image_tp4.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index c8ca576e739..200251ba6dc 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -127,9 +127,9 @@ "num-prompts": 96, "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.18741696812379248, - "latency_p99": 101.85202961772058, - "latency_mean": 79.03846078318437, + "throughput_qps": 0.1874, + "latency_p99": 101.852, + "latency_mean": 79.0385, "peak_memory_mb_max": 70000, "peak_memory_mb_mean": 70000 } diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json index adfb830992e..eb37f7a2261 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json @@ -23,9 +23,9 @@ "max-concurrency": 1, "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.21298869417358122, - "latency_mean": 4.695256627351046, - "latency_p99": 4.860056182866295, + "throughput_qps": 0.213, + "latency_mean": 4.6953, + "latency_p99": 4.8601, "peak_memory_mb_max": 56912.0, "peak_memory_mb_mean": 56912.0 } From 7a0a6de8af3ebe9bb1a04a90dd1b81d13b7c8a57 Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:22:13 +0800 Subject: [PATCH 14/19] Update qwen 3 tts baseline Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- tests/dfx/perf/tests/test_tts.json | 76 +++++++++++++++--------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 250e64c2a70..76d095bb35e 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -27,10 +27,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 147 + 122.4504 ], "median_audio_rtf": [ - 0.18 + 0.1472 ] } }, @@ -55,19 +55,19 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 247, - 697, - 4709 + 199.1197, + 647.1272, + 4777.2714 ], "median_audio_rtf": [ - 0.35, - 0.38, - 1.26 + 0.2899, + 0.3642, + 1.2282 ], "audio_throughput": [ - 21, - 39, - 41 + 20.5634, + 40.634, + 43.0758 ] } }, @@ -129,10 +129,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 53 + 43.5669 ], "median_audio_rtf": [ - 0.15 + 0.1247 ] } }, @@ -162,19 +162,19 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 79, - 743, - 6517 + 65.9174, + 713.7727, + 5000 ], "median_audio_rtf": [ - 0.2, - 0.34, - 1.27 + 0.1626, + 0.3148, + 1.1 ], "audio_throughput": [ - 35, - 47, - 43 + 31.9068, + 49.6268, + 48.3091 ] } }, @@ -228,13 +228,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 61 + 49.6105 ], "median_audio_rtf": [ - 0.17 + 0.1361 ], "audio_throughput": [ - 11 + 10.0881 ] } }, @@ -259,10 +259,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 55 + 45.5391 ], "median_audio_rtf": [ - 0.16 + 0.1271 ] } }, @@ -291,19 +291,19 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 78, - 616, - 5061 + 66.1471, + 587.5877, + 4960.3726 ], "median_audio_rtf": [ - 0.2, - 0.34, - 1.17 + 0.1637, + 0.3143, + 1.1703 ], "audio_throughput": [ - 39, - 47, - 47 + 35.1923, + 48.9046, + 48.0505 ] } }, @@ -328,10 +328,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 62 + 49.3466 ], "median_audio_rtf": [ - 0.17 + 0.1349 ], "audio_throughput": [ 9 From 53b750ad849a8feaa3abc8bae658e367f71564f6 Mon Sep 17 00:00:00 2001 From: NumberWan Date: Tue, 16 Jun 2026 09:27:00 +0000 Subject: [PATCH 15/19] Update Qwen Image and Qwen Image Layered perf baselines Signed-off-by: NumberWan --- .../test_qwen_image_layered_vllm_omni.json | 16 +++---- .../perf/tests/test_qwen_image_vllm_omni.json | 47 +++++++++---------- 2 files changed, 29 insertions(+), 34 deletions(-) diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index 7b449bb56b9..a94db56f5f6 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -21,10 +21,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.06, - "latency_mean": 16.24, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + "throughput_qps": 0.0703, + "latency_mean": 14.208, + "peak_memory_mb_max": 60543.5, + "peak_memory_mb_mean": 60543.5 } }, { @@ -38,10 +38,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.04, - "latency_mean": 27.60, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + "throughput_qps": 0.0417, + "latency_mean": 23.914, + "peak_memory_mb_max": 60543.5, + "peak_memory_mb_mean": 60543.5 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index 3f0771c14a7..5ca6ed61af0 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -21,10 +21,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.35, - "latency_mean": 2.85, - "peak_memory_mb_mean": 61000, - "peak_memory_mb_max": 61000 + "throughput_qps": 0.4153, + "latency_mean": 2.4065, + "peak_memory_mb_mean": 52289.9 } }, { @@ -38,10 +37,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.04, - "latency_mean": 27.32, - "peak_memory_mb_mean": 71000, - "peak_memory_mb_max": 71000 + "throughput_qps": 0.0423, + "latency_mean": 23.5991, + "peak_memory_mb_mean": 61360.5 } } ] @@ -69,8 +67,8 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.35, - "latency_mean": 2.82, + "throughput_qps": 0.4175, + "latency_mean": 2.3953, "peak_memory_mb_mean": 56000 } }, @@ -85,8 +83,8 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.04, - "latency_mean": 27.29, + "throughput_qps": 0.0423, + "latency_mean": 23.5768, "peak_memory_mb_mean": 65000 } } @@ -118,10 +116,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 9.6, - "peak_memory_mb_mean": 60000, - "peak_memory_mb_max": 60000 + "throughput_qps": 0.1214, + "latency_mean": 8.2211, + "peak_memory_mb_mean": 51704.7 } } ] @@ -162,10 +159,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.46, - "latency_mean": 2.17, - "peak_memory_mb_mean": 61000, - "peak_memory_mb_max": 61000 + "throughput_qps": 0.559, + "latency_mean": 1.794, + "peak_memory_mb_mean": 52397.6775 } }, { @@ -179,10 +175,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.16, - "latency_mean": 6.07, - "peak_memory_mb_mean": 71000, - "peak_memory_mb_max": 71000 + "throughput_qps": 0.1918, + "latency_mean": 5.2022, + "peak_memory_mb_mean": 61468.8 } } ] @@ -214,8 +209,8 @@ "warmup-num-inference-steps": 20, "enable-negative-prompt": true, "baseline": { - "throughput_qps": [0.36, 0.58, 0.57, 0.71], - "latency_mean": [2.76, 3.47, 6.86, 11.07], + "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], + "latency_mean": [2.4215, 2.992, 5.9293, 9.656], "peak_memory_mb_mean": [67000, 67000, 67000, 67000] } } From 7fda3e8a7447e3421f202442322f0b8e5ea2e61b Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:33:15 +0800 Subject: [PATCH 16/19] Update voxcpm2 baseline Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- tests/dfx/perf/tests/test_voxcpm2.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json index 4f24582815b..0aaca446fc3 100644 --- a/tests/dfx/perf/tests/test_voxcpm2.json +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -23,10 +23,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 145.22 + 113.9351 ], "median_audio_rtf": [ - 0.0527 + 0.0427 ] } }, @@ -47,13 +47,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 323.33 + 415.9187 ], "median_audio_rtf": [ - 0.156 + 0.1462 ], "audio_throughput": [ - 24.88 + 20.0128 ] } }, @@ -100,10 +100,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 85.56 + 66.2748 ], "median_audio_rtf": [ - 0.0448 + 0.0366 ] } }, @@ -127,13 +127,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 228.46 + 307.6373 ], "median_audio_rtf": [ - 0.1086 + 0.1056 ], "audio_throughput": [ - 72.68 + 64.5428 ] } }, @@ -157,13 +157,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 88.97 + 70.0958 ], "median_audio_rtf": [ - 0.0488 + 0.0424 ], "audio_throughput": [ - 16.69 + 16.075 ] } }, From 6d27b9b9f80ddee55fc80a19d6adf38850d0a8ab Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Tue, 16 Jun 2026 09:57:03 +0000 Subject: [PATCH 17/19] Fix Wan2.2 baseline update Signed-off-by: bjf-frz --- tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 1934b6138e2..77536d0b90f 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -32,7 +32,7 @@ "baseline": { "throughput_qps": 0.0421, "latency_mean": 23.6918, - "peak_memory_mb_mean": 80195.22 + "peak_memory_mb_mean": 72557.58 } } ] @@ -72,9 +72,9 @@ } ], "baseline": { - "throughput_qps": 0.0581, + "throughput_qps": 0.06, "latency_mean": 17.1966, - "peak_memory_mb_mean": 50557.92 + "peak_memory_mb_mean": 45742.88 } }, { @@ -99,7 +99,7 @@ "baseline": { "throughput_qps": 0.0116, "latency_mean": 86.0519, - "peak_memory_mb_mean": 59626.56 + "peak_memory_mb_mean": 53947.84 } } ] From 8b2b2b77f8a71909127ad3b2045bffbcc1f729a7 Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Wed, 17 Jun 2026 09:47:36 +0800 Subject: [PATCH 18/19] Update wan2.2 Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 77536d0b90f..07873a64b02 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -30,8 +30,8 @@ } ], "baseline": { - "throughput_qps": 0.0421, - "latency_mean": 23.6918, + "throughput_qps": 0.0465, + "latency_mean": 21.4354, "peak_memory_mb_mean": 72557.58 } } @@ -72,8 +72,8 @@ } ], "baseline": { - "throughput_qps": 0.06, - "latency_mean": 17.1966, + "throughput_qps": 0.0642, + "latency_mean": 15.5589, "peak_memory_mb_mean": 45742.88 } }, @@ -97,8 +97,8 @@ } ], "baseline": { - "throughput_qps": 0.0116, - "latency_mean": 86.0519, + "throughput_qps": 0.0128, + "latency_mean": 77.8565, "peak_memory_mb_mean": 53947.84 } } From 8352a34159cd8d0f44755add97f3a17e33145eab Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Wed, 17 Jun 2026 10:54:27 +0800 Subject: [PATCH 19/19] Update qwen3omni and qwen3tts Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- .../tests/test_qwen3_omni_async_chunk.json | 80 +++++++++---------- .../tests/test_qwen3_omni_no_async_chunk.json | 48 +++++------ .../perf/tests/test_qwen3_omni_vllm_text.json | 32 ++++---- tests/dfx/perf/tests/test_tts.json | 4 +- 4 files changed, 82 insertions(+), 82 deletions(-) diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 9a51dbee756..459554f6308 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -32,25 +32,25 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 146, - 222, - 350, - 462, - 807 + 135.4379, + 187.0896, + 400, + 435.675, + 724.6836 ], "mean_e2el_ms": [ - 21872, - 39929, - 48687, - 62009, - 93860 + 17619.9851, + 34811.8335, + 43656.9501, + 56657.1548, + 85097.6016 ], "mean_audio_rtf": [ - 0.15, - 0.26, - 0.33, - 0.43, - 0.65 + 0.1279, + 0.2238, + 0.2735, + 0.3712, + 0.5634 ] } }, @@ -79,13 +79,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 406 + 340.329 ], "mean_e2el_ms": [ - 5245 + 4305.477 ], "mean_audio_rtf": [ - 0.16 + 0.1305 ] } }, @@ -116,13 +116,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 271 + 220.0909 ], "mean_e2el_ms": [ - 6440 + 5475.9565 ], "mean_audio_rtf": [ - 0.22 + 0.1818 ] } }, @@ -155,13 +155,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 495 + 450.0155 ], "mean_e2el_ms": [ - 9229 + 8516.8316 ], "mean_audio_rtf": [ - 0.3 + 0.2717 ] } }, @@ -194,18 +194,18 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 86, - 228, - 390, - 706, - 1281 + 78.4278, + 195.8037, + 333.7318, + 561.805, + 1038.043 ], "mean_e2el_ms": [ - 4933, - 8532, - 11005, - 14591, - 21004 + 4288.3029, + 7429.9429, + 9451.6184, + 12607.5292, + 18153.2547 ] } }, @@ -239,10 +239,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 319 + 275.3875 ], "mean_e2el_ms": [ - 841 + 730.6995 ] } }, @@ -278,10 +278,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 214 + 166.2201 ], "mean_e2el_ms": [ - 802 + 673.5789 ] } }, @@ -319,10 +319,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 388 + 357.7781 ], "mean_e2el_ms": [ - 1071 + 953.3493 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index af94f607c63..d01ca06b43d 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -32,25 +32,25 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 136, - 283, - 461, - 455, - 829 + 109.1218, + 247.4411, + 354.9384, + 447.7885, + 694.7487 ], "mean_e2el_ms": [ - 27765, - 34669, - 43197, - 53022, - 76128 + 22413.667, + 30808.9715, + 35677.844, + 46345.8978, + 68350.4853 ], "mean_audio_rtf": [ - 0.2, - 0.23, - 0.27, - 0.38, - 0.54 + 0.1695, + 0.2033, + 0.2467, + 0.3234, + 0.4438 ] } }, @@ -79,13 +79,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 363 + 323.3573 ], "mean_e2el_ms": [ - 5364 + 4623.4173 ], "mean_audio_rtf": [ - 0.17 + 0.1406 ] } }, @@ -116,13 +116,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 257 + 215.4502 ], "mean_e2el_ms": [ - 5719 + 4924.149 ], "mean_audio_rtf": [ - 0.2 + 0.1637 ] } }, @@ -155,13 +155,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 1744 + 442.5815 ], "mean_e2el_ms": [ - 9817 + 6441.0986 ], "mean_audio_rtf": [ - 0.32 + 0.2091 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index 6d8e084a311..f608eca0e47 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -40,18 +40,18 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 90, - 355, - 348, - 514, - 703 + 69.1276, + 255.1882, + 310.639, + 422.0337, + 532.0582 ], "mean_e2el_ms": [ - 5212, - 8875, - 11085, - 15008, - 20113 + 4382.7211, + 7510.3815, + 9440.4854, + 12553.0186, + 16932.6065 ] } }, @@ -86,10 +86,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 31 + 26.3381 ], "mean_e2el_ms": [ - 579 + 485.2436 ] } }, @@ -126,10 +126,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 32 + 26.4517 ], "mean_e2el_ms": [ - 625 + 517.2017 ] } }, @@ -168,10 +168,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 35 + 27.6972 ], "mean_e2el_ms": [ - 681 + 561.3684 ] } } diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 76d095bb35e..ba1f9526143 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -164,12 +164,12 @@ "median_audio_ttfp_ms": [ 65.9174, 713.7727, - 5000 + 5941.7179 ], "median_audio_rtf": [ 0.1626, 0.3148, - 1.1 + 1.1662 ], "audio_throughput": [ 31.9068,