diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index b95aacd054e..200251ba6dc 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -41,7 +41,11 @@ "tensor_parallel_size": 2, "hf_overrides": { "rope_parameters": { - "mrope_section": [0, 32, 32], + "mrope_section": [ + 0, + 32, + 32 + ], "rope_type": "default" } }, @@ -58,7 +62,9 @@ "max_tokens": 8192, "detokenize": true, "skip_special_tokens": false, - "stop_token_ids": [128025] + "stop_token_ids": [ + 128025 + ] } }, { @@ -106,7 +112,9 @@ "content": [ { "prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。", - "image_urls": ["https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png"], + "image_urls": [ + "https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png" + ], "num_inference_steps": 8, "seed": 42, "guidance_scale": 1.0 @@ -119,10 +127,9 @@ "num-prompts": 96, "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.1, - "latency_p99": 44.46, - "latency_mean": 35.57, - "latency_std_dev": 0.5, + "throughput_qps": 0.1874, + "latency_p99": 101.852, + "latency_mean": 79.0385, "peak_memory_mb_max": 70000, "peak_memory_mb_mean": 70000 } diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json index 8f4c0ec5000..eb37f7a2261 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json @@ -23,10 +23,11 @@ "max-concurrency": 1, "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.21, - "latency_p99": 4.9235, - "peak_memory_mb_max": 57576, - "peak_memory_mb_mean": 57576 + "throughput_qps": 0.213, + "latency_mean": 4.6953, + "latency_p99": 4.8601, + "peak_memory_mb_max": 56912.0, + "peak_memory_mb_mean": 56912.0 } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index bc954cee59f..459554f6308 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -32,25 +32,25 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ + 135.4379, + 187.0896, 400, - 300, - 400, - 600, - 900 + 435.675, + 724.6836 ], "mean_e2el_ms": [ - 26000, - 43000, - 52000, - 72000, - 109000 + 17619.9851, + 34811.8335, + 43656.9501, + 56657.1548, + 85097.6016 ], "mean_audio_rtf": [ - 0.15, - 0.30, - 0.35, - 0.50, - 0.75 + 0.1279, + 0.2238, + 0.2735, + 0.3712, + 0.5634 ] } }, @@ -79,13 +79,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 600 + 340.329 ], "mean_e2el_ms": [ - 6000 + 4305.477 ], "mean_audio_rtf": [ - 0.20 + 0.1305 ] } }, @@ -116,13 +116,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 300 + 220.0909 ], "mean_e2el_ms": [ - 7000 + 5475.9565 ], "mean_audio_rtf": [ - 0.25 + 0.1818 ] } }, @@ -155,13 +155,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 700 + 450.0155 ], "mean_e2el_ms": [ - 11000 + 8516.8316 ], "mean_audio_rtf": [ - 0.35 + 0.2717 ] } }, @@ -194,18 +194,18 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 100, - 300, - 500, - 800, - 1500 + 78.4278, + 195.8037, + 333.7318, + 561.805, + 1038.043 ], "mean_e2el_ms": [ - 5000, - 8000, - 10000, - 14000, - 20000 + 4288.3029, + 7429.9429, + 9451.6184, + 12607.5292, + 18153.2547 ] } }, @@ -239,10 +239,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 500 + 275.3875 ], "mean_e2el_ms": [ - 1000 + 730.6995 ] } }, @@ -278,10 +278,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 200 + 166.2201 ], "mean_e2el_ms": [ - 800 + 673.5789 ] } }, @@ -319,10 +319,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 600 + 357.7781 ], "mean_e2el_ms": [ - 1300 + 953.3493 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index e57acecba4a..d01ca06b43d 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -32,25 +32,25 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 400, - 300, - 400, - 600, - 900 + 109.1218, + 247.4411, + 354.9384, + 447.7885, + 694.7487 ], "mean_e2el_ms": [ - 32000, - 34000, - 45000, - 61000, - 90000 + 22413.667, + 30808.9715, + 35677.844, + 46345.8978, + 68350.4853 ], "mean_audio_rtf": [ - 0.20, - 0.25, - 0.30, - 0.45, - 0.60 + 0.1695, + 0.2033, + 0.2467, + 0.3234, + 0.4438 ] } }, @@ -79,13 +79,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 500 + 323.3573 ], "mean_e2el_ms": [ - 6000 + 4623.4173 ], "mean_audio_rtf": [ - 0.20 + 0.1406 ] } }, @@ -116,13 +116,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 300 + 215.4502 ], "mean_e2el_ms": [ - 7000 + 4924.149 ], "mean_audio_rtf": [ - 0.20 + 0.1637 ] } }, @@ -155,13 +155,13 @@ "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [ - 700 + 442.5815 ], "mean_e2el_ms": [ - 9000 + 6441.0986 ], "mean_audio_rtf": [ - 0.30 + 0.2091 ] } } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index a973bb0e64b..f608eca0e47 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -40,18 +40,18 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 300, - 300, - 1000, - 600, - 800 + 69.1276, + 255.1882, + 310.639, + 422.0337, + 532.0582 ], "mean_e2el_ms": [ - 6000, - 8000, - 11000, - 13000, - 17000 + 4382.7211, + 7510.3815, + 9440.4854, + 12553.0186, + 16932.6065 ] } }, @@ -86,10 +86,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 30 + 26.3381 ], "mean_e2el_ms": [ - 540 + 485.2436 ] } }, @@ -126,10 +126,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 30 + 26.4517 ], "mean_e2el_ms": [ - 600 + 517.2017 ] } }, @@ -168,10 +168,10 @@ "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { "mean_ttft_ms": [ - 40 + 27.6972 ], "mean_e2el_ms": [ - 700 + 561.3684 ] } } diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 1903bc6708f..118fa6733a0 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -22,10 +22,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.06, - "latency_mean": 20, - "peak_memory_mb_max": 57000, - "peak_memory_mb_mean": 57000 + "throughput_qps": 0.0721, + "latency_mean": 13.8398, + "peak_memory_mb_max": 54731.4, + "peak_memory_mb_mean": 54731.4 } }, { @@ -40,10 +40,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.01, - "latency_mean": 60, - "peak_memory_mb_max": 67000, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.0186, + "latency_mean": 53.6942, + "peak_memory_mb_max": 64402.4, + "peak_memory_mb_mean": 64402.4 } } ] @@ -75,10 +75,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, - "latency_mean": 20, - "peak_memory_mb_max": 55000, - "peak_memory_mb_mean": 55000 + "throughput_qps": 0.0538, + "latency_mean": 18.5591, + "peak_memory_mb_max": 53485.0, + "peak_memory_mb_mean": 53485.0 } } ] @@ -120,10 +120,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.21, - "latency_mean": 10, - "peak_memory_mb_max": 57000, - "peak_memory_mb_mean": 57000 + "throughput_qps": 0.2776, + "latency_mean": 3.5938, + "peak_memory_mb_max": 55214.0, + "peak_memory_mb_mean": 55214.0 } }, { @@ -138,10 +138,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.08, - "latency_mean": 20, - "peak_memory_mb_max": 66000, - "peak_memory_mb_mean": 66000 + "throughput_qps": 0.1051, + "latency_mean": 9.4918, + "peak_memory_mb_max": 64265.6, + "peak_memory_mb_mean": 64265.6 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index 21695b7528c..da32881820d 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -21,10 +21,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 10, - "peak_memory_mb_max": 57000, - "peak_memory_mb_mean": 57000 + "throughput_qps": 0.1129, + "latency_mean": 8.8352, + "peak_memory_mb_max": 55356.5, + "peak_memory_mb_mean": 55356.5 } }, { @@ -38,10 +38,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.02, - "latency_mean": 50, - "peak_memory_mb_max": 65000, - "peak_memory_mb_mean": 65000 + "throughput_qps": 0.0240, + "latency_mean": 41.5686, + "peak_memory_mb_max": 62762.7, + "peak_memory_mb_mean": 62762.7 } } ] @@ -72,10 +72,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.06, - "latency_mean": 20, - "peak_memory_mb_max": 56000, - "peak_memory_mb_mean": 56000 + "throughput_qps": 0.0672, + "latency_mean": 14.8517, + "peak_memory_mb_max": 53931.5, + "peak_memory_mb_mean": 53931.5 } } ] @@ -116,10 +116,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.26, - "latency_mean": 10, - "peak_memory_mb_max": 58000, - "peak_memory_mb_mean": 58000 + "throughput_qps": 0.2879, + "latency_mean": 3.4701, + "peak_memory_mb_max": 55525.6, + "peak_memory_mb_mean": 55525.6 } }, { @@ -133,10 +133,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 10, - "peak_memory_mb_max": 67000, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.1113, + "latency_mean": 8.9656, + "peak_memory_mb_max": 63758.3, + "peak_memory_mb_mean": 63758.3 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index c0cd3b7c015..a94db56f5f6 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -21,10 +21,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, - "latency_mean": 20, - "peak_memory_mb_max": 63000, - "peak_memory_mb_mean": 63000 + "throughput_qps": 0.0703, + "latency_mean": 14.208, + "peak_memory_mb_max": 60543.5, + "peak_memory_mb_mean": 60543.5 } }, { @@ -38,10 +38,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.03, - "latency_mean": 30, - "peak_memory_mb_max": 63000, - "peak_memory_mb_mean": 63000 + "throughput_qps": 0.0417, + "latency_mean": 23.914, + "peak_memory_mb_max": 60543.5, + "peak_memory_mb_mean": 60543.5 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index a6eacf88e59..5ca6ed61af0 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -21,9 +21,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.35, - "latency_mean": 3, - "peak_memory_mb_mean": 56000 + "throughput_qps": 0.4153, + "latency_mean": 2.4065, + "peak_memory_mb_mean": 52289.9 } }, { @@ -37,9 +37,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.03, - "latency_mean": 26, - "peak_memory_mb_mean": 65000 + "throughput_qps": 0.0423, + "latency_mean": 23.5991, + "peak_memory_mb_mean": 61360.5 } } ] @@ -67,8 +67,8 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.35, - "latency_mean": 3, + "throughput_qps": 0.4175, + "latency_mean": 2.3953, "peak_memory_mb_mean": 56000 } }, @@ -83,8 +83,8 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.03, - "latency_mean": 26, + "throughput_qps": 0.0423, + "latency_mean": 23.5768, "peak_memory_mb_mean": 65000 } } @@ -116,9 +116,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.11, - "latency_mean": 9, - "peak_memory_mb_mean": 55000 + "throughput_qps": 0.1214, + "latency_mean": 8.2211, + "peak_memory_mb_mean": 51704.7 } } ] @@ -159,9 +159,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.46, - "latency_mean": 3, - "peak_memory_mb_mean": 56000 + "throughput_qps": 0.559, + "latency_mean": 1.794, + "peak_memory_mb_mean": 52397.6775 } }, { @@ -175,9 +175,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.18, - "latency_mean": 6, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.1918, + "latency_mean": 5.2022, + "peak_memory_mb_mean": 61468.8 } } ] @@ -209,8 +209,8 @@ "warmup-num-inference-steps": 20, "enable-negative-prompt": true, "baseline": { - "throughput_qps": [0.30, 0.50, 0.60, 0.70], - "latency_mean": [3.50, 4.00, 7.00, 12.00], + "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], + "latency_mean": [2.4215, 2.992, 5.9293, 9.656], "peak_memory_mb_mean": [67000, 67000, 67000, 67000] } } diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 9246b502f2f..ba1f9526143 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -27,10 +27,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 3000 + 122.4504 ], "median_audio_rtf": [ - 0.6 + 0.1472 ] } }, @@ -55,19 +55,19 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 1000, - 3000, - 14000 + 199.1197, + 647.1272, + 4777.2714 ], "median_audio_rtf": [ - 0.70, - 1.15, - 3.90 + 0.2899, + 0.3642, + 1.2282 ], "audio_throughput": [ - 12, - 14, - 14 + 20.5634, + 40.634, + 43.0758 ] } }, @@ -129,10 +129,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 2000 + 43.5669 ], "median_audio_rtf": [ - 0.40 + 0.1247 ] } }, @@ -162,19 +162,19 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 300, - 1000, - 5000 + 65.9174, + 713.7727, + 5941.7179 ], "median_audio_rtf": [ - 0.25, - 0.35, - 1.10 + 0.1626, + 0.3148, + 1.1662 ], "audio_throughput": [ - 30, - 40, - 40 + 31.9068, + 49.6268, + 48.3091 ] } }, @@ -228,13 +228,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 100 + 49.6105 ], "median_audio_rtf": [ - 0.20 + 0.1361 ], "audio_throughput": [ - 10 + 10.0881 ] } }, @@ -259,10 +259,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 100 + 45.5391 ], "median_audio_rtf": [ - 0.15 + 0.1271 ] } }, @@ -291,19 +291,19 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 100, - 1000, - 5000 + 66.1471, + 587.5877, + 4960.3726 ], "median_audio_rtf": [ - 0.25, - 0.40, - 1.20 + 0.1637, + 0.3143, + 1.1703 ], "audio_throughput": [ - 30, - 40, - 40 + 35.1923, + 48.9046, + 48.0505 ] } }, @@ -328,10 +328,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 100 + 49.3466 ], "median_audio_rtf": [ - 0.20 + 0.1349 ], "audio_throughput": [ 9 @@ -353,14 +353,24 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [20], - "max_concurrency": [1], + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "casual_male"}, + "extra_body": { + "voice": "casual_male" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [150], - "median_audio_rtf": [0.15] + "median_audio_ttfp_ms": [ + 150 + ], + "median_audio_rtf": [ + 0.15 + ] } }, { @@ -370,15 +380,27 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [80], - "max_concurrency": [8], + "num_prompts": [ + 80 + ], + "max_concurrency": [ + 8 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "casual_male"}, + "extra_body": { + "voice": "casual_male" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [1500], - "median_audio_rtf": [0.30], - "audio_throughput": [30.0] + "median_audio_ttfp_ms": [ + 1500 + ], + "median_audio_rtf": [ + 0.3 + ], + "audio_throughput": [ + 30.0 + ] } }, { @@ -388,15 +410,27 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [100], - "request_rate": [2.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 2.0 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "casual_male"}, + "extra_body": { + "voice": "casual_male" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [3000], - "median_audio_rtf": [0.25], - "audio_throughput": [4.0] + "median_audio_ttfp_ms": [ + 3000 + ], + "median_audio_rtf": [ + 0.25 + ], + "audio_throughput": [ + 4.0 + ] } } ] diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json index f19f95960c7..0aaca446fc3 100644 --- a/tests/dfx/perf/tests/test_voxcpm2.json +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -23,10 +23,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 600 + 113.9351 ], "median_audio_rtf": [ - 0.2 + 0.0427 ] } }, @@ -47,13 +47,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 3500 + 415.9187 ], "median_audio_rtf": [ - 0.85 + 0.1462 ], "audio_throughput": [ - 8 + 20.0128 ] } }, @@ -100,10 +100,10 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 350 + 66.2748 ], "median_audio_rtf": [ - 0.15 + 0.0366 ] } }, @@ -127,13 +127,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 3000 + 307.6373 ], "median_audio_rtf": [ - 0.5 + 0.1056 ], "audio_throughput": [ - 10 + 64.5428 ] } }, @@ -157,13 +157,13 @@ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "median_audio_ttfp_ms": [ - 500 + 70.0958 ], "median_audio_rtf": [ - 0.3 + 0.0424 ], "audio_throughput": [ - 5 + 16.075 ] } }, diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index d70a28efd80..07873a64b02 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -30,9 +30,9 @@ } ], "baseline": { - "throughput_qps": 0.04, - "latency_mean": 30, - "peak_memory_mb_mean": 75000 + "throughput_qps": 0.0465, + "latency_mean": 21.4354, + "peak_memory_mb_mean": 72557.58 } } ] @@ -72,9 +72,9 @@ } ], "baseline": { - "throughput_qps": 0.06, - "latency_mean": 20, - "peak_memory_mb_mean": 48000 + "throughput_qps": 0.0642, + "latency_mean": 15.5589, + "peak_memory_mb_mean": 45742.88 } }, { @@ -97,9 +97,9 @@ } ], "baseline": { - "throughput_qps": 0.01, - "latency_mean": 90, - "peak_memory_mb_mean": 56000 + "throughput_qps": 0.0128, + "latency_mean": 77.8565, + "peak_memory_mb_mean": 53947.84 } } ]