From d67a7aa8ad786538345d3187cdaf1eee65cdd5f6 Mon Sep 17 00:00:00 2001 From: y_null Date: Sat, 19 Sep 2026 02:15:37 +0800 Subject: [PATCH 1/2] [Perf] Restore MiniCPM-o 4.5 Seed-TTS perf to the chat-completions TTS path #7413 moved the Seed-TTS perf entry from /v1/chat/completions (with the TTS chat template) to /v1/realtime. A model-native duplex session only speaks once it has heard enough audio units, and the benchmark client feeds silence in real time, so every request pays a listening phase before the first audio packet, which the shared metric then counts in TTFT/e2el. Restore the entry to the chat-completions TTS path (revert the config to its pre-#7413 state). Measured on L20X, same harness: 1x32 mean_audio_rtf 1.35 -> 0.31, mean_ttft_ms 2461 -> 122, mean_e2el_ms 5472 -> 1260; 4x64 rtf 1.37 -> 0.62, ttft 2668 -> 181. --- tests/dfx/perf/tests/test_minicpmo_4_5.json | 86 +++++++++++++++++++-- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5.json b/tests/dfx/perf/tests/test_minicpmo_4_5.json index b384d61a0e6..7d7ca08adcd 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5.json @@ -26,8 +26,8 @@ { "dataset_name": "seed-tts", "dataset_path": "zhaochenyang20/seed-tts-eval", - "backend": "openai-realtime-tts", - "endpoint": "/v1/realtime", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", "num_prompts": [ 32, 64, @@ -42,7 +42,73 @@ "disable_shuffle": true, "trust_remote_code": true, "seed_tts_locale": "en", - "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration", + "baseline": { + "H100": { + "request_throughput": [ + 0.9731, + 1.1986, + 1.686 + ], + "mean_ttft_ms": [ + 152.5665, + 270.2559, + 301.455 + ], + "mean_e2el_ms": [ + 1027.2953, + 3306.3735, + 4686.5366 + ], + "mean_audio_ttfp_ms": [ + 458.1553, + 2030.2335, + 2068.6233 + ], + "mean_audio_rtf": [ + 0.2357, + 0.791, + 1.0324 + ] + }, + "A3": { + "request_throughput": [ + 0.5383, + 0.6042, + 0.7547 + ], + "mean_ttft_ms": [ + 333.2633, + 533.8656, + 547.3388 + ], + "mean_e2el_ms": [ + 1857.2154, + 6600.5101, + 10450.6811 + ], + "mean_audio_ttfp_ms": [ + 986.4666, + 3411.0754, + 3352.7515 + ], + "mean_audio_rtf": [ + 0.4423, + 1.5734, + 2.3024 + ] + } + } } ] }, @@ -72,8 +138,8 @@ { "dataset_name": "seed-tts", "dataset_path": "zhaochenyang20/seed-tts-eval", - "backend": "openai-realtime-tts", - "endpoint": "/v1/realtime", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", "num_prompts": [ 32, 64, @@ -88,6 +154,16 @@ "disable_shuffle": true, "trust_remote_code": true, "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" } ] From 5189873dcbf01a6fdb08c721cc3b6e9868b3693a Mon Sep 17 00:00:00 2001 From: y_null Date: Sat, 19 Sep 2026 02:20:51 +0800 Subject: [PATCH 2/2] Keep the Seed-TTS perf baseline unset The pre-#7413 baseline values (H100/A3) do not match what the restored chat-completions path measures on current hardware, so leave the field unset instead of restoring stale numbers. Only the request path and its extra_body are restored. --- tests/dfx/perf/tests/test_minicpmo_4_5.json | 98 +++++---------------- 1 file changed, 21 insertions(+), 77 deletions(-) diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5.json b/tests/dfx/perf/tests/test_minicpmo_4_5.json index 7d7ca08adcd..6ef867b1cd2 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5.json @@ -28,6 +28,16 @@ "dataset_path": "zhaochenyang20/seed-tts-eval", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, "num_prompts": [ 32, 64, @@ -42,73 +52,7 @@ "disable_shuffle": true, "trust_remote_code": true, "seed_tts_locale": "en", - "extra_body": { - "modalities": [ - "text", - "audio" - ], - "chat_template_kwargs": { - "enable_thinking": false, - "use_tts_template": true - } - }, - "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration", - "baseline": { - "H100": { - "request_throughput": [ - 0.9731, - 1.1986, - 1.686 - ], - "mean_ttft_ms": [ - 152.5665, - 270.2559, - 301.455 - ], - "mean_e2el_ms": [ - 1027.2953, - 3306.3735, - 4686.5366 - ], - "mean_audio_ttfp_ms": [ - 458.1553, - 2030.2335, - 2068.6233 - ], - "mean_audio_rtf": [ - 0.2357, - 0.791, - 1.0324 - ] - }, - "A3": { - "request_throughput": [ - 0.5383, - 0.6042, - 0.7547 - ], - "mean_ttft_ms": [ - 333.2633, - 533.8656, - 547.3388 - ], - "mean_e2el_ms": [ - 1857.2154, - 6600.5101, - 10450.6811 - ], - "mean_audio_ttfp_ms": [ - 986.4666, - 3411.0754, - 3352.7515 - ], - "mean_audio_rtf": [ - 0.4423, - 1.5734, - 2.3024 - ] - } - } + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" } ] }, @@ -140,6 +84,16 @@ "dataset_path": "zhaochenyang20/seed-tts-eval", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, "num_prompts": [ 32, 64, @@ -154,16 +108,6 @@ "disable_shuffle": true, "trust_remote_code": true, "seed_tts_locale": "en", - "extra_body": { - "modalities": [ - "text", - "audio" - ], - "chat_template_kwargs": { - "enable_thinking": false, - "use_tts_template": true - } - }, "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" } ]