[Frontend][Benchmark] Add duplex performance metrics for OmniInteract and Omni-DuplexEval benchmark

docs/cli/bench/serve.md

@@ -316,8 +316,12 @@ arrives, checkpointing the response's history position so a later committed user
 outputs are ineligible and omitted from the official manifest; `audio_clipped_bytes` records output beyond the rounded video
 horizon.
 
-TTFT, TTFP, and RTF start at client receipt of `response.created`. TPOT/ITL use engine stage-0 timing; ITL is emitted only when
-every token interval is present.
+Per-response TTFT and TTFP start when the server begins executing the native model-turn request that owns the response. RTF
+continues to use client receipt of `response.created` through the last audio packet, divided by emitted audio duration. Global
+TTFT, TTFP, and RTF cover the complete input-stream window. TPOT/ITL use engine stage-0 timing; ITL is emitted only when every
+token interval is present within a continuous generation segment. Model-unit pacing and gaps between generation segments are
+excluded from TPOT/ITL. Raw request metrics retain `response_created_to_first_text_ms` and
+`response_created_to_first_audio_ms` as client-envelope diagnostics.
 
 The checked-in local performance configuration measures four deterministic cases from each OmniInteract subset (12 videos
 total), with no benchmark warmups and a maximum concurrency of two. Each subset also sends one readiness request before its

tests/benchmarks/metrics/test_metrics.py

@@ -150,16 +150,16 @@ def test_unmeasured_duplex_latency_does_not_add_zero_samples():
     measured.audio_ttfp = 0.2
     measured.audio_rtf = 0.5
     measured.duplex_session_metrics = {
-        "mean_ttft_ms": 100.0,
-        "mean_ttfp_ms": 200.0,
-        "mean_rtf": 0.5,
+        "ttft_ms": 100.0,
+        "ttfp_ms": 200.0,
+        "rtf": 0.5,
     }
     listen_only = _make_output(100, output_tokens=0)
     listen_only.ttft = listen_only.audio_ttfp = listen_only.audio_rtf = 0.0
     listen_only.duplex_session_metrics = {
-        "mean_ttft_ms": None,
-        "mean_ttfp_ms": None,
-        "mean_rtf": None,
+        "ttft_ms": None,
+        "ttfp_ms": None,
+        "rtf": None,
     }
 
     metrics = _calculate_test_metrics([measured, listen_only], {"ttft": 150.0})
@@ -174,9 +174,9 @@ def test_unmeasured_duplex_latency_does_not_add_zero_samples():
 def test_all_unmeasured_duplex_latency_is_not_reported_as_zero():
     listen_only = _make_output(100, output_tokens=0)
     listen_only.duplex_session_metrics = {
-        "mean_ttft_ms": None,
-        "mean_ttfp_ms": None,
-        "mean_rtf": None,
+        "ttft_ms": None,
+        "ttfp_ms": None,
+        "rtf": None,
     }
 
     metrics = _calculate_test_metrics(
@@ -196,13 +196,13 @@ def test_unmeasured_duplex_tpot_does_not_add_zero_or_misalign_goodput():
     missing_tpot.itl = []
     missing_tpot.text_latency = missing_tpot.ttft = 0.1
     missing_tpot.tpot_measured = False
-    missing_tpot.duplex_session_metrics = {"mean_ttft_ms": 100.0}
+    missing_tpot.duplex_session_metrics = {"ttft_ms": 100.0}
 
     slow_ttft = _make_output(100, output_tokens=5)
     slow_ttft.ttft = 1.0
     slow_ttft.text_latency = 1.4
     slow_ttft.itl = [0.1] * 4
-    slow_ttft.duplex_session_metrics = {"mean_ttft_ms": 1000.0}
+    slow_ttft.duplex_session_metrics = {"ttft_ms": 1000.0}
 
     metrics = _calculate_test_metrics(
         [missing_tpot, slow_ttft],
@@ -219,7 +219,7 @@ def test_all_unmeasured_duplex_token_timing_is_not_reported_as_zero():
     output.itl = []
     output.text_latency = output.ttft
     output.tpot_measured = False
-    output.duplex_session_metrics = {"mean_ttft_ms": 100.0}
+    output.duplex_session_metrics = {"ttft_ms": 100.0}
 
     metrics = _calculate_test_metrics([output], {"tpot": float("inf")})
 
@@ -236,7 +236,7 @@ def test_duplex_response_timings_do_not_build_a_session_token_timeline():
         {"response_id": "r1", "stage0_tokens": {"itls_ms": [100.0, 100.0]}},
         {"response_id": "r2", "stage0_tokens": {"itls_ms": [100.0, 100.0]}},
     ]
-    output.duplex_session_metrics = {"mean_ttft_ms": 100.0}
+    output.duplex_session_metrics = {"ttft_ms": 100.0}
 
     metrics = _calculate_test_metrics([output])
 
@@ -252,7 +252,7 @@ def test_unmeasured_tpot_stays_missing_after_tokenizer_fallback():
     output.itl = []
     output.text_latency = output.ttft
     output.tpot_measured = False
-    output.duplex_session_metrics = {"mean_ttft_ms": 100.0}
+    output.duplex_session_metrics = {"ttft_ms": 100.0}
 
     def tokenizer(text, *, add_special_tokens):
         assert text == output.generated_text
@@ -282,7 +282,7 @@ def tokenizer(text, *, add_special_tokens):
 def test_zero_itl_does_not_create_zero_tpot():
     output = _make_output(100, output_tokens=3)
     output.itl = [0.0, 0.0]
-    output.duplex_session_metrics = {"mean_ttft_ms": 100.0}
+    output.duplex_session_metrics = {"ttft_ms": 100.0}
 
     metrics = _calculate_test_metrics([output], {"tpot": 1.0})
 
@@ -357,9 +357,9 @@ def test_single_token_responses_do_not_report_zero_tpot(capsys):
 def test_duplex_goodput_does_not_pair_measurements_from_different_requests():
     text_only, audio_only = _make_output(100), _make_output(100)
     text_only.ttft, text_only.audio_ttfp = 0.1, 0.0
-    text_only.duplex_session_metrics = {"mean_ttft_ms": 100.0, "mean_ttfp_ms": None, "mean_rtf": None}
+    text_only.duplex_session_metrics = {"ttft_ms": 100.0, "ttfp_ms": None, "rtf": None}
     audio_only.ttft, audio_only.audio_ttfp = 0.0, 0.2
-    audio_only.duplex_session_metrics = {"mean_ttft_ms": None, "mean_ttfp_ms": 200.0, "mean_rtf": None}
+    audio_only.duplex_session_metrics = {"ttft_ms": None, "ttfp_ms": 200.0, "rtf": None}
 
     metrics = _calculate_test_metrics([text_only, audio_only], {"ttft": 500.0, "audio_ttft": 500.0})
 

tests/benchmarks/patch/test_patch.py

@@ -287,20 +287,25 @@ async def close_session(self, **_kwargs):
                 "ttft": "conversation.item.create client send to first non-empty text delta",
                 "ttfp": "conversation.item.create client send to first audio packet",
                 "rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
+                "tpot": "Stage-0 engine mean time per output token",
             },
             "ttft_ms": pytest.approx(20.0, abs=2.0),
+            "tpot_ms": 10.0,
             "ttfp_ms": pytest.approx(30.0, abs=2.0),
             "rtf": pytest.approx(0.3, abs=0.03),
             "audio_generation_ms": pytest.approx(30.0, abs=2.0),
             "audio_duration_ms": 100.0,
         }
-    assert output.duplex_session_metrics == {
-        "session_id": session_id,
-        "audio_turn_count": 4,
-        "mean_ttft_ms": pytest.approx(20.0, abs=2.0),
-        "mean_ttfp_ms": pytest.approx(30.0, abs=2.0),
-        "mean_rtf": pytest.approx(0.3, abs=0.03),
-    }
+    session = output.duplex_session_metrics
+    assert session["session_id"] == session_id
+    assert session["audio_turn_count"] == 4
+    assert session["ttft_ms"]["count"] == 4
+    assert session["ttft_ms"]["mean"] == pytest.approx(20.0, abs=2.0)
+    assert session["tpot_ms"] == {"count": 4, "mean": 10.0, "p50": 10.0, "p99": 10.0}
+    assert session["ttfp_ms"]["count"] == 4
+    assert session["ttfp_ms"]["mean"] == pytest.approx(30.0, abs=2.0)
+    assert session["rtf"]["count"] == 4
+    assert session["rtf"]["mean"] == pytest.approx(0.3, abs=0.03)
 
 
 def test_stage0_token_timings_use_weighted_tpot_when_itls_are_incomplete():

tests/benchmarks/test_omniinteract.py

@@ -148,6 +148,72 @@ def _successful_output(
     return case, collector, result
 
 
+def test_response_metrics_include_engine_tpot_and_stream_window():
+    first_audio = _audio("r1")
+    first_audio["metadata"] = {
+        "audio_duration_ms": 100,
+        "vllm_omni": {
+            "stage_metrics": {
+                "0": {
+                    "num_tokens_out": 3,
+                    "vllm_tpot_ms": 10.0,
+                    "vllm_itls_ms": [10.0, 10.0],
+                }
+            }
+        },
+    }
+    second_audio = _audio("r2")
+    second_audio["metadata"] = {
+        "audio_duration_ms": 200,
+        "vllm_omni": {
+            "stage_metrics": {
+                "0": {
+                    "num_tokens_out": 2,
+                    "vllm_tpot_ms": 20.0,
+                    "vllm_itls_ms": [20.0],
+                }
+            }
+        },
+    }
+    collector = _collector(
+        (_created("r1"), 10.1),
+        (_text("r1", "first"), 10.2),
+        (first_audio, 10.3),
+        (_done("r1"), 10.4),
+        (_created("r2"), 12.1),
+        (_text("r2", "second"), 12.2),
+        (second_audio, 12.5),
+        (_done("r2"), 12.6),
+    )
+    result = oi.OmniInteractCaseResult("1q1a", "video.mp4", "", session_id="session")
+
+    oi._populate_response_metrics(result, collector, stream_start=10.0)
+
+    assert result.output_tokens == 5
+    assert [metric["tpot_ms"] for metric in result.duplex_request_metrics] == [10.0, 20.0]
+    assert result.duplex_session_metrics == {
+        "session_id": "session",
+        "audio_turn_count": 2,
+        "ttft_ms": {"count": 2, "mean": 100.0, "p50": 100.0, "p99": 100.0},
+        "tpot_ms": {"count": 2, "mean": 15.0, "p50": 10.0, "p99": 20.0},
+        "ttfp_ms": {"count": 2, "mean": 300.0, "p50": 200.0, "p99": 400.0},
+        "rtf": {"count": 2, "mean": 2.0, "p50": 2.0, "p99": 2.0},
+        "stream_ttft_ms": 200.0,
+        "stream_ttfp_ms": 300.0,
+        "stream_rtf": 8.333333,
+        "stream_audio_generation_ms": 2500.0,
+        "stream_audio_duration_ms": 300.0,
+        "stream_measurement_origin": {
+            "ttft": "input stream start to first non-empty text delta",
+            "ttfp": "input stream start to first audio packet",
+            "rtf": (
+                "input stream start-to-last-audio receive time divided by total emitted audio duration; "
+                "includes concurrent realtime input"
+            ),
+        },
+    }
+
+
 def _write_success(
     root: Path,
     case: data.OmniInteractCase,
@@ -645,7 +711,7 @@ async def run(_case_arg, config, **kwargs):
             transcript="timing metadata missing",
             output_tokens=0,
             duplex_request_metrics=[{"request_metrics": {"ttft_ms": 1.0}}],
-            duplex_session_metrics={"mean_ttft_ms": 10.0},
+            duplex_session_metrics={"ttft_ms": 10.0},
         )
 
     monkeypatch.setattr(benchmark_patch, "run_omniinteract_case", run)
@@ -713,7 +779,7 @@ async def run(*args, **kwargs):
             success=True,
             output_tokens=5,
             duplex_request_metrics=request_metrics,
-            duplex_session_metrics={"mean_ttft_ms": 100.0},
+            duplex_session_metrics={"ttft_ms": 100.0},
         )
 
     monkeypatch.setattr(benchmark_patch, "run_omniinteract_case", run)