[Bugfix][Doc] fix Prometheus per-stage audio metrics and drop RTF

docs/design/metrics.md

@@ -70,9 +70,9 @@ There are four independent paths for metric collection.
 
 **Path 2: Audio modality metrics (`vllm:omni_audio_*`)**
 
-`OmniModalityMetrics` registers seven audio families with `{model_name, stage, replica}` (plus an extra `threshold_ms` / `reason` label on the two extra-cardinality Counters). Three observation sites:
+`OmniModalityMetrics` registers six audio families with `{model_name, stage, replica}` (plus an extra `threshold_ms` / `reason` label on the two extra-cardinality Counters). Three observation sites:
 
-- `observe_modality_at_finalize(...)` — called from `omni_base._process_single_result` inside the existing `e2e_done` finalize guard. For `output_type == "audio"` it emits `audio_frames_total`, `audio_duration_s`, `audio_rtf` (or `audio_skipped_requests_total{reason="no_audio_data"}` when no audio was produced). Sample rate is resolved from `engine_outputs.multimodal_output` via `definitions.resolve_audio_sample_rate(...)` (fallback chain mirrors `serving_chat.py`'s audio response path).
+- `observe_modality_at_finalize(...)` — called from `omni_base._process_single_result` inside the existing `e2e_done` finalize guard. For `output_type == "audio"` it emits `audio_frames_total` and `audio_duration_s` (or `audio_skipped_requests_total{reason="no_audio_data"}` when no audio was produced). Sample rate is resolved from `engine_outputs.multimodal_output` via `definitions.resolve_audio_sample_rate(...)` (fallback chain mirrors `serving_chat.py`'s audio response path).
 - `observe_audio_first_packet(...)` — called from the OpenAI SSE audio branch in `serving_chat.py` on the first audio packet for a request. The once-per-request guard is held by `ClientRequestState.first_audio_ts`. The `request_arrival_ts` anchor is stored in `ClientRequestState` by `async_omni.generate()`, computed at request entry.
 - `observe_audio_streaming_finalize(...)` — called from `serving_chat.py` after the streaming chunk loop exhausts. It runs the per-chunk player simulation from `vllm_omni/benchmarks/audio_continuity.py` to compute the worst-case underrun and emits `audio_underrun_s` plus (when the request stayed below the threshold) `audio_continuity_ok_total{threshold_ms}`. Per-chunk PCM byte counts and arrival timestamps are recorded by the same audio branch that updates `first_audio_ts`.
 
@@ -167,15 +167,14 @@ second.
 | `vllm:omni_requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason ({stop, length, abort, ...}); aborts cover client-disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
 | `vllm:omni_e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end latency in seconds |
 
-### Audio (7)
+### Audio (6)
 
 Labels: `{model_name, stage, replica}` plus the listed extra label.
 
 | Metric | Type | Extra label | Description |
 |--------|------|-------------|-------------|
 | `vllm:omni_audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
 | `vllm:omni_audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
-| `vllm:omni_audio_rtf` | Histogram | — | Real-time factor `stage_gen_time_s / audio_duration_s` (SLO `< 1`); uses `RTF_BUCKETS` |
 | `vllm:omni_audio_frames_total` | Counter | — | Cumulative audio frames generated |
 | `vllm:omni_audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
 | `vllm:omni_audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |

docs/usage/metrics.md

@@ -43,7 +43,6 @@ Emitted at request finalize, except for `audio_ttfp_s` (streaming-hook at the fi
 |--------|------|-------------|-------------|
 | `vllm:omni_audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
 | `vllm:omni_audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
-| `vllm:omni_audio_rtf` | Histogram | — | Real-time factor (`stage_gen_time_s / audio_duration_s`); streaming TTS SLO red line `< 1`; uses `RTF_BUCKETS` |
 | `vllm:omni_audio_frames_total` | Counter | — | Cumulative audio frame count; throughput via `rate()` |
 | `vllm:omni_audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
 | `vllm:omni_audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |

tests/metrics/test_modality.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import pytest
+import torch
 from prometheus_client import REGISTRY, generate_latest
 
+from vllm_omni.engine.mm_outputs import MultimodalPayload
 from vllm_omni.metrics import definitions as defs
 from vllm_omni.metrics.modality import (
     OmniModalityMetrics,
@@ -37,7 +39,6 @@ def _sample_value(output: str, line_prefix: str) -> float | None:
 _EXPECTED_FAMILIES = [
     defs.AUDIO_TTFP_S,
     defs.AUDIO_DURATION_S,
-    defs.AUDIO_RTF_METRIC,
     defs.AUDIO_FRAMES_METRIC,
     defs.AUDIO_UNDERRUN_S,
     defs.AUDIO_CONTINUITY_OK_METRIC,
@@ -50,7 +51,6 @@ def test_all_locked_families_present(self, mod: OmniModalityMetrics) -> None:
         # Trigger at least one observation per family so the registry exposes them.
         mod.observe_audio_ttfp("s", "r", 0.1)
         mod.observe_audio_duration("s", "r", 1.0)
-        mod.observe_audio_rtf("s", "r", 0.5)
         mod.inc_audio_frames("s", "r", 1)
         mod.observe_audio_underrun("s", "r", 0.01)
         mod.inc_audio_continuity_ok("s", "r", 100)
@@ -81,13 +81,6 @@ def test_audio_duration_observed(self, mod: OmniModalityMetrics) -> None:
         prefix = f'{defs.AUDIO_DURATION_S}_sum{{model_name="{_MODEL}",replica="{replica}",stage="{stage}"}}'
         assert _sample_value(out, prefix) == 3.5
 
-    def test_audio_rtf_observed(self, mod: OmniModalityMetrics) -> None:
-        stage, replica = "talker_rtf", "0"
-        mod.observe_audio_rtf(stage, replica, 0.45)
-        out = generate_latest(REGISTRY).decode()
-        prefix = f'{defs.AUDIO_RTF_METRIC}_sum{{model_name="{_MODEL}",replica="{replica}",stage="{stage}"}}'
-        assert _sample_value(out, prefix) == 0.45
-
     def test_audio_frames_inc(self, mod: OmniModalityMetrics) -> None:
         stage, replica = "talker_frames", "0"
         mod.inc_audio_frames(stage, replica, 240)
@@ -171,9 +164,6 @@ def inc_audio_frames(self, s, r, n):
     def observe_audio_duration(self, s, r, d):
         self.calls.append(("observe_audio_duration", s, r, d))
 
-    def observe_audio_rtf(self, s, r, rtf):
-        self.calls.append(("observe_audio_rtf", s, r, rtf))
-
     def observe_audio_underrun(self, s, r, u):
         self.calls.append(("observe_audio_underrun", s, r, u))
 
@@ -192,8 +182,9 @@ def __init__(self, **kw):
 class TestObserveModalityAtFinalize:
     def test_audio_path_full(self):
         stub = _StubModMetrics()
-        stage_metrics = _Bag(stage_gen_time_ms=500.0, audio_generated_frames=24000)
-        engine_outputs = _Bag(multimodal_output={"audio_sample_rate": 24000})
+        stage_metrics = _Bag(stage_gen_time_ms=500.0, audio_generated_frames=0)
+        payload = MultimodalPayload(tensors={"audio": torch.zeros(24000)})
+        engine_outputs = _Bag(multimodal_output=payload)
 
         observe_modality_at_finalize(
             stub,
@@ -203,15 +194,33 @@ def test_audio_path_full(self):
             stage_metrics=stage_metrics,
             engine_outputs=engine_outputs,
         )
-        # 24000 frames / 24000 Hz = 1.0s duration; gen 0.5s → rtf 0.5
+        # 24000 frames / 24000 Hz = 1.0s duration
         assert ("inc_audio_frames", "1", "0", 24000) in stub.calls
         assert ("observe_audio_duration", "1", "0", 1.0) in stub.calls
-        assert ("observe_audio_rtf", "1", "0", 0.5) in stub.calls
         # Continuity/underrun NOT emitted from finalize — they come from the
         # streaming hook because they need the per-chunk arrival timeline.
         assert not any(c[0] == "observe_audio_underrun" for c in stub.calls)
         assert not any(c[0] == "inc_audio_continuity_ok" for c in stub.calls)
 
+    def test_audio_path_nested_multimodal_payload(self):
+        stub = _StubModMetrics()
+        payload = MultimodalPayload(tensors={"audio": torch.zeros(24000)})
+        engine_outputs = _Bag(
+            multimodal_output=MultimodalPayload(),
+            outputs=[_Bag(multimodal_output=payload)],
+        )
+
+        observe_modality_at_finalize(
+            stub,
+            output_type="audio",
+            stage_id=1,
+            replica_id=0,
+            stage_metrics=_Bag(stage_gen_time_ms=500.0, audio_generated_frames=0),
+            engine_outputs=engine_outputs,
+        )
+        assert ("inc_audio_frames", "1", "0", 24000) in stub.calls
+        assert ("observe_audio_duration", "1", "0", 1.0) in stub.calls
+
     def test_audio_path_zero_frames_emits_skipped(self):
         stub = _StubModMetrics()
         observe_modality_at_finalize(
@@ -220,11 +229,10 @@ def test_audio_path_zero_frames_emits_skipped(self):
             stage_id=1,
             replica_id=0,
             stage_metrics=_Bag(stage_gen_time_ms=300.0, audio_generated_frames=0),
-            engine_outputs=_Bag(multimodal_output={}),
+            engine_outputs=_Bag(multimodal_output=MultimodalPayload()),
         )
         assert ("inc_audio_frames", "1", "0", 0) in stub.calls
         assert not any(c[0] == "observe_audio_duration" for c in stub.calls)
-        assert not any(c[0] == "observe_audio_rtf" for c in stub.calls)
         assert ("inc_audio_skipped", "1", "0", "no_audio_data") in stub.calls
 
     def test_audio_uses_resolved_sample_rate_from_multimodal_output(self):
@@ -376,13 +384,6 @@ def test_replica_none_skipped(self):
 
 
 class TestBucketSelection:
-    def test_audio_rtf_uses_rtf_buckets(self, mod: OmniModalityMetrics) -> None:
-        stage, replica = "talker_buckets", "0"
-        mod.observe_audio_rtf(stage, replica, 0.5)
-        out = generate_latest(REGISTRY).decode()
-        rtf_marker = f'{defs.AUDIO_RTF_METRIC}_bucket{{le="0.9"'
-        assert rtf_marker in out, "audio_rtf should use RTF_BUCKETS containing le=0.9"
-
     def test_audio_ttfp_uses_seconds_buckets(self, mod: OmniModalityMetrics) -> None:
         stage, replica = "talker_seconds", "0"
         mod.observe_audio_ttfp(stage, replica, 0.1)

vllm_omni/metrics/definitions.py

@@ -127,7 +127,6 @@
 # ============================================================================
 AUDIO_TTFP_S = METRIC_PREFIX + AUDIO_TTFP + "_s"
 AUDIO_DURATION_S = METRIC_PREFIX + AUDIO_DURATION + "_s"
-AUDIO_RTF_METRIC = METRIC_PREFIX + AUDIO_RTF
 AUDIO_FRAMES_METRIC = METRIC_PREFIX + AUDIO_FRAMES
 AUDIO_UNDERRUN_S = METRIC_PREFIX + AUDIO_UNDERRUN + "_s"
 AUDIO_CONTINUITY_OK_METRIC = METRIC_PREFIX + AUDIO_CONTINUITY_OK
@@ -214,21 +213,6 @@
     60.0,
 )
 
-# RTF SLO red line is 1.0 — TTS must generate faster than playback.
-RTF_BUCKETS = (
-    0.1,
-    0.25,
-    0.5,
-    0.75,
-    0.9,
-    1.0,
-    1.25,
-    1.5,
-    2.0,
-    5.0,
-    10.0,
-)
-
 # Bytes bucket for transfer payload size.
 BYTES_BUCKETS = (
     1024,

vllm_omni/metrics/modality.py

@@ -47,12 +47,6 @@
     labelnames=_stage_labels,
     buckets=defs.SECONDS_BUCKETS,
 )
-_audio_rtf_family = Histogram(
-    defs.AUDIO_RTF_METRIC,
-    "Audio real-time factor (stage_gen_time_s / audio_duration_s); streaming TTS requires < 1.",
-    labelnames=_stage_labels,
-    buckets=defs.RTF_BUCKETS,
-)
 _audio_frames_family = Counter(
     defs.AUDIO_FRAMES_METRIC,
     "Cumulative audio frame count; per-model rate (not summable across models). Throughput recovered via rate().",
@@ -102,11 +96,6 @@ def observe_audio_duration(self, stage: str, replica: str, duration_seconds: flo
             duration_seconds
         )
 
-    def observe_audio_rtf(self, stage: str, replica: str, rtf: float) -> None:
-        if not self._log_stats:
-            return
-        _audio_rtf_family.labels(model_name=self._model_name, stage=stage, replica=replica).observe(rtf)
-
     def inc_audio_frames(self, stage: str, replica: str, n_frames: int) -> None:
         if not self._log_stats or n_frames <= 0:
             return
@@ -166,7 +155,6 @@ def observe_modality_at_finalize(
 
     stage_label = str(stage_id)
     replica_label = str(replica_id)
-    gen_time_s = float(getattr(stage_metrics, "stage_gen_time_ms", 0.0)) / 1000.0
     mm_out = extract_mm_output(engine_outputs)
 
     sample_rate = defs.resolve_audio_sample_rate(mm_out)
@@ -181,11 +169,6 @@ def observe_modality_at_finalize(
     duration_s = n_frames / sample_rate if sample_rate > 0 else 0.0
     if duration_s > 0:
         mod_metrics.observe_audio_duration(stage_label, replica_label, duration_s)
-        mod_metrics.observe_audio_rtf(
-            stage_label,
-            replica_label,
-            defs.compute_audio_rtf(gen_time_s, duration_s),
-        )
     else:
         # Request completed (finish_reason ∈ {stop, length} — error paths
         # don't reach finalize) but no audio samples were produced. Covers