diff --git a/components/src/dynamo/trtllm/args.py b/components/src/dynamo/trtllm/args.py index e362eec96fdb..8f464d7b1017 100644 --- a/components/src/dynamo/trtllm/args.py +++ b/components/src/dynamo/trtllm/args.py @@ -40,12 +40,27 @@ def _warn_deprecated(message: str) -> None: VALID_TRTLLM_CONNECTORS = {"none", "kvbm"} +def _forward_pass_metrics_enabled(config: DynamoRuntimeConfig) -> bool: + """Resolve the forward-pass metrics opt-in shared with vLLM and SGLang. + + TRT-LLM publishes in-process, so the port in DYN_FORWARDPASS_METRIC_PORT + is unused; setting it still opts in, matching the other backends. + """ + if os.environ.get("DYN_FORWARDPASS_METRIC_PORT"): + return True + return config.fpm_trace + + class Config(DynamoRuntimeConfig, DynamoTrtllmConfig): component: str # Whether this worker publishes KV events. Distinct from the router-side # `use_kv_events` on `router_advertisement`, which means the router # subscribes to them -- the reason the two live on separate objects. use_kv_events: bool + # Whether this worker streams forward-pass metrics to the Planner. The + # cross-backend opt-in turns it on; `--publish-metrics` implies it, since + # that already pays for the iteration statistics both read. + publish_forward_pass_metrics: bool # Routing this worker set advertises in its model card; None inherits the # frontend's configuration. router_advertisement: Optional[WorkerRouterConfig] = None @@ -55,6 +70,9 @@ def validate(self) -> None: DynamoRuntimeConfig.validate(self) DynamoTrtllmConfig.validate(self) self.use_kv_events = self.publish_kv_events + self.publish_forward_pass_metrics = ( + self.publish_metrics or _forward_pass_metrics_enabled(self) + ) # fix the connector as trtllm accepts only one connector and it should be in VALID_TRTLLM_CONNECTORS # while the runtime args accepts a list of connectors diff --git a/components/src/dynamo/trtllm/backend_args.py b/components/src/dynamo/trtllm/backend_args.py index 381b499927a5..c38ed3ec5322 100644 --- a/components/src/dynamo/trtllm/backend_args.py +++ b/components/src/dynamo/trtllm/backend_args.py @@ -183,7 +183,9 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: default=False, help=( "Publish KV cache events to the KV router. This does not enable " - "TensorRT-LLM metric reporting; use --publish-metrics for that." + "TensorRT-LLM iteration statistics: use --publish-metrics for " + "Prometheus metrics, and --fpm-trace or DYN_FORWARDPASS_METRIC_PORT " + "for the Planner's forward-pass metrics." ), dest="publish_kv_events", ) @@ -193,8 +195,11 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: env_var="DYN_TRTLLM_PUBLISH_METRICS", default=False, help=( - "Publish TensorRT-LLM iteration and Prometheus metrics. This does " - "not publish KV cache events; use --publish-kv-events for that." + "Expose TensorRT-LLM iteration and request metrics on this " + "worker's Prometheus endpoint, and publish the Planner's " + "forward-pass metrics, which read the same iteration " + "statistics. This does not publish KV cache events; use " + "--publish-kv-events for that." ), dest="publish_metrics", ) diff --git a/components/src/dynamo/trtllm/publisher.py b/components/src/dynamo/trtllm/publisher.py index 06d660c35001..66d9fa543c2e 100644 --- a/components/src/dynamo/trtllm/publisher.py +++ b/components/src/dynamo/trtllm/publisher.py @@ -494,7 +494,9 @@ class Publisher: Retrieves KV cache events and stats from TensorRT-LLM engine and publishes them: - KV Events: Routes to either ZMQ (if consolidator enabled) or NATS (if no consolidator) - - Metrics: Always publishes to NATS via WorkerMetricsPublisher + - Metrics: Worker-load samples via WorkerMetricsPublisher and, when opted in, + forward-pass metrics via FpmDirectPublisher; both read the engine's + iteration stats, which KV events never need Publisher Selection Logic: - If zmq_endpoint provided: Uses ZmqKvEventPublisher (ZMQ PUB) → Consolidator → NATS @@ -525,6 +527,7 @@ def __init__( kv_state_endpoint: Optional[str] = None, image_token_id: Optional[int] = None, publish_metrics: bool = True, + publish_forward_pass_metrics: bool = False, kv_event_publication_mode: KvEventPublicationMode = KvEventPublicationMode.DISABLED, streaming_kv_events_config: Optional[dict[str, Any]] = None, streaming_kv_events_gpus_per_node: Optional[int] = None, @@ -544,6 +547,7 @@ def __init__( self.kv_state_endpoint = kv_state_endpoint self.image_token_id = image_token_id self.publish_metrics = publish_metrics + self.publish_forward_pass_metrics = publish_forward_pass_metrics self.kv_event_publication_mode = kv_event_publication_mode self.streaming_kv_events_config = streaming_kv_events_config self.streaming_kv_events_gpus_per_node = streaming_kv_events_gpus_per_node @@ -604,7 +608,13 @@ async def _create_metrics_publisher_endpoint(self): await self.metrics_publisher.create_endpoint(self.endpoint) def initialize(self) -> None: - if self.publish_metrics: + # One stats stream feeds the Prometheus gauges, the metrics collector, + # the router's worker-load sample and the Planner's forward-pass + # metrics, so it runs when either opt-in is set. KV events never need + # it: the engine only produces iteration stats under + # enable_iter_perf_stats, at a per-iteration cost a KV-router benchmark + # should not pay. + if self.publish_metrics or self.publish_forward_pass_metrics: self.metrics_publisher = WorkerMetricsPublisher() self._init_publish_metrics_thread() task = asyncio.create_task(self._create_metrics_publisher_endpoint()) @@ -616,7 +626,7 @@ def initialize(self) -> None: # attention-DP rank. Non-attention-DP engines report size 1. Under # attention-DP, TRT-LLM emits one IterationStats row per rank and # Dynamo forwards attentionDpRank as the FPM dp_rank. - if self.publish_metrics: + if self.publish_forward_pass_metrics: try: fpm_dp_size = max(1, int(self.attention_dp_size or 1)) self.fpm_publisher = FpmDirectPublisher( @@ -849,9 +859,9 @@ def _disable_fpm_publisher(self) -> None: self.fpm_publisher = None async def _publish_stats_task(self): - """ - Publish stats to the metrics publisher. - """ + """Poll engine iteration stats into the Prometheus gauges, the metrics + collector, the worker-load publisher and, when opted in, the Planner's + forward-pass publisher.""" if self.engine is None: logging.error("LLM engine not initialized!") return @@ -1327,6 +1337,7 @@ async def get_publisher( kv_state_endpoint: Optional[str] = None, image_token_id: Optional[int] = None, publish_metrics: bool = True, + publish_forward_pass_metrics: bool = False, kv_event_publication_mode: KvEventPublicationMode = KvEventPublicationMode.DISABLED, streaming_kv_events_config: Optional[dict[str, Any]] = None, streaming_kv_events_gpus_per_node: Optional[int] = None, @@ -1346,6 +1357,7 @@ async def get_publisher( kv_state_endpoint=kv_state_endpoint, image_token_id=image_token_id, publish_metrics=publish_metrics, + publish_forward_pass_metrics=publish_forward_pass_metrics, kv_event_publication_mode=kv_event_publication_mode, streaming_kv_events_config=streaming_kv_events_config, streaming_kv_events_gpus_per_node=streaming_kv_events_gpus_per_node, diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_fpm_publisher.py b/components/src/dynamo/trtllm/tests/test_trtllm_fpm_publisher.py index 801a0bca91ee..33ea7f682e80 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_fpm_publisher.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_fpm_publisher.py @@ -295,7 +295,7 @@ def _build_publisher_stub( monkeypatch, *, attention_dp_size: int, - fpm_enabled: bool, + publish_forward_pass_metrics: bool, publish_metrics: bool = True, kv_event_publication_mode: publisher_mod.KvEventPublicationMode = publisher_mod.KvEventPublicationMode.POLLING, ): @@ -320,11 +320,11 @@ def _build_publisher_stub( pub.kv_state_endpoint = None pub.image_token_id = None pub.publish_metrics = publish_metrics + pub.publish_forward_pass_metrics = publish_forward_pass_metrics pub.kv_event_publication_mode = kv_event_publication_mode pub.streaming_kv_events_config = None pub.streaming_kv_events_gpus_per_node = None pub.attention_dp_size = attention_dp_size - pub.fpm_enabled = fpm_enabled pub.processing_initial_created_events = True pub.metrics_publisher = None pub.fpm_publisher = None @@ -372,7 +372,7 @@ def test_streaming_kv_events_create_direct_subscribers_without_engine_polling( pub, module, _ = _build_publisher_stub( monkeypatch, attention_dp_size=2, - fpm_enabled=True, + publish_forward_pass_metrics=True, ) pub.kv_event_publication_mode = publisher_mod.KvEventPublicationMode.STREAMING pub.streaming_kv_events_config = { @@ -419,10 +419,11 @@ def test_streaming_kv_event_endpoint_offset_matches_trtllm_streaming_manager( def test_publisher_initialize_constructs_fpm_direct_publisher_when_fpm_enabled( monkeypatch, ): - """Under non-attention-DP (attention_dp_size == 1, fpm_enabled == True), - Publisher.initialize() constructs FpmDirectPublisher with dp_size=1.""" + """Under non-attention-DP (attention_dp_size == 1), the forward-pass + metrics opt-in makes Publisher.initialize() construct FpmDirectPublisher + with dp_size=1.""" pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub( - monkeypatch, attention_dp_size=1, fpm_enabled=True + monkeypatch, attention_dp_size=1, publish_forward_pass_metrics=True ) pub.initialize() fake_fpm_cls.assert_called_once() @@ -432,35 +433,78 @@ def test_publisher_initialize_constructs_fpm_direct_publisher_when_fpm_enabled( assert pub.fpm_publisher is not None -def test_publisher_initialize_metrics_only_does_not_start_kv_events(monkeypatch): +def test_publisher_initialize_metrics_only_starts_stats_without_fpm(monkeypatch): + """The Publisher's two gates stay independent: the stats thread follows + either opt-in, the Planner publisher only the forward-pass one. Resolving + `--publish-metrics` into both is the config layer's job, covered by + test_trtllm_unit.py.""" pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub( monkeypatch, attention_dp_size=1, - fpm_enabled=True, + publish_forward_pass_metrics=False, publish_metrics=True, kv_event_publication_mode=publisher_mod.KvEventPublicationMode.DISABLED, ) pub.initialize() - fake_fpm_cls.assert_called_once() + fake_fpm_cls.assert_not_called() + assert pub.metrics_publisher is not None + assert pub.fpm_publisher is None pub._init_publish_metrics_thread.assert_called_once() pub._init_publish_kv_cache_events_thread.assert_not_called() -def test_publisher_initialize_kv_events_only_does_not_start_metrics(monkeypatch): +def test_publisher_initialize_kv_events_only_does_not_start_stats(monkeypatch): + """KV events never need iteration stats, so neither stats-fed publisher + nor the stats thread comes up.""" pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub( monkeypatch, attention_dp_size=1, - fpm_enabled=True, + publish_forward_pass_metrics=False, publish_metrics=False, ) pub.initialize() fake_fpm_cls.assert_not_called() + assert pub.metrics_publisher is None pub._init_publish_metrics_thread.assert_not_called() pub._init_publish_kv_cache_events_thread.assert_called_once() +def test_publisher_initialize_fpm_only_starts_stats_and_planner_publisher( + monkeypatch, +): + pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub( + monkeypatch, + attention_dp_size=1, + publish_forward_pass_metrics=True, + publish_metrics=False, + kv_event_publication_mode=publisher_mod.KvEventPublicationMode.DISABLED, + ) + pub.initialize() + + fake_fpm_cls.assert_called_once() + assert pub.metrics_publisher is not None + pub._init_publish_metrics_thread.assert_called_once() + pub._init_publish_kv_cache_events_thread.assert_not_called() + + +def test_publisher_initialize_without_any_opt_in_starts_no_threads(monkeypatch): + pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub( + monkeypatch, + attention_dp_size=1, + publish_forward_pass_metrics=False, + publish_metrics=False, + kv_event_publication_mode=publisher_mod.KvEventPublicationMode.DISABLED, + ) + pub.initialize() + + fake_fpm_cls.assert_not_called() + assert pub.metrics_publisher is None + pub._init_publish_metrics_thread.assert_not_called() + pub._init_publish_kv_cache_events_thread.assert_not_called() + + def _publisher_for_kv_event_test(): pub = publisher_mod.Publisher.__new__(publisher_mod.Publisher) pub.additional_metrics = None @@ -600,7 +644,7 @@ def test_publisher_initializes_fpm_publisher_under_attention_dp(monkeypatch): """Under attention-DP (attention_dp_size > 1), Publisher.initialize() constructs one FpmDirectPublisher channel per attention-DP rank.""" pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub( - monkeypatch, attention_dp_size=4, fpm_enabled=False + monkeypatch, attention_dp_size=4, publish_forward_pass_metrics=True ) pub.initialize() fake_fpm_cls.assert_called_once() diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_unit.py b/components/src/dynamo/trtllm/tests/test_trtllm_unit.py index 24e1fb59fecb..6a8e7ef45314 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_unit.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_unit.py @@ -513,19 +513,52 @@ def test_streaming_kv_events_accept_python_v2_cache_manager(monkeypatch): @pytest.mark.parametrize( - ("publish_flags", "expected_iter_perf_stats"), + ("cli_args", "fpm_port_env", "expected"), [ - (["--publish-kv-events"], False), - (["--publish-metrics"], True), - (["--publish-kv-events", "--publish-metrics"], True), - (["--publish-events-and-metrics"], True), + ([], None, False), + (["--fpm-trace"], None, True), + ([], "20380", True), + (["--publish-kv-events"], None, False), + (["--publish-metrics"], None, True), + (["--publish-kv-events", "--publish-metrics"], None, True), + ], +) +def test_config_publish_forward_pass_metrics_opt_in( + monkeypatch, cli_args, fpm_port_env, expected +): + """The Planner's forward-pass metrics follow the opt-in shared with vLLM + and SGLang, plus --publish-metrics, which already pays for the iteration + statistics both read. --publish-kv-events alone never enables them.""" + monkeypatch.delenv("DYN_FORWARDPASS_METRIC_PORT", raising=False) + # Config.validate writes DYN_FPM_TRACE back; setenv restores it on teardown. + monkeypatch.setenv("DYN_FPM_TRACE", "0") + if fpm_port_env is not None: + monkeypatch.setenv("DYN_FORWARDPASS_METRIC_PORT", fpm_port_env) + + config = parse_args(["--model", "fake-model", *cli_args]) + + assert config.publish_forward_pass_metrics is expected + + +@pytest.mark.parametrize( + ("publish_flags", "fpm_port_env", "expected_iter_perf_stats"), + [ + (["--publish-kv-events"], None, False), + (["--publish-metrics"], None, True), + (["--fpm-trace"], None, True), + ([], "20380", True), + ([], None, False), + (["--publish-kv-events", "--publish-metrics"], None, True), + (["--publish-events-and-metrics"], None, True), ], ) @pytest.mark.asyncio async def test_init_llm_worker_engine_args_without_overrides( - monkeypatch, publish_flags, expected_iter_perf_stats + monkeypatch, publish_flags, fpm_port_env, expected_iter_perf_stats ): - """KV events and metrics control their respective engine paths independently.""" + """Iteration stats follow --publish-metrics and the forward-pass metrics + opt-in, never --publish-kv-events alone; the engine-level + return_perf_metrics default stays off regardless.""" monkeypatch.delenv("DYN_TRTLLM_MAX_NUM_TOKENS", raising=False) monkeypatch.delenv("DYN_TRTLLM_MAX_BATCH_SIZE", raising=False) monkeypatch.delenv("DYN_TRTLLM_EXTRA_ENGINE_ARGS", raising=False) @@ -533,6 +566,10 @@ async def test_init_llm_worker_engine_args_without_overrides( monkeypatch.delenv("DYN_TRTLLM_PUBLISH_KV_EVENTS", raising=False) monkeypatch.delenv("DYN_TRTLLM_PUBLISH_METRICS", raising=False) monkeypatch.delenv("DYN_TRTLLM_PUBLISH_EVENTS_AND_METRICS", raising=False) + monkeypatch.delenv("DYN_FORWARDPASS_METRIC_PORT", raising=False) + monkeypatch.setenv("DYN_FPM_TRACE", "0") + if fpm_port_env is not None: + monkeypatch.setenv("DYN_FORWARDPASS_METRIC_PORT", fpm_port_env) if publish_flags == ["--publish-events-and-metrics"]: with pytest.warns( diff --git a/components/src/dynamo/trtllm/workers/llm_worker.py b/components/src/dynamo/trtllm/workers/llm_worker.py index cdd442275e00..f247cd5dcd1b 100644 --- a/components/src/dynamo/trtllm/workers/llm_worker.py +++ b/components/src/dynamo/trtllm/workers/llm_worker.py @@ -425,9 +425,14 @@ async def init_llm_worker( # and `--override-engine-args` are merged over `arg_map` below and can set # it back to true for custom instrumentation. "return_perf_metrics": False, - # Iteration stats drive the metrics-publishing path but are independent - # of KV-event publication. TensorRT backend always has this enabled. - "enable_iter_perf_stats": config.publish_metrics, + # Iteration stats feed the Prometheus surface and the Planner's + # forward-pass metrics, so either opt-in needs them. KV events do not: + # the engine only produces iteration stats under this flag, at a + # measurable per-iteration throughput cost. TensorRT backend always + # has this enabled. + "enable_iter_perf_stats": ( + config.publish_metrics or config.publish_forward_pass_metrics + ), "kv_connector_config": kv_connector_config, } @@ -989,6 +994,7 @@ async def init_llm_worker( if ( kv_event_publication_mode is not KvEventPublicationMode.DISABLED or config.publish_metrics + or config.publish_forward_pass_metrics ): # Initialize the independently gated KV-event and metrics publishers. # Use model as fallback if served_model_name is not provided @@ -1041,6 +1047,7 @@ async def init_llm_worker( kv_state_endpoint=config.kv_state_endpoint, image_token_id=image_token_id, publish_metrics=config.publish_metrics, + publish_forward_pass_metrics=config.publish_forward_pass_metrics, kv_event_publication_mode=kv_event_publication_mode, streaming_kv_events_config=streaming_kv_events_config, streaming_kv_events_gpus_per_node=gpus_per_node, diff --git a/docs/fern/pages/developer-guide/additional-resources/tensorrt-llm-details/dp-rank-routing.md b/docs/fern/pages/developer-guide/additional-resources/tensorrt-llm-details/dp-rank-routing.md index 72b76f7bad77..472dfb8063aa 100644 --- a/docs/fern/pages/developer-guide/additional-resources/tensorrt-llm-details/dp-rank-routing.md +++ b/docs/fern/pages/developer-guide/additional-resources/tensorrt-llm-details/dp-rank-routing.md @@ -70,10 +70,13 @@ python3 -m dynamo.frontend --router-mode kv The `--enable-attention-dp` flag sets `attention_dp_size = tensor_parallel_size` and configures Dynamo to publish KV events per DP rank. The router automatically creates routing targets for each `(worker_id, dp_rank)` combination. -`--publish-kv-events` enables KV-event generation and publication. Add -`--publish-metrics` when the worker should also publish TensorRT-LLM iteration -and Prometheus metrics. The deprecated `--publish-events-and-metrics` flag -enables both controls for one compatibility release. +`--publish-kv-events` enables KV-event generation and publication and nothing +else. Add `--publish-metrics` when the worker should also expose TensorRT-LLM +metrics on its own Prometheus endpoint and publish the Planner's forward-pass +metrics, or `--fpm-trace` or `DYN_FORWARDPASS_METRIC_PORT` when the Planner +needs those metrics without the Prometheus surface. The deprecated +`--publish-events-and-metrics` flag enables both publishing controls for one +compatibility release. > [!NOTE] > Attention DP requires TRT-LLM's PyTorch backend. AutoDeploy does not support attention DP. diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/observability.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/observability.md index c03fe1a02c58..6a40653a0783 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/observability.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/observability.md @@ -51,10 +51,23 @@ $ DYN_SYSTEM_PORT=8081 python -m dynamo.trtllm --model \ **Note:** The `backend` must be set to `"pytorch"` for metrics collection (enforced in `components/src/dynamo/trtllm/main.py`). TensorRT-LLM's `MetricsCollector` integration has only been tested/validated with the PyTorch backend. -KV-event publication is configured independently with `--publish-kv-events`. -Use both flags when a worker must publish both event and metric telemetry. For -backward compatibility, the deprecated `--publish-events-and-metrics` flag -continues to enable both controls for one release. +`--publish-metrics` exposes metrics on this worker's own endpoint. It also +publishes the per-rank used-block count that the frontend's optional +`--active-decode-blocks-threshold` reads. The KV router's per-worker scoring +(`worker_logit`) never reads that sample — it takes overlap from KV events and +load from its own active-sequence tracking — so enabling `--publish-metrics` +does not change KV-aware ranking. It is not inert, though: with that threshold +set, the published count is OR-ed with the router's own decode-block estimate, +and a worker marked overloaded leaves the candidate set until it recovers. + +Because the Planner's forward-pass metrics read the same iteration statistics, +`--publish-metrics` publishes those too; `--fpm-trace` and +`DYN_FORWARDPASS_METRIC_PORT`, the opt-in the other backends use, enable them +without the Prometheus surface. KV-event publication is configured +independently with `--publish-kv-events`. Use both publishing flags when a +worker must publish both event and metric telemetry. For backward +compatibility, the deprecated `--publish-events-and-metrics` flag continues to +enable both controls for one release. Wait for the TensorRT-LLM worker to start, then send requests and check metrics: @@ -217,7 +230,7 @@ TensorRT-LLM provides extensive performance data beyond the basic Prometheus met - **Prometheus Integration**: Uses the `MetricsCollector` class from `tensorrt_llm.metrics` (see [collector.py](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/metrics/collector.py)) - **Dynamo Integration**: Uses `register_engine_metrics_callback()` function with `metric_prefix_filter=["trtllm_"]` -- **Engine Configuration**: `LlmArgs.return_perf_metrics` remains `False`. It enables TensorRT-LLM's detailed per-step timing collector, whose `time_breakdown_metrics` payload Dynamo never reads and which costs measurable throughput. `--publish-metrics` enables only `enable_iter_perf_stats`, which Dynamo needs for iteration and Prometheus metrics. +- **Engine Configuration**: `LlmArgs.return_perf_metrics` remains `False`. It enables TensorRT-LLM's detailed per-step timing collector, whose `time_breakdown_metrics` payload Dynamo never reads and which costs measurable throughput. `--publish-metrics` and the forward-pass metrics opt-in each enable `enable_iter_perf_stats`: one stats stream feeds the Prometheus gauges, the worker-load publisher and the Planner's forward-pass metrics alike. `--publish-kv-events` alone leaves it off, because KV events do not need iteration statistics and collecting them costs per-iteration throughput. - **Per-request Metrics**: `SamplingParams.return_perf_metrics` is a separate switch that follows `--publish-metrics`. Token usage reporting, including `usage.prompt_tokens_details.cached_tokens`, is unaffected either way: it comes from the engine's own cached-token count, not from `request_perf_metrics`. - **Request Arrival Timestamp**: When engine-level metrics are off, TensorRT-LLM stamps request arrival during C++ request construction rather than at Python submission, so E2E latency, TTFT, and queue time exclude some submission and IPC time. Other request timing fields keep their existing anchors. - **Initialization**: Metrics appear after TensorRT-LLM engine initialization completes diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/overview.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/overview.md index 363925545890..3e05efb1355b 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/overview.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/overview.md @@ -159,7 +159,7 @@ Load-based scaling has the following known limitations. Throughput-based scaling **Requires ForwardPassMetrics (FPM).** Load-based scaling uses per-engine per-iteration metrics delivered via the Dynamo event plane (ForwardPassMetrics). The KV Router is **not** required for load-based scaling. FPM availability by backend: - **vLLM** — supported. Automatically enabled when the engine uses `InstrumentedScheduler` and `DYN_FORWARDPASS_METRIC_PORT` is set. -- **TensorRT-LLM** — supported, including attention-DP when the runtime emits the required iteration statistics. Dynamo publishes one FPM channel per attention-DP rank and disables FPM emission if the runtime schema is missing required fields. +- **TensorRT-LLM** — supported, including attention-DP. Enabled with `--fpm-trace`, `DYN_FORWARDPASS_METRIC_PORT` or `--publish-metrics`, each of which also turns on the engine's `enable_iter_perf_stats`; `--publish-kv-events` does not enable it. Dynamo publishes one FPM channel per attention-DP rank and disables FPM emission if the runtime schema is missing required fields. - **SGLang** — supported. Enabled when `DYN_FORWARDPASS_METRIC_PORT` is set; requires the upstream FPM module, which ships in the SGLang runtime as of `sglang==0.5.13.post1` (SGLang >= v0.5.13). See the [SGLang FPM section](../backends/sglang/observability.md#forward-pass-metrics-fpm). ### General diff --git a/docs/fern/pages/reference/backends/tensorrt-llm-configuration.mdx b/docs/fern/pages/reference/backends/tensorrt-llm-configuration.mdx index c2f0c5de7cd9..7298f21ccb30 100644 --- a/docs/fern/pages/reference/backends/tensorrt-llm-configuration.mdx +++ b/docs/fern/pages/reference/backends/tensorrt-llm-configuration.mdx @@ -155,13 +155,13 @@ Each field is both a CLI flag and an environment variable. The CLI flag takes pr - Publish KV cache events to the Dynamo KV router, enabling KV-aware routing and disaggregated KV transfer. This flag does not enable TensorRT-LLM metric reporting. The deprecated `--publish-events-and-metrics` flag enables both this flag and `--publish-metrics`. + Publish KV cache events to the Dynamo KV router, enabling KV-aware routing and disaggregated KV transfer. This flag does not enable TensorRT-LLM iteration statistics: use `--publish-metrics` for Prometheus metrics, and `--fpm-trace` or `DYN_FORWARDPASS_METRIC_PORT` for the Planner's forward-pass metrics. The deprecated `--publish-events-and-metrics` flag enables both this flag and `--publish-metrics`. Environment variable: `DYN_TRTLLM_PUBLISH_KV_EVENTS` - Publish TensorRT-LLM iteration and Prometheus metrics. This flag does not publish KV cache events. It enables `enable_iter_perf_stats` but keeps the engine's `return_perf_metrics` disabled. + Expose TensorRT-LLM iteration and request metrics on this worker's Prometheus endpoint, and publish the Planner's forward-pass metrics, which read the same iteration statistics. This flag does not publish KV cache events. It enables `enable_iter_perf_stats` and the per-request `SamplingParams.return_perf_metrics`, but keeps the engine's `return_perf_metrics` disabled. The same iteration statistics carry the per-rank used-block count read by the frontend's optional `--active-decode-blocks-threshold`. The KV router's per-worker scoring never reads that sample, so this flag does not change KV-aware ranking; with that threshold set, the count is OR-ed with the router's own decode-block estimate and can mark a worker overloaded, which removes it from the candidate set until it recovers. Environment variable: `DYN_TRTLLM_PUBLISH_METRICS` diff --git a/docs/fern/pages/reference/observability/forward-pass-metrics-traces.mdx b/docs/fern/pages/reference/observability/forward-pass-metrics-traces.mdx index 44ff82f684d7..7d2fdc854040 100644 --- a/docs/fern/pages/reference/observability/forward-pass-metrics-traces.mdx +++ b/docs/fern/pages/reference/observability/forward-pass-metrics-traces.mdx @@ -37,10 +37,12 @@ precedence when set. It cannot add a missing Dynamo relay to an unsupported topo | SGLang (`python -m dynamo.sglang`) | Aggregated, prefill, decode, or LLM diffusion, including `--enable-multimodal` without a dedicated encoder | Supported | | SGLang (`python -m dynamo.sglang`) | Embedding or dedicated multimodal encoder | Not supported; Dynamo warns and does not auto-enable FPM | | SGLang (`python -m dynamo.sglang`) | Image diffusion, video generation, or snapshot mode | Not supported | -| TensorRT-LLM and mocker | Direct-publisher paths | Persistence supported; no Python relay activation is required | +| TensorRT-LLM (`python -m dynamo.trtllm`) | Aggregated, prefill, or decode | Supported; also enables the engine's `enable_iter_perf_stats`. `--publish-metrics` enables FPM too, since it turns the same statistics on | +| Mocker | Direct-publisher path | Persistence supported; forward-pass metrics are always published | For vLLM, an explicit `DYN_FORWARDPASS_METRIC_PORT` wins; otherwise trace activation uses port -`20380`. SGLang uses its existing per-worker IPC endpoint. +`20380`. SGLang uses its existing per-worker IPC endpoint. TensorRT-LLM publishes in-process, so +the port value is unused, but setting the variable still opts the worker in. ## Capture Modes diff --git a/tests/utils/payloads.py b/tests/utils/payloads.py index 4919cd5966cc..363bb1e63835 100644 --- a/tests/utils/payloads.py +++ b/tests/utils/payloads.py @@ -2248,6 +2248,9 @@ class TRTLLMMetricsPayload(MetricsPayload): def _get_backend_specific_checks(self) -> list[MetricCheck]: """TRT-LLM-specific metric checks""" + component_prefix = prometheus_names.name_prefix.COMPONENT + total_blocks = f"{component_prefix}_{prometheus_names.kvstats.TOTAL_BLOCKS}" + checks = [ MetricCheck( # Check: Minimum count of unique trtllm_* metrics @@ -2263,7 +2266,54 @@ def _get_backend_specific_checks(self) -> list[MetricCheck]: f"SUCCESS: Found {len(set(value))} unique trtllm_* metrics (minimum required: 4)" ), multiline=True, - ) + ), + # The checks above and in the base class count metric *names* and + # accept a zero block count. Prometheus registers names when the + # collector is constructed, so both pass on a worker whose stats + # thread never published a sample and whose engine never recorded + # a request. The two checks below require an observation on each + # path, so one going dead cannot pass as the other still working. + MetricCheck( + # Stats path: the per-rank gauges are seeded at 0 by + # _init_publish_metrics_thread and only move when iteration + # stats arrive from the engine. Any rank reporting a positive + # block count proves the polling thread published. + name=f"{total_blocks} (positive on some rank)", + pattern=lambda name: ( + rf"{total_blocks}(?:\{{[^}}]*\}})?\s+([\d.eE+-]+)" + ), + validator=lambda value: any(float(v) > 0 for v in value), + error_msg=lambda name, value: ( + f"{name}: every rank reported a non-positive block count " + f"(values: {value}). The stats polling thread never " + f"published iteration stats." + ), + success_msg=lambda name, value: (f"SUCCESS: {name} (values: {value})"), + multiline=True, + ), + MetricCheck( + # Request path: TRT-LLM's own per-request series, recorded by + # MetricsCollector as requests finish. Several names are + # matched because they come from tensorrt_llm.metrics and one + # upstream rename should not silently void the check. + name="trtllm_* per-request observations", + pattern=lambda name: ( + r"trtllm_(?:request_success_total" + r"|e2e_request_latency_seconds_count" + r"|time_to_first_token_seconds_count" + r"|time_per_output_token_seconds_count" + r"|request_queue_time_seconds_count)" + r"(?:\{[^}]*\})?\s+([\d.eE+-]+)" + ), + validator=lambda value: any(float(v) > 0 for v in value), + error_msg=lambda name, value: ( + f"{name}: TRT-LLM recorded no per-request observations " + f"(values: {value}). The engine's request metrics are not " + f"reaching the collector." + ), + success_msg=lambda name, value: (f"SUCCESS: {name} (values: {value})"), + multiline=True, + ), ] # Check required labels: auto-injected (from prometheus_names.labels) + injected by backend