Skip to content
Merged
18 changes: 18 additions & 0 deletions components/src/dynamo/trtllm/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,27 @@ def _warn_deprecated(message: str) -> None:
VALID_TRTLLM_CONNECTORS = {"none", "kvbm"}


def _forward_pass_metrics_enabled(config: DynamoRuntimeConfig) -> bool:
"""Resolve the forward-pass metrics opt-in shared with vLLM and SGLang.

TRT-LLM publishes in-process, so the port in DYN_FORWARDPASS_METRIC_PORT
is unused; setting it still opts in, matching the other backends.
"""
if os.environ.get("DYN_FORWARDPASS_METRIC_PORT"):
return True
return config.fpm_trace


class Config(DynamoRuntimeConfig, DynamoTrtllmConfig):
component: str
# Whether this worker publishes KV events. Distinct from the router-side
# `use_kv_events` on `router_advertisement`, which means the router
# subscribes to them -- the reason the two live on separate objects.
use_kv_events: bool
# Whether this worker streams forward-pass metrics to the Planner. The
# cross-backend opt-in turns it on; `--publish-metrics` implies it, since
# that already pays for the iteration statistics both read.
publish_forward_pass_metrics: bool
# Routing this worker set advertises in its model card; None inherits the
# frontend's configuration.
router_advertisement: Optional[WorkerRouterConfig] = None
Expand All @@ -55,6 +70,9 @@ def validate(self) -> None:
DynamoRuntimeConfig.validate(self)
DynamoTrtllmConfig.validate(self)
self.use_kv_events = self.publish_kv_events
self.publish_forward_pass_metrics = (
self.publish_metrics or _forward_pass_metrics_enabled(self)
)

# fix the connector as trtllm accepts only one connector and it should be in VALID_TRTLLM_CONNECTORS
# while the runtime args accepts a list of connectors
Expand Down
11 changes: 8 additions & 3 deletions components/src/dynamo/trtllm/backend_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,9 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
default=False,
help=(
"Publish KV cache events to the KV router. This does not enable "
"TensorRT-LLM metric reporting; use --publish-metrics for that."
"TensorRT-LLM iteration statistics: use --publish-metrics for "
"Prometheus metrics, and --fpm-trace or DYN_FORWARDPASS_METRIC_PORT "
"for the Planner's forward-pass metrics."
),
dest="publish_kv_events",
)
Expand All @@ -193,8 +195,11 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
env_var="DYN_TRTLLM_PUBLISH_METRICS",
default=False,
help=(
"Publish TensorRT-LLM iteration and Prometheus metrics. This does "
"not publish KV cache events; use --publish-kv-events for that."
"Expose TensorRT-LLM iteration and request metrics on this "
"worker's Prometheus endpoint, and publish the Planner's "
"forward-pass metrics, which read the same iteration "
"statistics. This does not publish KV cache events; use "
"--publish-kv-events for that."
),
dest="publish_metrics",
)
Expand Down
24 changes: 18 additions & 6 deletions components/src/dynamo/trtllm/publisher.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,9 @@ class Publisher:

Retrieves KV cache events and stats from TensorRT-LLM engine and publishes them:
- KV Events: Routes to either ZMQ (if consolidator enabled) or NATS (if no consolidator)
- Metrics: Always publishes to NATS via WorkerMetricsPublisher
- Metrics: Worker-load samples via WorkerMetricsPublisher and, when opted in,
forward-pass metrics via FpmDirectPublisher; both read the engine's
iteration stats, which KV events never need

Publisher Selection Logic:
- If zmq_endpoint provided: Uses ZmqKvEventPublisher (ZMQ PUB) → Consolidator → NATS
Expand Down Expand Up @@ -525,6 +527,7 @@ def __init__(
kv_state_endpoint: Optional[str] = None,
image_token_id: Optional[int] = None,
publish_metrics: bool = True,
publish_forward_pass_metrics: bool = False,
kv_event_publication_mode: KvEventPublicationMode = KvEventPublicationMode.DISABLED,
streaming_kv_events_config: Optional[dict[str, Any]] = None,
streaming_kv_events_gpus_per_node: Optional[int] = None,
Expand All @@ -544,6 +547,7 @@ def __init__(
self.kv_state_endpoint = kv_state_endpoint
self.image_token_id = image_token_id
self.publish_metrics = publish_metrics
self.publish_forward_pass_metrics = publish_forward_pass_metrics
Comment thread
jorgeantonio21 marked this conversation as resolved.
Resolved
self.kv_event_publication_mode = kv_event_publication_mode
self.streaming_kv_events_config = streaming_kv_events_config
self.streaming_kv_events_gpus_per_node = streaming_kv_events_gpus_per_node
Expand Down Expand Up @@ -604,7 +608,13 @@ async def _create_metrics_publisher_endpoint(self):
await self.metrics_publisher.create_endpoint(self.endpoint)

def initialize(self) -> None:
if self.publish_metrics:
# One stats stream feeds the Prometheus gauges, the metrics collector,
# the router's worker-load sample and the Planner's forward-pass
# metrics, so it runs when either opt-in is set. KV events never need
# it: the engine only produces iteration stats under
# enable_iter_perf_stats, at a per-iteration cost a KV-router benchmark
# should not pay.
if self.publish_metrics or self.publish_forward_pass_metrics:
self.metrics_publisher = WorkerMetricsPublisher()
self._init_publish_metrics_thread()
task = asyncio.create_task(self._create_metrics_publisher_endpoint())
Expand All @@ -616,7 +626,7 @@ def initialize(self) -> None:
# attention-DP rank. Non-attention-DP engines report size 1. Under
# attention-DP, TRT-LLM emits one IterationStats row per rank and
# Dynamo forwards attentionDpRank as the FPM dp_rank.
if self.publish_metrics:
if self.publish_forward_pass_metrics:
Comment thread
jorgeantonio21 marked this conversation as resolved.
Resolved
try:
fpm_dp_size = max(1, int(self.attention_dp_size or 1))
self.fpm_publisher = FpmDirectPublisher(
Expand Down Expand Up @@ -849,9 +859,9 @@ def _disable_fpm_publisher(self) -> None:
self.fpm_publisher = None

async def _publish_stats_task(self):
"""
Publish stats to the metrics publisher.
"""
"""Poll engine iteration stats into the Prometheus gauges, the metrics
collector, the worker-load publisher and, when opted in, the Planner's
forward-pass publisher."""
if self.engine is None:
logging.error("LLM engine not initialized!")
return
Expand Down Expand Up @@ -1327,6 +1337,7 @@ async def get_publisher(
kv_state_endpoint: Optional[str] = None,
image_token_id: Optional[int] = None,
publish_metrics: bool = True,
publish_forward_pass_metrics: bool = False,
kv_event_publication_mode: KvEventPublicationMode = KvEventPublicationMode.DISABLED,
streaming_kv_events_config: Optional[dict[str, Any]] = None,
streaming_kv_events_gpus_per_node: Optional[int] = None,
Expand All @@ -1346,6 +1357,7 @@ async def get_publisher(
kv_state_endpoint=kv_state_endpoint,
image_token_id=image_token_id,
publish_metrics=publish_metrics,
publish_forward_pass_metrics=publish_forward_pass_metrics,
kv_event_publication_mode=kv_event_publication_mode,
streaming_kv_events_config=streaming_kv_events_config,
streaming_kv_events_gpus_per_node=streaming_kv_events_gpus_per_node,
Expand Down
68 changes: 56 additions & 12 deletions components/src/dynamo/trtllm/tests/test_trtllm_fpm_publisher.py
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ def _build_publisher_stub(
monkeypatch,
*,
attention_dp_size: int,
fpm_enabled: bool,
publish_forward_pass_metrics: bool,
publish_metrics: bool = True,
kv_event_publication_mode: publisher_mod.KvEventPublicationMode = publisher_mod.KvEventPublicationMode.POLLING,
):
Expand All @@ -320,11 +320,11 @@ def _build_publisher_stub(
pub.kv_state_endpoint = None
pub.image_token_id = None
pub.publish_metrics = publish_metrics
pub.publish_forward_pass_metrics = publish_forward_pass_metrics
pub.kv_event_publication_mode = kv_event_publication_mode
pub.streaming_kv_events_config = None
pub.streaming_kv_events_gpus_per_node = None
pub.attention_dp_size = attention_dp_size
pub.fpm_enabled = fpm_enabled
pub.processing_initial_created_events = True
pub.metrics_publisher = None
pub.fpm_publisher = None
Expand Down Expand Up @@ -372,7 +372,7 @@ def test_streaming_kv_events_create_direct_subscribers_without_engine_polling(
pub, module, _ = _build_publisher_stub(
monkeypatch,
attention_dp_size=2,
fpm_enabled=True,
publish_forward_pass_metrics=True,
)
pub.kv_event_publication_mode = publisher_mod.KvEventPublicationMode.STREAMING
pub.streaming_kv_events_config = {
Expand Down Expand Up @@ -419,10 +419,11 @@ def test_streaming_kv_event_endpoint_offset_matches_trtllm_streaming_manager(
def test_publisher_initialize_constructs_fpm_direct_publisher_when_fpm_enabled(
monkeypatch,
):
"""Under non-attention-DP (attention_dp_size == 1, fpm_enabled == True),
Publisher.initialize() constructs FpmDirectPublisher with dp_size=1."""
"""Under non-attention-DP (attention_dp_size == 1), the forward-pass
metrics opt-in makes Publisher.initialize() construct FpmDirectPublisher
with dp_size=1."""
pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub(
monkeypatch, attention_dp_size=1, fpm_enabled=True
monkeypatch, attention_dp_size=1, publish_forward_pass_metrics=True
)
pub.initialize()
fake_fpm_cls.assert_called_once()
Expand All @@ -432,35 +433,78 @@ def test_publisher_initialize_constructs_fpm_direct_publisher_when_fpm_enabled(
assert pub.fpm_publisher is not None


def test_publisher_initialize_metrics_only_does_not_start_kv_events(monkeypatch):
def test_publisher_initialize_metrics_only_starts_stats_without_fpm(monkeypatch):
"""The Publisher's two gates stay independent: the stats thread follows
either opt-in, the Planner publisher only the forward-pass one. Resolving
`--publish-metrics` into both is the config layer's job, covered by
test_trtllm_unit.py."""
pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub(
monkeypatch,
attention_dp_size=1,
fpm_enabled=True,
publish_forward_pass_metrics=False,
publish_metrics=True,
kv_event_publication_mode=publisher_mod.KvEventPublicationMode.DISABLED,
)
pub.initialize()

fake_fpm_cls.assert_called_once()
fake_fpm_cls.assert_not_called()
assert pub.metrics_publisher is not None
assert pub.fpm_publisher is None
pub._init_publish_metrics_thread.assert_called_once()
pub._init_publish_kv_cache_events_thread.assert_not_called()


def test_publisher_initialize_kv_events_only_does_not_start_metrics(monkeypatch):
def test_publisher_initialize_kv_events_only_does_not_start_stats(monkeypatch):
"""KV events never need iteration stats, so neither stats-fed publisher
nor the stats thread comes up."""
pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub(
monkeypatch,
attention_dp_size=1,
fpm_enabled=True,
publish_forward_pass_metrics=False,
publish_metrics=False,
)
pub.initialize()

fake_fpm_cls.assert_not_called()
assert pub.metrics_publisher is None
pub._init_publish_metrics_thread.assert_not_called()
pub._init_publish_kv_cache_events_thread.assert_called_once()


def test_publisher_initialize_fpm_only_starts_stats_and_planner_publisher(
monkeypatch,
):
pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub(
monkeypatch,
attention_dp_size=1,
publish_forward_pass_metrics=True,
publish_metrics=False,
kv_event_publication_mode=publisher_mod.KvEventPublicationMode.DISABLED,
)
pub.initialize()

fake_fpm_cls.assert_called_once()
assert pub.metrics_publisher is not None
pub._init_publish_metrics_thread.assert_called_once()
pub._init_publish_kv_cache_events_thread.assert_not_called()


def test_publisher_initialize_without_any_opt_in_starts_no_threads(monkeypatch):
pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub(
monkeypatch,
attention_dp_size=1,
publish_forward_pass_metrics=False,
publish_metrics=False,
kv_event_publication_mode=publisher_mod.KvEventPublicationMode.DISABLED,
)
pub.initialize()

fake_fpm_cls.assert_not_called()
assert pub.metrics_publisher is None
pub._init_publish_metrics_thread.assert_not_called()
pub._init_publish_kv_cache_events_thread.assert_not_called()


def _publisher_for_kv_event_test():
pub = publisher_mod.Publisher.__new__(publisher_mod.Publisher)
pub.additional_metrics = None
Expand Down Expand Up @@ -600,7 +644,7 @@ def test_publisher_initializes_fpm_publisher_under_attention_dp(monkeypatch):
"""Under attention-DP (attention_dp_size > 1), Publisher.initialize()
constructs one FpmDirectPublisher channel per attention-DP rank."""
pub, _publisher_mod, fake_fpm_cls = _build_publisher_stub(
monkeypatch, attention_dp_size=4, fpm_enabled=False
monkeypatch, attention_dp_size=4, publish_forward_pass_metrics=True
)
pub.initialize()
fake_fpm_cls.assert_called_once()
Expand Down
51 changes: 44 additions & 7 deletions components/src/dynamo/trtllm/tests/test_trtllm_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -513,26 +513,63 @@ def test_streaming_kv_events_accept_python_v2_cache_manager(monkeypatch):


@pytest.mark.parametrize(
("publish_flags", "expected_iter_perf_stats"),
("cli_args", "fpm_port_env", "expected"),
[
(["--publish-kv-events"], False),
(["--publish-metrics"], True),
(["--publish-kv-events", "--publish-metrics"], True),
(["--publish-events-and-metrics"], True),
([], None, False),
(["--fpm-trace"], None, True),
([], "20380", True),
(["--publish-kv-events"], None, False),
(["--publish-metrics"], None, True),
(["--publish-kv-events", "--publish-metrics"], None, True),
],
)
def test_config_publish_forward_pass_metrics_opt_in(
monkeypatch, cli_args, fpm_port_env, expected
):
"""The Planner's forward-pass metrics follow the opt-in shared with vLLM
and SGLang, plus --publish-metrics, which already pays for the iteration
statistics both read. --publish-kv-events alone never enables them."""
monkeypatch.delenv("DYN_FORWARDPASS_METRIC_PORT", raising=False)
# Config.validate writes DYN_FPM_TRACE back; setenv restores it on teardown.
monkeypatch.setenv("DYN_FPM_TRACE", "0")
if fpm_port_env is not None:
monkeypatch.setenv("DYN_FORWARDPASS_METRIC_PORT", fpm_port_env)

config = parse_args(["--model", "fake-model", *cli_args])

assert config.publish_forward_pass_metrics is expected


@pytest.mark.parametrize(
("publish_flags", "fpm_port_env", "expected_iter_perf_stats"),
[
(["--publish-kv-events"], None, False),
(["--publish-metrics"], None, True),
(["--fpm-trace"], None, True),
([], "20380", True),
([], None, False),
(["--publish-kv-events", "--publish-metrics"], None, True),
(["--publish-events-and-metrics"], None, True),
],
)
@pytest.mark.asyncio
async def test_init_llm_worker_engine_args_without_overrides(
monkeypatch, publish_flags, expected_iter_perf_stats
monkeypatch, publish_flags, fpm_port_env, expected_iter_perf_stats
):
"""KV events and metrics control their respective engine paths independently."""
"""Iteration stats follow --publish-metrics and the forward-pass metrics
opt-in, never --publish-kv-events alone; the engine-level
return_perf_metrics default stays off regardless."""
monkeypatch.delenv("DYN_TRTLLM_MAX_NUM_TOKENS", raising=False)
monkeypatch.delenv("DYN_TRTLLM_MAX_BATCH_SIZE", raising=False)
monkeypatch.delenv("DYN_TRTLLM_EXTRA_ENGINE_ARGS", raising=False)
monkeypatch.delenv("DYN_TRTLLM_OVERRIDE_ENGINE_ARGS", raising=False)
monkeypatch.delenv("DYN_TRTLLM_PUBLISH_KV_EVENTS", raising=False)
monkeypatch.delenv("DYN_TRTLLM_PUBLISH_METRICS", raising=False)
monkeypatch.delenv("DYN_TRTLLM_PUBLISH_EVENTS_AND_METRICS", raising=False)
monkeypatch.delenv("DYN_FORWARDPASS_METRIC_PORT", raising=False)
monkeypatch.setenv("DYN_FPM_TRACE", "0")
if fpm_port_env is not None:
monkeypatch.setenv("DYN_FORWARDPASS_METRIC_PORT", fpm_port_env)

if publish_flags == ["--publish-events-and-metrics"]:
with pytest.warns(
Expand Down
13 changes: 10 additions & 3 deletions components/src/dynamo/trtllm/workers/llm_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -425,9 +425,14 @@ async def init_llm_worker(
# and `--override-engine-args` are merged over `arg_map` below and can set
# it back to true for custom instrumentation.
"return_perf_metrics": False,
# Iteration stats drive the metrics-publishing path but are independent
# of KV-event publication. TensorRT backend always has this enabled.
"enable_iter_perf_stats": config.publish_metrics,
# Iteration stats feed the Prometheus surface and the Planner's
# forward-pass metrics, so either opt-in needs them. KV events do not:
# the engine only produces iteration stats under this flag, at a
# measurable per-iteration throughput cost. TensorRT backend always
# has this enabled.
"enable_iter_perf_stats": (
config.publish_metrics or config.publish_forward_pass_metrics
),
"kv_connector_config": kv_connector_config,
}

Expand Down Expand Up @@ -989,6 +994,7 @@ async def init_llm_worker(
if (
kv_event_publication_mode is not KvEventPublicationMode.DISABLED
or config.publish_metrics
or config.publish_forward_pass_metrics
):
# Initialize the independently gated KV-event and metrics publishers.
# Use model as fallback if served_model_name is not provided
Expand Down Expand Up @@ -1041,6 +1047,7 @@ async def init_llm_worker(
kv_state_endpoint=config.kv_state_endpoint,
image_token_id=image_token_id,
publish_metrics=config.publish_metrics,
publish_forward_pass_metrics=config.publish_forward_pass_metrics,
kv_event_publication_mode=kv_event_publication_mode,
streaming_kv_events_config=streaming_kv_events_config,
streaming_kv_events_gpus_per_node=gpus_per_node,
Expand Down
Loading
Loading