fix(health): skip canary health check for trtllm/vllm disagg decode workers

components/src/dynamo/trtllm/health_check.py

@@ -8,9 +8,10 @@
 """
 
 import logging
-from typing import Any
+from typing import Any, Optional
 
 from dynamo.health_check import HealthCheckPayload
+from dynamo.trtllm.constants import DisaggregationMode
 
 logger = logging.getLogger(__name__)
 
@@ -89,3 +90,51 @@ def __init__(self, tokenizer: Any = None) -> None:
             },
         }
         super().__init__()
+
+
+#: Reserved request key that marks a canary probe request for disagg decode.
+#: The handler detects this marker in `_setup_disaggregated_params_for_mode`
+#: and constructs a synthetic `LlmDisaggregatedParams(request_type=
+#: "context_and_generation")` so the engine runs the probe locally
+#: (full prefill + 1-token decode) without engaging the cache transceiver.
+#: Internal-only — not part of the public request protocol.
+CANARY_PROBE_KEY = "_canary_probe"
+
+
+class TrtllmDisaggDecodeHealthCheckPayload(TrtllmHealthCheckPayload):
+    """Canary payload for TRT-LLM disagg decode workers.
+
+    Sets a ``CANARY_PROBE_KEY`` marker so the handler bypasses the strict
+    "Disaggregated params are required for decode mode" guard and routes
+    the probe through `request_type="context_and_generation"` (local
+    prefill + decode, transceiver-free).
+
+    Mirrors SGLang's `SglangDisaggHealthCheckPayload` which uses
+    `FAKE_BOOTSTRAP_HOST` for the same purpose, and vLLM's natural
+    agg-style fallback when `prefill_result` is absent.
+    """
+
+    def __init__(self, tokenizer: Any = None) -> None:
+        super().__init__(tokenizer=tokenizer)
+        self.default_payload[CANARY_PROBE_KEY] = True
+
+
+def build_worker_health_check_payload(
+    disaggregation_mode: DisaggregationMode,
+    tokenizer: Any = None,
+) -> Optional[dict]:
+    """
+    Decide the health_check_payload for a TRT-LLM worker based on its disagg role.
+
+    Decode workers use a probe payload with ``CANARY_PROBE_KEY`` set; the
+    handler short-circuits in `_setup_disaggregated_params_for_mode` and
+    routes the probe as a local agg request (no cache transceiver, no real
+    prefill peer required). This gives disagg decode the same engine-level
+    canary coverage as aggregated / prefill workers.
+
+    Aggregated and prefill workers accept generic probes and register the
+    standard canary payload.
+    """
+    if disaggregation_mode == DisaggregationMode.DECODE:
+        return TrtllmDisaggDecodeHealthCheckPayload(tokenizer=tokenizer).to_dict()
+    return TrtllmHealthCheckPayload(tokenizer=tokenizer).to_dict()

components/src/dynamo/trtllm/request_handlers/handler_base.py

@@ -711,6 +711,22 @@ def _setup_disaggregated_params_for_mode(
         disaggregated_params = None
         epd_metadata: dict[str, Any] = {}
 
+        # Canary probe short-circuit: decode workers receive probes with a
+        # reserved `_canary_probe` marker (see TrtllmDisaggDecodeHealthCheckPayload
+        # in dynamo/trtllm/health_check.py). Build synthetic params with
+        # request_type="context_and_generation" so the engine runs the probe as
+        # full local prefill+decode — the cache transceiver activates only for
+        # request_type="generation_only", so this path is transceiver-free and
+        # doesn't need a real prefill peer. Mirrors sglang's FAKE_BOOTSTRAP_HOST.
+        if self.disaggregation_mode == DisaggregationMode.DECODE and request.get(
+            "_canary_probe"
+        ):
+            return (
+                LlmDisaggregatedParams(request_type="context_and_generation"),
+                None,
+                {},
+            )
+
         # PREFILL mode: setup context_only params
         if self.disaggregation_mode == DisaggregationMode.PREFILL:
             if ep_disaggregated_params:

components/src/dynamo/trtllm/tests/test_health_check_disagg.py

@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from dynamo.trtllm.constants import DisaggregationMode
+from dynamo.trtllm.health_check import (
+    CANARY_PROBE_KEY,
+    TrtllmDisaggDecodeHealthCheckPayload,
+    TrtllmHealthCheckPayload,
+    build_worker_health_check_payload,
+)
+
+pytestmark = [
+    pytest.mark.unit,
+    pytest.mark.trtllm,
+    pytest.mark.gpu_1,  # needs trtllm packages installed but does not use GPU
+    pytest.mark.profiled_vram_gib(0),
+    pytest.mark.pre_merge,
+]
+
+
+def test_trtllm_health_check_payload_has_no_disagg_params():
+    """Standard TrtllmHealthCheckPayload should NOT include disaggregated_params."""
+    payload = TrtllmHealthCheckPayload().to_dict()
+    assert "disaggregated_params" not in payload
+    assert "prefill_result" not in payload
+    assert "token_ids" in payload
+
+
+@pytest.mark.parametrize(
+    "mode",
+    [DisaggregationMode.AGGREGATED, DisaggregationMode.PREFILL],
+)
+def test_non_decode_modes_register_canary_payload(mode):
+    """Aggregated and prefill workers register the standard canary payload."""
+    payload = build_worker_health_check_payload(disaggregation_mode=mode)
+    assert payload is not None
+    assert "token_ids" in payload
+    assert "sampling_options" in payload
+
+
+def test_decode_mode_registers_probe_payload():
+    """Decode workers register a probe payload carrying CANARY_PROBE_KEY.
+
+    The handler detects the marker in `_setup_disaggregated_params_for_mode`
+    and routes the probe through `request_type="context_and_generation"`
+    so the engine runs it as a local agg request (no cache transceiver).
+    Pattern mirrors SGLang's FAKE_BOOTSTRAP_HOST.
+    """
+    payload = build_worker_health_check_payload(
+        disaggregation_mode=DisaggregationMode.DECODE
+    )
+    assert payload is not None
+    assert payload.get(CANARY_PROBE_KEY) is True
+    # Standard fields must still be present so the handler's downstream logic
+    # (sampling, stop conditions, token_ids) runs normally.
+    assert "token_ids" in payload
+    assert "sampling_options" in payload
+    assert "stop_conditions" in payload
+
+
+def test_disagg_decode_payload_class_sets_probe_marker():
+    """Direct construction of TrtllmDisaggDecodeHealthCheckPayload carries the marker."""
+    payload = TrtllmDisaggDecodeHealthCheckPayload().to_dict()
+    assert payload.get(CANARY_PROBE_KEY) is True
+    assert "disaggregated_params" not in payload  # real params built by handler

components/src/dynamo/trtllm/workers/llm_worker.py

@@ -54,7 +54,7 @@
 from dynamo.trtllm.args import Config
 from dynamo.trtllm.constants import DisaggregationMode, Modality
 from dynamo.trtllm.engine import Backend, get_llm_engine
-from dynamo.trtllm.health_check import TrtllmHealthCheckPayload
+from dynamo.trtllm.health_check import build_worker_health_check_payload
 from dynamo.trtllm.multimodal_processor import MultimodalRequestProcessor
 from dynamo.trtllm.publisher import DYNAMO_COMPONENT_REGISTRY, get_publisher
 from dynamo.trtllm.request_handlers.handlers import (
@@ -602,8 +602,18 @@ async def init_llm_worker(
                 custom_template_path=config.custom_jinja_template,
             )
 
-        # Get health check payload (checks env var and falls back to TensorRT-LLM default)
-        health_check_payload = TrtllmHealthCheckPayload(tokenizer=tokenizer).to_dict()
+        # Decode workers opt out of canary (the TRT-LLM decode handler strictly
+        # rejects canary probes without disaggregated_params); agg/prefill workers
+        # register the standard TRT-LLM payload. See build_worker_health_check_payload
+        # for the full reasoning.
+        health_check_payload = build_worker_health_check_payload(
+            disaggregation_mode=config.disaggregation_mode,
+            tokenizer=tokenizer,
+        )
+        if health_check_payload is None:
+            logging.info(
+                "Decode worker: canary health check disabled (no payload registered)"
+            )
 
         if config.publish_events_and_metrics:
             # Initialize and pass in the publisher to the request handler to

components/src/dynamo/vllm/tests/test_vllm_health_check.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Regression guard: vLLM workers must always register a canary payload.
+
+DIS-1737 originally routed disagg decode through a `payload = None` branch in
+`worker_factory.py`. That silently opted decode workers out of canary, which
+after DIS-1185 (canary = sole readiness authority) left them stuck NotReady.
+These tests ensure the payload constructor works for both agg and decode paths
+so no one re-introduces a DECODE-specific None branch.
+"""
+
+import pytest
+
+from dynamo.vllm.health_check import VllmHealthCheckPayload
+
+pytestmark = [
+    pytest.mark.unit,
+    pytest.mark.vllm,
+    pytest.mark.pre_merge,
+    pytest.mark.gpu_0,
+]
+
+
+@pytest.mark.parametrize("use_text_input", [False, True])
+def test_vllm_health_check_payload_is_non_none(use_text_input):
+    """VllmHealthCheckPayload.to_dict() returns a non-None dict regardless of
+    tokenizer mode. Worker code must always register a canary target; decode
+    workers rely on the vLLM handler's natural agg-style fallback when
+    `prefill_result` is absent (handlers.py::_generate_token_mode)."""
+    payload = VllmHealthCheckPayload(
+        engine_client=None, use_text_input=use_text_input
+    ).to_dict()
+
+    assert payload is not None
+    assert isinstance(payload, dict)
+    # Payload must contain some form of input the engine can decode.
+    assert "token_ids" in payload or "prompt" in payload

components/src/dynamo/vllm/worker_factory.py

@@ -392,6 +392,10 @@ async def _create_decode_worker(
             vllm_config,
         )
 
+        # vLLM's DecodeWorkerHandler._generate_token_mode handles requests without
+        # prefill_result by running them agg-style, so the standard canary payload
+        # works for decode workers too — the engine produces one token regardless
+        # of disaggregation mode. Always register a canary target.
         health_check_payload = VllmHealthCheckPayload(
             engine_client, use_text_input=config.use_vllm_tokenizer
         ).to_dict()

lib/runtime/src/system_health.rs

@@ -100,10 +100,21 @@ impl SystemHealth {
         self.health_check_enabled
     }
 
-    /// Signal endpoint transport registration. Sets Ready when canary is disabled;
-    /// no-op when canary is enabled (canary will set Ready after verification).
+    /// Signal endpoint transport registration.
+    ///
+    /// Marks the endpoint Ready when canary is disabled globally, OR when the
+    /// endpoint has no registered canary target (i.e. the caller opted out of
+    /// canary by not passing a `health_check_payload` to `serve_endpoint`).
+    /// Endpoints that DID register a canary target stay NotReady until the
+    /// canary verifies them — preserving DIS-1185's "canary is the authoritative
+    /// readiness signal for endpoints that opt in" contract.
     pub fn set_endpoint_registered(&self, endpoint: &str) {
-        if !self.health_check_enabled {
+        let has_target = self
+            .health_check_targets
+            .read()
+            .unwrap()
+            .contains_key(endpoint);
+        if !self.health_check_enabled || !has_target {
             self.set_endpoint_health_status(endpoint, HealthStatus::Ready);
         }
     }
@@ -141,20 +152,26 @@ impl SystemHealth {
                     .get(endpoint)
                     .is_some_and(|status| *status == HealthStatus::Ready)
             })
+        } else if !health_check_targets.is_empty() {
+            // Canary-opt-in endpoints exist: every one must be Ready.
+            health_check_targets
+                .iter()
+                .all(|(endpoint_subject, _target)| {
+                    endpoint_health
+                        .get(endpoint_subject)
+                        .is_some_and(|status| *status == HealthStatus::Ready)
+                })
+        } else if !endpoint_health.is_empty() {
+            // No canary targets, but endpoints have registered. Healthy when
+            // every registered endpoint is Ready. This covers workers that
+            // opt every endpoint out of canary (e.g. disagg decode workers,
+            // secondary operational endpoints).
+            endpoint_health
+                .values()
+                .all(|status| *status == HealthStatus::Ready)
         } else {
-            // If we have registered health check targets, use them to determine health
-            if !health_check_targets.is_empty() {
-                health_check_targets
-                    .iter()
-                    .all(|(endpoint_subject, _target)| {
-                        endpoint_health
-                            .get(endpoint_subject)
-                            .is_some_and(|status| *status == HealthStatus::Ready)
-                    })
-            } else {
-                // No health check targets registered, use simple system health
-                self.system_health == HealthStatus::Ready
-            }
+            // No endpoints registered at all — use simple system health.
+            self.system_health == HealthStatus::Ready
         };
 
         (healthy, endpoints)
@@ -298,3 +315,94 @@ impl SystemHealth {
         &self.live_path
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn sh(canary_enabled: bool) -> SystemHealth {
+        SystemHealth::new(
+            HealthStatus::NotReady,
+            vec![],
+            canary_enabled,
+            "/health".into(),
+            "/live".into(),
+        )
+    }
+
+    #[test]
+    fn endpoint_registered_auto_readies_without_target_when_canary_enabled() {
+        // Canary enabled but the endpoint did not register a canary target —
+        // it opted out of canary, so registration alone marks it Ready.
+        let h = sh(true);
+        h.set_endpoint_registered("generate");
+        assert_eq!(
+            h.get_endpoint_health_status("generate"),
+            Some(HealthStatus::Ready),
+            "endpoint with no canary target must auto-Ready"
+        );
+    }
+
+    #[test]
+    fn endpoint_registered_stays_notready_with_target_when_canary_enabled() {
+        // Canary enabled and the endpoint registered a target — registration
+        // does NOT flip Ready; canary verifies first (DIS-1185 contract).
+        let h = sh(true);
+        let instance = component::Instance {
+            component: "test".into(),
+            endpoint: "generate".into(),
+            namespace: "test".into(),
+            instance_id: 0,
+            transport: component::TransportType::Tcp("localhost:0".into()),
+            device_type: None,
+        };
+        h.health_check_targets.write().unwrap().insert(
+            "generate".to_string(),
+            HealthCheckTarget {
+                instance,
+                payload: serde_json::json!({}),
+            },
+        );
+        h.set_endpoint_registered("generate");
+        // With a registered target, set_endpoint_registered must NOT mark
+        // the endpoint Ready — canary is responsible. Either "no entry" or
+        // an explicit NotReady is acceptable (both make /health path #2 report
+        // unhealthy until canary succeeds).
+        assert_ne!(
+            h.get_endpoint_health_status("generate"),
+            Some(HealthStatus::Ready),
+            "endpoint with a canary target must wait for canary, not auto-Ready"
+        );
+    }
+
+    #[test]
+    fn endpoint_registered_readies_when_canary_disabled() {
+        let h = sh(false);
+        h.set_endpoint_registered("generate");
+        assert_eq!(
+            h.get_endpoint_health_status("generate"),
+            Some(HealthStatus::Ready),
+        );
+    }
+
+    #[test]
+    fn get_health_status_uses_endpoint_health_when_no_target() {
+        // No canary targets, no use_endpoint_health_status override.
+        // A single registered endpoint in Ready state should make /health healthy.
+        let h = sh(true);
+        h.set_endpoint_registered("generate");
+        let (healthy, _eps) = h.get_health_status();
+        assert!(
+            healthy,
+            "when no canary target exists and all registered endpoints are Ready, /health is healthy"
+        );
+    }
+
+    #[test]
+    fn get_health_status_notready_when_no_endpoints_and_system_notready() {
+        // No endpoints registered at all — falls back to system_health (NotReady by default).
+        let h = sh(true);
+        let (healthy, _eps) = h.get_health_status();
+        assert!(!healthy);
+    }
+}