From 0322d904813648925a63f59e72d91e8a92f210af Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 15 Sep 2026 09:28:55 -0700 Subject: [PATCH 01/18] chore(container): use pinned vLLM release image Signed-off-by: Shreyas Misra --- container/context.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/container/context.yaml b/container/context.yaml index 991a2e4b438a..207b64c5a291 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -75,9 +75,11 @@ dynamo: vllm: cuda13.0: base_image: nvcr.io/nvidia/cuda-dl-base - runtime_image: vllm/vllm-openai + runtime_image: public.ecr.aws/q9t5s3a7/vllm-release-repo base_image_tag: 25.11-cuda13.0-devel-ubuntu24.04 - runtime_image_tag: v0.28.0-ubuntu2404 + # vLLM commit 8a728663c1c3eeace834a95f5654fa653cc1998c. The multi-arch + # manifest resolves to sha256:f5df5cc3302b5f404848c4eca88d7bf7ed5226e151c056da22816d7734644d67. + runtime_image_tag: 8a728663c1c3eeace834a95f5654fa653cc1998c # Baseline is vllm-openai's TRUE base, nvidia/cuda:13.0.2-base-ubuntu24.04 # (Docker Hub), NOT a self-baseline of the third-party vllm/vllm-openai image. # vllm-openai is built FROM nvidia/cuda, so this attributes everything vLLM From f68593ceedf6776107b03af8bb43881e94a6b8b2 Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 15 Sep 2026 09:46:54 -0700 Subject: [PATCH 02/18] fix(container): pin tokenizers for vLLM-Omni Signed-off-by: Shreyas Misra --- container/context.yaml | 3 +++ container/templates/args.Dockerfile | 1 + container/templates/vllm_runtime.Dockerfile | 7 ++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/container/context.yaml b/container/context.yaml index 207b64c5a291..38b1e8743fdd 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -110,6 +110,9 @@ vllm: # this pin its dependency solve is unsatisfiable. Remove the pin once # vLLM-Omni lifts its <5.15 upper bound. transformers_version: "5.14.1" + # Transformers 5.14.1 caps tokenizers at 0.23.0, while the pinned vLLM + # runtime resolves its open-ended tokenizers dependency to 0.23.2. + tokenizers_version: "0.23.0" nixl_ref: v1.3.2 max_jobs: "10" enable_media_ffmpeg: "true" diff --git a/container/templates/args.Dockerfile b/container/templates/args.Dockerfile index fa19a44c036d..7927994ba4a5 100644 --- a/container/templates/args.Dockerfile +++ b/container/templates/args.Dockerfile @@ -99,6 +99,7 @@ ARG PLANNER_RUNTIME_IMAGE_TAG={{ context.dynamo.planner_runtime_image_tag }} {% if framework == "vllm" -%} ARG MAX_JOBS={{ context.vllm.max_jobs }} ARG TRANSFORMERS_VERSION={{ context.vllm.transformers_version }} +ARG TOKENIZERS_VERSION={{ context.vllm.tokenizers_version }} ARG VLLM_OMNI_REF={{ context.vllm[device_key].get("vllm_omni_ref", context.vllm.vllm_omni_ref) }} {% if device == "cuda" -%} diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 2454a4befb82..8df2aaf51e01 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -189,12 +189,13 @@ COPY --chmod=775 --chown=dynamo:0 --from=wheel_builder /opt/dynamo/dist/*.whl /o # The vLLM 0.28.0 release images resolve the unbounded `transformers>=5.5.3` # requirement to 5.15.1, but vLLM-Omni 0.28.0rc1 caps Transformers below 5.15. # Omni is layered against the installed Transformers version, so install the -# compatible release first and its dependency solve sees the final Transformers -# invariant instead of resolving against 5.15.1. +# compatible releases first so its dependency solve sees the final Transformers +# and Tokenizers invariants instead of the base image's incompatible versions. RUN --mount=type=cache,id=uv-root-{{ context.dynamo.uv_version }},target=/root/.cache/uv,sharing=locked \ export UV_CACHE_DIR=/root/.cache/uv && \ uv pip install {{ pip_target }} --no-deps \ - "transformers==${TRANSFORMERS_VERSION}" + "transformers==${TRANSFORMERS_VERSION}" \ + "tokenizers==${TOKENIZERS_VERSION}" {% if device != "cuda" %} # NIXL meta package always tries to find a cuda-backend From 49e0e3f33ba685248ea356163cb11b6ccf4fc41c Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 15 Sep 2026 10:22:19 -0700 Subject: [PATCH 03/18] fix(container): scope tokenizers build argument Signed-off-by: Shreyas Misra --- container/templates/vllm_runtime.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 8df2aaf51e01..216be74892ee 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -20,6 +20,7 @@ ARG ENABLE_KVBM ARG ENABLE_GPU_MEMORY_SERVICE ARG VLLM_OMNI_REF ARG TRANSFORMERS_VERSION +ARG TOKENIZERS_VERSION ARG NIXL_REF {% if device == "cuda" %} ARG CUDA_MAJOR From 24073518c410f4d51045a82ef845e77a545171d7 Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 15 Sep 2026 10:36:14 -0700 Subject: [PATCH 04/18] fix(container): use published tokenizers release Signed-off-by: Shreyas Misra --- container/context.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/context.yaml b/container/context.yaml index 38b1e8743fdd..8e307a03af4c 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -112,7 +112,7 @@ vllm: transformers_version: "5.14.1" # Transformers 5.14.1 caps tokenizers at 0.23.0, while the pinned vLLM # runtime resolves its open-ended tokenizers dependency to 0.23.2. - tokenizers_version: "0.23.0" + tokenizers_version: "0.22.2" nixl_ref: v1.3.2 max_jobs: "10" enable_media_ffmpeg: "true" From f6c6bc13d7df2d1bae99201221ba4d2ab4214873 Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 15 Sep 2026 10:49:46 -0700 Subject: [PATCH 05/18] fix(container): record InstantTensor license Signed-off-by: Shreyas Misra --- container/compliance/license_overrides.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/container/compliance/license_overrides.yaml b/container/compliance/license_overrides.yaml index 4e37b6e76bff..f42e8c24b4ec 100644 --- a/container/compliance/license_overrides.yaml +++ b/container/compliance/license_overrides.yaml @@ -173,6 +173,15 @@ overrides: license: LicenseRef-NVIDIA-Proprietary source: "Internal NVIDIA tool — distributed via NGC under the NVIDIA Software License Agreement" + # --- Open-source Python wheels with non-SPDX metadata --- + # instanttensor publishes `License: Apache License`, which the conservative + # metadata normalizer does not map to an SPDX identifier. Its release source + # archive and upstream repository both include the Apache-2.0 license text. + - ecosystem: python + name: instanttensor + license: Apache-2.0 + source: "InstantTensor v0.1.9 upstream LICENSE — https://github.com/scitix/InstantTensor/blob/v0.1.9/LICENSE" + # --- Intel oneAPI Python wheels --- # oneccl / oneccl-devel wheel METADATA sets License: "Intel Simplified Software # License" and Classifier: "License :: Other/Proprietary License" — neither From 76b18bfea93735c34c5898561abf94485a27318c Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Thu, 17 Sep 2026 09:35:52 -0700 Subject: [PATCH 06/18] fix(router): separate remote KV wait from prefill load Signed-off-by: Shreyas Misra --- .../configuration/groups/router_args.py | 23 ++++ .../src/dynamo/common/forward_pass_metrics.py | 10 ++ .../configuration/test_kv_router_args.py | 25 ++++ components/src/dynamo/vllm/args.py | 21 ++- .../src/dynamo/vllm/instrumented_scheduler.py | 11 ++ components/src/dynamo/vllm/main.py | 12 +- .../tests/test_vllm_instrumented_scheduler.py | 26 ++++ .../src/dynamo/vllm/tests/test_vllm_unit.py | 33 +++++ .../observability/metrics-catalog.mdx | 8 ++ lib/bindings/python/rust/llm/entrypoint.rs | 7 +- lib/bindings/python/rust/llm/kv.rs | 19 ++- lib/bindings/python/src/dynamo/_core.pyi | 4 + lib/kv-router/src/protocols.rs | 7 + lib/kv-router/src/scheduling/queue.rs | 28 +++- lib/llm/src/discovery/worker_monitor.rs | 120 +++++++++++++++++- lib/llm/src/fpm_publisher.rs | 116 ++++++++++++++++- lib/llm/src/http/service/busy_threshold.rs | 11 +- lib/llm/src/kv_router/metrics.rs | 84 +++++++++++- lib/llm/src/kv_router/metrics_subscriber.rs | 10 ++ .../src/kv_router/publisher/worker_metrics.rs | 1 + lib/runtime/src/metrics/prometheus_names.rs | 6 + 21 files changed, 559 insertions(+), 23 deletions(-) diff --git a/components/src/dynamo/common/configuration/groups/router_args.py b/components/src/dynamo/common/configuration/groups/router_args.py index cdf3bf2f50bb..29dec3b5b5a4 100644 --- a/components/src/dynamo/common/configuration/groups/router_args.py +++ b/components/src/dynamo/common/configuration/groups/router_args.py @@ -36,6 +36,7 @@ "active_decode_blocks_threshold", "active_prefill_tokens_threshold", "active_prefill_tokens_threshold_frac", + "remote_kv_waiting_tokens_threshold", "session_affinity_ttl_secs", "session_affinity_mode", ) @@ -82,6 +83,7 @@ class RouterConfigBase(ConfigBase): active_decode_blocks_threshold: Optional[float] active_prefill_tokens_threshold: Optional[int] active_prefill_tokens_threshold_frac: Optional[float] + remote_kv_waiting_tokens_threshold: Optional[int] def router_kwargs(self) -> dict: """Return a dict suitable for ``RouterConfig(mode, kv_config, **kwargs)``.""" @@ -109,6 +111,10 @@ def validate_rejection_thresholds(self) -> None: "--active-prefill-tokens-threshold-frac must be a finite value >= 0" ) + remote_kv_threshold = self.remote_kv_waiting_tokens_threshold + if remote_kv_threshold is not None and remote_kv_threshold < 0: + raise ValueError("--remote-kv-waiting-tokens-threshold must be >= 0") + def log_rejection_thresholds(self) -> None: """Log which independently configured rejection checks are active.""" configured = [ @@ -126,6 +132,10 @@ def log_rejection_thresholds(self) -> None: "--active-prefill-tokens-threshold-frac", self.active_prefill_tokens_threshold_frac, ), + ( + "--remote-kv-waiting-tokens-threshold", + self.remote_kv_waiting_tokens_threshold, + ), ) if value is not None ] @@ -310,6 +320,19 @@ def add_arguments(self, parser) -> None: ), arg_type=nullable_float, ) + add_argument( + g, + flag_name="--remote-kv-waiting-tokens-threshold", + env_var="DYN_REMOTE_KV_WAITING_TOKENS_THRESHOLD", + default=None, + help=( + "Literal token threshold for external-KV transfer backlog. A worker " + "remains eligible for prefix-affine routing until tokens in " + "WAITING_FOR_REMOTE_KVS exceed this value. Unset by default; pass " + "'None' to disable it." + ), + arg_type=nullable_int, + ) # CLI spelling -> `dynamo.llm.RouterMode` attribute name. diff --git a/components/src/dynamo/common/forward_pass_metrics.py b/components/src/dynamo/common/forward_pass_metrics.py index f656ee7bbeb0..1e0621e0f4f7 100644 --- a/components/src/dynamo/common/forward_pass_metrics.py +++ b/components/src/dynamo/common/forward_pass_metrics.py @@ -149,6 +149,16 @@ class QueuedRequestMetrics( # Population variance of KV context lengths for queued decode requests. var_decode_kv_tokens: float = 0.0 + # Requests whose destination KV blocks are allocated but whose external + # KV transfer has not completed. These are also included in the queued + # decode totals above for planner compatibility, but are broken out here + # so routing does not mistake transfer wait for prefill compute. + num_remote_kv_waiting_requests: int = 0 + + # Tokens currently being received by requests in + # WAITING_FOR_REMOTE_KVS. This is transfer backlog, not prefill work. + sum_remote_kv_waiting_tokens: int = 0 + class ForwardPassMetrics( msgspec.Struct, diff --git a/components/src/dynamo/common/tests/configuration/test_kv_router_args.py b/components/src/dynamo/common/tests/configuration/test_kv_router_args.py index 6e0a7cc9e634..fcc194ee9144 100644 --- a/components/src/dynamo/common/tests/configuration/test_kv_router_args.py +++ b/components/src/dynamo/common/tests/configuration/test_kv_router_args.py @@ -25,6 +25,7 @@ def _clear_rejection_threshold_env(monkeypatch: pytest.MonkeyPatch) -> None: "DYN_ACTIVE_DECODE_BLOCKS_THRESHOLD", "DYN_ACTIVE_PREFILL_TOKENS_THRESHOLD", "DYN_ACTIVE_PREFILL_TOKENS_THRESHOLD_FRAC", + "DYN_REMOTE_KV_WAITING_TOKENS_THRESHOLD", "DYN_ADMISSION_CONTROL", "DYN_ROUTER_QUEUE_THRESHOLD", "DYN_ROUTER_SESSION_AFFINITY_MODE", @@ -584,11 +585,13 @@ def test_frontend_rejection_thresholds_default_to_none( assert config.active_decode_blocks_threshold is None assert config.active_prefill_tokens_threshold is None assert config.active_prefill_tokens_threshold_frac is None + assert config.remote_kv_waiting_tokens_threshold is None assert config.router_queue_threshold is None assert config.router_kwargs() == { "active_decode_blocks_threshold": None, "active_prefill_tokens_threshold": None, "active_prefill_tokens_threshold_frac": None, + "remote_kv_waiting_tokens_threshold": None, "session_affinity_ttl_secs": None, "session_affinity_mode": "hard", } @@ -616,6 +619,12 @@ def test_frontend_rejection_thresholds_default_to_none( "active_prefill_tokens_threshold_frac", 2.0, ), + ( + "--remote-kv-waiting-tokens-threshold", + "8192", + "remote_kv_waiting_tokens_threshold", + 8192, + ), ], ) def test_each_cli_rejection_threshold_is_independently_opt_in( @@ -638,6 +647,7 @@ def test_each_cli_rejection_threshold_is_independently_opt_in( "active_decode_blocks_threshold": config.active_decode_blocks_threshold, "active_prefill_tokens_threshold": config.active_prefill_tokens_threshold, "active_prefill_tokens_threshold_frac": config.active_prefill_tokens_threshold_frac, + "remote_kv_waiting_tokens_threshold": config.remote_kv_waiting_tokens_threshold, } assert thresholds[field] == expected assert all(value is None for name, value in thresholds.items() if name != field) @@ -662,6 +672,11 @@ def test_each_cli_rejection_threshold_is_independently_opt_in( "active_prefill_tokens_threshold_frac", 3.0, ), + ( + "DYN_REMOTE_KV_WAITING_TOKENS_THRESHOLD", + "remote_kv_waiting_tokens_threshold", + 16384, + ), ], ) def test_each_environment_rejection_threshold_is_independently_opt_in( @@ -682,6 +697,7 @@ def test_each_environment_rejection_threshold_is_independently_opt_in( "active_decode_blocks_threshold": config.active_decode_blocks_threshold, "active_prefill_tokens_threshold": config.active_prefill_tokens_threshold, "active_prefill_tokens_threshold_frac": config.active_prefill_tokens_threshold_frac, + "remote_kv_waiting_tokens_threshold": config.remote_kv_waiting_tokens_threshold, } assert thresholds[field] == expected assert all(value is None for name, value in thresholds.items() if name != field) @@ -703,6 +719,8 @@ def test_all_rejection_thresholds_and_queue_override_are_forwarded( "1000", "--active-prefill-tokens-threshold-frac", "2.0", + "--remote-kv-waiting-tokens-threshold", + "8192", "--router-queue-threshold", "32.0", ] @@ -713,11 +731,13 @@ def test_all_rejection_thresholds_and_queue_override_are_forwarded( assert config.active_decode_blocks_threshold == 0.5 assert config.active_prefill_tokens_threshold == 1000 assert config.active_prefill_tokens_threshold_frac == 2.0 + assert config.remote_kv_waiting_tokens_threshold == 8192 assert config.router_queue_threshold == 32.0 assert config.router_kwargs() == { "active_decode_blocks_threshold": 0.5, "active_prefill_tokens_threshold": 1000, "active_prefill_tokens_threshold_frac": 2.0, + "remote_kv_waiting_tokens_threshold": 8192, "session_affinity_ttl_secs": None, "session_affinity_mode": "hard", } @@ -788,6 +808,7 @@ def test_admission_control_cli_flag_warns_and_is_ignored( assert config.active_decode_blocks_threshold is None assert config.active_prefill_tokens_threshold is None assert config.active_prefill_tokens_threshold_frac is None + assert config.remote_kv_waiting_tokens_threshold is None assert "--admission-control is no longer supported and is ignored" in caplog.text with pytest.raises(SystemExit): @@ -810,6 +831,7 @@ def test_removed_admission_control_environment_warns_and_is_ignored( assert config.active_decode_blocks_threshold is None assert config.active_prefill_tokens_threshold is None assert config.active_prefill_tokens_threshold_frac is None + assert config.remote_kv_waiting_tokens_threshold is None assert "DYN_ADMISSION_CONTROL is no longer supported and is ignored" in caplog.text @@ -819,6 +841,7 @@ def test_removed_admission_control_environment_warns_and_is_ignored( "--active-decode-blocks-threshold", "--active-prefill-tokens-threshold", "--active-prefill-tokens-threshold-frac", + "--remote-kv-waiting-tokens-threshold", ], ) def test_explicit_none_keeps_rejection_threshold_disabled( @@ -835,6 +858,7 @@ def test_explicit_none_keeps_rejection_threshold_disabled( assert config.active_decode_blocks_threshold is None assert config.active_prefill_tokens_threshold is None assert config.active_prefill_tokens_threshold_frac is None + assert config.remote_kv_waiting_tokens_threshold is None @pytest.mark.parametrize( @@ -846,6 +870,7 @@ def test_explicit_none_keeps_rejection_threshold_disabled( ("--active-prefill-tokens-threshold", "-1"), ("--active-prefill-tokens-threshold-frac", "-0.1"), ("--active-prefill-tokens-threshold-frac", "inf"), + ("--remote-kv-waiting-tokens-threshold", "-1"), ], ) def test_rejection_threshold_validation_rejects_invalid_values( diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index 6a9ced7de654..b31799627558 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -280,11 +280,20 @@ def _unsupported_fpm_trace_role(dynamo_config: Config) -> Optional[str]: return None -def _forward_pass_metrics_enabled(dynamo_config: Config) -> bool: +def _has_kv_transfer_connector(engine_config: object | None) -> bool: + """Whether an engine may enter WAITING_FOR_REMOTE_KVS.""" + if engine_config is None: + return False + return getattr(engine_config, "kv_transfer_config", None) is not None + + +def _forward_pass_metrics_enabled( + dynamo_config: Config, engine_config: object | None = None +) -> bool: """Resolve FPM activation without changing the legacy explicit-port path.""" if envs.is_set("DYN_FORWARDPASS_METRIC_PORT"): return True - if not dynamo_config.fpm_trace: + if not dynamo_config.fpm_trace and not _has_kv_transfer_connector(engine_config): return False unsupported_role = _unsupported_fpm_trace_role(dynamo_config) @@ -353,7 +362,7 @@ def update_engine_config_with_dynamo( f"(use_kv_events={dynamo_config.use_kv_events})" ) - fpm_enabled = _forward_pass_metrics_enabled(dynamo_config) + fpm_enabled = _forward_pass_metrics_enabled(dynamo_config, engine_config) if fpm_enabled: existing_cls = getattr(engine_config, "scheduler_cls", None) if existing_cls is None: @@ -368,7 +377,11 @@ def update_engine_config_with_dynamo( fpm_source = ( "DYN_FORWARDPASS_METRIC_PORT is set" if envs.is_set("DYN_FORWARDPASS_METRIC_PORT") - else "--fpm-trace/DYN_FPM_TRACE is enabled" + else ( + "--fpm-trace/DYN_FPM_TRACE is enabled" + if dynamo_config.fpm_trace + else "a KV transfer connector requires remote-wait telemetry" + ) ) logger.warning( f"{fpm_source} but scheduler_cls " diff --git a/components/src/dynamo/vllm/instrumented_scheduler.py b/components/src/dynamo/vllm/instrumented_scheduler.py index 62aae5f15a4a..4018a161b363 100644 --- a/components/src/dynamo/vllm/instrumented_scheduler.py +++ b/components/src/dynamo/vllm/instrumented_scheduler.py @@ -2153,10 +2153,14 @@ def _compute_queued(self) -> QueuedRequestMetrics: """ prefill = WelfordAccumulator() decode_kv = WelfordAccumulator() + remote_kv = WelfordAccumulator() for request in self.waiting: if request.status == RequestStatus.PREEMPTED: decode_kv.add(request.num_computed_tokens) + elif request.status == RequestStatus.WAITING_FOR_REMOTE_KVS: + decode_kv.add(request.num_computed_tokens) + remote_kv.add(request.num_computed_tokens) else: prefill.add(request.num_tokens) @@ -2166,6 +2170,11 @@ def _compute_queued(self) -> QueuedRequestMetrics: # engine and being transferred. Next schedule() step will # start generating -- count as queued decode. decode_kv.add(request.num_computed_tokens) + # Keep transfer backlog distinct from compute backlog. The + # router consumes this exact status-derived signal when + # deciding whether an affinity-hot worker's transfer queue is + # saturated. + remote_kv.add(request.num_computed_tokens) else: # Structured-output waits / WAITING_FOR_STREAMING_REQ: # no KV yet, essentially a queued prefill awaiting a @@ -2179,6 +2188,8 @@ def _compute_queued(self) -> QueuedRequestMetrics: num_decode_requests=decode_kv.n, sum_decode_kv_tokens=decode_kv.s, var_decode_kv_tokens=decode_kv.variance(), + num_remote_kv_waiting_requests=remote_kv.n, + sum_remote_kv_waiting_tokens=remote_kv.s, ) # ------------------------------------------------------------------ diff --git a/components/src/dynamo/vllm/main.py b/components/src/dynamo/vllm/main.py index b27f47f4062d..dc667eeb8d50 100644 --- a/components/src/dynamo/vllm/main.py +++ b/components/src/dynamo/vllm/main.py @@ -552,7 +552,17 @@ def setup_fpm_relay( Returns: List of FpmEventRelay instances, or None if FPM is not enabled. """ - if not (envs.is_set("DYN_FORWARDPASS_METRIC_PORT") or config.fpm_trace): + scheduler_cls = getattr(vllm_config, "scheduler_cls", None) + remote_kv_telemetry = ( + getattr(vllm_config, "kv_transfer_config", None) is not None + and scheduler_cls is not None + and "InstrumentedScheduler" in str(scheduler_cls) + ) + if not ( + envs.is_set("DYN_FORWARDPASS_METRIC_PORT") + or config.fpm_trace + or remote_kv_telemetry + ): return None try: diff --git a/components/src/dynamo/vllm/tests/test_vllm_instrumented_scheduler.py b/components/src/dynamo/vllm/tests/test_vllm_instrumented_scheduler.py index fee0c79edf4f..f9f14b25303f 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_instrumented_scheduler.py +++ b/components/src/dynamo/vllm/tests/test_vllm_instrumented_scheduler.py @@ -389,6 +389,28 @@ def test_skipped_waiting_for_remote_kvs_counts_as_queued_decode(): # Classified as decode with KV = num_computed_tokens. assert q.num_decode_requests == 2 assert q.sum_decode_kv_tokens == 1500 + assert q.num_remote_kv_waiting_requests == 2 + assert q.sum_remote_kv_waiting_tokens == 1500 + + +def test_main_waiting_queue_remote_kvs_uses_same_transfer_classification(): + """Be robust to vLLM queue-placement changes and transient states.""" + + q = _run_compute_queued( + waiting=[ + _make_request( + RequestStatus.WAITING_FOR_REMOTE_KVS, + num_tokens=1000, + num_computed_tokens=768, + ) + ], + skipped_waiting=[], + ) + + assert q.num_prefill_requests == 0 + assert q.num_decode_requests == 1 + assert q.num_remote_kv_waiting_requests == 1 + assert q.sum_remote_kv_waiting_tokens == 768 def test_skipped_waiting_for_structured_output_counts_as_queued_prefill(): @@ -458,6 +480,10 @@ def test_mixed_disagg_decode_engine_snapshot(): # 1 preempted (local decode evicted) + 3 remote-KV-waiting. assert q.num_decode_requests == 4 assert q.sum_decode_kv_tokens == 780 + 1024 + 2048 + 512 + # Only remote-KV waits contribute to transfer pressure; a local preemption + # remains decode backlog but is not a transfer. + assert q.num_remote_kv_waiting_requests == 3 + assert q.sum_remote_kv_waiting_tokens == 1024 + 2048 + 512 def test_mixed_prefill_engine_snapshot(): diff --git a/components/src/dynamo/vllm/tests/test_vllm_unit.py b/components/src/dynamo/vllm/tests/test_vllm_unit.py index d6d01b299813..09ce651bd40e 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_unit.py +++ b/components/src/dynamo/vllm/tests/test_vllm_unit.py @@ -1761,6 +1761,39 @@ def test_false_trace_does_not_enable_fpm(self, monkeypatch): assert engine_cfg.scheduler_cls is None + def test_kv_transfer_connector_enables_remote_wait_telemetry(self, monkeypatch): + monkeypatch.delenv("DYN_FORWARDPASS_METRIC_PORT", raising=False) + dynamo_cfg = _make_dynamo_config() + engine_cfg = _make_engine_config_with_runner( + scheduler_cls=None, + kv_transfer_config=SimpleNamespace(kv_connector="OffloadingConnector"), + ) + + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert ( + engine_cfg.scheduler_cls + == "dynamo.vllm.instrumented_scheduler.InstrumentedScheduler" + ) + assert envs.DYN_FORWARDPASS_METRIC_PORT == 20380 + + def test_custom_scheduler_does_not_start_implicit_remote_wait_relay( + self, monkeypatch + ): + vllm_main = _load_vllm_main() + monkeypatch.delenv("DYN_FORWARDPASS_METRIC_PORT", raising=False) + dynamo_cfg = _make_dynamo_config() + engine_cfg = _make_engine_config_with_runner( + scheduler_cls="example.CustomScheduler", + kv_transfer_config=SimpleNamespace(kv_connector="OffloadingConnector"), + ) + + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert vllm_main.setup_fpm_relay( + dynamo_cfg, SimpleNamespace(), engine_cfg + ) is None + def test_disabled_trace_does_not_start_relay(self, monkeypatch): vllm_main = _load_vllm_main() monkeypatch.delenv("DYN_FORWARDPASS_METRIC_PORT", raising=False) diff --git a/docs/fern/pages/reference/observability/metrics-catalog.mdx b/docs/fern/pages/reference/observability/metrics-catalog.mdx index 21cc3b8e1875..38bf4772eaf9 100644 --- a/docs/fern/pages/reference/observability/metrics-catalog.mdx +++ b/docs/fern/pages/reference/observability/metrics-catalog.mdx @@ -159,6 +159,14 @@ Counters for requests that end early: [cancellations](../../developer-guide/know Active prefill tokens queued per worker. + + Requests whose external KV transfer has not completed, per worker. This is transfer pressure, not prefill compute. + + + + KV context tokens belonging to requests whose external KV transfer has not completed, per worker. + + Last observed time to first token per worker, in seconds. diff --git a/lib/bindings/python/rust/llm/entrypoint.rs b/lib/bindings/python/rust/llm/entrypoint.rs index 39e3b20817f2..d2e44a2c91df 100644 --- a/lib/bindings/python/rust/llm/entrypoint.rs +++ b/lib/bindings/python/rust/llm/entrypoint.rs @@ -480,6 +480,7 @@ pub struct RouterConfig { active_prefill_tokens_threshold: Option, /// Threshold for active prefill tokens as fraction of max_num_batched_tokens active_prefill_tokens_threshold_frac: Option, + remote_kv_waiting_tokens_threshold: Option, session_affinity_ttl_secs: Option, session_affinity_mode: RsSessionAffinityMode, } @@ -488,7 +489,7 @@ pub struct RouterConfig { impl RouterConfig { #[new] #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (mode, config=None, active_decode_blocks_threshold=None, active_prefill_tokens_threshold=None, active_prefill_tokens_threshold_frac=None, enforce_disagg=false, session_affinity_ttl_secs=None, session_affinity_mode="hard"))] + #[pyo3(signature = (mode, config=None, active_decode_blocks_threshold=None, active_prefill_tokens_threshold=None, active_prefill_tokens_threshold_frac=None, enforce_disagg=false, session_affinity_ttl_secs=None, session_affinity_mode="hard", remote_kv_waiting_tokens_threshold=None))] pub fn new( mode: RouterMode, config: Option, @@ -498,6 +499,7 @@ impl RouterConfig { enforce_disagg: bool, session_affinity_ttl_secs: Option, session_affinity_mode: &str, + remote_kv_waiting_tokens_threshold: Option, ) -> PyResult { if enforce_disagg { static WARN_ONCE: std::sync::Once = std::sync::Once::new(); @@ -516,6 +518,7 @@ impl RouterConfig { active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold, } .validate() .map_err(PyValueError::new_err)?; @@ -528,6 +531,7 @@ impl RouterConfig { active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold, session_affinity_ttl_secs, session_affinity_mode, }) @@ -543,6 +547,7 @@ impl From for RsRouterConfig { active_decode_blocks_threshold: rc.active_decode_blocks_threshold, active_prefill_tokens_threshold: rc.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac: rc.active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold: rc.remote_kv_waiting_tokens_threshold, }, enforce_disagg: false, session_affinity_ttl_secs: rc.session_affinity_ttl_secs, diff --git a/lib/bindings/python/rust/llm/kv.rs b/lib/bindings/python/rust/llm/kv.rs index 91c86ce80878..fc3e85262af5 100644 --- a/lib/bindings/python/rust/llm/kv.rs +++ b/lib/bindings/python/rust/llm/kv.rs @@ -78,27 +78,32 @@ pub(crate) struct LoadThresholdConfig { active_prefill_tokens_threshold: Option, #[pyo3(get)] active_prefill_tokens_threshold_frac: Option, + #[pyo3(get)] + remote_kv_waiting_tokens_threshold: Option, } #[pymethods] impl LoadThresholdConfig { #[new] - #[pyo3(signature = (*, active_decode_blocks_threshold=None, active_prefill_tokens_threshold=None, active_prefill_tokens_threshold_frac=None))] + #[pyo3(signature = (*, active_decode_blocks_threshold=None, active_prefill_tokens_threshold=None, active_prefill_tokens_threshold_frac=None, remote_kv_waiting_tokens_threshold=None))] fn new( active_decode_blocks_threshold: Option, active_prefill_tokens_threshold: Option, active_prefill_tokens_threshold_frac: Option, + remote_kv_waiting_tokens_threshold: Option, ) -> PyResult { let config = validate_load_threshold_config( active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold, ) .map_err(PyValueError::new_err)?; Ok(Self { active_decode_blocks_threshold: config.active_decode_blocks_threshold, active_prefill_tokens_threshold: config.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac: config.active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold: config.remote_kv_waiting_tokens_threshold, }) } } @@ -109,6 +114,7 @@ impl LoadThresholdConfig { active_decode_blocks_threshold: self.active_decode_blocks_threshold, active_prefill_tokens_threshold: self.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac: self.active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold: self.remote_kv_waiting_tokens_threshold, } } } @@ -117,11 +123,13 @@ fn validate_load_threshold_config( active_decode_blocks_threshold: Option, active_prefill_tokens_threshold: Option, active_prefill_tokens_threshold_frac: Option, + remote_kv_waiting_tokens_threshold: Option, ) -> Result { let config = RsLoadThresholdConfig { active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold, }; config .validate() @@ -1828,6 +1836,7 @@ mod metric_worker_type_tests { active_decode_blocks_threshold: Some(0.8), active_prefill_tokens_threshold: Some(1024), active_prefill_tokens_threshold_frac: Some(0.5), + remote_kv_waiting_tokens_threshold: Some(8192), }; let (router, runtime) = standalone_encode_router( "python-standalone-encode-configured-load", @@ -1846,26 +1855,28 @@ mod load_threshold_config_tests { #[test] fn default_config_disables_overload_thresholds() { - let config = validate_load_threshold_config(None, None, None).unwrap(); + let config = validate_load_threshold_config(None, None, None, None).unwrap(); assert!(!config.is_configured()); } #[test] fn valid_config_preserves_all_thresholds() { - let config = validate_load_threshold_config(Some(0.75), Some(512), Some(0.5)).unwrap(); + let config = + validate_load_threshold_config(Some(0.75), Some(512), Some(0.5), Some(8192)).unwrap(); assert_eq!( config, RsLoadThresholdConfig { active_decode_blocks_threshold: Some(0.75), active_prefill_tokens_threshold: Some(512), active_prefill_tokens_threshold_frac: Some(0.5), + remote_kv_waiting_tokens_threshold: Some(8192), } ); } #[test] fn invalid_config_returns_validation_error() { - let error = validate_load_threshold_config(Some(1.1), None, None).unwrap_err(); + let error = validate_load_threshold_config(Some(1.1), None, None, None).unwrap_err(); assert!(error.contains( "invalid load threshold config: active_decode_blocks_threshold must be between 0.0 and 1.0" )); diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index 50706e497dfb..9ca914fadfab 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -1732,6 +1732,7 @@ class RouterConfig: enforce_disagg: bool = False, session_affinity_ttl_secs: Optional[int] = None, session_affinity_mode: str = "hard", + remote_kv_waiting_tokens_threshold: Optional[int] = None, ) -> None: """ Create a RouterConfig. @@ -1745,6 +1746,7 @@ class RouterConfig: enforce_disagg: Deprecated and ignored. Routing topology and readiness come from registered worker types. session_affinity_ttl_secs: Router-local session-affinity idle TTL in seconds. session_affinity_mode: Session binding behavior: ``hard`` or ``soft``. + remote_kv_waiting_tokens_threshold: Transfer-backlog token threshold for WAITING_FOR_REMOTE_KVS """ ... @@ -1754,6 +1756,7 @@ class LoadThresholdConfig: active_decode_blocks_threshold: Optional[float] active_prefill_tokens_threshold: Optional[int] active_prefill_tokens_threshold_frac: Optional[float] + remote_kv_waiting_tokens_threshold: Optional[int] def __init__( self, @@ -1761,6 +1764,7 @@ class LoadThresholdConfig: active_decode_blocks_threshold: Optional[float] = None, active_prefill_tokens_threshold: Optional[int] = None, active_prefill_tokens_threshold_frac: Optional[float] = None, + remote_kv_waiting_tokens_threshold: Optional[int] = None, ) -> None: ... diff --git a/lib/kv-router/src/protocols.rs b/lib/kv-router/src/protocols.rs index a8e96d3b0189..d2960d1fd873 100644 --- a/lib/kv-router/src/protocols.rs +++ b/lib/kv-router/src/protocols.rs @@ -1107,6 +1107,13 @@ pub struct ActiveLoad { pub active_decode_blocks: Option, /// Number of active prefill tokens (from scheduler's view). pub active_prefill_tokens: Option, + /// Number of requests whose external KV transfer is still in flight. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remote_kv_waiting_requests: Option, + /// Tokens currently waiting for external KV transfer completion. + /// This is transfer backlog and must not be interpreted as prefill compute. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remote_kv_waiting_tokens: Option, /// Total KV blocks currently in use on the worker. /// /// This is published by workers only and is the authoritative signal for diff --git a/lib/kv-router/src/scheduling/queue.rs b/lib/kv-router/src/scheduling/queue.rs index 6d64eec2848a..4843f0d3c1c0 100644 --- a/lib/kv-router/src/scheduling/queue.rs +++ b/lib/kv-router/src/scheduling/queue.rs @@ -44,6 +44,16 @@ pub const DEFAULT_MAX_BATCHED_TOKENS: u64 = 10_000_000; const ADMISSION_CHANNEL_CAPACITY: usize = 65_536; +fn selected_cached_tokens( + isl_tokens: usize, + selected_cached_blocks: u32, + block_size: u32, +) -> usize { + (selected_cached_blocks as usize) + .saturating_mul(block_size as usize) + .min(isl_tokens) +} + struct ClassQueueCounters { pending_count: AtomicUsize, pending_isl_tokens: AtomicUsize, @@ -1411,6 +1421,7 @@ impl< let target_cached_prefix_blocks = target_cached_prefix_blocks(&request, selected.selection.worker); + let selected_cached_blocks = selected.selected_worker_tiers.disk_blocks; let response = SchedulingResponse { best_worker: selected.selection.worker, effective_overlap_blocks: selected.selection.effective_overlap_blocks, @@ -1435,7 +1446,7 @@ impl< let prefill_load_hint = self.prefill_load_hint_for( request.isl_tokens, - selected.selection.cached_tokens, + selected_cached_blocks, request.track_prefill_tokens, ); @@ -1537,13 +1548,20 @@ impl< fn prefill_load_hint_for( &self, isl_tokens: usize, - cached_tokens: usize, + selected_cached_blocks: u32, track_prefill_tokens: bool, ) -> Option { if !track_prefill_tokens { return None; } + // Selection deliberately discounts lower-tier hits because transfer is + // slower than a device hit. Do not carry that discount into active + // prefill accounting: host/disk-resident tokens are transfer work, not + // GPU prefill compute. Remote-transfer pressure is reported separately + // by the worker from WAITING_FOR_REMOTE_KVS. + let cached_tokens = + selected_cached_tokens(isl_tokens, selected_cached_blocks, self.block_size); let effective_isl = effective_prefill_tokens(isl_tokens, cached_tokens); if effective_isl == 0 { return None; @@ -2250,6 +2268,12 @@ mod tests { (req, rx) } + #[test] + fn prefill_load_hint_excludes_all_selected_cached_tiers_from_compute() { + let cached_tokens = selected_cached_tokens(1_024, 48, 16); + assert_eq!(effective_prefill_tokens(1_024, cached_tokens), 256); + } + #[test] fn non_max_overlap_selection_ignores_ties_pins_and_ineligible_workers() { let workers = HashMap::from([ diff --git a/lib/llm/src/discovery/worker_monitor.rs b/lib/llm/src/discovery/worker_monitor.rs index 0e582d2f53a4..ec654f95df9e 100644 --- a/lib/llm/src/discovery/worker_monitor.rs +++ b/lib/llm/src/discovery/worker_monitor.rs @@ -41,6 +41,8 @@ fn cleanup_worker_metrics(worker_id: u64, dp_ranks: &[u32], worker_type: &str) { let labels = &[worker_id_str.as_str(), dp_rank_str.as_str(), worker_type]; let _ = m.active_decode_blocks.remove_label_values(labels); let _ = m.active_prefill_tokens.remove_label_values(labels); + let _ = m.remote_kv_waiting_requests.remove_label_values(labels); + let _ = m.remote_kv_waiting_tokens.remove_label_values(labels); let _ = WORKER_LAST_TIME_TO_FIRST_TOKEN_GAUGE.remove_label_values(labels); let _ = WORKER_LAST_INPUT_SEQUENCE_TOKENS_GAUGE.remove_label_values(labels); let _ = WORKER_LAST_INTER_TOKEN_LATENCY_GAUGE.remove_label_values(labels); @@ -110,6 +112,11 @@ pub struct LoadThresholdConfig { /// Worker is overloaded when `active_prefill_tokens > frac * max_num_batched_tokens`. #[serde(skip_serializing_if = "Option::is_none")] pub active_prefill_tokens_threshold_frac: Option, + + /// Absolute external-KV transfer backlog threshold. A worker is busy when + /// tokens in `WAITING_FOR_REMOTE_KVS` exceed this value. + #[serde(skip_serializing_if = "Option::is_none")] + pub remote_kv_waiting_tokens_threshold: Option, } impl LoadThresholdConfig { @@ -118,6 +125,7 @@ impl LoadThresholdConfig { self.active_decode_blocks_threshold.is_some() || self.active_prefill_tokens_threshold.is_some() || self.active_prefill_tokens_threshold_frac.is_some() + || self.remote_kv_waiting_tokens_threshold.is_some() } /// Validate threshold values shared by startup and dynamic configuration. @@ -166,6 +174,9 @@ impl LoadThresholdHandle { if let Some(value) = config.active_prefill_tokens_threshold_frac { current.active_prefill_tokens_threshold_frac = Some(value); } + if let Some(value) = config.remote_kv_waiting_tokens_threshold { + current.remote_kv_waiting_tokens_threshold = Some(value); + } } pub fn is_configured(&self) -> bool { @@ -197,6 +208,8 @@ struct RemoteActiveLoadSnapshot { active_decode_blocks: Option, active_prefill_tokens: Option, kv_used_blocks: Option, + remote_kv_waiting_requests: Option, + remote_kv_waiting_tokens: Option, } impl From for RemoteActiveLoadSnapshot { @@ -209,6 +222,8 @@ impl From for RemoteActiveLoadSnapshot { active_decode_blocks: load.active_decode_blocks, active_prefill_tokens: load.active_prefill_tokens, kv_used_blocks: load.kv_used_blocks, + remote_kv_waiting_requests: load.remote_kv_waiting_requests, + remote_kv_waiting_tokens: load.remote_kv_waiting_tokens, } } } @@ -227,6 +242,8 @@ impl LoadObservation { Option, Option, Option, + Option, + Option, ) { match self { Self::Scheduler(snapshot) => ( @@ -234,12 +251,16 @@ impl LoadObservation { Some(snapshot.active_decode_blocks), Some(snapshot.active_prefill_tokens), None, + None, + None, ), Self::Remote(snapshot) => ( snapshot.worker, snapshot.active_decode_blocks, snapshot.active_prefill_tokens, snapshot.kv_used_blocks, + snapshot.remote_kv_waiting_requests, + snapshot.remote_kv_waiting_tokens, ), } } @@ -251,6 +272,7 @@ pub struct WorkerLoadState { pub kv_used_blocks: HashMap, pub kv_total_blocks: HashMap, pub active_prefill_tokens: HashMap, + pub remote_kv_waiting_tokens: HashMap, /// max_num_batched_tokens from runtime config (same for all dp_ranks) pub max_num_batched_tokens: HashMap, /// The current router-visible ranks declared by this worker's runtime config. @@ -276,6 +298,8 @@ impl WorkerLoadState { .retain(|dp_rank, _| declared_dp_ranks.contains(dp_rank)); self.active_prefill_tokens .retain(|dp_rank, _| declared_dp_ranks.contains(dp_rank)); + self.remote_kv_waiting_tokens + .retain(|dp_rank, _| declared_dp_ranks.contains(dp_rank)); self.kv_total_blocks.clear(); if let Some(total_blocks) = total_kv_blocks { @@ -408,8 +432,14 @@ impl WorkerLoadState { observation: LoadObservation, active_decode_blocks_threshold: Option, ) -> bool { - let (worker, active_decode_blocks, active_prefill_tokens, kv_used_blocks) = - observation.parts(); + let ( + worker, + active_decode_blocks, + active_prefill_tokens, + kv_used_blocks, + _, + remote_kv_waiting_tokens, + ) = observation.parts(); let dp_rank = worker.dp_rank; if !self.accepts_dp_rank(dp_rank) { return false; @@ -424,6 +454,10 @@ impl WorkerLoadState { if let Some(active_tokens) = active_prefill_tokens { self.active_prefill_tokens.insert(dp_rank, active_tokens); } + if let Some(waiting_tokens) = remote_kv_waiting_tokens { + self.remote_kv_waiting_tokens + .insert(dp_rank, waiting_tokens); + } if let Some(threshold) = active_decode_blocks_threshold { self.update_decode_overload_latch( dp_rank, @@ -451,24 +485,42 @@ impl WorkerLoadState { /// /// Each threshold is `Option`. A `None` threshold means that check is /// skipped entirely — it cannot contribute to a dp_rank being overloaded. If all - /// three thresholds are `None`, no dp_rank is ever overloaded. + /// four thresholds are `None`, no dp_rank is ever overloaded. /// /// For each dp_rank, a dp_rank is overloaded if ANY of these conditions is met (OR logic): /// 1. `active_prefill_tokens > active_prefill_tokens_threshold` (absolute, if set) /// 2. `active_prefill_tokens > frac * max_num_batched_tokens` (fractional, if set) /// 3. decode overload latch set by either `kv_used_blocks` or `active_decode_blocks` (if set) + /// 4. `remote_kv_waiting_tokens > remote_kv_waiting_tokens_threshold` (if set) /// /// The worker is overloaded only if ALL dp_ranks are overloaded. + #[cfg(test)] pub fn is_overloaded( &self, active_decode_blocks_threshold: Option, active_prefill_tokens_threshold: Option, active_prefill_tokens_threshold_frac: Option, + ) -> bool { + self.is_overloaded_with_remote( + active_decode_blocks_threshold, + active_prefill_tokens_threshold, + active_prefill_tokens_threshold_frac, + None, + ) + } + + fn is_overloaded_with_remote( + &self, + active_decode_blocks_threshold: Option, + active_prefill_tokens_threshold: Option, + active_prefill_tokens_threshold_frac: Option, + remote_kv_waiting_tokens_threshold: Option, ) -> bool { // Short-circuit if all thresholds are unset (i.e. no overload check can fire) if active_decode_blocks_threshold.is_none() && active_prefill_tokens_threshold.is_none() && active_prefill_tokens_threshold_frac.is_none() + && remote_kv_waiting_tokens_threshold.is_none() { return false; } @@ -486,6 +538,7 @@ impl WorkerLoadState { .chain(self.kv_used_blocks.keys()) .chain(self.decode_overload_latches.keys()) .chain(self.active_prefill_tokens.keys()) + .chain(self.remote_kv_waiting_tokens.keys()) .copied() .collect(); &fallback_dp_ranks @@ -533,16 +586,29 @@ impl WorkerLoadState { } } + // Check 4: external-KV transfer backlog. This is intentionally + // independent of prefill compute pressure: prefix affinity keeps + // winning until the transfer queue crosses an explicit limit. + if let Some(threshold) = remote_kv_waiting_tokens_threshold + && self + .remote_kv_waiting_tokens + .get(&dp_rank) + .is_some_and(|&tokens| tokens > threshold) + { + return true; + } + // If we can't perform any check or no threshold exceeded, this dp_rank is free false }) } fn is_overloaded_for_config(&self, config: &LoadThresholdConfig) -> bool { - self.is_overloaded( + self.is_overloaded_with_remote( config.active_decode_blocks_threshold, config.active_prefill_tokens_threshold, config.active_prefill_tokens_threshold_frac, + config.remote_kv_waiting_tokens_threshold, ) } } @@ -937,7 +1003,8 @@ impl WorkerLoadMonitor for KvWorkerMonitor { let observation = LoadObservation::Remote(RemoteActiveLoadSnapshot::from(active_load)); - let (worker, _, _, _) = observation.parts(); + let (worker, _, _, _, remote_waiting_requests, remote_waiting_tokens) = + observation.parts(); if !known_workers.contains(&worker.worker_id) { tracing::debug!( worker_id = worker.worker_id, @@ -966,6 +1033,14 @@ impl WorkerLoadMonitor for KvWorkerMonitor { .or_default() .insert(worker.dp_rank); + WORKER_LOAD_METRICS.observe_remote_kv_wait( + worker.worker_id, + worker.dp_rank, + source.metric_label(), + remote_waiting_requests, + remote_waiting_tokens, + ); + // Snapshot thresholds once per event — rare writes (HTTP endpoint) // mean RwLock contention is effectively zero. let cfg = thresholds.get(); @@ -993,6 +1068,7 @@ impl WorkerLoadMonitor for KvWorkerMonitor { active_decode_blocks_threshold = ?cfg.active_decode_blocks_threshold, active_prefill_tokens_threshold = ?cfg.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac = ?cfg.active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold = ?cfg.remote_kv_waiting_tokens_threshold, worker_overloaded, "processed active load update" ); @@ -1172,6 +1248,8 @@ mod tests { active_decode_blocks: Some(10), active_prefill_tokens: Some(100), kv_used_blocks: None, + remote_kv_waiting_requests: None, + remote_kv_waiting_tokens: None, }); let mut local_then_remote = WorkerLoadState::default(); @@ -1197,6 +1275,8 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(8), + remote_kv_waiting_requests: None, + remote_kv_waiting_tokens: None, }), None, ); @@ -1208,6 +1288,8 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: Some(0), kv_used_blocks: None, + remote_kv_waiting_requests: None, + remote_kv_waiting_tokens: None, }), None, ); @@ -1320,6 +1402,7 @@ mod tests { active_decode_blocks_threshold: Some(0.85), active_prefill_tokens_threshold: Some(10_000), active_prefill_tokens_threshold_frac: Some(0.9), + remote_kv_waiting_tokens_threshold: Some(100_000), }; assert!(config.is_configured()); } @@ -1364,6 +1447,7 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(90), + ..Default::default() }, Some(0.6), ); @@ -1376,6 +1460,7 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(90), + ..Default::default() }, Some(0.6), ); @@ -1395,6 +1480,7 @@ mod tests { active_decode_blocks: Some(90), active_prefill_tokens: Some(900), kv_used_blocks: Some(90), + ..Default::default() }, Some(0.6), ); @@ -1417,6 +1503,8 @@ mod tests { active_decode_blocks: Some(100), active_prefill_tokens: Some(1_000), kv_used_blocks: Some(100), + remote_kv_waiting_requests: None, + remote_kv_waiting_tokens: None, }), Some(0.6), )); @@ -1440,6 +1528,7 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(90), + ..Default::default() }, Some(0.6), ); @@ -1459,6 +1548,7 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(90), + ..Default::default() }, Some(0.6), ); @@ -1471,6 +1561,7 @@ mod tests { active_decode_blocks: Some(10), active_prefill_tokens: None, kv_used_blocks: None, + ..Default::default() }, Some(0.6), ); @@ -1483,6 +1574,7 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(10), + ..Default::default() }, Some(0.6), ); @@ -1501,6 +1593,7 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(90), + ..Default::default() }, Some(0.6), ); @@ -1513,6 +1606,7 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(10), + ..Default::default() }, Some(0.6), ); @@ -1531,6 +1625,7 @@ mod tests { active_decode_blocks: Some(90), active_prefill_tokens: None, kv_used_blocks: None, + ..Default::default() }, Some(0.6), ); @@ -1543,6 +1638,7 @@ mod tests { active_decode_blocks: Some(10), active_prefill_tokens: None, kv_used_blocks: None, + ..Default::default() }, Some(0.6), ); @@ -1561,6 +1657,7 @@ mod tests { active_decode_blocks: Some(90), active_prefill_tokens: None, kv_used_blocks: None, + ..Default::default() }, Some(0.6), ); @@ -1573,6 +1670,7 @@ mod tests { active_decode_blocks: Some(10), active_prefill_tokens: None, kv_used_blocks: Some(10), + ..Default::default() }, Some(0.6), ); @@ -1611,6 +1709,7 @@ mod tests { active_decode_blocks: Some(90), active_prefill_tokens: None, kv_used_blocks: Some(90), + ..Default::default() }, Some(0.6), ); @@ -1629,6 +1728,7 @@ mod tests { active_decode_blocks: Some(90), active_prefill_tokens: None, kv_used_blocks: Some(90), + ..Default::default() }, Some(0.6), ); @@ -1653,6 +1753,16 @@ mod tests { assert!(state.is_overloaded(None, None, Some(2.0))); } + #[test] + fn remote_kv_wait_keeps_affinity_until_explicit_threshold_is_exceeded() { + let mut state = WorkerLoadState::default(); + state.remote_kv_waiting_tokens.insert(0, 8_192); + + assert!(!state.is_overloaded_with_remote(None, None, None, Some(8_192))); + assert!(state.is_overloaded_with_remote(None, None, None, Some(8_191))); + assert!(!state.is_overloaded(None, None, None)); + } + #[test] fn compute_overloaded_instances_flags_prefill_workers_over_token_threshold() { use dashmap::DashMap; diff --git a/lib/llm/src/fpm_publisher.rs b/lib/llm/src/fpm_publisher.rs index 2485de74764c..49cf3ffd8096 100644 --- a/lib/llm/src/fpm_publisher.rs +++ b/lib/llm/src/fpm_publisher.rs @@ -16,7 +16,7 @@ use std::time::Duration; use anyhow::Result; use futures::StreamExt; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; @@ -25,6 +25,10 @@ use dynamo_runtime::component::{Component, Endpoint}; use dynamo_runtime::traits::DistributedRuntimeProvider; use dynamo_runtime::transports::event_plane::EventPublisher; +use dynamo_kv_router::protocols::ActiveLoad; + +use crate::kv_router::KV_METRICS_SUBJECT; + use crate::utils::zmq::{connect_sub_socket, multipart_message}; const FPM_TOPIC: &str = "forward-pass-metrics"; @@ -117,9 +121,12 @@ impl FpmEventRelay { let publisher = rt.block_on(async { EventPublisher::for_endpoint(&endpoint, FPM_TOPIC).await })?; + let load_publisher = rt.block_on(async { + EventPublisher::for_endpoint(&endpoint, KV_METRICS_SUBJECT).await + })?; rt.spawn(async move { - Self::relay_loop(zmq_endpoint, publisher, cancel_clone, trace).await; + Self::relay_loop(zmq_endpoint, publisher, load_publisher, cancel_clone, trace).await; }); Ok(Self { cancel }) @@ -133,6 +140,7 @@ impl FpmEventRelay { async fn relay_loop( zmq_endpoint: String, publisher: EventPublisher, + load_publisher: EventPublisher, cancel: CancellationToken, trace: Option, ) { @@ -145,6 +153,7 @@ impl FpmEventRelay { }; let mut socket = socket; tracing::info!("FPM relay: connected to {zmq_endpoint}"); + let mut last_remote_kv_load = None; loop { tokio::select! { @@ -161,6 +170,26 @@ impl FpmEventRelay { if frames.len() == 3 { let payload = bytes::Bytes::from(frames.swap_remove(2)); tap_relay_fpm(&payload, trace.as_ref()); + match remote_kv_active_load(&payload) { + Ok(Some(load)) => { + let snapshot = ( + load.remote_kv_waiting_requests.unwrap_or_default(), + load.remote_kv_waiting_tokens.unwrap_or_default(), + ); + if last_remote_kv_load != Some(snapshot) { + match load_publisher.publish(&load).await { + Ok(()) => last_remote_kv_load = Some(snapshot), + Err(error) => { + tracing::warn!(%error, "FPM relay: remote KV load publish failed"); + } + } + } + } + Ok(None) => {} + Err(error) => { + tracing::warn!(%error, "FPM relay: failed to decode remote KV load telemetry"); + } + } if let Err(e) = publisher.publish_bytes_ref(&payload).await { tracing::warn!("FPM relay: event plane publish failed: {e}"); } @@ -186,6 +215,43 @@ impl FpmEventRelay { } } +#[derive(Debug, Default, Deserialize)] +struct QueuedRemoteKvTelemetry { + #[serde(default)] + num_remote_kv_waiting_requests: u64, + #[serde(default)] + sum_remote_kv_waiting_tokens: u64, +} + +#[derive(Debug, Deserialize)] +struct ForwardPassRemoteKvTelemetry { + worker_id: String, + dp_rank: u32, + #[serde(default)] + queued_requests: QueuedRemoteKvTelemetry, +} + +fn remote_kv_active_load(payload: &[u8]) -> Result> { + let telemetry: ForwardPassRemoteKvTelemetry = rmp_serde::from_slice(payload)?; + let worker_id = match telemetry.worker_id.parse::() { + Ok(worker_id) => worker_id, + Err(_) if telemetry.worker_id.is_empty() => return Ok(None), + Err(error) => { + return Err(anyhow::anyhow!( + "invalid worker_id {:?}: {error}", + telemetry.worker_id + )); + } + }; + Ok(Some(ActiveLoad { + worker_id, + dp_rank: telemetry.dp_rank, + remote_kv_waiting_requests: Some(telemetry.queued_requests.num_remote_kv_waiting_requests), + remote_kv_waiting_tokens: Some(telemetry.queued_requests.sum_remote_kv_waiting_tokens), + ..ActiveLoad::default() + })) +} + impl Drop for FpmEventRelay { fn drop(&mut self) { self.cancel.cancel(); @@ -220,6 +286,8 @@ struct QueuedRequestMetricsSer { num_decode_requests: i32, sum_decode_kv_tokens: i64, var_decode_kv_tokens: f64, + num_remote_kv_waiting_requests: i32, + sum_remote_kv_waiting_tokens: i64, } /// Top-level serialization struct matching Python `ForwardPassMetrics`. @@ -264,6 +332,8 @@ fn serialize_fpm_into( num_decode_requests: snapshot.num_queued_decode as i32, sum_decode_kv_tokens: snapshot.sum_queued_decode_kv_tokens as i64, var_decode_kv_tokens: snapshot.var_queued_decode_kv_tokens, + num_remote_kv_waiting_requests: 0, + sum_remote_kv_waiting_tokens: 0, }, }; metrics @@ -514,6 +584,8 @@ mod tests { num_decode_requests: i32, sum_decode_kv_tokens: i64, var_decode_kv_tokens: f64, + num_remote_kv_waiting_requests: i32, + sum_remote_kv_waiting_tokens: i64, } #[derive(Deserialize, Debug)] #[allow(dead_code)] @@ -545,6 +617,8 @@ mod tests { assert_eq!(decoded.queued_requests.num_prefill_requests, 1); assert_eq!(decoded.queued_requests.sum_prefill_tokens, 128); assert_eq!(decoded.queued_requests.num_decode_requests, 0); + assert_eq!(decoded.queued_requests.num_remote_kv_waiting_requests, 0); + assert_eq!(decoded.queued_requests.sum_remote_kv_waiting_tokens, 0); } #[test] @@ -608,7 +682,7 @@ mod tests { } /// Verify all 7 expected field names appear in scheduled_requests and - /// 6 in queued_requests — matching the Python schema exactly. + /// 8 in queued_requests — matching the Python schema exactly. #[test] fn test_serialize_fpm_field_names() { let snapshot = ForwardPassSnapshot::default(); @@ -650,6 +724,8 @@ mod tests { "num_decode_requests", "sum_decode_kv_tokens", "var_decode_kv_tokens", + "num_remote_kv_waiting_requests", + "sum_remote_kv_waiting_tokens", ]; for key in &expected_queued { assert!( @@ -663,4 +739,38 @@ mod tests { "queued_requests has unexpected extra fields" ); } + + #[test] + fn remote_kv_active_load_decodes_transfer_backlog() { + let payload = rmp_serde::to_vec_named(&serde_json::json!({ + "worker_id": "17", + "dp_rank": 2, + "queued_requests": { + "num_remote_kv_waiting_requests": 3, + "sum_remote_kv_waiting_tokens": 24_576 + } + })) + .unwrap(); + + let load = remote_kv_active_load(&payload).unwrap().unwrap(); + assert_eq!(load.worker_id, 17); + assert_eq!(load.dp_rank, 2); + assert_eq!(load.remote_kv_waiting_requests, Some(3)); + assert_eq!(load.remote_kv_waiting_tokens, Some(24_576)); + assert_eq!(load.active_prefill_tokens, None); + } + + #[test] + fn remote_kv_active_load_defaults_legacy_fpm_to_zero() { + let payload = rmp_serde::to_vec_named(&serde_json::json!({ + "worker_id": "17", + "dp_rank": 0, + "queued_requests": {} + })) + .unwrap(); + + let load = remote_kv_active_load(&payload).unwrap().unwrap(); + assert_eq!(load.remote_kv_waiting_requests, Some(0)); + assert_eq!(load.remote_kv_waiting_tokens, Some(0)); + } } diff --git a/lib/llm/src/http/service/busy_threshold.rs b/lib/llm/src/http/service/busy_threshold.rs index f009c8b5ef78..2aa7d0203fc8 100644 --- a/lib/llm/src/http/service/busy_threshold.rs +++ b/lib/llm/src/http/service/busy_threshold.rs @@ -68,6 +68,8 @@ pub struct BusyThresholdRequest { pub active_prefill_tokens_threshold: Option, /// The active prefill tokens threshold as fraction of max_num_batched_tokens, or null to just get the current value pub active_prefill_tokens_threshold_frac: Option, + /// External-KV transfer backlog token threshold, or null to just get the current value + pub remote_kv_waiting_tokens_threshold: Option, } /// Response for a threshold operation @@ -81,6 +83,8 @@ pub struct BusyThresholdResponse { pub active_prefill_tokens_threshold: Option, /// The active prefill tokens threshold as fraction of max_num_batched_tokens pub active_prefill_tokens_threshold_frac: Option, + /// External-KV transfer backlog token threshold + pub remote_kv_waiting_tokens_threshold: Option, } /// Response for listing all thresholds @@ -147,6 +151,7 @@ async fn busy_threshold_handler( active_decode_blocks_threshold: request.active_decode_blocks_threshold, active_prefill_tokens_threshold: request.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac: request.active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold: request.remote_kv_waiting_tokens_threshold, }; if let Err(error) = requested_config.validate() { return ( @@ -189,11 +194,13 @@ async fn busy_threshold_handler( active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, - ) = config.map_or((None, None, None), |c| { + remote_kv_waiting_tokens_threshold, + ) = config.map_or((None, None, None, None), |c| { ( c.active_decode_blocks_threshold, c.active_prefill_tokens_threshold, c.active_prefill_tokens_threshold_frac, + c.remote_kv_waiting_tokens_threshold, ) }); @@ -204,6 +211,7 @@ async fn busy_threshold_handler( active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold, })), ) } @@ -222,6 +230,7 @@ async fn list_busy_thresholds_handler( active_decode_blocks_threshold: config.active_decode_blocks_threshold, active_prefill_tokens_threshold: config.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac: config.active_prefill_tokens_threshold_frac, + remote_kv_waiting_tokens_threshold: config.remote_kv_waiting_tokens_threshold, }) .collect(), }; diff --git a/lib/llm/src/kv_router/metrics.rs b/lib/llm/src/kv_router/metrics.rs index cbce00c50b49..44367b7e4e60 100644 --- a/lib/llm/src/kv_router/metrics.rs +++ b/lib/llm/src/kv_router/metrics.rs @@ -5,7 +5,7 @@ //! //! This module centralizes all router-side Prometheus metric definitions: //! -//! - [`WorkerLoadMetrics`]: Per-worker active decode blocks and prefill tokens gauges. +//! - [`WorkerLoadMetrics`]: Per-worker active decode, prefill, and remote-KV-wait gauges. //! Registered on the frontend's own `prometheus::Registry` (default port 8000). //! Populated by `KvWorkerMonitor` in the frontend when receiving ActiveLoad events. //! - Frontend (aggregated and disaggregated): available on default port 8000 @@ -490,10 +490,12 @@ impl RouterWorkerStatusMetrics { // --------------------------------------------------------------------------- /// Per-worker active load gauges, published by `ActiveSequencesMultiWorker` -/// and cleaned up by `KvWorkerMonitor` when workers disappear. +/// and `KvWorkerMonitor`, then cleaned up by the monitor when workers disappear. pub struct WorkerLoadMetrics { pub active_decode_blocks: IntGaugeVec, pub active_prefill_tokens: IntGaugeVec, + pub remote_kv_waiting_requests: IntGaugeVec, + pub remote_kv_waiting_tokens: IntGaugeVec, } impl WorkerLoadMetrics { @@ -515,6 +517,29 @@ impl WorkerLoadMetrics { .with_label_values(labels) .set(active_tokens as i64); } + + pub fn observe_remote_kv_wait( + &self, + worker_id: u64, + dp_rank: u32, + worker_type: &str, + waiting_requests: Option, + waiting_tokens: Option, + ) { + let worker_id_str = worker_id.to_string(); + let dp_rank_str = dp_rank.to_string(); + let labels = &[worker_id_str.as_str(), dp_rank_str.as_str(), worker_type]; + if let Some(waiting_requests) = waiting_requests { + self.remote_kv_waiting_requests + .with_label_values(labels) + .set(waiting_requests as i64); + } + if let Some(waiting_tokens) = waiting_tokens { + self.remote_kv_waiting_tokens + .with_label_values(labels) + .set(waiting_tokens as i64); + } + } } pub static WORKER_LOAD_METRICS: LazyLock = LazyLock::new(|| WorkerLoadMetrics { @@ -542,6 +567,30 @@ pub static WORKER_LOAD_METRICS: LazyLock = LazyLock::new(|| W &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], ) .expect("Failed to create worker_active_prefill_tokens gauge"), + remote_kv_waiting_requests: IntGaugeVec::new( + Opts::new( + format!( + "{}_{}", + name_prefix::FRONTEND, + frontend_service::WORKER_REMOTE_KV_WAITING_REQUESTS + ), + "Requests waiting for an external KV transfer per worker", + ), + &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], + ) + .expect("Failed to create worker_remote_kv_waiting_requests gauge"), + remote_kv_waiting_tokens: IntGaugeVec::new( + Opts::new( + format!( + "{}_{}", + name_prefix::FRONTEND, + frontend_service::WORKER_REMOTE_KV_WAITING_TOKENS + ), + "KV context tokens waiting for an external transfer per worker", + ), + &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], + ) + .expect("Failed to create worker_remote_kv_waiting_tokens gauge"), }); /// Register the worker load gauges with the given Prometheus registry. @@ -552,6 +601,8 @@ pub fn register_worker_load_metrics( let m = &*WORKER_LOAD_METRICS; registry.register(Box::new(m.active_decode_blocks.clone()))?; registry.register(Box::new(m.active_prefill_tokens.clone()))?; + registry.register(Box::new(m.remote_kv_waiting_requests.clone()))?; + registry.register(Box::new(m.remote_kv_waiting_tokens.clone()))?; Ok(()) } @@ -1289,6 +1340,22 @@ mod tests { &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], ) .unwrap(), + remote_kv_waiting_requests: IntGaugeVec::new( + Opts::new( + "dynamo_frontend_worker_remote_kv_waiting_requests", + "Requests waiting for an external KV transfer per worker", + ), + &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], + ) + .unwrap(), + remote_kv_waiting_tokens: IntGaugeVec::new( + Opts::new( + "dynamo_frontend_worker_remote_kv_waiting_tokens", + "KV context tokens waiting for an external transfer per worker", + ), + &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], + ) + .unwrap(), }; registry .register(Box::new(metrics.active_decode_blocks.clone())) @@ -1296,8 +1363,15 @@ mod tests { registry .register(Box::new(metrics.active_prefill_tokens.clone())) .unwrap(); + registry + .register(Box::new(metrics.remote_kv_waiting_requests.clone())) + .unwrap(); + registry + .register(Box::new(metrics.remote_kv_waiting_tokens.clone())) + .unwrap(); metrics.observe(123, 0, "decode", 42, 100); + metrics.observe_remote_kv_wait(123, 0, "decode", Some(3), Some(24_576)); let output = gather_pef(®istry); let expected = "\ @@ -1307,6 +1381,12 @@ dynamo_frontend_worker_active_decode_blocks{dp_rank=\"0\",worker_id=\"123\",work # HELP dynamo_frontend_worker_active_prefill_tokens Active prefill tokens queued per worker # TYPE dynamo_frontend_worker_active_prefill_tokens gauge dynamo_frontend_worker_active_prefill_tokens{dp_rank=\"0\",worker_id=\"123\",worker_type=\"decode\"} 100 +# HELP dynamo_frontend_worker_remote_kv_waiting_requests Requests waiting for an external KV transfer per worker +# TYPE dynamo_frontend_worker_remote_kv_waiting_requests gauge +dynamo_frontend_worker_remote_kv_waiting_requests{dp_rank=\"0\",worker_id=\"123\",worker_type=\"decode\"} 3 +# HELP dynamo_frontend_worker_remote_kv_waiting_tokens KV context tokens waiting for an external transfer per worker +# TYPE dynamo_frontend_worker_remote_kv_waiting_tokens gauge +dynamo_frontend_worker_remote_kv_waiting_tokens{dp_rank=\"0\",worker_id=\"123\",worker_type=\"decode\"} 24576 "; assert_eq!( output, expected, diff --git a/lib/llm/src/kv_router/metrics_subscriber.rs b/lib/llm/src/kv_router/metrics_subscriber.rs index 47706c6452ff..d66a20f3b73f 100644 --- a/lib/llm/src/kv_router/metrics_subscriber.rs +++ b/lib/llm/src/kv_router/metrics_subscriber.rs @@ -97,6 +97,8 @@ impl PendingActiveLoads { dp_rank: _, active_decode_blocks, active_prefill_tokens, + remote_kv_waiting_requests, + remote_kv_waiting_tokens, kv_used_blocks, } = load; if active_decode_blocks.is_some() { @@ -105,6 +107,12 @@ impl PendingActiveLoads { if active_prefill_tokens.is_some() { pending.active_prefill_tokens = active_prefill_tokens; } + if remote_kv_waiting_requests.is_some() { + pending.remote_kv_waiting_requests = remote_kv_waiting_requests; + } + if remote_kv_waiting_tokens.is_some() { + pending.remote_kv_waiting_tokens = remote_kv_waiting_tokens; + } if kv_used_blocks.is_some() { pending.kv_used_blocks = kv_used_blocks; } @@ -341,6 +349,8 @@ mod tests { dp_rank, active_decode_blocks, active_prefill_tokens, + remote_kv_waiting_requests: None, + remote_kv_waiting_tokens: None, kv_used_blocks, } } diff --git a/lib/llm/src/kv_router/publisher/worker_metrics.rs b/lib/llm/src/kv_router/publisher/worker_metrics.rs index 1e6145257632..1b096eaee80f 100644 --- a/lib/llm/src/kv_router/publisher/worker_metrics.rs +++ b/lib/llm/src/kv_router/publisher/worker_metrics.rs @@ -184,6 +184,7 @@ impl WorkerMetricsPublisher { active_decode_blocks: metrics.active_decode_blocks, active_prefill_tokens: None, kv_used_blocks: metrics.kv_used_blocks, + ..ActiveLoad::default() }; if let Err(e) = sink.publish(active_load).await { diff --git a/lib/runtime/src/metrics/prometheus_names.rs b/lib/runtime/src/metrics/prometheus_names.rs index f06f43e36dc9..45795d60d09a 100644 --- a/lib/runtime/src/metrics/prometheus_names.rs +++ b/lib/runtime/src/metrics/prometheus_names.rs @@ -296,6 +296,12 @@ pub mod frontend_service { /// Gauge metric tracking current queued prefill tokens for each worker pub const WORKER_ACTIVE_PREFILL_TOKENS: &str = "worker_active_prefill_tokens"; + /// Requests waiting for external KV transfer completion per worker + pub const WORKER_REMOTE_KV_WAITING_REQUESTS: &str = "worker_remote_kv_waiting_requests"; + + /// KV context tokens waiting for external transfer completion per worker + pub const WORKER_REMOTE_KV_WAITING_TOKENS: &str = "worker_remote_kv_waiting_tokens"; + /// Last observed time to first token per worker (in seconds) /// Gauge metric tracking the most recent TTFT for each worker pub const WORKER_LAST_TIME_TO_FIRST_TOKEN_SECONDS: &str = From 2cb3b12f3d0d184b1f23af6099dc7b323809a84c Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Fri, 18 Sep 2026 09:40:36 -0700 Subject: [PATCH 07/18] fix(container): apply vLLM offloading hotfixes Signed-off-by: Shreyas Misra --- ...288_stop_final_sampled_token_offload.patch | 16 +++++ ...2_PR52771_fix_mtp_eagle_offload_hits.patch | 69 +++++++++++++++++++ container/templates/vllm_runtime.Dockerfile | 10 +++ 3 files changed, 95 insertions(+) create mode 100644 container/deps/vllm/patches/01_PR54288_stop_final_sampled_token_offload.patch create mode 100644 container/deps/vllm/patches/02_PR52771_fix_mtp_eagle_offload_hits.patch diff --git a/container/deps/vllm/patches/01_PR54288_stop_final_sampled_token_offload.patch b/container/deps/vllm/patches/01_PR54288_stop_final_sampled_token_offload.patch new file mode 100644 index 000000000000..d76ae9770524 --- /dev/null +++ b/container/deps/vllm/patches/01_PR54288_stop_final_sampled_token_offload.patch @@ -0,0 +1,16 @@ +diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +index da0aea06de5a432e98bcaeddf0b3a71a2f5b6795..b43df41f6761ca56e236353aa25b85e664c16af8 100644 +--- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py ++++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +@@ -1344,7 +1344,10 @@ class OffloadingConnectorScheduler: + if req.status is RequestStatus.FINISHED_ABORTED: + num_tokens_after_batch = req.num_computed_tokens + elif req.is_finished(): +- num_tokens_after_batch = req.num_tokens ++ # Clamp to the GPU prefix cache's commit point. The final sampled ++ # token's slot is never committed (under spec decode it holds a ++ # rejected draft's KV), so a block ending there must not be stored. ++ num_tokens_after_batch = max(req.num_prompt_tokens, req.num_tokens - 1) + else: + num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] + num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens diff --git a/container/deps/vllm/patches/02_PR52771_fix_mtp_eagle_offload_hits.patch b/container/deps/vllm/patches/02_PR52771_fix_mtp_eagle_offload_hits.patch new file mode 100644 index 000000000000..1f9bac312d80 --- /dev/null +++ b/container/deps/vllm/patches/02_PR52771_fix_mtp_eagle_offload_hits.patch @@ -0,0 +1,69 @@ +diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +index b43df41f6761ca56e236353aa25b85e664c16af8..2054b08da52a553261d6b9a79ef2223c87591139 100644 +--- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py ++++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +@@ -208,7 +208,21 @@ class SchedulerOffloadConfig(NamedTuple): + and vllm_config.speculative_config.use_eagle_block_drop() + ) + if use_eagle_block_drop and not eagle_groups: +- eagle_groups = set(range(len(kv_cache_config.kv_cache_groups))) ++ # No group is annotated as holding drafter layers. This happens ++ # for shared-group MTP models (e.g. Qwen3.5-style), whose drafter ++ # is a regular decoder layer merged into the target's ++ # full-attention group, and for separate-draft models that miss ++ # annotation. Treating every group as a drafter group here makes ++ # the volatile-tail pop apply to ALL groups, which can zero the ++ # whole request's offload hit whenever any group's servable ++ # window is a single chunk (issue #52735). Drafter KV served one ++ # chunk stale can only lower speculative acceptance rates — the ++ # target model verifies every draft — so fail toward serving. ++ logger.info_once( ++ "KV offloading: speculative decoding is enabled but no " ++ "KV-cache group is annotated as a drafter group; treating " ++ "all groups as non-draft for offloading." ++ ) + + if eagle_groups: + logger.info( +@@ -395,15 +409,19 @@ class RequestOffloadState: + accepted position may be rewritten after spec-token rejection. During + prefill the trailing chunk is stable (the draft input for a chunk's + last position is the next prompt token), so it is stored immediately. +- The exclusion must be applied consistently everywhere +- ``next_stored_chunk_idx`` is derived: otherwise the trailing chunk of +- each step is skipped on collection but jumped over by +- ``next_stored_chunk_idx``, so it is never re-considered and a +- permanent hole breaks prefix-reuse lookup. ++ Once the request has finished, no further spec-token rejection can ++ rewrite the tail, so the exclusion is lifted and the final chunk ++ becomes storable (issue #52735). The exclusion must be applied ++ consistently everywhere ``next_stored_chunk_idx`` is derived: ++ otherwise the trailing chunk of each step is skipped on collection but ++ jumped over by ``next_stored_chunk_idx``, so it is never re-considered ++ and a permanent hole breaks prefix-reuse lookup. ``is_finished`` is ++ monotonic, so the finish-time calls all see the lifted exclusion. + """ + num_chunks = num_offloadable_tokens // group_config.tokens_per_chunk + is_decoding = num_offloadable_tokens > self.req.num_prompt_tokens +- if group_config.is_eagle_group and is_decoding: ++ # Finished requests have no pending speculation. ++ if group_config.is_eagle_group and is_decoding and not self.req.is_finished(): + num_chunks = max(0, num_chunks - 1) + num_allocated_chunks = ( + len(group_state.block_ids) // self.config.blocks_per_chunk +@@ -761,9 +779,13 @@ class OffloadingConnectorScheduler: + ) + + # For eagle groups, query one extra chunk that will be popped. +- # We only need to increase the query size for sliding window groups. ++ # Widening applies to every group type: without it, the pop ++ # below shrinks max_hit_size_tokens past what was queried, ++ # which can push the confirmed boundary under a coarser ++ # sibling group's chunk granularity and zero the whole ++ # request's hit (issue #52735). + query_max = max_hit_size_tokens +- if is_eagle_unverified and sliding_window_size_in_chunks is not None: ++ if is_eagle_unverified: + query_max = min( + max_hit_size_tokens + tokens_per_chunk, + len(offload_keys) * tokens_per_chunk, diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 216be74892ee..072434f81e88 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -198,6 +198,16 @@ RUN --mount=type=cache,id=uv-root-{{ context.dynamo.uv_version }},target=/root/. "transformers==${TRANSFORMERS_VERSION}" \ "tokenizers==${TOKENIZERS_VERSION}" +{% if device == "cuda" %} +# Apply vLLM hotfixes to the installed package tree. Patches are applied with +# --fuzz=5 to tolerate minor line-number drift across nightly builds. +RUN --mount=type=bind,source=./container/deps/vllm/patches,target=/tmp/vllm_patches,readonly \ + SITE_PACKAGES="$(python3 -c 'import site; print(site.getsitepackages()[0])')" && \ + for p in /tmp/vllm_patches/*.patch; do \ + patch --batch --forward --fuzz=5 -p1 -d "${SITE_PACKAGES}" < "$p"; \ + done +{% endif %} + {% if device != "cuda" %} # NIXL meta package always tries to find a cuda-backend # https://github.com/ai-dynamo/nixl/blob/v1.1.0/src/bindings/python/nixl-meta/nixl/__init__.py From 81a496caafed9e1976dc10cab9db5ad7f62f9755 Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 22 Sep 2026 10:12:22 -0700 Subject: [PATCH 08/18] patches for functional issues Signed-off-by: Shreyas Misra --- ...nimax_m3_reasoning_structured_output.patch | 119 +++++++++++++ ...c_decode_structured_output_precommit.patch | 164 ++++++++++++++++++ .../06_minimax_m3_patch_embed_linear.patch | 29 ++++ 3 files changed, 312 insertions(+) create mode 100644 container/deps/vllm/patches/04_PR51230_minimax_m3_reasoning_structured_output.patch create mode 100644 container/deps/vllm/patches/05_PR43424_spec_decode_structured_output_precommit.patch create mode 100644 container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch diff --git a/container/deps/vllm/patches/04_PR51230_minimax_m3_reasoning_structured_output.patch b/container/deps/vllm/patches/04_PR51230_minimax_m3_reasoning_structured_output.patch new file mode 100644 index 000000000000..31a86d6472c1 --- /dev/null +++ b/container/deps/vllm/patches/04_PR51230_minimax_m3_reasoning_structured_output.patch @@ -0,0 +1,119 @@ +diff --git a/vllm/reasoning/minimax_m3_reasoning_parser.py b/vllm/reasoning/minimax_m3_reasoning_parser.py +index d49f9311f4..c86af61f5f 100644 +--- a/vllm/reasoning/minimax_m3_reasoning_parser.py ++++ b/vllm/reasoning/minimax_m3_reasoning_parser.py +@@ -46,6 +46,12 @@ class MiniMaxM3ReasoningParser(BaseThinkingReasoningParser): + self._pending_marker_streaming = False + self._last_streaming_delta_token_ids: tuple[int, ...] | None = None + self._last_streaming_content_token_ids: list[int] | None = None ++ # Incremental marker-scan cache for _reasoning_end_in_history. ++ # The parser is request-local and its token history is append-only, ++ # so each call only needs to scan tokens it has not seen yet. ++ self._history_scan_pos = 0 ++ self._history_last_start_pos = -1 ++ self._history_last_end_pos = -1 + + def _encode_text(self, text: str) -> list[int]: + try: +@@ -194,20 +200,96 @@ class MiniMaxM3ReasoningParser(BaseThinkingReasoningParser): + + return reasoning, (content_before + content_after) or None + ++ def _reasoning_end_in_history(self, input_ids: Sequence[int]) -> bool: ++ """Incremental equivalent of ``is_reasoning_end(input_ids)``. ++ ++ Tracks the last-seen start/end marker positions across calls, ++ scanning only tokens beyond the previous scan position (plus a ++ marker-length overlap for markers straddling the line). If the ++ sequence shrank since the last call - a new stream reusing the ++ parser - the cache resets and the scan restarts from zero. ++ ++ Cached positions are re-verified against the current tokens before ++ being trusted: the scheduler's bitmask path scans *simulated* ++ sequences that include scheduled speculative draft tokens, and a ++ draft that proposes a marker can be rejected by the verifier while ++ the same sequence length is later reached by different tokens ++ (accept-all-but-last plus bonus). A stale cached marker would then ++ end reasoning that never ended - engaging the grammar mid-thought ++ and leaving the response with empty content. Verification is ++ O(marker length); a mismatch triggers a full rescan. ++ """ ++ n = len(input_ids) ++ if n < self._history_scan_pos: ++ self._history_scan_pos = 0 ++ self._history_last_start_pos = -1 ++ self._history_last_end_pos = -1 ++ else: ++ for pos, marker in ( ++ (self._history_last_start_pos, self._start_token_ids), ++ (self._history_last_end_pos, self._end_token_ids), ++ ): ++ if pos >= 0 and ( ++ pos + len(marker) > n ++ or tuple(input_ids[pos : pos + len(marker)]) != marker ++ ): ++ self._history_scan_pos = 0 ++ self._history_last_start_pos = -1 ++ self._history_last_end_pos = -1 ++ break ++ overlap = max(len(self._start_token_ids), len(self._end_token_ids)) - 1 ++ begin = max(0, self._history_scan_pos - overlap) ++ window = input_ids[begin:n] if begin else input_ids ++ start_pos = self._rfind_token_sequence(window, self._start_token_ids) ++ if start_pos >= 0: ++ self._history_last_start_pos = max( ++ self._history_last_start_pos, begin + start_pos ++ ) ++ end_pos = self._rfind_token_sequence(window, self._end_token_ids) ++ if end_pos >= 0: ++ self._history_last_end_pos = max( ++ self._history_last_end_pos, begin + end_pos ++ ) ++ self._history_scan_pos = n ++ return ( ++ self._history_last_end_pos >= 0 ++ and self._history_last_end_pos > self._history_last_start_pos ++ ) ++ + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + if self._reasoning_ended_streaming: + return True + +- if self._reasoning_active_streaming or self._pending_marker_streaming: +- return False ++ # Check the token stream for the end marker before consulting the ++ # frontend streaming-state flags: the scheduler's structured-output ++ # gate (StructuredOutputManager.should_advance) calls this method on a ++ # request-local parser without ever calling ++ # extract_reasoning_streaming, which is the only place those flags are ++ # updated. With thinking_mode="enabled", ++ # ``_reasoning_active_streaming`` starts True and would otherwise gate ++ # detection off forever, so grammar constraints never engage ++ # (structured output is silently ignored). ++ # ++ # Ordering matters, not mere containment: ``input_ids`` spans prompt ++ # and output, and the chat template's block ++ # embeds literal marker text in the prompt. Reasoning has ended only ++ # when the last end marker follows the last start marker (the ++ # template's trailing ```` prefill keeps this False until ++ # the model really closes the block) - the same predicate as ++ # ``is_reasoning_end``, evaluated incrementally so per-token calls ++ # from the scheduler stay O(new tokens) instead of O(history). ++ if self._reasoning_end_in_history(input_ids): ++ return True + + delta_ids = tuple(delta_ids) + if self._contains_token_sequence(delta_ids, self._end_token_ids): + return True +- if self._contains_token_sequence(input_ids, self._end_token_ids): +- return True ++ ++ if self._reasoning_active_streaming or self._pending_marker_streaming: ++ return False ++ + if self._initial_in_reasoning: + return False + if self._ends_with_token_sequence_prefix(input_ids, self._start_token_ids): + diff --git a/container/deps/vllm/patches/05_PR43424_spec_decode_structured_output_precommit.patch b/container/deps/vllm/patches/05_PR43424_spec_decode_structured_output_precommit.patch new file mode 100644 index 000000000000..e73edf86840a --- /dev/null +++ b/container/deps/vllm/patches/05_PR43424_spec_decode_structured_output_precommit.patch @@ -0,0 +1,164 @@ +diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py +index d9d86668a6..0f2f00af9a 100644 +--- a/vllm/v1/core/sched/scheduler.py ++++ b/vllm/v1/core/sched/scheduler.py +@@ -1961,6 +1961,32 @@ class Scheduler(SchedulerInterface): + status_before_stop = request.status + num_output_tokens_before = len(request._output_token_ids) + ++ # Pre-commit grammar filter: when spec-decode × reasoning ++ # produces post-boundary bonus tokens that the verifier ++ # sampled WITHOUT the grammar mask engaged (the mask is ++ # gated on reasoning_ended, which is False until the ++ # boundary is observed — i.e. until *after* this step's ++ # tokens are sampled), drop the rejected trailing tokens ++ # before they are committed and fed to accept_tokens below. ++ # Mirrors the verifier-rejected-spec-token bookkeeping ++ # above: we decrement num_computed_tokens / ++ # num_output_placeholders by the rejection count, so the ++ # next step re-runs from the truncated position with the ++ # mask now correctly engaged. Stale outputs are skipped for ++ # the same reason as above: a stale rejection count ++ # predates the preemption rollback and must not apply. ++ if new_token_ids and not output_is_stale: ++ new_token_ids, num_grammar_rejected = ( ++ self.structured_output_manager.precommit_filter_tokens( ++ request, new_token_ids ++ ) ++ ) ++ if num_grammar_rejected > 0: ++ if request.num_computed_tokens > 0: ++ request.num_computed_tokens -= num_grammar_rejected ++ if request.num_output_placeholders > 0: ++ request.num_output_placeholders -= num_grammar_rejected ++ + # Check for stop and update request status. + if new_token_ids: + new_token_ids, stopped = self._update_request_with_output( +diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py +index d0ec080d84..6ea0213e53 100644 +--- a/vllm/v1/structured_output/__init__.py ++++ b/vllm/v1/structured_output/__init__.py +@@ -498,6 +498,121 @@ class StructuredOutputManager: + return new_token_ids + return new_token_ids[num_reasoning:] + ++ def _find_reasoning_end_in_tokens( ++ self, ++ reasoner: "ReasoningParser", ++ token_ids: list[int], ++ prior_token_ids: list[int], ++ ) -> int | None: ++ """Return the index in ``token_ids`` at which the reasoning-end ++ marker completes, or ``None`` if it does not complete inside this ++ batch. ++ ++ Scans progressively longer prefixes via the reasoner's ++ ``is_reasoning_end_streaming`` so that multi-token markers ++ (e.g. gpt-oss's ``<|channel|>final<|message|>``) which can ++ straddle the prior-context / current-batch line are still ++ detected. ``prior_token_ids`` are tokens already committed to ++ the request (passed by the caller from ``request.all_token_ids``). ++ ++ Unlike :meth:`_find_reasoning_end_index`, which conservatively ++ falls back to the final index, this returns ``None`` when the ++ marker does not complete in the batch — the pre-commit filter ++ must only act when the boundary is unambiguously inside ++ ``token_ids``. ++ """ ++ # Mutate a single growing buffer instead of rebuilding the prefix ++ # list each iteration: avoids O(L * N) allocation for long prior ++ # contexts and large speculative batches. ++ buf = list(prior_token_ids) ++ for i, token in enumerate(token_ids): ++ buf.append(token) ++ if reasoner.is_reasoning_end_streaming(buf, [token]): ++ return i ++ return None ++ ++ def precommit_filter_tokens( ++ self, ++ request: "Request", ++ new_token_ids: list[int], ++ ) -> tuple[list[int], int]: ++ """Pre-commit grammar filter for the reasoning-end boundary step. ++ ++ When reasoning ends mid-batch (i.e. the boundary marker ++ completes inside ``new_token_ids``), the verifier may have ++ sampled bonus tokens for positions past the boundary *without* ++ the grammar mask engaged: the mask is gated on ++ ``reasoning_ended`` (see :meth:`should_fill_bitmask`), and that ++ flag is only flipped to ``True`` after the boundary has been ++ observed — i.e. after this step's tokens were sampled. ++ ++ Without this filter, those grammar-invalid post-boundary bonus ++ tokens are committed to the request and then fed to ++ ``accept_tokens`` by the ``should_advance`` → ++ :meth:`trim_reasoning_for_advance` path in ++ ``Scheduler.update_from_output``, which terminates the request ++ with ``FINISHED_ERROR`` — a spec-decode-only failure mode that ++ grows with the speculative batch size and shows up most ++ prominently on reasoning models whose reasoning-end marker ++ spans multiple tokens (gpt-oss with the Harmony ++ ``openai_gptoss`` parser). ++ ++ Mechanism: dry-run grammar validation ++ (:meth:`StructuredOutputGrammar.validate_tokens` — stateless, ++ never advances the matcher) on the post-boundary slice, and ++ report the count of rejected trailing tokens. The caller ++ truncates ``new_token_ids`` and adjusts ++ ``num_computed_tokens`` / ``num_output_placeholders`` by the ++ rejected count (mirroring the existing verifier-rejected ++ spec-token bookkeeping), so the next step re-samples from the ++ boundary with the mask correctly engaged. The surviving tokens ++ are accepted into the matcher afterwards by the existing ++ post-commit advance path; this method never advances it. ++ ++ Returns ``(filtered_new_token_ids, num_rejected)``. When no ++ truncation is needed (no spec decode, no boundary in batch, or ++ all post-boundary tokens are grammar-valid), the original list ++ and ``0`` are returned. ++ """ ++ if self.vllm_config.speculative_config is None: ++ # Without spec decode there is at most one sampled token per ++ # step, so there are never post-boundary bonus tokens. ++ return new_token_ids, 0 ++ if not request.use_structured_output: ++ return new_token_ids, 0 ++ reasoner = self._get_reasoner(request) ++ if reasoner is None or self.enable_in_reasoning: ++ return new_token_ids, 0 ++ structured_req = request.structured_output_request ++ if structured_req is None or structured_req.grammar is None: ++ return new_token_ids, 0 ++ # Only act on the boundary step. Once reasoning has ended in a ++ # prior step the mask was correctly applied at sample time, and ++ # the post-commit accept path already handles validation. ++ if structured_req.reasoning_ended: ++ return new_token_ids, 0 ++ ++ # Pre-commit: request.all_token_ids does NOT yet include ++ # new_token_ids (we run before _update_request_with_output), so ++ # all_token_ids IS the prior context. ++ prior_token_ids = list(request.all_token_ids or []) ++ split_idx = self._find_reasoning_end_in_tokens( ++ reasoner, new_token_ids, prior_token_ids ++ ) ++ if split_idx is None: ++ return new_token_ids, 0 ++ ++ post_boundary = new_token_ids[split_idx + 1 :] ++ if not post_boundary: ++ return new_token_ids, 0 ++ ++ grammar = structured_req.grammar ++ validated = grammar.validate_tokens(post_boundary) ++ num_rejected = len(post_boundary) - len(validated) ++ if num_rejected == 0: ++ return new_token_ids, 0 ++ return new_token_ids[: split_idx + 1] + list(validated), num_rejected ++ + def clear_backend(self) -> None: + if self.backend is not None: + self.backend.destroy() + diff --git a/container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch b/container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch new file mode 100644 index 000000000000..fb5fb5c241d3 --- /dev/null +++ b/container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch @@ -0,0 +1,29 @@ +diff --git a/vllm/models/minimax_m3/common/vision_tower.py b/vllm/models/minimax_m3/common/vision_tower.py +--- a/vllm/models/minimax_m3/common/vision_tower.py ++++ b/vllm/models/minimax_m3/common/vision_tower.py +@@ -6,6 +6,7 @@ + import numpy as np + import torch + import torch.nn as nn ++import torch.nn.functional as F + from einops import rearrange + from transformers import PretrainedConfig + +@@ -69,9 +70,10 @@ class MiniMaxVLPatchEmbed(nn.Module): + self.patch_embedding = self.patch_embedding.to(pixel_values.dtype) +- x = pixel_values.reshape( +- pixel_values.shape[0], +- self.num_channels, +- self.temporal_patch_size, +- self.patch_size, +- self.patch_size, ++ # Each row is already exactly one non-overlapping Conv3d receptive ++ # field. The Conv3d output therefore has spatial shape (1, 1, 1) and ++ # is algebraically identical to this bias-free linear projection. ++ # Using GEMM also avoids lazy cuDNN initialization in the serving ++ # workers, which can fail while loading a cuDNN JIT module on GB200. ++ return F.linear( ++ pixel_values, ++ self.patch_embedding.weight.flatten(1), + ) +- return self.patch_embedding(x).reshape(x.shape[0], -1) From 24e42a66fb4bd8e786bae83fdabe51ff4b2fd304 Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 22 Sep 2026 10:18:57 -0700 Subject: [PATCH 09/18] remove F.linear patch Signed-off-by: Shreyas Misra --- .../06_minimax_m3_patch_embed_linear.patch | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch diff --git a/container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch b/container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch deleted file mode 100644 index fb5fb5c241d3..000000000000 --- a/container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch +++ /dev/null @@ -1,29 +0,0 @@ -diff --git a/vllm/models/minimax_m3/common/vision_tower.py b/vllm/models/minimax_m3/common/vision_tower.py ---- a/vllm/models/minimax_m3/common/vision_tower.py -+++ b/vllm/models/minimax_m3/common/vision_tower.py -@@ -6,6 +6,7 @@ - import numpy as np - import torch - import torch.nn as nn -+import torch.nn.functional as F - from einops import rearrange - from transformers import PretrainedConfig - -@@ -69,9 +70,10 @@ class MiniMaxVLPatchEmbed(nn.Module): - self.patch_embedding = self.patch_embedding.to(pixel_values.dtype) -- x = pixel_values.reshape( -- pixel_values.shape[0], -- self.num_channels, -- self.temporal_patch_size, -- self.patch_size, -- self.patch_size, -+ # Each row is already exactly one non-overlapping Conv3d receptive -+ # field. The Conv3d output therefore has spatial shape (1, 1, 1) and -+ # is algebraically identical to this bias-free linear projection. -+ # Using GEMM also avoids lazy cuDNN initialization in the serving -+ # workers, which can fail while loading a cuDNN JIT module on GB200. -+ return F.linear( -+ pixel_values, -+ self.patch_embedding.weight.flatten(1), - ) -- return self.patch_embedding(x).reshape(x.shape[0], -1) From 3a817e3537a908366cd638226eaf242348aa7344 Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Wed, 23 Sep 2026 15:12:59 -0700 Subject: [PATCH 10/18] finalize patches Signed-off-by: Shreyas Misra --- ...c_decode_structured_output_precommit.patch | 164 ------------------ .../06_minimax_m3_patch_embed_linear.patch | 29 ++++ 2 files changed, 29 insertions(+), 164 deletions(-) delete mode 100644 container/deps/vllm/patches/05_PR43424_spec_decode_structured_output_precommit.patch create mode 100644 container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch diff --git a/container/deps/vllm/patches/05_PR43424_spec_decode_structured_output_precommit.patch b/container/deps/vllm/patches/05_PR43424_spec_decode_structured_output_precommit.patch deleted file mode 100644 index e73edf86840a..000000000000 --- a/container/deps/vllm/patches/05_PR43424_spec_decode_structured_output_precommit.patch +++ /dev/null @@ -1,164 +0,0 @@ -diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py -index d9d86668a6..0f2f00af9a 100644 ---- a/vllm/v1/core/sched/scheduler.py -+++ b/vllm/v1/core/sched/scheduler.py -@@ -1961,6 +1961,32 @@ class Scheduler(SchedulerInterface): - status_before_stop = request.status - num_output_tokens_before = len(request._output_token_ids) - -+ # Pre-commit grammar filter: when spec-decode × reasoning -+ # produces post-boundary bonus tokens that the verifier -+ # sampled WITHOUT the grammar mask engaged (the mask is -+ # gated on reasoning_ended, which is False until the -+ # boundary is observed — i.e. until *after* this step's -+ # tokens are sampled), drop the rejected trailing tokens -+ # before they are committed and fed to accept_tokens below. -+ # Mirrors the verifier-rejected-spec-token bookkeeping -+ # above: we decrement num_computed_tokens / -+ # num_output_placeholders by the rejection count, so the -+ # next step re-runs from the truncated position with the -+ # mask now correctly engaged. Stale outputs are skipped for -+ # the same reason as above: a stale rejection count -+ # predates the preemption rollback and must not apply. -+ if new_token_ids and not output_is_stale: -+ new_token_ids, num_grammar_rejected = ( -+ self.structured_output_manager.precommit_filter_tokens( -+ request, new_token_ids -+ ) -+ ) -+ if num_grammar_rejected > 0: -+ if request.num_computed_tokens > 0: -+ request.num_computed_tokens -= num_grammar_rejected -+ if request.num_output_placeholders > 0: -+ request.num_output_placeholders -= num_grammar_rejected -+ - # Check for stop and update request status. - if new_token_ids: - new_token_ids, stopped = self._update_request_with_output( -diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py -index d0ec080d84..6ea0213e53 100644 ---- a/vllm/v1/structured_output/__init__.py -+++ b/vllm/v1/structured_output/__init__.py -@@ -498,6 +498,121 @@ class StructuredOutputManager: - return new_token_ids - return new_token_ids[num_reasoning:] - -+ def _find_reasoning_end_in_tokens( -+ self, -+ reasoner: "ReasoningParser", -+ token_ids: list[int], -+ prior_token_ids: list[int], -+ ) -> int | None: -+ """Return the index in ``token_ids`` at which the reasoning-end -+ marker completes, or ``None`` if it does not complete inside this -+ batch. -+ -+ Scans progressively longer prefixes via the reasoner's -+ ``is_reasoning_end_streaming`` so that multi-token markers -+ (e.g. gpt-oss's ``<|channel|>final<|message|>``) which can -+ straddle the prior-context / current-batch line are still -+ detected. ``prior_token_ids`` are tokens already committed to -+ the request (passed by the caller from ``request.all_token_ids``). -+ -+ Unlike :meth:`_find_reasoning_end_index`, which conservatively -+ falls back to the final index, this returns ``None`` when the -+ marker does not complete in the batch — the pre-commit filter -+ must only act when the boundary is unambiguously inside -+ ``token_ids``. -+ """ -+ # Mutate a single growing buffer instead of rebuilding the prefix -+ # list each iteration: avoids O(L * N) allocation for long prior -+ # contexts and large speculative batches. -+ buf = list(prior_token_ids) -+ for i, token in enumerate(token_ids): -+ buf.append(token) -+ if reasoner.is_reasoning_end_streaming(buf, [token]): -+ return i -+ return None -+ -+ def precommit_filter_tokens( -+ self, -+ request: "Request", -+ new_token_ids: list[int], -+ ) -> tuple[list[int], int]: -+ """Pre-commit grammar filter for the reasoning-end boundary step. -+ -+ When reasoning ends mid-batch (i.e. the boundary marker -+ completes inside ``new_token_ids``), the verifier may have -+ sampled bonus tokens for positions past the boundary *without* -+ the grammar mask engaged: the mask is gated on -+ ``reasoning_ended`` (see :meth:`should_fill_bitmask`), and that -+ flag is only flipped to ``True`` after the boundary has been -+ observed — i.e. after this step's tokens were sampled. -+ -+ Without this filter, those grammar-invalid post-boundary bonus -+ tokens are committed to the request and then fed to -+ ``accept_tokens`` by the ``should_advance`` → -+ :meth:`trim_reasoning_for_advance` path in -+ ``Scheduler.update_from_output``, which terminates the request -+ with ``FINISHED_ERROR`` — a spec-decode-only failure mode that -+ grows with the speculative batch size and shows up most -+ prominently on reasoning models whose reasoning-end marker -+ spans multiple tokens (gpt-oss with the Harmony -+ ``openai_gptoss`` parser). -+ -+ Mechanism: dry-run grammar validation -+ (:meth:`StructuredOutputGrammar.validate_tokens` — stateless, -+ never advances the matcher) on the post-boundary slice, and -+ report the count of rejected trailing tokens. The caller -+ truncates ``new_token_ids`` and adjusts -+ ``num_computed_tokens`` / ``num_output_placeholders`` by the -+ rejected count (mirroring the existing verifier-rejected -+ spec-token bookkeeping), so the next step re-samples from the -+ boundary with the mask correctly engaged. The surviving tokens -+ are accepted into the matcher afterwards by the existing -+ post-commit advance path; this method never advances it. -+ -+ Returns ``(filtered_new_token_ids, num_rejected)``. When no -+ truncation is needed (no spec decode, no boundary in batch, or -+ all post-boundary tokens are grammar-valid), the original list -+ and ``0`` are returned. -+ """ -+ if self.vllm_config.speculative_config is None: -+ # Without spec decode there is at most one sampled token per -+ # step, so there are never post-boundary bonus tokens. -+ return new_token_ids, 0 -+ if not request.use_structured_output: -+ return new_token_ids, 0 -+ reasoner = self._get_reasoner(request) -+ if reasoner is None or self.enable_in_reasoning: -+ return new_token_ids, 0 -+ structured_req = request.structured_output_request -+ if structured_req is None or structured_req.grammar is None: -+ return new_token_ids, 0 -+ # Only act on the boundary step. Once reasoning has ended in a -+ # prior step the mask was correctly applied at sample time, and -+ # the post-commit accept path already handles validation. -+ if structured_req.reasoning_ended: -+ return new_token_ids, 0 -+ -+ # Pre-commit: request.all_token_ids does NOT yet include -+ # new_token_ids (we run before _update_request_with_output), so -+ # all_token_ids IS the prior context. -+ prior_token_ids = list(request.all_token_ids or []) -+ split_idx = self._find_reasoning_end_in_tokens( -+ reasoner, new_token_ids, prior_token_ids -+ ) -+ if split_idx is None: -+ return new_token_ids, 0 -+ -+ post_boundary = new_token_ids[split_idx + 1 :] -+ if not post_boundary: -+ return new_token_ids, 0 -+ -+ grammar = structured_req.grammar -+ validated = grammar.validate_tokens(post_boundary) -+ num_rejected = len(post_boundary) - len(validated) -+ if num_rejected == 0: -+ return new_token_ids, 0 -+ return new_token_ids[: split_idx + 1] + list(validated), num_rejected -+ - def clear_backend(self) -> None: - if self.backend is not None: - self.backend.destroy() - diff --git a/container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch b/container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch new file mode 100644 index 000000000000..fb5fb5c241d3 --- /dev/null +++ b/container/deps/vllm/patches/06_minimax_m3_patch_embed_linear.patch @@ -0,0 +1,29 @@ +diff --git a/vllm/models/minimax_m3/common/vision_tower.py b/vllm/models/minimax_m3/common/vision_tower.py +--- a/vllm/models/minimax_m3/common/vision_tower.py ++++ b/vllm/models/minimax_m3/common/vision_tower.py +@@ -6,6 +6,7 @@ + import numpy as np + import torch + import torch.nn as nn ++import torch.nn.functional as F + from einops import rearrange + from transformers import PretrainedConfig + +@@ -69,9 +70,10 @@ class MiniMaxVLPatchEmbed(nn.Module): + self.patch_embedding = self.patch_embedding.to(pixel_values.dtype) +- x = pixel_values.reshape( +- pixel_values.shape[0], +- self.num_channels, +- self.temporal_patch_size, +- self.patch_size, +- self.patch_size, ++ # Each row is already exactly one non-overlapping Conv3d receptive ++ # field. The Conv3d output therefore has spatial shape (1, 1, 1) and ++ # is algebraically identical to this bias-free linear projection. ++ # Using GEMM also avoids lazy cuDNN initialization in the serving ++ # workers, which can fail while loading a cuDNN JIT module on GB200. ++ return F.linear( ++ pixel_values, ++ self.patch_embedding.weight.flatten(1), + ) +- return self.patch_embedding(x).reshape(x.shape[0], -1) From bea41d1a13ed64666a3703aa48523f7e7b2c815b Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Thu, 1 Oct 2026 08:37:24 -0700 Subject: [PATCH 11/18] fix(router): drop remote KV saturation protection Keep compute-only prefill accounting with unweighted cached tokens from the selected rank across all cache tiers. Remove the remote KV waiting threshold, saturation gate, and associated telemetry/configuration plumbing. Port the routing-only source change from c29a0fb7d3 while preserving container patches 1/2/3/4/6. Add regression coverage that the removed threshold CLI is rejected and its legacy environment variable is not forwarded. Signed-off-by: Shreyas Misra --- .../configuration/groups/router_args.py | 23 ---- .../src/dynamo/common/forward_pass_metrics.py | 10 -- .../configuration/test_kv_router_args.py | 42 +++--- components/src/dynamo/vllm/args.py | 21 +-- .../src/dynamo/vllm/instrumented_scheduler.py | 9 -- components/src/dynamo/vllm/main.py | 12 +- .../tests/test_vllm_instrumented_scheduler.py | 19 +-- .../src/dynamo/vllm/tests/test_vllm_unit.py | 33 ----- .../observability/metrics-catalog.mdx | 8 -- lib/bindings/python/rust/llm/entrypoint.rs | 7 +- lib/bindings/python/rust/llm/kv.rs | 19 +-- lib/bindings/python/src/dynamo/_core.pyi | 4 - lib/kv-router/src/protocols.rs | 7 - lib/kv-router/src/scheduling/local.rs | 10 +- lib/kv-router/src/scheduling/queue.rs | 48 +++++-- lib/llm/src/discovery/worker_monitor.rs | 120 +----------------- lib/llm/src/fpm_publisher.rs | 116 +---------------- lib/llm/src/http/service/busy_threshold.rs | 11 +- lib/llm/src/kv_router/metrics.rs | 84 +----------- lib/llm/src/kv_router/metrics_subscriber.rs | 10 -- .../src/kv_router/publisher/worker_metrics.rs | 1 - lib/runtime/src/metrics/prometheus_names.rs | 6 - 22 files changed, 90 insertions(+), 530 deletions(-) diff --git a/components/src/dynamo/common/configuration/groups/router_args.py b/components/src/dynamo/common/configuration/groups/router_args.py index 29dec3b5b5a4..cdf3bf2f50bb 100644 --- a/components/src/dynamo/common/configuration/groups/router_args.py +++ b/components/src/dynamo/common/configuration/groups/router_args.py @@ -36,7 +36,6 @@ "active_decode_blocks_threshold", "active_prefill_tokens_threshold", "active_prefill_tokens_threshold_frac", - "remote_kv_waiting_tokens_threshold", "session_affinity_ttl_secs", "session_affinity_mode", ) @@ -83,7 +82,6 @@ class RouterConfigBase(ConfigBase): active_decode_blocks_threshold: Optional[float] active_prefill_tokens_threshold: Optional[int] active_prefill_tokens_threshold_frac: Optional[float] - remote_kv_waiting_tokens_threshold: Optional[int] def router_kwargs(self) -> dict: """Return a dict suitable for ``RouterConfig(mode, kv_config, **kwargs)``.""" @@ -111,10 +109,6 @@ def validate_rejection_thresholds(self) -> None: "--active-prefill-tokens-threshold-frac must be a finite value >= 0" ) - remote_kv_threshold = self.remote_kv_waiting_tokens_threshold - if remote_kv_threshold is not None and remote_kv_threshold < 0: - raise ValueError("--remote-kv-waiting-tokens-threshold must be >= 0") - def log_rejection_thresholds(self) -> None: """Log which independently configured rejection checks are active.""" configured = [ @@ -132,10 +126,6 @@ def log_rejection_thresholds(self) -> None: "--active-prefill-tokens-threshold-frac", self.active_prefill_tokens_threshold_frac, ), - ( - "--remote-kv-waiting-tokens-threshold", - self.remote_kv_waiting_tokens_threshold, - ), ) if value is not None ] @@ -320,19 +310,6 @@ def add_arguments(self, parser) -> None: ), arg_type=nullable_float, ) - add_argument( - g, - flag_name="--remote-kv-waiting-tokens-threshold", - env_var="DYN_REMOTE_KV_WAITING_TOKENS_THRESHOLD", - default=None, - help=( - "Literal token threshold for external-KV transfer backlog. A worker " - "remains eligible for prefix-affine routing until tokens in " - "WAITING_FOR_REMOTE_KVS exceed this value. Unset by default; pass " - "'None' to disable it." - ), - arg_type=nullable_int, - ) # CLI spelling -> `dynamo.llm.RouterMode` attribute name. diff --git a/components/src/dynamo/common/forward_pass_metrics.py b/components/src/dynamo/common/forward_pass_metrics.py index 1e0621e0f4f7..f656ee7bbeb0 100644 --- a/components/src/dynamo/common/forward_pass_metrics.py +++ b/components/src/dynamo/common/forward_pass_metrics.py @@ -149,16 +149,6 @@ class QueuedRequestMetrics( # Population variance of KV context lengths for queued decode requests. var_decode_kv_tokens: float = 0.0 - # Requests whose destination KV blocks are allocated but whose external - # KV transfer has not completed. These are also included in the queued - # decode totals above for planner compatibility, but are broken out here - # so routing does not mistake transfer wait for prefill compute. - num_remote_kv_waiting_requests: int = 0 - - # Tokens currently being received by requests in - # WAITING_FOR_REMOTE_KVS. This is transfer backlog, not prefill work. - sum_remote_kv_waiting_tokens: int = 0 - class ForwardPassMetrics( msgspec.Struct, diff --git a/components/src/dynamo/common/tests/configuration/test_kv_router_args.py b/components/src/dynamo/common/tests/configuration/test_kv_router_args.py index fcc194ee9144..ea7a1f98c8ec 100644 --- a/components/src/dynamo/common/tests/configuration/test_kv_router_args.py +++ b/components/src/dynamo/common/tests/configuration/test_kv_router_args.py @@ -25,7 +25,6 @@ def _clear_rejection_threshold_env(monkeypatch: pytest.MonkeyPatch) -> None: "DYN_ACTIVE_DECODE_BLOCKS_THRESHOLD", "DYN_ACTIVE_PREFILL_TOKENS_THRESHOLD", "DYN_ACTIVE_PREFILL_TOKENS_THRESHOLD_FRAC", - "DYN_REMOTE_KV_WAITING_TOKENS_THRESHOLD", "DYN_ADMISSION_CONTROL", "DYN_ROUTER_QUEUE_THRESHOLD", "DYN_ROUTER_SESSION_AFFINITY_MODE", @@ -569,6 +568,23 @@ def test_conditional_disagg_config_rejects_invalid_json( kv_router_args._conditional_disagg_config_arg(config_json) +def test_removed_remote_kv_threshold_is_not_a_frontend_option( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _clear_rejection_threshold_env(monkeypatch) + monkeypatch.setenv("DYN_REMOTE_KV_WAITING_TOKENS_THRESHOLD", "8192") + parser = argparse.ArgumentParser() + FrontendArgGroup().add_arguments(parser) + + config = FrontendConfig.from_cli_args(parser.parse_args([])) + config.validate() + assert not hasattr(config, "remote_kv_waiting_tokens_threshold") + assert "remote_kv_waiting_tokens_threshold" not in config.router_kwargs() + with pytest.raises(SystemExit) as error: + parser.parse_args(["--remote-kv-waiting-tokens-threshold", "8192"]) + assert error.value.code == 2 + + def test_frontend_rejection_thresholds_default_to_none( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, @@ -585,13 +601,11 @@ def test_frontend_rejection_thresholds_default_to_none( assert config.active_decode_blocks_threshold is None assert config.active_prefill_tokens_threshold is None assert config.active_prefill_tokens_threshold_frac is None - assert config.remote_kv_waiting_tokens_threshold is None assert config.router_queue_threshold is None assert config.router_kwargs() == { "active_decode_blocks_threshold": None, "active_prefill_tokens_threshold": None, "active_prefill_tokens_threshold_frac": None, - "remote_kv_waiting_tokens_threshold": None, "session_affinity_ttl_secs": None, "session_affinity_mode": "hard", } @@ -619,12 +633,6 @@ def test_frontend_rejection_thresholds_default_to_none( "active_prefill_tokens_threshold_frac", 2.0, ), - ( - "--remote-kv-waiting-tokens-threshold", - "8192", - "remote_kv_waiting_tokens_threshold", - 8192, - ), ], ) def test_each_cli_rejection_threshold_is_independently_opt_in( @@ -647,7 +655,6 @@ def test_each_cli_rejection_threshold_is_independently_opt_in( "active_decode_blocks_threshold": config.active_decode_blocks_threshold, "active_prefill_tokens_threshold": config.active_prefill_tokens_threshold, "active_prefill_tokens_threshold_frac": config.active_prefill_tokens_threshold_frac, - "remote_kv_waiting_tokens_threshold": config.remote_kv_waiting_tokens_threshold, } assert thresholds[field] == expected assert all(value is None for name, value in thresholds.items() if name != field) @@ -672,11 +679,6 @@ def test_each_cli_rejection_threshold_is_independently_opt_in( "active_prefill_tokens_threshold_frac", 3.0, ), - ( - "DYN_REMOTE_KV_WAITING_TOKENS_THRESHOLD", - "remote_kv_waiting_tokens_threshold", - 16384, - ), ], ) def test_each_environment_rejection_threshold_is_independently_opt_in( @@ -697,7 +699,6 @@ def test_each_environment_rejection_threshold_is_independently_opt_in( "active_decode_blocks_threshold": config.active_decode_blocks_threshold, "active_prefill_tokens_threshold": config.active_prefill_tokens_threshold, "active_prefill_tokens_threshold_frac": config.active_prefill_tokens_threshold_frac, - "remote_kv_waiting_tokens_threshold": config.remote_kv_waiting_tokens_threshold, } assert thresholds[field] == expected assert all(value is None for name, value in thresholds.items() if name != field) @@ -719,8 +720,6 @@ def test_all_rejection_thresholds_and_queue_override_are_forwarded( "1000", "--active-prefill-tokens-threshold-frac", "2.0", - "--remote-kv-waiting-tokens-threshold", - "8192", "--router-queue-threshold", "32.0", ] @@ -731,13 +730,11 @@ def test_all_rejection_thresholds_and_queue_override_are_forwarded( assert config.active_decode_blocks_threshold == 0.5 assert config.active_prefill_tokens_threshold == 1000 assert config.active_prefill_tokens_threshold_frac == 2.0 - assert config.remote_kv_waiting_tokens_threshold == 8192 assert config.router_queue_threshold == 32.0 assert config.router_kwargs() == { "active_decode_blocks_threshold": 0.5, "active_prefill_tokens_threshold": 1000, "active_prefill_tokens_threshold_frac": 2.0, - "remote_kv_waiting_tokens_threshold": 8192, "session_affinity_ttl_secs": None, "session_affinity_mode": "hard", } @@ -808,7 +805,6 @@ def test_admission_control_cli_flag_warns_and_is_ignored( assert config.active_decode_blocks_threshold is None assert config.active_prefill_tokens_threshold is None assert config.active_prefill_tokens_threshold_frac is None - assert config.remote_kv_waiting_tokens_threshold is None assert "--admission-control is no longer supported and is ignored" in caplog.text with pytest.raises(SystemExit): @@ -831,7 +827,6 @@ def test_removed_admission_control_environment_warns_and_is_ignored( assert config.active_decode_blocks_threshold is None assert config.active_prefill_tokens_threshold is None assert config.active_prefill_tokens_threshold_frac is None - assert config.remote_kv_waiting_tokens_threshold is None assert "DYN_ADMISSION_CONTROL is no longer supported and is ignored" in caplog.text @@ -841,7 +836,6 @@ def test_removed_admission_control_environment_warns_and_is_ignored( "--active-decode-blocks-threshold", "--active-prefill-tokens-threshold", "--active-prefill-tokens-threshold-frac", - "--remote-kv-waiting-tokens-threshold", ], ) def test_explicit_none_keeps_rejection_threshold_disabled( @@ -858,7 +852,6 @@ def test_explicit_none_keeps_rejection_threshold_disabled( assert config.active_decode_blocks_threshold is None assert config.active_prefill_tokens_threshold is None assert config.active_prefill_tokens_threshold_frac is None - assert config.remote_kv_waiting_tokens_threshold is None @pytest.mark.parametrize( @@ -870,7 +863,6 @@ def test_explicit_none_keeps_rejection_threshold_disabled( ("--active-prefill-tokens-threshold", "-1"), ("--active-prefill-tokens-threshold-frac", "-0.1"), ("--active-prefill-tokens-threshold-frac", "inf"), - ("--remote-kv-waiting-tokens-threshold", "-1"), ], ) def test_rejection_threshold_validation_rejects_invalid_values( diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index b31799627558..6a9ced7de654 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -280,20 +280,11 @@ def _unsupported_fpm_trace_role(dynamo_config: Config) -> Optional[str]: return None -def _has_kv_transfer_connector(engine_config: object | None) -> bool: - """Whether an engine may enter WAITING_FOR_REMOTE_KVS.""" - if engine_config is None: - return False - return getattr(engine_config, "kv_transfer_config", None) is not None - - -def _forward_pass_metrics_enabled( - dynamo_config: Config, engine_config: object | None = None -) -> bool: +def _forward_pass_metrics_enabled(dynamo_config: Config) -> bool: """Resolve FPM activation without changing the legacy explicit-port path.""" if envs.is_set("DYN_FORWARDPASS_METRIC_PORT"): return True - if not dynamo_config.fpm_trace and not _has_kv_transfer_connector(engine_config): + if not dynamo_config.fpm_trace: return False unsupported_role = _unsupported_fpm_trace_role(dynamo_config) @@ -362,7 +353,7 @@ def update_engine_config_with_dynamo( f"(use_kv_events={dynamo_config.use_kv_events})" ) - fpm_enabled = _forward_pass_metrics_enabled(dynamo_config, engine_config) + fpm_enabled = _forward_pass_metrics_enabled(dynamo_config) if fpm_enabled: existing_cls = getattr(engine_config, "scheduler_cls", None) if existing_cls is None: @@ -377,11 +368,7 @@ def update_engine_config_with_dynamo( fpm_source = ( "DYN_FORWARDPASS_METRIC_PORT is set" if envs.is_set("DYN_FORWARDPASS_METRIC_PORT") - else ( - "--fpm-trace/DYN_FPM_TRACE is enabled" - if dynamo_config.fpm_trace - else "a KV transfer connector requires remote-wait telemetry" - ) + else "--fpm-trace/DYN_FPM_TRACE is enabled" ) logger.warning( f"{fpm_source} but scheduler_cls " diff --git a/components/src/dynamo/vllm/instrumented_scheduler.py b/components/src/dynamo/vllm/instrumented_scheduler.py index 4018a161b363..ca39449b7153 100644 --- a/components/src/dynamo/vllm/instrumented_scheduler.py +++ b/components/src/dynamo/vllm/instrumented_scheduler.py @@ -2153,14 +2153,12 @@ def _compute_queued(self) -> QueuedRequestMetrics: """ prefill = WelfordAccumulator() decode_kv = WelfordAccumulator() - remote_kv = WelfordAccumulator() for request in self.waiting: if request.status == RequestStatus.PREEMPTED: decode_kv.add(request.num_computed_tokens) elif request.status == RequestStatus.WAITING_FOR_REMOTE_KVS: decode_kv.add(request.num_computed_tokens) - remote_kv.add(request.num_computed_tokens) else: prefill.add(request.num_tokens) @@ -2170,11 +2168,6 @@ def _compute_queued(self) -> QueuedRequestMetrics: # engine and being transferred. Next schedule() step will # start generating -- count as queued decode. decode_kv.add(request.num_computed_tokens) - # Keep transfer backlog distinct from compute backlog. The - # router consumes this exact status-derived signal when - # deciding whether an affinity-hot worker's transfer queue is - # saturated. - remote_kv.add(request.num_computed_tokens) else: # Structured-output waits / WAITING_FOR_STREAMING_REQ: # no KV yet, essentially a queued prefill awaiting a @@ -2188,8 +2181,6 @@ def _compute_queued(self) -> QueuedRequestMetrics: num_decode_requests=decode_kv.n, sum_decode_kv_tokens=decode_kv.s, var_decode_kv_tokens=decode_kv.variance(), - num_remote_kv_waiting_requests=remote_kv.n, - sum_remote_kv_waiting_tokens=remote_kv.s, ) # ------------------------------------------------------------------ diff --git a/components/src/dynamo/vllm/main.py b/components/src/dynamo/vllm/main.py index dc667eeb8d50..b27f47f4062d 100644 --- a/components/src/dynamo/vllm/main.py +++ b/components/src/dynamo/vllm/main.py @@ -552,17 +552,7 @@ def setup_fpm_relay( Returns: List of FpmEventRelay instances, or None if FPM is not enabled. """ - scheduler_cls = getattr(vllm_config, "scheduler_cls", None) - remote_kv_telemetry = ( - getattr(vllm_config, "kv_transfer_config", None) is not None - and scheduler_cls is not None - and "InstrumentedScheduler" in str(scheduler_cls) - ) - if not ( - envs.is_set("DYN_FORWARDPASS_METRIC_PORT") - or config.fpm_trace - or remote_kv_telemetry - ): + if not (envs.is_set("DYN_FORWARDPASS_METRIC_PORT") or config.fpm_trace): return None try: diff --git a/components/src/dynamo/vllm/tests/test_vllm_instrumented_scheduler.py b/components/src/dynamo/vllm/tests/test_vllm_instrumented_scheduler.py index f9f14b25303f..6b5cae325c51 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_instrumented_scheduler.py +++ b/components/src/dynamo/vllm/tests/test_vllm_instrumented_scheduler.py @@ -389,28 +389,25 @@ def test_skipped_waiting_for_remote_kvs_counts_as_queued_decode(): # Classified as decode with KV = num_computed_tokens. assert q.num_decode_requests == 2 assert q.sum_decode_kv_tokens == 1500 - assert q.num_remote_kv_waiting_requests == 2 - assert q.sum_remote_kv_waiting_tokens == 1500 -def test_main_waiting_queue_remote_kvs_uses_same_transfer_classification(): - """Be robust to vLLM queue-placement changes and transient states.""" +def test_main_waiting_queue_remote_kvs_counts_as_queued_decode(): + """A transient remote-KV wait in the main queue is not prefill work.""" q = _run_compute_queued( waiting=[ _make_request( RequestStatus.WAITING_FOR_REMOTE_KVS, - num_tokens=1000, + num_tokens=1024, num_computed_tokens=768, - ) + ), ], skipped_waiting=[], ) - assert q.num_prefill_requests == 0 + assert q.sum_prefill_tokens == 0 assert q.num_decode_requests == 1 - assert q.num_remote_kv_waiting_requests == 1 - assert q.sum_remote_kv_waiting_tokens == 768 + assert q.sum_decode_kv_tokens == 768 def test_skipped_waiting_for_structured_output_counts_as_queued_prefill(): @@ -480,10 +477,6 @@ def test_mixed_disagg_decode_engine_snapshot(): # 1 preempted (local decode evicted) + 3 remote-KV-waiting. assert q.num_decode_requests == 4 assert q.sum_decode_kv_tokens == 780 + 1024 + 2048 + 512 - # Only remote-KV waits contribute to transfer pressure; a local preemption - # remains decode backlog but is not a transfer. - assert q.num_remote_kv_waiting_requests == 3 - assert q.sum_remote_kv_waiting_tokens == 1024 + 2048 + 512 def test_mixed_prefill_engine_snapshot(): diff --git a/components/src/dynamo/vllm/tests/test_vllm_unit.py b/components/src/dynamo/vllm/tests/test_vllm_unit.py index 09ce651bd40e..d6d01b299813 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_unit.py +++ b/components/src/dynamo/vllm/tests/test_vllm_unit.py @@ -1761,39 +1761,6 @@ def test_false_trace_does_not_enable_fpm(self, monkeypatch): assert engine_cfg.scheduler_cls is None - def test_kv_transfer_connector_enables_remote_wait_telemetry(self, monkeypatch): - monkeypatch.delenv("DYN_FORWARDPASS_METRIC_PORT", raising=False) - dynamo_cfg = _make_dynamo_config() - engine_cfg = _make_engine_config_with_runner( - scheduler_cls=None, - kv_transfer_config=SimpleNamespace(kv_connector="OffloadingConnector"), - ) - - update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) - - assert ( - engine_cfg.scheduler_cls - == "dynamo.vllm.instrumented_scheduler.InstrumentedScheduler" - ) - assert envs.DYN_FORWARDPASS_METRIC_PORT == 20380 - - def test_custom_scheduler_does_not_start_implicit_remote_wait_relay( - self, monkeypatch - ): - vllm_main = _load_vllm_main() - monkeypatch.delenv("DYN_FORWARDPASS_METRIC_PORT", raising=False) - dynamo_cfg = _make_dynamo_config() - engine_cfg = _make_engine_config_with_runner( - scheduler_cls="example.CustomScheduler", - kv_transfer_config=SimpleNamespace(kv_connector="OffloadingConnector"), - ) - - update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) - - assert vllm_main.setup_fpm_relay( - dynamo_cfg, SimpleNamespace(), engine_cfg - ) is None - def test_disabled_trace_does_not_start_relay(self, monkeypatch): vllm_main = _load_vllm_main() monkeypatch.delenv("DYN_FORWARDPASS_METRIC_PORT", raising=False) diff --git a/docs/fern/pages/reference/observability/metrics-catalog.mdx b/docs/fern/pages/reference/observability/metrics-catalog.mdx index 38bf4772eaf9..21cc3b8e1875 100644 --- a/docs/fern/pages/reference/observability/metrics-catalog.mdx +++ b/docs/fern/pages/reference/observability/metrics-catalog.mdx @@ -159,14 +159,6 @@ Counters for requests that end early: [cancellations](../../developer-guide/know Active prefill tokens queued per worker. - - Requests whose external KV transfer has not completed, per worker. This is transfer pressure, not prefill compute. - - - - KV context tokens belonging to requests whose external KV transfer has not completed, per worker. - - Last observed time to first token per worker, in seconds. diff --git a/lib/bindings/python/rust/llm/entrypoint.rs b/lib/bindings/python/rust/llm/entrypoint.rs index d2e44a2c91df..39e3b20817f2 100644 --- a/lib/bindings/python/rust/llm/entrypoint.rs +++ b/lib/bindings/python/rust/llm/entrypoint.rs @@ -480,7 +480,6 @@ pub struct RouterConfig { active_prefill_tokens_threshold: Option, /// Threshold for active prefill tokens as fraction of max_num_batched_tokens active_prefill_tokens_threshold_frac: Option, - remote_kv_waiting_tokens_threshold: Option, session_affinity_ttl_secs: Option, session_affinity_mode: RsSessionAffinityMode, } @@ -489,7 +488,7 @@ pub struct RouterConfig { impl RouterConfig { #[new] #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (mode, config=None, active_decode_blocks_threshold=None, active_prefill_tokens_threshold=None, active_prefill_tokens_threshold_frac=None, enforce_disagg=false, session_affinity_ttl_secs=None, session_affinity_mode="hard", remote_kv_waiting_tokens_threshold=None))] + #[pyo3(signature = (mode, config=None, active_decode_blocks_threshold=None, active_prefill_tokens_threshold=None, active_prefill_tokens_threshold_frac=None, enforce_disagg=false, session_affinity_ttl_secs=None, session_affinity_mode="hard"))] pub fn new( mode: RouterMode, config: Option, @@ -499,7 +498,6 @@ impl RouterConfig { enforce_disagg: bool, session_affinity_ttl_secs: Option, session_affinity_mode: &str, - remote_kv_waiting_tokens_threshold: Option, ) -> PyResult { if enforce_disagg { static WARN_ONCE: std::sync::Once = std::sync::Once::new(); @@ -518,7 +516,6 @@ impl RouterConfig { active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold, } .validate() .map_err(PyValueError::new_err)?; @@ -531,7 +528,6 @@ impl RouterConfig { active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold, session_affinity_ttl_secs, session_affinity_mode, }) @@ -547,7 +543,6 @@ impl From for RsRouterConfig { active_decode_blocks_threshold: rc.active_decode_blocks_threshold, active_prefill_tokens_threshold: rc.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac: rc.active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold: rc.remote_kv_waiting_tokens_threshold, }, enforce_disagg: false, session_affinity_ttl_secs: rc.session_affinity_ttl_secs, diff --git a/lib/bindings/python/rust/llm/kv.rs b/lib/bindings/python/rust/llm/kv.rs index fc3e85262af5..91c86ce80878 100644 --- a/lib/bindings/python/rust/llm/kv.rs +++ b/lib/bindings/python/rust/llm/kv.rs @@ -78,32 +78,27 @@ pub(crate) struct LoadThresholdConfig { active_prefill_tokens_threshold: Option, #[pyo3(get)] active_prefill_tokens_threshold_frac: Option, - #[pyo3(get)] - remote_kv_waiting_tokens_threshold: Option, } #[pymethods] impl LoadThresholdConfig { #[new] - #[pyo3(signature = (*, active_decode_blocks_threshold=None, active_prefill_tokens_threshold=None, active_prefill_tokens_threshold_frac=None, remote_kv_waiting_tokens_threshold=None))] + #[pyo3(signature = (*, active_decode_blocks_threshold=None, active_prefill_tokens_threshold=None, active_prefill_tokens_threshold_frac=None))] fn new( active_decode_blocks_threshold: Option, active_prefill_tokens_threshold: Option, active_prefill_tokens_threshold_frac: Option, - remote_kv_waiting_tokens_threshold: Option, ) -> PyResult { let config = validate_load_threshold_config( active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold, ) .map_err(PyValueError::new_err)?; Ok(Self { active_decode_blocks_threshold: config.active_decode_blocks_threshold, active_prefill_tokens_threshold: config.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac: config.active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold: config.remote_kv_waiting_tokens_threshold, }) } } @@ -114,7 +109,6 @@ impl LoadThresholdConfig { active_decode_blocks_threshold: self.active_decode_blocks_threshold, active_prefill_tokens_threshold: self.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac: self.active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold: self.remote_kv_waiting_tokens_threshold, } } } @@ -123,13 +117,11 @@ fn validate_load_threshold_config( active_decode_blocks_threshold: Option, active_prefill_tokens_threshold: Option, active_prefill_tokens_threshold_frac: Option, - remote_kv_waiting_tokens_threshold: Option, ) -> Result { let config = RsLoadThresholdConfig { active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold, }; config .validate() @@ -1836,7 +1828,6 @@ mod metric_worker_type_tests { active_decode_blocks_threshold: Some(0.8), active_prefill_tokens_threshold: Some(1024), active_prefill_tokens_threshold_frac: Some(0.5), - remote_kv_waiting_tokens_threshold: Some(8192), }; let (router, runtime) = standalone_encode_router( "python-standalone-encode-configured-load", @@ -1855,28 +1846,26 @@ mod load_threshold_config_tests { #[test] fn default_config_disables_overload_thresholds() { - let config = validate_load_threshold_config(None, None, None, None).unwrap(); + let config = validate_load_threshold_config(None, None, None).unwrap(); assert!(!config.is_configured()); } #[test] fn valid_config_preserves_all_thresholds() { - let config = - validate_load_threshold_config(Some(0.75), Some(512), Some(0.5), Some(8192)).unwrap(); + let config = validate_load_threshold_config(Some(0.75), Some(512), Some(0.5)).unwrap(); assert_eq!( config, RsLoadThresholdConfig { active_decode_blocks_threshold: Some(0.75), active_prefill_tokens_threshold: Some(512), active_prefill_tokens_threshold_frac: Some(0.5), - remote_kv_waiting_tokens_threshold: Some(8192), } ); } #[test] fn invalid_config_returns_validation_error() { - let error = validate_load_threshold_config(Some(1.1), None, None, None).unwrap_err(); + let error = validate_load_threshold_config(Some(1.1), None, None).unwrap_err(); assert!(error.contains( "invalid load threshold config: active_decode_blocks_threshold must be between 0.0 and 1.0" )); diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index 9ca914fadfab..50706e497dfb 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -1732,7 +1732,6 @@ class RouterConfig: enforce_disagg: bool = False, session_affinity_ttl_secs: Optional[int] = None, session_affinity_mode: str = "hard", - remote_kv_waiting_tokens_threshold: Optional[int] = None, ) -> None: """ Create a RouterConfig. @@ -1746,7 +1745,6 @@ class RouterConfig: enforce_disagg: Deprecated and ignored. Routing topology and readiness come from registered worker types. session_affinity_ttl_secs: Router-local session-affinity idle TTL in seconds. session_affinity_mode: Session binding behavior: ``hard`` or ``soft``. - remote_kv_waiting_tokens_threshold: Transfer-backlog token threshold for WAITING_FOR_REMOTE_KVS """ ... @@ -1756,7 +1754,6 @@ class LoadThresholdConfig: active_decode_blocks_threshold: Optional[float] active_prefill_tokens_threshold: Optional[int] active_prefill_tokens_threshold_frac: Optional[float] - remote_kv_waiting_tokens_threshold: Optional[int] def __init__( self, @@ -1764,7 +1761,6 @@ class LoadThresholdConfig: active_decode_blocks_threshold: Optional[float] = None, active_prefill_tokens_threshold: Optional[int] = None, active_prefill_tokens_threshold_frac: Optional[float] = None, - remote_kv_waiting_tokens_threshold: Optional[int] = None, ) -> None: ... diff --git a/lib/kv-router/src/protocols.rs b/lib/kv-router/src/protocols.rs index d2960d1fd873..a8e96d3b0189 100644 --- a/lib/kv-router/src/protocols.rs +++ b/lib/kv-router/src/protocols.rs @@ -1107,13 +1107,6 @@ pub struct ActiveLoad { pub active_decode_blocks: Option, /// Number of active prefill tokens (from scheduler's view). pub active_prefill_tokens: Option, - /// Number of requests whose external KV transfer is still in flight. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub remote_kv_waiting_requests: Option, - /// Tokens currently waiting for external KV transfer completion. - /// This is transfer backlog and must not be interpreted as prefill compute. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub remote_kv_waiting_tokens: Option, /// Total KV blocks currently in use on the worker. /// /// This is published by workers only and is the authoritative signal for diff --git a/lib/kv-router/src/scheduling/local.rs b/lib/kv-router/src/scheduling/local.rs index f480ffa9b8a7..3a03313ebb4f 100644 --- a/lib/kv-router/src/scheduling/local.rs +++ b/lib/kv-router/src/scheduling/local.rs @@ -973,7 +973,7 @@ mod tests { } #[tokio::test] - async fn test_schedule_uses_weighted_cached_tokens_for_active_tracking() { + async fn test_schedule_excludes_lower_tier_cached_tokens_from_active_tracking() { let mut workers = HashMap::new(); workers.insert( 0, @@ -984,13 +984,15 @@ mod tests { ); let (scheduler, slots, _cfg_tx, cancel_token) = make_scheduler(workers, None, true, None); let worker = WorkerWithDpRank::new(0, 0); + let mut tier_overlap_blocks = TierOverlapBlocks::default(); + tier_overlap_blocks.host_pinned.insert(worker, 1); let response = scheduler .schedule( Some("req-1".to_string()), 64, Some(vec![1, 2, 3, 4]), - TierOverlapBlocks::default(), + tier_overlap_blocks, HashMap::from([(worker, 0.75)]), HashMap::from([(worker, 48)]), None, @@ -1012,8 +1014,8 @@ mod tests { assert_eq!(response.effective_overlap_blocks, 0.75); assert_eq!( slots.active_tokens(Instant::now()).get(&worker).copied(), - Some(16), - "weighted cached tokens should reduce tracked prefill load", + Some(0), + "lower-tier cached tokens are transfer work, not active prefill compute", ); cancel_token.cancel(); diff --git a/lib/kv-router/src/scheduling/queue.rs b/lib/kv-router/src/scheduling/queue.rs index 4843f0d3c1c0..d04646a95470 100644 --- a/lib/kv-router/src/scheduling/queue.rs +++ b/lib/kv-router/src/scheduling/queue.rs @@ -27,7 +27,8 @@ use super::selector::{DefaultWorkerSelector, WorkerSelectionInput, WorkerSelecto use super::types::{ AdvisorySchedulingResponse, AdvisoryWorkerLoad, AttemptId, KvSchedulerError, NonMaxOverlapSelection, NonMaxOverlapSelectionObserver, OverloadedWorkerProvider, - SchedulingContext, SchedulingRequest, SchedulingResponse, WorkerAvailabilityProvider, + SchedulingContext, SchedulingRequest, SchedulingResponse, TierOverlapBlocks, + WorkerAvailabilityProvider, }; use crate::protocols::{ LocalBlockHash, PrefillLoadHint, WorkerConfigLike, WorkerId, WorkerSelectionResult, @@ -44,6 +45,22 @@ pub const DEFAULT_MAX_BATCHED_TOKENS: u64 = 10_000_000; const ADMISSION_CHANNEL_CAPACITY: usize = 65_536; +fn selected_rank_cached_blocks( + tier_overlap_blocks: &TierOverlapBlocks, + selected_worker: WorkerWithDpRank, +) -> u32 { + let blocks = [ + &tier_overlap_blocks.device, + &tier_overlap_blocks.host_pinned, + &tier_overlap_blocks.disk, + ] + .into_iter() + .map(|tier| tier.get(&selected_worker).copied().unwrap_or(0)) + .fold(0usize, usize::saturating_add); + + u32::try_from(blocks).unwrap_or(u32::MAX) +} + fn selected_cached_tokens( isl_tokens: usize, selected_cached_blocks: u32, @@ -1421,7 +1438,10 @@ impl< let target_cached_prefix_blocks = target_cached_prefix_blocks(&request, selected.selection.worker); - let selected_cached_blocks = selected.selected_worker_tiers.disk_blocks; + let selected_cached_blocks = selected_rank_cached_blocks( + &request.overlap.tier_overlap_blocks, + selected.selection.worker, + ); let response = SchedulingResponse { best_worker: selected.selection.worker, effective_overlap_blocks: selected.selection.effective_overlap_blocks, @@ -1555,11 +1575,9 @@ impl< return None; } - // Selection deliberately discounts lower-tier hits because transfer is - // slower than a device hit. Do not carry that discount into active - // prefill accounting: host/disk-resident tokens are transfer work, not - // GPU prefill compute. Remote-transfer pressure is reported separately - // by the worker from WAITING_FOR_REMOTE_KVS. + // Selection discounts lower-tier hits because transfer is slower than + // a device hit. Active prefill accounting must not carry that discount: + // host/disk-resident tokens are transfer work, not GPU prefill compute. let cached_tokens = selected_cached_tokens(isl_tokens, selected_cached_blocks, self.block_size); let effective_isl = effective_prefill_tokens(isl_tokens, cached_tokens); @@ -2269,7 +2287,21 @@ mod tests { } #[test] - fn prefill_load_hint_excludes_all_selected_cached_tiers_from_compute() { + fn selected_rank_cached_blocks_uses_only_selected_dp_rank() { + let selected = WorkerWithDpRank::new(7, 0); + let other_rank = WorkerWithDpRank::new(7, 1); + let mut tiers = TierOverlapBlocks::default(); + tiers.device.insert(selected, 2); + tiers.host_pinned.insert(selected, 3); + tiers.disk.insert(selected, 5); + tiers.device.insert(other_rank, 1_000); + + assert_eq!(selected_rank_cached_blocks(&tiers, selected), 10); + assert_eq!(selected_rank_cached_blocks(&tiers, other_rank), 1_000); + } + + #[test] + fn selected_cached_tokens_excludes_all_selected_tiers_from_compute() { let cached_tokens = selected_cached_tokens(1_024, 48, 16); assert_eq!(effective_prefill_tokens(1_024, cached_tokens), 256); } diff --git a/lib/llm/src/discovery/worker_monitor.rs b/lib/llm/src/discovery/worker_monitor.rs index ec654f95df9e..0e582d2f53a4 100644 --- a/lib/llm/src/discovery/worker_monitor.rs +++ b/lib/llm/src/discovery/worker_monitor.rs @@ -41,8 +41,6 @@ fn cleanup_worker_metrics(worker_id: u64, dp_ranks: &[u32], worker_type: &str) { let labels = &[worker_id_str.as_str(), dp_rank_str.as_str(), worker_type]; let _ = m.active_decode_blocks.remove_label_values(labels); let _ = m.active_prefill_tokens.remove_label_values(labels); - let _ = m.remote_kv_waiting_requests.remove_label_values(labels); - let _ = m.remote_kv_waiting_tokens.remove_label_values(labels); let _ = WORKER_LAST_TIME_TO_FIRST_TOKEN_GAUGE.remove_label_values(labels); let _ = WORKER_LAST_INPUT_SEQUENCE_TOKENS_GAUGE.remove_label_values(labels); let _ = WORKER_LAST_INTER_TOKEN_LATENCY_GAUGE.remove_label_values(labels); @@ -112,11 +110,6 @@ pub struct LoadThresholdConfig { /// Worker is overloaded when `active_prefill_tokens > frac * max_num_batched_tokens`. #[serde(skip_serializing_if = "Option::is_none")] pub active_prefill_tokens_threshold_frac: Option, - - /// Absolute external-KV transfer backlog threshold. A worker is busy when - /// tokens in `WAITING_FOR_REMOTE_KVS` exceed this value. - #[serde(skip_serializing_if = "Option::is_none")] - pub remote_kv_waiting_tokens_threshold: Option, } impl LoadThresholdConfig { @@ -125,7 +118,6 @@ impl LoadThresholdConfig { self.active_decode_blocks_threshold.is_some() || self.active_prefill_tokens_threshold.is_some() || self.active_prefill_tokens_threshold_frac.is_some() - || self.remote_kv_waiting_tokens_threshold.is_some() } /// Validate threshold values shared by startup and dynamic configuration. @@ -174,9 +166,6 @@ impl LoadThresholdHandle { if let Some(value) = config.active_prefill_tokens_threshold_frac { current.active_prefill_tokens_threshold_frac = Some(value); } - if let Some(value) = config.remote_kv_waiting_tokens_threshold { - current.remote_kv_waiting_tokens_threshold = Some(value); - } } pub fn is_configured(&self) -> bool { @@ -208,8 +197,6 @@ struct RemoteActiveLoadSnapshot { active_decode_blocks: Option, active_prefill_tokens: Option, kv_used_blocks: Option, - remote_kv_waiting_requests: Option, - remote_kv_waiting_tokens: Option, } impl From for RemoteActiveLoadSnapshot { @@ -222,8 +209,6 @@ impl From for RemoteActiveLoadSnapshot { active_decode_blocks: load.active_decode_blocks, active_prefill_tokens: load.active_prefill_tokens, kv_used_blocks: load.kv_used_blocks, - remote_kv_waiting_requests: load.remote_kv_waiting_requests, - remote_kv_waiting_tokens: load.remote_kv_waiting_tokens, } } } @@ -242,8 +227,6 @@ impl LoadObservation { Option, Option, Option, - Option, - Option, ) { match self { Self::Scheduler(snapshot) => ( @@ -251,16 +234,12 @@ impl LoadObservation { Some(snapshot.active_decode_blocks), Some(snapshot.active_prefill_tokens), None, - None, - None, ), Self::Remote(snapshot) => ( snapshot.worker, snapshot.active_decode_blocks, snapshot.active_prefill_tokens, snapshot.kv_used_blocks, - snapshot.remote_kv_waiting_requests, - snapshot.remote_kv_waiting_tokens, ), } } @@ -272,7 +251,6 @@ pub struct WorkerLoadState { pub kv_used_blocks: HashMap, pub kv_total_blocks: HashMap, pub active_prefill_tokens: HashMap, - pub remote_kv_waiting_tokens: HashMap, /// max_num_batched_tokens from runtime config (same for all dp_ranks) pub max_num_batched_tokens: HashMap, /// The current router-visible ranks declared by this worker's runtime config. @@ -298,8 +276,6 @@ impl WorkerLoadState { .retain(|dp_rank, _| declared_dp_ranks.contains(dp_rank)); self.active_prefill_tokens .retain(|dp_rank, _| declared_dp_ranks.contains(dp_rank)); - self.remote_kv_waiting_tokens - .retain(|dp_rank, _| declared_dp_ranks.contains(dp_rank)); self.kv_total_blocks.clear(); if let Some(total_blocks) = total_kv_blocks { @@ -432,14 +408,8 @@ impl WorkerLoadState { observation: LoadObservation, active_decode_blocks_threshold: Option, ) -> bool { - let ( - worker, - active_decode_blocks, - active_prefill_tokens, - kv_used_blocks, - _, - remote_kv_waiting_tokens, - ) = observation.parts(); + let (worker, active_decode_blocks, active_prefill_tokens, kv_used_blocks) = + observation.parts(); let dp_rank = worker.dp_rank; if !self.accepts_dp_rank(dp_rank) { return false; @@ -454,10 +424,6 @@ impl WorkerLoadState { if let Some(active_tokens) = active_prefill_tokens { self.active_prefill_tokens.insert(dp_rank, active_tokens); } - if let Some(waiting_tokens) = remote_kv_waiting_tokens { - self.remote_kv_waiting_tokens - .insert(dp_rank, waiting_tokens); - } if let Some(threshold) = active_decode_blocks_threshold { self.update_decode_overload_latch( dp_rank, @@ -485,42 +451,24 @@ impl WorkerLoadState { /// /// Each threshold is `Option`. A `None` threshold means that check is /// skipped entirely — it cannot contribute to a dp_rank being overloaded. If all - /// four thresholds are `None`, no dp_rank is ever overloaded. + /// three thresholds are `None`, no dp_rank is ever overloaded. /// /// For each dp_rank, a dp_rank is overloaded if ANY of these conditions is met (OR logic): /// 1. `active_prefill_tokens > active_prefill_tokens_threshold` (absolute, if set) /// 2. `active_prefill_tokens > frac * max_num_batched_tokens` (fractional, if set) /// 3. decode overload latch set by either `kv_used_blocks` or `active_decode_blocks` (if set) - /// 4. `remote_kv_waiting_tokens > remote_kv_waiting_tokens_threshold` (if set) /// /// The worker is overloaded only if ALL dp_ranks are overloaded. - #[cfg(test)] pub fn is_overloaded( &self, active_decode_blocks_threshold: Option, active_prefill_tokens_threshold: Option, active_prefill_tokens_threshold_frac: Option, - ) -> bool { - self.is_overloaded_with_remote( - active_decode_blocks_threshold, - active_prefill_tokens_threshold, - active_prefill_tokens_threshold_frac, - None, - ) - } - - fn is_overloaded_with_remote( - &self, - active_decode_blocks_threshold: Option, - active_prefill_tokens_threshold: Option, - active_prefill_tokens_threshold_frac: Option, - remote_kv_waiting_tokens_threshold: Option, ) -> bool { // Short-circuit if all thresholds are unset (i.e. no overload check can fire) if active_decode_blocks_threshold.is_none() && active_prefill_tokens_threshold.is_none() && active_prefill_tokens_threshold_frac.is_none() - && remote_kv_waiting_tokens_threshold.is_none() { return false; } @@ -538,7 +486,6 @@ impl WorkerLoadState { .chain(self.kv_used_blocks.keys()) .chain(self.decode_overload_latches.keys()) .chain(self.active_prefill_tokens.keys()) - .chain(self.remote_kv_waiting_tokens.keys()) .copied() .collect(); &fallback_dp_ranks @@ -586,29 +533,16 @@ impl WorkerLoadState { } } - // Check 4: external-KV transfer backlog. This is intentionally - // independent of prefill compute pressure: prefix affinity keeps - // winning until the transfer queue crosses an explicit limit. - if let Some(threshold) = remote_kv_waiting_tokens_threshold - && self - .remote_kv_waiting_tokens - .get(&dp_rank) - .is_some_and(|&tokens| tokens > threshold) - { - return true; - } - // If we can't perform any check or no threshold exceeded, this dp_rank is free false }) } fn is_overloaded_for_config(&self, config: &LoadThresholdConfig) -> bool { - self.is_overloaded_with_remote( + self.is_overloaded( config.active_decode_blocks_threshold, config.active_prefill_tokens_threshold, config.active_prefill_tokens_threshold_frac, - config.remote_kv_waiting_tokens_threshold, ) } } @@ -1003,8 +937,7 @@ impl WorkerLoadMonitor for KvWorkerMonitor { let observation = LoadObservation::Remote(RemoteActiveLoadSnapshot::from(active_load)); - let (worker, _, _, _, remote_waiting_requests, remote_waiting_tokens) = - observation.parts(); + let (worker, _, _, _) = observation.parts(); if !known_workers.contains(&worker.worker_id) { tracing::debug!( worker_id = worker.worker_id, @@ -1033,14 +966,6 @@ impl WorkerLoadMonitor for KvWorkerMonitor { .or_default() .insert(worker.dp_rank); - WORKER_LOAD_METRICS.observe_remote_kv_wait( - worker.worker_id, - worker.dp_rank, - source.metric_label(), - remote_waiting_requests, - remote_waiting_tokens, - ); - // Snapshot thresholds once per event — rare writes (HTTP endpoint) // mean RwLock contention is effectively zero. let cfg = thresholds.get(); @@ -1068,7 +993,6 @@ impl WorkerLoadMonitor for KvWorkerMonitor { active_decode_blocks_threshold = ?cfg.active_decode_blocks_threshold, active_prefill_tokens_threshold = ?cfg.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac = ?cfg.active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold = ?cfg.remote_kv_waiting_tokens_threshold, worker_overloaded, "processed active load update" ); @@ -1248,8 +1172,6 @@ mod tests { active_decode_blocks: Some(10), active_prefill_tokens: Some(100), kv_used_blocks: None, - remote_kv_waiting_requests: None, - remote_kv_waiting_tokens: None, }); let mut local_then_remote = WorkerLoadState::default(); @@ -1275,8 +1197,6 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(8), - remote_kv_waiting_requests: None, - remote_kv_waiting_tokens: None, }), None, ); @@ -1288,8 +1208,6 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: Some(0), kv_used_blocks: None, - remote_kv_waiting_requests: None, - remote_kv_waiting_tokens: None, }), None, ); @@ -1402,7 +1320,6 @@ mod tests { active_decode_blocks_threshold: Some(0.85), active_prefill_tokens_threshold: Some(10_000), active_prefill_tokens_threshold_frac: Some(0.9), - remote_kv_waiting_tokens_threshold: Some(100_000), }; assert!(config.is_configured()); } @@ -1447,7 +1364,6 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(90), - ..Default::default() }, Some(0.6), ); @@ -1460,7 +1376,6 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(90), - ..Default::default() }, Some(0.6), ); @@ -1480,7 +1395,6 @@ mod tests { active_decode_blocks: Some(90), active_prefill_tokens: Some(900), kv_used_blocks: Some(90), - ..Default::default() }, Some(0.6), ); @@ -1503,8 +1417,6 @@ mod tests { active_decode_blocks: Some(100), active_prefill_tokens: Some(1_000), kv_used_blocks: Some(100), - remote_kv_waiting_requests: None, - remote_kv_waiting_tokens: None, }), Some(0.6), )); @@ -1528,7 +1440,6 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(90), - ..Default::default() }, Some(0.6), ); @@ -1548,7 +1459,6 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(90), - ..Default::default() }, Some(0.6), ); @@ -1561,7 +1471,6 @@ mod tests { active_decode_blocks: Some(10), active_prefill_tokens: None, kv_used_blocks: None, - ..Default::default() }, Some(0.6), ); @@ -1574,7 +1483,6 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(10), - ..Default::default() }, Some(0.6), ); @@ -1593,7 +1501,6 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(90), - ..Default::default() }, Some(0.6), ); @@ -1606,7 +1513,6 @@ mod tests { active_decode_blocks: None, active_prefill_tokens: None, kv_used_blocks: Some(10), - ..Default::default() }, Some(0.6), ); @@ -1625,7 +1531,6 @@ mod tests { active_decode_blocks: Some(90), active_prefill_tokens: None, kv_used_blocks: None, - ..Default::default() }, Some(0.6), ); @@ -1638,7 +1543,6 @@ mod tests { active_decode_blocks: Some(10), active_prefill_tokens: None, kv_used_blocks: None, - ..Default::default() }, Some(0.6), ); @@ -1657,7 +1561,6 @@ mod tests { active_decode_blocks: Some(90), active_prefill_tokens: None, kv_used_blocks: None, - ..Default::default() }, Some(0.6), ); @@ -1670,7 +1573,6 @@ mod tests { active_decode_blocks: Some(10), active_prefill_tokens: None, kv_used_blocks: Some(10), - ..Default::default() }, Some(0.6), ); @@ -1709,7 +1611,6 @@ mod tests { active_decode_blocks: Some(90), active_prefill_tokens: None, kv_used_blocks: Some(90), - ..Default::default() }, Some(0.6), ); @@ -1728,7 +1629,6 @@ mod tests { active_decode_blocks: Some(90), active_prefill_tokens: None, kv_used_blocks: Some(90), - ..Default::default() }, Some(0.6), ); @@ -1753,16 +1653,6 @@ mod tests { assert!(state.is_overloaded(None, None, Some(2.0))); } - #[test] - fn remote_kv_wait_keeps_affinity_until_explicit_threshold_is_exceeded() { - let mut state = WorkerLoadState::default(); - state.remote_kv_waiting_tokens.insert(0, 8_192); - - assert!(!state.is_overloaded_with_remote(None, None, None, Some(8_192))); - assert!(state.is_overloaded_with_remote(None, None, None, Some(8_191))); - assert!(!state.is_overloaded(None, None, None)); - } - #[test] fn compute_overloaded_instances_flags_prefill_workers_over_token_threshold() { use dashmap::DashMap; diff --git a/lib/llm/src/fpm_publisher.rs b/lib/llm/src/fpm_publisher.rs index 49cf3ffd8096..2485de74764c 100644 --- a/lib/llm/src/fpm_publisher.rs +++ b/lib/llm/src/fpm_publisher.rs @@ -16,7 +16,7 @@ use std::time::Duration; use anyhow::Result; use futures::StreamExt; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; @@ -25,10 +25,6 @@ use dynamo_runtime::component::{Component, Endpoint}; use dynamo_runtime::traits::DistributedRuntimeProvider; use dynamo_runtime::transports::event_plane::EventPublisher; -use dynamo_kv_router::protocols::ActiveLoad; - -use crate::kv_router::KV_METRICS_SUBJECT; - use crate::utils::zmq::{connect_sub_socket, multipart_message}; const FPM_TOPIC: &str = "forward-pass-metrics"; @@ -121,12 +117,9 @@ impl FpmEventRelay { let publisher = rt.block_on(async { EventPublisher::for_endpoint(&endpoint, FPM_TOPIC).await })?; - let load_publisher = rt.block_on(async { - EventPublisher::for_endpoint(&endpoint, KV_METRICS_SUBJECT).await - })?; rt.spawn(async move { - Self::relay_loop(zmq_endpoint, publisher, load_publisher, cancel_clone, trace).await; + Self::relay_loop(zmq_endpoint, publisher, cancel_clone, trace).await; }); Ok(Self { cancel }) @@ -140,7 +133,6 @@ impl FpmEventRelay { async fn relay_loop( zmq_endpoint: String, publisher: EventPublisher, - load_publisher: EventPublisher, cancel: CancellationToken, trace: Option, ) { @@ -153,7 +145,6 @@ impl FpmEventRelay { }; let mut socket = socket; tracing::info!("FPM relay: connected to {zmq_endpoint}"); - let mut last_remote_kv_load = None; loop { tokio::select! { @@ -170,26 +161,6 @@ impl FpmEventRelay { if frames.len() == 3 { let payload = bytes::Bytes::from(frames.swap_remove(2)); tap_relay_fpm(&payload, trace.as_ref()); - match remote_kv_active_load(&payload) { - Ok(Some(load)) => { - let snapshot = ( - load.remote_kv_waiting_requests.unwrap_or_default(), - load.remote_kv_waiting_tokens.unwrap_or_default(), - ); - if last_remote_kv_load != Some(snapshot) { - match load_publisher.publish(&load).await { - Ok(()) => last_remote_kv_load = Some(snapshot), - Err(error) => { - tracing::warn!(%error, "FPM relay: remote KV load publish failed"); - } - } - } - } - Ok(None) => {} - Err(error) => { - tracing::warn!(%error, "FPM relay: failed to decode remote KV load telemetry"); - } - } if let Err(e) = publisher.publish_bytes_ref(&payload).await { tracing::warn!("FPM relay: event plane publish failed: {e}"); } @@ -215,43 +186,6 @@ impl FpmEventRelay { } } -#[derive(Debug, Default, Deserialize)] -struct QueuedRemoteKvTelemetry { - #[serde(default)] - num_remote_kv_waiting_requests: u64, - #[serde(default)] - sum_remote_kv_waiting_tokens: u64, -} - -#[derive(Debug, Deserialize)] -struct ForwardPassRemoteKvTelemetry { - worker_id: String, - dp_rank: u32, - #[serde(default)] - queued_requests: QueuedRemoteKvTelemetry, -} - -fn remote_kv_active_load(payload: &[u8]) -> Result> { - let telemetry: ForwardPassRemoteKvTelemetry = rmp_serde::from_slice(payload)?; - let worker_id = match telemetry.worker_id.parse::() { - Ok(worker_id) => worker_id, - Err(_) if telemetry.worker_id.is_empty() => return Ok(None), - Err(error) => { - return Err(anyhow::anyhow!( - "invalid worker_id {:?}: {error}", - telemetry.worker_id - )); - } - }; - Ok(Some(ActiveLoad { - worker_id, - dp_rank: telemetry.dp_rank, - remote_kv_waiting_requests: Some(telemetry.queued_requests.num_remote_kv_waiting_requests), - remote_kv_waiting_tokens: Some(telemetry.queued_requests.sum_remote_kv_waiting_tokens), - ..ActiveLoad::default() - })) -} - impl Drop for FpmEventRelay { fn drop(&mut self) { self.cancel.cancel(); @@ -286,8 +220,6 @@ struct QueuedRequestMetricsSer { num_decode_requests: i32, sum_decode_kv_tokens: i64, var_decode_kv_tokens: f64, - num_remote_kv_waiting_requests: i32, - sum_remote_kv_waiting_tokens: i64, } /// Top-level serialization struct matching Python `ForwardPassMetrics`. @@ -332,8 +264,6 @@ fn serialize_fpm_into( num_decode_requests: snapshot.num_queued_decode as i32, sum_decode_kv_tokens: snapshot.sum_queued_decode_kv_tokens as i64, var_decode_kv_tokens: snapshot.var_queued_decode_kv_tokens, - num_remote_kv_waiting_requests: 0, - sum_remote_kv_waiting_tokens: 0, }, }; metrics @@ -584,8 +514,6 @@ mod tests { num_decode_requests: i32, sum_decode_kv_tokens: i64, var_decode_kv_tokens: f64, - num_remote_kv_waiting_requests: i32, - sum_remote_kv_waiting_tokens: i64, } #[derive(Deserialize, Debug)] #[allow(dead_code)] @@ -617,8 +545,6 @@ mod tests { assert_eq!(decoded.queued_requests.num_prefill_requests, 1); assert_eq!(decoded.queued_requests.sum_prefill_tokens, 128); assert_eq!(decoded.queued_requests.num_decode_requests, 0); - assert_eq!(decoded.queued_requests.num_remote_kv_waiting_requests, 0); - assert_eq!(decoded.queued_requests.sum_remote_kv_waiting_tokens, 0); } #[test] @@ -682,7 +608,7 @@ mod tests { } /// Verify all 7 expected field names appear in scheduled_requests and - /// 8 in queued_requests — matching the Python schema exactly. + /// 6 in queued_requests — matching the Python schema exactly. #[test] fn test_serialize_fpm_field_names() { let snapshot = ForwardPassSnapshot::default(); @@ -724,8 +650,6 @@ mod tests { "num_decode_requests", "sum_decode_kv_tokens", "var_decode_kv_tokens", - "num_remote_kv_waiting_requests", - "sum_remote_kv_waiting_tokens", ]; for key in &expected_queued { assert!( @@ -739,38 +663,4 @@ mod tests { "queued_requests has unexpected extra fields" ); } - - #[test] - fn remote_kv_active_load_decodes_transfer_backlog() { - let payload = rmp_serde::to_vec_named(&serde_json::json!({ - "worker_id": "17", - "dp_rank": 2, - "queued_requests": { - "num_remote_kv_waiting_requests": 3, - "sum_remote_kv_waiting_tokens": 24_576 - } - })) - .unwrap(); - - let load = remote_kv_active_load(&payload).unwrap().unwrap(); - assert_eq!(load.worker_id, 17); - assert_eq!(load.dp_rank, 2); - assert_eq!(load.remote_kv_waiting_requests, Some(3)); - assert_eq!(load.remote_kv_waiting_tokens, Some(24_576)); - assert_eq!(load.active_prefill_tokens, None); - } - - #[test] - fn remote_kv_active_load_defaults_legacy_fpm_to_zero() { - let payload = rmp_serde::to_vec_named(&serde_json::json!({ - "worker_id": "17", - "dp_rank": 0, - "queued_requests": {} - })) - .unwrap(); - - let load = remote_kv_active_load(&payload).unwrap().unwrap(); - assert_eq!(load.remote_kv_waiting_requests, Some(0)); - assert_eq!(load.remote_kv_waiting_tokens, Some(0)); - } } diff --git a/lib/llm/src/http/service/busy_threshold.rs b/lib/llm/src/http/service/busy_threshold.rs index 2aa7d0203fc8..f009c8b5ef78 100644 --- a/lib/llm/src/http/service/busy_threshold.rs +++ b/lib/llm/src/http/service/busy_threshold.rs @@ -68,8 +68,6 @@ pub struct BusyThresholdRequest { pub active_prefill_tokens_threshold: Option, /// The active prefill tokens threshold as fraction of max_num_batched_tokens, or null to just get the current value pub active_prefill_tokens_threshold_frac: Option, - /// External-KV transfer backlog token threshold, or null to just get the current value - pub remote_kv_waiting_tokens_threshold: Option, } /// Response for a threshold operation @@ -83,8 +81,6 @@ pub struct BusyThresholdResponse { pub active_prefill_tokens_threshold: Option, /// The active prefill tokens threshold as fraction of max_num_batched_tokens pub active_prefill_tokens_threshold_frac: Option, - /// External-KV transfer backlog token threshold - pub remote_kv_waiting_tokens_threshold: Option, } /// Response for listing all thresholds @@ -151,7 +147,6 @@ async fn busy_threshold_handler( active_decode_blocks_threshold: request.active_decode_blocks_threshold, active_prefill_tokens_threshold: request.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac: request.active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold: request.remote_kv_waiting_tokens_threshold, }; if let Err(error) = requested_config.validate() { return ( @@ -194,13 +189,11 @@ async fn busy_threshold_handler( active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold, - ) = config.map_or((None, None, None, None), |c| { + ) = config.map_or((None, None, None), |c| { ( c.active_decode_blocks_threshold, c.active_prefill_tokens_threshold, c.active_prefill_tokens_threshold_frac, - c.remote_kv_waiting_tokens_threshold, ) }); @@ -211,7 +204,6 @@ async fn busy_threshold_handler( active_decode_blocks_threshold, active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold, })), ) } @@ -230,7 +222,6 @@ async fn list_busy_thresholds_handler( active_decode_blocks_threshold: config.active_decode_blocks_threshold, active_prefill_tokens_threshold: config.active_prefill_tokens_threshold, active_prefill_tokens_threshold_frac: config.active_prefill_tokens_threshold_frac, - remote_kv_waiting_tokens_threshold: config.remote_kv_waiting_tokens_threshold, }) .collect(), }; diff --git a/lib/llm/src/kv_router/metrics.rs b/lib/llm/src/kv_router/metrics.rs index 44367b7e4e60..cbce00c50b49 100644 --- a/lib/llm/src/kv_router/metrics.rs +++ b/lib/llm/src/kv_router/metrics.rs @@ -5,7 +5,7 @@ //! //! This module centralizes all router-side Prometheus metric definitions: //! -//! - [`WorkerLoadMetrics`]: Per-worker active decode, prefill, and remote-KV-wait gauges. +//! - [`WorkerLoadMetrics`]: Per-worker active decode blocks and prefill tokens gauges. //! Registered on the frontend's own `prometheus::Registry` (default port 8000). //! Populated by `KvWorkerMonitor` in the frontend when receiving ActiveLoad events. //! - Frontend (aggregated and disaggregated): available on default port 8000 @@ -490,12 +490,10 @@ impl RouterWorkerStatusMetrics { // --------------------------------------------------------------------------- /// Per-worker active load gauges, published by `ActiveSequencesMultiWorker` -/// and `KvWorkerMonitor`, then cleaned up by the monitor when workers disappear. +/// and cleaned up by `KvWorkerMonitor` when workers disappear. pub struct WorkerLoadMetrics { pub active_decode_blocks: IntGaugeVec, pub active_prefill_tokens: IntGaugeVec, - pub remote_kv_waiting_requests: IntGaugeVec, - pub remote_kv_waiting_tokens: IntGaugeVec, } impl WorkerLoadMetrics { @@ -517,29 +515,6 @@ impl WorkerLoadMetrics { .with_label_values(labels) .set(active_tokens as i64); } - - pub fn observe_remote_kv_wait( - &self, - worker_id: u64, - dp_rank: u32, - worker_type: &str, - waiting_requests: Option, - waiting_tokens: Option, - ) { - let worker_id_str = worker_id.to_string(); - let dp_rank_str = dp_rank.to_string(); - let labels = &[worker_id_str.as_str(), dp_rank_str.as_str(), worker_type]; - if let Some(waiting_requests) = waiting_requests { - self.remote_kv_waiting_requests - .with_label_values(labels) - .set(waiting_requests as i64); - } - if let Some(waiting_tokens) = waiting_tokens { - self.remote_kv_waiting_tokens - .with_label_values(labels) - .set(waiting_tokens as i64); - } - } } pub static WORKER_LOAD_METRICS: LazyLock = LazyLock::new(|| WorkerLoadMetrics { @@ -567,30 +542,6 @@ pub static WORKER_LOAD_METRICS: LazyLock = LazyLock::new(|| W &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], ) .expect("Failed to create worker_active_prefill_tokens gauge"), - remote_kv_waiting_requests: IntGaugeVec::new( - Opts::new( - format!( - "{}_{}", - name_prefix::FRONTEND, - frontend_service::WORKER_REMOTE_KV_WAITING_REQUESTS - ), - "Requests waiting for an external KV transfer per worker", - ), - &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], - ) - .expect("Failed to create worker_remote_kv_waiting_requests gauge"), - remote_kv_waiting_tokens: IntGaugeVec::new( - Opts::new( - format!( - "{}_{}", - name_prefix::FRONTEND, - frontend_service::WORKER_REMOTE_KV_WAITING_TOKENS - ), - "KV context tokens waiting for an external transfer per worker", - ), - &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], - ) - .expect("Failed to create worker_remote_kv_waiting_tokens gauge"), }); /// Register the worker load gauges with the given Prometheus registry. @@ -601,8 +552,6 @@ pub fn register_worker_load_metrics( let m = &*WORKER_LOAD_METRICS; registry.register(Box::new(m.active_decode_blocks.clone()))?; registry.register(Box::new(m.active_prefill_tokens.clone()))?; - registry.register(Box::new(m.remote_kv_waiting_requests.clone()))?; - registry.register(Box::new(m.remote_kv_waiting_tokens.clone()))?; Ok(()) } @@ -1340,22 +1289,6 @@ mod tests { &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], ) .unwrap(), - remote_kv_waiting_requests: IntGaugeVec::new( - Opts::new( - "dynamo_frontend_worker_remote_kv_waiting_requests", - "Requests waiting for an external KV transfer per worker", - ), - &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], - ) - .unwrap(), - remote_kv_waiting_tokens: IntGaugeVec::new( - Opts::new( - "dynamo_frontend_worker_remote_kv_waiting_tokens", - "KV context tokens waiting for an external transfer per worker", - ), - &[labels::WORKER_ID, labels::DP_RANK, labels::WORKER_TYPE], - ) - .unwrap(), }; registry .register(Box::new(metrics.active_decode_blocks.clone())) @@ -1363,15 +1296,8 @@ mod tests { registry .register(Box::new(metrics.active_prefill_tokens.clone())) .unwrap(); - registry - .register(Box::new(metrics.remote_kv_waiting_requests.clone())) - .unwrap(); - registry - .register(Box::new(metrics.remote_kv_waiting_tokens.clone())) - .unwrap(); metrics.observe(123, 0, "decode", 42, 100); - metrics.observe_remote_kv_wait(123, 0, "decode", Some(3), Some(24_576)); let output = gather_pef(®istry); let expected = "\ @@ -1381,12 +1307,6 @@ dynamo_frontend_worker_active_decode_blocks{dp_rank=\"0\",worker_id=\"123\",work # HELP dynamo_frontend_worker_active_prefill_tokens Active prefill tokens queued per worker # TYPE dynamo_frontend_worker_active_prefill_tokens gauge dynamo_frontend_worker_active_prefill_tokens{dp_rank=\"0\",worker_id=\"123\",worker_type=\"decode\"} 100 -# HELP dynamo_frontend_worker_remote_kv_waiting_requests Requests waiting for an external KV transfer per worker -# TYPE dynamo_frontend_worker_remote_kv_waiting_requests gauge -dynamo_frontend_worker_remote_kv_waiting_requests{dp_rank=\"0\",worker_id=\"123\",worker_type=\"decode\"} 3 -# HELP dynamo_frontend_worker_remote_kv_waiting_tokens KV context tokens waiting for an external transfer per worker -# TYPE dynamo_frontend_worker_remote_kv_waiting_tokens gauge -dynamo_frontend_worker_remote_kv_waiting_tokens{dp_rank=\"0\",worker_id=\"123\",worker_type=\"decode\"} 24576 "; assert_eq!( output, expected, diff --git a/lib/llm/src/kv_router/metrics_subscriber.rs b/lib/llm/src/kv_router/metrics_subscriber.rs index d66a20f3b73f..47706c6452ff 100644 --- a/lib/llm/src/kv_router/metrics_subscriber.rs +++ b/lib/llm/src/kv_router/metrics_subscriber.rs @@ -97,8 +97,6 @@ impl PendingActiveLoads { dp_rank: _, active_decode_blocks, active_prefill_tokens, - remote_kv_waiting_requests, - remote_kv_waiting_tokens, kv_used_blocks, } = load; if active_decode_blocks.is_some() { @@ -107,12 +105,6 @@ impl PendingActiveLoads { if active_prefill_tokens.is_some() { pending.active_prefill_tokens = active_prefill_tokens; } - if remote_kv_waiting_requests.is_some() { - pending.remote_kv_waiting_requests = remote_kv_waiting_requests; - } - if remote_kv_waiting_tokens.is_some() { - pending.remote_kv_waiting_tokens = remote_kv_waiting_tokens; - } if kv_used_blocks.is_some() { pending.kv_used_blocks = kv_used_blocks; } @@ -349,8 +341,6 @@ mod tests { dp_rank, active_decode_blocks, active_prefill_tokens, - remote_kv_waiting_requests: None, - remote_kv_waiting_tokens: None, kv_used_blocks, } } diff --git a/lib/llm/src/kv_router/publisher/worker_metrics.rs b/lib/llm/src/kv_router/publisher/worker_metrics.rs index 1b096eaee80f..1e6145257632 100644 --- a/lib/llm/src/kv_router/publisher/worker_metrics.rs +++ b/lib/llm/src/kv_router/publisher/worker_metrics.rs @@ -184,7 +184,6 @@ impl WorkerMetricsPublisher { active_decode_blocks: metrics.active_decode_blocks, active_prefill_tokens: None, kv_used_blocks: metrics.kv_used_blocks, - ..ActiveLoad::default() }; if let Err(e) = sink.publish(active_load).await { diff --git a/lib/runtime/src/metrics/prometheus_names.rs b/lib/runtime/src/metrics/prometheus_names.rs index 45795d60d09a..f06f43e36dc9 100644 --- a/lib/runtime/src/metrics/prometheus_names.rs +++ b/lib/runtime/src/metrics/prometheus_names.rs @@ -296,12 +296,6 @@ pub mod frontend_service { /// Gauge metric tracking current queued prefill tokens for each worker pub const WORKER_ACTIVE_PREFILL_TOKENS: &str = "worker_active_prefill_tokens"; - /// Requests waiting for external KV transfer completion per worker - pub const WORKER_REMOTE_KV_WAITING_REQUESTS: &str = "worker_remote_kv_waiting_requests"; - - /// KV context tokens waiting for external transfer completion per worker - pub const WORKER_REMOTE_KV_WAITING_TOKENS: &str = "worker_remote_kv_waiting_tokens"; - /// Last observed time to first token per worker (in seconds) /// Gauge metric tracking the most recent TTFT for each worker pub const WORKER_LAST_TIME_TO_FIRST_TOKEN_SECONDS: &str = From 8848b40b76c406013f411461b016f089f4bc4334 Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Mon, 5 Oct 2026 11:49:07 -0700 Subject: [PATCH 12/18] chore: trigger CI Signed-off-by: Shreyas Misra From 47538309b3e1153e8140615aac0a16204d2d2ef1 Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 6 Oct 2026 13:14:31 -0700 Subject: [PATCH 13/18] fix(ci): use mirrored vLLM base for MiniMax builds Signed-off-by: Shreyas Misra --- .github/workflows/pr.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 09428070b404..2818b70dce18 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -386,6 +386,8 @@ jobs: uses: $/.github/workflows/shared-build-image.yml with: framework: vllm + extra_build_args: | + RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base target: runtime cuda_version: '["13.0"]' platform: 'linux/amd64,linux/arm64' @@ -414,6 +416,8 @@ jobs: uses: $/.github/workflows/shared-build-image.yml with: framework: vllm + extra_build_args: | + RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base target: dev cuda_version: '["13.0"]' platform: 'linux/amd64,linux/arm64' @@ -437,6 +441,7 @@ jobs: build_timeout_minutes: 60 # local-dev remaps to the host user; ARG USER_UID/USER_GID have no defaults. extra_build_args: | + RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base USER_UID=1000 USER_GID=1000 secrets: inherit @@ -604,6 +609,8 @@ jobs: uses: $/.github/workflows/shared-build-image.yml with: framework: vllm + extra_build_args: | + RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base target: runtime cuda_version: '["13.0"]' platform: 'linux/amd64,linux/arm64' From 6c8d6038e90b44774f90d2553cc69a6a3c18503e Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 6 Oct 2026 14:25:46 -0700 Subject: [PATCH 14/18] fix(ci): pin MiniMax runtime tag in PR build arguments Signed-off-by: Shreyas Misra --- .github/workflows/pr.yaml | 4 ++++ container/context.yaml | 6 ++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2818b70dce18..1c14004c329e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -388,6 +388,7 @@ jobs: framework: vllm extra_build_args: | RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base + RUNTIME_IMAGE_TAG=vllm-nightly-8a728663c1c3 target: runtime cuda_version: '["13.0"]' platform: 'linux/amd64,linux/arm64' @@ -418,6 +419,7 @@ jobs: framework: vllm extra_build_args: | RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base + RUNTIME_IMAGE_TAG=vllm-nightly-8a728663c1c3 target: dev cuda_version: '["13.0"]' platform: 'linux/amd64,linux/arm64' @@ -442,6 +444,7 @@ jobs: # local-dev remaps to the host user; ARG USER_UID/USER_GID have no defaults. extra_build_args: | RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base + RUNTIME_IMAGE_TAG=vllm-nightly-8a728663c1c3 USER_UID=1000 USER_GID=1000 secrets: inherit @@ -611,6 +614,7 @@ jobs: framework: vllm extra_build_args: | RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base + RUNTIME_IMAGE_TAG=vllm-nightly-8a728663c1c3 target: runtime cuda_version: '["13.0"]' platform: 'linux/amd64,linux/arm64' diff --git a/container/context.yaml b/container/context.yaml index 8e307a03af4c..902c70abdb06 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -75,11 +75,9 @@ dynamo: vllm: cuda13.0: base_image: nvcr.io/nvidia/cuda-dl-base - runtime_image: public.ecr.aws/q9t5s3a7/vllm-release-repo + runtime_image: vllm/vllm-openai base_image_tag: 25.11-cuda13.0-devel-ubuntu24.04 - # vLLM commit 8a728663c1c3eeace834a95f5654fa653cc1998c. The multi-arch - # manifest resolves to sha256:f5df5cc3302b5f404848c4eca88d7bf7ed5226e151c056da22816d7734644d67. - runtime_image_tag: 8a728663c1c3eeace834a95f5654fa653cc1998c + runtime_image_tag: v0.28.0-ubuntu2404 # Baseline is vllm-openai's TRUE base, nvidia/cuda:13.0.2-base-ubuntu24.04 # (Docker Hub), NOT a self-baseline of the third-party vllm/vllm-openai image. # vllm-openai is built FROM nvidia/cuda, so this attributes everything vLLM From e423a0fde5fa1ff5443687a1d1bbd0ad5afe9e06 Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 6 Oct 2026 15:14:05 -0700 Subject: [PATCH 15/18] chore(ci): retrigger MiniMax M3 image builds Signed-off-by: Shreyas Misra From e01c0aa638b3d20e35b29f65c828fe7e52d033bf Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Tue, 6 Oct 2026 16:08:48 -0700 Subject: [PATCH 16/18] fix(vllm): align MiniMax nightly APIs and exclude Omni Signed-off-by: Shreyas Misra --- .github/actions/compliance-extract/action.yml | 18 ++++++++++ .github/workflows/pr.yaml | 4 +++ .github/workflows/shared-build-image.yml | 1 + components/src/dynamo/frontend/prepost.py | 6 +--- .../src/dynamo/frontend/vllm_protocol.py | 31 +++++++++++++++++ .../vllm/tests/test_vllm_api_contract.py | 33 +++++++++++-------- .../vllm/tests/test_vllm_kv_events_api.py | 28 +++++++++++++--- .../vllm/tests/test_vllm_renderer_api.py | 11 ++++++- container/deps/vllm/install_vllm_omni.sh | 17 ++++++++++ container/templates/args.Dockerfile | 1 + container/templates/vllm_runtime.Dockerfile | 1 + tests/frontend/test_prepost.py | 2 +- tests/frontend/test_prepost_mistral.py | 2 +- tests/report_pytest_markers.py | 1 + 14 files changed, 130 insertions(+), 26 deletions(-) create mode 100644 components/src/dynamo/frontend/vllm_protocol.py diff --git a/.github/actions/compliance-extract/action.yml b/.github/actions/compliance-extract/action.yml index b470a2fa6935..c14cbb04e838 100644 --- a/.github/actions/compliance-extract/action.yml +++ b/.github/actions/compliance-extract/action.yml @@ -57,6 +57,10 @@ inputs: description: 'EPP image reference (frontend only) — must equal what the build passed as EPP_IMAGE so the epp stage resolves the freshly-built ECR image rather than the released tag rendered from container/context.yaml. The compliance stages read /epp''s SBOM and license texts out of this stage, so a mismatch would attribute a different build of the binary than the one shipped. Empty for non-frontend targets, which have no epp stage.' required: false default: '' + extra_build_args: + description: 'Newline-separated build arguments; must match the image build, including base image and optional packages.' + required: false + default: '' sccache_bucket: description: 'SCCACHE S3 bucket — must equal what the build passed so wheel_builder_base''s SCCACHE_* ARGs/ENV match and the heavy Rust layer cache-hits. Normally the extract never runs sccache (everything cache-hits), but when the shared builder has evicted layers a stage re-runs here for real, so the extract also passes the IRSA secrets (see the buildx steps) to let that sccache authenticate.' required: false @@ -129,6 +133,7 @@ runs: SCCACHE_REGION: ${{ inputs.sccache_region }} GIT_SHA: ${{ inputs.git_sha }} EPP_IMAGE: ${{ inputs.epp_image }} + EXTRA_BUILD_ARGS: ${{ inputs.extra_build_args }} run: | mkdir -p /tmp/compliance # Reuse the warm cache the build just populated by matching ALL the @@ -174,6 +179,11 @@ runs: # --build-arg (it would clear the ARG and reintroduce the failure). EPP_IMAGE_ARG="" [ -n "${EPP_IMAGE:-}" ] && EPP_IMAGE_ARG="--build-arg EPP_IMAGE=${EPP_IMAGE}" + EXTRA_ARGS=() + while IFS= read -r build_arg || [ -n "$build_arg" ]; do + [ -z "$build_arg" ] && continue + EXTRA_ARGS+=(--build-arg "$build_arg") + done <<< "$EXTRA_BUILD_ARGS" docker buildx build \ --builder ${{ inputs.builder_name }} \ --platform ${{ inputs.platform }} \ @@ -183,6 +193,7 @@ runs: --build-arg SCCACHE_BUCKET="${SCCACHE_BUCKET}" \ --build-arg SCCACHE_REGION="${SCCACHE_REGION}" \ $EPP_IMAGE_ARG \ + "${EXTRA_ARGS[@]}" \ --output type=local,dest=/tmp/compliance \ --progress=plain \ -f "${DOCKERFILE_PATH}" \ @@ -356,6 +367,7 @@ runs: SCCACHE_REGION: ${{ inputs.sccache_region }} GIT_SHA: ${{ inputs.git_sha }} EPP_IMAGE: ${{ inputs.epp_image }} + EXTRA_BUILD_ARGS: ${{ inputs.extra_build_args }} run: | mkdir -p /tmp/sources # ENABLE_SOURCE_ARCHIVAL=true to actually run sources_collect; the other @@ -385,6 +397,11 @@ runs: # resolves the GC'd default staging tag. Empty for non-frontend targets. EPP_IMAGE_ARG="" [ -n "${EPP_IMAGE:-}" ] && EPP_IMAGE_ARG="--build-arg EPP_IMAGE=${EPP_IMAGE}" + EXTRA_ARGS=() + while IFS= read -r build_arg || [ -n "$build_arg" ]; do + [ -z "$build_arg" ] && continue + EXTRA_ARGS+=(--build-arg "$build_arg") + done <<< "$EXTRA_BUILD_ARGS" docker buildx build \ --builder ${{ inputs.builder_name }} \ --platform ${{ inputs.platform }} \ @@ -395,6 +412,7 @@ runs: --build-arg SCCACHE_REGION="${SCCACHE_REGION}" \ --build-arg ENABLE_SOURCE_ARCHIVAL=true \ $EPP_IMAGE_ARG \ + "${EXTRA_ARGS[@]}" \ --output type=local,dest=/tmp/sources \ --progress=plain \ -f "${DOCKERFILE_PATH}" \ diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1c14004c329e..50201b8de5b9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -389,6 +389,7 @@ jobs: extra_build_args: | RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base RUNTIME_IMAGE_TAG=vllm-nightly-8a728663c1c3 + ENABLE_VLLM_OMNI=false target: runtime cuda_version: '["13.0"]' platform: 'linux/amd64,linux/arm64' @@ -420,6 +421,7 @@ jobs: extra_build_args: | RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base RUNTIME_IMAGE_TAG=vllm-nightly-8a728663c1c3 + ENABLE_VLLM_OMNI=false target: dev cuda_version: '["13.0"]' platform: 'linux/amd64,linux/arm64' @@ -445,6 +447,7 @@ jobs: extra_build_args: | RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base RUNTIME_IMAGE_TAG=vllm-nightly-8a728663c1c3 + ENABLE_VLLM_OMNI=false USER_UID=1000 USER_GID=1000 secrets: inherit @@ -615,6 +618,7 @@ jobs: extra_build_args: | RUNTIME_IMAGE=${{ vars.ECR_REGISTRY }}/ai-dynamo/vllm-base RUNTIME_IMAGE_TAG=vllm-nightly-8a728663c1c3 + ENABLE_VLLM_OMNI=false target: runtime cuda_version: '["13.0"]' platform: 'linux/amd64,linux/arm64' diff --git a/.github/workflows/shared-build-image.yml b/.github/workflows/shared-build-image.yml index ace6d4d752fb..a6c55e158aef 100644 --- a/.github/workflows/shared-build-image.yml +++ b/.github/workflows/shared-build-image.yml @@ -578,6 +578,7 @@ jobs: # EPP_IMAGE so the epp stage cache-hits instead of resolving the # rendered default upstream staging tag, which gets garbage-collected. epp_image: ${{ steps.build-epp-image.outputs.epp_image_uri || '' }} + extra_build_args: ${{ inputs.extra_build_args }} # Match the build's sccache build-args so the extract reuses the warm # wheel_builder/pre_runtime cache instead of a cold rebuild. sccache_bucket: ${{ secrets.SCCACHE_S3_BUCKET }} diff --git a/components/src/dynamo/frontend/prepost.py b/components/src/dynamo/frontend/prepost.py index e6a6f0b5f1b2..fbec56615a52 100644 --- a/components/src/dynamo/frontend/prepost.py +++ b/components/src/dynamo/frontend/prepost.py @@ -19,11 +19,6 @@ ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, ) -from vllm.entrypoints.openai.engine.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, -) from vllm.reasoning import ReasoningParser from vllm.renderers import ChatParams, merge_kwargs from vllm.sampling_params import SamplingParams @@ -37,6 +32,7 @@ from .thinking import apply_default_thinking_mode_to_template_kwargs from .utils import legacy_guided_decoding +from .vllm_protocol import DeltaFunctionCall, DeltaMessage, DeltaToolCall if TYPE_CHECKING: from vllm.config import ModelConfig diff --git a/components/src/dynamo/frontend/vllm_protocol.py b/components/src/dynamo/frontend/vllm_protocol.py new file mode 100644 index 000000000000..aa3d86fee7ca --- /dev/null +++ b/components/src/dynamo/frontend/vllm_protocol.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Protocol types shared by the released and MiniMax nightly vLLM layouts.""" + +try: + from vllm.entrypoints.generate.base.protocol import ( + DeltaFunctionCall as DeltaFunctionCall, + ) + from vllm.entrypoints.generate.base.protocol import DeltaMessage as DeltaMessage + from vllm.entrypoints.generate.base.protocol import DeltaToolCall as DeltaToolCall + from vllm.entrypoints.generate.base.protocol import ( + FunctionDefinition as FunctionDefinition, + ) +except ModuleNotFoundError as exc: + # The nightly moved these types out of openai.engine. Only fall back for + # that module move; missing dependencies inside the new module must fail. + if exc.name not in { + "vllm.entrypoints.generate", + "vllm.entrypoints.generate.base", + "vllm.entrypoints.generate.base.protocol", + }: + raise + from vllm.entrypoints.openai.engine.protocol import ( + DeltaFunctionCall as DeltaFunctionCall, + ) + from vllm.entrypoints.openai.engine.protocol import DeltaMessage as DeltaMessage + from vllm.entrypoints.openai.engine.protocol import DeltaToolCall as DeltaToolCall + from vllm.entrypoints.openai.engine.protocol import ( + FunctionDefinition as FunctionDefinition, + ) diff --git a/components/src/dynamo/vllm/tests/test_vllm_api_contract.py b/components/src/dynamo/vllm/tests/test_vllm_api_contract.py index f6910fc2c2cb..9281f9706ed4 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_api_contract.py +++ b/components/src/dynamo/vllm/tests/test_vllm_api_contract.py @@ -94,20 +94,25 @@ def test_new_request_data_has_fields_instrumented_scheduler_sets(): ) -def test_scheduler_output_new_connector_fields_remain_optional(): - """Dynamo constructs SchedulerOutput directly without v0.27's new fields.""" - from vllm.v1.core.sched.output import SchedulerOutput - - fields = {field.name: field for field in dataclasses.fields(SchedulerOutput)} - for field_name in ("ec_manager_metadata", "partial_tail_offloads"): - assert field_name in fields, ( - f"vLLM SchedulerOutput.{field_name} is gone — re-audit Dynamo's " - "direct SchedulerOutput construction." - ) - assert fields[field_name].default is None, ( - f"vLLM SchedulerOutput.{field_name} became required — update every " - "InstrumentedScheduler constructor." - ) +def test_scheduler_output_construction_without_optional_connector_fields(): + """The benchmark scheduler can construct an empty output on both vLLM pins.""" + from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput + + output = SchedulerOutput( + scheduled_new_reqs=[], + scheduled_cached_reqs=CachedRequestData.make_empty(), + num_scheduled_tokens={}, + total_num_scheduled_tokens=0, + scheduled_spec_decode_tokens={}, + scheduled_encoder_inputs={}, + num_common_prefix_blocks=[0], + finished_req_ids=set(), + free_encoder_mm_hashes=[], + ) + assert output.total_num_scheduled_tokens == 0 + assert output.kv_connector_metadata is None + assert output.ec_connector_metadata is None + assert output.ec_manager_metadata is None def test_async_scheduler_has_methods_instrumented_scheduler_overrides(): diff --git a/components/src/dynamo/vllm/tests/test_vllm_kv_events_api.py b/components/src/dynamo/vllm/tests/test_vllm_kv_events_api.py index 2040e664a2cb..dfcfd700d840 100755 --- a/components/src/dynamo/vllm/tests/test_vllm_kv_events_api.py +++ b/components/src/dynamo/vllm/tests/test_vllm_kv_events_api.py @@ -12,7 +12,7 @@ This test is the early warning for vLLM KV-event wire-format changes. In the normal case, if this fails, first check whether -`lib/kv-router/src/zmq_wire.rs` already accepts the new upstream vLLM event +`lib/kv-router/src/zmq_wire/deserialize.rs` already accepts the new upstream vLLM event shape. If not, update that compatibility layer before updating this test. That file is Dynamo's compatibility layer for vLLM KV events: @@ -83,6 +83,8 @@ def test_block_stored_fields(self): 10. kv_cache_spec_kind (semantic cache type; optional for older vLLM) 11. kv_cache_spec_sliding_window (semantic cache window; optional for older vLLM) 12. locality (per-tier storage locality; optional for older vLLM) + 13. ownership (offloading tier; optional for older vLLM) + 14. session_id (request attribution; optional for older vLLM) If vLLM adds/removes/reorders fields, this test will fail. """ @@ -104,6 +106,10 @@ def test_block_stored_fields(self): expected_fields.append("kv_cache_spec_sliding_window") if _has_locality(BlockStored): expected_fields.append("locality") + if "ownership" in BlockStored.__struct_fields__: + expected_fields.append("ownership") + if "session_id" in BlockStored.__struct_fields__: + expected_fields.append("session_id") expected_fields = tuple(expected_fields) actual_fields = BlockStored.__struct_fields__ @@ -112,7 +118,7 @@ def test_block_stored_fields(self): f"Expected: {expected_fields}\n" f"Actual: {actual_fields}\n" f"Required follow-up:\n" - f" - Update lib/kv-router/src/zmq_wire.rs to match the new BlockStored wire format.\n" + f" - Update lib/kv-router/src/zmq_wire/deserialize.rs to match the new BlockStored wire format.\n" f" - Update this test's expected_fields and msgpack shape checks.\n" f" - If needed, add or update a regression test in lib/llm/src/kv_router/publisher.rs." ) @@ -131,6 +137,8 @@ def test_block_removed_fields(self): expected_fields.append("kv_cache_spec_sliding_window") if _has_locality(BlockRemoved): expected_fields.append("locality") + if "ownership" in BlockRemoved.__struct_fields__: + expected_fields.append("ownership") expected_fields = tuple(expected_fields) actual_fields = BlockRemoved.__struct_fields__ @@ -139,7 +147,7 @@ def test_block_removed_fields(self): f"Expected: {expected_fields}\n" f"Actual: {actual_fields}\n" f"Required follow-up:\n" - f" - Update lib/kv-router/src/zmq_wire.rs RawKvEvent::BlockRemoved seq deserializer.\n" + f" - Update lib/kv-router/src/zmq_wire/deserialize.rs RawKvEvent::BlockRemoved seq deserializer.\n" f" - Update this test's expected_fields." ) @@ -157,7 +165,7 @@ def test_event_batch_fields(self): f"Expected: {expected_fields}\n" f"Actual: {actual_fields}\n" f"Required follow-up:\n" - f" - Update lib/kv-router/src/zmq_wire.rs KvEventBatch Deserialize impl.\n" + f" - Update lib/kv-router/src/zmq_wire/deserialize.rs KvEventBatch Deserialize impl.\n" f" - Update subscriber.rs VllmEventBatch tuple if batch field order changes.\n" f" - Update this test's expected_fields." ) @@ -211,6 +219,10 @@ def test_block_stored_serialization_format(self): event_kwargs["kv_cache_spec_sliding_window"] = 128 if _has_locality(BlockStored): event_kwargs["locality"] = "LOCAL" + if "ownership" in BlockStored.__struct_fields__: + event_kwargs["ownership"] = "kvcr" + if "session_id" in BlockStored.__struct_fields__: + event_kwargs["session_id"] = "session-1" event = BlockStored(**event_kwargs) encoded = msgspec.msgpack.encode(event) @@ -234,6 +246,10 @@ def test_block_stored_serialization_format(self): assert decoded["kv_cache_spec_sliding_window"] == 128 if _has_locality(BlockStored): assert decoded["locality"] == "LOCAL" + if "ownership" in BlockStored.__struct_fields__: + assert decoded["ownership"] == "kvcr" + if "session_id" in BlockStored.__struct_fields__: + assert decoded["session_id"] == "session-1" def test_block_stored_tuple_extra_keys_serialization_format(self): """Verify multimodal tuple extra_keys keep the vLLM 0.19 wire shape.""" @@ -288,6 +304,8 @@ def test_block_removed_serialization_format(self): event_kwargs["kv_cache_spec_sliding_window"] = 128 if _has_locality(BlockRemoved): event_kwargs["locality"] = "REMOTE" + if "ownership" in BlockRemoved.__struct_fields__: + event_kwargs["ownership"] = "kvcr" event = BlockRemoved(**event_kwargs) decoded = msgspec.msgpack.decode(msgspec.msgpack.encode(event)) @@ -305,3 +323,5 @@ def test_block_removed_serialization_format(self): ), "kv_cache_spec_sliding_window has wrong value" if _has_locality(BlockRemoved): assert decoded["locality"] == "REMOTE" + if "ownership" in BlockRemoved.__struct_fields__: + assert decoded["ownership"] == "kvcr" diff --git a/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py b/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py index 44383e73eeee..4bfbab2bcefc 100755 --- a/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py +++ b/components/src/dynamo/vllm/tests/test_vllm_renderer_api.py @@ -22,7 +22,7 @@ _chat_protocol = importlib.import_module( "vllm.entrypoints.openai.chat_completion.protocol" ) -_engine_protocol = importlib.import_module("vllm.entrypoints.openai.engine.protocol") +_engine_protocol = importlib.import_module("dynamo.frontend.vllm_protocol") _inputs_data = importlib.import_module("vllm.inputs") _reasoning = importlib.import_module("vllm.reasoning") _sampling_params = importlib.import_module("vllm.sampling_params") @@ -480,6 +480,13 @@ def test_engine_core_struct_contract(self): core_output_fields = core_output_fields + tuple( fields + vllm_028_output_extra_fields for fields in core_output_fields ) + # MiniMax's nightly appends optional per-request speculative metrics. + # Dynamo reconstructs outputs without these engine-side metrics. + core_output_fields = core_output_fields + ( + base_output_fields + + vllm_028_output_extra_fields + + ("spec_decode_metrics",), + ) valid_output_fields = core_output_fields + tuple( fields + omni_output_extra_fields for fields in core_output_fields ) @@ -518,6 +525,8 @@ def test_engine_core_output_construction(self): assert output.mm_cache_miss_hashes is None if "new_sampling_mask" in EngineCoreOutput.__struct_fields__: assert output.new_sampling_mask is None + if "spec_decode_metrics" in EngineCoreOutput.__struct_fields__: + assert output.spec_decode_metrics is None assert output.finish_reason is FinishReason.STOP assert output.stop_reason == "eos" diff --git a/container/deps/vllm/install_vllm_omni.sh b/container/deps/vllm/install_vllm_omni.sh index 929e9a4821cd..37cb81630480 100755 --- a/container/deps/vllm/install_vllm_omni.sh +++ b/container/deps/vllm/install_vllm_omni.sh @@ -4,6 +4,23 @@ set -euo pipefail +# MiniMax nightly images exclude Omni because the released Omni package uses +# removed vLLM APIs. Remove an inherited installation as well as skipping ours. +case "${ENABLE_VLLM_OMNI:-true}" in + false) + PIP_TARGET_ARGS=() + if [ "${VLLM_OMNI_TARGET_DEVICE}" = "cuda" ]; then + PIP_TARGET_ARGS+=(--system) + fi + uv pip uninstall "${PIP_TARGET_ARGS[@]}" vllm-omni + python3 -c 'import importlib.util; assert importlib.util.find_spec("vllm_omni") is None, "vLLM-Omni must be absent when disabled"' + echo "vLLM-Omni explicitly disabled" + exit 0 + ;; + true) ;; + *) echo "ENABLE_VLLM_OMNI must be true or false" >&2; exit 1 ;; +esac + : "${VLLM_OMNI_REF:?VLLM_OMNI_REF must be set}" VLLM_OMNI_PROTECTED_PACKAGES_FILE="${VLLM_OMNI_PROTECTED_PACKAGES_FILE:-/tmp/vllm_omni_protected_packages.txt}" diff --git a/container/templates/args.Dockerfile b/container/templates/args.Dockerfile index 7927994ba4a5..4eaa02c8f1c8 100644 --- a/container/templates/args.Dockerfile +++ b/container/templates/args.Dockerfile @@ -100,6 +100,7 @@ ARG PLANNER_RUNTIME_IMAGE_TAG={{ context.dynamo.planner_runtime_image_tag }} ARG MAX_JOBS={{ context.vllm.max_jobs }} ARG TRANSFORMERS_VERSION={{ context.vllm.transformers_version }} ARG TOKENIZERS_VERSION={{ context.vllm.tokenizers_version }} +ARG ENABLE_VLLM_OMNI=true ARG VLLM_OMNI_REF={{ context.vllm[device_key].get("vllm_omni_ref", context.vllm.vllm_omni_ref) }} {% if device == "cuda" -%} diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 072434f81e88..aca0cc22216c 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -18,6 +18,7 @@ FROM ${RUNTIME_IMAGE}:${RUNTIME_IMAGE_TAG} AS pre_runtime ARG PYTHON_VERSION ARG ENABLE_KVBM ARG ENABLE_GPU_MEMORY_SERVICE +ARG ENABLE_VLLM_OMNI ARG VLLM_OMNI_REF ARG TRANSFORMERS_VERSION ARG TOKENIZERS_VERSION diff --git a/tests/frontend/test_prepost.py b/tests/frontend/test_prepost.py index 9a1b6b3a372e..0f8fef369162 100644 --- a/tests/frontend/test_prepost.py +++ b/tests/frontend/test_prepost.py @@ -24,12 +24,12 @@ ChatCompletionRequest, ChatCompletionToolsParam, ) - from vllm.entrypoints.openai.engine.protocol import FunctionDefinition from vllm.outputs import CompletionOutput from vllm.sampling_params import SamplingParams from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser from dynamo.frontend.prepost import StreamingPostProcessor, _prepare_request + from dynamo.frontend.vllm_protocol import FunctionDefinition else: # Fake some types so that `pre-commit` passes class CompletionOutput: diff --git a/tests/frontend/test_prepost_mistral.py b/tests/frontend/test_prepost_mistral.py index 1a034b7cac01..e16ad355fd9f 100644 --- a/tests/frontend/test_prepost_mistral.py +++ b/tests/frontend/test_prepost_mistral.py @@ -19,7 +19,6 @@ ChatCompletionRequest, ChatCompletionToolsParam, ) - from vllm.entrypoints.openai.engine.protocol import FunctionDefinition from vllm.outputs import CompletionOutput from vllm.reasoning import ReasoningParserManager from vllm.sampling_params import SamplingParams @@ -29,6 +28,7 @@ from dynamo.frontend.prepost import StreamingPostProcessor MistralReasoningParser = ReasoningParserManager.get_reasoning_parser("mistral") + from dynamo.frontend.vllm_protocol import FunctionDefinition else: # Fake some types so that `pre-commit` passes class MistralTokenizer: diff --git a/tests/report_pytest_markers.py b/tests/report_pytest_markers.py index fcdad9c0692f..c145835b8e22 100755 --- a/tests/report_pytest_markers.py +++ b/tests/report_pytest_markers.py @@ -203,6 +203,7 @@ "vllm.entrypoints.openai.chat_completion.protocol", "vllm.entrypoints.openai.engine", "vllm.entrypoints.openai.engine.protocol", + "vllm.entrypoints.generate.base.protocol", "vllm.entrypoints.pooling", "vllm.entrypoints.pooling.embed", "vllm.entrypoints.pooling.embed.protocol", From 09af0588991425f67886a37e9eebb1f17cfff13b Mon Sep 17 00:00:00 2001 From: NVShreyas <158103197+NVShreyas@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:51:29 -0700 Subject: [PATCH 17/18] Update container/templates/vllm_runtime.Dockerfile Co-authored-by: Dmitry Tokarev Signed-off-by: NVShreyas <158103197+NVShreyas@users.noreply.github.com> --- container/templates/vllm_runtime.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index aca0cc22216c..ceefebccfb48 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -205,7 +205,7 @@ RUN --mount=type=cache,id=uv-root-{{ context.dynamo.uv_version }},target=/root/. RUN --mount=type=bind,source=./container/deps/vllm/patches,target=/tmp/vllm_patches,readonly \ SITE_PACKAGES="$(python3 -c 'import site; print(site.getsitepackages()[0])')" && \ for p in /tmp/vllm_patches/*.patch; do \ - patch --batch --forward --fuzz=5 -p1 -d "${SITE_PACKAGES}" < "$p"; \ + patch --batch --forward --fuzz=5 -p1 -d "${SITE_PACKAGES}" < "$p" || exit 1; \ done {% endif %} From e46c79195aa3b138aa99e4a3e1cbddb0f4c54b14 Mon Sep 17 00:00:00 2001 From: Shreyas Misra Date: Wed, 7 Oct 2026 10:04:03 -0700 Subject: [PATCH 18/18] fix(frontend): use container duration for WebM decoding Fall back to container duration when the video stream duration is unknown. Cover the reported finalized VP9 WebM through the real decoder with all-frame, FPS-based, and fixed-frame sampling. Fixes #15811 Signed-off-by: Shreyas Misra --- .../src/preprocessor/media/decoders/video.rs | 62 +++++++++++++++++- lib/llm/tests/data/media/224p_6_vp9.webm | Bin 0 -> 846 bytes 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 lib/llm/tests/data/media/224p_6_vp9.webm diff --git a/lib/llm/src/preprocessor/media/decoders/video.rs b/lib/llm/src/preprocessor/media/decoders/video.rs index 3a5d7f26d860..4d3dcfc3a629 100644 --- a/lib/llm/src/preprocessor/media/decoders/video.rs +++ b/lib/llm/src/preprocessor/media/decoders/video.rs @@ -239,10 +239,17 @@ fn decode_video(config: &VideoDecoder, bytes: Vec) -> Result 0, "Cannot determine the video frame rate" ); + let source_duration = if input_stream.duration() > 0 { + Time::new(Some(input_stream.duration()), stream_time_base).as_secs() as f64 + } else { + // WebM may expose duration only at container level, in AV_TIME_BASE units. + // Clamp unknown (AV_NOPTS_VALUE) or nonpositive durations to zero. + input.duration().max(0) as f64 / f64::from(ffmpeg_next::ffi::AV_TIME_BASE) + }; ( input_stream.index(), stream_time_base, - Time::new(Some(input_stream.duration()), stream_time_base).as_secs() as f64, + source_duration, (frame_rate.numerator() as f32 / frame_rate.denominator() as f32) as f64, input_stream.frames().max(0) as u64, input_stream.parameters(), @@ -537,6 +544,59 @@ mod tests { assert!(msg.contains("libvpx-vp9"), "no re-encode hint in: {msg}"); } + #[rstest] + #[case(None, None, 6)] + #[case(Some(1.0), None, 3)] + #[case(None, Some(2), 2)] + fn test_decode_webm_with_container_duration( + #[case] fps: Option, + #[case] num_frames: Option, + #[case] expected_frames: usize, + ) { + // Finalized six-frame VP9 WebM from the reported data-URL request. + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/data/media/224p_6_vp9.webm" + ); + let input = ffmpeg_next::format::input(&path).unwrap(); + let stream = input + .streams() + .best(ffmpeg_next::media::Type::Video) + .unwrap(); + assert_eq!(stream.duration(), ffmpeg_next::ffi::AV_NOPTS_VALUE); + assert_eq!(stream.frames(), 0); + assert_eq!( + input.duration(), + 3 * i64::from(ffmpeg_next::ffi::AV_TIME_BASE) + ); + + let decoder = VideoDecoder { + fps, + num_frames, + strict: true, + ..Default::default() + }; + let decoded = decoder + .decode(EncodedMediaData::from_bytes(std::fs::read(path).unwrap())) + .unwrap(); + assert_eq!( + decoded.tensor_info.shape, + vec![expected_frames, 224, 224, 3] + ); + let Some(DecodedMediaMetadata::Video(metadata)) = decoded.tensor_info.metadata else { + panic!("missing video metadata"); + }; + assert_eq!(metadata.source_duration, 3.0); + assert_eq!(metadata.source_fps, 2.0); + assert_eq!(metadata.sampled_timestamps.len(), expected_frames); + assert!( + metadata + .sampled_timestamps + .iter() + .all(|t| (0.0..3.0).contains(t)) + ); + } + #[test] fn test_decode_video_num_frames() { let (encoded_data, width, height, _total_frames) = load_test_video("240p_10.mp4"); diff --git a/lib/llm/tests/data/media/224p_6_vp9.webm b/lib/llm/tests/data/media/224p_6_vp9.webm new file mode 100644 index 0000000000000000000000000000000000000000..180e668f6be2be0a2ac3295d0cf103e381d5ea6a GIT binary patch literal 846 zcmb1gy}x+AQ(GgW({~{L)X3uWxsk)EsiizMDc7mJk;$pGkx3%BA)S!{1eoOneP@^K z^4;AXyt+lyb7flan#P3?o><7bY#{HN@9rL;oKVQ&^x!p3j0zC0Oz>`?WD_&T8;Cg! zkznI!L4aJfLF@X)W_~9J--aHa#IiIqLp^gnLj!|wM-;A0XNSY`0NcC} zjf}-dJN_ML&B~Ce&7IT8ptN;Pb7@{mTWeT+Sb$}7Bct-;<`&trsSFP$Y-oJ2tMS3C zMyAjWEulLb89^R*6e}+NmVBTg{lrw)#ofM+Uj9MxPC<@-E-vjYnZ^0J=_jVSF7Ea9 z^aF~AxdsJ$`un-GH=7z7rJq>rx_F|qzi)tJh^Lc}D@0eYOM4GU8xR}i7MK{OpP1>o zxZTyy+26%A2q=f_#o}UznaKyblM&8OKQYgBagPVkb|2TEct1y9SD==#%#_r8kHox` zoYbQ96SG|xcZRw8xd5Hx33Ovmv_UK!rJq=eY}pK;Wy!$sG_(W-s6tL=QdvQTZdrk4 z`iX_Ei#uFGgMcmvd1tbLffWcD>lwfRgS_*$Qio&D8yOaFY-C_)X!2}!V&MJ2`;Eau zMMlC4D3bWiAVap=fkDVNy@64@aMrTSJw2_3Vyp9`7#2@&WcQA;q8F5+C+_aw+{m!K3z&00HZj?J HXk-QeKkyUm literal 0 HcmV?d00001