From 380b416ed3143b04373835017353f45606d6eccc Mon Sep 17 00:00:00 2001 From: Adit Ranadive Date: Mon, 2 Nov 2026 13:34:52 -0800 Subject: [PATCH] feat(kvcr): supervise Guard startup and health Signed-off-by: Adit Ranadive --- examples/backends/vllm/deploy/kvcr/README.md | 173 +++++++-- .../vllm/deploy/kvcr/agg-memory-service.yaml | 103 ++--- examples/backends/vllm/deploy/kvcr/deploy.sh | 81 +++- .../backends/vllm/deploy/kvcr/services.sh | 36 ++ tests/deploy/test_kvcr_guard.py | 115 +++++- tests/deploy/test_kvcr_manifests.py | 354 +++++++++++++++++- 6 files changed, 735 insertions(+), 127 deletions(-) create mode 100755 examples/backends/vllm/deploy/kvcr/services.sh diff --git a/examples/backends/vllm/deploy/kvcr/README.md b/examples/backends/vllm/deploy/kvcr/README.md index 01220b0a7230..a4bfadd6f5a3 100644 --- a/examples/backends/vllm/deploy/kvcr/README.md +++ b/examples/backends/vllm/deploy/kvcr/README.md @@ -23,30 +23,29 @@ KVCR-capable Dynamo vLLM runtime image for every process. and UCX builds. - Linux 6.5 or a kernel with equivalent `SO_PEERPIDFD` support for the memory-service variant. -- `envsubst` and `kubectl` on the deployment host. +- `envsubst`, `kubectl`, and Python 3 on the deployment host. The manifests default to the `rdma/shared_ib` extended resource. The process-local variant requests one share per GPU node; the memory-service variant requests two shares per node because both `main` and `kvcr-services` use RDMA. Set `DYNAMO_RDMA_RESOURCE=rdma/ib` or another cluster-specific resource name when -required. `UCX_TLS=rc_x,cuda` prevents TCP fallback. Set -`DYNAMO_UCX_NET_DEVICES` to the GPU-local HCA and port exposed by the selected -RDMA resource, such as `mlx5_0:1`; do not copy that example device name without -checking the target nodes. +required. `UCX_TLS=rc_x,cuda` prevents TCP payload fallback. Set +`DYNAMO_UCX_NET_DEVICES=all` for per-container single-HCA allocations, or pin +an HCA and port exposed in every engine and sidecar, such as `mlx5_0:1`. The image tag must contain a Dynamo semantic release version. Pin the same image by digest for every component, for example `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0@sha256:REPLACE_ME`. Until KVCR is available in a Dynamo release image, build that image from matching Dynamo, -KVCR, and vLLM revisions. Use [KVCR tag -`v0.1.0`](https://github.com/ai-dynamo/kvcr/tree/v0.1.0). Until -[vLLM PR 53624](https://github.com/vllm-project/vllm/pull/53624) is merged, pin -vLLM to the PR's immutable head SHA; at the time of this update it is -[`7015687c`](https://github.com/vllm-project/vllm/commit/7015687c390442fae384b78fc62e6621b3bf6e56). -After the PR merges, pin the commit that lands on vLLM `main`. Record the exact -Dynamo, KVCR, and vLLM commit IDs with the image digest used for every qualified -deployment. +KVCR, and vLLM revisions. This example targets KVCR **0.2.0** at +[`7ce17c358b867bd89f40a4a42bee9c76839506d0`](https://github.com/ai-dynamo/kvcr/commit/7ce17c358b867bd89f40a4a42bee9c76839506d0) +and vLLM's 0.2.0 adapter at +[`fed7de909f894fbee0c45130376aed2e2c9bf0d3`](https://github.com/vllm-project/vllm/commit/fed7de909f894fbee0c45130376aed2e2c9bf0d3). +KVCR requires NIXL 1.5.0 and the runtime's UCX, CUDA, and `libatomic` libraries. +Record the Dynamo, vLLM, and KVCR revisions and image digest for each deployment; +these source pins alone do not establish that an existing image contains a +compatible stack. `DYNAMO_KVCR_COMPATIBILITY_DIGEST` is an opaque layout version shared by the engine and memory service. Change it whenever model, dtype, block layout, or @@ -67,7 +66,26 @@ KVCR_MEMORY_SERVICE_ENABLED=false ./deploy.sh KVCR_MEMORY_SERVICE_ENABLED=true ./deploy.sh ``` -Run `./deploy.sh --render-only` to inspect the selected DGD. Both manifests +To test KVCR changes without rebuilding an otherwise compatible runtime image, +build a pure-Python wheel from the pinned KVCR tree and export +`KVCR_WHEEL=/absolute/path/to/kvcr-0.2.0-py3-none-any.whl` before deployment. +The script mounts it through `PYTHONPATH` in both containers; it does not install +packages, update vLLM, or change image environment settings. Rebuild the image +for native dependency or vLLM changes. Without `KVCR_WHEEL`, the installed KVCR +must be compatible with this example. A wheel overlay does not upgrade an old +vLLM adapter to the 0.2.0 API. For pinned registration, ensure +`UCX_REG_NONBLOCK_MEM_TYPES` and `UCX_IB_ODP_PREFETCH` are unset in both containers. + +`deploy.sh` applies an immutable, content-addressed ConfigMap containing +`services.sh`, Dynamo's shared `launch_utils.sh`, and the optional wheel before +applying the DGD. The bundle is limited to 1 MiB. It is retained after deployment +deletion because other deployments may share it; delete an old bundle only after +no Pods or deployment templates reference it. Do not replace a live bundle or +roll an incompatible KVCR wheel under engines that still map the old pools. + +Run `./deploy.sh --render-only` to inspect the selected DGD. For the memory-service +variant, `./deploy.sh --render-configmap` separately renders its prerequisite +ConfigMap; apply that first when deploying rendered files manually. Both manifests use required Pod anti-affinity, so the second worker remains Pending unless a second eligible node is available. After apply, the script also verifies that the operator materialized Grove as the DGD's workload provider. Each engine's @@ -92,11 +110,61 @@ supported, restart both workers to restore state tracking after worker 0 restarts. In `agg-memory-service.yaml`, `dynamo.vllm` runs in `main`. A regular -`kvcr-services` sidecar runs the KVCR memory service and state agent. Both -containers mount `/run/kvcr` and the memory-backed `/dev/shm/kvcr` volume. A -vLLM container restart leaves the service, Guard, and pool running. The -sidecar's probes check the KVCR Unix socket. The state agent runs in worker 0's -sidecar, and that sidecar's probe also checks state-agent health. Kubernetes +`kvcr-services` sidecar runs the KVCR memory service (`kvcr.service.server`) +and, on worker 0 only, the state agent. Both containers mount `/run/kvcr` for +the Unix socket. Only the sidecar mounts the memory-backed `/dev/shm/kvcr` +volume. Guard passes the pool and control-listener file descriptors over the +socket using `SCM_RIGHTS`; the engine maps the same backing pages without +opening a pool path. No shared pool mount, `hostPath`, or permission-setting +init container is needed in the engine. Both containers must use the same UID +to access the service socket. A vLLM container restart leaves the service, +Guard, and pool running. + +Both engines report residency to worker 0's state agent. Each Guard can serve +its preserved pool after its engine exits. The FD handoff +does not let a replacement service adopt a live engine's pool; the coupled +failure policy below still applies. + +`services.sh` starts KVCR and, on worker 0, the state agent. It uses Dynamo's +`wait_any_exit` and process-group cleanup helpers: any child exit ends the +wrapper and terminates/reaps its companion; SIGTERM does the same. Kubernetes' +Pod termination grace period (default: 30 seconds) is the final bound for stuck +children, not a timeout implemented by the shell helper. For a planned shutdown, +drain and stop engines before their service; the wrapper does not impose +cross-container shutdown ordering. + +Memory-service settings accept positive decimal integers without leading zeros: + +| Variable | Default | Controls | +|---|---:|---| +| `KVCR_POOL_SIZE_GIB` | 2 | Guard pool capacity per worker, in GiB | +| `KVCR_READY_TIMEOUT_SECONDS` | 120 | HTTP `/ready` wait before launching vLLM | +| `KVCR_GUARD_STARTUP_TIMEOUT_SECONDS` | 30 | Guard preparation and activation, each with this budget | +| `KVCR_TERMINATION_GRACE_PERIOD_SECONDS` | 30 | Pod termination grace for drain and cleanup | + +Timeouts are in seconds and independent. HTTP readiness is not inference readiness. +Engine-side KVCR initialization uses KVCR 0.2.0's default 60-second budget; +the pinned vLLM adapter does not expose an override. Increasing the Guard or +readiness timeout does not extend that budget. Pools whose engine-side memory +registration exceeds it require a vLLM adapter that exposes the startup timeout. + +Shared memory is pool size + 1 GiB; the sidecar requests pool size + 1% +(rounded up to GiB), with a limit 1 GiB above that. Engine CPU offload stays 2 GiB. +For a 1 TiB pool, reserve 1035 GiB per sidecar (1036 GiB limit), for example: + +```bash +KVCR_MEMORY_SERVICE_ENABLED=true KVCR_POOL_SIZE_GIB=1024 \ + KVCR_READY_TIMEOUT_SECONDS=600 KVCR_GUARD_STARTUP_TIMEOUT_SECONDS=600 \ + KVCR_TERMINATION_GRACE_PERIOD_SECONDS=180 \ + ./deploy.sh +``` + +The sidecar's HTTP readiness probe checks that the Unix accept +loop and Guard actor loops progress, plus worker 0's state-agent `/live` endpoint. +It does not require an attached engine or mutate a pool. No liveness/startup +restart probe is set on the sidecar: a failed probe must not restart it beneath +a live engine. Readiness does not itself fence engines or remove Dynamo routing +state. Kubernetes container discovery gives `main` and the real `kvcr-services` sidecar separate metadata writers. The explicit sidecar supplies its own downward-API Pod UID; the operator injects that identity only into its generated `main` container. @@ -106,11 +174,73 @@ deployment-fatal: a replacement service cannot adopt the original Guard and pool while vLLM remains alive. An external deployment controller must replace the entire affected worker Pod instead of restarting only the sidecar. If worker 0's sidecar restarts, recreate the worker group or DGD because its state -agent's routing state was also lost. +agent's routing state was also lost. State-agent loss alone does not destroy KV +payloads, but current engines bind a discovered agent instance and do not +automatically reselect its replacement. Group recreation is the conservative +way to restore those bindings. This example requires an external controller to +coordinate Pod or worker-group replacement. The state agent carries routing and residency information; it does not move KV payloads. KVCR uses NIXL and UCX for the remote payload transfer. +## Guard HugePages + +Guard supports native HugeTLB and transparent huge pages (THP), with base-page +fallback. Check Guard logs to confirm the backing in use. For allocation and +fallback details, see the [KVCR memory-service guide](https://github.com/ai-dynamo/kvcr/blob/7ce17c358b867bd89f40a4a42bee9c76839506d0/docs/dev-guide.md). + +### Native hugetlbfs + +Have the cluster administrator reserve huge pages on eligible nodes. Add a +`HugePages-2Mi` emptyDir mounted **only in `kvcr-services`**, with matching +`hugepages-2Mi` requests and limits. Kubernetes creates the mount; no `hostPath`, +privileged mount command, or permission-setting init container is needed. +See [Kubernetes HugePages configuration](https://kubernetes.io/docs/tasks/manage-hugepages/scheduling-hugepages/). + +Merge these additions into the worker's `podTemplate.spec` in +`agg-memory-service.yaml`; retain its existing containers, resources, and volumes: + +```yaml +containers: +- name: kvcr-services + resources: + requests: + hugepages-2Mi: 3Gi + limits: + hugepages-2Mi: 3Gi + volumeMounts: + - name: kvcr-hugepages + mountPath: /hugepages +volumes: +- name: kvcr-hugepages + emptyDir: + medium: HugePages-2Mi +``` + +This illustrates a 2 GiB pool with headroom: budget for the additional 100 MiB +journal per Guard, page-size rounding, and recovery snapshots. Adjust the +reservation for larger pools. For 1 GiB pages, change both resource keys to +`hugepages-1Gi` and the volume medium to `HugePages-1Gi`. + +The mount path is arbitrary; no mount-path CLI option is needed. For non-root +containers, use a suitable Pod `fsGroup` so Guard can write the volume, and +retain the same UID in Guard and vLLM for socket access. The engine needs neither +this mount nor a hugepage reservation for the pool received by FD. + +Keep the existing `/dev/shm/kvcr` volume, `--pool-dir`, and ordinary memory +budget for runtime fallback. A Pod whose hugepage request cannot be scheduled +stays Pending; KVCR's fallback only applies after it starts. To skip native +HugeTLB, add `--disable-hugepages` to the service command in `services.sh`. +That flag does not disable THP. + +### Transparent HugePages + +Retain the default memory-backed emptyDir at `/dev/shm/kvcr`; no extra volume +or `hugepages-*` resource is needed. THP uses the ordinary memory budget and +depends on the host's kernel and tmpfs/shmem policy. Have the cluster +administrator configure that policy at node level. THP is best-effort and +can fall back to base pages. + ## Verify Guard recovery Use the memory-service variant and the opt-in live-cluster test. The test adds @@ -134,4 +264,5 @@ the standard `test_output` directory). It verifies that the workers are on separate hosts, one source engine restarts while its KVCR sidecar remains up, the Guard serves the preserved cache to the other engine, the response matches, the KVCR transfer metrics increase, and the selected active HCA carries the -transfer. +transfer. With `DYNAMO_UCX_NET_DEVICES=all`, each measured container must expose +exactly one active RDMA port; its counters are read independently. diff --git a/examples/backends/vllm/deploy/kvcr/agg-memory-service.yaml b/examples/backends/vllm/deploy/kvcr/agg-memory-service.yaml index 116f1157d6ba..1c990caba935 100644 --- a/examples/backends/vllm/deploy/kvcr/agg-memory-service.yaml +++ b/examples/backends/vllm/deploy/kvcr/agg-memory-service.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: nvidia.com/v1beta1 @@ -42,16 +42,11 @@ spec: - args: - | set -euo pipefail - for _ in $(seq 1 60); do - if python3 -c "import socket; s=socket.socket(socket.AF_UNIX); s.settimeout(1); s.connect('/run/kvcr/memory.sock'); s.close()" >/dev/null 2>&1; then - break - fi - sleep 1 - done - if ! python3 -c "import socket; s=socket.socket(socket.AF_UNIX); s.settimeout(1); s.connect('/run/kvcr/memory.sock'); s.close()"; then - echo "KVCR memory service did not become ready" >&2 - exit 1 + source /opt/kvcr-startup/launch_utils.sh + if [ -f /opt/kvcr-startup/kvcr.whl ]; then + export PYTHONPATH="/opt/kvcr-startup/kvcr.whl${PYTHONPATH:+:$PYTHONPATH}" fi + wait_for_ready http://127.0.0.1:9092/ready "${KVCR_READY_TIMEOUT_SECONDS}" kv_transfer_config=$( python3 - <<'PY' @@ -152,50 +147,23 @@ spec: add: - IPC_LOCK volumeMounts: + - mountPath: /opt/kvcr-startup + name: kvcr-startup + readOnly: true - mountPath: /run/kvcr name: kvcr-socket - - mountPath: /dev/shm/kvcr - name: kvcr-memory workingDir: /workspace/examples/backends/vllm - args: - - | - set -euo pipefail - state_namespace=$DYN_NAMESPACE - if [ -n "${DYN_NAMESPACE_WORKER_SUFFIX:-}" ]; then - state_namespace="${state_namespace}-${DYN_NAMESPACE_WORKER_SUFFIX}" - fi - - kvcr_pid= - state_agent_pid= - cleanup() { - [ -z "$kvcr_pid" ] || kill -TERM "$kvcr_pid" 2>/dev/null || true - [ -z "$state_agent_pid" ] || kill -TERM "$state_agent_pid" 2>/dev/null || true - [ -z "$kvcr_pid" ] || wait "$kvcr_pid" 2>/dev/null || true - [ -z "$state_agent_pid" ] || wait "$state_agent_pid" 2>/dev/null || true - } - trap cleanup EXIT INT TERM - - python3 -m kvcr.kvcr_service \ - --socket-path /run/kvcr/memory.sock \ - --pool-dir /dev/shm/kvcr \ - --guard-count 1 \ - --pool-sizes-gb 2 \ - --compatibility-digest "${DYNAMO_KVCR_COMPATIBILITY_DIGEST}" & - kvcr_pid=$! - if [ "$POD_INDEX" = "0" ]; then - env DYN_NAMESPACE="$state_namespace" \ - DYN_HEALTH_CHECK_ENABLED=false \ - python3 -m dynamo.kv_state_agent \ - --max-slots "$KVCR_CACHE_SLOT_COUNT" & - state_agent_pid=$! - wait -n "$kvcr_pid" "$state_agent_pid" - else - wait "$kvcr_pid" - fi + - /opt/kvcr-startup/services.sh command: - /bin/bash - - -c env: + - name: DYNAMO_KVCR_COMPATIBILITY_DIGEST + value: "${DYNAMO_KVCR_COMPATIBILITY_DIGEST}" + - name: KVCR_POOL_SIZE_GIB + value: "${KVCR_POOL_SIZE_GIB}" + - name: KVCR_GUARD_STARTUP_TIMEOUT_SECONDS + value: "${KVCR_GUARD_STARTUP_TIMEOUT_SECONDS}" - name: CONTAINER_NAME value: kvcr-services - name: DYN_COMPONENT @@ -237,61 +205,50 @@ spec: - name: UCX_NET_DEVICES value: ${DYNAMO_UCX_NET_DEVICES} image: ${DYNAMO_VLLM_IMAGE} - livenessProbe: - exec: - command: - - python3 - - -c - - import os,socket,urllib.request; s=socket.socket(socket.AF_UNIX); s.settimeout(1); s.connect('/run/kvcr/memory.sock'); s.close(); os.environ['POD_INDEX'] != '0' or urllib.request.urlopen('http://127.0.0.1:9091/live', timeout=1).read() - failureThreshold: 3 - periodSeconds: 5 - timeoutSeconds: 4 name: kvcr-services + # Readiness observes failures without triggering an unsafe sidecar-only + # restart. A deployment controller must coordinate Pod/group replacement. readinessProbe: - exec: - command: - - python3 - - -c - - import os,socket,urllib.request; s=socket.socket(socket.AF_UNIX); s.settimeout(1); s.connect('/run/kvcr/memory.sock'); s.close(); os.environ['POD_INDEX'] != '0' or urllib.request.urlopen('http://127.0.0.1:9091/live', timeout=1).read() + httpGet: + path: /ready + port: 9092 failureThreshold: 3 periodSeconds: 5 timeoutSeconds: 4 resources: limits: ${DYNAMO_RDMA_RESOURCE}: "1" - memory: 4Gi + memory: ${KVCR_MEMORY_LIMIT_GIB}Gi requests: ${DYNAMO_RDMA_RESOURCE}: "1" cpu: "1" - memory: 3Gi + memory: ${KVCR_MEMORY_REQUEST_GIB}Gi securityContext: capabilities: add: - IPC_LOCK - startupProbe: - exec: - command: - - python3 - - -c - - import os,socket,urllib.request; s=socket.socket(socket.AF_UNIX); s.settimeout(1); s.connect('/run/kvcr/memory.sock'); s.close(); os.environ['POD_INDEX'] != '0' or urllib.request.urlopen('http://127.0.0.1:9091/live', timeout=1).read() - failureThreshold: 60 - periodSeconds: 2 - timeoutSeconds: 4 volumeMounts: + - mountPath: /opt/kvcr-startup + name: kvcr-startup + readOnly: true - mountPath: /run/kvcr name: kvcr-socket - mountPath: /dev/shm/kvcr name: kvcr-memory + terminationGracePeriodSeconds: ${KVCR_TERMINATION_GRACE_PERIOD_SECONDS} tolerations: - effect: NoSchedule key: nvidia.com/gpu operator: Exists volumes: + - configMap: + name: ${KVCR_STARTUP_CONFIG_MAP} + name: kvcr-startup - emptyDir: {} name: kvcr-socket - emptyDir: medium: Memory - sizeLimit: 3Gi + sizeLimit: ${KVCR_SHM_SIZE_GIB}Gi name: kvcr-memory replicas: 2 type: worker diff --git a/examples/backends/vllm/deploy/kvcr/deploy.sh b/examples/backends/vllm/deploy/kvcr/deploy.sh index 495903624fe3..838a7ba025d3 100755 --- a/examples/backends/vllm/deploy/kvcr/deploy.sh +++ b/examples/backends/vllm/deploy/kvcr/deploy.sh @@ -1,11 +1,11 @@ #!/bin/sh -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -eu : "${DYNAMO_VLLM_IMAGE:?set DYNAMO_VLLM_IMAGE to a KVCR-capable image}" -: "${DYNAMO_UCX_NET_DEVICES:?set DYNAMO_UCX_NET_DEVICES to the GPU-local UCX device}" +: "${DYNAMO_UCX_NET_DEVICES:?set DYNAMO_UCX_NET_DEVICES to all or an exposed GPU-local UCX device}" DYNAMO_RDMA_RESOURCE=${DYNAMO_RDMA_RESOURCE:-rdma/shared_ib} KVCR_MEMORY_SERVICE_ENABLED=${KVCR_MEMORY_SERVICE_ENABLED:-false} @@ -23,6 +23,28 @@ case "$KVCR_MEMORY_SERVICE_ENABLED" in manifest=$script_dir/agg-memory-service.yaml deployment_name=vllm-agg-kvcr-memory-service : "${DYNAMO_KVCR_COMPATIBILITY_DIGEST:?set a versioned KVCR compatibility value}" + KVCR_POOL_SIZE_GIB=${KVCR_POOL_SIZE_GIB:-2} + KVCR_READY_TIMEOUT_SECONDS=${KVCR_READY_TIMEOUT_SECONDS:-120} + KVCR_GUARD_STARTUP_TIMEOUT_SECONDS=${KVCR_GUARD_STARTUP_TIMEOUT_SECONDS:-30} + KVCR_TERMINATION_GRACE_PERIOD_SECONDS=${KVCR_TERMINATION_GRACE_PERIOD_SECONDS:-30} + for value in "$KVCR_POOL_SIZE_GIB" "$KVCR_READY_TIMEOUT_SECONDS" \ + "$KVCR_GUARD_STARTUP_TIMEOUT_SECONDS" \ + "$KVCR_TERMINATION_GRACE_PERIOD_SECONDS"; do + case "$value" in + ''|0*|*[!0-9]*) + echo "KVCR pool size and timeouts must be positive decimal integers without leading zeros" >&2 + exit 2 + ;; + esac + done + # Large tmpfs pools also charge kernel metadata to the allocating container. + KVCR_SHM_SIZE_GIB=$((KVCR_POOL_SIZE_GIB + 1)) + KVCR_MEMORY_REQUEST_GIB=$((KVCR_POOL_SIZE_GIB + (KVCR_POOL_SIZE_GIB + 99) / 100)) + KVCR_MEMORY_LIMIT_GIB=$((KVCR_MEMORY_REQUEST_GIB + 1)) + export KVCR_POOL_SIZE_GIB KVCR_READY_TIMEOUT_SECONDS + export KVCR_GUARD_STARTUP_TIMEOUT_SECONDS + export KVCR_TERMINATION_GRACE_PERIOD_SECONDS + export KVCR_SHM_SIZE_GIB KVCR_MEMORY_REQUEST_GIB KVCR_MEMORY_LIMIT_GIB case "$DYNAMO_KVCR_COMPATIBILITY_DIGEST" in *[!A-Za-z0-9._:/-]*) echo "DYNAMO_KVCR_COMPATIBILITY_DIGEST contains an unsupported character" >&2 @@ -36,10 +58,52 @@ case "$KVCR_MEMORY_SERVICE_ENABLED" in ;; esac +# Keep startup code and an optional development wheel out of the base image. +# Content-addressed, immutable ConfigMaps cannot change under a live engine. +startup_configmap() { + python3 - "$script_dir" "${KVCR_WHEEL:-}" "${1:-}" <<'PY' +import base64 +import hashlib +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +data = { + "services.sh": (root / "services.sh").read_text(), + "launch_utils.sh": (root.parents[3] / "common/launch_utils.sh").read_text(), +} +binary = {} +if sys.argv[2]: + binary["kvcr.whl"] = base64.b64encode(pathlib.Path(sys.argv[2]).read_bytes()).decode() +payload = json.dumps({"data": data, "binaryData": binary}, sort_keys=True) +if len(payload.encode()) > 1024 * 1024: + sys.exit("KVCR startup bundle exceeds 1 MiB; use a smaller pure-Python wheel") +name = "kvcr-startup-" + hashlib.sha256(payload.encode()).hexdigest()[:16] +bundle = { + "apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": name}, + "immutable": True, "data": data, +} +if binary: + bundle["binaryData"] = binary +print(name if sys.argv[3] == "name" else json.dumps(bundle)) +PY +} + +if [ "$KVCR_MEMORY_SERVICE_ENABLED" = true ]; then + KVCR_STARTUP_CONFIG_MAP=$(startup_configmap name) + export KVCR_STARTUP_CONFIG_MAP +fi + render() { substitution_vars='${DYNAMO_VLLM_IMAGE} ${DYNAMO_RDMA_RESOURCE}' substitution_vars="$substitution_vars "'${DYNAMO_UCX_NET_DEVICES}' substitution_vars="$substitution_vars "'${DYNAMO_KVCR_COMPATIBILITY_DIGEST}' + substitution_vars="$substitution_vars "'${KVCR_STARTUP_CONFIG_MAP}' + substitution_vars="$substitution_vars "'${KVCR_POOL_SIZE_GIB} ${KVCR_READY_TIMEOUT_SECONDS}' + substitution_vars="$substitution_vars "'${KVCR_GUARD_STARTUP_TIMEOUT_SECONDS}' + substitution_vars="$substitution_vars "'${KVCR_TERMINATION_GRACE_PERIOD_SECONDS}' + substitution_vars="$substitution_vars "'${KVCR_SHM_SIZE_GIB} ${KVCR_MEMORY_REQUEST_GIB} ${KVCR_MEMORY_LIMIT_GIB}' envsubst "$substitution_vars" < "$manifest" } @@ -47,12 +111,23 @@ case "${1:-}" in --render-only) render ;; + --render-configmap) + if [ "$KVCR_MEMORY_SERVICE_ENABLED" != true ]; then + echo "--render-configmap requires KVCR_MEMORY_SERVICE_ENABLED=true" >&2 + exit 2 + fi + startup_configmap + ;; "") if ! kubectl api-resources --api-group=grove.io -o name 2>/dev/null \ | grep -qx 'podcliquesets.grove.io'; then echo "The KVCR examples require a Dynamo operator with Grove enabled" >&2 exit 1 fi + if [ "$KVCR_MEMORY_SERVICE_ENABLED" = true ]; then + # Avoid the client-side last-applied annotation's 256 KiB limit for wheels. + startup_configmap | kubectl apply --server-side --field-manager=kvcr-startup -n "$NAMESPACE" -f - + fi render | kubectl apply -n "$NAMESPACE" -f - provider= for _ in $(seq 1 20); do @@ -70,7 +145,7 @@ case "${1:-}" in fi ;; *) - echo "usage: $0 [--render-only]" >&2 + echo "usage: $0 [--render-only|--render-configmap]" >&2 exit 2 ;; esac diff --git a/examples/backends/vllm/deploy/kvcr/services.sh b/examples/backends/vllm/deploy/kvcr/services.sh new file mode 100755 index 000000000000..4686607d3363 --- /dev/null +++ b/examples/backends/vllm/deploy/kvcr/services.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +source "$SCRIPT_DIR/launch_utils.sh" +trap dynamo_exit_trap EXIT +trap 'dynamo_reap_and_exit 0' TERM INT + +# The optional pure-Python wheel is mounted, not installed into the base image. +if [ -f "$SCRIPT_DIR/kvcr.whl" ]; then + export PYTHONPATH="$SCRIPT_DIR/kvcr.whl${PYTHONPATH:+:$PYTHONPATH}" +fi + +health_args=() +if [ "$POD_INDEX" = "0" ]; then + state_namespace="${DYN_NAMESPACE}${DYN_NAMESPACE_WORKER_SUFFIX:+-${DYN_NAMESPACE_WORKER_SUFFIX}}" + env DYN_NAMESPACE="$state_namespace" DYN_HEALTH_CHECK_ENABLED=false \ + python3 -m dynamo.kv_state_agent --max-slots "$KVCR_CACHE_SLOT_COUNT" & + health_args+=(--health-dependency-url "http://127.0.0.1:${DYN_SYSTEM_PORT:-9091}/live") +fi + +python3 -m kvcr.service.server \ + --socket-path /run/kvcr/memory.sock \ + --pool-dir /dev/shm/kvcr \ + --guard-count 1 \ + --pool-sizes-gb "$KVCR_POOL_SIZE_GIB" \ + --startup-timeout-seconds "$KVCR_GUARD_STARTUP_TIMEOUT_SECONDS" \ + --compatibility-digest "$DYNAMO_KVCR_COMPATIBILITY_DIGEST" \ + --health-host 0.0.0.0 --health-port 9092 \ + "${health_args[@]}" & + +# Use Dynamo's first-child-exit and process-group cleanup semantics. Kubernetes' +# terminationGracePeriodSeconds is the final bound for unresponsive children. +wait_any_exit diff --git a/tests/deploy/test_kvcr_guard.py b/tests/deploy/test_kvcr_guard.py index 50e6d2b8887d..ffb4d979cd2e 100644 --- a/tests/deploy/test_kvcr_guard.py +++ b/tests/deploy/test_kvcr_guard.py @@ -51,7 +51,7 @@ TIER_READ_BYTES = "vllm:kv_offload_tiering_read_bytes_total" TIER_WRITE_BYTES = "vllm:kv_offload_tiering_write_bytes_total" PROMPT_TOKENS = "vllm:prompt_tokens_by_source_total" -GUARD_INITIALIZED = "Initialized NIXL agent: KVCR-Guard-" +GUARD_PROMOTED = "KVCR_EVENT guard_promoted " # Large enough to make the transfer visible, while remaining comfortably below # the small model's context limit and the example's 2 GiB host-memory pool. LONG_PREFIX = ( @@ -152,6 +152,36 @@ def _rdma_counter( return int(result.stdout.strip()) * 4 +def _rdma_device(namespace: str, pod: str, container: str, configured: str) -> str: + # Device-plugin allocations can differ between containers in the same Pod. + snippet = """ +import json +from pathlib import Path +ports = [] +for verbs in Path('/sys/class/infiniband_verbs').glob('uverbs*'): + if not (Path('/dev/infiniband') / verbs.name).is_char_device(): + continue + device = (verbs / 'ibdev').read_text().strip() + for port in (Path('/sys/class/infiniband') / device / 'ports').iterdir(): + if (port / 'state').read_text().strip() == '4: ACTIVE': + ports.append(f'{device}:{port.name}') +print(json.dumps(ports)) +""" + ports = json.loads( + _exec(namespace, pod, container, "python3", "-c", snippet).stdout + ) + if configured == "all": + assert len(ports) == 1, ( + f"Counter validation needs one allocated active RDMA port in " + f"{pod}/{container}, found {ports}" + ) + return ports[0] + assert ( + configured in ports + ), f"{configured} is not exposed in {pod}/{container}: {ports}" + return configured + + def _wait_until(predicate, description: str, timeout: int) -> None: deadline = time.monotonic() + timeout last_error: subprocess.CalledProcessError | subprocess.TimeoutExpired | None = None @@ -192,6 +222,15 @@ def _render_manifest(tmp_path: Path, image: str) -> Path: timeout=RENDER_TIMEOUT, ) manifest = yaml.safe_load(result.stdout) + bundle = subprocess.run( + [str(deploy_script), "--render-configmap"], + check=True, + capture_output=True, + env=env, + text=True, + timeout=RENDER_TIMEOUT, + ) + (tmp_path / "kvcr-startup.json").write_text(bundle.stdout) worker = next( component for component in manifest["spec"]["components"] @@ -271,6 +310,27 @@ def fake_exec(*args, **_kwargs): _rdma_counter("test", "worker-0", "main", "mlx5_8:2", "port_rcv_data") +@pytest.mark.pre_merge +@pytest.mark.unit +@pytest.mark.gpu_0 +def test_rdma_device_respects_each_container_allocation(monkeypatch) -> None: + allocations = {"main": ["mlx5_0:1"], "kvcr-services": ["mlx5_10:1"]} + + def fake_exec(namespace, pod, container, *command): + return subprocess.CompletedProcess( + command, 0, stdout=json.dumps(allocations[container]) + ) + + monkeypatch.setattr(f"{__name__}._exec", fake_exec) + assert _rdma_device("test", "worker-0", "main", "all") == "mlx5_0:1" + assert _rdma_device("test", "worker-0", "kvcr-services", "all") == "mlx5_10:1" + with pytest.raises(AssertionError, match="is not exposed"): + _rdma_device("test", "worker-0", "kvcr-services", "mlx5_0:1") + allocations["main"].append("mlx5_1:1") + with pytest.raises(AssertionError, match="needs one allocated"): + _rdma_device("test", "worker-0", "main", "all") + + @pytest.mark.pre_merge @pytest.mark.unit @pytest.mark.gpu_0 @@ -332,12 +392,22 @@ async def test_kvcr_memory_service_guard_serves_after_engine_restart( "DYNAMO_KVCR_COMPATIBILITY_DIGEST" ), "DYNAMO_KVCR_COMPATIBILITY_DIGEST is required" ucx_device = os.environ.get("DYNAMO_UCX_NET_DEVICES", "") - assert re.fullmatch( + assert ucx_device == "all" or re.fullmatch( r"mlx5_\d+:\d+", ucx_device - ), "DYNAMO_UCX_NET_DEVICES must name one device and port, e.g. mlx5_0:1" + ), "DYNAMO_UCX_NET_DEVICES must be all or one device and port, e.g. mlx5_0:1" deployment_spec = DeploymentSpec(_render_manifest(tmp_path, image)) deployment_spec.namespace = namespace + # The deployment helper normally applies this immutable bundle before the DGD. + # Keep it, like deploy.sh does: other deployments can share its content hash. + _kubectl( + namespace, + "apply", + "--server-side", + "--field-manager=kvcr-startup", + "-f", + str(tmp_path / "kvcr-startup.json"), + ) logger.info( "KVCR_TEST start image=%s namespace=%s ucx_device=%s", image, @@ -457,9 +527,9 @@ def source_cache_persisted() -> bool: main_before = _container_status(namespace, source.name, MAIN) sidecar_before = _container_status(namespace, source.name, KVCR_SERVICES) - guard_initializations_before = _logs( - namespace, source.name, KVCR_SERVICES - ).count(GUARD_INITIALIZED) + guard_promotions_before = _logs(namespace, source.name, KVCR_SERVICES).count( + GUARD_PROMOTED + ) _exec( namespace, source.name, @@ -492,11 +562,9 @@ def engine_restarted() -> bool: engine_restarted, "the source vLLM container restart", FAILURE_TIMEOUT ) _wait_until( - lambda: _logs(namespace, source.name, KVCR_SERVICES).count( - GUARD_INITIALIZED - ) - > guard_initializations_before, - "KVCR Guard NIXL agent initialization", + lambda: _logs(namespace, source.name, KVCR_SERVICES).count(GUARD_PROMOTED) + > guard_promotions_before, + "KVCR Guard promotion after engine failure", FAILURE_TIMEOUT, ) sidecar_failed = _container_status(namespace, source.name, KVCR_SERVICES) @@ -517,18 +585,21 @@ def engine_restarted() -> bool: _capture_workers(deployment, pods, ".source-failed") target_before = _metrics(namespace, target.name, deployment_spec.system_port) + source_device = _rdma_device(namespace, source.name, KVCR_SERVICES, ucx_device) + target_device = _rdma_device(namespace, target.name, MAIN, ucx_device) + logger.info("KVCR_TEST RDMA source=%s target=%s", source_device, target_device) xmit_before = _rdma_counter( namespace, source.name, KVCR_SERVICES, - ucx_device, + source_device, "port_xmit_data", ) recv_before = _rdma_counter( namespace, target.name, MAIN, - ucx_device, + target_device, "port_rcv_data", ) recovered_content = _request(request_url) @@ -569,7 +640,7 @@ def target_metrics_published() -> bool: namespace, source.name, KVCR_SERVICES, - ucx_device, + source_device, "port_xmit_data", ) - xmit_before @@ -579,7 +650,7 @@ def target_metrics_published() -> bool: namespace, target.name, MAIN, - ucx_device, + target_device, "port_rcv_data", ) - recv_before @@ -618,6 +689,20 @@ def target_metrics_published() -> bool: "the source vLLM container to become ready", READINESS_TIMEOUT, ) + _wait_until( + lambda: sum_metric_samples( + _metrics(namespace, source.name, deployment_spec.system_port), + "vllm:kvcr_state", + {"resource": "block_records"}, + ) + >= blocks, + "preserved KVCR records after handback to the replacement engine", + 60, + ) + assert ( + _container_status(namespace, source.name, KVCR_SERVICES)["restartCount"] + == sidecar_before["restartCount"] + ) logger.info( "KVCR_TEST result=PASS source_main_restarts=%d " "source_sidecar_restarts=%d guard_blocks=%d guard_bytes=%d", diff --git a/tests/deploy/test_kvcr_manifests.py b/tests/deploy/test_kvcr_manifests.py index 9e21e0d009fe..7549f6f14503 100644 --- a/tests/deploy/test_kvcr_manifests.py +++ b/tests/deploy/test_kvcr_manifests.py @@ -1,10 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import base64 import json import os +import select import shutil +import signal import subprocess +import sys from pathlib import Path import pytest @@ -22,6 +26,97 @@ _MANIFESTS = ("agg.yaml", "agg-memory-service.yaml") +@pytest.fixture(autouse=True) +def _clear_deployment_options(monkeypatch): + for name in ( + "KVCR_POOL_SIZE_GIB", + "KVCR_READY_TIMEOUT_SECONDS", + "KVCR_GUARD_STARTUP_TIMEOUT_SECONDS", + "KVCR_TERMINATION_GRACE_PERIOD_SECONDS", + "KVCR_WHEEL", + ): + monkeypatch.delenv(name, raising=False) + + +@pytest.mark.parametrize( + "worker,exit_module,exit_code,guard_timeout", + [ + ("0", "kvcr.service.server", 7, "30"), + ("0", "dynamo.kv_state_agent", 0, "600"), + ("0", None, 0, "30"), + ("1", "kvcr.service.server", 7, "30"), + ], +) +def test_services_supervises_children( + tmp_path, worker, exit_module, exit_code, guard_timeout +): + """Exercise the actual wrapper with controllable children, without engines.""" + for source in ( + _MANIFEST_DIR / "services.sh", + _REPO_ROOT / "examples/common/launch_utils.sh", + ): + shutil.copy(source, tmp_path / source.name) + child = tmp_path / "python3" + child.write_text( + f"#!{sys.executable}\n" + + """ +import json, os, signal, sys +signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) +signal.signal(signal.SIGUSR1, lambda *_: sys.exit(int(os.environ["TEST_EXIT_CODE"]))) +print(json.dumps({"module": sys.argv[2], "pid": os.getpid(), "args": sys.argv[3:]}), flush=True) +signal.pause() +""" + ) + child.chmod(0o755) + env = { + **os.environ, + "PATH": f"{tmp_path}:{os.environ['PATH']}", + "POD_INDEX": worker, + "DYN_NAMESPACE": "test", + "KVCR_CACHE_SLOT_COUNT": "2", + "DYN_NAMESPACE_WORKER_SUFFIX": "", + "DYNAMO_KVCR_COMPATIBILITY_DIGEST": "test", + "TEST_EXIT_CODE": str(exit_code), + "KVCR_POOL_SIZE_GIB": "1024", + "KVCR_GUARD_STARTUP_TIMEOUT_SECONDS": guard_timeout, + } + process = subprocess.Popen( + ["bash", str(tmp_path / "services.sh")], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + start_new_session=True, + bufsize=0, + ) + children = {} + try: + for _ in range(2 if worker == "0" else 1): + assert select.select([process.stdout], [], [], 5)[0], "child did not start" + info = json.loads(process.stdout.readline()) + children[info["module"]] = info + guard_args = children["kvcr.service.server"]["args"] + assert guard_args[guard_args.index("--pool-sizes-gb") + 1] == "1024" + assert ( + guard_args[guard_args.index("--startup-timeout-seconds") + 1] + == guard_timeout + ) + assert guard_args[guard_args.index("--health-port") + 1] == "9092" + assert ("--health-dependency-url" in guard_args) == (worker == "0") + if exit_module is None: + process.terminate() + else: + os.kill(children[exit_module]["pid"], signal.SIGUSR1) + output, _ = process.communicate(timeout=5) + assert process.returncode == exit_code, output.decode() + for info in children.values(): + with pytest.raises(ProcessLookupError): + os.kill(info["pid"], 0) + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + + def _worker(manifest_name: str) -> dict: manifest = yaml.safe_load((_MANIFEST_DIR / manifest_name).read_text()) return next( @@ -129,7 +224,7 @@ def test_process_local_variant_couples_state_agent_and_vllm() -> None: assert "DYN_HEALTH_CHECK_ENABLED=false" in command assert '--max-slots "$KVCR_CACHE_SLOT_COUNT"' in command assert 'wait -n "$state_agent_pid" "$vllm_pid"' in command - assert "kvcr.kvcr_service" not in command + assert "kvcr.service.server" not in command assert 'case "$POD_INDEX" in' not in command assert '"cache_owner_ids": {"0": cache_owner_id()}' in command assert all(item["name"] != "POD_UID" for item in main["env"]) @@ -152,13 +247,14 @@ def test_memory_service_variant_keeps_guard_in_sidecar() -> None: pod_spec = _worker("agg-memory-service.yaml")["podTemplate"]["spec"] main, sidecar = pod_spec["containers"] main_command = main["args"][0] - sidecar_command = sidecar["args"][0] + assert sidecar["args"] == ["/opt/kvcr-startup/services.sh"] + sidecar_command = (_MANIFEST_DIR / "services.sh").read_text() assert sidecar["name"] == "kvcr-services" - assert "python3 -m kvcr.kvcr_service" in sidecar_command + assert "python3 -m kvcr.service.server" in sidecar_command assert "python3 -m dynamo.kv_state_agent" in sidecar_command assert "--guard-count 1" in sidecar_command - assert "--pool-sizes-gb 2" in sidecar_command + assert '--pool-sizes-gb "$KVCR_POOL_SIZE_GIB"' in sidecar_command assert 'if [ "$POD_INDEX" = "0" ]' in sidecar_command assert "DYN_HEALTH_CHECK_ENABLED=false" in sidecar_command assert "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS" not in sidecar_command @@ -167,7 +263,11 @@ def test_memory_service_variant_keeps_guard_in_sidecar() -> None: for item in sidecar["env"] ) assert '--max-slots "$KVCR_CACHE_SLOT_COUNT"' in sidecar_command - assert 'wait -n "$kvcr_pid" "$state_agent_pid"' in sidecar_command + assert "wait_any_exit" in sidecar_command + assert ( + 'wait_for_ready http://127.0.0.1:9092/ready "${KVCR_READY_TIMEOUT_SECONDS}"' + in main_command + ) assert '"kvcr_service_socket_path": "/run/kvcr/memory.sock"' in main_command assert "/run/kvcr/hold-engine-start" not in main_command assert 'case "$POD_INDEX" in' not in main_command @@ -180,24 +280,43 @@ def test_memory_service_variant_keeps_guard_in_sidecar() -> None: assert sidecar["resources"][resource_class]["${DYNAMO_RDMA_RESOURCE}"] == "1" assert "IPC_LOCK" in sidecar["securityContext"]["capabilities"]["add"] - for container in (main, sidecar): - mounts = {mount["name"] for mount in container["volumeMounts"]} - assert mounts == {"kvcr-memory", "kvcr-socket"} + assert {mount["name"] for mount in main["volumeMounts"]} == { + "kvcr-socket", + "kvcr-startup", + } + assert {mount["name"] for mount in sidecar["volumeMounts"]} == { + "kvcr-memory", + "kvcr-socket", + "kvcr-startup", + } memory = next( volume for volume in pod_spec["volumes"] if volume["name"] == "kvcr-memory" ) assert memory["emptyDir"]["medium"] == "Memory" - for probe_name in ("startupProbe", "livenessProbe", "readinessProbe"): - probe = sidecar[probe_name]["exec"]["command"][-1] - assert "/run/kvcr/memory.sock" in probe - assert "9091/live" in probe - assert "os.environ['POD_INDEX'] != '0' or" in probe + assert sidecar["readinessProbe"]["httpGet"] == {"path": "/ready", "port": 9092} + assert "startupProbe" not in sidecar + assert "livenessProbe" not in sidecar + assert "--health-dependency-url" in sidecar_command + assert "--health-port" in sidecar_command @pytest.mark.skipif(shutil.which("envsubst") is None, reason="envsubst is unavailable") -@pytest.mark.parametrize("memory_service", ("false", "true")) -def test_deploy_script_renders_selected_variant(memory_service: str) -> None: +@pytest.mark.parametrize( + "memory_service,pool_gib,memory_request_gib,guard_timeout,termination_grace", + [ + ("false", 2, 3, None, 30), + ("true", 2, 3, None, 30), + ("true", 1024, 1035, 600, 180), + ], +) +def test_deploy_script_renders_selected_variant( + memory_service: str, + pool_gib: int, + memory_request_gib: int, + guard_timeout: int | None, + termination_grace: int, +) -> None: env = os.environ.copy() env.update( { @@ -206,9 +325,17 @@ def test_deploy_script_renders_selected_variant(memory_service: str) -> None: "DYNAMO_KVCR_COMPATIBILITY_DIGEST": "qwen3-0.6b-example-v1", "DYNAMO_VLLM_IMAGE": "runtime:1.4.0@sha256:test", "KVCR_MEMORY_SERVICE_ENABLED": memory_service, + "KVCR_POOL_SIZE_GIB": str(pool_gib), + "KVCR_READY_TIMEOUT_SECONDS": "900", + "KVCR_TERMINATION_GRACE_PERIOD_SECONDS": str(termination_grace), } ) + if guard_timeout is not None: + env["KVCR_GUARD_STARTUP_TIMEOUT_SECONDS"] = str(guard_timeout) + else: + guard_timeout = 30 + result = subprocess.run( [str(_MANIFEST_DIR / "deploy.sh"), "--render-only"], check=True, @@ -220,6 +347,39 @@ def test_deploy_script_renders_selected_variant(memory_service: str) -> None: rendered = yaml.safe_load(result.stdout) assert rendered["kind"] == "DynamoGraphDeployment" + if memory_service == "true": + worker = next( + c for c in rendered["spec"]["components"] if c["name"] == "worker" + ) + pod_spec = worker["podTemplate"]["spec"] + main, sidecar = pod_spec["containers"] + assert 'wait_for_ready http://127.0.0.1:9092/ready "900"' in main["args"][0] + assert pod_spec["terminationGracePeriodSeconds"] == termination_grace + env_values = {e["name"]: e.get("value") for e in sidecar["env"]} + assert env_values["KVCR_POOL_SIZE_GIB"] == str(pool_gib) + assert env_values["KVCR_GUARD_STARTUP_TIMEOUT_SECONDS"] == str(guard_timeout) + main_env = {e["name"]: e["value"] for e in main["env"] if "value" in e} + config_result = subprocess.run( + [sys.executable, "-c", _kv_transfer_config_python(main["args"][0])], + check=True, + capture_output=True, + env={ + **env, + **main_env, + "KVCR_CACHE_SLOT": "0", + "POD_IP": "192.0.2.1", + }, + text=True, + timeout=30, + ) + tier = json.loads(config_result.stdout)["kv_connector_extra_config"][ + "secondary_tiers" + ][0] + assert "startup_timeout_seconds" not in tier + assert sidecar["resources"]["requests"]["memory"] == f"{memory_request_gib}Gi" + assert sidecar["resources"]["limits"]["memory"] == f"{memory_request_gib + 1}Gi" + memory = next(v for v in pod_spec["volumes"] if v["name"] == "kvcr-memory") + assert memory["emptyDir"]["sizeLimit"] == f"{pool_gib + 1}Gi" containers = [ container for component in rendered["spec"]["components"] @@ -239,12 +399,102 @@ def test_deploy_script_renders_selected_variant(memory_service: str) -> None: assert container["resources"]["limits"]["rdma/test"] == "1" if memory_service == "true": assert result.stdout.count("qwen3-0.6b-example-v1") == 2 + bundle = subprocess.run( + [str(_MANIFEST_DIR / "deploy.sh"), "--render-configmap"], + check=True, + capture_output=True, + env=env, + text=True, + timeout=30, + ) + configmap = json.loads(bundle.stdout) + assert configmap["immutable"] is True + assert "binaryData" not in configmap + assert ( + configmap["data"]["services.sh"] + == (_MANIFEST_DIR / "services.sh").read_text() + ) + assert ( + configmap["data"]["launch_utils.sh"] + == (_REPO_ROOT / "examples/common/launch_utils.sh").read_text() + ) + volume = next( + v + for v in worker["podTemplate"]["spec"]["volumes"] + if v["name"] == "kvcr-startup" + ) + assert volume["configMap"]["name"] == configmap["metadata"]["name"] expected_name = ( "vllm-agg-kvcr-memory-service" if memory_service == "true" else "vllm-agg-kvcr" ) assert f"name: {expected_name}" in result.stdout +@pytest.mark.skipif(shutil.which("envsubst") is None, reason="envsubst is unavailable") +@pytest.mark.parametrize("digest", ["2", "1.0", "true", "null"]) +def test_deploy_preserves_digest_as_string(digest): + result = subprocess.run( + [str(_MANIFEST_DIR / "deploy.sh"), "--render-only"], + check=True, + capture_output=True, + env={ + **os.environ, + "DYNAMO_UCX_NET_DEVICES": "test:1", + "DYNAMO_VLLM_IMAGE": "runtime:test", + "KVCR_MEMORY_SERVICE_ENABLED": "true", + "DYNAMO_KVCR_COMPATIBILITY_DIGEST": digest, + }, + text=True, + timeout=30, + ) + worker = next( + c + for c in yaml.safe_load(result.stdout)["spec"]["components"] + if c["name"] == "worker" + ) + sidecar = next( + c + for c in worker["podTemplate"]["spec"]["containers"] + if c["name"] == "kvcr-services" + ) + env = {e["name"]: e.get("value") for e in sidecar["env"]} + assert env["DYNAMO_KVCR_COMPATIBILITY_DIGEST"] == digest + + +@pytest.mark.parametrize( + "option,value", + [ + ("KVCR_POOL_SIZE_GIB", "0"), + ("KVCR_READY_TIMEOUT_SECONDS", "0"), + ("KVCR_GUARD_STARTUP_TIMEOUT_SECONDS", "0"), + ("KVCR_TERMINATION_GRACE_PERIOD_SECONDS", "0"), + ("KVCR_POOL_SIZE_GIB", "01"), + ("KVCR_POOL_SIZE_GIB", "-1"), + ("KVCR_POOL_SIZE_GIB", "1.5"), + ("KVCR_POOL_SIZE_GIB", "abc"), + ], +) +def test_deploy_script_rejects_invalid_pool_and_timeouts(option, value) -> None: + result = subprocess.run( + [str(_MANIFEST_DIR / "deploy.sh"), "--render-only"], + check=False, + capture_output=True, + env={ + **os.environ, + "DYNAMO_UCX_NET_DEVICES": "mlx5_test:1", + "DYNAMO_VLLM_IMAGE": "runtime:1.4.0@sha256:test", + "DYNAMO_KVCR_COMPATIBILITY_DIGEST": "test", + "KVCR_MEMORY_SERVICE_ENABLED": "true", + option: value, + }, + text=True, + timeout=30, + ) + assert result.returncode == 2 + assert result.stdout == "" + assert "positive decimal integers without leading zeros" in result.stderr + + def test_deploy_script_rejects_unknown_argument() -> None: env = os.environ.copy() env.update( @@ -268,6 +518,80 @@ def test_deploy_script_rejects_unknown_argument() -> None: assert "usage:" in result.stderr +@pytest.mark.skipif(shutil.which("envsubst") is None, reason="envsubst is unavailable") +@pytest.mark.parametrize("size", [256 * 1024, 1024 * 1024]) +def test_startup_bundle_wheel_overlay_and_size_limit(tmp_path, size): + wheel = tmp_path / "fixture.whl" + wheel.write_bytes(b"x" * size) + env = { + **os.environ, + "DYNAMO_UCX_NET_DEVICES": "test:1", + "DYNAMO_VLLM_IMAGE": "runtime:test", + "KVCR_MEMORY_SERVICE_ENABLED": "true", + "DYNAMO_KVCR_COMPATIBILITY_DIGEST": "test", + "KVCR_WHEEL": str(wheel), + "NAMESPACE": "kvcr-test", + } + result = subprocess.run( + [str(_MANIFEST_DIR / "deploy.sh"), "--render-configmap"], + check=False, + capture_output=True, + env=env, + text=True, + timeout=30, + ) + if size == 1024 * 1024: + assert result.returncode != 0 + assert "exceeds 1 MiB" in result.stderr + else: + assert result.returncode == 0, result.stderr + bundle = json.loads(result.stdout) + assert base64.b64decode(bundle["binaryData"]["kvcr.whl"]) == wheel.read_bytes() + without_wheel = subprocess.run( + [str(_MANIFEST_DIR / "deploy.sh"), "--render-configmap"], + check=True, + capture_output=True, + env={**env, "KVCR_WHEEL": ""}, + text=True, + timeout=30, + ) + assert ( + bundle["metadata"]["name"] + != json.loads(without_wheel.stdout)["metadata"]["name"] + ) + # This bundle fits the startup limit but not a client-side apply annotation. + assert 256 * 1024 < len(result.stdout.encode()) < 1024 * 1024 + kubectl = tmp_path / "kubectl" + kubectl.write_text( + f"#!{sys.executable}\n" + "import json, sys, yaml\n" + "args = sys.argv[1:]\n" + "if args[0] == 'api-resources':\n" + " print('podcliquesets.grove.io')\n" + "elif args[0] == 'apply':\n" + " print(json.dumps({'args': args, 'object': yaml.safe_load(sys.stdin)}))\n" + "elif args[0] == 'get':\n" + " print('grove')\n" + "else:\n" + " raise AssertionError(args)\n" + ) + kubectl.chmod(0o755) + applied = subprocess.run( + [str(_MANIFEST_DIR / "deploy.sh")], + check=True, + capture_output=True, + env={**env, "PATH": f"{tmp_path}{os.pathsep}{os.environ['PATH']}"}, + text=True, + timeout=30, + ) + configmap_call, deployment_call = map(json.loads, applied.stdout.splitlines()) + assert configmap_call["object"] == bundle + assert "--server-side" in configmap_call["args"] + assert deployment_call["object"]["kind"] == "DynamoGraphDeployment" + for call in (configmap_call, deployment_call): + assert call["args"][call["args"].index("-n") + 1] == "kvcr-test" + + def test_deploy_script_rejects_unsafe_compatibility_digest() -> None: env = os.environ.copy() env.update(