luckyq · Oct 27, 2026
diff --git a/.github/workflows/vllm-e2e.yml b/.github/workflows/vllm-e2e.yml
@@ -95,3 +95,56 @@ jobs:
           name: kvcr-phase1-${{ github.run_attempt }}
           path: artifacts/phase1/
           retention-days: 14
+
+  phase1-2p1d:
+    # 2P+1D on the two-GPU runner: prefill workers 0 and 1 share GPU 0 at
+    # --gpu-memory-utilization 0.35, the decode worker has GPU 1. Functional
+    # verdicts only (Guard takeover, hinted delivery, recovery); latencies are
+    # not comparable with a dedicated-GPU layout. kvcr-service sidecars run as
+    # background processes of this step, as the harness requires for disagg.
+    # Runs after phase1: both jobs share the runner host's Hugging Face cache,
+    # and two engines fetching the same model at once fail on its blob lock.
+    needs: [peer-transfer, phase1]
+    runs-on: prod-kvcr-tester-amd-gpu-2-v1
+    container:
+      image: ${{ inputs.image }}
+    timeout-minutes: 60
+    env:
+      HF_HOME: /scratch/huggingface
+      VLLM_CACHE_ROOT: /scratch/vllm
+      XDG_CACHE_HOME: /scratch/cache
+    steps:
+      - uses: actions/checkout@v4
+      - name: Show NVIDIA devices
+        run: nvidia-smi
+      - name: Run Phase 1 2P+1D (TC-01,TC-02,TC-03,TC-04,TC-06) with shared prefill GPU
+        shell: bash
+        env:
+          MODEL: ${{ inputs.model }}
+          DIGEST: phase1-2p1d-${{ github.run_id }}-${{ github.run_attempt }}
+        run: |
+          set -euo pipefail
+          SVC=/tmp/kvcr-svc; mkdir -p "$SVC" /dev/shm/kvcr-2p1d/0 /dev/shm/kvcr-2p1d/1
+          # KVCR 0.2.0 moved the service module to kvcr.service.server; older
+          # releases expose kvcr.kvcr_service. Pick by what the image provides.
+          SERVICE_PY="import importlib.util as u, runpy; runpy.run_module('kvcr.service.server' if u.find_spec('kvcr.service') else 'kvcr.kvcr_service', run_name='__main__')"
+          for i in 0 1; do
+            python3 -c "$SERVICE_PY" --socket-path "$SVC/memory-$i.sock" \
+              --pool-dir /dev/shm/kvcr-2p1d/$i --guard-count 1 --pool-sizes-gb 2 \
+              --compatibility-digest "$DIGEST" --log-level DEBUG \
+              > "$SVC/kvcr-service-$i.log" 2>&1 &
+          done
+          python3 -u tests/e2e/run_phase1.py --output-dir artifacts/phase1-2p1d/run \
+            --model "$MODEL" --mode guard --batch-invariant --timeout 2400 \
+            --topology disagg --gpus 0,0,1 --gpu-memory-utilization 0.35 \
+            --external-services "$SVC" --compatibility-digest "$DIGEST" \
+            --cases TC-01,TC-02,TC-03,TC-04,TC-06
+      - name: Save 2P+1D evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: kvcr-phase1-2p1d-${{ github.run_attempt }}
+          path: |
+            artifacts/phase1-2p1d/
+            /tmp/kvcr-svc/*.log
+          retention-days: 14
diff --git a/Dockerfile.quick-start-vllm b/Dockerfile.quick-start-vllm
@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ARG VLLM_IMAGE=vllm/vllm-openai:nightly-edad8b03c5285a1c12e7e4f83c1af709b319a01b@sha256:2e735ffeef26457f8a3ff44a62ae182870b10af37adc06ecbb08baca152365c6
+ARG DYNAMO_REF=318ab0390d8e3e1fa7517ea6a1ebf46a7854c236
+ARG KVCR_VLLM_REF=fed7de909f894fbee0c45130376aed2e2c9bf0d3
 
 # Keep the vLLM image, adapter source, and Dynamo revision compatible.
 FROM rust:1.93-bookworm AS build-base
@@ -14,12 +16,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # Pin the merged vLLM KVCR integration to an exact commit SHA.
 FROM build-base AS vllm-kvcr-source
 ARG KVCR_VLLM_REPO=https://github.com/vllm-project/vllm.git
-ARG KVCR_VLLM_REF=fed7de909f894fbee0c45130376aed2e2c9bf0d3
+ARG KVCR_VLLM_REF
 RUN git init /src/vllm-kvcr \
  && git -C /src/vllm-kvcr remote add origin "${KVCR_VLLM_REPO}" \
  && git -C /src/vllm-kvcr fetch --depth 1 origin "${KVCR_VLLM_REF}" \
  && git -C /src/vllm-kvcr checkout --detach FETCH_HEAD \
- && test "$(git -C /src/vllm-kvcr rev-parse HEAD)" = "${KVCR_VLLM_REF}"
+ && test "$(git -C /src/vllm-kvcr rev-parse HEAD)" = "${KVCR_VLLM_REF}" \
+ && git -C /src/vllm-kvcr rev-parse HEAD > /vllm-source.sha
 WORKDIR /src/vllm-kvcr
 # Use the public channel export.
 RUN sed -i 's/^from kvcr\.control_channels import ZmqPeerControlChannel$/from kvcr import ZmqPeerControlChannel/' \
@@ -37,10 +40,12 @@ RUN sed -i 's/^from kvcr\.control_channels import ZmqPeerControlChannel$/from kv
 # Pin a landed Dynamo revision with the versioned KV hint protocol and
 # per-rank KVCR control ports.
 FROM build-base AS dynamo-builder
-ARG DYNAMO_REF=318ab0390d8e3e1fa7517ea6a1ebf46a7854c236
+ARG DYNAMO_REF
 RUN git clone --filter=blob:none https://github.com/ai-dynamo/dynamo.git /src/dynamo \
  && git -C /src/dynamo fetch --depth 1 origin "${DYNAMO_REF}" \
- && git -C /src/dynamo checkout --detach "${DYNAMO_REF}"
+ && git -C /src/dynamo checkout --detach "${DYNAMO_REF}" \
+ && test "$(git -C /src/dynamo rev-parse HEAD)" = "${DYNAMO_REF}" \
+ && git -C /src/dynamo rev-parse HEAD > /dynamo-source.sha
 RUN python3 -m venv /venv \
  && /venv/bin/pip install --no-cache-dir "maturin[patchelf]"
 RUN cd /src/dynamo/lib/bindings/python \
@@ -89,6 +94,12 @@ ENV DYN_REQUEST_PLANE=tcp \
     PYTHONHASHSEED=0 \
     VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES=1
 
+# On-demand paging keeps Guard preparation off the synchronous registration
+# path. Fallback stays on so hardware without host ODP still registers; set
+# UCX_REG_NONBLOCK_FALLBACK=n to prove ODP is the path actually taken.
+ENV UCX_REG_NONBLOCK_MEM_TYPES=host \
+    UCX_IB_ODP_PREFETCH=n
+
 # Check serving-path compatibility; NCCL is pinned separately from torch.
 RUN kvcr-service --help
 RUN python3 - <<'PY'
@@ -158,10 +169,47 @@ for distribution in (
     print(f"{distribution}=={metadata.version(distribution)}")
 PY
 
+ARG VLLM_IMAGE
+ARG DYNAMO_REF
+ARG KVCR_VLLM_REF
+ARG KVCR_COMMIT=unknown
+
+LABEL org.opencontainers.image.revision="${KVCR_COMMIT}" \
+      io.ai-dynamo.kvcr.dynamo.sha="${DYNAMO_REF}" \
+      io.ai-dynamo.kvcr.vllm.sha="${KVCR_VLLM_REF}" \
+      io.ai-dynamo.kvcr.base-image="${VLLM_IMAGE}"
+
+# Keep the revisions observed inside the source stages in the final image.
+# Local quick-start builds may omit KVCR_COMMIT; CI supplies the PR merge SHA.
+COPY --from=dynamo-builder /dynamo-source.sha /opt/kvcr-build/dynamo.sha
+COPY --from=vllm-kvcr-source /vllm-source.sha /opt/kvcr-build/vllm.sha
+RUN python3 - <<'PY'
+import json
+import os
+from pathlib import Path
+import re
+
+out = Path("/opt/kvcr-build")
+kvcr_commit = os.environ["KVCR_COMMIT"]
+assert kvcr_commit == "unknown" or re.fullmatch(r"[0-9a-f]{40}", kvcr_commit)
+dynamo_commit = (out / "dynamo.sha").read_text().strip()
+vllm_commit = (out / "vllm.sha").read_text().strip()
+assert dynamo_commit == os.environ["DYNAMO_REF"]
+assert vllm_commit == os.environ["KVCR_VLLM_REF"]
+(out / "source-revisions.json").write_text(json.dumps({
+    "schema_version": 1,
+    "kvcr_commit": kvcr_commit,
+    "dynamo_commit": dynamo_commit,
+    "vllm_adapter_commit": vllm_commit,
+    "vllm_base_image": os.environ["VLLM_IMAGE"],
+}, indent=2) + "\n")
+PY
+
 ENV HF_HOME=/home/vllm/.cache/huggingface
 RUN mkdir -p "${HF_HOME}" \
  && chown -R vllm:0 "${HF_HOME}"
 
 WORKDIR /workspace
-USER vllm
+ARG KVCR_RUNTIME_USER=vllm
+USER ${KVCR_RUNTIME_USER}
 ENTRYPOINT []
diff --git a/scripts/dlcluster/build.sbatch b/scripts/dlcluster/build.sbatch
@@ -0,0 +1,35 @@
+#!/bin/bash
+#SBATCH --job-name=kvcr-p1-build
+#SBATCH --partition=general
+#SBATCH --gres=gpu:H100:1
+#SBATCH --constraint="x86_64&UB24.04"
+#SBATCH --cpus-per-task=16
+#SBATCH --mem=64G
+#SBATCH --time=02:00:00
+#SBATCH --output=/home/doxu/kvcr-runs/build-%j.log
+# Build the Phase 1 image for one KVCR ref and push it to ghcr (KVCR_PUSH=1).
+# A 1-GPU allocation only buys a fast start on a Docker-capable UB24.04 node;
+# the build itself is CPU-only. Needs: KVCR_BUNDLE, KVCR_REF, KVCR_IMAGE_REPO,
+# ~/.ghcr-token (user luckyq, write:packages).
+set -euo pipefail
+W=/tmp/doxu-kvcr-build-$SLURM_JOB_ID
+RUNS=/home/doxu/kvcr-runs/$SLURM_JOB_ID
+mkdir -p "$W" "$RUNS"
+log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*"; }
+trap 'rm -rf "$W"' EXIT
+log "node=$(hostname) kernel=$(uname -r) job=$SLURM_JOB_ID ref=$KVCR_REF repo=$KVCR_IMAGE_REPO"
+command -v docker >/dev/null || { log "FATAL: no docker on $(hostname)"; exit 3; }
+docker info --format 'docker {{.ServerVersion}}'
+git clone -q "$KVCR_BUNDLE" "$W/repo"
+git -C "$W/repo" checkout -q --detach "$KVCR_REF"
+test "$(git -C "$W/repo" rev-parse HEAD)" = "$KVCR_REF"
+cd "$W/repo"
+docker login ghcr.io -u luckyq --password-stdin < "$HOME/.ghcr-token" >/dev/null
+log "logged in to ghcr.io"
+export KVCR_E2E_IMAGE=$KVCR_IMAGE_REPO KVCR_PUSH=1 E2E_BUILD_OUTPUT="$RUNS/build" KVCR_PINNED_SOURCES=${KVCR_PINNED_SOURCES:-1}
+log "pinned sources: $KVCR_PINNED_SOURCES (1 = Dockerfile DYNAMO_REF / KVCR_VLLM_REF)"
+mkdir -p "$E2E_BUILD_OUTPUT"
+log "build+push start"
+bash scripts/docker/build_image.sh
+log "build+push done: $KVCR_IMAGE_REPO:$KVCR_REF"
+cat "$E2E_BUILD_OUTPUT"/*.json 2>/dev/null | head -60 || true
diff --git a/scripts/dlcluster/build_image.sh b/scripts/dlcluster/build_image.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+# Build and push the Phase 1 image for the current HEAD from a dlcluster node.
+#   scripts/dlcluster/build_image.sh [--repo ghcr.io/luckyq/kvcr-phase1]
+# Ships the ghcr token from the local `gh auth token` (account with write:packages).
+set -euo pipefail
+HOST=dlcluster; REPO=ghcr.io/luckyq/kvcr-phase1; PINNED=1
+while [[ $# -gt 0 ]]; do case $1 in --repo) REPO=$2; shift 2 ;; --ci-refs) PINNED=0; shift ;; *) echo "unknown arg $1" >&2; exit 2 ;; esac; done
+repo=$(git rev-parse --show-toplevel); cd "$repo"
+git diff --quiet HEAD -- . || { echo "commit your changes first (image build needs clean source)" >&2; exit 1; }
+ref=$(git rev-parse HEAD)
+bundle="kvcr-phase1-${ref:0:12}.bundle"
+git bundle create "/tmp/$bundle" HEAD >/dev/null
+gh auth token | ssh "$HOST" 'umask 077; cat > ~/.ghcr-token'
+scp -q "/tmp/$bundle" scripts/dlcluster/build.sbatch "$HOST:~/"
+ssh "$HOST" "mkdir -p ~/kvcr-runs && sbatch --export=ALL,KVCR_BUNDLE=\$HOME/$bundle,KVCR_REF=$ref,KVCR_IMAGE_REPO=$REPO,KVCR_PINNED_SOURCES=$PINNED ~/build.sbatch"
+echo "image will be $REPO:$ref"