From 2cf7a3aa092ad2506ecaf0902a544d6e3548c8ea Mon Sep 17 00:00:00 2001 From: Julien Darve Date: Tue, 8 Sep 2026 10:51:05 -0700 Subject: [PATCH 1/2] docs(epp): stop pinning vLLM on-ramp version Signed-off-by: Julien Darve --- deploy/inference-gateway/ext-proc/examples/onramp/README.md | 4 ++-- deploy/inference-gateway/ext-proc/examples/onramp/agg.yaml | 2 +- .../pages/kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/inference-gateway/ext-proc/examples/onramp/README.md b/deploy/inference-gateway/ext-proc/examples/onramp/README.md index c4ba59b4198f..aba2f1b78971 100644 --- a/deploy/inference-gateway/ext-proc/examples/onramp/README.md +++ b/deploy/inference-gateway/ext-proc/examples/onramp/README.md @@ -17,8 +17,8 @@ For the user-facing walkthrough, start with This on-ramp is available for aggregated serving only at this time. -The aggregated on-ramp uses the upstream `vllm/vllm-openai:v0.28.0` image. Replace it with the vLLM -image your platform standardizes on if you need another pinned or internally mirrored image. +The aggregated on-ramp uses the floating upstream `vllm/vllm-openai:latest` image. For reproducible +deployments, replace it with a pinned or internally mirrored image your platform standardizes on. KV-aware selection is provided by the runtime-free [selection service](../../../../../docs/fern/pages/developer-guide/knowledge-base/modular-components/router/standalone-selection.md), which the EPP runs **in-process**: the EPP and the selection service are compiled into one binary, diff --git a/deploy/inference-gateway/ext-proc/examples/onramp/agg.yaml b/deploy/inference-gateway/ext-proc/examples/onramp/agg.yaml index a6cafd4d0483..ef9fb807a42c 100644 --- a/deploy/inference-gateway/ext-proc/examples/onramp/agg.yaml +++ b/deploy/inference-gateway/ext-proc/examples/onramp/agg.yaml @@ -76,7 +76,7 @@ spec: spec: containers: - name: vllm - image: vllm/vllm-openai:v0.28.0 + image: vllm/vllm-openai:latest args: - --model - Qwen/Qwen3-0.6B diff --git a/docs/fern/pages/kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx b/docs/fern/pages/kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx index ad6f55d959dd..b218fdc43ae1 100644 --- a/docs/fern/pages/kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx +++ b/docs/fern/pages/kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx @@ -245,7 +245,7 @@ Compare a typical existing vLLM container with the changes required by the on-ra ```yaml containers: - name: vllm - image: vllm/vllm-openai:v0.28.0 + image: vllm/vllm-openai:latest args: - --model - Qwen/Qwen3-0.6B @@ -261,7 +261,7 @@ containers: ```yaml containers: - name: vllm - image: vllm/vllm-openai:v0.28.0 + image: vllm/vllm-openai:latest args: - --model - Qwen/Qwen3-0.6B From 4828b92f007a567fe841aaab4e3013ab7c56ffe6 Mon Sep 17 00:00:00 2001 From: Julien Darve Date: Wed, 9 Sep 2026 11:38:22 -0700 Subject: [PATCH 2/2] docs(epp): require an explicit vLLM image for on-ramp Signed-off-by: Julien Darve --- .../ext-proc/examples/onramp/README.md | 4 ++-- .../ext-proc/examples/onramp/agg.yaml | 6 +++--- .../kv-aware-routing/vanilla-vllm-onramp.mdx | 15 ++++++++++----- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/deploy/inference-gateway/ext-proc/examples/onramp/README.md b/deploy/inference-gateway/ext-proc/examples/onramp/README.md index aba2f1b78971..db94a4d3cc87 100644 --- a/deploy/inference-gateway/ext-proc/examples/onramp/README.md +++ b/deploy/inference-gateway/ext-proc/examples/onramp/README.md @@ -17,8 +17,8 @@ For the user-facing walkthrough, start with This on-ramp is available for aggregated serving only at this time. -The aggregated on-ramp uses the floating upstream `vllm/vllm-openai:latest` image. For reproducible -deployments, replace it with a pinned or internally mirrored image your platform standardizes on. +Replace `` in `agg.yaml` with the vLLM image your platform uses, including its release +tag or digest. Use a digest for reproducible deployments. KV-aware selection is provided by the runtime-free [selection service](../../../../../docs/fern/pages/developer-guide/knowledge-base/modular-components/router/standalone-selection.md), which the EPP runs **in-process**: the EPP and the selection service are compiled into one binary, diff --git a/deploy/inference-gateway/ext-proc/examples/onramp/agg.yaml b/deploy/inference-gateway/ext-proc/examples/onramp/agg.yaml index ef9fb807a42c..a7d1dcf74994 100644 --- a/deploy/inference-gateway/ext-proc/examples/onramp/agg.yaml +++ b/deploy/inference-gateway/ext-proc/examples/onramp/agg.yaml @@ -42,8 +42,8 @@ # This secret is required only for gated models; the public model used below # does not require it. # -# Replace `dynamo/dynamo-epp:dev` before applying this manifest unless that -# image is already available to the cluster. +# Replace `` and `dynamo/dynamo-epp:dev` with images available to +# the cluster before applying this manifest. # # Apply with: kubectl apply -n -f agg.yaml # Test: @@ -76,7 +76,7 @@ spec: spec: containers: - name: vllm - image: vllm/vllm-openai:latest + image: args: - --model - Qwen/Qwen3-0.6B diff --git a/docs/fern/pages/kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx b/docs/fern/pages/kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx index b218fdc43ae1..eff44badb87c 100644 --- a/docs/fern/pages/kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx +++ b/docs/fern/pages/kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx @@ -182,10 +182,13 @@ creates the workload layer in `$NAMESPACE`: - Two standalone EPP replicas and their gRPC and replica-synchronization Service. - An `InferencePool` and an `HTTPRoute` that attaches to the cross-namespace Gateway. -Apply the manifest from the repository root using the image you pushed: +Set `VLLM_IMAGE` to the vLLM image your platform uses, including its release tag or digest. Use a +digest for reproducible deployments. Apply the manifest from the repository root using that image +and the EPP image you pushed: ```bash -sed "s#dynamo/dynamo-epp:dev#$EPP_IMAGE#" \ +sed -e "s#dynamo/dynamo-epp:dev#${EPP_IMAGE:?Set EPP_IMAGE to the image you pushed}#" \ + -e "s##${VLLM_IMAGE:?Set VLLM_IMAGE to your chosen vLLM image}#" \ deploy/inference-gateway/ext-proc/examples/onramp/agg.yaml | kubectl apply -n "$NAMESPACE" -f - ``` @@ -238,14 +241,16 @@ kubectl patch deployment/vllm-qwen -n "$NAMESPACE" --type merge \ Do not use `kubectl label deployment` for this step. That command changes `Deployment.metadata.labels`, which does not label the worker pods. -Compare a typical existing vLLM container with the changes required by the on-ramp. +Compare a typical existing vLLM container with the changes required by the on-ramp. In both +snippets, replace `` with your existing vLLM image reference, including its release tag +or digest. **Before — existing vLLM container:** ```yaml containers: - name: vllm - image: vllm/vllm-openai:latest + image: args: - --model - Qwen/Qwen3-0.6B @@ -261,7 +266,7 @@ containers: ```yaml containers: - name: vllm - image: vllm/vllm-openai:latest + image: args: - --model - Qwen/Qwen3-0.6B