feat(operator): allow EPP replicas in Dynamo mode

deploy/inference-gateway/ext-proc/src/epp.rs

@@ -39,6 +39,18 @@ use crate::epp_router::{endpoint_in_subset, requested_policy_class};
 use crate::picker::{Endpoint, EndpointPicker, PickError, PickResult, RequestInfo, ResponseUsage};
 
 const BOOKKEEPING_TIMEOUT: Duration = Duration::from_secs(5);
+
+/// Formats a worker id as hex directly into the tracing subscriber's buffer.
+/// `format!("{id:x}")` at a per-request callsite allocates a `String` that is
+/// written once and dropped; the operator ships `RUST_LOG=info`, so that
+/// callsite is enabled and the field expression really does run per request.
+struct LowerHex(u64);
+
+impl std::fmt::Display for LowerHex {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:x}", self.0)
+    }
+}
 const DYN_KUBE_DISCOVERY_MODE: &str = "DYN_KUBE_DISCOVERY_MODE";
 
 /// `(token_ids, cache_namespace, priority_jump, strict_priority,
@@ -570,25 +582,26 @@ impl Router {
         is_disaggregated: bool,
         cache_namespace: Option<String>,
     ) -> Result<()> {
-        let decode_router = self.decode_router.clone();
-        let request_id = request_id.to_owned();
-        let tokens = tokens.to_vec();
+        // The timeout future is awaited here, so it can borrow `tokens` and
+        // `request_id` for its whole lifetime. Copying them in would put an
+        // O(prompt_len) allocation on every request for no benefit.
+        let decode_router = &self.decode_router;
 
         tokio::time::timeout(BOOKKEEPING_TIMEOUT, async {
             let worker = WorkerWithDpRank::new(worker_id, dp_rank);
             let router_config_override = decode_router_config_override(is_disaggregated);
 
             let overlap_blocks = decode_router
-                .get_overlap_blocks(&tokens, None, worker, None, cache_namespace.as_deref())
+                .get_overlap_blocks(tokens, None, worker, None, cache_namespace.as_deref())
                 .await
                 .map_err(|e| anyhow::anyhow!("get_overlap_blocks failed: {e:?}"))?;
 
             let cached_tokens = overlap_blocks as usize * decode_router.block_size() as usize;
 
             decode_router
                 .add_request(
-                    request_id,
-                    &tokens,
+                    request_id.to_owned(),
+                    tokens,
                     None,
                     cached_tokens,
                     None,
@@ -607,12 +620,11 @@ impl Router {
 
     /// Mark prefill as completed for a request.
     pub async fn mark_prefill_complete(&self, request_id: &str) -> Result<()> {
-        let decode_router = self.decode_router.clone();
-        let request_id = request_id.to_owned();
+        let decode_router = &self.decode_router;
 
         tokio::time::timeout(BOOKKEEPING_TIMEOUT, async {
             decode_router
-                .mark_prefill_completed(&request_id)
+                .mark_prefill_completed(request_id)
                 .await
                 .map_err(|e| anyhow::anyhow!("mark_prefill_completed failed: {e}"))
         })
@@ -622,12 +634,11 @@ impl Router {
 
     /// Free a request from the router's bookkeeping.
     pub async fn free_request(&self, request_id: &str) -> Result<()> {
-        let decode_router = self.decode_router.clone();
-        let request_id = request_id.to_owned();
+        let decode_router = &self.decode_router;
 
         tokio::time::timeout(BOOKKEEPING_TIMEOUT, async {
             decode_router
-                .free(&request_id)
+                .free(request_id)
                 .await
                 .map_err(|e| anyhow::anyhow!("free failed: {e}"))
         })
@@ -1584,7 +1595,7 @@ impl EndpointPicker for Router {
 
         tracing::info!(
             worker_id = decode_worker.worker_id,
-            worker_id_hex = format!("{:x}", decode_worker.worker_id),
+            worker_id_hex = %LowerHex(decode_worker.worker_id),
             dp_rank = decode_worker.dp_rank,
             is_disaggregated,
             endpoint = %endpoint,

deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go

@@ -1086,14 +1086,29 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) {
 			},
 		},
 		{
-			name: "v1alpha1 EPP requires one replica",
+			name: "v1alpha1 native Rust EPP accepts more than one replica",
 			deployment: alphaDCDWithSharedSpec(nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
 				ComponentType: consts.ComponentTypeEPP,
 				Replicas:      &validMinAvail,
 				ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{
 					MainContainer: &corev1.Container{Image: frontendImage150},
 				},
 			}),
+		},
+		{
+			name: "v1alpha1 legacy Go EPP still requires one replica",
+			deployment: alphaDCDWithSharedSpec(nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
+				ComponentType: consts.ComponentTypeEPP,
+				Replicas:      &validMinAvail,
+				EPPConfig: &nvidiacomv1alpha1.EPPConfig{
+					ConfigMapRef: &corev1.ConfigMapKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{Name: "legacy-epp-config"},
+					},
+				},
+				ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{
+					MainContainer: &corev1.Container{Image: legacyEPPImage140},
+				},
+			}),
 			wantWebhookErrs: []string{
 				"spec.replicas: Invalid value: 2: EPP component must have exactly 1 replica",
 			},

deploy/operator/internal/webhook/validation/shared_v1beta1.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ai-dynamo/dynamo/deploy/operator/internal/consts"
 	"github.com/ai-dynamo/dynamo/deploy/operator/internal/dra"
 	"github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo"
+	"github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/epp"
 	"github.com/ai-dynamo/dynamo/deploy/operator/internal/features"
 	"github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride"
 	corev1 "k8s.io/api/core/v1"
@@ -125,7 +126,11 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpec(
 				allErrs = append(allErrs, field.Forbidden(fldPath.Child("type"), fmt.Sprintf("cannot deploy EPP component: %v", err)))
 			}
 		}
-		if spec.Replicas != nil && *spec.Replicas != 1 {
+		// The native Rust EPP replicates: its replicas share active-sequence
+		// state through the Dynamo event plane. The deprecated Go EPP, selected
+		// by the presence of eppConfig, hosts no Dynamo KV router and has no
+		// equivalent, so it keeps the single-replica restriction it shipped with.
+		if epp.IsLegacyGoEPP(spec.EPPConfig) && spec.Replicas != nil && *spec.Replicas != 1 {
 			allErrs = append(allErrs, field.Invalid(
 				fldPath.Child("replicas"),
 				*spec.Replicas,

docs/fern/pages/reference/components/gateway-api-routing.mdx

@@ -9,6 +9,26 @@ Use this reference for a `DynamoGraphDeployment` (DGD) that routes requests thro
 Inference Extension (GAIE) and the Dynamo Endpoint Picker Plugin (EPP). For the procedure and
 topology, see [Using GAIE with Dynamo](../../kubernetes/kv-aware-routing/gateway-api.mdx).
 
+## EPP Modes
+
+The EPP runs in one of two modes, selected by the `DYN_EPP_MODE` environment variable. This reference
+covers Dynamo mode only.
+
+| Concern | Dynamo mode (`dynamo`, default) | Standalone mode (`standalone`) |
+|---|---|---|
+| Deployed by | Dynamo operator, from a DGD `epp` component | User-applied manifests |
+| Dynamo runtime | Required, with etcd and NATS | Not used |
+| Worker discovery | Dynamo runtime discovery | Kubernetes pod watch driven by `InferencePool.spec.selector` |
+| EPP ServiceAccount and RBAC | Created by the operator against the ClusterRole or Role from the `dynamo-platform` chart | Supplied with the deployment manifests |
+| EPP replication | Available. Replicas exchange active-sequence state through the Dynamo event plane, which the runtime addresses and discovers. It is ZMQ-backed by default and NATS when the runtime is configured for it | Available. Replicas exchange active-sequence state over ZMQ sockets the EPP binds itself, discovering peers from the Service named by `DYN_EPP_PEER_SERVICE` |
+
+Standalone mode does not install or run the Dynamo operator, so no operator-generated resource
+described on this page applies to it, including the EPP ServiceAccount and the chart's EPP RBAC. A
+standalone EPP carries its own ServiceAccount and Role, which grant the permissions that mode needs.
+Do not bind a standalone EPP to the operator's EPP ClusterRole; its rules cover only the resources a
+Dynamo-mode EPP reads. For the standalone path, see
+[Using GAIE with vanilla vLLM](../../kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx).
+
 ## DGD EPP Fields
 
 The EPP is a DGD component with `type: epp`. The Dynamo operator creates the EPP workload, Service,
@@ -31,7 +51,14 @@ and is configured through `DYN_*` environment variables. It implements the GAIE
 </ParamField>
 
 <ParamField path="spec.components[].replicas" type="integer" default="1">
-  Number of EPP pods. Increase this value when endpoint-picker request volume requires more EPP capacity.
+  Number of EPP pods. Increase it for endpoint-picker throughput or to survive the loss of a single
+  EPP pod. Above `1`, the routing accuracy of the fleet becomes approximate, and replicas do not share
+  active-load state unless `DYN_ROUTER_REPLICA_SYNC` is set. Read
+  [EPP Replication and Routing Accuracy](#epp-replication-and-routing-accuracy) before scaling it.
+
+  The deprecated Go EPP, selected by the presence of `eppConfig`, hosts no Dynamo KV router and has
+  no way to share state between replicas. It must have exactly `1` replica, and admission rejects any
+  other value.
 </ParamField>
 
 <ParamField path="spec.components[].frontendSidecar" type="string" required={true}>
@@ -89,7 +116,11 @@ Gateway topology.
 
 <ParamField path="DYN_ROUTER_REPLICA_SYNC" type="boolean" default="false">
   Publishes and consumes best-effort active-sequence state across EPP replicas through the Dynamo event
-  plane.
+  plane. It is off by default, matching the Frontend-hosted router, and is set through `podTemplate`
+  env. It has no peer to synchronize with at one replica, and the deprecated Go EPP cannot be
+  replicated at all. It reduces cross-replica divergence rather than removing it, and it carries a
+  throughput cost. See
+  [EPP Replication and Routing Accuracy](#epp-replication-and-routing-accuracy).
 </ParamField>
 
 <ParamField path="DYN_ROUTER_TRACK_ACTIVE_BLOCKS" type="boolean" default="true">
@@ -107,6 +138,124 @@ Gateway topology.
 For the full shared configuration surface and tuning guidance, see
 [Configuration and Tuning](../../developer-guide/knowledge-base/modular-components/router/configuration-and-tuning.md).
 
+## EPP Replication and Routing Accuracy
+
+Running more than one EPP replica trades exact load accounting for endpoint-picker throughput and
+fault tolerance. The trade is the same one the Frontend-hosted KV router already makes when it is
+replicated, described in
+[Router Operations](../../developer-guide/knowledge-base/modular-components/router/router-operations.md).
+This section states what a replicated EPP gets wrong, and by how much.
+
+The router keeps two kinds of state, and only one of them is affected.
+
+**Prefix-cache state converges on its own.** Workers publish KV `Stored` and `Removed` events to the
+Dynamo event plane, and every subscriber receives every event, so each replica builds an equivalent
+radix tree without any router-to-router traffic. Cache-overlap scoring, which dominates worker choice
+on prefix-heavy traffic, is therefore as accurate with ten replicas as with one. A replica that starts
+late rebuilds prefix state by querying each worker's local indexer.
+
+**Active-block state does not.** A booking is recorded by the replica that routed the request, so each
+replica sees only the load it placed itself. `DYN_ROUTER_REPLICA_SYNC` shares the request lifecycle —
+add, prefill-complete, and free — across replicas. It is off by default. That channel is
+fire-and-forget: its outbound queue drops the newest event when full, there is
+no acknowledgement, replay, or resynchronization, and replicas can disagree for a bounded period.
+Output-block growth is never shared, because it changes at a frequency that would consume
+disproportionate bandwidth, so decode-side load estimates drift between replicas whenever
+`DYN_ROUTER_TRACK_OUTPUT_BLOCKS` is on.
+
+Enabling it does not make active-load accounting correct. It changes the error from permanent to
+transient. Left off, each replica scores workers from its own slice of the traffic and understates load
+by roughly the replica count, and every replica is wrong in the same direction, so the errors compound
+rather than averaging out. Turned on, replicas converge on a fleet-wide view between the gaps above.
+Those gaps are not symmetric when the outbound queue drops an event: a dropped `add` hides a request
+from a peer for that request's lifetime, while a dropped `free` leaves a phantom booking on the peer
+until the reaper clears it.
+
+#### Choosing Whether to Enable Replica Synchronization
+
+It is off by default, which matches the Frontend-hosted router. Leaving it off is a real choice rather
+than an oversight, because sharing is not free.
+
+Every replica applies every other replica's lifecycle events, so the work of maintaining the shared
+view does not divide as replicas are added. A replica publishes the events for the requests it routed
+and receives the events for all the rest, which converges on the whole fleet's request rate no matter
+how many replicas there are. An `add` also carries one block hash per KV block of its prompt, so the
+per-event cost rises with prompt length. Adding replicas scales out request handling; it does not scale
+out this bookkeeping.
+
+Enable it when the cost is bounded and the benefit is real:
+
+- Traffic is unevenly distributed across replicas, so a local view is a poor sample of the fleet.
+- Active-load precision drives worker choice, as with long generations or tight latency targets.
+- Request rates and prompt lengths are modest enough that the added event volume is comfortable.
+
+Leave it off when traffic is already spread evenly across replicas, when cache-overlap scoring rather
+than active load drives most decisions, or when the deployment is scaled out for throughput at prompt
+lengths and request rates that would make fleet-wide event volume expensive.
+
+Measure rather than assume. The publisher and the ingress both expose Prometheus metrics, so the added
+volume and any dropped events are visible once it is on.
+
+### Booking Ownership on Disconnect and Retry
+
+The Frontend owns a request end to end and frees its booking when the work ends. The EPP only selects,
+so it has no direct view of the worker: it watches the response through `ext_proc` and frees when that
+stream ends.
+
+In practice the free lands where the Frontend's does. A stream that runs to completion ends when the
+response does. A stream that ends early takes the worker with it, because agentgateway drops fail-open
+once request headers are sent and has no response-side fail-open: an `ext_proc` failure mid-response
+ends the exchange, closing the stream to the worker, and the worker's Frontend sidecar cancels
+generation on the disconnect. Either way nothing keeps running that the EPP has stopped accounting
+for.
+
+Two effects remain:
+
+- **A retry can land on a different EPP.** The gateway load-balances `ext_proc` streams across EPP pods
+  with no request affinity, so a retried request re-enters admission on whichever replica it reaches.
+  Until the original `free` propagates, a replica that has not received it holds both bookings and
+  counts the request twice.
+- **Cancellation is not instant.** A disconnected decode request may hold router state for up to
+  `CLEANUP_DISPATCH_TIMEOUT` (120 seconds) while it finishes reaching its worker, so the free and the
+  release of worker capacity are close but not simultaneous.
+
+This reasoning depends on the gateway resetting the upstream. A gateway that instead lets the upstream
+run would leave the EPP accounting for nothing while the worker is still busy, which understates that
+worker across every replica that received the `free`.
+
+### Selection-to-Booking Race
+
+Replica synchronization is eventually consistent, so it cannot serialize selection. Two replicas can
+evaluate the same worker as least-loaded before either one's booking has propagated, and both send work
+to it. The size of the error grows with request arrival rate and with propagation delay, and no setting
+removes it. `DYN_ROUTER_TEMPERATURE` above `0` decorrelates replicas by sampling among near-equal
+workers, which reduces how often they converge on the same choice.
+
+### Queue and Admission State
+
+Replica synchronization covers active-sequence state. It does not cover the router queue, and that
+is a different kind of loss from the accuracy drift above.
+
+When router queueing is enabled, a request waits in the queue of the replica that received it. That
+queue is local to the replica and does not outlive it. No peer adopts the queued requests of a
+replica that goes away, and none of them is replayed. A replica also cannot see what its peers have
+queued, so admission decisions, fairness, and priority ordering hold within a replica rather than
+across the fleet. Making them fleet-wide would need strong consistency between replicas, which
+best-effort synchronization does not provide.
+
+Plan for this at the layer above. A client or gateway retry re-enters admission on whichever replica
+serves the retry, which is the recovery path available today. Weigh that against the workload's
+tolerance before enabling queueing and replication together: a queue that absorbs a long backlog
+concentrates more work in each replica, so more is lost when one is replaced during a rolling update
+or a node failure.
+
+### Choosing a Replica Count
+
+Use one replica when active-load precision matters more than endpoint-picker throughput, or when the
+workload is long-generation and sensitive to decode-side balance. Use several when the EPP itself is
+the bottleneck or its availability is, and the traffic is prefix-heavy enough that cache-overlap
+scoring — the part that stays accurate — drives most of the routing decision.
+
 ## Generated Resource Contract
 
 For a DGD named `<dgd-name>`, the operator generates an `InferencePool` named `<dgd-name>-pool` in