feat(kv-router): one session affinity config and validator, mode on every host

components/src/dynamo/frontend/frontend_args.py

@@ -21,6 +21,7 @@
     RouterArgGroup,
     RouterConfigBase,
 )
+from dynamo._core import validate_session_affinity_ttl_secs
 from dynamo.common.configuration.utils import (
     add_argument,
     add_negatable_bool_argument,
@@ -31,7 +32,6 @@
 from . import __version__
 
 _U32_MAX = 2**32 - 1
-_MAX_SESSION_AFFINITY_TTL_SECS = 31_536_000
 
 
 def validate_model_name(value: str) -> str:
@@ -141,12 +141,10 @@ def validate(self) -> None:
             )
         if self.min_initial_workers < 0:
             raise ValueError("--router-min-initial-workers must be >= 0")
-        if self.session_affinity_ttl_secs is not None and not (
-            1 <= self.session_affinity_ttl_secs <= _MAX_SESSION_AFFINITY_TTL_SECS
-        ):
-            raise ValueError(
-                "--router-session-affinity-ttl-secs must be between 1 and "
-                f"{_MAX_SESSION_AFFINITY_TTL_SECS}"
+        if self.session_affinity_ttl_secs is not None:
+            # The same range check the router applies, naming this flag.
+            validate_session_affinity_ttl_secs(
+                self.session_affinity_ttl_secs, "--router-session-affinity-ttl-secs"
             )
         if self.tokenizer_backend not in self._VALID_TOKENIZER_BACKENDS:
             raise ValueError(

deploy/inference-gateway/ext-proc/src/epp_standalone_config.rs

@@ -10,6 +10,7 @@
 //! [`EppStandaloneConfig::from_env`] reads envs, applies defaults, and calls
 //! [`EppStandaloneConfig::validate_config`] for field and cross-field checks.
 
+use dynamo_kv_router::services::selection::affinity::SessionAffinityMode;
 use validator::Validate;
 use validator::ValidationError;
 
@@ -176,13 +177,12 @@ pub struct EppStandaloneConfig {
     pub max_inflight_requests: usize,
     /// Pin each `x-dynamo-session-id` to the worker that served it for this
     /// long after its last request (`DYN_EPP_SESSION_AFFINITY_TTL_SECS`).
-    /// `None` disables session affinity.
-    #[validate(range(
-        min = 1.0,
-        max = 31536000.0,
-        message = "DYN_EPP_SESSION_AFFINITY_TTL_SECS must be between 1 and 31536000 seconds"
-    ))]
+    /// `None` disables session affinity. The range is checked by the shared
+    /// `SessionAffinityConfig` validator when the selector starts.
     pub session_affinity_ttl_secs: Option<f64>,
+    /// How a bound session treats a dispatch that landed elsewhere
+    /// (`DYN_EPP_SESSION_AFFINITY_MODE`, `hard` or `soft`); `hard` unless set.
+    pub session_affinity_mode: SessionAffinityMode,
 }
 
 impl EppStandaloneConfig {
@@ -247,6 +247,11 @@ impl EppStandaloneConfig {
             max_inflight_requests: opt_parse::<usize>(get, "DYN_EPP_MAX_INFLIGHT_REQUESTS")?
                 .unwrap_or(DEFAULT_MAX_INFLIGHT_REQUESTS),
             session_affinity_ttl_secs: opt_parse::<f64>(get, "DYN_EPP_SESSION_AFFINITY_TTL_SECS")?,
+            session_affinity_mode: opt_parse::<SessionAffinityMode>(
+                get,
+                "DYN_EPP_SESSION_AFFINITY_MODE",
+            )?
+            .unwrap_or_default(),
         })
     }
 

deploy/inference-gateway/ext-proc/src/selector.rs

@@ -14,6 +14,7 @@ use anyhow::{Context, Result, anyhow};
 
 use dynamo_kv_router::config::{KvRouterConfig, try_kv_router_config_from_dynamo_env};
 use dynamo_kv_router::protocols::RoutingConstraints;
+use dynamo_kv_router::services::selection::affinity::SessionAffinityConfig;
 use dynamo_kv_router::services::selection::{
     PromptRequest, SelectAndReserveRequest as CoreSelectAndReserveRequest, SelectionError,
     SelectionService, SelectionServiceBuilder, WorkerSelectionPolicyRegistry,
@@ -120,10 +121,12 @@ impl Selector {
             builder = builder.replica_sync(peer_replication.sync_port, Vec::new());
         }
         if let Some(ttl) = cfg.session_affinity_ttl_secs {
-            builder = builder.session_affinity(
-                std::time::Duration::try_from_secs_f64(ttl)
-                    .context("invalid session affinity TTL")?,
-            );
+            builder = builder
+                .session_affinity(
+                    SessionAffinityConfig::ttl_from_secs_f64(ttl)
+                        .context("invalid session affinity TTL")?,
+                )
+                .session_affinity_mode(cfg.session_affinity_mode);
         }
         let service = Arc::new(
             builder
@@ -363,6 +366,7 @@ models:
             max_num_batched_tokens: Some(8192),
             max_inflight_requests: 1024,
             session_affinity_ttl_secs: None,
+            session_affinity_mode: Default::default(),
         }
     }
 

deploy/inference-gateway/ext-proc/src/topology_adapter.rs

@@ -153,6 +153,7 @@ mod tests {
             max_num_batched_tokens: Some(8192),
             max_inflight_requests: 1024,
             session_affinity_ttl_secs: None,
+            session_affinity_mode: Default::default(),
         }
     }
 

docs/fern/pages/developer-guide/knowledge-base/modular-components/router/configuration-and-tuning.md

@@ -336,10 +336,12 @@ The first successfully dispatched request binds the session ID to its selected w
 
 | Mode | Behavior |
 |---|---|
-| `hard` | Default. Exact-dispatch to the stored target. If the worker or rank is no longer valid, invalidate the binding and retry normal selection once |
-| `soft` | Pass the stored target through the normal selection pipeline as an advisory target. The built-in selector retains it while eligible; a custom policy can choose another worker |
+| `hard` | Default. Selection is limited to the bound worker (and rank) while it is eligible, for every selection policy. A hard-bound request goes to its worker and queues there; the router holds it only when every eligible worker is busy, as for any request. A bound worker that discovery no longer knows drops the binding before selection and the session re-binds to the worker the request runs on. A bound worker that is live but excluded by the request's own constraints (allowed workers, taints, migration exclusions, or the overload detector) selects another worker and the request is rejected before dispatch as invalid, keeping the binding |
+| `soft` | The stored target enters the normal selection pipeline as a preference. The built-in selector keeps it while it is a candidate; a custom policy can choose another worker, and the session rebinds to the worker the request ran on |
 
-**Experimental.** Available since v1.6. When session affinity is enabled (`--router-session-affinity-ttl-secs`), a request that carries a parent session id binds under an internal key derived from that parent id instead of its own session, so the subagents of one parent share a binding while the parent keeps its own. Dynamo's affinity coordinator owns that binding: it commits only after a successful dispatch, is version-checked against concurrent updates, expires on the same TTL, and counts against the same global entry limit as any session binding. A request that carries an explicit worker target stays on its own session, so it is neither rejected against the group nor able to move it. Under the default `hard` mode the group is pinned to its first worker and does not migrate because of load; the binding resets only when the target becomes unusable or dispatch fails.
+Explicit request targets (`nvext.backend_instance_id`, `decode_worker_id`, `prefill_worker_id`) stay exact in both modes and must agree with an existing binding. When the affinity table is full (65,536 sessions per router), a new session routes without affinity rather than failing; the router counts these fallbacks.
+
+**Experimental.** Available since v1.6. When session affinity is enabled (`--router-session-affinity-ttl-secs`), a request that carries a parent session id binds under an internal key derived from that parent id instead of its own session, so the subagents of one parent share a binding while the parent keeps its own. Dynamo's session resolver owns that binding: it commits only after a successful dispatch, is version-checked against concurrent updates, expires on the same TTL, and counts against the same global entry limit as any session binding. A request that carries an explicit worker target stays on its own session, so it is neither rejected against the group nor able to move it. Under the default `hard` mode the group is pinned to its first worker and does not migrate because of load; the binding resets only when the bound worker leaves the pool.
 
 One behavior is known and unresolved: because siblings share one binding, a concurrent fan-out waits for the first sibling's dispatch to commit before the others are placed.
 
@@ -354,7 +356,7 @@ python -m dynamo.frontend \
   --router-session-affinity-mode soft
 ```
 
-Concurrent requests can share a binding. Versioned updates prevent an older concurrent request from replacing a newer soft rebind. Active requests prevent expiry. When a request lease ends after EOF, early drop, error, or cancellation, the idle timer restarts. A missing hard-bound worker or a non-cancellation hard-mode selection, setup, dispatch, or target-validation failure invalidates the binding.
+Concurrent requests can share a binding. Versioned updates prevent an older concurrent request from replacing a newer soft rebind. Active requests prevent expiry. When a request lease ends after EOF, early drop, error, or cancellation, the idle timer restarts. A hard-bound worker that leaves the pool invalidates the binding before the next request selects; a failed dispatch binds nothing and leaves an existing binding as it was. The same session layer serves the frontend (KV and builtin routing modes), the standalone selection service, and the EPP, so a binding means the same thing on every host.
 
 The configured value is the idle timeout. It is independent of
 `--router-ttl-secs` and `--router-predicted-ttl-secs`. Omit the session-affinity
@@ -368,7 +370,11 @@ When session affinity is enabled, routers synchronize affinity bindings through
 Runtime event plane. The origin publishes a binding after successful dispatch so
 concurrent requests can observe it, then publishes it again when the request lease
 ends so peer idle timers restart when the request becomes idle. The extra event
-fanout is an intentional tradeoff.
+fanout is an intentional tradeoff. Bindings travel on the `session_affinity_events_v2`
+subject, whose payload names the routing partition (model name and routing group)
+so a binding applies only to the partition it belongs to; for one release routers
+also publish and apply the older partition-less `session_affinity_events` subject so
+mixed-version replicas keep converging.
 
 Synchronization is advisory. Each replica owns its local idle TTL and uses the
 first live binding it observes. A matching update refreshes that local deadline,

docs/fern/pages/developer-guide/knowledge-base/modular-components/router/custom-worker-selection.mdx

@@ -415,7 +415,7 @@ flowchart LR
 
 Match both fields when a policy wants to recognize the target. A target with `dp_rank: None` matches every rank on its worker; a populated rank matches only that worker-rank pair. The target can be absent from the candidate table when the worker is unavailable or a custom filter rejects it.
 
-Hard affinity remains the default. A hard binding and every explicit request target take the exact-target path instead of advisory custom selection. A selection, setup, or dispatch failure before a response stream leaves the previous soft binding intact. An error or cancellation after the stream is returned does not roll back a completed rebind.
+Hard affinity remains the default. Under `hard`, the host limits the candidate set to the bound worker while that worker is eligible, for every policy: your filters, scorers, and picker still run, but only over that worker's data-parallel ranks, and `affinity_target()` names the binding they are already limited to. Explicit request targets are exact in both modes. A selection, setup, or dispatch failure before a response stream leaves the previous binding intact. An error or cancellation after the stream is returned does not roll back a completed rebind.
 
 The [custom policy examples](https://github.com/ai-dynamo/dynamo/blob/main/examples/router/custom-policy-example/README.md) include an overload-aware soft pinning policy that retains the advisory target until its active-request count exceeds a configured threshold, then selects the least-loaded alternative. Its two-Mocker walkthrough holds the first request open and verifies an `A -> B -> B` worker sequence: overload moves the second request to B, and the third request retains B after load drains, proving that the plugin-selected dispatch updated the binding.
 

docs/fern/pages/developer-guide/knowledge-base/modular-components/router/standalone-selection.md

@@ -68,7 +68,8 @@ APIs. Those bindings should wrap `SelectionService` rather than construct
 | `--indexer-peers` | none | Comma-separated HTTP URLs used for startup KV recovery through `/dump`. |
 | `--replica-sync-port` | none | Local ZMQ PUB port for active-load lifecycle events. The selector binds `tcp://*:<port>` internally. |
 | `--replica-sync-peers` | none | Comma-separated ZMQ PUB endpoints for selector peers. Requires `--replica-sync-port`. |
-| `--session-affinity-ttl-secs` | none | Pin each request `session_id` to the worker that served it for this long after its last request (1 to 31,536,000 seconds). When `--replica-sync-port` is configured, bindings replicate to peers over the replica mesh (`dynamo.session-affinity.v1` topic). |
+| `--session-affinity-ttl-secs` | none | Pin each request `session_id` to the worker that served it for this long after its last request (1 to 31,536,000 seconds). When `--replica-sync-port` is configured, bindings replicate to peers over the replica mesh (`dynamo.session-affinity.v2` topic, with `.v1` published and applied alongside it for one release). |
+| `--session-affinity-mode` | `hard` | How a bound session treats a dispatch that landed elsewhere. `hard` limits selection to the bound worker while it is eligible and rejects a booking elsewhere; `soft` offers the binding to the selection policy as a preference and rebinds to the worker booked. |
 | `--selection-cache-ttl-secs` | `120` | Seconds an unclaimed pending selection lives before eviction. |
 | `--selection-cache-max-entries` | `4096` | Maximum resident pending selections, evicting oldest first. |
 | `--selection-cache-max-bytes` | `268435456` | Approximate byte budget across resident pending selections. |
@@ -346,7 +347,7 @@ hash producer with the same algorithm, key, and key ID as the selector.
 
 ### `session_id`
 
-Both `POST /select` and `POST /select_and_reserve` accept an optional `session_id` string. With `--session-affinity-ttl-secs` enabled, each model and routing group has its own binding table. `/select_and_reserve` binds a new session to the selected worker and holds its lease until the reservation is released or expires. The idle TTL starts when the last lease is released. `/select` can use an existing binding but does not create one.
+Both `POST /select` and `POST /select_and_reserve` accept an optional `session_id` string. With `--session-affinity-ttl-secs` enabled, each model and routing group has its own binding table. `/select_and_reserve` binds a new session to the selected worker and holds its lease until the reservation is released or expires. The idle TTL starts when the last lease is released. `/select` can use an existing binding but does not create one. Under `--session-affinity-mode hard` (the default) selection is limited to the bound worker while it is eligible, for every policy; a bound worker that is no longer schedulable drops the binding and the session re-binds to the worker booked. Under `soft` the binding is a preference the policy may override.
 
 Bindings replicate over the configured replica mesh. An explicit `affinity_target` or `pinned_worker` takes precedence over the session binding. Without a configured TTL, `session_id` is only policy input: custom policies can read it through `WorkerSelectionContext::session_id()`, while the built-in selector ignores it. See [Write Custom Routing Strategies](custom-worker-selection.mdx).
 

docs/fern/pages/kubernetes/kv-aware-routing/vanilla-vllm-onramp.mdx

@@ -510,6 +510,7 @@ For a single EPP replica, set `spec.replicas: 1` and remove `DYN_EPP_PEER_SERVIC
 | `DYN_EPP_TOKENIZATION_TIMEOUT_MS` | No | Render request timeout; defaults to `5000`. |
 | `DYN_EPP_TOKENIZER_MAX_RESPONSE_BYTES` | No | Maximum render response size; defaults to `16777216`. |
 | `DYN_EPP_SESSION_AFFINITY_TTL_SECS` | No | Pin each `x-dynamo-session-id` to the worker that served it for this many seconds (1 to 31,536,000) after its last request; the binding replicates to peer EPPs over the replica mesh. Unset disables session affinity. |
+| `DYN_EPP_SESSION_AFFINITY_MODE` | No | `hard` (default) keeps a bound session on its worker while that worker is eligible; `soft` offers the binding to the selection policy as a preference and rebinds to the worker booked. |
 | `DYN_EPP_MAX_INFLIGHT_REQUESTS` | No | Concurrent EPP request guardrail; defaults to `1024`, and excess requests receive `503`. |
 | `DYN_EPP_PEER_SERVICE` | No | Existing EPP Service used to discover sibling replicas. Enables replica sync. |
 | `DYN_EPP_REPLICA_SYNC_PORT` | No | ZMQ port every replica binds and dials for replica sync; defaults to `9092`. All replicas selected by `DYN_EPP_PEER_SERVICE` must use the same value. |

docs/fern/pages/reference/components/frontend-configuration.mdx

@@ -127,6 +127,14 @@ router. See [Router Guide](../../developer-guide/knowledge-base/modular-componen
   Environment variable: `DYN_ROUTER_SESSION_AFFINITY_TTL_SECS`
 </ParamField>
 
+<ParamField path="--router-session-affinity-mode" type="string" default="hard">
+  How an existing session binding takes part in worker selection. `hard` limits selection to the
+  bound worker while it is eligible; `soft` offers the binding to the selection policy as a
+  preference. See [Session affinity](../../developer-guide/knowledge-base/modular-components/router/configuration-and-tuning.md#session-affinity).
+
+  Environment variable: `DYN_ROUTER_SESSION_AFFINITY_MODE`
+</ParamField>
+
 <ParamField path="--decode-fallback / --no-decode-fallback" type="boolean" default="false">
   Fall back to aggregated mode when prefill workers are unavailable.
 

lib/bindings/python/rust/lib.rs

@@ -317,6 +317,10 @@ fn register_core(m: &Bound<'_, PyModule>) -> PyResult<()> {
     )?;
 
     m.add_function(wrap_pyfunction!(llm::kv::compute_block_hash_for_seq_py, m)?)?;
+    m.add_function(wrap_pyfunction!(
+        llm::kv::validate_session_affinity_ttl_secs_py,
+        m
+    )?)?;
     m.add_function(wrap_pyfunction!(lora_name_to_id, m)?)?;
     #[cfg(feature = "mm-routing")]
     m.add_function(wrap_pyfunction!(resolve_routing_image_token_id, m)?)?;