diff --git a/core/schemas/bifrost.go b/core/schemas/bifrost.go index 246fe0c62b4..594dfcfb527 100644 --- a/core/schemas/bifrost.go +++ b/core/schemas/bifrost.go @@ -272,7 +272,7 @@ const ( BifrostContextKeyGovernanceRoutingRuleID BifrostContextKey = "bifrost-governance-routing-rule-id" // string (to store the routing rule ID (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceRoutingRuleName BifrostContextKey = "bifrost-governance-routing-rule-name" // string (to store the routing rule name (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceComplexityTier BifrostContextKey = "bifrost-governance-complexity-tier" // string (complexity tier computed for routing, e.g. "SIMPLE"/"MEDIUM"/"COMPLEX"; only present when a routing rule referenced complexity_tier and classification produced a tier (set by bifrost routing plugin - DO NOT SET THIS MANUALLY)) - BifrostContextKeyGovernanceComplexityMechanism BifrostContextKey = "bifrost-governance-complexity-mechanism" // string (how the complexity tier was classified: "semantic", or "skipped" when classification was demanded but produced no tier; only present when a routing rule referenced complexity_tier (set by bifrost routing plugin - DO NOT SET THIS MANUALLY)) + BifrostContextKeyGovernanceComplexityMechanism BifrostContextKey = "bifrost-governance-complexity-mechanism" // string (how the effective complexity tier was determined: "semantic", "llm", "session", or "skipped" when classification was demanded but produced no tier; only present when a routing rule referenced complexity_tier (set by bifrost routing plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyGovernanceComplexityScore BifrostContextKey = "bifrost-governance-complexity-score" // float64 (classifier score behind the tier: the semantic classifier's similarity to the nearest reference phrase; only present alongside a computed tier (set by bifrost routing plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeyRoutingPinnedAPIKeyID BifrostContextKey = "bifrost-routing-pinned-api-key-id" // string (provider key ID pinned by a matched routing rule target; resolved against the configured key pool during key selection and takes precedence over a caller-supplied pin (set by bifrost governance plugin - DO NOT SET THIS MANUALLY)) BifrostContextKeySelectedPromptName BifrostContextKey = "bifrost-selected-prompt-name" // string (display name of the selected prompt (set by prompts plugin - DO NOT SET THIS MANUALLY)) diff --git a/docs/deployment-guides/helm/governance.mdx b/docs/deployment-guides/helm/governance.mdx index 39c7b703ec3..b0005836ebc 100644 --- a/docs/deployment-guides/helm/governance.mdx +++ b/docs/deployment-guides/helm/governance.mdx @@ -406,12 +406,16 @@ bifrost: message_history_count: 1 count_toward_budgets: false vector_store: "embedded" + session: + enabled: true keywords: simple_keywords: ["what is a mutex?", "fix the grammar in this sentence."] medium_keywords: ["add api-key auth: hash the keys, reject revoked ones, and never log them."] complex_keywords: ["balance testing, prescribing rules, and staffing against rising resistant infections."] ``` +`session.enabled` is optional and defaults to `false`. When enabled, an identified session retains its highest observed tier for 24 hours of inactivity; normally sequential turns can escalate while lower proposals keep the stored tier. Overlapping requests for the same session are best-effort and resolve by last writer wins. The lifetime is built in and is separate from provider prompt-cache TTLs. + In the default split mode, runtime UI and API edits are preserved while the matching Helm-rendered section is unchanged. When Helm changes a section, keyword lists are merged additively with stored runtime phrases (union with duplicates removed), and the semantic block is replaced as one unit. Use `bifrost.sourceOfTruth: config.json` only when Helm should replace stored governance state. See [Source of Truth & Reconciliation](/deployment-guides/config-json/source-of-truth) for the full startup rules. diff --git a/docs/features/governance/complexity-router.mdx b/docs/features/governance/complexity-router.mdx index aa421498ec0..01a753cf397 100644 --- a/docs/features/governance/complexity-router.mdx +++ b/docs/features/governance/complexity-router.mdx @@ -16,7 +16,7 @@ complexity_tier in ["MEDIUM", "COMPLEX"] This lets you route simple greetings to a fast, cheap model and deep reasoning tasks to a frontier model automatically, with no changes to your application code. -Classification runs only when a routing rule actually references `complexity_tier`, so requests that never touch a complexity rule pay no embedding cost. Once semantic classification is configured, a request it cannot confidently match — a near miss, a timeout, an in-progress warmup — leaves `complexity_tier` unpublished. You can optionally configure an **LLM fallback classifier** to step in for exactly those requests — see [LLM fallback classifier](#llm-fallback-classifier). The fallback engages only after semantic classification has actually run; without semantic classification configured at all, Bifrost keeps the request on its existing routing path instead of guessing. +Classification runs only when a routing rule actually references `complexity_tier`, so requests that never touch a complexity rule pay no embedding cost. Once semantic classification is configured, a request it cannot confidently match, such as a near miss or timeout, leaves `complexity_tier` unpublished. You can optionally configure an **LLM fallback classifier** to step in after semantic classification has run and returned no tier. See [LLM fallback classifier](#llm-fallback-classifier). Without semantic classification configured at all, Bifrost keeps the request on its existing routing path instead of guessing. When [session-aware routing](#session-aware-routing) is enabled and the request carries a recognized session identity, a turn that still produces no tier of its own reuses the tier already retained for that session, so `complexity_tier` goes unpublished only when neither classification nor session state supplies one. Complexity classification is **semantic** (embedding-based). The older lexical keyword scorer is retired. See [Lexical keyword classifier (retired)](#lexical-keyword-classifier-retired) for details and migration guidance. @@ -55,16 +55,32 @@ With semantic classification configured, every tier must contain at least one ph `message_history_count` (default `1`) controls how many recent user messages are joined into the embedded text. Raising it lets a short follow-up like "and make it faster" inherit the intent of earlier turns, at the cost of diluting the latest message and embedding more tokens per request. Requests with fewer available turns embed what they have. +### Session-aware routing + +Enable **Session-aware routing** to balance cost and quality with an upward-only complexity ladder inside an agent conversation. The first classifiable user turn that produces a tier establishes the session tier. Each later sequential human turn is classified normally and can raise that tier from Simple to Medium or Complex, while an easier follow-up keeps the stored higher tier. This avoids unnecessary tier-driven model changes that can reduce provider prompt-cache reuse. Once a session reaches Complex, Bifrost reuses Complex without another classifier call. + +Session state expires after **24 hours of inactivity**. Each participating conversational turn refreshes that inactivity window. After expiry, the next classifiable human request starts a new session epoch and is classified normally. Bifrost stores only the effective tier under a scoped hash of the session identity; it does not store prompts, similarity scores, reference phrases, model choices, or turn history as session state. + +Bifrost uses the explicit `x-bf-session-id` when supplied. For recognized agent harnesses it can also use their native, User-Agent-gated identity: `x-codex-turn-metadata.session_id` for Codex and `x-claude-code-session-id` for Claude Code. Codex background work (`prewarm`, `compaction`, and `memory`) bypasses session state. Supported conversational continuations with no new human text may reuse an existing tier, but never initialize or escalate one. Requests with no valid identity retain ordinary per-request classification. + + +Complexity Router does not currently classify Codex requests sent through native WebSocket Responses mode, so session-aware routing does not apply on that path. Codex over HTTP/SSE Responses, and WebSocket requests using Bifrost's HTTP bridge, remain supported. + + + +Session-aware routing keeps the **complexity tier** stable; it does not pin a weighted routing target, provider key, or provider prompt-cache entry. Provider cache TTLs remain provider-owned and independent of the 24-hour routing-state lifetime. + + --- ## LLM fallback classifier By default, a request that matches no reference phrase confidently simply carries no `complexity_tier`. If you'd rather have a second opinion than let those requests fall through, set semantic classification's `fallback` to `llm` and configure a chat model to name the tier instead. -The LLM fallback runs **only after** semantic classification produces no tier — never as the primary classifier, and never in parallel with it. It never sees a request that semantic classification already resolved. +The LLM fallback runs **only after** semantic classification produces no tier: never as the primary classifier, and never in parallel with it. It never sees a request that semantic classification already resolved. -The cost of this classifier is latency, paid on every request it runs for. A request that reaches the fallback waits on one full chat completion from the configured model before it is routed. Pick a small, fast model, and use `timeout` to cap the wait — a timed-out classification skips complexity routing for that request, exactly like an unmatched semantic request without a fallback. +The cost of this classifier is latency, paid on every request it runs for. A request that reaches the fallback waits on one full chat completion from the configured model before it is routed. Pick a small, fast model, and use `timeout` to cap the wait. A timed-out classification skips complexity routing for that request unless session-aware routing can reuse a tier already retained for its session, exactly like an unmatched semantic request without a fallback. The fallback model is asked to answer with one of the three tier names, guided by a prompt you can edit (`prompt`, or **Fallback Classification Prompt** on the Complexity Router page). Bifrost always appends a fixed, non-editable section stating the tier names and the required JSON response shape, so your edits refine *what the tiers mean* to the model but can never break the response contract. Leaving `prompt` empty uses Bifrost's shipped default guidance. @@ -72,7 +88,7 @@ The fallback model is asked to answer with one of the three tier names, guided b `message_history_count` behaves the same way it does for semantic classification: it controls how many of the most recent user messages (oldest first) are sent to the fallback model, independent of the semantic classifier's own `message_history_count`. -An LLM-classified turn carries no similarity score — a chat completion has no equivalent of embedding-distance, and a synthetic one would invite comparisons against thresholds tuned for your vector backend. `complexity_score` is therefore absent on rows where `complexity_mechanism` is `llm`. See [Observability](#observability). +An LLM-classified turn carries no similarity score. A chat completion has no equivalent of embedding-distance, and a synthetic one would invite comparisons against thresholds tuned for your vector backend. `complexity_score` is therefore absent on rows where `complexity_mechanism` is `llm`. See [Observability](#observability). --- @@ -84,15 +100,15 @@ Semantic classification requires an embedding provider and model. The provider m +![Embedding configuration](../../media/ui-complexity-router-embedding-configuration.png) + Navigate to **Complexity Router** in the sidebar. - **Phrase to Tier Mapping**: add a phrase by typing it and pressing **Enter** in a tier's input; remove one with the × on its chip. Counts are shown per tier. -- **Edit embedding configuration**: opens the embedding sheet (provider, model, similarity floor, history window, timeout, budgets, and phrase storage: **Embedded** keeps phrase vectors in Bifrost's own memory; **Vector Store** keeps them in the configured vector store so they survive restarts, falling back to Embedded when none is available). Setting **When no phrase matches confidently** to **LLM classifier** reveals a **Fallback classifier** section further down the same sheet — provider, model, timeout, history window, and budgets for the fallback model. Setting it back to **None** hides that section again; its settings are preserved either way. +- **Session-aware routing**: retain the highest tier reached by each identified session for 24 hours of inactivity. The toggle is off by default and requires the semantic classifier. +- **Edit embedding configuration**: opens the embedding sheet (provider, model, similarity floor, history window, timeout, budgets, and phrase storage: **Embedded** keeps phrase vectors in Bifrost's own memory; **Vector Store** keeps them in the configured vector store so they survive restarts, falling back to Embedded when none is available). Setting **When no phrase matches confidently** to **LLM classifier** reveals a **Fallback classifier** section further down the same sheet: provider, model, timeout, history window, and budgets for the fallback model. Setting it back to **None** hides that section again; its settings are preserved either way. - When the fallback is on, a **Fallback Classification Prompt** section appears on the main page below the phrase lists, with a **Reset to default** button. The model itself is configured in the embedding sheet; only the prompt text lives here, since it needs room to iterate. - The **Classifier status** badge in the header shows whether the classifier is ready to serve (see [Classifier status and warmup](#classifier-status-and-warmup)). -- Click **Save changes** to apply (hot-reloaded, no restart), **Discard changes** to revert unsaved edits, or **Restore defaults** to restore the built-in reference phrases. Restore defaults keeps the embedding and fallback configuration. - -![Embedding configuration](../../media/ui-complexity-router-embedding-configuration.png) @@ -117,6 +133,9 @@ curl -X PUT http://localhost:8080/api/routing/complexity-analyzer-config \ "vector_store": "embedded", "fallback": "none" }, + "session": { + "enabled": true + }, "keywords": { "simple_keywords": ["what is a mutex?", "fix the grammar in this sentence."], "medium_keywords": ["add api-key auth: hash the keys, reject revoked ones, and never log them."], @@ -147,7 +166,7 @@ curl -X PUT http://localhost:8080/api/routing/complexity-analyzer-config \ } }' -# Check classifier status (includes llm readiness and the default prompt when llm is configured) +# Check classifier status (always includes llm readiness and the default prompt) curl http://localhost:8080/api/routing/complexity-analyzer-status # Restore built-in reference phrases (embedding configuration is preserved) @@ -181,6 +200,9 @@ Reference-phrase lists are stored in the existing `keywords` fields (`simple_key "message_history_count": 1, "count_toward_budgets": false }, + "session": { + "enabled": true + }, "keywords": { "simple_keywords": ["what is a mutex?", "fix the grammar in this sentence."], "medium_keywords": ["add api-key auth: hash the keys, reject revoked ones, and never log them."], @@ -202,11 +224,12 @@ Reference-phrase lists are stored in the existing `keywords` fields (`simple_key | `semantic.vector_store` | string | `embedded` | `embedded` uses Bifrost's built-in in-memory store and re-embeds phrases on restart. `vector_store` uses the configured top-level `vector_store`; if none is configured, it safely uses Embedded instead. | | `semantic.fallback` | string | `none` | What answers when semantic classification produces no tier: `none` records the request as skipped; `llm` asks the model configured in `llm` below. Requires `llm` to be set | | `llm.provider` | string | Required when `fallback` is `llm` | Provider used to run the classification chat completion; must have an enabled key | -| `llm.model` | string | Required when `fallback` is `llm` | Chat model asked to name the tier. Pick a small, fast one — every fallback classification waits on one completion | +| `llm.model` | string | Required when `fallback` is `llm` | Chat model asked to name the tier. Pick a small, fast one; every fallback classification waits on one completion | | `llm.timeout` | duration | `4s` | Ceiling on the classification completion; exceeding it skips tier routing for that request | | `llm.prompt` | string | Shipped default guidance | Replaces the shipped classification guidance (max 4,000 characters). The tier-name and response-format reinforcement is appended by Bifrost regardless and cannot be edited | | `llm.message_history_count` | integer | `1` | Number of recent user messages sent to the classifier, oldest first (1–10) | | `llm.count_toward_budgets` | boolean | `false` | Count classification completion cost toward virtual-key budgets (record-only, never enforced) | +| `session.enabled` | boolean | `false` | Retain the highest observed tier across normally sequential turns for 24 hours of inactivity. Requires `semantic`; overlapping requests for the same session are best-effort | | `keywords.simple_keywords` | string[] | 50 built-in phrases | Reference phrases for the Simple tier | | `keywords.medium_keywords` | string[] | 50 built-in phrases | Reference phrases for the Medium tier | | `keywords.complex_keywords` | string[] | 50 built-in phrases | Reference phrases for the Complex tier | @@ -243,7 +266,7 @@ The same response always also carries the LLM fallback classifier's own status, | Field | Values | Meaning | |---|---|---| -| `llm.state` | `disabled`, `ready` | `disabled` means no `llm` block is configured; `ready` means it is. Unlike semantic classification, the LLM fallback has no warmup — it makes its first provider call on the first classification it runs, so it is ready as soon as it is saved. | +| `llm.state` | `disabled`, `ready` | `disabled` means no `llm` block is configured; `ready` means it is. Unlike semantic classification, the LLM fallback has no warmup: it makes its first provider call on the first classification it runs, so it is ready as soon as it is saved. | | `llm_default_prompt` | string | The shipped classification guidance, served so a configuration client (like the **Fallback Classification Prompt** editor) can seed itself and offer a reset without holding a copy that drifts from the gateway's. Present regardless of whether an `llm` block is configured. | --- @@ -371,8 +394,8 @@ When a routing rule references `complexity_tier`, the classification outcome is | Field | Values | Meaning | |---|---|---| | `complexity_tier` | `SIMPLE`, `MEDIUM`, `COMPLEX` | The tier the request was classified into | -| `complexity_mechanism` | `semantic`, `llm`, `skipped` | How the tier was produced. `semantic` means an embedding match produced the tier; `llm` means the fallback chat model named the tier after semantic classification produced none; `skipped` means a rule demanded a tier but neither produced one (classifier not configured or not ready, unsupported input, failure/timeout, or a match below `min_similarity`) | -| `complexity_score` | 0.0 – 1.0 | The similarity score of the nearest reference phrase. Never set when `complexity_mechanism` is `llm` — a chat completion has no equivalent similarity score | +| `complexity_mechanism` | `semantic`, `llm`, `session`, `skipped` | How the effective tier was produced. `semantic` means an embedding match supplied it; `llm` means the fallback model named it; `session` means retained session state supplied it because the current turn was a continuation, proposed a lower tier, produced no tier, or followed the Complex ceiling; `skipped` means a rule demanded a tier but neither a classifier nor existing session state produced one | +| `complexity_score` | 0.0 – 1.0 | The similarity score of the nearest reference phrase. Set only when the effective decision is the current semantic match; absent for `llm`, `session`, and `skipped` | The routing decision logs also record the matched reference phrase alongside the tier and similarity, so you can tell a genuine match from an accidental one. Long phrases are truncated to 120 characters in the log line. @@ -399,19 +422,19 @@ curl "http://localhost:8080/api/logs?complexity_tiers=COMPLEX&complexity_mechani ``` -The raw `complexity_score` is displayed but not filterable; tier and mechanism are the supported filter dimensions. The mechanism filter offers `semantic`, `llm`, and `skipped`. Legacy `REASONING` tiers remain available in the logs filter. +The raw `complexity_score` is displayed but not filterable; tier and mechanism are the supported filter dimensions. The mechanism filter offers `semantic`, `llm`, `session`, and `skipped`. Legacy `REASONING` tiers remain available in the logs filter. ### In telemetry -The tier and mechanism are also emitted as the span attributes `bifrost.complexity_tier` and `bifrost.complexity_mechanism`, and as low-cardinality labels on Prometheus metrics. The raw score is deliberately excluded from metrics (unbounded cardinality). It lives only in the request logs. +The tier and mechanism are also emitted as the span attributes `bifrost.complexity_tier` and `bifrost.complexity_mechanism`, and as low-cardinality labels on Prometheus metrics. The raw score is emitted as the span attribute `bifrost.complexity_score` and stored in request logs, but deliberately excluded from metrics because it has unbounded cardinality. Semantic routing's own embedding overhead is tracked separately with two Prometheus counters, labeled by the embedding provider, model, and `phase` (`request` classification vs `warmup` exemplar embedding): - `bifrost_routing_embedding_requests_total` - `bifrost_routing_embedding_cost_total` (USD; recorded whether or not `count_toward_budgets` is set) -The LLM fallback classifier's own completion overhead is tracked separately too, with two Prometheus counters labeled by the fallback provider and model (no `phase` label — the fallback has no warmup): +The LLM fallback classifier's own completion overhead is tracked separately too, with two Prometheus counters labeled by the fallback provider and model (no `phase` label; the fallback has no warmup): - `bifrost_routing_llm_requests_total` - `bifrost_routing_llm_cost_total` (USD; recorded whether or not `count_toward_budgets` is set) @@ -424,7 +447,7 @@ See [Telemetry](../telemetry) and [Prometheus](../observability/prometheus) for ### No tier is ever published (everything is `skipped`) -The most common cause is that semantic classification is not configured. Without a configured semantic classifier, no fallback runs either — the LLM fallback only ever engages after semantic classification has actually been invoked, never as a substitute for missing semantic configuration. Check the classifier status badge or `GET /api/routing/complexity-analyzer-status`: +The most common cause is that semantic classification is not configured. Without a configured semantic classifier, no fallback runs either; the LLM fallback only ever engages after semantic classification has actually been invoked, never as a substitute for missing semantic configuration. Check the classifier status badge or `GET /api/routing/complexity-analyzer-status`: - `disabled`: set an embedding provider and model, and make sure the provider has an enabled key. - `warming`: warmup is embedding the reference phrases. If `serving_previous` is true, the last good generation remains available while it runs. @@ -436,11 +459,11 @@ Also verify a routing rule actually references `complexity_tier`; classification ### Setting `fallback` to `llm` is rejected -Semantic classification's `fallback` field requires a companion `llm` block with at least `provider` and `model` set — the update endpoint rejects `fallback: "llm"` without one. Configure the LLM fallback classifier (Web UI: the **Fallback classifier** section inside the embedding sheet; API/config.json: the `llm` block) before or in the same request that sets `fallback` to `llm`. +Semantic classification's `fallback` field requires a companion `llm` block with at least `provider` and `model` set; the update endpoint rejects `fallback: "llm"` without one. Configure the LLM fallback classifier (Web UI: the **Fallback classifier** section inside the embedding sheet; API/config.json: the `llm` block) before or in the same request that sets `fallback` to `llm`. ### LLM fallback times out or never runs -Check `llm.state` on `GET /api/routing/complexity-analyzer-status`: `disabled` means no `llm` block is saved. If it's `ready` but classifications still show `complexity_mechanism: skipped`, check `llm.timeout` — the fallback model may be too slow for the configured budget. Provider errors and timeouts are recorded in the routing decision logs alongside the cause. +Check `llm.state` on `GET /api/routing/complexity-analyzer-status`: `disabled` means no `llm` block is saved. If it's `ready` but classifications still show `complexity_mechanism: skipped`, check `llm.timeout`: the fallback model may be too slow for the configured budget. Provider errors and timeouts are recorded in the routing decision logs alongside the cause. ### Rule not matching when complexity_tier is set @@ -450,7 +473,7 @@ If classification is unavailable for a request (unsupported input, mixed-modal c ### Which request types are supported -Complexity routing currently runs only for **text-bearing** request families. This applies identically to the LLM fallback classifier — it shares the same input extraction as semantic classification, so a request semantic classification cannot analyze reaches the fallback in the same unclassifiable state. Supported inputs include: +Complexity routing currently runs only for **text-bearing** request families. This applies identically to the LLM fallback classifier. It shares the same input extraction as semantic classification, so a request semantic classification cannot analyze reaches the fallback in the same unclassifiable state. Supported inputs include: - Chat Completions and other messages-style requests with text-only user content - Text Completions requests using `prompt` diff --git a/docs/features/observability/datadog.mdx b/docs/features/observability/datadog.mdx index 487d1e49288..550566884f6 100644 --- a/docs/features/observability/datadog.mdx +++ b/docs/features/observability/datadog.mdx @@ -454,7 +454,7 @@ When a routing rule references `complexity_tier`, two additional tags are set: - `complexity_tier` - The complexity tier the request was classified into: `SIMPLE`, `MEDIUM`, or `COMPLEX` (Datadog normalizes tag values to lowercase, so query as `simple`/`medium`/`complex`) - `complexity_mechanism` - How the tier was classified: `semantic`, or `skipped` when classification ran but produced no tier -Use them to attribute cost, latency, and volume to classified complexity, e.g. `sum:bifrost.request.cost.usd{complexity_tier:complex} by {model}`. The raw complexity score is not exported as a tag (its cardinality is unbounded); it is available only in the log store. +Use them to attribute cost, latency, and volume to classified complexity, e.g. `sum:bifrost.request.cost.usd{complexity_tier:complex} by {model}`. The raw complexity score is not exported as a metric tag because its cardinality is unbounded; it remains available in request logs and trace attributes. --- diff --git a/docs/features/observability/prometheus.mdx b/docs/features/observability/prometheus.mdx index c998fa17e99..2a40dbd7e8d 100644 --- a/docs/features/observability/prometheus.mdx +++ b/docs/features/observability/prometheus.mdx @@ -239,7 +239,7 @@ Most request-level Bifrost LLM metrics include these labels (the `bifrost_key_ro - `routing_engine_used` - Comma-separated list of routing engines that contributed to the decision (e.g. `governance`, `routing-rule`, `loadbalancing`, `model-catalog`, `core`). `core` is emitted when the Bifrost orchestrator itself makes a routing decision — fallback transitions or retry transitions. - `routing_rule_id` / `routing_rule_name` - Routing rule that matched the request - `complexity_tier` - Complexity tier used for routing (`SIMPLE` / `MEDIUM` / `COMPLEX`); empty when no routing rule referenced `complexity_tier` -- `complexity_mechanism` - How the complexity tier was classified (`semantic` for the embedding-based classifier, or `skipped` when classification was demanded but produced no tier). The raw complexity score is deliberately not a label because it has unbounded cardinality and lives only in the request logs +- `complexity_mechanism` - How the effective complexity tier was determined (`semantic`, `llm`, `session`, or `skipped` when no tier was produced). The raw complexity score is deliberately not a label because it has unbounded cardinality; it remains available in request logs and trace attributes - `selected_key_id` / `selected_key_name` - API key that successfully served the request (`""` when all attempts failed) - `fallback_index` - Fallback position - `team_id` / `team_name` - Team identifiers (empty when governance is not used) diff --git a/docs/features/telemetry.mdx b/docs/features/telemetry.mdx index 61c8f58629a..e3d7a74bb98 100644 --- a/docs/features/telemetry.mdx +++ b/docs/features/telemetry.mdx @@ -76,7 +76,7 @@ Base Labels: - `routing_rule_id`: Routing rule ID that matched the request - `routing_rule_name`: Routing rule name that matched the request - `complexity_tier`: Complexity tier used for routing (`SIMPLE` / `MEDIUM` / `COMPLEX`); empty when no routing rule referenced `complexity_tier` -- `complexity_mechanism`: How the complexity tier was classified (`semantic` for the embedding-based classifier, or `skipped` when classification was demanded but produced no tier). The raw complexity score is deliberately not a label because it has unbounded cardinality; it is recorded in request logs and traces +- `complexity_mechanism`: How the effective complexity tier was determined (`semantic`, `llm`, `session`, or `skipped` when no tier was produced). The raw complexity score is deliberately not a label because it has unbounded cardinality; it is recorded in request logs and traces - `selected_key_id`: ID of the key that successfully served the request (empty string `""` on final errors) - `selected_key_name`: Name of the key that successfully served the request (empty string `""` on final errors) - `fallback_index`: Fallback index (0 for first attempt, 1 for second attempt, etc.) diff --git a/docs/media/ui-complexity-router-embedding-configuration.png b/docs/media/ui-complexity-router-embedding-configuration.png index e4a7497ebe1..99e1dc501c3 100644 Binary files a/docs/media/ui-complexity-router-embedding-configuration.png and b/docs/media/ui-complexity-router-embedding-configuration.png differ diff --git a/docs/media/ui-complexity-router-semantic.png b/docs/media/ui-complexity-router-semantic.png index d1a5681b7a9..cd8c93b9601 100644 Binary files a/docs/media/ui-complexity-router-semantic.png and b/docs/media/ui-complexity-router-semantic.png differ diff --git a/docs/openapi/openapi.json b/docs/openapi/openapi.json index e298700c7c7..0a1423f0bfe 100644 --- a/docs/openapi/openapi.json +++ b/docs/openapi/openapi.json @@ -68641,7 +68641,9 @@ "operationId": "getComplexityAnalyzerConfig", "summary": "Get complexity analyzer config", "description": "Returns the full complexity analyzer runtime config, including the semantic embedding configuration, the llm fallback classifier configuration, and per-tier reference phrase lists. Returns built-in defaults if none have been configured.", - "tags": ["Routing"], + "tags": [ + "Routing" + ], "security": [ { "ManagementBearerAuth": [] @@ -68659,6 +68661,56 @@ "required": [ "keywords" ], + "allOf": [ + { + "if": { + "properties": { + "session": { + "properties": { + "enabled": { + "const": true + } + }, + "required": [ + "enabled" + ] + } + }, + "required": [ + "session" + ] + }, + "then": { + "required": [ + "semantic" + ] + } + }, + { + "if": { + "properties": { + "semantic": { + "properties": { + "fallback": { + "const": "llm" + } + }, + "required": [ + "fallback" + ] + } + }, + "required": [ + "semantic" + ] + }, + "then": { + "required": [ + "llm" + ] + } + } + ], "properties": { "tier_boundaries": { "type": "object", @@ -68839,6 +68891,20 @@ "description": "Whether classification completion cost counts toward virtual-key budgets (record-only, never enforced; default false)" } } + }, + "session": { + "type": "object", + "description": "Session-aware complexity routing. When enabled and a request carries a supported session identity, Bifrost retains the highest tier observed across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins.", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable monotonic session tier retention. Requires semantic complexity classification; default is false." + } + } } } } @@ -68871,7 +68937,9 @@ "operationId": "updateComplexityAnalyzerConfig", "summary": "Update complexity analyzer config", "description": "Replaces the full complexity analyzer runtime config and hot-reloads the routing plugin. Changing the embedding configuration or reference phrases triggers a background re-warm of the classifier; unchanged phrases are not re-embedded. Setting semantic.fallback to llm requires the llm block to be present.", - "tags": ["Routing"], + "tags": [ + "Routing" + ], "requestBody": { "required": true, "content": { @@ -68883,6 +68951,56 @@ "required": [ "keywords" ], + "allOf": [ + { + "if": { + "properties": { + "session": { + "properties": { + "enabled": { + "const": true + } + }, + "required": [ + "enabled" + ] + } + }, + "required": [ + "session" + ] + }, + "then": { + "required": [ + "semantic" + ] + } + }, + { + "if": { + "properties": { + "semantic": { + "properties": { + "fallback": { + "const": "llm" + } + }, + "required": [ + "fallback" + ] + } + }, + "required": [ + "semantic" + ] + }, + "then": { + "required": [ + "llm" + ] + } + } + ], "properties": { "tier_boundaries": { "type": "object", @@ -69063,6 +69181,20 @@ "description": "Whether classification completion cost counts toward virtual-key budgets (record-only, never enforced; default false)" } } + }, + "session": { + "type": "object", + "description": "Session-aware complexity routing. When enabled and a request carries a supported session identity, Bifrost retains the highest tier observed across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins.", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable monotonic session tier retention. Requires semantic complexity classification; default is false." + } + } } } } @@ -69086,6 +69218,56 @@ "required": [ "keywords" ], + "allOf": [ + { + "if": { + "properties": { + "session": { + "properties": { + "enabled": { + "const": true + } + }, + "required": [ + "enabled" + ] + } + }, + "required": [ + "session" + ] + }, + "then": { + "required": [ + "semantic" + ] + } + }, + { + "if": { + "properties": { + "semantic": { + "properties": { + "fallback": { + "const": "llm" + } + }, + "required": [ + "fallback" + ] + } + }, + "required": [ + "semantic" + ] + }, + "then": { + "required": [ + "llm" + ] + } + } + ], "properties": { "tier_boundaries": { "type": "object", @@ -69266,6 +69448,20 @@ "description": "Whether classification completion cost counts toward virtual-key budgets (record-only, never enforced; default false)" } } + }, + "session": { + "type": "object", + "description": "Session-aware complexity routing. When enabled and a request carries a supported session identity, Bifrost retains the highest tier observed across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins.", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable monotonic session tier retention. Requires semantic complexity classification; default is false." + } + } } } } @@ -69310,7 +69506,9 @@ "operationId": "resetComplexityAnalyzerConfig", "summary": "Reset complexity analyzer config", "description": "Restores the built-in reference phrase lists and hot-reloads the routing plugin. The saved embedding provider, model, storage configuration, and llm fallback configuration are preserved.", - "tags": ["Routing"], + "tags": [ + "Routing" + ], "security": [ { "ManagementBearerAuth": [] @@ -69328,6 +69526,56 @@ "required": [ "keywords" ], + "allOf": [ + { + "if": { + "properties": { + "session": { + "properties": { + "enabled": { + "const": true + } + }, + "required": [ + "enabled" + ] + } + }, + "required": [ + "session" + ] + }, + "then": { + "required": [ + "semantic" + ] + } + }, + { + "if": { + "properties": { + "semantic": { + "properties": { + "fallback": { + "const": "llm" + } + }, + "required": [ + "fallback" + ] + } + }, + "required": [ + "semantic" + ] + }, + "then": { + "required": [ + "llm" + ] + } + } + ], "properties": { "tier_boundaries": { "type": "object", @@ -69508,6 +69756,109 @@ "description": "Whether classification completion cost counts toward virtual-key budgets (record-only, never enforced; default false)" } } + }, + "session": { + "type": "object", + "description": "Session-aware complexity routing. When enabled and a request carries a supported session identity, Bifrost retains the highest tier observed across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins.", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable monotonic session tier retention. Requires semantic complexity classification; default is false." + } + } + } + } + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + }, + "503": { + "description": "Config store not available", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BifrostError" + } + } + } + } + } + } + }, + "/api/routing/complexity-analyzer-status": { + "get": { + "operationId": "getComplexityAnalyzerStatus", + "summary": "Get complexity classifier status", + "description": "Returns the runtime status of the semantic complexity classifier (disabled, warming, ready, or failed), including warmup progress and whether a previous generation is still serving. When the llm fallback block is configured, also returns its readiness and the shipped default classification prompt.", + "tags": [ + "Routing" + ], + "responses": { + "200": { + "description": "Complexity classifier status retrieved successfully", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Runtime status of the semantic complexity classifier and, when configured, the LLM fallback classifier. Never contains phrases, embeddings, or provider secrets.", + "properties": { + "state": { + "type": "string", + "enum": [ + "disabled", + "warming", + "ready", + "failed" + ], + "description": "disabled = no embedding configuration; warming = reference phrases are being embedded; ready = serving the current configuration; failed = the desired configuration failed to warm" + }, + "loaded": { + "type": "integer", + "description": "Reference phrases embedded so far in the current warmup" + }, + "total": { + "type": "integer", + "description": "Total reference phrases to embed in the current warmup" + }, + "serving_previous": { + "type": "boolean", + "description": "When true with state=failed, the previous generation is still serving while the new one failed to warm" + }, + "error": { + "type": "string", + "description": "Warmup failure detail when state=failed" + }, + "llm": { + "type": "object", + "description": "Always present in the response. state is disabled when no llm block is configured.", + "properties": { + "state": { + "type": "string", + "enum": [ + "disabled", + "ready" + ], + "description": "disabled = no llm configuration; ready = the llm block is configured. The classifier has no warmup, so it is ready as soon as it is saved." + } + } + }, + "llm_default_prompt": { + "type": "string", + "description": "The shipped classification guidance, served so a configuration client can seed its prompt editor and offer a reset without holding a copy that drifts from the gateway's. Always present in the response, regardless of whether an llm block is configured." } } } @@ -69853,7 +70204,9 @@ "deprecated": true, "summary": "Get complexity analyzer config (deprecated path)", "description": "Returns the full complexity analyzer runtime config, including the semantic embedding configuration, the llm fallback classifier configuration, and per-tier reference phrase lists. Returns built-in defaults if none have been configured. Deprecated, use /api/routing/complexity-analyzer-config instead. This path is kept for backwards compatibility and behaves identically.", - "tags": ["Routing"], + "tags": [ + "Routing" + ], "security": [ { "ManagementBearerAuth": [] @@ -69871,6 +70224,56 @@ "required": [ "keywords" ], + "allOf": [ + { + "if": { + "properties": { + "session": { + "properties": { + "enabled": { + "const": true + } + }, + "required": [ + "enabled" + ] + } + }, + "required": [ + "session" + ] + }, + "then": { + "required": [ + "semantic" + ] + } + }, + { + "if": { + "properties": { + "semantic": { + "properties": { + "fallback": { + "const": "llm" + } + }, + "required": [ + "fallback" + ] + } + }, + "required": [ + "semantic" + ] + }, + "then": { + "required": [ + "llm" + ] + } + } + ], "properties": { "tier_boundaries": { "type": "object", @@ -70051,6 +70454,20 @@ "description": "Whether classification completion cost counts toward virtual-key budgets (record-only, never enforced; default false)" } } + }, + "session": { + "type": "object", + "description": "Session-aware complexity routing. When enabled and a request carries a supported session identity, Bifrost retains the highest tier observed across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins.", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable monotonic session tier retention. Requires semantic complexity classification; default is false." + } + } } } } @@ -70084,7 +70501,9 @@ "deprecated": true, "summary": "Update complexity analyzer config (deprecated path)", "description": "Replaces the full complexity analyzer runtime config and hot-reloads the routing plugin. Changing the embedding configuration or reference phrases triggers a background re-warm of the classifier; unchanged phrases are not re-embedded. Setting semantic.fallback to llm requires the llm block to be present. Deprecated, use /api/routing/complexity-analyzer-config instead. This path is kept for backwards compatibility and behaves identically.", - "tags": ["Routing"], + "tags": [ + "Routing" + ], "requestBody": { "required": true, "content": { @@ -70096,6 +70515,56 @@ "required": [ "keywords" ], + "allOf": [ + { + "if": { + "properties": { + "session": { + "properties": { + "enabled": { + "const": true + } + }, + "required": [ + "enabled" + ] + } + }, + "required": [ + "session" + ] + }, + "then": { + "required": [ + "semantic" + ] + } + }, + { + "if": { + "properties": { + "semantic": { + "properties": { + "fallback": { + "const": "llm" + } + }, + "required": [ + "fallback" + ] + } + }, + "required": [ + "semantic" + ] + }, + "then": { + "required": [ + "llm" + ] + } + } + ], "properties": { "tier_boundaries": { "type": "object", @@ -70276,6 +70745,20 @@ "description": "Whether classification completion cost counts toward virtual-key budgets (record-only, never enforced; default false)" } } + }, + "session": { + "type": "object", + "description": "Session-aware complexity routing. When enabled and a request carries a supported session identity, Bifrost retains the highest tier observed across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins.", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable monotonic session tier retention. Requires semantic complexity classification; default is false." + } + } } } } @@ -70299,6 +70782,56 @@ "required": [ "keywords" ], + "allOf": [ + { + "if": { + "properties": { + "session": { + "properties": { + "enabled": { + "const": true + } + }, + "required": [ + "enabled" + ] + } + }, + "required": [ + "session" + ] + }, + "then": { + "required": [ + "semantic" + ] + } + }, + { + "if": { + "properties": { + "semantic": { + "properties": { + "fallback": { + "const": "llm" + } + }, + "required": [ + "fallback" + ] + } + }, + "required": [ + "semantic" + ] + }, + "then": { + "required": [ + "llm" + ] + } + } + ], "properties": { "tier_boundaries": { "type": "object", @@ -70479,6 +71012,20 @@ "description": "Whether classification completion cost counts toward virtual-key budgets (record-only, never enforced; default false)" } } + }, + "session": { + "type": "object", + "description": "Session-aware complexity routing. When enabled and a request carries a supported session identity, Bifrost retains the highest tier observed across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins.", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable monotonic session tier retention. Requires semantic complexity classification; default is false." + } + } } } } @@ -70524,7 +71071,9 @@ "deprecated": true, "summary": "Reset complexity analyzer config (deprecated path)", "description": "Restores the built-in reference phrase lists and hot-reloads the routing plugin. The saved embedding provider, model, storage configuration, and llm fallback configuration are preserved. Deprecated, use /api/routing/complexity-analyzer-config/reset instead. This path is kept for backwards compatibility and behaves identically.", - "tags": ["Routing"], + "tags": [ + "Routing" + ], "security": [ { "ManagementBearerAuth": [] @@ -70542,6 +71091,56 @@ "required": [ "keywords" ], + "allOf": [ + { + "if": { + "properties": { + "session": { + "properties": { + "enabled": { + "const": true + } + }, + "required": [ + "enabled" + ] + } + }, + "required": [ + "session" + ] + }, + "then": { + "required": [ + "semantic" + ] + } + }, + { + "if": { + "properties": { + "semantic": { + "properties": { + "fallback": { + "const": "llm" + } + }, + "required": [ + "fallback" + ] + } + }, + "required": [ + "semantic" + ] + }, + "then": { + "required": [ + "llm" + ] + } + } + ], "properties": { "tier_boundaries": { "type": "object", @@ -70722,6 +71321,20 @@ "description": "Whether classification completion cost counts toward virtual-key budgets (record-only, never enforced; default false)" } } + }, + "session": { + "type": "object", + "description": "Session-aware complexity routing. When enabled and a request carries a supported session identity, Bifrost retains the highest tier observed across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins.", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable monotonic session tier retention. Requires semantic complexity classification; default is false." + } + } } } } @@ -70843,7 +71456,7 @@ { "name": "complexity_mechanisms", "in": "query", - "description": "Comma-separated list of complexity classification mechanisms to filter by (semantic or skipped)", + "description": "Comma-separated list of complexity decision mechanisms to filter by (semantic, llm, session, or skipped)", "schema": { "type": "string" } @@ -71455,7 +72068,7 @@ { "name": "complexity_mechanisms", "in": "query", - "description": "Comma-separated list of complexity classification mechanisms to filter by (semantic or skipped)", + "description": "Comma-separated list of complexity decision mechanisms to filter by (semantic, llm, session, or skipped)", "schema": { "type": "string" } @@ -82642,95 +83255,6 @@ } ] } - }, - "/api/routing/complexity-analyzer-status": { - "get": { - "operationId": "getComplexityAnalyzerStatus", - "summary": "Get complexity classifier status", - "description": "Returns the runtime status of the semantic complexity classifier (disabled, warming, ready, or failed), including warmup progress and whether a previous generation is still serving. When the llm fallback block is configured, also returns its readiness and the shipped default classification prompt.", - "tags": [ - "Routing" - ], - "responses": { - "200": { - "description": "Complexity classifier status retrieved successfully", - "content": { - "application/json": { - "schema": { - "type": "object", - "description": "Runtime status of the semantic complexity classifier and, when configured, the LLM fallback classifier. Never contains phrases, embeddings, or provider secrets.", - "properties": { - "state": { - "type": "string", - "enum": [ - "disabled", - "warming", - "ready", - "failed" - ], - "description": "disabled = no embedding configuration; warming = reference phrases are being embedded; ready = serving the current configuration; failed = the desired configuration failed to warm" - }, - "loaded": { - "type": "integer", - "description": "Reference phrases embedded so far in the current warmup" - }, - "total": { - "type": "integer", - "description": "Total reference phrases to embed in the current warmup" - }, - "serving_previous": { - "type": "boolean", - "description": "When true with state=failed, the previous generation is still serving while the new one failed to warm" - }, - "error": { - "type": "string", - "description": "Warmup failure detail when state=failed" - }, - "llm": { - "type": "object", - "description": "Always present in the response. state is disabled when no llm block is configured.", - "properties": { - "state": { - "type": "string", - "enum": [ - "disabled", - "ready" - ], - "description": "disabled = no llm configuration; ready = the llm block is configured. The classifier has no warmup, so it is ready as soon as it is saved." - } - } - }, - "llm_default_prompt": { - "type": "string", - "description": "The shipped classification guidance, served so a configuration client can seed its prompt editor and offer a reset without holding a copy that drifts from the gateway's. Always present in the response, regardless of whether an llm block is configured." - } - } - } - } - } - }, - "500": { - "description": "Internal server error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BifrostError" - } - } - } - }, - "503": { - "description": "Config store not available", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BifrostError" - } - } - } - } - } - } } }, "components": { @@ -101103,13 +101627,22 @@ "nullable": true }, "complexity_mechanism": { - "type": "string", - "description": "How the complexity tier was classified (semantic for the embedding-based classifier, or skipped when classification was demanded but produced no tier)", - "nullable": true + "type": [ + "string", + "null" + ], + "enum": [ + "semantic", + "llm", + "session", + "skipped", + null + ], + "description": "How the effective complexity tier was determined. session means retained session state supplied the tier; skipped means classification was demanded but produced no tier." }, "complexity_score": { "type": "number", - "description": "Similarity score of the nearest reference phrase behind the tier", + "description": "Similarity score of the nearest reference phrase behind the tier. Absent for llm, session, and skipped decisions.", "nullable": true }, "stream": { diff --git a/docs/openapi/paths/management/logging.yaml b/docs/openapi/paths/management/logging.yaml index 2c1a0a5e930..4ec22ced5c4 100644 --- a/docs/openapi/paths/management/logging.yaml +++ b/docs/openapi/paths/management/logging.yaml @@ -55,7 +55,7 @@ logs: type: string - name: complexity_mechanisms in: query - description: Comma-separated list of complexity classification mechanisms to filter by (semantic or skipped) + description: Comma-separated list of complexity decision mechanisms to filter by (semantic, llm, session, or skipped) schema: type: string - name: start_time @@ -238,7 +238,7 @@ logs-stats: type: string - name: complexity_mechanisms in: query - description: Comma-separated list of complexity classification mechanisms to filter by (semantic or skipped) + description: Comma-separated list of complexity decision mechanisms to filter by (semantic, llm, session, or skipped) schema: type: string - name: start_time @@ -835,7 +835,7 @@ _histogram-parameters: complexity_mechanisms: name: complexity_mechanisms in: query - description: Comma-separated list of complexity classification mechanisms to filter by (semantic or skipped) + description: Comma-separated list of complexity decision mechanisms to filter by (semantic, llm, session, or skipped) schema: type: string start_time: @@ -1874,4 +1874,4 @@ mcp-logs-histogram-top-tools: "400": $ref: "../../openapi.yaml#/components/responses/BadRequest" "500": - $ref: "../../openapi.yaml#/components/responses/InternalError" \ No newline at end of file + $ref: "../../openapi.yaml#/components/responses/InternalError" diff --git a/docs/openapi/schemas/management/governance.yaml b/docs/openapi/schemas/management/governance.yaml index 58051c78f55..98c1339463e 100644 --- a/docs/openapi/schemas/management/governance.yaml +++ b/docs/openapi/schemas/management/governance.yaml @@ -2404,6 +2404,33 @@ ComplexityAnalyzerConfig: additionalProperties: false required: - keywords + allOf: + - if: + properties: + session: + properties: + enabled: + const: true + required: + - enabled + required: + - session + then: + required: + - semantic + - if: + properties: + semantic: + properties: + fallback: + const: llm + required: + - fallback + required: + - semantic + then: + required: + - llm properties: tier_boundaries: $ref: '#/ComplexityTierBoundaries' @@ -2413,3 +2440,16 @@ ComplexityAnalyzerConfig: $ref: '#/ComplexitySemanticConfig' llm: $ref: '#/ComplexityLLMConfig' + session: + $ref: '#/ComplexitySessionConfig' + +ComplexitySessionConfig: + type: object + description: Session-aware complexity routing. When enabled and a request carries a supported session identity, Bifrost retains the highest tier observed across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins. + additionalProperties: false + required: + - enabled + properties: + enabled: + type: boolean + description: Enable monotonic session tier retention. Requires semantic complexity classification; default is false. diff --git a/docs/openapi/schemas/management/logging.yaml b/docs/openapi/schemas/management/logging.yaml index a1278a091ea..457e25b8899 100644 --- a/docs/openapi/schemas/management/logging.yaml +++ b/docs/openapi/schemas/management/logging.yaml @@ -55,12 +55,12 @@ LogEntry: description: Complexity tier used for routing; null when no routing rule referenced complexity_tier. REASONING is historical-only — it was merged into COMPLEX and is never emitted for new requests, but survives in log rows recorded under the old scheme. nullable: true complexity_mechanism: - type: string - description: How the complexity tier was classified (semantic for the embedding-based classifier, or skipped when classification was demanded but produced no tier) - nullable: true + type: [string, 'null'] + enum: [semantic, llm, session, skipped, null] + description: How the effective complexity tier was determined. session means retained session state supplied the tier; skipped means classification was demanded but produced no tier. complexity_score: type: number - description: Similarity score of the nearest reference phrase behind the tier + description: Similarity score of the nearest reference phrase behind the tier. Absent for llm, session, and skipped decisions. nullable: true stream: type: boolean diff --git a/framework/configstore/clientconfig.go b/framework/configstore/clientconfig.go index fde58ccbb1c..1cfa5f2a17e 100644 --- a/framework/configstore/clientconfig.go +++ b/framework/configstore/clientconfig.go @@ -1364,6 +1364,14 @@ func GenerateComplexityAnalyzerConfigHashes(config *ComplexityAnalyzerConfig) (C hashes.LLMSettings = settingsHash } + if normalized.Session != nil { + settingsHash, err := hashComplexityValue(normalized.Session) + if err != nil { + return ComplexityAnalyzerConfigHashes{}, fmt.Errorf("failed to hash session settings: %w", err) + } + hashes.SessionSettings = settingsHash + } + return hashes, nil } diff --git a/framework/configstore/complexityconfig.go b/framework/configstore/complexityconfig.go index e79785e364d..1096e8de064 100644 --- a/framework/configstore/complexityconfig.go +++ b/framework/configstore/complexityconfig.go @@ -216,11 +216,6 @@ type ComplexitySemanticConfig struct { // level because the fallback is meaningless without a primary — the LLM // classifier only ever runs after a semantic non-answer. // - // Session note: an LLM-classified turn carries no similarity score, so - // with a positive session switch_min_similarity it can never move a - // session tier (except through always_allow_escalation). That is - // deliberate: the LLM speaks exactly on the turns semantic was least - // confident about, which are the wrong turns to let re-pin a session. Fallback string `json:"fallback,omitempty"` } @@ -555,6 +550,51 @@ func (c *ComplexityLLMConfig) Validate() error { return nil } +// ComplexitySessionConfig controls monotonic complexity-tier retention across +// requests belonging to the same session. +// +// When enabled, Bifrost retains the highest tier reached across normally +// sequential session turns during the built-in inactivity window. Overlapping +// requests for the same session are best-effort because the runtime KV contract +// exposes separate reads and writes. The window is a routing-state retention +// policy and is intentionally independent of provider prompt-cache TTLs. +type ComplexitySessionConfig struct { + Enabled bool `json:"enabled"` +} + +// UnmarshalJSON rejects unknown fields so misspelled session settings cannot be +// accepted while silently leaving session routing disabled. +func (c *ComplexitySessionConfig) UnmarshalJSON(data []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + for field := range fields { + if field != "enabled" { + return fmt.Errorf("unknown complexity session field %q", field) + } + } + + var decoded struct { + Enabled *bool `json:"enabled"` + } + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + if decoded.Enabled == nil { + return fmt.Errorf("complexity session config requires enabled") + } + c.Enabled = *decoded.Enabled + return nil +} + +func (c *ComplexitySessionConfig) normalized() *ComplexitySessionConfig { + if c == nil { + return nil + } + return &ComplexitySessionConfig{Enabled: c.Enabled} +} + // ComplexityAnalyzerConfigHashes tracks the config.json hash for each editable // analyzer section. It is persisted with the config row, but not exposed through // API responses or config.json. @@ -570,7 +610,8 @@ type ComplexityAnalyzerConfigHashes struct { // LLMSettings covers the llm block (provider, model, timeout, prompt, // history window, budgets flag). The fallback selector rides the // SemanticSettings hash: it is a field of the semantic block. - LLMSettings string `json:"llm_settings,omitempty"` + LLMSettings string `json:"llm_settings,omitempty"` + SessionSettings string `json:"session_settings,omitempty"` } type legacyComplexityAnalyzerConfigHashes struct { @@ -640,7 +681,10 @@ type ComplexityAnalyzerConfig struct { // when Semantic.Fallback selects "llm". It may be present while the // fallback says "none": the block is retained so toggling the fallback // never loses settings. - LLM *ComplexityLLMConfig `json:"llm,omitempty"` + LLM *ComplexityLLMConfig `json:"llm,omitempty"` + // Session enables the built-in monotonic session tier. It is separate from + // semantic message_history_count: no message or turn history is persisted. + Session *ComplexitySessionConfig `json:"session,omitempty"` ConfigHashes ComplexityAnalyzerConfigHashes `json:"-"` // EmbeddingFingerprint is reserved for config-store implementations that // persist routing state. The semantic classifier verifies a VectorStore-side @@ -695,6 +739,7 @@ type complexitySemanticConfigRecord struct { Keywords ComplexityEditableKeywordConfig `json:"keywords"` Semantic *ComplexitySemanticConfig `json:"semantic,omitempty"` LLM *ComplexityLLMConfig `json:"llm,omitempty"` + Session *ComplexitySessionConfig `json:"session,omitempty"` ConfigHashes complexitySemanticRowHashes `json:"_config_hashes,omitempty"` EmbeddingFingerprint string `json:"_embedding_fingerprint,omitempty"` } @@ -707,6 +752,7 @@ type complexitySemanticRowHashes struct { ComplexKeywords string `json:"complex_keywords,omitempty"` SemanticSettings string `json:"semantic_settings,omitempty"` LLMSettings string `json:"llm_settings,omitempty"` + SessionSettings string `json:"session_settings,omitempty"` } // LLMFallbackEnabled reports whether a semantic non-answer should be retried @@ -719,6 +765,11 @@ func (c *ComplexityAnalyzerConfig) LLMFallbackEnabled() bool { c.LLM != nil } +// SessionRoutingEnabled reports whether monotonic session-tier retention is enabled. +func (c *ComplexityAnalyzerConfig) SessionRoutingEnabled() bool { + return c != nil && c.Session != nil && c.Session.Enabled +} + // Validate checks that the config is internally consistent. func (c *ComplexityAnalyzerConfig) Validate() error { if c == nil { @@ -755,6 +806,9 @@ func (c *ComplexityAnalyzerConfig) Validate() error { if c.Semantic != nil && c.Semantic.Fallback == ComplexitySemanticFallbackLLM && c.LLM == nil { return fmt.Errorf("semantic fallback %q requires an llm config block", ComplexitySemanticFallbackLLM) } + if c.SessionRoutingEnabled() && c.Semantic == nil { + return fmt.Errorf("complexity session routing requires a semantic config block") + } return nil } @@ -776,6 +830,7 @@ func (c *ComplexityAnalyzerConfig) Normalized() ComplexityAnalyzerConfig { }, Semantic: c.Semantic.normalized(), LLM: c.LLM.normalized(), + Session: c.Session.normalized(), ConfigHashes: c.ConfigHashes, EmbeddingFingerprint: c.EmbeddingFingerprint, } @@ -854,6 +909,7 @@ func MergeComplexityAnalyzerConfig(base, file *ComplexityAnalyzerConfig) (*Compl }, Semantic: mergeComplexitySemanticConfig(normalizedBase.Semantic, normalizedFile.Semantic), LLM: mergeComplexityLLMConfig(normalizedBase.LLM, normalizedFile.LLM), + Session: mergeComplexitySessionConfig(normalizedBase.Session, normalizedFile.Session), ConfigHashes: normalizedFile.ConfigHashes, EmbeddingFingerprint: normalizedBase.EmbeddingFingerprint, } @@ -882,6 +938,15 @@ func mergeComplexityLLMConfig(base, file *ComplexityLLMConfig) *ComplexityLLMCon return file.normalized() } +// mergeComplexitySessionConfig overlays file session settings. A nil file +// section keeps the persisted setting untouched. +func mergeComplexitySessionConfig(base, file *ComplexitySessionConfig) *ComplexitySessionConfig { + if file == nil { + return base.normalized() + } + return file.normalized() +} + // MergeComplexityAnalyzerConfigByHashes overlays only file-backed sections whose // config.json hash changed. Keyword sections are additive; tier boundaries replace. func MergeComplexityAnalyzerConfigByHashes(base, file *ComplexityAnalyzerConfig) (*ComplexityAnalyzerConfig, error) { @@ -936,6 +1001,14 @@ func MergeComplexityAnalyzerConfigByHashes(base, file *ComplexityAnalyzerConfig) merged.ConfigHashes.LLMSettings = normalizedFile.ConfigHashes.LLMSettings } } + // Session follows the same optional-section rule: omission means no file + // opinion, while an explicit enabled=false is a real override. + if normalizedFile.Session != nil { + if merged.Session == nil || merged.ConfigHashes.SessionSettings != normalizedFile.ConfigHashes.SessionSettings { + merged.Session = normalizedFile.Session.normalized() + merged.ConfigHashes.SessionSettings = normalizedFile.ConfigHashes.SessionSettings + } + } normalizedMerged := merged.Normalized() if err := normalizedMerged.Validate(); err != nil { return nil, err @@ -1019,12 +1092,14 @@ func encodeComplexitySemanticConfigRow(config ComplexityAnalyzerConfig) ([]byte, Keywords: config.Keywords, Semantic: config.Semantic, LLM: config.LLM, + Session: config.Session, ConfigHashes: complexitySemanticRowHashes{ SimpleKeywords: config.ConfigHashes.SimpleKeywords, MediumKeywords: config.ConfigHashes.MediumKeywords, ComplexKeywords: config.ConfigHashes.ComplexKeywords, SemanticSettings: config.ConfigHashes.SemanticSettings, LLMSettings: config.ConfigHashes.LLMSettings, + SessionSettings: config.ConfigHashes.SessionSettings, }, EmbeddingFingerprint: config.EmbeddingFingerprint, } @@ -1045,11 +1120,13 @@ func applyComplexitySemanticConfigRow(base *ComplexityAnalyzerConfig, row *compl combined.Keywords = row.Keywords combined.Semantic = row.Semantic combined.LLM = row.LLM + combined.Session = row.Session combined.ConfigHashes.SimpleKeywords = row.ConfigHashes.SimpleKeywords combined.ConfigHashes.MediumKeywords = row.ConfigHashes.MediumKeywords combined.ConfigHashes.ComplexKeywords = row.ConfigHashes.ComplexKeywords combined.ConfigHashes.SemanticSettings = row.ConfigHashes.SemanticSettings combined.ConfigHashes.LLMSettings = row.ConfigHashes.LLMSettings + combined.ConfigHashes.SessionSettings = row.ConfigHashes.SessionSettings combined.EmbeddingFingerprint = row.EmbeddingFingerprint return &combined } diff --git a/framework/configstore/complexityconfig_test.go b/framework/configstore/complexityconfig_test.go index 1aeefad6468..a2af5033dd2 100644 --- a/framework/configstore/complexityconfig_test.go +++ b/framework/configstore/complexityconfig_test.go @@ -29,6 +29,35 @@ func testSemanticAnalyzerConfig() *ComplexityAnalyzerConfig { return cfg } +func testSessionAnalyzerConfig() *ComplexityAnalyzerConfig { + cfg := testSemanticAnalyzerConfig() + cfg.Session = &ComplexitySessionConfig{Enabled: true} + return cfg +} + +func TestComplexitySessionConfigDecoding(t *testing.T) { + var cfg ComplexitySessionConfig + require.NoError(t, json.Unmarshal([]byte(`{"enabled":true}`), &cfg)) + assert.True(t, cfg.Enabled) + + err := json.Unmarshal([]byte(`{"enable":true}`), &cfg) + require.ErrorContains(t, err, `unknown complexity session field "enable"`) + + err = json.Unmarshal([]byte(`{}`), &cfg) + require.ErrorContains(t, err, "requires enabled") +} + +func TestComplexitySessionConfigRequiresSemanticWhenEnabled(t *testing.T) { + cfg := testComplexityAnalyzerConfig() + cfg.Session = &ComplexitySessionConfig{Enabled: true} + normalized := cfg.Normalized() + require.ErrorContains(t, normalized.Validate(), "requires a semantic config block") + + cfg.Session.Enabled = false + normalized = cfg.Normalized() + require.NoError(t, normalized.Validate()) +} + func TestComplexitySemanticConfigTimeoutDecoding(t *testing.T) { tests := []struct { name string @@ -244,12 +273,14 @@ func TestComplexityAnalyzerConfigSemanticPhraseValidation(t *testing.T) { func TestDecodeComplexityAnalyzerConfigSemanticRoundTrip(t *testing.T) { cfg := testSemanticAnalyzerConfig() + cfg.Session = &ComplexitySessionConfig{Enabled: true} cfg.ConfigHashes = ComplexityAnalyzerConfigHashes{ TierBoundaries: "tier-hash", SimpleKeywords: "simple-hash", MediumKeywords: "medium-hash", ComplexKeywords: "complex-hash", SemanticSettings: "settings-hash", + SessionSettings: "session-hash", } cfg.EmbeddingFingerprint = "fingerprint-1" @@ -262,13 +293,16 @@ func TestDecodeComplexityAnalyzerConfigSemanticRoundTrip(t *testing.T) { // row. Anything of it that leaks into the analyzer row is something an older // Bifrost would silently drop the next time it saved. assert.Contains(t, string(semanticRaw), `"_embedding_fingerprint":"fingerprint-1"`) + assert.Contains(t, string(semanticRaw), `"session":{"enabled":true}`) assert.NotContains(t, string(analyzerRaw), "_embedding_fingerprint") assert.NotContains(t, string(analyzerRaw), "semantic") + assert.NotContains(t, string(analyzerRaw), "session") decoded, err := roundTripComplexityAnalyzerConfig(t, cfg.Normalized()) require.NoError(t, err) require.NotNil(t, decoded.Semantic) assert.Equal(t, cfg.Normalized().Semantic, decoded.Semantic) + assert.Equal(t, cfg.Session, decoded.Session) assert.Equal(t, cfg.ConfigHashes, decoded.ConfigHashes) assert.Equal(t, "fingerprint-1", decoded.EmbeddingFingerprint) } @@ -312,6 +346,57 @@ func TestGenerateComplexityAnalyzerConfigHashesSemantic(t *testing.T) { assert.Empty(t, plainHashes.SemanticSettings) } +func TestGenerateComplexityAnalyzerConfigHashesSession(t *testing.T) { + enabled := testSessionAnalyzerConfig() + enabledHashes, err := GenerateComplexityAnalyzerConfigHashes(enabled) + require.NoError(t, err) + require.NotEmpty(t, enabledHashes.SessionSettings) + + disabled := testSessionAnalyzerConfig() + disabled.Session.Enabled = false + disabledHashes, err := GenerateComplexityAnalyzerConfigHashes(disabled) + require.NoError(t, err) + assert.NotEqual(t, enabledHashes.SessionSettings, disabledHashes.SessionSettings) + assert.Equal(t, enabledHashes.SemanticSettings, disabledHashes.SemanticSettings) + + withoutSession, err := GenerateComplexityAnalyzerConfigHashes(testSemanticAnalyzerConfig()) + require.NoError(t, err) + assert.Empty(t, withoutSession.SessionSettings) +} + +func TestMergeComplexityAnalyzerConfigByHashesSession(t *testing.T) { + withHashes := func(cfg *ComplexityAnalyzerConfig) *ComplexityAnalyzerConfig { + hashes, err := GenerateComplexityAnalyzerConfigHashes(cfg) + require.NoError(t, err) + cfg.ConfigHashes = hashes + return cfg + } + + t.Run("file omission preserves persisted session setting", func(t *testing.T) { + base := withHashes(testSessionAnalyzerConfig()) + file := withHashes(testSemanticAnalyzerConfig()) + + merged, err := MergeComplexityAnalyzerConfigByHashes(base, file) + require.NoError(t, err) + require.NotNil(t, merged.Session) + assert.True(t, merged.Session.Enabled) + assert.Equal(t, base.ConfigHashes.SessionSettings, merged.ConfigHashes.SessionSettings) + }) + + t.Run("explicit false overrides enabled", func(t *testing.T) { + base := withHashes(testSessionAnalyzerConfig()) + file := testSessionAnalyzerConfig() + file.Session.Enabled = false + withHashes(file) + + merged, err := MergeComplexityAnalyzerConfigByHashes(base, file) + require.NoError(t, err) + require.NotNil(t, merged.Session) + assert.False(t, merged.Session.Enabled) + assert.Equal(t, file.ConfigHashes.SessionSettings, merged.ConfigHashes.SessionSettings) + }) +} + func TestMergeComplexityAnalyzerConfigByHashesSemantic(t *testing.T) { fileConfig := func() *ComplexityAnalyzerConfig { cfg := testSemanticAnalyzerConfig() @@ -401,6 +486,36 @@ func TestRDBConfigStore_ComplexityAnalyzerConfigSemanticPersistence(t *testing.T assert.Equal(t, "fingerprint-1", got.EmbeddingFingerprint) } +func TestRDBConfigStore_ComplexitySessionPersistenceAndReset(t *testing.T) { + store := setupRDBTestStore(t) + ctx := context.Background() + + cfg := testSessionAnalyzerConfig() + hashes, err := GenerateComplexityAnalyzerConfigHashes(cfg) + require.NoError(t, err) + cfg.ConfigHashes = hashes + require.NoError(t, store.UpdateComplexityAnalyzerConfig(ctx, cfg)) + + got, err := store.GetComplexityAnalyzerConfig(ctx) + require.NoError(t, err) + require.NotNil(t, got.Session) + assert.True(t, got.Session.Enabled) + assert.Equal(t, hashes.SessionSettings, got.ConfigHashes.SessionSettings) + + // UI payloads omit internal hashes. The split-row carry-over path must keep + // the session hash beside the session setting in the semantic row. + update := testSessionAnalyzerConfig() + require.NoError(t, store.UpdateComplexityAnalyzerConfig(ctx, update)) + got, err = store.GetComplexityAnalyzerConfig(ctx) + require.NoError(t, err) + assert.Equal(t, hashes.SessionSettings, got.ConfigHashes.SessionSettings) + + restored, err := store.ResetComplexityAnalyzerConfig(ctx, testComplexityAnalyzerConfig()) + require.NoError(t, err) + require.NotNil(t, restored.Session) + assert.True(t, restored.Session.Enabled) +} + // A writer that carries ConfigHashes/EmbeddingFingerprint over from the stored row must not // clobber a concurrent writer that is setting fresh ones. The carry-over read and the save // have to be one atomic unit; if they are not, the carrying writer can read the pre-update diff --git a/framework/configstore/migrations.go b/framework/configstore/migrations.go index 6483ab14da2..b77de8e9c41 100644 --- a/framework/configstore/migrations.go +++ b/framework/configstore/migrations.go @@ -468,7 +468,7 @@ var configstoreMigrationSteps = []migrationStep{ {IDs: []string{"add_needs_session_stickiness_column"}, run: migrationAddNeedsSessionStickinessColumn}, {IDs: []string{"add_bedrock_endpoints_columns"}, run: migrationAddBedrockEndpointsColumns}, {IDs: []string{"add_cost_per_request_pricing_column"}, run: migrationAddCostPerRequestPricingColumn}, - {IDs: []string{backfillDefaultComplexityExemplarsMigrationID}, run: migrationBackfillDefaultComplexityExemplars}, + {IDs: []string{"backfill_default_complexity_exemplars_v2"}, run: migrationBackfillDefaultComplexityExemplars}, {IDs: []string{"add_notifications_table"}, run: migrationAddNotificationsTable}, {IDs: []string{"add_batch_jobs_table"}, run: migrationAddBatchJobsTable}, {IDs: []string{"add_image_megapixel_tier_pricing_columns"}, run: migrationAddImageMegapixelTierPricingColumns}, @@ -12302,18 +12302,6 @@ func readComplexityConfigRow(tx *gorm.DB, key string) (string, error) { return strings.TrimSpace(entry.Value), nil } -// backfillDefaultComplexityExemplarsMigrationID is shared by the migration and -// its registry entry on purpose. configstoreMigrationSteps is what the pending -// check reads, and triggerMigrations skips the entire run when nothing is -// pending — so a registry entry naming a different ID than the migration writes -// does not just mislabel the step, it can stop every migration from running. -// -// The ID is versioned because this migration was retargeted after the lexical -// and semantic configs moved into separate rows. Installations that ran the -// original never wrote a semantic row, and the migrator skips by ID, so reusing -// the old one would leave them with no exemplars at all. -const backfillDefaultComplexityExemplarsMigrationID = "backfill_default_complexity_exemplars_v2" - // preSplitComplexityAnalyzerRow is an analyzer row written before the lexical // and semantic configs moved into separate rows. type preSplitComplexityAnalyzerRow struct { @@ -12391,7 +12379,7 @@ func complexityConfigFromPreSplitAnalyzerRow(data []byte) (preSplitComplexityAna // exemplars to persisted complexity configurations created before those // defaults existed. Existing phrases and tier assignments always win. func migrationBackfillDefaultComplexityExemplars(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { - migrationName := backfillDefaultComplexityExemplarsMigrationID + migrationName := "backfill_default_complexity_exemplars_v2" logger.Info("[configstore] starting migration %s", migrationName) defer logger.Info("[configstore] finished migration %s", migrationName) diff --git a/framework/configstore/rdb.go b/framework/configstore/rdb.go index cee6506113f..df869b504b1 100644 --- a/framework/configstore/rdb.go +++ b/framework/configstore/rdb.go @@ -6448,6 +6448,8 @@ func (s *RDBConfigStore) readComplexityCarryOverWithDB(ctx context.Context, db * hashes.MediumKeywords = semanticRow.ConfigHashes.MediumKeywords hashes.ComplexKeywords = semanticRow.ConfigHashes.ComplexKeywords hashes.SemanticSettings = semanticRow.ConfigHashes.SemanticSettings + hashes.LLMSettings = semanticRow.ConfigHashes.LLMSettings + hashes.SessionSettings = semanticRow.ConfigHashes.SessionSettings return hashes, semanticRow.EmbeddingFingerprint, nil } diff --git a/framework/logstore/tables.go b/framework/logstore/tables.go index 65413769feb..a3362a061d8 100644 --- a/framework/logstore/tables.go +++ b/framework/logstore/tables.go @@ -59,7 +59,7 @@ type SearchFilters struct { VirtualKeyIDs []string `json:"virtual_key_ids,omitempty"` RoutingRuleIDs []string `json:"routing_rule_ids,omitempty"` ComplexityTiers []string `json:"complexity_tiers,omitempty"` // For filtering by routing complexity tier (SIMPLE, MEDIUM, COMPLEX) - ComplexityMechanisms []string `json:"complexity_mechanisms,omitempty"` // For filtering by complexity classification mechanism (lexical, skipped) + ComplexityMechanisms []string `json:"complexity_mechanisms,omitempty"` // For filtering by complexity decision mechanism (semantic, llm, session, skipped) TeamIDs []string `json:"team_ids,omitempty"` CustomerIDs []string `json:"customer_ids,omitempty"` UserIDs []string `json:"user_ids,omitempty"` diff --git a/helm-charts/bifrost/README.md b/helm-charts/bifrost/README.md index 392e7fef69f..8a497068daf 100644 --- a/helm-charts/bifrost/README.md +++ b/helm-charts/bifrost/README.md @@ -22,6 +22,7 @@ Official Helm charts for deploying [Bifrost](https://github.com/maximhq/bifrost) - Added `storage.logsStore.postgres` to point the logs store at a **separate external PostgreSQL** (different host and/or database) than the config store, instead of forcing both onto the shared top-level `postgresql` connection. Only applies when the logs store resolves to postgres; `enabled: false` (default) preserves existing behavior. Fields mirror `postgresql.external` (`host`, `port`, `user`, `password`, `passwordCommand`, `database`, `sslMode`, `connMaxLifetime`, `existingSecret`, `passwordKey`); with `existingSecret` the password is injected as `BIFROST_LOGS_POSTGRES_PASSWORD`. Renders into `logs_store.config`. - Updated `bifrost.governance.complexityAnalyzerConfig` for semantic Complexity Router configuration: set an embedding provider and model, add reference phrases for Simple, Medium, and Complex, and choose `embedded` or `vector_store` phrase storage. Bifrost detects the embedding dimension during warmup. Legacy four-tier lists remain valid: Simple stays Simple, Code and Technical merge into Medium, and Reasoning merges into Complex. Legacy `tier_boundaries` remain accepted during upgrades but are optional and ignored by semantic routing. Renders into `governance.complexity_analyzer_config`. - Added `vectorStore.type: chromem` plus a `vectorStore.chromem` block (`path`, `compress`) for the embedded in-process vector store used by semantic complexity routing. Renders into `vector_store.config`. +- Added `bifrost.governance.complexityAnalyzerConfig.session.enabled` for session-aware Complexity Router behavior. Identified sessions retain their highest observed tier across normally sequential turns for 24 hours of inactivity; overlapping requests for the same session are best-effort and resolve by last writer wins. Renders into `governance.complexity_analyzer_config.session.enabled`. ### 2.1.36 diff --git a/helm-charts/bifrost/values.schema.json b/helm-charts/bifrost/values.schema.json index 135936ba947..c6e3020ed2a 100644 --- a/helm-charts/bifrost/values.schema.json +++ b/helm-charts/bifrost/values.schema.json @@ -2186,8 +2186,20 @@ }, "complexityAnalyzerConfig": { "type": ["object", "null"], - "description": "Runtime configuration for complexity_tier CEL routing. Renders into governance.complexity_analyzer_config in config.json. Semantic (embedding-based) classification is the primary mechanism — without the semantic block no complexity_tier is published. The optional llm block is a chat-completion fallback engaged only when semantic.fallback is 'llm'.", + "description": "Runtime configuration for complexity_tier CEL routing. Renders into governance.complexity_analyzer_config in config.json. Semantic (embedding-based) classification is the primary mechanism — without the semantic block no complexity_tier is published. The optional llm block is a chat-completion fallback engaged only when semantic.fallback is 'llm'. The optional session block retains the highest tier observed for a session.", "properties": { + "session": { + "type": "object", + "description": "Session-aware complexity routing. Retains the highest observed tier across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable monotonic session tier retention. Requires semantic complexity classification; default: false." + } + }, + "required": ["enabled"], + "additionalProperties": false + }, "semantic": { "type": "object", "description": "Embedding-based (semantic) complexity classification settings. Presence of this block enables the classifier; it embeds the per-tier keyword lists as reference phrases.", @@ -2401,6 +2413,44 @@ ] } }, + "allOf": [ + { + "if": { + "properties": { + "session": { + "properties": { + "enabled": { + "const": true + } + }, + "required": ["enabled"] + } + }, + "required": ["session"] + }, + "then": { + "required": ["semantic"] + } + }, + { + "if": { + "properties": { + "semantic": { + "properties": { + "fallback": { + "const": "llm" + } + }, + "required": ["fallback"] + } + }, + "required": ["semantic"] + }, + "then": { + "required": ["llm"] + } + } + ], "required": ["keywords"], "additionalProperties": false } diff --git a/helm-charts/bifrost/values.yaml b/helm-charts/bifrost/values.yaml index afc2269a3f1..f26593a6854 100644 --- a/helm-charts/bifrost/values.yaml +++ b/helm-charts/bifrost/values.yaml @@ -934,6 +934,8 @@ bifrost: # prompt: "" # Replaces the shipped classification guidance (max 4000 chars); empty uses the shipped guidance # message_history_count: 1 # Recent user messages sent to the classifier, oldest first (1 = latest only) # count_toward_budgets: false # Count classification completion cost toward virtual-key budgets (record-only, never enforced) + # session: # Optional session-aware routing; requires semantic configuration above + # enabled: true # Retain the highest tier across sequential turns for 24 hours; overlapping requests are best-effort # keywords: # Required when complexityAnalyzerConfig is non-null; in split mode these phrases merge with stored defaults # simple_keywords: ["what is a mutex?", "fix the grammar in this sentence."] # medium_keywords: ["add api-key auth: hash the keys, reject revoked ones, and never log them."] diff --git a/plugins/logging/operations_test.go b/plugins/logging/operations_test.go index c27c6e3e351..b356d7873d5 100644 --- a/plugins/logging/operations_test.go +++ b/plugins/logging/operations_test.go @@ -859,7 +859,7 @@ func TestPostLLMHookCapturesComplexityRoutingContext(t *testing.T) { // Set by the governance plugin when a routing rule references complexity_tier. ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityTier, "COMPLEX") - ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityMechanism, "lexical") + ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityMechanism, "semantic") ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityScore, 0.42) statusCode := 500 @@ -888,8 +888,8 @@ func TestPostLLMHookCapturesComplexityRoutingContext(t *testing.T) { if entry.ComplexityTier == nil || *entry.ComplexityTier != "COMPLEX" { t.Fatalf("expected complexity_tier COMPLEX, got %v", entry.ComplexityTier) } - if entry.ComplexityMechanism == nil || *entry.ComplexityMechanism != "lexical" { - t.Fatalf("expected complexity_mechanism lexical, got %v", entry.ComplexityMechanism) + if entry.ComplexityMechanism == nil || *entry.ComplexityMechanism != "semantic" { + t.Fatalf("expected complexity_mechanism semantic, got %v", entry.ComplexityMechanism) } if entry.ComplexityScore == nil || *entry.ComplexityScore != 0.42 { t.Fatalf("expected complexity_score 0.42, got %v", entry.ComplexityScore) diff --git a/plugins/routing/complexity/config.go b/plugins/routing/complexity/config.go index 55d7d93934f..c8d3d4bb35c 100644 --- a/plugins/routing/complexity/config.go +++ b/plugins/routing/complexity/config.go @@ -38,8 +38,7 @@ const ( // tier. They surface in request logs (complexity_mechanism column) so admins can // see how each routing decision was classified. "skipped" means classification // was demanded but produced no tier (unsupported input, no signal, or the -// analyzer is disabled). Future classifiers add their own values here -// (e.g. "llm"). +// analyzer is disabled). // // "lexical" is not here: the keyword scorer no longer publishes a tier, so // nothing writes that value. It never reached a log either — the @@ -49,10 +48,13 @@ const ( MechanismSemantic = "semantic" // MechanismLLM means the chat-completion classifier published the tier. MechanismLLM = "llm" + // MechanismSession means a previously established session tier determined + // the effective tier for this turn. The current classifier either produced + // no tier, proposed a lower tier, or was intentionally skipped at COMPLEX. + MechanismSession = "session" // MechanismSkipped means classification was demanded by a routing rule but // produced no tier. MechanismSkipped = "skipped" - ) // Default boundaries are retained for the dormant lexical analyzer and its @@ -75,6 +77,9 @@ type SemanticConfig = configstore.ComplexitySemanticConfig // LLMConfig is the chat-completion classifier configuration. type LLMConfig = configstore.ComplexityLLMConfig +// SessionConfig controls monotonic complexity-tier retention across requests. +type SessionConfig = configstore.ComplexitySessionConfig + // AnalyzerConfig is the runtime configuration for the complexity analyzer. type AnalyzerConfig = configstore.ComplexityAnalyzerConfig diff --git a/plugins/routing/complexity/extract.go b/plugins/routing/complexity/extract.go index 12693df8273..fde99241749 100644 --- a/plugins/routing/complexity/extract.go +++ b/plugins/routing/complexity/extract.go @@ -67,45 +67,152 @@ var ( } ) -const codexTurnMetadataHeader = "x-codex-turn-metadata" +const ( + codexTurnMetadataHeader = "x-codex-turn-metadata" + claudeCodeSessionIDHeader = "x-claude-code-session-id" + maxComplexitySessionIDLength = 255 + claudeSessionEnvelopeOpen = "" + claudeResumeRecapPrefix = "The user stepped away and is coming back." +) type codexTurnMetadata struct { RequestKind string + SessionID string } -// buildComplexityInput extracts text from normalized BifrostRequest values for -// complexity_tier routing. It intentionally runs after the transport converters -// have produced Bifrost's typed request shape, so governance does not duplicate -// provider-specific raw payload parsing. +// InputDisposition describes how one request participates in session-aware +// complexity routing. +type InputDisposition uint8 + +const ( + // InputBypass means the operation is unsupported or explicitly belongs to a + // harness background workload. It neither classifies nor refreshes a session. + InputBypass InputDisposition = iota + // InputContinuation means a supported conversational request contains no new + // classifiable human text. It may reuse existing session state but cannot + // create or escalate it. + InputContinuation + // InputClassifiable means the request contains human-authored text that may + // initialize or escalate a session tier. + InputClassifiable +) + +// BuildInput extracts text from normalized BifrostRequest values for +// complexity_tier routing. It preserves the original boolean contract while +// BuildInputWithDisposition exposes the finer session-aware outcome. func BuildInput(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (ComplexityInput, bool) { + input, disposition := BuildInputWithDisposition(ctx, req) + return input, disposition == InputClassifiable +} + +// BuildInputWithDisposition extracts normalized classifier input and reports +// whether the request should be classified, should reuse existing session state, +// or should bypass session handling. Extraction runs after provider-specific +// transport conversion, so routing never reparses raw request payloads. +func BuildInputWithDisposition(ctx *schemas.BifrostContext, req *schemas.BifrostRequest) (ComplexityInput, InputDisposition) { if req == nil { - return ComplexityInput{}, false + return ComplexityInput{}, InputBypass } harness := detectComplexityHarness(ctx) if harness == complexityHarnessCodex && isCodexBackgroundRequest(ctx) { - return ComplexityInput{}, false + return ComplexityInput{}, InputBypass } switch req.RequestType { case schemas.ChatCompletionRequest, schemas.ChatCompletionStreamRequest: if req.ChatRequest == nil { - return ComplexityInput{}, false + return ComplexityInput{}, InputBypass + } + input, ok := extractFromChatMessages(req.ChatRequest.Input, harness) + if !ok { + return ComplexityInput{}, InputContinuation } - return extractFromChatMessages(req.ChatRequest.Input, harness) + if chatHasTrailingContinuation(req.ChatRequest.Input, harness) { + return ComplexityInput{}, InputContinuation + } + return input, InputClassifiable case schemas.TextCompletionRequest, schemas.TextCompletionStreamRequest: if req.TextCompletionRequest == nil { - return ComplexityInput{}, false + return ComplexityInput{}, InputBypass + } + input, ok := extractFromTextCompletionRequest(req.TextCompletionRequest) + if !ok { + return ComplexityInput{}, InputBypass } - return extractFromTextCompletionRequest(req.TextCompletionRequest) + return input, InputClassifiable case schemas.ResponsesRequest, schemas.ResponsesStreamRequest: if req.ResponsesRequest == nil { - return ComplexityInput{}, false + return ComplexityInput{}, InputBypass } - return extractFromResponsesRequest(req.ResponsesRequest, harness) + input, ok := extractFromResponsesRequest(req.ResponsesRequest, harness) + if !ok { + return ComplexityInput{}, InputContinuation + } + if responsesHasTrailingContinuation(req.ResponsesRequest.Input, harness) { + return ComplexityInput{}, InputContinuation + } + return input, InputClassifiable default: - return ComplexityInput{}, false + return ComplexityInput{}, InputBypass + } +} + +// chatHasTrailingContinuation distinguishes a fresh human turn from a tool or +// assistant continuation that merely replays an older human message in the +// request history. Context-only harness fragments and system instructions do +// not change which conversational actor came last. +func chatHasTrailingContinuation(messages []schemas.ChatMessage, harness complexityHarness) bool { + for i := len(messages) - 1; i >= 0; i-- { + msg := messages[i] + switch msg.Role { + case schemas.ChatMessageRoleSystem, schemas.ChatMessageRoleDeveloper: + continue + case schemas.ChatMessageRoleUser: + text, ok := extractChatTextOnly(msg.Content) + if !ok { + return true + } + _, kind := sanitizeUserText(text, harness) + if kind == complexityTextContextOnly || kind == complexityTextInvalid { + continue + } + return kind != complexityTextHuman + default: + return true + } } + return false +} + +// responsesHasTrailingContinuation is the Responses-API counterpart. Items +// without a role are tool, reasoning, or provider-native history items; when +// one follows the latest user message, this is a continuation rather than a +// new human turn. +func responsesHasTrailingContinuation(messages []schemas.ResponsesMessage, harness complexityHarness) bool { + for i := len(messages) - 1; i >= 0; i-- { + msg := messages[i] + if msg.Role == nil { + return true + } + switch *msg.Role { + case schemas.ResponsesInputMessageRoleSystem, schemas.ResponsesInputMessageRoleDeveloper: + continue + case schemas.ResponsesInputMessageRoleUser: + text, ok := extractResponsesTextOnly(msg.Content) + if !ok { + return true + } + _, kind := sanitizeUserText(text, harness) + if kind == complexityTextContextOnly || kind == complexityTextInvalid { + continue + } + return kind != complexityTextHuman + default: + return true + } + } + return false } // extractFromChatMessages builds a complexity input from chat messages by @@ -347,6 +454,45 @@ func detectComplexityHarness(ctx *schemas.BifrostContext) complexityHarness { } } +// ResolveComplexitySessionID resolves the trusted session identity used only by +// complexity routing. An explicit x-bf-session-id context value wins; otherwise +// native harness metadata is accepted only when the User-Agent identifies the +// corresponding Claude Code or Codex client. +// +// Native identities are not copied into BifrostContextKeySessionID because that +// key also enables core provider-key stickiness, which is a separate feature. +func ResolveComplexitySessionID(ctx *schemas.BifrostContext) (string, bool) { + if ctx == nil { + return "", false + } + if explicit, exists := ctx.Value(schemas.BifrostContextKeySessionID).(string); exists { + return normalizeComplexitySessionID(explicit) + } + + headers, _ := ctx.Value(schemas.BifrostContextKeyRequestHeaders).(map[string]string) + switch detectComplexityHarness(ctx) { + case complexityHarnessClaudeCode: + return normalizeComplexitySessionID(headers[claudeCodeSessionIDHeader]) + case complexityHarnessCodex: + metadata, ok := parseCodexTurnMetadata(ctx) + if !ok { + return "", false + } + return normalizeComplexitySessionID(metadata.SessionID) + default: + return "", false + } +} + +func normalizeComplexitySessionID(raw string) (string, bool) { + value := strings.TrimSpace(raw) + if value == "" || len(value) > maxComplexitySessionIDLength || + !utf8.ValidString(value) || strings.ContainsRune(value, '\x00') { + return "", false + } + return value, true +} + func isCodexBackgroundRequest(ctx *schemas.BifrostContext) bool { metadata, ok := parseCodexTurnMetadata(ctx) if !ok { @@ -376,6 +522,7 @@ func parseCodexTurnMetadata(ctx *schemas.BifrostContext) (codexTurnMetadata, boo var fields struct { RequestKind json.RawMessage `json:"request_kind"` + SessionID json.RawMessage `json:"session_id"` } if err := json.Unmarshal([]byte(rawMetadata), &fields); err != nil { return codexTurnMetadata{}, false @@ -387,6 +534,9 @@ func parseCodexTurnMetadata(ctx *schemas.BifrostContext) (codexTurnMetadata, boo if len(fields.RequestKind) > 0 { _ = json.Unmarshal(fields.RequestKind, &metadata.RequestKind) } + if len(fields.SessionID) > 0 { + _ = json.Unmarshal(fields.SessionID, &metadata.SessionID) + } return metadata, true } @@ -398,6 +548,9 @@ func sanitizeUserText(text string, harness complexityHarness) (string, complexit switch harness { case complexityHarnessClaudeCode: + if isClaudeCodeHousekeepingText(text) { + return "", complexityTextHousekeeping + } cleaned, removedContext := stripComplexityTags(text, claudeContextTags[:]) cleaned, removedHousekeeping := stripComplexityTags(cleaned, claudeHousekeepingTags[:]) return classifySanitizedText(cleaned, removedContext, removedHousekeeping) @@ -410,6 +563,17 @@ func sanitizeUserText(text string, harness complexityHarness) (string, complexit } } +// isClaudeCodeHousekeepingText recognizes complete user-role messages Claude +// Code injects for background session maintenance. These are not new human +// intent and therefore must not initialize or escalate session complexity. +// Detection is deliberately prefix-based and Claude-client-gated so a human +// request that merely mentions a session XML tag remains classifiable. +func isClaudeCodeHousekeepingText(text string) bool { + text = strings.TrimSpace(text) + return strings.HasPrefix(text, claudeSessionEnvelopeOpen) || + strings.HasPrefix(text, claudeResumeRecapPrefix) +} + func sanitizeSystemText(text string, harness complexityHarness) string { text = strings.TrimSpace(text) if text == "" { diff --git a/plugins/routing/complexity/extract_test.go b/plugins/routing/complexity/extract_test.go index ebb4cc156ae..275c517f0e4 100644 --- a/plugins/routing/complexity/extract_test.go +++ b/plugins/routing/complexity/extract_test.go @@ -2,6 +2,7 @@ package complexity import ( "context" + "strings" "testing" "time" @@ -439,6 +440,24 @@ func TestSanitizeUserText_ClaudeCodeWrappers(t *testing.T) { text: "focus on routing", wantKind: complexityTextHousekeeping, }, + { + name: "session_title_request", + text: "\nhello can u help me understand what sidekiq is\n\n\n" + + "Write the title in the predominant language of the session.", + wantKind: complexityTextHousekeeping, + }, + { + name: "resume_recap_request", + text: "The user stepped away and is coming back. Recap in under 40 words, 1-2 plain sentences, no markdown. " + + "Lead with the overall goal and current task, then the one next action.", + wantKind: complexityTextHousekeeping, + }, + { + name: "session_tag_mentioned_inside_human_request", + text: "How should I parse a XML element?", + wantText: "How should I parse a XML element?", + wantKind: complexityTextHuman, + }, { name: "wrapper_with_human_text", text: "build failed\nWhy did the build fail?", @@ -462,6 +481,38 @@ func TestSanitizeUserText_ClaudeCodeWrappers(t *testing.T) { } } +func TestBuildComplexityInput_ClaudeCodeInjectedMessagesAreContinuations(t *testing.T) { + claudeCtx := complexityHarnessContext(schemas.ClaudeCLI.String(), nil) + tests := []struct { + name string + text string + }{ + { + name: "session_title_request", + text: "\nDebug the distributed queue worker\n\n\n" + + "Write the title in the predominant language of the session.", + }, + { + name: "resume_recap_request", + text: "The user stepped away and is coming back. Recap in under 40 words, 1-2 plain sentences, no markdown.", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input, disposition := BuildInputWithDisposition(claudeCtx, &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: complexityChatString(tt.text)}, + }}, + }) + + assert.Equal(t, InputContinuation, disposition) + assert.Empty(t, input.LastUserText) + }) + } +} + func TestBuildComplexityInput_ClaudeCodeContextAndHousekeeping(t *testing.T) { claudeCtx := complexityHarnessContext(schemas.ClaudeCLI.String(), nil) @@ -635,6 +686,166 @@ func TestBuildComplexityInput_HarnessMarkersRequireMatchingUserAgent(t *testing. assert.Equal(t, markerText, input.LastUserText) } +func TestBuildInputWithDisposition(t *testing.T) { + userRole := schemas.ResponsesInputMessageRoleUser + tests := []struct { + name string + ctx *schemas.BifrostContext + req *schemas.BifrostRequest + want InputDisposition + }{ + { + name: "human turn is classifiable", + req: &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: complexityChatString("Explain vector clocks")}, + }}, + }, + want: InputClassifiable, + }, + { + name: "supported conversation without human text is a continuation", + req: &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleAssistant, Content: complexityChatString("Tool result received")}, + }}, + }, + want: InputContinuation, + }, + { + name: "chat replay followed by assistant output is a continuation", + req: &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: complexityChatString("Run the tests")}, + {Role: schemas.ChatMessageRoleAssistant, Content: complexityChatString("Calling the test tool")}, + {Role: schemas.ChatMessageRoleTool, Content: complexityChatString("Tests passed")}, + }}, + }, + want: InputContinuation, + }, + { + name: "responses replay followed by tool output is a continuation", + req: func() *schemas.BifrostRequest { + itemType := schemas.ResponsesMessageTypeFunctionCallOutput + return &schemas.BifrostRequest{ + RequestType: schemas.ResponsesRequest, + ResponsesRequest: &schemas.BifrostResponsesRequest{Input: []schemas.ResponsesMessage{ + {Role: &userRole, Content: complexityResponsesString("Run the tests")}, + {Type: &itemType}, + }}, + } + }(), + want: InputContinuation, + }, + { + name: "unsupported operation bypasses session state", + req: &schemas.BifrostRequest{RequestType: schemas.EmbeddingRequest}, + want: InputBypass, + }, + { + name: "codex background request bypasses session state", + ctx: complexityHarnessContext(schemas.CodexCLI.String(), map[string]string{ + codexTurnMetadataHeader: `{"request_kind":"compaction","session_id":"session-1"}`, + }), + req: &schemas.BifrostRequest{ + RequestType: schemas.ResponsesRequest, + ResponsesRequest: &schemas.BifrostResponsesRequest{Input: []schemas.ResponsesMessage{ + {Role: &userRole, Content: complexityResponsesString("Compact this conversation")}, + }}, + }, + want: InputBypass, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, got := BuildInputWithDisposition(tt.ctx, tt.req) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestResolveComplexitySessionID(t *testing.T) { + tests := []struct { + name string + ctx *schemas.BifrostContext + want string + wantFound bool + }{ + { + name: "explicit bifrost session wins over native metadata", + ctx: func() *schemas.BifrostContext { + ctx := complexityHarnessContext(schemas.CodexCLI.String(), map[string]string{ + codexTurnMetadataHeader: `{"request_kind":"turn","session_id":"native-session"}`, + }) + ctx.SetValue(schemas.BifrostContextKeySessionID, " explicit-session ") + return ctx + }(), + want: "explicit-session", + wantFound: true, + }, + { + name: "claude native header is accepted for claude code", + ctx: complexityHarnessContext(schemas.ClaudeCLI.String(), map[string]string{ + claudeCodeSessionIDHeader: "claude-session", + }), + want: "claude-session", + wantFound: true, + }, + { + name: "claude native header is rejected for a generic client", + ctx: complexityHarnessContext("generic-client/1.0", map[string]string{ + claudeCodeSessionIDHeader: "spoofed-session", + }), + }, + { + name: "codex native metadata is accepted for codex", + ctx: complexityHarnessContext(schemas.CodexDesktop.String(), map[string]string{ + codexTurnMetadataHeader: `{"request_kind":"turn","session_id":"codex-session"}`, + }), + want: "codex-session", + wantFound: true, + }, + { + name: "invalid request kind does not invalidate codex session identity", + ctx: complexityHarnessContext(schemas.CodexCLI.String(), map[string]string{ + codexTurnMetadataHeader: `{"request_kind":{},"session_id":"codex-session"}`, + }), + want: "codex-session", + wantFound: true, + }, + { + name: "oversized explicit identity is rejected without native fallback", + ctx: func() *schemas.BifrostContext { + ctx := complexityHarnessContext(schemas.CodexCLI.String(), map[string]string{ + codexTurnMetadataHeader: `{"session_id":"native-session"}`, + }) + ctx.SetValue(schemas.BifrostContextKeySessionID, strings.Repeat("x", maxComplexitySessionIDLength+1)) + return ctx + }(), + }, + { + name: "identity containing nul is rejected", + ctx: func() *schemas.BifrostContext { + ctx := complexityHarnessContext("", nil) + ctx.SetValue(schemas.BifrostContextKeySessionID, "session\x00suffix") + return ctx + }(), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, found := ResolveComplexitySessionID(tt.ctx) + assert.Equal(t, tt.wantFound, found) + assert.Equal(t, tt.want, got) + }) + } +} + func complexityHarnessContext(userAgent string, headers map[string]string) *schemas.BifrostContext { ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) if userAgent != "" { diff --git a/plugins/routing/complexity/prerequesthook_test.go b/plugins/routing/complexity/prerequesthook_test.go index 300fb726fc2..9258aff4ebb 100644 --- a/plugins/routing/complexity/prerequesthook_test.go +++ b/plugins/routing/complexity/prerequesthook_test.go @@ -3,6 +3,7 @@ package complexity_test import ( "context" "strings" + "sync/atomic" "testing" "time" @@ -11,6 +12,7 @@ import ( "github.com/maximhq/bifrost/core/schemas" "github.com/maximhq/bifrost/framework/configstore" configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/framework/kvstore" "github.com/maximhq/bifrost/plugins/routing" "github.com/maximhq/bifrost/plugins/routing/complexity" "github.com/maximhq/bifrost/plugins/routing/rules" @@ -23,6 +25,10 @@ func chatString(text string) *schemas.ChatMessageContent { // newComplexityRuleFixture builds a routing plugin whose store carries one // rule that fires only when a complexity tier was published. func newComplexityRuleFixture(t *testing.T) *routing.RoutingPlugin { + return newComplexityRuleFixtureWithConfig(t, nil) +} + +func newComplexityRuleFixtureWithConfig(t *testing.T, config *routing.Config) *routing.RoutingPlugin { t.Helper() logger := rules.NewMockLogger() provider := "openai" @@ -42,12 +48,20 @@ func newComplexityRuleFixture(t *testing.T) *routing.RoutingPlugin { Priority: 0, })) - plugin, err := routing.InitFromStore(context.Background(), nil, logger, nil, ruleStore, routing.NewMockGovernance()) + plugin, err := routing.InitFromStore(context.Background(), config, logger, nil, ruleStore, routing.NewMockGovernance()) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, plugin.Cleanup()) }) return plugin } +func newSessionComplexityRuleFixture(t *testing.T) *routing.RoutingPlugin { + t.Helper() + store, err := kvstore.New(kvstore.Config{CleanupInterval: time.Hour}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + return newComplexityRuleFixtureWithConfig(t, &routing.Config{KVStore: store}) +} + func chatRequest(text string) *schemas.BifrostRequest { return &schemas.BifrostRequest{ RequestType: schemas.ChatCompletionRequest, @@ -105,11 +119,45 @@ func testVectorForText(text string) []float64 { return []float64{0, 1, 0} case strings.Contains(text, "deep architectural tradeoff"): return []float64{0, 0, 1} + case strings.Contains(text, "medium request"): + return []float64{0, 1, 0} + case strings.Contains(text, "complex request"): + return []float64{0, 0, 1} default: // request text: nearest to the SIMPLE exemplar return []float64{0.9, 0.1, 0} } } +func sessionAnalyzerConfig() *complexity.AnalyzerConfig { + return &complexity.AnalyzerConfig{ + Keywords: configstore.ComplexityEditableKeywordConfig{ + SimpleKeywords: []string{"a casual greeting"}, + MediumKeywords: []string{"an implementation detail question"}, + ComplexKeywords: []string{"a deep architectural tradeoff analysis"}, + }, + Semantic: &configstore.ComplexitySemanticConfig{ + Provider: "openai", + EmbeddingModel: "test-embedding-model", + }, + Session: &configstore.ComplexitySessionConfig{Enabled: true}, + } +} + +func waitForSemanticClassifier(t *testing.T, plugin *routing.RoutingPlugin) { + t.Helper() + require.Eventually(t, func() bool { + return plugin.ComplexitySemanticStatus().State == complexity.SemanticStatusReady + }, 5*time.Second, 10*time.Millisecond, "semantic warmup should become ready") +} + +func complexitySessionContext(sessionID string) *schemas.BifrostContext { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + if sessionID != "" { + ctx.SetValue(schemas.BifrostContextKeySessionID, sessionID) + } + return ctx +} + func testEmbeddingExecutor(_ *schemas.BifrostContext, req *schemas.BifrostEmbeddingRequest) (*schemas.BifrostEmbeddingResponse, *schemas.BifrostError) { texts := req.Input.Texts if req.Input.Text != nil { @@ -168,6 +216,187 @@ func TestPreRequestHook_SemanticComplexityPublishesTierAndRoutes(t *testing.T) { require.Equal(t, "gpt-4o-mini", modelOut) } +func TestPreRequestHook_SessionComplexityOnlyEscalates(t *testing.T) { + plugin := newSessionComplexityRuleFixture(t) + plugin.SetEmbeddingRequestExecutor(testEmbeddingExecutor) + require.NoError(t, plugin.ReloadComplexityAnalyzerConfig(sessionAnalyzerConfig())) + waitForSemanticClassifier(t, plugin) + + tests := []struct { + requestText string + wantTier string + wantMechanism string + wantLogParts []string + }{ + { + requestText: "a simple request", + wantTier: complexity.TierSimple, + wantMechanism: complexity.MechanismSemantic, + wantLogParts: []string{"Session complexity initialized:", "effective=SIMPLE", "proposed=SIMPLE", "source=semantic", "proposed_similarity=", `proposed_matched="a casual greeting"`}, + }, + { + requestText: "a medium request", + wantTier: complexity.TierMedium, + wantMechanism: complexity.MechanismSemantic, + wantLogParts: []string{"Session complexity escalated:", "effective=MEDIUM", "previous=SIMPLE", "proposed=MEDIUM", "source=semantic", "proposed_similarity=", `proposed_matched="an implementation detail question"`}, + }, + { + requestText: "another medium request", + wantTier: complexity.TierMedium, + wantMechanism: complexity.MechanismSemantic, + wantLogParts: []string{"Session complexity confirmed:", "effective=MEDIUM", "proposed=MEDIUM", "source=semantic", "proposed_similarity=", `proposed_matched="an implementation detail question"`}, + }, + { + requestText: "another simple request", + wantTier: complexity.TierMedium, + wantMechanism: complexity.MechanismSession, + wantLogParts: []string{"Session complexity held:", "effective=MEDIUM", "proposed=SIMPLE", "source=semantic", "proposed_similarity=", `proposed_matched="a casual greeting"`}, + }, + { + requestText: "a complex request", + wantTier: complexity.TierComplex, + wantMechanism: complexity.MechanismSemantic, + wantLogParts: []string{"Session complexity escalated:", "effective=COMPLEX", "previous=MEDIUM", "proposed=COMPLEX", "source=semantic", "proposed_similarity=", `proposed_matched="a deep architectural tradeoff analysis"`}, + }, + { + requestText: "one more simple request", + wantTier: complexity.TierComplex, + wantMechanism: complexity.MechanismSession, + wantLogParts: []string{"Session complexity reused:", "effective=COMPLEX", "reason=complex-ceiling"}, + }, + } + + for _, tt := range tests { + ctx := complexitySessionContext("session-ladder") + require.NoError(t, plugin.PreRequestHook(ctx, chatRequest(tt.requestText))) + require.Equal(t, tt.wantTier, ctx.Value(schemas.BifrostContextKeyGovernanceComplexityTier)) + require.Equal(t, tt.wantMechanism, ctx.Value(schemas.BifrostContextKeyGovernanceComplexityMechanism)) + + var sessionLog string + for _, entry := range ctx.GetRoutingEngineLogs() { + if strings.HasPrefix(entry.Message, "Session complexity ") { + sessionLog = entry.Message + break + } + } + require.NotEmpty(t, sessionLog) + for _, part := range tt.wantLogParts { + require.Contains(t, sessionLog, part) + } + if strings.Contains(sessionLog, "proposed=") { + require.NotContains(t, sessionLog, " similarity=", "proposal evidence must not look like evidence for the effective tier") + require.NotContains(t, sessionLog, " matched=", "proposal evidence must not look like evidence for the effective tier") + } + } +} + +func TestPreRequestHook_SessionComplexityIsIsolatedBySessionID(t *testing.T) { + plugin := newSessionComplexityRuleFixture(t) + plugin.SetEmbeddingRequestExecutor(testEmbeddingExecutor) + require.NoError(t, plugin.ReloadComplexityAnalyzerConfig(sessionAnalyzerConfig())) + waitForSemanticClassifier(t, plugin) + + complexCtx := complexitySessionContext("session-a") + require.NoError(t, plugin.PreRequestHook(complexCtx, chatRequest("a complex request"))) + require.Equal(t, complexity.TierComplex, complexCtx.Value(schemas.BifrostContextKeyGovernanceComplexityTier)) + + simpleCtx := complexitySessionContext("session-b") + require.NoError(t, plugin.PreRequestHook(simpleCtx, chatRequest("a simple request"))) + require.Equal(t, complexity.TierSimple, simpleCtx.Value(schemas.BifrostContextKeyGovernanceComplexityTier)) + require.Equal(t, complexity.MechanismSemantic, simpleCtx.Value(schemas.BifrostContextKeyGovernanceComplexityMechanism)) +} + +func TestPreRequestHook_SessionModeWithoutIdentityRemainsPerRequest(t *testing.T) { + plugin := newSessionComplexityRuleFixture(t) + plugin.SetEmbeddingRequestExecutor(testEmbeddingExecutor) + require.NoError(t, plugin.ReloadComplexityAnalyzerConfig(sessionAnalyzerConfig())) + waitForSemanticClassifier(t, plugin) + + complexCtx := complexitySessionContext("") + require.NoError(t, plugin.PreRequestHook(complexCtx, chatRequest("a complex request"))) + require.Equal(t, complexity.TierComplex, complexCtx.Value(schemas.BifrostContextKeyGovernanceComplexityTier)) + + simpleCtx := complexitySessionContext("") + require.NoError(t, plugin.PreRequestHook(simpleCtx, chatRequest("a simple request"))) + require.Equal(t, complexity.TierSimple, simpleCtx.Value(schemas.BifrostContextKeyGovernanceComplexityTier)) + require.Equal(t, complexity.MechanismSemantic, simpleCtx.Value(schemas.BifrostContextKeyGovernanceComplexityMechanism)) +} + +func TestPreRequestHook_SessionStoreFailureFallsBackToCurrentClassification(t *testing.T) { + store, err := kvstore.New(kvstore.Config{CleanupInterval: time.Hour}) + require.NoError(t, err) + plugin := newComplexityRuleFixtureWithConfig(t, &routing.Config{KVStore: store}) + plugin.SetEmbeddingRequestExecutor(testEmbeddingExecutor) + require.NoError(t, plugin.ReloadComplexityAnalyzerConfig(sessionAnalyzerConfig())) + waitForSemanticClassifier(t, plugin) + require.NoError(t, store.Close()) + + ctx := complexitySessionContext("store-failure") + require.NoError(t, plugin.PreRequestHook(ctx, chatRequest("a medium request"))) + require.Equal(t, complexity.TierMedium, ctx.Value(schemas.BifrostContextKeyGovernanceComplexityTier)) + require.Equal(t, complexity.MechanismSemantic, ctx.Value(schemas.BifrostContextKeyGovernanceComplexityMechanism)) +} + +func TestPreRequestHook_SessionContinuationReusesButDoesNotInitializeTier(t *testing.T) { + plugin := newSessionComplexityRuleFixture(t) + plugin.SetEmbeddingRequestExecutor(testEmbeddingExecutor) + require.NoError(t, plugin.ReloadComplexityAnalyzerConfig(sessionAnalyzerConfig())) + waitForSemanticClassifier(t, plugin) + + continuationRequest := func() *schemas.BifrostRequest { + return &schemas.BifrostRequest{ + RequestType: schemas.ChatCompletionRequest, + ChatRequest: &schemas.BifrostChatRequest{ + Provider: schemas.OpenAI, + Model: "gpt-4o", + Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: chatString("a complex request")}, + {Role: schemas.ChatMessageRoleAssistant, Content: chatString("Calling the tool")}, + {Role: schemas.ChatMessageRoleTool, Content: chatString("Tool result received")}, + }, + }, + } + } + + absentCtx := complexitySessionContext("new-session") + require.NoError(t, plugin.PreRequestHook(absentCtx, continuationRequest())) + require.Nil(t, absentCtx.Value(schemas.BifrostContextKeyGovernanceComplexityTier)) + require.Equal(t, complexity.MechanismSkipped, absentCtx.Value(schemas.BifrostContextKeyGovernanceComplexityMechanism)) + + initialCtx := complexitySessionContext("existing-session") + require.NoError(t, plugin.PreRequestHook(initialCtx, chatRequest("a medium request"))) + require.Equal(t, complexity.TierMedium, initialCtx.Value(schemas.BifrostContextKeyGovernanceComplexityTier)) + + continuationCtx := complexitySessionContext("existing-session") + require.NoError(t, plugin.PreRequestHook(continuationCtx, continuationRequest())) + require.Equal(t, complexity.TierMedium, continuationCtx.Value(schemas.BifrostContextKeyGovernanceComplexityTier)) + require.Equal(t, complexity.MechanismSession, continuationCtx.Value(schemas.BifrostContextKeyGovernanceComplexityMechanism)) + require.Nil(t, continuationCtx.Value(schemas.BifrostContextKeyGovernanceComplexityScore)) +} + +func TestPreRequestHook_ComplexSessionSkipsLaterClassifierCalls(t *testing.T) { + plugin := newSessionComplexityRuleFixture(t) + var calls atomic.Int64 + plugin.SetEmbeddingRequestExecutor(func(ctx *schemas.BifrostContext, req *schemas.BifrostEmbeddingRequest) (*schemas.BifrostEmbeddingResponse, *schemas.BifrostError) { + calls.Add(1) + return testEmbeddingExecutor(ctx, req) + }) + require.NoError(t, plugin.ReloadComplexityAnalyzerConfig(sessionAnalyzerConfig())) + waitForSemanticClassifier(t, plugin) + + beforeRequest := calls.Load() + complexCtx := complexitySessionContext("complex-ceiling") + require.NoError(t, plugin.PreRequestHook(complexCtx, chatRequest("a complex request"))) + require.Equal(t, beforeRequest+1, calls.Load(), "the first human turn should be classified") + + secondCtx := complexitySessionContext("complex-ceiling") + require.NoError(t, plugin.PreRequestHook(secondCtx, chatRequest("a simple request"))) + require.Equal(t, beforeRequest+1, calls.Load(), "COMPLEX is the ceiling, so later turns need no classifier call") + require.Equal(t, complexity.TierComplex, secondCtx.Value(schemas.BifrostContextKeyGovernanceComplexityTier)) + require.Equal(t, complexity.MechanismSession, secondCtx.Value(schemas.BifrostContextKeyGovernanceComplexityMechanism)) + require.Nil(t, secondCtx.Value(schemas.BifrostContextKeyGovernanceComplexityScore)) +} + func TestPreRequestHook_SemanticComplexityNotReadyLogsInfo(t *testing.T) { plugin := newComplexityRuleFixture(t) warmupStarted := make(chan struct{}, 1) diff --git a/plugins/routing/complexityrouting.go b/plugins/routing/complexityrouting.go new file mode 100644 index 00000000000..885172c9061 --- /dev/null +++ b/plugins/routing/complexityrouting.go @@ -0,0 +1,321 @@ +package routing + +import ( + "errors" + "fmt" + + "github.com/maximhq/bifrost/core/schemas" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/plugins/routing/complexity" +) + +const noClassifiableComplexityInputLog = "Complexity analysis skipped: no routable human-authored text detected" + +// complexityProposal is one request classifier's answer before monotonic +// session state is applied. Score is nil for mechanisms, such as the LLM +// fallback, that do not produce a meaningful numeric confidence. +type complexityProposal struct { + Result *complexity.ComplexityResult + Mechanism string + Score *float64 + MatchedExemplar string + LogLevel schemas.LogLevel + LogMessage string +} + +func (p *RoutingPlugin) computeComplexity( + ctx *schemas.BifrostContext, + req *schemas.BifrostRequest, + virtualKey *configstoreTables.TableVirtualKey, +) *complexity.ComplexityResult { + input, disposition := complexity.BuildInputWithDisposition(ctx, req) + sessionID, hasSessionID := complexity.ResolveComplexitySessionID(ctx) + sessionActive := p.sessionEnabled.Load() && hasSessionID && p.sessionStore != nil + + if disposition != complexity.InputClassifiable { + if sessionActive && disposition == complexity.InputContinuation { + key := buildComplexitySessionKey(ctx, virtualKey, sessionID) + tier, found, err := p.sessionStore.load(key, true) + if err != nil { + p.logComplexitySessionStoreError("refresh continuation", err) + } else if found { + result := &complexity.ComplexityResult{Tier: tier} + publishComplexityDecision(ctx, result, complexity.MechanismSession, nil) + ctx.AppendRoutingEngineLog( + schemas.RoutingEngineRoutingRule, + schemas.LogLevelInfo, + fmt.Sprintf("Session complexity reused: effective=%s reason=continuation", tier), + ) + return result + } + } + + publishComplexityDecision(ctx, nil, complexity.MechanismSkipped, nil) + ctx.AppendRoutingEngineLog( + schemas.RoutingEngineRoutingRule, + schemas.LogLevelInfo, + noClassifiableComplexityInputLog, + ) + return nil + } + + if !sessionActive { + proposal := p.classifyComplexityInput(ctx, input) + publishComplexityProposal(ctx, proposal) + return proposal.Result + } + + key := buildComplexitySessionKey(ctx, virtualKey, sessionID) + priorTier, priorFound, loadErr := p.sessionStore.load(key, false) + if loadErr != nil { + p.logComplexitySessionStoreError("inspect", loadErr) + proposal := p.classifyComplexityInput(ctx, input) + publishComplexityProposal(ctx, proposal) + return proposal.Result + } + + if priorFound && priorTier == complexity.TierComplex { + resolution, err := p.sessionStore.resolve(key, "") + if err == nil && resolution.EffectiveTier != "" { + result := &complexity.ComplexityResult{Tier: resolution.EffectiveTier} + publishComplexityDecision(ctx, result, complexity.MechanismSession, nil) + ctx.AppendRoutingEngineLog( + schemas.RoutingEngineRoutingRule, + schemas.LogLevelInfo, + "Session complexity reused: effective=COMPLEX reason=complex-ceiling", + ) + return result + } + if err != nil { + p.logComplexitySessionStoreError("refresh complex ceiling", err) + } + // The state could have expired between inspection and refresh. Classify the + // current human turn instead of routing from a stale read. + } + + proposal := p.classifyComplexityInput(ctx, input) + proposedTier := "" + if proposal.Result != nil { + proposedTier = proposal.Result.Tier + } + resolution, err := p.sessionStore.resolve(key, proposedTier) + if err != nil { + p.logComplexitySessionStoreError("resolve", err) + return p.publishLocalSessionFallback(ctx, priorTier, priorFound, proposal) + } + + if resolution.EffectiveTier == "" { + publishComplexityProposal(ctx, proposal) + return proposal.Result + } + + if proposal.Result != nil && resolution.EffectiveTier == proposal.Result.Tier { + publishComplexityDecision(ctx, proposal.Result, proposal.Mechanism, proposal.Score) + event := "confirmed" + previousTier := "" + switch { + case !resolution.Existed: + event = "initialized" + case resolution.Escalated: + event = "escalated" + previousTier = resolution.PreviousTier + } + ctx.AppendRoutingEngineLog( + schemas.RoutingEngineRoutingRule, + schemas.LogLevelInfo, + formatSessionProposalLog(event, resolution.EffectiveTier, previousTier, proposal), + ) + return proposal.Result + } + + result := &complexity.ComplexityResult{Tier: resolution.EffectiveTier} + publishComplexityDecision(ctx, result, complexity.MechanismSession, nil) + if proposal.Result != nil { + ctx.AppendRoutingEngineLog( + schemas.RoutingEngineRoutingRule, + schemas.LogLevelInfo, + formatSessionProposalLog("held", resolution.EffectiveTier, "", proposal), + ) + } else { + ctx.AppendRoutingEngineLog( + schemas.RoutingEngineRoutingRule, + proposal.LogLevel, + fmt.Sprintf( + "Session complexity reused: effective=%s; current classifier produced no tier (%s)", + resolution.EffectiveTier, + proposal.LogMessage, + ), + ) + } + return result +} + +func (p *RoutingPlugin) classifyComplexityInput(ctx *schemas.BifrostContext, input complexity.ComplexityInput) complexityProposal { + if p.semanticClassifier == nil || !p.semanticClassifier.IsConfigured() { + if p.logger != nil { + p.logger.Debug("[Routing] %s", noSemanticClassifierLog) + } + return complexityProposal{ + Mechanism: complexity.MechanismSkipped, + LogLevel: schemas.LogLevelInfo, + LogMessage: noSemanticClassifierLog, + } + } + + semanticResult, err := p.semanticClassifier.Classify(ctx, input) + var rejectedResult *complexity.SemanticResult + var timedOut bool + if err != nil { + if p.logger != nil { + p.logger.Debug("[Routing] Semantic complexity classification unavailable: %v", err) + } + timedOut = errors.Is(err, ErrEmbeddingTimeout) + } else if semanticResult != nil && !semanticResult.Accepted { + rejectedResult = semanticResult + if p.logger != nil { + p.logger.Debug( + "[Routing] Semantic complexity below min_similarity: tier=%s similarity=%.2f min=%.2f", + semanticResult.Tier, + semanticResult.Score, + semanticResult.MinSimilarity, + ) + } + } else if semanticResult != nil { + score := semanticResult.Score + result := &complexity.ComplexityResult{Tier: semanticResult.Tier, Score: score} + return complexityProposal{ + Result: result, + Mechanism: complexity.MechanismSemantic, + Score: &score, + MatchedExemplar: semanticResult.MatchedExemplar, + LogLevel: schemas.LogLevelInfo, + LogMessage: withMatchedExemplar( + fmt.Sprintf("Semantic complexity: tier=%s similarity=%.2f", result.Tier, result.Score), + semanticResult.MatchedExemplar, + ), + } + } + + unavailableLevel := schemas.LogLevelWarn + unavailableCause := "Semantic complexity classification unavailable" + switch { + case err == nil && semanticResult == nil: + unavailableLevel = schemas.LogLevelInfo + case rejectedResult != nil: + unavailableLevel = schemas.LogLevelInfo + unavailableCause = withMatchedExemplar( + fmt.Sprintf( + "Semantic complexity rejected: nearest tier=%s similarity=%.2f below min_similarity=%.2f", + rejectedResult.Tier, + rejectedResult.Score, + rejectedResult.MinSimilarity, + ), + rejectedResult.MatchedExemplar, + ) + case timedOut: + unavailableCause = fmt.Sprintf( + "Semantic complexity classification timed out after %s", + p.semanticClassifier.Timeout(), + ) + } + + if p.llmClassifier != nil && p.llmClassifier.FallbackEnabled() { + ctx.AppendRoutingEngineLog( + schemas.RoutingEngineRoutingRule, + schemas.LogLevelInfo, + unavailableCause+"; falling back to the LLM classifier", + ) + return p.classifyLLMComplexity(ctx, input) + } + return complexityProposal{ + Mechanism: complexity.MechanismSkipped, + LogLevel: unavailableLevel, + LogMessage: unavailableCause + ", so no complexity tier is published", + } +} + +func (p *RoutingPlugin) publishLocalSessionFallback( + ctx *schemas.BifrostContext, + priorTier string, + priorFound bool, + proposal complexityProposal, +) *complexity.ComplexityResult { + if !priorFound { + publishComplexityProposal(ctx, proposal) + return proposal.Result + } + if proposal.Result != nil && !complexityTierAtLeast(priorTier, proposal.Result.Tier) { + publishComplexityProposal(ctx, proposal) + return proposal.Result + } + + result := &complexity.ComplexityResult{Tier: priorTier} + publishComplexityDecision(ctx, result, complexity.MechanismSession, nil) + message := "" + if proposal.Result != nil { + message = formatSessionProposalLog("reused after state-store failure", priorTier, "", proposal) + } else { + message = fmt.Sprintf( + "Session complexity reused after state-store failure: effective=%s; current classifier produced no tier (%s)", + priorTier, + proposal.LogMessage, + ) + } + ctx.AppendRoutingEngineLog( + schemas.RoutingEngineRoutingRule, + schemas.LogLevelWarn, + message, + ) + return result +} + +func (p *RoutingPlugin) logComplexitySessionStoreError(operation string, err error) { + if p.logger != nil { + p.logger.Warn("[Routing] complexity session store %s failed: %v", operation, err) + } +} + +func publishComplexityProposal(ctx *schemas.BifrostContext, proposal complexityProposal) { + publishComplexityDecision(ctx, proposal.Result, proposal.Mechanism, proposal.Score) + if proposal.LogMessage != "" { + ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, proposal.LogLevel, proposal.LogMessage) + } +} + +func publishComplexityDecision( + ctx *schemas.BifrostContext, + result *complexity.ComplexityResult, + mechanism string, + score *float64, +) { + ctx.ClearValue(schemas.BifrostContextKeyGovernanceComplexityTier) + ctx.ClearValue(schemas.BifrostContextKeyGovernanceComplexityScore) + ctx.ClearValue(schemas.BifrostContextKeyGovernanceComplexityMechanism) + if result != nil { + ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityTier, result.Tier) + } + if score != nil { + ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityScore, *score) + } + if mechanism != "" { + ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityMechanism, mechanism) + } +} + +// formatSessionProposalLog uses one field vocabulary for every session event +// backed by a current classifier proposal. Semantic evidence is explicitly +// proposal-scoped because the effective tier can be retained from prior state. +func formatSessionProposalLog(event, effectiveTier, previousTier string, proposal complexityProposal) string { + message := fmt.Sprintf("Session complexity %s: effective=%s", event, effectiveTier) + if previousTier != "" { + message += fmt.Sprintf(" previous=%s", previousTier) + } + message += fmt.Sprintf(" proposed=%s source=%s", proposal.Result.Tier, proposal.Mechanism) + if proposal.Score != nil { + message += fmt.Sprintf(" proposed_similarity=%.2f", *proposal.Score) + } + if matched := truncateExemplarForLog(proposal.MatchedExemplar); matched != "" { + message += fmt.Sprintf(" proposed_matched=%q", matched) + } + return message +} diff --git a/plugins/routing/complexitysession.go b/plugins/routing/complexitysession.go new file mode 100644 index 00000000000..2b4e2043721 --- /dev/null +++ b/plugins/routing/complexitysession.go @@ -0,0 +1,176 @@ +package routing + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "time" + + bifrost "github.com/maximhq/bifrost/core" + "github.com/maximhq/bifrost/core/schemas" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/framework/kvstore" + "github.com/maximhq/bifrost/plugins/routing/complexity" +) + +const ( + complexitySessionKeyPrefix = "complexity-session:v1:" + complexitySessionInactivityTTL = 24 * time.Hour +) + +var errInvalidComplexitySessionTier = errors.New("invalid stored complexity session tier") + +// complexitySessionStore retains one effective complexity tier per scoped +// session. It stores no request content, classifier scores, model choices, +// provider-cache information, or turn history. The router calculates the +// highest tier before refreshing the entry's TTL. Because KVStore exposes the +// read and write as separate operations, overlapping requests for the same +// session are best-effort and resolve by last writer wins. +type complexitySessionStore struct { + store schemas.KVStore + ttl time.Duration +} + +type complexitySessionResolution struct { + PreviousTier string + EffectiveTier string + Existed bool + Escalated bool +} + +func newComplexitySessionStore(store schemas.KVStore, ttl time.Duration) *complexitySessionStore { + return &complexitySessionStore{store: store, ttl: ttl} +} + +// load returns the current unexpired tier. When refresh is true, an existing +// entry receives a fresh inactivity TTL. It never creates an entry. +func (s *complexitySessionStore) load(key string, refresh bool) (string, bool, error) { + if s == nil || s.store == nil { + return "", false, nil + } + value, err := s.store.Get(key) + if errors.Is(err, kvstore.ErrNotFound) { + return "", false, nil + } + if err != nil { + return "", false, err + } + + tier, err := decodeStoredComplexityTier(value) + if err != nil { + return "", true, err + } + if refresh { + if err := s.store.SetWithTTL(key, tier, s.ttl); err != nil { + return "", true, err + } + } + return tier, true, nil +} + +// resolve stores max(current, proposed) and refreshes the inactivity TTL. An +// empty proposal refreshes an existing entry but never creates one. The tier +// comparison belongs here rather than in the generic KV store; Get and +// SetWithTTL are intentionally a best-effort read-modify-write sequence. +func (s *complexitySessionStore) resolve(key, proposed string) (complexitySessionResolution, error) { + if s == nil || s.store == nil { + return complexitySessionResolution{EffectiveTier: proposed}, nil + } + if proposed != "" { + if _, ok := complexityTierRank(proposed); !ok { + return complexitySessionResolution{}, fmt.Errorf("invalid proposed complexity tier %q", proposed) + } + } + + previous, existed, err := s.load(key, false) + if err != nil { + return complexitySessionResolution{}, err + } + if !existed && proposed == "" { + return complexitySessionResolution{}, nil + } + + effective := proposed + if existed && (effective == "" || complexityTierAtLeast(previous, effective)) { + effective = previous + } + if err := s.store.SetWithTTL(key, effective, s.ttl); err != nil { + return complexitySessionResolution{}, err + } + return complexitySessionResolution{ + PreviousTier: previous, + EffectiveTier: effective, + Existed: existed, + Escalated: existed && proposed != "" && effective != previous, + }, nil +} + +func complexityTierRank(tier string) (int, bool) { + switch tier { + case complexity.TierSimple: + return 1, true + case complexity.TierMedium: + return 2, true + case complexity.TierComplex: + return 3, true + default: + return 0, false + } +} + +func complexityTierAtLeast(left, right string) bool { + leftRank, leftOK := complexityTierRank(left) + rightRank, rightOK := complexityTierRank(right) + return leftOK && rightOK && leftRank >= rightRank +} + +func decodeStoredComplexityTier(value any) (string, error) { + var tier string + switch typed := value.(type) { + case string: + tier = typed + case []byte: + if err := json.Unmarshal(typed, &tier); err != nil { + tier = string(typed) + } + default: + return "", fmt.Errorf("%w: value has type %T", errInvalidComplexitySessionTier, value) + } + if _, ok := complexityTierRank(tier); !ok { + return "", fmt.Errorf("%w: %q", errInvalidComplexitySessionTier, tier) + } + return tier, nil +} + +// buildComplexitySessionKey isolates equal caller session IDs across virtual +// keys and authenticated users, then hashes the complete tuple so the in-memory +// key has bounded size and reveals no caller-provided identifier. +func buildComplexitySessionKey( + ctx *schemas.BifrostContext, + virtualKey *configstoreTables.TableVirtualKey, + sessionID string, +) string { + virtualKeyID := "" + if virtualKey != nil { + virtualKeyID = virtualKey.ID + } + userID := bifrost.GetStringFromContext(ctx, schemas.BifrostContextKeyUserID) + scopeKind := "deployment" + switch { + case virtualKeyID != "" && userID != "": + scopeKind = "virtual-key-user" + case virtualKeyID != "": + scopeKind = "virtual-key" + case userID != "": + scopeKind = "user" + } + + hash := sha256.New() + for _, part := range []string{scopeKind, virtualKeyID, userID, sessionID} { + _, _ = hash.Write([]byte(part)) + _, _ = hash.Write([]byte{0}) + } + return complexitySessionKeyPrefix + hex.EncodeToString(hash.Sum(nil)) +} diff --git a/plugins/routing/complexitysession_test.go b/plugins/routing/complexitysession_test.go new file mode 100644 index 00000000000..bee0f2079ee --- /dev/null +++ b/plugins/routing/complexitysession_test.go @@ -0,0 +1,215 @@ +package routing + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/configstore" + configstoreTables "github.com/maximhq/bifrost/framework/configstore/tables" + "github.com/maximhq/bifrost/framework/kvstore" + "github.com/maximhq/bifrost/plugins/routing/complexity" + "github.com/maximhq/bifrost/plugins/routing/rules" +) + +func testEnabledSessionAnalyzerConfig() *complexity.AnalyzerConfig { + cfg := complexity.DefaultAnalyzerConfig() + cfg.Semantic = &configstore.ComplexitySemanticConfig{ + Provider: schemas.OpenAI, + EmbeddingModel: "text-embedding-3-small", + } + cfg.Session = &configstore.ComplexitySessionConfig{Enabled: true} + return &cfg +} + +func TestRoutingPluginRequiresKVStoreForSessionMode(t *testing.T) { + logger := rules.NewMockLogger() + ruleStore, err := rules.NewLocalStore(context.Background(), logger, nil) + require.NoError(t, err) + + plugin, err := InitFromStore( + context.Background(), + &Config{ComplexityAnalyzerConfig: testEnabledSessionAnalyzerConfig()}, + logger, + nil, + ruleStore, + NewMockGovernance(), + ) + require.ErrorContains(t, err, "requires a KV store") + require.Nil(t, plugin) +} + +func newTestComplexitySessionStore(t *testing.T, ttl time.Duration) (*complexitySessionStore, *kvstore.Store) { + t.Helper() + store, err := kvstore.New(kvstore.Config{CleanupInterval: time.Hour}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + return newComplexitySessionStore(store, ttl), store +} + +type recordingSessionKVStore struct { + schemas.KVStore + setTTLs []time.Duration +} + +func (s *recordingSessionKVStore) SetWithTTL(key string, value any, ttl time.Duration) error { + s.setTTLs = append(s.setTTLs, ttl) + return s.KVStore.SetWithTTL(key, value, ttl) +} + +func TestComplexitySessionStoreMonotonicLadder(t *testing.T) { + sessions, _ := newTestComplexitySessionStore(t, time.Minute) + key := complexitySessionKeyPrefix + "ladder" + + resolution, err := sessions.resolve(key, complexity.TierSimple) + require.NoError(t, err) + require.False(t, resolution.Existed) + require.Equal(t, complexity.TierSimple, resolution.EffectiveTier) + + resolution, err = sessions.resolve(key, complexity.TierMedium) + require.NoError(t, err) + require.True(t, resolution.Escalated) + require.Equal(t, complexity.TierMedium, resolution.EffectiveTier) + + resolution, err = sessions.resolve(key, complexity.TierSimple) + require.NoError(t, err) + require.False(t, resolution.Escalated) + require.Equal(t, complexity.TierMedium, resolution.EffectiveTier) + + resolution, err = sessions.resolve(key, complexity.TierComplex) + require.NoError(t, err) + require.True(t, resolution.Escalated) + require.Equal(t, complexity.TierComplex, resolution.EffectiveTier) + + resolution, err = sessions.resolve(key, complexity.TierSimple) + require.NoError(t, err) + require.Equal(t, complexity.TierComplex, resolution.EffectiveTier) +} + +func TestComplexitySessionStoreEmptyProposalOnlyRefreshesExisting(t *testing.T) { + sessions, store := newTestComplexitySessionStore(t, time.Minute) + key := complexitySessionKeyPrefix + "continuation" + + resolution, err := sessions.resolve(key, "") + require.NoError(t, err) + require.Empty(t, resolution.EffectiveTier) + require.Equal(t, 0, store.Len()) + + _, err = sessions.resolve(key, complexity.TierMedium) + require.NoError(t, err) + resolution, err = sessions.resolve(key, "") + require.NoError(t, err) + require.True(t, resolution.Existed) + require.Equal(t, complexity.TierMedium, resolution.EffectiveTier) +} + +func TestComplexitySessionStoreRefreshesTTLWhenTierDoesNotChange(t *testing.T) { + store, err := kvstore.New(kvstore.Config{CleanupInterval: time.Hour}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + const sessionTTL = 45 * time.Minute + recordingStore := &recordingSessionKVStore{KVStore: store} + sessions := newComplexitySessionStore(recordingStore, sessionTTL) + key := complexitySessionKeyPrefix + "refresh" + + _, err = sessions.resolve(key, complexity.TierMedium) + require.NoError(t, err) + resolution, err := sessions.resolve(key, complexity.TierSimple) + require.NoError(t, err) + require.Equal(t, complexity.TierMedium, resolution.EffectiveTier) + tier, found, err := sessions.load(key, true) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, complexity.TierMedium, tier) + + require.Equal(t, []time.Duration{sessionTTL, sessionTTL, sessionTTL}, recordingStore.setTTLs) +} + +func TestComplexitySessionStoreExpiryStartsNewEpoch(t *testing.T) { + sessions, _ := newTestComplexitySessionStore(t, 10*time.Millisecond) + key := complexitySessionKeyPrefix + "expiry" + + _, err := sessions.resolve(key, complexity.TierComplex) + require.NoError(t, err) + time.Sleep(25 * time.Millisecond) + + tier, found, err := sessions.load(key, false) + require.NoError(t, err) + require.False(t, found) + require.Empty(t, tier) + + resolution, err := sessions.resolve(key, complexity.TierSimple) + require.NoError(t, err) + require.False(t, resolution.Existed) + require.Equal(t, complexity.TierSimple, resolution.EffectiveTier) +} + +func TestComplexitySessionStoreRejectsCorruptTier(t *testing.T) { + sessions, store := newTestComplexitySessionStore(t, time.Minute) + key := complexitySessionKeyPrefix + "corrupt" + require.NoError(t, store.SetWithTTL(key, "UNKNOWN", time.Minute)) + + _, _, err := sessions.load(key, false) + require.ErrorIs(t, err, errInvalidComplexitySessionTier) + _, err = sessions.resolve(key, complexity.TierSimple) + require.ErrorIs(t, err, errInvalidComplexitySessionTier) +} + +func TestComplexitySessionStoreDecodesReplicatedString(t *testing.T) { + sessions, store := newTestComplexitySessionStore(t, time.Minute) + key := complexitySessionKeyPrefix + "remote" + now := time.Now() + require.NoError(t, store.SetRemote(key, []byte(`"MEDIUM"`), now.UnixNano(), now.Add(time.Minute).UnixNano())) + + tier, found, err := sessions.load(key, false) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, complexity.TierMedium, tier) +} + +func TestBuildComplexitySessionKeyScopesAndHidesIdentity(t *testing.T) { + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + ctx.SetValue(schemas.BifrostContextKeyUserID, "user-1") + sessionID := "caller-session-secret" + + base := buildComplexitySessionKey(ctx, &configstoreTables.TableVirtualKey{ID: "vk-1"}, sessionID) + require.True(t, strings.HasPrefix(base, complexitySessionKeyPrefix)) + require.NotContains(t, base, sessionID) + require.NotContains(t, base, "user-1") + require.NotContains(t, base, "vk-1") + require.Equal(t, base, buildComplexitySessionKey(ctx, &configstoreTables.TableVirtualKey{ID: "vk-1"}, sessionID)) + + require.NotEqual(t, base, buildComplexitySessionKey(ctx, &configstoreTables.TableVirtualKey{ID: "vk-2"}, sessionID)) + ctx.SetValue(schemas.BifrostContextKeyUserID, "user-2") + require.NotEqual(t, base, buildComplexitySessionKey(ctx, &configstoreTables.TableVirtualKey{ID: "vk-1"}, sessionID)) + require.NotEqual(t, base, buildComplexitySessionKey(ctx, &configstoreTables.TableVirtualKey{ID: "vk-1"}, "another-session")) +} + +func TestPublishLocalSessionFallbackPreservesProposalEvidence(t *testing.T) { + score := 0.826 + proposal := complexityProposal{ + Result: &complexity.ComplexityResult{Tier: complexity.TierSimple}, + Mechanism: complexity.MechanismSemantic, + Score: &score, + MatchedExemplar: "a casual greeting", + } + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + result := (&RoutingPlugin{}).publishLocalSessionFallback(ctx, complexity.TierMedium, true, proposal) + + require.Equal(t, complexity.TierMedium, result.Tier) + require.Equal(t, complexity.MechanismSession, ctx.Value(schemas.BifrostContextKeyGovernanceComplexityMechanism)) + require.Nil(t, ctx.Value(schemas.BifrostContextKeyGovernanceComplexityScore)) + logs := ctx.GetRoutingEngineLogs() + require.Len(t, logs, 1) + require.Equal( + t, + `Session complexity reused after state-store failure: effective=MEDIUM proposed=SIMPLE source=semantic proposed_similarity=0.83 proposed_matched="a casual greeting"`, + logs[0].Message, + ) +} diff --git a/plugins/routing/llmclassify.go b/plugins/routing/llmclassify.go index 55a404442cf..f7652b78ac0 100644 --- a/plugins/routing/llmclassify.go +++ b/plugins/routing/llmclassify.go @@ -271,32 +271,28 @@ func llmClassifierShouldRetryWithResponses(bifrostErr *schemas.BifrostError) boo } } -// computeLLMComplexity runs the llm fallback classifier for one request — -// always after a semantic non-answer, never as the primary — and publishes -// its outcome, following the semantic branch's logging discipline: every -// failure funnels to one MechanismSkipped and one routing-engine log line -// naming the cause. -func (p *RoutingPlugin) computeLLMComplexity(ctx *schemas.BifrostContext, input complexity.ComplexityInput) *complexity.ComplexityResult { +// classifyLLMComplexity runs the llm fallback classifier for one request — +// always after a semantic non-answer, never as the primary — and returns a +// proposal without publishing context telemetry. The caller applies monotonic +// session state before publishing the effective tier. +func (p *RoutingPlugin) classifyLLMComplexity(ctx *schemas.BifrostContext, input complexity.ComplexityInput) complexityProposal { result, err := p.llmClassifier.Classify(ctx, input) if err == nil && result != nil { // No score is published: a chat completion has no similarity, and a // synthetic one would invite comparisons against thresholds tuned for // the vector backends. out := &complexity.ComplexityResult{Tier: result.Tier} - ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityTier, out.Tier) - ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityMechanism, complexity.MechanismLLM) - ctx.AppendRoutingEngineLog( - schemas.RoutingEngineRoutingRule, - schemas.LogLevelInfo, - fmt.Sprintf("LLM complexity: tier=%s", out.Tier), - ) - return out + return complexityProposal{ + Result: out, + Mechanism: complexity.MechanismLLM, + LogLevel: schemas.LogLevelInfo, + LogMessage: fmt.Sprintf("LLM complexity: tier=%s", out.Tier), + } } if err != nil && p.logger != nil { p.logger.Debug("[Governance] LLM complexity classification unavailable: %v", err) } - ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityMechanism, complexity.MechanismSkipped) // One line per decision, naming the cause. Every branch is an operator // problem — a budget to raise, a model that ignores the response contract, // or wiring that has not finished — so unlike the semantic branch there is @@ -323,8 +319,11 @@ func (p *RoutingPlugin) computeLLMComplexity(ctx *schemas.BifrostContext, input // not have. Provider error strings carry no secrets. unavailableLog = fmt.Sprintf("LLM complexity classification unavailable: %v; no complexity tier is published", err) } - ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelWarn, unavailableLog) - return nil + return complexityProposal{ + Mechanism: complexity.MechanismSkipped, + LogLevel: schemas.LogLevelWarn, + LogMessage: unavailableLog, + } } // chatResponseText extracts the assistant text from the first choice of a diff --git a/plugins/routing/main.go b/plugins/routing/main.go index eb94da8ad6f..fdaeedd89a2 100644 --- a/plugins/routing/main.go +++ b/plugins/routing/main.go @@ -11,7 +11,6 @@ package routing import ( "context" - "errors" "fmt" "strings" "sync" @@ -54,6 +53,9 @@ type Config struct { // ComplexityAnalyzerConfig overrides the analyzer defaults. When nil, the persisted // config is used, falling back to the built-in defaults. ComplexityAnalyzerConfig *complexity.AnalyzerConfig `json:"complexity_analyzer_config,omitempty"` + // KVStore is the runtime-only shared store used for complexity + // session state. It is injected by the host and is never serialized. + KVStore schemas.KVStore `json:"-"` } // chainMaxDepthOrDefault resolves the configured chain depth, falling back to the default. @@ -73,6 +75,8 @@ type RoutingPlugin struct { complexityAnalyzer atomic.Pointer[complexity.ComplexityAnalyzer] semanticClassifier *complexity.SemanticClassifier llmClassifier *complexity.LLMClassifier + sessionStore *complexitySessionStore + sessionEnabled atomic.Bool // governance supplies the virtual key, its live budget/rate-limit usage, and the provider // materialization that runs once rules have decided. Required: rules address budgets and @@ -99,7 +103,6 @@ type RoutingPlugin struct { // a chat completion is rejected because the judge model requires // /v1/responses; it stays nil until wired, in which case no fallback runs. responsesRequestExecutor atomic.Pointer[ResponsesRequestExecutor] - } // Init initializes and returns a routing plugin instance. @@ -162,12 +165,18 @@ func InitFromStore( semanticClassifier: complexity.NewSemanticClassifier(ctx, logger), llmClassifier: complexity.NewLLMClassifier(logger), } + if config != nil && config.KVStore != nil { + plugin.sessionStore = newComplexitySessionStore(config.KVStore, complexitySessionInactivityTTL) + } var analyzerOverride *complexity.AnalyzerConfig if config != nil { analyzerOverride = config.ComplexityAnalyzerConfig } - plugin.storeComplexityAnalyzerConfig(resolveAnalyzerConfigFromStoreOrArg(ctx, logger, configStore, analyzerOverride)) + if err := plugin.storeComplexityAnalyzerConfig(resolveAnalyzerConfigFromStoreOrArg(ctx, logger, configStore, analyzerOverride)); err != nil { + _ = plugin.Cleanup() + return nil, err + } return plugin, nil } @@ -182,11 +191,11 @@ func (p *RoutingPlugin) GetRuleStore() rules.Store { } // ReloadComplexityAnalyzerConfig swaps the analyzer used by complexity_tier routing. -func (p *RoutingPlugin) ReloadComplexityAnalyzerConfig(config *complexity.AnalyzerConfig) { - p.storeComplexityAnalyzerConfig(config) +func (p *RoutingPlugin) ReloadComplexityAnalyzerConfig(config *complexity.AnalyzerConfig) error { + return p.storeComplexityAnalyzerConfig(config) } -func (p *RoutingPlugin) storeComplexityAnalyzerConfig(config *complexity.AnalyzerConfig) { +func (p *RoutingPlugin) storeComplexityAnalyzerConfig(config *complexity.AnalyzerConfig) error { resolved, err := complexity.ValidateAndNormalize(config) if err != nil { if p.logger != nil { @@ -195,14 +204,18 @@ func (p *RoutingPlugin) storeComplexityAnalyzerConfig(config *complexity.Analyze defaults := complexity.DefaultAnalyzerConfig() resolved = &defaults } + if resolved.SessionRoutingEnabled() && p.sessionStore == nil { + return fmt.Errorf("complexity session routing requires a KV store") + } p.complexityAnalyzer.Store(complexity.NewComplexityAnalyzerWithConfig(resolved)) + p.sessionEnabled.Store(resolved.SessionRoutingEnabled()) if p.semanticClassifier != nil { p.semanticClassifier.Configure(resolved) } if p.llmClassifier != nil { p.llmClassifier.Configure(resolved) } - + return nil } // ComplexityLLMStatus returns the current llm classifier readiness. @@ -230,10 +243,19 @@ func (p *RoutingPlugin) RearmComplexitySemanticClassifier(provider schemas.Model // setting whose validity depends on live process state has a seam to hook // into; no semantic setting needs one today. func (p *RoutingPlugin) ValidateComplexityAnalyzerConfig(config *complexity.AnalyzerConfig) error { - if p.semanticClassifier == nil { - return nil + if p.semanticClassifier != nil { + if err := p.semanticClassifier.ValidateConfig(config); err != nil { + return err + } + } + resolved, err := complexity.ValidateAndNormalize(config) + if err != nil { + return err } - return p.semanticClassifier.ValidateConfig(config) + if resolved.SessionRoutingEnabled() && p.sessionStore == nil { + return fmt.Errorf("complexity session routing requires a KV store") + } + return nil } // ComplexitySemanticStatus returns the current semantic classifier readiness. @@ -356,126 +378,7 @@ func (p *RoutingPlugin) applyRoutingRules(ctx *schemas.BifrostContext, req *sche var computeComplexity func() *complexity.ComplexityResult if p.complexityAnalyzer.Load() != nil { computeComplexity = func() *complexity.ComplexityResult { - input, ok := complexity.BuildInput(ctx, req) - if !ok { - if p.logger != nil { - p.logger.Debug("[Routing] Complexity analysis skipped: no routable human-authored text detected") - } - ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityMechanism, complexity.MechanismSkipped) - ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelInfo, "Complexity analysis skipped: no routable human-authored text detected") - return nil - } - - // Semantic classification is the only mechanism. Without it there is no - // tier to publish: the lexical scorer still exists for historical - // configs but is never run, because the phrase lists an operator - // authors are semantic exemplars — whole sentences — and scoring them - // as literal keywords produces tiers that look authoritative while - // resting on matches the operator never intended. - if p.semanticClassifier == nil || !p.semanticClassifier.IsConfigured() { - if p.logger != nil { - p.logger.Debug("[Routing] %s", noSemanticClassifierLog) - } - ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityMechanism, complexity.MechanismSkipped) - ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, schemas.LogLevelInfo, noSemanticClassifierLog) - return nil - } - - // How much of the conversation is embedded is configuration - // (semantic.message_history_count); the classifier applies it from - // the same snapshot that owns the exemplars. - semanticResult, err := p.semanticClassifier.Classify(ctx, input) - // Carried to the single routing log below rather than logged here. Every - // one of these branches ends at the same outcome — no tier published — - // so logging per branch *and* at the outcome put two lines in the - // request log for one decision, the second restating the first. - var rejectedResult *complexity.SemanticResult - var timedOut bool - if err != nil { - if p.logger != nil { - p.logger.Debug("[Routing] Semantic complexity classification unavailable: %v", err) - } - timedOut = errors.Is(err, ErrEmbeddingTimeout) - } else if semanticResult != nil && !semanticResult.Accepted { - rejectedResult = semanticResult - if p.logger != nil { - p.logger.Debug( - "[Routing] Semantic complexity below min_similarity: tier=%s similarity=%.2f min=%.2f", - semanticResult.Tier, - semanticResult.Score, - semanticResult.MinSimilarity, - ) - } - } else if semanticResult != nil { - result := &complexity.ComplexityResult{Tier: semanticResult.Tier, Score: semanticResult.Score} - ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityTier, result.Tier) - ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityScore, result.Score) - ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityMechanism, complexity.MechanismSemantic) - // The exemplar is what makes the decision auditable: the tier alone - // cannot tell a reader whether the request genuinely resembled its - // nearest phrase or merely won an argmax over unrelated ones. - ctx.AppendRoutingEngineLog( - schemas.RoutingEngineRoutingRule, - schemas.LogLevelInfo, - withMatchedExemplar( - fmt.Sprintf("Semantic complexity: tier=%s similarity=%.2f", result.Tier, result.Score), - semanticResult.MatchedExemplar, - ), - ) - return result - } - // One line per decision, naming the cause. The level is part of the - // message: a classifier that could not run is an operator problem, - // while a request that resembled nothing in the tier lists is routine - // and would be noise at Warn. The cause and the outcome are built - // separately because a semantic non-answer now has two possible - // endings: skipped, or handed to the llm fallback. - unavailableLevel := schemas.LogLevelWarn - unavailableCause := "Semantic complexity classification unavailable" - switch { - case err == nil && semanticResult == nil: - unavailableLevel = schemas.LogLevelInfo - case rejectedResult != nil: - unavailableLevel = schemas.LogLevelInfo - // A near miss is the case where the exemplar matters most: it is - // the difference between "the floor is set too high for a phrase - // that genuinely fits" and "nothing in the tier lists resembles - // this request". - unavailableCause = withMatchedExemplar( - fmt.Sprintf( - "Semantic complexity rejected: nearest tier=%s similarity=%.2f below min_similarity=%.2f", - rejectedResult.Tier, - rejectedResult.Score, - rejectedResult.MinSimilarity, - ), - rejectedResult.MatchedExemplar, - ) - case timedOut: - // An exhausted budget is a tuning problem with an obvious remedy, - // and naming it as merely "unavailable" sends the operator hunting - // for a broken provider or an incomplete warmup instead of raising - // semantic.timeout. - unavailableCause = fmt.Sprintf( - "Semantic complexity classification timed out after %s", - p.semanticClassifier.Timeout(), - ) - } - // The fallback engages on every semantic non-answer alike — - // rejection, timeout, unfinished warmup, unwired executor — because - // each one leaves the same hole: a rule referencing complexity_tier - // that cannot match. Its own outcome is logged by - // computeLLMComplexity, so this line only records the handoff. - if p.llmClassifier != nil && p.llmClassifier.FallbackEnabled() { - ctx.AppendRoutingEngineLog( - schemas.RoutingEngineRoutingRule, - schemas.LogLevelInfo, - unavailableCause+"; falling back to the LLM classifier", - ) - return p.computeLLMComplexity(ctx, input) - } - ctx.SetValue(schemas.BifrostContextKeyGovernanceComplexityMechanism, complexity.MechanismSkipped) - ctx.AppendRoutingEngineLog(schemas.RoutingEngineRoutingRule, unavailableLevel, unavailableCause+", so no complexity tier is published") - return nil + return p.computeComplexity(ctx, req, virtualKey) } } diff --git a/transports/bifrost-http/handlers/routing_test.go b/transports/bifrost-http/handlers/routing_test.go index 6541347aca0..5d9eb1914c9 100644 --- a/transports/bifrost-http/handlers/routing_test.go +++ b/transports/bifrost-http/handlers/routing_test.go @@ -149,6 +149,12 @@ func TestComplexityAnalyzerConfigPutPersistsAndReloads(t *testing.T) { cfg.TierBoundaries.SimpleMedium = 0.12 cfg.TierBoundaries.MediumComplex = 0.34 cfg.Keywords.MediumKeywords = []string{" Function ", "api", "API"} + cfg.Semantic = &complexity.SemanticConfig{ + Provider: "openai", + EmbeddingModel: "text-embedding-3-small", + MinSimilarity: 0.65, + } + cfg.Session = &complexity.SessionConfig{Enabled: true} ctx := newTestRequestCtx(testComplexityAnalyzerPayload(t, cfg)) handler.updateComplexityAnalyzerConfig(ctx) @@ -170,6 +176,12 @@ func TestComplexityAnalyzerConfigPutPersistsAndReloads(t *testing.T) { if stored == nil || len(stored.Keywords.MediumKeywords) != 2 { t.Fatalf("expected normalized stored keywords, got %+v", stored) } + if stored.Semantic == nil || stored.Semantic.MinSimilarity != 0.65 { + t.Fatalf("expected semantic threshold to persist, got %+v", stored.Semantic) + } + if stored.Session == nil || !stored.Session.Enabled { + t.Fatalf("expected enabled session config to persist, got %+v", stored.Session) + } } func TestComplexityAnalyzerConfigPutRejectsInvalidPayloads(t *testing.T) { diff --git a/transports/bifrost-http/server/plugins.go b/transports/bifrost-http/server/plugins.go index 842692502ea..20272788e0d 100644 --- a/transports/bifrost-http/server/plugins.go +++ b/transports/bifrost-http/server/plugins.go @@ -104,6 +104,13 @@ func loadBuiltinPlugin(ctx context.Context, name string, pluginConfig any, bifro if err != nil { return nil, fmt.Errorf("failed to marshal routing plugin config: %w", err) } + if routingConfig == nil { + routingConfig = &routing.Config{} + } + // Session complexity state uses the same process-wide store as core + // session routing. The field is runtime-only and is never persisted in + // plugin configuration. + routingConfig.KVStore = bifrostConfig.KVStore // Routing rules read the virtual key and its live budget/rate-limit usage, so the // governance plugin must already be registered when this runs. governancePlugin, err := lib.FindPluginAs[governance.BaseGovernancePlugin](bifrostConfig, governancePluginNameFromContext(ctx)) diff --git a/transports/bifrost-http/server/server.go b/transports/bifrost-http/server/server.go index a0306171006..a8949a43ce6 100644 --- a/transports/bifrost-http/server/server.go +++ b/transports/bifrost-http/server/server.go @@ -1096,8 +1096,7 @@ func (s *BifrostHTTPServer) ReloadComplexityAnalyzerConfig(ctx context.Context, if err != nil { return fmt.Errorf("routing plugin not found: %w", err) } - routingPlugin.ReloadComplexityAnalyzerConfig(config) - return nil + return routingPlugin.ReloadComplexityAnalyzerConfig(config) } // ReloadRoutingRule reloads a routing rule from the database into the routing plugin's rule cache diff --git a/transports/config.schema.json b/transports/config.schema.json index 579623346aa..3c64c039fc0 100644 --- a/transports/config.schema.json +++ b/transports/config.schema.json @@ -3974,11 +3974,31 @@ }, "llm": { "$ref": "#/$defs/complexity_llm_config" + }, + "session": { + "$ref": "#/$defs/complexity_session_config" } }, "required": ["keywords"], - "required": ["keywords"], "allOf": [ + { + "if": { + "properties": { + "session": { + "properties": { + "enabled": { + "const": true + } + }, + "required": ["enabled"] + } + }, + "required": ["session"] + }, + "then": { + "required": ["semantic"] + } + }, { "description": "Selecting the llm fallback without configuring the classifier it names loads a semantic classifier whose non-answers have nothing to fall back to; the loader rejects it for the same reason.", "if": { @@ -4001,6 +4021,18 @@ ], "additionalProperties": false }, + "complexity_session_config": { + "type": "object", + "description": "Session-aware complexity routing. When enabled and a request carries a supported session identity, Bifrost retains the highest tier observed across normally sequential turns for 24 hours of inactivity. Overlapping requests for the same session are best-effort and resolve by last writer wins.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable monotonic session tier retention. Requires semantic complexity classification; default: false." + } + }, + "required": ["enabled"], + "additionalProperties": false + }, "complexity_semantic_config": { "type": "object", "description": "Embedding-based (semantic) complexity classification settings. Presence of this block enables the semantic classifier. The classifier embeds the analyzer's shared per-tier keyword lists as its exemplars.", diff --git a/transports/schema_test/config_schema_test.go b/transports/schema_test/config_schema_test.go index 5d1e1b80b1a..198fb8908a8 100644 --- a/transports/schema_test/config_schema_test.go +++ b/transports/schema_test/config_schema_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "time" @@ -359,6 +360,67 @@ func validateConfig(t *testing.T, schema *jsonschema.Schema, configJSON string) return schema.Validate(v) } +func TestSchemaComplexityAnalyzerDependencies(t *testing.T) { + compiled := compileSchema(t) + + tests := []struct { + name string + analyzer string + wantError bool + }{ + { + name: "base analyzer remains valid", + analyzer: complexityAnalyzerSchemaConfig(), + }, + { + name: "disabled session does not require semantic config", + analyzer: complexityAnalyzerSchemaConfig(`,"session":{"enabled":false}`), + }, + { + name: "enabled session requires semantic config", + analyzer: complexityAnalyzerSchemaConfig(`,"session":{"enabled":true}`), + wantError: true, + }, + { + name: "enabled session accepts semantic config", + analyzer: complexityAnalyzerSchemaConfig(`,"session":{"enabled":true}`, `,"semantic":{"provider":"openai","embedding_model":"text-embedding-3-small"}`), + }, + { + name: "semantic fallback none does not require llm config", + analyzer: complexityAnalyzerSchemaConfig(`,"semantic":{"provider":"openai","embedding_model":"text-embedding-3-small","fallback":"none"}`), + }, + { + name: "semantic fallback llm requires llm config", + analyzer: complexityAnalyzerSchemaConfig(`,"semantic":{"provider":"openai","embedding_model":"text-embedding-3-small","fallback":"llm"}`), + wantError: true, + }, + { + name: "semantic fallback llm accepts llm config", + analyzer: complexityAnalyzerSchemaConfig( + `,"llm":{"provider":"openai","model":"gpt-4o-mini"}`, + `,"semantic":{"provider":"openai","embedding_model":"text-embedding-3-small","fallback":"llm"}`, + ), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := fmt.Sprintf(`{"governance":{"complexity_analyzer_config":%s}}`, tt.analyzer) + err := validateConfig(t, compiled, config) + if tt.wantError && err == nil { + t.Fatal("expected schema validation to fail") + } + if !tt.wantError && err != nil { + t.Fatalf("expected schema validation to pass: %v", err) + } + }) + } +} + +func complexityAnalyzerSchemaConfig(fields ...string) string { + return `{"keywords":{"simple_keywords":["simple"],"medium_keywords":["medium"],"complex_keywords":["complex"]}` + strings.Join(fields, "") + `}` +} + // TestSchemaGuardrailRuleTarget verifies explicit MCP targets without breaking legacy LLM rules. func TestSchemaGuardrailRuleTarget(t *testing.T) { compiled := compileSchema(t) diff --git a/ui/app/workspace/complexity-router/formSchema.ts b/ui/app/workspace/complexity-router/formSchema.ts index c01cb88b76b..642093fca80 100644 --- a/ui/app/workspace/complexity-router/formSchema.ts +++ b/ui/app/workspace/complexity-router/formSchema.ts @@ -37,14 +37,13 @@ const semanticSchema = z.object({ timeout: z .string() .min(1, "Enter an embedding timeout") - .refine( - (value) => isPositiveDurationString(value), - `Enter a timeout greater than 0 and at most ${MAX_SEMANTIC_TIMEOUT_MS}ms`, - ) + .refine((value) => isPositiveDurationString(value), `Enter a timeout greater than 0 and at most ${MAX_SEMANTIC_TIMEOUT_MS}ms`) .optional(), min_similarity: z.number({ error: "Enter a number between 0 and 1" }).min(0, "Must be 0 or greater").lt(1, "Must be less than 1"), message_history_count: z - .number({ error: `Enter a number between ${MIN_SEMANTIC_MESSAGE_HISTORY} and ${MAX_SEMANTIC_MESSAGE_HISTORY}` }) + .number({ + error: `Enter a number between ${MIN_SEMANTIC_MESSAGE_HISTORY} and ${MAX_SEMANTIC_MESSAGE_HISTORY}`, + }) .int("Must be a whole number") .min(MIN_SEMANTIC_MESSAGE_HISTORY, `Must be at least ${MIN_SEMANTIC_MESSAGE_HISTORY}`) .max(MAX_SEMANTIC_MESSAGE_HISTORY, `Must be at most ${MAX_SEMANTIC_MESSAGE_HISTORY}`), @@ -60,14 +59,13 @@ const llmSchema = z.object({ timeout: z .string() .min(1, "Enter a classification timeout") - .refine( - (value) => isPositiveDurationString(value), - "Enter a timeout greater than 0", - ) + .refine((value) => isPositiveDurationString(value), "Enter a timeout greater than 0") .optional(), prompt: z.string().max(MAX_LLM_PROMPT_CHARACTERS, `Must be at most ${MAX_LLM_PROMPT_CHARACTERS} characters`), message_history_count: z - .number({ error: `Enter a number between ${MIN_LLM_MESSAGE_HISTORY} and ${MAX_LLM_MESSAGE_HISTORY}` }) + .number({ + error: `Enter a number between ${MIN_LLM_MESSAGE_HISTORY} and ${MAX_LLM_MESSAGE_HISTORY}`, + }) .int("Must be a whole number") .min(MIN_LLM_MESSAGE_HISTORY, `Must be at least ${MIN_LLM_MESSAGE_HISTORY}`) .max(MAX_LLM_MESSAGE_HISTORY, `Must be at most ${MAX_LLM_MESSAGE_HISTORY}`), @@ -83,6 +81,7 @@ export const analyzerConfigSchema = z }), semantic: semanticSchema, llm: llmSchema, + session: z.object({ enabled: z.boolean() }), }) .superRefine((data, ctx) => { // A blank provider and model means the classifier simply is not configured @@ -92,12 +91,27 @@ export const analyzerConfigSchema = z const hasModel = data.semantic.embedding_model.trim() !== ""; if (hasProvider || hasModel) { if (!hasProvider) { - ctx.addIssue({ code: "custom", message: "Select an embedding provider", path: ["semantic", "provider"] }); + ctx.addIssue({ + code: "custom", + message: "Select an embedding provider", + path: ["semantic", "provider"], + }); } if (!hasModel) { - ctx.addIssue({ code: "custom", message: "Select an embedding model", path: ["semantic", "embedding_model"] }); + ctx.addIssue({ + code: "custom", + message: "Select an embedding model", + path: ["semantic", "embedding_model"], + }); } } + if (data.session.enabled && (!hasProvider || !hasModel)) { + ctx.addIssue({ + code: "custom", + message: "Configure the semantic classifier before enabling session routing", + path: ["session", "enabled"], + }); + } // The llm block follows the same half-filled rule, with one addition: // switching the semantic fallback to "llm" makes the block mandatory, @@ -106,10 +120,18 @@ export const analyzerConfigSchema = z const hasLLMModel = data.llm.model.trim() !== ""; if (hasLLMProvider || hasLLMModel || data.semantic.fallback === "llm") { if (!hasLLMProvider) { - ctx.addIssue({ code: "custom", message: "Select a fallback provider", path: ["llm", "provider"] }); + ctx.addIssue({ + code: "custom", + message: "Select a fallback provider", + path: ["llm", "provider"], + }); } if (!hasLLMModel) { - ctx.addIssue({ code: "custom", message: "Select a fallback model", path: ["llm", "model"] }); + ctx.addIssue({ + code: "custom", + message: "Select a fallback model", + path: ["llm", "model"], + }); } } @@ -179,6 +201,7 @@ export const DEFAULT_FORM_VALUES: AnalyzerFormValues = { }, semantic: DEFAULT_SEMANTIC_FORM_VALUES, llm: DEFAULT_LLM_FORM_VALUES, + session: { enabled: false }, }; // Fills in the fields the API omitted so the semantic controls stay controlled. @@ -187,6 +210,7 @@ export function toFormValues(config: AnalyzerConfig): AnalyzerFormValues { const savedLLM = config.llm; return { keywords: config.keywords, + session: config.session ?? { enabled: false }, llm: savedLLM ? { ...DEFAULT_LLM_FORM_VALUES, @@ -210,6 +234,22 @@ export function toFormValues(config: AnalyzerConfig): AnalyzerFormValues { }; } +// Builds the replacement payload without writing a disabled session block. +// Session is additive to the complexity API, and nil already means disabled; +// omitting it keeps ordinary semantic edits compatible with gateways that +// predate session-aware routing. Once enabled, the block is sent explicitly. +export function toAnalyzerPayload(values: AnalyzerFormValues, saved?: AnalyzerConfig): AnalyzerConfig { + const semantic = values.semantic.provider && values.semantic.embedding_model ? values.semantic : (saved?.semantic ?? undefined); + const llm = values.llm.provider && values.llm.model ? values.llm : (saved?.llm ?? undefined); + + return { + keywords: values.keywords, + ...(values.session.enabled ? { session: values.session } : {}), + ...(semantic ? { semantic } : {}), + ...(llm ? { llm } : {}), + }; +} + // The timeout control edits milliseconds while the form value stays a Go // duration. A value this control wrote round-trips digit for digit, including a // "0" the operator is midway through typing, which the schema rejects rather @@ -226,4 +266,4 @@ export function llmTimeoutFieldValue(timeout: string | undefined): string | numb if (timeout === "") return ""; const millis = timeout?.trim().match(/^([0-9]*\.?[0-9]+)ms$/); return millis ? millis[1] : parseLLMTimeoutMs(timeout); -} +} \ No newline at end of file diff --git a/ui/app/workspace/complexity-router/page.tsx b/ui/app/workspace/complexity-router/page.tsx index 8daddf1fc66..b74328e692a 100644 --- a/ui/app/workspace/complexity-router/page.tsx +++ b/ui/app/workspace/complexity-router/page.tsx @@ -15,6 +15,7 @@ import { Button } from "@/components/ui/button"; import { ScrollArea } from "@/components/ui/scrollArea"; import { TagInput } from "@/components/ui/tagInput"; import { Textarea } from "@/components/ui/textarea"; +import { Switch } from "@/components/ui/switch"; import { EmbeddingSupportedProviders } from "@/lib/constants/logs"; import { getErrorMessage, useGetCoreConfigQuery, useGetProvidersQuery } from "@/lib/store"; import { useGetAllKeysQuery } from "@/lib/store/apis/providersApi"; @@ -24,12 +25,7 @@ import { useResetComplexityAnalyzerConfigMutation, useUpdateComplexityAnalyzerConfigMutation, } from "@/lib/store/apis/governanceApi"; -import { - AnalyzerConfig, - KeywordListKey, - MAX_LLM_PROMPT_CHARACTERS, - TIER_PHRASE_LIST_DEFINITIONS, -} from "@/lib/types/complexityRouter"; +import { KeywordListKey, MAX_LLM_PROMPT_CHARACTERS, TIER_PHRASE_LIST_DEFINITIONS } from "@/lib/types/complexityRouter"; import { ModelProvider } from "@/lib/types/config"; import { DBKey } from "@/lib/types/governance"; import { cn } from "@/lib/utils"; @@ -39,15 +35,10 @@ import { ExternalLink, Info, LoaderCircle, RotateCcw, Save, Settings2, TriangleA import { useEffect, useMemo, useState } from "react"; import { Controller, useForm } from "react-hook-form"; import { toast } from "sonner"; -import { - AnalyzerFormValues, - analyzerConfigSchema, - DEFAULT_FORM_VALUES, - toFormValues, -} from "./formSchema"; +import { AnalyzerFormValues, analyzerConfigSchema, DEFAULT_FORM_VALUES, toAnalyzerPayload, toFormValues } from "./formSchema"; import { ClassifierStatusBadge } from "./views/classifierStatusBadge"; import EmbeddingConfigSheet from "./views/embeddingConfigSheet"; -import { SectionHeading } from "./views/formPrimitives"; +import { FieldLabel, SectionHeading } from "./views/formPrimitives"; // Embedding-capable providers gate this page, matching the local cache screen's // rule: built-ins are listed in EmbeddingSupportedProviders, custom providers @@ -143,6 +134,7 @@ export default function ComplexityRouterPage() { const liveSemantic = watch("semantic"); const liveLLM = watch("llm"); + const liveSession = watch("session"); const liveKeywords = watch("keywords"); // Narrows the model list to what this provider's enabled keys can actually @@ -322,16 +314,10 @@ export default function ComplexityRouterPage() { // operator never opened. Nothing here removes it on purpose: the provider // select has no clear option, and Restore defaults goes through its own // endpoint. - const semantic = values.semantic.provider && values.semantic.embedding_model ? values.semantic : (data?.semantic ?? undefined); - // The llm block follows the same half-filled fallback as semantic: its - // controls also live in a sheet, so a save made without opening it must - // not silently drop a working block. - const llm = values.llm.provider && values.llm.model ? values.llm : (data?.llm ?? undefined); - const payload: AnalyzerConfig = { - keywords: values.keywords, - ...(semantic ? { semantic } : {}), - ...(llm ? { llm } : {}), - }; + // The helper preserves saved sheet-only settings and omits session when + // disabled. Nil is already the wire-level disabled state; avoiding the + // additive field keeps unrelated edits compatible with older gateways. + const payload = toAnalyzerPayload(values, data); updateConfig(payload) .unwrap() .then((res) => { @@ -384,7 +370,7 @@ export default function ComplexityRouterPage() { } const keywordErrors = errors.keywords; - const hasErrors = Boolean(keywordErrors || errors.semantic || errors.llm); + const hasErrors = Boolean(keywordErrors || errors.semantic || errors.llm || errors.session); const canSave = canUpdate && isDirty && !isResetting && !(isSubmitted && hasErrors); // Rendered on the page and again inside the sheet: the re-embed cost is a @@ -436,9 +422,9 @@ export default function ComplexityRouterPage() {
Each request is embedded and takes the tier of the nearest reference phrase, filling the{" "} - complexity_tier field that routing rules - target. + complexity_tier field that routing rules target. {isLLMFallbackEnabled ? " Requests matching no phrase confidently fall back to the LLM classifier." : ""} + {liveSession.enabled ? " Session-aware routing keeps the highest tier reached during the active session." : ""} {/* Status and embedding setup ride in the header rather than as @@ -464,7 +450,11 @@ export default function ComplexityRouterPage() { {isClassifierConfigured ? "Edit embedding configuration" : "Configure embedding"} {hasUnsavedEmbeddingConfigChanges && ( - + )}
- - {/* Fallback classifier budget attribution */} -
- - Count classification cost toward budgets - - ( - - )} - /> -
)} {/* Embedding budget attribution */} -
+
+ {/* Fallback classifier budget attribution */} + {isLLMFallbackSelected && ( +
+ + Count classification cost toward budgets + + ( + + )} + /> +
+ )} + {warning} )} diff --git a/ui/lib/types/complexityRouter.ts b/ui/lib/types/complexityRouter.ts index ba4c8ec9a4d..c570ef9982f 100644 --- a/ui/lib/types/complexityRouter.ts +++ b/ui/lib/types/complexityRouter.ts @@ -46,6 +46,10 @@ export interface SemanticConfig { fallback?: SemanticFallback; } +export interface SessionConfig { + enabled: boolean; +} + // The llm classifier has no warmup: it holds no corpus and makes its first // provider call on the first classified request, so it is simply ready the // moment its block is saved. @@ -53,7 +57,6 @@ export interface LLMStatusInfo { state: "disabled" | "ready"; } - export interface SemanticStatusInfo { state: "disabled" | "warming" | "ready" | "failed"; loaded: number; @@ -80,6 +83,10 @@ export interface AnalyzerConfig { // "llm". May be present while the fallback says "none": the block is // retained so toggling the fallback never loses settings. llm?: LLMConfig; + // When enabled, one scoped session retains its highest observed tier for a + // fixed 24-hour inactivity window. The gateway owns that policy; there are no + // client-tunable thresholds or downgrade controls. + session?: SessionConfig; } export type KeywordListKey = keyof EditableKeywordConfig; @@ -97,7 +104,7 @@ export const LEGACY_COMPLEXITY_TIER_VALUES = ["REASONING"] as const; // LEGACY_COMPLEXITY_TIER_VALUES): the complexity_mechanism column ships with the // semantic classifier, so no row was ever written with the retired "lexical" // mechanism and filtering on it could only ever return nothing. -export const COMPLEXITY_MECHANISM_VALUES = ["semantic", "llm", "skipped"] as const; +export const COMPLEXITY_MECHANISM_VALUES = ["semantic", "llm", "session", "skipped"] as const; // Labels cover "lexical" even though nothing filters on it. Rows predating the // structured columns record their decision only in the prose routing log, and @@ -107,6 +114,7 @@ export const COMPLEXITY_MECHANISM_LABELS: Record = { lexical: "Lexical", semantic: "Semantic", llm: "LLM", + session: "Session", skipped: "Skipped", }; @@ -276,4 +284,4 @@ export function parseLLMTimeoutMs(timeout: string | undefined): number { const unitToMs: Record = { ns: 1e-6, us: 1e-3, µs: 1e-3, ms: 1, s: 1000, m: 60000, h: 3600000 }; const milliseconds = value * unitToMs[match[2]]; return Number.isFinite(milliseconds) && milliseconds > 0 ? milliseconds : DEFAULT_LLM_TIMEOUT_MS; -} +} \ No newline at end of file diff --git a/ui/lib/types/logs.ts b/ui/lib/types/logs.ts index b02e03e2742..a64fa68841e 100644 --- a/ui/lib/types/logs.ts +++ b/ui/lib/types/logs.ts @@ -777,7 +777,7 @@ export interface LogFilters { status?: string[]; stop_reasons?: string[]; // For filtering by stop reason (stop, length, content_filter, refusal, tool_calls, etc.) complexity_tiers?: string[]; // For filtering by routing complexity tier (SIMPLE, MEDIUM, COMPLEX) - complexity_mechanisms?: string[]; // For filtering by complexity classification mechanism (semantic, llm, skipped) + complexity_mechanisms?: string[]; // For filtering by complexity decision mechanism (semantic, llm, session, skipped) objects?: string[]; // For filtering by request type (chat.completion, text.completion, embedding) start_time?: string; // RFC3339 format end_time?: string; // RFC3339 format