diff --git a/crates/adaptive/src/config.rs b/crates/adaptive/src/config.rs index 4ac432c36..ad7a83874 100644 --- a/crates/adaptive/src/config.rs +++ b/crates/adaptive/src/config.rs @@ -7,7 +7,7 @@ use nemo_relay::plugin::ConfigPolicy; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value as Json}; -use crate::response_cache::config::{BackendConfig, KEY_STRATEGY_EXACT_REQUEST, ToolCacheConfig}; +use crate::response_cache::config::{BackendConfig, ResponseCacheKeyStrategy, ToolCacheConfig}; /// Canonical config document for the adaptive plugin component. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -34,7 +34,7 @@ pub struct AdaptiveConfig { /// Adaptive Cache Governor settings. #[serde(default, skip_serializing_if = "Option::is_none")] pub acg: Option, - /// Opt-in exact-match LLM response and tool-result cache. When present, + /// Opt-in LLM response and tool-result cache. When present, /// the adaptive plugin installs the response-cache execution intercept(s). #[serde(default, skip_serializing_if = "Option::is_none")] pub response_cache: Option, @@ -191,7 +191,7 @@ impl Default for AcgComponentConfig { } } -/// Configuration for the adaptive plugin's exact-match LLM response and +/// Configuration for the adaptive plugin's LLM response and /// opt-in tool-result cache feature. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] @@ -212,8 +212,8 @@ pub struct ResponseCacheConfig { /// requests explicitly pinned deterministic (`temperature` = 0) — absent /// or unreadable temperatures count as nondeterministic. pub cache_nondeterministic: bool, - /// Key strategy. Only [`KEY_STRATEGY_EXACT_REQUEST`] is supported. - pub key_strategy: String, + /// Typed key-derivation strategy. + pub key_strategy: ResponseCacheKeyStrategy, /// Request headers (case-insensitive) folded into the key; never auth headers. pub header_allowlist: Vec, /// Storage backend selection. @@ -231,7 +231,7 @@ impl Default for ResponseCacheConfig { priority: 50, bypass_rate: 0.0, cache_nondeterministic: false, - key_strategy: KEY_STRATEGY_EXACT_REQUEST.to_string(), + key_strategy: ResponseCacheKeyStrategy::ExactRequest, header_allowlist: Vec::new(), backend: BackendConfig::default(), tools: None, @@ -402,7 +402,7 @@ nemo_relay::editor_config! { priority => { label: "priority", kind: Integer }, bypass_rate => { label: "bypass_rate", kind: Float }, cache_nondeterministic => { label: "cache_nondeterministic", kind: Boolean }, - key_strategy => { label: "key_strategy", kind: String }, + key_strategy => { label: "key_strategy", kind: Enum, values: ["exact_request", "logical"] }, header_allowlist => { label: "header_allowlist", kind: Json }, backend => { label: "backend", diff --git a/crates/adaptive/src/lib.rs b/crates/adaptive/src/lib.rs index 237dfbe8e..4ff3b35b7 100644 --- a/crates/adaptive/src/lib.rs +++ b/crates/adaptive/src/lib.rs @@ -57,8 +57,8 @@ pub use context_helpers::{ pub use error::{AdaptiveError, Result}; #[cfg(feature = "redis-backend")] pub use redis::RedisBackend; -pub use response_cache::RESPONSE_CACHE_MARK; pub use response_cache::config::{ToolCacheConfig, ToolClass, ToolOverride}; +pub use response_cache::{RESPONSE_CACHE_MARK, ResponseCacheKeyStrategy}; pub use runtime::features::AdaptiveRuntime; pub use storage::erased::AnyBackend; pub use storage::memory::InMemoryBackend; diff --git a/crates/adaptive/src/response_cache/config.rs b/crates/adaptive/src/response_cache/config.rs index 8d408ac44..a4d331c5e 100644 --- a/crates/adaptive/src/response_cache/config.rs +++ b/crates/adaptive/src/response_cache/config.rs @@ -11,11 +11,69 @@ use std::collections::BTreeMap; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::{Map, Value as Json}; -/// Exact-request key strategy identifier. -pub const KEY_STRATEGY_EXACT_REQUEST: &str = "exact_request"; +/// Strategy for deriving an LLM response-cache key. +/// +/// The `Unknown` variant preserves an unsupported JSON/TOML value long enough +/// for configuration validation to report it with a field-specific diagnostic. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum ResponseCacheKeyStrategy { + /// Key on the normalized request exactly. + #[default] + ExactRequest, + /// Normalize tool schemas structurally while preserving their interface. + Logical, + /// A wire value not supported by this Relay build. + Unknown(String), +} + +impl ResponseCacheKeyStrategy { + /// Stable JSON/TOML representation of this strategy. + pub fn as_str(&self) -> &str { + match self { + Self::ExactRequest => "exact_request", + Self::Logical => "logical", + Self::Unknown(value) => value, + } + } +} + +impl From<&str> for ResponseCacheKeyStrategy { + fn from(value: &str) -> Self { + match value { + "exact_request" => Self::ExactRequest, + "logical" => Self::Logical, + _ => Self::Unknown(value.to_string()), + } + } +} + +impl From for ResponseCacheKeyStrategy { + fn from(value: String) -> Self { + Self::from(value.as_str()) + } +} + +impl Serialize for ResponseCacheKeyStrategy { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for ResponseCacheKeyStrategy { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Ok(Self::from(value)) + } +} /// Default in-memory byte budget: 256 MiB. pub const DEFAULT_MAX_BYTES: usize = 256 * 1024 * 1024; diff --git a/crates/adaptive/src/response_cache/key.rs b/crates/adaptive/src/response_cache/key.rs index 33966930b..645a5c112 100644 --- a/crates/adaptive/src/response_cache/key.rs +++ b/crates/adaptive/src/response_cache/key.rs @@ -25,6 +25,7 @@ use serde_json::{Map, Value as Json, json}; use sha2::{Digest, Sha256}; use crate::config::ResponseCacheConfig; +use crate::response_cache::config::ResponseCacheKeyStrategy; use crate::response_cache::mark::CacheReason; use crate::response_cache::store::CACHE_SCHEMA_VERSION; @@ -77,6 +78,13 @@ pub fn build_cache_key( normalize_tool_call_ids(object); } + if config.key_strategy == ResponseCacheKeyStrategy::Logical + && let Some(object) = body.as_object_mut() + && let Some(tools) = object.get("tools").cloned() + { + object.insert("tools".to_string(), structural_tool_schema(&tools)); + } + let header_allowlist = normalized_header_allowlist(&config.header_allowlist); let headers = cache_key_headers(&request.headers, &header_allowlist); @@ -693,6 +701,48 @@ fn rewrite_id(id_value: &mut Json, mapping: &mut Map) { *id_value = Json::String(stable); } +/// Fingerprint of the tool set for the `logical` strategy: each tool keeps its +/// full definition minus string-valued `description` keys (stripped +/// recursively), and the array is sorted so tool order does not key. +fn structural_tool_schema(tools: &Json) -> Json { + let Some(array) = tools.as_array() else { + return tools.clone(); + }; + let mut entries: Vec = array + .iter() + .map(|tool| { + let mut entry = tool.clone(); + strip_descriptions(&mut entry); + entry + }) + .collect(); + entries + .sort_by_cached_key(|entry| serde_json_canonicalizer::to_string(entry).unwrap_or_default()); + Json::Array(entries) +} + +/// Removes every string-valued `description` key, at any depth. A non-string +/// value under that key (e.g. a schema property named `description`) is +/// interface, not prose, and stays. +fn strip_descriptions(value: &mut Json) { + match value { + Json::Object(object) => { + if object.get("description").is_some_and(Json::is_string) { + object.remove("description"); + } + for nested in object.values_mut() { + strip_descriptions(nested); + } + } + Json::Array(items) => { + for item in items { + strip_descriptions(item); + } + } + _ => {} + } +} + #[cfg(test)] #[path = "../../tests/unit/response_cache/key_tests.rs"] mod tests; diff --git a/crates/adaptive/src/response_cache/mod.rs b/crates/adaptive/src/response_cache/mod.rs index f8cb4b8b7..289b99916 100644 --- a/crates/adaptive/src/response_cache/mod.rs +++ b/crates/adaptive/src/response_cache/mod.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Opt-in exact-match cache for LLM responses and tool results: a feature of +//! Opt-in cache for LLM responses and tool results: a feature of //! the adaptive plugin, configured through //! [`crate::config::AdaptiveConfig::response_cache`]. //! @@ -23,9 +23,7 @@ pub mod store; pub(crate) mod tool; pub use crate::config::ResponseCacheConfig; -pub use crate::response_cache::config::{ - BackendConfig, KEY_STRATEGY_EXACT_REQUEST, ToolCacheConfig, -}; +pub use crate::response_cache::config::{BackendConfig, ResponseCacheKeyStrategy, ToolCacheConfig}; pub(crate) use crate::response_cache::intercept::{make_intercept, make_stream_intercept}; pub use crate::response_cache::mark::RESPONSE_CACHE_MARK; pub(crate) use crate::response_cache::store::build_store; diff --git a/crates/adaptive/src/runtime/validation.rs b/crates/adaptive/src/runtime/validation.rs index 1d825b63d..6d0e2a34f 100644 --- a/crates/adaptive/src/runtime/validation.rs +++ b/crates/adaptive/src/runtime/validation.rs @@ -9,7 +9,7 @@ use nemo_relay::plugin::{ use serde_json::Value as Json; use crate::config::{AdaptiveConfig, BackendSpec, ResponseCacheConfig}; -use crate::response_cache::config::{KEY_STRATEGY_EXACT_REQUEST, ToolCacheConfig, ToolClass}; +use crate::response_cache::config::{ResponseCacheKeyStrategy, ToolCacheConfig, ToolClass}; use crate::response_cache::tool::{is_supported_tool_pattern, wildcard_patterns_overlap}; pub fn validate_config(config: &AdaptiveConfig) -> ConfigReport { @@ -123,11 +123,14 @@ fn validate_response_cache(report: &mut ConfigReport, config: &ResponseCacheConf "bypass_rate must be in [0.0, 1.0]".to_string(), )); } - if config.key_strategy != KEY_STRATEGY_EXACT_REQUEST { + if matches!(config.key_strategy, ResponseCacheKeyStrategy::Unknown(_)) { report.diagnostics.push(response_cache_error( "response_cache.unsupported_key_strategy", Some("key_strategy"), - format!("unsupported key_strategy; only \"{KEY_STRATEGY_EXACT_REQUEST}\" is supported"), + format!( + "unsupported key_strategy '{}'; supported: \"exact_request\", \"logical\"", + config.key_strategy.as_str() + ), )); } // Auth material must never enter the key or the stored entries. diff --git a/crates/adaptive/tests/integration/response_cache_tests.rs b/crates/adaptive/tests/integration/response_cache_tests.rs index a5251cec7..5e9c406cc 100644 --- a/crates/adaptive/tests/integration/response_cache_tests.rs +++ b/crates/adaptive/tests/integration/response_cache_tests.rs @@ -33,8 +33,8 @@ use nemo_relay::plugin::{ }; use nemo_relay_adaptive::plugin_component::{ComponentSpec, register_adaptive_component}; use nemo_relay_adaptive::{ - AcgComponentConfig, AdaptiveConfig, BackendSpec, ResponseCacheConfig, StateConfig, - ToolCacheConfig, ToolClass, ToolOverride, + AcgComponentConfig, AdaptiveConfig, BackendSpec, ResponseCacheConfig, ResponseCacheKeyStrategy, + StateConfig, ToolCacheConfig, ToolClass, ToolOverride, }; use serde_json::{Value as Json, json}; use tokio::sync::Mutex; @@ -555,7 +555,7 @@ async fn invalid_config_is_rejected_by_validation() { response_cache: Some(ResponseCacheConfig { ttl_seconds: 0, bypass_rate: 2.0, - key_strategy: "semantic".to_string(), + key_strategy: ResponseCacheKeyStrategy::Unknown("semantic".to_string()), namespace: "invalid-config-test".to_string(), ..ResponseCacheConfig::default() }), @@ -647,7 +647,7 @@ async fn response_cache_validation_diagnostics_identify_the_invalid_setting() { let mut cache = ResponseCacheConfig { namespace: "diagnostic-contract-test".to_string(), - key_strategy: "semantic".to_string(), + key_strategy: ResponseCacheKeyStrategy::Unknown("semantic".to_string()), tools: Some(ToolCacheConfig { enabled: true, default: ToolClass { @@ -792,6 +792,50 @@ async fn hit_preserves_usage_on_the_end_event_and_reports_savings_on_the_mark() deregister_subscriber("response_cache_event_capture").unwrap(); } +#[tokio::test] +async fn logical_strategy_reuses_across_reworded_tool_descriptions() { + let _guard = TEST_MUTEX.lock().await; + reset_global(); + // `logical` must be accepted by validation (activate_cache asserts no + // diagnostics) and must reuse across a reworded tool description end-to-end. + activate_cache(ResponseCacheConfig { + namespace: "logical-key-integration-test".to_string(), + key_strategy: ResponseCacheKeyStrategy::Logical, + ..ResponseCacheConfig::default() + }) + .await; + + let calls = Arc::new(AtomicUsize::new(0)); + let provider = counting_provider(Arc::clone(&calls), sample_body()); + + let request_with_tool = |description: &str| LlmRequest { + headers: serde_json::Map::new(), + content: json!({ + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "messages": [{"role": "user", "content": "what is the weather?"}], + "temperature": 0.0, + "tools": [{"type": "function", "function": { + "name": "get_weather", + "description": description, + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}} + }}] + }), + }; + + call(&provider, request_with_tool("Get the weather for a city.")).await; + call( + &provider, + request_with_tool("Look up the current weather (reworded)."), + ) + .await; + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "logical keying must serve the reworded-tool repeat from cache" + ); +} + #[tokio::test] async fn errors_are_not_cached() { let _guard = TEST_MUTEX.lock().await; diff --git a/crates/adaptive/tests/unit/config_tests.rs b/crates/adaptive/tests/unit/config_tests.rs index 0dea14ecf..1436e20b3 100644 --- a/crates/adaptive/tests/unit/config_tests.rs +++ b/crates/adaptive/tests/unit/config_tests.rs @@ -7,7 +7,7 @@ use super::*; use nemo_relay::config_editor::{EditorConfig, EditorFieldKind}; use serde_json::json; -use crate::response_cache::config::{ToolCacheConfig, ToolClass}; +use crate::response_cache::config::{ResponseCacheKeyStrategy, ToolCacheConfig, ToolClass}; #[test] fn test_adaptive_config_defaults() { @@ -34,6 +34,10 @@ fn test_typed_section_helpers_default() { let response_cache = ResponseCacheConfig::default(); assert!(!response_cache.cache_nondeterministic); + assert_eq!( + response_cache.key_strategy, + ResponseCacheKeyStrategy::ExactRequest + ); let tools = ToolCacheConfig::default(); assert!(!tools.enabled); @@ -52,6 +56,28 @@ fn test_tool_cache_deserializes_explicit_error_caching_opt_in() { assert!(tools.cache_errors); } +#[test] +fn test_response_cache_key_strategy_roundtrips_and_preserves_unknown() { + let logical: ResponseCacheConfig = serde_json::from_value(json!({ + "key_strategy": "logical" + })) + .unwrap(); + assert_eq!(logical.key_strategy, ResponseCacheKeyStrategy::Logical); + assert_eq!( + serde_json::to_value(logical).unwrap()["key_strategy"], + json!("logical") + ); + + let unsupported: ResponseCacheConfig = serde_json::from_value(json!({ + "key_strategy": "future" + })) + .unwrap(); + assert_eq!( + unsupported.key_strategy, + ResponseCacheKeyStrategy::Unknown("future".to_string()) + ); +} + #[test] fn test_backend_spec_in_memory_helper_uses_empty_config() { let backend = BackendSpec::in_memory(); @@ -222,6 +248,16 @@ fn test_adaptive_editor_schema_covers_canonical_options() { ); } +#[test] +fn test_response_cache_key_strategy_editor_field_is_typed() { + let schema = AdaptiveConfig::editor_schema(); + let response_cache = schema.field("response_cache").unwrap().schema().unwrap(); + let key_strategy = response_cache.field("key_strategy").unwrap(); + + assert_eq!(key_strategy.kind, EditorFieldKind::Enum); + assert_eq!(key_strategy.enum_values, &["exact_request", "logical"]); +} + #[test] fn tool_class_editor_schema_exposes_optional_version() { let tool_class = ToolClass::editor_schema(); diff --git a/crates/adaptive/tests/unit/response_cache/key_tests.rs b/crates/adaptive/tests/unit/response_cache/key_tests.rs index b7b5a47b5..d4c0e4994 100644 --- a/crates/adaptive/tests/unit/response_cache/key_tests.rs +++ b/crates/adaptive/tests/unit/response_cache/key_tests.rs @@ -993,6 +993,13 @@ fn tool_key_with_error_policy( } } +fn logical_config() -> ResponseCacheConfig { + ResponseCacheConfig { + key_strategy: ResponseCacheKeyStrategy::Logical, + ..cache_all_config() + } +} + #[test] fn same_tool_and_args_yield_the_same_key() { let args = json!({"q": "weather", "units": "metric"}); @@ -1297,3 +1304,153 @@ fn decode_round_trip_guards_fall_back_to_raw_tool_and_message_shapes() { (legacy_message_request.content.clone(), None) ); } + +// --- `logical` key strategy (structural tool-schema hash) --------- + +const LOGICAL_KEY_MODEL: &str = "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning"; + +fn tool(name: &str, description: &str, param: &str, param_type: &str) -> Json { + json!({ + "type": "function", + "function": { + "name": name, + "description": description, + "parameters": { + "type": "object", + "properties": {param: {"type": param_type, "description": "a param"}} + } + } + }) +} + +#[test] +fn logical_ignores_tool_description_and_order() { + let a = request(json!({ + "model": LOGICAL_KEY_MODEL, + "messages": [{"role": "user", "content": "hi"}], + "tools": [tool("get_weather", "Get the weather.", "city", "string"), + tool("get_time", "Get the time.", "tz", "string")] + })); + let b = request(json!({ + "model": LOGICAL_KEY_MODEL, + "messages": [{"role": "user", "content": "hi"}], + "tools": [tool("get_time", "Return the current time.", "tz", "string"), + tool("get_weather", "Look up weather, reworded.", "city", "string")] + })); + assert_eq!( + key_of("openai", &a, &logical_config()), + key_of("openai", &b, &logical_config()), + "logical keying must ignore tool description text and tool order" + ); + assert_ne!( + key_of("openai", &a, &cache_all_config()), + key_of("openai", &b, &cache_all_config()), + "exact_request must not collapse reworded/reordered tools" + ); +} + +#[test] +fn structural_tool_schema_sorts_on_canonical_bytes() { + // RFC 8785 formats 1.0 and 1 identically, so two JCS-identical tool + // sets must sort (and therefore hash) identically; a serde_json + // Display sort key would order them differently around 1.5. + let a = json!([{"x": 1.0}, {"x": 1.5}]); + let b = json!([{"x": 1.5}, {"x": 1}]); + assert_eq!( + fingerprint(&structural_tool_schema(&a)), + fingerprint(&structural_tool_schema(&b)), + "JCS-identical tool sets must produce one key regardless of number formatting" + ); +} + +#[test] +fn logical_differs_on_changed_tool_interface() { + let cfg = logical_config(); + let base = request(json!({ + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}], + "tools": [tool("get_weather", "d", "city", "string")] + })); + let renamed = request(json!({ + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}], + "tools": [tool("get_weather", "d", "location", "string")] + })); + let retyped = request(json!({ + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}], + "tools": [tool("get_weather", "d", "city", "number")] + })); + assert_ne!( + key_of("openai", &base, &cfg), + key_of("openai", &renamed, &cfg), + "a renamed parameter must change the key" + ); + assert_ne!( + key_of("openai", &base, &cfg), + key_of("openai", &retyped, &cfg), + "a changed parameter type must change the key" + ); +} + +#[test] +fn logical_differs_on_distinct_builtin_tools() { + let cfg = logical_config(); + let with_builtin = |tool: Json| { + request(json!({ + "model": LOGICAL_KEY_MODEL, + "input": "search the docs", + "store": false, + "tools": [tool] + })) + }; + assert_ne!( + key_of( + "openai", + &with_builtin(json!({"type": "web_search_preview"})), + &cfg + ), + key_of( + "openai", + &with_builtin(json!({"type": "code_interpreter", "container": {"type": "auto"}})), + &cfg, + ), + "tools without a function schema must keep their definitions in the key" + ); +} + +#[test] +fn logical_differs_on_changed_parameter_enum() { + let cfg = logical_config(); + let with_units = |units: Json| { + request(json!({ + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}], + "tools": [{"type": "function", "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": { + "unit": {"type": "string", "enum": units} + }} + }}] + })) + }; + assert_ne!( + key_of( + "openai", + &with_units(json!(["celsius", "fahrenheit"])), + &cfg + ), + key_of("openai", &with_units(json!(["kelvin"])), &cfg), + "a changed parameter enum must change the key" + ); +} + +#[test] +fn logical_and_exact_do_not_collide() { + // Tool-less, so both strategies key the identical body and only the + // strategy field in the key document separates them. + let req = request(json!({ + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}] + })); + assert_ne!( + key_of("openai", &req, &logical_config()), + key_of("openai", &req, &cache_all_config()), + "logical and exact_request must not share keys (strategy is folded in)" + ); +} diff --git a/crates/node/adaptive.d.ts b/crates/node/adaptive.d.ts index a9fc1d17d..fa24a0f0d 100644 --- a/crates/node/adaptive.d.ts +++ b/crates/node/adaptive.d.ts @@ -6,6 +6,13 @@ import type { ConfigPolicy, ConfigDiagnostic, ConfigReport } from './plugin'; export { ConfigPolicy, ConfigDiagnostic, ConfigReport }; +/** Supported LLM response-cache key derivation strategies. */ +export const ResponseCacheKeyStrategy: { + readonly ExactRequest: 'exact_request'; + readonly Logical: 'logical'; +}; +export type ResponseCacheKeyStrategy = (typeof ResponseCacheKeyStrategy)[keyof typeof ResponseCacheKeyStrategy]; + /** Adaptive state backend selection. */ export interface BackendSpec { kind: string; @@ -52,7 +59,7 @@ export interface AcgConfig { stability_thresholds?: AcgStabilityThresholds; } -/** Opt-in exact-match LLM response and tool-result cache settings. */ +/** Opt-in LLM response and tool-result cache settings. */ export interface ResponseCacheConfig { ttlSeconds?: number; /** @@ -63,7 +70,7 @@ export interface ResponseCacheConfig { priority?: number; bypassRate?: number; cacheNondeterministic?: boolean; - keyStrategy?: string; + keyStrategy?: ResponseCacheKeyStrategy; headerAllowlist?: string[]; backend?: BackendSpec; /** Opt-in tool-result cache; omit to leave the tool surface off. */ @@ -77,7 +84,7 @@ interface ResponseCachePluginConfig { priority?: number; bypass_rate?: number; cache_nondeterministic?: boolean; - key_strategy?: string; + key_strategy?: ResponseCacheKeyStrategy; header_allowlist?: string[]; backend?: BackendSpec; tools?: ToolCachePluginConfig; @@ -340,7 +347,7 @@ export declare function acgConfig(config?: AcgConfig): AcgConfig; * Create response-cache settings with defaults applied. * * Merges caller-supplied overrides onto the opt-in LLM response and tool-result - * cache config shape (exact-match) used by the adaptive plugin. This is a section of + * cache config shape used by the adaptive plugin. This is a section of * the adaptive component, not a standalone plugin kind. * * @param config - Partial response-cache settings to override. diff --git a/crates/node/adaptive.js b/crates/node/adaptive.js index 0051270e4..fc1af15dd 100644 --- a/crates/node/adaptive.js +++ b/crates/node/adaptive.js @@ -8,6 +8,11 @@ const plugin = require('./plugin.js'); const ADAPTIVE_PLUGIN_KIND = 'adaptive'; +const ResponseCacheKeyStrategy = Object.freeze({ + ExactRequest: 'exact_request', + Logical: 'logical', +}); + /** * Create a default adaptive component config. * @@ -151,7 +156,7 @@ function acgConfig(config = {}) { * Create response-cache settings with defaults applied. * * Merges caller-supplied overrides onto the opt-in LLM response and tool-result - * cache config shape (exact-match) used by the adaptive plugin. This is a section of + * cache config shape used by the adaptive plugin. This is a section of * the adaptive component, not a standalone plugin kind. * * @param {object} [config={}] - Partial response-cache settings to override. @@ -170,7 +175,7 @@ function responseCacheConfig(config = {}) { priority: 50, bypassRate: 0, cacheNondeterministic: false, - keyStrategy: 'exact_request', + keyStrategy: ResponseCacheKeyStrategy.ExactRequest, headerAllowlist: [], backend: backend ?? inMemoryBackend(), ...rest, @@ -297,6 +302,7 @@ function setLatencySensitivity(value) { module.exports = { AdaptiveRuntime, ADAPTIVE_PLUGIN_KIND, + ResponseCacheKeyStrategy, defaultConfig, inMemoryBackend, redisBackend, diff --git a/crates/node/tests/adaptive_tests.mjs b/crates/node/tests/adaptive_tests.mjs index e17a49b7a..423d6bdc8 100644 --- a/crates/node/tests/adaptive_tests.mjs +++ b/crates/node/tests/adaptive_tests.mjs @@ -461,7 +461,7 @@ describe('adaptive helpers', () => { priority: 50, bypassRate: 0, cacheNondeterministic: false, - keyStrategy: 'exact_request', + keyStrategy: adaptive.ResponseCacheKeyStrategy.ExactRequest, headerAllowlist: [], backend: adaptive.inMemoryBackend(), }); @@ -502,6 +502,20 @@ describe('adaptive helpers', () => { }); }); + it('exports and serializes response-cache key strategy values', () => { + assert.deepEqual(adaptive.ResponseCacheKeyStrategy, { + ExactRequest: 'exact_request', + Logical: 'logical', + }); + const spec = adaptive.ComponentSpec({ + version: 1, + responseCache: { + keyStrategy: adaptive.ResponseCacheKeyStrategy.Logical, + }, + }); + assert.equal(spec.config.response_cache.key_strategy, 'logical'); + }); + it('serializes response-cache config at both native boundaries', () => { const unscoped = adaptive.validateConfig({ version: 1, responseCache: {} }); assert.ok(unscoped.diagnostics.some(({ code }) => code === 'response_cache.missing_namespace')); diff --git a/go/nemo_relay/adaptive.go b/go/nemo_relay/adaptive.go index 374392a50..3afcd07b6 100644 --- a/go/nemo_relay/adaptive.go +++ b/go/nemo_relay/adaptive.go @@ -67,6 +67,16 @@ type AcgConfig struct { StabilityThresholds *AcgStabilityThresholds `json:"stability_thresholds,omitempty"` } +// ResponseCacheKeyStrategy selects how LLM response-cache keys are derived. +type ResponseCacheKeyStrategy string + +const ( + // ResponseCacheKeyStrategyExactRequest keys on the normalized request exactly. + ResponseCacheKeyStrategyExactRequest ResponseCacheKeyStrategy = "exact_request" + // ResponseCacheKeyStrategyLogical ignores tool-description wording while preserving interfaces. + ResponseCacheKeyStrategyLogical ResponseCacheKeyStrategy = "logical" +) + // ResponseCacheConfig configures the opt-in LLM response and tool-result cache: a section // of the adaptive config (a sibling to acg/adaptive_hints/tool_parallelism), not a // standalone plugin kind. The Rust core validates and installs it from the adaptive @@ -87,8 +97,8 @@ type ResponseCacheConfig struct { // CacheNondeterministic lets requests that are not explicitly deterministic // use the cache (default false). CacheNondeterministic bool `json:"cache_nondeterministic"` - // KeyStrategy is the key strategy. Only "exact_request" is supported. - KeyStrategy string `json:"key_strategy,omitempty"` + // KeyStrategy is the typed LLM response-cache key derivation strategy. + KeyStrategy ResponseCacheKeyStrategy `json:"key_strategy,omitempty"` // HeaderAllowlist lists request headers folded into the key; never auth headers. HeaderAllowlist []string `json:"header_allowlist,omitempty"` // Backend selects the cache's own storage backend (distinct from the adaptive @@ -225,7 +235,7 @@ func NewResponseCacheConfig() ResponseCacheConfig { TTLSeconds: &ttlSeconds, Priority: &priority, CacheNondeterministic: false, - KeyStrategy: "exact_request", + KeyStrategy: ResponseCacheKeyStrategyExactRequest, } } diff --git a/go/nemo_relay/adaptive_runtime_test.go b/go/nemo_relay/adaptive_runtime_test.go index 4d6a48125..3a643162d 100644 --- a/go/nemo_relay/adaptive_runtime_test.go +++ b/go/nemo_relay/adaptive_runtime_test.go @@ -185,6 +185,7 @@ func TestResponseCacheConfigReachesTypedSurface(t *testing.T) { assertResponseCacheConstructorDefaults(t, rc) rc.Namespace = responseCacheTestNamespace rc.CacheNondeterministic = true + rc.KeyStrategy = ResponseCacheKeyStrategyLogical rc.Backend = &backend assertResponseCacheJSONSurface(t, rc) assertResponseCacheValidation(t, rc) @@ -198,6 +199,9 @@ func assertResponseCacheConstructorDefaults(t *testing.T, config ResponseCacheCo if config.Priority == nil || *config.Priority != 50 { t.Fatalf("constructor priority default mismatch: %#v", config.Priority) } + if config.KeyStrategy != ResponseCacheKeyStrategyExactRequest { + t.Fatalf("constructor key strategy default mismatch: %#v", config.KeyStrategy) + } } func assertResponseCacheJSONSurface(t *testing.T, responseCache ResponseCacheConfig) { @@ -225,6 +229,9 @@ func assertResponseCacheJSONSurface(t *testing.T, responseCache ResponseCacheCon if v, ok := section["cache_nondeterministic"].(bool); !ok || !v { t.Fatalf("explicit cache_nondeterministic=true was not preserved: %#v", section["cache_nondeterministic"]) } + if section["key_strategy"] != string(ResponseCacheKeyStrategyLogical) { + t.Fatalf("logical key strategy was not preserved: %#v", section["key_strategy"]) + } if b, ok := section["backend"].(map[string]any); !ok || b["kind"] != "in_memory" { t.Fatalf("backend not preserved: %#v", section["backend"]) } diff --git a/python/nemo_relay/adaptive.py b/python/nemo_relay/adaptive.py index 99469d606..fac6326da 100644 --- a/python/nemo_relay/adaptive.py +++ b/python/nemo_relay/adaptive.py @@ -10,6 +10,7 @@ from __future__ import annotations from dataclasses import dataclass, field, fields, is_dataclass +from enum import Enum from typing import Literal, Protocol, TypedDict, cast from nemo_relay import Json, JsonObject, UnsupportedBehavior @@ -38,6 +39,13 @@ class ConfigReport(TypedDict): diagnostics: list[ConfigDiagnostic] +class ResponseCacheKeyStrategy(str, Enum): + """Supported LLM response-cache key derivation strategies.""" + + EXACT_REQUEST = "exact_request" + LOGICAL = "logical" + + class _SupportsToDict(Protocol): def to_dict(self) -> JsonObject: ... @@ -370,7 +378,7 @@ def to_dict(self) -> JsonObject: @dataclass(slots=True) class ResponseCacheConfig: - """Opt-in exact-match LLM response and tool-result cache settings. + """Opt-in LLM response and tool-result cache settings. This is a section of the adaptive component, not a standalone plugin kind. When present, the adaptive plugin installs the response-cache execution @@ -386,7 +394,7 @@ class ResponseCacheConfig: bypass_rate: Probability in ``[0.0, 1.0]`` of skipping the cache and running live. cache_nondeterministic: Cache nondeterministic requests too; ``False`` caches only requests explicitly pinned deterministic (``temperature`` = 0). - key_strategy: Key strategy. Only ``"exact_request"`` is supported. + key_strategy: Typed key derivation strategy. header_allowlist: Request headers folded into the key; never auth headers. backend: Cache storage backend (``in_memory`` or ``redis``). tools: Opt-in tool-result cache; ``None`` leaves it off. @@ -397,7 +405,7 @@ class ResponseCacheConfig: priority: int = 50 bypass_rate: float = 0.0 cache_nondeterministic: bool = False - key_strategy: str = "exact_request" + key_strategy: ResponseCacheKeyStrategy = ResponseCacheKeyStrategy.EXACT_REQUEST header_allowlist: list[str] = field(default_factory=list) backend: BackendSpec = field(default_factory=BackendSpec.in_memory) tools: ToolCacheConfig | None = None @@ -411,7 +419,11 @@ def to_dict(self) -> JsonObject: "priority": self.priority, "bypass_rate": self.bypass_rate, "cache_nondeterministic": self.cache_nondeterministic, - "key_strategy": self.key_strategy, + "key_strategy": ( + self.key_strategy.value + if isinstance(self.key_strategy, ResponseCacheKeyStrategy) + else self.key_strategy + ), "header_allowlist": self.header_allowlist, "backend": _normalize(self.backend), "tools": _normalize(self.tools), @@ -557,6 +569,7 @@ def set_latency_sensitivity(level: int) -> None: "ConfigReport", "ComponentSpec", "ResponseCacheConfig", + "ResponseCacheKeyStrategy", "StateConfig", "TelemetryConfig", "ToolCacheConfig", diff --git a/python/nemo_relay/adaptive.pyi b/python/nemo_relay/adaptive.pyi index 59800a2eb..ef7c02789 100644 --- a/python/nemo_relay/adaptive.pyi +++ b/python/nemo_relay/adaptive.pyi @@ -9,6 +9,7 @@ helpers that summarize ACG observations into structured JSON payloads. """ from dataclasses import dataclass +from enum import Enum from typing import Literal, TypedDict from nemo_relay import JsonObject, ScopeHandle, UnsupportedBehavior @@ -31,6 +32,12 @@ class ConfigReport(TypedDict): diagnostics: list[ConfigDiagnostic] +class ResponseCacheKeyStrategy(str, Enum): + """Supported LLM response-cache key derivation strategies.""" + + EXACT_REQUEST: ResponseCacheKeyStrategy + LOGICAL: ResponseCacheKeyStrategy + @dataclass(slots=True) class ConfigPolicy: """Policy for unsupported adaptive configuration. @@ -227,7 +234,7 @@ class ToolCacheConfig: @dataclass(slots=True) class ResponseCacheConfig: - """Opt-in exact-match LLM response and tool-result cache settings. + """Opt-in LLM response and tool-result cache settings. A section of the adaptive component, not a standalone plugin kind. @@ -240,7 +247,7 @@ class ResponseCacheConfig: bypass_rate: Probability in ``[0.0, 1.0]`` of skipping the cache and running live. cache_nondeterministic: Cache nondeterministic requests too; ``False`` caches only requests explicitly pinned deterministic (``temperature`` = 0). - key_strategy: Key strategy. Only ``"exact_request"`` is supported. + key_strategy: Typed key derivation strategy. header_allowlist: Request headers folded into the key. backend: Cache storage backend (``in_memory`` or ``redis``). tools: Opt-in tool-result cache; ``None`` leaves the tool surface off. @@ -251,7 +258,7 @@ class ResponseCacheConfig: priority: int = ... bypass_rate: float = ... cache_nondeterministic: bool = ... - key_strategy: str = ... + key_strategy: ResponseCacheKeyStrategy = ... header_allowlist: list[str] = ... backend: BackendSpec = ... tools: ToolCacheConfig | None = ... diff --git a/python/tests/test_adaptive_config.py b/python/tests/test_adaptive_config.py index 7373c95e9..77c1fc8f7 100644 --- a/python/tests/test_adaptive_config.py +++ b/python/tests/test_adaptive_config.py @@ -16,6 +16,7 @@ ComponentSpec, ConfigPolicy, ResponseCacheConfig, + ResponseCacheKeyStrategy, StateConfig, TelemetryConfig, ToolCacheConfig, @@ -172,6 +173,14 @@ def test_response_cache_config_serializes_with_defaults(self): "backend": {"kind": "in_memory", "config": {}}, } + def test_response_cache_key_strategy_enum_serializes(self): + config = ResponseCacheConfig( + namespace="logical-cache", + key_strategy=ResponseCacheKeyStrategy.LOGICAL, + ) + + assert config.to_dict()["key_strategy"] == "logical" + def test_response_cache_default_preserves_positional_policy_argument(self): policy = ConfigPolicy(unknown_field="error") config = AdaptiveConfig(1, None, None, None, None, None, None, policy)