From fecab469a7e08edb37fe359b7b58c4299e062dba Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Fri, 10 Jul 2026 16:31:36 -0700 Subject: [PATCH 1/9] feat(response-cache): add the logical key strategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A higher-hit-rate `key_strategy`: tools are keyed by a structural, description- and order-insensitive schema hash, so rewording or reordering tool definitions no longer busts the cache — only a changed tool interface does. Every other field keys exactly as `exact_request`; the two strategies never share keys. Signed-off-by: Zhongxuan Wang --- crates/adaptive/src/config.rs | 2 +- crates/adaptive/src/response_cache/config.rs | 5 ++ crates/adaptive/src/response_cache/key.rs | 50 +++++++++++++++++++ crates/adaptive/src/response_cache/mod.rs | 4 +- crates/adaptive/src/runtime/validation.rs | 12 +++-- .../tests/integration/response_cache_tests.rs | 43 ++++++++++++++++ crates/node/adaptive.d.ts | 4 +- crates/node/adaptive.js | 2 +- python/nemo_relay/adaptive.py | 4 +- python/nemo_relay/adaptive.pyi | 4 +- 10 files changed, 117 insertions(+), 13 deletions(-) diff --git a/crates/adaptive/src/config.rs b/crates/adaptive/src/config.rs index 4ac432c36..36eb83a3d 100644 --- a/crates/adaptive/src/config.rs +++ b/crates/adaptive/src/config.rs @@ -212,7 +212,7 @@ pub struct ResponseCacheConfig { /// requests explicitly pinned deterministic (`temperature` = 0) — absent /// or unreadable temperatures count as nondeterministic. pub cache_nondeterministic: bool, - /// Key strategy. Only [`KEY_STRATEGY_EXACT_REQUEST`] is supported. + /// Key strategy: `exact_request` or `logical`. pub key_strategy: String, /// Request headers (case-insensitive) folded into the key; never auth headers. pub header_allowlist: Vec, diff --git a/crates/adaptive/src/response_cache/config.rs b/crates/adaptive/src/response_cache/config.rs index 8d408ac44..99160b22c 100644 --- a/crates/adaptive/src/response_cache/config.rs +++ b/crates/adaptive/src/response_cache/config.rs @@ -17,6 +17,11 @@ use serde_json::{Map, Value as Json}; /// Exact-request key strategy identifier. pub const KEY_STRATEGY_EXACT_REQUEST: &str = "exact_request"; +/// The "logical" key strategy: exact-match keying, but the tool set is keyed +/// on a structural, description- and order-insensitive fingerprint — so rewording +/// or reordering tools does not bust the cache; only a changed tool interface does. +pub const KEY_STRATEGY_LOGICAL: &str = "logical"; + /// Default in-memory byte budget: 256 MiB. pub const DEFAULT_MAX_BYTES: usize = 256 * 1024 * 1024; diff --git a/crates/adaptive/src/response_cache/key.rs b/crates/adaptive/src/response_cache/key.rs index 33966930b..d2042a4fc 100644 --- a/crates/adaptive/src/response_cache/key.rs +++ b/crates/adaptive/src/response_cache/key.rs @@ -25,6 +25,7 @@ use serde_json::{Map, Value as Json, json}; use sha2::{Digest, Sha256}; use crate::config::ResponseCacheConfig; +use crate::response_cache::config::KEY_STRATEGY_LOGICAL; use crate::response_cache::mark::CacheReason; use crate::response_cache::store::CACHE_SCHEMA_VERSION; @@ -77,6 +78,13 @@ pub fn build_cache_key( normalize_tool_call_ids(object); } + if config.key_strategy == KEY_STRATEGY_LOGICAL + && let Some(object) = body.as_object_mut() + && let Some(tools) = object.get("tools").cloned() + { + object.insert("tools".to_string(), structural_tool_schema(&tools)); + } + let header_allowlist = normalized_header_allowlist(&config.header_allowlist); let headers = cache_key_headers(&request.headers, &header_allowlist); @@ -693,6 +701,48 @@ fn rewrite_id(id_value: &mut Json, mapping: &mut Map) { *id_value = Json::String(stable); } +/// Fingerprint of the tool set for the `logical` strategy: each tool keeps its +/// full definition minus string-valued `description` keys (stripped +/// recursively), and the array is sorted so tool order does not key. +fn structural_tool_schema(tools: &Json) -> Json { + let Some(array) = tools.as_array() else { + return tools.clone(); + }; + let mut entries: Vec = array + .iter() + .map(|tool| { + let mut entry = tool.clone(); + strip_descriptions(&mut entry); + entry + }) + .collect(); + entries + .sort_by_cached_key(|entry| serde_json_canonicalizer::to_string(entry).unwrap_or_default()); + Json::Array(entries) +} + +/// Removes every string-valued `description` key, at any depth. A non-string +/// value under that key (e.g. a schema property named `description`) is +/// interface, not prose, and stays. +fn strip_descriptions(value: &mut Json) { + match value { + Json::Object(object) => { + if object.get("description").is_some_and(Json::is_string) { + object.remove("description"); + } + for nested in object.values_mut() { + strip_descriptions(nested); + } + } + Json::Array(items) => { + for item in items { + strip_descriptions(item); + } + } + _ => {} + } +} + #[cfg(test)] #[path = "../../tests/unit/response_cache/key_tests.rs"] mod tests; diff --git a/crates/adaptive/src/response_cache/mod.rs b/crates/adaptive/src/response_cache/mod.rs index f8cb4b8b7..086e61006 100644 --- a/crates/adaptive/src/response_cache/mod.rs +++ b/crates/adaptive/src/response_cache/mod.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Opt-in exact-match cache for LLM responses and tool results: a feature of +//! Opt-in cache for LLM responses and tool results: a feature of //! the adaptive plugin, configured through //! [`crate::config::AdaptiveConfig::response_cache`]. //! @@ -24,7 +24,7 @@ pub(crate) mod tool; pub use crate::config::ResponseCacheConfig; pub use crate::response_cache::config::{ - BackendConfig, KEY_STRATEGY_EXACT_REQUEST, ToolCacheConfig, + BackendConfig, KEY_STRATEGY_EXACT_REQUEST, KEY_STRATEGY_LOGICAL, ToolCacheConfig, }; pub(crate) use crate::response_cache::intercept::{make_intercept, make_stream_intercept}; pub use crate::response_cache::mark::RESPONSE_CACHE_MARK; diff --git a/crates/adaptive/src/runtime/validation.rs b/crates/adaptive/src/runtime/validation.rs index e7682a98a..035a141f9 100644 --- a/crates/adaptive/src/runtime/validation.rs +++ b/crates/adaptive/src/runtime/validation.rs @@ -9,7 +9,9 @@ use nemo_relay::plugin::{ use serde_json::Value as Json; use crate::config::{AdaptiveConfig, BackendSpec, ResponseCacheConfig}; -use crate::response_cache::config::{KEY_STRATEGY_EXACT_REQUEST, ToolCacheConfig}; +use crate::response_cache::config::{ + KEY_STRATEGY_EXACT_REQUEST, KEY_STRATEGY_LOGICAL, ToolCacheConfig, +}; use crate::response_cache::tool::{is_supported_tool_pattern, wildcard_patterns_overlap}; pub fn validate_config(config: &AdaptiveConfig) -> ConfigReport { @@ -123,11 +125,15 @@ fn validate_response_cache(report: &mut ConfigReport, config: &ResponseCacheConf "bypass_rate must be in [0.0, 1.0]".to_string(), )); } - if config.key_strategy != KEY_STRATEGY_EXACT_REQUEST { + if ![KEY_STRATEGY_EXACT_REQUEST, KEY_STRATEGY_LOGICAL].contains(&config.key_strategy.as_str()) { report.diagnostics.push(response_cache_error( "response_cache.unsupported_key_strategy", Some("key_strategy"), - format!("unsupported key_strategy; only \"{KEY_STRATEGY_EXACT_REQUEST}\" is supported"), + format!( + "unsupported key_strategy '{}'; supported: \"{KEY_STRATEGY_EXACT_REQUEST}\", \ + \"{KEY_STRATEGY_LOGICAL}\"", + config.key_strategy + ), )); } // Auth material must never enter the key or the stored entries. diff --git a/crates/adaptive/tests/integration/response_cache_tests.rs b/crates/adaptive/tests/integration/response_cache_tests.rs index a5251cec7..fe2b80b58 100644 --- a/crates/adaptive/tests/integration/response_cache_tests.rs +++ b/crates/adaptive/tests/integration/response_cache_tests.rs @@ -792,6 +792,49 @@ async fn hit_preserves_usage_on_the_end_event_and_reports_savings_on_the_mark() deregister_subscriber("response_cache_event_capture").unwrap(); } +#[tokio::test] +async fn logical_strategy_reuses_across_reworded_tool_descriptions() { + let _guard = TEST_MUTEX.lock().await; + reset_global(); + // `logical` must be accepted by validation (activate_cache asserts no + // diagnostics) and must reuse across a reworded tool description end-to-end. + activate_cache(ResponseCacheConfig { + key_strategy: "logical".to_string(), + ..ResponseCacheConfig::default() + }) + .await; + + let calls = Arc::new(AtomicUsize::new(0)); + let provider = counting_provider(Arc::clone(&calls), sample_body()); + + let request_with_tool = |description: &str| LlmRequest { + headers: serde_json::Map::new(), + content: json!({ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "what is the weather?"}], + "temperature": 0.0, + "tools": [{"type": "function", "function": { + "name": "get_weather", + "description": description, + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}} + }}] + }), + }; + + call(&provider, request_with_tool("Get the weather for a city.")).await; + call( + &provider, + request_with_tool("Look up the current weather (reworded)."), + ) + .await; + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "logical keying must serve the reworded-tool repeat from cache" + ); +} + #[tokio::test] async fn errors_are_not_cached() { let _guard = TEST_MUTEX.lock().await; diff --git a/crates/node/adaptive.d.ts b/crates/node/adaptive.d.ts index a9fc1d17d..ea1a66584 100644 --- a/crates/node/adaptive.d.ts +++ b/crates/node/adaptive.d.ts @@ -52,7 +52,7 @@ export interface AcgConfig { stability_thresholds?: AcgStabilityThresholds; } -/** Opt-in exact-match LLM response and tool-result cache settings. */ +/** Opt-in LLM response and tool-result cache settings. */ export interface ResponseCacheConfig { ttlSeconds?: number; /** @@ -340,7 +340,7 @@ export declare function acgConfig(config?: AcgConfig): AcgConfig; * Create response-cache settings with defaults applied. * * Merges caller-supplied overrides onto the opt-in LLM response and tool-result - * cache config shape (exact-match) used by the adaptive plugin. This is a section of + * cache config shape used by the adaptive plugin. This is a section of * the adaptive component, not a standalone plugin kind. * * @param config - Partial response-cache settings to override. diff --git a/crates/node/adaptive.js b/crates/node/adaptive.js index 0051270e4..24902f491 100644 --- a/crates/node/adaptive.js +++ b/crates/node/adaptive.js @@ -151,7 +151,7 @@ function acgConfig(config = {}) { * Create response-cache settings with defaults applied. * * Merges caller-supplied overrides onto the opt-in LLM response and tool-result - * cache config shape (exact-match) used by the adaptive plugin. This is a section of + * cache config shape used by the adaptive plugin. This is a section of * the adaptive component, not a standalone plugin kind. * * @param {object} [config={}] - Partial response-cache settings to override. diff --git a/python/nemo_relay/adaptive.py b/python/nemo_relay/adaptive.py index 99469d606..367d42cb3 100644 --- a/python/nemo_relay/adaptive.py +++ b/python/nemo_relay/adaptive.py @@ -370,7 +370,7 @@ def to_dict(self) -> JsonObject: @dataclass(slots=True) class ResponseCacheConfig: - """Opt-in exact-match LLM response and tool-result cache settings. + """Opt-in LLM response and tool-result cache settings. This is a section of the adaptive component, not a standalone plugin kind. When present, the adaptive plugin installs the response-cache execution @@ -386,7 +386,7 @@ class ResponseCacheConfig: bypass_rate: Probability in ``[0.0, 1.0]`` of skipping the cache and running live. cache_nondeterministic: Cache nondeterministic requests too; ``False`` caches only requests explicitly pinned deterministic (``temperature`` = 0). - key_strategy: Key strategy. Only ``"exact_request"`` is supported. + key_strategy: Key strategy: ``"exact_request"`` or ``"logical"``. header_allowlist: Request headers folded into the key; never auth headers. backend: Cache storage backend (``in_memory`` or ``redis``). tools: Opt-in tool-result cache; ``None`` leaves it off. diff --git a/python/nemo_relay/adaptive.pyi b/python/nemo_relay/adaptive.pyi index 59800a2eb..1709e443b 100644 --- a/python/nemo_relay/adaptive.pyi +++ b/python/nemo_relay/adaptive.pyi @@ -227,7 +227,7 @@ class ToolCacheConfig: @dataclass(slots=True) class ResponseCacheConfig: - """Opt-in exact-match LLM response and tool-result cache settings. + """Opt-in LLM response and tool-result cache settings. A section of the adaptive component, not a standalone plugin kind. @@ -240,7 +240,7 @@ class ResponseCacheConfig: bypass_rate: Probability in ``[0.0, 1.0]`` of skipping the cache and running live. cache_nondeterministic: Cache nondeterministic requests too; ``False`` caches only requests explicitly pinned deterministic (``temperature`` = 0). - key_strategy: Key strategy. Only ``"exact_request"`` is supported. + key_strategy: Key strategy: ``"exact_request"`` or ``"logical"``. header_allowlist: Request headers folded into the key. backend: Cache storage backend (``in_memory`` or ``redis``). tools: Opt-in tool-result cache; ``None`` leaves the tool surface off. From 6f21d18f118324adf5aeadad5403748f00e02a78 Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Mon, 17 Aug 2026 16:19:35 -0700 Subject: [PATCH 2/9] feat(adaptive): type response cache key strategy Signed-off-by: Zhongxuan Wang --- crates/adaptive/src/config.rs | 10 +-- crates/adaptive/src/lib.rs | 2 +- crates/adaptive/src/response_cache/config.rs | 67 +++++++++++++++++-- crates/adaptive/src/response_cache/key.rs | 4 +- crates/adaptive/src/response_cache/mod.rs | 4 +- crates/adaptive/src/runtime/validation.rs | 11 ++- .../tests/integration/response_cache_tests.rs | 11 +-- crates/adaptive/tests/unit/config_tests.rs | 36 +++++++++- .../tests/unit/response_cache/key_tests.rs | 7 ++ crates/node/adaptive.d.ts | 11 ++- crates/node/adaptive.js | 8 ++- crates/node/tests/adaptive_tests.mjs | 16 ++++- go/nemo_relay/adaptive.go | 16 ++++- go/nemo_relay/adaptive_runtime_test.go | 7 ++ python/nemo_relay/adaptive.py | 14 +++- python/nemo_relay/adaptive.pyi | 11 ++- python/tests/test_adaptive_config.py | 9 +++ 17 files changed, 201 insertions(+), 43 deletions(-) diff --git a/crates/adaptive/src/config.rs b/crates/adaptive/src/config.rs index 36eb83a3d..b4a29086a 100644 --- a/crates/adaptive/src/config.rs +++ b/crates/adaptive/src/config.rs @@ -7,7 +7,7 @@ use nemo_relay::plugin::ConfigPolicy; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value as Json}; -use crate::response_cache::config::{BackendConfig, KEY_STRATEGY_EXACT_REQUEST, ToolCacheConfig}; +use crate::response_cache::config::{BackendConfig, ResponseCacheKeyStrategy, ToolCacheConfig}; /// Canonical config document for the adaptive plugin component. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -212,8 +212,8 @@ pub struct ResponseCacheConfig { /// requests explicitly pinned deterministic (`temperature` = 0) — absent /// or unreadable temperatures count as nondeterministic. pub cache_nondeterministic: bool, - /// Key strategy: `exact_request` or `logical`. - pub key_strategy: String, + /// Typed key-derivation strategy. + pub key_strategy: ResponseCacheKeyStrategy, /// Request headers (case-insensitive) folded into the key; never auth headers. pub header_allowlist: Vec, /// Storage backend selection. @@ -231,7 +231,7 @@ impl Default for ResponseCacheConfig { priority: 50, bypass_rate: 0.0, cache_nondeterministic: false, - key_strategy: KEY_STRATEGY_EXACT_REQUEST.to_string(), + key_strategy: ResponseCacheKeyStrategy::ExactRequest, header_allowlist: Vec::new(), backend: BackendConfig::default(), tools: None, @@ -402,7 +402,7 @@ nemo_relay::editor_config! { priority => { label: "priority", kind: Integer }, bypass_rate => { label: "bypass_rate", kind: Float }, cache_nondeterministic => { label: "cache_nondeterministic", kind: Boolean }, - key_strategy => { label: "key_strategy", kind: String }, + key_strategy => { label: "key_strategy", kind: Enum, values: ["exact_request", "logical"] }, header_allowlist => { label: "header_allowlist", kind: Json }, backend => { label: "backend", diff --git a/crates/adaptive/src/lib.rs b/crates/adaptive/src/lib.rs index 237dfbe8e..4ff3b35b7 100644 --- a/crates/adaptive/src/lib.rs +++ b/crates/adaptive/src/lib.rs @@ -57,8 +57,8 @@ pub use context_helpers::{ pub use error::{AdaptiveError, Result}; #[cfg(feature = "redis-backend")] pub use redis::RedisBackend; -pub use response_cache::RESPONSE_CACHE_MARK; pub use response_cache::config::{ToolCacheConfig, ToolClass, ToolOverride}; +pub use response_cache::{RESPONSE_CACHE_MARK, ResponseCacheKeyStrategy}; pub use runtime::features::AdaptiveRuntime; pub use storage::erased::AnyBackend; pub use storage::memory::InMemoryBackend; diff --git a/crates/adaptive/src/response_cache/config.rs b/crates/adaptive/src/response_cache/config.rs index 99160b22c..a4d331c5e 100644 --- a/crates/adaptive/src/response_cache/config.rs +++ b/crates/adaptive/src/response_cache/config.rs @@ -11,16 +11,69 @@ use std::collections::BTreeMap; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::{Map, Value as Json}; -/// Exact-request key strategy identifier. -pub const KEY_STRATEGY_EXACT_REQUEST: &str = "exact_request"; +/// Strategy for deriving an LLM response-cache key. +/// +/// The `Unknown` variant preserves an unsupported JSON/TOML value long enough +/// for configuration validation to report it with a field-specific diagnostic. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum ResponseCacheKeyStrategy { + /// Key on the normalized request exactly. + #[default] + ExactRequest, + /// Normalize tool schemas structurally while preserving their interface. + Logical, + /// A wire value not supported by this Relay build. + Unknown(String), +} + +impl ResponseCacheKeyStrategy { + /// Stable JSON/TOML representation of this strategy. + pub fn as_str(&self) -> &str { + match self { + Self::ExactRequest => "exact_request", + Self::Logical => "logical", + Self::Unknown(value) => value, + } + } +} + +impl From<&str> for ResponseCacheKeyStrategy { + fn from(value: &str) -> Self { + match value { + "exact_request" => Self::ExactRequest, + "logical" => Self::Logical, + _ => Self::Unknown(value.to_string()), + } + } +} -/// The "logical" key strategy: exact-match keying, but the tool set is keyed -/// on a structural, description- and order-insensitive fingerprint — so rewording -/// or reordering tools does not bust the cache; only a changed tool interface does. -pub const KEY_STRATEGY_LOGICAL: &str = "logical"; +impl From for ResponseCacheKeyStrategy { + fn from(value: String) -> Self { + Self::from(value.as_str()) + } +} + +impl Serialize for ResponseCacheKeyStrategy { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for ResponseCacheKeyStrategy { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Ok(Self::from(value)) + } +} /// Default in-memory byte budget: 256 MiB. pub const DEFAULT_MAX_BYTES: usize = 256 * 1024 * 1024; diff --git a/crates/adaptive/src/response_cache/key.rs b/crates/adaptive/src/response_cache/key.rs index d2042a4fc..645a5c112 100644 --- a/crates/adaptive/src/response_cache/key.rs +++ b/crates/adaptive/src/response_cache/key.rs @@ -25,7 +25,7 @@ use serde_json::{Map, Value as Json, json}; use sha2::{Digest, Sha256}; use crate::config::ResponseCacheConfig; -use crate::response_cache::config::KEY_STRATEGY_LOGICAL; +use crate::response_cache::config::ResponseCacheKeyStrategy; use crate::response_cache::mark::CacheReason; use crate::response_cache::store::CACHE_SCHEMA_VERSION; @@ -78,7 +78,7 @@ pub fn build_cache_key( normalize_tool_call_ids(object); } - if config.key_strategy == KEY_STRATEGY_LOGICAL + if config.key_strategy == ResponseCacheKeyStrategy::Logical && let Some(object) = body.as_object_mut() && let Some(tools) = object.get("tools").cloned() { diff --git a/crates/adaptive/src/response_cache/mod.rs b/crates/adaptive/src/response_cache/mod.rs index 086e61006..289b99916 100644 --- a/crates/adaptive/src/response_cache/mod.rs +++ b/crates/adaptive/src/response_cache/mod.rs @@ -23,9 +23,7 @@ pub mod store; pub(crate) mod tool; pub use crate::config::ResponseCacheConfig; -pub use crate::response_cache::config::{ - BackendConfig, KEY_STRATEGY_EXACT_REQUEST, KEY_STRATEGY_LOGICAL, ToolCacheConfig, -}; +pub use crate::response_cache::config::{BackendConfig, ResponseCacheKeyStrategy, ToolCacheConfig}; pub(crate) use crate::response_cache::intercept::{make_intercept, make_stream_intercept}; pub use crate::response_cache::mark::RESPONSE_CACHE_MARK; pub(crate) use crate::response_cache::store::build_store; diff --git a/crates/adaptive/src/runtime/validation.rs b/crates/adaptive/src/runtime/validation.rs index 035a141f9..2ad418944 100644 --- a/crates/adaptive/src/runtime/validation.rs +++ b/crates/adaptive/src/runtime/validation.rs @@ -9,9 +9,7 @@ use nemo_relay::plugin::{ use serde_json::Value as Json; use crate::config::{AdaptiveConfig, BackendSpec, ResponseCacheConfig}; -use crate::response_cache::config::{ - KEY_STRATEGY_EXACT_REQUEST, KEY_STRATEGY_LOGICAL, ToolCacheConfig, -}; +use crate::response_cache::config::{ResponseCacheKeyStrategy, ToolCacheConfig}; use crate::response_cache::tool::{is_supported_tool_pattern, wildcard_patterns_overlap}; pub fn validate_config(config: &AdaptiveConfig) -> ConfigReport { @@ -125,14 +123,13 @@ fn validate_response_cache(report: &mut ConfigReport, config: &ResponseCacheConf "bypass_rate must be in [0.0, 1.0]".to_string(), )); } - if ![KEY_STRATEGY_EXACT_REQUEST, KEY_STRATEGY_LOGICAL].contains(&config.key_strategy.as_str()) { + if matches!(config.key_strategy, ResponseCacheKeyStrategy::Unknown(_)) { report.diagnostics.push(response_cache_error( "response_cache.unsupported_key_strategy", Some("key_strategy"), format!( - "unsupported key_strategy '{}'; supported: \"{KEY_STRATEGY_EXACT_REQUEST}\", \ - \"{KEY_STRATEGY_LOGICAL}\"", - config.key_strategy + "unsupported key_strategy '{}'; supported: \"exact_request\", \"logical\"", + config.key_strategy.as_str() ), )); } diff --git a/crates/adaptive/tests/integration/response_cache_tests.rs b/crates/adaptive/tests/integration/response_cache_tests.rs index fe2b80b58..74656a275 100644 --- a/crates/adaptive/tests/integration/response_cache_tests.rs +++ b/crates/adaptive/tests/integration/response_cache_tests.rs @@ -33,8 +33,8 @@ use nemo_relay::plugin::{ }; use nemo_relay_adaptive::plugin_component::{ComponentSpec, register_adaptive_component}; use nemo_relay_adaptive::{ - AcgComponentConfig, AdaptiveConfig, BackendSpec, ResponseCacheConfig, StateConfig, - ToolCacheConfig, ToolClass, ToolOverride, + AcgComponentConfig, AdaptiveConfig, BackendSpec, ResponseCacheConfig, ResponseCacheKeyStrategy, + StateConfig, ToolCacheConfig, ToolClass, ToolOverride, }; use serde_json::{Value as Json, json}; use tokio::sync::Mutex; @@ -555,7 +555,7 @@ async fn invalid_config_is_rejected_by_validation() { response_cache: Some(ResponseCacheConfig { ttl_seconds: 0, bypass_rate: 2.0, - key_strategy: "semantic".to_string(), + key_strategy: ResponseCacheKeyStrategy::Unknown("semantic".to_string()), namespace: "invalid-config-test".to_string(), ..ResponseCacheConfig::default() }), @@ -647,7 +647,7 @@ async fn response_cache_validation_diagnostics_identify_the_invalid_setting() { let mut cache = ResponseCacheConfig { namespace: "diagnostic-contract-test".to_string(), - key_strategy: "semantic".to_string(), + key_strategy: ResponseCacheKeyStrategy::Unknown("semantic".to_string()), tools: Some(ToolCacheConfig { enabled: true, default: ToolClass { @@ -799,7 +799,8 @@ async fn logical_strategy_reuses_across_reworded_tool_descriptions() { // `logical` must be accepted by validation (activate_cache asserts no // diagnostics) and must reuse across a reworded tool description end-to-end. activate_cache(ResponseCacheConfig { - key_strategy: "logical".to_string(), + namespace: "logical-key-integration-test".to_string(), + key_strategy: ResponseCacheKeyStrategy::Logical, ..ResponseCacheConfig::default() }) .await; diff --git a/crates/adaptive/tests/unit/config_tests.rs b/crates/adaptive/tests/unit/config_tests.rs index 0dea14ecf..d72a3d24a 100644 --- a/crates/adaptive/tests/unit/config_tests.rs +++ b/crates/adaptive/tests/unit/config_tests.rs @@ -7,7 +7,7 @@ use super::*; use nemo_relay::config_editor::{EditorConfig, EditorFieldKind}; use serde_json::json; -use crate::response_cache::config::{ToolCacheConfig, ToolClass}; +use crate::response_cache::config::{ResponseCacheKeyStrategy, ToolCacheConfig, ToolClass}; #[test] fn test_adaptive_config_defaults() { @@ -34,6 +34,10 @@ fn test_typed_section_helpers_default() { let response_cache = ResponseCacheConfig::default(); assert!(!response_cache.cache_nondeterministic); + assert_eq!( + response_cache.key_strategy, + ResponseCacheKeyStrategy::ExactRequest + ); let tools = ToolCacheConfig::default(); assert!(!tools.enabled); @@ -52,6 +56,28 @@ fn test_tool_cache_deserializes_explicit_error_caching_opt_in() { assert!(tools.cache_errors); } +#[test] +fn test_response_cache_key_strategy_roundtrips_and_preserves_unknown() { + let logical: ResponseCacheConfig = serde_json::from_value(json!({ + "key_strategy": "logical" + })) + .unwrap(); + assert_eq!(logical.key_strategy, ResponseCacheKeyStrategy::Logical); + assert_eq!( + serde_json::to_value(logical).unwrap()["key_strategy"], + json!("logical") + ); + + let unsupported: ResponseCacheConfig = serde_json::from_value(json!({ + "key_strategy": "future" + })) + .unwrap(); + assert_eq!( + unsupported.key_strategy, + ResponseCacheKeyStrategy::Unknown("future".to_string()) + ); +} + #[test] fn test_backend_spec_in_memory_helper_uses_empty_config() { let backend = BackendSpec::in_memory(); @@ -184,6 +210,14 @@ fn test_adaptive_editor_schema_covers_canonical_options() { response_cache.field("bypass_rate").unwrap().kind, EditorFieldKind::Float ); + assert_eq!( + response_cache.field("key_strategy").unwrap().kind, + EditorFieldKind::Enum + ); + assert_eq!( + response_cache.field("key_strategy").unwrap().enum_values, + &["exact_request", "logical"] + ); assert!( response_cache.field("skip_keys").is_none(), "exact-match cache config must not expose arbitrary key omission" diff --git a/crates/adaptive/tests/unit/response_cache/key_tests.rs b/crates/adaptive/tests/unit/response_cache/key_tests.rs index b7b5a47b5..2febd94ad 100644 --- a/crates/adaptive/tests/unit/response_cache/key_tests.rs +++ b/crates/adaptive/tests/unit/response_cache/key_tests.rs @@ -993,6 +993,13 @@ fn tool_key_with_error_policy( } } +fn logical_config() -> ResponseCacheConfig { + ResponseCacheConfig { + key_strategy: ResponseCacheKeyStrategy::Logical, + ..cache_all_config() + } +} + #[test] fn same_tool_and_args_yield_the_same_key() { let args = json!({"q": "weather", "units": "metric"}); diff --git a/crates/node/adaptive.d.ts b/crates/node/adaptive.d.ts index ea1a66584..fa24a0f0d 100644 --- a/crates/node/adaptive.d.ts +++ b/crates/node/adaptive.d.ts @@ -6,6 +6,13 @@ import type { ConfigPolicy, ConfigDiagnostic, ConfigReport } from './plugin'; export { ConfigPolicy, ConfigDiagnostic, ConfigReport }; +/** Supported LLM response-cache key derivation strategies. */ +export const ResponseCacheKeyStrategy: { + readonly ExactRequest: 'exact_request'; + readonly Logical: 'logical'; +}; +export type ResponseCacheKeyStrategy = (typeof ResponseCacheKeyStrategy)[keyof typeof ResponseCacheKeyStrategy]; + /** Adaptive state backend selection. */ export interface BackendSpec { kind: string; @@ -63,7 +70,7 @@ export interface ResponseCacheConfig { priority?: number; bypassRate?: number; cacheNondeterministic?: boolean; - keyStrategy?: string; + keyStrategy?: ResponseCacheKeyStrategy; headerAllowlist?: string[]; backend?: BackendSpec; /** Opt-in tool-result cache; omit to leave the tool surface off. */ @@ -77,7 +84,7 @@ interface ResponseCachePluginConfig { priority?: number; bypass_rate?: number; cache_nondeterministic?: boolean; - key_strategy?: string; + key_strategy?: ResponseCacheKeyStrategy; header_allowlist?: string[]; backend?: BackendSpec; tools?: ToolCachePluginConfig; diff --git a/crates/node/adaptive.js b/crates/node/adaptive.js index 24902f491..fc1af15dd 100644 --- a/crates/node/adaptive.js +++ b/crates/node/adaptive.js @@ -8,6 +8,11 @@ const plugin = require('./plugin.js'); const ADAPTIVE_PLUGIN_KIND = 'adaptive'; +const ResponseCacheKeyStrategy = Object.freeze({ + ExactRequest: 'exact_request', + Logical: 'logical', +}); + /** * Create a default adaptive component config. * @@ -170,7 +175,7 @@ function responseCacheConfig(config = {}) { priority: 50, bypassRate: 0, cacheNondeterministic: false, - keyStrategy: 'exact_request', + keyStrategy: ResponseCacheKeyStrategy.ExactRequest, headerAllowlist: [], backend: backend ?? inMemoryBackend(), ...rest, @@ -297,6 +302,7 @@ function setLatencySensitivity(value) { module.exports = { AdaptiveRuntime, ADAPTIVE_PLUGIN_KIND, + ResponseCacheKeyStrategy, defaultConfig, inMemoryBackend, redisBackend, diff --git a/crates/node/tests/adaptive_tests.mjs b/crates/node/tests/adaptive_tests.mjs index f81f0cfcf..661c3d0d0 100644 --- a/crates/node/tests/adaptive_tests.mjs +++ b/crates/node/tests/adaptive_tests.mjs @@ -392,7 +392,7 @@ describe('adaptive helpers', () => { priority: 50, bypassRate: 0, cacheNondeterministic: false, - keyStrategy: 'exact_request', + keyStrategy: adaptive.ResponseCacheKeyStrategy.ExactRequest, headerAllowlist: [], backend: adaptive.inMemoryBackend(), }); @@ -433,6 +433,20 @@ describe('adaptive helpers', () => { }); }); + it('exports and serializes response-cache key strategy values', () => { + assert.deepEqual(adaptive.ResponseCacheKeyStrategy, { + ExactRequest: 'exact_request', + Logical: 'logical', + }); + const spec = adaptive.ComponentSpec({ + version: 1, + responseCache: { + keyStrategy: adaptive.ResponseCacheKeyStrategy.Logical, + }, + }); + assert.equal(spec.config.response_cache.key_strategy, 'logical'); + }); + it('serializes response-cache config at both native boundaries', () => { const unscoped = adaptive.validateConfig({ version: 1, responseCache: {} }); assert.ok(unscoped.diagnostics.some(({ code }) => code === 'response_cache.missing_namespace')); diff --git a/go/nemo_relay/adaptive.go b/go/nemo_relay/adaptive.go index 374392a50..3afcd07b6 100644 --- a/go/nemo_relay/adaptive.go +++ b/go/nemo_relay/adaptive.go @@ -67,6 +67,16 @@ type AcgConfig struct { StabilityThresholds *AcgStabilityThresholds `json:"stability_thresholds,omitempty"` } +// ResponseCacheKeyStrategy selects how LLM response-cache keys are derived. +type ResponseCacheKeyStrategy string + +const ( + // ResponseCacheKeyStrategyExactRequest keys on the normalized request exactly. + ResponseCacheKeyStrategyExactRequest ResponseCacheKeyStrategy = "exact_request" + // ResponseCacheKeyStrategyLogical ignores tool-description wording while preserving interfaces. + ResponseCacheKeyStrategyLogical ResponseCacheKeyStrategy = "logical" +) + // ResponseCacheConfig configures the opt-in LLM response and tool-result cache: a section // of the adaptive config (a sibling to acg/adaptive_hints/tool_parallelism), not a // standalone plugin kind. The Rust core validates and installs it from the adaptive @@ -87,8 +97,8 @@ type ResponseCacheConfig struct { // CacheNondeterministic lets requests that are not explicitly deterministic // use the cache (default false). CacheNondeterministic bool `json:"cache_nondeterministic"` - // KeyStrategy is the key strategy. Only "exact_request" is supported. - KeyStrategy string `json:"key_strategy,omitempty"` + // KeyStrategy is the typed LLM response-cache key derivation strategy. + KeyStrategy ResponseCacheKeyStrategy `json:"key_strategy,omitempty"` // HeaderAllowlist lists request headers folded into the key; never auth headers. HeaderAllowlist []string `json:"header_allowlist,omitempty"` // Backend selects the cache's own storage backend (distinct from the adaptive @@ -225,7 +235,7 @@ func NewResponseCacheConfig() ResponseCacheConfig { TTLSeconds: &ttlSeconds, Priority: &priority, CacheNondeterministic: false, - KeyStrategy: "exact_request", + KeyStrategy: ResponseCacheKeyStrategyExactRequest, } } diff --git a/go/nemo_relay/adaptive_runtime_test.go b/go/nemo_relay/adaptive_runtime_test.go index c0d7dcd59..d6d32a1d0 100644 --- a/go/nemo_relay/adaptive_runtime_test.go +++ b/go/nemo_relay/adaptive_runtime_test.go @@ -182,6 +182,7 @@ func TestResponseCacheConfigReachesTypedSurface(t *testing.T) { assertResponseCacheConstructorDefaults(t, rc) rc.Namespace = responseCacheTestNamespace rc.CacheNondeterministic = true + rc.KeyStrategy = ResponseCacheKeyStrategyLogical rc.Backend = &backend assertResponseCacheJSONSurface(t, rc) assertResponseCacheValidation(t, rc) @@ -195,6 +196,9 @@ func assertResponseCacheConstructorDefaults(t *testing.T, config ResponseCacheCo if config.Priority == nil || *config.Priority != 50 { t.Fatalf("constructor priority default mismatch: %#v", config.Priority) } + if config.KeyStrategy != ResponseCacheKeyStrategyExactRequest { + t.Fatalf("constructor key strategy default mismatch: %#v", config.KeyStrategy) + } } func assertResponseCacheJSONSurface(t *testing.T, responseCache ResponseCacheConfig) { @@ -222,6 +226,9 @@ func assertResponseCacheJSONSurface(t *testing.T, responseCache ResponseCacheCon if v, ok := section["cache_nondeterministic"].(bool); !ok || !v { t.Fatalf("explicit cache_nondeterministic=true was not preserved: %#v", section["cache_nondeterministic"]) } + if section["key_strategy"] != string(ResponseCacheKeyStrategyLogical) { + t.Fatalf("logical key strategy was not preserved: %#v", section["key_strategy"]) + } if b, ok := section["backend"].(map[string]any); !ok || b["kind"] != "in_memory" { t.Fatalf("backend not preserved: %#v", section["backend"]) } diff --git a/python/nemo_relay/adaptive.py b/python/nemo_relay/adaptive.py index 367d42cb3..db20ec432 100644 --- a/python/nemo_relay/adaptive.py +++ b/python/nemo_relay/adaptive.py @@ -10,6 +10,7 @@ from __future__ import annotations from dataclasses import dataclass, field, fields, is_dataclass +from enum import Enum from typing import Literal, Protocol, TypedDict, cast from nemo_relay import Json, JsonObject, UnsupportedBehavior @@ -38,6 +39,13 @@ class ConfigReport(TypedDict): diagnostics: list[ConfigDiagnostic] +class ResponseCacheKeyStrategy(str, Enum): + """Supported LLM response-cache key derivation strategies.""" + + EXACT_REQUEST = "exact_request" + LOGICAL = "logical" + + class _SupportsToDict(Protocol): def to_dict(self) -> JsonObject: ... @@ -386,7 +394,7 @@ class ResponseCacheConfig: bypass_rate: Probability in ``[0.0, 1.0]`` of skipping the cache and running live. cache_nondeterministic: Cache nondeterministic requests too; ``False`` caches only requests explicitly pinned deterministic (``temperature`` = 0). - key_strategy: Key strategy: ``"exact_request"`` or ``"logical"``. + key_strategy: Typed key derivation strategy. header_allowlist: Request headers folded into the key; never auth headers. backend: Cache storage backend (``in_memory`` or ``redis``). tools: Opt-in tool-result cache; ``None`` leaves it off. @@ -397,7 +405,7 @@ class ResponseCacheConfig: priority: int = 50 bypass_rate: float = 0.0 cache_nondeterministic: bool = False - key_strategy: str = "exact_request" + key_strategy: ResponseCacheKeyStrategy = ResponseCacheKeyStrategy.EXACT_REQUEST header_allowlist: list[str] = field(default_factory=list) backend: BackendSpec = field(default_factory=BackendSpec.in_memory) tools: ToolCacheConfig | None = None @@ -411,7 +419,7 @@ def to_dict(self) -> JsonObject: "priority": self.priority, "bypass_rate": self.bypass_rate, "cache_nondeterministic": self.cache_nondeterministic, - "key_strategy": self.key_strategy, + "key_strategy": self.key_strategy.value, "header_allowlist": self.header_allowlist, "backend": _normalize(self.backend), "tools": _normalize(self.tools), diff --git a/python/nemo_relay/adaptive.pyi b/python/nemo_relay/adaptive.pyi index 1709e443b..80510a916 100644 --- a/python/nemo_relay/adaptive.pyi +++ b/python/nemo_relay/adaptive.pyi @@ -9,6 +9,7 @@ helpers that summarize ACG observations into structured JSON payloads. """ from dataclasses import dataclass +from enum import Enum from typing import Literal, TypedDict from nemo_relay import JsonObject, ScopeHandle, UnsupportedBehavior @@ -31,6 +32,12 @@ class ConfigReport(TypedDict): diagnostics: list[ConfigDiagnostic] +class ResponseCacheKeyStrategy(str, Enum): + """Supported LLM response-cache key derivation strategies.""" + + EXACT_REQUEST: str + LOGICAL: str + @dataclass(slots=True) class ConfigPolicy: """Policy for unsupported adaptive configuration. @@ -240,7 +247,7 @@ class ResponseCacheConfig: bypass_rate: Probability in ``[0.0, 1.0]`` of skipping the cache and running live. cache_nondeterministic: Cache nondeterministic requests too; ``False`` caches only requests explicitly pinned deterministic (``temperature`` = 0). - key_strategy: Key strategy: ``"exact_request"`` or ``"logical"``. + key_strategy: Typed key derivation strategy. header_allowlist: Request headers folded into the key. backend: Cache storage backend (``in_memory`` or ``redis``). tools: Opt-in tool-result cache; ``None`` leaves the tool surface off. @@ -251,7 +258,7 @@ class ResponseCacheConfig: priority: int = ... bypass_rate: float = ... cache_nondeterministic: bool = ... - key_strategy: str = ... + key_strategy: ResponseCacheKeyStrategy = ... header_allowlist: list[str] = ... backend: BackendSpec = ... tools: ToolCacheConfig | None = ... diff --git a/python/tests/test_adaptive_config.py b/python/tests/test_adaptive_config.py index 7373c95e9..77c1fc8f7 100644 --- a/python/tests/test_adaptive_config.py +++ b/python/tests/test_adaptive_config.py @@ -16,6 +16,7 @@ ComponentSpec, ConfigPolicy, ResponseCacheConfig, + ResponseCacheKeyStrategy, StateConfig, TelemetryConfig, ToolCacheConfig, @@ -172,6 +173,14 @@ def test_response_cache_config_serializes_with_defaults(self): "backend": {"kind": "in_memory", "config": {}}, } + def test_response_cache_key_strategy_enum_serializes(self): + config = ResponseCacheConfig( + namespace="logical-cache", + key_strategy=ResponseCacheKeyStrategy.LOGICAL, + ) + + assert config.to_dict()["key_strategy"] == "logical" + def test_response_cache_default_preserves_positional_policy_argument(self): policy = ConfigPolicy(unknown_field="error") config = AdaptiveConfig(1, None, None, None, None, None, None, policy) From e6db2984c635d2ccf513521bb3433ac83ac6951e Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 20 Aug 2026 08:29:47 -0700 Subject: [PATCH 3/9] test(response-cache): cover logical key behavior Signed-off-by: Zhongxuan Wang --- .../tests/unit/response_cache/key_tests.rs | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/crates/adaptive/tests/unit/response_cache/key_tests.rs b/crates/adaptive/tests/unit/response_cache/key_tests.rs index 2febd94ad..edabad6c1 100644 --- a/crates/adaptive/tests/unit/response_cache/key_tests.rs +++ b/crates/adaptive/tests/unit/response_cache/key_tests.rs @@ -1304,3 +1304,151 @@ fn decode_round_trip_guards_fall_back_to_raw_tool_and_message_shapes() { (legacy_message_request.content.clone(), None) ); } + +// --- `logical` key strategy (structural tool-schema hash) --------- + +fn tool(name: &str, description: &str, param: &str, param_type: &str) -> Json { + json!({ + "type": "function", + "function": { + "name": name, + "description": description, + "parameters": { + "type": "object", + "properties": {param: {"type": param_type, "description": "a param"}} + } + } + }) +} + +#[test] +fn logical_ignores_tool_description_and_order() { + let a = request(json!({ + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "tools": [tool("get_weather", "Get the weather.", "city", "string"), + tool("get_time", "Get the time.", "tz", "string")] + })); + let b = request(json!({ + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "tools": [tool("get_time", "Return the current time.", "tz", "string"), + tool("get_weather", "Look up weather, reworded.", "city", "string")] + })); + assert_eq!( + key_of("openai", &a, &logical_config()), + key_of("openai", &b, &logical_config()), + "logical keying must ignore tool description text and tool order" + ); + assert_ne!( + key_of("openai", &a, &cache_all_config()), + key_of("openai", &b, &cache_all_config()), + "exact_request must not collapse reworded/reordered tools" + ); +} + +#[test] +fn structural_tool_schema_sorts_on_canonical_bytes() { + // RFC 8785 formats 1.0 and 1 identically, so two JCS-identical tool + // sets must sort (and therefore hash) identically; a serde_json + // Display sort key would order them differently around 1.5. + let a = json!([{"x": 1.0}, {"x": 1.5}]); + let b = json!([{"x": 1.5}, {"x": 1}]); + assert_eq!( + fingerprint(&structural_tool_schema(&a)), + fingerprint(&structural_tool_schema(&b)), + "JCS-identical tool sets must produce one key regardless of number formatting" + ); +} + +#[test] +fn logical_differs_on_changed_tool_interface() { + let cfg = logical_config(); + let base = request(json!({ + "model": "m", "messages": [{"role": "user", "content": "hi"}], + "tools": [tool("get_weather", "d", "city", "string")] + })); + let renamed = request(json!({ + "model": "m", "messages": [{"role": "user", "content": "hi"}], + "tools": [tool("get_weather", "d", "location", "string")] + })); + let retyped = request(json!({ + "model": "m", "messages": [{"role": "user", "content": "hi"}], + "tools": [tool("get_weather", "d", "city", "number")] + })); + assert_ne!( + key_of("openai", &base, &cfg), + key_of("openai", &renamed, &cfg), + "a renamed parameter must change the key" + ); + assert_ne!( + key_of("openai", &base, &cfg), + key_of("openai", &retyped, &cfg), + "a changed parameter type must change the key" + ); +} + +#[test] +fn logical_differs_on_distinct_builtin_tools() { + let cfg = logical_config(); + let with_builtin = |tool: Json| { + request(json!({ + "model": "gpt-4o", + "input": "search the docs", + "store": false, + "tools": [tool] + })) + }; + assert_ne!( + key_of( + "openai", + &with_builtin(json!({"type": "web_search_preview"})), + &cfg + ), + key_of( + "openai", + &with_builtin(json!({"type": "code_interpreter", "container": {"type": "auto"}})), + &cfg, + ), + "tools without a function schema must keep their definitions in the key" + ); +} + +#[test] +fn logical_differs_on_changed_parameter_enum() { + let cfg = logical_config(); + let with_units = |units: Json| { + request(json!({ + "model": "m", "messages": [{"role": "user", "content": "hi"}], + "tools": [{"type": "function", "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": { + "unit": {"type": "string", "enum": units} + }} + }}] + })) + }; + assert_ne!( + key_of( + "openai", + &with_units(json!(["celsius", "fahrenheit"])), + &cfg + ), + key_of("openai", &with_units(json!(["kelvin"])), &cfg), + "a changed parameter enum must change the key" + ); +} + +#[test] +fn logical_and_exact_do_not_collide() { + // Tool-less, so both strategies key the identical body and only the + // strategy field in the key document separates them. + let req = request(json!({ + "model": "m", "messages": [{"role": "user", "content": "hi"}] + })); + assert_ne!( + key_of("openai", &req, &logical_config()), + key_of("openai", &req, &cache_all_config()), + "logical and exact_request must not share keys (strategy is folded in)" + ); +} From ee0e1eff72e062a7a01dbf985fb0b5294141ef9e Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 20 Aug 2026 08:32:28 -0700 Subject: [PATCH 4/9] feat(python): export response cache key strategy Signed-off-by: Zhongxuan Wang --- python/nemo_relay/adaptive.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/nemo_relay/adaptive.py b/python/nemo_relay/adaptive.py index db20ec432..d8ff3e064 100644 --- a/python/nemo_relay/adaptive.py +++ b/python/nemo_relay/adaptive.py @@ -565,6 +565,7 @@ def set_latency_sensitivity(level: int) -> None: "ConfigReport", "ComponentSpec", "ResponseCacheConfig", + "ResponseCacheKeyStrategy", "StateConfig", "TelemetryConfig", "ToolCacheConfig", From 23caa315b112024bbb802f21347fb38b1821cd1c Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 20 Aug 2026 08:40:47 -0700 Subject: [PATCH 5/9] test(adaptive): keep key strategy assertions focused Signed-off-by: Zhongxuan Wang --- crates/adaptive/tests/unit/config_tests.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/adaptive/tests/unit/config_tests.rs b/crates/adaptive/tests/unit/config_tests.rs index d72a3d24a..1436e20b3 100644 --- a/crates/adaptive/tests/unit/config_tests.rs +++ b/crates/adaptive/tests/unit/config_tests.rs @@ -210,14 +210,6 @@ fn test_adaptive_editor_schema_covers_canonical_options() { response_cache.field("bypass_rate").unwrap().kind, EditorFieldKind::Float ); - assert_eq!( - response_cache.field("key_strategy").unwrap().kind, - EditorFieldKind::Enum - ); - assert_eq!( - response_cache.field("key_strategy").unwrap().enum_values, - &["exact_request", "logical"] - ); assert!( response_cache.field("skip_keys").is_none(), "exact-match cache config must not expose arbitrary key omission" @@ -256,6 +248,16 @@ fn test_adaptive_editor_schema_covers_canonical_options() { ); } +#[test] +fn test_response_cache_key_strategy_editor_field_is_typed() { + let schema = AdaptiveConfig::editor_schema(); + let response_cache = schema.field("response_cache").unwrap().schema().unwrap(); + let key_strategy = response_cache.field("key_strategy").unwrap(); + + assert_eq!(key_strategy.kind, EditorFieldKind::Enum); + assert_eq!(key_strategy.enum_values, &["exact_request", "logical"]); +} + #[test] fn tool_class_editor_schema_exposes_optional_version() { let tool_class = ToolClass::editor_schema(); From 46238bc69229013962b355eef22ac805e7c8d950 Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 20 Aug 2026 08:48:14 -0700 Subject: [PATCH 6/9] fix(python): type response cache strategy enum members Signed-off-by: Zhongxuan Wang --- python/nemo_relay/adaptive.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/nemo_relay/adaptive.pyi b/python/nemo_relay/adaptive.pyi index 80510a916..ef7c02789 100644 --- a/python/nemo_relay/adaptive.pyi +++ b/python/nemo_relay/adaptive.pyi @@ -35,8 +35,8 @@ class ConfigReport(TypedDict): class ResponseCacheKeyStrategy(str, Enum): """Supported LLM response-cache key derivation strategies.""" - EXACT_REQUEST: str - LOGICAL: str + EXACT_REQUEST: ResponseCacheKeyStrategy + LOGICAL: ResponseCacheKeyStrategy @dataclass(slots=True) class ConfigPolicy: From a0f2b2b4407e20d131a72246f849f4918fcb3440 Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Thu, 20 Aug 2026 09:23:04 -0700 Subject: [PATCH 7/9] test(response-cache): use Nemotron logical key fixtures Signed-off-by: Zhongxuan Wang --- .../tests/integration/response_cache_tests.rs | 2 +- .../tests/unit/response_cache/key_tests.rs | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/adaptive/tests/integration/response_cache_tests.rs b/crates/adaptive/tests/integration/response_cache_tests.rs index 74656a275..5e9c406cc 100644 --- a/crates/adaptive/tests/integration/response_cache_tests.rs +++ b/crates/adaptive/tests/integration/response_cache_tests.rs @@ -811,7 +811,7 @@ async fn logical_strategy_reuses_across_reworded_tool_descriptions() { let request_with_tool = |description: &str| LlmRequest { headers: serde_json::Map::new(), content: json!({ - "model": "gpt-4o", + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", "messages": [{"role": "user", "content": "what is the weather?"}], "temperature": 0.0, "tools": [{"type": "function", "function": { diff --git a/crates/adaptive/tests/unit/response_cache/key_tests.rs b/crates/adaptive/tests/unit/response_cache/key_tests.rs index edabad6c1..d4c0e4994 100644 --- a/crates/adaptive/tests/unit/response_cache/key_tests.rs +++ b/crates/adaptive/tests/unit/response_cache/key_tests.rs @@ -1307,6 +1307,8 @@ fn decode_round_trip_guards_fall_back_to_raw_tool_and_message_shapes() { // --- `logical` key strategy (structural tool-schema hash) --------- +const LOGICAL_KEY_MODEL: &str = "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning"; + fn tool(name: &str, description: &str, param: &str, param_type: &str) -> Json { json!({ "type": "function", @@ -1324,13 +1326,13 @@ fn tool(name: &str, description: &str, param: &str, param_type: &str) -> Json { #[test] fn logical_ignores_tool_description_and_order() { let a = request(json!({ - "model": "m", + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}], "tools": [tool("get_weather", "Get the weather.", "city", "string"), tool("get_time", "Get the time.", "tz", "string")] })); let b = request(json!({ - "model": "m", + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}], "tools": [tool("get_time", "Return the current time.", "tz", "string"), tool("get_weather", "Look up weather, reworded.", "city", "string")] @@ -1365,15 +1367,15 @@ fn structural_tool_schema_sorts_on_canonical_bytes() { fn logical_differs_on_changed_tool_interface() { let cfg = logical_config(); let base = request(json!({ - "model": "m", "messages": [{"role": "user", "content": "hi"}], + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}], "tools": [tool("get_weather", "d", "city", "string")] })); let renamed = request(json!({ - "model": "m", "messages": [{"role": "user", "content": "hi"}], + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}], "tools": [tool("get_weather", "d", "location", "string")] })); let retyped = request(json!({ - "model": "m", "messages": [{"role": "user", "content": "hi"}], + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}], "tools": [tool("get_weather", "d", "city", "number")] })); assert_ne!( @@ -1393,7 +1395,7 @@ fn logical_differs_on_distinct_builtin_tools() { let cfg = logical_config(); let with_builtin = |tool: Json| { request(json!({ - "model": "gpt-4o", + "model": LOGICAL_KEY_MODEL, "input": "search the docs", "store": false, "tools": [tool] @@ -1419,7 +1421,7 @@ fn logical_differs_on_changed_parameter_enum() { let cfg = logical_config(); let with_units = |units: Json| { request(json!({ - "model": "m", "messages": [{"role": "user", "content": "hi"}], + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}], "tools": [{"type": "function", "function": { "name": "get_weather", "parameters": {"type": "object", "properties": { @@ -1444,7 +1446,7 @@ fn logical_and_exact_do_not_collide() { // Tool-less, so both strategies key the identical body and only the // strategy field in the key document separates them. let req = request(json!({ - "model": "m", "messages": [{"role": "user", "content": "hi"}] + "model": LOGICAL_KEY_MODEL, "messages": [{"role": "user", "content": "hi"}] })); assert_ne!( key_of("openai", &req, &logical_config()), From 6c08e10ec9e02776499e7300363656d4a1c83dd1 Mon Sep 17 00:00:00 2001 From: Zhongxuan Wang Date: Wed, 26 Aug 2026 15:13:40 -0400 Subject: [PATCH 8/9] fix(adaptive): drop stale KEY_STRATEGY_EXACT_REQUEST import The main merge kept the constant in validation.rs's import list after this branch replaced it with the ResponseCacheKeyStrategy enum, so nemo-relay-adaptive failed to compile (E0432) and every CI job that builds the workspace failed. Signed-off-by: Zhongxuan Wang --- crates/adaptive/src/runtime/validation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/adaptive/src/runtime/validation.rs b/crates/adaptive/src/runtime/validation.rs index b91fade14..6d0e2a34f 100644 --- a/crates/adaptive/src/runtime/validation.rs +++ b/crates/adaptive/src/runtime/validation.rs @@ -9,7 +9,7 @@ use nemo_relay::plugin::{ use serde_json::Value as Json; use crate::config::{AdaptiveConfig, BackendSpec, ResponseCacheConfig}; -use crate::response_cache::config::{KEY_STRATEGY_EXACT_REQUEST, ResponseCacheKeyStrategy, ToolCacheConfig, ToolClass}; +use crate::response_cache::config::{ResponseCacheKeyStrategy, ToolCacheConfig, ToolClass}; use crate::response_cache::tool::{is_supported_tool_pattern, wildcard_patterns_overlap}; pub fn validate_config(config: &AdaptiveConfig) -> ConfigReport { From 1c9765af324230bb644c8548fe8c05244a7841da Mon Sep 17 00:00:00 2001 From: "Zhongxuan (Daniel) Wang" <52872691+ZhongxuanWang@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:26:12 -0400 Subject: [PATCH 9/9] fix(adaptive): keep string key_strategy inputs serializable Address CodeRabbit review on #818. - `ResponseCacheConfig.to_dict()` in Python no longer raises `AttributeError` when `key_strategy` is a plain wire string. Enum members serialize through `.value`; strings pass through unchanged so unsupported values reach native validation with the `response_cache.unsupported_key_strategy` diagnostic, matching the Go string alias and the Node.js runtime. - Drop the stale "exact-match" wording from the two `response_cache` doc comments in `crates/adaptive/src/config.rs`. User-facing docs stay deferred to #819. Signed-off-by: Zhongxuan (Daniel) Wang <52872691+ZhongxuanWang@users.noreply.github.com> --- crates/adaptive/src/config.rs | 4 ++-- python/nemo_relay/adaptive.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/adaptive/src/config.rs b/crates/adaptive/src/config.rs index b4a29086a..ad7a83874 100644 --- a/crates/adaptive/src/config.rs +++ b/crates/adaptive/src/config.rs @@ -34,7 +34,7 @@ pub struct AdaptiveConfig { /// Adaptive Cache Governor settings. #[serde(default, skip_serializing_if = "Option::is_none")] pub acg: Option, - /// Opt-in exact-match LLM response and tool-result cache. When present, + /// Opt-in LLM response and tool-result cache. When present, /// the adaptive plugin installs the response-cache execution intercept(s). #[serde(default, skip_serializing_if = "Option::is_none")] pub response_cache: Option, @@ -191,7 +191,7 @@ impl Default for AcgComponentConfig { } } -/// Configuration for the adaptive plugin's exact-match LLM response and +/// Configuration for the adaptive plugin's LLM response and /// opt-in tool-result cache feature. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] diff --git a/python/nemo_relay/adaptive.py b/python/nemo_relay/adaptive.py index d8ff3e064..fac6326da 100644 --- a/python/nemo_relay/adaptive.py +++ b/python/nemo_relay/adaptive.py @@ -419,7 +419,11 @@ def to_dict(self) -> JsonObject: "priority": self.priority, "bypass_rate": self.bypass_rate, "cache_nondeterministic": self.cache_nondeterministic, - "key_strategy": self.key_strategy.value, + "key_strategy": ( + self.key_strategy.value + if isinstance(self.key_strategy, ResponseCacheKeyStrategy) + else self.key_strategy + ), "header_allowlist": self.header_allowlist, "backend": _normalize(self.backend), "tools": _normalize(self.tools),