Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 137 additions & 0 deletions src/config/llm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ impl LlmConfig {
request_timeout_secs: 120,
cheap_model: None,
smart_routing_cascade: false,
max_retries: 0,
circuit_breaker_threshold: None,
circuit_breaker_recovery_secs: 30,
response_cache_enabled: false,
response_cache_ttl_secs: 3600,
response_cache_max_entries: 100,
}
}

Expand Down Expand Up @@ -301,6 +307,62 @@ impl LlmConfig {
// Defaults to true. Overrides NearAI-specific smart_routing_cascade.
let smart_routing_cascade = parse_optional_env("SMART_ROUTING_CASCADE", true)?;

// Decorator chain settings — top-level `LLM_*` vars with fallback to
// existing backend-specific vars for backward compatibility.
let max_retries = optional_env("LLM_MAX_RETRIES")?
.map(|s| s.parse::<u32>())
.transpose()
.map_err(|e| ConfigError::InvalidValue {
key: "LLM_MAX_RETRIES".to_string(),
message: format!("must be a non-negative integer: {e}"),
})?
.unwrap_or(nearai.max_retries);

let circuit_breaker_threshold = optional_env("LLM_CIRCUIT_BREAKER_THRESHOLD")?
.map(|s| s.parse::<u32>())
.transpose()
.map_err(|e| ConfigError::InvalidValue {
key: "LLM_CIRCUIT_BREAKER_THRESHOLD".to_string(),
message: format!("must be a positive integer: {e}"),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The error message states that LLM_CIRCUIT_BREAKER_THRESHOLD must be a positive integer. However, a value of 0 is a valid u32 and represents a valid configuration (tripping the breaker on the first failure). To be more accurate, this message should indicate a "non-negative integer".

Suggested change
message: format!("must be a positive integer: {e}"),
message: format!("must be a non-negative integer: {e}"),

})?
.or(nearai.circuit_breaker_threshold);

let circuit_breaker_recovery_secs = optional_env("LLM_CIRCUIT_BREAKER_RECOVERY_SECS")?
.map(|s| s.parse::<u64>())
.transpose()
.map_err(|e| ConfigError::InvalidValue {
key: "LLM_CIRCUIT_BREAKER_RECOVERY_SECS".to_string(),
message: format!("must be a non-negative integer: {e}"),
})?
.unwrap_or(nearai.circuit_breaker_recovery_secs);

let response_cache_enabled = optional_env("LLM_RESPONSE_CACHE_ENABLED")?
.map(|s| s.parse::<bool>())
.transpose()
.map_err(|e| ConfigError::InvalidValue {
key: "LLM_RESPONSE_CACHE_ENABLED".to_string(),
message: format!("must be true or false: {e}"),
})?
.unwrap_or(nearai.response_cache_enabled);

let response_cache_ttl_secs = optional_env("LLM_RESPONSE_CACHE_TTL_SECS")?
.map(|s| s.parse::<u64>())
.transpose()
.map_err(|e| ConfigError::InvalidValue {
key: "LLM_RESPONSE_CACHE_TTL_SECS".to_string(),
message: format!("must be a non-negative integer: {e}"),
})?
.unwrap_or(nearai.response_cache_ttl_secs);

let response_cache_max_entries = optional_env("LLM_RESPONSE_CACHE_MAX_ENTRIES")?
.map(|s| s.parse::<usize>())
.transpose()
.map_err(|e| ConfigError::InvalidValue {
key: "LLM_RESPONSE_CACHE_MAX_ENTRIES".to_string(),
message: format!("must be a non-negative integer: {e}"),
})?
.unwrap_or(nearai.response_cache_max_entries);

Ok(Self {
backend: if is_nearai {
"nearai".to_string()
Expand All @@ -324,6 +386,12 @@ impl LlmConfig {
request_timeout_secs,
cheap_model,
smart_routing_cascade,
max_retries,
circuit_breaker_threshold,
circuit_breaker_recovery_secs,
response_cache_enabled,
response_cache_ttl_secs,
response_cache_max_entries,
})
}

Expand Down Expand Up @@ -1725,6 +1793,75 @@ mod tests {
}
}

fn clear_llm_decorator_env() {
// SAFETY: Only called under ENV_MUTEX in tests.
unsafe {
std::env::remove_var("LLM_MAX_RETRIES");
std::env::remove_var("NEARAI_MAX_RETRIES");
std::env::remove_var("LLM_CIRCUIT_BREAKER_THRESHOLD");
std::env::remove_var("CIRCUIT_BREAKER_THRESHOLD");
std::env::remove_var("LLM_RESPONSE_CACHE_ENABLED");
std::env::remove_var("RESPONSE_CACHE_ENABLED");
}
}

#[test]
fn llm_max_retries_overrides_nearai() {
let _guard = lock_env();
clear_llm_decorator_env();
// SAFETY: Under ENV_MUTEX.
unsafe {
std::env::set_var("NEARAI_MAX_RETRIES", "5");
std::env::set_var("LLM_MAX_RETRIES", "10");
}

let cfg = LlmConfig::resolve(&Settings::default()).expect("resolve");
assert_eq!(cfg.max_retries, 10);

unsafe {
std::env::remove_var("NEARAI_MAX_RETRIES");
std::env::remove_var("LLM_MAX_RETRIES");
}
}

#[test]
fn nearai_max_retries_used_as_fallback() {
let _guard = lock_env();
clear_llm_decorator_env();
// SAFETY: Under ENV_MUTEX.
unsafe {
std::env::set_var("NEARAI_MAX_RETRIES", "7");
}

let cfg = LlmConfig::resolve(&Settings::default()).expect("resolve");
assert_eq!(cfg.max_retries, 7);

unsafe {
std::env::remove_var("NEARAI_MAX_RETRIES");
}
}

#[test]
fn llm_max_retries_invalid_value_produces_error() {
let _guard = lock_env();
clear_llm_decorator_env();
// SAFETY: Under ENV_MUTEX.
unsafe {
std::env::set_var("LLM_MAX_RETRIES", "not-a-number");
}

let err = LlmConfig::resolve(&Settings::default()).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("LLM_MAX_RETRIES"),
"error should name the env var: {msg}"
);

unsafe {
std::env::remove_var("LLM_MAX_RETRIES");
}
}

// ── DB > ENV priority tests ─────────────────────────────────────

#[test]
Expand Down
10 changes: 5 additions & 5 deletions src/llm/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ Closed (normal)

**Transient vs non-transient errors:** Only `RequestFailed`, `RateLimited`, `InvalidResponse`, `SessionExpired`, `SessionRenewalFailed`, `Http`, and `Io` count toward the threshold. `AuthFailed`, `ContextLengthExceeded`, `ModelNotAvailable`, and `Json` errors never trip the breaker — they indicate caller problems, not backend degradation.

Configure via `NearAiConfig` fields: `circuit_breaker_threshold` (None = disabled), `circuit_breaker_recovery_secs` (default: 30).
Configure via `LlmConfig` fields: `circuit_breaker_threshold` (env: `LLM_CIRCUIT_BREAKER_THRESHOLD`, falls back to `CIRCUIT_BREAKER_THRESHOLD`; None = disabled), `circuit_breaker_recovery_secs` (env: `LLM_CIRCUIT_BREAKER_RECOVERY_SECS`; default: 30).

The circuit breaker wraps the entire provider chain. When open, it immediately returns `LlmError::RequestFailed` with a message including remaining cooldown seconds. The `FailoverProvider` sitting outside can then try a fallback model.

Expand All @@ -127,7 +127,7 @@ The circuit breaker wraps the entire provider chain. When open, it immediately r

**Backoff schedule:** base 1s doubled per attempt with ±25% jitter, minimum floor 100ms. Attempt 0: ~1s, attempt 1: ~2s, attempt 2: ~4s. For `RateLimited`, uses the `retry_after` duration from the error (provider-supplied) instead of backoff.

Configure via `NearAiConfig.max_retries` (env: `NEARAI_MAX_RETRIES`; default: 3). Set to 0 to disable.
Configure via `LlmConfig.max_retries` (env: `LLM_MAX_RETRIES`, falls back to `NEARAI_MAX_RETRIES`; default: 3). Set to 0 to disable.

## LlmProvider Trait

Expand Down Expand Up @@ -168,7 +168,7 @@ To add a new provider:

`CachedProvider` in `response_cache.rs` caches `complete()` responses. `complete_with_tools()` is never cached (side effects). Cache key is SHA-256 of `(model_name, messages_json, max_tokens, temperature, stop_sequences)`. LRU eviction when `max_entries` is reached; TTL-based expiry on access.

**Defaults:** TTL = 1 hour, max entries = 1000. Configure via `NearAiConfig` fields: `response_cache_enabled` (env: `NEARAI_RESPONSE_CACHE_ENABLED`), `response_cache_ttl_secs`, `response_cache_max_entries`. Cache is in-memory only — evicted on restart.
**Defaults:** TTL = 1 hour, max entries = 1000. Configure via `LlmConfig` fields: `response_cache_enabled` (env: `LLM_RESPONSE_CACHE_ENABLED`, falls back to `RESPONSE_CACHE_ENABLED`), `response_cache_ttl_secs` (env: `LLM_RESPONSE_CACHE_TTL_SECS`), `response_cache_max_entries` (env: `LLM_RESPONSE_CACHE_MAX_ENTRIES`). Cache is in-memory only — evicted on restart.

## OpenAI-Compatible Custom Headers

Expand Down Expand Up @@ -201,8 +201,8 @@ Raw provider
→ RetryProvider (per-provider backoff; wraps both primary and fallback)
→ SmartRoutingProvider (cheap/primary split when NEARAI_CHEAP_MODEL is set)
→ FailoverProvider (fallback model; only when NEARAI_FALLBACK_MODEL is set)
→ CircuitBreakerProvider (fast-fail; only when NEARAI_CIRCUIT_BREAKER_THRESHOLD is set)
→ CachedProvider (response cache; only when NEARAI_RESPONSE_CACHE_ENABLED=true)
→ CircuitBreakerProvider (fast-fail; only when LLM_CIRCUIT_BREAKER_THRESHOLD is set)
→ CachedProvider (response cache; only when LLM_RESPONSE_CACHE_ENABLED=true)
→ RecordingLlm (trace capture; only when IRONCLAW_RECORD_TRACE is set)
```

Expand Down
18 changes: 18 additions & 0 deletions src/llm/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,24 @@ pub struct LlmConfig {
/// Enable cascade mode for smart routing (retry with primary if cheap model
/// response seems uncertain). Default: true. Set via `SMART_ROUTING_CASCADE`.
pub smart_routing_cascade: bool,
/// Maximum number of retries for transient LLM errors.
/// Set via `LLM_MAX_RETRIES` (falls back to `NEARAI_MAX_RETRIES`). Default: 3.
pub max_retries: u32,
/// Consecutive failures before circuit breaker opens. None = disabled.
/// Set via `LLM_CIRCUIT_BREAKER_THRESHOLD` (falls back to `CIRCUIT_BREAKER_THRESHOLD`).
pub circuit_breaker_threshold: Option<u32>,
/// Seconds the circuit stays open before probing. Default: 30.
/// Set via `LLM_CIRCUIT_BREAKER_RECOVERY_SECS` (falls back to `CIRCUIT_BREAKER_RECOVERY_SECS`).
pub circuit_breaker_recovery_secs: u64,
/// Enable in-memory response caching. Default: false.
/// Set via `LLM_RESPONSE_CACHE_ENABLED` (falls back to `RESPONSE_CACHE_ENABLED`).
pub response_cache_enabled: bool,
/// TTL in seconds for cached responses. Default: 3600.
/// Set via `LLM_RESPONSE_CACHE_TTL_SECS` (falls back to `RESPONSE_CACHE_TTL_SECS`).
pub response_cache_ttl_secs: u64,
/// Max cached responses before LRU eviction. Default: 1000.
/// Set via `LLM_RESPONSE_CACHE_MAX_ENTRIES` (falls back to `RESPONSE_CACHE_MAX_ENTRIES`).
pub response_cache_max_entries: usize,
}

impl LlmConfig {
Expand Down
30 changes: 17 additions & 13 deletions src/llm/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -562,9 +562,10 @@ pub async fn build_provider_chain(
};
tracing::debug!("LLM provider initialized: {}", llm.model_name());

// 1. Retry
// 1. Retry — uses top-level LlmConfig fields (resolved from LLM_* env vars
// with fallback to NEARAI_* for backward compatibility).
let retry_config = RetryConfig {
max_retries: config.nearai.max_retries,
max_retries: config.max_retries,
};
let llm: Arc<dyn LlmProvider> = if retry_config.max_retries > 0 {
tracing::debug!(
Expand Down Expand Up @@ -645,18 +646,15 @@ pub async fn build_provider_chain(
};

// 4. Circuit breaker
let llm: Arc<dyn LlmProvider> = if let Some(threshold) = config.nearai.circuit_breaker_threshold
{
let llm: Arc<dyn LlmProvider> = if let Some(threshold) = config.circuit_breaker_threshold {
let cb_config = CircuitBreakerConfig {
failure_threshold: threshold,
recovery_timeout: std::time::Duration::from_secs(
config.nearai.circuit_breaker_recovery_secs,
),
recovery_timeout: std::time::Duration::from_secs(config.circuit_breaker_recovery_secs),
..CircuitBreakerConfig::default()
};
tracing::debug!(
threshold,
recovery_secs = config.nearai.circuit_breaker_recovery_secs,
recovery_secs = config.circuit_breaker_recovery_secs,
"LLM circuit breaker enabled"
);
Arc::new(CircuitBreakerProvider::new(llm, cb_config))
Expand All @@ -665,14 +663,14 @@ pub async fn build_provider_chain(
};

// 5. Response cache
let llm: Arc<dyn LlmProvider> = if config.nearai.response_cache_enabled {
let llm: Arc<dyn LlmProvider> = if config.response_cache_enabled {
let rc_config = ResponseCacheConfig {
ttl: std::time::Duration::from_secs(config.nearai.response_cache_ttl_secs),
max_entries: config.nearai.response_cache_max_entries,
ttl: std::time::Duration::from_secs(config.response_cache_ttl_secs),
max_entries: config.response_cache_max_entries,
};
tracing::debug!(
ttl_secs = config.nearai.response_cache_ttl_secs,
max_entries = config.nearai.response_cache_max_entries,
ttl_secs = config.response_cache_ttl_secs,
max_entries = config.response_cache_max_entries,
"LLM response cache enabled"
);
Arc::new(CachedProvider::new(llm, rc_config))
Expand Down Expand Up @@ -744,6 +742,12 @@ mod tests {
cheap_model: None,
smart_routing_cascade: true,
openai_codex: None,
max_retries: 3,
circuit_breaker_threshold: None,
circuit_breaker_recovery_secs: 30,
response_cache_enabled: false,
response_cache_ttl_secs: 3600,
response_cache_max_entries: 1000,
}
}

Expand Down
6 changes: 6 additions & 0 deletions src/llm/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -349,5 +349,11 @@ pub(crate) fn build_nearai_model_fetch_config() -> crate::config::LlmConfig {
cheap_model: None,
smart_routing_cascade: false,
openai_codex: None,
max_retries: 3,
circuit_breaker_threshold: None,
circuit_breaker_recovery_secs: 30,
response_cache_enabled: false,
response_cache_ttl_secs: 3600,
response_cache_max_entries: 1000,
}
}
Loading