From be0c1fbaca1594ed0d538b6f5506ae15bd62543a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 29 Jul 2026 11:32:07 +0200 Subject: [PATCH] feat(config): add per-operation reasoning_effort override (#2998) reasoning_effort was the only LLM request setting without a per-operation override: retain, reflect and consolidation all read the single global HINDSIGHT_API_LLM_REASONING_EFFORT. When one operation requires a specific value (e.g. reflect needs "none" for OpenAI reasoning models that reject function tools otherwise), that value is forced onto the others, silently degrading their generation quality. Add REASONING_EFFORT to the existing per-operation set, following the established fallback pattern: HINDSIGHT_API_RETAIN_LLM_REASONING_EFFORT HINDSIGHT_API_REFLECT_LLM_REASONING_EFFORT HINDSIGHT_API_CONSOLIDATION_LLM_REASONING_EFFORT Each falls back to HINDSIGHT_API_LLM_REASONING_EFFORT when unset. --- hindsight-api-slim/hindsight_api/config.py | 9 +++ .../hindsight_api/engine/memory_engine.py | 6 +- .../tests/test_per_operation_llm_config.py | 81 +++++++++++++++++++ .../docs/developer/configuration.md | 3 + .../references/developer/configuration.md | 3 + 5 files changed, 99 insertions(+), 3 deletions(-) diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 1811a57433..c68fed5381 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -314,6 +314,7 @@ def _parse_boolean_env(env_name: str, default: bool) -> bool: ENV_RETAIN_LLM_MAX_BACKOFF = "HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF" ENV_RETAIN_LLM_TIMEOUT = "HINDSIGHT_API_RETAIN_LLM_TIMEOUT" ENV_RETAIN_LLM_LITELLMROUTER_CONFIG = "HINDSIGHT_API_RETAIN_LLM_LITELLMROUTER_CONFIG" +ENV_RETAIN_LLM_REASONING_EFFORT = "HINDSIGHT_API_RETAIN_LLM_REASONING_EFFORT" # Fireworks AI batch inference. Fireworks' batch API is a proprietary # account-scoped dataset/job REST API on a control-plane host, distinct from the @@ -336,6 +337,7 @@ def _parse_boolean_env(env_name: str, default: bool) -> bool: ENV_REFLECT_LLM_MAX_BACKOFF = "HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF" ENV_REFLECT_LLM_TIMEOUT = "HINDSIGHT_API_REFLECT_LLM_TIMEOUT" ENV_REFLECT_LLM_LITELLMROUTER_CONFIG = "HINDSIGHT_API_REFLECT_LLM_LITELLMROUTER_CONFIG" +ENV_REFLECT_LLM_REASONING_EFFORT = "HINDSIGHT_API_REFLECT_LLM_REASONING_EFFORT" ENV_CONSOLIDATION_LLM_PROVIDER = "HINDSIGHT_API_CONSOLIDATION_LLM_PROVIDER" ENV_CONSOLIDATION_LLM_API_KEY = "HINDSIGHT_API_CONSOLIDATION_LLM_API_KEY" @@ -347,6 +349,7 @@ def _parse_boolean_env(env_name: str, default: bool) -> bool: ENV_CONSOLIDATION_LLM_MAX_BACKOFF = "HINDSIGHT_API_CONSOLIDATION_LLM_MAX_BACKOFF" ENV_CONSOLIDATION_LLM_TIMEOUT = "HINDSIGHT_API_CONSOLIDATION_LLM_TIMEOUT" ENV_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG = "HINDSIGHT_API_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG" +ENV_CONSOLIDATION_LLM_REASONING_EFFORT = "HINDSIGHT_API_CONSOLIDATION_LLM_REASONING_EFFORT" ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER" ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL" @@ -1859,6 +1862,7 @@ class HindsightConfig: retain_llm_max_backoff: float | None retain_llm_timeout: float | None retain_llm_litellmrouter_config: dict | None + retain_llm_reasoning_effort: str | None # Fireworks AI batch inference (static, server-level) fireworks_account_id: str | None @@ -1875,6 +1879,7 @@ class HindsightConfig: reflect_llm_max_backoff: float | None reflect_llm_timeout: float | None reflect_llm_litellmrouter_config: dict | None + reflect_llm_reasoning_effort: str | None consolidation_llm_provider: str | None consolidation_llm_api_key: str | None @@ -1886,6 +1891,7 @@ class HindsightConfig: consolidation_llm_max_backoff: float | None consolidation_llm_timeout: float | None consolidation_llm_litellmrouter_config: dict | None + consolidation_llm_reasoning_effort: str | None # Embeddings embeddings_provider: str @@ -2681,6 +2687,7 @@ def from_env(cls) -> "HindsightConfig": else None, retain_llm_timeout=float(os.getenv(ENV_RETAIN_LLM_TIMEOUT)) if os.getenv(ENV_RETAIN_LLM_TIMEOUT) else None, retain_llm_litellmrouter_config=_parse_llm_router_config(ENV_RETAIN_LLM_LITELLMROUTER_CONFIG), + retain_llm_reasoning_effort=os.getenv(ENV_RETAIN_LLM_REASONING_EFFORT) or None, reflect_llm_provider=os.getenv(ENV_REFLECT_LLM_PROVIDER) or None, reflect_llm_api_key=os.getenv(ENV_REFLECT_LLM_API_KEY) or None, reflect_llm_model=os.getenv(ENV_REFLECT_LLM_MODEL) @@ -2706,6 +2713,7 @@ def from_env(cls) -> "HindsightConfig": if os.getenv(ENV_REFLECT_LLM_TIMEOUT) else None, reflect_llm_litellmrouter_config=_parse_llm_router_config(ENV_REFLECT_LLM_LITELLMROUTER_CONFIG), + reflect_llm_reasoning_effort=os.getenv(ENV_REFLECT_LLM_REASONING_EFFORT) or None, consolidation_llm_provider=os.getenv(ENV_CONSOLIDATION_LLM_PROVIDER) or None, consolidation_llm_api_key=os.getenv(ENV_CONSOLIDATION_LLM_API_KEY) or None, consolidation_llm_model=os.getenv(ENV_CONSOLIDATION_LLM_MODEL) @@ -2731,6 +2739,7 @@ def from_env(cls) -> "HindsightConfig": if os.getenv(ENV_CONSOLIDATION_LLM_TIMEOUT) else None, consolidation_llm_litellmrouter_config=_parse_llm_router_config(ENV_CONSOLIDATION_LLM_LITELLMROUTER_CONFIG), + consolidation_llm_reasoning_effort=os.getenv(ENV_CONSOLIDATION_LLM_REASONING_EFFORT) or None, # Multi-LLM chains (indexed members + routing strategy) llm_members=_parse_llm_members(""), llm_strategy=_parse_llm_strategy(os.getenv(ENV_LLM_STRATEGY)), diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index 486b1965c8..7432d080d2 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -1182,7 +1182,7 @@ def pick(field: str) -> Any: api_key=retain_api_key, base_url=retain_base_url, model=retain_model, - reasoning_effort=config.llm_reasoning_effort, + reasoning_effort=config.retain_llm_reasoning_effort or config.llm_reasoning_effort, extra_body=config.llm_extra_body, default_headers=config.llm_default_headers, ollama_num_ctx=config.llm_ollama_num_ctx, @@ -1221,7 +1221,7 @@ def pick(field: str) -> Any: api_key=reflect_api_key, base_url=reflect_base_url, model=reflect_model, - reasoning_effort=config.llm_reasoning_effort, + reasoning_effort=config.reflect_llm_reasoning_effort or config.llm_reasoning_effort, extra_body=config.llm_extra_body, default_headers=config.llm_default_headers, ollama_num_ctx=config.llm_ollama_num_ctx, @@ -1260,7 +1260,7 @@ def pick(field: str) -> Any: api_key=consolidation_api_key, base_url=consolidation_base_url, model=consolidation_model, - reasoning_effort=config.llm_reasoning_effort, + reasoning_effort=config.consolidation_llm_reasoning_effort or config.llm_reasoning_effort, extra_body=config.llm_extra_body, default_headers=config.llm_default_headers, ollama_num_ctx=config.llm_ollama_num_ctx, diff --git a/hindsight-api-slim/tests/test_per_operation_llm_config.py b/hindsight-api-slim/tests/test_per_operation_llm_config.py index f7ca872d3c..0bfa5fbf35 100644 --- a/hindsight-api-slim/tests/test_per_operation_llm_config.py +++ b/hindsight-api-slim/tests/test_per_operation_llm_config.py @@ -430,3 +430,84 @@ def test_per_operation_retry_backoff_fallback_to_global(self): os.environ.pop("HINDSIGHT_API_LLM_INITIAL_BACKOFF", None) os.environ.pop("HINDSIGHT_API_LLM_MAX_BACKOFF", None) clear_config_cache() + + +class TestPerOperationReasoningEffort: + """Test the per-operation reasoning_effort override (issue #2998).""" + + _REASONING_EFFORT_ENV_VARS = ( + "HINDSIGHT_API_LLM_REASONING_EFFORT", + "HINDSIGHT_API_RETAIN_LLM_REASONING_EFFORT", + "HINDSIGHT_API_REFLECT_LLM_REASONING_EFFORT", + "HINDSIGHT_API_CONSOLIDATION_LLM_REASONING_EFFORT", + ) + + def _clear_reasoning_effort_env(self): + for key in self._REASONING_EFFORT_ENV_VARS: + os.environ.pop(key, None) + + def test_per_operation_reasoning_effort_from_env(self): + """Per-operation reasoning_effort overrides are loaded from environment.""" + from hindsight_api.config import clear_config_cache, get_config + + os.environ["HINDSIGHT_API_LLM_REASONING_EFFORT"] = "low" + os.environ["HINDSIGHT_API_RETAIN_LLM_REASONING_EFFORT"] = "high" + os.environ["HINDSIGHT_API_REFLECT_LLM_REASONING_EFFORT"] = "none" + os.environ["HINDSIGHT_API_CONSOLIDATION_LLM_REASONING_EFFORT"] = "medium" + + try: + clear_config_cache() + config = get_config() + + assert config.retain_llm_reasoning_effort == "high" + assert config.reflect_llm_reasoning_effort == "none" + assert config.consolidation_llm_reasoning_effort == "medium" + + # Global stays untouched. + assert config.llm_reasoning_effort == "low" + finally: + self._clear_reasoning_effort_env() + clear_config_cache() + + def test_per_operation_reasoning_effort_fallback_to_global(self): + """Unset per-operation reasoning_effort stays None (falls back at runtime).""" + from hindsight_api.config import clear_config_cache, get_config + + os.environ["HINDSIGHT_API_LLM_REASONING_EFFORT"] = "medium" + os.environ.pop("HINDSIGHT_API_RETAIN_LLM_REASONING_EFFORT", None) + os.environ.pop("HINDSIGHT_API_REFLECT_LLM_REASONING_EFFORT", None) + os.environ.pop("HINDSIGHT_API_CONSOLIDATION_LLM_REASONING_EFFORT", None) + + try: + clear_config_cache() + config = get_config() + + assert config.retain_llm_reasoning_effort is None + assert config.reflect_llm_reasoning_effort is None + assert config.consolidation_llm_reasoning_effort is None + assert config.llm_reasoning_effort == "medium" + finally: + self._clear_reasoning_effort_env() + clear_config_cache() + + def test_memory_engine_applies_per_operation_reasoning_effort(self): + """The engine threads each per-operation override into its LLM config, + and unset operations fall back to the global value.""" + from hindsight_api import MemoryEngine + from hindsight_api.config import clear_config_cache + + os.environ["HINDSIGHT_API_LLM_REASONING_EFFORT"] = "low" + os.environ["HINDSIGHT_API_REFLECT_LLM_REASONING_EFFORT"] = "none" + # retain/consolidation intentionally unset -> fall back to global "low". + + try: + clear_config_cache() + engine = MemoryEngine(skip_llm_verification=True) + + assert engine._llm_config.reasoning_effort == "low" + assert engine._reflect_llm_config.reasoning_effort == "none" + assert engine._retain_llm_config.reasoning_effort == "low" + assert engine._consolidation_llm_config.reasoning_effort == "low" + finally: + self._clear_reasoning_effort_env() + clear_config_cache() diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index fa81778360..13b94c647d 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -469,6 +469,7 @@ Different memory operations have different requirements. **Retain** (fact extrac | `HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF` | Initial backoff for retain retries (seconds) | Falls back to `HINDSIGHT_API_LLM_INITIAL_BACKOFF` | | `HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF` | Max backoff cap for retain retries (seconds) | Falls back to `HINDSIGHT_API_LLM_MAX_BACKOFF` | | `HINDSIGHT_API_RETAIN_LLM_TIMEOUT` | Timeout for retain requests (seconds) | Falls back to `HINDSIGHT_API_LLM_TIMEOUT` | +| `HINDSIGHT_API_RETAIN_LLM_REASONING_EFFORT` | Reasoning effort for retain operations | Falls back to `HINDSIGHT_API_LLM_REASONING_EFFORT` | | `HINDSIGHT_API_REFLECT_LLM_PROVIDER` | LLM provider for reflect operations | Falls back to `HINDSIGHT_API_LLM_PROVIDER` | | `HINDSIGHT_API_REFLECT_LLM_API_KEY` | API key for reflect LLM | Falls back to `HINDSIGHT_API_LLM_API_KEY` | | `HINDSIGHT_API_REFLECT_LLM_MODEL` | Model for reflect operations | Falls back to `HINDSIGHT_API_LLM_MODEL` | @@ -478,6 +479,7 @@ Different memory operations have different requirements. **Retain** (fact extrac | `HINDSIGHT_API_REFLECT_LLM_INITIAL_BACKOFF` | Initial backoff for reflect retries (seconds) | Falls back to `HINDSIGHT_API_LLM_INITIAL_BACKOFF` | | `HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF` | Max backoff cap for reflect retries (seconds) | Falls back to `HINDSIGHT_API_LLM_MAX_BACKOFF` | | `HINDSIGHT_API_REFLECT_LLM_TIMEOUT` | Timeout for reflect requests (seconds) | Falls back to `HINDSIGHT_API_LLM_TIMEOUT` | +| `HINDSIGHT_API_REFLECT_LLM_REASONING_EFFORT` | Reasoning effort for reflect operations | Falls back to `HINDSIGHT_API_LLM_REASONING_EFFORT` | | `HINDSIGHT_API_CONSOLIDATION_LLM_PROVIDER` | LLM provider for observation consolidation | Falls back to `HINDSIGHT_API_LLM_PROVIDER` | | `HINDSIGHT_API_CONSOLIDATION_LLM_API_KEY` | API key for consolidation LLM | Falls back to `HINDSIGHT_API_LLM_API_KEY` | | `HINDSIGHT_API_CONSOLIDATION_LLM_MODEL` | Model for consolidation operations | Falls back to `HINDSIGHT_API_LLM_MODEL` | @@ -487,6 +489,7 @@ Different memory operations have different requirements. **Retain** (fact extrac | `HINDSIGHT_API_CONSOLIDATION_LLM_INITIAL_BACKOFF` | Initial backoff for consolidation retries (seconds) | Falls back to `HINDSIGHT_API_LLM_INITIAL_BACKOFF` | | `HINDSIGHT_API_CONSOLIDATION_LLM_MAX_BACKOFF` | Max backoff cap for consolidation retries (seconds) | Falls back to `HINDSIGHT_API_LLM_MAX_BACKOFF` | | `HINDSIGHT_API_CONSOLIDATION_LLM_TIMEOUT` | Timeout for consolidation requests (seconds) | Falls back to `HINDSIGHT_API_LLM_TIMEOUT` | +| `HINDSIGHT_API_CONSOLIDATION_LLM_REASONING_EFFORT` | Reasoning effort for consolidation operations | Falls back to `HINDSIGHT_API_LLM_REASONING_EFFORT` | :::tip When to Use Per-Operation Config - **Retain**: Use models with strong structured output (e.g., GPT-4o, Claude) for accurate fact extraction diff --git a/skills/hindsight-docs/references/developer/configuration.md b/skills/hindsight-docs/references/developer/configuration.md index f23fa3d908..09ad81ead8 100644 --- a/skills/hindsight-docs/references/developer/configuration.md +++ b/skills/hindsight-docs/references/developer/configuration.md @@ -469,6 +469,7 @@ Different memory operations have different requirements. **Retain** (fact extrac | `HINDSIGHT_API_RETAIN_LLM_INITIAL_BACKOFF` | Initial backoff for retain retries (seconds) | Falls back to `HINDSIGHT_API_LLM_INITIAL_BACKOFF` | | `HINDSIGHT_API_RETAIN_LLM_MAX_BACKOFF` | Max backoff cap for retain retries (seconds) | Falls back to `HINDSIGHT_API_LLM_MAX_BACKOFF` | | `HINDSIGHT_API_RETAIN_LLM_TIMEOUT` | Timeout for retain requests (seconds) | Falls back to `HINDSIGHT_API_LLM_TIMEOUT` | +| `HINDSIGHT_API_RETAIN_LLM_REASONING_EFFORT` | Reasoning effort for retain operations | Falls back to `HINDSIGHT_API_LLM_REASONING_EFFORT` | | `HINDSIGHT_API_REFLECT_LLM_PROVIDER` | LLM provider for reflect operations | Falls back to `HINDSIGHT_API_LLM_PROVIDER` | | `HINDSIGHT_API_REFLECT_LLM_API_KEY` | API key for reflect LLM | Falls back to `HINDSIGHT_API_LLM_API_KEY` | | `HINDSIGHT_API_REFLECT_LLM_MODEL` | Model for reflect operations | Falls back to `HINDSIGHT_API_LLM_MODEL` | @@ -478,6 +479,7 @@ Different memory operations have different requirements. **Retain** (fact extrac | `HINDSIGHT_API_REFLECT_LLM_INITIAL_BACKOFF` | Initial backoff for reflect retries (seconds) | Falls back to `HINDSIGHT_API_LLM_INITIAL_BACKOFF` | | `HINDSIGHT_API_REFLECT_LLM_MAX_BACKOFF` | Max backoff cap for reflect retries (seconds) | Falls back to `HINDSIGHT_API_LLM_MAX_BACKOFF` | | `HINDSIGHT_API_REFLECT_LLM_TIMEOUT` | Timeout for reflect requests (seconds) | Falls back to `HINDSIGHT_API_LLM_TIMEOUT` | +| `HINDSIGHT_API_REFLECT_LLM_REASONING_EFFORT` | Reasoning effort for reflect operations | Falls back to `HINDSIGHT_API_LLM_REASONING_EFFORT` | | `HINDSIGHT_API_CONSOLIDATION_LLM_PROVIDER` | LLM provider for observation consolidation | Falls back to `HINDSIGHT_API_LLM_PROVIDER` | | `HINDSIGHT_API_CONSOLIDATION_LLM_API_KEY` | API key for consolidation LLM | Falls back to `HINDSIGHT_API_LLM_API_KEY` | | `HINDSIGHT_API_CONSOLIDATION_LLM_MODEL` | Model for consolidation operations | Falls back to `HINDSIGHT_API_LLM_MODEL` | @@ -487,6 +489,7 @@ Different memory operations have different requirements. **Retain** (fact extrac | `HINDSIGHT_API_CONSOLIDATION_LLM_INITIAL_BACKOFF` | Initial backoff for consolidation retries (seconds) | Falls back to `HINDSIGHT_API_LLM_INITIAL_BACKOFF` | | `HINDSIGHT_API_CONSOLIDATION_LLM_MAX_BACKOFF` | Max backoff cap for consolidation retries (seconds) | Falls back to `HINDSIGHT_API_LLM_MAX_BACKOFF` | | `HINDSIGHT_API_CONSOLIDATION_LLM_TIMEOUT` | Timeout for consolidation requests (seconds) | Falls back to `HINDSIGHT_API_LLM_TIMEOUT` | +| `HINDSIGHT_API_CONSOLIDATION_LLM_REASONING_EFFORT` | Reasoning effort for consolidation operations | Falls back to `HINDSIGHT_API_LLM_REASONING_EFFORT` | :::tip When to Use Per-Operation Config - **Retain**: Use models with strong structured output (e.g., GPT-4o, Claude) for accurate fact extraction