From 8b62b6be82a204e22c11b5e38796cee74e8bd882 Mon Sep 17 00:00:00 2001 From: DK09876 Date: Fri, 5 Dec 2025 13:42:06 -0700 Subject: [PATCH 01/10] Added hindsight_liteLLM implementation --- .../hindsight_api/engine/llm_wrapper.py | 39 + hindsight-integrations/litellm/README.md | 270 +++++++ .../litellm/hindsight_litellm/__init__.py | 605 +++++++++++++++ .../litellm/hindsight_litellm/callbacks.py | 615 +++++++++++++++ .../litellm/hindsight_litellm/config.py | 267 +++++++ .../litellm/hindsight_litellm/wrappers.py | 708 ++++++++++++++++++ hindsight-integrations/litellm/pyproject.toml | 59 ++ .../litellm/tests/__init__.py | 1 + .../litellm/tests/test_integration.py | 471 ++++++++++++ 9 files changed, 3035 insertions(+) create mode 100644 hindsight-integrations/litellm/README.md create mode 100644 hindsight-integrations/litellm/hindsight_litellm/__init__.py create mode 100644 hindsight-integrations/litellm/hindsight_litellm/callbacks.py create mode 100644 hindsight-integrations/litellm/hindsight_litellm/config.py create mode 100644 hindsight-integrations/litellm/hindsight_litellm/wrappers.py create mode 100644 hindsight-integrations/litellm/pyproject.toml create mode 100644 hindsight-integrations/litellm/tests/__init__.py create mode 100644 hindsight-integrations/litellm/tests/test_integration.py diff --git a/hindsight-api/hindsight_api/engine/llm_wrapper.py b/hindsight-api/hindsight_api/engine/llm_wrapper.py index d3ff78ed32..d9afec7a86 100644 --- a/hindsight-api/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api/hindsight_api/engine/llm_wrapper.py @@ -22,6 +22,35 @@ # Global semaphore to limit concurrent LLM requests across all instances _global_llm_semaphore = asyncio.Semaphore(32) +# Model-specific max output token limits +# These represent the maximum tokens a model can generate in a single response +MODEL_MAX_OUTPUT_TOKENS = { + # OpenAI models + "gpt-4o": 16384, + "gpt-4o-mini": 16384, + "gpt-4-turbo": 4096, + "gpt-4-turbo-preview": 4096, + "gpt-4": 8192, + "gpt-3.5-turbo": 4096, + "o1": 100000, + "o1-mini": 65536, + "o1-preview": 32768, + # Groq models + "llama-3.1-70b-versatile": 32768, + "llama-3.1-8b-instant": 8192, + "llama-3.3-70b-versatile": 32768, + "llama3-70b-8192": 8192, + "llama3-8b-8192": 8192, + "mixtral-8x7b-32768": 32768, + # Gemini models + "gemini-2.0-flash": 8192, + "gemini-1.5-pro": 8192, + "gemini-1.5-flash": 8192, +} + +# Conservative default for unknown models +DEFAULT_MAX_OUTPUT_TOKENS = 4096 + class OutputTooLongError(Exception): """ @@ -121,6 +150,16 @@ async def verify_connection(self) -> None: f"LLM connection verification failed for {self.provider}/{self.model}: {e}" ) from e + @property + def max_output_tokens(self) -> int: + """ + Get the max output tokens for the configured model. + + Returns the model-specific limit from MODEL_MAX_OUTPUT_TOKENS, + or DEFAULT_MAX_OUTPUT_TOKENS if the model is not in the mapping. + """ + return MODEL_MAX_OUTPUT_TOKENS.get(self.model, DEFAULT_MAX_OUTPUT_TOKENS) + async def call( self, messages: List[Dict[str, str]], diff --git a/hindsight-integrations/litellm/README.md b/hindsight-integrations/litellm/README.md new file mode 100644 index 0000000000..4bb6febd45 --- /dev/null +++ b/hindsight-integrations/litellm/README.md @@ -0,0 +1,270 @@ +# hindsight-litellm + +Universal LLM memory integration via LiteLLM. Add persistent memory to any LLM application with just a few lines of code. + +## Features + +- **Universal LLM Support** - Works with 100+ LLM providers via LiteLLM (OpenAI, Anthropic, Groq, Azure, AWS Bedrock, Google Vertex AI, and more) +- **Simple Integration** - Just configure, enable, and use `hindsight_litellm.completion()` +- **Automatic Memory Injection** - Relevant memories are injected into prompts before LLM calls +- **Automatic Conversation Storage** - Conversations are stored to Hindsight for future recall +- **Multi-User Support** - Entity ID scoping for isolated per-user memories +- **Session Management** - Group conversations into logical sessions +- **Direct Recall API** - Query memories manually without making LLM calls +- **Native Client Wrappers** - Alternative wrappers for OpenAI and Anthropic SDKs + +## Installation + +```bash +pip install hindsight-litellm +``` + +## Quick Start + +```python +import hindsight_litellm + +# Configure and enable memory integration +hindsight_litellm.configure( + hindsight_api_url="http://localhost:8888", + bank_id="my-agent", + entity_id="user-123", # Optional: for multi-user isolation +) +hindsight_litellm.enable() + +# Use the convenience wrapper - memory is automatically injected and stored +response = hindsight_litellm.completion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "What did we discuss about AI?"}] +) +``` + +## Configuration Options + +```python +hindsight_litellm.configure( + # Required + hindsight_api_url="http://localhost:8888", # Hindsight API server URL + bank_id="my-agent", # Memory bank ID + + # Optional - Multi-user and session management + entity_id="user-123", # User identifier for memory isolation + session_id="session-abc", # Session identifier for grouping + api_key="your-api-key", # Optional API key for authentication + + # Optional - Memory behavior + store_conversations=True, # Store conversations after LLM calls + inject_memories=True, # Inject relevant memories into prompts + max_memories=10, # Maximum memories to inject + max_memory_tokens=2000, # Maximum tokens for memory context + recall_budget="mid", # Recall budget: "low", "mid", "high" + fact_types=["world", "agent"], # Filter fact types to inject + + # Optional - Advanced + injection_mode="system_message", # or "prepend_user" + excluded_models=["gpt-3.5*"], # Exclude certain models + verbose=True, # Enable verbose logging +) +``` + +## Multi-Provider Support + +Works with any LiteLLM-supported provider: + +```python +import hindsight_litellm + +hindsight_litellm.configure( + hindsight_api_url="http://localhost:8888", + bank_id="my-agent", +) +hindsight_litellm.enable() + +# OpenAI +hindsight_litellm.completion(model="gpt-4o", messages=[...]) + +# Anthropic +hindsight_litellm.completion(model="claude-3-5-sonnet-20241022", messages=[...]) + +# Groq +hindsight_litellm.completion(model="groq/llama-3.1-70b-versatile", messages=[...]) + +# Azure OpenAI +hindsight_litellm.completion(model="azure/gpt-4", messages=[...]) + +# AWS Bedrock +hindsight_litellm.completion(model="bedrock/anthropic.claude-3", messages=[...]) + +# Google Vertex AI +hindsight_litellm.completion(model="vertex_ai/gemini-pro", messages=[...]) +``` + +## Direct Recall API + +Query memories manually without making an LLM call: + +```python +from hindsight_litellm import configure, recall + +configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") + +# Query memories +memories = recall("what projects am I working on?", limit=5) +for m in memories: + print(f"- [{m.fact_type}] {m.text}") + +# Output: +# - [world] User is building a FastAPI project +# - [opinion] User prefers Python over JavaScript +``` + +### Async Recall + +```python +from hindsight_litellm import arecall + +memories = await arecall("what do you know about me?", limit=10) +``` + +## Native Client Wrappers + +Alternative to LiteLLM callbacks for direct SDK integration: + +### OpenAI Wrapper + +```python +from openai import OpenAI +from hindsight_litellm import wrap_openai + +client = OpenAI() +wrapped = wrap_openai( + client, + bank_id="my-agent", + hindsight_api_url="http://localhost:8888", + entity_id="user-123", +) + +response = wrapped.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "What do you know about me?"}] +) +``` + +### Anthropic Wrapper + +```python +from anthropic import Anthropic +from hindsight_litellm import wrap_anthropic + +client = Anthropic() +wrapped = wrap_anthropic( + client, + bank_id="my-agent", + hindsight_api_url="http://localhost:8888", +) + +response = wrapped.messages.create( + model="claude-3-5-sonnet-20241022", + max_tokens=1024, + messages=[{"role": "user", "content": "Hello!"}] +) +``` + +## Session Management + +```python +from hindsight_litellm import configure, new_session, set_session, get_session + +configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") + +# Start a fresh conversation thread +session_id = new_session() +print(f"Started new session: {session_id}") + +# Resume a previous conversation +set_session("previous-session-id") + +# Get current session ID +current = get_session() +``` + +## Entity Management (Multi-User) + +```python +from hindsight_litellm import configure, set_entity, get_entity + +configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") + +# Switch between users +set_entity("user-alice") +# ... Alice's conversations and memories + +set_entity("user-bob") +# ... Bob's conversations and memories (isolated from Alice) +``` + +## Disabling and Cleanup + +```python +from hindsight_litellm import disable, cleanup + +# Temporarily disable memory integration +disable() + +# Clean up all resources (call when shutting down) +cleanup() +``` + +## API Reference + +### Main Functions + +| Function | Description | +|----------|-------------| +| `configure(...)` | Configure global Hindsight settings | +| `enable()` | Enable memory integration with LiteLLM | +| `disable()` | Disable memory integration | +| `is_enabled()` | Check if memory integration is enabled | +| `cleanup()` | Clean up all resources | + +### Configuration Functions + +| Function | Description | +|----------|-------------| +| `get_config()` | Get current configuration | +| `is_configured()` | Check if Hindsight is configured | +| `reset_config()` | Reset configuration to defaults | + +### Session/Entity Functions + +| Function | Description | +|----------|-------------| +| `new_session()` | Generate and set a new session ID | +| `set_session(id)` | Set a specific session ID | +| `get_session()` | Get current session ID | +| `set_entity(id)` | Set entity ID for multi-user isolation | +| `get_entity()` | Get current entity ID | + +### Recall Functions + +| Function | Description | +|----------|-------------| +| `recall(query, ...)` | Synchronously query memories | +| `arecall(query, ...)` | Asynchronously query memories | + +### Client Wrappers + +| Function | Description | +|----------|-------------| +| `wrap_openai(client, ...)` | Wrap OpenAI client with memory | +| `wrap_anthropic(client, ...)` | Wrap Anthropic client with memory | + +## Requirements + +- Python >= 3.10 +- litellm >= 1.40.0 +- A running Hindsight API server + +## License + +MIT diff --git a/hindsight-integrations/litellm/hindsight_litellm/__init__.py b/hindsight-integrations/litellm/hindsight_litellm/__init__.py new file mode 100644 index 0000000000..67c1224343 --- /dev/null +++ b/hindsight-integrations/litellm/hindsight_litellm/__init__.py @@ -0,0 +1,605 @@ +"""Hindsight-LiteLLM: Universal LLM memory integration via LiteLLM. + +This package provides automatic memory integration for any LLM provider +supported by LiteLLM (100+ providers including OpenAI, Anthropic, Groq, +Azure, AWS Bedrock, Google Vertex AI, and more). + +Features: +- Automatic memory injection before LLM calls +- Automatic conversation storage after LLM calls +- Works with any LiteLLM-supported provider +- Zero code changes to existing LiteLLM usage +- Multi-user support via entity_id +- Session management for conversation threading +- Direct recall API for manual memory queries +- Native client wrappers for OpenAI and Anthropic + +Basic usage: + >>> from hindsight_litellm import configure, enable + >>> + >>> # Configure Hindsight integration + >>> configure( + ... hindsight_api_url="http://localhost:8888", + ... bank_id="my-agent", + ... entity_id="user-123", # Multi-user support + ... store_conversations=True, + ... inject_memories=True, + ... ) + >>> + >>> # Enable memory integration + >>> enable() + >>> + >>> # Now use LiteLLM as normal - memory integration is automatic + >>> import litellm + >>> response = litellm.completion( + ... model="gpt-4", + ... messages=[{"role": "user", "content": "What did we discuss about AI?"}] + ... ) + +Direct recall API: + >>> from hindsight_litellm import configure, recall + >>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") + >>> + >>> # Query memories directly + >>> memories = recall("what projects am I working on?", limit=5) + >>> for m in memories: + ... print(f"- [{m.fact_type}] {m.text}") + +Native client wrappers: + >>> from openai import OpenAI + >>> from hindsight_litellm import wrap_openai + >>> + >>> client = OpenAI() + >>> wrapped = wrap_openai(client, bank_id="my-agent", entity_id="user-123") + >>> + >>> response = wrapped.chat.completions.create( + ... model="gpt-4", + ... messages=[{"role": "user", "content": "Hello!"}] + ... ) + +Session management: + >>> from hindsight_litellm import configure, new_session, set_session + >>> configure(bank_id="my-agent") + >>> + >>> session_id = new_session() # Start fresh conversation + >>> set_session("previous-id") # Resume previous conversation + +Works with any LiteLLM-supported provider: + >>> # OpenAI + >>> litellm.completion(model="gpt-4", messages=[...]) + >>> + >>> # Anthropic + >>> litellm.completion(model="claude-3-opus-20240229", messages=[...]) + >>> + >>> # Groq + >>> litellm.completion(model="groq/llama-3.1-70b-versatile", messages=[...]) + >>> + >>> # Azure OpenAI + >>> litellm.completion(model="azure/gpt-4", messages=[...]) + >>> + >>> # AWS Bedrock + >>> litellm.completion(model="bedrock/anthropic.claude-3", messages=[...]) + >>> + >>> # Google Vertex AI + >>> litellm.completion(model="vertex_ai/gemini-pro", messages=[...]) + +Context manager usage: + >>> from hindsight_litellm import hindsight_memory + >>> + >>> with hindsight_memory(bank_id="my-agent", entity_id="user-123"): + ... response = litellm.completion(model="gpt-4", messages=[...]) + >>> # Memory integration automatically disabled after context + +Configuration options: + - hindsight_api_url: URL of your Hindsight API server + - bank_id: Memory bank ID for memory operations (required) + - api_key: Optional API key for Hindsight authentication + - entity_id: User identifier for multi-user memory isolation + - session_id: Session identifier for conversation grouping + - store_conversations: Whether to store conversations (default: True) + - inject_memories: Whether to inject relevant memories (default: True) + - injection_mode: How to inject memories (system_message or prepend_user) + - max_memories: Maximum number of memories to inject (default: 10) + - recall_budget: Budget for memory recall (low, mid, high) + - excluded_models: List of model patterns to exclude from interception + - verbose: Enable verbose logging +""" + +from contextlib import contextmanager +from typing import Optional, List + +import litellm + +from .config import ( + configure, + get_config, + is_configured, + reset_config, + new_session, + set_session, + get_session, + set_entity, + get_entity, + HindsightConfig, + MemoryInjectionMode, +) +from .callbacks import ( + HindsightCallback, + get_callback, + cleanup_callback, +) +from .wrappers import ( + recall, + arecall, + RecallResult, + wrap_openai, + wrap_anthropic, + HindsightOpenAI, + HindsightAnthropic, +) + + +__version__ = "0.1.0" + +# Track whether we've registered with LiteLLM +_enabled = False + +# Store original functions for restoration +_original_completion = None +_original_acompletion = None + + +def _inject_memories(messages: List[dict]) -> List[dict]: + """Inject memories into messages list. + + Returns the modified messages list with memories injected into the system message. + """ + import logging + import requests + + if not is_configured(): + return messages + + config = get_config() + if not config or not config.enabled or not config.inject_memories: + return messages + + if not messages: + return messages + + # Extract user query from last user message + user_query = None + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content") + if isinstance(content, str): + user_query = content + break + + if not user_query: + return messages + + try: + # Build scoped bank_id + scoped_bank_id = config.bank_id + if config.entity_id: + scoped_bank_id = f"{config.bank_id}:{config.entity_id}" + + # Build recall request + url = f"{config.hindsight_api_url}/v1/default/banks/{scoped_bank_id}/memories/recall" + request_data = { + "query": user_query, + "budget": config.recall_budget or "mid", + "max_tokens": config.max_memory_tokens or 2000, + } + if config.fact_types: + request_data["types"] = config.fact_types + + headers = {"Content-Type": "application/json"} + if config.api_key: + headers["Authorization"] = f"Bearer {config.api_key}" + + response = requests.post(url, json=request_data, headers=headers, timeout=30) + response.raise_for_status() + response_data = response.json() + results = response_data.get("results", []) + + if not results: + return messages + + # Format memories + memory_lines = [] + for i, result in enumerate(results[:config.max_memories], 1): + text = result.get("text", "") + fact_type = result.get("type", result.get("fact_type", "world")) + if text: + type_label = fact_type.upper() if fact_type else "MEMORY" + memory_lines.append(f"{i}. [{type_label}] {text}") + + if not memory_lines: + return messages + + memory_context = ( + "# Relevant Memories\n" + "The following information from memory may be relevant:\n\n" + + "\n".join(memory_lines) + ) + + # Inject into messages + updated_messages = list(messages) + + # Find existing system message or create new one + found_system = False + for i, msg in enumerate(updated_messages): + if msg.get("role") == "system": + existing_content = msg.get("content", "") + updated_messages[i] = { + **msg, + "content": f"{existing_content}\n\n{memory_context}" + } + found_system = True + break + + if not found_system: + updated_messages.insert(0, { + "role": "system", + "content": memory_context + }) + + if config.verbose: + logger = logging.getLogger("hindsight_litellm") + logger.info(f"Injected {len(results)} memories into prompt") + + return updated_messages + + except Exception as e: + if config.verbose: + logging.getLogger("hindsight_litellm").warning(f"Failed to inject memories: {e}") + return messages + + +def _wrapped_completion(*args, **kwargs): + """Wrapper for litellm.completion that injects memories before the call.""" + # Inject memories into messages + if "messages" in kwargs: + kwargs["messages"] = _inject_memories(kwargs["messages"]) + elif args and len(args) > 1: + # messages might be second positional arg after model + args = list(args) + if isinstance(args[1], list): + args[1] = _inject_memories(args[1]) + args = tuple(args) + + # Call original + return _original_completion(*args, **kwargs) + + +async def _wrapped_acompletion(*args, **kwargs): + """Wrapper for litellm.acompletion that injects memories before the call.""" + # Inject memories into messages + if "messages" in kwargs: + kwargs["messages"] = _inject_memories(kwargs["messages"]) + elif args and len(args) > 1: + args = list(args) + if isinstance(args[1], list): + args[1] = _inject_memories(args[1]) + args = tuple(args) + + # Call original + return await _original_acompletion(*args, **kwargs) + + +def enable() -> None: + """Enable Hindsight memory integration with LiteLLM. + + This monkeypatches LiteLLM functions to: + 1. Inject relevant memories into prompts before LLM calls + 2. Store conversations to Hindsight after successful LLM calls + + Must be called after configure() to take effect. + + Example: + >>> from hindsight_litellm import configure, enable + >>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") + >>> enable() + >>> + >>> # Now all LiteLLM calls will have memory integration + >>> import litellm + >>> response = litellm.completion(model="gpt-4", messages=[...]) + """ + global _enabled, _original_completion, _original_acompletion + + if _enabled: + return # Already enabled + + if not is_configured(): + raise RuntimeError( + "Hindsight not configured. Call configure() before enable()." + ) + + # Store original functions and monkeypatch for memory injection + _original_completion = litellm.completion + _original_acompletion = litellm.acompletion + litellm.completion = _wrapped_completion + litellm.acompletion = _wrapped_acompletion + + # Get or create the callback instance for storing conversations + callback = get_callback() + + # Register callback using litellm.callbacks for conversation storage + if callback not in litellm.callbacks: + litellm.callbacks.append(callback) + + _enabled = True + + config = get_config() + if config and config.verbose: + print(f"Hindsight memory enabled for bank: {config.bank_id}") + + +def disable() -> None: + """Disable Hindsight memory integration with LiteLLM. + + This restores the original LiteLLM functions and removes callbacks, + stopping memory injection and conversation storage. + + Example: + >>> from hindsight_litellm import disable + >>> disable() # Stop memory integration + """ + global _enabled, _original_completion, _original_acompletion + + if not _enabled: + return # Already disabled + + # Restore original functions + if _original_completion is not None: + litellm.completion = _original_completion + _original_completion = None + if _original_acompletion is not None: + litellm.acompletion = _original_acompletion + _original_acompletion = None + + # Remove callback from litellm.callbacks + callback = get_callback() + if callback in litellm.callbacks: + litellm.callbacks.remove(callback) + + _enabled = False + + config = get_config() + if config and config.verbose: + print("Hindsight memory disabled") + + +def is_enabled() -> bool: + """Check if Hindsight memory integration is currently enabled. + + Returns: + True if enable() has been called and not subsequently disabled + """ + return _enabled + + +def cleanup() -> None: + """Clean up all Hindsight resources. + + This disables the integration and closes any open connections. + Call this when shutting down your application. + + Example: + >>> from hindsight_litellm import cleanup + >>> cleanup() # Clean up when done + """ + disable() + cleanup_callback() + reset_config() + + +# ============================================================================= +# Convenience wrappers - use hindsight_litellm.completion() directly +# ============================================================================= + +def completion(*args, **kwargs): + """Call LiteLLM completion with Hindsight memory integration. + + This is a convenience wrapper that delegates to litellm.completion(). + Memory injection and storage happen automatically if configured and enabled. + + Args: + *args: Positional arguments passed to litellm.completion() + **kwargs: Keyword arguments passed to litellm.completion() + + Returns: + LiteLLM ModelResponse object + + Example: + >>> import hindsight_litellm + >>> + >>> hindsight_litellm.configure( + ... hindsight_api_url="http://localhost:8888", + ... bank_id="my-agent", + ... ) + >>> hindsight_litellm.enable() + >>> + >>> # Use directly - no need to import litellm separately + >>> response = hindsight_litellm.completion( + ... model="gpt-4o-mini", + ... messages=[{"role": "user", "content": "Hello!"}] + ... ) + """ + return litellm.completion(*args, **kwargs) + + +async def acompletion(*args, **kwargs): + """Call LiteLLM async completion with Hindsight memory integration. + + This is a convenience wrapper that delegates to litellm.acompletion(). + Memory injection and storage happen automatically if configured and enabled. + + Args: + *args: Positional arguments passed to litellm.acompletion() + **kwargs: Keyword arguments passed to litellm.acompletion() + + Returns: + LiteLLM ModelResponse object + + Example: + >>> import hindsight_litellm + >>> import asyncio + >>> + >>> hindsight_litellm.configure( + ... hindsight_api_url="http://localhost:8888", + ... bank_id="my-agent", + ... ) + >>> hindsight_litellm.enable() + >>> + >>> async def main(): + ... response = await hindsight_litellm.acompletion( + ... model="gpt-4o-mini", + ... messages=[{"role": "user", "content": "Hello!"}] + ... ) + ... return response + >>> + >>> asyncio.run(main()) + """ + return await litellm.acompletion(*args, **kwargs) + + +@contextmanager +def hindsight_memory( + hindsight_api_url: str = "http://localhost:8888", + bank_id: Optional[str] = None, + api_key: Optional[str] = None, + entity_id: Optional[str] = None, + session_id: Optional[str] = None, + store_conversations: bool = True, + inject_memories: bool = True, + injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE, + max_memories: int = 10, + max_memory_tokens: int = 2000, + recall_budget: str = "mid", + fact_types: Optional[List[str]] = None, + document_id: Optional[str] = None, + excluded_models: Optional[List[str]] = None, + verbose: bool = False, +): + """Context manager for temporary Hindsight memory integration. + + Use this to enable memory integration for a specific block of code, + automatically cleaning up afterwards. + + Args: + hindsight_api_url: URL of the Hindsight API server + bank_id: Memory bank ID for memory operations (required) + api_key: Optional API key for Hindsight authentication + entity_id: User identifier for multi-user memory isolation + session_id: Session identifier for conversation grouping + store_conversations: Whether to store conversations + inject_memories: Whether to inject relevant memories + injection_mode: How to inject memories + max_memories: Maximum number of memories to inject + max_memory_tokens: Maximum tokens for memory context + recall_budget: Budget for memory recall (low, mid, high) + fact_types: List of fact types to filter (world, agent, opinion, observation) + document_id: Optional document ID for grouping conversations + excluded_models: List of model patterns to exclude + verbose: Enable verbose logging + + Example: + >>> from hindsight_litellm import hindsight_memory + >>> import litellm + >>> + >>> with hindsight_memory(bank_id="my-agent", entity_id="user-123"): + ... response = litellm.completion(model="gpt-4", messages=[...]) + >>> # Memory integration automatically disabled after context + """ + # Save previous state + was_enabled = is_enabled() + previous_config = get_config() + + try: + # Configure and enable + configure( + hindsight_api_url=hindsight_api_url, + bank_id=bank_id, + api_key=api_key, + entity_id=entity_id, + session_id=session_id, + store_conversations=store_conversations, + inject_memories=inject_memories, + injection_mode=injection_mode, + max_memories=max_memories, + max_memory_tokens=max_memory_tokens, + recall_budget=recall_budget, + fact_types=fact_types, + document_id=document_id, + excluded_models=excluded_models, + verbose=verbose, + ) + enable() + yield + finally: + # Restore previous state + disable() + if previous_config: + configure( + hindsight_api_url=previous_config.hindsight_api_url, + bank_id=previous_config.bank_id, + api_key=previous_config.api_key, + entity_id=previous_config.entity_id, + session_id=previous_config.session_id, + store_conversations=previous_config.store_conversations, + inject_memories=previous_config.inject_memories, + injection_mode=previous_config.injection_mode, + max_memories=previous_config.max_memories, + max_memory_tokens=previous_config.max_memory_tokens, + recall_budget=previous_config.recall_budget, + fact_types=previous_config.fact_types, + document_id=previous_config.document_id, + excluded_models=previous_config.excluded_models, + verbose=previous_config.verbose, + ) + if was_enabled: + enable() + else: + reset_config() + + +__all__ = [ + # Main API + "configure", + "enable", + "disable", + "is_enabled", + "cleanup", + "hindsight_memory", + # LLM completion wrappers (convenience) + "completion", + "acompletion", + # Session/Entity management + "new_session", + "set_session", + "get_session", + "set_entity", + "get_entity", + # Direct recall API + "recall", + "arecall", + "RecallResult", + # Native client wrappers + "wrap_openai", + "wrap_anthropic", + "HindsightOpenAI", + "HindsightAnthropic", + # Configuration + "get_config", + "is_configured", + "reset_config", + "HindsightConfig", + "MemoryInjectionMode", + # Callback (for advanced usage) + "HindsightCallback", + "get_callback", + "cleanup_callback", +] diff --git a/hindsight-integrations/litellm/hindsight_litellm/callbacks.py b/hindsight-integrations/litellm/hindsight_litellm/callbacks.py new file mode 100644 index 0000000000..c54e0ae554 --- /dev/null +++ b/hindsight-integrations/litellm/hindsight_litellm/callbacks.py @@ -0,0 +1,615 @@ +"""LiteLLM callback handlers for Hindsight memory integration. + +This module implements LiteLLM's CustomLogger interface to intercept +LLM calls and integrate with Hindsight for memory injection and storage. + +Uses direct HTTP calls via requests/httpx to avoid async event loop conflicts +when the hindsight_client's async methods are called from LiteLLM callbacks. +""" + +import logging +import fnmatch +import hashlib +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Set +import asyncio +import threading +import concurrent.futures + +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import ModelResponse + +from .config import get_config, is_configured, HindsightConfig, MemoryInjectionMode + +# Use requests for sync HTTP calls to avoid async event loop issues +try: + import requests + HAS_REQUESTS = True +except ImportError: + HAS_REQUESTS = False + +try: + import httpx + HAS_HTTPX = True +except ImportError: + HAS_HTTPX = False + + +logger = logging.getLogger(__name__) + +# Thread pool for running async operations in background +_executor = concurrent.futures.ThreadPoolExecutor(max_workers=4, thread_name_prefix="hindsight-") + + +class HindsightCallback(CustomLogger): + """LiteLLM custom logger that integrates with Hindsight memory system. + + This callback handler: + 1. Injects relevant memories into prompts before LLM calls + 2. Stores conversations to Hindsight after successful LLM calls + + Features: + - Works with 100+ LLM providers via LiteLLM + - Deduplication to avoid storing duplicate conversations + - Configurable memory injection modes + - Support for entity observations in recall + + Usage: + >>> from hindsight_litellm import configure, enable + >>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") + >>> enable() + >>> + >>> # Now all LiteLLM calls will have memory integration + >>> import litellm + >>> response = litellm.completion( + ... model="gpt-4", + ... messages=[{"role": "user", "content": "What did we discuss?"}] + ... ) + """ + + def __init__(self): + """Initialize the Hindsight callback handler.""" + super().__init__() + self._http_session = None + self._http_lock = threading.Lock() + # Track recently stored conversation hashes for deduplication + self._recent_hashes: Set[str] = set() + self._max_hash_cache = 1000 + + def _get_http_session(self): + """Get or create a requests Session (thread-safe).""" + if self._http_session is None: + with self._http_lock: + if self._http_session is None: + if HAS_REQUESTS: + self._http_session = requests.Session() + elif HAS_HTTPX: + self._http_session = httpx.Client(timeout=30.0) + else: + raise RuntimeError( + "Neither 'requests' nor 'httpx' is installed. " + "Please install one: pip install requests" + ) + return self._http_session + + def _http_post(self, url: str, json_data: dict, config: HindsightConfig) -> Optional[dict]: + """Make a synchronous HTTP POST request.""" + try: + session = self._get_http_session() + headers = {"Content-Type": "application/json"} + if config.api_key: + headers["Authorization"] = f"Bearer {config.api_key}" + + if HAS_REQUESTS: + response = session.post(url, json=json_data, headers=headers, timeout=30) + response.raise_for_status() + return response.json() + elif HAS_HTTPX: + response = session.post(url, json=json_data, headers=headers) + response.raise_for_status() + return response.json() + except Exception as e: + if config.verbose: + logger.warning(f"HTTP POST failed: {e}") + return None + + def _should_skip_model(self, model: str, config: HindsightConfig) -> bool: + """Check if this model should be excluded from interception.""" + for pattern in config.excluded_models: + if fnmatch.fnmatch(model.lower(), pattern.lower()): + return True + return False + + def _extract_user_query(self, messages: List[Dict[str, Any]]) -> Optional[str]: + """Extract the user's query from the last user message.""" + for msg in reversed(messages): + role = msg.get("role", "") + if role == "user": + content = msg.get("content") + if isinstance(content, str): + return content + elif isinstance(content, list): + # Handle structured content (e.g., vision messages) + text_parts = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + text_parts.append(item.get("text", "")) + if text_parts: + return " ".join(text_parts) + return None + + def _compute_conversation_hash( + self, + user_input: str, + assistant_output: str, + ) -> str: + """Compute a hash for deduplication.""" + content = f"{user_input.strip().lower()}|{assistant_output.strip().lower()}" + return hashlib.md5(content.encode()).hexdigest()[:16] + + def _is_duplicate(self, conv_hash: str) -> bool: + """Check if this conversation was recently stored.""" + if conv_hash in self._recent_hashes: + return True + + # Add to cache, evict oldest if full + self._recent_hashes.add(conv_hash) + if len(self._recent_hashes) > self._max_hash_cache: + # Remove oldest (arbitrary since set, but good enough) + self._recent_hashes.pop() + + return False + + def _format_memories( + self, + results: List[Any], + config: HindsightConfig + ) -> str: + """Format memory recall results into a context string. + + Results can be RecallResult objects (with .text, .type attributes) + or dicts (with get() method). + """ + if not results: + return "" + + memory_lines = [] + for i, result in enumerate(results[:config.max_memories], 1): + # Handle both RecallResult objects and dicts + if hasattr(result, 'text'): + text = result.text or "" + fact_type = getattr(result, 'type', 'world') or "world" + weight = getattr(result, 'weight', 0.0) or 0.0 + else: + text = result.get("text", "") + fact_type = result.get("type", result.get("fact_type", "world")) + weight = result.get("weight", 0.0) + + if text: + # Include metadata for context + type_label = fact_type.upper() if fact_type else "MEMORY" + line = f"{i}. [{type_label}] {text}" + if weight > 0 and config.verbose: + line += f" (relevance: {weight:.2f})" + memory_lines.append(line) + + if not memory_lines: + return "" + + return ( + "# Relevant Memories\n" + "The following information from memory may be relevant:\n\n" + + "\n".join(memory_lines) + ) + + def _inject_memories_into_messages( + self, + messages: List[Dict[str, Any]], + memory_context: str, + config: HindsightConfig, + ) -> List[Dict[str, Any]]: + """Inject memory context into the messages list.""" + if not memory_context: + return messages + + updated_messages = list(messages) # Make a copy + + if config.injection_mode == MemoryInjectionMode.SYSTEM_MESSAGE: + # Find existing system message or create new one + for i, msg in enumerate(updated_messages): + if msg.get("role") == "system": + # Append to existing system message + existing_content = msg.get("content", "") + updated_messages[i] = { + **msg, + "content": f"{existing_content}\n\n{memory_context}" + } + return updated_messages + + # No system message found, prepend one + updated_messages.insert(0, { + "role": "system", + "content": memory_context + }) + + elif config.injection_mode == MemoryInjectionMode.PREPEND_USER: + # Find the last user message and prepend context + for i in range(len(updated_messages) - 1, -1, -1): + if updated_messages[i].get("role") == "user": + original_content = updated_messages[i].get("content", "") + if isinstance(original_content, str): + updated_messages[i] = { + **updated_messages[i], + "content": f"{memory_context}\n\n---\n\n{original_content}" + } + break + + return updated_messages + + def _get_scoped_bank_id(self, config: HindsightConfig) -> str: + """Get bank_id with entity scoping if entity_id is set.""" + if config.entity_id: + return f"{config.bank_id}:{config.entity_id}" + return config.bank_id + + def _recall_memories_sync( + self, + query: str, + config: HindsightConfig + ) -> List[Dict[str, Any]]: + """Recall relevant memories from Hindsight (sync) using direct HTTP.""" + try: + scoped_bank_id = self._get_scoped_bank_id(config) + url = f"{config.hindsight_api_url}/v1/default/banks/{scoped_bank_id}/memories/recall" + + request_data = { + "query": query, + "budget": config.recall_budget or "mid", + "max_tokens": config.max_memory_tokens or 2000, + } + if config.fact_types: + request_data["types"] = config.fact_types + + response = self._http_post(url, request_data, config) + if response and "results" in response: + return response["results"] + return [] + + except Exception as e: + if config.verbose: + logger.warning(f"Failed to recall memories: {e}") + return [] + + async def _recall_memories_async( + self, + query: str, + config: HindsightConfig + ) -> List[Any]: + """Recall relevant memories from Hindsight (async). + + Uses thread pool executor with sync HTTP to avoid event loop conflicts. + """ + try: + loop = asyncio.get_running_loop() + results = await loop.run_in_executor( + _executor, + self._recall_memories_sync, + query, + config + ) + + return results if isinstance(results, list) else [] + + except Exception as e: + if config.verbose: + logger.warning(f"Failed to recall memories: {e}") + return [] + + def _store_conversation_sync( + self, + messages: List[Dict[str, Any]], + response: ModelResponse, + model: str, + config: HindsightConfig, + ) -> None: + """Store the conversation to Hindsight (sync) using direct HTTP.""" + try: + # Extract user input (last user message only) + user_input = self._extract_user_query(messages) + if not user_input: + return + + # Extract assistant response + assistant_output = "" + if response.choices and len(response.choices) > 0: + choice = response.choices[0] + if hasattr(choice, "message") and choice.message: + assistant_output = choice.message.content or "" + + if not assistant_output: + return + + # Skip if this looks like our injected memory context + if user_input.startswith("# Relevant Memories"): + return + + # Deduplication check + conv_hash = self._compute_conversation_hash(user_input, assistant_output) + if self._is_duplicate(conv_hash): + if config.verbose: + logger.debug(f"Skipping duplicate conversation: {conv_hash}") + return + + # Build conversation content for storage + # Format: Clear USER/ASSISTANT structure for Hindsight to extract facts from + conversation_text = f"USER: {user_input}\n\nASSISTANT: {assistant_output}" + + # Build metadata + metadata = { + "source": "litellm", + "model": model, + } + + # Add token usage if available + if hasattr(response, "usage") and response.usage: + if hasattr(response.usage, "total_tokens"): + metadata["tokens"] = str(response.usage.total_tokens) + + # Add session_id to metadata if set + if config.session_id: + metadata["session_id"] = config.session_id + + # Add entity_id to metadata if set + if config.entity_id: + metadata["entity_id"] = config.entity_id + + scoped_bank_id = self._get_scoped_bank_id(config) + url = f"{config.hindsight_api_url}/v1/default/banks/{scoped_bank_id}/memories" + + request_data = { + "items": [ + { + "content": conversation_text, + "context": f"conversation:litellm:{model}", + "metadata": metadata, + } + ], + } + if config.document_id: + request_data["document_id"] = config.document_id + + self._http_post(url, request_data, config) + + if config.verbose: + logger.info(f"Stored conversation to Hindsight bank: {config.bank_id}") + + except Exception as e: + if config.verbose: + logger.warning(f"Failed to store conversation: {e}") + + async def _store_conversation_async( + self, + messages: List[Dict[str, Any]], + response: ModelResponse, + model: str, + config: HindsightConfig, + ) -> None: + """Store the conversation to Hindsight (async). + + Uses thread pool executor with sync HTTP to avoid event loop conflicts. + """ + try: + loop = asyncio.get_running_loop() + await loop.run_in_executor( + _executor, + self._store_conversation_sync, + messages, + response, + model, + config + ) + except Exception as e: + if config.verbose: + logger.warning(f"Failed to store conversation: {e}") + + # ========== LiteLLM CustomLogger Interface ========== + + def log_pre_api_call( + self, + model: str, + messages: List[Dict[str, Any]], + kwargs: Dict[str, Any], + ) -> None: + """Called before making the API call (sync). + + This is where we inject memories into the messages. + """ + if not is_configured(): + return + + config = get_config() + if not config or not config.enabled or not config.inject_memories: + return + + if self._should_skip_model(model, config): + return + + # Extract user query + user_query = self._extract_user_query(messages) + if not user_query: + return + + # Recall relevant memories + memories = self._recall_memories_sync(user_query, config) + if not memories: + return + + # Format and inject memories + memory_context = self._format_memories(memories, config) + updated_messages = self._inject_memories_into_messages( + messages, memory_context, config + ) + + # Modify messages list IN-PLACE (don't just reassign kwargs) + messages.clear() + messages.extend(updated_messages) + + if config.verbose: + logger.info(f"Injected {len(memories)} memories into prompt") + + async def async_log_pre_api_call( + self, + model: str, + messages: List[Dict[str, Any]], + kwargs: Dict[str, Any], + ) -> None: + """Called before making the API call (async). + + This is where we inject memories into the messages. + """ + if not is_configured(): + return + + config = get_config() + if not config or not config.enabled or not config.inject_memories: + return + + if self._should_skip_model(model, config): + return + + # Extract user query + user_query = self._extract_user_query(messages) + if not user_query: + return + + # Recall relevant memories + memories = await self._recall_memories_async(user_query, config) + if not memories: + return + + # Format and inject memories + memory_context = self._format_memories(memories, config) + updated_messages = self._inject_memories_into_messages( + messages, memory_context, config + ) + + # Modify messages list IN-PLACE (don't just reassign kwargs) + messages.clear() + messages.extend(updated_messages) + + if config.verbose: + logger.info(f"Injected {len(memories)} memories into prompt") + + def log_success_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: float, + end_time: float, + ) -> None: + """Called after successful API call (sync). + + This is where we store the conversation. + """ + if not is_configured(): + return + + config = get_config() + if not config or not config.enabled or not config.store_conversations: + return + + model = kwargs.get("model", "unknown") + if self._should_skip_model(model, config): + return + + messages = kwargs.get("messages", []) + if not messages: + return + + # Store the conversation + self._store_conversation_sync(messages, response_obj, model, config) + + async def async_log_success_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: float, + end_time: float, + ) -> None: + """Called after successful API call (async). + + This is where we store the conversation. + """ + if not is_configured(): + return + + config = get_config() + if not config or not config.enabled or not config.store_conversations: + return + + model = kwargs.get("model", "unknown") + if self._should_skip_model(model, config): + return + + messages = kwargs.get("messages", []) + if not messages: + return + + # Store the conversation + await self._store_conversation_async(messages, response_obj, model, config) + + def log_failure_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: float, + end_time: float, + ) -> None: + """Called after failed API call (sync).""" + # We don't store failed conversations + pass + + async def async_log_failure_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: float, + end_time: float, + ) -> None: + """Called after failed API call (async).""" + # We don't store failed conversations + pass + + def close(self) -> None: + """Clean up resources.""" + with self._http_lock: + if self._http_session is not None: + try: + if HAS_REQUESTS: + self._http_session.close() + elif HAS_HTTPX: + self._http_session.close() + except Exception: + pass + self._http_session = None + self._recent_hashes.clear() + + +# Global callback instance +_callback: Optional[HindsightCallback] = None + + +def get_callback() -> HindsightCallback: + """Get the global callback instance, creating it if necessary.""" + global _callback + if _callback is None: + _callback = HindsightCallback() + return _callback + + +def cleanup_callback() -> None: + """Clean up the global callback instance.""" + global _callback + if _callback is not None: + _callback.close() + _callback = None diff --git a/hindsight-integrations/litellm/hindsight_litellm/config.py b/hindsight-integrations/litellm/hindsight_litellm/config.py new file mode 100644 index 0000000000..8a112c36d0 --- /dev/null +++ b/hindsight-integrations/litellm/hindsight_litellm/config.py @@ -0,0 +1,267 @@ +"""Global configuration for Hindsight-LiteLLM integration.""" + +from typing import Optional, List +from dataclasses import dataclass, field +from enum import Enum +from uuid import uuid4 + + +class MemoryInjectionMode(str, Enum): + """How memories should be injected into the prompt.""" + SYSTEM_MESSAGE = "system_message" # Add as system message + PREPEND_USER = "prepend_user" # Prepend to user message + DISABLED = "disabled" # Don't inject memories + + +@dataclass +class HindsightConfig: + """Configuration for Hindsight integration with LiteLLM. + + Attributes: + hindsight_api_url: URL of the Hindsight API server + bank_id: Memory bank ID for memory operations (required) + api_key: Optional API key for Hindsight authentication + entity_id: User/entity identifier for memory scoping (multi-user support) + session_id: Session identifier for conversation grouping + store_conversations: Whether to store conversations to Hindsight + inject_memories: Whether to inject relevant memories into prompts + injection_mode: How to inject memories (system_message or prepend_user) + max_memories: Maximum number of memories to inject + max_memory_tokens: Maximum tokens for injected memory context + recall_budget: Budget level for memory recall (low, mid, high) + fact_types: List of fact types to filter recall (world, agent, opinion, observation) + document_id: Optional document ID for grouping stored conversations + enabled: Master switch to enable/disable Hindsight integration + excluded_models: List of model patterns to exclude from interception + verbose: Enable verbose logging + """ + + hindsight_api_url: str = "http://localhost:8888" + bank_id: Optional[str] = None + api_key: Optional[str] = None + entity_id: Optional[str] = None # User identifier for multi-user memory isolation + session_id: Optional[str] = None # Session identifier for conversation grouping + store_conversations: bool = True + inject_memories: bool = True + injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE + max_memories: int = 10 + max_memory_tokens: int = 2000 + recall_budget: str = "mid" # low, mid, high + fact_types: Optional[List[str]] = None # world, agent, opinion, observation + document_id: Optional[str] = None + enabled: bool = True + excluded_models: List[str] = field(default_factory=list) + verbose: bool = False + + +# Global configuration instance +_global_config: Optional[HindsightConfig] = None + + +def configure( + hindsight_api_url: str = "http://localhost:8888", + bank_id: Optional[str] = None, + api_key: Optional[str] = None, + entity_id: Optional[str] = None, + session_id: Optional[str] = None, + store_conversations: bool = True, + inject_memories: bool = True, + injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE, + max_memories: int = 10, + max_memory_tokens: int = 2000, + recall_budget: str = "mid", + fact_types: Optional[List[str]] = None, + document_id: Optional[str] = None, + enabled: bool = True, + excluded_models: Optional[List[str]] = None, + verbose: bool = False, +) -> HindsightConfig: + """Configure global Hindsight integration settings for LiteLLM. + + This function sets up the global configuration that will be used by the + LiteLLM callbacks to inject memories and store conversations. + + Args: + hindsight_api_url: URL of the Hindsight API server + bank_id: Memory bank ID for memory operations (required) + api_key: Optional API key for Hindsight authentication + entity_id: User/entity identifier for multi-user memory isolation + session_id: Session identifier for conversation grouping + store_conversations: Whether to store conversations to Hindsight + inject_memories: Whether to inject relevant memories into prompts + injection_mode: How to inject memories into the prompt + max_memories: Maximum number of memories to inject + max_memory_tokens: Maximum tokens for injected memory context + recall_budget: Budget level for memory recall (low, mid, high) + fact_types: List of fact types to filter (world, agent, opinion, observation) + document_id: Optional document ID for grouping stored conversations + enabled: Master switch to enable/disable Hindsight integration + excluded_models: List of model patterns to exclude from interception + verbose: Enable verbose logging + + Returns: + The configured HindsightConfig instance + + Example: + >>> from hindsight_litellm import configure, enable + >>> configure( + ... hindsight_api_url="http://localhost:8888", + ... bank_id="my-agent", + ... entity_id="user-123", # Multi-user support + ... store_conversations=True, + ... inject_memories=True, + ... ) + >>> enable() # Register callbacks with LiteLLM + """ + global _global_config + + _global_config = HindsightConfig( + hindsight_api_url=hindsight_api_url, + bank_id=bank_id, + api_key=api_key, + entity_id=entity_id, + session_id=session_id, + store_conversations=store_conversations, + inject_memories=inject_memories, + injection_mode=injection_mode, + max_memories=max_memories, + max_memory_tokens=max_memory_tokens, + recall_budget=recall_budget, + fact_types=fact_types, + document_id=document_id, + enabled=enabled, + excluded_models=excluded_models or [], + verbose=verbose, + ) + + return _global_config + + +def get_config() -> Optional[HindsightConfig]: + """Get the current global configuration. + + Returns: + The current HindsightConfig instance, or None if not configured + """ + return _global_config + + +def is_configured() -> bool: + """Check if Hindsight has been configured. + + Returns: + True if configure() has been called with a valid bank_id + """ + return ( + _global_config is not None + and _global_config.enabled + and _global_config.bank_id is not None + ) + + +def reset_config() -> None: + """Reset the global configuration to None.""" + global _global_config + _global_config = None + + +def new_session() -> str: + """Generate and set a new session ID. + + Creates a new UUID-based session ID and updates the global config. + This is useful for starting fresh conversation threads. + + Returns: + The new session ID string + + Raises: + RuntimeError: If Hindsight has not been configured + + Example: + >>> from hindsight_litellm import configure, new_session + >>> configure(bank_id="my-agent") + >>> session_id = new_session() + >>> print(f"Started new session: {session_id}") + """ + global _global_config + + if _global_config is None: + raise RuntimeError( + "Hindsight not configured. Call configure() before new_session()." + ) + + new_id = str(uuid4()) + _global_config.session_id = new_id + return new_id + + +def set_session(session_id: str) -> None: + """Set a specific session ID. + + Use this to resume a previous conversation session. + + Args: + session_id: The session ID to set + + Raises: + RuntimeError: If Hindsight has not been configured + + Example: + >>> from hindsight_litellm import configure, set_session + >>> configure(bank_id="my-agent") + >>> set_session("previous-session-id") # Resume conversation + """ + global _global_config + + if _global_config is None: + raise RuntimeError( + "Hindsight not configured. Call configure() before set_session()." + ) + + _global_config.session_id = session_id + + +def get_session() -> Optional[str]: + """Get the current session ID. + + Returns: + The current session ID, or None if not set + """ + if _global_config is None: + return None + return _global_config.session_id + + +def set_entity(entity_id: str) -> None: + """Set the entity ID for multi-user memory isolation. + + Args: + entity_id: The entity/user identifier + + Raises: + RuntimeError: If Hindsight has not been configured + + Example: + >>> from hindsight_litellm import configure, set_entity + >>> configure(bank_id="my-agent") + >>> set_entity("user-123") # Switch to this user's memories + """ + global _global_config + + if _global_config is None: + raise RuntimeError( + "Hindsight not configured. Call configure() before set_entity()." + ) + + _global_config.entity_id = entity_id + + +def get_entity() -> Optional[str]: + """Get the current entity ID. + + Returns: + The current entity ID, or None if not set + """ + if _global_config is None: + return None + return _global_config.entity_id diff --git a/hindsight-integrations/litellm/hindsight_litellm/wrappers.py b/hindsight-integrations/litellm/hindsight_litellm/wrappers.py new file mode 100644 index 0000000000..9021734bcc --- /dev/null +++ b/hindsight-integrations/litellm/hindsight_litellm/wrappers.py @@ -0,0 +1,708 @@ +"""Native client wrappers for Hindsight memory integration. + +This module provides wrappers for native LLM client SDKs (OpenAI, Anthropic) +that automatically integrate with Hindsight for memory injection and storage. + +This is an alternative to the LiteLLM callback approach, providing direct +integration with native client libraries. +""" + +import logging +from typing import Any, Dict, List, Optional, Union +from dataclasses import dataclass + +from .config import get_config, is_configured, HindsightConfig + + +logger = logging.getLogger(__name__) + + +@dataclass +class RecallResult: + """A single memory recall result.""" + text: str + fact_type: str + weight: float + metadata: Optional[Dict[str, Any]] = None + + def __str__(self) -> str: + return self.text + + +def recall( + query: str, + limit: int = 10, + bank_id: Optional[str] = None, + entity_id: Optional[str] = None, + fact_types: Optional[List[str]] = None, + budget: Optional[str] = None, + max_tokens: Optional[int] = None, + hindsight_api_url: Optional[str] = None, +) -> List[RecallResult]: + """Recall memories from Hindsight. + + This function allows you to manually query memories without making an LLM call. + Useful for debugging, building custom UIs, or pre-filtering memories. + + Args: + query: The query string to search memories for + limit: Maximum number of memories to return (default: 10) + bank_id: Override the configured bank_id + entity_id: Override the configured entity_id for multi-user isolation + fact_types: Filter by fact types (world, agent, opinion, observation) + budget: Recall budget level (low, mid, high) + max_tokens: Maximum tokens for memory context + hindsight_api_url: Override the configured API URL + + Returns: + List of RecallResult objects containing matched memories + + Raises: + RuntimeError: If Hindsight is not configured and no overrides provided + + Example: + >>> from hindsight_litellm import configure, recall + >>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") + >>> + >>> # Query memories + >>> memories = recall("what projects am I working on?", limit=5) + >>> for m in memories: + ... print(f"- [{m.fact_type}] {m.text}") + - [world] User is building a FastAPI project + - [opinion] User prefers Python over JavaScript + """ + # Get config or use overrides + config = get_config() + + api_url = hindsight_api_url or (config.hindsight_api_url if config else None) + target_bank_id = bank_id or (config.bank_id if config else None) + target_entity_id = entity_id or (config.entity_id if config else None) + target_fact_types = fact_types or (config.fact_types if config else None) + target_budget = budget or (config.recall_budget if config else "mid") + target_max_tokens = max_tokens or (config.max_memory_tokens if config else 2000) + + if not api_url or not target_bank_id: + raise RuntimeError( + "Hindsight not configured. Call configure() or provide bank_id and hindsight_api_url." + ) + + try: + from hindsight_client import Hindsight + + client = Hindsight(base_url=api_url, timeout=30.0) + + # Build bank_id with entity scoping if entity_id is set + scoped_bank_id = target_bank_id + if target_entity_id: + scoped_bank_id = f"{target_bank_id}:{target_entity_id}" + + # Call recall API + results = client.recall( + bank_id=scoped_bank_id, + query=query, + types=target_fact_types, + budget=target_budget, + max_tokens=target_max_tokens, + ) + + # Convert to RecallResult objects + recall_results = [] + if results: + for r in results[:limit]: + if hasattr(r, 'text'): + # Object with attributes + fact_type = getattr(r, 'type', None) or getattr(r, 'fact_type', 'unknown') + recall_results.append(RecallResult( + text=r.text, + fact_type=fact_type, + weight=getattr(r, 'weight', 0.0), + metadata=getattr(r, 'metadata', None), + )) + elif isinstance(r, dict): + # Dict from API response - API returns 'type' not 'fact_type' + fact_type = r.get('type') or r.get('fact_type', 'unknown') + recall_results.append(RecallResult( + text=r.get('text', str(r)), + fact_type=fact_type, + weight=r.get('weight', 0.0), + metadata=r.get('metadata'), + )) + + return recall_results + + except ImportError as e: + raise RuntimeError(f"hindsight-client not installed: {e}") + except Exception as e: + if config and config.verbose: + logger.warning(f"Failed to recall memories: {e}") + raise + + +async def arecall( + query: str, + limit: int = 10, + bank_id: Optional[str] = None, + entity_id: Optional[str] = None, + fact_types: Optional[List[str]] = None, + budget: Optional[str] = None, + max_tokens: Optional[int] = None, + hindsight_api_url: Optional[str] = None, +) -> List[RecallResult]: + """Async version of recall(). + + See recall() for full documentation. + """ + import asyncio + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: recall( + query=query, + limit=limit, + bank_id=bank_id, + entity_id=entity_id, + fact_types=fact_types, + budget=budget, + max_tokens=max_tokens, + hindsight_api_url=hindsight_api_url, + ) + ) + + +class HindsightOpenAI: + """Wrapper for OpenAI client with Hindsight memory integration. + + This wraps the native OpenAI client to automatically inject memories + and store conversations. + + Example: + >>> from openai import OpenAI + >>> from hindsight_litellm import wrap_openai + >>> + >>> client = OpenAI() + >>> wrapped = wrap_openai(client, bank_id="my-agent") + >>> + >>> response = wrapped.chat.completions.create( + ... model="gpt-4", + ... messages=[{"role": "user", "content": "What do you know about me?"}] + ... ) + """ + + def __init__( + self, + client: Any, + bank_id: str, + hindsight_api_url: str = "http://localhost:8888", + entity_id: Optional[str] = None, + session_id: Optional[str] = None, + store_conversations: bool = True, + inject_memories: bool = True, + max_memories: int = 10, + recall_budget: str = "mid", + verbose: bool = False, + ): + """Initialize the wrapped OpenAI client. + + Args: + client: The OpenAI client instance to wrap + bank_id: Memory bank ID for memory operations + hindsight_api_url: URL of the Hindsight API server + entity_id: User identifier for multi-user memory isolation + session_id: Session identifier for conversation grouping + store_conversations: Whether to store conversations + inject_memories: Whether to inject relevant memories + max_memories: Maximum number of memories to inject + recall_budget: Budget level for memory recall (low, mid, high) + verbose: Enable verbose logging + """ + self._client = client + self._bank_id = bank_id + self._api_url = hindsight_api_url + self._entity_id = entity_id + self._session_id = session_id + self._store_conversations = store_conversations + self._inject_memories = inject_memories + self._max_memories = max_memories + self._recall_budget = recall_budget + self._verbose = verbose + self._hindsight_client = None + + # Create wrapped chat.completions interface + self.chat = _WrappedChat(self) + + def _get_hindsight_client(self): + """Get or create the Hindsight client.""" + if self._hindsight_client is None: + from hindsight_client import Hindsight + self._hindsight_client = Hindsight( + base_url=self._api_url, + timeout=30.0, + ) + return self._hindsight_client + + def _get_scoped_bank_id(self) -> str: + """Get bank_id with entity scoping if set.""" + if self._entity_id: + return f"{self._bank_id}:{self._entity_id}" + return self._bank_id + + def _recall_memories(self, query: str) -> str: + """Recall and format memories for injection.""" + if not self._inject_memories: + return "" + + try: + client = self._get_hindsight_client() + results = client.recall( + bank_id=self._get_scoped_bank_id(), + query=query, + budget=self._recall_budget, + max_tokens=self._max_memories * 200, + ) + + if not results: + return "" + + memory_lines = [] + for i, r in enumerate(results[:self._max_memories], 1): + text = r.text if hasattr(r, 'text') else str(r) + fact_type = r.fact_type if hasattr(r, 'fact_type') else 'memory' + memory_lines.append(f"{i}. [{fact_type.upper()}] {text}") + + if not memory_lines: + return "" + + return ( + "# Relevant Memories\n" + "The following information from memory may be relevant:\n\n" + + "\n".join(memory_lines) + ) + + except Exception as e: + if self._verbose: + logger.warning(f"Failed to recall memories: {e}") + return "" + + def _store_conversation(self, user_input: str, assistant_output: str, model: str): + """Store the conversation to Hindsight.""" + if not self._store_conversations: + return + + try: + client = self._get_hindsight_client() + conversation_text = f"USER: {user_input}\n\nASSISTANT: {assistant_output}" + + metadata = { + "source": "openai-wrapper", + "model": model, + } + if self._session_id: + metadata["session_id"] = self._session_id + + client.retain( + bank_id=self._get_scoped_bank_id(), + content=conversation_text, + context=f"conversation:openai:{model}", + metadata=metadata, + ) + + if self._verbose: + logger.info(f"Stored conversation to Hindsight") + + except Exception as e: + if self._verbose: + logger.warning(f"Failed to store conversation: {e}") + + # Proxy other attributes to the underlying client + def __getattr__(self, name: str) -> Any: + return getattr(self._client, name) + + +class _WrappedChat: + """Wrapped chat interface for OpenAI client.""" + + def __init__(self, wrapper: HindsightOpenAI): + self._wrapper = wrapper + self.completions = _WrappedCompletions(wrapper) + + +class _WrappedCompletions: + """Wrapped completions interface for OpenAI client.""" + + def __init__(self, wrapper: HindsightOpenAI): + self._wrapper = wrapper + + def create(self, **kwargs) -> Any: + """Create a chat completion with memory integration.""" + messages = list(kwargs.get("messages", [])) + model = kwargs.get("model", "gpt-4") + + # Extract user query + user_query = None + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content") + if isinstance(content, str): + user_query = content + break + + # Inject memories + if user_query and self._wrapper._inject_memories: + memory_context = self._wrapper._recall_memories(user_query) + if memory_context: + # Find system message and append, or prepend new one + found_system = False + for i, msg in enumerate(messages): + if msg.get("role") == "system": + messages[i] = { + **msg, + "content": f"{msg.get('content', '')}\n\n{memory_context}" + } + found_system = True + break + + if not found_system: + messages.insert(0, {"role": "system", "content": memory_context}) + + kwargs["messages"] = messages + + # Make the actual API call + response = self._wrapper._client.chat.completions.create(**kwargs) + + # Store conversation + if user_query and self._wrapper._store_conversations: + if response.choices and response.choices[0].message: + assistant_output = response.choices[0].message.content or "" + if assistant_output: + self._wrapper._store_conversation(user_query, assistant_output, model) + + return response + + +class HindsightAnthropic: + """Wrapper for Anthropic client with Hindsight memory integration. + + This wraps the native Anthropic client to automatically inject memories + and store conversations. + + Example: + >>> from anthropic import Anthropic + >>> from hindsight_litellm import wrap_anthropic + >>> + >>> client = Anthropic() + >>> wrapped = wrap_anthropic(client, bank_id="my-agent") + >>> + >>> response = wrapped.messages.create( + ... model="claude-3-5-sonnet-20241022", + ... max_tokens=1024, + ... messages=[{"role": "user", "content": "What do you know about me?"}] + ... ) + """ + + def __init__( + self, + client: Any, + bank_id: str, + hindsight_api_url: str = "http://localhost:8888", + entity_id: Optional[str] = None, + session_id: Optional[str] = None, + store_conversations: bool = True, + inject_memories: bool = True, + max_memories: int = 10, + recall_budget: str = "mid", + verbose: bool = False, + ): + """Initialize the wrapped Anthropic client. + + Args: + client: The Anthropic client instance to wrap + bank_id: Memory bank ID for memory operations + hindsight_api_url: URL of the Hindsight API server + entity_id: User identifier for multi-user memory isolation + session_id: Session identifier for conversation grouping + store_conversations: Whether to store conversations + inject_memories: Whether to inject relevant memories + max_memories: Maximum number of memories to inject + recall_budget: Budget level for memory recall (low, mid, high) + verbose: Enable verbose logging + """ + self._client = client + self._bank_id = bank_id + self._api_url = hindsight_api_url + self._entity_id = entity_id + self._session_id = session_id + self._store_conversations = store_conversations + self._inject_memories = inject_memories + self._max_memories = max_memories + self._recall_budget = recall_budget + self._verbose = verbose + self._hindsight_client = None + + # Create wrapped messages interface + self.messages = _WrappedAnthropicMessages(self) + + def _get_hindsight_client(self): + """Get or create the Hindsight client.""" + if self._hindsight_client is None: + from hindsight_client import Hindsight + self._hindsight_client = Hindsight( + base_url=self._api_url, + timeout=30.0, + ) + return self._hindsight_client + + def _get_scoped_bank_id(self) -> str: + """Get bank_id with entity scoping if set.""" + if self._entity_id: + return f"{self._bank_id}:{self._entity_id}" + return self._bank_id + + def _recall_memories(self, query: str) -> str: + """Recall and format memories for injection.""" + if not self._inject_memories: + return "" + + try: + client = self._get_hindsight_client() + results = client.recall( + bank_id=self._get_scoped_bank_id(), + query=query, + budget=self._recall_budget, + max_tokens=self._max_memories * 200, + ) + + if not results: + return "" + + memory_lines = [] + for i, r in enumerate(results[:self._max_memories], 1): + text = r.text if hasattr(r, 'text') else str(r) + fact_type = r.fact_type if hasattr(r, 'fact_type') else 'memory' + memory_lines.append(f"{i}. [{fact_type.upper()}] {text}") + + if not memory_lines: + return "" + + return ( + "# Relevant Memories\n" + "The following information from memory may be relevant:\n\n" + + "\n".join(memory_lines) + ) + + except Exception as e: + if self._verbose: + logger.warning(f"Failed to recall memories: {e}") + return "" + + def _store_conversation(self, user_input: str, assistant_output: str, model: str): + """Store the conversation to Hindsight.""" + if not self._store_conversations: + return + + try: + client = self._get_hindsight_client() + conversation_text = f"USER: {user_input}\n\nASSISTANT: {assistant_output}" + + metadata = { + "source": "anthropic-wrapper", + "model": model, + } + if self._session_id: + metadata["session_id"] = self._session_id + + client.retain( + bank_id=self._get_scoped_bank_id(), + content=conversation_text, + context=f"conversation:anthropic:{model}", + metadata=metadata, + ) + + if self._verbose: + logger.info(f"Stored conversation to Hindsight") + + except Exception as e: + if self._verbose: + logger.warning(f"Failed to store conversation: {e}") + + # Proxy other attributes to the underlying client + def __getattr__(self, name: str) -> Any: + return getattr(self._client, name) + + +class _WrappedAnthropicMessages: + """Wrapped messages interface for Anthropic client.""" + + def __init__(self, wrapper: HindsightAnthropic): + self._wrapper = wrapper + + def create(self, **kwargs) -> Any: + """Create a message with memory integration.""" + messages = list(kwargs.get("messages", [])) + model = kwargs.get("model", "claude-3-5-sonnet-20241022") + system = kwargs.get("system", "") + + # Extract user query + user_query = None + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content") + if isinstance(content, str): + user_query = content + break + elif isinstance(content, list): + # Handle structured content + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + user_query = item.get("text", "") + break + if user_query: + break + + # Inject memories into system prompt + if user_query and self._wrapper._inject_memories: + memory_context = self._wrapper._recall_memories(user_query) + if memory_context: + if system: + kwargs["system"] = f"{system}\n\n{memory_context}" + else: + kwargs["system"] = memory_context + + # Make the actual API call + response = self._wrapper._client.messages.create(**kwargs) + + # Store conversation + if user_query and self._wrapper._store_conversations: + if response.content: + assistant_output = "" + for block in response.content: + if hasattr(block, 'text'): + assistant_output += block.text + if assistant_output: + self._wrapper._store_conversation(user_query, assistant_output, model) + + return response + + +def wrap_openai( + client: Any, + bank_id: str, + hindsight_api_url: str = "http://localhost:8888", + entity_id: Optional[str] = None, + session_id: Optional[str] = None, + store_conversations: bool = True, + inject_memories: bool = True, + max_memories: int = 10, + recall_budget: str = "mid", + verbose: bool = False, +) -> HindsightOpenAI: + """Wrap an OpenAI client with Hindsight memory integration. + + This creates a wrapped client that automatically injects memories + and stores conversations when making chat completion calls. + + Args: + client: The OpenAI client instance to wrap + bank_id: Memory bank ID for memory operations + hindsight_api_url: URL of the Hindsight API server + entity_id: User identifier for multi-user memory isolation + session_id: Session identifier for conversation grouping + store_conversations: Whether to store conversations + inject_memories: Whether to inject relevant memories + max_memories: Maximum number of memories to inject + recall_budget: Budget level for memory recall (low, mid, high) + verbose: Enable verbose logging + + Returns: + Wrapped OpenAI client with memory integration + + Example: + >>> from openai import OpenAI + >>> from hindsight_litellm import wrap_openai + >>> + >>> client = OpenAI() + >>> wrapped = wrap_openai( + ... client, + ... bank_id="my-agent", + ... entity_id="user-123", # Multi-user support + ... ) + >>> + >>> response = wrapped.chat.completions.create( + ... model="gpt-4", + ... messages=[{"role": "user", "content": "What do you know about me?"}] + ... ) + """ + return HindsightOpenAI( + client=client, + bank_id=bank_id, + hindsight_api_url=hindsight_api_url, + entity_id=entity_id, + session_id=session_id, + store_conversations=store_conversations, + inject_memories=inject_memories, + max_memories=max_memories, + recall_budget=recall_budget, + verbose=verbose, + ) + + +def wrap_anthropic( + client: Any, + bank_id: str, + hindsight_api_url: str = "http://localhost:8888", + entity_id: Optional[str] = None, + session_id: Optional[str] = None, + store_conversations: bool = True, + inject_memories: bool = True, + max_memories: int = 10, + recall_budget: str = "mid", + verbose: bool = False, +) -> HindsightAnthropic: + """Wrap an Anthropic client with Hindsight memory integration. + + This creates a wrapped client that automatically injects memories + and stores conversations when making message calls. + + Args: + client: The Anthropic client instance to wrap + bank_id: Memory bank ID for memory operations + hindsight_api_url: URL of the Hindsight API server + entity_id: User identifier for multi-user memory isolation + session_id: Session identifier for conversation grouping + store_conversations: Whether to store conversations + inject_memories: Whether to inject relevant memories + max_memories: Maximum number of memories to inject + recall_budget: Budget level for memory recall (low, mid, high) + verbose: Enable verbose logging + + Returns: + Wrapped Anthropic client with memory integration + + Example: + >>> from anthropic import Anthropic + >>> from hindsight_litellm import wrap_anthropic + >>> + >>> client = Anthropic() + >>> wrapped = wrap_anthropic( + ... client, + ... bank_id="my-agent", + ... entity_id="user-123", # Multi-user support + ... ) + >>> + >>> response = wrapped.messages.create( + ... model="claude-3-5-sonnet-20241022", + ... max_tokens=1024, + ... messages=[{"role": "user", "content": "What do you know about me?"}] + ... ) + """ + return HindsightAnthropic( + client=client, + bank_id=bank_id, + hindsight_api_url=hindsight_api_url, + entity_id=entity_id, + session_id=session_id, + store_conversations=store_conversations, + inject_memories=inject_memories, + max_memories=max_memories, + recall_budget=recall_budget, + verbose=verbose, + ) diff --git a/hindsight-integrations/litellm/pyproject.toml b/hindsight-integrations/litellm/pyproject.toml new file mode 100644 index 0000000000..b3b4c3626c --- /dev/null +++ b/hindsight-integrations/litellm/pyproject.toml @@ -0,0 +1,59 @@ +[project] +name = "hindsight-litellm" +version = "0.1.0" +description = "Universal LLM memory integration via LiteLLM - works with 100+ providers" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [ + { name = "Vectorize", email = "support@vectorize.io" } +] +keywords = [ + "ai", + "memory", + "llm", + "litellm", + "openai", + "anthropic", + "groq", + "langchain", + "agents", + "hindsight", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +dependencies = [ + "litellm>=1.40.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "pytest-mock>=3.10.0", +] + +[project.urls] +Homepage = "https://github.com/vectorize-io/hindsight" +Documentation = "https://github.com/vectorize-io/hindsight/tree/main/hindsight-integrations/litellm" +Repository = "https://github.com/vectorize-io/hindsight" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["hindsight_litellm"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/hindsight-integrations/litellm/tests/__init__.py b/hindsight-integrations/litellm/tests/__init__.py new file mode 100644 index 0000000000..abc1ec6c35 --- /dev/null +++ b/hindsight-integrations/litellm/tests/__init__.py @@ -0,0 +1 @@ +# Tests for hindsight-litellm diff --git a/hindsight-integrations/litellm/tests/test_integration.py b/hindsight-integrations/litellm/tests/test_integration.py new file mode 100644 index 0000000000..164d4f8aac --- /dev/null +++ b/hindsight-integrations/litellm/tests/test_integration.py @@ -0,0 +1,471 @@ +"""Integration tests for hindsight-litellm.""" + +import pytest +from unittest.mock import Mock, patch, MagicMock +from typing import List, Dict, Any + +from hindsight_litellm import ( + configure, + enable, + disable, + is_enabled, + cleanup, + get_config, + is_configured, + reset_config, + HindsightConfig, + MemoryInjectionMode, +) +from hindsight_litellm.callbacks import HindsightCallback, get_callback, cleanup_callback + + +class TestConfiguration: + """Tests for configuration management.""" + + def setup_method(self): + """Reset config before each test.""" + reset_config() + disable() + + def teardown_method(self): + """Clean up after each test.""" + cleanup() + + def test_configure_creates_config(self): + """Test that configure creates a config object.""" + config = configure( + bank_id="test-agent", + hindsight_api_url="http://localhost:8888", + ) + + assert config is not None + assert config.bank_id == "test-agent" + assert config.hindsight_api_url == "http://localhost:8888" + assert config.enabled is True + + def test_configure_with_all_options(self): + """Test configure with all options.""" + config = configure( + hindsight_api_url="http://custom:9999", + bank_id="custom-agent", + api_key="secret-key", + store_conversations=False, + inject_memories=False, + injection_mode=MemoryInjectionMode.PREPEND_USER, + max_memories=5, + max_memory_tokens=1000, + recall_budget="high", + fact_types=["world", "opinion"], + document_id="doc-123", + enabled=True, + excluded_models=["gpt-3.5*"], + verbose=True, + ) + + assert config.hindsight_api_url == "http://custom:9999" + assert config.bank_id == "custom-agent" + assert config.api_key == "secret-key" + assert config.store_conversations is False + assert config.inject_memories is False + assert config.injection_mode == MemoryInjectionMode.PREPEND_USER + assert config.max_memories == 5 + assert config.max_memory_tokens == 1000 + assert config.recall_budget == "high" + assert config.fact_types == ["world", "opinion"] + assert config.document_id == "doc-123" + assert config.excluded_models == ["gpt-3.5*"] + assert config.verbose is True + + def test_is_configured_without_bank_id(self): + """Test is_configured returns False without bank_id.""" + configure(hindsight_api_url="http://localhost:8888") + assert is_configured() is False + + def test_is_configured_with_bank_id(self): + """Test is_configured returns True with bank_id.""" + configure(bank_id="test-agent") + assert is_configured() is True + + def test_reset_config(self): + """Test reset_config clears the configuration.""" + configure(bank_id="test-agent") + assert is_configured() is True + + reset_config() + assert get_config() is None + assert is_configured() is False + + +class TestEnableDisable: + """Tests for enable/disable functionality.""" + + def setup_method(self): + """Reset state before each test.""" + cleanup() + + def teardown_method(self): + """Clean up after each test.""" + cleanup() + + def test_enable_without_config_raises(self): + """Test enable raises error without configuration.""" + with pytest.raises(RuntimeError, match="not configured"): + enable() + + def test_enable_registers_callback(self): + """Test enable registers callback with LiteLLM.""" + import litellm + + configure(bank_id="test-agent") + enable() + + callback = get_callback() + assert callback in litellm.callbacks + assert is_enabled() is True + + def test_disable_removes_callback(self): + """Test disable removes callback from LiteLLM.""" + import litellm + + configure(bank_id="test-agent") + enable() + assert is_enabled() is True + + disable() + callback = get_callback() + assert callback not in litellm.callbacks + assert is_enabled() is False + + def test_enable_idempotent(self): + """Test enable is idempotent (can be called multiple times).""" + import litellm + + configure(bank_id="test-agent") + + # Enable multiple times + enable() + enable() + enable() + + # Should only have one callback + callback = get_callback() + assert litellm.callbacks.count(callback) == 1 + + +class TestCallback: + """Tests for the HindsightCallback class.""" + + def setup_method(self): + """Reset state before each test.""" + cleanup() + + def teardown_method(self): + """Clean up after each test.""" + cleanup() + + def test_extract_user_query_simple(self): + """Test extracting user query from simple messages.""" + callback = HindsightCallback() + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + query = callback._extract_user_query(messages) + assert query == "What is the capital of France?" + + def test_extract_user_query_from_last_user_message(self): + """Test extracting query from last user message.""" + callback = HindsightCallback() + messages = [ + {"role": "user", "content": "First question"}, + {"role": "assistant", "content": "First answer"}, + {"role": "user", "content": "Second question"}, + ] + + query = callback._extract_user_query(messages) + assert query == "Second question" + + def test_extract_user_query_structured_content(self): + """Test extracting query from structured content (vision).""" + callback = HindsightCallback() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": "http://example.com/img.png"}}, + ], + }, + ] + + query = callback._extract_user_query(messages) + assert query == "What's in this image?" + + def test_extract_user_query_multiple_text_parts(self): + """Test extracting query with multiple text parts.""" + callback = HindsightCallback() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "First part."}, + {"type": "text", "text": "Second part."}, + ], + }, + ] + + query = callback._extract_user_query(messages) + assert query == "First part. Second part." + + def test_format_memories(self): + """Test formatting memories into context string.""" + callback = HindsightCallback() + config = HindsightConfig(bank_id="test", max_memories=10, verbose=False) + + memories = [ + {"text": "User likes Python", "fact_type": "world", "weight": 0.95}, + {"text": "User works at Google", "fact_type": "world", "weight": 0.8}, + ] + + formatted = callback._format_memories(memories, config) + + assert "Relevant Memories" in formatted + assert "User likes Python" in formatted + assert "User works at Google" in formatted + assert "[WORLD]" in formatted + + def test_format_memories_with_verbose(self): + """Test formatting memories with verbose mode shows weights.""" + callback = HindsightCallback() + config = HindsightConfig(bank_id="test", max_memories=10, verbose=True) + + memories = [ + {"text": "User likes Python", "fact_type": "world", "weight": 0.95}, + ] + + formatted = callback._format_memories(memories, config) + + assert "relevance: 0.95" in formatted + + def test_inject_memories_as_system_message(self): + """Test injecting memories as system message.""" + callback = HindsightCallback() + config = HindsightConfig( + bank_id="test", + injection_mode=MemoryInjectionMode.SYSTEM_MESSAGE, + ) + + messages = [ + {"role": "user", "content": "Hello"}, + ] + memory_context = "# Relevant Memories\n1. User is John" + + result = callback._inject_memories_into_messages(messages, memory_context, config) + + assert len(result) == 2 + assert result[0]["role"] == "system" + assert "Relevant Memories" in result[0]["content"] + assert result[1]["role"] == "user" + + def test_inject_memories_prepend_to_existing_system(self): + """Test injecting memories appends to existing system message.""" + callback = HindsightCallback() + config = HindsightConfig( + bank_id="test", + injection_mode=MemoryInjectionMode.SYSTEM_MESSAGE, + ) + + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"}, + ] + memory_context = "# Relevant Memories\n1. User is John" + + result = callback._inject_memories_into_messages(messages, memory_context, config) + + assert len(result) == 2 + assert result[0]["role"] == "system" + assert "You are helpful." in result[0]["content"] + assert "Relevant Memories" in result[0]["content"] + + def test_inject_memories_prepend_user_mode(self): + """Test injecting memories in prepend_user mode.""" + callback = HindsightCallback() + config = HindsightConfig( + bank_id="test", + injection_mode=MemoryInjectionMode.PREPEND_USER, + ) + + messages = [ + {"role": "user", "content": "What's my name?"}, + ] + memory_context = "# Relevant Memories\n1. User is John" + + result = callback._inject_memories_into_messages(messages, memory_context, config) + + assert len(result) == 1 + assert result[0]["role"] == "user" + assert "Relevant Memories" in result[0]["content"] + assert "What's my name?" in result[0]["content"] + + def test_should_skip_model_exact_match(self): + """Test model exclusion with exact match.""" + callback = HindsightCallback() + config = HindsightConfig( + bank_id="test", + excluded_models=["gpt-3.5-turbo"], + ) + + assert callback._should_skip_model("gpt-3.5-turbo", config) is True + assert callback._should_skip_model("gpt-4", config) is False + + def test_should_skip_model_wildcard(self): + """Test model exclusion with wildcard pattern.""" + callback = HindsightCallback() + config = HindsightConfig( + bank_id="test", + excluded_models=["gpt-3.5*", "claude-instant-*"], + ) + + assert callback._should_skip_model("gpt-3.5-turbo", config) is True + assert callback._should_skip_model("gpt-3.5-turbo-16k", config) is True + assert callback._should_skip_model("claude-instant-1.2", config) is True + assert callback._should_skip_model("gpt-4", config) is False + assert callback._should_skip_model("claude-3-opus", config) is False + + +class TestDeduplication: + """Tests for conversation deduplication.""" + + def setup_method(self): + """Reset state before each test.""" + cleanup() + + def teardown_method(self): + """Clean up after each test.""" + cleanup() + + def test_compute_conversation_hash(self): + """Test computing conversation hash.""" + callback = HindsightCallback() + + hash1 = callback._compute_conversation_hash("Hello", "Hi there!") + hash2 = callback._compute_conversation_hash("Hello", "Hi there!") + hash3 = callback._compute_conversation_hash("Hello", "Different response") + + # Same content should produce same hash + assert hash1 == hash2 + # Different content should produce different hash + assert hash1 != hash3 + + def test_compute_conversation_hash_case_insensitive(self): + """Test that hash is case insensitive.""" + callback = HindsightCallback() + + hash1 = callback._compute_conversation_hash("HELLO", "HI THERE!") + hash2 = callback._compute_conversation_hash("hello", "hi there!") + + assert hash1 == hash2 + + def test_is_duplicate_first_time(self): + """Test first occurrence is not a duplicate.""" + callback = HindsightCallback() + + result = callback._is_duplicate("abc123") + + assert result is False + + def test_is_duplicate_second_time(self): + """Test second occurrence is a duplicate.""" + callback = HindsightCallback() + + callback._is_duplicate("abc123") # First time + result = callback._is_duplicate("abc123") # Second time + + assert result is True + + def test_is_duplicate_different_hashes(self): + """Test different hashes are not duplicates.""" + callback = HindsightCallback() + + callback._is_duplicate("abc123") + result = callback._is_duplicate("xyz789") + + assert result is False + + +class TestContextManager: + """Tests for the hindsight_memory context manager.""" + + def setup_method(self): + """Reset state before each test.""" + cleanup() + + def teardown_method(self): + """Clean up after each test.""" + cleanup() + + def test_context_manager_enables_and_disables(self): + """Test context manager enables and disables correctly.""" + from hindsight_litellm import hindsight_memory + + assert is_enabled() is False + + with hindsight_memory(bank_id="test-agent"): + assert is_enabled() is True + assert get_config().bank_id == "test-agent" + + assert is_enabled() is False + + def test_context_manager_restores_previous_config(self): + """Test context manager restores previous configuration.""" + from hindsight_litellm import hindsight_memory + + # Set up initial config + configure(bank_id="original-agent") + enable() + assert get_config().bank_id == "original-agent" + + # Use context manager with different config + with hindsight_memory(bank_id="temporary-agent"): + assert get_config().bank_id == "temporary-agent" + + # Should restore original config + assert get_config().bank_id == "original-agent" + assert is_enabled() is True + + def test_context_manager_with_fact_types(self): + """Test context manager with fact_types parameter.""" + from hindsight_litellm import hindsight_memory + + with hindsight_memory(bank_id="test-agent", fact_types=["world", "opinion"]): + config = get_config() + assert config.fact_types == ["world", "opinion"] + + +class TestFactTypes: + """Tests for fact_types configuration.""" + + def setup_method(self): + """Reset config before each test.""" + reset_config() + + def teardown_method(self): + """Clean up after each test.""" + cleanup() + + def test_configure_with_fact_types(self): + """Test configuring with fact_types.""" + config = configure( + bank_id="test-agent", + fact_types=["world", "agent", "opinion"], + ) + + assert config.fact_types == ["world", "agent", "opinion"] + + def test_configure_without_fact_types(self): + """Test configuring without fact_types defaults to None.""" + config = configure(bank_id="test-agent") + + assert config.fact_types is None From b05e3756e5791c85a48b50e96a9c2238460e2756 Mon Sep 17 00:00:00 2001 From: DK09876 Date: Mon, 8 Dec 2025 10:05:16 -0700 Subject: [PATCH 02/10] Add instructions for entity vs bank id --- hindsight-integrations/litellm/README.md | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/hindsight-integrations/litellm/README.md b/hindsight-integrations/litellm/README.md index 4bb6febd45..75ade2ee1a 100644 --- a/hindsight-integrations/litellm/README.md +++ b/hindsight-integrations/litellm/README.md @@ -19,6 +19,46 @@ Universal LLM memory integration via LiteLLM. Add persistent memory to any LLM a pip install hindsight-litellm ``` +## Core Concepts + +### bank_id vs entity_id + +These two identifiers control how memories are organized and isolated: + +| Identifier | Represents | Example | +|------------|-----------|---------| +| `bank_id` | Your agent or application | `"customer-support-bot"` | +| `entity_id` | The end user being served | `"user-alice"`, `"user-bob"` | + +**Why entity_id matters**: When building multi-user applications, you need memory isolation between users. Without it, memories leak across users: + +```python +# Without entity_id - memories are shared (dangerous for multi-user apps!) +configure(bank_id="my-bot") + +# User Alice says: "I'm allergic to peanuts" +# User Bob asks: "What am I allergic to?" +# Bob gets told: "You're allergic to peanuts" -- Memory leak! +``` + +```python +# With entity_id - memories are isolated per user +configure(bank_id="my-bot", entity_id="alice") +# Alice's memories stay with Alice + +set_entity("bob") +# Bob has his own isolated memory space +``` + +**When to use entity_id**: +- Multi-tenant SaaS applications +- Customer support agents serving many customers +- Any app where multiple users interact with the same agent + +**When you DON'T need entity_id**: +- Single-user applications (personal CLI tools, local assistants) +- Shared knowledge bases where all users should see the same memories + ## Quick Start ```python From ebdcf78f09a6dbb23595f8f8a3d9c348d01ba794 Mon Sep 17 00:00:00 2001 From: DK09876 Date: Mon, 8 Dec 2025 10:58:52 -0700 Subject: [PATCH 03/10] Add another line about entity --- hindsight-integrations/litellm/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hindsight-integrations/litellm/README.md b/hindsight-integrations/litellm/README.md index 75ade2ee1a..7d7cc136af 100644 --- a/hindsight-integrations/litellm/README.md +++ b/hindsight-integrations/litellm/README.md @@ -59,6 +59,8 @@ set_entity("bob") - Single-user applications (personal CLI tools, local assistants) - Shared knowledge bases where all users should see the same memories +**How it works**: When you set `entity_id`, it's combined with `bank_id` to create a scoped bank ID (e.g., `"my-bot:alice"`). This means each user effectively gets their own memory bank, providing complete isolation. + ## Quick Start ```python From 6d71dec4a15975575e81f6a063575d7db2fc35fb Mon Sep 17 00:00:00 2001 From: DK09876 Date: Wed, 10 Dec 2025 12:44:32 -0600 Subject: [PATCH 04/10] Address PR review comments and enhance litellm integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove deprecated limit parameter from recall() and arecall() functions since Hindsight uses budget/max_tokens for result control - Remove dead MODEL_MAX_OUTPUT_TOKENS dict and max_output_tokens property from LLMProvider (superseded by hardcoded max_completion_tokens) - Add test-litellm-integration job to CI workflow - Add reflect API support with use_reflect config option - Add verbose mode debug info via get_last_injection_debug() - Add entity_id support for multi-user memory isolation - Add retain() and reflect() wrapper functions - Update docstrings and examples 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .github/workflows/test.yml | 29 ++ .../hindsight_api/engine/llm_wrapper.py | 39 -- .../litellm/hindsight_litellm/__init__.py | 342 +++++++++++++--- .../litellm/hindsight_litellm/callbacks.py | 65 ++- .../litellm/hindsight_litellm/config.py | 77 ++++ .../litellm/hindsight_litellm/wrappers.py | 373 +++++++++++++++++- 6 files changed, 817 insertions(+), 108 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 831b262661..6947fe3aac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -441,3 +441,32 @@ jobs: run: | echo "=== API Server Logs ===" cat /tmp/api-server.log || echo "No API server log found" + + test-litellm-integration: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + prune-cache: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: ".python-version" + + - name: Build litellm integration + working-directory: ./hindsight-integrations/litellm + run: uv build + + - name: Install dependencies + working-directory: ./hindsight-integrations/litellm + run: uv sync --extra dev + + - name: Run tests + working-directory: ./hindsight-integrations/litellm + run: uv run pytest tests -v \ No newline at end of file diff --git a/hindsight-api/hindsight_api/engine/llm_wrapper.py b/hindsight-api/hindsight_api/engine/llm_wrapper.py index d9afec7a86..d3ff78ed32 100644 --- a/hindsight-api/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api/hindsight_api/engine/llm_wrapper.py @@ -22,35 +22,6 @@ # Global semaphore to limit concurrent LLM requests across all instances _global_llm_semaphore = asyncio.Semaphore(32) -# Model-specific max output token limits -# These represent the maximum tokens a model can generate in a single response -MODEL_MAX_OUTPUT_TOKENS = { - # OpenAI models - "gpt-4o": 16384, - "gpt-4o-mini": 16384, - "gpt-4-turbo": 4096, - "gpt-4-turbo-preview": 4096, - "gpt-4": 8192, - "gpt-3.5-turbo": 4096, - "o1": 100000, - "o1-mini": 65536, - "o1-preview": 32768, - # Groq models - "llama-3.1-70b-versatile": 32768, - "llama-3.1-8b-instant": 8192, - "llama-3.3-70b-versatile": 32768, - "llama3-70b-8192": 8192, - "llama3-8b-8192": 8192, - "mixtral-8x7b-32768": 32768, - # Gemini models - "gemini-2.0-flash": 8192, - "gemini-1.5-pro": 8192, - "gemini-1.5-flash": 8192, -} - -# Conservative default for unknown models -DEFAULT_MAX_OUTPUT_TOKENS = 4096 - class OutputTooLongError(Exception): """ @@ -150,16 +121,6 @@ async def verify_connection(self) -> None: f"LLM connection verification failed for {self.provider}/{self.model}: {e}" ) from e - @property - def max_output_tokens(self) -> int: - """ - Get the max output tokens for the configured model. - - Returns the model-specific limit from MODEL_MAX_OUTPUT_TOKENS, - or DEFAULT_MAX_OUTPUT_TOKENS if the model is not in the mapping. - """ - return MODEL_MAX_OUTPUT_TOKENS.get(self.model, DEFAULT_MAX_OUTPUT_TOKENS) - async def call( self, messages: List[Dict[str, str]], diff --git a/hindsight-integrations/litellm/hindsight_litellm/__init__.py b/hindsight-integrations/litellm/hindsight_litellm/__init__.py index 67c1224343..b3aa366cc3 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/__init__.py +++ b/hindsight-integrations/litellm/hindsight_litellm/__init__.py @@ -41,7 +41,7 @@ >>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") >>> >>> # Query memories directly - >>> memories = recall("what projects am I working on?", limit=5) + >>> memories = recall("what projects am I working on?") >>> for m in memories: ... print(f"- [{m.fact_type}] {m.text}") @@ -103,10 +103,20 @@ - recall_budget: Budget for memory recall (low, mid, high) - excluded_models: List of model patterns to exclude from interception - verbose: Enable verbose logging + - bank_name: Display name for the memory bank + - background: Instructions that help Hindsight understand what to remember + +Background example: + >>> configure( + ... bank_id="routing-agent", + ... background="This agent routes customer requests to support channels. " + ... "Remember which types of issues should go to which channels.", + ... ) """ from contextlib import contextmanager -from typing import Optional, List +from dataclasses import dataclass +from typing import Optional, List, Any import litellm @@ -132,6 +142,16 @@ recall, arecall, RecallResult, + RecallResponse, + RecallDebugInfo, + reflect, + areflect, + ReflectResult, + ReflectDebugInfo, + retain, + aretain, + RetainResult, + RetainDebugInfo, wrap_openai, wrap_anthropic, HindsightOpenAI, @@ -149,13 +169,86 @@ _original_acompletion = None +@dataclass +class InjectionDebugInfo: + """Debug information from a memory injection operation. + + This is populated when verbose=True in the config and can be retrieved + via get_last_injection_debug() after a completion() call. + + Attributes: + mode: The injection mode used ("reflect" or "recall") + query: The user query used for memory lookup + bank_id: The bank ID used + scoped_bank_id: The bank ID with entity scoping applied + entity_id: The entity ID used (if any) + memory_context: The formatted memory context that was injected + reflect_text: The raw reflect text (when mode="reflect") + reflect_facts: The facts used to generate the reflect response (when reflect_include_facts=True) + recall_results: The raw recall results (when mode="recall") + results_count: Number of memories/results found + injected: Whether memories were actually injected into the prompt + error: Error message if injection failed (None on success) + """ + mode: str # "reflect" or "recall" + query: str + bank_id: str + scoped_bank_id: str + entity_id: Optional[str] + memory_context: str # The formatted context that was injected + reflect_text: Optional[str] = None # Raw reflect response text + reflect_facts: Optional[List[dict]] = None # Facts used by reflect (when reflect_include_facts=True) + recall_results: Optional[List[dict]] = None # Raw recall results + results_count: int = 0 + injected: bool = False + error: Optional[str] = None # Error message if injection failed + + +# Store the last injection debug info (populated when verbose=True) +_last_injection_debug: Optional[InjectionDebugInfo] = None + + +def get_last_injection_debug() -> Optional[InjectionDebugInfo]: + """Get debug info from the last memory injection operation. + + When verbose=True in the config, this returns information about + what memories were injected into the last completion() call. + + Returns: + InjectionDebugInfo if verbose mode captured injection info, None otherwise + + Example: + >>> from hindsight_litellm import configure, enable, completion, get_last_injection_debug + >>> configure(bank_id="my-agent", verbose=True, use_reflect=True) + >>> enable() + >>> response = completion(model="gpt-4o-mini", messages=[...]) + >>> debug = get_last_injection_debug() + >>> if debug: + ... print(f"Injected {debug.results_count} memories via {debug.mode}") + ... print(f"Reflect text: {debug.reflect_text}") + """ + return _last_injection_debug + + +def clear_injection_debug() -> None: + """Clear the stored injection debug info.""" + global _last_injection_debug + _last_injection_debug = None + + def _inject_memories(messages: List[dict]) -> List[dict]: """Inject memories into messages list. Returns the modified messages list with memories injected into the system message. + Uses reflect API when config.use_reflect=True, otherwise uses recall API. + + When verbose=True in config, stores debug info retrievable via get_last_injection_debug(). """ + global _last_injection_debug import logging - import requests + + # Clear previous debug info + _last_injection_debug = None if not is_configured(): return messages @@ -180,50 +273,146 @@ def _inject_memories(messages: List[dict]) -> List[dict]: return messages try: + from hindsight_client import Hindsight + # Build scoped bank_id scoped_bank_id = config.bank_id if config.entity_id: scoped_bank_id = f"{config.bank_id}:{config.entity_id}" - # Build recall request - url = f"{config.hindsight_api_url}/v1/default/banks/{scoped_bank_id}/memories/recall" - request_data = { - "query": user_query, - "budget": config.recall_budget or "mid", - "max_tokens": config.max_memory_tokens or 2000, - } - if config.fact_types: - request_data["types"] = config.fact_types - - headers = {"Content-Type": "application/json"} - if config.api_key: - headers["Authorization"] = f"Bearer {config.api_key}" - - response = requests.post(url, json=request_data, headers=headers, timeout=30) - response.raise_for_status() - response_data = response.json() - results = response_data.get("results", []) - - if not results: - return messages - - # Format memories - memory_lines = [] - for i, result in enumerate(results[:config.max_memories], 1): - text = result.get("text", "") - fact_type = result.get("type", result.get("fact_type", "world")) - if text: - type_label = fact_type.upper() if fact_type else "MEMORY" - memory_lines.append(f"{i}. [{type_label}] {text}") - - if not memory_lines: - return messages - - memory_context = ( - "# Relevant Memories\n" - "The following information from memory may be relevant:\n\n" - + "\n".join(memory_lines) - ) + # Track debug info + mode = "reflect" if config.use_reflect else "recall" + reflect_text = None + reflect_facts = None + recall_results = None + results_count = 0 + memory_context = "" + + # Create client + client = Hindsight(base_url=config.hindsight_api_url, timeout=30.0) + + # Use reflect API if use_reflect is enabled + if config.use_reflect: + # If reflect_include_facts is enabled, use the API directly to include facts + if config.reflect_include_facts: + from hindsight_client_api.models import reflect_request, reflect_include_options + request_obj = reflect_request.ReflectRequest( + query=user_query, + budget=config.recall_budget or "mid", + include=reflect_include_options.ReflectIncludeOptions(facts={}), + ) + import asyncio + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + result = loop.run_until_complete(client._api.reflect(scoped_bank_id, request_obj)) + # Extract facts from based_on + if hasattr(result, 'based_on') and result.based_on: + reflect_facts = [ + { + "text": f.text if hasattr(f, 'text') else str(f), + "type": getattr(f, 'type', None), + "context": getattr(f, 'context', None), + } + for f in result.based_on + ] + else: + result = client.reflect( + bank_id=scoped_bank_id, + query=user_query, + budget=config.recall_budget or "mid", + ) + reflect_text = result.text if hasattr(result, 'text') else str(result) + + if not reflect_text: + # Store debug info for empty result + if config.verbose: + _last_injection_debug = InjectionDebugInfo( + mode=mode, + query=user_query, + bank_id=config.bank_id, + scoped_bank_id=scoped_bank_id, + entity_id=config.entity_id, + memory_context="", + reflect_text="", + reflect_facts=reflect_facts, + results_count=0, + injected=False, + ) + return messages + + results_count = 1 # reflect returns a single synthesized response + memory_context = ( + "# Relevant Context from Memory\n" + f"{reflect_text}" + ) + else: + # Use recall API (original behavior) + result = client.recall( + bank_id=scoped_bank_id, + query=user_query, + budget=config.recall_budget or "mid", + max_tokens=config.max_memory_tokens or 2000, + types=config.fact_types, + ) + results = result.results if hasattr(result, 'results') else [] + # Convert to dicts for debug info + recall_results = [ + { + "text": r.text if hasattr(r, 'text') else str(r), + "type": getattr(r, 'type', 'world'), + } + for r in results + ] + + if not results: + # Store debug info for empty result + if config.verbose: + _last_injection_debug = InjectionDebugInfo( + mode=mode, + query=user_query, + bank_id=config.bank_id, + scoped_bank_id=scoped_bank_id, + entity_id=config.entity_id, + memory_context="", + recall_results=[], + results_count=0, + injected=False, + ) + return messages + + # Format memories + memory_lines = [] + for i, r in enumerate(results[:config.max_memories], 1): + text = r.text if hasattr(r, 'text') else str(r) + fact_type = getattr(r, 'type', 'world') + if text: + type_label = fact_type.upper() if fact_type else "MEMORY" + memory_lines.append(f"{i}. [{type_label}] {text}") + + if not memory_lines: + if config.verbose: + _last_injection_debug = InjectionDebugInfo( + mode=mode, + query=user_query, + bank_id=config.bank_id, + scoped_bank_id=scoped_bank_id, + entity_id=config.entity_id, + memory_context="", + recall_results=recall_results, + results_count=0, + injected=False, + ) + return messages + + results_count = len(memory_lines) + memory_context = ( + "# Relevant Memories\n" + "The following information from memory may be relevant:\n\n" + + "\n".join(memory_lines) + ) # Inject into messages updated_messages = list(messages) @@ -246,15 +435,62 @@ def _inject_memories(messages: List[dict]) -> List[dict]: "content": memory_context }) + # Store debug info when verbose if config.verbose: + _last_injection_debug = InjectionDebugInfo( + mode=mode, + query=user_query, + bank_id=config.bank_id, + scoped_bank_id=scoped_bank_id, + entity_id=config.entity_id, + memory_context=memory_context, + reflect_text=reflect_text, + reflect_facts=reflect_facts, + recall_results=recall_results, + results_count=results_count, + injected=True, + ) logger = logging.getLogger("hindsight_litellm") - logger.info(f"Injected {len(results)} memories into prompt") + logger.info(f"Injected memories using {mode} into prompt") return updated_messages + except ImportError as e: + if config.verbose: + logging.getLogger("hindsight_litellm").warning( + f"hindsight_client not installed: {e}. Install with: pip install hindsight-client" + ) + _last_injection_debug = InjectionDebugInfo( + mode="reflect" if config.use_reflect else "recall", + query=user_query or "", + bank_id=config.bank_id or "", + scoped_bank_id=scoped_bank_id if 'scoped_bank_id' in dir() else config.bank_id or "", + entity_id=config.entity_id, + memory_context="", + results_count=0, + injected=False, + error=f"hindsight_client not installed: {e}", + ) + return messages except Exception as e: + # Always set debug info on error when verbose mode is on if config.verbose: logging.getLogger("hindsight_litellm").warning(f"Failed to inject memories: {e}") + # Build scoped bank_id for debug info + scoped_bank_id = config.bank_id + if config.entity_id: + scoped_bank_id = f"{config.bank_id}:{config.entity_id}" + _last_injection_debug = InjectionDebugInfo( + mode="reflect" if config.use_reflect else "recall", + query=user_query or "", + bank_id=config.bank_id or "", + scoped_bank_id=scoped_bank_id or "", + entity_id=config.entity_id, + memory_context="", + results_count=0, + injected=False, + error=str(e), + ) return messages @@ -483,6 +719,8 @@ def hindsight_memory( document_id: Optional[str] = None, excluded_models: Optional[List[str]] = None, verbose: bool = False, + bank_name: Optional[str] = None, + background: Optional[str] = None, ): """Context manager for temporary Hindsight memory integration. @@ -505,6 +743,8 @@ def hindsight_memory( document_id: Optional document ID for grouping conversations excluded_models: List of model patterns to exclude verbose: Enable verbose logging + bank_name: Optional display name for the memory bank + background: Optional background/instructions for memory extraction Example: >>> from hindsight_litellm import hindsight_memory @@ -536,6 +776,8 @@ def hindsight_memory( document_id=document_id, excluded_models=excluded_models, verbose=verbose, + bank_name=bank_name, + background=background, ) enable() yield @@ -559,6 +801,8 @@ def hindsight_memory( document_id=previous_config.document_id, excluded_models=previous_config.excluded_models, verbose=previous_config.verbose, + bank_name=previous_config.bank_name, + background=previous_config.background, ) if was_enabled: enable() @@ -583,10 +827,16 @@ def hindsight_memory( "get_session", "set_entity", "get_entity", - # Direct recall API + # Direct memory APIs "recall", "arecall", "RecallResult", + "reflect", + "areflect", + "ReflectResult", + "retain", + "aretain", + "RetainResult", # Native client wrappers "wrap_openai", "wrap_anthropic", @@ -598,6 +848,10 @@ def hindsight_memory( "reset_config", "HindsightConfig", "MemoryInjectionMode", + # Injection debug (verbose mode) + "get_last_injection_debug", + "clear_injection_debug", + "InjectionDebugInfo", # Callback (for advanced usage) "HindsightCallback", "get_callback", diff --git a/hindsight-integrations/litellm/hindsight_litellm/callbacks.py b/hindsight-integrations/litellm/hindsight_litellm/callbacks.py index c54e0ae554..8304159405 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/callbacks.py +++ b/hindsight-integrations/litellm/hindsight_litellm/callbacks.py @@ -312,14 +312,16 @@ def _store_conversation_sync( model: str, config: HindsightConfig, ) -> None: - """Store the conversation to Hindsight (sync) using direct HTTP.""" - try: - # Extract user input (last user message only) - user_input = self._extract_user_query(messages) - if not user_input: - return + """Store the conversation to Hindsight (sync) using direct HTTP. + + By default, stores the full conversation history passed to the LLM. + Each message is stored as a separate item, all linked by document_id + (using session_id if set, otherwise a generated one). - # Extract assistant response + Hindsight will process the document as a whole for memory extraction. + """ + try: + # Extract assistant response from the LLM response assistant_output = "" if response.choices and len(response.choices) > 0: choice = response.choices[0] @@ -329,10 +331,43 @@ def _store_conversation_sync( if not assistant_output: return - # Skip if this looks like our injected memory context - if user_input.startswith("# Relevant Memories"): + # Build conversation items - each message becomes a separate item + # All linked by document_id for Hindsight to process together + items = [] + for msg in messages: + role = msg.get("role", "").upper() + content = msg.get("content", "") + + # Skip system messages - they're instructions, not conversation + if role == "SYSTEM": + continue + + # Skip if this looks like our injected memory context + if isinstance(content, str) and content.startswith("# Relevant Memories"): + continue + + # Handle structured content (e.g., vision messages) + if isinstance(content, list): + text_parts = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + text_parts.append(item.get("text", "")) + content = " ".join(text_parts) + + if content: + # Map roles to clearer labels + label = "USER" if role == "USER" else "ASSISTANT" + items.append(f"{label}: {content}") + + # Add the new assistant response + items.append(f"ASSISTANT: {assistant_output}") + + if not items: return + # Use last user message for deduplication hash + user_input = self._extract_user_query(messages) or "" + # Deduplication check conv_hash = self._compute_conversation_hash(user_input, assistant_output) if self._is_duplicate(conv_hash): @@ -340,9 +375,12 @@ def _store_conversation_sync( logger.debug(f"Skipping duplicate conversation: {conv_hash}") return - # Build conversation content for storage - # Format: Clear USER/ASSISTANT structure for Hindsight to extract facts from - conversation_text = f"USER: {user_input}\n\nASSISTANT: {assistant_output}" + # Use session_id as document_id if set, otherwise use config.document_id + doc_id = config.document_id or config.session_id + + # Build the full conversation as a single item for now + # (Future: could store each message as separate item in same document) + conversation_text = "\n\n".join(items) # Build metadata metadata = { @@ -372,11 +410,10 @@ def _store_conversation_sync( "content": conversation_text, "context": f"conversation:litellm:{model}", "metadata": metadata, + "document_id": doc_id, # Group by session/document } ], } - if config.document_id: - request_data["document_id"] = config.document_id self._http_post(url, request_data, config) diff --git a/hindsight-integrations/litellm/hindsight_litellm/config.py b/hindsight-integrations/litellm/hindsight_litellm/config.py index 8a112c36d0..a4698c671f 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/config.py +++ b/hindsight-integrations/litellm/hindsight_litellm/config.py @@ -34,6 +34,9 @@ class HindsightConfig: enabled: Master switch to enable/disable Hindsight integration excluded_models: List of model patterns to exclude from interception verbose: Enable verbose logging + bank_name: Optional display name for the memory bank + background: Optional background/instructions for memory extraction + use_reflect: Use reflect API instead of recall for memory injection (synthesizes answer) """ hindsight_api_url: str = "http://localhost:8888" @@ -52,6 +55,10 @@ class HindsightConfig: enabled: bool = True excluded_models: List[str] = field(default_factory=list) verbose: bool = False + bank_name: Optional[str] = None # Display name for the memory bank + background: Optional[str] = None # Background/instructions for memory extraction + use_reflect: bool = False # Use reflect instead of recall for memory injection + reflect_include_facts: bool = False # Include facts used by reflect in debug info # Global configuration instance @@ -75,6 +82,10 @@ def configure( enabled: bool = True, excluded_models: Optional[List[str]] = None, verbose: bool = False, + bank_name: Optional[str] = None, + background: Optional[str] = None, + use_reflect: bool = False, + reflect_include_facts: bool = False, ) -> HindsightConfig: """Configure global Hindsight integration settings for LiteLLM. @@ -98,6 +109,16 @@ def configure( enabled: Master switch to enable/disable Hindsight integration excluded_models: List of model patterns to exclude from interception verbose: Enable verbose logging + bank_name: Optional display name for the memory bank + background: Optional background/instructions that help Hindsight understand + what information is important to extract and remember from conversations. + This is passed to create_bank() to configure the memory bank. + use_reflect: Use reflect API instead of recall for memory injection. + When True, Hindsight will synthesize a contextual answer based on + memories rather than returning raw memory facts. + reflect_include_facts: When use_reflect=True, include the facts that + were used to generate the reflect response in the debug info. + This is useful for debugging what memories the reflect API used. Returns: The configured HindsightConfig instance @@ -110,6 +131,8 @@ def configure( ... entity_id="user-123", # Multi-user support ... store_conversations=True, ... inject_memories=True, + ... background="This agent routes customer requests to support channels. " + ... "Remember which types of issues should go to which channels.", ... ) >>> enable() # Register callbacks with LiteLLM """ @@ -132,11 +155,65 @@ def configure( enabled=enabled, excluded_models=excluded_models or [], verbose=verbose, + bank_name=bank_name, + background=background, + use_reflect=use_reflect, + reflect_include_facts=reflect_include_facts, ) + # If background or bank_name is provided, create/update the bank + if bank_id and (background or bank_name): + _create_or_update_bank( + hindsight_api_url=hindsight_api_url, + bank_id=bank_id, + name=bank_name, + background=background, + verbose=verbose, + ) + return _global_config +def _create_or_update_bank( + hindsight_api_url: str, + bank_id: str, + name: Optional[str] = None, + background: Optional[str] = None, + verbose: bool = False, +) -> None: + """Create or update a memory bank with the given configuration. + + This is called automatically by configure() when background or bank_name is provided. + """ + try: + from hindsight_client import Hindsight + + client = Hindsight(hindsight_api_url) + client.create_bank( + bank_id=bank_id, + name=name, + background=background, + ) + if verbose: + import logging + logging.getLogger("hindsight_litellm").info( + f"Created/updated bank '{bank_id}' with background" + ) + except ImportError: + if verbose: + import logging + logging.getLogger("hindsight_litellm").warning( + "hindsight_client not installed. Cannot create bank with background. " + "Install with: pip install hindsight-client" + ) + except Exception as e: + if verbose: + import logging + logging.getLogger("hindsight_litellm").warning( + f"Failed to create/update bank: {e}" + ) + + def get_config() -> Optional[HindsightConfig]: """Get the current global configuration. diff --git a/hindsight-integrations/litellm/hindsight_litellm/wrappers.py b/hindsight-integrations/litellm/hindsight_litellm/wrappers.py index 9021734bcc..8b5f47ea9f 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/wrappers.py +++ b/hindsight-integrations/litellm/hindsight_litellm/wrappers.py @@ -29,16 +29,48 @@ def __str__(self) -> str: return self.text +@dataclass +class RecallDebugInfo: + """Debug information from a recall operation.""" + query: str + bank_id: str + scoped_bank_id: str + entity_id: Optional[str] + budget: str + max_tokens: int + fact_types: Optional[List[str]] + results_count: int + api_url: str + + +@dataclass +class RecallResponse: + """Response from a recall operation, including results and optional debug info.""" + results: List[RecallResult] + debug: Optional[RecallDebugInfo] = None + + def __iter__(self): + return iter(self.results) + + def __len__(self): + return len(self.results) + + def __getitem__(self, key): + return self.results[key] + + def __bool__(self): + return bool(self.results) + + def recall( query: str, - limit: int = 10, bank_id: Optional[str] = None, entity_id: Optional[str] = None, fact_types: Optional[List[str]] = None, budget: Optional[str] = None, max_tokens: Optional[int] = None, hindsight_api_url: Optional[str] = None, -) -> List[RecallResult]: +) -> RecallResponse: """Recall memories from Hindsight. This function allows you to manually query memories without making an LLM call. @@ -46,16 +78,16 @@ def recall( Args: query: The query string to search memories for - limit: Maximum number of memories to return (default: 10) bank_id: Override the configured bank_id entity_id: Override the configured entity_id for multi-user isolation fact_types: Filter by fact types (world, agent, opinion, observation) - budget: Recall budget level (low, mid, high) + budget: Recall budget level (low, mid, high) - controls how many memories are returned max_tokens: Maximum tokens for memory context hindsight_api_url: Override the configured API URL Returns: - List of RecallResult objects containing matched memories + RecallResponse containing matched memories (iterable like a list). + When verbose=True in config, includes debug info via .debug attribute. Raises: RuntimeError: If Hindsight is not configured and no overrides provided @@ -65,11 +97,17 @@ def recall( >>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") >>> >>> # Query memories - >>> memories = recall("what projects am I working on?", limit=5) + >>> memories = recall("what projects am I working on?") >>> for m in memories: ... print(f"- [{m.fact_type}] {m.text}") - [world] User is building a FastAPI project - [opinion] User prefers Python over JavaScript + >>> + >>> # With verbose mode, access debug info + >>> configure(bank_id="my-agent", verbose=True) + >>> memories = recall("what projects am I working on?") + >>> if memories.debug: + ... print(f"Queried bank: {memories.debug.scoped_bank_id}") """ # Get config or use overrides config = get_config() @@ -108,7 +146,7 @@ def recall( # Convert to RecallResult objects recall_results = [] if results: - for r in results[:limit]: + for r in results: if hasattr(r, 'text'): # Object with attributes fact_type = getattr(r, 'type', None) or getattr(r, 'fact_type', 'unknown') @@ -128,7 +166,22 @@ def recall( metadata=r.get('metadata'), )) - return recall_results + # Include debug info if verbose + debug_info = None + if config and config.verbose: + debug_info = RecallDebugInfo( + query=query, + bank_id=target_bank_id, + scoped_bank_id=scoped_bank_id, + entity_id=target_entity_id, + budget=target_budget, + max_tokens=target_max_tokens, + fact_types=target_fact_types, + results_count=len(recall_results), + api_url=api_url, + ) + + return RecallResponse(results=recall_results, debug=debug_info) except ImportError as e: raise RuntimeError(f"hindsight-client not installed: {e}") @@ -140,14 +193,13 @@ def recall( async def arecall( query: str, - limit: int = 10, bank_id: Optional[str] = None, entity_id: Optional[str] = None, fact_types: Optional[List[str]] = None, budget: Optional[str] = None, max_tokens: Optional[int] = None, hindsight_api_url: Optional[str] = None, -) -> List[RecallResult]: +) -> RecallResponse: """Async version of recall(). See recall() for full documentation. @@ -158,7 +210,6 @@ async def arecall( None, lambda: recall( query=query, - limit=limit, bank_id=bank_id, entity_id=entity_id, fact_types=fact_types, @@ -169,6 +220,306 @@ async def arecall( ) +@dataclass +class ReflectDebugInfo: + """Debug information from a reflect operation.""" + query: str + bank_id: str + scoped_bank_id: str + entity_id: Optional[str] + budget: str + context: Optional[str] + api_url: str + + +@dataclass +class ReflectResult: + """Result from a reflect operation.""" + text: str + based_on: Optional[Dict[str, List[Any]]] = None + debug: Optional[ReflectDebugInfo] = None + + def __str__(self) -> str: + return self.text + + +def reflect( + query: str, + bank_id: Optional[str] = None, + entity_id: Optional[str] = None, + budget: Optional[str] = None, + context: Optional[str] = None, + hindsight_api_url: Optional[str] = None, +) -> ReflectResult: + """Generate a contextual answer based on memories. + + Unlike recall() which returns raw memory facts, reflect() uses an LLM + to synthesize a coherent answer based on the bank's memories. + + Args: + query: The question or prompt to answer + bank_id: Override the configured bank_id + entity_id: Override the configured entity_id for multi-user isolation + budget: Budget level for reflection (low, mid, high) + context: Additional context to include in the reflection + hindsight_api_url: Override the configured API URL + + Returns: + ReflectResult with synthesized answer text + + Raises: + RuntimeError: If Hindsight is not configured and no overrides provided + + Example: + >>> from hindsight_litellm import configure, reflect + >>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") + >>> + >>> # Get a synthesized answer based on memories + >>> result = reflect("What projects am I working on?") + >>> print(result.text) + Based on our conversations, you're working on a FastAPI project... + """ + config = get_config() + + api_url = hindsight_api_url or (config.hindsight_api_url if config else None) + target_bank_id = bank_id or (config.bank_id if config else None) + target_entity_id = entity_id or (config.entity_id if config else None) + target_budget = budget or (config.recall_budget if config else "mid") + + if not api_url or not target_bank_id: + raise RuntimeError( + "Hindsight not configured. Call configure() or provide bank_id and hindsight_api_url." + ) + + try: + from hindsight_client import Hindsight + + client = Hindsight(base_url=api_url, timeout=30.0) + + # Build bank_id with entity scoping if entity_id is set + scoped_bank_id = target_bank_id + if target_entity_id: + scoped_bank_id = f"{target_bank_id}:{target_entity_id}" + + # Call reflect API + result = client.reflect( + bank_id=scoped_bank_id, + query=query, + budget=target_budget, + context=context, + ) + + # Convert to ReflectResult + text = result.text if hasattr(result, 'text') else str(result) + based_on = getattr(result, 'based_on', None) + + # Include debug info if verbose + debug_info = None + if config and config.verbose: + debug_info = ReflectDebugInfo( + query=query, + bank_id=target_bank_id, + scoped_bank_id=scoped_bank_id, + entity_id=target_entity_id, + budget=target_budget, + context=context, + api_url=api_url, + ) + + return ReflectResult(text=text, based_on=based_on, debug=debug_info) + + except ImportError as e: + raise RuntimeError(f"hindsight-client not installed: {e}") + except Exception as e: + if config and config.verbose: + logger.warning(f"Failed to reflect: {e}") + raise + + +async def areflect( + query: str, + bank_id: Optional[str] = None, + entity_id: Optional[str] = None, + budget: Optional[str] = None, + context: Optional[str] = None, + hindsight_api_url: Optional[str] = None, +) -> ReflectResult: + """Async version of reflect(). + + See reflect() for full documentation. + """ + import asyncio + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: reflect( + query=query, + bank_id=bank_id, + entity_id=entity_id, + budget=budget, + context=context, + hindsight_api_url=hindsight_api_url, + ) + ) + + +@dataclass +class RetainDebugInfo: + """Debug information from a retain operation.""" + content: str + bank_id: str + scoped_bank_id: str + entity_id: Optional[str] + context: Optional[str] + document_id: Optional[str] + metadata: Optional[Dict[str, str]] + api_url: str + + +@dataclass +class RetainResult: + """Result from a retain operation.""" + success: bool + items_count: int = 0 + debug: Optional[RetainDebugInfo] = None + + def __bool__(self) -> bool: + return self.success + + +def retain( + content: str, + bank_id: Optional[str] = None, + entity_id: Optional[str] = None, + context: Optional[str] = None, + document_id: Optional[str] = None, + metadata: Optional[Dict[str, str]] = None, + hindsight_api_url: Optional[str] = None, +) -> RetainResult: + """Store content to Hindsight memory. + + This function allows you to manually store content to memory without + making an LLM call. Useful for storing feedback, user preferences, + or any other information you want the system to remember. + + Args: + content: The text content to store + bank_id: Override the configured bank_id + entity_id: Override the configured entity_id for multi-user isolation + context: Context description for the memory (e.g., "customer_feedback") + document_id: Optional document ID for grouping related memories + metadata: Optional key-value metadata to attach to the memory + hindsight_api_url: Override the configured API URL + + Returns: + RetainResult indicating success + + Raises: + RuntimeError: If Hindsight is not configured and no overrides provided + + Example: + >>> from hindsight_litellm import configure, retain + >>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888") + >>> + >>> # Store feedback + >>> retain("User prefers dark mode", context="user_preference") + >>> + >>> # Store with metadata + >>> retain( + ... "Customer reported billing issue resolved", + ... context="support_ticket", + ... metadata={"ticket_id": "12345", "status": "resolved"} + ... ) + """ + config = get_config() + + api_url = hindsight_api_url or (config.hindsight_api_url if config else None) + target_bank_id = bank_id or (config.bank_id if config else None) + target_entity_id = entity_id or (config.entity_id if config else None) + target_document_id = document_id or (config.document_id if config else None) + + if not api_url or not target_bank_id: + raise RuntimeError( + "Hindsight not configured. Call configure() or provide bank_id and hindsight_api_url." + ) + + try: + from hindsight_client import Hindsight + + client = Hindsight(base_url=api_url, timeout=30.0) + + # Build bank_id with entity scoping if entity_id is set + scoped_bank_id = target_bank_id + if target_entity_id: + scoped_bank_id = f"{target_bank_id}:{target_entity_id}" + + # Call retain API + result = client.retain( + bank_id=scoped_bank_id, + content=content, + context=context, + document_id=target_document_id, + metadata=metadata, + ) + + # Check success + success = getattr(result, 'success', True) + items_count = getattr(result, 'items_count', 1) + + # Include debug info if verbose + debug_info = None + if config and config.verbose: + logger.info(f"Stored content to Hindsight bank: {target_bank_id}") + debug_info = RetainDebugInfo( + content=content, + bank_id=target_bank_id, + scoped_bank_id=scoped_bank_id, + entity_id=target_entity_id, + context=context, + document_id=target_document_id, + metadata=metadata, + api_url=api_url, + ) + + return RetainResult(success=success, items_count=items_count, debug=debug_info) + + except ImportError as e: + raise RuntimeError(f"hindsight-client not installed: {e}") + except Exception as e: + if config and config.verbose: + logger.warning(f"Failed to retain: {e}") + raise + + +async def aretain( + content: str, + bank_id: Optional[str] = None, + entity_id: Optional[str] = None, + context: Optional[str] = None, + document_id: Optional[str] = None, + metadata: Optional[Dict[str, str]] = None, + hindsight_api_url: Optional[str] = None, +) -> RetainResult: + """Async version of retain(). + + See retain() for full documentation. + """ + import asyncio + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + lambda: retain( + content=content, + bank_id=bank_id, + entity_id=entity_id, + context=context, + document_id=document_id, + metadata=metadata, + hindsight_api_url=hindsight_api_url, + ) + ) + + class HindsightOpenAI: """Wrapper for OpenAI client with Hindsight memory integration. From 77be122d455a386bb95b2573d148f7bbc6f6f148 Mon Sep 17 00:00:00 2001 From: DK09876 Date: Thu, 11 Dec 2025 10:03:39 -0600 Subject: [PATCH 05/10] Make max_memories optional to allow unlimited memory injection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Change max_memories default from 10 to None (no limit) - When max_memories is None, all results from the API are used - Fix recall result handling to properly detect list vs object return - Update wrappers (OpenAI, Anthropic) with same optional behavior This allows users to control memory limits via max_memory_tokens and recall_budget without an artificial count limit. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .../litellm/hindsight_litellm/__init__.py | 13 +++++++--- .../litellm/hindsight_litellm/callbacks.py | 4 ++- .../litellm/hindsight_litellm/config.py | 4 +-- .../litellm/hindsight_litellm/wrappers.py | 26 ++++++++++--------- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/hindsight-integrations/litellm/hindsight_litellm/__init__.py b/hindsight-integrations/litellm/hindsight_litellm/__init__.py index b3aa366cc3..5b55392191 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/__init__.py +++ b/hindsight-integrations/litellm/hindsight_litellm/__init__.py @@ -357,7 +357,13 @@ def _inject_memories(messages: List[dict]) -> List[dict]: max_tokens=config.max_memory_tokens or 2000, types=config.fact_types, ) - results = result.results if hasattr(result, 'results') else [] + # client.recall() returns a list directly, not an object with .results + if isinstance(result, list): + results = result + elif hasattr(result, 'results'): + results = result.results + else: + results = [] # Convert to dicts for debug info recall_results = [ { @@ -383,9 +389,10 @@ def _inject_memories(messages: List[dict]) -> List[dict]: ) return messages - # Format memories + # Format memories (apply limit if set, otherwise use all) + results_to_use = results[:config.max_memories] if config.max_memories else results memory_lines = [] - for i, r in enumerate(results[:config.max_memories], 1): + for i, r in enumerate(results_to_use, 1): text = r.text if hasattr(r, 'text') else str(r) fact_type = getattr(r, 'type', 'world') if text: diff --git a/hindsight-integrations/litellm/hindsight_litellm/callbacks.py b/hindsight-integrations/litellm/hindsight_litellm/callbacks.py index 8304159405..623e1441b7 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/callbacks.py +++ b/hindsight-integrations/litellm/hindsight_litellm/callbacks.py @@ -173,8 +173,10 @@ def _format_memories( if not results: return "" + # Apply limit if set, otherwise use all results + results_to_use = results[:config.max_memories] if config.max_memories else results memory_lines = [] - for i, result in enumerate(results[:config.max_memories], 1): + for i, result in enumerate(results_to_use, 1): # Handle both RecallResult objects and dicts if hasattr(result, 'text'): text = result.text or "" diff --git a/hindsight-integrations/litellm/hindsight_litellm/config.py b/hindsight-integrations/litellm/hindsight_litellm/config.py index a4698c671f..85b90e901b 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/config.py +++ b/hindsight-integrations/litellm/hindsight_litellm/config.py @@ -47,7 +47,7 @@ class HindsightConfig: store_conversations: bool = True inject_memories: bool = True injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE - max_memories: int = 10 + max_memories: Optional[int] = None # None = no limit (use all results from API) max_memory_tokens: int = 2000 recall_budget: str = "mid" # low, mid, high fact_types: Optional[List[str]] = None # world, agent, opinion, observation @@ -74,7 +74,7 @@ def configure( store_conversations: bool = True, inject_memories: bool = True, injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE, - max_memories: int = 10, + max_memories: Optional[int] = None, max_memory_tokens: int = 2000, recall_budget: str = "mid", fact_types: Optional[List[str]] = None, diff --git a/hindsight-integrations/litellm/hindsight_litellm/wrappers.py b/hindsight-integrations/litellm/hindsight_litellm/wrappers.py index 8b5f47ea9f..312c1e07ea 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/wrappers.py +++ b/hindsight-integrations/litellm/hindsight_litellm/wrappers.py @@ -548,7 +548,7 @@ def __init__( session_id: Optional[str] = None, store_conversations: bool = True, inject_memories: bool = True, - max_memories: int = 10, + max_memories: Optional[int] = None, recall_budget: str = "mid", verbose: bool = False, ): @@ -562,7 +562,7 @@ def __init__( session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations inject_memories: Whether to inject relevant memories - max_memories: Maximum number of memories to inject + max_memories: Maximum number of memories to inject (None = no limit) recall_budget: Budget level for memory recall (low, mid, high) verbose: Enable verbose logging """ @@ -608,14 +608,15 @@ def _recall_memories(self, query: str) -> str: bank_id=self._get_scoped_bank_id(), query=query, budget=self._recall_budget, - max_tokens=self._max_memories * 200, + max_tokens=self._max_memories * 200 if self._max_memories else 2000, ) if not results: return "" + results_to_use = results[:self._max_memories] if self._max_memories else results memory_lines = [] - for i, r in enumerate(results[:self._max_memories], 1): + for i, r in enumerate(results_to_use, 1): text = r.text if hasattr(r, 'text') else str(r) fact_type = r.fact_type if hasattr(r, 'fact_type') else 'memory' memory_lines.append(f"{i}. [{fact_type.upper()}] {text}") @@ -759,7 +760,7 @@ def __init__( session_id: Optional[str] = None, store_conversations: bool = True, inject_memories: bool = True, - max_memories: int = 10, + max_memories: Optional[int] = None, recall_budget: str = "mid", verbose: bool = False, ): @@ -773,7 +774,7 @@ def __init__( session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations inject_memories: Whether to inject relevant memories - max_memories: Maximum number of memories to inject + max_memories: Maximum number of memories to inject (None = no limit) recall_budget: Budget level for memory recall (low, mid, high) verbose: Enable verbose logging """ @@ -819,14 +820,15 @@ def _recall_memories(self, query: str) -> str: bank_id=self._get_scoped_bank_id(), query=query, budget=self._recall_budget, - max_tokens=self._max_memories * 200, + max_tokens=self._max_memories * 200 if self._max_memories else 2000, ) if not results: return "" + results_to_use = results[:self._max_memories] if self._max_memories else results memory_lines = [] - for i, r in enumerate(results[:self._max_memories], 1): + for i, r in enumerate(results_to_use, 1): text = r.text if hasattr(r, 'text') else str(r) fact_type = r.fact_type if hasattr(r, 'fact_type') else 'memory' memory_lines.append(f"{i}. [{fact_type.upper()}] {text}") @@ -942,7 +944,7 @@ def wrap_openai( session_id: Optional[str] = None, store_conversations: bool = True, inject_memories: bool = True, - max_memories: int = 10, + max_memories: Optional[int] = None, recall_budget: str = "mid", verbose: bool = False, ) -> HindsightOpenAI: @@ -959,7 +961,7 @@ def wrap_openai( session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations inject_memories: Whether to inject relevant memories - max_memories: Maximum number of memories to inject + max_memories: Maximum number of memories to inject (None = no limit) recall_budget: Budget level for memory recall (low, mid, high) verbose: Enable verbose logging @@ -1004,7 +1006,7 @@ def wrap_anthropic( session_id: Optional[str] = None, store_conversations: bool = True, inject_memories: bool = True, - max_memories: int = 10, + max_memories: Optional[int] = None, recall_budget: str = "mid", verbose: bool = False, ) -> HindsightAnthropic: @@ -1021,7 +1023,7 @@ def wrap_anthropic( session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations inject_memories: Whether to inject relevant memories - max_memories: Maximum number of memories to inject + max_memories: Maximum number of memories to inject (None = no limit) recall_budget: Budget level for memory recall (low, mid, high) verbose: Enable verbose logging From 3aad275dec56aaeb654be98db23eae4e3a92d803 Mon Sep 17 00:00:00 2001 From: DK09876 Date: Thu, 11 Dec 2025 12:34:17 -0600 Subject: [PATCH 06/10] Remove entity_id from hindsight_litellm; add gpt-4o token cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-user support now uses separate bank_ids per user instead of entity_id scoping (e.g., bank_id=f"user-{user_id}"). This simplifies the API and aligns with the Hindsight architecture. Also fixes max_completion_tokens error for gpt-4o models by capping the value at 16384 (gpt-4o's limit) instead of sending the default 65000 which exceeds the model's supported maximum. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .../hindsight_api/engine/llm_wrapper.py | 6 +- .../litellm/hindsight_litellm/__init__.py | 74 ++++-------- .../litellm/hindsight_litellm/callbacks.py | 18 +-- .../litellm/hindsight_litellm/config.py | 47 +------- .../litellm/hindsight_litellm/wrappers.py | 109 ++++-------------- 5 files changed, 62 insertions(+), 192 deletions(-) diff --git a/hindsight-api/hindsight_api/engine/llm_wrapper.py b/hindsight-api/hindsight_api/engine/llm_wrapper.py index d3ff78ed32..4d87c33f7e 100644 --- a/hindsight-api/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api/hindsight_api/engine/llm_wrapper.py @@ -175,9 +175,13 @@ async def call( is_reasoning_model = any(x in model_lower for x in ["gpt-5", "o1", "o3"]) # For GPT-4 and GPT-4.1 models, cap max_completion_tokens to 32000 + # For GPT-4o models, cap to 16384 is_gpt4_model = any(x in model_lower for x in ["gpt-4.1", "gpt-4-"]) + is_gpt4o_model = "gpt-4o" in model_lower if max_completion_tokens is not None: - if is_gpt4_model and max_completion_tokens > 32000: + if is_gpt4o_model and max_completion_tokens > 16384: + max_completion_tokens = 16384 + elif is_gpt4_model and max_completion_tokens > 32000: max_completion_tokens = 32000 # For reasoning models, max_completion_tokens includes reasoning + output tokens # Enforce minimum of 16000 to ensure enough space for both diff --git a/hindsight-integrations/litellm/hindsight_litellm/__init__.py b/hindsight-integrations/litellm/hindsight_litellm/__init__.py index 5b55392191..e5c402646f 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/__init__.py +++ b/hindsight-integrations/litellm/hindsight_litellm/__init__.py @@ -9,7 +9,7 @@ - Automatic conversation storage after LLM calls - Works with any LiteLLM-supported provider - Zero code changes to existing LiteLLM usage -- Multi-user support via entity_id +- Multi-user support via separate bank_ids - Session management for conversation threading - Direct recall API for manual memory queries - Native client wrappers for OpenAI and Anthropic @@ -20,8 +20,7 @@ >>> # Configure Hindsight integration >>> configure( ... hindsight_api_url="http://localhost:8888", - ... bank_id="my-agent", - ... entity_id="user-123", # Multi-user support + ... bank_id="user-123", # Use separate bank_ids for multi-user support ... store_conversations=True, ... inject_memories=True, ... ) @@ -50,7 +49,7 @@ >>> from hindsight_litellm import wrap_openai >>> >>> client = OpenAI() - >>> wrapped = wrap_openai(client, bank_id="my-agent", entity_id="user-123") + >>> wrapped = wrap_openai(client, bank_id="user-123") >>> >>> response = wrapped.chat.completions.create( ... model="gpt-4", @@ -86,20 +85,20 @@ Context manager usage: >>> from hindsight_litellm import hindsight_memory >>> - >>> with hindsight_memory(bank_id="my-agent", entity_id="user-123"): + >>> with hindsight_memory(bank_id="user-123"): ... response = litellm.completion(model="gpt-4", messages=[...]) >>> # Memory integration automatically disabled after context Configuration options: - hindsight_api_url: URL of your Hindsight API server - - bank_id: Memory bank ID for memory operations (required) + - bank_id: Memory bank ID for memory operations (required). For multi-user + support, use different bank_ids per user (e.g., f"user-{user_id}") - api_key: Optional API key for Hindsight authentication - - entity_id: User identifier for multi-user memory isolation - session_id: Session identifier for conversation grouping - store_conversations: Whether to store conversations (default: True) - inject_memories: Whether to inject relevant memories (default: True) - injection_mode: How to inject memories (system_message or prepend_user) - - max_memories: Maximum number of memories to inject (default: 10) + - max_memories: Maximum number of memories to inject (None = unlimited) - recall_budget: Budget for memory recall (low, mid, high) - excluded_models: List of model patterns to exclude from interception - verbose: Enable verbose logging @@ -128,8 +127,6 @@ new_session, set_session, get_session, - set_entity, - get_entity, HindsightConfig, MemoryInjectionMode, ) @@ -180,8 +177,6 @@ class InjectionDebugInfo: mode: The injection mode used ("reflect" or "recall") query: The user query used for memory lookup bank_id: The bank ID used - scoped_bank_id: The bank ID with entity scoping applied - entity_id: The entity ID used (if any) memory_context: The formatted memory context that was injected reflect_text: The raw reflect text (when mode="reflect") reflect_facts: The facts used to generate the reflect response (when reflect_include_facts=True) @@ -193,8 +188,6 @@ class InjectionDebugInfo: mode: str # "reflect" or "recall" query: str bank_id: str - scoped_bank_id: str - entity_id: Optional[str] memory_context: str # The formatted context that was injected reflect_text: Optional[str] = None # Raw reflect response text reflect_facts: Optional[List[dict]] = None # Facts used by reflect (when reflect_include_facts=True) @@ -275,10 +268,8 @@ def _inject_memories(messages: List[dict]) -> List[dict]: try: from hindsight_client import Hindsight - # Build scoped bank_id - scoped_bank_id = config.bank_id - if config.entity_id: - scoped_bank_id = f"{config.bank_id}:{config.entity_id}" + # Use bank_id directly (no entity scoping) + bank_id = config.bank_id # Track debug info mode = "reflect" if config.use_reflect else "recall" @@ -307,7 +298,7 @@ def _inject_memories(messages: List[dict]) -> List[dict]: except RuntimeError: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - result = loop.run_until_complete(client._api.reflect(scoped_bank_id, request_obj)) + result = loop.run_until_complete(client._api.reflect(bank_id, request_obj)) # Extract facts from based_on if hasattr(result, 'based_on') and result.based_on: reflect_facts = [ @@ -320,7 +311,7 @@ def _inject_memories(messages: List[dict]) -> List[dict]: ] else: result = client.reflect( - bank_id=scoped_bank_id, + bank_id=bank_id, query=user_query, budget=config.recall_budget or "mid", ) @@ -332,9 +323,7 @@ def _inject_memories(messages: List[dict]) -> List[dict]: _last_injection_debug = InjectionDebugInfo( mode=mode, query=user_query, - bank_id=config.bank_id, - scoped_bank_id=scoped_bank_id, - entity_id=config.entity_id, + bank_id=bank_id, memory_context="", reflect_text="", reflect_facts=reflect_facts, @@ -351,7 +340,7 @@ def _inject_memories(messages: List[dict]) -> List[dict]: else: # Use recall API (original behavior) result = client.recall( - bank_id=scoped_bank_id, + bank_id=bank_id, query=user_query, budget=config.recall_budget or "mid", max_tokens=config.max_memory_tokens or 2000, @@ -379,9 +368,7 @@ def _inject_memories(messages: List[dict]) -> List[dict]: _last_injection_debug = InjectionDebugInfo( mode=mode, query=user_query, - bank_id=config.bank_id, - scoped_bank_id=scoped_bank_id, - entity_id=config.entity_id, + bank_id=bank_id, memory_context="", recall_results=[], results_count=0, @@ -404,9 +391,7 @@ def _inject_memories(messages: List[dict]) -> List[dict]: _last_injection_debug = InjectionDebugInfo( mode=mode, query=user_query, - bank_id=config.bank_id, - scoped_bank_id=scoped_bank_id, - entity_id=config.entity_id, + bank_id=bank_id, memory_context="", recall_results=recall_results, results_count=0, @@ -447,9 +432,7 @@ def _inject_memories(messages: List[dict]) -> List[dict]: _last_injection_debug = InjectionDebugInfo( mode=mode, query=user_query, - bank_id=config.bank_id, - scoped_bank_id=scoped_bank_id, - entity_id=config.entity_id, + bank_id=bank_id, memory_context=memory_context, reflect_text=reflect_text, reflect_facts=reflect_facts, @@ -471,8 +454,6 @@ def _inject_memories(messages: List[dict]) -> List[dict]: mode="reflect" if config.use_reflect else "recall", query=user_query or "", bank_id=config.bank_id or "", - scoped_bank_id=scoped_bank_id if 'scoped_bank_id' in dir() else config.bank_id or "", - entity_id=config.entity_id, memory_context="", results_count=0, injected=False, @@ -483,16 +464,10 @@ def _inject_memories(messages: List[dict]) -> List[dict]: # Always set debug info on error when verbose mode is on if config.verbose: logging.getLogger("hindsight_litellm").warning(f"Failed to inject memories: {e}") - # Build scoped bank_id for debug info - scoped_bank_id = config.bank_id - if config.entity_id: - scoped_bank_id = f"{config.bank_id}:{config.entity_id}" _last_injection_debug = InjectionDebugInfo( mode="reflect" if config.use_reflect else "recall", query=user_query or "", bank_id=config.bank_id or "", - scoped_bank_id=scoped_bank_id or "", - entity_id=config.entity_id, memory_context="", results_count=0, injected=False, @@ -714,12 +689,11 @@ def hindsight_memory( hindsight_api_url: str = "http://localhost:8888", bank_id: Optional[str] = None, api_key: Optional[str] = None, - entity_id: Optional[str] = None, session_id: Optional[str] = None, store_conversations: bool = True, inject_memories: bool = True, injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE, - max_memories: int = 10, + max_memories: Optional[int] = None, max_memory_tokens: int = 2000, recall_budget: str = "mid", fact_types: Optional[List[str]] = None, @@ -736,14 +710,14 @@ def hindsight_memory( Args: hindsight_api_url: URL of the Hindsight API server - bank_id: Memory bank ID for memory operations (required) + bank_id: Memory bank ID for memory operations (required). For multi-user + support, use different bank_ids per user (e.g., f"user-{user_id}") api_key: Optional API key for Hindsight authentication - entity_id: User identifier for multi-user memory isolation session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations inject_memories: Whether to inject relevant memories injection_mode: How to inject memories - max_memories: Maximum number of memories to inject + max_memories: Maximum number of memories to inject (None = unlimited) max_memory_tokens: Maximum tokens for memory context recall_budget: Budget for memory recall (low, mid, high) fact_types: List of fact types to filter (world, agent, opinion, observation) @@ -757,7 +731,7 @@ def hindsight_memory( >>> from hindsight_litellm import hindsight_memory >>> import litellm >>> - >>> with hindsight_memory(bank_id="my-agent", entity_id="user-123"): + >>> with hindsight_memory(bank_id="user-123"): ... response = litellm.completion(model="gpt-4", messages=[...]) >>> # Memory integration automatically disabled after context """ @@ -771,7 +745,6 @@ def hindsight_memory( hindsight_api_url=hindsight_api_url, bank_id=bank_id, api_key=api_key, - entity_id=entity_id, session_id=session_id, store_conversations=store_conversations, inject_memories=inject_memories, @@ -796,7 +769,6 @@ def hindsight_memory( hindsight_api_url=previous_config.hindsight_api_url, bank_id=previous_config.bank_id, api_key=previous_config.api_key, - entity_id=previous_config.entity_id, session_id=previous_config.session_id, store_conversations=previous_config.store_conversations, inject_memories=previous_config.inject_memories, @@ -828,12 +800,10 @@ def hindsight_memory( # LLM completion wrappers (convenience) "completion", "acompletion", - # Session/Entity management + # Session management "new_session", "set_session", "get_session", - "set_entity", - "get_entity", # Direct memory APIs "recall", "arecall", diff --git a/hindsight-integrations/litellm/hindsight_litellm/callbacks.py b/hindsight-integrations/litellm/hindsight_litellm/callbacks.py index 623e1441b7..49cdee9810 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/callbacks.py +++ b/hindsight-integrations/litellm/hindsight_litellm/callbacks.py @@ -248,10 +248,8 @@ def _inject_memories_into_messages( return updated_messages - def _get_scoped_bank_id(self, config: HindsightConfig) -> str: - """Get bank_id with entity scoping if entity_id is set.""" - if config.entity_id: - return f"{config.bank_id}:{config.entity_id}" + def _get_bank_id(self, config: HindsightConfig) -> str: + """Get the bank_id for API calls.""" return config.bank_id def _recall_memories_sync( @@ -261,8 +259,8 @@ def _recall_memories_sync( ) -> List[Dict[str, Any]]: """Recall relevant memories from Hindsight (sync) using direct HTTP.""" try: - scoped_bank_id = self._get_scoped_bank_id(config) - url = f"{config.hindsight_api_url}/v1/default/banks/{scoped_bank_id}/memories/recall" + bank_id = self._get_bank_id(config) + url = f"{config.hindsight_api_url}/v1/default/banks/{bank_id}/memories/recall" request_data = { "query": query, @@ -399,12 +397,8 @@ def _store_conversation_sync( if config.session_id: metadata["session_id"] = config.session_id - # Add entity_id to metadata if set - if config.entity_id: - metadata["entity_id"] = config.entity_id - - scoped_bank_id = self._get_scoped_bank_id(config) - url = f"{config.hindsight_api_url}/v1/default/banks/{scoped_bank_id}/memories" + bank_id = self._get_bank_id(config) + url = f"{config.hindsight_api_url}/v1/default/banks/{bank_id}/memories" request_data = { "items": [ diff --git a/hindsight-integrations/litellm/hindsight_litellm/config.py b/hindsight-integrations/litellm/hindsight_litellm/config.py index 85b90e901b..a8966013cd 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/config.py +++ b/hindsight-integrations/litellm/hindsight_litellm/config.py @@ -19,9 +19,9 @@ class HindsightConfig: Attributes: hindsight_api_url: URL of the Hindsight API server - bank_id: Memory bank ID for memory operations (required) + bank_id: Memory bank ID for memory operations (required). For multi-user + support, use different bank_ids per user (e.g., f"user-{user_id}") api_key: Optional API key for Hindsight authentication - entity_id: User/entity identifier for memory scoping (multi-user support) session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations to Hindsight inject_memories: Whether to inject relevant memories into prompts @@ -42,7 +42,6 @@ class HindsightConfig: hindsight_api_url: str = "http://localhost:8888" bank_id: Optional[str] = None api_key: Optional[str] = None - entity_id: Optional[str] = None # User identifier for multi-user memory isolation session_id: Optional[str] = None # Session identifier for conversation grouping store_conversations: bool = True inject_memories: bool = True @@ -69,7 +68,6 @@ def configure( hindsight_api_url: str = "http://localhost:8888", bank_id: Optional[str] = None, api_key: Optional[str] = None, - entity_id: Optional[str] = None, session_id: Optional[str] = None, store_conversations: bool = True, inject_memories: bool = True, @@ -94,9 +92,9 @@ def configure( Args: hindsight_api_url: URL of the Hindsight API server - bank_id: Memory bank ID for memory operations (required) + bank_id: Memory bank ID for memory operations (required). For multi-user + support, use different bank_ids per user (e.g., f"user-{user_id}") api_key: Optional API key for Hindsight authentication - entity_id: User/entity identifier for multi-user memory isolation session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations to Hindsight inject_memories: Whether to inject relevant memories into prompts @@ -127,8 +125,7 @@ def configure( >>> from hindsight_litellm import configure, enable >>> configure( ... hindsight_api_url="http://localhost:8888", - ... bank_id="my-agent", - ... entity_id="user-123", # Multi-user support + ... bank_id="user-123", # Per-user bank for multi-user support ... store_conversations=True, ... inject_memories=True, ... background="This agent routes customer requests to support channels. " @@ -142,7 +139,6 @@ def configure( hindsight_api_url=hindsight_api_url, bank_id=bank_id, api_key=api_key, - entity_id=entity_id, session_id=session_id, store_conversations=store_conversations, inject_memories=inject_memories, @@ -309,36 +305,3 @@ def get_session() -> Optional[str]: return _global_config.session_id -def set_entity(entity_id: str) -> None: - """Set the entity ID for multi-user memory isolation. - - Args: - entity_id: The entity/user identifier - - Raises: - RuntimeError: If Hindsight has not been configured - - Example: - >>> from hindsight_litellm import configure, set_entity - >>> configure(bank_id="my-agent") - >>> set_entity("user-123") # Switch to this user's memories - """ - global _global_config - - if _global_config is None: - raise RuntimeError( - "Hindsight not configured. Call configure() before set_entity()." - ) - - _global_config.entity_id = entity_id - - -def get_entity() -> Optional[str]: - """Get the current entity ID. - - Returns: - The current entity ID, or None if not set - """ - if _global_config is None: - return None - return _global_config.entity_id diff --git a/hindsight-integrations/litellm/hindsight_litellm/wrappers.py b/hindsight-integrations/litellm/hindsight_litellm/wrappers.py index 312c1e07ea..baa0e8426a 100644 --- a/hindsight-integrations/litellm/hindsight_litellm/wrappers.py +++ b/hindsight-integrations/litellm/hindsight_litellm/wrappers.py @@ -34,8 +34,6 @@ class RecallDebugInfo: """Debug information from a recall operation.""" query: str bank_id: str - scoped_bank_id: str - entity_id: Optional[str] budget: str max_tokens: int fact_types: Optional[List[str]] @@ -65,7 +63,6 @@ def __bool__(self): def recall( query: str, bank_id: Optional[str] = None, - entity_id: Optional[str] = None, fact_types: Optional[List[str]] = None, budget: Optional[str] = None, max_tokens: Optional[int] = None, @@ -78,8 +75,8 @@ def recall( Args: query: The query string to search memories for - bank_id: Override the configured bank_id - entity_id: Override the configured entity_id for multi-user isolation + bank_id: Override the configured bank_id. For multi-user support, + use different bank_ids per user (e.g., f"user-{user_id}") fact_types: Filter by fact types (world, agent, opinion, observation) budget: Recall budget level (low, mid, high) - controls how many memories are returned max_tokens: Maximum tokens for memory context @@ -107,14 +104,13 @@ def recall( >>> configure(bank_id="my-agent", verbose=True) >>> memories = recall("what projects am I working on?") >>> if memories.debug: - ... print(f"Queried bank: {memories.debug.scoped_bank_id}") + ... print(f"Queried bank: {memories.debug.bank_id}") """ # Get config or use overrides config = get_config() api_url = hindsight_api_url or (config.hindsight_api_url if config else None) target_bank_id = bank_id or (config.bank_id if config else None) - target_entity_id = entity_id or (config.entity_id if config else None) target_fact_types = fact_types or (config.fact_types if config else None) target_budget = budget or (config.recall_budget if config else "mid") target_max_tokens = max_tokens or (config.max_memory_tokens if config else 2000) @@ -129,14 +125,9 @@ def recall( client = Hindsight(base_url=api_url, timeout=30.0) - # Build bank_id with entity scoping if entity_id is set - scoped_bank_id = target_bank_id - if target_entity_id: - scoped_bank_id = f"{target_bank_id}:{target_entity_id}" - # Call recall API results = client.recall( - bank_id=scoped_bank_id, + bank_id=target_bank_id, query=query, types=target_fact_types, budget=target_budget, @@ -172,8 +163,6 @@ def recall( debug_info = RecallDebugInfo( query=query, bank_id=target_bank_id, - scoped_bank_id=scoped_bank_id, - entity_id=target_entity_id, budget=target_budget, max_tokens=target_max_tokens, fact_types=target_fact_types, @@ -194,7 +183,6 @@ def recall( async def arecall( query: str, bank_id: Optional[str] = None, - entity_id: Optional[str] = None, fact_types: Optional[List[str]] = None, budget: Optional[str] = None, max_tokens: Optional[int] = None, @@ -211,7 +199,6 @@ async def arecall( lambda: recall( query=query, bank_id=bank_id, - entity_id=entity_id, fact_types=fact_types, budget=budget, max_tokens=max_tokens, @@ -225,8 +212,6 @@ class ReflectDebugInfo: """Debug information from a reflect operation.""" query: str bank_id: str - scoped_bank_id: str - entity_id: Optional[str] budget: str context: Optional[str] api_url: str @@ -246,7 +231,6 @@ def __str__(self) -> str: def reflect( query: str, bank_id: Optional[str] = None, - entity_id: Optional[str] = None, budget: Optional[str] = None, context: Optional[str] = None, hindsight_api_url: Optional[str] = None, @@ -258,8 +242,8 @@ def reflect( Args: query: The question or prompt to answer - bank_id: Override the configured bank_id - entity_id: Override the configured entity_id for multi-user isolation + bank_id: Override the configured bank_id. For multi-user support, + use different bank_ids per user (e.g., f"user-{user_id}") budget: Budget level for reflection (low, mid, high) context: Additional context to include in the reflection hindsight_api_url: Override the configured API URL @@ -283,7 +267,6 @@ def reflect( api_url = hindsight_api_url or (config.hindsight_api_url if config else None) target_bank_id = bank_id or (config.bank_id if config else None) - target_entity_id = entity_id or (config.entity_id if config else None) target_budget = budget or (config.recall_budget if config else "mid") if not api_url or not target_bank_id: @@ -296,14 +279,9 @@ def reflect( client = Hindsight(base_url=api_url, timeout=30.0) - # Build bank_id with entity scoping if entity_id is set - scoped_bank_id = target_bank_id - if target_entity_id: - scoped_bank_id = f"{target_bank_id}:{target_entity_id}" - # Call reflect API result = client.reflect( - bank_id=scoped_bank_id, + bank_id=target_bank_id, query=query, budget=target_budget, context=context, @@ -319,8 +297,6 @@ def reflect( debug_info = ReflectDebugInfo( query=query, bank_id=target_bank_id, - scoped_bank_id=scoped_bank_id, - entity_id=target_entity_id, budget=target_budget, context=context, api_url=api_url, @@ -339,7 +315,6 @@ def reflect( async def areflect( query: str, bank_id: Optional[str] = None, - entity_id: Optional[str] = None, budget: Optional[str] = None, context: Optional[str] = None, hindsight_api_url: Optional[str] = None, @@ -355,7 +330,6 @@ async def areflect( lambda: reflect( query=query, bank_id=bank_id, - entity_id=entity_id, budget=budget, context=context, hindsight_api_url=hindsight_api_url, @@ -368,8 +342,6 @@ class RetainDebugInfo: """Debug information from a retain operation.""" content: str bank_id: str - scoped_bank_id: str - entity_id: Optional[str] context: Optional[str] document_id: Optional[str] metadata: Optional[Dict[str, str]] @@ -390,7 +362,6 @@ def __bool__(self) -> bool: def retain( content: str, bank_id: Optional[str] = None, - entity_id: Optional[str] = None, context: Optional[str] = None, document_id: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, @@ -404,8 +375,8 @@ def retain( Args: content: The text content to store - bank_id: Override the configured bank_id - entity_id: Override the configured entity_id for multi-user isolation + bank_id: Override the configured bank_id. For multi-user support, + use different bank_ids per user (e.g., f"user-{user_id}") context: Context description for the memory (e.g., "customer_feedback") document_id: Optional document ID for grouping related memories metadata: Optional key-value metadata to attach to the memory @@ -435,7 +406,6 @@ def retain( api_url = hindsight_api_url or (config.hindsight_api_url if config else None) target_bank_id = bank_id or (config.bank_id if config else None) - target_entity_id = entity_id or (config.entity_id if config else None) target_document_id = document_id or (config.document_id if config else None) if not api_url or not target_bank_id: @@ -448,14 +418,9 @@ def retain( client = Hindsight(base_url=api_url, timeout=30.0) - # Build bank_id with entity scoping if entity_id is set - scoped_bank_id = target_bank_id - if target_entity_id: - scoped_bank_id = f"{target_bank_id}:{target_entity_id}" - # Call retain API result = client.retain( - bank_id=scoped_bank_id, + bank_id=target_bank_id, content=content, context=context, document_id=target_document_id, @@ -473,8 +438,6 @@ def retain( debug_info = RetainDebugInfo( content=content, bank_id=target_bank_id, - scoped_bank_id=scoped_bank_id, - entity_id=target_entity_id, context=context, document_id=target_document_id, metadata=metadata, @@ -494,7 +457,6 @@ def retain( async def aretain( content: str, bank_id: Optional[str] = None, - entity_id: Optional[str] = None, context: Optional[str] = None, document_id: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, @@ -511,7 +473,6 @@ async def aretain( lambda: retain( content=content, bank_id=bank_id, - entity_id=entity_id, context=context, document_id=document_id, metadata=metadata, @@ -544,7 +505,6 @@ def __init__( client: Any, bank_id: str, hindsight_api_url: str = "http://localhost:8888", - entity_id: Optional[str] = None, session_id: Optional[str] = None, store_conversations: bool = True, inject_memories: bool = True, @@ -556,9 +516,9 @@ def __init__( Args: client: The OpenAI client instance to wrap - bank_id: Memory bank ID for memory operations + bank_id: Memory bank ID for memory operations. For multi-user support, + use different bank_ids per user (e.g., f"user-{user_id}") hindsight_api_url: URL of the Hindsight API server - entity_id: User identifier for multi-user memory isolation session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations inject_memories: Whether to inject relevant memories @@ -569,7 +529,6 @@ def __init__( self._client = client self._bank_id = bank_id self._api_url = hindsight_api_url - self._entity_id = entity_id self._session_id = session_id self._store_conversations = store_conversations self._inject_memories = inject_memories @@ -591,12 +550,6 @@ def _get_hindsight_client(self): ) return self._hindsight_client - def _get_scoped_bank_id(self) -> str: - """Get bank_id with entity scoping if set.""" - if self._entity_id: - return f"{self._bank_id}:{self._entity_id}" - return self._bank_id - def _recall_memories(self, query: str) -> str: """Recall and format memories for injection.""" if not self._inject_memories: @@ -605,7 +558,7 @@ def _recall_memories(self, query: str) -> str: try: client = self._get_hindsight_client() results = client.recall( - bank_id=self._get_scoped_bank_id(), + bank_id=self._bank_id, query=query, budget=self._recall_budget, max_tokens=self._max_memories * 200 if self._max_memories else 2000, @@ -652,7 +605,7 @@ def _store_conversation(self, user_input: str, assistant_output: str, model: str metadata["session_id"] = self._session_id client.retain( - bank_id=self._get_scoped_bank_id(), + bank_id=self._bank_id, content=conversation_text, context=f"conversation:openai:{model}", metadata=metadata, @@ -756,7 +709,6 @@ def __init__( client: Any, bank_id: str, hindsight_api_url: str = "http://localhost:8888", - entity_id: Optional[str] = None, session_id: Optional[str] = None, store_conversations: bool = True, inject_memories: bool = True, @@ -768,9 +720,9 @@ def __init__( Args: client: The Anthropic client instance to wrap - bank_id: Memory bank ID for memory operations + bank_id: Memory bank ID for memory operations. For multi-user support, + use different bank_ids per user (e.g., f"user-{user_id}") hindsight_api_url: URL of the Hindsight API server - entity_id: User identifier for multi-user memory isolation session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations inject_memories: Whether to inject relevant memories @@ -781,7 +733,6 @@ def __init__( self._client = client self._bank_id = bank_id self._api_url = hindsight_api_url - self._entity_id = entity_id self._session_id = session_id self._store_conversations = store_conversations self._inject_memories = inject_memories @@ -803,12 +754,6 @@ def _get_hindsight_client(self): ) return self._hindsight_client - def _get_scoped_bank_id(self) -> str: - """Get bank_id with entity scoping if set.""" - if self._entity_id: - return f"{self._bank_id}:{self._entity_id}" - return self._bank_id - def _recall_memories(self, query: str) -> str: """Recall and format memories for injection.""" if not self._inject_memories: @@ -817,7 +762,7 @@ def _recall_memories(self, query: str) -> str: try: client = self._get_hindsight_client() results = client.recall( - bank_id=self._get_scoped_bank_id(), + bank_id=self._bank_id, query=query, budget=self._recall_budget, max_tokens=self._max_memories * 200 if self._max_memories else 2000, @@ -864,7 +809,7 @@ def _store_conversation(self, user_input: str, assistant_output: str, model: str metadata["session_id"] = self._session_id client.retain( - bank_id=self._get_scoped_bank_id(), + bank_id=self._bank_id, content=conversation_text, context=f"conversation:anthropic:{model}", metadata=metadata, @@ -940,7 +885,6 @@ def wrap_openai( client: Any, bank_id: str, hindsight_api_url: str = "http://localhost:8888", - entity_id: Optional[str] = None, session_id: Optional[str] = None, store_conversations: bool = True, inject_memories: bool = True, @@ -955,9 +899,9 @@ def wrap_openai( Args: client: The OpenAI client instance to wrap - bank_id: Memory bank ID for memory operations + bank_id: Memory bank ID for memory operations. For multi-user support, + use different bank_ids per user (e.g., f"user-{user_id}") hindsight_api_url: URL of the Hindsight API server - entity_id: User identifier for multi-user memory isolation session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations inject_memories: Whether to inject relevant memories @@ -975,8 +919,7 @@ def wrap_openai( >>> client = OpenAI() >>> wrapped = wrap_openai( ... client, - ... bank_id="my-agent", - ... entity_id="user-123", # Multi-user support + ... bank_id=f"user-{user_id}", # Multi-user support via separate banks ... ) >>> >>> response = wrapped.chat.completions.create( @@ -988,7 +931,6 @@ def wrap_openai( client=client, bank_id=bank_id, hindsight_api_url=hindsight_api_url, - entity_id=entity_id, session_id=session_id, store_conversations=store_conversations, inject_memories=inject_memories, @@ -1002,7 +944,6 @@ def wrap_anthropic( client: Any, bank_id: str, hindsight_api_url: str = "http://localhost:8888", - entity_id: Optional[str] = None, session_id: Optional[str] = None, store_conversations: bool = True, inject_memories: bool = True, @@ -1017,9 +958,9 @@ def wrap_anthropic( Args: client: The Anthropic client instance to wrap - bank_id: Memory bank ID for memory operations + bank_id: Memory bank ID for memory operations. For multi-user support, + use different bank_ids per user (e.g., f"user-{user_id}") hindsight_api_url: URL of the Hindsight API server - entity_id: User identifier for multi-user memory isolation session_id: Session identifier for conversation grouping store_conversations: Whether to store conversations inject_memories: Whether to inject relevant memories @@ -1037,8 +978,7 @@ def wrap_anthropic( >>> client = Anthropic() >>> wrapped = wrap_anthropic( ... client, - ... bank_id="my-agent", - ... entity_id="user-123", # Multi-user support + ... bank_id=f"user-{user_id}", # Multi-user support via separate banks ... ) >>> >>> response = wrapped.messages.create( @@ -1051,7 +991,6 @@ def wrap_anthropic( client=client, bank_id=bank_id, hindsight_api_url=hindsight_api_url, - entity_id=entity_id, session_id=session_id, store_conversations=store_conversations, inject_memories=inject_memories, From 4773d76fbe21d8f688094a95651210d706e41a4d Mon Sep 17 00:00:00 2001 From: DK09876 Date: Thu, 11 Dec 2025 16:35:17 -0600 Subject: [PATCH 07/10] Fix dark mode styling across Control Plane UI components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improvements to ensure proper text visibility and contrast in both light and dark modes: - Add global CSS rules for datetime-local calendar picker icon visibility using filter: invert() for both light (0.5) and dark (1) modes - Fix text colors in dialog components to use theme-aware foreground colors - Update memory detail panel, document/chunk modals, and data views to use proper dark mode text classes (text-foreground, text-card-foreground) - Fix form labels, headings, and content text in bank selector dialogs - Update entities view and documents view table styling for dark mode - Bump package versions to 0.1.4 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- hindsight-control-plane/src/app/globals.css | 9 ++++++ .../src/components/bank-profile-view.tsx | 4 +-- .../src/components/bank-selector.tsx | 15 +++++----- .../src/components/data-view.tsx | 24 ++++++++-------- .../src/components/document-chunk-modal.tsx | 22 +++++++-------- .../src/components/documents-view.tsx | 28 +++++++++---------- .../src/components/memory-detail-panel.tsx | 18 ++++++------ .../src/components/ui/dialog.tsx | 4 +-- 8 files changed, 67 insertions(+), 57 deletions(-) diff --git a/hindsight-control-plane/src/app/globals.css b/hindsight-control-plane/src/app/globals.css index e41afbd5c6..b58199f181 100644 --- a/hindsight-control-plane/src/app/globals.css +++ b/hindsight-control-plane/src/app/globals.css @@ -180,4 +180,13 @@ code, pre { -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text; +} + +/* Fix datetime-local calendar icon visibility in both light and dark modes */ +input[type="datetime-local"]::-webkit-calendar-picker-indicator { + filter: invert(0.5); +} + +.dark input[type="datetime-local"]::-webkit-calendar-picker-indicator { + filter: invert(1); } \ No newline at end of file diff --git a/hindsight-control-plane/src/components/bank-profile-view.tsx b/hindsight-control-plane/src/components/bank-profile-view.tsx index 7a4896ce83..aa823d86ce 100644 --- a/hindsight-control-plane/src/components/bank-profile-view.tsx +++ b/hindsight-control-plane/src/components/bank-profile-view.tsx @@ -239,7 +239,7 @@ export function BankProfileView() {
{editMode ? ( <> - diff --git a/hindsight-control-plane/src/components/bank-selector.tsx b/hindsight-control-plane/src/components/bank-selector.tsx index 76b3aa601a..8ff4573dc0 100644 --- a/hindsight-control-plane/src/components/bank-selector.tsx +++ b/hindsight-control-plane/src/components/bank-selector.tsx @@ -266,7 +266,7 @@ function BankSelectorInner() {