Implement Memory Guard Lite for OSS

.gitignore

@@ -61,3 +61,6 @@ hindsight-integrations/_drafts/
 
 blog-post*
 .worktrees/
+
+# Local-only plan + scratch directory (not part of the repo)
+docs/

hindsight-api-slim/hindsight_api/alembic/versions/bb22cc33dd44_add_status_to_memory_units.py

@@ -0,0 +1,61 @@
+"""Add status column to memory_units (Memory Guard quarantine).
+
+Revision ID: bb22cc33dd44
+Revises: z1u2v3w4x5y6
+Create Date: 2026-06-04
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "bb22cc33dd44"
+down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_VALID = "('active','quarantined','released')"
+
+
+def _pg_schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _pg_upgrade() -> None:
+    s = _pg_schema_prefix()
+    op.execute(f"ALTER TABLE {s}memory_units ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active'")
+    op.execute(f"ALTER TABLE {s}memory_units ADD CONSTRAINT memory_units_status_check CHECK (status IN {_VALID})")
+    op.execute(
+        f"CREATE INDEX IF NOT EXISTS idx_memory_units_status "
+        f"ON {s}memory_units (bank_id, status) WHERE status <> 'active'"
+    )
+
+
+def _pg_downgrade() -> None:
+    s = _pg_schema_prefix()
+    op.execute(f"DROP INDEX IF EXISTS {s}idx_memory_units_status")
+    op.execute(f"ALTER TABLE {s}memory_units DROP CONSTRAINT IF EXISTS memory_units_status_check")
+    op.execute(f"ALTER TABLE {s}memory_units DROP COLUMN IF EXISTS status")
+
+
+def _oracle_upgrade() -> None:
+    op.execute("ALTER TABLE memory_units ADD status VARCHAR2(32) DEFAULT 'active' NOT NULL")
+    op.execute(f"ALTER TABLE memory_units ADD CONSTRAINT memory_units_status_check CHECK (status IN {_VALID})")
+    op.execute("CREATE INDEX idx_memory_units_status ON memory_units (bank_id, status)")
+
+
+def _oracle_downgrade() -> None:
+    op.execute("DROP INDEX idx_memory_units_status")
+    op.execute("ALTER TABLE memory_units DROP CONSTRAINT memory_units_status_check")
+    op.execute("ALTER TABLE memory_units DROP COLUMN status")
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade, oracle=_oracle_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade, oracle=_oracle_downgrade)

hindsight-api-slim/hindsight_api/alembic/versions/dd44ee55ff66_merge_memory_guard_and_llm_tracing_heads.py

@@ -0,0 +1,39 @@
+"""merge status / llm tracing / history-split heads
+
+Originally a two-way merge for the memory-guard and llm-tracing branches.
+The memory-guard branch (``cc33dd44ee55`` add_security_events_table and the
+``b5e7f1a2c3d4`` verified-columns follow-up) was deleted when Memory Defense
+moved out of api-slim, so this migration now absorbs the work the deleted
+``b5e7f1a2c3d4`` merge used to do: collapse the three remaining heads
+(``bb22cc33dd44`` add_status, ``d3e4f5a6b7c8`` add_llm_requests_table,
+``a7b8c9d0e1f2`` split_history_into_own_tables) into one.
+
+Revision ID: dd44ee55ff66
+Revises: bb22cc33dd44, d3e4f5a6b7c8, a7b8c9d0e1f2
+Create Date: 2026-06-04
+"""
+
+from collections.abc import Sequence
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "dd44ee55ff66"
+down_revision: str | Sequence[str] | None = ("bb22cc33dd44", "a7b8c9d0e1f2")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _pg_upgrade() -> None:
+    pass  # merge migration — no schema changes
+
+
+def _pg_downgrade() -> None:
+    pass  # merge migration — no schema changes
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade)

hindsight-api-slim/hindsight_api/alembic/versions/f1e2d3c4b5a6_merge_maintenance_repair_and_memory_guard_heads.py

@@ -0,0 +1,46 @@
+"""Merge maintenance routines repair and memory guard heads.
+
+Revision ID: f1e2d3c4b5a6
+Revises: b2d4f6a8c1e3, dd44ee55ff66
+Create Date: 2026-06-08
+
+The rebase of feat/memory-defense-extension onto main produced two parallel
+Alembic heads: b2d4f6a8c1e3 (server-side maintenance routine repair, from
+main) and dd44ee55ff66 (Memory Guard quarantine status column + history
+split merge, from the feature branch). Structural merge revision with no
+schema changes; only job is to unify the DAG so ``alembic upgrade head``
+is unambiguous again.
+"""
+
+from collections.abc import Sequence
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "f1e2d3c4b5a6"
+down_revision: str | Sequence[str] | None = ("b2d4f6a8c1e3", "dd44ee55ff66")
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _pg_upgrade() -> None:
+    pass
+
+
+def _pg_downgrade() -> None:
+    pass
+
+
+def _oracle_upgrade() -> None:
+    pass
+
+
+def _oracle_downgrade() -> None:
+    pass
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade, oracle=_oracle_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade, oracle=_oracle_downgrade)

hindsight-api-slim/hindsight_api/api/http.py

@@ -476,6 +476,10 @@ class MemoryItem(BaseModel):
         default=None,
         description="Optional tags for visibility scoping. Memories with tags can be filtered during recall.",
     )
+    receipt_uri: str | None = Field(
+        default=None,
+        description="Optional URI referencing a security receipt for this memory item.",
+    )
 
     @field_validator("content")
     @classmethod
@@ -5627,6 +5631,15 @@ async def api_update_bank_config(
                     app.state.memory._operation_validator.validate_bank_write(ctx)
                 )
 
+            # Validate Memory Defense policy shape before persisting.
+            if "memory_defense" in request.updates and request.updates["memory_defense"] is not None:
+                from hindsight_api.extensions.memory_defense import parse_policy
+
+                try:
+                    parse_policy(request.updates["memory_defense"])
+                except ValueError as exc:
+                    raise HTTPException(status_code=422, detail=f"invalid memory_defense policy: {exc}")
+
             # Update config via config resolver (validates configurable fields and permissions)
             await app.state.memory._config_resolver.update_bank_config(bank_id, request.updates, request_context)
 
@@ -6156,6 +6169,10 @@ async def api_retain(
         except (AuthenticationError, HTTPException):
             raise
         except Exception as e:
+            from hindsight_api.engine.retain.orchestrator import MemoryDefenseAllBlockedError
+
+            if isinstance(e, MemoryDefenseAllBlockedError):
+                raise HTTPException(status_code=422, detail={"violations": e.violations})
             import traceback
 
             # Create a summary of the input for debugging

hindsight-api-slim/hindsight_api/config.py

@@ -458,6 +458,9 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]:
 ENV_WEBHOOK_EVENT_TYPES = "HINDSIGHT_API_WEBHOOK_EVENT_TYPES"
 ENV_WEBHOOK_DELIVERY_POLL_INTERVAL_SECONDS = "HINDSIGHT_API_WEBHOOK_DELIVERY_POLL_INTERVAL_SECONDS"
 
+# Memory Defense configuration (server-level defaults)
+ENV_MEMORY_DEFENSE_ENABLED_DEFAULT = "HINDSIGHT_API_MEMORY_DEFENSE_ENABLED_DEFAULT"
+
 # Built-in llama.cpp configuration (for provider=llamacpp)
 ENV_LLAMACPP_MODEL_PATH = "HINDSIGHT_API_LLAMACPP_MODEL_PATH"
 ENV_LLAMACPP_GPU_LAYERS = "HINDSIGHT_API_LLAMACPP_GPU_LAYERS"
@@ -881,6 +884,9 @@ def _parse_strategy_boosts(raw: str | None) -> dict[str, str]:
 DEFAULT_OBSERVATIONS_MISSION = None  # Declarative spec of what observations are for this bank
 DEFAULT_MAX_OBSERVATIONS_PER_SCOPE = -1  # Max observations per tag scope (-1 = unlimited)
 
+# Memory Defense defaults (server-level configuration)
+DEFAULT_MEMORY_DEFENSE_ENABLED_DEFAULT = False  # Default: new banks have memory_defense disabled
+
 # Database migrations
 DEFAULT_RUN_MIGRATIONS_ON_STARTUP = True
 
@@ -1470,6 +1476,10 @@ class HindsightConfig:
     # When False: only label entities are extracted (or no entities at all if no labels configured)
     entities_allow_free_form: bool
 
+    # Memory Defense policy (dict matching DefensePolicy schema — validated on write)
+    # None = Memory Defense disabled / not configured for this bank
+    memory_defense: dict | None
+
     # Reflect agent settings
     reflect_mission: str | None
     reflect_source_facts_max_tokens: int
@@ -1554,6 +1564,9 @@ class HindsightConfig:
     # eligible-but-unscheduled facts. 0 = disabled.
     consolidation_reconcile_interval_seconds: int
 
+    # Memory Defense configuration (static - server-level defaults for new banks)
+    memory_defense_enabled_default: bool  # Default: new banks have memory_defense disabled
+
     # Webhook configuration (static - server-level only, not per-bank)
     webhook_url: str | None  # Global webhook URL (None = disabled)
     webhook_secret: str | None  # HMAC signing secret (None = unsigned)
@@ -1664,6 +1677,8 @@ class HindsightConfig:
         "disposition_empathy",
         # Gemini safety settings (controls content filtering for Gemini/VertexAI providers)
         "llm_gemini_safety_settings",
+        # Memory Defense policy (validated against DefensePolicy schema on write)
+        "memory_defense",
     }
 
     @property
@@ -2387,6 +2402,7 @@ def from_env(cls) -> "HindsightConfig":
             ),
             entity_labels=None,
             entities_allow_free_form=True,
+            memory_defense=None,
             # Database migrations
             run_migrations_on_startup=os.getenv(ENV_RUN_MIGRATIONS_ON_STARTUP, "true").lower() == "true",
             # Database connection pool
@@ -2496,6 +2512,11 @@ def from_env(cls) -> "HindsightConfig":
                     str(DEFAULT_CONSOLIDATION_RECONCILE_INTERVAL_SECONDS),
                 )
             ),
+            # Memory Defense configuration (static, server-level defaults)
+            memory_defense_enabled_default=os.getenv(
+                ENV_MEMORY_DEFENSE_ENABLED_DEFAULT, str(DEFAULT_MEMORY_DEFENSE_ENABLED_DEFAULT)
+            ).lower()
+            == "true",
             # Webhook configuration (static, server-level only)
             webhook_url=os.getenv(ENV_WEBHOOK_URL) or DEFAULT_WEBHOOK_URL,
             webhook_secret=os.getenv(ENV_WEBHOOK_SECRET) or DEFAULT_WEBHOOK_SECRET,

hindsight-api-slim/hindsight_api/engine/memory_engine.py

@@ -976,6 +976,34 @@ def __init__(
             tenant_extension = DefaultTenantExtension(config={})
         self._tenant_extension = tenant_extension
 
+        # Load memory defense extension; default to Lite when env var is unset.
+        # Lazy imports avoid a circular dependency: extensions/__init__ imports
+        # MCPExtension which imports MemoryEngine at module level.
+        from ..extensions.builtin.memory_defense_lite import (  # noqa: PLC0415
+            MemoryDefenseLiteExtension,
+        )
+        from ..extensions.context import DefaultExtensionContext  # noqa: PLC0415
+        from ..extensions.loader import load_extension  # noqa: PLC0415
+        from ..extensions.memory_defense import MemoryDefenseExtension  # noqa: PLC0415
+
+        # Build the extension context now; webhook_manager is populated later in
+        # initialize() once the pool is ready.  current_schema is a per-request
+        # value written by _authenticate() and execute_task().
+        self._ext_ctx = DefaultExtensionContext(
+            database_url=config.database_url or "",
+            memory_engine=self,
+            webhook_manager=None,
+            current_schema=None,
+        )
+
+        loaded = load_extension("MEMORY_DEFENSE", MemoryDefenseExtension, context=self._ext_ctx)
+        if loaded is not None:
+            self._memory_defense: MemoryDefenseExtension = loaded
+        else:
+            lite = MemoryDefenseLiteExtension({})
+            lite.set_context(self._ext_ctx)
+            self._memory_defense = lite
+
         # Cache for get_bank_stats — short TTL + concurrent-loader coalescing.
         # The query joins memory_links to memory_units and can be a multi-second
         # parallel scan on large banks; a single polling client used to be able
@@ -1055,6 +1083,7 @@ async def _authenticate_tenant(self, request_context: "RequestContext | None") -
         tenant_context = await self._tenant_extension.authenticate(request_context)
 
         _current_schema.set(tenant_context.schema_name)
+        self._ext_ctx.current_schema = tenant_context.schema_name
         return tenant_context.schema_name
 
     async def _handle_import_documents(self, task_dict: dict[str, Any]):
@@ -1540,6 +1569,7 @@ async def execute_task(self, task_dict: dict[str, Any]):
         schema = task_dict.pop("_schema", None)
         if schema:
             _current_schema.set(schema)
+            self._ext_ctx.current_schema = schema
 
         # Check if operation was cancelled (only for tasks with operation_id)
         if operation_id:
@@ -2634,6 +2664,9 @@ async def _init_connection(conn: asyncpg.Connection) -> None:
             global_webhooks=webhook_global,
             tenant_extension=self._tenant_extension,
         )
+        # Propagate the now-ready webhook manager to the extension context so
+        # that the Memory Defense extension can fire webhooks.
+        self._ext_ctx.webhook_manager = self._webhook_manager
         logger.debug("Webhook manager initialized")
 
         # Long-lived HTTP client for webhook delivery tasks
@@ -3350,6 +3383,8 @@ async def _retain_batch_async_internal(
                 # Stream chunk-level "storing N/total" progress to the operation row as
                 # the document's chunks commit (more useful than the coarse sub-batch tick).
                 progress_callback=self._write_operation_progress,
+                webhook_manager=self._webhook_manager,
+                memory_defense_extension=self._memory_defense,
             )
             # Map the created facts onto this retain's trace so the trace view can
             # show which memories the ingestion produced. result[0] is the
@@ -4386,6 +4421,21 @@ def to_tuple_format(results):
                     {"reranker_type": rerank_kind, "candidates_reranked": len(scored_results)},
                 )
 
+            # Step 4.8: Post-filter by quarantine status.
+            # Applied after reranking so score ordering is preserved.
+            # Quarantined rows are never surfaced via recall; they remain in the DB
+            # as audit-only orphans. None status (e.g. graph-expanded rows) is
+            # treated as active.
+            pre_filter_len = len(scored_results)
+            filtered: list = []
+            for sr in scored_results:
+                if sr.retrieval.status == "quarantined":
+                    continue
+                filtered.append(sr)
+            if len(filtered) != pre_filter_len:
+                log_buffer.append(f"  [4.8] Quarantine filter: {pre_filter_len} -> {len(filtered)}")
+            scored_results = filtered
+
             # Step 5: Truncate to thinking_budget * 2 for token filtering
             rerank_limit = thinking_budget * 2
             top_scored = scored_results[:rerank_limit]
@@ -4710,6 +4760,7 @@ def _make_source_fact(sid: str, r: Any) -> MemoryFact:
                         chunk_id=result_dict.get("chunk_id"),
                         tags=result_dict.get("tags"),
                         source_fact_ids=source_fact_ids_by_obs.get(result_id) if include_source_facts else None,
+                        status=result_dict.get("status"),
                     )
                 )
 
@@ -6191,7 +6242,9 @@ async def list_memory_units(
 
             units = await conn.fetch(
                 f"""
-                SELECT id, text, event_date, context, fact_type, mentioned_at, occurred_start, occurred_end, chunk_id, proof_count, tags, consolidated_at, consolidation_failed_at
+                SELECT id, text, event_date, context, fact_type, status, document_id,
+                       mentioned_at, occurred_start, occurred_end, chunk_id, proof_count,
+                       tags, consolidated_at, consolidation_failed_at
                 FROM {fq_table("memory_units")}
                 {where_clause}
                 ORDER BY mentioned_at DESC NULLS LAST, created_at DESC
@@ -6238,6 +6291,8 @@ async def list_memory_units(
                         "context": row["context"] if row["context"] else "",
                         "date": row["event_date"].isoformat() if row["event_date"] else "",
                         "fact_type": row["fact_type"],
+                        "status": row["status"],
+                        "document_id": row["document_id"],
                         "mentioned_at": row["mentioned_at"].isoformat() if row["mentioned_at"] else None,
                         "occurred_start": row["occurred_start"].isoformat() if row["occurred_start"] else None,
                         "occurred_end": row["occurred_end"].isoformat() if row["occurred_end"] else None,

hindsight-api-slim/hindsight_api/engine/response_models.py

@@ -181,6 +181,9 @@ def parse_metadata(cls, v: Any) -> dict[str, str] | None:
         None,
         description="IDs of source facts this observation was derived from (observation type only, when source_facts is enabled)",
     )
+    status: str | None = Field(
+        None, description="Memory Defense: lifecycle status (active, quarantined, pending_review)"
+    )
 
 
 class ChunkInfo(BaseModel):