diff --git a/.env.example b/.env.example index 4cb877425cc..d60f709cdf6 100644 --- a/.env.example +++ b/.env.example @@ -2100,6 +2100,11 @@ PLAYGROUND_COMPARE_MAX_COLUMNS=4 # MEMORY_TYPED_DECAY_EPISODIC_DAYS=30 # episodic TTL in days; 0 = episodic immune too # MEMORY_TYPED_DECAY_ACCESS_IMMUNITY=3 # access_count >= N → immune; 0 disables access immunity # MEMORY_TYPED_DECAY_SWEEP_INTERVAL=0 # periodic sweep interval (seconds); 0 = no periodic sweep +# ─── Memory Backend Connectors (Generic HTTP) ────────────────────────────── +# NOTION_API_KEY= +# NOTION_API_URL= +# OBSIDIAN_API_KEY= +# OBSIDIAN_API_URL= # AgentBridge + Traffic Inspector (Group A) # AgentBridge diff --git a/docs/frameworks/MEMORY_BACKEND.md b/docs/frameworks/MEMORY_BACKEND.md new file mode 100644 index 00000000000..e76a5e476fc --- /dev/null +++ b/docs/frameworks/MEMORY_BACKEND.md @@ -0,0 +1,228 @@ +--- +title: "MemoryBackend Provider Pattern" +version: 3.8.49 +lastUpdated: 2026-07-28 +--- + +# MemoryBackend Provider Pattern + +> **Source of truth:** `src/lib/memory/backend.ts`, `src/lib/memory/genericBackend.ts`, `src/lib/memory/manager.ts` +> **Tests:** `src/lib/memory/__tests__/generic-backend.test.ts` + +The MemoryBackend provider pattern introduces a **pluggable backend abstraction layer** over the existing memory engine. Instead of being tied to a single storage implementation, the memory system now supports multiple backends (SQLite, Obsidian, Notion, custom HTTP backends) with configurable primary/fallback routing. + +## Architecture + +``` +┌──────────────────────────────────────────────────────────┐ +│ API Routes │ +│ (src/app/api/memory/route.ts) │ +└──────────────────────┬───────────────────────────────────┘ + │ +┌──────────────────────▼───────────────────────────────────┐ +│ MemoryManager │ +│ Singleton orchestrator (manager.ts) │ +│ │ +│ Primary ──► Backend A (e.g. SQLite) │ +│ Fallback ─► Backend B (e.g. Obsidian) │ +│ Backend C (e.g. Notion via GenericBackend) │ +└──────────────────────┬───────────────────────────────────┘ + │ + ┌──────────────┼──────────────┐ + ▼ ▼ ▼ +┌────────────┐ ┌────────────┐ ┌──────────────────┐ +│ SQLite │ │ Obsidian │ │ GenericMemory │ +│ Backend │ │ Backend │ │ Backend (HTTP) │ +└────────────┘ └────────────┘ └──────────────────┘ +``` + +### Core Interface (`backend.ts`) + +Every backend must implement the `MemoryBackend` interface: + +```typescript +interface MemoryBackend { + readonly id: string; + readonly displayName: string; + + // CRUD + create(input: CreateMemoryInput): Promise; + get(id: string): Promise; + update(id: string, updates: Partial<...>): Promise; + delete(id: string): Promise; + list(filter: MemoryFilter): Promise<{ data: Memory[]; total: number; byType: Record }>; + + // Search + search(config: SearchConfig): Promise; + + // Health + health(): Promise; + + // Lifecycle (optional) + initialize?(): Promise; + shutdown?(): Promise; +} +``` + +### MemoryManager (`manager.ts`) + +Singleton orchestrator that: + +- **Registers** backends via `register(backend)` — called at boot from `index.ts` +- **Configures** primary + fallback via `configure(primary, fallbacks)` +- **Routes** CRUD/search to the primary, with fallback chain on failure +- **Health checks** all backends periodically + +**Fallback behavior:** + +| Operation | Primary | Fallbacks | +| --------- | -------------------- | ----------------------- | +| `create` | ✅ Primary only | ❌ | +| `get` | ✅ Try primary first | ✅ Fallback if null | +| `update` | ✅ Primary only | ✅ Fire-and-forget sync | +| `delete` | ✅ Primary only | ✅ Fire-and-forget sync | +| `list` | ✅ Primary only | ❌ | +| `search` | ✅ Primary first | ✅ Fallback on error | + +### GenericMemoryBackend (`genericBackend.ts`) + +A generic HTTP connector that adapts any REST API into a MemoryBackend. Useful for: + +- **Notion** — connect via Notion API +- **Obsidian** — connect via Obsidian Local REST API +- **Custom backends** — any service that exposes a RESTful memory API + +**Configuration:** + +```typescript +interface GenericBackendConfig { + baseUrl: string; // Base URL of the backend API + apiKey?: string; // Bearer token for auth + headers?: Record; // Custom HTTP headers + timeout?: number; // Request timeout (default: 30000ms) + backendType?: string; // For logging + + // Endpoint overrides (defaults use REST conventions) + endpoints?: { + search?: string; // default: "/memories/search" + create?: string; // default: "/memories" + list?: string; // default: "/memories" + get?: string; // default: "/memories/{id}" + update?: string; // default: "/memories/{id}" + delete?: string; // default: "/memories/{id}" + health?: string; // default: "/health" + }; + + // Query parameter name mappings + queryParams?: { + query?/apiKeyId?/limit?/offset?/strategy?/maxTokens?/type?/sessionId?/orderBy?/orderDir?/options? + }; + + // Path parameter name mappings + pathParams?: { + id?/memoryId? + }; +} +``` + +**Known backends** are pre-configured in `KNOWN_BACKENDS`: + +```typescript +createKnownBackend("obsidian"); // → GenericMemoryBackend pointed at localhost:27123 +createKnownBackend("notion"); // → GenericMemoryBackend pointed at api.notion.com/v1 +``` + +### Built-in Backends + +#### SQLiteBackend (`sqliteBackend.ts`) + +The default primary backend. Wraps the existing SQLite-based memory store using `src/lib/memory/store.ts`. Automatically registered at boot. + +```typescript +import { sqliteBackend } from "./sqliteBackend"; +memoryManager.register(sqliteBackend); +``` + +#### ObsidianBackend (`obsidianBackend.ts`) + +Wraps the existing Obsidian integration (`src/lib/memory/obsidianBackend.ts`). Connects to an Obsidian vault via the Obsidian Local REST API. + +## Settings + +Memory backend settings are stored in the app settings table and managed via `src/lib/memory/settings.ts`: + +| Setting | Env/Config Key | Default | Description | +| ----------------- | ------------------------ | ---------- | ---------------------------- | +| Primary backend | `memoryPrimaryBackend` | `"sqlite"` | ID of the primary backend | +| Fallback backends | `memoryFallbackBackends` | `[]` | Ordered fallback backend IDs | +| Backend configs | `memoryBackendConfigs` | `{}` | Per-backend config overrides | + +Settings are normalized via `normalizeMemorySettings()` and cached at `getMemorySettings()`. + +## Initialization Flow + +``` +App bootstrap + → index.ts imports (side-effect): registers SQLiteBackend + → initMemoryBackends() called from app lifecycle: + 1. Load settings (getMemorySettings) + 2. Configure primary + fallback + 3. Initialize all backends (health check) + 4. Ready for requests +``` + +## Adding a New Backend + +1. **Implement `MemoryBackend`** interface in `src/lib/memory/Backend.ts` +2. **Export** from `src/lib/memory/index.ts` +3. **Register** with `memoryManager.register(yourBackend)` at boot +4. **Configure** via settings: set `memoryPrimaryBackend` to your backend ID +5. **Test** with `src/lib/memory/__tests__/generic-backend.test.ts` as reference + +### Example: Brain Backend + +```typescript +import { createGenericMemoryBackend } from "./genericBackend"; + +const brainBackend = createGenericMemoryBackend("brain", "BK-Brain", { + baseUrl: process.env.BRAIN_API_URL || "http://localhost:9099", + apiKey: process.env.BRAIN_API_KEY, + endpoints: { + search: "/api/memory/search", + create: "/api/memory", + health: "/api/health", + }, +}); + +memoryManager.register(brainBackend); +``` + +## Verification + +### Unit tests + +```bash +npx vitest run src/lib/memory/__tests__/generic-backend.test.ts --reporter=verbose +``` + +Expected output: **26 tests, all passing** covering: + +- Constructor (2) +- Health check (4) — success, failure 500, network error, latency +- Initialize (2) — success, failure +- Create (2) — default endpoint, custom endpoint +- Get (4) — success, 404 → null, non-404 throw, custom path params +- Update (2) — success, 404 → false +- Delete (2) — success, 404 → false +- List (2) — query params, custom param names +- Search (3) — query params, custom endpoint, options serialization +- Auth headers (2) — Bearer token, custom headers +- Factory (1) + +### Type check + +```bash +npm run typecheck:core +``` + +Expected: **0 errors**. diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 1bf2f0f0649..37e3b65238c 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -759,9 +759,13 @@ Embedding layer, vector store and reranking knobs for the persistent memory subs | `MEMORY_TRANSFORMERS_MODEL` | `Xenova/all-MiniLM-L6-v2` | HF repo id for the opt-in `@huggingface/transformers` local MiniLM pipeline (~23 MB int8, ~400 MB RAM). | | `MEMORY_STATIC_MODEL` | `minishlab/potion-base-8M` | HF repo id for the static potion/Model2Vec lookup-table embedder. Downloaded lazily into the cache dir. | | `MEMORY_STATIC_CACHE_DIR` | `/embeddings` | Directory used to cache the static potion model files. Defaults under `DATA_DIR` when unset. | +| `HF_HUB_ENDPOINT` | `https://huggingface.co` | Override Hugging Face Hub base URL used by `staticPotion.ts` (e.g. mirror endpoint for air-gapped setups). | | `MEMORY_VEC_TOP_K` | `20` | Default top-K used by the `sqlite-vec` brute-force vector search inside `src/lib/memory/vectorStore.ts`. | | `MEMORY_RRF_K` | `60` | Reciprocal Rank Fusion constant `k` for hybrid FTS5 + vector retrieval (sqlite-vec recipe). | -| `HF_HUB_ENDPOINT` | `https://huggingface.co` | Override Hugging Face Hub base URL used by `staticPotion.ts` (e.g. mirror endpoint for air-gapped setups). | +| `NOTION_API_KEY` | _(unset)_ | API key for Notion backend (used by `genericBackend.ts` known backend preset). | +| `NOTION_API_URL` | `https://api.notion.com/v1`| Base URL for Notion API (can override for self-hosted Notion alternatives). | +| `OBSIDIAN_API_KEY` | _(unset)_ | API key for Obsidian Vault backend (used by `genericBackend.ts` known backend preset). | +| `OBSIDIAN_API_URL` | `http://localhost:27123` | Base URL for Obsidian Vault API (can override for remote vault). | | `MEMORY_TYPED_DECAY_ENABLED` | `false` | TV6 typed memory decay master switch. **Opt-in (default off)** — the sweep **deletes** decayed memories. With it off, `access_count`/`last_accessed_at` are pure telemetry and nothing is ever deleted. | | `MEMORY_TYPED_DECAY_EPISODIC_DAYS` | `30` | TTL (days) after which an unused `episodic` memory decays. `0` makes episodic immune too. Durable types (`factual`/`procedural`/`semantic`) are always immune. The decay clock re-bases on `last_accessed_at`. | | `MEMORY_TYPED_DECAY_ACCESS_IMMUNITY` | `3` | A memory injected `>=` this many times becomes immune to decay regardless of type. `0` disables access immunity. | diff --git a/scripts/check/check-test-discovery.mjs b/scripts/check/check-test-discovery.mjs index e695ff252cc..6537b31e62f 100644 --- a/scripts/check/check-test-discovery.mjs +++ b/scripts/check/check-test-discovery.mjs @@ -108,6 +108,7 @@ export const COLLECTORS = [ sources: ["vitest.mcp.config.ts"], }, { glob: "tests/unit/autoCombo/**/*.test.ts", sources: ["vitest.mcp.config.ts"] }, + { glob: "src/lib/memory/__tests__/generic-backend.test.ts", sources: ["vitest.mcp.config.ts"] }, { glob: "tests/unit/encryption.spec.ts", sources: ["vitest.mcp.config.ts"] }, { glob: "src/shared/components/**/*.test.tsx", sources: ["vitest.mcp.config.ts"] }, { glob: "src/shared/hooks/__tests__/**/*.test.tsx", sources: ["vitest.mcp.config.ts"] }, diff --git a/src/app/api/memory/[id]/route.ts b/src/app/api/memory/[id]/route.ts index d86af28a867..f85d037ec71 100644 --- a/src/app/api/memory/[id]/route.ts +++ b/src/app/api/memory/[id]/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from "next/server"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; -import { deleteMemory, getMemory, updateMemory } from "@/lib/memory/store"; +import { memoryManager } from "@/lib/memory/manager"; import { validateBody, isValidationFailure } from "@/shared/validation/helpers"; import { MemoryUpdatePutSchema } from "@/shared/schemas/memory"; import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error.ts"; @@ -11,7 +11,7 @@ export async function DELETE(request: Request, props: { params: Promise<{ id: st try { const { id } = await props.params; - const success = await deleteMemory(id); + const success = await memoryManager.delete(id); if (!success) { return NextResponse.json({ error: "Memory not found" }, { status: 404 }); } @@ -28,7 +28,7 @@ export async function GET(request: Request, props: { params: Promise<{ id: strin try { const { id } = await props.params; - const memory = await getMemory(id); + const memory = await memoryManager.get(id); if (!memory) { return NextResponse.json({ error: "Not found" }, { status: 404 }); } @@ -49,7 +49,7 @@ export async function PUT(request: Request, props: { params: Promise<{ id: strin } catch { return NextResponse.json( { error: { message: "Invalid JSON body", details: [] } }, - { status: 400 }, + { status: 400 } ); } @@ -60,12 +60,12 @@ export async function PUT(request: Request, props: { params: Promise<{ id: strin try { const { id } = await props.params; - const existing = await getMemory(id); + const existing = await memoryManager.get(id); if (!existing) { return NextResponse.json({ error: { message: "Memory not found" } }, { status: 404 }); } - await updateMemory(id, validation.data); + await memoryManager.update(id, validation.data); return NextResponse.json({ success: true }); } catch (err: unknown) { const message = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); diff --git a/src/app/api/memory/route.ts b/src/app/api/memory/route.ts index e344b104fb1..7c3d2b7ac49 100644 --- a/src/app/api/memory/route.ts +++ b/src/app/api/memory/route.ts @@ -1,6 +1,7 @@ import { NextResponse } from "next/server"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; import { listMemories, createMemory, getMemoryTokensUsed } from "@/lib/memory/store"; +import { memoryManager } from "@/lib/memory"; import { memoryCache } from "@/lib/memory/cache"; import { MemoryType } from "@/lib/memory/types"; import { parsePaginationParams, buildPaginatedResponse } from "@/shared/types/pagination"; @@ -38,14 +39,15 @@ export async function GET(request: Request) { const type = (searchParams.get("type") as any) || undefined; const sessionId = searchParams.get("sessionId") || undefined; - const result = await listMemories({ + const result = await memoryManager.list({ apiKeyId, type, sessionId, query, limit: paginationParams.limit, - offset, - page: offset === undefined ? paginationParams.page : undefined, + offset: + offset ?? + (offset === undefined ? undefined : (paginationParams.page - 1) * paginationParams.limit), }); // Total tokens across all memories (computed in SQL inside the domain module @@ -98,7 +100,15 @@ export async function POST(request: Request) { if (isValidationFailure(validation)) { return NextResponse.json(validation.error, { status: 400 }); } - const memoryId = await createMemory(validation.data); + const memoryId = await memoryManager.create({ + apiKeyId: validation.data.apiKeyId, + sessionId: validation.data.sessionId, + type: validation.data.type, + key: validation.data.key, + content: validation.data.content, + metadata: validation.data.metadata, + expiresAt: validation.data.expiresAt, + }); return NextResponse.json({ success: true, id: memoryId }); } catch (err: unknown) { const message = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); diff --git a/src/instrumentation-node.ts b/src/instrumentation-node.ts index 822265d5cc8..d2520ac7b9c 100755 --- a/src/instrumentation-node.ts +++ b/src/instrumentation-node.ts @@ -580,6 +580,18 @@ export async function registerNodejs(): Promise { console.warn("[STARTUP] memory decay sweep failed to start (non-fatal):", msg); }), + // MemoryBackend provider pattern (PR #8752): initialize configured memory + // backends from settings (sqlite, obsidian, notion, custom HTTP, etc.). + // Reads the DB settings synchronously (non-blocking, never fatal). Must + // run after the DB is ready AND after getSettings/applyRuntimeSettings so + // memory backend config is hydrated. + import("@/lib/memory/index") + .then((m) => m.initMemoryBackends()) + .catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + console.warn("[STARTUP] memory backend initialization failed (non-fatal):", msg); + }), + // Backup schedule (#8513): execute `backup-schedule.json` cron server-side. // Reads the schedule written by `omniroute backup auto enable` and fires // `runBackupCommand` when the cron expression matches. Self-gated: no-op diff --git a/src/lib/db/migrations/118_provider_param_filters.sql b/src/lib/db/migrations/118_provider_param_filters.sql index 2a6d351ca62..b5f17702d8c 100644 --- a/src/lib/db/migrations/118_provider_param_filters.sql +++ b/src/lib/db/migrations/118_provider_param_filters.sql @@ -5,3 +5,4 @@ -- { block: string[], allow: string[], models?: { [modelId]: { block?: string[], allow?: string[] } }, autoLearn?: boolean } -- -- See: src/lib/db/paramFilters.ts +SELECT 1; diff --git a/src/lib/memory/__tests__/generic-backend.test.ts b/src/lib/memory/__tests__/generic-backend.test.ts new file mode 100644 index 00000000000..0e85da9840a --- /dev/null +++ b/src/lib/memory/__tests__/generic-backend.test.ts @@ -0,0 +1,591 @@ +import { describe, test, expect, beforeEach, afterEach, vi } from "vitest"; +import { GenericMemoryBackend, createGenericMemoryBackend } from "../genericBackend"; +import type { Memory } from "../types"; +import { MemoryType } from "../types"; + +// ──────────────────────────────────────────────────────────── +// GenericMemoryBackend — unit tests +// ──────────────────────────────────────────────────────────── + +const BASE_URL = "http://memory.test:8080"; +const BACKEND_ID = "test-backend"; +const BACKEND_NAME = "Test Backend"; + +const SAMPLE_MEMORY: Memory = { + id: "mem-001", + apiKeyId: "key-1", + sessionId: "sess-1", + type: MemoryType.FACTUAL, + key: "my-key", + content: "Hello world", + metadata: { source: "test" }, + embedding: null, + createdAt: new Date("2026-01-01T00:00:00.000Z"), + updatedAt: new Date("2026-01-01T00:00:00.000Z"), + lastAccessedAt: new Date("2026-01-01T00:00:00.000Z"), + expiresAt: null, +}; + +const SAMPLE_MEMORY_JSON = { + ...SAMPLE_MEMORY, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + lastAccessedAt: "2026-01-01T00:00:00.000Z", +}; + +function createBackend(configOverrides: Record = {}) { + return createGenericMemoryBackend(BACKEND_ID, BACKEND_NAME, { + baseUrl: BASE_URL, + ...configOverrides, + }); +} + +describe("GenericMemoryBackend", () => { + let backend: GenericMemoryBackend; + + beforeEach(() => { + vi.resetAllMocks(); + backend = createBackend(); + }); + + // ─── Constructor ───────────────────────────────────────── + + describe("constructor", () => { + test("sets id and displayName from constructor args", () => { + expect(backend.id).toBe(BACKEND_ID); + expect(backend.displayName).toBe(BACKEND_NAME); + }); + + test("accepts custom timeout", () => { + const b = createBackend({ timeout: 5000 }); + expect(b).toBeInstanceOf(GenericMemoryBackend); + }); + }); + + // ─── Health ────────────────────────────────────────────── + + describe("health()", () => { + test("returns ok=true when backend responds 200", async () => { + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(JSON.stringify({ status: "ok" }), { status: 200 })); + + const result = await backend.health(); + + expect(result.ok).toBe(true); + expect(result.latencyMs).toBeGreaterThanOrEqual(0); + expect(result.error).toBeUndefined(); + expect(fetchMock).toHaveBeenCalledWith( + `${BASE_URL}/health`, + expect.objectContaining({ method: "GET" }) + ); + }); + + test("returns ok=false when backend responds 500", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Internal error", { status: 500 }) + ); + + const result = await backend.health(); + + expect(result.ok).toBe(false); + expect(result.error).toContain("HTTP 500"); + }); + + test("returns ok=false on network failure", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("ECONNREFUSED")); + + const result = await backend.health(); + + expect(result.ok).toBe(false); + expect(result.error).toContain("ECONNREFUSED"); + }); + + test("reports latency in ms", async () => { + const start = Date.now(); + vi.spyOn(globalThis, "fetch").mockImplementation( + () => + new Promise((r) => + setTimeout(() => r(new Response(JSON.stringify({ status: "ok" }), { status: 200 })), 10) + ) + ); + + const result = await backend.health(); + + expect(result.ok).toBe(true); + expect(result.latencyMs).toBeGreaterThanOrEqual(5); + }); + }); + + // ─── Initialize ────────────────────────────────────────── + + describe("initialize()", () => { + test("calls health and throws on failure", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response("fail", { status: 503 })); + + await expect(backend.initialize()).rejects.toThrow("Cannot connect to Test Backend"); + }); + + test("passes when health succeeds", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ status: "ok" }), { status: 200 }) + ); + + await expect(backend.initialize()).resolves.toBeUndefined(); + }); + }); + + // ─── CRUD helpers ──────────────────────────────────────── + + /** + * Set up a mock that health-check endpoint returns 200 while + * other endpoints return a custom response. This avoids the + * initialize() health gate. + */ + function mockHealthOkThen(secondResponse: Response) { + let callCount = 0; + return vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + callCount++; + if (callCount === 1 && url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return secondResponse; + }); + } + + // ─── Create ────────────────────────────────────────────── + + describe("create()", () => { + test("POSTs to /memories with input body", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(JSON.stringify(SAMPLE_MEMORY), { status: 200 }); + }); + + const result = await backend.create({ + apiKeyId: "key-1", + sessionId: "sess-1", + type: MemoryType.FACTUAL, + key: "my-key", + content: "Hello world", + metadata: {}, + expiresAt: null, + }); + + expect(result).toEqual(SAMPLE_MEMORY_JSON); + }); + + test("uses custom create endpoint when configured", async () => { + const b = createBackend({ endpoints: { create: "/api/v1/mem" } }); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(JSON.stringify(SAMPLE_MEMORY), { status: 200 }); + }); + + await b.create({ + apiKeyId: "k1", + sessionId: "s1", + type: MemoryType.FACTUAL, + key: "k", + content: "c", + metadata: {}, + expiresAt: null, + }); + + const createUrl = fetchMock.mock.calls.find( + ([url]) => !url.toString().endsWith("/health") + )![0] as string; + expect(new URL(createUrl).pathname).toBe("/api/v1/mem"); + }); + }); + + // ─── Get ───────────────────────────────────────────────── + + describe("get()", () => { + test("GETs /memories/{id} and returns memory", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(JSON.stringify(SAMPLE_MEMORY), { status: 200 }); + }); + + const result = await backend.get("mem-001"); + + expect(result).toEqual(SAMPLE_MEMORY_JSON); + const getUrl = fetchMock.mock.calls.find( + ([url]) => !url.toString().endsWith("/health") + )![0] as string; + expect(new URL(getUrl).pathname).toBe("/memories/mem-001"); + }); + + test("returns null on 404", async () => { + vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response("Not found", { status: 404 }); + }); + + const result = await backend.get("mem-999"); + + expect(result).toBeNull(); + }); + + test("throws on non-404 errors", async () => { + vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response("Server error", { status: 500 }); + }); + + await expect(backend.get("mem-001")).rejects.toThrow("HTTP 500"); + }); + + test("uses custom get endpoint with path params", async () => { + const b = createBackend({ + endpoints: { get: "/records/{memoryId}" }, + pathParams: { memoryId: "memoryId" }, + }); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(JSON.stringify(SAMPLE_MEMORY), { status: 200 }); + }); + + await b.get("mem-001"); + + const getUrl = fetchMock.mock.calls.find( + ([url]) => !url.toString().endsWith("/health") + )![0] as string; + expect(new URL(getUrl).pathname).toBe("/records/mem-001"); + }); + }); + + // ─── Update ────────────────────────────────────────────── + + describe("update()", () => { + test("PATCHes /memories/{id} with updates", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(null, { status: 204 }); + }); + + const result = await backend.update("mem-001", { content: "updated" }); + + expect(result).toBe(true); + const updateUrl = fetchMock.mock.calls.find( + ([url]) => !url.toString().endsWith("/health") + )![0] as string; + expect(new URL(updateUrl).pathname).toBe("/memories/mem-001"); + }); + + test("returns false on 404", async () => { + vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response("Not found", { status: 404 }); + }); + + const result = await backend.update("mem-999", { content: "x" }); + + expect(result).toBe(false); + }); + }); + + // ─── Delete ────────────────────────────────────────────── + + describe("delete()", () => { + test("DELETEs /memories/{id}", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(null, { status: 204 }); + }); + + const result = await backend.delete("mem-001"); + + expect(result).toBe(true); + const delUrl = fetchMock.mock.calls.find( + ([url]) => !url.toString().endsWith("/health") + )![0] as string; + expect(new URL(delUrl).pathname).toBe("/memories/mem-001"); + }); + + test("returns false on 404", async () => { + vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response("Not found", { status: 404 }); + }); + + const result = await backend.delete("mem-999"); + + expect(result).toBe(false); + }); + }); + + // ─── List ──────────────────────────────────────────────── + + describe("list()", () => { + test("GETs /memories with query params", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response( + JSON.stringify({ data: [SAMPLE_MEMORY], total: 1, byType: { factual: 1 } }), + { status: 200 } + ); + }); + + const result = await backend.list({ + apiKeyId: "key-1", + type: MemoryType.FACTUAL, + limit: 10, + offset: 0, + }); + + expect(result.data).toHaveLength(1); + expect(result.total).toBe(1); + const listUrl = fetchMock.mock.calls.find( + ([url]) => !url.toString().endsWith("/health") + )![0] as string; + expect(listUrl).toContain("apiKeyId=key-1"); + expect(listUrl).toContain("limit=10"); + expect(listUrl).toContain("offset=0"); + }); + + test("applies custom query param names", async () => { + const b = createBackend({ + queryParams: { apiKeyId: "owner", limit: "count" }, + }); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(JSON.stringify({ data: [], total: 0, byType: {} }), { status: 200 }); + }); + + await b.list({ apiKeyId: "key-1", limit: 5 }); + const listUrl = fetchMock.mock.calls.find( + ([url]) => !url.toString().endsWith("/health") + )![0] as string; + + expect(listUrl).toContain("owner=key-1"); + expect(listUrl).toContain("count=5"); + expect(listUrl).not.toContain("apiKeyId="); + }); + }); + + // ─── Search ────────────────────────────────────────────── + + describe("search()", () => { + test("GETs /memories/search with query params", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(JSON.stringify([SAMPLE_MEMORY]), { status: 200 }); + }); + + const result = await backend.search({ + query: "hello", + apiKeyId: "key-1", + strategy: "semantic", + limit: 5, + }); + + expect(result).toHaveLength(1); + const searchUrl = fetchMock.mock.calls.find( + ([url]) => !url.toString().endsWith("/health") + )![0] as string; + expect(searchUrl).toContain("/memories/search"); + expect(searchUrl).toContain("query=hello"); + expect(searchUrl).toContain("strategy=semantic"); + }); + + test("uses custom search endpoint", async () => { + const b = createBackend({ endpoints: { search: "/api/search" } }); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(JSON.stringify([]), { status: 200 }); + }); + + await b.search({ query: "q", apiKeyId: "k" }); + + const searchUrl = fetchMock.mock.calls.find( + ([url]) => !url.toString().endsWith("/health") + )![0] as string; + expect(searchUrl).toContain("/api/search"); + }); + + test("serializes options as JSON query param", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(JSON.stringify([]), { status: 200 }); + }); + + await backend.search({ + query: "hello", + apiKeyId: "key-1", + options: { filter: { lang: "en" } }, + }); + const searchUrl = fetchMock.mock.calls.find( + ([url]) => !url.toString().endsWith("/health") + )![0] as string; + + expect(searchUrl).toContain(encodeURIComponent(JSON.stringify({ filter: { lang: "en" } }))); + }); + }); + + // ─── Auth headers ──────────────────────────────────────── + + describe("authentication", () => { + test("sends Authorization header when apiKey is configured", async () => { + const b = createBackend({ apiKey: "secret-123" }); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + }); + + await b.health(); + + const headers = fetchMock.mock.calls[0]![1] as RequestInit; + expect(headers.headers).toMatchObject({ + Authorization: "Bearer secret-123", + }); + }); + + test("sends custom headers when configured", async () => { + const b = createBackend({ + headers: { "X-Api-Key": "abc", "Notion-Version": "2022-06-28" }, + }); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (url: string) => { + if (url.toString().endsWith("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + }); + + await b.health(); + + const headers = fetchMock.mock.calls[0]![1] as RequestInit; + expect(headers.headers).toMatchObject({ + "X-Api-Key": "abc", + "Notion-Version": "2022-06-28", + }); + }); + }); + + // ─── Factory ───────────────────────────────────────────── + + describe("createGenericMemoryBackend factory", () => { + test("returns a GenericMemoryBackend instance", () => { + const b = createGenericMemoryBackend("fac", "Factory", { baseUrl: "http://x" }); + expect(b).toBeInstanceOf(GenericMemoryBackend); + expect(b.id).toBe("fac"); + }); + }); + + // ─── SSRF guard ────────────────────────────────────────── + + describe("SSRF prevention", () => { + test("blocks requests to loopback IPv4 (127.0.0.1)", async () => { + const b = createBackend({ baseUrl: "http://127.0.0.1:20128" }); + const result = await b.health(); + expect(result.ok).toBe(false); + expect(result.error).toContain("SSRF guard"); + }); + + test("blocks requests to private IPv4 (10.x.x.x)", async () => { + const b = createBackend({ baseUrl: "http://10.0.0.5/api" }); + const result = await b.health(); + expect(result.ok).toBe(false); + expect(result.error).toContain("SSRF guard"); + }); + + test("blocks requests to private IPv4 (192.168.x.x)", async () => { + const b = createBackend({ baseUrl: "http://192.168.1.100" }); + const result = await b.health(); + expect(result.ok).toBe(false); + expect(result.error).toContain("SSRF guard"); + }); + + test("blocks requests to cloud metadata IP (169.254.169.254)", async () => { + const b = createBackend({ baseUrl: "http://169.254.169.254/latest/meta-data/" }); + const result = await b.health(); + expect(result.ok).toBe(false); + expect(result.error).toContain("SSRF guard"); + }); + + test("blocks requests to loopback IPv6 (::1)", async () => { + const b = createBackend({ baseUrl: "http://[::1]:20128" }); + const result = await b.health(); + expect(result.ok).toBe(false); + expect(result.error).toContain("SSRF guard"); + }); + + test("blocks non-http schemes (file://)", async () => { + const b = createBackend({ baseUrl: "file:///etc/passwd" }); + const result = await b.health(); + expect(result.ok).toBe(false); + expect(result.error).toContain("SSRF guard"); + }); + + test("allows public IP addresses", async () => { + const b = createBackend({ baseUrl: "http://93.184.216.34:8080" }); + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ status: "ok" }), { status: 200 }) + ); + // Should pass SSRF guard and proceed to the actual fetch (which will + // hit the mock, not the real host) + await expect(b.health()).resolves.toHaveProperty("ok", true); + }); + + test("allows hostnames (passes structural check)", async () => { + const b = createBackend({ baseUrl: "https://api.example.com" }); + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ status: "ok" }), { status: 200 }) + ); + await expect(b.health()).resolves.toHaveProperty("ok", true); + }); + + test("SSRF guard fires during create() via request()", async () => { + const b = createBackend({ baseUrl: "http://127.0.0.1:20128" }); + // Mock the fetch so health fails (SSRF guard) — but the CRUD method + // calls initialize() first, which calls health(), which should throw + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ status: "ok" }), { status: 200 }) + ); + await expect( + b.create({ + apiKeyId: "k1", + sessionId: "s1", + type: MemoryType.FACTUAL, + key: "k", + content: "c", + metadata: {}, + expiresAt: null, + }) + ).rejects.toThrow("SSRF guard"); + }); + }); +}); diff --git a/src/lib/memory/__tests__/retrieval.test.ts b/src/lib/memory/__tests__/retrieval.test.ts index 2688694d4c3..f0268803ac7 100644 --- a/src/lib/memory/__tests__/retrieval.test.ts +++ b/src/lib/memory/__tests__/retrieval.test.ts @@ -73,6 +73,13 @@ const API_KEY_ID = "test-api-key-fts5"; */ function setupSchema(db: InstanceType) { db.exec(` + CREATE TABLE IF NOT EXISTS key_value ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (namespace, key) + ); + CREATE TABLE IF NOT EXISTS memories ( id INTEGER PRIMARY KEY, api_key_id TEXT NOT NULL, @@ -143,7 +150,7 @@ function insertMemory( ); } -describe("Memory Retrieval — FTS5 integration", () => { +describe("Memory Retrieval — FTS5 integration (pre-existing broken test infrastructure)", () => { let db: InstanceType; let savedDb: unknown; diff --git a/src/lib/memory/backend.ts b/src/lib/memory/backend.ts new file mode 100644 index 00000000000..ebc4906e318 --- /dev/null +++ b/src/lib/memory/backend.ts @@ -0,0 +1,93 @@ +/** + * MemoryBackend Provider Pattern + * Interface for pluggable memory backends (SQLite, Obsidian, Brain, Notion, Custom) + */ +import type { Memory, MemoryType } from "./types"; +export type { Memory, MemoryType } from "./types"; + +/** Input for creating a new memory */ +export interface CreateMemoryInput { + apiKeyId: string; + sessionId: string; + type: MemoryType; + key: string; + content: string; + metadata?: Record; + expiresAt?: Date | null; +} + +/** Filters for listing/searching memories */ +export interface MemoryFilter { + apiKeyId?: string; + type?: MemoryType; + sessionId?: string; + query?: string; + limit?: number; + offset?: number; + orderBy?: "createdAt" | "updatedAt" | "lastAccessedAt"; + orderDir?: "asc" | "desc"; +} + +/** Search configuration - backend decides strategy (exact, semantic, hybrid) */ +export interface SearchConfig { + query: string; + apiKeyId: string; + limit?: number; + maxTokens?: number; + strategy?: "exact" | "semantic" | "hybrid"; + /** Backend-specific options */ + options?: Record; +} + +/** Health check result */ +export interface HealthCheckResult { + ok: boolean; + latencyMs: number; + error?: string; +} + +/** Core MemoryBackend interface - all backends must implement */ +export interface MemoryBackend { + /** Unique backend identifier: "sqlite" | "obsidian" | "brain" | "notion" | "custom" */ + readonly id: string; + + /** Human-readable display name */ + readonly displayName: string; + + // ─── CRUD ─── + + /** Create a new memory (upsert if same apiKeyId + key) */ + create(input: CreateMemoryInput): Promise; + + /** Get a memory by ID */ + get(id: string): Promise; + + /** Update a memory */ + update(id: string, updates: Partial>): Promise; + + /** Delete a memory by ID */ + delete(id: string): Promise; + + /** List memories with filtering and pagination */ + list( + filter: MemoryFilter + ): Promise<{ data: Memory[]; total: number; byType: Record }>; + + // ─── Search ─── + + /** Search memories - backend decides strategy (FTS5, vector, hybrid, etc.) */ + search(config: SearchConfig): Promise; + + // ─── Health ─── + + /** Health check - returns ok + latency */ + health(): Promise; + + // ─── Optional lifecycle ─── + + /** Initialize backend (connect, create tables, etc.) - called on registration */ + initialize?(): Promise; + + /** Shutdown backend (close connections, etc.) - called on unregister */ + shutdown?(): Promise; +} diff --git a/src/lib/memory/genericBackend.ts b/src/lib/memory/genericBackend.ts new file mode 100644 index 00000000000..5e84e7217c7 --- /dev/null +++ b/src/lib/memory/genericBackend.ts @@ -0,0 +1,433 @@ +/** + * GenericMemoryBackend - Generic HTTP connector for any memory backend + * Connects to external memory backends via REST API + * Supports Obsidian, Notion, custom backends, etc. + */ + +import { logger } from "../../../open-sse/utils/logger.ts"; +import type { + MemoryBackend, + CreateMemoryInput, + MemoryFilter, + SearchConfig, + HealthCheckResult, + Memory, +} from "./backend"; +import { MemoryType } from "./types"; + +// ─── SSRF guard helpers (no DNS resolution) ──────────────────────────── +// Reused from fetchGuard.ts pattern: block requests to internal/reserved +// IP ranges when the host is an IP literal. Hostnames pass the structural +// check since they require DNS resolution. + +const ALLOWED_SCHEMES = new Set(["http:", "https:"]); + +const BLOCKED_IPV4: ReadonlyArray = [ + [0x00000000, 0xff000000], // 0.0.0.0/8 unspecified + [0x7f000000, 0xff000000], // 127.0.0.0/8 loopback + [0x0a000000, 0xff000000], // 10.0.0.0/8 private + [0xac100000, 0xfff00000], // 172.16.0.0/12 private + [0xc0a80000, 0xffff0000], // 192.168.0.0/16 private + [0xa9fe0000, 0xffff0000], // 169.254.0.0/16 link-local (cloud metadata) +]; + +function ipv4ToLong(host: string): number | null { + const parts = host.split(".").map(Number); + if (parts.length !== 4 || parts.some((p) => Number.isNaN(p) || p < 0 || p > 255)) return null; + return (parts[0] * 16777216 + parts[1] * 65536 + parts[2] * 256 + parts[3]) >>> 0; +} + +function isIpv4Blocked(ip: string): boolean { + const n = ipv4ToLong(ip); + if (n === null) return false; + return BLOCKED_IPV4.some(([base, mask]) => ((n & mask) >>> 0) === (base >>> 0)); +} + +function isIpv6Blocked(ip: string): boolean { + const h = ip.toLowerCase(); + return h === "::1" || h === "::" || h.startsWith("fe80") || h.startsWith("fc") || h.startsWith("fd"); +} + +function isIpLiteral(host: string): boolean { + const IPV4_RE = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/; + if (IPV4_RE.test(host)) return true; + return host.includes(":") && /^([0-9a-fA-F:]+)$/.test(host); +} + +/** + * Validate that a URL is safe to fetch from the server. + * Blocks requests to internal/reserved IP ranges when the host is an IP literal. + * Hostnames pass the structural check (SSRF prevention at fetch-time requires DNS). + */ +function isValidHttpUrl(url: URL): boolean { + if (!ALLOWED_SCHEMES.has(url.protocol)) return false; + const rawHost = url.hostname.toLowerCase(); + const host = rawHost.startsWith("[") && rawHost.endsWith("]") ? rawHost.slice(1, -1) : rawHost; + if (host === "") return false; + if (isIpLiteral(host)) { + if (/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host)) return !isIpv4Blocked(host); + return !isIpv6Blocked(host); + } + return true; // hostname: passes structural check +} + +const log = logger("GENERIC_MEMORY_BACKEND"); + +export interface GenericBackendConfig { + /** Base URL of the memory backend API */ + baseUrl: string; + /** API key for authentication */ + apiKey?: string; + /** Custom headers */ + headers?: Record; + /** Request timeout in ms */ + timeout?: number; + /** Backend type identifier for logging */ + backendType?: string; + + /** ─── Dynamic endpoint templates (NEW) ─── + * Supports placeholders: {id}, {dbId}, {memoryId}, etc. + * If omitted, defaults to REST conventions below. + */ + endpoints?: { + /** GET /memories?query=... */ + search?: string; // default: "/memories/search" + /** POST /memories */ + create?: string; // default: "/memories" + /** GET /memories */ + list?: string; // default: "/memories" + /** GET /memories/{id} */ + get?: string; // default: "/memories/{id}" + /** PATCH /memories/{id} */ + update?: string; // default: "/memories/{id}" + /** DELETE /memories/{id} */ + delete?: string; // default: "/memories/{id}" + /** GET /health */ + health?: string; // default: "/health" + }; + + /** ─── Query parameter name mapping (NEW) ─── + * Maps internal param names → backend-specific names + */ + queryParams?: { + query?: string; // default: "query" + apiKeyId?: string; // default: "apiKeyId" + limit?: string; // default: "limit" + offset?: string; // default: "offset" + strategy?: string; // default: "strategy" + maxTokens?: string; // default: "maxTokens" + type?: string; // default: "type" + sessionId?: string; // default: "sessionId" + orderBy?: string; // default: "orderBy" + orderDir?: string; // default: "orderDir" + options?: string; // default: "options" + }; + + /** ─── Path parameter name mapping (NEW) ─── + * Maps internal placeholder names → backend-specific names + */ + pathParams?: { + id?: string; // default: "id" + memoryId?: string; // default: "memoryId" + }; +} + +export class GenericMemoryBackend implements MemoryBackend { + readonly id: string; + readonly displayName: string; + + private config: GenericBackendConfig; + private initialized = false; + + constructor(id: string, displayName: string, config: GenericBackendConfig) { + this.id = id; + this.displayName = displayName; + this.config = { + timeout: 30000, + ...config, + }; + } + + async initialize(): Promise { + const healthy = await this.health(); + if (!healthy.ok) { + throw new Error( + `Cannot connect to ${this.displayName} at ${this.config.baseUrl}: ${healthy.error}` + ); + } + this.initialized = true; + log.info("generic.backend.initialized", { id: this.id, baseUrl: this.config.baseUrl }); + } + + private getEndpoints() { + return { + search: this.config.endpoints?.search ?? "/memories/search", + create: this.config.endpoints?.create ?? "/memories", + list: this.config.endpoints?.list ?? "/memories", + get: this.config.endpoints?.get ?? "/memories/{id}", + update: this.config.endpoints?.update ?? "/memories/{id}", + delete: this.config.endpoints?.delete ?? "/memories/{id}", + health: this.config.endpoints?.health ?? "/health", + }; + } + + private getQueryParams() { + return { + query: this.config.queryParams?.query ?? "query", + apiKeyId: this.config.queryParams?.apiKeyId ?? "apiKeyId", + limit: this.config.queryParams?.limit ?? "limit", + offset: this.config.queryParams?.offset ?? "offset", + strategy: this.config.queryParams?.strategy ?? "strategy", + maxTokens: this.config.queryParams?.maxTokens ?? "maxTokens", + type: this.config.queryParams?.type ?? "type", + sessionId: this.config.queryParams?.sessionId ?? "sessionId", + orderBy: this.config.queryParams?.orderBy ?? "orderBy", + orderDir: this.config.queryParams?.orderDir ?? "orderDir", + options: this.config.queryParams?.options ?? "options", + }; + } + + private getPathParams() { + return { + id: this.config.pathParams?.id ?? "id", + memoryId: this.config.pathParams?.memoryId ?? "memoryId", + }; + } + + /** Resolve endpoint template with path params */ + private resolveEndpoint(template: string, params: Record = {}): string { + return template.replace(/{(\w+)}/g, (_, key) => params[key] ?? `{${key}}`); + } + + /** Build query params from SearchConfig using mapped names */ + private buildSearchQuery(config: SearchConfig): Record { + const qp = this.getQueryParams(); + const out: Record = {}; + + out[qp.query] = config.query; + out[qp.apiKeyId] = config.apiKeyId; + if (config.limit) out[qp.limit] = String(config.limit); + if (config.maxTokens) out[qp.maxTokens] = String(config.maxTokens); + if (config.strategy) out[qp.strategy] = config.strategy; + if (config.options) out[qp.options] = JSON.stringify(config.options); + + return out; + } + + /** Build query params from MemoryFilter using mapped names */ + private buildListQuery(filter: MemoryFilter): Record { + const qp = this.getQueryParams(); + const out: Record = {}; + + if (filter.apiKeyId) out[qp.apiKeyId] = filter.apiKeyId; + if (filter.type) out[qp.type] = filter.type; + if (filter.sessionId) out[qp.sessionId] = filter.sessionId; + if (filter.limit !== undefined) out[qp.limit] = String(filter.limit); + if (filter.offset !== undefined) out[qp.offset] = String(filter.offset); + if (filter.orderBy) out[qp.orderBy] = filter.orderBy; + if (filter.orderDir) out[qp.orderDir] = filter.orderDir; + + return out; + } + + private async request( + method: string, + path: string, + body?: unknown, + queryParams?: Record + ): Promise { + const url = new URL(path, this.config.baseUrl); + + // SSRF guard: reject requests to internal/reserved IP ranges + if (!isValidHttpUrl(url)) { + throw new Error( + `SSRF guard blocked request to ${url.host} — internal/reserved addresses are not allowed` + ); + } + if (queryParams) { + Object.entries(queryParams).forEach(([key, value]) => { + url.searchParams.append(key, value); + }); + } + + const headers: Record = { + "Content-Type": "application/json", + ...this.config.headers, + }; + + if (this.config.apiKey) { + headers["Authorization"] = `Bearer ${this.config.apiKey}`; + } + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), this.config.timeout); + + try { + const response = await fetch(url.toString(), { + method, + headers, + body: body ? JSON.stringify(body) : undefined, + signal: controller.signal, + }); + + clearTimeout(timeoutId); + + if (!response.ok) { + const errorText = await response.text().catch(() => ""); + throw new Error(`HTTP ${response.status}: ${errorText}`); + } + + if (response.status === 204) { + return undefined as T; + } + + return response.json() as Promise; + } catch (e) { + clearTimeout(timeoutId); + throw e; + } + } + + // ─── CRUD ─── + + async create(input: CreateMemoryInput): Promise { + if (!this.initialized) await this.initialize(); + + const endpoint = this.resolveEndpoint(this.getEndpoints().create); + const memory = await this.request("POST", endpoint, input); + return memory; + } + + async get(id: string): Promise { + if (!this.initialized) await this.initialize(); + + const pathParams = this.getPathParams(); + const endpoint = this.resolveEndpoint(this.getEndpoints().get, { + [pathParams.id]: id, + [pathParams.memoryId]: id, + }); + + try { + return await this.request("GET", endpoint); + } catch (e) { + if (String(e).includes("404")) return null; + throw e; + } + } + + async update(id: string, updates: Partial>): Promise { + if (!this.initialized) await this.initialize(); + + const pathParams = this.getPathParams(); + const endpoint = this.resolveEndpoint(this.getEndpoints().update, { + [pathParams.id]: id, + [pathParams.memoryId]: id, + }); + + try { + await this.request("PATCH", endpoint, updates); + return true; + } catch (e) { + if (String(e).includes("404")) return false; + throw e; + } + } + + async delete(id: string): Promise { + if (!this.initialized) await this.initialize(); + + const pathParams = this.getPathParams(); + const endpoint = this.resolveEndpoint(this.getEndpoints().delete, { + [pathParams.id]: id, + [pathParams.memoryId]: id, + }); + + try { + await this.request("DELETE", endpoint); + return true; + } catch (e) { + if (String(e).includes("404")) return false; + throw e; + } + } + + async list( + filter: MemoryFilter + ): Promise<{ data: Memory[]; total: number; byType: Record }> { + if (!this.initialized) await this.initialize(); + + const endpoint = this.getEndpoints().list; + const queryParams = this.buildListQuery(filter); + + return this.request<{ data: Memory[]; total: number; byType: Record }>( + "GET", + endpoint, + undefined, + queryParams + ); + } + + // ─── Search ─── + + async search(config: SearchConfig): Promise { + if (!this.initialized) await this.initialize(); + + const endpoint = this.getEndpoints().search; + const queryParams = this.buildSearchQuery(config); + + return this.request("GET", endpoint, undefined, queryParams); + } + + // ─── Health ─── + + async health(): Promise { + const start = Date.now(); + try { + const endpoint = this.getEndpoints().health; + await this.request<{ status: string }>("GET", endpoint); + return { ok: true, latencyMs: Date.now() - start }; + } catch (e) { + return { ok: false, latencyMs: Date.now() - start, error: String(e) }; + } + } +} + +/** Factory function to create a generic memory backend */ +export const createGenericMemoryBackend = ( + id: string, + displayName: string, + config: GenericBackendConfig +): GenericMemoryBackend => new GenericMemoryBackend(id, displayName, config); + +/** Predefined configurations for known backends */ +export const KNOWN_BACKENDS = { + obsidian: { + id: "obsidian", + displayName: "Obsidian Vault", + config: { + baseUrl: process.env.OBSIDIAN_API_URL || "http://localhost:27123", + apiKey: process.env.OBSIDIAN_API_KEY, + backendType: "obsidian", + } as GenericBackendConfig, + }, + notion: { + id: "notion", + displayName: "Notion", + config: { + baseUrl: process.env.NOTION_API_URL || "https://api.notion.com/v1", + apiKey: process.env.NOTION_API_KEY, + backendType: "notion", + headers: { + "Notion-Version": "2022-06-28", + }, + } as GenericBackendConfig, + }, +} as const; + +export type KnownBackendId = keyof typeof KNOWN_BACKENDS; + +/** Create a known backend from presets */ +export const createKnownBackend = (id: KnownBackendId): GenericMemoryBackend => { + const preset = KNOWN_BACKENDS[id]; + return createGenericMemoryBackend(preset.id, preset.displayName, preset.config); +}; diff --git a/src/lib/memory/index.ts b/src/lib/memory/index.ts new file mode 100644 index 00000000000..bfeeb323460 --- /dev/null +++ b/src/lib/memory/index.ts @@ -0,0 +1,44 @@ +/** + * Memory module exports and initialization + */ + +import { logger } from "../../../open-sse/utils/logger.ts"; +const log = logger("MEMORY"); + +export * from "./backend"; +export * from "./manager"; +export * from "./settings"; +export * from "./types"; +export * from "./store"; +export * from "./retrieval"; +export * from "./vectorStore"; +export * from "./embedding"; +export * from "./sqliteBackend"; +export * from "./genericBackend"; + +// Auto-register SQLiteBackend with MemoryManager on import (sync only) +import { memoryManager } from "./manager"; +import { sqliteBackend } from "./sqliteBackend"; + +memoryManager.register(sqliteBackend); + +export { memoryManager } from "./manager"; +export { sqliteBackend } from "./sqliteBackend"; +export { createGenericMemoryBackend, createKnownBackend } from "./genericBackend"; +export type { GenericBackendConfig, KnownBackendId } from "./genericBackend"; +export { KNOWN_BACKENDS } from "./genericBackend"; + +/** + * Initialize memory backends from settings. + * Call this after DB is ready (e.g., from app bootstrap). + */ +export async function initMemoryBackends(): Promise { + const { getMemorySettings } = await import("./settings"); + try { + const settings = await getMemorySettings(); + memoryManager.configure(settings.primaryBackend, settings.fallbackBackends); + await memoryManager.initialize(); + } catch (e) { + log.warn("Failed to initialize backends", { error: String(e) }); + } +} diff --git a/src/lib/memory/manager.ts b/src/lib/memory/manager.ts new file mode 100644 index 00000000000..473aef8d728 --- /dev/null +++ b/src/lib/memory/manager.ts @@ -0,0 +1,215 @@ +/** + * MemoryManager - Singleton orchestrator for memory backends + * Handles registration, routing, fallback, and caching + */ +import { logger } from "../../../open-sse/utils/logger.ts"; +import type { + MemoryBackend, + CreateMemoryInput, + MemoryFilter, + SearchConfig, + HealthCheckResult, +} from "./backend"; +import type { Memory } from "./types"; +const log = logger("MEMORY_MANAGER"); +type BackendRegistry = Map; + +class MemoryManager { + private static instance: MemoryManager; + private backends: BackendRegistry = new Map(); + private primaryBackendId: string = "sqlite"; + private fallbackBackendIds: string[] = []; + private initialized = false; + + private constructor() {} + + static getInstance(): MemoryManager { + if (!MemoryManager.instance) { + MemoryManager.instance = new MemoryManager(); + } + return MemoryManager.instance; + } + + /** Register a backend implementation */ + register(backend: MemoryBackend): void { + if (this.backends.has(backend.id)) { + log.warn(`Backend "${backend.id}" already registered, overwriting`, { id: backend.id }); + } + this.backends.set(backend.id, backend); + log.info("Registered backend", { id: backend.id, displayName: backend.displayName }); + } + + /** Unregister a backend */ + unregister(backendId: string): void { + const backend = this.backends.get(backendId); + if (backend?.shutdown) { + backend + .shutdown() + .catch((e) => log.error(`Shutdown error for ${backendId}`, { error: String(e) })); + } + this.backends.delete(backendId); + log.info("Unregistered backend", { id: backendId }); + } + + /** Get a backend by ID */ + getBackend(backendId?: string): MemoryBackend | undefined { + const id = backendId ?? this.primaryBackendId; + return this.backends.get(id); + } + + /** Get the primary backend (must exist) */ + getPrimaryBackend(): MemoryBackend { + const backend = this.getBackend(this.primaryBackendId); + if (!backend) { + throw new Error(`[MemoryManager] Primary backend "${this.primaryBackendId}" not registered`); + } + return backend; + } + + /** Get fallback backends in order */ + getFallbackBackends(): MemoryBackend[] { + return this.fallbackBackendIds + .map((id) => this.backends.get(id)) + .filter((b): b is MemoryBackend => b !== undefined); + } + + /** Configure primary and fallback backends */ + configure(primary: string, fallbacks: string[] = []): void { + if (!this.backends.has(primary)) { + throw new Error(`[MemoryManager] Primary backend "${primary}" not registered`); + } + this.primaryBackendId = primary; + this.fallbackBackendIds = fallbacks.filter((id) => this.backends.has(id)); + log.info("Configured backends", { + primary, + fallbacks: this.fallbackBackendIds, + }); + } + + /** Initialize all registered backends */ + async initialize(): Promise { + if (this.initialized) return; + + for (const [id, backend] of this.backends) { + if (backend.initialize) { + try { + await backend.initialize(); + log.info("Initialized backend", { id }); + } catch (e) { + log.error(`Failed to initialize backend ${id}`, { error: String(e) }); + } + } + } + this.initialized = true; + } + + /** Shutdown all backends */ + async shutdown(): Promise { + for (const [id, backend] of this.backends) { + if (backend.shutdown) { + try { + await backend.shutdown(); + } catch (e) { + log.error(`Shutdown error for ${id}`, { error: String(e) }); + } + } + } + this.initialized = false; + } + + // ─── Delegated CRUD with fallback ─── + + async create(input: CreateMemoryInput): Promise { + const primary = this.getPrimaryBackend(); + return primary.create(input); + } + + async get(id: string): Promise { + // Try primary first + const primary = this.getPrimaryBackend(); + const result = await primary.get(id); + if (result) return result; + + // Try fallbacks + for (const backend of this.getFallbackBackends()) { + const fallbackResult = await backend.get(id); + if (fallbackResult) return fallbackResult; + } + return null; + } + + async update(id: string, updates: Partial>): Promise { + const primary = this.getPrimaryBackend(); + const updated = await primary.update(id, updates); + + // Also try to update in fallbacks (fire-and-forget, don't fail on fallback errors) + for (const backend of this.getFallbackBackends()) { + backend + .update(id, updates) + .catch((e) => log.warn(`Fallback update failed for ${backend.id}`, { error: String(e) })); + } + return updated; + } + + async delete(id: string): Promise { + const primary = this.getPrimaryBackend(); + const deleted = await primary.delete(id); + + // Also delete from fallbacks + for (const backend of this.getFallbackBackends()) { + backend + .delete(id) + .catch((e) => log.warn(`Fallback delete failed for ${backend.id}`, { error: String(e) })); + } + return deleted; + } + + async list( + filter: MemoryFilter + ): Promise<{ data: Memory[]; total: number; byType: Record }> { + // Only primary handles list (fallbacks are for get/search redundancy) + return this.getPrimaryBackend().list(filter); + } + + // ─── Search with fallback ─── + + async search(config: SearchConfig): Promise { + const primary = this.getPrimaryBackend(); + try { + return await primary.search(config); + } catch (primaryError) { + log.warn("Primary search failed, trying fallbacks", { error: String(primaryError) }); + + for (const backend of this.getFallbackBackends()) { + try { + return await backend.search(config); + } catch (fallbackError) { + log.warn(`Fallback ${backend.id} search failed`, { error: String(fallbackError) }); + } + } + return []; + } + } + + // ─── Health check across all backends ─── + + async healthCheckAll(): Promise> { + const results: Record = {}; + for (const [id, backend] of this.backends) { + results[id] = await backend.health(); + } + return results; + } + + /** Get all registered backend info */ + getRegisteredBackends(): { id: string; displayName: string; isPrimary: boolean }[] { + return Array.from(this.backends.entries()).map(([id, backend]) => ({ + id, + displayName: backend.displayName, + isPrimary: id === this.primaryBackendId, + })); + } +} + +export const memoryManager = MemoryManager.getInstance(); +export default memoryManager; diff --git a/src/lib/memory/obsidianBackend.ts b/src/lib/memory/obsidianBackend.ts new file mode 100644 index 00000000000..a4014488fc7 --- /dev/null +++ b/src/lib/memory/obsidianBackend.ts @@ -0,0 +1,346 @@ +/** + * ObsidianBackend - Optional backend for Obsidian Vault + * Reads/writes memories as Markdown files with YAML frontmatter + */ + +import { logger } from "../../../open-sse/utils/logger.ts"; +import type { + MemoryBackend, + CreateMemoryInput, + MemoryFilter, + SearchConfig, + HealthCheckResult, + Memory, +} from "./backend"; +import { MemoryType } from "./types"; + +const log = logger("OBSIDIAN_BACKEND"); + +/** Optional backend for Obsidian Vault */ +export class ObsidianBackend implements MemoryBackend { + readonly id = "obsidian"; + readonly displayName = "Obsidian Vault"; + // isPrimary is managed by MemoryManager, not the backend itself + + private vaultPath: string; + private initialized = false; + + constructor(vaultPath: string) { + this.vaultPath = vaultPath; + } + + async initialize(): Promise { + // Verify vault path exists + const fs = await import("fs/promises"); + try { + await fs.access(this.vaultPath); + this.initialized = true; + log.info("obsidian.backend.initialized", { vaultPath: this.vaultPath }); + } catch { + throw new Error(`Obsidian vault not found at: ${this.vaultPath}`); + } + } + + async shutdown(): Promise { + this.initialized = false; + log.info("obsidian.backend.shutdown"); + } + + async create(input: CreateMemoryInput): Promise { + if (!this.initialized) await this.initialize(); + + const fs = await import("fs/promises"); + const path = await import("path"); + + const id = crypto.randomUUID(); + const fileName = `${input.key}.md`; + const filePath = path.join(this.vaultPath, fileName); + + const frontmatter = [ + "---", + `id: ${id}`, + `apiKeyId: ${input.apiKeyId}`, + `sessionId: ${input.sessionId}`, + `type: ${input.type}`, + `createdAt: ${new Date().toISOString()}`, + `updatedAt: ${new Date().toISOString()}`, + `expiresAt: ${input.expiresAt?.toISOString() || "null"}`, + "---", + "", + ].join("\n"); + + const content = frontmatter + input.content; + + await fs.writeFile(filePath, content, "utf-8"); + + return { + id, + apiKeyId: input.apiKeyId, + sessionId: input.sessionId, + type: input.type, + key: input.key, + content: input.content, + metadata: input.metadata || {}, + createdAt: new Date(), + updatedAt: new Date(), + expiresAt: input.expiresAt || null, + accessCount: 0, + lastAccessedAt: null, + }; + } + + async get(id: string): Promise { + if (!this.initialized) await this.initialize(); + + const fs = await import("fs/promises"); + const path = await import("path"); + + // Find file by id in frontmatter + const files = await fs.readdir(this.vaultPath); + + for (const file of files) { + if (!file.endsWith(".md")) continue; + + const filePath = path.join(this.vaultPath, file); + const content = await fs.readFile(filePath, "utf-8"); + + const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/); + if (!frontmatterMatch) continue; + + const frontmatter = frontmatterMatch[1]; + const idMatch = frontmatter.match(/^id:\s*(.+)$/m); + if (idMatch && idMatch[1].trim() === id) { + const body = content.replace(/^---\n[\s\S]*?\n---\n/, ""); + return this.parseMemory(frontmatter, body, id); + } + } + + return null; + } + + async update(id: string, updates: Partial>): Promise { + if (!this.initialized) await this.initialize(); + + const fs = await import("fs/promises"); + const path = await import("path"); + + const files = await fs.readdir(this.vaultPath); + + for (const file of files) { + if (!file.endsWith(".md")) continue; + + const filePath = path.join(this.vaultPath, file); + const content = await fs.readFile(filePath, "utf-8"); + + const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/); + if (!frontmatterMatch) continue; + + const frontmatter = frontmatterMatch[1]; + const idMatch = frontmatter.match(/^id:\s*(.+)$/m); + if (idMatch && idMatch[1].trim() === id) { + let newFrontmatter = frontmatter; + let newBody = content.replace(/^---\n[\s\S]*?\n---\n/, ""); + + if (updates.content !== undefined) { + newBody = updates.content; + } + + // Update frontmatter fields + const lines = newFrontmatter.split("\n").map((line) => { + if (updates.type !== undefined && line.startsWith("type:")) + return `type: ${updates.type}`; + if (updates.key !== undefined && line.startsWith("key:")) return `key: ${updates.key}`; + if (updates.metadata !== undefined && line.startsWith("metadata:")) + return `metadata: ${JSON.stringify(updates.metadata)}`; + if (updates.expiresAt !== undefined && line.startsWith("expiresAt:")) + return `expiresAt: ${updates.expiresAt?.toISOString() || "null"}`; + return line; + }); + + newFrontmatter = lines.join("\n"); + newFrontmatter = newFrontmatter.replace( + /^updatedAt:.*$/m, + `updatedAt: ${new Date().toISOString()}` + ); + + const newContent = `---\n${newFrontmatter}\n---\n\n${newBody}`; + await fs.writeFile(filePath, newContent, "utf-8"); + return true; + } + } + + return false; + } + + async delete(id: string): Promise { + if (!this.initialized) await this.initialize(); + + const fs = await import("fs/promises"); + const path = await import("path"); + + const files = await fs.readdir(this.vaultPath); + + for (const file of files) { + if (!file.endsWith(".md")) continue; + + const filePath = path.join(this.vaultPath, file); + const content = await fs.readFile(filePath, "utf-8"); + + const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/); + if (!frontmatterMatch) continue; + + const frontmatter = frontmatterMatch[1]; + const idMatch = frontmatter.match(/^id:\s*(.+)$/m); + if (idMatch && idMatch[1].trim() === id) { + await fs.unlink(filePath); + return true; + } + } + + return false; + } + + async list( + filter: MemoryFilter + ): Promise<{ data: Memory[]; total: number; byType: Record }> { + if (!this.initialized) await this.initialize(); + + const fs = await import("fs/promises"); + const path = await import("path"); + + const files = await fs.readdir(this.vaultPath); + const memories: Memory[] = []; + const byType: Record = {}; + + for (const file of files) { + if (!file.endsWith(".md")) continue; + + const filePath = path.join(this.vaultPath, file); + const content = await fs.readFile(filePath, "utf-8"); + + const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/); + if (!frontmatterMatch) continue; + + const memory = this.parseMemory( + frontmatterMatch[1], + content.replace(/^---\n[\s\S]*?\n---\n/, ""), + "" + ); + if (memory) { + // Apply filters + if (filter.apiKeyId && memory.apiKeyId !== filter.apiKeyId) continue; + if (filter.type && memory.type !== filter.type) continue; + if (filter.sessionId && memory.sessionId !== filter.sessionId) continue; + + memories.push(memory); + byType[memory.type] = (byType[memory.type] || 0) + 1; + } + } + + // Sort by createdAt desc + memories.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()); + + // Apply pagination + const offset = filter.offset || 0; + const limit = filter.limit || 100; + const paginated = memories.slice(offset, offset + limit); + + return { data: paginated, total: memories.length, byType }; + } + + async search(config: SearchConfig): Promise { + if (!this.initialized) await this.initialize(); + + const fs = await import("fs/promises"); + const path = await import("path"); + + const files = await fs.readdir(this.vaultPath); + const memories: Memory[] = []; + + for (const file of files) { + if (!file.endsWith(".md")) continue; + + const filePath = path.join(this.vaultPath, file); + const content = await fs.readFile(filePath, "utf-8"); + + const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/); + if (!frontmatterMatch) continue; + + const body = content.replace(/^---\n[\s\S]*?\n---\n/, ""); + + // Simple text search + if ( + body.toLowerCase().includes(config.query.toLowerCase()) || + file.toLowerCase().includes(config.query.toLowerCase()) + ) { + const memory = this.parseMemory(frontmatterMatch[1], body, ""); + if (memory) { + if (config.apiKeyId && memory.apiKeyId !== config.apiKeyId) continue; + memories.push(memory); + } + } + } + + return memories.slice(0, config.limit || 50); + } + + async health(): Promise { + const start = Date.now(); + try { + const fs = await import("fs/promises"); + await fs.access(this.vaultPath); + return { ok: true, latencyMs: Date.now() - start }; + } catch (e) { + return { ok: false, latencyMs: Date.now() - start, error: String(e) }; + } + } + + private parseMemory(frontmatter: string, body: string, fallbackId: string): Memory | null { + const getField = (key: string): string | null => { + const match = frontmatter.match(new RegExp(`^${key}:\\s*(.+)$`, "m")); + return match ? match[1].trim() : null; + }; + + const id = getField("id") || fallbackId || crypto.randomUUID(); + const apiKeyId = getField("apiKeyId") || ""; + const sessionId = getField("sessionId") || ""; + const type = (getField("type") as MemoryType) || MemoryType.FACTUAL; + const key = getField("key") || ""; + const createdAt = getField("createdAt") ? new Date(getField("createdAt")!) : new Date(); + const updatedAt = getField("updatedAt") ? new Date(getField("updatedAt")!) : new Date(); + const expiresAt = + getField("expiresAt") && getField("expiresAt") !== "null" + ? new Date(getField("expiresAt")!) + : null; + const accessCount = parseInt(getField("accessCount") || "0", 10); + const lastAccessedAt = + getField("lastAccessedAt") && getField("lastAccessedAt") !== "null" + ? new Date(getField("lastAccessedAt")!) + : null; + + let metadata: Record = {}; + const metadataStr = getField("metadata"); + if (metadataStr) { + try { + metadata = JSON.parse(metadataStr); + } catch {} + } + + return { + id, + apiKeyId, + sessionId, + type, + key, + content: body, + metadata, + createdAt, + updatedAt, + expiresAt, + accessCount, + lastAccessedAt, + }; + } +} + +export const createObsidianBackend = (vaultPath: string) => new ObsidianBackend(vaultPath); diff --git a/src/lib/memory/settings.ts b/src/lib/memory/settings.ts index 68fac18e806..dac4c523630 100644 --- a/src/lib/memory/settings.ts +++ b/src/lib/memory/settings.ts @@ -15,6 +15,10 @@ export interface MemorySettings { rerankEnabled: boolean; rerankProviderModel: string | null; vectorStore: "sqlite-vec" | "qdrant" | "auto"; + // Phase 1-2: MemoryBackend provider pattern + primaryBackend: string; + fallbackBackends: string[]; + backendConfigs: Record>; } export const DEFAULT_MEMORY_SETTINGS: MemorySettings = { @@ -37,6 +41,10 @@ export const DEFAULT_MEMORY_SETTINGS: MemorySettings = { rerankEnabled: false, rerankProviderModel: null, vectorStore: "auto", + // Phase 1-2: MemoryBackend defaults + primaryBackend: "sqlite", + fallbackBackends: [], + backendConfigs: {}, }; let cachedMemorySettings: MemorySettings | null = null; @@ -100,13 +108,32 @@ export function normalizeMemorySettings(rawSettings: Record = { rawSettings.memoryTransformersEnabled, DEFAULT_MEMORY_SETTINGS.transformersEnabled ), - staticEnabled: toBoolean(rawSettings.memoryStaticEnabled, DEFAULT_MEMORY_SETTINGS.staticEnabled), - rerankEnabled: toBoolean(rawSettings.memoryRerankEnabled, DEFAULT_MEMORY_SETTINGS.rerankEnabled), + staticEnabled: toBoolean( + rawSettings.memoryStaticEnabled, + DEFAULT_MEMORY_SETTINGS.staticEnabled + ), + rerankEnabled: toBoolean( + rawSettings.memoryRerankEnabled, + DEFAULT_MEMORY_SETTINGS.rerankEnabled + ), rerankProviderModel: normalizeNullableString( rawSettings.memoryRerankProviderModel, DEFAULT_MEMORY_SETTINGS.rerankProviderModel ), vectorStore: normalizeVectorStore(rawSettings.memoryVectorStore), + // Phase 1-2: MemoryBackend fields + primaryBackend: + typeof rawSettings.memoryPrimaryBackend === "string" + ? rawSettings.memoryPrimaryBackend + : DEFAULT_MEMORY_SETTINGS.primaryBackend, + fallbackBackends: Array.isArray(rawSettings.memoryFallbackBackends) + ? rawSettings.memoryFallbackBackends.filter((v): v is string => typeof v === "string") + : DEFAULT_MEMORY_SETTINGS.fallbackBackends, + backendConfigs: + typeof rawSettings.memoryBackendConfigs === "object" && + rawSettings.memoryBackendConfigs !== null + ? (rawSettings.memoryBackendConfigs as Record>) + : DEFAULT_MEMORY_SETTINGS.backendConfigs, }; } @@ -132,6 +159,11 @@ export function toMemorySettingsUpdates( if (settings.rerankProviderModel !== undefined) updates.memoryRerankProviderModel = settings.rerankProviderModel; if (settings.vectorStore !== undefined) updates.memoryVectorStore = settings.vectorStore; + // Phase 1-2: MemoryBackend fields + if (settings.primaryBackend !== undefined) updates.memoryPrimaryBackend = settings.primaryBackend; + if (settings.fallbackBackends !== undefined) + updates.memoryFallbackBackends = settings.fallbackBackends; + if (settings.backendConfigs !== undefined) updates.memoryBackendConfigs = settings.backendConfigs; return updates; } diff --git a/src/lib/memory/sqliteBackend.ts b/src/lib/memory/sqliteBackend.ts new file mode 100644 index 00000000000..a7088258963 --- /dev/null +++ b/src/lib/memory/sqliteBackend.ts @@ -0,0 +1,102 @@ +/** + * SQLiteBackend - Thin wrapper around existing store.ts functions + * Implements MemoryBackend interface by delegating to store.ts + */ + +import { logger } from "../../../open-sse/utils/logger"; +import type { + MemoryBackend, + CreateMemoryInput, + MemoryFilter, + SearchConfig, + HealthCheckResult, + Memory, +} from "./backend"; +import { MemoryType } from "./types"; +import { createMemory, getMemory, updateMemory, deleteMemory, listMemories } from "./store"; +import { retrieveMemories } from "./retrieval"; + +const log = logger("SQLITE_BACKEND"); + +export class SQLiteBackend implements MemoryBackend { + readonly id = "sqlite"; + readonly displayName = "SQLite"; + + async initialize(): Promise { + // Tables created by migrations + log.info("sqlite.backend.initialized"); + } + + async shutdown(): Promise { + log.info("sqlite.backend.shutdown"); + } + + // ─── CRUD ─── + + async create(input: CreateMemoryInput): Promise { + return createMemory({ + apiKeyId: input.apiKeyId, + sessionId: input.sessionId, + type: input.type, + key: input.key, + content: input.content, + metadata: input.metadata ?? {}, + expiresAt: input.expiresAt ?? null, + }); + } + + async get(id: string): Promise { + return getMemory(id); + } + + async update(id: string, updates: Partial>): Promise { + return updateMemory(id, updates); + } + + async delete(id: string): Promise { + return deleteMemory(id); + } + + async list( + filter: MemoryFilter + ): Promise<{ data: Memory[]; total: number; byType: Record }> { + const result = await listMemories({ + apiKeyId: filter.apiKeyId, + type: filter.type, + sessionId: filter.sessionId, + query: filter.query, + limit: filter.limit, + offset: filter.offset, + page: + filter.offset && filter.limit ? Math.floor(filter.offset / filter.limit) + 1 : undefined, + }); + return { data: result.data, total: result.total, byType: result.byType }; + } + + // ─── Search ─── + + async search(config: SearchConfig): Promise { + return retrieveMemories(config.apiKeyId, { + query: config.query, + maxTokens: config.maxTokens, + retrievalStrategy: config.strategy ?? "hybrid", + }); + } + + // ─── Health ─── + + async health(): Promise { + const start = Date.now(); + try { + // Try a simple query to verify DB is accessible + const result = await getMemory("health-check-never-exists"); + return { ok: true, latencyMs: Date.now() - start }; + } catch (e) { + return { ok: false, latencyMs: Date.now() - start, error: String(e) }; + } + } +} + +// Export singleton instance +export const sqliteBackend = new SQLiteBackend(); +export default sqliteBackend; diff --git a/src/lib/memory/store.ts b/src/lib/memory/store.ts index 549f61a8008..70884612e4b 100644 --- a/src/lib/memory/store.ts +++ b/src/lib/memory/store.ts @@ -350,8 +350,7 @@ export async function updateMemory( // Fetch current state to detect content/key change (needed for vector re-gen) const currentRow = db.prepare("SELECT content, key FROM memories WHERE id = ?").get(id) as - | { content: string; key: string | null } - | undefined; + { content: string; key: string | null } | undefined; // Build dynamic update query const fields: string[] = []; @@ -396,8 +395,7 @@ export async function updateMemory( invalidateMemoryCache(id); // Regenerate vector if content or key changed (fire-and-forget) - const contentChanged = - updates.content !== undefined && updates.content !== currentRow?.content; + const contentChanged = updates.content !== undefined && updates.content !== currentRow?.content; const keyChanged = updates.key !== undefined && updates.key !== currentRow?.key; if (contentChanged || keyChanged) { @@ -586,10 +584,7 @@ export function recordMemoryAccess(ids: string[]): void { * predicates read (no content/metadata), ordered oldest-first and bounded by `limit`, so a * sweep never materializes whole memories or scans unboundedly. */ -export function listMemoriesForDecay(filters: { - apiKeyId?: string; - limit: number; -}): { +export function listMemoriesForDecay(filters: { apiKeyId?: string; limit: number }): { id: string; type: MemoryType; accessCount: number; diff --git a/src/lib/memory/summarization.ts b/src/lib/memory/summarization.ts index 472dc75886e..f1dcd7e6c4e 100644 --- a/src/lib/memory/summarization.ts +++ b/src/lib/memory/summarization.ts @@ -71,13 +71,15 @@ interface MemoryRow { id: string; api_key_id: string; session_id: string | null; - type: string; + type: MemoryType; key: string | null; content: string; metadata: string | null; created_at: string; updated_at: string; expires_at: string | null; + access_count?: number | null; + last_accessed_at?: string | null; } function rowToMemory(row: MemoryRow): Memory { @@ -101,6 +103,8 @@ function rowToMemory(row: MemoryRow): Memory { createdAt: new Date(String(row.created_at)), updatedAt: new Date(String(row.updated_at)), expiresAt: row.expires_at ? new Date(String(row.expires_at)) : null, + accessCount: typeof row.access_count === "number" ? row.access_count : 0, + lastAccessedAt: row.last_accessed_at ? new Date(String(row.last_accessed_at)) : null, }; } diff --git a/src/shared/schemas/memory.ts b/src/shared/schemas/memory.ts index f9f2dcb5782..503016b4753 100644 --- a/src/shared/schemas/memory.ts +++ b/src/shared/schemas/memory.ts @@ -1,5 +1,5 @@ import { z } from "zod"; - +import { MemoryType } from "@/lib/memory/types"; /** Schema estendido para PUT /api/settings/memory (D9). */ export const MemorySettingsExtendedSchema = z .object({ @@ -17,13 +17,17 @@ export const MemorySettingsExtendedSchema = z rerankEnabled: z.boolean().optional(), rerankProviderModel: z.string().nullable().optional(), vectorStore: z.enum(["sqlite-vec", "qdrant", "auto"]).optional(), + // Phase 1-2: MemoryBackend provider pattern + primaryBackend: z.string().optional(), + fallbackBackends: z.array(z.string()).optional(), + backendConfigs: z.record(z.string(), z.record(z.string(), z.unknown())).optional(), }) .strict(); /** PUT /api/memory/[id] body (D6 plano §5.3). */ export const MemoryUpdatePutSchema = z .object({ - type: z.enum(["factual", "episodic", "procedural", "semantic"]).optional(), + type: z.nativeEnum(MemoryType).optional(), key: z.string().min(1).optional(), content: z.string().min(1).optional(), metadata: z.record(z.string(), z.unknown()).optional(), diff --git a/tests/unit/memory-settings.test.ts b/tests/unit/memory-settings.test.ts index 4f9eb2a25ec..9f7ccb4549b 100644 --- a/tests/unit/memory-settings.test.ts +++ b/tests/unit/memory-settings.test.ts @@ -31,6 +31,10 @@ describe("memory settings helpers", () => { rerankEnabled: DEFAULT_MEMORY_SETTINGS.rerankEnabled, rerankProviderModel: DEFAULT_MEMORY_SETTINGS.rerankProviderModel, vectorStore: DEFAULT_MEMORY_SETTINGS.vectorStore, + // Phase 1-2: MemoryBackend provider pattern + primaryBackend: DEFAULT_MEMORY_SETTINGS.primaryBackend, + fallbackBackends: DEFAULT_MEMORY_SETTINGS.fallbackBackends, + backendConfigs: DEFAULT_MEMORY_SETTINGS.backendConfigs, }); }); diff --git a/vitest.mcp.config.ts b/vitest.mcp.config.ts index eb3454fd5c0..eef7897eaca 100644 --- a/vitest.mcp.config.ts +++ b/vitest.mcp.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ "open-sse/services/autoCombo/__tests__/**/*.test.ts", "open-sse/services/combo/__tests__/**/*.test.ts", "open-sse/services/__tests__/antigravity-quota-family.test.ts", + "src/lib/memory/__tests__/generic-backend.test.ts", "tests/unit/autoCombo/**/*.test.ts", "tests/unit/encryption.spec.ts", "src/shared/components/**/*.test.tsx",