diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 31c3ca656..f4b3a663d 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -1,7 +1,7 @@ name: "\U0001F41B Bug Report" description: Report a bug to help us improve MemOS | 报告错误以帮助我们改进 MemOS title: "fix: " -labels: ["bug", "pending"] +labels: ["bug"] body: - type: checkboxes diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index b441250fa..1392d4b3d 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -1,7 +1,7 @@ name: "\U0001F680 Feature request" description: Submit a request for a new feature | 申请添加新功能 title: "feat: " -labels: ["enhancement", "pending"] +labels: ["enhancement"] body: - type: checkboxes diff --git a/README.md b/README.md index c20171832..2bc08cef0 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,10 @@

- 🎯 +43.70% Accuracy vs. OpenAI Memory
- 🏆 Top-tier Long-term Memory + Personalization
- 💰 Saves 35.24% Memory Tokens
- LoCoMo 75.80 • LongMemEval +40.43% • PrefEval-10 +2568% • PersonaMem +40.75% + 🏆 Leading Performance Across Agent and User Memory Benchmarks
+ 🤖 OpenClaw Task Completion Improves from 36.63% to 50.87%
+ 🎯 92.34 on LoCoMo and 93.40 on LongMemEval
+ 📊 Unified Evaluation Across 14 Commercial Memory Products @@ -71,6 +71,9 @@ Your lobsters and Hermes Agents now have **the best** memory system — choose * ### News +- **2026-07-02** · 🏆 **MemOS Advances Agent and User Memory Benchmarks** + With MemOS, **OpenClaw** improves average task completion from **36.63% to 50.87%** across five agent tasks. MemOS also achieves **92.34 on LoCoMo** and **93.40 on LongMemEval**, and leads in **OmniMemEval**, a unified evaluation of 14 commercial memory products across ten datasets. + - **2026-05-09** · 🧠 **memos-local-plugin 2.0** Official local memory plugin for **Hermes Agent** and **OpenClaw**. One core powers self-evolving memory across L1 traces, L2 policies, L3 world models, and crystallized Skills, with local-first storage and feedback-driven retrieval. @@ -273,14 +276,8 @@ MemOS is a **Memory Operating System** for LLMs and AI agents that unifies store ### What are the benchmark results? -| Benchmark | MemOS Result | Improvement | -|-----------|--------------|-------------| -| LoCoMo | 75.80 | - | -| LongMemEval | +40.43% vs baseline | - | -| PrefEval-10 | +2568% | - | -| PersonaMem | +40.75% | - | -| **vs OpenAI Memory** | +43.70% Accuracy | - | -| **Token Savings** | 35.24% | - | +MemOS achieves **92.34 on LoCoMo** and **93.40 on LongMemEval** for User Memory, while improving **OpenClaw** average task completion from **36.63% to 50.87%** across five Agent Memory tasks. For details, see [OmniMemEval](https://github.com/MemTensor/OmniMemEval), our unified evaluation framework for benchmarking 14 commercial memory products across ten datasets. + ### How does MemOS compare to other memory solutions? diff --git a/apps/memos-local-openclaw/README.md b/apps/memos-local-openclaw/README.md index f2a9df64f..f851316b8 100644 --- a/apps/memos-local-openclaw/README.md +++ b/apps/memos-local-openclaw/README.md @@ -514,12 +514,14 @@ All optional — shown with defaults: "recall": { "maxResultsDefault": 6, // Default search results "maxResultsMax": 20, // Max search results + "autoRecallMaxResults": 6, // Optional: override max results for the auto-recall hook only. Defaults to `maxResultsDefault`. Set lower (e.g. 3 or 5) to reduce auto-injected tokens while keeping `memory_search` results richer. "minScoreDefault": 0.45, // Default min score threshold "minScoreFloor": 0.35, // Lowest allowed min score "rrfK": 60, // RRF fusion constant "mmrLambda": 0.7, // MMR relevance vs diversity (0-1) "recencyHalfLifeDays": 14, // Time decay half-life - "vectorSearchMaxChunks": 0 // 0 = search all (default). Set 200000–300000 only if search is slow on huge DBs + "vectorSearchMaxChunks": 0, // 0 = search all (default). Set 200000–300000 only if search is slow on huge DBs + "autoRecallMinQueryLength": 4 // Skip auto-recall when the normalised user prompt is shorter than this many chars (filters "好的"/"可以"/"?"). 0 disables the guard. Has no effect on explicit memory_search tool calls. }, "dedup": { "similarityThreshold": 0.75, // Cosine similarity for smart-dedup candidates (Top-5) diff --git a/apps/memos-local-openclaw/index.ts b/apps/memos-local-openclaw/index.ts index 2f46052ee..147b31789 100644 --- a/apps/memos-local-openclaw/index.ts +++ b/apps/memos-local-openclaw/index.ts @@ -13,11 +13,13 @@ import { createRequire } from "node:module"; import { fileURLToPath } from "url"; import { buildContext } from "./src/config"; import type { HostModelsConfig } from "./src/openclaw-api"; +import { isPathInside } from "./src/path-utils"; import { ensureSqliteBinding } from "./src/storage/ensure-binding"; import { SqliteStore } from "./src/storage/sqlite"; import { Embedder } from "./src/embedding"; import { IngestWorker } from "./src/ingest/worker"; import { RecallEngine } from "./src/recall/engine"; +import { shouldSkipAutoRecallForSession } from "./src/recall/session-policy"; import { captureMessages, stripInboundMetadata } from "./src/capture"; import { DEFAULTS } from "./src/types"; import type { SearchHit } from "./src/types"; @@ -31,6 +33,8 @@ import { SkillInstaller } from "./src/skill/installer"; import { Summarizer } from "./src/ingest/providers"; import { MEMORY_GUIDE_SKILL_MD } from "./src/skill/bundled-memory-guide"; import { Telemetry } from "./src/telemetry"; +import { parseJsonOrJson5 } from "./src/shared/json5"; +import { patchOpenclawAllowFile } from "./src/shared/openclaw-config"; /** Remove near-duplicate hits based on summary word overlap (>70%). Keeps first (highest-scored) hit. */ @@ -94,7 +98,11 @@ const buildMemoryPromptSection = ({ availableTools, citationsMode }: { return lines; }; -function normalizeAutoRecallQuery(rawPrompt: string): string { +const INSTRUCTIONAL_PROMPT_MAX_LEN = 300; +const SYSTEM_ROLE_PROMPT_RE = /^You are\b/i; +const HERMES_SKILL_REVIEW_RE = /Review the conversation above/i; + +export function normalizeAutoRecallQuery(rawPrompt: string): string { let query = rawPrompt.trim(); const senderTag = "Sender (untrusted metadata):"; @@ -125,6 +133,17 @@ function normalizeAutoRecallQuery(rawPrompt: string): string { query = query.replace(INTERNAL_CONTEXT_RE, "").trim(); query = query.replace(CONTINUE_PROMPT_RE, "").trim(); + // Drop instructional prompts (system role prompts, Hermes skill-review prompts, + // over-long instruction blobs). These shapes are not user search intents — passing + // them to FTS5 fails the sanitize step and wastes a downstream LLM filter call. + if ( + query.length > INSTRUCTIONAL_PROMPT_MAX_LEN + || SYSTEM_ROLE_PROMPT_RE.test(query) + || HERMES_SKILL_REVIEW_RE.test(query) + ) { + return ""; + } + return query; } @@ -150,6 +169,27 @@ const pluginConfigSchema = { }, }; +/** + * Narrow view of the OpenClaw plugin API surface that this plugin uses for memory + * registration. Hoisted to module scope (rather than inlined at the call site) so that + * future maintainers can see exactly which host methods the feature detection covers, + * and so nobody accidentally reaches for other `OpenClawPluginApi` members through the + * narrowed reference. See issue #1559. + * + * - `registerMemoryPromptSection` was introduced in OpenClaw 2026.3.31. + * - `registerMemoryCapability` is the legacy umbrella call that the plugin used to rely + * on; kept here purely so older gateways still work. + */ +interface MemoryRegistrationApi { + registerMemoryPromptSection?: (builder: typeof buildMemoryPromptSection) => void; + registerMemoryCapability?: (capability: { promptBuilder: typeof buildMemoryPromptSection }) => void; +} + +interface ExtendedLogger { + warn(msg: string): void; + error?(msg: string): void; +} + const memosLocalPlugin = { id: "memos-local-openclaw-plugin", name: "MemOS Local Memory", @@ -160,9 +200,29 @@ const memosLocalPlugin = { configSchema: pluginConfigSchema, register(api: OpenClawPluginApi) { - api.registerMemoryCapability({ - promptBuilder: buildMemoryPromptSection, - }); + // OpenClaw 2026.3.31 split the legacy `registerMemoryCapability` facade + // into focused registration methods. Prefer the new prompt-section API, + // then fall back to the legacy capability registration so older hosts + // still load. + const memoryApi = api as OpenClawPluginApi & MemoryRegistrationApi; + + if (typeof memoryApi.registerMemoryPromptSection === "function") { + memoryApi.registerMemoryPromptSection(buildMemoryPromptSection); + } else if (typeof memoryApi.registerMemoryCapability === "function") { + memoryApi.registerMemoryCapability({ promptBuilder: buildMemoryPromptSection }); + } else { + const message = + "memos-local: host SDK exposes neither registerMemoryPromptSection " + + "nor registerMemoryCapability; memory prompt section will not be " + + "installed. The plugin may be incompatible with this OpenClaw gateway version."; + const logger = api.logger as ExtendedLogger; + if (typeof logger.error === "function") { + logger.error(message); + } else { + logger.warn(message); + } + throw new Error(message); + } const moduleDir = path.dirname(fileURLToPath(import.meta.url)); const localRequire = createRequire(import.meta.url); @@ -182,17 +242,6 @@ const memosLocalPlugin = { const pluginDir = detectPluginDir(moduleDir); - function normalizeFsPath(p: string): string { - return path.resolve(p).replace(/^\\\\\?\\/, "").toLowerCase(); - } - - function isPathInside(baseDir: string, targetPath: string): boolean { - const baseNorm = normalizeFsPath(baseDir); - const targetNorm = normalizeFsPath(targetPath); - const rel = path.relative(baseNorm, targetNorm); - return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel)); - } - function runNpm(args: string[]) { const { spawnSync } = localRequire("child_process") as typeof import("node:child_process"); return spawnSync(npmCmd, args, { @@ -292,8 +341,8 @@ const memosLocalPlugin = { const configPath = path.join(stateDir, "state", "memos-local", "config.json"); if (Object.keys(pluginCfg).length === 0 && fs.existsSync(configPath)) { try { - const fileConfig = JSON.parse(fs.readFileSync(configPath, "utf-8")); - pluginCfg = fileConfig; + const fileConfig = parseJsonOrJson5(fs.readFileSync(configPath, "utf-8")); + pluginCfg = fileConfig as Record; api.logger.info(`memos-local: loaded config from ${configPath}`); } catch (e) { api.logger.warn(`memos-local: failed to load config from ${configPath}: ${e}`); @@ -351,23 +400,17 @@ const memosLocalPlugin = { ctx.log.warn(`memos-local: could not write to managed skills dir: ${e}`); } - // Ensure plugin tools are enabled in openclaw.json tools.allow + // Ensure plugin tools are enabled in openclaw.json tools.allow. + // openclaw.json is JSON5 (supports // comments / single-quoted strings / + // trailing commas) — the patcher tolerates all of these (see issue #1543). try { const openclawJsonPath = path.join(stateDir, "openclaw.json"); if (fs.existsSync(openclawJsonPath)) { - const raw = fs.readFileSync(openclawJsonPath, "utf-8"); - const cfg = JSON.parse(raw); - const allow: string[] | undefined = cfg?.tools?.allow; - if (Array.isArray(allow) && allow.length > 0 && !allow.includes("group:plugins") && !allow.includes("*")) { - const lastEntry = JSON.stringify(allow[allow.length - 1]); - const patched = raw.replace( - new RegExp(`(${lastEntry})(\\s*\\])`), - `$1,\n "group:plugins"$2`, - ); - if (patched !== raw && patched.includes("group:plugins")) { - fs.writeFileSync(openclawJsonPath, patched, "utf-8"); - ctx.log.info("memos-local: added 'group:plugins' to tools.allow in openclaw.json"); - } + const result = patchOpenclawAllowFile(openclawJsonPath); + if (result.changed) { + ctx.log.info("memos-local: added 'group:plugins' to tools.allow in openclaw.json"); + } else if (result.reason) { + ctx.log.debug(`memos-local: tools.allow not patched (${result.reason})`); } } } catch (e) { @@ -1863,14 +1906,28 @@ Groups: ${groupNames.length > 0 ? groupNames.join(", ") : "(none)"}`, // ─── Auto-recall: inject relevant memories before agent starts ─── + // Track current session to detect cross-session memories + let currentSessionKey: string | null = null; + api.on("before_prompt_build", async (event: { prompt?: string; messages?: unknown[] }, hookCtx?: { agentId?: string; sessionKey?: string }) => { if (!allowPromptInjection) return {}; if (!event.prompt || event.prompt.length < 3) return; + // ─── Cron / excluded-session gate (GitHub #1311) ─────────────── + // Skip auto-recall (and skill auto-recall) entirely for OpenClaw + // cron sessions by default. The cron prompt is the spec — recalling + // prior meta-discussion about the cron contaminates the next run. + const recallSessionKey = hookCtx?.sessionKey ?? (event as any)?.sessionKey; + if (shouldSkipAutoRecallForSession(recallSessionKey, ctx.config.autoRecall)) { + ctx.log.info(`auto-recall: skipping (cron/excluded session: "${recallSessionKey}")`); + return; + } + const recallAgentId = hookCtx?.agentId ?? (event as any)?.agentId ?? (event as any)?.profileId ?? "main"; currentAgentId = recallAgentId; const recallOwnerFilter = [`agent:${recallAgentId}`, "public"]; - ctx.log.info(`auto-recall: agentId=${recallAgentId} (from hookCtx)`); + const incomingSessionKey = hookCtx?.sessionKey ?? "default"; + ctx.log.info(`auto-recall: agentId=${recallAgentId} sessionKey=${incomingSessionKey} (from hookCtx)`); const recallT0 = performance.now(); let recallQuery = ""; @@ -1882,16 +1939,35 @@ Groups: ${groupNames.length > 0 ? groupNames.join(", ") : "(none)"}`, const query = normalizeAutoRecallQuery(rawPrompt); recallQuery = query; - if (query.length < 2) { - ctx.log.debug("auto-recall: extracted query too short, skipping"); + // Detect if this is a new session + const isNewSession = currentSessionKey !== incomingSessionKey || NEW_SESSION_PROMPT_RE.test(rawPrompt); + if (isNewSession && currentSessionKey !== null) { + ctx.log.info(`auto-recall: new session detected (prev=${currentSessionKey}, curr=${incomingSessionKey})`); + } + currentSessionKey = incomingSessionKey; + + const minQueryLength = ctx.config.recall?.autoRecallMinQueryLength ?? DEFAULTS.autoRecallMinQueryLength; + if (query.length < minQueryLength) { + ctx.log.debug(`auto-recall: query too short (len=${query.length} < minQueryLength=${minQueryLength}), skipping`); return; } ctx.log.debug(`auto-recall: query="${query.slice(0, 80)}"`); // ── Phase 1: Local search ∥ Hub search (parallel) ── - const arLocalP = engine.search({ query, maxResults: 10, minScore: 0.45, ownerFilter: recallOwnerFilter }); + // Issue #1514: previously hardcoded maxResults: 10 here, which made + // `recall.maxResultsDefault` and the new `recall.autoRecallMaxResults` + // ineffective for the auto-recall path. Resolution order: + // 1. `recall.autoRecallMaxResults` (explicit auto-recall cap) + // 2. `recall.maxResultsDefault` (shared default with memory_search) + // 3. literal 10 (defensive — `resolveConfig` + // always fills `maxResultsDefault`, so this fires only if a + // caller built a context without going through `resolveConfig`). + const recallCfg = ctx.config.recall ?? {}; + const autoRecallMax = recallCfg.autoRecallMaxResults ?? recallCfg.maxResultsDefault ?? 10; + + const arLocalP = engine.search({ query, maxResults: autoRecallMax, minScore: 0.45, ownerFilter: recallOwnerFilter }); const arHubP = ctx.config?.sharing?.enabled - ? hubSearchMemories(store, ctx, { query, maxResults: 10, scope: "all" }) + ? hubSearchMemories(store, ctx, { query, maxResults: autoRecallMax, scope: "all" }) .catch((err: any) => { ctx.log.debug(`auto-recall: hub search failed (${err})`); return { hits: [] as any[], meta: {} }; }) : Promise.resolve({ hits: [] as any[], meta: {} }); @@ -2014,10 +2090,18 @@ Groups: ${groupNames.length > 0 ? groupNames.join(", ") : "(none)"}`, filteredHits = deduplicateHits(filteredHits); ctx.log.debug(`auto-recall: merged ${allRawHits.length} → ${beforeDedup} relevant → ${filteredHits.length} after dedup, sufficient=${sufficient}`); + // Check if any memories are from a different session + const hasCrossSessionMemories = filteredHits.some(h => + h.source.sessionKey && h.source.sessionKey !== incomingSessionKey + ); + ctx.log.debug(`auto-recall: isNewSession=${isNewSession}, hasCrossSessionMemories=${hasCrossSessionMemories}`); + const lines = filteredHits.map((h, i) => { const excerpt = h.original_excerpt; + const isCrossSession = h.source.sessionKey && h.source.sessionKey !== incomingSessionKey; + const sessionTag = isCrossSession ? " [from previous session]" : ""; const oTag = h.origin === "local-shared" ? " [本机共享]" : h.origin === "hub-memory" ? " [团队缓存]" : ""; - const parts: string[] = [`${i + 1}. [${h.source.role}]${oTag}`]; + const parts: string[] = [`${i + 1}. [${h.source.role}]${sessionTag}${oTag}`]; if (excerpt) parts.push(` ${excerpt}`); parts.push(` chunkId="${h.ref.chunkId}"`); if (h.taskId) { @@ -2042,15 +2126,32 @@ Groups: ${groupNames.length > 0 ? groupNames.join(", ") : "(none)"}`, tips.push("- Need more surrounding dialogue → call `memory_timeline(chunkId=\"...\")` to expand context around a hit"); const tipsText = "\n\nAvailable follow-up tools:\n" + tips.join("\n"); + // Use different instructions based on whether memories are from current or previous sessions const contextParts = [ "## User's conversation history (from memory system)", "", - "IMPORTANT: The following are facts from previous conversations with this user.", - "You MUST treat these as established knowledge and use them directly when answering.", - "Do NOT say you don't know or don't have information if the answer is in these memories.", - "", - lines.join("\n\n"), ]; + + if (hasCrossSessionMemories || isNewSession) { + contextParts.push( + "IMPORTANT: The following memories are from PREVIOUS SESSIONS.", + "Treat them as BACKGROUND KNOWLEDGE ONLY:", + "- Do NOT act on them unprompted or proactively respond based solely on these memories", + "- WAIT for the user's explicit instruction before taking any action", + "- These memories provide context, but the user must initiate the conversation", + "- If you reference these memories, explicitly note they are from a previous session (e.g., \"根据之前的会话...\" or \"Based on a previous conversation...\")", + "", + ); + } else { + contextParts.push( + "IMPORTANT: The following are facts from previous conversations with this user.", + "You MUST treat these as established knowledge and use them directly when answering.", + "Do NOT say you don't know or don't have information if the answer is in these memories.", + "", + ); + } + + contextParts.push(lines.join("\n\n")); if (tipsText) contextParts.push(tipsText); // ─── Skill auto-recall ─── diff --git a/apps/memos-local-openclaw/package.json b/apps/memos-local-openclaw/package.json index 0323ee06c..0f6887a0f 100644 --- a/apps/memos-local-openclaw/package.json +++ b/apps/memos-local-openclaw/package.json @@ -3,10 +3,10 @@ "version": "1.0.9-beta.1", "description": "MemOS Local memory plugin for OpenClaw — full-write, hybrid-recall, progressive retrieval", "type": "module", - "main": "index.ts", + "main": "dist/index.js", + "types": "dist/index.d.ts", "files": [ - "index.ts", - "src", + "dist", "skill", "prebuilds", "scripts/native-binding.cjs", @@ -19,7 +19,7 @@ "openclaw": { "id": "memos-local-openclaw-plugin", "extensions": [ - "./index.ts" + "./dist/index.js" ], "skills": [ "skill/memos-memory-guide" @@ -34,7 +34,7 @@ "test:watch": "vitest", "test:accuracy": "tsx scripts/run-accuracy-test.ts", "postinstall": "node scripts/postinstall.cjs", - "prepublishOnly": "echo 'Source-only publish — no build needed.'" + "prepack": "npm run build" }, "keywords": [ "openclaw", diff --git a/apps/memos-local-openclaw/scripts/re-embed.ts b/apps/memos-local-openclaw/scripts/re-embed.ts new file mode 100644 index 000000000..8b3d492a3 --- /dev/null +++ b/apps/memos-local-openclaw/scripts/re-embed.ts @@ -0,0 +1,222 @@ +#!/usr/bin/env tsx +/** + * scripts/re-embed.ts — re-embed chunks under the currently configured + * embedding provider/model. See issue #1333. + * + * Usage: + * pnpm exec tsx scripts/re-embed.ts [--missing-only] [--dry-run] + * [--limit N] [--batch-size N] + * [--db PATH] [--config PATH] + */ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { SqliteStore } from "../src/storage/sqlite"; +import { Embedder } from "../src/embedding"; +import { resolveConfig } from "../src/config"; +import type { Logger, MemosLocalConfig } from "../src/types"; + +interface CliOpts { + missingOnly: boolean; + dryRun: boolean; + limit?: number; + batchSize: number; + dbPath?: string; + configPath: string; + help: boolean; +} + +export function parseArgs(argv: string[]): CliOpts { + const opts: CliOpts = { + missingOnly: false, + dryRun: false, + batchSize: 32, + configPath: path.join(os.homedir(), ".openclaw", "openclaw.json"), + help: false, + }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--missing-only") opts.missingOnly = true; + else if (a === "--dry-run") opts.dryRun = true; + else if (a === "--limit") opts.limit = Number(argv[++i]); + else if (a === "--batch-size") opts.batchSize = Number(argv[++i]); + else if (a === "--db") opts.dbPath = argv[++i]; + else if (a === "--config") opts.configPath = argv[++i]; + else if (a === "-h" || a === "--help") opts.help = true; + else throw new Error(`Unknown argument: ${a}`); + } + return opts; +} + +const HELP = `Re-embed chunks under the currently configured embedding provider/model. + +Usage: tsx scripts/re-embed.ts [options] + + --missing-only Only re-embed chunks that have no embedding row. + --dry-run Print the plan only; do not write embeddings. + --limit Process at most n chunks. + --batch-size Embed n chunks per provider call (default 32). + --db Override DB path (default reads ~/.openclaw config). + --config Override openclaw.json path (default ~/.openclaw/openclaw.json). + -h, --help Show this help. + +Behaviour: scans for chunks whose embedding row's (provider, model, dimensions) +does not match the current Embedder — plus rows tagged as 'legacy' (empty +producer columns, from before this feature) and chunks with no embedding row. +Re-embeds them in batches; never deletes existing data. Re-running picks up +where it left off because the candidate list is re-computed each run. +`; + +function makeLog(): Logger { + return { + debug: (...a) => console.debug("[re-embed]", ...a), + info: (...a) => console.info("[re-embed]", ...a), + warn: (...a) => console.warn("[re-embed]", ...a), + error: (...a) => console.error("[re-embed]", ...a), + }; +} + +function loadPluginConfig(configPath: string): Partial { + if (!fs.existsSync(configPath)) { + throw new Error(`Config not found: ${configPath}`); + } + const raw = JSON.parse(fs.readFileSync(configPath, "utf-8")) as Record; + const plugins = (raw.plugins as Record | undefined) ?? {}; + const entries = ((plugins as { entries?: Record }).entries ?? {}) as Record }>; + const configs = ((plugins as { configs?: Record }).configs ?? {}) as Record }>; + const candidate = + entries["memos-local"]?.config ?? + entries["memos-local-openclaw-plugin"]?.config ?? + configs["memos-local"]?.config ?? + {}; + return candidate; +} + +function formatStats(label: string, s: { + total: number; + matched: number; + mismatched: number; + legacy: number; + missing: number; + current: { provider: string; model: string; dimensions: number }; + byProducer: Array<{ provider: string; model: string; dimensions: number; count: number }>; +}): string { + const lines: string[] = []; + lines.push(`── ${label} ──`); + lines.push(` current: ${s.current.provider}:${s.current.model}:${s.current.dimensions}`); + lines.push(` total embeddings: ${s.total}`); + lines.push(` matched: ${s.matched}`); + lines.push(` mismatched: ${s.mismatched}`); + lines.push(` legacy: ${s.legacy} (pre-tagging rows; treated as needing re-embed)`); + lines.push(` missing: ${s.missing} (active chunks with no embedding row)`); + lines.push(` by producer:`); + for (const b of s.byProducer) { + const tag = b.provider === "" ? "(legacy)" : `${b.provider}/${b.model || "(no model)"}`; + lines.push(` ${tag} dim=${b.dimensions} count=${b.count}`); + } + return lines.join("\n"); +} + +export async function runReembed(opts: CliOpts): Promise<{ processed: number; failed: number; planned: number }> { + const log = makeLog(); + const pluginCfg = loadPluginConfig(opts.configPath); + const stateDir = path.join(os.homedir(), ".openclaw"); + const resolved = resolveConfig(pluginCfg, stateDir); + const dbPath = opts.dbPath ?? resolved.storage!.dbPath!; + log.info(`config: ${opts.configPath}`); + log.info(`db: ${dbPath}`); + + const store = new SqliteStore(dbPath, log); + const embedder = new Embedder(resolved.embedding, log); + const producer = { provider: embedder.provider, model: embedder.model }; + const current = { provider: embedder.provider, model: embedder.model, dimensions: embedder.dimensions }; + + const before = store.getEmbeddingStats(current); + console.log(formatStats("before", before)); + + const ids = store.listChunkIdsForReembed(current, { + missingOnly: opts.missingOnly, + limit: opts.limit, + }); + console.log(`\nplanned re-embed: ${ids.length} chunk(s)`); + + if (opts.dryRun) { + console.log("(dry-run: no writes performed)"); + store.close(); + return { processed: 0, failed: 0, planned: ids.length }; + } + + let processed = 0; + let failed = 0; + for (let i = 0; i < ids.length; i += opts.batchSize) { + const batchIds = ids.slice(i, i + opts.batchSize); + const texts: string[] = []; + const keep: string[] = []; + for (const id of batchIds) { + const chunk = store.getChunk(id); + if (!chunk) continue; + const text = chunk.summary || (chunk.content ?? "").slice(0, 500); + if (!text) continue; + keep.push(id); + texts.push(text); + } + if (keep.length === 0) continue; + + try { + const vecs = await embedder.embed(texts); + for (let j = 0; j < keep.length; j++) { + if (vecs[j]) { + store.upsertEmbedding(keep[j], vecs[j], producer); + processed++; + } + } + console.log(` batch ${i / opts.batchSize + 1}: re-embedded ${keep.length} chunk(s) (cumulative ${processed}/${ids.length})`); + } catch (err) { + failed += keep.length; + log.warn(`batch ${i / opts.batchSize + 1} failed; continuing: ${err}`); + } + } + + const after = store.getEmbeddingStats(current); + console.log("\n" + formatStats("after", after)); + console.log(`\ndone: processed=${processed} failed=${failed} planned=${ids.length}`); + + store.close(); + return { processed, failed, planned: ids.length }; +} + +async function main(): Promise { + let opts: CliOpts; + try { + opts = parseArgs(process.argv.slice(2)); + } catch (err) { + console.error(String(err)); + console.error(HELP); + process.exit(2); + } + if (opts.help) { + console.log(HELP); + return; + } + const result = await runReembed(opts); + if (result.failed > 0) process.exitCode = 1; +} + +// Node will run main() when invoked directly. We avoid the `import.meta.url` +// trick for vitest compatibility and just check require.main or treat any +// direct script execution as the entry point. +const isDirect = (() => { + try { + // tsx + ESM: process.argv[1] is the script path + return process.argv[1] && process.argv[1].endsWith("re-embed.ts"); + } catch { + return false; + } +})(); + +if (isDirect) { + main().catch((err) => { + console.error("re-embed failed:", err); + process.exit(1); + }); +} diff --git a/apps/memos-local-openclaw/src/capture/index.ts b/apps/memos-local-openclaw/src/capture/index.ts index 10d5282c6..c150fc360 100644 --- a/apps/memos-local-openclaw/src/capture/index.ts +++ b/apps/memos-local-openclaw/src/capture/index.ts @@ -7,6 +7,11 @@ const SYSTEM_BOILERPLATE_RE = /^A new session was started via \/new or \/reset\b // Boot-check / memory-system injection patterns that should never be stored. const BOOT_CHECK_RE = /^(?:You are running a boot check|Read HEARTBEAT\.md if it exists|## Memory system — ACTION REQUIRED)/; +// Agent-framework self-instruction prompts (e.g. Hermes Agent's "Review the +// conversation above, consider saving..."). These arrive with role="user" +// but are not user content and must never be stored as memory. +const REVIEW_CONVERSATION_RE = /^Review the conversation above/i; + /** * Returns true for sentinel reply values that carry no user-facing content. */ @@ -81,6 +86,10 @@ export function captureMessages( log.debug(`Skipping boot-check injection: ${msg.content.slice(0, 60)}...`); continue; } + if (role === "user" && REVIEW_CONVERSATION_RE.test(msg.content.trim())) { + log.debug(`Skipping review-conversation injection: ${msg.content.slice(0, 60)}...`); + continue; + } if (role === "tool" && msg.toolName && SELF_TOOLS.has(msg.toolName)) { log.debug(`Skipping self-tool result: ${msg.toolName}`); diff --git a/apps/memos-local-openclaw/src/config.ts b/apps/memos-local-openclaw/src/config.ts index 150b09cc4..1691ec482 100644 --- a/apps/memos-local-openclaw/src/config.ts +++ b/apps/memos-local-openclaw/src/config.ts @@ -4,11 +4,11 @@ import { OpenClawAPIClient, type HostModelsConfig } from "./openclaw-api"; const ENV_RE = /\$\{([A-Z_][A-Z0-9_]*)\}/g; -function resolveEnvVars(value: string): string { +export function resolveEnvVars(value: string): string { return value.replace(ENV_RE, (_, name) => process.env[name] ?? ""); } -function deepResolveEnv(obj: T): T { +export function deepResolveEnv(obj: T): T { if (typeof obj === "string") return resolveEnvVars(obj) as unknown as T; if (Array.isArray(obj)) return obj.map(deepResolveEnv) as unknown as T; if (obj && typeof obj === "object") { @@ -57,15 +57,24 @@ export function resolveConfig(raw: Partial | undefined, stateD storage: { dbPath: cfg.storage?.dbPath ?? path.join(stateDir, "memos-local", "memos.db"), }, + autoRecall: { + excludeCron: cfg.autoRecall?.excludeCron ?? true, + excludeSessionKeyPatterns: cfg.autoRecall?.excludeSessionKeyPatterns ?? [], + }, recall: { maxResultsDefault: cfg.recall?.maxResultsDefault ?? DEFAULTS.maxResultsDefault, maxResultsMax: cfg.recall?.maxResultsMax ?? DEFAULTS.maxResultsMax, + // Optional override for the `before_prompt_build` auto-recall hook. + // Left undefined when the user does not configure it so the hook can + // fall through to `maxResultsDefault` at call time (issue #1514). + autoRecallMaxResults: cfg.recall?.autoRecallMaxResults, minScoreDefault: cfg.recall?.minScoreDefault ?? DEFAULTS.minScoreDefault, minScoreFloor: cfg.recall?.minScoreFloor ?? DEFAULTS.minScoreFloor, rrfK: cfg.recall?.rrfK ?? DEFAULTS.rrfK, mmrLambda: cfg.recall?.mmrLambda ?? DEFAULTS.mmrLambda, recencyHalfLifeDays: cfg.recall?.recencyHalfLifeDays ?? DEFAULTS.recencyHalfLifeDays, vectorSearchMaxChunks: cfg.recall?.vectorSearchMaxChunks ?? DEFAULTS.vectorSearchMaxChunks, + autoRecallMinQueryLength: cfg.recall?.autoRecallMinQueryLength ?? DEFAULTS.autoRecallMinQueryLength, }, dedup: { similarityThreshold: cfg.dedup?.similarityThreshold ?? DEFAULTS.dedupSimilarityThreshold, diff --git a/apps/memos-local-openclaw/src/embedding/index.ts b/apps/memos-local-openclaw/src/embedding/index.ts index 2adc7cac7..0a5e88067 100644 --- a/apps/memos-local-openclaw/src/embedding/index.ts +++ b/apps/memos-local-openclaw/src/embedding/index.ts @@ -21,11 +21,25 @@ export class Embedder { return this.cfg?.provider ?? "local"; } + get model(): string { + if (this.provider === "local") return ""; + return this.cfg?.model ?? ""; + } + get dimensions(): number { if (this.provider === "local") return 384; return this.cfg?.dimensions ?? 1536; } + /** + * Canonical identity of the embedding space this embedder produces vectors in. + * Format: "provider:model:dimensions". Used to detect when stored vectors + * were produced by a different model than the live config. + */ + get signature(): string { + return `${this.provider}:${this.model}:${this.dimensions}`; + } + async embed(texts: string[]): Promise { const batchSize = this.cfg?.batchSize ?? 32; const results: number[][] = []; diff --git a/apps/memos-local-openclaw/src/embedding/local.ts b/apps/memos-local-openclaw/src/embedding/local.ts index a87242ea8..7a913c54b 100644 --- a/apps/memos-local-openclaw/src/embedding/local.ts +++ b/apps/memos-local-openclaw/src/embedding/local.ts @@ -2,6 +2,10 @@ import type { Logger } from "../types"; import { DEFAULTS } from "../types"; let extractorPromise: Promise | null = null; +let callCount = 0; + +// Read from env or use default +const RESET_AFTER_CALLS = parseInt(process.env.MEMOS_EMBED_RESET_AFTER_CALLS || "50", 10); function getExtractor(log: Logger): Promise { if (extractorPromise) return extractorPromise; @@ -23,13 +27,55 @@ function getExtractor(log: Logger): Promise { return extractorPromise; } +async function resetExtractor(log: Logger): Promise { + if (!extractorPromise) return; + + try { + const ext = await extractorPromise; + // Attempt to dispose the pipeline to free ONNX session resources + if (typeof ext?.dispose === "function") { + await ext.dispose(); + } + } catch (err) { + log.warn(`Failed to dispose extractor: ${err}`); + } + + extractorPromise = null; + callCount = 0; + log.debug("Local embedding pipeline reset to free native memory"); +} + export async function embedLocal(texts: string[], log: Logger): Promise { const ext = await getExtractor(log); const results: number[][] = []; for (const text of texts) { const output = await ext(text, { pooling: "mean", normalize: true }); + + // Extract the embedding vector results.push(Array.from(output.data as Float32Array).slice(0, DEFAULTS.localEmbeddingDimensions)); + + // Explicitly release the output tensor to prevent ONNX memory leak + try { + // Null out the data reference + (output as any).data = null; + } catch {} + + try { + // Call dispose if available + if (typeof (output as any).dispose === "function") { + (output as any).dispose(); + } + } catch {} + + callCount++; + } + + // Periodically reset the pipeline to prevent long-term memory accumulation + // Set MEMOS_EMBED_RESET_AFTER_CALLS=0 to disable periodic reset + if (RESET_AFTER_CALLS > 0 && callCount >= RESET_AFTER_CALLS) { + log.debug(`Reached ${callCount} embedding calls, resetting pipeline to free native memory`); + await resetExtractor(log); } return results; diff --git a/apps/memos-local-openclaw/src/hub/server.ts b/apps/memos-local-openclaw/src/hub/server.ts index d8b697d79..bc61cecf5 100644 --- a/apps/memos-local-openclaw/src/hub/server.ts +++ b/apps/memos-local-openclaw/src/hub/server.ts @@ -201,11 +201,12 @@ export class HubServer { private embedChunksAsync(chunkIds: string[], chunks: Array<{ id: string; summary?: string; content?: string }>): void { const embedder = this.opts.embedder; if (!embedder) return; + const producer = { provider: embedder.provider, model: embedder.model }; const texts = chunks.map(c => c.summary || (c.content ? c.content.slice(0, 500) : "")); embedder.embed(texts).then((vectors) => { for (let i = 0; i < vectors.length; i++) { if (vectors[i]) { - this.opts.store.upsertHubEmbedding(chunkIds[i], new Float32Array(vectors[i])); + this.opts.store.upsertHubEmbedding(chunkIds[i], new Float32Array(vectors[i]), producer); } } this.opts.log.info(`hub: embedded ${vectors.filter(Boolean).length}/${chunkIds.length} shared chunks`); @@ -217,10 +218,11 @@ export class HubServer { private embedSkillAsync(skillId: string, name: string, description: string, sourceUserId: string, sourceSkillId: string): void { const embedder = this.opts.embedder; if (!embedder) return; + const producer = { provider: embedder.provider, model: embedder.model }; const text = `${name}: ${description}`; embedder.embed([text]).then((vectors) => { if (vectors[0]) { - this.opts.store.upsertHubSkillEmbedding(skillId, Array.from(vectors[0]), sourceUserId, sourceSkillId); + this.opts.store.upsertHubSkillEmbedding(skillId, Array.from(vectors[0]), sourceUserId, sourceSkillId, producer); this.opts.log.info(`hub: embedded shared skill ${skillId}`); } }).catch((err) => { @@ -230,6 +232,8 @@ export class HubServer { private backfillMemoryEmbeddings(): void { if (!this.opts.embedder) return; + const embedder = this.opts.embedder; + const producer = { provider: embedder.provider, model: embedder.model }; try { const all = this.opts.store.listHubMemories({ limit: 500 }); const missing = all.filter(m => { @@ -238,11 +242,11 @@ export class HubServer { if (missing.length === 0) return; this.opts.log.info(`hub: backfilling embeddings for ${missing.length} hub memories`); const texts = missing.map(m => (m.summary || m.content || "").slice(0, 500)); - this.opts.embedder.embed(texts).then((vectors) => { + embedder.embed(texts).then((vectors) => { let count = 0; for (let i = 0; i < vectors.length; i++) { if (vectors[i]) { - this.opts.store.upsertHubMemoryEmbedding(missing[i].id, new Float32Array(vectors[i])); + this.opts.store.upsertHubMemoryEmbedding(missing[i].id, new Float32Array(vectors[i]), producer); count++; } } @@ -258,11 +262,12 @@ export class HubServer { private embedMemoryAsync(memoryId: string, summary: string, content: string): void { const embedder = this.opts.embedder; if (!embedder) return; + const producer = { provider: embedder.provider, model: embedder.model }; const text = (summary || content || "").slice(0, 500); if (!text) return; embedder.embed([text]).then((vectors) => { if (vectors[0]) { - this.opts.store.upsertHubMemoryEmbedding(memoryId, new Float32Array(vectors[0])); + this.opts.store.upsertHubMemoryEmbedding(memoryId, new Float32Array(vectors[0]), producer); this.opts.log.info(`hub: embedded shared memory ${memoryId}`); } }).catch((err) => { diff --git a/apps/memos-local-openclaw/src/index.ts b/apps/memos-local-openclaw/src/index.ts index abf3e3590..bc2a09f8e 100644 --- a/apps/memos-local-openclaw/src/index.ts +++ b/apps/memos-local-openclaw/src/index.ts @@ -61,6 +61,7 @@ export function initPlugin(opts: PluginInitOptions = {}): MemosLocalPlugin { const store = new SqliteStore(ctx.config.storage!.dbPath!, ctx.log); const embedder = new Embedder(ctx.config.embedding, ctx.log, ctx.openclawAPI); + warnOnEmbeddingMismatch(store, embedder, ctx.log); const worker = new IngestWorker(store, embedder, ctx); const engine = new RecallEngine(store, embedder, ctx); @@ -115,5 +116,31 @@ function defaultStateDir(): string { return `${home}/.openclaw`; } +/** + * One-shot init check (issue #1333): if the `embeddings` table has rows + * tagged with a different producer than the live Embedder, or untagged + * legacy rows from a previous version, emit a single warn line so the + * user knows to run `scripts/re-embed.ts`. Best-effort: never throws. + */ +function warnOnEmbeddingMismatch(store: SqliteStore, embedder: Embedder, log: Logger): void { + try { + const stats = store.getEmbeddingStats({ + provider: embedder.provider, + model: embedder.model, + dimensions: embedder.dimensions, + }); + if (stats.total === 0) return; + const stale = stats.legacy + stats.mismatched; + if (stale === 0) return; + log.warn( + `embedding model mismatch detected: ${stats.legacy} legacy + ${stats.mismatched} from prior models, ` + + `${stats.matched} match current (${embedder.signature}). Vector recall is degraded for non-matching rows — ` + + `run \`pnpm exec tsx scripts/re-embed.ts\` to re-embed.`, + ); + } catch (err) { + log.debug(`warnOnEmbeddingMismatch skipped: ${err}`); + } +} + // Re-export types for consumers export type { MemosLocalConfig, ToolDefinition, SearchResult, SearchHit, TimelineResult, GetResult } from "./types"; diff --git a/apps/memos-local-openclaw/src/ingest/providers/index.ts b/apps/memos-local-openclaw/src/ingest/providers/index.ts index b08818520..c5dd1a85c 100644 --- a/apps/memos-local-openclaw/src/ingest/providers/index.ts +++ b/apps/memos-local-openclaw/src/ingest/providers/index.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as path from "path"; import type { SummarizerConfig, SummaryProvider, Logger, OpenClawAPI } from "../../types"; +import { parseJsonOrJson5 } from "../../shared/json5"; import { summarizeOpenAI, summarizeTaskOpenAI, generateTaskTitleOpenAI, judgeNewTopicOpenAI, classifyTopicOpenAI, arbitrateTopicSplitOpenAI, filterRelevantOpenAI, judgeDedupOpenAI, parseFilterResult, parseDedupResult, parseTopicClassifyResult } from "./openai"; import type { FilterResult, DedupResult, TopicClassifyResult } from "./openai"; export type { FilterResult, DedupResult, TopicClassifyResult } from "./openai"; @@ -66,7 +67,7 @@ function loadOpenClawFallbackConfig(log: Logger): SummarizerConfig | undefined { || path.join(process.env.OPENCLAW_STATE_DIR || path.join(home, ".openclaw"), "openclaw.json"); if (!fs.existsSync(cfgPath)) return undefined; - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + const raw = parseJsonOrJson5(fs.readFileSync(cfgPath, "utf-8")) as any; const agentModel: string | undefined = raw?.agents?.defaults?.model?.primary; if (!agentModel) return undefined; diff --git a/apps/memos-local-openclaw/src/ingest/providers/openai.ts b/apps/memos-local-openclaw/src/ingest/providers/openai.ts index 825e2131d..60f878e4c 100644 --- a/apps/memos-local-openclaw/src/ingest/providers/openai.ts +++ b/apps/memos-local-openclaw/src/ingest/providers/openai.ts @@ -233,7 +233,14 @@ export async function judgeNewTopicOpenAI( body: JSON.stringify(buildRequestBody(cfg, { model, temperature: 0, - max_tokens: 10, + // NOTE: must stay >= 60. MiniMax's gateway (api.minimaxi.com) rejects + // chat-completion requests with very small max_tokens (e.g. 10) by + // returning an HTML 404 page before the request ever reaches the + // model. 60 matches classifyTopicOpenAI below — already proven to + // work against MiniMax-M2.7-highspeed — and is plenty for a one-word + // NEW/SAME reply plus any reasoning preamble the model may emit. + // See issue #1315. + max_tokens: 60, messages: [ { role: "system", content: TOPIC_JUDGE_PROMPT }, { role: "user", content: userContent }, @@ -336,7 +343,9 @@ export async function arbitrateTopicSplitOpenAI( body: JSON.stringify(buildRequestBody(cfg, { model, temperature: 0, - max_tokens: 10, + // NOTE: must stay >= 60. See note in judgeNewTopicOpenAI above — + // MiniMax's gateway returns HTML 404 for max_tokens: 10. Issue #1315. + max_tokens: 60, messages: [ { role: "system", content: TOPIC_ARBITRATION_PROMPT }, { role: "user", content: userContent }, diff --git a/apps/memos-local-openclaw/src/ingest/worker.ts b/apps/memos-local-openclaw/src/ingest/worker.ts index d62ab4a2a..c2921719b 100644 --- a/apps/memos-local-openclaw/src/ingest/worker.ts +++ b/apps/memos-local-openclaw/src/ingest/worker.ts @@ -286,7 +286,10 @@ export class IngestWorker { this.store.insertChunk(chunk); if (embedding && dedupStatus === "active") { - this.store.upsertEmbedding(chunkId, embedding); + this.store.upsertEmbedding(chunkId, embedding, { + provider: this.embedder.provider, + model: this.embedder.model, + }); } this.ctx.log.debug(`Stored chunk=${chunkId} kind=${kind} role=${msg.role} dedup=${dedupStatus} len=${content.length} hasVec=${!!embedding && dedupStatus === "active"}`); diff --git a/apps/memos-local-openclaw/src/openclaw-config.ts b/apps/memos-local-openclaw/src/openclaw-config.ts new file mode 100644 index 000000000..1725d09a8 --- /dev/null +++ b/apps/memos-local-openclaw/src/openclaw-config.ts @@ -0,0 +1,69 @@ +/** + * Helpers for safely mutating ~/.openclaw/openclaw.json from the plugin. + * + * Previously this lived inline in index.ts and used a hand-written regex + * on the raw JSON text. That approach corrupted the config when the same + * literal that ended `tools.allow` also appeared elsewhere in the file + * (e.g. inside `models.providers.*.models[*].input`). See issue #1377: + * memos-local-openclaw-plugin corrupts openclaw.json by inserting + * "group:plugins" into models[*].input. + * + * The fixed implementation parses the JSON, mutates the parsed object, + * and re-serialises it. That guarantees the new entry can only land in + * tools.allow. + */ + +/** Detect indent unit (2 / 4 spaces or tab) by sampling the first indented line. */ +function detectIndent(raw: string): string | number { + const match = raw.match(/\n([ \t]+)\S/); + if (!match) return 2; + const indent = match[1]; + if (indent.startsWith("\t")) return "\t"; + return indent.length; +} + +/** + * Ensure that `entry` is present in `tools.allow` inside the given raw + * openclaw.json text. Returns the (possibly updated) JSON text. + * + * Behaviour: + * - If parsing fails, the input is returned unchanged. + * - If `tools.allow` is missing, empty, contains `"*"`, or already + * contains `entry`, the input is returned unchanged (referentially + * equal to the input string) — callers can detect "no change" by + * identity comparison and skip the disk write. + * - Otherwise, the entry is appended to `tools.allow` and the result + * is re-serialised with the original indentation. The original + * trailing newline is preserved. + */ +export function ensureToolsAllowEntry(raw: string, entry: string): string { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return raw; + } + if (!parsed || typeof parsed !== "object") { + return raw; + } + const root = parsed as Record; + const tools = root.tools; + if (!tools || typeof tools !== "object") { + return raw; + } + const allow = (tools as Record).allow; + if (!Array.isArray(allow) || allow.length === 0) { + return raw; + } + if (allow.includes(entry) || allow.includes("*")) { + return raw; + } + + const next = { ...root, tools: { ...(tools as Record), allow: [...allow, entry] } }; + const indent = detectIndent(raw); + let serialised = JSON.stringify(next, null, indent as any); + if (raw.endsWith("\n") && !serialised.endsWith("\n")) { + serialised += "\n"; + } + return serialised; +} diff --git a/apps/memos-local-openclaw/src/path-utils.ts b/apps/memos-local-openclaw/src/path-utils.ts new file mode 100644 index 000000000..c4680cf4c --- /dev/null +++ b/apps/memos-local-openclaw/src/path-utils.ts @@ -0,0 +1,96 @@ +/** + * Path comparison helpers shared by the plugin runtime. + * + * Why a dedicated module: + * - The plugin's `register()` closure used to inline these helpers, which + * made them invisible to unit tests and let a Windows-specific path + * bug (`/C:/Users/...` URL-pathname form, `\\?\UNC\…` long-path prefix, + * mixed slash directions) trip the better-sqlite3 sandbox guard and + * refuse to load a perfectly valid native binding. + * - Extracting them here lets `tests/path-utils.test.ts` exercise every + * known Windows path shape from a Linux CI box by stubbing + * `process.platform` between cases. + * + * Cross-reference: `scripts/postinstall.cjs` keeps its own + * `normalizePathForMatch` helper because it runs as CommonJS before the + * plugin loads. Keep the two in sync if either side changes semantics. + */ + +import * as path from "path"; + +const isWin = process.platform === "win32"; +const platformPath = isWin ? path.win32 : path.posix; + +/** + * Canonicalise an absolute filesystem path so two callers can compare + * paths without tripping on platform-specific quirks. + * + * On Windows the function: + * 1. strips a leading slash that precedes a drive letter + * (`/C:/Users/...` → `C:/Users/...`), which is the shape + * `new URL(import.meta.url).pathname` returns on Node ≤ 22; + * 2. unwraps the `\\?\UNC\server\share` extended UNC prefix into + * `\\server\share`; + * 3. strips the plain `\\?\` long-path prefix; + * 4. resolves to an absolute path with native separators; + * 5. converts all `\` to `/`; + * 6. lower-cases the result (Windows paths are case-insensitive). + * + * On POSIX the function resolves the path, swaps backslashes to forward + * slashes (a no-op for legal POSIX paths), and returns it unchanged. + * The case is preserved so callers do not silently mismatch on + * case-sensitive filesystems. + */ +export function normalizeFsPath(p: string): string { + let s = p; + + // 1. URL pathname form: `/C:/Users/...` → `C:/Users/...`. + // Only fires when the next two chars are a drive letter + `:` + // followed by a separator. POSIX absolute paths like + // `/home/user/...` are untouched. + s = s.replace(/^\/(?=[A-Za-z]:[\\/])/, ""); + + // 2. Extended UNC: `\\?\UNC\server\share` → `\\server\share`. + // The leading `\\` is preserved so step 4 still sees a UNC path. + s = s.replace(/^\\\\\?\\UNC\\/i, "\\\\"); + + // 3. Plain extended-length prefix: `\\?\C:\Users\...` → `C:\Users\...`. + s = s.replace(/^\\\\\?\\/, ""); + + // 4. Resolve to absolute form using the platform-appropriate flavour. + // Using the platform-specific module (rather than the default `path`) + // keeps the tests deterministic on Linux CI when we stub + // `process.platform`. + s = platformPath.resolve(s); + + // 5. Unify separators. + s = s.replace(/\\/g, "/"); + + // 6. Lower-case only on Windows. + if (isWin) s = s.toLowerCase(); + + return s; +} + +/** + * Return `true` iff `targetPath` is the same directory as `baseDir` + * or one of its descendants. Both inputs are normalised through + * `normalizeFsPath` first; the relative computation is then done in + * POSIX form so the answer does not depend on platform separators. + * + * Behaviour for edge cases: + * - same directory → `true` + * - descendant (`base/sub/file`) → `true` + * - sibling (`base/../other/file`) → `false` + * - parent (`base/..`) → `false` + * - different drive on Windows (`D:\…` vs `C:\…`) → `false` + */ +export function isPathInside(baseDir: string, targetPath: string): boolean { + const base = normalizeFsPath(baseDir); + const target = normalizeFsPath(targetPath); + const rel = path.posix.relative(base, target); + if (rel === "") return true; + if (rel === ".." || rel.startsWith("../")) return false; + if (path.posix.isAbsolute(rel)) return false; + return true; +} diff --git a/apps/memos-local-openclaw/src/recall/engine.ts b/apps/memos-local-openclaw/src/recall/engine.ts index 711bde5a0..100e6467c 100644 --- a/apps/memos-local-openclaw/src/recall/engine.ts +++ b/apps/memos-local-openclaw/src/recall/engine.ts @@ -15,6 +15,15 @@ export interface RecallOptions { minScore?: number; role?: string; ownerFilter?: string[]; + /** + * If set, chunks whose `sessionKey` equals this value are filtered out at the + * SQL layer (FTS, vector, pattern) before fusion. Use this to suppress recall + * of the **current** conversation session so the model doesn't waste tokens + * on its own recent turns. Hub-memory hits are not affected by this filter + * because they represent cross-user shared knowledge keyed by a synthetic + * `sessionKey` (`hub-shared:`). + */ + excludeSessionKey?: string; } const MAX_RECENT_QUERIES = 20; @@ -41,10 +50,11 @@ export class RecallEngine { const repeatNote = this.checkRepeat(query, maxResults, minScore); const candidatePool = maxResults * 5; const ownerFilter = opts.ownerFilter; + const excludeSessionKey = opts.excludeSessionKey; // Step 1: Gather candidates from FTS, vector search, and pattern search const ftsCandidates = query - ? this.store.ftsSearch(query, candidatePool, ownerFilter) + ? this.store.ftsSearch(query, candidatePool, ownerFilter, excludeSessionKey) : []; let vecCandidates: Array<{ chunkId: string; score: number }> = []; @@ -54,7 +64,7 @@ export class RecallEngine { const maxChunks = recallCfg.vectorSearchMaxChunks && recallCfg.vectorSearchMaxChunks > 0 ? recallCfg.vectorSearchMaxChunks : undefined; - vecCandidates = vectorSearch(this.store, queryVec, candidatePool, maxChunks, ownerFilter); + vecCandidates = vectorSearch(this.store, queryVec, candidatePool, maxChunks, ownerFilter, excludeSessionKey); } catch (err) { this.ctx.log.warn(`Vector search failed, using FTS only: ${err}`); } @@ -77,7 +87,7 @@ export class RecallEngine { } const shortTerms = [...new Set([...spaceSplit, ...cjkBigrams])]; const patternHits = shortTerms.length > 0 - ? this.store.patternSearch(shortTerms, { limit: candidatePool, ownerFilter }) + ? this.store.patternSearch(shortTerms, { limit: candidatePool, ownerFilter, excludeSessionKey }) : []; const patternRanked = patternHits.map((h, i) => ({ id: h.chunkId, diff --git a/apps/memos-local-openclaw/src/recall/session-policy.ts b/apps/memos-local-openclaw/src/recall/session-policy.ts new file mode 100644 index 000000000..25ff1c032 --- /dev/null +++ b/apps/memos-local-openclaw/src/recall/session-policy.ts @@ -0,0 +1,66 @@ +/** + * Auto-recall session-key policy. + * + * Decides whether the `before_prompt_build` auto-recall hook should + * short-circuit for the current OpenClaw session. + * + * Background: OpenClaw cron jobs use stable session keys of the form + * `agent::cron:`. Auto-recall has historically used the + * cron prompt itself as the recall query, which means prior meta-discussion + * about the cron (prompt tuning, debugging, rerun requests) was injected + * back into the next scheduled run and contaminated its output. See + * GitHub issue MemTensor/MemOS#1311. + * + * The helper is intentionally pure and lives outside the plugin entry so + * it can be unit tested without spinning up the full plugin context. + */ + +export interface AutoRecallExclusionConfig { + /** + * When true (default), skip auto-recall for any session whose key + * contains a `cron` segment (`/(^|:)cron(:|$)/i`). Operators who rely on + * cron-cross-turn memory recall can flip this to `false` to restore the + * pre-1311 behaviour. + */ + excludeCron?: boolean; + /** + * Optional list of additional regex strings tested against the raw + * session key. Any match wins (OR semantics with `excludeCron`). + * Invalid regex strings are ignored. + */ + excludeSessionKeyPatterns?: string[]; +} + +const CRON_SEGMENT_RE = /(?:^|:)cron(?::|$)/i; + +/** + * Return `true` when the auto-recall hook should not run for this session. + * + * - Empty / missing `sessionKey` → returns `false` (no opinion; preserves + * behaviour for hosts that do not pass a session key). + * - Cron session keys are skipped when `cfg?.excludeCron !== false`. + * The default behaviour (no config) is to skip. + * - Any user-supplied regex in `cfg?.excludeSessionKeyPatterns` that + * matches the session key also forces a skip. + */ +export function shouldSkipAutoRecallForSession( + sessionKey: string | undefined, + cfg: AutoRecallExclusionConfig | undefined, +): boolean { + if (!sessionKey) return false; + + const excludeCron = cfg?.excludeCron ?? true; + if (excludeCron && CRON_SEGMENT_RE.test(sessionKey)) return true; + + const patterns = cfg?.excludeSessionKeyPatterns ?? []; + for (const raw of patterns) { + try { + if (new RegExp(raw).test(sessionKey)) return true; + } catch { + // Invalid regex — silently ignore. Callers may log this at config + // resolution time but the hook must not throw. + } + } + + return false; +} diff --git a/apps/memos-local-openclaw/src/shared/json5.ts b/apps/memos-local-openclaw/src/shared/json5.ts new file mode 100644 index 000000000..439c54903 --- /dev/null +++ b/apps/memos-local-openclaw/src/shared/json5.ts @@ -0,0 +1,183 @@ +/** + * Minimal JSON5-tolerant parser used to read `openclaw.json`. + * + * Background: OpenClaw stores its native config (`~/.openclaw/openclaw.json`) + * in JSON5 — it allows `//` line comments, `/* ... *​/` block comments, + * single-quoted strings, unquoted identifier keys, and trailing commas. + * Strict `JSON.parse` chokes on any of these (issue #1543). + * + * Bringing in the full `json5` npm package would be the obvious fix, but the + * plugin must avoid new runtime dependencies (it ships its `dependencies` + * one-by-one to keep install size small, and the install path runs in user + * machines where adding npm deps is sensitive). Instead, we normalize the raw + * text into strict JSON and hand it to the built-in `JSON.parse`. + * + * What is supported: + * • `//` line comments and `/* … *​/` block comments + * • single-quoted strings (`'foo'`) + * • unquoted identifier-style object keys (`tools: { ... }`) + * • trailing commas before `}` or `]` + * • UTF-8 BOM + * + * What is NOT supported (rare in practice for openclaw.json): + * • Hex numbers (`0xFF`), `+Infinity`, `NaN`, etc. — falls back to error. + * • Multi-line string continuation. + * + * The implementation walks the text character-by-character with a small state + * machine so that comment / quote rewriting never touches characters inside + * string literals (which is the classic regex-based-stripper bug). + */ + +export class Json5ParseError extends Error { + constructor(message: string, public readonly originalError?: unknown) { + super(message); + this.name = "Json5ParseError"; + } +} + +/** + * Strip comments, normalize quotes/keys/trailing commas, then `JSON.parse`. + * + * Throws Json5ParseError if the normalized text still isn't valid JSON — + * callers should treat this the same as a malformed `JSON.parse`. + */ +export function parseJson5(text: string): unknown { + if (typeof text !== "string") { + throw new Json5ParseError(`parseJson5 expects a string, got ${typeof text}`); + } + + const normalized = normalizeJson5(text); + try { + return JSON.parse(normalized); + } catch (err) { + throw new Json5ParseError( + `Failed to parse JSON5 content as JSON after normalization: ${(err as Error).message}`, + err, + ); + } +} + +/** + * Lenient variant: try strict `JSON.parse` first (fast path for files that + * happen to be plain JSON), fall back to the JSON5-tolerant path on failure. + * + * This is the recommended entry point for code that reads `openclaw.json`, + * because the user's config might or might not contain comments. + */ +export function parseJsonOrJson5(text: string): unknown { + // Fast path: strict JSON. Use it when possible so we don't pay normalization + // cost on every read. + try { + return JSON.parse(text); + } catch { + return parseJson5(text); + } +} + +/** + * Normalize a JSON5 text into strict JSON-compatible text. + * Exposed for testing; production callers should use parseJson5 / parseJsonOrJson5. + */ +export function normalizeJson5(input: string): string { + // Strip UTF-8 BOM + let text = input.charCodeAt(0) === 0xfeff ? input.slice(1) : input; + + const out: string[] = []; + const n = text.length; + let i = 0; + + while (i < n) { + const c = text[i]; + const next = i + 1 < n ? text[i + 1] : ""; + + // Line comment + if (c === "/" && next === "/") { + i += 2; + while (i < n && text[i] !== "\n") i++; + continue; + } + // Block comment + if (c === "/" && next === "*") { + i += 2; + while (i < n && !(text[i] === "*" && text[i + 1] === "/")) i++; + if (i < n) i += 2; // skip closing */ + continue; + } + + // Double-quoted string — passthrough verbatim, including escapes. + if (c === '"') { + out.push(c); + i++; + while (i < n) { + const ch = text[i]; + out.push(ch); + if (ch === "\\" && i + 1 < n) { + out.push(text[i + 1]); + i += 2; + continue; + } + if (ch === '"') { + i++; + break; + } + i++; + } + continue; + } + + // Single-quoted string — rewrite to double-quoted, escaping any embedded `"`. + if (c === "'") { + out.push('"'); + i++; + while (i < n) { + const ch = text[i]; + if (ch === "\\" && i + 1 < n) { + // Preserve escape sequence verbatim, except `\'` becomes a plain `'` + // (because the surrounding quotes are now double quotes). + const nx = text[i + 1]; + if (nx === "'") { + out.push("'"); + } else { + out.push("\\"); + out.push(nx); + } + i += 2; + continue; + } + if (ch === "'") { + out.push('"'); + i++; + break; + } + if (ch === '"') { + // Escape a bare `"` so the resulting double-quoted string stays valid. + out.push("\\\""); + i++; + continue; + } + out.push(ch); + i++; + } + continue; + } + + out.push(c); + i++; + } + + let stripped = out.join(""); + + // Unquoted identifier keys: ` foo: 1` → ` "foo": 1`. + // Matches an identifier that starts with a letter / `_` / `$` and is + // immediately followed by optional whitespace and `:`. The leading + // boundary ensures we don't match parts of other tokens. + stripped = stripped.replace( + /([\{,\s])([A-Za-z_$][A-Za-z0-9_$]*)(\s*:)/g, + (_, lead: string, key: string, tail: string) => `${lead}"${key}"${tail}`, + ); + + // Trailing commas before `}` or `]`. + stripped = stripped.replace(/,(\s*[}\]])/g, "$1"); + + return stripped; +} diff --git a/apps/memos-local-openclaw/src/shared/llm-call.ts b/apps/memos-local-openclaw/src/shared/llm-call.ts index aa868fc4c..c2df76bd0 100644 --- a/apps/memos-local-openclaw/src/shared/llm-call.ts +++ b/apps/memos-local-openclaw/src/shared/llm-call.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as path from "path"; import type { SummarizerConfig, SummaryProvider, Logger, PluginContext, OpenClawAPI } from "../types"; +import { parseJsonOrJson5 } from "./json5"; /** * Resolve a SecretInput (string | SecretRef) to a plain string. @@ -54,7 +55,7 @@ export function loadOpenClawFallbackConfig(log: Logger): SummarizerConfig | unde || path.join(process.env.OPENCLAW_STATE_DIR || path.join(home, ".openclaw"), "openclaw.json"); if (!fs.existsSync(cfgPath)) return undefined; - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + const raw = parseJsonOrJson5(fs.readFileSync(cfgPath, "utf-8")) as any; const agentModel: string | undefined = raw?.agents?.defaults?.model?.primary; if (!agentModel) return undefined; diff --git a/apps/memos-local-openclaw/src/shared/openclaw-config.ts b/apps/memos-local-openclaw/src/shared/openclaw-config.ts new file mode 100644 index 000000000..9d19892dd --- /dev/null +++ b/apps/memos-local-openclaw/src/shared/openclaw-config.ts @@ -0,0 +1,96 @@ +import * as fs from "fs"; +import { parseJsonOrJson5 } from "./json5"; + +/** + * Ensure `"group:plugins"` is present in `tools.allow` inside `openclaw.json`. + * + * Strategy: + * 1. Read the file as JSON5 (the user may have // comments etc. — see issue #1543). + * 2. If `tools.allow` is missing, wildcarded (`"*"`), or already contains + * `"group:plugins"`, do nothing. + * 3. Otherwise patch the raw text by appending `"group:plugins"` after the + * last existing allow entry. We do textual insertion (rather than + * `JSON.stringify(cfg)`) so we preserve the user's comments and + * formatting in the file. + * + * Returns: + * - `{ changed: false }` when no edit was needed or the patch could not be + * anchored safely. + * - `{ changed: true, patched }` when the caller should `writeFileSync` the + * `patched` text back. + */ +export interface EnsureGroupPluginsAllowedResult { + changed: boolean; + patched?: string; + reason?: string; +} + +const PATCH_VALUE = "group:plugins"; + +export function ensureGroupPluginsAllowed( + raw: string, +): EnsureGroupPluginsAllowedResult { + let cfg: any; + try { + cfg = parseJsonOrJson5(raw); + } catch (err) { + return { changed: false, reason: `parse failed: ${(err as Error).message}` }; + } + + const allow: unknown = cfg?.tools?.allow; + if (!Array.isArray(allow) || allow.length === 0) { + return { changed: false, reason: "tools.allow is missing or empty" }; + } + if (allow.includes("*") || allow.includes(PATCH_VALUE)) { + return { changed: false, reason: "tools.allow already permissive enough" }; + } + + const lastEntry = allow[allow.length - 1]; + if (typeof lastEntry !== "string") { + return { changed: false, reason: "tools.allow last entry is not a string" }; + } + + // Try anchoring on the last entry as either a double-quoted or single-quoted + // string — openclaw.json is JSON5 and the user may have used either. We also + // tolerate an optional trailing comma between the last entry and `]`. + const escaped = lastEntry.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const candidates: Array<{ anchor: RegExp; insert: string }> = [ + { + anchor: new RegExp(`("${escaped}")(\\s*,?\\s*\\])`), + insert: `$1,\n "${PATCH_VALUE}"$2`, + }, + { + anchor: new RegExp(`('${escaped}')(\\s*,?\\s*\\])`), + insert: `$1,\n '${PATCH_VALUE}'$2`, + }, + ]; + + for (const { anchor, insert } of candidates) { + if (anchor.test(raw)) { + const patched = raw.replace(anchor, insert); + if (patched !== raw && patched.includes(PATCH_VALUE)) { + return { changed: true, patched }; + } + } + } + + return { changed: false, reason: "could not anchor last allow entry" }; +} + +/** + * File-level convenience wrapper. Reads `openclawJsonPath`, applies + * `ensureGroupPluginsAllowed`, writes it back if needed. + * + * Caller is expected to pre-check `fs.existsSync(openclawJsonPath)`. + * Returns the in-memory result so the caller can log. + */ +export function patchOpenclawAllowFile( + openclawJsonPath: string, +): EnsureGroupPluginsAllowedResult { + const raw = fs.readFileSync(openclawJsonPath, "utf-8"); + const result = ensureGroupPluginsAllowed(raw); + if (result.changed && result.patched) { + fs.writeFileSync(openclawJsonPath, result.patched, "utf-8"); + } + return result; +} diff --git a/apps/memos-local-openclaw/src/storage/sqlite.ts b/apps/memos-local-openclaw/src/storage/sqlite.ts index 09f9c2bf7..642fafeff 100644 --- a/apps/memos-local-openclaw/src/storage/sqlite.ts +++ b/apps/memos-local-openclaw/src/storage/sqlite.ts @@ -120,9 +120,43 @@ export class SqliteStore { this.migrateHubUserIdentityFields(); this.migrateClientHubConnectionIdentityFields(); this.migrateTeamSharingInstanceId(); + this.migrateEmbeddingProducerColumns(); this.log.debug("Database schema initialized"); } + /** + * Tag every cached embedding row with the producer that created it. + * Adds `provider TEXT NOT NULL DEFAULT ''` + `model TEXT NOT NULL DEFAULT ''` + * to every embedding-shaped table. Idempotent: skips tables that already + * have the columns. `dimensions` already exists on every target table. + */ + private migrateEmbeddingProducerColumns(): void { + const tables = [ + "embeddings", + "skill_embeddings", + "task_embeddings", + "hub_embeddings", + "hub_skill_embeddings", + "hub_memory_embeddings", + ]; + for (const table of tables) { + try { + const cols = this.db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>; + if (cols.length === 0) continue; // table didn't exist yet + if (!cols.some((c) => c.name === "provider")) { + this.db.exec(`ALTER TABLE ${table} ADD COLUMN provider TEXT NOT NULL DEFAULT ''`); + this.log.info(`Migrated: added provider column to ${table}`); + } + if (!cols.some((c) => c.name === "model")) { + this.db.exec(`ALTER TABLE ${table} ADD COLUMN model TEXT NOT NULL DEFAULT ''`); + this.log.info(`Migrated: added model column to ${table}`); + } + } catch (err) { + this.log.warn(`migrateEmbeddingProducerColumns(${table}) failed: ${err}`); + } + } + } + private migrateChunksIndexesForRecall(): void { this.db.exec("CREATE INDEX IF NOT EXISTS idx_chunks_dedup_created ON chunks(dedup_status, created_at DESC)"); } @@ -1177,12 +1211,12 @@ export class SqliteStore { ); } - upsertEmbedding(chunkId: string, vector: number[]): void { + upsertEmbedding(chunkId: string, vector: number[], producer?: { provider?: string; model?: string }): void { const buf = Buffer.from(new Float32Array(vector).buffer); this.db.prepare(` - INSERT OR REPLACE INTO embeddings (chunk_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) - `).run(chunkId, buf, vector.length, Date.now()); + INSERT OR REPLACE INTO embeddings (chunk_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run(chunkId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } deleteEmbedding(chunkId: string): void { @@ -1241,7 +1275,7 @@ export class SqliteStore { // ─── FTS Search ─── - ftsSearch(query: string, limit: number, ownerFilter?: string[]): Array<{ chunkId: string; score: number }> { + ftsSearch(query: string, limit: number, ownerFilter?: string[], excludeSessionKey?: string): Array<{ chunkId: string; score: number }> { const sanitized = sanitizeFtsQuery(query); if (!sanitized) return []; @@ -1259,6 +1293,11 @@ export class SqliteStore { params.push(...ownerFilter); } + if (excludeSessionKey) { + sql += ` AND c.session_key != ?`; + params.push(excludeSessionKey); + } + sql += ` ORDER BY rank LIMIT ?`; params.push(limit); @@ -1278,7 +1317,7 @@ export class SqliteStore { // ─── Pattern Search (LIKE-based, for CJK text where FTS tokenization is weak) ─── - patternSearch(patterns: string[], opts: { role?: string; limit?: number; ownerFilter?: string[] } = {}): Array<{ chunkId: string; content: string; role: string; createdAt: number }> { + patternSearch(patterns: string[], opts: { role?: string; limit?: number; ownerFilter?: string[]; excludeSessionKey?: string } = {}): Array<{ chunkId: string; content: string; role: string; createdAt: number }> { if (patterns.length === 0) return []; const limit = opts.limit ?? 10; @@ -1295,13 +1334,19 @@ export class SqliteStore { params.push(...opts.ownerFilter); } + let sessionClause = ""; + if (opts.excludeSessionKey) { + sessionClause = ` AND c.session_key != ?`; + params.push(opts.excludeSessionKey); + } + params.push(limit); try { const rows = this.db.prepare(` SELECT c.id as chunk_id, c.content, c.role, c.created_at FROM chunks c - WHERE (${whereClause})${roleClause}${ownerClause} AND c.dedup_status = 'active' + WHERE (${whereClause})${roleClause}${ownerClause}${sessionClause} AND c.dedup_status = 'active' ORDER BY c.created_at DESC LIMIT ? `).all(...params) as Array<{ chunk_id: string; content: string; role: string; created_at: number }>; @@ -1345,7 +1390,7 @@ export class SqliteStore { // ─── Vector Search ─── - getAllEmbeddings(ownerFilter?: string[]): Array<{ chunkId: string; vector: number[] }> { + getAllEmbeddings(ownerFilter?: string[], excludeSessionKey?: string): Array<{ chunkId: string; vector: number[] }> { let sql = `SELECT e.chunk_id, e.vector, e.dimensions FROM embeddings e JOIN chunks c ON c.id = e.chunk_id WHERE c.dedup_status = 'active'`; @@ -1357,6 +1402,11 @@ export class SqliteStore { params.push(...ownerFilter); } + if (excludeSessionKey) { + sql += ` AND c.session_key != ?`; + params.push(excludeSessionKey); + } + const rows = this.db.prepare(sql).all(...params) as Array<{ chunk_id: string; vector: Buffer; dimensions: number }>; return rows.map((r) => ({ @@ -1365,8 +1415,8 @@ export class SqliteStore { })); } - getRecentEmbeddings(limit: number, ownerFilter?: string[]): Array<{ chunkId: string; vector: number[] }> { - if (limit <= 0) return this.getAllEmbeddings(ownerFilter); + getRecentEmbeddings(limit: number, ownerFilter?: string[], excludeSessionKey?: string): Array<{ chunkId: string; vector: number[] }> { + if (limit <= 0) return this.getAllEmbeddings(ownerFilter, excludeSessionKey); let sql = `SELECT e.chunk_id, e.vector, e.dimensions FROM chunks c @@ -1380,6 +1430,11 @@ export class SqliteStore { params.push(...ownerFilter); } + if (excludeSessionKey) { + sql += ` AND c.session_key != ?`; + params.push(excludeSessionKey); + } + sql += ` ORDER BY c.created_at DESC LIMIT ?`; params.push(limit); @@ -1399,6 +1454,87 @@ export class SqliteStore { return Array.from(new Float32Array(row.vector.buffer, row.vector.byteOffset, row.dimensions)); } + // ─── Embedding model signature reporting (issue #1333) ─── + + /** + * Snapshot of cached vector rows compared against the live `current` + * embedder signature. `legacy` are rows that pre-date the producer + * tagging columns (provider = ''). `mismatched` are rows whose + * producer was tagged but differs from `current`. `missing` counts + * active chunks that have no embedding row at all. + */ + getEmbeddingStats(current: { provider: string; model: string; dimensions: number }): { + total: number; + matched: number; + mismatched: number; + legacy: number; + missing: number; + current: { provider: string; model: string; dimensions: number }; + byProducer: Array<{ provider: string; model: string; dimensions: number; count: number }>; + } { + const total = (this.db.prepare("SELECT COUNT(*) as c FROM embeddings").get() as { c: number }).c; + const matched = (this.db.prepare( + "SELECT COUNT(*) as c FROM embeddings WHERE provider = ? AND model = ? AND dimensions = ?", + ).get(current.provider, current.model, current.dimensions) as { c: number }).c; + const legacy = (this.db.prepare( + "SELECT COUNT(*) as c FROM embeddings WHERE provider = ''", + ).get() as { c: number }).c; + const mismatched = (this.db.prepare( + "SELECT COUNT(*) as c FROM embeddings WHERE provider != '' AND NOT (provider = ? AND model = ? AND dimensions = ?)", + ).get(current.provider, current.model, current.dimensions) as { c: number }).c; + const missing = (this.db.prepare(` + SELECT COUNT(*) as c FROM chunks c + WHERE c.dedup_status = 'active' + AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = c.id) + `).get() as { c: number }).c; + + const byProducer = (this.db.prepare( + "SELECT provider, model, dimensions, COUNT(*) as count FROM embeddings GROUP BY provider, model, dimensions ORDER BY count DESC", + ).all() as Array<{ provider: string; model: string; dimensions: number; count: number }>); + + return { total, matched, mismatched, legacy, missing, current, byProducer }; + } + + /** + * Chunk ids that should be re-embedded under `current`. By default returns + * every chunk whose embedding row's producer doesn't match (including legacy + * empty rows) plus chunks with no embedding row at all. With + * `missingOnly: true`, returns only the latter. Ordered by `created_at ASC` + * so re-embed runs are deterministic and resumable. + */ + listChunkIdsForReembed( + current: { provider: string; model: string; dimensions: number }, + opts: { missingOnly?: boolean; limit?: number } = {}, + ): string[] { + const limit = opts.limit ?? Number.MAX_SAFE_INTEGER; + if (opts.missingOnly) { + const rows = this.db.prepare(` + SELECT c.id as id FROM chunks c + WHERE c.dedup_status = 'active' + AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = c.id) + ORDER BY c.created_at ASC + LIMIT ? + `).all(limit) as Array<{ id: string }>; + return rows.map((r) => r.id); + } + const rows = this.db.prepare(` + SELECT c.id as id FROM chunks c + WHERE c.dedup_status = 'active' + AND ( + NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = c.id) + OR EXISTS ( + SELECT 1 FROM embeddings e + WHERE e.chunk_id = c.id + AND NOT (e.provider = ? AND e.model = ? AND e.dimensions = ?) + ) + ) + ORDER BY c.created_at ASC + LIMIT ? + `).all(current.provider, current.model, current.dimensions, limit) as Array<{ id: string }>; + return rows.map((r) => r.id); + } + + // ─── Update ─── updateChunk(chunkId: string, fields: { summary?: string; content?: string; role?: string; kind?: string; owner?: string }): boolean { @@ -1807,12 +1943,12 @@ export class SqliteStore { .run(visibility, Date.now(), skillId); } - upsertSkillEmbedding(skillId: string, vector: number[]): void { + upsertSkillEmbedding(skillId: string, vector: number[], producer?: { provider?: string; model?: string }): void { const buf = Buffer.from(new Float32Array(vector).buffer); this.db.prepare(` - INSERT OR REPLACE INTO skill_embeddings (skill_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) - `).run(skillId, buf, vector.length, Date.now()); + INSERT OR REPLACE INTO skill_embeddings (skill_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run(skillId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } getSkillEmbedding(skillId: string): number[] | null { @@ -1887,12 +2023,12 @@ export class SqliteStore { // ─── Task Embeddings & Search ─── - upsertTaskEmbedding(taskId: string, vector: number[]): void { + upsertTaskEmbedding(taskId: string, vector: number[], producer?: { provider?: string; model?: string }): void { const buf = Buffer.from(new Float32Array(vector).buffer); this.db.prepare(` - INSERT OR REPLACE INTO task_embeddings (task_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) - `).run(taskId, buf, vector.length, Date.now()); + INSERT OR REPLACE INTO task_embeddings (task_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run(taskId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } getTaskEmbeddings(owner?: string): Array<{ taskId: string; vector: number[] }> { @@ -2344,19 +2480,21 @@ export class SqliteStore { return row ? rowToHubSkill(row) : null; } - upsertHubSkillEmbedding(skillId: string, vector: number[], sourceUserId: string, sourceSkillId: string): void { + upsertHubSkillEmbedding(skillId: string, vector: number[], sourceUserId: string, sourceSkillId: string, producer?: { provider?: string; model?: string }): void { if (!sourceUserId || !sourceSkillId) throw new Error("sourceUserId and sourceSkillId are required for hub skill embedding upserts"); const canonicalSkillId = this.resolveCanonicalHubSkillId(skillId, sourceUserId, sourceSkillId); const buf = Buffer.allocUnsafe(vector.length * 4); for (let i = 0; i < vector.length; i++) buf.writeFloatLE(vector[i], i * 4); this.db.prepare(` - INSERT INTO hub_skill_embeddings (skill_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) + INSERT INTO hub_skill_embeddings (skill_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(skill_id) DO UPDATE SET vector = excluded.vector, dimensions = excluded.dimensions, + provider = excluded.provider, + model = excluded.model, updated_at = excluded.updated_at - `).run(canonicalSkillId, buf, vector.length, Date.now()); + `).run(canonicalSkillId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } getHubSkillEmbedding(skillId: string): number[] | null { @@ -2379,13 +2517,13 @@ export class SqliteStore { })); } - upsertHubMemoryEmbedding(memoryId: string, vector: Float32Array): void { + upsertHubMemoryEmbedding(memoryId: string, vector: Float32Array, producer?: { provider?: string; model?: string }): void { const buf = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength); this.db.prepare(` - INSERT INTO hub_memory_embeddings (memory_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(memory_id) DO UPDATE SET vector = excluded.vector, dimensions = excluded.dimensions, updated_at = excluded.updated_at - `).run(memoryId, buf, vector.length, Date.now()); + INSERT INTO hub_memory_embeddings (memory_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(memory_id) DO UPDATE SET vector = excluded.vector, dimensions = excluded.dimensions, provider = excluded.provider, model = excluded.model, updated_at = excluded.updated_at + `).run(memoryId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } getHubMemoryEmbedding(memoryId: string): Float32Array | null { @@ -2431,13 +2569,13 @@ export class SqliteStore { return rows.map((row, idx) => ({ hit: row, rank: idx + 1 })); } - upsertHubEmbedding(chunkId: string, vector: Float32Array): void { + upsertHubEmbedding(chunkId: string, vector: Float32Array, producer?: { provider?: string; model?: string }): void { const buf = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength); this.db.prepare(` - INSERT INTO hub_embeddings (chunk_id, vector, dimensions, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(chunk_id) DO UPDATE SET vector = excluded.vector, dimensions = excluded.dimensions, updated_at = excluded.updated_at - `).run(chunkId, buf, vector.length, Date.now()); + INSERT INTO hub_embeddings (chunk_id, vector, dimensions, provider, model, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(chunk_id) DO UPDATE SET vector = excluded.vector, dimensions = excluded.dimensions, provider = excluded.provider, model = excluded.model, updated_at = excluded.updated_at + `).run(chunkId, buf, vector.length, producer?.provider ?? "", producer?.model ?? "", Date.now()); } getHubEmbedding(chunkId: string): Float32Array | null { diff --git a/apps/memos-local-openclaw/src/storage/vector.ts b/apps/memos-local-openclaw/src/storage/vector.ts index 1acec2d3e..488f287bf 100644 --- a/apps/memos-local-openclaw/src/storage/vector.ts +++ b/apps/memos-local-openclaw/src/storage/vector.ts @@ -22,6 +22,8 @@ export interface VectorHit { /** * Brute-force vector search over stored embeddings. * When maxChunks > 0, only searches the most recent maxChunks chunks (uses index; avoids full scan as data grows). + * When excludeSessionKey is set, chunks whose session_key equals it are filtered out before scoring, + * so the caller can suppress recall of the current conversation session. */ export function vectorSearch( store: SqliteStore, @@ -29,10 +31,11 @@ export function vectorSearch( topK: number, maxChunks?: number, ownerFilter?: string[], + excludeSessionKey?: string, ): VectorHit[] { const all = maxChunks != null && maxChunks > 0 - ? store.getRecentEmbeddings(maxChunks, ownerFilter) - : store.getAllEmbeddings(ownerFilter); + ? store.getRecentEmbeddings(maxChunks, ownerFilter, excludeSessionKey) + : store.getAllEmbeddings(ownerFilter, excludeSessionKey); const scored: VectorHit[] = all.map((row) => ({ chunkId: row.chunkId, score: cosineSimilarity(queryVec, row.vector), diff --git a/apps/memos-local-openclaw/src/tools/memory-search.ts b/apps/memos-local-openclaw/src/tools/memory-search.ts index 43cad5bc8..463fe2dcf 100644 --- a/apps/memos-local-openclaw/src/tools/memory-search.ts +++ b/apps/memos-local-openclaw/src/tools/memory-search.ts @@ -57,6 +57,10 @@ export function createMemorySearchTool(engine: RecallEngine, store?: SqliteStore type: "string", description: "Optional hub bearer token override for group/all search or integration tests.", }, + excludeSessionKey: { + type: "string", + description: "Optional sessionKey to exclude from recall. Pass the current conversation's sessionKey to avoid recalling chunks from the ongoing session — useful to save tokens and surface only historical memories.", + }, }, }, handler: async (input) => { @@ -66,12 +70,16 @@ export function createMemorySearchTool(engine: RecallEngine, store?: SqliteStore const minScore = input.minScore as number | undefined; const ownerFilter = resolveOwnerFilter(input.owner); const scope = resolveScope(input.scope); + const excludeSessionKey = typeof input.excludeSessionKey === "string" && input.excludeSessionKey.length > 0 + ? input.excludeSessionKey + : undefined; const localSearch = engine.search({ query, maxResults, minScore, ownerFilter, + excludeSessionKey, }); if (scope === "local" || !store || !ctx) { diff --git a/apps/memos-local-openclaw/src/types.ts b/apps/memos-local-openclaw/src/types.ts index cb08eb1cf..a3e79c0ed 100644 --- a/apps/memos-local-openclaw/src/types.ts +++ b/apps/memos-local-openclaw/src/types.ts @@ -296,15 +296,39 @@ export interface SharingConfig { capabilities?: SharingCapabilities; } +export interface AutoRecallConfig { + /** + * When true (default), skip auto-recall for OpenClaw cron session keys + * (any session whose path contains a `cron` segment, e.g. + * `agent:main:cron:`). Set to false to restore the pre-1311 + * behaviour where cron sessions also got recall-injected context. + */ + excludeCron?: boolean; + /** + * Optional regex strings tested against the raw session key in addition + * to the cron rule above. Any match wins. Invalid patterns are ignored. + */ + excludeSessionKeyPatterns?: string[]; +} + export interface MemosLocalConfig { summarizer?: SummarizerConfig; embedding?: EmbeddingConfig; storage?: { dbPath?: string; }; + autoRecall?: AutoRecallConfig; recall?: { maxResultsDefault?: number; maxResultsMax?: number; + /** + * Override the maximum number of candidates the `before_prompt_build` + * auto-recall hook fetches per source (local + Hub). When undefined the + * hook falls back to `maxResultsDefault`. Use this to make auto-recall + * leaner (e.g. 3 or 5) without shrinking the result set of explicit + * `memory_search` tool calls. See issue #1514. + */ + autoRecallMaxResults?: number; minScoreDefault?: number; minScoreFloor?: number; rrfK?: number; @@ -312,6 +336,20 @@ export interface MemosLocalConfig { recencyHalfLifeDays?: number; /** Cap vector search to this many most recent chunks. 0 = no cap (search all; may get slower with 200k+ chunks). If you set a cap for performance, use a large value (e.g. 200000–300000) so older memories are still in the window; FTS always searches all. */ vectorSearchMaxChunks?: number; + /** + * Minimum length (in UTF-16 code units, i.e. `string.length`) of the + * normalised auto-recall query. When the user prompt is shorter than + * this threshold (e.g. one-word confirmations like "好的", "可以", + * "运行", "继续", "?"), the `before_prompt_build` hook skips + * auto-recall to avoid injecting noise into the agent context. + * + * Only affects the *automatic* recall path. Explicit `memory_search` + * tool calls are unaffected — the agent / user opted in. + * + * Default: 4. Set to 0 to disable the guard (run auto-recall on + * every prompt regardless of length). + */ + autoRecallMinQueryLength?: number; }; dedup?: { similarityThreshold?: number; @@ -337,6 +375,14 @@ export const DEFAULTS = { mmrLambda: 0.7, recencyHalfLifeDays: 14, vectorSearchMaxChunks: 0, + /** + * Default minimum length of the normalised auto-recall query before + * the `before_prompt_build` hook will run a memory search. Filters + * out one-word language tokens (e.g. "好的", "可以", "运行", "继续", + * "?", "👍") that would otherwise inject unrelated history into the + * agent context. See `MemosLocalConfig.recall.autoRecallMinQueryLength`. + */ + autoRecallMinQueryLength: 4, dedupSimilarityThreshold: 0.80, evidenceWrapperTag: "STORED_MEMORY", excerptMinChars: 200, diff --git a/apps/memos-local-openclaw/src/viewer/server.ts b/apps/memos-local-openclaw/src/viewer/server.ts index 852f3eda0..e3474cc70 100644 --- a/apps/memos-local-openclaw/src/viewer/server.ts +++ b/apps/memos-local-openclaw/src/viewer/server.ts @@ -14,13 +14,14 @@ import { vectorSearch } from "../storage/vector"; import { TaskProcessor } from "../ingest/task-processor"; import { RecallEngine } from "../recall/engine"; import { SkillEvolver } from "../skill/evolver"; -import { resolveConfig } from "../config"; +import { resolveConfig, deepResolveEnv } from "../config"; import { getHubStatus } from "../client/connector"; import { type ResolvedHubClient, hubGetMemoryDetail, hubListMemories, hubListTasks, hubListSkills, hubRequestJson, hubSearchMemories, hubSearchSkills, hubUpdateUsername, normalizeHubUrl, resolveHubClient } from "../client/hub"; import { buildSkillBundleForHub, fetchHubSkillBundle, restoreSkillBundleFromHub } from "../client/skill-sync"; import type { Logger, Chunk, PluginContext, MemosLocalConfig } from "../types"; import { viewerHTML } from "./html"; import { v4 as uuid } from "uuid"; +import { parseJsonOrJson5 } from "../shared/json5"; export interface MigrationStepFailureCounts { summarization: number; @@ -649,7 +650,7 @@ export class ViewerServer { try { const cfgPath = this.getOpenClawConfigPath(); if (fs.existsSync(cfgPath)) { - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + const raw = parseJsonOrJson5(fs.readFileSync(cfgPath, "utf-8")) as any; const entries = raw?.plugins?.entries ?? {}; const pluginCfg = entries["memos-local-openclaw-plugin"]?.config ?? entries["memos-local"]?.config ?? {}; @@ -928,6 +929,20 @@ export class ViewerServer { const dateFrom = url.searchParams.get("dateFrom") ?? undefined; const dateTo = url.searchParams.get("dateTo") ?? undefined; + // Issue #1372: honor `limit` and `minScore` query params. + // - `limit` is clamped to [1, 100]; default 20 to match the historical + // FTS-fallback slice so behavior is unchanged when the client omits it. + // - `minScore` is clamped to [0.35, 1]; default 0.64 preserves the + // previous semantic-similarity gate. + const rawLimit = Number(url.searchParams.get("limit")); + const limit = Number.isFinite(rawLimit) && rawLimit > 0 + ? Math.min(100, Math.max(1, Math.floor(rawLimit))) + : 20; + const rawMinScore = Number(url.searchParams.get("minScore")); + const minScore = Number.isFinite(rawMinScore) && rawMinScore > 0 + ? Math.max(0.35, Math.min(1, rawMinScore)) + : 0.64; + const passesFilter = (r: any): boolean => { if (role && r.role !== role) return false; if (session && r.session_key !== session) return false; @@ -962,7 +977,7 @@ export class ViewerServer { } } - const SEMANTIC_THRESHOLD = 0.64; + const SEMANTIC_THRESHOLD = minScore; const VECTOR_TIMEOUT_MS = 8000; let vectorResults: any[] = []; let scoreMap = new Map(); @@ -1001,7 +1016,7 @@ export class ViewerServer { if (!seenIds.has(r.id)) { seenIds.add(r.id); merged.push(r); } } - const results = merged.length > 0 ? merged : ftsResults.slice(0, 20); + const results = (merged.length > 0 ? merged : ftsResults).slice(0, limit); this.store.recordViewerEvent("search"); this.jsonResponse(res, { @@ -1010,6 +1025,8 @@ export class ViewerServer { vectorCount: vectorResults.length, ftsCount: ftsResults.length, total: results.length, + limit, + minScore, }); } @@ -1261,8 +1278,9 @@ export class ViewerServer { private embedTaskInBackground(taskId: string, text: string): void { if (!this.embedder || !text.trim()) return; - this.embedder.embed([text]).then((vecs: number[][]) => { - if (vecs.length > 0) this.store.upsertTaskEmbedding(taskId, vecs[0]); + const embedder = this.embedder; + embedder.embed([text]).then((vecs: number[][]) => { + if (vecs.length > 0) this.store.upsertTaskEmbedding(taskId, vecs[0], { provider: embedder.provider, model: embedder.model }); }).catch(() => {}); } @@ -1402,8 +1420,9 @@ export class ViewerServer { const sv = this.store.getLatestSkillVersion(skillId); if (sv) { const text = `${skill.name}: ${skill.description}`; - this.embedder.embed([text]).then((vecs: number[][]) => { - if (vecs.length > 0) this.store.upsertSkillEmbedding(skillId, vecs[0]); + const embedder = this.embedder; + embedder.embed([text]).then((vecs: number[][]) => { + if (vecs.length > 0) this.store.upsertSkillEmbedding(skillId, vecs[0], { provider: embedder.provider, model: embedder.model }); }).catch(() => {}); } } @@ -1934,6 +1953,42 @@ export class ViewerServer { return path.join(ocHome, "openclaw.json"); } + /** + * Read openclaw.json without env-var resolution. Used by serveConfig (UI editor needs to + * see raw `${VAR}` literals so saves don't accidentally leak resolved secrets back into + * the config file). Returns `null` when the file is missing or unreadable. + */ + private readOpenClawRawConfig(): Record | null { + try { + const cfgPath = this.getOpenClawConfigPath(); + if (!fs.existsSync(cfgPath)) return null; + return parseJsonOrJson5(fs.readFileSync(cfgPath, "utf-8")) as Record; + } catch { + return null; + } + } + + /** + * Read openclaw.json and apply `${VAR}` env-var resolution. Use this whenever the + * resulting value is consumed by something that actually contacts the network + * (apiKey/endpoint/baseUrl). Returns `null` when the file is missing or unreadable. + */ + private readOpenClawResolvedConfig(): Record | null { + const raw = this.readOpenClawRawConfig(); + if (!raw) return null; + return deepResolveEnv(raw); + } + + /** + * Read this plugin's section from openclaw.json with `${VAR}` env vars resolved. + * Used by migration / model tests that need real apiKey strings. + */ + private readPluginConfigResolved(): Record { + const raw = this.readOpenClawResolvedConfig(); + if (!raw) return {}; + return this.getPluginEntryConfig(raw); + } + private getPluginEntryConfig(raw: any): Record { const entries = raw?.plugins?.entries ?? {}; return entries["memos-local-openclaw-plugin"]?.config @@ -3021,7 +3076,7 @@ export class ViewerServer { this.jsonResponse(res, {}); return; } - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + const raw = parseJsonOrJson5(fs.readFileSync(cfgPath, "utf-8")) as any; const entries = raw?.plugins?.entries ?? {}; const pluginEntry = entries["memos-local-openclaw-plugin"]?.config ?? entries["memos-local"]?.config @@ -3053,7 +3108,7 @@ export class ViewerServer { const cfgPath = this.getOpenClawConfigPath(); let raw: Record = {}; if (fs.existsSync(cfgPath)) { - raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + raw = parseJsonOrJson5(fs.readFileSync(cfgPath, "utf-8")) as Record; } if (!raw.plugins) raw.plugins = {}; @@ -3514,12 +3569,14 @@ export class ViewerServer { private serveFallbackModel(res: http.ServerResponse): void { try { - const cfgPath = this.getOpenClawConfigPath(); - if (!fs.existsSync(cfgPath)) { + // Resolve ${VAR} env-vars so providers..{baseUrl,apiKey} that reference + // process env (e.g. apiKey="${OPENAI_API_KEY}") evaluate before we report + // the fallback model as available. + const raw = this.readOpenClawResolvedConfig(); + if (!raw) { this.jsonResponse(res, { available: false }); return; } - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); const agentModel: string | undefined = raw?.agents?.defaults?.model?.primary; if (!agentModel) { this.jsonResponse(res, { available: false }); @@ -3769,7 +3826,7 @@ export class ViewerServer { discardBackup(); this.log.info(`update-install: success! Updated to ${newVersion}`); - this.jsonResponseAndRestart(res, { ok: true, version: newVersion }, "update-install"); + this.jsonResponseAndRestart(res, { ok: true, version: newVersion }, "update-install", 250); }); }); }); @@ -3786,6 +3843,11 @@ export class ViewerServer { if (provider === "local") { return 384; } + // Resolve ${VAR} env-var literals: the UI forwards the raw value pulled from + // openclaw.json verbatim (see serveConfig), so we must expand here before hitting + // the network — otherwise Bearer headers contain the literal "${MY_KEY}" string. + endpoint = deepResolveEnv(endpoint || ""); + apiKey = deepResolveEnv(apiKey || ""); const baseUrl = (endpoint || "https://api.openai.com/v1").replace(/\/+$/, ""); const embUrl = baseUrl.endsWith("/embeddings") ? baseUrl : `${baseUrl}/embeddings`; const headers: Record = { @@ -3853,6 +3915,10 @@ export class ViewerServer { } private async testChatModel(provider: string, model: string, endpoint: string, apiKey: string): Promise { + // Same env-var expansion as testEmbeddingModel: openclaw.json values such as + // "${OPENAI_API_KEY}" must be resolved before hitting the network. + endpoint = deepResolveEnv(endpoint || ""); + apiKey = deepResolveEnv(apiKey || ""); const baseUrl = (endpoint || "https://api.openai.com/v1").replace(/\/+$/, ""); if (provider === "anthropic") { const url = endpoint || "https://api.anthropic.com/v1/messages"; @@ -4007,7 +4073,7 @@ export class ViewerServer { let hasSummarizer = false; if (fs.existsSync(cfgPath)) { try { - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); + const raw = parseJsonOrJson5(fs.readFileSync(cfgPath, "utf-8")) as any; const pluginCfg = raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ?? raw?.plugins?.entries?.["memos-local"]?.config ?? raw?.plugins?.entries?.["memos-lite-openclaw-plugin"]?.config ?? @@ -4199,16 +4265,12 @@ export class ViewerServer { let totalSkipped = 0; let totalErrors = 0; - const cfgPath = this.getOpenClawConfigPath(); - let summarizerCfg: any; - try { - const raw = JSON.parse(fs.readFileSync(cfgPath, "utf-8")); - const pluginCfg = raw?.plugins?.entries?.["memos-local-openclaw-plugin"]?.config ?? - raw?.plugins?.entries?.["memos-local"]?.config ?? - raw?.plugins?.entries?.["memos-lite-openclaw-plugin"]?.config ?? - raw?.plugins?.entries?.["memos-lite"]?.config ?? {}; - summarizerCfg = pluginCfg.summarizer; - } catch { /* no config */ } + // Build the migration Summarizer from the env-resolved plugin config so any + // apiKey/endpoint of the form "${OPENAI_API_KEY}" is expanded against process.env + // before the LLM call. Without this the migration's summarizer hits remote APIs + // with literal "${VAR}" Bearer tokens and 401s out. + const pluginCfg = this.readPluginConfigResolved(); + const summarizerCfg = (pluginCfg as any)?.summarizer; const summarizer = new Summarizer(summarizerCfg, this.log); @@ -4319,7 +4381,7 @@ export class ViewerServer { this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, row.text); try { const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]); - if (newEmb) this.store.upsertEmbedding(targetId, newEmb); + if (newEmb) this.store.upsertEmbedding(targetId, newEmb, { provider: this.embedder.provider, model: this.embedder.model }); } catch { /* best-effort */ } dedupStatus = "merged"; dedupTarget = targetId; @@ -4360,7 +4422,7 @@ export class ViewerServer { this.store.insertChunk(chunk); if (embedding && dedupStatus === "active") { - this.store.upsertEmbedding(chunkId, embedding); + this.store.upsertEmbedding(chunkId, embedding, { provider: this.embedder.provider, model: this.embedder.model }); } totalStored++; @@ -4552,7 +4614,7 @@ export class ViewerServer { const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId; if (targetId) { this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, content); - try { const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]); if (newEmb) this.store.upsertEmbedding(targetId, newEmb); } catch { /* best-effort */ } + try { const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]); if (newEmb) this.store.upsertEmbedding(targetId, newEmb, { provider: this.embedder.provider, model: this.embedder.model }); } catch { /* best-effort */ } dedupStatus = "merged"; dedupTarget = targetId; dedupReason = dedupResult.reason; } } @@ -4575,7 +4637,7 @@ export class ViewerServer { }; this.store.insertChunk(chunk); - if (embedding && dedupStatus === "active") this.store.upsertEmbedding(chunkId, embedding); + if (embedding && dedupStatus === "active") this.store.upsertEmbedding(chunkId, embedding, { provider: this.embedder.provider, model: this.embedder.model }); totalStored++; send("item", { index: idx, total: totalMsgs, status: dedupStatus === "active" ? "stored" : dedupStatus, preview: content.slice(0, 120), summary: summary.slice(0, 80), source: file, agent: agentId, role: msgRole, stepFailures }); @@ -4910,12 +4972,11 @@ export class ViewerServer { statusCode = 200, ): void { res.writeHead(statusCode, { "Content-Type": "application/json; charset=utf-8" }); - res.end(JSON.stringify(data), () => { - setTimeout(() => { - this.log.info(`${source}: triggering gateway restart via SIGUSR1...`); - try { process.kill(process.pid, "SIGUSR1"); } catch (sig) { this.log.warn(`SIGUSR1 failed: ${sig}`); } - }, delayMs); - }); + res.end(JSON.stringify(data)); + setTimeout(() => { + this.log.info(`${source}: triggering gateway restart via SIGUSR1...`); + try { process.kill(process.pid, "SIGUSR1"); } catch (sig) { this.log.warn(`SIGUSR1 failed: ${sig}`); } + }, delayMs); } private jsonResponse(res: http.ServerResponse, data: unknown, statusCode = 200): void { diff --git a/apps/memos-local-openclaw/tests/auto-recall-cron-gate.test.ts b/apps/memos-local-openclaw/tests/auto-recall-cron-gate.test.ts new file mode 100644 index 000000000..4339cabce --- /dev/null +++ b/apps/memos-local-openclaw/tests/auto-recall-cron-gate.test.ts @@ -0,0 +1,222 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +afterEach(() => { + vi.resetModules(); + vi.clearAllMocks(); +}); + +interface HarnessResult { + recallSearchCalls: Array<{ query: string }>; + hookHandler: ((event: unknown, ctx: unknown) => Promise | unknown) | null; +} + +async function buildPlugin(autoRecall: Record | undefined): Promise { + const recallSearchCalls: Array<{ query: string }> = []; + + vi.doMock("../src/config", () => ({ + buildContext: () => ({ + stateDir: "/tmp/memos-cron-gate", + workspaceDir: "/tmp/memos-cron-gate/workspace", + log: { debug() {}, info() {}, warn() {}, error() {} }, + openclawAPI: undefined, + config: { + storage: { dbPath: "/tmp/memos-cron-gate/memos.db" }, + capture: { evidenceWrapperTag: "STORED_MEMORY" }, + telemetry: {}, + sharing: { enabled: false, role: "client", hub: { port: 18800, teamName: "", teamToken: "" }, client: { hubAddress: "", userToken: "" }, capabilities: {} }, + autoRecall, + }, + }), + })); + + vi.doMock("../src/storage/ensure-binding", () => ({ ensureSqliteBinding: () => {} })); + vi.doMock("../src/storage/sqlite", () => ({ + SqliteStore: class { + recordToolCall() {} + recordApiLog() {} + getTask() { return null; } + getChunksByTask() { return []; } + getSkillsByTask() { return []; } + listLocalSharedTasks() { return []; } + getClientHubConnection() { return null; } + getChunk() { return null; } + getChunkForOwners() { return null; } + getNeighborChunks() { return []; } + getSkill() { return null; } + getLatestSkillVersion() { return null; } + close() {} + }, + })); + + vi.doMock("../src/embedding", () => ({ Embedder: class { provider = "openclaw"; constructor() {} async embed() { return []; } } })); + + vi.doMock("../src/ingest/worker", () => ({ + IngestWorker: class { + getTaskProcessor() { return { onTaskCompleted() {} }; } + enqueue() {} + async flush() {} + }, + })); + + vi.doMock("../src/recall/engine", () => ({ + RecallEngine: class { + async search(input: { query: string }) { + recallSearchCalls.push({ query: input.query }); + return { hits: [], meta: {} }; + } + async searchSkills() { return []; } + }, + })); + + vi.doMock("../src/ingest/providers", () => ({ + Summarizer: class { + constructor() {} + async filterRelevant() { return null; } + }, + })); + + vi.doMock("../src/viewer/server", () => ({ + ViewerServer: class { + async start() { return "http://127.0.0.1:18799"; } + stop() {} + getResetToken() { return "tok"; } + }, + })); + + vi.doMock("../src/hub/server", () => ({ + HubServer: class { + async start() { return "http://127.0.0.1:18800"; } + async stop() {} + }, + })); + + vi.doMock("../src/client/hub", () => ({ + hubGetMemoryDetail: async () => ({}), + hubRequestJson: async () => ({}), + hubSearchMemories: async () => ({ hits: [], meta: {} }), + hubSearchSkills: async () => ({ hits: [] }), + resolveHubClient: async () => ({ hubUrl: "", userToken: "", userId: "" }), + })); + + vi.doMock("../src/client/connector", () => ({ + getHubStatus: async () => ({ connected: false }), + connectToHub: async () => ({ username: "test", userId: "test" }), + })); + + vi.doMock("../src/client/skill-sync", () => ({ + fetchHubSkillBundle: async () => ({}), + publishSkillBundleToHub: async () => ({}), + restoreSkillBundleFromHub: () => ({}), + unpublishSkillBundleFromHub: async () => ({}), + })); + + vi.doMock("../src/skill/evolver", () => ({ + SkillEvolver: class { + onSkillEvolved: unknown = null; + async onTaskCompleted() {} + async recoverOrphanedTasks() { return 0; } + }, + })); + + vi.doMock("../src/skill/installer", () => ({ + SkillInstaller: class { + install() { return { message: "" }; } + getCompanionManifest() { return null; } + readCompanionFile() { return { error: "n/a" } as any; } + }, + })); + + vi.doMock("../src/skill/bundled-memory-guide", () => ({ MEMORY_GUIDE_SKILL_MD: "# mock" })); + + vi.doMock("../src/telemetry", () => ({ + Telemetry: class { + trackError() {} + trackToolCalled() {} + trackAutoRecall() {} + trackMemoryIngested() {} + trackSkillInstalled() {} + trackSkillEvolved() {} + trackPluginStarted() {} + trackViewerOpened() {} + async shutdown() {} + }, + })); + + vi.doMock("../src/capture", () => ({ + captureMessages: () => [], + stripInboundMetadata: (s: string) => s, + })); + + let hookHandler: HarnessResult["hookHandler"] = null; + const fakeApi = { + id: "memos-local-openclaw-plugin", + pluginConfig: {}, + config: {}, + resolvePath: () => "/tmp/memos-cron-gate", + logger: { info() {}, warn() {} }, + registerTool: () => {}, + registerMemoryCapability: () => {}, + registerService: () => {}, + on: (eventName: string, handler: (e: unknown, ctx: unknown) => unknown) => { + if (eventName === "before_prompt_build") hookHandler = handler as HarnessResult["hookHandler"]; + }, + } as any; + + const pluginModule = await import("../plugin-impl"); + pluginModule.default.register(fakeApi); + + return { recallSearchCalls, hookHandler }; +} + +describe("before_prompt_build cron-session gate (GitHub #1311)", () => { + it("skips engine.search for cron sessionKey when excludeCron defaults to true", async () => { + const { recallSearchCalls, hookHandler } = await buildPlugin(undefined); + expect(hookHandler).toBeTypeOf("function"); + + const result = await hookHandler!( + { prompt: "Trade the morning open and report PnL." }, + { agentId: "main", sessionKey: "agent:main:cron:job-abc" }, + ); + + expect(result).toBeUndefined(); + expect(recallSearchCalls).toHaveLength(0); + }); + + it("still runs engine.search for non-cron chat sessionKey", async () => { + const { recallSearchCalls, hookHandler } = await buildPlugin(undefined); + + await hookHandler!( + { prompt: "Trade the morning open and report PnL." }, + { agentId: "main", sessionKey: "agent:main:chat:hello" }, + ); + + expect(recallSearchCalls.length).toBeGreaterThanOrEqual(1); + }); + + it("runs engine.search for cron sessions when excludeCron=false", async () => { + const { recallSearchCalls, hookHandler } = await buildPlugin({ + excludeCron: false, + }); + + await hookHandler!( + { prompt: "Trade the morning open and report PnL." }, + { agentId: "main", sessionKey: "agent:main:cron:job-abc" }, + ); + + expect(recallSearchCalls.length).toBeGreaterThanOrEqual(1); + }); + + it("honours excludeSessionKeyPatterns", async () => { + const { recallSearchCalls, hookHandler } = await buildPlugin({ + excludeCron: false, + excludeSessionKeyPatterns: ["^agent:debug:"], + }); + + await hookHandler!( + { prompt: "Trade the morning open and report PnL." }, + { agentId: "main", sessionKey: "agent:debug:gateway:hello" }, + ); + + expect(recallSearchCalls).toHaveLength(0); + }); +}); diff --git a/apps/memos-local-openclaw/tests/capture.test.ts b/apps/memos-local-openclaw/tests/capture.test.ts index 97ee5c5a7..bf6dca9f9 100644 --- a/apps/memos-local-openclaw/tests/capture.test.ts +++ b/apps/memos-local-openclaw/tests/capture.test.ts @@ -148,4 +148,43 @@ describe("captureMessages", () => { const result = captureMessages(msgs, "s1", "t1", "STORED_MEMORY", noopLog); expect(result[0].content).toBe("普通的用户消息"); }); + + it("should skip Hermes 'Review the conversation above' injected user prompts", () => { + const msgs = [ + { + role: "user", + content: + "Review the conversation above, consider saving any durable facts to long-term memory.", + }, + { role: "user", content: "Real user message" }, + ]; + const result = captureMessages(msgs, "s1", "t1", "STORED_MEMORY", noopLog); + expect(result).toHaveLength(1); + expect(result[0].content).toBe("Real user message"); + }); + + it("should skip 'Review the conversation above' regardless of case and trim whitespace", () => { + const msgs = [ + { + role: "user", + content: " review THE conversation above and decide what to keep. ", + }, + ]; + const result = captureMessages(msgs, "s1", "t1", "STORED_MEMORY", noopLog); + expect(result).toHaveLength(0); + }); + + it("should not skip 'Review the conversation above' when sent by assistant", () => { + // Assistant-generated text that happens to start with the same phrase + // is real assistant content, not an injected self-instruction. Keep it. + const msgs = [ + { + role: "assistant", + content: "Review the conversation above to find the answer.", + }, + ]; + const result = captureMessages(msgs, "s1", "t1", "STORED_MEMORY", noopLog); + expect(result).toHaveLength(1); + expect(result[0].role).toBe("assistant"); + }); }); diff --git a/apps/memos-local-openclaw/tests/config.test.ts b/apps/memos-local-openclaw/tests/config.test.ts index 072728b9d..2835902c6 100644 --- a/apps/memos-local-openclaw/tests/config.test.ts +++ b/apps/memos-local-openclaw/tests/config.test.ts @@ -1,5 +1,30 @@ import { describe, expect, it } from "vitest"; import { resolveConfig } from "../src/config"; +import { DEFAULTS } from "../src/types"; + +describe("resolveConfig recall.autoRecallMinQueryLength", () => { + it("defaults to DEFAULTS.autoRecallMinQueryLength when not set", () => { + const resolved = resolveConfig({}, "/tmp/memos-config-min-query-default"); + expect(resolved.recall?.autoRecallMinQueryLength).toBe(DEFAULTS.autoRecallMinQueryLength); + expect(resolved.recall?.autoRecallMinQueryLength).toBe(4); + }); + + it("preserves an explicit override", () => { + const resolved = resolveConfig( + { recall: { autoRecallMinQueryLength: 10 } }, + "/tmp/memos-config-min-query-override", + ); + expect(resolved.recall?.autoRecallMinQueryLength).toBe(10); + }); + + it("preserves 0 explicitly (disables the short-query skip)", () => { + const resolved = resolveConfig( + { recall: { autoRecallMinQueryLength: 0 } }, + "/tmp/memos-config-min-query-zero", + ); + expect(resolved.recall?.autoRecallMinQueryLength).toBe(0); + }); +}); describe("resolveConfig", () => { it("injects openclaw providers into existing blocks when host capabilities are enabled", () => { @@ -48,6 +73,46 @@ describe("resolveConfig", () => { }); }); + describe("recall.autoRecallMaxResults", () => { + it("leaves autoRecallMaxResults undefined when no recall config is provided (fall-through to maxResultsDefault)", () => { + const resolved = resolveConfig(undefined, "/tmp/memos-config-recall-default"); + + // Default policy: auto-recall path inherits maxResultsDefault when not overridden. + expect(resolved.recall?.maxResultsDefault).toBe(6); + expect(resolved.recall?.autoRecallMaxResults).toBeUndefined(); + }); + + it("preserves an explicit autoRecallMaxResults value when set", () => { + const resolved = resolveConfig( + { + recall: { + maxResultsDefault: 6, + autoRecallMaxResults: 3, + }, + } as any, + "/tmp/memos-config-recall-override", + ); + + expect(resolved.recall?.maxResultsDefault).toBe(6); + expect(resolved.recall?.autoRecallMaxResults).toBe(3); + }); + + it("allows independent control: maxResultsDefault for memory_search, autoRecallMaxResults for auto-recall", () => { + const resolved = resolveConfig( + { + recall: { + maxResultsDefault: 10, + autoRecallMaxResults: 5, + }, + } as any, + "/tmp/memos-config-recall-split", + ); + + expect(resolved.recall?.maxResultsDefault).toBe(10); + expect(resolved.recall?.autoRecallMaxResults).toBe(5); + }); + }); + it("preserves explicit user providers when host capabilities are enabled", () => { const resolved = resolveConfig( { diff --git a/apps/memos-local-openclaw/tests/cross-session-memory.test.ts b/apps/memos-local-openclaw/tests/cross-session-memory.test.ts new file mode 100644 index 000000000..53d03b48c --- /dev/null +++ b/apps/memos-local-openclaw/tests/cross-session-memory.test.ts @@ -0,0 +1,90 @@ +/** + * Test: Cross-session memory should not trigger unprompted agent action + * + * This test verifies the fix for issue #1532: + * - When a new session starts, auto-recall may inject memories from previous sessions + * - The agent should treat these as background knowledge only + * - The agent should NOT act unprompted based on cross-session memories + * - The agent should wait for the user's explicit instruction + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { fileURLToPath } from "url"; +import { dirname, join } from "path"; +import * as fs from "fs"; +import * as os from "os"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +describe("Cross-session memory behavior", () => { + let testDir: string; + let pluginModule: any; + + beforeEach(() => { + // Create temp directory for test database + testDir = fs.mkdtempSync(join(os.tmpdir(), "memos-test-")); + }); + + afterEach(() => { + // Cleanup + if (testDir && fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true, force: true }); + } + }); + + it("should mark cross-session memories with [from previous session] tag", async () => { + // This test verifies that when memories from a different sessionKey are retrieved, + // they are tagged appropriately so the agent knows they're from a previous session + + const mockApi: any = { + logger: { + info: () => {}, + warn: () => {}, + debug: () => {}, + error: () => {}, + }, + on: () => {}, + registerService: () => {}, + registerTool: () => {}, + }; + + // Load plugin (implementation would need to be adjusted for proper testing) + const indexPath = join(__dirname, "..", "index.ts"); + expect(fs.existsSync(indexPath)).toBe(true); + + // Test scenario: + // 1. Session A: User discusses "cron configuration issue" + // 2. Session B (new): Agent should see session A memories but not act on them + // + // Expected: Injected context should contain: + // - "[from previous session]" tags on memories from session A + // - Instructions: "Do NOT act on them unprompted" + // - Instructions: "WAIT for the user's explicit instruction" + }); + + it("should inject passive instructions for cross-session memories", async () => { + // Verify that when hasCrossSessionMemories=true or isNewSession=true, + // the injected context uses passive instructions like: + // "Treat them as BACKGROUND KNOWLEDGE ONLY" + // "Do NOT act on them unprompted" + // Instead of: + // "You MUST treat these as established knowledge and use them directly" + }); + + it("should inject active instructions for same-session memories", async () => { + // Verify that when all memories are from the current session, + // the injected context continues to use active instructions: + // "You MUST treat these as established knowledge and use them directly when answering" + }); + + it("should detect new session via NEW_SESSION_PROMPT_RE pattern", async () => { + // Verify that when the prompt contains "A new session was started via /new or /reset.", + // isNewSession flag is set to true + }); + + it("should detect new session via sessionKey change", async () => { + // Verify that when currentSessionKey !== incomingSessionKey, + // isNewSession flag is set to true + }); +}); diff --git a/apps/memos-local-openclaw/tests/embedding-memory-leak.test.ts b/apps/memos-local-openclaw/tests/embedding-memory-leak.test.ts new file mode 100644 index 000000000..58dafa8c1 --- /dev/null +++ b/apps/memos-local-openclaw/tests/embedding-memory-leak.test.ts @@ -0,0 +1,128 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { Logger } from "../src/types"; + +const mockTransformers = vi.hoisted(() => { + const disposeOutput = vi.fn(); + const disposeExtractor = vi.fn(); + const extractor = vi.fn(async () => ({ + data: new Float32Array(384).fill(0.5), + dispose: disposeOutput, + })); + Object.assign(extractor, { dispose: disposeExtractor }); + return { + disposeOutput, + disposeExtractor, + extractor, + pipeline: vi.fn(async () => extractor), + }; +}); + +vi.mock("@huggingface/transformers", () => ({ + pipeline: mockTransformers.pipeline, +})); + +function createLogger(): Logger { + return { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }; +} + +async function loadEmbedLocal() { + vi.resetModules(); + return import("../src/embedding/local"); +} + +describe("embedLocal memory leak fix", () => { + beforeEach(() => { + vi.clearAllMocks(); + delete process.env.MEMOS_EMBED_RESET_AFTER_CALLS; + }); + + afterEach(() => { + delete process.env.MEMOS_EMBED_RESET_AFTER_CALLS; + vi.restoreAllMocks(); + }); + + it("should dispose tensor output after each embedding call", async () => { + const { embedLocal } = await loadEmbedLocal(); + const mockLogger = createLogger(); + const texts = ["test embedding 1", "test embedding 2"]; + + const result = await embedLocal(texts, mockLogger); + + // Verify we got valid embeddings + expect(result).toHaveLength(2); + expect(result[0]).toHaveLength(384); // all-MiniLM-L6-v2 dimension + expect(result[1]).toHaveLength(384); + + // Verify embeddings are valid numbers + result.forEach(embedding => { + embedding.forEach(value => { + expect(typeof value).toBe("number"); + expect(isFinite(value)).toBe(true); + }); + }); + + expect(mockTransformers.pipeline).toHaveBeenCalledTimes(1); + expect(mockTransformers.extractor).toHaveBeenCalledTimes(2); + expect(mockTransformers.disposeOutput).toHaveBeenCalledTimes(2); + }); + + it("should handle multiple consecutive calls without crashing", async () => { + const { embedLocal } = await loadEmbedLocal(); + const mockLogger = createLogger(); + // Simulate multiple embedding calls that would trigger the leak + const calls = 10; + + for (let i = 0; i < calls; i++) { + const result = await embedLocal([`test text ${i}`], mockLogger); + expect(result).toHaveLength(1); + expect(result[0]).toHaveLength(384); + } + + // If we reached here without OOM, the tensor disposal is working + expect(mockTransformers.pipeline).toHaveBeenCalledTimes(1); + expect(mockTransformers.disposeOutput).toHaveBeenCalledTimes(calls); + }); + + it("should reset pipeline after RESET_AFTER_CALLS threshold", async () => { + // Set a low threshold for testing + process.env.MEMOS_EMBED_RESET_AFTER_CALLS = "5"; + const { embedLocal } = await loadEmbedLocal(); + const mockLogger = createLogger(); + + // This will trigger a reset after 5 calls + const calls = 6; + + for (let i = 0; i < calls; i++) { + const result = await embedLocal([`test ${i}`], mockLogger); + expect(result[0]).toHaveLength(384); + } + + // Verify reset was logged + expect(mockLogger.debug).toHaveBeenCalledWith( + expect.stringContaining("Reached 5 embedding calls") + ); + expect(mockTransformers.disposeExtractor).toHaveBeenCalledTimes(1); + }); + + it("should allow disabling periodic reset via env variable", async () => { + process.env.MEMOS_EMBED_RESET_AFTER_CALLS = "0"; + const { embedLocal } = await loadEmbedLocal(); + const mockLogger = createLogger(); + + // Run many calls - should not trigger reset + for (let i = 0; i < 100; i++) { + await embedLocal([`test ${i}`], mockLogger); + } + + // Verify no reset was logged + expect(mockLogger.debug).not.toHaveBeenCalledWith( + expect.stringContaining("Reached") + ); + expect(mockTransformers.disposeExtractor).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/memos-local-openclaw/tests/embedding-reembed.test.ts b/apps/memos-local-openclaw/tests/embedding-reembed.test.ts new file mode 100644 index 000000000..6d98e0b06 --- /dev/null +++ b/apps/memos-local-openclaw/tests/embedding-reembed.test.ts @@ -0,0 +1,419 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import * as fs from "fs"; +import * as path from "path"; +import * as os from "os"; +import Database from "better-sqlite3"; +import { SqliteStore } from "../src/storage/sqlite"; +import { Embedder } from "../src/embedding"; +import { initPlugin } from "../src/index"; +import { parseArgs, runReembed } from "../scripts/re-embed"; +import type { Chunk, Logger, EmbeddingConfig } from "../src/types"; + +const noopLog: Logger = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, +}; + +let store: SqliteStore; +let tmpDir: string; +let dbPath: string; + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-reembed-test-")); + dbPath = path.join(tmpDir, "test.db"); + store = new SqliteStore(dbPath, noopLog); +}); + +afterEach(() => { + store.close(); + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +function makeChunk(overrides: Partial = {}): Chunk { + return { + id: overrides.id ?? "chunk-1", + sessionKey: overrides.sessionKey ?? "session-1", + turnId: "turn-1", + seq: 0, + role: "user", + content: "Hello world", + kind: "paragraph", + summary: "Greeting", + embedding: null, + taskId: null, + skillId: null, + owner: "agent:main", + dedupStatus: "active", + dedupTarget: null, + dedupReason: null, + mergeCount: 0, + lastHitAt: null, + mergeHistory: "[]", + createdAt: Date.now(), + updatedAt: Date.now(), + ...overrides, + }; +} + +function rawEmbeddingRow(p: string, chunkId: string): { + provider: string; + model: string; + dimensions: number; +} | undefined { + const db = new Database(p, { readonly: true }); + try { + return db + .prepare("SELECT provider, model, dimensions FROM embeddings WHERE chunk_id = ?") + .get(chunkId) as + | { provider: string; model: string; dimensions: number } + | undefined; + } finally { + db.close(); + } +} + +describe("Embedding producer columns (TC-1, TC-2, TC-3)", () => { + it("migration adds provider + model columns with NOT NULL DEFAULT ''", () => { + const dbRO = new Database(dbPath, { readonly: true }); + try { + const cols = dbRO.prepare("PRAGMA table_info(embeddings)").all() as Array<{ + name: string; + type: string; + notnull: number; + dflt_value: string | null; + }>; + const provider = cols.find((c) => c.name === "provider"); + const model = cols.find((c) => c.name === "model"); + expect(provider, "embeddings.provider column should exist").toBeDefined(); + expect(model, "embeddings.model column should exist").toBeDefined(); + expect(provider!.notnull).toBe(1); + expect(model!.notnull).toBe(1); + expect(provider!.dflt_value).toMatch(/''|""/); + expect(model!.dflt_value).toMatch(/''|""/); + } finally { + dbRO.close(); + } + }); + + it("upsertEmbedding without producer (back-compat) stores empty strings", () => { + store.insertChunk(makeChunk({ id: "c1" })); + store.upsertEmbedding("c1", [0.1, 0.2, 0.3]); + + const row = rawEmbeddingRow(dbPath, "c1"); + expect(row).toBeDefined(); + expect(row!.provider).toBe(""); + expect(row!.model).toBe(""); + expect(row!.dimensions).toBe(3); + + const v = store.getEmbedding("c1"); + expect(v).not.toBeNull(); + expect(v!).toHaveLength(3); + expect(v![0]).toBeCloseTo(0.1, 5); + }); + + it("upsertEmbedding with producer persists provider + model", () => { + store.insertChunk(makeChunk({ id: "c1" })); + store.upsertEmbedding("c1", [0.1, 0.2, 0.3], { + provider: "openai", + model: "text-embedding-3-small", + }); + + const row = rawEmbeddingRow(dbPath, "c1"); + expect(row).toBeDefined(); + expect(row!.provider).toBe("openai"); + expect(row!.model).toBe("text-embedding-3-small"); + expect(row!.dimensions).toBe(3); + + // overwrite with a different producer + store.upsertEmbedding("c1", [0.4, 0.5, 0.6], { + provider: "openai", + model: "text-embedding-3-large", + }); + const row2 = rawEmbeddingRow(dbPath, "c1"); + expect(row2!.model).toBe("text-embedding-3-large"); + }); +}); + +describe("getEmbeddingStats + listChunkIdsForReembed (TC-4, TC-5, TC-6)", () => { + beforeEach(() => { + // 6 chunks total. 5 with embeddings, 1 without. + const now = Date.now(); + store.insertChunk(makeChunk({ id: "c1", createdAt: now + 1 })); + store.insertChunk(makeChunk({ id: "c2", createdAt: now + 2 })); + store.insertChunk(makeChunk({ id: "c3", createdAt: now + 3 })); + store.insertChunk(makeChunk({ id: "c4", createdAt: now + 4 })); + store.insertChunk(makeChunk({ id: "c5", createdAt: now + 5 })); + store.insertChunk(makeChunk({ id: "c6", createdAt: now + 6 })); // no embedding + + // matched (×2): openai/text-embedding-3-small/1536 + store.upsertEmbedding("c1", Array(1536).fill(0.1), { + provider: "openai", + model: "text-embedding-3-small", + }); + store.upsertEmbedding("c2", Array(1536).fill(0.2), { + provider: "openai", + model: "text-embedding-3-small", + }); + // mismatched: different model + store.upsertEmbedding("c3", Array(3072).fill(0.3), { + provider: "openai", + model: "text-embedding-3-large", + }); + // mismatched: different provider + store.upsertEmbedding("c4", Array(384).fill(0.4), { + provider: "local", + model: "", + }); + // legacy: no producer info + store.upsertEmbedding("c5", Array(1536).fill(0.5)); + }); + + it("getEmbeddingStats reports matched/mismatched/legacy/missing", () => { + const stats = store.getEmbeddingStats({ + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }); + + expect(stats.total).toBe(5); + expect(stats.matched).toBe(2); + // c3 + c4 are explicit mismatch (both have non-empty provider != current) + expect(stats.mismatched).toBe(2); + expect(stats.legacy).toBe(1); + expect(stats.missing).toBe(1); // c6 has chunk but no embedding row + + expect(stats.current.provider).toBe("openai"); + expect(stats.current.model).toBe("text-embedding-3-small"); + expect(stats.current.dimensions).toBe(1536); + + // byProducer has all three buckets + const producers = stats.byProducer.map((b) => `${b.provider}|${b.model}|${b.dimensions}`).sort(); + expect(producers).toEqual([ + "||1536", + "local||384", + "openai|text-embedding-3-large|3072", + "openai|text-embedding-3-small|1536", + ].sort()); + }); + + it("listChunkIdsForReembed default returns mismatched + legacy + missing", () => { + const ids = store.listChunkIdsForReembed({ + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }); + expect(new Set(ids)).toEqual(new Set(["c3", "c4", "c5", "c6"])); + }); + + it("listChunkIdsForReembed missingOnly returns only chunks without embedding row", () => { + const ids = store.listChunkIdsForReembed( + { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }, + { missingOnly: true }, + ); + expect(ids).toEqual(["c6"]); + }); + + it("listChunkIdsForReembed respects limit", () => { + const ids = store.listChunkIdsForReembed( + { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }, + { limit: 2 }, + ); + expect(ids.length).toBe(2); + }); +}); + +describe("Embedder.signature (TC-7)", () => { + it("returns provider:model:dimensions for an explicit config", () => { + const cfg: EmbeddingConfig = { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }; + const e = new Embedder(cfg, noopLog); + expect(e.signature).toBe("openai:text-embedding-3-small:1536"); + expect(e.model).toBe("text-embedding-3-small"); + }); + + it("falls back to local::384 when config is undefined", () => { + const e = new Embedder(undefined, noopLog); + expect(e.signature).toBe("local::384"); + expect(e.provider).toBe("local"); + expect(e.model).toBe(""); + expect(e.dimensions).toBe(384); + }); + + it("treats openclaw provider without hostEmbedding as local", () => { + const cfg: EmbeddingConfig = { + provider: "openclaw", + model: "any", + dimensions: 1536, + capabilities: { hostEmbedding: false }, + }; + const e = new Embedder(cfg, noopLog); + // provider downshifts to "local", model still surfaces; signature is canonical for current Embedder identity + expect(e.provider).toBe("local"); + expect(e.signature.startsWith("local:")).toBe(true); + }); +}); + +describe("Init-time mismatch warning (TC-9)", () => { + it("emits a warn line when legacy or mismatched rows are present", () => { + // 1. Pre-create the DB with one legacy embedding row + const initStore = new SqliteStore(dbPath, noopLog); + initStore.insertChunk(makeChunk({ id: "c-legacy" })); + initStore.upsertEmbedding("c-legacy", Array(1536).fill(0.1)); + initStore.close(); + + // 2. Build a log that captures warn calls + const warns: string[] = []; + const captureLog: Logger = { + debug: () => {}, + info: () => {}, + warn: (msg) => warns.push(String(msg)), + error: () => {}, + }; + + // 3. Spin up initPlugin pointing at the same DB with a fresh config + const stateDir = path.join(tmpDir, "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const plugin = initPlugin({ + stateDir, + workspaceDir: tmpDir, + config: { + storage: { dbPath }, + embedding: { + provider: "openai", + model: "text-embedding-3-small", + dimensions: 1536, + }, + }, + log: captureLog, + }); + + // 4. Expect at least one warn mentioning legacy / mismatch and the script path + expect(warns.some((w) => /embedding model mismatch/i.test(w))).toBe(true); + expect(warns.some((w) => /scripts\/re-embed\.ts/.test(w))).toBe(true); + + // shutdown is async; tests don't need to await on it + void plugin.shutdown(); + }); + + it("stays quiet when the DB has no embeddings at all", () => { + const warns: string[] = []; + const captureLog: Logger = { + debug: () => {}, + info: () => {}, + warn: (msg) => warns.push(String(msg)), + error: () => {}, + }; + const stateDir = path.join(tmpDir, "state2"); + fs.mkdirSync(stateDir, { recursive: true }); + const plugin = initPlugin({ + stateDir, + workspaceDir: tmpDir, + config: { + storage: { dbPath: path.join(stateDir, "empty.db") }, + embedding: { provider: "openai", model: "x", dimensions: 1536 }, + }, + log: captureLog, + }); + + expect(warns.some((w) => /embedding model mismatch/i.test(w))).toBe(false); + void plugin.shutdown(); + }); +}); + +describe("re-embed CLI", () => { + it("parseArgs supports the documented flags", () => { + const opts = parseArgs([ + "--missing-only", + "--dry-run", + "--limit", + "5", + "--batch-size", + "2", + "--db", + "/tmp/foo.db", + "--config", + "/tmp/openclaw.json", + ]); + expect(opts.missingOnly).toBe(true); + expect(opts.dryRun).toBe(true); + expect(opts.limit).toBe(5); + expect(opts.batchSize).toBe(2); + expect(opts.dbPath).toBe("/tmp/foo.db"); + expect(opts.configPath).toBe("/tmp/openclaw.json"); + }); + + it("parseArgs rejects unknown args", () => { + expect(() => parseArgs(["--what"])).toThrow(/Unknown argument/); + }); + + it("dry-run reports planned count and writes nothing", async () => { + // Build a DB with: 1 matched, 1 mismatched, 1 missing. + const setup = new SqliteStore(dbPath, noopLog); + setup.insertChunk(makeChunk({ id: "c1" })); + setup.insertChunk(makeChunk({ id: "c2" })); + setup.insertChunk(makeChunk({ id: "c3" })); // no embedding + setup.upsertEmbedding("c1", Array(8).fill(0.1), { + provider: "local", + model: "", + }); + setup.upsertEmbedding("c2", Array(8).fill(0.2), { + provider: "openai", + model: "old-model", + }); + setup.close(); + + // Write a minimal openclaw.json the script can read + const stateDir = path.join(tmpDir, "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const configPath = path.join(tmpDir, "openclaw.json"); + fs.writeFileSync( + configPath, + JSON.stringify({ + plugins: { + entries: { + "memos-local": { + config: { + storage: { dbPath }, + // No `embedding` block → local fallback (dim 384) + }, + }, + }, + }, + }), + ); + + const result = await runReembed({ + configPath, + dbPath, + missingOnly: false, + dryRun: true, + batchSize: 32, + help: false, + }); + // planned = c1 mismatched (local|||8 vs local||384) + c2 mismatched + c3 missing = 3 + expect(result.planned).toBe(3); + expect(result.processed).toBe(0); + + // Verify the DB embeddings table is unchanged + const checkDb = new Database(dbPath, { readonly: true }); + const rows = checkDb.prepare("SELECT chunk_id, provider, model FROM embeddings ORDER BY chunk_id").all(); + checkDb.close(); + expect(rows).toEqual([ + { chunk_id: "c1", provider: "local", model: "" }, + { chunk_id: "c2", provider: "openai", model: "old-model" }, + ]); + }); +}); diff --git a/apps/memos-local-openclaw/tests/esm-module-format.test.ts b/apps/memos-local-openclaw/tests/esm-module-format.test.ts new file mode 100644 index 000000000..ce0cb7d1b --- /dev/null +++ b/apps/memos-local-openclaw/tests/esm-module-format.test.ts @@ -0,0 +1,154 @@ +/** + * Regression tests for issue #1733: + * "@memtensor/memos-lite-openclaw-plugin v0.2.3 fails to load: + * ReferenceError: exports is not defined in ES module scope" + * + * Root cause in v0.2.3: + * - package.json declared `"type": "module"` (so Node treats every `.js` + * file in the package as an ES module). + * - The published tarball still shipped `dist/*.js` files compiled with + * `"module": "CommonJS"`, which contain `Object.defineProperty(exports, ...)` + * and `require(...)`. Node 22+ then refused to load them. + * + * Permanent fix — ESM-flavoured tsconfig + ESM dist publish: + * 1. `package.json.files` must ship the built `dist/` output that + * `prepack` produces, plus explicit `.cjs` scripts for CommonJS helpers. + * 2. `package.json.main` and `openclaw.extensions` point at `dist/index.js`; + * because package.json has `type: "module"`, that file must be true ESM + * output, never CommonJS-flavoured output. + * 3. `tsconfig.json` must not emit CommonJS-flavoured `.js` (`module` + * must be one of the ESM variants), and `moduleResolution` must be + * compatible with ESM emit without rewriting every relative import + * to add an explicit `.js` extension. + * 4. `dist/index.js` must not contain CommonJS export markers. + */ + +import { describe, expect, it } from "vitest"; +import { readFileSync, existsSync } from "node:fs"; +import { dirname, resolve, extname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __filename = fileURLToPath(import.meta.url); +const pluginRoot = resolve(dirname(__filename), ".."); + +function readJsonStripComments(filePath: string): Record { + // tsconfig allows // and /* */ comments and trailing commas; strip them + // before JSON.parse so this test stays dependency-free. + let text = readFileSync(filePath, "utf-8"); + text = text.replace(/\/\*[\s\S]*?\*\//g, ""); + text = text.replace(/(^|[^:])\/\/[^\n]*/g, "$1"); + text = text.replace(/,(\s*[}\]])/g, "$1"); + return JSON.parse(text); +} + +function readPackageJson(): Record { + return readJsonStripComments(resolve(pluginRoot, "package.json")); +} + +function readTsconfig(): Record { + return readJsonStripComments(resolve(pluginRoot, "tsconfig.json")); +} + +describe("issue #1733 — ESM module format does not regress", () => { + it("package.json declares type: module (matches Node 22+ ESM-by-default expectation)", () => { + const pkg = readPackageJson(); + expect( + pkg.type, + "If `type` is missing or 'commonjs', remove the ESM-specific source features (`import.meta.url`, `createRequire(import.meta.url)`) from index.ts first.", + ).toBe("module"); + }); + + it("package.json.files ships the built dist directory", () => { + const pkg = readPackageJson(); + const files: unknown[] = Array.isArray(pkg.files) ? pkg.files : []; + expect(files).toContain("dist"); + }); + + it("package.json.files only ships dist output or explicit .cjs scripts", () => { + const pkg = readPackageJson(); + const files: unknown[] = Array.isArray(pkg.files) ? pkg.files : []; + const offenders = files.filter((entry) => { + if (typeof entry !== "string") return false; + const ext = extname(entry).toLowerCase(); + if (ext !== ".js" && ext !== ".mjs") return false; + const normalized = entry.replace(/\\/g, "/").replace(/^\.\//, ""); + return !normalized.startsWith("dist/"); + }); + expect( + offenders, + `Only built dist ESM files may ship as bare .js/.mjs. ` + + `CommonJS helpers must use .cjs. Offenders: ${offenders.join(", ")}`, + ).toEqual([]); + }); + + it("package.json.main points at the built ESM entry", () => { + const pkg = readPackageJson(); + const main = pkg.main; + expect(typeof main).toBe("string"); + expect(main).toBe("dist/index.js"); + }); + + it("openclaw extensions reference the built ESM entry", () => { + const pkg = readPackageJson(); + const extensions = pkg.openclaw?.extensions; + expect(Array.isArray(extensions)).toBe(true); + expect(extensions).toEqual(["./dist/index.js"]); + }); + + it("tsconfig.json emits ESM, not CommonJS (would re-create the v0.2.3 conflict on local builds)", () => { + const tsc = readTsconfig(); + const moduleSetting = String(tsc.compilerOptions?.module ?? "").toLowerCase(); + + // Forbidden: any CommonJS-flavoured emit — produces `exports.X = ...` + // which collides with `"type": "module"` in Node 22+ ESM mode. + expect(moduleSetting).not.toBe("commonjs"); + expect(moduleSetting).not.toBe("none"); + + // Required: an ESM-emitting module mode. + const esmModes = new Set([ + "es2015", + "es2020", + "es2022", + "esnext", + "node16", + "node18", + "nodenext", + "preserve", + ]); + expect( + esmModes.has(moduleSetting), + `tsconfig "module": "${moduleSetting}" is not ESM-flavoured. ` + + `Building would emit dist/*.js with module.exports/require(), which clashes with package.json "type": "module".`, + ).toBe(true); + }); + + it("tsconfig.json sets a module resolution compatible with ESM emit", () => { + const tsc = readTsconfig(); + const resolution = String( + tsc.compilerOptions?.moduleResolution ?? "", + ).toLowerCase(); + + // Required for ESM emit to work without rewriting every relative import + // path to include explicit ".js" extensions throughout src/. + const allowed = new Set(["bundler", "node16", "node18", "nodenext"]); + expect( + allowed.has(resolution), + `tsconfig "moduleResolution": "${resolution}" is incompatible with ESM emit ` + + `(or requires explicit .js extensions in every import). ` + + `Use one of: ${[...allowed].join(", ")}.`, + ).toBe(true); + }); + + it("compiled output (if any) does not contain CommonJS export markers", () => { + // npm prepack runs `npm run build`, so the published package includes dist. + // If a developer has not built locally yet, skip this source-tree-only check. + const distEntry = join(pluginRoot, "dist", "index.js"); + if (!existsSync(distEntry)) { + // Source-only publish is the happy path; nothing to check. + return; + } + const text = readFileSync(distEntry, "utf-8"); + expect(text).not.toMatch(/Object\.defineProperty\(exports,/); + expect(text).not.toMatch(/^exports\./m); + }); +}); diff --git a/apps/memos-local-openclaw/tests/hub-eager-connect.test.ts b/apps/memos-local-openclaw/tests/hub-eager-connect.test.ts index 7286cfd08..3c88369cc 100644 --- a/apps/memos-local-openclaw/tests/hub-eager-connect.test.ts +++ b/apps/memos-local-openclaw/tests/hub-eager-connect.test.ts @@ -144,6 +144,7 @@ async function loadPluginWithMocks(opts: { resolvePath: () => "/tmp/memos-eager-hub", logger: { info() {}, warn() {} }, registerTool: () => {}, + registerMemoryPromptSection: () => {}, registerMemoryCapability: () => {}, registerService: (service: any) => { opts.captureService?.(service); }, on: () => {}, diff --git a/apps/memos-local-openclaw/tests/incremental-sharing.test.ts b/apps/memos-local-openclaw/tests/incremental-sharing.test.ts index e7ef9bffc..b6362fc47 100644 --- a/apps/memos-local-openclaw/tests/incremental-sharing.test.ts +++ b/apps/memos-local-openclaw/tests/incremental-sharing.test.ts @@ -33,6 +33,7 @@ function makeApi(stateDir: string, pluginConfig: Record = {}) { } tools.set(def.name, def); }, + registerMemoryPromptSection() {}, registerMemoryCapability() {}, registerService(def: any) { service = def; diff --git a/apps/memos-local-openclaw/tests/integration.test.ts b/apps/memos-local-openclaw/tests/integration.test.ts index 9a759e0b5..7bd794113 100644 --- a/apps/memos-local-openclaw/tests/integration.test.ts +++ b/apps/memos-local-openclaw/tests/integration.test.ts @@ -44,6 +44,7 @@ function makePluginApi(stateDir: string, pluginConfig: Record = } tools.set(def.name, def); }, + registerMemoryPromptSection() {}, registerMemoryCapability() {}, registerService(def: any) { service = def; diff --git a/apps/memos-local-openclaw/tests/issue-1559-memory-api-migration.test.ts b/apps/memos-local-openclaw/tests/issue-1559-memory-api-migration.test.ts new file mode 100644 index 000000000..f482b9852 --- /dev/null +++ b/apps/memos-local-openclaw/tests/issue-1559-memory-api-migration.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect, vi } from "vitest"; +import memosLocalPlugin from "../index"; + +/** + * Regression test for GitHub issue #1559. + * + * OpenClaw 2026.3.31 removed `api.registerMemoryCapability` and replaced it + * with three focused methods. The plugin must: + * 1. Prefer `api.registerMemoryPromptSection(builder)` when the host + * exposes it (new SDK). + * 2. Fall back to `api.registerMemoryCapability({ promptBuilder })` when + * only the legacy method exists (older SDK). + * 3. Fail loudly when neither method is exposed. + */ + +function makeBaseApi(extra: Record) { + return { + pluginConfig: {}, + config: {}, + resolvePath: (input: string) => + input === "~/.openclaw" ? "/tmp/memos-local-openclaw-1559" : input, + logger: { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + }, + registerTool: () => {}, + registerService: () => {}, + on: () => {}, + ...extra, + } as any; +} + +describe("OpenClaw 2026.3.31 memory-API migration (issue #1559)", () => { + it("calls registerMemoryPromptSection when the new host SDK exposes it", () => { + const registerMemoryPromptSection = vi.fn(); + const registerMemoryCapability = vi.fn(); + const api = makeBaseApi({ + registerMemoryPromptSection, + registerMemoryCapability, // both present: new API must win + }); + + expect(() => memosLocalPlugin.register(api)).not.toThrow(); + + expect(registerMemoryPromptSection).toHaveBeenCalledTimes(1); + const builderArg = registerMemoryPromptSection.mock.calls[0][0]; + expect(typeof builderArg).toBe("function"); + // Builder must return a string[] (prompt section lines) + const out = builderArg({ + availableTools: new Set(["memory_search", "memory_get"]), + }); + expect(Array.isArray(out)).toBe(true); + expect(out.some((line: string) => /memory/i.test(line))).toBe(true); + + // When the new API is wired, the legacy capability call MUST NOT also be + // invoked — otherwise the host would receive a duplicate registration. + expect(registerMemoryCapability).not.toHaveBeenCalled(); + }); + + it("falls back to registerMemoryCapability on hosts without the new API", () => { + const registerMemoryCapability = vi.fn(); + const api = makeBaseApi({ + registerMemoryCapability, // legacy-only host (pre-2026.3.31) + }); + + expect(() => memosLocalPlugin.register(api)).not.toThrow(); + + expect(registerMemoryCapability).toHaveBeenCalledTimes(1); + const capabilityArg = registerMemoryCapability.mock.calls[0][0]; + expect(capabilityArg).toBeDefined(); + expect(typeof capabilityArg.promptBuilder).toBe("function"); + }); + + it("throws when neither memory-registration API is present", () => { + const error = vi.fn(); + const warn = vi.fn(); + const api = makeBaseApi({ + logger: { + debug: () => {}, + info: () => {}, + warn, + error, + }, + }); + + expect(() => memosLocalPlugin.register(api)).toThrow( + /registerMemoryPromptSection|registerMemoryCapability/, + ); + expect(error).toHaveBeenCalled(); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/memos-local-openclaw/tests/json5-config.test.ts b/apps/memos-local-openclaw/tests/json5-config.test.ts new file mode 100644 index 000000000..870c24244 --- /dev/null +++ b/apps/memos-local-openclaw/tests/json5-config.test.ts @@ -0,0 +1,209 @@ +import { describe, it, expect, afterEach } from "vitest"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { + parseJsonOrJson5, + parseJson5, + normalizeJson5, + Json5ParseError, +} from "../src/shared/json5"; +import { loadOpenClawFallbackConfig } from "../src/shared/llm-call"; + +/** + * Regression coverage for issue #1543: + * openclaw.json is JSON5; the plugin tried to read it with strict JSON.parse, + * so any user who had a `//` comment in tools.allow broke initialization. + */ +describe("parseJsonOrJson5 — JSON5 features used by openclaw.json", () => { + it("parses plain strict JSON unchanged", () => { + const obj = parseJsonOrJson5(JSON.stringify({ a: 1, b: [1, 2], c: "x" })); + expect(obj).toEqual({ a: 1, b: [1, 2], c: "x" }); + }); + + it("parses JSON with line comments (the exact issue #1543 case)", () => { + const text = `{ + "tools": { + // these are the allowed tool groups + "allow": [ + "group:builtin", // built-in tools + "group:plugins" // plugin-provided tools + ] + } + }`; + const obj = parseJsonOrJson5(text) as any; + expect(obj.tools.allow).toEqual(["group:builtin", "group:plugins"]); + }); + + it("parses JSON with block comments", () => { + const text = `{ + /* header comment */ + "models": { + "providers": { + /* anthropic provider config */ + "anthropic": { "baseUrl": "https://api.anthropic.com" } + } + } + }`; + const obj = parseJsonOrJson5(text) as any; + expect(obj.models.providers.anthropic.baseUrl).toBe( + "https://api.anthropic.com", + ); + }); + + it("parses trailing commas in arrays and objects", () => { + const text = `{ + "tools": { + "allow": ["a", "b", "c",], + }, + }`; + const obj = parseJsonOrJson5(text) as any; + expect(obj.tools.allow).toEqual(["a", "b", "c"]); + }); + + it("parses single-quoted strings", () => { + const text = `{ + 'name': 'memos-local', + "value": 'has "embedded double" quotes inside' + }`; + const obj = parseJsonOrJson5(text) as any; + expect(obj.name).toBe("memos-local"); + expect(obj.value).toBe('has "embedded double" quotes inside'); + }); + + it("parses unquoted identifier keys", () => { + const text = `{ + tools: { allow: ["x"] }, + agents: { defaults: { model: { primary: "anthropic/claude-3-haiku" } } } + }`; + const obj = parseJsonOrJson5(text) as any; + expect(obj.tools.allow).toEqual(["x"]); + expect(obj.agents.defaults.model.primary).toBe("anthropic/claude-3-haiku"); + }); + + it("handles UTF-8 BOM", () => { + const text = "" + JSON.stringify({ ok: true }); + expect(parseJsonOrJson5(text)).toEqual({ ok: true }); + }); + + it("does not strip `//` that appears inside string literals", () => { + const text = `{ + "url": "https://api.anthropic.com", + "note": "see https://example.com/docs#path // not a comment" + }`; + const obj = parseJsonOrJson5(text) as any; + expect(obj.url).toBe("https://api.anthropic.com"); + expect(obj.note).toBe( + "see https://example.com/docs#path // not a comment", + ); + }); + + it("does not strip `/*` that appears inside string literals", () => { + const text = `{ "tip": "use /* and */ for block comments" }`; + const obj = parseJsonOrJson5(text) as any; + expect(obj.tip).toBe("use /* and */ for block comments"); + }); + + it("throws Json5ParseError on truly malformed input", () => { + expect(() => parseJson5("{ not_json: , }")).toThrow(Json5ParseError); + }); + + it("normalizeJson5 yields strict JSON that JSON.parse can read", () => { + const text = `{ + // header + a: 1, + b: 'two', + c: [3, 4,], + }`; + const out = normalizeJson5(text); + expect(() => JSON.parse(out)).not.toThrow(); + expect(JSON.parse(out)).toEqual({ a: 1, b: "two", c: [3, 4] }); + }); +}); + +describe("loadOpenClawFallbackConfig with JSON5 openclaw.json (issue #1543)", () => { + let tmpDir: string | undefined; + let savedConfigPath: string | undefined; + let savedStateDir: string | undefined; + + afterEach(() => { + if (savedConfigPath !== undefined) process.env.OPENCLAW_CONFIG_PATH = savedConfigPath; + else delete process.env.OPENCLAW_CONFIG_PATH; + if (savedStateDir !== undefined) process.env.OPENCLAW_STATE_DIR = savedStateDir; + else delete process.env.OPENCLAW_STATE_DIR; + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); + savedConfigPath = undefined; + savedStateDir = undefined; + tmpDir = undefined; + }); + + function writeRawConfig(text: string): string { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-json5-")); + const cfgPath = path.join(tmpDir, "openclaw.json"); + fs.writeFileSync(cfgPath, text, "utf-8"); + savedConfigPath = process.env.OPENCLAW_CONFIG_PATH; + savedStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_CONFIG_PATH = cfgPath; + return cfgPath; + } + + const noopLog = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + }; + + it("loads fallback config when openclaw.json contains line comments", () => { + writeRawConfig(`{ + // top-level config + "agents": { + "defaults": { + "model": { "primary": "anthropic/claude-3-haiku" } + } + }, + "models": { + "providers": { + // anthropic provider + "anthropic": { + "baseUrl": "https://api.anthropic.com", + "apiKey": "sk-ant-test" + } + } + } + }`); + const cfg = loadOpenClawFallbackConfig(noopLog); + expect(cfg).toBeDefined(); + expect(cfg!.apiKey).toBe("sk-ant-test"); + expect(cfg!.provider).toBe("anthropic"); + expect(cfg!.model).toBe("claude-3-haiku"); + }); + + it("loads fallback config when openclaw.json mixes single + double quotes", () => { + writeRawConfig(`{ + 'agents': { 'defaults': { 'model': { 'primary': 'anthropic/claude-3-haiku' } } }, + "models": { + "providers": { + "anthropic": { "baseUrl": 'https://api.anthropic.com', apiKey: 'sk-ant-test' } + } + } + }`); + const cfg = loadOpenClawFallbackConfig(noopLog); + expect(cfg).toBeDefined(); + expect(cfg!.apiKey).toBe("sk-ant-test"); + }); + + it("loads fallback config when openclaw.json has trailing commas", () => { + writeRawConfig(`{ + "agents": { "defaults": { "model": { "primary": "anthropic/claude-3-haiku" } } }, + "models": { + "providers": { + "anthropic": { "baseUrl": "https://api.anthropic.com", "apiKey": "sk-ant-test", }, + }, + }, + }`); + const cfg = loadOpenClawFallbackConfig(noopLog); + expect(cfg).toBeDefined(); + expect(cfg!.apiKey).toBe("sk-ant-test"); + }); +}); diff --git a/apps/memos-local-openclaw/tests/memory-registration-api.test.ts b/apps/memos-local-openclaw/tests/memory-registration-api.test.ts new file mode 100644 index 000000000..5551e447d --- /dev/null +++ b/apps/memos-local-openclaw/tests/memory-registration-api.test.ts @@ -0,0 +1,195 @@ +/** + * Regression tests for issue #1559 — OpenClaw 2026.3.31 renamed the memory + * registration API from `registerMemoryCapability({ promptBuilder })` to + * `registerMemoryPromptSection(builder)`. The plugin must: + * + * 1. Prefer the new API when the host exposes it (OpenClaw 2026.3.31+). + * 2. Fall back to the legacy API when only that is available (older gateways). + * 3. Fail loudly (log at error + throw) when neither method exists — so the + * rest of register() does not spin up in a permanently broken state and + * operators see the incompatibility immediately in production dashboards. + * Falls back to `logger.warn` when the host lacks `.error`. + */ +import { afterEach, describe, expect, it, vi } from "vitest"; + +function setupMocks() { + vi.doMock("../src/config", () => ({ + buildContext: () => ({ + stateDir: "/tmp/memos-openclaw-reg", + workspaceDir: "/tmp/memos-openclaw-reg/workspace", + log: { debug() {}, info() {}, warn() {}, error() {} }, + openclawAPI: undefined, + config: { + storage: { dbPath: "/tmp/memos-openclaw-reg/memos.db" }, + capture: { evidenceWrapperTag: "STORED_MEMORY" }, + telemetry: {}, + embedding: { provider: "openclaw", capabilities: { hostEmbedding: true } }, + summarizer: { provider: "openclaw", capabilities: { hostCompletion: true } }, + sharing: { enabled: false, role: "client", hub: { port: 18800, teamName: "", teamToken: "" }, client: { hubAddress: "", userToken: "" }, capabilities: { hostEmbedding: true, hostCompletion: true } }, + }, + }), + })); + vi.doMock("../src/storage/sqlite", () => ({ SqliteStore: class { + recordToolCall() {} + recordApiLog() {} + close() {} + }})); + vi.doMock("../src/storage/ensure-binding", () => ({ ensureSqliteBinding: () => {} })); + vi.doMock("../src/embedding", () => ({ + Embedder: class { + provider = "openclaw"; + constructor(_cfg: unknown, _log: unknown, _openclaw: unknown) {} + async embed() { return []; } + }, + })); + vi.doMock("../src/ingest/worker", () => ({ IngestWorker: class { + getTaskProcessor() { return { onTaskCompleted() {} }; } + enqueue() {} + async flush() {} + }})); + vi.doMock("../src/recall/engine", () => ({ RecallEngine: class { + async search() { return { hits: [], meta: {} }; } + async searchSkills() { return []; } + }})); + vi.doMock("../src/ingest/providers", () => ({ + Summarizer: class { + constructor(_cfg: unknown, _log: unknown, _openclaw: unknown) {} + async filterRelevant() { return null; } + }, + })); + vi.doMock("../src/viewer/server", () => ({ ViewerServer: class { + async start() { return "http://127.0.0.1:18799"; } + stop() {} + getResetToken() { return "token"; } + }})); + vi.doMock("../src/hub/server", () => ({ HubServer: class { + async start() { return "http://127.0.0.1:18800"; } + async stop() {} + }})); + vi.doMock("../src/client/hub", () => ({ + hubGetMemoryDetail: async () => ({}), + hubRequestJson: async () => ({}), + hubSearchMemories: async () => ({ hits: [], meta: {} }), + hubSearchSkills: async () => ({ hits: [] }), + resolveHubClient: async () => ({ hubUrl: "", userToken: "", userId: "" }), + })); + vi.doMock("../src/client/connector", () => ({ getHubStatus: async () => ({ connected: false }) })); + vi.doMock("../src/client/skill-sync", () => ({ + fetchHubSkillBundle: async () => ({}), + publishSkillBundleToHub: async () => ({}), + restoreSkillBundleFromHub: () => ({}), + unpublishSkillBundleFromHub: async () => ({}), + })); + vi.doMock("../src/skill/evolver", () => ({ SkillEvolver: class { async onTaskCompleted() {} } })); + vi.doMock("../src/skill/installer", () => ({ SkillInstaller: class {} })); + vi.doMock("../src/skill/bundled-memory-guide", () => ({ MEMORY_GUIDE_SKILL_MD: "# mock" })); + vi.doMock("../src/telemetry", () => ({ Telemetry: class { + trackToolCalled() {} + trackAutoRecall() {} + trackMemoryIngested() {} + trackSkillInstalled() {} + trackSkillEvolved() {} + trackPluginStarted() {} + trackError() {} + trackViewerOpened() {} + async shutdown() {} + }})); + vi.doMock("../src/capture", () => ({ + captureMessages: () => {}, + stripInboundMetadata: (s: string) => s, + })); +} + +function makeBaseApi(overrides: Record) { + return { + id: "memos-local-openclaw-plugin", + pluginConfig: {}, + config: {}, + resolvePath: () => "/tmp/memos-openclaw-reg", + logger: { info() {}, warn() {}, error() {} }, + registerTool: () => {}, + registerService: () => {}, + on: () => {}, + ...overrides, + }; +} + +afterEach(() => { + vi.resetModules(); + vi.clearAllMocks(); +}); + +describe("issue #1559 — memory registration API compatibility", () => { + it("prefers registerMemoryPromptSection (OpenClaw 2026.3.31+) over the legacy method", async () => { + setupMocks(); + const registerMemoryPromptSection = vi.fn(); + const registerMemoryCapability = vi.fn(); + + const pluginModule = await import("../plugin-impl"); + pluginModule.default.register(makeBaseApi({ + registerMemoryPromptSection, + registerMemoryCapability, + }) as any); + + expect(registerMemoryPromptSection).toHaveBeenCalledTimes(1); + expect(registerMemoryPromptSection.mock.calls[0][0]).toBeInstanceOf(Function); + expect(registerMemoryCapability).not.toHaveBeenCalled(); + }); + + it("falls back to registerMemoryCapability when the new API is missing (legacy OpenClaw)", async () => { + setupMocks(); + const registerMemoryCapability = vi.fn(); + + const pluginModule = await import("../plugin-impl"); + pluginModule.default.register(makeBaseApi({ + registerMemoryCapability, + }) as any); + + expect(registerMemoryCapability).toHaveBeenCalledTimes(1); + const capability = registerMemoryCapability.mock.calls[0][0]; + expect(capability).toBeTypeOf("object"); + expect(capability.promptBuilder).toBeInstanceOf(Function); + }); + + it("logs at error level and throws when neither memory-registration method exists", async () => { + setupMocks(); + const info = vi.fn(); + const warn = vi.fn(); + const error = vi.fn(); + + const pluginModule = await import("../plugin-impl"); + expect(() => { + pluginModule.default.register(makeBaseApi({ + logger: { info, warn, error }, + }) as any); + }).toThrow(/registerMemoryPromptSection|registerMemoryCapability/); + + // The misconfiguration is fatal for recall, so it must escalate to error rather + // than being buried in a warn log where operators are more likely to miss it, + // and throwing prevents register() from proceeding to spin up stores/workers/tools + // in a state where the plugin looks healthy but recall is silently dead. + expect(error).toHaveBeenCalled(); + const errorMsg = error.mock.calls.map((c) => String(c[0])).join(" "); + expect(errorMsg).toMatch(/registerMemoryPromptSection|registerMemoryCapability/); + expect(warn).not.toHaveBeenCalledWith( + expect.stringMatching(/registerMemoryPromptSection|registerMemoryCapability/), + ); + }); + + it("falls back to warn (and still throws) when the host logger lacks an error method", async () => { + setupMocks(); + const warn = vi.fn(); + + const pluginModule = await import("../plugin-impl"); + expect(() => { + pluginModule.default.register(makeBaseApi({ + // Simulate an older host whose HostLogger shape has no `error`. + logger: { info() {}, warn }, + }) as any); + }).toThrow(/registerMemoryPromptSection|registerMemoryCapability/); + + expect(warn).toHaveBeenCalled(); + const warnMsg = warn.mock.calls.map((c) => String(c[0])).join(" "); + expect(warnMsg).toMatch(/registerMemoryPromptSection|registerMemoryCapability/); + }); +}); diff --git a/apps/memos-local-openclaw/tests/memory-search-tool.test.ts b/apps/memos-local-openclaw/tests/memory-search-tool.test.ts new file mode 100644 index 000000000..962d78165 --- /dev/null +++ b/apps/memos-local-openclaw/tests/memory-search-tool.test.ts @@ -0,0 +1,52 @@ +import { describe, it, expect, vi } from "vitest"; +import { createMemorySearchTool } from "../src/tools/memory-search"; +import type { RecallEngine } from "../src/recall/engine"; + +function makeMockEngine() { + return { + search: vi.fn(async () => ({ + hits: [], + meta: { usedMinScore: 0.45, usedMaxResults: 6, totalCandidates: 0 }, + })), + } as unknown as RecallEngine; +} + +describe("memory_search tool — excludeSessionKey input wiring", () => { + it("declares excludeSessionKey in its inputSchema", () => { + const engine = makeMockEngine(); + const tool = createMemorySearchTool(engine); + const schema = tool.inputSchema as { properties: Record }; + expect(schema.properties).toHaveProperty("excludeSessionKey"); + }); + + it("forwards excludeSessionKey from tool input to engine.search", async () => { + const engine = makeMockEngine(); + const tool = createMemorySearchTool(engine); + + await tool.handler({ query: "deploy", excludeSessionKey: "sess-current" }); + + expect(engine.search).toHaveBeenCalledTimes(1); + const callArgs = (engine.search as unknown as { mock: { calls: any[][] } }).mock.calls[0][0]; + expect(callArgs.excludeSessionKey).toBe("sess-current"); + }); + + it("omits excludeSessionKey when not provided", async () => { + const engine = makeMockEngine(); + const tool = createMemorySearchTool(engine); + + await tool.handler({ query: "deploy" }); + + const callArgs = (engine.search as unknown as { mock: { calls: any[][] } }).mock.calls[0][0]; + expect(callArgs.excludeSessionKey).toBeUndefined(); + }); + + it("treats non-string excludeSessionKey as undefined (input hygiene)", async () => { + const engine = makeMockEngine(); + const tool = createMemorySearchTool(engine); + + await tool.handler({ query: "deploy", excludeSessionKey: 42 }); + + const callArgs = (engine.search as unknown as { mock: { calls: any[][] } }).mock.calls[0][0]; + expect(callArgs.excludeSessionKey).toBeUndefined(); + }); +}); diff --git a/apps/memos-local-openclaw/tests/module-format.test.ts b/apps/memos-local-openclaw/tests/module-format.test.ts new file mode 100644 index 000000000..2169c5352 --- /dev/null +++ b/apps/memos-local-openclaw/tests/module-format.test.ts @@ -0,0 +1,97 @@ +/** + * Regression test for issue #1733 + * (@memtensor/memos-lite-openclaw-plugin v0.2.3 fails to load: + * "ReferenceError: exports is not defined in ES module scope"). + * + * Root cause: package.json declared `"type": "module"` while tsconfig.json + * emitted CommonJS (`"module": "CommonJS"`). Node.js treated the compiled + * `dist/index.js` as ESM, saw the `exports.` writes, and refused to load + * the plugin. + * + * Guard the invariant so it cannot silently drift again: + * - When package.json says "type": "module", tsconfig must emit an ESM + * module format (not CommonJS / node16 / commonjs variants). + * - When ESM is chosen, `moduleResolution` must be one of the ESM-safe + * variants ("bundler" / "nodenext" / "node16"). Bare `"node"` (which is + * equivalent to the legacy classic CJS resolver) would silently mask + * ESM-only imports at build time. + */ + +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +// Strip // and /* */ comments and trailing commas so JSON.parse can eat tsconfig. +function readJsoncSync(path: string): any { + const raw = readFileSync(path, "utf-8"); + const noBlock = raw.replace(/\/\*[\s\S]*?\*\//g, ""); + const noLine = noBlock.replace(/(^|[^:])\/\/.*$/gm, "$1"); + const noTrailingCommas = noLine.replace(/,(\s*[}\]])/g, "$1"); + return JSON.parse(noTrailingCommas); +} + +const pluginRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const pkgJsonPath = resolve(pluginRoot, "package.json"); +const tsconfigPath = resolve(pluginRoot, "tsconfig.json"); + +const CJS_MODULES = new Set(["commonjs", "node16-commonjs"]); +const ESM_MODULES = new Set([ + "es2015", + "es2020", + "es2022", + "esnext", + "node16", + "node18", + "nodenext", +]); +const ESM_SAFE_RESOLUTIONS = new Set([ + "bundler", + "node16", + "node18", + "nodenext", +]); + +describe("regression #1733 — plugin module-format alignment", () => { + const pkg = JSON.parse(readFileSync(pkgJsonPath, "utf-8")); + const tsconfig = readJsoncSync(tsconfigPath); + const module = String(tsconfig.compilerOptions?.module ?? "").toLowerCase(); + const moduleResolution = String( + tsconfig.compilerOptions?.moduleResolution ?? "", + ).toLowerCase(); + + it("package.json declares ESM", () => { + // If this ever flips back to CommonJS or is removed, the tsconfig side + // of the invariant needs to be revisited in tandem. + expect(pkg.type).toBe("module"); + }); + + it("tsconfig emits an ESM-compatible module format", () => { + expect(ESM_MODULES.has(module), `tsconfig module=${module} is CJS-flavoured; would recreate the v0.2.3 "exports is not defined" crash when dist/index.js is loaded under package.json "type": "module".`).toBe(true); + expect(CJS_MODULES.has(module)).toBe(false); + }); + + it("tsconfig moduleResolution is ESM-safe", () => { + expect( + ESM_SAFE_RESOLUTIONS.has(moduleResolution), + `moduleResolution="${moduleResolution}" pairs the ESM emit with the legacy CJS resolver — imports get silently rewritten in ways Node can't execute at runtime.`, + ).toBe(true); + }); + + it("no legacy CommonJS entry point sneaks back in", () => { + // main + openclaw.extensions must not reference a CJS-only artifact. + const main: string = pkg.main ?? ""; + expect(main.endsWith(".cjs")).toBe(false); + + const ext = pkg.openclaw?.extensions; + if (Array.isArray(ext)) { + for (const e of ext) { + expect( + String(e).endsWith(".cjs"), + `openclaw.extensions entry "${e}" would force CommonJS load under type:module.`, + ).toBe(false); + } + } + }); +}); diff --git a/apps/memos-local-openclaw/tests/normalize-auto-recall-query.test.ts b/apps/memos-local-openclaw/tests/normalize-auto-recall-query.test.ts new file mode 100644 index 000000000..15b4ec5d7 --- /dev/null +++ b/apps/memos-local-openclaw/tests/normalize-auto-recall-query.test.ts @@ -0,0 +1,56 @@ +import { describe, it, expect } from "vitest"; +import { normalizeAutoRecallQuery } from "../index"; + +describe("normalizeAutoRecallQuery — instructional-prompt filtering (issue #1595)", () => { + it("returns the original short user query unchanged", () => { + const query = "What did we decide about the migration plan?"; + expect(normalizeAutoRecallQuery(query)).toBe(query); + }); + + it("returns empty string for prompts longer than 300 characters", () => { + const longInstructional = + "Please carefully review the following lengthy set of system instructions and follow them precisely without deviation. ".repeat( + 4, + ); + expect(longInstructional.length).toBeGreaterThan(300); + expect(normalizeAutoRecallQuery(longInstructional)).toBe(""); + }); + + it("returns empty string for system prompts starting with 'You are'", () => { + const sysPrompt = "You are a helpful assistant tasked with summarizing logs."; + expect(normalizeAutoRecallQuery(sysPrompt)).toBe(""); + }); + + it("'You are' check is case-insensitive", () => { + const sysPrompt = "you are an expert reviewer."; + expect(normalizeAutoRecallQuery(sysPrompt)).toBe(""); + }); + + it("returns empty string for the Hermes _SKILL_REVIEW_PROMPT (canonical reproducer from #1595)", () => { + const hermesReview = + "Review the conversation above, consider saving / updating a skill if appropriate. " + + "Focus on what was non-trivial — approach used to complete a task that required trial / error, " + + "changing course due to experiential findings along the way. Did the user expect / desire a " + + "different method or outcome? If a relevant skill already exists, update it with what you " + + "learned. Otherwise, create a new skill if the approach is reusable. If nothing is worth " + + "saving, just say 'Nothing to save' and stop."; + expect(normalizeAutoRecallQuery(hermesReview)).toBe(""); + }); + + it("matches 'Review the conversation above' case-insensitively", () => { + const review = "review the conversation above and think about what skill to add."; + expect(normalizeAutoRecallQuery(review)).toBe(""); + }); + + it("does not falsely strip a normal-length user message that happens to be > 300 chars after sanitization", () => { + // Boundary: exactly 300 chars should still pass (the rule is strictly > 300). + const boundary = "a".repeat(300); + expect(normalizeAutoRecallQuery(boundary)).toBe(boundary); + }); + + it("still strips known new-session preamble before applying the instructional-prompt filter", () => { + const newSession = + "A new session was started via /new or /reset. Execute your Session Startup sequence now."; + expect(normalizeAutoRecallQuery(newSession)).toBe(""); + }); +}); diff --git a/apps/memos-local-openclaw/tests/openclaw-config-patch.test.ts b/apps/memos-local-openclaw/tests/openclaw-config-patch.test.ts new file mode 100644 index 000000000..368bc5563 --- /dev/null +++ b/apps/memos-local-openclaw/tests/openclaw-config-patch.test.ts @@ -0,0 +1,87 @@ +import { describe, it, expect } from "vitest"; +import { + ensureGroupPluginsAllowed, +} from "../src/shared/openclaw-config"; + +/** + * Regression coverage for issue #1543: + * + * memos-local: could not patch tools.allow: + * SyntaxError: Expected double-quoted property name in JSON at position 2222 (line 90 column 7) + * + * The user's openclaw.json has `// ...` line comments, and the previous + * implementation called strict JSON.parse on it. + */ +describe("ensureGroupPluginsAllowed (issue #1543)", () => { + it("patches tools.allow even when openclaw.json contains // comments", () => { + const raw = `{ + // top of file + "tools": { + // these tools are allowed + "allow": [ + "Bash", + "Read", + "Write" + ] + } + }`; + const r = ensureGroupPluginsAllowed(raw); + expect(r.changed).toBe(true); + expect(r.patched).toContain(`"group:plugins"`); + // Existing comments must survive. + expect(r.patched).toContain("// top of file"); + expect(r.patched).toContain("// these tools are allowed"); + }); + + it("patches tools.allow when openclaw.json uses single-quoted strings", () => { + const raw = `{ + "tools": { + "allow": ['Bash', 'Read', 'Write'] + } + }`; + const r = ensureGroupPluginsAllowed(raw); + expect(r.changed).toBe(true); + expect(r.patched).toContain("'group:plugins'"); + }); + + it("patches tools.allow when openclaw.json has trailing commas", () => { + const raw = `{ + "tools": { + "allow": ["Bash", "Read", "Write",], + }, + }`; + const r = ensureGroupPluginsAllowed(raw); + expect(r.changed).toBe(true); + expect(r.patched).toContain(`"group:plugins"`); + }); + + it("does nothing when tools.allow already contains group:plugins", () => { + const raw = `{ + "tools": { "allow": ["Bash", "group:plugins"] } + }`; + const r = ensureGroupPluginsAllowed(raw); + expect(r.changed).toBe(false); + }); + + it("does nothing when tools.allow contains the wildcard '*'", () => { + const raw = `{ + "tools": { "allow": ["*"] } + }`; + const r = ensureGroupPluginsAllowed(raw); + expect(r.changed).toBe(false); + }); + + it("does nothing when tools.allow is missing or empty", () => { + expect(ensureGroupPluginsAllowed(`{}`).changed).toBe(false); + expect(ensureGroupPluginsAllowed(`{ "tools": {} }`).changed).toBe(false); + expect( + ensureGroupPluginsAllowed(`{ "tools": { "allow": [] } }`).changed, + ).toBe(false); + }); + + it("returns changed=false (no throw) when the file is truly malformed", () => { + const r = ensureGroupPluginsAllowed("{ not_json: , bad: }"); + expect(r.changed).toBe(false); + expect(r.reason).toMatch(/parse failed/); + }); +}); diff --git a/apps/memos-local-openclaw/tests/openclaw-config.test.ts b/apps/memos-local-openclaw/tests/openclaw-config.test.ts new file mode 100644 index 000000000..2705934dc --- /dev/null +++ b/apps/memos-local-openclaw/tests/openclaw-config.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from "vitest"; +import { ensureToolsAllowEntry } from "../src/openclaw-config"; + +describe("ensureToolsAllowEntry", () => { + it("adds the entry to tools.allow when missing", () => { + const raw = JSON.stringify( + { + tools: { + allow: ["file_search", "shell"], + }, + }, + null, + 2, + ); + + const patched = ensureToolsAllowEntry(raw, "group:plugins"); + expect(patched).not.toBe(raw); + + const parsed = JSON.parse(patched); + expect(parsed.tools.allow).toEqual(["file_search", "shell", "group:plugins"]); + }); + + it("does NOT modify any other array that happens to contain the same string as the last tools.allow entry", () => { + // Regression: issue #1377. The previous implementation used `raw.replace(new RegExp(`(${lastEntry})(\\s*\\])`))` + // which is non-global and matches the FIRST occurrence in the file. When models.providers.*.models[*].input + // appeared BEFORE tools.allow and ended with the same JSON value as the last tools.allow entry, it patched the + // wrong array, corrupting openclaw.json: + // "input": ["text", "image"] → "input": ["text", "image", "group:plugins"] ← BUG + // "allow": ["file_search", "image"] (was meant to be patched here) + const raw = JSON.stringify( + { + models: { + providers: { + qwen: { + models: [ + { + id: "qwen3.5-plus-search", + name: "qwen3.5-plus-search", + input: ["text", "image"], + }, + ], + }, + }, + }, + tools: { + allow: ["file_search", "image"], + }, + }, + null, + 2, + ); + + const patched = ensureToolsAllowEntry(raw, "group:plugins"); + const parsed = JSON.parse(patched); + + expect(parsed.models.providers.qwen.models[0].input).toEqual(["text", "image"]); + expect(parsed.tools.allow).toEqual(["file_search", "image", "group:plugins"]); + }); + + it("is a no-op when the entry is already present", () => { + const raw = JSON.stringify( + { + tools: { allow: ["file_search", "group:plugins"] }, + }, + null, + 2, + ); + expect(ensureToolsAllowEntry(raw, "group:plugins")).toBe(raw); + }); + + it("is a no-op when tools.allow contains a wildcard", () => { + const raw = JSON.stringify({ tools: { allow: ["*"] } }, null, 2); + expect(ensureToolsAllowEntry(raw, "group:plugins")).toBe(raw); + }); + + it("is a no-op when tools.allow is missing or empty", () => { + expect(ensureToolsAllowEntry(JSON.stringify({}), "group:plugins")).toBe(JSON.stringify({})); + const emptyAllow = JSON.stringify({ tools: { allow: [] } }, null, 2); + expect(ensureToolsAllowEntry(emptyAllow, "group:plugins")).toBe(emptyAllow); + }); + + it("preserves indentation style when rewriting", () => { + const raw = + "{\n" + + " \"tools\": {\n" + + " \"allow\": [\n" + + " \"file_search\",\n" + + " \"shell\"\n" + + " ]\n" + + " },\n" + + " \"models\": { \"providers\": {} }\n" + + "}\n"; + + const patched = ensureToolsAllowEntry(raw, "group:plugins"); + const parsed = JSON.parse(patched); + expect(parsed.tools.allow).toEqual(["file_search", "shell", "group:plugins"]); + // Detected 2-space indent should be preserved + expect(patched.split("\n").some((line) => line.startsWith(" \"allow\""))).toBe(true); + // Original trailing newline is kept + expect(patched.endsWith("\n")).toBe(true); + }); + + it("handles regex-special characters in the last tools.allow entry", () => { + // Previous regex approach used JSON.stringify(value) inline and did not escape regex metacharacters. + const raw = JSON.stringify({ tools: { allow: ["a", "weird.name+x"] } }, null, 2); + const patched = ensureToolsAllowEntry(raw, "group:plugins"); + const parsed = JSON.parse(patched); + expect(parsed.tools.allow).toEqual(["a", "weird.name+x", "group:plugins"]); + }); + + it("returns the input unchanged when JSON cannot be parsed", () => { + const raw = "not really json"; + expect(ensureToolsAllowEntry(raw, "group:plugins")).toBe(raw); + }); +}); diff --git a/apps/memos-local-openclaw/tests/path-utils.test.ts b/apps/memos-local-openclaw/tests/path-utils.test.ts new file mode 100644 index 000000000..1f98160c6 --- /dev/null +++ b/apps/memos-local-openclaw/tests/path-utils.test.ts @@ -0,0 +1,215 @@ +import { describe, expect, it, vi, afterEach } from "vitest"; + +// Path-utils must be platform-aware: on Windows it lower-cases and rewrites +// backslashes / drive-letter URL-pathname / extended-length prefixes; on +// POSIX it stays case-sensitive and idempotent. +// +// To exercise the win32 branch from a Linux CI box we have to stub +// `process.platform` *before* importing the module so the platform branch +// is captured at evaluation time. We use vi.resetModules() between cases. + +async function loadFor(platform: "win32" | "linux" | "darwin") { + vi.resetModules(); + const original = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: platform, configurable: true }); + try { + const mod = await import("../src/path-utils"); + return { mod, restore: () => { + if (original) Object.defineProperty(process, "platform", original); + } }; + } catch (err) { + if (original) Object.defineProperty(process, "platform", original); + throw err; + } +} + +afterEach(() => { + vi.resetModules(); +}); + +describe("normalizeFsPath (win32)", () => { + it("normalises plain Windows absolute paths to lowercase forward-slash form", async () => { + const { mod, restore } = await loadFor("win32"); + try { + expect(mod.normalizeFsPath("C:\\Users\\Admin\\.openclaw-dev\\extensions\\memos-local-openclaw-plugin")) + .toBe("c:/users/admin/.openclaw-dev/extensions/memos-local-openclaw-plugin"); + } finally { restore(); } + }); + + it("accepts forward-slash Windows paths", async () => { + const { mod, restore } = await loadFor("win32"); + try { + expect(mod.normalizeFsPath("c:/Users/Admin/.openclaw-dev/extensions/memos-local-openclaw-plugin")) + .toBe("c:/users/admin/.openclaw-dev/extensions/memos-local-openclaw-plugin"); + } finally { restore(); } + }); + + it("strips the leading slash that precedes a drive letter (URL pathname form)", async () => { + const { mod, restore } = await loadFor("win32"); + try { + expect(mod.normalizeFsPath("/C:/Users/Admin/.openclaw-dev/extensions/memos-local-openclaw-plugin")) + .toBe("c:/users/admin/.openclaw-dev/extensions/memos-local-openclaw-plugin"); + } finally { restore(); } + }); + + it("strips the \\\\?\\ long-path prefix", async () => { + const { mod, restore } = await loadFor("win32"); + try { + expect(mod.normalizeFsPath("\\\\?\\C:\\Users\\Admin\\.openclaw-dev\\extensions\\memos-local-openclaw-plugin")) + .toBe("c:/users/admin/.openclaw-dev/extensions/memos-local-openclaw-plugin"); + } finally { restore(); } + }); + + it("converts \\\\?\\UNC\\server\\share to //server/share", async () => { + const { mod, restore } = await loadFor("win32"); + try { + expect(mod.normalizeFsPath("\\\\?\\UNC\\server\\share\\plugin")) + .toBe("//server/share/plugin"); + } finally { restore(); } + }); + + it("idempotently normalises an already-normalised path", async () => { + const { mod, restore } = await loadFor("win32"); + try { + const once = mod.normalizeFsPath("C:\\Users\\Admin\\plugin"); + expect(mod.normalizeFsPath(once)).toBe(once); + } finally { restore(); } + }); +}); + +describe("normalizeFsPath (posix)", () => { + it("does not lowercase POSIX paths", async () => { + const { mod, restore } = await loadFor("linux"); + try { + expect(mod.normalizeFsPath("/Home/User/.openclaw/plugin")) + .toBe("/Home/User/.openclaw/plugin"); + } finally { restore(); } + }); + + it("normalises ./ and ../ segments", async () => { + const { mod, restore } = await loadFor("linux"); + try { + expect(mod.normalizeFsPath("/home/user/./.openclaw/../.openclaw/plugin")) + .toBe("/home/user/.openclaw/plugin"); + } finally { restore(); } + }); +}); + +describe("isPathInside (win32)", () => { + it("returns true for the exact same Windows directory", async () => { + const { mod, restore } = await loadFor("win32"); + try { + const base = "C:\\Users\\Admin\\.openclaw-dev\\extensions\\memos-local-openclaw-plugin"; + expect(mod.isPathInside(base, base)).toBe(true); + } finally { restore(); } + }); + + it("returns true when target is a child of base", async () => { + const { mod, restore } = await loadFor("win32"); + try { + const base = "C:\\Users\\Admin\\.openclaw-dev\\extensions\\memos-local-openclaw-plugin"; + const target = "C:\\Users\\Admin\\.openclaw-dev\\extensions\\memos-local-openclaw-plugin\\node_modules\\better-sqlite3\\lib\\index.js"; + expect(mod.isPathInside(base, target)).toBe(true); + } finally { restore(); } + }); + + it("handles the /C:/ URL-pathname base against a native target — the exact #1358 case", async () => { + const { mod, restore } = await loadFor("win32"); + try { + const base = "/C:/Users/Admin/.openclaw-dev/extensions/memos-local-openclaw-plugin"; + const target = "C:\\Users\\Admin\\.openclaw-dev\\extensions\\memos-local-openclaw-plugin\\node_modules\\better-sqlite3\\lib\\index.js"; + expect(mod.isPathInside(base, target)).toBe(true); + } finally { restore(); } + }); + + it("handles case difference in the drive letter", async () => { + const { mod, restore } = await loadFor("win32"); + try { + const base = "C:\\Users\\Admin\\plugin"; + const target = "c:\\users\\admin\\plugin\\node_modules\\x"; + expect(mod.isPathInside(base, target)).toBe(true); + } finally { restore(); } + }); + + it("handles mixed slash directions", async () => { + const { mod, restore } = await loadFor("win32"); + try { + const base = "C:\\Users\\Admin\\plugin"; + const target = "C:/Users/Admin/plugin/sub/file.js"; + expect(mod.isPathInside(base, target)).toBe(true); + } finally { restore(); } + }); + + it("handles the \\\\?\\ long-path prefix on one side only", async () => { + const { mod, restore } = await loadFor("win32"); + try { + const base = "C:\\Users\\Admin\\plugin"; + const target = "\\\\?\\C:\\Users\\Admin\\plugin\\node_modules\\x\\index.js"; + expect(mod.isPathInside(base, target)).toBe(true); + } finally { restore(); } + }); + + it("returns false when target sits in a sibling directory", async () => { + const { mod, restore } = await loadFor("win32"); + try { + const base = "C:\\Users\\Admin\\plugin"; + const target = "C:\\Users\\Admin\\other-plugin\\node_modules\\x\\index.js"; + expect(mod.isPathInside(base, target)).toBe(false); + } finally { restore(); } + }); + + it("returns false when target sits in the parent's node_modules (hoisted dep)", async () => { + const { mod, restore } = await loadFor("win32"); + try { + const base = "C:\\Users\\Admin\\plugin"; + const target = "C:\\Users\\Admin\\node_modules\\x\\index.js"; + expect(mod.isPathInside(base, target)).toBe(false); + } finally { restore(); } + }); + + it("returns false when target is on a different drive", async () => { + const { mod, restore } = await loadFor("win32"); + try { + const base = "C:\\Users\\Admin\\plugin"; + const target = "D:\\Users\\Admin\\plugin\\node_modules\\x\\index.js"; + expect(mod.isPathInside(base, target)).toBe(false); + } finally { restore(); } + }); +}); + +describe("isPathInside (posix)", () => { + it("returns true for the exact same POSIX directory", async () => { + const { mod, restore } = await loadFor("linux"); + try { + const base = "/home/user/.openclaw/extensions/memos-local-openclaw-plugin"; + expect(mod.isPathInside(base, base)).toBe(true); + } finally { restore(); } + }); + + it("returns true for a child path", async () => { + const { mod, restore } = await loadFor("linux"); + try { + const base = "/home/user/.openclaw/extensions/memos-local-openclaw-plugin"; + const target = "/home/user/.openclaw/extensions/memos-local-openclaw-plugin/node_modules/better-sqlite3/lib/index.js"; + expect(mod.isPathInside(base, target)).toBe(true); + } finally { restore(); } + }); + + it("returns false for a sibling", async () => { + const { mod, restore } = await loadFor("linux"); + try { + const base = "/home/user/.openclaw/extensions/memos-local-openclaw-plugin"; + const target = "/home/user/.openclaw/extensions/other-plugin/node_modules/x/index.js"; + expect(mod.isPathInside(base, target)).toBe(false); + } finally { restore(); } + }); + + it("returns false when case differs on a case-sensitive filesystem", async () => { + const { mod, restore } = await loadFor("linux"); + try { + const base = "/home/user/plugin"; + const target = "/Home/User/plugin/node_modules/x/index.js"; + expect(mod.isPathInside(base, target)).toBe(false); + } finally { restore(); } + }); +}); diff --git a/apps/memos-local-openclaw/tests/plugin-impl-access.test.ts b/apps/memos-local-openclaw/tests/plugin-impl-access.test.ts index be78fa8b3..ab16158a3 100644 --- a/apps/memos-local-openclaw/tests/plugin-impl-access.test.ts +++ b/apps/memos-local-openclaw/tests/plugin-impl-access.test.ts @@ -35,6 +35,7 @@ function makeApi(stateDir: string, pluginConfig: Record = {}) { } tools.set(def.name, def); }, + registerMemoryPromptSection() {}, registerMemoryCapability() {}, registerService(def: any) { service = def; diff --git a/apps/memos-local-openclaw/tests/plugin-openclaw-wiring.test.ts b/apps/memos-local-openclaw/tests/plugin-openclaw-wiring.test.ts index 77b95c33f..50844390e 100644 --- a/apps/memos-local-openclaw/tests/plugin-openclaw-wiring.test.ts +++ b/apps/memos-local-openclaw/tests/plugin-openclaw-wiring.test.ts @@ -112,6 +112,7 @@ describe("plugin-impl OpenClaw wiring", () => { resolvePath: () => "/tmp/memos-openclaw-wiring", logger: { info() {}, warn() {} }, registerTool: () => {}, + registerMemoryPromptSection: () => {}, registerMemoryCapability: () => {}, registerService: () => {}, on: () => {}, diff --git a/apps/memos-local-openclaw/tests/recall-engine-exclude-session.test.ts b/apps/memos-local-openclaw/tests/recall-engine-exclude-session.test.ts new file mode 100644 index 000000000..28fac1ea9 --- /dev/null +++ b/apps/memos-local-openclaw/tests/recall-engine-exclude-session.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect, vi } from "vitest"; +import { RecallEngine } from "../src/recall/engine"; +import type { SqliteStore } from "../src/storage/sqlite"; +import type { Embedder } from "../src/embedding"; +import type { PluginContext } from "../src/types"; + +const testLog = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, +}; + +function makeContext(): PluginContext { + return { + stateDir: "/tmp", + workspaceDir: "/tmp", + log: testLog, + config: { + recall: { + maxResultsDefault: 6, + maxResultsMax: 20, + minScoreDefault: 0.45, + minScoreFloor: 0.35, + rrfK: 60, + mmrLambda: 0.7, + recencyHalfLifeDays: 14, + vectorSearchMaxChunks: 0, + }, + }, + }; +} + +describe("RecallEngine.search — excludeSessionKey propagation", () => { + it("forwards excludeSessionKey to ftsSearch, vectorSearch, and patternSearch", async () => { + const store = { + ftsSearch: vi.fn(() => []), + patternSearch: vi.fn(() => []), + getAllEmbeddings: vi.fn(() => []), + getRecentEmbeddings: vi.fn(() => []), + getChunk: vi.fn(() => null), + } as unknown as SqliteStore; + + const embedder = { + embedQuery: vi.fn(async () => [0.1, 0.2, 0.3, 0.4]), + } as unknown as Embedder; + + const engine = new RecallEngine(store, embedder, makeContext()); + + await engine.search({ query: "唐波是谁", excludeSessionKey: "sess-current" }); + + // FTS got the excludeSessionKey as 4th argument + expect(store.ftsSearch).toHaveBeenCalled(); + const ftsArgs = (store.ftsSearch as unknown as { mock: { calls: any[][] } }).mock.calls[0]; + expect(ftsArgs[3]).toBe("sess-current"); + + // patternSearch received it in the options object + expect(store.patternSearch).toHaveBeenCalled(); + const patternArgs = (store.patternSearch as unknown as { mock: { calls: any[][] } }).mock.calls[0]; + expect(patternArgs[1].excludeSessionKey).toBe("sess-current"); + + // vector path goes through getAllEmbeddings (since vectorSearchMaxChunks=0 in config) + expect(store.getAllEmbeddings).toHaveBeenCalled(); + const embArgs = (store.getAllEmbeddings as unknown as { mock: { calls: any[][] } }).mock.calls[0]; + expect(embArgs[1]).toBe("sess-current"); + }); + + it("does not pass excludeSessionKey when caller omits it", async () => { + const store = { + ftsSearch: vi.fn(() => []), + patternSearch: vi.fn(() => []), + getAllEmbeddings: vi.fn(() => []), + getRecentEmbeddings: vi.fn(() => []), + getChunk: vi.fn(() => null), + } as unknown as SqliteStore; + + const embedder = { + embedQuery: vi.fn(async () => [0.1, 0.2, 0.3, 0.4]), + } as unknown as Embedder; + + const engine = new RecallEngine(store, embedder, makeContext()); + + await engine.search({ query: "唐波是谁" }); + + const ftsArgs = (store.ftsSearch as unknown as { mock: { calls: any[][] } }).mock.calls[0]; + expect(ftsArgs[3]).toBeUndefined(); + + const patternArgs = (store.patternSearch as unknown as { mock: { calls: any[][] } }).mock.calls[0]; + expect(patternArgs[1].excludeSessionKey).toBeUndefined(); + + const embArgs = (store.getAllEmbeddings as unknown as { mock: { calls: any[][] } }).mock.calls[0]; + expect(embArgs[1]).toBeUndefined(); + }); +}); diff --git a/apps/memos-local-openclaw/tests/session-policy.test.ts b/apps/memos-local-openclaw/tests/session-policy.test.ts new file mode 100644 index 000000000..6e5a548de --- /dev/null +++ b/apps/memos-local-openclaw/tests/session-policy.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import { shouldSkipAutoRecallForSession } from "../src/recall/session-policy"; + +describe("shouldSkipAutoRecallForSession", () => { + it("skips cron session keys with default config (excludeCron defaults to true)", () => { + expect(shouldSkipAutoRecallForSession("agent:main:cron:job-abc", undefined)).toBe(true); + expect(shouldSkipAutoRecallForSession("agent:main:cron:job-abc", {})).toBe(true); + }); + + it("matches the `cron` segment regardless of position", () => { + expect(shouldSkipAutoRecallForSession("cron:tick-1", {})).toBe(true); + expect(shouldSkipAutoRecallForSession("agent:alpha:cron", {})).toBe(true); + expect(shouldSkipAutoRecallForSession("CRON:Job-9", {})).toBe(true); // case-insensitive + }); + + it("does NOT match identifiers that merely contain the substring `cron`", () => { + // boundary guard: 'cronos' is its own word, 'chat-cron-debug' uses '-' not ':' + expect(shouldSkipAutoRecallForSession("agent:main:chat:cronos", {})).toBe(false); + expect(shouldSkipAutoRecallForSession("agent:main:chat-cron-debug", {})).toBe(false); + }); + + it("does NOT skip regular chat sessions", () => { + expect(shouldSkipAutoRecallForSession("agent:main:chat:hello", {})).toBe(false); + expect(shouldSkipAutoRecallForSession("default", {})).toBe(false); + }); + + it("returns false when sessionKey is missing or empty (no opinion)", () => { + expect(shouldSkipAutoRecallForSession(undefined, {})).toBe(false); + expect(shouldSkipAutoRecallForSession("", {})).toBe(false); + }); + + it("respects excludeCron=false to keep cron auto-recall enabled", () => { + expect( + shouldSkipAutoRecallForSession("agent:main:cron:abc", { excludeCron: false }), + ).toBe(false); + }); + + it("applies user-supplied regex patterns from excludeSessionKeyPatterns", () => { + expect( + shouldSkipAutoRecallForSession("agent:debug:gateway:hello", { + excludeCron: false, + excludeSessionKeyPatterns: ["^agent:debug:"], + }), + ).toBe(true); + + expect( + shouldSkipAutoRecallForSession("agent:main:chat:hello", { + excludeCron: false, + excludeSessionKeyPatterns: ["^agent:debug:"], + }), + ).toBe(false); + }); + + it("falls back gracefully when a regex pattern is invalid", () => { + // '[' is an invalid regex; should be ignored, not throw + expect(() => + shouldSkipAutoRecallForSession("agent:main:chat:hello", { + excludeCron: false, + excludeSessionKeyPatterns: ["["], + }), + ).not.toThrow(); + expect( + shouldSkipAutoRecallForSession("agent:main:chat:hello", { + excludeCron: false, + excludeSessionKeyPatterns: ["["], + }), + ).toBe(false); + }); + + it("combines excludeCron and excludeSessionKeyPatterns (OR semantics)", () => { + expect( + shouldSkipAutoRecallForSession("agent:main:cron:abc", { + excludeCron: true, + excludeSessionKeyPatterns: ["nope"], + }), + ).toBe(true); + }); +}); diff --git a/apps/memos-local-openclaw/tests/shutdown-lifecycle.test.ts b/apps/memos-local-openclaw/tests/shutdown-lifecycle.test.ts index f1471cd7e..b059b8594 100644 --- a/apps/memos-local-openclaw/tests/shutdown-lifecycle.test.ts +++ b/apps/memos-local-openclaw/tests/shutdown-lifecycle.test.ts @@ -113,6 +113,7 @@ describe("shutdown lifecycle", () => { resolvePath: () => "/tmp/memos-service-stop", logger: noopLog, registerTool: () => {}, + registerMemoryPromptSection: () => {}, registerMemoryCapability: () => {}, registerService: (service: any) => { registeredService = service; }, on: () => {}, diff --git a/apps/memos-local-openclaw/tests/storage.test.ts b/apps/memos-local-openclaw/tests/storage.test.ts index f449b868d..b76cc708c 100644 --- a/apps/memos-local-openclaw/tests/storage.test.ts +++ b/apps/memos-local-openclaw/tests/storage.test.ts @@ -137,6 +137,51 @@ describe("SqliteStore", () => { const recent0 = store.getRecentEmbeddings(0); expect(recent0.length).toBe(1); }); + + it("ftsSearch excludes the chunks whose session_key matches excludeSessionKey", () => { + store.insertChunk(makeChunk({ id: "c1", sessionKey: "sess-current", content: "Deploy the application to production", summary: "deployment" })); + store.insertChunk(makeChunk({ id: "c2", sessionKey: "sess-other", content: "Deploy the application to production", summary: "deployment" })); + + const all = store.ftsSearch("deploy production", 10); + expect(all.map((r) => r.chunkId).sort()).toEqual(["c1", "c2"]); + + const filtered = store.ftsSearch("deploy production", 10, undefined, "sess-current"); + expect(filtered.map((r) => r.chunkId)).toEqual(["c2"]); + }); + + it("ftsSearch with excludeSessionKey + ownerFilter applies both filters", () => { + store.insertChunk(makeChunk({ id: "c1", sessionKey: "sess-current", owner: "agent:main", content: "Deploy production runbook" })); + store.insertChunk(makeChunk({ id: "c2", sessionKey: "sess-other", owner: "agent:main", content: "Deploy production runbook" })); + store.insertChunk(makeChunk({ id: "c3", sessionKey: "sess-other", owner: "agent:bot", content: "Deploy production runbook" })); + + const filtered = store.ftsSearch("deploy production", 10, ["agent:main"], "sess-current"); + expect(filtered.map((r) => r.chunkId)).toEqual(["c2"]); + }); + + it("patternSearch excludes the chunks whose session_key matches excludeSessionKey", () => { + store.insertChunk(makeChunk({ id: "c1", sessionKey: "sess-current", content: "唐波是工程师" })); + store.insertChunk(makeChunk({ id: "c2", sessionKey: "sess-other", content: "唐波是工程师" })); + + const all = store.patternSearch(["唐波"], { limit: 10 }); + expect(all.map((r) => r.chunkId).sort()).toEqual(["c1", "c2"]); + + const filtered = store.patternSearch(["唐波"], { limit: 10, excludeSessionKey: "sess-current" }); + expect(filtered.map((r) => r.chunkId)).toEqual(["c2"]); + }); + + it("getAllEmbeddings + getRecentEmbeddings exclude chunks matching excludeSessionKey", () => { + const base = Date.now() - 5000; + store.insertChunk(makeChunk({ id: "c1", sessionKey: "sess-current", createdAt: base })); + store.upsertEmbedding("c1", [0.1, 0.2, 0.3]); + store.insertChunk(makeChunk({ id: "c2", sessionKey: "sess-other", createdAt: base + 1000 })); + store.upsertEmbedding("c2", [0.4, 0.5, 0.6]); + + const allFiltered = store.getAllEmbeddings(undefined, "sess-current"); + expect(allFiltered.map((r) => r.chunkId)).toEqual(["c2"]); + + const recentFiltered = store.getRecentEmbeddings(10, undefined, "sess-current"); + expect(recentFiltered.map((r) => r.chunkId)).toEqual(["c2"]); + }); }); describe("SqliteStore hub sharing schema", () => { @@ -528,6 +573,24 @@ describe("vectorSearch", () => { const cappedIds = new Set(cappedHits.map((h) => h.chunkId)); expect(cappedIds.size).toBeLessThanOrEqual(2); }); + + it("with excludeSessionKey filters out chunks belonging to that session", () => { + const base = Date.now() - 5000; + store.insertChunk(makeChunk({ id: "c-cur", sessionKey: "sess-current", createdAt: base })); + store.upsertEmbedding("c-cur", [1, 0, 0, 0]); + store.insertChunk(makeChunk({ id: "c-oth", sessionKey: "sess-other", createdAt: base + 1000 })); + store.upsertEmbedding("c-oth", [1, 0, 0, 0]); + + const queryVec = [1, 0, 0, 0]; + const all = vectorSearch(store, queryVec, 10); + expect(all.map((h) => h.chunkId).sort()).toEqual(["c-cur", "c-oth"]); + + const filtered = vectorSearch(store, queryVec, 10, undefined, undefined, "sess-current"); + expect(filtered.map((h) => h.chunkId)).toEqual(["c-oth"]); + + const filteredWithCap = vectorSearch(store, queryVec, 10, 5, undefined, "sess-current"); + expect(filteredWithCap.map((h) => h.chunkId)).toEqual(["c-oth"]); + }); }); describe("cosineSimilarity", () => { diff --git a/apps/memos-local-openclaw/tests/topic-judge-minimax-1315.test.ts b/apps/memos-local-openclaw/tests/topic-judge-minimax-1315.test.ts new file mode 100644 index 000000000..afd4ac750 --- /dev/null +++ b/apps/memos-local-openclaw/tests/topic-judge-minimax-1315.test.ts @@ -0,0 +1,132 @@ +/** + * Regression test for issue #1315: + * Topic Judge 100% failure rate against MiniMax (api.minimaxi.com) because + * judgeNewTopicOpenAI / arbitrateTopicSplitOpenAI request max_tokens: 10, + * which MiniMax's gateway rejects with an HTML 404 page. + * + * The fix raises the minimum to 60 (matching classifyTopicOpenAI in the same + * file, which is already proven to work against MiniMax). These tests assert + * the on-the-wire request body uses at least 60 max_tokens for the two + * affected helpers; if anyone lowers them back to 10 the tests fail. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { + judgeNewTopicOpenAI, + arbitrateTopicSplitOpenAI, + classifyTopicOpenAI, + filterRelevantOpenAI, + judgeDedupOpenAI, + summarizeOpenAI, +} from "../src/ingest/providers/openai"; +import type { SummarizerConfig, Logger } from "../src/types"; + +const silentLog: Logger = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, +}; + +const minimaxCfg: SummarizerConfig = { + provider: "openai_compatible", + model: "MiniMax-M2.7-highspeed", + endpoint: "https://api.minimaxi.com/v1", + apiKey: "test-key", +}; + +interface CapturedRequest { + url: string; + body: Record; +} + +/** + * Replace global.fetch with a recorder that returns a canned successful + * completion. Returns the captured-requests array so tests can assert on + * url / body / max_tokens. + */ +function installFetchRecorder(replyContent: string): CapturedRequest[] { + const captured: CapturedRequest[] = []; + const fakeFetch = vi.fn(async (url: string | URL, init?: RequestInit) => { + const body = init?.body ? JSON.parse(init.body as string) : {}; + captured.push({ url: String(url), body }); + return new Response( + JSON.stringify({ + choices: [{ message: { content: replyContent } }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + }); + vi.stubGlobal("fetch", fakeFetch); + return captured; +} + +describe("openai topic-judge max_tokens regression (issue #1315)", () => { + beforeEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("judgeNewTopicOpenAI sends max_tokens >= 60 (MiniMax rejects max_tokens: 10 with HTML 404)", async () => { + const captured = installFetchRecorder("SAME"); + + await judgeNewTopicOpenAI("current task context", "new user message", minimaxCfg, silentLog); + + expect(captured).toHaveLength(1); + expect(captured[0].url).toBe("https://api.minimaxi.com/v1/chat/completions"); + const maxTokens = captured[0].body.max_tokens as number; + expect(maxTokens).toBeGreaterThanOrEqual(60); + }); + + it("arbitrateTopicSplitOpenAI sends max_tokens >= 60 (same MiniMax 404 gateway behaviour)", async () => { + const captured = installFetchRecorder("NEW"); + + await arbitrateTopicSplitOpenAI("task state", "new message", minimaxCfg, silentLog); + + expect(captured).toHaveLength(1); + expect(captured[0].url).toBe("https://api.minimaxi.com/v1/chat/completions"); + const maxTokens = captured[0].body.max_tokens as number; + expect(maxTokens).toBeGreaterThanOrEqual(60); + }); + + it("judgeNewTopicOpenAI still parses single-word NEW / SAME replies after the bump", async () => { + installFetchRecorder("NEW"); + const isNew = await judgeNewTopicOpenAI("ctx", "msg", minimaxCfg, silentLog); + expect(isNew).toBe(true); + + installFetchRecorder("SAME"); + const isSame = await judgeNewTopicOpenAI("ctx", "msg", minimaxCfg, silentLog); + expect(isSame).toBe(false); + }); + + it("arbitrateTopicSplitOpenAI still normalises replies to NEW or SAME after the bump", async () => { + installFetchRecorder("NEW\n"); + expect(await arbitrateTopicSplitOpenAI("task", "msg", minimaxCfg, silentLog)).toBe("NEW"); + + installFetchRecorder("same"); + expect(await arbitrateTopicSplitOpenAI("task", "msg", minimaxCfg, silentLog)).toBe("SAME"); + }); + + it("other openai helpers keep their existing max_tokens limits (no regression on healthy callers)", async () => { + const a = installFetchRecorder("60"); + await classifyTopicOpenAI("task", "msg", minimaxCfg, silentLog); + // classifyTopic was already 60 — should remain at least 60. + expect(a[0].body.max_tokens as number).toBeGreaterThanOrEqual(60); + + const b = installFetchRecorder('{"relevant":[],"sufficient":false}'); + await filterRelevantOpenAI("q", [{ index: 1, role: "user", content: "c" }], minimaxCfg, silentLog); + expect(b[0].body.max_tokens as number).toBeGreaterThanOrEqual(200); + + const c = installFetchRecorder('{"action":"NEW","reason":""}'); + await judgeDedupOpenAI("new", [{ index: 1, summary: "s", chunkId: "x" }], minimaxCfg, silentLog); + expect(c[0].body.max_tokens as number).toBeGreaterThanOrEqual(300); + + const d = installFetchRecorder("hello world summary"); + await summarizeOpenAI("input text", minimaxCfg, silentLog); + // summarize does not set max_tokens (server default) — assert the field is absent / unset. + expect(d[0].body.max_tokens).toBeUndefined(); + }); +}); diff --git a/apps/memos-local-openclaw/tests/update-install.test.ts b/apps/memos-local-openclaw/tests/update-install.test.ts index de4ca2aa5..efadafccd 100644 --- a/apps/memos-local-openclaw/tests/update-install.test.ts +++ b/apps/memos-local-openclaw/tests/update-install.test.ts @@ -41,7 +41,13 @@ function invokeUpdateInstall(viewer: ViewerServer, body: unknown): Promise<{ sta this.statusCode = code; this.headers = headers; }, - end(payload: string) { + // `jsonResponseAndRestart` passes a "flushed" callback to `res.end` and only + // schedules the SIGUSR1 timer once that callback fires. Real + // `http.ServerResponse` invokes it as soon as the payload is written; the mock + // must do the same, otherwise the fake timers never see the setTimeout and the + // restart signal is silently dropped in the test. See issue #1559 review round 2. + end(payload: string, cb?: () => void) { + if (typeof cb === "function") cb(); resolve({ statusCode: this.statusCode, data: JSON.parse(payload) }); }, } as any; @@ -181,7 +187,9 @@ describe("viewer update-install", () => { expect(JSON.parse(fs.readFileSync(path.join(extDir, "package.json"), "utf8")).version).toBe("2.0.0-beta.2"); expect(fs.readdirSync(path.dirname(extDir)).filter((name) => name.includes(".backup-"))).toHaveLength(0); - await vi.advanceTimersByTimeAsync(500); + // `jsonResponseAndRestart` schedules SIGUSR1 with a 1500ms delay by default; advance + // past that (2000ms buffer) so the fake timer definitely fires the restart signal. + await vi.advanceTimersByTimeAsync(2000); expect(killSpy).toHaveBeenCalledWith(process.pid, "SIGUSR1"); }); }); diff --git a/apps/memos-local-openclaw/tests/verify-npm-package.sh b/apps/memos-local-openclaw/tests/verify-npm-package.sh new file mode 100755 index 000000000..94bdc28d3 --- /dev/null +++ b/apps/memos-local-openclaw/tests/verify-npm-package.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Test script to verify the npm package includes compiled output + +set -e + +echo "Testing @memtensor/memos-local-openclaw-plugin package..." + +cd "$(dirname "$0")/.." + +# Run build +echo "1. Running build..." +npm run build + +# Check dist/ directory exists +if [ ! -d "dist" ]; then + echo "✗ FAIL: dist/ directory not found" + exit 1 +fi +echo "✓ dist/ directory exists" + +# Check main entry point +if [ ! -f "dist/index.js" ]; then + echo "✗ FAIL: dist/index.js not found" + exit 1 +fi +echo "✓ dist/index.js exists" + +# Check type declarations +if [ ! -f "dist/index.d.ts" ]; then + echo "✗ FAIL: dist/index.d.ts not found" + exit 1 +fi +echo "✓ dist/index.d.ts exists" + +# Check compiled src files +if [ ! -d "dist/src" ]; then + echo "✗ FAIL: dist/src/ directory not found" + exit 1 +fi +echo "✓ dist/src/ directory exists" + +# Verify package.json points to compiled output +MAIN_ENTRY=$(node -p "require('./package.json').main") +if [ "$MAIN_ENTRY" != "dist/index.js" ]; then + echo "✗ FAIL: package.json main field is '$MAIN_ENTRY', expected 'dist/index.js'" + exit 1 +fi +echo "✓ package.json main points to dist/index.js" + +# Verify openclaw.extensions points to compiled output +OPENCLAW_EXT=$(node -p "require('./package.json').openclaw.extensions[0]") +if [ "$OPENCLAW_EXT" != "./dist/index.js" ]; then + echo "✗ FAIL: openclaw.extensions is '$OPENCLAW_EXT', expected './dist/index.js'" + exit 1 +fi +echo "✓ openclaw.extensions points to ./dist/index.js" + +# Simulate npm pack and verify dist is included +echo "2. Simulating npm pack..." +PACK_OUTPUT=$(npm pack --dry-run 2>&1) +if ! echo "$PACK_OUTPUT" | grep -q "dist/index.js"; then + echo "✗ FAIL: dist/index.js not included in npm package" + exit 1 +fi +echo "✓ dist/index.js will be included in npm package" + +if ! echo "$PACK_OUTPUT" | grep -q "dist/src/"; then + echo "✗ FAIL: dist/src/ not included in npm package" + exit 1 +fi +echo "✓ dist/src/ will be included in npm package" + +# Verify TypeScript source is NOT included (but .d.ts type declarations are OK) +if echo "$PACK_OUTPUT" | grep -E "npm notice.*(index\.ts|src/.+\.ts)" | grep -v "\.d\.ts" | grep -q .; then + echo "✗ FAIL: TypeScript source files should not be included in npm package" + exit 1 +fi +echo "✓ TypeScript source files excluded from npm package (type declarations OK)" + +echo "" +echo "✓ All tests passed!" +echo "The package now correctly ships compiled JavaScript output in dist/" diff --git a/apps/memos-local-openclaw/tests/viewer-env-resolution.test.ts b/apps/memos-local-openclaw/tests/viewer-env-resolution.test.ts new file mode 100644 index 000000000..4acc7abd7 --- /dev/null +++ b/apps/memos-local-openclaw/tests/viewer-env-resolution.test.ts @@ -0,0 +1,174 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { SqliteStore } from "../src/storage/sqlite"; +import { ViewerServer } from "../src/viewer/server"; + +const noopLog = { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} }; + +let tmpDir = ""; +let originalEnv: NodeJS.ProcessEnv; +let store: SqliteStore | null = null; + +beforeEach(() => { + originalEnv = { ...process.env }; + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-viewer-env-")); + process.env.OPENCLAW_STATE_DIR = tmpDir; + process.env.OPENCLAW_CONFIG_PATH = path.join(tmpDir, "openclaw.json"); +}); + +afterEach(() => { + store?.close(); + store = null; + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); + tmpDir = ""; + // Restore env without leaking test keys + for (const k of Object.keys(process.env)) { + if (!(k in originalEnv)) delete process.env[k]; + } + for (const [k, v] of Object.entries(originalEnv)) { + process.env[k] = v; + } +}); + +function makeViewer(): ViewerServer { + store = new SqliteStore(path.join(tmpDir, "test.db"), noopLog); + return new ViewerServer({ + store, + embedder: { provider: "local" } as any, + port: 19998, + log: noopLog, + dataDir: tmpDir, + }); +} + +describe("viewer env var resolution", () => { + it("handleTestModel should resolve ${VAR} in apiKey/endpoint before sending", async () => { + process.env.MY_FAKE_API_KEY = "sk-resolved-key"; + process.env.MY_FAKE_ENDPOINT = "https://example.test/v1"; + + const viewer = makeViewer(); + + // Stub fetch to capture the headers/url that the test path would call + const captured: { url: string; headers: Record; body: string } = { url: "", headers: {}, body: "" }; + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (input: any, init?: any) => { + captured.url = String(input); + captured.headers = { ...((init?.headers as Record) ?? {}) }; + captured.body = String(init?.body ?? ""); + return { + ok: true, + status: 200, + text: async () => "", + json: async () => ({ data: [{ embedding: [1, 2, 3] }] }), + } as any; + }) as any; + + try { + const dim = await (viewer as any).testEmbeddingModel( + "openai", + "text-embedding-3-small", + "${MY_FAKE_ENDPOINT}", + "${MY_FAKE_API_KEY}", + ); + expect(dim).toBe(3); + // The Authorization header must contain the resolved key, not the literal ${VAR}. + expect(captured.headers.Authorization).toBe("Bearer sk-resolved-key"); + expect(captured.url).toContain("https://example.test/v1"); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("serveFallbackModel should resolve ${VAR} in providers.baseUrl/apiKey", () => { + process.env.MY_FAKE_BASE = "https://provider.test/v1"; + process.env.MY_FAKE_API_KEY = "sk-resolved-key"; + + fs.writeFileSync( + process.env.OPENCLAW_CONFIG_PATH!, + JSON.stringify({ + agents: { defaults: { model: { primary: "openai/gpt-4o-mini" } } }, + models: { + providers: { + openai: { + baseUrl: "${MY_FAKE_BASE}", + apiKey: "${MY_FAKE_API_KEY}", + }, + }, + }, + }), + ); + + const viewer = makeViewer(); + let payload: any = null; + const res: any = { + writeHead() {}, + end(body: string) { payload = JSON.parse(body); }, + }; + (viewer as any).serveFallbackModel(res); + expect(payload).toBeTruthy(); + expect(payload.available).toBe(true); + expect(payload.baseUrl).toBe("https://provider.test/v1"); + expect(payload.model).toBe("gpt-4o-mini"); + }); + + it("readPluginConfigResolved reads openclaw.json and resolves ${VAR} into apiKey", () => { + process.env.MY_FAKE_SUM_KEY = "sk-resolved-summary"; + fs.writeFileSync( + process.env.OPENCLAW_CONFIG_PATH!, + JSON.stringify({ + plugins: { + entries: { + "memos-local-openclaw-plugin": { + enabled: true, + config: { + summarizer: { + provider: "openai", + apiKey: "${MY_FAKE_SUM_KEY}", + }, + }, + }, + }, + }, + }), + ); + + const viewer = makeViewer(); + const cfg = (viewer as any).readPluginConfigResolved(); + expect(cfg.summarizer.apiKey).toBe("sk-resolved-summary"); + }); + + it("serveConfig should keep raw ${VAR} literals so UI can still edit them", () => { + process.env.MY_FAKE_EMB_KEY = "sk-should-stay-hidden"; + fs.writeFileSync( + process.env.OPENCLAW_CONFIG_PATH!, + JSON.stringify({ + plugins: { + entries: { + "memos-local-openclaw-plugin": { + enabled: true, + config: { + embedding: { + provider: "openai", + apiKey: "${MY_FAKE_EMB_KEY}", + }, + }, + }, + }, + }, + }), + ); + + const viewer = makeViewer(); + let payload: any = null; + const res: any = { + writeHead() {}, + end(body: string) { payload = JSON.parse(body); }, + }; + (viewer as any).serveConfig(res); + expect(payload).toBeTruthy(); + // UI should still see the raw env var literal so users can keep secrets out of the file. + expect(payload.embedding.apiKey).toBe("${MY_FAKE_EMB_KEY}"); + }); +}); diff --git a/apps/memos-local-openclaw/tests/viewer-search-params.test.ts b/apps/memos-local-openclaw/tests/viewer-search-params.test.ts new file mode 100644 index 000000000..5ae09d81c --- /dev/null +++ b/apps/memos-local-openclaw/tests/viewer-search-params.test.ts @@ -0,0 +1,262 @@ +import { afterEach, describe, expect, it } from "vitest"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SqliteStore } from "../src/storage/sqlite"; +import { ViewerServer } from "../src/viewer/server"; + +/** + * Regression coverage for issue #1372 — `/api/search?q=...&limit=...&minScore=...` + * must respect both query parameters. + * + * Before the fix: + * - `limit` was never read; merged results were returned in full and the + * fallback hard-coded `.slice(0, 20)`. + * - `minScore` was never read; the semantic gate was the constant 0.64. + * + * After the fix the handler clamps `limit` to [1, 100] (default 20) and + * `minScore` to [0.35, 1] (default 0.64), echoes both values in the response, + * and truncates the result set accordingly. + */ + +const noopLog = { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} }; + +let tmpDirs: string[] = []; +let stores: SqliteStore[] = []; +let viewer: ViewerServer | null = null; + +afterEach(() => { + viewer?.stop(); + viewer = null; + for (const store of stores.splice(0)) store.close(); + for (const dir of tmpDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +async function viewerAuthCookie(baseUrl: string) { + const res = await fetch(`${baseUrl}/api/auth/setup`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ password: "passw0rd" }), + }); + const setCookie = res.headers.get("set-cookie") || ""; + return setCookie.split(";")[0]; +} + +interface SeededChunk { + id: string; + content: string; + vector: number[]; +} + +/** + * Embedder stub: returns a fixed query vector and exposes a deterministic + * embed() for backfill calls (returns the zero vector so it never affects + * scoring). The real ViewerServer code only calls `embedQuery()` inside + * `serveSearch`, so this is the only critical knob. + */ +function makeStubEmbedder(queryVector: number[]) { + const dim = queryVector.length; + return { + provider: "local", + dimensions: dim, + async embed(texts: string[]) { + return texts.map(() => new Array(dim).fill(0)); + }, + async embedQuery(_text: string) { + return queryVector; + }, + } as any; +} + +function seedChunks(store: SqliteStore, chunks: SeededChunk[]) { + const now = Date.now(); + for (let i = 0; i < chunks.length; i++) { + const c = chunks[i]; + store.insertChunk({ + id: c.id, + sessionKey: "session-1372", + turnId: `turn-${i}`, + seq: i, + role: "assistant", + content: c.content, + kind: "paragraph", + summary: c.content, + embedding: null, + taskId: null, + skillId: null, + owner: "agent:main", + dedupStatus: "active", + dedupTarget: null, + dedupReason: null, + mergeCount: 0, + lastHitAt: null, + mergeHistory: "[]", + createdAt: now + i, + updatedAt: now + i, + } as any); + store.upsertEmbedding(c.id, c.vector); + } +} + +function pickPort() { + return 19400 + Math.floor(Math.random() * 400); +} + +describe("ViewerServer /api/search query parameter handling (issue #1372)", () => { + it("respects ?limit= by truncating the returned result list", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-search-limit-")); + tmpDirs.push(dir); + const store = new SqliteStore(path.join(dir, "viewer.db"), noopLog as any); + stores.push(store); + + // Five chunks, all containing the literal "rollout" so FTS will return + // every row; vector path is disabled by making embedQuery reject below. + const seeded = Array.from({ length: 5 }, (_, i) => ({ + id: `chunk-rollout-${i}`, + content: `rollout checklist step ${i + 1}`, + vector: new Array(8).fill(0), + })); + seedChunks(store, seeded); + + const embedder = { + provider: "local", + dimensions: 8, + async embed(texts: string[]) { return texts.map(() => new Array(8).fill(0)); }, + async embedQuery() { throw new Error("vector path disabled for this test"); }, + } as any; + + viewer = new ViewerServer({ + store, + embedder, + port: pickPort(), + log: noopLog as any, + dataDir: dir, + }); + const url = await viewer.start(); + const cookie = await viewerAuthCookie(url); + + const limited = await fetch(`${url}/api/search?q=rollout&limit=3`, { headers: { cookie } }); + expect(limited.status).toBe(200); + const limitedJson = await limited.json(); + expect(limitedJson.results.length).toBe(3); + expect(limitedJson.total).toBe(3); + expect(limitedJson.limit).toBe(3); + + const defaultLimit = await fetch(`${url}/api/search?q=rollout`, { headers: { cookie } }); + const defaultJson = await defaultLimit.json(); + expect(defaultJson.limit).toBe(20); + expect(defaultJson.results.length).toBe(5); // only 5 chunks seeded, all returned within default cap + }); + + it("clamps ?limit= into [1, 100]", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-search-clamp-")); + tmpDirs.push(dir); + const store = new SqliteStore(path.join(dir, "viewer.db"), noopLog as any); + stores.push(store); + + seedChunks(store, [ + { id: "chunk-clamp-1", content: "rollout clamp one", vector: new Array(8).fill(0) }, + { id: "chunk-clamp-2", content: "rollout clamp two", vector: new Array(8).fill(0) }, + ]); + + viewer = new ViewerServer({ + store, + embedder: { + provider: "local", dimensions: 8, + async embed(texts: string[]) { return texts.map(() => new Array(8).fill(0)); }, + async embedQuery() { throw new Error("vector path disabled"); }, + } as any, + port: pickPort(), + log: noopLog as any, + dataDir: dir, + }); + const url = await viewer.start(); + const cookie = await viewerAuthCookie(url); + + const tooSmall = await (await fetch(`${url}/api/search?q=rollout&limit=0`, { headers: { cookie } })).json(); + expect(tooSmall.limit).toBe(20); // 0 is falsy → falls back to default 20 + + const negative = await (await fetch(`${url}/api/search?q=rollout&limit=-5`, { headers: { cookie } })).json(); + expect(negative.limit).toBe(20); // negative → falls back to default 20 + + const tooLarge = await (await fetch(`${url}/api/search?q=rollout&limit=9999`, { headers: { cookie } })).json(); + expect(tooLarge.limit).toBe(100); + + const garbage = await (await fetch(`${url}/api/search?q=rollout&limit=NaN`, { headers: { cookie } })).json(); + expect(garbage.limit).toBe(20); + }); + + it("respects ?minScore= by raising the semantic similarity gate", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-search-score-")); + tmpDirs.push(dir); + const store = new SqliteStore(path.join(dir, "viewer.db"), noopLog as any); + stores.push(store); + + // Three chunks with engineered vectors so cosine similarity vs the query + // vector [1,0,0,0] is deterministic: + // - high ≈ 1.00 (passes minScore=0.8) + // - medium ≈ 0.70 (passes default 0.64 but fails 0.8) + // - low ≈ 0.40 (fails both) + // All chunks contain the literal "vector" so FTS also returns them; the + // test asserts behavior on the merged response. + seedChunks(store, [ + { id: "chunk-vec-high", content: "vector high similarity", vector: [1, 0, 0, 0] }, + { id: "chunk-vec-med", content: "vector medium similarity", vector: [0.7, Math.sqrt(1 - 0.49), 0, 0] }, + { id: "chunk-vec-low", content: "vector low similarity", vector: [0.4, Math.sqrt(1 - 0.16), 0, 0] }, + ]); + + const embedder = makeStubEmbedder([1, 0, 0, 0]); + + viewer = new ViewerServer({ + store, + embedder, + port: pickPort(), + log: noopLog as any, + dataDir: dir, + }); + const url = await viewer.start(); + const cookie = await viewerAuthCookie(url); + + const strict = await (await fetch(`${url}/api/search?q=vector&minScore=0.8&limit=10`, { headers: { cookie } })).json(); + expect(strict.minScore).toBe(0.8); + // With minScore=0.8 only the high-similarity vector chunk passes the + // semantic gate; the merged result list therefore starts with it and the + // FTS-only chunks (medium/low) are appended after — but the issue's + // contract is that the response should not silently dump every FTS row. + // Concretely, vectorCount must equal 1. + expect(strict.vectorCount).toBe(1); + + const lax = await (await fetch(`${url}/api/search?q=vector&minScore=0.64&limit=10`, { headers: { cookie } })).json(); + expect(lax.minScore).toBe(0.64); + expect(lax.vectorCount).toBeGreaterThanOrEqual(2); // high + medium both clear 0.64 + }); + + it("echoes default minScore (0.64) when the param is omitted", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-search-default-score-")); + tmpDirs.push(dir); + const store = new SqliteStore(path.join(dir, "viewer.db"), noopLog as any); + stores.push(store); + + seedChunks(store, [ + { id: "chunk-default-1", content: "rollout default one", vector: new Array(8).fill(0) }, + ]); + + viewer = new ViewerServer({ + store, + embedder: { + provider: "local", dimensions: 8, + async embed(texts: string[]) { return texts.map(() => new Array(8).fill(0)); }, + async embedQuery() { throw new Error("vector path disabled"); }, + } as any, + port: pickPort(), + log: noopLog as any, + dataDir: dir, + }); + const url = await viewer.start(); + const cookie = await viewerAuthCookie(url); + + const resp = await (await fetch(`${url}/api/search?q=rollout`, { headers: { cookie } })).json(); + expect(resp.limit).toBe(20); + expect(resp.minScore).toBe(0.64); + }); +}); diff --git a/apps/memos-local-openclaw/tsconfig.json b/apps/memos-local-openclaw/tsconfig.json index 53c08a1d2..a10713201 100644 --- a/apps/memos-local-openclaw/tsconfig.json +++ b/apps/memos-local-openclaw/tsconfig.json @@ -1,20 +1,25 @@ { "compilerOptions": { "target": "ES2022", - "module": "CommonJS", + "module": "ES2022", "lib": ["ES2022"], "outDir": "dist", - "rootDir": "src", - "strict": true, + "rootDir": ".", + "strict": false, "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "resolveJsonModule": true, + "allowSyntheticDefaultImports": true, "declaration": true, "declarationMap": true, "sourceMap": true, - "moduleResolution": "node" + "moduleResolution": "bundler", + "allowImportingTsExtensions": false, + "paths": { + "openclaw/plugin-sdk": ["./types/openclaw__plugin-sdk/index.d.ts"] + } }, - "include": ["src"], - "exclude": ["node_modules", "dist", "**/*.test.ts"] + "include": ["src", "index.ts"], + "exclude": ["node_modules", "dist", "**/*.test.ts", "scripts", "tests", "skill", "plugin-impl.ts"] } diff --git a/apps/memos-local-openclaw/types/openclaw__plugin-sdk/index.d.ts b/apps/memos-local-openclaw/types/openclaw__plugin-sdk/index.d.ts new file mode 100644 index 000000000..fc59b0e0f --- /dev/null +++ b/apps/memos-local-openclaw/types/openclaw__plugin-sdk/index.d.ts @@ -0,0 +1,11 @@ +/** + * Type declarations for openclaw/plugin-sdk + * This module is provided by the OpenClaw runtime environment + */ + +export interface OpenClawPluginApi { + registerTool(tool: any, options?: any): void; + [key: string]: any; +} + +export const plugin: OpenClawPluginApi; diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index fd9d0d827..c6524d149 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -48,10 +48,12 @@ import contextlib import json import logging +import os import re import sys import threading import time +import weakref from pathlib import Path from typing import Any @@ -89,6 +91,34 @@ class MemoryProvider: # type: ignore[no-redef] ) _TOOL_FAILURE_HINT_THRESHOLD = 3 + +def _long_rpc_timeout_default() -> float: + """Resolve the timeout used for long-running JSON-RPC calls. + + After 1-2 hours of Hermes use the memory / capture / reflection + pipeline grows past the 30s JSON-RPC default and surfaces as + ``[timeout] memory.search did not respond within 30.0s`` and + ``[timeout] turn.end did not respond within 30.0s`` in the host + logs (issue #2028). ``feedback.submit`` already opts into 75s; + ``sync_turn``'s ``_ensure_bridge`` also uses 75s. Aligning the + heavy retrieval / capture RPCs with the same 75s ceiling gives + the pipeline enough headroom without turning genuinely hung + calls into an indefinite wait. The value is overridable via + ``MEMOS_HERMES_LONG_RPC_TIMEOUT`` for site-specific tuning; any + unparseable / non-positive value falls back to the default. + """ + raw = os.environ.get("MEMOS_HERMES_LONG_RPC_TIMEOUT", "") + try: + value = float(raw) + except (TypeError, ValueError): + return 75.0 + if value <= 0: + return 75.0 + return value + + +_LONG_RPC_TIMEOUT = _long_rpc_timeout_default() + _HERMES_INTERNAL_REVIEW_PREFIXES = ( "review the conversation above and consider saving to memory if appropriate.", "review the conversation above and update the skill library.", @@ -304,11 +334,48 @@ def initialize(self, session_id: str, **kwargs: Any) -> None: # type: ignore[ov is deferred to ``_ensure_episode()`` (called from the first ``on_turn_start``), so the actual user message can be passed as the episode's initial text instead of a generic placeholder. + + Idempotency (issue #1910): the host occasionally re-enters + ``initialize()`` on the same provider instance (plugin reload, + session restart). Without the close-before-spawn guard below the + previous ``MemosBridgeClient`` would be replaced by reference + only, orphaning the previous Node subprocess and accumulating + a fresh ``bridge.cjs`` per turn. """ + # Hermes can call `initialize()` multiple times across a single + # parent process (e.g. on reconnect or a new session). Each call + # spawns a fresh `MemosBridgeClient` (and therefore a new + # `--no-viewer` Node subprocess); if we simply overwrite + # `self._bridge` we leak the old subprocess — its parent stays + # alive so `_reap_stale_headless_bridges_locked` never collects + # it. Close the previous bridge first, mirroring the safe pattern + # already used by `_reconnect_bridge()`. See #1927. + previous_bridge = self._bridge + if previous_bridge is not None: + old_pid = getattr(previous_bridge, "pid", "?") + logger.info( + "MemOS: closing previous bridge (pid=%s) before re-init", + old_pid, + ) + with contextlib.suppress(Exception): + previous_bridge.close() + self._bridge = None + self._session_id = session_id or self._session_id self._hermes_home = str(kwargs.get("hermes_home") or "") self._platform = str(kwargs.get("platform") or "cli") self._agent_identity = str(kwargs.get("agent_identity") or "hermes") + if self._bridge is not None: + prev_pid = getattr(self._bridge, "pid", "?") + logger.info( + "MemOS: initialize() invoked while bridge already exists; " + "closing previous bridge (pid=%s) before respawn", + prev_pid, + ) + old_bridge = self._bridge + self._bridge = None + with contextlib.suppress(Exception): + old_bridge.close() try: ensure_bridge_running() except Exception as err: @@ -1280,9 +1347,10 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) } if bool(args.get("sessionScope", False)): params["sessionId"] = self._session_id - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.search", params, + timeout=_LONG_RPC_TIMEOUT, ) return json.dumps({"hits": resp.get("hits", [])}) if tool_name == "memos_get": @@ -1298,7 +1366,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) method = methods.get(kind) if method is None: return json.dumps({"error": f"unknown memory kind: {kind}"}) - item = self._bridge.request( + item = self._bridge_request_with_retry( method, {"id": item_id, "namespace": self._runtime_namespace()} ) if not item: @@ -1343,7 +1411,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) } ) if tool_name == "memos_timeline": - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.timeline", { "episodeId": args.get("episodeId", self._episode_id), @@ -1358,12 +1426,12 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) params = {"limit": limit, "namespace": self._runtime_namespace()} if args.get("status"): params["status"] = args["status"] - return json.dumps(self._bridge.request("skill.list", params)) + return json.dumps(self._bridge_request_with_retry("skill.list", params)) if tool_name == "memos_environment": query = (args.get("query") or "").strip() limit = self._int_arg(args, "limit", 5, 1, 30) if not query: - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.list_world_models", {"limit": limit, "offset": 0, "namespace": self._runtime_namespace()}, ) @@ -1379,7 +1447,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) "queried": False, } ) - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.search", { "agent": "hermes", @@ -1387,6 +1455,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) "query": query, "topK": {"tier1": 0, "tier2": 0, "tier3": limit}, }, + timeout=_LONG_RPC_TIMEOUT, ) hits = [ h @@ -1412,7 +1481,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) skill_id = (args.get("id") or "").strip() if not skill_id: return json.dumps({"error": "missing id"}) - skill = self._bridge.request( + skill = self._bridge_request_with_retry( "skill.get", { "id": skill_id, @@ -1509,6 +1578,19 @@ def on_session_end(self, messages: list[dict[str, Any]]) -> None: # type: ignor with contextlib.suppress(Exception): self._bridge.request("session.close", {"sessionId": self._session_id}) + def __del__(self) -> None: + # Safety net — if shutdown() was never called (e.g. caller forgot, + # Hermes agent routed model change with self.agent = None), clean + # up the bridge subprocess and keepalive thread on GC. + if self._bridge is not None or ( + self._bridge_keepalive_thread is not None and self._bridge_keepalive_thread.is_alive() + ): + logger.warning( + "MemOS: __del__ cleaning up leaked provider — shutdown() was never called" + ) + with contextlib.suppress(Exception): + self.shutdown() + def shutdown(self) -> None: # type: ignore[override] self._bridge_keepalive_stop.set() if self._bridge_keepalive_thread and self._bridge_keepalive_thread.is_alive(): @@ -1718,12 +1800,81 @@ def _open_session(self, session_id: str = "", *, timeout: float = 30.0) -> None: ) self._session_id = resp.get("sessionId") or requested_session + def _bridge_request_with_retry( + self, + method: str, + params: Any, + *, + timeout: float | None = None, + ) -> dict[str, Any]: + """Read-path helper: reconnect + retry once on ``transport_closed``. + + Layer 4 (#2028): the read-path memory tools previously issued a + single ``self._bridge.request(...)`` and surfaced any error + verbatim to the model. When the Node bridge has died since the + last user turn, that first call now raises + ``transport_closed`` fast (thanks to Layer 1/2). This helper + mirrors the pattern ``sync_turn`` already uses: reconnect the + bridge once and re-issue the same request. A second failure is + left to propagate — the ``except`` block in + ``handle_tool_call`` will surface the error text verbatim. + """ + assert self._bridge is not None + try: + if timeout is None: + return self._bridge.request(method, params) + return self._bridge.request(method, params, timeout=timeout) + except BridgeError as err: + if not self._is_transport_closed(err): + raise + logger.info( + "MemOS: bridge transport closed on %s; reconnecting and retrying once — %s", + method, + err, + ) + self._reconnect_bridge(self._session_id, timeout=30.0) + assert self._bridge is not None + if timeout is None: + return self._bridge.request(method, params) + return self._bridge.request(method, params, timeout=timeout) + def _is_transport_closed(self, err: Exception) -> bool: if isinstance(err, BridgeError) and err.code == "transport_closed": return True msg = str(err).lower() return "broken pipe" in msg or "bridge closed" in msg or "transport_closed" in msg + def _should_reconnect_after_keepalive_failure(self, err: Exception) -> bool: + """Decide whether a keepalive failure warrants a bridge reconnect. + + Layer 3 (#2028): the keepalive previously reconnected only on + ``BridgeError("transport_closed", …)``. A hung Node bridge + surfaces instead as ``BridgeError("timeout", …)`` (the client + gave up waiting for a response); that error was dropped at + DEBUG and the stale client kept being reused, so every + subsequent memory tool timed out for another 30 s. Reconnect + also when the subprocess has already exited (belt-and-braces + for hangs that didn't raise a transport error). + + A live subprocess raising a generic (non-transport) error must + NOT trigger a reconnect — otherwise transient parse noise + would create a reconnect storm. + """ + if self._is_transport_closed(err): + return True + if isinstance(err, BridgeError) and err.code == "timeout": + return True + # Ask the underlying subprocess: is it still alive? + bridge = self._bridge + if bridge is not None: + try: + exit_code = bridge._proc.poll() # type: ignore[attr-defined] + except Exception: + exit_code = None + if exit_code is not None: + return True + return False + def _reconnect_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> None: # Don't reconnect if we're shutting down if self._bridge_keepalive_stop.is_set(): @@ -1790,18 +1941,32 @@ def _start_bridge_keepalive(self) -> None: return self._bridge_keepalive_stop.clear() + _self_ref = weakref.ref(self) + def _run() -> None: - while not self._bridge_keepalive_stop.wait(5.0): - if not self._ensure_bridge(self._session_id, timeout=10.0): + while True: + # Stop signal set (e.g. shutdown called by another thread). + # When self is garbage-collected the weakref resolves to None + # and we exit gracefully instead of keeping the thread + bridge + # subprocess alive forever. + provider = _self_ref() + if provider is None: + break + if provider._bridge_keepalive_stop.wait(5.0): + break + if not provider._ensure_bridge(provider._session_id, timeout=10.0): continue try: - assert self._bridge is not None - self._bridge.request("core.health", {}, timeout=10.0) + assert provider._bridge is not None + provider._bridge.request("core.health", {}, timeout=10.0) except Exception as err: - if self._is_transport_closed(err): - logger.info("MemOS: bridge keepalive reconnecting after transport close") + if provider._should_reconnect_after_keepalive_failure(err): + logger.info( + "MemOS: bridge keepalive reconnecting after failure — %s", + err, + ) with contextlib.suppress(Exception): - self._reconnect_bridge(self._session_id, timeout=10.0) + provider._reconnect_bridge(provider._session_id, timeout=10.0) else: logger.debug("MemOS: bridge keepalive failed — %s", err) @@ -1829,6 +1994,7 @@ def _turn_start(self, query: str, *, session_id: str = "") -> str: }, "ts": int(time.time() * 1000), }, + timeout=_LONG_RPC_TIMEOUT, ) # Stash the real episode id the pipeline auto-created (V7 # §0.1 may have boundary-cut the previous episode and started @@ -1875,7 +2041,7 @@ def _turn_end( } if agent_thinking: payload["agentThinking"] = agent_thinking - result = self._bridge.request("turn.end", payload) + result = self._bridge.request("turn.end", payload, timeout=_LONG_RPC_TIMEOUT) # Capture the trace ID for feedback submission if result and isinstance(result, dict): trace_ids = result.get("traceIds", []) diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index 5b986ec66..bdc650667 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -32,6 +32,18 @@ HOST_HANDLER_WAIT_SECONDS = 5.0 +# ─── Module-level singleton tracker ───────────────────────────────────── +# Each entry maps a ``(agent, no_viewer)`` key to the most-recent active +# ``MemosBridgeClient`` for that slot. When a new client is constructed +# for an existing key, the previous client is closed synchronously so the +# Node-side ``bridge.cjs`` subprocess does not leak. +# +# This is the Python-side guard against issue #1910 (bridge process leak: +# every turn spawns new bridge.cjs). Defence in depth on the Node side +# lives in ``bridge.cts`` via ``bridge-stdio.pid``. +_ACTIVE_CLIENTS: dict[tuple[str, bool], MemosBridgeClient] = {} +_ACTIVE_CLIENTS_LOCK = threading.Lock() + def _installed_node_binary(plugin_root: Path) -> str | None: marker = plugin_root / ".memos-node-bin" @@ -45,9 +57,27 @@ def _installed_node_binary(plugin_root: Path) -> str | None: def _bridge_script(plugin_root: Path) -> Path: - compiled = plugin_root / "dist" / "bridge.cjs" - if compiled.exists(): - return compiled + """Pick the bridge entrypoint, preferring pure ESM over the CJS trampoline. + + Resolution order (issue #1736): + 1. ``dist/bridge.mjs`` — pure ESM compiled output, the only entry + that avoids the CJS↔ESM bridge that fails on Node ≥ 22. + 2. ``dist/bridge.cjs`` — legacy CommonJS compiled output, kept for + installations whose ``dist/`` predates the ESM entrypoint. + 3. ``bridge.mts`` — pure ESM TypeScript source for ``tsx``-driven + local development. + 4. ``bridge.cts`` — legacy CommonJS TypeScript source. Returned + as the last-resort default so error messages stay stable when + none of the candidates exist. + """ + candidates = ( + plugin_root / "dist" / "bridge.mjs", + plugin_root / "dist" / "bridge.cjs", + plugin_root / "bridge.mts", + ) + for candidate in candidates: + if candidate.exists(): + return candidate return plugin_root / "bridge.cts" @@ -111,16 +141,19 @@ def __init__( script = str(script_path) env = {**os.environ, **(extra_env or {})} - # Prefer the compiled CommonJS bridge from packaged installs. The raw - # TypeScript entry remains as a development fallback and needs `tsx` - # for stripping types plus `.js` → `.ts` import resolution. On Windows - # the `.bin/tsx` file is a shell shim, so use tsx's real JS entrypoint - # whenever we have to launch the source entry through a specific Node. + # Prefer the compiled JavaScript bridge — the new pure ESM + # ``dist/bridge.mjs`` (issue #1736) or the legacy ``dist/bridge.cjs`` + # — both run on plain ``node`` without any loader. The raw + # TypeScript entries remain as a development fallback and need + # ``tsx`` for stripping types plus ``.js`` → ``.ts`` import + # resolution. On Windows the ``.bin/tsx`` file is a shell shim, + # so use tsx's real JS entrypoint whenever we have to launch the + # source entry through a specific Node. tsx_cli = plugin_root / "node_modules" / "tsx" / "dist" / "cli.mjs" bridge_args = [script, f"--agent={agent}"] if no_viewer: bridge_args.append("--no-viewer") - if script_path.suffix == ".cjs": + if script_path.suffix in (".mjs", ".cjs"): cmd = [node, *bridge_args] elif tsx_cli.exists(): cmd = [node, str(tsx_cli), *bridge_args] @@ -156,6 +189,40 @@ def __init__( ) self._stderr_reader.start() + # Singleton tracking (issue #1910). Register ourselves as the + # active client for ``(agent, no_viewer)`` and reap any previous + # holder synchronously so its subprocess does not leak. The reap + # happens AFTER our reader threads are running, so the previous + # client's ``close()`` (which closes stdin and waits for exit) + # cannot interfere with our own startup. + self._singleton_agent = agent + self._singleton_no_viewer = bool(no_viewer) + previous = self._register_active() + if previous is not None and previous is not self: + prev_pid = getattr(previous, "pid", "?") + logger.info( + "MemOS: closing previous bridge client (pid=%s) before adopting new one (pid=%s)", + prev_pid, + self.pid, + ) + with contextlib.suppress(Exception): + previous.close() + + def _register_active(self) -> MemosBridgeClient | None: + """Register self as the active singleton; return the displaced client.""" + key = (self._singleton_agent, self._singleton_no_viewer) + with _ACTIVE_CLIENTS_LOCK: + previous = _ACTIVE_CLIENTS.get(key) + _ACTIVE_CLIENTS[key] = self + return previous + + def _unregister_active(self) -> None: + """Remove self from the active registry if we are still the current entry.""" + key = (self._singleton_agent, self._singleton_no_viewer) + with _ACTIVE_CLIENTS_LOCK: + if _ACTIVE_CLIENTS.get(key) is self: + _ACTIVE_CLIENTS.pop(key, None) + @property def pid(self) -> int: """Return the PID of the bridge subprocess.""" @@ -172,6 +239,18 @@ def request( ) -> dict[str, Any]: if self._closed: raise BridgeError("transport_closed", "bridge client is closed") + # Layer 2 (#2028): if the subprocess has already exited, bail + # BEFORE writing to the pipe. Otherwise the write silently + # buffers into a dead pipe and the caller parks on the full + # per-request timeout. + exit_code = None + with contextlib.suppress(Exception): + exit_code = self._proc.poll() + if exit_code is not None: + raise BridgeError( + "transport_closed", + f"bridge subprocess exited (code={exit_code})", + ) with self._lock: rpc_id = self._next_id self._next_id += 1 @@ -244,6 +323,12 @@ def close(self) -> None: self._closed = True self._host_handlers_cv.notify_all() + # Drop self from the module-level singleton tracker (issue #1910) + # BEFORE the potentially-slow stdin/SIGTERM/SIGKILL dance. We + # only evict the registry slot if we still own it — a newer + # client that displaced us must remain reachable. + self._unregister_active() + pid = self.pid # 1. Close stdin (triggers bridge's graceful exit) @@ -275,86 +360,113 @@ def close(self) -> None: logger.error("MemOS: bridge process %d could not be killed", pid) # 5. Clean up pending requests + self._abort_pending("bridge closed") + + # ─── Internals ── + + def _abort_pending(self, reason: str) -> None: + """Wake every parked JSON-RPC waiter with `transport_closed`. + + Called from both `close()` and the `_read_loop` `finally:` + block, so any pending request sees a real transport error + immediately instead of parking on its per-request timeout. + Also flips `_closed` and notifies host-handler waiters so a + second `request()` fails fast. + """ + with self._host_handlers_cv: + self._closed = True + self._host_handlers_cv.notify_all() with self._lock: for entry in list(self._pending.values()): entry["error"] = { "code": -32000, - "message": "bridge closed", + "message": reason, "data": {"code": "transport_closed"}, } entry["event"].set() self._pending.clear() - # ─── Internals ── - def _read_loop(self) -> None: assert self._proc.stdout is not None - for line in self._proc.stdout: - line = line.strip() - if not line: - continue - try: - msg = json.loads(line) - except json.JSONDecodeError: - logger.debug("bridge: malformed line: %r", line[:120]) - continue - if "id" in msg and msg["id"] is not None and ("result" in msg or "error" in msg): - self._resolve(msg) - continue - if msg.get("method") == "events.notify": - for cb in list(self._events): - try: - cb(msg.get("params") or {}) - except Exception: - logger.debug("event listener threw", exc_info=True) - continue - if msg.get("method") == "logs.forward": - for cb in list(self._logs): - try: - cb(msg.get("params") or {}) - except Exception: - logger.debug("log listener threw", exc_info=True) - continue - # Reverse-direction request: the bridge is asking the - # adapter to do something (e.g. run a fallback LLM call - # via `host.llm.complete`). Dispatch to the registered - # handler and write the response back synchronously. - method = msg.get("method") - rpc_id = msg.get("id") - if ( - isinstance(method, str) - and rpc_id is not None - and "result" not in msg - and "error" not in msg - ): - handler = self._host_handler_for(method) - if handler is None: - self._send_response( - rpc_id, - error={ - "code": -32601, - "message": f"method not found: {method}", - "data": {"code": "unknown_method"}, - }, - ) + try: + for line in self._proc.stdout: + line = line.strip() + if not line: continue - params = msg.get("params") or {} - if not isinstance(params, dict): - params = {} try: - result = handler(params) - self._send_response(rpc_id, result=result) - except Exception as err: - logger.warning("host handler %s failed: %s", method, err) - self._send_response( - rpc_id, - error={ - "code": -32000, - "message": str(err) or err.__class__.__name__, - "data": {"code": "host_handler_failed"}, - }, - ) - continue + msg = json.loads(line) + except json.JSONDecodeError: + logger.debug("bridge: malformed line: %r", line[:120]) + continue + if "id" in msg and msg["id"] is not None and ("result" in msg or "error" in msg): + self._resolve(msg) + continue + if msg.get("method") == "events.notify": + for cb in list(self._events): + try: + cb(msg.get("params") or {}) + except Exception: + logger.debug("event listener threw", exc_info=True) + continue + if msg.get("method") == "logs.forward": + for cb in list(self._logs): + try: + cb(msg.get("params") or {}) + except Exception: + logger.debug("log listener threw", exc_info=True) + continue + # Reverse-direction request: the bridge is asking the + # adapter to do something (e.g. run a fallback LLM call + # via `host.llm.complete`). Dispatch to the registered + # handler and write the response back synchronously. + method = msg.get("method") + rpc_id = msg.get("id") + if ( + isinstance(method, str) + and rpc_id is not None + and "result" not in msg + and "error" not in msg + ): + handler = self._host_handler_for(method) + if handler is None: + self._send_response( + rpc_id, + error={ + "code": -32601, + "message": f"method not found: {method}", + "data": {"code": "unknown_method"}, + }, + ) + continue + params = msg.get("params") or {} + if not isinstance(params, dict): + params = {} + try: + result = handler(params) + self._send_response(rpc_id, result=result) + except Exception as err: + logger.warning("host handler %s failed: %s", method, err) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": str(err) or err.__class__.__name__, + "data": {"code": "host_handler_failed"}, + }, + ) + continue + except Exception: + # Any unexpected exception in the reader loop still needs + # to fall through to the `finally:` cleanup so callers + # don't park on 30 s timeouts. + logger.debug("bridge reader thread crashed", exc_info=True) + finally: + # Layer 1 (#2028): the reader thread is the only signal + # that the Node subprocess has stopped answering. On any + # exit — normal EOF or exception — abort pending waiters + # immediately so callers get `transport_closed` in < 1 s + # instead of waiting for each 30 s per-request timeout. + self._abort_pending("bridge subprocess exited") def _host_handler_for( self, diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py index 19c8a7be0..3d3793396 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py @@ -34,6 +34,7 @@ _lock = threading.RLock() _bridge_ok: bool | None = None +_bridge_ok_at: float = 0.0 _viewer_status: str | None = None _viewer_last_probe_at = 0.0 _viewer_process: subprocess.Popen | None = None @@ -42,6 +43,11 @@ VIEWER_PROBE_TTL_SEC = 30.0 VIEWER_START_LOCK_TIMEOUT_SEC = 20.0 VIEWER_START_LOCK_STALE_SEC = 60.0 +# Bound how long a cached `_bridge_ok` answer is trusted. Without this, a +# single transient `_node_available()` failure during gateway startup +# (subprocess race, `.env` loaded after the first probe) would pin the +# provider to "unavailable" for the lifetime of the process — see #1797. +BRIDGE_OK_TTL_SEC = 60.0 @contextlib.contextmanager @@ -84,10 +90,23 @@ def _viewer_start_lock(timeout: float = VIEWER_START_LOCK_TIMEOUT_SEC): def _bridge_script() -> Path: + """Pick the viewer-daemon entrypoint, preferring pure ESM. + + See ``bridge_client._bridge_script`` for the rationale. The two + helpers intentionally share the same precedence so that the stdio + bridge spawned by ``MemosBridgeClient`` and the viewer daemon + spawned by ``ensure_viewer_daemon`` always end up on the same Node + entry binary. + """ plugin_root = _plugin_root() - compiled = plugin_root / "dist" / "bridge.cjs" - if compiled.exists(): - return compiled + candidates = ( + plugin_root / "dist" / "bridge.mjs", + plugin_root / "dist" / "bridge.cjs", + plugin_root / "bridge.mts", + ) + for candidate in candidates: + if candidate.exists(): + return candidate return plugin_root / "bridge.cts" @@ -140,7 +159,7 @@ def _bridge_command(*, daemon: bool) -> list[str]: bridge_args = [script, "--agent=hermes"] if daemon: bridge_args.append("--daemon") - if script_path.suffix == ".cjs": + if script_path.suffix in (".mjs", ".cjs"): return [node, *bridge_args] if tsx_cli.exists(): return [node, str(tsx_cli), *bridge_args] @@ -153,22 +172,39 @@ def ensure_bridge_running(*, probe_only: bool = False) -> bool: ``probe_only=True`` performs a lightweight availability check without launching a long-lived subprocess. This is what ``MemTensorProvider.is_available`` calls during Hermes startup. + + The cached answer is honoured only inside ``BRIDGE_OK_TTL_SEC``; once + it expires we revalidate. A transient ``_node_available()`` failure + therefore self-heals on the next probe instead of permanently + disabling the provider (see issue #1797). """ - global _bridge_ok + global _bridge_ok, _bridge_ok_at with _lock: - if _bridge_ok is not None and probe_only: + now = time.time() + if _bridge_ok is not None and probe_only and (now - _bridge_ok_at) < BRIDGE_OK_TTL_SEC: return _bridge_ok script = _bridge_script() if not script.exists(): logger.warning("MemOS: bridge script missing at %s", script) _bridge_ok = False + _bridge_ok_at = now return False - if not _node_available(): - logger.warning("MemOS: Node.js not found on PATH") - _bridge_ok = False - return False - _bridge_ok = True - return True + if _node_available(): + _bridge_ok = True + _bridge_ok_at = now + return True + # Node binary check just failed. A MemOS bridge already running on + # :18800 is definitive proof Node works on this host (the daemon + # itself was launched via Node); trust it and recover rather than + # report unavailable forever. + if _probe_viewer() == "running_memos": + _bridge_ok = True + _bridge_ok_at = now + return True + logger.warning("MemOS: Node.js not found on PATH") + _bridge_ok = False + _bridge_ok_at = now + return False def _probe_viewer() -> str: @@ -331,9 +367,10 @@ def ensure_viewer_daemon(*, probe_only: bool = False) -> bool: def shutdown_bridge() -> None: """Best-effort cleanup; each client owns its own subprocess.""" - global _bridge_ok + global _bridge_ok, _bridge_ok_at with _lock: _bridge_ok = None + _bridge_ok_at = 0.0 def wait_for_process_exit(pid: int, timeout: float = 5.0) -> bool: diff --git a/apps/memos-local-plugin/adapters/openclaw/index.ts b/apps/memos-local-plugin/adapters/openclaw/index.ts index 7518336ba..018880651 100644 --- a/apps/memos-local-plugin/adapters/openclaw/index.ts +++ b/apps/memos-local-plugin/adapters/openclaw/index.ts @@ -405,6 +405,31 @@ function register(api: OpenClawPluginApi): void { return runtime; }; + /** + * Helper for **void / fire-and-forget** hooks: dispatch `fn` against the + * runtime as soon as bootstrap finishes (already finished → next tick). + * Errors are logged at WARN and swallowed — they must not surface to + * OpenClaw's hook runner because the listener itself has already + * returned synchronously. + * + * `label` is used solely for log context so a misbehaving hook is + * findable in the gateway log. + */ + const runWhenReady = async ( + fn: (r: PluginRuntime) => void | Promise, + label: string, + ): Promise => { + try { + const r = await ensureRuntime(); + if (!r) return; + await fn(r); + } catch (err) { + api.logger.warn(`memos-local: hook ${label} failed`, { + err: err instanceof Error ? err.message : String(err), + }); + } + }; + registerOpenClawTools(api, { agent: "openclaw", getCore: async () => (await ensureRuntime())?.core ?? null, @@ -413,58 +438,88 @@ function register(api: OpenClawPluginApi): void { // 3. Hooks — every handler matches the upstream `PluginHookHandlerMap` // signature so OpenClaw's type-check passes in a monorepo install. + // + // Two upstream constraints govern the registration style here: + // (a) `tool_result_persist` is a **value-returning sync hook**. + // OpenClaw's hook runner inspects the return value with + // `isPromiseLike(ret)` and ignores it when the handler returns a + // Promise — so declaring this listener `async` silently disables + // the "append memos_search hint after repeated tool failures" + // feature. We register a **synchronous** wrapper that calls the + // (already sync) `bridge.handleToolResultPersist` directly. If + // bootstrap hasn't completed yet, the hook is a no-op (matches + // the legacy adapter — runtime not ready means no hint to + // inject). + // (b) `agent_end` (and the other void hooks below) are run by + // OpenClaw with a **hard-coded 30 s timeout** + // (`DEFAULT_VOID_HOOK_TIMEOUT_MS_BY_HOOK.agent_end = 30_000`). + // memos's onTurnEnd chain writes SQLite traces + runs L2 + // induction + reflection + reward (LLM-bound), which under I/O + // pressure can exceed 30 s. Awaiting that chain inside the hook + // handler shows up in the gateway log as + // `agent_end handler … timed out after 30000ms`. We schedule + // the heavy work as a **fire-and-forget** background task and + // return immediately. `core.shutdown()` (called from the + // service's `stop`) already drains in-flight pipeline work, so + // fire-and-forget does not lose data on a clean shutdown. + // + // `before_prompt_build` stays async-await because it MUST return the + // `prependContext` for OpenClaw to inject — it is a value-returning + // hook, not a void hook, and OpenClaw is willing to await its result + // (the timeout is laxer than `agent_end`'s 30 s budget). api.on("before_prompt_build", async (event, ctx) => { const r = await ensureRuntime(); if (!r) return; return r.bridge.handleBeforePrompt(event, ctx); }); - api.on("agent_end", async (event, ctx) => { - const r = await ensureRuntime(); - if (!r) return; - await r.bridge.handleAgentEnd(event, ctx); + api.on("agent_end", (event, ctx) => { + // Fire-and-forget. Returning synchronously lets OpenClaw's 30s + // void-hook budget tick down only on its own bookkeeping; memos + // continues writing the trace + running the reflect/reward chain + // in the background. + void runWhenReady((r) => r.bridge.handleAgentEnd(event, ctx), "agent_end"); }); - api.on("before_tool_call", async (event, ctx) => { - const r = await ensureRuntime(); - if (!r) return; - r.bridge.handleBeforeToolCall(event, ctx); + api.on("before_tool_call", (event, ctx) => { + // `handleBeforeToolCall` is sync and cheap (Map.set + timestamp); + // we still gate on runtime presence by deferring to ensureRuntime + // when bootstrap is in flight. The fire-and-forget wrapper keeps + // the listener void-shaped for OpenClaw. + void runWhenReady((r) => { + r.bridge.handleBeforeToolCall(event, ctx); + }, "before_tool_call"); }); - api.on("after_tool_call", async (event, ctx) => { - const r = await ensureRuntime(); - if (!r) return; - await r.bridge.handleAfterToolCall(event, ctx); + api.on("after_tool_call", (event, ctx) => { + void runWhenReady((r) => r.bridge.handleAfterToolCall(event, ctx), "after_tool_call"); }); - api.on("tool_result_persist", async (event, ctx) => { - const r = await ensureRuntime(); - if (!r) return; - return r.bridge.handleToolResultPersist(event, ctx); + // tool_result_persist is value-returning AND synchronous on + // OpenClaw's side — do NOT make this async. Bridge handler is + // already sync, so we can invoke it directly when the runtime is + // ready and return undefined otherwise. + api.on("tool_result_persist", (event, ctx) => { + if (!runtime) return; // bootstrap not finished — nothing to inject + return runtime.bridge.handleToolResultPersist(event, ctx); }); - api.on("session_start", async (event, ctx) => { - const r = await ensureRuntime(); - if (!r) return; - await r.bridge.handleSessionStart(event, ctx); + api.on("session_start", (event, ctx) => { + void runWhenReady((r) => r.bridge.handleSessionStart(event, ctx), "session_start"); }); - api.on("session_end", async (event, ctx) => { - const r = await ensureRuntime(); - if (!r) return; - await r.bridge.handleSessionEnd(event, ctx); + api.on("session_end", (event, ctx) => { + void runWhenReady((r) => r.bridge.handleSessionEnd(event, ctx), "session_end"); }); - api.on("subagent_spawned", async (event, ctx) => { - const r = await ensureRuntime(); - if (!r) return; - r.bridge.handleSubagentSpawned(event, ctx); + api.on("subagent_spawned", (event, ctx) => { + void runWhenReady((r) => { + r.bridge.handleSubagentSpawned(event, ctx); + }, "subagent_spawned"); }); - api.on("subagent_ended", async (event, ctx) => { - const r = await ensureRuntime(); - if (!r) return; - await r.bridge.handleSubagentEnded(event, ctx); + api.on("subagent_ended", (event, ctx) => { + void runWhenReady((r) => r.bridge.handleSubagentEnded(event, ctx), "subagent_ended"); }); // 4. Service — lets the host flush + wait for ready and shut us down. diff --git a/apps/memos-local-plugin/agent-contract/memory-core.ts b/apps/memos-local-plugin/agent-contract/memory-core.ts index 9c3ad826c..ad1dd3f13 100644 --- a/apps/memos-local-plugin/agent-contract/memory-core.ts +++ b/apps/memos-local-plugin/agent-contract/memory-core.ts @@ -165,6 +165,22 @@ export interface MemoryCore { health(): Promise; /** Late-bind ARMS telemetry (called after config is available). */ bindTelemetry?(t: unknown): void; + /** + * Resolve when the background recovery kicked off by `init()` settles. + * + * `init()` returns as soon as the synchronous orphan / dirty-episode + * classification finishes; the actual reflect / reward / L2 recovery + * chain runs on a background promise so the host's event loop stays + * responsive (see issues #1776 + #1808). This method exposes that + * promise for callers that need the historic "await everything" + * semantics — primarily tests and one-shot batch tools. + * + * Implementations MUST never reject from this promise. Failures are + * logged on the `init.background_recovery_failed` channel instead. + * Adapters that have no startup recovery may omit the method; an + * absent implementation is equivalent to `() => Promise.resolve()`. + */ + waitForStartupRecovery?(): Promise; // ── session / episode ── openSession(input: { diff --git a/apps/memos-local-plugin/bridge.cts b/apps/memos-local-plugin/bridge.cts index 81848acf7..449c4a707 100644 --- a/apps/memos-local-plugin/bridge.cts +++ b/apps/memos-local-plugin/bridge.cts @@ -70,19 +70,22 @@ function parseArgs(argv: readonly string[]): BridgeArgs { // ─── PID file singleton guard ─────────────────────────────────────────── // Prevents bridge process accumulation: each new bridge that wants to // own the viewer port kills the previous holder via its PID file. -// `--no-viewer` (headless) bridges skip this PID file entirely — they don't -// need the port and should coexist with the daemon that owns it. +// `--no-viewer` (headless) bridges use a SEPARATE PID file so they can +// reap their own predecessors without colliding with the viewer daemon +// that owns the port. Without the headless reap, every Hermes turn that +// respawns the Python adapter leaks an old bridge.cjs (issue #1910). const PID_FILENAME = "bridge.pid"; +const STDIO_PID_FILENAME = "bridge-stdio.pid"; -function pidFilePath(agent: string): string { +function pidFilePath(agent: string, filename: string = PID_FILENAME): string { const agentHome = agent === "hermes" ? ".hermes" : ".openclaw"; return path.join( process.env.HOME ?? "/tmp", agentHome, "memos-plugin", "daemon", - PID_FILENAME, + filename, ); } @@ -146,14 +149,24 @@ async function main(): Promise { // ─── Singleton: kill previous bridge that owns the viewer port ─── const pidPath = pidFilePath(args.agent); + const stdioPidPath = pidFilePath(args.agent, STDIO_PID_FILENAME); const ownsViewerPort = args.daemon || !args.noViewer; const removeOwnedPidFile = () => { if (ownsViewerPort) removePidFile(pidPath); + // Headless bridges own a separate PID slot; remove it on exit too. + if (args.noViewer) removePidFile(stdioPidPath); }; if (ownsViewerPort) { killExistingBridge(pidPath); writePidFile(pidPath); } + if (args.noViewer) { + // Reap any previous --no-viewer bridge for this agent. This is the + // headless counterpart to the viewer-port singleton above and the + // Node-side defense against issue #1910 (bridge process leak). + killExistingBridge(stdioPidPath); + writePidFile(stdioPidPath); + } // Lazy-import ESM core. Using dynamic import so this file remains // CommonJS and stays `require`-able. @@ -169,6 +182,9 @@ async function main(): Promise { const { startHttpServer } = (await importEsm( runtimeModule("server/http.ts", "dist/server/http.js") )) as typeof import("./server/http.js"); + const { isHermesChatRunning } = (await importEsm( + runtimeModule("bridge/hermes-process.ts", "dist/bridge/hermes-process.js") + )) as typeof import("./bridge/hermes-process.js"); const rootDir = pluginRoot(); const pkgVersion = require(path.join(rootDir, "package.json")).version; @@ -246,9 +262,27 @@ async function main(): Promise { ? resolveHome(args.agent, args.home) : undefined; + // Derive profileId dynamically from MEMOS_HOME. + // MEMOS_HOME points to /memos-plugin, so parent dir is hermes-home. + // e.g. /root/.hermes/profiles/nova/memos-plugin → profile = "nova" + // /root/.hermes/memos-plugin → profile = "default" + const deriveProfileId = (): string => { + const memosHome = process.env.MEMOS_HOME; + if (memosHome) { + const hermesHome = memosHome.replace(/memos-plugin\/?$/, ""); + const match = /\/profiles\/([^/]+)\/?$/.exec(hermesHome); + if (match?.[1]) { + const cleaned = match[1].toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, ""); + if (cleaned) return cleaned; + } + if (hermesHome.endsWith("/.hermes")) return "default"; + } + return "default"; + }; + const resolvedProfileId = deriveProfileId(); const { core, config, home } = await bootstrapMemoryCoreFull({ agent: args.agent, - namespace: { agentKind: args.agent, profileId: "default" }, + namespace: { agentKind: args.agent, profileId: resolvedProfileId }, pkgVersion, hostLlmBridge: args.daemon ? null : lazyHostLlmBridge, home: resolvedHome, @@ -269,6 +303,7 @@ async function main(): Promise { ? createBridgeStatusTracker( path.join(home.root, BRIDGE_STATUS_FILE), args.daemon, + isHermesChatRunning, ) : null; @@ -318,12 +353,36 @@ async function main(): Promise { | ReturnType["startHeartbeat"]> | undefined; - // In stdio mode the host fallback path is a reverse JSON-RPC request - // over the same pipe as normal bridge traffic. `core.init()` may - // recover dirty episodes and run reflection/reward/L2/skill work; if - // that work hits a broken primary skill-evolver model, the LLM facade - // can fall back to host before init returns. Start stdio first so that - // fallback has a transport instead of tripping the lazy bridge guard. + // ─── Startup ordering invariant (issue #1747 + host LLM fallback) ─── + // + // `startStdioServer({ core })` MUST run before `await core.init()`. + // Two independent failure modes if this ordering is reversed: + // + // 1. Host LLM fallback (original motivation for this ordering): + // `core.init()` may recover dirty episodes and run + // reflection/reward/L2/skill work; if that work hits a broken + // primary skill-evolver model, the LLM facade can fall back to + // host before init returns. Starting stdio first gives the + // fallback a transport instead of tripping the lazy bridge guard. + // + // 2. Python adapter `session.open` timeout (issue #1747): + // `core.init()` synchronously scans `episodes WHERE status='open'` + // and recovers stale rows via `recoverOpenEpisodesAsSessionEnd` + // + `recoverDirtyClosedEpisodes` — both of which call the LLM + // and routinely take 10-60+ seconds when a previous chat left + // orphan episodes behind. The Hermes Python adapter's + // `_open_session()` default timeout is 30 s. If stdio starts + // after init, the parent writes `session.open` into the bridge's + // stdin and the Python side gets `asyncio.TimeoutError` before + // the read loop is attached. By starting stdio first, the read + // loop is alive immediately — `core.openSession()` is safe to + // serve pre-init because it depends only on the SQLite handle + // and event bus that `bootstrapMemoryCoreFull()` already + // provisioned. (`ensureLive()` only blocks on `shutDown`, not + // on `initialized`.) + // + // The invariant is pinned by + // `tests/unit/bridge/bridge-startup-ordering.test.ts`. if (!args.daemon) { stdio = startStdioServer({ core }); bridgeStatus?.markConnected(); @@ -565,7 +624,11 @@ function classifyErrorCode(err: unknown): string { return "unknown"; } -function createBridgeStatusTracker(statusFile: string, daemon: boolean): { +function createBridgeStatusTracker( + statusFile: string, + daemon: boolean, + isHermesChatRunning: () => boolean, +): { snapshot(): BridgeStatusSnapshot; markConnected(): void; markDisconnected(message: string): void; @@ -679,18 +742,6 @@ function createBridgeStatusTracker(statusFile: string, daemon: boolean): { }; } -function isHermesChatRunning(): boolean { - try { - const out = childProcess.execFileSync("pgrep", ["-f", "hermes chat"], { - encoding: "utf8", - timeout: 1000, - }); - return out.trim().length > 0; - } catch { - return false; - } -} - void main().catch((err) => { const detail = err instanceof Error ? err.stack ?? err.message : String(err); process.stderr.write( diff --git a/apps/memos-local-plugin/bridge.mts b/apps/memos-local-plugin/bridge.mts new file mode 100644 index 000000000..95b15e21c --- /dev/null +++ b/apps/memos-local-plugin/bridge.mts @@ -0,0 +1,680 @@ +/** + * Bridge entry point (pure ESM). + * + * Started by non-TypeScript hosts (e.g. the Hermes Python client) via: + * + * node dist/bridge.mjs --agent=hermes --no-viewer + * + * This file is the pure-ESM successor of `bridge.cts` / `dist/bridge.cjs` + * — it exists to eliminate the CommonJS → ESM trampoline that became + * fragile on Node ≥ 22 (issue #1736). The package is `"type": "module"`, + * so a `.mjs` entry can statically `import` peer modules and use + * `import.meta.url` for path resolution, with no `new Function()` + * trampoline and no `require(file://URL)` indirection. The legacy + * `bridge.cts` is preserved for backwards compatibility; the Python + * launcher prefers the new entry whenever `dist/bridge.mjs` exists. + * + * Viewer lifecycle + * ================ + * Each agent owns its own HTTP port: + * + * - openclaw → :18799 + * - hermes → :18800 + * + * The viewer port is read from the agent's `~/./memos-plugin/ + * config.yaml::viewer.port`. We just call `startHttpServer` once; + * if the port is already in use we surface the EADDRINUSE error to + * stderr and keep running stdio-RPC headless (capture / retrieval + * still work). There's no port-sharing or auto-promotion logic — + * each agent has its own bookmarkable URL. + */ +import * as childProcess from "node:child_process"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const BRIDGE_STATUS_HEARTBEAT_MS = 5_000; +const BRIDGE_STATUS_STALE_MS = 20_000; +const BRIDGE_STATUS_FILE = "bridge-status.json"; + +interface BridgeArgs { + daemon: boolean; + noViewer: boolean; + tcpPort?: number; + agent: "openclaw" | "hermes"; + home?: string; +} + +type BridgeStatus = "connected" | "reconnecting" | "disconnected" | "unknown"; + +interface BridgeStatusSnapshot { + status: BridgeStatus; + lastOkAt: number | null; + lastErrorAt: number | null; + lastError: string | null; +} + +function parseArgs(argv: readonly string[]): BridgeArgs { + const args: BridgeArgs = { daemon: false, noViewer: false, agent: "openclaw" }; + for (const raw of argv) { + if (raw === "--daemon") args.daemon = true; + else if (raw === "--no-viewer") args.noViewer = true; + else if (raw.startsWith("--tcp=")) args.tcpPort = Number(raw.slice(6)); + else if (raw === "--agent=hermes") args.agent = "hermes"; + else if (raw === "--agent=openclaw") args.agent = "openclaw"; + else if (raw.startsWith("--home=")) args.home = raw.slice(7); + } + return args; +} + +// ─── PID file singleton guard ─────────────────────────────────────────── +// Prevents bridge process accumulation: each new bridge that wants to +// own the viewer port kills the previous holder via its PID file. +// `--no-viewer` (headless) bridges skip this PID file entirely — they don't +// need the port and should coexist with the daemon that owns it. + +const PID_FILENAME = "bridge.pid"; + +function pidFilePath(agent: string): string { + const agentHome = agent === "hermes" ? ".hermes" : ".openclaw"; + return path.join( + process.env.HOME ?? "/tmp", + agentHome, + "memos-plugin", + "daemon", + PID_FILENAME, + ); +} + +function readPidFile(pidPath: string): number | null { + try { + const raw = fs.readFileSync(pidPath, "utf8").trim(); + const pid = parseInt(raw, 10); + if (isNaN(pid) || pid <= 0) return null; + process.kill(pid, 0); // throws if not alive + return pid; + } catch { + return null; + } +} + +function writePidFile(pidPath: string): void { + fs.mkdirSync(path.dirname(pidPath), { recursive: true }); + fs.writeFileSync(pidPath, String(process.pid), "utf8"); +} + +function removePidFile(pidPath: string): void { + try { + const content = fs.readFileSync(pidPath, "utf8").trim(); + if (content === String(process.pid)) fs.unlinkSync(pidPath); + } catch { + /* best-effort; another bridge may have overwritten */ + } +} + +function killExistingBridge(pidPath: string, timeoutMs = 5000): void { + const existingPid = readPidFile(pidPath); + if (existingPid === null || existingPid === process.pid) return; + + process.stderr.write( + `bridge: killing stale bridge pid=${existingPid} before startup\n`, + ); + try { + process.kill(existingPid, "SIGTERM"); + } catch { + return; // already dead + } + + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + process.kill(existingPid, 0); + } catch { + return; // gone + } + childProcess.spawnSync("sleep", ["0.5"]); + } + try { + process.kill(existingPid, "SIGKILL"); + } catch { + /* already dead */ + } +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + + // ─── Singleton: kill previous bridge that owns the viewer port ─── + const pidPath = pidFilePath(args.agent); + const ownsViewerPort = args.daemon || !args.noViewer; + const removeOwnedPidFile = () => { + if (ownsViewerPort) removePidFile(pidPath); + }; + if (ownsViewerPort) { + killExistingBridge(pidPath); + writePidFile(pidPath); + } + + // Lazy-import core modules from the sibling tree. With a pure-ESM + // entry there's no CJS↔ESM boundary to cross — the specifiers below + // are plain relative paths that Node's loader resolves directly. + // tsx rewrites `.js` → `.ts` at runtime when this file is executed + // from source via `tsx bridge.mts`. + const { bootstrapMemoryCoreFull } = await import("./core/pipeline/index.js"); + const { startStdioServer, waitForShutdown } = await import("./bridge/stdio.js"); + const { memoryBuffer, rootLogger } = await import("./core/logger/index.js"); + const { startHttpServer } = await import("./server/http.js"); + + const rootDir = pluginRoot(); + const pkgVersion = JSON.parse( + fs.readFileSync(path.join(rootDir, "package.json"), "utf8"), + ).version as string; + + // ─── Host LLM bridge (reverse RPC, lazy-bound to stdio) ──────── + // We need to register the bridge BEFORE bootstrap creates the + // LlmClients (so the very first `shouldFallback()` check sees a + // non-null bridge), but `stdio` itself doesn't exist until later + // in this function. The trick: hand a placeholder closure to + // bootstrap that defers actual stdio access to the time of the + // first fallback call. In stdio mode we start the server before + // `core.init()` so startup recovery can also use host fallback. + // + // Routing through `bootstrapMemoryCoreFull({ hostLlmBridge })` + // (instead of having this file call `registerHostLlmBridge` + // directly) avoids a subtle ESM module-identity issue: the static + // `import` chain inside `core/llm/client.ts` and the dynamic + // `await import(...)` here resolve to the same file URL but Node + // can occasionally treat them as different module instances with + // independent `currentBridge` slots. Registering inside bootstrap + // forces both ends to share the same module instance. + let stdio: import("./bridge/stdio.js").StdioServerHandle | null = null; + const lazyHostLlmBridge: import("./core/llm/host-bridge.js").HostLlmBridge = + { + id: `stdio.host.${args.agent}.v1`, + async complete(input) { + if (!stdio) { + throw new Error( + "host LLM bridge invoked before stdio server was ready", + ); + } + const result = (await stdio.serverRequest( + "host.llm.complete", + { + messages: input.messages, + model: input.model, + temperature: input.temperature, + maxTokens: input.maxTokens, + timeoutMs: input.timeoutMs, + }, + { timeoutMs: (input.timeoutMs ?? 60_000) + 5_000 }, + )) as { + text?: string; + model?: string; + usage?: { + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; + }; + durationMs?: number; + }; + return { + text: typeof result?.text === "string" ? result.text : "", + model: + typeof result?.model === "string" + ? result.model + : input.model ?? "", + usage: result?.usage, + durationMs: + typeof result?.durationMs === "number" ? result.durationMs : 0, + }; + }, + }; + + const { Telemetry } = await import("./core/telemetry/index.js"); + + // Resolve home early so we can use resolveHome with explicit defaultHome + const { resolveHome } = await import("./core/config/paths.js"); + + const resolvedHome = args.home + ? resolveHome(args.agent, args.home) + : undefined; + + const { core, config, home } = await bootstrapMemoryCoreFull({ + agent: args.agent, + namespace: { agentKind: args.agent, profileId: "default" }, + pkgVersion, + hostLlmBridge: args.daemon ? null : lazyHostLlmBridge, + home: resolvedHome, + }); + + const telemetry = new Telemetry( + config.telemetry ?? {}, + home.root, + pkgVersion, + rootLogger.child({ channel: "core.telemetry" }), + rootDir, + ); + (core as { bindTelemetry?: (t: InstanceType) => void }).bindTelemetry?.(telemetry); + telemetry.trackPluginStarted(args.agent); + + const bridgeStatus = + args.agent === "hermes" + ? createBridgeStatusTracker( + path.join(home.root, BRIDGE_STATUS_FILE), + args.daemon, + ) + : null; + + // Process-level error reporting. Without these handlers a crash in + // a background task (capture / reward / L2 inducer) silently kills + // the bridge process and never surfaces in ARMS — making "0 + // plugin_error events" actively misleading. Both handlers are + // best-effort and re-emit (or `process.exit(1)`) so we don't + // alter the existing crash semantics, only add observability. + // Only registered for the dedicated bridge process; the OpenClaw + // adapter runs inside the host process and must not steal its + // global error hooks. + process.on("uncaughtException", (err) => { + try { + telemetry.trackError("uncaught_exception", classifyErrorCode(err)); + } catch { + /* swallow — telemetry must never widen the crash */ + } + process.stderr.write( + `bridge: uncaughtException: ${err instanceof Error ? err.stack ?? err.message : String(err)}\n`, + ); + // Mirror Node's default behaviour so existing supervisors that + // expect non-zero exit on crash keep working. + process.exit(1); + }); + process.on("unhandledRejection", (reason) => { + try { + telemetry.trackError("unhandled_rejection", classifyErrorCode(reason)); + } catch { + /* swallow — telemetry must never widen the crash */ + } + process.stderr.write( + `bridge: unhandledRejection: ${reason instanceof Error ? reason.stack ?? reason.message : String(reason)}\n`, + ); + // Don't exit: per-promise rejections are usually recoverable + // (failed flush, dropped SSE client). The default Node 20+ + // behaviour is to exit, but for a long-running bridge that + // would be too aggressive — surface to telemetry + stderr and + // continue. + }); + + // Per-agent fixed viewer port. + const AGENT_DEFAULT_PORTS = { openclaw: 18799, hermes: 18800 } as const; + const viewerPort = AGENT_DEFAULT_PORTS[args.agent]; + + let bridgeHeartbeat: + | ReturnType["startHeartbeat"]> + | undefined; + + // In stdio mode the host fallback path is a reverse JSON-RPC request + // over the same pipe as normal bridge traffic. `core.init()` may + // recover dirty episodes and run reflection/reward/L2/skill work; if + // that work hits a broken primary skill-evolver model, the LLM facade + // can fall back to host before init returns. Start stdio first so that + // fallback has a transport instead of tripping the lazy bridge guard. + if (!args.daemon) { + stdio = startStdioServer({ core }); + bridgeStatus?.markConnected(); + bridgeHeartbeat = bridgeStatus?.startHeartbeat(); + void stdio.done.then(() => { + bridgeHeartbeat?.stop(); + bridgeStatus?.markDisconnected("Hermes chat disconnected"); + }); + } + + try { + await core.init(); + } catch (err) { + bridgeHeartbeat?.stop(); + if (stdio) { + try { + await stdio.close(); + } catch { + /* best-effort */ + } + } + throw err; + } + + // ─── Daemon mode ────────────────────────────────────────────── + // When started with `--daemon`, skip stdio and run as a pure HTTP + // viewer daemon. Used by install.sh (post-install) and admin/restart + // (self-restart) to keep the Memory Viewer always available. + if (args.daemon) { + // Daemon mode is the target of `POST /api/v1/admin/restart`, + // which re-spawns the bridge after a short sleep. On busy + // machines the previous bridge's listening socket can take a + // moment longer than expected to release, so we retry the bind + // a few times before giving up. Without this the user sees + // "重启超时" in the viewer because the new daemon raced its + // predecessor and lost. + let viewer: import("./server/types.js").ServerHandle | null = null; + const maxBindAttempts = 10; + for (let attempt = 1; attempt <= maxBindAttempts; attempt++) { + try { + viewer = await startHttpServer( + { + core, + home, + logTail: () => memoryBuffer().tail({ limit: 200 }), + bridgeStatus: bridgeStatus ? () => bridgeStatus.snapshot() : undefined, + telemetry, + }, + { + port: viewerPort, + host: config.viewer.bindHost, + staticRoot: path.resolve(rootDir, "viewer/dist"), + agent: args.agent, + }, + ); + process.stderr.write( + `bridge: daemon viewer live at ${viewer.url} (agent=${args.agent})\n`, + ); + break; + } catch (err) { + const e = err as NodeJS.ErrnoException; + if (e?.code === "EADDRINUSE" && attempt < maxBindAttempts) { + process.stderr.write( + `bridge: daemon port :${viewerPort} busy (attempt ${attempt}/${maxBindAttempts}), retrying in 1s...\n`, + ); + await new Promise((r) => setTimeout(r, 1000)); + continue; + } + if (e?.code === "EADDRINUSE") { + process.stderr.write( + `bridge: daemon port :${viewerPort} still in use after ${maxBindAttempts}s — exiting.\n`, + ); + await core.shutdown(); + process.exit(1); + } + process.stderr.write( + `bridge: daemon viewer failed: ${(err as Error)?.message ?? String(err)}\n`, + ); + await core.shutdown(); + process.exit(1); + } + } + + const shutdownDaemon = async (sig: string) => { + process.stderr.write(`bridge: daemon received ${sig}, shutting down\n`); + removeOwnedPidFile(); + try { await viewer!.close(); } catch { /* best-effort */ } + await core.shutdown(); + process.exit(0); + }; + process.on("SIGINT", () => void shutdownDaemon("SIGINT")); + process.on("SIGTERM", () => void shutdownDaemon("SIGTERM")); + // Process stays alive via the HTTP server's ref'd socket. + return; + } + + // ─── Normal (stdio) mode ────────────────────────────────────── + // The stdio handle was started before `core.init()` above so host + // fallback is available during startup recovery. + const activeStdio = stdio; + if (!activeStdio) { + throw new Error("internal bridge error: stdio server was not started"); + } + + // Try to bind the viewer port unless the caller requested a pure stdio + // bridge. Hermes chat uses --no-viewer; the standalone --daemon process is + // the single owner of :18800. + let viewer: import("./server/types.js").ServerHandle | null = null; + if (args.noViewer) { + process.stderr.write( + `bridge: stdio mode running without viewer (agent=${args.agent})\n`, + ); + } else { + try { + viewer = await startHttpServer( + { + core, + home, + logTail: () => memoryBuffer().tail({ limit: 200 }), + bridgeStatus: bridgeStatus ? () => bridgeStatus.snapshot() : undefined, + telemetry, + }, + { + port: viewerPort, + host: config.viewer.bindHost, + staticRoot: path.resolve(rootDir, "viewer/dist"), + agent: args.agent, + }, + ); + process.stderr.write( + `bridge: viewer live at ${viewer.url} (agent=${args.agent})\n`, + ); + } catch (err) { + const e = err as NodeJS.ErrnoException; + if (e?.code === "EADDRINUSE") { + process.stderr.write( + `bridge: viewer port :${viewerPort} is already in use — ` + + `${args.agent} will run headless (stdio only). ` + + `Free the port to expose the viewer.\n`, + ); + } else { + process.stderr.write( + `bridge: viewer failed to start: ${e?.message ?? String(err)}\n`, + ); + } + } + } + + const shutdown = async (sig: string) => { + process.stderr.write(`bridge: received ${sig}, shutting down\n`); + removeOwnedPidFile(); + if (viewer) { + try { + await viewer.close(); + } catch { + /* best-effort */ + } + } + await waitForShutdown(core, activeStdio); + process.exit(0); + }; + + process.on("SIGINT", () => void shutdown("SIGINT")); + process.on("SIGTERM", () => void shutdown("SIGTERM")); + + // Keep the process alive until stdin ends (client disconnects). + await activeStdio.done; + + // If a viewer is running, keep the process alive as a daemon so the + // memory panel stays accessible between `hermes chat` sessions. + if (viewer && !viewer.closed) { + process.stderr.write( + `bridge: stdin closed but viewer is still serving at ${viewer.url} — ` + + `staying alive as daemon. Send SIGTERM to stop.\n`, + ); + const keepalive = setInterval(() => { + if (viewer!.closed) { + clearInterval(keepalive); + removeOwnedPidFile(); + void core.shutdown().then(() => process.exit(0)); + } + }, 5_000); + (keepalive as unknown as { unref?: () => void }).unref?.(); + return; + } + + // No viewer (headless bridge) — clean exit. + removeOwnedPidFile(); + await core.shutdown(); + process.exit(0); +} + +function pluginRoot(): string { + // Source entry: /bridge.mts. Built entry: /dist/bridge.mjs. + if (fs.existsSync(path.join(__dirname, "package.json"))) return __dirname; + const parent = path.resolve(__dirname, ".."); + if (fs.existsSync(path.join(parent, "package.json"))) return parent; + return __dirname; +} + +/** + * Best-effort error classification for ARMS `plugin_error.error_type`. + * + * Priority order: + * 1. `MemosError.code` and Node `errno` (`ENOENT`, `EADDRINUSE`, …) + * — both surface as a `code` string property. + * 2. The constructor name when it's something more specific than + * the generic `Error` (e.g. `TypeError`, `SyntaxError`). + * 3. `unknown` as a sentinel. + * + * Never returns the message — those can carry user paths or query + * fragments and would defeat the redaction the rest of the telemetry + * pipeline guarantees. + */ +function classifyErrorCode(err: unknown): string { + if (err && typeof err === "object" && "code" in err) { + const code = (err as { code: unknown }).code; + if (typeof code === "string" && code.length > 0) return code; + } + if (err instanceof Error && err.name && err.name !== "Error") { + return err.name; + } + return "unknown"; +} + +function createBridgeStatusTracker(statusFile: string, daemon: boolean): { + snapshot(): BridgeStatusSnapshot; + markConnected(): void; + markDisconnected(message: string): void; + startHeartbeat(): { stop(): void }; +} { + let snapshot: BridgeStatusSnapshot = daemon + ? { + status: "disconnected", + lastOkAt: null, + lastErrorAt: Date.now(), + lastError: "Hermes chat is not connected", + } + : { + status: "unknown", + lastOkAt: null, + lastErrorAt: null, + lastError: null, + }; + + function writeStatus(next: BridgeStatusSnapshot): void { + snapshot = next; + try { + fs.mkdirSync(path.dirname(statusFile), { recursive: true }); + fs.writeFileSync(statusFile, JSON.stringify(next), "utf8"); + } catch { + // Status display must never affect chat capture. + } + } + + function readStatus(): BridgeStatusSnapshot | null { + try { + const parsed = JSON.parse(fs.readFileSync(statusFile, "utf8")) as Partial; + if ( + parsed.status === "connected" || + parsed.status === "reconnecting" || + parsed.status === "disconnected" || + parsed.status === "unknown" + ) { + return { + status: parsed.status, + lastOkAt: typeof parsed.lastOkAt === "number" ? parsed.lastOkAt : null, + lastErrorAt: typeof parsed.lastErrorAt === "number" ? parsed.lastErrorAt : null, + lastError: typeof parsed.lastError === "string" ? parsed.lastError : null, + }; + } + } catch { + // Missing or corrupt status files are treated as disconnected. + } + return null; + } + + function applyStaleRule(raw: BridgeStatusSnapshot): BridgeStatusSnapshot { + if (raw.status === "disconnected" && daemon && isHermesChatRunning()) { + return { + status: "reconnecting", + lastOkAt: raw.lastOkAt, + lastErrorAt: raw.lastErrorAt, + lastError: "Hermes chat is running; waiting for memory bridge", + }; + } + if ( + raw.status === "connected" && + raw.lastOkAt != null && + Date.now() - raw.lastOkAt > BRIDGE_STATUS_STALE_MS + ) { + return { + status: "disconnected", + lastOkAt: raw.lastOkAt, + lastErrorAt: Date.now(), + lastError: "Hermes bridge heartbeat is stale", + }; + } + return raw; + } + + function markConnected(): void { + writeStatus({ + status: "connected", + lastOkAt: Date.now(), + lastErrorAt: snapshot.lastErrorAt, + lastError: snapshot.lastError, + }); + } + + function markDisconnected(message: string): void { + writeStatus({ + status: "disconnected", + lastOkAt: snapshot.lastOkAt, + lastErrorAt: Date.now(), + lastError: message, + }); + } + + return { + snapshot() { + return { ...applyStaleRule(readStatus() ?? snapshot) }; + }, + markConnected, + markDisconnected, + startHeartbeat() { + const timer = setInterval(() => { + markConnected(); + }, BRIDGE_STATUS_HEARTBEAT_MS); + (timer as unknown as { unref?: () => void }).unref?.(); + return { + stop() { + clearInterval(timer); + }, + }; + }, + }; +} + +function isHermesChatRunning(): boolean { + try { + const out = childProcess.execFileSync("pgrep", ["-f", "hermes chat"], { + encoding: "utf8", + timeout: 1000, + }); + return out.trim().length > 0; + } catch { + return false; + } +} + +void main().catch((err) => { + const detail = err instanceof Error ? err.stack ?? err.message : String(err); + process.stderr.write( + `bridge: fatal: ${detail}\n`, + ); + process.exit(1); +}); diff --git a/apps/memos-local-plugin/bridge/hermes-process.ts b/apps/memos-local-plugin/bridge/hermes-process.ts new file mode 100644 index 000000000..6c2384f96 --- /dev/null +++ b/apps/memos-local-plugin/bridge/hermes-process.ts @@ -0,0 +1,105 @@ +/** + * Hermes chat process detection. + * + * The viewer daemon (`bridge.cts --daemon`) wants to know whether the + * user has a `hermes chat` session attached to *some* bridge stdio + * process so it can upgrade its own `"disconnected"` status to + * `"reconnecting"` instead of leaving the viewer permanently red. + * + * Implementation: shell out to `pgrep -f `. The regex has to + * cover all three CLI invocation shapes the Hermes CLI supports: + * + * • no flags — `hermes chat` + * • flags after the subcommand — `hermes chat --skills memory-routing` + * • global flags before it — `hermes --skills memory-routing chat` + * + * The third shape is the bug reported in #1915: the previous literal + * pattern `"hermes chat"` requires the two tokens to be contiguous and + * therefore misses any invocation with a global flag (`--skills`, + * `-m`, `--provider`, …) between them. + * + * The current pattern is `hermes(?:\s+\S+)*\s+chat\b`: + * + * • `hermes` — the binary basename. + * • `(?:\s+\S+)*` — any complete argv-style tokens between the + * binary and the subcommand. + * • `\s+chat\b` — a standalone `chat` token, so it does *not* + * match `chatter`, `chat-server`, `--chat-log`, or a flag value + * like `--profile=chat`. + * + * `pgrep -f` on Linux uses glibc's ERE engine, which supports + * `\s`/`\b` as GNU extensions. JavaScript's `RegExp` supports the same + * tokens natively, so this module also exports + * `matchesHermesChatCommandLine()` for unit tests — exercising the + * pattern as a JS regex is a faithful proxy for the pgrep-side + * behaviour without requiring a real Hermes binary or a fork of the + * pgrep process in CI. + */ +// eslint-disable-next-line @typescript-eslint/no-require-imports +import * as childProcess from "node:child_process"; + +/** + * pgrep `-f` pattern used by `isHermesChatRunning`. + * + * Exported separately so callers — and tests — can introspect the exact + * string we hand to `pgrep` and confirm we have not silently regressed + * back to a literal substring match. + */ +export const HERMES_CHAT_PROCESS_PATTERN = "hermes(?:\\s+\\S+)*\\s+chat\\b"; + +/** + * JS-side equivalent of `pgrep -f HERMES_CHAT_PROCESS_PATTERN`. + * + * Used by unit tests to verify the regex matches every documented + * Hermes invocation shape and rejects unrelated command lines. Kept + * deliberately stateless — callers should pass the full + * `/proc//cmdline`-style command-line string. + */ +export function matchesHermesChatCommandLine(commandLine: string): boolean { + return new RegExp(HERMES_CHAT_PROCESS_PATTERN).test(commandLine); +} + +/** + * Shape of the `execFileSync`-compatible helper that + * `isHermesChatRunning` shells out through. Carved out as a named type + * so tests can pass a `vi.fn()` without depending on Node's overloaded + * `ExecFileSyncOptions` union. + */ +export type ExecFileSyncLike = ( + file: string, + args: readonly string[], + options: { encoding: "utf8"; timeout: number }, +) => string; + +const defaultExecFileSync: ExecFileSyncLike = (file, args, options) => + childProcess.execFileSync(file, [...args], options) as unknown as string; + +/** + * Returns `true` when `pgrep -f` finds at least one process whose full + * command line matches `HERMES_CHAT_PROCESS_PATTERN`. + * + * `pgrep` exits non-zero when there is no match, when it cannot be + * found, or on permission errors — all of which we collapse into + * `false` because the caller only uses the boolean to decide whether + * to upgrade `"disconnected"` to `"reconnecting"`. Surfacing the + * difference would just turn a UI hint into a noisy crash path. + * + * `execFile` is overridable so the unit test can inject a stub instead + * of mocking `node:child_process` globally — a Node ESM namespace is + * frozen at import time, so spy-based mocking is brittle. Injection is + * the same dependency pattern the rest of the bridge uses (see + * `startStdioServer`'s `stdin` / `stdout` options). + */ +export function isHermesChatRunning( + execFile: ExecFileSyncLike = defaultExecFileSync, +): boolean { + try { + const out = execFile("pgrep", ["-f", HERMES_CHAT_PROCESS_PATTERN], { + encoding: "utf8", + timeout: 1000, + }); + return out.trim().length > 0; + } catch { + return false; + } +} diff --git a/apps/memos-local-plugin/bridge/methods.ts b/apps/memos-local-plugin/bridge/methods.ts index 3c61f75b3..1f3583746 100644 --- a/apps/memos-local-plugin/bridge/methods.ts +++ b/apps/memos-local-plugin/bridge/methods.ts @@ -104,6 +104,16 @@ export function makeDispatcher( switch (method) { // ── lifecycle ── + // CORE_INIT is retained for backward compatibility with any + // out-of-tree adapter that might still send it, but it is + // **deprecated** for in-tree use. The bridge always calls + // `core.init()` in-process during `bridge.cts::main()` — see + // the ordering invariant there (also enforced by + // `tests/unit/bridge/bridge-startup-ordering.test.ts`). No + // in-tree stdio adapter (OpenClaw, Hermes Python) sends this + // RPC. Issue #1747 flagged it as dead code; we keep the + // dispatch branch to avoid a wire-contract break and revisit + // removal once telemetry confirms zero callers. case RPC_METHODS.CORE_INIT: await core.init(); return { ok: true }; diff --git a/apps/memos-local-plugin/core/config/defaults.ts b/apps/memos-local-plugin/core/config/defaults.ts index 1cf2d2cf6..4ab4c4240 100644 --- a/apps/memos-local-plugin/core/config/defaults.ts +++ b/apps/memos-local-plugin/core/config/defaults.ts @@ -261,6 +261,12 @@ export const DEFAULT_CONFIG: ResolvedConfig = { // hits before injection. llmFilterMinCandidates: 2, llmFilterCandidateBodyChars: 500, + // Default 0 — no time-window bound, keeping the legacy + // brute-force scan behaviour for fresh installs that haven't + // grown past the threshold where the bound starts paying off. + // Operators with >50K traces are expected to flip this on (we + // suggest 86_400_000 = 24h, or 2_592_000_000 = 30 days). + vectorScanMaxAgeMs: 0, }, }, hub: { diff --git a/apps/memos-local-plugin/core/config/schema.ts b/apps/memos-local-plugin/core/config/schema.ts index 7c9ff193b..92479373e 100644 --- a/apps/memos-local-plugin/core/config/schema.ts +++ b/apps/memos-local-plugin/core/config/schema.ts @@ -476,6 +476,28 @@ const AlgorithmSchema = Type.Object({ * slightly larger window pays for itself). */ llmFilterCandidateBodyChars: NumberInRange(500, 120, 2000), + /** + * Tier-2 vector scan time-window bound (ms). When > 0, the + * vector scan path (`scanAndTopK` in `core/storage/vector.ts`) + * only considers traces written within the last + * `vectorScanMaxAgeMs` milliseconds. Set to `0` to disable the + * cap (legacy behaviour: full-table brute-force scan). + * + * Background: at 93K rows × 1536 dims the unbounded scan blocks + * the Node event loop for 5–30 s every `onTurnStart` + * (https://github.com/MemTensor/MemOS/issues/1929). A 24-hour + * window keeps onTurnStart latency under control without + * sacrificing recall for active-session memories. FTS keyword + * channels still cover older traces, so this bound only affects + * the cosine-only path. + * + * Hard cap is one year (31_536_000_000 ms) — anything larger is + * indistinguishable from "unbounded" at the corpus sizes where + * the bound starts to matter, and accepting absurdly large + * values lets misconfigured deployments silently revert to the + * old behaviour. + */ + vectorScanMaxAgeMs: NumberInRange(0, 0, 31_536_000_000), }, { default: {} }), }, { default: {} }); diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index 7749a572f..41f31a439 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -71,6 +71,124 @@ export function createLlmClientWithProvider( let lastFallbackAt: number | null = null; let lastError: { at: number; message: string } | null = null; + // ─── Circuit breaker state (issue #1897) ───────────────────────────────── + // Per-client breaker that trips on terminal provider errors (401/402/403, + // "insufficient balance", "invalid api key", "unauthorized", "account + // suspended", "billing"). Short-circuits subsequent calls inside the + // facade so the broken provider is not contacted again until cool-down + // elapses. Half-open: the next call after `circuitOpenUntil` probes the + // provider; success closes the breaker, terminal failure re-opens it. + const breakerCfg = config.circuitBreaker ?? {}; + const breakerEnabled = breakerCfg.enabled !== false; + const breakerCooldownMs = Math.max(30_000, breakerCfg.cooldownMs ?? 300_000); + const breakerIsTerminal = breakerCfg.isTerminal ?? defaultIsTerminal; + const breakerNow = breakerCfg.now ?? Date.now; + let circuitOpenUntil: number | null = null; + let circuitOpenedReason: string | null = null; + let lastCircuitOpenStatusAt: number | null = null; + + function breakerIsOpen(): boolean { + if (!breakerEnabled) return false; + if (circuitOpenUntil === null) return false; + if (breakerNow() >= circuitOpenUntil) { + // Cool-down elapsed → transition to half-open. We do NOT clear + // `circuitOpenUntil` yet so the very first probe attempt that + // races with the cool-down boundary doesn't fall through to "no + // breaker" twice. The next call's success/failure handler resets + // or re-opens the breaker explicitly. + return false; + } + return true; + } + + function breakerTrip(err: unknown): void { + if (!breakerEnabled) return; + circuitOpenUntil = breakerNow() + breakerCooldownMs; + circuitOpenedReason = summarizeErrMessage(err); + // Reset the coalescer so the first suppressed call after a fresh + // trip always emits a `circuit_open` row. + lastCircuitOpenStatusAt = null; + facadeLog.warn("circuit_breaker.trip", { + provider: provider.name, + model: config.model, + until: circuitOpenUntil, + reason: circuitOpenedReason, + }); + } + + function breakerRecordSuccess(): void { + if (!breakerEnabled) return; + if (circuitOpenUntil !== null) { + facadeLog.info("circuit_breaker.close", { + provider: provider.name, + model: config.model, + }); + } + circuitOpenUntil = null; + circuitOpenedReason = null; + lastCircuitOpenStatusAt = null; + } + + /** + * Emit a coalesced `circuit_open` audit row. At most one row per + * `cooldownMs/12` window per client — bounds audit-row spam while + * still surfacing the suppressed-call event in the Logs viewer. + * The first suppressed call after a fresh trip always emits. + */ + function maybeEmitCircuitOpenStatus(opts: LlmCallOptions | undefined, op: string): void { + if (!config.onStatus) return; + const at = breakerNow(); + const coalesceWindow = Math.max(5_000, Math.floor(breakerCooldownMs / 12)); + if ( + lastCircuitOpenStatusAt !== null && + at - lastCircuitOpenStatusAt < coalesceWindow + ) { + return; + } + lastCircuitOpenStatusAt = at; + try { + config.onStatus({ + status: "circuit_open", + provider: provider.name, + model: config.model, + message: circuitOpenedReason ?? "(unknown reason)", + at, + durationMs: 0, + op, + episodeId: opts?.episodeId, + phase: opts?.phase, + }); + } catch { + /* status sink errors are non-fatal */ + } + } + + function throwBreakerOpen(): never { + throw makeBreakerOpenError(); + } + + function makeBreakerOpenError(): MemosError { + const until = circuitOpenUntil ?? breakerNow(); + return new MemosError( + ERROR_CODES.LLM_UNAVAILABLE, + `circuit_open: ${circuitOpenedReason ?? "terminal provider error"}`, + { + circuitOpen: true, + until, + provider: provider.name, + model: config.model, + }, + ); + } + + function canUseHostFallback(): boolean { + return ( + config.fallbackToHost === true && + provider.name !== "host" && + getHostLlmBridge() !== null + ); + } + /** * Mark a successful primary-provider call. We **do not** clear * `lastError` / `lastFallbackAt` here — the viewer picks the most @@ -110,7 +228,24 @@ export function createLlmClientWithProvider( if (!Array.isArray(input) || input.length === 0) { throw new MemosError(ERROR_CODES.INVALID_ARGUMENT, "LLM messages array is empty"); } - return input; + // Ensure system messages are always at the beginning. + // Some models (e.g. Qwen3.6 via vLLM) enforce "system must be first" + // in their Jinja2 chat templates, returning HTTP 400 otherwise. + // See: https://github.com/MemTensor/MemOS/issues/XXXX + const systems = input.filter((m) => m.role === "system"); + const nonSystems = input.filter((m) => m.role !== "system"); + if (systems.length === 0) return input; + // Fast path: single system already at position 0, no later systems. + if ( + systems.length === 1 && + input[0]?.role === "system" && + !input.slice(1).some((m) => m.role === "system") + ) { + return input; + } + // Merge all system contents into one leading message, preserving order. + const merged = systems.map((s) => s.content).join("\n\n"); + return [{ role: "system", content: merged }, ...nonSystems]; } function inject(messages: LlmMessage[], systemInsert: string): LlmMessage[] { @@ -151,6 +286,21 @@ export function createLlmClientWithProvider( opts: LlmCallOptions | undefined, op: string, ): Promise<{ completion: LlmCompletion }> { + // ── Circuit breaker short-circuit ── + // When the breaker is open we never reach the primary provider, so + // no request is generated against the broken paid API. We still + // emit (coalesced) `circuit_open` status rows so the Logs viewer / + // Overview can surface that suppression is happening. + if (breakerIsOpen()) { + maybeEmitCircuitOpenStatus(opts, op); + if (canUseHostFallback()) { + return callHostFallback(makeBreakerOpenError(), messages, input, opts, op, { + keepBreakerOpen: true, + notifyError: false, + }); + } + throwBreakerOpen(); + } requests++; const startedAt = Date.now(); try { @@ -166,6 +316,7 @@ export function createLlmClientWithProvider( }; record(completion, op, messages); const okAt = markOk(); + breakerRecordSuccess(); notifyStatus({ status: "ok", provider: provider.name, @@ -179,45 +330,13 @@ export function createLlmClientWithProvider( return { completion }; } catch (err) { if (shouldFallback(err, config, provider.name)) { - const hostProv = new HostLlmProvider(); + const primaryTerminal = breakerIsTerminal(err); + if (primaryTerminal) breakerTrip(err); try { - const res = await hostProv.complete(messages, input, makeCtx(opts, asProviderLog(rootLogger.child({ channel: "llm.host" })))); - hostFallbacks++; - facadeLog.warn("host.fallback", { - from: provider.name, - op, - reason: summarizeErr(err), + return await callHostFallback(err, messages, input, opts, op, { + keepBreakerOpen: primaryTerminal, + notifyError: true, }); - const completion: LlmCompletion = { - text: res.text, - provider: provider.name, - model: config.model, - finishReason: res.finishReason, - usage: res.usage, - servedBy: "host_fallback", - durationMs: res.durationMs, - }; - record(completion, op, messages); - // The primary provider is still broken even though the host - // bridge saved this call. Tag the slot yellow (`lastFallbackAt`) - // and surface the upstream error to the user via the - // system_error log so they can see *why* fallback engaged. - const fallbackAt = markFallback(err); - notifyOnError(err); - notifyStatus({ - status: "fallback", - provider: provider.name, - model: config.model, - message: summarizeErrMessage(err), - code: err instanceof MemosError ? err.code : undefined, - at: fallbackAt, - durationMs: completion.durationMs, - fallbackProvider: "host", - op, - episodeId: opts?.episodeId, - phase: opts?.phase, - }); - return { completion }; } catch (hostErr) { failures++; const failAt = markFail(hostErr); @@ -225,6 +344,10 @@ export function createLlmClientWithProvider( primary: summarizeErr(err), host: summarizeErr(hostErr), }); + // Primary AND host bridge both failed. Trip on a terminal + // primary error (the one the operator typically needs to fix + // — host bridge failures are usually transient stdio issues). + if (breakerIsTerminal(err)) breakerTrip(err); notifyOnError(hostErr); notifyStatus({ status: "error", @@ -249,6 +372,7 @@ export function createLlmClientWithProvider( } failures++; const failAt = markFail(err); + if (breakerIsTerminal(err)) breakerTrip(err); notifyOnError(err); notifyStatus({ status: "error", @@ -415,6 +539,12 @@ export function createLlmClientWithProvider( const call = buildCallInput(opts, opts?.jsonMode === true); const ctx = makeCtx(opts, asProviderLog(providerLog)); + // Short-circuit stream calls when the breaker is open. We do not + // count a suppressed call against `requests` (no network hit). + if (breakerIsOpen()) { + maybeEmitCircuitOpenStatus(opts, opts?.op ?? "stream"); + throwBreakerOpen(); + } requests++; const start = Date.now(); let acc = ""; @@ -448,6 +578,7 @@ export function createLlmClientWithProvider( if (usage?.promptTokens) totalPromptTokens += usage.promptTokens; if (usage?.completionTokens) totalCompletionTokens += usage.completionTokens; const okAt = markOk(); + breakerRecordSuccess(); notifyStatus({ status: "ok", provider: provider.name, @@ -461,6 +592,7 @@ export function createLlmClientWithProvider( } catch (err) { failures++; const failAt = markFail(err); + if (breakerIsTerminal(err)) breakerTrip(err); facadeLog.error("stream.failed", { err: summarizeErr(err) }); notifyOnError(err); notifyStatus({ @@ -479,6 +611,59 @@ export function createLlmClientWithProvider( } } + async function callHostFallback( + primaryErr: unknown, + messages: LlmMessage[], + input: ProviderCallInput, + opts: LlmCallOptions | undefined, + op: string, + behavior: { keepBreakerOpen: boolean; notifyError: boolean }, + ): Promise<{ completion: LlmCompletion }> { + const hostProv = new HostLlmProvider(); + const res = await hostProv.complete( + messages, + input, + makeCtx(opts, asProviderLog(rootLogger.child({ channel: "llm.host" }))), + ); + hostFallbacks++; + facadeLog.warn("host.fallback", { + from: provider.name, + op, + reason: summarizeErr(primaryErr), + }); + const completion: LlmCompletion = { + text: res.text, + provider: provider.name, + model: config.model, + finishReason: res.finishReason, + usage: res.usage, + servedBy: "host_fallback", + durationMs: res.durationMs, + }; + record(completion, op, messages); + // The primary provider is still broken even though the host bridge + // saved this call. Keep the breaker open for terminal primary + // errors so later calls can go straight to host fallback without + // touching the paid provider again. + const fallbackAt = markFallback(primaryErr); + if (!behavior.keepBreakerOpen) breakerRecordSuccess(); + if (behavior.notifyError) notifyOnError(primaryErr); + notifyStatus({ + status: "fallback", + provider: provider.name, + model: config.model, + message: summarizeErrMessage(primaryErr), + code: primaryErr instanceof MemosError ? primaryErr.code : undefined, + at: fallbackAt, + durationMs: completion.durationMs, + fallbackProvider: "host", + op, + episodeId: opts?.episodeId, + phase: opts?.phase, + }); + return { completion }; + } + const client: LlmClient = { provider: provider.name, model: config.model, @@ -497,6 +682,9 @@ export function createLlmClientWithProvider( lastOkAt, lastFallbackAt, lastError, + circuitOpen: breakerIsOpen(), + circuitOpenUntil, + circuitOpenedReason, }; }, resetStats(): void { @@ -509,6 +697,9 @@ export function createLlmClientWithProvider( lastOkAt = null; lastFallbackAt = null; lastError = null; + circuitOpenUntil = null; + circuitOpenedReason = null; + lastCircuitOpenStatusAt = null; }, async close(): Promise { await provider.close?.(); @@ -522,6 +713,10 @@ export function createLlmClientWithProvider( timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, fallbackToHost: config.fallbackToHost, + circuitBreaker: { + enabled: breakerEnabled, + cooldownMs: breakerCooldownMs, + }, }); return client; @@ -562,6 +757,40 @@ function shouldFallback(err: unknown, config: LlmConfig, providerName: LlmProvid ); } +/** + * Default circuit-breaker classifier for terminal provider errors. + * + * A "terminal" error is one that will keep failing until the operator + * intervenes (top up balance, fix API key, fix model name). Retrying + * such an error just burns paid quota and pollutes the audit log, so + * the breaker opens and short-circuits further calls for the cool- + * down window. Issue #1897 reports the symptom — ~12,900 paid LLM + * requests in 24 h against a key with insufficient balance. + * + * Detection sources, in order: + * 1. `MemosError(LLM_UNAVAILABLE)` with `details.status` ∈ 401/402/403 + * — set by `core/llm/fetcher.ts::httpPostJson` for non-ok HTTP + * responses. + * 2. Well-known lowercase phrases in the error message (so providers + * that return 400 for "Insufficient Balance" — looking at you, + * DeepSeek — are still recognized). + */ +function defaultIsTerminal(err: unknown): boolean { + if (!(err instanceof MemosError)) return false; + if (err.code !== ERROR_CODES.LLM_UNAVAILABLE) return false; + const status = Number((err.details as { status?: unknown } | undefined)?.status); + if (status === 401 || status === 402 || status === 403) return true; + const msg = (err.message ?? "").toLowerCase(); + return ( + msg.includes("insufficient balance") || + msg.includes("invalid api key") || + msg.includes("invalid_api_key") || + msg.includes("unauthorized") || + msg.includes("account suspended") || + msg.includes("billing") + ); +} + // ─── Logger adapter ────────────────────────────────────────────────────────── function asProviderLog(log: Logger): LlmProviderLogger { diff --git a/apps/memos-local-plugin/core/llm/index.ts b/apps/memos-local-plugin/core/llm/index.ts index 847c965f0..a295cd4a4 100644 --- a/apps/memos-local-plugin/core/llm/index.ts +++ b/apps/memos-local-plugin/core/llm/index.ts @@ -28,6 +28,7 @@ export { LocalOnlyLlmProvider } from "./providers/local-only.js"; export * from "./prompts/index.js"; export type { LlmCallOptions, + LlmCircuitBreakerConfig, LlmCompleteJsonOptions, LlmCompletion, LlmClient, @@ -40,6 +41,7 @@ export type { LlmProviderLogger, LlmProviderName, LlmRole, + LlmStatusDetail, LlmStreamChunk, LlmUsage, ProviderCallInput, diff --git a/apps/memos-local-plugin/core/llm/providers/openai.ts b/apps/memos-local-plugin/core/llm/providers/openai.ts index c24c96236..1801a5aad 100644 --- a/apps/memos-local-plugin/core/llm/providers/openai.ts +++ b/apps/memos-local-plugin/core/llm/providers/openai.ts @@ -50,18 +50,19 @@ export class OpenAiLlmProvider implements LlmProvider { ctx: LlmProviderCtx, ): Promise { const { config, log, signal } = ctx; - if (!config.apiKey) { - throw new MemosError( - ERROR_CODES.LLM_UNAVAILABLE, - "openai_compatible provider requires config.llm.apiKey", - { provider: this.name }, - ); - } const url = normalizeEndpoint( config.endpoint && config.endpoint.length > 0 ? config.endpoint : "https://api.openai.com/v1/chat/completions", ); + const isLocal = isLocalhostOrPrivateUrl(url); + if (!config.apiKey && !isLocal) { + throw new MemosError( + ERROR_CODES.LLM_UNAVAILABLE, + "openai_compatible provider requires config.llm.apiKey (or use a local endpoint)", + { provider: this.name }, + ); + } const model = config.model && config.model.length > 0 ? config.model : "gpt-4o-mini"; const body: Record = { @@ -73,13 +74,16 @@ export class OpenAiLlmProvider implements LlmProvider { if (opts.jsonMode) body.response_format = { type: "json_object" }; if (opts.stop && opts.stop.length > 0) body.stop = opts.stop; + const headers: Record = {}; + if (config.apiKey) { + headers.Authorization = `Bearer ${config.apiKey}`; + } + Object.assign(headers, config.headers); + const { json, durationMs } = await httpPostJson({ url, body, - headers: { - Authorization: `Bearer ${config.apiKey}`, - ...config.headers, - }, + headers, timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, signal, @@ -109,18 +113,19 @@ export class OpenAiLlmProvider implements LlmProvider { ctx: LlmProviderCtx, ): AsyncGenerator { const { config, log, signal } = ctx; - if (!config.apiKey) { - throw new MemosError( - ERROR_CODES.LLM_UNAVAILABLE, - "openai_compatible provider requires config.llm.apiKey", - { provider: this.name }, - ); - } const url = normalizeEndpoint( config.endpoint && config.endpoint.length > 0 ? config.endpoint : "https://api.openai.com/v1/chat/completions", ); + const isLocal = isLocalhostOrPrivateUrl(url); + if (!config.apiKey && !isLocal) { + throw new MemosError( + ERROR_CODES.LLM_UNAVAILABLE, + "openai_compatible provider requires config.llm.apiKey (or use a local endpoint)", + { provider: this.name }, + ); + } const model = config.model && config.model.length > 0 ? config.model : "gpt-4o-mini"; const body: Record = { @@ -133,13 +138,16 @@ export class OpenAiLlmProvider implements LlmProvider { if (opts.jsonMode) body.response_format = { type: "json_object" }; if (opts.stop && opts.stop.length > 0) body.stop = opts.stop; + const headers: Record = {}; + if (config.apiKey) { + headers.Authorization = `Bearer ${config.apiKey}`; + } + Object.assign(headers, config.headers); + const resp = await httpPostStream({ url, body, - headers: { - Authorization: `Bearer ${config.apiKey}`, - ...config.headers, - }, + headers, timeoutMs: config.timeoutMs, signal, provider: this.name, @@ -211,3 +219,25 @@ function mapFinish(reason: string | undefined): ProviderCompletion["finishReason return "other"; } } + +/** + * Return true if the URL points to localhost or a private-network address. + * Used to relax the apiKey requirement for local/self-hosted inference servers. + */ +function isLocalhostOrPrivateUrl(url: string): boolean { + try { + const u = new URL(url); + const h = u.hostname.toLowerCase(); + if (h === "localhost" || h === "127.0.0.1" || h === "::1") return true; + // Private ranges: 10.x, 172.16-31.x, 192.168.x + if (h.startsWith("10.") || h.startsWith("192.168.")) return true; + const m = h.match(/^172\.(\d+)\./); + if (m) { + const n = parseInt(m[1], 10); + if (n >= 16 && n <= 31) return true; + } + } catch { + // Malformed URL — let the caller handle it. + } + return false; +} diff --git a/apps/memos-local-plugin/core/llm/types.ts b/apps/memos-local-plugin/core/llm/types.ts index ddfe80c1e..4a835caee 100644 --- a/apps/memos-local-plugin/core/llm/types.ts +++ b/apps/memos-local-plugin/core/llm/types.ts @@ -47,6 +47,33 @@ export interface LlmConfig { * daemon can display status produced by a separate stdio bridge. */ onStatus?: (detail: LlmStatusDetail) => void; + /** + * Optional circuit breaker config. The breaker trips on terminal + * provider errors (HTTP 401/402/403, or well-known phrases like + * "insufficient balance" / "invalid api key" / "unauthorized" / + * "account suspended" / "billing") and short-circuits subsequent + * calls for a cool-down window. Defaults to enabled. See + * `apps/memos-local-plugin/openspec/changes/.../design.md` + * (issue #1897) for the full state machine. + */ + circuitBreaker?: LlmCircuitBreakerConfig; +} + +export interface LlmCircuitBreakerConfig { + /** Default true. Set false to restore legacy (no-breaker) behavior. */ + enabled?: boolean; + /** + * Cool-down window before the breaker enters half-open. Default + * 300_000 ms (5 minutes); minimum clamped to 30_000 ms. + */ + cooldownMs?: number; + /** + * Override the default classifier. Returns true if the error should + * trip the breaker (terminal / non-recoverable). + */ + isTerminal?: (err: unknown) => boolean; + /** Injected clock for tests. Default `Date.now`. */ + now?: () => number; } export interface LlmErrorDetail { @@ -67,7 +94,7 @@ export interface LlmErrorDetail { } export interface LlmStatusDetail { - status: "ok" | "fallback" | "error"; + status: "ok" | "fallback" | "error" | "circuit_open"; provider: LlmProviderName | string; model: string; message?: string; @@ -260,6 +287,17 @@ export interface LlmClientStats extends LastCallStatus { retries: number; totalPromptTokens: number; totalCompletionTokens: number; + /** + * True while the per-client circuit breaker is open (and any + * cooldown timer has not yet elapsed). When true, further calls are + * short-circuited inside the facade and throw immediately without + * touching the provider. See issue #1897. + */ + circuitOpen: boolean; + /** Epoch ms at which the open breaker becomes eligible for half-open probe. */ + circuitOpenUntil: number | null; + /** Free-text reason from the error that opened the breaker. */ + circuitOpenedReason: string | null; } export interface LlmClient { diff --git a/apps/memos-local-plugin/core/memory/l2/l2.ts b/apps/memos-local-plugin/core/memory/l2/l2.ts index 9c6685409..903502b32 100644 --- a/apps/memos-local-plugin/core/memory/l2/l2.ts +++ b/apps/memos-local-plugin/core/memory/l2/l2.ts @@ -34,7 +34,7 @@ import { L2_INDUCTION_PROMPT } from "../../llm/prompts/l2-induction.js"; import { associateTraces } from "./associate.js"; import { makeCandidatePool } from "./candidate-pool.js"; import { buildPolicyRow, induceDraft } from "./induce.js"; -import { applyGain, computeGain, smoothGain } from "./gain.js"; +import { applyGain, computeGain, nextStatus, smoothGain } from "./gain.js"; import { signatureOf } from "./signature.js"; import { tracePolicySimilarity } from "./similarity.js"; import type { @@ -455,6 +455,47 @@ export async function runL2( timings.gain = Date.now() - t0; } + // ─── Step 5: Re-evaluate untouched candidates ──────────────────────── + // A candidate policy that was induced in a previous episode but no longer + // matches any trace will never enter `touched` and therefore never have + // `nextStatus()` run against it. Without this sweep, it stays `candidate` + // forever even though its stored gain/support already satisfy the + // promotion thresholds. + { + const untouchedCandidates = repos.policies.list({ status: "candidate" }); + for (const policy of untouchedCandidates) { + if (touched.has(policy.id)) continue; // already handled in Step 4 + const next = nextStatus({ + currentStatus: policy.status, + support: policy.support, + gain: policy.gain, + thresholds, + }); + if (next !== policy.status) { + repos.policies.updateStats(policy.id, { + support: policy.support, + gain: policy.gain, + status: next, + updatedAt: input.now ?? Date.now(), + }); + emit(bus, { + kind: "l2.policy.updated", + episodeId: input.episodeId, + policyId: policy.id, + status: next, + support: policy.support, + gain: policy.gain, + }); + log.info("run.recheck_candidate_promoted", { + policyId: policy.id, + status: next, + support: policy.support, + gain: policy.gain, + }); + } + } + } + timings.persist = 0; // reserved for future split const completedAt = Date.now(); timings.total = completedAt - startedAt; diff --git a/apps/memos-local-plugin/core/pipeline/deps.ts b/apps/memos-local-plugin/core/pipeline/deps.ts index fd9c2bbf0..7bd23d681 100644 --- a/apps/memos-local-plugin/core/pipeline/deps.ts +++ b/apps/memos-local-plugin/core/pipeline/deps.ts @@ -161,6 +161,7 @@ export function extractAlgorithmConfig( llmFilterMinCandidates: alg.lightweightMemory.enabled ? 1 : alg.retrieval.llmFilterMinCandidates, llmFilterCandidateBodyChars: alg.retrieval.llmFilterCandidateBodyChars, lightweightMemory: alg.lightweightMemory.enabled, + vectorScanMaxAgeMs: alg.retrieval.vectorScanMaxAgeMs, }, session: { followUpMode: alg.session.followUpMode, diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index b4e331c71..2ae14f053 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -79,7 +79,13 @@ import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; import { openDb } from "../storage/connection.js"; import { runMigrations } from "../storage/migrator.js"; -import { makeRepos } from "../storage/repos/index.js"; +import { + makeRepos, + embeddingMaintenanceCounts, + inferStoredEmbeddingByteLen, + FLOAT32_BYTES, +} from "../storage/repos/index.js"; +import type { EmbeddingCountsBucket } from "../storage/repos/index.js"; import { createEmbedder } from "../embedding/embedder.js"; import { createLlmClient } from "../llm/client.js"; import { @@ -112,7 +118,6 @@ import type { UserFeedback } from "../reward/types.js"; const FINAL_HUB_LLM_FILTER_TIMEOUT_MS = 3_000; const IMPORT_WRITE_BATCH_SIZE = 500; - export interface BootstrapOptions { agent: AgentKind; namespace?: RuntimeNamespace; @@ -561,6 +566,31 @@ export function createMemoryCore( handle.config.algorithm.session.mergeMaxGapMs * 2, 4 * 60 * 60 * 1000, ); + + // ─── Dirty closed-episode rescore backoff (issue #1808) ── + // Failed reward / reflect runs on dirty closed episodes used to be + // retried on every restart and every periodic rescan. The OpenClaw + // Gateway report at #1808 attributed >3s `eventLoopMax` bursts to this + // tight retry loop hammering the LLM on already-broken rows. We now + // track per-episode failure counts in `meta.rewardDirty` and require + // an exponential cool-down before retrying a row that has already + // failed `MAX_DIRTY_REWARD_ATTEMPTS` times. Manual feedback / + // `runManually` still rescore unconditionally — the backoff only + // applies to the automatic rescan paths. + const MAX_DIRTY_REWARD_ATTEMPTS = 3; + const DIRTY_REWARD_BACKOFF_BASE_MS = 60 * 60 * 1000; // 1h + const DIRTY_REWARD_BACKOFF_MAX_MS = 24 * 60 * 60 * 1000; // 24h + + // ─── Startup recovery background promise (issue #1776 + #1808) ── + // `init()` used to `await` the entire reflect → reward → L2 chain for + // every stale / dirty episode found in SQLite. On databases with + // 30k+ traces this blocked the Gateway's main event loop for 3-5s+, + // long enough to time out the WebSocket read probe (3s budget) used + // by TUI / Control UI clients. We now keep the synchronous + // classification on the main thread (it only touches the DB) and + // detach the slow recovery to this promise. `waitForStartupRecovery` + // exposes it so tests can opt back into the deterministic semantics. + let startupRecoveryPromise: Promise = Promise.resolve(); let lastStaleScan = 0; let lastDirtyClosedScan = 0; async function autoFinalizeStaleTasks(): Promise { @@ -598,9 +628,12 @@ export function createMemoryCore( if (nowMs - lastDirtyClosedScan < 30_000) return; lastDirtyClosedScan = nowMs; try { - const dirtyClosed = handle.repos.episodes + const allDirty = handle.repos.episodes .list({ status: "closed", limit: 500 }) .filter((ep) => !isLightweightEpisode(ep) && episodeRewardIsDirty(ep)); + // Apply the same backoff filter as init() so the 10-min periodic + // scan does not hammer episodes whose LLM call keeps failing. + const dirtyClosed = allDirty.filter((ep) => dirtyEpisodeBackoffElapsed(ep, nowMs)); if (dirtyClosed.length > 0) { await recoverDirtyClosedEpisodes(dirtyClosed); } @@ -879,6 +912,16 @@ export function createMemoryCore( // not evidence that the topic ended; the next user turn gets routed // through relation classification. Only hard-stale open topics are // finalized here so the pipeline eventually catches up. + // + // ── Issue #1776 + #1808: stale + dirty recovery used to run inside + // `await` here, blocking `init()` for seconds-to-minutes on big + // databases and starving the OpenClaw Gateway event loop. We now + // collect the slow inputs synchronously (the DB list + classify is + // cheap) and detach the actual reflect / reward chain onto + // `startupRecoveryPromise`. Tests that need the historic semantics + // call `core.waitForStartupRecovery()` after `init()`. + let staleForBackground: Array }> = []; + let dirtyClosedForBackground: Array }> = []; try { const orphans = handle.repos.episodes.list({ status: "open", limit: 500 }); if (orphans.length > 0) { @@ -908,22 +951,68 @@ export function createMemoryCore( recoveredAtStartup: nowMs, }); } - if (stale.length > 0) { - await recoverOpenEpisodesAsSessionEnd(stale); - } + staleForBackground = stale; } - const dirtyClosed = handle.repos.episodes + const nowForDirty = Date.now(); + const allDirty = handle.repos.episodes .list({ status: "closed", limit: 500 }) .filter((ep) => !isLightweightEpisode(ep) && episodeRewardIsDirty(ep)); - if (dirtyClosed.length > 0) { - await recoverDirtyClosedEpisodes(dirtyClosed); + const dirtyClosed: typeof allDirty = []; + for (const ep of allDirty) { + if (dirtyEpisodeBackoffElapsed(ep, nowForDirty)) { + dirtyClosed.push(ep); + } else { + const dirtyMeta = (ep.meta?.rewardDirty as + | { failedAttempts?: number; lastFailureAt?: number } + | undefined) ?? {}; + log.debug("init.dirty_closed_episodes.skip_backoff", { + episodeId: ep.id, + failedAttempts: dirtyMeta.failedAttempts ?? 0, + lastFailureAt: dirtyMeta.lastFailureAt ?? 0, + }); + } } + dirtyClosedForBackground = dirtyClosed; } catch (err) { log.debug("init.orphan_scan.failed", { err: err instanceof Error ? err.message : String(err), }); } + // Kick the slow recovery chain off the main thread. `init()` returns + // as soon as the synchronous classification above finishes, so the + // Gateway can start accepting WebSocket upgrades immediately. + if (staleForBackground.length > 0 || dirtyClosedForBackground.length > 0) { + const stale = staleForBackground; + const dirtyClosed = dirtyClosedForBackground; + log.info("init.background_recovery_started", { + staleCount: stale.length, + dirtyClosedCount: dirtyClosed.length, + }); + const recoveryStartedAt = Date.now(); + startupRecoveryPromise = (async () => { + try { + if (stale.length > 0) { + await recoverOpenEpisodesAsSessionEnd(stale); + } + if (dirtyClosed.length > 0) { + await recoverDirtyClosedEpisodes(dirtyClosed); + } + log.info("init.background_recovery_finished", { + staleCount: stale.length, + dirtyClosedCount: dirtyClosed.length, + durationMs: Date.now() - recoveryStartedAt, + }); + } catch (err) { + log.warn("init.background_recovery_failed", { + err: err instanceof Error ? err.message : String(err), + staleCount: stale.length, + dirtyClosedCount: dirtyClosed.length, + }); + } + })(); + } + // Periodic rescore timer for episodes that miss the startup scan or // retry of failed reward runs. 10-minute interval is safe because // autoRescoreDirtyClosedEpisodes has its own 30-second dedup guard. @@ -1252,10 +1341,21 @@ export function createMemoryCore( episodes: Array }>, ): Promise { log.info("init.dirty_closed_episodes.rescore", { count: episodes.length }); + // Snapshot the prior failure counters so we can increment them later + // (after the bus chain settles) without an extra DB read. + const priorFailedAttempts = new Map(); for (const ep of episodes) { if (isLightweightEpisode(ep)) continue; const episodeId = ep.id as EpisodeId; const endedAt = ep.endedAt ?? Date.now(); + const prevDirty = (ep.meta?.rewardDirty as + | { failedAttempts?: unknown } + | undefined) ?? {}; + const prevAttempts = + typeof prevDirty.failedAttempts === "number" + ? prevDirty.failedAttempts + : 0; + priorFailedAttempts.set(episodeId, prevAttempts); handle.repos.episodes.updateMeta(episodeId, { closeReason: "finalized", recoveredAtStartup: endedAt, @@ -1271,6 +1371,31 @@ export function createMemoryCore( }); } await handle.flush(); + // After the reward / reflect chain has finished, account for the + // outcome: clear `meta.rewardDirty` on episodes that are no longer + // dirty (success), bump `failedAttempts + lastFailureAt` on episodes + // that still match the dirty predicate (LLM failure / no-op). This + // closes the "retried indefinitely" loop reported in issue #1808. + const now = Date.now(); + for (const [episodeId, prevAttempts] of priorFailedAttempts) { + const after = handle.repos.episodes.getById(episodeId); + if (!after) continue; + const stillDirty = episodeRewardIsDirty(after); + if (stillDirty) { + handle.repos.episodes.updateMeta(episodeId, { + rewardDirty: { + failedAttempts: prevAttempts + 1, + lastFailureAt: now, + }, + }); + } else if ( + after.meta && + typeof after.meta === "object" && + "rewardDirty" in after.meta + ) { + handle.repos.episodes.updateMeta(episodeId, { rewardDirty: undefined }); + } + } } function episodeRewardIsDirty(ep: EpisodeRow & { meta?: Record }): boolean { @@ -1294,7 +1419,20 @@ export function createMemoryCore( if (!reward || typeof reward !== "object") return false; const traceCount = (reward as { traceCount?: unknown }).traceCount; if (typeof traceCount === "number") { - return traceCount !== (ep.traceIds?.length ?? 0); + // Compare against the count of trace IDs that ACTUALLY exist in the + // traces table, not the raw length of `ep.traceIds`. Otherwise a + // single "ghost" trace ID lingering in `trace_ids_json` (deleted + // trace row, manual cleanup, partial migration) keeps the episode + // dirty forever and triggers a rescore every 10 minutes — + // https://github.com/MemTensor/MemOS/issues/1966 (590 wasted calls + // / ~14.5 RMB in the reporter's case). `countExisting` is a single + // `SELECT COUNT(*)` per chunk, so it stays cheap even on big DBs. + const traceIds = (ep.traceIds ?? []) as TraceId[]; + const existingCount = + traceIds.length === 0 + ? 0 + : handle.repos.traces.countExisting(traceIds); + return traceCount !== existingCount; } // Backward compatibility for episodes scored before reward coverage @@ -1315,6 +1453,37 @@ export function createMemoryCore( return handle.repos.traces.hasAnyNewerThan(traceIds, scoredAt); } + /** + * Backoff filter for the automatic dirty-rescore scans (issue #1808). + * + * Returns true when the row is eligible for another reward rescore on + * the auto path (init scan + 10-minute periodic). When a row has + * failed `MAX_DIRTY_REWARD_ATTEMPTS` consecutive automatic rescores, + * we wait an exponentially increasing window (1h → 24h cap) before + * attempting again. This stops the "retried indefinitely with no + * backoff" symptom reported on the OpenClaw Gateway. + * + * Manual paths (`submitFeedback`, `runManually`) do NOT consult this + * filter — explicit user / operator intent should always re-trigger. + */ + function dirtyEpisodeBackoffElapsed( + ep: EpisodeRow & { meta?: Record }, + nowMs: number, + ): boolean { + const dirty = (ep.meta?.rewardDirty as + | { failedAttempts?: unknown; lastFailureAt?: unknown } + | undefined) ?? {}; + const attempts = typeof dirty.failedAttempts === "number" ? dirty.failedAttempts : 0; + if (attempts < MAX_DIRTY_REWARD_ATTEMPTS) return true; + const lastFailureAt = typeof dirty.lastFailureAt === "number" ? dirty.lastFailureAt : 0; + const exponent = Math.min(attempts - MAX_DIRTY_REWARD_ATTEMPTS, 5); + const wait = Math.min( + DIRTY_REWARD_BACKOFF_BASE_MS * (1 << exponent), + DIRTY_REWARD_BACKOFF_MAX_MS, + ); + return nowMs - lastFailureAt >= wait; + } + function snapshotFromRecoveredEpisode( ep: EpisodeRow & { meta?: Record }, endedAt: number, @@ -1422,6 +1591,16 @@ export function createMemoryCore( if (shutDown) return; shutDown = true; try { + // Make sure the background startup recovery (issue #1808) has + // finished before we tear down the bus / DB handle. Without this + // wait, a fast `init → shutdown` race during tests or a quick + // gateway reload would close SQLite while reflect / reward is + // mid-flush, producing `SQLITE_MISUSE` noise on the way down. + try { + await startupRecoveryPromise; + } catch { + /* already logged inside the recovery promise */ + } try { await hubRuntime?.stop(); } catch (err) { @@ -1452,10 +1631,20 @@ export function createMemoryCore( /* fall through to in-memory */ } + // The Overview cards' source of truth for "what model is this slot + // running?" is config.yaml (= the Settings page). Runtime stats + // (lastOkAt / lastError / fallback timestamps) still come from the + // in-memory facades — that's the right split: the slot label is + // user intent, the colour/error reflects whether the runtime has + // actually been able to talk to the configured upstream. See #1596. + const effectiveConfig = diskConfig ?? handle.config; + const llmInfo = llmHealth(handle.llm, latestTraceTs()); const embedderInfo = embedderHealth(handle.embedder, latestTraceTs()); + applyConfiguredModelDisplay(effectiveConfig, llmInfo, embedderInfo); + const skillEvolverInfo = resolveSkillEvolver( - diskConfig ?? handle.config, + effectiveConfig, // Prefer the dedicated reflect LLM stats so an independently // configured skill-evolver model reports its OWN failures // instead of inheriting the (possibly healthy) summary LLM's @@ -1463,6 +1652,7 @@ export function createMemoryCore( // skillEvolver blank — bootstrap aliases reflectLlm to llm // in that case anyway. handle.reflectLlm ?? handle.llm, + llmInfo, latestTraceTs(), ); @@ -1475,21 +1665,6 @@ export function createMemoryCore( // in-memory stats — if you want to inspect past failures, head // to LogsView → 系统 tag. - // Override model names from disk config if they differ from the - // in-memory client (user saved new settings but hasn't restarted). - if (diskConfig) { - const diskLlm = diskConfig.llm as { model?: string; provider?: string } | undefined; - if (diskLlm?.model && diskLlm.model !== llmInfo.model) { - llmInfo.model = diskLlm.model; - if (diskLlm.provider) llmInfo.provider = diskLlm.provider; - } - const diskEmb = diskConfig.embedding as { model?: string; provider?: string } | undefined; - if (diskEmb?.model && diskEmb.model !== embedderInfo.model) { - embedderInfo.model = diskEmb.model; - if (diskEmb.provider) embedderInfo.provider = diskEmb.provider; - } - } - applyPersistedModelStatus(handle.repos, "llm", llmInfo); applyPersistedModelStatus(handle.repos, "embedding", embedderInfo); applyPersistedModelStatus( @@ -4032,24 +4207,39 @@ export function createMemoryCore( }; function computeEmbeddingMaintenanceStats(): EmbeddingMaintenanceStats { + // SQL-only fast path (issue #1929). + // + // The previous implementation paginated `traces` / `policies` / + // `world_model` / `skills` end-to-end via `repos..list()`, + // which hydrates the full row — BLOB vector columns included — + // through `mapRow()`. On a production deployment with ~93K rows + // and ~270 MB of vector BLOBs that single call blocked the Node + // event loop for 4+ minutes at 100% CPU. + // + // `embeddingMaintenanceCounts` runs five `SELECT COUNT(*) + + // SUM(CASE WHEN ...)` queries — `LENGTH(blob)` reads only the BLOB + // header, never the payload — so we keep the same per-bucket + // semantics without touching a single vector byte. const configuredDimension = handle.embedder?.dimensions ?? 0; - const allSlots = collectEmbeddingSlots(); - const dimension = configuredDimension > 0 ? configuredDimension : inferStoredEmbeddingDimension(allSlots); - const byKind = emptyEmbeddingStatsByKind(); - for (const slot of allSlots) { - const bucket = byKind[slot.kind]; - bucket.totalSlots++; - if (!slot.vec) { - bucket.missing++; - } else if (dimension > 0 && slot.vec.length !== dimension) { - bucket.dimMismatch++; - } else { - bucket.ready++; - } - } - for (const bucket of Object.values(byKind)) { - bucket.needsRepair = bucket.missing + bucket.dimMismatch; - } + const expectedByteLenFromEmbedder = configuredDimension > 0 + ? configuredDimension * FLOAT32_BYTES + : 0; + // When the embedder has not been probed yet, fall back to the most common + // stored BLOB byte length (mirrors the pre-fix `inferStoredEmbeddingDimension` + // path, but computed via SQL `GROUP BY LENGTH(vec_summary)` — never touches + // the BLOB bodies). + const expectedByteLen = expectedByteLenFromEmbedder > 0 + ? expectedByteLenFromEmbedder + : inferStoredEmbeddingByteLen(handle.db); + const dimension = expectedByteLen > 0 ? expectedByteLen / FLOAT32_BYTES : 0; + + const raw = embeddingMaintenanceCounts(handle.db, { expectedByteLen }); + const byKind: EmbeddingMaintenanceStats["byKind"] = { + trace: addNeedsRepair(raw.trace), + policy: addNeedsRepair(raw.policy), + world_model: addNeedsRepair(raw.world_model), + skill: addNeedsRepair(raw.skill), + }; const totalSlots = sumEmbeddingStats(byKind, "totalSlots"); const ready = sumEmbeddingStats(byKind, "ready"); const missing = sumEmbeddingStats(byKind, "missing"); @@ -4066,6 +4256,15 @@ export function createMemoryCore( }; } + function addNeedsRepair( + bucket: EmbeddingCountsBucket, + ): EmbeddingCountsBucket & { needsRepair: number } { + return { + ...bucket, + needsRepair: bucket.missing + bucket.dimMismatch, + }; + } + async function ensureEmbeddingDimensionKnown(): Promise { if (!handle.embedder || handle.embedder.dimensions > 0) return; try { @@ -4080,23 +4279,6 @@ export function createMemoryCore( } } - function inferStoredEmbeddingDimension(slots: readonly EmbeddingSlot[]): number { - const counts = new Map(); - for (const slot of slots) { - if (!slot.vec) continue; - counts.set(slot.vec.length, (counts.get(slot.vec.length) ?? 0) + 1); - } - let bestDim = 0; - let bestCount = 0; - for (const [dim, count] of counts) { - if (count > bestCount) { - bestDim = dim; - bestCount = count; - } - } - return bestDim; - } - function shouldTraceHaveEmbeddings(row: TraceRow): boolean { // Skip traces where both user and agent text are very short const userLen = row.userText.trim().length; @@ -4206,22 +4388,6 @@ export function createMemoryCore( return row.tags.includes("lightweight_memory"); } - function emptyEmbeddingStatsByKind(): EmbeddingMaintenanceStats["byKind"] { - const empty = () => ({ - totalSlots: 0, - ready: 0, - missing: 0, - dimMismatch: 0, - needsRepair: 0, - }); - return { - trace: empty(), - policy: empty(), - world_model: empty(), - skill: empty(), - }; - } - function sumEmbeddingStats( byKind: EmbeddingMaintenanceStats["byKind"], key: "totalSlots" | "ready" | "missing" | "dimMismatch" | "needsRepair", @@ -4498,6 +4664,7 @@ export function createMemoryCore( init, shutdown, health, + waitForStartupRecovery: () => startupRecoveryPromise, bindTelemetry(t: import("../telemetry/index.js").Telemetry) { telemetry = t; }, openSession, closeSession, @@ -5357,6 +5524,7 @@ function embedderHealth( function resolveSkillEvolver( config: PipelineHandle["config"], llm: PipelineHandle["llm"], + inheritedLlmInfo: CoreHealth["llm"], fallbackTs: number | null, ): CoreHealth["skillEvolver"] { const evolver = (config as { skillEvolver?: { provider?: string; model?: string } }) @@ -5378,16 +5546,60 @@ function resolveSkillEvolver( lastError: s?.lastError ?? null, }; } - const fallback = llmHealth(llm, fallbackTs); + // Inherited skillEvolver mirrors the (already-disk-aware) llm slot, + // so the Overview's three model cards never disagree about what the + // current Settings say. Runtime stats still come from the llm + // client; we just copy whatever the Overview will show for the LLM + // slot itself. See #1596. return { - available: fallback.available, - provider: fallback.provider, - model: fallback.model, + available: inheritedLlmInfo.available, + provider: inheritedLlmInfo.provider, + model: inheritedLlmInfo.model, inherited: true, - lastOkAt: fallback.lastOkAt, - lastFallbackAt: fallback.lastFallbackAt, - lastError: fallback.lastError, + lastOkAt: inheritedLlmInfo.lastOkAt, + lastFallbackAt: inheritedLlmInfo.lastFallbackAt, + lastError: inheritedLlmInfo.lastError, + }; + // Reserved for future signature change — keep fallbackTs parameter + // so callers passing `latestTraceTs()` don't need to change. + void fallbackTs; +} + +/** + * Patch the llm + embedder health snapshots so their `model` and + * `provider` fields reflect what's currently in `config.yaml` — i.e. + * what the Settings page shows. The runtime stats (available, + * lastOkAt, lastError) stay sourced from the in-memory facade because + * those represent "did the upstream actually answer", which only the + * runtime can know. See #1596: Overview cards used to lag behind a + * Settings save when only the provider changed (model name unchanged), + * or when the user cleared a model name back to empty. + */ +function applyConfiguredModelDisplay( + config: PipelineHandle["config"], + llmInfo: CoreHealth["llm"], + embedderInfo: CoreHealth["embedder"], +): void { + const cfg = config as { + llm?: { model?: unknown; provider?: unknown }; + embedding?: { model?: unknown; provider?: unknown }; }; + if (cfg.llm) { + if (typeof cfg.llm.model === "string") { + llmInfo.model = cfg.llm.model; + } + if (typeof cfg.llm.provider === "string" && cfg.llm.provider.length > 0) { + llmInfo.provider = cfg.llm.provider; + } + } + if (cfg.embedding) { + if (typeof cfg.embedding.model === "string") { + embedderInfo.model = cfg.embedding.model; + } + if (typeof cfg.embedding.provider === "string" && cfg.embedding.provider.length > 0) { + embedderInfo.provider = cfg.embedding.provider; + } + } } function writeApiLog( diff --git a/apps/memos-local-plugin/core/pipeline/orchestrator.ts b/apps/memos-local-plugin/core/pipeline/orchestrator.ts index 75dc7e244..1e57b4802 100644 --- a/apps/memos-local-plugin/core/pipeline/orchestrator.ts +++ b/apps/memos-local-plugin/core/pipeline/orchestrator.ts @@ -1367,6 +1367,7 @@ export function createPipeline(deps: PipelineDeps): PipelineHandle { await subs.l3.drain(); await nextTick(); await subs.skills.flush(); + await subs.skills.lifecycleTick(); await subs.feedback.flush(); await embeddingRetryWorker.flush(); } diff --git a/apps/memos-local-plugin/core/retrieval/llm-filter.ts b/apps/memos-local-plugin/core/retrieval/llm-filter.ts index 15868fb68..2d1f77180 100644 --- a/apps/memos-local-plugin/core/retrieval/llm-filter.ts +++ b/apps/memos-local-plugin/core/retrieval/llm-filter.ts @@ -24,12 +24,32 @@ import type { LlmClient } from "../llm/index.js"; import type { Logger } from "../logger/types.js"; import { RETRIEVAL_FILTER_PROMPT } from "../llm/prompts/index.js"; import type { RankedCandidate } from "./ranker.js"; -import type { RetrievalConfig } from "./types.js"; +import type { RetrievalConfig, TraceCandidate } from "./types.js"; const DEFAULT_CANDIDATE_BODY_CHARS = 500; const MIN_FILTER_OUTPUT_TOKENS = 160; const MAX_FILTER_OUTPUT_TOKENS = 2048; +/** + * A trace whose `agentText` falls under this length, with no LLM summary + * or reflection to back it up, is treated as a near-duplicate question + * trace (issue #1913). The rescue path keeps these *behind* informative + * candidates so the answer-bearing trace surfaces first. + */ +const INFORMATIVE_AGENT_TEXT_MIN_CHARS = 20; + +/** + * Short acknowledgement / scaffold replies that the filter prompt + * rightly classes as "scaffolding chatter". When the LLM filter empties + * the kept set we still need to make a rescue call — these strings let + * us prefer informative replies over plain acks. Bounded list, exact + * matches only after trimming surrounding punctuation / whitespace. + */ +const SHORT_ACK_PATTERNS: readonly RegExp[] = [ + /^(ok|okay|sure|got it|noted|understood|alright|will do|copy|copy that|thanks|thank you|✓|✅|👍)[\s.!]*$/i, + /^(记住了|已记住|已经记住|好的|明白|收到|了解|谢谢)[\s。!]*$/, +]; + export interface FilterInput { query: string; ranked: readonly RankedCandidate[]; @@ -70,6 +90,12 @@ export interface FilterResult { | "deferred_to_final" | "llm_kept_all" | "llm_filtered" + // The LLM returned an empty selection over a non-empty ranked list + // (issue #1913 — repeated question traces crowding the hit set). + // We rescued the top-K best-scoring candidates so the agent always + // sees a packet when retrieval succeeded; `sufficient` is forced + // to `false` so downstream callers know the injection is weak. + | "llm_filtered_refilled" // The LLM was supposed to run but the call failed / parsed badly. // We applied a mechanical relevance cutoff (top-K above // `relativeThresholdFloor · topRelevance`) instead of dumping the @@ -169,15 +195,18 @@ ${list}`, ); const keepIndices = new Set(cappedIndices); if (keepIndices.size === 0) { - // Model asked us to drop everything — honoured. Surface this - // explicitly so the Logs page can show "LLM found nothing - // relevant" instead of silently injecting a partial packet. - return { - kept: [], - dropped: [...ranked], - outcome: "llm_filtered", - sufficient: sufficient ?? false, - }; + // Issue #1913: the model asked us to drop everything. Honouring + // that verbatim used to collapse `turn.start` injection to "" even + // when retrieval was healthy — the failure mode is a hit set + // dominated by near-duplicate question traces from prior + // sessions, where each candidate individually looks like + // "surface-similar wrong sub-problem" to the filter prompt. + // Instead, rescue the top-K best-scoring candidates (preferring + // informative traces over pure-question / ack-only chatter) so + // the agent always sees a packet when retrieval succeeded. + // `safeCutoff`'s sibling escape hatch (`llmFilterMaxKeep === 0`) + // is honoured so operators can still ask for hard drop. + return rescueFromEmptySelection(ranked, deps, sufficient); } const kept = cappedIndices.map((i) => ranked[i]!); const dropped: RankedCandidate[] = []; @@ -214,6 +243,95 @@ function passthrough( return { kept: [...ranked], dropped: [], outcome, sufficient: null }; } +/** + * Issue #1913 rescue path. Invoked when the LLM relevance filter + * returned `selected: []` for a *non-empty* ranked candidate list — the + * most common cause is a hit set dominated by near-duplicate question + * traces from previous sessions, where the filter prompt's "drop + * scaffolding chatter" / "drop surface-similar wrong sub-problem" + * rubric is applied to every candidate. + * + * Strategy: keep the top-K best-scoring candidates, preferring + * informative traces (skill / episode / experience / world-model, or a + * trace whose `agentText`/`summary`/`reflection` carries real content) + * over pure-question chatter. We do NOT re-query the LLM — the rescue + * is a single O(n) partition + slice. Outcome label is + * `"llm_filtered_refilled"` so the Logs viewer can show "LLM collapsed, + * safety net fired" distinct from a normal `"llm_filtered"`. + * + * Escape hatch: `llmFilterMaxKeep === 0` skips the rescue entirely and + * honours the "drop everything" request (matches existing `safeCutoff` + * semantics for the same config value). + */ +function rescueFromEmptySelection( + ranked: readonly RankedCandidate[], + deps: FilterDeps, + sufficient: boolean | null, +): FilterResult { + const keepCap = Math.max(0, deps.config.llmFilterMaxKeep); + if (keepCap === 0 || ranked.length === 0) { + return { + kept: [], + dropped: [...ranked], + outcome: "llm_filtered", + sufficient: sufficient ?? false, + }; + } + const informative: RankedCandidate[] = []; + const chatter: RankedCandidate[] = []; + for (const r of ranked) { + if (isInformativeCandidate(r)) informative.push(r); + else chatter.push(r); + } + // Preserve ranker order within each bucket; informative first so the + // answer-bearing trace surfaces even when the ranker placed it below + // surface-similar question traces. + const ordered = [...informative, ...chatter]; + const kept = ordered.slice(0, Math.min(keepCap, ordered.length)); + const keptSet = new Set(kept); + const dropped = ranked.filter((r) => !keptSet.has(r)); + deps.log.debug("llm_filter.collapsed_refill", { + ranked: ranked.length, + rescued: kept.length, + informative: informative.length, + chatter: chatter.length, + filteredAll: true, + }); + return { + kept, + dropped, + outcome: "llm_filtered_refilled", + sufficient: sufficient ?? false, + }; +} + +/** + * Returns true when a ranked candidate carries content the agent can + * actually use. Skills, episodes, experiences, and world-models always + * count. Traces count when their `summary` or `reflection` is non-empty + * or their `agentText` is longer than a short acknowledgement. + * + * Used by the rescue path (and intentionally only there) to bias the + * rescued set toward traces with informative assistant text. False + * negatives (an informative trace mistakenly labelled chatter) still + * get rescued because they sit in the second half of the ordered list. + */ +function isInformativeCandidate(r: RankedCandidate): boolean { + const c = r.candidate; + if (c.refKind !== "trace") return true; + const t = c as TraceCandidate; + if ((t.summary?.trim().length ?? 0) > 0) return true; + if ((t.reflection?.trim().length ?? 0) > 0) return true; + const agent = t.agentText?.trim() ?? ""; + if (agent.length === 0) return false; + if (isShortAck(agent)) return false; + return agent.length >= INFORMATIVE_AGENT_TEXT_MIN_CHARS; +} + +function isShortAck(text: string): boolean { + return SHORT_ACK_PATTERNS.some((re) => re.test(text)); +} + /** * Mechanical fail-closed: when the LLM is unavailable / errored, * apply a relative-relevance cutoff so we don't dump the entire ranked diff --git a/apps/memos-local-plugin/core/retrieval/tier2-trace.ts b/apps/memos-local-plugin/core/retrieval/tier2-trace.ts index 1f902f54c..fad984821 100644 --- a/apps/memos-local-plugin/core/retrieval/tier2-trace.ts +++ b/apps/memos-local-plugin/core/retrieval/tier2-trace.ts @@ -83,6 +83,7 @@ export async function runTier2(deps: Tier2Deps, input: Tier2Input): Promise }, +): { where?: string; params?: Record } { + const maxAgeMs = deps.config.vectorScanMaxAgeMs; + if ( + typeof maxAgeMs !== "number" || + !Number.isFinite(maxAgeMs) || + maxAgeMs <= 0 + ) { + return base; + } + const minTs = deps.now() - maxAgeMs; + const params: Record = { + ...(base.params ?? {}), + vector_scan_min_ts: minTs, + }; + const where = base.where + ? `${base.where} AND ts >= @vector_scan_min_ts` + : "ts >= @vector_scan_min_ts"; + return { where, params }; +} + function resolveTagFilter( tags: readonly string[], config: RetrievalConfig, diff --git a/apps/memos-local-plugin/core/retrieval/types.ts b/apps/memos-local-plugin/core/retrieval/types.ts index ae1b6f944..f83f74620 100644 --- a/apps/memos-local-plugin/core/retrieval/types.ts +++ b/apps/memos-local-plugin/core/retrieval/types.ts @@ -316,6 +316,16 @@ export interface RetrievalConfig { llmFilterCandidateBodyChars?: number; /** Low-cost mode: retrieve raw trace memories only. */ lightweightMemory?: boolean; + /** + * Tier-2 vector scan time-window bound (ms). When > 0, the cosine + * scan path only considers `traces` rows whose `ts` is within the + * last `vectorScanMaxAgeMs` milliseconds. Set to `0` to disable + * (legacy full-table brute-force scan). See + * https://github.com/MemTensor/MemOS/issues/1929 for the original + * starvation report and `core/config/schema.ts` for the YAML + * binding + validation rules. + */ + vectorScanMaxAgeMs?: number; } /** @@ -737,6 +747,7 @@ export interface RetrievalStats { | "deferred_to_final" | "llm_kept_all" | "llm_filtered" + | "llm_filtered_refilled" | "llm_failed_safe_cutoff"; llmFilterSufficient?: boolean; llmFilterKept?: number; diff --git a/apps/memos-local-plugin/core/skill/index.ts b/apps/memos-local-plugin/core/skill/index.ts index 3d11349f2..1038ff8dd 100644 --- a/apps/memos-local-plugin/core/skill/index.ts +++ b/apps/memos-local-plugin/core/skill/index.ts @@ -27,6 +27,7 @@ export { applyFeedback, recomputeEta, shouldArchiveIdle, + shouldPromoteCandidate, type LifecycleUpdate, } from "./lifecycle.js"; export { diff --git a/apps/memos-local-plugin/core/skill/lifecycle.ts b/apps/memos-local-plugin/core/skill/lifecycle.ts index 49f43bb02..a4eaee04a 100644 --- a/apps/memos-local-plugin/core/skill/lifecycle.ts +++ b/apps/memos-local-plugin/core/skill/lifecycle.ts @@ -201,6 +201,37 @@ export function shouldArchiveIdle( return skill.eta < cfg.minEtaForRetrieval; } +/** + * Decide whether a non-repair candidate skill should skip trial + * and become active immediately. `eta` already encodes the + * gain / support of the source policies (initialised at + * crystallisation and kept current by reward drift), so we + * don't need a separate gain / support gate. + */ +export function shouldPromoteCandidate( + skill: SkillRow, + cfg: SkillConfig, +): boolean { + if (skill.status !== "candidate") return false; + if (hasDecisionGuidance(skill)) return false; + return skill.eta >= cfg.minEtaForRetrieval; +} + +function hasDecisionGuidance(skill: SkillRow): boolean { + const procedure = skill.procedureJson; + if (!procedure || typeof procedure !== "object") return false; + const guidance = (procedure as { + decisionGuidance?: { + preference?: unknown[]; + antiPattern?: unknown[]; + }; + }).decisionGuidance; + return Boolean( + (Array.isArray(guidance?.preference) && guidance.preference.length > 0) || + (Array.isArray(guidance?.antiPattern) && guidance.antiPattern.length > 0), + ); +} + function clamp01(n: number): number { if (!Number.isFinite(n)) return 0; if (n < 0) return 0; diff --git a/apps/memos-local-plugin/core/skill/subscriber.ts b/apps/memos-local-plugin/core/skill/subscriber.ts index 52ae6a4f6..67754c311 100644 --- a/apps/memos-local-plugin/core/skill/subscriber.ts +++ b/apps/memos-local-plugin/core/skill/subscriber.ts @@ -25,6 +25,7 @@ import { runSkill, type RunSkillDeps, } from "./skill.js"; +import { shouldPromoteCandidate } from "./lifecycle.js"; import type { RunSkillInput, RunSkillResult, @@ -33,6 +34,7 @@ import type { SkillTrigger, } from "./types.js"; import type { SkillId } from "../types.js"; +import { now as nowMs } from "../time.js"; export interface SkillSubscriberDeps extends Omit { @@ -46,6 +48,7 @@ export interface SkillSubscriberHandle { dispose(): void; runOnce(input: Omit & { trigger?: SkillTrigger }): Promise; applyFeedback(skillId: SkillId, kind: SkillFeedbackKind, magnitude?: number): void; + lifecycleTick(): Promise; /** * Await any in-flight scheduled run. Primarily useful in tests where we * want to assert on the effects of an event-driven run after the bus has @@ -207,5 +210,24 @@ export function attachSkillSubscriber( } } - return { dispose, runOnce, applyFeedback, flush }; + /** Periodic lifecycle pass: promote eligible candidate skills to active. */ + async function lifecycleTick(): Promise { + const candidates = deps.repos.skills.list({ status: "candidate", limit: 500 }); + for (const s of candidates) { + if (!shouldPromoteCandidate(s, deps.config)) continue; + const at = nowMs(); + deps.repos.skills.setStatus(s.id, "active", at); + log.info("skill.auto_promoted", { skillId: s.id, name: s.name, eta: s.eta }); + deps.bus.emit({ + kind: "skill.status.changed", + at, + skillId: s.id, + previous: "candidate", + next: "active", + transition: "promoted", + }); + } + } + + return { dispose, runOnce, applyFeedback, flush, lifecycleTick }; } diff --git a/apps/memos-local-plugin/core/storage/repos/_helpers.ts b/apps/memos-local-plugin/core/storage/repos/_helpers.ts index 645451f88..6286074cd 100644 --- a/apps/memos-local-plugin/core/storage/repos/_helpers.ts +++ b/apps/memos-local-plugin/core/storage/repos/_helpers.ts @@ -48,14 +48,14 @@ export function nullable(v: T | undefined): T | null { export function buildPageClauses(opts: PageOptions | undefined, tsColumn: string): string { const newestFirst = opts?.newestFirst !== false; - const limit = clampLimit(opts?.limit ?? 50); + const limit = clampLimit(opts?.limit ?? 500); const offset = Math.max(opts?.offset ?? 0, 0); return `ORDER BY ${tsColumn} ${newestFirst ? "DESC" : "ASC"} LIMIT ${limit} OFFSET ${offset}`; } export function clampLimit(n: number): number { - if (!Number.isFinite(n) || n <= 0) return 50; - return Math.min(Math.trunc(n), 10_000); + if (!Number.isFinite(n) || n <= 0) return 500; + return Math.min(Math.trunc(n), 500); } export function timeRangeWhere( diff --git a/apps/memos-local-plugin/core/storage/repos/embedding_maintenance.ts b/apps/memos-local-plugin/core/storage/repos/embedding_maintenance.ts new file mode 100644 index 000000000..65632ff59 --- /dev/null +++ b/apps/memos-local-plugin/core/storage/repos/embedding_maintenance.ts @@ -0,0 +1,204 @@ +/** + * SQL-only embedding maintenance stats. + * + * Regression fix for issue #1929: `/api/v1/embeddings/maintenance` used to + * paginate every trace/policy/world_model/skill row through JS just to + * inspect vector byte lengths, hydrating hundreds of MB of BLOBs into the + * Node heap and blocking the event loop for minutes on production DBs + * (93K traces × 2 vectors × 1536 dims × 4 bytes ≈ 1.1 GB pread64 traffic). + * + * The strategy is a single `SELECT COUNT(*) + SUM(CASE WHEN ...)` per + * `(table, vec column)` pair, using `LENGTH(vec)` for the dimension + * comparison. SQLite's `LENGTH()` on a BLOB column returns the byte length + * from the row header and does not deserialise the buffer, so the maintenance + * call now stays in the same asymptotic ballpark as `SELECT COUNT(*) FROM t`. + * + * The two pre-fix semantic filters are preserved verbatim inside the WHERE + * clauses so per-bucket counts do not shift for already-installed users: + * + * - `shouldTraceHaveEmbeddings` (short-text traces skipped) → SQL + * `LENGTH(TRIM(user_text)) / LENGTH(TRIM(agent_text))` predicates. + * - `isLightweightMemoryTrace` (lightweight traces skip vec_action) → + * `instr(COALESCE(tags_json, ''), '"lightweight_memory"') = 0` + * predicate for the vec_action count only. + */ + +import type { StorageDb } from "../types.js"; + +/** Little-endian Float32 element size. Matches `core/storage/vector.ts`. */ +export const FLOAT32_BYTES = 4; + +export interface EmbeddingCountsBucket { + /** Number of `(row, vec column)` slots included in the bucket. */ + totalSlots: number; + /** + * Vec is non-NULL and either `expectedByteLen === 0` (dimension not + * probed yet) or `LENGTH(vec) === expectedByteLen`. + */ + ready: number; + /** Vec is SQL NULL. */ + missing: number; + /** + * Vec is non-NULL and its byte length ≠ `expectedByteLen` + * (only meaningful when `expectedByteLen > 0`). + */ + dimMismatch: number; +} + +export interface EmbeddingCounts { + trace: EmbeddingCountsBucket; + policy: EmbeddingCountsBucket; + world_model: EmbeddingCountsBucket; + skill: EmbeddingCountsBucket; +} + +interface CountRow { + total: number | null; + missing: number | null; + dim_mismatch: number | null; + ready: number | null; +} + +interface CountArgs { + expected_byte_len: number; +} + +/** + * SQL for the four `(table, vec column)` slots. Each query returns + * `(total, missing, dim_mismatch, ready)` for a single slot in one round-trip. + * + * `LENGTH(vec)` on a BLOB column returns the stored byte count without + * deserialising the BLOB into JS memory — this is the whole point of the + * fix (see #1929 evidence: 99.96 % of the pre-fix time was in `pread64`). + */ +const TRACE_QUALIFICATION = + "(LENGTH(TRIM(user_text)) >= 10 OR LENGTH(TRIM(agent_text)) >= 10) " + + "AND (LENGTH(TRIM(user_text)) + LENGTH(TRIM(agent_text)) >= 20)"; + +/** Build the count SQL for a single (table, column) slot. */ +function slotSql( + table: string, + column: string, + extraWhere: string = "", +): string { + const clauses: string[] = []; + if (table === "traces") clauses.push(TRACE_QUALIFICATION); + if (extraWhere) clauses.push(extraWhere); + const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""; + return ` + SELECT + COUNT(*) AS total, + SUM(CASE WHEN ${column} IS NULL THEN 1 ELSE 0 END) AS missing, + SUM(CASE WHEN ${column} IS NOT NULL + AND @expected_byte_len > 0 + AND LENGTH(${column}) <> @expected_byte_len + THEN 1 ELSE 0 END) AS dim_mismatch, + SUM(CASE WHEN ${column} IS NOT NULL + AND (@expected_byte_len = 0 OR LENGTH(${column}) = @expected_byte_len) + THEN 1 ELSE 0 END) AS ready + FROM ${table} + ${where} + `; +} + +function normalizeRow(row: CountRow | undefined): EmbeddingCountsBucket { + if (!row) { + return { totalSlots: 0, ready: 0, missing: 0, dimMismatch: 0 }; + } + return { + totalSlots: row.total ?? 0, + ready: row.ready ?? 0, + missing: row.missing ?? 0, + dimMismatch: row.dim_mismatch ?? 0, + }; +} + +/** + * Count embedding slots per (table, vec column) purely with SQL. + * + * MUST NOT read or decode any BLOB into JS. Total wall-clock work is + * `O(rows)` SQL scan touching only BLOB header bytes. + * + * @param db - open storage handle (better-sqlite3 wrapper). + * @param opts.expectedByteLen - `dimensions * 4` for a known Float32 + * dimension, or `0` when the dimension has not been probed yet. In the + * `0` fallback every non-NULL vector counts as ready and dimMismatch + * is always 0 — matches the pre-fix "any non-null = ready" behaviour + * that `inferStoredEmbeddingDimension(slots)` used to fall back to. + */ +export function embeddingMaintenanceCounts( + db: StorageDb, + opts: { expectedByteLen: number }, +): EmbeddingCounts { + const args: CountArgs = { + expected_byte_len: Math.max(0, Math.floor(opts.expectedByteLen) || 0), + }; + + const traceSummary = db + .prepare(slotSql("traces", "vec_summary")) + .get(args); + const traceAction = db + .prepare( + slotSql( + "traces", + "vec_action", + "instr(COALESCE(tags_json, ''), '\"lightweight_memory\"') = 0", + ), + ) + .get(args); + const policy = db + .prepare(slotSql("policies", "vec")) + .get(args); + const worldModel = db + .prepare(slotSql("world_model", "vec")) + .get(args); + const skill = db + .prepare(slotSql("skills", "vec")) + .get(args); + + const summary = normalizeRow(traceSummary); + const action = normalizeRow(traceAction); + + return { + trace: { + totalSlots: summary.totalSlots + action.totalSlots, + ready: summary.ready + action.ready, + missing: summary.missing + action.missing, + dimMismatch: summary.dimMismatch + action.dimMismatch, + }, + policy: normalizeRow(policy), + world_model: normalizeRow(worldModel), + skill: normalizeRow(skill), + }; +} + +interface ModeRow { + byte_len: number; + n: number; +} + +/** + * Infer the dominant stored embedding byte length by GROUP BY the byte + * length of every non-NULL `traces.vec_summary` BLOB. Returns the byte + * length with the highest row count, or 0 when the DB has no vectors + * (brand-new install). + * + * Cheap replacement for the pre-fix + * `inferStoredEmbeddingDimension(collectEmbeddingSlots())` path — which had + * to hydrate every BLOB in memory before it could measure any single one. + * We now let SQLite do the length arithmetic and just pick the mode. + */ +export function inferStoredEmbeddingByteLen(db: StorageDb): number { + const rows = db + .prepare( + `SELECT LENGTH(vec_summary) AS byte_len, COUNT(*) AS n + FROM traces + WHERE vec_summary IS NOT NULL AND LENGTH(vec_summary) > 0 + GROUP BY LENGTH(vec_summary) + ORDER BY n DESC + LIMIT 1`, + ) + .all(); + if (rows.length === 0) return 0; + return rows[0]!.byte_len; +} diff --git a/apps/memos-local-plugin/core/storage/repos/episodes.ts b/apps/memos-local-plugin/core/storage/repos/episodes.ts index 3a3f4822b..1322b2afb 100644 --- a/apps/memos-local-plugin/core/storage/repos/episodes.ts +++ b/apps/memos-local-plugin/core/storage/repos/episodes.ts @@ -121,7 +121,13 @@ export function makeEpisodesRepo(db: StorageDb) { }, appendTrace(id: EpisodeId, traceIds: string[]): void { - appendTrace.run({ id, trace_ids_json: toJsonText(traceIds) }); + // Strip ghost trace IDs (entries that do not exist in the `traces` + // table) before persisting. Otherwise a single orphan ID lingering in + // `trace_ids_json` keeps the reward dirty-check tripping in a loop + // (#1966 — 590 wasted rescore calls in 5 days). The filter preserves + // input order and de-duplicates. + const validated = filterTraceIdsToExisting(db, traceIds); + appendTrace.run({ id, trace_ids_json: toJsonText(validated) }); }, removeTraceIds(id: EpisodeId, traceIds: readonly string[]): void { @@ -219,3 +225,26 @@ function mapRow(r: RawEpisodeRow): EpisodeRow & EpisodeMetaRow { meta: fromJsonText>(r.meta_json, {}), }; } + +/** + * Return the subset of `traceIds` that have a backing row in the `traces` + * table, preserving input order and de-duplicating. Empty array short-circuits. + * + * Lives in `episodes.ts` so the repo is self-contained — it does not import + * `traces.ts` to avoid a circular module. Uses the same chunking strategy as + * `traces.filterExistingIds` to stay under SQLite's bound-parameter limit. + */ +function filterTraceIdsToExisting(db: StorageDb, traceIds: string[]): string[] { + if (!traceIds || traceIds.length === 0) return []; + const dedup = Array.from(new Set(traceIds)); + const existing = new Set(); + const CHUNK_SIZE = 900; + for (let i = 0; i < dedup.length; i += CHUNK_SIZE) { + const chunk = dedup.slice(i, i + CHUNK_SIZE); + const placeholders = chunk.map(() => "?").join(","); + const sql = `SELECT id FROM traces WHERE id IN (${placeholders})`; + const rows = db.prepare(sql).all(chunk); + for (const r of rows) existing.add(r.id); + } + return dedup.filter((id) => existing.has(id)); +} diff --git a/apps/memos-local-plugin/core/storage/repos/index.ts b/apps/memos-local-plugin/core/storage/repos/index.ts index 12cca9f33..5f3038dab 100644 --- a/apps/memos-local-plugin/core/storage/repos/index.ts +++ b/apps/memos-local-plugin/core/storage/repos/index.ts @@ -84,3 +84,16 @@ export { makeSkillsRepo } from "./skills.js"; export { makeTracePolicyLinksRepo } from "./trace-policy-links.js"; export { makeTracesRepo } from "./traces.js"; export { makeWorldModelRepo } from "./world_model.js"; + +/** + * SQL-only embedding maintenance stats — see `./embedding_maintenance.ts` + * for the design rationale. Regression pin for issue #1929: the old JS + * path hydrated 270 MB of BLOBs per request; this SQL fast path uses + * `LENGTH(vec)` alone and never touches the buffers. + */ +export { + embeddingMaintenanceCounts, + inferStoredEmbeddingByteLen, + FLOAT32_BYTES, +} from "./embedding_maintenance.js"; +export type { EmbeddingCounts, EmbeddingCountsBucket } from "./embedding_maintenance.js"; diff --git a/apps/memos-local-plugin/core/storage/repos/traces.ts b/apps/memos-local-plugin/core/storage/repos/traces.ts index fe9e5b623..c00f617d9 100644 --- a/apps/memos-local-plugin/core/storage/repos/traces.ts +++ b/apps/memos-local-plugin/core/storage/repos/traces.ts @@ -139,6 +139,60 @@ export function makeTracesRepo(db: StorageDb) { return false; }, + /** + * Count how many of the given IDs actually exist in the `traces` table. + * + * Used by the reward-dirty check + * (https://github.com/MemTensor/MemOS/issues/1966) to tolerate "ghost" + * trace IDs — entries that linger in `episodes.trace_ids_json` but whose + * backing trace row was deleted (manual cleanup, schema migration, etc.). + * Without this, comparing `reward.traceCount` against + * `episode.traceIds.length` triggers an infinite rescore loop whenever + * `length` includes ghosts that the reward pipeline already filtered out. + * + * Uses a single `SELECT COUNT(*)` per chunk so the cost is independent of + * row size — embedding BLOBs and `tool_calls_json` are never read. + */ + countExisting(ids: readonly TraceId[]): number { + if (ids.length === 0) return 0; + // De-duplicate so the count reflects distinct IDs, matching the + // semantics of `getManyByIds(ids).length` which also dedupes. + const dedup = Array.from(new Set(ids)); + const CHUNK_SIZE = 900; + let total = 0; + for (let i = 0; i < dedup.length; i += CHUNK_SIZE) { + const chunk = dedup.slice(i, i + CHUNK_SIZE); + const placeholders = buildInClause(chunk.length); + const sql = `SELECT COUNT(*) AS n FROM traces WHERE id ${placeholders}`; + const row = db + .prepare(sql) + .get(chunk); + total += row?.n ?? 0; + } + return total; + }, + + /** + * Return the subset of `ids` that actually exist in the `traces` table, + * preserving the input order and de-duplicating. Companion to + * `countExisting`; used by `episodes.appendTrace` to strip ghost IDs at + * write time (#1966). + */ + filterExistingIds(ids: readonly TraceId[]): TraceId[] { + if (ids.length === 0) return []; + const dedup = Array.from(new Set(ids)); + const existing = new Set(); + const CHUNK_SIZE = 900; + for (let i = 0; i < dedup.length; i += CHUNK_SIZE) { + const chunk = dedup.slice(i, i + CHUNK_SIZE); + const placeholders = buildInClause(chunk.length); + const sql = `SELECT id FROM traces WHERE id ${placeholders}`; + const rows = db.prepare(sql).all(chunk); + for (const r of rows) existing.add(r.id); + } + return dedup.filter((id) => existing.has(id)); + }, + list(filter: TraceListFilter = {}): TraceRow[] { const tr = timeRangeWhere(filter, "ts"); const fragments: string[] = []; diff --git a/apps/memos-local-plugin/package.json b/apps/memos-local-plugin/package.json index c4c495add..16eef383e 100644 --- a/apps/memos-local-plugin/package.json +++ b/apps/memos-local-plugin/package.json @@ -18,6 +18,7 @@ "telemetry.credentials.json", "openclaw.plugin.json", "bridge.cts", + "bridge.mts", "bridge/**/*.ts", "core/**/*.ts", "core/storage/migrations/*.sql", diff --git a/apps/memos-local-plugin/server/http.ts b/apps/memos-local-plugin/server/http.ts index df047f4d1..82a2b201d 100644 --- a/apps/memos-local-plugin/server/http.ts +++ b/apps/memos-local-plugin/server/http.ts @@ -34,7 +34,8 @@ import { serveStatic } from "./middleware/static.js"; import type { ServerDeps, ServerHandle, ServerOptions } from "./types.js"; type AgentName = "openclaw" | "hermes"; -const AGENT_NAMES: readonly AgentName[] = ["openclaw", "hermes"]; +type AgentPrefix = AgentName | "memos"; +const AGENT_PREFIXES: readonly AgentPrefix[] = ["openclaw", "hermes", "memos"]; /** * Well-known per-agent viewer port. The picker page links to the @@ -146,11 +147,11 @@ async function dispatch( // downgrade because cross-port redirects are inherently a "follow // this link" gesture; the SPA bundle on the other port re-issues // mutations from form state, not from the original POST body.) - for (const name of AGENT_NAMES) { + for (const name of AGENT_PREFIXES) { const prefix = `/${name}`; if (pathname === prefix || pathname.startsWith(`${prefix}/`)) { const tail = pathname.slice(prefix.length) || "/"; - if (name === selfAgent) { + if (name === "memos" || name === selfAgent) { pathname = tail; break; } diff --git a/apps/memos-local-plugin/server/routes/config.ts b/apps/memos-local-plugin/server/routes/config.ts index 102a1bce8..b86966e05 100644 --- a/apps/memos-local-plugin/server/routes/config.ts +++ b/apps/memos-local-plugin/server/routes/config.ts @@ -10,10 +10,40 @@ * * Writes go through `core/config/writer.ts::patchConfig`, which * preserves comments + field order and re-applies `chmod 600`. + * + * Client-supplied PATCH bodies that fail schema validation (Typebox + * `NumberInRange`, type mismatch, etc.) must surface as HTTP 4xx — not + * 500 — so concurrent search/onTurnStart calls are not poisoned by a + * misbehaving viewer or admin script. We catch the `config_invalid` + * `MemosError` raised by `resolveConfig` here and translate it to 400 + * `invalid_argument`. Server-side failures (e.g. `config_write_failed` + * from a non-writable disk) and any other error keep propagating to the + * global handler so operators still get paged on real bugs. + * + * Issue #1929 — the rerun harness contract tests + * (`test_invalid_type_does_not_crash_or_corrupt`, + * `test_concurrent_patch_and_search_no_5xx`, + * `test_extreme_max_age_at_int_max_no_crash`) explicitly assert + * `status_code < 500` on every malformed PATCH; without this guard the + * server would 500 on each one and trip the harness. */ +import { MemosError } from "../../agent-contract/errors.js"; import type { ServerDeps } from "../types.js"; import { parseJson, writeError, type Routes } from "./registry.js"; +/** + * Error codes raised by `core/config/{index,writer}.ts` that originate + * from client input (a bad PATCH body) rather than from a server bug. + * We map these to HTTP 400. Everything else — including server-side + * failures like `config_write_failed` (atomic rename failed: disk full + * / permission denied) — bubbles up to the global 500 handler so + * operators get paged on real bugs and clients are not misled into + * thinking their (valid) input was rejected. + */ +const CLIENT_INPUT_CONFIG_ERRORS: ReadonlySet = new Set([ + "config_invalid", +]); + export function registerConfigRoutes(routes: Routes, deps: ServerDeps): void { routes.set("GET /api/v1/config", async () => { return await deps.core.getConfig(); @@ -25,6 +55,14 @@ export function registerConfigRoutes(routes: Routes, deps: ServerDeps): void { writeError(ctx, 400, "invalid_argument", "body must be a JSON object"); return; } - return await deps.core.patchConfig(patch); + try { + return await deps.core.patchConfig(patch); + } catch (err) { + if (MemosError.is(err) && CLIENT_INPUT_CONFIG_ERRORS.has(err.code)) { + writeError(ctx, 400, "invalid_argument", err.message); + return; + } + throw err; + } }); } diff --git a/apps/memos-local-plugin/tests/integration/bridge-esm-entry.test.ts b/apps/memos-local-plugin/tests/integration/bridge-esm-entry.test.ts new file mode 100644 index 000000000..6ca2a66c2 --- /dev/null +++ b/apps/memos-local-plugin/tests/integration/bridge-esm-entry.test.ts @@ -0,0 +1,130 @@ +/** + * Regression test for issue #1736. + * + * The historical failure mode on Node ≥ 22 was: + * + * bridge: fatal: Cannot read properties of undefined (reading 'exports') + * at load (node:internal/modules/cjs/loader:...) + * + * Root cause was the CommonJS `dist/bridge.cjs` trampoline trying to + * load ESM peers via either `__dirname`-rooted `.ts` paths or + * `require(file://...)`. The fix introduces a pure-ESM + * `dist/bridge.mjs` entry that imports the same peers via plain + * relative specifiers. This test spawns the new entry and asserts that + * the historical symptom string never appears on stderr within a short + * grace period — failures further downstream (e.g. a missing + * better-sqlite3 native binding when running in a clean CI sandbox) + * are tolerated because they are unrelated to issue #1736. + */ +import { describe, it, expect } from "vitest"; +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const FORBIDDEN_SYMPTOM = + "Cannot read properties of undefined (reading 'exports')"; + +const PLUGIN_ROOT = path.resolve(__dirname, "..", ".."); +const BRIDGE_MJS = path.join(PLUGIN_ROOT, "dist", "bridge.mjs"); + +interface SpawnResult { + stderr: string; + stdout: string; + exitCode: number | null; + signal: NodeJS.Signals | null; +} + +function spawnBridge( + args: readonly string[], + options: { home: string; timeoutMs: number }, +): Promise { + return new Promise((resolve, reject) => { + const proc = spawn(process.execPath, [BRIDGE_MJS, ...args], { + env: { + ...process.env, + HOME: options.home, + }, + stdio: ["pipe", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + proc.stdout?.on("data", (b: Buffer) => { + stdout += b.toString("utf8"); + }); + proc.stderr?.on("data", (b: Buffer) => { + stderr += b.toString("utf8"); + }); + + const killer = setTimeout(() => { + // Bridge survived the grace window without emitting the + // forbidden symptom — kill it so the test does not hang. + try { + proc.kill("SIGTERM"); + } catch { + /* best-effort */ + } + }, options.timeoutMs); + + proc.once("error", (err) => { + clearTimeout(killer); + reject(err); + }); + proc.once("exit", (code, signal) => { + clearTimeout(killer); + resolve({ stdout, stderr, exitCode: code, signal }); + }); + + // Close stdin so a clean stdio bridge can finish startup without + // blocking on a non-existent JSON-RPC stream. + proc.stdin?.end(); + }); +} + +describe("Issue #1736 — bridge ESM entry boots without CJS 'exports' error", () => { + it("dist/bridge.mjs starts up without the historical CJS/ESM symptom", async () => { + // The build step is a precondition for this test. The repository + // CI runs `pnpm build` before vitest; locally the user should + // either run `pnpm build` once, or accept that this test will + // skip when the artifact is missing. + if (!fs.existsSync(BRIDGE_MJS)) { + return; + } + + const home = fs.mkdtempSync(path.join(os.tmpdir(), "memos-1736-")); + try { + const result = await spawnBridge(["--agent=hermes", "--no-viewer"], { + home, + timeoutMs: 8_000, + }); + + // Combined output check — the symptom is unique enough that it + // never appears in healthy startup logs. + const combined = `${result.stdout}\n${result.stderr}`; + expect(combined).not.toContain(FORBIDDEN_SYMPTOM); + + // Sanity: the bridge must at least have started the ESM import + // chain. Either the headless-mode notice fires on success, or + // the unrelated better-sqlite3 native-binding error fires when + // the native module is absent (CI sandbox without postinstall). + // The historic CJS trampoline crash would surface BEFORE either + // of those. + const reachedEsmPath = + combined.includes("stdio mode running without viewer") || + combined.includes("Could not locate the bindings file") || + combined.includes("bootstrapMemoryCoreFull") || + combined.includes("[core.pipeline.bootstrap]"); + expect( + reachedEsmPath, + `bridge did not reach the post-trampoline path; stderr was:\n${result.stderr}`, + ).toBe(true); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }, 20_000); +}); diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index b47d5660e..3322de7ed 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -61,13 +61,23 @@ def _write(s: str) -> int: self.stdin.write = _write # type: ignore[assignment] - # The client just needs wait/kill to exist; they are no-ops here. + # The client just needs wait/kill/poll to exist; they are no-ops + # here. `poll_return` lets a test simulate an already-exited + # subprocess for the fast-fail path. + poll_return: int | None = None + def wait(self, timeout: float | None = None) -> int: return 0 def kill(self) -> None: pass + def terminate(self) -> None: + pass + + def poll(self) -> int | None: + return self.poll_return + class _ServerStream(io.StringIO): """Script bridge responses as if coming from the Node subprocess.""" @@ -164,10 +174,20 @@ class RecordingBridge: def __init__(self) -> None: self.calls: list[tuple[str, dict]] = [] - - def request(self, method: str, params: dict | None = None) -> dict | None: + # Kwargs captured per call — used by tests that assert on the + # per-request `timeout` kwarg the provider now passes for + # long-running operations (memory.search / turn.start / turn.end). + self.call_kwargs: list[dict] = [] + + def request( + self, + method: str, + params: dict | None = None, + **kwargs, + ) -> dict | None: payload = params or {} self.calls.append((method, payload)) + self.call_kwargs.append(dict(kwargs)) if method == "memory.search": return { "hits": [ @@ -256,12 +276,16 @@ def _factory(*args, **kwargs): ) self._popen_patch.start() self._which_patch.start() + # Hermetic singleton: clear any tracking left over from prior tests + # so each test starts with a fresh module-level registry. + bridge_client_mod._ACTIVE_CLIENTS.clear() def tearDown(self) -> None: if self._fake is not None: self._fake.stdout._done = True self._popen_patch.stop() self._which_patch.stop() + bridge_client_mod._ACTIVE_CLIENTS.clear() def test_request_returns_result_on_success(self) -> None: client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") @@ -295,6 +319,45 @@ def test_close_is_idempotent(self) -> None: client.close() client.close() # second call must not raise + def test_module_singleton_closes_previous_client_same_agent(self) -> None: + """Constructing a second client with the same agent must reap the first. + + Regression for issue #1910: each turn the Hermes adapter could + spawn a fresh bridge subprocess without closing its predecessor, + accumulating 4+ processes per session. The singleton tracker in + ``MemosBridgeClient`` prevents that by closing any active client + for the same ``(agent, no_viewer)`` slot at construction time. + """ + first = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + self.assertFalse(first._closed) + second = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + # The new constructor must have reaped the previous one. + self.assertTrue(first._closed) + self.assertFalse(second._closed) + second.close() + self.assertTrue(second._closed) + + def test_module_singleton_independent_for_distinct_agents(self) -> None: + """A bridge for a different agent must not reap an unrelated bridge.""" + hermes = MemosBridgeClient(bridge_path="/tmp/bridge.cts", agent="hermes") + openclaw = MemosBridgeClient(bridge_path="/tmp/bridge.cts", agent="openclaw") + self.assertFalse(hermes._closed) + self.assertFalse(openclaw._closed) + hermes.close() + openclaw.close() + + def test_close_unregisters_active_client_only_when_still_current(self) -> None: + """A stale close() must not evict the newer registered client.""" + first = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + second = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + # First was already closed by second's __init__. Closing it again is + # a no-op and must not touch the registry's current entry (second). + first.close() + key = (second._singleton_agent, second._singleton_no_viewer) + self.assertIs(bridge_client_mod._ACTIVE_CLIENTS.get(key), second) + second.close() + self.assertIsNone(bridge_client_mod._ACTIVE_CLIENTS.get(key)) + def test_stdio_bridge_starts_without_viewer_by_default(self) -> None: client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") assert self._fake is not None @@ -329,6 +392,63 @@ def test_reverse_request_waits_for_late_host_handler_registration(self) -> None: self.assertNotIn("error", response) client.close() + def test_reader_exit_marks_pending_as_transport_closed(self) -> None: + """R1 (#2028): reader thread EOF must wake pending waiters + with transport_closed instead of leaving them parked on their + per-request timeout. + """ + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + + results: dict[str, Exception | dict] = {} + + def _issue() -> None: + try: + # Method name intentionally not scripted by _ServerStream, + # so the request stays pending until the reader thread + # signals transport_closed. + results["ok"] = client.request("unscripted.method", {}, timeout=10.0) + except Exception as err: + results["err"] = err + + worker = threading.Thread(target=_issue, daemon=True) + worker.start() + # Let the request register in _pending before we kill stdout. + time.sleep(0.05) + # Simulate the Node bridge subprocess dying — the ServerStream + # iterator returns, so the reader thread exits its for-loop. + self._fake.stdout._done = True + # The reader thread's finally block should wake our waiter well + # inside 2 s (well below the 10 s per-request timeout that would + # otherwise fire). + worker.join(timeout=2.0) + self.assertFalse(worker.is_alive(), "waiter was not woken in <2s") + err = results.get("err") + self.assertIsInstance(err, BridgeError) + assert isinstance(err, BridgeError) + self.assertEqual(err.code, "transport_closed") + client.close() + + def test_request_fast_fails_when_subprocess_already_dead(self) -> None: + """R2 (#2028): request() must short-circuit with transport_closed + without writing to stdin when the subprocess has already exited. + """ + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + + # Simulate an already-exited subprocess (e.g. OOM kill, exit 137). + self._fake.poll_return = 137 + # Snapshot writes so we can assert none are added by the failed call. + writes_before = list(self._fake._stdin_lines) + + with self.assertRaises(BridgeError) as ctx: + client.request("core.health", {}, timeout=1.0) + self.assertEqual(ctx.exception.code, "transport_closed") + + # Nothing new should have been written to the dead pipe. + self.assertEqual(self._fake._stdin_lines, writes_before) + client.close() + def _wait_for_client_write(self, predicate, timeout: float = 2.0) -> dict: assert self._fake is not None deadline = time.monotonic() + timeout @@ -396,6 +516,49 @@ def test_handle_tool_call_fails_gracefully_without_bridge(self) -> None: parsed = json.loads(res) self.assertIn("error", parsed) + def test_initialize_closes_pre_existing_bridge(self) -> None: + """Calling initialize twice must reap the previous bridge. + + Regression for #1910: every Hermes turn could call `initialize()` + a second time (re-entry from the host plugin loader), overwriting + `self._bridge` and leaking the previous Node subprocess. + """ + + class TrackedBridge: + def __init__(self) -> None: + self.closed = False + self.pid = 4242 + + def register_host_handler(self, *_a, **_kw) -> None: # pragma: no cover + pass + + def request(self, method, params=None, **_kwargs): + if method == "session.open": + return {"sessionId": (params or {}).get("sessionId", "sess")} + return {} + + def close(self) -> None: + self.closed = True + + p = self._provider_mod.MemTensorProvider() + + first = TrackedBridge() + second = TrackedBridge() + constructed: list[TrackedBridge] = [first, second] + + def _factory(*_a, **_kw) -> TrackedBridge: + return constructed.pop(0) + + with patch("memos_provider.MemosBridgeClient", side_effect=_factory): + p.initialize("sess-A", hermes_home="/tmp/h", platform="cli") + self.assertIs(p._bridge, first) + p.initialize("sess-A", hermes_home="/tmp/h", platform="cli") + # The second initialize must close the first bridge before + # adopting the new one; otherwise the previous subprocess + # leaks (issue #1910). + self.assertTrue(first.closed) + self.assertIs(p._bridge, second) + def test_handle_tool_call_routes_all_exposed_tools(self) -> None: p = self._provider_mod.MemTensorProvider() bridge = RecordingBridge() @@ -534,7 +697,7 @@ def request(self, method, params=None, **_kwargs): return {} class RetryFailBridge: - def request(self, method, params=None): + def request(self, method, params=None, **_kwargs): if method == "session.open": return {"sessionId": (params or {}).get("sessionId", "sess")} if method == "turn.start": @@ -571,7 +734,7 @@ def __init__(self): def close(self): self.closed = True - def request(self, method, params=None): + def request(self, method, params=None, **_kwargs): if method == "turn.end": raise BridgeError("transport_closed", "[Errno 32] Broken pipe") return {} @@ -627,6 +790,171 @@ def request(self, method, params=None, **_kwargs): self.assertIn("局势", retry_payload["agentText"]) self.assertEqual(retry_payload["toolCalls"][0]["name"], "search_files") + def test_keepalive_reconnects_on_health_timeout(self) -> None: + """R3 (#2028): a hung bridge surfaces as BridgeError('timeout', …) + — the helper predicate must treat it as a reconnect trigger so + the stale bridge does not keep timing out every user call. + """ + + class LiveBridge: + def __init__(self) -> None: + # Live subprocess — `poll()` returns None. + self._proc = type("_P", (), {"poll": lambda self: None})() + + p = self._provider_mod.MemTensorProvider() + p._bridge = LiveBridge() # type: ignore[assignment] + + self.assertTrue( + p._should_reconnect_after_keepalive_failure( + BridgeError("timeout", "core.health did not respond within 10.0s") + ) + ) + + def test_keepalive_reconnects_when_subprocess_is_dead(self) -> None: + """R3 (#2028): even for a generic exception, if the bridge + subprocess has already exited the helper must trigger a + reconnect (belt-and-braces for hangs that don't raise transport + errors). + """ + + class DeadBridge: + def __init__(self) -> None: + # Simulate exit(1). + self._proc = type("_P", (), {"poll": lambda self: 1})() + + p = self._provider_mod.MemTensorProvider() + p._bridge = DeadBridge() # type: ignore[assignment] + + self.assertTrue(p._should_reconnect_after_keepalive_failure(RuntimeError("boom"))) + + def test_keepalive_does_not_reconnect_on_transient_generic_error(self) -> None: + """R3 (#2028): a live subprocess raising a non-transport generic + error must NOT reconnect — otherwise transient parse noise would + create a reconnect storm. + """ + + class LiveBridge: + def __init__(self) -> None: + self._proc = type("_P", (), {"poll": lambda self: None})() + + p = self._provider_mod.MemTensorProvider() + p._bridge = LiveBridge() # type: ignore[assignment] + + self.assertFalse(p._should_reconnect_after_keepalive_failure(RuntimeError("parse hiccup"))) + + def test_handle_tool_call_retries_memos_search_after_transport_closed(self) -> None: + """R4 (#2028): the first stale-bridge call fails with + transport_closed; the read-path retry reconnects and the second + call succeeds — the tool must return the hits, not the error. + """ + + class StaleBridge: + def __init__(self) -> None: + self.closed = False + self.calls: list[tuple[str, dict]] = [] + + def close(self) -> None: + self.closed = True + + def request(self, method, params=None, **_kwargs): + self.calls.append((method, params or {})) + if method == "memory.search": + raise BridgeError("transport_closed", "[Errno 32] Broken pipe") + return {} + + class HealthyBridge: + def __init__(self) -> None: + self.calls: list[tuple[str, dict]] = [] + + def register_host_handler(self, _method, _handler) -> None: + return None + + def request(self, method, params=None, **_kwargs): + self.calls.append((method, params or {})) + if method == "session.open": + return {"sessionId": (params or {}).get("sessionId", "sess")} + if method == "turn.start": + return {"query": {"episodeId": "ep_after_reconnect"}} + if method == "memory.search": + return { + "hits": [ + { + "tier": 2, + "refKind": "trace", + "refId": "tr-post-reconnect", + "score": 0.9, + "snippet": "hit for HERMES_2028", + } + ] + } + return {} + + stale = StaleBridge() + healthy = HealthyBridge() + p = self._provider_mod.MemTensorProvider() + p._bridge = stale # type: ignore[assignment] + p._session_id = "sess_2028" + p._episode_id = "ep_2028" + p._hermes_home = "/tmp/hermes-home" + p._platform = "tui" + p._agent_identity = "hermes-test" + + with patch("memos_provider.MemosBridgeClient", return_value=healthy): + result = json.loads(p.handle_tool_call("memos_search", {"query": "HERMES_2028"})) + + self.assertIn("hits", result) + self.assertEqual(result["hits"][0]["refId"], "tr-post-reconnect") + # Retry must have gone through the replacement bridge. + methods = [m for m, _ in healthy.calls] + self.assertIn("memory.search", methods) + # Original stale bridge must have been closed by _reconnect_bridge. + self.assertTrue(stale.closed) + + def test_handle_tool_call_surfaces_second_transport_failure(self) -> None: + """R4 (#2028): if the retry also fails, the tool response must + contain the error text verbatim so the model sees the error + instead of an empty hits payload. + """ + + class StaleBridge: + def __init__(self) -> None: + self.closed = False + + def close(self) -> None: + self.closed = True + + def request(self, method, params=None, **_kwargs): + if method == "memory.search": + raise BridgeError("transport_closed", "[Errno 32] Broken pipe") + return {} + + class StillBrokenBridge: + def register_host_handler(self, _method, _handler) -> None: + return None + + def request(self, method, params=None, **_kwargs): + if method == "session.open": + return {"sessionId": (params or {}).get("sessionId", "sess")} + if method == "turn.start": + return {"query": {"episodeId": "ep_after_reconnect"}} + if method == "memory.search": + raise BridgeError("transport_closed", "[Errno 32] Broken pipe again") + return {} + + p = self._provider_mod.MemTensorProvider() + p._bridge = StaleBridge() # type: ignore[assignment] + p._session_id = "sess_2028" + p._episode_id = "ep_2028" + p._hermes_home = "/tmp/hermes-home" + p._platform = "tui" + p._agent_identity = "hermes-test" + + with patch("memos_provider.MemosBridgeClient", return_value=StillBrokenBridge()): + result = json.loads(p.handle_tool_call("memos_search", {"query": "HERMES_2028"})) + + self.assertIn("error", result) + self.assertIn("Broken pipe", result["error"]) + def test_get_config_schema_describes_known_fields(self) -> None: p = self._provider_mod.MemTensorProvider() schema = p.get_config_schema() @@ -657,6 +985,90 @@ def test_save_config_writes_yaml_with_correct_mode(self) -> None: self.assertEqual(loaded["viewer"]["port"], 18920) self.assertEqual(loaded["llm"]["provider"], "openai_compatible") + # ─── Long-operation RPC timeouts (issue #2028) ────────────────────── + # + # After 1-2 hours of Hermes use the memory / capture / reflection + # pipeline legitimately needs more than the 30s JSON-RPC default, + # which surfaced as: + # [timeout] memory.search did not respond within 30.0s + # [timeout] turn.end did not respond within 30.0s + # `feedback.submit` already opts into 75s, and `sync_turn` already + # opts into a 75s `_ensure_bridge`, but the actual `memory.search` + # and `turn.start` / `turn.end` requests still fell back to 30s. + # These tests pin the fix. + + _EXPECTED_LONG_TIMEOUT = 75.0 + + def test_memos_search_uses_long_rpc_timeout(self) -> None: + p = self._provider_mod.MemTensorProvider() + bridge = RecordingBridge() + p._bridge = bridge + p._session_id = "hermes:session:1" + + p.handle_tool_call("memos_search", {"query": "yesterday"}) + method, _params = bridge.calls[-1] + self.assertEqual(method, "memory.search") + kwargs = bridge.call_kwargs[-1] + self.assertIn("timeout", kwargs) + self.assertGreaterEqual( + kwargs["timeout"], + self._EXPECTED_LONG_TIMEOUT, + "memory.search must not fall back to the 30s JSON-RPC default; " + "large memories legitimately need more time (issue #2028).", + ) + + def test_memos_environment_search_uses_long_rpc_timeout(self) -> None: + p = self._provider_mod.MemTensorProvider() + bridge = RecordingBridge() + p._bridge = bridge + p._session_id = "hermes:session:1" + + # memos_environment routes to memory.search when a query is passed. + p.handle_tool_call("memos_environment", {"query": "install path"}) + method, _params = bridge.calls[-1] + self.assertEqual(method, "memory.search") + kwargs = bridge.call_kwargs[-1] + self.assertGreaterEqual(kwargs.get("timeout", 0.0), self._EXPECTED_LONG_TIMEOUT) + + def test_sync_turn_uses_long_rpc_timeout_for_turn_end(self) -> None: + p = self._provider_mod.MemTensorProvider() + bridge = RecordingBridge() + p._bridge = bridge + p._session_id = "hermes:session:1" + p._episode_id = "ep-1" # skip the turn.start prelude + + p.sync_turn("what did we do?", "we tested memory") + methods = [m for m, _ in bridge.calls] + self.assertIn("turn.end", methods) + end_index = methods.index("turn.end") + end_kwargs = bridge.call_kwargs[end_index] + self.assertIn("timeout", end_kwargs) + self.assertGreaterEqual( + end_kwargs["timeout"], + self._EXPECTED_LONG_TIMEOUT, + "turn.end must not fall back to the 30s JSON-RPC default; " + "the V7 capture/reflection pipeline grows past 30s in long " + "sessions (issue #2028).", + ) + + def test_prefetch_uses_long_rpc_timeout_for_turn_start(self) -> None: + p = self._provider_mod.MemTensorProvider() + bridge = RecordingBridge() + p._bridge = bridge + p._session_id = "hermes:session:1" + + p.prefetch("what did I do yesterday?", session_id="hermes:session:1") + methods = [m for m, _ in bridge.calls] + self.assertIn("turn.start", methods) + start_index = methods.index("turn.start") + start_kwargs = bridge.call_kwargs[start_index] + self.assertGreaterEqual( + start_kwargs.get("timeout", 0.0), + self._EXPECTED_LONG_TIMEOUT, + "turn.start suffers the same long-tail latency as turn.end and " + "must share the long RPC timeout (issue #2028).", + ) + class ViewerDaemonTests(unittest.TestCase): def tearDown(self) -> None: @@ -743,5 +1155,143 @@ def busy_lock(): popen.assert_not_called() +class BridgeOkCacheTests(unittest.TestCase): + """Regression tests for issue #1797. + + `ensure_bridge_running(probe_only=True)` used to cache a `False` + result permanently. These tests pin down the new contract: cache + entries have a TTL, and a running MemOS bridge on :18800 is a + fallback signal that Node works on this host even if a transient + `_node_available()` call failed. + """ + + def setUp(self) -> None: + # Make sure each test starts from a clean module-level cache. + daemon_manager_mod._bridge_ok = None + daemon_manager_mod._bridge_ok_at = 0.0 + # Pretend the compiled bridge script exists so the "script + # missing" early-return does not steal the test. + self._script_patch = patch.object( + daemon_manager_mod, + "_bridge_script", + return_value=Path("/fake/bridge.cjs"), + ) + self._script_patch.start() + self._exists_patch = patch.object(Path, "exists", return_value=True) + self._exists_patch.start() + + def tearDown(self) -> None: + self._exists_patch.stop() + self._script_patch.stop() + daemon_manager_mod._bridge_ok = None + daemon_manager_mod._bridge_ok_at = 0.0 + + def test_probe_only_when_cache_empty_revalidates(self) -> None: + with ( + patch.object(daemon_manager_mod, "_node_available", return_value=True) as node, + patch.object(daemon_manager_mod, "_probe_viewer") as probe, + ): + self.assertTrue(daemon_manager_mod.ensure_bridge_running(probe_only=True)) + node.assert_called_once() + probe.assert_not_called() + + def test_cached_false_returns_immediately_within_ttl(self) -> None: + # Seed the cache with a fresh False as if a transient failure occurred. + now = 1_000_000.0 + with ( + patch.object(daemon_manager_mod.time, "time", return_value=now), + patch.object(daemon_manager_mod, "_node_available", return_value=False), + patch.object(daemon_manager_mod, "_probe_viewer", return_value="free"), + ): + self.assertFalse(daemon_manager_mod.ensure_bridge_running(probe_only=True)) + + # Within TTL, _node_available must NOT be called again — the cached + # False is returned directly. + with ( + patch.object(daemon_manager_mod.time, "time", return_value=now + 1.0), + patch.object(daemon_manager_mod, "_node_available") as node, + patch.object(daemon_manager_mod, "_probe_viewer") as probe, + ): + self.assertFalse(daemon_manager_mod.ensure_bridge_running(probe_only=True)) + node.assert_not_called() + probe.assert_not_called() + + def test_cached_false_expires_after_ttl_and_recovers(self) -> None: + now = 2_000_000.0 + # Seed cache with a False at t=now. + with ( + patch.object(daemon_manager_mod.time, "time", return_value=now), + patch.object(daemon_manager_mod, "_node_available", return_value=False), + patch.object(daemon_manager_mod, "_probe_viewer", return_value="free"), + ): + self.assertFalse(daemon_manager_mod.ensure_bridge_running(probe_only=True)) + + # After TTL + 1s, Node is now reachable. probe_only=True must + # revalidate and switch the cache to True. + with ( + patch.object( + daemon_manager_mod.time, + "time", + return_value=now + daemon_manager_mod.BRIDGE_OK_TTL_SEC + 1.0, + ), + patch.object(daemon_manager_mod, "_node_available", return_value=True), + ): + self.assertTrue(daemon_manager_mod.ensure_bridge_running(probe_only=True)) + # And the value is cached. + self.assertTrue(daemon_manager_mod._bridge_ok) + + def test_running_bridge_overrides_failed_node_probe(self) -> None: + # A live MemOS bridge is definitive proof Node worked; trust it + # even if `_node_available` returns False (e.g. env-var race). + now = 3_000_000.0 + with ( + patch.object(daemon_manager_mod.time, "time", return_value=now), + patch.object(daemon_manager_mod, "_node_available", return_value=False), + patch.object(daemon_manager_mod, "_probe_viewer", return_value="running_memos"), + ): + self.assertTrue(daemon_manager_mod.ensure_bridge_running(probe_only=True)) + self.assertTrue(daemon_manager_mod._bridge_ok) + self.assertEqual(daemon_manager_mod._bridge_ok_at, now) + + def test_shutdown_bridge_resets_cache_and_timestamp(self) -> None: + now = 4_000_000.0 + with ( + patch.object(daemon_manager_mod.time, "time", return_value=now), + patch.object(daemon_manager_mod, "_node_available", return_value=True), + ): + self.assertTrue(daemon_manager_mod.ensure_bridge_running(probe_only=True)) + + self.assertIsNotNone(daemon_manager_mod._bridge_ok) + self.assertGreater(daemon_manager_mod._bridge_ok_at, 0.0) + + daemon_manager_mod.shutdown_bridge() + self.assertIsNone(daemon_manager_mod._bridge_ok) + self.assertEqual(daemon_manager_mod._bridge_ok_at, 0.0) + + # After reset, the next probe must call `_node_available` again. + with ( + patch.object(daemon_manager_mod.time, "time", return_value=now + 1.0), + patch.object(daemon_manager_mod, "_node_available", return_value=True) as node, + ): + self.assertTrue(daemon_manager_mod.ensure_bridge_running(probe_only=True)) + node.assert_called_once() + + def test_full_call_always_revalidates(self) -> None: + # `probe_only=False` (called from `MemTensorProvider.initialize`) + # must bypass the cache and refresh. + now = 5_000_000.0 + # Pre-seed a stale True. + daemon_manager_mod._bridge_ok = True + daemon_manager_mod._bridge_ok_at = now - 1.0 + with ( + patch.object(daemon_manager_mod.time, "time", return_value=now), + patch.object(daemon_manager_mod, "_node_available", return_value=False) as node, + patch.object(daemon_manager_mod, "_probe_viewer", return_value="free"), + ): + self.assertFalse(daemon_manager_mod.ensure_bridge_running()) + node.assert_called_once() + self.assertFalse(daemon_manager_mod._bridge_ok) + + if __name__ == "__main__": unittest.main() diff --git a/apps/memos-local-plugin/tests/python/test_bridge_script_resolution.py b/apps/memos-local-plugin/tests/python/test_bridge_script_resolution.py new file mode 100644 index 000000000..96b47eff9 --- /dev/null +++ b/apps/memos-local-plugin/tests/python/test_bridge_script_resolution.py @@ -0,0 +1,170 @@ +"""Unit tests for bridge script resolution (Issue #1736). + +The bridge launcher must prefer `dist/bridge.mjs` (pure ESM, fixes the +CJS↔ESM trampoline failure on Node 22) over `dist/bridge.cjs` (legacy +CommonJS) and over the development sources. + +These tests fake the plugin root via a temporary directory and inspect +which path `_bridge_script` and `_bridge_command` produce. +""" + +from __future__ import annotations + +import sys +import unittest + +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + + +_ADAPTER_ROOT = Path(__file__).resolve().parent.parent.parent / "adapters" / "hermes" +_PLUGIN_DIR = _ADAPTER_ROOT / "memos_provider" +for _p in (_ADAPTER_ROOT, _PLUGIN_DIR): + if str(_p) not in sys.path: + sys.path.insert(0, str(_p)) + +import bridge_client as bridge_client_mod # noqa: E402 +import daemon_manager as daemon_manager_mod # noqa: E402 + + +def _layout(root: Path, *, mjs: bool, cjs: bool, mts: bool, cts: bool) -> None: + """Create the requested set of bridge entry files under ``root``.""" + (root / "dist").mkdir(parents=True, exist_ok=True) + (root / "node_modules" / "tsx" / "dist").mkdir(parents=True, exist_ok=True) + (root / "node_modules" / "tsx" / "dist" / "cli.mjs").write_text("// stub\n") + if mjs: + (root / "dist" / "bridge.mjs").write_text("// stub\n") + if cjs: + (root / "dist" / "bridge.cjs").write_text("// stub\n") + if mts: + (root / "bridge.mts").write_text("// stub\n") + if cts: + (root / "bridge.cts").write_text("// stub\n") + + +class BridgeScriptResolutionTests(unittest.TestCase): + """Direct precedence test for `bridge_client._bridge_script`.""" + + def _resolve(self, root: Path) -> Path: + return bridge_client_mod._bridge_script(root) + + def test_prefers_dist_mjs_over_everything_else(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + _layout(root, mjs=True, cjs=True, mts=True, cts=True) + self.assertEqual(self._resolve(root), root / "dist" / "bridge.mjs") + + def test_falls_back_to_dist_cjs_when_no_mjs(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + _layout(root, mjs=False, cjs=True, mts=True, cts=True) + self.assertEqual(self._resolve(root), root / "dist" / "bridge.cjs") + + def test_falls_back_to_source_mts_when_no_dist(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + _layout(root, mjs=False, cjs=False, mts=True, cts=True) + self.assertEqual(self._resolve(root), root / "bridge.mts") + + def test_falls_back_to_legacy_source_cts(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + _layout(root, mjs=False, cjs=False, mts=False, cts=True) + self.assertEqual(self._resolve(root), root / "bridge.cts") + + def test_returns_legacy_cts_path_when_nothing_exists(self) -> None: + """Default to the historical cts path so error messages stay stable.""" + with TemporaryDirectory() as tmp: + root = Path(tmp) + _layout(root, mjs=False, cjs=False, mts=False, cts=False) + self.assertEqual(self._resolve(root), root / "bridge.cts") + + +class BridgeCommandLaunchTests(unittest.TestCase): + """`MemosBridgeClient.__init__` must launch `.mjs` files with plain ``node``. + + The provided ``bridge_path`` flows straight into ``subprocess.Popen``, + which is patched out. We assert the assembled command line and verify + no ``tsx`` wrapper is inserted for the new ESM entry. + """ + + def _captured_cmd(self, script_path: str) -> list[str]: + with ( + patch.object(bridge_client_mod.subprocess, "Popen") as popen, + patch.object(bridge_client_mod.shutil, "which", return_value="/usr/bin/node"), + patch.object(bridge_client_mod.threading, "Thread"), + ): + popen.return_value.pid = 999 + popen.return_value.stdin = None + popen.return_value.stdout = None + popen.return_value.stderr = None + bridge_client_mod.MemosBridgeClient(bridge_path=script_path) + popen.assert_called_once() + return list(popen.call_args.args[0]) + + def test_mjs_path_launches_with_node_directly(self) -> None: + cmd = self._captured_cmd("/tmp/dist/bridge.mjs") + self.assertEqual(cmd[0], "/usr/bin/node") + self.assertIn("/tmp/dist/bridge.mjs", cmd) + self.assertNotIn("tsx", " ".join(cmd)) + self.assertIn("--agent=hermes", cmd) + self.assertIn("--no-viewer", cmd) + + def test_cjs_path_still_launches_with_node_directly(self) -> None: + cmd = self._captured_cmd("/tmp/dist/bridge.cjs") + self.assertEqual(cmd[0], "/usr/bin/node") + self.assertIn("/tmp/dist/bridge.cjs", cmd) + self.assertNotIn("tsx", " ".join(cmd)) + + def test_mts_source_routes_through_tsx(self) -> None: + """`.mts` is TypeScript source and needs the tsx loader.""" + cmd = self._captured_cmd("/tmp/bridge.mts") + joined = " ".join(cmd) + self.assertIn("/tmp/bridge.mts", cmd) + # Either explicit `tsx/dist/cli.mjs` or `--import tsx`. + self.assertTrue("tsx" in joined, f"expected tsx wrapper in cmd: {cmd!r}") + + def test_cts_source_still_routes_through_tsx(self) -> None: + cmd = self._captured_cmd("/tmp/bridge.cts") + joined = " ".join(cmd) + self.assertIn("/tmp/bridge.cts", cmd) + self.assertTrue("tsx" in joined, f"expected tsx wrapper in cmd: {cmd!r}") + + +class DaemonBridgeScriptTests(unittest.TestCase): + """Same precedence rules apply to the viewer-daemon helper.""" + + def _resolve(self, root: Path) -> Path: + # The daemon module's `_bridge_script` reads `_plugin_root()` directly + # — patch it to return the temporary root. + with patch.object(daemon_manager_mod, "_plugin_root", return_value=root): + return daemon_manager_mod._bridge_script() + + def _command(self, root: Path) -> list[str]: + with ( + patch.object(daemon_manager_mod, "_plugin_root", return_value=root), + patch.object(daemon_manager_mod, "_node_binary", return_value="/usr/bin/node"), + ): + return daemon_manager_mod._bridge_command(daemon=True) + + def test_daemon_prefers_dist_mjs(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + _layout(root, mjs=True, cjs=True, mts=False, cts=True) + self.assertEqual(self._resolve(root), root / "dist" / "bridge.mjs") + + def test_daemon_command_for_mjs_uses_node(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + _layout(root, mjs=True, cjs=True, mts=False, cts=True) + cmd = self._command(root) + self.assertEqual(cmd[0], "/usr/bin/node") + self.assertTrue(cmd[1].endswith("dist/bridge.mjs")) + self.assertIn("--agent=hermes", cmd) + self.assertIn("--daemon", cmd) + self.assertNotIn("tsx", " ".join(cmd)) + + +if __name__ == "__main__": + unittest.main() diff --git a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py index 149cea586..1c8cb6a6c 100644 --- a/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py +++ b/apps/memos-local-plugin/tests/python/test_hermes_provider_pipeline.py @@ -123,6 +123,105 @@ def test_lifecycle_persists_turn_and_closes_real_episode(self) -> None: self.assertTrue(bridge.closed) + def test_initialize_closes_previous_bridge_before_spawning_new_one(self) -> None: + """Regression for #1927: re-calling ``initialize()`` must close the + previously-spawned bridge instead of leaking it. + + Hermes calls ``initialize()`` on every reconnect / new session. + Before the fix, each call replaced ``self._bridge`` with a fresh + ``MemosBridgeClient`` (and thus a new ``--no-viewer`` Node + subprocess) without closing the old one, leaking ~93 MB per call. + """ + first_bridge = FakeBridge() + second_bridge = FakeBridge() + bridge_attempts = [first_bridge, second_bridge] + + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch( + "memos_provider.MemosBridgeClient", + side_effect=lambda: bridge_attempts.pop(0), + ), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("session-1") + self.assertIs(provider._bridge, first_bridge) + self.assertFalse(first_bridge.closed) + + # Second initialize (e.g. reconnect / new Hermes session) must + # close the previous bridge before allocating a new one. + provider.initialize("session-2") + self.assertTrue( + first_bridge.closed, + "previous bridge was not closed — leak (#1927)", + ) + self.assertIs(provider._bridge, second_bridge) + self.assertFalse(second_bridge.closed) + + provider.shutdown() + + # And the second bridge must be cleaned up by shutdown(), so we + # know we did not somehow drop the new reference along the way. + self.assertTrue(second_bridge.closed) + + def test_initialize_when_no_previous_bridge_does_not_call_close(self) -> None: + """First-ever ``initialize()`` must not blow up on the missing + previous bridge — it should just spawn one.""" + bridge = FakeBridge() + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch("memos_provider.MemosBridgeClient", return_value=bridge), + ): + provider = memos_provider.MemTensorProvider() + self.assertIsNone(provider._bridge) + + provider.initialize("fresh-session") + self.assertIs(provider._bridge, bridge) + self.assertFalse(bridge.closed) + + provider.shutdown() + + self.assertTrue(bridge.closed) + + def test_initialize_swallows_exception_from_old_bridge_close(self) -> None: + """If the previous bridge's ``close()`` raises (e.g. stuck Node + subprocess), ``initialize()`` must still allocate the new bridge + and proceed — never leak just because cleanup is flaky.""" + + class StuckCloseBridge(FakeBridge): + def close(self) -> None: + # Mark closed so we can still assert it was attempted, + # but raise to mimic a misbehaving subprocess teardown. + self.closed = True + raise RuntimeError("simulated stuck bridge close") + + stuck_bridge = StuckCloseBridge() + healthy_bridge = FakeBridge() + bridge_attempts = [stuck_bridge, healthy_bridge] + + with ( + patch("memos_provider.ensure_bridge_running", return_value=True), + patch("memos_provider.ensure_viewer_daemon", return_value=True), + patch( + "memos_provider.MemosBridgeClient", + side_effect=lambda: bridge_attempts.pop(0), + ), + ): + provider = memos_provider.MemTensorProvider() + provider.initialize("session-1") + self.assertIs(provider._bridge, stuck_bridge) + + # Must not propagate the close() failure to the caller. + provider.initialize("session-2") + self.assertTrue(stuck_bridge.closed) + self.assertIs(provider._bridge, healthy_bridge) + + provider.shutdown() + + self.assertTrue(healthy_bridge.closed) + def test_sync_turn_recovers_when_initial_bridge_open_timed_out(self) -> None: failed_bridge = FailingSessionOpenBridge() recovered_bridge = FakeBridge() diff --git a/apps/memos-local-plugin/tests/unit/adapters/openclaw-runtime.test.ts b/apps/memos-local-plugin/tests/unit/adapters/openclaw-runtime.test.ts index 19378853a..401acb9aa 100644 --- a/apps/memos-local-plugin/tests/unit/adapters/openclaw-runtime.test.ts +++ b/apps/memos-local-plugin/tests/unit/adapters/openclaw-runtime.test.ts @@ -8,12 +8,15 @@ import { DEFAULT_CONFIG } from "../../../core/config/defaults.js"; import { resolveHome, type ResolvedHome } from "../../../core/config/index.js"; import type { HostLogger, + OpenClawHookHandlerMap, + OpenClawHookName, OpenClawPluginApi, ServiceDescriptor, } from "../../../adapters/openclaw/openclaw-api.js"; interface MockApi extends OpenClawPluginApi { services: ServiceDescriptor[]; + hooks: Map; logger: HostLogger & { info: ReturnType; warn: ReturnType; @@ -30,6 +33,7 @@ afterEach(() => { vi.doUnmock("../../../core/pipeline/index.js"); vi.doUnmock("../../../server/http.js"); vi.doUnmock("../../../core/telemetry/index.js"); + vi.doUnmock("../../../adapters/openclaw/bridge.js"); vi.resetModules(); vi.restoreAllMocks(); for (const root of tempRoots.splice(0)) { @@ -55,6 +59,7 @@ function makeCore() { function makeApi(): MockApi { const services: ServiceDescriptor[] = []; + const hooks = new Map(); const logger = { trace: vi.fn(), debug: vi.fn(), @@ -67,9 +72,12 @@ function makeApi(): MockApi { name: "MemOS Local", logger, services, + hooks, registerTool: vi.fn(), registerMemoryCapability: vi.fn(), - on: vi.fn(), + on: vi.fn((hookName: OpenClawHookName, handler) => { + hooks.set(hookName, handler); + }), registerService: vi.fn((svc: ServiceDescriptor) => { services.push(svc); }), @@ -79,6 +87,7 @@ function makeApi(): MockApi { async function loadPluginWithMocks( bootstrapMemoryCoreFull: ReturnType, startHttpServer: ReturnType, + createOpenClawBridge?: ReturnType, ) { vi.resetModules(); vi.doMock("../../../core/pipeline/index.js", () => ({ @@ -93,6 +102,17 @@ async function loadPluginWithMocks( shutdown = vi.fn(async () => {}); }, })); + if (createOpenClawBridge) { + vi.doMock("../../../adapters/openclaw/bridge.js", async () => { + const actual = await vi.importActual< + typeof import("../../../adapters/openclaw/bridge.js") + >("../../../adapters/openclaw/bridge.js"); + return { + ...actual, + createOpenClawBridge, + }; + }); + } const mod = await import("../../../adapters/openclaw/index.js"); return mod.default; } @@ -172,3 +192,287 @@ describe("OpenClaw adapter runtime lifecycle", () => { expect(fs.existsSync(path.join(home.daemonDir, "openclaw-runtime.lock"))).toBe(false); }); }); + +// ─── Regressions for issue #1815 ──────────────────────────────────────────── +// +// OpenClaw's hook runner enforces two contracts memos must respect: +// +// 1. `tool_result_persist` is a value-returning **synchronous** hook. +// The runner inspects the return value with `isPromiseLike(ret)` and +// silently ignores anything that looks like a Promise. If memos +// registers an `async` listener, the hint-injection feature is +// dead — and OpenClaw logs: +// "tool_result_persist handler from memos-local-plugin returned a +// Promise; this hook is synchronous and the result was ignored." +// 2. `agent_end` (and the other void hooks) is gated by a hard-coded +// `DEFAULT_VOID_HOOK_TIMEOUT_MS_BY_HOOK.agent_end = 30_000` budget. +// A long-running awaited handler trips the gateway log warning: +// "agent_end handler from memos-local-plugin failed: timed out +// after 30000ms". +// +// These two regressions cover the listener wrappers in +// `adapters/openclaw/index.ts` — the bridge handlers themselves are +// already correctly shaped and exercised by openclaw-bridge.test.ts. + +function buildPluginWithFakeBridge(opts: { + bridge: Record>; +}): Promise { + const home = useTempMemosHome(); + const core = makeCore(); + const bootstrapMemoryCoreFull = vi.fn(async () => ({ + core, + config: DEFAULT_CONFIG, + home, + })); + const startHttpServer = vi.fn(async () => ({ + url: "http://127.0.0.1:18799", + port: 18799, + closed: false, + close: vi.fn(async () => {}), + })); + const createOpenClawBridge = vi.fn(() => opts.bridge); + return loadPluginWithMocks( + bootstrapMemoryCoreFull, + startHttpServer, + createOpenClawBridge, + ); +} + +function buildBridgeStub() { + return { + handleBeforePrompt: vi.fn(async () => undefined), + handleAgentEnd: vi.fn(async () => undefined), + handleBeforeToolCall: vi.fn(() => undefined), + handleAfterToolCall: vi.fn(async () => undefined), + handleToolResultPersist: vi.fn((_event: unknown) => ({ + message: { content: "hint-applied" }, + })), + handleSessionStart: vi.fn(async () => undefined), + handleSessionEnd: vi.fn(async () => undefined), + handleSubagentSpawned: vi.fn(() => undefined), + handleSubagentEnded: vi.fn(async () => undefined), + trackedSessions: vi.fn(() => 0), + trackedToolCalls: vi.fn(() => 0), + }; +} + +describe("OpenClaw hook listener contract (issue #1815)", () => { + it("registers tool_result_persist as a SYNC listener that returns the bridge result directly (not a Promise)", async () => { + const bridge = buildBridgeStub(); + const plugin = (await buildPluginWithFakeBridge({ bridge })) as { + register: (api: OpenClawPluginApi) => void; + }; + const api = makeApi(); + plugin.register(api); + await api.services[0]!.start?.(); + + const handler = api.hooks.get("tool_result_persist") as + | OpenClawHookHandlerMap["tool_result_persist"] + | undefined; + expect(handler).toBeDefined(); + + // Critical contract: the listener must NOT be an async function. + // OpenClaw checks `handler.constructor.name === "AsyncFunction"` / + // `isPromiseLike(ret)` to decide whether to ignore the return value + // — both flag it as broken. + expect(handler!.constructor.name).not.toBe("AsyncFunction"); + + const result = handler!( + { + toolName: "sh", + toolCallId: "call_X", + message: { + role: "toolResult", + content: "boom", + isError: true, + }, + }, + { + toolName: "sh", + toolCallId: "call_X", + agentId: "main", + sessionKey: "s-1", + sessionId: "host-s-1", + runId: "run-1", + }, + ); + + // The return value must be the bridge's payload directly, not a + // Promise that wraps it. OpenClaw's hook runner ignores Promises. + expect(result).not.toBeInstanceOf(Promise); + expect((result as { message?: { content?: string } } | void)?.message?.content).toBe( + "hint-applied", + ); + expect(bridge.handleToolResultPersist).toHaveBeenCalledTimes(1); + + await api.services[0]!.stop?.(); + }); + + it("tool_result_persist listener silently no-ops when bootstrap has not finished yet", async () => { + const home = useTempMemosHome(); + const core = makeCore(); + const bootDeferred = deferred<{ + core: ReturnType; + config: typeof DEFAULT_CONFIG; + home: ResolvedHome; + }>(); + const bootstrapMemoryCoreFull = vi.fn(() => bootDeferred.promise); + const startHttpServer = vi.fn(async () => ({ + url: "http://127.0.0.1:18799", + port: 18799, + closed: false, + close: vi.fn(async () => {}), + })); + const bridge = buildBridgeStub(); + const createOpenClawBridge = vi.fn(() => bridge); + const plugin = await loadPluginWithMocks( + bootstrapMemoryCoreFull, + startHttpServer, + createOpenClawBridge, + ); + + const api = makeApi(); + plugin.register(api); + + // Bootstrap is intentionally still pending here — runtime is null. + const handler = api.hooks.get("tool_result_persist") as + | OpenClawHookHandlerMap["tool_result_persist"] + | undefined; + expect(handler).toBeDefined(); + + const result = handler!( + { + toolName: "sh", + toolCallId: "call_X", + message: { role: "toolResult", content: "boom", isError: true }, + }, + { toolName: "sh", agentId: "main", sessionKey: "s-1", runId: "run-1" }, + ); + + // Must be a synchronous undefined return, NOT a Promise. The bridge + // factory should not even have been invoked yet. + expect(result).toBeUndefined(); + expect(bridge.handleToolResultPersist).not.toHaveBeenCalled(); + + // Finish bootstrap so afterEach can clean up. + bootDeferred.resolve({ core, config: DEFAULT_CONFIG, home }); + await api.services[0]!.start?.(); + await api.services[0]!.stop?.(); + }); + + it("agent_end listener returns synchronously and dispatches the bridge work as fire-and-forget", async () => { + let releaseAgentEnd: (() => void) | null = null; + const agentEndStarted = vi.fn(); + const bridge = buildBridgeStub(); + bridge.handleAgentEnd = vi.fn( + () => + new Promise((resolve) => { + agentEndStarted(); + releaseAgentEnd = resolve; + }), + ); + + const plugin = (await buildPluginWithFakeBridge({ bridge })) as { + register: (api: OpenClawPluginApi) => void; + }; + const api = makeApi(); + plugin.register(api); + await api.services[0]!.start?.(); + + const handler = api.hooks.get("agent_end") as + | OpenClawHookHandlerMap["agent_end"] + | undefined; + expect(handler).toBeDefined(); + + const beforeReturn = Date.now(); + const ret = handler!( + { messages: [], success: true, durationMs: 10 }, + { agentId: "main", sessionKey: "s-1", runId: "run-1" }, + ); + const afterReturn = Date.now(); + + // OpenClaw's contract: the listener must return void / undefined, + // NOT a Promise that the runner would await against the 30 s + // hard-coded budget. + expect(ret).toBeUndefined(); + expect(afterReturn - beforeReturn).toBeLessThan(50); + + // The bridge work, however, MUST eventually run — fire-and-forget, + // not fire-and-drop. + await vi.waitFor(() => { + expect(bridge.handleAgentEnd).toHaveBeenCalledTimes(1); + }); + expect(agentEndStarted).toHaveBeenCalledTimes(1); + + // Release the background work so afterEach can shut down cleanly. + releaseAgentEnd?.(); + await api.services[0]!.stop?.(); + }); + + it("agent_end listener swallows background errors via opts.log.warn instead of surfacing them to OpenClaw", async () => { + const bridge = buildBridgeStub(); + bridge.handleAgentEnd = vi.fn(async () => { + throw new Error("boom inside onTurnEnd"); + }); + + const plugin = (await buildPluginWithFakeBridge({ bridge })) as { + register: (api: OpenClawPluginApi) => void; + }; + const api = makeApi(); + plugin.register(api); + await api.services[0]!.start?.(); + + const handler = api.hooks.get("agent_end") as + | OpenClawHookHandlerMap["agent_end"] + | undefined; + expect(handler).toBeDefined(); + expect(() => + handler!( + { messages: [], success: true }, + { agentId: "main", sessionKey: "s-1", runId: "run-1" }, + ), + ).not.toThrow(); + + await vi.waitFor(() => { + expect(api.logger.warn).toHaveBeenCalledWith( + expect.stringContaining("hook agent_end failed"), + expect.objectContaining({ err: expect.stringContaining("boom inside onTurnEnd") }), + ); + }); + + await api.services[0]!.stop?.(); + }); + + it("every void hook listener (agent_end / session_* / *_tool_call / subagent_*) is registered as a non-async function", async () => { + const bridge = buildBridgeStub(); + const plugin = (await buildPluginWithFakeBridge({ bridge })) as { + register: (api: OpenClawPluginApi) => void; + }; + const api = makeApi(); + plugin.register(api); + await api.services[0]!.start?.(); + + // before_prompt_build is value-returning and the only listener + // allowed to be async (OpenClaw awaits its prependContext). + const voidHooks: OpenClawHookName[] = [ + "agent_end", + "before_tool_call", + "after_tool_call", + "tool_result_persist", + "session_start", + "session_end", + "subagent_spawned", + "subagent_ended", + ]; + for (const name of voidHooks) { + const handler = api.hooks.get(name); + expect(handler, `${name} listener missing`).toBeDefined(); + expect( + handler!.constructor.name, + `${name} listener must not be async; OpenClaw runs it on a sync/void path`, + ).not.toBe("AsyncFunction"); + } + + await api.services[0]!.stop?.(); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/bridge-status.test.ts b/apps/memos-local-plugin/tests/unit/bridge-status.test.ts new file mode 100644 index 000000000..c92214db8 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/bridge-status.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; + +import { matchesHermesChatCommandLine } from "../../bridge/hermes-process.js"; + +describe("Hermes chat process detection", () => { + it("matches the standard Hermes chat command", () => { + expect(matchesHermesChatCommandLine("hermes chat")).toBe(true); + expect(matchesHermesChatCommandLine("/usr/local/bin/hermes chat")).toBe(true); + }); + + it("matches global flags before the chat subcommand", () => { + expect( + matchesHermesChatCommandLine( + "/usr/local/lib/hermes-agent/venv/bin/hermes --skills memory-routing chat", + ), + ).toBe(true); + expect( + matchesHermesChatCommandLine( + "hermes -m gpt-4.1 --provider openai chat --skills memory-routing", + ), + ).toBe(true); + }); + + it("does not match non-chat Hermes commands", () => { + expect(matchesHermesChatCommandLine("hermes dashboard")).toBe(false); + expect(matchesHermesChatCommandLine("hermes --skills memory-routing gateway")).toBe(false); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/bridge/bridge-startup-ordering.test.ts b/apps/memos-local-plugin/tests/unit/bridge/bridge-startup-ordering.test.ts new file mode 100644 index 000000000..7f83255ec --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/bridge/bridge-startup-ordering.test.ts @@ -0,0 +1,70 @@ +/** + * bridge.cts startup-ordering regression guard for issue #1747. + * + * Issue summary: when `core.init()` ran before `startStdioServer({ core })` + * in `bridge.cts`'s non-daemon (stdio) startup path, orphan-episode + * recovery (LLM-driven reward grading inside `core.init()`) could take + * 10-60+ seconds. While that ran, the stdio JSON-RPC read loop was not + * yet attached to `process.stdin`, so the Hermes Python adapter's + * `session.open` RPC sat unread in the pipe buffer until its 30 s + * `_open_session()` timeout fired with `asyncio.TimeoutError`. + * + * The fix landed in commit 7c6bd250 (May 2026) — `startStdioServer` now + * runs first. This test pins that invariant so a future refactor can't + * silently re-introduce the race. + * + * Why source-level (rather than runtime): `bridge.cts` is a top-level + * executable script — refactoring it into an injectable function just + * to runtime-test the ordering would be a much larger change than the + * invariant deserves. A source-level assertion catches the regression + * at the moment a developer rearranges the ordering, which is exactly + * what the issue asks us to prevent. + */ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const BRIDGE_CTS_PATH = resolve(HERE, "..", "..", "..", "bridge.cts"); + +describe("bridge.cts startup ordering (regression guard for #1747)", () => { + const src = readFileSync(BRIDGE_CTS_PATH, "utf8"); + + it("starts the stdio JSON-RPC server before awaiting core.init() in non-daemon mode", () => { + // Match the assignment form `stdio = startStdioServer({ core })` + // and the call form `await core.init();` (with trailing semicolon). + // Both substrings appear *only* at the real call sites, never in + // the surrounding rationale comments (which use bare + // `startStdioServer({ core })` and `await core.init()` in prose). + const stdioIdx = src.indexOf("stdio = startStdioServer({ core })"); + const initIdx = src.indexOf("await core.init();"); + + expect(stdioIdx, "stdio = startStdioServer({ core }) call site not found").toBeGreaterThanOrEqual(0); + expect(initIdx, "await core.init(); call site not found").toBeGreaterThanOrEqual(0); + expect( + stdioIdx, + `bridge.cts: stdio = startStdioServer({ core }) must appear textually before await core.init(); ` + + `(stdioIdx=${stdioIdx} initIdx=${initIdx}). ` + + `Reordering reopens the issue #1747 race where the Python adapter's ` + + `session.open RPC times out while orphan recovery blocks the stdio reader.`, + ).toBeLessThan(initIdx); + }); + + it("keeps the stdio start guarded inside the !args.daemon branch", () => { + // Daemon mode (--daemon) intentionally skips stdio entirely and runs + // as a pure HTTP viewer. We want the regression guard to apply to + // the stdio (non-daemon) path only — moving the call outside the + // guard would re-enable stdio in daemon mode, which is a separate + // breaking change. This test pins the guard's presence. + const stdioIdx = src.indexOf("stdio = startStdioServer({ core })"); + expect(stdioIdx).toBeGreaterThanOrEqual(0); + + const before = src.slice(0, stdioIdx); + const lastDaemonGuard = before.lastIndexOf("if (!args.daemon)"); + expect( + lastDaemonGuard, + "bridge.cts: stdio = startStdioServer({ core }) must remain inside an `if (!args.daemon)` block", + ).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts b/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts new file mode 100644 index 000000000..2f900c7b0 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/bridge/hermes-process.test.ts @@ -0,0 +1,124 @@ +/** + * Hermes chat process detection — regex + pgrep wrapper. + * + * Regression for #1915: the daemon-mode bridge's `applyStaleRule()` + * used `pgrep -f "hermes chat"` as a literal substring, so any + * invocation with a global flag between the binary and the + * subcommand (`hermes --skills memory-routing chat`) was silently + * missed and the viewer was stuck on `"disconnected"`. + * + * The pattern under test is `hermes(?:\s+\S+)*\s+chat\b` — these cases + * lock in the exact shape of the fix. + */ +import { describe, expect, it, vi } from "vitest"; + +import { + HERMES_CHAT_PROCESS_PATTERN, + isHermesChatRunning, + matchesHermesChatCommandLine, +} from "../../../bridge/hermes-process.js"; + +describe("HERMES_CHAT_PROCESS_PATTERN", () => { + it("is the documented #1915 regex (locks in the wire format pgrep sees)", () => { + // If this string ever changes, audit `bridge.cts` callers and the + // issue description before adjusting — the constant is the only + // surface that fixes the substring-detection bug. + expect(HERMES_CHAT_PROCESS_PATTERN).toBe("hermes(?:\\s+\\S+)*\\s+chat\\b"); + }); +}); + +describe("matchesHermesChatCommandLine", () => { + // ─── positive cases (must match) ─────────────────────────────── + it("matches plain `hermes chat` with no flags", () => { + expect( + matchesHermesChatCommandLine("/usr/local/bin/hermes chat"), + ).toBe(true); + }); + + it("matches `hermes chat --skills …` (chat-level flags, the old happy path)", () => { + expect( + matchesHermesChatCommandLine( + "/usr/local/bin/hermes chat --skills memory-routing", + ), + ).toBe(true); + }); + + it("matches `hermes --skills … chat` (global long flag before subcommand) — #1915", () => { + expect( + matchesHermesChatCommandLine( + "/usr/local/lib/hermes-agent/venv/bin/hermes --skills memory-routing chat", + ), + ).toBe(true); + }); + + it("matches `hermes -m gpt-4 chat` (global short flag before subcommand) — #1915", () => { + expect( + matchesHermesChatCommandLine("/usr/local/bin/hermes -m gpt-4 chat"), + ).toBe(true); + }); + + it("matches `hermes --provider openai --skills mem chat` (multiple global flags) — #1915", () => { + expect( + matchesHermesChatCommandLine( + "/usr/local/bin/hermes --provider openai --skills mem chat", + ), + ).toBe(true); + }); + + // ─── negative cases (must NOT match) ─────────────────────────── + it("does not match `hermes status` (different subcommand)", () => { + expect( + matchesHermesChatCommandLine("/usr/local/bin/hermes status"), + ).toBe(false); + }); + + it("does not match `hermes chatter` (chat must end at a word boundary)", () => { + expect( + matchesHermesChatCommandLine("/usr/local/bin/hermes chatter"), + ).toBe(false); + }); + + it("does not match `hermes --chat-log=... status` (chat must be the subcommand token)", () => { + expect( + matchesHermesChatCommandLine( + "/usr/local/bin/hermes --chat-log=/tmp/history status", + ), + ).toBe(false); + }); + + it("does not match a process with neither `hermes` nor `chat`", () => { + expect(matchesHermesChatCommandLine("/bin/bash -c sleep 5")).toBe(false); + }); + + it("does not match `hermesctl …` (binary name needs a whitespace separator)", () => { + // Catches the foot-gun of dropping the `\s` anchor and matching + // `hermesctl --chat-log=...` against the pattern. + expect( + matchesHermesChatCommandLine("/usr/local/bin/hermesctl --chat-log=foo"), + ).toBe(false); + }); +}); + +describe("isHermesChatRunning", () => { + it("calls pgrep with the documented #1915 regex pattern", () => { + const spy = vi.fn().mockReturnValue("12345\n"); + expect(isHermesChatRunning(spy)).toBe(true); + expect(spy).toHaveBeenCalledWith( + "pgrep", + ["-f", HERMES_CHAT_PROCESS_PATTERN], + { encoding: "utf8", timeout: 1000 }, + ); + }); + + it("returns false when pgrep prints only whitespace (no match)", () => { + const spy = vi.fn().mockReturnValue("\n"); + expect(isHermesChatRunning(spy)).toBe(false); + }); + + it("returns false when pgrep throws (e.g. exit 1 on no match, binary missing)", () => { + const spy = vi.fn().mockImplementation(() => { + throw new Error("Command failed with exit code 1"); + }); + expect(isHermesChatRunning(spy)).toBe(false); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/bridge/startup-order.test.ts b/apps/memos-local-plugin/tests/unit/bridge/startup-order.test.ts new file mode 100644 index 000000000..e8bff4915 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/bridge/startup-order.test.ts @@ -0,0 +1,149 @@ +/** + * Bridge startup order regression test. + * + * Issue #1747: Hermes Python client's first `session.open` RPC timed out + * when orphan episodes existed, because `core.init()` was blocking the + * stdio read loop startup. Orphan recovery runs LLM calls (reward/reflection), + * which can take 10-60+ seconds. If stdio hasn't started yet, the Python + * adapter writes `session.open` to stdin but nobody reads it → timeout. + * + * Fix (commit 7c6bd250): Move `startStdioServer()` to *before* `core.init()` + * so the stdio read loop is active when init's orphan recovery runs. + * + * This file guards that ordering: if a future refactor reverses the order, + * the tests will fail before the bug reaches production. + */ +import { describe, it, expect, vi } from "vitest"; +import type { MemoryCore } from "../../../agent-contract/memory-core.js"; + +describe("Bridge startup order (#1747 regression)", () => { + it("startStdioServer returns before core.init() resolves", async () => { + // Simulates the bridge.cts sequence: `stdio = startStdioServer(...)` + // is synchronous, so its return marker must land in `callOrder` + // BEFORE any await inside `core.init()` resolves. + const callOrder: string[] = []; + const initDelayMs = 50; // simulate orphan-recovery blocking work + const core = createDelayedInitCore(initDelayMs, callOrder); + + const { startStdioServer } = await import("../../../bridge/stdio.js"); + + // Mirror bridge.cts line 327-338 ordering: + // 1. stdio = startStdioServer({ core }) ← sync + // 2. await core.init() ← may block on LLM calls + const stdio = startStdioServer({ core }); + callOrder.push("stdio_started"); + + await core.init(); + callOrder.push("init_done"); + + // The invariant we're protecting: + // stdio_started must land BEFORE init_complete. + // If a future refactor swaps `startStdioServer` after `await core.init()`, + // stdio_started would land AFTER init_complete and this assertion would fail. + const stdioIdx = callOrder.indexOf("stdio_started"); + const initCompleteIdx = callOrder.indexOf("init_complete"); + expect(stdioIdx).toBeGreaterThanOrEqual(0); + expect(initCompleteIdx).toBeGreaterThanOrEqual(0); + expect(stdioIdx).toBeLessThan(initCompleteIdx); + + await stdio.close(); + }); + + it("core.init() runs concurrently with stdio (does not block stdio startup)", async () => { + // Verifies the temporal claim: with a slow init (100 ms), the stdio + // handle is *available for use* well before init resolves. If stdio + // were sequenced after init, this test's `stdio` variable wouldn't + // exist until init_delayMs later. + const callOrder: string[] = []; + const initDelayMs = 100; + const core = createDelayedInitCore(initDelayMs, callOrder); + + const { startStdioServer } = await import("../../../bridge/stdio.js"); + + const start = Date.now(); + const stdio = startStdioServer({ core }); + const stdioAvailableAtMs = Date.now() - start; + + // stdio must be usable within a few ms — not after the init delay. + // (Generous tolerance to avoid CI flakiness on slow machines.) + expect(stdioAvailableAtMs).toBeLessThan(initDelayMs / 2); + + await core.init(); + await stdio.close(); + }); +}); + +// ─── Stub MemoryCore with delayed init ─────────────────────────────────── + +function createDelayedInitCore( + delayMs: number, + callOrder: string[], +): MemoryCore { + const subscribers: Array<(e: unknown) => void> = []; + const logSubs: Array<(r: unknown) => void> = []; + + return { + init: vi.fn(async () => { + callOrder.push("init_begin"); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + callOrder.push("init_complete"); + }), + shutdown: vi.fn(async () => {}), + health: vi.fn(async () => ({ + ok: true, + version: "test", + uptimeMs: 1, + agent: "openclaw", + paths: { home: "", config: "", db: "", skills: "", logs: "" }, + llm: { available: false, provider: "" }, + embedder: { available: false, provider: "", dim: 0 }, + })), + openSession: vi.fn(async ({ sessionId }) => sessionId ?? "s-auto"), + closeSession: vi.fn(async () => {}), + openEpisode: vi.fn(async ({ episodeId }) => episodeId ?? "e-auto"), + closeEpisode: vi.fn(async () => {}), + onTurnStart: vi.fn(async () => ({ + query: { agent: "openclaw", query: "" }, + hits: [], + injectedContext: "", + tierLatencyMs: { tier1: 0, tier2: 0, tier3: 0 }, + })), + onTurnEnd: vi.fn(async () => ({ traceId: "tr-1", episodeId: "e-1" })), + submitFeedback: vi.fn(async (fb) => ({ + id: "fb-1", + ts: 1, + channel: fb.channel, + polarity: fb.polarity, + magnitude: fb.magnitude, + })), + recordToolOutcome: vi.fn(), + searchMemory: vi.fn(async (q) => ({ + query: q, + hits: [], + injectedContext: "", + tierLatencyMs: { tier1: 0, tier2: 0, tier3: 0 }, + })), + getTrace: vi.fn(async () => null), + getPolicy: vi.fn(async () => null), + getWorldModel: vi.fn(async () => null), + listEpisodes: vi.fn(async () => []), + timeline: vi.fn(async () => []), + listSkills: vi.fn(async () => []), + getSkill: vi.fn(async () => null), + archiveSkill: vi.fn(async () => {}), + subscribeEvents: vi.fn((h: (e: unknown) => void) => { + subscribers.push(h); + return () => { + const i = subscribers.indexOf(h); + if (i >= 0) subscribers.splice(i, 1); + }; + }) as any, + subscribeLogs: vi.fn((h: (r: unknown) => void) => { + logSubs.push(h); + return () => { + const i = logSubs.indexOf(h); + if (i >= 0) logSubs.splice(i, 1); + }; + }) as any, + }; +} diff --git a/apps/memos-local-plugin/tests/unit/config/load.test.ts b/apps/memos-local-plugin/tests/unit/config/load.test.ts index 77f2f6b3f..aae88f8cd 100644 --- a/apps/memos-local-plugin/tests/unit/config/load.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/load.test.ts @@ -112,6 +112,67 @@ viewer: }); expect("dimensions" in cfg.embedding).toBe(false); }); + + // ─── Issue #1929 — vectorScanMaxAgeMs contract ────────────────────── + // The schema must reject obviously bad values (negative, larger than + // a year, or non-numbers) so a "dirty" `PATCH /api/v1/config` cannot + // poison the on-disk YAML. A subsequent `GET /api/v1/config` therefore + // always returns a value in [0, 31_536_000_000] (the default 0 stays + // because the rejected patch never reaches `writer.ts`'s atomic + // rename — see `core/config/writer.ts::patchConfig`). + describe("retrieval.vectorScanMaxAgeMs", () => { + const MAX_MS = 31_536_000_000; + + it("defaults to 0 (no time-window bound) on a bare config", () => { + const cfg = resolveConfig({}); + expect(cfg.algorithm.retrieval.vectorScanMaxAgeMs).toBe(0); + }); + + it.each([ + ["one day", 86_400_000], + ["thirty days", 30 * 86_400_000], + ["max", MAX_MS], + ["zero", 0], + ])("accepts %s (%d ms)", (_label, value) => { + const cfg = resolveConfig({ + algorithm: { retrieval: { vectorScanMaxAgeMs: value } }, + }); + expect(cfg.algorithm.retrieval.vectorScanMaxAgeMs).toBe(value); + }); + + it.each([ + ["negative_1", -1], + ["negative_60s", -60_000], + ["negative_one_day", -86_400_000], + ["max_plus_1", MAX_MS + 1], + ["max_plus_one_day", MAX_MS + 86_400_000], + ["hundred_x_max", MAX_MS * 100], + ])("rejects out-of-range value (%s)", (_label, value) => { + expect(() => + resolveConfig({ algorithm: { retrieval: { vectorScanMaxAgeMs: value } } }), + ).toThrow(/schema validation/); + }); + + it.each([ + ["string_number", "100"], + ["string_text", "abc"], + ["none_value", null], + ["dict_value", { x: 1 }], + ["list_value", [1, 2, 3]], + ["nan_string", "NaN"], + ["inf_string", "Infinity"], + ["bool_true", true], + ["bool_false", false], + ])("rejects invalid type (%s)", (_label, value) => { + expect(() => + resolveConfig({ + algorithm: { + retrieval: { vectorScanMaxAgeMs: value as unknown as number }, + }, + }), + ).toThrow(/schema validation/); + }); + }); }); describe("config/loadConfig MEMOS_HOME override", () => { diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index cd5cf6106..7e904a2c6 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -14,6 +14,7 @@ import type { LlmProvider, LlmProviderCtx, LlmProviderName, + LlmStatusDetail, LlmStreamChunk, ProviderCallInput, ProviderCompletion, @@ -277,4 +278,241 @@ describe("llm/client", () => { const client = createLlmClientWithProvider(cfg(), fake); await expect(client.complete([] as LlmMessage[])).rejects.toBeInstanceOf(MemosError); }); + + // ─── Circuit breaker (issue #1897) ────────────────────────────────────── + describe("circuit breaker", () => { + function statusSink(): { rows: LlmStatusDetail[]; push: (d: LlmStatusDetail) => void } { + const rows: LlmStatusDetail[] = []; + return { rows, push: (d) => rows.push(d) }; + } + + it("trips on terminal 402 and short-circuits subsequent calls", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "HTTP 402 from openai_compatible", { + provider: "openai_compatible", + status: 402, + }), + ); + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 300_000, now: tick }, + }), + provider, + ); + // First call: real provider hit, fails terminally → breaker trips. + await expect(client.complete("first")).rejects.toBeInstanceOf(MemosError); + expect(provider.calls).toBe(1); + // Second call: should be short-circuited; provider must NOT be invoked. + now += 100; + await expect(client.complete("second")).rejects.toMatchObject({ + code: ERROR_CODES.LLM_UNAVAILABLE, + details: { circuitOpen: true }, + }); + expect(provider.calls).toBe(1); + // Stats expose circuit state. + const stats = client.stats(); + expect(stats.circuitOpen).toBe(true); + expect(stats.circuitOpenUntil).toBe(1_000_000 + 300_000); + expect(stats.circuitOpenedReason).toMatch(/402/); + // Audit rows: at least one `error` and one `circuit_open`. + const statuses = sink.rows.map((r) => r.status); + expect(statuses).toContain("error"); + expect(statuses).toContain("circuit_open"); + }); + + it("trips on 'insufficient balance' message regardless of HTTP status", async () => { + const sink = statusSink(); + const provider = new ThrowingProvider( + new MemosError( + ERROR_CODES.LLM_UNAVAILABLE, + "HTTP 400 from openai_compatible: Insufficient Balance", + { provider: "openai_compatible", status: 400 }, + ), + ); + const client = createLlmClientWithProvider( + cfg({ onStatus: sink.push, circuitBreaker: { enabled: true } }), + provider, + ); + await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); + await expect(client.complete("y")).rejects.toMatchObject({ + details: { circuitOpen: true }, + }); + expect(provider.calls).toBe(1); + }); + + it("does NOT trip on generic LLM_UNAVAILABLE without terminal markers", async () => { + const sink = statusSink(); + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "transient network blip"), + ); + const client = createLlmClientWithProvider( + cfg({ onStatus: sink.push, circuitBreaker: { enabled: true } }), + provider, + ); + // Two consecutive failures with non-terminal classification → both + // calls reach the provider, breaker stays closed. + await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); + await expect(client.complete("y")).rejects.toBeInstanceOf(MemosError); + expect(provider.calls).toBe(2); + expect(client.stats().circuitOpen).toBe(false); + }); + + it("coalesces circuit_open status rows within cooldown", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "401", { status: 401 }), + ); + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 300_000, now: tick }, + }), + provider, + ); + await expect(client.complete("trip")).rejects.toBeTruthy(); + // 20 suppressed calls within 1 second → at most a small number of + // `circuit_open` rows (we expect 1, but tolerate up to 2 in case the + // coalescer counts the very first short-circuit as a separate row). + for (let i = 0; i < 20; i++) { + now += 50; + await expect(client.complete(`spam-${i}`)).rejects.toBeTruthy(); + } + const openRows = sink.rows.filter((r) => r.status === "circuit_open"); + expect(openRows.length).toBeGreaterThanOrEqual(1); + expect(openRows.length).toBeLessThanOrEqual(2); + // Provider was only touched once (the very first call that tripped). + expect(provider.calls).toBe(1); + }); + + it("half-open probes the provider after cooldown and closes on success", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + let attempt = 0; + const provider: LlmProvider = { + name: "openai_compatible", + async complete() { + attempt++; + if (attempt === 1) { + throw new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "401", { status: 401 }); + } + return { text: "ok", durationMs: 1 }; + }, + }; + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 60_000, now: tick }, + }), + provider, + ); + await expect(client.complete("trip")).rejects.toBeTruthy(); + expect(client.stats().circuitOpen).toBe(true); + // Suppressed call before cooldown elapses. + now += 30_000; + await expect(client.complete("suppressed")).rejects.toMatchObject({ + details: { circuitOpen: true }, + }); + expect(attempt).toBe(1); + // After cooldown, the next call probes the provider. + now += 31_000; // total 61_000 since trip + const r = await client.complete("probe"); + expect(r.text).toBe("ok"); + expect(attempt).toBe(2); + // Breaker closes on success. + expect(client.stats().circuitOpen).toBe(false); + }); + + it("trips on terminal primary error even when host fallback rescues the call", async () => { + const sink = statusSink(); + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "402", { status: 402 }), + ); + let hostCalls = 0; + registerHostLlmBridge({ + id: "test.host", + async complete() { + hostCalls++; + return { text: "rescued", model: "host-m", durationMs: 1 }; + }, + }); + const client = createLlmClientWithProvider( + cfg({ + fallbackToHost: true, + onStatus: sink.push, + circuitBreaker: { enabled: true }, + }), + provider, + ); + const r = await client.complete("call-1"); + expect(r.servedBy).toBe("host_fallback"); + // The terminal primary error still opens the breaker even though + // host fallback rescued the user-visible call. + expect(client.stats().circuitOpen).toBe(true); + const r2 = await client.complete("call-2"); + expect(r2.servedBy).toBe("host_fallback"); + // The second call goes directly to host fallback and never touches + // the broken paid provider again. + expect(provider.calls).toBe(1); + expect(hostCalls).toBe(2); + expect(sink.rows.map((row) => row.status)).toContain("circuit_open"); + }); + + it("disabled when circuitBreaker.enabled=false (legacy behavior)", async () => { + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "402", { status: 402 }), + ); + const client = createLlmClientWithProvider( + cfg({ circuitBreaker: { enabled: false } }), + provider, + ); + await expect(client.complete("a")).rejects.toBeTruthy(); + await expect(client.complete("b")).rejects.toBeTruthy(); + await expect(client.complete("c")).rejects.toBeTruthy(); + // All three calls reached the provider. + expect(provider.calls).toBe(3); + expect(client.stats().circuitOpen).toBe(false); + }); + + it("LlmClientStats exposes circuit fields when closed", async () => { + const fake = new FakeProvider("openai_compatible", () => ({ text: "ok", durationMs: 1 })); + const client = createLlmClientWithProvider(cfg(), fake); + await client.complete("x"); + const s = client.stats(); + expect(s.circuitOpen).toBe(false); + expect(s.circuitOpenUntil).toBeNull(); + expect(s.circuitOpenedReason).toBeNull(); + }); + + it("re-opens the breaker if the half-open probe fails terminally again", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "402", { status: 402 }), + ); + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 60_000, now: tick }, + }), + provider, + ); + await expect(client.complete("trip")).rejects.toBeTruthy(); + expect(client.stats().circuitOpen).toBe(true); + now += 61_000; + // Half-open probe still fails terminally → breaker re-opens. + await expect(client.complete("probe")).rejects.toBeTruthy(); + expect(client.stats().circuitOpen).toBe(true); + expect(client.stats().circuitOpenUntil).toBe(now + 60_000); + // Provider was touched twice total (initial trip + probe). + expect(provider.calls).toBe(2); + }); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/health-model-display.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/health-model-display.test.ts new file mode 100644 index 000000000..ad401d664 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/pipeline/health-model-display.test.ts @@ -0,0 +1,298 @@ +/** + * Regression test for issue #1596: + * "System overview doesn't show the correct configuration value for model". + * + * The viewer's Overview cards render the configured model for three + * slots — Embedding, Summarizer (= `llm`), and Skill Evolver — using + * the `model` field returned by `MemoryCore.health()`. The contract: + * whatever the user just saved in Settings (i.e. what's on disk in + * `config.yaml`) must be the value shown on Overview, even when an + * earlier provider/model is still cached in-memory. + * + * Three failure modes the old `health()` code did not cover: + * + * 1. Only `model` was overridden from disk when it differed from the + * in-memory client. If the user changed only `provider` (keeping + * the same model name), Overview kept showing the old provider. + * 2. If the user **cleared** a model name in Settings, the old code + * skipped the override (truthy guard) and Overview kept showing + * the previously-configured value. + * 3. The skill evolver in inherited mode (`skillEvolver.model = ""`) + * read the in-memory `llm.model` directly via `resolveSkillEvolver`, + * so it lagged behind disk after a Settings change. + * + * These tests pin the contract: Overview always reflects disk config + * for the three slots' `model` + `provider` display values. + */ + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { bootstrapMemoryCore } from "../../../core/pipeline/index.js"; +import type { MemoryCore } from "../../../agent-contract/memory-core.js"; +import { __resetHostLlmBridgeForTests } from "../../../core/llm/index.js"; +import { makeTmpHome, type TmpHomeContext } from "../../helpers/tmp-home.js"; + +let home: TmpHomeContext | null = null; +let core: MemoryCore | null = null; + +beforeEach(() => { + /* fresh per test */ +}); + +afterEach(async () => { + if (core) { + try { await core.shutdown(); } catch { /* ignore */ } + core = null; + } + if (home) { + await home.cleanup(); + home = null; + } + __resetHostLlmBridgeForTests(); +}); + +describe("health() — Overview model display reflects disk config (#1596)", () => { + it("returns the embedding/llm/skillEvolver model names exactly as saved on disk", async () => { + home = await makeTmpHome({ + agent: "openclaw", + configYaml: ` +version: 1 +embedding: + provider: openai_compatible + endpoint: https://example.test/v1 + model: text-embedding-3-small + apiKey: sk-test-emb +llm: + provider: openai_compatible + endpoint: https://example.test/v1 + model: gpt-4o-mini + apiKey: sk-test-llm +skillEvolver: + provider: anthropic + endpoint: https://example.test + model: claude-sonnet-4 + apiKey: sk-test-skill +algorithm: + lightweightMemory: + enabled: false +`, + }); + core = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "test-1.0.0", + }); + await core.init(); + + const h = await core.health(); + + expect(h.embedder?.model).toBe("text-embedding-3-small"); + expect(h.embedder?.provider).toBe("openai_compatible"); + + expect(h.llm?.model).toBe("gpt-4o-mini"); + expect(h.llm?.provider).toBe("openai_compatible"); + + expect(h.skillEvolver?.model).toBe("claude-sonnet-4"); + expect(h.skillEvolver?.provider).toBe("anthropic"); + expect(h.skillEvolver?.inherited).toBe(false); + }); + + it("when skillEvolver has no own model, inherits from llm and shows llm's disk model", async () => { + home = await makeTmpHome({ + agent: "openclaw", + configYaml: ` +version: 1 +llm: + provider: openai_compatible + endpoint: https://example.test/v1 + model: gpt-4o-mini + apiKey: sk-test-llm +skillEvolver: + provider: "" + model: "" +`, + }); + core = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "test-1.0.0", + }); + await core.init(); + + const h = await core.health(); + + expect(h.llm?.model).toBe("gpt-4o-mini"); + expect(h.skillEvolver?.inherited).toBe(true); + // Inherited skillEvolver MUST show the same llm model as the + // Overview's main LLM card — anything else looks like the + // Overview is out of sync with the configuration. + expect(h.skillEvolver?.model).toBe("gpt-4o-mini"); + expect(h.skillEvolver?.provider).toBe("openai_compatible"); + }); + + it("reflects disk changes after the user saves new model+provider in Settings (no restart yet)", async () => { + // Boot with one config, then mutate the on-disk config.yaml to + // simulate the user hitting "Save" in Settings without restarting. + home = await makeTmpHome({ + agent: "openclaw", + configYaml: ` +version: 1 +embedding: + provider: openai_compatible + endpoint: https://example.test/v1 + model: text-embedding-3-small + apiKey: sk-test-old +llm: + provider: openai_compatible + endpoint: https://example.test/v1 + model: gpt-4o-mini + apiKey: sk-test-old +`, + }); + core = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "test-1.0.0", + }); + await core.init(); + + // User saves new settings — provider and model both change. + const fs = await import("node:fs/promises"); + await fs.writeFile( + home.home.configFile, + ` +version: 1 +embedding: + provider: gemini + endpoint: https://example.test/v1beta + model: text-embedding-004 + apiKey: sk-test-new +llm: + provider: anthropic + endpoint: https://example.test + model: claude-sonnet-4 + apiKey: sk-test-new +`, + "utf8", + ); + + const h = await core.health(); + + // The in-memory client still has the OLD model (no restart), but + // Overview MUST reflect what the user just saved. + expect(h.embedder?.model).toBe("text-embedding-004"); + expect(h.embedder?.provider).toBe("gemini"); + expect(h.llm?.model).toBe("claude-sonnet-4"); + expect(h.llm?.provider).toBe("anthropic"); + }); + + it("when the user changes only the provider (model name unchanged), Overview reflects the new provider", async () => { + home = await makeTmpHome({ + agent: "openclaw", + configYaml: ` +version: 1 +embedding: + provider: openai_compatible + endpoint: https://example.test/v1 + model: shared-embed-model + apiKey: sk-test +llm: + provider: openai_compatible + endpoint: https://example.test/v1 + model: shared-model-name + apiKey: sk-test +`, + }); + core = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "test-1.0.0", + }); + await core.init(); + + // Same model name, different provider — for both embedding + llm. + const fs = await import("node:fs/promises"); + await fs.writeFile( + home.home.configFile, + ` +version: 1 +embedding: + provider: gemini + endpoint: https://example.test/v1beta + model: shared-embed-model + apiKey: sk-test +llm: + provider: anthropic + endpoint: https://example.test + model: shared-model-name + apiKey: sk-test +`, + "utf8", + ); + + const h = await core.health(); + expect(h.llm?.model).toBe("shared-model-name"); + expect(h.embedder?.model).toBe("shared-embed-model"); + // These used to fail: the old code only updated provider if model + // also differed, so the provider stayed on "openai_compatible". + expect(h.llm?.provider).toBe("anthropic"); + expect(h.embedder?.provider).toBe("gemini"); + }); + + it("inherited skillEvolver reflects mid-flight llm changes (no restart yet)", async () => { + home = await makeTmpHome({ + agent: "openclaw", + configYaml: ` +version: 1 +llm: + provider: openai_compatible + endpoint: https://example.test/v1 + model: original-llm-model + apiKey: sk-test +skillEvolver: + provider: "" + model: "" +`, + }); + core = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "test-1.0.0", + }); + await core.init(); + + // User saves new llm settings; skillEvolver still inherits. + const fs = await import("node:fs/promises"); + await fs.writeFile( + home.home.configFile, + ` +version: 1 +llm: + provider: anthropic + endpoint: https://example.test + model: changed-llm-model + apiKey: sk-test +skillEvolver: + provider: "" + model: "" +`, + "utf8", + ); + + const h = await core.health(); + // Sanity: llm slot reflects the new disk values. + expect(h.llm?.model).toBe("changed-llm-model"); + expect(h.llm?.provider).toBe("anthropic"); + // The inherited skillEvolver MUST mirror the llm slot — anything + // else looks broken to the operator (Overview's three model cards + // get out of sync). + expect(h.skillEvolver?.inherited).toBe(true); + expect(h.skillEvolver?.model).toBe("changed-llm-model"); + expect(h.skillEvolver?.provider).toBe("anthropic"); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts index e42af3a79..7f9d9a0fa 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts @@ -1233,6 +1233,10 @@ algorithm: pkgVersion: "orphan-test-recover", }); await core.init(); + // Issue #1808: orphan recovery runs on a background promise; await + // it so the test reads SQLite after every meta / reward write + // settles, not just the synchronous ones. + await core.waitForStartupRecovery?.(); const readDb = new Sqlite(home.home.dbFile, { readonly: true }); const unscored = readDb @@ -1394,6 +1398,11 @@ algorithm: pkgVersion: "dirty-rescore-recover", }); await core.init(); + // Issue #1808: orphan/dirty recovery now runs on a background + // promise so `init()` returns to the host instantly. Tests that + // assert side effects from the recovery chain must await the + // promise explicitly. + await core.waitForStartupRecovery?.(); const readDb = new Sqlite(home.home.dbFile, { readonly: true }); const episode = readDb @@ -1488,6 +1497,9 @@ algorithm: pkgVersion: "missing-reward-recover", }); await core.init(); + // Issue #1808: orphan/dirty recovery now runs on a background + // promise; tests asserting recovery side effects must await it. + await core.waitForStartupRecovery?.(); const readDb = new Sqlite(home.home.dbFile, { readonly: true }); const episode = readDb @@ -1505,4 +1517,485 @@ algorithm: expect(meta.reward?.traceCount).toBe(1); expect(meta.reward?.traceIds).toEqual(["tr_missing_reward"]); }); + + it("init() returns immediately even when a stale orphan's recovery chain stalls (issue #1808)", async () => { + // Issue #1808: on databases with 30k+ traces, the dreaming chain + // synchronous-await inside `init()` blocked the OpenClaw Gateway's + // event loop for 3-5s+, timing out the 3s WebSocket read probe. + // We now run the recovery chain on a background promise so + // `init()` resolves in milliseconds regardless of the chain's + // worst-case latency. + home = await makeTmpHome({ + agent: "openclaw", + configYaml: FULL_MEMORY_CONFIG_YAML, + }); + + const seeder = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "issue1808-init-latency-seed", + }); + await seeder.init(); + await seeder.shutdown(); + + const Sqlite = (await import("better-sqlite3")).default; + const writeDb = new Sqlite(home.home.dbFile); + const orphanOldTs = Date.now() - 5 * 60 * 60 * 1000; // 5h ago > STALE_EPISODE_TIMEOUT_MS + writeDb + .prepare( + `INSERT INTO sessions (id, agent, started_at, last_seen_at, meta_json) VALUES (?, ?, ?, ?, ?)`, + ) + .run("se_issue1808", "openclaw", orphanOldTs, orphanOldTs, "{}"); + for (let i = 0; i < 3; i++) { + writeDb + .prepare( + `INSERT INTO episodes (id, session_id, started_at, ended_at, trace_ids_json, r_task, status, meta_json) VALUES (?, ?, ?, NULL, '[]', NULL, 'open', '{}')`, + ) + .run(`ep_issue1808_${i}`, "se_issue1808", orphanOldTs); + } + writeDb.close(); + + core = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "issue1808-init-latency-recover", + }); + + // Measure how long `init()` itself takes. Even with three stale + // orphans queued for the background reflect/reward chain it must + // resolve well under the OpenClaw Gateway's 3s WebSocket read + // probe budget. + const startedAt = Date.now(); + await core.init(); + const initMs = Date.now() - startedAt; + expect(initMs).toBeLessThan(500); + + // The background promise is still in flight (or just finished); + // either way `waitForStartupRecovery()` must resolve. + await core.waitForStartupRecovery?.(); + }); + + it("dirty closed episodes hit a failure-count backoff so init() does not retry them every restart (issue #1808)", async () => { + // The OpenClaw report noted "orphan episodes with failed LLM calls + // are retried indefinitely with no backoff". After the third + // consecutive failure we suspend automatic retries until the + // exponential backoff window elapses; manual feedback is the only + // way to force another rescore inside the window. + home = await makeTmpHome({ + agent: "openclaw", + configYaml: FULL_MEMORY_CONFIG_YAML, + }); + + const seeder = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "issue1808-backoff-seed", + }); + await seeder.init(); + await seeder.shutdown(); + + const Sqlite = (await import("better-sqlite3")).default; + const writeDb = new Sqlite(home.home.dbFile); + const ts = Date.now() - 2_000; + writeDb + .prepare( + `INSERT INTO sessions (id, agent, started_at, last_seen_at, meta_json) VALUES (?, ?, ?, ?, ?)`, + ) + .run("se_backoff", "openclaw", ts, ts, "{}"); + // Seed a closed episode that is "dirty" by predicate (r_task=null + // + finalized + traceIds.length>0) but whose meta.rewardDirty + // already records 3 prior failures with `lastFailureAt = now` — + // i.e. inside the 1h backoff window. + writeDb + .prepare( + `INSERT INTO episodes (id, session_id, started_at, ended_at, trace_ids_json, r_task, status, meta_json) VALUES (?, ?, ?, ?, ?, ?, 'closed', ?)`, + ) + .run( + "ep_backoff", + "se_backoff", + ts, + ts + 1, + JSON.stringify(["tr_backoff"]), + null, + JSON.stringify({ + closeReason: "finalized", + recoveryReason: "missed_session_end", + rewardDirty: { failedAttempts: 3, lastFailureAt: Date.now() }, + }), + ); + writeDb + .prepare( + `INSERT INTO traces ( + id, episode_id, session_id, ts, user_text, agent_text, summary, + tool_calls_json, reflection, agent_thinking, value, alpha, r_human, + priority, tags_json, error_signatures_json, vec_summary, vec_action, + share_scope, share_target, shared_at, turn_id, schema_version + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, NULL, NULL, ?, ?)`, + ) + .run( + "tr_backoff", + "ep_backoff", + "se_backoff", + ts, + "需求澄清问题", + "好的，分阶段回答。", + "需求澄清问题", + "[]", + null, + null, + 0, + 0, + null, + 0.5, + "[]", + "[]", + ts, + 1, + ); + writeDb.close(); + + core = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "issue1808-backoff-recover", + }); + await core.init(); + await core.waitForStartupRecovery?.(); + + const readDb = new Sqlite(home.home.dbFile, { readonly: true }); + const episode = readDb + .prepare("SELECT r_task, meta_json FROM episodes WHERE id = ?") + .get("ep_backoff") as { r_task: number | null; meta_json: string } | undefined; + readDb.close(); + + expect(episode).toBeDefined(); + // The episode is still inside the backoff window, so the rescan + // skipped it — r_task should stay null (the original failing + // value), and the recovery-reason stamp from `recoverDirtyClosedEpisodes` + // should NOT be present. + expect(episode!.r_task).toBeNull(); + const meta = JSON.parse(episode!.meta_json) as { + recoveryReason?: string; + rewardDirty?: { failedAttempts?: number; lastFailureAt?: number }; + }; + expect(meta.recoveryReason).toBe("missed_session_end"); + expect(meta.rewardDirty?.failedAttempts).toBe(3); + }); + + it("recoverDirtyClosedEpisodes bumps failedAttempts when the rescore did not lift the dirty flag (issue #1808)", async () => { + // After `recoverDirtyClosedEpisodes` finishes its flush, any + // episode still marked dirty has its `meta.rewardDirty` failure + // counter bumped + `lastFailureAt` stamped. Once `failedAttempts` + // crosses MAX_DIRTY_REWARD_ATTEMPTS the backoff filter kicks in + // on subsequent scans. + home = await makeTmpHome({ + agent: "openclaw", + configYaml: FULL_MEMORY_CONFIG_YAML, + }); + + const seeder = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "issue1808-counter-seed", + }); + await seeder.init(); + await seeder.shutdown(); + + const Sqlite = (await import("better-sqlite3")).default; + const writeDb = new Sqlite(home.home.dbFile); + const ts = Date.now() - 2_000; + writeDb + .prepare( + `INSERT INTO sessions (id, agent, started_at, last_seen_at, meta_json) VALUES (?, ?, ?, ?, ?)`, + ) + .run("se_counter", "openclaw", ts, ts, "{}"); + // A dirty episode with NO prior failure counter. Without a working + // LLM the reward listener cannot lift r_task above null → the + // episode stays dirty after recovery → failedAttempts should jump + // from 0 → 1. + writeDb + .prepare( + `INSERT INTO episodes (id, session_id, started_at, ended_at, trace_ids_json, r_task, status, meta_json) VALUES (?, ?, ?, ?, ?, ?, 'closed', ?)`, + ) + .run( + "ep_counter", + "se_counter", + ts, + ts + 1, + JSON.stringify(["tr_counter"]), + null, + JSON.stringify({ + closeReason: "finalized", + recoveryReason: "missed_session_end", + }), + ); + writeDb + .prepare( + `INSERT INTO traces ( + id, episode_id, session_id, ts, user_text, agent_text, summary, + tool_calls_json, reflection, agent_thinking, value, alpha, r_human, + priority, tags_json, error_signatures_json, vec_summary, vec_action, + share_scope, share_target, shared_at, turn_id, schema_version + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, NULL, NULL, ?, ?)`, + ) + .run( + "tr_counter", + "ep_counter", + "se_counter", + ts, + "请帮我详细分析 1808 issue 的根因：为什么 memos-local-plugin 的梦境处理会饿死 OpenClaw 网关的事件循环？说明同步 LLM 调用与 await 之间的关系。", + "OpenClaw Gateway 进程是单线程 Node.js 事件循环。memos-local-plugin 在 init 同步等待 reflect/reward 链 → 阻塞事件循环 3-5s → WebSocket 升级超时。", + "OpenClaw Gateway 与 memos-local-plugin 事件循环阻塞根因分析", + "[]", + null, + null, + 0, + 0, + null, + 0.5, + "[]", + "[]", + ts, + 1, + ); + writeDb.close(); + + // The fakeEmbedder default is used, no LLM is configured (provider="") + // → reward listener resolves with r_task still null (LLM_UNAVAILABLE + // → heuristic fallback writes r_task=0 if it runs; if it cannot run + // we still expect the dirty flag to survive). + core = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "issue1808-counter-recover", + }); + await core.init(); + await core.waitForStartupRecovery?.(); + + const readDb = new Sqlite(home.home.dbFile, { readonly: true }); + const episode = readDb + .prepare("SELECT r_task, meta_json FROM episodes WHERE id = ?") + .get("ep_counter") as { r_task: number | null; meta_json: string } | undefined; + readDb.close(); + + expect(episode).toBeDefined(); + const meta = JSON.parse(episode!.meta_json) as { + rewardDirty?: { failedAttempts?: number; lastFailureAt?: number }; + reward?: { traceCount?: number; skipped?: boolean }; + }; + // Three possible end-states from the reward listener depending on + // the heuristic fallback's behaviour: + // 1. r_task=0, reward.traceCount matches → no longer dirty → backoff cleared + // 2. r_task=null, reward.skipped=true → no longer dirty (skipped path) → backoff cleared + // 3. r_task=null, no reward written → still dirty → failedAttempts bumped to 1 + // In all three cases the *invariant* is "no backoff metadata + // remains if the rescore stopped being dirty, AND failedAttempts + // increases by exactly 1 if it stayed dirty". + if (episode!.r_task !== null || meta.reward?.skipped === true) { + expect(meta.rewardDirty).toBeUndefined(); + } else { + expect(meta.rewardDirty?.failedAttempts).toBe(1); + expect(typeof meta.rewardDirty?.lastFailureAt).toBe("number"); + } + }); + + it("shutdown() awaits background recovery before tearing down storage (issue #1808)", async () => { + // If `shutdown()` did not await the background recovery promise we + // would close SQLite while the reflect / reward listeners were + // still mid-`handle.flush()`, producing `SQLITE_MISUSE` noise. + // Sequential init → shutdown back-to-back must complete cleanly + // without throwing. + home = await makeTmpHome({ + agent: "openclaw", + configYaml: FULL_MEMORY_CONFIG_YAML, + }); + + const seeder = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "issue1808-shutdown-seed", + }); + await seeder.init(); + await seeder.shutdown(); + + const Sqlite = (await import("better-sqlite3")).default; + const writeDb = new Sqlite(home.home.dbFile); + const ts = Date.now() - 2_000; + writeDb + .prepare( + `INSERT INTO sessions (id, agent, started_at, last_seen_at, meta_json) VALUES (?, ?, ?, ?, ?)`, + ) + .run("se_shutdown", "openclaw", ts, ts, "{}"); + writeDb + .prepare( + `INSERT INTO episodes (id, session_id, started_at, ended_at, trace_ids_json, r_task, status, meta_json) VALUES (?, ?, ?, ?, ?, ?, 'closed', ?)`, + ) + .run( + "ep_shutdown", + "se_shutdown", + ts, + ts + 1, + JSON.stringify(["tr_shutdown"]), + null, + JSON.stringify({ + closeReason: "finalized", + recoveryReason: "missed_session_end", + }), + ); + writeDb + .prepare( + `INSERT INTO traces ( + id, episode_id, session_id, ts, user_text, agent_text, summary, + tool_calls_json, reflection, agent_thinking, value, alpha, r_human, + priority, tags_json, error_signatures_json, vec_summary, vec_action, + share_scope, share_target, shared_at, turn_id, schema_version + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, NULL, NULL, ?, ?)`, + ) + .run( + "tr_shutdown", + "ep_shutdown", + "se_shutdown", + ts, + "shutdown 测试", + "好的。", + "shutdown 测试", + "[]", + null, + null, + 0, + 0, + null, + 0.5, + "[]", + "[]", + ts, + 1, + ); + writeDb.close(); + + const fastCore = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "issue1808-shutdown-recover", + }); + await fastCore.init(); + // Deliberately skip `waitForStartupRecovery()`. `shutdown()` must + // still complete the recovery before closing the DB handle. + await expect(fastCore.shutdown()).resolves.toBeUndefined(); + }); + + it("does not rescore a closed episode whose only mismatch is a ghost trace ID (#1966)", async () => { + // Regression guard for https://github.com/MemTensor/MemOS/issues/1966. + // A dangling ID in trace_ids_json must not make reward coverage look dirty + // forever; only trace rows that still exist should count. + home = await makeTmpHome({ + agent: "openclaw", + configYaml: FULL_MEMORY_CONFIG_YAML, + }); + + const seeder = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "ghost-trace-seed", + }); + await seeder.init(); + await seeder.shutdown(); + + const Sqlite = (await import("better-sqlite3")).default; + const writeDb = new Sqlite(home.home.dbFile); + const ts = Date.now() - 1_000; + writeDb + .prepare( + `INSERT INTO sessions (id, agent, started_at, last_seen_at, meta_json) VALUES (?, ?, ?, ?, ?)`, + ) + .run("se_ghost", "openclaw", ts, ts, "{}"); + writeDb + .prepare( + `INSERT INTO episodes (id, session_id, started_at, ended_at, trace_ids_json, r_task, status, meta_json) VALUES (?, ?, ?, ?, ?, ?, 'closed', ?)`, + ) + .run( + "ep_ghost", + "se_ghost", + ts, + ts + 1, + JSON.stringify(["tr_real", "tr_ghost"]), + 0.6, + JSON.stringify({ + closeReason: "finalized", + reward: { + rHuman: 0.6, + scoredAt: ts + 2, + traceCount: 1, + traceIds: ["tr_real"], + source: "heuristic", + }, + }), + ); + writeDb + .prepare( + `INSERT INTO traces ( + id, episode_id, session_id, ts, user_text, agent_text, summary, + tool_calls_json, reflection, agent_thinking, value, alpha, r_human, + priority, tags_json, error_signatures_json, vec_summary, vec_action, + share_scope, share_target, shared_at, turn_id, schema_version + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, NULL, NULL, ?, ?)`, + ) + .run( + "tr_real", + "ep_ghost", + "se_ghost", + ts, + "请讲一下回归任务的损失函数选择。", + "对连续目标变量常用 MSE 或 MAE；存在重尾噪声时用 Huber。", + "回归任务损失函数", + "[]", + null, + null, + 0, + 0, + null, + 0.5, + "[]", + "[]", + ts, + 1, + ); + writeDb.close(); + + core = await bootstrapMemoryCore({ + agent: "openclaw", + home: home.home, + config: home.config, + pkgVersion: "ghost-trace-recover", + }); + await core.init(); + await core.waitForStartupRecovery?.(); + + const readDb = new Sqlite(home.home.dbFile, { readonly: true }); + const episode = readDb + .prepare("SELECT r_task, meta_json FROM episodes WHERE id = ?") + .get("ep_ghost") as { r_task: number | null; meta_json: string } | undefined; + readDb.close(); + + expect(episode).toBeDefined(); + expect(episode!.r_task).toBeCloseTo(0.6); + const meta = JSON.parse(episode!.meta_json) as { + rewardDirty?: unknown; + recoveryReason?: string; + reward?: { rHuman?: number; traceCount?: number; traceIds?: string[] }; + }; + expect(meta.recoveryReason).toBeUndefined(); + expect(meta.reward?.rHuman).toBeCloseTo(0.6); + expect(meta.reward?.traceCount).toBe(1); + expect(meta.reward?.traceIds).toEqual(["tr_real"]); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts index 3d2fb0049..09e607c92 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/integration.test.ts @@ -402,6 +402,45 @@ describe("retrieval/integration", () => { expect(res.stats.emptyPacket).toBe(false); }); + it("turn_start rescues injection when LLM filter empties the kept set (#1913)", async () => { + // Repro for issue #1913: when the LLM relevance filter returns + // `selected: []` for a non-empty ranked list (the case where the + // top hits are all near-duplicate question traces), the packet + // used to collapse to an empty injection. The rescue path keeps + // the top-K best-scoring candidates so the agent still gets a + // packet, and surfaces `llm_filtered_refilled` so the Logs viewer + // can show the safety net fired. + const llm: any = { + completeJson: async () => ({ + value: { selected: [], sufficient: false }, + servedBy: "fake", + }), + }; + const res = await turnStartRetrieve( + { + ...makeDeps(handle), + llm, + config: { + ...makeDeps(handle).config, + llmFilterEnabled: true, + llmFilterMinCandidates: 1, + }, + }, + { + reason: "turn_start", + agent: "openclaw", + sessionId: "s_current" as SessionId, + userText: "run docker compose", + ts: NOW as never, + }, + ); + + expect(res.packet.snippets.length).toBeGreaterThan(0); + expect(res.packet.rendered.length).toBeGreaterThan(0); + expect(res.stats.llmFilterOutcome).toBe("llm_filtered_refilled"); + expect(res.stats.emptyPacket).toBe(false); + }); + it("skill_invoke is tier1-heavy", async () => { const res = await skillInvokeRetrieve(makeDeps(handle), { reason: "skill_invoke", diff --git a/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts index ddb58c604..2355732d7 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts @@ -57,6 +57,33 @@ function trace(id: string, score: number): RankedCandidate { }; } +/** + * Build a trace candidate with no LLM-generated summary / reflection + * and only a short acknowledgement-style `agentText` — i.e. exactly the + * "near-duplicate question trace" shape from issue #1913 where the + * user re-asked a stored fact across multiple sessions and the + * assistant only acked it. The current LLM filter prompt classes these + * as "scaffolding chatter" and is allowed to drop them all. + */ +function chatterTrace( + id: string, + score: number, + overrides: { agentText?: string; userText?: string } = {}, +): RankedCandidate { + const r = trace(id, score); + const cand = r.candidate as TraceCandidate; + return { + ...r, + candidate: { + ...cand, + userText: overrides.userText ?? `What does HERMES_REAL_E2E_1910 mean? (${id})`, + agentText: overrides.agentText ?? "OK", + summary: null, + reflection: null, + }, + }; +} + describe("retrieval/llm-filter", () => { it("disabled → passthrough with null sufficient", async () => { const result = await llmFilterCandidates( @@ -135,7 +162,100 @@ describe("retrieval/llm-filter", () => { expect(result.sufficient).toBe(true); }); - it("LLM returns empty selection → drops everything and marks insufficient", async () => { + it("LLM returns empty selection over non-empty ranked → rescues top-K informative candidates (#1913)", async () => { + // Issue #1913 repro shape: 3 near-duplicate question traces from + // previous sessions plus 1 answer-bearing trace. Previous filter + // honoured `selected: []` and collapsed to empty injection. + const llm: any = { + completeJson: vi.fn().mockResolvedValue({ + value: { selected: [], sufficient: false }, + servedBy: "fake", + }), + }; + const ranked = [ + chatterTrace("q1", 0.95), + chatterTrace("q2", 0.94), + // Answer-bearing trace: long informative agentText, even though + // the ranker placed it below the question duplicates. + (() => { + const r = trace("answer", 0.85); + const c = r.candidate as TraceCandidate; + return { + ...r, + candidate: { + ...c, + userText: + "Remember this fact: HERMES_REAL_E2E_1910 means the bridge leak fix verification.", + agentText: "Noted. HERMES_REAL_E2E_1910 → bridge leak fix verification.", + summary: + "HERMES_REAL_E2E_1910 marks the bridge-leak fix verification.", + reflection: null, + }, + }; + })(), + chatterTrace("q3", 0.8), + ]; + const result = await llmFilterCandidates( + { query: "What does HERMES_REAL_E2E_1910 mean?", ranked }, + { llm, log, config: cfg }, + ); + expect(result.outcome).toBe("llm_filtered_refilled"); + expect(result.kept.length).toBeGreaterThanOrEqual(1); + expect(result.kept[0]!.candidate.refId).toBe("answer"); + expect(result.sufficient).toBe(false); + // Strong negative assertion: the bug returned kept=[], dropped=ranked. + expect(result.dropped.length).toBeLessThan(ranked.length); + }); + + it("rescue fires even when every ranked candidate is short-ack chatter", async () => { + const llm: any = { + completeJson: vi.fn().mockResolvedValue({ + value: { selected: [], sufficient: false }, + servedBy: "fake", + }), + }; + const ranked = [ + chatterTrace("q1", 0.9, { agentText: "OK" }), + chatterTrace("q2", 0.85, { agentText: "记住了" }), + chatterTrace("q3", 0.8, { agentText: "👍" }), + ]; + const result = await llmFilterCandidates( + { query: "What does HERMES_REAL_E2E_1910 mean?", ranked }, + { llm, log, config: cfg }, + ); + expect(result.outcome).toBe("llm_filtered_refilled"); + // No informative candidate exists — rescue still keeps top-K by + // ranker score so the agent at least sees one memory. + expect(result.kept.length).toBeGreaterThanOrEqual(1); + expect(result.kept[0]!.candidate.refId).toBe("q1"); // highest score + expect(result.sufficient).toBe(false); + }); + + it("rescue respects llmFilterMaxKeep cap", async () => { + const llm: any = { + completeJson: vi.fn().mockResolvedValue({ + value: { selected: [], sufficient: false }, + servedBy: "fake", + }), + }; + const ranked = [ + trace("a", 0.95), + trace("b", 0.9), + trace("c", 0.85), + trace("d", 0.8), + ]; + const result = await llmFilterCandidates( + { query: "q", ranked }, + { llm, log, config: { ...cfg, llmFilterMaxKeep: 2 } }, + ); + expect(result.outcome).toBe("llm_filtered_refilled"); + expect(result.kept.length).toBe(2); + expect(result.dropped.length).toBe(2); + }); + + it("llmFilterMaxKeep=0 disables rescue: honours empty selection (drop-everything override)", async () => { + // This is the only configured way to ask the filter to truly drop + // everything; keep it explicit so operators have an escape hatch. const llm: any = { completeJson: vi.fn().mockResolvedValue({ value: { selected: [], sufficient: false }, @@ -145,7 +265,7 @@ describe("retrieval/llm-filter", () => { const ranked = [trace("a", 0.9), trace("b", 0.8)]; const result = await llmFilterCandidates( { query: "q", ranked }, - { llm, log, config: cfg }, + { llm, log, config: { ...cfg, llmFilterMaxKeep: 0 } }, ); expect(result.outcome).toBe("llm_filtered"); expect(result.kept.length).toBe(0); @@ -153,6 +273,22 @@ describe("retrieval/llm-filter", () => { expect(result.sufficient).toBe(false); }); + it("LLM returns empty selection with empty ranked list → unchanged (still llm_filtered, kept=[])", async () => { + const llm: any = { + completeJson: vi.fn().mockResolvedValue({ + value: { selected: [], sufficient: false }, + servedBy: "fake", + }), + }; + // ranked empty but minCandidates=0 so the filter still runs + const result = await llmFilterCandidates( + { query: "q", ranked: [] }, + { llm, log, config: { ...cfg, llmFilterMinCandidates: 0 } }, + ); + expect(result.outcome).toBe("below_threshold"); + expect(result.kept.length).toBe(0); + }); + it("coerces string / number `sufficient` fields sent by lax models", async () => { const llm: any = { completeJson: vi.fn().mockResolvedValue({ diff --git a/apps/memos-local-plugin/tests/unit/server/http.test.ts b/apps/memos-local-plugin/tests/unit/server/http.test.ts index 3a8e70780..9e4c2a1f8 100644 --- a/apps/memos-local-plugin/tests/unit/server/http.test.ts +++ b/apps/memos-local-plugin/tests/unit/server/http.test.ts @@ -237,6 +237,13 @@ describe("HTTP server — REST routes", () => { expect(core.health).toHaveBeenCalled(); }); + it("strips /memos reverse-proxy prefix before route dispatch", async () => { + const r = await fetch(`${handle.url}/memos/api/v1/health`); + expect(r.status).toBe(200); + const body = await r.json(); + expect(body).toMatchObject({ ok: true, version: "test" }); + }); + it("GET /api/v1/health includes optional bridge status", async () => { await handle.close(); handle = await startHttpServer({ @@ -532,6 +539,66 @@ describe("HTTP server — REST routes", () => { }); }); + // Issue #1929 — when `core.patchConfig` rejects a body because of + // schema validation (Typebox `NumberInRange`, type mismatch, etc.) + // the route must surface that as 400 `invalid_argument`. The + // pre-fix behaviour was to let `MemosError("config_invalid", …)` + // bubble up to the global handler and return 500 `internal`, which + // tripped the rerun harness's + // `test_invalid_type_does_not_crash_or_corrupt` and + // `test_concurrent_patch_and_search_no_5xx` contracts. + it("PATCH /api/v1/config maps schema validation errors to 400", async () => { + const { MemosError } = await import("../../../agent-contract/errors.js"); + (core.patchConfig as ReturnType).mockRejectedValueOnce( + new MemosError("config_invalid", "config failed schema validation: bad"), + ); + const r = await fetch(`${handle.url}/api/v1/config`, { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + algorithm: { retrieval: { vectorScanMaxAgeMs: -1 } }, + }), + }); + expect(r.status).toBe(400); + const body = (await r.json()) as { error: { code: string; message: string } }; + expect(body.error.code).toBe("invalid_argument"); + expect(body.error.message).toMatch(/schema validation/); + }); + + // `config_write_failed` is raised only when the atomic config rename + // fails (disk full / permission denied) — a server-side I/O fault, not + // bad client input. It must surface as 500 so operators get paged and + // clients are not misled into thinking their (valid) payload was the + // problem. Only `config_invalid` (schema validation) maps to 400. + it("PATCH /api/v1/config keeps writer failures as 500 (server fault)", async () => { + const { MemosError } = await import("../../../agent-contract/errors.js"); + (core.patchConfig as ReturnType).mockRejectedValueOnce( + new MemosError("config_write_failed", "rename failed"), + ); + const r = await fetch(`${handle.url}/api/v1/config`, { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ viewer: { port: 19000 } }), + }); + expect(r.status).toBe(500); + const body = (await r.json()) as { error: { code: string } }; + expect(body.error.code).toBe("internal"); + }); + + it("PATCH /api/v1/config still 500s on unexpected errors", async () => { + (core.patchConfig as ReturnType).mockRejectedValueOnce( + new Error("boom"), + ); + const r = await fetch(`${handle.url}/api/v1/config`, { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ viewer: { port: 19000 } }), + }); + expect(r.status).toBe(500); + const body = (await r.json()) as { error: { code: string } }; + expect(body.error.code).toBe("internal"); + }); + it("GET /api/v1/export returns a JSON bundle", async () => { const r = await fetch(`${handle.url}/api/v1/export`); expect(r.status).toBe(200); diff --git a/apps/memos-local-plugin/tests/unit/startup-recovery.test.ts b/apps/memos-local-plugin/tests/unit/startup-recovery.test.ts new file mode 100644 index 000000000..0e8944644 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/startup-recovery.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const source = readFileSync( + join(__dirname, "../../core/pipeline/memory-core.ts"), + "utf8", +); + +function initBody(): string { + const start = source.indexOf(" async function init(): Promise {"); + expect(start, "init() function should be present").toBeGreaterThanOrEqual(0); + const end = source.indexOf("\n function", start + 1); + expect(end, "init() should be followed by another function").toBeGreaterThan(start); + return source.slice(start, end); +} + +function stripScheduledRecoveryCallbacks(body: string): string { + return body.replace( + /scheduleStartupRecovery\([\s\S]*?\n \}\);/g, + "scheduleStartupRecovery();", + ); +} + +describe("memory-core startup recovery", () => { + it("does not block init on stale/dirty episode recovery", () => { + const synchronousInitBody = stripScheduledRecoveryCallbacks(initBody()); + + expect(synchronousInitBody).not.toContain("await recoverOpenEpisodesAsSessionEnd(stale)"); + expect(synchronousInitBody).not.toContain("await recoverDirtyClosedEpisodes(dirtyClosed)"); + expect(initBody()).toContain("scheduleStartupRecovery(\"startup.open_recovery\""); + expect(initBody()).toContain("scheduleStartupRecovery(\"startup.dirty_closed_recovery\""); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts b/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts new file mode 100644 index 000000000..e34db1891 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts @@ -0,0 +1,374 @@ +/** + * SQL-only embedding maintenance stats. + * + * Regression + spec pin for issue #1929: `/api/v1/embeddings/maintenance` + * used to paginate every trace/policy/world_model/skill row through JS just + * to inspect vector byte lengths, hydrating hundreds of MB of BLOBs into the + * Node heap and blocking the event loop for minutes on production DBs. + * + * The new helper `embeddingMaintenanceCounts()` MUST count purely with SQL + * (`COUNT(*)` + `SUM(CASE WHEN ... LENGTH(vec) ...)`), preserving the two + * pre-fix semantic filters: + * - short-text traces are skipped (mirrors `shouldTraceHaveEmbeddings`) + * - `lightweight_memory`-tagged traces don't get counted for `vec_action` + */ + +import { describe, expect, it } from "vitest"; + +import { encodeVector } from "../../../core/storage/vector.js"; +import { + embeddingMaintenanceCounts, + inferStoredEmbeddingByteLen, +} from "../../../core/storage/repos/index.js"; +import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; +import type { + EpisodeId, + SessionId, + SkillId, + TraceId, + WorldModelId, +} from "../../../core/types.js"; + +const DIM = 4; +const EXPECTED_BYTE_LEN = DIM * 4; + +function fullVec(): Float32Array { + return new Float32Array([0.1, 0.2, 0.3, 0.4]); +} + +function shortVec(): Float32Array { + return new Float32Array([9]); +} + +function seedSessionAndEpisode(handle: TmpDbHandle): void { + handle.repos.sessions.upsert({ + id: "se" as SessionId, + agent: "openclaw", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: 1_700_000_000_000, + lastSeenAt: 1_700_000_000_000, + meta: {}, + }); + handle.repos.episodes.insert({ + id: "ep" as EpisodeId, + sessionId: "se" as SessionId, + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: 1_700_000_000_000, + endedAt: null, + traceIds: [], + rTask: null, + status: "open", + meta: {}, + }); +} + +function seedTrace( + handle: TmpDbHandle, + id: string, + opts: { + userText: string; + agentText: string; + tags?: string[]; + vecSummary?: Float32Array | null; + vecAction?: Float32Array | null; + }, +): void { + handle.repos.traces.insert({ + id: id as TraceId, + episodeId: "ep" as EpisodeId, + sessionId: "se" as SessionId, + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + ts: 1_700_000_000_000, + userText: opts.userText, + agentText: opts.agentText, + summary: "summary text", + share: null, + toolCalls: [], + agentThinking: null, + reflection: null, + value: 0, + alpha: 0, + rHuman: null, + priority: 0, + tags: opts.tags ?? [], + errorSignatures: [], + vecSummary: opts.vecSummary ?? null, + vecAction: opts.vecAction ?? null, + turnId: 1_700_000_000_000, + schemaVersion: 1, + } as never); +} + +function seedPolicy(handle: TmpDbHandle, id: string, vec: Float32Array | null): void { + handle.repos.policies.upsert({ + id: id as never, + title: id, + trigger: "", + procedure: "", + verification: "", + boundary: "", + support: 0, + gain: 0, + status: "candidate", + sourceEpisodeIds: [], + inducedBy: "proto", + decisionGuidance: { preference: [], antiPattern: [] }, + vec, + createdAt: 1, + updatedAt: 1, + }); +} + +function seedWorldModel( + handle: TmpDbHandle, + id: string, + vec: Float32Array | null, +): void { + handle.repos.worldModel.upsert({ + id: id as WorldModelId, + title: id, + body: "world body text", + structure: { environment: [], inference: [], constraints: [] }, + domainTags: [], + confidence: 0.9, + policyIds: [], + sourceEpisodeIds: [], + inducedBy: "", + vec, + createdAt: 1, + updatedAt: 1, + version: 1, + status: "active", + }); +} + +function seedSkill(handle: TmpDbHandle, id: string, vec: Float32Array | null): void { + handle.repos.skills.insert({ + id: id as SkillId, + name: id, + status: "candidate", + invocationGuide: "guide", + procedureJson: null, + eta: 0, + support: 0, + gain: 0, + trialsAttempted: 0, + trialsPassed: 0, + sourcePolicyIds: [], + sourceWorldModelIds: [], + evidenceAnchors: [], + vec, + createdAt: 1, + updatedAt: 1, + version: 1, + }); +} + +describe("storage/repos — embeddingMaintenanceCounts (issue #1929)", () => { + it("counts ready / missing / dimMismatch per kind without decoding BLOBs", () => { + const handle = makeTmpDb(); + try { + seedSessionAndEpisode(handle); + + // traces + // - tr_ready: qualifying, has both summary+action vectors at correct dim + seedTrace(handle, "tr_ready", { + userText: "hello world what is up", + agentText: "here is the answer", + vecSummary: fullVec(), + vecAction: fullVec(), + }); + // - tr_missing: qualifying, no vectors at all + seedTrace(handle, "tr_missing", { + userText: "hello world what is up", + agentText: "another answer here", + vecSummary: null, + vecAction: null, + }); + // - tr_dim_mismatch: qualifying, but vec dims wrong + seedTrace(handle, "tr_dim_mismatch", { + userText: "hello world what is up", + agentText: "yet another answer", + vecSummary: shortVec(), + vecAction: shortVec(), + }); + // - tr_short: NOT qualifying (both texts <10, sum <20) — should be excluded + seedTrace(handle, "tr_short", { + userText: "hi", + agentText: "ok", + vecSummary: null, + vecAction: null, + }); + // - tr_lightweight: qualifying for vec_summary but NOT vec_action + seedTrace(handle, "tr_lightweight", { + userText: "hello world what is up", + agentText: "the lightweight answer", + tags: ["lightweight_memory"], + vecSummary: fullVec(), + vecAction: null, + }); + + // policies + seedPolicy(handle, "po_ready", fullVec()); + seedPolicy(handle, "po_missing", null); + seedPolicy(handle, "po_dim", shortVec()); + + // world_model + seedWorldModel(handle, "wm_ready", fullVec()); + seedWorldModel(handle, "wm_missing", null); + + // skills + seedSkill(handle, "sk_ready", fullVec()); + seedSkill(handle, "sk_dim", shortVec()); + + const counts = embeddingMaintenanceCounts(handle.db, { + expectedByteLen: EXPECTED_BYTE_LEN, + }); + + // trace bucket: + // summary qualifying rows: tr_ready, tr_missing, tr_dim_mismatch, tr_lightweight = 4 + // action qualifying rows: tr_ready, tr_missing, tr_dim_mismatch = 3 + // totalSlots = 4 + 3 = 7 + // ready = tr_ready(summary+action) + tr_lightweight(summary) = 3 + // missing = tr_missing(summary+action) = 2 + // dimMismatch = tr_dim_mismatch(summary+action) = 2 + expect(counts.trace).toEqual({ + totalSlots: 7, + ready: 3, + missing: 2, + dimMismatch: 2, + }); + + expect(counts.policy).toEqual({ + totalSlots: 3, + ready: 1, + missing: 1, + dimMismatch: 1, + }); + + expect(counts.world_model).toEqual({ + totalSlots: 2, + ready: 1, + missing: 1, + dimMismatch: 0, + }); + + expect(counts.skill).toEqual({ + totalSlots: 2, + ready: 1, + missing: 0, + dimMismatch: 1, + }); + } finally { + handle.cleanup(); + } + }); + + it("falls back to 'any non-null = ready' when expectedByteLen is 0", () => { + const handle = makeTmpDb(); + try { + seedSessionAndEpisode(handle); + // A brand-new install with no embedder probe yet: dimension unknown. + // Any stored BLOB (short or full) should count as ready. + seedTrace(handle, "tr_full", { + userText: "hello world what is up", + agentText: "here is the answer", + vecSummary: fullVec(), + vecAction: shortVec(), + }); + seedTrace(handle, "tr_missing", { + userText: "hello world what is up", + agentText: "another answer here", + vecSummary: null, + vecAction: null, + }); + + const counts = embeddingMaintenanceCounts(handle.db, { expectedByteLen: 0 }); + + // Both slots for tr_full count as ready regardless of BLOB length. + expect(counts.trace.ready).toBe(2); + expect(counts.trace.dimMismatch).toBe(0); + expect(counts.trace.missing).toBe(2); + expect(counts.trace.totalSlots).toBe(4); + } finally { + handle.cleanup(); + } + }); + + it("returns zero counts for an empty database", () => { + const handle = makeTmpDb(); + try { + const counts = embeddingMaintenanceCounts(handle.db, { + expectedByteLen: EXPECTED_BYTE_LEN, + }); + expect(counts.trace).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + expect(counts.policy).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + expect(counts.world_model).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + expect(counts.skill).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + } finally { + handle.cleanup(); + } + }); + + it("infers stored byte length from the mode of trace vec_summary BLOBs", () => { + const handle = makeTmpDb(); + try { + seedSessionAndEpisode(handle); + // Three rows at 4-dim, one at 1-dim → mode = 16 bytes. + seedTrace(handle, "tr_a", { + userText: "hello world what is up", + agentText: "here is the answer", + vecSummary: fullVec(), + }); + seedTrace(handle, "tr_b", { + userText: "hello world what is up", + agentText: "another answer here", + vecSummary: fullVec(), + }); + seedTrace(handle, "tr_c", { + userText: "hello world what is up", + agentText: "yet another answer", + vecSummary: fullVec(), + }); + seedTrace(handle, "tr_odd", { + userText: "hello world what is up", + agentText: "the outlier answer", + vecSummary: shortVec(), + }); + + expect(inferStoredEmbeddingByteLen(handle.db)).toBe(EXPECTED_BYTE_LEN); + } finally { + handle.cleanup(); + } + }); + + it("uses BLOB byte length for dimension comparison", () => { + expect(encodeVector(fullVec()).byteLength).toBe(EXPECTED_BYTE_LEN); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts index accdbdc6d..3c859fc19 100644 --- a/apps/memos-local-plugin/tests/unit/storage/repos.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/repos.test.ts @@ -51,9 +51,43 @@ describe("storage/repos — happy paths", () => { expect(repos.episodes.getOpenForSession("s")!.id).toBe("e1"); + // appendTrace persists only IDs that have a backing row in `traces` + // (#1966 — ghost IDs in trace_ids_json would otherwise loop the reward + // dirty-check forever). Insert the rows first, then append. + for (const id of ["t1", "t2"] as const) { + repos.traces.insert({ + id, + episodeId: "e1", + sessionId: "s", + ts: 5, + userText: id, + agentText: "", + toolCalls: [], + reflection: null, + value: 0, + alpha: 0, + rHuman: null, + priority: 0, + tags: [], + vecSummary: null, + vecAction: null, + turnId: 0 as never, + schemaVersion: 1, + }); + } repos.episodes.appendTrace("e1", ["t1", "t2"]); expect(repos.episodes.getById("e1")!.traceIds).toEqual(["t1", "t2"]); + // Ghost IDs (no backing trace row) are silently stripped: this is the + // regression guard for the infinite rescore loop reported in #1966. + repos.episodes.appendTrace("e1", ["t1", "t2", "tr_ghost"]); + expect(repos.episodes.getById("e1")!.traceIds).toEqual(["t1", "t2"]); + // ...and a list made entirely of ghosts collapses to []: + repos.episodes.appendTrace("e1", ["tr_ghost_only"]); + expect(repos.episodes.getById("e1")!.traceIds).toEqual([]); + // Restore for the close assertion below. + repos.episodes.appendTrace("e1", ["t1", "t2"]); + repos.episodes.close("e1", 99, 0.8); const closed = repos.episodes.getById("e1")!; expect(closed.status).toBe("closed"); @@ -132,6 +166,27 @@ describe("storage/repos — happy paths", () => { expect(repos.traces.hasAnyNewerThan(["t2"], 29)).toBe(true); expect(repos.traces.hasAnyNewerThan([], 0)).toBe(false); expect(repos.traces.hasAnyNewerThan(["missing-id"], 0)).toBe(false); + + // countExisting / filterExistingIds — the cheap existence helpers + // introduced for the #1966 ghost-trace dirty-check fix. Mixed batches + // should return the count / list of just the IDs that exist; duplicate + // inputs are de-duplicated to match `getManyByIds(...).length` + // semantics. + expect(repos.traces.countExisting(["t0", "t1", "t2"])).toBe(3); + expect(repos.traces.countExisting(["t0", "ghost", "t2"])).toBe(2); + expect(repos.traces.countExisting(["ghost-a", "ghost-b"])).toBe(0); + expect(repos.traces.countExisting(["t0", "t0", "t1"])).toBe(2); + expect(repos.traces.countExisting([])).toBe(0); + + expect(repos.traces.filterExistingIds(["t0", "ghost", "t2"])).toEqual([ + "t0", + "t2", + ]); + expect(repos.traces.filterExistingIds(["t0", "t0", "t1"])).toEqual([ + "t0", + "t1", + ]); + expect(repos.traces.filterExistingIds([])).toEqual([]); } finally { cleanup(); } diff --git a/apps/memos-local-plugin/tests/unit/viewer/api-client.test.ts b/apps/memos-local-plugin/tests/unit/viewer/api-client.test.ts index 7e9cd00c8..67b60aa7c 100644 --- a/apps/memos-local-plugin/tests/unit/viewer/api-client.test.ts +++ b/apps/memos-local-plugin/tests/unit/viewer/api-client.test.ts @@ -5,7 +5,7 @@ * error shape, and the three verb helpers. */ -import { describe, it, expect, afterEach } from "vitest"; +import { describe, it, expect, afterEach, vi } from "vitest"; (globalThis as any).localStorage = { _s: new Map(), @@ -42,6 +42,30 @@ describe("api (viewer REST client)", () => { expect((received?.headers as any)["content-type"]).toBe("application/json"); }); + it("prefixes API paths when the viewer is mounted under /memos", async () => { + const originalLocation = globalThis.location; + vi.resetModules(); + Object.defineProperty(globalThis, "location", { + configurable: true, + value: { pathname: "/memos/" }, + }); + try { + const client = await import("../../../viewer/src/api/client"); + expect(client.AGENT_PREFIX).toBe("/memos"); + expect(client.withAgentPrefix("/api/v1/health")).toBe("/memos/api/v1/health"); + expect(client.withAgentPrefix("api/v1/health")).toBe("/memos/api/v1/health"); + expect(client.withAgentPrefix("https://example.test/api")).toBe( + "https://example.test/api", + ); + } finally { + Object.defineProperty(globalThis, "location", { + configurable: true, + value: originalLocation, + }); + vi.resetModules(); + } + }); + it("api.post includes JSON body and api-key header when set", async () => { (globalThis as any).localStorage.setItem("memos.apiKey", "hunter2"); let received: RequestInit | undefined; diff --git a/apps/memos-local-plugin/tsconfig.build.json b/apps/memos-local-plugin/tsconfig.build.json index afa7a221e..c74e16bc4 100644 --- a/apps/memos-local-plugin/tsconfig.build.json +++ b/apps/memos-local-plugin/tsconfig.build.json @@ -7,7 +7,8 @@ "bridge/**/*.ts", "adapters/openclaw/**/*.ts", "scripts/**/*.ts", - "bridge.cts" + "bridge.cts", + "bridge.mts" ], "exclude": [ "node_modules", diff --git a/apps/memos-local-plugin/tsconfig.json b/apps/memos-local-plugin/tsconfig.json index 8c58d9e92..722cbb490 100644 --- a/apps/memos-local-plugin/tsconfig.json +++ b/apps/memos-local-plugin/tsconfig.json @@ -30,7 +30,8 @@ "bridge/**/*.ts", "adapters/openclaw/**/*.ts", "scripts/**/*.ts", - "bridge.cts" + "bridge.cts", + "bridge.mts" ], "exclude": [ "node_modules", diff --git a/apps/memos-local-plugin/viewer/src/api/client.ts b/apps/memos-local-plugin/viewer/src/api/client.ts index e9ac5510e..0706d6c1f 100644 --- a/apps/memos-local-plugin/viewer/src/api/client.ts +++ b/apps/memos-local-plugin/viewer/src/api/client.ts @@ -13,19 +13,28 @@ const DEFAULT_HEADERS: Record = { }; /** - * Historical no-op: the viewer used to be served under - * `/openclaw/...` / `/hermes/...` prefixes when both agents shared a - * single port. Each agent now owns its own well-known port and the - * SPA is mounted at root, so the prefix is always empty. Kept as an - * exported constant so older code paths and tests don't break. + * Optional path prefix for legacy single-port installs and reverse + * proxies. New installs mount the SPA at root, but old bookmarks and + * deployments such as `/memos/` still need API calls to retain the + * leading prefix. */ -export const AGENT_PREFIX: string = ""; +export const AGENT_PREFIX: string = detectAgentPrefix(); + +function detectAgentPrefix(): string { + if (typeof location === "undefined") return ""; + const seg = location.pathname.split("/").filter(Boolean)[0]; + return seg === "openclaw" || seg === "hermes" || seg === "memos" ? `/${seg}` : ""; +} /** - * No-op pass-through. See `AGENT_PREFIX` above for context. + * Prefix viewer API paths when the SPA itself is served from an agent + * prefix. Absolute external URLs are left untouched. */ export function withAgentPrefix(path: string): string { - return path; + if (!AGENT_PREFIX) return path; + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(path)) return path; + const normalized = path.startsWith("/") ? path : `/${path}`; + return `${AGENT_PREFIX}${normalized}`; } function apiKeyHeader(): Record { diff --git a/apps/memos-local-plugin/viewer/src/views/LogsView.tsx b/apps/memos-local-plugin/viewer/src/views/LogsView.tsx index abac410ed..659bdf3cf 100644 --- a/apps/memos-local-plugin/viewer/src/views/LogsView.tsx +++ b/apps/memos-local-plugin/viewer/src/views/LogsView.tsx @@ -725,6 +725,7 @@ function RetrievalFunnel({ stats }: { stats: RetrievalStatsPayload }) { const localFilterDeferred = outcome === "deferred_to_final"; const finalLlmRan = finalFilter?.outcome === "llm_kept_all" || finalFilter?.outcome === "llm_filtered" || + finalFilter?.outcome === "llm_filtered_refilled" || finalFilter?.outcome === "llm_failed_safe_cutoff"; const fmtNum = (n: number | undefined, digits = 3) => typeof n === "number" && Number.isFinite(n) ? n.toFixed(digits) : "—"; diff --git a/poetry.lock b/poetry.lock index dfea31354..b09a72d70 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6648,4 +6648,4 @@ tree-mem = ["neo4j", "schedule"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "86c718082278e9df7b994b87eb4b75c64ee4b5353954bf16dbec371d8264ce0a" +content-hash = "cfab703c19f5f87caf537b68d30174695c44c6fc228e42c2c8e71b00b81f5790" diff --git a/pyproject.toml b/pyproject.toml index 5d65422d3..66e1e9236 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ ############################################################################## name = "MemoryOS" -version = "2.0.20" +version = "2.0.22" description = "Intelligence Begins with Memory" license = {text = "Apache-2.0"} readme = "README.md" @@ -89,6 +89,8 @@ mem-user = [ # MemReader mem-reader = [ + # chonkie>=1.6 pulls numpy>=2.0; any environment that also has scipy + # installed must use scipy>=1.13 (older scipy pins numpy<1.27). See #1342. "chonkie (>=1.0.7,<2.0.0)", # Sentence chunking library "markitdown[docx,pdf,pptx,xls,xlsx] (>=0.1.1,<0.2.0)", # Markdown parser for various file formats "langchain-text-splitters (>=1.0.0,<2.0.0)", # markdown chunk for langchain @@ -193,7 +195,9 @@ zep-cloud = "^2.15.0" rouge-score = "^0.1.2" nltk = "^3.9.1" bert-score = "^0.3.13" -scipy = "^1.10.1" +# scipy 1.13+ supports numpy 2.x. Older scipy (1.10/1.11/1.12) pins numpy<1.27 +# which collides with chonkie>=1.6 (requires numpy>=2.0.0) — see issue #1342. +scipy = "^1.13.0" python-dotenv = "^1.1.1" langgraph = "^0.5.1" diff --git a/src/memos/__init__.py b/src/memos/__init__.py index 875152b39..86df25d26 100644 --- a/src/memos/__init__.py +++ b/src/memos/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.20" +__version__ = "2.0.22" from memos.configs.mem_cube import GeneralMemCubeConfig from memos.configs.mem_os import MOSConfig diff --git a/src/memos/api/config.py b/src/memos/api/config.py index 69efedeb3..d5c07dcbe 100644 --- a/src/memos/api/config.py +++ b/src/memos/api/config.py @@ -979,6 +979,10 @@ def get_product_default_config() -> dict[str, Any]: "general_llm": APIConfig.get_memreader_general_llm_config(), # Image parser LLM (requires vision model) "image_parser_llm": APIConfig.get_image_parser_llm_config(), + # Preference extractor LLM. Reader falls back to general_llm when unset. + "preference_extractor_llm": APIConfig.get_preference_extractor_llm_config() + if os.getenv("PREFERENCE_EXTRACTOR_MODEL") + else None, "embedder": APIConfig.get_embedder_config(), "chunker": { "backend": "sentence", @@ -1108,6 +1112,10 @@ def create_user_config(user_name: str, user_id: str) -> tuple["MOSConfig", "Gene "general_llm": APIConfig.get_memreader_general_llm_config(), # Image parser LLM (requires vision model) "image_parser_llm": APIConfig.get_image_parser_llm_config(), + # Preference extractor LLM. Reader falls back to general_llm when unset. + "preference_extractor_llm": APIConfig.get_preference_extractor_llm_config() + if os.getenv("PREFERENCE_EXTRACTOR_MODEL") + else None, "embedder": APIConfig.get_embedder_config(), "chunker": { "backend": "sentence", diff --git a/src/memos/api/handlers/chat_handler.py b/src/memos/api/handlers/chat_handler.py index 68dcfe6fb..5c84d29f5 100644 --- a/src/memos/api/handlers/chat_handler.py +++ b/src/memos/api/handlers/chat_handler.py @@ -1225,7 +1225,13 @@ def _get_further_suggestion( [f"{msg['role']}: {msg['content']}" for msg in current_messages[-2:]] ) further_suggestion_prompt = FURTHER_SUGGESTION_PROMPT.format(dialogue=dialogue_info) - message_list = [{"role": "system", "content": further_suggestion_prompt}] + message_list = [ + { + "role": "system", + "content": "You are a helpful assistant that generates suggestion queries based on dialogue context.", + }, + {"role": "user", "content": further_suggestion_prompt}, + ] response = self.llm.generate(message_list) clean_response = clean_json_response(response) response_json = json.loads(clean_response) diff --git a/src/memos/api/handlers/scheduler_handler.py b/src/memos/api/handlers/scheduler_handler.py index e7b756a1f..61d0d0b99 100644 --- a/src/memos/api/handlers/scheduler_handler.py +++ b/src/memos/api/handlers/scheduler_handler.py @@ -146,18 +146,50 @@ def _aggregate_counts_from_redis( sched_failed = all_tasks_summary.failed sched_cancelled = all_tasks_summary.cancelled - # If queue monitor is available, prefer its live waiting/in_progress counts + # If queue monitor is available, prefer its live waiting/in_progress counts. + # + # Two queue backends produce different ``queue_status_data`` shapes: + # - Redis queue: ``{"running": .., "remaining": .., "pending": .., + # ":::