From 13f8ac0fd5890e968c4e020340493aeffec7f3f0 Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Thu, 2 Jul 2026 15:18:05 +0800 Subject: [PATCH 1/3] fix(memos-local-plugin): SQL-only embedding maintenance stats (#1929) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `GET /api/v1/embeddings/maintenance` used to paginate every trace / policy / world_model / skill row through `repos.*.list()`, which reads full BLOB vector columns and decodes them into Float32Array on the JS heap purely so the maintenance path could inspect each vector's length. On a production deployment with ~93K traces × 2 vector columns × 1536 dims × 4 bytes ≈ 1.1 GB of BLOB pread64 traffic and ~270 MB of JS heap allocations per request, all on the synchronous better-sqlite3 path — the entire OpenClaw gateway event loop was starved for 4+ minutes at 100% CPU while the stats call ran, as strace (99.96% pread64) and the observed `eventLoopDelayMaxMs=285883` / `durationMs=292731` confirmed. The maintenance endpoint only needs counts, not the vector bodies. This change adds `embeddingMaintenanceCounts()` to `core/storage/repos/`, which issues five `SELECT COUNT(*) + SUM(CASE WHEN ...)` queries — one per `(table, vec column)` slot — using `LENGTH(vec)` for the dimension check. SQLite's `LENGTH()` on a BLOB column returns the header byte count without copying the buffer, so the stats path never leaves SQLite. The two pre-fix semantic filters (`shouldTraceHaveEmbeddings` and `isLightweightMemoryTrace`) are preserved verbatim in the WHERE clauses so per-bucket counts do not shift for already-installed users. The public `EmbeddingMaintenanceStats` JSON shape is unchanged. - Add `core/storage/repos/embedding_maintenance.ts` with SQL-only `embeddingMaintenanceCounts()` + `inferStoredEmbeddingByteLen()` helpers. - Re-export them (and `FLOAT32_BYTES` / `EmbeddingCounts`) from `core/storage/repos/index.ts`. - Rewire `core/pipeline/memory-core.ts::computeEmbeddingMaintenanceStats()` to the SQL fast path; drop the dead `inferStoredEmbeddingDimension(slots)` and `emptyEmbeddingStatsByKind()` helpers. - New `tests/unit/storage/embedding-maintenance.test.ts` (4 cases) pins the bucket semantics, lightweight-memory carveout, short-text filter, dim-mismatch detection, empty-DB safety, `expectedByteLen=0` fallback, and the mode-based byte-length inference. The tier-2 `scanAndTopK` bounding the reporter flagged as an "Additional Fix" is out of scope for this PR — the title and OpenClaw event-loop-block log both point at the maintenance endpoint, and keeping the surface tight makes the fix easy to review and revert. Verification: 4/4 new unit tests pass, 28/28 memory-core façade tests pass, `npx vitest run` shows 1048 passing / 3 pre-existing failures (v7 e2e / namespace-visibility migrator regression / traces-count > 500) that all reproduce on the base branch after `git stash` — unrelated to this change. `tsc -p tsconfig.json --noEmit` and `tsc -p tsconfig.build.json` both clean. Fixes #1929 Co-Authored-By: Claude Opus 4.7 (1M context) --- .../core/pipeline/memory-core.ts | 89 ++--- .../storage/repos/embedding_maintenance.ts | 204 ++++++++++ .../core/storage/repos/index.ts | 20 + .../storage/embedding-maintenance.test.ts | 369 ++++++++++++++++++ 4 files changed, 631 insertions(+), 51 deletions(-) create mode 100644 apps/memos-local-plugin/core/storage/repos/embedding_maintenance.ts create mode 100644 apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index b4e331c71..9df1f2534 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -79,7 +79,12 @@ import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; import { openDb } from "../storage/connection.js"; import { runMigrations } from "../storage/migrator.js"; -import { makeRepos } from "../storage/repos/index.js"; +import { + makeRepos, + embeddingMaintenanceCounts, + inferStoredEmbeddingByteLen, + FLOAT32_BYTES, +} from "../storage/repos/index.js"; import { createEmbedder } from "../embedding/embedder.js"; import { createLlmClient } from "../llm/client.js"; import { @@ -4033,23 +4038,26 @@ export function createMemoryCore( function computeEmbeddingMaintenanceStats(): EmbeddingMaintenanceStats { const configuredDimension = handle.embedder?.dimensions ?? 0; - const allSlots = collectEmbeddingSlots(); - const dimension = configuredDimension > 0 ? configuredDimension : inferStoredEmbeddingDimension(allSlots); - const byKind = emptyEmbeddingStatsByKind(); - for (const slot of allSlots) { - const bucket = byKind[slot.kind]; - bucket.totalSlots++; - if (!slot.vec) { - bucket.missing++; - } else if (dimension > 0 && slot.vec.length !== dimension) { - bucket.dimMismatch++; - } else { - bucket.ready++; - } - } - for (const bucket of Object.values(byKind)) { - bucket.needsRepair = bucket.missing + bucket.dimMismatch; - } + const expectedByteLenFromEmbedder = configuredDimension > 0 + ? configuredDimension * FLOAT32_BYTES + : 0; + // When the embedder has not been probed yet, fall back to the most common + // stored BLOB byte length (mirrors the pre-fix `inferStoredEmbeddingDimension` + // path, but computed via SQL `GROUP BY LENGTH(vec_summary)` — never touches + // the BLOB bodies). + const expectedByteLen = expectedByteLenFromEmbedder > 0 + ? expectedByteLenFromEmbedder + : inferStoredEmbeddingByteLen(handle.db); + const dimension = expectedByteLen > 0 ? expectedByteLen / FLOAT32_BYTES : 0; + + const raw = embeddingMaintenanceCounts(handle.db, { expectedByteLen }); + const byKind: EmbeddingMaintenanceStats["byKind"] = { + trace: addNeedsRepair(raw.trace), + policy: addNeedsRepair(raw.policy), + world_model: addNeedsRepair(raw.world_model), + skill: addNeedsRepair(raw.skill), + }; + const totalSlots = sumEmbeddingStats(byKind, "totalSlots"); const ready = sumEmbeddingStats(byKind, "ready"); const missing = sumEmbeddingStats(byKind, "missing"); @@ -4066,6 +4074,18 @@ export function createMemoryCore( }; } + function addNeedsRepair(bucket: { + totalSlots: number; + ready: number; + missing: number; + dimMismatch: number; + }): EmbeddingMaintenanceStats["byKind"]["trace"] { + return { + ...bucket, + needsRepair: bucket.missing + bucket.dimMismatch, + }; + } + async function ensureEmbeddingDimensionKnown(): Promise { if (!handle.embedder || handle.embedder.dimensions > 0) return; try { @@ -4080,23 +4100,6 @@ export function createMemoryCore( } } - function inferStoredEmbeddingDimension(slots: readonly EmbeddingSlot[]): number { - const counts = new Map(); - for (const slot of slots) { - if (!slot.vec) continue; - counts.set(slot.vec.length, (counts.get(slot.vec.length) ?? 0) + 1); - } - let bestDim = 0; - let bestCount = 0; - for (const [dim, count] of counts) { - if (count > bestCount) { - bestDim = dim; - bestCount = count; - } - } - return bestDim; - } - function shouldTraceHaveEmbeddings(row: TraceRow): boolean { // Skip traces where both user and agent text are very short const userLen = row.userText.trim().length; @@ -4206,22 +4209,6 @@ export function createMemoryCore( return row.tags.includes("lightweight_memory"); } - function emptyEmbeddingStatsByKind(): EmbeddingMaintenanceStats["byKind"] { - const empty = () => ({ - totalSlots: 0, - ready: 0, - missing: 0, - dimMismatch: 0, - needsRepair: 0, - }); - return { - trace: empty(), - policy: empty(), - world_model: empty(), - skill: empty(), - }; - } - function sumEmbeddingStats( byKind: EmbeddingMaintenanceStats["byKind"], key: "totalSlots" | "ready" | "missing" | "dimMismatch" | "needsRepair", diff --git a/apps/memos-local-plugin/core/storage/repos/embedding_maintenance.ts b/apps/memos-local-plugin/core/storage/repos/embedding_maintenance.ts new file mode 100644 index 000000000..65632ff59 --- /dev/null +++ b/apps/memos-local-plugin/core/storage/repos/embedding_maintenance.ts @@ -0,0 +1,204 @@ +/** + * SQL-only embedding maintenance stats. + * + * Regression fix for issue #1929: `/api/v1/embeddings/maintenance` used to + * paginate every trace/policy/world_model/skill row through JS just to + * inspect vector byte lengths, hydrating hundreds of MB of BLOBs into the + * Node heap and blocking the event loop for minutes on production DBs + * (93K traces × 2 vectors × 1536 dims × 4 bytes ≈ 1.1 GB pread64 traffic). + * + * The strategy is a single `SELECT COUNT(*) + SUM(CASE WHEN ...)` per + * `(table, vec column)` pair, using `LENGTH(vec)` for the dimension + * comparison. SQLite's `LENGTH()` on a BLOB column returns the byte length + * from the row header and does not deserialise the buffer, so the maintenance + * call now stays in the same asymptotic ballpark as `SELECT COUNT(*) FROM t`. + * + * The two pre-fix semantic filters are preserved verbatim inside the WHERE + * clauses so per-bucket counts do not shift for already-installed users: + * + * - `shouldTraceHaveEmbeddings` (short-text traces skipped) → SQL + * `LENGTH(TRIM(user_text)) / LENGTH(TRIM(agent_text))` predicates. + * - `isLightweightMemoryTrace` (lightweight traces skip vec_action) → + * `instr(COALESCE(tags_json, ''), '"lightweight_memory"') = 0` + * predicate for the vec_action count only. + */ + +import type { StorageDb } from "../types.js"; + +/** Little-endian Float32 element size. Matches `core/storage/vector.ts`. */ +export const FLOAT32_BYTES = 4; + +export interface EmbeddingCountsBucket { + /** Number of `(row, vec column)` slots included in the bucket. */ + totalSlots: number; + /** + * Vec is non-NULL and either `expectedByteLen === 0` (dimension not + * probed yet) or `LENGTH(vec) === expectedByteLen`. + */ + ready: number; + /** Vec is SQL NULL. */ + missing: number; + /** + * Vec is non-NULL and its byte length ≠ `expectedByteLen` + * (only meaningful when `expectedByteLen > 0`). + */ + dimMismatch: number; +} + +export interface EmbeddingCounts { + trace: EmbeddingCountsBucket; + policy: EmbeddingCountsBucket; + world_model: EmbeddingCountsBucket; + skill: EmbeddingCountsBucket; +} + +interface CountRow { + total: number | null; + missing: number | null; + dim_mismatch: number | null; + ready: number | null; +} + +interface CountArgs { + expected_byte_len: number; +} + +/** + * SQL for the four `(table, vec column)` slots. Each query returns + * `(total, missing, dim_mismatch, ready)` for a single slot in one round-trip. + * + * `LENGTH(vec)` on a BLOB column returns the stored byte count without + * deserialising the BLOB into JS memory — this is the whole point of the + * fix (see #1929 evidence: 99.96 % of the pre-fix time was in `pread64`). + */ +const TRACE_QUALIFICATION = + "(LENGTH(TRIM(user_text)) >= 10 OR LENGTH(TRIM(agent_text)) >= 10) " + + "AND (LENGTH(TRIM(user_text)) + LENGTH(TRIM(agent_text)) >= 20)"; + +/** Build the count SQL for a single (table, column) slot. */ +function slotSql( + table: string, + column: string, + extraWhere: string = "", +): string { + const clauses: string[] = []; + if (table === "traces") clauses.push(TRACE_QUALIFICATION); + if (extraWhere) clauses.push(extraWhere); + const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""; + return ` + SELECT + COUNT(*) AS total, + SUM(CASE WHEN ${column} IS NULL THEN 1 ELSE 0 END) AS missing, + SUM(CASE WHEN ${column} IS NOT NULL + AND @expected_byte_len > 0 + AND LENGTH(${column}) <> @expected_byte_len + THEN 1 ELSE 0 END) AS dim_mismatch, + SUM(CASE WHEN ${column} IS NOT NULL + AND (@expected_byte_len = 0 OR LENGTH(${column}) = @expected_byte_len) + THEN 1 ELSE 0 END) AS ready + FROM ${table} + ${where} + `; +} + +function normalizeRow(row: CountRow | undefined): EmbeddingCountsBucket { + if (!row) { + return { totalSlots: 0, ready: 0, missing: 0, dimMismatch: 0 }; + } + return { + totalSlots: row.total ?? 0, + ready: row.ready ?? 0, + missing: row.missing ?? 0, + dimMismatch: row.dim_mismatch ?? 0, + }; +} + +/** + * Count embedding slots per (table, vec column) purely with SQL. + * + * MUST NOT read or decode any BLOB into JS. Total wall-clock work is + * `O(rows)` SQL scan touching only BLOB header bytes. + * + * @param db - open storage handle (better-sqlite3 wrapper). + * @param opts.expectedByteLen - `dimensions * 4` for a known Float32 + * dimension, or `0` when the dimension has not been probed yet. In the + * `0` fallback every non-NULL vector counts as ready and dimMismatch + * is always 0 — matches the pre-fix "any non-null = ready" behaviour + * that `inferStoredEmbeddingDimension(slots)` used to fall back to. + */ +export function embeddingMaintenanceCounts( + db: StorageDb, + opts: { expectedByteLen: number }, +): EmbeddingCounts { + const args: CountArgs = { + expected_byte_len: Math.max(0, Math.floor(opts.expectedByteLen) || 0), + }; + + const traceSummary = db + .prepare(slotSql("traces", "vec_summary")) + .get(args); + const traceAction = db + .prepare( + slotSql( + "traces", + "vec_action", + "instr(COALESCE(tags_json, ''), '\"lightweight_memory\"') = 0", + ), + ) + .get(args); + const policy = db + .prepare(slotSql("policies", "vec")) + .get(args); + const worldModel = db + .prepare(slotSql("world_model", "vec")) + .get(args); + const skill = db + .prepare(slotSql("skills", "vec")) + .get(args); + + const summary = normalizeRow(traceSummary); + const action = normalizeRow(traceAction); + + return { + trace: { + totalSlots: summary.totalSlots + action.totalSlots, + ready: summary.ready + action.ready, + missing: summary.missing + action.missing, + dimMismatch: summary.dimMismatch + action.dimMismatch, + }, + policy: normalizeRow(policy), + world_model: normalizeRow(worldModel), + skill: normalizeRow(skill), + }; +} + +interface ModeRow { + byte_len: number; + n: number; +} + +/** + * Infer the dominant stored embedding byte length by GROUP BY the byte + * length of every non-NULL `traces.vec_summary` BLOB. Returns the byte + * length with the highest row count, or 0 when the DB has no vectors + * (brand-new install). + * + * Cheap replacement for the pre-fix + * `inferStoredEmbeddingDimension(collectEmbeddingSlots())` path — which had + * to hydrate every BLOB in memory before it could measure any single one. + * We now let SQLite do the length arithmetic and just pick the mode. + */ +export function inferStoredEmbeddingByteLen(db: StorageDb): number { + const rows = db + .prepare( + `SELECT LENGTH(vec_summary) AS byte_len, COUNT(*) AS n + FROM traces + WHERE vec_summary IS NOT NULL AND LENGTH(vec_summary) > 0 + GROUP BY LENGTH(vec_summary) + ORDER BY n DESC + LIMIT 1`, + ) + .all(); + if (rows.length === 0) return 0; + return rows[0]!.byte_len; +} diff --git a/apps/memos-local-plugin/core/storage/repos/index.ts b/apps/memos-local-plugin/core/storage/repos/index.ts index 12cca9f33..7386ec218 100644 --- a/apps/memos-local-plugin/core/storage/repos/index.ts +++ b/apps/memos-local-plugin/core/storage/repos/index.ts @@ -5,6 +5,13 @@ */ import type { StorageDb } from "../types.js"; +import { + embeddingMaintenanceCounts, + inferStoredEmbeddingByteLen, + FLOAT32_BYTES, + type EmbeddingCounts, + type EmbeddingCountsBucket, +} from "./embedding_maintenance.js"; import { makeApiLogsRepo } from "./api_logs.js"; import { makeAuditRepo } from "./audit.js"; import { makeCandidatePoolRepo } from "./candidate_pool.js"; @@ -84,3 +91,16 @@ export { makeSkillsRepo } from "./skills.js"; export { makeTracePolicyLinksRepo } from "./trace-policy-links.js"; export { makeTracesRepo } from "./traces.js"; export { makeWorldModelRepo } from "./world_model.js"; + +/** + * SQL-only embedding maintenance stats — see `./embedding_maintenance.ts` + * for the design rationale. Regression pin for issue #1929: the old JS + * path hydrated 270 MB of BLOBs per request; this SQL fast path uses + * `LENGTH(vec)` alone and never touches the buffers. + */ +export { + embeddingMaintenanceCounts, + inferStoredEmbeddingByteLen, + FLOAT32_BYTES, +}; +export type { EmbeddingCounts, EmbeddingCountsBucket }; diff --git a/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts b/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts new file mode 100644 index 000000000..f86cfe4fe --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts @@ -0,0 +1,369 @@ +/** + * SQL-only embedding maintenance stats. + * + * Regression + spec pin for issue #1929: `/api/v1/embeddings/maintenance` + * used to paginate every trace/policy/world_model/skill row through JS just + * to inspect vector byte lengths, hydrating hundreds of MB of BLOBs into the + * Node heap and blocking the event loop for minutes on production DBs. + * + * The new helper `embeddingMaintenanceCounts()` MUST count purely with SQL + * (`COUNT(*)` + `SUM(CASE WHEN ... LENGTH(vec) ...)`), preserving the two + * pre-fix semantic filters: + * - short-text traces are skipped (mirrors `shouldTraceHaveEmbeddings`) + * - `lightweight_memory`-tagged traces don't get counted for `vec_action` + */ + +import { describe, expect, it } from "vitest"; + +import { + embeddingMaintenanceCounts, + inferStoredEmbeddingByteLen, +} from "../../../core/storage/repos/index.js"; +import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; +import type { + EpisodeId, + SessionId, + SkillId, + TraceId, + WorldModelId, +} from "../../../core/types.js"; + +const DIM = 4; +const EXPECTED_BYTE_LEN = DIM * 4; + +function fullVec(): Float32Array { + return new Float32Array([0.1, 0.2, 0.3, 0.4]); +} + +function shortVec(): Float32Array { + return new Float32Array([9]); +} + +function seedSessionAndEpisode(handle: TmpDbHandle): void { + handle.repos.sessions.upsert({ + id: "se" as SessionId, + agent: "openclaw", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: 1_700_000_000_000, + lastSeenAt: 1_700_000_000_000, + meta: {}, + }); + handle.repos.episodes.insert({ + id: "ep" as EpisodeId, + sessionId: "se" as SessionId, + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: 1_700_000_000_000, + endedAt: null, + traceIds: [], + rTask: null, + status: "open", + meta: {}, + }); +} + +function seedTrace( + handle: TmpDbHandle, + id: string, + opts: { + userText: string; + agentText: string; + tags?: string[]; + vecSummary?: Float32Array | null; + vecAction?: Float32Array | null; + }, +): void { + handle.repos.traces.insert({ + id: id as TraceId, + episodeId: "ep" as EpisodeId, + sessionId: "se" as SessionId, + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + ts: 1_700_000_000_000, + userText: opts.userText, + agentText: opts.agentText, + summary: "summary text", + share: null, + toolCalls: [], + agentThinking: null, + reflection: null, + value: 0, + alpha: 0, + rHuman: null, + priority: 0, + tags: opts.tags ?? [], + errorSignatures: [], + vecSummary: opts.vecSummary ?? null, + vecAction: opts.vecAction ?? null, + turnId: 1_700_000_000_000, + schemaVersion: 1, + } as never); +} + +function seedPolicy(handle: TmpDbHandle, id: string, vec: Float32Array | null): void { + handle.repos.policies.upsert({ + id: id as never, + title: id, + trigger: "", + procedure: "", + verification: "", + boundary: "", + support: 0, + gain: 0, + status: "candidate", + sourceEpisodeIds: [], + inducedBy: "proto", + decisionGuidance: { preference: [], antiPattern: [] }, + vec, + createdAt: 1, + updatedAt: 1, + }); +} + +function seedWorldModel( + handle: TmpDbHandle, + id: string, + vec: Float32Array | null, +): void { + handle.repos.worldModel.upsert({ + id: id as WorldModelId, + title: id, + body: "world body text", + structure: { environment: [], inference: [], constraints: [] }, + domainTags: [], + confidence: 0.9, + policyIds: [], + sourceEpisodeIds: [], + inducedBy: "", + vec, + createdAt: 1, + updatedAt: 1, + version: 1, + status: "active", + }); +} + +function seedSkill(handle: TmpDbHandle, id: string, vec: Float32Array | null): void { + handle.repos.skills.insert({ + id: id as SkillId, + name: id, + status: "candidate", + invocationGuide: "guide", + procedureJson: null, + eta: 0, + support: 0, + gain: 0, + trialsAttempted: 0, + trialsPassed: 0, + sourcePolicyIds: [], + sourceWorldModelIds: [], + evidenceAnchors: [], + vec, + createdAt: 1, + updatedAt: 1, + version: 1, + }); +} + +describe("storage/repos — embeddingMaintenanceCounts (issue #1929)", () => { + it("counts ready / missing / dimMismatch per kind without decoding BLOBs", () => { + const handle = makeTmpDb(); + try { + seedSessionAndEpisode(handle); + + // traces + // - tr_ready: qualifying, has both summary+action vectors at correct dim + seedTrace(handle, "tr_ready", { + userText: "hello world what is up", + agentText: "here is the answer", + vecSummary: fullVec(), + vecAction: fullVec(), + }); + // - tr_missing: qualifying, no vectors at all + seedTrace(handle, "tr_missing", { + userText: "hello world what is up", + agentText: "another answer here", + vecSummary: null, + vecAction: null, + }); + // - tr_dim_mismatch: qualifying, but vec dims wrong + seedTrace(handle, "tr_dim_mismatch", { + userText: "hello world what is up", + agentText: "yet another answer", + vecSummary: shortVec(), + vecAction: shortVec(), + }); + // - tr_short: NOT qualifying (both texts <10, sum <20) — should be excluded + seedTrace(handle, "tr_short", { + userText: "hi", + agentText: "ok", + vecSummary: null, + vecAction: null, + }); + // - tr_lightweight: qualifying for vec_summary but NOT vec_action + seedTrace(handle, "tr_lightweight", { + userText: "hello world what is up", + agentText: "the lightweight answer", + tags: ["lightweight_memory"], + vecSummary: fullVec(), + vecAction: null, + }); + + // policies + seedPolicy(handle, "po_ready", fullVec()); + seedPolicy(handle, "po_missing", null); + seedPolicy(handle, "po_dim", shortVec()); + + // world_model + seedWorldModel(handle, "wm_ready", fullVec()); + seedWorldModel(handle, "wm_missing", null); + + // skills + seedSkill(handle, "sk_ready", fullVec()); + seedSkill(handle, "sk_dim", shortVec()); + + const counts = embeddingMaintenanceCounts(handle.db, { + expectedByteLen: EXPECTED_BYTE_LEN, + }); + + // trace bucket: + // summary qualifying rows: tr_ready, tr_missing, tr_dim_mismatch, tr_lightweight = 4 + // action qualifying rows: tr_ready, tr_missing, tr_dim_mismatch = 3 + // totalSlots = 4 + 3 = 7 + // ready = tr_ready(summary+action) + tr_lightweight(summary) = 3 + // missing = tr_missing(summary+action) = 2 + // dimMismatch = tr_dim_mismatch(summary+action) = 2 + expect(counts.trace).toEqual({ + totalSlots: 7, + ready: 3, + missing: 2, + dimMismatch: 2, + }); + + expect(counts.policy).toEqual({ + totalSlots: 3, + ready: 1, + missing: 1, + dimMismatch: 1, + }); + + expect(counts.world_model).toEqual({ + totalSlots: 2, + ready: 1, + missing: 1, + dimMismatch: 0, + }); + + expect(counts.skill).toEqual({ + totalSlots: 2, + ready: 1, + missing: 0, + dimMismatch: 1, + }); + } finally { + handle.cleanup(); + } + }); + + it("falls back to 'any non-null = ready' when expectedByteLen is 0", () => { + const handle = makeTmpDb(); + try { + seedSessionAndEpisode(handle); + // A brand-new install with no embedder probe yet: dimension unknown. + // Any stored BLOB (short or full) should count as ready. + seedTrace(handle, "tr_full", { + userText: "hello world what is up", + agentText: "here is the answer", + vecSummary: fullVec(), + vecAction: shortVec(), + }); + seedTrace(handle, "tr_missing", { + userText: "hello world what is up", + agentText: "another answer here", + vecSummary: null, + vecAction: null, + }); + + const counts = embeddingMaintenanceCounts(handle.db, { expectedByteLen: 0 }); + + // Both slots for tr_full count as ready regardless of BLOB length. + expect(counts.trace.ready).toBe(2); + expect(counts.trace.dimMismatch).toBe(0); + expect(counts.trace.missing).toBe(2); + expect(counts.trace.totalSlots).toBe(4); + } finally { + handle.cleanup(); + } + }); + + it("returns zero counts for an empty database", () => { + const handle = makeTmpDb(); + try { + const counts = embeddingMaintenanceCounts(handle.db, { + expectedByteLen: EXPECTED_BYTE_LEN, + }); + expect(counts.trace).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + expect(counts.policy).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + expect(counts.world_model).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + expect(counts.skill).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + } finally { + handle.cleanup(); + } + }); + + it("infers stored byte length from the mode of trace vec_summary BLOBs", () => { + const handle = makeTmpDb(); + try { + seedSessionAndEpisode(handle); + // Three rows at 4-dim, one at 1-dim → mode = 16 bytes. + seedTrace(handle, "tr_a", { + userText: "hello world what is up", + agentText: "here is the answer", + vecSummary: fullVec(), + }); + seedTrace(handle, "tr_b", { + userText: "hello world what is up", + agentText: "another answer here", + vecSummary: fullVec(), + }); + seedTrace(handle, "tr_c", { + userText: "hello world what is up", + agentText: "yet another answer", + vecSummary: fullVec(), + }); + seedTrace(handle, "tr_odd", { + userText: "hello world what is up", + agentText: "the outlier answer", + vecSummary: shortVec(), + }); + + expect(inferStoredEmbeddingByteLen(handle.db)).toBe(EXPECTED_BYTE_LEN); + } finally { + handle.cleanup(); + } + }); +}); From b8fb35e9bb5bbe619bb9f7f06b209878bd736fb1 Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Thu, 2 Jul 2026 15:39:52 +0800 Subject: [PATCH 2/3] refactor(memos-local-plugin): use precise EmbeddingCountsBucket type in addNeedsRepair MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return type was pinned to `EmbeddingMaintenanceStats["byKind"]["trace"]`, but the helper is reused for all four bucket kinds (trace, policy, world_model, skill). If any bucket shape ever diverged, TypeScript would silently accept a structurally compatible but semantically wrong type on the non-trace buckets. Switch the return type to `EmbeddingCountsBucket & { needsRepair: number }` — the shared bucket shape from `embedding_maintenance.ts` — which is what the helper actually produces and does not implicitly couple to the trace slot. Fixes Open Code Review finding on PR #2038. --- apps/memos-local-plugin/core/pipeline/memory-core.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index 9df1f2534..a7010738f 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -85,6 +85,7 @@ import { inferStoredEmbeddingByteLen, FLOAT32_BYTES, } from "../storage/repos/index.js"; +import type { EmbeddingCountsBucket } from "../storage/repos/index.js"; import { createEmbedder } from "../embedding/embedder.js"; import { createLlmClient } from "../llm/client.js"; import { @@ -4079,7 +4080,7 @@ export function createMemoryCore( ready: number; missing: number; dimMismatch: number; - }): EmbeddingMaintenanceStats["byKind"]["trace"] { + }): EmbeddingCountsBucket & { needsRepair: number } { return { ...bucket, needsRepair: bucket.missing + bucket.dimMismatch, From 9abaae822a4fd9ebcde4fd8670adb39a5ce2f86b Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Thu, 2 Jul 2026 15:58:20 +0800 Subject: [PATCH 3/3] refactor(memos-local-plugin): tidy embedding_maintenance re-exports and reuse EmbeddingCountsBucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to PR #2038 Open Code Review: 1. `core/storage/repos/index.ts`: the `embedding_maintenance` symbols were imported and then re-exported in two separate statements, while every other barrel entry in this file uses the direct `export { … } from "…"` form (lines 77–93). Collapse to the same shape: export { embeddingMaintenanceCounts, inferStoredEmbeddingByteLen, FLOAT32_BYTES, } from "./embedding_maintenance.js"; export type { EmbeddingCounts, EmbeddingCountsBucket } from "./embedding_maintenance.js"; 2. `core/pipeline/memory-core.ts`: `addNeedsRepair()` declared its `bucket` parameter as an inline literal `{ totalSlots; ready; missing; dimMismatch }` that is byte-for-byte identical to the already-imported `EmbeddingCountsBucket`. Replace the inline literal with the named type so the structural contract lives in one place. Behaviour unchanged — pure type / re-export tidy-up. --- apps/memos-local-plugin/core/pipeline/memory-core.ts | 9 +++------ apps/memos-local-plugin/core/storage/repos/index.ts | 11 ++--------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index a7010738f..92032e305 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -4075,12 +4075,9 @@ export function createMemoryCore( }; } - function addNeedsRepair(bucket: { - totalSlots: number; - ready: number; - missing: number; - dimMismatch: number; - }): EmbeddingCountsBucket & { needsRepair: number } { + function addNeedsRepair( + bucket: EmbeddingCountsBucket, + ): EmbeddingCountsBucket & { needsRepair: number } { return { ...bucket, needsRepair: bucket.missing + bucket.dimMismatch, diff --git a/apps/memos-local-plugin/core/storage/repos/index.ts b/apps/memos-local-plugin/core/storage/repos/index.ts index 7386ec218..5f3038dab 100644 --- a/apps/memos-local-plugin/core/storage/repos/index.ts +++ b/apps/memos-local-plugin/core/storage/repos/index.ts @@ -5,13 +5,6 @@ */ import type { StorageDb } from "../types.js"; -import { - embeddingMaintenanceCounts, - inferStoredEmbeddingByteLen, - FLOAT32_BYTES, - type EmbeddingCounts, - type EmbeddingCountsBucket, -} from "./embedding_maintenance.js"; import { makeApiLogsRepo } from "./api_logs.js"; import { makeAuditRepo } from "./audit.js"; import { makeCandidatePoolRepo } from "./candidate_pool.js"; @@ -102,5 +95,5 @@ export { embeddingMaintenanceCounts, inferStoredEmbeddingByteLen, FLOAT32_BYTES, -}; -export type { EmbeddingCounts, EmbeddingCountsBucket }; +} from "./embedding_maintenance.js"; +export type { EmbeddingCounts, EmbeddingCountsBucket } from "./embedding_maintenance.js";