diff --git a/README.md b/README.md index 3e8222e2..3322a907 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ ST-->>PL: pack text PL->>OC: inject codemem context ``` -**Retrieval** combines two strategies: keyword search via SQLite FTS5 with BM25 scoring and semantic similarity via sqlite-vec embeddings. In the pack-building path, results from both are merged, deduplicated, and re-ranked using recency and memory-kind boosts. +**Retrieval** combines two strategies: keyword search via SQLite FTS5 with BM25 scoring and semantic similarity via sqlite-vec embeddings. In the pack-building path, results from both are merged, exactly deduplicated, and re-ranked using recency and memory-kind boosts. Near-related memories stay fully rendered by default; use compact rendering or `CODEMEM_PACK_COMPRESSION=ids` only when you intentionally want ID-based expansion via `memory_get_observations`. **Injection** happens automatically. The plugin builds a query from the current session context (first prompt, latest prompt, project, recently modified files), calls `build_memory_pack`, and appends the result to the system prompt via `experimental.chat.system.transform`. @@ -142,6 +142,8 @@ For architecture details, see [docs/architecture.md](docs/architecture.md). Run `codemem --help` for the full list. Legacy top-level aliases (`export-memories`, `import-memories`, `show`, `forget`, `remember`) still work but are hidden from help. +Pack rendering defaults to self-contained context. For token-constrained experiments, `codemem pack --compact` renders an index plus top details. Near-related compression is controlled by `--compression-mode off|compact|ids` (or `CODEMEM_PACK_COMPRESSION`); MCP `memory_pack` exposes the same setting as `compression_mode`. Use `ids` only when the agent can follow up with `memory_get_observations`. + ## MCP tools To give the LLM direct access to memory tools (search, timeline, pack, remember, forget): diff --git a/packages/cli/src/commands/pack-shared.ts b/packages/cli/src/commands/pack-shared.ts index 27e94ca4..074319f7 100644 --- a/packages/cli/src/commands/pack-shared.ts +++ b/packages/cli/src/commands/pack-shared.ts @@ -36,7 +36,11 @@ export function addPackRequestOptions(command: Command): Command { .option("--project ", "project identifier (defaults to git repo root)") .option("--all-projects", "search across all projects") .option("--compact", "render a scannable index with full detail only for top N items") - .option("--compact-detail ", "items to show in full detail in compact mode (default 3)"); + .option("--compact-detail ", "items to show in full detail in compact mode (default 3)") + .option( + "--compression-mode ", + "near-related compression mode: off, compact, or ids (default: CODEMEM_PACK_COMPRESSION or compact)", + ); } export function buildPackRequestOptions( @@ -49,6 +53,7 @@ export function buildPackRequestOptions( allProjects?: boolean; compact?: boolean; compactDetail?: string; + compressionMode?: string; }, ctx: { cwd?: string; @@ -76,19 +81,29 @@ export function buildPackRequestOptions( } let renderOptions: PackRenderOptions | undefined; - if (opts.compact || opts.compactDetail != null) { - renderOptions = { compact: true }; + if (opts.compact || opts.compactDetail != null || opts.compressionMode != null) { + renderOptions = {}; + if (opts.compact || opts.compactDetail != null) renderOptions.compact = true; if (opts.compactDetail != null) { renderOptions.compactDetailCount = parseNonNegativeInt( opts.compactDetail, "compact detail count", ); } + if (opts.compressionMode != null) { + renderOptions.compressionMode = parseCompressionMode(opts.compressionMode); + } } return { limit, budget, filters, renderOptions }; } +function parseCompressionMode(value: string): "off" | "compact" | "ids" { + const normalized = value.trim().toLowerCase(); + if (normalized === "off" || normalized === "compact" || normalized === "ids") return normalized; + throw new PackUsageError("compression mode must be one of: off, compact, ids"); +} + function parsePositiveInt(value: string, label: string): number { if (!/^\d+$/.test(value.trim())) { throw new PackUsageError(`${label} must be a positive integer`); diff --git a/packages/cli/src/commands/pack.test.ts b/packages/cli/src/commands/pack.test.ts index e61ad54d..e4f44ba7 100644 --- a/packages/cli/src/commands/pack.test.ts +++ b/packages/cli/src/commands/pack.test.ts @@ -242,6 +242,49 @@ describe("pack command", () => { expect(JSON.parse(String(logSpy.mock.calls.at(-1)?.[0]))).toMatchObject({ items: [] }); }); + it("passes explicit compression mode through main pack command", async () => { + buildMemoryPackAsync.mockResolvedValue({ + items: [], + metrics: { + total_items: 0, + pack_tokens: 0, + fallback_used: false, + sources: { fts: 0, semantic: 0, fuzzy: 0 }, + }, + pack_text: "", + }); + + await parsePackCommand(["continue viewer health work", "--compression-mode", "ids"]); + + expect(buildMemoryPackAsync).toHaveBeenCalledWith( + "continue viewer health work", + 10, + undefined, + expect.any(Object), + { compressionMode: "ids" }, + ); + }); + + it("rejects invalid compression mode", async () => { + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + await parsePackCommand([ + "continue viewer health work", + "--json", + "--compression-mode", + "banana", + ]); + + expect(JSON.parse(String(logSpy.mock.calls.at(-1)?.[0]))).toEqual({ + error: "usage_error", + message: "compression mode must be one of: off, compact, ids", + }); + expect(process.exitCode).toBe(2); + expect(errorSpy).not.toHaveBeenCalled(); + expect(buildMemoryPackAsync).not.toHaveBeenCalled(); + }); + it("emits structured json errors for pack failures", async () => { buildMemoryPackAsync.mockRejectedValue(new Error("pack blew up")); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); diff --git a/packages/core/src/pack.test.ts b/packages/core/src/pack.test.ts index 564d17ad..1ef204da 100644 --- a/packages/core/src/pack.test.ts +++ b/packages/core/src/pack.test.ts @@ -1,7 +1,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { connect } from "./db.js"; import { buildMemoryPack, buildMemoryPackTrace, estimateTokens } from "./pack.js"; import { MemoryStore } from "./store.js"; @@ -1577,7 +1577,7 @@ describe("buildMemoryPack compact mode", () => { expect(pack.pack_text).toContain("## Index"); expect(pack.pack_text).toContain("## Detail"); expect(pack.pack_text).toContain("memory_get"); - expect(pack.pack_text).toContain("memory_search"); + expect(pack.pack_text).toContain("memory_get_observations"); }); it("shows index lines for all items", () => { @@ -1761,7 +1761,26 @@ describe("buildMemoryPack compact mode", () => { 0.7, ); - const pack = buildMemoryPack(store, "thread local memory store pooling mcp server", 10, 5); + const unbudgeted = buildMemoryPack( + store, + "thread local memory store pooling mcp server", + 10, + null, + undefined, + undefined, + { compressionMode: "ids" }, + ); + expect(unbudgeted.items.some((item) => (item.compressed_ids?.length ?? 0) > 0)).toBe(true); + + const pack = buildMemoryPack( + store, + "thread local memory store pooling mcp server", + 10, + 5, + undefined, + undefined, + { compressionMode: "ids" }, + ); expect(pack.items).toHaveLength(0); expect(pack.item_ids).toHaveLength(0); @@ -1792,7 +1811,17 @@ describe("buildMemoryPack compact mode", () => { 0.9, ); - const pack = buildMemoryPack(store, "remember what happened previously", 10); + const pack = buildMemoryPack( + store, + "remember what happened previously", + 10, + null, + undefined, + undefined, + { + compressionMode: "ids", + }, + ); expect(pack.item_ids).toEqual(expect.arrayContaining([summaryId, idA, idB])); expect(pack.items.find((item) => item.id === idB)?.compressed_ids).toEqual([idA]); @@ -1819,11 +1848,12 @@ describe("buildMemoryPack cluster compression", () => { }); afterEach(() => { + vi.unstubAllEnvs(); store.close(); rmSync(tmpDir, { recursive: true, force: true }); }); - it("compresses related items into one representative in default mode", () => { + it("keeps related items self-contained in default mode", () => { const idA = store.remember( sessionId, "discovery", @@ -1841,7 +1871,42 @@ describe("buildMemoryPack cluster compression", () => { const pack = buildMemoryPack(store, "sync pass orchestrator typescript", 10); - expect(pack.pack_text).toContain("(+1 related)"); + expect(pack.pack_text).not.toContain("(+1 related)"); + expect(pack.pack_text).toContain("Sync pass orchestrator moved from Python to TypeScript"); + expect(pack.pack_text).toContain("Sync pass orchestrator ported to TypeScript"); + expect(pack.item_ids).toEqual(expect.arrayContaining([idA, idB])); + expect(pack.items.map((item) => item.id)).toEqual(expect.arrayContaining([idA, idB])); + }); + + it("compresses related items into one representative when ids mode is requested", () => { + const idA = store.remember( + sessionId, + "discovery", + "Sync pass orchestrator ported to TypeScript", + "Moved the sync pass orchestrator from Python to TypeScript.", + 0.6, + ); + const idB = store.remember( + sessionId, + "feature", + "Sync pass orchestrator moved from Python to TypeScript", + "The orchestrator now coordinates sync in TypeScript.", + 0.9, + ); + + const pack = buildMemoryPack( + store, + "sync pass orchestrator typescript", + 10, + null, + undefined, + undefined, + { + compressionMode: "ids", + }, + ); + + expect(pack.pack_text).toContain(`(+1 related: [${idA}])`); expect(pack.pack_text).toContain("Sync pass orchestrator moved from Python to TypeScript"); expect(pack.pack_text).not.toContain("Sync pass orchestrator ported to TypeScript -"); expect(pack.item_ids).toEqual(expect.arrayContaining([idA, idB])); @@ -1853,6 +1918,29 @@ describe("buildMemoryPack cluster compression", () => { }); }); + it("uses CODEMEM_PACK_COMPRESSION to select ids compression mode", () => { + const idA = store.remember( + sessionId, + "change", + "Replication retention plan updated for bounded history", + "Updated the retention plan.", + 0.4, + ); + store.remember( + sessionId, + "decision", + "Replication retention plan revised for bounded history", + "Revised retention plan and documented bounds.", + 0.95, + ); + + vi.stubEnv("CODEMEM_PACK_COMPRESSION", "ids"); + const pack = buildMemoryPack(store, "replication retention bounded history", 10); + + expect(pack.pack_text).toContain(`(+1 related: [${idA}])`); + expect(pack.items[0]?.compressed_ids).toEqual([idA]); + }); + it("chooses highest-confidence representative on clustered items", () => { const idLow = store.remember( sessionId, @@ -1869,7 +1957,17 @@ describe("buildMemoryPack cluster compression", () => { 0.95, ); - const pack = buildMemoryPack(store, "replication retention bounded history", 10); + const pack = buildMemoryPack( + store, + "replication retention bounded history", + 10, + null, + undefined, + undefined, + { + compressionMode: "ids", + }, + ); expect(pack.items[0]?.id).toBe(idHigh); expect(pack.items[0]?.compressed_ids).toEqual([idLow]); }); @@ -1925,7 +2023,8 @@ describe("buildMemoryPack cluster compression", () => { }, ); - expect(pack.pack_text).toContain("(+1 related)"); + expect(pack.pack_text).toContain(`(+1 related: [${idA}])`); + expect(pack.pack_text).toContain("memory_get_observations"); expect(pack.item_ids).toEqual(expect.arrayContaining([idA, idB])); expect(pack.items).toHaveLength(1); }); @@ -1990,7 +2089,15 @@ describe("buildMemoryPack cluster compression", () => { 0.9, ); - const trace = buildMemoryPackTrace(store, "peer deletion cursor leak sync worker", 10); + const trace = buildMemoryPackTrace( + store, + "peer deletion cursor leak sync worker", + 10, + null, + undefined, + undefined, + { compressionMode: "ids" }, + ); expect(trace.assembly.compressed_clusters).toHaveLength(1); expect(trace.assembly.compressed_clusters[0]).toMatchObject({ diff --git a/packages/core/src/pack.ts b/packages/core/src/pack.ts index 6472f07c..37d8ccb6 100644 --- a/packages/core/src/pack.ts +++ b/packages/core/src/pack.ts @@ -105,13 +105,52 @@ function parseFacts(raw: string | null): string[] | null { * available. Falls back to the original single-line format when neither * structured field exists. */ -function relatedSuffix(item: MemoryResult, clusterState?: ClusterCompressionState): string { - const relatedCount = clusterState?.compressedByRepresentative.get(item.id)?.size ?? 0; - return relatedCount > 0 ? ` (+${relatedCount} related)` : ""; +type PackCompressionMode = "off" | "compact" | "ids"; + +const PACK_COMPRESSION_MODE_ENV = "CODEMEM_PACK_COMPRESSION"; +const DEFAULT_PACK_COMPRESSION_MODE: PackCompressionMode = "compact"; + +function parsePackCompressionMode(value: string | undefined): PackCompressionMode | null { + const normalized = value?.trim().toLowerCase(); + if (!normalized) return null; + if (["0", "false", "none", "off", "disabled"].includes(normalized)) return "off"; + if (["compact", "compact-only", "compact_only", "default"].includes(normalized)) return "compact"; + if (["1", "true", "on", "ids", "all", "legacy"].includes(normalized)) return "ids"; + return null; +} + +function resolvePackCompressionMode(explicit?: PackCompressionMode): PackCompressionMode { + return ( + explicit ?? + parsePackCompressionMode(process.env[PACK_COMPRESSION_MODE_ENV]) ?? + DEFAULT_PACK_COMPRESSION_MODE + ); } -function formatItem(item: MemoryResult, clusterState?: ClusterCompressionState): string { - const header = `[${item.id}] (${item.kind}) ${item.title}${relatedSuffix(item, clusterState)}`; +function relatedSuffix( + item: MemoryResult, + clusterState?: ClusterCompressionState, + options: { includeIds?: boolean } = {}, +): string { + const related = clusterState?.compressedByRepresentative.get(item.id); + const relatedCount = related?.size ?? 0; + if (relatedCount === 0) return ""; + if (!options.includeIds) return ` (+${relatedCount} related)`; + const ids = [...(related ?? [])] + .sort((a, b) => a - b) + .map((id) => `[${id}]`) + .join(", "); + return ` (+${relatedCount} related: ${ids})`; +} + +function formatItem( + item: MemoryResult, + clusterState?: ClusterCompressionState, + options: { includeRelatedIds?: boolean } = {}, +): string { + const header = `[${item.id}] (${item.kind}) ${item.title}${relatedSuffix(item, clusterState, { + includeIds: options.includeRelatedIds, + })}`; const narrative = item.narrative || null; const facts = parseFacts(item.facts); @@ -138,10 +177,11 @@ function formatSection( header: string, items: MemoryResult[], clusterState?: ClusterCompressionState, + options: { includeRelatedIds?: boolean } = {}, ): string { const heading = `## ${header}`; if (items.length === 0) return `${heading}\n`; - return [heading, ...items.map((item) => formatItem(item, clusterState))].join("\n"); + return [heading, ...items.map((item) => formatItem(item, clusterState, options))].join("\n"); } // --------------------------------------------------------------------------- @@ -149,11 +189,14 @@ function formatSection( // --------------------------------------------------------------------------- const DEFAULT_COMPACT_DETAIL_COUNT = 3; -const COMPACT_FOOTER = "Use `memory_get` or `memory_search` to fetch detail for any item by [ID]."; +const COMPACT_FOOTER = + "Use `memory_get` for one item, or `memory_get_observations(ids=[...])` for related IDs shown in the index."; /** Single-line index entry for compact mode. */ function formatIndexLine(item: MemoryResult, clusterState?: ClusterCompressionState): string { - return `[${item.id}] (${item.kind}) ${item.title}${relatedSuffix(item, clusterState)}`; + return `[${item.id}] (${item.kind}) ${item.title}${relatedSuffix(item, clusterState, { + includeIds: true, + })}`; } /** @@ -174,7 +217,9 @@ function renderCompactPack( const detailItems = items.filter((item) => detailIds.has(item.id)); const detailSection = detailItems.length > 0 - ? `## Detail\n${detailItems.map((item) => formatItem(item, clusterState)).join("\n\n")}` + ? `## Detail\n${detailItems + .map((item) => formatItem(item, clusterState, { includeRelatedIds: true })) + .join("\n\n")}` : "## Detail\n(no items)"; return `${indexSection}\n\n${detailSection}\n\n${COMPACT_FOOTER}`; @@ -1236,7 +1281,12 @@ function buildPackArtifacts( tokenBudget: number | null = null, filters?: MemoryFilters, semanticResults?: MemoryResult[], - options: { recordUsage: boolean; compact?: boolean; compactDetailCount?: number } = { + options: { + recordUsage: boolean; + compact?: boolean; + compactDetailCount?: number; + compressionMode?: PackCompressionMode; + } = { recordUsage: true, }, ): PackArtifacts { @@ -1482,19 +1532,26 @@ function buildPackArtifacts( timelineItems = collapseExactDuplicates(timelineItems, dedupeState); observationItems = collapseExactDuplicates(observationItems, dedupeState); + const compact = options.compact ?? false; + const compactDetailCount = options.compactDetailCount ?? DEFAULT_COMPACT_DETAIL_COUNT; + const compressionMode = resolvePackCompressionMode(options.compressionMode); + const compressRelated = compressionMode === "ids" || (compressionMode === "compact" && compact); + const clusterState: ClusterCompressionState = { compressedByRepresentative: new Map(), representativeByCompressedId: new Map(), clusters: [], }; - const compressionPoolSeen = new Set(); - const compressionPool: MemoryResult[] = []; - for (const item of [...summaryItems, ...timelineItems, ...observationItems]) { - if (compressionPoolSeen.has(item.id)) continue; - compressionPoolSeen.add(item.id); - compressionPool.push(item); + if (compressRelated) { + const compressionPoolSeen = new Set(); + const compressionPool: MemoryResult[] = []; + for (const item of [...summaryItems, ...timelineItems, ...observationItems]) { + if (compressionPoolSeen.has(item.id)) continue; + compressionPoolSeen.add(item.id); + compressionPool.push(item); + } + compressClusters(compressionPool, modeLabel, clusterState); } - compressClusters(compressionPool, modeLabel, clusterState); const compressedSectionIds = new Set(flattenCompressedIds(clusterState)); summaryItems = summaryItems.filter((item) => !compressedSectionIds.has(item.id)); timelineItems = timelineItems.filter((item) => !compressedSectionIds.has(item.id)); @@ -1507,9 +1564,6 @@ function buildPackArtifacts( // index-line costs for items beyond the detail count, and renders a // scannable Index + Detail layout instead of Summary/Timeline/Observations. - const compact = options.compact ?? false; - const compactDetailCount = options.compactDetailCount ?? DEFAULT_COMPACT_DETAIL_COUNT; - let budgetedSummary: MemoryResult[]; let budgetedTimeline: MemoryResult[]; let budgetedObservations: MemoryResult[]; @@ -1535,7 +1589,8 @@ function buildPackArtifacts( const indexCost = estimateTokens(formatIndexLine(item, clusterState)); if (detailSlots > 0) { // Detail items appear in both Index and Detail — charge both. - const fullCost = indexCost + estimateTokens(formatItem(item, clusterState)); + const fullCost = + indexCost + estimateTokens(formatItem(item, clusterState, { includeRelatedIds: true })); if (tokensUsed + fullCost <= tokenBudget) { tokensUsed += fullCost; budgetedItems.push(item); @@ -1569,10 +1624,11 @@ function buildPackArtifacts( if (tokenBudget != null && tokenBudget > 0) { let tokensUsed = 0; + const formatOptions = { includeRelatedIds: compressionMode === "ids" }; budgetedSummary = []; for (const item of summaryItems) { - const cost = estimateTokens(formatItem(item, clusterState)); + const cost = estimateTokens(formatItem(item, clusterState, formatOptions)); if (tokensUsed + cost > tokenBudget) break; tokensUsed += cost; budgetedSummary.push(item); @@ -1580,7 +1636,7 @@ function buildPackArtifacts( budgetedTimeline = []; for (const item of timelineItems) { - const cost = estimateTokens(formatItem(item, clusterState)); + const cost = estimateTokens(formatItem(item, clusterState, formatOptions)); if (tokensUsed + cost > tokenBudget) break; tokensUsed += cost; budgetedTimeline.push(item); @@ -1588,7 +1644,7 @@ function buildPackArtifacts( budgetedObservations = []; for (const item of observationItems) { - const cost = estimateTokens(formatItem(item, clusterState)); + const cost = estimateTokens(formatItem(item, clusterState, formatOptions)); if (tokensUsed + cost > tokenBudget) break; tokensUsed += cost; budgetedObservations.push(item); @@ -1596,9 +1652,15 @@ function buildPackArtifacts( } const sections = [ - formatSection("Summary", budgetedSummary, clusterState), - formatSection("Timeline", budgetedTimeline, clusterState), - formatSection("Observations", budgetedObservations, clusterState), + formatSection("Summary", budgetedSummary, clusterState, { + includeRelatedIds: compressionMode === "ids", + }), + formatSection("Timeline", budgetedTimeline, clusterState, { + includeRelatedIds: compressionMode === "ids", + }), + formatSection("Observations", budgetedObservations, clusterState, { + includeRelatedIds: compressionMode === "ids", + }), ]; packText = sections.join("\n\n"); } @@ -1890,6 +1952,7 @@ export function buildMemoryPack( recordUsage: true, compact: renderOptions?.compact, compactDetailCount: renderOptions?.compactDetailCount, + compressionMode: renderOptions?.compressionMode, }).response; } @@ -1906,6 +1969,7 @@ export function buildMemoryPackTrace( recordUsage: false, compact: renderOptions?.compact, compactDetailCount: renderOptions?.compactDetailCount, + compressionMode: renderOptions?.compressionMode, }).trace; } @@ -1949,6 +2013,7 @@ export async function buildMemoryPackAsync( recordUsage: true, compact: renderOptions?.compact, compactDetailCount: renderOptions?.compactDetailCount, + compressionMode: renderOptions?.compressionMode, }).response; } @@ -1976,5 +2041,6 @@ export async function buildMemoryPackTraceAsync( recordUsage: false, compact: renderOptions?.compact, compactDetailCount: renderOptions?.compactDetailCount, + compressionMode: renderOptions?.compressionMode, }).trace; } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 4401ccff..27af4ee1 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -731,4 +731,11 @@ export interface PackRenderOptions { * Ignored when compact is false. Default: 3. */ compactDetailCount?: number; + /** + * Controls near-related pack compression. Default: "compact". + * - "off": never compress related items + * - "compact": compress related items only in compact rendering mode + * - "ids": compress related items in all modes and render related IDs as a retrieval hint + */ + compressionMode?: "off" | "compact" | "ids"; } diff --git a/packages/mcp-server/src/index.ts b/packages/mcp-server/src/index.ts index 86962bc2..d4585b24 100644 --- a/packages/mcp-server/src/index.ts +++ b/packages/mcp-server/src/index.ts @@ -546,14 +546,24 @@ async function main() { .max(50) .optional() .describe("Number of items to show in full detail in compact mode (default 3)"), + compression_mode: z + .enum(["off", "compact", "ids"]) + .optional() + .describe( + "Near-related compression mode: off disables it, compact applies only to compact rendering, ids applies in all modes. Defaults to CODEMEM_PACK_COMPRESSION or compact.", + ), ...filterSchema, }, async (args) => { try { const filters = buildFilters(args); const renderOptions = - args.compact || args.compact_detail_count != null - ? { compact: args.compact ?? true, compactDetailCount: args.compact_detail_count } + args.compact || args.compact_detail_count != null || args.compression_mode != null + ? { + compact: args.compact ?? (args.compact_detail_count != null ? true : undefined), + compactDetailCount: args.compact_detail_count, + compressionMode: args.compression_mode, + } : undefined; const result = await store.buildMemoryPackAsync( args.context,