fix(memory): recall relevant topics beyond scan cap

docs/design/2026-08-09-bounded-memory-recall-candidates.md

@@ -0,0 +1,56 @@
+# Bounded Memory Recall Candidates
+
+## Problem
+
+The project and user memory scanners enumerate, read, and parse every topic,
+then return only the 200 most recent documents. Recall uses those shared scanner
+APIs, so an older relevant document outside either 200-document window cannot
+reach the heuristic or model selector even though the expensive scan work has
+already happened.
+
+The same capped APIs are also used by Forget, Indexer, Status, and Extraction.
+Removing their limit globally would widen unrelated behavior.
+
+## Decision
+
+Keep the existing scanner APIs and their 200-document limit unchanged. Add
+explicit all-topic variants used only by recall.
+
+Recall ranks the combined project and user pool before model selection:
+
+- retain up to 180 documents with a lexical match using the existing scorer;
+- fill the remaining candidate slots by recency, preserving at least 20 recent
+  opportunities when enough lexical matches exist;
+- send at most 200 candidates to the model selector;
+- append manifest entries only while their cumulative UTF-8 size remains at or
+  below 25,000 bytes;
+- validate selector output only against documents actually present in that
+  bounded manifest.
+
+The heuristic fallback continues to score the complete recall pool and still
+returns at most five documents. Existing body and prompt limits remain
+unchanged.
+
+## Failure and compatibility boundaries
+
+Project scanning remains required. User scanning remains best-effort. Invalid
+or unreadable files keep the existing skip behavior. Empty candidate manifests
+return no model selection rather than sending an unbounded request.
+
+There is no public setting, persistent index, new dependency, provider API, or
+Fast/Refined state machine. Recall still performs an O(n) local pass over the
+already parsed documents; a persistent catalog requires separate measurement
+and evidence.
+
+## Verification
+
+- A deliberately old relevant topic beyond the regular 200-document result is
+  recalled from a real temporary memory tree.
+- The regular scanner still returns 200 documents and omits that topic.
+- The model candidate set contains the lexical target and recent reserve while
+  remaining at 200 documents.
+- A manifest built from large multibyte descriptions stays within 25,000 UTF-8
+  bytes.
+- The deterministic CLI E2E selects the overflow topic and exposes its unique
+  marker at the ToolResult delivery point after the bounded initial wait
+  expires.

packages/core/src/memory/memoryLifecycle.integration.test.ts

@@ -229,4 +229,65 @@ describe('managed auto-memory lifecycle integration', () => {
     expect(recall.prompt).toContain('user/');
     expect(recall.prompt).toContain('reference/');
   });
+
+  it('recalls a relevant topic beyond the general 200-document scan cap', async () => {
+    const referenceDir = path.dirname(
+      getAutoMemoryFilePath(projectRoot, 'reference/filler-000.md'),
+    );
+    await fs.mkdir(referenceDir, { recursive: true });
+    await Promise.all(
+      Array.from({ length: 200 }, (_, index) =>
+        fs.writeFile(
+          path.join(
+            referenceDir,
+            `filler-${String(index).padStart(3, '0')}.md`,
+          ),
+          [
+            '---',
+            'type: reference',
+            `name: Filler ${index}`,
+            'description: Unrelated historical note',
+            '---',
+            '',
+            'No matching content.',
+          ].join('\n'),
+          'utf-8',
+        ),
+      ),
+    );
+
+    const targetPath = getAutoMemoryFilePath(
+      projectRoot,
+      'reference/overflow-target.md',
+    );
+    await fs.writeFile(
+      targetPath,
+      [
+        '---',
+        'type: reference',
+        'name: Overflow Zephyr Marker',
+        'description: Unique recall target beyond the general scan cap',
+        '---',
+        '',
+        'The saved codeword is OVERFLOW-ZEPHYR-7040.',
+      ].join('\n'),
+      'utf-8',
+    );
+    await fs.utimes(targetPath, new Date(0), new Date(0));
+
+    const cappedDocs = await scanAutoMemoryTopicDocuments(projectRoot);
+    expect(cappedDocs).toHaveLength(200);
+    expect(cappedDocs.some((doc) => doc.filePath === targetPath)).toBe(false);
+
+    const recall = await resolveRelevantAutoMemoryPromptForQuery(
+      projectRoot,
+      'What is the overflow zephyr codeword?',
+    );
+
+    expect(recall.strategy).toBe('heuristic');
+    expect(recall.selectedDocs.map((doc) => doc.filePath)).toContain(
+      targetPath,
+    );
+    expect(recall.prompt).toContain('OVERFLOW-ZEPHYR-7040');
+  });
 });

packages/core/src/memory/recall.test.ts

@@ -12,19 +12,19 @@ import {
 } from './recall.js';
 import type { ScannedAutoMemoryDocument } from './scan.js';
 import type { Config } from '../config/config.js';
-import { scanAutoMemoryTopicDocuments } from './scan.js';
+import { scanAllAutoMemoryTopicDocuments } from './scan.js';
 import { selectRelevantAutoMemoryDocumentsByModel } from './relevanceSelector.js';
 
 vi.mock('./scan.js', async (importOriginal) => {
   const actual = await importOriginal<typeof import('./scan.js')>();
   return {
     ...actual,
-    scanAutoMemoryTopicDocuments: vi.fn(),
+    scanAllAutoMemoryTopicDocuments: vi.fn(),
     // Explicit mock — recall now unions user-level docs into the pool, so
     // leaving this on the real implementation would silently fall through
     // to the filesystem (only "works" because the path doesn't exist and
     // listMarkdownFiles swallows ENOENT). Defaults to an empty pool.
-    scanUserAutoMemoryTopicDocuments: vi.fn().mockResolvedValue([]),
+    scanAllUserAutoMemoryTopicDocuments: vi.fn().mockResolvedValue([]),
   };
 });
 
@@ -413,7 +413,7 @@ describe('auto-memory relevant recall', () => {
   });
 
   it('uses model-driven selection when config is provided', async () => {
-    vi.mocked(scanAutoMemoryTopicDocuments).mockResolvedValue(docs);
+    vi.mocked(scanAllAutoMemoryTopicDocuments).mockResolvedValue(docs);
     vi.mocked(selectRelevantAutoMemoryDocumentsByModel).mockResolvedValue([
       docs[0],
     ]);
@@ -431,8 +431,63 @@ describe('auto-memory relevant recall', () => {
     expect(result.prompt).toContain('Reference Memory (reference.md)');
   });
 
+  it('bounds model candidates while retaining lexical and recent documents', async () => {
+    const lexicalDocs = Array.from({ length: 200 }, (_, index) => ({
+      ...memoryDoc(
+        `lexical-${String(index).padStart(3, '0')}.md`,
+        'reference',
+        `Overflow memory ${index}`,
+        'Matching historical context',
+        '',
+      ),
+      mtimeMs: 0,
+    }));
+    const recentDocs = Array.from({ length: 20 }, (_, index) => ({
+      ...memoryDoc(
+        `recent-${String(index).padStart(2, '0')}.md`,
+        'reference',
+        `General memory ${index}`,
+        'Unrelated recent context',
+        '',
+      ),
+      mtimeMs: 20 - index,
+    }));
+    const lexicalTarget = {
+      ...memoryDoc(
+        'overflow-target.md',
+        'reference',
+        'Overflow Zephyr Marker',
+        'Unique semantic target',
+        '',
+      ),
+      mtimeMs: 0,
+    };
+    vi.mocked(scanAllAutoMemoryTopicDocuments).mockResolvedValue([
+      ...lexicalDocs,
+      ...recentDocs,
+      lexicalTarget,
+    ]);
+    vi.mocked(selectRelevantAutoMemoryDocumentsByModel).mockImplementation(
+      async (_config, _query, candidates) =>
+        candidates.includes(lexicalTarget) ? [lexicalTarget] : [],
+    );
+
+    const result = await resolveRelevantAutoMemoryPromptForQuery(
+      '/tmp/project',
+      'find the overflow zephyr marker',
+      { config: {} as Config },
+    );
+
+    const modelCandidates = vi.mocked(selectRelevantAutoMemoryDocumentsByModel)
+      .mock.calls[0]![2];
+    expect(modelCandidates).toHaveLength(200);
+    expect(modelCandidates[0]).toBe(lexicalTarget);
+    expect(modelCandidates.slice(-20)).toEqual(recentDocs);
+    expect(result.selectedDocs).toEqual([lexicalTarget]);
+  });
+
   it('falls back to heuristic selection when model-driven selection fails', async () => {
-    vi.mocked(scanAutoMemoryTopicDocuments).mockResolvedValue(docs);
+    vi.mocked(scanAllAutoMemoryTopicDocuments).mockResolvedValue(docs);
     vi.mocked(selectRelevantAutoMemoryDocumentsByModel).mockRejectedValue(
       new Error('selector failed'),
     );
@@ -456,9 +511,15 @@ describe('auto-memory relevant recall', () => {
   });
 
   it('keeps active tool schemas out of heuristic fallback', async () => {
-    vi.mocked(scanAutoMemoryTopicDocuments).mockResolvedValue(activeToolDocs);
-    vi.mocked(selectRelevantAutoMemoryDocumentsByModel).mockRejectedValue(
-      new Error('selector failed'),
+    vi.mocked(scanAllAutoMemoryTopicDocuments).mockResolvedValue(
+      activeToolDocs,
+    );
+    let modelCandidates: ScannedAutoMemoryDocument[] = [];
+    vi.mocked(selectRelevantAutoMemoryDocumentsByModel).mockImplementation(
+      async (_config, _query, candidates) => {
+        modelCandidates = candidates;
+        throw new Error('selector failed');
+      },
     );
 
     const result = await resolveRelevantAutoMemoryPromptForQuery(
@@ -470,6 +531,12 @@ describe('auto-memory relevant recall', () => {
       },
     );
 
+    expect(modelCandidates.map((doc) => doc.filePath)).not.toContain(
+      '/tmp/ata-tool.md',
+    );
+    expect(modelCandidates.map((doc) => doc.filePath)).toContain(
+      '/tmp/ata-gotcha.md',
+    );
     expect(result.strategy).toBe('heuristic');
     expect(result.selectedDocs.map((doc) => doc.filePath)).not.toContain(
       '/tmp/ata-tool.md',

packages/core/src/memory/recall.ts

@@ -8,8 +8,8 @@ import * as path from 'node:path';
 import type { Config } from '../config/config.js';
 import { createDebugLogger } from '../utils/debugLogger.js';
 import {
-  scanAutoMemoryTopicDocuments,
-  scanUserAutoMemoryTopicDocuments,
+  scanAllAutoMemoryTopicDocuments,
+  scanAllUserAutoMemoryTopicDocuments,
   type ScannedAutoMemoryDocument,
 } from './scan.js';
 import { memoryAge, memoryFreshnessText } from './memoryAge.js';
@@ -19,6 +19,8 @@ import { logMemoryRecall, MemoryRecallEvent } from '../telemetry/index.js';
 const MAX_RELEVANT_DOCS = 5;
 const MAX_DOC_BODY_CHARS = 1_200;
 const MAX_HEURISTIC_QUERY_TOKENS = 64;
+const MAX_MODEL_CANDIDATE_DOCS = 200;
+const RECENT_MODEL_CANDIDATE_RESERVE = 20;
 const debugLogger = createDebugLogger('AUTO_MEMORY_RECALL');
 
 const ACTIVE_TOOL_USAGE_MEMORY_MARKERS = [
@@ -221,6 +223,27 @@ export function selectRelevantAutoMemoryDocuments(
     .map(({ doc }) => doc);
 }
 
+function selectModelCandidateDocuments(
+  query: string,
+  docs: ScannedAutoMemoryDocument[],
+  recentTools: readonly string[],
+): ScannedAutoMemoryDocument[] {
+  const eligible = docs.filter(
+    (doc) => !isActiveToolUsageMemory(doc, recentTools),
+  );
+  const lexical = selectRelevantAutoMemoryDocuments(
+    query,
+    eligible,
+    MAX_MODEL_CANDIDATE_DOCS - RECENT_MODEL_CANDIDATE_RESERVE,
+  );
+  const selected = new Set(lexical.map((doc) => doc.filePath));
+  const recent = eligible
+    .filter((doc) => !selected.has(doc.filePath))
+    .sort((a, b) => b.mtimeMs - a.mtimeMs)
+    .slice(0, MAX_MODEL_CANDIDATE_DOCS - lexical.length);
+  return [...lexical, ...recent];
+}
+
 function truncateBody(body: string): string {
   const normalized = normalizeBody(body);
   if (normalized.length <= MAX_DOC_BODY_CHARS) {
@@ -299,19 +322,16 @@ export async function resolveRelevantAutoMemoryPromptForQuery(
   // recall returns nothing at all for the rest of the session. Project-
   // level scan failures still bubble — they're the only mandatory side.
   const [projectDocs, userDocs] = await Promise.all([
-    scanAutoMemoryTopicDocuments(projectRoot),
-    scanUserAutoMemoryTopicDocuments().catch((error: unknown) => {
+    scanAllAutoMemoryTopicDocuments(projectRoot),
+    scanAllUserAutoMemoryTopicDocuments().catch((error: unknown) => {
       debugLogger.warn(
         `User-level auto-memory scan failed; project-level recall continues: ${error instanceof Error ? error.message : String(error)}`,
       );
       return [];
     }),
   ]);
-  // Project-level docs come first as a soft hint to the model-based
-  // selector and, in the heuristic fallback (`selectRelevantAutoMemoryDocuments`),
-  // as the stable-sort tie-breaker — matching the PR's "project shadows
-  // user" precedence. The model selector ranks by its own judgement so
-  // this ordering is advisory there, not enforced.
+  // Project-level docs come first as the stable tie-break when later ranking
+  // keys match.
   const docs = filterExcludedAutoMemoryDocuments(
     [...projectDocs, ...userDocs],
     options.excludedFilePaths,
@@ -340,10 +360,15 @@ export async function resolveRelevantAutoMemoryPromptForQuery(
 
   if (options.config) {
     try {
+      const modelCandidates = selectModelCandidateDocuments(
+        query,
+        docs,
+        options.recentTools ?? [],
+      );
       const selectedDocs = await selectRelevantAutoMemoryDocumentsByModel(
         options.config,
         query,
-        docs,
+        modelCandidates,
         limit,
         options.recentTools ?? [],
         options.abortSignal,