From 9104d79b8d8218e228373f45fed2cfb340cb9dd2 Mon Sep 17 00:00:00 2001 From: Alex Lavaee Date: Thu, 23 Apr 2026 03:49:04 +0000 Subject: [PATCH 1/2] feat(deep-research): wire D5 prose-guard + D2 history-brief into copilot and opencode SDKs - Import deriveHistoryBrief, PROSE_GUARD_RETRY_PROMPT, queryWithProseGuard from ../../_context/index.ts in both copilot/index.ts and opencode/index.ts - D5 (prose-guard retry): wrap all 6 specialist send/prompt call sites in queryWithProseGuard so empty/whitespace responses trigger one retry with PROSE_GUARD_RETRY_PROMPT. Stages covered: history-locator, history-analyzer, locator-i, pattern-finder-i, analyzer-i, online-researcher-i. - D2 (history brief): derive priorResearchBrief = deriveHistoryBrief(historyOverview) after the history pipeline completes; pass it into buildLocatorPrompt and buildAnalyzerPrompt for every per-partition stage. - Copilot: prose-guard wraps s.session.send + getMessages; getText uses existing getAssistantText helper; save uses final getMessages() call. - OpenCode: prose-guard captures lastResult via mutable variable so s.save receives the real result.data! without an extra network round-trip; getText uses existing extractResponseText helper. Also stage prerequisite changes: _context/ helpers module, updated prompts.ts (priorResearchBrief types), and claude/index.ts reference implementation. All checks pass: bun typecheck, bun lint (0 warnings), bun test _context/ (26/26) --- .../builtin/_context/_context.test.ts | 220 +++++++++++++ src/sdk/workflows/builtin/_context/budget.ts | 48 +++ src/sdk/workflows/builtin/_context/index.ts | 3 + src/sdk/workflows/builtin/_context/masking.ts | 284 +++++++++++++++++ .../workflows/builtin/_context/scratchpad.ts | 218 +++++++++++++ .../deep-research-codebase/claude/index.ts | 217 ++++++++----- .../deep-research-codebase/copilot/index.ts | 181 +++++++---- .../deep-research-codebase/helpers/prompts.ts | 106 +++++++ .../deep-research-codebase/opencode/index.ts | 293 ++++++++++++------ 9 files changed, 1340 insertions(+), 230 deletions(-) create mode 100644 src/sdk/workflows/builtin/_context/_context.test.ts create mode 100644 src/sdk/workflows/builtin/_context/budget.ts create mode 100644 src/sdk/workflows/builtin/_context/index.ts create mode 100644 src/sdk/workflows/builtin/_context/masking.ts create mode 100644 src/sdk/workflows/builtin/_context/scratchpad.ts diff --git a/src/sdk/workflows/builtin/_context/_context.test.ts b/src/sdk/workflows/builtin/_context/_context.test.ts new file mode 100644 index 000000000..82d33d94c --- /dev/null +++ b/src/sdk/workflows/builtin/_context/_context.test.ts @@ -0,0 +1,220 @@ +import { describe, test, expect } from "bun:test"; +import { + compactDiffStat, + compactReminder, + compactUncommitted, + deriveHistoryBrief, + infraInvalidationPaths, + isInfraPath, + maskChangeset, + shouldReRunInfraDiscovery, + truncateMarkdownReport, +} from "./masking.ts"; +import { + appendToSection, + detectSpecPath, + extractSection, + replaceSection, +} from "./scratchpad.ts"; + +describe("isInfraPath", () => { + test("matches lockfiles, manifests, configs, CI, agent instructions", () => { + const hits = [ + "package.json", + "packages/foo/package.json", + "bun.lock", + "bun.lockb", + "go.sum", + "Cargo.lock", + "pnpm-lock.yaml", + "tsconfig.json", + "tsconfig.build.json", + "vitest.config.ts", + "jest.config.cjs", + "eslint.config.mjs", + "biome.json", + ".eslintrc.json", + ".github/workflows/ci.yml", + ".gitlab-ci.yml", + "Jenkinsfile", + "CLAUDE.md", + "AGENTS.md", + ".claude/settings.json", + ".github/copilot-instructions.md", + ".github/agents/reviewer.md", + ".agents/skills/foo.md", + ]; + for (const p of hits) expect(isInfraPath(p)).toBe(true); + }); + + test("does NOT match ordinary source files", () => { + const misses = [ + "src/index.ts", + "src/package/foo.ts", + "docs/readme.md", + "tests/auth.test.ts", + ]; + for (const p of misses) expect(isInfraPath(p)).toBe(false); + }); +}); + +describe("infraInvalidationPaths / shouldReRunInfraDiscovery", () => { + const cs = (opts: Partial<{ nameStatus: string; uncommitted: string }>) => ({ + baseBranch: "main", + diffStat: "", + uncommitted: opts.uncommitted ?? "", + nameStatus: opts.nameStatus ?? "", + errors: [], + }); + + test("untracked infra file in `git status -s` triggers invalidation", () => { + const result = infraInvalidationPaths(cs({ uncommitted: "?? package.json" })); + expect(result).toEqual(["package.json"]); + expect(shouldReRunInfraDiscovery(cs({ uncommitted: "?? package.json" }))).toBe(true); + }); + + test("renames use destination path", () => { + const result = infraInvalidationPaths( + cs({ nameStatus: "R100\told.ts\tnew.ts\nM\ttsconfig.json" }), + ); + expect(result).toContain("tsconfig.json"); + }); + + test("pure source-file changes do NOT trigger invalidation", () => { + expect( + shouldReRunInfraDiscovery( + cs({ + nameStatus: "M\tsrc/auth.ts\nA\tsrc/newfile.ts", + uncommitted: " M src/auth.ts", + }), + ), + ).toBe(false); + }); +}); + +describe("compactDiffStat", () => { + test("preserves output when under top-N", () => { + const input = " file.ts | 5 +++++\n 1 file changed, 5 insertions(+)"; + expect(compactDiffStat(input)).toBe(input); + }); + + test("keeps top-N by churn + elision marker", () => { + const lines: string[] = []; + for (let i = 0; i < 30; i++) lines.push(` file${i}.ts | ${i + 1} +++`); + lines.push(" 30 files changed, 465 insertions(+)"); + const out = compactDiffStat(lines.join("\n")); + expect(out).toContain("30 files changed"); + expect(out).toContain("additional files elided"); + expect(out).toContain("file29.ts"); + }); +}); + +describe("compactUncommitted — untracked safety", () => { + test("ALL ?? entries retained", () => { + const lines: string[] = []; + for (let i = 0; i < 50; i++) lines.push(` M staged${i}.ts`); + lines.push("?? untracked-1.ts"); + lines.push("?? untracked-2.ts"); + const out = compactUncommitted(lines.join("\n")); + expect(out).toContain("?? untracked-1.ts"); + expect(out).toContain("?? untracked-2.ts"); + expect(out).toContain("additional staged/modified entries elided"); + }); + + test("no elision when under threshold", () => { + const input = " M a.ts\n M b.ts\n?? c.ts"; + expect(compactUncommitted(input)).toBe(input); + }); +}); + +describe("maskChangeset", () => { + test("short-circuits for small changesets", () => { + const cs = { + baseBranch: "main", + diffStat: "small", + uncommitted: "", + nameStatus: "", + errors: [], + }; + expect(maskChangeset(cs)).toBe(cs); + }); +}); + +describe("truncateMarkdownReport", () => { + test("returns verbatim when under cap", () => { + expect(truncateMarkdownReport("short", 100)).toBe("short"); + }); + + test("inserts marker with head + tail when over cap", () => { + const big = "a".repeat(20_000); + const out = truncateMarkdownReport(big, 1000); + expect(out.length).toBeLessThanOrEqual(1200); + expect(out).toContain("[… truncated"); + }); +}); + +describe("deriveHistoryBrief", () => { + test("prefers Synthesis section", () => { + const input = `### Documents Reviewed\n- \`a.md\`\n\n### Synthesis\nthe real content lives here`; + const out = deriveHistoryBrief(input, 50); + expect(out).toBe("the real content lives here"); + }); + + test("caps at maxWords", () => { + const many = Array(200).fill("word").join(" "); + const out = deriveHistoryBrief(many, 10); + expect(out.endsWith("…")).toBe(true); + }); + + test("empty input → empty output", () => { + expect(deriveHistoryBrief("")).toBe(""); + }); +}); + +describe("compactReminder", () => { + test("joins with pipes", () => { + expect( + compactReminder({ intent: "refactor auth", iteration: 3, extra: "+5 files" }), + ).toBe("iteration 3 | refactor auth | +5 files"); + }); +}); + +describe("scratchpad section utilities", () => { + const sample = `# Heading\n\n## A\nfirst\n\n## B\nsecond\n\n## C\nthird\n`; + + test("extractSection", () => { + expect(extractSection(sample, "B")).toBe("second"); + expect(extractSection(sample, "missing")).toBe(""); + }); + + test("replaceSection preserves order", () => { + const out = replaceSection(sample, "B", "REPLACED"); + expect(extractSection(out, "B")).toBe("REPLACED"); + expect(extractSection(out, "A")).toBe("first"); + expect(extractSection(out, "C")).toBe("third"); + }); + + test("appendToSection concatenates", () => { + const out = appendToSection(sample, "A", "appended"); + expect(extractSection(out, "A")).toBe("first\n\nappended"); + }); +}); + +describe("detectSpecPath", () => { + test.each([ + ["specs/foo.md", "specs/foo.md"], + ["./notes.txt", "./notes.txt"], + ["/abs/path.md", "/abs/path.md"], + ["~/docs/thing.rst", "~/docs/thing.rst"], + ])("detects path %p", (input, expected) => { + expect(detectSpecPath(input)).toBe(expected); + }); + + test.each([ + ["# Full RFC\n\nThis is inline."], + ["Multi-line\nprose content."], + [""], + ])("rejects non-path %p", (input) => { + expect(detectSpecPath(input)).toBeNull(); + }); +}); diff --git a/src/sdk/workflows/builtin/_context/budget.ts b/src/sdk/workflows/builtin/_context/budget.ts new file mode 100644 index 000000000..e94c004ac --- /dev/null +++ b/src/sdk/workflows/builtin/_context/budget.ts @@ -0,0 +1,48 @@ +/** + * Thresholds and budget utilities shared by builtin workflows. + * + * All thresholds are **heuristics** — tuned from practical observation of + * Ralph and deep-research-codebase runs, not derived from a model-specific + * benchmark. Keep them together here so a single change in policy updates + * every consumer. + */ + +/** Approx chars-per-token for English + code mix. Used for budget guards only. */ +export const CHARS_PER_TOKEN = 4; + +/** Estimate token count from a string. Deliberately conservative. */ +export function approxTokens(text: string): number { + return Math.ceil(text.length / CHARS_PER_TOKEN); +} + +/** Fraction of effective window at which compaction should trigger. */ +export const COMPACT_TRIGGER_FRACTION = 0.7; + +/** Partition file-LOC above which a file is flagged as "huge" for readers. */ +export const HUGE_FILE_LOC = 5_000; + +/** + * Aggregator pre-flight: when the sum of per-partition scratch-file sizes + * exceeds this many chars, compact each oversized scratch before the + * aggregator reads them. Empirically chosen — ~150K chars ≈ 37K tokens. + */ +export const SCRATCH_COMPACT_THRESHOLD = 150_000; + +/** + * Debugger report fallback cap. When `extractMarkdownBlock` can't find a + * fenced ```markdown block, we keep head+tail up to this many chars to + * avoid injecting a 30K-token raw transcript into the next planner prompt. + */ +export const MAX_DEBUGGER_REPORT_CHARS = 8_000; + +/** Changeset-masking trigger: above this char count we start compacting. */ +export const CHANGESET_MASK_THRESHOLD = 8_000; + +/** Max diff-stat entries retained verbatim after masking. */ +export const DIFF_STAT_TOP_N = 20; + +/** Max staged-modification `uncommitted` entries retained verbatim. */ +export const UNCOMMITTED_STAGED_TOP_N = 30; + +/** History-brief cap for deep-research-codebase explorer injection. */ +export const HISTORY_BRIEF_MAX_WORDS = 150; diff --git a/src/sdk/workflows/builtin/_context/index.ts b/src/sdk/workflows/builtin/_context/index.ts new file mode 100644 index 000000000..2405fb4fa --- /dev/null +++ b/src/sdk/workflows/builtin/_context/index.ts @@ -0,0 +1,3 @@ +export * from "./budget.ts"; +export * from "./masking.ts"; +export * from "./scratchpad.ts"; diff --git a/src/sdk/workflows/builtin/_context/masking.ts b/src/sdk/workflows/builtin/_context/masking.ts new file mode 100644 index 000000000..4a19100aa --- /dev/null +++ b/src/sdk/workflows/builtin/_context/masking.ts @@ -0,0 +1,284 @@ +/** + * Shared masking + guard utilities used by Ralph and deep-research-codebase. + * + * Keep every function here **pure** (no I/O except the explicit session + * wrappers) so they can be unit-tested with `bun test` without spinning + * up SDK clients. + */ + +import { + CHANGESET_MASK_THRESHOLD, + DIFF_STAT_TOP_N, + HISTORY_BRIEF_MAX_WORDS, + MAX_DEBUGGER_REPORT_CHARS, + UNCOMMITTED_STAGED_TOP_N, +} from "./budget.ts"; + +// ============================================================================ +// CHANGESET MASKING +// ============================================================================ + +export interface BranchChangeset { + baseBranch: string; + diffStat: string; + uncommitted: string; + nameStatus: string; + errors: string[]; +} + +/** + * Compact a large branch changeset while preserving correctness-critical + * signal. The key invariant is that **untracked files** (`??` entries in + * `git status -s`) only ever appear in the `uncommitted` field, because + * `git diff --stat` doesn't surface them. Wholesale dropping `uncommitted` + * on later iterations would hide brand-new untracked source/test files + * from the reviewer — a silent correctness regression. + */ +export function maskChangeset(cs: BranchChangeset): BranchChangeset { + const total = + cs.diffStat.length + cs.uncommitted.length + cs.nameStatus.length; + if (total <= CHANGESET_MASK_THRESHOLD) return cs; + + return { + ...cs, + diffStat: compactDiffStat(cs.diffStat), + uncommitted: compactUncommitted(cs.uncommitted), + // nameStatus is authoritative + cheap — leave verbatim. + }; +} + +export function compactDiffStat(diffStat: string): string { + if (!diffStat.trim()) return diffStat; + const lines = diffStat.split("\n"); + const fileLines: { raw: string; churn: number }[] = []; + const summaryLines: string[] = []; + + for (const line of lines) { + if (/\d+\s+files?\s+changed/.test(line)) { + summaryLines.push(line); + continue; + } + const match = line.match(/\|\s*(\d+)/); + if (!match || !match[1]) continue; + fileLines.push({ raw: line, churn: parseInt(match[1], 10) }); + } + + if (fileLines.length <= DIFF_STAT_TOP_N) return diffStat; + + fileLines.sort((a, b) => b.churn - a.churn); + const kept = fileLines.slice(0, DIFF_STAT_TOP_N); + const elided = fileLines.length - kept.length; + + const parts = [ + ...kept.map((f) => f.raw), + ` … ${elided} additional files elided by masker …`, + ...summaryLines, + ]; + return parts.join("\n"); +} + +export function compactUncommitted(uncommitted: string): string { + if (!uncommitted.trim()) return uncommitted; + const lines = uncommitted.split("\n").filter((l) => l.length > 0); + const untracked: string[] = []; + const others: string[] = []; + + for (const line of lines) { + if (line.startsWith("?? ")) untracked.push(line); + else others.push(line); + } + + if (others.length <= UNCOMMITTED_STAGED_TOP_N) return uncommitted; + + const keptOthers = others.slice(0, UNCOMMITTED_STAGED_TOP_N); + const elidedOthers = others.length - keptOthers.length; + const summary = ` … ${elidedOthers} additional staged/modified entries elided (all untracked entries retained) …`; + + return [...untracked, ...keptOthers, summary].join("\n"); +} + +// ============================================================================ +// INFRA-DISCOVERY INVALIDATION +// ============================================================================ + +const INFRA_PATH_PATTERNS: RegExp[] = [ + // Manifests + /(^|\/)package\.json$/, + /(^|\/)pyproject\.toml$/, + /(^|\/)Cargo\.toml$/, + /(^|\/)go\.mod$/, + /(^|\/)Gemfile$/, + /(^|\/)composer\.json$/, + // Lockfiles + /(^|\/)bun\.lockb?$/, + /(^|\/)package-lock\.json$/, + /(^|\/)yarn\.lock$/, + /(^|\/)pnpm-lock\.yaml$/, + /(^|\/)Cargo\.lock$/, + /(^|\/)go\.sum$/, + /(^|\/)Gemfile\.lock$/, + /(^|\/)composer\.lock$/, + /(^|\/)uv\.lock$/, + /(^|\/)poetry\.lock$/, + // Build configs + /(^|\/)tsconfig[^/]*\.json$/, + /(^|\/)vite\.config\.[cm]?[jt]sx?$/, + /(^|\/)esbuild\.(config|mjs|js)[^/]*$/, + /(^|\/)webpack\.config\.[cm]?[jt]sx?$/, + /(^|\/)rollup\.config\.[cm]?[jt]sx?$/, + /(^|\/)Makefile$/, + // Test configs + /(^|\/)(vitest|jest|playwright)\.config\.[cm]?[jt]sx?$/, + /(^|\/)\.mocharc\.[^/]+$/, + /(^|\/)pytest\.ini$/, + /(^|\/)tox\.ini$/, + // Lint/format configs + /(^|\/)\.eslintrc(\.[^/]+)?$/, + /(^|\/)eslint\.config\.[cm]?[jt]sx?$/, + /(^|\/)biome\.json$/, + /(^|\/)\.prettierrc(\.[^/]+)?$/, + /(^|\/)oxlint\.json$/, + /(^|\/)\.editorconfig$/, + // CI + /^\.github\/workflows\//, + /^\.gitlab-ci\.yml$/, + /^Jenkinsfile$/, + /^\.circleci\//, + /^\.buildkite\//, + // Agent instructions + /(^|\/)CLAUDE\.md$/, + /(^|\/)AGENTS\.md$/, + /^\.claude\//, + /^\.opencode\//, + /^\.github\/copilot-instructions\.md$/, + /^\.github\/agents\//, + /^\.agents\//, +]; + +export function isInfraPath(path: string): boolean { + return INFRA_PATH_PATTERNS.some((re) => re.test(path)); +} + +function parseNameStatus(nameStatus: string): string[] { + const paths: string[] = []; + for (const line of nameStatus.split("\n")) { + if (!line.trim()) continue; + const parts = line.split("\t"); + if (parts.length >= 2) { + const p = parts[parts.length - 1]; + if (p) paths.push(p); + } + } + return paths; +} + +function parseStatusS(statusS: string): string[] { + const paths: string[] = []; + for (const line of statusS.split("\n")) { + if (line.length < 4) continue; + const rest = line.slice(3); + const renameMatch = rest.match(/^(.+)\s->\s(.+)$/); + if (renameMatch && renameMatch[2]) { + paths.push(renameMatch[2].trim()); + } else { + paths.push(rest.trim()); + } + } + return paths; +} + +export function infraInvalidationPaths(cs: BranchChangeset): string[] { + const touched = new Set([ + ...parseNameStatus(cs.nameStatus), + ...parseStatusS(cs.uncommitted), + ]); + const hits: string[] = []; + for (const p of touched) { + if (isInfraPath(p)) hits.push(p); + } + return hits; +} + +export function shouldReRunInfraDiscovery(cs: BranchChangeset): boolean { + return infraInvalidationPaths(cs).length > 0; +} + +// ============================================================================ +// MARKDOWN-REPORT TRUNCATION +// ============================================================================ + +export function truncateMarkdownReport( + content: string, + maxChars: number = MAX_DEBUGGER_REPORT_CHARS, +): string { + if (content.length <= maxChars) return content; + const headLen = Math.floor(maxChars * 0.6); + const tailLen = maxChars - headLen - 128; + const elided = content.length - headLen - tailLen; + const head = content.slice(0, headLen); + const tail = content.slice(-tailLen); + return ( + head + + `\n\n[… truncated ${elided} chars by truncateMarkdownReport — original was ${content.length} chars …]\n\n` + + tail + ); +} + +// ============================================================================ +// HISTORY BRIEF (deep-research) +// ============================================================================ + +export function deriveHistoryBrief( + historyOverview: string, + maxWords: number = HISTORY_BRIEF_MAX_WORDS, +): string { + const trimmed = historyOverview.trim(); + if (!trimmed) return ""; + + const synthMatch = trimmed.match(/###\s+Synthesis\s*\n([\s\S]*?)(?=\n###\s|$)/); + const src = synthMatch?.[1]?.trim() || trimmed; + + const words = src.split(/\s+/).filter(Boolean); + if (words.length <= maxWords) return src; + return words.slice(0, maxWords).join(" ") + " …"; +} + +// ============================================================================ +// COMPACT REMINDER +// ============================================================================ + +export function compactReminder(opts: { + intent: string; + iteration?: number; + extra?: string; +}): string { + const parts: string[] = []; + if (opts.iteration !== undefined) parts.push(`iteration ${opts.iteration}`); + parts.push(opts.intent.trim().replace(/\s+/g, " ").slice(0, 200)); + if (opts.extra) parts.push(opts.extra); + return parts.join(" | "); +} + +// ============================================================================ +// PROSE GUARD (specialist transcript retry) +// ============================================================================ + +export interface ProseGuardOpts { + query: () => Promise; + getText: (result: T) => string; + retry: () => Promise; +} + +export async function queryWithProseGuard( + opts: ProseGuardOpts, +): Promise<{ text: string; attempts: 1 | 2 }> { + const first = await opts.query(); + const firstText = opts.getText(first); + if (firstText.trim().length > 0) return { text: firstText, attempts: 1 }; + const second = await opts.retry(); + return { text: opts.getText(second), attempts: 2 }; +} + +export const PROSE_GUARD_RETRY_PROMPT = + "Emit the required prose summary now — do not end on a tool call. " + + "A short markdown report following the earlier output format is sufficient."; diff --git a/src/sdk/workflows/builtin/_context/scratchpad.ts b/src/sdk/workflows/builtin/_context/scratchpad.ts new file mode 100644 index 000000000..def373250 --- /dev/null +++ b/src/sdk/workflows/builtin/_context/scratchpad.ts @@ -0,0 +1,218 @@ +/** + * Ralph per-run scratchpad. + * + * Persistent markdown artifact at `.atomic/ralph//state.md`. + * Survives every iteration of the plan → orchestrate → review → debug loop + * so the next-iteration planner can see the full prior-design history, + * cumulative files modified, rejected approaches, and open questions — + * rather than re-deriving them from just the latest debugger report. + * + * **Single-writer discipline:** all appenders run inside the outer + * `.run()` callback AFTER each stage resolves. Never append from inside + * a headless fan-out — that re-introduces race conditions. + */ + +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export type ScratchpadHandle = { + filePath: string; + dir: string; + sessionId: string; +}; + +export async function initScratchpad(opts: { + sessionId: string; + projectRoot: string; + originalSpec: string; +}): Promise { + const dir = path.join(opts.projectRoot, ".atomic", "ralph", opts.sessionId); + await mkdir(dir, { recursive: true }); + const filePath = path.join(dir, "state.md"); + + try { + await readFile(filePath, "utf8"); + } catch { + const seed = [ + `# Ralph Run State (session \`${opts.sessionId}\`)`, + ``, + `_Persistent scratchpad shared across all iterations. Written only by_`, + `_the outer \`.run()\` callback — single-writer discipline._`, + ``, + `## Session Intent`, + opts.originalSpec.trim(), + ``, + `## Prior Spec Path`, + `_(set if the planner short-circuited to a file path)_`, + ``, + `## Prior RFCs`, + `_(append-only, newest last; each wrapped in a \`\`\`markdown block)_`, + ``, + `## Files Modified`, + `_(cumulative, deduped by repo-relative path)_`, + ``, + `## Decisions Made`, + ``, + `## Rejected Approaches`, + ``, + `## Open Questions`, + ``, + `## Debugger Reports`, + `_(one per iteration that produced findings)_`, + ``, + ].join("\n"); + await writeFile(filePath, seed, "utf8"); + } + + return { filePath, dir, sessionId: opts.sessionId }; +} + +export async function readScratchpad(h: ScratchpadHandle): Promise { + return readFile(h.filePath, "utf8"); +} + +/** + * Heuristic: matches the `Spec Path Short-Circuit` contract defined in + * `ralph/helpers/prompts.ts`. Returns trimmed path or null. + */ +export function detectSpecPath(plannerOutput: string): string | null { + const trimmed = plannerOutput.trim(); + if (!trimmed) return null; + if (/\n/.test(trimmed)) return null; + const looksLikePath = + /^(\/|\.\/|~\/)/.test(trimmed) || + /\.(md|txt|rst|adoc|org)$/i.test(trimmed); + return looksLikePath ? trimmed : null; +} + +export async function recordPlannerOutput( + h: ScratchpadHandle, + iteration: number, + plannerOutput: string, +): Promise { + const pathLike = detectSpecPath(plannerOutput); + const current = await readScratchpad(h); + + if (pathLike) { + const updated = replaceSection( + current, + "Prior Spec Path", + `\`${pathLike}\` _(from iteration ${iteration})_`, + ); + await writeFile(h.filePath, updated, "utf8"); + return; + } + + const block = [ + `### Iteration ${iteration}`, + "```markdown", + plannerOutput.trim(), + "```", + ].join("\n"); + const updated = appendToSection(current, "Prior RFCs", block); + await writeFile(h.filePath, updated, "utf8"); +} + +export async function recordDebuggerReport( + h: ScratchpadHandle, + iteration: number, + report: string, +): Promise { + const current = await readScratchpad(h); + const block = [ + `### Iteration ${iteration}`, + "```markdown", + report.trim(), + "```", + ].join("\n"); + const updated = appendToSection(current, "Debugger Reports", block); + await writeFile(h.filePath, updated, "utf8"); +} + +export async function recordFilesModified( + h: ScratchpadHandle, + iteration: number, + paths: string[], +): Promise { + if (paths.length === 0) return; + const current = await readScratchpad(h); + const existing = extractSection(current, "Files Modified") + .split("\n") + .map((l) => l.match(/^- `([^`]+)`/)?.[1]) + .filter((p): p is string => !!p); + const set = new Set(existing); + const added: string[] = []; + for (const p of paths) { + if (!set.has(p)) { + set.add(p); + added.push(p); + } + } + if (added.length === 0) return; + const newBody = [...set] + .map((p) => `- \`${p}\``) + .concat([` _(iteration ${iteration} added ${added.length})_`]) + .join("\n"); + const updated = replaceSection(current, "Files Modified", newBody); + await writeFile(h.filePath, updated, "utf8"); +} + +export async function latestPriorRFC( + h: ScratchpadHandle, +): Promise { + const current = await readScratchpad(h); + const body = extractSection(current, "Prior RFCs"); + const blocks = [...body.matchAll(/```markdown\s*\n([\s\S]*?)\n```/g)]; + const last = blocks[blocks.length - 1]?.[1]?.trim(); + return last && last.length > 0 ? last : null; +} + +export async function priorSpecPath( + h: ScratchpadHandle, +): Promise { + const current = await readScratchpad(h); + const body = extractSection(current, "Prior Spec Path"); + const match = body.match(/`([^`]+)`/); + return match?.[1] ?? null; +} + +// ============================================================================ +// SECTION UTILITIES +// ============================================================================ + +export function extractSection(content: string, name: string): string { + const re = new RegExp( + `(^|\\n)##\\s+${escapeRe(name)}\\s*\\n([\\s\\S]*?)(?=\\n##\\s|$)`, + ); + const match = content.match(re); + return match?.[2]?.trim() ?? ""; +} + +export function replaceSection( + content: string, + name: string, + newBody: string, +): string { + const re = new RegExp( + `((^|\\n)##\\s+${escapeRe(name)}\\s*\\n)([\\s\\S]*?)(?=\\n##\\s|$)`, + ); + if (re.test(content)) { + return content.replace(re, `$1${newBody}\n`); + } + const suffix = content.endsWith("\n") ? "" : "\n"; + return `${content}${suffix}\n## ${name}\n${newBody}\n`; +} + +export function appendToSection( + content: string, + name: string, + block: string, +): string { + const existing = extractSection(content, name); + const body = existing ? `${existing}\n\n${block}` : block; + return replaceSection(content, name, body); +} + +function escapeRe(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/src/sdk/workflows/builtin/deep-research-codebase/claude/index.ts b/src/sdk/workflows/builtin/deep-research-codebase/claude/index.ts index 94896f7ac..1d5d9f38e 100644 --- a/src/sdk/workflows/builtin/deep-research-codebase/claude/index.ts +++ b/src/sdk/workflows/builtin/deep-research-codebase/claude/index.ts @@ -82,6 +82,11 @@ import { slugifyPrompt, } from "../helpers/prompts.ts"; import { writeExplorerScratchFile } from "../helpers/scratch.ts"; +import { + deriveHistoryBrief, + PROSE_GUARD_RETRY_PROMPT, + queryWithProseGuard, +} from "../../_context/index.ts"; /** * Shared SDK options for every sub-agent dispatch. `permissionMode` + @@ -188,35 +193,53 @@ export default defineWorkflow({ {}, {}, async (s) => { - const result = await s.session.query( - buildHistoryLocatorPrompt({ question: prompt }), - { agent: "codebase-research-locator", ...SUBAGENT_OPTS }, - ); - s.save(s.sessionId); - return extractAssistantText(result, 0); - }, - ); + const { text } = await queryWithProseGuard({ + query: () => + s.session.query(buildHistoryLocatorPrompt({ question: prompt }), { + agent: "codebase-research-locator", + ...SUBAGENT_OPTS, + }), + getText: (r) => extractAssistantText(r, 0), + retry: () => + s.session.query(PROSE_GUARD_RETRY_PROMPT, { + agent: "codebase-research-locator", + ...SUBAGENT_OPTS, + }), + }); + s.save(s.sessionId); + return text; + }, + ); - const historyAnalyzer = await ctx.stage( - { - name: "history-analyzer", - headless: true, - description: "Synthesize prior research (codebase-research-analyzer)", - }, - {}, - {}, - async (s) => { - const result = await s.session.query( - buildHistoryAnalyzerPrompt({ - question: prompt, - locatorOutput: historyLocator.result, + const historyAnalyzer = await ctx.stage( + { + name: "history-analyzer", + headless: true, + description: "Synthesize prior research (codebase-research-analyzer)", + }, + {}, + {}, + async (s) => { + const { text } = await queryWithProseGuard({ + query: () => + s.session.query( + buildHistoryAnalyzerPrompt({ + question: prompt, + locatorOutput: historyLocator.result, + }), + { agent: "codebase-research-analyzer", ...SUBAGENT_OPTS }, + ), + getText: (r) => extractAssistantText(r, 0), + retry: () => + s.session.query(PROSE_GUARD_RETRY_PROMPT, { + agent: "codebase-research-analyzer", + ...SUBAGENT_OPTS, }), - { agent: "codebase-research-analyzer", ...SUBAGENT_OPTS }, - ); - s.save(s.sessionId); - return extractAssistantText(result, 0); - }, - ); + }); + s.save(s.sessionId); + return text; + }, + ); return historyAnalyzer.result; })(), @@ -225,6 +248,10 @@ export default defineWorkflow({ const { partitions, explorerCount, scratchDir, totalLoc, totalFiles } = scout.result; + // D2: derive a short brief from the history-analyzer output to inject as + // into per-partition locator + analyzer prompts. + const priorResearchBrief = deriveHistoryBrief(historyOverview); + // Pull the scout transcript ONCE so every per-partition specialist can // embed the architectural orientation in its prompt. The scout has // completed by the time we get here (we're past Promise.all), so this @@ -259,19 +286,29 @@ export default defineWorkflow({ {}, {}, async (s) => { - const result = await s.session.query( - buildLocatorPrompt({ - question: prompt, - partition, - scoutOverview, - index: i, - total: explorerCount, + const { text } = await queryWithProseGuard({ + query: () => + s.session.query( + buildLocatorPrompt({ + question: prompt, + partition, + scoutOverview, + index: i, + total: explorerCount, + priorResearchBrief, + }), + { agent: "codebase-locator", ...SUBAGENT_OPTS }, + ), + getText: (r) => extractAssistantText(r, 0), + retry: () => + s.session.query(PROSE_GUARD_RETRY_PROMPT, { + agent: "codebase-locator", + ...SUBAGENT_OPTS, }), - { agent: "codebase-locator", ...SUBAGENT_OPTS }, - ); - s.save(s.sessionId); - return extractAssistantText(result, 0); - }, + }); + s.save(s.sessionId); + return text; + }, ), ctx.stage( { @@ -282,19 +319,28 @@ export default defineWorkflow({ {}, {}, async (s) => { - const result = await s.session.query( - buildPatternFinderPrompt({ - question: prompt, - partition, - scoutOverview, - index: i, - total: explorerCount, + const { text } = await queryWithProseGuard({ + query: () => + s.session.query( + buildPatternFinderPrompt({ + question: prompt, + partition, + scoutOverview, + index: i, + total: explorerCount, + }), + { agent: "codebase-pattern-finder", ...SUBAGENT_OPTS }, + ), + getText: (r) => extractAssistantText(r, 0), + retry: () => + s.session.query(PROSE_GUARD_RETRY_PROMPT, { + agent: "codebase-pattern-finder", + ...SUBAGENT_OPTS, }), - { agent: "codebase-pattern-finder", ...SUBAGENT_OPTS }, - ); - s.save(s.sessionId); - return extractAssistantText(result, 0); - }, + }); + s.save(s.sessionId); + return text; + }, ), ]); @@ -312,20 +358,30 @@ export default defineWorkflow({ {}, {}, async (s) => { - const result = await s.session.query( - buildAnalyzerPrompt({ - question: prompt, - partition, - locatorOutput, - scoutOverview, - index: i, - total: explorerCount, + const { text } = await queryWithProseGuard({ + query: () => + s.session.query( + buildAnalyzerPrompt({ + question: prompt, + partition, + locatorOutput, + scoutOverview, + index: i, + total: explorerCount, + priorResearchBrief, + }), + { agent: "codebase-analyzer", ...SUBAGENT_OPTS }, + ), + getText: (r) => extractAssistantText(r, 0), + retry: () => + s.session.query(PROSE_GUARD_RETRY_PROMPT, { + agent: "codebase-analyzer", + ...SUBAGENT_OPTS, }), - { agent: "codebase-analyzer", ...SUBAGENT_OPTS }, - ); - s.save(s.sessionId); - return extractAssistantText(result, 0); - }, + }); + s.save(s.sessionId); + return text; + }, ), ctx.stage( { @@ -336,19 +392,28 @@ export default defineWorkflow({ {}, {}, async (s) => { - const result = await s.session.query( - buildOnlineResearcherPrompt({ - question: prompt, - partition, - locatorOutput, - index: i, - total: explorerCount, + const { text } = await queryWithProseGuard({ + query: () => + s.session.query( + buildOnlineResearcherPrompt({ + question: prompt, + partition, + locatorOutput, + index: i, + total: explorerCount, + }), + { agent: "codebase-online-researcher", ...SUBAGENT_OPTS }, + ), + getText: (r) => extractAssistantText(r, 0), + retry: () => + s.session.query(PROSE_GUARD_RETRY_PROMPT, { + agent: "codebase-online-researcher", + ...SUBAGENT_OPTS, }), - { agent: "codebase-online-researcher", ...SUBAGENT_OPTS }, - ); - s.save(s.sessionId); - return extractAssistantText(result, 0); - }, + }); + s.save(s.sessionId); + return text; + }, ), ]); diff --git a/src/sdk/workflows/builtin/deep-research-codebase/copilot/index.ts b/src/sdk/workflows/builtin/deep-research-codebase/copilot/index.ts index becdf8277..ca88f94c8 100644 --- a/src/sdk/workflows/builtin/deep-research-codebase/copilot/index.ts +++ b/src/sdk/workflows/builtin/deep-research-codebase/copilot/index.ts @@ -52,6 +52,11 @@ import { slugifyPrompt, } from "../helpers/prompts.ts"; import { writeExplorerScratchFile } from "../helpers/scratch.ts"; +import { + deriveHistoryBrief, + PROSE_GUARD_RETRY_PROMPT, + queryWithProseGuard, +} from "../../_context/index.ts"; /** * Concatenate every top-level assistant turn's non-empty content. The final @@ -157,12 +162,21 @@ export default defineWorkflow({ {}, { agent: "codebase-research-locator" }, async (s) => { - await s.session.send({ - prompt: buildHistoryLocatorPrompt({ question: prompt }), + const { text } = await queryWithProseGuard({ + query: async () => { + await s.session.send({ + prompt: buildHistoryLocatorPrompt({ question: prompt }), + }); + return await s.session.getMessages(); + }, + getText: getAssistantText, + retry: async () => { + await s.session.send({ prompt: PROSE_GUARD_RETRY_PROMPT }); + return await s.session.getMessages(); + }, }); - const messages = await s.session.getMessages(); - s.save(messages); - return getAssistantText(messages); + s.save(await s.session.getMessages()); + return text; }, ); @@ -175,15 +189,24 @@ export default defineWorkflow({ {}, { agent: "codebase-research-analyzer" }, async (s) => { - await s.session.send({ - prompt: buildHistoryAnalyzerPrompt({ - question: prompt, - locatorOutput: historyLocator.result, - }), + const { text } = await queryWithProseGuard({ + query: async () => { + await s.session.send({ + prompt: buildHistoryAnalyzerPrompt({ + question: prompt, + locatorOutput: historyLocator.result, + }), + }); + return await s.session.getMessages(); + }, + getText: getAssistantText, + retry: async () => { + await s.session.send({ prompt: PROSE_GUARD_RETRY_PROMPT }); + return await s.session.getMessages(); + }, }); - const messages = await s.session.getMessages(); - s.save(messages); - return getAssistantText(messages); + s.save(await s.session.getMessages()); + return text; }, ); @@ -194,6 +217,10 @@ export default defineWorkflow({ const { partitions, explorerCount, scratchDir, totalLoc, totalFiles } = scout.result; + // D2: derive a short brief from the history-analyzer output to inject as + // into per-partition locator + analyzer prompts. + const priorResearchBrief = deriveHistoryBrief(historyOverview); + const scoutOverview = (await ctx.transcript(scout)).content; // ── Stage 2: per-partition specialist fan-out ───────────────────────── @@ -213,18 +240,28 @@ export default defineWorkflow({ {}, { agent: "codebase-locator" }, async (s) => { - await s.session.send({ - prompt: buildLocatorPrompt({ - question: prompt, - partition, - scoutOverview, - index: i, - total: explorerCount, - }), + const { text } = await queryWithProseGuard({ + query: async () => { + await s.session.send({ + prompt: buildLocatorPrompt({ + question: prompt, + partition, + scoutOverview, + index: i, + total: explorerCount, + priorResearchBrief, + }), + }); + return await s.session.getMessages(); + }, + getText: getAssistantText, + retry: async () => { + await s.session.send({ prompt: PROSE_GUARD_RETRY_PROMPT }); + return await s.session.getMessages(); + }, }); - const messages = await s.session.getMessages(); - s.save(messages); - return getAssistantText(messages); + s.save(await s.session.getMessages()); + return text; }, ), ctx.stage( @@ -236,18 +273,27 @@ export default defineWorkflow({ {}, { agent: "codebase-pattern-finder" }, async (s) => { - await s.session.send({ - prompt: buildPatternFinderPrompt({ - question: prompt, - partition, - scoutOverview, - index: i, - total: explorerCount, - }), + const { text } = await queryWithProseGuard({ + query: async () => { + await s.session.send({ + prompt: buildPatternFinderPrompt({ + question: prompt, + partition, + scoutOverview, + index: i, + total: explorerCount, + }), + }); + return await s.session.getMessages(); + }, + getText: getAssistantText, + retry: async () => { + await s.session.send({ prompt: PROSE_GUARD_RETRY_PROMPT }); + return await s.session.getMessages(); + }, }); - const messages = await s.session.getMessages(); - s.save(messages); - return getAssistantText(messages); + s.save(await s.session.getMessages()); + return text; }, ), ]); @@ -266,19 +312,29 @@ export default defineWorkflow({ {}, { agent: "codebase-analyzer" }, async (s) => { - await s.session.send({ - prompt: buildAnalyzerPrompt({ - question: prompt, - partition, - locatorOutput, - scoutOverview, - index: i, - total: explorerCount, - }), + const { text } = await queryWithProseGuard({ + query: async () => { + await s.session.send({ + prompt: buildAnalyzerPrompt({ + question: prompt, + partition, + locatorOutput, + scoutOverview, + index: i, + total: explorerCount, + priorResearchBrief, + }), + }); + return await s.session.getMessages(); + }, + getText: getAssistantText, + retry: async () => { + await s.session.send({ prompt: PROSE_GUARD_RETRY_PROMPT }); + return await s.session.getMessages(); + }, }); - const messages = await s.session.getMessages(); - s.save(messages); - return getAssistantText(messages); + s.save(await s.session.getMessages()); + return text; }, ), ctx.stage( @@ -290,18 +346,27 @@ export default defineWorkflow({ {}, { agent: "codebase-online-researcher" }, async (s) => { - await s.session.send({ - prompt: buildOnlineResearcherPrompt({ - question: prompt, - partition, - locatorOutput, - index: i, - total: explorerCount, - }), + const { text } = await queryWithProseGuard({ + query: async () => { + await s.session.send({ + prompt: buildOnlineResearcherPrompt({ + question: prompt, + partition, + locatorOutput, + index: i, + total: explorerCount, + }), + }); + return await s.session.getMessages(); + }, + getText: getAssistantText, + retry: async () => { + await s.session.send({ prompt: PROSE_GUARD_RETRY_PROMPT }); + return await s.session.getMessages(); + }, }); - const messages = await s.session.getMessages(); - s.save(messages); - return getAssistantText(messages); + s.save(await s.session.getMessages()); + return text; }, ), ]); diff --git a/src/sdk/workflows/builtin/deep-research-codebase/helpers/prompts.ts b/src/sdk/workflows/builtin/deep-research-codebase/helpers/prompts.ts index 24c51be3d..dda4b6a14 100644 --- a/src/sdk/workflows/builtin/deep-research-codebase/helpers/prompts.ts +++ b/src/sdk/workflows/builtin/deep-research-codebase/helpers/prompts.ts @@ -38,6 +38,95 @@ */ import type { PartitionUnit } from "./scout.ts"; +import { deriveHistoryBrief, HUGE_FILE_LOC } from "../../_context/index.ts"; + +/** + * Render a `` block for inclusion in explorer prompts. + * Caller is responsible for passing the already-bounded brief string + * (`deriveHistoryBrief(historyOverview)` is the usual producer). Returns + * an empty string when the brief is empty so the caller can inline the + * result without conditional glue. + */ +function renderPriorResearchHint(brief: string | undefined): string { + const trimmed = brief?.trim() ?? ""; + if (!trimmed) return ""; + return [ + ``, + ``, + `Previous research on this topic (kept brief; below the distraction`, + `threshold). Use as a weak prior — trust fresh investigation when they`, + `disagree:`, + ``, + trimmed, + ``, + ].join("\n"); +} + +/** + * Build a `` block listing files in the partition whose LOC + * exceeds {@link HUGE_FILE_LOC}. Readers should offset/limit these rather + * than reading in full. Empty partitions or no-huge-files → empty string. + * + * Note: `PartitionUnit.files` does not carry per-file LOC currently — the + * deterministic producer passes `hugeFiles` directly. This helper renders + * whatever list the caller supplies. + */ +function renderHugeFiles(hugeFiles: string[] | undefined): string { + if (!hugeFiles || hugeFiles.length === 0) return ""; + return [ + ``, + ``, + `The following files in your partition exceed ${HUGE_FILE_LOC.toLocaleString()} lines.`, + `Read them via offset/limit (e.g. Read with \`offset\` + \`limit\`) — reading`, + `them in full will blow your context window:`, + ``, + ...hugeFiles.map((f) => `- \`${f}\``), + ``, + ].join("\n"); +} + +// Re-export for SDK index call sites that need to compute a brief. +export { deriveHistoryBrief, HUGE_FILE_LOC }; + +/** + * Prepended to every stage prompt. Compresses agent prose output to cut + * tokens without losing technical substance. Code blocks, tool-call + * arguments, and structured file outputs remain verbatim. + */ +const CAVEMAN_PREAMBLE = `# Response Style (applies to all prose output) + +Respond terse like smart caveman. All technical substance stay. Only fluff die. + +## Persistence + +ACTIVE EVERY RESPONSE. No revert after many turns. No filler drift. Still active if unsure. + +## Rules + +Drop: articles (a/an/the), filler (just/really/basically/actually/simply), pleasantries (sure/certainly/of course/happy to), hedging. Fragments OK. Short synonyms (big not extensive, fix not "implement a solution for"). Technical terms exact. Code blocks unchanged. Errors quoted exact. File paths, line numbers, symbol names exact. + +Pattern: \`[thing] [action] [reason]. [next step].\` + +Not: "Sure! I'd be happy to help you with that. The issue you're experiencing is likely caused by..." +Yes: "Bug in auth middleware. Token expiry check use \`<\` not \`<=\`. Fix:" + +## Intensity + +Drop articles, fragments OK, short synonyms. + +Example — "Why React component re-render?" +"New object ref each render. Inline object prop = new ref = re-render. Wrap in \`useMemo\`." + +## Auto-Clarity + +Drop caveman for: security warnings, irreversible action confirmations, multi-step sequences where fragment order risks misread, user asks to clarify or repeats question. Resume caveman after clear part done. + +## Boundaries + +Code/commits/PRs: write normal. Structured file outputs (scratch file sections, required headers/templates): write normal — schema wins. Section headers and required output shapes unchanged. + +--- +`; const DOCUMENTARIAN_DISCLAIMER = "You are a documentarian, not a critic. Document what EXISTS — do not " + @@ -98,6 +187,7 @@ export function buildScoutPrompt(opts: { .join("\n"); return [ + CAVEMAN_PREAMBLE, ``, opts.question, ``, @@ -167,6 +257,8 @@ export function buildLocatorPrompt(opts: { scoutOverview: string; index: number; total: number; + /** ≤150-word brief derived from the history-analyzer output (D2). */ + priorResearchBrief?: string; }): string { const assignment = renderPartitionAssignment(opts.partition); const dirs = renderPartitionDirs(opts.partition); @@ -176,6 +268,7 @@ export function buildLocatorPrompt(opts: { : "(scout overview unavailable — proceed without)"; return [ + CAVEMAN_PREAMBLE, ``, opts.question, ``, @@ -189,6 +282,7 @@ export function buildLocatorPrompt(opts: { ``, orientation, ``, + renderPriorResearchHint(opts.priorResearchBrief), ``, ``, `Search ONLY within these directories. Other partitions cover the rest of`, @@ -256,6 +350,7 @@ export function buildPatternFinderPrompt(opts: { : "(scout overview unavailable — proceed without)"; return [ + CAVEMAN_PREAMBLE, ``, opts.question, ``, @@ -313,6 +408,10 @@ export function buildAnalyzerPrompt(opts: { scoutOverview: string; index: number; total: number; + /** Files in this partition whose LOC exceeds HUGE_FILE_LOC (D1). */ + hugeFiles?: string[]; + /** ≤150-word brief derived from the history-analyzer output (D2). */ + priorResearchBrief?: string; }): string { const assignment = renderPartitionAssignment(opts.partition); const orientation = @@ -325,6 +424,7 @@ export function buildAnalyzerPrompt(opts: { : "(locator returned no files — analyse the partition directly)"; return [ + CAVEMAN_PREAMBLE, ``, opts.question, ``, @@ -340,10 +440,12 @@ export function buildAnalyzerPrompt(opts: { ``, orientation, ``, + renderPriorResearchHint(opts.priorResearchBrief), ``, ``, assignment, ``, + renderHugeFiles(opts.hugeFiles), ``, ``, `Verbatim output from the codebase-locator sibling for this partition —`, @@ -420,6 +522,7 @@ export function buildOnlineResearcherPrompt(opts: { : "(locator returned no files)"; return [ + CAVEMAN_PREAMBLE, ``, opts.question, ``, @@ -498,6 +601,7 @@ export function buildHistoryLocatorPrompt(opts: { question: string; }): string { return [ + CAVEMAN_PREAMBLE, ``, opts.question, ``, @@ -561,6 +665,7 @@ export function buildHistoryAnalyzerPrompt(opts: { : "(no prior research surfaced)"; return [ + CAVEMAN_PREAMBLE, ``, opts.question, ``, @@ -647,6 +752,7 @@ export function buildAggregatorPrompt(opts: { : "(no historical research surfaced)"; return [ + CAVEMAN_PREAMBLE, ``, opts.question, ``, diff --git a/src/sdk/workflows/builtin/deep-research-codebase/opencode/index.ts b/src/sdk/workflows/builtin/deep-research-codebase/opencode/index.ts index 778323c1f..d66b4e911 100644 --- a/src/sdk/workflows/builtin/deep-research-codebase/opencode/index.ts +++ b/src/sdk/workflows/builtin/deep-research-codebase/opencode/index.ts @@ -51,6 +51,11 @@ import { slugifyPrompt, } from "../helpers/prompts.ts"; import { writeExplorerScratchFile } from "../helpers/scratch.ts"; +import { + deriveHistoryBrief, + PROSE_GUARD_RETRY_PROMPT, + queryWithProseGuard, +} from "../../_context/index.ts"; /** Filter for text parts only — non-text parts produce [object Object]. */ function extractResponseText( @@ -156,20 +161,35 @@ export default defineWorkflow({ {}, { title: "history-locator" }, async (s) => { - const result = await s.client.session.prompt({ - sessionID: s.session.id, - parts: [ - { - type: "text", - text: buildHistoryLocatorPrompt({ - question: prompt, - }), - }, - ], - agent: "codebase-research-locator", + let lastResult: Awaited>; + const { text } = await queryWithProseGuard({ + query: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [ + { + type: "text", + text: buildHistoryLocatorPrompt({ + question: prompt, + }), + }, + ], + agent: "codebase-research-locator", + }); + return lastResult; + }, + getText: (r) => extractResponseText(r.data!.parts), + retry: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [{ type: "text", text: PROSE_GUARD_RETRY_PROMPT }], + agent: "codebase-research-locator", + }); + return lastResult; + }, }); - s.save(result.data!); - return extractResponseText(result.data!.parts); + s.save(lastResult!.data!); + return text; }, ); @@ -182,21 +202,36 @@ export default defineWorkflow({ {}, { title: "history-analyzer" }, async (s) => { - const result = await s.client.session.prompt({ - sessionID: s.session.id, - parts: [ - { - type: "text", - text: buildHistoryAnalyzerPrompt({ - question: prompt, - locatorOutput: historyLocator.result, - }), - }, - ], - agent: "codebase-research-analyzer", + let lastResult: Awaited>; + const { text } = await queryWithProseGuard({ + query: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [ + { + type: "text", + text: buildHistoryAnalyzerPrompt({ + question: prompt, + locatorOutput: historyLocator.result, + }), + }, + ], + agent: "codebase-research-analyzer", + }); + return lastResult; + }, + getText: (r) => extractResponseText(r.data!.parts), + retry: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [{ type: "text", text: PROSE_GUARD_RETRY_PROMPT }], + agent: "codebase-research-analyzer", + }); + return lastResult; + }, }); - s.save(result.data!); - return extractResponseText(result.data!.parts); + s.save(lastResult!.data!); + return text; }, ); @@ -207,6 +242,10 @@ export default defineWorkflow({ const { partitions, explorerCount, scratchDir, totalLoc, totalFiles } = scout.result; + // D2: derive a short brief from the history-analyzer output to inject as + // into per-partition locator + analyzer prompts. + const priorResearchBrief = deriveHistoryBrief(historyOverview); + const scoutOverview = (await ctx.transcript(scout)).content; // ── Stage 2: per-partition specialist fan-out ───────────────────────── @@ -226,24 +265,40 @@ export default defineWorkflow({ {}, { title: `locator-${i}` }, async (s) => { - const result = await s.client.session.prompt({ - sessionID: s.session.id, - parts: [ - { - type: "text", - text: buildLocatorPrompt({ - question: prompt, - partition, - scoutOverview, - index: i, - total: explorerCount, - }), - }, - ], - agent: "codebase-locator", + let lastResult: Awaited>; + const { text } = await queryWithProseGuard({ + query: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [ + { + type: "text", + text: buildLocatorPrompt({ + question: prompt, + partition, + scoutOverview, + index: i, + total: explorerCount, + priorResearchBrief, + }), + }, + ], + agent: "codebase-locator", + }); + return lastResult; + }, + getText: (r) => extractResponseText(r.data!.parts), + retry: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [{ type: "text", text: PROSE_GUARD_RETRY_PROMPT }], + agent: "codebase-locator", + }); + return lastResult; + }, }); - s.save(result.data!); - return extractResponseText(result.data!.parts); + s.save(lastResult!.data!); + return text; }, ), ctx.stage( @@ -255,24 +310,39 @@ export default defineWorkflow({ {}, { title: `pattern-finder-${i}` }, async (s) => { - const result = await s.client.session.prompt({ - sessionID: s.session.id, - parts: [ - { - type: "text", - text: buildPatternFinderPrompt({ - question: prompt, - partition, - scoutOverview, - index: i, - total: explorerCount, - }), - }, - ], - agent: "codebase-pattern-finder", + let lastResult: Awaited>; + const { text } = await queryWithProseGuard({ + query: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [ + { + type: "text", + text: buildPatternFinderPrompt({ + question: prompt, + partition, + scoutOverview, + index: i, + total: explorerCount, + }), + }, + ], + agent: "codebase-pattern-finder", + }); + return lastResult; + }, + getText: (r) => extractResponseText(r.data!.parts), + retry: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [{ type: "text", text: PROSE_GUARD_RETRY_PROMPT }], + agent: "codebase-pattern-finder", + }); + return lastResult; + }, }); - s.save(result.data!); - return extractResponseText(result.data!.parts); + s.save(lastResult!.data!); + return text; }, ), ]); @@ -291,25 +361,41 @@ export default defineWorkflow({ {}, { title: `analyzer-${i}` }, async (s) => { - const result = await s.client.session.prompt({ - sessionID: s.session.id, - parts: [ - { - type: "text", - text: buildAnalyzerPrompt({ - question: prompt, - partition, - locatorOutput, - scoutOverview, - index: i, - total: explorerCount, - }), - }, - ], - agent: "codebase-analyzer", + let lastResult: Awaited>; + const { text } = await queryWithProseGuard({ + query: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [ + { + type: "text", + text: buildAnalyzerPrompt({ + question: prompt, + partition, + locatorOutput, + scoutOverview, + index: i, + total: explorerCount, + priorResearchBrief, + }), + }, + ], + agent: "codebase-analyzer", + }); + return lastResult; + }, + getText: (r) => extractResponseText(r.data!.parts), + retry: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [{ type: "text", text: PROSE_GUARD_RETRY_PROMPT }], + agent: "codebase-analyzer", + }); + return lastResult; + }, }); - s.save(result.data!); - return extractResponseText(result.data!.parts); + s.save(lastResult!.data!); + return text; }, ), ctx.stage( @@ -321,24 +407,39 @@ export default defineWorkflow({ {}, { title: `online-researcher-${i}` }, async (s) => { - const result = await s.client.session.prompt({ - sessionID: s.session.id, - parts: [ - { - type: "text", - text: buildOnlineResearcherPrompt({ - question: prompt, - partition, - locatorOutput, - index: i, - total: explorerCount, - }), - }, - ], - agent: "codebase-online-researcher", + let lastResult: Awaited>; + const { text } = await queryWithProseGuard({ + query: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [ + { + type: "text", + text: buildOnlineResearcherPrompt({ + question: prompt, + partition, + locatorOutput, + index: i, + total: explorerCount, + }), + }, + ], + agent: "codebase-online-researcher", + }); + return lastResult; + }, + getText: (r) => extractResponseText(r.data!.parts), + retry: async () => { + lastResult = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [{ type: "text", text: PROSE_GUARD_RETRY_PROMPT }], + agent: "codebase-online-researcher", + }); + return lastResult; + }, }); - s.save(result.data!); - return extractResponseText(result.data!.parts); + s.save(lastResult!.data!); + return text; }, ), ]); From 9b3365cf4493da4f89e2d87e1eadd12483badaff Mon Sep 17 00:00:00 2001 From: Alex Lavaee Date: Thu, 23 Apr 2026 04:14:56 +0000 Subject: [PATCH 2/2] feat(workflows): apply context-engineering refinements to ralph + deep-research MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refines both builtin workflows across all three SDKs (claude/copilot/opencode) using shared, SDK-agnostic helpers under `_context/`. Ralph (per-iteration coding loop): - R1 infra-discovery hoist with `shouldReRunInfraDiscovery` invalidation — re-run only when infra files (lockfiles, manifests, configs, CI, agent instructions) change, otherwise reuse the cached overview. - R2 persistent scratchpad at `.atomic/ralph//state.md` records planner output, debugger reports, and modified files; planner reads the latest prior RFC + session intent from it, enabling cheap re-plans. - R4 SPEC_REMINDER positioned at planner + reviewer prompt heads so the primacy slot reinforces the contract on every iteration. - R5 reviewer + debugger prompts run their changeset through `maskChangeset` — top-N by churn for diff-stat, top-N for staged uncommitted, ALL `??` untracked entries preserved verbatim (correctness invariant). - R6 debugger report capped at MAX_DEBUGGER_REPORT_CHARS via `truncateMarkdownReport` (head + tail with elision marker). Deep-research-codebase (scout → fan-out → aggregator): - D2 history-analyzer output condensed via `deriveHistoryBrief` (≤150 words) and injected as `` into per-partition locator + analyzer prompts. - D3 aggregator pre-flight: when the sum of partition scratch sizes exceeds SCRATCH_COMPACT_THRESHOLD, each oversized scratch is rewritten with `compactScratchFile` (head/tail truncation that preserves every `## ` / `### ` heading verbatim so the aggregator's reading contract still holds). - D5 prose-guard: every specialist call (history-locator/analyzer + per-partition locator/pattern-finder/analyzer/online-researcher) wrapped in `queryWithProseGuard` — retries once with a short reminder prompt when the assistant ends on a tool call instead of emitting prose. New shared module `_context/`: - `budget.ts` — token approximator + thresholds. - `masking.ts` — pure functions: maskChangeset, compactDiffStat, compactUncommitted, isInfraPath, infraInvalidationPaths, shouldReRunInfraDiscovery, truncateMarkdownReport, deriveHistoryBrief, compactReminder, queryWithProseGuard, compactScratchFile. - `scratchpad.ts` — single-writer Ralph state-file helpers (initScratchpad, recordPlannerOutput, recordDebuggerReport, recordFilesModified, latestPriorRFC, detectSpecPath). - 29 unit tests covering the pure helpers. Verification: bun typecheck ✓, bun lint 0 warnings, bun test 1191/1191 pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../builtin/_context/_context.test.ts | 32 +++ src/sdk/workflows/builtin/_context/budget.ts | 3 - src/sdk/workflows/builtin/_context/masking.ts | 64 ++++++ .../deep-research-codebase/claude/index.ts | 10 +- .../deep-research-codebase/copilot/index.ts | 7 +- .../deep-research-codebase/helpers/prompts.ts | 30 +-- .../deep-research-codebase/helpers/scratch.ts | 49 ++++- .../deep-research-codebase/opencode/index.ts | 7 +- .../workflows/builtin/ralph/claude/index.ts | 187 ++++++++++++------ .../workflows/builtin/ralph/copilot/index.ts | 149 +++++++++----- .../builtin/ralph/helpers/prompts.ts | 140 +++++++++++-- .../workflows/builtin/ralph/opencode/index.ts | 164 ++++++++++----- 12 files changed, 635 insertions(+), 207 deletions(-) diff --git a/src/sdk/workflows/builtin/_context/_context.test.ts b/src/sdk/workflows/builtin/_context/_context.test.ts index 82d33d94c..28cc653c0 100644 --- a/src/sdk/workflows/builtin/_context/_context.test.ts +++ b/src/sdk/workflows/builtin/_context/_context.test.ts @@ -2,6 +2,7 @@ import { describe, test, expect } from "bun:test"; import { compactDiffStat, compactReminder, + compactScratchFile, compactUncommitted, deriveHistoryBrief, infraInvalidationPaths, @@ -218,3 +219,34 @@ describe("detectSpecPath", () => { expect(detectSpecPath(input)).toBeNull(); }); }); + +describe("compactScratchFile", () => { + test("returns verbatim when under cap", () => { + const small = "## A\nbody\n## B\nbody"; + expect(compactScratchFile(small, 1000)).toBe(small); + }); + + test("preserves all ## headings even when bodies are elided", () => { + const big = + "## Scope\n" + + "x".repeat(2000) + + "\n## Files in Scope\n" + + "y".repeat(2000) + + "\n## How It Works\n" + + "z".repeat(2000); + const out = compactScratchFile(big, 600); + expect(out).toContain("## Scope"); + expect(out).toContain("## Files in Scope"); + expect(out).toContain("## How It Works"); + expect(out).toContain("elided"); + expect(out.length).toBeLessThan(big.length); + }); + + test("preserves ### sub-headings", () => { + const input = + "## A\n" + "p".repeat(500) + "\n### Sub\n" + "q".repeat(500); + const out = compactScratchFile(input, 200); + expect(out).toContain("## A"); + expect(out).toContain("### Sub"); + }); +}); diff --git a/src/sdk/workflows/builtin/_context/budget.ts b/src/sdk/workflows/builtin/_context/budget.ts index e94c004ac..783796836 100644 --- a/src/sdk/workflows/builtin/_context/budget.ts +++ b/src/sdk/workflows/builtin/_context/budget.ts @@ -18,9 +18,6 @@ export function approxTokens(text: string): number { /** Fraction of effective window at which compaction should trigger. */ export const COMPACT_TRIGGER_FRACTION = 0.7; -/** Partition file-LOC above which a file is flagged as "huge" for readers. */ -export const HUGE_FILE_LOC = 5_000; - /** * Aggregator pre-flight: when the sum of per-partition scratch-file sizes * exceeds this many chars, compact each oversized scratch before the diff --git a/src/sdk/workflows/builtin/_context/masking.ts b/src/sdk/workflows/builtin/_context/masking.ts index 4a19100aa..2ec80e2a5 100644 --- a/src/sdk/workflows/builtin/_context/masking.ts +++ b/src/sdk/workflows/builtin/_context/masking.ts @@ -282,3 +282,67 @@ export async function queryWithProseGuard( export const PROSE_GUARD_RETRY_PROMPT = "Emit the required prose summary now — do not end on a tool call. " + "A short markdown report following the earlier output format is sufficient."; + +// ============================================================================ +// SCRATCH FILE COMPACTION (D3, deep-research aggregator pre-flight) +// ============================================================================ + +/** + * Schema-preserving head/tail truncation of a markdown scratch file. Keeps + * every `## ` and `### ` heading verbatim and truncates section bodies to + * fit the budget. Path anchors and `file:line` refs that appear within the + * preserved head/tail of each section are kept; only the middle of long + * sections is collapsed. + * + * The aggregator's reading contract (helpers/scratch.ts schema) requires + * the same heading layout — locator/patterns/analyzer/online sections — so + * we never drop a heading even if its body becomes empty. + */ +export function compactScratchFile(content: string, maxChars: number): string { + if (content.length <= maxChars) return content; + + const lines = content.split("\n"); + const sections: { heading: string; body: string[] }[] = []; + let current: { heading: string; body: string[] } | null = null; + for (const line of lines) { + if (/^##\s/.test(line) || /^###\s/.test(line)) { + if (current) sections.push(current); + current = { heading: line, body: [] }; + } else if (current) { + current.body.push(line); + } else { + // Pre-heading preamble — treat as a synthetic leading section so we + // don't lose the file's title block. + current = { heading: "", body: [line] }; + } + } + if (current) sections.push(current); + + const headingCost = sections.reduce( + (sum, s) => sum + s.heading.length + 1, + 0, + ); + const budgetForBodies = Math.max(0, maxChars - headingCost); + const perSection = Math.floor(budgetForBodies / Math.max(1, sections.length)); + + const out: string[] = []; + for (const s of sections) { + if (s.heading) out.push(s.heading); + const body = s.body.join("\n"); + if (body.length <= perSection) { + out.push(body); + continue; + } + const half = Math.floor((perSection - 32) / 2); + if (half <= 0) { + out.push(`[… ${body.length} chars elided …]`); + continue; + } + out.push( + body.slice(0, half) + + `\n\n[… ${body.length - perSection} chars elided …]\n\n` + + body.slice(body.length - half), + ); + } + return out.join("\n"); +} diff --git a/src/sdk/workflows/builtin/deep-research-codebase/claude/index.ts b/src/sdk/workflows/builtin/deep-research-codebase/claude/index.ts index 1d5d9f38e..1de4c5b39 100644 --- a/src/sdk/workflows/builtin/deep-research-codebase/claude/index.ts +++ b/src/sdk/workflows/builtin/deep-research-codebase/claude/index.ts @@ -81,7 +81,7 @@ import { buildScoutPrompt, slugifyPrompt, } from "../helpers/prompts.ts"; -import { writeExplorerScratchFile } from "../helpers/scratch.ts"; +import { compactScratchFilesForAggregator, writeExplorerScratchFile } from "../helpers/scratch.ts"; import { deriveHistoryBrief, PROSE_GUARD_RETRY_PROMPT, @@ -445,6 +445,14 @@ export default defineWorkflow({ `${isoDate}-${slug}.md`, ); + // D3: pre-flight scratch compaction. If the sum of partition scratch + // sizes exceeds SCRATCH_COMPACT_THRESHOLD, rewrite each oversized file + // with a head/tail-truncated, schema-preserving summary so the + // aggregator's effective context window stays bounded. + await compactScratchFilesForAggregator( + explorerHandles.map((e) => e.scratchPath), + ); + await ctx.stage( { name: "aggregator", diff --git a/src/sdk/workflows/builtin/deep-research-codebase/copilot/index.ts b/src/sdk/workflows/builtin/deep-research-codebase/copilot/index.ts index ca88f94c8..a142593f8 100644 --- a/src/sdk/workflows/builtin/deep-research-codebase/copilot/index.ts +++ b/src/sdk/workflows/builtin/deep-research-codebase/copilot/index.ts @@ -51,7 +51,7 @@ import { buildScoutPrompt, slugifyPrompt, } from "../helpers/prompts.ts"; -import { writeExplorerScratchFile } from "../helpers/scratch.ts"; +import { compactScratchFilesForAggregator, writeExplorerScratchFile } from "../helpers/scratch.ts"; import { deriveHistoryBrief, PROSE_GUARD_RETRY_PROMPT, @@ -393,6 +393,11 @@ export default defineWorkflow({ `${isoDate}-${slug}.md`, ); + // D3: pre-flight scratch compaction (see Claude index for rationale). + await compactScratchFilesForAggregator( + explorerHandles.map((e) => e.scratchPath), + ); + await ctx.stage( { name: "aggregator", diff --git a/src/sdk/workflows/builtin/deep-research-codebase/helpers/prompts.ts b/src/sdk/workflows/builtin/deep-research-codebase/helpers/prompts.ts index dda4b6a14..46597ab5b 100644 --- a/src/sdk/workflows/builtin/deep-research-codebase/helpers/prompts.ts +++ b/src/sdk/workflows/builtin/deep-research-codebase/helpers/prompts.ts @@ -38,7 +38,7 @@ */ import type { PartitionUnit } from "./scout.ts"; -import { deriveHistoryBrief, HUGE_FILE_LOC } from "../../_context/index.ts"; +import { deriveHistoryBrief } from "../../_context/index.ts"; /** * Render a `` block for inclusion in explorer prompts. @@ -62,31 +62,8 @@ function renderPriorResearchHint(brief: string | undefined): string { ].join("\n"); } -/** - * Build a `` block listing files in the partition whose LOC - * exceeds {@link HUGE_FILE_LOC}. Readers should offset/limit these rather - * than reading in full. Empty partitions or no-huge-files → empty string. - * - * Note: `PartitionUnit.files` does not carry per-file LOC currently — the - * deterministic producer passes `hugeFiles` directly. This helper renders - * whatever list the caller supplies. - */ -function renderHugeFiles(hugeFiles: string[] | undefined): string { - if (!hugeFiles || hugeFiles.length === 0) return ""; - return [ - ``, - ``, - `The following files in your partition exceed ${HUGE_FILE_LOC.toLocaleString()} lines.`, - `Read them via offset/limit (e.g. Read with \`offset\` + \`limit\`) — reading`, - `them in full will blow your context window:`, - ``, - ...hugeFiles.map((f) => `- \`${f}\``), - ``, - ].join("\n"); -} - // Re-export for SDK index call sites that need to compute a brief. -export { deriveHistoryBrief, HUGE_FILE_LOC }; +export { deriveHistoryBrief }; /** * Prepended to every stage prompt. Compresses agent prose output to cut @@ -408,8 +385,6 @@ export function buildAnalyzerPrompt(opts: { scoutOverview: string; index: number; total: number; - /** Files in this partition whose LOC exceeds HUGE_FILE_LOC (D1). */ - hugeFiles?: string[]; /** ≤150-word brief derived from the history-analyzer output (D2). */ priorResearchBrief?: string; }): string { @@ -445,7 +420,6 @@ export function buildAnalyzerPrompt(opts: { ``, assignment, ``, - renderHugeFiles(opts.hugeFiles), ``, ``, `Verbatim output from the codebase-locator sibling for this partition —`, diff --git a/src/sdk/workflows/builtin/deep-research-codebase/helpers/scratch.ts b/src/sdk/workflows/builtin/deep-research-codebase/helpers/scratch.ts index 82053f976..6b329727c 100644 --- a/src/sdk/workflows/builtin/deep-research-codebase/helpers/scratch.ts +++ b/src/sdk/workflows/builtin/deep-research-codebase/helpers/scratch.ts @@ -20,9 +20,13 @@ * References"), or the aggregator will look for sections that don't exist. */ -import { writeFile } from "node:fs/promises"; +import { readFile, stat, writeFile } from "node:fs/promises"; import path from "node:path"; import type { PartitionUnit } from "./scout.ts"; +import { + compactScratchFile, + SCRATCH_COMPACT_THRESHOLD, +} from "../../_context/index.ts"; export type ExplorerSections = { index: number; @@ -113,3 +117,46 @@ export async function writeExplorerScratchFile( await writeFile(abs, md, "utf8"); return abs; } + +/** + * D3: aggregator pre-flight compaction. If the **sum** of scratch file + * sizes exceeds `SCRATCH_COMPACT_THRESHOLD`, each file is rewritten using + * `compactScratchFile` so the aggregator can read them without overflowing + * its effective context window. Heading schema is preserved verbatim, so + * the aggregator's reading contract still holds. + * + * Returns the list of paths actually compacted (empty when the run was + * under threshold), so callers can log it. + */ +export async function compactScratchFilesForAggregator( + paths: string[], + threshold: number = SCRATCH_COMPACT_THRESHOLD, +): Promise { + const sizes = await Promise.all( + paths.map(async (p) => { + try { + return (await stat(p)).size; + } catch { + return 0; + } + }), + ); + const total = sizes.reduce((a, b) => a + b, 0); + if (total <= threshold) return []; + + // Per-file budget: divide threshold across N files, leaving a 10% headroom + // for headings + delimiters the aggregator emits between sections. + const perFileBudget = Math.floor((threshold * 0.9) / Math.max(1, paths.length)); + + const compacted: string[] = []; + await Promise.all( + paths.map(async (p, i) => { + if ((sizes[i] ?? 0) <= perFileBudget) return; + const content = await readFile(p, "utf8"); + const next = compactScratchFile(content, perFileBudget); + await writeFile(p, next, "utf8"); + compacted.push(p); + }), + ); + return compacted; +} diff --git a/src/sdk/workflows/builtin/deep-research-codebase/opencode/index.ts b/src/sdk/workflows/builtin/deep-research-codebase/opencode/index.ts index d66b4e911..dcb573740 100644 --- a/src/sdk/workflows/builtin/deep-research-codebase/opencode/index.ts +++ b/src/sdk/workflows/builtin/deep-research-codebase/opencode/index.ts @@ -50,7 +50,7 @@ import { buildScoutPrompt, slugifyPrompt, } from "../helpers/prompts.ts"; -import { writeExplorerScratchFile } from "../helpers/scratch.ts"; +import { compactScratchFilesForAggregator, writeExplorerScratchFile } from "../helpers/scratch.ts"; import { deriveHistoryBrief, PROSE_GUARD_RETRY_PROMPT, @@ -466,6 +466,11 @@ export default defineWorkflow({ `${isoDate}-${slug}.md`, ); + // D3: pre-flight scratch compaction (see Claude index for rationale). + await compactScratchFilesForAggregator( + explorerHandles.map((e) => e.scratchPath), + ); + await ctx.stage( { name: "aggregator", diff --git a/src/sdk/workflows/builtin/ralph/claude/index.ts b/src/sdk/workflows/builtin/ralph/claude/index.ts index 335f17cf6..cd4eeefbd 100644 --- a/src/sdk/workflows/builtin/ralph/claude/index.ts +++ b/src/sdk/workflows/builtin/ralph/claude/index.ts @@ -35,9 +35,43 @@ import { } from "../helpers/prompts.ts"; import { hasActionableFindings } from "../helpers/review.ts"; import { captureBranchChangeset } from "../helpers/git.ts"; +import { + initScratchpad, + latestPriorRFC, + recordDebuggerReport, + recordFilesModified, + recordPlannerOutput, + shouldReRunInfraDiscovery, +} from "../../_context/index.ts"; const DEFAULT_MAX_LOOPS = 10; +/** Deterministic intent extraction — first sentence/paragraph or 200 chars. */ +function deriveSessionIntent(prompt: string): string { + const trimmed = prompt.trim(); + if (!trimmed) return "(no prompt)"; + const firstPara = trimmed.split(/\n\n+/)[0] ?? trimmed; + const oneLine = firstPara.split("\n").join(" ").replace(/\s+/g, " "); + return oneLine.length > 200 ? oneLine.slice(0, 200) + "…" : oneLine; +} + +/** + * Parse repo-relative paths from `git diff --name-status` output. Used to + * tell the scratchpad which files this iteration touched. + */ +function parseFilesFromNameStatus(nameStatus: string): string[] { + const paths: string[] = []; + for (const line of nameStatus.split("\n")) { + if (!line.trim()) continue; + const parts = line.split("\t"); + if (parts.length >= 2) { + const p = parts[parts.length - 1]; + if (p) paths.push(p); + } + } + return paths; +} + // The orchestrator stage implements the actual code changes and can run for // a very long time on large tasks. Completion is detected via session file // watching for idle and result events from Claude's own SDK — no manual @@ -86,11 +120,23 @@ export default defineWorkflow({ .run(async (ctx) => { const prompt = ctx.inputs.prompt ?? ""; const maxLoops = ctx.inputs.max_loops ?? DEFAULT_MAX_LOOPS; + const runId = crypto.randomUUID(); + const sessionIntent = deriveSessionIntent(prompt); + const scratchpad = await initScratchpad({ + sessionId: runId, + projectRoot: process.cwd(), + originalSpec: prompt, + }); + let debuggerReport = ""; + let priorRfc: string | null = null; + // Hoisted infra-discovery cache; populated lazily on iteration 1, then + // re-run only when a build/test/lint/CI/agent-instruction file changes. + let discoveryContext: string | null = null; for (let iteration = 1; iteration <= maxLoops; iteration++) { // ── Plan ──────────────────────────────────────────────────────────── - await ctx.stage( + const plannerStage = await ctx.stage( { name: `planner-${iteration}` }, { chatFlags: [ @@ -102,16 +148,22 @@ export default defineWorkflow({ }, {}, async (s) => { - await s.session.query( + const result = await s.session.query( buildPlannerPrompt(prompt, { iteration, debuggerReport: debuggerReport || undefined, + priorRfc: priorRfc ?? undefined, + sessionIntent, }), ); s.save(s.sessionId); + return extractAssistantText(result, 0); }, ); + await recordPlannerOutput(scratchpad, iteration, plannerStage.result); + priorRfc = await latestPriorRFC(scratchpad); + // ── Orchestrate ───────────────────────────────────────────────────── await ctx.stage( { name: `orchestrator-${iteration}` }, @@ -130,72 +182,88 @@ export default defineWorkflow({ }, ); - // ── Infrastructure Discovery (three parallel sub-agent stages) ──── + // ── Capture changeset (needed before infra-invalidation + reviewer) ─ const changeset = await captureBranchChangeset(); - const discoveryPrompts = buildInfraDiscoveryPrompts(); - const [locatorResult, analyzerResult, patternResult] = await Promise.all([ - ctx.stage( - { name: `infra-locate-${iteration}`, headless: true }, - {}, - {}, - async (s) => { - const result = await s.session.query(discoveryPrompts.locator, { - agent: "codebase-locator", - permissionMode: "bypassPermissions", - allowDangerouslySkipPermissions: true, - }); - s.save(s.sessionId); - return extractAssistantText(result, 0); - }, - ), - ctx.stage( - { name: `infra-analyze-${iteration}`, headless: true }, - {}, - {}, - async (s) => { - const result = await s.session.query(discoveryPrompts.analyzer, { - agent: "codebase-analyzer", - permissionMode: "bypassPermissions", - allowDangerouslySkipPermissions: true, - }); - s.save(s.sessionId); - return extractAssistantText(result, 0); - }, - ), - ctx.stage( - { name: `infra-patterns-${iteration}`, headless: true }, - {}, - {}, - async (s) => { - const result = await s.session.query( - discoveryPrompts.patternFinder, - { - agent: "codebase-pattern-finder", - permissionMode: "bypassPermissions", - allowDangerouslySkipPermissions: true, + await recordFilesModified( + scratchpad, + iteration, + parseFilesFromNameStatus(changeset.nameStatus), + ); + + // ── Infrastructure Discovery (hoisted; re-runs on infra-file edits) ─ + const needsRediscovery = + discoveryContext === null || shouldReRunInfraDiscovery(changeset); + if (needsRediscovery) { + const discoveryPrompts = buildInfraDiscoveryPrompts(); + const [locatorResult, analyzerResult, patternResult] = + await Promise.all([ + ctx.stage( + { name: `infra-locate-${iteration}`, headless: true }, + {}, + {}, + async (s) => { + const result = await s.session.query(discoveryPrompts.locator, { + agent: "codebase-locator", + permissionMode: "bypassPermissions", + allowDangerouslySkipPermissions: true, + }); + s.save(s.sessionId); + return extractAssistantText(result, 0); }, - ); - s.save(s.sessionId); - return extractAssistantText(result, 0); - }, - ), - ]); + ), + ctx.stage( + { name: `infra-analyze-${iteration}`, headless: true }, + {}, + {}, + async (s) => { + const result = await s.session.query( + discoveryPrompts.analyzer, + { + agent: "codebase-analyzer", + permissionMode: "bypassPermissions", + allowDangerouslySkipPermissions: true, + }, + ); + s.save(s.sessionId); + return extractAssistantText(result, 0); + }, + ), + ctx.stage( + { name: `infra-patterns-${iteration}`, headless: true }, + {}, + {}, + async (s) => { + const result = await s.session.query( + discoveryPrompts.patternFinder, + { + agent: "codebase-pattern-finder", + permissionMode: "bypassPermissions", + allowDangerouslySkipPermissions: true, + }, + ); + s.save(s.sessionId); + return extractAssistantText(result, 0); + }, + ), + ]); - const discoveryContext = [ - "### Infrastructure Files (codebase-locator)\n\n" + - locatorResult.result, - "### Infrastructure Analysis (codebase-analyzer)\n\n" + - analyzerResult.result, - "### Build & Test Patterns (codebase-pattern-finder)\n\n" + - patternResult.result, - ].join("\n\n---\n\n"); + discoveryContext = [ + "### Infrastructure Files (codebase-locator)\n\n" + + locatorResult.result, + "### Infrastructure Analysis (codebase-analyzer)\n\n" + + analyzerResult.result, + "### Build & Test Patterns (codebase-pattern-finder)\n\n" + + patternResult.result, + ].join("\n\n---\n\n"); + } // ── Review (two parallel headless passes with schema enforcement) ── const reviewPrompt = buildReviewPrompt(prompt, { changeset, iteration, - discoveryContext, + discoveryContext: discoveryContext ?? undefined, + sessionIntent, }); const runReviewer = (name: string) => @@ -259,6 +327,7 @@ export default defineWorkflow({ ); debuggerReport = extractMarkdownBlock(debugger_.result); + await recordDebuggerReport(scratchpad, iteration, debuggerReport); } } }) diff --git a/src/sdk/workflows/builtin/ralph/copilot/index.ts b/src/sdk/workflows/builtin/ralph/copilot/index.ts index 441baeb52..ed08ef572 100644 --- a/src/sdk/workflows/builtin/ralph/copilot/index.ts +++ b/src/sdk/workflows/builtin/ralph/copilot/index.ts @@ -34,9 +34,38 @@ import { } from "../helpers/prompts.ts"; import { hasActionableFindings } from "../helpers/review.ts"; import { captureBranchChangeset } from "../helpers/git.ts"; +import { + initScratchpad, + latestPriorRFC, + recordDebuggerReport, + recordFilesModified, + recordPlannerOutput, + shouldReRunInfraDiscovery, +} from "../../_context/index.ts"; const DEFAULT_MAX_LOOPS = 10; +function deriveSessionIntent(prompt: string): string { + const trimmed = prompt.trim(); + if (!trimmed) return "(no prompt)"; + const firstPara = trimmed.split(/\n\n+/)[0] ?? trimmed; + const oneLine = firstPara.split("\n").join(" ").replace(/\s+/g, " "); + return oneLine.length > 200 ? oneLine.slice(0, 200) + "…" : oneLine; +} + +function parseFilesFromNameStatus(nameStatus: string): string[] { + const paths: string[] = []; + for (const line of nameStatus.split("\n")) { + if (!line.trim()) continue; + const parts = line.split("\t"); + if (parts.length >= 2) { + const p = parts[parts.length - 1]; + if (p) paths.push(p); + } + } + return paths; +} + const SUBMIT_REVIEW_DESCRIPTION = "Submit the structured code review result. You MUST call this tool " + "exactly once with your complete review. Do not output the review as " + @@ -97,7 +126,17 @@ export default defineWorkflow({ .run(async (ctx) => { const userPromptText = ctx.inputs.prompt ?? ""; const maxLoops = ctx.inputs.max_loops ?? DEFAULT_MAX_LOOPS; + const runId = crypto.randomUUID(); + const sessionIntent = deriveSessionIntent(userPromptText); + const scratchpad = await initScratchpad({ + sessionId: runId, + projectRoot: process.cwd(), + originalSpec: userPromptText, + }); + let debuggerReport = ""; + let priorRfc: string | null = null; + let discoveryContext: string | null = null; for (let iteration = 1; iteration <= maxLoops; iteration++) { // ── Plan ────────────────────────────────────────────────────────── @@ -110,6 +149,8 @@ export default defineWorkflow({ prompt: buildPlannerPrompt(userPromptText, { iteration, debuggerReport: debuggerReport || undefined, + priorRfc: priorRfc ?? undefined, + sessionIntent, }), }); const messages = await s.session.getMessages(); @@ -118,6 +159,9 @@ export default defineWorkflow({ }, ); + await recordPlannerOutput(scratchpad, iteration, planner.result); + priorRfc = await latestPriorRFC(scratchpad); + // ── Orchestrate ─────────────────────────────────────────────────── await ctx.stage( { name: `orchestrator-${iteration}` }, @@ -133,61 +177,73 @@ export default defineWorkflow({ }, ); - // ── Infrastructure Discovery (three parallel sub-agent stages) ── + // ── Capture changeset ───────────────────────────────────────────── const changeset = await captureBranchChangeset(); - const discoveryPrompts = buildInfraDiscoveryPrompts(); + await recordFilesModified( + scratchpad, + iteration, + parseFilesFromNameStatus(changeset.nameStatus), + ); - const [locatorResult, analyzerResult, patternResult] = await Promise.all([ - ctx.stage( - { name: `infra-locate-${iteration}`, headless: true }, - {}, - { agent: "codebase-locator" }, - async (s) => { - await s.session.send({ prompt: discoveryPrompts.locator }); - const messages = await s.session.getMessages(); - s.save(messages); - return getAssistantText(messages); - }, - ), - ctx.stage( - { name: `infra-analyze-${iteration}`, headless: true }, - {}, - { agent: "codebase-analyzer" }, - async (s) => { - await s.session.send({ prompt: discoveryPrompts.analyzer }); - const messages = await s.session.getMessages(); - s.save(messages); - return getAssistantText(messages); - }, - ), - ctx.stage( - { name: `infra-patterns-${iteration}`, headless: true }, - {}, - { agent: "codebase-pattern-finder" }, - async (s) => { - await s.session.send({ prompt: discoveryPrompts.patternFinder }); - const messages = await s.session.getMessages(); - s.save(messages); - return getAssistantText(messages); - }, - ), - ]); + // ── Infrastructure Discovery (hoisted; re-runs on infra edits) ── + const needsRediscovery = + discoveryContext === null || shouldReRunInfraDiscovery(changeset); + if (needsRediscovery) { + const discoveryPrompts = buildInfraDiscoveryPrompts(); + const [locatorResult, analyzerResult, patternResult] = + await Promise.all([ + ctx.stage( + { name: `infra-locate-${iteration}`, headless: true }, + {}, + { agent: "codebase-locator" }, + async (s) => { + await s.session.send({ prompt: discoveryPrompts.locator }); + const messages = await s.session.getMessages(); + s.save(messages); + return getAssistantText(messages); + }, + ), + ctx.stage( + { name: `infra-analyze-${iteration}`, headless: true }, + {}, + { agent: "codebase-analyzer" }, + async (s) => { + await s.session.send({ prompt: discoveryPrompts.analyzer }); + const messages = await s.session.getMessages(); + s.save(messages); + return getAssistantText(messages); + }, + ), + ctx.stage( + { name: `infra-patterns-${iteration}`, headless: true }, + {}, + { agent: "codebase-pattern-finder" }, + async (s) => { + await s.session.send({ prompt: discoveryPrompts.patternFinder }); + const messages = await s.session.getMessages(); + s.save(messages); + return getAssistantText(messages); + }, + ), + ]); - const discoveryContext = [ - "### Infrastructure Files (codebase-locator)\n\n" + - locatorResult.result, - "### Infrastructure Analysis (codebase-analyzer)\n\n" + - analyzerResult.result, - "### Build & Test Patterns (codebase-pattern-finder)\n\n" + - patternResult.result, - ].join("\n\n---\n\n"); + discoveryContext = [ + "### Infrastructure Files (codebase-locator)\n\n" + + locatorResult.result, + "### Infrastructure Analysis (codebase-analyzer)\n\n" + + analyzerResult.result, + "### Build & Test Patterns (codebase-pattern-finder)\n\n" + + patternResult.result, + ].join("\n\n---\n\n"); + } // ── Review (two parallel passes) ────────────────────────────────── const reviewPrompt = buildReviewPrompt(userPromptText, { changeset, iteration, useSubmitTool: true, - discoveryContext, + discoveryContext: discoveryContext ?? undefined, + sessionIntent, }); // Each parallel reviewer gets its own tool + capture ref so they @@ -273,6 +329,7 @@ export default defineWorkflow({ ); debuggerReport = extractMarkdownBlock(debugger_.result); + await recordDebuggerReport(scratchpad, iteration, debuggerReport); } } }) diff --git a/src/sdk/workflows/builtin/ralph/helpers/prompts.ts b/src/sdk/workflows/builtin/ralph/helpers/prompts.ts index 97b664e11..3f4c10460 100644 --- a/src/sdk/workflows/builtin/ralph/helpers/prompts.ts +++ b/src/sdk/workflows/builtin/ralph/helpers/prompts.ts @@ -14,6 +14,69 @@ import { z } from "zod"; +import { + compactReminder, + MAX_DEBUGGER_REPORT_CHARS, + maskChangeset, + truncateMarkdownReport, +} from "../../_context/index.ts"; + +// ============================================================================ +// TOKEN-REDUCTION META-PROMPT +// ============================================================================ + +/** + * Prepended to every Ralph stage prompt. Compresses agent output to cut + * tokens while preserving all technical substance. Code blocks, commits, + * PRs, and structured outputs (JSON/markdown templates) are exempt — the + * individual stage prompts override this where verbatim formatting matters. + */ +export const CAVEMAN_PREAMBLE = `# Response Style (applies to all prose output) + +Respond terse like smart caveman. All technical substance stay. Only fluff die. + +## Persistence + +ACTIVE EVERY RESPONSE. No revert after many turns. No filler drift. Still active if unsure. + +## Rules + +Drop: articles (a/an/the), filler (just/really/basically/actually/simply), pleasantries (sure/certainly/of course/happy to), hedging. Fragments OK. Short synonyms (big not extensive, fix not "implement a solution for"). Technical terms exact. Code blocks unchanged. Errors quoted exact. + +Pattern: \`[thing] [action] [reason]. [next step].\` + +Not: "Sure! I'd be happy to help you with that. The issue you're experiencing is likely caused by..." +Yes: "Bug in auth middleware. Token expiry check use \`<\` not \`<=\`. Fix:" + +## Intensity + +Drop articles, fragments OK, short synonyms. + +Example — "Why React component re-render?" +"New object ref each render. Inline object prop = new ref = re-render. Wrap in \`useMemo\`." + +Example — "Explain database connection pooling." +"Pool reuse open DB connections. No new connection per request. Skip handshake overhead." + +## Auto-Clarity + +Drop caveman for: security warnings, irreversible action confirmations, multi-step sequences where fragment order risks misread, user asks to clarify or repeats question. Resume caveman after clear part done. + +Example — destructive op: +> **Warning:** This will permanently delete all rows in the \`users\` table and cannot be undone. +> \`\`\`sql +> DROP TABLE users; +> \`\`\` +> Caveman resume. Verify backup exist first. + +## Boundaries + +Code/commits/PRs: write normal. Structured outputs (JSON, required markdown templates, schema-enforced fields): write normal — schema wins. + +--- + +`; + // ============================================================================ // STRUCTURED OUTPUT SCHEMAS // ============================================================================ @@ -159,6 +222,18 @@ export interface PlannerContext { iteration: number; /** Markdown report from the previous iteration's debugger sub-agent. */ debuggerReport?: string; + /** + * Previous iteration's full inline RFC (if any), captured from the + * per-run scratchpad. Only injected on re-plan iterations when the prior + * planner did NOT short-circuit to a file path. Precedence on re-plan: + * `debuggerReport` > `priorRfc`. + */ + priorRfc?: string; + /** + * Session-level one-line intent used for the `` anchor + * at the bottom of the prompt. Deterministic — never the full spec. + */ + sessionIntent?: string; } /** @@ -175,7 +250,9 @@ export function buildPlannerPrompt( context: PlannerContext = { iteration: 1 }, ): string { const debuggerReport = context.debuggerReport?.trim() ?? ""; - const isReplan = context.iteration > 1 && debuggerReport.length > 0; + const priorRfc = context.priorRfc?.trim() ?? ""; + const isReplan = + context.iteration > 1 && (debuggerReport.length > 0 || priorRfc.length > 0); const header = isReplan ? `# Technical Design Revision (Iteration ${context.iteration}) @@ -192,7 +269,23 @@ Author a Technical Design Document / RFC for the specification below.`; ${spec} `; - const debuggerBlock = isReplan + const priorRfcBlock = + isReplan && priorRfc.length > 0 + ? ` + +## Prior RFC (previous iteration — for continuity) + + +${priorRfc} + + +_Note: the **Debugger Report below takes precedence** over the Prior RFC._ +_The Prior RFC is provided for continuity so you do not re-derive the_ +_design from scratch. Keep decisions, alternatives, and open questions_ +_that are still valid; revise what the debugger report invalidates._` + : ""; + + const debuggerBlock = isReplan && debuggerReport.length > 0 ? ` ## Debugger Report (authoritative) @@ -216,9 +309,16 @@ Fold every issue in the debugger report into the revised RFC: uncertainty the debugger flagged as unresolved.` : ""; - return `${header} + const specReminder = context.sessionIntent + ? `\n\n\n${compactReminder({ + intent: context.sessionIntent, + iteration: context.iteration, + })}\n` + : ""; + + return `${CAVEMAN_PREAMBLE}${header} -${specBlock}${debuggerBlock} +${specBlock}${priorRfcBlock}${debuggerBlock} ${ isReplan @@ -279,7 +379,7 @@ forward the path. Duplicating the spec wastes tokens and introduces drift.` - Output nothing else after the RFC (or path) — no meta-commentary, no summary. The document (or path) stands on its own. - Match depth to stakes: a greenfield service warrants deep sections 5-7; a - small refactor can abbreviate them, but every section header must be present.`; + small refactor can abbreviate them, but every section header must be present.${specReminder}`; } // ============================================================================ // ORCHESTRATOR @@ -320,7 +420,7 @@ ${plannerNotes} (empty — fall back to the Original User Specification below) `; - return `You are the workflow orchestrator. You run a three-phase loop: + return `${CAVEMAN_PREAMBLE}You are the workflow orchestrator. You run a three-phase loop: 1. **Decompose** the design document into a task list. 2. **Execute** the tasks by spawning parallel worker sub-agents. @@ -472,7 +572,7 @@ export interface InfraDiscoveryPrompts { */ export function buildInfraDiscoveryPrompts(): InfraDiscoveryPrompts { return { - locator: `# Locate Build & Test Infrastructure Files + locator: `${CAVEMAN_PREAMBLE}# Locate Build & Test Infrastructure Files Find ALL files in this repository that define or configure the build, test, lint, type-check, and CI/CD infrastructure. Report their paths and a @@ -500,7 +600,7 @@ Be exhaustive. Do NOT skip files just because they seem minor — CI configs and agent instruction files often contain the authoritative command list. End with a brief trailing summary (1-2 sentences) of what you found.`, - analyzer: `# Analyze Build & Test Infrastructure + analyzer: `${CAVEMAN_PREAMBLE}# Analyze Build & Test Infrastructure Examine this repository's build, test, lint, and type-check infrastructure. Your goal is to produce a concise reference that tells a reviewer exactly @@ -543,7 +643,7 @@ Be specific — include the exact invocation string (e.g. \`bun test\`, not just "run tests"). If a command has variants (e.g. test:unit, test:e2e), list each separately. End with a brief trailing summary.`, - patternFinder: `# Find Build & Test Patterns + patternFinder: `${CAVEMAN_PREAMBLE}# Find Build & Test Patterns Search this repository for existing patterns that show how code is built, tested, and validated. A reviewer needs to know not just WHAT commands exist, @@ -629,6 +729,8 @@ export interface ReviewContext { * repository's build/test/lint commands as part of verification. */ discoveryContext?: string; + /** Session-level one-line intent used for the `` anchor. */ + sessionIntent?: string; } /** @@ -640,7 +742,7 @@ export function buildReviewPrompt( spec: string, context: ReviewContext, ): string { - const { changeset } = context; + const changeset = maskChangeset(context.changeset); const hasChanges = changeset.diffStat.length > 0 || changeset.uncommitted.length > 0; const hasErrors = changeset.errors.length > 0; @@ -748,7 +850,7 @@ ${context.discoveryContext} // ── Full prompt ──────────────────────────────────────────────────────── - return `${header} + return `${CAVEMAN_PREAMBLE}${header} ## Original Specification @@ -830,7 +932,15 @@ ${outputSection} - **overall_explanation**: Summary of overall quality, correctness, and any patterns observed. -Begin your review now.`; +Begin your review now.${ + context.sessionIntent + ? `\n\n\n${compactReminder({ + intent: context.sessionIntent, + iteration: context.iteration, + extra: `changeset: ${changeset.nameStatus.split("\n").filter(Boolean).length} paths`, + })}\n` + : "" + }`; } // ============================================================================ @@ -892,7 +1002,7 @@ ${trimmed} : `(no reviewer output captured)`; } - const { changeset } = context; + const changeset = maskChangeset(context.changeset); const hasChanges = changeset.nameStatus.length > 0 || changeset.uncommitted.length > 0; const hasErrors = changeset.errors.length > 0; @@ -928,7 +1038,7 @@ ${trimmed} changesetSection = "(no changes detected)"; } - return `# Debugging Report Request (Iteration ${context.iteration}) + return `${CAVEMAN_PREAMBLE}# Debugging Report Request (Iteration ${context.iteration}) The reviewer flagged the issues below. Investigate them as a debugger and produce a structured report that the planner will consume on the next loop @@ -1018,5 +1128,5 @@ export function extractMarkdownBlock(content: string): string { if (match[1]) last = match[1]; } if (last !== null) return last.trim(); - return content.trim(); + return truncateMarkdownReport(content.trim(), MAX_DEBUGGER_REPORT_CHARS); } diff --git a/src/sdk/workflows/builtin/ralph/opencode/index.ts b/src/sdk/workflows/builtin/ralph/opencode/index.ts index 1e8901982..aba45760d 100644 --- a/src/sdk/workflows/builtin/ralph/opencode/index.ts +++ b/src/sdk/workflows/builtin/ralph/opencode/index.ts @@ -31,9 +31,38 @@ import { } from "../helpers/prompts.ts"; import { hasActionableFindings } from "../helpers/review.ts"; import { captureBranchChangeset } from "../helpers/git.ts"; +import { + initScratchpad, + latestPriorRFC, + recordDebuggerReport, + recordFilesModified, + recordPlannerOutput, + shouldReRunInfraDiscovery, +} from "../../_context/index.ts"; const DEFAULT_MAX_LOOPS = 10; +function deriveSessionIntent(prompt: string): string { + const trimmed = prompt.trim(); + if (!trimmed) return "(no prompt)"; + const firstPara = trimmed.split(/\n\n+/)[0] ?? trimmed; + const oneLine = firstPara.split("\n").join(" ").replace(/\s+/g, " "); + return oneLine.length > 200 ? oneLine.slice(0, 200) + "…" : oneLine; +} + +function parseFilesFromNameStatus(nameStatus: string): string[] { + const paths: string[] = []; + for (const line of nameStatus.split("\n")) { + if (!line.trim()) continue; + const parts = line.split("\t"); + if (parts.length >= 2) { + const p = parts[parts.length - 1]; + if (p) paths.push(p); + } + } + return paths; +} + /** Concatenate the text-typed parts of an OpenCode response. */ function extractResponseText( parts: Array<{ type: string; [key: string]: unknown }>, @@ -87,7 +116,17 @@ export default defineWorkflow({ .run(async (ctx) => { const prompt = ctx.inputs.prompt ?? ""; const maxLoops = ctx.inputs.max_loops ?? DEFAULT_MAX_LOOPS; + const runId = crypto.randomUUID(); + const sessionIntent = deriveSessionIntent(prompt); + const scratchpad = await initScratchpad({ + sessionId: runId, + projectRoot: process.cwd(), + originalSpec: prompt, + }); + let debuggerReport = ""; + let priorRfc: string | null = null; + let discoveryContext: string | null = null; for (let iteration = 1; iteration <= maxLoops; iteration++) { // ── Plan ──────────────────────────────────────────────────────────── @@ -104,6 +143,8 @@ export default defineWorkflow({ text: buildPlannerPrompt(prompt, { iteration, debuggerReport: debuggerReport || undefined, + priorRfc: priorRfc ?? undefined, + sessionIntent, }), }, ], @@ -114,6 +155,9 @@ export default defineWorkflow({ }, ); + await recordPlannerOutput(scratchpad, iteration, planner.result); + priorRfc = await latestPriorRFC(scratchpad); + // ── Orchestrate ───────────────────────────────────────────────────── await ctx.stage( { name: `orchestrator-${iteration}` }, @@ -136,66 +180,81 @@ export default defineWorkflow({ }, ); - // ── Infrastructure Discovery (three parallel sub-agent stages) ──── + // ── Capture changeset ─────────────────────────────────────────────── const changeset = await captureBranchChangeset(); - const discoveryPrompts = buildInfraDiscoveryPrompts(); + await recordFilesModified( + scratchpad, + iteration, + parseFilesFromNameStatus(changeset.nameStatus), + ); - const [locatorResult, analyzerResult, patternResult] = await Promise.all([ - ctx.stage( - { name: `infra-locate-${iteration}`, headless: true }, - {}, - { title: `infra-locate-${iteration}` }, - async (s) => { - const result = await s.client.session.prompt({ - sessionID: s.session.id, - parts: [{ type: "text", text: discoveryPrompts.locator }], - agent: "codebase-locator", - }); - s.save(result.data!); - return extractResponseText(result.data!.parts); - }, - ), - ctx.stage( - { name: `infra-analyze-${iteration}`, headless: true }, - {}, - { title: `infra-analyze-${iteration}` }, - async (s) => { - const result = await s.client.session.prompt({ - sessionID: s.session.id, - parts: [{ type: "text", text: discoveryPrompts.analyzer }], - agent: "codebase-analyzer", - }); - s.save(result.data!); - return extractResponseText(result.data!.parts); - }, - ), - ctx.stage( - { name: `infra-patterns-${iteration}`, headless: true }, - {}, - { title: `infra-patterns-${iteration}` }, - async (s) => { - const result = await s.client.session.prompt({ - sessionID: s.session.id, - parts: [{ type: "text", text: discoveryPrompts.patternFinder }], - agent: "codebase-pattern-finder", - }); - s.save(result.data!); - return extractResponseText(result.data!.parts); - }, - ), - ]); + // ── Infrastructure Discovery (hoisted; re-runs on infra edits) ──── + const needsRediscovery = + discoveryContext === null || shouldReRunInfraDiscovery(changeset); + if (needsRediscovery) { + const discoveryPrompts = buildInfraDiscoveryPrompts(); + const [locatorResult, analyzerResult, patternResult] = + await Promise.all([ + ctx.stage( + { name: `infra-locate-${iteration}`, headless: true }, + {}, + { title: `infra-locate-${iteration}` }, + async (s) => { + const result = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [{ type: "text", text: discoveryPrompts.locator }], + agent: "codebase-locator", + }); + s.save(result.data!); + return extractResponseText(result.data!.parts); + }, + ), + ctx.stage( + { name: `infra-analyze-${iteration}`, headless: true }, + {}, + { title: `infra-analyze-${iteration}` }, + async (s) => { + const result = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [{ type: "text", text: discoveryPrompts.analyzer }], + agent: "codebase-analyzer", + }); + s.save(result.data!); + return extractResponseText(result.data!.parts); + }, + ), + ctx.stage( + { name: `infra-patterns-${iteration}`, headless: true }, + {}, + { title: `infra-patterns-${iteration}` }, + async (s) => { + const result = await s.client.session.prompt({ + sessionID: s.session.id, + parts: [{ type: "text", text: discoveryPrompts.patternFinder }], + agent: "codebase-pattern-finder", + }); + s.save(result.data!); + return extractResponseText(result.data!.parts); + }, + ), + ]); - const discoveryContext = [ - "### Infrastructure Files (codebase-locator)\n\n" + locatorResult.result, - "### Infrastructure Analysis (codebase-analyzer)\n\n" + analyzerResult.result, - "### Build & Test Patterns (codebase-pattern-finder)\n\n" + patternResult.result, - ].join("\n\n---\n\n"); + discoveryContext = [ + "### Infrastructure Files (codebase-locator)\n\n" + + locatorResult.result, + "### Infrastructure Analysis (codebase-analyzer)\n\n" + + analyzerResult.result, + "### Build & Test Patterns (codebase-pattern-finder)\n\n" + + patternResult.result, + ].join("\n\n---\n\n"); + } // ── Review (two parallel passes) ──────────────────────────────────── const reviewPrompt = buildReviewPrompt(prompt, { changeset, iteration, - discoveryContext, + discoveryContext: discoveryContext ?? undefined, + sessionIntent, }); const reviewStage = async (name: string) => @@ -261,6 +320,7 @@ export default defineWorkflow({ ); debuggerReport = extractMarkdownBlock(debugger_.result); + await recordDebuggerReport(scratchpad, iteration, debuggerReport); } } })