From dbc296967c90ed2cd00562114d56c8905f9b0668 Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 22 Jun 2026 17:16:30 -0700 Subject: [PATCH] feat: add autonomous quality gates --- packages/coding-agent/src/cli/args.ts | 22 +++++ .../coding-agent/src/core/agent-session.ts | 4 +- packages/coding-agent/src/core/autonomous.ts | 87 +++++++++++++++++-- packages/coding-agent/src/main.ts | 15 +++- packages/coding-agent/test/args.test.ts | 21 +++++ .../suite/agent-session-autonomous.test.ts | 40 +++++++++ 6 files changed, 182 insertions(+), 7 deletions(-) diff --git a/packages/coding-agent/src/cli/args.ts b/packages/coding-agent/src/cli/args.ts index ce770ec7af..67e486f8c8 100644 --- a/packages/coding-agent/src/cli/args.ts +++ b/packages/coding-agent/src/cli/args.ts @@ -43,6 +43,9 @@ export interface Args { noThemes?: boolean; noContextFiles?: boolean; autonomous?: boolean; + autonomousGates?: string[]; + autonomousGateRetries?: number; + autonomousGateTimeoutMs?: number; listModels?: string | true; offline?: boolean; verbose?: boolean; @@ -169,6 +172,13 @@ export function parseArgs(args: string[]): Args { result.noContextFiles = true; } else if (arg === "--autonomous") { result.autonomous = true; + } else if (arg === "--autonomous-gate" && i + 1 < args.length) { + result.autonomousGates = result.autonomousGates ?? []; + result.autonomousGates.push(args[++i]); + } else if (arg === "--autonomous-gate-retries" && i + 1 < args.length) { + result.autonomousGateRetries = parsePositiveInt(args[++i], "--autonomous-gate-retries", result); + } else if (arg === "--autonomous-gate-timeout-ms" && i + 1 < args.length) { + result.autonomousGateTimeoutMs = parsePositiveInt(args[++i], "--autonomous-gate-timeout-ms", result); } else if (arg === "--list-models") { // Check if next arg is a search pattern (not a flag or file arg) if (i + 1 < args.length && !args[i + 1].startsWith("-") && !args[i + 1].startsWith("@")) { @@ -266,6 +276,9 @@ ${chalk.bold("Options:")} --no-themes Disable theme discovery and loading --no-context-files, -nc Disable AGENTS.md and CLAUDE.md discovery and loading --autonomous Continue autonomously on help requests or soft blockers + --autonomous-gate Run a command before autonomous mode may finish (repeatable) + --autonomous-gate-retries Max autonomous retries per failed gate (default: 3) + --autonomous-gate-timeout-ms Timeout per autonomous gate command in milliseconds --export Export session file to HTML and exit --list-models [search] List available models (with optional fuzzy search) --verbose Force verbose startup (overrides quietStartup setting) @@ -376,3 +389,12 @@ ${chalk.bold("Built-in Tool Names:")} edit - Edit files with find/replace (off by default) `); } + +function parsePositiveInt(flagValue: string, flagName: string, result: Args): number | undefined { + const parsed = Number(flagValue); + if (!Number.isInteger(parsed) || parsed <= 0) { + result.diagnostics.push({ type: "error", message: `${flagName} requires a positive integer` }); + return undefined; + } + return parsed; +} diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index f5b4b88717..00949fa349 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -1569,7 +1569,9 @@ export class AgentSession { if (goalMessages.length > 0 || signal?.aborted) { return goalMessages; } - const autonomousMessage = nextAutonomousContinuation(this._autonomousState, context.message, { cwd: this._cwd }); + const autonomousMessage = await nextAutonomousContinuation(this._autonomousState, context.message, { + cwd: this._cwd, + }); return autonomousMessage ? [autonomousMessage] : []; } diff --git a/packages/coding-agent/src/core/autonomous.ts b/packages/coding-agent/src/core/autonomous.ts index 48d4a60aa1..d80158487b 100644 --- a/packages/coding-agent/src/core/autonomous.ts +++ b/packages/coding-agent/src/core/autonomous.ts @@ -6,6 +6,13 @@ export interface AgentAutonomousFinishContractConfig { continuationPrompt?: string; } +export interface AgentAutonomousGateConfig { + commands?: string[]; + onFail?: "feed_output_back"; + maxRetries?: number; + timeoutMs?: number; +} + export interface AgentAutonomousConfig { enabled?: boolean; maxContinuations?: number; @@ -14,6 +21,7 @@ export interface AgentAutonomousConfig { timeoutMs?: number; continuationPrompt?: string; finishContract?: AgentAutonomousFinishContractConfig; + gates?: AgentAutonomousGateConfig; } export interface AgentAutonomousStatus { @@ -22,7 +30,7 @@ export interface AgentAutonomousStatus { turnsUsed: number; tokensUsed: number; startedAt?: number; - limits: Required>; + limits: Required>; } export const DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT = @@ -32,7 +40,7 @@ export const DEFAULT_AUTONOMOUS_FINISH_PROMPT = "Autonomous finish contract is not satisfied. Do not stop with 'I think it works'. Continue until one of these is true: a clean git patch exists, configured tests passed, an explicit blocker artifact is written, or a no-op is justified with evidence. Inspect the repo and run the relevant checks now."; export const DEFAULT_AUTONOMOUS_LIMITS: Required< - Omit + Omit > = { maxContinuations: 3, maxTurns: 12, @@ -46,9 +54,11 @@ export interface AutonomousRuntimeState { turnsUsed: number; tokensUsed: number; startedAt?: number; - limits: Required>; + limits: Required>; continuationPrompt: string; finishContract: Required; + gates: Required; + gateAttempts: Record; } export interface AutonomousDecision { @@ -75,6 +85,13 @@ export function createAutonomousRuntimeState(config?: AgentAutonomousConfig): Au enabled: config?.finishContract?.enabled ?? true, continuationPrompt: config?.finishContract?.continuationPrompt?.trim() || DEFAULT_AUTONOMOUS_FINISH_PROMPT, }, + gates: { + commands: [...(config?.gates?.commands ?? [])], + onFail: config?.gates?.onFail ?? "feed_output_back", + maxRetries: normalizeLimit(config?.gates?.maxRetries, 3), + timeoutMs: normalizeLimit(config?.gates?.timeoutMs, 5 * 60 * 1000), + }, + gateAttempts: {}, }; } @@ -109,12 +126,12 @@ export function addAutonomousUsage(state: AutonomousRuntimeState, usage: Usage | state.tokensUsed += usage?.totalTokens ?? 0; } -export function nextAutonomousContinuation( +export async function nextAutonomousContinuation( state: AutonomousRuntimeState, message: AssistantMessage, options: { cwd?: string } = {}, now = Date.now(), -): UserMessage | undefined { +): Promise { if (!state.enabled) { return undefined; } @@ -122,6 +139,15 @@ export function nextAutonomousContinuation( if (!decision.shouldContinue) { return undefined; } + if (decision.reason === "finish_contract" && options.cwd) { + const gateContinuation = runAutonomousQualityGates(state, options.cwd, now); + if (gateContinuation || state.gates.commands.length > 0) { + if (gateContinuation) { + state.continuationsUsed++; + } + return gateContinuation; + } + } state.continuationsUsed++; return { role: "user", @@ -249,6 +275,57 @@ export function hasAutonomousFinishEvidence(text: string, cwd?: string): boolean ); } +function runAutonomousQualityGates( + state: AutonomousRuntimeState, + cwd: string, + timestamp: number, +): UserMessage | undefined { + for (const command of state.gates.commands) { + const result = spawnSync(command, { + cwd, + encoding: "utf8", + shell: true, + timeout: state.gates.timeoutMs, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status === 0 && !result.error) { + state.gateAttempts[command] = 0; + continue; + } + + const attempt = (state.gateAttempts[command] ?? 0) + 1; + state.gateAttempts[command] = attempt; + if (attempt > state.gates.maxRetries) { + return undefined; + } + const exitText = + result.error?.message ?? + (result.signal ? `terminated by ${result.signal}` : `exited ${result.status ?? "unknown"}`); + const output = truncateGateOutput([result.stdout, result.stderr].filter(Boolean).join("\n").trim()); + return { + role: "user", + content: [ + { + type: "text", + text: + `Autonomous quality gate failed (attempt ${attempt}/${state.gates.maxRetries}): \`${command}\` ${exitText}.\n` + + (output ? `\nOutput:\n${output}\n` : "\n") + + "\nFix the failure and continue. Do not finish until quality gates pass or you have a concrete external blocker with evidence.", + }, + ], + timestamp, + }; + } + return undefined; +} + +function truncateGateOutput(output: string, maxChars = 6000): string { + if (output.length <= maxChars) { + return output; + } + return `${output.slice(0, maxChars)}\n... [truncated ${output.length - maxChars} chars]`; +} + function hasGitWorktreeChanges(cwd: string): boolean { const result = spawnSync("git", ["--no-optional-locks", "status", "--porcelain"], { cwd, diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index 4d915c4399..fd6a85f159 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -640,6 +640,7 @@ function runtimeConfigFromArgs( agentDir: string, sessionDir: string | undefined, ): AgentSessionRuntimeConfig { + const hasAutonomousGates = (parsed.autonomousGates?.length ?? 0) > 0; return { cwd, agentDir, @@ -663,7 +664,19 @@ function runtimeConfigFromArgs( themes: resolveCliPaths(cwd, parsed.themes), noThemes: parsed.noThemes, noContextFiles: parsed.noContextFiles, - autonomous: parsed.autonomous ? { enabled: true } : undefined, + autonomous: + parsed.autonomous || hasAutonomousGates + ? { + enabled: true, + gates: hasAutonomousGates + ? { + commands: parsed.autonomousGates, + maxRetries: parsed.autonomousGateRetries, + timeoutMs: parsed.autonomousGateTimeoutMs, + } + : undefined, + } + : undefined, extensionFlagValues: parsed.unknownFlags.size > 0 ? Object.fromEntries(parsed.unknownFlags.entries()) : undefined, }; } diff --git a/packages/coding-agent/test/args.test.ts b/packages/coding-agent/test/args.test.ts index 40eb9df13f..c4534aef28 100644 --- a/packages/coding-agent/test/args.test.ts +++ b/packages/coding-agent/test/args.test.ts @@ -286,6 +286,27 @@ describe("parseArgs", () => { }); }); + describe("autonomous gate flags", () => { + test("parses repeatable --autonomous-gate flags", () => { + const result = parseArgs(["--autonomous-gate", "npm test", "--autonomous-gate", "npm run lint"]); + expect(result.autonomousGates).toEqual(["npm test", "npm run lint"]); + }); + + test("parses autonomous gate retry and timeout limits", () => { + const result = parseArgs(["--autonomous-gate-retries", "4", "--autonomous-gate-timeout-ms", "1234"]); + expect(result.autonomousGateRetries).toBe(4); + expect(result.autonomousGateTimeoutMs).toBe(1234); + }); + + test("reports invalid autonomous gate limits", () => { + const result = parseArgs(["--autonomous-gate-retries", "nope"]); + expect(result.diagnostics).toContainEqual({ + type: "error", + message: "--autonomous-gate-retries requires a positive integer", + }); + }); + }); + describe("tool flags", () => { test("parses --no-tools flag", () => { const result = parseArgs(["--no-tools"]); diff --git a/packages/coding-agent/test/suite/agent-session-autonomous.test.ts b/packages/coding-agent/test/suite/agent-session-autonomous.test.ts index 0feadd9299..48cce355f5 100644 --- a/packages/coding-agent/test/suite/agent-session-autonomous.test.ts +++ b/packages/coding-agent/test/suite/agent-session-autonomous.test.ts @@ -135,6 +135,46 @@ describe("AgentSession autonomous mode", () => { expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(0); }); + it("accepts passing quality gates as finish evidence", async () => { + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + gates: { commands: [`${process.execPath} -e "process.exit(0)"`] }, + }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("Done.")]); + + await harness.session.prompt("make the change"); + + expect(getUserTexts(harness)).toEqual(["make the change"]); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(0); + }); + + it("feeds failing quality gate output back into autonomous mode", async () => { + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + gates: { + commands: [`${process.execPath} -e "console.error('gate failed'); process.exit(1)"`], + maxRetries: 1, + }, + }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("Done."), fauxAssistantMessage("I wrote BLOCKER.md with evidence.")]); + + await harness.session.prompt("make the change"); + + const users = getUserTexts(harness); + expect(users[0]).toBe("make the change"); + expect(users[1]).toContain("Autonomous quality gate failed"); + expect(users[1]).toContain("gate failed"); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + }); + it("classifies soft blockers separately from real external blockers", () => { const state = createAutonomousRuntimeState({ enabled: true });