diff --git a/packages/coding-agent/src/cli/args.ts b/packages/coding-agent/src/cli/args.ts index f1a7453506..2ced33883e 100644 --- a/packages/coding-agent/src/cli/args.ts +++ b/packages/coding-agent/src/cli/args.ts @@ -42,6 +42,14 @@ export interface Args { themes?: string[]; noThemes?: boolean; noContextFiles?: boolean; + autonomous?: boolean; + autonomousGates?: string[]; + autonomousGateRetries?: number; + autonomousGateTimeoutMs?: number; + autonomousMaxContinuations?: number; + autonomousMaxTurns?: number; + autonomousMaxTokens?: number; + autonomousTimeoutMs?: number; listModels?: string | true; offline?: boolean; verbose?: boolean; @@ -166,6 +174,23 @@ export function parseArgs(args: string[]): Args { result.noThemes = true; } else if (arg === "--no-context-files" || arg === "-nc") { result.noContextFiles = true; + } else if (arg === "--autonomous") { + result.autonomous = true; + } else if (arg === "--autonomous-gate" && i + 1 < args.length) { + result.autonomousGates = result.autonomousGates ?? []; + result.autonomousGates.push(args[++i]); + } else if (arg === "--autonomous-gate-retries" && i + 1 < args.length) { + result.autonomousGateRetries = parsePositiveInt(args[++i], "--autonomous-gate-retries", result); + } else if (arg === "--autonomous-gate-timeout-ms" && i + 1 < args.length) { + result.autonomousGateTimeoutMs = parsePositiveInt(args[++i], "--autonomous-gate-timeout-ms", result); + } else if (arg === "--autonomous-max-continuations" && i + 1 < args.length) { + result.autonomousMaxContinuations = parsePositiveInt(args[++i], "--autonomous-max-continuations", result); + } else if (arg === "--autonomous-max-turns" && i + 1 < args.length) { + result.autonomousMaxTurns = parsePositiveInt(args[++i], "--autonomous-max-turns", result); + } else if (arg === "--autonomous-max-tokens" && i + 1 < args.length) { + result.autonomousMaxTokens = parsePositiveInt(args[++i], "--autonomous-max-tokens", result); + } else if (arg === "--autonomous-timeout-ms" && i + 1 < args.length) { + result.autonomousTimeoutMs = parsePositiveInt(args[++i], "--autonomous-timeout-ms", result); } else if (arg === "--list-models") { // Check if next arg is a search pattern (not a flag or file arg) if (i + 1 < args.length && !args[i + 1].startsWith("-") && !args[i + 1].startsWith("@")) { @@ -262,6 +287,14 @@ ${chalk.bold("Options:")} --theme Load a theme file or directory (can be used multiple times) --no-themes Disable theme discovery and loading --no-context-files, -nc Disable AGENTS.md and CLAUDE.md discovery and loading + --autonomous Continue autonomously until host-observable terminal evidence exists + --autonomous-gate Run a command before autonomous mode may finish (repeatable) + --autonomous-gate-retries Max autonomous retries per failed gate (default: 3) + --autonomous-gate-timeout-ms Timeout per autonomous gate command in milliseconds + --autonomous-max-continuations Max autonomous follow-up messages (default: 3) + --autonomous-max-turns Max assistant turns while autonomous mode is active (default: 12) + --autonomous-max-tokens Max tokens while autonomous mode is active (default: 80000) + --autonomous-timeout-ms Max autonomous wall-clock time in milliseconds (default: 1800000) --export Export session file to HTML and exit --list-models [search] List available models (with optional fuzzy search) --verbose Force verbose startup (overrides quietStartup setting) @@ -372,3 +405,12 @@ ${chalk.bold("Built-in Tool Names:")} edit - Edit files with find/replace (off by default) `); } + +function parsePositiveInt(value: string, flag: string, result: Args): number | undefined { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + result.diagnostics.push({ type: "error", message: `${flag} must be a positive integer` }); + return undefined; + } + return parsed; +} diff --git a/packages/coding-agent/src/core/agent-session-config.ts b/packages/coding-agent/src/core/agent-session-config.ts index e4d2d6c417..3833e0d84d 100644 --- a/packages/coding-agent/src/core/agent-session-config.ts +++ b/packages/coding-agent/src/core/agent-session-config.ts @@ -1,4 +1,5 @@ import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; +import type { AgentAutonomousConfig } from "./autonomous.js"; export interface AgentSessionRuntimeConfig { cwd?: string; @@ -23,6 +24,7 @@ export interface AgentSessionRuntimeConfig { themes?: string[]; noThemes?: boolean; noContextFiles?: boolean; + autonomous?: AgentAutonomousConfig; extensionFlagValues?: Record; } @@ -56,6 +58,7 @@ export function mergeAgentSessionRuntimeConfig( themes: cloneArray(override.themes ?? base.themes), noThemes: override.noThemes ?? base.noThemes, noContextFiles: override.noContextFiles ?? base.noContextFiles, + autonomous: mergeAutonomousConfig(base.autonomous, override.autonomous), extensionFlagValues: base.extensionFlagValues || override.extensionFlagValues ? { ...(base.extensionFlagValues ?? {}), ...(override.extensionFlagValues ?? {}) } @@ -73,10 +76,24 @@ function cloneAgentSessionRuntimeConfig(config: AgentSessionRuntimeConfig): Agen skills: cloneArray(config.skills), promptTemplates: cloneArray(config.promptTemplates), themes: cloneArray(config.themes), + autonomous: config.autonomous ? { ...config.autonomous } : undefined, extensionFlagValues: config.extensionFlagValues ? { ...config.extensionFlagValues } : undefined, }; } +function mergeAutonomousConfig( + base: AgentAutonomousConfig | undefined, + override: AgentAutonomousConfig | undefined, +): AgentAutonomousConfig | undefined { + if (!base && !override) { + return undefined; + } + return { + ...(base ?? {}), + ...(override ?? {}), + }; +} + function cloneArray(value: T[] | undefined): T[] | undefined { return value ? [...value] : undefined; } diff --git a/packages/coding-agent/src/core/agent-session-services.ts b/packages/coding-agent/src/core/agent-session-services.ts index 16af3b0fc5..cc277063aa 100644 --- a/packages/coding-agent/src/core/agent-session-services.ts +++ b/packages/coding-agent/src/core/agent-session-services.ts @@ -4,6 +4,7 @@ import type { Model } from "@earendil-works/pi-ai"; import { getAgentDir } from "../config.js"; import { installAgentTraceUpload } from "./agent-traces.js"; import { AuthStorage } from "./auth-storage.js"; +import type { AgentAutonomousConfig } from "./autonomous.js"; import type { AgentRlmHeartbeatController } from "./cron-jobs.js"; import type { SessionStartEvent, ToolDefinition } from "./extensions/index.js"; import { ModelRegistry } from "./model-registry.js"; @@ -59,6 +60,7 @@ export interface AgentSessionCreationOptions { subagentRuntimeHost?: SubagentRuntimeHost; rlmHeartbeatController?: AgentRlmHeartbeatController; prewarmIpythonKernel?: boolean; + autonomous?: AgentAutonomousConfig; } /** @@ -224,5 +226,6 @@ export async function createAgentSessionFromServices( rlmHeartbeatController: options.rlmHeartbeatController, sessionStartEvent: options.sessionStartEvent, prewarmIpythonKernel: options.prewarmIpythonKernel, + autonomous: options.autonomous, }); } diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 325718bcfc..f4b6cca508 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -50,6 +50,16 @@ import { stripFrontmatter } from "../utils/frontmatter.js"; import { sleep } from "../utils/sleep.js"; import { ensureTool, MISSING_RIPGREP_MESSAGE } from "../utils/tools-manager.js"; import { formatNoApiKeyFoundMessage, formatNoModelSelectedMessage } from "./auth-guidance.js"; +import { + type AgentAutonomousConfig, + type AgentAutonomousStatus, + type AutonomousRuntimeState, + addAutonomousUsage, + autonomousStatus, + createAutonomousRuntimeState, + nextAutonomousContinuation, + setAutonomousEnabled, +} from "./autonomous.js"; import { type BashResult, executeBashWithOperations } from "./bash-executor.js"; import { type CompactionResult, @@ -326,6 +336,8 @@ export interface AgentSessionConfig { rlmParentNodeId?: string; /** Host responsible for creating RLM subagent runtimes. */ subagentRuntimeHost?: SubagentRuntimeHost; + /** Host-side autonomous continuation policy. */ + autonomous?: AgentAutonomousConfig; /** * Boot the IPython kernel in the background as soon as the session is created, * so the first ipython tool call doesn't pay the kernel cold start. @@ -384,6 +396,8 @@ type GoalSlashCommand = | { kind: "resume" } | { kind: "start"; objective: string; tokenBudget?: number }; +type AutonomousSlashCommand = { kind: "status" } | { kind: "on" } | { kind: "off" }; + interface RlmChildRun { id: string; prompt: string; @@ -665,12 +679,15 @@ export class AgentSession { private _goalAccountingStartedAt: number | undefined = undefined; private _goalAccountedAssistantMessages = new WeakSet(); private _goalAbortInProgress = false; + private _autonomousState: AutonomousRuntimeState; // Compaction state private _compactionAbortController: AbortController | undefined = undefined; private _autoCompactionAbortController: AbortController | undefined = undefined; private _overflowRecoveryAttempted = false; private _continueAfterThresholdCompaction = false; + private _postCompactionContinuePromise: Promise | undefined = undefined; + private _postCompactionContinuationMessages: AgentMessage[] = []; // Branch summarization state private _branchSummaryAbortController: AbortController | undefined = undefined; @@ -756,6 +773,7 @@ export class AgentSession { this._rlmSessionDir = config.rlmSessionDir; this._rlmParentNodeId = config.rlmParentNodeId; this._subagentRuntimeHost = config.subagentRuntimeHost; + this._autonomousState = createAutonomousRuntimeState(config.autonomous, { cwd: this._cwd }); this._goalState = this._loadPersistedGoalState(); if (this._goalState.status === "active") { this._goalAccountingStartedAt = Date.now(); @@ -870,7 +888,7 @@ export class AgentSession { } private _installAgentContinuationHook(): void { - this.agent.getContinuationMessages = (context, signal) => this._getGoalContinuationMessages(context, signal); + this.agent.getContinuationMessages = (context, signal) => this._getContinuationMessages(context, signal); } private _installAgentTurnHook(): void { @@ -1131,6 +1149,55 @@ export class AgentSession { return { kind: "start", objective: validateGoalObjective(objective), tokenBudget }; } + private _parseAutonomousSlashCommand(text: string): AutonomousSlashCommand | undefined { + if (text !== "/autonomous" && !text.startsWith("/autonomous ")) { + return undefined; + } + const rest = text.slice("/autonomous".length).trim().toLowerCase(); + if (!rest || rest === "status") { + return { kind: "status" }; + } + if (rest === "on" || rest === "enable" || rest === "enabled") { + return { kind: "on" }; + } + if (rest === "off" || rest === "disable" || rest === "disabled") { + return { kind: "off" }; + } + throw new Error("Usage: /autonomous [on|off|status]"); + } + + private _formatAutonomousStatus(): string { + const status = this.getAutonomousStatus(); + const state = status.enabled ? "on" : "off"; + return `Autonomous mode: ${state}. Continuations: ${status.continuationsUsed}/${status.limits.maxContinuations}. Turns: ${status.turnsUsed}/${status.limits.maxTurns}. Tokens: ${status.tokensUsed}/${status.limits.maxTokens}.`; + } + + private async _emitAutonomousStatus(): Promise { + await this.sendCustomMessage( + { + customType: "autonomous_status", + content: this._formatAutonomousStatus(), + display: true, + details: this.getAutonomousStatus(), + }, + { triggerTurn: false }, + ); + } + + private async _handleAutonomousSlashCommand(text: string): Promise { + const command = this._parseAutonomousSlashCommand(text); + if (!command) { + return false; + } + if (command.kind === "on") { + setAutonomousEnabled(this._autonomousState, true, { cwd: this._cwd }); + } else if (command.kind === "off") { + setAutonomousEnabled(this._autonomousState, false); + } + await this._emitAutonomousStatus(); + return true; + } + private async _validateCanStartAgentRun(): Promise { if (!this.model) { throw new Error(formatNoModelSelectedMessage()); @@ -1309,11 +1376,26 @@ export class AgentSession { return false; } + if (this._queueAutonomousContinuationForThresholdCompaction(context.message)) { + this._continueAfterThresholdCompaction = true; + return true; + } + const lastMessage = this.agent.state.messages[this.agent.state.messages.length - 1]; this._continueAfterThresholdCompaction = lastMessage !== undefined && lastMessage.role !== "assistant"; return true; } + private _queueAutonomousContinuationForThresholdCompaction(message: AssistantMessage): boolean { + const autonomousMessage = nextAutonomousContinuation(this._autonomousState, message, { cwd: this._cwd }); + if (!autonomousMessage) { + return false; + } + this._postCompactionContinuationMessages.push(autonomousMessage); + this.agent.followUp(autonomousMessage); + return true; + } + /** * Handle a goal.* request from the IPython kernel host bridge (the bundled * goal skill). All goal state stays host-side; the kernel only sees the @@ -1496,6 +1578,18 @@ export class AgentSession { } } + private async _getContinuationMessages( + context: GetContinuationMessagesContext, + signal?: AbortSignal, + ): Promise { + const goalMessages = await this._getGoalContinuationMessages(context, signal); + if (goalMessages.length > 0 || signal?.aborted) { + return goalMessages; + } + const autonomousMessage = nextAutonomousContinuation(this._autonomousState, context.message, { cwd: this._cwd }); + return autonomousMessage ? [autonomousMessage] : []; + } + // Track last assistant message for auto-compaction check private _lastAssistantMessage: AssistantMessage | undefined = undefined; @@ -1619,6 +1713,7 @@ export class AgentSession { if (this._accountGoalUsageForAssistantMessage(assistantMsg)) { this.agent.steer(createGoalContextMessage(this._goalState, "budget_limit")); } + addAutonomousUsage(this._autonomousState, assistantMsg.usage); } } @@ -1965,6 +2060,10 @@ export class AgentSession { return { ...this._goalWithCurrentWallClock() }; } + getAutonomousStatus(): AgentAutonomousStatus { + return autonomousStatus(this._autonomousState); + } + /** Scoped models for cycling (from --models flag) */ get scopedModels(): ReadonlyArray<{ model: Model; thinkingLevel?: ThinkingLevel }> { return this._scopedModels; @@ -2066,6 +2165,11 @@ export class AgentSession { let currentImages = options?.images; if (expandPromptTemplates) { + const handledAutonomousCommand = await this._handleAutonomousSlashCommand(currentText); + if (handledAutonomousCommand) { + preflightResult?.(true); + return; + } const handledGoalCommand = await this._handleGoalSlashCommand(currentText, currentImages); if (handledGoalCommand) { preflightResult?.(true); @@ -2207,8 +2311,13 @@ export class AgentSession { } preflightResult?.(true); + // `isStreaming` can become false before the lower-level Agent has fully + // completed its run lifecycle. Wait for that lifecycle to settle before + // starting the next print-mode/autonomous gate prompt. + await this.agent.waitForIdle(); await this.agent.prompt(messages); await this.waitForRetry(); + await this._waitForPostCompactionContinuations(); } /** @@ -3076,11 +3185,51 @@ export class AgentSession { const contextTokens = this._getThresholdContextTokens(assistantMessage, compactionTimestamp); if (contextTokens === undefined) return false; if (shouldCompact(contextTokens, contextWindow, settings)) { + if (this._queueAutonomousContinuationForThresholdCompaction(assistantMessage)) { + this._continueAfterThresholdCompaction = true; + } return await this._runAutoCompaction("threshold", false); } return false; } + private _schedulePostCompactionContinue(): void { + if (this._postCompactionContinuePromise) { + return; + } + let scheduledPromise: Promise; + scheduledPromise = new Promise((resolve) => { + setTimeout(() => { + this._runPostCompactionContinue() + .catch(() => {}) + .finally(resolve); + }, 100); + }); + this._postCompactionContinuePromise = scheduledPromise.finally(() => { + if (this._postCompactionContinuePromise === scheduledPromise) { + this._postCompactionContinuePromise = undefined; + } + }); + } + + private async _runPostCompactionContinue(): Promise { + const continuationMessages = this._postCompactionContinuationMessages.splice(0); + const lastMessage = this.agent.state.messages[this.agent.state.messages.length - 1]; + if (continuationMessages.length > 0 && lastMessage?.role !== "assistant") { + const continuationMessageSet = new Set(continuationMessages); + this.agent.removeQueuedMessages((message) => continuationMessageSet.has(message)); + await this.agent.prompt(continuationMessages); + return; + } + await this.agent.continue(); + } + + private async _waitForPostCompactionContinuations(): Promise { + while (this._postCompactionContinuePromise) { + await this._postCompactionContinuePromise; + } + } + /** * Internal: Run auto-compaction with events. */ @@ -3130,6 +3279,10 @@ export class AgentSession { errorMessage: "Auto-compaction skipped: nothing to summarize outside the recent-context window", errorSeverity: "warning", }); + if (shouldContinueAfterThreshold || this.agent.hasQueuedMessages()) { + this._schedulePostCompactionContinue(); + return true; + } return false; } @@ -3235,16 +3388,13 @@ export class AgentSession { this.agent.state.messages = messages.slice(0, -1); } - setTimeout(() => { - this.agent.continue().catch(() => {}); - }, 100); + this._schedulePostCompactionContinue(); return true; } else if (shouldContinueAfterThreshold || this.agent.hasQueuedMessages()) { // Threshold compaction can intentionally stop a tool loop between turns. // Queued follow-up/steering/custom messages can also be waiting. - setTimeout(() => { - this.agent.continue().catch(() => {}); - }, 100); + this._schedulePostCompactionContinue(); + return true; } return false; } catch (error) { diff --git a/packages/coding-agent/src/core/autonomous.ts b/packages/coding-agent/src/core/autonomous.ts new file mode 100644 index 0000000000..713748751e --- /dev/null +++ b/packages/coding-agent/src/core/autonomous.ts @@ -0,0 +1,371 @@ +import { spawnSync } from "node:child_process"; +import type { AssistantMessage, TextContent, Usage, UserMessage } from "@earendil-works/pi-ai"; + +export interface AgentAutonomousConfig { + enabled?: boolean; + maxContinuations?: number; + maxTurns?: number; + maxTokens?: number; + timeoutMs?: number; + continuationPrompt?: string; + gates?: AgentAutonomousGateConfig; +} + +export interface AgentAutonomousGateConfig { + commands?: string[]; + maxRetries?: number; + timeoutMs?: number; +} + +export interface AgentAutonomousGateFailure { + command: string; + attempt: number; + exitText: string; + output: string; +} + +export interface AgentAutonomousStatus { + enabled: boolean; + continuationsUsed: number; + turnsUsed: number; + tokensUsed: number; + startedAt?: number; + limits: Required>; + gates: Required; + gateAttempts: Record; + lastGateFailure?: AgentAutonomousGateFailure; +} + +export const DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT = + "No human input is available in autonomous mode. Continue working. If you were asking the user a question, make a reasonable assumption and verify it. If you believe you are blocked, prove it with host-observable evidence. If you believe the task is complete, produce completion evidence: passing gates or a patch relative to the autonomous baseline."; + +export const DEFAULT_AUTONOMOUS_LIMITS: Required< + Omit +> = { + maxContinuations: 3, + maxTurns: 12, + maxTokens: 80_000, + timeoutMs: 30 * 60 * 1000, +}; + +export const DEFAULT_AUTONOMOUS_GATES: Required = { + commands: [], + maxRetries: 3, + timeoutMs: 5 * 60 * 1000, +}; + +export interface AutonomousRuntimeState { + enabled: boolean; + continuationsUsed: number; + turnsUsed: number; + tokensUsed: number; + startedAt?: number; + limits: Required>; + continuationPrompt: string; + gates: Required; + gateAttempts: Record; + lastGateFailure?: GateFailure; + lastGateFailureSnapshot?: GitWorktreeSnapshot; + gitBaseline?: GitWorktreeSnapshot; +} + +export interface AutonomousDecision { + shouldContinue: boolean; + reason: "missing_terminal_evidence" | "gate_failed" | "not_needed" | "limit_reached"; +} + +interface GitWorktreeSnapshot { + status: string; + diff: string; +} + +type GateFailure = AgentAutonomousGateFailure; + +export function createAutonomousRuntimeState( + config?: AgentAutonomousConfig, + options: { cwd?: string } = {}, +): AutonomousRuntimeState { + const enabled = config?.enabled === true; + return { + enabled, + continuationsUsed: 0, + turnsUsed: 0, + tokensUsed: 0, + startedAt: enabled ? Date.now() : undefined, + limits: { + maxContinuations: normalizeLimit(config?.maxContinuations, DEFAULT_AUTONOMOUS_LIMITS.maxContinuations), + maxTurns: normalizeLimit(config?.maxTurns, DEFAULT_AUTONOMOUS_LIMITS.maxTurns), + maxTokens: normalizeLimit(config?.maxTokens, DEFAULT_AUTONOMOUS_LIMITS.maxTokens), + timeoutMs: normalizeLimit(config?.timeoutMs, DEFAULT_AUTONOMOUS_LIMITS.timeoutMs), + }, + continuationPrompt: config?.continuationPrompt?.trim() || DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT, + gates: { + commands: [...(config?.gates?.commands ?? DEFAULT_AUTONOMOUS_GATES.commands)], + maxRetries: normalizeLimit(config?.gates?.maxRetries, DEFAULT_AUTONOMOUS_GATES.maxRetries), + timeoutMs: normalizeLimit(config?.gates?.timeoutMs, DEFAULT_AUTONOMOUS_GATES.timeoutMs), + }, + gateAttempts: {}, + lastGateFailure: undefined, + lastGateFailureSnapshot: undefined, + gitBaseline: enabled ? captureGitWorktreeSnapshot(options.cwd) : undefined, + }; +} + +export function setAutonomousEnabled( + state: AutonomousRuntimeState, + enabled: boolean, + options: { cwd?: string } = {}, +): void { + state.enabled = enabled; + if (enabled) { + state.continuationsUsed = 0; + state.turnsUsed = 0; + state.tokensUsed = 0; + state.startedAt = Date.now(); + state.gateAttempts = {}; + state.lastGateFailure = undefined; + state.lastGateFailureSnapshot = undefined; + state.gitBaseline = captureGitWorktreeSnapshot(options.cwd); + } else { + state.startedAt = undefined; + state.gateAttempts = {}; + state.lastGateFailure = undefined; + state.lastGateFailureSnapshot = undefined; + state.gitBaseline = undefined; + } +} + +export function autonomousStatus(state: AutonomousRuntimeState): AgentAutonomousStatus { + return { + enabled: state.enabled, + continuationsUsed: state.continuationsUsed, + turnsUsed: state.turnsUsed, + tokensUsed: state.tokensUsed, + startedAt: state.startedAt, + limits: { ...state.limits }, + gates: { ...state.gates, commands: [...state.gates.commands] }, + gateAttempts: { ...state.gateAttempts }, + lastGateFailure: state.lastGateFailure ? { ...state.lastGateFailure } : undefined, + }; +} + +export function addAutonomousUsage(state: AutonomousRuntimeState, usage: Usage | undefined): void { + if (!state.enabled) { + return; + } + state.turnsUsed++; + state.tokensUsed += autonomousTokenDelta(usage); +} + +function autonomousTokenDelta(usage: Usage | undefined): number { + if (!usage) { + return 0; + } + // Cache-read tokens are repeated context served from provider cache. Counting them + // cumulatively makes long autonomous verifier loops exhaust their host-side token + // budget far before the non-cached work reaches the configured cap. + return usage.input + usage.output + usage.cacheWrite; +} + +export function nextAutonomousContinuation( + state: AutonomousRuntimeState, + message: AssistantMessage, + options: { cwd?: string } = {}, + now = Date.now(), +): UserMessage | undefined { + if (!state.enabled) { + return undefined; + } + const decision = shouldAutonomouslyContinue(state, message, options, now); + if (!decision.shouldContinue) { + return undefined; + } + state.continuationsUsed++; + return { + role: "user", + content: [ + { + type: "text", + text: + decision.reason === "gate_failed" + ? (buildGateFailureContinuation(state, now) ?? state.continuationPrompt) + : state.continuationPrompt, + }, + ], + timestamp: now, + }; +} + +export function shouldAutonomouslyContinue( + state: AutonomousRuntimeState, + message: AssistantMessage, + options: { cwd?: string } = {}, + now = Date.now(), +): AutonomousDecision { + if (!state.enabled || message.stopReason === "error" || message.stopReason === "aborted") { + return { shouldContinue: false, reason: "not_needed" }; + } + if (autonomousLimitReached(state, now)) { + return { shouldContinue: false, reason: "limit_reached" }; + } + if (state.gates.commands.length > 0) { + const gateResult = runAutonomousQualityGates(state, options.cwd); + if (gateResult === "passed" || gateResult === "retry_exhausted") { + return { shouldContinue: false, reason: "not_needed" }; + } + return { shouldContinue: true, reason: "gate_failed" }; + } + if (hasGitWorktreeChangedSinceBaseline(state, options.cwd)) { + return { shouldContinue: false, reason: "not_needed" }; + } + return { shouldContinue: true, reason: "missing_terminal_evidence" }; +} + +export function assistantText(message: AssistantMessage): string { + return message.content + .filter((part): part is TextContent => part.type === "text") + .map((part) => part.text) + .join("\n") + .trim(); +} + +function autonomousLimitReached(state: AutonomousRuntimeState, now: number): boolean { + if (state.continuationsUsed >= state.limits.maxContinuations) { + return true; + } + if (state.turnsUsed >= state.limits.maxTurns) { + return true; + } + if (state.tokensUsed >= state.limits.maxTokens) { + return true; + } + return state.startedAt !== undefined && now - state.startedAt >= state.limits.timeoutMs; +} + +type GateResult = "passed" | "failed" | "retry_exhausted"; + +function runAutonomousQualityGates(state: AutonomousRuntimeState, cwd: string | undefined): GateResult { + if (!cwd) { + return "failed"; + } + for (const command of state.gates.commands) { + const currentSnapshot = captureGitWorktreeSnapshot(cwd); + if ( + state.lastGateFailure?.command === command && + state.lastGateFailureSnapshot && + gitWorktreeSnapshotsEqual(currentSnapshot, state.lastGateFailureSnapshot) + ) { + state.lastGateFailure = { + ...state.lastGateFailure, + exitText: "not rerun: workspace unchanged since previous failed gate", + output: truncateGateOutput( + `${state.lastGateFailure.output}\n\nThe autonomous gate was not rerun because the workspace has not changed since this failure. Edit source files, tests, or a blocker artifact before attempting to finish again.`, + ), + }; + return "failed"; + } + const result = spawnSync(command, { + cwd, + encoding: "utf8", + shell: true, + timeout: state.gates.timeoutMs, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status === 0 && !result.error) { + state.gateAttempts[command] = 0; + if (state.lastGateFailure?.command === command) { + state.lastGateFailure = undefined; + state.lastGateFailureSnapshot = undefined; + } + continue; + } + const attempt = (state.gateAttempts[command] ?? 0) + 1; + state.gateAttempts[command] = attempt; + const exitText = + result.error?.message ?? + (result.signal ? `terminated by ${result.signal}` : `exited ${result.status ?? "unknown"}`); + state.lastGateFailure = { + command, + attempt, + exitText, + output: truncateGateOutput([result.stdout, result.stderr].filter(Boolean).join("\n").trim()), + }; + state.lastGateFailureSnapshot = currentSnapshot; + return attempt > state.gates.maxRetries ? "retry_exhausted" : "failed"; + } + state.lastGateFailure = undefined; + state.lastGateFailureSnapshot = undefined; + return "passed"; +} + +function buildGateFailureContinuation(state: AutonomousRuntimeState, timestamp: number): string | undefined { + const failure = state.lastGateFailure; + if (!failure) { + return undefined; + } + return ( + `Autonomous quality gate failed (attempt ${failure.attempt}/${state.gates.maxRetries}): \`${failure.command}\` ${failure.exitText}.\n` + + (failure.output ? `\nOutput:\n${failure.output}\n` : "\n") + + `\nContinue working. Fix the failure, then produce terminal evidence. Timestamp: ${new Date(timestamp).toISOString()}.` + ); +} + +function hasGitWorktreeChangedSinceBaseline(state: AutonomousRuntimeState, cwd: string | undefined): boolean { + const current = captureGitWorktreeSnapshot(cwd); + if (!current || !state.gitBaseline) { + return false; + } + return current.status !== state.gitBaseline.status || current.diff !== state.gitBaseline.diff; +} + + +function gitWorktreeSnapshotsEqual(a: GitWorktreeSnapshot | undefined, b: GitWorktreeSnapshot | undefined): boolean { + return !!a && !!b && a.status === b.status && a.diff === b.diff; +} + +function captureGitWorktreeSnapshot(cwd: string | undefined): GitWorktreeSnapshot | undefined { + if (!cwd) { + return undefined; + } + const pathspec = [ + "--", + ".", + ":(exclude)verification", + ":(exclude)target", + ":(exclude).vf-prime-agent", + ":(exclude)Cargo.lock", + ":(exclude)submission.tar.gz", + ":(exclude)runner_args.log", + ]; + const status = spawnSync("git", ["--no-optional-locks", "status", "--porcelain=v1", "-uall", ...pathspec], { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + if (status.status !== 0 || typeof status.stdout !== "string") { + return undefined; + } + const diff = spawnSync("git", ["--no-optional-locks", "diff", "--no-ext-diff", "--binary", "HEAD", ...pathspec], { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + return { + status: status.stdout, + diff: diff.status === 0 && typeof diff.stdout === "string" ? diff.stdout : "", + }; +} + +function truncateGateOutput(output: string, maxChars = 6000): string { + if (output.length <= maxChars) { + return output; + } + return `${output.slice(0, maxChars)}\n... [truncated ${output.length - maxChars} chars]`; +} + +function normalizeLimit(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value) || value === undefined || value <= 0) { + return fallback; + } + return Math.trunc(value); +} diff --git a/packages/coding-agent/src/core/sdk.ts b/packages/coding-agent/src/core/sdk.ts index 2695f8e06a..43e560d329 100644 --- a/packages/coding-agent/src/core/sdk.ts +++ b/packages/coding-agent/src/core/sdk.ts @@ -6,6 +6,7 @@ import { AgentSession } from "./agent-session.js"; import type { AgentSessionCreationOptions } from "./agent-session-services.js"; import { formatNoModelsAvailableMessage } from "./auth-guidance.js"; import { AuthStorage } from "./auth-storage.js"; +import type { AgentAutonomousConfig } from "./autonomous.js"; import { DEFAULT_THINKING_LEVEL } from "./defaults.js"; import type { ExtensionRunner, LoadExtensionsResult, SessionStartEvent, ToolDefinition } from "./extensions/index.js"; import { convertToLlm } from "./messages.js"; @@ -65,6 +66,8 @@ export interface CreateAgentSessionOptions extends AgentSessionCreationOptions { settingsManager?: SettingsManager; /** Session start event metadata for extension runtime startup. */ sessionStartEvent?: SessionStartEvent; + /** Host-side autonomous continuation policy. */ + autonomous?: AgentAutonomousConfig; } /** Result from createAgentSession */ @@ -358,6 +361,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} subagentRuntimeHost: options.subagentRuntimeHost, sessionStartEvent: options.sessionStartEvent, prewarmIpythonKernel: options.prewarmIpythonKernel, + autonomous: options.autonomous, }); const extensionsResult = resourceLoader.getExtensions(); diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index 6ec9ace2dc..b83f2af4f8 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -623,6 +623,9 @@ function buildSessionOptions( if (config.tools) { options.tools = [...config.tools]; } + if (config.autonomous) { + options.autonomous = { ...config.autonomous }; + } return { options, cliThinkingFromModel, diagnostics }; } @@ -660,6 +663,23 @@ function runtimeConfigFromArgs( themes: resolveCliPaths(cwd, parsed.themes), noThemes: parsed.noThemes, noContextFiles: parsed.noContextFiles, + autonomous: parsed.autonomous + ? { + enabled: true, + maxContinuations: parsed.autonomousMaxContinuations, + maxTurns: parsed.autonomousMaxTurns, + maxTokens: parsed.autonomousMaxTokens, + timeoutMs: parsed.autonomousTimeoutMs, + gates: + parsed.autonomousGates || parsed.autonomousGateRetries || parsed.autonomousGateTimeoutMs + ? { + commands: parsed.autonomousGates, + maxRetries: parsed.autonomousGateRetries, + timeoutMs: parsed.autonomousGateTimeoutMs, + } + : undefined, + } + : undefined, extensionFlagValues: parsed.unknownFlags.size > 0 ? Object.fromEntries(parsed.unknownFlags.entries()) : undefined, }; } @@ -687,6 +707,7 @@ export function resolveRuntimeSessionOptions( allowedToolNames: runtimeSessionOptions?.allowedToolNames, includeGoals: runtimeSessionOptions?.includeGoals, rlmHeartbeatController: runtimeSessionOptions?.rlmHeartbeatController, + autonomous: runtimeSessionOptions?.autonomous ?? sessionOptions.autonomous, rlmDepth: runtimeSessionOptions?.rlmDepth, rlmMaxDepth: runtimeSessionOptions?.rlmMaxDepth, rlmSessionDir: runtimeSessionOptions?.rlmSessionDir, diff --git a/packages/coding-agent/src/modes/print-mode.ts b/packages/coding-agent/src/modes/print-mode.ts index 2f9fb048e2..8dfe026a79 100644 --- a/packages/coding-agent/src/modes/print-mode.ts +++ b/packages/coding-agent/src/modes/print-mode.ts @@ -8,6 +8,7 @@ import type { AssistantMessage, ImageContent } from "@earendil-works/pi-ai"; import type { AgentSessionRuntime } from "../core/agent-session-runtime.js"; +import type { AgentAutonomousGateFailure, AgentAutonomousStatus } from "../core/autonomous.js"; import { flushRawStdout, writeRawStdout } from "../core/output-guard.js"; import { killTrackedDetachedChildren } from "../utils/shell.js"; @@ -25,6 +26,58 @@ export interface PrintModeOptions { initialImages?: ImageContent[]; } +function latestGateAttempt(status: AgentAutonomousStatus): number { + return Math.max(status.lastGateFailure?.attempt ?? 0, 0, ...Object.values(status.gateAttempts)); +} + +function autonomousLimitsReached(status: AgentAutonomousStatus, now = Date.now()): boolean { + if (status.continuationsUsed >= status.limits.maxContinuations) return true; + if (status.turnsUsed >= status.limits.maxTurns) return true; + if (status.tokensUsed >= status.limits.maxTokens) return true; + return status.startedAt !== undefined && now - status.startedAt >= status.limits.timeoutMs; +} + +function shouldContinuePrintModeAutonomousGates(status: AgentAutonomousStatus): boolean { + if (!status.enabled || status.gates.commands.length === 0 || !status.lastGateFailure) return false; + if (autonomousLimitsReached(status)) return false; + return latestGateAttempt(status) <= status.gates.maxRetries; +} + +function buildPrintModeGateContinuation( + failure: AgentAutonomousGateFailure, + attempt: number, + maxRetries: number, +): string { + return ( + `Autonomous quality gate failed (attempt ${attempt}/${maxRetries}): \`${failure.command}\` ${failure.exitText}.\n` + + (failure.output ? `\nOutput:\n${failure.output}\n` : "\n") + + `\nContinue working. Fix the failure, then produce terminal evidence. Timestamp: ${new Date().toISOString()}.` + ); +} + +async function waitForPrintModeIdleWithAutonomousGates( + getSession: () => AgentSessionRuntime["session"], +): Promise { + let lastPromptedGateAttempt = 0; + while (true) { + const session = getSession(); + await session.agent.waitForIdle(); + const status = session.getAutonomousStatus(); + const attempt = latestGateAttempt(status); + if ( + !shouldContinuePrintModeAutonomousGates(status) || + !status.lastGateFailure || + attempt <= lastPromptedGateAttempt + ) { + return; + } + lastPromptedGateAttempt = attempt; + await session.prompt(buildPrintModeGateContinuation(status.lastGateFailure, attempt, status.gates.maxRetries), { + streamingBehavior: "followUp", + }); + } +} + /** * Run in print (single-shot) mode. * Sends prompts to the agent and outputs the result. @@ -125,6 +178,8 @@ export async function runPrintMode(runtimeHost: AgentSessionRuntime, options: Pr await session.prompt(message); } + await waitForPrintModeIdleWithAutonomousGates(() => session); + if (mode === "text") { const state = session.state; const lastMessage = state.messages[state.messages.length - 1]; @@ -144,6 +199,14 @@ export async function runPrintMode(runtimeHost: AgentSessionRuntime, options: Pr } } + const autonomousStatus = session.getAutonomousStatus(); + if (autonomousStatus.enabled && autonomousStatus.gates.commands.length > 0 && autonomousStatus.lastGateFailure) { + console.error( + `Autonomous quality gate still failing after attempt ${latestGateAttempt(autonomousStatus)}/${autonomousStatus.gates.maxRetries}: ${autonomousStatus.lastGateFailure.exitText}`, + ); + exitCode = 1; + } + return exitCode; } catch (error: unknown) { console.error(error instanceof Error ? error.message : String(error)); diff --git a/packages/coding-agent/src/utils/child-process.ts b/packages/coding-agent/src/utils/child-process.ts index 9f75a285d8..b5843ddd40 100644 --- a/packages/coding-agent/src/utils/child-process.ts +++ b/packages/coding-agent/src/utils/child-process.ts @@ -25,8 +25,8 @@ export function waitForChildProcess(child: ChildProcess): Promise let exited = false; let exitCode: number | null = null; let postExitTimer: NodeJS.Timeout | undefined; - let stdoutEnded = child.stdout === null; - let stderrEnded = child.stderr === null; + let stdoutEnded = child.stdout === null || child.stdout.readableEnded; + let stderrEnded = child.stderr === null || child.stderr.readableEnded; const cleanup = () => { if (postExitTimer) { @@ -91,5 +91,9 @@ export function waitForChildProcess(child: ChildProcess): Promise child.once("error", onError); child.once("exit", onExit); child.once("close", onClose); + + if (child.exitCode !== null || child.signalCode !== null) { + onExit(child.exitCode); + } }); } diff --git a/packages/coding-agent/test/args.test.ts b/packages/coding-agent/test/args.test.ts index 53744843f8..0f17226f1a 100644 --- a/packages/coding-agent/test/args.test.ts +++ b/packages/coding-agent/test/args.test.ts @@ -279,6 +279,50 @@ describe("parseArgs", () => { }); }); + describe("--autonomous flag", () => { + test("parses --autonomous flag", () => { + const result = parseArgs(["--autonomous"]); + expect(result.autonomous).toBe(true); + }); + + test("parses autonomous gate flags", () => { + const result = parseArgs([ + "--autonomous", + "--autonomous-gate", + "npm test", + "--autonomous-gate", + "npm run lint", + "--autonomous-gate-retries", + "2", + "--autonomous-gate-timeout-ms", + "1000", + ]); + expect(result.autonomous).toBe(true); + expect(result.autonomousGates).toEqual(["npm test", "npm run lint"]); + expect(result.autonomousGateRetries).toBe(2); + expect(result.autonomousGateTimeoutMs).toBe(1000); + }); + + test("parses autonomous limit flags", () => { + const result = parseArgs([ + "--autonomous", + "--autonomous-max-continuations", + "20", + "--autonomous-max-turns", + "80", + "--autonomous-max-tokens", + "500000", + "--autonomous-timeout-ms", + "1800000", + ]); + expect(result.autonomous).toBe(true); + expect(result.autonomousMaxContinuations).toBe(20); + expect(result.autonomousMaxTurns).toBe(80); + expect(result.autonomousMaxTokens).toBe(500000); + expect(result.autonomousTimeoutMs).toBe(1800000); + }); + }); + describe("tool flags", () => { test("parses --no-tools flag", () => { const result = parseArgs(["--no-tools"]); diff --git a/packages/coding-agent/test/print-mode.test.ts b/packages/coding-agent/test/print-mode.test.ts index bbf6756e04..cd4a549712 100644 --- a/packages/coding-agent/test/print-mode.test.ts +++ b/packages/coding-agent/test/print-mode.test.ts @@ -1,5 +1,6 @@ import type { AssistantMessage, ImageContent } from "@earendil-works/pi-ai"; import { afterEach, describe, expect, it, vi } from "vitest"; +import type { AgentAutonomousStatus } from "../src/core/autonomous.js"; import type { SessionShutdownEvent } from "../src/index.js"; import { runPrintMode } from "../src/modes/print-mode.js"; @@ -12,13 +13,14 @@ type FakeExtensionRunner = { type FakeSession = { sessionManager: { getHeader: () => object | undefined }; - agent: { waitForIdle: () => Promise }; + agent: { waitForIdle: ReturnType Promise>> }; state: { messages: AssistantMessage[] }; extensionRunner: FakeExtensionRunner; bindExtensions: ReturnType; subscribe: ReturnType; prompt: ReturnType; reload: ReturnType; + getAutonomousStatus: ReturnType; }; type FakeRuntimeHost = { @@ -55,7 +57,18 @@ function createAssistantMessage(options?: { }; } -function createRuntimeHost(assistantMessage: AssistantMessage): FakeRuntimeHost { +function createRuntimeHost( + assistantMessage: AssistantMessage, + autonomousStatus: AgentAutonomousStatus = { + enabled: false, + continuationsUsed: 0, + turnsUsed: 0, + tokensUsed: 0, + limits: { maxContinuations: 3, maxTurns: 12, maxTokens: 80_000, timeoutMs: 1_800_000 }, + gates: { commands: [], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: {}, + }, +): FakeRuntimeHost { const extensionRunner: FakeExtensionRunner = { hasHandlers: (eventType: string) => eventType === "session_shutdown", emit: vi.fn(async () => {}), @@ -65,13 +78,14 @@ function createRuntimeHost(assistantMessage: AssistantMessage): FakeRuntimeHost const session: FakeSession = { sessionManager: { getHeader: () => undefined }, - agent: { waitForIdle: async () => {} }, + agent: { waitForIdle: vi.fn(async () => {}) }, state, extensionRunner, bindExtensions: vi.fn(async () => {}), subscribe: vi.fn(() => () => {}), prompt: vi.fn(async () => {}), reload: vi.fn(async () => {}), + getAutonomousStatus: vi.fn(() => autonomousStatus), }; return { @@ -139,4 +153,143 @@ describe("runPrintMode", () => { expect(session.extensionRunner.emit).toHaveBeenCalledTimes(1); expect(session.extensionRunner.emit).toHaveBeenCalledWith({ type: "session_shutdown", reason: "quit" }); }); + + it("returns non-zero when autonomous gates are still failing", async () => { + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { + enabled: true, + continuationsUsed: 34, + turnsUsed: 92, + tokensUsed: 215_535, + limits: { maxContinuations: 999, maxTurns: 1000, maxTokens: 2_000_000, timeoutMs: 1_800_000 }, + gates: { commands: ["verify-public"], maxRetries: 999, timeoutMs: 3_600_000 }, + gateAttempts: { "verify-public": 34 }, + lastGateFailure: { + command: "verify-public", + attempt: 34, + exitText: "exited 1", + output: "0/43", + }, + }); + const { session } = runtimeHost; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(1); + expect(errorSpy).toHaveBeenCalledWith("Autonomous quality gate still failing after attempt 34/999: exited 1"); + expect(session.extensionRunner.emit).toHaveBeenCalledWith({ type: "session_shutdown", reason: "quit" }); + }); + + it("keeps prompting while autonomous gates fail below retry limits", async () => { + const statuses: AgentAutonomousStatus[] = [ + { + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "exited 1", + output: "0/9", + }, + }, + { + enabled: true, + continuationsUsed: 2, + turnsUsed: 3, + tokensUsed: 200, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 2 }, + lastGateFailure: { + command: "verify-public", + attempt: 2, + exitText: "exited 1", + output: "0/9 summary", + }, + }, + { + enabled: true, + continuationsUsed: 2, + turnsUsed: 4, + tokensUsed: 250, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 2 }, + }, + ]; + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still working" }), statuses[0]); + const { session } = runtimeHost; + let statusIndex = 0; + session.getAutonomousStatus.mockImplementation( + () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, + ); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(0); + expect(session.agent.waitForIdle).toHaveBeenCalledBefore(session.prompt); + expect(session.prompt).toHaveBeenCalledTimes(2); + expect(session.prompt.mock.calls[0][0]).toContain("Autonomous quality gate failed (attempt 1/3)"); + expect(session.prompt.mock.calls[0][0]).toContain("0/9"); + expect(session.prompt.mock.calls[0][1]).toEqual({ streamingBehavior: "followUp" }); + expect(session.prompt.mock.calls[1][0]).toContain("Autonomous quality gate failed (attempt 2/3)"); + expect(session.prompt.mock.calls[1][0]).toContain("0/9 summary"); + expect(session.prompt.mock.calls[1][1]).toEqual({ streamingBehavior: "followUp" }); + }); + + it("does not synchronously spin when a re-prompt does not advance gate attempts", async () => { + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "exited 1", + output: "0/9", + }, + }); + const { session } = runtimeHost; + session.getAutonomousStatus.mockImplementation(() => ({ + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "exited 1", + output: "0/9", + }, + })); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(1); + expect(session.agent.waitForIdle).toHaveBeenCalledTimes(2); + expect(session.prompt).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/coding-agent/test/suite/agent-session-autonomous.test.ts b/packages/coding-agent/test/suite/agent-session-autonomous.test.ts new file mode 100644 index 0000000000..cd2f4c5f26 --- /dev/null +++ b/packages/coding-agent/test/suite/agent-session-autonomous.test.ts @@ -0,0 +1,243 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { fauxAssistantMessage } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it } from "vitest"; +import { + addAutonomousUsage, + createAutonomousRuntimeState, + DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT, + nextAutonomousContinuation, + shouldAutonomouslyContinue, +} from "../../src/core/autonomous.js"; +import { createHarness, getAssistantTexts, getMessageText, getUserTexts, type Harness } from "./harness.js"; + +describe("AgentSession autonomous mode", () => { + const harnesses: Harness[] = []; + + afterEach(() => { + while (harnesses.length > 0) { + harnesses.pop()?.cleanup(); + } + }); + + it("injects a host-side continuation when the assistant asks the user for help", async () => { + const harness = await createHarness({ + autonomous: { enabled: true, maxContinuations: 1 }, + }); + harnesses.push(harness); + harness.setResponses([ + fauxAssistantMessage("Which package manager should I use?"), + fauxAssistantMessage("I inspected the repo and used npm."), + ]); + + await harness.session.prompt("fix the project"); + + expect(getAssistantTexts(harness)).toEqual([ + "Which package manager should I use?", + "I inspected the repo and used npm.", + ]); + expect(getUserTexts(harness)).toEqual(["fix the project", DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT]); + expect(harness.session.getAutonomousStatus()).toMatchObject({ + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + }); + }); + + it("continues through a claimed external blocker instead of trusting prose", async () => { + const harness = await createHarness({ + autonomous: { enabled: true, maxContinuations: 1 }, + }); + harnesses.push(harness); + harness.setResponses([ + fauxAssistantMessage("Blocked: this requires an API key credential from the user."), + fauxAssistantMessage( + "I will inspect the environment and verify whether the credential is actually unavailable.", + ), + ]); + + await harness.session.prompt("run the private eval"); + + expect(getUserTexts(harness)).toEqual(["run the private eval", DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT]); + expect(harness.session.getAutonomousStatus()).toMatchObject({ + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + }); + }); + + it("stops after the configured autonomous continuation limit", async () => { + const harness = await createHarness({ + autonomous: { enabled: true, maxContinuations: 1 }, + }); + harnesses.push(harness); + harness.setResponses([ + fauxAssistantMessage("Can you confirm the test command?"), + fauxAssistantMessage("Can you confirm whether to run lint too?"), + ]); + + await harness.session.prompt("make the change"); + + expect(getAssistantTexts(harness)).toEqual([ + "Can you confirm the test command?", + "Can you confirm whether to run lint too?", + ]); + expect(getUserTexts(harness)).toEqual(["make the change", DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT]); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + }); + + it("supports /autonomous on and off without calling the model", async () => { + const harness = await createHarness(); + harnesses.push(harness); + + await harness.session.prompt("/autonomous on"); + await harness.session.prompt("/autonomous off"); + + expect(harness.getPendingResponseCount()).toBe(0); + expect(harness.session.getAutonomousStatus().enabled).toBe(false); + const statusMessages = harness.session.messages.filter( + (message) => message.role === "custom" && message.customType === "autonomous_status", + ); + expect(statusMessages).toHaveLength(2); + }); + + it("continues when the assistant tries to finish without terminal evidence", async () => { + const harness = await createHarness({ + autonomous: { enabled: true, maxContinuations: 1 }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("Done."), fauxAssistantMessage("I will collect concrete evidence.")]); + + await harness.session.prompt("make the change"); + + expect(getUserTexts(harness)).toEqual(["make the change", DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT]); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + }); + + it("accepts a git worktree change relative to the autonomous baseline as terminal evidence", async () => { + const harness = await createHarness(); + harnesses.push(harness); + execFileSync("git", ["init"], { cwd: harness.tempDir, stdio: "ignore" }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: harness.tempDir }); + execFileSync("git", ["config", "user.name", "Test User"], { cwd: harness.tempDir }); + const path = join(harness.tempDir, "file.txt"); + writeFileSync(path, "before\n"); + execFileSync("git", ["add", "file.txt"], { cwd: harness.tempDir }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "--no-gpg-sign", "-m", "initial"], { + cwd: harness.tempDir, + stdio: "ignore", + }); + await harness.session.prompt("/autonomous on"); + writeFileSync(path, "after\n"); + harness.setResponses([fauxAssistantMessage("Done.")]); + + await harness.session.prompt("make the change"); + + expect(getUserTexts(harness)).toEqual(["make the change"]); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(0); + }); + + it("accepts passing autonomous gates as terminal evidence", async () => { + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 1, + gates: { commands: [`${process.execPath} -e "process.exit(0)"`] }, + }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("Done.")]); + + await harness.session.prompt("make the change"); + + expect(getUserTexts(harness)).toEqual(["make the change"]); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(0); + }); + + it("feeds failing autonomous gate output back into the session", async () => { + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 1, + gates: { + commands: [`${process.execPath} -e "console.error('gate failed'); process.exit(1)"`], + maxRetries: 2, + }, + }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("Done."), fauxAssistantMessage("I will fix the gate failure.")]); + + await harness.session.prompt("make the change"); + + const users = getUserTexts(harness); + expect(users[1]).toContain("Autonomous quality gate failed"); + expect(users[1]).toContain("gate failed"); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + }); + + it("does not rerun a failed autonomous gate until the workspace changes", () => { + const tempDir = join(process.cwd(), `.tmp-autonomous-gate-${Date.now()}-${Math.random().toString(36).slice(2)}`); + execFileSync("mkdir", ["-p", join(tempDir, "verification")]); + execFileSync("git", ["init"], { cwd: tempDir, stdio: "ignore" }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: tempDir }); + execFileSync("git", ["config", "user.name", "Test User"], { cwd: tempDir }); + writeFileSync(join(tempDir, "src.rs"), "initial\n"); + execFileSync("git", ["add", "src.rs"], { cwd: tempDir }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "--no-gpg-sign", "-m", "initial"], { cwd: tempDir, stdio: "ignore" }); + try { + const counter = join(tempDir, "verification", "public_feedback_scores.jsonl"); + const gate = `${process.execPath} -e "const fs=require('fs'); const p='${counter}'; const n=fs.existsSync(p)?fs.readFileSync(p,'utf8').trim().split(/\\n/).filter(Boolean).length:0; fs.appendFileSync(p,JSON.stringify({run:n+1,score:0})+'\\n'); process.exit(1);"`; + const state = createAutonomousRuntimeState({ enabled: true, maxContinuations: 3, gates: { commands: [gate], maxRetries: 3 } }, { cwd: tempDir }); + + const first = nextAutonomousContinuation(state, fauxAssistantMessage("Done."), { cwd: tempDir }); + writeFileSync(join(tempDir, "Cargo.lock"), "generated lockfile\n"); + const second = nextAutonomousContinuation(state, fauxAssistantMessage("Still done."), { cwd: tempDir }); + + expect(first).toBeDefined(); + expect(second).toBeDefined(); + expect(getMessageText(second)).toContain("workspace has not changed"); + expect(getMessageText(second)).toContain("Edit source files"); + expect(readFileSync(counter, "utf8").trim().split(/\n/)).toHaveLength(1); + expect(state.gateAttempts[gate]).toBe(1); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("does not count cache-read tokens against the autonomous token budget", () => { + const state = createAutonomousRuntimeState({ enabled: true, maxTokens: 10 }); + + addAutonomousUsage(state, { + input: 2, + output: 3, + cacheRead: 1_000, + cacheWrite: 4, + totalTokens: 1_009, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }); + + expect(state.tokensUsed).toBe(9); + expect(shouldAutonomouslyContinue(state, fauxAssistantMessage("Done."))).toMatchObject({ + shouldContinue: true, + }); + }); + + it("does not use assistant prose as terminal blocker evidence", () => { + const state = createAutonomousRuntimeState({ enabled: true }); + + expect( + shouldAutonomouslyContinue(state, fauxAssistantMessage("I'm blocked. What should I try next?")), + ).toMatchObject({ + shouldContinue: true, + reason: "missing_terminal_evidence", + }); + expect( + shouldAutonomouslyContinue(state, fauxAssistantMessage("Blocked: this requires OAuth login from the user.")), + ).toMatchObject({ + shouldContinue: true, + reason: "missing_terminal_evidence", + }); + }); +}); diff --git a/packages/coding-agent/test/suite/agent-session-compaction.test.ts b/packages/coding-agent/test/suite/agent-session-compaction.test.ts index a21014894e..a0917b63f9 100644 --- a/packages/coding-agent/test/suite/agent-session-compaction.test.ts +++ b/packages/coding-agent/test/suite/agent-session-compaction.test.ts @@ -1,7 +1,7 @@ import type { AgentMessage, ShouldStopAfterTurnContext } from "@earendil-works/pi-agent-core"; import { type AssistantMessage, fauxAssistantMessage, type Model, type ToolResultMessage } from "@earendil-works/pi-ai"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { createHarness, type Harness } from "./harness.js"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createHarness, getMessageText, type Harness } from "./harness.js"; type SessionWithCompactionInternals = { _checkCompaction: (assistantMessage: AssistantMessage, skipAbortedCheck?: boolean) => Promise; @@ -43,9 +43,17 @@ function createAssistant( }; } +function failingGateCommand(): string { + return `${process.execPath} -e "console.error('gate failed'); process.exit(1)"`; +} + describe("AgentSession compaction characterization", () => { const harnesses: Harness[] = []; + beforeEach(() => { + vi.useRealTimers(); + }); + afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); @@ -342,6 +350,169 @@ describe("AgentSession compaction characterization", () => { expect(shouldStop).toBe(true); }); + it("queues a failing autonomous gate continuation before threshold compaction stops a tool loop", async () => { + vi.useFakeTimers(); + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 1 } }, + models: [{ id: "faux-1", contextWindow: 200_000 }], + extensionFactories: [ + (pi) => { + pi.on("session_before_compact", async (event) => ({ + compaction: { + summary: "auto compacted", + firstKeptEntryId: event.preparation.firstKeptEntryId, + tokensBefore: event.preparation.tokensBefore, + details: {}, + }, + })); + }, + ], + }); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as SessionWithCompactionInternals; + const successfulAssistant = createAssistant(harness, { + stopReason: "toolUse", + totalTokens: 10_000, + timestamp: Date.now(), + }); + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: "call-1", + toolName: "large-context", + content: [{ type: "text", text: "x".repeat(800_000) }], + isError: false, + timestamp: Date.now() + 500, + }; + const currentUser = { + role: "user", + content: [{ type: "text", text: "hello" }], + timestamp: Date.now() - 1000, + } satisfies Parameters[0]; + const oldUser = { + role: "user", + content: [{ type: "text", text: "old" }], + timestamp: Date.now() - 3000, + } satisfies Parameters[0]; + const oldAssistant = createAssistant(harness, { + stopReason: "stop", + totalTokens: 100, + timestamp: Date.now() - 2000, + }); + const oldMessages: AgentMessage[] = [oldUser, oldAssistant]; + const messages: AgentMessage[] = [currentUser, successfulAssistant, toolResult]; + for (const message of [oldUser, oldAssistant, currentUser, successfulAssistant]) { + harness.sessionManager.appendMessage(message); + } + harness.session.agent.state.messages = [...oldMessages, ...messages]; + + const continueSpy = vi.spyOn(harness.session.agent, "continue").mockResolvedValue(); + const followUpSpy = vi.spyOn(harness.session.agent, "followUp"); + + const shouldStop = await sessionInternals._shouldStopAfterTurn({ + message: successfulAssistant, + toolResults: [toolResult], + context: { systemPrompt: harness.session.systemPrompt, messages, tools: [] }, + newMessages: [successfulAssistant, toolResult], + }); + + expect(shouldStop).toBe(true); + expect(harness.session.getAutonomousStatus()).toMatchObject({ + continuationsUsed: 1, + gates: expect.objectContaining({ maxRetries: 5 }), + }); + expect(followUpSpy).toHaveBeenCalledTimes(1); + const queuedText = getMessageText(followUpSpy.mock.calls[0]?.[0]); + expect(queuedText).toContain("Autonomous quality gate failed (attempt 1/5)"); + expect(queuedText).toContain("gate failed"); + + await sessionInternals._runAutoCompaction("threshold", false); + await vi.advanceTimersByTimeAsync(100); + + expect(continueSpy).toHaveBeenCalledTimes(1); + }); + + it("queues a failing autonomous gate continuation before post-turn threshold compaction", async () => { + vi.useFakeTimers(); + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 1 } }, + models: [{ id: "faux-1", contextWindow: 200_000 }], + }); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as SessionWithCompactionInternals; + const successfulAssistant = createAssistant(harness, { + stopReason: "stop", + totalTokens: 10_000, + timestamp: Date.now(), + }); + const largeToolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: "call-1", + toolName: "large-context", + content: [{ type: "text", text: "x".repeat(800_000) }], + isError: false, + timestamp: Date.now() + 500, + }; + harness.session.agent.state.messages = [ + { role: "user", content: [{ type: "text", text: "hello" }], timestamp: Date.now() - 1000 }, + successfulAssistant, + largeToolResult, + ]; + + const runCompactionSpy = vi.spyOn(sessionInternals, "_runAutoCompaction").mockResolvedValue(); + const followUpSpy = vi.spyOn(harness.session.agent, "followUp"); + + await sessionInternals._checkCompaction(successfulAssistant, false); + + expect(runCompactionSpy).toHaveBeenCalledWith("threshold", false); + expect(followUpSpy).toHaveBeenCalledTimes(1); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + const queuedText = getMessageText(followUpSpy.mock.calls[0]?.[0]); + expect(queuedText).toContain("Autonomous quality gate failed (attempt 1/5)"); + expect(queuedText).toContain("gate failed"); + }); + + it("waits for threshold-compaction autonomous continuations before finishing prompt", async () => { + vi.useFakeTimers(); + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 1, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 1 } }, + models: [{ id: "faux-1", contextWindow: 200_000 }], + }); + harnesses.push(harness); + const highUsageDone = { + ...fauxAssistantMessage("done"), + usage: createUsage(10_000), + }; + harness.setResponses([highUsageDone, fauxAssistantMessage("retry")]); + const promptPromise = harness.session.prompt("make the change"); + + await vi.waitFor(() => expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1)); + await vi.advanceTimersByTimeAsync(100); + await promptPromise; + + expect(harness.session.getAutonomousStatus()).toMatchObject({ + continuationsUsed: 1, + turnsUsed: 2, + }); + }); + it("does not trigger threshold compaction for error messages when no prior usage exists", async () => { const harness = await createHarness(); harnesses.push(harness); diff --git a/packages/coding-agent/test/suite/harness.ts b/packages/coding-agent/test/suite/harness.ts index 16a8182ec4..edbaecc62a 100644 --- a/packages/coding-agent/test/suite/harness.ts +++ b/packages/coding-agent/test/suite/harness.ts @@ -11,6 +11,7 @@ import type { FauxModelDefinition, FauxProviderRegistration, FauxResponseStep, M import { registerFauxProvider } from "@earendil-works/pi-ai"; import { AgentSession, type AgentSessionEvent } from "../../src/core/agent-session.js"; import { AuthStorage } from "../../src/core/auth-storage.js"; +import type { AgentAutonomousConfig } from "../../src/core/autonomous.js"; import type { ExtensionRunner } from "../../src/core/extensions/index.js"; import { convertToLlm } from "../../src/core/messages.js"; import { ModelRegistry } from "../../src/core/model-registry.js"; @@ -63,6 +64,7 @@ export interface HarnessOptions { resourceLoader?: ResourceLoader; extensionFactories?: Array; withConfiguredAuth?: boolean; + autonomous?: AgentAutonomousConfig; } export interface Harness { @@ -174,6 +176,7 @@ export async function createHarness(options: HarnessOptions = {}): Promise