diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 8a1dd7f4936..10a2275a849 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Added autonomous mode with host-side continuations, configurable limits, and quality gates for evaluator-controlled runs ([#278](https://github.com/PrimeIntellect-ai/prime-agent/pull/278) by [@sethkarten](https://github.com/sethkarten)). - Fixed Prime Inference credential and team-header precedence to prefer `PRIME_API_KEY`, then the Prime CLI config, then `auth.json`. ## [0.2.9] - 2026-07-13 diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index 813147f8f6b..1723d38aa18 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -17,7 +17,7 @@ Edit directly or use `/settings` for common options. |---------|------|---------|-------------| | `defaultProvider` | string | - | Default provider (e.g., `"anthropic"`, `"openai"`) | | `defaultModel` | string | - | Default model ID | -| `defaultThinkingLevel` | string | - | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"` | +| `defaultThinkingLevel` | string | `"xhigh"` | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"` | | `hideThinkingBlock` | boolean | `false` | Hide thinking blocks in output | | `thinkingBudgets` | object | - | Custom token budgets per thinking level | @@ -258,7 +258,7 @@ See [packages.md](packages.md) for package management details. { "defaultProvider": "anthropic", "defaultModel": "claude-sonnet-4-20250514", - "defaultThinkingLevel": "medium", + "defaultThinkingLevel": "xhigh", "theme": "dark", "compaction": { "enabled": true, diff --git a/packages/coding-agent/skills/agent-observe/SKILL.md b/packages/coding-agent/skills/agent-observe/SKILL.md index 1bae5b30c4a..d213cf54fde 100644 --- a/packages/coding-agent/skills/agent-observe/SKILL.md +++ b/packages/coding-agent/skills/agent-observe/SKILL.md @@ -26,9 +26,9 @@ recent = await agent_observe.recent_messages("worker", limit=6) message preview. This includes subagents shown in the agents view; filter by `runtimeKind == "subagent"` and `parentSessionId` or `parentActiveSessionId` to inspect subagents that belong to your session. -- `await agent_observe.get_agent(target)` returns one agent summary. `target` - is resolved like other live-session selectors: active id, session id/name, or - unambiguous suffix. +- `await agent_observe.get_agent(target)` returns `agent`, where `agent` + contains one agent summary. `target` is resolved like other live-session + selectors: active id, session id/name, or unambiguous suffix. - `await agent_observe.recent_messages(target, limit=8, max_chars=800)` returns up to `limit` recent bounded message previews for the target session. `limit` must be 1-50, and `max_chars` must be 80-2000. diff --git a/packages/coding-agent/skills/orchestration-heartbeat/src/orchestration_heartbeat/__init__.py b/packages/coding-agent/skills/orchestration-heartbeat/src/orchestration_heartbeat/__init__.py index 5bf6929fed4..b58e4854130 100644 --- a/packages/coding-agent/skills/orchestration-heartbeat/src/orchestration_heartbeat/__init__.py +++ b/packages/coding-agent/skills/orchestration-heartbeat/src/orchestration_heartbeat/__init__.py @@ -24,14 +24,23 @@ def _first_value(agent: dict[str, Any], *keys: str) -> Any: return None +def _safe_field(value: Any) -> str | None: + if value is None: + return None + text = str(value) + sanitized = "".join(" " if ord(char) < 32 or ord(char) == 127 else char for char in text) + sanitized = " ".join(sanitized.split()) + return sanitized or None + + def _session_label(agent: dict[str, Any]) -> str: - name = _first_value(agent, "sessionName", "name") - session_id = _first_value(agent, "sessionId", "session_id") - active_session_id = _first_value(agent, "activeSessionId", "active_session_id") - cwd = agent.get("cwd") - status = agent.get("status") - streaming = _first_value(agent, "isStreaming", "streaming") - pending = _first_value(agent, "pendingMessageCount", "pending_message_count") + name = _safe_field(_first_value(agent, "sessionName", "name")) + session_id = _safe_field(_first_value(agent, "sessionId", "session_id")) + active_session_id = _safe_field(_first_value(agent, "activeSessionId", "active_session_id")) + cwd = _safe_field(agent.get("cwd")) + status = _safe_field(agent.get("status")) + streaming = _safe_field(_first_value(agent, "isStreaming", "streaming")) + pending = _safe_field(_first_value(agent, "pendingMessageCount", "pending_message_count")) parts = [ f"name={name}" if name else None, f"session_id={session_id}" if session_id else None, diff --git a/packages/coding-agent/src/cli/args.ts b/packages/coding-agent/src/cli/args.ts index d246b049f46..2f115d411f0 100644 --- a/packages/coding-agent/src/cli/args.ts +++ b/packages/coding-agent/src/cli/args.ts @@ -42,6 +42,14 @@ export interface Args { themes?: string[]; noThemes?: boolean; noContextFiles?: boolean; + autonomous?: boolean; + autonomousGates?: string[]; + autonomousGateRetries?: number; + autonomousGateTimeoutMs?: number; + autonomousMaxContinuations?: number; + autonomousMaxTurns?: number; + autonomousMaxTokens?: number; + autonomousTimeoutMs?: number; listModels?: string | true; offline?: boolean; verbose?: boolean; @@ -197,6 +205,44 @@ export function parseArgs(args: string[]): Args { result.noThemes = true; } else if (arg === "--no-context-files" || arg === "-nc") { result.noContextFiles = true; + } else if (arg === "--autonomous") { + result.autonomous = true; + } else if (arg === "--autonomous-gate") { + result.autonomous = true; + if (hasRequiredOptionValue(args, i, arg, result)) { + result.autonomousGates = result.autonomousGates ?? []; + result.autonomousGates.push(args[++i]); + } + } else if (arg === "--autonomous-gate-retries") { + result.autonomous = true; + if (hasRequiredOptionValue(args, i, arg, result)) { + result.autonomousGateRetries = parsePositiveInt(args[++i], "--autonomous-gate-retries", result); + } + } else if (arg === "--autonomous-gate-timeout-ms") { + result.autonomous = true; + if (hasRequiredOptionValue(args, i, arg, result)) { + result.autonomousGateTimeoutMs = parsePositiveInt(args[++i], "--autonomous-gate-timeout-ms", result); + } + } else if (arg === "--autonomous-max-continuations") { + result.autonomous = true; + if (hasRequiredOptionValue(args, i, arg, result)) { + result.autonomousMaxContinuations = parsePositiveInt(args[++i], "--autonomous-max-continuations", result); + } + } else if (arg === "--autonomous-max-turns") { + result.autonomous = true; + if (hasRequiredOptionValue(args, i, arg, result)) { + result.autonomousMaxTurns = parsePositiveInt(args[++i], "--autonomous-max-turns", result); + } + } else if (arg === "--autonomous-max-tokens") { + result.autonomous = true; + if (hasRequiredOptionValue(args, i, arg, result)) { + result.autonomousMaxTokens = parsePositiveInt(args[++i], "--autonomous-max-tokens", result); + } + } else if (arg === "--autonomous-timeout-ms") { + result.autonomous = true; + if (hasRequiredOptionValue(args, i, arg, result)) { + result.autonomousTimeoutMs = parsePositiveInt(args[++i], "--autonomous-timeout-ms", result); + } } else if (arg === "--list-models") { // Check if next arg is a search pattern (not a flag or file arg) if (i + 1 < args.length && !args[i + 1].startsWith("-") && !args[i + 1].startsWith("@")) { @@ -292,6 +338,14 @@ ${chalk.bold("Options:")} --theme Load a theme file or directory (can be used multiple times) --no-themes Disable theme discovery and loading --no-context-files, -nc Disable AGENTS.md and CLAUDE.md discovery and loading + --autonomous Continue autonomously until host-observable terminal evidence exists + --autonomous-gate Run a command before autonomous mode may finish (repeatable) + --autonomous-gate-retries Max autonomous retries per failed gate (default: 3) + --autonomous-gate-timeout-ms Timeout per autonomous gate command in milliseconds + --autonomous-max-continuations Max autonomous follow-up messages (default: 3) + --autonomous-max-turns Max assistant turns while autonomous mode is active (default: 12) + --autonomous-max-tokens Max tokens while autonomous mode is active (default: 80000) + --autonomous-timeout-ms Max autonomous wall-clock time in milliseconds (default: 1800000) --export Export session file to HTML and exit --list-models [search] List available models (with optional fuzzy search) --verbose Force verbose startup (overrides quietStartup setting) @@ -404,3 +458,21 @@ ${chalk.bold("Built-in Tool Names:")} ipython - Execute Python in a persistent IPython kernel `); } + +function hasRequiredOptionValue(args: string[], index: number, flag: string, result: Args): boolean { + const next = args[index + 1]; + if (next === undefined || next.startsWith("--")) { + result.diagnostics.push({ type: "error", message: `${flag} requires a value` }); + return false; + } + return true; +} + +function parsePositiveInt(value: string, flag: string, result: Args): number | undefined { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + result.diagnostics.push({ type: "error", message: `${flag} must be a positive integer` }); + return undefined; + } + return parsed; +} diff --git a/packages/coding-agent/src/core/agent-session-config.ts b/packages/coding-agent/src/core/agent-session-config.ts index e4d2d6c4176..16dd2393881 100644 --- a/packages/coding-agent/src/core/agent-session-config.ts +++ b/packages/coding-agent/src/core/agent-session-config.ts @@ -1,4 +1,5 @@ import type { ThinkingLevel } from "@earendil-works/pi-agent-core"; +import type { AgentAutonomousConfig } from "./autonomous.js"; export interface AgentSessionRuntimeConfig { cwd?: string; @@ -23,6 +24,7 @@ export interface AgentSessionRuntimeConfig { themes?: string[]; noThemes?: boolean; noContextFiles?: boolean; + autonomous?: AgentAutonomousConfig; extensionFlagValues?: Record; } @@ -56,6 +58,7 @@ export function mergeAgentSessionRuntimeConfig( themes: cloneArray(override.themes ?? base.themes), noThemes: override.noThemes ?? base.noThemes, noContextFiles: override.noContextFiles ?? base.noContextFiles, + autonomous: mergeAutonomousConfig(base.autonomous, override.autonomous), extensionFlagValues: base.extensionFlagValues || override.extensionFlagValues ? { ...(base.extensionFlagValues ?? {}), ...(override.extensionFlagValues ?? {}) } @@ -73,10 +76,42 @@ function cloneAgentSessionRuntimeConfig(config: AgentSessionRuntimeConfig): Agen skills: cloneArray(config.skills), promptTemplates: cloneArray(config.promptTemplates), themes: cloneArray(config.themes), + autonomous: mergeAutonomousConfig(undefined, config.autonomous), extensionFlagValues: config.extensionFlagValues ? { ...config.extensionFlagValues } : undefined, }; } +export function mergeAutonomousConfig( + base: AgentAutonomousConfig | undefined, + override: AgentAutonomousConfig | undefined, +): AgentAutonomousConfig | undefined { + if (!base && !override) { + return undefined; + } + const gates = mergeAutonomousGateConfig(base?.gates, override?.gates); + return { + ...(base ?? {}), + ...(override ?? {}), + ...(gates ? { gates } : {}), + }; +} + +function mergeAutonomousGateConfig( + base: AgentAutonomousConfig["gates"] | undefined, + override: AgentAutonomousConfig["gates"] | undefined, +): AgentAutonomousConfig["gates"] | undefined { + if (!base && !override) { + return undefined; + } + return { + ...(base ?? {}), + ...(override ?? {}), + ...(override?.commands !== undefined || base?.commands !== undefined + ? { commands: cloneArray(override?.commands ?? base?.commands) } + : {}), + }; +} + function cloneArray(value: T[] | undefined): T[] | undefined { return value ? [...value] : undefined; } diff --git a/packages/coding-agent/src/core/agent-session-services.ts b/packages/coding-agent/src/core/agent-session-services.ts index fc0d28b4439..94dd9c6191b 100644 --- a/packages/coding-agent/src/core/agent-session-services.ts +++ b/packages/coding-agent/src/core/agent-session-services.ts @@ -6,6 +6,7 @@ import type { AgentSessionMessageController } from "./agent-messages.js"; import type { AgentObserveController } from "./agent-observe.js"; import { installAgentTraceUpload } from "./agent-traces.js"; import { AuthStorage } from "./auth-storage.js"; +import type { AgentAutonomousConfig } from "./autonomous.js"; import type { AgentRlmHeartbeatController } from "./cron-jobs.js"; import { createHerdrAgentStateExtension } from "./extensions/builtin/herdr-agent-state.js"; import type { SessionStartEvent, ToolDefinition } from "./extensions/index.js"; @@ -73,6 +74,7 @@ export interface AgentSessionCreationOptions { subagentRuntimeHost?: SubagentRuntimeHost; rlmHeartbeatController?: AgentRlmHeartbeatController; prewarmIpythonKernel?: boolean; + autonomous?: AgentAutonomousConfig; } /** @@ -268,5 +270,6 @@ export async function createAgentSessionFromServices( rlmHeartbeatController: options.rlmHeartbeatController, sessionStartEvent: options.sessionStartEvent, prewarmIpythonKernel: options.prewarmIpythonKernel, + autonomous: options.autonomous, }); } diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 53864575d1c..308b40164f2 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -76,6 +76,18 @@ import { isLikelyAuthenticationError, } from "./auth-guidance.js"; import type { AuthSourceToken } from "./auth-storage.js"; +import { + type AgentAutonomousConfig, + type AgentAutonomousStatus, + type AutonomousRuntimeState, + addAutonomousContinuation, + addAutonomousUsage, + autonomousStatus, + createAutonomousRuntimeState, + nextAutonomousContinuation, + refreshAutonomousQualityGates, + setAutonomousEnabled, +} from "./autonomous.js"; import { type BashResult, executeBashWithOperations } from "./bash-executor.js"; import { COMPACT_SKILL_NAME, @@ -364,6 +376,8 @@ export interface AgentSessionConfig { rlmParentNodeId?: string; /** Host responsible for creating RLM subagent runtimes. */ subagentRuntimeHost?: SubagentRuntimeHost; + /** Host-side autonomous continuation policy. */ + autonomous?: AgentAutonomousConfig; /** * Boot the IPython kernel in the background as soon as the session is created, * so the first ipython tool call doesn't pay the kernel cold start. @@ -405,6 +419,10 @@ export interface PromptOptions { preflightResult?: (success: boolean, queued?: boolean) => void; /** Queue instead of starting immediately when the session is idle but already has queued work. */ queueIfBusy?: boolean; + /** Host-generated prompt that must bypass extension/slash/template input interception. */ + internalPrompt?: boolean; + /** Prevent host-driven prompts from causing autonomous continuation injection. */ + suppressAutonomousContinuation?: boolean; /** Skip extension input handlers for replaying already-accepted input. */ skipInputHandlers?: boolean; agentMessageId?: string; @@ -618,6 +636,13 @@ type GoalSlashCommand = | { kind: "resume" } | { kind: "start"; objective: string; tokenBudget?: number }; +type AutonomousSlashCommand = { kind: "status" } | { kind: "on" } | { kind: "off" }; + +type AutonomousRuntimeSnapshot = Pick< + AutonomousRuntimeState, + "continuationsUsed" | "gateAttempts" | "lastGateFailure" | "lastGateFailureSnapshot" +>; + interface RlmChildRun { id: string; prompt: string; @@ -748,6 +773,9 @@ export class AgentSession { private _goalAccountingStartedAt: number | undefined = undefined; private _goalAccountedAssistantMessages = new WeakSet(); private _goalAbortInProgress = false; + private _autonomousState: AutonomousRuntimeState; + private _autonomousContinuationSuppressionDepth = 0; + private _autonomousContinuationSuppressedMessages = new WeakSet(); // Compaction state private _compactionAbortController: AbortController | undefined = undefined; @@ -854,6 +882,10 @@ export class AgentSession { private _turnIntervalAutoRefinePending = false; private _postCompactionContinuationScheduled = false; private _postCompactionContinuationTimer: ReturnType | undefined; + private _postCompactionContinuationMessages: AgentMessage[] = []; + private _queuedAutonomousThresholdContinuations = new WeakMap(); + private _queuedAutonomousContinuationSnapshots = new WeakMap(); + private _pendingThresholdCompactionAutonomousMessages: AgentMessage[] = []; private _pendingAutoRefineReview: { reason: AutoRefineReason; review: AutoRefineReview } | undefined; private _autoRefineBranchVersion = 0; private _autoRefineReviewAbort?: AbortController; @@ -890,6 +922,7 @@ export class AgentSession { this._rlmSessionDir = config.rlmSessionDir; this._rlmParentNodeId = config.rlmParentNodeId; this._subagentRuntimeHost = config.subagentRuntimeHost; + this._autonomousState = createAutonomousRuntimeState(config.autonomous, { cwd: this._cwd }); this._goalState = this._loadPersistedGoalState(); this._restoreLateIpythonSentAgentMessages(); if (this._goalState.status === "active") { @@ -1001,7 +1034,7 @@ export class AgentSession { } private _installAgentContinuationHook(): void { - this.agent.getContinuationMessages = (context, signal) => this._getGoalContinuationMessages(context, signal); + this.agent.getContinuationMessages = (context, signal) => this._getContinuationMessages(context, signal); } private _installAgentTurnHook(): void { @@ -1311,6 +1344,64 @@ export class AgentSession { return { kind: "start", objective: validateGoalObjective(objective), tokenBudget }; } + private _parseAutonomousSlashCommand(text: string): AutonomousSlashCommand | undefined { + if (text !== "/autonomous" && !text.startsWith("/autonomous ")) { + return undefined; + } + const rest = text.slice("/autonomous".length).trim().toLowerCase(); + if (!rest || rest === "status") { + return { kind: "status" }; + } + if (rest === "on" || rest === "enable" || rest === "enabled") { + return { kind: "on" }; + } + if (rest === "off" || rest === "disable" || rest === "disabled") { + return { kind: "off" }; + } + throw new Error("Usage: /autonomous [on|off|status]"); + } + + private _formatAutonomousStatus(): string { + const status = this.getAutonomousStatus(); + const state = status.enabled ? "on" : "off"; + return `Autonomous mode: ${state}. Continuations: ${status.continuationsUsed}/${status.limits.maxContinuations}. Turns: ${status.turnsUsed}/${status.limits.maxTurns}. Tokens: ${status.tokensUsed}/${status.limits.maxTokens}.`; + } + + private _emitAutonomousStatus(): void { + const message = { + role: "custom" as const, + customType: "autonomous_status", + content: this._formatAutonomousStatus(), + display: true, + details: this.getAutonomousStatus(), + timestamp: Date.now(), + } satisfies CustomMessage; + this.agent.state.messages.push(message); + this.sessionManager.appendCustomMessageEntry( + message.customType, + message.content, + message.display, + message.details, + ); + this._emit({ type: "message_start", message }); + this._emit({ type: "message_end", message }); + } + + private async _handleAutonomousSlashCommand(text: string): Promise { + const command = this._parseAutonomousSlashCommand(text); + if (!command) { + return false; + } + if (command.kind === "on") { + setAutonomousEnabled(this._autonomousState, true, { cwd: this._cwd }); + } else if (command.kind === "off") { + setAutonomousEnabled(this._autonomousState, false); + this._clearQueuedAutonomousContinuations(); + } + this._emitAutonomousStatus(); + return true; + } + private async _validateCanStartAgentRun(): Promise { if (!this.model) { throw new Error(formatNoModelSelectedMessage()); @@ -1454,7 +1545,7 @@ export class AgentSession { return true; } - private _shouldStopAfterTurn(context: ShouldStopAfterTurnContext): boolean { + private async _shouldStopAfterTurn(context: ShouldStopAfterTurnContext): Promise { if (this._stopGoalContinuationForTerminalMessage(context.message)) { return true; } @@ -1465,15 +1556,15 @@ export class AgentSession { } catch { // Goal accounting must not interrupt the core agent loop. } - if (this._shouldStopForThresholdCompaction(context)) { + if (await this._shouldStopForThresholdCompaction(context)) { return true; } return false; } - private _shouldStopForThresholdCompaction(context: ShouldStopAfterTurnContext): boolean { + private async _shouldStopForThresholdCompaction(context: ShouldStopAfterTurnContext): Promise { this._continueAfterThresholdCompaction = false; - if (this._pendingRequestedCompaction === undefined && !this._thresholdCompactionNeeded(context)) { + if (this._pendingRequestedCompaction === undefined && !(await this._thresholdCompactionNeeded(context))) { return false; } @@ -1482,7 +1573,7 @@ export class AgentSession { return true; } - private _thresholdCompactionNeeded(context: ShouldStopAfterTurnContext): boolean { + private async _thresholdCompactionNeeded(context: ShouldStopAfterTurnContext): Promise { const settings = this.settingsManager.getCompactionSettings(); if (!settings.enabled) return false; @@ -1494,7 +1585,104 @@ export class AgentSession { } const contextTokens = this._getThresholdContextTokens(context.message, compactionTimestamp); - return contextTokens !== undefined && shouldCompact(contextTokens, contextWindow, settings); + if (contextTokens === undefined || !shouldCompact(contextTokens, contextWindow, settings)) { + return false; + } + + if (await this._queueAutonomousContinuationForThresholdCompaction(context.message)) { + this._continueAfterThresholdCompaction = true; + } + return true; + } + + private _snapshotAutonomousRuntimeState(): AutonomousRuntimeSnapshot { + return { + continuationsUsed: this._autonomousState.continuationsUsed, + gateAttempts: { ...this._autonomousState.gateAttempts }, + lastGateFailure: this._autonomousState.lastGateFailure + ? { ...this._autonomousState.lastGateFailure } + : undefined, + lastGateFailureSnapshot: this._autonomousState.lastGateFailureSnapshot + ? { ...this._autonomousState.lastGateFailureSnapshot } + : undefined, + }; + } + + private _restoreAutonomousRuntimeSnapshot(snapshot: AutonomousRuntimeSnapshot): void { + this._autonomousState.continuationsUsed = snapshot.continuationsUsed; + this._autonomousState.gateAttempts = { ...snapshot.gateAttempts }; + this._autonomousState.lastGateFailure = snapshot.lastGateFailure ? { ...snapshot.lastGateFailure } : undefined; + this._autonomousState.lastGateFailureSnapshot = snapshot.lastGateFailureSnapshot + ? { ...snapshot.lastGateFailureSnapshot } + : undefined; + } + + private async _queueAutonomousContinuationForThresholdCompaction( + message: AssistantMessage, + ): Promise { + const queuedMessage = this._queuedAutonomousThresholdContinuations.get(message); + if (queuedMessage && this._postCompactionContinuationMessages.includes(queuedMessage)) { + return queuedMessage; + } + const snapshot = this._snapshotAutonomousRuntimeState(); + const autonomousMessage = await nextAutonomousContinuation(this._autonomousState, message, { cwd: this._cwd }); + if (!autonomousMessage) { + return undefined; + } + this._queuedAutonomousThresholdContinuations.set(message, autonomousMessage); + this._queuedAutonomousContinuationSnapshots.set(autonomousMessage, snapshot); + this._postCompactionContinuationMessages.push(autonomousMessage); + this._pendingThresholdCompactionAutonomousMessages.push(autonomousMessage); + this.agent.followUp(autonomousMessage); + return autonomousMessage; + } + + private _clearQueuedAutonomousContinuations( + options: { restoreAutonomousState?: boolean; messages?: AgentMessage[] } = {}, + ): void { + const requestedMessages = options.messages ?? [...this._postCompactionContinuationMessages]; + const requestedMessageSet = new Set(requestedMessages); + const queuedMessages = this._postCompactionContinuationMessages.filter((message) => + requestedMessageSet.has(message), + ); + if (queuedMessages.length === 0) { + return; + } + const queuedMessageSet = new Set(queuedMessages); + this._postCompactionContinuationMessages = this._postCompactionContinuationMessages.filter( + (message) => !queuedMessageSet.has(message), + ); + this.agent.removeQueuedMessages((message) => queuedMessageSet.has(message)); + if (options.restoreAutonomousState) { + for (const queuedMessage of queuedMessages) { + const snapshot = this._queuedAutonomousContinuationSnapshots.get(queuedMessage); + if (snapshot) { + this._restoreAutonomousRuntimeSnapshot(snapshot); + break; + } + } + } + for (const queuedMessage of queuedMessages) { + this._queuedAutonomousContinuationSnapshots.delete(queuedMessage); + } + this._pendingThresholdCompactionAutonomousMessages = this._pendingThresholdCompactionAutonomousMessages.filter( + (message) => !queuedMessageSet.has(message), + ); + if (options.messages === undefined) { + this._continueAfterThresholdCompaction = false; + } + if (!this.agent.hasQueuedMessages()) { + this._cancelPostCompactionContinue(); + } + } + + private _clearQueuedAutonomousContinuationsAfterSkippedThresholdCompaction( + shouldContinueAfterThreshold: boolean, + queuedMessages: AgentMessage[], + ): void { + if (shouldContinueAfterThreshold) { + this._clearQueuedAutonomousContinuations({ restoreAutonomousState: true, messages: queuedMessages }); + } } /** @@ -1799,6 +1987,26 @@ export class AgentSession { } } + private async _getContinuationMessages( + context: GetContinuationMessagesContext, + signal?: AbortSignal, + ): Promise { + const goalMessages = await this._getGoalContinuationMessages(context, signal); + if (goalMessages.length > 0 || signal?.aborted) { + return goalMessages; + } + if ( + this._autonomousContinuationSuppressionDepth > 0 || + context.newMessages.some((message) => this._autonomousContinuationSuppressedMessages.has(message)) + ) { + return []; + } + const autonomousMessage = await nextAutonomousContinuation(this._autonomousState, context.message, { + cwd: this._cwd, + }); + return autonomousMessage ? [autonomousMessage] : []; + } + // Track last assistant message for auto-compaction check private _lastAssistantMessage: AssistantMessage | undefined = undefined; @@ -2040,9 +2248,14 @@ export class AgentSession { // Track assistant message for auto-compaction (checked on agent_end) if (event.message.role === "assistant") { this._lastAssistantMessage = event.message; - this._assistantTurnsSinceAutoRefine++; const assistantMsg = event.message as AssistantMessage; + if (assistantMsg.stopReason !== "error") { + addAutonomousUsage(this._autonomousState, assistantMsg.usage); + } + if (assistantMsg.stopReason !== "error" && assistantMsg.stopReason !== "aborted") { + this._assistantTurnsSinceAutoRefine++; + } if (assistantMsg.stopReason !== "error") { this._overflowRecoveryAttempted = false; } @@ -2487,6 +2700,31 @@ export class AgentSession { return { ...this._goalWithCurrentWallClock() }; } + getAutonomousStatus(): AgentAutonomousStatus { + return autonomousStatus(this._autonomousState); + } + + recordHostAutonomousContinuation(): void { + addAutonomousContinuation(this._autonomousState); + } + + async refreshAutonomousGates(): Promise { + await refreshAutonomousQualityGates(this._autonomousState, { cwd: this._cwd }); + } + + private async _runWithAutonomousContinuationSuppressed(fn: () => Promise): Promise { + this._autonomousContinuationSuppressionDepth++; + try { + return await fn(); + } finally { + this._autonomousContinuationSuppressionDepth--; + } + } + + private _markAutonomousContinuationSuppressed(message: AgentMessage): void { + this._autonomousContinuationSuppressedMessages.add(message); + } + /** Scoped models for cycling (from --models flag) */ get scopedModels(): ReadonlyArray<{ model: Model; thinkingLevel?: ThinkingLevel }> { return this._scopedModels; @@ -2656,7 +2894,11 @@ export class AgentSession { text, message, options.streamingBehavior, - { queueKey: options.followUpQueueKey, previewLabel }, + { + queueKey: options.followUpQueueKey, + previewLabel, + suppressAutonomousContinuation: options.suppressAutonomousContinuation, + }, ); if (!queued) { reportPreflight(false); @@ -2681,7 +2923,7 @@ export class AgentSession { const lastAssistant = this._findLastAssistantMessage(); if (lastAssistant) { - await this._checkCompaction(lastAssistant, false); + await this._checkCompaction(lastAssistant, false, false); } const pendingModelSelectEmit = this._pendingModelSelectEmit(); @@ -2746,7 +2988,11 @@ export class AgentSession { text, message, options.streamingBehavior, - { queueKey: options.followUpQueueKey, previewLabel }, + { + queueKey: options.followUpQueueKey, + previewLabel, + suppressAutonomousContinuation: options.suppressAutonomousContinuation, + }, ); if (!queued) { reportPreflight(false); @@ -2757,7 +3003,12 @@ export class AgentSession { } try { - await this.agent.prompt(messages); + if (options?.suppressAutonomousContinuation) { + this._markAutonomousContinuationSuppressed(message); + await this._runWithAutonomousContinuationSuppressed(() => this.agent.prompt(messages)); + } else { + await this.agent.prompt(messages); + } } catch (error) { this._pendingNextTurnMessages.unshift(...drainedNextTurnMessages.map((pending) => ({ ...pending }))); throw error; @@ -2767,7 +3018,8 @@ export class AgentSession { } private async _prompt(text: string, options?: InternalPromptOptions): Promise { - const expandPromptTemplates = options?.expandPromptTemplates ?? true; + const isInternalPrompt = options?.internalPrompt === true; + const expandPromptTemplates = isInternalPrompt ? false : (options?.expandPromptTemplates ?? true); const preflightResult = options?.preflightResult; let preflightSettled = false; const reportPreflight = (success: boolean, queued = false) => { @@ -2784,8 +3036,36 @@ export class AgentSession { try { let currentText = text; + const hasQueueIfBusyBackpressure = () => + options?.queueIfBusy === true && + (this.pendingMessageCount > 0 || + this.isCompacting || + this.isRetrying || + this.isBashRunning || + this.hasAcceptedPromptInFlight); - if (expandPromptTemplates) { + const shouldHandleBuiltInSlashCommands = !isInternalPrompt && !options?.skipPrePromptWork; + const isBuiltInSlashCommand = + shouldHandleBuiltInSlashCommands && + (currentText === "/autonomous" || + currentText.startsWith("/autonomous ") || + currentText === "/goal" || + currentText.startsWith("/goal ")); + if (!this.isStreaming && isBuiltInSlashCommand && hasQueueIfBusyBackpressure()) { + reportPreflight(false); + throw new Error("Agent has queued work. Retry the slash command after pending work finishes."); + } + if ( + shouldHandleBuiltInSlashCommands && + (currentText === "/autonomous" || currentText.startsWith("/autonomous ")) + ) { + const handledAutonomousCommand = await this._handleAutonomousSlashCommand(currentText); + if (handledAutonomousCommand) { + reportPreflight(true); + return; + } + } + if (shouldHandleBuiltInSlashCommands && (currentText === "/goal" || currentText.startsWith("/goal "))) { const handledGoalCommand = await this._handleGoalSlashCommand(currentText, currentImages); if (handledGoalCommand) { reportPreflight(true); @@ -2805,9 +3085,9 @@ export class AgentSession { } // Emit input event for extension interception (before skill/template expansion). - // Agent-to-agent messages use acceptAgentMessagePrompt(), which bypasses - // input handlers so extensions cannot rewrite or swallow direct delivery. - if (!options?.skipInputHandlers && this._extensionRunner.hasHandlers("input")) { + // Agent-to-agent and internal host prompts bypass input handlers so + // extensions cannot rewrite or swallow direct delivery. + if (!isInternalPrompt && !options?.skipInputHandlers && this._extensionRunner.hasHandlers("input")) { const inputResult = await this._extensionRunner.emitInput( currentText, currentImages, @@ -2833,13 +3113,7 @@ export class AgentSession { // If streaming, or a caller explicitly asked to respect existing queued work, // enqueue according to the requested behavior. const shouldQueueForStreaming = this.isStreaming; - const shouldQueueForPendingWork = - options?.queueIfBusy === true && - (this.pendingMessageCount > 0 || - this.isCompacting || - this.isRetrying || - this.isBashRunning || - this.hasAcceptedPromptInFlight); + const shouldQueueForPendingWork = hasQueueIfBusyBackpressure(); if (shouldQueueForStreaming || shouldQueueForPendingWork) { if (!options?.streamingBehavior) { const stateDescription = shouldQueueForStreaming @@ -2856,6 +3130,7 @@ export class AgentSession { { queueKey: options.followUpQueueKey, agentMessageId: options.agentMessageId, + suppressAutonomousContinuation: options.suppressAutonomousContinuation, customMessage: options.customMessage, }, ); @@ -2938,7 +3213,7 @@ export class AgentSession { // Check if we need to compact before sending (catches aborted responses) const lastAssistant = this._findLastAssistantMessage(); if (lastAssistant) { - await this._checkCompaction(lastAssistant, false); + await this._checkCompaction(lastAssistant, false, false); } // Build messages array (custom message if any, then user message) @@ -3048,6 +3323,7 @@ export class AgentSession { { queueKey: options.followUpQueueKey, agentMessageId: options.agentMessageId, + suppressAutonomousContinuation: options.suppressAutonomousContinuation, customMessage: options.customMessage, }, ); @@ -3058,7 +3334,14 @@ export class AgentSession { reportPreflight(true, true); return; } - const promptPromise = this.agent.prompt(messages); + if (options?.suppressAutonomousContinuation) { + for (const message of messages) { + this._markAutonomousContinuationSuppressed(message); + } + } + const promptPromise = options?.suppressAutonomousContinuation + ? this._runWithAutonomousContinuationSuppressed(() => this.agent.prompt(messages)) + : this.agent.prompt(messages); const promptAccepted = Symbol("promptAccepted"); const acceptance = acceptedAgentMessagePrompt ? acceptedAgentMessagePrompt.accepted.then( @@ -3294,7 +3577,12 @@ export class AgentSession { text: string, images: ImageContent[] | undefined, streamingBehavior: "steer" | "followUp", - options: { queueKey?: string; agentMessageId?: string; customMessage?: CustomMessage } = {}, + options: { + queueKey?: string; + agentMessageId?: string; + customMessage?: CustomMessage; + suppressAutonomousContinuation?: boolean; + } = {}, ): Promise { const pendingNextTurnMessages = this._pendingNextTurnMessages; this._pendingNextTurnMessages = []; @@ -3316,6 +3604,7 @@ export class AgentSession { queueKey: options.queueKey, message: options.customMessage, prefixMessages: pendingNextTurnMessages, + suppressAutonomousContinuation: options.suppressAutonomousContinuation, }); return true; } catch (error) { @@ -3328,7 +3617,7 @@ export class AgentSession { text: string, message: CustomMessage, streamingBehavior: "steer" | "followUp", - options: { queueKey?: string; previewLabel?: string } = {}, + options: { queueKey?: string; previewLabel?: string; suppressAutonomousContinuation?: boolean } = {}, ): Promise { const pendingNextTurnMessages = this._pendingNextTurnMessages; this._pendingNextTurnMessages = []; @@ -3339,6 +3628,7 @@ export class AgentSession { message, prefixMessages: pendingNextTurnMessages, previewLabel: options.previewLabel, + suppressAutonomousContinuation: options.suppressAutonomousContinuation, }); if (!queued) { this._pendingNextTurnMessages.unshift(...pendingNextTurnMessages); @@ -3350,6 +3640,7 @@ export class AgentSession { prefixMessages: pendingNextTurnMessages, previewLabel: options.previewLabel, queueKey: options.queueKey, + suppressAutonomousContinuation: options.suppressAutonomousContinuation, }); return true; } catch (error) { @@ -3371,6 +3662,7 @@ export class AgentSession { message?: QueuedAgentMessage; prefixMessages?: CustomMessage[]; previewLabel?: string; + suppressAutonomousContinuation?: boolean; } = {}, ): Promise { const content = options.content ?? this._buildPromptContent(text, images); @@ -3381,6 +3673,9 @@ export class AgentSession { content, timestamp: Date.now(), } satisfies UserMessage); + if (options.suppressAutonomousContinuation) { + this._markAutonomousContinuationSuppressed(message); + } this._steeringMessages.push({ text, previewLabel: options.previewLabel, @@ -3406,6 +3701,7 @@ export class AgentSession { message?: QueuedAgentMessage; prefixMessages?: CustomMessage[]; previewLabel?: string; + suppressAutonomousContinuation?: boolean; } = {}, ): Promise { if (options.queueKey && this._followUpMessages.some((message) => message.queueKey === options.queueKey)) { @@ -3419,6 +3715,9 @@ export class AgentSession { content, timestamp: Date.now(), } satisfies UserMessage); + if (options.suppressAutonomousContinuation) { + this._markAutonomousContinuationSuppressed(message); + } this._followUpMessages.push({ text, previewLabel: options.previewLabel, @@ -4362,8 +4661,10 @@ export class AgentSession { } this._postCompactionContinuationScheduled = false; + const continuationMessages = [...this._postCompactionContinuationMessages]; try { await this.agent.continue(); + this._forgetConsumedPostCompactionContinuations(continuationMessages); } catch (error) { const message = error instanceof Error ? error.message : String(error); if (message.includes("already processing")) { @@ -4372,6 +4673,25 @@ export class AgentSession { } } + private _forgetConsumedPostCompactionContinuations(continuationMessages: AgentMessage[]): void { + if (continuationMessages.length === 0) { + return; + } + const continuationMessageSet = new Set(continuationMessages); + const stillQueued = new Set(this.agent.removeQueuedMessages((message) => continuationMessageSet.has(message))); + for (const message of stillQueued) { + this.agent.followUp(message); + } + for (const message of continuationMessages) { + if (!stillQueued.has(message)) { + this._queuedAutonomousContinuationSnapshots.delete(message); + } + } + this._postCompactionContinuationMessages = this._postCompactionContinuationMessages.filter( + (message) => !continuationMessageSet.has(message) || stillQueued.has(message), + ); + } + private _shouldSkipAutoRefineForActiveAgent(): boolean { return this.isStreaming || this.isCompacting; } @@ -4756,7 +5076,11 @@ export class AgentSession { return calculateContextTokens(assistantMessage.usage); } - private async _checkCompaction(assistantMessage: AssistantMessage, skipAbortedCheck = true): Promise { + private async _checkCompaction( + assistantMessage: AssistantMessage, + skipAbortedCheck = true, + queueAutonomousContinuation = true, + ): Promise { // An abort drops any compaction the model requested this turn, even on the // pre-prompt path (skipAbortedCheck=false) which continues to threshold checks. if (assistantMessage.stopReason === "aborted") { @@ -4826,6 +5150,12 @@ export class AgentSession { const contextTokens = this._getThresholdContextTokens(assistantMessage, compactionTimestamp); if (contextTokens === undefined) return false; if (shouldCompact(contextTokens, contextWindow, settings)) { + if ( + queueAutonomousContinuation && + (await this._queueAutonomousContinuationForThresholdCompaction(assistantMessage)) + ) { + this._continueAfterThresholdCompaction = true; + } return await this._runAutoCompaction("threshold", false); } return false; @@ -4846,6 +5176,10 @@ export class AgentSession { const customInstructions = pending?.customInstructions; const shouldContinueAfterCompaction = (reason === "threshold" || reason === "requested") && this._continueAfterThresholdCompaction; + const queuedAutonomousContinuationsForThisCompaction = + reason === "threshold" && shouldContinueAfterCompaction + ? this._pendingThresholdCompactionAutonomousMessages.splice(0) + : []; this._continueAfterThresholdCompaction = false; // A requested compaction stopped the loop on purpose; don't stall if it fails. @@ -4867,6 +5201,10 @@ export class AgentSession { aborted: false, willRetry: false, }); + this._clearQueuedAutonomousContinuationsAfterSkippedThresholdCompaction( + reason === "threshold" && shouldContinueAfterCompaction, + queuedAutonomousContinuationsForThisCompaction, + ); resumeAfterFailure(); return false; } @@ -4880,6 +5218,10 @@ export class AgentSession { aborted: false, willRetry: false, }); + this._clearQueuedAutonomousContinuationsAfterSkippedThresholdCompaction( + reason === "threshold" && shouldContinueAfterCompaction, + queuedAutonomousContinuationsForThisCompaction, + ); resumeAfterFailure(); return false; } @@ -4916,6 +5258,10 @@ export class AgentSession { } return false; } catch (error) { + this._clearQueuedAutonomousContinuationsAfterSkippedThresholdCompaction( + reason === "threshold" && shouldContinueAfterCompaction, + queuedAutonomousContinuationsForThisCompaction, + ); const errorMessage = error instanceof Error ? error.message : "compaction failed"; const aborted = errorMessage === "Compaction cancelled" || (error instanceof Error && error.name === "AbortError"); diff --git a/packages/coding-agent/src/core/autonomous.ts b/packages/coding-agent/src/core/autonomous.ts new file mode 100644 index 00000000000..c9be6a4ac24 --- /dev/null +++ b/packages/coding-agent/src/core/autonomous.ts @@ -0,0 +1,534 @@ +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { lstat, readlink } from "node:fs/promises"; +import { resolve } from "node:path"; +import type { AssistantMessage, Usage, UserMessage } from "@earendil-works/pi-ai"; +import { waitForChildProcess } from "../utils/child-process.js"; +import { killProcessTree, trackDetachedChildPid, untrackDetachedChildPid } from "../utils/shell.js"; + +export interface AgentAutonomousConfig { + enabled?: boolean; + maxContinuations?: number; + maxTurns?: number; + maxTokens?: number; + timeoutMs?: number; + continuationPrompt?: string; + gates?: AgentAutonomousGateConfig; +} + +export interface AgentAutonomousGateConfig { + commands?: string[]; + maxRetries?: number; + timeoutMs?: number; +} + +export interface AgentAutonomousGateFailure { + command: string; + attempt: number; + exitText: string; + output: string; +} + +export interface AgentAutonomousStatus { + enabled: boolean; + continuationsUsed: number; + turnsUsed: number; + tokensUsed: number; + startedAt?: number; + limits: Required>; + gates: Required; + gateAttempts: Record; + lastGateFailure?: AgentAutonomousGateFailure; +} + +export const DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT = + "No human input is available in autonomous mode. Continue working until the host evaluator, verifier, or configured autonomous limits stop the run. If you were asking the user a question, make a reasonable assumption and verify it. If you believe you are blocked, prove it with host-observable evidence, preserve that evidence, and keep looking for safe progress while budget remains. Do not end the session yourself; the verifier/evaluator decides completion when configured gates pass."; + +export const DEFAULT_AUTONOMOUS_LIMITS: Required< + Omit +> = { + maxContinuations: 3, + maxTurns: 12, + maxTokens: 80_000, + timeoutMs: 30 * 60 * 1000, +}; + +export const DEFAULT_AUTONOMOUS_GATES: Required = { + commands: [], + maxRetries: 3, + timeoutMs: 5 * 60 * 1000, +}; + +const MAX_GATE_OUTPUT_CHARS = 6000; +const MAX_CHILD_PROCESS_OUTPUT_CHARS = 1024 * 1024; + +export interface AutonomousRuntimeState { + enabled: boolean; + continuationsUsed: number; + turnsUsed: number; + tokensUsed: number; + startedAt?: number; + limits: Required>; + continuationPrompt: string; + gates: Required; + gateAttempts: Record; + lastGateFailure?: GateFailure; + lastGateFailureSnapshot?: GitWorktreeSnapshot; +} + +export type AutonomousLimitReason = "maxContinuations" | "maxTurns" | "maxTokens" | "timeoutMs"; +export type AutonomousGateResult = "passed" | "failed" | "retry_exhausted"; + +type AutonomousLimitState = Pick< + AgentAutonomousStatus, + "continuationsUsed" | "turnsUsed" | "tokensUsed" | "startedAt" | "limits" +>; + +export interface AutonomousDecision { + shouldContinue: boolean; + reason: "missing_terminal_evidence" | "gate_failed" | "not_needed" | "limit_reached"; +} + +interface GitWorktreeSnapshot { + status: string; + diff: string; + untrackedHash: string; +} + +type GateFailure = AgentAutonomousGateFailure; + +export function createAutonomousRuntimeState( + config?: AgentAutonomousConfig, + _options: { cwd?: string } = {}, +): AutonomousRuntimeState { + const enabled = config?.enabled === true; + return { + enabled, + continuationsUsed: 0, + turnsUsed: 0, + tokensUsed: 0, + startedAt: enabled ? Date.now() : undefined, + limits: { + maxContinuations: normalizeLimit(config?.maxContinuations, DEFAULT_AUTONOMOUS_LIMITS.maxContinuations), + maxTurns: normalizeLimit(config?.maxTurns, DEFAULT_AUTONOMOUS_LIMITS.maxTurns), + maxTokens: normalizeLimit(config?.maxTokens, DEFAULT_AUTONOMOUS_LIMITS.maxTokens), + timeoutMs: normalizeLimit(config?.timeoutMs, DEFAULT_AUTONOMOUS_LIMITS.timeoutMs), + }, + continuationPrompt: config?.continuationPrompt?.trim() || DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT, + gates: { + commands: [...(config?.gates?.commands ?? DEFAULT_AUTONOMOUS_GATES.commands)], + maxRetries: normalizeLimit(config?.gates?.maxRetries, DEFAULT_AUTONOMOUS_GATES.maxRetries), + timeoutMs: normalizeLimit(config?.gates?.timeoutMs, DEFAULT_AUTONOMOUS_GATES.timeoutMs), + }, + gateAttempts: {}, + lastGateFailure: undefined, + lastGateFailureSnapshot: undefined, + }; +} + +export function setAutonomousEnabled( + state: AutonomousRuntimeState, + enabled: boolean, + _options: { cwd?: string } = {}, +): void { + state.enabled = enabled; + if (enabled) { + state.continuationsUsed = 0; + state.turnsUsed = 0; + state.tokensUsed = 0; + state.startedAt = Date.now(); + state.gateAttempts = {}; + state.lastGateFailure = undefined; + state.lastGateFailureSnapshot = undefined; + } else { + state.startedAt = undefined; + state.gateAttempts = {}; + state.lastGateFailure = undefined; + state.lastGateFailureSnapshot = undefined; + } +} + +export function autonomousStatus(state: AutonomousRuntimeState): AgentAutonomousStatus { + return { + enabled: state.enabled, + continuationsUsed: state.continuationsUsed, + turnsUsed: state.turnsUsed, + tokensUsed: state.tokensUsed, + startedAt: state.startedAt, + limits: { ...state.limits }, + gates: { ...state.gates, commands: [...state.gates.commands] }, + gateAttempts: { ...state.gateAttempts }, + lastGateFailure: state.lastGateFailure ? { ...state.lastGateFailure } : undefined, + }; +} + +export function addAutonomousUsage(state: AutonomousRuntimeState, usage: Usage | undefined): void { + if (!state.enabled) { + return; + } + state.turnsUsed++; + state.tokensUsed += autonomousTokenDelta(usage); +} + +export function addAutonomousContinuation(state: AutonomousRuntimeState): void { + if (!state.enabled) { + return; + } + state.continuationsUsed++; +} + +function autonomousTokenDelta(usage: Usage | undefined): number { + if (!usage) { + return 0; + } + // Cache-read tokens are repeated context served from provider cache. Counting them + // cumulatively makes long autonomous verifier loops exhaust their host-side token + // budget far before the non-cached work reaches the configured cap. + return usage.input + usage.output + usage.cacheWrite; +} + +export async function nextAutonomousContinuation( + state: AutonomousRuntimeState, + message: AssistantMessage, + options: { cwd?: string } = {}, + now = Date.now(), +): Promise { + if (!state.enabled) { + return undefined; + } + const decision = await shouldAutonomouslyContinue(state, message, options, now); + if (!decision.shouldContinue) { + return undefined; + } + state.continuationsUsed++; + return { + role: "user", + content: [ + { + type: "text", + text: + decision.reason === "gate_failed" + ? (buildGateFailureContinuation(state, now) ?? state.continuationPrompt) + : state.continuationPrompt, + }, + ], + timestamp: now, + }; +} + +export async function shouldAutonomouslyContinue( + state: AutonomousRuntimeState, + message: AssistantMessage, + options: { cwd?: string } = {}, + now = Date.now(), +): Promise { + if (!state.enabled || message.stopReason === "error" || message.stopReason === "aborted") { + return { shouldContinue: false, reason: "not_needed" }; + } + const gateResult = await refreshAutonomousQualityGates(state, options); + if (gateResult) { + if (gateResult === "passed") { + return { shouldContinue: false, reason: "not_needed" }; + } + if (gateResult === "retry_exhausted" || autonomousLimitReason(state, now)) { + return { shouldContinue: false, reason: "limit_reached" }; + } + return { shouldContinue: true, reason: "gate_failed" }; + } + if (autonomousLimitReason(state, now)) { + return { shouldContinue: false, reason: "limit_reached" }; + } + return { shouldContinue: true, reason: "missing_terminal_evidence" }; +} + +export function autonomousLimitReason( + state: AutonomousLimitState, + now = Date.now(), +): AutonomousLimitReason | undefined { + if (state.continuationsUsed >= state.limits.maxContinuations) { + return "maxContinuations"; + } + if (state.turnsUsed >= state.limits.maxTurns) { + return "maxTurns"; + } + if (state.tokensUsed >= state.limits.maxTokens) { + return "maxTokens"; + } + if (state.startedAt !== undefined && now - state.startedAt >= state.limits.timeoutMs) { + return "timeoutMs"; + } + return undefined; +} + +export async function refreshAutonomousQualityGates( + state: AutonomousRuntimeState, + options: { cwd?: string } = {}, +): Promise { + if (!state.enabled || state.gates.commands.length === 0) { + return undefined; + } + return await runAutonomousQualityGates(state, options.cwd); +} + +async function runAutonomousQualityGates( + state: AutonomousRuntimeState, + cwd: string | undefined, +): Promise { + if (!cwd) { + return "failed"; + } + for (const command of state.gates.commands) { + const currentSnapshot = await captureGitWorktreeSnapshot(cwd); + if ( + state.lastGateFailure?.command === command && + state.lastGateFailureSnapshot && + gitWorktreeSnapshotsEqual(currentSnapshot, state.lastGateFailureSnapshot) + ) { + const attempt = (state.gateAttempts[command] ?? state.lastGateFailure.attempt) + 1; + state.gateAttempts[command] = attempt; + state.lastGateFailure = { + ...state.lastGateFailure, + attempt, + exitText: "not rerun: workspace unchanged since previous failed gate", + output: + "The autonomous gate was not rerun because the workspace has not changed since this failure. Edit source files, tests, or a blocker artifact before attempting to finish again.", + }; + return attempt > state.gates.maxRetries ? "retry_exhausted" : "failed"; + } + const result = await runChildProcess(command, [], { + cwd, + shell: true, + timeoutMs: state.gates.timeoutMs, + maxOutputChars: MAX_GATE_OUTPUT_CHARS, + }); + const postRunSnapshot = await captureGitWorktreeSnapshot(cwd); + if (result.status === 0 && !result.error && !result.timedOut) { + state.gateAttempts[command] = 0; + if (state.lastGateFailure?.command === command) { + state.lastGateFailure = undefined; + state.lastGateFailureSnapshot = undefined; + } + continue; + } + const attempt = (state.gateAttempts[command] ?? 0) + 1; + state.gateAttempts[command] = attempt; + const exitText = formatProcessExit(result); + state.lastGateFailure = { + command, + attempt, + exitText, + output: truncateGateOutput( + [result.stdout, result.stderr].filter(Boolean).join("\n").trim(), + result.outputTruncated, + ), + }; + state.lastGateFailureSnapshot = postRunSnapshot; + return attempt > state.gates.maxRetries ? "retry_exhausted" : "failed"; + } + state.lastGateFailure = undefined; + state.lastGateFailureSnapshot = undefined; + return "passed"; +} + +function buildGateFailureContinuation(state: AutonomousRuntimeState, timestamp: number): string | undefined { + const failure = state.lastGateFailure; + if (!failure) { + return undefined; + } + return ( + `Autonomous quality gate failed (attempt ${failure.attempt}/${state.gates.maxRetries}): \`${failure.command}\` ${failure.exitText}.\n` + + (failure.output ? `\nOutput:\n${failure.output}\n` : "\n") + + `\nContinue working. Fix the failure, then produce terminal evidence. Timestamp: ${new Date(timestamp).toISOString()}.` + ); +} + +function gitWorktreeSnapshotsEqual(a: GitWorktreeSnapshot | undefined, b: GitWorktreeSnapshot | undefined): boolean { + return !!a && !!b && a.status === b.status && a.diff === b.diff && a.untrackedHash === b.untrackedHash; +} + +async function captureGitWorktreeSnapshot(cwd: string | undefined): Promise { + if (!cwd) { + return undefined; + } + const pathspec = [ + "--", + ".", + ":(exclude)verification", + ":(exclude)target", + ":(exclude).vf-prime-agent", + ":(exclude)Cargo.lock", + ":(exclude)submission.tar.gz", + ":(exclude)runner_args.log", + ]; + const status = await runChildProcess( + "git", + ["--no-optional-locks", "status", "--porcelain=v1", "-z", "-uall", "--no-renames", ...pathspec], + { + cwd, + timeoutMs: 10_000, + }, + ); + if (status.status !== 0 || status.error || status.timedOut || status.outputTruncated) { + return undefined; + } + const diff = await runChildProcess( + "git", + ["--no-optional-locks", "diff", "--no-ext-diff", "--binary", "HEAD", ...pathspec], + { + cwd, + timeoutMs: 10_000, + }, + ); + if (diff.status !== 0 || diff.error || diff.timedOut || diff.outputTruncated) { + return undefined; + } + return { + status: status.stdout, + diff: diff.stdout, + untrackedHash: await hashUntrackedFiles(cwd, status.stdout), + }; +} + +function untrackedPathsFromStatus(status: string): string[] { + return status + .split("\0") + .filter((entry) => entry.startsWith("?? ")) + .map((entry) => entry.slice(3)) + .sort(); +} + +async function hashUntrackedFiles(cwd: string, status: string): Promise { + const aggregate = createHash("sha256"); + for (const path of untrackedPathsFromStatus(status)) { + aggregate.update(path); + aggregate.update("\0"); + aggregate.update(await hashUntrackedPath(resolve(cwd, path))); + aggregate.update("\0"); + } + return aggregate.digest("hex"); +} + +async function hashUntrackedPath(path: string): Promise { + try { + const stat = await lstat(path); + if (stat.isSymbolicLink()) { + return `symlink:${await readlink(path)}`; + } + if (!stat.isFile()) { + return `other:${stat.mode}:${stat.size}:${stat.mtimeMs}`; + } + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) { + hash.update(chunk); + } + return `file:${hash.digest("hex")}`; + } catch (error) { + return `error:${error instanceof Error ? error.message : String(error)}`; + } +} + +interface ChildProcessResult { + status: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; + error?: Error; + timedOut?: boolean; + outputTruncated: boolean; +} + +function runChildProcess( + command: string, + args: string[], + options: { cwd?: string; shell?: boolean; timeoutMs?: number; maxOutputChars?: number } = {}, +): Promise { + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd: options.cwd, + detached: process.platform !== "win32", + shell: options.shell === true, + stdio: ["ignore", "pipe", "pipe"], + }); + if (child.pid) { + trackDetachedChildPid(child.pid); + } + let stdout = ""; + let stderr = ""; + let error: Error | undefined; + let timedOut = false; + let outputTruncated = false; + let settled = false; + const maxOutputChars = options.maxOutputChars ?? MAX_CHILD_PROCESS_OUTPUT_CHARS; + const finish = (result: Pick) => { + if (settled) { + return; + } + settled = true; + if (timer) { + clearTimeout(timer); + } + if (child.pid) { + untrackDetachedChildPid(child.pid); + } + resolve({ ...result, stdout, stderr, error, timedOut, outputTruncated }); + }; + const timer = options.timeoutMs + ? setTimeout(() => { + timedOut = true; + if (child.pid) { + killProcessTree(child.pid); + } else { + child.kill("SIGKILL"); + } + }, options.timeoutMs) + : undefined; + child.stdout?.setEncoding("utf8"); + child.stderr?.setEncoding("utf8"); + child.stdout?.on("data", (chunk: string) => { + const remaining = maxOutputChars - stdout.length; + if (remaining > 0) { + stdout += chunk.slice(0, remaining); + } + outputTruncated ||= chunk.length > remaining; + }); + child.stderr?.on("data", (chunk: string) => { + const remaining = maxOutputChars - stderr.length; + if (remaining > 0) { + stderr += chunk.slice(0, remaining); + } + outputTruncated ||= chunk.length > remaining; + }); + void waitForChildProcess(child).then( + (status) => finish({ status, signal: child.signalCode }), + (err: Error) => { + error = err; + finish({ status: child.exitCode, signal: child.signalCode }); + }, + ); + }); +} + +function formatProcessExit(result: ChildProcessResult): string { + if (result.timedOut) { + return "timed out"; + } + if (result.error) { + return result.error.message; + } + return result.signal ? `terminated by ${result.signal}` : `exited ${result.status ?? "unknown"}`; +} + +function truncateGateOutput(output: string, outputAlreadyTruncated = false, maxChars = MAX_GATE_OUTPUT_CHARS): string { + if (output.length <= maxChars && !outputAlreadyTruncated) { + return output; + } + return `${output.slice(0, maxChars)}\n... [truncated]`; +} + +function normalizeLimit(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value) || value === undefined || value <= 0) { + return fallback; + } + return Math.trunc(value); +} diff --git a/packages/coding-agent/src/core/sdk.ts b/packages/coding-agent/src/core/sdk.ts index 432774c3718..91e5187605d 100644 --- a/packages/coding-agent/src/core/sdk.ts +++ b/packages/coding-agent/src/core/sdk.ts @@ -6,6 +6,7 @@ import { AgentSession } from "./agent-session.js"; import type { AgentSessionCreationOptions } from "./agent-session-services.js"; import { formatNoModelsAvailableMessage } from "./auth-guidance.js"; import { AuthStorage } from "./auth-storage.js"; +import type { AgentAutonomousConfig } from "./autonomous.js"; import { DEFAULT_THINKING_LEVEL } from "./defaults.js"; import type { ExtensionRunner, LoadExtensionsResult, SessionStartEvent, ToolDefinition } from "./extensions/index.js"; import { McpManager } from "./mcp/mcp-manager.js"; @@ -69,6 +70,8 @@ export interface CreateAgentSessionOptions extends AgentSessionCreationOptions { settingsManager?: SettingsManager; /** Session start event metadata for extension runtime startup. */ sessionStartEvent?: SessionStartEvent; + /** Host-side autonomous continuation policy. */ + autonomous?: AgentAutonomousConfig; } /** Result from createAgentSession */ @@ -379,6 +382,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} subagentRuntimeHost: options.subagentRuntimeHost, sessionStartEvent: options.sessionStartEvent, prewarmIpythonKernel: options.prewarmIpythonKernel, + autonomous: options.autonomous, }); const extensionsResult = resourceLoader.getExtensions(); diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index cfbb715f068..b4bcec74f1d 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -27,7 +27,11 @@ import { buildInitialMessage } from "./cli/initial-message.js"; import { listModels } from "./cli/list-models.js"; import { selectSession } from "./cli/session-picker.js"; import { expandTildePath, getAgentDir, getSessionDirEnvOverride, VERSION } from "./config.js"; -import { type AgentSessionRuntimeConfig, mergeAgentSessionRuntimeConfig } from "./core/agent-session-config.js"; +import { + type AgentSessionRuntimeConfig, + mergeAgentSessionRuntimeConfig, + mergeAutonomousConfig, +} from "./core/agent-session-config.js"; import { type CreateAgentSessionRuntimeFactory, createAgentSessionRuntime } from "./core/agent-session-runtime.js"; import { type AgentSessionRuntimeDiagnostic, @@ -640,6 +644,9 @@ function buildSessionOptions( if (config.tools) { options.tools = [...config.tools]; } + if (config.autonomous) { + options.autonomous = mergeAutonomousConfig(undefined, config.autonomous); + } return { options, cliThinkingFromModel, diagnostics }; } @@ -648,6 +655,39 @@ function resolveCliPaths(cwd: string, paths: string[] | undefined): string[] | u return paths?.map((value) => (isLocalPath(value) ? resolve(cwd, value) : value)); } +function runtimeAutonomousConfigFromArgs(parsed: Args): AgentSessionRuntimeConfig["autonomous"] { + const hasAutonomousOptions = + parsed.autonomous === true || + parsed.autonomousGates !== undefined || + parsed.autonomousGateRetries !== undefined || + parsed.autonomousGateTimeoutMs !== undefined || + parsed.autonomousMaxContinuations !== undefined || + parsed.autonomousMaxTurns !== undefined || + parsed.autonomousMaxTokens !== undefined || + parsed.autonomousTimeoutMs !== undefined; + if (!hasAutonomousOptions) { + return undefined; + } + const hasGateOptions = + parsed.autonomousGates !== undefined || + parsed.autonomousGateRetries !== undefined || + parsed.autonomousGateTimeoutMs !== undefined; + return { + enabled: true, + maxContinuations: parsed.autonomousMaxContinuations, + maxTurns: parsed.autonomousMaxTurns, + maxTokens: parsed.autonomousMaxTokens, + timeoutMs: parsed.autonomousTimeoutMs, + gates: hasGateOptions + ? { + commands: parsed.autonomousGates, + maxRetries: parsed.autonomousGateRetries, + timeoutMs: parsed.autonomousGateTimeoutMs, + } + : undefined, + }; +} + function runtimeConfigFromArgs( parsed: Args, cwd: string, @@ -677,6 +717,7 @@ function runtimeConfigFromArgs( themes: resolveCliPaths(cwd, parsed.themes), noThemes: parsed.noThemes, noContextFiles: parsed.noContextFiles, + autonomous: runtimeAutonomousConfigFromArgs(parsed), extensionFlagValues: parsed.unknownFlags.size > 0 ? Object.fromEntries(parsed.unknownFlags.entries()) : undefined, }; } @@ -707,6 +748,10 @@ export function resolveRuntimeSessionOptions( rlmHeartbeatController: runtimeSessionOptions?.rlmHeartbeatController, agentMessageController: runtimeSessionOptions?.agentMessageController, agentObserveController: runtimeSessionOptions?.agentObserveController, + autonomous: + (runtimeSessionOptions?.rlmDepth ?? 0) > 0 + ? mergeAutonomousConfig(sessionOptions.autonomous, { ...runtimeSessionOptions?.autonomous, enabled: false }) + : mergeAutonomousConfig(sessionOptions.autonomous, runtimeSessionOptions?.autonomous), rlmDepth: runtimeSessionOptions?.rlmDepth, rlmMaxDepth: runtimeSessionOptions?.rlmMaxDepth, rlmSessionDir: runtimeSessionOptions?.rlmSessionDir, diff --git a/packages/coding-agent/src/modes/print-mode.ts b/packages/coding-agent/src/modes/print-mode.ts index 2f9fb048e24..49b70591354 100644 --- a/packages/coding-agent/src/modes/print-mode.ts +++ b/packages/coding-agent/src/modes/print-mode.ts @@ -8,6 +8,12 @@ import type { AssistantMessage, ImageContent } from "@earendil-works/pi-ai"; import type { AgentSessionRuntime } from "../core/agent-session-runtime.js"; +import { + type AgentAutonomousGateFailure, + type AgentAutonomousStatus, + type AutonomousLimitReason, + autonomousLimitReason, +} from "../core/autonomous.js"; import { flushRawStdout, writeRawStdout } from "../core/output-guard.js"; import { killTrackedDetachedChildren } from "../utils/shell.js"; @@ -25,6 +31,112 @@ export interface PrintModeOptions { initialImages?: ImageContent[]; } +function latestGateAttempt(status: AgentAutonomousStatus): number { + return Math.max(status.lastGateFailure?.attempt ?? 0, 0, ...Object.values(status.gateAttempts)); +} + +function describeAutonomousLimit(status: AgentAutonomousStatus, reason: AutonomousLimitReason): string { + if (reason === "maxContinuations") { + return `maxContinuations reached (${status.continuationsUsed}/${status.limits.maxContinuations})`; + } + if (reason === "maxTurns") { + return `maxTurns reached (${status.turnsUsed}/${status.limits.maxTurns})`; + } + if (reason === "maxTokens") { + return `maxTokens reached (${status.tokensUsed}/${status.limits.maxTokens})`; + } + const elapsed = status.startedAt === undefined ? 0 : Math.max(0, Date.now() - status.startedAt); + return `timeoutMs reached (${elapsed}/${status.limits.timeoutMs})`; +} + +function shouldContinuePrintModeAutonomousGates(status: AgentAutonomousStatus): boolean { + return ( + status.enabled && + status.gates.commands.length > 0 && + !!status.lastGateFailure && + latestGateAttempt(status) <= status.gates.maxRetries && + !autonomousLimitReason(status) + ); +} + +function buildPrintModeGateContinuation( + failure: AgentAutonomousGateFailure, + attempt: number, + maxRetries: number, +): string { + return ( + `Autonomous quality gate failed (attempt ${attempt}/${maxRetries}): \`${failure.command}\` ${failure.exitText}.\n` + + (failure.output ? `\nOutput:\n${failure.output}\n` : "\n") + + `\nContinue working. Fix the failure, then produce terminal evidence. Timestamp: ${new Date().toISOString()}.` + ); +} + +function printModeAutonomousProgressKey(status: AgentAutonomousStatus): string { + return [ + latestGateAttempt(status), + status.continuationsUsed, + status.turnsUsed, + status.tokensUsed, + status.lastGateFailure?.exitText ?? "", + ].join(":"); +} + +async function waitForPrintModeIdleWithAutonomousGates( + getSession: () => AgentSessionRuntime["session"], +): Promise { + let lastPromptedProgressKey: string | undefined; + let repeatedProgressPrompts = 0; + while (true) { + const session = getSession(); + await session.agent.waitForIdle(); + const status = session.getAutonomousStatus(); + if (!shouldContinuePrintModeAutonomousGates(status) || !status.lastGateFailure) { + return; + } + const progressKey = printModeAutonomousProgressKey(status); + if (progressKey === lastPromptedProgressKey) { + repeatedProgressPrompts++; + } else { + repeatedProgressPrompts = 0; + lastPromptedProgressKey = progressKey; + } + // Print mode is the host loop for autonomous verifier runs. A still-failing + // gate is not terminal while the configured autonomous budgets remain. Keep + // feeding the failure back even if the observable status key repeats; the + // normal autonomous limits above bound retries, turns, tokens, and wall time. + // The small yield prevents a tight loop if a test double or broken session + // returns immediately without advancing state. + if (repeatedProgressPrompts > 0) { + await new Promise((resolve) => setTimeout(resolve, Math.min(1000, repeatedProgressPrompts * 50))); + } + session.recordHostAutonomousContinuation(); + await session.prompt( + buildPrintModeGateContinuation(status.lastGateFailure, latestGateAttempt(status), status.gates.maxRetries), + { + streamingBehavior: "followUp", + internalPrompt: true, + suppressAutonomousContinuation: true, + }, + ); + // followUp prompts can be accepted/queued before the actual retry turn has + // produced a new assistant message. Wait for the session to become idle again + // so an earlier assistant error does not get mistaken for this retry result. + await session.agent.waitForIdle(); + await session.refreshAutonomousGates(); + const lastMessage = session.state.messages[session.state.messages.length - 1]; + if (lastMessage?.role === "assistant") { + const assistantMessage = lastMessage as AssistantMessage; + if (assistantMessage.stopReason === "error" || assistantMessage.stopReason === "aborted") { + const postErrorStatus = session.getAutonomousStatus(); + if (shouldContinuePrintModeAutonomousGates(postErrorStatus) && postErrorStatus.lastGateFailure) { + continue; + } + return; + } + } + } +} + /** * Run in print (single-shot) mode. * Sends prompts to the agent and outputs the result. @@ -125,6 +237,8 @@ export async function runPrintMode(runtimeHost: AgentSessionRuntime, options: Pr await session.prompt(message); } + await waitForPrintModeIdleWithAutonomousGates(() => session); + if (mode === "text") { const state = session.state; const lastMessage = state.messages[state.messages.length - 1]; @@ -144,6 +258,23 @@ export async function runPrintMode(runtimeHost: AgentSessionRuntime, options: Pr } } + const autonomousStatus = session.getAutonomousStatus(); + const autonomousLimit = autonomousLimitReason(autonomousStatus); + if (autonomousStatus.enabled && autonomousStatus.gates.commands.length > 0 && autonomousStatus.lastGateFailure) { + const limitText = autonomousLimit + ? `; autonomous limit reached: ${describeAutonomousLimit(autonomousStatus, autonomousLimit)}` + : ""; + console.error( + `Autonomous quality gate still failing after attempt ${latestGateAttempt(autonomousStatus)}/${autonomousStatus.gates.maxRetries}: ${autonomousStatus.lastGateFailure.exitText}${limitText}`, + ); + exitCode = 1; + } else if (autonomousStatus.enabled && autonomousStatus.gates.commands.length === 0 && autonomousLimit) { + console.error( + `Autonomous run stopped before terminal evidence; ${describeAutonomousLimit(autonomousStatus, autonomousLimit)}`, + ); + exitCode = 1; + } + return exitCode; } catch (error: unknown) { console.error(error instanceof Error ? error.message : String(error)); diff --git a/packages/coding-agent/src/utils/child-process.ts b/packages/coding-agent/src/utils/child-process.ts index 9f75a285d80..7e91bc3718d 100644 --- a/packages/coding-agent/src/utils/child-process.ts +++ b/packages/coding-agent/src/utils/child-process.ts @@ -1,4 +1,5 @@ import type { ChildProcess } from "node:child_process"; +import { constants } from "node:os"; import { basename } from "node:path"; const EXIT_STDIO_GRACE_MS = 100; @@ -19,14 +20,25 @@ export function shouldUseWindowsShell(command: string): boolean { * though the original process is already gone. We wait briefly for stdio to end, * then forcibly stop tracking the inherited handles. */ +function signalExitCode(signal: NodeJS.Signals | null): number | null { + if (!signal) return null; + const signalNumber = constants.signals[signal]; + return signalNumber === undefined ? 1 : 128 + signalNumber; +} + +function normalizedExitCode(code: number | null, signal: NodeJS.Signals | null): number | null { + return code ?? signalExitCode(signal); +} + export function waitForChildProcess(child: ChildProcess): Promise { return new Promise((resolve, reject) => { let settled = false; let exited = false; let exitCode: number | null = null; + let exitSignal: NodeJS.Signals | null = null; let postExitTimer: NodeJS.Timeout | undefined; - let stdoutEnded = child.stdout === null; - let stderrEnded = child.stderr === null; + let stdoutEnded = child.stdout === null || child.stdout.readableEnded; + let stderrEnded = child.stderr === null || child.stderr.readableEnded; const cleanup = () => { if (postExitTimer) { @@ -52,7 +64,7 @@ export function waitForChildProcess(child: ChildProcess): Promise const maybeFinalizeAfterExit = () => { if (!exited || settled) return; if (stdoutEnded && stderrEnded) { - finalize(exitCode); + finalize(normalizedExitCode(exitCode, exitSignal)); } }; @@ -73,17 +85,18 @@ export function waitForChildProcess(child: ChildProcess): Promise reject(err); }; - const onExit = (code: number | null) => { + const onExit = (code: number | null, signal: NodeJS.Signals | null = null) => { exited = true; exitCode = code; + exitSignal = signal; maybeFinalizeAfterExit(); if (!settled) { - postExitTimer = setTimeout(() => finalize(code), EXIT_STDIO_GRACE_MS); + postExitTimer = setTimeout(() => finalize(normalizedExitCode(code, signal)), EXIT_STDIO_GRACE_MS); } }; - const onClose = (code: number | null) => { - finalize(code); + const onClose = (code: number | null, signal: NodeJS.Signals | null = null) => { + finalize(normalizedExitCode(code, signal)); }; child.stdout?.once("end", onStdoutEnd); @@ -91,5 +104,9 @@ export function waitForChildProcess(child: ChildProcess): Promise child.once("error", onError); child.once("exit", onExit); child.once("close", onClose); + + if (child.exitCode !== null || child.signalCode !== null) { + onExit(child.exitCode, child.signalCode); + } }); } diff --git a/packages/coding-agent/test/agent-session-concurrent.test.ts b/packages/coding-agent/test/agent-session-concurrent.test.ts index 075ba400729..34eafd087e5 100644 --- a/packages/coding-agent/test/agent-session-concurrent.test.ts +++ b/packages/coding-agent/test/agent-session-concurrent.test.ts @@ -360,6 +360,68 @@ describe("AgentSession concurrent prompt guard", () => { expect(receivedUserText).toBe("agent-to-agent payload"); }); + it("delivers internal prompts without extension input interception", async () => { + const model = getModel("anthropic", "claude-sonnet-4-5")!; + let inputCalls = 0; + let receivedUserText: string | undefined; + const agent = new Agent({ + getApiKey: () => "test-key", + initialState: { + model, + systemPrompt: "Test", + tools: [], + }, + streamFn: (_model, context) => { + const stream = new MockAssistantStream(); + queueMicrotask(() => { + const userMessages = context.messages.filter((message) => message.role === "user"); + const user = userMessages.at(-1); + if (user && typeof user.content !== "string") { + receivedUserText = user.content + .filter( + (part): part is TextContent => + typeof part === "object" && part !== null && part.type === "text", + ) + .map((part) => part.text) + .join("\n"); + } + stream.push({ type: "start", partial: createAssistantMessage("") }); + stream.push({ type: "done", reason: "stop", message: createAssistantMessage("Delivered") }); + }); + return stream; + }, + }); + + const sessionManager = SessionManager.inMemory(); + const settingsManager = SettingsManager.create(tempDir, tempDir); + const authStorage = AuthStorage.create(join(tempDir, "auth.json")); + const modelRegistry = ModelRegistry.create(authStorage, tempDir); + authStorage.setRuntimeApiKey("anthropic", "test-key"); + const extensionsResult = await createTestExtensionsResult([ + (pi) => { + pi.on("input", async () => { + inputCalls++; + return { action: "handled" }; + }); + }, + ]); + + session = new AgentSession({ + agent, + sessionManager, + settingsManager, + cwd: tempDir, + modelRegistry, + resourceLoader: createTestResourceLoader({ extensionsResult }), + }); + + await session.prompt("host gate follow-up", { internalPrompt: true }); + await session.agent.waitForIdle(); + + expect(inputCalls).toBe(0); + expect(receivedUserText).toBe("host gate follow-up"); + }); + it("should allow prompt() after previous completes", async () => { // Create session with a stream that completes immediately const model = getModel("anthropic", "claude-sonnet-4-5")!; diff --git a/packages/coding-agent/test/agent-session-config.test.ts b/packages/coding-agent/test/agent-session-config.test.ts index e610fb2c1a1..f9b330a4679 100644 --- a/packages/coding-agent/test/agent-session-config.test.ts +++ b/packages/coding-agent/test/agent-session-config.test.ts @@ -65,6 +65,33 @@ describe("mergeAgentSessionRuntimeConfig", () => { expect(merged.extensions).not.toBe(defaults.extensions); }); + it("deep-merges autonomous gate overrides", () => { + const defaults: AgentSessionRuntimeConfig = { + autonomous: { + enabled: true, + maxTurns: 20, + gates: { commands: ["npm test"], maxRetries: 3 }, + }, + }; + + const overrides: AgentSessionRuntimeConfig = { + autonomous: { + maxContinuations: 5, + gates: { timeoutMs: 1000 }, + }, + }; + + const merged = mergeAgentSessionRuntimeConfig(defaults, overrides); + + expect(merged.autonomous).toEqual({ + enabled: true, + maxTurns: 20, + maxContinuations: 5, + gates: { commands: ["npm test"], maxRetries: 3, timeoutMs: 1000 }, + }); + expect(merged.autonomous?.gates?.commands).not.toBe(defaults.autonomous?.gates?.commands); + }); + it("ignores undefined override values", () => { const defaults: AgentSessionRuntimeConfig = { cwd: "/repo/default", diff --git a/packages/coding-agent/test/args.test.ts b/packages/coding-agent/test/args.test.ts index 1b62ad5efd5..0c13740e2b4 100644 --- a/packages/coding-agent/test/args.test.ts +++ b/packages/coding-agent/test/args.test.ts @@ -330,6 +330,98 @@ describe("parseArgs", () => { }); }); + describe("--autonomous flag", () => { + test("parses --autonomous flag", () => { + const result = parseArgs(["--autonomous"]); + expect(result.autonomous).toBe(true); + }); + + test("parses autonomous gate flags", () => { + const result = parseArgs([ + "--autonomous", + "--autonomous-gate", + "npm test", + "--autonomous-gate", + "npm run lint", + "--autonomous-gate-retries", + "2", + "--autonomous-gate-timeout-ms", + "1000", + ]); + expect(result.autonomous).toBe(true); + expect(result.autonomousGates).toEqual(["npm test", "npm run lint"]); + expect(result.autonomousGateRetries).toBe(2); + expect(result.autonomousGateTimeoutMs).toBe(1000); + }); + + test("parses autonomous limit flags", () => { + const result = parseArgs([ + "--autonomous", + "--autonomous-max-continuations", + "20", + "--autonomous-max-turns", + "80", + "--autonomous-max-tokens", + "500000", + "--autonomous-timeout-ms", + "1800000", + ]); + expect(result.autonomous).toBe(true); + expect(result.autonomousMaxContinuations).toBe(20); + expect(result.autonomousMaxTurns).toBe(80); + expect(result.autonomousMaxTokens).toBe(500000); + expect(result.autonomousTimeoutMs).toBe(1800000); + }); + + test("auto-enables autonomous mode when autonomous sub-options are supplied", () => { + const result = parseArgs(["--autonomous-max-turns", "1", "--autonomous-gate", "npm test"]); + expect(result.autonomous).toBe(true); + expect(result.autonomousMaxTurns).toBe(1); + expect(result.autonomousGates).toEqual(["npm test"]); + }); + + test("reports missing autonomous option values", () => { + const result = parseArgs(["--autonomous-max-turns", "--autonomous-gate", "npm test"]); + + expect(result.autonomous).toBe(true); + expect(result.autonomousMaxTurns).toBeUndefined(); + expect(result.autonomousGates).toEqual(["npm test"]); + expect(result.unknownFlags.size).toBe(0); + expect(result.diagnostics).toContainEqual({ + type: "error", + message: "--autonomous-max-turns requires a value", + }); + }); + + test("does not consume another autonomous flag as a gate value", () => { + const result = parseArgs(["--autonomous-gate", "--autonomous-max-turns", "3"]); + + expect(result.autonomous).toBe(true); + expect(result.autonomousGates).toBeUndefined(); + expect(result.autonomousMaxTurns).toBe(3); + expect(result.diagnostics).toContainEqual({ + type: "error", + message: "--autonomous-gate requires a value", + }); + }); + + test.each([ + "--autonomous-gate", + "--autonomous-gate-retries", + "--autonomous-gate-timeout-ms", + "--autonomous-max-continuations", + "--autonomous-max-turns", + "--autonomous-max-tokens", + "--autonomous-timeout-ms", + ])("reports when %s has no value", (flag) => { + const result = parseArgs([flag]); + + expect(result.autonomous).toBe(true); + expect(result.unknownFlags.size).toBe(0); + expect(result.diagnostics).toContainEqual({ type: "error", message: `${flag} requires a value` }); + }); + }); + describe("tool flags", () => { test("parses --no-tools flag", () => { const result = parseArgs(["--no-tools"]); diff --git a/packages/coding-agent/test/child-process.test.ts b/packages/coding-agent/test/child-process.test.ts new file mode 100644 index 00000000000..8000c2f9cef --- /dev/null +++ b/packages/coding-agent/test/child-process.test.ts @@ -0,0 +1,17 @@ +import type { ChildProcess } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { describe, expect, it } from "vitest"; +import { waitForChildProcess } from "../src/utils/child-process.js"; + +describe("waitForChildProcess", () => { + it("reports signaled already-exited children as failures", async () => { + const child = Object.assign(new EventEmitter(), { + stdout: null, + stderr: null, + exitCode: null, + signalCode: "SIGTERM" as NodeJS.Signals, + }); + + await expect(waitForChildProcess(child as unknown as ChildProcess)).resolves.toBe(143); + }); +}); diff --git a/packages/coding-agent/test/main-interactive-routing.test.ts b/packages/coding-agent/test/main-interactive-routing.test.ts index 21d92be9a6c..ae83d27c11d 100644 --- a/packages/coding-agent/test/main-interactive-routing.test.ts +++ b/packages/coding-agent/test/main-interactive-routing.test.ts @@ -295,6 +295,57 @@ describe("runtime session option resolution", () => { rlmSessionDir: "/tmp/rlm-session", }); }); + + test("deep-merges autonomous runtime session overrides", () => { + const resolved = resolveRuntimeSessionOptions( + { + autonomous: { + enabled: true, + maxTurns: 20, + gates: { commands: ["npm test"], maxRetries: 3 }, + }, + }, + { + autonomous: { + maxContinuations: 5, + gates: { timeoutMs: 1000 }, + }, + }, + ); + + expect(resolved.autonomous).toEqual({ + enabled: true, + maxTurns: 20, + maxContinuations: 5, + gates: { commands: ["npm test"], maxRetries: 3, timeoutMs: 1000 }, + }); + }); + + test("disables autonomous mode for subagent runtime sessions", () => { + const resolved = resolveRuntimeSessionOptions( + { + autonomous: { + enabled: true, + maxTurns: 20, + gates: { commands: ["npm test"], maxRetries: 3 }, + }, + }, + { + rlmDepth: 1, + autonomous: { + maxContinuations: 5, + gates: { timeoutMs: 1000 }, + }, + }, + ); + + expect(resolved.autonomous).toEqual({ + enabled: false, + maxTurns: 20, + maxContinuations: 5, + gates: { commands: ["npm test"], maxRetries: 3, timeoutMs: 1000 }, + }); + }); }); function makeSessionSummary(overrides: Partial): SessionSummary { diff --git a/packages/coding-agent/test/model-resolver.test.ts b/packages/coding-agent/test/model-resolver.test.ts index faf73af3230..45583992a86 100644 --- a/packages/coding-agent/test/model-resolver.test.ts +++ b/packages/coding-agent/test/model-resolver.test.ts @@ -436,6 +436,22 @@ describe("default model selection", () => { expect(result.model?.id).toBe("openai/ghost-model"); }); + test("findInitialModel uses medium as the built-in default thinking level", async () => { + const reasoningModel = mockModels[0]; + const registry = { + getAvailable: async () => [reasoningModel], + } as unknown as Parameters[0]["modelRegistry"]; + + const result = await findInitialModel({ + scopedModels: [], + isContinuing: false, + modelRegistry: registry, + }); + + expect(result.model).toBe(reasoningModel); + expect(result.thinkingLevel).toBe("medium"); + }); + test("findInitialModel selects ai-gateway default when available", async () => { const aiGatewayModel: Model<"anthropic-messages"> = { id: "anthropic/claude-opus-4-6", diff --git a/packages/coding-agent/test/print-mode.test.ts b/packages/coding-agent/test/print-mode.test.ts index bbf6756e046..2149b1a2517 100644 --- a/packages/coding-agent/test/print-mode.test.ts +++ b/packages/coding-agent/test/print-mode.test.ts @@ -1,5 +1,6 @@ import type { AssistantMessage, ImageContent } from "@earendil-works/pi-ai"; import { afterEach, describe, expect, it, vi } from "vitest"; +import type { AgentAutonomousStatus } from "../src/core/autonomous.js"; import type { SessionShutdownEvent } from "../src/index.js"; import { runPrintMode } from "../src/modes/print-mode.js"; @@ -12,13 +13,16 @@ type FakeExtensionRunner = { type FakeSession = { sessionManager: { getHeader: () => object | undefined }; - agent: { waitForIdle: () => Promise }; + agent: { waitForIdle: ReturnType Promise>> }; state: { messages: AssistantMessage[] }; extensionRunner: FakeExtensionRunner; bindExtensions: ReturnType; subscribe: ReturnType; prompt: ReturnType; reload: ReturnType; + getAutonomousStatus: ReturnType; + recordHostAutonomousContinuation: ReturnType; + refreshAutonomousGates: ReturnType; }; type FakeRuntimeHost = { @@ -55,7 +59,18 @@ function createAssistantMessage(options?: { }; } -function createRuntimeHost(assistantMessage: AssistantMessage): FakeRuntimeHost { +function createRuntimeHost( + assistantMessage: AssistantMessage, + autonomousStatus: AgentAutonomousStatus = { + enabled: false, + continuationsUsed: 0, + turnsUsed: 0, + tokensUsed: 0, + limits: { maxContinuations: 3, maxTurns: 12, maxTokens: 80_000, timeoutMs: 1_800_000 }, + gates: { commands: [], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: {}, + }, +): FakeRuntimeHost { const extensionRunner: FakeExtensionRunner = { hasHandlers: (eventType: string) => eventType === "session_shutdown", emit: vi.fn(async () => {}), @@ -65,13 +80,16 @@ function createRuntimeHost(assistantMessage: AssistantMessage): FakeRuntimeHost const session: FakeSession = { sessionManager: { getHeader: () => undefined }, - agent: { waitForIdle: async () => {} }, + agent: { waitForIdle: vi.fn(async () => {}) }, state, extensionRunner, bindExtensions: vi.fn(async () => {}), subscribe: vi.fn(() => () => {}), prompt: vi.fn(async () => {}), reload: vi.fn(async () => {}), + getAutonomousStatus: vi.fn(() => autonomousStatus), + recordHostAutonomousContinuation: vi.fn(), + refreshAutonomousGates: vi.fn(), }; return { @@ -139,4 +157,593 @@ describe("runPrintMode", () => { expect(session.extensionRunner.emit).toHaveBeenCalledTimes(1); expect(session.extensionRunner.emit).toHaveBeenCalledWith({ type: "session_shutdown", reason: "quit" }); }); + + it("stops host-driven gate retries once gate maxRetries is exhausted", async () => { + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 4 }, + lastGateFailure: { + command: "verify-public", + attempt: 4, + exitText: "not rerun: workspace unchanged since previous failed gate", + output: "edit source files before attempting to finish again", + }, + }); + const { session } = runtimeHost; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(1); + expect(session.prompt).not.toHaveBeenCalled(); + expect(session.recordHostAutonomousContinuation).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith( + "Autonomous quality gate still failing after attempt 4/3: not rerun: workspace unchanged since previous failed gate", + ); + }); + + it("refreshes autonomous gates after host-driven gate retries", async () => { + const failingStatus: AgentAutonomousStatus = { + enabled: true, + continuationsUsed: 0, + turnsUsed: 1, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 3, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "exited 1", + output: "0/9", + }, + }; + const passingStatus: AgentAutonomousStatus = { + ...failingStatus, + continuationsUsed: 1, + turnsUsed: 2, + tokensUsed: 200, + gateAttempts: { "verify-public": 0 }, + lastGateFailure: undefined, + }; + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "fixed the gate" }), failingStatus); + const { session } = runtimeHost; + let currentStatus = failingStatus; + session.getAutonomousStatus.mockImplementation(() => currentStatus); + session.refreshAutonomousGates.mockImplementation(() => { + currentStatus = passingStatus; + }); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(0); + expect(session.prompt).toHaveBeenCalledTimes(1); + expect(session.recordHostAutonomousContinuation).toHaveBeenCalledTimes(1); + expect(session.refreshAutonomousGates).toHaveBeenCalledTimes(1); + }); + + it("keeps autonomous gate prompting after a transient assistant error while limits remain", async () => { + const statuses: AgentAutonomousStatus[] = [ + { + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "exited 1", + output: "0/9", + }, + }, + { + enabled: true, + continuationsUsed: 1, + turnsUsed: 3, + tokensUsed: 200, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 2 }, + lastGateFailure: { + command: "verify-public", + attempt: 2, + exitText: "exited 1", + output: "0/9", + }, + }, + { + enabled: true, + continuationsUsed: 1, + turnsUsed: 4, + tokensUsed: 300, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 2 }, + lastGateFailure: { + command: "verify-public", + attempt: 2, + exitText: "exited 1", + output: "0/9", + }, + }, + { + enabled: true, + continuationsUsed: 1, + turnsUsed: 5, + tokensUsed: 400, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 2 }, + }, + ]; + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), statuses[0]); + const { session } = runtimeHost; + let statusIndex = 0; + session.getAutonomousStatus.mockImplementation( + () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, + ); + session.prompt.mockImplementationOnce(async () => { + session.state.messages = [createAssistantMessage({ stopReason: "error", errorMessage: "provider down" })]; + }); + session.prompt.mockImplementationOnce(async () => { + session.state.messages = [createAssistantMessage({ text: "still failing" })]; + }); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(0); + expect(session.prompt).toHaveBeenCalledTimes(2); + expect(session.recordHostAutonomousContinuation).toHaveBeenCalledTimes(2); + expect(errorSpy).not.toHaveBeenCalledWith("provider down"); + expect(session.prompt.mock.calls[0][0]).toContain("Autonomous quality gate failed"); + expect(session.prompt.mock.calls[1][0]).toContain("Autonomous quality gate failed"); + }); + + it("waits for a queued follow-up turn before evaluating transient assistant errors", async () => { + const statuses: AgentAutonomousStatus[] = [ + { + enabled: true, + continuationsUsed: 0, + turnsUsed: 1, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 1, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "exited 1", + output: "0/9", + }, + }, + { + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + tokensUsed: 200, + startedAt: Date.now(), + limits: { maxContinuations: 1, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + }, + ]; + const runtimeHost = createRuntimeHost( + createAssistantMessage({ stopReason: "error", errorMessage: "provider down" }), + statuses[0], + ); + const { session } = runtimeHost; + let statusIndex = 0; + session.getAutonomousStatus.mockImplementation( + () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, + ); + let waitCount = 0; + session.agent.waitForIdle.mockImplementation(async () => { + waitCount++; + if (waitCount === 2) { + session.state.messages = [createAssistantMessage({ text: "queued retry completed" })]; + } + }); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(0); + expect(session.prompt).toHaveBeenCalledTimes(1); + expect(session.recordHostAutonomousContinuation).toHaveBeenCalledTimes(1); + expect(session.agent.waitForIdle).toHaveBeenCalledTimes(3); + expect(errorSpy).not.toHaveBeenCalledWith("provider down"); + }); + + it("does not issue host-driven gate prompts once maxContinuations is reached", async () => { + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { + enabled: true, + continuationsUsed: 3, + turnsUsed: 2, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 3, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "exited 1", + output: "0/9", + }, + }); + const { session } = runtimeHost; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(1); + expect(session.prompt).not.toHaveBeenCalled(); + expect(session.recordHostAutonomousContinuation).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith( + "Autonomous quality gate still failing after attempt 1/3: exited 1; autonomous limit reached: maxContinuations reached (3/3)", + ); + }); + + it("returns non-zero when autonomous gates are still failing", async () => { + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { + enabled: true, + continuationsUsed: 999, + turnsUsed: 92, + tokensUsed: 215_535, + limits: { maxContinuations: 999, maxTurns: 1000, maxTokens: 2_000_000, timeoutMs: 1_800_000 }, + gates: { commands: ["verify-public"], maxRetries: 999, timeoutMs: 3_600_000 }, + gateAttempts: { "verify-public": 34 }, + lastGateFailure: { + command: "verify-public", + attempt: 34, + exitText: "exited 1", + output: "0/43", + }, + }); + const { session } = runtimeHost; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(1); + expect(errorSpy).toHaveBeenCalledWith( + "Autonomous quality gate still failing after attempt 34/999: exited 1; autonomous limit reached: maxContinuations reached (999/999)", + ); + expect(session.extensionRunner.emit).toHaveBeenCalledWith({ type: "session_shutdown", reason: "quit" }); + }); + + it("reports the exact autonomous limit that stopped a still-failing gate", async () => { + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { + enabled: true, + continuationsUsed: 34, + turnsUsed: 92, + tokensUsed: 2_000_000, + limits: { maxContinuations: 999, maxTurns: 1000, maxTokens: 2_000_000, timeoutMs: 1_800_000 }, + gates: { commands: ["verify-public"], maxRetries: 999, timeoutMs: 3_600_000 }, + gateAttempts: { "verify-public": 34 }, + lastGateFailure: { + command: "verify-public", + attempt: 34, + exitText: "exited 1", + output: "0/43", + }, + }); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(1); + expect(errorSpy).toHaveBeenCalledWith( + "Autonomous quality gate still failing after attempt 34/999: exited 1; autonomous limit reached: maxTokens reached (2000000/2000000)", + ); + }); + + it("returns non-zero when ungated autonomous runs stop at a limit", async () => { + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "I still need more work." }), { + enabled: true, + continuationsUsed: 3, + turnsUsed: 4, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 3, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: [], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: {}, + }); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(1); + expect(errorSpy).toHaveBeenCalledWith( + "Autonomous run stopped before terminal evidence; maxContinuations reached (3/3)", + ); + }); + + it("keeps prompting while autonomous gates fail below retry limits", async () => { + const statuses: AgentAutonomousStatus[] = [ + { + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "exited 1", + output: "0/9", + }, + }, + { + enabled: true, + continuationsUsed: 2, + turnsUsed: 3, + tokensUsed: 200, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 2 }, + lastGateFailure: { + command: "verify-public", + attempt: 2, + exitText: "exited 1", + output: "0/9 summary", + }, + }, + { + enabled: true, + continuationsUsed: 2, + turnsUsed: 4, + tokensUsed: 250, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 2 }, + }, + ]; + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still working" }), statuses[0]); + const { session } = runtimeHost; + let statusIndex = 0; + session.getAutonomousStatus.mockImplementation( + () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, + ); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(0); + expect(session.agent.waitForIdle).toHaveBeenCalledBefore(session.prompt); + expect(session.prompt).toHaveBeenCalledTimes(2); + expect(session.prompt.mock.calls[0][0]).toContain("Autonomous quality gate failed (attempt 1/3)"); + expect(session.prompt.mock.calls[0][0]).toContain("0/9"); + expect(session.prompt.mock.calls[0][1]).toEqual({ + streamingBehavior: "followUp", + internalPrompt: true, + suppressAutonomousContinuation: true, + }); + expect(session.prompt.mock.calls[1][0]).toContain("Autonomous quality gate failed (attempt 2/3)"); + expect(session.prompt.mock.calls[1][0]).toContain("0/9 summary"); + expect(session.prompt.mock.calls[1][1]).toEqual({ + streamingBehavior: "followUp", + internalPrompt: true, + suppressAutonomousContinuation: true, + }); + }); + + it("continues prompting when gate attempts do not advance but autonomous usage does", async () => { + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), { + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "exited 1", + output: "0/9", + }, + }); + const { session } = runtimeHost; + const statuses: AgentAutonomousStatus[] = [ + { + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + tokensUsed: 100, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { command: "verify-public", attempt: 1, exitText: "exited 1", output: "0/9" }, + }, + { + enabled: true, + continuationsUsed: 2, + turnsUsed: 3, + tokensUsed: 150, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "not rerun: workspace unchanged since previous failed gate", + output: "edit source files before attempting to finish again", + }, + }, + { + enabled: true, + continuationsUsed: 3, + turnsUsed: 4, + tokensUsed: 150, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "not rerun: workspace unchanged since previous failed gate", + output: "edit source files before attempting to finish again", + }, + }, + { + enabled: true, + continuationsUsed: 10, + turnsUsed: 5, + tokensUsed: 150, + startedAt: Date.now(), + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 3, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 1 }, + lastGateFailure: { + command: "verify-public", + attempt: 1, + exitText: "not rerun: workspace unchanged since previous failed gate", + output: "edit source files before attempting to finish again", + }, + }, + ]; + let statusIndex = 0; + session.getAutonomousStatus.mockImplementation( + () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, + ); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(1); + expect(session.agent.waitForIdle).toHaveBeenCalledTimes(7); + expect(session.prompt).toHaveBeenCalledTimes(3); + expect(session.recordHostAutonomousContinuation).toHaveBeenCalledTimes(3); + expect(session.prompt.mock.calls[1][0]).toContain("workspace unchanged"); + expect(session.prompt.mock.calls[2][0]).toContain("workspace unchanged"); + }); + + it("keeps prompting on repeated gate progress until autonomous limits stop the run", async () => { + const startedAt = Date.now(); + const statuses: AgentAutonomousStatus[] = [ + { + enabled: true, + continuationsUsed: 7, + turnsUsed: 8, + tokensUsed: 100, + startedAt, + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 20, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 7 }, + lastGateFailure: { + command: "verify-public", + attempt: 7, + exitText: "not rerun: workspace unchanged since previous failed gate", + output: "edit source files before attempting to finish again", + }, + }, + { + enabled: true, + continuationsUsed: 8, + turnsUsed: 9, + tokensUsed: 100, + startedAt, + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 20, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 7 }, + lastGateFailure: { + command: "verify-public", + attempt: 7, + exitText: "not rerun: workspace unchanged since previous failed gate", + output: "edit source files before attempting to finish again", + }, + }, + { + enabled: true, + continuationsUsed: 9, + turnsUsed: 10, + tokensUsed: 100, + startedAt, + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 20, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 7 }, + lastGateFailure: { + command: "verify-public", + attempt: 7, + exitText: "not rerun: workspace unchanged since previous failed gate", + output: "edit source files before attempting to finish again", + }, + }, + { + enabled: true, + continuationsUsed: 10, + turnsUsed: 11, + tokensUsed: 100, + startedAt, + limits: { maxContinuations: 10, maxTurns: 20, maxTokens: 100_000, timeoutMs: 60_000 }, + gates: { commands: ["verify-public"], maxRetries: 20, timeoutMs: 300_000 }, + gateAttempts: { "verify-public": 7 }, + lastGateFailure: { + command: "verify-public", + attempt: 7, + exitText: "not rerun: workspace unchanged since previous failed gate", + output: "edit source files before attempting to finish again", + }, + }, + ]; + const runtimeHost = createRuntimeHost(createAssistantMessage({ text: "still failing" }), statuses[0]); + const { session } = runtimeHost; + let statusIndex = 0; + session.getAutonomousStatus.mockImplementation( + () => statuses[Math.min(statusIndex++, statuses.length - 1)] as AgentAutonomousStatus, + ); + + const exitCode = await runPrintMode(runtimeHost as unknown as Parameters[0], { + mode: "text", + }); + + expect(exitCode).toBe(1); + expect(session.prompt).toHaveBeenCalledTimes(3); + expect(session.prompt.mock.calls[0][0]).toContain("workspace unchanged"); + expect(session.prompt.mock.calls[2][0]).toContain("workspace unchanged"); + }); }); diff --git a/packages/coding-agent/test/suite/agent-session-autonomous.test.ts b/packages/coding-agent/test/suite/agent-session-autonomous.test.ts new file mode 100644 index 00000000000..92668e17b03 --- /dev/null +++ b/packages/coding-agent/test/suite/agent-session-autonomous.test.ts @@ -0,0 +1,543 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { fauxAssistantMessage } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it } from "vitest"; +import { + addAutonomousUsage, + createAutonomousRuntimeState, + DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT, + nextAutonomousContinuation, + shouldAutonomouslyContinue, +} from "../../src/core/autonomous.js"; +import type { AgentCronJob } from "../../src/core/cron-jobs.js"; +import { createHarness, getAssistantTexts, getMessageText, getUserTexts, type Harness } from "./harness.js"; + +function isProcessRunning(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitForProcessExit(pid: number, timeoutMs = 2000): Promise { + const deadline = Date.now() + timeoutMs; + while (isProcessRunning(pid) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + return !isProcessRunning(pid); +} + +describe("AgentSession autonomous mode", () => { + const harnesses: Harness[] = []; + + afterEach(() => { + while (harnesses.length > 0) { + harnesses.pop()?.cleanup(); + } + }); + + it("injects a host-side continuation when the assistant asks the user for help", async () => { + const harness = await createHarness({ + autonomous: { enabled: true, maxContinuations: 1 }, + }); + harnesses.push(harness); + harness.setResponses([ + fauxAssistantMessage("Which package manager should I use?"), + fauxAssistantMessage("I inspected the repo and used npm."), + ]); + + await harness.session.prompt("fix the project"); + + expect(getAssistantTexts(harness)).toEqual([ + "Which package manager should I use?", + "I inspected the repo and used npm.", + ]); + expect(getUserTexts(harness)).toEqual(["fix the project", DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT]); + expect(harness.session.getAutonomousStatus()).toMatchObject({ + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + }); + }); + + it("continues through a claimed external blocker instead of trusting prose", async () => { + const harness = await createHarness({ + autonomous: { enabled: true, maxContinuations: 1 }, + }); + harnesses.push(harness); + harness.setResponses([ + fauxAssistantMessage("Blocked: this requires an API key credential from the user."), + fauxAssistantMessage( + "I will inspect the environment and verify whether the credential is actually unavailable.", + ), + ]); + + await harness.session.prompt("run the private eval"); + + expect(getUserTexts(harness)).toEqual(["run the private eval", DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT]); + expect(harness.session.getAutonomousStatus()).toMatchObject({ + enabled: true, + continuationsUsed: 1, + turnsUsed: 2, + }); + }); + + it("stops after the configured autonomous continuation limit", async () => { + const harness = await createHarness({ + autonomous: { enabled: true, maxContinuations: 1 }, + }); + harnesses.push(harness); + harness.setResponses([ + fauxAssistantMessage("Can you confirm the test command?"), + fauxAssistantMessage("Can you confirm whether to run lint too?"), + ]); + + await harness.session.prompt("make the change"); + + expect(getAssistantTexts(harness)).toEqual([ + "Can you confirm the test command?", + "Can you confirm whether to run lint too?", + ]); + expect(getUserTexts(harness)).toEqual(["make the change", DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT]); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + }); + + it("does not count failed assistant messages against autonomous usage limits", async () => { + const harness = await createHarness({ + autonomous: { enabled: true, maxTurns: 1 }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("failed", { stopReason: "error", errorMessage: "provider failed" })]); + + await harness.session.prompt("try once"); + + expect(harness.session.getAutonomousStatus()).toMatchObject({ + turnsUsed: 0, + tokensUsed: 0, + continuationsUsed: 0, + }); + }); + + it("counts aborted assistant messages against autonomous usage limits", async () => { + const harness = await createHarness({ + autonomous: { enabled: true, maxTurns: 1 }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("aborted", { stopReason: "aborted" })]); + + await harness.session.prompt("try once"); + + expect(harness.session.getAutonomousStatus()).toMatchObject({ + turnsUsed: 1, + continuationsUsed: 0, + }); + }); + + it("supports /autonomous on and off without calling the model", async () => { + const harness = await createHarness(); + harnesses.push(harness); + + await harness.session.prompt("/autonomous on"); + await harness.session.prompt("/autonomous off"); + + expect(harness.getPendingResponseCount()).toBe(0); + expect(harness.session.getAutonomousStatus().enabled).toBe(false); + const statusMessages = harness.session.messages.filter( + (message) => message.role === "custom" && message.customType === "autonomous_status", + ); + expect(statusMessages).toHaveLength(2); + }); + + it("continues when the assistant tries to finish without terminal evidence", async () => { + const harness = await createHarness({ + autonomous: { enabled: true, maxContinuations: 1 }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("Done."), fauxAssistantMessage("I will collect concrete evidence.")]); + + await harness.session.prompt("make the change"); + + expect(getUserTexts(harness)).toEqual(["make the change", DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT]); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + }); + + it("continues after a git worktree change instead of letting the agent self-terminate", async () => { + const harness = await createHarness(); + harnesses.push(harness); + execFileSync("git", ["init"], { cwd: harness.tempDir, stdio: "ignore" }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: harness.tempDir }); + execFileSync("git", ["config", "user.name", "Test User"], { cwd: harness.tempDir }); + const path = join(harness.tempDir, "file.txt"); + writeFileSync(path, "before\n"); + execFileSync("git", ["add", "file.txt"], { cwd: harness.tempDir }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "--no-gpg-sign", "-m", "initial"], { + cwd: harness.tempDir, + stdio: "ignore", + }); + await harness.session.prompt("/autonomous on"); + writeFileSync(path, "after\n"); + harness.setResponses([ + fauxAssistantMessage("Done."), + fauxAssistantMessage("Continuing until the evaluator stops me."), + ]); + + await harness.session.prompt("make the change"); + + expect(getUserTexts(harness)[0]).toBe("make the change"); + expect(getUserTexts(harness).slice(1)).toContain(DEFAULT_AUTONOMOUS_CONTINUATION_PROMPT); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBeGreaterThan(0); + }); + + it("runs autonomous gates before applying usage limits", async () => { + const state = createAutonomousRuntimeState({ + enabled: true, + maxTurns: 1, + gates: { commands: [`${process.execPath} -e "process.exit(0)"`] }, + }); + state.turnsUsed = 1; + state.lastGateFailure = { + command: "stale gate", + attempt: 1, + exitText: "exited 1", + output: "stale failure", + }; + + expect( + await shouldAutonomouslyContinue(state, fauxAssistantMessage("Done."), { cwd: process.cwd() }), + ).toMatchObject({ + shouldContinue: false, + reason: "not_needed", + }); + expect(state.lastGateFailure).toBeUndefined(); + }); + + it("lets passing autonomous gates complete the run under verifier control", async () => { + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 1, + gates: { commands: [`${process.execPath} -e "process.exit(0)"`] }, + }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("Done.")]); + + await harness.session.prompt("make the change"); + + expect(getUserTexts(harness)).toEqual(["make the change"]); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(0); + }); + + it("feeds failing autonomous gate output back into the session", async () => { + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 1, + gates: { + commands: [`${process.execPath} -e "console.error('gate failed'); process.exit(1)"`], + maxRetries: 2, + }, + }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("Done."), fauxAssistantMessage("I will fix the gate failure.")]); + + await harness.session.prompt("make the change"); + + const users = getUserTexts(harness); + expect(users[1]).toContain("Autonomous quality gate failed"); + expect(users[1]).toContain("gate failed"); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + }); + + it("suppresses autonomous continuation injection for host-driven gate prompts", async () => { + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + gates: { + commands: [`${process.execPath} -e "console.error('gate failed'); process.exit(1)"`], + maxRetries: 2, + }, + }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("Still failing.")]); + + harness.session.recordHostAutonomousContinuation(); + await harness.session.prompt("host gate follow-up", { + internalPrompt: true, + suppressAutonomousContinuation: true, + }); + + expect(getUserTexts(harness)).toEqual(["host gate follow-up"]); + expect(getAssistantTexts(harness)).toEqual(["Still failing."]); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + }); + + it("suppresses autonomous continuation injection for queued host-driven prompts", async () => { + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + gates: { + commands: [`${process.execPath} -e "console.error('gate failed'); process.exit(1)"`], + maxRetries: 2, + }, + }, + }); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("Still failing.")]); + const sessionInternals = harness.session as unknown as { + _compactionAbortController?: AbortController; + }; + sessionInternals._compactionAbortController = new AbortController(); + const heartbeatJob = { + id: "heartbeat-test", + status: "active", + activeSessionId: "active-test", + sessionId: harness.session.sessionId, + sessionFile: harness.session.sessionFile ?? "session.jsonl", + cwd: harness.tempDir, + prompt: "host gate follow-up", + schedule: { kind: "interval", expression: "every 5m", intervalMs: 300_000 }, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + runCount: 0, + } satisfies AgentCronJob; + + await harness.session.promptHeartbeat(heartbeatJob, { + queueIfBusy: true, + streamingBehavior: "followUp", + suppressAutonomousContinuation: true, + }); + sessionInternals._compactionAbortController = undefined; + await harness.session.agent.continue(); + + expect(getAssistantTexts(harness)).toEqual(["Still failing."]); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(0); + expect(harness.getPendingResponseCount()).toBe(0); + }); + + it("advances retry budget without rerunning a failed autonomous gate until the workspace changes", async () => { + const tempDir = join(process.cwd(), `.tmp-autonomous-gate-${Date.now()}-${Math.random().toString(36).slice(2)}`); + execFileSync("mkdir", ["-p", join(tempDir, "verification")]); + execFileSync("git", ["init"], { cwd: tempDir, stdio: "ignore" }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: tempDir }); + execFileSync("git", ["config", "user.name", "Test User"], { cwd: tempDir }); + writeFileSync(join(tempDir, "src.rs"), "initial\n"); + execFileSync("git", ["add", "src.rs"], { cwd: tempDir }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "--no-gpg-sign", "-m", "initial"], { + cwd: tempDir, + stdio: "ignore", + }); + try { + const counter = join(tempDir, "verification", "public_feedback_scores.jsonl"); + const gate = `${process.execPath} -e "const fs=require('fs'); const p='${counter}'; const n=fs.existsSync(p)?fs.readFileSync(p,'utf8').trim().split(/\\n/).filter(Boolean).length:0; fs.appendFileSync(p,JSON.stringify({run:n+1,score:0})+'\\n'); process.exit(1);"`; + const state = createAutonomousRuntimeState( + { enabled: true, maxContinuations: 3, gates: { commands: [gate], maxRetries: 3 } }, + { cwd: tempDir }, + ); + + const first = await nextAutonomousContinuation(state, fauxAssistantMessage("Done."), { cwd: tempDir }); + writeFileSync(join(tempDir, "Cargo.lock"), "generated lockfile\n"); + const second = await nextAutonomousContinuation(state, fauxAssistantMessage("Still done."), { cwd: tempDir }); + + expect(first).toBeDefined(); + expect(second).toBeDefined(); + expect(getMessageText(second)).toContain("workspace has not changed"); + expect(getMessageText(second)).toContain("Edit source files"); + expect(readFileSync(counter, "utf8").trim().split(/\n/)).toHaveLength(1); + expect(state.gateAttempts[gate]).toBe(2); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("reruns a failed autonomous gate when untracked file contents change", async () => { + const tempDir = join( + process.cwd(), + `.tmp-autonomous-untracked-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + execFileSync("mkdir", ["-p", tempDir]); + execFileSync("git", ["init"], { cwd: tempDir, stdio: "ignore" }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: tempDir }); + execFileSync("git", ["config", "user.name", "Test User"], { cwd: tempDir }); + writeFileSync(join(tempDir, "src.rs"), "initial\n"); + execFileSync("git", ["add", "src.rs"], { cwd: tempDir }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "--no-gpg-sign", "-m", "initial"], { + cwd: tempDir, + stdio: "ignore", + }); + try { + const candidate = join(tempDir, "candidate.txt"); + writeFileSync(candidate, "bad\n"); + const gate = `${process.execPath} -e "const fs=require('fs'); process.exit(fs.readFileSync('candidate.txt','utf8').trim()==='good'?0:1)"`; + const state = createAutonomousRuntimeState({ + enabled: true, + maxContinuations: 3, + gates: { commands: [gate], maxRetries: 3 }, + }); + + const first = await nextAutonomousContinuation(state, fauxAssistantMessage("Done."), { cwd: tempDir }); + writeFileSync(candidate, "good\n"); + const second = await nextAutonomousContinuation(state, fauxAssistantMessage("Still done."), { + cwd: tempDir, + }); + + expect(first).toBeDefined(); + expect(second).toBeUndefined(); + expect(state.lastGateFailure).toBeUndefined(); + expect(state.gateAttempts[gate]).toBe(0); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("bounds captured autonomous gate output", async () => { + const gate = `${process.execPath} -e "process.stdout.write('x'.repeat(20000)); process.exit(1)"`; + const state = createAutonomousRuntimeState({ + enabled: true, + maxContinuations: 1, + gates: { commands: [gate], maxRetries: 1 }, + }); + + await nextAutonomousContinuation(state, fauxAssistantMessage("Done."), { cwd: process.cwd() }); + + expect(state.lastGateFailure?.output).toContain("... [truncated]"); + expect(state.lastGateFailure?.output.length).toBeLessThan(6100); + }); + + it("terminates the autonomous gate process tree when the timeout expires", async () => { + const tempDir = join( + process.cwd(), + `.tmp-autonomous-process-tree-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + execFileSync("mkdir", ["-p", tempDir]); + execFileSync("git", ["init"], { cwd: tempDir, stdio: "ignore" }); + const pidFile = join(tempDir, "descendant.pid"); + const script = join(tempDir, "gate.cjs"); + writeFileSync( + script, + `const { spawn } = require("node:child_process");\n` + + `const { writeFileSync } = require("node:fs");\n` + + `const child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: "inherit" });\n` + + `writeFileSync(${JSON.stringify(pidFile)}, String(child.pid));\n` + + `setTimeout(() => {}, 60000);\n`, + ); + let descendantPid: number | undefined; + try { + const state = createAutonomousRuntimeState({ + enabled: true, + maxContinuations: 1, + gates: { commands: [`${process.execPath} gate.cjs`], maxRetries: 1, timeoutMs: 250 }, + }); + const startedAt = Date.now(); + + await nextAutonomousContinuation(state, fauxAssistantMessage("Done."), { cwd: tempDir }); + descendantPid = Number.parseInt(readFileSync(pidFile, "utf8"), 10); + + expect(Date.now() - startedAt).toBeLessThan(3000); + expect(state.lastGateFailure?.exitText).toBe("timed out"); + expect(await waitForProcessExit(descendantPid)).toBe(true); + } finally { + if (descendantPid && isProcessRunning(descendantPid)) { + process.kill(descendantPid, "SIGKILL"); + } + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("stops autonomous continuation once gate retries are exhausted", async () => { + const state = createAutonomousRuntimeState({ + enabled: true, + maxContinuations: 5, + gates: { commands: [`${process.execPath} -e "process.exit(1)"`], maxRetries: 1 }, + }); + + const first = await nextAutonomousContinuation(state, fauxAssistantMessage("Done."), { cwd: process.cwd() }); + const second = await nextAutonomousContinuation(state, fauxAssistantMessage("Still done."), { + cwd: process.cwd(), + }); + + expect(first).toBeDefined(); + expect(second).toBeUndefined(); + expect(state.continuationsUsed).toBe(1); + }); + + it("records the post-failure worktree snapshot for gate rerun suppression", async () => { + const tempDir = join( + process.cwd(), + `.tmp-autonomous-post-snapshot-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + execFileSync("mkdir", ["-p", tempDir]); + execFileSync("git", ["init"], { cwd: tempDir, stdio: "ignore" }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: tempDir }); + execFileSync("git", ["config", "user.name", "Test User"], { cwd: tempDir }); + writeFileSync(join(tempDir, "src.rs"), "initial\n"); + execFileSync("git", ["add", "src.rs"], { cwd: tempDir }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "--no-gpg-sign", "-m", "initial"], { + cwd: tempDir, + stdio: "ignore", + }); + try { + const generated = join(tempDir, "generated.txt"); + const gate = `${process.execPath} -e "const fs=require('fs'); fs.appendFileSync('${generated}', 'run\\n'); process.exit(1);"`; + const state = createAutonomousRuntimeState( + { enabled: true, maxContinuations: 3, gates: { commands: [gate], maxRetries: 3 } }, + { cwd: tempDir }, + ); + + const first = await nextAutonomousContinuation(state, fauxAssistantMessage("Done."), { cwd: tempDir }); + const second = await nextAutonomousContinuation(state, fauxAssistantMessage("Still done."), { cwd: tempDir }); + + expect(first).toBeDefined(); + expect(second).toBeDefined(); + expect(getMessageText(second)).toContain("workspace has not changed"); + expect(readFileSync(generated, "utf8").trim().split(/\n/)).toHaveLength(1); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("does not count cache-read tokens against the autonomous token budget", async () => { + const state = createAutonomousRuntimeState({ enabled: true, maxTokens: 10 }); + + addAutonomousUsage(state, { + input: 2, + output: 3, + cacheRead: 1_000, + cacheWrite: 4, + totalTokens: 1_009, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }); + + expect(state.tokensUsed).toBe(9); + expect(await shouldAutonomouslyContinue(state, fauxAssistantMessage("Done."))).toMatchObject({ + shouldContinue: true, + }); + }); + + it("does not use assistant prose as terminal blocker evidence", async () => { + const state = createAutonomousRuntimeState({ enabled: true }); + + expect( + await shouldAutonomouslyContinue(state, fauxAssistantMessage("I'm blocked. What should I try next?")), + ).toMatchObject({ + shouldContinue: true, + reason: "missing_terminal_evidence", + }); + expect( + await shouldAutonomouslyContinue( + state, + fauxAssistantMessage("Blocked: this requires OAuth login from the user."), + ), + ).toMatchObject({ + shouldContinue: true, + reason: "missing_terminal_evidence", + }); + }); +}); diff --git a/packages/coding-agent/test/suite/agent-session-compact-skill.test.ts b/packages/coding-agent/test/suite/agent-session-compact-skill.test.ts index 64a835d4b2d..b1b1425bc10 100644 --- a/packages/coding-agent/test/suite/agent-session-compact-skill.test.ts +++ b/packages/coding-agent/test/suite/agent-session-compact-skill.test.ts @@ -5,7 +5,7 @@ import { createHarness, type Harness } from "./harness.js"; type SessionInternals = { _checkCompaction: (assistantMessage: AssistantMessage, skipAbortedCheck?: boolean) => Promise; - _shouldStopAfterTurn: (context: ShouldStopAfterTurnContext) => boolean; + _shouldStopAfterTurn: (context: ShouldStopAfterTurnContext) => Promise; _createKernelHostHandlers: () => Record; }; @@ -109,9 +109,9 @@ describe("AgentSession compact skill host requests", () => { setStreaming(harness, false); const internals = harness.session as unknown as SessionInternals; - expect(internals._shouldStopAfterTurn({ message: createAssistant(harness) } as ShouldStopAfterTurnContext)).toBe( - true, - ); + await expect( + internals._shouldStopAfterTurn({ message: createAssistant(harness) } as ShouldStopAfterTurnContext), + ).resolves.toBe(true); await internals._checkCompaction(createAssistant(harness)); const compactionEntries = harness.sessionManager.getEntries().filter((entry) => entry.type === "compaction"); diff --git a/packages/coding-agent/test/suite/agent-session-compaction.test.ts b/packages/coding-agent/test/suite/agent-session-compaction.test.ts index e92233152ac..d84f188f6aa 100644 --- a/packages/coding-agent/test/suite/agent-session-compaction.test.ts +++ b/packages/coding-agent/test/suite/agent-session-compaction.test.ts @@ -1,11 +1,15 @@ import type { AgentMessage, ShouldStopAfterTurnContext } from "@earendil-works/pi-agent-core"; import { type AssistantMessage, fauxAssistantMessage, type Model, type ToolResultMessage } from "@earendil-works/pi-ai"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { createHarness, type Harness } from "./harness.js"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createHarness, getMessageText, type Harness } from "./harness.js"; type SessionWithCompactionInternals = { - _checkCompaction: (assistantMessage: AssistantMessage, skipAbortedCheck?: boolean) => Promise; - _runAutoCompaction: (reason: "overflow" | "threshold", willRetry: boolean) => Promise; + _checkCompaction: ( + assistantMessage: AssistantMessage, + skipAbortedCheck?: boolean, + queueAutonomousContinuation?: boolean, + ) => Promise; + _runAutoCompaction: (reason: "overflow" | "threshold" | "requested", willRetry: boolean) => Promise; _shouldStopAfterTurn: (context: ShouldStopAfterTurnContext) => boolean | Promise; }; @@ -43,9 +47,17 @@ function createAssistant( }; } +function failingGateCommand(): string { + return `${process.execPath} -e "console.error('gate failed'); process.exit(1)"`; +} + describe("AgentSession compaction characterization", () => { const harnesses: Harness[] = []; + beforeEach(() => { + vi.useRealTimers(); + }); + afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); @@ -198,6 +210,95 @@ describe("AgentSession compaction characterization", () => { expect(continueSpy).toHaveBeenCalledTimes(1); }); + it("keeps prior autonomous continuations when later threshold compaction is skipped", async () => { + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 4, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + }); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as { + _queueAutonomousContinuationForThresholdCompaction( + message: AssistantMessage, + ): Promise; + _clearQueuedAutonomousContinuationsAfterSkippedThresholdCompaction( + shouldContinueAfterThreshold: boolean, + queuedMessages: AgentMessage[], + ): void; + _postCompactionContinuationMessages: AgentMessage[]; + }; + const firstAssistant = createAssistant(harness, { stopReason: "toolUse", totalTokens: 10_000 }); + const secondAssistant = createAssistant(harness, { stopReason: "toolUse", totalTokens: 10_000 }); + + const firstQueued = await sessionInternals._queueAutonomousContinuationForThresholdCompaction(firstAssistant); + const secondQueued = await sessionInternals._queueAutonomousContinuationForThresholdCompaction(secondAssistant); + + expect(firstQueued).toBeDefined(); + expect(secondQueued).toBeDefined(); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(2); + sessionInternals._clearQueuedAutonomousContinuationsAfterSkippedThresholdCompaction(true, [secondQueued!]); + + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + expect(sessionInternals._postCompactionContinuationMessages).toEqual([firstQueued]); + expect( + harness.session.agent.removeQueuedMessages((message) => message === firstQueued || message === secondQueued), + ).toEqual([firstQueued]); + }); + + it("clears queued autonomous continuations when threshold compaction is skipped", async () => { + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { keepRecentTokens: 1 } }, + }); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as SessionWithCompactionInternals; + const successfulAssistant = createAssistant(harness, { + stopReason: "toolUse", + totalTokens: 10_000, + timestamp: Date.now(), + }); + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: "call-1", + toolName: "large-context", + content: [{ type: "text", text: "x".repeat(800_000) }], + isError: false, + timestamp: Date.now() + 500, + }; + harness.session.agent.state.messages = [ + { role: "user", content: [{ type: "text", text: "hello" }], timestamp: Date.now() - 1000 }, + successfulAssistant, + toolResult, + ]; + + const shouldStop = await sessionInternals._shouldStopAfterTurn({ + message: successfulAssistant, + toolResults: [toolResult], + context: { + systemPrompt: harness.session.systemPrompt, + messages: [successfulAssistant, toolResult], + tools: [], + }, + newMessages: [successfulAssistant, toolResult], + }); + expect(shouldStop).toBe(true); + expect(harness.session.agent.hasQueuedMessages()).toBe(true); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + + await sessionInternals._runAutoCompaction("threshold", false); + + expect(harness.session.agent.hasQueuedMessages()).toBe(false); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(0); + }); + it("emits a warning when auto-compaction has nothing to summarize", async () => { const harness = await createHarness(); harnesses.push(harness); @@ -378,6 +479,375 @@ describe("AgentSession compaction characterization", () => { expect(shouldStop).toBe(true); }); + it("queues a failing autonomous gate continuation before threshold compaction stops a tool loop", async () => { + vi.useFakeTimers(); + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 1 } }, + models: [{ id: "faux-1", contextWindow: 200_000 }], + extensionFactories: [ + (pi) => { + pi.on("session_before_compact", async (event) => ({ + compaction: { + summary: "auto compacted", + firstKeptEntryId: event.preparation.firstKeptEntryId, + tokensBefore: event.preparation.tokensBefore, + details: {}, + }, + })); + }, + ], + }); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as SessionWithCompactionInternals; + const successfulAssistant = createAssistant(harness, { + stopReason: "toolUse", + totalTokens: 10_000, + timestamp: Date.now(), + }); + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: "call-1", + toolName: "large-context", + content: [{ type: "text", text: "x".repeat(800_000) }], + isError: false, + timestamp: Date.now() + 500, + }; + const currentUser = { + role: "user", + content: [{ type: "text", text: "hello" }], + timestamp: Date.now() - 1000, + } satisfies Parameters[0]; + const oldUser = { + role: "user", + content: [{ type: "text", text: "old" }], + timestamp: Date.now() - 3000, + } satisfies Parameters[0]; + const oldAssistant = createAssistant(harness, { + stopReason: "stop", + totalTokens: 100, + timestamp: Date.now() - 2000, + }); + const oldMessages: AgentMessage[] = [oldUser, oldAssistant]; + const messages: AgentMessage[] = [currentUser, successfulAssistant, toolResult]; + for (const message of [oldUser, oldAssistant, currentUser, successfulAssistant]) { + harness.sessionManager.appendMessage(message); + } + harness.session.agent.state.messages = [...oldMessages, ...messages]; + + const continueSpy = vi.spyOn(harness.session.agent, "continue").mockResolvedValue(); + const followUpSpy = vi.spyOn(harness.session.agent, "followUp"); + + const shouldStop = await sessionInternals._shouldStopAfterTurn({ + message: successfulAssistant, + toolResults: [toolResult], + context: { systemPrompt: harness.session.systemPrompt, messages, tools: [] }, + newMessages: [successfulAssistant, toolResult], + }); + + expect(shouldStop).toBe(true); + expect(harness.session.getAutonomousStatus()).toMatchObject({ + continuationsUsed: 1, + gates: expect.objectContaining({ maxRetries: 5 }), + }); + expect(followUpSpy).toHaveBeenCalledTimes(1); + const queuedText = getMessageText(followUpSpy.mock.calls[0]?.[0]); + expect(queuedText).toContain("Autonomous quality gate failed (attempt 1/5)"); + expect(queuedText).toContain("gate failed"); + + await sessionInternals._runAutoCompaction("threshold", false); + await vi.advanceTimersByTimeAsync(100); + + expect(continueSpy).toHaveBeenCalledTimes(1); + }); + + it("keeps autonomous continuation bookkeeping when only steering queue is drained", async () => { + vi.useFakeTimers(); + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 1 } }, + models: [{ id: "faux-1", contextWindow: 200_000 }], + }); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as { + _schedulePostCompactionContinue(): void; + _postCompactionContinuationMessages: AgentMessage[]; + }; + const steeringMessage = { + role: "user", + content: [{ type: "text", text: "steer first" }], + timestamp: Date.now(), + } satisfies AgentMessage; + const autonomousMessage = { + role: "user", + content: [{ type: "text", text: "autonomous follow-up" }], + timestamp: Date.now(), + } satisfies AgentMessage; + sessionInternals._postCompactionContinuationMessages = [autonomousMessage]; + harness.session.agent.state.messages = [{ ...fauxAssistantMessage("done"), timestamp: Date.now() - 1000 }]; + harness.session.agent.steer(steeringMessage); + harness.session.agent.followUp(autonomousMessage); + const continueSpy = vi.spyOn(harness.session.agent, "continue").mockResolvedValue(); + const followUpSpy = vi.spyOn(harness.session.agent, "followUp"); + + sessionInternals._schedulePostCompactionContinue(); + await vi.advanceTimersByTimeAsync(100); + + expect(continueSpy).toHaveBeenCalledTimes(1); + expect(sessionInternals._postCompactionContinuationMessages).toEqual([autonomousMessage]); + expect(followUpSpy).toHaveBeenCalledWith(autonomousMessage); + }); + + it("keeps autonomous threshold continuations when post-compaction continue must retry", async () => { + vi.useFakeTimers(); + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 1 } }, + models: [{ id: "faux-1", contextWindow: 200_000 }], + }); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as { + _schedulePostCompactionContinue(): void; + _postCompactionContinuationMessages: AgentMessage[]; + _postCompactionContinuationScheduled: boolean; + }; + const queuedMessage = { + role: "user", + content: [{ type: "text", text: "autonomous follow-up" }], + timestamp: Date.now(), + } satisfies AgentMessage; + sessionInternals._postCompactionContinuationMessages = [queuedMessage]; + harness.session.agent.state.messages = [ + { role: "user", content: [{ type: "text", text: "hello" }], timestamp: Date.now() - 1000 }, + ]; + const continueSpy = vi + .spyOn(harness.session.agent, "continue") + .mockRejectedValueOnce(new Error("already processing")); + + sessionInternals._schedulePostCompactionContinue(); + await vi.advanceTimersByTimeAsync(100); + + expect(continueSpy).toHaveBeenCalledTimes(1); + expect(sessionInternals._postCompactionContinuationMessages).toEqual([queuedMessage]); + expect(sessionInternals._postCompactionContinuationScheduled).toBe(true); + }); + + it("clears queued autonomous threshold continuations when autonomous mode is disabled", async () => { + vi.useFakeTimers(); + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 1 } }, + models: [{ id: "faux-1", contextWindow: 200_000 }], + extensionFactories: [ + (pi) => { + pi.on("session_before_compact", async (event) => ({ + compaction: { + summary: "auto compacted", + firstKeptEntryId: event.preparation.firstKeptEntryId, + tokensBefore: event.preparation.tokensBefore, + details: {}, + }, + })); + }, + ], + }); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as SessionWithCompactionInternals; + const successfulAssistant = createAssistant(harness, { + stopReason: "toolUse", + totalTokens: 10_000, + timestamp: Date.now(), + }); + const toolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: "call-1", + toolName: "large-context", + content: [{ type: "text", text: "x".repeat(800_000) }], + isError: false, + timestamp: Date.now() + 500, + }; + const oldUser = { + role: "user", + content: [{ type: "text", text: "old" }], + timestamp: Date.now() - 3000, + } satisfies Parameters[0]; + const oldAssistant = createAssistant(harness, { + stopReason: "stop", + totalTokens: 100, + timestamp: Date.now() - 2000, + }); + const currentUser = { + role: "user", + content: [{ type: "text", text: "hello" }], + timestamp: Date.now() - 1000, + } satisfies Parameters[0]; + for (const message of [oldUser, oldAssistant, currentUser, successfulAssistant]) { + harness.sessionManager.appendMessage(message); + } + harness.session.agent.state.messages = [oldUser, oldAssistant, currentUser, successfulAssistant, toolResult]; + const continueSpy = vi.spyOn(harness.session.agent, "continue").mockResolvedValue(); + + await sessionInternals._shouldStopAfterTurn({ + message: successfulAssistant, + toolResults: [toolResult], + context: { + systemPrompt: harness.session.systemPrompt, + messages: [currentUser, successfulAssistant, toolResult], + tools: [], + }, + newMessages: [successfulAssistant, toolResult], + }); + expect(harness.session.agent.hasQueuedMessages()).toBe(true); + + await harness.session.prompt("/autonomous off"); + expect(harness.session.getAutonomousStatus().enabled).toBe(false); + expect(harness.session.agent.hasQueuedMessages()).toBe(false); + + await sessionInternals._runAutoCompaction("threshold", false); + await vi.advanceTimersByTimeAsync(100); + + expect(continueSpy).not.toHaveBeenCalled(); + }); + + it("queues a failing autonomous gate continuation before post-turn threshold compaction", async () => { + vi.useFakeTimers(); + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 1 } }, + models: [{ id: "faux-1", contextWindow: 200_000 }], + }); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as SessionWithCompactionInternals; + const successfulAssistant = createAssistant(harness, { + stopReason: "stop", + totalTokens: 10_000, + timestamp: Date.now(), + }); + const largeToolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: "call-1", + toolName: "large-context", + content: [{ type: "text", text: "x".repeat(800_000) }], + isError: false, + timestamp: Date.now() + 500, + }; + harness.session.agent.state.messages = [ + { role: "user", content: [{ type: "text", text: "hello" }], timestamp: Date.now() - 1000 }, + successfulAssistant, + largeToolResult, + ]; + + const runCompactionSpy = vi.spyOn(sessionInternals, "_runAutoCompaction").mockResolvedValue(); + const followUpSpy = vi.spyOn(harness.session.agent, "followUp"); + + await sessionInternals._checkCompaction(successfulAssistant, false); + + expect(runCompactionSpy).toHaveBeenCalledWith("threshold", false); + expect(followUpSpy).toHaveBeenCalledTimes(1); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1); + const queuedText = getMessageText(followUpSpy.mock.calls[0]?.[0]); + expect(queuedText).toContain("Autonomous quality gate failed (attempt 1/5)"); + expect(queuedText).toContain("gate failed"); + }); + + it("does not queue autonomous gate continuations for pre-prompt threshold compaction", async () => { + vi.useFakeTimers(); + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 2, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 1 } }, + models: [{ id: "faux-1", contextWindow: 200_000 }], + }); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as SessionWithCompactionInternals; + const successfulAssistant = createAssistant(harness, { + stopReason: "stop", + totalTokens: 10_000, + timestamp: Date.now(), + }); + const largeToolResult: ToolResultMessage = { + role: "toolResult", + toolCallId: "call-1", + toolName: "large-context", + content: [{ type: "text", text: "x".repeat(800_000) }], + isError: false, + timestamp: Date.now() + 500, + }; + harness.session.agent.state.messages = [ + { role: "user", content: [{ type: "text", text: "hello" }], timestamp: Date.now() - 1000 }, + successfulAssistant, + largeToolResult, + ]; + + const runCompactionSpy = vi.spyOn(sessionInternals, "_runAutoCompaction").mockResolvedValue(); + const followUpSpy = vi.spyOn(harness.session.agent, "followUp"); + + await sessionInternals._checkCompaction(successfulAssistant, false, false); + + expect(runCompactionSpy).toHaveBeenCalledWith("threshold", false); + expect(followUpSpy).not.toHaveBeenCalled(); + expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(0); + }); + + it("waits for threshold-compaction autonomous continuations before finishing prompt", async () => { + vi.useFakeTimers(); + const harness = await createHarness({ + autonomous: { + enabled: true, + maxContinuations: 1, + maxTurns: 100, + gates: { commands: [failingGateCommand()], maxRetries: 5 }, + }, + settings: { compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 1 } }, + models: [{ id: "faux-1", contextWindow: 200_000 }], + }); + harnesses.push(harness); + const highUsageDone = { + ...fauxAssistantMessage("done"), + usage: createUsage(10_000), + }; + harness.setResponses([highUsageDone, fauxAssistantMessage("retry")]); + const promptPromise = harness.session.prompt("make the change"); + + await vi.waitFor(() => expect(harness.session.getAutonomousStatus().continuationsUsed).toBe(1)); + await vi.advanceTimersByTimeAsync(100); + await promptPromise; + + expect(harness.session.getAutonomousStatus()).toMatchObject({ + continuationsUsed: 1, + turnsUsed: 2, + }); + }); + it("does not trigger threshold compaction for error messages when no prior usage exists", async () => { const harness = await createHarness(); harnesses.push(harness); diff --git a/packages/coding-agent/test/suite/agent-session-prompt.test.ts b/packages/coding-agent/test/suite/agent-session-prompt.test.ts index 327f0f60fe8..c7dcb020017 100644 --- a/packages/coding-agent/test/suite/agent-session-prompt.test.ts +++ b/packages/coding-agent/test/suite/agent-session-prompt.test.ts @@ -775,6 +775,62 @@ stale extension instructions`, ).toBeUndefined(); }); + it("handles autonomous slash commands when template expansion is disabled", async () => { + const harness = await createHarness(); + harnesses.push(harness); + + await harness.session.prompt("/autonomous on", { expandPromptTemplates: false }); + + expect(harness.session.getAutonomousStatus().enabled).toBe(true); + expect(harness.getPendingResponseCount()).toBe(0); + expect(harness.session.messages.some((message) => message.role === "custom")).toBe(true); + }); + + it("handles goal slash commands when template expansion is disabled", async () => { + const harness = await createHarness(); + harnesses.push(harness); + + await harness.session.prompt("/goal status", { expandPromptTemplates: false }); + + expect(harness.eventsOfType("goal_update")).toHaveLength(1); + expect(harness.getPendingResponseCount()).toBe(0); + expect(getAssistantTexts(harness)).toEqual([]); + }); + + it("does not run built-in slash commands immediately while queueIfBusy backpressure is active", async () => { + const harness = await createHarness(); + harnesses.push(harness); + const sessionInternals = harness.session as unknown as { + _compactionAbortController?: AbortController; + }; + sessionInternals._compactionAbortController = new AbortController(); + + await expect( + harness.session.prompt("/autonomous on", { + queueIfBusy: true, + streamingBehavior: "followUp", + }), + ).rejects.toThrow("Agent has queued work"); + sessionInternals._compactionAbortController = undefined; + + expect(harness.session.getAutonomousStatus().enabled).toBe(false); + expect(harness.session.getFollowUpMessages()).toEqual([]); + expect(harness.getPendingResponseCount()).toBe(0); + }); + + it("keeps built-in slash commands literal for accepted agent messages", async () => { + const harness = await createHarness(); + harnesses.push(harness); + harness.setResponses([fauxAssistantMessage("literal")]); + + await harness.session.acceptAgentMessagePrompt("/autonomous on", { expandPromptTemplates: false }); + await harness.session.agent.waitForIdle(); + + expect(harness.session.getAutonomousStatus().enabled).toBe(false); + expect(getUserTexts(harness)).toEqual(["/autonomous on"]); + expect(getAssistantTexts(harness)).toEqual(["literal"]); + }); + it("queues accepted agent messages without expanding slash commands or prompt templates", async () => { const template: PromptTemplate = { name: "review", diff --git a/packages/coding-agent/test/suite/agent-session-queue.test.ts b/packages/coding-agent/test/suite/agent-session-queue.test.ts index 7a204020a94..efe0561d154 100644 --- a/packages/coding-agent/test/suite/agent-session-queue.test.ts +++ b/packages/coding-agent/test/suite/agent-session-queue.test.ts @@ -118,6 +118,19 @@ describe("AgentSession queue characterization", () => { } }); + it("does not count failed assistant messages toward the auto-refine interval", async () => { + const harness = await createAutoRefineHarness({ + settings: { autoRefine: { enabled: true, turnInterval: 1, cooldownMs: 0 } }, + }); + harnesses.push(harness); + const internals = harness.session as unknown as AutoRefineInternals; + harness.setResponses([fauxAssistantMessage("failed", { stopReason: "error", errorMessage: "provider failed" })]); + + await harness.session.prompt("fail once"); + + expect(internals._assistantTurnsSinceAutoRefine).toBe(0); + }); + it("auto-refine review runs after the configured turn interval", async () => { const reviewer = vi.fn(async () => ({ shouldRefine: true, diff --git a/packages/coding-agent/test/suite/harness.ts b/packages/coding-agent/test/suite/harness.ts index e7eef404ebf..8a6e63152f6 100644 --- a/packages/coding-agent/test/suite/harness.ts +++ b/packages/coding-agent/test/suite/harness.ts @@ -12,6 +12,7 @@ import { registerFauxProvider } from "@earendil-works/pi-ai"; import type { AgentObserveController } from "../../src/core/agent-observe.js"; import { AgentSession, type AgentSessionEvent, type AutoRefineReviewer } from "../../src/core/agent-session.js"; import { AuthStorage } from "../../src/core/auth-storage.js"; +import type { AgentAutonomousConfig } from "../../src/core/autonomous.js"; import type { ExtensionRunner } from "../../src/core/extensions/index.js"; import { convertToLlm } from "../../src/core/messages.js"; import { ModelRegistry } from "../../src/core/model-registry.js"; @@ -68,6 +69,7 @@ export interface HarnessOptions { agentObserveController?: AgentObserveController; persistSession?: boolean; rlmDepth?: number; + autonomous?: AgentAutonomousConfig; autoRefineReviewer?: AutoRefineReviewer; } @@ -185,6 +187,7 @@ export async function createHarness(options: HarnessOptions = {}): Promise