Autonomous Eval

packages/coding-agent/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Added autonomous mode with host-side continuations, configurable limits, and quality gates for evaluator-controlled runs ([#278](https://github.com/PrimeIntellect-ai/prime-agent/pull/278) by [@sethkarten](https://github.com/sethkarten)).
 - Changed agent-to-agent messages to render as directional rows, with received messages expandable in chat and sent messages shown below their Python cell ([ENG-4531](https://linear.app/primeintellect/issue/ENG-4531/collapse-and-simplify-agent2agent-messages-in-chat-tui)).
 - Fixed IPython state restore notices rendering as full user messages when prompts were queued or restored ([ENG-4530](https://linear.app/primeintellect/issue/ENG-4530/collapse-ipython-state-restore-messages-in-chat-tui)).
 - Changed bare `/mcp` to open the Services menu while preserving explicit `list`, `login`, and `logout` subcommands ([ENG-4535](https://linear.app/primeintellect/issue/ENG-4535/open-services-mcp-menu-from-mcp)).
@@ -29,6 +30,7 @@
 - Changed `/model` to sort featured flagship models above a provider's long tail (with a numeric-aware alphabetical tiebreak), so the full Prime Inference catalog doesn't flood the picker.
 - Fixed selector prompts and choices filling their background through the terminal's right edge.
 - Changed automatic harness refinement to be enabled by default while keeping `autoRefine.enabled: false` as the opt-out.
+- Changed the default thinking level to `xhigh` for models that support it ([#278](https://github.com/PrimeIntellect-ai/prime-agent/pull/278) by [@sethkarten](https://github.com/sethkarten)).
 - Fixed non-numeric `autoRefine.turnInterval` and `autoRefine.cooldownMs` settings falling back to defaults instead of silently enabling a noisy auto-refine loop.
 - Fixed all session-resume entry points to share a searchable full-screen picker, stream results while loading, and support renaming ([ENG-4513](https://linear.app/primeintellect/issue/ENG-4513/resume-in-agents-view-is-broken)).
 
@@ -48,6 +50,7 @@
 - Fixed provider errors being surfaced instead of retried within the retry budget ([ENG-4503](https://linear.app/primeintellect/issue/ENG-4503/restarting-old-session-returns-empty-model-response)).
 - Fixed Agents View returning from fullscreen sessions without flashing primary scrollback ([ENG-4508](https://linear.app/primeintellect/issue/ENG-4508/fullscreen-mode-agents-view-scroll)).
 
+
 ## [0.2.6] - 2026-07-06
 
 - Fixed the installer splash flickering during animation and resize by stabilizing full-screen redraws and removing misleading synthetic percentages ([ENG-4481](https://linear.app/primeintellect/issue/ENG-4481/installer-screen-is-unstable-and-flickery)).

packages/coding-agent/docs/settings.md

@@ -17,7 +17,7 @@ Edit directly or use `/settings` for common options.
 |---------|------|---------|-------------|
 | `defaultProvider` | string | - | Default provider (e.g., `"anthropic"`, `"openai"`) |
 | `defaultModel` | string | - | Default model ID |
-| `defaultThinkingLevel` | string | - | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"` |
+| `defaultThinkingLevel` | string | `"xhigh"` | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"` |
 | `hideThinkingBlock` | boolean | `false` | Hide thinking blocks in output |
 | `thinkingBudgets` | object | - | Custom token budgets per thinking level |
 
@@ -258,7 +258,7 @@ See [packages.md](packages.md) for package management details.
 {
   "defaultProvider": "anthropic",
   "defaultModel": "claude-sonnet-4-20250514",
-  "defaultThinkingLevel": "medium",
+  "defaultThinkingLevel": "xhigh",
   "theme": "dark",
   "compaction": {
     "enabled": true,

packages/coding-agent/skills/agent-observe/SKILL.md

@@ -26,9 +26,9 @@ recent = await agent_observe.recent_messages("worker", limit=6)
   message preview. This includes subagents shown in the agents view; filter
   by `runtimeKind == "subagent"` and `parentSessionId` or
   `parentActiveSessionId` to inspect subagents that belong to your session.
-- `await agent_observe.get_agent(target)` returns one agent summary. `target`
-  is resolved like other live-session selectors: active id, session id/name, or
-  unambiguous suffix.
+- `await agent_observe.get_agent(target)` returns `agent`, where `agent`
+  contains one agent summary. `target` is resolved like other live-session
+  selectors: active id, session id/name, or unambiguous suffix.
 - `await agent_observe.recent_messages(target, limit=8, max_chars=800)`
   returns up to `limit` recent bounded message previews for the target session.
   `limit` must be 1-50, and `max_chars` must be 80-2000.

packages/coding-agent/skills/orchestration-heartbeat/src/orchestration_heartbeat/__init__.py

@@ -24,14 +24,23 @@ def _first_value(agent: dict[str, Any], *keys: str) -> Any:
     return None
 
 
+def _safe_field(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value)
+    sanitized = "".join(" " if ord(char) < 32 or ord(char) == 127 else char for char in text)
+    sanitized = " ".join(sanitized.split())
+    return sanitized or None
+
+
 def _session_label(agent: dict[str, Any]) -> str:
-    name = _first_value(agent, "sessionName", "name")
-    session_id = _first_value(agent, "sessionId", "session_id")
-    active_session_id = _first_value(agent, "activeSessionId", "active_session_id")
-    cwd = agent.get("cwd")
-    status = agent.get("status")
-    streaming = _first_value(agent, "isStreaming", "streaming")
-    pending = _first_value(agent, "pendingMessageCount", "pending_message_count")
+    name = _safe_field(_first_value(agent, "sessionName", "name"))
+    session_id = _safe_field(_first_value(agent, "sessionId", "session_id"))
+    active_session_id = _safe_field(_first_value(agent, "activeSessionId", "active_session_id"))
+    cwd = _safe_field(agent.get("cwd"))
+    status = _safe_field(agent.get("status"))
+    streaming = _safe_field(_first_value(agent, "isStreaming", "streaming"))
+    pending = _safe_field(_first_value(agent, "pendingMessageCount", "pending_message_count"))
     parts = [
         f"name={name}" if name else None,
         f"session_id={session_id}" if session_id else None,

packages/coding-agent/src/cli/args.ts

@@ -42,6 +42,14 @@ export interface Args {
 	themes?: string[];
 	noThemes?: boolean;
 	noContextFiles?: boolean;
+	autonomous?: boolean;
+	autonomousGates?: string[];
+	autonomousGateRetries?: number;
+	autonomousGateTimeoutMs?: number;
+	autonomousMaxContinuations?: number;
+	autonomousMaxTurns?: number;
+	autonomousMaxTokens?: number;
+	autonomousTimeoutMs?: number;
 	listModels?: string | true;
 	offline?: boolean;
 	verbose?: boolean;
@@ -197,6 +205,44 @@ export function parseArgs(args: string[]): Args {
 			result.noThemes = true;
 		} else if (arg === "--no-context-files" || arg === "-nc") {
 			result.noContextFiles = true;
+		} else if (arg === "--autonomous") {
+			result.autonomous = true;
+		} else if (arg === "--autonomous-gate") {
+			result.autonomous = true;
+			if (hasRequiredOptionValue(args, i, arg, result)) {
+				result.autonomousGates = result.autonomousGates ?? [];
+				result.autonomousGates.push(args[++i]);
+			}
+		} else if (arg === "--autonomous-gate-retries") {
+			result.autonomous = true;
+			if (hasRequiredOptionValue(args, i, arg, result)) {
+				result.autonomousGateRetries = parsePositiveInt(args[++i], "--autonomous-gate-retries", result);
+			}
+		} else if (arg === "--autonomous-gate-timeout-ms") {
+			result.autonomous = true;
+			if (hasRequiredOptionValue(args, i, arg, result)) {
+				result.autonomousGateTimeoutMs = parsePositiveInt(args[++i], "--autonomous-gate-timeout-ms", result);
+			}
+		} else if (arg === "--autonomous-max-continuations") {
+			result.autonomous = true;
+			if (hasRequiredOptionValue(args, i, arg, result)) {
+				result.autonomousMaxContinuations = parsePositiveInt(args[++i], "--autonomous-max-continuations", result);
+			}
+		} else if (arg === "--autonomous-max-turns") {
+			result.autonomous = true;
+			if (hasRequiredOptionValue(args, i, arg, result)) {
+				result.autonomousMaxTurns = parsePositiveInt(args[++i], "--autonomous-max-turns", result);
+			}
+		} else if (arg === "--autonomous-max-tokens") {
+			result.autonomous = true;
+			if (hasRequiredOptionValue(args, i, arg, result)) {
+				result.autonomousMaxTokens = parsePositiveInt(args[++i], "--autonomous-max-tokens", result);
+			}
+		} else if (arg === "--autonomous-timeout-ms") {
+			result.autonomous = true;
+			if (hasRequiredOptionValue(args, i, arg, result)) {
+				result.autonomousTimeoutMs = parsePositiveInt(args[++i], "--autonomous-timeout-ms", result);
+			}
 		} else if (arg === "--list-models") {
 			// Check if next arg is a search pattern (not a flag or file arg)
 			if (i + 1 < args.length && !args[i + 1].startsWith("-") && !args[i + 1].startsWith("@")) {
@@ -292,6 +338,14 @@ ${chalk.bold("Options:")}
   --theme <path>                 Load a theme file or directory (can be used multiple times)
   --no-themes                    Disable theme discovery and loading
   --no-context-files, -nc        Disable AGENTS.md and CLAUDE.md discovery and loading
+  --autonomous                   Continue autonomously until host-observable terminal evidence exists
+  --autonomous-gate <command>    Run a command before autonomous mode may finish (repeatable)
+  --autonomous-gate-retries <n>  Max autonomous retries per failed gate (default: 3)
+  --autonomous-gate-timeout-ms <n> Timeout per autonomous gate command in milliseconds
+  --autonomous-max-continuations <n> Max autonomous follow-up messages (default: 3)
+  --autonomous-max-turns <n>     Max assistant turns while autonomous mode is active (default: 12)
+  --autonomous-max-tokens <n>    Max tokens while autonomous mode is active (default: 80000)
+  --autonomous-timeout-ms <n>    Max autonomous wall-clock time in milliseconds (default: 1800000)
   --export <file>                Export session file to HTML and exit
   --list-models [search]         List available models (with optional fuzzy search)
   --verbose                      Force verbose startup (overrides quietStartup setting)
@@ -404,3 +458,21 @@ ${chalk.bold("Built-in Tool Names:")}
   ipython - Execute Python in a persistent IPython kernel
 `);
 }
+
+function hasRequiredOptionValue(args: string[], index: number, flag: string, result: Args): boolean {
+	const next = args[index + 1];
+	if (next === undefined || next.startsWith("--")) {
+		result.diagnostics.push({ type: "error", message: `${flag} requires a value` });
+		return false;
+	}
+	return true;
+}
+
+function parsePositiveInt(value: string, flag: string, result: Args): number | undefined {
+	const parsed = Number(value);
+	if (!Number.isInteger(parsed) || parsed <= 0) {
+		result.diagnostics.push({ type: "error", message: `${flag} must be a positive integer` });
+		return undefined;
+	}
+	return parsed;
+}

packages/coding-agent/src/core/agent-session-config.ts

@@ -1,4 +1,5 @@
 import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
+import type { AgentAutonomousConfig } from "./autonomous.js";
 
 export interface AgentSessionRuntimeConfig {
 	cwd?: string;
@@ -23,6 +24,7 @@ export interface AgentSessionRuntimeConfig {
 	themes?: string[];
 	noThemes?: boolean;
 	noContextFiles?: boolean;
+	autonomous?: AgentAutonomousConfig;
 	extensionFlagValues?: Record<string, boolean | string>;
 }
 
@@ -56,6 +58,7 @@ export function mergeAgentSessionRuntimeConfig(
 		themes: cloneArray(override.themes ?? base.themes),
 		noThemes: override.noThemes ?? base.noThemes,
 		noContextFiles: override.noContextFiles ?? base.noContextFiles,
+		autonomous: mergeAutonomousConfig(base.autonomous, override.autonomous),
 		extensionFlagValues:
 			base.extensionFlagValues || override.extensionFlagValues
 				? { ...(base.extensionFlagValues ?? {}), ...(override.extensionFlagValues ?? {}) }
@@ -73,10 +76,42 @@ function cloneAgentSessionRuntimeConfig(config: AgentSessionRuntimeConfig): Agen
 		skills: cloneArray(config.skills),
 		promptTemplates: cloneArray(config.promptTemplates),
 		themes: cloneArray(config.themes),
+		autonomous: mergeAutonomousConfig(undefined, config.autonomous),
 		extensionFlagValues: config.extensionFlagValues ? { ...config.extensionFlagValues } : undefined,
 	};
 }
 
+export function mergeAutonomousConfig(
+	base: AgentAutonomousConfig | undefined,
+	override: AgentAutonomousConfig | undefined,
+): AgentAutonomousConfig | undefined {
+	if (!base && !override) {
+		return undefined;
+	}
+	const gates = mergeAutonomousGateConfig(base?.gates, override?.gates);
+	return {
+		...(base ?? {}),
+		...(override ?? {}),
+		...(gates ? { gates } : {}),
+	};
+}
+
+function mergeAutonomousGateConfig(
+	base: AgentAutonomousConfig["gates"] | undefined,
+	override: AgentAutonomousConfig["gates"] | undefined,
+): AgentAutonomousConfig["gates"] | undefined {
+	if (!base && !override) {
+		return undefined;
+	}
+	return {
+		...(base ?? {}),
+		...(override ?? {}),
+		...(override?.commands !== undefined || base?.commands !== undefined
+			? { commands: cloneArray(override?.commands ?? base?.commands) }
+			: {}),
+	};
+}
+
 function cloneArray<T>(value: T[] | undefined): T[] | undefined {
 	return value ? [...value] : undefined;
 }

packages/coding-agent/src/core/agent-session-services.ts

@@ -6,6 +6,7 @@ import type { AgentSessionMessageController } from "./agent-messages.js";
 import type { AgentObserveController } from "./agent-observe.js";
 import { installAgentTraceUpload } from "./agent-traces.js";
 import { AuthStorage } from "./auth-storage.js";
+import type { AgentAutonomousConfig } from "./autonomous.js";
 import type { AgentRlmHeartbeatController } from "./cron-jobs.js";
 import { createHerdrAgentStateExtension } from "./extensions/builtin/herdr-agent-state.js";
 import type { SessionStartEvent, ToolDefinition } from "./extensions/index.js";
@@ -73,6 +74,7 @@ export interface AgentSessionCreationOptions {
 	subagentRuntimeHost?: SubagentRuntimeHost;
 	rlmHeartbeatController?: AgentRlmHeartbeatController;
 	prewarmIpythonKernel?: boolean;
+	autonomous?: AgentAutonomousConfig;
 }
 
 /**
@@ -268,5 +270,6 @@ export async function createAgentSessionFromServices(
 		rlmHeartbeatController: options.rlmHeartbeatController,
 		sessionStartEvent: options.sessionStartEvent,
 		prewarmIpythonKernel: options.prewarmIpythonKernel,
+		autonomous: options.autonomous,
 	});
 }