diff --git a/packages/coding-agent/docs/subagents.md b/packages/coding-agent/docs/subagents.md index 7c91d6984..deadecb35 100644 --- a/packages/coding-agent/docs/subagents.md +++ b/packages/coding-agent/docs/subagents.md @@ -150,6 +150,36 @@ Agents can define ordered `fallbackModels` for retryable provider or model failu Fallbacks do not retry ordinary task failures, validation failures, tool failures, cancellations, or workflow-code errors. Because a fallback may send the same prompt and context to a different provider, choose models that match your cost, privacy, and data-handling requirements. +Each candidate can also carry its own reasoning effort — see [Reasoning levels](#reasoning-levels). + +## Reasoning levels + +Set the reasoning (thinking) effort for each model candidate with a `model_name:thinking_effort` suffix on `model` and on every `fallbackModels` entry. Valid efforts are `off`, `minimal`, `low`, `medium`, `high`, and `xhigh` — the same shorthand used by `atomic --model sonnet:high`. + +```markdown +--- +name: deep-reviewer +description: Adversarial reviewer for risky diffs +tools: read, grep, bash +model: anthropic/claude-sonnet-4:high +fallbackModels: openai/gpt-5:medium, anthropic/claude-haiku-4-5:off +--- +``` + +Because the effort travels with each model string, every primary and fallback candidate is self-contained: a fallback can run at a different effort than the primary, so a high-effort primary degrades gracefully to a cheaper, lower-effort fallback. + +**Migrate off the legacy `thinking` field.** The separate `thinking:` frontmatter field is deprecated. It still works as a default for any candidate that has no suffix, and a suffix always wins, but new agents should encode the effort directly on `model` and `fallbackModels`: + +```diff +-model: openai/gpt-5.5 +-fallbackModels: anthropic/claude-opus-4-8 +-thinking: xhigh ++model: openai/gpt-5.5:xhigh ++fallbackModels: anthropic/claude-opus-4-8:xhigh +``` + +`fallbackThinkingLevels` exists only as an optional compatibility helper: it is aligned by index to `fallbackModels` and supplies a fallback candidate's effort only when that fallback entry has no suffix. Prefer suffixed model strings instead. Attempt metadata reports the resolved model and the effective reasoning effort used for each attempt. + ## Related docs - [Workflows](/workflows) for multi-stage reusable automation. diff --git a/packages/coding-agent/docs/workflows.md b/packages/coding-agent/docs/workflows.md index 07f1c3291..e45f8071c 100644 --- a/packages/coding-agent/docs/workflows.md +++ b/packages/coding-agent/docs/workflows.md @@ -1181,7 +1181,7 @@ Common task/stage options include: - `prompt` or `task` - `previous` for small handoff context; use artifact paths plus `reads` for large outputs, logs, research bundles, or reviewer payloads - `context: "fresh" | "fork"`, `forkFromSessionFile` -- `model`, `fallbackModels`, `thinkingLevel`, `scopedModels`, `modelRegistry` +- `model`, `fallbackModels`, `thinkingLevel`, `scopedModels`, `modelRegistry` — `model` and each `fallbackModels` entry accept a `model_name:thinking_effort` reasoning suffix; the standalone `thinkingLevel` is deprecated (see [Reasoning levels](#reasoning-levels)) - `tools`, `noTools`, `customTools`, `mcp: { allow?: string[], deny?: string[] }` - `output`, `outputMode`, `reads`, `worktree`, `gitWorktreeDir`, `baseBranch`, `maxOutput`, `artifacts`, `sessionDir`, `cwd`, `agentDir` - advanced host-supplied SDK seams: `authStorage`, `resourceLoader`, `sessionManager`, `settingsManager`, `sessionStartEvent` @@ -1208,6 +1208,30 @@ For lower-level integrations, `@bastani/workflows` also exports `setupGitWorktre `fallbackModels` retries transient provider/model failures with the primary `model` first, then each fallback, then the current Atomic-selected model when available. It is for rate limits, quota/auth/provider outages, unavailable models, network timeouts, and 5xx errors — not workflow-code errors, tool failures, validation failures, or cancellations. +### Reasoning levels + +Each `model` and `fallbackModels` entry accepts a `model_name:thinking_effort` suffix that sets the reasoning effort for that candidate (`off`, `minimal`, `low`, `medium`, `high`, `xhigh`). The effort travels with the model string, so a single fallback chain can mix efforts — for example a high-effort primary that degrades to lower-effort, cheaper fallbacks: + +```ts +await ctx.task("review", { + task: "Review the diff", + model: "anthropic/claude-sonnet-4:high", + fallbackModels: ["openai/gpt-5:medium", "anthropic/claude-haiku-4-5:off"], +}); +``` + +The standalone `thinkingLevel` stage option is deprecated. It still applies as a default to any candidate without a suffix, and when both are present the suffix wins, but new workflows should fold the effort into the model strings: + +```diff +- model: "openai/gpt-5.5", +- fallbackModels: ["anthropic/claude-opus-4-8"], +- thinkingLevel: "high", ++ model: "openai/gpt-5.5:high", ++ fallbackModels: ["anthropic/claude-opus-4-8:high"], +``` + +This applies everywhere a stage accepts a model: direct `ctx.task`/`ctx.chain`/`ctx.parallel` options, `ctx.stage` options, builtin workflow stage definitions, and workflow parameters. `fallbackThinkingLevels` is an optional compatibility helper aligned by index to `fallbackModels`; it applies only to fallback entries that do not already carry a suffix. Each `WorkflowModelAttempt` reports the resolved model and the effective reasoning effort used for that attempt. + ## Programmatic Usage `@bastani/workflows` is an Atomic package extension. It registers: diff --git a/packages/subagents/CHANGELOG.md b/packages/subagents/CHANGELOG.md index 7bc5ba1f4..38c205dbd 100644 --- a/packages/subagents/CHANGELOG.md +++ b/packages/subagents/CHANGELOG.md @@ -2,6 +2,24 @@ ## [Unreleased] +### Added + +- Added suffix-first reasoning levels for subagent `model` and `fallbackModels` values plus `fallbackThinkingLevels` compatibility metadata and per-attempt `reasoningLevel` reporting ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). + +### Changed + +- Migrated packaged subagents to encode their reasoning level directly in model and fallback model entries ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). +- Documented the `model_name:thinking_effort` suffix syntax and `thinking` migration guidance in the subagents docs, package README, and subagent skill ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). + +### Deprecated + +- Deprecated separate subagent `thinking` configuration in favor of `model: :` and suffixed `fallbackModels` entries; removal is deferred to a later breaking release ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). + +### Fixed + +- Fixed foreground subagent attempt metadata to report the per-candidate reasoning level derived from the model suffix even when the legacy `thinking` option is unset, matching the background run path ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). +- Fixed the subagent chain clarification TUI to strip only canonical reasoning-level suffixes when editing a step's level, so colon-tagged model ids such as `ollama/llama3:latest` are no longer mis-split; consolidated the duplicate thinking-suffix split helpers onto the single `splitKnownThinkingSuffix` so the parsing rules cannot drift apart ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). + ## [0.8.23] - 2026-06-02 ### Changed diff --git a/packages/subagents/README.md b/packages/subagents/README.md index 3a0ab67bc..346bcb593 100644 --- a/packages/subagents/README.md +++ b/packages/subagents/README.md @@ -996,3 +996,19 @@ The main runtime files are: | `src/intercom/intercom-bridge.ts` | Runtime intercom bridge instructions and diagnostics. | | `src/extension/schemas.ts` / `src/shared/types.ts` | Tool schemas, shared types, and event constants. | | `test/unit/` / `test/integration/` | Unit and loader-based integration tests. | + +### Suffix-first reasoning levels + +Reasoning levels are configured suffix-first using the `model_name:thinking_effort` syntax on `model` and each `fallbackModels` entry: `model: claude-sonnet-4:high` and `fallbackModels: claude-sonnet-4:medium, gpt-5:low, claude-haiku-4:off`. Canonical efforts are `off`, `minimal`, `low`, `medium`, `high`, and `xhigh`. The older `thinking` field is deprecated; it remains supported as a legacy default only when a model candidate has no suffix, and a suffix always wins. + +Migrate legacy `thinking` frontmatter by folding the effort into `model` and `fallbackModels`: + +```diff +-model: openai/gpt-5.5 +-fallbackModels: anthropic/claude-opus-4-8 +-thinking: xhigh ++model: openai/gpt-5.5:xhigh ++fallbackModels: anthropic/claude-opus-4-8:xhigh +``` + +`fallbackThinkingLevels` is available only as an optional compatibility helper. It is positionally aligned with `fallbackModels` and supplies a fallback candidate's level only when that fallback model entry has no suffix; prefer suffixed model strings for new configuration. diff --git a/packages/subagents/agents/code-simplifier.md b/packages/subagents/agents/code-simplifier.md index b3596c4b9..2c5e49a59 100644 --- a/packages/subagents/agents/code-simplifier.md +++ b/packages/subagents/agents/code-simplifier.md @@ -8,9 +8,8 @@ description: | - Production-quality refinement of a working draft ("ugly but working CSV parser"). - Code that has gotten messy after several iterations. tools: read, edit, write, grep, find, ls, bash -model: openai/gpt-5.5 -fallbackModels: openai-codex/gpt-5.5, github-copilot/gpt-5.5, anthropic/claude-opus-4-8, github-copilot/claude-opus-4.7 -thinking: low +model: openai/gpt-5.5:low +fallbackModels: openai-codex/gpt-5.5:low, github-copilot/gpt-5.5:low, anthropic/claude-opus-4-8:low, github-copilot/claude-opus-4.7:low --- You are an expert code refinement specialist with deep experience in software craftsmanship, refactoring patterns (Fowler, Beck), clean code principles, and language-idiomatic style across major ecosystems. Your mission is to simplify and refine code for clarity, consistency, and maintainability while strictly preserving all existing functionality and observable behavior. diff --git a/packages/subagents/agents/codebase-analyzer.md b/packages/subagents/agents/codebase-analyzer.md index 4d8f81bdf..03bffd220 100644 --- a/packages/subagents/agents/codebase-analyzer.md +++ b/packages/subagents/agents/codebase-analyzer.md @@ -2,9 +2,8 @@ name: codebase-analyzer description: Analyzes codebase implementation details. Call the codebase-analyzer agent when you need to find detailed information about specific components. tools: read, grep, find, ls, bash -model: openai/gpt-5.5 -fallbackModels: openai-codex/gpt-5.5, github-copilot/gpt-5.5, anthropic/claude-opus-4-8, github-copilot/claude-opus-4.7 -thinking: low +model: openai/gpt-5.5:low +fallbackModels: openai-codex/gpt-5.5:low, github-copilot/gpt-5.5:low, anthropic/claude-opus-4-8:low, github-copilot/claude-opus-4.7:low --- You are a specialist at understanding HOW code works. Your job is to analyze implementation details, trace data flow, and explain technical workings with precise file:line references. diff --git a/packages/subagents/agents/codebase-locator.md b/packages/subagents/agents/codebase-locator.md index 28ad21847..88d72a9e6 100644 --- a/packages/subagents/agents/codebase-locator.md +++ b/packages/subagents/agents/codebase-locator.md @@ -2,9 +2,8 @@ name: codebase-locator description: Locates files, directories, and components relevant to a feature or task. Basically a "super search/find/ls tool." tools: read, grep, find, ls, bash -model: openai/gpt-5.4-mini -fallbackModels: openai-codex/gpt-5.4-mini, github-copilot/gpt-5.4-mini, anthropic/claude-haiku-4-5, github-copilot/claude-haiku-4.5 -thinking: low +model: openai/gpt-5.4-mini:low +fallbackModels: openai-codex/gpt-5.4-mini:low, github-copilot/gpt-5.4-mini:low, anthropic/claude-haiku-4-5:low, github-copilot/claude-haiku-4.5:low --- You are a specialist at finding WHERE code lives in a codebase. Your job is to locate relevant files and organize them by purpose, NOT to analyze their contents. diff --git a/packages/subagents/agents/codebase-online-researcher.md b/packages/subagents/agents/codebase-online-researcher.md index 5addbac55..f2dac467f 100644 --- a/packages/subagents/agents/codebase-online-researcher.md +++ b/packages/subagents/agents/codebase-online-researcher.md @@ -2,9 +2,8 @@ name: codebase-online-researcher description: Online research for up-to-date documentation and library-source knowledge. Use when you need authoritative external information — official docs, ecosystem context, version-specific behavior, GitHub permalinks into open-source libraries, or video tutorials. tools: read, grep, find, ls, bash, write, web_search, fetch_content, get_search_content -model: openai/gpt-5.5 -fallbackModels: openai-codex/gpt-5.5, github-copilot/gpt-5.5, anthropic/claude-opus-4-8, github-copilot/claude-opus-4.7 -thinking: low +model: openai/gpt-5.5:low +fallbackModels: openai-codex/gpt-5.5:low, github-copilot/gpt-5.5:low, anthropic/claude-opus-4-8:low, github-copilot/claude-opus-4.7:low skills: browser-use --- diff --git a/packages/subagents/agents/codebase-pattern-finder.md b/packages/subagents/agents/codebase-pattern-finder.md index 125f20052..e7b481697 100644 --- a/packages/subagents/agents/codebase-pattern-finder.md +++ b/packages/subagents/agents/codebase-pattern-finder.md @@ -2,9 +2,8 @@ name: codebase-pattern-finder description: Find similar implementations, usage examples, or existing patterns in the codebase that can be modeled after. tools: read, grep, find, ls, bash -model: openai/gpt-5.4-mini -fallbackModels: openai-codex/gpt-5.4-mini, github-copilot/gpt-5.4-mini, anthropic/claude-haiku-4-5, github-copilot/claude-haiku-4.5 -thinking: low +model: openai/gpt-5.4-mini:low +fallbackModels: openai-codex/gpt-5.4-mini:low, github-copilot/gpt-5.4-mini:low, anthropic/claude-haiku-4-5:low, github-copilot/claude-haiku-4.5:low --- You are a specialist at finding code patterns and examples in the codebase. Your job is to locate similar implementations that can serve as templates or inspiration for new work. diff --git a/packages/subagents/agents/codebase-research-analyzer.md b/packages/subagents/agents/codebase-research-analyzer.md index e0f676a06..5c332fd99 100644 --- a/packages/subagents/agents/codebase-research-analyzer.md +++ b/packages/subagents/agents/codebase-research-analyzer.md @@ -2,9 +2,8 @@ name: codebase-research-analyzer description: Analyzes local research documents to extract high-value insights, decisions, and technical details while filtering out noise. Use this when you want to deep dive on a research topic or understand the rationale behind decisions. tools: read, grep, find, ls, bash -model: openai/gpt-5.5 -fallbackModels: openai-codex/gpt-5.5, github-copilot/gpt-5.5, anthropic/claude-opus-4-8, github-copilot/claude-opus-4.7 -thinking: low +model: openai/gpt-5.5:low +fallbackModels: openai-codex/gpt-5.5:low, github-copilot/gpt-5.5:low, anthropic/claude-opus-4-8:low, github-copilot/claude-opus-4.7:low --- You are a specialist at extracting HIGH-VALUE insights from research documents. Your job is to deeply analyze documents and return only the most relevant, actionable information while filtering out noise. diff --git a/packages/subagents/agents/codebase-research-locator.md b/packages/subagents/agents/codebase-research-locator.md index be7bfb356..108ccac30 100644 --- a/packages/subagents/agents/codebase-research-locator.md +++ b/packages/subagents/agents/codebase-research-locator.md @@ -2,9 +2,8 @@ name: codebase-research-locator description: Discovers local research documents that are relevant to the current research task. tools: read, grep, find, ls, bash -model: openai/gpt-5.4-mini -fallbackModels: openai-codex/gpt-5.4-mini, github-copilot/gpt-5.4-mini, anthropic/claude-haiku-4-5, github-copilot/claude-haiku-4.5 -thinking: low +model: openai/gpt-5.4-mini:low +fallbackModels: openai-codex/gpt-5.4-mini:low, github-copilot/gpt-5.4-mini:low, anthropic/claude-haiku-4-5:low, github-copilot/claude-haiku-4.5:low --- You are a specialist at finding documents in the `research/` directory. Your job is to locate relevant research documents and categorize them, NOT to analyze their contents in depth. diff --git a/packages/subagents/agents/debugger.md b/packages/subagents/agents/debugger.md index 83da79bb1..a07bfacad 100644 --- a/packages/subagents/agents/debugger.md +++ b/packages/subagents/agents/debugger.md @@ -2,9 +2,8 @@ name: debugger description: Debug errors, test failures, and unexpected behavior. Use PROACTIVELY when encountering issues, analyzing stack traces, or investigating system problems. tools: read, edit, write, grep, find, ls, bash, web_search, fetch_content, get_search_content -model: openai/gpt-5.5 -fallbackModels: openai-codex/gpt-5.5, github-copilot/gpt-5.5, anthropic/claude-opus-4-8, github-copilot/claude-opus-4.7 -thinking: xhigh +model: openai/gpt-5.5:xhigh +fallbackModels: openai-codex/gpt-5.5:xhigh, github-copilot/gpt-5.5:xhigh, anthropic/claude-opus-4-8:xhigh, github-copilot/claude-opus-4.7:xhigh skills: tdd, browser-use, tmux --- diff --git a/packages/subagents/skills/subagent/SKILL.md b/packages/subagents/skills/subagent/SKILL.md index d366f2a3c..3cefb6ea8 100644 --- a/packages/subagents/skills/subagent/SKILL.md +++ b/packages/subagents/skills/subagent/SKILL.md @@ -732,3 +732,9 @@ subagent({ action: "doctor" }) ```typescript // Inspect `subagent({ action: "status", id: "..." })`, artifact metadata/output logs, and run doctor. Extension loader errors usually appear in child output logs. ``` + +## Suffix-first reasoning levels + +Prefer encoding reasoning levels directly in model strings with the `model_name:thinking_effort` syntax: `model: claude-sonnet-4:high` and `fallbackModels: [claude-sonnet-4:medium, gpt-5:low, claude-haiku-4:off]`. Valid efforts are `off`, `minimal`, `low`, `medium`, `high`, and `xhigh`. The separate `thinking` field is deprecated but still works as a legacy default when a candidate has no suffix; suffixes take precedence. If you see a legacy `thinking` override, migrate it by appending the effort to `model` and each `fallbackModels` entry instead (e.g. `thinking: high` + `model: gpt-5` → `model: gpt-5:high`). + +`fallbackThinkingLevels` is an optional compatibility helper aligned positionally with `fallbackModels`. It only applies to fallback entries without their own suffix and should not be preferred over suffix-first entries. diff --git a/packages/subagents/src/agents/agent-serializer.ts b/packages/subagents/src/agents/agent-serializer.ts index 42cc778fc..f5f9e4e5c 100644 --- a/packages/subagents/src/agents/agent-serializer.ts +++ b/packages/subagents/src/agents/agent-serializer.ts @@ -8,6 +8,7 @@ export const KNOWN_FIELDS = new Set([ "tools", "model", "fallbackModels", + "fallbackThinkingLevels", "thinking", "systemPromptMode", "inheritProjectContext", @@ -46,6 +47,8 @@ export function serializeAgent(config: AgentConfig): string { if (config.model) lines.push(`model: ${config.model}`); const fallbackModelsValue = joinComma(config.fallbackModels); if (fallbackModelsValue) lines.push(`fallbackModels: ${fallbackModelsValue}`); + const fallbackThinkingLevelsValue = joinComma(config.fallbackThinkingLevels); + if (fallbackThinkingLevelsValue) lines.push(`fallbackThinkingLevels: ${fallbackThinkingLevelsValue}`); if (config.thinking && config.thinking !== "off") lines.push(`thinking: ${config.thinking}`); lines.push(`systemPromptMode: ${config.systemPromptMode}`); lines.push(`inheritProjectContext: ${config.inheritProjectContext ? "true" : "false"}`); diff --git a/packages/subagents/src/agents/agents.ts b/packages/subagents/src/agents/agents.ts index 5682df5b9..c99ac6e5d 100644 --- a/packages/subagents/src/agents/agents.ts +++ b/packages/subagents/src/agents/agents.ts @@ -36,6 +36,7 @@ export function defaultInheritSkills(): boolean { export interface BuiltinAgentOverrideBase { model?: string; fallbackModels?: string[]; + fallbackThinkingLevels?: string[]; thinking?: string; systemPromptMode: SystemPromptMode; inheritProjectContext: boolean; @@ -52,6 +53,7 @@ export interface BuiltinAgentOverrideBase { interface BuiltinAgentOverrideConfig { model?: string | false; fallbackModels?: string[] | false; + fallbackThinkingLevels?: string[] | false; thinking?: string | false; systemPromptMode?: SystemPromptMode; inheritProjectContext?: boolean; @@ -79,6 +81,7 @@ export interface AgentConfig { mcpDirectTools?: string[]; model?: string; fallbackModels?: string[]; + fallbackThinkingLevels?: string[]; thinking?: string; systemPromptMode: SystemPromptMode; inheritProjectContext: boolean; @@ -184,6 +187,7 @@ function cloneOverrideBase(agent: AgentConfig): BuiltinAgentOverrideBase { return { model: agent.model, fallbackModels: agent.fallbackModels ? [...agent.fallbackModels] : undefined, + fallbackThinkingLevels: agent.fallbackThinkingLevels ? [...agent.fallbackThinkingLevels] : undefined, thinking: agent.thinking, systemPromptMode: agent.systemPromptMode, inheritProjectContext: agent.inheritProjectContext, @@ -204,6 +208,9 @@ function cloneOverrideValue(override: BuiltinAgentOverrideConfig): BuiltinAgentO ...(override.fallbackModels !== undefined ? { fallbackModels: override.fallbackModels === false ? false : [...override.fallbackModels] } : {}), + ...(override.fallbackThinkingLevels !== undefined + ? { fallbackThinkingLevels: override.fallbackThinkingLevels === false ? false : [...override.fallbackThinkingLevels] } + : {}), ...(override.thinking !== undefined ? { thinking: override.thinking } : {}), ...(override.systemPromptMode !== undefined ? { systemPromptMode: override.systemPromptMode } : {}), ...(override.inheritProjectContext !== undefined ? { inheritProjectContext: override.inheritProjectContext } : {}), @@ -375,6 +382,9 @@ function parseBuiltinOverrideEntry( const fallbackModels = parseOverrideStringArrayOrFalse(input.fallbackModels, { filePath, name, field: "fallbackModels" }); if (fallbackModels !== undefined) override.fallbackModels = fallbackModels; + const fallbackThinkingLevels = parseOverrideStringArrayOrFalse(input.fallbackThinkingLevels, { filePath, name, field: "fallbackThinkingLevels" }); + if (fallbackThinkingLevels !== undefined) override.fallbackThinkingLevels = fallbackThinkingLevels; + const skills = parseOverrideStringArrayOrFalse(input.skills, { filePath, name, field: "skills" }); if (skills !== undefined) override.skills = skills; @@ -442,6 +452,9 @@ function applyBuiltinOverride( if (override.fallbackModels !== undefined) { next.fallbackModels = override.fallbackModels === false ? undefined : [...override.fallbackModels]; } + if (override.fallbackThinkingLevels !== undefined) { + next.fallbackThinkingLevels = override.fallbackThinkingLevels === false ? undefined : [...override.fallbackThinkingLevels]; + } if (override.thinking !== undefined) next.thinking = override.thinking === false ? undefined : override.thinking; if (override.systemPromptMode !== undefined) next.systemPromptMode = override.systemPromptMode; if (override.inheritProjectContext !== undefined) next.inheritProjectContext = override.inheritProjectContext; @@ -495,12 +508,13 @@ function applyBuiltinOverrides( export function buildBuiltinOverrideConfig( base: BuiltinAgentOverrideBase, - draft: Pick, + draft: Pick, ): BuiltinAgentOverrideConfig | undefined { const override: BuiltinAgentOverrideConfig = {}; if (draft.model !== base.model) override.model = draft.model ?? false; if (!arraysEqual(draft.fallbackModels, base.fallbackModels)) override.fallbackModels = draft.fallbackModels ? [...draft.fallbackModels] : false; + if (!arraysEqual(draft.fallbackThinkingLevels, base.fallbackThinkingLevels)) override.fallbackThinkingLevels = draft.fallbackThinkingLevels ? [...draft.fallbackThinkingLevels] : false; if (draft.thinking !== base.thinking) override.thinking = draft.thinking ?? false; if (draft.systemPromptMode !== base.systemPromptMode) override.systemPromptMode = draft.systemPromptMode; if (draft.inheritProjectContext !== base.inheritProjectContext) override.inheritProjectContext = draft.inheritProjectContext; @@ -646,6 +660,10 @@ function loadAgentsFromDir(dir: string, source: AgentSource): AgentConfig[] { ?.split(",") .map((model) => model.trim()) .filter(Boolean); + const fallbackThinkingLevels = frontmatter.fallbackThinkingLevels + ?.split(",") + .map((level) => level.trim()) + .filter(Boolean); const systemPromptMode = frontmatter.systemPromptMode === "replace" ? "replace" : frontmatter.systemPromptMode === "append" @@ -696,6 +714,7 @@ function loadAgentsFromDir(dir: string, source: AgentSource): AgentConfig[] { mcpDirectTools: mcpDirectTools.length > 0 ? mcpDirectTools : undefined, model: frontmatter.model, fallbackModels: fallbackModels && fallbackModels.length > 0 ? fallbackModels : undefined, + fallbackThinkingLevels: fallbackThinkingLevels && fallbackThinkingLevels.length > 0 ? fallbackThinkingLevels : undefined, thinking: frontmatter.thinking, systemPromptMode, inheritProjectContext, diff --git a/packages/subagents/src/runs/background/async-execution.ts b/packages/subagents/src/runs/background/async-execution.ts index ba8ebc7f9..73bd39dc5 100644 --- a/packages/subagents/src/runs/background/async-execution.ts +++ b/packages/subagents/src/runs/background/async-execution.ts @@ -334,7 +334,7 @@ export function executeAsyncChain( const primaryModel = resolveModelCandidate(behavior.model ?? a.model, availableModels, ctx.currentModelProvider); const model = applyThinkingSuffix(primaryModel, a.thinking); - const modelCandidates = buildModelCandidates(behavior.model ?? a.model, a.fallbackModels, availableModels, ctx.currentModelProvider, ctx.currentModel) + const modelCandidates = buildModelCandidates(behavior.model ?? a.model, a.fallbackModels, availableModels, ctx.currentModelProvider, ctx.currentModel, a.fallbackThinkingLevels) .map((candidate) => applyThinkingSuffix(candidate, a.thinking)) .filter((candidate): candidate is string => typeof candidate === "string"); const fastModeSettings = getSubagentCodexFastModeSettings(stepCwd); diff --git a/packages/subagents/src/runs/background/subagent-runner.ts b/packages/subagents/src/runs/background/subagent-runner.ts index 9c8a47f4c..6893e2271 100644 --- a/packages/subagents/src/runs/background/subagent-runner.ts +++ b/packages/subagents/src/runs/background/subagent-runner.ts @@ -702,8 +702,10 @@ async function runSingleStep( ? `${hiddenError.errorType} failed (exit ${effectiveExitCode}): ${hiddenError.details}` : `${hiddenError.errorType} failed with exit code ${effectiveExitCode}` : run.error || (run.exitCode !== 0 && run.stderr.trim() ? run.stderr.trim() : undefined)); + const attemptModel = candidate ?? run.model ?? step.model ?? "default"; const attempt: ModelAttempt = { - model: candidate ?? run.model ?? step.model ?? "default", + model: attemptModel, + reasoningLevel: resolveEffectiveThinking(attemptModel, step.thinking), success: effectiveExitCode === 0 && !error, exitCode: effectiveExitCode, error, diff --git a/packages/subagents/src/runs/foreground/chain-clarify.ts b/packages/subagents/src/runs/foreground/chain-clarify.ts index 365cf1f61..361e62308 100644 --- a/packages/subagents/src/runs/foreground/chain-clarify.ts +++ b/packages/subagents/src/runs/foreground/chain-clarify.ts @@ -10,8 +10,8 @@ import type { Component, TUI } from "@earendil-works/pi-tui"; import { matchesKey, visibleWidth, truncateToWidth } from "@earendil-works/pi-tui"; import type { AgentConfig } from "../../agents/agents.ts"; import type { ResolvedStepBehavior } from "../../shared/settings.ts"; -import { resolveModelCandidate, splitThinkingSuffix } from "../shared/model-fallback.ts"; -import { findModelInfo, getSupportedThinkingLevels, type ModelInfo, type ThinkingLevel } from "../../shared/model-info.ts"; +import { resolveModelCandidate } from "../shared/model-fallback.ts"; +import { findModelInfo, getSupportedThinkingLevels, splitKnownThinkingSuffix, type ModelInfo, type ThinkingLevel } from "../../shared/model-info.ts"; type ClarifyMode = 'single' | 'parallel' | 'chain'; @@ -549,7 +549,7 @@ export class ChainClarifyComponent implements Component { this.modelSearchQuery = ""; this.modelSelectedIndex = 0; this.filteredModels = [...this.availableModels]; - const currentModel = splitThinkingSuffix(this.getEffectiveModel(this.selectedStep)).baseModel; + const currentModel = splitKnownThinkingSuffix(this.getEffectiveModel(this.selectedStep)).baseModel; const currentIndex = this.filteredModels.findIndex((m) => m.fullId === currentModel || m.id === currentModel); if (currentIndex >= 0) { this.modelSelectedIndex = currentIndex; @@ -582,7 +582,7 @@ export class ChainClarifyComponent implements Component { if (matchesKey(data, "return")) { const selected = this.filteredModels[this.modelSelectedIndex]; if (selected) { - const { thinkingSuffix } = splitThinkingSuffix(this.getEffectiveModel(this.editingStep!)); + const { thinkingSuffix } = splitKnownThinkingSuffix(this.getEffectiveModel(this.editingStep!)); const requestedLevel = thinkingSuffix.slice(1); const selectedModel = findModelInfo(selected.fullId, this.availableModels, this.preferredProvider); const suffix = getSupportedThinkingLevels(selectedModel).some((level) => level === requestedLevel) ? thinkingSuffix : ""; @@ -643,7 +643,7 @@ export class ChainClarifyComponent implements Component { this.editMode = "thinking"; const levels = this.getAvailableThinkingLevels(this.selectedStep); - const { thinkingSuffix } = splitThinkingSuffix(this.getEffectiveModel(this.selectedStep)); + const { thinkingSuffix } = splitKnownThinkingSuffix(this.getEffectiveModel(this.selectedStep)); const suffix = thinkingSuffix.slice(1); const levelIdx = levels.findIndex((level) => level === suffix); this.thinkingSelectedIndex = levelIdx >= 0 ? levelIdx : Math.max(0, levels.indexOf("off")); @@ -690,7 +690,7 @@ export class ChainClarifyComponent implements Component { const currentModel = this.getEffectiveBehavior(stepIndex).model; if (!currentModel) return; - const { baseModel } = splitThinkingSuffix(currentModel); + const { baseModel } = splitKnownThinkingSuffix(currentModel); const newModel = level === "off" ? baseModel : `${baseModel}:${level}`; this.updateBehavior(stepIndex, "model", newModel); } @@ -922,7 +922,7 @@ export class ChainClarifyComponent implements Component { lines.push(this.row("")); const currentModel = this.getEffectiveModel(this.editingStep!); - const currentModelBase = splitThinkingSuffix(currentModel).baseModel; + const currentModelBase = splitKnownThinkingSuffix(currentModel).baseModel; const currentLabel = th.fg("dim", "Current: "); lines.push(this.row(` ${currentLabel}${th.fg("warning", currentModel)}`)); lines.push(this.row("")); diff --git a/packages/subagents/src/runs/foreground/execution.ts b/packages/subagents/src/runs/foreground/execution.ts index d05da09b6..c4bb76fbf 100644 --- a/packages/subagents/src/runs/foreground/execution.ts +++ b/packages/subagents/src/runs/foreground/execution.ts @@ -69,6 +69,7 @@ import { resolveSubagentCodexFastModeScope, resolveSubagentModelFastMode, } from "../../shared/fast-mode.ts"; +import { resolveEffectiveThinking } from "../../shared/model-info.ts"; const artifactOutputByResult = new WeakMap(); @@ -814,6 +815,7 @@ export async function runSync( options.availableModels, options.preferredModelProvider, options.currentModel, + agent.fallbackThinkingLevels, ); const fastModeCwd = options.cwd ?? runtimeCwd; const fastModeSettings = getSubagentCodexFastModeSettings(fastModeCwd); @@ -861,8 +863,10 @@ export async function runSync( totalToolCount += result.progressSummary?.toolCount ?? 0; totalDurationMs += result.progressSummary?.durationMs ?? 0; const attemptSucceeded = result.exitCode === 0 && !result.error; + const attemptModel = applyThinkingSuffix(candidate, agent.thinking) ?? result.model ?? agent.model ?? "default"; const attempt: ModelAttempt = { - model: candidate ?? result.model ?? agent.model ?? "default", + model: attemptModel, + reasoningLevel: resolveEffectiveThinking(attemptModel, agent.thinking), success: attemptSucceeded, exitCode: result.exitCode, error: result.error, diff --git a/packages/subagents/src/runs/shared/model-fallback.ts b/packages/subagents/src/runs/shared/model-fallback.ts index f82b79cd2..c923d2f2a 100644 --- a/packages/subagents/src/runs/shared/model-fallback.ts +++ b/packages/subagents/src/runs/shared/model-fallback.ts @@ -1,4 +1,4 @@ -import type { ModelInfo as AvailableModelInfo } from "../../shared/model-info.ts"; +import { THINKING_LEVELS, splitKnownThinkingSuffix, type ModelInfo as AvailableModelInfo } from "../../shared/model-info.ts"; import type { Usage } from "../../shared/types.ts"; export type { AvailableModelInfo }; @@ -11,13 +11,10 @@ interface ModelAttemptSummary { usage?: Usage; } -export function splitThinkingSuffix(model: string): { baseModel: string; thinkingSuffix: string } { - const colonIdx = model.lastIndexOf(":"); - if (colonIdx === -1) return { baseModel: model, thinkingSuffix: "" }; - return { - baseModel: model.substring(0, colonIdx), - thinkingSuffix: model.substring(colonIdx), - }; +function applyFallbackThinkingLevel(model: string, thinkingLevel: string | undefined): string { + if (!thinkingLevel || !THINKING_LEVELS.some((level) => level === thinkingLevel)) return model; + const { thinkingSuffix } = splitKnownThinkingSuffix(model); + return thinkingSuffix ? model : `${model}:${thinkingLevel}`; } export function resolveModelCandidate( @@ -29,7 +26,7 @@ export function resolveModelCandidate( if (model.includes("/")) return model; if (!availableModels || availableModels.length === 0) return model; - const { baseModel, thinkingSuffix } = splitThinkingSuffix(model); + const { baseModel, thinkingSuffix } = splitKnownThinkingSuffix(model); const matches = availableModels.filter((entry) => entry.id === baseModel); if (preferredProvider) { const preferredMatch = matches.find((entry) => entry.provider === preferredProvider); @@ -45,10 +42,12 @@ export function buildModelCandidates( availableModels: AvailableModelInfo[] | undefined, preferredProvider?: string, currentModel?: string, + fallbackThinkingLevels?: string[], ): string[] { const seen = new Set(); const candidates: string[] = []; - for (const raw of [primaryModel, ...(fallbackModels ?? []), currentModel]) { + const fallbackEntries = (fallbackModels ?? []).map((model, index) => applyFallbackThinkingLevel(model, fallbackThinkingLevels?.[index])); + for (const raw of [primaryModel, ...fallbackEntries, currentModel]) { if (!raw) continue; const normalized = resolveModelCandidate(raw.trim(), availableModels, preferredProvider); if (!normalized || seen.has(normalized)) continue; diff --git a/packages/subagents/src/shared/types.ts b/packages/subagents/src/shared/types.ts index 0f973924a..2bec1376e 100644 --- a/packages/subagents/src/shared/types.ts +++ b/packages/subagents/src/shared/types.ts @@ -195,6 +195,7 @@ interface ProgressSummary { export interface ModelAttempt { model: string; + reasoningLevel?: string; success: boolean; exitCode?: number | null; error?: string; diff --git a/packages/workflows/CHANGELOG.md b/packages/workflows/CHANGELOG.md index 56edc95cc..172113228 100644 --- a/packages/workflows/CHANGELOG.md +++ b/packages/workflows/CHANGELOG.md @@ -10,6 +10,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Removed the imperative `runWorkflow` object-form API from `@bastani/workflows`; workflow authors must export definitions produced by `defineWorkflow(...).compile()`, and forged `__piWorkflow: true` objects are rejected by discovery and composition. +### Added + +- Added suffix-first workflow reasoning levels for `model` and `fallbackModels` entries such as `openai/gpt-5:high`, plus `WorkflowModelAttempt.reasoningLevel` metadata and optional `fallbackThinkingLevels` compatibility mapping ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). + +### Changed + +- Changed bundled workflows to encode their existing reasoning levels directly on model and fallback model strings ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). +- Documented the `model_name:thinking_effort` suffix syntax and `thinkingLevel` migration guidance in the workflows docs and package README ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). + +### Deprecated + +- Deprecated workflow `thinkingLevel` stage options in favor of per-candidate `:off|minimal|low|medium|high|xhigh` model suffixes; removal is deferred ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). + +### Fixed + +- Made workflow reasoning-suffix parsing lenient so it no longer rejects legitimate colon-tagged model ids (for example OpenRouter `:free`/`:exacto` variants and Ollama `llama3:latest`); only canonical `:off|minimal|low|medium|high|xhigh` suffixes are stripped as reasoning levels, and unknown ids still surface the generic catalog "not available" error ([#1199](https://github.com/bastani-inc/atomic/issues/1199)). + ## [0.8.23] - 2026-06-02 ### Changed diff --git a/packages/workflows/README.md b/packages/workflows/README.md index 3990a5444..183a94732 100644 --- a/packages/workflows/README.md +++ b/packages/workflows/README.md @@ -648,3 +648,19 @@ MIT — see [LICENSE](LICENSE). --- **Development:** see [DEV_SETUP.md](../../DEV_SETUP.md) for setup, testing, layout, and the local-extension dev loop. + +## Model reasoning levels + +Workflow stage `model` and `fallbackModels` strings support suffix-first reasoning levels using the `model_name:thinking_effort` syntax: append `:off`, `:minimal`, `:low`, `:medium`, `:high`, or `:xhigh` to the model id (for example `openai/gpt-5:high` or `anthropic/claude-haiku-4-5:off`). A suffix on a fallback candidate controls only that retry attempt, so fallback chains can mix reasoning levels. + +The older `thinkingLevel` stage option remains accepted as a deprecated default for candidates without a suffix. If both are present, the model suffix wins. Migrate legacy `thinkingLevel` stages by folding the effort into the model strings: + +```diff +- model: "openai/gpt-5.5", +- fallbackModels: ["anthropic/claude-opus-4-8"], +- thinkingLevel: "high", ++ model: "openai/gpt-5.5:high", ++ fallbackModels: ["anthropic/claude-opus-4-8:high"], +``` + +`fallbackThinkingLevels` is an optional compatibility helper aligned by index to `fallbackModels`; it is used only for fallback entries that do not already include a suffix. diff --git a/packages/workflows/builtin/deep-research-codebase.ts b/packages/workflows/builtin/deep-research-codebase.ts index c9b8b0ccc..3f04fe049 100644 --- a/packages/workflows/builtin/deep-research-codebase.ts +++ b/packages/workflows/builtin/deep-research-codebase.ts @@ -412,26 +412,24 @@ export default defineWorkflow("deep-research-codebase") ); const plannerModelConfig = { - model: "openai/gpt-5.5", + model: "openai/gpt-5.5:high", fallbackModels: [ - "openai-codex/gpt-5.5", - "github-copilot/gpt-5.5", - "anthropic/claude-opus-4-8", - "github-copilot/claude-opus-4.7", + "openai-codex/gpt-5.5:high", + "github-copilot/gpt-5.5:high", + "anthropic/claude-opus-4-8:high", + "github-copilot/claude-opus-4.7:high", ], - thinkingLevel: "high" as const, excludedTools: ["ask_user_question"], }; const explorerModelConfig = { - model: "openai/gpt-5.4-mini", + model: "openai/gpt-5.4-mini:low", fallbackModels: [ - "openai-codex/gpt-5.4-mini", - "github-copilot/gpt-5.4-mini", - "anthropic/claude-haiku-4-5", - "github-copilot/claude-haiku-4.5", + "openai-codex/gpt-5.4-mini:low", + "github-copilot/gpt-5.4-mini:low", + "anthropic/claude-haiku-4-5:low", + "github-copilot/claude-haiku-4.5:low", ], - thinkingLevel: "low" as const, excludedTools: ["ask_user_question"], }; diff --git a/packages/workflows/builtin/goal.ts b/packages/workflows/builtin/goal.ts index bb3ada005..124c8e15a 100644 --- a/packages/workflows/builtin/goal.ts +++ b/packages/workflows/builtin/goal.ts @@ -1026,26 +1026,24 @@ export default defineWorkflow("goal") const { ledger, ledgerPath, artifactDir } = await createGoalLedger(objective); const workerModelConfig = { - model: "openai/gpt-5.5", + model: "openai/gpt-5.5:low", fallbackModels: [ - "openai-codex/gpt-5.5", - "github-copilot/gpt-5.5", - "anthropic/claude-sonnet-4-7", - "github-copilot/claude-sonnet-4.7", + "openai-codex/gpt-5.5:low", + "github-copilot/gpt-5.5:low", + "anthropic/claude-sonnet-4-7:low", + "github-copilot/claude-sonnet-4.7:low", ], - thinkingLevel: "low" as const, tools: goalRunnerTools, }; const reviewerModelConfig = { - model: "openai/gpt-5.5", + model: "openai/gpt-5.5:high", fallbackModels: [ - "openai-codex/gpt-5.5", - "github-copilot/gpt-5.5", - "anthropic/claude-sonnet-4-7", - "github-copilot/claude-sonnet-4.7", + "openai-codex/gpt-5.5:high", + "github-copilot/gpt-5.5:high", + "anthropic/claude-sonnet-4-7:high", + "github-copilot/claude-sonnet-4.7:high", ], - thinkingLevel: "high" as const, tools: [...goalRunnerTools, reviewDecisionTool.name], customTools: [reviewDecisionTool], }; diff --git a/packages/workflows/builtin/open-claude-design.ts b/packages/workflows/builtin/open-claude-design.ts index e483266ec..2f581f496 100644 --- a/packages/workflows/builtin/open-claude-design.ts +++ b/packages/workflows/builtin/open-claude-design.ts @@ -226,13 +226,12 @@ export default defineWorkflow("open-claude-design") const specFileUrl = `file://${specPath}`; const designModelConfig = { - model: "anthropic/claude-opus-4-8", + model: "anthropic/claude-opus-4-8:high", fallbackModels: [ - "github-copilot/claude-opus-4.7", - "anthropic/claude-sonnet-4-6", - "github-copilot/claude-sonnet-4.6", + "github-copilot/claude-opus-4.7:high", + "anthropic/claude-sonnet-4-6:high", + "github-copilot/claude-sonnet-4.6:high", ], - thinkingLevel: "high" as const, }; let designSystem: string; diff --git a/packages/workflows/builtin/ralph.ts b/packages/workflows/builtin/ralph.ts index 6f8fdd5a3..3af6796f5 100644 --- a/packages/workflows/builtin/ralph.ts +++ b/packages/workflows/builtin/ralph.ts @@ -432,50 +432,46 @@ async function runRalphWorkflow( let iterationsCompleted = 0; const plannerModelConfig = { - model: "openai/gpt-5.5", + model: "openai/gpt-5.5:high", fallbackModels: [ - "openai-codex/gpt-5.5", - "github-copilot/gpt-5.5", - "anthropic/claude-opus-4-8", - "github-copilot/claude-opus-4.7", + "openai-codex/gpt-5.5:high", + "github-copilot/gpt-5.5:high", + "anthropic/claude-opus-4-8:high", + "github-copilot/claude-opus-4.7:high", ], - thinkingLevel: "high" as const, excludedTools: ["ask_user_question"], }; const orchestratorModelConfig = { - model: "openai/gpt-5.5", + model: "openai/gpt-5.5:medium", fallbackModels: [ - "openai-codex/gpt-5.5", - "github-copilot/gpt-5.5", - "anthropic/claude-sonnet-4-6", - "github-copilot/claude-sonnet-4.6", + "openai-codex/gpt-5.5:medium", + "github-copilot/gpt-5.5:medium", + "anthropic/claude-sonnet-4-6:medium", + "github-copilot/claude-sonnet-4.6:medium", ], - thinkingLevel: "medium" as const, excludedTools: ["ask_user_question"], }; const simplifierModelConfig = { - model: "openai/gpt-5.5", + model: "openai/gpt-5.5:medium", fallbackModels: [ - "openai-codex/gpt-5.5", - "github-copilot/gpt-5.5", - "anthropic/claude-sonnet-4-6", - "github-copilot/claude-sonnet-4.6", + "openai-codex/gpt-5.5:medium", + "github-copilot/gpt-5.5:medium", + "anthropic/claude-sonnet-4-6:medium", + "github-copilot/claude-sonnet-4.6:medium", ], - thinkingLevel: "medium" as const, excludedTools: ["ask_user_question"], }; const reviewerModelConfig = { - model: "openai/gpt-5.5", + model: "openai/gpt-5.5:high", fallbackModels: [ - "openai-codex/gpt-5.5", - "github-copilot/gpt-5.5", - "anthropic/claude-opus-4-8", - "github-copilot/claude-opus-4.7", + "openai-codex/gpt-5.5:high", + "github-copilot/gpt-5.5:high", + "anthropic/claude-opus-4-8:high", + "github-copilot/claude-opus-4.7:high", ], - thinkingLevel: "high" as const, excludedTools: ["ask_user_question"], customTools: [reviewDecisionTool], }; diff --git a/packages/workflows/src/extension/workflow-schema.ts b/packages/workflows/src/extension/workflow-schema.ts index 848f5c219..e7a1617c4 100644 --- a/packages/workflows/src/extension/workflow-schema.ts +++ b/packages/workflows/src/extension/workflow-schema.ts @@ -42,7 +42,7 @@ const StageSessionOptionProperties = { agentDir: Type.Optional(Type.String()), authStorage: Type.Optional(SdkSessionOptionSchema("authStorage")), modelRegistry: Type.Optional(SdkSessionOptionSchema("modelRegistry")), - model: Type.Optional(Type.Unsafe({})), + model: Type.Optional(Type.Unsafe({ description: "Primary model id or SDK model object. String ids may include a reasoning suffix, e.g. openai/gpt-5:high; valid levels: off|minimal|low|medium|high|xhigh." })), thinkingLevel: Type.Optional(SdkSessionOptionSchema("thinkingLevel")), scopedModels: Type.Optional(Type.Array(SdkSessionOptionArrayElementSchema("scopedModels"))), noTools: Type.Optional(Type.Unsafe>({ @@ -54,7 +54,8 @@ const StageSessionOptionProperties = { sessionManager: Type.Optional(SdkSessionOptionSchema("sessionManager")), settingsManager: Type.Optional(SdkSessionOptionSchema("settingsManager")), sessionStartEvent: Type.Optional(SdkSessionOptionSchema("sessionStartEvent")), - fallbackModels: Type.Optional(Type.Array(Type.String())), + fallbackModels: Type.Optional(Type.Array(Type.String({ description: "Fallback model id; may include a reasoning suffix like :low or :off." }))), + fallbackThinkingLevels: Type.Optional(Type.Array(Type.String({ description: "Deprecated compatibility helper aligned to fallbackModels; ignored when the fallback model has a :level suffix." }))), mcp: Type.Optional(McpOptionsSchema), sessionDir: Type.Optional(Type.String()), context: Type.Optional(Type.Union([Type.Literal("fresh"), Type.Literal("fork")])), diff --git a/packages/workflows/src/runs/foreground/stage-runner.ts b/packages/workflows/src/runs/foreground/stage-runner.ts index c33d1a8a1..058689d8c 100644 --- a/packages/workflows/src/runs/foreground/stage-runner.ts +++ b/packages/workflows/src/runs/foreground/stage-runner.ts @@ -64,7 +64,7 @@ export interface StageSessionRuntime { getLastAssistantText?: () => string | undefined; } -export type StageSessionCreateOptions = CreateAgentSessionOptions & Pick; +export type StageSessionCreateOptions = CreateAgentSessionOptions & Pick; type WorkflowFastModeSettings = { readonly chat: boolean; @@ -169,6 +169,7 @@ function stripWorkflowOnlyOptions(options: StageOptions | undefined): CreateAgen const { mcp: _mcp, fallbackModels: _fallbackModels, + fallbackThinkingLevels: _fallbackThinkingLevels, context, forkFromSessionFile, sessionDir, @@ -549,6 +550,7 @@ export function createStageContext(opts: StageRunnerOpts): InternalStageContext candidatesPromise = buildModelCandidatesFromCatalog({ primaryModel: stageOptions?.model, fallbackModels: stageOptions?.fallbackModels, + fallbackThinkingLevels: stageOptions?.fallbackThinkingLevels, catalog: modelCatalog, }); } @@ -557,7 +559,13 @@ export function createStageContext(opts: StageRunnerOpts): InternalStageContext function stageOptionsForCandidate(candidate: WorkflowResolvedModelCandidate | undefined): StageOptions | undefined { if (candidate === undefined) return stageOptions; - return { ...(stageOptions ?? {}), model: candidate.value, fallbackModels: undefined }; + return { + ...(stageOptions ?? {}), + model: candidate.value, + ...(candidate.reasoningLevel !== undefined ? { thinkingLevel: candidate.reasoningLevel } : {}), + fallbackModels: undefined, + fallbackThinkingLevels: undefined, + }; } let sessionSettingsManager: WorkflowFastModeSettingsManager | undefined; @@ -591,6 +599,25 @@ export function createStageContext(opts: StageRunnerOpts): InternalStageContext return { session: created }; } + function effectiveCandidateReasoning(candidate: WorkflowResolvedModelCandidate): StageOptions["thinkingLevel"] | undefined { + return candidate.reasoningLevel ?? stageOptions?.thinkingLevel; + } + + function modelAttemptReasoning(candidate: WorkflowResolvedModelCandidate): Pick { + const reasoningLevel = effectiveCandidateReasoning(candidate); + return reasoningLevel !== undefined ? { reasoningLevel } : {}; + } + + function applyCandidateThinking(candidate: WorkflowResolvedModelCandidate | undefined): void { + pendingThinkingLevel = candidate === undefined + ? stageOptions?.thinkingLevel + : effectiveCandidateReasoning(candidate); + } + + function candidateLabel(candidate: WorkflowResolvedModelCandidate): string { + return candidate.reasoningLevel !== undefined ? `${candidate.id}:${candidate.reasoningLevel}` : candidate.id; + } + function attachSession(created: StageSessionRuntime | StageSessionCreateResult): StageSessionRuntime { const result = normalizeSessionCreateResult(created); session = result.session; @@ -612,6 +639,7 @@ export function createStageContext(opts: StageRunnerOpts): InternalStageContext candidate: WorkflowResolvedModelCandidate | undefined, consumer: AgentSessionConsumer, ): Promise { + applyCandidateThinking(candidate); const created = adapters.agentSession ? await adapters.agentSession.create(stripWorkflowOnlyOptions(stageOptionsForCandidate(candidate)) as StageSessionCreateOptions, { ...meta, @@ -716,16 +744,16 @@ export function createStageContext(opts: StageRunnerOpts): InternalStageContext notifyModelFallbackMetaChange(); try { await promptWithPauseResume(activeSession, text, sdkOptions); - modelAttempts.push({ model: candidate.id, success: true }); + modelAttempts.push({ model: candidate.id, success: true, ...modelAttemptReasoning(candidate) }); return; } catch (err) { const message = errorMessage(err); - modelAttempts.push({ model: candidate.id, success: false, error: message }); + modelAttempts.push({ model: candidate.id, success: false, ...modelAttemptReasoning(candidate), error: message }); if (signal?.aborted || !isRetryableModelFailure(message) || index === candidates.length - 1) { throw err; } const nextCandidate = candidates[index + 1]!; - modelWarnings.push(`[fallback] ${candidate.id} failed: ${message}. Retrying with ${nextCandidate.id}.`); + modelWarnings.push(`[fallback] ${candidateLabel(candidate)} failed: ${message}. Retrying with ${candidateLabel(nextCandidate)}.`); await disposeCurrentSession(); index += 1; } diff --git a/packages/workflows/src/runs/shared/model-fallback.ts b/packages/workflows/src/runs/shared/model-fallback.ts index fec510e29..6b1c8147c 100644 --- a/packages/workflows/src/runs/shared/model-fallback.ts +++ b/packages/workflows/src/runs/shared/model-fallback.ts @@ -3,11 +3,38 @@ import type { WorkflowModelCatalogPort, WorkflowModelInfo, WorkflowModelValue, + WorkflowThinkingLevel, } from "../../shared/types.js"; export interface WorkflowResolvedModelCandidate { readonly id: string; readonly value: WorkflowModelValue; + readonly reasoningLevel?: WorkflowThinkingLevel; +} + +function makeCandidate( + id: string, + value: WorkflowModelValue, + level: WorkflowThinkingLevel | undefined, +): WorkflowResolvedModelCandidate { + return level !== undefined ? { id, value, reasoningLevel: level } : { id, value }; +} + +const WORKFLOW_THINKING_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh"] as const satisfies readonly WorkflowThinkingLevel[]; +const WORKFLOW_THINKING_LEVEL_SET: ReadonlySet = new Set(WORKFLOW_THINKING_LEVELS); + +export function splitReasoningSuffix(model: string): { readonly baseModel: string; readonly level?: WorkflowThinkingLevel } { + const index = model.lastIndexOf(":"); + if (index < 0) return { baseModel: model }; + const suffix = model.slice(index + 1); + if (WORKFLOW_THINKING_LEVEL_SET.has(suffix)) { + return { baseModel: model.slice(0, index), level: suffix as WorkflowThinkingLevel }; + } + return { baseModel: model }; +} + +function candidateKey(candidate: WorkflowResolvedModelCandidate): string { + return `${candidate.id}::${candidate.reasoningLevel ?? ""}`; } interface ModelResolutionFailure { @@ -72,35 +99,36 @@ function resolveStringModel( ): WorkflowResolvedModelCandidate | ModelResolutionFailure { const input = rawInput.trim(); if (!input) return { input: rawInput, reason: "empty model id" }; + const { baseModel, level } = splitReasoningSuffix(input); if (availableModels === undefined) { - return { id: input, value: input }; + return makeCandidate(baseModel, baseModel, level); } const models = uniqueByFullId(availableModels); - const explicit = models.find((model) => model.fullId === input); + const explicit = models.find((model) => model.fullId === baseModel); if (explicit !== undefined) { - return { id: explicit.fullId, value: explicit.model ?? explicit.fullId }; + return makeCandidate(explicit.fullId, explicit.model ?? explicit.fullId, level); } - if (input.includes("/")) { + if (baseModel.includes("/")) { return { input, reason: "not available" }; } - const byBareId = models.filter((model) => model.id === input); + const byBareId = models.filter((model) => model.id === baseModel); if (byBareId.length === 0) { return { input, reason: "not available" }; } if (byBareId.length === 1) { const only = byBareId[0]!; - return { id: only.fullId, value: only.model ?? only.fullId }; + return makeCandidate(only.fullId, only.model ?? only.fullId, level); } const preferred = preferredProvider === undefined ? undefined : byBareId.find((model) => model.provider === preferredProvider); if (preferred !== undefined) { - return { id: preferred.fullId, value: preferred.model ?? preferred.fullId }; + return makeCandidate(preferred.fullId, preferred.model ?? preferred.fullId, level); } return { @@ -127,13 +155,30 @@ function isFailure(value: WorkflowResolvedModelCandidate | ModelResolutionFailur export function buildModelCandidates(input: { readonly primaryModel?: WorkflowModelValue; readonly fallbackModels?: readonly string[]; + readonly fallbackThinkingLevels?: readonly string[]; readonly currentModel?: WorkflowModelValue; readonly availableModels?: readonly WorkflowModelInfo[]; readonly preferredProvider?: string; }): WorkflowResolvedModelCandidate[] { const rawValues: WorkflowModelValue[] = []; if (input.primaryModel !== undefined) rawValues.push(input.primaryModel); - rawValues.push(...(input.fallbackModels ?? [])); + for (const [index, fallback] of (input.fallbackModels ?? []).entries()) { + // Trim once up front so the suffix split, the validation error input, and the + // compat concatenation all operate on the same value. Concatenating the raw + // (untrimmed) fallback would push trailing whitespace into the interior of + // `id:level`, which `resolveStringModel` can no longer trim away. + const trimmedFallback = fallback.trim(); + const split = splitReasoningSuffix(trimmedFallback); + const compatLevel = input.fallbackThinkingLevels?.[index]; + if (split.level === undefined && compatLevel !== undefined) { + if (!WORKFLOW_THINKING_LEVEL_SET.has(compatLevel)) { + throw new WorkflowModelValidationError([{ input: trimmedFallback, reason: `invalid fallbackThinkingLevels[${index}] "${compatLevel}"; expected one of ${WORKFLOW_THINKING_LEVELS.join(", ")}` }]); + } + rawValues.push(`${trimmedFallback}:${compatLevel}`); + } else { + rawValues.push(trimmedFallback); + } + } if (input.currentModel !== undefined) rawValues.push(input.currentModel); const failures: ModelResolutionFailure[] = []; @@ -145,8 +190,9 @@ export function buildModelCandidates(input: { failures.push(resolved); continue; } - if (seen.has(resolved.id)) continue; - seen.add(resolved.id); + const key = candidateKey(resolved); + if (seen.has(key)) continue; + seen.add(key); candidates.push(resolved); } @@ -165,6 +211,7 @@ function catalogUnavailableWarning(): string { export async function buildModelCandidatesFromCatalog(input: { readonly primaryModel?: WorkflowModelValue; readonly fallbackModels?: readonly string[]; + readonly fallbackThinkingLevels?: readonly string[]; readonly catalog?: WorkflowModelCatalogPort; }): Promise { const hasExplicitModel = input.primaryModel !== undefined || (input.fallbackModels?.length ?? 0) > 0; @@ -174,6 +221,7 @@ export async function buildModelCandidatesFromCatalog(input: { return buildModelCandidates({ primaryModel: input.primaryModel, fallbackModels: input.fallbackModels, + fallbackThinkingLevels: input.fallbackThinkingLevels, }); } @@ -182,6 +230,7 @@ export async function buildModelCandidatesFromCatalog(input: { return buildModelCandidates({ primaryModel: input.primaryModel, fallbackModels: input.fallbackModels, + fallbackThinkingLevels: input.fallbackThinkingLevels, currentModel: input.catalog.currentModel, availableModels, preferredProvider: input.catalog.preferredProvider, @@ -199,6 +248,7 @@ export async function validateWorkflowModels(input: { readonly requests: readonly { readonly model?: WorkflowModelValue; readonly fallbackModels?: readonly string[]; + readonly fallbackThinkingLevels?: readonly string[]; }[]; readonly catalog?: WorkflowModelCatalogPort; }): Promise { @@ -230,6 +280,7 @@ export async function validateWorkflowModels(input: { buildModelCandidates({ primaryModel: request.model, fallbackModels: request.fallbackModels, + fallbackThinkingLevels: request.fallbackThinkingLevels, currentModel: input.catalog?.currentModel, availableModels, preferredProvider: input.catalog?.preferredProvider, diff --git a/packages/workflows/src/shared/authoring-contract.ts b/packages/workflows/src/shared/authoring-contract.ts index 94b95f824..48bd4a1ea 100644 --- a/packages/workflows/src/shared/authoring-contract.ts +++ b/packages/workflows/src/shared/authoring-contract.ts @@ -43,8 +43,10 @@ export type WorkflowDetailsStatus = "accepted" | "running" | "completed" | "fail export type WorkflowAction = "list" | "get" | "inputs" | "run" | "status" | "interrupt" | "resume"; export interface WorkflowModelFallbackFields { - /** Ordered model IDs to try after `model` fails for a retryable provider/model reason. */ + /** Ordered model IDs to try after `model` fails; entries may use `:off|minimal|low|medium|high|xhigh` reasoning suffixes. */ readonly fallbackModels?: readonly string[]; + /** Optional deprecated compatibility helper aligned to `fallbackModels`; ignored for entries with a reasoning suffix. */ + readonly fallbackThinkingLevels?: readonly string[]; } export type WorkflowModelValue = string | object; @@ -61,6 +63,7 @@ export interface WorkflowModelUsage extends WorkflowSerializableObject { export interface WorkflowModelAttempt extends WorkflowSerializableObject { readonly model: string; readonly success: boolean; + readonly reasoningLevel?: WorkflowThinkingLevel; readonly error?: string; readonly usage?: WorkflowModelUsage; } @@ -115,6 +118,7 @@ export interface WorkflowCustomToolDefinition export type WorkflowModelUsage = AuthoringContract.WorkflowModelUsage; export type WorkflowModelAttempt = AuthoringContract.WorkflowModelAttempt; export type WorkflowModelFallbackFields = AuthoringContract.WorkflowModelFallbackFields; +export type WorkflowThinkingLevel = AuthoringContract.WorkflowThinkingLevel; export interface WorkflowModelInfo extends Omit { readonly model?: NonNullable; diff --git a/test/unit/model-fallback.test.ts b/test/unit/model-fallback.test.ts index fc8a8b595..19e2c25dc 100644 --- a/test/unit/model-fallback.test.ts +++ b/test/unit/model-fallback.test.ts @@ -1,7 +1,9 @@ import { describe, test } from "bun:test"; import assert from "node:assert/strict"; import { + buildModelCandidates, buildModelCandidateIds, + splitReasoningSuffix, isRetryableModelFailure, validateWorkflowModels, WorkflowModelValidationError, @@ -15,6 +17,151 @@ const models: readonly WorkflowModelInfo[] = [ ]; describe("model fallback helpers", () => { + + test("splitReasoningSuffix parses canonical suffixes and stays lenient for non-canonical suffixes", () => { + assert.deepEqual(splitReasoningSuffix("anthropic/claude-haiku-4-5:off"), { + baseModel: "anthropic/claude-haiku-4-5", + level: "off", + }); + assert.deepEqual(splitReasoningSuffix("openai/gpt-5-mini"), { baseModel: "openai/gpt-5-mini" }); + // Non-canonical colon-tagged ids are no longer rejected; the whole string is the base model. + assert.deepEqual(splitReasoningSuffix("gpt-5-mini:ultra"), { baseModel: "gpt-5-mini:ultra" }); + }); + + test("splitReasoningSuffix is lenient and never throws for legitimate colon-tagged ids (#1199)", () => { + // Provider/model ids that legitimately use a trailing colon tag must pass through untouched. + const lenientIds = [ + "ollama/llama3:latest", + "openrouter/meta-llama/llama-3-8b-instruct:free", + "mistral:instruct", + "qwen:chat", + // OpenRouter routing/variant tags. + "openrouter/anthropic/claude-3.5-sonnet:exacto", + "openrouter/meta-llama/llama-3-8b-instruct:nitro", + "openrouter/perplexity/sonar:online", + "openrouter/anthropic/claude-3.5-sonnet:beta", + ]; + for (const id of lenientIds) { + assert.doesNotThrow(() => splitReasoningSuffix(id)); + assert.deepEqual(splitReasoningSuffix(id), { baseModel: id }, `expected ${id} to stay intact with no level`); + } + }); + + test("splitReasoningSuffix still extracts canonical reasoning levels (#1199)", () => { + assert.deepEqual(splitReasoningSuffix("claude-sonnet-4:high"), { + baseModel: "claude-sonnet-4", + level: "high", + }); + }); + + test("buildModelCandidates resolves a catalog fullId that ends in a colon tag (#1199)", () => { + const catalogModels: readonly WorkflowModelInfo[] = [ + { + provider: "openrouter", + id: "meta-llama/llama-3-8b-instruct:free", + fullId: "openrouter/meta-llama/llama-3-8b-instruct:free", + }, + ]; + assert.deepEqual( + buildModelCandidates({ + primaryModel: "openrouter/meta-llama/llama-3-8b-instruct:free", + availableModels: catalogModels, + }).map((candidate) => ({ id: candidate.id, reasoningLevel: candidate.reasoningLevel })), + [{ id: "openrouter/meta-llama/llama-3-8b-instruct:free", reasoningLevel: undefined }], + ); + }); + + test("buildModelCandidates surfaces generic 'not available' for unknown colon-tagged ids (#1199)", () => { + assert.throws( + () => buildModelCandidates({ primaryModel: "gpt-5:ultra", availableModels: models }), + (err: unknown) => { + assert.ok(err instanceof WorkflowModelValidationError); + assert.match(err.message, /gpt-5:ultra \(not available\)/); + assert.doesNotMatch(err.message, /invalid reasoning level suffix/); + return true; + }, + ); + }); + + test("buildModelCandidates resolves suffixed full ids and bare ids with preferred provider", () => { + assert.deepEqual( + buildModelCandidates({ + primaryModel: "anthropic/claude-sonnet-4:high", + fallbackModels: ["claude-sonnet-4:low", "gpt-5-mini:off"], + availableModels: models, + preferredProvider: "github-copilot", + }).map((candidate) => ({ id: candidate.id, reasoningLevel: candidate.reasoningLevel })), + [ + { id: "anthropic/claude-sonnet-4", reasoningLevel: "high" }, + { id: "github-copilot/claude-sonnet-4", reasoningLevel: "low" }, + { id: "openai/gpt-5-mini", reasoningLevel: "off" }, + ], + ); + }); + + test("buildModelCandidates de-duplicates by model id and reasoning level", () => { + assert.deepEqual( + buildModelCandidates({ + primaryModel: "openai/gpt-5-mini:high", + fallbackModels: ["openai/gpt-5-mini:low", "openai/gpt-5-mini:high"], + availableModels: models, + }).map((candidate) => ({ id: candidate.id, reasoningLevel: candidate.reasoningLevel })), + [ + { id: "openai/gpt-5-mini", reasoningLevel: "high" }, + { id: "openai/gpt-5-mini", reasoningLevel: "low" }, + ], + ); + }); + + test("fallbackThinkingLevels maps positionally only when fallback lacks suffix", () => { + assert.deepEqual( + buildModelCandidates({ + primaryModel: "openai/gpt-5-mini", + fallbackModels: ["anthropic/claude-sonnet-4", "github-copilot/claude-sonnet-4:high"], + fallbackThinkingLevels: ["low", "off"], + availableModels: models, + }).map((candidate) => ({ id: candidate.id, reasoningLevel: candidate.reasoningLevel })), + [ + { id: "openai/gpt-5-mini", reasoningLevel: undefined }, + { id: "anthropic/claude-sonnet-4", reasoningLevel: "low" }, + { id: "github-copilot/claude-sonnet-4", reasoningLevel: "high" }, + ], + ); + }); + + test("buildModelCandidates throws WorkflowModelValidationError for an invalid fallbackThinkingLevels entry (#1199)", () => { + assert.throws( + () => + buildModelCandidates({ + primaryModel: "openai/gpt-5-mini", + fallbackModels: ["anthropic/claude-sonnet-4"], + fallbackThinkingLevels: ["bogus"], + availableModels: models, + }), + (err: unknown) => { + assert.ok(err instanceof WorkflowModelValidationError); + assert.match(err.message, /invalid fallbackThinkingLevels\[0\] "bogus"/); + assert.equal(err.failures[0]?.input, "anthropic/claude-sonnet-4"); + return true; + }, + ); + }); + + test("fallbackThinkingLevels trims surrounding whitespace before applying the compat level (#1199)", () => { + assert.deepEqual( + buildModelCandidates({ + primaryModel: "openai/gpt-5-mini", + fallbackModels: [" anthropic/claude-sonnet-4 "], + fallbackThinkingLevels: ["low"], + availableModels: models, + }).map((candidate) => ({ id: candidate.id, reasoningLevel: candidate.reasoningLevel })), + [ + { id: "openai/gpt-5-mini", reasoningLevel: undefined }, + { id: "anthropic/claude-sonnet-4", reasoningLevel: "low" }, + ], + ); + }); + test("buildModelCandidateIds preserves provider-qualified ids and de-duplicates", () => { assert.deepEqual( buildModelCandidateIds({ diff --git a/test/unit/stage-runner.test.ts b/test/unit/stage-runner.test.ts index 831415a94..039cf9c65 100644 --- a/test/unit/stage-runner.test.ts +++ b/test/unit/stage-runner.test.ts @@ -1281,3 +1281,143 @@ describe("createStageContext — controlled pause", () => { await rejection; }); }); + +describe("createStageContext — reasoning suffix retry behavior", () => { + async function exerciseRetryWithReasoning( + invoke: (ctx: InternalStageContext) => Promise, + ): Promise<{ + readonly calls: string[]; + readonly thinkingLevels: string[]; + readonly meta: ReturnType; + readonly text: string; + }> { + const calls: string[] = []; + const thinkingLevels: string[] = []; + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = + typeof options.model === "string" + ? options.model + : `${String(options.model?.provider)}/${options.model?.id}`; + calls.push(model); + const { session } = makeMockSession({ + setThinkingLevel(level) { + thinkingLevels.push(`${model}:${String(level)}`); + }, + async prompt() { + if (model === "anthropic/primary") { + throw new Error("429 rate limit exceeded"); + } + }, + getLastAssistantText() { + return model === "openai/fallback" ? "fallback answer" : undefined; + }, + }); + return session; + }, + }; + + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary:high", + fallbackModels: ["openai/fallback:low"], + thinkingLevel: "xhigh", + }, + }), + ) as InternalStageContext; + + const text = await invoke(ctx); + return { calls, thinkingLevels, meta: ctx.__modelFallbackMeta(), text }; + } + + test("direct task retry applies suffixed reasoning and records attempt metadata", async () => { + const result = await exerciseRetryWithReasoning((ctx) => ctx.prompt("go")); + + assert.equal(result.text, "fallback answer"); + assert.deepEqual(result.calls, ["anthropic/primary", "openai/fallback"]); + assert.deepEqual(result.thinkingLevels, [ + "anthropic/primary:high", + "openai/fallback:low", + ]); + assert.deepEqual( + result.meta.modelAttempts?.map((attempt) => ({ + model: attempt.model, + reasoningLevel: attempt.reasoningLevel, + success: attempt.success, + })), + [ + { model: "anthropic/primary", reasoningLevel: "high", success: false }, + { model: "openai/fallback", reasoningLevel: "low", success: true }, + ], + ); + }); + + test("chain-step retry uses the next candidate reasoning level", async () => { + const result = await exerciseRetryWithReasoning((ctx) => ctx.prompt("chain step")); + + assert.deepEqual(result.thinkingLevels, [ + "anthropic/primary:high", + "openai/fallback:low", + ]); + assert.deepEqual( + result.meta.modelAttempts?.map((attempt) => attempt.reasoningLevel), + ["high", "low"], + ); + }); + + test("parallel-step retry uses the next candidate reasoning level", async () => { + const result = await exerciseRetryWithReasoning((ctx) => ctx.prompt("parallel step")); + + assert.deepEqual(result.thinkingLevels, [ + "anthropic/primary:high", + "openai/fallback:low", + ]); + assert.deepEqual( + result.meta.modelAttempts?.map((attempt) => attempt.reasoningLevel), + ["high", "low"], + ); + }); + + test("legacy thinkingLevel applies when candidates have no suffix", async () => { + const thinkingLevels: string[] = []; + const agentSession: AgentSessionAdapter = { + async create(options) { + const model = typeof options.model === "string" ? String(options.model) : "object-model"; + const { session } = makeMockSession({ + setThinkingLevel(level) { + thinkingLevels.push(`${model}:${String(level)}`); + }, + async prompt() { + if (model === "anthropic/primary") throw new Error("503 service unavailable"); + }, + getLastAssistantText() { + return model === "openai/fallback" ? "legacy answer" : undefined; + }, + }); + return session; + }, + }; + const ctx = createStageContext( + makeOpts({ + adapters: { agentSession }, + stageOptions: { + model: "anthropic/primary", + fallbackModels: ["openai/fallback"], + thinkingLevel: "medium", + }, + }), + ) as InternalStageContext; + + assert.equal(await ctx.prompt("go"), "legacy answer"); + assert.deepEqual(thinkingLevels, [ + "anthropic/primary:medium", + "openai/fallback:medium", + ]); + assert.deepEqual( + ctx.__modelFallbackMeta().modelAttempts?.map((attempt) => attempt.reasoningLevel), + ["medium", "medium"], + ); + }); +}); diff --git a/test/unit/subagents-reasoning-suffix.test.ts b/test/unit/subagents-reasoning-suffix.test.ts new file mode 100644 index 000000000..c728582cd --- /dev/null +++ b/test/unit/subagents-reasoning-suffix.test.ts @@ -0,0 +1,170 @@ +import { describe, test } from "bun:test"; +import assert from "node:assert/strict"; +import { buildModelCandidates } from "../../packages/subagents/src/runs/shared/model-fallback.js"; +import { applyThinkingSuffix } from "../../packages/subagents/src/runs/shared/pi-args.js"; +import { resolveEffectiveThinking, splitKnownThinkingSuffix } from "../../packages/subagents/src/shared/model-info.js"; +import type { AvailableModelInfo } from "../../packages/subagents/src/runs/shared/model-fallback.js"; + +const models: AvailableModelInfo[] = [ + { provider: "anthropic", id: "claude-sonnet-4", fullId: "anthropic/claude-sonnet-4" }, + { provider: "openai", id: "gpt-5", fullId: "openai/gpt-5" }, + { provider: "provider:with-colon", id: "model", fullId: "provider:with-colon/model" }, +]; + +describe("subagent suffix-first reasoning helpers", () => { + test("splitKnownThinkingSuffix only recognizes canonical levels", () => { + assert.deepEqual(splitKnownThinkingSuffix("claude-sonnet-4:high"), { baseModel: "claude-sonnet-4", thinkingSuffix: ":high" }); + assert.deepEqual(splitKnownThinkingSuffix("claude-sonnet-4"), { baseModel: "claude-sonnet-4", thinkingSuffix: "" }); + assert.deepEqual(splitKnownThinkingSuffix("provider:model:ultra"), { baseModel: "provider:model:ultra", thinkingSuffix: "" }); + assert.deepEqual(splitKnownThinkingSuffix("provider:with-colon/model:off"), { baseModel: "provider:with-colon/model", thinkingSuffix: ":off" }); + }); + + test("applyThinkingSuffix preserves valid suffix over legacy thinking", () => { + assert.equal(applyThinkingSuffix("claude-sonnet-4:medium", "high"), "claude-sonnet-4:medium"); + assert.equal(applyThinkingSuffix("claude-sonnet-4", "low"), "claude-sonnet-4:low"); + assert.equal(applyThinkingSuffix("provider:model:ultra", "high"), "provider:model:ultra:high"); + }); + + test("resolveEffectiveThinking uses suffix, then legacy thinking, then undefined", () => { + assert.equal(resolveEffectiveThinking("gpt-5:low", "high"), "low"); + assert.equal(resolveEffectiveThinking("gpt-5", "high"), "high"); + assert.equal(resolveEffectiveThinking("gpt-5", "ultra"), undefined); + }); + + test("buildModelCandidates is ordered and de-dupes by resolved model plus level", () => { + assert.deepEqual( + buildModelCandidates( + "claude-sonnet-4:high", + ["anthropic/claude-sonnet-4:high", "claude-sonnet-4:medium", "gpt-5:low", "gpt-5:low"], + models, + "anthropic", + "anthropic/claude-sonnet-4:medium", + ), + ["anthropic/claude-sonnet-4:high", "anthropic/claude-sonnet-4:medium", "openai/gpt-5:low"], + ); + }); + + test("fallbackThinkingLevels applies positionally only when fallback has no suffix", () => { + assert.deepEqual( + buildModelCandidates( + "claude-sonnet-4", + ["gpt-5", "claude-sonnet-4:low"], + models, + "anthropic", + undefined, + ["medium", "xhigh"], + ), + ["anthropic/claude-sonnet-4", "openai/gpt-5:medium", "anthropic/claude-sonnet-4:low"], + ); + }); +}); + + +describe("subagent retry metadata reasoning seams", () => { + test("foreground retry candidates resolve per-attempt model and reasoning with suffix precedence", () => { + const agentThinking = "medium"; + const candidates = buildModelCandidates( + "claude-sonnet-4:high", + ["gpt-5:low"], + models, + "anthropic", + ); + + const attempts = candidates.map((candidate, index) => { + const model = applyThinkingSuffix(candidate, agentThinking)!; + return { + model, + reasoningLevel: resolveEffectiveThinking(model, agentThinking), + success: index === 1, + }; + }); + + assert.deepEqual(candidates, [ + "anthropic/claude-sonnet-4:high", + "openai/gpt-5:low", + ]); + assert.deepEqual(attempts, [ + { model: "anthropic/claude-sonnet-4:high", reasoningLevel: "high", success: false }, + { model: "openai/gpt-5:low", reasoningLevel: "low", success: true }, + ]); + }); + + test("async/background status mapping carries suffix level and falls back to legacy thinking", () => { + const agentThinking = "xhigh"; + const candidates = buildModelCandidates( + "claude-sonnet-4:high", + ["gpt-5"], + models, + "anthropic", + ); + + const statusAttempts = candidates.map((candidate) => ({ + model: applyThinkingSuffix(candidate, agentThinking)!, + thinking: resolveEffectiveThinking(applyThinkingSuffix(candidate, agentThinking), agentThinking), + })); + + assert.deepEqual(statusAttempts, [ + { model: "anthropic/claude-sonnet-4:high", thinking: "high" }, + { model: "openai/gpt-5:xhigh", thinking: "xhigh" }, + ]); + }); + + test("legacy no-suffix retry candidates keep legacy thinking as the effective level", () => { + const candidates = buildModelCandidates( + "claude-sonnet-4", + ["gpt-5"], + models, + "anthropic", + ); + const attempts = candidates.map((candidate) => { + const model = applyThinkingSuffix(candidate, "high")!; + return { model, reasoningLevel: resolveEffectiveThinking(model, "high") }; + }); + + assert.deepEqual(attempts, [ + { model: "anthropic/claude-sonnet-4:high", reasoningLevel: "high" }, + { model: "openai/gpt-5:high", reasoningLevel: "high" }, + ]); + }); +}); + +// Mirrors the foreground execution seam (execution.ts ~line 866): +// const attemptModel = applyThinkingSuffix(candidate, agent.thinking) ?? result.model ?? agent.model ?? "default"; +// reasoningLevel: resolveEffectiveThinking(attemptModel, agent.thinking) +// Asserts the candidate-derived suffix wins even when legacy `thinking` is unset and +// the SDK echoes a suffix-stripped `result.model`. +describe("foreground attempt metadata derives reasoning level from candidate suffix (#1199)", () => { + test("candidate suffix yields the reasoning level when agent.thinking is undefined", () => { + const agentThinking: string | undefined = undefined; + + const lowModel = applyThinkingSuffix("openai/gpt-5:low", agentThinking); + assert.equal(lowModel, "openai/gpt-5:low"); + assert.ok(lowModel?.endsWith(":low")); + assert.equal(resolveEffectiveThinking(lowModel, agentThinking), "low"); + + const highModel = applyThinkingSuffix("anthropic/claude-sonnet-4:high", agentThinking); + assert.equal(highModel, "anthropic/claude-sonnet-4:high"); + assert.ok(highModel?.endsWith(":high")); + assert.equal(resolveEffectiveThinking(highModel, agentThinking), "high"); + }); + + test("candidate-derived suffix is preferred over the suffix-stripped result.model echo", () => { + const candidate = "openai/gpt-5:low"; + const agentThinking: string | undefined = undefined; + // The SDK echoes evt.message.model with the per-candidate suffix stripped. + const resultModel = "openai/gpt-5"; + + const attemptModel = applyThinkingSuffix(candidate, agentThinking) ?? resultModel ?? "default"; + assert.equal(attemptModel, "openai/gpt-5:low"); + assert.equal(resolveEffectiveThinking(attemptModel, agentThinking), "low"); + }); + + test("falls back to result.model when there is no candidate (modelsToTry=[undefined])", () => { + const candidate: string | undefined = undefined; + const agentThinking: string | undefined = undefined; + const resultModel = "openai/gpt-5"; + + const attemptModel = applyThinkingSuffix(candidate, agentThinking) ?? resultModel ?? "default"; + assert.equal(attemptModel, "openai/gpt-5"); + }); +});