diff --git a/README.md b/README.md index 24a47e93d4..6f0d0ad273 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,7 @@ Beginner-friendly guides: Advanced and source-build guides: - [Advanced Setup](docs/advanced-setup.md) +- [Smart Auto-Routing](docs/smart-routing.md) - [Android Install](ANDROID_INSTALL.md) ## Supported Providers diff --git a/docs/smart-routing.md b/docs/smart-routing.md new file mode 100644 index 0000000000..021cce5eaf --- /dev/null +++ b/docs/smart-routing.md @@ -0,0 +1,57 @@ +# Smart auto-routing + +Smart routing is an opt-in mode that classifies each user turn as **simple** or **strong** and sends it to your configured **simple** or **strong** model accordingly, so trivial turns ("ok", "rename this", "what does this do?") can go to a cheaper model while the strong model handles everything non-trivial. Whether the simple role is actually cheaper depends on how your provider bills it. OpenClaude routes to the role you set and does not verify your provider's pricing. + +It is **off by default** and **experimental** — the classifier is a fast heuristic (prompt length, code blocks, reasoning/planning keywords, first turn of a session), not a perfect judge. When in doubt it routes to the strong model, so the failure mode is "no savings on a turn that could have been cheap," never a silently degraded answer on a turn you cared about. + +Smart routing is provider-agnostic: it swaps the model within your current provider. It works against any backend where you have both a cheaper and a stronger model configured. It does not read your provider's, gateway's, or account's pricing, so any savings or cost estimates it shows are based on a first-party reference table and may not match what you are actually billed. + +## Setup + +Both roles point at `agentModels` keys (or bare model ids). For example, in `~/.openclaude.json`: + +```json +{ + "agentModels": { + "mini": { "model": "gpt-5-mini" }, + "main": { "model": "gpt-5" } + }, + "smartRouting": { + "enabled": true, + "simpleModel": "mini", + "strongModel": "main" + } +} +``` + +Optional tuning fields: `simpleMaxChars` and `simpleMaxWords` raise or lower the size threshold for "simple". + +## The `/smartroute` command + +| Command | Effect | +| --- | --- | +| `/smartroute` | Show status (enabled/disabled, the two roles, available `agentModels` keys). | +| `/smartroute on` | Enable (requires both roles set). | +| `/smartroute off` | Disable. | +| `/smartroute simple ` | Set the simple-turn model to an `agentModels` key. | +| `/smartroute strong ` | Set the strong-turn model. | + +When you set roles, the command warns if the simple model is not actually priced below the strong model (for models with known first-party pricing). + +## Environment variables + +These set a startup default. An explicit `smartRouting` block in settings always overrides them. + +| Variable | Meaning | +| --- | --- | +| `OPENCLAUDE_SMART_ROUTING` | `1` or `true` enables routing at startup. | +| `OPENCLAUDE_SMART_ROUTING_SIMPLE` | `agentModels` key or model id for simple turns. | +| `OPENCLAUDE_SMART_ROUTING_STRONG` | `agentModels` key or model id for strong turns. | + +## Behavior notes + +- **One decision per turn.** The model is chosen once when your message arrives and held for the whole turn (including its tool calls), so it does not flap mid-turn. +- **Fallback.** If a simple-routed turn's model call errors (transport or server error), it retries once on the strong model. Aborts and auth/permission/bad-request errors are not retried. +- **Allowlist.** Any model smart routing selects is checked against your org model allowlist (`availableModels`). A disallowed model is coerced to strong; if strong is also disallowed, routing disables itself for the session and the default model is used. Running `/smartroute on` re-enables routing and clears that session disable. +- **Same-provider only.** Roles must be model-only `agentModels` entries (or bare model ids). If a role resolves to a cross-provider entry (one with `base_url`/`api_key`), routing silently disables — cross-provider routing is not supported yet. +- **Auditing.** `/cost` shows a routing summary: how many turns went simple vs strong, how many escalated to strong via fallback, and an estimated savings line when both models appear in the first-party reference pricing table. That estimate is reference pricing only and may not reflect what your provider/gateway/account actually bills. diff --git a/src/commands.ts b/src/commands.ts index a806deb87f..1dc952962f 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -167,6 +167,7 @@ import chrome from './commands/chrome/index.js' import stickers from './commands/stickers/index.js' import advisor from './commands/advisor.js' import ads from './commands/ads.js' +import smartroute from './commands/smartroute/index.js' import { logError } from './utils/log.js' import { toError } from './utils/errors.js' import { logForDebugging } from './utils/debug.js' @@ -278,6 +279,7 @@ const COMMANDS = memoize((): Command[] => [ addDir, advisor, ads, + smartroute, agents, autoFix, branch, diff --git a/src/commands/login/login.tsx b/src/commands/login/login.tsx index 4dcaa87828..319d902511 100644 --- a/src/commands/login/login.tsx +++ b/src/commands/login/login.tsx @@ -1,7 +1,9 @@ import { feature } from 'bun:bundle' import * as React from 'react' -import { resetCostState } from '../../bootstrap/state.js' +// Use the cost-tracker wrapper (not the raw bootstrap reset) so the routing +// tally is cleared alongside cost counters on an account switch. +import { resetCostState } from '../../cost-tracker.js' import { clearTrustedDeviceToken, enrollTrustedDevice, diff --git a/src/commands/smartroute/index.test.ts b/src/commands/smartroute/index.test.ts new file mode 100644 index 0000000000..f47febe3e0 --- /dev/null +++ b/src/commands/smartroute/index.test.ts @@ -0,0 +1,182 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test' +import command from './index.js' +import * as settingsModule from '../../utils/settings/settings.js' +import type { SettingsJson } from '../../utils/settings/types.js' +import type { LocalCommandResult } from '../../types/command.js' + +// The command always returns a `text` result; narrow the union so `.value` is +// accessible without an `as` cast (and assert that contract while we're here). +function expectText(res: LocalCommandResult): Extract { + if (res.type !== 'text') throw new Error(`expected a text result, got ${res.type}`) + return res +} + +// Two model-only agentModels keys with first-party-priced models so the +// cheaper-than warning can be exercised: haiku (cheap) vs opus (expensive). +const AGENT_MODELS = { + mini: { model: 'claude-haiku-4-5' }, + main: { model: 'claude-opus-4-5' }, +} +const SMART_ROUTING_ENV_KEYS = [ + 'OPENCLAUDE_SMART_ROUTING', + 'OPENCLAUDE_SMART_ROUTING_SIMPLE', + 'OPENCLAUDE_SMART_ROUTING_STRONG', +] as const + +function makeContext(initial: Partial = {}) { + let state = { + settings: { agentModels: AGENT_MODELS, ...initial } as SettingsJson, + } + return { + getAppState: () => state as never, + setAppState: (updater: (s: typeof state) => typeof state) => { + state = updater(state) + }, + _state: () => state, + } as unknown as Parameters>['call']>[1] & { + _state: () => typeof state + } +} + +describe('/smartroute command', () => { + let writeSpy: ReturnType + let call: Awaited>['call'] + let savedEnv: Record<(typeof SMART_ROUTING_ENV_KEYS)[number], string | undefined> + + beforeEach(async () => { + savedEnv = Object.fromEntries(SMART_ROUTING_ENV_KEYS.map(key => [key, process.env[key]])) as typeof savedEnv + for (const key of SMART_ROUTING_ENV_KEYS) delete process.env[key] + writeSpy = spyOn(settingsModule, 'updateSettingsForSource').mockImplementation(() => ({ error: null })) + call = (await command.load()).call + }) + afterEach(() => { + writeSpy.mockRestore() + for (const key of SMART_ROUTING_ENV_KEYS) { + const value = savedEnv[key] + if (value == null) delete process.env[key] + else process.env[key] = value + } + }) + + test('status with no config shows disabled and available keys', async () => { + const ctx = makeContext() + const res = expectText(await call('', ctx)) + expect(res.value).toContain('status: disabled') + expect(res.value).toContain('mini, main') + }) + + test('status shows env-backed role values when settings have no smartRouting block', async () => { + process.env.OPENCLAUDE_SMART_ROUTING = '1' + process.env.OPENCLAUDE_SMART_ROUTING_SIMPLE = 'mini' + process.env.OPENCLAUDE_SMART_ROUTING_STRONG = 'main' + const ctx = makeContext() + const res = expectText(await call('', ctx)) + expect(res.value).toContain('status: enabled') + expect(res.value).toContain('simple: mini') + expect(res.value).toContain('strong: main') + }) + + test('on without both roles set is rejected', async () => { + const ctx = makeContext({ smartRouting: { enabled: false, simpleModel: 'mini' } }) + const res = expectText(await call('on', ctx)) + expect(res.value).toContain('Set both roles first') + expect(writeSpy).not.toHaveBeenCalled() + }) + + test('on accepts env-backed roles and persists the normalized settings block', async () => { + process.env.OPENCLAUDE_SMART_ROUTING = '1' + process.env.OPENCLAUDE_SMART_ROUTING_SIMPLE = 'mini' + process.env.OPENCLAUDE_SMART_ROUTING_STRONG = 'main' + const ctx = makeContext() + const res = expectText(await call('on', ctx)) + expect(res.value).toContain('Smart routing enabled') + expect(writeSpy).toHaveBeenCalledWith('userSettings', { + smartRouting: { enabled: true, simpleModel: 'mini', strongModel: 'main' }, + }) + expect(ctx._state().settings.smartRouting).toEqual({ + enabled: true, + simpleModel: 'mini', + strongModel: 'main', + }) + }) + + test('setting simple/strong to a valid key persists', async () => { + const ctx = makeContext() + await call('simple mini', ctx) + expect(writeSpy).toHaveBeenCalledWith('userSettings', { smartRouting: { simpleModel: 'mini' } }) + expect((ctx as never as { _state: () => { settings: SettingsJson } })._state().settings.smartRouting).toEqual({ + simpleModel: 'mini', + }) + }) + + test('setting a role reports persistence errors without mutating app state', async () => { + writeSpy.mockImplementation(() => ({ error: new Error('settings are read-only') })) + const ctx = makeContext() + const res = expectText(await call('simple mini', ctx)) + expect(res.value).toContain('Failed to update smart routing settings: settings are read-only') + expect(ctx._state().settings.smartRouting).toBeUndefined() + }) + + test('setting the strong role to a valid key persists', async () => { + const ctx = makeContext() + await call('strong main', ctx) + expect(writeSpy).toHaveBeenCalledWith('userSettings', { smartRouting: { strongModel: 'main' } }) + }) + + test('setting one role preserves env-backed defaults instead of shadowing them with a partial block', async () => { + process.env.OPENCLAUDE_SMART_ROUTING = '1' + process.env.OPENCLAUDE_SMART_ROUTING_SIMPLE = 'mini' + process.env.OPENCLAUDE_SMART_ROUTING_STRONG = 'main' + const ctx = makeContext() + await call('simple main', ctx) + expect(writeSpy).toHaveBeenCalledWith('userSettings', { + smartRouting: { enabled: true, simpleModel: 'main', strongModel: 'main' }, + }) + }) + + test('simple/strong with no value argument is rejected', async () => { + const ctx = makeContext() + const res = expectText(await call('simple', ctx)) + expect(res.value).toContain('Specify an agentModels key') + expect(writeSpy).not.toHaveBeenCalled() + }) + + test('setting a role to an unknown key is rejected with available keys', async () => { + const ctx = makeContext() + const res = expectText(await call('simple nope', ctx)) + expect(res.value).toContain('not a configured agentModels key') + expect(res.value).toContain('mini, main') + expect(writeSpy).not.toHaveBeenCalled() + }) + + test('enabling with simple cheaper than strong gives no warning', async () => { + const ctx = makeContext({ smartRouting: { enabled: false, simpleModel: 'mini', strongModel: 'main' } }) + const res = expectText(await call('on', ctx)) + expect(res.value).toContain('Smart routing enabled') + expect(res.value).not.toContain('Heads up') + expect(writeSpy).toHaveBeenCalledWith('userSettings', { + smartRouting: { enabled: true, simpleModel: 'mini', strongModel: 'main' }, + }) + }) + + test('warns when the simple model is not cheaper than the strong model', async () => { + // Swap roles: simple=opus (expensive), strong=haiku (cheap). + const ctx = makeContext({ smartRouting: { enabled: false, simpleModel: 'main', strongModel: 'mini' } }) + const res = expectText(await call('on', ctx)) + expect(res.value).toContain('Heads up') + expect(res.value).toContain('not cheaper') + // The warning must be hedged as first-party reference pricing, since the + // active provider may bill these models differently (jatmn P2). + expect(res.value).toContain('first-party reference pricing') + expect(res.value).toContain('provider may bill differently') + }) + + test('off disables', async () => { + const ctx = makeContext({ smartRouting: { enabled: true, simpleModel: 'mini', strongModel: 'main' } }) + const res = expectText(await call('off', ctx)) + expect(res.value).toContain('disabled') + expect(writeSpy).toHaveBeenCalledWith('userSettings', { + smartRouting: { enabled: false, simpleModel: 'mini', strongModel: 'main' }, + }) + }) +}) diff --git a/src/commands/smartroute/index.ts b/src/commands/smartroute/index.ts new file mode 100644 index 0000000000..6bd29b88ae --- /dev/null +++ b/src/commands/smartroute/index.ts @@ -0,0 +1,134 @@ +import type { Command } from '../../commands.js' +import type { LocalCommandCall } from '../../types/command.js' +import { updateSettingsForSource } from '../../utils/settings/settings.js' +import type { SettingsJson } from '../../utils/settings/types.js' +import { readSmartRouting } from '../../services/api/smartRouting/settings.js' +import { + clearSmartRoutingSessionDisable, + getKnownInputCost, + isSmartRoutingDisabledForSession, + resolveSmartRoutingRoleModelString, +} from '../../services/api/smartRouting/index.js' +import { getSessionId } from '../../bootstrap/state.js' + +type SmartRoutingSettings = NonNullable + +const HELP = + 'Usage:\n' + + ' /smartroute show status\n' + + ' /smartroute on|off enable / disable\n' + + ' /smartroute simple \n' + + ' /smartroute strong ' + +function text(value: string) { + return { type: 'text' as const, value } +} + +/** + * Warn when both roles have first-party reference pricing and, by that pricing, + * simple is not actually cheaper. The numbers are first-party list prices, not + * the active provider's, so the warning is hedged accordingly. + */ +function cheaperWarning(s: SmartRoutingSettings, settings: SettingsJson | null): string { + const simple = getKnownInputCost(resolveSmartRoutingRoleModelString(s.simpleModel, settings) ?? '') + const strong = getKnownInputCost(resolveSmartRoutingRoleModelString(s.strongModel, settings) ?? '') + if (simple != null && strong != null && simple >= strong) { + return `\nHeads up: by first-party reference pricing the simple model is not cheaper than the strong model (${simple} vs ${strong} per Mtok input); your provider may bill differently. Smart routing may not save money.` + } + return '' +} + +function formatPersistError(error: Error): string { + return `Failed to update smart routing settings: ${error.message}` +} + +function readCurrentSmartRouting(settings: SettingsJson): SmartRoutingSettings { + if (settings.smartRouting !== undefined) return { ...settings.smartRouting } + const normalized = readSmartRouting(settings) + if (!normalized.enabled) return {} + return { ...normalized } +} + +const call: LocalCommandCall = async (args, context) => { + const arg = args.trim() + const settings = context.getAppState().settings as unknown as SettingsJson + const current = readCurrentSmartRouting(settings) + const agentModelKeys = Object.keys(settings?.agentModels ?? {}) + + const persist = (next: SmartRoutingSettings): Error | null => { + const { error } = updateSettingsForSource('userSettings', { smartRouting: next }) + if (error) return error + context.setAppState(s => ({ + ...s, + settings: { ...s.settings, smartRouting: next }, + })) + return null + } + + // Status (no args). + if (!arg) { + const normalized = readSmartRouting(settings) + const disabledForSession = isSmartRoutingDisabledForSession(getSessionId()) + const lines = [ + 'Smart routing (experimental)', + ` status: ${normalized.enabled ? 'enabled' : 'disabled'}${ + disabledForSession ? ' (auto-disabled this session: both models outside the org allowlist)' : '' + }`, + ` simple: ${normalized.simpleModel ?? '(unset)'}`, + ` strong: ${normalized.strongModel ?? '(unset)'}`, + ] + if (agentModelKeys.length > 0) lines.push(` available agentModels keys: ${agentModelKeys.join(', ')}`) + return text(lines.join('\n') + '\n\n' + HELP) + } + + const [sub, value] = arg.split(/\s+/, 2) + const lower = sub.toLowerCase() + + if (lower === 'on') { + if (!current.strongModel || !current.simpleModel) { + return text('Set both roles first: /smartroute simple and /smartroute strong .') + } + const next = { ...current, enabled: true } + const error = persist(next) + if (error) return text(formatPersistError(error)) + // Re-enabling clears any session auto-disable. + clearSmartRoutingSessionDisable(getSessionId()) + return text(`Smart routing enabled (simple=${next.simpleModel}, strong=${next.strongModel}).${cheaperWarning(next, settings)}`) + } + + if (lower === 'off') { + const error = persist({ ...current, enabled: false }) + if (error) return text(formatPersistError(error)) + return text('Smart routing disabled.') + } + + if (lower === 'simple' || lower === 'strong') { + if (!value) return text(`Specify an agentModels key: /smartroute ${lower} .`) + if (!agentModelKeys.includes(value)) { + return text( + `"${value}" is not a configured agentModels key.` + + (agentModelKeys.length ? ` Available: ${agentModelKeys.join(', ')}.` : ' Configure agentModels first.'), + ) + } + const next: SmartRoutingSettings = + lower === 'simple' ? { ...current, simpleModel: value } : { ...current, strongModel: value } + const error = persist(next) + if (error) return text(formatPersistError(error)) + return text(`Set ${lower} model to "${value}".${cheaperWarning(next, settings)}`) + } + + return text(HELP) +} + +const smartroute = { + type: 'local', + name: 'smartroute', + description: 'Configure smart auto-routing (experimental): route simple turns to your configured simple model', + argumentHint: '[on|off|simple |strong ]', + isEnabled: () => true, + isHidden: false, + supportsNonInteractive: true, + load: () => Promise.resolve({ call }), +} satisfies Command + +export default smartroute diff --git a/src/cost-tracker.ts b/src/cost-tracker.ts index 5eaa4cbda4..73864f054b 100644 --- a/src/cost-tracker.ts +++ b/src/cost-tracker.ts @@ -9,6 +9,8 @@ import { resetSessionCacheStats, } from './services/api/cacheStatsTracker.js' import { getAPIProvider, isGithubNativeAnthropicMode } from './utils/model/providers.js' +import { getRoutingSummaryForDisplay, resetRoutingTally } from './services/api/smartRouting/index.js' +import { getSettings_DEPRECATED } from './utils/settings/settings.js' import { addToTotalCostState, addToTotalLinesChanged, @@ -83,6 +85,9 @@ export { export function resetCostState(): void { baseResetCostState() resetSessionCacheStats() + // Keep the routing summary scoped to the same session window as the cost + // block it renders beside — otherwise /cost shows stale cross-session counts. + resetRoutingTally() } type StoredCostState = { @@ -286,9 +291,12 @@ Total duration (wall): ${formatDuration(getTotalDuration())} Total code changes: ${getTotalLinesAdded()} ${getTotalLinesAdded() === 1 ? 'line' : 'lines'} added, ${getTotalLinesRemoved()} ${getTotalLinesRemoved() === 1 ? 'line' : 'lines'} removed`, ) + const routingSummary = getRoutingSummaryForDisplay(getSettings_DEPRECATED()) + const routingSection = routingSummary ? `\n\n${chalk.dim(routingSummary)}` : '' + return `${statsBlock}${tokenSection} -${modelUsageDisplay}` +${modelUsageDisplay}${routingSection}` } function round(number: number, precision: number): number { diff --git a/src/query.ts b/src/query.ts index 70cf690199..2d43e7a826 100644 --- a/src/query.ts +++ b/src/query.ts @@ -114,7 +114,7 @@ import { queryCheckpoint } from './utils/queryProfiler.js' import { runTools } from './services/tools/toolOrchestration.js' import { applyToolResultBudget } from './utils/toolResultStorage.js' import { resolveNextFallbackProviderFromState } from './utils/providerFallback.js' -import { setActiveProviderProfile } from './utils/providerProfiles.js' +import { setActiveProviderProfile, getActiveProviderProfile } from './utils/providerProfiles.js' import { getPrimaryModel } from './utils/providerModels.js' import { recordContentReplacement } from './utils/sessionStorage.js' import { handleStopHooks } from './query/stopHooks.js' @@ -135,7 +135,20 @@ import { getCurrentTurnTokenBudget, getTurnOutputTokens, incrementBudgetContinuationCount, + getSessionId, } from './bootstrap/state.js' +import { stripThinkingBlocksIfProviderAllows } from './utils/conversationRecovery.js' +import { + decideTurnModel, + deriveUserTurnNumber, + extractLatestUserText, + isRetryableRoutedModelError, + latestUserMessageHasNonTextContent, + recordRoutingDecision, + recordRoutingEscalation, + shouldDropPinForProviderSwap, + type TurnRoutingDecision, +} from './services/api/smartRouting/index.js' import { createBudgetTracker, checkTokenBudget } from './query/tokenBudget.js' import { count } from './utils/array.js' /* eslint-disable @typescript-eslint/no-require-imports */ @@ -555,6 +568,16 @@ async function* queryLoop( // trigger point. Loop-local (not on State) to avoid touching the 7 continue // sites. let taskBudgetRemaining: number | undefined = undefined + // Smart-routing decision, pinned once per user turn (transition===undefined) + // and reused on every continuation pass. Loop-local (not on State) so it + // survives the State rebuilds at the continue sites for free — mirrors + // taskBudgetRemaining above. + let pinnedTurnRoute: TurnRoutingDecision | undefined = undefined + // Provider profile the pinned route's model was resolved against. If a + // mid-turn provider-fallback swap changes the active provider, the pinned + // model (a model-only route keyed to the old provider) must not be replayed + // at the new endpoint — KTD6 in the plan. + let pinnedRouteProviderId: string | undefined = undefined const toolFailureGuardState = createToolFailureLoopGuardState() // Snapshot immutable env/statsig/session state once at entry. See QueryConfig @@ -979,6 +1002,62 @@ async function* queryLoop( doesMostRecentAssistantMessageExceed200k(messagesForQuery), }) + // Smart routing (opt-in): classify once per user turn (transition===undefined) + // and pin the decision; reuse the pin on every continuation pass. Applied + // BEFORE the blocking-limit math below so the token-budget guard and the + // model call agree on the model. Disabled/misconfigured → pin is `routed:false` + // and currentModel keeps today's resolution (byte-for-byte unchanged). + if (state.transition === undefined) { + pinnedTurnRoute = decideTurnModel({ + settings: appState.settings as unknown as Parameters[0]['settings'], + parentModel: currentModel, + permissionMode, + input: { + userText: extractLatestUserText(messagesForQuery), + hasNonTextContent: latestUserMessageHasNonTextContent(messagesForQuery), + turnNumber: deriveUserTurnNumber(messagesForQuery), + }, + sessionId: getSessionId(), + }) + if (pinnedTurnRoute.routed === false && pinnedTurnRoute.justDisabledForSession) { + yield createSystemMessage( + 'Smart routing disabled for this session: both configured models are outside the org allowlist. Using the default model.', + 'warning', + ) + } + if (pinnedTurnRoute.routed) { + recordRoutingDecision(pinnedTurnRoute.complexity) + pinnedRouteProviderId = getActiveProviderProfile()?.id + } + } else if ( + shouldDropPinForProviderSwap( + pinnedTurnRoute, + pinnedRouteProviderId, + getActiveProviderProfile()?.id, + ) + ) { + // A provider-fallback swap happened mid-turn: the pinned model belongs to + // the previous provider. Drop the pin and let today's resolution (already + // re-derived to the new provider's model above) stand for the rest of the + // turn rather than sending a stale model id to the new endpoint. + pinnedTurnRoute = undefined + } + // Apply whatever pin survived the guard above (may be undefined after an + // invalidation, in which case currentModel keeps today's resolution). + if (pinnedTurnRoute?.routed) { + const priorModel = currentModel + currentModel = pinnedTurnRoute.model + toolUseContext.options.mainLoopModel = pinnedTurnRoute.model + // A model change at the turn boundary would replay a prior model's + // thinking signature; strip it under the provider gate (never for + // preserve-reasoning providers, which 400 on a stripped block). + if (pinnedTurnRoute.model !== priorModel) { + messagesForQuery = stripThinkingBlocksIfProviderAllows( + messagesForQuery as unknown as Parameters[0], + ) as unknown as typeof messagesForQuery + } + } + queryCheckpoint('query_setup_end') // Create fetch wrapper once per query session to avoid memory retention. @@ -1127,6 +1206,13 @@ async function* queryLoop( const toolsForModel = agentStepLimit?.summaryRequested ? [] : toolUseContext.options.tools + // Once-only guard for the smart-routing routed-error fallback (U4): a + // simple-routed call that errors retries once on the strong model; a second + // failure propagates normally rather than re-routing. Intentionally scoped + // per user turn (here, outside the while(attemptWithFallback) retry loop) — + // moving it inside would reset it every attempt and defeat the once-only + // guarantee. + let routedFallbackUsed = false queryCheckpoint('query_api_loop_start') try { @@ -1461,6 +1547,64 @@ async function* queryLoop( continue } + // Smart-routing routed-error fallback (U4): a simple-routed call that + // errors retries once on the strong model. Reuses this same + // attemptWithFallback retry loop — not a new retry mechanism. Aborts + // and 4xx client errors (auth/permission/bad-request) are NOT retried. + if ( + pinnedTurnRoute?.routed && + pinnedTurnRoute.complexity === 'simple' && + !routedFallbackUsed && + !(innerError instanceof FallbackTriggeredError) && + !toolUseContext.abortController.signal.aborted && + isRetryableRoutedModelError(innerError) + ) { + const strongModel = pinnedTurnRoute.strongModel + routedFallbackUsed = true + attemptWithFallback = true + recordRoutingEscalation() + // Re-pin to strong so this turn's later continuation passes (next_turn) + // don't re-route to the failing simple model and fall back again. + pinnedTurnRoute = { + routed: true, + model: strongModel, + complexity: 'strong', + reason: 'fell back from simple model', + strongModel, + } + + yield* yieldMissingToolResultBlocks( + assistantMessages, + 'Smart-routing fallback to strong model', + ) + assistantMessages.length = 0 + toolResults.length = 0 + toolUseBlocks.length = 0 + needsFollowUp = false + + if (streamingToolExecutor) { + streamingToolExecutor.discard() + streamingToolExecutor = new StreamingToolExecutor( + toolUseContext.options.tools, + canUseTool, + toolUseContext, + ) + } + + currentModel = strongModel + toolUseContext.options.mainLoopModel = strongModel + // Strip prior-model thinking before retrying on the strong model, + // under the provider gate (never for preserve-reasoning providers). + messagesForQuery = stripThinkingBlocksIfProviderAllows( + messagesForQuery as unknown as Parameters[0], + ) as unknown as typeof messagesForQuery + + yield createSystemMessage( + `Smart routing: retrying on ${renderModelName(strongModel)} after the simple model failed`, + 'warning', + ) + continue + } throw innerError } } diff --git a/src/services/api/agentRouting.ts b/src/services/api/agentRouting.ts index 01eb464b2c..8f46da8d71 100644 --- a/src/services/api/agentRouting.ts +++ b/src/services/api/agentRouting.ts @@ -160,7 +160,7 @@ export function resolveAgentModelProvider( * sent literally and failing with a provider "model not found". A real model id * (a configured agentModels key for the active provider) passes through as-is. */ -function resolveModelOnlyModel( +export function resolveModelOnlyModel( model: string, parentModel: string, permissionMode?: PermissionMode, diff --git a/src/services/api/openaiShim.ts b/src/services/api/openaiShim.ts index 8ecbf30a05..94ab7e1f8e 100644 --- a/src/services/api/openaiShim.ts +++ b/src/services/api/openaiShim.ts @@ -20,6 +20,11 @@ * OPENAI_MODEL=gpt-4o — default model override * CODEX_API_KEY / ~/.codex/auth.json — Codex auth for codexplan/codexspark * + * Smart auto-routing (opt-in; startup defaults, overridden by settings.smartRouting): + * OPENCLAUDE_SMART_ROUTING=1|true — route simple turns to a cheaper model + * OPENCLAUDE_SMART_ROUTING_SIMPLE= — agentModels key or model id for simple turns + * OPENCLAUDE_SMART_ROUTING_STRONG= — agentModels key or model id for strong turns + * * GitHub Copilot API (api.githubcopilot.com), OpenAI-compatible: * CLAUDE_CODE_USE_GITHUB=1 — enable GitHub inference (no need for USE_OPENAI) * GITHUB_TOKEN or GH_TOKEN — Copilot API token (mapped to Bearer auth) diff --git a/src/services/api/smartModelRouting.ts b/src/services/api/smartModelRouting.ts index 65f378de06..f2e4670e4b 100644 --- a/src/services/api/smartModelRouting.ts +++ b/src/services/api/smartModelRouting.ts @@ -38,6 +38,8 @@ export type RoutingDecision = { export type RoutingInput = { /** The user's message text for this turn. */ userText: string + /** True when the latest user turn includes image/document or other non-text blocks. */ + hasNonTextContent?: boolean /** * Optional: how many tool-use blocks the assistant has emitted in the * recent conversation. High values correlate with "continue this work" @@ -146,6 +148,14 @@ export function routeModel( const text = input.userText ?? '' const trimmed = text.trim() + if (input.hasNonTextContent) { + return { + model: config.strongModel, + complexity: 'strong', + reason: 'contains non-text content', + } + } + if (!trimmed) { // Empty input (e.g. resuming a tool-use chain) — cheap by default. return { diff --git a/src/services/api/smartRouting/index.test.ts b/src/services/api/smartRouting/index.test.ts new file mode 100644 index 0000000000..7bbf6e8dfb --- /dev/null +++ b/src/services/api/smartRouting/index.test.ts @@ -0,0 +1,395 @@ +import { afterEach, describe, expect, spyOn, test } from 'bun:test' +import { + clearSmartRoutingSessionDisable, + decideTurnModel, + deriveUserTurnNumber, + extractLatestUserText, + formatRoutingSummary, + getRoutingSummaryForDisplay, + getRoutingTally, + isRetryableRoutedModelError, + isSmartRoutingDisabledForSession, + latestUserMessageHasNonTextContent, + recordRoutingDecision, + recordRoutingEscalation, + resetRoutingTally, + shouldDropPinForProviderSwap, + type TurnRoutingDecision, +} from './index.js' +import * as modelAllowlistModule from '../../../utils/model/modelAllowlist.js' +import type { SettingsJson } from '../../../utils/settings/types.js' + +// Control isModelAllowed directly rather than the global settings singleton it +// reads. This keeps these tests deterministic even when another test file leaks +// a mock.module of modelAllowlist (e.g. agent.test.ts), and the afterEach +// restore guarantees we never leak our own spy if an assertion throws first. +let activeAllowlistSpy: ReturnType | undefined +afterEach(() => { + activeAllowlistSpy?.mockRestore() + activeAllowlistSpy = undefined +}) + +/** Force isModelAllowed to allow only the given models (exact membership). */ +function mockGlobalAllowlist(availableModels: string[] | undefined) { + activeAllowlistSpy = spyOn(modelAllowlistModule, 'isModelAllowed').mockImplementation( + (model: string) => (availableModels ? availableModels.includes(model) : true), + ) + return activeAllowlistSpy +} + +const PARENT = 'gpt-5' + +function settings(overrides: Record): SettingsJson { + return overrides as unknown as SettingsJson +} + +// Two model-only agentModels keys + an opt-in smartRouting block. No availableModels +// allowlist, so isModelAllowed returns true for everything. +function enabledSettings(extra: Record = {}): SettingsJson { + return settings({ + agentModels: { mini: { model: 'gpt-5-mini' }, main: { model: 'gpt-5' } }, + smartRouting: { enabled: true, simpleModel: 'mini', strongModel: 'main' }, + ...extra, + }) +} + +const userMsg = (text: string, isMeta = false) => ({ + type: 'user', + isMeta, + message: { role: 'user', content: text }, +}) +const toolResultMsg = () => ({ + type: 'user', + message: { role: 'user', content: [{ type: 'tool_result', content: 'ok' }] }, +}) +const imageMsg = () => ({ + type: 'user', + message: { + role: 'user', + content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'abc' } }], + }, +}) +const assistantMsg = () => ({ type: 'assistant', message: { role: 'assistant', content: 'hi' } }) + +describe('deriveUserTurnNumber', () => { + test('counts only real user messages (not isMeta, not tool-results)', () => { + const msgs = [ + userMsg('first real turn'), + assistantMsg(), + toolResultMsg(), + userMsg('continue', true), // isMeta nudge + userMsg('second real turn'), + ] + expect(deriveUserTurnNumber(msgs)).toBe(2) + }) + + test('empty conversation is zero', () => { + expect(deriveUserTurnNumber([])).toBe(0) + }) +}) + +describe('extractLatestUserText', () => { + test('returns the most recent real user message text', () => { + const msgs = [userMsg('old'), assistantMsg(), userMsg('newest')] + expect(extractLatestUserText(msgs)).toBe('newest') + }) + + test('skips isMeta and tool-result messages', () => { + const msgs = [userMsg('the real one'), assistantMsg(), toolResultMsg(), userMsg('nudge', true)] + expect(extractLatestUserText(msgs)).toBe('the real one') + }) + + test('joins text blocks of array content', () => { + const msgs = [{ type: 'user', message: { role: 'user', content: [{ type: 'text', text: 'a' }, { type: 'text', text: 'b' }] } }] + expect(extractLatestUserText(msgs)).toBe('a\nb') + }) +}) + +describe('latestUserMessageHasNonTextContent', () => { + test('detects image/document-style blocks on the latest real user turn', () => { + expect(latestUserMessageHasNonTextContent([userMsg('old'), imageMsg()])).toBe(true) + expect( + latestUserMessageHasNonTextContent([ + imageMsg(), + { type: 'user', message: { content: [{ type: 'text', text: 'plain follow-up' }] } }, + ]), + ).toBe(false) + }) + + test('skips meta and tool-result carriers', () => { + const msgs = [userMsg('plain'), imageMsg(), toolResultMsg(), userMsg('nudge', true)] + expect(latestUserMessageHasNonTextContent(msgs)).toBe(true) + }) +}) + +describe('decideTurnModel', () => { + afterEach(() => { + clearSmartRoutingSessionDisable('sess-1') + clearSmartRoutingSessionDisable('sess-2') + }) + + test('disabled settings → routed:false', () => { + const d = decideTurnModel({ + settings: settings({}), + parentModel: PARENT, + input: { userText: 'hi', turnNumber: 2 }, + }) + expect(d.routed).toBe(false) + }) + + test('short non-first turn routes simple', () => { + mockGlobalAllowlist(undefined) // allow all; immune to a leaked cross-file allowlist mock + const d = decideTurnModel({ + settings: enabledSettings(), + parentModel: PARENT, + input: { userText: 'ok thanks', turnNumber: 3 }, + }) + expect(d).toMatchObject({ routed: true, complexity: 'simple', model: 'gpt-5-mini', strongModel: 'gpt-5' }) + }) + + test('first turn routes strong (routeModel turnNumber===1 guard)', () => { + mockGlobalAllowlist(undefined) + const d = decideTurnModel({ + settings: enabledSettings(), + parentModel: PARENT, + input: { userText: 'ok', turnNumber: 1 }, + }) + expect(d).toMatchObject({ routed: true, complexity: 'strong', model: 'gpt-5' }) + }) + + test('strong-signal prompt routes strong', () => { + mockGlobalAllowlist(undefined) + const d = decideTurnModel({ + settings: enabledSettings(), + parentModel: PARENT, + input: { userText: 'refactor the auth module please', turnNumber: 4 }, + }) + expect(d).toMatchObject({ routed: true, complexity: 'strong', model: 'gpt-5' }) + }) + + test('non-text user content routes strong even when the text extractor is empty', () => { + mockGlobalAllowlist(undefined) + const d = decideTurnModel({ + settings: enabledSettings(), + parentModel: PARENT, + input: { userText: '', hasNonTextContent: true, turnNumber: 4 }, + }) + expect(d).toMatchObject({ routed: true, complexity: 'strong', model: 'gpt-5' }) + }) + + test('disallowed simple model coerces to strong', () => { + // Distinct, non-prefix-colliding model ids so the allowlist genuinely blocks + // simple while permitting strong. + const s = settings({ + agentModels: { mini: { model: 'alpha-mini' }, main: { model: 'beta-big' } }, + smartRouting: { enabled: true, simpleModel: 'mini', strongModel: 'main' }, + }) + const spy = mockGlobalAllowlist(['beta-big']) + const d = decideTurnModel({ + settings: s, + parentModel: PARENT, + input: { userText: 'ok thanks', turnNumber: 3 }, + }) + expect(d).toMatchObject({ routed: true, model: 'beta-big', complexity: 'strong' }) + spy.mockRestore() + }) + + test('both models disallowed → routing disabled for session, fires once', () => { + const spy = mockGlobalAllowlist(['some-other-model']) + const cfg = { + settings: enabledSettings(), + parentModel: PARENT, + input: { userText: 'ok thanks', turnNumber: 3 }, + sessionId: 'sess-1', + } + const first = decideTurnModel(cfg) + expect(first).toEqual({ routed: false, justDisabledForSession: true }) + expect(isSmartRoutingDisabledForSession('sess-1')).toBe(true) + + // Second call: still disabled, but the one-time flag is not re-raised. + const second = decideTurnModel(cfg) + expect(second).toEqual({ routed: false }) + spy.mockRestore() + }) + + test('both disallowed with no sessionId stays silent (no per-turn notice storm)', () => { + const spy = mockGlobalAllowlist(['some-other-model']) + const d = decideTurnModel({ + settings: enabledSettings(), + parentModel: PARENT, + input: { userText: 'ok thanks', turnNumber: 3 }, + // no sessionId + }) + expect(d).toEqual({ routed: false }) + spy.mockRestore() + }) + + test('a session disable does not leak into another session', () => { + const spy = mockGlobalAllowlist(['x']) + decideTurnModel({ + settings: enabledSettings(), + parentModel: PARENT, + input: { userText: 'ok', turnNumber: 3 }, + sessionId: 'sess-1', + }) + expect(isSmartRoutingDisabledForSession('sess-1')).toBe(true) + expect(isSmartRoutingDisabledForSession('sess-2')).toBe(false) + spy.mockRestore() + }) + + test('clearSmartRoutingSessionDisable re-enables a disabled session (the /smartroute on path)', () => { + const spy = mockGlobalAllowlist(['x']) + decideTurnModel({ + settings: enabledSettings(), + parentModel: PARENT, + input: { userText: 'ok', turnNumber: 3 }, + sessionId: 'sess-1', + }) + expect(isSmartRoutingDisabledForSession('sess-1')).toBe(true) + clearSmartRoutingSessionDisable('sess-1') + expect(isSmartRoutingDisabledForSession('sess-1')).toBe(false) + spy.mockRestore() + }) +}) + +describe('isRetryableRoutedModelError', () => { + test('4xx client errors (bad request / auth / permission) are not retryable', () => { + expect(isRetryableRoutedModelError({ status: 400 })).toBe(false) + expect(isRetryableRoutedModelError({ status: 401 })).toBe(false) + expect(isRetryableRoutedModelError({ statusCode: 403 })).toBe(false) + }) + + test('404 and 429 are retryable by design (the fallback switches to a different model)', () => { + expect(isRetryableRoutedModelError({ status: 404 })).toBe(true) + expect(isRetryableRoutedModelError({ status: 429 })).toBe(true) + }) + + test('5xx, network, and unclassified errors are retryable', () => { + expect(isRetryableRoutedModelError({ status: 500 })).toBe(true) + expect(isRetryableRoutedModelError({ status: 529 })).toBe(true) + expect(isRetryableRoutedModelError(new Error('socket hang up'))).toBe(true) + expect(isRetryableRoutedModelError(undefined)).toBe(true) + }) +}) + +describe('shouldDropPinForProviderSwap', () => { + const routed: TurnRoutingDecision = { + routed: true, + model: 'mini', + complexity: 'simple', + reason: 'x', + strongModel: 'main', + } + + test('no pin -> never drop', () => { + expect(shouldDropPinForProviderSwap(undefined, 'p1', 'p2')).toBe(false) + }) + + test('routed pin, same provider -> keep', () => { + expect(shouldDropPinForProviderSwap(routed, 'p1', 'p1')).toBe(false) + }) + + test('routed pin, provider changed -> drop', () => { + expect(shouldDropPinForProviderSwap(routed, 'p1', 'p2')).toBe(true) + }) + + test('no provider profiles (both undefined) -> keep', () => { + expect(shouldDropPinForProviderSwap(routed, undefined, undefined)).toBe(false) + }) + + test('non-routed pin -> never drop', () => { + expect(shouldDropPinForProviderSwap({ routed: false }, 'p1', 'p2')).toBe(false) + }) +}) + +describe('routing tally', () => { + afterEach(() => resetRoutingTally()) + + test('records decisions and escalations', () => { + resetRoutingTally() + recordRoutingDecision('simple') + recordRoutingDecision('simple') + recordRoutingDecision('strong') + recordRoutingEscalation() + expect(getRoutingTally()).toEqual({ simple: 2, strong: 1, escalations: 1 }) + }) + + test('reset clears the tally', () => { + recordRoutingDecision('simple') + resetRoutingTally() + expect(getRoutingTally()).toEqual({ simple: 0, strong: 0, escalations: 0 }) + }) +}) + +describe('formatRoutingSummary', () => { + test('null when nothing routed', () => { + expect(formatRoutingSummary({ simple: 0, strong: 0, escalations: 0 })).toBeNull() + }) + + test('shows split and escalations', () => { + const out = formatRoutingSummary({ simple: 5, strong: 2, escalations: 1 }) + expect(out).toContain('5 simple, 2 strong') + expect(out).toContain('1 escalated to strong') + }) + + test('estimated savings line when both models priced and simple cheaper', () => { + const out = formatRoutingSummary({ simple: 3, strong: 1, escalations: 0 }, { + simpleInputCost: 1, + strongInputCost: 5, + }) + expect(out).toContain('~80% lower') + // The estimate must disclose it is first-party reference pricing, not the + // active provider's billed rate (jatmn P2). + expect(out).toContain('first-party reference pricing') + expect(out).toContain('may bill differently') + }) + + test('savings unavailable when a price is unknown', () => { + const out = formatRoutingSummary({ simple: 3, strong: 1, escalations: 0 }, { strongInputCost: 5 }) + expect(out).toContain('Estimated savings unavailable') + // The unavailable line must name first-party reference pricing as the source. + expect(out).toContain('no first-party reference pricing') + }) + + test('notes no savings when simple is not cheaper', () => { + const out = formatRoutingSummary({ simple: 3, strong: 1, escalations: 0 }, { + simpleInputCost: 5, + strongInputCost: 5, + }) + expect(out).toContain('not cheaper') + // The not-cheaper branch must carry the same first-party reference hedge. + expect(out).toContain('first-party reference pricing') + expect(out).toContain('may bill differently') + }) +}) + +describe('getRoutingSummaryForDisplay', () => { + afterEach(() => resetRoutingTally()) + + test('uses env-backed smart-routing roles for pricing display', () => { + const previous = { + OPENCLAUDE_SMART_ROUTING: process.env.OPENCLAUDE_SMART_ROUTING, + OPENCLAUDE_SMART_ROUTING_SIMPLE: process.env.OPENCLAUDE_SMART_ROUTING_SIMPLE, + OPENCLAUDE_SMART_ROUTING_STRONG: process.env.OPENCLAUDE_SMART_ROUTING_STRONG, + } + try { + process.env.OPENCLAUDE_SMART_ROUTING = '1' + process.env.OPENCLAUDE_SMART_ROUTING_SIMPLE = 'mini' + process.env.OPENCLAUDE_SMART_ROUTING_STRONG = 'main' + recordRoutingDecision('simple') + const out = getRoutingSummaryForDisplay( + settings({ + agentModels: { mini: { model: 'claude-haiku-4-5' }, main: { model: 'claude-opus-4-5' } }, + }), + ) + expect(out).toContain('Smart routing: 1 simple, 0 strong') + expect(out).toContain('first-party reference pricing') + expect(out).not.toContain('Estimated savings unavailable') + } finally { + for (const [key, value] of Object.entries(previous)) { + if (value == null) delete process.env[key] + else process.env[key] = value + } + } + }) +}) diff --git a/src/services/api/smartRouting/index.ts b/src/services/api/smartRouting/index.ts new file mode 100644 index 0000000000..71dc57fae7 --- /dev/null +++ b/src/services/api/smartRouting/index.ts @@ -0,0 +1,295 @@ +import type { PermissionMode } from '../../../utils/permissions/PermissionMode.js' +import type { SettingsJson } from '../../../utils/settings/types.js' +import { isModelAllowed } from '../../../utils/model/modelAllowlist.js' +import { getCanonicalName } from '../../../utils/model/model.js' +import { MODEL_COSTS } from '../../../utils/modelCost.js' +import { routeModel, type RoutingInput } from '../smartModelRouting.js' +import { readSmartRouting, type NormalizedSmartRouting } from './settings.js' +import { resolveSmartRoutingConfig } from './resolveConfig.js' + +/** + * Per-million input-token price for a model when it is in the first-party + * pricing table, else `undefined` (third-party / gateway models have no known + * price). Used for the "simple isn't cheaper" warning (U5) and to gate the + * estimated-savings line (U6). + */ +export function getKnownInputCost(model: string): number | undefined { + return MODEL_COSTS[getCanonicalName(model)]?.inputTokens +} + +export { readSmartRouting, type NormalizedSmartRouting } +export { resolveSmartRoutingConfig } from './resolveConfig.js' + +/** Resolve an agentModels key to its underlying model string, or return bare model ids unchanged. */ +export function resolveSmartRoutingRoleModelString( + key: string | undefined, + settings: SettingsJson | null, +): string | undefined { + if (!key) return undefined + return settings?.agentModels?.[key]?.model ?? key +} + +/** + * Outcome of a per-turn routing decision. + * + * `routed: false` means the caller must use today's normal model resolution — + * smart routing is disabled, misconfigured, disabled-for-session, or both roles + * are outside the org allowlist. `justDisabledForSession` is set the first time + * an allowlist conflict disables routing, so the caller emits one notice. + */ +export type TurnRoutingDecision = + | { routed: false; justDisabledForSession?: boolean } + | { + routed: true + model: string + complexity: 'simple' | 'strong' + reason: string + /** The resolved strong model, for the routed-error fallback (U4). */ + strongModel: string + } + +/** + * Whether a pinned route must be dropped because the active provider changed + * since the route was pinned (a mid-turn provider-fallback swap). A model-only + * route is keyed to the provider it resolved against, so replaying it at a new + * endpoint would send an unknown model id (KTD6). + */ +export function shouldDropPinForProviderSwap( + pinned: TurnRoutingDecision | undefined, + pinnedProviderId: string | undefined, + currentProviderId: string | undefined, +): boolean { + return !!pinned && pinned.routed && currentProviderId !== pinnedProviderId +} + +// Session-scoped disable set. Keyed by session id so a disable never leaks into +// an unrelated session in a long-lived host (gRPC/SDK) — a new session has a new +// id and is therefore never in the set. NOT a process-global boolean. +// Capped so a long-lived host with a persistent allowlist conflict can't grow it +// without bound; clearing is safe (a cleared session re-evaluates and re-disables). +const MAX_DISABLED_SESSIONS = 1024 +const disabledSessions = new Set() + +function markSessionDisabled(sessionId: string): void { + if (disabledSessions.size >= MAX_DISABLED_SESSIONS) disabledSessions.clear() + disabledSessions.add(sessionId) +} + +/** Clear the session-disable flag (e.g. on explicit `/smartroute enable`). */ +export function clearSmartRoutingSessionDisable(sessionId: string | undefined): void { + if (sessionId) disabledSessions.delete(sessionId) +} + +/** Whether smart routing has been auto-disabled for this session (allowlist conflict). */ +export function isSmartRoutingDisabledForSession(sessionId: string | undefined): boolean { + return sessionId ? disabledSessions.has(sessionId) : false +} + +export interface DecideTurnModelInput { + settings: SettingsJson | null + parentModel: string + permissionMode?: PermissionMode + input: RoutingInput + sessionId?: string +} + +/** + * Decide the model for this user turn: resolve config, classify via `routeModel`, + * then enforce the org allowlist by calling `isModelAllowed` directly and + * unconditionally (NOT the change-gated `shouldEnforceModelAllowlist`, which + * short-circuits when the routed model equals the session model). A disallowed + * routed model is coerced to strong; if strong is also disallowed, routing is + * disabled for the session and the caller falls back to today's resolution. + */ +export function decideTurnModel({ + settings, + parentModel, + permissionMode, + input, + sessionId, +}: DecideTurnModelInput): TurnRoutingDecision { + if (isSmartRoutingDisabledForSession(sessionId)) return { routed: false } + + const config = resolveSmartRoutingConfig({ settings, parentModel, permissionMode }) + if (!config.enabled) return { routed: false } + + const decision = routeModel(input, config) + + let model = decision.model + let complexity = decision.complexity + if (!isModelAllowed(model)) { + // Coerce a disallowed model to strong. + model = config.strongModel + complexity = 'strong' + if (!isModelAllowed(model)) { + // Both roles outside the allowlist — disable for the session and let the + // caller use today's (allowlist-clean) resolution. Notice fires once per + // session; with no sessionId we can't dedupe, so stay silent rather than + // emit the notice on every turn. + const first = sessionId ? !disabledSessions.has(sessionId) : false + if (sessionId) markSessionDisabled(sessionId) + return first ? { routed: false, justDisabledForSession: true } : { routed: false } + } + } + + return { routed: true, model, complexity, reason: decision.reason, strongModel: config.strongModel } +} + +/** + * Whether a routed model-call error is worth retrying on the strong model. + * 400/401/403 (bad request, auth, permission) are about the request itself and + * a different model won't fix them, so they propagate. Everything else retries, + * including 404 and 429 by design: the fallback switches to a *different* model + * (strong), so a 404 ("simple model not found", e.g. a typo'd role) or a 429 + * (the simple model rate-limited) is recovered by completing the turn on strong + * rather than failing it. Callers must check abort separately (an aborted turn + * must not retry). + */ +export function isRetryableRoutedModelError(err: unknown): boolean { + const status = + (err as { status?: number })?.status ?? (err as { statusCode?: number })?.statusCode + if (status === 400 || status === 401 || status === 403) return false + return true +} + +/** Session-level routing tally, rendered by the cost/status surfaces (U6). */ +export interface RoutingTally { + simple: number + strong: number + /** Simple-routed turns that fell back to strong on a routed-model error. */ + escalations: number +} + +const tally: RoutingTally = { simple: 0, strong: 0, escalations: 0 } + +/** Record a pinned routing decision (once per user turn). */ +export function recordRoutingDecision(complexity: 'simple' | 'strong'): void { + tally[complexity]++ +} + +/** Record a simple→strong fallback escalation (U4). */ +export function recordRoutingEscalation(): void { + tally.escalations++ +} + +export function getRoutingTally(): RoutingTally { + return { ...tally } +} + +export function resetRoutingTally(): void { + tally.simple = 0 + tally.strong = 0 + tally.escalations = 0 +} + +/** + * Format the session routing summary for `/cost`. Returns null when nothing has + * been routed. The split and escalation count are always shown (the falsifiable + * core); the estimated-savings line is gated on first-party pricing for both + * roles and is annotated as unavailable when either price is unknown. + */ +export function formatRoutingSummary( + t: RoutingTally, + pricing?: { simpleInputCost?: number; strongInputCost?: number }, +): string | null { + if (t.simple + t.strong === 0) return null + const head = + `Smart routing: ${t.simple} simple, ${t.strong} strong` + + (t.escalations ? `, ${t.escalations} escalated to strong` : '') + const s = pricing?.simpleInputCost + const st = pricing?.strongInputCost + let savings: string + if (s != null && st != null && st > 0) { + savings = + s < st + ? ` Estimated (first-party reference pricing): simple turns use a model priced ~${Math.round((1 - s / st) * 100)}% lower per input token. Your provider/gateway/account may bill differently; actual savings depend on token mix and cache.` + : ' Estimated (first-party reference pricing): simple model is not cheaper than strong, so no savings expected. Your provider may bill differently.' + } else { + savings = ' Estimated savings unavailable: one or both models have no first-party reference pricing.' + } + return `${head}\n${savings}` +} + +/** + * Build the `/cost` routing summary from the live tally and the configured + * roles' first-party pricing. + */ +export function getRoutingSummaryForDisplay(settings: SettingsJson | null): string | null { + const sr = readSmartRouting(settings) + const simpleModelId = resolveSmartRoutingRoleModelString(sr.simpleModel, settings) + const strongModelId = resolveSmartRoutingRoleModelString(sr.strongModel, settings) + return formatRoutingSummary(getRoutingTally(), { + simpleInputCost: simpleModelId ? getKnownInputCost(simpleModelId) : undefined, + strongInputCost: strongModelId ? getKnownInputCost(strongModelId) : undefined, + }) +} + +/** Minimal message shape for turn counting — avoids importing heavy message types. */ +interface TurnCountMessage { + type: string + isMeta?: boolean + message?: { content?: unknown } +} + +function isToolResultCarrier(content: unknown): boolean { + return ( + Array.isArray(content) && + content.some( + block => + typeof block === 'object' && block !== null && (block as { type?: string }).type === 'tool_result', + ) + ) +} + +function isRealUserMessage(m: TurnCountMessage): boolean { + return m.type === 'user' && !m.isMeta && !isToolResultCarrier(m.message?.content) +} + +/** + * Count real user turns in the conversation: user-role messages that are neither + * `isMeta` (injected nudges/system reminders) nor tool-result carriers + * (continuation passes). This is the session-level turn number for + * `RoutingInput.turnNumber` — distinct from the loop's per-`query()` `turnCount`. + */ +export function deriveUserTurnNumber(messages: readonly TurnCountMessage[]): number { + let count = 0 + for (const m of messages) { + if (isRealUserMessage(m)) count++ + } + return count +} + +function textOfContent(content: unknown): string { + if (typeof content === 'string') return content + if (Array.isArray(content)) { + return content + .filter( + block => typeof block === 'object' && block !== null && (block as { type?: string }).type === 'text', + ) + .map(block => (block as { text?: string }).text ?? '') + .join('\n') + } + return '' +} + +/** Whether the most recent real user turn includes image/document or other non-text content. */ +export function latestUserMessageHasNonTextContent(messages: readonly TurnCountMessage[]): boolean { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] + if (!isRealUserMessage(m)) continue + const content = m.message?.content + return Array.isArray(content) && content.some( + block => !(typeof block === 'object' && block !== null && (block as { type?: string }).type === 'text'), + ) + } + return false +} + +/** Text of the most recent real user message (for `RoutingInput.userText`). */ +export function extractLatestUserText(messages: readonly TurnCountMessage[]): string { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] + if (isRealUserMessage(m)) return textOfContent(m.message?.content) + } + return '' +} diff --git a/src/services/api/smartRouting/resolveConfig.test.ts b/src/services/api/smartRouting/resolveConfig.test.ts new file mode 100644 index 0000000000..2e22d60f6e --- /dev/null +++ b/src/services/api/smartRouting/resolveConfig.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from 'bun:test' +import { resolveSmartRoutingConfig } from './resolveConfig.js' +import type { SettingsJson } from '../../../utils/settings/types.js' + +const PARENT = 'gpt-5' + +function settings(overrides: Record): SettingsJson { + return overrides as unknown as SettingsJson +} + +describe('resolveSmartRoutingConfig', () => { + test('both roles as model-only agentModels keys resolve to their model strings', () => { + const s = settings({ + agentModels: { + mini: { model: 'gpt-5-mini' }, + main: { model: 'gpt-5' }, + }, + smartRouting: { enabled: true, simpleModel: 'mini', strongModel: 'main' }, + }) + const config = resolveSmartRoutingConfig({ settings: s, parentModel: PARENT }) + expect(config.enabled).toBe(true) + expect(config.simpleModel).toBe('gpt-5-mini') + expect(config.strongModel).toBe('gpt-5') + }) + + test('simpleModel resolving to a cross-provider override collapses to strong, no credential leak', () => { + const s = settings({ + agentModels: { + ds: { base_url: 'https://api.deepseek.com/v1', api_key: 'sk-secret' }, + main: { model: 'gpt-5' }, + }, + smartRouting: { enabled: true, simpleModel: 'ds', strongModel: 'main' }, + }) + const config = resolveSmartRoutingConfig({ settings: s, parentModel: PARENT }) + expect(config.enabled).toBe(true) + expect(config.simpleModel).toBe('gpt-5') // collapsed to strong + expect(config.strongModel).toBe('gpt-5') + // No credential field leaked into the returned config. + const json = JSON.stringify(config) + expect(json).not.toContain('sk-secret') + expect(json).not.toContain('api_key') + expect(json).not.toContain('base_url') + }) + + test('strong role resolving to a cross-provider override disables routing (no safe fallback)', () => { + const s = settings({ + agentModels: { + ds: { base_url: 'https://api.deepseek.com/v1', api_key: 'sk-secret' }, + mini: { model: 'gpt-5-mini' }, + }, + smartRouting: { enabled: true, simpleModel: 'mini', strongModel: 'ds' }, + }) + const config = resolveSmartRoutingConfig({ settings: s, parentModel: PARENT }) + expect(config.enabled).toBe(false) + expect(JSON.stringify(config)).not.toContain('sk-secret') + }) + + test('missing simpleModel collapses to strong (routeModel treats equal models as always-strong)', () => { + const s = settings({ + agentModels: { main: { model: 'gpt-5' } }, + smartRouting: { enabled: true, strongModel: 'main' }, + }) + const config = resolveSmartRoutingConfig({ settings: s, parentModel: PARENT }) + expect(config.enabled).toBe(true) + expect(config.simpleModel).toBe('gpt-5') + expect(config.strongModel).toBe('gpt-5') + }) + + test('bare model ids (not agentModels keys) pass through as model strings', () => { + const s = settings({ + smartRouting: { enabled: true, simpleModel: 'qwen2.5-coder:7b', strongModel: 'qwen2.5-coder:32b' }, + }) + const config = resolveSmartRoutingConfig({ settings: s, parentModel: PARENT }) + expect(config.enabled).toBe(true) + expect(config.simpleModel).toBe('qwen2.5-coder:7b') + expect(config.strongModel).toBe('qwen2.5-coder:32b') + }) + + test('thresholds carry through from settings', () => { + const s = settings({ + smartRouting: { enabled: true, simpleModel: 'a', strongModel: 'b', simpleMaxChars: 200, simpleMaxWords: 40 }, + }) + const config = resolveSmartRoutingConfig({ settings: s, parentModel: PARENT }) + expect(config.simpleMaxChars).toBe(200) + expect(config.simpleMaxWords).toBe(40) + }) + + test('disabled settings produce a disabled config', () => { + const s = settings({ smartRouting: { enabled: false, simpleModel: 'a', strongModel: 'b' } }) + expect(resolveSmartRoutingConfig({ settings: s, parentModel: PARENT }).enabled).toBe(false) + expect(resolveSmartRoutingConfig({ settings: null, parentModel: PARENT }).enabled).toBe(false) + }) +}) diff --git a/src/services/api/smartRouting/resolveConfig.ts b/src/services/api/smartRouting/resolveConfig.ts new file mode 100644 index 0000000000..497e8be354 --- /dev/null +++ b/src/services/api/smartRouting/resolveConfig.ts @@ -0,0 +1,83 @@ +import type { PermissionMode } from '../../../utils/permissions/PermissionMode.js' +import type { SettingsJson } from '../../../utils/settings/types.js' +import type { SmartRoutingConfig } from '../smartModelRouting.js' +import { isProviderOverride, resolveAgentModelProvider, resolveModelOnlyModel } from '../agentRouting.js' +import { readSmartRouting } from './settings.js' + +export interface ResolveSmartRoutingConfigInput { + settings: SettingsJson | null + parentModel: string + permissionMode?: PermissionMode +} + +/** + * Resolve a single role (an agentModels key or a bare model id) to the concrete + * model string smart routing should send. + * + * Returns `null` when the role resolves to a cross-provider `ProviderOverride`: + * B1 is model-only within the current provider, so cross-provider routes are + * deferred. The whole override object is discarded — no field (`apiKey`, + * `baseURL`) is ever read or surfaced. + */ +function resolveRoleToModelOnly( + roleKey: string, + settings: SettingsJson | null, + parentModel: string, + permissionMode?: PermissionMode, +): string | null { + const route = resolveAgentModelProvider(roleKey, settings) + if (route) { + // Cross-provider override: discard the whole object, defer to strong default. + if (isProviderOverride(route)) return null + return resolveModelOnlyModel(route.model, parentModel, permissionMode) + } + // Not an agentModels key — treat as a bare model id (alias/inherit aware). + return resolveModelOnlyModel(roleKey, parentModel, permissionMode) +} + +/** + * Build a `SmartRoutingConfig` (concrete model strings) from `settings.smartRouting`. + * + * Strong-default rules: + * - Disabled or misconfigured settings → `{ enabled: false }`-shaped config; the + * caller must fall back to today's model resolution rather than route. + * - `strongModel` unresolvable (missing, or a cross-provider override) → disabled, + * because there is no safe model to fall back to. + * - `simpleModel` unresolvable → collapsed to the strong model, which `routeModel` + * treats as "always strong" (`simpleModel === strongModel`). + * + * Never throws. + */ +export function resolveSmartRoutingConfig({ + settings, + parentModel, + permissionMode, +}: ResolveSmartRoutingConfigInput): SmartRoutingConfig { + const norm = readSmartRouting(settings) + + const disabled = (strongModel = ''): SmartRoutingConfig => ({ + enabled: false, + simpleModel: strongModel, + strongModel, + }) + + if (!norm.enabled || !norm.strongModel) return disabled(norm.strongModel ?? '') + + const strong = resolveRoleToModelOnly(norm.strongModel, settings, parentModel, permissionMode) + // No usable strong model (e.g. strong role is a cross-provider override) — do + // not route; the caller uses today's resolution. + if (!strong) return disabled(norm.strongModel) + + const simple = norm.simpleModel + ? resolveRoleToModelOnly(norm.simpleModel, settings, parentModel, permissionMode) + : null + + return { + enabled: true, + // Unresolvable simple → collapse to strong (routeModel then always picks strong). + simpleModel: simple ?? strong, + strongModel: strong, + simpleMaxChars: norm.simpleMaxChars, + simpleMaxWords: norm.simpleMaxWords, + } +} diff --git a/src/services/api/smartRouting/settings.test.ts b/src/services/api/smartRouting/settings.test.ts new file mode 100644 index 0000000000..b827642df6 --- /dev/null +++ b/src/services/api/smartRouting/settings.test.ts @@ -0,0 +1,111 @@ +import { afterEach, describe, expect, spyOn, test } from 'bun:test' +import { readSmartRouting } from './settings.js' +import type { SettingsJson } from '../../../utils/settings/types.js' + +function settingsWith(smartRouting: unknown): SettingsJson { + return { smartRouting } as unknown as SettingsJson +} + +describe('readSmartRouting', () => { + afterEach(() => { + // Restore any console spies between tests. + }) + + test('absent block normalizes to disabled, no warning', () => { + const errSpy = spyOn(console, 'error').mockImplementation(() => {}) + expect(readSmartRouting(null)).toEqual({ enabled: false }) + expect(readSmartRouting({} as SettingsJson)).toEqual({ enabled: false }) + expect(errSpy).not.toHaveBeenCalled() + errSpy.mockRestore() + }) + + test('explicitly disabled normalizes to disabled, no warning', () => { + const errSpy = spyOn(console, 'error').mockImplementation(() => {}) + expect(readSmartRouting(settingsWith({ enabled: false, simpleModel: 'mini', strongModel: 'main' }))).toEqual({ + enabled: false, + }) + expect(errSpy).not.toHaveBeenCalled() + errSpy.mockRestore() + }) + + test('enabled with both roles carries both and passes thresholds through', () => { + const result = readSmartRouting( + settingsWith({ enabled: true, simpleModel: 'mini', strongModel: 'main', simpleMaxChars: 200, simpleMaxWords: 40 }), + ) + expect(result).toEqual({ + enabled: true, + simpleModel: 'mini', + strongModel: 'main', + simpleMaxChars: 200, + simpleMaxWords: 40, + }) + }) + + test('enabled but strongModel missing normalizes to disabled with one warning', () => { + const errSpy = spyOn(console, 'error').mockImplementation(() => {}) + expect(readSmartRouting(settingsWith({ enabled: true, simpleModel: 'mini' }))).toEqual({ enabled: false }) + expect(errSpy).toHaveBeenCalledTimes(1) + errSpy.mockRestore() + }) + + test('strongModel present but simpleModel missing stays enabled (routeModel collapses to strong)', () => { + const result = readSmartRouting(settingsWith({ enabled: true, strongModel: 'main' })) + expect(result.enabled).toBe(true) + expect(result.strongModel).toBe('main') + expect(result.simpleModel).toBeUndefined() + }) + + test('non-numeric or non-positive thresholds are dropped so classifier defaults apply', () => { + const result = readSmartRouting( + settingsWith({ + enabled: true, + simpleModel: 'mini', + strongModel: 'main', + simpleMaxChars: -5, + simpleMaxWords: Number.NaN, + }), + ) + expect(result.simpleMaxChars).toBeUndefined() + expect(result.simpleMaxWords).toBeUndefined() + }) + + test('whitespace-only role strings are treated as absent', () => { + const errSpy = spyOn(console, 'error').mockImplementation(() => {}) + // strong is whitespace -> treated as missing -> disabled + warning + expect(readSmartRouting(settingsWith({ enabled: true, simpleModel: 'mini', strongModel: ' ' }))).toEqual({ + enabled: false, + }) + errSpy.mockRestore() + }) + + describe('env fallback (settings override env)', () => { + test('env enables routing when settings say nothing', () => { + const env = { + OPENCLAUDE_SMART_ROUTING: '1', + OPENCLAUDE_SMART_ROUTING_SIMPLE: 'mini', + OPENCLAUDE_SMART_ROUTING_STRONG: 'main', + } as unknown as NodeJS.ProcessEnv + expect(readSmartRouting(null, env)).toEqual({ + enabled: true, + simpleModel: 'mini', + strongModel: 'main', + simpleMaxChars: undefined, + simpleMaxWords: undefined, + }) + }) + + test('settings block present (even disabled) overrides env', () => { + const env = { + OPENCLAUDE_SMART_ROUTING: '1', + OPENCLAUDE_SMART_ROUTING_SIMPLE: 'mini', + OPENCLAUDE_SMART_ROUTING_STRONG: 'main', + } as unknown as NodeJS.ProcessEnv + // settings explicitly disables -> env's enable does not apply + expect(readSmartRouting(settingsWith({ enabled: false }), env)).toEqual({ enabled: false }) + }) + + test('no env and no settings is disabled', () => { + expect(readSmartRouting(null, {} as NodeJS.ProcessEnv)).toEqual({ enabled: false }) + }) + }) +}) diff --git a/src/services/api/smartRouting/settings.ts b/src/services/api/smartRouting/settings.ts new file mode 100644 index 0000000000..fab2d9ac4a --- /dev/null +++ b/src/services/api/smartRouting/settings.ts @@ -0,0 +1,83 @@ +import type { SettingsJson } from '../../../utils/settings/types.js' + +/** + * Normalized smart-routing configuration read from settings. + * + * This is the shape callers (the role resolver, the CLI surface) consume. It + * carries the raw role keys and thresholds — it does NOT resolve role keys to + * concrete model strings (that is `resolveConfig.ts`'s job). + * + * `enabled` reflects the strong-default rule: a config that opts in but omits + * `strongModel` is normalized to disabled, because routing with no strong model + * to fall back to is a misconfiguration, not a usable state. + */ +export interface NormalizedSmartRouting { + enabled: boolean + /** agentModels key or bare model id for "simple" turns. */ + simpleModel?: string + /** agentModels key or bare model id for "strong" turns and any unsure case. */ + strongModel?: string + simpleMaxChars?: number + simpleMaxWords?: number +} + +const DISABLED: NormalizedSmartRouting = { enabled: false } + +/** Keep a positive finite number, otherwise drop it so the classifier default applies. */ +function sanitizeThreshold(value: number | undefined): number | undefined { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + return undefined + } + return value +} + +/** Startup defaults from env. Used only when `settings.smartRouting` is absent. */ +function readEnvSmartRouting(env: NodeJS.ProcessEnv): SettingsJson['smartRouting'] | undefined { + if (env.OPENCLAUDE_SMART_ROUTING == null) return undefined + const enabled = env.OPENCLAUDE_SMART_ROUTING === '1' || env.OPENCLAUDE_SMART_ROUTING === 'true' + return { + enabled, + simpleModel: env.OPENCLAUDE_SMART_ROUTING_SIMPLE?.trim() || undefined, + strongModel: env.OPENCLAUDE_SMART_ROUTING_STRONG?.trim() || undefined, + } +} + +/** + * Read and normalize smart-routing config. Precedence: `settings.smartRouting` + * (if present at all, even disabled) wins over env, so org-managed settings + * override an env default. Env (`OPENCLAUDE_SMART_ROUTING*`) is the startup + * default when settings say nothing. + * + * Returns a disabled config when the block is absent, disabled, or + * misconfigured (enabled but missing `strongModel`). Warns once on the + * misconfiguration case, mirroring the one-sided-route warning in + * `agentRouting.ts`. + */ +export function readSmartRouting( + settings: SettingsJson | null, + env: NodeJS.ProcessEnv = process.env, +): NormalizedSmartRouting { + // Settings take precedence over env: only fall back to env when the settings + // block is entirely absent (undefined). + const raw = settings?.smartRouting ?? readEnvSmartRouting(env) + if (!raw || !raw.enabled) return DISABLED + + const strongModel = raw.strongModel?.trim() || undefined + if (!strongModel) { + console.error( + '[smartRouting] Warning: smartRouting is enabled but strongModel is missing; ' + + 'smart routing needs a strong model to fall back to. Disabling smart routing.', + ) + return DISABLED + } + + const simpleModel = raw.simpleModel?.trim() || undefined + + return { + enabled: true, + simpleModel, + strongModel, + simpleMaxChars: sanitizeThreshold(raw.simpleMaxChars), + simpleMaxWords: sanitizeThreshold(raw.simpleMaxWords), + } +} diff --git a/src/utils/conversationRecovery.test.ts b/src/utils/conversationRecovery.test.ts index 5d6043e9d6..58fddbc7e7 100644 --- a/src/utils/conversationRecovery.test.ts +++ b/src/utils/conversationRecovery.test.ts @@ -12,6 +12,20 @@ import { } from '../test/sharedMutationLock.js' import * as realUdsClient from './udsClient.js' import * as realProviders from './model/providers.js' +import type { NormalizedMessage } from '../types/message.js' + +// Typed fixture for the thinking-strip gate tests. The full NormalizedMessage +// shape carries fields these tests don't exercise, so the cast is centralized +// here once rather than re-spelled as `as any` at each call site. +function assistantThinkingMessage(): NormalizedMessage { + return { + type: 'assistant', + message: { + role: 'assistant', + content: [{ type: 'thinking', thinking: 'x' }, { type: 'text', text: 'answer' }], + }, + } as unknown as NormalizedMessage +} const tempDirs: string[] = [] const originalSimple = process.env.CLAUDE_CODE_SIMPLE @@ -686,6 +700,32 @@ test('deserializeMessages preserves thinking blocks for DeepSeek 3P provider (#9 expect(content.some(block => block.type === 'thinking')).toBe(true) }) +test('stripThinkingBlocksIfProviderAllows preserves thinking for preserve-reasoning 3P (Z.AI GLM / DeepSeek)', async () => { + // Smart routing reuses this gate on a per-turn model swap. A preserve-reasoning + // provider 400s if the thinking block is stripped, so the gate must leave it. + clearProviderEnv() + process.env.CLAUDE_CODE_USE_OPENAI = '1' + process.env.OPENAI_BASE_URL = 'https://api.deepseek.com/v1' + process.env.OPENAI_MODEL = 'deepseek-v4-flash' + const { stripThinkingBlocksIfProviderAllows } = await importFreshConversationRecovery() + + const result = stripThinkingBlocksIfProviderAllows([assistantThinkingMessage()]) + const content = (result[0] as any)?.message?.content as Array<{ type: string }> + expect(content.some(block => block.type === 'thinking')).toBe(true) +}) + +test('stripThinkingBlocksIfProviderAllows strips thinking for generic OpenAI 3P', async () => { + clearProviderEnv() + process.env.CLAUDE_CODE_USE_OPENAI = '1' + process.env.OPENAI_BASE_URL = 'https://api.openai.com/v1' + process.env.OPENAI_MODEL = 'gpt-5-mini' + const { stripThinkingBlocksIfProviderAllows } = await importFreshConversationRecovery() + + const result = stripThinkingBlocksIfProviderAllows([assistantThinkingMessage()]) + const content = (result[0] as any)?.message?.content as Array<{ type: string }> + expect(content.some(block => block.type === 'thinking')).toBe(false) +}) + test('deserializeMessages still strips thinking blocks for generic OpenAI 3P (no preserveReasoningContent)', async () => { // Counter-test: providers that don't set preserveReasoningContent keep the // original strip behaviour from #248; thinking blocks were causing 400s diff --git a/src/utils/conversationRecovery.ts b/src/utils/conversationRecovery.ts index f1c9e523fa..b77c15cb7e 100644 --- a/src/utils/conversationRecovery.ts +++ b/src/utils/conversationRecovery.ts @@ -252,6 +252,35 @@ function shouldPreserveThinkingBlocksForProviderReplay(): boolean { ) } +/** + * Strip protected-thinking blocks from history only when the active provider + * tolerates it. Mirrors the gate used during session-resume deserialization: + * strip for non-preserve third-party providers, leave Anthropic-native and + * preserve-reasoning providers (DeepSeek/Kimi/Z.AI GLM — which 400 on a + * stripped block, issue #957) untouched. + * + * Exposed for callers that rewrite history across a model change (e.g. smart + * routing's per-turn model swap), so a model-bound thinking signature is never + * replayed to a different model without re-creating the preserve-reasoning 400. + */ +export function stripThinkingBlocksIfProviderAllows( + messages: NormalizedMessage[], +): NormalizedMessage[] { + const provider = getAPIProvider() + const isAnthropicNativeTransport = usesAnthropicNativeMessageFormat({ + processEnv: process.env, + model: process.env.OPENAI_MODEL, + providerCategory: provider as NonNullable< + Parameters[0] + >['providerCategory'], + }) + const isThirdPartyProvider = provider !== 'foundry' && !isAnthropicNativeTransport + if (isThirdPartyProvider && !shouldPreserveThinkingBlocksForProviderReplay()) { + return stripThinkingBlocks(messages) + } + return messages +} + function parsePrIdentifier(value: string): number | null { const directNumber = parseInt(value, 10) if (!isNaN(directNumber) && directNumber > 0) { @@ -347,22 +376,7 @@ export function deserializeMessagesWithInterruptDetection( // outgoing OpenAI-format message. Stripping the block leaves the shim with // no reasoning text to attach, and the provider 400s with // "reasoning_content in the thinking mode must be passed back" (issue #957). - const provider = getAPIProvider() - const isAnthropicNativeTransport = usesAnthropicNativeMessageFormat({ - processEnv: process.env, - model: process.env.OPENAI_MODEL, - // runtimeMetadata's inline providerCategory union predates the newer - // 'xai'/'xiaomi-mimo' categories; they take the same third-party path. - providerCategory: provider as NonNullable< - Parameters[0] - >['providerCategory'], - }) - const isThirdPartyProvider = - provider !== 'foundry' && !isAnthropicNativeTransport - const thinkingStripped = - isThirdPartyProvider && !shouldPreserveThinkingBlocksForProviderReplay() - ? stripThinkingBlocks(filteredThinking) - : filteredThinking + const thinkingStripped = stripThinkingBlocksIfProviderAllows(filteredThinking) // Filter out assistant messages with only whitespace text content. // This can happen when model outputs "\n\n" before thinking, user cancels mid-stream. diff --git a/src/utils/settings/types.ts b/src/utils/settings/types.ts index d405829201..b144121cdd 100644 --- a/src/utils/settings/types.ts +++ b/src/utils/settings/types.ts @@ -780,6 +780,32 @@ export const SettingsSchema = lazySchema(() => 'Use "default" key as fallback. Model name must exist in agentModels. ' + 'Example: { "Explore": "deepseek-chat", "general-purpose": "gpt-4o", "default": "gpt-4o" }', ), + smartRouting: z + .object({ + enabled: z.boolean().optional().describe('Opt in to per-turn simple-vs-strong model routing. Off by default.'), + simpleModel: z + .string() + .optional() + .describe('agentModels key (or bare model id) used for turns classified "simple".'), + strongModel: z + .string() + .optional() + .describe('agentModels key (or bare model id) used for "strong" turns and whenever routing is unsure.'), + simpleMaxChars: z + .number() + .optional() + .describe('Max characters in user input to qualify as "simple". Passed to routeModel.'), + simpleMaxWords: z + .number() + .optional() + .describe('Max whitespace-separated words to qualify as "simple". Passed to routeModel.'), + }) + .optional() + .describe( + 'Opt-in smart routing: classify each user turn and route simple turns to the configured simple model. ' + + 'simpleModel/strongModel are agentModels keys (or bare model ids). ' + + 'Example: { "enabled": true, "simpleModel": "mini", "strongModel": "main" }', + ), providerFallbackChain: z .array(z.string()) .optional() diff --git a/web/src/data/commands.ts b/web/src/data/commands.ts index ad78fa7d3d..e49194e032 100644 --- a/web/src/data/commands.ts +++ b/web/src/data/commands.ts @@ -92,6 +92,7 @@ export const commands: SlashCommand[] = [ { name: 'model', description: 'Set the AI model for the session', category: 'models', args: '[model]' }, { name: 'provider', description: 'Manage API provider profiles', category: 'models' }, { name: 'effort', description: 'Set effort level for model usage', category: 'models', args: '[low|medium|high|max|auto]' }, + { name: 'smartroute', description: 'Configure smart auto-routing (experimental): route simple turns to your configured simple model', category: 'models', args: '[on|off|simple |strong ]' }, { name: 'login', description: 'Sign in with your Anthropic account', category: 'models' }, { name: 'logout', description: 'Sign out from your Anthropic account', category: 'models' }, { name: 'onboard-github', description: 'Interactive setup for GitHub Copilot: OAuth device login stored in secure storage', category: 'models' }, diff --git a/web/src/data/configuration.ts b/web/src/data/configuration.ts index 277ba4dbcf..e7140c1aad 100644 --- a/web/src/data/configuration.ts +++ b/web/src/data/configuration.ts @@ -50,6 +50,7 @@ export const settingOptions: SettingOption[] = [ { key: 'verbose', description: 'Verbose output by default.' }, { key: 'allowAutoUpdates', description: 'Enable or disable the auto-updater.' }, { key: 'hooks', description: 'Shell hooks that run on tool events (PreToolUse, PostToolUse, …).' }, + { key: 'smartRouting', description: 'Opt-in smart auto-routing: { enabled, simpleModel, strongModel } route simple turns to the configured simple model. Configure with /smartroute.' }, ] export interface EnvVar { @@ -74,4 +75,7 @@ export const envVars: EnvVar[] = [ { name: 'HTTP_PROXY / HTTPS_PROXY', description: 'Route API traffic through a proxy.' }, { name: 'NODE_EXTRA_CA_CERTS', description: 'Extra CA certificates for corporate TLS interception.' }, { name: 'CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC', description: 'Disable non-essential network traffic.' }, + { name: 'OPENCLAUDE_SMART_ROUTING', description: 'Set to 1/true to enable smart auto-routing as a startup default (settings.smartRouting overrides it).' }, + { name: 'OPENCLAUDE_SMART_ROUTING_SIMPLE', description: 'agentModels key or model id used for turns classified "simple".' }, + { name: 'OPENCLAUDE_SMART_ROUTING_STRONG', description: 'agentModels key or model id used for "strong" turns and as the routed-error fallback.' }, ]