Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
4d54d5d
fix: type safety, defensive defaults, and unbounded retry prevention
skyhighbg22-jpg Jun 5, 2026
aed3744
test: enable UNATTENDED_RETRY feature in bun test scripts
skyhighbg22-jpg Jun 5, 2026
0243028
test: cover persistent retry cap driven through real gate
skyhighbg22-jpg Jun 5, 2026
3d0b227
test: defensively clear leaked env vars in client.test.ts
skyhighbg22-jpg Jun 5, 2026
cd2dac6
test: isolate countMcpToolTokens tests from mcp.ts env-var side effect
skyhighbg22-jpg Jun 5, 2026
9e1d401
fix: validate against EXTERNAL_PERMISSION_MODES; confine persistent r…
skyhighbg22-jpg Jun 5, 2026
d61b253
feat: emit telemetry when persistent retry cap is reached
skyhighbg22-jpg Jun 6, 2026
ff28014
fix: surface changelog cache-write failures in migration
skyhighbg22-jpg Jun 6, 2026
2d44e7c
fix: split mkdir and writeFile in changelog migration
skyhighbg22-jpg Jun 6, 2026
33f9ed9
fix: stop overriding getGlobalConfig in user.test.ts mock
skyhighbg22-jpg Jun 7, 2026
aa9f9fd
Merge branch 'main' into fix/code-quality-and-defensive-defaults
skyhighbg22-jpg Jun 11, 2026
ab5a7a7
Merge branch 'main' into fix/code-quality-and-defensive-defaults
skyhighbg22-jpg Jun 12, 2026
fc870de
Fix leftover conflict marker in QueryEngine.ts
skyhighbg22-jpg Jun 10, 2026
8eb7219
fix: remove stale retry guard and handle mkdir EEXIST
skyhighbg22-jpg Jun 13, 2026
3b9032e
test: don't restore OPENAI auth header env vars in afterEach
skyhighbg22-jpg Jun 14, 2026
e542390
fix: remove duplicate OPENAI_AUTH_* keys in originalEnv (TS1117)
skyhighbg22-jpg Jun 14, 2026
1414b6c
fix: restore OPENAI auth header snapshot and move MCP lock to top-level
skyhighbg22-jpg Jun 14, 2026
79072d8
fix: use splice for atomic array replacement in snip replay
skyhighbg22-jpg Jun 14, 2026
197dd22
ci: add UNATTENDED_RETRY feature flag to release workflow test command
skyhighbg22-jpg Jun 15, 2026
38b953c
fix: clear auth env vars in shared setup; add telemetry at persistent…
skyhighbg22-jpg Jun 16, 2026
bac72d6
fix: make persistent retry cap test pass without --feature=UNATTENDED…
skyhighbg22-jpg Jun 16, 2026
c034103
fix: normalize REPL bridge permissionMode against EXTERNAL_PERMISSION…
skyhighbg22-jpg Jun 16, 2026
a6fb1d2
fix: export isPersistentRetryEnabled for test-side feature-gate asser…
skyhighbg22-jpg Jun 17, 2026
85ea136
fix: use isPersistentRetryEnabled() as real feature gate in retry cap…
skyhighbg22-jpg Jun 17, 2026
d287c2f
fix: restore missing retryableRateLimit declaration in persistent ret…
skyhighbg22-jpg Jun 17, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@
"web:build": "bun run --cwd web build",
"web:preview": "bun run --cwd web preview",
"web:typecheck": "bun run --cwd web typecheck",
"test": "bun test",
"test:full": "bun test --max-concurrency=1",
"test:coverage": "bun test --coverage --coverage-reporter=lcov --coverage-dir=coverage --max-concurrency=1 && bun run scripts/render-coverage-heatmap.ts",
"test": "bun test --feature=UNATTENDED_RETRY",
"test:full": "bun test --feature=UNATTENDED_RETRY --max-concurrency=1",
"test:coverage": "bun test --feature=UNATTENDED_RETRY --coverage --coverage-reporter=lcov --coverage-dir=coverage --max-concurrency=1 && bun run scripts/render-coverage-heatmap.ts",
"test:coverage:ui": "bun run scripts/render-coverage-heatmap.ts",
"security:pr-scan": "bun run scripts/pr-intent-scan.ts",
"test:provider-recommendation": "bun test src/utils/providerRecommendation.test.ts src/utils/providerProfile.test.ts",
Expand All @@ -62,7 +62,7 @@
"check": "bun run smoke && bun run typecheck:type-tests && bun run test:full",
"verify:privacy": "bun run scripts/verify-no-phone-home.ts",
"build:verified": "bun run build && bun run verify:privacy",
"test:provider": "bun test --max-concurrency=1 src/services/api/*.test.ts src/utils/context.test.ts",
"test:provider": "bun test --feature=UNATTENDED_RETRY --max-concurrency=1 src/services/api/*.test.ts src/utils/context.test.ts",
"doctor:runtime": "bun run scripts/system-check.ts",
"doctor:runtime:json": "bun run scripts/system-check.ts --json",
"doctor:report": "bun run scripts/system-check.ts --out reports/doctor-runtime.json",
Expand Down
18 changes: 14 additions & 4 deletions src/QueryEngine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import type {
SDKStatus,
SDKUserMessageReplay,
} from 'src/entrypoints/agentSdkTypes.js'
import { EXTERNAL_PERMISSION_MODES } from 'src/types/permissions.js'
import { accumulateUsage, updateUsage } from 'src/services/api/claude.js'
import type { NonNullableUsage } from 'src/services/api/logging.js'
import { EMPTY_USAGE } from 'src/services/api/logging.js'
Expand Down Expand Up @@ -548,12 +549,18 @@ export class QueryEngine {
])
headlessProfilerCheckpoint('after_skills_plugins')

const rawPermissionMode = initialAppState.toolPermissionContext.mode
const validPermissionMode: PermissionMode = (
EXTERNAL_PERMISSION_MODES as readonly string[]
).includes(rawPermissionMode)
? (rawPermissionMode as PermissionMode)
: 'default'

yield buildSystemInitMessage({
tools,
mcpClients,
model: mainLoopModel,
permissionMode: initialAppState.toolPermissionContext
.mode as PermissionMode, // TODO: avoid the cast
permissionMode: validPermissionMode,
commands,
agents,
skills,
Expand Down Expand Up @@ -936,8 +943,11 @@ export class QueryEngine {
)
if (snipResult !== undefined) {
if (snipResult.executed) {
this.mutableMessages.length = 0
this.mutableMessages.push(...snipResult.messages)
this.mutableMessages.splice(
0,
this.mutableMessages.length,
...snipResult.messages,
)
}
break
}
Expand Down
8 changes: 6 additions & 2 deletions src/main.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -346,8 +346,12 @@ function runMigrations(): void {
});
}
// Async migration - fire and forget since it's non-blocking
migrateChangelogFromConfig().catch(() => {
// Silently ignore migration errors - will retry on next startup
migrateChangelogFromConfig().catch(error => {
logError(
new Error(
`Changelog migration failed; will retry on next startup: ${errorMessage(error)}`,
),
)
});
}

Expand Down
29 changes: 29 additions & 0 deletions src/services/api/client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,16 @@ const originalEnv = {
OPENAI_BASE_URL: process.env.OPENAI_BASE_URL,
OPENAI_API_BASE: process.env.OPENAI_API_BASE,
OPENAI_API_FORMAT: process.env.OPENAI_API_FORMAT,
OPENAI_AUTH_HEADER: process.env.OPENAI_AUTH_HEADER,
OPENAI_AUTH_SCHEME: process.env.OPENAI_AUTH_SCHEME,
OPENAI_AUTH_HEADER_VALUE: process.env.OPENAI_AUTH_HEADER_VALUE,
OPENAI_MODEL: process.env.OPENAI_MODEL,
MINIMAX_API_KEY: process.env.MINIMAX_API_KEY,
XAI_API_KEY: process.env.XAI_API_KEY,
MIMO_API_KEY: process.env.MIMO_API_KEY,
VENICE_API_KEY: process.env.VENICE_API_KEY,
NVIDIA_NIM: process.env.NVIDIA_NIM,
NVIDIA_API_KEY: process.env.NVIDIA_API_KEY,
ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
ANTHROPIC_AUTH_TOKEN: process.env.ANTHROPIC_AUTH_TOKEN,
ANTHROPIC_BASE_URL: process.env.ANTHROPIC_BASE_URL,
Expand Down Expand Up @@ -75,8 +81,15 @@ function clearEnvForMiniMaxOnlyTest(): void {
delete process.env.OPENAI_BASE_URL
delete process.env.OPENAI_API_BASE
delete process.env.OPENAI_MODEL
delete process.env.OPENAI_API_FORMAT
delete process.env.OPENAI_AUTH_HEADER
delete process.env.OPENAI_AUTH_SCHEME
delete process.env.OPENAI_AUTH_HEADER_VALUE
delete process.env.XAI_API_KEY
delete process.env.MIMO_API_KEY
delete process.env.VENICE_API_KEY
delete process.env.NVIDIA_NIM
delete process.env.NVIDIA_API_KEY
delete process.env.ANTHROPIC_API_KEY
delete process.env.ANTHROPIC_AUTH_TOKEN
delete process.env.ANTHROPIC_BASE_URL
Expand Down Expand Up @@ -108,7 +121,10 @@ beforeEach(async () => {
delete process.env.OPENAI_MODEL
delete process.env.MINIMAX_API_KEY
delete process.env.XAI_API_KEY
delete process.env.MIMO_API_KEY
delete process.env.VENICE_API_KEY
delete process.env.NVIDIA_NIM
delete process.env.NVIDIA_API_KEY
delete process.env.ANTHROPIC_API_KEY
delete process.env.ANTHROPIC_AUTH_TOKEN
delete process.env.ANTHROPIC_BASE_URL
Expand Down Expand Up @@ -141,10 +157,16 @@ afterEach(() => {
restoreEnv('OPENAI_BASE_URL', originalEnv.OPENAI_BASE_URL)
restoreEnv('OPENAI_API_BASE', originalEnv.OPENAI_API_BASE)
restoreEnv('OPENAI_API_FORMAT', originalEnv.OPENAI_API_FORMAT)
restoreEnv('OPENAI_AUTH_HEADER', originalEnv.OPENAI_AUTH_HEADER)
restoreEnv('OPENAI_AUTH_SCHEME', originalEnv.OPENAI_AUTH_SCHEME)
restoreEnv('OPENAI_AUTH_HEADER_VALUE', originalEnv.OPENAI_AUTH_HEADER_VALUE)
restoreEnv('OPENAI_MODEL', originalEnv.OPENAI_MODEL)
restoreEnv('MINIMAX_API_KEY', originalEnv.MINIMAX_API_KEY)
restoreEnv('XAI_API_KEY', originalEnv.XAI_API_KEY)
restoreEnv('MIMO_API_KEY', originalEnv.MIMO_API_KEY)
restoreEnv('VENICE_API_KEY', originalEnv.VENICE_API_KEY)
restoreEnv('NVIDIA_NIM', originalEnv.NVIDIA_NIM)
restoreEnv('NVIDIA_API_KEY', originalEnv.NVIDIA_API_KEY)
restoreEnv('ANTHROPIC_API_KEY', originalEnv.ANTHROPIC_API_KEY)
restoreEnv('ANTHROPIC_AUTH_TOKEN', originalEnv.ANTHROPIC_AUTH_TOKEN)
restoreEnv('ANTHROPIC_BASE_URL', originalEnv.ANTHROPIC_BASE_URL)
Expand Down Expand Up @@ -183,7 +205,14 @@ test('first-party Anthropic requests execute the configured fetch wrapper withou
delete process.env.OPENAI_API_BASE
delete process.env.OPENAI_MODEL
delete process.env.NVIDIA_NIM
delete process.env.NVIDIA_API_KEY
delete process.env.XAI_API_KEY
delete process.env.MIMO_API_KEY
delete process.env.VENICE_API_KEY
delete process.env.ANTHROPIC_API_KEY
delete process.env.ANTHROPIC_AUTH_TOKEN
delete process.env.ANTHROPIC_BASE_URL
delete process.env.ANTHROPIC_MODEL

const fetchOverride = (async (_input, init) => {
capturedHeaders = new Headers(init?.headers)
Expand Down
53 changes: 46 additions & 7 deletions src/services/api/withRetry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,12 @@ type ProvidersModule = typeof import('../../utils/model/providers.js')
// Helper to build a mock APIError with specific headers
function makeError(headers: Record<string, string>): APIError {
const headersObj = new Headers(headers)
return {
headers: headersObj,
status: 429,
message: 'rate limit exceeded',
name: 'APIError',
error: {},
} as unknown as APIError
return new APIError(
429,
{ error: { type: 'rate_limit_error', message: 'rate limit exceeded' } },
'rate limit exceeded',
headersObj,
)
}

// Save/restore env vars between tests
Expand All @@ -26,6 +25,7 @@ const envKeys = [
'CLAUDE_CODE_USE_BEDROCK',
'CLAUDE_CODE_USE_VERTEX',
'CLAUDE_CODE_USE_FOUNDRY',
'CLAUDE_CODE_UNATTENDED_RETRY',
'CLAUDE_CODE_MAX_RETRIES',
'OPENCLAUDE_MAX_RETRIES',
'OPENCLAUDE_RETRY_DELAY_MS',
Expand Down Expand Up @@ -75,6 +75,9 @@ async function importFreshWithRetryModule(
) {
mock.restore()
originalProvidersModule ??= await importActualProviders()
mock.module('src/utils/sleep.js', () => ({
sleep: async () => undefined,
}))
mock.module('src/utils/model/providers.js', () => ({
...originalProvidersModule!,
getAPIProvider: () => provider,
Expand Down Expand Up @@ -362,3 +365,39 @@ describe('parseOpenRouterAffordableMaxTokensError (#1125)', () => {
expect(shouldRetry(err)).toBe(true)
})
})

describe('persistent retry cap', () => {
test('persistent retries stop after 100 retryable 429s', async () => {
// Drive the real persistent retry gate — no runtime override. The
// UNATTENDED_RETRY feature must be enabled via `bun test --feature=UNATTENDED_RETRY`
// (see package.json), and the env var must be truthy, otherwise
// isPersistentRetryEnabled() returns false and the cap never triggers.
process.env.CLAUDE_CODE_UNATTENDED_RETRY = '1'
const retryModule = await importFreshWithRetryModule('firstParty')
const { CannotRetryError, withRetry, _PERSISTENT_MAX_ATTEMPTS_FOR_TEST } = retryModule

expect(_PERSISTENT_MAX_ATTEMPTS_FOR_TEST).toBe(100)

const retryableRateLimit = makeError({ 'retry-after': '1' })
const operation = mock(async () => {
throw retryableRateLimit
})

const runRetries = async () => {
for await (const _ of withRetry(
async () => ({} as never),
operation,
{
maxRetries: 0,
model: 'claude-sonnet-4-6',
thinkingConfig: { type: 'disabled' },
},
)) {
void _
}
}

await expect(runRetries()).rejects.toBeInstanceOf(CannotRetryError)
expect(operation).toHaveBeenCalledTimes(101)
})
})
35 changes: 29 additions & 6 deletions src/services/api/withRetry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,12 @@ function shouldRetry529(querySource: QuerySource | undefined): boolean {
const PERSISTENT_MAX_BACKOFF_MS = 5 * 60 * 1000
const PERSISTENT_RESET_CAP_MS = 6 * 60 * 60 * 1000
const HEARTBEAT_INTERVAL_MS = 30_000
const PERSISTENT_MAX_ATTEMPTS = 100
// Exposed for unit-test assertion only. The persistent retry cap itself is
// driven by isPersistentRetryEnabled() — there is no runtime override seam
// (tests must enable UNATTENDED_RETRY via `bun test --feature=UNATTENDED_RETRY`
// and set CLAUDE_CODE_UNATTENDED_RETRY to exercise this path).
export { PERSISTENT_MAX_ATTEMPTS as _PERSISTENT_MAX_ATTEMPTS_FOR_TEST }

function isPersistentRetryEnabled(): boolean {
return feature('UNATTENDED_RETRY')
Expand Down Expand Up @@ -190,6 +196,7 @@ export async function* withRetry<T>(
options: RetryOptions,
): AsyncGenerator<SystemAPIErrorMessage, T> {
const maxRetries = getMaxRetries(options)
const persistentRetryEnabled = isPersistentRetryEnabled()
const retryContext: RetryContext = {
model: options.model,
thinkingConfig: options.thinkingConfig,
Expand Down Expand Up @@ -289,7 +296,7 @@ export async function* withRetry<T>(
// keep-alive path instead of fast-mode cache-preservation anyway.
if (
wasFastModeActive &&
!isPersistentRetryEnabled() &&
!persistentRetryEnabled &&
error instanceof APIError &&
(error.status === 429 || is529Error(error))
) {
Expand Down Expand Up @@ -376,7 +383,7 @@ export async function* withRetry<T>(
if (
process.env.USER_TYPE === 'external' &&
!process.env.IS_SANDBOX &&
!isPersistentRetryEnabled()
!persistentRetryEnabled
) {
logEvent('tengu_api_custom_529_overloaded_error', {})
throw new CannotRetryError(
Expand All @@ -389,17 +396,33 @@ export async function* withRetry<T>(

// Only retry if the error indicates we should
const persistent =
isPersistentRetryEnabled() && isTransientCapacityError(error)
persistentRetryEnabled && isTransientCapacityError(error)
if (attempt > maxRetries && !persistent) {
throw new CannotRetryError(error, retryContext)
}
// Cap persistent retries to prevent unbounded loops (100 attempts * ~5min max backoff ≈ 8 hours).
// The "~8 hours" estimate refers only to the exponential backoff path; actual wall-clock
// time depends on retry-after headers (server directives) which can add significant delay.
if (persistent && persistentAttempt >= PERSISTENT_MAX_ATTEMPTS) {
logEvent('tengu_api_persistent_retry_cap_reached', {
error: (error as APIError).message as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
status: (error as APIError).status,
model: retryContext.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
persistentAttempt,
PERSISTENT_MAX_BACKOFF_MS,
PERSISTENT_MAX_ATTEMPTS,
provider: getAPIProviderForStatsig(),
})
throw new CannotRetryError(error, retryContext)
}

// AWS/GCP errors aren't always APIError, but can be retried
const handledCloudAuthError =
handleAwsCredentialError(error) || handleGcpCredentialError(error)
if (
!handledCloudAuthError &&
(!(error instanceof APIError) || !shouldRetry(error))
(!(error instanceof APIError) ||
!shouldRetry(error, persistentRetryEnabled))
) {
throw new CannotRetryError(error, retryContext)
}
Expand Down Expand Up @@ -776,15 +799,15 @@ function handleGcpCredentialError(error: unknown): boolean {
return false
}

function shouldRetry(error: APIError): boolean {
function shouldRetry(error: APIError, persistentRetryEnabled: boolean): boolean {
// Never retry mock errors - they're from /mock-limits command for testing
if (isMockRateLimitError(error)) {
return false
}

// Persistent mode: 429/529 always retryable, bypass subscriber gates and
// x-should-retry header.
if (isPersistentRetryEnabled() && isTransientCapacityError(error)) {
if (persistentRetryEnabled && isTransientCapacityError(error)) {
return true
}

Expand Down
36 changes: 31 additions & 5 deletions src/services/compact/autoCompact.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -247,11 +247,37 @@ describe('getAutoCompactThreshold', () => {
})

describe('getAutoCompactFailureCooldownMs', () => {
test('uses valid positive integer override', async () => {
process.env.OPENCLAUDE_AUTOCOMPACT_FAILURE_COOLDOWN_MS = ' 5000 '
test('uses valid positive integer override above the floor', async () => {
process.env.OPENCLAUDE_AUTOCOMPACT_FAILURE_COOLDOWN_MS = ' 15000 '
const { getAutoCompactFailureCooldownMs } = await importAutoCompact()

expect(getAutoCompactFailureCooldownMs()).toBe(5000)
expect(getAutoCompactFailureCooldownMs()).toBe(15000)
})

test('rejects overrides below the minimum cooldown floor', async () => {
const {
AUTOCOMPACT_FAILURE_COOLDOWN_MS,
getAutoCompactFailureCooldownMs,
MIN_AUTOCOMPACT_FAILURE_COOLDOWN_MS,
} = await importAutoCompact()

// 5000 is below the 10_000ms floor — must fall back to the default
// rather than being accepted as a valid test override.
process.env.OPENCLAUDE_AUTOCOMPACT_FAILURE_COOLDOWN_MS = '5000'
expect(getAutoCompactFailureCooldownMs()).toBe(
AUTOCOMPACT_FAILURE_COOLDOWN_MS,
)
expect(MIN_AUTOCOMPACT_FAILURE_COOLDOWN_MS).toBe(10_000)

// Boundary: exactly the floor value is accepted.
process.env.OPENCLAUDE_AUTOCOMPACT_FAILURE_COOLDOWN_MS = '10000'
expect(getAutoCompactFailureCooldownMs()).toBe(10_000)

// One below the floor is rejected.
process.env.OPENCLAUDE_AUTOCOMPACT_FAILURE_COOLDOWN_MS = '9999'
expect(getAutoCompactFailureCooldownMs()).toBe(
AUTOCOMPACT_FAILURE_COOLDOWN_MS,
)
})

test('ignores partial or invalid override values', async () => {
Expand Down Expand Up @@ -453,7 +479,7 @@ describe('resolveAutoCompactCircuitBreakerState', () => {
describe('autoCompactIfNeeded circuit breaker', () => {
beforeEach(() => {
process.env.CLAUDE_AUTOCOMPACT_PCT_OVERRIDE = '1'
process.env.OPENCLAUDE_AUTOCOMPACT_FAILURE_COOLDOWN_MS = '5000'
process.env.OPENCLAUDE_AUTOCOMPACT_FAILURE_COOLDOWN_MS = '15000'
})

test('trips after three non-user failures and records a retry time', async () => {
Expand Down Expand Up @@ -648,7 +674,7 @@ describe('autoCompactIfNeeded circuit breaker', () => {
)

expect(result.lastFailureAtMs).toBe(106_000)
expect(result.nextRetryAtMs).toBe(111_000)
expect(result.nextRetryAtMs).toBe(121_000)
} finally {
Date.now = originalDateNow
}
Expand Down
Loading
Loading