diff --git a/packages/cli/src/nonInteractiveCli.ts b/packages/cli/src/nonInteractiveCli.ts index 4e3b04f79f5..5c370ae65c1 100644 --- a/packages/cli/src/nonInteractiveCli.ts +++ b/packages/cli/src/nonInteractiveCli.ts @@ -109,6 +109,12 @@ const LOOP_TYPE_LABELS: Record = { 'the model spent too many consecutive calls reading files without making progress', [LoopType.ACTION_STAGNATION]: 'the model kept calling the same tool without making progress', + [LoopType.GLOBAL_TOOL_CALL_DUPLICATE]: + 'the model repeated the same tool call across the turn, even when not back-to-back', + [LoopType.ALTERNATING_TOOL_CALL_PATTERN]: + 'the model alternated between the same two tool calls in a repeating pattern', + [LoopType.TURN_TOOL_CALL_CAP]: + 'the model exceeded the maximum number of tool calls allowed in a single turn', }; function emitLoopDetectedMessage( @@ -122,9 +128,13 @@ function emitLoopDetectedMessage( } const reason = loopType ? LOOP_TYPE_LABELS[loopType] : undefined; const detail = reason ? ` (${loopType}: ${reason})` : ''; - process.stderr.write( - `Loop detection halted the run${detail}. Set the \`model.skipLoopDetection\` setting to true to disable.\n`, - ); + // The turn cap runs before the skipLoopDetection gate, so that setting can't + // disable it — don't suggest it for TURN_TOOL_CALL_CAP. + const hint = + loopType === LoopType.TURN_TOOL_CALL_CAP + ? ' This is an always-on per-turn tool-call cap and cannot be disabled via `model.skipLoopDetection`.' + : ' Set the `model.skipLoopDetection` setting to true to disable.'; + process.stderr.write(`Loop detection halted the run${detail}.${hint}\n`); } /** diff --git a/packages/core/src/core/client.test.ts b/packages/core/src/core/client.test.ts index f71066b2dc9..bfcf47fd379 100644 --- a/packages/core/src/core/client.test.ts +++ b/packages/core/src/core/client.test.ts @@ -4286,6 +4286,82 @@ hello expect(client['pendingMemoryPrefetch']).toBeUndefined(); }); + it('should halt via the always-on turn cap before the skipLoopDetection gate', async () => { + let abortHandlerInvoked = false; + mockMemoryManager.recall.mockImplementation((_root, _query, opts) => { + opts.abortSignal?.addEventListener('abort', () => { + abortHandlerInvoked = true; + }); + return new Promise(() => {}); + }); + + const mockChat: Partial = { + addHistory: vi.fn(), + getHistory: vi.fn().mockReturnValue([]), + getHistoryLength: vi.fn().mockReturnValue(0), + }; + client['chat'] = mockChat as GeminiChat; + + // The always-on cap trips on the first event — it runs before (and + // independently of) the gated detectors. + const loopDetector = client['loopDetector']; + const alwaysOnSpy = vi + .spyOn(loopDetector, 'checkAlwaysOnSafeties') + .mockReturnValue(true); + const deterministicSpy = vi.spyOn( + loopDetector, + 'addAndCheckDeterministicToolCallLoop', + ); + vi.spyOn(loopDetector, 'getLastLoopType').mockReturnValue( + LoopType.TURN_TOOL_CALL_CAP, + ); + + // `run` is invoked as `turn.run(...)`, so `this` is the live Turn — + // populate pendingToolCalls the way the real Turn.run does as it streams + // ToolCallRequest chunks, so the halt's clear runs against a non-empty + // array (not a trivially-empty one). + mockTurnRunFn.mockImplementation(async function* (this: { + pendingToolCalls: unknown[]; + }) { + this.pendingToolCalls.push( + { name: 'read_file', args: { path: 'a.ts' } }, + { name: 'read_file', args: { path: 'b.ts' } }, + ); + yield { type: 'content', value: 'looping' }; + }); + + const stream = client.sendMessageStream( + [{ text: 'trigger the cap' }], + new AbortController().signal, + 'prompt-id-cap', + { type: SendMessageType.UserQuery }, + ); + const events = []; + let result = await stream.next(); + while (!result.done) { + events.push(result.value); + result = await stream.next(); + } + const returnedTurn = result.value as + | { pendingToolCalls: unknown[] } + | undefined; + + // Always-on cap fires and short-circuits before the gated detectors run. + expect(alwaysOnSpy).toHaveBeenCalled(); + expect(deterministicSpy).not.toHaveBeenCalled(); + const loopEvent = events.find( + (e) => e.type === GeminiEventType.LoopDetected, + ); + expect(loopEvent?.value?.loopType).toBe(LoopType.TURN_TOOL_CALL_CAP); + // The two pending calls collected before the cap tripped are dropped, so + // the halt doesn't spawn a continuation that re-trips the cap and + // double-prints the message. + expect(returnedTurn?.pendingToolCalls).toHaveLength(0); + // The mid-stream memory prefetch is cancelled. + expect(abortHandlerInvoked).toBe(true); + expect(client['pendingMemoryPrefetch']).toBeUndefined(); + }); + it('should PRESERVE the pending prefetch when next-speaker continueTurn returns', async () => { // Self-inflicted-regression guard for the round-4 finding: // the bottom-of-try `normalCompletion = true` doesn't cover the @@ -6182,6 +6258,7 @@ Other open files: // Replace loop detector with spies const ldMock = { + checkAlwaysOnSafeties: vi.fn().mockReturnValue(false), addAndCheckDeterministicToolCallLoop: vi.fn().mockReturnValue(false), addAndCheckHeuristicLoops: vi.fn().mockReturnValue(false), reset: vi.fn(), @@ -6211,7 +6288,8 @@ Other open files: // consume stream } - // Assert - neither detector path runs when skipLoopDetection is true + // Assert - always-on safeties still run, but opt-in detectors don't + expect(ldMock.checkAlwaysOnSafeties).toHaveBeenCalled(); expect( ldMock.addAndCheckDeterministicToolCallLoop, ).not.toHaveBeenCalled(); diff --git a/packages/core/src/core/client.ts b/packages/core/src/core/client.ts index c59b59eeef2..c79ce048c96 100644 --- a/packages/core/src/core/client.ts +++ b/packages/core/src/core/client.ts @@ -2123,6 +2123,31 @@ export class GeminiClient { didUpdateIdeContextState = true; } + // Always-on safety checks (turn tool-call cap). These fire before + // the skipLoopDetection gate so they cannot be bypassed by + // configuration. + const alwaysOnLoop = this.loopDetector.checkAlwaysOnSafeties(event); + if (alwaysOnLoop) { + // The tripping response may carry several tool calls collected + // before the cap fired. Drop them so the run halts here instead of + // executing them, spawning a continuation, and re-tripping the cap + // (which would double-print the halt message and waste a request). + turn.pendingToolCalls.length = 0; + const loopType = this.loopDetector.getLastLoopType(); + yield { + type: GeminiEventType.LoopDetected, + ...(loopType && { value: { loopType } }), + }; + if (arenaAgentClient) { + await arenaAgentClient.reportError('Loop detected'); + } + this.lastApiCompletionTimestamp = Date.now(); + if (isTopLevelInteraction) + endInteractionSpan('error', { errorMessage: 'loop detected' }); + this.cancelPendingMemoryPrefetch(); + return turn; + } + // Loop detection is opt-in: `model.skipLoopDetection` defaults to true // (see settingsSchema) to avoid false-positive interruptions. Keep BOTH // the deterministic identical-tool-call check and the heuristic checks diff --git a/packages/core/src/services/loopDetectionService.test.ts b/packages/core/src/services/loopDetectionService.test.ts index d4ea53ee045..7b9d673499f 100644 --- a/packages/core/src/services/loopDetectionService.test.ts +++ b/packages/core/src/services/loopDetectionService.test.ts @@ -14,6 +14,7 @@ import type { } from '../core/turn.js'; import { GeminiEventType } from '../core/turn.js'; import * as loggers from '../telemetry/loggers.js'; +import { LoopType } from '../telemetry/types.js'; import { LoopDetectionService } from './loopDetectionService.js'; vi.mock('../telemetry/loggers.js', () => ({ @@ -27,6 +28,9 @@ const CONTENT_CHUNK_SIZE = 50; // Mirrored from loopDetectionService.ts. Kept local so the test is // self-describing and failures point to the constant that changed. const FILE_READ_WINDOW = 15; +const GLOBAL_DUPLICATE_THRESHOLD = 6; +const ALTERNATING_PATTERN_CYCLES = 3; +const TURN_TOOL_CALL_CAP = 100; describe('LoopDetectionService', () => { let service: LoopDetectionService; @@ -1021,4 +1025,342 @@ describe('LoopDetectionService', () => { } }); }); + + describe('Turn Tool Call Cap (Always-On Circuit Breaker)', () => { + it('should not fire when total calls are below the cap', () => { + service.reset(''); + for (let i = 0; i < TURN_TOOL_CALL_CAP; i++) { + const isLoop = service.checkAlwaysOnSafeties( + createToolCallRequestEvent('any_tool', { i }), + ); + expect(isLoop).toBe(false); + } + }); + + it('should fire on the call that exceeds the cap', () => { + service.reset(''); + for (let i = 0; i < TURN_TOOL_CALL_CAP; i++) { + service.checkAlwaysOnSafeties( + createToolCallRequestEvent('any_tool', { i }), + ); + } + const isLoop = service.checkAlwaysOnSafeties( + createToolCallRequestEvent('any_tool', { extra: true }), + ); + expect(isLoop).toBe(true); + expect(loggers.logLoopDetected).toHaveBeenCalledTimes(1); + // The turn cap reports its own loop type, not consecutive-identical. + expect(loggers.logLoopDetected).toHaveBeenCalledWith( + mockConfig, + expect.objectContaining({ + loop_type: 'turn_tool_call_cap', + }), + ); + expect(service.getLastLoopType()).toBe(LoopType.TURN_TOOL_CALL_CAP); + }); + + it('should fire regardless of disabledForSession', () => { + service.reset(''); + service.disableForSession(); + // disableForSession prevents heuristic checks, but not the turn cap + for (let i = 0; i < TURN_TOOL_CALL_CAP; i++) { + service.checkAlwaysOnSafeties( + createToolCallRequestEvent('any_tool', { i }), + ); + } + const isLoop = service.checkAlwaysOnSafeties( + createToolCallRequestEvent('any_tool', { extra: true }), + ); + // disabledForSession blocks non-ToolCallRequest events in + // checkAlwaysOnSafeties, but this IS a ToolCallRequest so the cap + // still fires. + expect(isLoop).toBe(true); + }); + + const retryEvent = { + type: GeminiEventType.Retry, + } as ServerGeminiStreamEvent; + const finishedEvent = { + type: GeminiEventType.Finished, + value: { reason: 'STOP' }, + } as unknown as ServerGeminiStreamEvent; + + it('rolls back a failed attempt on retry so its calls do not count', () => { + service.reset(''); + // Attempt makes 60 calls, then the API retries (no round-trip committed + // yet, so the rollback floor is 0). + for (let i = 0; i < 60; i++) { + service.checkAlwaysOnSafeties(createToolCallRequestEvent('t', { i })); + } + service.checkAlwaysOnSafeties(retryEvent); + // The 60 discarded calls must not count: a full cap's worth of fresh + // calls stays under the limit, and only the (cap+1)-th fires. + for (let i = 0; i < TURN_TOOL_CALL_CAP; i++) { + expect( + service.checkAlwaysOnSafeties( + createToolCallRequestEvent('t', { j: i }), + ), + ).toBe(false); + } + expect( + service.checkAlwaysOnSafeties( + createToolCallRequestEvent('t', { last: true }), + ), + ).toBe(true); + expect(loggers.logLoopDetected).toHaveBeenCalledTimes(1); + }); + + it('preserves committed round-trip counts when a later attempt retries', () => { + service.reset(''); + // Round-trip 1: 60 calls, then Finished commits them as the floor. + for (let i = 0; i < 60; i++) { + service.checkAlwaysOnSafeties(createToolCallRequestEvent('t', { i })); + } + service.checkAlwaysOnSafeties(finishedEvent); + // Round-trip 2: 30 calls, then a retry discards only these 30. + for (let i = 0; i < 30; i++) { + service.checkAlwaysOnSafeties( + createToolCallRequestEvent('t', { k: i }), + ); + } + service.checkAlwaysOnSafeties(retryEvent); + // Total is back to the committed 60 (NOT zero): 40 more reach exactly the + // cap without firing, and the next call trips it. + for (let i = 0; i < TURN_TOOL_CALL_CAP - 60; i++) { + expect( + service.checkAlwaysOnSafeties( + createToolCallRequestEvent('t', { m: i }), + ), + ).toBe(false); + } + expect( + service.checkAlwaysOnSafeties( + createToolCallRequestEvent('t', { last: true }), + ), + ).toBe(true); + }); + + it('still accumulates across committed round-trips to trip the cap', () => { + service.reset(''); + let fired = false; + // 11 calls/round-trip; the cap (100) is crossed partway through. + for (let rt = 0; rt < 12 && !fired; rt++) { + for (let i = 0; i < 11 && !fired; i++) { + fired = service.checkAlwaysOnSafeties( + createToolCallRequestEvent('t', { rt, i }), + ); + } + if (!fired) { + service.checkAlwaysOnSafeties(finishedEvent); + } + } + expect(fired).toBe(true); + expect(service.getLastLoopType()).toBe(LoopType.TURN_TOOL_CALL_CAP); + }); + }); + + describe('Global Tool Call Duplicate Detection', () => { + it('should not fire when same call appears fewer than threshold times', () => { + service.reset(''); + const event = createToolCallRequestEvent('stuck_tool', { + param: 'same', + }); + for (let i = 0; i < GLOBAL_DUPLICATE_THRESHOLD - 1; i++) { + const isLoop = service.addAndCheckHeuristicLoops(event); + expect(isLoop).toBe(false); + } + }); + + it('should fire when same (tool, args) appears threshold times non-consecutively', () => { + service.reset(''); + const stuckEvent = createToolCallRequestEvent('stuck_tool', { + param: 'same', + }); + const otherEvents = [ + createToolCallRequestEvent('other_a', { x: 1 }), + createToolCallRequestEvent('other_b', { y: 2 }), + createToolCallRequestEvent('other_c', { z: 3 }), + ]; + + // Interleave: stuck, other_a, stuck, other_b, stuck, other_c, ... + // GLOBAL_DUPLICATE_THRESHOLD total stuck calls with different calls between + let otherIdx = 0; + for (let i = 0; i < GLOBAL_DUPLICATE_THRESHOLD - 1; i++) { + expect(service.addAndCheckHeuristicLoops(stuckEvent)).toBe(false); + expect( + service.addAndCheckHeuristicLoops( + otherEvents[otherIdx % otherEvents.length], + ), + ).toBe(false); + otherIdx++; + } + // The threshold-th stuck call should fire + const isLoop = service.addAndCheckHeuristicLoops(stuckEvent); + expect(isLoop).toBe(true); + expect(loggers.logLoopDetected).toHaveBeenCalledWith( + mockConfig, + expect.objectContaining({ + loop_type: 'global_tool_call_duplicate', + }), + ); + // getLastLoopType() is the getter the client uses to populate the + // bubbled LoopDetected event, so assert it too — not just the logged one. + expect(service.getLastLoopType()).toBe( + LoopType.GLOBAL_TOOL_CALL_DUPLICATE, + ); + }); + + it('should not fire for different (tool, args) pairs', () => { + service.reset(''); + for (let i = 0; i < GLOBAL_DUPLICATE_THRESHOLD; i++) { + const isLoop = service.addAndCheckHeuristicLoops( + createToolCallRequestEvent('stuck_tool', { param: i }), + ); + expect(isLoop).toBe(false); + } + expect(loggers.logLoopDetected).not.toHaveBeenCalled(); + }); + + it('should fire for consecutive identical calls via both detectors', () => { + // The heuristic path also runs checkGlobalDuplicate on every + // ToolCallRequest, so a consecutive run of 5 identical calls trips + // the consecutive detector first (threshold 5 < global 6). This test + // verifies the global path would also fire if the consecutive + // detector were disabled. + service.reset(''); + const event = createToolCallRequestEvent('stuck_tool', { + param: 'same', + }); + for (let i = 0; i < GLOBAL_DUPLICATE_THRESHOLD - 1; i++) { + service.addAndCheckHeuristicLoops(event); + } + const isLoop = service.addAndCheckHeuristicLoops(event); + expect(isLoop).toBe(true); + expect(loggers.logLoopDetected).toHaveBeenCalledWith( + mockConfig, + expect.objectContaining({ + loop_type: 'global_tool_call_duplicate', + }), + ); + }); + + it('does not count a retried replay toward the global-duplicate threshold', () => { + service.reset(''); + const stuck = createToolCallRequestEvent('stuck_tool', { param: 'same' }); + const retry = { type: GeminiEventType.Retry } as ServerGeminiStreamEvent; + // Failed attempt streams (threshold - 3) identical calls, then retries. + for (let i = 0; i < GLOBAL_DUPLICATE_THRESHOLD - 3; i++) { + expect(service.addAndCheckHeuristicLoops(stuck)).toBe(false); + } + service.addAndCheckHeuristicLoops(retry); + // The replay streams the same calls again. Without the Retry reset the + // pre- and post-retry counts would sum to the threshold and false-fire. + for (let i = 0; i < GLOBAL_DUPLICATE_THRESHOLD - 3; i++) { + expect(service.addAndCheckHeuristicLoops(stuck)).toBe(false); + } + expect(loggers.logLoopDetected).not.toHaveBeenCalled(); + }); + }); + + describe('Alternating Tool Call Pattern Detection', () => { + it('should fire for a clean ABABAB alternating pattern', () => { + service.reset(''); + const eventA = createToolCallRequestEvent('tool_a', { param: 'a' }); + const eventB = createToolCallRequestEvent('tool_b', { param: 'b' }); + + // ALTERNATING_PATTERN_CYCLES cycles = 2*CYCLES calls. Build up to + // one call short of the trigger. + const totalCycles = ALTERNATING_PATTERN_CYCLES; + for (let i = 0; i < totalCycles - 1; i++) { + expect(service.addAndCheckHeuristicLoops(eventA)).toBe(false); + expect(service.addAndCheckHeuristicLoops(eventB)).toBe(false); + } + // First call of the final cycle + expect(service.addAndCheckHeuristicLoops(eventA)).toBe(false); + // Second call of the final cycle completes the pattern + const isLoop = service.addAndCheckHeuristicLoops(eventB); + expect(isLoop).toBe(true); + expect(loggers.logLoopDetected).toHaveBeenCalledWith( + mockConfig, + expect.objectContaining({ + loop_type: 'alternating_tool_call_pattern', + }), + ); + expect(service.getLastLoopType()).toBe( + LoopType.ALTERNATING_TOOL_CALL_PATTERN, + ); + }); + + it('should not fire when calls alternate but with varying keys', () => { + service.reset(''); + // Alternating tool names but different args each time → different + // keys → no clean ABAB because the keys keep changing. + const totalCycles = ALTERNATING_PATTERN_CYCLES + 2; + for (let i = 0; i < totalCycles; i++) { + expect( + service.addAndCheckHeuristicLoops( + createToolCallRequestEvent('tool_a', { param: i }), + ), + ).toBe(false); + expect( + service.addAndCheckHeuristicLoops( + createToolCallRequestEvent('tool_b', { param: i }), + ), + ).toBe(false); + } + expect(loggers.logLoopDetected).not.toHaveBeenCalled(); + }); + + it('should not fire for a single tool repeated (consecutive, not alternating)', () => { + service.reset(''); + const event = createToolCallRequestEvent('tool_a', { param: 'a' }); + const totalCalls = 2 * ALTERNATING_PATTERN_CYCLES; + for (let i = 0; i < totalCalls; i++) { + // The consecutive identical detector would fire at threshold 5, + // but we only check the heuristic path here. At 6 calls the + // global duplicate detector fires. This test just confirms the + // alternating detector doesn't false-positive on a repeated key. + service.addAndCheckHeuristicLoops(event); + } + // Either global_duplicate or consecutive_identical fires — we just + // verify the alternating pattern detector didn't fire. + const logged = vi.mocked(loggers.logLoopDetected).mock.calls; + const alternatingFired = logged.some((call) => { + const event = call[1] as unknown as Record; + return 'loop_type' in event + ? event['loop_type'] === 'alternating_tool_call_pattern' + : false; + }); + expect(alternatingFired).toBe(false); + }); + + it('should reset alternating window after a different third pattern', () => { + service.reset(''); + const eventA = createToolCallRequestEvent('tool_a', { param: 'a' }); + const eventB = createToolCallRequestEvent('tool_b', { param: 'b' }); + const eventC = createToolCallRequestEvent('tool_c', { param: 'c' }); + + // Build up ABAB + service.addAndCheckHeuristicLoops(eventA); + service.addAndCheckHeuristicLoops(eventB); + service.addAndCheckHeuristicLoops(eventA); + service.addAndCheckHeuristicLoops(eventB); + // Insert C to break the pattern + service.addAndCheckHeuristicLoops(eventC); + // Restart ABAB from here — need 6 calls (3 cycles) after the break + service.addAndCheckHeuristicLoops(eventA); + service.addAndCheckHeuristicLoops(eventB); + service.addAndCheckHeuristicLoops(eventA); + service.addAndCheckHeuristicLoops(eventB); + expect(service.addAndCheckHeuristicLoops(eventA)).toBe(false); + const isLoop = service.addAndCheckHeuristicLoops(eventB); + expect(isLoop).toBe(true); + expect(loggers.logLoopDetected).toHaveBeenCalledWith( + mockConfig, + expect.objectContaining({ + loop_type: 'alternating_tool_call_pattern', + }), + ); + }); + }); }); diff --git a/packages/core/src/services/loopDetectionService.ts b/packages/core/src/services/loopDetectionService.ts index f6e4d909332..a9128092da7 100644 --- a/packages/core/src/services/loopDetectionService.ts +++ b/packages/core/src/services/loopDetectionService.ts @@ -44,6 +44,20 @@ const FILE_READ_WINDOW = 15; // Action stagnation tracking const STAGNATION_THRESHOLD = 8; +// Global tool call duplicate tracking: how many times the same (tool, args) +// pair must appear across the entire turn (not necessarily consecutively) +// before it is treated as a loop. +const GLOBAL_DUPLICATE_THRESHOLD = 6; + +// Alternating pattern detection: number of complete AB cycles needed to +// trip the detector (3 cycles = 6 calls: A B A B A B). +const ALTERNATING_PATTERN_CYCLES = 3; + +// Hard per-turn tool call cap. Always-on circuit breaker — not gated by +// skipLoopDetection. If a single turn exceeds this many tool calls the +// turn is halted regardless of loop-detection configuration. +const TURN_TOOL_CALL_CAP = 100; + /** * Service for detecting and preventing infinite loops in AI responses. * Monitors tool call repetitions and content sentence repetitions. @@ -85,6 +99,27 @@ export class LoopDetectionService { // exploration rather than loop evidence. Resets per-prompt in reset(). private hasSeenNonReadTool = false; + // Non-consecutive global duplicate tracking: counts every (tool, args) + // pair seen across the entire turn. When any pair reaches + // GLOBAL_DUPLICATE_THRESHOLD, the turn is halted. + private globalToolCallCounts = new Map(); + + // Sliding window of recent tool-call keys for alternating-pattern + // detection (ABABAB…). Kept at 2 * ALTERNATING_PATTERN_CYCLES entries. + private recentToolCallKeys: string[] = []; + + // Total tool calls emitted in the current turn. Always-on circuit breaker; + // exceeds TURN_TOOL_CALL_CAP → hard-stop. Accumulates across ToolResult + // continuations within a turn (reset() only runs for top-level interactions). + private turnToolCallTotal = 0; + + // Rollback floor for turnToolCallTotal: the committed total as of the last + // completed round-trip (Finished event). A retry re-streams the failed + // attempt's tool calls (Turn clears pendingToolCalls on retry), so on Retry + // we roll back to this floor — discarding only the failed attempt, not the + // counts from prior completed round-trips. + private turnToolCallTotalCommitted = 0; + // Loop type of the most recent firing. Bubbled up through the // LoopDetected event so callers (non-interactive CLI, telemetry) can tell // the user which detector actually fired. @@ -152,10 +187,26 @@ export class LoopDetectionService { this.thoughtHistory = []; this.trackToolCall(event.value); + const toolCallKey = this.getToolCallKey(event.value); + const globalDup = this.checkGlobalDuplicate(toolCallKey); + const alternating = this.checkAlternatingPattern(toolCallKey); const readFileLoop = this.checkReadFileLoop(); const actionStagnation = this.checkActionStagnation(); - this.loopDetected = readFileLoop || actionStagnation; + this.loopDetected = + globalDup || alternating || readFileLoop || actionStagnation; + break; + } + case GeminiEventType.Retry: { + // A retry replays the failed attempt's tool calls (Turn clears + // pendingToolCalls on retry), so drop the counters this PR added to + // avoid the heuristic detectors firing on a duplicated replay — e.g. + // 3 identical calls + Retry + 3 more would otherwise hit the + // global-duplicate threshold of 6. Mirrors the deterministic path's + // resetToolCallCount() on Retry; the always-on cap keeps its own + // counter accurate via commit/rollback instead. + this.globalToolCallCounts.clear(); + this.recentToolCallKeys = []; break; } case GeminiEventType.Content: { @@ -198,6 +249,44 @@ export class LoopDetectionService { return this.loopDetected; } + /** + * Always-on safety checks that fire regardless of skipLoopDetection. + * Currently enforces the per-turn tool call cap. Call this before the + * gated checks so the hard cap cannot be bypassed by configuration. + */ + checkAlwaysOnSafeties(event: ServerGeminiStreamEvent): boolean { + if (this.loopDetected) { + return true; + } + + // A model response (round-trip) finished cleanly: commit its tool-call + // count as the rollback floor. The per-turn total accumulates across + // ToolResult continuations, so the floor must track the last committed + // round-trip rather than resetting to zero. + if (event.type === GeminiEventType.Finished) { + this.turnToolCallTotalCommitted = this.turnToolCallTotal; + return false; + } + + // A retry re-streams the failed attempt's tool calls, which would + // double-count against the cap. Roll back to the last committed round-trip + // so only executed calls count — never below it (prior round-trips stay). + if (event.type === GeminiEventType.Retry) { + this.turnToolCallTotal = this.turnToolCallTotalCommitted; + return false; + } + + if (event.type !== GeminiEventType.ToolCallRequest) { + return false; + } + + if (this.checkTurnToolCallCap()) { + this.loopDetected = true; + return true; + } + return false; + } + private checkToolCallLoop(toolCall: { name: string; args: object }): boolean { const key = this.getToolCallKey(toolCall); if (this.lastToolCallKey === key) { @@ -550,6 +639,89 @@ export class LoopDetectionService { return false; } + /** + * Always-on hard cap: if the turn exceeds TURN_TOOL_CALL_CAP tool calls + * the turn is halted. This is a safety net independent of + * skipLoopDetection and fires on the very next tool call that pushes the + * total past the cap, not retroactively. + */ + private checkTurnToolCallCap(): boolean { + this.turnToolCallTotal++; + if (this.turnToolCallTotal > TURN_TOOL_CALL_CAP) { + this.lastLoopType = LoopType.TURN_TOOL_CALL_CAP; + logLoopDetected( + this.config, + new LoopDetectedEvent(LoopType.TURN_TOOL_CALL_CAP, this.promptId), + ); + return true; + } + return false; + } + + /** + * Non-consecutive global duplicate detection: the SAME (tool, args) pair + * need not appear consecutively — if it appears GLOBAL_DUPLICATE_THRESHOLD + * times anywhere in the turn, it is treated as a loop. This catches models + * that intersperse the stuck call among other actions. + */ + private checkGlobalDuplicate(toolCallKey: string): boolean { + const count = (this.globalToolCallCounts.get(toolCallKey) ?? 0) + 1; + this.globalToolCallCounts.set(toolCallKey, count); + + if (count >= GLOBAL_DUPLICATE_THRESHOLD) { + this.lastLoopType = LoopType.GLOBAL_TOOL_CALL_DUPLICATE; + logLoopDetected( + this.config, + new LoopDetectedEvent( + LoopType.GLOBAL_TOOL_CALL_DUPLICATE, + this.promptId, + ), + ); + return true; + } + return false; + } + + /** + * Alternating-pattern detection: catches ABABAB… patterns where the model + * flips between two distinct tool calls. Tracked via a sliding window of + * tool-call keys; when the window fills with alternating A/B values the + * turn is halted. + */ + private checkAlternatingPattern(toolCallKey: string): boolean { + const maxLen = 2 * ALTERNATING_PATTERN_CYCLES; + this.recentToolCallKeys.push(toolCallKey); + if (this.recentToolCallKeys.length > maxLen) { + this.recentToolCallKeys.shift(); + } + + if (this.recentToolCallKeys.length < maxLen) { + return false; + } + + // Extract the two alternating keys. If there are more than two distinct + // keys in the window, there is no clean ABAB pattern. + const [a, b] = this.recentToolCallKeys; + if (a === b) return false; // not alternating, same tool + + for (let i = 0; i < maxLen; i++) { + const expected = i % 2 === 0 ? a : b; + if (this.recentToolCallKeys[i] !== expected) { + return false; + } + } + + this.lastLoopType = LoopType.ALTERNATING_TOOL_CALL_PATTERN; + logLoopDetected( + this.config, + new LoopDetectedEvent( + LoopType.ALTERNATING_TOOL_CALL_PATTERN, + this.promptId, + ), + ); + return true; + } + /** * Resets all loop detection state. */ @@ -566,6 +738,10 @@ export class LoopDetectionService { this.lastSeenToolName = null; this.hasSeenNonReadTool = false; this.lastLoopType = null; + this.globalToolCallCounts.clear(); + this.recentToolCallKeys = []; + this.turnToolCallTotal = 0; + this.turnToolCallTotalCommitted = 0; } private resetToolCallCount(): void { diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts index f33880c9815..ed698c7ae36 100644 --- a/packages/core/src/telemetry/types.ts +++ b/packages/core/src/telemetry/types.ts @@ -424,6 +424,12 @@ export enum LoopType { REPETITIVE_THOUGHTS = 'repetitive_thoughts', READ_FILE_LOOP = 'read_file_loop', ACTION_STAGNATION = 'action_stagnation', + /** Same (tool, args) pair appears N times across the entire turn, not necessarily consecutively. */ + GLOBAL_TOOL_CALL_DUPLICATE = 'global_tool_call_duplicate', + /** Two tools alternating in a fixed pattern (A B A B A B ...). */ + ALTERNATING_TOOL_CALL_PATTERN = 'alternating_tool_call_pattern', + /** Total tool calls in a single turn exceeded the always-on hard cap, regardless of pattern. */ + TURN_TOOL_CALL_CAP = 'turn_tool_call_cap', } export class LoopDetectedEvent implements BaseTelemetryEvent {