From 90ed1329c772e238098e9088d5f38bac03232c33 Mon Sep 17 00:00:00 2001 From: Kentaro Wakayama Date: Mon, 31 Aug 2026 00:01:56 +0200 Subject: [PATCH 1/2] test: reproduce repeated error-level retry logs past the mirror flush threshold (issue-inbox#821) --- .../conversation/run-chunk-mirror.test.ts | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/src/agent/conversation/run-chunk-mirror.test.ts b/src/agent/conversation/run-chunk-mirror.test.ts index de83ee8d8d..4d24714f15 100644 --- a/src/agent/conversation/run-chunk-mirror.test.ts +++ b/src/agent/conversation/run-chunk-mirror.test.ts @@ -620,4 +620,93 @@ describe("agent/conversation-run-chunk-mirror", () => { "an oversized-event stop must be reported at error level with the cursor metadata", ); }); + + // VERYFRONT-AGENT-3 (veryfront-issue-inbox#821): the threshold added for the + // first incident stopped the first four retries from paging, but every + // attempt at or past the threshold still logged at error level. Retry + // backoff caps at ~5s, so one persistent append outage resumes emitting a + // Sentry error every ~5s per active run once the streak reaches five. + // Escalation must fire once per failure streak; the remaining attempts in + // the same streak stay at warn. A successful flush ends the streak, so the + // next persistent outage must escalate again. + it("escalates a persistent retry streak to error once, not on every attempt", async () => { + const logs: Array<{ level: "warn" | "error"; message: string; consecutiveFailures: unknown }> = + []; + let failing = true; + const flakyFetch = (() => + Promise.resolve( + failing + ? new Response( + JSON.stringify({ detail: "upstream unavailable" }), + { status: 503, headers: { "Content-Type": "application/json" } }, + ) + : new Response( + JSON.stringify({ + latestEventId: 1, + latestExternalEventSequence: 1, + appendedCount: 1, + run: { + runId: "run-1", + conversationId: "11111111-1111-4111-8111-111111111111", + latestEventId: 1, + latestExternalEventSequence: 1, + }, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + )) as typeof fetch; + const mirror = createHostedConversationRunChunkMirror({ + authToken: "token", + apiUrl: "https://api.example.test", + conversationId: "11111111-1111-4111-8111-111111111111", + runId: "run-1", + latestEventId: 0, + fetch: flakyFetch, + instrumentation: { + warn: (message, metadata) => { + logs.push({ level: "warn", message, consecutiveFailures: metadata.consecutiveFailures }); + }, + error: (message, metadata) => { + logs.push({ level: "error", message, consecutiveFailures: metadata.consecutiveFailures }); + }, + }, + }); + + await mirror.appendEvents([{ type: "TEXT_MESSAGE_CONTENT", delta: "persisted" }]); + for (let attempt = 0; attempt < 8; attempt += 1) { + await mirror.flush(); + } + + failing = false; + await mirror.flush(); + + failing = true; + await mirror.appendEvents([{ type: "TEXT_MESSAGE_CONTENT", delta: "queued again" }]); + for (let attempt = 0; attempt < 5; attempt += 1) { + await mirror.flush(); + } + mirror.dispose(); + + assertEquals( + logs, + [ + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 1 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 2 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 3 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 4 }, + { level: "error", message: RETRY_LOG_MESSAGE, consecutiveFailures: 5 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 6 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 7 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 8 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 1 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 2 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 3 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 4 }, + { level: "error", message: RETRY_LOG_MESSAGE, consecutiveFailures: 5 }, + ], + "a failure streak must escalate to error exactly once at the threshold; " + + "later attempts in the same streak stay at warn, and a recovered flush " + + "resets the streak so the next persistent outage escalates again", + ); + }); }); From 3a22b254c3533a056b8752cb8bb3e318aa7c43c8 Mon Sep 17 00:00:00 2001 From: Kentaro Wakayama Date: Mon, 31 Aug 2026 00:43:17 +0200 Subject: [PATCH 2/2] fix(agent): escalate mirror flush retry log to error once per failure streak The retry-scheduled log compared consecutiveFailures against the threshold with >=, so every attempt at or past the fifth logged at error level. With retry backoff capped at ~5s, one persistent append outage emitted a Sentry error every few seconds per active run (VERYFRONT-AGENT-3). Compare with strict equality instead: the streak counter increments by one per scheduled retry and resets to zero on a successful flush, so the escalation fires exactly once per failure streak and re-arms after recovery. Attempts 1-4 and 6+ of a streak stay at warn; terminal stop/disable paths still report at error level. --- src/agent/conversation/run-chunk-mirror.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/agent/conversation/run-chunk-mirror.ts b/src/agent/conversation/run-chunk-mirror.ts index b507469fbb..29a40ad68f 100644 --- a/src/agent/conversation/run-chunk-mirror.ts +++ b/src/agent/conversation/run-chunk-mirror.ts @@ -270,9 +270,12 @@ async function runHostedChunkMirrorTrace( // Retries are self-healing and back off to only a few seconds, so logging // every attempt at error level turns one degraded append window into a Sentry -// error every few seconds per active run (VERYFRONT-AGENT-3). Escalate only -// once the failure streak suggests the outage is persistent; terminal -// stop/disable paths report at error level separately. +// error every few seconds per active run (VERYFRONT-AGENT-3). Escalate +// exactly once per failure streak, at the attempt where the streak first +// reaches this threshold: the counter increments by one per scheduled retry +// and resets to zero on a successful flush, so strict equality fires once +// and re-arms after recovery. Terminal stop/disable paths report at error +// level separately. const HOSTED_CHUNK_MIRROR_RETRY_ERROR_THRESHOLD = 5; function recordHostedChunkMirrorRetryScheduled(input: { @@ -290,7 +293,7 @@ function recordHostedChunkMirrorRetryScheduled(input: { consecutiveFailures: input.flushAttempt.consecutiveFailures, }); const message = "Durable run mirror flush failed; queued for retry"; - if (input.flushAttempt.consecutiveFailures >= HOSTED_CHUNK_MIRROR_RETRY_ERROR_THRESHOLD) { + if (input.flushAttempt.consecutiveFailures === HOSTED_CHUNK_MIRROR_RETRY_ERROR_THRESHOLD) { input.instrumentation?.error?.(message, metadata); return; }