From f7b036362cab0aceb6cc545e324d2928c611950d Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 14 Aug 2026 18:34:29 +0200 Subject: [PATCH 1/2] fix(agent): downgrade per-attempt run mirror retry logs to warn Every retry_scheduled flush outcome of the hosted run chunk mirror logged at error level, and node-sentry converts error-level framework logs into Sentry captures. With retry backoff capped at a few seconds, one degraded window of the run-event append endpoint emitted a Sentry error every ~5s per active run. The condition is self-healing, so log each retry attempt at warn and escalate to error only once consecutiveFailures reaches 5. Terminal stop/disable paths keep their existing error-level reporting. Fixes VERYFRONT-AGENT-3 --- .../conversation/run-chunk-mirror.test.ts | 48 +++++++++++++++++++ src/agent/conversation/run-chunk-mirror.ts | 33 ++++++++----- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/src/agent/conversation/run-chunk-mirror.test.ts b/src/agent/conversation/run-chunk-mirror.test.ts index 211fc39a9b..76338088d3 100644 --- a/src/agent/conversation/run-chunk-mirror.test.ts +++ b/src/agent/conversation/run-chunk-mirror.test.ts @@ -51,6 +51,8 @@ function createQueueController(): ConversationRunEventQueueController & { }; } +const RETRY_LOG_MESSAGE = "Durable run mirror flush failed; queued for retry"; + describe("agent/conversation-run-chunk-mirror", () => { it("prepares UI chunks into durable events and enqueues them", async () => { const queueController = createQueueController(); @@ -300,6 +302,52 @@ describe("agent/conversation-run-chunk-mirror", () => { } }); + // VERYFRONT-AGENT-3: every retry_scheduled flush logged at error level, so a + // degraded append endpoint emitted a Sentry error per ~5s retry per run. The + // per-attempt log must stay at warn and escalate to error only once the + // failure streak signals the condition is not self-healing. + it("warns on early retry attempts and escalates to error at the failure threshold", async () => { + const logs: Array<{ level: "warn" | "error"; message: string; consecutiveFailures: unknown }> = + []; + const failingFetch = (() => + Promise.resolve( + new Response( + JSON.stringify({ detail: "upstream unavailable" }), + { status: 503, headers: { "Content-Type": "application/json" } }, + ), + )) as typeof fetch; + const mirror = createHostedConversationRunChunkMirror({ + authToken: "token", + apiUrl: "https://api.example.test", + conversationId: "11111111-1111-4111-8111-111111111111", + runId: "run-1", + latestEventId: 0, + fetch: failingFetch, + instrumentation: { + warn: (message, metadata) => { + logs.push({ level: "warn", message, consecutiveFailures: metadata.consecutiveFailures }); + }, + error: (message, metadata) => { + logs.push({ level: "error", message, consecutiveFailures: metadata.consecutiveFailures }); + }, + }, + }); + + await mirror.appendEvents([{ type: "TEXT_MESSAGE_CONTENT", delta: "persisted" }]); + for (let attempt = 0; attempt < 5; attempt += 1) { + await mirror.flush(); + } + mirror.dispose(); + + assertEquals(logs, [ + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 1 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 2 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 3 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 4 }, + { level: "error", message: RETRY_LOG_MESSAGE, consecutiveFailures: 5 }, + ]); + }); + it("records a terminal auth rejection instead of retrying forever", async () => { const originalFetch = globalThis.fetch; const errors: Array<{ message: string; metadata: Record }> = []; diff --git a/src/agent/conversation/run-chunk-mirror.ts b/src/agent/conversation/run-chunk-mirror.ts index 7ac2361af6..7cac549b0d 100644 --- a/src/agent/conversation/run-chunk-mirror.ts +++ b/src/agent/conversation/run-chunk-mirror.ts @@ -257,23 +257,34 @@ async function runHostedChunkMirrorTrace( return await operation(); } +// Retries are self-healing and back off to only a few seconds, so logging +// every attempt at error level turns one degraded append window into a Sentry +// error every few seconds per active run (VERYFRONT-AGENT-3). Escalate only +// once the failure streak suggests the outage is persistent; terminal +// stop/disable paths report at error level separately. +const HOSTED_CHUNK_MIRROR_RETRY_ERROR_THRESHOLD = 5; + function recordHostedChunkMirrorRetryScheduled(input: { instrumentation: HostedConversationRunChunkMirrorInstrumentation | undefined; conversationId: string; runId: string; flushAttempt: ConversationRunMirrorRetryScheduledState; }): void { - input.instrumentation?.error?.( - "Durable run mirror flush failed; queued for retry", - createHostedChunkMirrorRetryMetadata({ - conversationId: input.conversationId, - runId: input.runId, - errorMessage: input.flushAttempt.errorMessage ?? "Conversation run append failed", - retryDelayMs: input.flushAttempt.retryDelayMs, - pendingEventCount: input.flushAttempt.pendingEventCount, - consecutiveFailures: input.flushAttempt.consecutiveFailures, - }), - ); + const metadata = createHostedChunkMirrorRetryMetadata({ + conversationId: input.conversationId, + runId: input.runId, + errorMessage: input.flushAttempt.errorMessage ?? "Conversation run append failed", + retryDelayMs: input.flushAttempt.retryDelayMs, + pendingEventCount: input.flushAttempt.pendingEventCount, + consecutiveFailures: input.flushAttempt.consecutiveFailures, + }); + const message = "Durable run mirror flush failed; queued for retry"; + if (input.flushAttempt.consecutiveFailures >= HOSTED_CHUNK_MIRROR_RETRY_ERROR_THRESHOLD) { + input.instrumentation?.error?.(message, metadata); + return; + } + + input.instrumentation?.warn?.(message, metadata); } function recordHostedChunkMirrorHighBacklog(input: { From b3472e166374e14b40b42cb4bc251e2de9785c69 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Fri, 14 Aug 2026 20:27:34 +0200 Subject: [PATCH 2/2] Stabilize PR 3722 against CI shard and docs gates The retry-log change moved the generated API reference line number and added a test file that reshuffled coverage shard 2. That exposed repository-hardening tests that read repository files through process cwd while sibling parallel tests can temporarily chdir. Anchor those reads to the test module URL and refresh the pinned API reference line. Constraint: CI runs docs generation on Deno 2.7.7/Linux; local newer Deno reported broad line-number drift.\nRejected: Regenerate all API docs locally | produced toolchain-only churn unrelated to the PR.\nConfidence: high\nScope-risk: narrow\nTested: deno test --preload=src/testing/preload.ts --no-check --parallel --allow-all --unstable-worker-options --unstable-net src/security/repository-hardening.test.ts src/testing/cwd.test.ts\nTested: deno test --preload=src/testing/preload.ts --no-check --allow-all --unstable-worker-options --unstable-net src/agent/conversation/run-chunk-mirror.test.ts\nTested: deno task lint:cwd-relative-test-reads\nTested: deno fmt --check src/security/repository-hardening.test.ts src/agent/conversation/run-chunk-mirror.ts src/agent/conversation/run-chunk-mirror.test.ts docs/api-reference/veryfront/agent.md\nTested: docker run --rm -u "501:20" -e DENO_DIR=/tmp/deno-cache -v "/private/tmp/veryfront-open-pr-review.CGPobM/pr-3722":/work -w /work denoland/deno:2.7.7 deno task docs:api-reference:check\nTested: deno task coverage:ci:shard -- --shard=2/8 --coverage-dir=/tmp/pr3722-coverage-shard-2.k9XkY0\nTested: deno check src/security/repository-hardening.test.ts src/agent/conversation/run-chunk-mirror.ts\nTested: git diff --check\nNot-tested: Full CI locally. --- docs/api-reference/veryfront/agent.md | 2 +- src/security/repository-hardening.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api-reference/veryfront/agent.md b/docs/api-reference/veryfront/agent.md index 8e099361db..41c21df255 100644 --- a/docs/api-reference/veryfront/agent.md +++ b/docs/api-reference/veryfront/agent.md @@ -699,7 +699,7 @@ Input delivered to a hosted agent-service detached execution callback. | `createHostedChildMirrorContext` | Context for create hosted child mirror. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-mirror.ts#L233) | | `createHostedChildPendingToolLifecycle` | Create hosted child pending tool lifecycle. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-pending-tool-lifecycle.ts#L98) | | `createHostedChildPendingToolLifecycleLogger` | Create hosted child pending tool lifecycle logger. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-pending-tool-lifecycle.ts#L55) | -| `createHostedConversationRunChunkMirror` | Create hosted conversation run chunk mirror. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/conversation/run-chunk-mirror.ts#L363) | +| `createHostedConversationRunChunkMirror` | Create hosted conversation run chunk mirror. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/conversation/run-chunk-mirror.ts#L374) | | `createHostedDurableChildForkRunContext` | Context for create hosted durable child fork run. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-fork-run-context.ts#L179) | | `createHostedDurableChildInvokeTraceRecorder` | Create hosted durable child invoke trace recorder. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/durable-child-fork-execution.ts#L312) | | `createHostedFormInputTool` | Create hosted form input tool. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/form-input-tool.ts#L34) | diff --git a/src/security/repository-hardening.test.ts b/src/security/repository-hardening.test.ts index b11a60614b..aa2b0dcffa 100644 --- a/src/security/repository-hardening.test.ts +++ b/src/security/repository-hardening.test.ts @@ -7,7 +7,7 @@ const CREATE_APP_TOKEN_ACTION = "actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1"; async function readText(path: string): Promise { - return await Deno.readTextFile(path); + return await Deno.readTextFile(new URL(`../../${path}`, import.meta.url)); } function stripComments(text: string): string {