diff --git a/docs/api-reference/veryfront/agent.md b/docs/api-reference/veryfront/agent.md index 8e099361db..41c21df255 100644 --- a/docs/api-reference/veryfront/agent.md +++ b/docs/api-reference/veryfront/agent.md @@ -699,7 +699,7 @@ Input delivered to a hosted agent-service detached execution callback. | `createHostedChildMirrorContext` | Context for create hosted child mirror. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-mirror.ts#L233) | | `createHostedChildPendingToolLifecycle` | Create hosted child pending tool lifecycle. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-pending-tool-lifecycle.ts#L98) | | `createHostedChildPendingToolLifecycleLogger` | Create hosted child pending tool lifecycle logger. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-pending-tool-lifecycle.ts#L55) | -| `createHostedConversationRunChunkMirror` | Create hosted conversation run chunk mirror. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/conversation/run-chunk-mirror.ts#L363) | +| `createHostedConversationRunChunkMirror` | Create hosted conversation run chunk mirror. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/conversation/run-chunk-mirror.ts#L374) | | `createHostedDurableChildForkRunContext` | Context for create hosted durable child fork run. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-fork-run-context.ts#L179) | | `createHostedDurableChildInvokeTraceRecorder` | Create hosted durable child invoke trace recorder. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/durable-child-fork-execution.ts#L312) | | `createHostedFormInputTool` | Create hosted form input tool. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/form-input-tool.ts#L34) | diff --git a/src/agent/conversation/run-chunk-mirror.test.ts b/src/agent/conversation/run-chunk-mirror.test.ts index 211fc39a9b..76338088d3 100644 --- a/src/agent/conversation/run-chunk-mirror.test.ts +++ b/src/agent/conversation/run-chunk-mirror.test.ts @@ -51,6 +51,8 @@ function createQueueController(): ConversationRunEventQueueController & { }; } +const RETRY_LOG_MESSAGE = "Durable run mirror flush failed; queued for retry"; + describe("agent/conversation-run-chunk-mirror", () => { it("prepares UI chunks into durable events and enqueues them", async () => { const queueController = createQueueController(); @@ -300,6 +302,52 @@ describe("agent/conversation-run-chunk-mirror", () => { } }); + // VERYFRONT-AGENT-3: every retry_scheduled flush logged at error level, so a + // degraded append endpoint emitted a Sentry error per ~5s retry per run. The + // per-attempt log must stay at warn and escalate to error only once the + // failure streak signals the condition is not self-healing. + it("warns on early retry attempts and escalates to error at the failure threshold", async () => { + const logs: Array<{ level: "warn" | "error"; message: string; consecutiveFailures: unknown }> = + []; + const failingFetch = (() => + Promise.resolve( + new Response( + JSON.stringify({ detail: "upstream unavailable" }), + { status: 503, headers: { "Content-Type": "application/json" } }, + ), + )) as typeof fetch; + const mirror = createHostedConversationRunChunkMirror({ + authToken: "token", + apiUrl: "https://api.example.test", + conversationId: "11111111-1111-4111-8111-111111111111", + runId: "run-1", + latestEventId: 0, + fetch: failingFetch, + instrumentation: { + warn: (message, metadata) => { + logs.push({ level: "warn", message, consecutiveFailures: metadata.consecutiveFailures }); + }, + error: (message, metadata) => { + logs.push({ level: "error", message, consecutiveFailures: metadata.consecutiveFailures }); + }, + }, + }); + + await mirror.appendEvents([{ type: "TEXT_MESSAGE_CONTENT", delta: "persisted" }]); + for (let attempt = 0; attempt < 5; attempt += 1) { + await mirror.flush(); + } + mirror.dispose(); + + assertEquals(logs, [ + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 1 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 2 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 3 }, + { level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 4 }, + { level: "error", message: RETRY_LOG_MESSAGE, consecutiveFailures: 5 }, + ]); + }); + it("records a terminal auth rejection instead of retrying forever", async () => { const originalFetch = globalThis.fetch; const errors: Array<{ message: string; metadata: Record }> = []; diff --git a/src/agent/conversation/run-chunk-mirror.ts b/src/agent/conversation/run-chunk-mirror.ts index 7ac2361af6..7cac549b0d 100644 --- a/src/agent/conversation/run-chunk-mirror.ts +++ b/src/agent/conversation/run-chunk-mirror.ts @@ -257,23 +257,34 @@ async function runHostedChunkMirrorTrace( return await operation(); } +// Retries are self-healing and back off to only a few seconds, so logging +// every attempt at error level turns one degraded append window into a Sentry +// error every few seconds per active run (VERYFRONT-AGENT-3). Escalate only +// once the failure streak suggests the outage is persistent; terminal +// stop/disable paths report at error level separately. +const HOSTED_CHUNK_MIRROR_RETRY_ERROR_THRESHOLD = 5; + function recordHostedChunkMirrorRetryScheduled(input: { instrumentation: HostedConversationRunChunkMirrorInstrumentation | undefined; conversationId: string; runId: string; flushAttempt: ConversationRunMirrorRetryScheduledState; }): void { - input.instrumentation?.error?.( - "Durable run mirror flush failed; queued for retry", - createHostedChunkMirrorRetryMetadata({ - conversationId: input.conversationId, - runId: input.runId, - errorMessage: input.flushAttempt.errorMessage ?? "Conversation run append failed", - retryDelayMs: input.flushAttempt.retryDelayMs, - pendingEventCount: input.flushAttempt.pendingEventCount, - consecutiveFailures: input.flushAttempt.consecutiveFailures, - }), - ); + const metadata = createHostedChunkMirrorRetryMetadata({ + conversationId: input.conversationId, + runId: input.runId, + errorMessage: input.flushAttempt.errorMessage ?? "Conversation run append failed", + retryDelayMs: input.flushAttempt.retryDelayMs, + pendingEventCount: input.flushAttempt.pendingEventCount, + consecutiveFailures: input.flushAttempt.consecutiveFailures, + }); + const message = "Durable run mirror flush failed; queued for retry"; + if (input.flushAttempt.consecutiveFailures >= HOSTED_CHUNK_MIRROR_RETRY_ERROR_THRESHOLD) { + input.instrumentation?.error?.(message, metadata); + return; + } + + input.instrumentation?.warn?.(message, metadata); } function recordHostedChunkMirrorHighBacklog(input: { diff --git a/src/security/repository-hardening.test.ts b/src/security/repository-hardening.test.ts index b11a60614b..aa2b0dcffa 100644 --- a/src/security/repository-hardening.test.ts +++ b/src/security/repository-hardening.test.ts @@ -7,7 +7,7 @@ const CREATE_APP_TOKEN_ACTION = "actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1"; async function readText(path: string): Promise { - return await Deno.readTextFile(path); + return await Deno.readTextFile(new URL(`../../${path}`, import.meta.url)); } function stripComments(text: string): string {