Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/api-reference/veryfront/agent.md
Original file line number Diff line number Diff line change
Expand Up @@ -699,7 +699,7 @@ Input delivered to a hosted agent-service detached execution callback.
| `createHostedChildMirrorContext` | Context for create hosted child mirror. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-mirror.ts#L233) |
| `createHostedChildPendingToolLifecycle` | Create hosted child pending tool lifecycle. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-pending-tool-lifecycle.ts#L98) |
| `createHostedChildPendingToolLifecycleLogger` | Create hosted child pending tool lifecycle logger. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-pending-tool-lifecycle.ts#L55) |
| `createHostedConversationRunChunkMirror` | Create hosted conversation run chunk mirror. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/conversation/run-chunk-mirror.ts#L363) |
| `createHostedConversationRunChunkMirror` | Create hosted conversation run chunk mirror. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/conversation/run-chunk-mirror.ts#L374) |
| `createHostedDurableChildForkRunContext` | Context for create hosted durable child fork run. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-fork-run-context.ts#L179) |
| `createHostedDurableChildInvokeTraceRecorder` | Create hosted durable child invoke trace recorder. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/durable-child-fork-execution.ts#L312) |
| `createHostedFormInputTool` | Create hosted form input tool. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/form-input-tool.ts#L34) |
Expand Down
48 changes: 48 additions & 0 deletions src/agent/conversation/run-chunk-mirror.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ function createQueueController(): ConversationRunEventQueueController & {
};
}

const RETRY_LOG_MESSAGE = "Durable run mirror flush failed; queued for retry";

describe("agent/conversation-run-chunk-mirror", () => {
it("prepares UI chunks into durable events and enqueues them", async () => {
const queueController = createQueueController();
Expand Down Expand Up @@ -300,6 +302,52 @@ describe("agent/conversation-run-chunk-mirror", () => {
}
});

// VERYFRONT-AGENT-3: every retry_scheduled flush logged at error level, so a
// degraded append endpoint emitted a Sentry error per ~5s retry per run. The
// per-attempt log must stay at warn and escalate to error only once the
// failure streak signals the condition is not self-healing.
it("warns on early retry attempts and escalates to error at the failure threshold", async () => {
const logs: Array<{ level: "warn" | "error"; message: string; consecutiveFailures: unknown }> =
[];
const failingFetch = (() =>
Promise.resolve(
new Response(
JSON.stringify({ detail: "upstream unavailable" }),
{ status: 503, headers: { "Content-Type": "application/json" } },
),
)) as typeof fetch;
const mirror = createHostedConversationRunChunkMirror({
authToken: "token",
apiUrl: "https://api.example.test",
conversationId: "11111111-1111-4111-8111-111111111111",
runId: "run-1",
latestEventId: 0,
fetch: failingFetch,
instrumentation: {
warn: (message, metadata) => {
logs.push({ level: "warn", message, consecutiveFailures: metadata.consecutiveFailures });
},
error: (message, metadata) => {
logs.push({ level: "error", message, consecutiveFailures: metadata.consecutiveFailures });
},
},
});

await mirror.appendEvents([{ type: "TEXT_MESSAGE_CONTENT", delta: "persisted" }]);
for (let attempt = 0; attempt < 5; attempt += 1) {
await mirror.flush();
}
mirror.dispose();

assertEquals(logs, [
{ level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 1 },
{ level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 2 },
{ level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 3 },
{ level: "warn", message: RETRY_LOG_MESSAGE, consecutiveFailures: 4 },
{ level: "error", message: RETRY_LOG_MESSAGE, consecutiveFailures: 5 },
]);
});

it("records a terminal auth rejection instead of retrying forever", async () => {
const originalFetch = globalThis.fetch;
const errors: Array<{ message: string; metadata: Record<string, unknown> }> = [];
Expand Down
33 changes: 22 additions & 11 deletions src/agent/conversation/run-chunk-mirror.ts
Original file line number Diff line number Diff line change
Expand Up @@ -257,23 +257,34 @@ async function runHostedChunkMirrorTrace<T>(
return await operation();
}

// Retries are self-healing and back off to only a few seconds, so logging
// every attempt at error level turns one degraded append window into a Sentry
// error every few seconds per active run (VERYFRONT-AGENT-3). Escalate only
// once the failure streak suggests the outage is persistent; terminal
// stop/disable paths report at error level separately.
const HOSTED_CHUNK_MIRROR_RETRY_ERROR_THRESHOLD = 5;

function recordHostedChunkMirrorRetryScheduled(input: {
instrumentation: HostedConversationRunChunkMirrorInstrumentation | undefined;
conversationId: string;
runId: string;
flushAttempt: ConversationRunMirrorRetryScheduledState;
}): void {
input.instrumentation?.error?.(
"Durable run mirror flush failed; queued for retry",
createHostedChunkMirrorRetryMetadata({
conversationId: input.conversationId,
runId: input.runId,
errorMessage: input.flushAttempt.errorMessage ?? "Conversation run append failed",
retryDelayMs: input.flushAttempt.retryDelayMs,
pendingEventCount: input.flushAttempt.pendingEventCount,
consecutiveFailures: input.flushAttempt.consecutiveFailures,
}),
);
const metadata = createHostedChunkMirrorRetryMetadata({
conversationId: input.conversationId,
runId: input.runId,
errorMessage: input.flushAttempt.errorMessage ?? "Conversation run append failed",
retryDelayMs: input.flushAttempt.retryDelayMs,
pendingEventCount: input.flushAttempt.pendingEventCount,
consecutiveFailures: input.flushAttempt.consecutiveFailures,
});
const message = "Durable run mirror flush failed; queued for retry";
if (input.flushAttempt.consecutiveFailures >= HOSTED_CHUNK_MIRROR_RETRY_ERROR_THRESHOLD) {
input.instrumentation?.error?.(message, metadata);
return;
}

input.instrumentation?.warn?.(message, metadata);
}

function recordHostedChunkMirrorHighBacklog(input: {
Expand Down
2 changes: 1 addition & 1 deletion src/security/repository-hardening.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ const CREATE_APP_TOKEN_ACTION =
"actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1";

async function readText(path: string): Promise<string> {
return await Deno.readTextFile(path);
return await Deno.readTextFile(new URL(`../../${path}`, import.meta.url));
}

function stripComments(text: string): string {
Expand Down