Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/errors/error-handlers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ describe("error-handlers", () => {
if (attempts < 2) throw new Error("fail");
return "success";
},
{ maxRetries: 3, initialDelay: 1 },
{ maxAttempts: 3, initialDelay: 1 },
);

assertEquals(result, "success");
Expand All @@ -94,7 +94,7 @@ describe("error-handlers", () => {
attempts++;
throw new Error("always fails");
},
{ maxRetries: 2, initialDelay: 1 },
{ maxAttempts: 2, initialDelay: 1 },
),
Error,
"always fails",
Expand Down
78 changes: 58 additions & 20 deletions src/errors/error-handlers.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { serverLogger } from "#veryfront/utils/logger/logger.ts";
import { sleep } from "#veryfront/utils/sleep.ts";
import {
DEFAULT_RETRY_INITIAL_DELAY_MS,
DEFAULT_RETRY_MAX_ATTEMPTS,
Expand Down Expand Up @@ -44,41 +45,78 @@ export function handleErrorWithFallbackSync<T>(
}
}

export interface RetryWithBackoffOptions {
/** Total number of attempts (first try included). */
maxAttempts?: number;
initialDelay?: number;
maxDelay?: number;
logger?: typeof serverLogger;
/** Per-attempt timeout; aborts the attempt's signal with an AbortError. */
timeoutMs?: number;
Comment thread
kojiwakayama marked this conversation as resolved.
Outdated
/** Return false to rethrow immediately without further attempts (default: always retry). */
shouldRetry?: (error: unknown, attempt: number) => boolean;
/** Override the exponential backoff delay for the wait after `attempt` (0-based). */
computeDelay?: (attempt: number, error: unknown) => number;
/** Called before each backoff wait; replaces the default warn log. */
onRetry?: (info: { error: Error; attempt: number; delay: number; isTimeout: boolean }) => void;
/** Wrap the terminal error once all attempts are exhausted (default: rethrow as-is). */
Comment thread
kojiwakayama marked this conversation as resolved.
Outdated
wrapFinalError?: (lastError: Error, lastAttempt: number) => Error;
Comment thread
kojiwakayama marked this conversation as resolved.
Outdated
}

export async function retryWithBackoff<T>(
fn: () => Promise<T>,
options: {
maxRetries?: number;
initialDelay?: number;
maxDelay?: number;
logger?: typeof serverLogger;
} = {},
fn: (signal: AbortSignal | undefined, attempt: number) => Promise<T>,
options: RetryWithBackoffOptions = {},
): Promise<T> {
const {
maxRetries = DEFAULT_RETRY_MAX_ATTEMPTS,
maxAttempts = DEFAULT_RETRY_MAX_ATTEMPTS,
Comment thread
kojiwakayama marked this conversation as resolved.
initialDelay = DEFAULT_RETRY_INITIAL_DELAY_MS,
maxDelay = DEFAULT_RETRY_MAX_DELAY_MS,
logger: retryLogger = serverLogger,
timeoutMs,
shouldRetry,
computeDelay,
onRetry,
wrapFinalError,
} = options;

let lastError: unknown;
let delay = initialDelay;
let lastError: Error | undefined;

Comment thread
kojiwakayama marked this conversation as resolved.
for (let attempt = 0; attempt < maxAttempts; attempt++) {
const controller = timeoutMs === undefined ? undefined : new AbortController();
const timeoutId = controller === undefined
? undefined
: setTimeout(() => controller.abort(), timeoutMs);

for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
return await fn();
return await fn(controller?.signal, attempt);
} catch (error) {
lastError = error;
safeLog(() => retryLogger.warn(`Attempt ${attempt + 1} failed, retrying...`, error));
lastError = error instanceof Error ? error : new Error(String(error));

if (shouldRetry && !shouldRetry(error, attempt)) {
throw error;
}

if (attempt >= maxAttempts - 1) {
break;
}

const delay = computeDelay
? computeDelay(attempt, error)
: Math.min(initialDelay * 2 ** attempt, maxDelay);
const isTimeout = lastError.name === "AbortError";
Comment thread
kojiwakayama marked this conversation as resolved.

if (attempt >= maxRetries - 1) {
continue;
if (onRetry) {
onRetry({ error: lastError, attempt, delay, isTimeout });
} else {
safeLog(() => retryLogger.warn(`Attempt ${attempt + 1} failed, retrying...`, lastError));
}

// no cleanup needed: one-shot
await new Promise<void>((resolve) => setTimeout(resolve, delay));
delay = Math.min(delay * 2, maxDelay);
await sleep(delay);
} finally {
clearTimeout(timeoutId);
}
}

throw lastError;
const finalError = lastError ?? new Error("Retry failed without capturing an error");
throw wrapFinalError ? wrapFinalError(finalError, Math.max(0, maxAttempts - 1)) : finalError;
}
1 change: 1 addition & 0 deletions src/errors/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ export {
handleErrorWithFallback,
handleErrorWithFallbackSync,
retryWithBackoff,
type RetryWithBackoffOptions,
} from "./error-handlers.ts";

export {
Expand Down
58 changes: 27 additions & 31 deletions src/platform/adapters/fs/github/github-api-client.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { createError, toError } from "#veryfront/errors";
import { createError, retryWithBackoff, toError } from "#veryfront/errors";
import { logger } from "#veryfront/utils";
import type { ResolvedGitHubConfig } from "./types.ts";
import {
Expand Down Expand Up @@ -81,12 +81,11 @@ export class GitHubApiClient {
return this.rateLimitInfo;
}

private async request(endpoint: string): Promise<unknown> {
private request(endpoint: string): Promise<unknown> {
const url = `${this.baseUrl}${endpoint}`;
let lastError: Error | null = null;

for (let attempt = 1; attempt <= this.config.retry.maxRetries; attempt++) {
try {
return retryWithBackoff(
async () => {
const response = await fetch(url, {
headers: {
Authorization: `Bearer ${this.config.token}`,
Expand All @@ -103,27 +102,29 @@ export class GitHubApiClient {
throw this.createAPIError(response.status, errorBody, endpoint);
}

return await response.json();
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));

if (this.isClientError(lastError) && !this.isRateLimitError(lastError)) {
throw lastError;
}

if (attempt >= this.config.retry.maxRetries) break;

const delay = this.calculateRetryDelay(attempt, lastError);
logger.warn(`${LOG_PREFIX} Request failed, retrying`, {
attempt,
delay,
error: lastError.message,
});
await this.sleep(delay);
}
}

throw lastError ?? new Error("Request failed after retries");
return response.json();
},
{
maxAttempts: this.config.retry.maxRetries,
initialDelay: this.config.retry.initialDelay,
maxDelay: this.config.retry.maxDelay,
Comment thread
kojiwakayama marked this conversation as resolved.
shouldRetry: (error) => {
const err = error instanceof Error ? error : new Error(String(error));
return !(this.isClientError(err) && !this.isRateLimitError(err));
},
computeDelay: (attempt, error) => {
const err = error instanceof Error ? error : new Error(String(error));
return this.calculateRetryDelay(attempt + 1, err);
},
onRetry: ({ error, attempt, delay }) => {
logger.warn(`${LOG_PREFIX} Request failed, retrying`, {
attempt: attempt + 1,
delay,
error: error.message,
});
},
},
);
}

private updateRateLimitInfo(response: Response): void {
Expand Down Expand Up @@ -221,9 +222,4 @@ export class GitHubApiClient {

return delay + Math.random() * RETRY_JITTER_MAX_MS;
}

// no cleanup needed: one-shot
private sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
}
27 changes: 12 additions & 15 deletions src/platform/adapters/fs/veryfront/retry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
*/

import { logger as baseLogger } from "#veryfront/utils";
import { retryWithBackoff } from "#veryfront/errors/error-handlers.ts";

const logger = baseLogger.component("fs-retry");

Expand Down Expand Up @@ -57,22 +58,18 @@ function isTransientError(error: unknown): boolean {
* Execute an async function with a single retry for transient errors.
* Non-transient errors (4xx, validation) are thrown immediately.
*/
export async function withRetryOnTransient<T>(
export function withRetryOnTransient<T>(
fn: () => Promise<T>,
context: string,
): Promise<T> {
try {
return await fn();
} catch (error) {
if (!isTransientError(error)) throw error;

logger.warn(`${context} — transient error, retrying once`, {
error: error instanceof Error ? error.message : String(error),
});

// no cleanup needed: one-shot
await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS));

return fn();
}
return retryWithBackoff(() => fn(), {
maxAttempts: 2,
computeDelay: () => RETRY_DELAY_MS,
shouldRetry: (error) => isTransientError(error),
onRetry: ({ error }) => {
logger.warn(`${context} — transient error, retrying once`, {
error: error.message,
});
},
});
}
101 changes: 46 additions & 55 deletions src/platform/adapters/token/veryfront/api-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { injectContext } from "#veryfront/observability/tracing/otlp-setup.ts";
import { type VeryfrontTokenConfig } from "./types.ts";
import { TOKEN_STORAGE_ERROR } from "#veryfront/errors/error-registry.ts";
import { VeryfrontError } from "#veryfront/errors/types.ts";
import { retryWithBackoff } from "#veryfront/errors/error-handlers.ts";

const logger = baseLogger.component("token-storage-api-client");

Expand Down Expand Up @@ -190,72 +191,62 @@ export class TokenStorageApiClient {
return TOKEN_STORAGE_ERROR.create({ detail: `${prefixMessage}: ${message}` });
}

private logTimedOut(url: string, timeoutMs: number, attempt: number): void {
logger.warn("Request timed out", {
url: url.replace(/token=[^&]+/, "token=***"),
timeoutMs,
attempt: attempt + 1,
});
}

private async fetchWithRetry(url: string, init: RequestInit): Promise<Response> {
const { maxRetries, initialDelay, maxDelay } = this.config.retry;
const timeoutMs = this.config.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
let lastError: Error | undefined;

for (let attempt = 0; attempt <= maxRetries; attempt++) {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);

try {
return retryWithBackoff(
(signal) => {
const headers = new Headers(init.headers);
injectContext(headers);

const response = await fetch(url, {
...init,
headers,
signal: controller.signal,
});

if (response.status >= 400 && response.status < 500 && response.status !== 429) {
return response;
}

if (!response.ok && (response.status >= 500 || response.status === 429)) {
throw TOKEN_STORAGE_ERROR.create({
detail: `Server error: ${response.status}`,
status: response.status,
});
}
return fetch(url, { ...init, headers, signal }).then((response) => {
if (response.status >= 400 && response.status < 500 && response.status !== 429) {
return response;
}

return response;
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
if (!response.ok && (response.status >= 500 || response.status === 429)) {
throw TOKEN_STORAGE_ERROR.create({
detail: `Server error: ${response.status}`,
status: response.status,
});
}

const isTimeout = error instanceof Error && error.name === "AbortError";
if (isTimeout) {
logger.warn("Request timed out", {
url: url.replace(/token=[^&]+/, "token=***"),
timeoutMs,
return response;
});
},
{
maxAttempts: maxRetries + 1,
initialDelay,
maxDelay,
timeoutMs,
onRetry: ({ error, attempt, delay, isTimeout }) => {
if (isTimeout) this.logTimedOut(url, timeoutMs, attempt);

logger.warn("Request failed, retrying...", {
attempt: attempt + 1,
maxRetries,
delay,
error: error.message,
timeout: isTimeout,
});
}

if (attempt >= maxRetries) {
break;
}

const delay = Math.min(initialDelay * Math.pow(2, attempt), maxDelay);

logger.warn("Request failed, retrying...", {
attempt: attempt + 1,
maxRetries,
delay,
error: lastError.message,
timeout: isTimeout,
});

// no cleanup needed: one-shot
await new Promise<void>((resolve) => setTimeout(resolve, delay));
} finally {
clearTimeout(timeoutId);
}
}
},
wrapFinalError: (lastError, lastAttempt) => {
if (lastError.name === "AbortError") this.logTimedOut(url, timeoutMs, lastAttempt);

throw TOKEN_STORAGE_ERROR.create({
detail: `Request failed after ${maxRetries} retries: ${lastError?.message}`,
});
return TOKEN_STORAGE_ERROR.create({
detail: `Request failed after ${maxRetries} retries: ${lastError.message}`,
});
},
},
);
}
}
Loading