diff --git a/packages/cli/src/utils/publishProject.test.ts b/packages/cli/src/utils/publishProject.test.ts index 65255cc57a..8cff808289 100644 --- a/packages/cli/src/utils/publishProject.test.ts +++ b/packages/cli/src/utils/publishProject.test.ts @@ -146,6 +146,16 @@ function directFetch(completeData?: Record) { .mockResolvedValueOnce(publishedResponse(completeData)); } +function networkFailure( + code: string, + message: string, + metadata: Record = {}, +): TypeError { + return new TypeError("fetch failed", { + cause: Object.assign(new Error(message), { code, ...metadata }), + }); +} + /** Asserts the Nth fetch call, always requiring an AbortSignal alongside the given init. */ function expectFetchCall( fetchMock: ReturnType, @@ -729,6 +739,190 @@ describe("publishProjectArchive", () => { rmSync(dir, { recursive: true, force: true }); } }); + + it("retries a transient presigned upload network failure once", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse()) + .mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected")) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockResolvedValueOnce(publishedResponse()); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const result = await publishProjectArchive(dir); + + expect(result.projectId).toBe("hfp_123"); + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(fetchMock.mock.calls[1]![0]).toBe("https://s3.example.com/upload"); + expect(fetchMock.mock.calls[2]![0]).toBe("https://s3.example.com/upload"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("waits briefly before retrying a transport failure", async () => { + vi.useFakeTimers(); + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse()) + .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "temporary DNS failure")) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockResolvedValueOnce(publishedResponse()); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const publish = publishProjectArchive(dir); + await vi.advanceTimersByTimeAsync(0); + expect(fetchMock).toHaveBeenCalledTimes(2); + + await vi.advanceTimersByTimeAsync(199); + expect(fetchMock).toHaveBeenCalledTimes(2); + + await vi.advanceTimersByTimeAsync(1); + await expect(publish).resolves.toMatchObject({ projectId: "hfp_123" }); + expect(fetchMock).toHaveBeenCalledTimes(4); + } finally { + await vi.runAllTimersAsync(); + vi.useRealTimers(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the upload stage and transport cause after the retry is exhausted", async () => { + const dir = makeProjectDir(); + const signedUrl = + "https://s3.example.com/upload?X-Amz-Credential=secret&X-Amz-Signature=do-not-print"; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse({ upload_url: signedUrl })) + .mockRejectedValueOnce( + networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com", { + errno: -3001, + syscall: "getaddrinfo", + }), + ) + .mockRejectedValueOnce( + networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com", { + errno: -3001, + syscall: "getaddrinfo", + }), + ); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const promise = publishProjectArchive(dir); + await expect(promise).rejects.toThrow( + "Failed to upload project archive after 2 attempts: fetch failed (EAI_AGAIN, syscall=getaddrinfo, errno=-3001: getaddrinfo EAI_AGAIN s3.example.com)", + ); + await expect(promise).rejects.not.toThrow("do-not-print"); + expect(fetchMock).toHaveBeenCalledTimes(3); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the prepare stage and explains disabled Node proxy support", async () => { + const dir = makeProjectDir(); + vi.stubEnv("HTTPS_PROXY", "http://proxy.example.com:8080"); + vi.stubEnv("NODE_USE_ENV_PROXY", ""); + const fetchMock = vi + .fn() + .mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable")) + .mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable")); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + await expect(publishProjectArchive(dir)).rejects.toThrow( + "Failed to prepare project upload after 2 attempts: fetch failed (ENETUNREACH: network is unreachable). Proxy variables are set but ignored by Node fetch; if this network requires them, retry with NODE_USE_ENV_PROXY=1", + ); + expect(fetchMock).toHaveBeenCalledTimes(2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("retries a transient prepare-upload network failure once", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockRejectedValueOnce(networkFailure("EAI_AGAIN", "temporary DNS failure")) + .mockResolvedValueOnce(uploadResponse()) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockResolvedValueOnce(publishedResponse()); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + const result = await publishProjectArchive(dir); + + expect(result.projectId).toBe("hfp_123"); + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(fetchMock.mock.calls[0]![0]).toBe( + "https://api2.heygen.com/v1/hyperframes/projects/publish/upload", + ); + expect(fetchMock.mock.calls[1]![0]).toBe( + "https://api2.heygen.com/v1/hyperframes/projects/publish/upload", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("does not retry a request that reached its timeout", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockRejectedValueOnce( + new DOMException("The operation was aborted due to timeout", "TimeoutError"), + ) + .mockResolvedValueOnce(uploadResponse()) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockResolvedValueOnce(publishedResponse()); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + await expect(publishProjectArchive(dir)).rejects.toThrow( + "Failed to prepare project upload: The operation was aborted due to timeout", + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports the finalize stage and transport cause", async () => { + const dir = makeProjectDir(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(uploadResponse()) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected")); + vi.stubGlobal("fetch", fetchMock); + + try { + writeFileSync(join(dir, "index.html"), "", "utf-8"); + + await expect(publishProjectArchive(dir)).rejects.toThrow( + "Failed to finalize project publish: fetch failed (ECONNRESET: socket disconnected)", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); /** Staged flow returning an already-owned, already-claimed stable project. */ diff --git a/packages/cli/src/utils/publishProject.ts b/packages/cli/src/utils/publishProject.ts index f9bf5f4851..89c2585c80 100644 --- a/packages/cli/src/utils/publishProject.ts +++ b/packages/cli/src/utils/publishProject.ts @@ -15,6 +15,8 @@ const DEFAULT_PROJECT_IGNORE = ["/renders/", "/snapshots/"]; const PUBLISH_CONTENT_TYPE = "application/zip"; const PUBLISH_METADATA_TIMEOUT_MS = 30_000; const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000; +const PUBLISH_TRANSPORT_ATTEMPTS = 2; +const PUBLISH_RETRY_DELAY_MS = 200; // Conservative floor — most connections are faster, but this prevents // premature aborts on slow/unstable networks (hotel wifi, tethering). const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000; @@ -166,6 +168,83 @@ async function readErrorMessage(response: Response, fallback: string): Promise + Boolean(process.env[key]?.trim()), + ); + const proxyEnabled = + process.env["NODE_USE_ENV_PROXY"] === "1" || + process.execArgv.includes("--use-env-proxy") || + process.env["NODE_OPTIONS"]?.split(/\s+/u).includes("--use-env-proxy") === true; + if (!proxyConfigured || proxyEnabled) return ""; + return ( + ". Proxy variables are set but ignored by Node fetch; if this network requires them, retry with " + + "NODE_USE_ENV_PROXY=1 (Node 22.21+)" + ); +} + +function describeFetchFailure(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + const cause = error instanceof Error ? error.cause : undefined; + const causeMessage = cause instanceof Error ? cause.message : ""; + const metadata = [...systemErrorMetadata(cause), ...systemErrorMetadata(error)].filter( + (value, index, all) => all.indexOf(value) === index, + ); + const distinctCauseMessage = causeMessage && causeMessage !== message ? causeMessage : ""; + const detail = [metadata.join(", "), distinctCauseMessage].filter(Boolean).join(": "); + return `${redactUrlQuery(message)}${detail ? ` (${redactUrlQuery(detail)})` : ""}${proxySupportHint()}`; +} + +function isRequestTimeout(error: unknown): boolean { + return ( + error instanceof DOMException && (error.name === "TimeoutError" || error.name === "AbortError") + ); +} + +function waitBeforePublishRetry(): Promise { + return new Promise((resolve) => setTimeout(resolve, PUBLISH_RETRY_DELAY_MS)); +} + +async function fetchForPublish( + input: string, + createInit: () => RequestInit, + failureStage: string, + attempts = 1, +): Promise { + if (attempts < 1) throw new RangeError("Publish fetch attempts must be at least 1"); + let lastError: unknown; + let attemptsMade = 0; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + attemptsMade = attempt; + try { + return await fetch(input, createInit()); + } catch (error) { + lastError = error; + if (isRequestTimeout(error) || attempt === attempts) break; + await waitBeforePublishRetry(); + } + } + const attemptDetail = attemptsMade > 1 ? ` after ${attemptsMade} attempts` : ""; + throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(lastError)}`, { + cause: lastError instanceof Error ? lastError : undefined, + }); +} + export function uploadTimeoutMs(byteLength: number): number { return Math.max( PUBLISH_UPLOAD_MIN_TIMEOUT_MS, @@ -483,12 +562,16 @@ async function publishProjectArchiveDirect( heygen_route: "canary", }; - const response = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish`, { - method: "POST", - body, - headers, - signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), - }); + const response = await fetchForPublish( + `${apiBaseUrl}/v1/hyperframes/projects/publish`, + () => ({ + method: "POST", + body, + headers, + signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), + }), + "Failed to publish project", + ); const payload = await readJson(response); const publishedProject = parsePublishedProjectResponse(payload); @@ -504,14 +587,19 @@ async function uploadArchiveToPresignedUrl( archive: PublishArchiveResult, ): Promise { const presignedUrlTtlMs = stagedUpload.expiresInSeconds * 1000 - PUBLISH_METADATA_TIMEOUT_MS; - const s3Response = await fetch(stagedUpload.uploadUrl, { - method: "PUT", - body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }), - headers: stagedUpload.uploadHeaders, - signal: AbortSignal.timeout( - Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs), - ), - }); + const s3Response = await fetchForPublish( + stagedUpload.uploadUrl, + () => ({ + method: "PUT", + body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }), + headers: stagedUpload.uploadHeaders, + signal: AbortSignal.timeout( + Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs), + ), + }), + "Failed to upload project archive", + PUBLISH_TRANSPORT_ATTEMPTS, + ); if (!s3Response.ok) { throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive")); } @@ -526,20 +614,25 @@ async function publishProjectArchiveStaged( projectId: string | undefined, ): Promise { const fileName = `${title}.zip`; - const uploadResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/upload`, { - method: "POST", - body: JSON.stringify({ - file_name: fileName, - content_type: PUBLISH_CONTENT_TYPE, - content_length: archive.buffer.byteLength, + const uploadResponse = await fetchForPublish( + `${apiBaseUrl}/v1/hyperframes/projects/publish/upload`, + () => ({ + method: "POST", + body: JSON.stringify({ + file_name: fileName, + content_type: PUBLISH_CONTENT_TYPE, + content_length: archive.buffer.byteLength, + }), + headers: { + ...authHeaders, + "content-type": "application/json", + heygen_route: "canary", + }, + signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS), }), - headers: { - ...authHeaders, - "content-type": "application/json", - heygen_route: "canary", - }, - signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS), - }); + "Failed to prepare project upload", + PUBLISH_TRANSPORT_ATTEMPTS, + ); if (uploadResponse.status === 404 || uploadResponse.status === 405) { return null; @@ -553,22 +646,26 @@ async function publishProjectArchiveStaged( await uploadArchiveToPresignedUrl(stagedUpload, archive); - const completeResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/complete`, { - method: "POST", - body: JSON.stringify({ - upload_key: stagedUpload.uploadKey, - file_name: fileName, - title, - ...(isPublic ? { is_public: true } : {}), - ...(projectId ? { project_id: projectId } : {}), + const completeResponse = await fetchForPublish( + `${apiBaseUrl}/v1/hyperframes/projects/publish/complete`, + () => ({ + method: "POST", + body: JSON.stringify({ + upload_key: stagedUpload.uploadKey, + file_name: fileName, + title, + ...(isPublic ? { is_public: true } : {}), + ...(projectId ? { project_id: projectId } : {}), + }), + headers: { + ...authHeaders, + "content-type": "application/json", + heygen_route: "canary", + }, + signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), }), - headers: { - ...authHeaders, - "content-type": "application/json", - heygen_route: "canary", - }, - signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)), - }); + "Failed to finalize project publish", + ); const completePayload = await readJson(completeResponse); const publishedProject = parsePublishedProjectResponse(completePayload);