From b7a9ce60244e18b74533aaeeff6ae282a82892f1 Mon Sep 17 00:00:00 2001 From: Greg Brimble Date: Thu, 26 May 2022 12:09:15 +0100 Subject: [PATCH] Use new bulk upload API for 'pages publish' (#1028) * Use new bulk upload API for 'pages publish' * Only upload missing files * Get PAGES_API_HOST from Env * Using new /pages/assets endpoints * Added tests for pages bulk upload endpoints * fixing imports * removed authOverride by relaxing the constraint about overriding Authorization header * tweaked based on review comments * switched p-queue to be a devDep of Wrangler, since it'll be bundled Co-authored-by: Sid Chatterjee Co-authored-by: Glen Maddern --- .changeset/unlucky-rocks-obey.md | 7 + package-lock.json | 90 +++++++-- packages/wrangler/package.json | 3 +- packages/wrangler/src/__tests__/pages.test.ts | 101 ++++++++-- packages/wrangler/src/cfetch/internal.ts | 11 +- packages/wrangler/src/pages.tsx | 186 ++++++++++++------ 6 files changed, 303 insertions(+), 95 deletions(-) create mode 100644 .changeset/unlucky-rocks-obey.md diff --git a/.changeset/unlucky-rocks-obey.md b/.changeset/unlucky-rocks-obey.md new file mode 100644 index 000000000000..f1bab2449385 --- /dev/null +++ b/.changeset/unlucky-rocks-obey.md @@ -0,0 +1,7 @@ +--- +"wrangler": patch +--- + +feat: Use new bulk upload API for 'wrangler pages publish' + +This raises the file limit back up to 20k for a deployment. diff --git a/package-lock.json b/package-lock.json index 245d7f52775e..dc48f9bed35a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8018,6 +8018,12 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true + }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -14793,19 +14799,19 @@ } }, "node_modules/mime-db": { - "version": "1.51.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", - "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.34", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", - "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "dependencies": { - "mime-db": "1.51.0" + "mime-db": "1.52.0" }, "engines": { "node": ">= 0.6" @@ -15743,6 +15749,34 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-7.2.0.tgz", + "integrity": "sha512-Kvv7p13M46lTYLQ/PsZdaj/1Vj6u/8oiIJgyQyx4oVkOfHdd7M2EZvXigDvcsSzRwanCzQirV5bJPQFoSQt5MA==", + "dev": true, + "dependencies": { + "eventemitter3": "^4.0.7", + "p-timeout": "^5.0.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-queue/node_modules/p-timeout": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-5.0.2.tgz", + "integrity": "sha512-sEmji9Yaq+Tw+STwsGAE56hf7gMy9p0tQfJojIAamB7WHJYJKf1qlsg9jqBWG8q9VCxKPhZaP/AcXwEoBcYQhQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-timeout": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", @@ -20578,6 +20612,7 @@ "jest-websocket-mock": "^2.3.0", "mime": "^3.0.0", "open": "^8.4.0", + "p-queue": "^7.2.0", "pretty-bytes": "^6.0.0", "prompts": "^2.4.2", "react": "^17.0.2", @@ -26492,6 +26527,12 @@ "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", "dev": true }, + "eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true + }, "events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -31145,16 +31186,16 @@ "version": "3.0.0" }, "mime-db": { - "version": "1.51.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", - "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==" + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, "mime-types": { - "version": "2.1.34", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", - "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "requires": { - "mime-db": "1.51.0" + "mime-db": "1.52.0" } }, "mimic-fn": { @@ -31796,6 +31837,24 @@ "aggregate-error": "^3.0.0" } }, + "p-queue": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-7.2.0.tgz", + "integrity": "sha512-Kvv7p13M46lTYLQ/PsZdaj/1Vj6u/8oiIJgyQyx4oVkOfHdd7M2EZvXigDvcsSzRwanCzQirV5bJPQFoSQt5MA==", + "dev": true, + "requires": { + "eventemitter3": "^4.0.7", + "p-timeout": "^5.0.2" + }, + "dependencies": { + "p-timeout": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-5.0.2.tgz", + "integrity": "sha512-sEmji9Yaq+Tw+STwsGAE56hf7gMy9p0tQfJojIAamB7WHJYJKf1qlsg9jqBWG8q9VCxKPhZaP/AcXwEoBcYQhQ==", + "dev": true + } + } + }, "p-timeout": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", @@ -35330,7 +35389,7 @@ "fsevents": "~2.3.2", "get-port": "^6.1.2", "glob-to-regexp": "0.4.1", - "http-terminator": "*", + "http-terminator": "^3.2.0", "ignore": "^5.2.0", "ink": "^3.2.0", "ink-select-input": "^4.2.1", @@ -35344,6 +35403,7 @@ "miniflare": "2.4.0", "nanoid": "^3.3.3", "open": "^8.4.0", + "p-queue": "^7.2.0", "path-to-regexp": "^6.2.0", "pretty-bytes": "^6.0.0", "prompts": "^2.4.2", diff --git a/packages/wrangler/package.json b/packages/wrangler/package.json index 39621f47a909..fac718963496 100644 --- a/packages/wrangler/package.json +++ b/packages/wrangler/package.json @@ -87,6 +87,7 @@ "jest-websocket-mock": "^2.3.0", "mime": "^3.0.0", "open": "^8.4.0", + "p-queue": "^7.2.0", "pretty-bytes": "^6.0.0", "prompts": "^2.4.2", "react": "^17.0.2", @@ -132,7 +133,7 @@ "testTimeout": 30000, "testRegex": ".*.(test|spec)\\.[jt]sx?$", "transformIgnorePatterns": [ - "node_modules/(?!find-up|locate-path|p-locate|p-limit|yocto-queue|path-exists|execa|strip-final-newline|npm-run-path|path-key|onetime|mimic-fn|human-signals|is-stream|get-port|supports-color|pretty-bytes)" + "node_modules/(?!find-up|locate-path|p-locate|p-limit|p-timeout|p-queue|yocto-queue|path-exists|execa|strip-final-newline|npm-run-path|path-key|onetime|mimic-fn|human-signals|is-stream|get-port|supports-color|pretty-bytes)" ], "moduleNameMapper": { "clipboardy": "/src/__tests__/helpers/clipboardy-mock.js", diff --git a/packages/wrangler/src/__tests__/pages.test.ts b/packages/wrangler/src/__tests__/pages.test.ts index bb5633283691..749f94ea5b29 100644 --- a/packages/wrangler/src/__tests__/pages.test.ts +++ b/packages/wrangler/src/__tests__/pages.test.ts @@ -5,7 +5,7 @@ import { mockConsoleMethods } from "./helpers/mock-console"; import { runInTempDir } from "./helpers/run-in-tmp"; import { runWrangler } from "./helpers/run-wrangler"; import type { Project, Deployment } from "../pages"; -import type { File, FormData } from "undici"; +import type { FormData } from "undici"; describe("pages", () => { runInTempDir(); @@ -279,21 +279,53 @@ describe("pages", () => { writeFileSync("logo.png", "foobar"); setMockResponse( - "/accounts/:accountId/pages/projects/foo/file", - async ([_url, accountId], init) => { + "/accounts/:accountId/pages/projects/foo/upload-token", + async ([_url, accountId]) => { expect(accountId).toEqual("some-account-id"); - expect(init.method).toEqual("POST"); - const body = init.body as FormData; - const logoPNGFile = body.get("file") as File; - expect(await logoPNGFile.text()).toEqual("foobar"); - expect(logoPNGFile.name).toEqual("logo.png"); return { - id: "2082190357cfd3617ccfe04f340c6247", + jwt: "<>", }; } ); + setMockResponse( + "/pages/assets/check-missing", + "POST", + async (_, init) => { + expect(init.headers).toMatchObject({ + Authorization: "Bearer <>", + }); + const body = JSON.parse(init.body as string) as { hashes: string[] }; + expect(body).toMatchObject({ + hashes: ["2082190357cfd3617ccfe04f340c6247"], + }); + return body.hashes; + } + ); + + setMockResponse("/pages/assets/upload", "POST", async (_, init) => { + expect(init.headers).toMatchObject({ + Authorization: "Bearer <>", + }); + const body = JSON.parse(init.body as string) as { + key: string; + value: string; + metadata: { contentType: string }; + base64: boolean; + }[]; + expect(body).toMatchObject([ + { + key: "2082190357cfd3617ccfe04f340c6247", + value: Buffer.from("foobar").toString("base64"), + metadata: { + contentType: "image/png", + }, + base64: true, + }, + ]); + }); + setMockResponse( "/accounts/:accountId/pages/projects/foo/deployments", async ([_url, accountId], init) => { @@ -326,15 +358,58 @@ describe("pages", () => { it("should not error when directory names contain periods and houses a extensionless file", async () => { mkdirSync(".well-known"); + // Note: same content as previous test, but since it's a different extension, + // it hashes to a different value writeFileSync(".well-known/foobar", "foobar"); setMockResponse( - "/accounts/:accountId/pages/projects/foo/file", - async () => ({ - id: "7b764dacfd211bebd8077828a7ddefd7", - }) + "/accounts/:accountId/pages/projects/foo/upload-token", + async ([_url, accountId]) => { + expect(accountId).toEqual("some-account-id"); + + return { + jwt: "<>", + }; + } + ); + + setMockResponse( + "/pages/assets/check-missing", + "POST", + async (_, init) => { + expect(init.headers).toMatchObject({ + Authorization: "Bearer <>", + }); + const body = JSON.parse(init.body as string) as { hashes: string[] }; + expect(body).toMatchObject({ + hashes: ["7b764dacfd211bebd8077828a7ddefd7"], + }); + return body.hashes; + } ); + setMockResponse("/pages/assets/upload", "POST", async (_, init) => { + expect(init.headers).toMatchObject({ + Authorization: "Bearer <>", + }); + const body = JSON.parse(init.body as string) as { + key: string; + value: string; + metadata: { contentType: string }; + base64: boolean; + }[]; + expect(body).toMatchObject([ + { + key: "7b764dacfd211bebd8077828a7ddefd7", + value: Buffer.from("foobar").toString("base64"), + metadata: { + contentType: "application/octet-stream", + }, + base64: true, + }, + ]); + }); + setMockResponse( "/accounts/:accountId/pages/projects/foo/deployments", async () => ({ diff --git a/packages/wrangler/src/cfetch/internal.ts b/packages/wrangler/src/cfetch/internal.ts index 9203d0e525e3..72eead7348ca 100644 --- a/packages/wrangler/src/cfetch/internal.ts +++ b/packages/wrangler/src/cfetch/internal.ts @@ -32,7 +32,7 @@ export async function fetchInternal( await requireLoggedIn(); const apiToken = requireApiToken(); const headers = cloneHeaders(init.headers); - addAuthorizationHeader(headers, apiToken); + addAuthorizationHeaderIfUnspecified(headers, apiToken); const queryString = queryParams ? `?${queryParams.toString()}` : ""; const method = init.method ?? "GET"; @@ -96,16 +96,13 @@ function requireApiToken(): string { return authToken; } -function addAuthorizationHeader( +function addAuthorizationHeaderIfUnspecified( headers: Record, apiToken: string ): void { - if ("Authorization" in headers) { - throw new Error( - "The request already specifies an authorisation header - cannot add a new one." - ); + if (!("Authorization" in headers)) { + headers["Authorization"] = `Bearer ${apiToken}`; } - headers["Authorization"] = `Bearer ${apiToken}`; } /** diff --git a/packages/wrangler/src/pages.tsx b/packages/wrangler/src/pages.tsx index 18e67fdacd92..a0c96e641715 100644 --- a/packages/wrangler/src/pages.tsx +++ b/packages/wrangler/src/pages.tsx @@ -14,6 +14,7 @@ import SelectInput from "ink-select-input"; import Spinner from "ink-spinner"; import Table from "ink-table"; import { getType } from "mime"; +import PQueue from "p-queue"; import prettyBytes from "pretty-bytes"; import React from "react"; import { format as timeagoFormat } from "timeago.js"; @@ -32,7 +33,11 @@ import openInBrowser from "./open-in-browser"; import { toUrlPath } from "./paths"; import { requireAuth } from "./user"; import type { Config } from "../pages/functions/routes"; -import type { Headers, Request, fetch } from "@miniflare/core"; +import type { + Headers as MiniflareHeaders, + Request as MiniflareRequest, + fetch as miniflareFetch, +} from "@miniflare/core"; import type { BuildResult } from "esbuild"; import type { MiniflareOptions } from "miniflare"; import type { BuilderCallback, CommandModule } from "yargs"; @@ -284,7 +289,7 @@ function generateRulesMatcher( T ][]; - return ({ request }: { request: Request }) => { + return ({ request }: { request: MiniflareRequest }) => { const { pathname, host } = new URL(request.url); return compiledRules @@ -357,7 +362,7 @@ function generateHeadersMatcher(headersFile: string) { ) ); - return (request: Request) => { + return (request: MiniflareRequest) => { const matches = rulesMatcher({ request, }); @@ -406,7 +411,7 @@ function generateRedirectsMatcher(redirectsFile: string) { }) ); - return (request: Request) => { + return (request: MiniflareRequest) => { const match = rulesMatcher({ request, })[0]; @@ -461,7 +466,9 @@ function hasFileExtension(pathname: string) { return /\/.+\.[a-z0-9]+$/i.test(pathname); } -async function generateAssetsFetch(directory: string): Promise { +async function generateAssetsFetch( + directory: string +): Promise { // Defer importing miniflare until we really need it const { Headers, Request, Response } = await import("@miniflare/core"); @@ -510,12 +517,12 @@ async function generateAssetsFetch(directory: string): Promise { return readFileSync(file); }; - const generateResponse = (request: Request) => { + const generateResponse = (request: MiniflareRequest) => { const url = new URL(request.url); const deconstructedResponse: { status: number; - headers: Headers; + headers: MiniflareHeaders; body?: Buffer; } = { status: 200, @@ -667,8 +674,12 @@ async function generateAssetsFetch(directory: string): Promise { }; const attachHeaders = ( - request: Request, - deconstructedResponse: { status: number; headers: Headers; body?: Buffer } + request: MiniflareRequest, + deconstructedResponse: { + status: number; + headers: MiniflareHeaders; + body?: Buffer; + } ) => { const headers = deconstructedResponse.headers; const newHeaders = new Headers({}); @@ -1031,12 +1042,9 @@ const createDeployment: CommandModule< builtFunctions = readFileSync(outfile, "utf-8"); } - type File = { - content: Buffer; - metadata: Metadata; - }; - - type Metadata = { + type FileContainer = { + content: string; + contentType: string; sizeInBytes: number; hash: string; }; @@ -1051,7 +1059,7 @@ const createDeployment: CommandModule< const walk = async ( dir: string, - fileMap: Map = new Map(), + fileMap: Map = new Map(), depth = 0 ) => { const files = await readdir(dir); @@ -1085,8 +1093,6 @@ const createDeployment: CommandModule< const base64Content = fileContent.toString("base64"); const extension = extname(basename(name)).substring(1); - const content = base64Content + extension; - if (filestat.size > 25 * 1024 * 1024) { throw new Error( `Error: Pages only supports files up to ${prettyBytes( @@ -1096,11 +1102,12 @@ const createDeployment: CommandModule< } fileMap.set(name, { - content: fileContent, - metadata: { - sizeInBytes: filestat.size, - hash: hash(content).toString("hex").slice(0, 32), - }, + content: base64Content, + contentType: getType(name) || "application/octet-stream", + sizeInBytes: filestat.size, + hash: hash(base64Content + extension) + .toString("hex") + .slice(0, 32), }); } }) @@ -1111,51 +1118,112 @@ const createDeployment: CommandModule< const fileMap = await walk(directory); + if (fileMap.size > 20000) { + throw new FatalError( + `Error: Pages only supports up to 20,000 files in a deployment. Ensure you have specified your build output directory correctly.`, + 1 + ); + } + + const files = [...fileMap.values()]; + + const { jwt } = await fetchResult<{ jwt: string }>( + `/accounts/${accountId}/pages/projects/${projectName}/upload-token` + ); + const start = Date.now(); - const files: Array> = []; + const missingHashes = await fetchResult( + `/pages/assets/check-missing`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${jwt}`, + }, + body: JSON.stringify({ + hashes: files.map(({ hash }) => hash), + }), + } + ); - if (fileMap.size > 1000) { - throw new Error( - `Error: Pages only supports up to 1,000 files in a deployment at the moment.\nTry a smaller project perhaps?` - ); - } + const sortedFiles = files + .filter((file) => missingHashes.includes(file.hash)) + .sort((a, b) => b.sizeInBytes - a.sizeInBytes); + + const buckets: { + files: FileContainer[]; + remainingSize: number; + }[] = []; + + const MAX_BUCKET_SIZE = 50 * 1024 * 1024; + const MAX_BUCKET_FILE_COUNT = 5000; + + for (const file of sortedFiles) { + let inserted = false; + + for (const bucket of buckets) { + if ( + bucket.remainingSize >= file.sizeInBytes && + bucket.files.length < MAX_BUCKET_FILE_COUNT + ) { + bucket.files.push(file); + bucket.remainingSize -= file.sizeInBytes; + inserted = true; + break; + } + } - let counter = 0; + if (!inserted) { + buckets.push({ + files: [file], + remainingSize: MAX_BUCKET_SIZE - file.sizeInBytes, + }); + } + } + let counter = fileMap.size - sortedFiles.length; const { rerender, unmount } = render( ); - fileMap.forEach((file: File, name: string) => { - const form = new FormData(); - form.append( - "file", - new File([new Uint8Array(file.content.buffer)], name) - ); - - // TODO: Consider a retry + const queue = new PQueue({ concurrency: 10 }); - const promise = fetchResult<{ id: string }>( - `/accounts/${accountId}/pages/projects/${projectName}/file`, - { + for (const bucket of buckets) { + const payload = bucket.files.map((file) => ({ + key: file.hash, + value: file.content, + metadata: { + contentType: file.contentType, + }, + base64: true, + })); + queue.add(() => + fetchResult(`/pages/assets/upload`, { method: "POST", - body: form, - } - ).then((response) => { - counter++; - rerender(); - if (response.id != file.metadata.hash) { - throw new Error( - `Looks like there was an issue uploading '${name}'. Try again perhaps?` - ); - } - }); - - files.push(promise); - }); + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${jwt}`, + }, + body: JSON.stringify(payload), + }).then( + () => { + counter += bucket.files.length; + rerender(); + }, + (error) => { + return Promise.reject( + new FatalError( + "Failed to upload files. Please try again.", + error.code || 1 + ) + ); + } + ) + ); + } - await Promise.all(files); + await queue.onIdle(); unmount(); @@ -1173,7 +1241,7 @@ const createDeployment: CommandModule< Object.fromEntries( [...fileMap.entries()].map(([fileName, file]) => [ `/${fileName}`, - file.metadata.hash, + file.hash, ]) ) ) @@ -1477,7 +1545,7 @@ export const pages: BuilderCallback = (yargs) => { // env.ASSETS.fetch serviceBindings: { - async ASSETS(request: Request) { + async ASSETS(request: MiniflareRequest) { if (proxyPort) { try { const url = new URL(request.url); @@ -1869,7 +1937,7 @@ export const pages: BuilderCallback = (yargs) => { } as CommandModule); }; -const invalidAssetsFetch: typeof fetch = () => { +const invalidAssetsFetch: typeof miniflareFetch = () => { throw new Error( "Trying to fetch assets directly when there is no `directory` option specified, and not in `local` mode." );