Use new bulk upload API for 'pages publish' (#1028)

* Use new bulk upload API for 'pages publish'

* Only upload missing files

* Get PAGES_API_HOST from Env

* Using new /pages/assets endpoints

* Added tests for pages bulk upload endpoints

* fixing imports

* removed authOverride by relaxing the constraint about overriding Authorization header

* tweaked based on review comments

* switched p-queue to be a devDep of Wrangler, since it'll be bundled

Co-authored-by: Sid Chatterjee <[email protected]>
Co-authored-by: Glen Maddern <[email protected]>

Author: 3 people

.changeset/unlucky-rocks-obey.md

@@ -0,0 +1,7 @@
+---
+"wrangler": patch
+---
+
+feat: Use new bulk upload API for 'wrangler pages publish'
+
+This raises the file limit back up to 20k for a deployment.

package-lock.json

@@ -87,6 +87,7 @@
     "jest-websocket-mock": "^2.3.0",
     "mime": "^3.0.0",
     "open": "^8.4.0",
+    "p-queue": "^7.2.0",
     "pretty-bytes": "^6.0.0",
     "prompts": "^2.4.2",
     "react": "^17.0.2",
@@ -132,7 +133,7 @@
     "testTimeout": 30000,
     "testRegex": ".*.(test|spec)\\.[jt]sx?$",
     "transformIgnorePatterns": [
-      "node_modules/(?!find-up|locate-path|p-locate|p-limit|yocto-queue|path-exists|execa|strip-final-newline|npm-run-path|path-key|onetime|mimic-fn|human-signals|is-stream|get-port|supports-color|pretty-bytes)"
+      "node_modules/(?!find-up|locate-path|p-locate|p-limit|p-timeout|p-queue|yocto-queue|path-exists|execa|strip-final-newline|npm-run-path|path-key|onetime|mimic-fn|human-signals|is-stream|get-port|supports-color|pretty-bytes)"
     ],
     "moduleNameMapper": {
       "clipboardy": "<rootDir>/src/__tests__/helpers/clipboardy-mock.js",

packages/wrangler/package.json

@@ -5,7 +5,7 @@ import { mockConsoleMethods } from "./helpers/mock-console";
 import { runInTempDir } from "./helpers/run-in-tmp";
 import { runWrangler } from "./helpers/run-wrangler";
 import type { Project, Deployment } from "../pages";
-import type { File, FormData } from "undici";
+import type { FormData } from "undici";
 
 describe("pages", () => {
   runInTempDir();
@@ -279,21 +279,53 @@ describe("pages", () => {
       writeFileSync("logo.png", "foobar");
 
       setMockResponse(
-        "/accounts/:accountId/pages/projects/foo/file",
-        async ([_url, accountId], init) => {
+        "/accounts/:accountId/pages/projects/foo/upload-token",
+        async ([_url, accountId]) => {
           expect(accountId).toEqual("some-account-id");
-          expect(init.method).toEqual("POST");
-          const body = init.body as FormData;
-          const logoPNGFile = body.get("file") as File;
-          expect(await logoPNGFile.text()).toEqual("foobar");
-          expect(logoPNGFile.name).toEqual("logo.png");
 
           return {
-            id: "2082190357cfd3617ccfe04f340c6247",
+            jwt: "<<funfetti-auth-jwt>>",
           };
         }
       );
 
+      setMockResponse(
+        "/pages/assets/check-missing",
+        "POST",
+        async (_, init) => {
+          expect(init.headers).toMatchObject({
+            Authorization: "Bearer <<funfetti-auth-jwt>>",
+          });
+          const body = JSON.parse(init.body as string) as { hashes: string[] };
+          expect(body).toMatchObject({
+            hashes: ["2082190357cfd3617ccfe04f340c6247"],
+          });
+          return body.hashes;
+        }
+      );
+
+      setMockResponse("/pages/assets/upload", "POST", async (_, init) => {
+        expect(init.headers).toMatchObject({
+          Authorization: "Bearer <<funfetti-auth-jwt>>",
+        });
+        const body = JSON.parse(init.body as string) as {
+          key: string;
+          value: string;
+          metadata: { contentType: string };
+          base64: boolean;
+        }[];
+        expect(body).toMatchObject([
+          {
+            key: "2082190357cfd3617ccfe04f340c6247",
+            value: Buffer.from("foobar").toString("base64"),
+            metadata: {
+              contentType: "image/png",
+            },
+            base64: true,
+          },
+        ]);
+      });
+
       setMockResponse(
         "/accounts/:accountId/pages/projects/foo/deployments",
         async ([_url, accountId], init) => {
@@ -326,15 +358,58 @@ describe("pages", () => {
 
     it("should not error when directory names contain periods and houses a extensionless file", async () => {
       mkdirSync(".well-known");
+      // Note: same content as previous test, but since it's a different extension,
+      // it hashes to a different value
       writeFileSync(".well-known/foobar", "foobar");
 
       setMockResponse(
-        "/accounts/:accountId/pages/projects/foo/file",
-        async () => ({
-          id: "7b764dacfd211bebd8077828a7ddefd7",
-        })
+        "/accounts/:accountId/pages/projects/foo/upload-token",
+        async ([_url, accountId]) => {
+          expect(accountId).toEqual("some-account-id");
+
+          return {
+            jwt: "<<funfetti-auth-jwt>>",
+          };
+        }
+      );
+
+      setMockResponse(
+        "/pages/assets/check-missing",
+        "POST",
+        async (_, init) => {
+          expect(init.headers).toMatchObject({
+            Authorization: "Bearer <<funfetti-auth-jwt>>",
+          });
+          const body = JSON.parse(init.body as string) as { hashes: string[] };
+          expect(body).toMatchObject({
+            hashes: ["7b764dacfd211bebd8077828a7ddefd7"],
+          });
+          return body.hashes;
+        }
       );
 
+      setMockResponse("/pages/assets/upload", "POST", async (_, init) => {
+        expect(init.headers).toMatchObject({
+          Authorization: "Bearer <<funfetti-auth-jwt>>",
+        });
+        const body = JSON.parse(init.body as string) as {
+          key: string;
+          value: string;
+          metadata: { contentType: string };
+          base64: boolean;
+        }[];
+        expect(body).toMatchObject([
+          {
+            key: "7b764dacfd211bebd8077828a7ddefd7",
+            value: Buffer.from("foobar").toString("base64"),
+            metadata: {
+              contentType: "application/octet-stream",
+            },
+            base64: true,
+          },
+        ]);
+      });
+
       setMockResponse(
         "/accounts/:accountId/pages/projects/foo/deployments",
         async () => ({

packages/wrangler/src/__tests__/pages.test.ts

@@ -32,7 +32,7 @@ export async function fetchInternal<ResponseType>(
   await requireLoggedIn();
   const apiToken = requireApiToken();
   const headers = cloneHeaders(init.headers);
-  addAuthorizationHeader(headers, apiToken);
+  addAuthorizationHeaderIfUnspecified(headers, apiToken);
 
   const queryString = queryParams ? `?${queryParams.toString()}` : "";
   const method = init.method ?? "GET";
@@ -96,16 +96,13 @@ function requireApiToken(): string {
   return authToken;
 }
 
-function addAuthorizationHeader(
+function addAuthorizationHeaderIfUnspecified(
   headers: Record<string, string>,
   apiToken: string
 ): void {
-  if ("Authorization" in headers) {
-    throw new Error(
-      "The request already specifies an authorisation header - cannot add a new one."
-    );
+  if (!("Authorization" in headers)) {
+    headers["Authorization"] = `Bearer ${apiToken}`;
   }
-  headers["Authorization"] = `Bearer ${apiToken}`;
 }
 
 /**