WIP: proper no-bundle in Pages

Author: GregBrimble

fixtures/pages-workerjs-directory/package.json

@@ -0,0 +1,18 @@
+{
+	"name": "pages-workerjs-directory",
+	"version": "0.0.0",
+	"private": true,
+	"sideEffects": false,
+	"scripts": {
+		"check:type": "tsc",
+		"dev": "npx wrangler pages dev public --port 8794",
+		"test": "npx vitest",
+		"test:ci": "npx vitest"
+	},
+	"devDependencies": {
+		"undici": "^5.9.1"
+	},
+	"engines": {
+		"node": ">=16.13"
+	}
+}

fixtures/pages-workerjs-directory/public/_worker.js/index.js

@@ -0,0 +1,11 @@
+export default {
+	async fetch(request, env) {
+		switch (new URL(request.url).pathname) {
+			case "/worker": {
+				return new Response(require("./other-script").default);
+			}
+		}
+
+		return env.ASSETS.fetch(request);
+	},
+};

fixtures/pages-workerjs-directory/public/_worker.js/other-script.js

@@ -0,0 +1 @@
+export default "test";

fixtures/pages-workerjs-directory/public/index.html

@@ -0,0 +1 @@
+<h1>Hello, world!</h1>

fixtures/pages-workerjs-directory/tests/index.test.ts

@@ -0,0 +1,21 @@
+import { resolve } from "node:path";
+import { fetch } from "undici";
+import { describe, it } from "vitest";
+import { runWranglerPagesDev } from "../../shared/src/run-wrangler-long-lived";
+
+describe.concurrent("Pages _worker.js/ directory", () => {
+	it("should support non-bundling with 'dev'", async ({ expect }) => {
+		const { ip, port, stop } = await runWranglerPagesDev(
+			resolve(__dirname, ".."),
+			"public",
+			["--port=0"]
+		);
+		await expect(
+			fetch(`http://${ip}:${port}/`).then((resp) => resp.text())
+		).resolves.toContain("Hello, world!");
+		await expect(
+			fetch(`http://${ip}:${port}/worker`).then((resp) => resp.text())
+		).resolves.toContain("test");
+		await stop();
+	});
+});

fixtures/pages-workerjs-directory/tsconfig.json

@@ -0,0 +1,12 @@
+{
+	"compilerOptions": {
+		"target": "ES2020",
+		"esModuleInterop": true,
+		"module": "CommonJS",
+		"lib": ["ES2020"],
+		"types": ["node"],
+		"moduleResolution": "node",
+		"noEmit": true
+	},
+	"include": ["tests", "../../node-types.d.ts"]
+}

packages/wrangler/src/__tests__/pages/functions-build.test.ts

@@ -449,4 +449,67 @@ export default {
 		hello.js:2:36: ERROR: Could not resolve \\"node:async_hooks\\""
 	`);
 	});
+
+	it("should compile a _worker.js/ directory", async () => {
+		mkdirSync("public");
+		mkdirSync("public/_worker.js");
+		writeFileSync(
+			"public/_worker.js/index.js",
+			`
+export default {
+  async fetch(request, env) {
+		return new Response("Hello from _worker.js/index.js");
+	},
+};`
+		);
+		writeFileSync(
+			"public/_worker.js/cat.js",
+			`
+export const cat = "cat";`
+		);
+
+		await runWrangler(
+			`pages functions build --outfile=public/_worker.bundle --compatibility-flag=nodejs_compat`
+		);
+
+		expect(existsSync("public/_worker.bundle")).toBe(true);
+		expect(std.out).toMatchInlineSnapshot(
+			`"🚧 'wrangler pages <command>' is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose"`
+		);
+
+		const workerBundleContents = readFileSync("public/_worker.bundle", "utf-8");
+		const workerBundleWithConstantData = replaceRandomWithConstantData(
+			workerBundleContents,
+			[
+				[/------formdata-undici-0.[0-9]*/g, "------formdata-undici-0.test"],
+				[/functionsWorker-0.[0-9]*.js/g, "functionsWorker-0.test.js"],
+			]
+		);
+
+		expect(workerBundleWithConstantData).toMatchInlineSnapshot(`
+		"------formdata-undici-0.test
+		Content-Disposition: form-data; name=\\"metadata\\"
+
+		{\\"main_module\\":\\"index.js\\"}
+		------formdata-undici-0.test
+		Content-Disposition: form-data; name=\\"index.js\\"; filename=\\"index.js\\"
+		Content-Type: application/javascript+module
+
+
+		export default {
+		  async fetch(request, env) {
+				return new Response(\\"Hello from _worker.js/index.js\\");
+			},
+		};
+		------formdata-undici-0.test
+		Content-Disposition: form-data; name=\\"cat.js\\"; filename=\\"cat.js\\"
+		Content-Type: application/javascript+module
+
+
+		export const cat = \\"cat\\";
+		------formdata-undici-0.test--"
+	`);
+
+		expect(std.err).toMatchInlineSnapshot(`""`);
+	});
 });

packages/wrangler/src/api/pages/publish.tsx

@@ -1,6 +1,6 @@
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, lstatSync, readFileSync } from "node:fs";
 import { tmpdir } from "node:os";
-import { join, resolve as resolvePath } from "node:path";
+import { join, resolve, resolve as resolvePath } from "node:path";
 import { cwd } from "node:process";
 import { File, FormData } from "undici";
 import { fetchResult } from "../../cfetch";
@@ -17,6 +17,7 @@ import {
 } from "../../pages/functions/buildWorker";
 import { validateRoutes } from "../../pages/functions/routes-validation";
 import { upload } from "../../pages/upload";
+import traverseModuleGraph from "../../traverse-module-graph";
 import { createUploadWorkerBundleContents } from "./create-worker-bundle-contents";
 import type { BundleResult } from "../../bundle";
 import type { Project, Deployment } from "@cloudflare/types";
@@ -95,9 +96,10 @@ export async function publish({
 		_redirects: string | undefined,
 		_routesGenerated: string | undefined,
 		_routesCustom: string | undefined,
+		_workerJSDirectory = false,
 		_workerJS: string | undefined;
 
-	const workerScriptPath = resolvePath(directory, "_worker.js");
+	const _workerPath = resolvePath(directory, "_worker.js");
 
 	try {
 		_headers = readFileSync(join(directory, "_headers"), "utf-8");
@@ -116,7 +118,10 @@ export async function publish({
 	} catch {}
 
 	try {
-		_workerJS = readFileSync(workerScriptPath, "utf-8");
+		_workerJSDirectory = lstatSync(_workerPath).isDirectory();
+		if (!_workerJSDirectory) {
+			_workerJS = readFileSync(_workerPath, "utf-8");
+		}
 	} catch {}
 
 	// Grab the bindings from the API, we need these for shims and other such hacky inserts
@@ -243,13 +248,28 @@ export async function publish({
 	 * When using a _worker.js file, the entire /functions directory is ignored
 	 * – this includes its routing and middleware characteristics.
 	 */
-	if (_workerJS) {
+	if (_workerJSDirectory) {
+		workerBundle = await traverseModuleGraph(
+			{
+				file: resolve(join(_workerPath, "index.js")),
+				directory: resolve(_workerPath),
+				format: "modules",
+				moduleRoot: resolve(_workerPath),
+			},
+			[
+				{
+					type: "ESModule",
+					globs: ["**/*.js"],
+				},
+			]
+		);
+	} else if (_workerJS) {
 		if (bundle) {
 			const outfile = join(tmpdir(), `./bundledWorker-${Math.random()}.mjs`);
 			workerBundle = await buildRawWorker({
-				workerScriptPath,
+				workerScriptPath: _workerPath,
 				outfile,
-				directory: directory ?? ".",
+				directory,
 				local: false,
 				sourcemap: true,
 				watch: false,
@@ -258,13 +278,13 @@ export async function publish({
 				nodejsCompat,
 			});
 		} else {
-			await checkRawWorker(workerScriptPath, () => {});
-			// TODO: Replace this with the cool new no-bundle stuff when that lands: https://github.com/cloudflare/workers-sdk/pull/2769
+			await checkRawWorker(_workerPath, () => {});
+			// TODO: Let users configure this in the future.
 			workerBundle = {
 				modules: [],
 				dependencies: {},
 				stop: undefined,
-				resolvedEntryPointPath: workerScriptPath,
+				resolvedEntryPointPath: _workerPath,
 				bundleType: "esm",
 			};
 		}

packages/wrangler/src/pages/build.ts

@@ -1,9 +1,17 @@
-import { existsSync, mkdirSync, writeFileSync } from "node:fs";
-import { basename, dirname, relative, resolve as resolvePath } from "node:path";
+import { existsSync, lstatSync, mkdirSync, writeFileSync } from "node:fs";
+import {
+	basename,
+	dirname,
+	join,
+	relative,
+	resolve,
+	resolve as resolvePath,
+} from "node:path";
 import { createUploadWorkerBundleContents } from "../api/pages/create-worker-bundle-contents";
 import { FatalError } from "../errors";
 import { logger } from "../logger";
 import * as metrics from "../metrics";
+import traverseModuleGraph from "../traverse-module-graph";
 import { buildFunctions } from "./buildFunctions";
 import { isInPagesCI } from "./constants";
 import {
@@ -208,21 +216,38 @@ export const Handler = async (args: PagesBuildArgs) => {
 		 * and if we were able to resolve _worker.js
 		 */
 		if (workerScriptPath) {
-			/**
-			 * `buildRawWorker` builds `_worker.js`, but doesn't give us the bundle
-			 * we want to return, which includes the external dependencies (like wasm,
-			 * binary, text). Let's output that build result to memory and only write
-			 * to disk once we have the final bundle
-			 */
-			bundle = await buildRawWorker({
-				workerScriptPath,
-				outdir,
-				directory: buildOutputDirectory,
-				local: false,
-				sourcemap,
-				watch,
-				betaD1Shims: d1Databases,
-			});
+			if (lstatSync(workerScriptPath).isDirectory()) {
+				bundle = await traverseModuleGraph(
+					{
+						file: resolve(join(workerScriptPath, "index.js")),
+						directory: resolve(workerScriptPath),
+						format: "modules",
+						moduleRoot: resolve(workerScriptPath),
+					},
+					[
+						{
+							type: "ESModule",
+							globs: ["**/*.js"],
+						},
+					]
+				);
+			} else {
+				/**
+				 * `buildRawWorker` builds `_worker.js`, but doesn't give us the bundle
+				 * we want to return, which includes the external dependencies (like wasm,
+				 * binary, text). Let's output that build result to memory and only write
+				 * to disk once we have the final bundle
+				 */
+				bundle = await buildRawWorker({
+					workerScriptPath,
+					outdir,
+					directory: buildOutputDirectory,
+					local: false,
+					sourcemap,
+					watch,
+					betaD1Shims: d1Databases,
+				});
+			}
 		} else {
 			try {
 				/**