WIP: proper no-bundle in Pages

Author: GregBrimble

fixtures/pages-workerjs-directory/package.json

@@ -0,0 +1,18 @@
+{
+	"name": "pages-workerjs-directory",
+	"version": "0.0.0",
+	"private": true,
+	"sideEffects": false,
+	"scripts": {
+		"check:type": "tsc",
+		"dev": "npx wrangler pages dev public --port 8794",
+		"test": "npx vitest",
+		"test:ci": "npx vitest"
+	},
+	"devDependencies": {
+		"undici": "^5.9.1"
+	},
+	"engines": {
+		"node": ">=16.13"
+	}
+}

fixtures/pages-workerjs-directory/public/_worker.js/index.js

@@ -0,0 +1,10 @@
+export default {
+	async fetch(request, env) {
+		const { pathname } = new URL(request.url);
+		if (pathname !== "/") {
+			return new Response((await import(`./${pathname.slice(1)}`)).default);
+		}
+
+		return env.ASSETS.fetch(request);
+	},
+};

fixtures/pages-workerjs-directory/public/_worker.js/other-script.js

@@ -0,0 +1 @@
+export default "test";

fixtures/pages-workerjs-directory/public/index.html

@@ -0,0 +1 @@
+<h1>Hello, world!</h1>

fixtures/pages-workerjs-directory/tests/index.test.ts

@@ -0,0 +1,21 @@
+import { resolve } from "node:path";
+import { fetch } from "undici";
+import { describe, it } from "vitest";
+import { runWranglerPagesDev } from "../../shared/src/run-wrangler-long-lived";
+
+describe.concurrent("Pages _worker.js/ directory", () => {
+	it("should support non-bundling with 'dev'", async ({ expect }) => {
+		const { ip, port, stop } = await runWranglerPagesDev(
+			resolve(__dirname, ".."),
+			"public",
+			["--port=0"]
+		);
+		await expect(
+			fetch(`http://${ip}:${port}/`).then((resp) => resp.text())
+		).resolves.toContain("Hello, world!");
+		await expect(
+			fetch(`http://${ip}:${port}/other-script`).then((resp) => resp.text())
+		).resolves.toContain("test");
+		await stop();
+	});
+});

fixtures/pages-workerjs-directory/tsconfig.json

@@ -0,0 +1,12 @@
+{
+	"compilerOptions": {
+		"target": "ES2020",
+		"esModuleInterop": true,
+		"module": "CommonJS",
+		"lib": ["ES2020"],
+		"types": ["node"],
+		"moduleResolution": "node",
+		"noEmit": true
+	},
+	"include": ["tests", "../../node-types.d.ts"]
+}

packages/wrangler/src/__tests__/pages/functions-build.test.ts

@@ -449,4 +449,67 @@ export default {
 		hello.js:2:36: ERROR: Could not resolve \\"node:async_hooks\\""
 	`);
 	});
+
+	it("should compile a _worker.js/ directory", async () => {
+		mkdirSync("public");
+		mkdirSync("public/_worker.js");
+		writeFileSync(
+			"public/_worker.js/index.js",
+			`
+export default {
+  async fetch(request, env) {
+		return new Response("Hello from _worker.js/index.js");
+	},
+};`
+		);
+		writeFileSync(
+			"public/_worker.js/cat.js",
+			`
+export const cat = "cat";`
+		);
+
+		await runWrangler(
+			`pages functions build --outfile=public/_worker.bundle --compatibility-flag=nodejs_compat`
+		);
+
+		expect(existsSync("public/_worker.bundle")).toBe(true);
+		expect(std.out).toMatchInlineSnapshot(
+			`"🚧 'wrangler pages <command>' is a beta command. Please report any issues to https://github.com/cloudflare/workers-sdk/issues/new/choose"`
+		);
+
+		const workerBundleContents = readFileSync("public/_worker.bundle", "utf-8");
+		const workerBundleWithConstantData = replaceRandomWithConstantData(
+			workerBundleContents,
+			[
+				[/------formdata-undici-0.[0-9]*/g, "------formdata-undici-0.test"],
+				[/functionsWorker-0.[0-9]*.js/g, "functionsWorker-0.test.js"],
+			]
+		);
+
+		expect(workerBundleWithConstantData).toMatchInlineSnapshot(`
+		"------formdata-undici-0.test
+		Content-Disposition: form-data; name=\\"metadata\\"
+
+		{\\"main_module\\":\\"index.js\\"}
+		------formdata-undici-0.test
+		Content-Disposition: form-data; name=\\"index.js\\"; filename=\\"index.js\\"
+		Content-Type: application/javascript+module
+
+
+		export default {
+		  async fetch(request, env) {
+				return new Response(\\"Hello from _worker.js/index.js\\");
+			},
+		};
+		------formdata-undici-0.test
+		Content-Disposition: form-data; name=\\"cat.js\\"; filename=\\"cat.js\\"
+		Content-Type: application/javascript+module
+
+
+		export const cat = \\"cat\\";
+		------formdata-undici-0.test--"
+	`);
+
+		expect(std.err).toMatchInlineSnapshot(`""`);
+	});
 });

packages/wrangler/src/api/dev.ts

@@ -3,6 +3,7 @@ import { startApiDev, startDev } from "../dev";
 import { logger } from "../logger";
 
 import type { Environment } from "../config";
+import type { Rule } from "../config/environment";
 import type { EnablePagesAssetsServiceBindingOptions } from "../miniflare-cli/types";
 import type { RequestInit, Response, RequestInfo } from "undici";
 
@@ -42,6 +43,9 @@ export interface UnstableDevOptions {
 		bucket_name: string;
 		preview_bucket_name?: string;
 	}[];
+	bundleEntrypoint?: boolean;
+	moduleRoot?: string;
+	rules?: Rule[];
 	logLevel?: "none" | "info" | "error" | "log" | "warn" | "debug"; // Specify logging level  [choices: "debug", "info", "log", "warn", "error", "none"] [default: "log"]
 	inspect?: boolean;
 	local?: boolean;
@@ -150,6 +154,7 @@ export async function unstable_dev(
 					},
 					config: options?.config,
 					env: options?.env,
+					bundleEntrypoint: !!options?.bundleEntrypoint,
 					bundle: options?.bundle,
 					compatibilityDate: options?.compatibilityDate,
 					compatibilityFlags: options?.compatibilityFlags,

packages/wrangler/src/api/pages/publish.tsx

@@ -1,4 +1,4 @@
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, lstatSync, readFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join, resolve as resolvePath } from "node:path";
 import { cwd } from "node:process";
@@ -17,6 +17,7 @@ import {
 } from "../../pages/functions/buildWorker";
 import { validateRoutes } from "../../pages/functions/routes-validation";
 import { upload } from "../../pages/upload";
+import traverseModuleGraph from "../../traverse-module-graph";
 import { createUploadWorkerBundleContents } from "./create-worker-bundle-contents";
 import type { BundleResult } from "../../bundle";
 import type { Project, Deployment } from "@cloudflare/types";
@@ -95,9 +96,10 @@ export async function publish({
 		_redirects: string | undefined,
 		_routesGenerated: string | undefined,
 		_routesCustom: string | undefined,
+		_workerJSDirectory = false,
 		_workerJS: string | undefined;
 
-	const workerScriptPath = resolvePath(directory, "_worker.js");
+	const _workerPath = resolvePath(directory, "_worker.js");
 
 	try {
 		_headers = readFileSync(join(directory, "_headers"), "utf-8");
@@ -116,7 +118,10 @@ export async function publish({
 	} catch {}
 
 	try {
-		_workerJS = readFileSync(workerScriptPath, "utf-8");
+		_workerJSDirectory = lstatSync(_workerPath).isDirectory();
+		if (!_workerJSDirectory) {
+			_workerJS = readFileSync(_workerPath, "utf-8");
+		}
 	} catch {}
 
 	// Grab the bindings from the API, we need these for shims and other such hacky inserts
@@ -243,13 +248,28 @@ export async function publish({
 	 * When using a _worker.js file, the entire /functions directory is ignored
 	 * – this includes its routing and middleware characteristics.
 	 */
-	if (_workerJS) {
+	if (_workerJSDirectory) {
+		workerBundle = await traverseModuleGraph(
+			{
+				file: resolvePath(join(_workerPath, "index.js")),
+				directory: resolvePath(_workerPath),
+				format: "modules",
+				moduleRoot: resolvePath(_workerPath),
+			},
+			[
+				{
+					type: "ESModule",
+					globs: ["**/*.js"],
+				},
+			]
+		);
+	} else if (_workerJS) {
 		if (bundle) {
 			const outfile = join(tmpdir(), `./bundledWorker-${Math.random()}.mjs`);
 			workerBundle = await buildRawWorker({
-				workerScriptPath,
+				workerScriptPath: _workerPath,
 				outfile,
-				directory: directory ?? ".",
+				directory,
 				local: false,
 				sourcemap: true,
 				watch: false,
@@ -258,13 +278,13 @@ export async function publish({
 				nodejsCompat,
 			});
 		} else {
-			await checkRawWorker(workerScriptPath, () => {});
-			// TODO: Replace this with the cool new no-bundle stuff when that lands: https://github.com/cloudflare/workers-sdk/pull/2769
+			await checkRawWorker(_workerPath, () => {});
+			// TODO: Let users configure this in the future.
 			workerBundle = {
 				modules: [],
 				dependencies: {},
 				stop: undefined,
-				resolvedEntryPointPath: workerScriptPath,
+				resolvedEntryPointPath: _workerPath,
 				bundleType: "esm",
 			};
 		}

packages/wrangler/src/bundle.ts

@@ -117,6 +117,7 @@ export async function bundleWorker(
 	entry: Entry,
 	destination: string,
 	options: {
+		bundle?: boolean;
 		serveAssetsFromWorker: boolean;
 		assets?: StaticAssetsConfig;
 		betaD1Shims?: string[];
@@ -149,6 +150,7 @@ export async function bundleWorker(
 	}
 ): Promise<BundleResult> {
 	const {
+		bundle = true,
 		serveAssetsFromWorker,
 		betaD1Shims,
 		doBindings,
@@ -350,7 +352,7 @@ export async function bundleWorker(
 
 	const buildOptions: esbuild.BuildOptions & { metafile: true } = {
 		entryPoints: [inputEntry.file],
-		bundle: true,
+		bundle,
 		absWorkingDir: entry.directory,
 		outdir: destination,
 		entryNames: entryName || path.parse(entry.file).name,

packages/wrangler/src/dev.tsx

@@ -28,7 +28,7 @@ import {
 	printWranglerBanner,
 } from "./index";
 import type { Config, Environment } from "./config";
-import type { Route } from "./config/environment";
+import type { Route, Rule } from "./config/environment";
 import type { LoggerLevel } from "./logger";
 import type { EnablePagesAssetsServiceBindingOptions } from "./miniflare-cli/types";
 import type { CfWorkerInit } from "./worker";
@@ -334,6 +334,9 @@ export type AdditionalDevProps = {
 		preview_bucket_name?: string;
 	}[];
 	d1Databases?: Environment["d1_databases"];
+	bundleEntrypoint?: boolean;
+	moduleRoot?: string;
+	rules?: Rule[];
 };
 
 type StartDevOptions = DevArguments &
@@ -424,7 +427,8 @@ export async function startDev(args: StartDevOptions) {
 					zone={zoneId}
 					host={host}
 					routes={routes}
-					rules={getRules(configParam)}
+					bundleEntrypoint={!!args.bundleEntrypoint}
+					rules={args.rules ?? getRules(configParam)}
 					legacyEnv={isLegacyEnv(configParam)}
 					minify={args.minify ?? configParam.minify}
 					legacyNodeCompat={legacyNodeCompat}
@@ -560,7 +564,8 @@ export async function startApiDev(args: StartDevOptions) {
 			zone: zoneId,
 			host: host,
 			routes: routes,
-			rules: getRules(configParam),
+			bundleEntrypoint: !!args.bundleEntrypoint,
+			rules: args.rules ?? getRules(configParam),
 			legacyEnv: isLegacyEnv(configParam),
 			minify: args.minify ?? configParam.minify,
 			legacyNodeCompat,
@@ -687,7 +692,7 @@ async function validateDevServerSettings(
 	config: Config
 ) {
 	const entry = await getEntry(
-		{ assets: args.assets, script: args.script },
+		{ assets: args.assets, script: args.script, moduleRoot: args.moduleRoot },
 		config,
 		"dev"
 	);

packages/wrangler/src/dev/dev.tsx

@@ -116,6 +116,7 @@ export type DevProps = {
 	initialPort: number;
 	initialIp: string;
 	inspectorPort: number;
+	bundleEntrypoint: boolean;
 	rules: Config["rules"];
 	accountId: string | undefined;
 	initialMode: "local" | "remote";
@@ -271,6 +272,7 @@ function DevSession(props: DevSessionProps) {
 		entry: props.entry,
 		destination: directory,
 		jsxFactory: props.jsxFactory,
+		bundleEntrypoint: props.bundleEntrypoint,
 		rules: props.rules,
 		jsxFragment: props.jsxFragment,
 		serveAssetsFromWorker: Boolean(
@@ -294,6 +296,7 @@ function DevSession(props: DevSessionProps) {
 		testScheduled: props.testScheduled ?? false,
 		experimentalLocal: props.experimentalLocal,
 	});
+	console.log(bundle);
 
 	// TODO(queues) support remote wrangler dev
 	if (