Remove es-module-lexer (#7073)

Author: penalosa

.changeset/tasty-elephants-develop.md

@@ -0,0 +1,5 @@
+---
+"wrangler": patch
+---
+
+Internal refactor to remove `es-module-lexer` and support `wrangler types` for Workers with Durable Objects & JSX

packages/wrangler/package.json

@@ -130,7 +130,6 @@
 		"concurrently": "^8.2.2",
 		"devtools-protocol": "^0.0.1182435",
 		"dotenv": "^16.0.0",
-		"es-module-lexer": "^1.3.0",
 		"execa": "^6.1.0",
 		"express": "^4.18.1",
 		"find-up": "^6.3.0",

packages/wrangler/src/__tests__/api/startDevWorker/BundleController.test.ts

@@ -79,6 +79,7 @@ describe("BundleController", () => {
 					define: {},
 					format: "modules",
 					moduleRoot: path.resolve("src"),
+					exports: [],
 				},
 			};
 
@@ -151,6 +152,7 @@ describe("BundleController", () => {
 					define: {},
 					format: "modules",
 					moduleRoot: path.resolve("src"),
+					exports: [],
 				},
 			};
 
@@ -219,6 +221,7 @@ describe("BundleController", () => {
 					define: {},
 					format: "modules",
 					moduleRoot: path.resolve("."),
+					exports: [],
 				},
 			};
 
@@ -295,6 +298,7 @@ describe("BundleController", () => {
 				define: {},
 				format: "modules",
 				moduleRoot: path.resolve("src"),
+				exports: [],
 			},
 		});
 
@@ -353,6 +357,7 @@ describe("BundleController", () => {
 					define: {},
 					format: "modules",
 					moduleRoot: path.resolve("src"),
+					exports: [],
 				},
 			};
 
@@ -400,6 +405,7 @@ describe("BundleController", () => {
 					define: {},
 					format: "modules",
 					moduleRoot: process.cwd(),
+					exports: [],
 				},
 				legacy: {},
 			};
@@ -473,6 +479,7 @@ describe("BundleController", () => {
 					define: {},
 					format: "modules",
 					moduleRoot: process.cwd(),
+					exports: [],
 				},
 			};
 
@@ -518,6 +525,7 @@ describe("BundleController", () => {
 					define: {},
 					format: "modules",
 					moduleRoot: path.resolve("src"),
+					exports: [],
 				},
 			};
 

packages/wrangler/src/__tests__/api/startDevWorker/LocalRuntimeController.test.ts

@@ -92,6 +92,7 @@ function makeEsbuildBundle(testBundle: TestBundle): Bundle {
 			format: "modules",
 			moduleRoot: "/virtual",
 			name: undefined,
+			exports: [],
 		},
 		dependencies: {},
 		sourceMapPath: undefined,
@@ -235,6 +236,7 @@ describe("LocalRuntimeController", () => {
 					format: "modules",
 					moduleRoot: "/virtual",
 					name: undefined,
+					exports: [],
 				},
 				dependencies: {},
 				sourceMapPath: undefined,
@@ -348,6 +350,7 @@ describe("LocalRuntimeController", () => {
 					format: "service-worker",
 					moduleRoot: "/virtual",
 					name: undefined,
+					exports: [],
 				},
 				dependencies: {},
 				sourceMapPath: undefined,

packages/wrangler/src/__tests__/find-additional-modules.test.ts

@@ -42,6 +42,7 @@ describe("traverse module graph", () => {
 				directory: process.cwd(),
 				format: "modules",
 				moduleRoot: process.cwd(),
+				exports: [],
 			},
 			[]
 		);
@@ -77,6 +78,7 @@ describe("traverse module graph", () => {
 				directory: process.cwd(),
 				format: "modules",
 				moduleRoot: process.cwd(),
+				exports: [],
 			},
 			[{ type: type as ConfigModuleRuleType, globs: ["**/*.js"] }]
 		);
@@ -111,6 +113,7 @@ describe("traverse module graph", () => {
 				format: "modules",
 				// The default module root is dirname(file)
 				moduleRoot: path.join(process.cwd(), "./src/nested"),
+				exports: [],
 			},
 			[{ type: "ESModule", globs: ["**/*.js"] }]
 		);
@@ -145,6 +148,7 @@ describe("traverse module graph", () => {
 				format: "modules",
 				// The default module root is dirname(file)
 				moduleRoot: path.join(process.cwd(), "./src"),
+				exports: [],
 			},
 			[{ type: "ESModule", globs: ["**/*.js"] }]
 		);
@@ -179,6 +183,7 @@ describe("traverse module graph", () => {
 				format: "modules",
 				// The default module root is dirname(file)
 				moduleRoot: path.join(process.cwd(), "./src"),
+				exports: [],
 			},
 			[{ type: "ESModule", globs: ["**/*.mjs"] }]
 		);
@@ -213,6 +218,7 @@ describe("traverse module graph", () => {
 				format: "modules",
 				// The default module root is dirname(file)
 				moduleRoot: path.join(process.cwd(), "./src"),
+				exports: [],
 			},
 			[]
 		);
@@ -247,6 +253,7 @@ describe("traverse module graph", () => {
 					format: "modules",
 					// The default module root is dirname(file)
 					moduleRoot: path.join(process.cwd(), "./src"),
+					exports: [],
 				},
 				[
 					{ type: "Text", globs: ["**/*.txt"] },

packages/wrangler/src/__tests__/guess-worker-format.test.ts

@@ -16,7 +16,7 @@ describe("guess worker format", () => {
 			process.cwd(),
 			undefined
 		);
-		expect(guess).toBe("modules");
+		expect(guess.format).toBe("modules");
 	});
 
 	it('should detect a "service-worker" worker', async () => {
@@ -29,7 +29,7 @@ describe("guess worker format", () => {
 			process.cwd(),
 			undefined
 		);
-		expect(guess).toBe("service-worker");
+		expect(guess.format).toBe("service-worker");
 	});
 
 	it('should detect a "service-worker" worker using `typeof module`', async () => {
@@ -39,7 +39,7 @@ describe("guess worker format", () => {
 			process.cwd(),
 			undefined
 		);
-		expect(guess).toBe("service-worker");
+		expect(guess.format).toBe("service-worker");
 	});
 
 	it('should detect a "service-worker" worker using imports', async () => {
@@ -64,7 +64,7 @@ describe("guess worker format", () => {
 			process.cwd(),
 			undefined
 		);
-		expect(guess).toBe("service-worker");
+		expect(guess.format).toBe("service-worker");
 	});
 
 	it("should throw an error when the hint doesn't match the guess (modules - service-worker)", async () => {
@@ -100,7 +100,7 @@ describe("guess worker format", () => {
 			process.cwd(),
 			undefined
 		);
-		expect(guess).toBe("service-worker");
+		expect(guess.format).toBe("service-worker");
 	});
 
 	it("logs a warning when a worker has exports, but not a default one", async () => {
@@ -110,11 +110,26 @@ describe("guess worker format", () => {
 			process.cwd(),
 			undefined
 		);
-		expect(guess).toBe("service-worker");
+		expect(guess.format).toBe("service-worker");
 		expect(std.warn).toMatchInlineSnapshot(`
       "▲ [WARNING] The entrypoint index.ts has exports like an ES Module, but hasn't defined a default export like a module worker normally would. Building the worker using \\"service-worker\\" format...
 
       "
     `);
 	});
+
+	it("should list exports", async () => {
+		await writeFile(
+			"./index.ts",
+			"export default {}; export const Hello ='world'"
+		);
+		// Note that this isn't actually a valid worker, because it's missing
+		// a fetch handler. Regardless, our heuristic is simply to check for exports.
+		const guess = await guessWorkerFormat(
+			path.join(process.cwd(), "./index.ts"),
+			process.cwd(),
+			undefined
+		);
+		expect(guess.exports).toStrictEqual(["Hello", "default"]);
+	});
 });

packages/wrangler/src/__tests__/navigator-user-agent.test.ts

@@ -107,6 +107,7 @@ describe("defineNavigatorUserAgent is respected", () => {
 				directory: process.cwd(),
 				format: "modules",
 				moduleRoot: path.dirname(path.resolve("src/index.js")),
+				exports: [],
 			},
 			path.resolve("dist"),
 			{
@@ -168,6 +169,7 @@ describe("defineNavigatorUserAgent is respected", () => {
 				directory: process.cwd(),
 				format: "modules",
 				moduleRoot: path.dirname(path.resolve("src/index.js")),
+				exports: [],
 			},
 			path.resolve("dist"),
 			{

packages/wrangler/src/api/startDevWorker/BundlerController.ts

@@ -77,6 +77,7 @@ export class BundlerController extends Controller<BundlerControllerEventMap> {
 				directory: config.directory,
 				format: config.build.format,
 				moduleRoot: config.build.moduleRoot,
+				exports: config.build.exports,
 			};
 
 			const entryDirectory = path.dirname(config.entrypoint);
@@ -214,6 +215,7 @@ export class BundlerController extends Controller<BundlerControllerEventMap> {
 			directory: config.directory,
 			format: config.build.format,
 			moduleRoot: config.build.moduleRoot,
+			exports: config.build.exports,
 		};
 		const { bindings } = await convertBindingsToCfWorkerInitBindings(
 			config.bindings

packages/wrangler/src/api/startDevWorker/ConfigController.ts

@@ -284,6 +284,7 @@ async function resolveConfig(
 			jsxFactory: input.build?.jsxFactory || config.jsx_factory,
 			jsxFragment: input.build?.jsxFragment || config.jsx_fragment,
 			tsconfig: input.build?.tsconfig ?? config.tsconfig,
+			exports: entry.exports,
 		},
 		dev: await resolveDevConfig(config, input),
 		legacy: {

packages/wrangler/src/api/startDevWorker/types.ts

@@ -179,6 +179,7 @@ export type StartDevWorkerOptions = Omit<StartDevWorkerInput, "assets"> & {
 		moduleRules: Rule[];
 		define: Record<string, string>;
 		additionalModules: CfModule[];
+		exports: string[];
 
 		processEntrypoint: boolean;
 	};

packages/wrangler/src/deployment-bundle/entry.ts

@@ -31,6 +31,9 @@ export type Entry = {
 	 * A worker's name
 	 */
 	name?: string | undefined;
+
+	/** Export from a Worker's entrypoint */
+	exports: string[];
 };
 
 /**
@@ -78,7 +81,7 @@ export async function getEntry(
 	}
 	await runCustomBuild(paths.absolutePath, paths.relativePath, config.build);
 
-	const format = await guessWorkerFormat(
+	const { format, exports } = await guessWorkerFormat(
 		paths.absolutePath,
 		directory,
 		args.format ?? config.build?.upload?.format,
@@ -119,6 +122,7 @@ export async function getEntry(
 		moduleRoot:
 			args.moduleRoot ?? config.base_dir ?? path.dirname(paths.absolutePath),
 		name: config.name ?? "worker",
+		exports,
 	};
 }
 

packages/wrangler/src/deployment-bundle/guess-worker-format.ts

@@ -19,7 +19,7 @@ export default async function guessWorkerFormat(
 	entryWorkingDirectory: string,
 	hint: CfScriptFormat | undefined,
 	tsconfig?: string | undefined
-): Promise<CfScriptFormat> {
+): Promise<{ format: CfScriptFormat; exports: string[] }> {
 	const parsedEntryPath = path.parse(entryFile);
 	if (parsedEntryPath.ext == ".py") {
 		logger.warn(
@@ -28,7 +28,7 @@ export default async function guessWorkerFormat(
 				entryFile
 			)} defines a Python worker, support for Python workers is currently experimental. Python workers with a requirements.txt file can only be run locally and cannot be deployed.`
 		);
-		return "modules";
+		return { format: "modules", exports: [] };
 	}
 
 	const result = await esbuild.build({
@@ -77,5 +77,5 @@ export default async function guessWorkerFormat(
 			}
 		}
 	}
-	return guessedWorkerFormat;
+	return { format: guessedWorkerFormat, exports };
 }

packages/wrangler/src/dev/dev.tsx

@@ -372,6 +372,7 @@ function DevSession(props: DevSessionProps) {
 				nodejsCompatMode: props.nodejsCompatMode ?? null,
 				format: props.entry.format,
 				moduleRoot: props.entry.moduleRoot,
+				exports: props.entry.exports,
 			},
 			dev: {
 				auth: async () => {

packages/wrangler/src/dev/miniflare.ts

@@ -1,7 +1,6 @@
 import assert from "node:assert";
 import { randomUUID } from "node:crypto";
 import path from "node:path";
-import * as esmLexer from "es-module-lexer";
 import {
 	CoreHeaders,
 	HttpOptions_Style,
@@ -259,14 +258,6 @@ export function buildLog(): Log {
 	return new WranglerLog(level, { prefix: "wrangler-UserWorker" });
 }
 
-async function getEntrypointNames(entrypointSource: string) {
-	await esmLexer.init;
-	const [_imports, exports] = esmLexer.parse(entrypointSource);
-	// TODO(soon): support `export * from "...";` with `--no-bundle`. Without
-	//  `--no-bundle`, `esbuild` will bundle these, so they'll be picked up here.
-	return exports.map(({ n }) => n);
-}
-
 async function buildSourceOptions(
 	config: Omit<ConfigBundle, "rules">
 ): Promise<{ sourceOptions: SourceOptions; entrypointNames: string[] }> {
@@ -285,9 +276,7 @@ async function buildSourceOptions(
 					config.bundle.modules
 				);
 
-		const entrypointNames = isPython
-			? []
-			: await getEntrypointNames(entrypointSource);
+		const entrypointNames = isPython ? [] : config.bundle.entry.exports;
 
 		const modulesRoot = path.dirname(scriptPath);
 		const sourceOptions: SourceOptions = {

packages/wrangler/src/pages/functions/buildPlugin.ts

@@ -31,6 +31,7 @@ export function buildPluginFromFunctions({
 		directory: functionsDirectory,
 		format: "modules",
 		moduleRoot: functionsDirectory,
+		exports: [],
 	};
 	const moduleCollector = createModuleCollector({
 		entry,