feat: Create very basic Asset Worker and plumb it into wrangler dev

This commit does the ground work needed in order to
add Assets support for Workers in `wrangler dev`.
Amongst others, it implements the following:

- it creates a new package called `workers-shared`
that hosts the `Asset Server Worker`, and the
`Router Worker`in the future
- it scaffolds the `Asset Server Worker` in some very
basic form, with basic configuration. Further behaviour
implementation will follow in a subsequent PR
- it does the ground work of plumbing ASW into Miniflare

.changeset/orange-icons-build.md

@@ -0,0 +1,41 @@
+---
+"@cloudflare/workers-shared": minor
+"wrangler": minor
+---
+
+feat: Create very basic Asset Server Worker and plumb it into `wrangler dev`
+
+These changes do the ground work needed in order to add Assets support for Workers in `wrangler dev`. They introduce the following components, and the following conceptual "client pipeline":
+
+<!-- prettier-ignore-start -->
+                   |
+              wrangler dev
+                   |              workerd
++------------------+---------------------+
+|                  |                     |
+|      +-----------v--------------+      |
+|      |                          |      |
+|      |     NOOP Entry Worker    |      |
+|      |                          |      |
+|      +------------+-------------+      |
+|                   |                    |
+|           env.ASSET_SERVER             |
+|                   |                    |
+|      +------------v-------------+      |
+|      |                          |      |
+|      |    Asset Server Worker   |      |
+|      |                          |      |
+|      +------------+-------------+      |
+|                   |                    |
++-------------------+--------------------+
+                    |
+                    v
+    	"Hello world from Asset Server"
+
+<!-- prettier-ignore-end -->
+
+Based on the above architectural diagram, this body of work implements the following:
+
+- it creates a new package called `workers-shared` that hosts the `Asset Server Worker`, and the `Router Worker`in the future
+- it scaffolds the `Asset Server Worker` in some very basic form, with basic configuration. Further behaviour implementation will follow in a subsequent PR
+- it does the ground work of plumbing ASW into Miniflare

fixtures/workers-with-assets/README.md

@@ -0,0 +1,23 @@
+# workers-with-assets
+
+`workers-with-assets` is a test fixture that showcases Workers with Assets. This particular fixture sets up an assets-only Workers project.
+
+## dev
+
+To start a dev session you can run
+
+```
+wrangler dev
+```
+
+or
+
+```
+wrangler dev --experimental-assets=./public
+```
+
+## Run tests
+
+```
+npm run test
+```

fixtures/workers-with-assets/package.json

@@ -0,0 +1,20 @@
+{
+	"name": "assets-worker",
+	"private": true,
+	"scripts": {
+		"check:type": "tsc",
+		"dev": "npx wrangler dev",
+		"test:ci": "vitest run",
+		"test:watch": "vitest",
+		"type:tests": "tsc -p ./tests/tsconfig.json"
+	},
+	"devDependencies": {
+		"@cloudflare/workers-tsconfig": "workspace:*",
+		"@cloudflare/workers-types": "^4.20240725.0",
+		"undici": "^5.28.4",
+		"wrangler": "workspace:*"
+	},
+	"volta": {
+		"extends": "../../package.json"
+	}
+}

fixtures/workers-with-assets/public/index.html

@@ -0,0 +1 @@
+<h1>Hello Workers + Assets World!</h1>

fixtures/workers-with-assets/tests/index.test.ts

@@ -0,0 +1,26 @@
+import { resolve } from "node:path";
+import { fetch } from "undici";
+import { afterAll, beforeAll, describe, it } from "vitest";
+import { runWranglerDev } from "../../shared/src/run-wrangler-long-lived";
+
+describe("[Workers + Assets] `wrangler dev`", () => {
+	let ip: string, port: number, stop: (() => Promise<unknown>) | undefined;
+
+	beforeAll(async () => {
+		({ ip, port, stop } = await runWranglerDev(resolve(__dirname, ".."), [
+			"--port=0",
+			"--inspector-port=0",
+		]));
+	});
+
+	afterAll(async () => {
+		await stop?.();
+	});
+
+	it("renders ", async ({ expect }) => {
+		const response = await fetch(`http://${ip}:${port}/`);
+		const text = await response.text();
+		expect(response.status).toBe(200);
+		expect(text).toContain(`Hello from Asset Server Worker 🚀`);
+	});
+});

fixtures/workers-with-assets/tests/tsconfig.json

@@ -0,0 +1,7 @@
+{
+	"extends": "@cloudflare/workers-tsconfig/tsconfig.json",
+	"compilerOptions": {
+		"types": ["node"]
+	},
+	"include": ["**/*.ts", "../../../node-types.d.ts"]
+}

fixtures/workers-with-assets/tsconfig.json

@@ -0,0 +1,13 @@
+{
+	"compilerOptions": {
+		"target": "ES2020",
+		"esModuleInterop": true,
+		"module": "CommonJS",
+		"lib": ["ES2020"],
+		"types": ["node"],
+		"moduleResolution": "node",
+		"noEmit": true,
+		"skipLibCheck": true
+	},
+	"include": ["tests", "../../node-types.d.ts"]
+}

fixtures/workers-with-assets/wrangler.toml

@@ -0,0 +1,5 @@
+name = "assets-worker"
+compatibility_date = "2024-01-01"
+
+[experimental_assets]
+directory = "./public"

packages/workers-shared/README.md

@@ -0,0 +1,13 @@
+# `@cloudflare/workers-shared`
+
+This is a package that is used at Cloudflare to power some internal features of [Cloudflare Workers](https://developers.cloudflare.com/workers/), as well as their open-source equivalents here in workers-sdk and Wrangler.
+
+## `asset-server`
+
+The Asset Server Worker.
+
+For more details please refer to the dedicated README file.
+
+> [!NOTE]
+> Since code in this package is used by the Workers infrastructure, it is important that PRs are given careful review with regards to how they could cause a failure in production.
+> Ideally, there should be comprehensive tests for changes being made to give extra confidence about the behavior.

packages/workers-shared/asset-server-worker/README.md

@@ -0,0 +1,3 @@
+# `asset-server`
+
+The Asset Server is a [Cloudflare Worker](https://developers.cloudflare.com/workers/) that is responsible of serving assets for Workers deployed on the edge, that contain static assets as well.

packages/workers-shared/asset-server-worker/src/index.ts

@@ -0,0 +1,5 @@
+export default {
+	async fetch(request, env) {
+		return new Response("Hello from Asset Server Worker 🚀");
+	},
+};

packages/workers-shared/asset-server-worker/wrangler.toml

@@ -0,0 +1,12 @@
+##
+# Configuration file for the Asset Server Worker
+#
+# Please note that wrangler has a dependency on this file, and will
+# attempt to read it as part of setting up a new Miniflare instance
+# in developemnt mode. We should ensure that any configuration changes
+# to this file are persisted in wrangler as well, when necessary.
+# (see packages/wrangler/src/dev/miniflare.ts -> buildMiniflareOptions())
+##
+name = "asset-server"
+main = "src/index.ts"
+compatibility_date = "2024-07-31"

packages/workers-shared/package.json

@@ -0,0 +1,17 @@
+{
+	"name": "@cloudflare/workers-shared",
+	"version": "0.0.1",
+	"description": "Package that is used at Cloudflare to power some internal features of Cloudflare Workers.",
+	"scripts": {
+		"build": "pnpm run clean && pnpm run bundle:asset-server",
+		"bundle:asset-server": "esbuild asset-server-worker/src/index.ts --format=esm --bundle --outfile=dist/asset-server-worker.mjs --sourcemap=external",
+		"clean": "rimraf dist",
+		"dev": "pnpm run clean && concurrently -n bundle:asset-server -c blue \"pnpm run bundle:asset-server --watch\""
+	},
+	"devDependencies": {
+		"concurrently": "^8.2.2",
+		"esbuild": "0.17.19",
+		"rimraf": "^6.0.1",
+		"typescript": "^5.5.4"
+	}
+}

packages/workers-shared/turbo.json

@@ -0,0 +1,9 @@
+{
+	"$schema": "http://turbo.build/schema.json",
+	"extends": ["//"],
+	"pipeline": {
+		"build": {
+			"outputs": ["dist/**"]
+		}
+	}
+}

packages/wrangler/package.json

@@ -92,6 +92,7 @@
 		"@cloudflare/eslint-config-worker": "workspace:*",
 		"@cloudflare/pages-shared": "workspace:^",
 		"@cloudflare/types": "^6.18.4",
+		"@cloudflare/workers-shared": "workspace:*",
 		"@cloudflare/workers-tsconfig": "workspace:*",
 		"@cloudflare/workers-types": "^4.20240725.0",
 		"@cspotcode/source-map-support": "0.8.1",

packages/wrangler/scripts/deps.ts

@@ -22,6 +22,7 @@ export const EXTERNAL_DEPENDENCIES = [
 	// and read when we are bundling the worker application
 	"unenv",
 	"workerd/worker.mjs",
+	"@cloudflare/workers-shared",
 ];
 
 const pathToPackageJson = path.resolve(__dirname, "..", "package.json");

packages/wrangler/src/__tests__/deploy.test.ts

@@ -4363,7 +4363,7 @@ addEventListener('fetch', event => {});`
 			});
 			writeWorkerSource();
 			mockUploadWorkerRequest({
-				expectedMainModule: "no-op-worker.js",
+				expectedMainModule: "no-op-assets-worker.js",
 			});
 			mockSubDomainRequest();
 			await runWrangler("deploy");

packages/wrangler/src/__tests__/dev.test.tsx

@@ -1303,7 +1303,7 @@ describe("wrangler dev", () => {
 			await expect(
 				runWrangler("dev --legacy-assets abc --site xyz")
 			).rejects.toThrowErrorMatchingInlineSnapshot(
-				`[Error: Cannot use Assets and Workers Sites in the same Worker.]`
+				`[Error: Cannot use Legacy Assets and Workers Sites in the same Worker.]`
 			);
 		});
 
@@ -1318,7 +1318,7 @@ describe("wrangler dev", () => {
 			await expect(
 				runWrangler("dev --legacy-assets abc")
 			).rejects.toThrowErrorMatchingInlineSnapshot(
-				`[Error: Cannot use Assets and Workers Sites in the same Worker.]`
+				`[Error: Cannot use Legacy Assets and Workers Sites in the same Worker.]`
 			);
 		});
 
@@ -1331,7 +1331,7 @@ describe("wrangler dev", () => {
 			await expect(
 				runWrangler("dev --site xyz")
 			).rejects.toThrowErrorMatchingInlineSnapshot(
-				`[Error: Cannot use Assets and Workers Sites in the same Worker.]`
+				`[Error: Cannot use Legacy Assets and Workers Sites in the same Worker.]`
 			);
 		});
 
@@ -1347,7 +1347,7 @@ describe("wrangler dev", () => {
 			await expect(
 				runWrangler("dev --legacy-assets abc")
 			).rejects.toThrowErrorMatchingInlineSnapshot(
-				`[Error: Cannot use Assets and Workers Sites in the same Worker.]`
+				`[Error: Cannot use Legacy Assets and Workers Sites in the same Worker.]`
 			);
 		});
 
@@ -1441,11 +1441,11 @@ describe("wrangler dev", () => {
 			await expect(
 				runWrangler("dev")
 			).rejects.toThrowErrorMatchingInlineSnapshot(
-				`[Error: Cannot use Assets and Workers Sites in the same Worker.]`
+				`[Error: Cannot use Experimental Assets and Workers Sites in the same Worker.]`
 			);
 		});
 
-		it("should error if --experimental-assets and config.site are used together", async () => {
+		it("should error if config.site and --experimental-assets are used together", async () => {
 			writeWranglerToml({
 				main: "./index.js",
 				site: {
@@ -1457,7 +1457,74 @@ describe("wrangler dev", () => {
 			await expect(
 				runWrangler("dev --experimental-assets assets")
 			).rejects.toThrowErrorMatchingInlineSnapshot(
-				`[Error: Cannot use Assets and Workers Sites in the same Worker.]`
+				`[Error: Cannot use Experimental Assets and Workers Sites in the same Worker.]`
+			);
+		});
+
+		it("should error if config.experimental_assets and config.legacy_assets are used together", async () => {
+			writeWranglerToml({
+				main: "./index.js",
+				experimental_assets: { directory: "assets" },
+				legacy_assets: {
+					bucket: "xyz",
+					include: [],
+					exclude: [],
+					browser_TTL: undefined,
+					serve_single_page_app: true,
+				},
+			});
+			fs.writeFileSync("index.js", `export default {};`);
+			fs.openSync("assets", "w");
+			await expect(
+				runWrangler("dev")
+			).rejects.toThrowErrorMatchingInlineSnapshot(
+				`[Error: Cannot use Legacy Assets and Experimental Assets in the same Worker.]`
+			);
+		});
+
+		it("should error if --experimental-assets and --legacy-assets are used together", async () => {
+			fs.writeFileSync("index.js", `export default {};`);
+			fs.openSync("assets", "w");
+			await expect(
+				runWrangler("dev --experimental-assets assets --legacy-assets assets")
+			).rejects.toThrowErrorMatchingInlineSnapshot(
+				`[Error: Cannot use Legacy Assets and Experimental Assets in the same Worker.]`
+			);
+		});
+
+		it("should error if --experimental-assets and config.legacy_assets are used together", async () => {
+			writeWranglerToml({
+				main: "./index.js",
+				legacy_assets: {
+					bucket: "xyz",
+					include: [],
+					exclude: [],
+					browser_TTL: undefined,
+					serve_single_page_app: true,
+				},
+			});
+			fs.writeFileSync("index.js", `export default {};`);
+			fs.openSync("assets", "w");
+			await expect(
+				runWrangler("dev --experimental-assets assets")
+			).rejects.toThrowErrorMatchingInlineSnapshot(
+				`[Error: Cannot use Legacy Assets and Experimental Assets in the same Worker.]`
+			);
+		});
+
+		it("should error if config.experimental_assets and --legacy-assets are used together", async () => {
+			writeWranglerToml({
+				main: "./index.js",
+				experimental_assets: {
+					directory: "xyz",
+				},
+			});
+			fs.writeFileSync("index.js", `export default {};`);
+			fs.openSync("xyz", "w");
+			await expect(
+				runWrangler("dev --legacy-assets xyz")
+			).rejects.toThrowErrorMatchingInlineSnapshot(
+				`[Error: Cannot use Legacy Assets and Experimental Assets in the same Worker.]`
 			);
 		});
 
@@ -1475,7 +1542,7 @@ describe("wrangler dev", () => {
 			);
 		});
 
-		it("should error if directory specified by 'experimental_assets' configuration key does not exist", async () => {
+		it("should error if directory specified by '[experimental_assets]' configuration key does not exist", async () => {
 			writeWranglerToml({
 				main: "./index.js",
 				experimental_assets: {

packages/wrangler/src/api/dev.ts

@@ -189,7 +189,7 @@ export async function unstable_dev(
 		localProtocol: options?.localProtocol,
 		httpsKeyPath: options?.httpsKeyPath,
 		httpsCertPath: options?.httpsCertPath,
-		experimentalAssets: options?.experimentalAssets,
+		experimentalAssets: undefined,
 		legacyAssets: options?.legacyAssets,
 		site: options?.site, // Root folder of static assets for Workers Sites
 		siteInclude: options?.siteInclude, // Array of .gitignore-style patterns that match file or directory names from the sites directory. Only matched items will be uploaded.

packages/wrangler/src/api/startDevWorker/ConfigController.ts

@@ -204,6 +204,10 @@ async function resolveConfig(
 			legacyAssets: Boolean(legacyAssets),
 			script: input.entrypoint,
 			moduleRoot: input.build?.moduleRoot,
+			// getEntry only needs to know if experimental_assets was specified.
+			// The actualy value is not relevant here, which is why not passing
+			// the entire ExperimentalAssets object is fine.
+			experimentalAssets: input?.experimental?.assets?.directory,
 		},
 		config,
 		"dev"
@@ -257,6 +261,9 @@ async function resolveConfig(
 			capnp: input.unsafe?.capnp ?? unsafe?.capnp,
 			metadata: input.unsafe?.metadata ?? unsafe?.metadata,
 		},
+		experimental: {
+			assets: input?.experimental?.assets,
+		},
 	} satisfies StartDevWorkerOptions;
 
 	if (resolved.legacy.legacyAssets && resolved.legacy.site) {