fix: skip unwanted files and directories when publishing site assets

In keeping with Wrangler 1, we now skip node_modules and hidden files and directories.

An exception is made for `.well-known`. See https://datatracker.ietf.org/doc/html/rfc8615.

The tests also prove that the asset uploader will walk directories in general.

Author: petebacondarwin

.changeset/violet-wasps-reply.md

@@ -0,0 +1,11 @@
+---
+"wrangler": patch
+---
+
+fix: skip unwanted files and directories when publishing site assets
+
+In keeping with Wrangler 1, we now skip node_modules and hidden files and directories.
+
+An exception is made for `.well-known`. See https://datatracker.ietf.org/doc/html/rfc8615.
+
+The tests also prove that the asset uploader will walk directories in general.

packages/wrangler/src/__tests__/publish.test.ts

@@ -383,6 +383,94 @@ describe("publish", () => {
       expect(error).toMatchInlineSnapshot(`undefined`);
     });
 
+    it("should walk directories except node_modules", async () => {
+      const assets = [
+        {
+          filePath: "assets/directory-1/file-1.txt",
+          content: "Content of file-1",
+        },
+        {
+          filePath: "assets/node_modules/file-2.txt",
+          content: "Content of file-2",
+        },
+      ];
+      const kvNamespace = {
+        title: "__test-name_sites_assets",
+        id: "__test-name_sites_assets-id",
+      };
+      writeWranglerToml("./index.js", "assets");
+      writeEsmWorkerSource();
+      writeAssets(assets);
+      mockUploadWorkerRequest();
+      mockSubDomainRequest();
+      mockListKVNamespacesRequest(kvNamespace);
+      mockKeyListRequest(kvNamespace.id, []);
+      // Only expect file-1 to be uploaded
+      mockUploadAssetsToKVRequest(kvNamespace.id, assets.slice(0, 1));
+      const { stdout, stderr, error } = await runWrangler("publish");
+
+      expect(stripTimings(stdout)).toMatchInlineSnapshot(`
+        "reading assets/directory-1/file-1.txt...
+        uploading as assets/directory-1/file-1.2ca234f380.txt...
+        Uploaded
+        test-name
+        (TIMINGS)
+        Deployed
+        test-name
+        (TIMINGS)
+         
+        test-name.test-sub-domain.workers.dev"
+      `);
+      expect(stderr).toMatchInlineSnapshot(`""`);
+      expect(error).toMatchInlineSnapshot(`undefined`);
+    });
+
+    it("should skip hidden files and directories except `.well-known`", async () => {
+      const assets = [
+        {
+          filePath: "assets/.hidden-file.txt",
+          content: "Content of hidden-file",
+        },
+        {
+          filePath: "assets/.hidden/file-1.txt",
+          content: "Content of file-1",
+        },
+        {
+          filePath: "assets/.well-known/file-2.txt",
+          content: "Content of file-2",
+        },
+      ];
+      const kvNamespace = {
+        title: "__test-name_sites_assets",
+        id: "__test-name_sites_assets-id",
+      };
+      writeWranglerToml("./index.js", "assets");
+      writeEsmWorkerSource();
+      writeAssets(assets);
+      mockUploadWorkerRequest();
+      mockSubDomainRequest();
+      mockListKVNamespacesRequest(kvNamespace);
+      mockKeyListRequest(kvNamespace.id, []);
+      // Only expect file-2 to be uploaded
+      mockUploadAssetsToKVRequest(kvNamespace.id, assets.slice(2));
+      const { stdout, stderr, error } = await runWrangler("publish");
+
+      expect(stripTimings(stdout)).toMatchInlineSnapshot(`
+        "reading assets/.well-known/file-2.txt...
+        uploading as assets/.well-known/file-2.5938485188.txt...
+        Uploaded
+        test-name
+        (TIMINGS)
+        Deployed
+        test-name
+        (TIMINGS)
+         
+        test-name.test-sub-domain.workers.dev"
+      `);
+      expect(stderr).toMatchInlineSnapshot(`""`);
+      expect(error).toMatchInlineSnapshot(`undefined`);
+    });
+
     it("should error if the asset is over 25Mb", async () => {
       const assets = [
         {

packages/wrangler/src/sites.tsx

@@ -6,11 +6,26 @@ import { fetchResult } from "./cfetch";
 import type { AssetPaths } from "./config";
 import { listNamespaceKeys, listNamespaces, putBulkKeyValue } from "./kv";
 
+/** Paths to always ignore. */
+const ALWAYS_IGNORE = ["node_modules"];
+const HIDDEN_FILES_TO_INCLUDE = [
+  ".well-known", // See https://datatracker.ietf.org/doc/html/rfc8615
+];
+
 async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   const files = await readdir(dirPath, { withFileTypes: true });
   for (const file of files) {
-    // TODO: always ignore `node_modules`
-    // TODO: ignore hidden files (starting with .) but not .well-known??
+    // Skip files that we never want to process.
+    if (ALWAYS_IGNORE.some((p) => file.name === p)) {
+      continue;
+    }
+    // Skip hidden files (starting with .) except for some special ones
+    if (
+      file.name.startsWith(".") &&
+      !HIDDEN_FILES_TO_INCLUDE.some((p) => file.name === p)
+    ) {
+      continue;
+    }
     // TODO: follow symlinks??
     if (file.isDirectory()) {
       yield* await getFilesInFolder(path.join(dirPath, file.name));