Web Worker support for esbuild flow

.changeset/fifty-goats-shave.md

@@ -0,0 +1,5 @@
+---
+"modular-scripts": minor
+---
+
+Web Worker support and docs for esbuild.

docs/building-apps/index.md

@@ -0,0 +1,7 @@
+---
+has_children: true
+title: Building your Apps
+nav_order: 500
+---
+
+# Building your Apps

docs/building-apps/web workers.md

@@ -0,0 +1,51 @@
+---
+parent: Building your Apps
+title: Adding web workers
+---
+
+esbuild {: .label .label-yellow }
+
+It is possible to add
+[web workers](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers)
+to your application just by writing them as normal typescript modules. There are
+some rules to follow to write a worker:
+
+- Your worker module must follow the `<filename>.worker.[ts|js|jsx|tsx]` name
+  pattern for Modular to build it as a worker.
+- Worker extension must be explicitly included in the import statement for the
+  typechecker to correctly type it. `import Worker from './my.worker.ts'` is ok,
+  `import Worker from './my.worker'` is not.
+- A worker can only `import` other modules. Trying to `import` files that have a
+  different extension than `[ts|js|jsx|tsx]` will trigger a build error.
+- If a worker doesn't `import` any other module, it should `export {}` or
+  `export default {}` to avoid being marked as global module by the type
+  checker.
+
+Importing a worker will return a `Class` that, when instantiated, returns a
+worker instance. For example:
+
+```ts
+// ./index.ts
+import DateFormatterCls from './worker/dateFormatter.worker.ts';
+
+// Instantiate the worker
+const worker = new DateFormatterCls();
+
+worker.current.onmessage = (message) =>
+  console.log('Received a message from worker', message.data);
+worker.postMessage(new Date.now());
+```
+
+```ts
+// ./worker/dateFormatter.worker.ts
+import { wait, format } from '../utils/date-utils';
+// These imports are allowed because they refer to other modules
+
+globalThis.self.onmessage = async (message: { data: number }) => {
+  postMessage(`Hello there. Processing date...`);
+  // Simulate work
+  await wait(500);
+  // Send back the formatter date
+  postMessage(`Date is: ${format(message.data)}`);
+};
+```

packages/modular-scripts/react-app-env.d.ts

@@ -11,6 +11,14 @@ declare namespace NodeJS {
   }
 }
 
+declare module '*.worker.ts' {
+  class WebWorkerClass extends Worker {
+    constructor();
+  }
+
+  export default WebWorkerClass;
+}
+
 declare module '*.avif' {
   const src: string;
   export default src;

packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.ts

@@ -0,0 +1,3 @@
+globalThis.self.postMessage("I'm alive!");
+
+export {};

packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/index.ts

@@ -0,0 +1,7 @@
+import WorkerCls from './alive.worker.ts';
+
+const worker = new WorkerCls();
+
+worker.addEventListener('message', (event: MessageEvent<string>) => {
+  console.log(`Received message from worker: ${event.data}`);
+});

packages/modular-scripts/src/__tests__/esbuild-scripts/__snapshots__/workerPlugin.test.ts.snap

@@ -0,0 +1,31 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`WHEN running esbuild with the workerFactoryPlugin WHEN there's a url import SHOULD ouput the correct alive.worker-[hash].ts file 1`] = `
+"// packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.ts
+globalThis.self.postMessage(\\"I'm alive!\\");
+"
+`;
+
+exports[`WHEN running esbuild with the workerFactoryPlugin WHEN there's a url import SHOULD ouput the correct index.js 1`] = `
+"// worker-url:packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.js
+var alive_worker_default = \\"./alive.worker-T4TLN6IN.js\\";
+
+// web-worker:packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.js
+var workerPath = new URL(alive_worker_default, import.meta.url);
+var importSrc = 'import \\"' + workerPath + '\\";';
+var blob = new Blob([importSrc], {
+  type: \\"text/javascript\\"
+});
+var alive_worker_default2 = class {
+  constructor() {
+    return new Worker(URL.createObjectURL(blob), { type: \\"module\\" });
+  }
+};
+
+// packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/index.ts
+var worker = new alive_worker_default2();
+worker.addEventListener(\\"message\\", (event) => {
+  console.log(\`Received message from worker: \${event.data}\`);
+});
+"
+`;

packages/modular-scripts/src/__tests__/esbuild-scripts/workerPlugin.test.ts

@@ -0,0 +1,64 @@
+import * as esbuild from 'esbuild';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import * as tmp from 'tmp';
+import tree from 'tree-view-for-tests';
+import webworkerPlugin from '../../esbuild-scripts/plugins/workerFactoryPlugin';
+import getModularRoot from '../../utils/getModularRoot';
+
+describe('WHEN running esbuild with the workerFactoryPlugin', () => {
+  describe("WHEN there's a url import", () => {
+    let tmpDir: tmp.DirResult;
+    let result: esbuild.BuildResult;
+    let outdir: string;
+
+    beforeAll(async () => {
+      tmpDir = tmp.dirSync();
+      outdir = path.join(tmpDir.name, 'output');
+      result = await esbuild.build({
+        entryPoints: [
+          path.join(__dirname, '__fixtures__', 'worker-plugin', 'index.ts'),
+        ],
+        plugins: [webworkerPlugin()],
+        outdir,
+        sourceRoot: getModularRoot(),
+        bundle: true,
+        splitting: true,
+        format: 'esm',
+        target: 'es2021',
+      });
+    });
+
+    afterAll(async () => {
+      await fs.emptyDir(tmpDir.name);
+      tmpDir.removeCallback();
+    });
+
+    it('SHOULD be successful', () => {
+      expect(result.errors).toEqual([]);
+      expect(result.warnings).toEqual([]);
+    });
+
+    it('SHOULD have the correct output structure', () => {
+      expect(tree(outdir)).toMatchInlineSnapshot(`
+        "output
+        ├─ alive.worker-T4TLN6IN.js #y0mybi
+        └─ index.js #1kx9oa0"
+      `);
+    });
+
+    it('SHOULD ouput the correct index.js', () => {
+      let content = String(fs.readFileSync(path.join(outdir, 'index.js')));
+      content = content.replaceAll(getModularRoot(), '');
+      expect(content).toMatchSnapshot();
+    });
+
+    it('SHOULD ouput the correct alive.worker-[hash].ts file', () => {
+      let content = String(
+        fs.readFileSync(path.join(outdir, 'alive.worker-T4TLN6IN.js')),
+      );
+      content = content.replaceAll(getModularRoot(), '');
+      expect(content).toMatchSnapshot();
+    });
+  });
+});

packages/modular-scripts/src/esbuild-scripts/config/createEsbuildConfig.ts

@@ -8,15 +8,18 @@ import * as logger from '../../utils/logger';
 
 import moduleScopePlugin from '../plugins/moduleScopePlugin';
 import svgrPlugin from '../plugins/svgr';
+import workerFactoryPlugin from '../plugins/workerFactoryPlugin';
 
 export default function createEsbuildConfig(
   paths: Paths,
   config: Partial<esbuild.BuildOptions> = {},
 ): esbuild.BuildOptions {
   const { plugins: configPlugins, ...partialConfig } = config;
+
   const plugins: esbuild.Plugin[] = [
     moduleScopePlugin(paths),
     svgrPlugin(),
+    workerFactoryPlugin(),
   ].concat(configPlugins || []);
 
   const define = Object.assign(

packages/modular-scripts/src/esbuild-scripts/plugins/extensionAllowList.ts

@@ -0,0 +1,45 @@
+import * as esbuild from 'esbuild';
+import * as path from 'path';
+
+// This plugin resolves all the files in the project and excepts if one of the extensions is not in the allow list
+// Please note that implicit (empty) extensions in the importer are always valid.
+
+interface ExtensionPluginConf {
+  allowedExtensions?: string[];
+  reason?: string;
+}
+
+function createExtensionAllowlistPlugin({
+  reason,
+  allowedExtensions = ['.js', '.jsx', '.ts', '.tsx'],
+}: ExtensionPluginConf): esbuild.Plugin {
+  return {
+    name: 'extension-allow-list-plugin',
+    setup(build) {
+      // No lookbehind in Go regexp; need to look at all the files and do the check manually.
+      build.onResolve({ filter: /.*/ }, (args) => {
+        // Extract the extension; if not in the allow list, return an error.
+        const extension = path.extname(args.path);
+        if (extension && !allowedExtensions.includes(extension)) {
+          const errorReason = reason ? ` Reason: ${reason}` : '';
+          return {
+            errors: [
+              {
+                pluginName: 'extension-allow-list-plugin',
+                text: `Extension not allowed`,
+                detail: `Extension for file "${args.path}", imported by "${
+                  args.importer
+                }", is not allowed. Permitted extensions are: ${JSON.stringify(
+                  allowedExtensions,
+                )}.${errorReason}`,
+              },
+            ],
+          };
+        }
+        return undefined;
+      });
+    },
+  };
+}
+
+export default createExtensionAllowlistPlugin;

packages/modular-scripts/src/esbuild-scripts/plugins/workerFactoryPlugin.ts

@@ -0,0 +1,117 @@
+import * as esbuild from 'esbuild';
+import * as path from 'path';
+import getModularRoot from '../../utils/getModularRoot';
+import createExtensionAllowlistPlugin from './extensionAllowList';
+
+// This plugin builds Web Workers on the fly and exports them to use like worker-loader for Webpack 4: https://v4.webpack.js.org/loaders/worker-loader/
+// The workers are not inlined, a new file is generated in the bundle. Only files *imported* with the *.worker.[jt]sx pattern are matched.
+// The workers are trampolined to avoid CORS errors.
+// This will be deprecated in the future when esbuild supports the Worker signature: see https://github.com/evanw/esbuild/issues/312
+// And will probably end up being compatible with Webpack 5 support https://webpack.js.org/guides/web-workers
+
+function createPlugin(): esbuild.Plugin {
+  const plugin: esbuild.Plugin = {
+    name: 'worker-factory-plugin',
+    setup(build) {
+      // This stores built workers for later use
+      const workerBuildCache: Map<string, esbuild.BuildResult> = new Map();
+
+      build.onResolve({ filter: /.*\.worker.[jt]sx?$/ }, (args) => {
+        const importPath = args.path;
+        const importAbsolutePath = path.join(args.resolveDir, importPath);
+
+        // Pin the file extension to .js
+        const workerAbsolutePath = path.join(
+          path.dirname(importAbsolutePath),
+          path.basename(importAbsolutePath).replace(/\.[jt]sx?$/, '.js'),
+        );
+
+        const relativePath = path.relative(
+          getModularRoot(),
+          workerAbsolutePath,
+        );
+
+        return {
+          path: relativePath,
+          namespace: 'web-worker',
+        };
+      });
+
+      build.onLoad({ filter: /.*/, namespace: 'web-worker' }, async (args) => {
+        // Build the worker file with the same format, target and definitions of the bundle
+        try {
+          const result = await esbuild.build({
+            format: build.initialOptions.format,
+            target: build.initialOptions.target,
+            define: build.initialOptions.define,
+            minify: build.initialOptions.minify,
+            entryPoints: [path.join(getModularRoot(), args.path)],
+            plugins: [
+              createExtensionAllowlistPlugin({
+                reason: 'Web workers can only import other modules.',
+              }),
+            ],
+            bundle: true,
+            write: false,
+          });
+
+          // Store the file in the build cache for later use, since we need to emit a file and trampoline it transparently to the user
+          workerBuildCache.set(args.path, result);
+
+          // Trampoline the worker within the bundle, to avoid CORS errors
+          return {
+            contents: `
+                  // Web worker bundled by worker-factory-plugin, mimicking the Worker constructor
+                  import workerUrl from '${args.path}:__worker-url';
+
+                  const workerPath = new URL(workerUrl, import.meta.url);
+                  const importSrc = 'import "' + workerPath + '";';
+
+                  const blob = new Blob([importSrc], {
+                    type: "text/javascript",
+                  });
+
+                  export default class {
+                    constructor() {
+                      return new Worker(URL.createObjectURL(blob), { type: "module" });
+                    }
+                  }
+                `,
+          };
+        } catch (e) {
+          console.error('Error building worker script:', e);
+        }
+      });
+
+      build.onResolve({ filter: /.*:__worker-url/ }, (args) => {
+        return {
+          path: args.path.split(':__worker-url')[0],
+          namespace: 'worker-url',
+        };
+      });
+
+      build.onLoad({ filter: /.*/, namespace: 'worker-url' }, (args) => {
+        const result = workerBuildCache.get(args.path);
+        workerBuildCache.delete(args.path);
+        if (result) {
+          const { outputFiles } = result;
+          if (outputFiles?.length === 1) {
+            const outputFile = outputFiles[0];
+            return {
+              contents: outputFile.contents,
+              loader: 'file',
+            };
+          } else {
+            throw new Error(`Could not read output files`);
+          }
+        } else {
+          throw new Error(`Could not find result for ${args.path}`);
+        }
+      });
+    },
+  };
+
+  return plugin;
+}
+
+export default createPlugin;

packages/modular-scripts/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "target": "es5",
     "downlevelIteration": true,
-    "lib": ["dom", "dom.iterable", "esnext"],
+    "lib": ["dom", "dom.iterable", "esnext", "WebWorker"],
     "allowJs": true,
     "skipLibCheck": true,
     "esModuleInterop": true,

yarn.lock

@@ -13149,7 +13149,7 @@ ts-node@10.4.0:
     make-error "^1.1.1"
     yn "3.1.1"
 
-ts-pnp@1.2.0, ts-pnp@^1.1.6:
+ts-pnp@^1.1.6:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/ts-pnp/-/ts-pnp-1.2.0.tgz#a500ad084b0798f1c3071af391e65912c86bca92"
   integrity sha512-csd+vJOb/gkzvcCHgTGSChYpy5f1/XKNsmvBGO4JXS+z1v2HobugDz4s1IeFXM3wZB44uczs+eazB5Q/ccdhQw==