Custom bundle loaders + WASM support -> master

.eslintignore

@@ -10,6 +10,8 @@
 /test/integration/dynamic-references-raw/index.js
 /test/integration/dynamic-references-raw/local.js
 /test/integration/hmr-dynamic/index.js
+/test/integration/wasm-async/index.js
+/test/integration/wasm-dynamic/index.js
 
 # Generated by the build
 lib

package.json

@@ -15,6 +15,7 @@
     "babylon-walk": "^1.0.2",
     "browser-resolve": "^1.11.2",
     "chalk": "^2.1.0",
+    "child-process-promise": "^2.2.1",
     "chokidar": "^1.7.0",
     "commander": "^2.11.0",
     "cross-spawn": "^5.1.0",

src/Bundle.js

@@ -15,7 +15,20 @@ class Bundle {
     this.entryAsset = null;
     this.assets = new Set();
     this.childBundles = new Set();
-    this.siblingBundles = new Map();
+    this.siblingBundles = new Set;
+    this.siblingBundlesMap = new Map();
+  }
+
+  static createWithAsset(asset, parentBundle) {
+    let bundle = new Bundle(
+      asset.type,
+      Path.join(asset.options.outDir, asset.generateBundleName()),
+      parentBundle
+    );
+
+    bundle.entryAsset = asset;
+    bundle.addAsset(asset);
+    return bundle;
   }
 
   addAsset(asset) {
@@ -33,26 +46,36 @@ class Bundle {
       return this;
     }
 
-    if (!this.siblingBundles.has(type)) {
-      let bundle = this.createChildBundle(
+    if (!this.siblingBundlesMap.has(type)) {
+      let bundle = new Bundle(
         type,
         Path.join(
           Path.dirname(this.name),
           Path.basename(this.name, Path.extname(this.name)) + '.' + type
-        )
+        ),
+        this
       );
-      this.siblingBundles.set(type, bundle);
+
+      this.childBundles.add(bundle);
+      this.siblingBundles.add(bundle);
+      this.siblingBundlesMap.set(type, bundle);
     }
 
-    return this.siblingBundles.get(type);
+    return this.siblingBundlesMap.get(type);
   }
 
-  createChildBundle(type, name) {
-    let bundle = new Bundle(type, name, this);
+  createChildBundle(entryAsset) {
+    let bundle = Bundle.createWithAsset(entryAsset, this);
     this.childBundles.add(bundle);
     return bundle;
   }
 
+  createSiblingBundle(entryAsset) {
+    let bundle = this.createChildBundle(entryAsset);
+    this.siblingBundles.add(bundle);
+    return bundle;
+  }
+
   get isEmpty() {
     return this.assets.size === 0;
   }

src/Bundler.js

@@ -15,6 +15,7 @@ const localRequire = require('./utils/localRequire');
 const config = require('./utils/config');
 const emoji = require('./utils/emoji');
 const loadEnv = require('./utils/env');
+const PromiseQueue = require('./utils/PromiseQueue');
 
 /**
  * The Bundler is the main entry point. It resolves and loads assets,
@@ -32,6 +33,17 @@ class Bundler extends EventEmitter {
     this.cache = this.options.cache ? new FSCache(this.options) : null;
     this.logger = new Logger(this.options);
     this.delegate = options.delegate || {};
+    this.bundleLoaders = {};
+
+    this.addBundleLoader(
+      'wasm',
+      require.resolve('./builtins/loaders/wasm-loader')
+    );
+    this.addBundleLoader(
+      'css',
+      require.resolve('./builtins/loaders/css-loader')
+    );
+    this.addBundleLoader('js', require.resolve('./builtins/loaders/js-loader'));
 
     this.pending = false;
     this.loadedAssets = new Map();
@@ -41,7 +53,7 @@ class Bundler extends EventEmitter {
     this.hmr = null;
     this.bundleHashes = null;
     this.errored = false;
-    this.buildQueue = new Set();
+    this.buildQueue = new PromiseQueue(this.processAsset.bind(this));
     this.rebuildTimeout = null;
   }
 
@@ -92,6 +104,18 @@ class Bundler extends EventEmitter {
     this.packagers.add(type, packager);
   }
 
+  addBundleLoader(type, path) {
+    if (typeof path !== 'string') {
+      throw new Error('Bundle loader should be a module path.');
+    }
+
+    if (this.farm) {
+      throw new Error('Bundle loaders must be added before bundling.');
+    }
+
+    this.bundleLoaders[type] = path;
+  }
+
   async loadPlugins() {
     let pkg = await config.load(this.mainFile, ['package.json']);
     if (!pkg) {
@@ -141,8 +165,26 @@ class Bundler extends EventEmitter {
         this.buildQueue.add(this.mainAsset);
       }
 
-      // Build the queued assets, and produce a bundle tree.
-      let bundle = await this.buildQueuedAssets(isInitialBundle);
+      // Build the queued assets.
+      let loadedAssets = await this.buildQueue.run();
+
+      // Emit an HMR update for any new assets (that don't have a parent bundle yet)
+      // plus the asset that actually changed.
+      if (this.hmr && !isInitialBundle) {
+        this.hmr.emitUpdate([...this.findOrphanAssets(), ...loadedAssets]);
+      }
+
+      // Invalidate bundles
+      for (let asset of this.loadedAssets.values()) {
+        asset.invalidateBundle();
+      }
+
+      // Create a new bundle tree and package everything up.
+      let bundle = this.createBundleTree(this.mainAsset);
+      this.bundleHashes = await bundle.package(this, this.bundleHashes);
+
+      // Unload any orphaned assets
+      this.unloadOrphanedAssets();
 
       let buildTime = Date.now() - startTime;
       let time =
@@ -151,6 +193,7 @@ class Bundler extends EventEmitter {
           : `${(buildTime / 1000).toFixed(2)}s`;
       this.logger.status(emoji.success, `Built in ${time}.`, 'green');
 
+      this.emit('bundled', bundle);
       return bundle;
     } catch (err) {
       this.errored = true;
@@ -182,8 +225,8 @@ class Bundler extends EventEmitter {
     await loadEnv(this.mainFile);
 
     this.options.extensions = Object.assign({}, this.parser.extensions);
+    this.options.bundleLoaders = this.bundleLoaders;
     this.options.env = process.env;
-    this.farm = WorkerFarm.getShared(this.options);
 
     if (this.options.watch) {
       // FS events on macOS are flakey in the tests, which write lots of files very quickly
@@ -199,6 +242,8 @@ class Bundler extends EventEmitter {
       this.hmr = new HMRServer();
       this.options.hmrPort = await this.hmr.start(this.options.hmrPort);
     }
+
+    this.farm = WorkerFarm.getShared(this.options);
   }
 
   stop() {
@@ -215,49 +260,11 @@ class Bundler extends EventEmitter {
     }
   }
 
-  async buildQueuedAssets(isInitialBundle = false) {
-    // Consume the rebuild queue until it is empty.
-    let loadedAssets = new Set();
-    while (this.buildQueue.size > 0) {
-      let promises = [];
-      for (let asset of this.buildQueue) {
-        // Invalidate the asset, unless this is the initial bundle
-        if (!isInitialBundle) {
-          asset.invalidate();
-          if (this.cache) {
-            this.cache.invalidate(asset.name);
-          }
-        }
-
-        promises.push(this.loadAsset(asset));
-        loadedAssets.add(asset);
-      }
-
-      // Wait for all assets to load. If there are more added while
-      // these are processing, they'll be loaded in the next batch.
-      await Promise.all(promises);
-    }
-
-    // Emit an HMR update for any new assets (that don't have a parent bundle yet)
-    // plus the asset that actually changed.
-    if (this.hmr && !isInitialBundle) {
-      this.hmr.emitUpdate([...this.findOrphanAssets(), ...loadedAssets]);
-    }
-
-    // Invalidate bundles
-    for (let asset of this.loadedAssets.values()) {
-      asset.invalidateBundle();
-    }
-
-    // Create a new bundle tree and package everything up.
-    let bundle = this.createBundleTree(this.mainAsset);
-    this.bundleHashes = await bundle.package(this, this.bundleHashes);
-
-    // Unload any orphaned assets
-    this.unloadOrphanedAssets();
-
-    this.emit('bundled', bundle);
-    return bundle;
+  async getAsset(name, parent) {
+    let asset = await this.resolveAsset(name, parent);
+    this.buildQueue.add(asset);
+    await this.buildQueue.run();
+    return asset;
   }
 
   async resolveAsset(name, parent) {
@@ -328,9 +335,19 @@ class Bundler extends EventEmitter {
     }
   }
 
+  async processAsset(asset, isRebuild) {
+    if (isRebuild) {
+      asset.invalidate();
+      if (this.cache) {
+        this.cache.invalidate(asset.name);
+      }
+    }
+
+    await this.loadAsset(asset);
+  }
+
   async loadAsset(asset) {
     if (asset.processed) {
-      this.buildQueue.delete(asset);
       return;
     }
 
@@ -386,8 +403,6 @@ class Bundler extends EventEmitter {
         asset.depAssets.set(dep, assetDep);
       }
     });
-
-    this.buildQueue.delete(asset);
   }
 
   createBundleTree(asset, dep, bundle, parentBundles = new Set()) {
@@ -416,27 +431,20 @@ class Bundler extends EventEmitter {
       }
     }
 
-    // Create the root bundle if it doesn't exist
     if (!bundle) {
-      bundle = new Bundle(
-        asset.type,
-        Path.join(this.options.outDir, asset.generateBundleName())
-      );
-      bundle.entryAsset = asset;
+      // Create the root bundle if it doesn't exist
+      bundle = Bundle.createWithAsset(asset);
+    } else if (dep && dep.dynamic) {
+      // Create a new bundle for dynamic imports
+      bundle = bundle.createChildBundle(asset);
+    } else if (asset.type && !this.packagers.has(asset.type)) {
+      // No packager is available for this asset type. Create a new bundle with only this asset.
+      bundle.createSiblingBundle(asset);
+    } else {
+      // Add the asset to the common bundle of the asset's type
+      bundle.getSiblingBundle(asset.type).addAsset(asset);
     }
 
-    // Create a new bundle for dynamic imports
-    if (dep && dep.dynamic) {
-      bundle = bundle.createChildBundle(
-        asset.type,
-        Path.join(this.options.outDir, asset.generateBundleName())
-      );
-      bundle.entryAsset = asset;
-    }
-
-    // Add the asset to the bundle of the asset's type
-    bundle.getSiblingBundle(asset.type).addAsset(asset);
-
     // If the asset generated a representation for the parent bundle type, also add it there
     if (asset.generated[bundle.type] != null) {
       bundle.addAsset(asset);
@@ -523,7 +531,7 @@ class Bundler extends EventEmitter {
 
     // Add the asset to the rebuild queue, and reset the timeout.
     for (let asset of assets) {
-      this.buildQueue.add(asset);
+      this.buildQueue.add(asset, true);
     }
 
     clearTimeout(this.rebuildTimeout);

src/Server.js

@@ -5,6 +5,10 @@ const getPort = require('get-port');
 const serverErrors = require('./utils/customErrors').serverErrors;
 const generateCertificate = require('./utils/generateCertificate');
 
+serveStatic.mime.define({
+  'application/wasm': ['wasm']
+});
+
 function middleware(bundler) {
   const serve = serveStatic(bundler.options.outDir, {index: false});
 

src/assets/HTMLAsset.js

@@ -47,7 +47,9 @@ class HTMLAsset extends Asset {
             continue;
           }
           if (elements && elements.includes(node.tag)) {
-            let assetPath = this.addURLDependency(decodeURIComponent(node.attrs[attr]));
+            let assetPath = this.addURLDependency(
+              decodeURIComponent(node.attrs[attr])
+            );
             if (!isURL(assetPath)) {
               assetPath = urlJoin(this.options.publicURL, assetPath);
             }

src/assets/RawAsset.js

@@ -6,6 +6,12 @@ class RawAsset extends Asset {
   load() {}
 
   generate() {
+    // Don't return a URL to the JS bundle if there is a bundle loader defined for this asset type.
+    // This will cause the actual asset to be automatically preloaded prior to the JS bundle running.
+    if (this.options.bundleLoaders[this.type]) {
+      return {};
+    }
+
     const pathToAsset = urlJoin(
       this.options.publicURL,
       this.generateBundleName()