fix: make forking work in the browser

src/chains/ethereum/ethereum/src/blockchain.ts

@@ -594,7 +594,7 @@ export default class Blockchain extends Emittery<BlockchainTypedEvents> {
     await this.#blockBeingSavedPromise;
     return {
       transactions,
-      blockNumber: nextBlock.header.number.toBuffer()
+      blockNumber: nextBlock.header.number.toArrayLike(Buffer)
     };
   };
 

src/chains/ethereum/ethereum/src/forking/handlers/http-handler.ts

@@ -42,10 +42,13 @@ export class HttpHandler extends BaseHandler implements Handler {
       });
     }
   }
-  private handleLengthedResponse(res: http.IncomingMessage, length: number) {
-    let buffer = Buffer.allocUnsafe(length);
-    let offset = 0;
-    return new Promise<Buffer>((resolve, reject) => {
+  private async handleLengthedResponse(
+    res: http.IncomingMessage,
+    length: number
+  ) {
+    return await new Promise<Buffer>((resolve, reject) => {
+      const buffer = Buffer.allocUnsafe(length);
+      let offset = 0;
       function data(message: Buffer) {
         const messageLength = message.length;
         // note: Node will NOT send us more data than the content-length header
@@ -54,9 +57,9 @@ export class HttpHandler extends BaseHandler implements Handler {
         offset += messageLength;
       }
       function end() {
-        // note: Node doesn't check if the content-length matches, so we do that
-        // here
-        if (offset !== buffer.length) {
+        // note: Node doesn't check if the content-length matches (we might
+        // receive less data than expected), so we do that here
+        if (offset !== length) {
           // if we didn't receive enough data, throw
           reject(new Error("content-length mismatch"));
         } else {
@@ -65,24 +68,18 @@ export class HttpHandler extends BaseHandler implements Handler {
       }
       res.on("data", data);
       res.on("end", end);
+      res.on("error", reject);
     });
   }
-  private handleChunkedResponse(res: http.IncomingMessage) {
-    let buffer: Buffer;
-    return new Promise<Buffer>(resolve => {
-      res.on("data", (message: Buffer) => {
-        const chunk = message;
-        if (buffer) {
-          buffer = Buffer.concat([buffer, chunk], buffer.length + chunk.length);
-        } else {
-          buffer = Buffer.concat([chunk], chunk.length);
-        }
-      });
 
-      res.on("end", () => {
-        resolve(buffer);
-      });
-    });
+  private async handleChunkedResponse(res: http.IncomingMessage) {
+    const chunks = [];
+    let totalLength = 0;
+    for await (let chunk of res) {
+      chunks.push(chunk);
+      totalLength += chunk.length;
+    }
+    return chunks.length === 1 ? chunks[0] : Buffer.concat(chunks, totalLength);
   }
 
   public async request<T>(
@@ -119,17 +116,24 @@ export class HttpHandler extends BaseHandler implements Handler {
         const { headers } = res;
 
         let buffer: Promise<Buffer>;
-        // if we have a transfer-encoding we don't care about "content-length"
-        // (per HTTP spec). We also don't care about invalid lengths
-        if ("transfer-encoding" in headers) {
+        // in the browser we can't detect if the response is compressed (gzip),
+        // but it doesn't matter since the browser has decompressed already
+        // anyway
+        if (process.env.IS_BROWSER) {
           buffer = this.handleChunkedResponse(res);
         } else {
-          const length = (headers["content-length"] as any) / 1;
-          if (isNaN(length) || length <= 0) {
+          // if we have a transfer-encoding we don't care about "content-length"
+          // (per HTTP spec). We also don't care about invalid lengths
+          if ("transfer-encoding" in headers) {
             buffer = this.handleChunkedResponse(res);
           } else {
-            // we have a content-length; use it to pre-allocate the required memory
-            buffer = this.handleLengthedResponse(res, length);
+            const length = (headers["content-length"] as any) / 1;
+            if (isNaN(length) || length <= 0) {
+              buffer = this.handleChunkedResponse(res);
+            } else {
+              // we have a content-length; use it to pre-allocate the required memory
+              buffer = this.handleLengthedResponse(res, length);
+            }
           }
         }
 

src/packages/flavors/index.ts

@@ -71,7 +71,6 @@ export function GetConnector<Flavor extends FlavorName>(
         const f = eval("require")(flavor);
         const Connector: FilecoinConnector =
           typeof f.default != "undefined" ? f.default.Connector : f.Connector;
-        console.log(Connector, f);
         // @ts-ignore
         return new Connector(providerOptions, executor);
       }

src/packages/ganache/README.md

@@ -212,8 +212,6 @@ const options = {};
 const provider = Ganache.provider(options);
 ```
 
-NOTE: currently forking does not work in the browser, but we plan to add support in the future.
-
 ## Documentation
 
 New interactive RPC documentation coming soon!

src/packages/ganache/index.ts

@@ -19,6 +19,11 @@ export type {
 export { EthereumProvider } from "@ganache/core";
 import type { ConnectorsByName } from "@ganache/flavors";
 export type FilecoinProvider = ConnectorsByName["filecoin"];
+
+// polyfill "setImmediate" for the browser
+// this is removed by webpack for our Node.js build
+require("setimmediate");
+
 export {
   server,
   provider,

src/packages/ganache/package.json

@@ -68,7 +68,7 @@
     "cross-env": "7.0.3",
     "crypto-browserify": "3.12.0",
     "events": "3.3.0",
-    "isomorphic-ws": "4.0.1",
+    "https-browserify": "1.0.0",
     "level-js": "6.1.0",
     "mcl-wasm": "0.9.0",
     "mocha": "9.1.3",
@@ -77,9 +77,11 @@
     "path-browserify": "1.0.1",
     "process": "0.11.10",
     "scrypt-js": "3.0.1",
+    "setimmediate": "1.0.5",
     "shebang-loader": "0.0.1",
     "shx": "0.3.3",
     "stream-browserify": "3.0.0",
+    "stream-http": "3.2.0",
     "terser-webpack-plugin": "5.2.5",
     "ts-loader": "9.2.6",
     "ts-node": "10.4.0",

src/packages/ganache/webpack/polyfills/url.ts

@@ -0,0 +1,6 @@
+// We use node's built-in `URL` module on the Node.js side but this is already
+// available in the browser, so we just need to export it so webpack can
+// resolve it.
+
+// my magnum opus:
+export const URL = globalThis.URL;

src/packages/ganache/webpack/polyfills/ws.ts

@@ -0,0 +1,14 @@
+// We use the Node.js `ws` module on the Node.js side but it doesn't work in the
+// browser, so we need to export the browser `WebSocket` for the browser.
+import type { URL } from "url";
+
+export default class WebSocket {
+  // the browser `WebSocket` class has a second parameter, `protocols`, which is
+  // incompatible with the `ws` module's second parameter, `options`.
+  constructor(url: string | URL, options?: { protocol?: string | undefined }) {
+    return new globalThis.WebSocket(
+      url,
+      options ? options.protocol : undefined
+    );
+  }
+}

src/packages/ganache/webpack/webpack.browser.config.ts

@@ -17,8 +17,11 @@ const config: webpack.Configuration = merge({}, base, {
       events: require.resolve("events/"),
       buffer: require.resolve("buffer/"),
       fs: false,
-      http: false,
-      https: false
+      // Taken from https://webpack.js.org/configuration/resolve/#resolvefallback
+      http: require.resolve("stream-http"),
+      https: require.resolve("https-browserify"),
+      // not needed by the browser as the browser does the work
+      zlib: false
       //#endregion node polyfills
     },
     alias: {
@@ -29,13 +32,13 @@ const config: webpack.Configuration = merge({}, base, {
       leveldown: require.resolve("level-js/"),
       // browser version can't start a server, so just remove the websocket server since it can't work anyway
       "@trufflesuite/uws-js-unofficial": false,
+      // replace URL with a browser version -- sorta. just look at the polyfill code
+      url: require.resolve("./polyfills/url"),
       "@ganache/filecoin": false,
-      // `url` is already a global property in browser
-      url: false,
       // mcl-wasm may be needed when creating a new @ethereumjs/vm and requires a browser version for browsers
       "mcl-wasm": require.resolve("mcl-wasm/browser"),
-      // ws doesn't work in the browser, isomorphic-ws does
-      ws: require.resolve("isomorphic-ws/"),
+      // ws doesn't work in the browser so we polyfill it
+      ws: require.resolve("./polyfills/ws"),
       // we don't use the debug module internally, so let's just not include it
       // in any package.
       debug: require.resolve("./polyfills/debug")
@@ -47,7 +50,12 @@ const config: webpack.Configuration = merge({}, base, {
   },
   plugins: [
     new webpack.ProvidePlugin({ Buffer: ["buffer", "Buffer"] }),
-    new webpack.ProvidePlugin({ process: ["process"] })
+    new webpack.ProvidePlugin({ process: ["process"] }),
+    // replace process.env.IS_BROWSER with `true` to cause the minifier to
+    // remove code blocks that require `process.env.IS_BROWSER != false`
+    new webpack.EnvironmentPlugin({
+      IS_BROWSER: true
+    })
   ]
 });
 

src/packages/ganache/webpack/webpack.common.config.ts

@@ -1,13 +1,14 @@
 import webpack from "webpack";
 import TerserPlugin from "terser-webpack-plugin";
-import path from "path";
+import { join } from "path";
 
-const VERSION = require(path.join(__dirname, "../package.json")).version;
-const CLI_VERSION = require(path.join(__dirname, "../../cli/package.json"))
-  .version;
-const CORE_VERSION = require(path.join(__dirname, "../../core/package.json"))
-  .version;
-const GANACHE_FILECOIN_VERSION = require(path.join(
+const VERSION = require(join(__dirname, "../package.json")).version;
+const CLI_VERSION = require(join(__dirname, "../../cli/package.json")).version;
+const CORE_VERSION = require(join(
+  __dirname,
+  "../../core/package.json"
+)).version;
+const GANACHE_FILECOIN_VERSION = require(join(
   __dirname,
   "../../../chains/filecoin/filecoin/package.json"
 )).version;
@@ -73,7 +74,15 @@ const base: webpack.Configuration = {
           sourceMap: true,
           // Truffle needs our stack traces in its tests:
           // https://github.com/trufflesuite/truffle/blob/b2742bc1187a3c1513173d19c58ce0d3a8fe969b/packages/contract-tests/test/errors.js#L280
-          keep_fnames: true
+          keep_fnames: true,
+          output: {
+            // terser will take strings like "\ufffd" (REPLACEMENT CHARACTER)
+            // and compress them into their single character representation: "�"
+            // (that should render as a question mark within a diamond). This is
+            // nice, but Chromium extensions don't like it and error with "It
+            // isn't UTF-8 encoded".
+            ascii_only: true
+          }
         }
       })
     ]

src/packages/ganache/webpack/webpack.node.config.ts

@@ -19,7 +19,9 @@ const config: webpack.Configuration = merge({}, base, {
     alias: {
       // we don't use the debug module internally, so let's just not include it
       // in any package.
-      debug: require.resolve("./polyfills/debug")
+      debug: require.resolve("./polyfills/debug"),
+      // the `setimmediate` package is only used in the browser
+      setimmediate: false
     }
   },
   plugins: [
@@ -30,7 +32,12 @@ const config: webpack.Configuration = merge({}, base, {
       banner: "#!/usr/bin/env node",
       raw: true
     }),
-    new DeduplicatePlugin()
+    new DeduplicatePlugin(),
+    // replace process.env.IS_BROWSER with `false` to cause the minifier to
+    // remove code blocks that require `process.env.IS_BROWSER != true`
+    new webpack.EnvironmentPlugin({
+      IS_BROWSER: false
+    })
   ],
   optimization: {
     splitChunks: {