Improve base64 encoding/decoding speeds (#1985)

Adds new base64 encoding and decoding utilities that have faster
performance characteristics than the current implementation. This also
fixes an issue where the client would sometimes get unresponsive while
encoding large files.

Fixes #1984 

```
async base64 encoding of 98095974 bytes took 596 ms
sync base64 encoding of 98095974 bytes took 86004 ms
async base64 decoding of 98095974 bytes took 5525 ms
sync base64 decoding of 98095974 bytes took 73044 ms
```

packages/snaps-controllers/coverage.json

@@ -1,6 +1,6 @@
 {
   "branches": 90.11,
-  "functions": 96.34,
-  "lines": 97.29,
+  "functions": 96.35,
+  "lines": 97.3,
   "statements": 96.98
 }

packages/snaps-controllers/src/snaps/SnapController.test.ts

@@ -41,7 +41,13 @@ import {
   getMockSnapFilesWithUpdatedChecksum,
 } from '@metamask/snaps-utils/test-utils';
 import type { SemVerRange, SemVerVersion } from '@metamask/utils';
-import { assert, AssertionError, stringToBytes } from '@metamask/utils';
+import {
+  assert,
+  AssertionError,
+  base64ToBytes,
+  stringToBytes,
+} from '@metamask/utils';
+import { File } from 'buffer';
 import fetchMock from 'jest-fetch-mock';
 import { createEngineStream } from 'json-rpc-middleware-stream';
 import { pipeline } from 'readable-stream';
@@ -6577,6 +6583,25 @@ describe('SnapController', () => {
     });
 
     it('supports hex encoding', async () => {
+      fetchMock.disableMocks();
+
+      // We can remove this once we drop Node 18
+      Object.defineProperty(globalThis, 'File', {
+        value: File,
+      });
+
+      // Because jest-fetch-mock replaces native fetch, we mock it here
+      Object.defineProperty(globalThis, 'fetch', {
+        value: async (dataUrl: string) => {
+          const base64 = dataUrl.replace(
+            'data:application/octet-stream;base64,',
+            '',
+          );
+          const u8 = base64ToBytes(base64);
+          return new File([u8], '');
+        },
+      });
+
       const auxiliaryFile = new VirtualFile({
         path: 'src/foo.json',
         value: stringToBytes('{ "foo" : "bar" }'),
@@ -6613,6 +6638,8 @@ describe('SnapController', () => {
         ),
       ).toStrictEqual(auxiliaryFile.toString('hex'));
 
+      fetchMock.enableMocks();
+
       snapController.destroy();
     });
 

packages/snaps-controllers/src/snaps/SnapController.ts

@@ -69,6 +69,7 @@ import {
   unwrapError,
   OnHomePageResponseStruct,
   getValidatedLocalizationFiles,
+  encodeBase64,
 } from '@metamask/snaps-utils';
 import type { Json, NonEmptyArray, SemVerRange } from '@metamask/utils';
 import {
@@ -983,7 +984,7 @@ export class SnapController extends BaseController<
 
     this.messagingSystem.registerActionHandler(
       `${controllerName}:getFile`,
-      (...args) => this.getSnapFile(...args),
+      async (...args) => this.getSnapFile(...args),
     );
   }
 
@@ -1423,11 +1424,11 @@ export class SnapController extends BaseController<
    * @param encoding - An optional requested file encoding.
    * @returns The file requested in the chosen file encoding or null if the file is not found.
    */
-  getSnapFile(
+  async getSnapFile(
     snapId: SnapId,
     path: string,
     encoding: AuxiliaryFileEncoding = AuxiliaryFileEncoding.Base64,
-  ): string | null {
+  ): Promise<string | null> {
     const snap = this.getExpect(snapId);
     const normalizedPath = normalizeRelative(path);
     const value = snap.auxiliaryFiles?.find(
@@ -2284,10 +2285,13 @@ export class SnapController extends BaseController<
       `Invalid source code for snap "${snapId}".`,
     );
 
-    const auxiliaryFiles = rawAuxiliaryFiles.map((file) => ({
-      path: file.path,
-      value: file.toString('base64'),
-    }));
+    const auxiliaryFiles = rawAuxiliaryFiles.map((file) => {
+      assert(typeof file.data.base64 === 'string');
+      return {
+        path: file.path,
+        value: file.data.base64,
+      };
+    });
 
     const snapsState = this.state.snaps;
 
@@ -2373,6 +2377,14 @@ export class SnapController extends BaseController<
         manifest.result.source.files,
       );
 
+      await Promise.all(
+        auxiliaryFiles.map(async (file) => {
+          // This should still be safe
+          // eslint-disable-next-line require-atomic-updates
+          file.data.base64 = await encodeBase64(file);
+        }),
+      );
+
       const localizationFiles = await getSnapFiles(
         location,
         manifest.result.source.locales,

packages/snaps-simulator/src/features/simulation/hooks.test.ts

@@ -1,6 +1,7 @@
 import { AuxiliaryFileEncoding, DialogType, text } from '@metamask/snaps-sdk';
 import { VirtualFile, normalizeRelative } from '@metamask/snaps-utils';
-import { stringToBytes } from '@metamask/utils';
+import { base64ToBytes, stringToBytes } from '@metamask/utils';
+import { File } from 'buffer';
 import { expectSaga } from 'redux-saga-test-plan';
 
 import { addNotification } from '../notifications';
@@ -38,6 +39,15 @@ jest.mock('@reduxjs/toolkit', () => ({
   nanoid: () => 'foo',
 }));
 
+// Because jest-fetch-mock replaces native fetch, we mock it here
+Object.defineProperty(globalThis, 'fetch', {
+  value: async (dataUrl: string) => {
+    const base64 = dataUrl.replace('data:application/octet-stream;base64,', '');
+    const u8 = base64ToBytes(base64);
+    return new File([u8], '');
+  },
+});
+
 const snapId = 'local:http://localhost:8080';
 
 describe('showDialog', () => {

packages/snaps-simulator/src/features/simulation/hooks.ts

@@ -196,5 +196,5 @@ export function* getSnapFile(
     return null;
   }
 
-  return encodeAuxiliaryFile(base64, encoding);
+  return yield call(encodeAuxiliaryFile, base64, encoding);
 }

packages/snaps-utils/coverage.json

@@ -1,6 +1,6 @@
 {
-  "branches": 96,
-  "functions": 99,
-  "lines": 98.68,
-  "statements": 95.58
+  "branches": 96.01,
+  "functions": 98.53,
+  "lines": 98.6,
+  "statements": 95.55
 }

packages/snaps-utils/src/auxiliary-files.test.ts

@@ -5,24 +5,26 @@ import { base64 } from '@scure/base';
 import { encodeAuxiliaryFile } from './auxiliary-files';
 
 describe('encodeAuxiliaryFile', () => {
-  it('returns value without modifying it for base64', () => {
+  it('returns value without modifying it for base64', async () => {
     const value = base64.encode(stringToBytes('foo'));
     expect(
-      encodeAuxiliaryFile(value, AuxiliaryFileEncoding.Base64),
+      await encodeAuxiliaryFile(value, AuxiliaryFileEncoding.Base64),
     ).toStrictEqual(value);
   });
 
-  it('re-encodes to hex when requested', () => {
+  it('re-encodes to hex when requested', async () => {
     const bytes = stringToBytes('foo');
     const value = base64.encode(bytes);
-    expect(encodeAuxiliaryFile(value, AuxiliaryFileEncoding.Hex)).toStrictEqual(
-      bytesToHex(bytes),
-    );
+    expect(
+      await encodeAuxiliaryFile(value, AuxiliaryFileEncoding.Hex),
+    ).toStrictEqual(bytesToHex(bytes));
   });
 
-  it('returns plaintext when requested', () => {
+  it('returns plaintext when requested', async () => {
     const bytes = stringToBytes('foo');
     const value = base64.encode(bytes);
-    expect(encodeAuxiliaryFile(value, AuxiliaryFileEncoding.Utf8)).toBe('foo');
+    expect(await encodeAuxiliaryFile(value, AuxiliaryFileEncoding.Utf8)).toBe(
+      'foo',
+    );
   });
 });

packages/snaps-utils/src/auxiliary-files.ts

@@ -1,6 +1,7 @@
 import { AuxiliaryFileEncoding } from '@metamask/snaps-sdk';
 import { bytesToHex, bytesToString } from '@metamask/utils';
-import { base64 } from '@scure/base';
+
+import { decodeBase64 } from './base64';
 
 /**
  * Re-encodes an auxiliary file if needed depending on the requested file encoding.
@@ -9,7 +10,7 @@ import { base64 } from '@scure/base';
  * @param encoding - The chosen encoding.
  * @returns The file encoded in the requested encoding.
  */
-export function encodeAuxiliaryFile(
+export async function encodeAuxiliaryFile(
   value: string,
   encoding: AuxiliaryFileEncoding,
 ) {
@@ -19,7 +20,7 @@ export function encodeAuxiliaryFile(
   }
 
   // TODO: Use @metamask/utils for this
-  const decoded = base64.decode(value);
+  const decoded = await decodeBase64(value);
   if (encoding === AuxiliaryFileEncoding.Utf8) {
     return bytesToString(decoded);
   }

packages/snaps-utils/src/base64.test.ts

@@ -0,0 +1,67 @@
+import { bytesToBase64, stringToBytes } from '@metamask/utils';
+import { File } from 'buffer';
+
+import { decodeBase64, encodeBase64 } from './base64';
+import { VirtualFile } from './virtual-file';
+
+// Very basic mock that mimics the base64 encoding logic of the browser
+class MockFileReader {
+  onload?: () => any;
+
+  onerror?: () => any;
+
+  result?: any;
+
+  error?: any;
+
+  readAsDataURL(file: File) {
+    file
+      .arrayBuffer()
+      .then((buffer) => {
+        const u8 = new Uint8Array(buffer);
+
+        this.result = `data:application/octet-stream;base64,${bytesToBase64(
+          u8,
+        )}`;
+
+        this.onload?.();
+      })
+      .catch((error) => {
+        this.error = error;
+        this.onerror?.();
+      });
+  }
+}
+
+describe('encodeBase64', () => {
+  // We can remove this once we drop Node 18
+  Object.defineProperty(globalThis, 'File', {
+    value: File,
+  });
+
+  it('encodes vfile to base64', async () => {
+    const vfile = new VirtualFile(
+      stringToBytes(JSON.stringify({ foo: 'bar' })),
+    );
+    expect(await encodeBase64(vfile)).toBe('eyJmb28iOiJiYXIifQ==');
+  });
+
+  it('uses FileReader API when available', async () => {
+    Object.defineProperty(globalThis, 'FileReader', {
+      value: MockFileReader,
+    });
+
+    const vfile = new VirtualFile(
+      stringToBytes(JSON.stringify({ foo: 'bar' })),
+    );
+    expect(await encodeBase64(vfile)).toBe('eyJmb28iOiJiYXIifQ==');
+  });
+});
+
+describe('decodeBase64', () => {
+  it('decodes base64 string to bytes', async () => {
+    expect(await decodeBase64('eyJmb28iOiJiYXIifQ==')).toStrictEqual(
+      stringToBytes(JSON.stringify({ foo: 'bar' })),
+    );
+  });
+});

packages/snaps-utils/src/base64.ts

@@ -0,0 +1,46 @@
+import { bytesToBase64 } from '@metamask/utils';
+
+import { getBytes } from './bytes';
+import type { VirtualFile } from './virtual-file';
+
+/**
+ * Provides fast, asynchronous base64 encoding.
+ *
+ * @param input - The input value, assumed to be coercable to bytes.
+ * @returns A base64 string.
+ */
+export async function encodeBase64(input: Uint8Array | VirtualFile | string) {
+  const bytes = getBytes(input);
+  // In the browser, FileReader is much faster than bytesToBase64.
+  if ('FileReader' in globalThis) {
+    return await new Promise((resolve, reject) => {
+      const reader = Object.assign(new FileReader(), {
+        onload: () =>
+          resolve(
+            (reader.result as string).replace(
+              'data:application/octet-stream;base64,',
+              '',
+            ),
+          ),
+        onerror: () => reject(reader.error),
+      });
+      reader.readAsDataURL(
+        new File([bytes], '', { type: 'application/octet-stream' }),
+      );
+    });
+  }
+  return bytesToBase64(bytes);
+}
+
+/**
+ * Provides fast, asynchronous base64 decoding.
+ *
+ * @param base64 - A base64 string.
+ * @returns A Uint8Array of bytes.
+ */
+export async function decodeBase64(base64: string) {
+  const response = await fetch(
+    `data:application/octet-stream;base64,${base64}`,
+  );
+  return new Uint8Array(await response.arrayBuffer());
+}

packages/snaps-utils/src/bytes.test.ts

@@ -0,0 +1,24 @@
+import { getBytes } from './bytes';
+import { VirtualFile } from './virtual-file';
+
+describe('getBytes', () => {
+  const FOO_BAR_STR = 'foo bar';
+  const FOO_BAR_UINT8 = new Uint8Array([
+    0x66, 0x6f, 0x6f, 0x20, 0x62, 0x61, 0x72,
+  ]);
+
+  it('handles Uint8Array', () => {
+    expect(getBytes(FOO_BAR_UINT8)).toStrictEqual(FOO_BAR_UINT8);
+  });
+
+  it('handles strings', () => {
+    expect(getBytes(FOO_BAR_STR)).toStrictEqual(FOO_BAR_UINT8);
+  });
+
+  it('handles virtual files', () => {
+    expect(getBytes(new VirtualFile(FOO_BAR_UINT8))).toStrictEqual(
+      FOO_BAR_UINT8,
+    );
+    expect(getBytes(new VirtualFile(FOO_BAR_STR))).toStrictEqual(FOO_BAR_UINT8);
+  });
+});

packages/snaps-utils/src/bytes.ts

@@ -0,0 +1,21 @@
+import { stringToBytes } from '@metamask/utils';
+
+import { VirtualFile } from './virtual-file';
+
+/**
+ * Convert a bytes-like input value to a Uint8Array.
+ *
+ * @param bytes - A bytes-like value.
+ * @returns The input value converted to a Uint8Array if necessary.
+ */
+export function getBytes(bytes: VirtualFile | Uint8Array | string): Uint8Array {
+  // Unwrap VirtualFiles to extract the content
+  // The content is then either a string or Uint8Array
+  const unwrapped = bytes instanceof VirtualFile ? bytes.value : bytes;
+
+  if (typeof unwrapped === 'string') {
+    return stringToBytes(unwrapped);
+  }
+
+  return unwrapped;
+}