feat: stream crs data to disk

barretenberg/ts/src/crs/net_crs.ts

@@ -21,40 +21,32 @@ export class NetCrs {
     await this.downloadG2Data();
   }
 
-  async downloadG1Data() {
-    // Skip the download if numPoints is 0 (would download the entire file due to bad range header otherwise)
-    if (this.numPoints === 0) {
-      return (this.data = new Uint8Array([]));
-    }
-
-    const g1End = this.numPoints * 64 - 1;
+  /**
+   * Opens up a ReadableStream to the points data
+   */
+  async streamG1Data(): Promise<ReadableStream<Uint8Array>> {
+    const response = await this.fetchG1Data();
+    return response.body!;
+  }
 
-    const response = await retry(
-      () =>
-        fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g1.dat', {
-          headers: {
-            Range: `bytes=0-${g1End}`,
-          },
-          cache: 'force-cache',
-        }),
-      makeBackoff([5, 5, 5]),
-    );
+  /**
+   * Opens up a ReadableStream to the points data
+   */
+  async streamG2Data(): Promise<ReadableStream<Uint8Array>> {
+    const response = await this.fetchG2Data();
+    return response.body!;
+  }
 
+  async downloadG1Data() {
+    const response = await this.fetchG1Data();
     return (this.data = new Uint8Array(await response.arrayBuffer()));
   }
 
   /**
    * Download the G2 points data.
    */
   async downloadG2Data() {
-    const response2 = await retry(
-      () =>
-        fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g2.dat', {
-          cache: 'force-cache',
-        }),
-      makeBackoff([5, 5, 5]),
-    );
-
+    const response2 = await this.fetchG2Data();
     return (this.g2Data = new Uint8Array(await response2.arrayBuffer()));
   }
 
@@ -73,6 +65,41 @@ export class NetCrs {
   getG2Data(): Uint8Array {
     return this.g2Data;
   }
+
+  /**
+   * Fetches the appropriate range of points from a remote source
+   */
+  private async fetchG1Data(): Promise<Response> {
+    // Skip the download if numPoints is 0 (would download the entire file due to bad range header otherwise)
+    if (this.numPoints === 0) {
+      return new Response(new Uint8Array([]));
+    }
+
+    const g1End = this.numPoints * 64 - 1;
+    return await retry(
+      () =>
+        fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g1.dat', {
+          headers: {
+            Range: `bytes=0-${g1End}`,
+          },
+          cache: 'force-cache',
+        }),
+      makeBackoff([5, 5, 5]),
+    );
+  }
+
+  /**
+   * Fetches the appropriate range of points from a remote source
+   */
+  private async fetchG2Data(): Promise<Response> {
+    return await retry(
+      () =>
+        fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g2.dat', {
+          cache: 'force-cache',
+        }),
+      makeBackoff([5, 5, 5]),
+    );
+  }
 }
 
 /**
@@ -96,29 +123,43 @@ export class NetGrumpkinCrs {
   }
 
   async downloadG1Data() {
+    const response = await this.fetchG1Data();
+    return (this.data = new Uint8Array(await response.arrayBuffer()));
+  }
+
+  /**
+   * Opens up a ReadableStream to the points data
+   */
+  async streamG1Data(): Promise<ReadableStream<Uint8Array>> {
+    const response = await this.fetchG1Data();
+    return response.body!;
+  }
+
+  /**
+   * G1 points data for prover key.
+   * @returns The points data.
+   */
+  getG1Data(): Uint8Array {
+    return this.data;
+  }
+
+  /**
+   * Fetches the appropriate range of points from a remote source
+   */
+  private async fetchG1Data(): Promise<Response> {
     // Skip the download if numPoints is 0 (would download the entire file due to bad range header otherwise)
     if (this.numPoints === 0) {
-      return (this.data = new Uint8Array([]));
+      return new Response(new Uint8Array([]));
     }
 
     const g1Start = 28;
     const g1End = g1Start + (this.numPoints * 64 - 1);
 
-    const response = await fetch('https://aztec-ignition.s3.amazonaws.com/TEST%20GRUMPKIN/monomial/transcript00.dat', {
+    return await fetch('https://aztec-ignition.s3.amazonaws.com/TEST%20GRUMPKIN/monomial/transcript00.dat', {
       headers: {
         Range: `bytes=${g1Start}-${g1End}`,
       },
       cache: 'force-cache',
     });
-
-    return (this.data = new Uint8Array(await response.arrayBuffer()));
-  }
-
-  /**
-   * G1 points data for prover key.
-   * @returns The points data.
-   */
-  getG1Data(): Uint8Array {
-    return this.data;
   }
 }

barretenberg/ts/src/crs/node/index.ts

@@ -1,8 +1,10 @@
 import { NetCrs, NetGrumpkinCrs } from '../net_crs.js';
-import { closeSync, mkdirSync, openSync, readFileSync, readSync, writeFileSync } from 'fs';
+import { closeSync, mkdirSync, openSync, readFileSync, readSync, writeFileSync, createWriteStream } from 'fs';
 import { stat } from 'fs/promises';
+import { Readable } from 'stream';
 import createDebug from 'debug';
 import { homedir } from 'os';
+import { finished } from 'stream/promises';
 
 /**
  * Generic CRS finder utility class.
@@ -24,7 +26,7 @@ export class Crs {
     return crs;
   }
 
-  async init() {
+  async init(): Promise<void> {
     mkdirSync(this.path, { recursive: true });
 
     const g1FileSize = await stat(this.path + '/bn254_g1.dat')
@@ -41,9 +43,12 @@ export class Crs {
 
     this.logger(`Downloading CRS of size ${this.numPoints} into ${this.path}`);
     const crs = new NetCrs(this.numPoints);
-    await crs.init();
-    writeFileSync(this.path + '/bn254_g1.dat', crs.getG1Data());
-    writeFileSync(this.path + '/bn254_g2.dat', crs.getG2Data());
+    const [g1, g2] = await Promise.all([crs.streamG1Data(), crs.streamG2Data()]);
+
+    await Promise.all([
+      finished(Readable.fromWeb(g1 as any).pipe(createWriteStream(this.path + '/bn254_g1.dat'))),
+      finished(Readable.fromWeb(g2 as any).pipe(createWriteStream(this.path + '/bn254_g2.dat'))),
+    ]);
   }
 
   /**
@@ -90,7 +95,7 @@ export class GrumpkinCrs {
     return crs;
   }
 
-  async init() {
+  async init(): Promise<void> {
     mkdirSync(this.path, { recursive: true });
 
     const g1FileSize = await stat(this.path + '/grumpkin_g1.dat')
@@ -104,8 +109,9 @@ export class GrumpkinCrs {
 
     this.logger(`Downloading Grumpkin CRS of size ${this.numPoints} into ${this.path}`);
     const crs = new NetGrumpkinCrs(this.numPoints);
-    await crs.init();
-    writeFileSync(this.path + '/grumpkin_g1.dat', crs.getG1Data());
+    const stream = await crs.streamG1Data();
+
+    await finished(Readable.fromWeb(stream as any).pipe(createWriteStream(this.path + '/grumpkin_g1.dat')));
     writeFileSync(this.path + '/grumpkin_size', String(crs.numPoints));
   }