From 137ceed2378f5753d667e45d3cc9c9dab38062ef Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Mon, 24 Mar 2025 18:43:25 +0000 Subject: [PATCH] feat: stream crs data to disk --- barretenberg/ts/src/crs/net_crs.ts | 115 +++++++++++++++++--------- barretenberg/ts/src/crs/node/index.ts | 22 +++-- 2 files changed, 92 insertions(+), 45 deletions(-) diff --git a/barretenberg/ts/src/crs/net_crs.ts b/barretenberg/ts/src/crs/net_crs.ts index 8f1d2789146b..50708b9106e9 100644 --- a/barretenberg/ts/src/crs/net_crs.ts +++ b/barretenberg/ts/src/crs/net_crs.ts @@ -21,25 +21,24 @@ export class NetCrs { await this.downloadG2Data(); } - async downloadG1Data() { - // Skip the download if numPoints is 0 (would download the entire file due to bad range header otherwise) - if (this.numPoints === 0) { - return (this.data = new Uint8Array([])); - } - - const g1End = this.numPoints * 64 - 1; + /** + * Opens up a ReadableStream to the points data + */ + async streamG1Data(): Promise> { + const response = await this.fetchG1Data(); + return response.body!; + } - const response = await retry( - () => - fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g1.dat', { - headers: { - Range: `bytes=0-${g1End}`, - }, - cache: 'force-cache', - }), - makeBackoff([5, 5, 5]), - ); + /** + * Opens up a ReadableStream to the points data + */ + async streamG2Data(): Promise> { + const response = await this.fetchG2Data(); + return response.body!; + } + async downloadG1Data() { + const response = await this.fetchG1Data(); return (this.data = new Uint8Array(await response.arrayBuffer())); } @@ -47,14 +46,7 @@ export class NetCrs { * Download the G2 points data. */ async downloadG2Data() { - const response2 = await retry( - () => - fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g2.dat', { - cache: 'force-cache', - }), - makeBackoff([5, 5, 5]), - ); - + const response2 = await this.fetchG2Data(); return (this.g2Data = new Uint8Array(await response2.arrayBuffer())); } @@ -73,6 +65,41 @@ export class NetCrs { getG2Data(): Uint8Array { return this.g2Data; } + + /** + * Fetches the appropriate range of points from a remote source + */ + private async fetchG1Data(): Promise { + // Skip the download if numPoints is 0 (would download the entire file due to bad range header otherwise) + if (this.numPoints === 0) { + return new Response(new Uint8Array([])); + } + + const g1End = this.numPoints * 64 - 1; + return await retry( + () => + fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g1.dat', { + headers: { + Range: `bytes=0-${g1End}`, + }, + cache: 'force-cache', + }), + makeBackoff([5, 5, 5]), + ); + } + + /** + * Fetches the appropriate range of points from a remote source + */ + private async fetchG2Data(): Promise { + return await retry( + () => + fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/flat/g2.dat', { + cache: 'force-cache', + }), + makeBackoff([5, 5, 5]), + ); + } } /** @@ -96,29 +123,43 @@ export class NetGrumpkinCrs { } async downloadG1Data() { + const response = await this.fetchG1Data(); + return (this.data = new Uint8Array(await response.arrayBuffer())); + } + + /** + * Opens up a ReadableStream to the points data + */ + async streamG1Data(): Promise> { + const response = await this.fetchG1Data(); + return response.body!; + } + + /** + * G1 points data for prover key. + * @returns The points data. + */ + getG1Data(): Uint8Array { + return this.data; + } + + /** + * Fetches the appropriate range of points from a remote source + */ + private async fetchG1Data(): Promise { // Skip the download if numPoints is 0 (would download the entire file due to bad range header otherwise) if (this.numPoints === 0) { - return (this.data = new Uint8Array([])); + return new Response(new Uint8Array([])); } const g1Start = 28; const g1End = g1Start + (this.numPoints * 64 - 1); - const response = await fetch('https://aztec-ignition.s3.amazonaws.com/TEST%20GRUMPKIN/monomial/transcript00.dat', { + return await fetch('https://aztec-ignition.s3.amazonaws.com/TEST%20GRUMPKIN/monomial/transcript00.dat', { headers: { Range: `bytes=${g1Start}-${g1End}`, }, cache: 'force-cache', }); - - return (this.data = new Uint8Array(await response.arrayBuffer())); - } - - /** - * G1 points data for prover key. - * @returns The points data. - */ - getG1Data(): Uint8Array { - return this.data; } } diff --git a/barretenberg/ts/src/crs/node/index.ts b/barretenberg/ts/src/crs/node/index.ts index 2eee8cb6f6db..996da5f80d3b 100644 --- a/barretenberg/ts/src/crs/node/index.ts +++ b/barretenberg/ts/src/crs/node/index.ts @@ -1,8 +1,10 @@ import { NetCrs, NetGrumpkinCrs } from '../net_crs.js'; -import { closeSync, mkdirSync, openSync, readFileSync, readSync, writeFileSync } from 'fs'; +import { closeSync, mkdirSync, openSync, readFileSync, readSync, writeFileSync, createWriteStream } from 'fs'; import { stat } from 'fs/promises'; +import { Readable } from 'stream'; import createDebug from 'debug'; import { homedir } from 'os'; +import { finished } from 'stream/promises'; /** * Generic CRS finder utility class. @@ -24,7 +26,7 @@ export class Crs { return crs; } - async init() { + async init(): Promise { mkdirSync(this.path, { recursive: true }); const g1FileSize = await stat(this.path + '/bn254_g1.dat') @@ -41,9 +43,12 @@ export class Crs { this.logger(`Downloading CRS of size ${this.numPoints} into ${this.path}`); const crs = new NetCrs(this.numPoints); - await crs.init(); - writeFileSync(this.path + '/bn254_g1.dat', crs.getG1Data()); - writeFileSync(this.path + '/bn254_g2.dat', crs.getG2Data()); + const [g1, g2] = await Promise.all([crs.streamG1Data(), crs.streamG2Data()]); + + await Promise.all([ + finished(Readable.fromWeb(g1 as any).pipe(createWriteStream(this.path + '/bn254_g1.dat'))), + finished(Readable.fromWeb(g2 as any).pipe(createWriteStream(this.path + '/bn254_g2.dat'))), + ]); } /** @@ -90,7 +95,7 @@ export class GrumpkinCrs { return crs; } - async init() { + async init(): Promise { mkdirSync(this.path, { recursive: true }); const g1FileSize = await stat(this.path + '/grumpkin_g1.dat') @@ -104,8 +109,9 @@ export class GrumpkinCrs { this.logger(`Downloading Grumpkin CRS of size ${this.numPoints} into ${this.path}`); const crs = new NetGrumpkinCrs(this.numPoints); - await crs.init(); - writeFileSync(this.path + '/grumpkin_g1.dat', crs.getG1Data()); + const stream = await crs.streamG1Data(); + + await finished(Readable.fromWeb(stream as any).pipe(createWriteStream(this.path + '/grumpkin_g1.dat'))); writeFileSync(this.path + '/grumpkin_size', String(crs.numPoints)); }