Skip to content

fix: async blob #20559

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alexghr merged 1 commit into from
Feb 16, 2026
Merged

fix: async blob #20559

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion yarn-project/archiver/src/archiver-sync.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ describe('Archiver Sync', () => {
});

// Create a random blob that doesn't match the checkpoint
const randomBlob = makeRandomBlob(3);
const randomBlob = await makeRandomBlob(3);

// Override blob client to return the random blob instead of the correct one
blobClient.getBlobSidecar.mockResolvedValue([randomBlob]);
Expand Down
18 changes: 9 additions & 9 deletions yarn-project/archiver/src/test/fake_l1_state.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,9 +195,9 @@ export class FakeL1State {
this.addMessages(checkpointNumber, messagesL1BlockNumber, messages);

// Create the transaction and blobs
const tx = this.makeRollupTx(checkpoint, signers);
const blobHashes = this.makeVersionedBlobHashes(checkpoint);
const blobs = this.makeBlobsFromCheckpoint(checkpoint);
const tx = await this.makeRollupTx(checkpoint, signers);
const blobHashes = await this.makeVersionedBlobHashes(checkpoint);
const blobs = await this.makeBlobsFromCheckpoint(checkpoint);

// Store the checkpoint data
this.checkpoints.push({
Expand Down Expand Up @@ -539,14 +539,14 @@ export class FakeL1State {
}));
}

private makeRollupTx(checkpoint: Checkpoint, signers: Secp256k1Signer[]): Transaction {
private async makeRollupTx(checkpoint: Checkpoint, signers: Secp256k1Signer[]): Promise<Transaction> {
const attestations = signers
.map(signer => makeCheckpointAttestationFromCheckpoint(checkpoint, signer))
.map(attestation => CommitteeAttestation.fromSignature(attestation.signature))
.map(committeeAttestation => committeeAttestation.toViem());

const header = checkpoint.header.toViem();
const blobInput = getPrefixedEthBlobCommitments(getBlobsPerL1Block(checkpoint.toBlobFields()));
const blobInput = getPrefixedEthBlobCommitments(await getBlobsPerL1Block(checkpoint.toBlobFields()));
const archive = toHex(checkpoint.archive.root.toBuffer());
const attestationsAndSigners = new CommitteeAttestationsAndSigners(
attestations.map(attestation => CommitteeAttestation.fromViem(attestation)),
Expand Down Expand Up @@ -595,13 +595,13 @@ export class FakeL1State {
} as Transaction<bigint, number>;
}

private makeVersionedBlobHashes(checkpoint: Checkpoint): `0x${string}`[] {
return getBlobsPerL1Block(checkpoint.toBlobFields()).map(
private async makeVersionedBlobHashes(checkpoint: Checkpoint): Promise<`0x${string}`[]> {
return (await getBlobsPerL1Block(checkpoint.toBlobFields())).map(
b => `0x${b.getEthVersionedBlobHash().toString('hex')}` as `0x${string}`,
);
}

private makeBlobsFromCheckpoint(checkpoint: Checkpoint): Blob[] {
return getBlobsPerL1Block(checkpoint.toBlobFields());
private async makeBlobsFromCheckpoint(checkpoint: Checkpoint): Promise<Blob[]> {
return await getBlobsPerL1Block(checkpoint.toBlobFields());
}
}
18 changes: 9 additions & 9 deletions yarn-project/blob-client/src/blobstore/blob_store_test_suite.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
it('should store and retrieve a blob by hash', async () => {
// Create a test blob with random fields
const testFields = [Fr.random(), Fr.random(), Fr.random()];
const blob = Blob.fromFields(testFields);
const blob = await Blob.fromFields(testFields);
const blobHash = blob.getEthVersionedBlobHash();

// Store the blob
Expand All @@ -29,8 +29,8 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {

it('should handle multiple blobs stored and retrieved by their hashes', async () => {
// Create two different blobs
const blob1 = Blob.fromFields([Fr.random(), Fr.random()]);
const blob2 = Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]);
const blob1 = await Blob.fromFields([Fr.random(), Fr.random()]);
const blob2 = await Blob.fromFields([Fr.random(), Fr.random(), Fr.random()]);

const blobHash1 = blob1.getEthVersionedBlobHash();
const blobHash2 = blob2.getEthVersionedBlobHash();
Expand All @@ -57,9 +57,9 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {

it('should handle retrieving subset of stored blobs', async () => {
// Store multiple blobs
const blob1 = Blob.fromFields([Fr.random()]);
const blob2 = Blob.fromFields([Fr.random()]);
const blob3 = Blob.fromFields([Fr.random()]);
const blob1 = await Blob.fromFields([Fr.random()]);
const blob2 = await Blob.fromFields([Fr.random()]);
const blob3 = await Blob.fromFields([Fr.random()]);

await blobStore.addBlobs([blob1, blob2, blob3]);

Expand All @@ -75,7 +75,7 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
});

it('should handle duplicate blob hashes in request', async () => {
const blob = Blob.fromFields([Fr.random()]);
const blob = await Blob.fromFields([Fr.random()]);
const blobHash = blob.getEthVersionedBlobHash();

await blobStore.addBlobs([blob]);
Expand All @@ -91,8 +91,8 @@ export function describeBlobStore(getBlobStore: () => Promise<BlobStore>) {
it('should overwrite blob when storing with same hash', async () => {
// Create two blobs that will have the same hash (same content)
const fields = [Fr.random(), Fr.random()];
const blob1 = Blob.fromFields(fields);
const blob2 = Blob.fromFields(fields);
const blob1 = await Blob.fromFields(fields);
const blob2 = await Blob.fromFields(fields);

const blobHash = blob1.getEthVersionedBlobHash();

Expand Down
20 changes: 10 additions & 10 deletions yarn-project/blob-client/src/client/http.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { HttpBlobClient } from './http.js';
describe('HttpBlobClient', () => {
it('should handle no sources configured', async () => {
const client = new HttpBlobClient({});
const blob = Blob.fromFields([Fr.random()]);
const blob = await Blob.fromFields([Fr.random()]);
const blobHash = blob.getEthVersionedBlobHash();

const success = await client.sendBlobsToFilestore([blob]);
Expand All @@ -40,11 +40,11 @@ describe('HttpBlobClient', () => {
let latestSlotNumber: number;
let missedSlots: number[];

beforeEach(() => {
beforeEach(async () => {
latestSlotNumber = 1;
missedSlots = [];

testBlobs = Array.from({ length: 2 }, () => makeRandomBlob(3));
testBlobs = await Promise.all(Array.from({ length: 2 }, () => makeRandomBlob(3)));
testBlobsHashes = testBlobs.map(b => b.getEthVersionedBlobHash());

blobData = testBlobs.map(b => b.toJSON());
Expand Down Expand Up @@ -292,7 +292,7 @@ describe('HttpBlobClient', () => {
});

// Create a blob that has mismatch data and commitment.
const randomBlobs = Array.from({ length: 2 }, () => makeRandomBlob(3));
const randomBlobs = await Promise.all(Array.from({ length: 2 }, () => makeRandomBlob(3)));
const incorrectBlob = new Blob(randomBlobs[0].data, randomBlobs[1].commitment);
const incorrectBlobHash = incorrectBlob.getEthVersionedBlobHash();
// Update blobData to include the incorrect blob
Expand All @@ -312,7 +312,7 @@ describe('HttpBlobClient', () => {

it('should accumulate blobs across all three sources (filestore, consensus, archive)', async () => {
// Create three blobs for testing
const blobs = Array.from({ length: 3 }, () => makeRandomBlob(3));
const blobs = await Promise.all(Array.from({ length: 3 }, () => makeRandomBlob(3)));
const blobHashes = blobs.map(b => b.getEthVersionedBlobHash());

// Blob 0 only in filestore
Expand Down Expand Up @@ -368,7 +368,7 @@ describe('HttpBlobClient', () => {

it('should preserve blob order when requesting multiple blobs', async () => {
// Create three distinct blobs
const blobs = Array.from({ length: 3 }, () => makeRandomBlob(3));
const blobs = await Promise.all(Array.from({ length: 3 }, () => makeRandomBlob(3)));
const blobHashes = blobs.map(b => b.getEthVersionedBlobHash());

// Add all blobs to filestore
Expand Down Expand Up @@ -477,7 +477,7 @@ describe('HttpBlobClient', () => {

it('should return only one blob when multiple blobs with the same blobHash exist on a block', async () => {
// Create a blob data array with two blobs that have the same commitment (thus same blobHash)
const blob = makeRandomBlob(3);
const blob = await makeRandomBlob(3);
const blobHash = blob.getEthVersionedBlobHash();
const duplicateBlobData = [blob.toJSON(), blob.toJSON()];

Expand All @@ -503,7 +503,7 @@ describe('HttpBlobClient', () => {
l1ConsensusHostUrls: [`http://localhost:${consensusHostPort}`],
});

const blob = makeRandomBlob(3);
const blob = await makeRandomBlob(3);
const blobHash = blob.getEthVersionedBlobHash();
const blobJson = blob.toJSON();

Expand Down Expand Up @@ -616,8 +616,8 @@ describe('HttpBlobClient FileStore Integration', () => {
let testBlobs: Blob[];
let testBlobsHashes: Buffer[];

beforeEach(() => {
testBlobs = Array.from({ length: 2 }, () => makeRandomBlob(3));
beforeEach(async () => {
testBlobs = await Promise.all(Array.from({ length: 2 }, () => makeRandomBlob(3)));
testBlobsHashes = testBlobs.map(b => b.getEthVersionedBlobHash());
});

Expand Down
22 changes: 13 additions & 9 deletions yarn-project/blob-client/src/client/http.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,8 +215,8 @@ export class HttpBlobClient implements BlobClientInterface {
const getFilledBlobs = (): Blob[] => resultBlobs.filter((b): b is Blob => b !== undefined);

// Helper to fill in results from fetched blobs
const fillResults = (fetchedBlobs: BlobJson[]): Blob[] => {
const blobs = processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
const fillResults = async (fetchedBlobs: BlobJson[]): Promise<Blob[]> => {
const blobs = await processFetchedBlobs(fetchedBlobs, blobHashes, this.log);
// Fill in any missing positions with matching blobs
for (let i = 0; i < blobHashes.length; i++) {
if (resultBlobs[i] === undefined) {
Expand Down Expand Up @@ -269,7 +269,7 @@ export class HttpBlobClient implements BlobClientInterface {
...ctx,
});
const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex);
const result = fillResults(blobs);
const result = await fillResults(blobs);
this.log.debug(
`Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`,
{ slotNumber, l1ConsensusHostUrl, ...ctx },
Expand Down Expand Up @@ -312,7 +312,7 @@ export class HttpBlobClient implements BlobClientInterface {
this.log.debug('No blobs found from archive client', archiveCtx);
} else {
this.log.trace(`Got ${allBlobs.length} blobs from archive client before filtering`, archiveCtx);
const result = fillResults(allBlobs);
const result = await fillResults(allBlobs);
this.log.debug(
`Got ${allBlobs.length} blobs from archive client (total: ${result.length}/${blobHashes.length})`,
archiveCtx,
Expand Down Expand Up @@ -345,7 +345,7 @@ export class HttpBlobClient implements BlobClientInterface {
*/
private async tryFileStores(
getMissingBlobHashes: () => Buffer[],
fillResults: (blobs: BlobJson[]) => Blob[],
fillResults: (blobs: BlobJson[]) => Promise<Blob[]>,
ctx: { blockHash: string; blobHashes: string[] },
): Promise<void> {
// Shuffle clients for load distribution
Expand All @@ -366,7 +366,7 @@ export class HttpBlobClient implements BlobClientInterface {
});
const blobs = await client.getBlobsByHashes(blobHashStrings);
if (blobs.length > 0) {
const result = fillResults(blobs);
const result = await fillResults(blobs);
this.log.debug(
`Got ${blobs.length} blobs from filestore (total: ${result.length}/${ctx.blobHashes.length})`,
{
Expand All @@ -388,7 +388,7 @@ export class HttpBlobClient implements BlobClientInterface {
l1ConsensusHostIndex?: number,
): Promise<Blob[]> {
const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex);
return processFetchedBlobs(blobs, blobHashes, this.log).filter((b): b is Blob => b !== undefined);
return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b): b is Blob => b !== undefined);
}

public async getBlobsFromHost(
Expand Down Expand Up @@ -616,7 +616,11 @@ function parseBlobJson(data: any): BlobJson {

// Returns an array that maps each blob hash to the corresponding blob, or undefined if the blob is not found
// or the data does not match the commitment.
function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Logger): (Blob | undefined)[] {
async function processFetchedBlobs(
blobs: BlobJson[],
blobHashes: Buffer[],
logger: Logger,
): Promise<(Blob | undefined)[]> {
const requestedBlobHashes = new Set<string>(blobHashes.map(bufferToHex));
const hashToBlob = new Map<string, Blob>();
for (const blobJson of blobs) {
Expand All @@ -626,7 +630,7 @@ function processFetchedBlobs(blobs: BlobJson[], blobHashes: Buffer[], logger: Lo
}

try {
const blob = Blob.fromJson(blobJson);
const blob = await Blob.fromJson(blobJson);
hashToBlob.set(hashHex, blob);
} catch (err) {
// If the above throws, it's likely that the blob commitment does not match the hash of the blob data.
Expand Down
4 changes: 2 additions & 2 deletions yarn-project/blob-client/src/client/tests.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ export function runBlobClientTests(
});

it('should send and retrieve blobs by hash', async () => {
const blob = makeRandomBlob(5);
const blob = await makeRandomBlob(5);
const blobHash = blob.getEthVersionedBlobHash();

await client.sendBlobsToFilestore([blob]);
Expand All @@ -39,7 +39,7 @@ export function runBlobClientTests(
});

it('should handle multiple blobs', async () => {
const blobs = Array.from({ length: 3 }, () => makeRandomBlob(7));
const blobs = await Promise.all(Array.from({ length: 3 }, () => makeRandomBlob(7)));
const blobHashes = blobs.map(blob => blob.getEthVersionedBlobHash());

await client.sendBlobsToFilestore(blobs);
Expand Down
24 changes: 12 additions & 12 deletions yarn-project/blob-client/src/filestore/filestore_blob_client.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ describe('FileStoreBlobClient', () => {

describe('saveBlob', () => {
it('should save a blob to the filestore', async () => {
const blob = Blob.fromFields([Fr.random(), Fr.random()]);
const blob = await Blob.fromFields([Fr.random(), Fr.random()]);
const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`;

await client.saveBlob(blob);
Expand All @@ -88,7 +88,7 @@ describe('FileStoreBlobClient', () => {
});

it('should skip saving if blob already exists and skipIfExists=true', async () => {
const blob = Blob.fromFields([Fr.random()]);
const blob = await Blob.fromFields([Fr.random()]);
const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`;

// Save first time
Expand All @@ -107,7 +107,7 @@ describe('FileStoreBlobClient', () => {
});

it('should overwrite if skipIfExists=false', async () => {
const blob = Blob.fromFields([Fr.random()]);
const blob = await Blob.fromFields([Fr.random()]);
const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`;

// Save first time
Expand All @@ -130,8 +130,8 @@ describe('FileStoreBlobClient', () => {

describe('saveBlobs', () => {
it('should save multiple blobs', async () => {
const blob1 = Blob.fromFields([Fr.random()]);
const blob2 = Blob.fromFields([Fr.random()]);
const blob1 = await Blob.fromFields([Fr.random()]);
const blob2 = await Blob.fromFields([Fr.random()]);

await client.saveBlobs([blob1, blob2]);

Expand All @@ -145,7 +145,7 @@ describe('FileStoreBlobClient', () => {

describe('getBlobsByHashes', () => {
it('should retrieve blobs by their versioned hashes', async () => {
const blob = Blob.fromFields([Fr.random(), Fr.random()]);
const blob = await Blob.fromFields([Fr.random(), Fr.random()]);
const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`;

await client.saveBlob(blob);
Expand All @@ -163,8 +163,8 @@ describe('FileStoreBlobClient', () => {
});

it('should retrieve multiple blobs', async () => {
const blob1 = Blob.fromFields([Fr.random()]);
const blob2 = Blob.fromFields([Fr.random()]);
const blob1 = await Blob.fromFields([Fr.random()]);
const blob2 = await Blob.fromFields([Fr.random()]);

await client.saveBlobs([blob1, blob2]);

Expand All @@ -177,7 +177,7 @@ describe('FileStoreBlobClient', () => {
});

it('should skip blobs that fail to parse', async () => {
const blob = Blob.fromFields([Fr.random()]);
const blob = await Blob.fromFields([Fr.random()]);
const hash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`;

// Save invalid JSON
Expand All @@ -191,7 +191,7 @@ describe('FileStoreBlobClient', () => {

describe('exists', () => {
it('should return true if blob exists', async () => {
const blob = Blob.fromFields([Fr.random()]);
const blob = await Blob.fromFields([Fr.random()]);
const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`;

await client.saveBlob(blob);
Expand Down Expand Up @@ -240,14 +240,14 @@ describe('FileStoreBlobClient', () => {
const readOnlyStore = new MockReadOnlyFileStore();
const readOnlyClient = new FileStoreBlobClient(readOnlyStore, basePath);

const blob = Blob.fromFields([Fr.random()]);
const blob = await Blob.fromFields([Fr.random()]);

await expect(readOnlyClient.saveBlob(blob)).rejects.toThrow('FileStore is read-only');
});

it('should be able to read from read-only store', async () => {
const files = new Map<string, Buffer>();
const blob = Blob.fromFields([Fr.random()]);
const blob = await Blob.fromFields([Fr.random()]);
const versionedHash = `0x${blob.getEthVersionedBlobHash().toString('hex')}`;
const path = `${basePath}/blobs/${versionedHash}.data`;

Expand Down
Loading
Loading