diff --git a/yarn-project/archiver/src/l1/data_retrieval.ts b/yarn-project/archiver/src/l1/data_retrieval.ts index 4f5a529f1aae..599a33bc1bfb 100644 --- a/yarn-project/archiver/src/l1/data_retrieval.ts +++ b/yarn-project/archiver/src/l1/data_retrieval.ts @@ -265,6 +265,9 @@ async function processCheckpointProposedLogs( checkpointNumber, expectedHashes, ); + const { timestamp, parentBeaconBlockRoot } = await getL1Block(publicClient, log.l1BlockNumber); + const l1 = new L1PublishedData(log.l1BlockNumber, timestamp, log.l1BlockHash.toString()); + const checkpointBlobData = await getCheckpointBlobDataFromBlobs( blobClient, checkpoint.blockHash, @@ -272,12 +275,8 @@ async function processCheckpointProposedLogs( checkpointNumber, logger, isHistoricalSync, - ); - - const l1 = new L1PublishedData( - log.l1BlockNumber, - await getL1BlockTime(publicClient, log.l1BlockNumber), - log.l1BlockHash.toString(), + parentBeaconBlockRoot, + timestamp, ); retrievedCheckpoints.push({ ...checkpoint, checkpointBlobData, l1, chainId, version }); @@ -298,9 +297,12 @@ async function processCheckpointProposedLogs( return retrievedCheckpoints; } -export async function getL1BlockTime(publicClient: ViemPublicClient, blockNumber: bigint): Promise { +export async function getL1Block( + publicClient: ViemPublicClient, + blockNumber: bigint, +): Promise<{ timestamp: bigint; parentBeaconBlockRoot: string | undefined }> { const block = await publicClient.getBlock({ blockNumber, includeTransactions: false }); - return block.timestamp; + return { timestamp: block.timestamp, parentBeaconBlockRoot: block.parentBeaconBlockRoot }; } export async function getCheckpointBlobDataFromBlobs( @@ -310,8 +312,14 @@ export async function getCheckpointBlobDataFromBlobs( checkpointNumber: CheckpointNumber, logger: Logger, isHistoricalSync: boolean, + parentBeaconBlockRoot?: string, + l1BlockTimestamp?: bigint, ): Promise { - const blobBodies = await blobClient.getBlobSidecar(blockHash, blobHashes, { isHistoricalSync }); + const blobBodies = await blobClient.getBlobSidecar(blockHash, blobHashes, { + isHistoricalSync, + parentBeaconBlockRoot, + l1BlockTimestamp, + }); if (blobBodies.length === 0) { throw new NoBlobBodiesFoundError(checkpointNumber); } diff --git a/yarn-project/blob-client/src/client/http.test.ts b/yarn-project/blob-client/src/client/http.test.ts index c5c7d8eb832f..ceb41961c796 100644 --- a/yarn-project/blob-client/src/client/http.test.ts +++ b/yarn-project/blob-client/src/client/http.test.ts @@ -85,16 +85,22 @@ describe('HttpBlobClient', () => { return; } - if (req.url?.includes('/eth/v1/beacon/headers/')) { + if (req.url?.includes('/eth/v1/config/genesis')) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ data: { genesisTime: '1000' } })); + } else if (req.url?.includes('/eth/v1/config/spec')) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ data: { secondsPerSlot: '12' } })); + } else if (req.url?.includes('/eth/v1/beacon/headers/')) { res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ data: { header: { message: { slot: latestSlotNumber } } } })); - } else if (req.url?.includes('/eth/v1/beacon/blob_sidecars/')) { + } else if (req.url?.includes('/eth/v1/beacon/blobs/')) { if (missedSlots.some(slot => req.url?.includes(`/${slot}`))) { res.writeHead(404, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: 'Not Found' })); } else { res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ data: blobData })); + res.end(JSON.stringify({ data: blobData.map(b => b.blob) })); } } else { res.writeHead(404, { 'Content-Type': 'application/json' }); @@ -135,6 +141,61 @@ describe('HttpBlobClient', () => { expect(retrievedBlobs[1].commitment).toEqual(testBlobs[1].commitment); }); + it('should compute slot from l1BlockTimestamp without headers call when genesis config is cached', async () => { + await startExecutionHostServer(); + await startConsensusHostServer(); + + const client = new HttpBlobClient({ + l1RpcUrls: [`http://localhost:${executionHostPort}`], + l1ConsensusHostUrls: [`http://localhost:${consensusHostPort}`], + }); + + // Call start() to fetch and cache genesis config (genesis_time=1000, SECONDS_PER_SLOT=12) + await client.start(); + + const fetchSpy = jest.spyOn(client as any, 'fetch'); + + // slot = (l1BlockTimestamp - genesis_time) / seconds_per_slot = (1024 - 1000) / 12 = 2 + // so blobs should be fetched at slot 2 + const retrievedBlobs = await client.getBlobSidecar('0x1234', testBlobsHashes, { + l1BlockTimestamp: 1024n, + }); + + expect(retrievedBlobs).toHaveLength(2); + expect(retrievedBlobs[0].commitment).toEqual(testBlobs[0].commitment); + expect(retrievedBlobs[1].commitment).toEqual(testBlobs[1].commitment); + + // Headers call for slot resolution should NOT have been made + expect(fetchSpy).not.toHaveBeenCalledWith( + expect.stringContaining('/eth/v1/beacon/headers/0x'), + expect.anything(), + ); + // Blobs fetched at the computed slot (2) + expect(fetchSpy).toHaveBeenCalledWith(expect.stringContaining('/eth/v1/beacon/blobs/2'), expect.anything()); + }); + + it('should fall back to headers call when l1BlockTimestamp is not provided', async () => { + await startExecutionHostServer(); + await startConsensusHostServer(); + + const client = new HttpBlobClient({ + l1RpcUrls: [`http://localhost:${executionHostPort}`], + l1ConsensusHostUrls: [`http://localhost:${consensusHostPort}`], + }); + + // Call start() to cache genesis config, but do NOT pass l1BlockTimestamp + await client.start(); + + const fetchSpy = jest.spyOn(client as any, 'fetch'); + + // No l1BlockTimestamp — should fall back to headers call + const retrievedBlobs = await client.getBlobSidecar('0x1234', testBlobsHashes); + + expect(retrievedBlobs).toHaveLength(2); + // Headers call for slot resolution SHOULD have been made (via parentBeaconBlockRoot from execution RPC) + expect(fetchSpy).toHaveBeenCalledWith(expect.stringContaining('/eth/v1/beacon/headers/'), expect.anything()); + }); + it('should handle when multiple consensus hosts are provided', async () => { await startExecutionHostServer(); await startConsensusHostServer(); @@ -423,11 +484,11 @@ describe('HttpBlobClient', () => { // Verify we hit the 404 for slot 33 before trying slot 34, and that we use the api key header // (see issue https://github.com/AztecProtocol/aztec-packages/issues/13415) expect(fetchSpy).toHaveBeenCalledWith( - expect.stringContaining('/eth/v1/beacon/blob_sidecars/33'), + expect.stringContaining('/eth/v1/beacon/blobs/33'), expect.objectContaining({ headers: { ['X-API-KEY']: 'my-api-key' } }), ); expect(fetchSpy).toHaveBeenCalledWith( - expect.stringContaining('/eth/v1/beacon/blob_sidecars/34'), + expect.stringContaining('/eth/v1/beacon/blobs/34'), expect.objectContaining({ headers: { ['X-API-KEY']: 'my-api-key' } }), ); }); @@ -458,10 +519,7 @@ describe('HttpBlobClient', () => { expect(fetchSpy).toHaveBeenCalledTimes(latestSlotNumber - 33 + 2); for (let i = 33; i <= latestSlotNumber; i++) { - expect(fetchSpy).toHaveBeenCalledWith( - expect.stringContaining(`/eth/v1/beacon/blob_sidecars/${i}`), - expect.anything(), - ); + expect(fetchSpy).toHaveBeenCalledWith(expect.stringContaining(`/eth/v1/beacon/blobs/${i}`), expect.anything()); } }); @@ -519,27 +577,12 @@ describe('HttpBlobClient', () => { ]; expect(await client.getBlobSidecar('0x1234', [blobHash])).toEqual([]); - // Incorrect bytes for the commitment. - blobData = [ - ...originalBlobData, - { - ...blobJson, - // eslint-disable-next-line camelcase - kzg_commitment: 'abcdefghijk', - }, - ]; - expect(await client.getBlobSidecar('0x1234', [blobHash])).toEqual([]); - - // Commitment does not exist. - blobData = [ - ...originalBlobData, - { - blob: blobJson.blob, - } as BlobJson, - ]; + // Blob from a different hash, commitment is computed correctly but doesn't match requested hash. + const otherBlob = await makeRandomBlob(3); + blobData = [...originalBlobData, otherBlob.toJSON()]; expect(await client.getBlobSidecar('0x1234', [blobHash])).toEqual([]); - // Correct blob json. + // Correct blob hex json. blobData = [...originalBlobData, blobJson]; const result = await client.getBlobSidecar('0x1234', [blobHash]); expect(result).toHaveLength(1); @@ -906,9 +949,9 @@ describe('HttpBlobClient FileStore Integration', () => { if (req.url?.includes('/eth/v1/beacon/headers/')) { res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ data: { header: { message: { slot: 1 } } } })); - } else if (req.url?.includes('/eth/v1/beacon/blob_sidecars/')) { + } else if (req.url?.includes('/eth/v1/beacon/blobs/')) { res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ data: blobData })); + res.end(JSON.stringify({ data: blobData.map(b => b.blob) })); } else { res.writeHead(404, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: 'Not Found' })); @@ -951,8 +994,8 @@ describe('HttpBlobClient FileStore Integration', () => { const retrievedBlobs = await client.getBlobSidecar('0x1234', testBlobsHashes); expect(retrievedBlobs).toHaveLength(2); - // Consensus should not be called for blob_sidecars since filestore had all blobs - expect(fetchSpy).not.toHaveBeenCalledWith(expect.stringContaining('blob_sidecars'), expect.anything()); + // Consensus should not be called for blobs since filestore had all blobs + expect(fetchSpy).not.toHaveBeenCalledWith(expect.stringContaining('/beacon/blobs/'), expect.anything()); }); it('should fall back to consensus when filestore has partial blobs', async () => { diff --git a/yarn-project/blob-client/src/client/http.ts b/yarn-project/blob-client/src/client/http.ts index 5d626933261a..ffa48d2cbe74 100644 --- a/yarn-project/blob-client/src/client/http.ts +++ b/yarn-project/blob-client/src/client/http.ts @@ -24,6 +24,11 @@ export class HttpBlobClient implements BlobClientInterface { private disabled = false; private healthcheckUploadIntervalId?: NodeJS.Timeout; + /** Cached beacon genesis time (seconds since Unix epoch). Fetched once at startup. */ + private beaconGenesisTime?: bigint; + /** Cached beacon slot duration in seconds. Fetched once at startup. */ + private beaconSecondsPerSlot?: number; + constructor( config?: BlobClientConfig, private readonly opts: { @@ -251,7 +256,7 @@ export class HttpBlobClient implements BlobClientInterface { // The beacon api can query by slot number, so we get that first const consensusCtx = { l1ConsensusHostUrls, ...ctx }; this.log.trace(`Attempting to get slot number for block hash`, consensusCtx); - const slotNumber = await this.getSlotNumber(blockHash); + const slotNumber = await this.getSlotNumber(blockHash, opts?.parentBeaconBlockRoot, opts?.l1BlockTimestamp); this.log.debug(`Got slot number ${slotNumber} from consensus host for querying blobs`, consensusCtx); if (slotNumber) { @@ -268,7 +273,12 @@ export class HttpBlobClient implements BlobClientInterface { l1ConsensusHostUrl, ...ctx, }); - const blobs = await this.getBlobsFromHost(l1ConsensusHostUrl, slotNumber, l1ConsensusHostIndex); + const blobs = await this.getBlobsFromHost( + l1ConsensusHostUrl, + slotNumber, + l1ConsensusHostIndex, + getMissingBlobHashes(), + ); const result = await fillResults(blobs); this.log.debug( `Got ${blobs.length} blobs from consensus host (total: ${result.length}/${blobHashes.length})`, @@ -387,7 +397,7 @@ export class HttpBlobClient implements BlobClientInterface { blobHashes: Buffer[] = [], l1ConsensusHostIndex?: number, ): Promise { - const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex); + const blobs = await this.getBlobsFromHost(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes); return (await processFetchedBlobs(blobs, blobHashes, this.log)).filter((b): b is Blob => b !== undefined); } @@ -395,11 +405,12 @@ export class HttpBlobClient implements BlobClientInterface { hostUrl: string, blockHashOrSlot: string | number, l1ConsensusHostIndex?: number, + blobHashes?: Buffer[], ): Promise { try { - let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex); + let res = await this.fetchBlobSidecars(hostUrl, blockHashOrSlot, l1ConsensusHostIndex, blobHashes); if (res.ok) { - return parseBlobJsonsFromResponse(await res.json(), this.log); + return await parseBlobJsonsFromResponse(await res.json(), this.log); } if (res.status === 404 && typeof blockHashOrSlot === 'number') { @@ -414,9 +425,9 @@ export class HttpBlobClient implements BlobClientInterface { let currentSlot = blockHashOrSlot + 1; while (res.status === 404 && maxRetries > 0 && latestSlot !== undefined && currentSlot <= latestSlot) { this.log.debug(`Trying slot ${currentSlot}`); - res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex); + res = await this.fetchBlobSidecars(hostUrl, currentSlot, l1ConsensusHostIndex, blobHashes); if (res.ok) { - return parseBlobJsonsFromResponse(await res.json(), this.log); + return await parseBlobJsonsFromResponse(await res.json(), this.log); } currentSlot++; maxRetries--; @@ -439,8 +450,17 @@ export class HttpBlobClient implements BlobClientInterface { hostUrl: string, blockHashOrSlot: string | number, l1ConsensusHostIndex?: number, + blobHashes?: Buffer[], ): Promise { - const baseUrl = `${hostUrl}/eth/v1/beacon/blob_sidecars/${blockHashOrSlot}`; + let baseUrl = `${hostUrl}/eth/v1/beacon/blobs/${blockHashOrSlot}`; + + if (blobHashes && blobHashes.length > 0) { + const params = new URLSearchParams(); + for (const hash of blobHashes) { + params.append('versioned_hashes', `0x${hash.toString('hex')}`); + } + baseUrl += `?${params.toString()}`; + } const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config, l1ConsensusHostIndex); this.log.debug(`Fetching blob sidecar for ${blockHashOrSlot}`, { url, ...options }); @@ -482,34 +502,50 @@ export class HttpBlobClient implements BlobClientInterface { * @param blockHash - The block hash * @returns The slot number */ - private async getSlotNumber(blockHash: `0x${string}`): Promise { + private async getSlotNumber( + blockHash: `0x${string}`, + parentBeaconBlockRoot?: string, + l1BlockTimestamp?: bigint, + ): Promise { const { l1ConsensusHostUrls, l1RpcUrls } = this.config; if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) { this.log.debug('No consensus host url configured'); return undefined; } - if (!l1RpcUrls || l1RpcUrls.length === 0) { - this.log.debug('No execution host url configured'); - return undefined; + // Primary path: compute slot from timestamp if genesis config is cached (no network call needed) + if ( + l1BlockTimestamp !== undefined && + this.beaconGenesisTime !== undefined && + this.beaconSecondsPerSlot !== undefined + ) { + const slot = Number((l1BlockTimestamp - this.beaconGenesisTime) / BigInt(this.beaconSecondsPerSlot)); + this.log.debug(`Computed slot ${slot} from L1 block timestamp`, { l1BlockTimestamp }); + return slot; } - // Ping execution node to get the parentBeaconBlockRoot for this block - let parentBeaconBlockRoot: string | undefined; - const client = createPublicClient({ - transport: fallback(l1RpcUrls.map(url => http(url, { batch: false }))), - }); - try { - const res: RpcBlock = await client.request({ - method: 'eth_getBlockByHash', - params: [blockHash, /*tx flag*/ false], + if (!parentBeaconBlockRoot) { + // parentBeaconBlockRoot not provided by caller — fetch it from the execution RPC + if (!l1RpcUrls || l1RpcUrls.length === 0) { + this.log.debug('No execution host url configured'); + return undefined; + } + + const client = createPublicClient({ + transport: fallback(l1RpcUrls.map(url => http(url, { batch: false }))), }); + try { + const res: RpcBlock = await client.request({ + method: 'eth_getBlockByHash', + params: [blockHash, /*tx flag*/ false], + }); - if (res.parentBeaconBlockRoot) { - parentBeaconBlockRoot = res.parentBeaconBlockRoot; + if (res.parentBeaconBlockRoot) { + parentBeaconBlockRoot = res.parentBeaconBlockRoot; + } + } catch (err) { + this.log.error(`Error getting parent beacon block root`, err); } - } catch (err) { - this.log.error(`Error getting parent beacon block root`, err); } if (!parentBeaconBlockRoot) { @@ -555,9 +591,12 @@ export class HttpBlobClient implements BlobClientInterface { /** * Start the blob client. - * Uploads the initial healthcheck file (awaited) and starts periodic uploads. + * Fetches and caches beacon genesis config for timestamp-based slot resolution, + * then uploads the initial healthcheck file (awaited) and starts periodic uploads. */ public async start(): Promise { + await this.fetchBeaconConfig(); + if (!this.fileStoreUploadClient) { return; } @@ -582,6 +621,53 @@ export class HttpBlobClient implements BlobClientInterface { }, intervalMs); } + /** + * Fetches and caches beacon genesis time and slot duration from the first available consensus host. + * These static values enable timestamp-based slot resolution, eliminating the per-fetch headers call. + * Logs a warning and leaves fields undefined if all hosts fail, callers fall back gracefully. + */ + private async fetchBeaconConfig(): Promise { + const { l1ConsensusHostUrls } = this.config; + if (!l1ConsensusHostUrls || l1ConsensusHostUrls.length === 0) { + return; + } + + for (let i = 0; i < l1ConsensusHostUrls.length; i++) { + try { + const { url: genesisUrl, ...genesisOptions } = getBeaconNodeFetchOptions( + `${l1ConsensusHostUrls[i]}/eth/v1/config/genesis`, + this.config, + i, + ); + const { url: specUrl, ...specOptions } = getBeaconNodeFetchOptions( + `${l1ConsensusHostUrls[i]}/eth/v1/config/spec`, + this.config, + i, + ); + + const [genesisRes, specRes] = await Promise.all([ + this.fetch(genesisUrl, genesisOptions), + this.fetch(specUrl, specOptions), + ]); + + if (genesisRes.ok && specRes.ok) { + const genesis = await genesisRes.json(); + const spec = await specRes.json(); + this.beaconGenesisTime = BigInt(genesis.data.genesisTime); + this.beaconSecondsPerSlot = parseInt(spec.data.secondsPerSlot); + this.log.debug(`Fetched beacon genesis config`, { + genesisTime: this.beaconGenesisTime, + secondsPerSlot: this.beaconSecondsPerSlot, + }); + return; + } + } catch (err) { + this.log.warn(`Failed to fetch beacon config from host ${l1ConsensusHostUrls[i]}`, err); + } + } + this.log.warn('Could not fetch beacon genesis config from any consensus host — will use headers call fallback'); + } + /** * Stop the blob client, clearing any periodic tasks. */ @@ -593,10 +679,9 @@ export class HttpBlobClient implements BlobClientInterface { } } -function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] { +async function parseBlobJsonsFromResponse(response: any, logger: Logger): Promise { try { - const blobs = response.data.map(parseBlobJson); - return blobs; + return await Promise.all((response.data as string[]).map(parseBlobJson)); } catch (err) { logger.error(`Error parsing blob json from response`, err); return []; @@ -607,10 +692,9 @@ function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] { // https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlobSidecars // Here we attempt to parse the response data to Buffer, and check the lengths (via Blob's constructor), to avoid // throwing an error down the line when calling Blob.fromJson(). -function parseBlobJson(data: any): BlobJson { - const blobBuffer = Buffer.from(data.blob.slice(2), 'hex'); - const commitmentBuffer = Buffer.from(data.kzg_commitment.slice(2), 'hex'); - const blob = new Blob(blobBuffer, commitmentBuffer); +async function parseBlobJson(rawHex: string): Promise { + const blobBuffer = Buffer.from(rawHex.slice(2), 'hex'); + const blob = await Blob.fromBlobBuffer(blobBuffer); return blob.toJSON(); } diff --git a/yarn-project/blob-client/src/client/interface.ts b/yarn-project/blob-client/src/client/interface.ts index b9dfd8e30728..6959e31edbdc 100644 --- a/yarn-project/blob-client/src/client/interface.ts +++ b/yarn-project/blob-client/src/client/interface.ts @@ -11,6 +11,17 @@ export interface GetBlobSidecarOptions { * - Near tip: FileStore first with no retries (data should exist), L1 consensus second (freshest data), then FileStore with retries, then archive (eg. blobscan) */ isHistoricalSync?: boolean; + /** + * The parent beacon block root for the L1 block containing the blobs. + * If provided, skips the eth_getBlockByHash execution RPC call inside getSlotNumber. + */ + parentBeaconBlockRoot?: string; + /** + * The timestamp of the L1 execution block containing the blobs. + * When provided alongside a cached beacon genesis config (fetched at startup), allows computing + * the beacon slot directly via timestamp math, skipping the beacon headers network call entirely. + */ + l1BlockTimestamp?: bigint; } export interface BlobClientInterface {