-
Notifications
You must be signed in to change notification settings - Fork 106
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adds configurable block providers to allow using bitswap but also other methods such as trustless gateways and any yet-to-be-invented way of resolving a CID to a block..
- Loading branch information
1 parent
51316ba
commit d264154
Showing
12 changed files
with
363 additions
and
72 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 59 additions & 0 deletions
59
packages/helia/src/block-providers/bitswap-block-provider.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| import { createBitswap } from 'ipfs-bitswap' | ||
| import type { BlockProvider } from '@helia/interface/blocks' | ||
| import type { Libp2p } from '@libp2p/interface' | ||
| import type { Startable } from '@libp2p/interface/startable' | ||
| import type { Blockstore } from 'interface-blockstore' | ||
| import type { AbortOptions } from 'interface-store' | ||
| import type { Bitswap, BitswapNotifyProgressEvents, BitswapWantBlockProgressEvents } from 'ipfs-bitswap' | ||
| import type { CID } from 'multiformats/cid' | ||
| import type { MultihashHasher } from 'multiformats/hashes/interface' | ||
| import type { ProgressOptions } from 'progress-events' | ||
|
|
||
| export class BitswapBlockProvider implements BlockProvider< | ||
| ProgressOptions<BitswapNotifyProgressEvents>, | ||
| ProgressOptions<BitswapWantBlockProgressEvents> | ||
| >, Startable { | ||
| private readonly bitswap: Bitswap | ||
| private started: boolean | ||
|
|
||
| constructor (libp2p: Libp2p, blockstore: Blockstore, hashers: MultihashHasher[]) { | ||
| this.bitswap = createBitswap(libp2p, blockstore, { | ||
| hashLoader: { | ||
| getHasher: async (codecOrName: string | number): Promise<MultihashHasher<number>> => { | ||
| const hasher = hashers.find(hasher => { | ||
| return hasher.code === codecOrName || hasher.name === codecOrName | ||
| }) | ||
|
|
||
| if (hasher != null) { | ||
| return hasher | ||
| } | ||
|
|
||
| throw new Error(`Could not load hasher for code/name "${codecOrName}"`) | ||
| } | ||
| } | ||
| }) | ||
| this.started = false | ||
| } | ||
|
|
||
| isStarted (): boolean { | ||
| return this.started | ||
| } | ||
|
|
||
| async start (): Promise<void> { | ||
| await this.bitswap.start() | ||
| this.started = true | ||
| } | ||
|
|
||
| async stop (): Promise<void> { | ||
| await this.bitswap.stop() | ||
| this.started = false | ||
| } | ||
|
|
||
| notify (cid: CID, block: Uint8Array, options?: ProgressOptions<BitswapNotifyProgressEvents>): void { | ||
| this.bitswap.notify(cid, block, options) | ||
| } | ||
|
|
||
| async get (cid: CID, options?: AbortOptions & ProgressOptions<BitswapWantBlockProgressEvents>): Promise<Uint8Array> { | ||
| return this.bitswap.want(cid, options) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| export { BitswapBlockProvider } from './bitswap-block-provider.js' | ||
| export { TrustedGatewayBlockProvider } from './trustless-gateway-block-provider.js' |
83 changes: 83 additions & 0 deletions
83
packages/helia/src/block-providers/trustless-gateway-block-provider.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| import { logger } from '@libp2p/logger' | ||
| import type { BlockProvider } from '@helia/interface/blocks' | ||
| import type { AbortOptions } from 'interface-store' | ||
| import type { CID } from 'multiformats/cid' | ||
| import type { ProgressEvent, ProgressOptions } from 'progress-events' | ||
|
|
||
| const log = logger('helia:trustless-gateway-block-provider') | ||
|
|
||
| export type TrustlessGatewayGetBlockProgressEvents = | ||
| ProgressEvent<'trustless-gateway:get-block:fetch', URL> | ||
|
|
||
| /** | ||
| * A BlockProvider that accepts a list of trustless gateways that are queried | ||
| * for blocks. Individual gateways are randomly chosen. | ||
| */ | ||
| export class TrustedGatewayBlockProvider implements BlockProvider< | ||
| ProgressOptions, | ||
| ProgressOptions<TrustlessGatewayGetBlockProgressEvents> | ||
| > { | ||
| private readonly gateways: URL[] | ||
|
|
||
| constructor (urls: string[]) { | ||
| this.gateways = urls.map(url => new URL(url.toString())) | ||
| } | ||
|
|
||
| notify (cid: CID, block: Uint8Array, options?: ProgressOptions): void { | ||
| // no-op | ||
| } | ||
|
|
||
| async get (cid: CID, options: AbortOptions & ProgressOptions<TrustlessGatewayGetBlockProgressEvents> = {}): Promise<Uint8Array> { | ||
| // choose a gateway | ||
| const url = this.gateways[Math.floor(Math.random() * this.gateways.length)] | ||
|
|
||
| log('getting block for %c from %s', cid, url) | ||
|
|
||
| try { | ||
| const block = await getRawBlockFromGateway(url, cid, options.signal) | ||
| log('got block for %c from %s', cid, url) | ||
|
|
||
| return block | ||
| } catch (err: any) { | ||
| log.error('failed to get block for %c from %s', cid, url, err) | ||
|
|
||
| throw err | ||
| } | ||
| } | ||
| } | ||
|
|
||
| async function getRawBlockFromGateway (url: URL, cid: CID, signal?: AbortSignal): Promise<Uint8Array> { | ||
| const gwUrl = new URL(url) | ||
| gwUrl.pathname = `/ipfs/${cid.toString()}` | ||
|
|
||
| // necessary as not every gateway supports dag-cbor, but every should support | ||
| // sending raw block as-is | ||
| gwUrl.search = '?format=raw' | ||
|
|
||
| if (signal?.aborted === true) { | ||
| throw new Error(`Signal to fetch raw block for CID ${cid} from gateway ${gwUrl.toString()} was aborted prior to fetch`) | ||
| } | ||
|
|
||
| try { | ||
| const res = await fetch(gwUrl.toString(), { | ||
| signal, | ||
| headers: { | ||
| // also set header, just in case ?format= is filtered out by some | ||
| // reverse proxy | ||
| Accept: 'application/vnd.ipld.raw' | ||
| }, | ||
| cache: 'force-cache' | ||
| }) | ||
| if (!res.ok) { | ||
| throw new Error(`unable to fetch raw block for CID ${cid} from gateway ${gwUrl.toString()}`) | ||
| } | ||
| return new Uint8Array(await res.arrayBuffer()) | ||
| } catch (cause) { | ||
| // @ts-expect-error - TS thinks signal?.aborted can only be false now | ||
| // because it was checked for true above. | ||
| if (signal?.aborted === true) { | ||
| throw new Error(`fetching raw block for CID ${cid} from gateway ${gwUrl.toString()} was aborted`) | ||
| } | ||
| throw new Error(`unable to fetch raw block for CID ${cid}`) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.