From 635d7a2938111ccc53f8defbd9b8f8f8ea3e8e6a Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 5 Dec 2023 10:50:37 +0000 Subject: [PATCH] deps!: updates to libp2p v1 (#320) Updates all deps to libp2p v1. BREAKING CHANGE: The libp2p API has changed in a couple of places - please see the [upgrade guide](https://github.com/libp2p/js-libp2p/blob/main/doc/migrations/v0.46-v1.0.0.md) --- packages/helia/.aegir.js | 9 ++-- packages/helia/package.json | 39 ++++++++------- packages/helia/src/block-brokers/bitswap.ts | 3 +- .../block-brokers/trustless-gateway/broker.ts | 20 ++++---- .../block-brokers/trustless-gateway/index.ts | 9 +++- packages/helia/src/helia.ts | 24 +++++----- packages/helia/src/index.ts | 48 ++++++++----------- packages/helia/src/storage.ts | 4 +- .../src/utils/libp2p-defaults.browser.ts | 41 +++++++++------- packages/helia/src/utils/libp2p-defaults.ts | 47 ++++++++++-------- packages/helia/src/utils/libp2p.ts | 48 +++++++++++++++---- packages/helia/src/utils/networked-storage.ts | 33 ++++++++----- .../test/block-brokers/block-broker.spec.ts | 11 ++++- .../block-brokers/trustless-gateway.spec.ts | 6 ++- packages/helia/test/factory.spec.ts | 29 +++++++++-- packages/helia/test/fixtures/create-helia.ts | 6 +-- packages/helia/test/index.spec.ts | 4 +- .../test/utils/networked-storage.spec.ts | 6 ++- packages/interface/package.json | 4 +- packages/interface/src/index.ts | 7 ++- packages/interop/package.json | 13 ++--- .../test/fixtures/create-helia.browser.ts | 4 +- .../interop/test/fixtures/create-helia.ts | 4 +- 23 files changed, 266 insertions(+), 153 deletions(-) diff --git a/packages/helia/.aegir.js b/packages/helia/.aegir.js index a1cde6b0..84e4b05d 100644 --- a/packages/helia/.aegir.js +++ b/packages/helia/.aegir.js @@ -1,5 +1,5 @@ -import { circuitRelayServer } from 'libp2p/circuit-relay' -import { identifyService } from 'libp2p/identify' +import { circuitRelayServer } from '@libp2p/circuit-relay-v2' +import { identify } from '@libp2p/identify' import { WebSockets } from '@multiformats/mafmt' import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' @@ -23,8 +23,11 @@ const options = { `/ip4/127.0.0.1/tcp/0/ws` ] }, + connectionManager: { + inboundConnectionThreshold: Infinity + }, services: { - identify: identifyService(), + identify: identify(), relay: circuitRelayServer({ reservations: { maxReservations: Infinity, diff --git a/packages/helia/package.json b/packages/helia/package.json index 1360dc11..51fdae5f 100644 --- a/packages/helia/package.json +++ b/packages/helia/package.json @@ -78,24 +78,30 @@ "prepublishOnly": "node scripts/update-version.js && npm run build" }, "dependencies": { - "@chainsafe/libp2p-gossipsub": "^10.0.0", - "@chainsafe/libp2p-noise": "^13.0.0", - "@chainsafe/libp2p-yamux": "^5.0.0", + "@chainsafe/libp2p-gossipsub": "^11.0.0", + "@chainsafe/libp2p-noise": "^14.0.0", + "@chainsafe/libp2p-yamux": "^6.0.1", "@helia/delegated-routing-v1-http-api-client": "^1.1.0", "@helia/interface": "^2.1.0", "@ipld/dag-cbor": "^9.0.0", "@ipld/dag-json": "^10.0.1", "@ipld/dag-pb": "^4.0.3", - "@libp2p/bootstrap": "^9.0.2", - "@libp2p/interface": "^0.1.1", - "@libp2p/kad-dht": "^10.0.2", - "@libp2p/logger": "^3.0.1", - "@libp2p/mdns": "^9.0.2", - "@libp2p/mplex": "^9.0.2", - "@libp2p/tcp": "^8.0.2", - "@libp2p/webrtc": "^3.1.3", - "@libp2p/websockets": "^7.0.2", - "@libp2p/webtransport": "^3.0.3", + "@libp2p/autonat": "^1.0.1", + "@libp2p/bootstrap": "^10.0.2", + "@libp2p/circuit-relay-v2": "^1.0.2", + "@libp2p/dcutr": "^1.0.1", + "@libp2p/identify": "^1.0.1", + "@libp2p/interface": "^1.0.1", + "@libp2p/kad-dht": "^11.0.2", + "@libp2p/keychain": "^4.0.2", + "@libp2p/mdns": "^10.0.2", + "@libp2p/mplex": "^10.0.2", + "@libp2p/ping": "^1.0.1", + "@libp2p/tcp": "^9.0.2", + "@libp2p/upnp-nat": "^1.0.1", + "@libp2p/webrtc": "^4.0.3", + "@libp2p/websockets": "^8.0.2", + "@libp2p/webtransport": "^4.0.3", "any-signal": "^4.1.1", "blockstore-core": "^4.0.0", "cborg": "^4.0.1", @@ -103,13 +109,13 @@ "interface-blockstore": "^5.0.0", "interface-datastore": "^8.0.0", "interface-store": "^5.0.1", - "ipfs-bitswap": "^19.0.0", + "ipfs-bitswap": "^20.0.0", "ipns": "^7.0.1", "it-all": "^3.0.2", "it-drain": "^3.0.1", "it-filter": "^3.0.1", "it-foreach": "^2.0.2", - "libp2p": "^0.46.6", + "libp2p": "^1.0.3", "mortice": "^3.0.1", "multiformats": "^12.0.1", "p-defer": "^4.0.0", @@ -118,9 +124,10 @@ "uint8arrays": "^4.0.3" }, "devDependencies": { + "@libp2p/logger": "^4.0.1", "@multiformats/mafmt": "^12.1.5", "@multiformats/multiaddr": "^12.1.7", - "@types/sinon": "^10.0.14", + "@types/sinon": "^17.0.2", "aegir": "^41.0.0", "delay": "^6.0.0", "sinon": "^17.0.0", diff --git a/packages/helia/src/block-brokers/bitswap.ts b/packages/helia/src/block-brokers/bitswap.ts index 54f19b33..d73331c7 100644 --- a/packages/helia/src/block-brokers/bitswap.ts +++ b/packages/helia/src/block-brokers/bitswap.ts @@ -1,7 +1,6 @@ import { createBitswap } from 'ipfs-bitswap' import type { BlockAnnouncer, BlockBroker, BlockRetrievalOptions, BlockRetriever } from '@helia/interface/blocks' -import type { Libp2p } from '@libp2p/interface' -import type { Startable } from '@libp2p/interface/startable' +import type { Libp2p, Startable } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' import type { Bitswap, BitswapNotifyProgressEvents, BitswapOptions, BitswapWantBlockProgressEvents } from 'ipfs-bitswap' import type { CID } from 'multiformats/cid' diff --git a/packages/helia/src/block-brokers/trustless-gateway/broker.ts b/packages/helia/src/block-brokers/trustless-gateway/broker.ts index 918676e8..433a690d 100644 --- a/packages/helia/src/block-brokers/trustless-gateway/broker.ts +++ b/packages/helia/src/block-brokers/trustless-gateway/broker.ts @@ -1,13 +1,11 @@ -import { logger } from '@libp2p/logger' import { TrustlessGateway } from './trustless-gateway.js' import { DEFAULT_TRUSTLESS_GATEWAYS } from './index.js' -import type { TrustlessGatewayBlockBrokerInit, TrustlessGatewayGetBlockProgressEvents } from './index.js' +import type { TrustlessGatewayBlockBrokerInit, TrustlessGatewayComponents, TrustlessGatewayGetBlockProgressEvents } from './index.js' import type { BlockRetrievalOptions, BlockRetriever } from '@helia/interface/blocks' +import type { Logger } from '@libp2p/interface' import type { CID } from 'multiformats/cid' import type { ProgressOptions } from 'progress-events' -const log = logger('helia:trustless-gateway-block-broker') - /** * A class that accepts a list of trustless gateways that are queried * for blocks. @@ -16,8 +14,10 @@ export class TrustlessGatewayBlockBroker implements BlockRetriever< ProgressOptions > { private readonly gateways: TrustlessGateway[] + private readonly log: Logger - constructor (init: TrustlessGatewayBlockBrokerInit = {}) { + constructor (components: TrustlessGatewayComponents, init: TrustlessGatewayBlockBrokerInit = {}) { + this.log = components.logger.forComponent('helia:trustless-gateway-block-broker') this.gateways = (init.gateways ?? DEFAULT_TRUSTLESS_GATEWAYS) .map((gatewayOrUrl) => { return new TrustlessGateway(gatewayOrUrl) @@ -31,14 +31,14 @@ ProgressOptions const aggregateErrors: Error[] = [] for (const gateway of sortedGateways) { - log('getting block for %c from %s', cid, gateway.url) + this.log('getting block for %c from %s', cid, gateway.url) try { const block = await gateway.getRawBlock(cid, options.signal) - log.trace('got block for %c from %s', cid, gateway.url) + this.log.trace('got block for %c from %s', cid, gateway.url) try { await options.validateFn?.(block) } catch (err) { - log.error('failed to validate block for %c from %s', cid, gateway.url, err) + this.log.error('failed to validate block for %c from %s', cid, gateway.url, err) gateway.incrementInvalidBlocks() throw new Error(`unable to validate block for CID ${cid} from gateway ${gateway.url}`) @@ -46,7 +46,7 @@ ProgressOptions return block } catch (err: unknown) { - log.error('failed to get block for %c from %s', cid, gateway.url, err) + this.log.error('failed to get block for %c from %s', cid, gateway.url, err) if (err instanceof Error) { aggregateErrors.push(err) } else { @@ -54,7 +54,7 @@ ProgressOptions } // if signal was aborted, exit the loop if (options.signal?.aborted === true) { - log.trace('request aborted while fetching raw block for CID %c from gateway %s', cid, gateway.url) + this.log.trace('request aborted while fetching raw block for CID %c from gateway %s', cid, gateway.url) break } } diff --git a/packages/helia/src/block-brokers/trustless-gateway/index.ts b/packages/helia/src/block-brokers/trustless-gateway/index.ts index 846bb68d..2a97b463 100644 --- a/packages/helia/src/block-brokers/trustless-gateway/index.ts +++ b/packages/helia/src/block-brokers/trustless-gateway/index.ts @@ -1,5 +1,6 @@ import { TrustlessGatewayBlockBroker } from './broker.js' import type { BlockRetriever } from '@helia/interface/src/blocks.js' +import type { ComponentLogger } from '@libp2p/interface' import type { ProgressEvent } from 'progress-events' export const DEFAULT_TRUSTLESS_GATEWAYS = [ @@ -23,6 +24,10 @@ export interface TrustlessGatewayBlockBrokerInit { gateways?: Array } -export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): () => BlockRetriever { - return () => new TrustlessGatewayBlockBroker(init) +export interface TrustlessGatewayComponents { + logger: ComponentLogger +} + +export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): (components: TrustlessGatewayComponents) => BlockRetriever { + return (components) => new TrustlessGatewayBlockBroker(components, init) } diff --git a/packages/helia/src/helia.ts b/packages/helia/src/helia.ts index 8e7bf052..6e4ba230 100644 --- a/packages/helia/src/helia.ts +++ b/packages/helia/src/helia.ts @@ -1,5 +1,4 @@ -import { start, stop } from '@libp2p/interface/startable' -import { logger } from '@libp2p/logger' +import { start, stop } from '@libp2p/interface' import drain from 'it-drain' import { CustomProgressEvent } from 'progress-events' import { bitswap, trustlessGateway } from './block-brokers/index.js' @@ -11,13 +10,11 @@ import { NetworkedStorage } from './utils/networked-storage.js' import type { HeliaInit } from '.' import type { GCOptions, Helia } from '@helia/interface' import type { Pins } from '@helia/interface/pins' -import type { Libp2p } from '@libp2p/interface' +import type { ComponentLogger, Libp2p, Logger } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' import type { Datastore } from 'interface-datastore' import type { CID } from 'multiformats/cid' -const log = logger('helia') - interface HeliaImplInit extends HeliaInit { libp2p: T blockstore: Blockstore @@ -29,25 +26,30 @@ export class HeliaImpl implements Helia { public blockstore: BlockStorage public datastore: Datastore public pins: Pins + public logger: ComponentLogger + private readonly log: Logger constructor (init: HeliaImplInit) { + this.logger = init.libp2p.logger + this.log = this.logger.forComponent('helia') const hashers = defaultHashers(init.hashers) const components = { blockstore: init.blockstore, datastore: init.datastore, libp2p: init.libp2p, - hashers + hashers, + logger: init.libp2p.logger } const blockBrokers = init.blockBrokers?.map((fn) => { return fn(components) }) ?? [ bitswap()(components), - trustlessGateway()() + trustlessGateway()(components) ] - const networkedStorage = new NetworkedStorage(init.blockstore, { + const networkedStorage = new NetworkedStorage(components, { blockBrokers, hashers }) @@ -79,7 +81,7 @@ export class HeliaImpl implements Helia { const helia = this const blockstore = this.blockstore.unwrap() - log('gc start') + this.log('gc start') await drain(blockstore.deleteMany((async function * (): AsyncGenerator { for await (const { cid } of blockstore.getAll()) { @@ -92,7 +94,7 @@ export class HeliaImpl implements Helia { options.onProgress?.(new CustomProgressEvent('helia:gc:deleted', cid)) } catch (err) { - log.error('Error during gc', err) + helia.log.error('Error during gc', err) options.onProgress?.(new CustomProgressEvent('helia:gc:error', err)) } } @@ -101,6 +103,6 @@ export class HeliaImpl implements Helia { releaseLock() } - log('gc finished') + this.log('gc finished') } } diff --git a/packages/helia/src/index.ts b/packages/helia/src/index.ts index 0b0755c5..ff41c238 100644 --- a/packages/helia/src/index.ts +++ b/packages/helia/src/index.ts @@ -19,16 +19,15 @@ * ``` */ -import { logger } from '@libp2p/logger' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { HeliaImpl } from './helia.js' import { createLibp2p } from './utils/libp2p.js' -import { name, version } from './version.js' import type { DefaultLibp2pServices } from './utils/libp2p-defaults.js' import type { Helia } from '@helia/interface' import type { BlockBroker } from '@helia/interface/blocks' -import type { Libp2p } from '@libp2p/interface' +import type { ComponentLogger, Libp2p } from '@libp2p/interface' +import type { KeychainInit } from '@libp2p/keychain' import type { Blockstore } from 'interface-blockstore' import type { Datastore } from 'interface-datastore' import type { Libp2pOptions } from 'libp2p' @@ -41,8 +40,6 @@ export * from '@helia/interface' export * from '@helia/interface/blocks' export * from '@helia/interface/pins' -const log = logger('helia') - /** * DAGWalkers take a block and yield CIDs encoded in that block */ @@ -117,6 +114,18 @@ export interface HeliaInit { * webworker), pass true here to hold the gc lock in this process. */ holdGcLock?: boolean + + /** + * An optional logging component to pass to libp2p. If not specified the + * default implementation from libp2p will be used. + */ + logger?: ComponentLogger + + /** + * By default Helia stores the node's PeerId in an encrypted form in a + * libp2p keystore. These options control how that keystore is configured. + */ + keychain?: KeychainInit } /** @@ -133,23 +142,24 @@ export async function createHelia (init: HeliaInit = {}): Promise if (isLibp2p(init.libp2p)) { libp2p = init.libp2p } else { - libp2p = await createLibp2p(datastore, init.libp2p) + libp2p = await createLibp2p({ + ...init, + libp2p: init.libp2p, + datastore + }) } const helia = new HeliaImpl({ ...init, + libp2p, datastore, - blockstore, - libp2p + blockstore }) if (init.start !== false) { await helia.start() } - // add helia to agent version - addHeliaToAgentVersion(helia) - return helia } @@ -164,19 +174,3 @@ function isLibp2p (obj: any): obj is Libp2p { // if these are all functions it's probably a libp2p object return funcs.every(m => typeof obj[m] === 'function') } - -function addHeliaToAgentVersion (helia: Helia): void { - // add helia to agent version - try { - const existingAgentVersion = helia.libp2p.services.identify.host.agentVersion - - if (existingAgentVersion.match(/js-libp2p\/\d+\.\d+\.\d+\sUserAgent=/) == null) { - // the user changed the agent version - return - } - - helia.libp2p.services.identify.host.agentVersion = `${name}/${version} ${helia.libp2p.services.identify.host.agentVersion}` - } catch (err) { - log.error('could not add Helia to agent version', err) - } -} diff --git a/packages/helia/src/storage.ts b/packages/helia/src/storage.ts index 0575cdf6..909c2e89 100644 --- a/packages/helia/src/storage.ts +++ b/packages/helia/src/storage.ts @@ -1,8 +1,8 @@ -import { start, stop, type Startable } from '@libp2p/interface/startable' +import { start, stop } from '@libp2p/interface' import createMortice from 'mortice' import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions } from '@helia/interface/blocks' import type { Pins } from '@helia/interface/pins' -import type { AbortOptions } from '@libp2p/interface' +import type { AbortOptions, Startable } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' import type { AwaitIterable } from 'interface-store' import type { Mortice } from 'mortice' diff --git a/packages/helia/src/utils/libp2p-defaults.browser.ts b/packages/helia/src/utils/libp2p-defaults.browser.ts index 2e704a1d..9dd2c7f0 100644 --- a/packages/helia/src/utils/libp2p-defaults.browser.ts +++ b/packages/helia/src/utils/libp2p-defaults.browser.ts @@ -2,35 +2,41 @@ import { gossipsub } from '@chainsafe/libp2p-gossipsub' import { noise } from '@chainsafe/libp2p-noise' import { yamux } from '@chainsafe/libp2p-yamux' import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' +import { autoNAT } from '@libp2p/autonat' import { bootstrap } from '@libp2p/bootstrap' +import { circuitRelayTransport } from '@libp2p/circuit-relay-v2' +import { dcutr } from '@libp2p/dcutr' +import { type Identify, identify } from '@libp2p/identify' import { type DualKadDHT, kadDHT } from '@libp2p/kad-dht' +import { keychain, type Keychain } from '@libp2p/keychain' import { mplex } from '@libp2p/mplex' +import { ping, type PingService } from '@libp2p/ping' import { webRTC, webRTCDirect } from '@libp2p/webrtc' import { webSockets } from '@libp2p/websockets' import { webTransport } from '@libp2p/webtransport' import { ipnsSelector } from 'ipns/selector' import { ipnsValidator } from 'ipns/validator' -import { autoNATService } from 'libp2p/autonat' -import { circuitRelayTransport } from 'libp2p/circuit-relay' -import { dcutrService } from 'libp2p/dcutr' -import { type IdentifyService, identifyService } from 'libp2p/identify' -import { pingService, type PingService } from 'libp2p/ping' +import * as libp2pInfo from 'libp2p/version' +import { name, version } from '../version.js' import { bootstrapConfig } from './bootstrappers.js' -import type { PubSub } from '@libp2p/interface/pubsub' +import type { Libp2pDefaultsOptions } from './libp2p.js' +import type { PubSub } from '@libp2p/interface' import type { Libp2pOptions } from 'libp2p' export interface DefaultLibp2pServices extends Record { - dht: DualKadDHT - delegatedRouting: unknown - pubsub: PubSub - identify: IdentifyService autoNAT: unknown dcutr: unknown + delegatedRouting: unknown + dht: DualKadDHT + identify: Identify + keychain: Keychain ping: PingService + pubsub: PubSub } -export function libp2pDefaults (): Libp2pOptions { +export function libp2pDefaults (options: Libp2pDefaultsOptions): Libp2pOptions { return { + peerId: options.peerId, addresses: { listen: [ '/webrtc' @@ -56,10 +62,8 @@ export function libp2pDefaults (): Libp2pOptions { bootstrap(bootstrapConfig) ], services: { - identify: identifyService(), - autoNAT: autoNATService(), - pubsub: gossipsub(), - dcutr: dcutrService(), + autoNAT: autoNAT(), + dcutr: dcutr(), delegatedRouting: () => createDelegatedRoutingV1HttpApiClient('https://delegated-ipfs.dev'), dht: kadDHT({ clientMode: true, @@ -70,7 +74,12 @@ export function libp2pDefaults (): Libp2pOptions { ipns: ipnsSelector } }), - ping: pingService() + identify: identify({ + agentVersion: `${name}/${version} ${libp2pInfo.name}/${libp2pInfo.version} UserAgent=${globalThis.navigator.userAgent}` + }), + keychain: keychain(options.keychain), + ping: ping(), + pubsub: gossipsub() } } } diff --git a/packages/helia/src/utils/libp2p-defaults.ts b/packages/helia/src/utils/libp2p-defaults.ts index 0f6ce3f8..a7167307 100644 --- a/packages/helia/src/utils/libp2p-defaults.ts +++ b/packages/helia/src/utils/libp2p-defaults.ts @@ -2,39 +2,45 @@ import { gossipsub } from '@chainsafe/libp2p-gossipsub' import { noise } from '@chainsafe/libp2p-noise' import { yamux } from '@chainsafe/libp2p-yamux' import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' +import { autoNAT } from '@libp2p/autonat' import { bootstrap } from '@libp2p/bootstrap' +import { circuitRelayTransport, circuitRelayServer, type CircuitRelayService } from '@libp2p/circuit-relay-v2' +import { dcutr } from '@libp2p/dcutr' +import { type Identify, identify } from '@libp2p/identify' import { type DualKadDHT, kadDHT } from '@libp2p/kad-dht' +import { keychain, type Keychain } from '@libp2p/keychain' import { mdns } from '@libp2p/mdns' import { mplex } from '@libp2p/mplex' +import { ping, type PingService } from '@libp2p/ping' import { tcp } from '@libp2p/tcp' +import { uPnPNAT } from '@libp2p/upnp-nat' import { webRTC, webRTCDirect } from '@libp2p/webrtc' import { webSockets } from '@libp2p/websockets' import { ipnsSelector } from 'ipns/selector' import { ipnsValidator } from 'ipns/validator' -import { autoNATService } from 'libp2p/autonat' -import { circuitRelayTransport, circuitRelayServer, type CircuitRelayService } from 'libp2p/circuit-relay' -import { dcutrService } from 'libp2p/dcutr' -import { type IdentifyService, identifyService } from 'libp2p/identify' -import { pingService, type PingService } from 'libp2p/ping' -import { uPnPNATService } from 'libp2p/upnp-nat' +import * as libp2pInfo from 'libp2p/version' +import { name, version } from '../version.js' import { bootstrapConfig } from './bootstrappers.js' -import type { PubSub } from '@libp2p/interface/pubsub' +import type { Libp2pDefaultsOptions } from './libp2p.js' +import type { PubSub } from '@libp2p/interface' import type { Libp2pOptions } from 'libp2p' export interface DefaultLibp2pServices extends Record { - dht: DualKadDHT + autoNAT: unknown + dcutr: unknown delegatedRouting: unknown + dht: DualKadDHT + identify: Identify + keychain: Keychain + ping: PingService pubsub: PubSub relay: CircuitRelayService - identify: IdentifyService - autoNAT: unknown upnp: unknown - dcutr: unknown - ping: PingService } -export function libp2pDefaults (): Libp2pOptions { +export function libp2pDefaults (options: Libp2pDefaultsOptions): Libp2pOptions { return { + peerId: options.peerId, addresses: { listen: [ '/ip4/0.0.0.0/tcp/0', @@ -63,11 +69,8 @@ export function libp2pDefaults (): Libp2pOptions { bootstrap(bootstrapConfig) ], services: { - identify: identifyService(), - autoNAT: autoNATService(), - upnp: uPnPNATService(), - pubsub: gossipsub(), - dcutr: dcutrService(), + autoNAT: autoNAT(), + dcutr: dcutr(), delegatedRouting: () => createDelegatedRoutingV1HttpApiClient('https://delegated-ipfs.dev'), dht: kadDHT({ validators: { @@ -77,10 +80,16 @@ export function libp2pDefaults (): Libp2pOptions { ipns: ipnsSelector } }), + identify: identify({ + agentVersion: `${name}/${version} ${libp2pInfo.name}/${libp2pInfo.version} UserAgent=${globalThis.process.version}` + }), + keychain: keychain(options.keychain), + ping: ping(), + pubsub: gossipsub(), relay: circuitRelayServer({ advertise: true }), - ping: pingService() + upnp: uPnPNAT() } } } diff --git a/packages/helia/src/utils/libp2p.ts b/packages/helia/src/utils/libp2p.ts index 354f81e5..e78a1f1f 100644 --- a/packages/helia/src/utils/libp2p.ts +++ b/packages/helia/src/utils/libp2p.ts @@ -1,21 +1,53 @@ -import { createLibp2p as create, type Libp2pOptions } from 'libp2p' -import { type DefaultLibp2pServices, libp2pDefaults } from './libp2p-defaults.js' -import type { Libp2p } from '@libp2p/interface' +import { keychain } from '@libp2p/keychain' +import { defaultLogger } from '@libp2p/logger' +import { Key } from 'interface-datastore' +import { createLibp2p as create } from 'libp2p' +import { libp2pDefaults } from './libp2p-defaults.js' +import type { DefaultLibp2pServices } from './libp2p-defaults.js' +import type { ComponentLogger, Libp2p, PeerId } from '@libp2p/interface' +import type { KeychainInit } from '@libp2p/keychain' import type { Datastore } from 'interface-datastore' +import type { Libp2pOptions } from 'libp2p' -export interface CreateLibp2pOptions { +export interface CreateLibp2pOptions> { datastore: Datastore + libp2p?: Libp2pOptions + logger?: ComponentLogger + keychain?: KeychainInit start?: boolean } -export async function createLibp2p (datastore: Datastore, options?: Libp2pOptions): Promise> { - const defaults = libp2pDefaults() +export interface Libp2pDefaultsOptions { + peerId?: PeerId + keychain?: KeychainInit +} + +export async function createLibp2p = DefaultLibp2pServices> (options: CreateLibp2pOptions): Promise> { + let peerId = options.libp2p?.peerId + const logger = options.logger ?? defaultLogger() + + // if no peer id was passed, try to load it from the keychain + if (peerId == null) { + const chain = keychain(options.keychain)({ + datastore: options.datastore, + logger + }) + + const selfKey = new Key('/pkcs8/self') + + if (await options.datastore.has(selfKey)) { + // load the peer id from the keychain + peerId = await chain.exportPeerId('self') + } + } + + const defaults = libp2pDefaults(options) options = options ?? {} + // @ts-expect-error derived ServiceMap is not compatible with ServiceFactoryMap return create({ - datastore, ...defaults, - ...options, + ...options.libp2p, start: false }) } diff --git a/packages/helia/src/utils/networked-storage.ts b/packages/helia/src/utils/networked-storage.ts index bf02e5ed..f8e78b4b 100644 --- a/packages/helia/src/utils/networked-storage.ts +++ b/packages/helia/src/utils/networked-storage.ts @@ -1,20 +1,16 @@ -import { CodeError } from '@libp2p/interface/errors' -import { start, stop, type Startable } from '@libp2p/interface/startable' -import { logger } from '@libp2p/logger' +import { CodeError, start, stop } from '@libp2p/interface' import { anySignal } from 'any-signal' import filter from 'it-filter' import forEach from 'it-foreach' import { CustomProgressEvent, type ProgressOptions } from 'progress-events' import { equals as uint8ArrayEquals } from 'uint8arrays/equals' import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetriever, BlockAnnouncer, BlockRetrievalOptions } from '@helia/interface/blocks' -import type { AbortOptions } from '@libp2p/interface' +import type { AbortOptions, ComponentLogger, Logger, LoggerOptions, Startable } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' import type { AwaitIterable } from 'interface-store' import type { CID } from 'multiformats/cid' import type { MultihashHasher } from 'multiformats/hashes/interface' -const log = logger('helia:networked-storage') - export interface NetworkedStorageStorageInit { blockBrokers?: BlockBroker[] hashers?: MultihashHasher[] @@ -32,6 +28,11 @@ function isBlockAnnouncer (b: any): b is BlockAnnouncer { return typeof b.announce === 'function' } +export interface NetworkedStorageComponents { + blockstore: Blockstore + logger: ComponentLogger +} + /** * Networked storage wraps a regular blockstore - when getting blocks if the * blocks are not present Bitswap will be used to fetch them from network peers. @@ -42,12 +43,14 @@ export class NetworkedStorage implements Blocks, Startable { private readonly blockAnnouncers: BlockAnnouncer[] private readonly hashers: MultihashHasher[] private started: boolean + private readonly log: Logger /** * Create a new BlockStorage */ - constructor (blockstore: Blockstore, init: NetworkedStorageStorageInit) { - this.child = blockstore + constructor (components: NetworkedStorageComponents, init: NetworkedStorageStorageInit) { + this.log = components.logger.forComponent('helia:networked-storage') + this.child = components.blockstore this.blockRetrievers = (init.blockBrokers ?? []).filter(isBlockRetriever) this.blockAnnouncers = (init.blockBrokers ?? []).filter(isBlockAnnouncer) this.hashers = init.hashers ?? [] @@ -124,7 +127,10 @@ export class NetworkedStorage implements Blocks, Startable { if (options.offline !== true && !(await this.child.has(cid))) { // we do not have the block locally, get it from a block provider options.onProgress?.(new CustomProgressEvent('blocks:get:providers:get', cid)) - const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers, options) + const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers, { + ...options, + log: this.log + }) options.onProgress?.(new CustomProgressEvent('blocks:get:blockstore:put', cid)) await this.child.put(cid, block, options) @@ -152,7 +158,10 @@ export class NetworkedStorage implements Blocks, Startable { if (options.offline !== true && !(await this.child.has(cid))) { // we do not have the block locally, get it from a block provider options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:get', cid)) - const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers, options) + const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers, { + ...options, + log: this.log + }) options.onProgress?.(new CustomProgressEvent('blocks:get-many:blockstore:put', cid)) await this.child.put(cid, block, options) @@ -218,7 +227,7 @@ export const getCidBlockVerifierFunction = (cid: CID, hashers: MultihashHasher[] * Race block providers cancelling any pending requests once the block has been * found. */ -async function raceBlockRetrievers (cid: CID, providers: BlockRetriever[], hashers: MultihashHasher[], options: AbortOptions): Promise { +async function raceBlockRetrievers (cid: CID, providers: BlockRetriever[], hashers: MultihashHasher[], options: AbortOptions & LoggerOptions): Promise { const validateFn = getCidBlockVerifierFunction(cid, hashers) const controller = new AbortController() @@ -246,7 +255,7 @@ async function raceBlockRetrievers (cid: CID, providers: BlockRetriever[], hashe return block } catch (err) { - log.error('could not retrieve verified block for %c', cid, err) + options.log.error('could not retrieve verified block for %c', cid, err) throw err } }) diff --git a/packages/helia/test/block-brokers/block-broker.spec.ts b/packages/helia/test/block-brokers/block-broker.spec.ts index 3eb121e3..e10eb7fc 100644 --- a/packages/helia/test/block-brokers/block-broker.spec.ts +++ b/packages/helia/test/block-brokers/block-broker.spec.ts @@ -1,5 +1,6 @@ /* eslint-env mocha */ +import { defaultLogger } from '@libp2p/logger' import { expect } from 'aegir/chai' import { MemoryBlockstore } from 'blockstore-core' import delay from 'delay' @@ -31,7 +32,10 @@ describe('block-broker', () => { blockstore = new MemoryBlockstore() bitswapBlockBroker = stubInterface() gatewayBlockBroker = stubInterface() - storage = new NetworkedStorage(blockstore, { + storage = new NetworkedStorage({ + blockstore, + logger: defaultLogger() + }, { blockBrokers: [ bitswapBlockBroker, gatewayBlockBroker @@ -112,7 +116,10 @@ describe('block-broker', () => { it('handles incorrect bytes from a gateway', async () => { const { cid } = blocks[0] const block = blocks[1].block - storage = new NetworkedStorage(blockstore, { + storage = new NetworkedStorage({ + blockstore, + logger: defaultLogger() + }, { blockBrokers: [ gatewayBlockBroker ], diff --git a/packages/helia/test/block-brokers/trustless-gateway.spec.ts b/packages/helia/test/block-brokers/trustless-gateway.spec.ts index 513a5b91..2b1af781 100644 --- a/packages/helia/test/block-brokers/trustless-gateway.spec.ts +++ b/packages/helia/test/block-brokers/trustless-gateway.spec.ts @@ -1,4 +1,6 @@ /* eslint-env mocha */ + +import { defaultLogger } from '@libp2p/logger' import { expect } from 'aegir/chai' import * as raw from 'multiformats/codecs/raw' import Sinon from 'sinon' @@ -39,7 +41,9 @@ describe('trustless-gateway-block-broker', () => { stubConstructor(TrustlessGateway, 'http://localhost:8082'), stubConstructor(TrustlessGateway, 'http://localhost:8083') ] - gatewayBlockBroker = new TrustlessGatewayBlockBroker() + gatewayBlockBroker = new TrustlessGatewayBlockBroker({ + logger: defaultLogger() + }) // must copy the array because the broker calls .sort which mutates in-place ;(gatewayBlockBroker as any).gateways = [...gateways] }) diff --git a/packages/helia/test/factory.spec.ts b/packages/helia/test/factory.spec.ts index 284adafb..a7de6740 100644 --- a/packages/helia/test/factory.spec.ts +++ b/packages/helia/test/factory.spec.ts @@ -1,10 +1,11 @@ /* eslint-env mocha */ +import { identify } from '@libp2p/identify' import { webSockets } from '@libp2p/websockets' import { expect } from 'aegir/chai' +import { MemoryDatastore } from 'datastore-core' import { Key } from 'interface-datastore' import { createLibp2p } from 'libp2p' -import { identifyService } from 'libp2p/identify' import { CID } from 'multiformats/cid' import { createHelia } from '../src/index.js' import type { Helia } from '@helia/interface' @@ -23,7 +24,7 @@ describe('helia factory', () => { start: false }) - expect(helia.libp2p.isStarted()).to.be.false() + expect(helia.libp2p.status).to.equal('stopped') }) it('does not require any constructor args', async () => { @@ -53,7 +54,7 @@ describe('helia factory', () => { webSockets() ], services: { - identify: identifyService({ + identify: identify({ agentVersion: 'my custom agent version' }) } @@ -78,4 +79,26 @@ describe('helia factory', () => { expect(agentVersionBuf).to.be.undefined() }) + + it('reuses peer id if reusing datastore', async () => { + const datastore = new MemoryDatastore() + + helia = await createHelia({ + datastore, + start: false + }) + + const peerId = helia.libp2p.peerId + + await helia.stop() + + await createHelia({ + datastore, + start: false + }) + + const otherPeerId = helia.libp2p.peerId + + expect(peerId.toString()).to.equal(otherPeerId.toString()) + }) }) diff --git a/packages/helia/test/fixtures/create-helia.ts b/packages/helia/test/fixtures/create-helia.ts index e4ac8de6..1ac670e4 100644 --- a/packages/helia/test/fixtures/create-helia.ts +++ b/packages/helia/test/fixtures/create-helia.ts @@ -1,7 +1,7 @@ +import { circuitRelayTransport } from '@libp2p/circuit-relay-v2' +import { identify } from '@libp2p/identify' import { webSockets } from '@libp2p/websockets' import * as Filters from '@libp2p/websockets/filters' -import { circuitRelayTransport } from 'libp2p/circuit-relay' -import { identifyService } from 'libp2p/identify' import { bitswap } from '../../src/block-brokers/index.js' import { createHelia as createNode } from '../../src/index.js' import type { Helia } from '@helia/interface' @@ -27,7 +27,7 @@ export async function createHelia (): Promise { denyDialMultiaddr: async () => false }, services: { - identify: identifyService() + identify: identify() } } }) diff --git a/packages/helia/test/index.spec.ts b/packages/helia/test/index.spec.ts index d70d041b..56dfd190 100644 --- a/packages/helia/test/index.spec.ts +++ b/packages/helia/test/index.spec.ts @@ -37,11 +37,11 @@ describe('helia', () => { }) it('stops and starts', async () => { - expect(helia.libp2p.isStarted()).to.be.true() + expect(helia.libp2p.status).to.equal('started') await helia.stop() - expect(helia.libp2p.isStarted()).to.be.false() + expect(helia.libp2p.status).to.equal('stopped') }) it('should have a blockstore', async () => { diff --git a/packages/helia/test/utils/networked-storage.spec.ts b/packages/helia/test/utils/networked-storage.spec.ts index 44762125..b4072738 100644 --- a/packages/helia/test/utils/networked-storage.spec.ts +++ b/packages/helia/test/utils/networked-storage.spec.ts @@ -1,5 +1,6 @@ /* eslint-env mocha */ +import { defaultLogger } from '@libp2p/logger' import { expect } from 'aegir/chai' import { MemoryBlockstore } from 'blockstore-core' import delay from 'delay' @@ -30,7 +31,10 @@ describe('networked-storage', () => { blockstore = new MemoryBlockstore() bitswap = stubInterface() - storage = new NetworkedStorage(blockstore, { + storage = new NetworkedStorage({ + blockstore, + logger: defaultLogger() + }, { blockBrokers: [ bitswap ], diff --git a/packages/interface/package.json b/packages/interface/package.json index 37886acc..0f95da5d 100644 --- a/packages/interface/package.json +++ b/packages/interface/package.json @@ -70,11 +70,11 @@ "build": "aegir build" }, "dependencies": { - "@libp2p/interface": "^0.1.1", + "@libp2p/interface": "^1.0.1", "interface-blockstore": "^5.0.0", "interface-datastore": "^8.0.0", "interface-store": "^5.0.1", - "ipfs-bitswap": "^19.0.0", + "ipfs-bitswap": "^20.0.0", "multiformats": "^12.0.1", "progress-events": "^1.0.0" }, diff --git a/packages/interface/src/index.ts b/packages/interface/src/index.ts index 72a03c11..1173ff06 100644 --- a/packages/interface/src/index.ts +++ b/packages/interface/src/index.ts @@ -16,7 +16,7 @@ import type { Blocks } from './blocks.js' import type { Pins } from './pins.js' -import type { Libp2p, AbortOptions } from '@libp2p/interface' +import type { Libp2p, AbortOptions, ComponentLogger } from '@libp2p/interface' import type { Datastore } from 'interface-datastore' import type { CID } from 'multiformats/cid' import type { ProgressEvent, ProgressOptions } from 'progress-events' @@ -47,6 +47,11 @@ export interface Helia { */ pins: Pins + /** + * A logging component that can be reused by consumers + */ + logger: ComponentLogger + /** * Starts the Helia node */ diff --git a/packages/interop/package.json b/packages/interop/package.json index 6b6a73af..588b5407 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -53,11 +53,12 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@chainsafe/libp2p-noise": "^13.0.0", - "@chainsafe/libp2p-yamux": "^5.0.0", + "@chainsafe/libp2p-noise": "^14.0.0", + "@chainsafe/libp2p-yamux": "^6.0.1", "@helia/interface": "^2.1.0", - "@libp2p/tcp": "^8.0.2", - "@libp2p/websockets": "^7.0.2", + "@libp2p/identify": "^1.0.1", + "@libp2p/tcp": "^9.0.1", + "@libp2p/websockets": "^8.0.1", "@multiformats/sha3": "^3.0.0", "aegir": "^41.0.0", "blockstore-core": "^4.0.0", @@ -66,9 +67,9 @@ "ipfsd-ctl": "^13.0.0", "it-all": "^3.0.2", "it-to-buffer": "^4.0.1", - "kubo": "^0.23.0", + "kubo": "^0.24.0", "kubo-rpc-client": "^3.0.1", - "libp2p": "^0.46.6", + "libp2p": "^1.0.1", "multiformats": "^12.0.1" }, "browser": { diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 28a03223..6f42ff96 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -1,5 +1,6 @@ import { noise } from '@chainsafe/libp2p-noise' import { yamux } from '@chainsafe/libp2p-yamux' +import { identify } from '@libp2p/identify' import { webSockets } from '@libp2p/websockets' import { all } from '@libp2p/websockets/filters' import { MemoryBlockstore } from 'blockstore-core' @@ -7,7 +8,6 @@ import { MemoryDatastore } from 'datastore-core' import { createHelia, type HeliaInit } from 'helia' import { bitswap } from 'helia/block-brokers' import { createLibp2p } from 'libp2p' -import { identifyService } from 'libp2p/identify' import type { Helia } from '@helia/interface' export async function createHeliaNode (init?: Partial): Promise { @@ -29,7 +29,7 @@ export async function createHeliaNode (init?: Partial): Promise): Promise { @@ -30,7 +30,7 @@ export async function createHeliaNode (init?: Partial): Promise