feat: async peerstore backed by datastores (#1058)

We have a peerstore that keeps all data for all observed peers in memory with no eviction. This is fine when you don't discover many peers but when using the DHT you encounter a significant number of peers so our peer storage grows and grows over time. We have a persistent peer store, but it just periodically writes peers into the datastore to be read at startup, still keeping them in memory. It also means a restart doesn't give you any temporary reprieve from the memory leak as the previously observed peer data is read into memory at startup. This change refactors the peerstore to use a datastore by default, reading and writing peer info as it arrives. It can be configured with a MemoryDatastore if desired. It was necessary to change the peerstore and *book interfaces to be asynchronous since the datastore api is asynchronous. BREAKING CHANGE: `libp2p.handle`, `libp2p.registrar.register` and the peerstore methods have become async
libp2p · Jan 20, 2022 · 978eb36 · 978eb36
1 parent 0a4dc54
commit 978eb36
Show file tree

Hide file tree

Showing 94 changed files with 3,212 additions and 3,988 deletions.
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -12,29 +12,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest]
         node: [16]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 16
-      - uses: actions/cache@v2
-        id: cache
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-           npm install
-           npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
 
   check:
     needs: build
@@ -43,23 +28,8 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-node@v2
       with:
-        node-version: 16
-    - uses: actions/cache@v2
-      id: cache
-      env:
-        CACHE_NAME: cache-node-modules
-      with:
-        path: |
-          ~/.cache
-          ~/.npm
-          ./node_modules
-          ./dist
-        key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-    - name: Install Dependencies
-      if: steps.cache.outputs.cache-hit != 'true'
-      run: |
-        npm install
-        npm run build
+        node-version: lts/*
+    - uses: ipfs/aegir/actions/cache-node-modules@master
     - run: npx aegir lint
     - run: npx aegir dep-check
     - uses: ipfs/aegir/actions/bundle-size@master
@@ -80,23 +50,7 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node }}
-      - uses: actions/cache@v2
-        id: cache
-        if: matrix.os != 'windows-latest'
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          npm install
-          npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run test:node -- --cov --bail
       - uses: codecov/codecov-action@v1
   test-chrome:
@@ -107,22 +61,7 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: lts/*
-      - uses: actions/cache@v2
-        id: cache
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          npm install
-          npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run test:browser -- -t browser -t webworker --bail
   test-firefox:
     needs: check
@@ -132,22 +71,7 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: lts/*
-      - uses: actions/cache@v2
-        id: cache
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          npm install
-          npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run test:browser -- -t browser -t webworker --bail -- --browser firefox
   test-ts:
     needs: check
@@ -157,22 +81,7 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: lts/*
-      - uses: actions/cache@v2
-        id: cache
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          npm install
-          npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run test:ts
   test-interop:
     needs: check
@@ -182,20 +91,5 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: lts/*
-      - uses: actions/cache@v2
-        id: cache
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          npm install
-          npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run test:interop -- --bail -- --exit
diff --git a/examples/connection-encryption/1.js b/examples/connection-encryption/1.js
@@ -30,7 +30,7 @@ const createNode = async () => {
     createNode()
   ])
 
-  node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
+  await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
 
   node2.handle('/a-protocol', ({ stream }) => {
     pipe(

diff --git a/examples/delegated-routing/package.json b/examples/delegated-routing/package.json
@@ -3,16 +3,16 @@
   "version": "0.1.0",
   "private": true,
   "dependencies": {
-    "ipfs": "~0.34.4",
-    "libp2p": "github:libp2p/js-libp2p#master",
-    "libp2p-delegated-content-routing": "~0.2.2",
-    "libp2p-delegated-peer-routing": "~0.2.2",
-    "libp2p-kad-dht": "^0.26.5",
-    "libp2p-mplex": "~0.8.5",
-    "libp2p-secio": "~0.11.1",
-    "libp2p-webrtc-star": "~0.15.8",
-    "libp2p-websocket-star": "~0.10.2",
-    "libp2p-websockets": "~0.12.2",
+    "@chainsafe/libp2p-noise": "^5.0.2",
+    "ipfs-core": "^0.13.0",
+    "libp2p": "../../",
+    "libp2p-delegated-content-routing": "^0.11.0",
+    "libp2p-delegated-peer-routing": "^0.11.1",
+    "libp2p-kad-dht": "^0.28.6",
+    "libp2p-mplex": "^0.10.4",
+    "libp2p-webrtc-star": "^0.25.0",
+    "libp2p-websocket-star": "^0.10.2",
+    "libp2p-websockets": "^0.16.2",
     "react": "^16.8.6",
     "react-dom": "^16.8.6",
     "react-scripts": "2.1.8"

diff --git a/examples/delegated-routing/src/App.js b/examples/delegated-routing/src/App.js
@@ -2,7 +2,7 @@
 'use strict'
 
 import React from 'react'
-import Ipfs from 'ipfs'
+import Ipfs from 'ipfs-core'
 import libp2pBundle from './libp2p-bundle'
 const Component = React.Component
 
@@ -70,7 +70,7 @@ class App extends Component {
   }
 
   componentDidMount () {
-    window.ipfs = this.ipfs = new Ipfs({
+    window.ipfs = this.ipfs = Ipfs.create({
       config: {
         Addresses: {
           Swarm: []

diff --git a/examples/delegated-routing/src/libp2p-bundle.js b/examples/delegated-routing/src/libp2p-bundle.js
@@ -6,7 +6,7 @@ const Websockets = require('libp2p-websockets')
 const WebSocketStar = require('libp2p-websocket-star')
 const WebRTCStar = require('libp2p-webrtc-star')
 const MPLEX = require('libp2p-mplex')
-const SECIO = require('libp2p-secio')
+const { NOISE } = require('@chainsafe/libp2p-noise')
 const KadDHT = require('libp2p-kad-dht')
 const DelegatedPeerRouter = require('libp2p-delegated-peer-routing')
 const DelegatedContentRouter = require('libp2p-delegated-content-routing')
@@ -48,7 +48,7 @@ export default function Libp2pBundle ({peerInfo, peerBook}) {
         MPLEX
       ],
       connEncryption: [
-        SECIO
+        NOISE
       ],
       dht: KadDHT
     },

diff --git a/examples/discovery-mechanisms/3.js b/examples/discovery-mechanisms/3.js
@@ -5,7 +5,7 @@ const Libp2p = require('../../')
 const TCP = require('libp2p-tcp')
 const Mplex = require('libp2p-mplex')
 const { NOISE } = require('@chainsafe/libp2p-noise')
-const Gossipsub = require('libp2p-gossipsub')
+const Gossipsub = require('@achingbrain/libp2p-gossipsub')
 const Bootstrap = require('libp2p-bootstrap')
 const PubsubPeerDiscovery = require('libp2p-pubsub-peer-discovery')
 

diff --git a/examples/discovery-mechanisms/bootstrapers.js b/examples/discovery-mechanisms/bootstrapers.js
@@ -1,13 +1,13 @@
 'use strict'
 
-// Find this list at: https://github.com/ipfs/js-ipfs/blob/master/packages/ipfs-core/src/runtime/config-nodejs.js
+// Find this list at: https://github.com/ipfs/js-ipfs/blob/master/packages/ipfs-core-config/src/config.js
 const bootstrapers = [
   '/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ',
   '/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN',
   '/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb',
   '/dnsaddr/bootstrap.libp2p.io/p2p/QmZa1sAxajnQjVM8WjWXoMbmPd7NsWhfKsPkErzpm9wGkp',
   '/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa',
-  '/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt'
+  '/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt',
 ]
 
 module.exports = bootstrapers
diff --git a/examples/discovery-mechanisms/test-1.js b/examples/discovery-mechanisms/test-1.js
@@ -2,41 +2,27 @@
 
 const path = require('path')
 const execa = require('execa')
-const pWaitFor = require('p-wait-for')
 const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
-const bootstrapers = require('./bootstrapers')
-
-const discoveredCopy = 'Discovered:'
-const connectedCopy = 'Connection established to:'
 
 async function test () {
-  const discoveredNodes = []
-  const connectedNodes = []
-
   process.stdout.write('1.js\n')
 
   const proc = execa('node', [path.join(__dirname, '1.js')], {
     cwd: path.resolve(__dirname),
     all: true
   })
 
+  let output = ''
+
   proc.all.on('data', async (data) => {
     process.stdout.write(data)
-    const line = uint8ArrayToString(data)
+    output += uint8ArrayToString(data)
 
-    // Discovered or Connected
-    if (line.includes(discoveredCopy)) {
-      const id = line.trim().split(discoveredCopy)[1]
-      discoveredNodes.push(id)
-    } else if (line.includes(connectedCopy)) {
-      const id = line.trim().split(connectedCopy)[1]
-      connectedNodes.push(id)
+    // Discovered and connected
+    if (output.includes('Connection established to:')) {
+      proc.kill()
     }
   })
-
-  await pWaitFor(() => discoveredNodes.length === bootstrapers.length && connectedNodes.length === bootstrapers.length)
-
-  proc.kill()
 }
 
 module.exports = test
diff --git a/examples/libp2p-in-the-browser/.babelrc b/examples/libp2p-in-the-browser/.babelrc
@@ -1,4 +1,3 @@
 {
-  "presets": ["@babel/preset-env"],
   "plugins": ["syntax-async-functions","transform-regenerator"]
 }
diff --git a/examples/libp2p-in-the-browser/package.json b/examples/libp2p-in-the-browser/package.json
@@ -14,12 +14,11 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@babel/preset-env": "^7.13.0",
+    "@chainsafe/libp2p-noise": "^5.0.2",
     "libp2p": "../../",
-    "libp2p-bootstrap": "^0.13.0",
+    "libp2p-bootstrap": "^0.14.0",
     "libp2p-mplex": "^0.10.4",
-    "@chainsafe/libp2p-noise": "^4.1.0",
-    "libp2p-webrtc-star": "^0.23.0",
+    "libp2p-webrtc-star": "^0.25.0",
     "libp2p-websockets": "^0.16.1"
   },
   "devDependencies": {

diff --git a/examples/package.json b/examples/package.json
@@ -8,12 +8,12 @@
   },
   "license": "MIT",
   "dependencies": {
+    "@achingbrain/libp2p-gossipsub": "^0.12.2",
     "execa": "^2.1.0",
     "fs-extra": "^8.1.0",
     "libp2p": "../src",
     "libp2p-pubsub-peer-discovery": "^4.0.0",
     "libp2p-relay-server": "^0.3.0",
-    "libp2p-gossipsub": "^0.11.0",
     "p-defer": "^3.0.0",
     "uint8arrays": "^3.0.0",
     "which": "^2.0.1"

diff --git a/examples/peer-and-content-routing/1.js b/examples/peer-and-content-routing/1.js
@@ -38,8 +38,8 @@ const createNode = async () => {
     createNode()
   ])
 
-  node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
-  node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
+  await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
+  await node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
 
   await Promise.all([
     node1.dial(node2.peerId),

diff --git a/examples/peer-and-content-routing/2.js b/examples/peer-and-content-routing/2.js
@@ -40,8 +40,8 @@ const createNode = async () => {
     createNode()
   ])
 
-  node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
-  node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
+  await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
+  await node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
 
   await Promise.all([
     node1.dial(node2.peerId),

diff --git a/examples/peer-and-content-routing/README.md b/examples/peer-and-content-routing/README.md
@@ -43,8 +43,8 @@ const node1 = nodes[0]
 const node2 = nodes[1]
 const node3 = nodes[2]
 
-node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
-node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
+await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
+await node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
 
 await Promise.all([
   node1.dial(node2.peerId),