Skip to content

Commit

Permalink
fix: prevent interop timeouts with fast fixture loading (#73)
Browse files Browse the repository at this point in the history
* deps: update ipfsd-ctl and kubo-rpc-client

* chore: apply suggestions from code review

* tmp: get some debug logs from CI jobs

* fix: disable delegated routing in vfetch-interop tests

* fix: test:node on windows

* Revert "tmp: get some debug logs from CI jobs"

This reverts commit 3654a28.
  • Loading branch information
SgtPooki authored May 8, 2024
1 parent aeb5a89 commit a43d994
Show file tree
Hide file tree
Showing 10 changed files with 70 additions and 139 deletions.
54 changes: 19 additions & 35 deletions packages/interop/.aegir.js
Original file line number Diff line number Diff line change
@@ -1,46 +1,30 @@
import getPort from 'aegir/get-port'
import { createServer } from 'ipfsd-ctl'
import * as kuboRpcClient from 'kubo-rpc-client'
import { resolve } from 'node:path'
import { tmpdir } from 'node:os'

const IPFS_PATH = resolve(tmpdir(), 'verified-fetch-interop-ipfs-repo')

/** @type {import('aegir').PartialOptions} */
export default {
test: {
files: './dist/src/*.spec.js',
before: async (options) => {
if (options.runner !== 'node') {
const ipfsdPort = await getPort()
const ipfsdServer = await createServer({
host: '127.0.0.1',
port: ipfsdPort
}, {
ipfsBin: (await import('kubo')).default.path(),
kuboRpcModule: kuboRpcClient,
ipfsOptions: {
config: {
Addresses: {
Swarm: [
"/ip4/0.0.0.0/tcp/0",
"/ip4/0.0.0.0/tcp/0/ws"
]
}
}
}
}).start()
before: async () => {

return {
env: {
IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}`
},
ipfsdServer
}
}
const { createKuboNode } = await import('./dist/src/fixtures/create-kubo.js')
const kuboNode = await createKuboNode(IPFS_PATH)

return {}
},
after: async (options, beforeResult) => {
if (options.runner !== 'node') {
await beforeResult.ipfsdServer.stop()
await kuboNode.start()

// requires aegir build to be run first, which it will by default.
const { loadFixtures } = await import('./dist/src/fixtures/load-fixtures.js')

await loadFixtures(IPFS_PATH)

return {
kuboNode
}
},
after: async (_options, beforeResult) => {
await beforeResult.kuboNode.stop()
}
}
}
7 changes: 4 additions & 3 deletions packages/interop/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,11 @@
"dependencies": {
"@helia/verified-fetch": "1.3.14",
"aegir": "^42.2.5",
"ipfsd-ctl": "^13.0.0",
"it-drain": "^3.0.5",
"execa": "^8.0.1",
"fast-glob": "^3.3.2",
"ipfsd-ctl": "^14.1.0",
"kubo": "^0.27.0",
"kubo-rpc-client": "^3.0.4",
"kubo-rpc-client": "^4.1.1",
"magic-bytes.js": "^1.10.0",
"multiformats": "^13.1.0"
},
Expand Down
1 change: 0 additions & 1 deletion packages/interop/src/bin.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#! /usr/bin/env node
/* eslint-disable no-console */

import { spawn } from 'node:child_process'
import { dirname, resolve } from 'node:path'
Expand Down
30 changes: 0 additions & 30 deletions packages/interop/src/fixtures/create-kubo.browser.ts

This file was deleted.

17 changes: 9 additions & 8 deletions packages/interop/src/fixtures/create-kubo.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */
import { createController, type Controller } from 'ipfsd-ctl'
import { createNode, type KuboNode } from 'ipfsd-ctl'
import { path as kuboPath } from 'kubo'
import * as kuboRpcClient from 'kubo-rpc-client'
import { create } from 'kubo-rpc-client'

export async function createKuboNode (): Promise<Controller> {
return createController({
kuboRpcModule: kuboRpcClient,
ipfsBin: kuboPath(),
export async function createKuboNode (repoPath = undefined): Promise<KuboNode> {
return createNode({
type: 'kubo',
rpc: create,
bin: kuboPath(),
test: true,
ipfsOptions: {
repo: repoPath,
init: {
config: {
Addresses: {
Swarm: [
Expand Down
9 changes: 0 additions & 9 deletions packages/interop/src/fixtures/load-fixture-data.ts

This file was deleted.

25 changes: 25 additions & 0 deletions packages/interop/src/fixtures/load-fixtures.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import { $ } from 'execa'
import fg from 'fast-glob'
import { path as kuboPath } from 'kubo'

/**
* Only callable from node (intended to be consumed by .aegir.js)
* but the fixtures loaded by this function are also used by browser tests.
*/
export async function loadFixtures (IPFS_PATH = undefined): Promise<void> {
const kuboBinary = process.env.KUBO_BINARY ?? kuboPath()
/**
* fast-glob does not like windows paths, see https://github.com/mrmlnc/fast-glob/issues/237
* fast-glob performs search from process.cwd() by default, which will be:
* 1. the root of the monorepo when running tests in CI
* 2. the package root when running tests in the package directory
*/
let globRoot = process.cwd().replace(/\\/g, '/')
if (!globRoot.includes('packages/interop')) {
// we only want car files from the interop package
globRoot = [...globRoot.split('/'), 'packages/interop'].join('/')
}
for (const carFile of await fg.glob('src/fixtures/data/*.car', { cwd: globRoot })) {
await $({ env: { IPFS_PATH } })`${kuboBinary} dag import --pin-roots=false --offline ${carFile}`
}
}
12 changes: 2 additions & 10 deletions packages/interop/src/json.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,23 @@
import { createVerifiedFetch } from '@helia/verified-fetch'
import { expect } from 'aegir/chai'
import { CID } from 'multiformats/cid'
import { createKuboNode } from './fixtures/create-kubo.js'
import { loadFixtureDataCar } from './fixtures/load-fixture-data.js'
import type { Controller } from 'ipfsd-ctl'

describe('@helia/verified-fetch - json', () => {
describe('unixfs - multiblock', () => {
let controller: Controller<'go'>
let verifiedFetch: Awaited<ReturnType<typeof createVerifiedFetch>>

before(async () => {
controller = await createKuboNode()
await controller.start()
// As of 2024-01-18, https://cloudflare-ipfs.com/ipns/tokens.uniswap.org resolves to:
// root: QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr
// child1: QmNik5N4ryNwzzXYq5hCYKGcRjAf9QtigxtiJh9o8aXXbG // partial JSON
// child2: QmWNBJX6fZyNTLWNYBHxAHpBctCP43R2zeqV2G8uavqFZn // partial JSON
await loadFixtureDataCar(controller, 'QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car')
verifiedFetch = await createVerifiedFetch({
gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
gateways: ['http://127.0.0.1:8180'],
routers: []
})
})

after(async () => {
await controller.stop()
await verifiedFetch.stop()
})

Expand Down
33 changes: 7 additions & 26 deletions packages/interop/src/unixfs-dir.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,35 +2,24 @@
import { createVerifiedFetch } from '@helia/verified-fetch'
import { expect } from 'aegir/chai'
import { filetypemime } from 'magic-bytes.js'
import { createKuboNode } from './fixtures/create-kubo.js'
import { loadFixtureDataCar } from './fixtures/load-fixture-data.js'
import type { VerifiedFetch } from '@helia/verified-fetch'
import type { Controller } from 'ipfsd-ctl'

describe('@helia/verified-fetch - unixfs directory', () => {
let controller: Controller
let verifiedFetch: VerifiedFetch

before(async () => {
controller = await createKuboNode()
await controller.start()

verifiedFetch = await createVerifiedFetch({
gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
gateways: ['http://127.0.0.1:8180'],
routers: []
})
verifiedFetch = await createVerifiedFetch()
})

after(async () => {
await controller.stop()
await verifiedFetch.stop()
})

describe('unixfs-dir-redirect', () => {
before(async () => {
await loadFixtureDataCar(controller, 'gateway-conformance-fixtures.car')
});

[
'https://example.com/ipfs/bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q',
'ipfs://bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q',
Expand All @@ -45,12 +34,8 @@ describe('@helia/verified-fetch - unixfs directory', () => {
})
})

// This tests the content of https://explore.ipld.io/#/explore/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1%20-%20Barrel%20-%20Part%201
describe('XKCD Barrel Part 1', () => {
before(async () => {
// This is the content of https://explore.ipld.io/#/explore/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1%20-%20Barrel%20-%20Part%201
await loadFixtureDataCar(controller, 'QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR-xkcd-Barrel-part-1.car')
})

it('fails to load when passed the root', async () => {
// The spec says we should generate HTML with directory listings, but we don't do that yet, so expect a failure
const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR')
Expand Down Expand Up @@ -78,8 +63,8 @@ describe('@helia/verified-fetch - unixfs directory', () => {
before(async () => {
await verifiedFetch.stop()
verifiedFetch = await createVerifiedFetch({
gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
gateways: ['http://127.0.0.1:8180'],
routers: []
}, {
contentTypeParser: (bytes) => {
return filetypemime(bytes)?.[0]
Expand All @@ -94,12 +79,8 @@ describe('@helia/verified-fetch - unixfs directory', () => {
})
})

// from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car
describe('HAMT-sharded directory', () => {
before(async () => {
// from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car
await loadFixtureDataCar(controller, 'bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i-single-layer-hamt-with-multi-block-files.car')
})

it('loads path /ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt', async () => {
const resp = await verifiedFetch('ipfs://bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt')
expect(resp).to.be.ok()
Expand Down
21 changes: 4 additions & 17 deletions packages/interop/src/websites.spec.ts
Original file line number Diff line number Diff line change
@@ -1,28 +1,20 @@
/* eslint-env mocha */
import { createVerifiedFetch } from '@helia/verified-fetch'
import { expect } from 'aegir/chai'
import { createKuboNode } from './fixtures/create-kubo.js'
import { loadFixtureDataCar } from './fixtures/load-fixture-data.js'
import type { Controller } from 'ipfsd-ctl'

describe('@helia/verified-fetch - websites', () => {
describe('helia-identify.on.fleek.co', () => {
let controller: Controller<'go'>
let verifiedFetch: Awaited<ReturnType<typeof createVerifiedFetch>>

before(async () => {
controller = await createKuboNode()
await controller.start()
// 2024-01-22 CID for _dnslink.helia-identify.on.fleek.co
await loadFixtureDataCar(controller, 'QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv-helia-identify-website.car')
verifiedFetch = await createVerifiedFetch({
gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
gateways: ['http://127.0.0.1:8180'],
routers: []
})
})

after(async () => {
await controller.stop()
await verifiedFetch.stop()
})

Expand Down Expand Up @@ -55,21 +47,16 @@ describe('@helia/verified-fetch - websites', () => {
* ```
*/
describe('fake blog.libp2p.io', () => {
let controller: Controller<'go'>
let verifiedFetch: Awaited<ReturnType<typeof createVerifiedFetch>>

before(async () => {
controller = await createKuboNode()
await controller.start()
await loadFixtureDataCar(controller, 'QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw-fake-blog.libp2p.io.car')
verifiedFetch = await createVerifiedFetch({
gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
gateways: ['http://127.0.0.1:8180'],
routers: []
})
})

after(async () => {
await controller.stop()
await verifiedFetch.stop()
})

Expand Down

0 comments on commit a43d994

Please sign in to comment.