From bb50c3e4d401bf5212283945066764d098187928 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Mon, 20 May 2024 17:43:15 -0700
Subject: [PATCH 01/25] feat: resolve IPFS_NS_MAP with @multiformats/dns

---
 packages/gateway-conformance/.aegir.js        |  3 +-
 .../src/conformance.spec.ts                   |  8 ++-
 .../gateway-conformance/src/demo-server.ts    |  5 +-
 .../src/fixtures/basic-server.ts              | 55 ++++++++++++++++++-
 4 files changed, 64 insertions(+), 7 deletions(-)

diff --git a/packages/gateway-conformance/.aegir.js b/packages/gateway-conformance/.aegir.js
index c5e6076..abb3e53 100644
--- a/packages/gateway-conformance/.aegir.js
+++ b/packages/gateway-conformance/.aegir.js
@@ -22,7 +22,8 @@ export default {
       const SERVER_PORT = await getPort(3441)
       const stopBasicServer = await startBasicServer({
         serverPort: SERVER_PORT,
-        kuboGateway
+        kuboGateway,
+        IPFS_NS_MAP
       })
 
       const { startReverseProxy } = await import('./dist/src/fixtures/reverse-proxy.js')
diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index 97f5ee8..ba871cc 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -319,11 +319,15 @@ describe('@helia/verified-fetch - gateway conformance', function () {
       }
     })
 
-    tests.forEach(({ name, spec, skip, run, successRate: minSuccessRate }) => {
+    tests.forEach(({ name, spec, skip, run, timeout, successRate: minSuccessRate }) => {
       const log = logger.forComponent(name)
       const expectedSuccessRate = process.env.SUCCESS_RATE != null ? Number.parseFloat(process.env.SUCCESS_RATE) : minSuccessRate
 
       it(`${name} has a success rate of at least ${expectedSuccessRate}%`, async function () {
+        if (timeout != null) {
+          this.timeout(timeout)
+        }
+
         const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs(name,
           [
             ...(spec != null ? ['--specs', spec] : [])
@@ -368,7 +372,7 @@ describe('@helia/verified-fetch - gateway conformance', function () {
 
       expect(failureCount).to.be.lessThanOrEqual(1134)
       expect(successCount).to.be.greaterThanOrEqual(262)
-      expect(successRate).to.be.greaterThanOrEqual(18.77)
+      expect(successRate).to.be.greaterThanOrEqual(18.79)
     })
   })
 })
diff --git a/packages/gateway-conformance/src/demo-server.ts b/packages/gateway-conformance/src/demo-server.ts
index 39698a3..2a8459d 100644
--- a/packages/gateway-conformance/src/demo-server.ts
+++ b/packages/gateway-conformance/src/demo-server.ts
@@ -14,12 +14,13 @@ const { node: controller, gatewayUrl, repoPath } = await createKuboNode(await ge
 
 const kuboGateway = gatewayUrl
 await controller.start()
-await loadKuboFixtures(repoPath)
+const IPFS_NS_MAP = await loadKuboFixtures(repoPath)
 
 const SERVER_PORT = await getPort(3441)
 await startBasicServer({
   serverPort: SERVER_PORT,
-  kuboGateway
+  kuboGateway,
+  IPFS_NS_MAP
 })
 
 const PROXY_PORT = await getPort(3442)
diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index 45ca9b3..b6c22e6 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -1,5 +1,6 @@
 import { createServer } from 'node:http'
 import { logger } from '@libp2p/logger'
+import { type DNSResolver } from '@multiformats/dns/resolvers'
 import { contentTypeParser } from './content-type-parser.js'
 import { createVerifiedFetch } from './create-verified-fetch.js'
 
@@ -13,9 +14,56 @@ const log = logger('basic-server')
 export interface BasicServerOptions {
   kuboGateway?: string
   serverPort: number
+
+  /**
+   * @see https://github.com/ipfs/kubo/blob/5de5b77168be347186dbc9f1586c2deb485ca2ef/docs/environment-variables.md#ipfs_ns_map
+   */
+  IPFS_NS_MAP: string
 }
 
-export async function startBasicServer ({ kuboGateway, serverPort }: BasicServerOptions): Promise<() => Promise<void>> {
+const getLocalDnsResolver = (ipfsNsMap: string): DNSResolver => {
+  const log = logger('basic-server:dns')
+  const nsMap = new Map<string, string>()
+  const keyVals = ipfsNsMap.split(',')
+  for (const keyVal of keyVals) {
+    const [key, val] = keyVal.split(':')
+    log('Setting entry: %s="%s"', key, val)
+    nsMap.set(key, val)
+  }
+  return async (domain, options) => {
+    log.trace('Querying "%s" for types %O', domain, options?.types)
+    const actualDomainKey = domain.replace('_dnslink.', '')
+    const nsValue = nsMap.get(actualDomainKey)
+    if (nsValue == null) {
+      log.error('No IPFS_NS_MAP entry for domain "%s"', actualDomainKey)
+      throw new Error(`No IPFS_NS_MAP entry for domain "${actualDomainKey}"`)
+    }
+    const data = `dnslink=${nsValue}`
+    log.trace('Returning DNS response for %s: %s', domain, data)
+
+    return {
+      Status: 0,
+      TC: false,
+      RD: false,
+      RA: false,
+      AD: true,
+      CD: true,
+      Question: [{
+        name: domain,
+        type: 16
+      }],
+      Answer: [{
+        name: domain,
+        type: 16,
+        TTL: 180,
+        data
+        // data: 'dnslink=/ipfs/bafkqac3imvwgy3zao5xxe3de'
+      }]
+    }
+  }
+}
+
+export async function startBasicServer ({ kuboGateway, serverPort, IPFS_NS_MAP }: BasicServerOptions): Promise<() => Promise<void>> {
   kuboGateway = kuboGateway ?? process.env.KUBO_GATEWAY
   const useSessions = process.env.USE_SESSIONS !== 'false'
 
@@ -24,11 +72,14 @@ export async function startBasicServer ({ kuboGateway, serverPort }: BasicServer
   if (kuboGateway == null) {
     throw new Error('options.kuboGateway or KUBO_GATEWAY env var is required')
   }
+
+  const localDnsResolver = getLocalDnsResolver(IPFS_NS_MAP)
   const verifiedFetch = await createVerifiedFetch({
     gateways: [kuboGateway],
     routers: [],
     allowInsecure: true,
-    allowLocal: true
+    allowLocal: true,
+    dnsResolvers: [localDnsResolver]
   }, {
     contentTypeParser
   })

From cb724f40446d33d8ada36cfa3e72e1eeea89a55e Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Tue, 21 May 2024 13:05:26 -0700
Subject: [PATCH 02/25] fix: various fixes to gateway conformance

---
 packages/gateway-conformance/.aegir.js        |  23 +-
 packages/gateway-conformance/package.json     |   5 +-
 .../src/conformance.spec.ts                   |  85 +++---
 .../gateway-conformance/src/demo-server.ts    |   9 +-
 .../src/fixtures/basic-server.ts              | 260 +++++++++++-------
 .../src/fixtures/get-local-dns-resolver.ts    | 112 ++++++++
 .../src/fixtures/ipns-record-datastore.ts     |  11 +
 .../src/fixtures/kubo-mgmt.ts                 |  36 ++-
 .../src/fixtures/reverse-proxy.ts             |  19 +-
 packages/verified-fetch/package.json          |   6 +-
 10 files changed, 400 insertions(+), 166 deletions(-)
 create mode 100644 packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts
 create mode 100644 packages/gateway-conformance/src/fixtures/ipns-record-datastore.ts

diff --git a/packages/gateway-conformance/.aegir.js b/packages/gateway-conformance/.aegir.js
index abb3e53..698b527 100644
--- a/packages/gateway-conformance/.aegir.js
+++ b/packages/gateway-conformance/.aegir.js
@@ -1,5 +1,7 @@
 // @ts-check
 import getPort from 'aegir/get-port'
+import { logger } from '@libp2p/logger'
+const log = logger('aegir')
 
 /** @type {import('aegir').PartialOptions} */
 export default {
@@ -12,26 +14,30 @@ export default {
 
       const { createKuboNode } = await import('./dist/src/fixtures/create-kubo.js')
       const KUBO_PORT = await getPort(3440)
+      const SERVER_PORT = await getPort(3441)
+      const PROXY_PORT = await getPort(3442)
       const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_PORT)
       await controller.start()
       const { loadKuboFixtures } = await import('./dist/src/fixtures/kubo-mgmt.js')
-      const IPFS_NS_MAP = await loadKuboFixtures(repoPath)
+      const IPFS_NS_MAP = await loadKuboFixtures(repoPath, PROXY_PORT)
       const kuboGateway = gatewayUrl
 
       const { startBasicServer } = await import('./dist/src/fixtures/basic-server.js')
-      const SERVER_PORT = await getPort(3441)
       const stopBasicServer = await startBasicServer({
         serverPort: SERVER_PORT,
         kuboGateway,
         IPFS_NS_MAP
+      }).catch((err) => {
+        log.error(err)
       })
 
       const { startReverseProxy } = await import('./dist/src/fixtures/reverse-proxy.js')
-      const PROXY_PORT = await getPort(3442)
       const stopReverseProxy = await startReverseProxy({
         backendPort: SERVER_PORT,
         targetHost: 'localhost',
         proxyPort: PROXY_PORT
+      }).catch((err) => {
+        log.error(err)
       })
 
       const CONFORMANCE_HOST = 'localhost'
@@ -51,12 +57,19 @@ export default {
       }
     },
     after: async (options, beforeResult) => {
+      log('aegir test after hook')
+      // @ts-expect-error - broken aegir types
+      await beforeResult.controller.stop()
+      log('controller stopped')
+
       // @ts-expect-error - broken aegir types
       await beforeResult.stopReverseProxy()
+      log('reverse proxy stopped')
+
       // @ts-expect-error - broken aegir types
       await beforeResult.stopBasicServer()
-      // @ts-expect-error - broken aegir types
-      await beforeResult.controller.stop()
+      log('basic server stopped')
+
     }
   }
 }
diff --git a/packages/gateway-conformance/package.json b/packages/gateway-conformance/package.json
index 9a3b05a..dac0e49 100644
--- a/packages/gateway-conformance/package.json
+++ b/packages/gateway-conformance/package.json
@@ -52,9 +52,12 @@
     "test": "aegir test -t node"
   },
   "dependencies": {
-    "@helia/interface": "^4.3.0",
+    "@helia/block-brokers": "^3.0.0-f46700f",
+    "@helia/interface": "^4.3.0-f46700f",
+    "@helia/utils": "^0.3.1",
     "@helia/verified-fetch": "1.4.2",
     "@libp2p/logger": "^4.0.11",
+    "@multiformats/dns": "^1.0.6",
     "@sgtpooki/file-type": "^1.0.1",
     "aegir": "^42.2.5",
     "execa": "^8.0.1",
diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index ba871cc..7a02c50 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -16,6 +16,7 @@ interface TestConfig {
   skip?: string[]
   run?: string[]
   successRate: number
+  timeout?: number
 }
 
 function getGatewayConformanceBinaryPath (): string {
@@ -68,11 +69,14 @@ const tests: TestConfig[] = [
   {
     name: 'TestPathing',
     run: ['TestPathing'],
-    successRate: 23.53
+    successRate: 26.67
   },
   {
     name: 'TestDNSLinkGatewayUnixFSDirectoryListing',
     run: ['TestDNSLinkGatewayUnixFSDirectoryListing'],
+    skip: [
+      'TestDNSLinkGatewayUnixFSDirectoryListing/.*TODO:_cleanup_Kubo-specifics'
+    ],
     successRate: 0
   },
   {
@@ -89,7 +93,8 @@ const tests: TestConfig[] = [
   // {
   //   name: 'TestNativeDag',
   //   run: ['TestNativeDag'],
-  //   successRate: 100
+  //   successRate: 100,
+  //   timeout: 120000
   // },
   {
     name: 'TestGatewayJSONCborAndIPNS',
@@ -142,26 +147,27 @@ const tests: TestConfig[] = [
     run: ['TestTrustlessCarDagScopeAll'],
     successRate: 36.36
   },
-  {
-    name: 'TestTrustlessCarDagScopeEntity',
-    run: ['TestTrustlessCarDagScopeEntity'],
-    successRate: 34.57
-  },
-  {
-    name: 'TestTrustlessCarDagScopeBlock',
-    run: ['TestTrustlessCarDagScopeBlock'],
-    successRate: 34.69
-  },
+  // {
+  //   name: 'TestTrustlessCarDagScopeEntity',
+  //   run: ['TestTrustlessCarDagScopeEntity'],
+  //   successRate: 34.57
+  // },
+  // {
+  //   name: 'TestTrustlessCarDagScopeBlock',
+  //   run: ['TestTrustlessCarDagScopeBlock'],
+  //   successRate: 34.69
+  // },
   {
     name: 'TestTrustlessCarPathing',
     run: ['TestTrustlessCarPathing'],
-    successRate: 33.85
-  },
-  {
-    name: 'TestSubdomainGatewayDNSLinkInlining',
-    run: ['TestSubdomainGatewayDNSLinkInlining'],
-    successRate: 0
+    successRate: 35,
+    timeout: 240000
   },
+  // {
+  //   name: 'TestSubdomainGatewayDNSLinkInlining',
+  //   run: ['TestSubdomainGatewayDNSLinkInlining'],
+  //   successRate: 0
+  // },
   {
     name: 'TestGatewaySubdomainAndIPNS',
     run: ['TestGatewaySubdomainAndIPNS'],
@@ -213,18 +219,22 @@ const tests: TestConfig[] = [
     run: ['TestGatewayCacheWithIPNS'],
     successRate: 35.71
   },
-  // times out
-  // {
-  //   name: 'TestGatewayCache',
-  //   run: ['TestGatewayCache'],
-  //   successRate: 100
-  // },
-  // times out
-  // {
-  //   name: 'TestUnixFSDirectoryListing',
-  //   run: ['TestUnixFSDirectoryListing'],
-  //   successRate: 100
-  // },
+  {
+    name: 'TestGatewayCache',
+    run: ['TestGatewayCache'],
+    successRate: 60.71,
+    timeout: 1200000
+  },
+  {
+    name: 'TestUnixFSDirectoryListing',
+    run: ['TestUnixFSDirectoryListing'],
+    skip: [
+      'TestUnixFSDirectoryListingOnSubdomainGateway',
+      'TestUnixFSDirectoryListing/.*TODO:_cleanup_Kubo-specifics'
+    ],
+    successRate: 16.67,
+    timeout: 1200000
+  },
   {
     name: 'TestTar',
     run: ['TestTar'],
@@ -336,7 +346,7 @@ describe('@helia/verified-fetch - gateway conformance', function () {
             ...((skip != null) ? ['-skip', `${skip.join('|')}`] : []),
             ...((run != null) ? ['-run', `${run.join('|')}`] : [])
           ]
-        ), { reject: false })
+        ), { reject: false, signal: timeout != null ? AbortSignal.timeout(timeout) : undefined })
 
         log(stdout)
         log.error(stderr)
@@ -352,6 +362,7 @@ describe('@helia/verified-fetch - gateway conformance', function () {
      * as this test does.
      */
     it('has expected total failures and successes', async function () {
+      this.timeout(200000)
       const log = logger.forComponent('all')
 
       // TODO: unskip when verified-fetch is no longer infinitely looping on requests.
@@ -360,19 +371,19 @@ describe('@helia/verified-fetch - gateway conformance', function () {
         'TestTrustlessCarEntityBytes',
         'TestUnixFSDirectoryListingOnSubdomainGateway',
         'TestGatewayCache',
-        'TestUnixFSDirectoryListing'
+        'TestUnixFSDirectoryListing',
+        '.*/.*TODO:_cleanup_Kubo-specifics'
       ]
+      const skip = ['-skip', toSkip.join('|')]
 
-      const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], ['-skip', toSkip.join('|')]), { reject: false })
+      const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], skip), { reject: false, signal: AbortSignal.timeout(200000) })
 
       log(stdout)
       log.error(stderr)
 
-      const { failureCount, successCount, successRate } = await getReportDetails('gwc-report-all.json')
+      const { successRate } = await getReportDetails('gwc-report-all.json')
 
-      expect(failureCount).to.be.lessThanOrEqual(1134)
-      expect(successCount).to.be.greaterThanOrEqual(262)
-      expect(successRate).to.be.greaterThanOrEqual(18.79)
+      expect(successRate).to.be.greaterThanOrEqual(15.7)
     })
   })
 })
diff --git a/packages/gateway-conformance/src/demo-server.ts b/packages/gateway-conformance/src/demo-server.ts
index 2a8459d..d183bb0 100644
--- a/packages/gateway-conformance/src/demo-server.ts
+++ b/packages/gateway-conformance/src/demo-server.ts
@@ -10,20 +10,21 @@ import { startReverseProxy } from './fixtures/reverse-proxy.js'
 
 const log = logger('demo-server')
 
-const { node: controller, gatewayUrl, repoPath } = await createKuboNode(await getPort(3440))
+const KUBO_GATEWAY_PORT = await getPort(3440)
+const SERVER_PORT = await getPort(3441)
+const PROXY_PORT = await getPort(3442)
+const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_GATEWAY_PORT)
 
 const kuboGateway = gatewayUrl
 await controller.start()
-const IPFS_NS_MAP = await loadKuboFixtures(repoPath)
+const IPFS_NS_MAP = await loadKuboFixtures(repoPath, PROXY_PORT)
 
-const SERVER_PORT = await getPort(3441)
 await startBasicServer({
   serverPort: SERVER_PORT,
   kuboGateway,
   IPFS_NS_MAP
 })
 
-const PROXY_PORT = await getPort(3442)
 await startReverseProxy({
   backendPort: SERVER_PORT,
   targetHost: 'localhost',
diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index b6c22e6..ad095dc 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -1,8 +1,17 @@
-import { createServer } from 'node:http'
+import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'
+import { trustlessGateway } from '@helia/block-brokers'
+import { createHeliaHTTP } from '@helia/http'
+import { httpGatewayRouting } from '@helia/routers'
 import { logger } from '@libp2p/logger'
-import { type DNSResolver } from '@multiformats/dns/resolvers'
+import { dns } from '@multiformats/dns'
+import { MemoryBlockstore } from 'blockstore-core'
 import { contentTypeParser } from './content-type-parser.js'
 import { createVerifiedFetch } from './create-verified-fetch.js'
+import { getLocalDnsResolver } from './get-local-dns-resolver.js'
+import { getIpnsRecordDatastore } from './ipns-record-datastore.js'
+import type { DNSResolver } from '@multiformats/dns/resolvers'
+import type { Blockstore } from 'interface-blockstore'
+import type { Datastore } from 'interface-datastore'
 
 const log = logger('basic-server')
 /**
@@ -21,45 +30,133 @@ export interface BasicServerOptions {
   IPFS_NS_MAP: string
 }
 
-const getLocalDnsResolver = (ipfsNsMap: string): DNSResolver => {
-  const log = logger('basic-server:dns')
-  const nsMap = new Map<string, string>()
-  const keyVals = ipfsNsMap.split(',')
-  for (const keyVal of keyVals) {
-    const [key, val] = keyVal.split(':')
-    log('Setting entry: %s="%s"', key, val)
-    nsMap.set(key, val)
+type Response = ServerResponse<IncomingMessage> & {
+  req: IncomingMessage
+}
+
+interface CreateHeliaOptions {
+  gateways: string[]
+  dnsResolvers: DNSResolver[]
+  blockstore: Blockstore
+  datastore: Datastore
+}
+
+/**
+ * We need to create helia manually so we can stub some of the things...
+ */
+async function createHelia (init: CreateHeliaOptions): Promise<ReturnType<typeof createHeliaHTTP>> {
+  return createHeliaHTTP({
+    blockBrokers: [
+      trustlessGateway({
+        allowInsecure: true,
+        allowLocal: true
+      })
+    ],
+    routers: [
+      httpGatewayRouting({
+        gateways: init.gateways
+      })
+    ],
+    dns: dns({
+      resolvers: {
+        '.': init.dnsResolvers
+      }
+    })
+  })
+}
+
+async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response, { serverPort, useSessions, verifiedFetch, kuboGateway, localDnsResolver }: any): Promise<void> {
+  const log = logger('basic-server:request')
+  if (req.method === 'OPTIONS') {
+    res.writeHead(200)
+    res.end()
+    return
+  }
+
+  if (req.url == null) {
+    // this should never happen
+    log.error('No URL provided, returning 400 Bad Request')
+    res.writeHead(400)
+    res.end('Bad Request')
+    return
+  }
+
+  const hostname = req.headers.host?.split(':')[0]
+  const host = req.headers['x-forwarded-for'] ?? `${hostname}:${serverPort}`
+
+  const fullUrlHref = req.headers.referer ?? `http://${host}${req.url}`
+  const urlLog = logger(`basic-server:request:${host}${req.url}`)
+  urlLog('configuring request')
+  urlLog.trace('req.headers: %O', req.headers)
+  let requestController: AbortController | null = new AbortController()
+  // we need to abort the request if the client disconnects
+  const onReqEnd = (): void => {
+    urlLog('client disconnected, aborting request')
+    requestController?.abort()
+  }
+  req.on('end', onReqEnd)
+
+  const reqTimeout = setTimeout(() => {
+    /**
+     * Abort the request because it's taking too long.
+     * This is only needed for when @helia/verified-fetch is not correctly
+     * handling a request and should not be needed once we have 100% gateway
+     * conformance coverage.
+     */
+    urlLog.error('timing out request')
+    requestController?.abort()
+  }, 2000)
+  reqTimeout.unref() // don't keep the process alive just for this timeout
+
+  const onResFinish = (): void => {
+    urlLog.trace('response finished, aborting signal')
+    requestController?.abort()
   }
-  return async (domain, options) => {
-    log.trace('Querying "%s" for types %O', domain, options?.types)
-    const actualDomainKey = domain.replace('_dnslink.', '')
-    const nsValue = nsMap.get(actualDomainKey)
-    if (nsValue == null) {
-      log.error('No IPFS_NS_MAP entry for domain "%s"', actualDomainKey)
-      throw new Error(`No IPFS_NS_MAP entry for domain "${actualDomainKey}"`)
+  res.on('finish', onResFinish)
+
+  try {
+    urlLog.trace('calling verified-fetch')
+    const resp = await verifiedFetch(fullUrlHref, { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true })
+    urlLog.trace('verified-fetch response status: %d', resp.status)
+
+    // loop over headers and set them on the response
+    const headers: Record<string, string> = {}
+    for (const [key, value] of resp.headers.entries()) {
+      headers[key] = value
     }
-    const data = `dnslink=${nsValue}`
-    log.trace('Returning DNS response for %s: %s', domain, data)
-
-    return {
-      Status: 0,
-      TC: false,
-      RD: false,
-      RA: false,
-      AD: true,
-      CD: true,
-      Question: [{
-        name: domain,
-        type: 16
-      }],
-      Answer: [{
-        name: domain,
-        type: 16,
-        TTL: 180,
-        data
-        // data: 'dnslink=/ipfs/bafkqac3imvwgy3zao5xxe3de'
-      }]
+
+    res.writeHead(resp.status, headers)
+    if (resp.body == null) {
+      // need to convert ArrayBuffer to Buffer or Uint8Array
+      res.write(Buffer.from(await resp.arrayBuffer()))
+      urlLog.trace('wrote response')
+    } else {
+      // read the body of the response and write it to the response from the server
+      const reader = resp.body.getReader()
+      while (true) {
+        const { done, value } = await reader.read()
+        if (done) {
+          urlLog.trace('response stream finished')
+          break
+        }
+
+        res.write(Buffer.from(value))
+      }
     }
+    res.end()
+  } catch (e: any) {
+    urlLog.error('Problem with request: %s', e.message, e)
+    if (!res.headersSent) {
+      res.writeHead(500)
+    }
+    res.end(`Internal Server Error: ${e.message}`)
+  } finally {
+    urlLog.trace('Cleaning up request')
+    clearTimeout(reqTimeout)
+    requestController.abort()
+    requestController = null
+    req.off('end', onReqEnd)
+    res.off('finish', onResFinish)
   }
 }
 
@@ -73,79 +170,34 @@ export async function startBasicServer ({ kuboGateway, serverPort, IPFS_NS_MAP }
     throw new Error('options.kuboGateway or KUBO_GATEWAY env var is required')
   }
 
-  const localDnsResolver = getLocalDnsResolver(IPFS_NS_MAP)
-  const verifiedFetch = await createVerifiedFetch({
-    gateways: [kuboGateway],
-    routers: [],
-    allowInsecure: true,
-    allowLocal: true,
-    dnsResolvers: [localDnsResolver]
-  }, {
+  const blockstore = new MemoryBlockstore()
+  const datastore = getIpnsRecordDatastore()
+  const localDnsResolver = getLocalDnsResolver(IPFS_NS_MAP, kuboGateway)
+
+  const helia = await createHelia({ gateways: [kuboGateway], dnsResolvers: [localDnsResolver], blockstore, datastore })
+
+  const verifiedFetch = await createVerifiedFetch(helia, {
     contentTypeParser
   })
 
   const server = createServer((req, res) => {
-    if (req.method === 'OPTIONS') {
-      res.writeHead(200)
-      res.end()
-      return
-    }
-
-    if (req.url == null) {
-      // this should never happen
-      res.writeHead(400)
-      res.end('Bad Request')
-      return
-    }
-
-    log.trace('req.headers: %O', req.headers)
-    const hostname = req.headers.host?.split(':')[0]
-    const host = req.headers['x-forwarded-for'] ?? `${hostname}:${serverPort}`
+    try {
+      void createAndCallVerifiedFetch(req, res, { serverPort, useSessions, kuboGateway, localDnsResolver, verifiedFetch }).catch((err) => {
+        log.error('Error in createAndCallVerifiedFetch', err)
 
-    const fullUrlHref = req.headers.referer ?? `http://${host}${req.url}`
-    log('fetching %s', fullUrlHref)
-
-    const requestController = new AbortController()
-    // we need to abort the request if the client disconnects
-    req.on('close', () => {
-      log('client disconnected, aborting request')
-      requestController.abort()
-    })
-
-    void verifiedFetch(fullUrlHref, { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true }).then(async (resp) => {
-      // loop over headers and set them on the response
-      const headers: Record<string, string> = {}
-      for (const [key, value] of resp.headers.entries()) {
-        headers[key] = value
-      }
-
-      res.writeHead(resp.status, headers)
-      if (resp.body == null) {
-        // need to convert ArrayBuffer to Buffer or Uint8Array
-        res.write(Buffer.from(await resp.arrayBuffer()))
-      } else {
-        // read the body of the response and write it to the response from the server
-        const reader = resp.body.getReader()
-        while (true) {
-          const { done, value } = await reader.read()
-          if (done) {
-            break
-          }
-          log('typeof value: %s', typeof value)
-
-          res.write(Buffer.from(value))
+        if (!res.headersSent) {
+          res.writeHead(500)
         }
-      }
-      res.end()
-    }).catch((e) => {
-      log.error('Problem with request: %s', e.message, e)
+        res.end('Internal Server Error')
+      })
+    } catch (err) {
+      log.error('Error in createServer', err)
+
       if (!res.headersSent) {
         res.writeHead(500)
       }
-      res.end(`Internal Server Error: ${e.message}`)
-    }).finally(() => {
-      requestController.abort()
-    })
+      res.end('Internal Server Error')
+    }
   })
 
   server.listen(serverPort, () => {
@@ -153,7 +205,11 @@ export async function startBasicServer ({ kuboGateway, serverPort, IPFS_NS_MAP }
   })
 
   return async () => {
+    log('Stopping...')
     await new Promise<void>((resolve, reject) => {
+      // no matter what happens, we need to kill the server
+      server.closeAllConnections()
+      log('Closed all connections')
       server.close((err: any) => {
         if (err != null) {
           reject(err)
diff --git a/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts b/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts
new file mode 100644
index 0000000..70d444b
--- /dev/null
+++ b/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts
@@ -0,0 +1,112 @@
+import { logger } from '@libp2p/logger'
+import { type Answer, type Question } from '@multiformats/dns'
+import { type DNSResolver } from '@multiformats/dns/resolvers'
+
+export function getLocalDnsResolver (ipfsNsMap: string, kuboGateway: string): DNSResolver {
+  const log = logger('basic-server:dns')
+  const nsMap = new Map<string, string>()
+  const keyVals = ipfsNsMap.split(',')
+  for (const keyVal of keyVals) {
+    const [key, val] = keyVal.split(':')
+    log('Setting entry: %s="%s"', key, val)
+    nsMap.set(key, val)
+  }
+
+  // async function getNameFromKubo (name: string): Promise<string> {
+  //   try {
+  //     log.trace('Fetching peer record for %s from Kubo', name)
+  //     const peerResponse = await fetch(`${kuboGateway}/api/v0/name/resolve?arg=${name}`, { method: 'POST' })
+  //     // invalid .json(), see https://github.com/ipfs/kubo/issues/10428
+  //     const text = (await peerResponse.text()).trim()
+  //     log('response from Kubo: %s', text)
+  //     const peerJson = JSON.parse(text)
+  //     return peerJson.Path
+  //   } catch (err: any) {
+  //     log.error('Problem fetching peer record from kubo: %s', err.message, err)
+  //     // process.exit(1)
+  //     throw err
+  //   }
+  // }
+
+  // /**
+  //  * @see https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-resolve
+  //  */
+  // async function getPeerRecordFromKubo (peerId: string): Promise<string> {
+  //   try {
+  //     log.trace('Fetching peer record for %s from Kubo', peerId)
+  //     const peerResponse = await fetch(`${kuboGateway}/api/v0/resolve/${peerId}`, { method: 'POST' })
+  //     // invalid .json(), see https://github.com/ipfs/kubo/issues/10428
+  //     const text = (await peerResponse.text()).trim()
+  //     log('response from Kubo: %s', text)
+  //     const peerJson = JSON.parse(text)
+  //     return peerJson.Path
+  //   } catch (err: any) {
+  //     log.error('Problem fetching peer record from kubo: %s', err.message, err)
+  //     // process.exit(1)
+  //     return getNameFromKubo(peerId)
+  //   }
+  // }
+
+  return async (domain, options) => {
+    const questions: Question[] = []
+    const answers: Answer[] = []
+
+    if (Array.isArray(options?.types)) {
+      options?.types?.forEach?.((type) => {
+        questions.push({ name: domain, type })
+      })
+    } else {
+      questions.push({ name: domain, type: options?.types ?? 16 })
+    }
+    // TODO: do we need to do anything with CNAME resolution...?
+    // if (questions.some((q) => q.type === 5)) {
+    //   answers.push({
+    //     name: domain,
+    //     type: 5,
+    //     TTL: 180,
+    //     data: ''
+    //   })
+    // }
+    if (questions.some((q) => q.type === 16)) {
+      log.trace('Querying "%s" for types %O', domain, options?.types)
+      const actualDomainKey = domain.replace('_dnslink.', '')
+      const nsValue = nsMap.get(actualDomainKey)
+      // try {
+      //   await getPeerRecordFromKubo(actualDomainKey)
+      // await getNameFromKubo(actualDomainKey)
+      if (nsValue == null) {
+        log.error('No IPFS_NS_MAP entry for domain "%s"', actualDomainKey)
+        // try to query kubo for the record
+        // temporarily disabled because it can cause an infinite loop
+        // await getPeerRecordFromKubo(actualDomainKey)
+
+        throw new Error('No IPFS_NS_MAP entry for domain')
+      }
+      const data = `dnslink=${nsValue}`
+      answers.push({
+        name: domain,
+        type: 16,
+        TTL: 180,
+        data // should be in the format 'dnslink=/ipfs/bafkqac3imvwgy3zao5xxe3de'
+      })
+      // } catch (err: any) {
+      //   log.error('Problem resolving record: %s', err.message, err)
+      // }
+    }
+
+    const dnsResponse = {
+      Status: 0,
+      TC: false,
+      RD: false,
+      RA: false,
+      AD: true,
+      CD: true,
+      Question: questions,
+      Answer: answers
+    }
+
+    log.trace('Returning DNS response for %s: %O', domain, dnsResponse)
+
+    return dnsResponse
+  }
+}
diff --git a/packages/gateway-conformance/src/fixtures/ipns-record-datastore.ts b/packages/gateway-conformance/src/fixtures/ipns-record-datastore.ts
new file mode 100644
index 0000000..37dc134
--- /dev/null
+++ b/packages/gateway-conformance/src/fixtures/ipns-record-datastore.ts
@@ -0,0 +1,11 @@
+import { MemoryDatastore } from 'datastore-core'
+import type { Datastore } from 'interface-datastore'
+
+const datastore = new MemoryDatastore()
+/**
+ * We need a normalized datastore so we can set custom records
+ * from the IPFS_NS_MAP like kubo does.
+ */
+export function getIpnsRecordDatastore (): Datastore {
+  return datastore
+}
diff --git a/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts b/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts
index 1423da5..faed104 100644
--- a/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts
+++ b/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts
@@ -10,11 +10,17 @@
 import { readFile } from 'node:fs/promises'
 import { dirname, relative, posix, basename } from 'node:path'
 import { fileURLToPath } from 'node:url'
+import { Record as DhtRecord } from '@libp2p/kad-dht'
 import { logger } from '@libp2p/logger'
+import { peerIdFromString } from '@libp2p/peer-id'
 import { $ } from 'execa'
 import fg from 'fast-glob'
+import { Key } from 'interface-datastore'
+import { peerIdToRoutingKey } from 'ipns'
 import { path } from 'kubo'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { GWC_IMAGE } from '../constants.js'
+import { getIpnsRecordDatastore } from './ipns-record-datastore.js'
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 const __dirname = dirname(fileURLToPath(import.meta.url))
@@ -28,10 +34,10 @@ export const GWC_FIXTURES_PATH = posix.resolve(__dirname, 'gateway-conformance-f
 /**
  * use `createKuboNode' to start a kubo node prior to loading fixtures.
  */
-export async function loadKuboFixtures (kuboRepoDir: string): Promise<string> {
+export async function loadKuboFixtures (kuboRepoDir: string, proxyPort: number): Promise<string> {
   await downloadFixtures()
 
-  return loadFixtures(kuboRepoDir)
+  return loadFixtures(kuboRepoDir, proxyPort)
 }
 
 function getExecaOptions ({ cwd, ipfsNsMap, kuboRepoDir }: { cwd?: string, ipfsNsMap?: string, kuboRepoDir?: string } = {}): { cwd: string, env: Record<string, string | undefined> } {
@@ -66,7 +72,7 @@ async function downloadFixtures (force = false): Promise<void> {
   }
 }
 
-export async function loadFixtures (kuboRepoDir: string): Promise<string> {
+export async function loadFixtures (kuboRepoDir: string, proxyPort: number): Promise<string> {
   const execaOptions = getExecaOptions({ kuboRepoDir })
 
   const carPath = `${GWC_FIXTURES_PATH}/**/*.car`
@@ -85,20 +91,24 @@ export async function loadFixtures (kuboRepoDir: string): Promise<string> {
     throw new Error('No *.car fixtures found')
   }
 
-  // TODO: fix in CI. See https://github.com/ipfs/helia-verified-fetch/actions/runs/9022946675/job/24793649918?pr=67#step:7:19
-  if (process.env.CI == null) {
-    for (const ipnsRecord of await fg.glob([`${GWC_FIXTURES_PATH}/**/*.ipns-record`])) {
-      const key = basename(ipnsRecord, '.ipns-record')
-      const relativePath = relative(GWC_FIXTURES_PATH, ipnsRecord)
-      log('Loading *.ipns-record fixture %s', relativePath)
-      const { stdout } = await $(({ ...execaOptions }))`cd ${GWC_FIXTURES_PATH} && ${kuboBinary} routing put --allow-offline "/ipns/${key}" "${relativePath}"`
-      stdout.split('\n').forEach(log)
-    }
+  const datastore = getIpnsRecordDatastore()
+
+  for (const fsIpnsRecord of await fg.glob([`${GWC_FIXTURES_PATH}/**/*.ipns-record`])) {
+    const peerIdString = basename(fsIpnsRecord, '.ipns-record').split('_')[0]
+    const relativePath = relative(GWC_FIXTURES_PATH, fsIpnsRecord)
+    log('Loading *.ipns-record fixture %s', relativePath)
+    const key = peerIdFromString(peerIdString)
+    const customRoutingKey = peerIdToRoutingKey(key)
+    const dhtKey = new Key('/dht/record/' + uint8ArrayToString(customRoutingKey, 'base32'), false)
+
+    const dhtRecord = new DhtRecord(customRoutingKey, await readFile(fsIpnsRecord, null), new Date(Date.now() + 9999999))
+
+    await datastore.put(dhtKey, dhtRecord.serialize())
   }
 
   const json = await readFile(`${GWC_FIXTURES_PATH}/dnslinks.json`, 'utf-8')
   const { subdomains, domains } = JSON.parse(json)
-  const subdomainDnsLinks = Object.entries(subdomains).map(([key, value]) => `${key}.example.com:${value}`).join(',')
+  const subdomainDnsLinks = Object.entries(subdomains).map(([key, value]) => `${key}.localhost:${value}`).join(',')
   const domainDnsLinks = Object.entries(domains).map(([key, value]) => `${key}:${value}`).join(',')
   const ipfsNsMap = `${domainDnsLinks},${subdomainDnsLinks}`
 
diff --git a/packages/gateway-conformance/src/fixtures/reverse-proxy.ts b/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
index d942883..00f61b9 100644
--- a/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
+++ b/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
@@ -64,6 +64,11 @@ const makeRequest = (options: RequestOptions, req: IncomingMessage, res: ServerR
     res.writeHead(500)
     res.end(`Internal Server Error: ${e.message}`)
   })
+
+  proxyReq.on('close', () => {
+    log.trace('Proxy request closed; ending response')
+    res.end()
+  })
 }
 
 export interface ReverseProxyOptions {
@@ -108,6 +113,18 @@ export async function startReverseProxy (options?: ReverseProxyOptions): Promise
   })
 
   return async function stopReverseProxy (): Promise<void> {
-    proxyServer?.close()
+    log('Stopping...')
+    await new Promise<void>((resolve, reject) => {
+      // no matter what happens, we need to kill the server
+      proxyServer.closeAllConnections()
+      log('Closed all connections')
+      proxyServer.close((err: any) => {
+        if (err != null) {
+          reject(err)
+        } else {
+          resolve()
+        }
+      })
+    })
   }
 }
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
index 4400856..598c0d5 100644
--- a/packages/verified-fetch/package.json
+++ b/packages/verified-fetch/package.json
@@ -57,10 +57,10 @@
     "release": "aegir release"
   },
   "dependencies": {
-    "@helia/block-brokers": "^3.0.0",
+    "@helia/block-brokers": "^3.0.0-f46700f",
     "@helia/car": "^3.1.5",
     "@helia/http": "^1.0.7",
-    "@helia/interface": "^4.3.0",
+    "@helia/interface": "^4.3.0-f46700f",
     "@helia/ipns": "^7.2.2",
     "@helia/routers": "^1.1.0",
     "@ipld/dag-cbor": "^9.2.0",
@@ -89,7 +89,7 @@
     "@helia/dag-json": "^3.0.4",
     "@helia/json": "^3.0.4",
     "@helia/unixfs": "^3.0.6",
-    "@helia/utils": "^0.3.0",
+    "@helia/utils": "^0.3.1",
     "@ipld/car": "^5.3.0",
     "@libp2p/interface-compliance-tests": "^5.3.4",
     "@libp2p/logger": "^4.0.9",

From b3019ad53dc28281f797ff8eb67559e68a8b86b8 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Tue, 21 May 2024 13:05:58 -0700
Subject: [PATCH 03/25] fix: gateway-conformance expects 301 to path

---
 packages/verified-fetch/src/verified-fetch.ts | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index dc61bce..2136ce8 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -324,8 +324,11 @@ export class VerifiedFetch {
           this.log('could not redirect to %s/ as redirect option was set to "error"', resource)
           throw new TypeError('Failed to fetch')
         } else if (options?.redirect === 'manual') {
-          this.log('returning 301 permanent redirect to %s/', resource)
-          return movedPermanentlyResponse(resource, `${resource}/`)
+          const url = new URL(resource)
+          const redirectPath = `${url.pathname}/`
+          this.log('returning 301 permanent redirect to %s', redirectPath)
+
+          return movedPermanentlyResponse(resource, url.pathname)
         }
 
         // fall-through simulates following the redirect?

From 38839edda50f9968e62b8fb6546b36f3e581924f Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Tue, 21 May 2024 15:31:11 -0700
Subject: [PATCH 04/25] fix: gateway conformance subdomain handling

---
 .../src/utils/handle-redirects.ts             | 66 +++++++++++++++++++
 packages/verified-fetch/src/verified-fetch.ts |  8 ++-
 2 files changed, 73 insertions(+), 1 deletion(-)
 create mode 100644 packages/verified-fetch/src/utils/handle-redirects.ts

diff --git a/packages/verified-fetch/src/utils/handle-redirects.ts b/packages/verified-fetch/src/utils/handle-redirects.ts
new file mode 100644
index 0000000..a2752d2
--- /dev/null
+++ b/packages/verified-fetch/src/utils/handle-redirects.ts
@@ -0,0 +1,66 @@
+import { type AbortOptions, type ComponentLogger } from '@libp2p/interface'
+import { type VerifiedFetchInit, type Resource } from '../index.js'
+import { matchURLString } from './parse-url-string.js'
+import { movedPermanentlyResponse } from './responses.js'
+
+interface GetRedirectResponse {
+  resource: Resource
+  options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions
+  logger: ComponentLogger
+
+}
+
+export async function getRedirectResponse ({ resource, options, logger }: GetRedirectResponse): Promise<null | Response> {
+  const log = logger.forComponent('helia:verified-fetch:get-redirect-response')
+  if (typeof resource !== 'string' || options == null) {
+    return null
+  }
+  log.trace('checking for redirect info')
+  // if x-forwarded-host is passed, we need to set the location header to the subdomain
+  // so that the browser can redirect to the correct subdomain
+  try {
+    // TODO: handle checking if subdomains are enabled and set location to subdomain host instead.
+    const headers = new Headers(options?.headers)
+    // if (headers.get('x-forwarded-host') != null) {
+    const urlParts = matchURLString(resource)
+    const reqUrl = new URL(resource)
+    const actualHost = headers.get('x-forwarded-host') ?? reqUrl.host
+    // const subdomainUrl = new URL(reqUrl, `${reqUrl.protocol}//${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`)
+    const subdomainUrl = new URL(reqUrl)
+    subdomainUrl.host = `${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`
+
+    log.trace('headers.get(\'host\')=%s', headers.get('host'))
+    log.trace('headers.get(\'x-forwarded-host\')=%s', headers.get('x-forwarded-host'))
+    log.trace('headers.get(\'x-forwarded-for\')=%s', headers.get('x-forwarded-for'))
+    if (headers.get('host') != null && headers.get('host') === reqUrl.host) {
+      // log.trace('host header is the same as the request url host, not setting location header')
+      log.trace('host header is the same as the request url host')
+      // return null
+    } else {
+      log.trace('host header is different from the request url host')
+    }
+
+    subdomainUrl.pathname = reqUrl.pathname.replace(`/${urlParts.cidOrPeerIdOrDnsLink}`, '').replace(`/${urlParts.protocol}`, '')
+    // log.trace('subdomain url %s, given input: %s', subdomainUrl.href, `${reqUrl.protocol}//${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`)
+    log.trace('subdomain url %s', subdomainUrl.href)
+    const pathUrl = new URL(reqUrl, `${reqUrl.protocol}//${actualHost}`)
+    log.trace('path url %s', pathUrl.href)
+    // const url = new URL(reqUrl, `${reqUrl.protocol}//${actualHost}`)
+    // try to query subdomain with HEAD request to see if it's supported
+    try {
+      const subdomainTest = await fetch(subdomainUrl, { method: 'HEAD' })
+      if (subdomainTest.ok) {
+        log('subdomain supported, redirecting to subdomain')
+        return movedPermanentlyResponse(resource.toString(), subdomainUrl.href)
+      }
+    } catch (err: any) {
+      log('subdomain not supported, redirecting to path', err)
+      return movedPermanentlyResponse(resource.toString(), pathUrl.href)
+    }
+    // }
+  } catch (e) {
+    // if it's not a full URL, we have nothing left to do.
+    log.error('error setting location header for x-forwarded-host', e)
+  }
+  return null
+}
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index 2136ce8..f8b682d 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -24,7 +24,9 @@ import { getETag } from './utils/get-e-tag.js'
 import { getResolvedAcceptHeader } from './utils/get-resolved-accept-header.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { tarStream } from './utils/get-tar-stream.js'
+import { getRedirectResponse } from './utils/handle-redirects.js'
 import { parseResource } from './utils/parse-resource.js'
+import { type ParsedUrlStringResults } from './utils/parse-url-string.js'
 import { resourceToSessionCacheKey } from './utils/resource-to-cache-key.js'
 import { setCacheControlHeader, setIpfsRoots } from './utils/response-headers.js'
 import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse } from './utils/responses.js'
@@ -32,7 +34,6 @@ import { selectOutputType } from './utils/select-output-type.js'
 import { handlePathWalking, isObjectNode } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, CreateVerifiedFetchOptions, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
 import type { FetchHandlerFunctionArg, RequestFormatShorthand } from './types.js'
-import type { ParsedUrlStringResults } from './utils/parse-url-string'
 import type { Helia, SessionBlockstore } from '@helia/interface'
 import type { Blockstore } from 'interface-blockstore'
 import type { ObjectNode } from 'ipfs-unixfs-exporter'
@@ -511,6 +512,11 @@ export class VerifiedFetch {
     let response: Response
     let reqFormat: RequestFormatShorthand | undefined
 
+    const redirectResponse = await getRedirectResponse({ resource, options, logger: this.helia.logger })
+    if (redirectResponse != null) {
+      return redirectResponse
+    }
+
     const handlerArgs: FetchHandlerFunctionArg = { resource: resource.toString(), cid, path, accept, session: options?.session ?? true, options }
 
     if (accept === 'application/vnd.ipfs.ipns-record') {

From 95fff645886628067ecccf3d5dc821eb639e156b Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Tue, 21 May 2024 16:15:12 -0700
Subject: [PATCH 05/25] fix: more testgatewaysubdomain passing tests

---
 .../src/conformance.spec.ts                   | 20 +++++++++----
 .../src/fixtures/basic-server.ts              | 27 +++++++++++++----
 .../src/fixtures/reverse-proxy.ts             |  1 +
 .../src/utils/handle-redirects.ts             | 29 ++++++++++++++-----
 packages/verified-fetch/src/verified-fetch.ts |  2 +-
 5 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index 7a02c50..1b63798 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -30,10 +30,12 @@ function getGatewayConformanceBinaryPath (): string {
 function getConformanceTestArgs (name: string, gwcArgs: string[] = [], goTestArgs: string[] = []): string[] {
   return [
     'test',
-    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    `--gateway-url=http://${process.env.CONFORMANCE_HOST!}:${process.env.PROXY_PORT!}`,
-    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    `--subdomain-url=http://${process.env.CONFORMANCE_HOST!}:${process.env.PROXY_PORT!}`,
+    // `--gateway-url=http://${process.env.CONFORMANCE_HOST!}:${process.env.PROXY_PORT!}`, // eslint-disable-line @typescript-eslint/no-non-null-assertion
+    '--gateway-url=http://127.0.0.1:3441', // eslint-disable-line @typescript-eslint/no-non-null-assertion
+    // `--gateway-url=http://${process.env.CONFORMANCE_HOST!}`, // eslint-disable-line @typescript-eslint/no-non-null-assertion
+    // `--subdomain-url=http://${process.env.CONFORMANCE_HOST!}:${process.env.PROXY_PORT!}`, // eslint-disable-line @typescript-eslint/no-non-null-assertion
+    `--subdomain-url=http://${process.env.CONFORMANCE_HOST!}:3441`, // eslint-disable-line @typescript-eslint/no-non-null-assertion
+    // `--subdomain-url=http://${process.env.CONFORMANCE_HOST!}`, // eslint-disable-line @typescript-eslint/no-non-null-assertion
     '--verbose',
     '--json', `gwc-report-${name}.json`,
     ...gwcArgs,
@@ -175,7 +177,15 @@ const tests: TestConfig[] = [
   },
   {
     name: 'TestGatewaySubdomains',
-    run: ['TestGatewaySubdomains'],
+    run: [
+      'TestGatewaySubdomains'
+      // 100%
+      // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv0%7D_redirects_to_CIDv1_representation_in_subdomain_%28direct_HTTP%29/Header_Location'
+      // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D_redirects_to_subdomain_%28direct_HTTP%29/Status_code'
+    ],
+    skip: [
+      'TestGatewaySubdomains/.*HTTP_proxy_tunneling_via_CONNECT' // verified fetch should not be doing HTTP proxy tunneling.
+    ],
     successRate: 7.17
   },
   // times out
diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index ad095dc..9aadc57 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -5,6 +5,7 @@ import { httpGatewayRouting } from '@helia/routers'
 import { logger } from '@libp2p/logger'
 import { dns } from '@multiformats/dns'
 import { MemoryBlockstore } from 'blockstore-core'
+import { Agent, setGlobalDispatcher } from 'undici'
 import { contentTypeParser } from './content-type-parser.js'
 import { createVerifiedFetch } from './create-verified-fetch.js'
 import { getLocalDnsResolver } from './get-local-dns-resolver.js'
@@ -72,6 +73,11 @@ async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response,
     res.end()
     return
   }
+  if (req.method === 'HEAD') {
+    res.writeHead(200)
+    res.end()
+    return
+  }
 
   if (req.url == null) {
     // this should never happen
@@ -80,12 +86,16 @@ async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response,
     res.end('Bad Request')
     return
   }
+  // const url = new URL(kuboGateway)
+  // const host = url.host
 
-  const hostname = req.headers.host?.split(':')[0]
-  const host = req.headers['x-forwarded-for'] ?? `${hostname}:${serverPort}`
+  // const hostname = req.headers.host?.split(':')[0]
+  // const host = req.headers['x-forwarded-host'] ?? `localhost:${serverPort}` // `${hostname}:${serverPort}`
+  const fullUrlHref = new URL(req.url, `http://${req.headers.host}`)
+  // req.headers['x-forwarded-host'] = `localhost:${serverPort}`
 
-  const fullUrlHref = req.headers.referer ?? `http://${host}${req.url}`
-  const urlLog = logger(`basic-server:request:${host}${req.url}`)
+  // const fullUrlHref = req.headers.referer ?? `http://${host}${req.url}`
+  const urlLog = logger(`basic-server:request:${fullUrlHref}`)
   urlLog('configuring request')
   urlLog.trace('req.headers: %O', req.headers)
   let requestController: AbortController | null = new AbortController()
@@ -116,7 +126,8 @@ async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response,
 
   try {
     urlLog.trace('calling verified-fetch')
-    const resp = await verifiedFetch(fullUrlHref, { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true })
+    const resp = await verifiedFetch(fullUrlHref.toString(), { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true, headers: req.headers })
+    // const resp = await verifiedFetch(fullUrlHref.toString(), { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true, headers: req.headers })
     urlLog.trace('verified-fetch response status: %d', resp.status)
 
     // loop over headers and set them on the response
@@ -161,6 +172,12 @@ async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response,
 }
 
 export async function startBasicServer ({ kuboGateway, serverPort, IPFS_NS_MAP }: BasicServerOptions): Promise<() => Promise<void>> {
+  const staticDnsAgent = new Agent({
+    connect: {
+      lookup: (_hostname, _options, callback) => { callback(null, [{ address: '0.0.0.0', family: 4 }]) }
+    }
+  })
+  setGlobalDispatcher(staticDnsAgent)
   kuboGateway = kuboGateway ?? process.env.KUBO_GATEWAY
   const useSessions = process.env.USE_SESSIONS !== 'false'
 
diff --git a/packages/gateway-conformance/src/fixtures/reverse-proxy.ts b/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
index 00f61b9..d3d308b 100644
--- a/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
+++ b/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
@@ -1,4 +1,5 @@
 import { request, createServer, type RequestOptions, type IncomingMessage, type ServerResponse } from 'node:http'
+import { connect } from 'node:net'
 import { logger } from '@libp2p/logger'
 
 const log = logger('reverse-proxy')
diff --git a/packages/verified-fetch/src/utils/handle-redirects.ts b/packages/verified-fetch/src/utils/handle-redirects.ts
index a2752d2..ca67825 100644
--- a/packages/verified-fetch/src/utils/handle-redirects.ts
+++ b/packages/verified-fetch/src/utils/handle-redirects.ts
@@ -2,16 +2,19 @@ import { type AbortOptions, type ComponentLogger } from '@libp2p/interface'
 import { type VerifiedFetchInit, type Resource } from '../index.js'
 import { matchURLString } from './parse-url-string.js'
 import { movedPermanentlyResponse } from './responses.js'
+import type { CID } from 'multiformats/cid'
 
 interface GetRedirectResponse {
+  cid: CID
   resource: Resource
   options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions
   logger: ComponentLogger
 
 }
 
-export async function getRedirectResponse ({ resource, options, logger }: GetRedirectResponse): Promise<null | Response> {
+export async function getRedirectResponse ({ resource, options, logger, cid }: GetRedirectResponse): Promise<null | Response> {
   const log = logger.forComponent('helia:verified-fetch:get-redirect-response')
+  const headers = new Headers(options?.headers)
   if (typeof resource !== 'string' || options == null) {
     return null
   }
@@ -20,24 +23,34 @@ export async function getRedirectResponse ({ resource, options, logger }: GetRed
   // so that the browser can redirect to the correct subdomain
   try {
     // TODO: handle checking if subdomains are enabled and set location to subdomain host instead.
-    const headers = new Headers(options?.headers)
     // if (headers.get('x-forwarded-host') != null) {
     const urlParts = matchURLString(resource)
     const reqUrl = new URL(resource)
-    const actualHost = headers.get('x-forwarded-host') ?? reqUrl.host
+    const forwardedHost = headers.get('x-forwarded-host')
+    const actualHost = forwardedHost ?? reqUrl.host
     // const subdomainUrl = new URL(reqUrl, `${reqUrl.protocol}//${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`)
     const subdomainUrl = new URL(reqUrl)
-    subdomainUrl.host = `${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`
+    if (urlParts.protocol === 'ipfs' && cid.version === 0) {
+      subdomainUrl.host = `${cid.toV1()}.ipfs.${actualHost}`
+    } else {
+      subdomainUrl.host = `${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`
+    }
 
     log.trace('headers.get(\'host\')=%s', headers.get('host'))
     log.trace('headers.get(\'x-forwarded-host\')=%s', headers.get('x-forwarded-host'))
     log.trace('headers.get(\'x-forwarded-for\')=%s', headers.get('x-forwarded-for'))
-    if (headers.get('host') != null && headers.get('host') === reqUrl.host) {
+    const headerHost = headers.get('host')
+
+    if (headerHost != null && !subdomainUrl.host.includes(headerHost)) {
+      log.trace('host header is not the same as the subdomain url host, not setting location header')
+      return null
+    }
+    if (reqUrl.host === subdomainUrl.host) {
       // log.trace('host header is the same as the request url host, not setting location header')
-      log.trace('host header is the same as the request url host')
-      // return null
+      log.trace('req url is the same as the subdomain url, not setting location header')
+      return null
     } else {
-      log.trace('host header is different from the request url host')
+      log.trace('req url is different from the subdomain url, attempting to set the location header')
     }
 
     subdomainUrl.pathname = reqUrl.pathname.replace(`/${urlParts.cidOrPeerIdOrDnsLink}`, '').replace(`/${urlParts.protocol}`, '')
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index f8b682d..f5cf43f 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -512,7 +512,7 @@ export class VerifiedFetch {
     let response: Response
     let reqFormat: RequestFormatShorthand | undefined
 
-    const redirectResponse = await getRedirectResponse({ resource, options, logger: this.helia.logger })
+    const redirectResponse = await getRedirectResponse({ resource, options, logger: this.helia.logger, cid })
     if (redirectResponse != null) {
       return redirectResponse
     }

From bce22393a32c80a7ccfe0092b3cd5c787742d6a7 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Tue, 21 May 2024 16:23:10 -0700
Subject: [PATCH 06/25] fix: more testgatewaysubdomain passing tests

---
 packages/gateway-conformance/src/conformance.spec.ts  | 3 ++-
 packages/verified-fetch/src/utils/handle-redirects.ts | 7 ++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index 1b63798..e745761 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -180,7 +180,8 @@ const tests: TestConfig[] = [
     run: [
       'TestGatewaySubdomains'
       // 100%
-      // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv0%7D_redirects_to_CIDv1_representation_in_subdomain_%28direct_HTTP%29/Header_Location'
+      // 'TestGatewaySubdomains/request_for_%7BCID%7D.ipfs.example.com_should_return_expected_payload_%28direct_HTTP%29/Status_code',
+      // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv0%7D_redirects_to_CIDv1_representation_in_subdomain_%28direct_HTTP%29/Header_Location',
       // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D_redirects_to_subdomain_%28direct_HTTP%29/Status_code'
     ],
     skip: [
diff --git a/packages/verified-fetch/src/utils/handle-redirects.ts b/packages/verified-fetch/src/utils/handle-redirects.ts
index ca67825..06bd0af 100644
--- a/packages/verified-fetch/src/utils/handle-redirects.ts
+++ b/packages/verified-fetch/src/utils/handle-redirects.ts
@@ -27,6 +27,7 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
     const urlParts = matchURLString(resource)
     const reqUrl = new URL(resource)
     const forwardedHost = headers.get('x-forwarded-host')
+    const headerHost = headers.get('host')
     const actualHost = forwardedHost ?? reqUrl.host
     // const subdomainUrl = new URL(reqUrl, `${reqUrl.protocol}//${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`)
     const subdomainUrl = new URL(reqUrl)
@@ -36,10 +37,14 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
       subdomainUrl.host = `${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`
     }
 
+    if (headerHost?.includes(urlParts.protocol) === true && subdomainUrl.host.includes(headerHost)) {
+      log.trace('request was for a subdomain already, not setting location header')
+      return null
+    }
+
     log.trace('headers.get(\'host\')=%s', headers.get('host'))
     log.trace('headers.get(\'x-forwarded-host\')=%s', headers.get('x-forwarded-host'))
     log.trace('headers.get(\'x-forwarded-for\')=%s', headers.get('x-forwarded-for'))
-    const headerHost = headers.get('host')
 
     if (headerHost != null && !subdomainUrl.host.includes(headerHost)) {
       log.trace('host header is not the same as the subdomain url host, not setting location header')

From 1b96aa2147de00e02ffcf58045ea499b9ebaa5e3 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Tue, 21 May 2024 17:00:51 -0700
Subject: [PATCH 07/25] fix: some redirect and url parsing

---
 .../src/utils/handle-redirects.ts             | 22 +++++++++++++------
 .../src/utils/parse-url-string.ts             |  2 +-
 packages/verified-fetch/src/verified-fetch.ts | 11 ++++++----
 .../test/utils/parse-url-string.spec.ts       | 14 ++++++++++++
 4 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/packages/verified-fetch/src/utils/handle-redirects.ts b/packages/verified-fetch/src/utils/handle-redirects.ts
index 06bd0af..2416b98 100644
--- a/packages/verified-fetch/src/utils/handle-redirects.ts
+++ b/packages/verified-fetch/src/utils/handle-redirects.ts
@@ -14,10 +14,19 @@ interface GetRedirectResponse {
 
 export async function getRedirectResponse ({ resource, options, logger, cid }: GetRedirectResponse): Promise<null | Response> {
   const log = logger.forComponent('helia:verified-fetch:get-redirect-response')
+
+  if (typeof resource !== 'string' || options == null || ['ipfs://', 'ipns://'].some((prefix) => resource.startsWith(prefix))) {
+    return null
+  }
   const headers = new Headers(options?.headers)
-  if (typeof resource !== 'string' || options == null) {
+  const forwardedHost = headers.get('x-forwarded-host')
+  const headerHost = headers.get('host')
+  const forwardedFor = headers.get('x-forwarded-for')
+  if (forwardedFor == null && forwardedHost == null && headerHost == null) {
+    log.trace('no redirect info found in headers')
     return null
   }
+
   log.trace('checking for redirect info')
   // if x-forwarded-host is passed, we need to set the location header to the subdomain
   // so that the browser can redirect to the correct subdomain
@@ -26,8 +35,6 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
     // if (headers.get('x-forwarded-host') != null) {
     const urlParts = matchURLString(resource)
     const reqUrl = new URL(resource)
-    const forwardedHost = headers.get('x-forwarded-host')
-    const headerHost = headers.get('host')
     const actualHost = forwardedHost ?? reqUrl.host
     // const subdomainUrl = new URL(reqUrl, `${reqUrl.protocol}//${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`)
     const subdomainUrl = new URL(reqUrl)
@@ -42,9 +49,9 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
       return null
     }
 
-    log.trace('headers.get(\'host\')=%s', headers.get('host'))
-    log.trace('headers.get(\'x-forwarded-host\')=%s', headers.get('x-forwarded-host'))
-    log.trace('headers.get(\'x-forwarded-for\')=%s', headers.get('x-forwarded-for'))
+    log.trace('headers.get(\'host\')=%s', headerHost)
+    log.trace('headers.get(\'x-forwarded-host\')=%s', forwardedHost)
+    log.trace('headers.get(\'x-forwarded-for\')=%s', forwardedFor)
 
     if (headerHost != null && !subdomainUrl.host.includes(headerHost)) {
       log.trace('host header is not the same as the subdomain url host, not setting location header')
@@ -58,10 +65,11 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
       log.trace('req url is different from the subdomain url, attempting to set the location header')
     }
 
-    subdomainUrl.pathname = reqUrl.pathname.replace(`/${urlParts.cidOrPeerIdOrDnsLink}`, '').replace(`/${urlParts.protocol}`, '')
+    subdomainUrl.pathname = reqUrl.pathname.replace(`/${urlParts.cidOrPeerIdOrDnsLink}`, '').replace(`/${urlParts.protocol}`, '') + '/'
     // log.trace('subdomain url %s, given input: %s', subdomainUrl.href, `${reqUrl.protocol}//${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`)
     log.trace('subdomain url %s', subdomainUrl.href)
     const pathUrl = new URL(reqUrl, `${reqUrl.protocol}//${actualHost}`)
+    pathUrl.pathname = reqUrl.pathname + '/'
     log.trace('path url %s', pathUrl.href)
     // const url = new URL(reqUrl, `${reqUrl.protocol}//${actualHost}`)
     // try to query subdomain with HEAD request to see if it's supported
diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts
index 63e761d..65e652b 100644
--- a/packages/verified-fetch/src/utils/parse-url-string.ts
+++ b/packages/verified-fetch/src/utils/parse-url-string.ts
@@ -72,7 +72,7 @@ function matchUrlGroupsGuard (groups?: null | { [key in string]: string; } | Mat
 }
 
 export function matchURLString (urlString: string): MatchUrlGroups {
-  for (const pattern of [URL_REGEX, PATH_REGEX, PATH_GATEWAY_REGEX, SUBDOMAIN_GATEWAY_REGEX]) {
+  for (const pattern of [SUBDOMAIN_GATEWAY_REGEX, URL_REGEX, PATH_GATEWAY_REGEX, PATH_REGEX]) {
     const match = urlString.match(pattern)
 
     if (matchUrlGroupsGuard(match?.groups)) {
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index f5cf43f..d8524be 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -325,11 +325,14 @@ export class VerifiedFetch {
           this.log('could not redirect to %s/ as redirect option was set to "error"', resource)
           throw new TypeError('Failed to fetch')
         } else if (options?.redirect === 'manual') {
-          const url = new URL(resource)
-          const redirectPath = `${url.pathname}/`
-          this.log('returning 301 permanent redirect to %s', redirectPath)
+          // const url = new URL(resource)
+          // const redirectPath = `${url.pathname}/`
+          // this.log('returning 301 permanent redirect to %s', redirectPath)
 
-          return movedPermanentlyResponse(resource, url.pathname)
+          // return movedPermanentlyResponse(resource, url.pathname)
+
+          this.log('returning 301 permanent redirect to %s/', resource)
+          return movedPermanentlyResponse(resource, `${resource}/`)
         }
 
         // fall-through simulates following the redirect?
diff --git a/packages/verified-fetch/test/utils/parse-url-string.spec.ts b/packages/verified-fetch/test/utils/parse-url-string.spec.ts
index c541862..08e2a49 100644
--- a/packages/verified-fetch/test/utils/parse-url-string.spec.ts
+++ b/packages/verified-fetch/test/utils/parse-url-string.spec.ts
@@ -931,4 +931,18 @@ describe('parseUrlString', () => {
       })
     })
   })
+
+  describe('subdomainURLs with paths', () => {
+    it('should correctly parse a subdomain that also has /ipfs in the path', async () => {
+      // straight from gateway-conformance test: http://bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:3441/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am
+      await assertMatchUrl(
+        'http://bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:3441/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am', {
+          protocol: 'ipfs',
+          cid: 'bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am',
+          path: 'ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am',
+          query: {}
+        }
+      )
+    })
+  })
 })

From cd9db44b2ed57d1ffab2131f132d58daa54cc5e1 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Tue, 21 May 2024 18:02:39 -0700
Subject: [PATCH 08/25] test: make sure test constructs proper unixfs data

---
 packages/verified-fetch/src/verified-fetch.ts       | 12 +++++++++++-
 .../verified-fetch/test/content-type-parser.spec.ts | 13 +++++++++----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index d8524be..9d37641 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -29,7 +29,7 @@ import { parseResource } from './utils/parse-resource.js'
 import { type ParsedUrlStringResults } from './utils/parse-url-string.js'
 import { resourceToSessionCacheKey } from './utils/resource-to-cache-key.js'
 import { setCacheControlHeader, setIpfsRoots } from './utils/response-headers.js'
-import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse } from './utils/responses.js'
+import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse, notFoundResponse } from './utils/responses.js'
 import { selectOutputType } from './utils/select-output-type.js'
 import { handlePathWalking, isObjectNode } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, CreateVerifiedFetchOptions, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
@@ -410,6 +410,16 @@ export class VerifiedFetch {
   }
 
   private async handleRaw ({ resource, cid, path, session, options, accept }: FetchHandlerFunctionArg): Promise<Response> {
+    /**
+     * if we have a path, we can't walk it, so we need to return a 404.
+     *
+     * @see https://github.com/ipfs/gateway-conformance/blob/26994cfb056b717a23bf694ce4e94386728748dd/tests/subdomain_gateway_ipfs_test.go#L198-L204
+     */
+    if (path !== '') {
+      this.log.trace('404-ing raw codec request for %c/%s', cid, path)
+      return notFoundResponse(resource, 'Raw codec does not support paths')
+    }
+
     const byteRangeContext = new ByteRangeContext(this.helia.logger, options?.headers)
     const blockstore = this.getBlockstore(cid, resource, session, options)
     const result = await blockstore.get(cid, options)
diff --git a/packages/verified-fetch/test/content-type-parser.spec.ts b/packages/verified-fetch/test/content-type-parser.spec.ts
index 54deb03..fb6f787 100644
--- a/packages/verified-fetch/test/content-type-parser.spec.ts
+++ b/packages/verified-fetch/test/content-type-parser.spec.ts
@@ -84,10 +84,14 @@ describe('content-type-parser', () => {
 
   it('is passed a filename from a deep traversal if it is available', async () => {
     const fs = unixfs(helia)
-    const deepDirCid = await fs.addFile({
-      path: 'foo/bar/a-file.html',
-      content: uint8ArrayFromString('<html><body>Hello world</body></html>')
-    })
+
+    let barDir = await fs.addDirectory({ path: './bar' })
+    const aFileHtml = await fs.addFile({ path: './bar/a-file.html', content: uint8ArrayFromString('<html><body>Hello world</body></html>') })
+    barDir = await fs.cp(aFileHtml, barDir, 'a-file.html')
+    let fooDir = await fs.addDirectory({ path: './foo' })
+    fooDir = await fs.cp(barDir, fooDir, 'bar')
+    let deepDirCid = await fs.addDirectory()
+    deepDirCid = await fs.cp(fooDir, deepDirCid, 'foo')
 
     verifiedFetch = new VerifiedFetch({
       helia
@@ -95,6 +99,7 @@ describe('content-type-parser', () => {
       contentTypeParser: async (data, fileName) => fileName
     })
     const resp = await verifiedFetch.fetch(`ipfs://${deepDirCid}/foo/bar/a-file.html`)
+    expect(resp.status).to.equal(200)
     expect(resp.headers.get('content-type')).to.equal('a-file.html')
   })
 

From b329b2dbde2e9dbe2c9de0c53991be922021dc1a Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Tue, 21 May 2024 18:50:07 -0700
Subject: [PATCH 09/25] fix: more gwc improvements for testgatewaysubdomains

---
 .../src/conformance.spec.ts                   |  5 +++-
 .../src/fixtures/basic-server.ts              | 30 ++++++++++++++-----
 .../src/fixtures/kubo-mgmt.ts                 |  2 +-
 .../src/utils/handle-redirects.ts             | 16 ++++++++--
 4 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index e745761..b8994aa 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -180,6 +180,9 @@ const tests: TestConfig[] = [
     run: [
       'TestGatewaySubdomains'
       // 100%
+      // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D%2F%7Bfilename_with_percent_encoding%7D_redirects_to_subdomain_%28direct_HTTP%29'
+      // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D%2F%7Bfilename_with_percent_encoding%7D_redirects_to_subdomain_%28direct_HTTP%29/Status_code'
+      // 'TestGatewaySubdomains/request_for_%7BCID%7D.ipfs.example.com%2Fipfs%2F%7BCID%7D_should_return_HTTP_404_%28direct_HTTP%29/Status_code'
       // 'TestGatewaySubdomains/request_for_%7BCID%7D.ipfs.example.com_should_return_expected_payload_%28direct_HTTP%29/Status_code',
       // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv0%7D_redirects_to_CIDv1_representation_in_subdomain_%28direct_HTTP%29/Header_Location',
       // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D_redirects_to_subdomain_%28direct_HTTP%29/Status_code'
@@ -187,7 +190,7 @@ const tests: TestConfig[] = [
     skip: [
       'TestGatewaySubdomains/.*HTTP_proxy_tunneling_via_CONNECT' // verified fetch should not be doing HTTP proxy tunneling.
     ],
-    successRate: 7.17
+    successRate: 41.35
   },
   // times out
   // {
diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index 9aadc57..eef951a 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -86,15 +86,17 @@ async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response,
     res.end('Bad Request')
     return
   }
-  // const url = new URL(kuboGateway)
-  // const host = url.host
 
-  // const hostname = req.headers.host?.split(':')[0]
-  // const host = req.headers['x-forwarded-host'] ?? `localhost:${serverPort}` // `${hostname}:${serverPort}`
+  // @see https://github.com/ipfs/gateway-conformance/issues/185#issuecomment-2123708150
+  let fixingGwcAnnoyance = false
+  if (req.headers.host != null && (req.headers.host === 'localhost' || req.headers.Host === 'localhost')) {
+    log.trace('set fixingGwcAnnoyance to true')
+    fixingGwcAnnoyance = true
+    req.headers.host = `localhost:${serverPort}`
+  }
+
   const fullUrlHref = new URL(req.url, `http://${req.headers.host}`)
-  // req.headers['x-forwarded-host'] = `localhost:${serverPort}`
 
-  // const fullUrlHref = req.headers.referer ?? `http://${host}${req.url}`
   const urlLog = logger(`basic-server:request:${fullUrlHref}`)
   urlLog('configuring request')
   urlLog.trace('req.headers: %O', req.headers)
@@ -127,13 +129,25 @@ async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response,
   try {
     urlLog.trace('calling verified-fetch')
     const resp = await verifiedFetch(fullUrlHref.toString(), { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true, headers: req.headers })
-    // const resp = await verifiedFetch(fullUrlHref.toString(), { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true, headers: req.headers })
     urlLog.trace('verified-fetch response status: %d', resp.status)
 
     // loop over headers and set them on the response
     const headers: Record<string, string> = {}
     for (const [key, value] of resp.headers.entries()) {
-      headers[key] = value
+      if (fixingGwcAnnoyance) {
+        urlLog.trace('need to fix GWC annoyance.')
+        if (value.includes(`localhost:${serverPort}`) === true) {
+          const newValue = value.replace(`localhost:${serverPort}`, 'localhost')
+          urlLog.trace('fixing GWC annoyance. Replacing Header[%s] value of "%s" with "%s"', key, value, newValue)
+          // we need to fix any Location, or other headers that have localhost without port in them.
+          headers[key] = newValue
+        } else {
+          urlLog.trace('NOT fixing GWC annoyance. Setting Header[%s] value of "%s"', key, value)
+          headers[key] = value
+        }
+      } else {
+        headers[key] = value
+      }
     }
 
     res.writeHead(resp.status, headers)
diff --git a/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts b/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts
index faed104..23901d4 100644
--- a/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts
+++ b/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts
@@ -108,7 +108,7 @@ export async function loadFixtures (kuboRepoDir: string, proxyPort: number): Pro
 
   const json = await readFile(`${GWC_FIXTURES_PATH}/dnslinks.json`, 'utf-8')
   const { subdomains, domains } = JSON.parse(json)
-  const subdomainDnsLinks = Object.entries(subdomains).map(([key, value]) => `${key}.localhost:${value}`).join(',')
+  const subdomainDnsLinks = Object.entries(subdomains).map(([key, value]) => `${key}.localhost%3A${3441}:${value}`).join(',')
   const domainDnsLinks = Object.entries(domains).map(([key, value]) => `${key}:${value}`).join(',')
   const ipfsNsMap = `${domainDnsLinks},${subdomainDnsLinks}`
 
diff --git a/packages/verified-fetch/src/utils/handle-redirects.ts b/packages/verified-fetch/src/utils/handle-redirects.ts
index 2416b98..1cff2cb 100644
--- a/packages/verified-fetch/src/utils/handle-redirects.ts
+++ b/packages/verified-fetch/src/utils/handle-redirects.ts
@@ -12,12 +12,21 @@ interface GetRedirectResponse {
 
 }
 
+function maybeAddTraillingSlash (path: string): string {
+  // if it has an extension-like ending, don't add a trailing slash
+  if (path.match(/\.[a-zA-Z0-9]{1,4}$/) != null) {
+    return path
+  }
+  return path.endsWith('/') ? path : `${path}/`
+}
+
 export async function getRedirectResponse ({ resource, options, logger, cid }: GetRedirectResponse): Promise<null | Response> {
   const log = logger.forComponent('helia:verified-fetch:get-redirect-response')
 
   if (typeof resource !== 'string' || options == null || ['ipfs://', 'ipns://'].some((prefix) => resource.startsWith(prefix))) {
     return null
   }
+
   const headers = new Headers(options?.headers)
   const forwardedHost = headers.get('x-forwarded-host')
   const headerHost = headers.get('host')
@@ -65,11 +74,11 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
       log.trace('req url is different from the subdomain url, attempting to set the location header')
     }
 
-    subdomainUrl.pathname = reqUrl.pathname.replace(`/${urlParts.cidOrPeerIdOrDnsLink}`, '').replace(`/${urlParts.protocol}`, '') + '/'
+    subdomainUrl.pathname = maybeAddTraillingSlash(reqUrl.pathname.replace(`/${urlParts.cidOrPeerIdOrDnsLink}`, '').replace(`/${urlParts.protocol}`, ''))
     // log.trace('subdomain url %s, given input: %s', subdomainUrl.href, `${reqUrl.protocol}//${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`)
     log.trace('subdomain url %s', subdomainUrl.href)
     const pathUrl = new URL(reqUrl, `${reqUrl.protocol}//${actualHost}`)
-    pathUrl.pathname = reqUrl.pathname + '/'
+    pathUrl.pathname = maybeAddTraillingSlash(reqUrl.pathname)
     log.trace('path url %s', pathUrl.href)
     // const url = new URL(reqUrl, `${reqUrl.protocol}//${actualHost}`)
     // try to query subdomain with HEAD request to see if it's supported
@@ -78,6 +87,9 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
       if (subdomainTest.ok) {
         log('subdomain supported, redirecting to subdomain')
         return movedPermanentlyResponse(resource.toString(), subdomainUrl.href)
+      } else {
+        log('subdomain not supported, subdomain failed with status %s %s', subdomainTest.status, subdomainTest.statusText)
+        throw new Error('subdomain not supported')
       }
     } catch (err: any) {
       log('subdomain not supported, redirecting to path', err)

From ba5f6a0cff550ea4980fd1f6d18789455e1f55fe Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Tue, 21 May 2024 19:08:10 -0700
Subject: [PATCH 10/25] test: some adjustments of default tests enabled

---
 packages/gateway-conformance/src/conformance.spec.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index b8994aa..fd30ada 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -189,6 +189,10 @@ const tests: TestConfig[] = [
     ],
     skip: [
       'TestGatewaySubdomains/.*HTTP_proxy_tunneling_via_CONNECT' // verified fetch should not be doing HTTP proxy tunneling.
+      // TODO: add directory listing support to verified-fetch
+      // 'TestGatewaySubdomains/.*directory_listing_at_%7Bcid%7D.ipfs.example.com%2Fsub%2Fdir_%28direct_HTTP%29',
+      // 'TestGatewaySubdomains/valid_file_and_subdirectory_paths_in_directory_listing_at_%7Bcid%7D.ipfs.example.com_%28direct_HTTP%29/Status_code',
+      // 'TestGatewaySubdomains/valid_file_and_subdirectory_paths_in_directory_listing_at_%7Bcid%7D.ipfs.example.com_%28direct_HTTP%29/Body'
     ],
     successRate: 41.35
   },

From 58d6835684cc4d6d28e11471d393411e769b8ee0 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Tue, 21 May 2024 19:14:45 -0700
Subject: [PATCH 11/25] test: adjust total success expectation

---
 packages/gateway-conformance/src/conformance.spec.ts       | 2 +-
 packages/gateway-conformance/src/fixtures/basic-server.ts  | 2 +-
 packages/gateway-conformance/src/fixtures/reverse-proxy.ts | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index fd30ada..45f6935 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -401,7 +401,7 @@ describe('@helia/verified-fetch - gateway conformance', function () {
 
       const { successRate } = await getReportDetails('gwc-report-all.json')
 
-      expect(successRate).to.be.greaterThanOrEqual(15.7)
+      expect(successRate).to.be.greaterThanOrEqual(22.61)
     })
   })
 })
diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index eef951a..b13d20b 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -90,7 +90,7 @@ async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response,
   // @see https://github.com/ipfs/gateway-conformance/issues/185#issuecomment-2123708150
   let fixingGwcAnnoyance = false
   if (req.headers.host != null && (req.headers.host === 'localhost' || req.headers.Host === 'localhost')) {
-    log.trace('set fixingGwcAnnoyance to true')
+    log.trace('set fixingGwcAnnoyance to true for %s', req.url)
     fixingGwcAnnoyance = true
     req.headers.host = `localhost:${serverPort}`
   }
diff --git a/packages/gateway-conformance/src/fixtures/reverse-proxy.ts b/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
index d3d308b..00f61b9 100644
--- a/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
+++ b/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
@@ -1,5 +1,4 @@
 import { request, createServer, type RequestOptions, type IncomingMessage, type ServerResponse } from 'node:http'
-import { connect } from 'node:net'
 import { logger } from '@libp2p/logger'
 
 const log = logger('reverse-proxy')

From e3c11bd7500e48c134e0f4e731c2fbfb1f67e314 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Wed, 22 May 2024 09:06:27 -0700
Subject: [PATCH 12/25] test: update latest sucess rates

---
 .../src/conformance.spec.ts                   | 57 ++++++++++---------
 1 file changed, 31 insertions(+), 26 deletions(-)

diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index 45f6935..20ea3ef 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -61,7 +61,7 @@ const tests: TestConfig[] = [
   {
     name: 'TestDagPbConversion',
     run: ['TestDagPbConversion'],
-    successRate: 35.38
+    successRate: 26.15
   },
   {
     name: 'TestPlainCodec',
@@ -71,7 +71,7 @@ const tests: TestConfig[] = [
   {
     name: 'TestPathing',
     run: ['TestPathing'],
-    successRate: 26.67
+    successRate: 40
   },
   {
     name: 'TestDNSLinkGatewayUnixFSDirectoryListing',
@@ -89,7 +89,7 @@ const tests: TestConfig[] = [
   {
     name: 'TestGatewayJsonCbor',
     run: ['TestGatewayJsonCbor'],
-    successRate: 44.44
+    successRate: 22.22
   },
   // currently results in an infinite loop without verified-fetch stopping the request whether sessions are enabled or not.
   // {
@@ -116,7 +116,7 @@ const tests: TestConfig[] = [
   {
     name: 'TestGatewayBlock',
     run: ['TestGatewayBlock'],
-    successRate: 37.93
+    successRate: 20.69
   },
   {
     name: 'TestTrustlessRawRanges',
@@ -126,7 +126,8 @@ const tests: TestConfig[] = [
   {
     name: 'TestTrustlessRaw',
     run: ['TestTrustlessRaw'],
-    successRate: 55.56
+    skip: ['TestTrustlessRawRanges'],
+    successRate: 70.83
   },
   {
     name: 'TestGatewayIPNSRecord',
@@ -136,7 +137,7 @@ const tests: TestConfig[] = [
   {
     name: 'TestTrustlessCarOrderAndDuplicates',
     run: ['TestTrustlessCarOrderAndDuplicates'],
-    successRate: 13.79
+    successRate: 44.83
   },
   // times out
   // {
@@ -147,7 +148,7 @@ const tests: TestConfig[] = [
   {
     name: 'TestTrustlessCarDagScopeAll',
     run: ['TestTrustlessCarDagScopeAll'],
-    successRate: 36.36
+    successRate: 54.55
   },
   // {
   //   name: 'TestTrustlessCarDagScopeEntity',
@@ -159,12 +160,13 @@ const tests: TestConfig[] = [
   //   run: ['TestTrustlessCarDagScopeBlock'],
   //   successRate: 34.69
   // },
-  {
-    name: 'TestTrustlessCarPathing',
-    run: ['TestTrustlessCarPathing'],
-    successRate: 35,
-    timeout: 240000
-  },
+  // {
+  //   // passes at the set successRate, but takes incredibly long (consistently ~2m).. disabling for now.
+  //   name: 'TestTrustlessCarPathing',
+  //   run: ['TestTrustlessCarPathing'],
+  //   successRate: 35,
+  //   timeout: 130000
+  // },
   // {
   //   name: 'TestSubdomainGatewayDNSLinkInlining',
   //   run: ['TestSubdomainGatewayDNSLinkInlining'],
@@ -215,7 +217,8 @@ const tests: TestConfig[] = [
   {
     name: 'TestRedirectsFileSupport',
     run: ['TestRedirectsFileSupport'],
-    successRate: 2.33
+    skip: ['TestRedirectsFileSupportWithDNSLink'],
+    successRate: 0
   },
   {
     name: 'TestPathGatewayMiscellaneous',
@@ -225,7 +228,7 @@ const tests: TestConfig[] = [
   {
     name: 'TestGatewayUnixFSFileRanges',
     run: ['TestGatewayUnixFSFileRanges'],
-    successRate: 40
+    successRate: 46.67
   },
   {
     name: 'TestGatewaySymlink',
@@ -237,12 +240,14 @@ const tests: TestConfig[] = [
     run: ['TestGatewayCacheWithIPNS'],
     successRate: 35.71
   },
-  {
-    name: 'TestGatewayCache',
-    run: ['TestGatewayCache'],
-    successRate: 60.71,
-    timeout: 1200000
-  },
+  // {
+  //   // passes at the set successRate, but takes incredibly long (consistently ~2m).. disabling for now.
+  //   name: 'TestGatewayCache',
+  //   run: ['TestGatewayCache'],
+  //   skip: ['TestGatewayCacheWithIPNS'],
+  //   successRate: 59.38,
+  //   timeout: 1200000
+  // },
   {
     name: 'TestUnixFSDirectoryListing',
     run: ['TestUnixFSDirectoryListing'],
@@ -250,13 +255,13 @@ const tests: TestConfig[] = [
       'TestUnixFSDirectoryListingOnSubdomainGateway',
       'TestUnixFSDirectoryListing/.*TODO:_cleanup_Kubo-specifics'
     ],
-    successRate: 16.67,
+    successRate: 50,
     timeout: 1200000
   },
   {
     name: 'TestTar',
     run: ['TestTar'],
-    successRate: 50
+    successRate: 62.5
   }
 ]
 
@@ -310,9 +315,9 @@ describe('@helia/verified-fetch - gateway conformance', function () {
   describe('smokeTests', () => {
     [
       ['basic server path request works', `http://localhost:${process.env.SERVER_PORT}/ipfs/bafkqabtimvwgy3yk`],
-      ['proxy server path request works', `http://localhost:${process.env.PROXY_PORT}/ipfs/bafkqabtimvwgy3yk`],
-      ['basic server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.SERVER_PORT}`],
-      ['proxy server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.PROXY_PORT}`]
+      // ['proxy server path request works', `http://localhost:${process.env.PROXY_PORT}/ipfs/bafkqabtimvwgy3yk`],
+      ['basic server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.SERVER_PORT}`]
+      // ['proxy server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.PROXY_PORT}`]
     ].forEach(([name, url]) => {
       it(name, async () => {
         const resp = await fetch(url)

From ddd69004d21a8fcd2f1899cc8ee6b969afcd85bd Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Wed, 22 May 2024 09:26:23 -0700
Subject: [PATCH 13/25] test: update 'all' test success rate and remove reverse
 proxy

---
 packages/gateway-conformance/.aegir.js        |  18 +--
 .../src/conformance.spec.ts                   |  32 ++---
 .../gateway-conformance/src/demo-server.ts    |  15 +-
 .../src/fixtures/kubo-mgmt.ts                 |   6 +-
 .../src/fixtures/reverse-proxy.ts             | 130 ------------------
 5 files changed, 17 insertions(+), 184 deletions(-)
 delete mode 100644 packages/gateway-conformance/src/fixtures/reverse-proxy.ts

diff --git a/packages/gateway-conformance/.aegir.js b/packages/gateway-conformance/.aegir.js
index 698b527..49a3ca9 100644
--- a/packages/gateway-conformance/.aegir.js
+++ b/packages/gateway-conformance/.aegir.js
@@ -15,11 +15,10 @@ export default {
       const { createKuboNode } = await import('./dist/src/fixtures/create-kubo.js')
       const KUBO_PORT = await getPort(3440)
       const SERVER_PORT = await getPort(3441)
-      const PROXY_PORT = await getPort(3442)
       const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_PORT)
       await controller.start()
       const { loadKuboFixtures } = await import('./dist/src/fixtures/kubo-mgmt.js')
-      const IPFS_NS_MAP = await loadKuboFixtures(repoPath, PROXY_PORT)
+      const IPFS_NS_MAP = await loadKuboFixtures(repoPath)
       const kuboGateway = gatewayUrl
 
       const { startBasicServer } = await import('./dist/src/fixtures/basic-server.js')
@@ -31,26 +30,15 @@ export default {
         log.error(err)
       })
 
-      const { startReverseProxy } = await import('./dist/src/fixtures/reverse-proxy.js')
-      const stopReverseProxy = await startReverseProxy({
-        backendPort: SERVER_PORT,
-        targetHost: 'localhost',
-        proxyPort: PROXY_PORT
-      }).catch((err) => {
-        log.error(err)
-      })
-
       const CONFORMANCE_HOST = 'localhost'
 
       return {
         controller,
-        stopReverseProxy,
         stopBasicServer,
         env: {
           IPFS_NS_MAP,
           CONFORMANCE_HOST,
           KUBO_PORT: `${KUBO_PORT}`,
-          PROXY_PORT: `${PROXY_PORT}`,
           SERVER_PORT: `${SERVER_PORT}`,
           KUBO_GATEWAY: kuboGateway
         }
@@ -62,10 +50,6 @@ export default {
       await beforeResult.controller.stop()
       log('controller stopped')
 
-      // @ts-expect-error - broken aegir types
-      await beforeResult.stopReverseProxy()
-      log('reverse proxy stopped')
-
       // @ts-expect-error - broken aegir types
       await beforeResult.stopBasicServer()
       log('basic server stopped')
diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index 20ea3ef..cf55d85 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-unused-vars */
 /* eslint-env mocha */
 import { readFile } from 'node:fs/promises'
 import { homedir } from 'node:os'
@@ -30,12 +29,8 @@ function getGatewayConformanceBinaryPath (): string {
 function getConformanceTestArgs (name: string, gwcArgs: string[] = [], goTestArgs: string[] = []): string[] {
   return [
     'test',
-    // `--gateway-url=http://${process.env.CONFORMANCE_HOST!}:${process.env.PROXY_PORT!}`, // eslint-disable-line @typescript-eslint/no-non-null-assertion
-    '--gateway-url=http://127.0.0.1:3441', // eslint-disable-line @typescript-eslint/no-non-null-assertion
-    // `--gateway-url=http://${process.env.CONFORMANCE_HOST!}`, // eslint-disable-line @typescript-eslint/no-non-null-assertion
-    // `--subdomain-url=http://${process.env.CONFORMANCE_HOST!}:${process.env.PROXY_PORT!}`, // eslint-disable-line @typescript-eslint/no-non-null-assertion
-    `--subdomain-url=http://${process.env.CONFORMANCE_HOST!}:3441`, // eslint-disable-line @typescript-eslint/no-non-null-assertion
-    // `--subdomain-url=http://${process.env.CONFORMANCE_HOST!}`, // eslint-disable-line @typescript-eslint/no-non-null-assertion
+    `--gateway-url=http://127.0.0.1:${process.env.SERVER_PORT}`,
+    `--subdomain-url=http://${process.env.CONFORMANCE_HOST}:${process.env.SERVER_PORT}`,
     '--verbose',
     '--json', `gwc-report-${name}.json`,
     ...gwcArgs,
@@ -293,9 +288,6 @@ describe('@helia/verified-fetch - gateway conformance', function () {
     if (process.env.KUBO_GATEWAY == null) {
       throw new Error('KUBO_GATEWAY env var is required')
     }
-    if (process.env.PROXY_PORT == null) {
-      throw new Error('PROXY_PORT env var is required')
-    }
     if (process.env.SERVER_PORT == null) {
       throw new Error('SERVER_PORT env var is required')
     }
@@ -303,7 +295,7 @@ describe('@helia/verified-fetch - gateway conformance', function () {
       throw new Error('CONFORMANCE_HOST env var is required')
     }
     // see https://stackoverflow.com/questions/71074255/use-custom-dns-resolver-for-any-request-in-nodejs
-    // EVERY undici/fetch request host resolves to local IP. Node.js does not resolve reverse-proxy requests properly
+    // EVERY undici/fetch request host resolves to local IP. Without this, Node.js does not resolve subdomain requests properly
     const staticDnsAgent = new Agent({
       connect: {
         lookup: (_hostname, _options, callback) => { callback(null, [{ address: '0.0.0.0', family: 4 }]) }
@@ -388,25 +380,17 @@ describe('@helia/verified-fetch - gateway conformance', function () {
       this.timeout(200000)
       const log = logger.forComponent('all')
 
-      // TODO: unskip when verified-fetch is no longer infinitely looping on requests.
-      const toSkip = [
-        'TestNativeDag',
-        'TestTrustlessCarEntityBytes',
-        'TestUnixFSDirectoryListingOnSubdomainGateway',
-        'TestGatewayCache',
-        'TestUnixFSDirectoryListing',
-        '.*/.*TODO:_cleanup_Kubo-specifics'
-      ]
-      const skip = ['-skip', toSkip.join('|')]
-
-      const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], skip), { reject: false, signal: AbortSignal.timeout(200000) })
+      const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], []), { reject: false, signal: AbortSignal.timeout(200000) })
 
       log(stdout)
       log.error(stderr)
 
       const { successRate } = await getReportDetails('gwc-report-all.json')
+      const knownSuccessRate = 39.19
+      // check latest success rate with `SUCCESS_RATE=100 npm run test -- -g 'total'`
+      const expectedSuccessRate = process.env.SUCCESS_RATE != null ? Number.parseFloat(process.env.SUCCESS_RATE) : knownSuccessRate
 
-      expect(successRate).to.be.greaterThanOrEqual(22.61)
+      expect(successRate).to.be.greaterThanOrEqual(expectedSuccessRate)
     })
   })
 })
diff --git a/packages/gateway-conformance/src/demo-server.ts b/packages/gateway-conformance/src/demo-server.ts
index d183bb0..8f4a4b4 100644
--- a/packages/gateway-conformance/src/demo-server.ts
+++ b/packages/gateway-conformance/src/demo-server.ts
@@ -6,32 +6,27 @@ import getPort from 'aegir/get-port'
 import { startBasicServer } from './fixtures/basic-server.js'
 import { createKuboNode } from './fixtures/create-kubo.js'
 import { loadKuboFixtures } from './fixtures/kubo-mgmt.js'
-import { startReverseProxy } from './fixtures/reverse-proxy.js'
 
 const log = logger('demo-server')
 
 const KUBO_GATEWAY_PORT = await getPort(3440)
 const SERVER_PORT = await getPort(3441)
-const PROXY_PORT = await getPort(3442)
 const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_GATEWAY_PORT)
 
 const kuboGateway = gatewayUrl
 await controller.start()
-const IPFS_NS_MAP = await loadKuboFixtures(repoPath, PROXY_PORT)
+const IPFS_NS_MAP = await loadKuboFixtures(repoPath)
 
-await startBasicServer({
+const stopServer = await startBasicServer({
   serverPort: SERVER_PORT,
   kuboGateway,
   IPFS_NS_MAP
 })
 
-await startReverseProxy({
-  backendPort: SERVER_PORT,
-  targetHost: 'localhost',
-  proxyPort: PROXY_PORT
-})
-
 process.on('exit', () => {
+  stopServer().catch((err) => {
+    log.error('Failed to stop server', err)
+  })
   controller.stop().catch((err) => {
     log.error('Failed to stop controller', err)
     process.exit(1)
diff --git a/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts b/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts
index 23901d4..8a88c6b 100644
--- a/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts
+++ b/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts
@@ -34,10 +34,10 @@ export const GWC_FIXTURES_PATH = posix.resolve(__dirname, 'gateway-conformance-f
 /**
  * use `createKuboNode' to start a kubo node prior to loading fixtures.
  */
-export async function loadKuboFixtures (kuboRepoDir: string, proxyPort: number): Promise<string> {
+export async function loadKuboFixtures (kuboRepoDir: string): Promise<string> {
   await downloadFixtures()
 
-  return loadFixtures(kuboRepoDir, proxyPort)
+  return loadFixtures(kuboRepoDir)
 }
 
 function getExecaOptions ({ cwd, ipfsNsMap, kuboRepoDir }: { cwd?: string, ipfsNsMap?: string, kuboRepoDir?: string } = {}): { cwd: string, env: Record<string, string | undefined> } {
@@ -72,7 +72,7 @@ async function downloadFixtures (force = false): Promise<void> {
   }
 }
 
-export async function loadFixtures (kuboRepoDir: string, proxyPort: number): Promise<string> {
+export async function loadFixtures (kuboRepoDir: string): Promise<string> {
   const execaOptions = getExecaOptions({ kuboRepoDir })
 
   const carPath = `${GWC_FIXTURES_PATH}/**/*.car`
diff --git a/packages/gateway-conformance/src/fixtures/reverse-proxy.ts b/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
deleted file mode 100644
index 00f61b9..0000000
--- a/packages/gateway-conformance/src/fixtures/reverse-proxy.ts
+++ /dev/null
@@ -1,130 +0,0 @@
-import { request, createServer, type RequestOptions, type IncomingMessage, type ServerResponse } from 'node:http'
-import { logger } from '@libp2p/logger'
-
-const log = logger('reverse-proxy')
-
-let TARGET_HOST: string
-let backendPort: number
-let proxyPort: number
-let subdomain: undefined | string
-let prefixPath: undefined | string
-let disableTryFiles: boolean
-let X_FORWARDED_HOST: undefined | string
-
-const makeRequest = (options: RequestOptions, req: IncomingMessage, res: ServerResponse & { req: IncomingMessage }, attemptRootFallback = false): void => {
-  options.headers = options.headers ?? {}
-  options.headers.Host = TARGET_HOST
-  const clientIp = req.socket.remoteAddress
-  options.headers['X-Forwarded-For'] = req.headers.host ?? clientIp
-
-  // override path to include prefixPath if set
-  if (prefixPath != null) {
-    options.path = `${prefixPath}${options.path}`
-  }
-  if (subdomain != null) {
-    options.headers.Host = `${subdomain}.${TARGET_HOST}`
-  }
-  if (X_FORWARDED_HOST != null) {
-    options.headers['X-Forwarded-Host'] = X_FORWARDED_HOST
-  }
-
-  // log where we're making the request to
-  log('Proxying request to %s:%s%s', options.headers.Host, options.port, options.path)
-
-  const proxyReq = request(options, (proxyRes) => {
-    if (!disableTryFiles && proxyRes.statusCode === 404) { // poor mans attempt to implement nginx style try_files
-      if (!attemptRootFallback) {
-        // Split the path and pop the last segment
-        const pathSegments = options.path?.split('/') ?? []
-        const lastSegment = pathSegments.pop() ?? ''
-
-        // Attempt to request the last segment at the root
-        makeRequest({ ...options, path: `/${lastSegment}` }, req, res, true)
-      } else {
-        // If already attempted a root fallback, serve index.html
-        makeRequest({ ...options, path: '/index.html' }, req, res)
-      }
-    } else {
-      // setCommonHeaders(res)
-      if (proxyRes.statusCode == null) {
-        log.error('No status code received from proxy')
-        res.writeHead(500)
-        res.end('Internal Server Error')
-        return
-      }
-      res.writeHead(proxyRes.statusCode, proxyRes.headers)
-      proxyRes.pipe(res, { end: true })
-    }
-  })
-
-  req.pipe(proxyReq, { end: true })
-
-  proxyReq.on('error', (e) => {
-    log.error(`Problem with request: ${e.message}`)
-    res.writeHead(500)
-    res.end(`Internal Server Error: ${e.message}`)
-  })
-
-  proxyReq.on('close', () => {
-    log.trace('Proxy request closed; ending response')
-    res.end()
-  })
-}
-
-export interface ReverseProxyOptions {
-  targetHost?: string
-  backendPort?: number
-  proxyPort?: number
-  subdomain?: string
-  prefixPath?: string
-  disableTryFiles?: boolean
-  xForwardedHost?: string
-}
-export async function startReverseProxy (options?: ReverseProxyOptions): Promise<() => Promise<void>> {
-  TARGET_HOST = options?.targetHost ?? process.env.TARGET_HOST ?? 'localhost'
-  backendPort = options?.backendPort ?? Number(process.env.BACKEND_PORT ?? 3000)
-  proxyPort = options?.proxyPort ?? Number(process.env.PROXY_PORT ?? 3333)
-  subdomain = options?.subdomain ?? process.env.SUBDOMAIN
-  prefixPath = options?.prefixPath ?? process.env.PREFIX_PATH
-  disableTryFiles = options?.disableTryFiles ?? process.env.DISABLE_TRY_FILES === 'true'
-  X_FORWARDED_HOST = options?.xForwardedHost ?? process.env.X_FORWARDED_HOST
-
-  const proxyServer = createServer((req, res) => {
-    if (req.method === 'OPTIONS') {
-      res.writeHead(200)
-      res.end()
-      return
-    }
-    log('req.headers: %O', req.headers)
-
-    const options: RequestOptions = {
-      hostname: TARGET_HOST,
-      port: backendPort,
-      path: req.url,
-      method: req.method,
-      headers: { ...req.headers }
-    }
-
-    makeRequest(options, req, res)
-  })
-
-  proxyServer.listen(proxyPort, () => {
-    log(`Proxy server listening on port ${proxyPort}`)
-  })
-
-  return async function stopReverseProxy (): Promise<void> {
-    log('Stopping...')
-    await new Promise<void>((resolve, reject) => {
-      // no matter what happens, we need to kill the server
-      proxyServer.closeAllConnections()
-      log('Closed all connections')
-      proxyServer.close((err: any) => {
-        if (err != null) {
-          reject(err)
-        } else {
-          resolve()
-        }
-      })
-    })
-  }
-}

From 39ceb6da0fb4768b4ff2463ec6e421d53bf651db Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Wed, 22 May 2024 10:04:26 -0700
Subject: [PATCH 14/25] fix: pass datastore and blockstore to helia instance

---
 .../gateway-conformance/src/conformance.spec.ts    | 14 ++++++--------
 .../src/fixtures/basic-server.ts                   |  2 ++
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index cf55d85..47e5abc 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -96,12 +96,12 @@ const tests: TestConfig[] = [
   {
     name: 'TestGatewayJSONCborAndIPNS',
     run: ['TestGatewayJSONCborAndIPNS'],
-    successRate: 24.24
+    successRate: 51.52
   },
   {
     name: 'TestGatewayIPNSPath',
     run: ['TestGatewayIPNSPath'],
-    successRate: 27.27
+    successRate: 100
   },
   {
     name: 'TestRedirectCanonicalIPNS',
@@ -127,7 +127,7 @@ const tests: TestConfig[] = [
   {
     name: 'TestGatewayIPNSRecord',
     run: ['TestGatewayIPNSRecord'],
-    successRate: 0
+    successRate: 17.39
   },
   {
     name: 'TestTrustlessCarOrderAndDuplicates',
@@ -170,7 +170,7 @@ const tests: TestConfig[] = [
   {
     name: 'TestGatewaySubdomainAndIPNS',
     run: ['TestGatewaySubdomainAndIPNS'],
-    successRate: 0
+    successRate: 31.58
   },
   {
     name: 'TestGatewaySubdomains',
@@ -233,7 +233,7 @@ const tests: TestConfig[] = [
   {
     name: 'TestGatewayCacheWithIPNS',
     run: ['TestGatewayCacheWithIPNS'],
-    successRate: 35.71
+    successRate: 66.67
   },
   // {
   //   // passes at the set successRate, but takes incredibly long (consistently ~2m).. disabling for now.
@@ -307,9 +307,7 @@ describe('@helia/verified-fetch - gateway conformance', function () {
   describe('smokeTests', () => {
     [
       ['basic server path request works', `http://localhost:${process.env.SERVER_PORT}/ipfs/bafkqabtimvwgy3yk`],
-      // ['proxy server path request works', `http://localhost:${process.env.PROXY_PORT}/ipfs/bafkqabtimvwgy3yk`],
       ['basic server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.SERVER_PORT}`]
-      // ['proxy server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.PROXY_PORT}`]
     ].forEach(([name, url]) => {
       it(name, async () => {
         const resp = await fetch(url)
@@ -386,7 +384,7 @@ describe('@helia/verified-fetch - gateway conformance', function () {
       log.error(stderr)
 
       const { successRate } = await getReportDetails('gwc-report-all.json')
-      const knownSuccessRate = 39.19
+      const knownSuccessRate = 42.47
       // check latest success rate with `SUCCESS_RATE=100 npm run test -- -g 'total'`
       const expectedSuccessRate = process.env.SUCCESS_RATE != null ? Number.parseFloat(process.env.SUCCESS_RATE) : knownSuccessRate
 
diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index b13d20b..5a190aa 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -47,6 +47,8 @@ interface CreateHeliaOptions {
  */
 async function createHelia (init: CreateHeliaOptions): Promise<ReturnType<typeof createHeliaHTTP>> {
   return createHeliaHTTP({
+    blockstore: init.blockstore,
+    datastore: init.datastore,
     blockBrokers: [
       trustlessGateway({
         allowInsecure: true,

From 93c1ca50331f84acbe00a24d973608987bbfb9ea Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Wed, 22 May 2024 10:18:04 -0700
Subject: [PATCH 15/25] chore: lint fix and header types

---
 packages/gateway-conformance/package.json      | 11 ++++++++++-
 .../src/fixtures/basic-server.ts               | 17 ++++++++++++-----
 .../src/fixtures/header-utils.ts               | 18 ++++++++++++++++++
 3 files changed, 40 insertions(+), 6 deletions(-)
 create mode 100644 packages/gateway-conformance/src/fixtures/header-utils.ts

diff --git a/packages/gateway-conformance/package.json b/packages/gateway-conformance/package.json
index dac0e49..5b03cba 100644
--- a/packages/gateway-conformance/package.json
+++ b/packages/gateway-conformance/package.json
@@ -53,18 +53,27 @@
   },
   "dependencies": {
     "@helia/block-brokers": "^3.0.0-f46700f",
+    "@helia/http": "^1.0.8",
     "@helia/interface": "^4.3.0-f46700f",
-    "@helia/utils": "^0.3.1",
+    "@helia/routers": "^1.1.0",
     "@helia/verified-fetch": "1.4.2",
+    "@libp2p/kad-dht": "^12.0.17",
     "@libp2p/logger": "^4.0.11",
+    "@libp2p/peer-id": "^4.1.2",
     "@multiformats/dns": "^1.0.6",
     "@sgtpooki/file-type": "^1.0.1",
     "aegir": "^42.2.5",
+    "blockstore-core": "^4.4.1",
+    "datastore-core": "^9.2.9",
     "execa": "^8.0.1",
     "fast-glob": "^3.3.2",
+    "interface-blockstore": "^5.2.10",
+    "interface-datastore": "^8.2.11",
     "ipfsd-ctl": "^14.1.0",
+    "ipns": "^9.1.0",
     "kubo": "^0.27.0",
     "kubo-rpc-client": "^4.1.1",
+    "uint8arrays": "^5.1.0",
     "undici": "^6.15.0"
   },
   "browser": {
diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index 5a190aa..2f0816b 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -9,6 +9,7 @@ import { Agent, setGlobalDispatcher } from 'undici'
 import { contentTypeParser } from './content-type-parser.js'
 import { createVerifiedFetch } from './create-verified-fetch.js'
 import { getLocalDnsResolver } from './get-local-dns-resolver.js'
+import { convertNodeJsHeadersToFetchHeaders } from './header-utils.js'
 import { getIpnsRecordDatastore } from './ipns-record-datastore.js'
 import type { DNSResolver } from '@multiformats/dns/resolvers'
 import type { Blockstore } from 'interface-blockstore'
@@ -68,7 +69,13 @@ async function createHelia (init: CreateHeliaOptions): Promise<ReturnType<typeof
   })
 }
 
-async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response, { serverPort, useSessions, verifiedFetch, kuboGateway, localDnsResolver }: any): Promise<void> {
+interface CallVerifiedFetchOptions {
+  serverPort: number
+  useSessions: boolean
+  verifiedFetch: Awaited<ReturnType<typeof createVerifiedFetch>>
+}
+
+async function callVerifiedFetch (req: IncomingMessage, res: Response, { serverPort, useSessions, verifiedFetch }: CallVerifiedFetchOptions): Promise<void> {
   const log = logger('basic-server:request')
   if (req.method === 'OPTIONS') {
     res.writeHead(200)
@@ -130,7 +137,7 @@ async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response,
 
   try {
     urlLog.trace('calling verified-fetch')
-    const resp = await verifiedFetch(fullUrlHref.toString(), { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true, headers: req.headers })
+    const resp = await verifiedFetch(fullUrlHref.toString(), { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true, headers: convertNodeJsHeadersToFetchHeaders(req.headers) })
     urlLog.trace('verified-fetch response status: %d', resp.status)
 
     // loop over headers and set them on the response
@@ -138,7 +145,7 @@ async function createAndCallVerifiedFetch (req: IncomingMessage, res: Response,
     for (const [key, value] of resp.headers.entries()) {
       if (fixingGwcAnnoyance) {
         urlLog.trace('need to fix GWC annoyance.')
-        if (value.includes(`localhost:${serverPort}`) === true) {
+        if (value.includes(`localhost:${serverPort}`)) {
           const newValue = value.replace(`localhost:${serverPort}`, 'localhost')
           urlLog.trace('fixing GWC annoyance. Replacing Header[%s] value of "%s" with "%s"', key, value, newValue)
           // we need to fix any Location, or other headers that have localhost without port in them.
@@ -215,8 +222,8 @@ export async function startBasicServer ({ kuboGateway, serverPort, IPFS_NS_MAP }
 
   const server = createServer((req, res) => {
     try {
-      void createAndCallVerifiedFetch(req, res, { serverPort, useSessions, kuboGateway, localDnsResolver, verifiedFetch }).catch((err) => {
-        log.error('Error in createAndCallVerifiedFetch', err)
+      void callVerifiedFetch(req, res, { serverPort, useSessions, verifiedFetch }).catch((err) => {
+        log.error('Error in callVerifiedFetch', err)
 
         if (!res.headersSent) {
           res.writeHead(500)
diff --git a/packages/gateway-conformance/src/fixtures/header-utils.ts b/packages/gateway-conformance/src/fixtures/header-utils.ts
new file mode 100644
index 0000000..58230c8
--- /dev/null
+++ b/packages/gateway-conformance/src/fixtures/header-utils.ts
@@ -0,0 +1,18 @@
+import type { IncomingHttpHeaders } from 'undici/types/header'
+
+export function convertNodeJsHeadersToFetchHeaders (headers: IncomingHttpHeaders): HeadersInit {
+  const fetchHeaders = new Headers()
+  for (const [key, value] of Object.entries(headers)) {
+    if (value == null) {
+      continue
+    }
+    if (Array.isArray(value)) {
+      for (const v of value) {
+        fetchHeaders.append(key, v)
+      }
+    } else {
+      fetchHeaders.append(key, value)
+    }
+  }
+  return fetchHeaders
+}

From 8307ec046a8c89427b4d593384c32b02512da81e Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Wed, 22 May 2024 10:23:49 -0700
Subject: [PATCH 16/25] deps: update deps

---
 package.json                                  |  2 +-
 packages/gateway-conformance/package.json     | 14 +++----
 .../src/conformance.spec.ts                   |  4 +-
 packages/interop/package.json                 |  6 +--
 packages/verified-fetch/package.json          | 42 +++++++++----------
 5 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/package.json b/package.json
index 471e0e3..f7d640e 100644
--- a/package.json
+++ b/package.json
@@ -122,7 +122,7 @@
     "docs:no-publish": "aegir docs --publish false"
   },
   "devDependencies": {
-    "aegir": "^42.2.5",
+    "aegir": "^42.2.11",
     "npm-run-all": "^4.1.5"
   },
   "type": "module",
diff --git a/packages/gateway-conformance/package.json b/packages/gateway-conformance/package.json
index 5b03cba..86cbf7f 100644
--- a/packages/gateway-conformance/package.json
+++ b/packages/gateway-conformance/package.json
@@ -52,29 +52,29 @@
     "test": "aegir test -t node"
   },
   "dependencies": {
-    "@helia/block-brokers": "^3.0.0-f46700f",
+    "@helia/block-brokers": "^3.0.1",
     "@helia/http": "^1.0.8",
-    "@helia/interface": "^4.3.0-f46700f",
+    "@helia/interface": "^4.3.0",
     "@helia/routers": "^1.1.0",
     "@helia/verified-fetch": "1.4.2",
     "@libp2p/kad-dht": "^12.0.17",
-    "@libp2p/logger": "^4.0.11",
+    "@libp2p/logger": "^4.0.13",
     "@libp2p/peer-id": "^4.1.2",
     "@multiformats/dns": "^1.0.6",
     "@sgtpooki/file-type": "^1.0.1",
-    "aegir": "^42.2.5",
+    "aegir": "^42.2.11",
     "blockstore-core": "^4.4.1",
     "datastore-core": "^9.2.9",
-    "execa": "^8.0.1",
+    "execa": "^9.1.0",
     "fast-glob": "^3.3.2",
     "interface-blockstore": "^5.2.10",
     "interface-datastore": "^8.2.11",
     "ipfsd-ctl": "^14.1.0",
     "ipns": "^9.1.0",
-    "kubo": "^0.27.0",
+    "kubo": "^0.28.0",
     "kubo-rpc-client": "^4.1.1",
     "uint8arrays": "^5.1.0",
-    "undici": "^6.15.0"
+    "undici": "^6.18.1"
   },
   "browser": {
     "./dist/src/fixtures/create-kubo.js": "./dist/src/fixtures/create-kubo.browser.js",
diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index 47e5abc..4ca6abf 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -359,7 +359,7 @@ describe('@helia/verified-fetch - gateway conformance', function () {
             ...((skip != null) ? ['-skip', `${skip.join('|')}`] : []),
             ...((run != null) ? ['-run', `${run.join('|')}`] : [])
           ]
-        ), { reject: false, signal: timeout != null ? AbortSignal.timeout(timeout) : undefined })
+        ), { reject: false, cancelSignal: timeout != null ? AbortSignal.timeout(timeout) : undefined })
 
         log(stdout)
         log.error(stderr)
@@ -378,7 +378,7 @@ describe('@helia/verified-fetch - gateway conformance', function () {
       this.timeout(200000)
       const log = logger.forComponent('all')
 
-      const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], []), { reject: false, signal: AbortSignal.timeout(200000) })
+      const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], []), { reject: false, cancelSignal: AbortSignal.timeout(200000) })
 
       log(stdout)
       log.error(stderr)
diff --git a/packages/interop/package.json b/packages/interop/package.json
index be241e3..112bec3 100644
--- a/packages/interop/package.json
+++ b/packages/interop/package.json
@@ -58,11 +58,11 @@
   },
   "dependencies": {
     "@helia/verified-fetch": "1.4.2",
-    "aegir": "^42.2.5",
-    "execa": "^8.0.1",
+    "aegir": "^42.2.11",
+    "execa": "^9.1.0",
     "fast-glob": "^3.3.2",
     "ipfsd-ctl": "^14.1.0",
-    "kubo": "^0.27.0",
+    "kubo": "^0.28.0",
     "kubo-rpc-client": "^4.1.1",
     "magic-bytes.js": "^1.10.0",
     "multiformats": "^13.1.0"
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
index 598c0d5..b67e225 100644
--- a/packages/verified-fetch/package.json
+++ b/packages/verified-fetch/package.json
@@ -57,32 +57,32 @@
     "release": "aegir release"
   },
   "dependencies": {
-    "@helia/block-brokers": "^3.0.0-f46700f",
+    "@helia/block-brokers": "^3.0.1",
     "@helia/car": "^3.1.5",
-    "@helia/http": "^1.0.7",
-    "@helia/interface": "^4.3.0-f46700f",
+    "@helia/http": "^1.0.8",
+    "@helia/interface": "^4.3.0",
     "@helia/ipns": "^7.2.2",
     "@helia/routers": "^1.1.0",
     "@ipld/dag-cbor": "^9.2.0",
     "@ipld/dag-json": "^10.2.0",
     "@ipld/dag-pb": "^4.1.0",
-    "@libp2p/interface": "^1.1.6",
-    "@libp2p/kad-dht": "^12.0.11",
-    "@libp2p/peer-id": "^4.0.9",
+    "@libp2p/interface": "^1.4.0",
+    "@libp2p/kad-dht": "^12.0.17",
+    "@libp2p/peer-id": "^4.1.2",
     "@multiformats/dns": "^1.0.6",
     "cborg": "^4.2.0",
     "hashlru": "^2.3.0",
     "interface-blockstore": "^5.2.10",
     "interface-datastore": "^8.2.11",
     "ipfs-unixfs-exporter": "^13.5.0",
-    "it-map": "^3.0.5",
+    "it-map": "^3.1.0",
     "it-pipe": "^3.0.1",
     "it-tar": "^6.0.5",
-    "it-to-browser-readablestream": "^2.0.6",
-    "lru-cache": "^10.2.0",
+    "it-to-browser-readablestream": "^2.0.9",
+    "lru-cache": "^10.2.2",
     "multiformats": "^13.1.0",
     "progress-events": "^1.0.0",
-    "uint8arrays": "^5.0.3"
+    "uint8arrays": "^5.1.0"
   },
   "devDependencies": {
     "@helia/dag-cbor": "^3.0.4",
@@ -91,25 +91,25 @@
     "@helia/unixfs": "^3.0.6",
     "@helia/utils": "^0.3.1",
     "@ipld/car": "^5.3.0",
-    "@libp2p/interface-compliance-tests": "^5.3.4",
-    "@libp2p/logger": "^4.0.9",
-    "@libp2p/peer-id-factory": "^4.0.9",
+    "@libp2p/interface-compliance-tests": "^5.4.5",
+    "@libp2p/logger": "^4.0.13",
+    "@libp2p/peer-id-factory": "^4.1.2",
     "@sgtpooki/file-type": "^1.0.1",
     "@types/sinon": "^17.0.3",
-    "aegir": "^42.2.5",
+    "aegir": "^42.2.11",
     "blockstore-core": "^4.4.1",
-    "browser-readablestream-to-it": "^2.0.5",
+    "browser-readablestream-to-it": "^2.0.7",
     "datastore-core": "^9.2.9",
-    "helia": "^4.2.1",
+    "helia": "^4.2.2",
     "ipfs-unixfs-importer": "^15.2.5",
     "ipns": "^9.1.0",
-    "it-all": "^3.0.4",
-    "it-drain": "^3.0.5",
-    "it-last": "^3.0.4",
-    "it-to-buffer": "^4.0.5",
+    "it-all": "^3.0.6",
+    "it-drain": "^3.0.7",
+    "it-last": "^3.0.6",
+    "it-to-buffer": "^4.0.7",
     "magic-bytes.js": "^1.10.0",
     "p-defer": "^4.0.1",
-    "sinon": "^17.0.1",
+    "sinon": "^18.0.0",
     "sinon-ts": "^2.0.0"
   },
   "sideEffects": false

From 03ac4420c0379989e050b1a202734c2060060343 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 23 May 2024 12:24:01 -0700
Subject: [PATCH 17/25] chore: modify log prefixes slightly

---
 .../src/conformance.spec.ts                   | 21 ++++++++++++-------
 .../src/fixtures/basic-server.ts              |  4 ++--
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index 4ca6abf..2ca2334 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -7,7 +7,7 @@ import { expect } from 'aegir/chai'
 import { execa } from 'execa'
 import { Agent, setGlobalDispatcher } from 'undici'
 
-const logger = prefixLogger('conformance-tests')
+const logger = prefixLogger('gateway-conformance')
 
 interface TestConfig {
   name: string
@@ -334,16 +334,21 @@ describe('@helia/verified-fetch - gateway conformance', function () {
 
     after(async () => {
       const log = logger.forComponent('after')
-      try {
-        await execa('rm', [binaryPath])
-        log('gateway-conformance binary successfully uninstalled.')
-      } catch (error) {
-        log.error(`Error removing "${binaryPath}"`, error)
+
+      if (process.env.GATEWAY_CONFORMANCE_BINARY == null) {
+        try {
+          await execa('rm', [binaryPath])
+          log('gateway-conformance binary successfully uninstalled.')
+        } catch (error) {
+          log.error(`Error removing "${binaryPath}"`, error)
+        }
+      } else {
+        log('Not removing custom gateway-conformance binary at %s', binaryPath)
       }
     })
 
     tests.forEach(({ name, spec, skip, run, timeout, successRate: minSuccessRate }) => {
-      const log = logger.forComponent(name)
+      const log = logger.forComponent(`output:${name}`)
       const expectedSuccessRate = process.env.SUCCESS_RATE != null ? Number.parseFloat(process.env.SUCCESS_RATE) : minSuccessRate
 
       it(`${name} has a success rate of at least ${expectedSuccessRate}%`, async function () {
@@ -376,7 +381,7 @@ describe('@helia/verified-fetch - gateway conformance', function () {
      */
     it('has expected total failures and successes', async function () {
       this.timeout(200000)
-      const log = logger.forComponent('all')
+      const log = logger.forComponent('output:all')
 
       const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], []), { reject: false, cancelSignal: AbortSignal.timeout(200000) })
 
diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index 2f0816b..fe1e21c 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -98,8 +98,8 @@ async function callVerifiedFetch (req: IncomingMessage, res: Response, { serverP
 
   // @see https://github.com/ipfs/gateway-conformance/issues/185#issuecomment-2123708150
   let fixingGwcAnnoyance = false
-  if (req.headers.host != null && (req.headers.host === 'localhost' || req.headers.Host === 'localhost')) {
-    log.trace('set fixingGwcAnnoyance to true for %s', req.url)
+  if (req.headers.host != null && req.headers.host === 'localhost') {
+    log.trace('set fixingGwcAnnoyance to true for %s', new URL(req.url, `http://${req.headers.host}`).href)
     fixingGwcAnnoyance = true
     req.headers.host = `localhost:${serverPort}`
   }

From 3abd9603dc08bd8cfce3cfbcf0d487aa57cae655 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 23 May 2024 12:46:29 -0700
Subject: [PATCH 18/25] chore: cleanup and re-enable tests not timing out

---
 .../src/conformance.spec.ts                   | 42 +++++++------------
 1 file changed, 16 insertions(+), 26 deletions(-)

diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index 2ca2334..30ca86f 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -86,13 +86,11 @@ const tests: TestConfig[] = [
     run: ['TestGatewayJsonCbor'],
     successRate: 22.22
   },
-  // currently results in an infinite loop without verified-fetch stopping the request whether sessions are enabled or not.
-  // {
-  //   name: 'TestNativeDag',
-  //   run: ['TestNativeDag'],
-  //   successRate: 100,
-  //   timeout: 120000
-  // },
+  {
+    name: 'TestNativeDag',
+    run: ['TestNativeDag'],
+    successRate: 60.71
+  },
   {
     name: 'TestGatewayJSONCborAndIPNS',
     run: ['TestGatewayJSONCborAndIPNS'],
@@ -134,8 +132,8 @@ const tests: TestConfig[] = [
     run: ['TestTrustlessCarOrderAndDuplicates'],
     successRate: 44.83
   },
-  // times out
   // {
+  //   // currently timing out
   //   name: 'TestTrustlessCarEntityBytes',
   //   run: ['TestTrustlessCarEntityBytes'],
   //   successRate: 100
@@ -146,11 +144,13 @@ const tests: TestConfig[] = [
     successRate: 54.55
   },
   // {
+  //   // currently timing out
   //   name: 'TestTrustlessCarDagScopeEntity',
   //   run: ['TestTrustlessCarDagScopeEntity'],
   //   successRate: 34.57
   // },
   // {
+  //   // currently timing out
   //   name: 'TestTrustlessCarDagScopeBlock',
   //   run: ['TestTrustlessCarDagScopeBlock'],
   //   successRate: 34.69
@@ -163,9 +163,10 @@ const tests: TestConfig[] = [
   //   timeout: 130000
   // },
   // {
+  //   // currently timing out
   //   name: 'TestSubdomainGatewayDNSLinkInlining',
   //   run: ['TestSubdomainGatewayDNSLinkInlining'],
-  //   successRate: 0
+  //   successRate: 100
   // },
   {
     name: 'TestGatewaySubdomainAndIPNS',
@@ -173,32 +174,21 @@ const tests: TestConfig[] = [
     successRate: 31.58
   },
   {
+    // TODO: add directory listing support to verified-fetch
     name: 'TestGatewaySubdomains',
     run: [
       'TestGatewaySubdomains'
-      // 100%
-      // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D%2F%7Bfilename_with_percent_encoding%7D_redirects_to_subdomain_%28direct_HTTP%29'
-      // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D%2F%7Bfilename_with_percent_encoding%7D_redirects_to_subdomain_%28direct_HTTP%29/Status_code'
-      // 'TestGatewaySubdomains/request_for_%7BCID%7D.ipfs.example.com%2Fipfs%2F%7BCID%7D_should_return_HTTP_404_%28direct_HTTP%29/Status_code'
-      // 'TestGatewaySubdomains/request_for_%7BCID%7D.ipfs.example.com_should_return_expected_payload_%28direct_HTTP%29/Status_code',
-      // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv0%7D_redirects_to_CIDv1_representation_in_subdomain_%28direct_HTTP%29/Header_Location',
-      // 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D_redirects_to_subdomain_%28direct_HTTP%29/Status_code'
     ],
     skip: [
       'TestGatewaySubdomains/.*HTTP_proxy_tunneling_via_CONNECT' // verified fetch should not be doing HTTP proxy tunneling.
-      // TODO: add directory listing support to verified-fetch
-      // 'TestGatewaySubdomains/.*directory_listing_at_%7Bcid%7D.ipfs.example.com%2Fsub%2Fdir_%28direct_HTTP%29',
-      // 'TestGatewaySubdomains/valid_file_and_subdirectory_paths_in_directory_listing_at_%7Bcid%7D.ipfs.example.com_%28direct_HTTP%29/Status_code',
-      // 'TestGatewaySubdomains/valid_file_and_subdirectory_paths_in_directory_listing_at_%7Bcid%7D.ipfs.example.com_%28direct_HTTP%29/Body'
     ],
     successRate: 41.35
   },
-  // times out
-  // {
-  //   name: 'TestUnixFSDirectoryListingOnSubdomainGateway',
-  //   run: ['TestUnixFSDirectoryListingOnSubdomainGateway'],
-  //   successRate: 100
-  // },
+  {
+    name: 'TestUnixFSDirectoryListingOnSubdomainGateway',
+    run: ['TestUnixFSDirectoryListingOnSubdomainGateway'],
+    successRate: 10.26
+  },
   {
     name: 'TestRedirectsFileWithIfNoneMatchHeader',
     run: ['TestRedirectsFileWithIfNoneMatchHeader'],

From d2dbb62b2ebf62e105c76de24fc0b7bc0cf3e541 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 23 May 2024 13:00:28 -0700
Subject: [PATCH 19/25] fix: pull out fetch->nodejs header logic

---
 .../src/fixtures/basic-server.ts              | 21 ++-----------
 .../src/fixtures/header-utils.ts              | 30 +++++++++++++++++++
 2 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index fe1e21c..7565766 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -9,7 +9,7 @@ import { Agent, setGlobalDispatcher } from 'undici'
 import { contentTypeParser } from './content-type-parser.js'
 import { createVerifiedFetch } from './create-verified-fetch.js'
 import { getLocalDnsResolver } from './get-local-dns-resolver.js'
-import { convertNodeJsHeadersToFetchHeaders } from './header-utils.js'
+import { convertFetchHeadersToNodeJsHeaders, convertNodeJsHeadersToFetchHeaders } from './header-utils.js'
 import { getIpnsRecordDatastore } from './ipns-record-datastore.js'
 import type { DNSResolver } from '@multiformats/dns/resolvers'
 import type { Blockstore } from 'interface-blockstore'
@@ -140,24 +140,7 @@ async function callVerifiedFetch (req: IncomingMessage, res: Response, { serverP
     const resp = await verifiedFetch(fullUrlHref.toString(), { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true, headers: convertNodeJsHeadersToFetchHeaders(req.headers) })
     urlLog.trace('verified-fetch response status: %d', resp.status)
 
-    // loop over headers and set them on the response
-    const headers: Record<string, string> = {}
-    for (const [key, value] of resp.headers.entries()) {
-      if (fixingGwcAnnoyance) {
-        urlLog.trace('need to fix GWC annoyance.')
-        if (value.includes(`localhost:${serverPort}`)) {
-          const newValue = value.replace(`localhost:${serverPort}`, 'localhost')
-          urlLog.trace('fixing GWC annoyance. Replacing Header[%s] value of "%s" with "%s"', key, value, newValue)
-          // we need to fix any Location, or other headers that have localhost without port in them.
-          headers[key] = newValue
-        } else {
-          urlLog.trace('NOT fixing GWC annoyance. Setting Header[%s] value of "%s"', key, value)
-          headers[key] = value
-        }
-      } else {
-        headers[key] = value
-      }
-    }
+    const headers = convertFetchHeadersToNodeJsHeaders({ resp, log: urlLog, fixingGwcAnnoyance, serverPort })
 
     res.writeHead(resp.status, headers)
     if (resp.body == null) {
diff --git a/packages/gateway-conformance/src/fixtures/header-utils.ts b/packages/gateway-conformance/src/fixtures/header-utils.ts
index 58230c8..23409c0 100644
--- a/packages/gateway-conformance/src/fixtures/header-utils.ts
+++ b/packages/gateway-conformance/src/fixtures/header-utils.ts
@@ -1,3 +1,4 @@
+import type { Logger } from '@libp2p/logger'
 import type { IncomingHttpHeaders } from 'undici/types/header'
 
 export function convertNodeJsHeadersToFetchHeaders (headers: IncomingHttpHeaders): HeadersInit {
@@ -16,3 +17,32 @@ export function convertNodeJsHeadersToFetchHeaders (headers: IncomingHttpHeaders
   }
   return fetchHeaders
 }
+
+export interface ConvertFetchHeadersToNodeJsHeadersOptions {
+  // headers: Headers
+  resp: Response
+  log: Logger
+  fixingGwcAnnoyance: boolean
+  serverPort: number
+}
+
+export function convertFetchHeadersToNodeJsHeaders ({ resp, log, fixingGwcAnnoyance, serverPort }: ConvertFetchHeadersToNodeJsHeadersOptions): IncomingHttpHeaders {
+  const headers: Record<string, string> = {}
+  for (const [key, value] of resp.headers.entries()) {
+    if (fixingGwcAnnoyance) {
+      log.trace('need to fix GWC annoyance.')
+      if (value.includes(`localhost:${serverPort}`)) {
+        const newValue = value.replace(`localhost:${serverPort}`, 'localhost')
+        log.trace('fixing GWC annoyance. Replacing Header[%s] value of "%s" with "%s"', key, value, newValue)
+        // we need to fix any Location, or other headers that have localhost without port in them.
+        headers[key] = newValue
+      } else {
+        log.trace('NOT fixing GWC annoyance. Setting Header[%s] value of "%s"', key, value)
+        headers[key] = value
+      }
+    } else {
+      headers[key] = value
+    }
+  }
+  return headers
+}

From a5c2e83f6d1e5d52ef9cffe89ae309a22a3052b3 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 23 May 2024 13:13:06 -0700
Subject: [PATCH 20/25] chore: apply suggestions from code review

---
 packages/gateway-conformance/.aegir.js        |  1 -
 .../src/conformance.spec.ts                   |  1 -
 .../src/fixtures/get-local-dns-resolver.ts    | 43 -------------------
 .../src/fixtures/header-utils.ts              |  1 -
 .../src/utils/handle-redirects.ts             | 13 ------
 packages/verified-fetch/src/verified-fetch.ts |  6 ---
 6 files changed, 65 deletions(-)

diff --git a/packages/gateway-conformance/.aegir.js b/packages/gateway-conformance/.aegir.js
index 49a3ca9..7f2c3f3 100644
--- a/packages/gateway-conformance/.aegir.js
+++ b/packages/gateway-conformance/.aegir.js
@@ -45,7 +45,6 @@ export default {
       }
     },
     after: async (options, beforeResult) => {
-      log('aegir test after hook')
       // @ts-expect-error - broken aegir types
       await beforeResult.controller.stop()
       log('controller stopped')
diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index 30ca86f..5b0bced 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -241,7 +241,6 @@ const tests: TestConfig[] = [
       'TestUnixFSDirectoryListing/.*TODO:_cleanup_Kubo-specifics'
     ],
     successRate: 50,
-    timeout: 1200000
   },
   {
     name: 'TestTar',
diff --git a/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts b/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts
index 70d444b..892052c 100644
--- a/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts
+++ b/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts
@@ -12,40 +12,6 @@ export function getLocalDnsResolver (ipfsNsMap: string, kuboGateway: string): DN
     nsMap.set(key, val)
   }
 
-  // async function getNameFromKubo (name: string): Promise<string> {
-  //   try {
-  //     log.trace('Fetching peer record for %s from Kubo', name)
-  //     const peerResponse = await fetch(`${kuboGateway}/api/v0/name/resolve?arg=${name}`, { method: 'POST' })
-  //     // invalid .json(), see https://github.com/ipfs/kubo/issues/10428
-  //     const text = (await peerResponse.text()).trim()
-  //     log('response from Kubo: %s', text)
-  //     const peerJson = JSON.parse(text)
-  //     return peerJson.Path
-  //   } catch (err: any) {
-  //     log.error('Problem fetching peer record from kubo: %s', err.message, err)
-  //     // process.exit(1)
-  //     throw err
-  //   }
-  // }
-
-  // /**
-  //  * @see https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-resolve
-  //  */
-  // async function getPeerRecordFromKubo (peerId: string): Promise<string> {
-  //   try {
-  //     log.trace('Fetching peer record for %s from Kubo', peerId)
-  //     const peerResponse = await fetch(`${kuboGateway}/api/v0/resolve/${peerId}`, { method: 'POST' })
-  //     // invalid .json(), see https://github.com/ipfs/kubo/issues/10428
-  //     const text = (await peerResponse.text()).trim()
-  //     log('response from Kubo: %s', text)
-  //     const peerJson = JSON.parse(text)
-  //     return peerJson.Path
-  //   } catch (err: any) {
-  //     log.error('Problem fetching peer record from kubo: %s', err.message, err)
-  //     // process.exit(1)
-  //     return getNameFromKubo(peerId)
-  //   }
-  // }
 
   return async (domain, options) => {
     const questions: Question[] = []
@@ -71,14 +37,8 @@ export function getLocalDnsResolver (ipfsNsMap: string, kuboGateway: string): DN
       log.trace('Querying "%s" for types %O', domain, options?.types)
       const actualDomainKey = domain.replace('_dnslink.', '')
       const nsValue = nsMap.get(actualDomainKey)
-      // try {
-      //   await getPeerRecordFromKubo(actualDomainKey)
-      // await getNameFromKubo(actualDomainKey)
       if (nsValue == null) {
         log.error('No IPFS_NS_MAP entry for domain "%s"', actualDomainKey)
-        // try to query kubo for the record
-        // temporarily disabled because it can cause an infinite loop
-        // await getPeerRecordFromKubo(actualDomainKey)
 
         throw new Error('No IPFS_NS_MAP entry for domain')
       }
@@ -89,9 +49,6 @@ export function getLocalDnsResolver (ipfsNsMap: string, kuboGateway: string): DN
         TTL: 180,
         data // should be in the format 'dnslink=/ipfs/bafkqac3imvwgy3zao5xxe3de'
       })
-      // } catch (err: any) {
-      //   log.error('Problem resolving record: %s', err.message, err)
-      // }
     }
 
     const dnsResponse = {
diff --git a/packages/gateway-conformance/src/fixtures/header-utils.ts b/packages/gateway-conformance/src/fixtures/header-utils.ts
index 23409c0..6b11ea4 100644
--- a/packages/gateway-conformance/src/fixtures/header-utils.ts
+++ b/packages/gateway-conformance/src/fixtures/header-utils.ts
@@ -19,7 +19,6 @@ export function convertNodeJsHeadersToFetchHeaders (headers: IncomingHttpHeaders
 }
 
 export interface ConvertFetchHeadersToNodeJsHeadersOptions {
-  // headers: Headers
   resp: Response
   log: Logger
   fixingGwcAnnoyance: boolean
diff --git a/packages/verified-fetch/src/utils/handle-redirects.ts b/packages/verified-fetch/src/utils/handle-redirects.ts
index 1cff2cb..b3dff41 100644
--- a/packages/verified-fetch/src/utils/handle-redirects.ts
+++ b/packages/verified-fetch/src/utils/handle-redirects.ts
@@ -40,12 +40,9 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
   // if x-forwarded-host is passed, we need to set the location header to the subdomain
   // so that the browser can redirect to the correct subdomain
   try {
-    // TODO: handle checking if subdomains are enabled and set location to subdomain host instead.
-    // if (headers.get('x-forwarded-host') != null) {
     const urlParts = matchURLString(resource)
     const reqUrl = new URL(resource)
     const actualHost = forwardedHost ?? reqUrl.host
-    // const subdomainUrl = new URL(reqUrl, `${reqUrl.protocol}//${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`)
     const subdomainUrl = new URL(reqUrl)
     if (urlParts.protocol === 'ipfs' && cid.version === 0) {
       subdomainUrl.host = `${cid.toV1()}.ipfs.${actualHost}`
@@ -58,29 +55,20 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
       return null
     }
 
-    log.trace('headers.get(\'host\')=%s', headerHost)
-    log.trace('headers.get(\'x-forwarded-host\')=%s', forwardedHost)
-    log.trace('headers.get(\'x-forwarded-for\')=%s', forwardedFor)
-
     if (headerHost != null && !subdomainUrl.host.includes(headerHost)) {
       log.trace('host header is not the same as the subdomain url host, not setting location header')
       return null
     }
     if (reqUrl.host === subdomainUrl.host) {
-      // log.trace('host header is the same as the request url host, not setting location header')
       log.trace('req url is the same as the subdomain url, not setting location header')
       return null
-    } else {
-      log.trace('req url is different from the subdomain url, attempting to set the location header')
     }
 
     subdomainUrl.pathname = maybeAddTraillingSlash(reqUrl.pathname.replace(`/${urlParts.cidOrPeerIdOrDnsLink}`, '').replace(`/${urlParts.protocol}`, ''))
-    // log.trace('subdomain url %s, given input: %s', subdomainUrl.href, `${reqUrl.protocol}//${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}`)
     log.trace('subdomain url %s', subdomainUrl.href)
     const pathUrl = new URL(reqUrl, `${reqUrl.protocol}//${actualHost}`)
     pathUrl.pathname = maybeAddTraillingSlash(reqUrl.pathname)
     log.trace('path url %s', pathUrl.href)
-    // const url = new URL(reqUrl, `${reqUrl.protocol}//${actualHost}`)
     // try to query subdomain with HEAD request to see if it's supported
     try {
       const subdomainTest = await fetch(subdomainUrl, { method: 'HEAD' })
@@ -95,7 +83,6 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
       log('subdomain not supported, redirecting to path', err)
       return movedPermanentlyResponse(resource.toString(), pathUrl.href)
     }
-    // }
   } catch (e) {
     // if it's not a full URL, we have nothing left to do.
     log.error('error setting location header for x-forwarded-host', e)
diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
index 9d37641..2ca9939 100644
--- a/packages/verified-fetch/src/verified-fetch.ts
+++ b/packages/verified-fetch/src/verified-fetch.ts
@@ -325,12 +325,6 @@ export class VerifiedFetch {
           this.log('could not redirect to %s/ as redirect option was set to "error"', resource)
           throw new TypeError('Failed to fetch')
         } else if (options?.redirect === 'manual') {
-          // const url = new URL(resource)
-          // const redirectPath = `${url.pathname}/`
-          // this.log('returning 301 permanent redirect to %s', redirectPath)
-
-          // return movedPermanentlyResponse(resource, url.pathname)
-
           this.log('returning 301 permanent redirect to %s/', resource)
           return movedPermanentlyResponse(resource, `${resource}/`)
         }

From d5c44df89587ef3eaaa3fa95a5d150bff4d2b945 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Thu, 23 May 2024 13:24:32 -0700
Subject: [PATCH 21/25] chore: lint fix after gh pr suggestions

---
 packages/gateway-conformance/src/conformance.spec.ts            | 2 +-
 .../gateway-conformance/src/fixtures/get-local-dns-resolver.ts  | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts
index 5b0bced..c59d0db 100644
--- a/packages/gateway-conformance/src/conformance.spec.ts
+++ b/packages/gateway-conformance/src/conformance.spec.ts
@@ -240,7 +240,7 @@ const tests: TestConfig[] = [
       'TestUnixFSDirectoryListingOnSubdomainGateway',
       'TestUnixFSDirectoryListing/.*TODO:_cleanup_Kubo-specifics'
     ],
-    successRate: 50,
+    successRate: 50
   },
   {
     name: 'TestTar',
diff --git a/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts b/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts
index 892052c..cebc5ff 100644
--- a/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts
+++ b/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts
@@ -12,7 +12,6 @@ export function getLocalDnsResolver (ipfsNsMap: string, kuboGateway: string): DN
     nsMap.set(key, val)
   }
 
-
   return async (domain, options) => {
     const questions: Question[] = []
     const answers: Answer[] = []

From e5d6ae4f87e03219a3114b9f283120f13bbf20a0 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Fri, 24 May 2024 10:40:21 -0700
Subject: [PATCH 22/25] chore: apply suggestions from code review

Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com>
---
 packages/gateway-conformance/.aegir.js                | 3 ++-
 packages/verified-fetch/src/utils/handle-redirects.ts | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/gateway-conformance/.aegir.js b/packages/gateway-conformance/.aegir.js
index 7f2c3f3..911e360 100644
--- a/packages/gateway-conformance/.aegir.js
+++ b/packages/gateway-conformance/.aegir.js
@@ -15,6 +15,7 @@ export default {
       const { createKuboNode } = await import('./dist/src/fixtures/create-kubo.js')
       const KUBO_PORT = await getPort(3440)
       const SERVER_PORT = await getPort(3441)
+      // The Kubo gateway will be passed to the VerifiedFetch config
       const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_PORT)
       await controller.start()
       const { loadKuboFixtures } = await import('./dist/src/fixtures/kubo-mgmt.js')
@@ -22,7 +23,7 @@ export default {
       const kuboGateway = gatewayUrl
 
       const { startBasicServer } = await import('./dist/src/fixtures/basic-server.js')
-      const stopBasicServer = await startBasicServer({
+      const stopBasicServer = await startVerifiedFetchGateway({
         serverPort: SERVER_PORT,
         kuboGateway,
         IPFS_NS_MAP
diff --git a/packages/verified-fetch/src/utils/handle-redirects.ts b/packages/verified-fetch/src/utils/handle-redirects.ts
index b3dff41..90fba6f 100644
--- a/packages/verified-fetch/src/utils/handle-redirects.ts
+++ b/packages/verified-fetch/src/utils/handle-redirects.ts
@@ -20,6 +20,7 @@ function maybeAddTraillingSlash (path: string): string {
   return path.endsWith('/') ? path : `${path}/`
 }
 
+// See https://specs.ipfs.tech/http-gateways/path-gateway/#location-response-header
 export async function getRedirectResponse ({ resource, options, logger, cid }: GetRedirectResponse): Promise<null | Response> {
   const log = logger.forComponent('helia:verified-fetch:get-redirect-response')
 

From 967a4e34b6580477151bd7a57e0c4771c9cba0d2 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Fri, 24 May 2024 10:48:21 -0700
Subject: [PATCH 23/25] chore: apply suggestions from code review

Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com>
---
 packages/gateway-conformance/src/fixtures/basic-server.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index 7565766..b7bee85 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -150,6 +150,7 @@ async function callVerifiedFetch (req: IncomingMessage, res: Response, { serverP
     } else {
       // read the body of the response and write it to the response from the server
       const reader = resp.body.getReader()
+      reader.pipeTo(res)
       while (true) {
         const { done, value } = await reader.read()
         if (done) {
@@ -177,7 +178,7 @@ async function callVerifiedFetch (req: IncomingMessage, res: Response, { serverP
   }
 }
 
-export async function startBasicServer ({ kuboGateway, serverPort, IPFS_NS_MAP }: BasicServerOptions): Promise<() => Promise<void>> {
+export async function startVerifiedFetchGateway ({ kuboGateway, serverPort, IPFS_NS_MAP }: BasicServerOptions): Promise<() => Promise<void>> {
   const staticDnsAgent = new Agent({
     connect: {
       lookup: (_hostname, _options, callback) => { callback(null, [{ address: '0.0.0.0', family: 4 }]) }

From 2be475c954b8f1c2d2c8f5da1173aecf46e81144 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Fri, 24 May 2024 11:33:29 -0700
Subject: [PATCH 24/25] test: add handle-redirects tests

---
 .../src/utils/handle-redirects.ts             | 13 ++-
 .../test/utils/handle-redirects.spec.ts       | 84 +++++++++++++++++++
 2 files changed, 95 insertions(+), 2 deletions(-)
 create mode 100644 packages/verified-fetch/test/utils/handle-redirects.spec.ts

diff --git a/packages/verified-fetch/src/utils/handle-redirects.ts b/packages/verified-fetch/src/utils/handle-redirects.ts
index 90fba6f..b123444 100644
--- a/packages/verified-fetch/src/utils/handle-redirects.ts
+++ b/packages/verified-fetch/src/utils/handle-redirects.ts
@@ -10,6 +10,10 @@ interface GetRedirectResponse {
   options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions
   logger: ComponentLogger
 
+  /**
+   * Only used in testing.
+   */
+  fetch?: typeof globalThis.fetch
 }
 
 function maybeAddTraillingSlash (path: string): string {
@@ -21,7 +25,7 @@ function maybeAddTraillingSlash (path: string): string {
 }
 
 // See https://specs.ipfs.tech/http-gateways/path-gateway/#location-response-header
-export async function getRedirectResponse ({ resource, options, logger, cid }: GetRedirectResponse): Promise<null | Response> {
+export async function getRedirectResponse ({ resource, options, logger, cid, fetch = globalThis.fetch }: GetRedirectResponse): Promise<null | Response> {
   const log = logger.forComponent('helia:verified-fetch:get-redirect-response')
 
   if (typeof resource !== 'string' || options == null || ['ipfs://', 'ipns://'].some((prefix) => resource.startsWith(prefix))) {
@@ -81,7 +85,12 @@ export async function getRedirectResponse ({ resource, options, logger, cid }: G
         throw new Error('subdomain not supported')
       }
     } catch (err: any) {
-      log('subdomain not supported, redirecting to path', err)
+      log('subdomain not supported', err)
+      if (pathUrl.href === reqUrl.href) {
+        log('path url is the same as the request url, not setting location header')
+        return null
+      }
+      // pathUrl is different from request URL (maybe even with just a trailing slash)
       return movedPermanentlyResponse(resource.toString(), pathUrl.href)
     }
   } catch (e) {
diff --git a/packages/verified-fetch/test/utils/handle-redirects.spec.ts b/packages/verified-fetch/test/utils/handle-redirects.spec.ts
new file mode 100644
index 0000000..173cf4c
--- /dev/null
+++ b/packages/verified-fetch/test/utils/handle-redirects.spec.ts
@@ -0,0 +1,84 @@
+import { prefixLogger } from '@libp2p/logger'
+import { expect } from 'aegir/chai'
+import { CID } from 'multiformats/cid'
+import Sinon from 'sinon'
+import { getRedirectResponse } from '../../src/utils/handle-redirects.js'
+
+const logger = prefixLogger('test:handle-redirects')
+describe('handle-redirects', () => {
+  describe('getRedirectResponse', () => {
+    const sandbox = Sinon.createSandbox()
+    const cid = CID.parse('bafkqabtimvwgy3yk')
+
+    let fetchStub: Sinon.SinonStub
+
+    beforeEach(() => {
+      fetchStub = sandbox.stub(globalThis, 'fetch')
+    })
+
+    afterEach(() => {
+      sandbox.restore()
+    })
+
+    const nullResponses = [
+      { resource: cid, options: {}, logger, cid, testTitle: 'should return null if resource is not a string' },
+      { resource: 'http://ipfs.io/ipfs/bafkqabtimvwgy3yk', options: undefined, logger, cid, testTitle: 'should return null if options is undefined' },
+      { resource: 'ipfs://', options: {}, logger, cid, testTitle: 'should return null for ipfs:// protocol urls' },
+      { resource: 'ipns://', options: {}, logger, cid, testTitle: 'should return null for ipns:// protocol urls' }
+    ]
+
+    nullResponses.forEach(({ resource, options, logger, cid, testTitle }) => {
+      it(testTitle, async () => {
+        const response = await getRedirectResponse({ resource, options, logger, cid })
+        expect(response).to.be.null()
+      })
+    })
+
+    it('should attempt to get the current host from the headers', async () => {
+      const resource = 'http://ipfs.io/ipfs/bafkqabtimvwgy3yk'
+      const options = { headers: new Headers({ 'x-forwarded-host': 'localhost:3931' }) }
+      fetchStub.returns(Promise.resolve(new Response(null, { status: 200 })))
+
+      const response = await getRedirectResponse({ resource, options, logger, cid, fetch: fetchStub })
+      expect(fetchStub.calledOnce).to.be.true()
+      expect(response).to.not.be.null()
+      expect(response).to.have.property('status', 301)
+      const location = response?.headers.get('location')
+      expect(location).to.equal('http://bafkqabtimvwgy3yk.ipfs.localhost:3931/')
+    })
+
+    it('should return redirect response to requested host with trailing slash when HEAD fetch fails', async () => {
+      const resource = 'http://ipfs.io/ipfs/bafkqabtimvwgy3yk'
+      const options = { headers: new Headers({ 'x-forwarded-host': 'localhost:3931' }) }
+      fetchStub.returns(Promise.reject(new Response(null, { status: 404 })))
+
+      const response = await getRedirectResponse({ resource, options, logger, cid, fetch: fetchStub })
+      expect(fetchStub.calledOnce).to.be.true()
+      expect(response).to.not.be.null()
+      expect(response).to.have.property('status', 301)
+      const location = response?.headers.get('location')
+      // note that the URL returned in location header has trailing slash.
+      expect(location).to.equal('http://ipfs.io/ipfs/bafkqabtimvwgy3yk/')
+    })
+
+    it('should not return redirect response to x-forwarded-host if HEAD fetch fails', async () => {
+      const resource = 'http://ipfs.io/ipfs/bafkqabtimvwgy3yk/file.txt'
+      const options = { headers: new Headers({ 'x-forwarded-host': 'localhost:3931' }) }
+      fetchStub.returns(Promise.reject(new Response(null, { status: 404 })))
+
+      const response = await getRedirectResponse({ resource, options, logger, cid, fetch: fetchStub })
+      expect(fetchStub.calledOnce).to.be.true()
+      expect(response).to.be.null()
+    })
+
+    it('should not return redirect response to x-forwarded-host when HEAD fetch fails and trailing slash already exists', async () => {
+      const resource = 'http://ipfs.io/ipfs/bafkqabtimvwgy3yk/'
+      const options = { headers: new Headers({ 'x-forwarded-host': 'localhost:3931' }) }
+      fetchStub.returns(Promise.reject(new Response(null, { status: 404 })))
+
+      const response = await getRedirectResponse({ resource, options, logger, cid, fetch: fetchStub })
+      expect(fetchStub.calledOnce).to.be.true()
+      expect(response).to.be.null()
+    })
+  })
+})

From f9bfa62e9de45334ac4cca19923599a5685b3c59 Mon Sep 17 00:00:00 2001
From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
Date: Fri, 24 May 2024 11:49:24 -0700
Subject: [PATCH 25/25] chore: fix build after applying pr suggestions

---
 packages/gateway-conformance/.aegir.js                    | 2 +-
 packages/gateway-conformance/src/demo-server.ts           | 4 ++--
 packages/gateway-conformance/src/fixtures/basic-server.ts | 1 -
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/packages/gateway-conformance/.aegir.js b/packages/gateway-conformance/.aegir.js
index 911e360..12b43e9 100644
--- a/packages/gateway-conformance/.aegir.js
+++ b/packages/gateway-conformance/.aegir.js
@@ -22,7 +22,7 @@ export default {
       const IPFS_NS_MAP = await loadKuboFixtures(repoPath)
       const kuboGateway = gatewayUrl
 
-      const { startBasicServer } = await import('./dist/src/fixtures/basic-server.js')
+      const { startVerifiedFetchGateway } = await import('./dist/src/fixtures/basic-server.js')
       const stopBasicServer = await startVerifiedFetchGateway({
         serverPort: SERVER_PORT,
         kuboGateway,
diff --git a/packages/gateway-conformance/src/demo-server.ts b/packages/gateway-conformance/src/demo-server.ts
index 8f4a4b4..9c5e44e 100644
--- a/packages/gateway-conformance/src/demo-server.ts
+++ b/packages/gateway-conformance/src/demo-server.ts
@@ -3,7 +3,7 @@
  */
 import { logger } from '@libp2p/logger'
 import getPort from 'aegir/get-port'
-import { startBasicServer } from './fixtures/basic-server.js'
+import { startVerifiedFetchGateway } from './fixtures/basic-server.js'
 import { createKuboNode } from './fixtures/create-kubo.js'
 import { loadKuboFixtures } from './fixtures/kubo-mgmt.js'
 
@@ -17,7 +17,7 @@ const kuboGateway = gatewayUrl
 await controller.start()
 const IPFS_NS_MAP = await loadKuboFixtures(repoPath)
 
-const stopServer = await startBasicServer({
+const stopServer = await startVerifiedFetchGateway({
   serverPort: SERVER_PORT,
   kuboGateway,
   IPFS_NS_MAP
diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts
index b7bee85..4f94958 100644
--- a/packages/gateway-conformance/src/fixtures/basic-server.ts
+++ b/packages/gateway-conformance/src/fixtures/basic-server.ts
@@ -150,7 +150,6 @@ async function callVerifiedFetch (req: IncomingMessage, res: Response, { serverP
     } else {
       // read the body of the response and write it to the response from the server
       const reader = resp.body.getReader()
-      reader.pipeTo(res)
       while (true) {
         const { done, value } = await reader.read()
         if (done) {