Colossus/route to ipfs gateway

pioneer/packages/joy-media/src/DiscoveryProvider.tsx

@@ -16,7 +16,7 @@ export type BootstrapNodes = {
 };
 
 export type DiscoveryProvider = {
-  resolveAssetEndpoint: (provider: StorageProviderId, contentId?: string, cancelToken?: CancelToken) => Promise<string>;
+  resolveAssetEndpoint: (provider: StorageProviderId, operation: 'download'| 'upload', contentId?: string, cancelToken?: CancelToken) => Promise<string>;
   reportUnreachable: (provider: StorageProviderId) => void;
 };
 
@@ -44,7 +44,12 @@ type ProviderStats = {
 function newDiscoveryProvider ({ bootstrapNodes }: BootstrapNodes): DiscoveryProvider {
   const stats = new Map<string, ProviderStats>();
 
-  const resolveAssetEndpoint = async (storageProvider: StorageProviderId, contentId?: string, cancelToken?: CancelToken) => {
+  const resolveAssetEndpoint = async (
+    storageProvider: StorageProviderId,
+    operation: 'download' | 'upload',
+    contentId?: string,
+    cancelToken?: CancelToken
+  ) => {
     const providerKey = storageProvider.toString();
 
     let stat = stats.get(providerKey);
@@ -113,7 +118,9 @@ function newDiscoveryProvider ({ bootstrapNodes }: BootstrapNodes): DiscoveryPro
     console.log(stat);
 
     if (stat) {
-      return `${stat.assetApiEndpoint}/asset/v0/${contentId || ''}`;
+      const ver = operation === 'download' ? 'v1' : 'v0';
+
+      return `${stat.assetApiEndpoint}/asset/${ver}/${contentId || ''}`;
     }
 
     throw new Error('Resolving failed.');

pioneer/packages/joy-media/src/Upload.tsx

@@ -350,7 +350,7 @@ class Upload extends React.PureComponent<Props, State> {
     let url: string;
 
     try {
-      url = await discoveryProvider.resolveAssetEndpoint(storageProvider, contentId, cancelSource.token);
+      url = await discoveryProvider.resolveAssetEndpoint(storageProvider, 'upload', contentId, cancelSource.token);
     } catch (err) {
       return this.setState({
         error: `Failed to contact storage provider: ${normalizeError(err)}`,

pioneer/packages/joy-media/src/common/MediaPlayerWithResolver.tsx

@@ -80,7 +80,7 @@ function InnerComponent (props: Props) {
       let assetUrl: string | undefined;
 
       try {
-        assetUrl = await discoveryProvider.resolveAssetEndpoint(provider, contentId.encode(), cancelSource.token);
+        assetUrl = await discoveryProvider.resolveAssetEndpoint(provider, 'download', contentId.encode(), cancelSource.token);
       } catch (err) {
         if (axios.isCancel(err)) {
           return;

storage-node/packages/cli/src/commands/base.ts

@@ -5,17 +5,17 @@ import { ContentId } from '@joystream/types/media'
 // Commands base abstract class. Contains reusable methods.
 export abstract class BaseCommand {
   // Creates the Colossus asset URL and logs it.
-  protected createAndLogAssetUrl(url: string, contentId: string | ContentId): string {
+  protected createAndLogAssetUrl(url: string, operation: 'download' | 'upload', contentId: string | ContentId): string {
     let normalizedContentId: string
 
     if (typeof contentId === 'string') {
       normalizedContentId = contentId
     } else {
       normalizedContentId = contentId.encode()
     }
-
+    const ver = operation === 'download' ? 'v1' : 'v0'
     const normalizedUrl = removeEndingForwardSlash(url)
-    const assetUrl = `${normalizedUrl}/asset/v0/${normalizedContentId}`
+    const assetUrl = `${normalizedUrl}/asset/${ver}/${normalizedContentId}`
     console.log(chalk.yellow('Generated asset URL:', assetUrl))
 
     return assetUrl

storage-node/packages/cli/src/commands/download.ts

@@ -46,7 +46,7 @@ export class DownloadCommand extends BaseCommand {
     // Checks for input parameters, shows usage if they are invalid.
     if (!this.assertParameters()) return
 
-    const assetUrl = this.createAndLogAssetUrl(this.storageNodeUrl, this.contentId)
+    const assetUrl = this.createAndLogAssetUrl(this.storageNodeUrl, 'download', this.contentId)
     console.log(chalk.yellow('File path:', this.outputFilePath))
 
     // Create file write stream and set error handler.

storage-node/packages/cli/src/commands/head.ts

@@ -36,7 +36,7 @@ export class HeadCommand extends BaseCommand {
     // Checks for input parameters, shows usage if they are invalid.
     if (!this.assertParameters()) return
 
-    const assetUrl = this.createAndLogAssetUrl(this.storageNodeUrl, this.contentId)
+    const assetUrl = this.createAndLogAssetUrl(this.storageNodeUrl, 'download', this.contentId)
 
     try {
       const response = await axios.head(assetUrl)

storage-node/packages/cli/src/commands/upload.ts

@@ -214,7 +214,7 @@ export class UploadCommand extends BaseCommand {
     const colossusEndpoint = await this.discoverStorageProviderEndpoint(dataObject.liaison.toString())
     debug(`Discovered storage node endpoint: ${colossusEndpoint}`)
 
-    const assetUrl = this.createAndLogAssetUrl(colossusEndpoint, addContentParams.contentId)
+    const assetUrl = this.createAndLogAssetUrl(colossusEndpoint, 'upload', addContentParams.contentId)
     await this.uploadFile(assetUrl)
   }
 }

storage-node/packages/colossus/lib/app.js

@@ -32,6 +32,7 @@ const yaml = require('js-yaml')
 // Project requires
 const validateResponses = require('./middleware/validate_responses')
 const fileUploads = require('./middleware/file_uploads')
+const ipfsGateway = require('./middleware/ipfs_proxy')
 const pagination = require('@joystream/storage-utils/pagination')
 
 // Configure app
@@ -62,6 +63,9 @@ function createApp(projectRoot, storage, runtime) {
     },
   })
 
+  // Proxy asset GET and HEAD directly to the IPFS gateway
+  app.use('/asset/v1/:id', ipfsGateway.createProxy(storage))
+
   // If no other handler gets triggered (errors), respond with the
   // error serialized to JSON.
   // Disable lint because we need such function signature.

storage-node/packages/colossus/lib/middleware/ipfs_proxy.js

@@ -0,0 +1,59 @@
+const { createProxyMiddleware } = require('http-proxy-middleware')
+const debug = require('debug')('joystream:ipfs-proxy')
+
+/* 
+For this proxying to work correctly, ensure IPFS HTTP Gateway is configured as a path gateway:
+This can be done manually with the following command:
+
+  $ ipfs config --json Gateway.PublicGateways '{"localhost": null }' 
+
+The implicit default config is below which is not what we want!
+
+  $ ipfs config --json Gateway.PublicGateways '{
+    "localhost": {
+        "Paths": ["/ipfs", "/ipns"],
+        "UseSubdomains": true
+      }
+    }'
+
+https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#gateway
+*/
+
+const pathFilter = function (path, req) {
+  return path.match('^/asset/v1/') && (req.method === 'GET' || req.method === 'HEAD')
+}
+
+const createPathRewriter = (resolve) => {
+  return async (_path, req) => {
+    const hash = await resolve(req.params.id)
+    return `/ipfs/${hash}`
+  }
+}
+
+const createResolver = (storage) => {
+  return async (id) => await storage.resolveContentIdWithTimeout(5000, id)
+}
+
+const createProxy = (storage) => {
+  const pathRewrite = createPathRewriter(createResolver(storage))
+
+  return createProxyMiddleware(pathFilter, {
+    // Default path to local IPFS HTTP GATEWAY
+    target: 'http://localhost:8080/',
+    pathRewrite,
+    // capture redirect when IPFS HTTP Gateway is configured with 'UseDomains':true
+    // and treat it as an error.
+    onProxyRes: function (proxRes, _req, res) {
+      if (proxRes.statusCode === 301) {
+        debug('IPFS HTTP Gateway is allowing UseSubdomains. Killing stream')
+        proxRes.destroy()
+        // TODO: Maybe redirect - temporary to /v0/asset/contentId ?
+        res.status(500).end()
+      }
+    },
+  })
+}
+
+module.exports = {
+  createProxy,
+}

storage-node/packages/colossus/package.json

@@ -50,14 +50,15 @@
     "temp": "^0.9.0"
   },
   "dependencies": {
-    "@joystream/storage-runtime-api": "^0.1.0",
     "@joystream/storage-node-backend": "^0.1.0",
+    "@joystream/storage-runtime-api": "^0.1.0",
     "@joystream/storage-utils": "^0.1.0",
     "body-parser": "^1.19.0",
     "chalk": "^2.4.2",
     "cors": "^2.8.5",
     "express-openapi": "^4.6.1",
     "figlet": "^1.2.1",
+    "http-proxy-middleware": "^1.0.5",
     "js-yaml": "^3.13.1",
     "lodash": "^4.17.11",
     "meow": "^7.0.1",

storage-node/packages/helios/bin/cli.js

@@ -26,7 +26,7 @@ function mapInfoToStatus(providers, currentHeight) {
 
 function makeAssetUrl(contentId, source) {
   source = stripEndingSlash(source)
-  return `${source}/asset/v0/${encodeAddress(contentId)}`
+  return `${source}/asset/v1/${encodeAddress(contentId)}`
 }
 
 async function assetRelationshipState(api, contentId, providers) {

storage-node/packages/storage/storage.js

@@ -224,6 +224,10 @@ class Storage {
         debug(`Warning IPFS daemon not running: ${err.message}`)
       } else {
         debug(`IPFS node is up with identity: ${identity.id}`)
+        // TODO: wait for IPFS daemon to be online for this to be effective..?
+        // set the IPFS HTTP Gateway config we desire.. operator might need
+        // to restart their daemon if the config was changed.
+        this.ipfs.config.set('Gateway.PublicGateways', { 'localhost': null })
       }
     })
   }

storage-node/scripts/compose/devchain-and-ipfs-node/docker-compose.yaml

@@ -4,8 +4,16 @@ services:
     image: ipfs/go-ipfs:latest
     ports:
       - '127.0.0.1:5001:5001'
+      - '127.0.0.1:8080:8080'
     volumes:
       - ipfs-data:/data/ipfs
+    entrypoint: ''
+    command: |
+      /bin/sh -c "
+        set -e
+        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
+        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
+      "
   chain:
     image: joystream/node:latest
     ports:

yarn.lock

@@ -13588,7 +13588,7 @@ [email protected]:
     lodash "^4.17.11"
     micromatch "^3.1.10"
 
-http-proxy-middleware@^1.0.3:
+http-proxy-middleware@^1.0.3, http-proxy-middleware@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-1.0.5.tgz#4c6e25d95a411e3d750bc79ccf66290675176dc2"
   integrity sha512-CKzML7u4RdGob8wuKI//H8Ein6wNTEQR7yjVEzPbhBLGdOfkfvgTnp2HLnniKBDP9QW4eG10/724iTWLBeER3g==