diff --git a/pioneer/packages/joy-media/src/DiscoveryProvider.tsx b/pioneer/packages/joy-media/src/DiscoveryProvider.tsx index 0b4c165e21..f1378fb2b7 100644 --- a/pioneer/packages/joy-media/src/DiscoveryProvider.tsx +++ b/pioneer/packages/joy-media/src/DiscoveryProvider.tsx @@ -44,7 +44,11 @@ type ProviderStats = { function newDiscoveryProvider ({ bootstrapNodes }: BootstrapNodes): DiscoveryProvider { const stats = new Map(); - const resolveAssetEndpoint = async (storageProvider: StorageProviderId, contentId?: string, cancelToken?: CancelToken) => { + const resolveAssetEndpoint = async ( + storageProvider: StorageProviderId, + contentId?: string, + cancelToken?: CancelToken + ) => { const providerKey = storageProvider.toString(); let stat = stats.get(providerKey); diff --git a/storage-node/packages/colossus/lib/middleware/ipfs_proxy.js b/storage-node/packages/colossus/lib/middleware/ipfs_proxy.js new file mode 100644 index 0000000000..c1d0ee40d6 --- /dev/null +++ b/storage-node/packages/colossus/lib/middleware/ipfs_proxy.js @@ -0,0 +1,85 @@ +const { createProxyMiddleware } = require('http-proxy-middleware') +const debug = require('debug')('joystream:ipfs-proxy') +const mime = require('mime-types') + +/* +For this proxying to work correctly, ensure IPFS HTTP Gateway is configured as a path gateway: +This can be done manually with the following command: + + $ ipfs config --json Gateway.PublicGateways '{"localhost": null }' + +The implicit default config is below which is not what we want! + + $ ipfs config --json Gateway.PublicGateways '{ + "localhost": { + "Paths": ["/ipfs", "/ipns"], + "UseSubdomains": true + } + }' + +https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#gateway +*/ + +const pathFilter = function (path, req) { + // we get the full path here so it needs to match the path where + // it is used by the openapi initializer + return path.match('^/asset/v0') && (req.method === 'GET' || req.method === 'HEAD') +} + +const createPathRewriter = (resolve) => { + return async (_path, req) => { + // we expect the handler was used in openapi/express path with an id in the path: + // "/asset/v0/:id" + // TODO: catch and deal with hash == undefined if content id not found + const contentId = req.params.id + const hash = await resolve(contentId) + return `/ipfs/${hash}` + } +} + +const createResolver = (storage) => { + return async (id) => await storage.resolveContentIdWithTimeout(5000, id) +} + +const createProxy = (storage) => { + const pathRewrite = createPathRewriter(createResolver(storage)) + + return createProxyMiddleware(pathFilter, { + // Default path to local IPFS HTTP GATEWAY + target: 'http://localhost:8080/', + pathRewrite, + onProxyRes: function (proxRes, req, res) { + /* + Make sure the reverse proxy used infront of colosss (nginx/caddy) Does not duplicate + these headers to prevent some browsers getting confused especially + with duplicate access-control-allow-origin headers! + 'accept-ranges': 'bytes', + 'access-control-allow-headers': 'Content-Type, Range, User-Agent, X-Requested-With', + 'access-control-allow-methods': 'GET', + 'access-control-allow-origin': '*', + 'access-control-expose-headers': 'Content-Range, X-Chunked-Output, X-Stream-Output', + */ + + if (proxRes.statusCode === 301) { + // capture redirect when IPFS HTTP Gateway is configured with 'UseDomains':true + // and treat it as an error. + console.error('IPFS HTTP Gateway is configured for "UseSubdomains". Killing stream') + res.status(500).end() + proxRes.destroy() + } else { + // Handle downloading as attachment /asset/v0/:id?download + if (req.query.download) { + const contentId = req.params.id + const contentType = proxRes.headers['content-type'] + const ext = mime.extension(contentType) || 'bin' + const fileName = `${contentId}.${ext}` + proxRes.headers['Content-Disposition'] = `attachment; filename=${fileName}` + } + } + }, + }) +} + +module.exports = { + createProxy, +} diff --git a/storage-node/packages/colossus/package.json b/storage-node/packages/colossus/package.json index 7ee0a0029d..035f694db4 100644 --- a/storage-node/packages/colossus/package.json +++ b/storage-node/packages/colossus/package.json @@ -50,17 +50,19 @@ "temp": "^0.9.0" }, "dependencies": { - "@joystream/storage-runtime-api": "^0.1.0", "@joystream/storage-node-backend": "^0.1.0", + "@joystream/storage-runtime-api": "^0.1.0", "@joystream/storage-utils": "^0.1.0", "body-parser": "^1.19.0", "chalk": "^2.4.2", "cors": "^2.8.5", "express-openapi": "^4.6.1", "figlet": "^1.2.1", + "http-proxy-middleware": "^1.0.5", "js-yaml": "^3.13.1", "lodash": "^4.17.11", "meow": "^7.0.1", + "mime-types": "^2.1.27", "multer": "^1.4.1", "si-prefix": "^0.2.0" } diff --git a/storage-node/packages/colossus/paths/asset/v0/{id}.js b/storage-node/packages/colossus/paths/asset/v0/{id}.js index 993070870f..fc50100e17 100644 --- a/storage-node/packages/colossus/paths/asset/v0/{id}.js +++ b/storage-node/packages/colossus/paths/asset/v0/{id}.js @@ -18,12 +18,9 @@ 'use strict' -const path = require('path') - const debug = require('debug')('joystream:colossus:api:asset') - -const utilRanges = require('@joystream/storage-utils/ranges') const filter = require('@joystream/storage-node-backend/filter') +const ipfsProxy = require('../../../lib/middleware/ipfs_proxy') function errorHandler(response, err, code) { debug(err) @@ -31,6 +28,9 @@ function errorHandler(response, err, code) { } module.exports = function (storage, runtime) { + // Creat the IPFS HTTP Gateway proxy middleware + const proxy = ipfsProxy.createProxy(storage) + const doc = { // parameters for all operations in this path parameters: [ @@ -45,34 +45,6 @@ module.exports = function (storage, runtime) { }, ], - // Head: report that ranges are OK - async head(req, res) { - const id = req.params.id - - // Open file - try { - const size = await storage.size(id) - const stream = await storage.open(id, 'r') - const type = stream.fileInfo.mimeType - - // Close the stream; we don't need to fetch the file (if we haven't - // already). Then return result. - stream.destroy() - - res.status(200) - res.contentType(type) - res.header('Content-Disposition', 'inline') - res.header('Content-Transfer-Encoding', 'binary') - res.header('Accept-Ranges', 'bytes') - if (size > 0) { - res.header('Content-Length', size) - } - res.send() - } catch (err) { - errorHandler(res, err, err.code) - } - }, - // Put for uploads async put(req, res) { const id = req.params.id // content id @@ -184,61 +156,21 @@ module.exports = function (storage, runtime) { } }, - // Get content async get(req, res) { - const id = req.params.id - const download = req.query.download - - // Parse range header - let ranges - if (!download) { - try { - const rangeHeader = req.headers.range - ranges = utilRanges.parse(rangeHeader) - } catch (err) { - // Do nothing; it's ok to ignore malformed ranges and respond with the - // full content according to https://www.rfc-editor.org/rfc/rfc7233.txt - } - if (ranges && ranges.unit !== 'bytes') { - // Ignore ranges that are not byte units. - ranges = undefined - } - } - debug('Requested range(s) is/are', ranges) - - // Open file - try { - const size = await storage.size(id) - const stream = await storage.open(id, 'r') - - // Add a file extension to download requests if necessary. If the file - // already contains an extension, don't add one. - let sendName = id - const type = stream.fileInfo.mimeType - if (download) { - let ext = path.extname(sendName) - if (!ext) { - ext = stream.fileInfo.ext - if (ext) { - sendName = `${sendName}.${ext}` - } - } - } + proxy(req, res) + }, - const opts = { - name: sendName, - type, - size, - ranges, - download, - } - utilRanges.send(res, stream, opts) - } catch (err) { - errorHandler(res, err, err.code) - } + async head(req, res) { + proxy(req, res) }, } + // doc.get = proxy + // doc.head = proxy + // Note: Adding the middleware this way is causing problems! + // We are loosing some information from the request, specifically req.query.download parameters for some reason. + // Does it have to do with how/when the apiDoc is being processed? binding issue? + // OpenAPI specs doc.get.apiDoc = { description: 'Download an asset.', diff --git a/storage-node/packages/colossus/paths/discover/v0/{id}.js b/storage-node/packages/colossus/paths/discover/v0/{id}.js index 6467a0f8b6..a24c1a91df 100644 --- a/storage-node/packages/colossus/paths/discover/v0/{id}.js +++ b/storage-node/packages/colossus/paths/discover/v0/{id}.js @@ -12,7 +12,7 @@ module.exports = function (runtime) { name: 'id', in: 'path', required: true, - description: 'Actor accouuntId', + description: 'Storage Provider Id', schema: { type: 'string', // integer ? }, diff --git a/storage-node/packages/helios/bin/cli.js b/storage-node/packages/helios/bin/cli.js index ee133bdef8..9e84c4db2b 100755 --- a/storage-node/packages/helios/bin/cli.js +++ b/storage-node/packages/helios/bin/cli.js @@ -26,7 +26,7 @@ function mapInfoToStatus(providers, currentHeight) { function makeAssetUrl(contentId, source) { source = stripEndingSlash(source) - return `${source}/asset/v0/${encodeAddress(contentId)}` + return `${source}/asset/v1/${encodeAddress(contentId)}` } async function assetRelationshipState(api, contentId, providers) { diff --git a/storage-node/packages/storage/storage.js b/storage-node/packages/storage/storage.js index f3e3f0764d..ae92582375 100644 --- a/storage-node/packages/storage/storage.js +++ b/storage-node/packages/storage/storage.js @@ -224,6 +224,10 @@ class Storage { debug(`Warning IPFS daemon not running: ${err.message}`) } else { debug(`IPFS node is up with identity: ${identity.id}`) + // TODO: wait for IPFS daemon to be online for this to be effective..? + // set the IPFS HTTP Gateway config we desire.. operator might need + // to restart their daemon if the config was changed. + this.ipfs.config.set('Gateway.PublicGateways', { 'localhost': null }) } }) } diff --git a/storage-node/scripts/compose/devchain-and-ipfs-node/docker-compose.yaml b/storage-node/scripts/compose/devchain-and-ipfs-node/docker-compose.yaml index 73f6a7ecf8..4130757e1e 100644 --- a/storage-node/scripts/compose/devchain-and-ipfs-node/docker-compose.yaml +++ b/storage-node/scripts/compose/devchain-and-ipfs-node/docker-compose.yaml @@ -4,8 +4,16 @@ services: image: ipfs/go-ipfs:latest ports: - '127.0.0.1:5001:5001' + - '127.0.0.1:8080:8080' volumes: - ipfs-data:/data/ipfs + entrypoint: '' + command: | + /bin/sh -c " + set -e + /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }' + /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true + " chain: image: joystream/node:latest ports: diff --git a/yarn.lock b/yarn.lock index f308cf2ee8..5cc15dd2a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13595,7 +13595,7 @@ http-proxy-middleware@0.19.1: lodash "^4.17.11" micromatch "^3.1.10" -http-proxy-middleware@^1.0.3: +http-proxy-middleware@^1.0.3, http-proxy-middleware@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-1.0.5.tgz#4c6e25d95a411e3d750bc79ccf66290675176dc2" integrity sha512-CKzML7u4RdGob8wuKI//H8Ein6wNTEQR7yjVEzPbhBLGdOfkfvgTnp2HLnniKBDP9QW4eG10/724iTWLBeER3g== @@ -17753,7 +17753,7 @@ mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24: dependencies: mime-db "1.42.0" -mime-types@^2.1.18, mime-types@^2.1.22, mime-types@^2.1.26, mime-types@~2.1.17: +mime-types@^2.1.18, mime-types@^2.1.22, mime-types@^2.1.26, mime-types@^2.1.27, mime-types@~2.1.17: version "2.1.27" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==