From 399c9869f8de364c150e2f2b42b9fae77496ba1f Mon Sep 17 00:00:00 2001 From: Pierre Lehnen Date: Mon, 21 Dec 2020 13:22:18 -0300 Subject: [PATCH 1/2] Storj Integration --- app/file-upload/server/config/Storj.js | 78 +++++++++ .../server/config/_configUploadStorage.js | 1 + app/file-upload/server/lib/FileUpload.js | 1 + app/file-upload/server/startup/settings.js | 38 +++++ app/file-upload/ufs/Storj/server.js | 153 ++++++++++++++++++ packages/rocketchat-i18n/i18n/en.i18n.json | 3 + 6 files changed, 274 insertions(+) create mode 100644 app/file-upload/server/config/Storj.js create mode 100644 app/file-upload/ufs/Storj/server.js diff --git a/app/file-upload/server/config/Storj.js b/app/file-upload/server/config/Storj.js new file mode 100644 index 0000000000000..c9a4d20eaf8e5 --- /dev/null +++ b/app/file-upload/server/config/Storj.js @@ -0,0 +1,78 @@ +import http from 'http'; +import https from 'https'; + +import { Meteor } from 'meteor/meteor'; +import _ from 'underscore'; + +import { settings } from '../../../settings'; +import { FileUploadClass, FileUpload } from '../lib/FileUpload'; +import '../../ufs/Storj/server.js'; + +const get = function(file, req, res) { + const forceDownload = typeof req.query.download !== 'undefined'; + + this.store.getRedirectURL(file, forceDownload, (err, fileUrl) => { + if (err) { + return console.error(err); + } + + if (!fileUrl) { + return res.end(); + } + + return FileUpload.redirectToFile(fileUrl, req, res); + }); +}; + +const copy = function(file, out) { + const fileUrl = this.store.getRedirectURL(file); + + if (fileUrl) { + const request = /^https:/.test(fileUrl) ? https : http; + request.get(fileUrl, (fileRes) => fileRes.pipe(out)); + } else { + out.end(); + } +}; + +// This is the class that is retrieved when using FileUpload.getStoreByName / FileUpload.getStore +const StorjUploads = new FileUploadClass({ + name: 'Storj:Uploads', + get, + copy, + // store setted bellow +}); + +const StorjAvatars = new FileUploadClass({ + name: 'Storj:Avatars', + get, + copy, + // store setted bellow +}); + +const StorjUserDataFiles = new FileUploadClass({ + name: 'Storj:UserDataFiles', + get, + copy, + // store setted bellow +}); + +const configure = Meteor.bindEnvironment(_.debounce(function() { + const bucketName = settings.get('FileUpload_Storj_Bucket'); + const accessKey = settings.get('FileUpload_Storj_AccessKey'); + + if (!bucketName || !accessKey) { + return; + } + + const config = { + accessKey, + bucketName, + }; + + StorjUploads.store = FileUpload.configureUploadsStore('Storj', StorjUploads.name, config); + StorjAvatars.store = FileUpload.configureUploadsStore('Storj', StorjAvatars.name, config); + StorjUserDataFiles.store = FileUpload.configureUploadsStore('Storj', StorjUserDataFiles.name, config); +}, 500)); + +settings.get(/^FileUpload_Storj_/, configure); diff --git a/app/file-upload/server/config/_configUploadStorage.js b/app/file-upload/server/config/_configUploadStorage.js index 3fb54925db717..8da6c56a83db3 100644 --- a/app/file-upload/server/config/_configUploadStorage.js +++ b/app/file-upload/server/config/_configUploadStorage.js @@ -7,6 +7,7 @@ import './FileSystem.js'; import './GoogleStorage.js'; import './GridFS.js'; import './Webdav.js'; +import './Storj.js'; const configStore = _.debounce(() => { const store = settings.get('FileUpload_Storage_Type'); diff --git a/app/file-upload/server/lib/FileUpload.js b/app/file-upload/server/lib/FileUpload.js index 73c5ba261ac3f..5e06ae017c03e 100644 --- a/app/file-upload/server/lib/FileUpload.js +++ b/app/file-upload/server/lib/FileUpload.js @@ -50,6 +50,7 @@ export const FileUpload = { const stores = UploadFS.getStores(); delete stores[name]; + // the class that is instantiated here is the one from ../../ufs/*/server return new UploadFS.store[store](Object.assign({ name, }, options, FileUpload[`default${ type }`]())); diff --git a/app/file-upload/server/startup/settings.js b/app/file-upload/server/startup/settings.js index 4b228108e86b7..93527868f93d1 100644 --- a/app/file-upload/server/startup/settings.js +++ b/app/file-upload/server/startup/settings.js @@ -71,6 +71,9 @@ settings.addGroup('FileUpload', function() { }, { key: 'FileSystem', i18nLabel: 'FileSystem', + }, { + key: 'Storj', + i18nLabel: 'Storj', }], public: true, }); @@ -212,6 +215,41 @@ settings.addGroup('FileUpload', function() { }); }); + this.section('Storj Storage', function() { + this.add('FileUpload_Storj_Bucket', '', { + type: 'string', + private: true, + enableQuery: { + _id: 'FileUpload_Storage_Type', + value: 'Storj', + }, + secret: true, + }); + this.add('FileUpload_Storj_AccessKey', '', { + type: 'string', + private: true, + enableQuery: { + _id: 'FileUpload_Storage_Type', + value: 'Storj', + }, + secret: true, + }); + // this.add('FileUpload_Storj_Proxy_Avatars', false, { + // type: 'boolean', + // enableQuery: { + // _id: 'FileUpload_Storage_Type', + // value: 'Storj', + // }, + // }); + // this.add('FileUpload_Storj_Proxy_Uploads', false, { + // type: 'boolean', + // enableQuery: { + // _id: 'FileUpload_Storage_Type', + // value: 'Storj', + // }, + // }); + }); + this.section('File System', function() { this.add('FileUpload_FileSystemPath', '', { type: 'string', diff --git a/app/file-upload/ufs/Storj/server.js b/app/file-upload/ufs/Storj/server.js new file mode 100644 index 0000000000000..e3d6e075c6dd1 --- /dev/null +++ b/app/file-upload/ufs/Storj/server.js @@ -0,0 +1,153 @@ +import { Meteor } from 'meteor/meteor'; +import { check } from 'meteor/check'; +import { UploadFS } from 'meteor/jalik:ufs'; +import { Random } from 'meteor/random'; +// import uplink from 'uplink-js'; + +/** + * Storj store + * @param options + * @constructor + */ + +export class StorjStore extends UploadFS.Store { + constructor(options) { + // Default options + // options.accessKey, + // options.bucketName, + super(options); + + options.getPath = options.getPath || function(file) { + return file._id; + }; + + this.bucketName = options.bucketName; + // uplink.parseAccess(options.accessKey).then((access) => access.openProject()).then((project) => { + // this.project = project; + // }); + + this.getPath = function(file) { + if (file.Storj) { + return file.Storj.path; + } + }; + + this.getRedirectURL = function(file, forceDownload = false, callback) { + // const download = this.project.downloadObject(this.bucketName, encodeURI(file.name)); + + + // const params = { + // action: 'read', + // responseDisposition: forceDownload ? 'attachment' : 'inline', + // expires: Date.now() + this.options.URLExpiryTimeSpan * 1000, + // }; + + // this.bucket.file(this.getPath(file)).getSignedUrl(params, callback); + + // const params = { + // Key: this.getPath(file), + // Expires: classOptions.URLExpiryTimeSpan, + // ResponseContentDisposition: `${ forceDownload ? 'attachment' : 'inline' }; filename="${ encodeURI(file.name) }"`, + // }; + + // return s3.getSignedUrl('getObject', params, callback); + }; + + /** + * Creates the file in the collection + * @param file + * @param callback + * @return {string} + */ + this.create = function(file, callback) { + check(file, Object); + + if (file._id == null) { + file._id = Random.id(); + } + + file.Storj = { + path: this.options.getPath(file), + }; + + file.store = this.options.name; // assign store to file + return this.getCollection().insert(file, callback); + }; + + /** + * Removes the file + * @param fileId + * @param callback + */ + this.delete = function(fileId, callback) { + // const file = this.getCollection().findOne({ _id: fileId }); + + // this.project.deleteObject(this.bucketName, fileId).then((...args) => { + // console.log('deleteObject', ...args); + // callback && callback(); + // }); + }; + + /** + * Returns the file read stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + + this.lateReadStream = async function(fileId, file, options = {}) { + Meteor.wrapAsync((cb) => { + setTimeout(cb, 10000); + })(); + console.log('lateReadStream'); + const download = await this.project.downloadObject(this.bucketName, fileId); + return download; + }; + + this.getReadStream = function(fileId, file, options = {}) { + console.log('getReadStream', fileId); + // const download = Promise.await(this.lateReadStream(fileId, file, options)); + const download = Promise.await(this.project.downloadObject(this.bucketName, fileId)); + return download.stream(); + }; + + /** + * Returns the file write stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + this.getWriteStream = function(fileId, file/* , options*/) { + console.log('getWriteStream'); + const upload = Promise.await(this.project.uploadObject(this.bucketName, fileId)); + const stream = upload.stream(); + stream.on('finish', () => { + console.log('finished writing'); + }); + stream.on('close', () => { + console.log('close'); + }); + stream.on('error', () => { + console.log('error'); + }); + stream.on('drain', () => { + console.log('drain'); + }); + stream.on('pipe', () => { + console.log('pipe'); + }); + stream.on('unpipe', () => { + console.log('unpipe'); + }); + stream.on('end', () => { + console.log('end'); + }); + return stream; + }; + } +} + +// Add store to UFS namespace +UploadFS.store.Storj = StorjStore; diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json index e2a620ee35c75..e6b761dea9ace 100644 --- a/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/packages/rocketchat-i18n/i18n/en.i18n.json @@ -1726,6 +1726,8 @@ "FileUpload_S3_URLExpiryTimeSpan": "URLs Expiration Timespan", "FileUpload_S3_URLExpiryTimeSpan_Description": "Time after which Amazon S3 generated URLs will no longer be valid (in seconds). If set to less than 5 seconds, this field will be ignored.", "FileUpload_Storage_Type": "Storage Type", + "FileUpload_Storj_AccessKey": "Access Key", + "FileUpload_Storj_Bucket": "Bucket Name", "FileUpload_Webdav_Password": "WebDAV Password", "FileUpload_Webdav_Proxy_Avatars": "Proxy Avatars", "FileUpload_Webdav_Proxy_Avatars_Description": "Proxy avatar file transmissions through your server instead of direct access to the asset's URL", @@ -3529,6 +3531,7 @@ "Stop_Recording": "Stop Recording", "Store_Last_Message": "Store Last Message", "Store_Last_Message_Sent_per_Room": "Store last message sent on each room.", + "Storj": "Storj Storage", "Stream_Cast": "Stream Cast", "Stream_Cast_Address": "Stream Cast Address", "Stream_Cast_Address_Description": "IP or Host of your Rocket.Chat central Stream Cast. E.g. `192.168.1.1:3000` or `localhost:4000`", From 20898d9a44fda3e896168f3c5e240d961a354ea0 Mon Sep 17 00:00:00 2001 From: Pierre Lehnen Date: Mon, 21 Dec 2020 19:38:57 -0300 Subject: [PATCH 2/2] remvoed unused code --- app/file-upload/ufs/Storj/server.js | 75 +++-------------------------- 1 file changed, 6 insertions(+), 69 deletions(-) diff --git a/app/file-upload/ufs/Storj/server.js b/app/file-upload/ufs/Storj/server.js index e3d6e075c6dd1..cb8e730e369a3 100644 --- a/app/file-upload/ufs/Storj/server.js +++ b/app/file-upload/ufs/Storj/server.js @@ -1,4 +1,3 @@ -import { Meteor } from 'meteor/meteor'; import { check } from 'meteor/check'; import { UploadFS } from 'meteor/jalik:ufs'; import { Random } from 'meteor/random'; @@ -22,9 +21,6 @@ export class StorjStore extends UploadFS.Store { }; this.bucketName = options.bucketName; - // uplink.parseAccess(options.accessKey).then((access) => access.openProject()).then((project) => { - // this.project = project; - // }); this.getPath = function(file) { if (file.Storj) { @@ -32,27 +28,6 @@ export class StorjStore extends UploadFS.Store { } }; - this.getRedirectURL = function(file, forceDownload = false, callback) { - // const download = this.project.downloadObject(this.bucketName, encodeURI(file.name)); - - - // const params = { - // action: 'read', - // responseDisposition: forceDownload ? 'attachment' : 'inline', - // expires: Date.now() + this.options.URLExpiryTimeSpan * 1000, - // }; - - // this.bucket.file(this.getPath(file)).getSignedUrl(params, callback); - - // const params = { - // Key: this.getPath(file), - // Expires: classOptions.URLExpiryTimeSpan, - // ResponseContentDisposition: `${ forceDownload ? 'attachment' : 'inline' }; filename="${ encodeURI(file.name) }"`, - // }; - - // return s3.getSignedUrl('getObject', params, callback); - }; - /** * Creates the file in the collection * @param file @@ -80,12 +55,9 @@ export class StorjStore extends UploadFS.Store { * @param callback */ this.delete = function(fileId, callback) { - // const file = this.getCollection().findOne({ _id: fileId }); - - // this.project.deleteObject(this.bucketName, fileId).then((...args) => { - // console.log('deleteObject', ...args); - // callback && callback(); - // }); + this.project.deleteObject(this.bucketName, fileId).then(() => { + callback && callback(); + }); }; /** @@ -95,19 +67,7 @@ export class StorjStore extends UploadFS.Store { * @param options * @return {*} */ - - this.lateReadStream = async function(fileId, file, options = {}) { - Meteor.wrapAsync((cb) => { - setTimeout(cb, 10000); - })(); - console.log('lateReadStream'); - const download = await this.project.downloadObject(this.bucketName, fileId); - return download; - }; - - this.getReadStream = function(fileId, file, options = {}) { - console.log('getReadStream', fileId); - // const download = Promise.await(this.lateReadStream(fileId, file, options)); + this.getReadStream = function(fileId/* , file, options = {}*/) { const download = Promise.await(this.project.downloadObject(this.bucketName, fileId)); return download.stream(); }; @@ -119,32 +79,9 @@ export class StorjStore extends UploadFS.Store { * @param options * @return {*} */ - this.getWriteStream = function(fileId, file/* , options*/) { - console.log('getWriteStream'); + this.getWriteStream = function(fileId/* , file, options*/) { const upload = Promise.await(this.project.uploadObject(this.bucketName, fileId)); - const stream = upload.stream(); - stream.on('finish', () => { - console.log('finished writing'); - }); - stream.on('close', () => { - console.log('close'); - }); - stream.on('error', () => { - console.log('error'); - }); - stream.on('drain', () => { - console.log('drain'); - }); - stream.on('pipe', () => { - console.log('pipe'); - }); - stream.on('unpipe', () => { - console.log('unpipe'); - }); - stream.on('end', () => { - console.log('end'); - }); - return stream; + return upload.stream(); }; } }