Feat/add multer s3 (#3854)

* add multer s3 * add types multer s3 * update multer s3 implementation * Revert "update multer s3 implementation" This reverts commit 9a25bf5. * update storage utils * update multer storage type on routes * revert getMulterStorage * revert getMulterStorage * update getmulterstorage * update getmulterstorage * update getmulterstorage
FlowiseAI · Jan 12, 2025 · d60242c · d60242c
1 parent 15d06ec
commit d60242c
Show file tree

Hide file tree

Showing 19 changed files with 289 additions and 62 deletions.
diff --git a/packages/components/src/storageUtils.ts b/packages/components/src/storageUtils.ts
@@ -120,6 +120,37 @@ export const addSingleFileToStorage = async (mime: string, bf: Buffer, fileName:
     }
 }
 
+export const getFileFromUpload = async (filePath: string): Promise<Buffer> => {
+    const storageType = getStorageType()
+    if (storageType === 's3') {
+        const { s3Client, Bucket } = getS3Config()
+
+        let Key = filePath
+        // remove the first '/' if it exists
+        if (Key.startsWith('/')) {
+            Key = Key.substring(1)
+        }
+        const getParams = {
+            Bucket,
+            Key
+        }
+
+        const response = await s3Client.send(new GetObjectCommand(getParams))
+        const body = response.Body
+        if (body instanceof Readable) {
+            const streamToString = await body.transformToString('base64')
+            if (streamToString) {
+                return Buffer.from(streamToString, 'base64')
+            }
+        }
+        // @ts-ignore
+        const buffer = Buffer.concat(response.Body.toArray())
+        return buffer
+    } else {
+        return fs.readFileSync(filePath)
+    }
+}
+
 export const getFileFromStorage = async (file: string, ...paths: string[]): Promise<Buffer> => {
     const storageType = getStorageType()
     const sanitizedFilename = _sanitizeFilename(file)
@@ -183,6 +214,20 @@ export const removeFilesFromStorage = async (...paths: string[]) => {
     }
 }
 
+export const removeSpecificFileFromUpload = async (filePath: string) => {
+    const storageType = getStorageType()
+    if (storageType === 's3') {
+        let Key = filePath
+        // remove the first '/' if it exists
+        if (Key.startsWith('/')) {
+            Key = Key.substring(1)
+        }
+        await _deleteS3Folder(Key)
+    } else {
+        fs.unlinkSync(filePath)
+    }
+}
+
 export const removeSpecificFileFromStorage = async (...paths: string[]) => {
     const storageType = getStorageType()
     if (storageType === 's3') {

diff --git a/packages/server/package.json b/packages/server/package.json
@@ -91,6 +91,7 @@
         "moment": "^2.29.3",
         "moment-timezone": "^0.5.34",
         "multer": "^1.4.5-lts.1",
+        "multer-s3": "^3.0.1",
         "mysql2": "^3.11.3",
         "openai": "^4.57.3",
         "pg": "^8.11.1",
@@ -110,6 +111,7 @@
         "@types/cors": "^2.8.12",
         "@types/crypto-js": "^4.1.1",
         "@types/multer": "^1.4.7",
+        "@types/multer-s3": "^3.0.3",
         "@types/sanitize-html": "^2.9.5",
         "concurrently": "^7.1.0",
         "cypress": "^13.13.0",

diff --git a/packages/server/src/controllers/openai-assistants-vector-store/index.ts b/packages/server/src/controllers/openai-assistants-vector-store/index.ts
@@ -143,7 +143,7 @@ const uploadFilesToAssistantVectorStore = async (req: Request, res: Response, ne
                 // Address file name with special characters: https://github.com/expressjs/multer/issues/1104
                 file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
                 uploadFiles.push({
-                    filePath: file.path,
+                    filePath: file.path ?? file.key,
                     fileName: file.originalname
                 })
             }

diff --git a/packages/server/src/controllers/openai-assistants/index.ts b/packages/server/src/controllers/openai-assistants/index.ts
@@ -84,7 +84,7 @@ const uploadAssistantFiles = async (req: Request, res: Response, next: NextFunct
                 // Address file name with special characters: https://github.com/expressjs/multer/issues/1104
                 file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
                 uploadFiles.push({
-                    filePath: file.path,
+                    filePath: file.path ?? file.key,
                     fileName: file.originalname
                 })
             }

diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts
@@ -33,6 +33,20 @@ declare global {
         interface Request {
             io?: Server
         }
+        namespace Multer {
+            interface File {
+                bucket: string
+                key: string
+                acl: string
+                contentType: string
+                contentDisposition: null
+                storageClass: string
+                serverSideEncryption: null
+                metadata: any
+                location: string
+                etag: string
+            }
+        }
     }
 }
 

diff --git a/packages/server/src/routes/attachments/index.ts b/packages/server/src/routes/attachments/index.ts
@@ -1,13 +1,10 @@
 import express from 'express'
-import multer from 'multer'
 import attachmentsController from '../../controllers/attachments'
-import { getUploadPath } from '../../utils'
+import { getMulterStorage } from '../../utils'
 
 const router = express.Router()
 
-const upload = multer({ dest: getUploadPath() })
-
 // CREATE
-router.post('/:chatflowId/:chatId', upload.array('files'), attachmentsController.createAttachment)
+router.post('/:chatflowId/:chatId', getMulterStorage().array('files'), attachmentsController.createAttachment)
 
 export default router
diff --git a/packages/server/src/routes/documentstore/index.ts b/packages/server/src/routes/documentstore/index.ts
@@ -1,12 +1,10 @@
 import express from 'express'
-import multer from 'multer'
-import { getUploadPath } from '../../utils'
 import documentStoreController from '../../controllers/documentstore'
+import { getMulterStorage } from '../../utils'
 
 const router = express.Router()
-const upload = multer({ dest: getUploadPath() })
 
-router.post(['/upsert/', '/upsert/:id'], upload.array('files'), documentStoreController.upsertDocStoreMiddleware)
+router.post(['/upsert/', '/upsert/:id'], getMulterStorage().array('files'), documentStoreController.upsertDocStoreMiddleware)
 
 router.post(['/refresh/', '/refresh/:id'], documentStoreController.refreshDocStoreMiddleware)
 

diff --git a/packages/server/src/routes/openai-assistants-files/index.ts b/packages/server/src/routes/openai-assistants-files/index.ts
@@ -1,12 +1,10 @@
 import express from 'express'
-import multer from 'multer'
 import openaiAssistantsController from '../../controllers/openai-assistants'
-import { getUploadPath } from '../../utils'
+import { getMulterStorage } from '../../utils'
 
 const router = express.Router()
-const upload = multer({ dest: getUploadPath() })
 
 router.post('/download/', openaiAssistantsController.getFileFromAssistant)
-router.post('/upload/', upload.array('files'), openaiAssistantsController.uploadAssistantFiles)
+router.post('/upload/', getMulterStorage().array('files'), openaiAssistantsController.uploadAssistantFiles)
 
 export default router
diff --git a/packages/server/src/routes/openai-assistants-vector-store/index.ts b/packages/server/src/routes/openai-assistants-vector-store/index.ts
@@ -1,10 +1,8 @@
 import express from 'express'
-import multer from 'multer'
 import openaiAssistantsVectorStoreController from '../../controllers/openai-assistants-vector-store'
-import { getUploadPath } from '../../utils'
+import { getMulterStorage } from '../../utils'
 
 const router = express.Router()
-const upload = multer({ dest: getUploadPath() })
 
 // CREATE
 router.post('/', openaiAssistantsVectorStoreController.createAssistantVectorStore)
@@ -22,7 +20,7 @@ router.put(['/', '/:id'], openaiAssistantsVectorStoreController.updateAssistantV
 router.delete(['/', '/:id'], openaiAssistantsVectorStoreController.deleteAssistantVectorStore)
 
 // POST
-router.post('/:id', upload.array('files'), openaiAssistantsVectorStoreController.uploadFilesToAssistantVectorStore)
+router.post('/:id', getMulterStorage().array('files'), openaiAssistantsVectorStoreController.uploadFilesToAssistantVectorStore)
 
 // DELETE
 router.patch(['/', '/:id'], openaiAssistantsVectorStoreController.deleteFilesFromAssistantVectorStore)

diff --git a/packages/server/src/routes/predictions/index.ts b/packages/server/src/routes/predictions/index.ts
@@ -1,13 +1,15 @@
 import express from 'express'
-import multer from 'multer'
 import predictionsController from '../../controllers/predictions'
-import { getUploadPath } from '../../utils'
+import { getMulterStorage } from '../../utils'
 
 const router = express.Router()
 
-const upload = multer({ dest: getUploadPath() })
-
 // CREATE
-router.post(['/', '/:id'], upload.array('files'), predictionsController.getRateLimiterMiddleware, predictionsController.createPrediction)
+router.post(
+    ['/', '/:id'],
+    getMulterStorage().array('files'),
+    predictionsController.getRateLimiterMiddleware,
+    predictionsController.createPrediction
+)
 
 export default router
diff --git a/packages/server/src/routes/vectors/index.ts b/packages/server/src/routes/vectors/index.ts
@@ -1,19 +1,16 @@
 import express from 'express'
-import multer from 'multer'
 import vectorsController from '../../controllers/vectors'
-import { getUploadPath } from '../../utils'
+import { getMulterStorage } from '../../utils'
 
 const router = express.Router()
 
-const upload = multer({ dest: getUploadPath() })
-
 // CREATE
 router.post(
     ['/upsert/', '/upsert/:id'],
-    upload.array('files'),
+    getMulterStorage().array('files'),
     vectorsController.getRateLimiterMiddleware,
     vectorsController.upsertVectorMiddleware
 )
-router.post(['/internal-upsert/', '/internal-upsert/:id'], upload.array('files'), vectorsController.createInternalUpsert)
+router.post(['/internal-upsert/', '/internal-upsert/:id'], getMulterStorage().array('files'), vectorsController.createInternalUpsert)
 
 export default router
diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts
@@ -1,17 +1,18 @@
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { DocumentStore } from '../../database/entities/DocumentStore'
-import * as fs from 'fs'
 import * as path from 'path'
 import {
     addArrayFilesToStorage,
     addSingleFileToStorage,
     getFileFromStorage,
+    getFileFromUpload,
     ICommonObject,
     IDocument,
     mapExtToInputField,
     mapMimeTypeToInputField,
     removeFilesFromStorage,
-    removeSpecificFileFromStorage
+    removeSpecificFileFromStorage,
+    removeSpecificFileFromUpload
 } from 'flowise-components'
 import {
     addLoaderSource,
@@ -1441,7 +1442,7 @@ const upsertDocStoreMiddleware = async (
         const filesLoaderConfig: ICommonObject = {}
         for (const file of files) {
             const fileNames: string[] = []
-            const fileBuffer = fs.readFileSync(file.path)
+            const fileBuffer = await getFileFromUpload(file.path ?? file.key)
             // Address file name with special characters: https://github.com/expressjs/multer/issues/1104
             file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
 
@@ -1481,7 +1482,7 @@ const upsertDocStoreMiddleware = async (
                 filesLoaderConfig[fileInputField] = JSON.stringify([storagePath])
             }
 
-            fs.unlinkSync(file.path)
+            await removeSpecificFileFromUpload(file.path ?? file.key)
         }
 
         loaderConfig = {

diff --git a/packages/server/src/services/openai-assistants-vector-store/index.ts b/packages/server/src/services/openai-assistants-vector-store/index.ts
@@ -1,11 +1,11 @@
 import OpenAI from 'openai'
 import { StatusCodes } from 'http-status-codes'
-import fs from 'fs'
 import { Credential } from '../../database/entities/Credential'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { getErrorMessage } from '../../errors/utils'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { decryptCredentialData } from '../../utils'
+import { getFileFromUpload, removeSpecificFileFromUpload } from 'flowise-components'
 
 const getAssistantVectorStore = async (credentialId: string, vectorStoreId: string) => {
     try {
@@ -178,13 +178,14 @@ const uploadFilesToAssistantVectorStore = async (
         const openai = new OpenAI({ apiKey: openAIApiKey })
         const uploadedFiles = []
         for (const file of files) {
-            const toFile = await OpenAI.toFile(fs.readFileSync(file.filePath), file.fileName)
+            const fileBuffer = await getFileFromUpload(file.filePath)
+            const toFile = await OpenAI.toFile(fileBuffer, file.fileName)
             const createdFile = await openai.files.create({
                 file: toFile,
                 purpose: 'assistants'
             })
             uploadedFiles.push(createdFile)
-            fs.unlinkSync(file.filePath)
+            await removeSpecificFileFromUpload(file.filePath)
         }
 
         const file_ids = [...uploadedFiles.map((file) => file.id)]

diff --git a/packages/server/src/services/openai-assistants/index.ts b/packages/server/src/services/openai-assistants/index.ts
@@ -1,11 +1,11 @@
 import OpenAI from 'openai'
-import fs from 'fs'
 import { StatusCodes } from 'http-status-codes'
 import { decryptCredentialData } from '../../utils'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { Credential } from '../../database/entities/Credential'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { getErrorMessage } from '../../errors/utils'
+import { getFileFromUpload, removeSpecificFileFromUpload } from 'flowise-components'
 
 // ----------------------------------------
 // Assistants
@@ -101,13 +101,14 @@ const uploadFilesToAssistant = async (credentialId: string, files: { filePath: s
     const uploadedFiles = []
 
     for (const file of files) {
-        const toFile = await OpenAI.toFile(fs.readFileSync(file.filePath), file.fileName)
+        const fileBuffer = await getFileFromUpload(file.filePath)
+        const toFile = await OpenAI.toFile(fileBuffer, file.fileName)
         const createdFile = await openai.files.create({
             file: toFile,
             purpose: 'assistants'
         })
         uploadedFiles.push(createdFile)
-        fs.unlinkSync(file.filePath)
+        await removeSpecificFileFromUpload(file.filePath)
     }
 
     return uploadedFiles

diff --git a/packages/server/src/utils/buildChatflow.ts b/packages/server/src/utils/buildChatflow.ts
@@ -9,7 +9,9 @@ import {
     mapMimeTypeToInputField,
     mapExtToInputField,
     generateFollowUpPrompts,
-    IServerSideEventStreamer
+    IServerSideEventStreamer,
+    getFileFromUpload,
+    removeSpecificFileFromUpload
 } from 'flowise-components'
 import { StatusCodes } from 'http-status-codes'
 import {
@@ -49,7 +51,6 @@ import { validateChatflowAPIKey } from './validateKey'
 import { databaseEntities } from '.'
 import { v4 as uuidv4 } from 'uuid'
 import { omit } from 'lodash'
-import * as fs from 'fs'
 import logger from './logger'
 import { utilAddChatMessage } from './addChatMesage'
 import { buildAgentGraph } from './buildAgentGraph'
@@ -162,7 +163,7 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
             const overrideConfig: ICommonObject = { ...req.body }
             const fileNames: string[] = []
             for (const file of files) {
-                const fileBuffer = fs.readFileSync(file.path)
+                const fileBuffer = await getFileFromUpload(file.path ?? file.key)
                 // Address file name with special characters: https://github.com/expressjs/multer/issues/1104
                 file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
                 const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid)
@@ -195,7 +196,7 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals
                     overrideConfig[fileInputField] = storagePath
                 }
 
-                fs.unlinkSync(file.path)
+                await removeSpecificFileFromUpload(file.path ?? file.key)
             }
             if (overrideConfig.vars && typeof overrideConfig.vars === 'string') {
                 overrideConfig.vars = JSON.parse(overrideConfig.vars)

diff --git a/packages/server/src/utils/createAttachment.ts b/packages/server/src/utils/createAttachment.ts
@@ -1,7 +1,13 @@
 import { Request } from 'express'
 import * as path from 'path'
-import * as fs from 'fs'
-import { addArrayFilesToStorage, IDocument, mapExtToInputField, mapMimeTypeToInputField } from 'flowise-components'
+import {
+    addArrayFilesToStorage,
+    getFileFromUpload,
+    IDocument,
+    mapExtToInputField,
+    mapMimeTypeToInputField,
+    removeSpecificFileFromUpload
+} from 'flowise-components'
 import { getRunningExpressApp } from './getRunningExpressApp'
 import { getErrorMessage } from '../errors/utils'
 
@@ -41,7 +47,7 @@ export const createFileAttachment = async (req: Request) => {
     if (files.length) {
         const isBase64 = req.body.base64
         for (const file of files) {
-            const fileBuffer = fs.readFileSync(file.path)
+            const fileBuffer = await getFileFromUpload(file.path ?? file.key)
             const fileNames: string[] = []
 
             // Address file name with special characters: https://github.com/expressjs/multer/issues/1104
@@ -63,7 +69,7 @@ export const createFileAttachment = async (req: Request) => {
                 fileInputField = fileInputFieldFromExt
             }
 
-            fs.unlinkSync(file.path)
+            await removeSpecificFileFromUpload(file.path ?? file.key)
 
             try {
                 const nodeData = {