Skip to content

Commit 56f9208

Browse files
chungyau97HenryHengZJ
chungyau97
and
HenryHengZJ
authored
Feature/export import stage 2 (#3063)
* add export all function * modify exportAll to reuse existing code from other services * modify routes of export-import * add exportAll function into UI * add errorhandler * add importAll Function into UI for ChatFlow * modify importAll Function to import tools * remove appServer variable * modify exportAll to exportData for new requirement in backend * chore modify type camelCase to PascalCase in exportImportService * add import export for variables, assistants, and checkboxes for UI --------- Co-authored-by: Henry <[email protected]>
1 parent 40a1064 commit 56f9208

File tree

18 files changed

+644
-89
lines changed

18 files changed

+644
-89
lines changed

‎packages/server/src/Interface.ts

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
import { IAction } from 'flowise-components'
2-
import { ICommonObject, IFileUpload, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components'
1+
import { IAction, ICommonObject, IFileUpload, INode, INodeData as INodeDataFromComponent, INodeParams } from 'flowise-components'
32

43
export type MessageType = 'apiMessage' | 'userMessage'
54

‎packages/server/src/controllers/export-import/index.ts

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import { NextFunction, Request, Response } from 'express'
2+
import exportImportService from '../../services/export-import'
3+
4+
const exportData = async (req: Request, res: Response, next: NextFunction) => {
5+
try {
6+
const apiResponse = await exportImportService.exportData(exportImportService.convertExportInput(req.body))
7+
return res.json(apiResponse)
8+
} catch (error) {
9+
next(error)
10+
}
11+
}
12+
13+
const importData = async (req: Request, res: Response, next: NextFunction) => {
14+
try {
15+
const importData = req.body
16+
await exportImportService.importData(importData)
17+
return res.json({ message: 'success' })
18+
} catch (error) {
19+
next(error)
20+
}
21+
}
22+
23+
export default {
24+
exportData,
25+
importData
26+
}

‎packages/server/src/routes/export-import/index.ts

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import express from 'express'
2+
import exportImportController from '../../controllers/export-import'
3+
const router = express.Router()
4+
5+
router.post('/export', exportImportController.exportData)
6+
7+
router.post('/import', exportImportController.importData)
8+
9+
export default router

‎packages/server/src/routes/index.ts

+2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import componentsCredentialsRouter from './components-credentials'
99
import componentsCredentialsIconRouter from './components-credentials-icon'
1010
import credentialsRouter from './credentials'
1111
import documentStoreRouter from './documentstore'
12+
import exportImportRouter from './export-import'
1213
import feedbackRouter from './feedback'
1314
import fetchLinksRouter from './fetch-links'
1415
import flowConfigRouter from './flow-config'
@@ -53,6 +54,7 @@ router.use('/components-credentials-icon', componentsCredentialsIconRouter)
5354
router.use('/chatflows-uploads', chatflowsUploadsRouter)
5455
router.use('/credentials', credentialsRouter)
5556
router.use('/document-store', documentStoreRouter)
57+
router.use('/export-import', exportImportRouter)
5658
router.use('/feedback', feedbackRouter)
5759
router.use('/fetch-links', fetchLinksRouter)
5860
router.use('/flow-config', flowConfigRouter)

‎packages/server/src/services/assistants/index.ts

+51-1
Original file line numberDiff line numberDiff line change
@@ -289,10 +289,60 @@ const updateAssistant = async (assistantId: string, requestBody: any): Promise<A
289289
}
290290
}
291291

292+
const importAssistants = async (newAssistants: Partial<Assistant>[]): Promise<any> => {
293+
try {
294+
const appServer = getRunningExpressApp()
295+
296+
// step 1 - check whether array is zero
297+
if (newAssistants.length == 0) return
298+
299+
// step 2 - check whether ids are duplicate in database
300+
let ids = '('
301+
let count: number = 0
302+
const lastCount = newAssistants.length - 1
303+
newAssistants.forEach((newAssistant) => {
304+
ids += `'${newAssistant.id}'`
305+
if (lastCount != count) ids += ','
306+
if (lastCount == count) ids += ')'
307+
count += 1
308+
})
309+
310+
const selectResponse = await appServer.AppDataSource.getRepository(Assistant)
311+
.createQueryBuilder('assistant')
312+
.select('assistant.id')
313+
.where(`assistant.id IN ${ids}`)
314+
.getMany()
315+
const foundIds = selectResponse.map((response) => {
316+
return response.id
317+
})
318+
319+
// step 3 - remove ids that are only duplicate
320+
const prepVariables: Partial<Assistant>[] = newAssistants.map((newAssistant) => {
321+
let id: string = ''
322+
if (newAssistant.id) id = newAssistant.id
323+
if (foundIds.includes(id)) {
324+
newAssistant.id = undefined
325+
}
326+
return newAssistant
327+
})
328+
329+
// step 4 - transactional insert array of entities
330+
const insertResponse = await appServer.AppDataSource.getRepository(Assistant).insert(prepVariables)
331+
332+
return insertResponse
333+
} catch (error) {
334+
throw new InternalFlowiseError(
335+
StatusCodes.INTERNAL_SERVER_ERROR,
336+
`Error: variableService.importVariables - ${getErrorMessage(error)}`
337+
)
338+
}
339+
}
340+
292341
export default {
293342
createAssistant,
294343
deleteAssistant,
295344
getAllAssistants,
296345
getAssistantById,
297-
updateAssistant
346+
updateAssistant,
347+
importAssistants
298348
}

‎packages/server/src/services/chatflows/index.ts

+9-7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { removeFolderFromStorage } from 'flowise-components'
22
import { StatusCodes } from 'http-status-codes'
3-
import { ChatflowType, IChatFlow, IReactFlowObject } from '../../Interface'
3+
import { ChatflowType, IReactFlowObject } from '../../Interface'
44
import { ChatFlow } from '../../database/entities/ChatFlow'
55
import { ChatMessage } from '../../database/entities/ChatMessage'
66
import { ChatMessageFeedback } from '../../database/entities/ChatMessageFeedback'
@@ -103,14 +103,17 @@ const deleteChatflow = async (chatflowId: string): Promise<any> => {
103103
}
104104
}
105105

106-
const getAllChatflows = async (type?: ChatflowType): Promise<IChatFlow[]> => {
106+
const getAllChatflows = async (type?: ChatflowType): Promise<ChatFlow[]> => {
107107
try {
108108
const appServer = getRunningExpressApp()
109109
const dbResponse = await appServer.AppDataSource.getRepository(ChatFlow).find()
110110
if (type === 'MULTIAGENT') {
111-
return dbResponse.filter((chatflow) => chatflow.type === type)
111+
return dbResponse.filter((chatflow) => chatflow.type === 'MULTIAGENT')
112+
} else if (type === 'CHATFLOW') {
113+
// fetch all chatflows that are not agentflow
114+
return dbResponse.filter((chatflow) => chatflow.type === 'CHATFLOW' || !chatflow.type)
112115
}
113-
return dbResponse.filter((chatflow) => chatflow.type === 'CHATFLOW' || !chatflow.type)
116+
return dbResponse
114117
} catch (error) {
115118
throw new InternalFlowiseError(
116119
StatusCodes.INTERNAL_SERVER_ERROR,
@@ -202,7 +205,7 @@ const importChatflows = async (newChatflows: Partial<ChatFlow>[]): Promise<any>
202205
const appServer = getRunningExpressApp()
203206

204207
// step 1 - check whether file chatflows array is zero
205-
if (newChatflows.length == 0) throw new Error('No chatflows in this file.')
208+
if (newChatflows.length == 0) return
206209

207210
// step 2 - check whether ids are duplicate in database
208211
let ids = '('
@@ -232,9 +235,8 @@ const importChatflows = async (newChatflows: Partial<ChatFlow>[]): Promise<any>
232235
if (newChatflow.flowData) flowData = newChatflow.flowData
233236
if (foundIds.includes(id)) {
234237
newChatflow.id = undefined
235-
newChatflow.name += ' with new id'
238+
newChatflow.name += ' (1)'
236239
}
237-
newChatflow.type = 'CHATFLOW'
238240
newChatflow.flowData = JSON.stringify(JSON.parse(flowData))
239241
return newChatflow
240242
})

‎packages/server/src/services/documentstore/index.ts

+14
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,19 @@ const getAllDocumentStores = async () => {
6262
}
6363
}
6464

65+
const getAllDocumentFileChunks = async () => {
66+
try {
67+
const appServer = getRunningExpressApp()
68+
const entities = await appServer.AppDataSource.getRepository(DocumentStoreFileChunk).find()
69+
return entities
70+
} catch (error) {
71+
throw new InternalFlowiseError(
72+
StatusCodes.INTERNAL_SERVER_ERROR,
73+
`Error: documentStoreServices.getAllDocumentFileChunks - ${getErrorMessage(error)}`
74+
)
75+
}
76+
}
77+
6578
const deleteLoaderFromDocumentStore = async (storeId: string, loaderId: string) => {
6679
try {
6780
const appServer = getRunningExpressApp()
@@ -1225,6 +1238,7 @@ export default {
12251238
createDocumentStore,
12261239
deleteLoaderFromDocumentStore,
12271240
getAllDocumentStores,
1241+
getAllDocumentFileChunks,
12281242
getDocumentStoreById,
12291243
getUsedChatflowNames,
12301244
getDocumentStoreFileChunks,

‎packages/server/src/services/export-import/index.ts

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import { StatusCodes } from 'http-status-codes'
2+
import { ChatFlow } from '../../database/entities/ChatFlow'
3+
import { Tool } from '../../database/entities/Tool'
4+
import { Variable } from '../../database/entities/Variable'
5+
import { Assistant } from '../../database/entities/Assistant'
6+
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
7+
import { getErrorMessage } from '../../errors/utils'
8+
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
9+
import chatflowService from '../chatflows'
10+
import toolsService from '../tools'
11+
import variableService from '../variables'
12+
import assistantService from '../assistants'
13+
14+
type ExportInput = {
15+
tool: boolean
16+
chatflow: boolean
17+
agentflow: boolean
18+
variable: boolean
19+
assistant: boolean
20+
}
21+
22+
type ExportData = {
23+
Tool: Tool[]
24+
ChatFlow: ChatFlow[]
25+
AgentFlow: ChatFlow[]
26+
Variable: Variable[]
27+
Assistant: Assistant[]
28+
}
29+
30+
const convertExportInput = (body: any): ExportInput => {
31+
try {
32+
if (!body || typeof body !== 'object') throw new Error('Invalid ExportInput object in request body')
33+
if (body.tool && typeof body.tool !== 'boolean') throw new Error('Invalid tool property in ExportInput object')
34+
if (body.chatflow && typeof body.chatflow !== 'boolean') throw new Error('Invalid chatflow property in ExportInput object')
35+
if (body.agentflow && typeof body.agentflow !== 'boolean') throw new Error('Invalid agentflow property in ExportInput object')
36+
if (body.variable && typeof body.variable !== 'boolean') throw new Error('Invalid variable property in ExportInput object')
37+
if (body.assistant && typeof body.assistant !== 'boolean') throw new Error('Invalid assistant property in ExportInput object')
38+
return body as ExportInput
39+
} catch (error) {
40+
throw new InternalFlowiseError(
41+
StatusCodes.INTERNAL_SERVER_ERROR,
42+
`Error: exportImportService.convertExportInput - ${getErrorMessage(error)}`
43+
)
44+
}
45+
}
46+
47+
const FileDefaultName = 'ExportData.json'
48+
const exportData = async (exportInput: ExportInput): Promise<{ FileDefaultName: string } & ExportData> => {
49+
try {
50+
// step 1 - get all Tool
51+
let allTool: Tool[] = []
52+
if (exportInput.tool === true) allTool = await toolsService.getAllTools()
53+
54+
// step 2 - get all ChatFlow
55+
let allChatflow: ChatFlow[] = []
56+
if (exportInput.chatflow === true) allChatflow = await chatflowService.getAllChatflows('CHATFLOW')
57+
58+
// step 3 - get all MultiAgent
59+
let allMultiAgent: ChatFlow[] = []
60+
if (exportInput.agentflow === true) allMultiAgent = await chatflowService.getAllChatflows('MULTIAGENT')
61+
62+
let allVars: Variable[] = []
63+
if (exportInput.variable === true) allVars = await variableService.getAllVariables()
64+
65+
let allAssistants: Assistant[] = []
66+
if (exportInput.assistant === true) allAssistants = await assistantService.getAllAssistants()
67+
68+
return {
69+
FileDefaultName,
70+
Tool: allTool,
71+
ChatFlow: allChatflow,
72+
AgentFlow: allMultiAgent,
73+
Variable: allVars,
74+
Assistant: allAssistants
75+
}
76+
} catch (error) {
77+
throw new InternalFlowiseError(
78+
StatusCodes.INTERNAL_SERVER_ERROR,
79+
`Error: exportImportService.exportData - ${getErrorMessage(error)}`
80+
)
81+
}
82+
}
83+
84+
const importData = async (importData: ExportData) => {
85+
try {
86+
const appServer = getRunningExpressApp()
87+
const queryRunner = appServer.AppDataSource.createQueryRunner()
88+
89+
try {
90+
queryRunner.startTransaction()
91+
92+
// step 1 - importTools
93+
if (importData.Tool.length > 0) await toolsService.importTools(importData.Tool)
94+
// step 2 - importChatflows
95+
if (importData.ChatFlow.length > 0) await chatflowService.importChatflows(importData.ChatFlow)
96+
// step 3 - importAgentlows
97+
if (importData.AgentFlow.length > 0) await chatflowService.importChatflows(importData.AgentFlow)
98+
if (importData.Variable.length > 0) await variableService.importVariables(importData.Variable)
99+
if (importData.Assistant.length > 0) await assistantService.importAssistants(importData.Assistant)
100+
queryRunner.commitTransaction()
101+
} catch (error) {
102+
queryRunner.rollbackTransaction()
103+
throw error
104+
} finally {
105+
queryRunner.release()
106+
}
107+
} catch (error) {
108+
throw new InternalFlowiseError(
109+
StatusCodes.INTERNAL_SERVER_ERROR,
110+
`Error: exportImportService.importAll - ${getErrorMessage(error)}`
111+
)
112+
}
113+
}
114+
115+
export default {
116+
convertExportInput,
117+
exportData,
118+
importData
119+
}

‎packages/server/src/services/tools/index.ts

+52-4
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import { StatusCodes } from 'http-status-codes'
2-
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
32
import { Tool } from '../../database/entities/Tool'
4-
import { getAppVersion } from '../../utils'
53
import { InternalFlowiseError } from '../../errors/internalFlowiseError'
64
import { getErrorMessage } from '../../errors/utils'
5+
import { getAppVersion } from '../../utils'
6+
import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
77

88
const createTool = async (requestBody: any): Promise<any> => {
99
try {
@@ -35,7 +35,7 @@ const deleteTool = async (toolId: string): Promise<any> => {
3535
}
3636
}
3737

38-
const getAllTools = async (): Promise<any> => {
38+
const getAllTools = async (): Promise<Tool[]> => {
3939
try {
4040
const appServer = getRunningExpressApp()
4141
const dbResponse = await appServer.AppDataSource.getRepository(Tool).find()
@@ -79,10 +79,58 @@ const updateTool = async (toolId: string, toolBody: any): Promise<any> => {
7979
}
8080
}
8181

82+
const importTools = async (newTools: Partial<Tool>[]) => {
83+
try {
84+
const appServer = getRunningExpressApp()
85+
86+
// step 1 - check whether file tools array is zero
87+
if (newTools.length == 0) return
88+
89+
// step 2 - check whether ids are duplicate in database
90+
let ids = '('
91+
let count: number = 0
92+
const lastCount = newTools.length - 1
93+
newTools.forEach((newTools) => {
94+
ids += `'${newTools.id}'`
95+
if (lastCount != count) ids += ','
96+
if (lastCount == count) ids += ')'
97+
count += 1
98+
})
99+
100+
const selectResponse = await appServer.AppDataSource.getRepository(Tool)
101+
.createQueryBuilder('t')
102+
.select('t.id')
103+
.where(`t.id IN ${ids}`)
104+
.getMany()
105+
const foundIds = selectResponse.map((response) => {
106+
return response.id
107+
})
108+
109+
// step 3 - remove ids that are only duplicate
110+
const prepTools: Partial<Tool>[] = newTools.map((newTool) => {
111+
let id: string = ''
112+
if (newTool.id) id = newTool.id
113+
if (foundIds.includes(id)) {
114+
newTool.id = undefined
115+
newTool.name += ' (1)'
116+
}
117+
return newTool
118+
})
119+
120+
// step 4 - transactional insert array of entities
121+
const insertResponse = await appServer.AppDataSource.getRepository(Tool).insert(prepTools)
122+
123+
return insertResponse
124+
} catch (error) {
125+
throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Error: toolsService.importTools - ${getErrorMessage(error)}`)
126+
}
127+
}
128+
82129
export default {
83130
createTool,
84131
deleteTool,
85132
getAllTools,
86133
getToolById,
87-
updateTool
134+
updateTool,
135+
importTools
88136
}

0 commit comments

Comments
 (0)