Feature/update upsert API (#3836)

* update upsert API

* add fix for override files in upsert vector

packages/api-documentation/src/yml/swagger.yml

@@ -628,8 +628,8 @@ paths:
                 - document-store
             security:
                 - bearerAuth: []
-            summary: Upsert new document to document store
-            description: Upsert new document to document store
+            summary: Upsert document to document store
+            description: Upsert document to document store
             operationId: upsertDocument
             parameters:
                 - in: path
@@ -2326,6 +2326,13 @@ components:
                     type: string
                     format: uuid
                     description: Document ID within the store. If provided, existing configuration from the document will be used for the new document
+                metadata:
+                    type: object
+                    description: Metadata associated with the document
+                    example: { 'foo': 'bar' }
+                replaceExisting:
+                    type: boolean
+                    description: Whether to replace existing document loader with the new upserted chunks. However this does not delete the existing embeddings in the vector store
                 loader:
                     type: object
                     properties:

packages/server/src/Interface.DocumentStore.ts

@@ -70,6 +70,8 @@ export interface IDocumentStoreLoaderForPreview extends IDocumentStoreLoader {
 
 export interface IDocumentStoreUpsertData {
     docId: string
+    metadata?: string | object
+    replaceExisting?: boolean
     loader?: {
         name: string
         config: ICommonObject

packages/server/src/controllers/documentstore/index.ts

@@ -428,6 +428,27 @@ const generateDocStoreToolDesc = async (req: Request, res: Response, next: NextF
     }
 }
 
+const getDocStoreConfigs = async (req: Request, res: Response, next: NextFunction) => {
+    try {
+        if (typeof req.params.id === 'undefined' || req.params.id === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.getDocStoreConfigs - storeId not provided!`
+            )
+        }
+        if (typeof req.params.loaderId === 'undefined' || req.params.loaderId === '') {
+            throw new InternalFlowiseError(
+                StatusCodes.PRECONDITION_FAILED,
+                `Error: documentStoreController.getDocStoreConfigs - doc loader Id not provided!`
+            )
+        }
+        const apiResponse = await documentStoreService.findDocStoreAvailableConfigs(req.params.id, req.params.loaderId)
+        return res.json(apiResponse)
+    } catch (error) {
+        next(error)
+    }
+}
+
 export default {
     deleteDocumentStore,
     createDocumentStore,
@@ -453,5 +474,6 @@ export default {
     upsertDocStoreMiddleware,
     refreshDocStoreMiddleware,
     saveProcessingLoader,
-    generateDocStoreToolDesc
+    generateDocStoreToolDesc,
+    getDocStoreConfigs
 }

packages/server/src/index.ts

@@ -25,6 +25,7 @@ import { validateAPIKey } from './utils/validateKey'
 import { IMetricsProvider } from './Interface.Metrics'
 import { Prometheus } from './metrics/Prometheus'
 import { OpenTelemetry } from './metrics/OpenTelemetry'
+import { WHITELIST_URLS } from './utils/constants'
 import 'global-agent/bootstrap'
 
 declare global {
@@ -124,27 +125,7 @@ export class App {
             next()
         })
 
-        const whitelistURLs = [
-            '/api/v1/verify/apikey/',
-            '/api/v1/chatflows/apikey/',
-            '/api/v1/public-chatflows',
-            '/api/v1/public-chatbotConfig',
-            '/api/v1/prediction/',
-            '/api/v1/vector/upsert/',
-            '/api/v1/node-icon/',
-            '/api/v1/components-credentials-icon/',
-            '/api/v1/chatflows-streaming',
-            '/api/v1/chatflows-uploads',
-            '/api/v1/openai-assistants-file/download',
-            '/api/v1/feedback',
-            '/api/v1/leads',
-            '/api/v1/get-upload-file',
-            '/api/v1/ip',
-            '/api/v1/ping',
-            '/api/v1/version',
-            '/api/v1/attachments',
-            '/api/v1/metrics'
-        ]
+        const whitelistURLs = WHITELIST_URLS
         const URL_CASE_INSENSITIVE_REGEX: RegExp = /\/api\/v1\//i
         const URL_CASE_SENSITIVE_REGEX: RegExp = /\/api\/v1\//
 

packages/server/src/routes/documentstore/index.ts

@@ -21,6 +21,8 @@ router.get('/store/:id', documentStoreController.getDocumentStoreById)
 router.put('/store/:id', documentStoreController.updateDocumentStore)
 // Delete documentStore
 router.delete('/store/:id', documentStoreController.deleteDocumentStore)
+// Get document store configs
+router.get('/store-configs/:id/:loaderId', documentStoreController.getDocStoreConfigs)
 
 /** Component Nodes = Document Store - Loaders */
 // Get all loaders

packages/server/src/services/assistants/index.ts

@@ -15,6 +15,7 @@ import { DocumentStore } from '../../database/entities/DocumentStore'
 import { ICommonObject } from 'flowise-components'
 import logger from '../../utils/logger'
 import { ASSISTANT_PROMPT_GENERATOR } from '../../utils/prompt'
+import { INPUT_PARAMS_TYPE } from '../../utils/constants'
 
 const createAssistant = async (requestBody: any): Promise<Assistant> => {
     try {
@@ -425,26 +426,11 @@ const getDocumentStores = async (): Promise<any> => {
 const getTools = async (): Promise<any> => {
     try {
         const tools = await nodesService.getAllNodesForCategory('Tools')
-        const whitelistTypes = [
-            'asyncOptions',
-            'options',
-            'multiOptions',
-            'datagrid',
-            'string',
-            'number',
-            'boolean',
-            'password',
-            'json',
-            'code',
-            'date',
-            'file',
-            'folder',
-            'tabs'
-        ]
+
         // filter out those tools that input params type are not in the list
         const filteredTools = tools.filter((tool) => {
             const inputs = tool.inputs || []
-            return inputs.every((input) => whitelistTypes.includes(input.type))
+            return inputs.every((input) => INPUT_PARAMS_TYPE.includes(input.type))
         })
         return filteredTools
     } catch (error) {

packages/server/src/services/documentstore/index.ts

@@ -24,7 +24,8 @@ import {
     IDocumentStoreRefreshData,
     IDocumentStoreUpsertData,
     IDocumentStoreWhereUsed,
-    INodeData
+    INodeData,
+    IOverrideConfig
 } from '../../Interface'
 import { DocumentStoreFileChunk } from '../../database/entities/DocumentStoreFileChunk'
 import { v4 as uuidv4 } from 'uuid'
@@ -41,6 +42,7 @@ import { UpsertHistory } from '../../database/entities/UpsertHistory'
 import { cloneDeep, omit } from 'lodash'
 import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../../Interface.Metrics'
 import { DOCUMENTSTORE_TOOL_DESCRIPTION_PROMPT_GENERATOR } from '../../utils/prompt'
+import { INPUT_PARAMS_TYPE } from '../../utils/constants'
 
 const DOCUMENT_STORE_BASE_FOLDER = 'docustore'
 
@@ -1323,6 +1325,15 @@ const upsertDocStoreMiddleware = async (
 ) => {
     const appServer = getRunningExpressApp()
     const docId = data.docId
+    let metadata = {}
+    if (data.metadata) {
+        try {
+            metadata = typeof data.metadata === 'string' ? JSON.parse(data.metadata) : data.metadata
+        } catch (error) {
+            throw new InternalFlowiseError(StatusCodes.BAD_REQUEST, `Error: Invalid metadata`)
+        }
+    }
+    const replaceExisting = data.replaceExisting ?? false
     const newLoader = typeof data.loader === 'string' ? JSON.parse(data.loader) : data.loader
     const newSplitter = typeof data.splitter === 'string' ? JSON.parse(data.splitter) : data.splitter
     const newVectorStore = typeof data.vectorStore === 'string' ? JSON.parse(data.vectorStore) : data.vectorStore
@@ -1479,6 +1490,13 @@ const upsertDocStoreMiddleware = async (
         }
     }
 
+    if (Object.keys(metadata).length > 0) {
+        loaderConfig = {
+            ...loaderConfig,
+            metadata
+        }
+    }
+
     // Step 4: Verification for must have components
     if (!loaderName || !loaderId || !loaderConfig) {
         throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Loader not configured`)
@@ -1503,7 +1521,7 @@ const upsertDocStoreMiddleware = async (
         splitterConfig
     }
 
-    if (isRefreshExisting) {
+    if (isRefreshExisting || replaceExisting) {
         processData.id = docId
     }
 
@@ -1629,6 +1647,146 @@ const generateDocStoreToolDesc = async (docStoreId: string, selectedChatModel: I
     }
 }
 
+export const findDocStoreAvailableConfigs = async (storeId: string, docId: string) => {
+    // find the document store
+    const appServer = getRunningExpressApp()
+    const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({ id: storeId })
+
+    if (!entity) {
+        throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`)
+    }
+
+    const loaders = JSON.parse(entity.loaders)
+    const loader = loaders.find((ldr: IDocumentStoreLoader) => ldr.id === docId)
+    if (!loader) {
+        throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document loader ${docId} not found`)
+    }
+
+    const nodes = []
+    const componentCredentials = appServer.nodesPool.componentCredentials
+
+    const loaderName = loader.loaderId
+    const loaderLabel = appServer.nodesPool.componentNodes[loaderName].label
+
+    const loaderInputs =
+        appServer.nodesPool.componentNodes[loaderName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
+    nodes.push({
+        label: loaderLabel,
+        nodeId: `${loaderName}_0`,
+        inputParams: loaderInputs
+    })
+
+    const splitterName = loader.splitterId
+    if (splitterName) {
+        const splitterLabel = appServer.nodesPool.componentNodes[splitterName].label
+        const splitterInputs =
+            appServer.nodesPool.componentNodes[splitterName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
+        nodes.push({
+            label: splitterLabel,
+            nodeId: `${splitterName}_0`,
+            inputParams: splitterInputs
+        })
+    }
+
+    if (entity.vectorStoreConfig) {
+        const vectorStoreName = JSON.parse(entity.vectorStoreConfig || '{}').name
+        const vectorStoreLabel = appServer.nodesPool.componentNodes[vectorStoreName].label
+        const vectorStoreInputs =
+            appServer.nodesPool.componentNodes[vectorStoreName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
+        nodes.push({
+            label: vectorStoreLabel,
+            nodeId: `${vectorStoreName}_0`,
+            inputParams: vectorStoreInputs
+        })
+    }
+
+    if (entity.embeddingConfig) {
+        const embeddingName = JSON.parse(entity.embeddingConfig || '{}').name
+        const embeddingLabel = appServer.nodesPool.componentNodes[embeddingName].label
+        const embeddingInputs =
+            appServer.nodesPool.componentNodes[embeddingName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
+        nodes.push({
+            label: embeddingLabel,
+            nodeId: `${embeddingName}_0`,
+            inputParams: embeddingInputs
+        })
+    }
+
+    if (entity.recordManagerConfig) {
+        const recordManagerName = JSON.parse(entity.recordManagerConfig || '{}').name
+        const recordManagerLabel = appServer.nodesPool.componentNodes[recordManagerName].label
+        const recordManagerInputs =
+            appServer.nodesPool.componentNodes[recordManagerName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? []
+        nodes.push({
+            label: recordManagerLabel,
+            nodeId: `${recordManagerName}_0`,
+            inputParams: recordManagerInputs
+        })
+    }
+
+    const configs: IOverrideConfig[] = []
+    for (const node of nodes) {
+        const inputParams = node.inputParams
+        for (const inputParam of inputParams) {
+            let obj: IOverrideConfig
+            if (inputParam.type === 'file') {
+                obj = {
+                    node: node.label,
+                    nodeId: node.nodeId,
+                    label: inputParam.label,
+                    name: 'files',
+                    type: inputParam.fileType ?? inputParam.type
+                }
+            } else if (inputParam.type === 'options') {
+                obj = {
+                    node: node.label,
+                    nodeId: node.nodeId,
+                    label: inputParam.label,
+                    name: inputParam.name,
+                    type: inputParam.options
+                        ? inputParam.options
+                              ?.map((option) => {
+                                  return option.name
+                              })
+                              .join(', ')
+                        : 'string'
+                }
+            } else if (inputParam.type === 'credential') {
+                // get component credential inputs
+                for (const name of inputParam.credentialNames ?? []) {
+                    if (Object.prototype.hasOwnProperty.call(componentCredentials, name)) {
+                        const inputs = componentCredentials[name]?.inputs ?? []
+                        for (const input of inputs) {
+                            obj = {
+                                node: node.label,
+                                nodeId: node.nodeId,
+                                label: input.label,
+                                name: input.name,
+                                type: input.type === 'password' ? 'string' : input.type
+                            }
+                            configs.push(obj)
+                        }
+                    }
+                }
+                continue
+            } else {
+                obj = {
+                    node: node.label,
+                    nodeId: node.nodeId,
+                    label: inputParam.label,
+                    name: inputParam.name,
+                    type: inputParam.type === 'password' ? 'string' : inputParam.type
+                }
+            }
+            if (!configs.some((config) => JSON.stringify(config) === JSON.stringify(obj))) {
+                configs.push(obj)
+            }
+        }
+    }
+
+    return configs
+}
+
 export default {
     updateDocumentStoreUsage,
     deleteDocumentStore,
@@ -1656,5 +1814,6 @@ export default {
     updateVectorStoreConfigOnly,
     upsertDocStoreMiddleware,
     refreshDocStoreMiddleware,
-    generateDocStoreToolDesc
+    generateDocStoreToolDesc,
+    findDocStoreAvailableConfigs
 }