diff --git a/packages/api-documentation/src/yml/swagger.yml b/packages/api-documentation/src/yml/swagger.yml index 5a92d4bf2ef..36662d0206b 100644 --- a/packages/api-documentation/src/yml/swagger.yml +++ b/packages/api-documentation/src/yml/swagger.yml @@ -628,8 +628,8 @@ paths: - document-store security: - bearerAuth: [] - summary: Upsert new document to document store - description: Upsert new document to document store + summary: Upsert document to document store + description: Upsert document to document store operationId: upsertDocument parameters: - in: path @@ -2326,6 +2326,13 @@ components: type: string format: uuid description: Document ID within the store. If provided, existing configuration from the document will be used for the new document + metadata: + type: object + description: Metadata associated with the document + example: { 'foo': 'bar' } + replaceExisting: + type: boolean + description: Whether to replace existing document loader with the new upserted chunks. However this does not delete the existing embeddings in the vector store loader: type: object properties: diff --git a/packages/server/src/Interface.DocumentStore.ts b/packages/server/src/Interface.DocumentStore.ts index 44bb10dc449..fca62d4faf1 100644 --- a/packages/server/src/Interface.DocumentStore.ts +++ b/packages/server/src/Interface.DocumentStore.ts @@ -70,6 +70,8 @@ export interface IDocumentStoreLoaderForPreview extends IDocumentStoreLoader { export interface IDocumentStoreUpsertData { docId: string + metadata?: string | object + replaceExisting?: boolean loader?: { name: string config: ICommonObject diff --git a/packages/server/src/controllers/documentstore/index.ts b/packages/server/src/controllers/documentstore/index.ts index 2499cb301df..9eb80b80fc4 100644 --- a/packages/server/src/controllers/documentstore/index.ts +++ b/packages/server/src/controllers/documentstore/index.ts @@ -428,6 +428,27 @@ const generateDocStoreToolDesc = async (req: Request, res: Response, next: NextF } } +const getDocStoreConfigs = async (req: Request, res: Response, next: NextFunction) => { + try { + if (typeof req.params.id === 'undefined' || req.params.id === '') { + throw new InternalFlowiseError( + StatusCodes.PRECONDITION_FAILED, + `Error: documentStoreController.getDocStoreConfigs - storeId not provided!` + ) + } + if (typeof req.params.loaderId === 'undefined' || req.params.loaderId === '') { + throw new InternalFlowiseError( + StatusCodes.PRECONDITION_FAILED, + `Error: documentStoreController.getDocStoreConfigs - doc loader Id not provided!` + ) + } + const apiResponse = await documentStoreService.findDocStoreAvailableConfigs(req.params.id, req.params.loaderId) + return res.json(apiResponse) + } catch (error) { + next(error) + } +} + export default { deleteDocumentStore, createDocumentStore, @@ -453,5 +474,6 @@ export default { upsertDocStoreMiddleware, refreshDocStoreMiddleware, saveProcessingLoader, - generateDocStoreToolDesc + generateDocStoreToolDesc, + getDocStoreConfigs } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 3b6742f6f59..1ed6a749fb8 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -25,6 +25,7 @@ import { validateAPIKey } from './utils/validateKey' import { IMetricsProvider } from './Interface.Metrics' import { Prometheus } from './metrics/Prometheus' import { OpenTelemetry } from './metrics/OpenTelemetry' +import { WHITELIST_URLS } from './utils/constants' import 'global-agent/bootstrap' declare global { @@ -124,27 +125,7 @@ export class App { next() }) - const whitelistURLs = [ - '/api/v1/verify/apikey/', - '/api/v1/chatflows/apikey/', - '/api/v1/public-chatflows', - '/api/v1/public-chatbotConfig', - '/api/v1/prediction/', - '/api/v1/vector/upsert/', - '/api/v1/node-icon/', - '/api/v1/components-credentials-icon/', - '/api/v1/chatflows-streaming', - '/api/v1/chatflows-uploads', - '/api/v1/openai-assistants-file/download', - '/api/v1/feedback', - '/api/v1/leads', - '/api/v1/get-upload-file', - '/api/v1/ip', - '/api/v1/ping', - '/api/v1/version', - '/api/v1/attachments', - '/api/v1/metrics' - ] + const whitelistURLs = WHITELIST_URLS const URL_CASE_INSENSITIVE_REGEX: RegExp = /\/api\/v1\//i const URL_CASE_SENSITIVE_REGEX: RegExp = /\/api\/v1\// diff --git a/packages/server/src/routes/documentstore/index.ts b/packages/server/src/routes/documentstore/index.ts index 1d1bad89e0a..b53ceb3c5cc 100644 --- a/packages/server/src/routes/documentstore/index.ts +++ b/packages/server/src/routes/documentstore/index.ts @@ -21,6 +21,8 @@ router.get('/store/:id', documentStoreController.getDocumentStoreById) router.put('/store/:id', documentStoreController.updateDocumentStore) // Delete documentStore router.delete('/store/:id', documentStoreController.deleteDocumentStore) +// Get document store configs +router.get('/store-configs/:id/:loaderId', documentStoreController.getDocStoreConfigs) /** Component Nodes = Document Store - Loaders */ // Get all loaders diff --git a/packages/server/src/services/assistants/index.ts b/packages/server/src/services/assistants/index.ts index 424440bafd8..0681376bc53 100644 --- a/packages/server/src/services/assistants/index.ts +++ b/packages/server/src/services/assistants/index.ts @@ -15,6 +15,7 @@ import { DocumentStore } from '../../database/entities/DocumentStore' import { ICommonObject } from 'flowise-components' import logger from '../../utils/logger' import { ASSISTANT_PROMPT_GENERATOR } from '../../utils/prompt' +import { INPUT_PARAMS_TYPE } from '../../utils/constants' const createAssistant = async (requestBody: any): Promise => { try { @@ -425,26 +426,11 @@ const getDocumentStores = async (): Promise => { const getTools = async (): Promise => { try { const tools = await nodesService.getAllNodesForCategory('Tools') - const whitelistTypes = [ - 'asyncOptions', - 'options', - 'multiOptions', - 'datagrid', - 'string', - 'number', - 'boolean', - 'password', - 'json', - 'code', - 'date', - 'file', - 'folder', - 'tabs' - ] + // filter out those tools that input params type are not in the list const filteredTools = tools.filter((tool) => { const inputs = tool.inputs || [] - return inputs.every((input) => whitelistTypes.includes(input.type)) + return inputs.every((input) => INPUT_PARAMS_TYPE.includes(input.type)) }) return filteredTools } catch (error) { diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts index 3ac7d10a8f5..db2ecf322a2 100644 --- a/packages/server/src/services/documentstore/index.ts +++ b/packages/server/src/services/documentstore/index.ts @@ -24,7 +24,8 @@ import { IDocumentStoreRefreshData, IDocumentStoreUpsertData, IDocumentStoreWhereUsed, - INodeData + INodeData, + IOverrideConfig } from '../../Interface' import { DocumentStoreFileChunk } from '../../database/entities/DocumentStoreFileChunk' import { v4 as uuidv4 } from 'uuid' @@ -41,6 +42,7 @@ import { UpsertHistory } from '../../database/entities/UpsertHistory' import { cloneDeep, omit } from 'lodash' import { FLOWISE_COUNTER_STATUS, FLOWISE_METRIC_COUNTERS } from '../../Interface.Metrics' import { DOCUMENTSTORE_TOOL_DESCRIPTION_PROMPT_GENERATOR } from '../../utils/prompt' +import { INPUT_PARAMS_TYPE } from '../../utils/constants' const DOCUMENT_STORE_BASE_FOLDER = 'docustore' @@ -1323,6 +1325,15 @@ const upsertDocStoreMiddleware = async ( ) => { const appServer = getRunningExpressApp() const docId = data.docId + let metadata = {} + if (data.metadata) { + try { + metadata = typeof data.metadata === 'string' ? JSON.parse(data.metadata) : data.metadata + } catch (error) { + throw new InternalFlowiseError(StatusCodes.BAD_REQUEST, `Error: Invalid metadata`) + } + } + const replaceExisting = data.replaceExisting ?? false const newLoader = typeof data.loader === 'string' ? JSON.parse(data.loader) : data.loader const newSplitter = typeof data.splitter === 'string' ? JSON.parse(data.splitter) : data.splitter const newVectorStore = typeof data.vectorStore === 'string' ? JSON.parse(data.vectorStore) : data.vectorStore @@ -1479,6 +1490,13 @@ const upsertDocStoreMiddleware = async ( } } + if (Object.keys(metadata).length > 0) { + loaderConfig = { + ...loaderConfig, + metadata + } + } + // Step 4: Verification for must have components if (!loaderName || !loaderId || !loaderConfig) { throw new InternalFlowiseError(StatusCodes.INTERNAL_SERVER_ERROR, `Loader not configured`) @@ -1503,7 +1521,7 @@ const upsertDocStoreMiddleware = async ( splitterConfig } - if (isRefreshExisting) { + if (isRefreshExisting || replaceExisting) { processData.id = docId } @@ -1629,6 +1647,146 @@ const generateDocStoreToolDesc = async (docStoreId: string, selectedChatModel: I } } +export const findDocStoreAvailableConfigs = async (storeId: string, docId: string) => { + // find the document store + const appServer = getRunningExpressApp() + const entity = await appServer.AppDataSource.getRepository(DocumentStore).findOneBy({ id: storeId }) + + if (!entity) { + throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${storeId} not found`) + } + + const loaders = JSON.parse(entity.loaders) + const loader = loaders.find((ldr: IDocumentStoreLoader) => ldr.id === docId) + if (!loader) { + throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document loader ${docId} not found`) + } + + const nodes = [] + const componentCredentials = appServer.nodesPool.componentCredentials + + const loaderName = loader.loaderId + const loaderLabel = appServer.nodesPool.componentNodes[loaderName].label + + const loaderInputs = + appServer.nodesPool.componentNodes[loaderName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? [] + nodes.push({ + label: loaderLabel, + nodeId: `${loaderName}_0`, + inputParams: loaderInputs + }) + + const splitterName = loader.splitterId + if (splitterName) { + const splitterLabel = appServer.nodesPool.componentNodes[splitterName].label + const splitterInputs = + appServer.nodesPool.componentNodes[splitterName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? [] + nodes.push({ + label: splitterLabel, + nodeId: `${splitterName}_0`, + inputParams: splitterInputs + }) + } + + if (entity.vectorStoreConfig) { + const vectorStoreName = JSON.parse(entity.vectorStoreConfig || '{}').name + const vectorStoreLabel = appServer.nodesPool.componentNodes[vectorStoreName].label + const vectorStoreInputs = + appServer.nodesPool.componentNodes[vectorStoreName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? [] + nodes.push({ + label: vectorStoreLabel, + nodeId: `${vectorStoreName}_0`, + inputParams: vectorStoreInputs + }) + } + + if (entity.embeddingConfig) { + const embeddingName = JSON.parse(entity.embeddingConfig || '{}').name + const embeddingLabel = appServer.nodesPool.componentNodes[embeddingName].label + const embeddingInputs = + appServer.nodesPool.componentNodes[embeddingName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? [] + nodes.push({ + label: embeddingLabel, + nodeId: `${embeddingName}_0`, + inputParams: embeddingInputs + }) + } + + if (entity.recordManagerConfig) { + const recordManagerName = JSON.parse(entity.recordManagerConfig || '{}').name + const recordManagerLabel = appServer.nodesPool.componentNodes[recordManagerName].label + const recordManagerInputs = + appServer.nodesPool.componentNodes[recordManagerName].inputs?.filter((input) => INPUT_PARAMS_TYPE.includes(input.type)) ?? [] + nodes.push({ + label: recordManagerLabel, + nodeId: `${recordManagerName}_0`, + inputParams: recordManagerInputs + }) + } + + const configs: IOverrideConfig[] = [] + for (const node of nodes) { + const inputParams = node.inputParams + for (const inputParam of inputParams) { + let obj: IOverrideConfig + if (inputParam.type === 'file') { + obj = { + node: node.label, + nodeId: node.nodeId, + label: inputParam.label, + name: 'files', + type: inputParam.fileType ?? inputParam.type + } + } else if (inputParam.type === 'options') { + obj = { + node: node.label, + nodeId: node.nodeId, + label: inputParam.label, + name: inputParam.name, + type: inputParam.options + ? inputParam.options + ?.map((option) => { + return option.name + }) + .join(', ') + : 'string' + } + } else if (inputParam.type === 'credential') { + // get component credential inputs + for (const name of inputParam.credentialNames ?? []) { + if (Object.prototype.hasOwnProperty.call(componentCredentials, name)) { + const inputs = componentCredentials[name]?.inputs ?? [] + for (const input of inputs) { + obj = { + node: node.label, + nodeId: node.nodeId, + label: input.label, + name: input.name, + type: input.type === 'password' ? 'string' : input.type + } + configs.push(obj) + } + } + } + continue + } else { + obj = { + node: node.label, + nodeId: node.nodeId, + label: inputParam.label, + name: inputParam.name, + type: inputParam.type === 'password' ? 'string' : inputParam.type + } + } + if (!configs.some((config) => JSON.stringify(config) === JSON.stringify(obj))) { + configs.push(obj) + } + } + } + + return configs +} + export default { updateDocumentStoreUsage, deleteDocumentStore, @@ -1656,5 +1814,6 @@ export default { updateVectorStoreConfigOnly, upsertDocStoreMiddleware, refreshDocStoreMiddleware, - generateDocStoreToolDesc + generateDocStoreToolDesc, + findDocStoreAvailableConfigs } diff --git a/packages/server/src/utils/constants.ts b/packages/server/src/utils/constants.ts new file mode 100644 index 00000000000..5db77b9aa6b --- /dev/null +++ b/packages/server/src/utils/constants.ts @@ -0,0 +1,38 @@ +export const WHITELIST_URLS = [ + '/api/v1/verify/apikey/', + '/api/v1/chatflows/apikey/', + '/api/v1/public-chatflows', + '/api/v1/public-chatbotConfig', + '/api/v1/prediction/', + '/api/v1/vector/upsert/', + '/api/v1/node-icon/', + '/api/v1/components-credentials-icon/', + '/api/v1/chatflows-streaming', + '/api/v1/chatflows-uploads', + '/api/v1/openai-assistants-file/download', + '/api/v1/feedback', + '/api/v1/leads', + '/api/v1/get-upload-file', + '/api/v1/ip', + '/api/v1/ping', + '/api/v1/version', + '/api/v1/attachments', + '/api/v1/metrics' +] + +export const INPUT_PARAMS_TYPE = [ + 'asyncOptions', + 'options', + 'multiOptions', + 'datagrid', + 'string', + 'number', + 'boolean', + 'password', + 'json', + 'code', + 'date', + 'file', + 'folder', + 'tabs' +] diff --git a/packages/server/src/utils/upsertVector.ts b/packages/server/src/utils/upsertVector.ts index de1e564130e..d92ab41d357 100644 --- a/packages/server/src/utils/upsertVector.ts +++ b/packages/server/src/utils/upsertVector.ts @@ -161,16 +161,29 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) => const availableVariables = await appServer.AppDataSource.getRepository(Variable).find() const { nodeOverrides, variableOverrides, apiOverrideStatus } = getAPIOverrideConfig(chatflow) - // For "files" input, add a new node override with the actual input name such as pdfFile, txtFile, etc. + // For "files" input, add a new node override with the actual input name such as pdfFile, txtFile, etc, to allow overriding the input for (const nodeLabel in nodeOverrides) { const params = nodeOverrides[nodeLabel] const enabledFileParam = params.find((param) => param.enabled && param.name === 'files') if (enabledFileParam) { - const fileInputFieldFromExt = mapExtToInputField(enabledFileParam.type) - nodeOverrides[nodeLabel].push({ - ...enabledFileParam, - name: fileInputFieldFromExt - }) + if (enabledFileParam.type.includes(',')) { + const fileInputFieldsFromExt = enabledFileParam.type.split(',').map((fileType) => mapExtToInputField(fileType.trim())) + for (const fileInputFieldFromExt of fileInputFieldsFromExt) { + if (nodeOverrides[nodeLabel].some((param) => param.name === fileInputFieldFromExt)) { + continue + } + nodeOverrides[nodeLabel].push({ + ...enabledFileParam, + name: fileInputFieldFromExt + }) + } + } else { + const fileInputFieldFromExt = mapExtToInputField(enabledFileParam.type) + nodeOverrides[nodeLabel].push({ + ...enabledFileParam, + name: fileInputFieldFromExt + }) + } } } diff --git a/packages/ui/src/api/documentstore.js b/packages/ui/src/api/documentstore.js index e53ab50c69e..cb6211b9706 100644 --- a/packages/ui/src/api/documentstore.js +++ b/packages/ui/src/api/documentstore.js @@ -6,6 +6,7 @@ const getSpecificDocumentStore = (id) => client.get(`/document-store/store/${id} const createDocumentStore = (body) => client.post(`/document-store/store`, body) const updateDocumentStore = (id, body) => client.put(`/document-store/store/${id}`, body) const deleteDocumentStore = (id) => client.delete(`/document-store/store/${id}`) +const getDocumentStoreConfig = (storeId, loaderId) => client.get(`/document-store/store-configs/${storeId}/${loaderId}`) const deleteLoaderFromStore = (id, fileId) => client.delete(`/document-store/loader/${id}/${fileId}`) const deleteChunkFromStore = (storeId, loaderId, chunkId) => client.delete(`/document-store/chunks/${storeId}/${loaderId}/${chunkId}`) @@ -52,5 +53,6 @@ export default { updateVectorStoreConfig, saveProcessingLoader, refreshLoader, - generateDocStoreToolDesc + generateDocStoreToolDesc, + getDocumentStoreConfig } diff --git a/packages/ui/src/views/chatflows/APICodeDialog.jsx b/packages/ui/src/views/chatflows/APICodeDialog.jsx index ea1e1caefeb..a7ae54d9d21 100644 --- a/packages/ui/src/views/chatflows/APICodeDialog.jsx +++ b/packages/ui/src/views/chatflows/APICodeDialog.jsx @@ -38,7 +38,7 @@ import cURLSVG from '@/assets/images/cURL.svg' import EmbedSVG from '@/assets/images/embed.svg' import ShareChatbotSVG from '@/assets/images/sharing.png' import settingsSVG from '@/assets/images/settings.svg' -import { IconBulb, IconBox, IconVariable } from '@tabler/icons-react' +import { IconBulb, IconBox, IconVariable, IconExclamationCircle } from '@tabler/icons-react' // API import apiKeyApi from '@/api/apikey' @@ -726,9 +726,44 @@ formData.append("openAIApiKey[openAIEmbeddings_0]", "sk-my-openai-2nd-key")` {checkboxVal && getConfigApi.data && getConfigApi.data.length > 0 && ( <> - + You can override existing input configuration of the chatflow with overrideConfig property. +
+
+ + + { + 'For security reason, override config is disabled by default. You can change this by going into Chatflow Configuration -> Security tab, and enable the property you want to override.' + } +  Refer{' '} + + here + {' '} + for more details + +
+
diff --git a/packages/ui/src/views/docstore/DocStoreAPIDialog.jsx b/packages/ui/src/views/docstore/DocStoreAPIDialog.jsx new file mode 100644 index 00000000000..0cf56496c25 --- /dev/null +++ b/packages/ui/src/views/docstore/DocStoreAPIDialog.jsx @@ -0,0 +1,396 @@ +import { createPortal } from 'react-dom' +import { useState, useEffect } from 'react' +import PropTypes from 'prop-types' +import rehypeMathjax from 'rehype-mathjax' +import rehypeRaw from 'rehype-raw' +import remarkGfm from 'remark-gfm' +import remarkMath from 'remark-math' +import { MemoizedReactMarkdown } from '@/ui-component/markdown/MemoizedReactMarkdown' +import { CodeBlock } from '@/ui-component/markdown/CodeBlock' +import { Typography, Stack, Card, Accordion, AccordionSummary, AccordionDetails, Dialog, DialogContent, DialogTitle } from '@mui/material' +import { TableViewOnly } from '@/ui-component/table/Table' +import documentstoreApi from '@/api/documentstore' +import useApi from '@/hooks/useApi' +import { useTheme } from '@mui/material/styles' +import ExpandMoreIcon from '@mui/icons-material/ExpandMore' + +const DocStoreAPIDialog = ({ show, dialogProps, onCancel }) => { + const [nodeConfig, setNodeConfig] = useState({}) + const [values, setValues] = useState('') + const theme = useTheme() + const [nodeConfigExpanded, setNodeConfigExpanded] = useState({}) + + const getConfigApi = useApi(documentstoreApi.getDocumentStoreConfig) + + const formDataRequest = () => { + return `With the Upsert API, you can choose an existing document and reuse the same configuration for upserting. + +\`\`\`python +import requests +import json + +API_URL = "http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId}" +API_KEY = "your_api_key_here" + +# use form data to upload files +form_data = { + "files": ('my-another-file.pdf', open('my-another-file.pdf', 'rb')) +} + +body_data = { + "docId": "${dialogProps.loaderId}", + "metadata": {}, # Add additional metadata to the document chunks + "replaceExisting": True, # Replace existing document with the new upserted chunks + "splitter": json.dumps({"config":{"chunkSize":20000}}) # Override existing configuration + # "loader": "", + # "vectorStore": "", + # "embedding": "", + # "recordManager": "", +} + +headers = { + "Authorization": f"Bearer {BEARER_TOKEN}" +} + +def query(form_data): + response = requests.post(API_URL, files=form_data, data=body_data, headers=headers) + print(response) + return response.json() + +output = query(form_data) +print(output) +\`\`\` + +\`\`\`javascript +// use FormData to upload files +let formData = new FormData(); +formData.append("files", input.files[0]); +formData.append("docId", "${dialogProps.loaderId}"); +formData.append("splitter", JSON.stringify({"config":{"chunkSize":20000}})); +// Add additional metadata to the document chunks +formData.append("metadata", "{}"); +// Replace existing document with the new upserted chunks +formData.append("replaceExisting", "true"); +// Override existing configuration +// formData.append("loader", ""); +// formData.append("embedding", ""); +// formData.append("vectorStore", ""); +// formData.append("recordManager", ""); + +async function query(formData) { + const response = await fetch( + "http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId}", + { + method: "POST", + headers: { + "Authorization": "Bearer " + }, + body: formData + } + ); + const result = await response.json(); + return result; +} + +query(formData).then((response) => { + console.log(response); +}); +\`\`\` + +\`\`\`bash +curl -X POST http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId} \\ + -H "Authorization: Bearer " \\ + -F "files=@" \\ + -F "docId=${dialogProps.loaderId}" \\ + -F "splitter={"config":{"chunkSize":20000}}" \\ + -F "metadata={}" \\ + -F "replaceExisting=true" \\ + # Override existing configuration: + # -F "loader=" \\ + # -F "embedding=" \\ + # -F "vectorStore=" \\ + # -F "recordManager=" +\`\`\` +` + } + + const jsonDataRequest = () => { + return `With the Upsert API, you can choose an existing document and reuse the same configuration for upserting. + +\`\`\`python +import requests + +API_URL = "http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId}" +API_KEY = "your_api_key_here" + +headers = { + "Authorization": f"Bearer {BEARER_TOKEN}" +} + +def query(payload): + response = requests.post(API_URL, json=payload, headers=headers) + return response.json() + +output = query({ + "docId": "${dialogProps.loaderId}", + "metadata": "{}", # Add additional metadata to the document chunks + "replaceExisting": True, # Replace existing document with the new upserted chunks + # Override existing configuration + "loader": { + "config": { + "text": "This is a new text" + } + }, + "splitter": { + "config": { + "chunkSize": 20000 + } + }, + # embedding: {}, + # vectorStore: {}, + # recordManager: {} +}) +print(output) +\`\`\` + +\`\`\`javascript +async function query(data) { + const response = await fetch( + "http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId}", + { + method: "POST", + headers: { + "Content-Type": "application/json", + "Authorization": "Bearer " + }, + body: JSON.stringify(data) + } + ); + const result = await response.json(); + return result; +} + +query({ + "docId": "${dialogProps.loaderId}, + "metadata": "{}", // Add additional metadata to the document chunks + "replaceExisting": true, // Replace existing document with the new upserted chunks + // Override existing configuration + "loader": { + "config": { + "text": "This is a new text" + } + }, + "splitter": { + "config": { + "chunkSize": 20000 + } + }, + // embedding: {}, + // vectorStore: {}, + // recordManager: {} +}).then((response) => { + console.log(response); +}); +\`\`\` + +\`\`\`bash +curl -X POST http://localhost:3000/api/v1/document-store/upsert/${dialogProps.storeId} \\ + -H "Content-Type: application/json" \\ + -H "Authorization: Bearer " \\ + -d '{ + "docId": "${dialogProps.loaderId}", + "metadata": "{}", + "replaceExisting": true, + "loader": { + "config": { + "text": "This is a new text" + } + }, + "splitter": { + "config": { + "chunkSize": 20000 + } + } + // Override existing configuration + // "embedding": {}, + // "vectorStore": {}, + // "recordManager": {} + }' + +\`\`\` +` + } + + const groupByNodeLabel = (nodes) => { + const result = {} + const seenNodes = new Set() + let isFormDataBody = false + + nodes.forEach((item) => { + const { node, nodeId, label, name, type } = item + if (name === 'files') isFormDataBody = true + seenNodes.add(node) + + if (!result[node]) { + result[node] = { + nodeIds: [], + params: [] + } + } + + if (!result[node].nodeIds.includes(nodeId)) result[node].nodeIds.push(nodeId) + + const param = { label, name, type } + + if (!result[node].params.some((existingParam) => JSON.stringify(existingParam) === JSON.stringify(param))) { + result[node].params.push(param) + } + }) + + // Sort the nodeIds array + for (const node in result) { + result[node].nodeIds.sort() + } + setNodeConfig(result) + + if (isFormDataBody) { + setValues(formDataRequest()) + } else { + setValues(jsonDataRequest()) + } + } + + const handleAccordionChange = (nodeLabel) => (event, isExpanded) => { + const accordianNodes = { ...nodeConfigExpanded } + accordianNodes[nodeLabel] = isExpanded + setNodeConfigExpanded(accordianNodes) + } + + useEffect(() => { + if (getConfigApi.data) { + groupByNodeLabel(getConfigApi.data) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getConfigApi.data]) + + useEffect(() => { + if (show && dialogProps) { + getConfigApi.request(dialogProps.storeId, dialogProps.loaderId) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [show, dialogProps]) + + const portalElement = document.getElementById('portal') + + const component = show ? ( + + + {dialogProps.title} + + + + ) : ( + + {children} + + ) + } + }} + > + {values} + + + You can override existing configurations: + + + + {Object.keys(nodeConfig) + .sort() + .map((nodeLabel) => ( + + } + aria-controls={`nodes-accordian-${nodeLabel}`} + id={`nodes-accordian-header-${nodeLabel}`} + > + + {nodeLabel} + {nodeConfig[nodeLabel].nodeIds.length > 0 && + nodeConfig[nodeLabel].nodeIds.map((nodeId, index) => ( +
+ + {nodeId} + +
+ ))} + + + + { + // eslint-disable-next-line + const { node, nodeId, ...rest } = obj + return rest + })} + columns={Object.keys(nodeConfig[nodeLabel].params[0]).slice(-3)} + /> + + + ))} + + + + + ) : null + + return createPortal(component, portalElement) +} + +DocStoreAPIDialog.propTypes = { + show: PropTypes.bool, + dialogProps: PropTypes.object, + onCancel: PropTypes.func +} + +export default DocStoreAPIDialog diff --git a/packages/ui/src/views/docstore/DocumentStoreDetail.jsx b/packages/ui/src/views/docstore/DocumentStoreDetail.jsx index ad1864b1f4f..457e2cf4c7d 100644 --- a/packages/ui/src/views/docstore/DocumentStoreDetail.jsx +++ b/packages/ui/src/views/docstore/DocumentStoreDetail.jsx @@ -37,6 +37,7 @@ import ViewHeader from '@/layout/MainLayout/ViewHeader' import DeleteDocStoreDialog from './DeleteDocStoreDialog' import DocumentStoreStatus from '@/views/docstore/DocumentStoreStatus' import ConfirmDialog from '@/ui-component/dialog/ConfirmDialog' +import DocStoreAPIDialog from './DocStoreAPIDialog' // API import documentsApi from '@/api/documentstore' @@ -56,6 +57,7 @@ import FileChunksIcon from '@mui/icons-material/AppRegistration' import NoteAddIcon from '@mui/icons-material/NoteAdd' import SearchIcon from '@mui/icons-material/Search' import RefreshIcon from '@mui/icons-material/Refresh' +import CodeIcon from '@mui/icons-material/Code' import doc_store_details_emptySVG from '@/assets/images/doc_store_details_empty.svg' // store @@ -142,6 +144,8 @@ const DocumentStoreDetails = () => { const [documentLoaderListDialogProps, setDocumentLoaderListDialogProps] = useState({}) const [showDeleteDocStoreDialog, setShowDeleteDocStoreDialog] = useState(false) const [deleteDocStoreDialogProps, setDeleteDocStoreDialogProps] = useState({}) + const [showDocStoreAPIDialog, setShowDocStoreAPIDialog] = useState(false) + const [docStoreAPIDialogProps, setDocStoreAPIDialogProps] = useState({}) const [anchorEl, setAnchorEl] = useState(null) const open = Boolean(anchorEl) @@ -374,6 +378,16 @@ const DocumentStoreDetails = () => { setAnchorEl(event.currentTarget) } + const onViewUpsertAPI = (storeId, loaderId) => { + const props = { + title: `Upsert API`, + storeId, + loaderId + } + setDocStoreAPIDialogProps(props) + setShowDocStoreAPIDialog(true) + } + const handleClose = () => { setAnchorEl(null) } @@ -650,6 +664,7 @@ const DocumentStoreDetails = () => { onChunkUpsert={() => navigate(`/document-stores/vector/${documentStore.id}/${loader.id}`) } + onViewUpsertAPI={() => onViewUpsertAPI(documentStore.id, loader.id)} /> ))} @@ -695,6 +710,13 @@ const DocumentStoreDetails = () => { onDelete={onDocStoreDelete} /> )} + {showDocStoreAPIDialog && ( + setShowDocStoreAPIDialog(false)} + /> + )} {isBackdropLoading && } @@ -784,6 +806,10 @@ function LoaderRow(props) { Upsert Chunks + + + View API + @@ -805,6 +831,7 @@ LoaderRow.propTypes = { onViewChunksClick: PropTypes.func, onEditClick: PropTypes.func, onDeleteClick: PropTypes.func, - onChunkUpsert: PropTypes.func + onChunkUpsert: PropTypes.func, + onViewUpsertAPI: PropTypes.func } export default DocumentStoreDetails diff --git a/packages/ui/src/views/vectorstore/VectorStoreDialog.jsx b/packages/ui/src/views/vectorstore/VectorStoreDialog.jsx index 9d0075c93f6..658db8c2c29 100644 --- a/packages/ui/src/views/vectorstore/VectorStoreDialog.jsx +++ b/packages/ui/src/views/vectorstore/VectorStoreDialog.jsx @@ -23,7 +23,7 @@ import { CheckboxInput } from '@/ui-component/checkbox/Checkbox' import { BackdropLoader } from '@/ui-component/loading/BackdropLoader' import { TableViewOnly } from '@/ui-component/table/Table' -import { IconX, IconBulb } from '@tabler/icons-react' +import { IconX, IconBulb, IconExclamationCircle } from '@tabler/icons-react' import ExpandMoreIcon from '@mui/icons-material/ExpandMore' import pythonSVG from '@/assets/images/python.svg' import javascriptSVG from '@/assets/images/javascript.svg' @@ -545,6 +545,47 @@ formData.append("openAIApiKey[openAIEmbeddings_0]", "sk-my-openai-2nd-key")` showLineNumbers={false} wrapLines /> +
+
+ + + { + 'For security reason, override config is disabled by default. You can change this by going into Chatflow Configuration -> Security tab, and enable the property you want to override.' + } +  Refer{' '} + + here + {' '} + for more details + +
+