Feature/DocumentStore

packages/components/nodes/documentloaders/DocumentStore/DocStoreLoader.ts

@@ -0,0 +1,100 @@
+import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeOutputsValue, INodeParams } from '../../../src/Interface'
+import { DataSource } from 'typeorm'
+import { Document } from '@langchain/core/documents'
+
+class DocStore_DocumentLoaders implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    outputs: INodeOutputsValue[]
+    badge: string
+
+    constructor() {
+        this.label = 'Document Store'
+        this.name = 'documentStore'
+        this.version = 1.0
+        this.type = 'Document'
+        this.icon = 'dstore.svg'
+        this.badge = 'NEW'
+
+        this.category = 'Document Loaders'
+        this.description = `Load data from pre-configured document stores`
+        this.baseClasses = [this.type]
+        this.inputs = [
+            {
+                label: 'Select Store',
+                name: 'selectedStore',
+                type: 'asyncOptions',
+                loadMethod: 'listStores'
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Document',
+                name: 'document',
+                description: 'Array of document objects containing metadata and pageContent',
+                baseClasses: [...this.baseClasses, 'json']
+            },
+            {
+                label: 'Text',
+                name: 'text',
+                description: 'Concatenated string from pageContent of documents',
+                baseClasses: ['string', 'json']
+            }
+        ]
+    }
+
+    //@ts-ignore
+    loadMethods = {
+        async listStores(_: INodeData, options: ICommonObject): Promise<INodeOptionsValue[]> {
+            const returnData: INodeOptionsValue[] = []
+
+            const appDataSource = options.appDataSource as DataSource
+            const databaseEntities = options.databaseEntities as IDatabaseEntity
+
+            if (appDataSource === undefined || !appDataSource) {
+                return returnData
+            }
+
+            const stores = await appDataSource.getRepository(databaseEntities['DocumentStore']).find()
+            for (const store of stores) {
+                if (store.status === 'SYNC') {
+                    const obj = {
+                        name: store.id,
+                        label: store.name,
+                        description: store.description
+                    }
+                    returnData.push(obj)
+                }
+            }
+            return returnData
+        }
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        // if (options.isUpsert) {
+        const selectedStore = nodeData.inputs?.selectedStore as string
+        const appDataSource = options.appDataSource as DataSource
+        const databaseEntities = options.databaseEntities as IDatabaseEntity
+        const chunks = await appDataSource
+            .getRepository(databaseEntities['DocumentStoreFileChunk'])
+            .find({ where: { storeId: selectedStore } })
+
+        const finalDocs = []
+        for (const chunk of chunks) {
+            finalDocs.push(new Document({ pageContent: chunk.pageContent, metadata: JSON.parse(chunk.metadata) }))
+        }
+        return finalDocs
+        // }
+        //
+        // return {}
+    }
+}
+
+module.exports = { nodeClass: DocStore_DocumentLoaders }

packages/components/src/documentStoreProcessor.ts

@@ -0,0 +1,154 @@
+import { CharacterTextSplitter, MarkdownTextSplitter, RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
+import path from 'path'
+import { PDFLoader } from 'langchain/document_loaders/fs/pdf'
+import { TextLoader } from 'langchain/document_loaders/fs/text'
+import fs from 'fs'
+import { TextSplitter } from 'langchain/dist/text_splitter'
+import { DocxLoader } from 'langchain/document_loaders/fs/docx'
+
+export class DocumentStoreProcessor {
+    constructor() {}
+    public async splitIntoChunks(id: string, config: any, uploadedFiles: any[]): Promise<object> {
+        let splitter: TextSplitter
+        switch (config.splitter) {
+            case 'character-splitter':
+                splitter = new CharacterTextSplitter({
+                    chunkSize: config.chunkSize ?? 1000,
+                    chunkOverlap: config.chunkOverlap ?? 50,
+                    separator: config.separator ?? '\n'
+                })
+
+                break
+            case 'recursive-splitter':
+                splitter = new RecursiveCharacterTextSplitter({
+                    chunkSize: config.chunkSize ?? 1000,
+                    chunkOverlap: config.chunkOverlap ?? 50
+                })
+
+                break
+            case 'token-splitter':
+                break
+            case 'code-splitter':
+                splitter = RecursiveCharacterTextSplitter.fromLanguage(config.codeLanguage, {
+                    chunkSize: config.chunkSize ?? 1000,
+                    chunkOverlap: config.chunkOverlap ?? 50
+                })
+                break
+            case 'html-to-markdown-splitter':
+                break
+            case 'markdown-splitter':
+                splitter = new MarkdownTextSplitter({
+                    chunkSize: config.chunkSize ?? 1000,
+                    chunkOverlap: config.chunkOverlap ?? 50
+                })
+                break
+        }
+
+        for (let i = 0; i < uploadedFiles.length; i++) {
+            let fileObj = uploadedFiles[i]
+            if (fileObj.status === 'NEW') {
+                const filename = fileObj.name
+                let loader = null
+                const blob = await fs.openAsBlob(fileObj.path)
+                switch (path.extname(filename)) {
+                    case '.pdf':
+                        loader = new PDFLoader(blob)
+                        break
+                    default:
+                        loader = new TextLoader(blob)
+                        break
+                }
+                // @ts-ignore
+                if (splitter && loader) {
+                    const docs = await loader.loadAndSplit(splitter)
+                    fileObj.chunks = docs
+                    fileObj.totalChunks = docs.length
+                }
+            }
+        }
+
+        return { id: id, uploadedFiles: uploadedFiles }
+    }
+
+    public async split(config: any, fileObj: any): Promise<object> {
+        let splitter: TextSplitter
+        switch (config.splitter) {
+            case 'character-splitter':
+                splitter = new CharacterTextSplitter({
+                    chunkSize: config.chunkSize ?? 1000,
+                    chunkOverlap: config.chunkOverlap ?? 50,
+                    separator: config.separator ?? '\n'
+                })
+
+                break
+            case 'recursive-splitter':
+                splitter = new RecursiveCharacterTextSplitter({
+                    chunkSize: config.chunkSize ?? 1000,
+                    chunkOverlap: config.chunkOverlap ?? 50
+                })
+
+                break
+            case 'token-splitter':
+                break
+            case 'code-splitter':
+                splitter = RecursiveCharacterTextSplitter.fromLanguage(config.codeLanguage, {
+                    chunkSize: config.chunkSize ?? 1000,
+                    chunkOverlap: config.chunkOverlap ?? 50
+                })
+                break
+            case 'html-to-markdown-splitter':
+                break
+            case 'markdown-splitter':
+                splitter = new MarkdownTextSplitter({
+                    chunkSize: config.chunkSize ?? 1000,
+                    chunkOverlap: config.chunkOverlap ?? 50
+                })
+                break
+        }
+
+        const filename = fileObj.name
+        let loader = null
+        const blob = await fs.openAsBlob(fileObj.path)
+        switch (path.extname(filename)) {
+            case '.pdf': {
+                let legacy = config.pdfLegacyBuild
+                if (config.pdfUsage === 'perFile') {
+                    loader = new PDFLoader(blob, {
+                        splitPages: false,
+                        pdfjs: () =>
+                            // @ts-ignore
+                            legacy ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
+                    })
+                } else {
+                    loader = new PDFLoader(blob, {
+                        pdfjs: () =>
+                            // @ts-ignore
+                            legacy ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
+                    })
+                }
+                break
+            }
+            case '.docx': {
+                loader = new DocxLoader(blob)
+                break
+            }
+            default:
+                loader = new TextLoader(blob)
+                break
+        }
+        let chunks: any[] = []
+        // @ts-ignore
+        if (splitter && loader) {
+            chunks = await loader.loadAndSplit(splitter)
+        }
+        const totalChunks = chunks.length
+        // if -1, return all chunks
+        if (config.previewChunkCount === -1) config.previewChunkCount = totalChunks
+        // return all docs if the user ask for more than we have
+        if (totalChunks <= config.previewChunkCount) config.previewChunkCount = totalChunks
+        // return only the first n chunks
+        if (totalChunks > config.previewChunkCount) chunks = chunks.slice(0, config.previewChunkCount)
+
+        return { chunks: chunks, totalChunks: totalChunks, previewChunkCount: config.previewChunkCount }
+    }
+}

packages/components/src/index.ts

@@ -7,3 +7,4 @@ dotenv.config({ path: envPath, override: true })
 export * from './Interface'
 export * from './utils'
 export * from './speechToText'
+export * from './documentStoreProcessor'

packages/server/src/Interface.ts

@@ -242,3 +242,32 @@ export interface IUploadFileSizeAndTypes {
     fileTypes: string[]
     maxUploadSize: number
 }
+
+// DocumentStore related
+export enum DocumentStoreStatus {
+    EMPTY_SYNC = 'EMPTY',
+    SYNC = 'SYNC',
+    SYNCING = 'SYNCING',
+    STALE = 'STALE',
+    NEW = 'NEW'
+}
+
+export interface IDocumentStore {
+    id: string
+    name: string
+    description: string
+    subFolder: string
+    files: string // JSON string
+    metrics: string // JSON string
+    whereUsed: string // JSON string
+    updatedDate: Date
+    createdDate: Date
+    status: DocumentStoreStatus
+}
+export interface IDocumentStoreFileChunk {
+    id: string
+    docId: string
+    storeId: string
+    pageContent: string
+    metadata: string
+}