expand text file supported file types

hemati · Oct 31, 2023 · 173429a · 173429a
1 parent e745cc2
commit 173429a
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 11 deletions.
diff --git a/packages/components/nodes/documentloaders/Text/Text.ts b/packages/components/nodes/documentloaders/Text/Text.ts
@@ -19,7 +19,7 @@ class Text_DocumentLoaders implements INode {
     constructor() {
         this.label = 'Text File'
         this.name = 'textFile'
-        this.version = 2.0
+        this.version = 3.0
         this.type = 'Document'
         this.icon = 'textFile.svg'
         this.category = 'Document Loaders'
@@ -30,7 +30,8 @@ class Text_DocumentLoaders implements INode {
                 label: 'Txt File',
                 name: 'txtFile',
                 type: 'file',
-                fileType: '.txt'
+                fileType:
+                    '.txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml'
             },
             {
                 label: 'Text Splitter',

diff --git a/...ents/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts b/...ents/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts
@@ -43,6 +43,7 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode {
                 rows: 4,
                 description: 'Array of custom separators to determine when to split the text, will override the default separators',
                 placeholder: `["|", "##", ">", "-"]`,
+                additionalParams: true,
                 optional: true
             }
         ]

diff --git a/packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json b/packages/server/marketplaces/chatflows/Conversational Retrieval QA Chain.json
@@ -368,7 +368,7 @@
                 "id": "recursiveCharacterTextSplitter_0",
                 "label": "Recursive Character Text Splitter",
                 "name": "recursiveCharacterTextSplitter",
-                "version": 1,
+                "version": 2,
                 "type": "RecursiveCharacterTextSplitter",
                 "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"],
                 "category": "Text Splitters",
@@ -388,6 +388,17 @@
                         "type": "number",
                         "optional": true,
                         "id": "recursiveCharacterTextSplitter_0-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separators",
+                        "name": "separators",
+                        "type": "string",
+                        "rows": 4,
+                        "description": "Array of custom separators to determine when to split the text, will override the default separators",
+                        "placeholder": "[\"|\", \"##\", \">\", \"-\"]",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "recursiveCharacterTextSplitter_0-input-separators-string"
                     }
                 ],
                 "inputAnchors": [],
@@ -426,7 +437,7 @@
                 "id": "textFile_0",
                 "label": "Text File",
                 "name": "textFile",
-                "version": 2,
+                "version": 3,
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
@@ -436,7 +447,7 @@
                         "label": "Txt File",
                         "name": "txtFile",
                         "type": "file",
-                        "fileType": ".txt",
+                        "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml",
                         "id": "textFile_0-input-txtFile-file"
                     },
                     {

diff --git a/packages/server/marketplaces/chatflows/Local QnA.json b/packages/server/marketplaces/chatflows/Local QnA.json
@@ -14,7 +14,7 @@
                 "id": "recursiveCharacterTextSplitter_1",
                 "label": "Recursive Character Text Splitter",
                 "name": "recursiveCharacterTextSplitter",
-                "version": 1,
+                "version": 2,
                 "type": "RecursiveCharacterTextSplitter",
                 "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"],
                 "category": "Text Splitters",
@@ -34,6 +34,17 @@
                         "type": "number",
                         "optional": true,
                         "id": "recursiveCharacterTextSplitter_1-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separators",
+                        "name": "separators",
+                        "type": "string",
+                        "rows": 4,
+                        "description": "Array of custom separators to determine when to split the text, will override the default separators",
+                        "placeholder": "[\"|\", \"##\", \">\", \"-\"]",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "recursiveCharacterTextSplitter_1-input-separators-string"
                     }
                 ],
                 "inputAnchors": [],
@@ -373,7 +384,7 @@
                 "id": "textFile_0",
                 "label": "Text File",
                 "name": "textFile",
-                "version": 1,
+                "version": 3,
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
@@ -383,7 +394,7 @@
                         "label": "Txt File",
                         "name": "txtFile",
                         "type": "file",
-                        "fileType": ".txt",
+                        "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml",
                         "id": "textFile_0-input-txtFile-file"
                     },
                     {

diff --git a/packages/server/marketplaces/chatflows/Metadata Filter Upsert.json b/packages/server/marketplaces/chatflows/Metadata Filter Upsert.json
@@ -14,7 +14,7 @@
                 "id": "recursiveCharacterTextSplitter_1",
                 "label": "Recursive Character Text Splitter",
                 "name": "recursiveCharacterTextSplitter",
-                "version": 1,
+                "version": 2,
                 "type": "RecursiveCharacterTextSplitter",
                 "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"],
                 "category": "Text Splitters",
@@ -34,6 +34,17 @@
                         "type": "number",
                         "optional": true,
                         "id": "recursiveCharacterTextSplitter_1-input-chunkOverlap-number"
+                    },
+                    {
+                        "label": "Custom Separators",
+                        "name": "separators",
+                        "type": "string",
+                        "rows": 4,
+                        "description": "Array of custom separators to determine when to split the text, will override the default separators",
+                        "placeholder": "[\"|\", \"##\", \">\", \"-\"]",
+                        "additionalParams": true,
+                        "optional": true,
+                        "id": "recursiveCharacterTextSplitter_1-input-separators-string"
                     }
                 ],
                 "inputAnchors": [],
@@ -72,7 +83,7 @@
                 "id": "textFile_0",
                 "label": "Text File",
                 "name": "textFile",
-                "version": 1,
+                "version": 3,
                 "type": "Document",
                 "baseClasses": ["Document"],
                 "category": "Document Loaders",
@@ -82,7 +93,7 @@
                         "label": "Txt File",
                         "name": "txtFile",
                         "type": "file",
-                        "fileType": ".txt",
+                        "fileType": ".txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml",
                         "id": "textFile_0-input-txtFile-file"
                     },
                     {