[Usntructured loaders] Make Unstructured API URL optional when enviro…

…nment variable is present (FlowiseAI#3414) * Make Unstructured API URL optional when environment variable is present * Fix empty apiUrl option in Unsctructured flowise loader * Add focumentation for env vars
cheviader · Nov 1, 2024 · a159da5 · a159da5
1 parent 15d59a9
commit a159da5
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 9 deletions.
diff --git a/packages/components/nodes/documentloaders/S3File/README.md b/packages/components/nodes/documentloaders/S3File/README.md
@@ -0,0 +1,13 @@
+# S3 File Loader
+
+DS File Loarder integration for Flowise
+
+## 🌱 Env Variables
+
+| Variable                     | Description                                                                                     | Type                                             | Default                             |
+| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
+| UNSTRUCTURED_API_URL | Default `unstructuredApiUrl` for S3 File Loader                                            | String                                                                    |  http://localhost:8000/general/v0/general          |
+
+## License
+
+Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
diff --git a/packages/components/nodes/documentloaders/S3File/S3File.ts b/packages/components/nodes/documentloaders/S3File/S3File.ts
@@ -70,7 +70,8 @@ class S3_DocumentLoaders implements INode {
                 description:
                     'Your Unstructured.io URL. Read <a target="_blank" href="https://unstructured-io.github.io/unstructured/introduction.html#getting-started">more</a> on how to get started',
                 type: 'string',
-                default: 'http://localhost:8000/general/v0/general'
+                placeholder: process.env.UNSTRUCTURED_API_URL || 'http://localhost:8000/general/v0/general',
+                optional: !!process.env.UNSTRUCTURED_API_URL
             },
             {
                 label: 'Unstructured API KEY',

diff --git a/packages/components/nodes/documentloaders/Unstructured/README.md b/packages/components/nodes/documentloaders/Unstructured/README.md
@@ -0,0 +1,13 @@
+# Unstructured File/Floder Loader
+
+Unstructured File Loader integration for Flowise
+
+## 🌱 Env Variables
+
+| Variable                     | Description                                                                                     | Type                                             | Default                             |
+| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
+| UNSTRUCTURED_API_URL | Default `apiUrl` for Unstructured File/Floder Loader                                            | String                                                                    |  http://localhost:8000/general/v0/general          |
+
+## License
+
+Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
diff --git a/packages/components/nodes/documentloaders/Unstructured/Unstructured.ts b/packages/components/nodes/documentloaders/Unstructured/Unstructured.ts
@@ -29,9 +29,9 @@ type Element = {
 export class UnstructuredLoader extends BaseDocumentLoader {
     public filePath: string
 
-    private apiUrl = 'https://api.unstructuredapp.io/general/v0/general'
+    private apiUrl = process.env.UNSTRUCTURED_API_URL || 'https://api.unstructuredapp.io/general/v0/general'
 
-    private apiKey?: string
+    private apiKey: string | undefined = process.env.UNSTRUCTURED_API_KEY
 
     private strategy: StringWithAutocomplete<UnstructuredLoaderStrategy> = 'hi_res'
 
@@ -66,10 +66,10 @@ export class UnstructuredLoader extends BaseDocumentLoader {
 
         const options = optionsOrLegacyFilePath
         this.apiKey = options.apiKey
-        this.apiUrl = options.apiUrl ?? this.apiUrl
-        this.strategy = options.strategy ?? this.strategy
+        this.apiUrl = options.apiUrl || this.apiUrl
+        this.strategy = options.strategy || this.strategy
         this.encoding = options.encoding
-        this.ocrLanguages = options.ocrLanguages ?? this.ocrLanguages
+        this.ocrLanguages = options.ocrLanguages || this.ocrLanguages
         this.coordinates = options.coordinates
         this.pdfInferTableStructure = options.pdfInferTableStructure
         this.xmlKeepTags = options.xmlKeepTags
@@ -128,7 +128,7 @@ export class UnstructuredLoader extends BaseDocumentLoader {
         }
 
         const headers = {
-            'UNSTRUCTURED-API-KEY': this.apiKey ?? ''
+            'UNSTRUCTURED-API-KEY': this.apiKey || ''
         }
 
         const response = await fetch(this.apiUrl, {

diff --git a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts
@@ -63,7 +63,8 @@ class UnstructuredFile_DocumentLoaders implements INode {
                 description:
                     'Unstructured API URL. Read <a target="_blank" href="https://docs.unstructured.io/api-reference/api-services/saas-api-development-guide">more</a> on how to get started',
                 type: 'string',
-                default: 'http://localhost:8000/general/v0/general'
+                placeholder: process.env.UNSTRUCTURED_API_URL || 'http://localhost:8000/general/v0/general',
+                optional: !!process.env.UNSTRUCTURED_API_URL
             },
             {
                 label: 'Strategy',

diff --git a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts
@@ -51,7 +51,8 @@ class UnstructuredFolder_DocumentLoaders implements INode {
                 description:
                     'Unstructured API URL. Read <a target="_blank" href="https://unstructured-io.github.io/unstructured/introduction.html#getting-started">more</a> on how to get started',
                 type: 'string',
-                default: 'http://localhost:8000/general/v0/general'
+                placeholder: process.env.UNSTRUCTURED_API_URL || 'http://localhost:8000/general/v0/general',
+                optional: !!process.env.UNSTRUCTURED_API_URL
             },
             {
                 label: 'Strategy',