From 01559d4da36ba512fc04f88480634eaff97e1dd4 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Sat, 16 Mar 2024 09:39:09 +0530 Subject: [PATCH 1/2] Adding support for AzureChatOpenAI --- .../AzureChatOpenAI/AzureChatOpenAI.ts | 56 +++++++++++++++++-- packages/server/src/index.ts | 2 +- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index ea924fd0f2f..1500c251170 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -1,8 +1,9 @@ -import { AzureOpenAIInput, ChatOpenAI, OpenAIChatInput } from '@langchain/openai' +import { AzureOpenAIInput, ChatOpenAI as LangchainChatOpenAI, OpenAIChatInput } from '@langchain/openai' import { BaseCache } from '@langchain/core/caches' import { BaseLLMParams } from '@langchain/core/language_models/llms' -import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { ICommonObject, IMultiModalOption, INode, INodeData, INodeParams } from '../../../src/Interface' import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { ChatOpenAI } from '../ChatOpenAI/FlowiseChatOpenAI' class AzureChatOpenAI_ChatModels implements INode { label: string @@ -24,7 +25,7 @@ class AzureChatOpenAI_ChatModels implements INode { this.icon = 'Azure.svg' this.category = 'Chat Models' this.description = 'Wrapper around Azure OpenAI large language models that use the Chat endpoint' - this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)] + this.baseClasses = [this.type, ...getBaseClasses(LangchainChatOpenAI)] this.credential = { label: 'Connect Credential', name: 'credential', @@ -58,6 +59,10 @@ class AzureChatOpenAI_ChatModels implements INode { { label: 'gpt-35-turbo-16k', name: 'gpt-35-turbo-16k' + }, + { + label: 'gpt-4-vision-preview', + name: 'gpt-4-vision-preview' } ], default: 'gpt-35-turbo', @@ -102,6 +107,38 @@ class AzureChatOpenAI_ChatModels implements INode { step: 1, optional: true, additionalParams: true + }, + { + label: 'Allow Image Uploads', + name: 'allowImageUploads', + type: 'boolean', + description: + 'Automatically uses gpt-4-vision-preview when image is being uploaded from chat. Only works with LLMChain, Conversation Chain, ReAct Agent, and Conversational Agent', + default: false, + optional: true + }, + { + label: 'Image Resolution', + description: 'This parameter controls the resolution in which the model views the image.', + name: 'imageResolution', + type: 'options', + options: [ + { + label: 'Low', + name: 'low' + }, + { + label: 'High', + name: 'high' + }, + { + label: 'Auto', + name: 'auto' + } + ], + default: 'low', + optional: false, + additionalParams: true } ] } @@ -122,6 +159,9 @@ class AzureChatOpenAI_ChatModels implements INode { const azureOpenAIApiDeploymentName = getCredentialParam('azureOpenAIApiDeploymentName', credentialData, nodeData) const azureOpenAIApiVersion = getCredentialParam('azureOpenAIApiVersion', credentialData, nodeData) + const allowImageUploads = nodeData.inputs?.allowImageUploads as boolean + const imageResolution = nodeData.inputs?.imageResolution as string + const obj: Partial & BaseLLMParams & Partial = { temperature: parseFloat(temperature), modelName, @@ -138,7 +178,15 @@ class AzureChatOpenAI_ChatModels implements INode { if (timeout) obj.timeout = parseInt(timeout, 10) if (cache) obj.cache = cache - const model = new ChatOpenAI(obj) + const multiModalOption: IMultiModalOption = { + image: { + allowImageUploads: allowImageUploads ?? false, + imageResolution + } + } + + const model = new ChatOpenAI(nodeData.id, obj) + model.setMultiModalOption(multiModalOption) return model } } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 44855eb733b..6d7e07284d8 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1675,7 +1675,7 @@ export class App { if (!chatflow) return `Chatflow ${chatflowid} not found` const uploadAllowedNodes = ['llmChain', 'conversationChain', 'mrklAgentChat', 'conversationalAgent'] - const uploadProcessingNodes = ['chatOpenAI', 'chatAnthropic', 'awsChatBedrock'] + const uploadProcessingNodes = ['chatOpenAI', 'chatAnthropic', 'awsChatBedrock', 'azureChatOpenAI'] const flowObj = JSON.parse(chatflow.flowData) const imgUploadSizeAndTypes: IUploadFileSizeAndTypes[] = [] From 0c649c9ce377f70d556be3bfd17dcee3d31d2547 Mon Sep 17 00:00:00 2001 From: vinodkiran Date: Sat, 16 Mar 2024 10:03:55 +0530 Subject: [PATCH 2/2] update the version number for AzureChatOpenAI --- .../nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts index 1500c251170..a78a5bd818d 100644 --- a/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts @@ -20,7 +20,7 @@ class AzureChatOpenAI_ChatModels implements INode { constructor() { this.label = 'Azure ChatOpenAI' this.name = 'azureChatOpenAI' - this.version = 2.0 + this.version = 3.0 this.type = 'AzureChatOpenAI' this.icon = 'Azure.svg' this.category = 'Chat Models'