Feat/sdk vision support

api/controllers/service_api/app/file.py

@@ -26,6 +26,9 @@ def post(self, app_model, end_user):
         if 'file' not in request.files:
             raise NoFileUploadedError()
 
+        if not file.mimetype:
+            raise UnsupportedFileTypeError()
+
         if len(request.files) > 1:
             raise TooManyFilesError()
 

sdks/nodejs-client/README.md

@@ -14,27 +14,33 @@ import { DifyClient, ChatClient, CompletionClient } from 'dify-client'
 
 const API_KEY = 'your-api-key-here'
 const user = `random-user-id`
-const inputs = {
-  name: 'test name a'
-}
-const query = "Please tell me a short story in 10 words or less."
+const query = 'Please tell me a short story in 10 words or less.'
+const remote_url_files = [{
+    type: 'image',
+    transfer_method: 'remote_url',
+    url: 'your_url_addresss'
+}]
 
 // Create a completion client
 const completionClient = new CompletionClient(API_KEY)
 // Create a completion message
-completionClient.createCompletionMessage(inputs, query, responseMode, user)
+completionClient.createCompletionMessage({'query': query}, user)
+// Create a completion message with vision model
+completionClient.createCompletionMessage({'query': 'Describe the picture.'}, user, false, remote_url_files)
 
 // Create a chat client
 const chatClient = new ChatClient(API_KEY)
 // Create a chat message in stream mode
-const response = await chatClient.createChatMessage(inputs, query, user, true, null)
+const response = await chatClient.createChatMessage({}, query, user, true, null)
 const stream = response.data;
 stream.on('data', data => {
     console.log(data);
 });
 stream.on('end', () => {
-    console.log("stream done");
+    console.log('stream done');
 });
+// Create a chat message with vision model
+chatClient.createChatMessage({}, 'Describe the picture.', user, false, null, remote_url_files)
 // Fetch conversations
 chatClient.getConversations(user)
 // Fetch conversation messages

sdks/nodejs-client/index.js

@@ -107,12 +107,12 @@ export class DifyClient {
 }
 
 export class CompletionClient extends DifyClient {
-  createCompletionMessage(inputs, query, user, stream = false) {
+  createCompletionMessage(inputs, user, stream = false, files = null) {
     const data = {
       inputs,
-      query,
       user,
       response_mode: stream ? "streaming" : "blocking",
+      files,
     };
     return this.sendRequest(
       routes.createCompletionMessage.method,
@@ -130,13 +130,15 @@ export class ChatClient extends DifyClient {
     query,
     user,
     stream = false,
-    conversation_id = null
+    conversation_id = null,
+    files = null
   ) {
     const data = {
       inputs,
       query,
       user,
       response_mode: stream ? "streaming" : "blocking",
+      files,
     };
     if (conversation_id) data.conversation_id = conversation_id;
 

sdks/nodejs-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dify-client",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "This is the Node.js SDK for the Dify.AI API, which allows you to easily integrate Dify.AI into your Node.js applications.",
   "main": "index.js",
   "type": "module",

sdks/php-client/README.md

@@ -26,11 +26,11 @@ $difyClient = new DifyClient($apiKey);
 
 // Create a completion client
 $completionClient = new CompletionClient($apiKey);
-$response = $completionClient->create_completion_message($inputs, $query, $response_mode, $user);
+$response = $completionClient->create_completion_message($inputs, $response_mode, $user, $files);
 
 // Create a chat client
 $chatClient = new ChatClient($apiKey);
-$response = $chatClient->create_chat_message($inputs, $query, $user, $response_mode, $conversation_id);
+$response = $chatClient->create_chat_message($inputs, $query, $user, $response_mode, $conversation_id, $files);
 
 // Fetch application parameters
 $response = $difyClient->get_application_parameters($user);

sdks/php-client/dify-client.php

@@ -47,24 +47,25 @@ public function get_application_parameters($user) {
 }
 
 class CompletionClient extends DifyClient {
-    public function create_completion_message($inputs, $query, $response_mode, $user) {
+    public function create_completion_message($inputs, $response_mode, $user, $files = null) {
         $data = [
             'inputs' => $inputs,
-            'query' => $query,
             'response_mode' => $response_mode,
             'user' => $user,
+            'files' => $files,
         ];
         return $this->send_request('POST', 'completion-messages', $data, null, $response_mode === 'streaming');
     }
 }
 
 class ChatClient extends DifyClient {
-    public function create_chat_message($inputs, $query, $user, $response_mode = 'blocking', $conversation_id = null) {
+    public function create_chat_message($inputs, $query, $user, $response_mode = 'blocking', $conversation_id = null, $files = null) {
         $data = [
             'inputs' => $inputs,
             'query' => $query,
             'user' => $user,
             'response_mode' => $response_mode,
+            'files' => $files,
         ];
         if ($conversation_id) {
             $data['conversation_id'] = $conversation_id;

sdks/python-client/README.md

@@ -14,28 +14,59 @@ Write your code with sdk:
 
 - completion generate with `blocking` response_mode
 
+```python
+from dify_client import CompletionClient
+
+api_key = "your_api_key"
+
+# Initialize CompletionClient
+completion_client = CompletionClient(api_key)
+
+# Create Completion Message using CompletionClient
+completion_response = completion_client.create_completion_message(inputs={"query": "What's the weather like today?"},
+                                                                  response_mode="blocking", user="user_id")
+completion_response.raise_for_status()
+
+result = completion_response.json()
+
+print(result.get('answer'))
 ```
-import json
+
+- completion using vision model, like gpt-4-vision
+
+```python
 from dify_client import CompletionClient
 
 api_key = "your_api_key"
 
 # Initialize CompletionClient
 completion_client = CompletionClient(api_key)
 
+files = [{
+    "type": "image",
+    "transfer_method": "remote_url",
+    "url": "your_image_url"
+}]
+
+# files = [{
+#     "type": "image",
+#     "transfer_method": "local_file",
+#     "upload_file_id": "your_file_id"
+# }]
+
 # Create Completion Message using CompletionClient
-completion_response = completion_client.create_completion_message(inputs={}, query="Hello", response_mode="blocking", user="user_id")
+completion_response = completion_client.create_completion_message(inputs={"query": "Describe the picture."},
+                                                                  response_mode="blocking", user="user_id", files=files)
 completion_response.raise_for_status()
 
-result = completion_response.text
-result = json.loads(result)
+result = completion_response.json()
 
 print(result.get('answer'))
 ```
 
 - chat generate with `streaming` response_mode
 
-```
+```python
 import json
 from dify_client import ChatClient
 
@@ -55,10 +86,67 @@ for line in chat_response.iter_lines(decode_unicode=True):
         print(line.get('answer'))
 ```
 
-- Others
+- chat using vision model, like gpt-4-vision
+
+```python
+from dify_client import ChatClient
 
+api_key = "your_api_key"
+
+# Initialize ChatClient
+chat_client = ChatClient(api_key)
+
+files = [{
+    "type": "image",
+    "transfer_method": "remote_url",
+    "url": "your_image_url"
+}]
+
+# files = [{
+#     "type": "image",
+#     "transfer_method": "local_file",
+#     "upload_file_id": "your_file_id"
+# }]
+
+# Create Chat Message using ChatClient
+chat_response = chat_client.create_chat_message(inputs={}, query="Describe the picture.", user="user_id",
+                                                response_mode="blocking", files=files)
+chat_response.raise_for_status()
+
+result = chat_response.json()
+
+print(result.get("answer"))
 ```
-import json
+
+- upload file when using vision model
+
+```python
+from dify_client import DifyClient
+
+api_key = "your_api_key"
+
+# Initialize Client
+dify_client = DifyClient(api_key)
+
+file_path = "your_image_file_path"
+file_name = "panda.jpeg"
+mime_type = "image/jpeg"
+
+with open(file_path, "rb") as file:
+    files = {
+        "file": (file_name, file, mime_type)
+    }
+    response = dify_client.file_upload("user_id", files)
+
+    result = response.json()
+    print(f'upload_file_id: {result.get("id")}')
+```
+
+
+
+- Others
+
+```python
 from dify_client import ChatClient
 
 api_key = "your_api_key"
@@ -69,32 +157,29 @@ client = ChatClient(api_key)
 # Get App parameters
 parameters = client.get_application_parameters(user="user_id")
 parameters.raise_for_status()
-parameters = json.loads(parameters.text)
 
 print('[parameters]')
-print(parameters)
+print(parameters.json())
 
 # Get Conversation List (only for chat)
 conversations = client.get_conversations(user="user_id")
 conversations.raise_for_status()
-conversations = json.loads(conversations.text)
 
 print('[conversations]')
-print(conversations)
+print(conversations.json())
 
 # Get Message List (only for chat)
 messages = client.get_conversation_messages(user="user_id", conversation_id="conversation_id")
 messages.raise_for_status()
-messages = json.loads(messages.text)
 
 print('[messages]')
-print(messages)
+print(messages.json())
 
 # Rename Conversation (only for chat)
-rename_conversation_response = client.rename_conversation(conversation_id="conversation_id", name="new_name", user="user_id")
+rename_conversation_response = client.rename_conversation(conversation_id="conversation_id",
+                                                          name="new_name", user="user_id")
 rename_conversation_response.raise_for_status()
-rename_conversation_result = json.loads(rename_conversation_response.text)
 
 print('[rename result]')
-print(rename_conversation_result)
+print(rename_conversation_response.json())
 ```

sdks/python-client/dify_client/__init__.py

@@ -1 +1 @@
-from dify_client.client import ChatClient, CompletionClient
+from dify_client.client import ChatClient, CompletionClient, DifyClient

sdks/python-client/dify_client/client.py

@@ -6,14 +6,24 @@ def __init__(self, api_key):
         self.api_key = api_key
         self.base_url = "https://api.dify.ai/v1"
 
-    def _send_request(self, method, endpoint, data=None, params=None, stream=False):
+    def _send_request(self, method, endpoint, json=None, params=None, stream=False):
         headers = {
             "Authorization": f"Bearer {self.api_key}",
             "Content-Type": "application/json"
         }
 
         url = f"{self.base_url}{endpoint}"
-        response = requests.request(method, url, json=data, params=params, headers=headers, stream=stream)
+        response = requests.request(method, url, json=json, params=params, headers=headers, stream=stream)
+
+        return response
+
+    def _send_request_with_files(self, method, endpoint, data, files):
+        headers = {
+            "Authorization": f"Bearer {self.api_key}"
+        }
+
+        url = f"{self.base_url}{endpoint}"
+        response = requests.request(method, url, data=data, headers=headers, files=files)
 
         return response
 
@@ -28,30 +38,39 @@ def get_application_parameters(self, user):
         params = {"user": user}
         return self._send_request("GET", "/parameters", params=params)
 
+    def file_upload(self, user, files):
+        data = {
+            "user": user
+        }
+        return self._send_request_with_files("POST", "/files/upload", data=data, files=files)
+
 
 class CompletionClient(DifyClient):
-    def create_completion_message(self, inputs, query, response_mode, user):
+    def create_completion_message(self, inputs, response_mode, user, files=None):
         data = {
             "inputs": inputs,
-            "query": query,
             "response_mode": response_mode,
-            "user": user
+            "user": user,
+            "files": files
         }
-        return self._send_request("POST", "/completion-messages", data, stream=True if response_mode == "streaming" else False)
+        return self._send_request("POST", "/completion-messages", data,
+                                  stream=True if response_mode == "streaming" else False)
 
 
 class ChatClient(DifyClient):
-    def create_chat_message(self, inputs, query, user, response_mode="blocking", conversation_id=None):
+    def create_chat_message(self, inputs, query, user, response_mode="blocking", conversation_id=None, files=None):
         data = {
             "inputs": inputs,
             "query": query,
             "user": user,
-            "response_mode": response_mode
+            "response_mode": response_mode,
+            "files": files
         }
         if conversation_id:
             data["conversation_id"] = conversation_id
 
-        return self._send_request("POST", "/chat-messages", data, stream=True if response_mode == "streaming" else False)
+        return self._send_request("POST", "/chat-messages", data,
+                                  stream=True if response_mode == "streaming" else False)
 
     def get_conversation_messages(self, user, conversation_id=None, first_id=None, limit=None):
         params = {"user": user}

sdks/python-client/setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name="dify-client",
-    version="0.1.8",
+    version="0.1.10",
     author="Dify",
     author_email="[email protected]",
     description="A package for interacting with the Dify Service-API",