Add embedding input functionality for disabled modalities

example_embedding_only_curl.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""
+Helper script to generate a curl command for testing embedding-only mode.
+
+This script creates an example embedding and outputs a curl command that you can run.
+"""
+
+import json
+import torch
+from vllm.utils.serial_utils import tensor2base64
+
+def create_example_image_embedding(
+    num_images: int = 1,
+    feature_size: int = 256,
+    hidden_size: int = 4096,
+    dtype: torch.dtype = torch.float16
+) -> torch.Tensor:
+    """Create a dummy image embedding tensor."""
+    embedding = torch.randn(
+        num_images, 
+        feature_size, 
+        hidden_size,
+        dtype=dtype
+    )
+    return embedding
+
+def main():
+    # Create example embedding
+    image_embeds = create_example_image_embedding(
+        num_images=1,
+        feature_size=256,
+        hidden_size=4096,
+        dtype=torch.float16
+    )
+
+    # Convert to base64
+    base64_image_embedding = tensor2base64(image_embeds)
+
+    # Create the request payload
+    payload = {
+        "model": "Qwen/Qwen2.5-VL-3B-Instruct",
+        "messages": [
+            {
+                "role": "system",
+                "content": "You are a helpful assistant."
+            },
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "What do you see in this image? Describe it briefly."
+                    },
+                    {
+                        "type": "image_embeds",
+                        "image_embeds": base64_image_embedding,
+                    },
+                ],
+            },
+        ],
+        "max_tokens": 100,
+        "temperature": 0.7,
+    }
+
+    # Print curl command
+    print("="*70)
+    print("CURL COMMAND:")
+    print("="*70)
+    print("\ncurl -X POST http://localhost:8000/v1/chat/completions \\")
+    print("  -H 'Content-Type: application/json' \\")
+    print("  -d '{}'".format(json.dumps(payload, indent=2)))
+    print("\n" + "="*70)
+    print("\nNote: The embedding is included in the JSON payload above.")
+    print("Embedding shape:", image_embeds.shape)
+    print("="*70)
+
+if __name__ == "__main__":
+    main()
+

example_embedding_only_serve.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""
+Example: Running vLLM in embedding-only mode (text mode) with embedding inputs
+
+This example demonstrates:
+1. How to start the server with embedding-only mode for memory savings
+2. How to create and send embedding inputs via the OpenAI API
+"""
+
+import json
+import torch
+from openai import OpenAI
+from vllm.utils.serial_utils import tensor2base64
+
+# ============================================================================
+# STEP 1: Start the server with this command:
+# ============================================================================
+"""
+vllm serve Qwen/Qwen2.5-VL-7B-Instruct \
+    --enable-mm-embeds \
+    --limit-mm-per-prompt '{"image": 0, "video": 0}' \
+    --port 8000
+"""
+
+# ============================================================================
+# STEP 2: Create a client and example embedding
+# ============================================================================
+
+def create_example_image_embedding(
+    num_images: int = 1,
+    feature_size: int = 256,  # Typical image feature size
+    hidden_size: int = 4096,   # Hidden size for Qwen2.5-VL models
+    dtype: torch.dtype = torch.float16
+) -> torch.Tensor:
+    """
+    Create a dummy image embedding tensor.
+
+    Shape: (num_images, feature_size, hidden_size)
+    For Qwen2.5-VL models: (1, 256, 4096)
+    """
+    # Create random embeddings (in practice, these would come from your encoder)
+    embedding = torch.randn(
+        num_images, 
+        feature_size, 
+        hidden_size,
+        dtype=dtype
+    )
+    return embedding
+
+
+def main():
+    # Initialize OpenAI client
+    client = OpenAI(
+        api_key="EMPTY",  # Not needed for local server
+        base_url="http://localhost:8000/v1",
+    )
+
+    # Get the model name from the server (must match the model the server was started with)
+    try:
+        models = client.models.list()
+        if models.data:
+            model_name = models.data[0].id
+            print(f"Using model from server: {model_name}")
+        else:
+            raise ValueError("No models found on server")
+    except Exception as e:
+        print(f"Warning: Could not get model from server: {e}")
+        print("Using default model name. Make sure it matches your server.")
+        model_name = "Qwen/Qwen2.5-VL-7B-Instruct"  # Fallback: must match server
+
+    # Create example image embedding
+    # Shape: (num_images, feature_size, hidden_size)
+    # For Qwen2.5-VL models, typical shape is (1, 256, 4096)
+    print("Creating example image embedding...")
+    image_embeds = create_example_image_embedding(
+        num_images=1,
+        feature_size=256,
+        hidden_size=4096,
+        dtype=torch.float16
+    )
+    print(f"Embedding shape: {image_embeds.shape}")
+
+    # Convert to base64 for API transmission
+    base64_image_embedding = tensor2base64(image_embeds)
+    print("Embedding serialized to base64")
+
+    # Send request with embedding input
+    print("\nSending chat completion request...")
+    chat_completion = client.chat.completions.create(
+        model=model_name,
+        messages=[
+            {
+                "role": "system",
+                "content": "You are a helpful assistant."
+            },
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "What do you see in this image? Describe it briefly."
+                    },
+                    {
+                        "type": "image_embeds",
+                        "image_embeds": base64_image_embedding,
+                    },
+                ],
+            },
+        ],
+        max_tokens=100,
+        temperature=0.7,
+    )
+
+    print("\n" + "="*60)
+    print("Response:")
+    print("="*60)
+    print(chat_completion.choices[0].message.content)
+    print("="*60)
+
+
+if __name__ == "__main__":
+    main()
+

test_embedding_only_fix.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""
+Combined test script: Starts server and runs embedding-only test
+This allows agentic access to test the fix in one command
+"""
+
+import subprocess
+import time
+import sys
+import signal
+import os
+import threading
+from pathlib import Path
+
+try:
+    import requests
+except ImportError:
+    print("Installing requests...")
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "requests", "-q"])
+    import requests
+
+# Import the test client
+from example_embedding_only_serve import main as run_client_test
+from openai import OpenAI
+
+MODEL = "Qwen/Qwen2.5-VL-7B-Instruct"
+PORT = 8000
+SERVER_URL = f"http://localhost:{PORT}"
+
+def check_server_ready(max_wait=300, process=None):
+    """Check if server is ready by polling /health endpoint"""
+    print(f"Waiting up to {max_wait} seconds for server to be ready...")
+    for i in range(max_wait):
+        # Check if process is still alive
+        if process and process.poll() is not None:
+            print(f"\n✗ Server process died unexpectedly (exit code: {process.returncode})")
+            # Try to read any output
+            try:
+                output = process.stdout.read()
+                if output:
+                    print("Server output:")
+                    print(output[-2000:])  # Last 2000 chars
+            except:
+                pass
+            return False
+
+        try:
+            response = requests.get(f"{SERVER_URL}/health", timeout=2)
+            if response.status_code == 200:
+                print(f"✓ Server is ready (took {i+1}s)")
+                return True
+        except:
+            pass
+
+        # Show progress every 10 seconds
+        if (i + 1) % 10 == 0:
+            print(f"  Still waiting... ({i+1}s/{max_wait}s)")
+
+        time.sleep(1)
+
+    print(f"\n✗ Server failed to start within {max_wait} seconds")
+    if process:
+        # Try to read any output
+        try:
+            output = process.stdout.read()
+            if output:
+                print("\nLast server output:")
+                print(output[-2000:])  # Last 2000 chars
+        except:
+            pass
+    return False
+
+def start_server():
+    """Start vLLM server in background"""
+    print("Starting vLLM server...")
+    print(f"Model: {MODEL}")
+    print(f"Port: {PORT}")
+
+    # Use existing environment (including default HuggingFace cache)
+    import os
+    env = os.environ.copy()
+
+    cmd = [
+        sys.executable, "-m", "vllm.entrypoints.openai.api_server",
+        "--model", MODEL,
+        "--enable-mm-embeds",
+        "--limit-mm-per-prompt", '{"image": 0, "video": 0}',
+        "--port", str(PORT),
+        "--max-model-len", "2048",  # Smaller for faster startup
+    ]
+
+    # Start server process - write to both file and capture
+    log_file = open("/tmp/vllm_server.log", "w")
+    process = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+        env=env
+    )
+
+    print(f"Server process started (PID: {process.pid})")
+    print("Server logs are being written to /tmp/vllm_server.log")
+    print("You can monitor progress with: tail -f /tmp/vllm_server.log")
+
+    # Start a thread to copy output to log file
+    def log_output():
+        try:
+            for line in process.stdout:
+                log_file.write(line)
+                log_file.flush()
+                # Also print important lines
+                if any(keyword in line.lower() for keyword in ["error", "exception", "ready", "uvicorn", "started"]):
+                    print(f"  [SERVER] {line.rstrip()}")
+        except:
+            pass
+
+    log_thread = threading.Thread(target=log_output, daemon=True)
+    log_thread.start()
+
+    if not check_server_ready(max_wait=300, process=process):
+        print("✗ Server failed to start within timeout")
+        log_file.close()
+        process.terminate()
+        return None
+
+    log_file.close()
+    return process
+
+def run_test():
+    """Run the client test"""
+    print("\n" + "="*60)
+    print("Running client test...")
+    print("="*60 + "\n")
+
+    try:
+        run_client_test()
+        print("\n✓ Test completed successfully!")
+        return True
+    except Exception as e:
+        print(f"\n✗ Test failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+def main():
+    """Main test runner"""
+    server_process = None
+
+    try:
+        # Start server
+        server_process = start_server()
+        if not server_process:
+            print("Failed to start server")
+            return 1
+
+        # Run test
+        success = run_test()
+
+        return 0 if success else 1
+
+    except KeyboardInterrupt:
+        print("\n\nInterrupted by user")
+        return 130
+    finally:
+        # Cleanup: kill server
+        if server_process:
+            print("\nShutting down server...")
+            try:
+                server_process.terminate()
+                server_process.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                print("Force killing server...")
+                server_process.kill()
+                server_process.wait()
+            print("Server stopped")
+
+if __name__ == "__main__":
+    sys.exit(main())
+