From 7016dd650634940dff91946bf458efd90b2d37fe Mon Sep 17 00:00:00 2001 From: zxy Date: Fri, 26 Jun 2026 14:05:48 +0800 Subject: [PATCH] refactor: rename vl package to multimodal --- docs/en/faq.md | 2 +- docs/en/get_started/ascend/get_started.md | 2 +- docs/en/get_started/camb/get_started.md | 2 +- docs/en/get_started/get_started.md | 4 +-- docs/en/get_started/maca/get_started.md | 2 +- docs/en/multi_modal/cogvlm.md | 2 +- docs/en/multi_modal/deepseek_vl2.md | 2 +- docs/en/multi_modal/gemma3.md | 2 +- docs/en/multi_modal/index.rst | 5 +++- docs/en/multi_modal/internvl.md | 10 +++---- docs/en/multi_modal/llava.md | 2 +- docs/en/multi_modal/minicpmv.md | 4 +-- docs/en/multi_modal/molmo.md | 2 +- docs/en/multi_modal/multimodal_inputs.md | 10 +++---- docs/en/multi_modal/phi3.md | 2 +- docs/en/multi_modal/qwen2_5_vl.md | 6 ++--- docs/en/multi_modal/qwen2_vl.md | 2 +- docs/en/multi_modal/vl_pipeline.md | 26 +++++++++---------- docs/zh_cn/faq.md | 2 +- docs/zh_cn/get_started/ascend/get_started.md | 2 +- docs/zh_cn/get_started/camb/get_started.md | 2 +- docs/zh_cn/get_started/get_started.md | 4 +-- docs/zh_cn/get_started/maca/get_started.md | 2 +- docs/zh_cn/multi_modal/cogvlm.md | 2 +- docs/zh_cn/multi_modal/deepseek_vl2.md | 2 +- docs/zh_cn/multi_modal/gemma3.md | 2 +- docs/zh_cn/multi_modal/index.rst | 4 ++- docs/zh_cn/multi_modal/internvl.md | 10 +++---- docs/zh_cn/multi_modal/llava.md | 2 +- docs/zh_cn/multi_modal/minicpmv.md | 4 +-- docs/zh_cn/multi_modal/molmo.md | 2 +- docs/zh_cn/multi_modal/multimodal_inputs.md | 10 +++---- docs/zh_cn/multi_modal/phi3.md | 2 +- docs/zh_cn/multi_modal/qwen2_5_vl.md | 6 ++--- docs/zh_cn/multi_modal/qwen2_vl.md | 2 +- docs/zh_cn/multi_modal/vl_pipeline.md | 26 +++++++++---------- lmdeploy/api.py | 2 +- lmdeploy/lite/apis/calibrate.py | 2 +- lmdeploy/multimodal/__init__.py | 22 ++++++++++++++++ lmdeploy/{vl => multimodal}/constants.py | 0 lmdeploy/{vl => multimodal}/engine.py | 2 +- lmdeploy/{vl => multimodal}/media/__init__.py | 0 lmdeploy/{vl => multimodal}/media/audio.py | 0 lmdeploy/{vl => multimodal}/media/base.py | 0 .../{vl => multimodal}/media/connection.py | 0 lmdeploy/{vl => multimodal}/media/image.py | 0 .../{vl => multimodal}/media/time_series.py | 0 lmdeploy/{vl => multimodal}/media/video.py | 0 .../{vl => multimodal}/media/video_loader.py | 0 lmdeploy/{vl => multimodal}/model/__init__.py | 0 lmdeploy/{vl => multimodal}/model/base.py | 4 +-- lmdeploy/{vl => multimodal}/model/builder.py | 2 +- lmdeploy/{vl => multimodal}/model/cogvlm.py | 2 +- lmdeploy/{vl => multimodal}/model/deepseek.py | 4 +-- .../{vl => multimodal}/model/deepseek_vl2.py | 2 +- .../{vl => multimodal}/model/gemma3_vl.py | 2 +- lmdeploy/{vl => multimodal}/model/glm4_1v.py | 2 +- lmdeploy/{vl => multimodal}/model/glm4_v.py | 2 +- .../{vl => multimodal}/model/interns1_pro.py | 4 +-- lmdeploy/{vl => multimodal}/model/internvl.py | 4 +-- .../{vl => multimodal}/model/internvl3_hf.py | 4 +-- lmdeploy/{vl => multimodal}/model/llama4.py | 2 +- lmdeploy/{vl => multimodal}/model/llava.py | 4 +-- lmdeploy/{vl => multimodal}/model/llava_hf.py | 4 +-- .../{vl => multimodal}/model/llava_next.py | 4 +-- lmdeploy/{vl => multimodal}/model/minicpmv.py | 4 +-- lmdeploy/{vl => multimodal}/model/molmo.py | 4 +-- .../{vl => multimodal}/model/phi3_vision.py | 2 +- .../model/preprocess_utils.py | 4 +-- lmdeploy/{vl => multimodal}/model/qwen2.py | 4 +-- lmdeploy/{vl => multimodal}/model/qwen3.py | 2 +- lmdeploy/{vl => multimodal}/model/qwen3_5.py | 6 ++--- .../{vl => multimodal}/model/qwen3_omni.py | 2 +- lmdeploy/{vl => multimodal}/model/utils.py | 0 lmdeploy/{vl => multimodal}/tools/__init__.py | 0 lmdeploy/{vl => multimodal}/utils.py | 0 lmdeploy/pytorch/messages.py | 2 +- lmdeploy/pytorch/models/interns1_pro.py | 2 +- lmdeploy/pytorch/models/qwen3_5.py | 2 +- .../pytorch/models/qwen3_omni_moe_thinker.py | 2 +- lmdeploy/pytorch/models/qwen3_vl.py | 2 +- lmdeploy/pytorch/models/utils/model.py | 2 +- lmdeploy/pytorch/multimodal/data_type.py | 2 +- lmdeploy/serve/core/vl_async_engine.py | 2 +- lmdeploy/serve/openai/api_server.py | 2 +- lmdeploy/serve/processors/multimodal.py | 14 +++++----- lmdeploy/turbomind/models/qwen3_5.py | 2 +- lmdeploy/vl/__init__.py | 23 ++-------------- tests/pytorch/paging/test_block_trie.py | 2 +- tests/test_lmdeploy/test_content_merge.py | 2 +- .../test_hf_chat_template.py | 2 +- .../test_multimodal_encode.py} | 2 +- .../test_nonhf_chat_template.py | 2 +- .../test_preprocess_utils.py | 4 +-- .../test_qwen3_omni_processor.py | 8 +++--- .../test_qwen3vl_processor.py | 6 ++--- .../test_safe_url.py | 4 +-- 97 files changed, 186 insertions(+), 178 deletions(-) create mode 100644 lmdeploy/multimodal/__init__.py rename lmdeploy/{vl => multimodal}/constants.py (100%) rename lmdeploy/{vl => multimodal}/engine.py (99%) rename lmdeploy/{vl => multimodal}/media/__init__.py (100%) rename lmdeploy/{vl => multimodal}/media/audio.py (100%) rename lmdeploy/{vl => multimodal}/media/base.py (100%) rename lmdeploy/{vl => multimodal}/media/connection.py (100%) rename lmdeploy/{vl => multimodal}/media/image.py (100%) rename lmdeploy/{vl => multimodal}/media/time_series.py (100%) rename lmdeploy/{vl => multimodal}/media/video.py (100%) rename lmdeploy/{vl => multimodal}/media/video_loader.py (100%) rename lmdeploy/{vl => multimodal}/model/__init__.py (100%) rename lmdeploy/{vl => multimodal}/model/base.py (99%) rename lmdeploy/{vl => multimodal}/model/builder.py (98%) rename lmdeploy/{vl => multimodal}/model/cogvlm.py (98%) rename lmdeploy/{vl => multimodal}/model/deepseek.py (98%) rename lmdeploy/{vl => multimodal}/model/deepseek_vl2.py (99%) rename lmdeploy/{vl => multimodal}/model/gemma3_vl.py (98%) rename lmdeploy/{vl => multimodal}/model/glm4_1v.py (92%) rename lmdeploy/{vl => multimodal}/model/glm4_v.py (98%) rename lmdeploy/{vl => multimodal}/model/interns1_pro.py (96%) rename lmdeploy/{vl => multimodal}/model/internvl.py (99%) rename lmdeploy/{vl => multimodal}/model/internvl3_hf.py (97%) rename lmdeploy/{vl => multimodal}/model/llama4.py (99%) rename lmdeploy/{vl => multimodal}/model/llava.py (99%) rename lmdeploy/{vl => multimodal}/model/llava_hf.py (97%) rename lmdeploy/{vl => multimodal}/model/llava_next.py (98%) rename lmdeploy/{vl => multimodal}/model/minicpmv.py (98%) rename lmdeploy/{vl => multimodal}/model/molmo.py (98%) rename lmdeploy/{vl => multimodal}/model/phi3_vision.py (95%) rename lmdeploy/{vl => multimodal}/model/preprocess_utils.py (98%) rename lmdeploy/{vl => multimodal}/model/qwen2.py (98%) rename lmdeploy/{vl => multimodal}/model/qwen3.py (94%) rename lmdeploy/{vl => multimodal}/model/qwen3_5.py (97%) rename lmdeploy/{vl => multimodal}/model/qwen3_omni.py (94%) rename lmdeploy/{vl => multimodal}/model/utils.py (100%) rename lmdeploy/{vl => multimodal}/tools/__init__.py (100%) rename lmdeploy/{vl => multimodal}/utils.py (100%) rename tests/test_lmdeploy/{test_vl => test_multimodal}/test_hf_chat_template.py (99%) rename tests/test_lmdeploy/{test_vl/test_vl_encode.py => test_multimodal/test_multimodal_encode.py} (99%) rename tests/test_lmdeploy/{test_vl => test_multimodal}/test_nonhf_chat_template.py (99%) rename tests/test_lmdeploy/{test_vl => test_multimodal}/test_preprocess_utils.py (95%) rename tests/test_lmdeploy/{test_vl => test_multimodal}/test_qwen3_omni_processor.py (97%) rename tests/test_lmdeploy/{test_vl => test_multimodal}/test_qwen3vl_processor.py (97%) rename tests/test_lmdeploy/{test_vl => test_multimodal}/test_safe_url.py (92%) diff --git a/docs/en/faq.md b/docs/en/faq.md index 39563d5d01..345f508ce8 100644 --- a/docs/en/faq.md +++ b/docs/en/faq.md @@ -94,7 +94,7 @@ lmdeploy serve api_server internlm/internlm2_5-7b-chat --cache-max-entry-count 0 ### Api Server Fetch Timeout The image URL fetch timeout for the API server can be configured via the environment variable `LMDEPLOY_FETCH_TIMEOUT`. -By default, requests may take up to 10 seconds before timing out. See [lmdeploy/vl/utils.py](https://github.com/InternLM/lmdeploy/blob/7b6876eafcb842633e0efe8baabe5906d7beeeea/lmdeploy/vl/utils.py#L31) for usage. +By default, requests may take up to 10 seconds before timing out. See [lmdeploy/multimodal/utils.py](https://github.com/InternLM/lmdeploy/blob/main/lmdeploy/multimodal/utils.py) for usage. ## Quantization diff --git a/docs/en/get_started/ascend/get_started.md b/docs/en/get_started/ascend/get_started.md index 376548b0dc..e88e02e865 100644 --- a/docs/en/get_started/ascend/get_started.md +++ b/docs/en/get_started/ascend/get_started.md @@ -50,7 +50,7 @@ Set `device_type="ascend"` in the `PytorchEngineConfig`: ```python from lmdeploy import pipeline, PytorchEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-2B', backend_config=PytorchEngineConfig(tp=1, device_type='ascend')) image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') diff --git a/docs/en/get_started/camb/get_started.md b/docs/en/get_started/camb/get_started.md index 5b6e622667..0c1c35d302 100644 --- a/docs/en/get_started/camb/get_started.md +++ b/docs/en/get_started/camb/get_started.md @@ -43,7 +43,7 @@ Set `device_type="camb"` in the `PytorchEngineConfig`: ```python from lmdeploy import pipeline, PytorchEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-2B', backend_config=PytorchEngineConfig(tp=1, device_type='camb')) image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') diff --git a/docs/en/get_started/get_started.md b/docs/en/get_started/get_started.md index 8650858d12..b588f3a6b5 100644 --- a/docs/en/get_started/get_started.md +++ b/docs/en/get_started/get_started.md @@ -83,7 +83,7 @@ For example, you can utilize the following code snippet to perform the inference ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-8B') @@ -96,7 +96,7 @@ In VLM pipeline, the default image processing batch size is 1. This can be adjus ```python from lmdeploy import pipeline, VisionConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-8B', vision_config=VisionConfig( diff --git a/docs/en/get_started/maca/get_started.md b/docs/en/get_started/maca/get_started.md index 5c647a379e..47b622de7a 100644 --- a/docs/en/get_started/maca/get_started.md +++ b/docs/en/get_started/maca/get_started.md @@ -33,7 +33,7 @@ Set `device_type="maca"` in the `PytorchEngineConfig`: ```python from lmdeploy import pipeline, PytorchEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-2B', backend_config=PytorchEngineConfig(tp=1, device_type='maca')) image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') diff --git a/docs/en/multi_modal/cogvlm.md b/docs/en/multi_modal/cogvlm.md index 0a4e7849c7..a02cc62ba5 100644 --- a/docs/en/multi_modal/cogvlm.md +++ b/docs/en/multi_modal/cogvlm.md @@ -26,7 +26,7 @@ The following sample code shows the basic usage of VLM pipeline. For more exampl ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image if __name__ == "__main__": diff --git a/docs/en/multi_modal/deepseek_vl2.md b/docs/en/multi_modal/deepseek_vl2.md index c0c2129bea..71a592c889 100644 --- a/docs/en/multi_modal/deepseek_vl2.md +++ b/docs/en/multi_modal/deepseek_vl2.md @@ -30,7 +30,7 @@ To construct valid DeepSeek-VL2 prompts with image inputs, users should insert ` ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image if __name__ == "__main__": diff --git a/docs/en/multi_modal/gemma3.md b/docs/en/multi_modal/gemma3.md index 20905bfe97..5e644b6ea0 100644 --- a/docs/en/multi_modal/gemma3.md +++ b/docs/en/multi_modal/gemma3.md @@ -18,7 +18,7 @@ The following sample code shows the basic usage of VLM pipeline. For more exampl ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image if __name__ == "__main__": diff --git a/docs/en/multi_modal/index.rst b/docs/en/multi_modal/index.rst index 92356eec16..db3df4bf3e 100644 --- a/docs/en/multi_modal/index.rst +++ b/docs/en/multi_modal/index.rst @@ -1,6 +1,9 @@ -Vision-Language Models +Multimodal Models ================================= +Use ``lmdeploy.multimodal`` for multimodal helper APIs such as media loading +and local-file encoding. + .. toctree:: :maxdepth: 2 :caption: Examples diff --git a/docs/en/multi_modal/internvl.md b/docs/en/multi_modal/internvl.md index 896db2e5d9..8ffed56616 100644 --- a/docs/en/multi_modal/internvl.md +++ b/docs/en/multi_modal/internvl.md @@ -42,7 +42,7 @@ The following sample code shows the basic usage of VLM pipeline. For detailed in ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-8B') @@ -60,7 +60,7 @@ More examples are listed below: ```python from lmdeploy import pipeline, GenerationConfig -from lmdeploy.vl.constants import IMAGE_TOKEN +from lmdeploy.multimodal.constants import IMAGE_TOKEN pipe = pipeline('OpenGVLab/InternVL2-8B', log_level='INFO') messages = [ @@ -86,7 +86,7 @@ out = pipe(messages, gen_config=GenerationConfig(top_k=1)) ```python from lmdeploy import pipeline, GenerationConfig -from lmdeploy.vl.constants import IMAGE_TOKEN +from lmdeploy.multimodal.constants import IMAGE_TOKEN pipe = pipeline('OpenGVLab/InternVL2-8B', log_level='INFO') messages = [ @@ -114,8 +114,8 @@ out = pipe(messages, gen_config=GenerationConfig(top_k=1)) import numpy as np from lmdeploy import pipeline, GenerationConfig from decord import VideoReader, cpu -from lmdeploy.vl.constants import IMAGE_TOKEN -from lmdeploy.vl import encode_image_base64 +from lmdeploy.multimodal.constants import IMAGE_TOKEN +from lmdeploy.multimodal import encode_image_base64 from PIL import Image pipe = pipeline('OpenGVLab/InternVL2-8B', log_level='INFO') diff --git a/docs/en/multi_modal/llava.md b/docs/en/multi_modal/llava.md index c374b67121..d63d22a912 100644 --- a/docs/en/multi_modal/llava.md +++ b/docs/en/multi_modal/llava.md @@ -33,7 +33,7 @@ The following sample code shows the basic usage of VLM pipeline. For detailed in ```python from lmdeploy import GenerationConfig, TurbomindEngineConfig, pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline("llava-hf/llava-interleave-qwen-7b-hf", backend_config=TurbomindEngineConfig(cache_max_entry_count=0.5), diff --git a/docs/en/multi_modal/minicpmv.md b/docs/en/multi_modal/minicpmv.md index 0f2bf176b9..f7da7238e8 100644 --- a/docs/en/multi_modal/minicpmv.md +++ b/docs/en/multi_modal/minicpmv.md @@ -19,7 +19,7 @@ The following sample code shows the basic usage of VLM pipeline. For detailed in ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('openbmb/MiniCPM-V-2_6') @@ -97,7 +97,7 @@ print(out.text) ```python from lmdeploy import pipeline, GenerationConfig -from lmdeploy.vl import encode_image_base64 +from lmdeploy.multimodal import encode_image_base64 import torch from PIL import Image from transformers import AutoModel, AutoTokenizer diff --git a/docs/en/multi_modal/molmo.md b/docs/en/multi_modal/molmo.md index dfff43dc64..8c98f38098 100644 --- a/docs/en/multi_modal/molmo.md +++ b/docs/en/multi_modal/molmo.md @@ -19,7 +19,7 @@ The following sample code shows the basic usage of VLM pipeline. For detailed in ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('allenai/Molmo-7B-D-0924') diff --git a/docs/en/multi_modal/multimodal_inputs.md b/docs/en/multi_modal/multimodal_inputs.md index 4f78ad2504..2b14addb86 100644 --- a/docs/en/multi_modal/multimodal_inputs.md +++ b/docs/en/multi_modal/multimodal_inputs.md @@ -398,7 +398,7 @@ In addition to HTTP URLs, lmdeploy accepts: - **Local file paths** via `file://` scheme: `file:///absolute/path/to/file.jpg` - **Base64-encoded data** via data URLs: `data:;base64,` -Use the helpers in `lmdeploy.vl.utils` to encode local files: +Use the helpers in `lmdeploy.multimodal.utils` to encode local files:
Local file path example @@ -434,7 +434,7 @@ print(response.choices[0].message.content) ```python from openai import OpenAI -from lmdeploy.vl.utils import encode_image_base64 +from lmdeploy.multimodal.utils import encode_image_base64 client = OpenAI(api_key='EMPTY', base_url='http://localhost:23333/v1') model_name = client.models.list().data[0].id @@ -465,7 +465,7 @@ print(response.choices[0].message.content) ```python from openai import OpenAI -from lmdeploy.vl.utils import encode_video_base64 +from lmdeploy.multimodal.utils import encode_video_base64 client = OpenAI(api_key='EMPTY', base_url='http://localhost:23333/v1') model_name = client.models.list().data[0].id @@ -497,7 +497,7 @@ print(response.choices[0].message.content) ```python from openai import OpenAI -from lmdeploy.vl.utils import encode_audio_base64 +from lmdeploy.multimodal.utils import encode_audio_base64 client = OpenAI(api_key='EMPTY', base_url='http://localhost:23333/v1') model_name = client.models.list().data[0].id @@ -528,7 +528,7 @@ print(response.choices[0].message.content) ```python from openai import OpenAI -from lmdeploy.vl.utils import encode_time_series_base64 +from lmdeploy.multimodal.utils import encode_time_series_base64 client = OpenAI(api_key='EMPTY', base_url='http://localhost:23333/v1') model_name = client.models.list().data[0].id diff --git a/docs/en/multi_modal/phi3.md b/docs/en/multi_modal/phi3.md index a7ad0237e2..8767b88254 100644 --- a/docs/en/multi_modal/phi3.md +++ b/docs/en/multi_modal/phi3.md @@ -26,7 +26,7 @@ The following sample code shows the basic usage of VLM pipeline. For more exampl ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('microsoft/Phi-3.5-vision-instruct') diff --git a/docs/en/multi_modal/qwen2_5_vl.md b/docs/en/multi_modal/qwen2_5_vl.md index ac2ffa2ce6..d42c9d9552 100644 --- a/docs/en/multi_modal/qwen2_5_vl.md +++ b/docs/en/multi_modal/qwen2_5_vl.md @@ -25,7 +25,7 @@ The following sample code shows the basic usage of the VLM pipeline. For detaile ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('Qwen/Qwen2.5-VL-7B-Instruct') @@ -98,8 +98,8 @@ out = pipe(messages, gen_config=GenerationConfig(top_k=1)) import numpy as np from lmdeploy import pipeline, GenerationConfig from decord import VideoReader, cpu -from lmdeploy.vl.constants import IMAGE_TOKEN -from lmdeploy.vl import encode_image_base64 +from lmdeploy.multimodal.constants import IMAGE_TOKEN +from lmdeploy.multimodal import encode_image_base64 from PIL import Image pipe = pipeline('Qwen/Qwen2.5-VL-7B-Instruct', log_level='INFO') diff --git a/docs/en/multi_modal/qwen2_vl.md b/docs/en/multi_modal/qwen2_vl.md index 425d5d8f28..6a4d78ad0e 100644 --- a/docs/en/multi_modal/qwen2_vl.md +++ b/docs/en/multi_modal/qwen2_vl.md @@ -36,7 +36,7 @@ The following sample code shows the basic usage of VLM pipeline. For detailed in ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('Qwen/Qwen2-VL-2B-Instruct') diff --git a/docs/en/multi_modal/vl_pipeline.md b/docs/en/multi_modal/vl_pipeline.md index 4972ba91d5..6b3f506255 100644 --- a/docs/en/multi_modal/vl_pipeline.md +++ b/docs/en/multi_modal/vl_pipeline.md @@ -16,7 +16,7 @@ Using the pipeline interface to infer other VLM models is similar, with the main ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B') @@ -53,7 +53,7 @@ Tensor paramllelism can be activated by setting the engine parameter `tp` ```python from lmdeploy import pipeline, TurbomindEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(tp=2)) @@ -69,7 +69,7 @@ When creating the pipeline, you can customize the size of the context window by ```python from lmdeploy import pipeline, TurbomindEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(session_len=8192)) @@ -85,7 +85,7 @@ You can change the default sampling parameters of pipeline by passing `Generatio ```python from lmdeploy import pipeline, GenerationConfig, TurbomindEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(tp=2, session_len=8192)) @@ -101,8 +101,8 @@ By default, LMDeploy inserts the special image token into the user prompt follow ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image -from lmdeploy.vl.constants import IMAGE_TOKEN +from lmdeploy.multimodal import load_image +from lmdeploy.multimodal.constants import IMAGE_TOKEN pipe = pipeline('deepseek-ai/deepseek-vl-1.3b-chat') @@ -117,7 +117,7 @@ While performing inference, LMDeploy identifies an appropriate chat template fro ```python from lmdeploy import pipeline, ChatTemplateConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('local_model_folder', chat_template_config=ChatTemplateConfig(model_name='llava-v1')) image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') @@ -133,7 +133,7 @@ The default parameters of the visual model can be modified by setting `VisionCon ```python from lmdeploy import pipeline, VisionConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image vision_config=VisionConfig(max_batch_size=16) pipe = pipeline('liuhaotian/llava-v1.5-7b', vision_config=vision_config) image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') @@ -145,7 +145,7 @@ print(response) ```python from lmdeploy import pipeline, GenerationConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B') image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') @@ -162,7 +162,7 @@ When dealing with multiple images, you can put them all in one list. Keep in min ```python from lmdeploy import pipeline, TurbomindEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(session_len=8192)) @@ -183,7 +183,7 @@ Conducting inference with batch prompts is quite straightforward; just place the ```python from lmdeploy import pipeline, TurbomindEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(session_len=8192)) @@ -203,7 +203,7 @@ There are two ways to do the multi-turn conversations with the pipeline. One is ```python from lmdeploy import pipeline, TurbomindEngineConfig, GenerationConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(session_len=8192)) @@ -224,7 +224,7 @@ You can release the pipeline explicitly by calling its `close()` method, or alte from lmdeploy import pipeline from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image with pipeline('OpenGVLab/InternVL2_5-8B') as pipe: image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') diff --git a/docs/zh_cn/faq.md b/docs/zh_cn/faq.md index 1822d079e6..9853729350 100644 --- a/docs/zh_cn/faq.md +++ b/docs/zh_cn/faq.md @@ -94,7 +94,7 @@ lmdeploy serve api_server internlm/internlm2_5-7b-chat --cache-max-entry-count 0 API 服务器的图像 URL 获取超时可通过环境变量 `LMDEPLOY_FETCH_TIMEOUT` 进行配置。默认情况下，请求可能需要长达 10 秒才会超时。 -请参阅 [lmdeploy/vl/utils.py](https://github.com/InternLM/lmdeploy/blob/7b6876eafcb842633e0efe8baabe5906d7beeeea/lmdeploy/vl/utils.py#L31) 了解用法。 +请参阅 [lmdeploy/multimodal/utils.py](https://github.com/InternLM/lmdeploy/blob/main/lmdeploy/multimodal/utils.py) 了解用法。 ## 量化 diff --git a/docs/zh_cn/get_started/ascend/get_started.md b/docs/zh_cn/get_started/ascend/get_started.md index bae1503470..a9559ed944 100644 --- a/docs/zh_cn/get_started/ascend/get_started.md +++ b/docs/zh_cn/get_started/ascend/get_started.md @@ -43,7 +43,7 @@ print(response) ```python from lmdeploy import pipeline, PytorchEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-2B', backend_config=PytorchEngineConfig(tp=1, device_type='ascend')) image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') diff --git a/docs/zh_cn/get_started/camb/get_started.md b/docs/zh_cn/get_started/camb/get_started.md index 4f3043ccce..393330b08a 100644 --- a/docs/zh_cn/get_started/camb/get_started.md +++ b/docs/zh_cn/get_started/camb/get_started.md @@ -40,7 +40,7 @@ print(response) ```python from lmdeploy import pipeline, PytorchEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-2B', backend_config=PytorchEngineConfig(tp=1, device_type='camb')) image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') diff --git a/docs/zh_cn/get_started/get_started.md b/docs/zh_cn/get_started/get_started.md index 51d5f0ff81..8e8de9b135 100644 --- a/docs/zh_cn/get_started/get_started.md +++ b/docs/zh_cn/get_started/get_started.md @@ -81,7 +81,7 @@ VLM 推理 pipeline 与 LLM 类似，但增加了使用 pipeline 处理图像数 ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-8B') @@ -94,7 +94,7 @@ print(response) ```python from lmdeploy import pipeline, VisionConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-8B', vision_config=VisionConfig( diff --git a/docs/zh_cn/get_started/maca/get_started.md b/docs/zh_cn/get_started/maca/get_started.md index bbe57caf7f..2466fffcb1 100644 --- a/docs/zh_cn/get_started/maca/get_started.md +++ b/docs/zh_cn/get_started/maca/get_started.md @@ -31,7 +31,7 @@ print(response) ```python from lmdeploy import pipeline, PytorchEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-2B', backend_config=PytorchEngineConfig(tp=1, device_type='maca')) image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') diff --git a/docs/zh_cn/multi_modal/cogvlm.md b/docs/zh_cn/multi_modal/cogvlm.md index 5101f249f7..41acaf0303 100644 --- a/docs/zh_cn/multi_modal/cogvlm.md +++ b/docs/zh_cn/multi_modal/cogvlm.md @@ -25,7 +25,7 @@ huggingface-cli download lmsys/vicuna-7b-v1.5 special_tokens_map.json tokenizer. ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image if __name__ == "__main__": diff --git a/docs/zh_cn/multi_modal/deepseek_vl2.md b/docs/zh_cn/multi_modal/deepseek_vl2.md index 3e21288c55..86a4489e24 100644 --- a/docs/zh_cn/multi_modal/deepseek_vl2.md +++ b/docs/zh_cn/multi_modal/deepseek_vl2.md @@ -30,7 +30,7 @@ pip install attrdict timm 'transformers<4.48.0' ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image if __name__ == "__main__": diff --git a/docs/zh_cn/multi_modal/gemma3.md b/docs/zh_cn/multi_modal/gemma3.md index 5ad624424e..106a9076bd 100644 --- a/docs/zh_cn/multi_modal/gemma3.md +++ b/docs/zh_cn/multi_modal/gemma3.md @@ -18,7 +18,7 @@ Gemma 是 Google 推出的轻量级、最先进的开放模型系列，采用与 ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image if __name__ == "__main__": diff --git a/docs/zh_cn/multi_modal/index.rst b/docs/zh_cn/multi_modal/index.rst index 0344770a3e..d6e1f7cde1 100644 --- a/docs/zh_cn/multi_modal/index.rst +++ b/docs/zh_cn/multi_modal/index.rst @@ -1,6 +1,8 @@ -视觉语言模型 +多模态模型 ================================= +请使用 ``lmdeploy.multimodal`` 访问多模态辅助 API，例如媒体加载和本地文件编码。 + .. toctree:: :maxdepth: 2 :caption: 示例 diff --git a/docs/zh_cn/multi_modal/internvl.md b/docs/zh_cn/multi_modal/internvl.md index 6af6f45b0f..30f8d97a28 100644 --- a/docs/zh_cn/multi_modal/internvl.md +++ b/docs/zh_cn/multi_modal/internvl.md @@ -42,7 +42,7 @@ docker build --build-arg CUDA_VERSION=cu11 -t openmmlab/lmdeploy:internvl . -f . ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2-8B') @@ -60,7 +60,7 @@ print(response) ```python from lmdeploy import pipeline, GenerationConfig -from lmdeploy.vl.constants import IMAGE_TOKEN +from lmdeploy.multimodal.constants import IMAGE_TOKEN pipe = pipeline('OpenGVLab/InternVL2-8B', log_level='INFO') messages = [ @@ -86,7 +86,7 @@ out = pipe(messages, gen_config=GenerationConfig(top_k=1)) ```python from lmdeploy import pipeline, GenerationConfig -from lmdeploy.vl.constants import IMAGE_TOKEN +from lmdeploy.multimodal.constants import IMAGE_TOKEN pipe = pipeline('OpenGVLab/InternVL2-8B', log_level='INFO') messages = [ @@ -114,8 +114,8 @@ out = pipe(messages, gen_config=GenerationConfig(top_k=1)) import numpy as np from lmdeploy import pipeline, GenerationConfig from decord import VideoReader, cpu -from lmdeploy.vl.constants import IMAGE_TOKEN -from lmdeploy.vl import encode_image_base64 +from lmdeploy.multimodal.constants import IMAGE_TOKEN +from lmdeploy.multimodal import encode_image_base64 from PIL import Image pipe = pipeline('OpenGVLab/InternVL2-8B', log_level='INFO') diff --git a/docs/zh_cn/multi_modal/llava.md b/docs/zh_cn/multi_modal/llava.md index 6538d1b861..857030828f 100644 --- a/docs/zh_cn/multi_modal/llava.md +++ b/docs/zh_cn/multi_modal/llava.md @@ -33,7 +33,7 @@ docker pull openmmlab/lmdeploy:latest ```python from lmdeploy import GenerationConfig, TurbomindEngineConfig, pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline("llava-hf/llava-interleave-qwen-7b-hf", backend_config=TurbomindEngineConfig(cache_max_entry_count=0.5), gen_config=GenerationConfig(max_new_tokens=512)) diff --git a/docs/zh_cn/multi_modal/minicpmv.md b/docs/zh_cn/multi_modal/minicpmv.md index eb2a168cdb..784f3404ad 100644 --- a/docs/zh_cn/multi_modal/minicpmv.md +++ b/docs/zh_cn/multi_modal/minicpmv.md @@ -19,7 +19,7 @@ LMDeploy 支持 MiniCPM-V 系列模型，具体如下： ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('openbmb/MiniCPM-V-2_6') @@ -97,7 +97,7 @@ print(out.text) ```python from lmdeploy import pipeline, GenerationConfig -from lmdeploy.vl import encode_image_base64 +from lmdeploy.multimodal import encode_image_base64 import torch from PIL import Image from transformers import AutoModel, AutoTokenizer diff --git a/docs/zh_cn/multi_modal/molmo.md b/docs/zh_cn/multi_modal/molmo.md index 1dc8f8f79b..5e5f93a83f 100644 --- a/docs/zh_cn/multi_modal/molmo.md +++ b/docs/zh_cn/multi_modal/molmo.md @@ -19,7 +19,7 @@ LMDeploy 支持 Molmo 系列模型，具体如下： ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('allenai/Molmo-7B-D-0924') diff --git a/docs/zh_cn/multi_modal/multimodal_inputs.md b/docs/zh_cn/multi_modal/multimodal_inputs.md index d87b19e068..a9a2114eb3 100644 --- a/docs/zh_cn/multi_modal/multimodal_inputs.md +++ b/docs/zh_cn/multi_modal/multimodal_inputs.md @@ -397,7 +397,7 @@ ______________________________________________________________________ - **本地文件路径**，使用 `file://` 协议：`file:///absolute/path/to/file.jpg` - **Base64 编码数据**，使用 data URL：`data:;base64,` -可使用 `lmdeploy.vl.utils` 中的工具函数对本地文件进行编码： +可使用 `lmdeploy.multimodal.utils` 中的工具函数对本地文件进行编码：
本地文件路径示例 @@ -433,7 +433,7 @@ print(response.choices[0].message.content) ```python from openai import OpenAI -from lmdeploy.vl.utils import encode_image_base64 +from lmdeploy.multimodal.utils import encode_image_base64 client = OpenAI(api_key='EMPTY', base_url='http://localhost:23333/v1') model_name = client.models.list().data[0].id @@ -464,7 +464,7 @@ print(response.choices[0].message.content) ```python from openai import OpenAI -from lmdeploy.vl.utils import encode_video_base64 +from lmdeploy.multimodal.utils import encode_video_base64 client = OpenAI(api_key='EMPTY', base_url='http://localhost:23333/v1') model_name = client.models.list().data[0].id @@ -496,7 +496,7 @@ print(response.choices[0].message.content) ```python from openai import OpenAI -from lmdeploy.vl.utils import encode_audio_base64 +from lmdeploy.multimodal.utils import encode_audio_base64 client = OpenAI(api_key='EMPTY', base_url='http://localhost:23333/v1') model_name = client.models.list().data[0].id @@ -527,7 +527,7 @@ print(response.choices[0].message.content) ```python from openai import OpenAI -from lmdeploy.vl.utils import encode_time_series_base64 +from lmdeploy.multimodal.utils import encode_time_series_base64 client = OpenAI(api_key='EMPTY', base_url='http://localhost:23333/v1') model_name = client.models.list().data[0].id diff --git a/docs/zh_cn/multi_modal/phi3.md b/docs/zh_cn/multi_modal/phi3.md index b5545d30b6..880fe6f2ef 100644 --- a/docs/zh_cn/multi_modal/phi3.md +++ b/docs/zh_cn/multi_modal/phi3.md @@ -26,7 +26,7 @@ pip install flash-attn ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('microsoft/Phi-3.5-vision-instruct') diff --git a/docs/zh_cn/multi_modal/qwen2_5_vl.md b/docs/zh_cn/multi_modal/qwen2_5_vl.md index 2b1d81c0a4..91f0f2d4ba 100644 --- a/docs/zh_cn/multi_modal/qwen2_5_vl.md +++ b/docs/zh_cn/multi_modal/qwen2_5_vl.md @@ -25,7 +25,7 @@ pip install qwen-vl-utils[decord]==0.0.8 ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('Qwen/Qwen2.5-VL-7B-Instruct') @@ -98,8 +98,8 @@ out = pipe(messages, gen_config=GenerationConfig(top_k=1)) import numpy as np from lmdeploy import pipeline, GenerationConfig from decord import VideoReader, cpu -from lmdeploy.vl.constants import IMAGE_TOKEN -from lmdeploy.vl import encode_image_base64 +from lmdeploy.multimodal.constants import IMAGE_TOKEN +from lmdeploy.multimodal import encode_image_base64 from PIL import Image pipe = pipeline('Qwen/Qwen2.5-VL-7B-Instruct', log_level='INFO') diff --git a/docs/zh_cn/multi_modal/qwen2_vl.md b/docs/zh_cn/multi_modal/qwen2_vl.md index 1d2d18f30a..10af3cc66f 100644 --- a/docs/zh_cn/multi_modal/qwen2_vl.md +++ b/docs/zh_cn/multi_modal/qwen2_vl.md @@ -36,7 +36,7 @@ docker build --build-arg CUDA_VERSION=cu11 -t openmmlab/lmdeploy:qwen2vl . -f ./ ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('Qwen/Qwen2-VL-2B-Instruct') diff --git a/docs/zh_cn/multi_modal/vl_pipeline.md b/docs/zh_cn/multi_modal/vl_pipeline.md index 9662bcc569..d2d2c54c89 100644 --- a/docs/zh_cn/multi_modal/vl_pipeline.md +++ b/docs/zh_cn/multi_modal/vl_pipeline.md @@ -16,7 +16,7 @@ LMDeploy 把视觉-语言模型（VLM）复杂的推理过程，抽象为简单 ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B') @@ -53,7 +53,7 @@ print(response) ```python from lmdeploy import pipeline, TurbomindEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(tp=2)) @@ -69,7 +69,7 @@ print(response) ```python from lmdeploy import pipeline, TurbomindEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(session_len=8192)) @@ -85,7 +85,7 @@ print(response) ```python from lmdeploy import pipeline, GenerationConfig, TurbomindEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(tp=2, session_len=8192)) @@ -101,8 +101,8 @@ print(response) ```python from lmdeploy import pipeline -from lmdeploy.vl import load_image -from lmdeploy.vl.constants import IMAGE_TOKEN +from lmdeploy.multimodal import load_image +from lmdeploy.multimodal.constants import IMAGE_TOKEN pipe = pipeline('deepseek-ai/deepseek-vl-1.3b-chat') @@ -117,7 +117,7 @@ print(response) ```python from lmdeploy import pipeline, ChatTemplateConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('local_model_folder', chat_template_config=ChatTemplateConfig(model_name='llava-v1')) image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') @@ -133,7 +133,7 @@ print(response) ```python from lmdeploy import pipeline, VisionConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image vision_config=VisionConfig(max_batch_size=16) pipe = pipeline('liuhaotian/llava-v1.5-7b', vision_config=vision_config) image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') @@ -145,7 +145,7 @@ print(response) ```python from lmdeploy import pipeline, GenerationConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B') image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') @@ -162,7 +162,7 @@ print(logits) ```python from lmdeploy import pipeline, TurbomindEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(session_len=8192)) @@ -183,7 +183,7 @@ print(response) ```python from lmdeploy import pipeline, TurbomindEngineConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(session_len=8192)) @@ -203,7 +203,7 @@ pipeline 进行多轮对话有两种方式，一种是按照 openai 的格式来 ```python from lmdeploy import pipeline, TurbomindEngineConfig, GenerationConfig -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image pipe = pipeline('OpenGVLab/InternVL2_5-8B', backend_config=TurbomindEngineConfig(session_len=8192)) @@ -224,7 +224,7 @@ print(sess.response.text) from lmdeploy import pipeline from lmdeploy import pipeline -from lmdeploy.vl import load_image +from lmdeploy.multimodal import load_image with pipeline('OpenGVLab/InternVL2_5-8B') as pipe: image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg') diff --git a/lmdeploy/api.py b/lmdeploy/api.py index 6db813b9cc..66a66a13c3 100644 --- a/lmdeploy/api.py +++ b/lmdeploy/api.py @@ -59,7 +59,7 @@ def pipeline(model_path: str, print(response) # VLM - from lmdeploy.vl import load_image + from lmdeploy.multimodal import load_image from lmdeploy import pipeline, TurbomindEngineConfig, ChatTemplateConfig pipe = pipeline('liuhaotian/llava-v1.5-7b', backend_config=TurbomindEngineConfig(session_len=8192), diff --git a/lmdeploy/lite/apis/calibrate.py b/lmdeploy/lite/apis/calibrate.py index 4b09fca9d4..ff23f39ba2 100644 --- a/lmdeploy/lite/apis/calibrate.py +++ b/lmdeploy/lite/apis/calibrate.py @@ -9,7 +9,7 @@ from lmdeploy.archs import get_model_arch from lmdeploy.lite.quantization import CalibrationContext, CalibrationContextV2 from lmdeploy.lite.utils import collect_target_modules, get_calib_loaders, load_hf_from_pretrained -from lmdeploy.vl.model.builder import load_vl_model +from lmdeploy.multimodal.model.builder import load_vl_model LAYER_TYPE_MAP = { 'InternLM2ForCausalLM': 'InternLM2DecoderLayer', diff --git a/lmdeploy/multimodal/__init__.py b/lmdeploy/multimodal/__init__.py new file mode 100644 index 0000000000..5dab7b9a83 --- /dev/null +++ b/lmdeploy/multimodal/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .utils import ( + encode_audio_base64, + encode_image_base64, + encode_time_series_base64, + encode_video_base64, + load_audio, + load_image, + load_time_series, + load_video, +) + +__all__ = [ + 'load_image', + 'load_video', + 'load_audio', + 'load_time_series', + 'encode_image_base64', + 'encode_video_base64', + 'encode_audio_base64', + 'encode_time_series_base64', +] diff --git a/lmdeploy/vl/constants.py b/lmdeploy/multimodal/constants.py similarity index 100% rename from lmdeploy/vl/constants.py rename to lmdeploy/multimodal/constants.py diff --git a/lmdeploy/vl/engine.py b/lmdeploy/multimodal/engine.py similarity index 99% rename from lmdeploy/vl/engine.py rename to lmdeploy/multimodal/engine.py index f2a5f62ccf..3a664595fa 100644 --- a/lmdeploy/vl/engine.py +++ b/lmdeploy/multimodal/engine.py @@ -9,7 +9,7 @@ from lmdeploy.messages import PytorchEngineConfig, TurbomindEngineConfig, VisionConfig from lmdeploy.utils import is_bf16_supported -from lmdeploy.vl.model.builder import load_vl_model +from lmdeploy.multimodal.model.builder import load_vl_model def _get_hf_config_mm_feature_dtype(hf_config) -> torch.dtype | None: diff --git a/lmdeploy/vl/media/__init__.py b/lmdeploy/multimodal/media/__init__.py similarity index 100% rename from lmdeploy/vl/media/__init__.py rename to lmdeploy/multimodal/media/__init__.py diff --git a/lmdeploy/vl/media/audio.py b/lmdeploy/multimodal/media/audio.py similarity index 100% rename from lmdeploy/vl/media/audio.py rename to lmdeploy/multimodal/media/audio.py diff --git a/lmdeploy/vl/media/base.py b/lmdeploy/multimodal/media/base.py similarity index 100% rename from lmdeploy/vl/media/base.py rename to lmdeploy/multimodal/media/base.py diff --git a/lmdeploy/vl/media/connection.py b/lmdeploy/multimodal/media/connection.py similarity index 100% rename from lmdeploy/vl/media/connection.py rename to lmdeploy/multimodal/media/connection.py diff --git a/lmdeploy/vl/media/image.py b/lmdeploy/multimodal/media/image.py similarity index 100% rename from lmdeploy/vl/media/image.py rename to lmdeploy/multimodal/media/image.py diff --git a/lmdeploy/vl/media/time_series.py b/lmdeploy/multimodal/media/time_series.py similarity index 100% rename from lmdeploy/vl/media/time_series.py rename to lmdeploy/multimodal/media/time_series.py diff --git a/lmdeploy/vl/media/video.py b/lmdeploy/multimodal/media/video.py similarity index 100% rename from lmdeploy/vl/media/video.py rename to lmdeploy/multimodal/media/video.py diff --git a/lmdeploy/vl/media/video_loader.py b/lmdeploy/multimodal/media/video_loader.py similarity index 100% rename from lmdeploy/vl/media/video_loader.py rename to lmdeploy/multimodal/media/video_loader.py diff --git a/lmdeploy/vl/model/__init__.py b/lmdeploy/multimodal/model/__init__.py similarity index 100% rename from lmdeploy/vl/model/__init__.py rename to lmdeploy/multimodal/model/__init__.py diff --git a/lmdeploy/vl/model/base.py b/lmdeploy/multimodal/model/base.py similarity index 99% rename from lmdeploy/vl/model/base.py rename to lmdeploy/multimodal/model/base.py index 47de994b5a..10ac159861 100644 --- a/lmdeploy/vl/model/base.py +++ b/lmdeploy/multimodal/model/base.py @@ -10,8 +10,8 @@ from transformers import AutoConfig, AutoTokenizer from lmdeploy.archs import get_model_arch -from lmdeploy.vl.constants import Modality -from lmdeploy.vl.model.preprocess_utils import ( +from lmdeploy.multimodal.constants import Modality +from lmdeploy.multimodal.model.preprocess_utils import ( get_expanded_input_ids, get_expanded_mm_items, get_mm_items_offset, diff --git a/lmdeploy/vl/model/builder.py b/lmdeploy/multimodal/model/builder.py similarity index 98% rename from lmdeploy/vl/model/builder.py rename to lmdeploy/multimodal/model/builder.py index 6b2152bc09..91c8235ebf 100644 --- a/lmdeploy/vl/model/builder.py +++ b/lmdeploy/multimodal/model/builder.py @@ -6,7 +6,7 @@ from lmdeploy.archs import get_model_arch from lmdeploy.messages import PytorchEngineConfig, TurbomindEngineConfig from lmdeploy.utils import get_logger, get_model -from lmdeploy.vl.model.base import VISION_MODELS +from lmdeploy.multimodal.model.base import VISION_MODELS from .cogvlm import CogVLMVisionModel # noqa F401 from .deepseek import DeepSeekVisionModel # noqa F401 diff --git a/lmdeploy/vl/model/cogvlm.py b/lmdeploy/multimodal/model/cogvlm.py similarity index 98% rename from lmdeploy/vl/model/cogvlm.py rename to lmdeploy/multimodal/model/cogvlm.py index dd35c907b7..587bdf85bb 100644 --- a/lmdeploy/vl/model/cogvlm.py +++ b/lmdeploy/multimodal/model/cogvlm.py @@ -1,7 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/deepseek.py b/lmdeploy/multimodal/model/deepseek.py similarity index 98% rename from lmdeploy/vl/model/deepseek.py rename to lmdeploy/multimodal/model/deepseek.py index bd0f68c2e3..081d0a300b 100644 --- a/lmdeploy/vl/model/deepseek.py +++ b/lmdeploy/multimodal/model/deepseek.py @@ -5,8 +5,8 @@ from transformers import AutoModelForCausalLM from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel -from lmdeploy.vl.model.utils import disable_logging +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.utils import disable_logging logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/deepseek_vl2.py b/lmdeploy/multimodal/model/deepseek_vl2.py similarity index 99% rename from lmdeploy/vl/model/deepseek_vl2.py rename to lmdeploy/multimodal/model/deepseek_vl2.py index c17b02c954..0f592cafdc 100644 --- a/lmdeploy/vl/model/deepseek_vl2.py +++ b/lmdeploy/multimodal/model/deepseek_vl2.py @@ -6,7 +6,7 @@ from transformers import AutoConfig from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/gemma3_vl.py b/lmdeploy/multimodal/model/gemma3_vl.py similarity index 98% rename from lmdeploy/vl/model/gemma3_vl.py rename to lmdeploy/multimodal/model/gemma3_vl.py index d957184621..a554c1135b 100644 --- a/lmdeploy/vl/model/gemma3_vl.py +++ b/lmdeploy/multimodal/model/gemma3_vl.py @@ -5,7 +5,7 @@ from transformers.processing_utils import ImagesKwargs, ProcessingKwargs from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/glm4_1v.py b/lmdeploy/multimodal/model/glm4_1v.py similarity index 92% rename from lmdeploy/vl/model/glm4_1v.py rename to lmdeploy/multimodal/model/glm4_1v.py index f534a592aa..acd4edf88e 100644 --- a/lmdeploy/vl/model/glm4_1v.py +++ b/lmdeploy/multimodal/model/glm4_1v.py @@ -3,7 +3,7 @@ from transformers import AutoConfig from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, MultimodalSpecialTokens, VisionModel +from lmdeploy.multimodal.model.base import VISION_MODELS, MultimodalSpecialTokens, VisionModel logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/glm4_v.py b/lmdeploy/multimodal/model/glm4_v.py similarity index 98% rename from lmdeploy/vl/model/glm4_v.py rename to lmdeploy/multimodal/model/glm4_v.py index 0892b02dc2..cd4ce1ac2c 100644 --- a/lmdeploy/vl/model/glm4_v.py +++ b/lmdeploy/multimodal/model/glm4_v.py @@ -3,7 +3,7 @@ from transformers import AutoConfig from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/interns1_pro.py b/lmdeploy/multimodal/model/interns1_pro.py similarity index 96% rename from lmdeploy/vl/model/interns1_pro.py rename to lmdeploy/multimodal/model/interns1_pro.py index d1f18e1828..4851fc4810 100644 --- a/lmdeploy/vl/model/interns1_pro.py +++ b/lmdeploy/multimodal/model/interns1_pro.py @@ -5,8 +5,8 @@ import torch from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, MultimodalSpecialTokens -from lmdeploy.vl.model.qwen3 import Qwen3VLModel +from lmdeploy.multimodal.model.base import VISION_MODELS, MultimodalSpecialTokens +from lmdeploy.multimodal.model.qwen3 import Qwen3VLModel logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/internvl.py b/lmdeploy/multimodal/model/internvl.py similarity index 99% rename from lmdeploy/vl/model/internvl.py rename to lmdeploy/multimodal/model/internvl.py index 1534ad3388..3ecb853db5 100644 --- a/lmdeploy/vl/model/internvl.py +++ b/lmdeploy/multimodal/model/internvl.py @@ -6,8 +6,8 @@ from transformers import AutoConfig, AutoModel, AutoTokenizer, CLIPImageProcessor from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel -from lmdeploy.vl.model.utils import disable_logging +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.utils import disable_logging logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/internvl3_hf.py b/lmdeploy/multimodal/model/internvl3_hf.py similarity index 97% rename from lmdeploy/vl/model/internvl3_hf.py rename to lmdeploy/multimodal/model/internvl3_hf.py index 3816cfe491..fed5605688 100644 --- a/lmdeploy/vl/model/internvl3_hf.py +++ b/lmdeploy/multimodal/model/internvl3_hf.py @@ -5,8 +5,8 @@ from transformers.processing_utils import ImagesKwargs, ProcessingKwargs from lmdeploy.utils import get_logger -from lmdeploy.vl.model.internvl import VISION_MODELS, InternVLVisionModel -from lmdeploy.vl.model.utils import disable_logging +from lmdeploy.multimodal.model.internvl import VISION_MODELS, InternVLVisionModel +from lmdeploy.multimodal.model.utils import disable_logging logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/llama4.py b/lmdeploy/multimodal/model/llama4.py similarity index 99% rename from lmdeploy/vl/model/llama4.py rename to lmdeploy/multimodal/model/llama4.py index 92199f458c..023861e61e 100644 --- a/lmdeploy/vl/model/llama4.py +++ b/lmdeploy/multimodal/model/llama4.py @@ -4,7 +4,7 @@ from transformers import AutoConfig from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/llava.py b/lmdeploy/multimodal/model/llava.py similarity index 99% rename from lmdeploy/vl/model/llava.py rename to lmdeploy/multimodal/model/llava.py index 803b2067cb..30842b82d3 100644 --- a/lmdeploy/vl/model/llava.py +++ b/lmdeploy/multimodal/model/llava.py @@ -11,8 +11,8 @@ from transformers import AutoConfig, AutoModelForCausalLM from lmdeploy.utils import get_logger -from lmdeploy.vl.model.llava_hf import VISION_MODELS, LlavaHfVisionModel -from lmdeploy.vl.model.utils import disable_logging, rewrite_ctx +from lmdeploy.multimodal.model.llava_hf import VISION_MODELS, LlavaHfVisionModel +from lmdeploy.multimodal.model.utils import disable_logging, rewrite_ctx logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/llava_hf.py b/lmdeploy/multimodal/model/llava_hf.py similarity index 97% rename from lmdeploy/vl/model/llava_hf.py rename to lmdeploy/multimodal/model/llava_hf.py index 08dfa3dbc6..ca83d94659 100644 --- a/lmdeploy/vl/model/llava_hf.py +++ b/lmdeploy/multimodal/model/llava_hf.py @@ -5,8 +5,8 @@ from transformers import AutoProcessor from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel -from lmdeploy.vl.model.utils import disable_logging +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.utils import disable_logging logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/llava_next.py b/lmdeploy/multimodal/model/llava_next.py similarity index 98% rename from lmdeploy/vl/model/llava_next.py rename to lmdeploy/multimodal/model/llava_next.py index e55bbe6f25..aa7c9636a0 100644 --- a/lmdeploy/vl/model/llava_next.py +++ b/lmdeploy/multimodal/model/llava_next.py @@ -5,8 +5,8 @@ import torch from lmdeploy.utils import get_logger -from lmdeploy.vl.model.llava_hf import VISION_MODELS, LlavaHfVisionModel -from lmdeploy.vl.model.utils import disable_logging +from lmdeploy.multimodal.model.llava_hf import VISION_MODELS, LlavaHfVisionModel +from lmdeploy.multimodal.model.utils import disable_logging logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/minicpmv.py b/lmdeploy/multimodal/model/minicpmv.py similarity index 98% rename from lmdeploy/vl/model/minicpmv.py rename to lmdeploy/multimodal/model/minicpmv.py index e64278244a..cd34d4271f 100644 --- a/lmdeploy/vl/model/minicpmv.py +++ b/lmdeploy/multimodal/model/minicpmv.py @@ -7,8 +7,8 @@ from transformers import AutoConfig, AutoModelForCausalLM from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel -from lmdeploy.vl.model.utils import disable_logging +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.utils import disable_logging logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/molmo.py b/lmdeploy/multimodal/model/molmo.py similarity index 98% rename from lmdeploy/vl/model/molmo.py rename to lmdeploy/multimodal/model/molmo.py index 1bad7474cb..cea6bb7e62 100644 --- a/lmdeploy/vl/model/molmo.py +++ b/lmdeploy/multimodal/model/molmo.py @@ -5,8 +5,8 @@ from transformers import AutoModelForCausalLM, AutoProcessor from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel -from lmdeploy.vl.model.utils import disable_logging +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.utils import disable_logging logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/phi3_vision.py b/lmdeploy/multimodal/model/phi3_vision.py similarity index 95% rename from lmdeploy/vl/model/phi3_vision.py rename to lmdeploy/multimodal/model/phi3_vision.py index 56ce69ce40..ae2fe2ace4 100644 --- a/lmdeploy/vl/model/phi3_vision.py +++ b/lmdeploy/multimodal/model/phi3_vision.py @@ -3,7 +3,7 @@ from transformers import AutoProcessor -from lmdeploy.vl.model.llava_hf import VISION_MODELS, LlavaHfVisionModel +from lmdeploy.multimodal.model.llava_hf import VISION_MODELS, LlavaHfVisionModel @VISION_MODELS.register_module() diff --git a/lmdeploy/vl/model/preprocess_utils.py b/lmdeploy/multimodal/model/preprocess_utils.py similarity index 98% rename from lmdeploy/vl/model/preprocess_utils.py rename to lmdeploy/multimodal/model/preprocess_utils.py index f267722ff7..e342cf7845 100644 --- a/lmdeploy/vl/model/preprocess_utils.py +++ b/lmdeploy/multimodal/model/preprocess_utils.py @@ -5,10 +5,10 @@ import torch.nn.functional as F from lmdeploy.utils import get_logger -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality if TYPE_CHECKING: - from lmdeploy.vl.model.base import MultimodalSpecialTokens + from lmdeploy.multimodal.model.base import MultimodalSpecialTokens logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/qwen2.py b/lmdeploy/multimodal/model/qwen2.py similarity index 98% rename from lmdeploy/vl/model/qwen2.py rename to lmdeploy/multimodal/model/qwen2.py index 98e3f8cd09..af80807d81 100644 --- a/lmdeploy/vl/model/qwen2.py +++ b/lmdeploy/multimodal/model/qwen2.py @@ -1,8 +1,8 @@ # Copyright (c) OpenMMLab. All rights reserved. import torch -from lmdeploy.vl.model.base import VISION_MODELS, VisionModel -from lmdeploy.vl.model.utils import disable_logging +from lmdeploy.multimodal.model.base import VISION_MODELS, VisionModel +from lmdeploy.multimodal.model.utils import disable_logging def check_qwen_vl_deps_install(): diff --git a/lmdeploy/vl/model/qwen3.py b/lmdeploy/multimodal/model/qwen3.py similarity index 94% rename from lmdeploy/vl/model/qwen3.py rename to lmdeploy/multimodal/model/qwen3.py index 733fd54750..484996baa9 100644 --- a/lmdeploy/vl/model/qwen3.py +++ b/lmdeploy/multimodal/model/qwen3.py @@ -3,7 +3,7 @@ from transformers import AutoProcessor from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, MultimodalSpecialTokens, VisionModel +from lmdeploy.multimodal.model.base import VISION_MODELS, MultimodalSpecialTokens, VisionModel logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/qwen3_5.py b/lmdeploy/multimodal/model/qwen3_5.py similarity index 97% rename from lmdeploy/vl/model/qwen3_5.py rename to lmdeploy/multimodal/model/qwen3_5.py index efcc971ac3..5aa810068e 100644 --- a/lmdeploy/vl/model/qwen3_5.py +++ b/lmdeploy/multimodal/model/qwen3_5.py @@ -5,9 +5,9 @@ import torch from lmdeploy.utils import get_logger -from lmdeploy.vl.model.base import VISION_MODELS, MultimodalSpecialTokens -from lmdeploy.vl.model.qwen3 import Qwen3VLModel -from lmdeploy.vl.model.utils import disable_logging +from lmdeploy.multimodal.model.base import VISION_MODELS, MultimodalSpecialTokens +from lmdeploy.multimodal.model.qwen3 import Qwen3VLModel +from lmdeploy.multimodal.model.utils import disable_logging logger = get_logger('lmdeploy') diff --git a/lmdeploy/vl/model/qwen3_omni.py b/lmdeploy/multimodal/model/qwen3_omni.py similarity index 94% rename from lmdeploy/vl/model/qwen3_omni.py rename to lmdeploy/multimodal/model/qwen3_omni.py index c654b375d0..6b2481dac4 100644 --- a/lmdeploy/vl/model/qwen3_omni.py +++ b/lmdeploy/multimodal/model/qwen3_omni.py @@ -1,7 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. from transformers import AutoProcessor -from lmdeploy.vl.model.base import VISION_MODELS, MultimodalSpecialTokens, VisionModel +from lmdeploy.multimodal.model.base import VISION_MODELS, MultimodalSpecialTokens, VisionModel def check_transformers(): diff --git a/lmdeploy/vl/model/utils.py b/lmdeploy/multimodal/model/utils.py similarity index 100% rename from lmdeploy/vl/model/utils.py rename to lmdeploy/multimodal/model/utils.py diff --git a/lmdeploy/vl/tools/__init__.py b/lmdeploy/multimodal/tools/__init__.py similarity index 100% rename from lmdeploy/vl/tools/__init__.py rename to lmdeploy/multimodal/tools/__init__.py diff --git a/lmdeploy/vl/utils.py b/lmdeploy/multimodal/utils.py similarity index 100% rename from lmdeploy/vl/utils.py rename to lmdeploy/multimodal/utils.py diff --git a/lmdeploy/pytorch/messages.py b/lmdeploy/pytorch/messages.py index 00888bfdbb..246bf380ec 100644 --- a/lmdeploy/pytorch/messages.py +++ b/lmdeploy/pytorch/messages.py @@ -12,7 +12,7 @@ from lmdeploy.pytorch.disagg.conn.protocol import MigrationRequest from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs, make_multimodal_content_hash from lmdeploy.utils import get_logger -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality from .block import LogicalTokenBlocks diff --git a/lmdeploy/pytorch/models/interns1_pro.py b/lmdeploy/pytorch/models/interns1_pro.py index 9a03f30640..29b7be2edf 100644 --- a/lmdeploy/pytorch/models/interns1_pro.py +++ b/lmdeploy/pytorch/models/interns1_pro.py @@ -10,7 +10,7 @@ from lmdeploy.pytorch.model_inputs import StepContext, StepContextManager from lmdeploy.pytorch.multimodal.data_type import MultiModalData from lmdeploy.pytorch.weight_loader.model_weight_loader import load_weight -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality from .interns1_pro_time_series import InternS1ProTimeSeriesModel from .patch import add_prefix, get_build_model_context diff --git a/lmdeploy/pytorch/models/qwen3_5.py b/lmdeploy/pytorch/models/qwen3_5.py index 732d9b1166..1ce581d164 100644 --- a/lmdeploy/pytorch/models/qwen3_5.py +++ b/lmdeploy/pytorch/models/qwen3_5.py @@ -25,7 +25,7 @@ ) from lmdeploy.pytorch.nn.rotary_embedding import get_rope_parameters from lmdeploy.pytorch.weight_loader.model_weight_loader import default_weight_loader, load_weight -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality from .patch import add_prefix, get_build_model_context from .qwen2_5_vl import Qwen2_5_VisionRotaryEmbedding as Qwen3_5VisionRotaryEmbedding diff --git a/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py b/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py index 733c323dbf..eaf068f897 100644 --- a/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py +++ b/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py @@ -18,7 +18,7 @@ from lmdeploy.pytorch.nn import ApplyRotaryEmb, FlashAttention, LayerNorm from lmdeploy.pytorch.nn.linear import build_colwise_linear, build_qkv_proj, build_rowwise_linear from lmdeploy.pytorch.weight_loader.model_weight_loader import load_weight -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality from .qwen3_vl import Qwen3VLVisionBlock, Qwen3VLVisionPatchEmbed, Qwen3VLVisionRotaryEmbedding from .qwen3_vl_moe import Qwen3VLMoeTextModel diff --git a/lmdeploy/pytorch/models/qwen3_vl.py b/lmdeploy/pytorch/models/qwen3_vl.py index 9a0185e13b..5f082a5baa 100644 --- a/lmdeploy/pytorch/models/qwen3_vl.py +++ b/lmdeploy/pytorch/models/qwen3_vl.py @@ -17,7 +17,7 @@ from lmdeploy.pytorch.nn.linear import build_colwise_linear, build_rowwise_linear from lmdeploy.pytorch.nn.rotary_embedding import get_rope_parameters from lmdeploy.pytorch.weight_loader.model_weight_loader import load_weight -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality from .patch import add_prefix from .qwen2_5_vl import Qwen2_5_VisionRotaryEmbedding as Qwen3VLVisionRotaryEmbedding diff --git a/lmdeploy/pytorch/models/utils/model.py b/lmdeploy/pytorch/models/utils/model.py index 4b140fa692..0a01b7aeb4 100644 --- a/lmdeploy/pytorch/models/utils/model.py +++ b/lmdeploy/pytorch/models/utils/model.py @@ -11,7 +11,7 @@ from lmdeploy.pytorch.multimodal.data_type import MultiModalData from lmdeploy.pytorch.nn.embedding import ParallelEmbedding from lmdeploy.pytorch.nn.linear import build_rowwise_linear -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality class BaseModelMetaProcessor: diff --git a/lmdeploy/pytorch/multimodal/data_type.py b/lmdeploy/pytorch/multimodal/data_type.py index f778c2aeb5..4bc75dbddc 100644 --- a/lmdeploy/pytorch/multimodal/data_type.py +++ b/lmdeploy/pytorch/multimodal/data_type.py @@ -8,7 +8,7 @@ import torch from torch import Tensor -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality NestedTensor = Tensor | list[Tensor] diff --git a/lmdeploy/serve/core/vl_async_engine.py b/lmdeploy/serve/core/vl_async_engine.py index d246a20f75..3700b51d36 100644 --- a/lmdeploy/serve/core/vl_async_engine.py +++ b/lmdeploy/serve/core/vl_async_engine.py @@ -21,7 +21,7 @@ def __init__(self, **kwargs) -> None: from lmdeploy.serve.processors import MultimodalProcessor from lmdeploy.utils import try_import_deeplink - from lmdeploy.vl.engine import ImageEncoder + from lmdeploy.multimodal.engine import ImageEncoder if backend == 'pytorch': try_import_deeplink(backend_config.device_type) diff --git a/lmdeploy/serve/openai/api_server.py b/lmdeploy/serve/openai/api_server.py index 37fb86d7bc..e6037ca17c 100644 --- a/lmdeploy/serve/openai/api_server.py +++ b/lmdeploy/serve/openai/api_server.py @@ -1392,7 +1392,7 @@ def dummy_get_device_id(): return 0 if int(os.environ.get('LOCAL_RANK', -1)) > 0: - from lmdeploy.vl.model.utils import _set_func + from lmdeploy.multimodal.model.utils import _set_func # the replacement can't be recovered _set_func('mmengine.logging.logger._get_device_id', dummy_get_device_id) diff --git a/lmdeploy/serve/processors/multimodal.py b/lmdeploy/serve/processors/multimodal.py index cf2452935e..4b4c67a85a 100644 --- a/lmdeploy/serve/processors/multimodal.py +++ b/lmdeploy/serve/processors/multimodal.py @@ -7,12 +7,12 @@ from lmdeploy.model import MODELS, BaseChatTemplate from lmdeploy.tokenizer import Tokenizer from lmdeploy.utils import get_logger -from lmdeploy.vl.constants import Modality -from lmdeploy.vl.media.audio import AudioMediaIO -from lmdeploy.vl.media.connection import load_from_url -from lmdeploy.vl.media.image import ImageMediaIO -from lmdeploy.vl.media.time_series import TimeSeriesMediaIO -from lmdeploy.vl.media.video import VideoMediaIO +from lmdeploy.multimodal.constants import Modality +from lmdeploy.multimodal.media.audio import AudioMediaIO +from lmdeploy.multimodal.media.connection import load_from_url +from lmdeploy.multimodal.media.image import ImageMediaIO +from lmdeploy.multimodal.media.time_series import TimeSeriesMediaIO +from lmdeploy.multimodal.media.video import VideoMediaIO logger = get_logger('lmdeploy') @@ -299,7 +299,7 @@ def _is_image_list(obj) -> bool: @staticmethod def _re_format_prompt_images_pair(prompt: tuple) -> dict: """Reformat the prompt to openai message format.""" - from lmdeploy.vl import load_image + from lmdeploy.multimodal import load_image messages = {'role': 'user', 'content': []} prompt, images = prompt diff --git a/lmdeploy/turbomind/models/qwen3_5.py b/lmdeploy/turbomind/models/qwen3_5.py index 0d13f372ae..b87ad8bb03 100644 --- a/lmdeploy/turbomind/models/qwen3_5.py +++ b/lmdeploy/turbomind/models/qwen3_5.py @@ -26,7 +26,7 @@ import _turbomind as _tm import torch -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality from ..builders import ( AttentionBuilder, diff --git a/lmdeploy/vl/__init__.py b/lmdeploy/vl/__init__.py index 5dab7b9a83..63dd255fe8 100644 --- a/lmdeploy/vl/__init__.py +++ b/lmdeploy/vl/__init__.py @@ -1,22 +1,3 @@ # Copyright (c) OpenMMLab. All rights reserved. -from .utils import ( - encode_audio_base64, - encode_image_base64, - encode_time_series_base64, - encode_video_base64, - load_audio, - load_image, - load_time_series, - load_video, -) - -__all__ = [ - 'load_image', - 'load_video', - 'load_audio', - 'load_time_series', - 'encode_image_base64', - 'encode_video_base64', - 'encode_audio_base64', - 'encode_time_series_base64', -] +raise ImportError('lmdeploy.vl has been renamed to lmdeploy.multimodal. ' + 'Please update imports to use lmdeploy.multimodal.') diff --git a/tests/pytorch/paging/test_block_trie.py b/tests/pytorch/paging/test_block_trie.py index 8e5cf21033..9fa95fe50f 100644 --- a/tests/pytorch/paging/test_block_trie.py +++ b/tests/pytorch/paging/test_block_trie.py @@ -7,7 +7,7 @@ from lmdeploy.pytorch.messages import SamplingParam, SequenceMeta, UpdateTokenMode from lmdeploy.pytorch.multimodal.data_type import MultiModalData from lmdeploy.pytorch.paging import Scheduler -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality class TestBlockTire: diff --git a/tests/test_lmdeploy/test_content_merge.py b/tests/test_lmdeploy/test_content_merge.py index d494a8fcbd..49767f5a2a 100644 --- a/tests/test_lmdeploy/test_content_merge.py +++ b/tests/test_lmdeploy/test_content_merge.py @@ -5,7 +5,7 @@ from PIL import Image from lmdeploy.serve.processors import MultimodalProcessor -from lmdeploy.vl.constants import Modality +from lmdeploy.multimodal.constants import Modality multimodal_module = sys.modules[MultimodalProcessor.__module__] diff --git a/tests/test_lmdeploy/test_vl/test_hf_chat_template.py b/tests/test_lmdeploy/test_multimodal/test_hf_chat_template.py similarity index 99% rename from tests/test_lmdeploy/test_vl/test_hf_chat_template.py rename to tests/test_lmdeploy/test_multimodal/test_hf_chat_template.py index b6eabf8a54..e09b767287 100644 --- a/tests/test_lmdeploy/test_vl/test_hf_chat_template.py +++ b/tests/test_lmdeploy/test_multimodal/test_hf_chat_template.py @@ -3,7 +3,7 @@ import pytest from lmdeploy.model import MODELS -from lmdeploy.vl.model.builder import load_vl_model +from lmdeploy.multimodal.model.builder import load_vl_model def get_model_and_chat_template(model_path, trust_remote_code=False): diff --git a/tests/test_lmdeploy/test_vl/test_vl_encode.py b/tests/test_lmdeploy/test_multimodal/test_multimodal_encode.py similarity index 99% rename from tests/test_lmdeploy/test_vl/test_vl_encode.py rename to tests/test_lmdeploy/test_multimodal/test_multimodal_encode.py index 276d3cb936..82da21b95f 100644 --- a/tests/test_lmdeploy/test_vl/test_vl_encode.py +++ b/tests/test_lmdeploy/test_multimodal/test_multimodal_encode.py @@ -2,7 +2,7 @@ import numpy as np -from lmdeploy.vl import ( +from lmdeploy.multimodal import ( encode_audio_base64, encode_image_base64, encode_time_series_base64, diff --git a/tests/test_lmdeploy/test_vl/test_nonhf_chat_template.py b/tests/test_lmdeploy/test_multimodal/test_nonhf_chat_template.py similarity index 99% rename from tests/test_lmdeploy/test_vl/test_nonhf_chat_template.py rename to tests/test_lmdeploy/test_multimodal/test_nonhf_chat_template.py index bf6399647b..73327743ed 100644 --- a/tests/test_lmdeploy/test_vl/test_nonhf_chat_template.py +++ b/tests/test_lmdeploy/test_multimodal/test_nonhf_chat_template.py @@ -3,7 +3,7 @@ import pytest from lmdeploy.model import MODELS -from lmdeploy.vl.model.builder import load_vl_model +from lmdeploy.multimodal.model.builder import load_vl_model def get_model_and_chat_template(model_path, trust_remote_code=False): diff --git a/tests/test_lmdeploy/test_vl/test_preprocess_utils.py b/tests/test_lmdeploy/test_multimodal/test_preprocess_utils.py similarity index 95% rename from tests/test_lmdeploy/test_vl/test_preprocess_utils.py rename to tests/test_lmdeploy/test_multimodal/test_preprocess_utils.py index 22084f0e88..a060f4ae28 100644 --- a/tests/test_lmdeploy/test_vl/test_preprocess_utils.py +++ b/tests/test_lmdeploy/test_multimodal/test_preprocess_utils.py @@ -2,8 +2,8 @@ import torch -from lmdeploy.vl.constants import Modality -from lmdeploy.vl.model.preprocess_utils import get_expanded_mm_items +from lmdeploy.multimodal.constants import Modality +from lmdeploy.multimodal.model.preprocess_utils import get_expanded_mm_items class _Tokens: diff --git a/tests/test_lmdeploy/test_vl/test_qwen3_omni_processor.py b/tests/test_lmdeploy/test_multimodal/test_qwen3_omni_processor.py similarity index 97% rename from tests/test_lmdeploy/test_vl/test_qwen3_omni_processor.py rename to tests/test_lmdeploy/test_multimodal/test_qwen3_omni_processor.py index 596467c1a2..0b88300672 100644 --- a/tests/test_lmdeploy/test_vl/test_qwen3_omni_processor.py +++ b/tests/test_lmdeploy/test_multimodal/test_qwen3_omni_processor.py @@ -5,10 +5,10 @@ from lmdeploy.pytorch.models.qwen3_omni_moe_thinker import Qwen3OmniInputProcessor from lmdeploy.pytorch.models.utils.model import DeployModelMixinV1 from lmdeploy.pytorch.multimodal.data_type import MultiModalData -from lmdeploy.vl.constants import Modality -from lmdeploy.vl.model.base import MultimodalSpecialTokens -from lmdeploy.vl.model.preprocess_utils import get_expanded_mm_items -from lmdeploy.vl.model.qwen3_omni import Qwen3OmniModel +from lmdeploy.multimodal.constants import Modality +from lmdeploy.multimodal.model.base import MultimodalSpecialTokens +from lmdeploy.multimodal.model.preprocess_utils import get_expanded_mm_items +from lmdeploy.multimodal.model.qwen3_omni import Qwen3OmniModel class FakeQwen3OmniProcessor: diff --git a/tests/test_lmdeploy/test_vl/test_qwen3vl_processor.py b/tests/test_lmdeploy/test_multimodal/test_qwen3vl_processor.py similarity index 97% rename from tests/test_lmdeploy/test_vl/test_qwen3vl_processor.py rename to tests/test_lmdeploy/test_multimodal/test_qwen3vl_processor.py index 5e13d4b4bf..59bd74d1c9 100644 --- a/tests/test_lmdeploy/test_vl/test_qwen3vl_processor.py +++ b/tests/test_lmdeploy/test_multimodal/test_qwen3vl_processor.py @@ -1,8 +1,8 @@ import pytest -from lmdeploy.vl import load_image, load_video -from lmdeploy.vl.constants import Modality -from lmdeploy.vl.model.qwen3 import Qwen3VLModel +from lmdeploy.multimodal import load_image, load_video +from lmdeploy.multimodal.constants import Modality +from lmdeploy.multimodal.model.qwen3 import Qwen3VLModel QWEN3VL_MODELS = [ 'Qwen/Qwen3-VL-4B-Instruct', diff --git a/tests/test_lmdeploy/test_vl/test_safe_url.py b/tests/test_lmdeploy/test_multimodal/test_safe_url.py similarity index 92% rename from tests/test_lmdeploy/test_vl/test_safe_url.py rename to tests/test_lmdeploy/test_multimodal/test_safe_url.py index f919b3f0b9..1f168f01dd 100644 --- a/tests/test_lmdeploy/test_vl/test_safe_url.py +++ b/tests/test_lmdeploy/test_multimodal/test_safe_url.py @@ -4,7 +4,7 @@ import pytest -from lmdeploy.vl.media.connection import _is_safe_url, _load_http_url +from lmdeploy.multimodal.media.connection import _is_safe_url, _load_http_url @pytest.mark.parametrize( @@ -35,7 +35,7 @@ def test_is_safe_url(url, expected_safe, mock_ips): @patch('requests.Session.get') -@patch('lmdeploy.vl.media.connection._is_safe_url', return_value=(True, '')) +@patch('lmdeploy.multimodal.media.connection._is_safe_url', return_value=(True, '')) def test_load_http_url_logic(mock_safe, mock_get): media_io = MagicMock() url_spec = urlparse('https://example.com/img.jpg')