[mypy] Enable following imports for entrypoints

.github/workflows/mypy.yaml

@@ -38,7 +38,6 @@ jobs:
         mypy vllm/core --follow-imports skip
         mypy vllm/distributed --follow-imports skip
         mypy vllm/engine  --follow-imports skip
-        mypy vllm/entrypoints --follow-imports skip
         mypy vllm/executor --follow-imports skip
         mypy vllm/lora --follow-imports skip
         mypy vllm/model_executor  --follow-imports skip

format.sh

@@ -102,7 +102,6 @@ mypy vllm/attention --follow-imports skip
 mypy vllm/core --follow-imports skip
 mypy vllm/distributed --follow-imports skip
 mypy vllm/engine  --follow-imports skip
-mypy vllm/entrypoints --follow-imports skip
 mypy vllm/executor --follow-imports skip
 mypy vllm/lora --follow-imports skip
 mypy vllm/model_executor  --follow-imports skip

pyproject.toml

@@ -56,6 +56,7 @@ files = [
     "vllm/*.py",
     "vllm/adapter_commons",
     "vllm/assets",
+    "vllm/entrypoints",
     "vllm/inputs",
     "vllm/logging",
     "vllm/multimodal",

tests/entrypoints/openai/test_completion.py

@@ -298,14 +298,9 @@ async def test_prompt_logprobs_chat(client: openai.AsyncOpenAI,
     if prompt_logprobs is not None:
         params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
 
-    if prompt_logprobs and prompt_logprobs < 0:
-        with pytest.raises(BadRequestError) as err_info:
+    if prompt_logprobs is not None and prompt_logprobs < 0:
+        with pytest.raises(BadRequestError):
             await client.chat.completions.create(**params)
-        expected_err_string = (
-            "Error code: 400 - {'object': 'error', 'message': "
-            "'Prompt_logprobs set to invalid negative value: -1',"
-            " 'type': 'BadRequestError', 'param': None, 'code': 400}")
-        assert str(err_info.value) == expected_err_string
     else:
         completion = await client.chat.completions.create(**params)
         if prompt_logprobs and prompt_logprobs > 0:
@@ -369,14 +364,9 @@ async def test_prompt_logprobs_completion(client: openai.AsyncOpenAI,
     if prompt_logprobs is not None:
         params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
 
-    if prompt_logprobs and prompt_logprobs < 0:
-        with pytest.raises(BadRequestError) as err_info:
+    if prompt_logprobs is not None and prompt_logprobs < 0:
+        with pytest.raises(BadRequestError):
             await client.completions.create(**params)
-        expected_err_string = (
-            "Error code: 400 - {'object': 'error', 'message': "
-            "'Prompt_logprobs set to invalid negative value: -1',"
-            " 'type': 'BadRequestError', 'param': None, 'code': 400}")
-        assert str(err_info.value) == expected_err_string
     else:
         completion = await client.completions.create(**params)
         if prompt_logprobs and prompt_logprobs > 0:

vllm/engine/async_llm_engine.py

@@ -4,7 +4,6 @@
 from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
                     Optional, Set, Tuple, Type, Union)
 
-from transformers import PreTrainedTokenizer
 from typing_extensions import assert_never
 
 import vllm.envs as envs
@@ -28,6 +27,7 @@
 from vllm.prompt_adapter.request import PromptAdapterRequest
 from vllm.sampling_params import SamplingParams
 from vllm.sequence import ExecuteModelRequest, SamplerOutput
+from vllm.transformers_utils.tokenizer import AnyTokenizer
 from vllm.usage.usage_lib import UsageContext
 from vllm.utils import print_warning_once
 
@@ -308,8 +308,8 @@ async def _tokenize_prompt_async(
         lora_request: Optional[LoRARequest],
     ) -> List[int]:
         """Async version of :meth:`_tokenize_prompt`."""
-        tokenizer = self.get_tokenizer_group("prompts must be None if "
-                                             "skip_tokenizer_init is True")
+        tokenizer = self.get_tokenizer_group(
+            missing_msg="prompts must be None if skip_tokenizer_init is True")
 
         return await tokenizer.encode_async(request_id=request_id,
                                             prompt=prompt,
@@ -652,7 +652,7 @@ def _error_callback(self, exc: Exception) -> None:
     async def get_tokenizer(
         self,
         lora_request: Optional[LoRARequest] = None,
-    ) -> "PreTrainedTokenizer":
+    ) -> "AnyTokenizer":
         if self.engine_use_ray:
             return await self.engine.get_tokenizer.remote(  # type: ignore
                 lora_request)

vllm/engine/llm_engine.py

@@ -3,9 +3,9 @@
 from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
                     Mapping, Optional)
 from typing import Sequence as GenericSequence
-from typing import Set, Tuple, Type, TypeVar, Union
+from typing import Set, Tuple, Type, Union
 
-from typing_extensions import assert_never
+from typing_extensions import TypeVar, assert_never
 
 import vllm.envs as envs
 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
@@ -44,8 +44,9 @@
                           init_tracer)
 from vllm.transformers_utils.config import try_get_generation_config
 from vllm.transformers_utils.detokenizer import Detokenizer
+from vllm.transformers_utils.tokenizer import AnyTokenizer
 from vllm.transformers_utils.tokenizer_group import (
-    AnyTokenizer, BaseTokenizerGroup, init_tokenizer_from_configs)
+    BaseTokenizerGroup, init_tokenizer_from_configs)
 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
                                   usage_message)
 from vllm.utils import Counter
@@ -68,6 +69,7 @@ def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
     return config.to_diff_dict()
 
 
+_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
 _O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
 
 PromptComponents = Tuple[Optional[str], List[int],
@@ -488,12 +490,21 @@ def __del__(self):
                                    "skip_tokenizer_init is True")
 
     def get_tokenizer_group(
-            self,
-            fail_msg: str = MISSING_TOKENIZER_GROUP_MSG) -> BaseTokenizerGroup:
-        if self.tokenizer is None:
-            raise ValueError(fail_msg)
+        self,
+        group_type: Type[_G] = BaseTokenizerGroup,
+        *,
+        missing_msg: str = MISSING_TOKENIZER_GROUP_MSG,
+    ) -> _G:
+        tokenizer_group = self.tokenizer
+
+        if tokenizer_group is None:
+            raise ValueError(missing_msg)
+        if not isinstance(tokenizer_group, group_type):
+            raise TypeError("Invalid type of tokenizer group. "
+                            f"Expected type: {group_type}, but "
+                            f"found type: {type(tokenizer_group)}")
 
-        return self.tokenizer
+        return tokenizer_group
 
     def get_tokenizer(
         self,
@@ -688,8 +699,8 @@ def _tokenize_prompt(
         * prompt token ids
         '''
 
-        tokenizer = self.get_tokenizer_group("prompts must be None if "
-                                             "skip_tokenizer_init is True")
+        tokenizer = self.get_tokenizer_group(
+            missing_msg="prompts must be None if skip_tokenizer_init is True")
 
         return tokenizer.encode(request_id=request_id,
                                 prompt=prompt,

vllm/engine/output_processor/interfaces.py

@@ -1,13 +1,12 @@
 from abc import ABC, abstractmethod
 from typing import Callable, List
 
-from transformers import PreTrainedTokenizer
-
 from vllm.config import SchedulerConfig
 from vllm.core.scheduler import Scheduler
 from vllm.engine.output_processor.stop_checker import StopChecker
 from vllm.sequence import Sequence, SequenceGroup, SequenceGroupOutput
 from vllm.transformers_utils.detokenizer import Detokenizer
+from vllm.transformers_utils.tokenizer import AnyTokenizer
 from vllm.utils import Counter
 
 
@@ -29,7 +28,7 @@ def create_output_processor(
         detokenizer: Detokenizer,
         scheduler: List[Scheduler],
         seq_counter: Counter,
-        get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
+        get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer],
         stop_checker: "StopChecker",
     ):
         """Create an output processor.

vllm/engine/output_processor/multi_step.py

@@ -1,8 +1,6 @@
 import functools
 from typing import Callable, List
 
-from transformers import PreTrainedTokenizer
-
 from vllm.core.scheduler import Scheduler
 from vllm.engine.output_processor.interfaces import (
     SequenceGroupOutputProcessor)
@@ -12,6 +10,7 @@
 from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
                            SequenceOutput, SequenceStatus)
 from vllm.transformers_utils.detokenizer import Detokenizer
+from vllm.transformers_utils.tokenizer import AnyTokenizer
 from vllm.utils import Counter
 
 logger = init_logger(__name__)
@@ -36,7 +35,7 @@ def __init__(
         detokenizer: Detokenizer,
         scheduler: List[Scheduler],
         seq_counter: Counter,
-        get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
+        get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer],
         stop_checker: StopChecker,
     ):
         self.detokenizer = detokenizer

vllm/engine/output_processor/stop_checker.py

@@ -1,10 +1,9 @@
 from typing import Callable, Optional
 
-from transformers import PreTrainedTokenizer
-
 from vllm.lora.request import LoRARequest
 from vllm.sampling_params import SamplingParams
 from vllm.sequence import Sequence, SequenceStatus
+from vllm.transformers_utils.tokenizer import AnyTokenizer
 
 
 class StopChecker:
@@ -15,8 +14,7 @@ class StopChecker:
     """
 
     def __init__(self, max_model_len: int,
-                 get_tokenizer_for_seq: Callable[[Sequence],
-                                                 PreTrainedTokenizer]):
+                 get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer]):
         # Do not use it directly, but use `self._get_max_model_len`.
         self._max_model_len = max_model_len
         self.get_tokenizer_for_seq = get_tokenizer_for_seq

vllm/engine/protocol.py

@@ -1,8 +1,6 @@
 from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
                     runtime_checkable)
 
-from transformers import PreTrainedTokenizer
-
 from vllm.config import DecodingConfig, ModelConfig
 from vllm.core.scheduler import SchedulerOutputs
 from vllm.inputs.data import PromptInputs
@@ -12,6 +10,7 @@
 from vllm.prompt_adapter.request import PromptAdapterRequest
 from vllm.sampling_params import SamplingParams
 from vllm.sequence import SamplerOutput
+from vllm.transformers_utils.tokenizer import AnyTokenizer
 
 
 @runtime_checkable
@@ -40,6 +39,7 @@ def generate(
         prompt_adapter_request: Optional[PromptAdapterRequest] = None
     ) -> AsyncGenerator[RequestOutput, None]:
         """Generates outputs for a request"""
+        ...
 
     def encode(
         self,
@@ -50,6 +50,7 @@ def encode(
         trace_headers: Optional[Mapping[str, str]] = None,
     ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
         """Generate outputs for a request from an embedding model."""
+        ...
 
     async def abort(self, request_id: str) -> None:
         """Abort a request.
@@ -60,25 +61,29 @@ async def abort(self, request_id: str) -> None:
 
     async def get_model_config(self) -> ModelConfig:
         """Get the model configuration of the vLLM engine."""
+        ...
 
     async def get_decoding_config(self) -> DecodingConfig:
+        ...
         """Get the decoding configuration of the vLLM engine."""
 
     async def get_tokenizer(
         self,
         lora_request: Optional[LoRARequest] = None,
-    ) -> PreTrainedTokenizer:
-        """Get the appropriate Tokenizer for the request"""
+    ) -> AnyTokenizer:
+        """Get the appropriate tokenizer for the request"""
+        ...
 
     async def is_tracing_enabled(self) -> bool:
-        pass
+        ...
 
     async def do_log_stats(
         self,
         scheduler_outputs: Optional[SchedulerOutputs] = None,
         model_output: Optional[List[SamplerOutput]] = None,
     ) -> None:
-        pass
+        ...
 
     async def check_health(self) -> None:
         """Raise if unhealthy"""
+        ...

vllm/entrypoints/api_server.py

@@ -61,6 +61,7 @@ async def generate(request: Request) -> Response:
     async def stream_results() -> AsyncGenerator[bytes, None]:
         async for request_output in results_generator:
             prompt = request_output.prompt
+            assert prompt is not None
             text_outputs = [
                 prompt + output.text for output in request_output.outputs
             ]
@@ -80,6 +81,7 @@ async def stream_results() -> AsyncGenerator[bytes, None]:
 
     assert final_output is not None
     prompt = final_output.prompt
+    assert prompt is not None
     text_outputs = [prompt + output.text for output in final_output.outputs]
     ret = {"text": text_outputs}
     return JSONResponse(ret)
@@ -115,6 +117,7 @@ async def run_server(args: Namespace,
     logger.info("args: %s", args)
 
     app = await init_app(args, llm_engine)
+    assert engine is not None
 
     shutdown_task = await serve_http(
         app,