Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion tools/pre_commit/mypy.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,8 @@
r"vllm/model_executor/models/[qQ]",
r"vllm/model_executor/models/[rR]",
r"vllm/model_executor/models/[sS]",
r"vllm/model_executor/models/[tT]",
# `[tT]` is otherwise clean; the `transformers` package is still excluded.
r"vllm/model_executor/models/transformers",

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
# `[tT]` is otherwise clean; the `transformers` package is still excluded.
r"vllm/model_executor/models/transformers",
r"vllm/model_executor/models/transformers",

Comment not needed

r"vllm/model_executor/models/[uU]",
r"vllm/model_executor/models/[vV]",
r"vllm/model_executor/models/[wW]",
Expand Down
21 changes: 14 additions & 7 deletions vllm/model_executor/models/telechat2.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from vllm.config import VllmConfig
from vllm.model_executor.models.llama import LlamaForCausalLM, LlamaModel

from .llama import LlamaDecoderLayer
from .llama import LlamaAttention, LlamaDecoderLayer, LlamaMLP
from .utils import AutoWeightsLoader, PPMissingLayer, WeightsMapper


Expand All @@ -46,7 +46,9 @@ class TeleChat2Model(LlamaModel):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
hf_config = vllm_config.model_config.hf_config

vllm_config.model_config.hf_config.attribute_map = {
# `attribute_map` is a ClassVar on `PretrainedConfig`; shadowing it on the
# instance is intentional here so LlamaModel can read TeleChat2's fields.
vllm_config.model_config.hf_config.attribute_map = { # type: ignore[misc]

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
# `attribute_map` is a ClassVar on `PretrainedConfig`; shadowing it on the
# instance is intentional here so LlamaModel can read TeleChat2's fields.
vllm_config.model_config.hf_config.attribute_map = { # type: ignore[misc]
vllm_config.model_config.hf_config.attribute_map = { # type: ignore[misc]

comment not needed

"num_hidden_layers": "n_layer",
"num_attention_heads": "n_head",
"intermediate_size": "ffn_hidden_size",
Expand All @@ -63,11 +65,16 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
# Telechat2's gate_up_proj and qkv_proj don't have bias
# see: https://github.com/vllm-project/vllm/pull/10311#issuecomment-2490297566
for layer in self.layers:
if not isinstance(layer, PPMissingLayer):
layer.self_attn.qkv_proj.bias = None
layer.self_attn.qkv_proj.skip_bias_add = True
layer.mlp.gate_up_proj.bias = None
layer.mlp.gate_up_proj.skip_bias_add = True
if isinstance(layer, PPMissingLayer):
continue
self_attn = layer.self_attn
mlp = layer.mlp
assert isinstance(self_attn, LlamaAttention)
assert isinstance(mlp, LlamaMLP)
self_attn.qkv_proj.register_parameter("bias", None)
self_attn.qkv_proj.skip_bias_add = True
mlp.gate_up_proj.register_parameter("bias", None)
mlp.gate_up_proj.skip_bias_add = True

def _split_key_value(
self, weights: Iterable[tuple[str, torch.Tensor]]
Expand Down
2 changes: 1 addition & 1 deletion vllm/model_executor/models/terratorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ def forward(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_list = []
model_buffers = dict(self.named_buffers())
loaded_buffers = []
loaded_buffers: list[str] = []
for key, value in weights:
if isinstance(value, (dict, OrderedDict)):
if key == "state_dict":
Expand Down
Loading