diff --git a/tests/model_executor/model_loader/test_attn_sink_reload.py b/tests/model_executor/model_loader/test_attn_sink_reload.py new file mode 100644 index 000000000000..73d85297a447 --- /dev/null +++ b/tests/model_executor/model_loader/test_attn_sink_reload.py @@ -0,0 +1,360 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Layerwise reload invariants for checkpoint-backed attention sinks. + +The defect these tests guard against is a model ``load_weights`` that writes a +sink with a bare ``copy_``. Such a write is invisible to +``online_process_loader``, so during a live update it lands on the temporary +parameter that the reload materializes and is then discarded when the saved +kernel tensor is put back. Testing a mock ``weight_loader`` invocation would not +catch that, so these tests drive the real layerwise reload lifecycle. + +The tests drive a real layerwise reload and check the resulting sink values, +padding, and parameter storage. +""" + +from collections.abc import Iterable +from types import SimpleNamespace + +import pytest +import torch +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink + +from vllm.model_executor.model_loader.reload import layerwise +from vllm.model_executor.model_loader.reload.layerwise import ( + finalize_layerwise_reload, + initialize_layerwise_reload, + record_metadata_for_reloading, +) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader + +PADDED_HEADS = 8 +LOCAL_HEADS = 4 +GLOBAL_HEADS = 16 +PROJ_SIZE = 256 +# The fake layers only require dtype during deferred attention finalization. +MODEL_CONFIG = SimpleNamespace(dtype=torch.float32) + + +def _make_padded_sink() -> torch.nn.Parameter: + sink = torch.nn.Parameter( + torch.full((PADDED_HEADS,), -float("inf"), dtype=torch.float32), + requires_grad=False, + ) + sink.weight_loader = default_weight_loader + return sink + + +def _checkpoint_a() -> list[tuple[str, torch.Tensor]]: + return [ + ("sink", torch.arange(GLOBAL_HEADS, dtype=torch.float32)), + ("proj", torch.arange(PROJ_SIZE, dtype=torch.float32)), + ] + + +def _checkpoint_b() -> list[tuple[str, torch.Tensor]]: + return [ + ("sink", -torch.arange(GLOBAL_HEADS, dtype=torch.float32) - 1.0), + ("proj", -torch.arange(PROJ_SIZE, dtype=torch.float32) - 1.0), + ] + + +def _expected_local(weights: list[tuple[str, torch.Tensor]], start: int, end: int): + sink = next(w for n, w in weights if n == "sink") + return sink[start:end] + + +class _SinkLayer(torch.nn.Module): + """Minimal layer with the sink loading shape the models use.""" + + def __init__(self) -> None: + super().__init__() + self.sink = _make_padded_sink() + self.proj = torch.nn.Parameter(torch.zeros(PROJ_SIZE)) + self.proj.weight_loader = default_weight_loader + # Sinks are consumed by attention backends, which defer processing. + self.process_weights_after_loading = lambda dtype: None + + def load_weights( + self, + weights: Iterable[tuple[str, torch.Tensor]], + head_start: int, + head_end: int, + ) -> set[str]: + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if "sink" in name: + param = params_dict[name] + load_padded_attn_sink(param, loaded_weight, head_start, head_end) + loaded_params.add(name) + continue + weight_loader = getattr( + params_dict[name], "weight_loader", default_weight_loader + ) + weight_loader(params_dict[name], loaded_weight) + loaded_params.add(name) + return loaded_params + + +class _ExactShapeSinkLayer(torch.nn.Module): + """Local runtime sink shape used by HY-V4, MiMo-V2, and GPT-OSS.""" + + def __init__(self) -> None: + super().__init__() + self.sink = torch.nn.Parameter(torch.zeros(LOCAL_HEADS), requires_grad=False) + self.sink.weight_loader = default_weight_loader + + def load_weights( + self, loaded_weight: torch.Tensor, head_start: int, head_end: int + ) -> None: + param = self.sink + local_weight = loaded_weight[head_start:head_end] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, local_weight) + + +def _live_reload(layer, head_start: int, head_end: int, weights) -> None: + """Drive the real layerwise reload lifecycle for one weight update.""" + record_metadata_for_reloading(layer) + initialize_layerwise_reload(layer) + layer.load_weights(weights, head_start, head_end) + finalize_layerwise_reload(layer, MODEL_CONFIG) + + +@pytest.mark.parametrize("tp_rank", [0, 1]) +def test_padded_sink_live_update_reaches_kernel_tensor(tp_rank: int): + """A live update must land in kernel tensor storage with padding intact.""" + head_start = LOCAL_HEADS * tp_rank + head_end = head_start + LOCAL_HEADS + + layer = _SinkLayer() + layer.load_weights(_checkpoint_a(), head_start, head_end) + kernel_ptr = layer.sink.data_ptr() + + _live_reload(layer, head_start, head_end, _checkpoint_b()) + + expected = _expected_local(_checkpoint_b(), head_start, head_end) + assert torch.equal(layer.sink.data[:LOCAL_HEADS], expected) + assert torch.isneginf(layer.sink.data[LOCAL_HEADS:]).all() + assert layer.sink.data_ptr() == kernel_ptr + + +def test_padded_sink_padding_tail_not_polluted_after_reload(): + """Polluted padding must be restored to -inf by the live update itself.""" + layer = _SinkLayer() + layer.load_weights(_checkpoint_a(), 0, LOCAL_HEADS) + with torch.no_grad(): + layer.sink.data[LOCAL_HEADS:] = 0.0 + + _live_reload(layer, 0, LOCAL_HEADS, _checkpoint_b()) + + assert torch.isneginf(layer.sink.data[LOCAL_HEADS:]).all() + + +def test_padded_sink_survives_repeated_reloads(): + """Reloading twice keeps loading through the restored original loader.""" + layer = _SinkLayer() + layer.load_weights(_checkpoint_a(), 0, LOCAL_HEADS) + kernel_ptr = layer.sink.data_ptr() + + _live_reload(layer, 0, LOCAL_HEADS, _checkpoint_b()) + _live_reload(layer, 0, LOCAL_HEADS, _checkpoint_a()) + + expected = _expected_local(_checkpoint_a(), 0, LOCAL_HEADS) + assert torch.equal(layer.sink.data[:LOCAL_HEADS], expected) + assert torch.isneginf(layer.sink.data[LOCAL_HEADS:]).all() + assert layer.sink.data_ptr() == kernel_ptr + + +@pytest.mark.parametrize("tp_rank", [0, 1]) +def test_padded_sink_deferred_attention_finalize(monkeypatch, tp_rank: int): + """Deferred attention replays the sink at finalize into stable storage.""" + head_start = LOCAL_HEADS * tp_rank + head_end = head_start + LOCAL_HEADS + layer = _SinkLayer() + layer.load_weights(_checkpoint_a(), head_start, head_end) + kernel_ptr = layer.sink.data_ptr() + post_load_dtypes: list[torch.dtype] = [] + layer.process_weights_after_loading = post_load_dtypes.append + monkeypatch.setattr( + layerwise, "is_deferred_attention_layer", lambda candidate: candidate is layer + ) + + record_metadata_for_reloading(layer) + initialize_layerwise_reload(layer) + layer.load_weights(_checkpoint_b(), head_start, head_end) + assert not post_load_dtypes # Deferred until _finalize_attention_layer. + finalize_layerwise_reload(layer, MODEL_CONFIG) + + assert post_load_dtypes == [torch.float32] + assert torch.equal( + layer.sink.data[:LOCAL_HEADS], + _expected_local(_checkpoint_b(), head_start, head_end), + ) + assert torch.isneginf(layer.sink.data[LOCAL_HEADS:]).all() + assert layer.sink.data_ptr() == kernel_ptr + + +@pytest.mark.parametrize("tp_rank", [0, 1]) +def test_exact_shape_sink_live_reload(tp_rank: int): + """A TP-local sink without padding also reloads into stable storage.""" + head_start = LOCAL_HEADS * tp_rank + head_end = head_start + LOCAL_HEADS + checkpoint_a = _checkpoint_a()[0][1] + checkpoint_b = _checkpoint_b()[0][1] + layer = _ExactShapeSinkLayer() + layer.load_weights(checkpoint_a, head_start, head_end) + kernel_ptr = layer.sink.data_ptr() + + _live_reload(layer, head_start, head_end, checkpoint_b) + + assert torch.equal(layer.sink.data, checkpoint_b[head_start:head_end]) + assert layer.sink.data_ptr() == kernel_ptr + + +def _make_param(padded_heads: int = PADDED_HEADS) -> torch.nn.Parameter: + return torch.nn.Parameter( + torch.full((padded_heads,), -float("inf"), dtype=torch.float32), + requires_grad=False, + ) + + +def _checkpoint() -> torch.Tensor: + return torch.arange(GLOBAL_HEADS, dtype=torch.float32) + + +@pytest.mark.parametrize("tp_rank", [0, 1, 2, 3]) +def test_load_padded_attn_sink_writes_local_heads_and_keeps_padding_at_neginf( + tp_rank: int, +): + """Every rank takes its own heads and keeps the padded tail disabled.""" + param = _make_param() + loaded_weight = _checkpoint() + head_start = LOCAL_HEADS * tp_rank + head_end = head_start + LOCAL_HEADS + + load_padded_attn_sink(param, loaded_weight, head_start, head_end) + + expected_local = loaded_weight[head_start:head_end] + assert torch.equal(param.data[:LOCAL_HEADS], expected_local) + assert torch.isneginf(param.data[LOCAL_HEADS:]).all() + + +def test_load_padded_attn_sink_resets_polluted_padding_tail(): + """A reload must restore the padding tail even if it was polluted.""" + param = _make_param() + with torch.no_grad(): + param.data[LOCAL_HEADS:] = 0.0 + + load_padded_attn_sink(param, _checkpoint(), 0, LOCAL_HEADS) + + assert torch.isneginf(param.data[LOCAL_HEADS:]).all() + + +def test_load_padded_attn_sink_preserves_parameter_storage(): + """The helper must load in place so kernel tensor references stay valid.""" + param = _make_param() + ptr = param.data.data_ptr() + + load_padded_attn_sink(param, _checkpoint(), 0, LOCAL_HEADS) + + assert param.data.data_ptr() == ptr + + +def test_load_padded_attn_sink_uses_current_weight_loader(): + """The load must be routed through the parameter's current loader. + + Layerwise reload replaces ``weight_loader`` with ``online_process_loader``; + reading it dynamically is what makes a live update observable. + """ + param = _make_param() + calls = [] + + def recording_loader(p, w): + calls.append((p, w)) + + param.weight_loader = recording_loader + + load_padded_attn_sink(param, _checkpoint(), 0, LOCAL_HEADS) + + assert len(calls) == 1 + called_param, called_weight = calls[0] + assert called_param is param + # The loader receives the full padded runtime representation so that the + # element counts match and the replay re-establishes the padding. + assert called_weight.shape == param.shape + assert torch.equal(called_weight[:LOCAL_HEADS], _checkpoint()[:LOCAL_HEADS]) + assert torch.isneginf(called_weight[LOCAL_HEADS:]).all() + + +def test_load_padded_attn_sink_degenerate_head_range(): + """A rank with no heads keeps the whole sink disabled and stays loadable.""" + param = _make_param() + + load_padded_attn_sink(param, _checkpoint(), 0, 0) + + assert torch.isneginf(param.data).all() + + +def test_load_padded_attn_sink_builds_runtime_weight_in_param_dtype(): + """The reconstructed tensor must match the parameter, not the checkpoint. + + Padding is written by the load, so a dtype mismatch between the checkpoint + sink and the runtime parameter must be resolved before the load. + """ + param = _make_param().to(torch.bfloat16) + loader_inputs = [] + param.weight_loader = lambda p, w: loader_inputs.append(w) + + load_padded_attn_sink(param, _checkpoint(), 0, LOCAL_HEADS) + + assert loader_inputs[0].dtype == torch.bfloat16 + assert torch.equal( + loader_inputs[0][:LOCAL_HEADS], _checkpoint()[:LOCAL_HEADS].to(torch.bfloat16) + ) + assert torch.isneginf(loader_inputs[0][LOCAL_HEADS:]).all() + + +def test_load_padded_attn_sink_rejects_head_range_wider_than_param(): + """A rank claiming more heads than the padded parameter must fail loudly. + + Silently truncating would load the wrong heads, and the online loader + rejects a source larger than its destination anyway. + """ + param = _make_param(padded_heads=2) + + with pytest.raises(ValueError, match="does not fit the runtime sink parameter"): + load_padded_attn_sink(param, _checkpoint(), 0, LOCAL_HEADS) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") +@pytest.mark.parametrize("tp_rank", [0, 1]) +def test_padded_sink_live_update_reaches_kernel_tensor_on_cuda(tp_rank: int): + """A live update must survive the reload when the sink lives on a device. + + The reload moves real storage: the parameter is restored onto the meta + device and materialized again before the kernel tensor is refreshed. On CPU + that is incidental; on CUDA the update has to traverse a device boundary to + reach the tensor the attention kernel reads. + """ + device = torch.device("cuda") + head_start = LOCAL_HEADS * tp_rank + head_end = head_start + LOCAL_HEADS + + def to_device(weights): + return [(name, weight.to(device)) for name, weight in weights] + + layer = _SinkLayer().to(device) + layer.load_weights(to_device(_checkpoint_a()), head_start, head_end) + kernel_ptr = layer.sink.data_ptr() + + _live_reload(layer, head_start, head_end, to_device(_checkpoint_b())) + + assert layer.sink.data_ptr() == kernel_ptr + assert layer.sink.is_cuda + assert torch.equal( + layer.sink.data[:LOCAL_HEADS], + _expected_local(_checkpoint_b(), head_start, head_end).to(device), + ) + assert torch.isneginf(layer.sink.data[LOCAL_HEADS:]).all() diff --git a/vllm/models/deepseek_v4/amd/dspark.py b/vllm/models/deepseek_v4/amd/dspark.py index c3f249fa80d5..239f934ae506 100644 --- a/vllm/models/deepseek_v4/amd/dspark.py +++ b/vllm/models/deepseek_v4/amd/dspark.py @@ -42,6 +42,7 @@ ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.qwen3_dspark import ( DSparkConfidenceHead, @@ -473,8 +474,12 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: break else: if "attn_sink" in name: - narrow = loaded_weight[head_start:head_end] - params_dict[name][: narrow.shape[0]].copy_(narrow) + load_padded_attn_sink( + params_dict[name], + loaded_weight, + head_start, + head_end, + ) loaded_params.add(name) continue if ".shared_experts.w2" in name: diff --git a/vllm/models/deepseek_v4/amd/model.py b/vllm/models/deepseek_v4/amd/model.py index f8306dd12c23..158009eb0397 100644 --- a/vllm/models/deepseek_v4/amd/model.py +++ b/vllm/models/deepseek_v4/amd/model.py @@ -56,6 +56,7 @@ ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import ( EagleModelMixin, @@ -1280,9 +1281,12 @@ def _resolve_param_name(name: str) -> str: elif "attn_sink" in name: if is_pp_missing_parameter(name, self): continue - narrow_weight = loaded_weight[head_rank_start:head_rank_end] - n = narrow_weight.shape[0] - params_dict[name][:n].copy_(narrow_weight) + load_padded_attn_sink( + params_dict[name], + loaded_weight, + head_rank_start, + head_rank_end, + ) loaded_params.add(name) continue else: diff --git a/vllm/models/deepseek_v4/amd/mtp.py b/vllm/models/deepseek_v4/amd/mtp.py index 045a731321b9..4e0b817acb4e 100644 --- a/vllm/models/deepseek_v4/amd/mtp.py +++ b/vllm/models/deepseek_v4/amd/mtp.py @@ -32,6 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink from vllm.model_executor.model_loader.mtp_validation import ( is_mtp_completeness_check_enabled, ) @@ -454,9 +455,12 @@ def _resolve_scale_name(name: str) -> str: break continue elif "attn_sink" in name: - narrow_weight = loaded_weight[head_rank_start:head_rank_end] - n = narrow_weight.shape[0] - params_dict[name][:n].copy_(narrow_weight) + load_padded_attn_sink( + params_dict[name], + loaded_weight, + head_rank_start, + head_rank_end, + ) loaded_params.add(name) continue else: diff --git a/vllm/models/deepseek_v4/cpu/model.py b/vllm/models/deepseek_v4/cpu/model.py index 6e88d21c2794..f2fbb9e69aaa 100644 --- a/vllm/models/deepseek_v4/cpu/model.py +++ b/vllm/models/deepseek_v4/cpu/model.py @@ -43,6 +43,7 @@ ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import ( EagleModelMixin, @@ -758,9 +759,12 @@ def _resolve_param_name(name: str) -> str: elif "attn_sink" in name: if is_pp_missing_parameter(name, self): continue - narrow_weight = loaded_weight[head_rank_start:head_rank_end] - n = narrow_weight.shape[0] - params_dict[name][:n].copy_(narrow_weight) + load_padded_attn_sink( + params_dict[name], + loaded_weight, + head_rank_start, + head_rank_end, + ) loaded_params.add(name) continue else: diff --git a/vllm/models/deepseek_v4/nvidia/dspark.py b/vllm/models/deepseek_v4/nvidia/dspark.py index 987909702cf4..6003b54718d6 100644 --- a/vllm/models/deepseek_v4/nvidia/dspark.py +++ b/vllm/models/deepseek_v4/nvidia/dspark.py @@ -41,6 +41,7 @@ ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.qwen3_dspark import ( DSparkConfidenceHead, @@ -543,8 +544,12 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: break else: if "attn_sink" in name: - narrow = loaded_weight[head_start:head_end] - params_dict[name][: narrow.shape[0]].copy_(narrow) + load_padded_attn_sink( + params_dict[name], + loaded_weight, + head_start, + head_end, + ) loaded_params.add(name) continue if name.endswith(".ffn.gate.bias"): diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index 7edff8ae0202..6c25c0b112ba 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -51,6 +51,7 @@ ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import ( EagleModelMixin, @@ -1750,9 +1751,12 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: elif "attn_sink" in name: if is_pp_missing_parameter(name, self): continue - narrow_weight = loaded_weight[head_rank_start:head_rank_end] - n = narrow_weight.shape[0] - params_dict[name][:n].copy_(narrow_weight) + load_padded_attn_sink( + params_dict[name], + loaded_weight, + head_rank_start, + head_rank_end, + ) loaded_params.add(name) continue else: diff --git a/vllm/models/deepseek_v4/nvidia/mtp.py b/vllm/models/deepseek_v4/nvidia/mtp.py index 50886b0afab7..c145651b546a 100644 --- a/vllm/models/deepseek_v4/nvidia/mtp.py +++ b/vllm/models/deepseek_v4/nvidia/mtp.py @@ -40,6 +40,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink from vllm.model_executor.model_loader.mtp_validation import ( is_mtp_completeness_check_enabled, ) @@ -487,9 +488,12 @@ def _find_mtp_layer_idx(name: str) -> int: break continue elif "attn_sink" in name: - narrow_weight = loaded_weight[head_rank_start:head_rank_end] - n = narrow_weight.shape[0] - params_dict[name][:n].copy_(narrow_weight) + load_padded_attn_sink( + params_dict[name], + loaded_weight, + head_rank_start, + head_rank_end, + ) loaded_params.add(name) continue else: diff --git a/vllm/models/deepseek_v4/xpu/dspark.py b/vllm/models/deepseek_v4/xpu/dspark.py index 63333a9c0a2e..1afffc90fcfd 100644 --- a/vllm/models/deepseek_v4/xpu/dspark.py +++ b/vllm/models/deepseek_v4/xpu/dspark.py @@ -31,6 +31,7 @@ ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead from vllm.model_executor.models.utils import maybe_prefix @@ -386,8 +387,12 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: if "attn_sink" in name: if name not in params_dict: continue - narrow = loaded_weight[head_start:head_end] - params_dict[name][: narrow.shape[0]].copy_(narrow) + load_padded_attn_sink( + params_dict[name], + loaded_weight, + head_start, + head_end, + ) loaded_params.add(name) continue if ".shared_experts.w2" in name: diff --git a/vllm/models/deepseek_v4/xpu/model.py b/vllm/models/deepseek_v4/xpu/model.py index f3e468ebb1f5..8b05aee4ff1c 100644 --- a/vllm/models/deepseek_v4/xpu/model.py +++ b/vllm/models/deepseek_v4/xpu/model.py @@ -42,6 +42,7 @@ ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import ( EagleModelMixin, @@ -1222,9 +1223,12 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: elif "attn_sink" in name: if is_pp_missing_parameter(name, self): continue - narrow_weight = loaded_weight[head_rank_start:head_rank_end] - n = narrow_weight.shape[0] - params_dict[name][:n].copy_(narrow_weight) + load_padded_attn_sink( + params_dict[name], + loaded_weight, + head_rank_start, + head_rank_end, + ) loaded_params.add(name) continue else: diff --git a/vllm/models/deepseek_v4/xpu/mtp.py b/vllm/models/deepseek_v4/xpu/mtp.py index a19d0d578220..edd082ab4037 100644 --- a/vllm/models/deepseek_v4/xpu/mtp.py +++ b/vllm/models/deepseek_v4/xpu/mtp.py @@ -32,6 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.attention_sink import load_padded_attn_sink from vllm.model_executor.model_loader.mtp_validation import ( is_mtp_completeness_check_enabled, ) @@ -442,9 +443,12 @@ def _find_mtp_layer_idx(name: str) -> int: break continue elif "attn_sink" in name: - narrow_weight = loaded_weight[head_rank_start:head_rank_end] - n = narrow_weight.shape[0] - params_dict[name][:n].copy_(narrow_weight) + load_padded_attn_sink( + params_dict[name], + loaded_weight, + head_rank_start, + head_rank_end, + ) loaded_params.add(name) continue else: