Fix added tokens config with sensible filter

python/sglang/srt/lora/lora.py

@@ -128,8 +128,8 @@ def _process_weight(self, name: str, loaded_weight: torch.Tensor):
             # added/extra token emb
             self.added_tokens_embeddings[name] = loaded_weight.cpu()
             assert loaded_weight.shape[0] == self.config.lora_added_tokens_size, (
-                f"LoRA adapter {self.uid} has extra_vocab_size {self.config.extra_vocab_size} specified in the config, "
-                f"but the loaded weight has {loaded_weight.shape[0]} extra vocab size"
+                f"LoRA adapter {self.uid} has lora_added_tokens_size {self.config.lora_added_tokens_size} specified in the config, "
+                f"but the loaded weight '{name}' has shape {loaded_weight.shape[0]} in first dimension"
             )
 
     def _normalize_weights(self):

python/sglang/srt/lora/lora_config.py

@@ -13,11 +13,14 @@
 # ==============================================================================
 
 import json
+import logging
 import os
 from typing import Dict, Optional
 
 from huggingface_hub import snapshot_download
 
+logger = logging.getLogger(__name__)
+
 
 class LoRAConfig:
     def __init__(
@@ -42,6 +45,35 @@ def __init__(
             len(self.added_tokens_config) if self.added_tokens_config is not None else 0
         )
 
+    def filter_added_tokens(self, base_vocab_size: int) -> None:
+        """
+        Filter added_tokens_config to only include truly added tokens.
+
+        Tokens with ID < base_vocab_size are already part of the base model's
+        vocabulary and should not be treated as added tokens. This commonly
+        happens when added_tokens.json is copied from the base model's tokenizer.
+
+        Args:
+            base_vocab_size: The vocabulary size of the base model.
+        """
+        if not self.added_tokens_config:
+            return
+
+        original_count = len(self.added_tokens_config)
+        self.added_tokens_config = {
+            token: token_id
+            for token, token_id in self.added_tokens_config.items()
+            if token_id >= base_vocab_size
+        }
+        self.lora_added_tokens_size = len(self.added_tokens_config)
+
+        filtered_count = original_count - self.lora_added_tokens_size
+        if filtered_count > 0:
+            logger.debug(
+                f"Filtered {filtered_count} tokens from added_tokens_config "
+                f"(ID < {base_vocab_size}). Remaining: {self.lora_added_tokens_size}"
+            )
+
     @classmethod
     def from_dict(
         cls,

python/sglang/srt/lora/lora_manager.py

@@ -135,6 +135,8 @@ def load_lora_adapter(self, lora_ref: LoRARef) -> LoRAUpdateOutput:
         try:
             # load configs
             new_adapter = LoRAConfig(lora_ref.lora_path)
+            # Filter fake added tokens before validation (ID >= base vocab_size)
+            new_adapter.filter_added_tokens(self.base_hf_config.vocab_size)
             self.validate_new_adapter(new_adapter, lora_ref)
             self.configs[lora_ref.lora_id] = new_adapter
 
@@ -489,6 +491,11 @@ def init_lora_shapes(
                 default=0,
             )
 
+        # Filter fake added tokens from startup adapters (ID >= base vocab_size)
+        base_vocab_size = self.base_hf_config.vocab_size
+        for config in self.configs.values():
+            config.filter_added_tokens(base_vocab_size)
+
         # Auto-infer self.lora_added_vocab_size from loaded LoRA configs
         # This happens automatically without requiring user input
         # if self.lora_added_vocab_size is None:
@@ -561,6 +568,8 @@ def load_lora_adapter_from_tensors(
 
         try:
             new_adapter = LoRAConfig.from_dict(config_dict, added_tokens_config)
+            # Filter fake added tokens before validation (ID >= base vocab_size)
+            new_adapter.filter_added_tokens(self.base_hf_config.vocab_size)
             self.validate_new_adapter(new_adapter, lora_ref)
             self.configs[lora_ref.lora_id] = new_adapter
 

test/registered/lora/test_lora_config.py

@@ -0,0 +1,138 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Unit tests for LoRAConfig functionality."""
+
+import unittest
+
+from sglang.srt.lora.lora_config import LoRAConfig
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=10, suite="stage-a-cpu-only")
+
+
+class TestLoRAConfigFilterAddedTokens(unittest.TestCase):
+    """Test cases for LoRAConfig.filter_added_tokens method."""
+
+    def _create_config_with_added_tokens(self, added_tokens: dict) -> LoRAConfig:
+        """Helper to create a LoRAConfig with specified added_tokens."""
+        config_dict = {
+            "target_modules": ["q_proj", "k_proj", "v_proj"],
+            "r": 8,
+            "lora_alpha": 16,
+        }
+        return LoRAConfig.from_dict(config_dict, added_tokens_config=added_tokens)
+
+    def test_filter_no_added_tokens(self):
+        """Test filtering when there are no added tokens."""
+        config = self._create_config_with_added_tokens(None)
+        self.assertEqual(config.lora_added_tokens_size, 0)
+
+        config.filter_added_tokens(base_vocab_size=32000)
+        self.assertEqual(config.lora_added_tokens_size, 0)
+        self.assertIsNone(config.added_tokens_config)
+
+    def test_filter_empty_added_tokens(self):
+        """Test filtering when added_tokens is empty dict."""
+        config = self._create_config_with_added_tokens({})
+        self.assertEqual(config.lora_added_tokens_size, 0)
+
+        config.filter_added_tokens(base_vocab_size=32000)
+        self.assertEqual(config.lora_added_tokens_size, 0)
+
+    def test_filter_all_fake_tokens(self):
+        """Test filtering when all tokens are fake (ID < base_vocab_size)."""
+        # These tokens have IDs less than base_vocab_size (32000)
+        added_tokens = {
+            "<pad>": 0,
+            "<eos>": 2,
+            "<bos>": 1,
+        }
+        config = self._create_config_with_added_tokens(added_tokens)
+        self.assertEqual(config.lora_added_tokens_size, 3)
+
+        config.filter_added_tokens(base_vocab_size=32000)
+        self.assertEqual(config.lora_added_tokens_size, 0)
+        self.assertEqual(config.added_tokens_config, {})
+
+    def test_filter_all_real_tokens(self):
+        """Test filtering when all tokens are real (ID >= base_vocab_size)."""
+        base_vocab_size = 32000
+        # These tokens have IDs >= base_vocab_size
+        added_tokens = {
+            "<new_token_1>": 32000,
+            "<new_token_2>": 32001,
+            "<new_token_3>": 32002,
+        }
+        config = self._create_config_with_added_tokens(added_tokens)
+        self.assertEqual(config.lora_added_tokens_size, 3)
+
+        config.filter_added_tokens(base_vocab_size=base_vocab_size)
+        self.assertEqual(config.lora_added_tokens_size, 3)
+        self.assertEqual(config.added_tokens_config, added_tokens)
+
+    def test_filter_mixed_tokens(self):
+        """Test filtering with both fake and real tokens."""
+        base_vocab_size = 32000
+        added_tokens = {
+            # Fake tokens (ID < base_vocab_size)
+            "<pad>": 0,
+            "<eos>": 2,
+            # Real tokens (ID >= base_vocab_size)
+            "<new_token_1>": 32000,
+            "<new_token_2>": 32001,
+        }
+        config = self._create_config_with_added_tokens(added_tokens)
+        self.assertEqual(config.lora_added_tokens_size, 4)
+
+        config.filter_added_tokens(base_vocab_size=base_vocab_size)
+        self.assertEqual(config.lora_added_tokens_size, 2)
+        self.assertEqual(
+            config.added_tokens_config,
+            {"<new_token_1>": 32000, "<new_token_2>": 32001},
+        )
+
+    def test_filter_boundary_token(self):
+        """Test token exactly at base_vocab_size boundary."""
+        base_vocab_size = 32000
+        added_tokens = {
+            "<at_boundary>": 32000,  # Exactly at boundary, should be kept
+            "<below_boundary>": 31999,  # Just below, should be filtered
+        }
+        config = self._create_config_with_added_tokens(added_tokens)
+
+        config.filter_added_tokens(base_vocab_size=base_vocab_size)
+        self.assertEqual(config.lora_added_tokens_size, 1)
+        self.assertEqual(config.added_tokens_config, {"<at_boundary>": 32000})
+
+    def test_filter_idempotent(self):
+        """Test that calling filter_added_tokens multiple times is safe."""
+        base_vocab_size = 32000
+        added_tokens = {
+            "<fake>": 100,
+            "<real>": 32000,
+        }
+        config = self._create_config_with_added_tokens(added_tokens)
+
+        # First call
+        config.filter_added_tokens(base_vocab_size=base_vocab_size)
+        self.assertEqual(config.lora_added_tokens_size, 1)
+
+        # Second call should not change anything
+        config.filter_added_tokens(base_vocab_size=base_vocab_size)
+        self.assertEqual(config.lora_added_tokens_size, 1)
+        self.assertEqual(config.added_tokens_config, {"<real>": 32000})
+
+
+if __name__ == "__main__":
+    unittest.main(warnings="ignore")