fix(tokenizer): register deepseek_v32 config alias for DeepSeek-V3.2-Exp

docs/reference/tokenizer-auto-detection.md

@@ -134,6 +134,18 @@ If a tokenizer fails during service initialization, AIPerf walks the `__cause__`
 | `TimeoutError` | Network Timeout | Pre-download and use: `--tokenizer ./local-path` |
 | `OSError` | Tokenizer Load Error | Clear cache and retry |
 
+## Model Compatibility Shims
+
+Some checkpoints ship a `model_type` in `config.json` that the installed `transformers` release does not yet recognize. When the config also lacks an `auto_map`, there is no remote class for `transformers` to import, so `--tokenizer-trust-remote-code` cannot help and tokenizer loading aborts before any benchmark traffic.
+
+AIPerf registers a narrow config alias for these cases before loading the tokenizer:
+
+| Model type | Aliased to | Notes |
+|---|---|---|
+| `deepseek_v32` (DeepSeek-V3.2-Exp) | `DeepseekV3Config` | V3.2 reuses the V3 config schema. Same approach as vLLM and SGLang. |
+
+The shim is best-effort and idempotent: it is a no-op on `transformers` releases that already register the model type natively, and it never raises (loading falls through to the normal error path if the expected base config class is unavailable). Native `deepseek_v32` support landed in `transformers` via [huggingface/transformers#41251](https://github.com/huggingface/transformers/pull/41251); this alias covers the older releases in AIPerf's supported range (`transformers>=4.56`) that predate it, and can be removed once that floor moves past it.
+
 ## CLI Options
 
 | Option | Description |

src/aiperf/common/tokenizer.py

@@ -376,6 +376,51 @@ def _missing_tokenizer_class_hint(error: Exception) -> str | None:
     )
 
 
+_DEEPSEEK_V32_MODEL_TYPE = "deepseek_v32"
+
+
+def _ensure_deepseek_v32_config_registered() -> None:
+    """Register a ``deepseek_v32`` config alias when transformers lacks one.
+
+    DeepSeek-V3.2-Exp ships ``config.json`` with ``model_type: "deepseek_v32"``
+    and no ``auto_map``. On transformers releases without a native
+    ``deepseek_v32`` entry (the ``>=4.56`` floor we still support), the
+    ``AutoConfig`` lookup that ``AutoTokenizer`` performs internally falls back
+    to an empty config and tokenizer loading aborts -- and
+    ``--tokenizer-trust-remote-code`` cannot help, since with no ``auto_map``
+    there is no remote config class to import. V3.2 reuses the V3 config schema,
+    so we alias it onto ``DeepseekV3Config`` (vLLM and SGLang do the same).
+
+    Idempotent and best-effort: a no-op once the model type is known (native
+    support on newer transformers, or a prior call), and silent when the
+    expected DeepSeek-V3 config class is unavailable so tokenizer loading
+    proceeds to its normal error path. Native support landed in transformers
+    via huggingface/transformers#41251; this shim covers the older releases in
+    our supported range (``>=4.56``) that predate it, and self-disables where
+    native support exists, so it can be removed once that floor moves past it.
+    """
+    try:
+        from transformers import AutoConfig, DeepseekV3Config
+        from transformers.models.auto.configuration_auto import CONFIG_MAPPING
+
+        if _DEEPSEEK_V32_MODEL_TYPE in CONFIG_MAPPING:
+            return
+
+        class _DeepseekV32ConfigAlias(DeepseekV3Config):
+            model_type = _DEEPSEEK_V32_MODEL_TYPE
+
+        AutoConfig.register(
+            _DEEPSEEK_V32_MODEL_TYPE, _DeepseekV32ConfigAlias, exist_ok=True
+        )
+    except (ImportError, TypeError, ValueError) as e:
+        # ImportError: DeepseekV3Config / CONFIG_MAPPING moved or renamed in a
+        # transformers version we don't pin. TypeError: base config unavailable
+        # (e.g. patched to None) so subclassing fails. ValueError: register()
+        # rejected the model type. All mean "leave the registry untouched and
+        # let tokenizer loading hit its normal error path."
+        _logger.debug(f"deepseek_v32 config alias registration skipped: {e!r}")
+
+
 class Tokenizer:
     """Simplified interface for HuggingFace tokenizers with sensible defaults."""
 
@@ -480,6 +525,8 @@ def _load_from_hub(
     ) -> "Tokenizer":
         from transformers import AutoTokenizer
 
+        _ensure_deepseek_v32_config_registered()
+
         if _is_offline_mode():
             tokenizer_instance = cls._from_pretrained_local(
                 AutoTokenizer.from_pretrained,

tests/unit/common/test_tokenizer_deepseek_v32.py

@@ -0,0 +1,165 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tests for the ``deepseek_v32`` config-alias compatibility shim.
+
+DeepSeek-V3.2-Exp ships ``model_type: "deepseek_v32"`` with no ``auto_map``.
+On transformers releases without native support, the ``AutoConfig`` lookup that
+``AutoTokenizer`` performs internally aborts tokenizer loading. The shim
+registers a ``DeepseekV3Config`` alias so loading proceeds; once transformers
+ships native support it must stay a no-op.
+"""
+
+from collections.abc import Iterator
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from aiperf.common.tokenizer import (
+    _DEEPSEEK_V32_MODEL_TYPE,
+    _ensure_deepseek_v32_config_registered,
+)
+
+
+class _FakeDeepseekV3Config:
+    """Stand-in for ``transformers.DeepseekV3Config`` that supports subclassing."""
+
+    model_type = "deepseek_v3"
+
+
+@pytest.fixture
+def isolated_transformers_registry() -> Iterator[None]:
+    """Snapshot and restore the real transformers config registry.
+
+    Tests that mutate the global ``CONFIG_MAPPING`` (registering or removing
+    ``deepseek_v32``) must not leak that state into other tests, since the
+    registry is process-global and unaffected by the singleton-reset fixtures.
+    """
+    from transformers.models.auto.configuration_auto import (
+        CONFIG_MAPPING,
+        CONFIG_MAPPING_NAMES,
+    )
+
+    saved_names = dict(CONFIG_MAPPING_NAMES)
+    saved_extra = dict(CONFIG_MAPPING._extra_content)
+    try:
+        yield
+    finally:
+        CONFIG_MAPPING_NAMES.clear()
+        CONFIG_MAPPING_NAMES.update(saved_names)
+        CONFIG_MAPPING._extra_content.clear()
+        CONFIG_MAPPING._extra_content.update(saved_extra)
+
+
+class TestEnsureDeepseekV32ConfigRegistered:
+    def test_registers_alias_when_model_type_absent(self) -> None:
+        auto_config = MagicMock()
+        with (
+            patch("transformers.AutoConfig", auto_config),
+            patch("transformers.DeepseekV3Config", _FakeDeepseekV3Config),
+            patch(
+                "transformers.models.auto.configuration_auto.CONFIG_MAPPING",
+                {},  # deepseek_v32 not present
+            ),
+        ):
+            _ensure_deepseek_v32_config_registered()
+
+        auto_config.register.assert_called_once()
+        args, kwargs = auto_config.register.call_args
+        assert args[0] == _DEEPSEEK_V32_MODEL_TYPE
+        registered_cls = args[1]
+        assert issubclass(registered_cls, _FakeDeepseekV3Config)
+        assert registered_cls.model_type == _DEEPSEEK_V32_MODEL_TYPE
+        assert kwargs.get("exist_ok") is True
+
+    def test_no_op_when_model_type_already_present(self) -> None:
+        auto_config = MagicMock()
+        with (
+            patch("transformers.AutoConfig", auto_config),
+            patch("transformers.DeepseekV3Config", _FakeDeepseekV3Config),
+            patch(
+                "transformers.models.auto.configuration_auto.CONFIG_MAPPING",
+                {_DEEPSEEK_V32_MODEL_TYPE: object()},  # native support present
+            ),
+        ):
+            _ensure_deepseek_v32_config_registered()
+
+        auto_config.register.assert_not_called()
+
+    def test_swallows_errors_and_does_not_raise(self) -> None:
+        # Simulate an old/renamed transformers where DeepseekV3Config is absent:
+        # the shim must degrade silently so loading reaches its normal error path.
+        with (
+            patch(
+                "transformers.models.auto.configuration_auto.CONFIG_MAPPING",
+                {},
+            ),
+            patch("transformers.AutoConfig", MagicMock()),
+            patch(
+                "transformers.DeepseekV3Config",
+                new=None,
+                create=True,
+            ),
+        ):
+            # DeepseekV3Config = None -> subclassing raises TypeError,
+            # which the shim must swallow.
+            _ensure_deepseek_v32_config_registered()
+
+    def test_real_transformers_round_trip(
+        self, isolated_transformers_registry: None
+    ) -> None:
+        # End-to-end against the installed transformers: force the
+        # "no native support" state, then verify the shim makes AutoConfig
+        # resolve deepseek_v32 to a DeepseekV3Config subclass.
+        from transformers import AutoConfig, DeepseekV3Config
+        from transformers.models.auto.configuration_auto import (
+            CONFIG_MAPPING,
+            CONFIG_MAPPING_NAMES,
+        )
+
+        CONFIG_MAPPING._extra_content.pop(_DEEPSEEK_V32_MODEL_TYPE, None)
+        CONFIG_MAPPING_NAMES.pop(_DEEPSEEK_V32_MODEL_TYPE, None)
+        assert _DEEPSEEK_V32_MODEL_TYPE not in CONFIG_MAPPING
+
+        with pytest.raises(ValueError):
+            AutoConfig.for_model(_DEEPSEEK_V32_MODEL_TYPE)
+
+        _ensure_deepseek_v32_config_registered()
+
+        assert _DEEPSEEK_V32_MODEL_TYPE in CONFIG_MAPPING
+        config = AutoConfig.for_model(_DEEPSEEK_V32_MODEL_TYPE)
+        assert isinstance(config, DeepseekV3Config)
+        assert config.model_type == _DEEPSEEK_V32_MODEL_TYPE
+
+        # Second call is idempotent (exist_ok=True), no raise.
+        _ensure_deepseek_v32_config_registered()
+
+
+class TestLoadFromHubRegistersDeepseekV32:
+    def test_load_from_hub_invokes_registration_hook(self) -> None:
+        # The shim must run before AutoTokenizer.from_pretrained on every load
+        # path. Drive the cache-warm branch (simplest: no alias resolution).
+        sentinel = object()
+        with (
+            patch(
+                "aiperf.common.tokenizer._ensure_deepseek_v32_config_registered"
+            ) as ensure_mock,
+            patch("aiperf.common.tokenizer._is_offline_mode", return_value=False),
+            patch("aiperf.common.tokenizer._is_hf_cached", return_value=True),
+            patch(
+                "aiperf.common.tokenizer.Tokenizer._build_with_kwargs",
+                return_value=sentinel,
+            ),
+            patch("transformers.AutoTokenizer"),
+        ):
+            from aiperf.common.tokenizer import Tokenizer
+
+            result = Tokenizer._load_from_hub(
+                "deepseek-ai/DeepSeek-V3.2-Exp",
+                trust_remote_code=False,
+                revision="main",
+                resolve_alias=True,
+            )
+
+        assert result is sentinel
+        ensure_mock.assert_called_once()