From 1d8aed1f0b4b9bb4cc6d672bd7474dacd738acb0 Mon Sep 17 00:00:00 2001 From: Sunghoon Choi Date: Sun, 31 May 2026 06:29:44 +0000 Subject: [PATCH 1/4] Support Hunyuan Dense V1 export --- src/python/py/models/builders/hunyuan.py | 28 +++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/python/py/models/builders/hunyuan.py b/src/python/py/models/builders/hunyuan.py index c2f9a0c100..437367b18a 100644 --- a/src/python/py/models/builders/hunyuan.py +++ b/src/python/py/models/builders/hunyuan.py @@ -29,16 +29,26 @@ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): # base = rope_theta * alpha ^ (head_dim / (head_dim - 2)) # With alpha=1000, head_dim=128: # effective_theta ≈ 10000 * 1000^(128/126) ≈ 10,359,000 - if hasattr(config, "rope_scaling") and config.rope_scaling is not None: - alpha = config.rope_scaling.get("alpha", 1.0) + # Transformers versions have used both rope_scaling and rope_parameters + # for RoPE metadata, so accept either shape before passing config to the base builder. + rope_config = getattr(config, "rope_scaling", None) or getattr(config, "rope_parameters", None) + if rope_config is not None: + base_theta = getattr(config, "rope_theta", None) or rope_config.get("rope_theta") + if base_theta is not None: + config.rope_theta = base_theta + + alpha = rope_config.get("alpha", 1.0) head_dim = getattr(config, "head_dim", None) if head_dim is None: head_dim = config.hidden_size // config.num_attention_heads - if alpha != 1.0 and head_dim is not None and head_dim > 2: - config.rope_theta = config.rope_theta * (alpha ** (head_dim / (head_dim - 2))) + if alpha != 1.0 and base_theta is not None and head_dim is not None and head_dim > 2: + config.rope_theta = base_theta * (alpha ** (head_dim / (head_dim - 2))) - # Disable rope_scaling: effective theta is now baked into config.rope_theta above. + # Disable generic RoPE scaling: Hunyuan's effective theta is now baked into config.rope_theta above. + # Leaving these fields set would let the base builder apply another, non-Hunyuan scaling path. config.rope_scaling = None + if hasattr(config, "rope_parameters"): + config.rope_parameters = None super().__init__(config, io_dtype, onnx_dtype, ep, cache_dir, extra_options) @@ -63,10 +73,12 @@ def make_attention_qk_rope_and_norm(self, layer_id, attention, **kwargs): query_layernorm / key_layernorm are aliased to q_norm / k_norm so the existing make_qk_norm() infrastructure can be reused. """ - # Alias Hunyuan weight names to what make_qk_norm expects - if not hasattr(attention, "q_norm"): + # Alias Hunyuan weight names to what make_qk_norm expects. + # Some Transformers versions expose q_norm/k_norm as None while the + # real modules live under query_layernorm/key_layernorm. + if getattr(getattr(attention, "q_norm", None), "weight", None) is None and hasattr(attention, "query_layernorm"): attention.q_norm = attention.query_layernorm - if not hasattr(attention, "k_norm"): + if getattr(getattr(attention, "k_norm", None), "weight", None) is None and hasattr(attention, "key_layernorm"): attention.k_norm = attention.key_layernorm # RoPE first, then QK norm From 12fe457d52c1fb1e11e9c3fbdd7cc2feec7dcc06 Mon Sep 17 00:00:00 2001 From: Sunghoon Choi Date: Sun, 31 May 2026 06:40:28 +0000 Subject: [PATCH 2/4] Add Hunyuan builder RoPE test --- test/python/test_hunyuan_builder.py | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 test/python/test_hunyuan_builder.py diff --git a/test/python/test_hunyuan_builder.py b/test/python/test_hunyuan_builder.py new file mode 100644 index 0000000000..59f9baf093 --- /dev/null +++ b/test/python/test_hunyuan_builder.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import math +import importlib.util +import sys +import types +from pathlib import Path + +import pytest + +BUILDERS_DIR = Path(__file__).parents[2] / "src" / "python" / "py" / "models" / "builders" +sys.path.insert(0, str(BUILDERS_DIR.parents[1])) + + +def _load_builder_module(module_name): + spec = importlib.util.spec_from_file_location(f"models.builders.{module_name}", BUILDERS_DIR / f"{module_name}.py") + module = importlib.util.module_from_spec(spec) + sys.modules[f"models.builders.{module_name}"] = module + spec.loader.exec_module(module) + return module + + +sys.modules.setdefault("models", types.ModuleType("models")) +builders_package = sys.modules.setdefault("models.builders", types.ModuleType("models.builders")) +builders_package.__path__ = [str(BUILDERS_DIR)] + +base_module = _load_builder_module("base") +hunyuan_module = _load_builder_module("hunyuan") +HunyuanDenseV1Model = hunyuan_module.HunyuanDenseV1Model +Model = base_module.Model + + +def _build_hunyuan_config(config, monkeypatch): + def base_init(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): + self.config = config + + monkeypatch.setattr(Model, "__init__", base_init) + return HunyuanDenseV1Model(config, None, None, "cpu", None, {}) + + +@pytest.mark.parametrize("rope_field", ["rope_scaling", "rope_parameters"]) +def test_hunyuan_dynamic_alpha_rope_is_baked_into_theta(rope_field, monkeypatch): + rope_config = {"type": "dynamic", "alpha": 1000.0, "rope_theta": 10000.0} + config_kwargs = { + "hidden_size": 2048, + "num_attention_heads": 16, + "head_dim": 128, + "rope_theta": None if rope_field == "rope_parameters" else 10000.0, + "rope_scaling": rope_config if rope_field == "rope_scaling" else None, + "rope_parameters": rope_config if rope_field == "rope_parameters" else None, + } + config = types.SimpleNamespace(**config_kwargs) + + _build_hunyuan_config(config, monkeypatch) + + expected_theta = 10000.0 * (1000.0 ** (128 / 126)) + assert math.isclose(config.rope_theta, expected_theta) + assert config.rope_scaling is None + assert config.rope_parameters is None \ No newline at end of file From b4313286f80e9096ab50fed06908a1d69db29182 Mon Sep 17 00:00:00 2001 From: Sunghoon Choi Date: Tue, 9 Jun 2026 17:19:48 +0000 Subject: [PATCH 3/4] Move Hunyuan builder test under model tests --- .../{ => models/hunyuan_v1_dense}/test_hunyuan_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename test/python/{ => models/hunyuan_v1_dense}/test_hunyuan_builder.py (97%) diff --git a/test/python/test_hunyuan_builder.py b/test/python/models/hunyuan_v1_dense/test_hunyuan_builder.py similarity index 97% rename from test/python/test_hunyuan_builder.py rename to test/python/models/hunyuan_v1_dense/test_hunyuan_builder.py index 59f9baf093..11cb23f31d 100644 --- a/test/python/test_hunyuan_builder.py +++ b/test/python/models/hunyuan_v1_dense/test_hunyuan_builder.py @@ -1,14 +1,14 @@ from __future__ import annotations -import math import importlib.util +import math import sys import types from pathlib import Path import pytest -BUILDERS_DIR = Path(__file__).parents[2] / "src" / "python" / "py" / "models" / "builders" +BUILDERS_DIR = Path(__file__).parents[4] / "src" / "python" / "py" / "models" / "builders" sys.path.insert(0, str(BUILDERS_DIR.parents[1])) From e79bda56fddd280b4814f35d6e1a50f538a97aa7 Mon Sep 17 00:00:00 2001 From: Sunghoon Choi Date: Tue, 9 Jun 2026 19:12:02 +0000 Subject: [PATCH 4/4] Move Hunyuan test under builder tests --- .../test_hunyuan_builder.py => builder/test_hunyuan.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename test/python/{models/hunyuan_v1_dense/test_hunyuan_builder.py => builder/test_hunyuan.py} (97%) diff --git a/test/python/models/hunyuan_v1_dense/test_hunyuan_builder.py b/test/python/builder/test_hunyuan.py similarity index 97% rename from test/python/models/hunyuan_v1_dense/test_hunyuan_builder.py rename to test/python/builder/test_hunyuan.py index 11cb23f31d..d5177b5821 100644 --- a/test/python/models/hunyuan_v1_dense/test_hunyuan_builder.py +++ b/test/python/builder/test_hunyuan.py @@ -8,7 +8,7 @@ import pytest -BUILDERS_DIR = Path(__file__).parents[4] / "src" / "python" / "py" / "models" / "builders" +BUILDERS_DIR = Path(__file__).parents[3] / "src" / "python" / "py" / "models" / "builders" sys.path.insert(0, str(BUILDERS_DIR.parents[1]))