From 9b65380dfd380139914fd33a603d7ecaf9f94452 Mon Sep 17 00:00:00 2001 From: Copilot Date: Thu, 12 Mar 2026 15:47:33 +0000 Subject: [PATCH] fix: resolve rope_theta from rope_parameters dict in HF config validation Some models (e.g., GLM-4.6V) store rope_theta inside the rope_parameters dict rather than as a top-level config attribute. The previous code compared against config.rope_theta which returned a stale class default (10000) instead of the actual value (500000 from rope_parameters). This caused validation failures when launching GLM-4.6V training with correct --rotary-base 500000. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- slime/backends/megatron_utils/arguments.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/slime/backends/megatron_utils/arguments.py b/slime/backends/megatron_utils/arguments.py index d724b899ca..68a7aad715 100644 --- a/slime/backends/megatron_utils/arguments.py +++ b/slime/backends/megatron_utils/arguments.py @@ -40,6 +40,15 @@ def equal(x, y): if hasattr(hf_config, "text_config"): hf_config = hf_config.text_config + # Some models store rope_theta inside rope_parameters dict rather than + # as a top-level attribute. Prefer the dict value when available so + # the validation doesn't compare against a stale class default. + rope_params = getattr(hf_config, "rope_parameters", None) + if isinstance(rope_params, dict) and "rope_theta" in rope_params: + _hf_rope_theta = rope_params["rope_theta"] + else: + _hf_rope_theta = getattr(hf_config, "rope_theta", None) + for hf_config_name, megatron_config_name, compare_fn in [ ("hidden_size", "hidden_size", equal), ("num_attention_heads", "num_attention_heads", equal), @@ -47,7 +56,6 @@ def equal(x, y): ("intermediate_size", "ffn_hidden_size", equal), ("tie_word_embeddings", "untie_embeddings_and_output_weights", lambda x, y: not x == y), ("rms_norm_eps", "norm_epsilon", equal), - ("rope_theta", "rotary_base", equal), ]: if hasattr(hf_config, hf_config_name): if not compare_fn(getattr(hf_config, hf_config_name), getattr(args, megatron_config_name)): @@ -56,6 +64,14 @@ def equal(x, y): f"{megatron_config_name} {getattr(args, megatron_config_name)}, please check the config." ) + # Validate rope_theta separately using the resolved value + if _hf_rope_theta is not None: + if not equal(_hf_rope_theta, getattr(args, "rotary_base", None)): + errors.append( + f"rope_theta in hf config {_hf_rope_theta} is not equal to " + f"rotary_base {getattr(args, 'rotary_base', None)}, please check the config." + ) + if len(errors) > 0: raise AssertionError("hf_validate_args failed: " + "; ".join(errors))