Skip to content
29 changes: 21 additions & 8 deletions verl/utils/qat/compressed_tensors.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@


_HIGH_PRECISION_DTYPES = (torch.bfloat16, torch.float16, torch.float32)
# Trainers export these checkpoints under their own names, in their mixed FP8/FP4 layout.
_DEEPSEEK_V4_MODEL_TYPES = frozenset({"deepseek_v4", "deepseek_v41"})


def _check_scheme(kind: str, scheme: Optional[str]) -> Optional[str]:
Expand Down Expand Up @@ -446,7 +448,7 @@ class Fp8ConfigWeightQuantizer:

def __init__(self, fp8_config: Mapping[str, Any], *, model_types: Iterable[Optional[str]] = ()):
quant_method = str(fp8_config.get("quant_method", "")).lower()
self.passthrough = bool({"deepseek_v4", "deepseek_v41"} & set(model_types))
self.passthrough = bool(_DEEPSEEK_V4_MODEL_TYPES & set(model_types))
if self.passthrough:
return
activation_scheme = fp8_config.get("activation_scheme")
Expand Down Expand Up @@ -547,7 +549,20 @@ def dispatch_rollout_weight_quantizer(
has_override = bool(linear or moe or config_file)

hf_config = model_config.hf_config
native = getattr(hf_config, "quantization_config", None)
if native is None:
native = getattr(getattr(hf_config, "text_config", None), "quantization_config", None)
model_types = {
getattr(hf_config, "model_type", None),
getattr(getattr(hf_config, "text_config", None), "model_type", None),
}
if has_override:
if model_types & _DEEPSEEK_V4_MODEL_TYPES:
raise NotImplementedError(
"rollout.quantization overrides are not supported for DeepSeek-V4: the trainer exports its "
"checkpoint names (e.g. ffn.experts) and its own FP8/FP4 layout, which the override's "
"compressed-tensors targets do not match. Drop the override to sync the native quantization."
)
quantization_config = build_compressed_tensors_config(
linear=linear,
moe=moe,
Expand All @@ -556,9 +571,6 @@ def dispatch_rollout_weight_quantizer(
model_path=model_config.local_path,
)
else:
native = getattr(hf_config, "quantization_config", None)
if native is None:
native = getattr(getattr(hf_config, "text_config", None), "quantization_config", None)
quantization_config = _as_quantization_config(native)

qat_mode = qat_config.mode if qat_config is not None and qat_config.enable else None
Expand All @@ -569,6 +581,11 @@ def dispatch_rollout_weight_quantizer(
"quantization_config or set rollout.quantization.linear / moe / config_file"
)
return None
if getattr(getattr(rollout_config, "checkpoint_engine", None), "backend", None) == "delta_sharded":
raise NotImplementedError(
"checkpoint_engine.backend='delta_sharded' does not support quantized rollouts: its steady-state "
"deltas bypass the trainer-side rollout weight quantizer"
)

quant_method = str(quantization_config.get("quant_method", "")).lower()
if quant_method == "compressed-tensors":
Expand All @@ -581,10 +598,6 @@ def dispatch_rollout_weight_quantizer(
quantizer = CompressedTensorsWeightQuantizer(quantization_config)
elif "fp8" in quant_method:
qat_modes = {"fp8"}
model_types = {
getattr(hf_config, "model_type", None),
getattr(getattr(hf_config, "text_config", None), "model_type", None),
}
quantizer = Fp8ConfigWeightQuantizer(quantization_config, model_types=model_types)
else:
raise NotImplementedError(
Expand Down
18 changes: 17 additions & 1 deletion verl/workers/engine/veomni/transformer_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,7 @@ def __init__(

self.pad_to_length: bool = self.engine_config.pad_to_length
self.pad_to_length_bucket: int = self.engine_config.pad_to_length_bucket
self._rollout_weight_quantizer = None

def initialize(self):
"""
Expand Down Expand Up @@ -273,7 +274,7 @@ def initialize(self):
processing_class=self.model_config.get_processor(),
checkpoint_config=self.checkpoint_config,
trust_remote_code=self.model_config.trust_remote_code,
per_tensor_param_fn=self.get_per_tensor_param,
per_tensor_param_fn=self._get_hf_export_per_tensor_param,
hf_export_dtype=hf_export_dtype,
)

Expand Down Expand Up @@ -643,6 +644,11 @@ def _hf_delta_entry(self, name, spec, place, lidx, lval):
# get_per_tensor_param_shard and _hf_delta_entry overrides.

def get_per_tensor_param(self, **kwargs):
per_tensor_param, peft_config = self._get_hf_export_per_tensor_param()
return self._quantize_for_rollout(per_tensor_param), peft_config

def _get_hf_export_per_tensor_param(self):
"""Full HF-format weights without the rollout weight quantizer: the checkpoint 'hf_model' export."""
# FSDP2 CPUOffloadPolicy owns CPU<->accelerator placement; calling model.to(device)
# here leaves the module half-moved and crashes state_dict() below (#5995). The
# per-DTensor .to(device).full_tensor() in param_generator() below stages each
Expand Down Expand Up @@ -693,6 +699,16 @@ def param_generator():
# TODO: support VeOmni LoRA
return param_generator(), None

def set_rollout_weight_quantizer(self, quantizer) -> None:
self._rollout_weight_quantizer = quantizer

def _quantize_for_rollout(self, per_tensor_param):
# A converter that exports the checkpoint's own quantized layout (DeepSeek-V4's FP8 / FP4
# experts) meets a pass-through quantizer for that model type, as with Megatron-Bridge.
if self._rollout_weight_quantizer is None:
return per_tensor_param
return self._rollout_weight_quantizer(per_tensor_param)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved


class EngineEvalModeCtx(BaseEngineCtx):
def __init__(self, engine: VeOmniEngine, **kwargs):
Expand Down