Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion tensorrt_llm/_torch/model_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -641,7 +641,9 @@ def load_hf_quant_config(hf_quant_config, moe_backend, checkpoint_dir=None):
or hf_quant_config.get("activation_scheme") == "static"):
quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES

block_size = hf_quant_config.get("weight_block_size", [128, 128])
block_size = hf_quant_config.get("weight_block_size")
if block_size is None:
block_size = [128, 128]
assert tuple(block_size) == (
128, 128), "FP8_BLOCK_SCALES only supports block_size=(128,128)"
quant_config.group_size = block_size[0]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ def test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks(moe_lora_mode: s
"""
cuda_graph_config = CudaGraphConfig(max_batch_size=10) if moe_lora_mode == "cudagraph" else None

model_dir = f"{llm_models_root()}/Mistral-Small-4-119B-2603"
model_dir = f"{llm_models_root()}/Mistral-Small-4-119B-2603/Mistral-Small-4-119B-2603"

ranks = _RANKS
max_rank = max(ranks)
Expand Down
17 changes: 17 additions & 0 deletions tests/unittest/_torch/test_hf_quant_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,20 @@ def test_load_hf_quant_config_parses_nvfp4_with_kv_cache_scheme():
assert quant_config.group_size == 16
assert quant_config.kv_cache_quant_algo == QuantAlgo.FP8
assert set(quant_config.exclude_modules) == {gate_exclude, "lm_head"}


def test_load_hf_quant_config_defaults_null_fp8_weight_block_size():
quant_config, layer_quant_config = ModelConfig.load_hf_quant_config(
{
"activation_scheme": "static",
"modules_to_not_convert": ["model.vision_tower", "lm_head"],
"quant_method": "fp8",
"weight_block_size": None,
},
moe_backend="CUTLASS",
)

assert layer_quant_config is None
assert quant_config.quant_algo == QuantAlgo.FP8_BLOCK_SCALES
assert quant_config.group_size == 128
assert {"model.vision_tower", "lm_head"} <= set(quant_config.exclude_modules)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
Loading