From 158c820644fdf3f59cc819d29acf8156c8f27d2b Mon Sep 17 00:00:00 2001 From: kunal-vaishnavi <115581922+kunal-vaishnavi@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:38:37 -0700 Subject: [PATCH] Update quantized_model.py --- src/python/py/models/quantized_model.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/python/py/models/quantized_model.py b/src/python/py/models/quantized_model.py index 84d712e05d..b41b16d90d 100644 --- a/src/python/py/models/quantized_model.py +++ b/src/python/py/models/quantized_model.py @@ -317,6 +317,11 @@ def __init__(self, quant_type, input_path, bits, group_size, q_size, kv_size, in # model.layers.layer_id.mlp.dense_h_to_4h.g_idx module.mlp.gate_proj.g_idx = tensor module.mlp.up_proj.g_idx = tensor + elif bool(re.match(r"^model.layers\.\d+\.mlp.(gate_up_proj|dense_h_to_4h)\.bias$", name)): + # model.layers.layer_id.mlp.gate_up_proj.bias + # model.layers.layer_id.mlp.dense_h_to_4h.bias + module.mlp.gate_proj.bias = tensor[: intermediate_size] + module.mlp.down_proj.bias = tensor[intermediate_size: ] else: raise NotImplementedError(f"{name} in your quantized model is not recognized.")