diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 0f614e4df3d..3eb15f2af74 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -10966,7 +10966,7 @@ def set_gguf_parameters(self):
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         # these layers act as MLM head, so we don't need them
-        if name.startswith("decoder."):
+        if name.startswith("decoder.") or name.startswith("head.norm"):
             return
 
         if name.startswith("model."):