diff --git a/src/transformers/models/llava/configuration_llava.py b/src/transformers/models/llava/configuration_llava.py
index d70175c4738a..a9394d5c646d 100644
--- a/src/transformers/models/llava/configuration_llava.py
+++ b/src/transformers/models/llava/configuration_llava.py
@@ -137,6 +137,7 @@ def __init__(
         self.text_config = text_config
         self._vocab_size = self.text_config.vocab_size
 
+        kwargs.pop("vocab_size", None)
         super().__init__(**kwargs)
 
     @property