diff --git a/farm/modeling/language_model.py b/farm/modeling/language_model.py
index 9f6ce5e2e..28f0a9327 100644
--- a/farm/modeling/language_model.py
+++ b/farm/modeling/language_model.py
@@ -1275,6 +1275,7 @@ def load(cls, pretrained_model_name_or_path, language=None, **kwargs):
         config.summary_last_dropout = 0
         config.summary_type = 'first'
         config.summary_activation = 'gelu'
+        config.summary_use_proj = False
         electra.pooler = SequenceSummary(config)
         electra.pooler.apply(electra.model._init_weights)
         return electra