diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py
index 6d8234f9c3fd..6ad0a6ccd210 100644
--- a/src/transformers/data/data_collator.py
+++ b/src/transformers/data/data_collator.py
@@ -261,7 +261,7 @@ def __call__(
                 batch["input_ids"], special_tokens_mask=special_tokens_mask
             )
         else:
-            labels = batch["input_ids"]
+            labels = batch["input_ids"].clone()
             if self.tokenizer.pad_token_id is not None:
                 labels[labels == self.tokenizer.pad_token_id] = -100
             batch["labels"] = labels