diff --git a/examples/text-generation/run_generation.py b/examples/text-generation/run_generation.py
index 274aa591cd..4da935e477 100755
--- a/examples/text-generation/run_generation.py
+++ b/examples/text-generation/run_generation.py
@@ -682,7 +682,7 @@ def preprocess_function(examples):
             # Tokenize the texts
             return tokenizer(
                 examples[column_name],
-                padding="max_length",
+                padding="max_length" if prompt_length > 0 else False,
                 max_length=prompt_length if prompt_length > 0 else None,
                 truncation=prompt_length > 0,
             )