diff --git a/applications/DeepSpeed-Chat/training/utils/data/raw_datasets.py b/applications/DeepSpeed-Chat/training/utils/data/raw_datasets.py
index e8c519a8f..3c84f4b07 100644
--- a/applications/DeepSpeed-Chat/training/utils/data/raw_datasets.py
+++ b/applications/DeepSpeed-Chat/training/utils/data/raw_datasets.py
@@ -689,8 +689,10 @@ def get_prompt_and_chosen(self, sample):
             'positive_passages'][0]['text']
 
     def get_prompt_and_rejected(self, sample):
-        return " Human: " + sample['query'] + " Assistant: " + sample[
-            'negative_passages'][0]['text']
+        if len(sample['negative_passages']) > 0:
+            return " Human: " + sample['query'] + " Assistant: " + sample[
+                'negative_passages'][0]['text']
+        return None
 
 
 # Japanese dataset