fix dataset

Author: penguine-ip

b.py

@@ -13,11 +13,12 @@
 from deepeval.test_case.llm_test_case import LLMTestCaseParams
 
 test_case = ConversationalTestCase(
+    chatbot_role="A programmer",
     turns=[
         LLMTestCase(
             input="Message input", actual_output="Message actual output"
         )
-    ]
+    ],
 )
 test_case2 = ConversationalTestCase(
     turns=[
@@ -36,46 +37,46 @@
 
 from deepeval.metrics import GEval
 
-correctness_metric = GEval(
-    name="Correctness",
-    criteria="Determine whether the actual output is factually correct based on the expected output.",
-    # NOTE: you can only provide either criteria or evaluation_steps, and not both
-    evaluation_steps=[
-        "Check whether the facts in 'actual output' contradicts any facts in 'expected output'",
-        "You should also heavily penalize omission of detail",
-        "Vague language, or contradicting OPINIONS, are OK",
-    ],
-    evaluation_params=[
-        LLMTestCaseParams.INPUT,
-        LLMTestCaseParams.ACTUAL_OUTPUT,
-    ],
-)
+# correctness_metric = GEval(
+#     name="Correctness",
+#     criteria="Determine whether the actual output is factually correct based on the expected output.",
+#     # NOTE: you can only provide either criteria or evaluation_steps, and not both
+#     evaluation_steps=[
+#         "Check whether the facts in 'actual output' contradicts any facts in 'expected output'",
+#         "You should also heavily penalize omission of detail",
+#         "Vague language, or contradicting OPINIONS, are OK",
+#     ],
+#     evaluation_params=[
+#         LLMTestCaseParams.INPUT,
+#         LLMTestCaseParams.ACTUAL_OUTPUT,
+#     ],
+# )
 
-evaluate(
-    test_cases=[
-        LLMTestCase(
-            input="Message input number 1!",
-            actual_output="Message actual output number 1...",
-            retrieval_context=["I love dogs"],
-        ),
-        LLMTestCase(
-            input="Message input 2, this is just a test",
-            actual_output="Message actual output 2, this is just a test",
-            retrieval_context=["I love dogs"],
-        ),
-    ],
-    metrics=[
-        # correctness_metric,
-        # AnswerRelevancyMetric(),
-        # BiasMetric(),
-        SummarizationMetric(verbose_mode=True, truths_extraction_limit=3),
-        FaithfulnessMetric(verbose_mode=True, truths_extraction_limit=3),
-    ],
-    # throttle_value=10,
-    # max_concurrent=1,
-)
+# evaluate(
+#     test_cases=[
+#         LLMTestCase(
+#             input="Message input number 1!",
+#             actual_output="Message actual output number 1...",
+#             retrieval_context=["I love dogs"],
+#         ),
+#         LLMTestCase(
+#             input="Message input 2, this is just a test",
+#             actual_output="Message actual output 2, this is just a test",
+#             retrieval_context=["I love dogs"],
+#         ),
+#     ],
+#     metrics=[
+#         # correctness_metric,
+#         # AnswerRelevancyMetric(),
+#         # BiasMetric(),
+#         SummarizationMetric(verbose_mode=True, truths_extraction_limit=3),
+#         FaithfulnessMetric(verbose_mode=True, truths_extraction_limit=3),
+#     ],
+#     # throttle_value=10,
+#     # max_concurrent=1,
+# )
 
-# confident_evaluate(experiment_name="Convo", test_cases=[test_case])
+confident_evaluate(experiment_name="Convo", test_cases=[test_case])
 
 
 # evaluate(

deepeval/dataset/dataset.py

@@ -544,14 +544,20 @@ def add_goldens_from_json_file(
                 )
             )
 
-    def push(self, alias: str, overwrite: Optional[bool] = None):
+    def push(
+        self,
+        alias: str,
+        overwrite: Optional[bool] = None,
+        auto_convert_test_cases_to_goldens: bool = False,
+    ):
         if len(self.test_cases) == 0 and len(self.goldens) == 0:
             raise ValueError(
                 "Unable to push empty dataset to Confident AI, there must be at least one test case or golden in dataset"
             )
         if is_confident():
             goldens = self.goldens
-            goldens.extend(convert_test_cases_to_goldens(self.test_cases))
+            if auto_convert_test_cases_to_goldens:
+                goldens.extend(convert_test_cases_to_goldens(self.test_cases))
             api_dataset = APIDataset(
                 alias=alias,
                 overwrite=overwrite,