Nathan add logging to metrics (#157)

what this PR does:

If you want to log out something comming from the metrics, simply return it in the metric dict.
for example, if you want to log out the judge response when using llm_as_judge, simply return the response in the dict.

```
{
   "score": score,
   "judgement": judge_response
}
````

the `judgement` field is a string and will not be aggregated. however, it will be logged in the details for each sample.

---------

Co-authored-by: Nathan Habib <[email protected]>

Author: NathanHB

src/lighteval/evaluator.py

@@ -117,22 +117,8 @@ def evaluate(  # noqa: C901
         # using a deep copy here because process results pops from the model responses
         metrics = task.process_results(doc, copy.deepcopy(model_responses))
 
-        # Remove the user_prompt from the metrics in case of llm-as-judge metric
-        if "user_prompt" in metrics:
-            user_prompt = metrics["user_prompt"]
-            del metrics["user_prompt"]
-        else:
-            user_prompt = None
-        if "judgement" in metrics:
-            judgement = metrics["judgement"]
-            del metrics["judgement"]
-        else:
-            judgement = None
-
         evaluation_tracker.metrics_logger.log(task_example_id.task_name, metrics)
-        evaluation_tracker.details_logger.log(
-            task_example_id.task_name, task, doc, model_responses, metrics, (user_prompt, judgement)
-        )
+        evaluation_tracker.details_logger.log(task_example_id.task_name, task, doc, model_responses, metrics)
 
     return evaluation_tracker
 

src/lighteval/logging/info_loggers.py

@@ -205,8 +205,6 @@ class Detail:
         choices: list = field(default_factory=list)
         gold_index: list = field(default_factory=list)
         metrics: dict = field(default_factory=dict)
-        judement_prompt: str = None
-        judgement: str = None
         specifics: dict = field(default_factory=dict)
 
     @dataclass
@@ -367,11 +365,16 @@ def log(
             detail.choices = doc.choices
             detail.gold_index = as_list(doc.gold_index)
             pred_saved = True
-        if task.has_metric_category[MetricCategory.GENERATIVE_MULTI_TURN]:
+        if (
+            task.has_metric_category[MetricCategory.LLM_AS_JUDGE_MULTI_TURN]
+            or task.has_metric_category[MetricCategory.LLM_AS_JUDGE]
+        ):
+            detail.choices = doc.choices
+            detail.gold_index = as_list(doc.gold_index)
             pred_saved = True
-            detail.judement_prompt = llm_as_prompt_judgement[0]
-            detail.judgement = llm_as_prompt_judgement[1]
+
         detail.specifics = doc.specific
+
         if not pred_saved:
             raise NotImplementedError(
                 "No metric prediction saved."
@@ -487,6 +490,8 @@ def aggregate(self, task_dict: dict[str, LightevalTask], bootstrap_iters: int =
                 except OverflowError:
                     hlog_warn(f"{task_name}, {metric_name} got an OVERFLOW ERROR when aggregating.")
                     metric_result = float("nan")
+                except KeyError:
+                    continue
 
                 if isinstance(metric_result, dict):  # For some corpus level grouping metrics
                     self.metric_aggregated[task_name].update(metric_result)

src/lighteval/metrics/__init__.py

@@ -148,12 +148,15 @@ def apply_multichoice_metric_one_token(results: list[ModelReturn], formatted_doc
     return results, outputs
 
 
-def apply_generative_multi_turn_metric(results: list[ModelReturn], formatted_doc: Doc, metrics: list[str]):
+def apply_llm_as_judge_metric(results: list[ModelReturn], formatted_doc: Doc, metrics: list[str]):
     outputs = {}
     predictions = results.pop(0).result
 
     for metric in metrics:
-        if Metrics[metric].value.category == MetricCategory.GENERATIVE_MULTI_TURN:
+        if (
+            Metrics[metric].value.category == MetricCategory.LLM_AS_JUDGE_MULTI_TURN
+            or Metrics[metric].value.category == MetricCategory.LLM_AS_JUDGE
+        ):
             outputs.update(Metrics[metric].value.compute(predictions=predictions, formatted_doc=formatted_doc))
 
     return results, outputs

src/lighteval/metrics/metrics.py

@@ -228,7 +228,7 @@ class Metrics(Enum):
     llm_judge_multi_turn = SampleLevelMetricGrouping(
         metric=["single_turn", "multi_turn"],
         higher_is_better=True,
-        category=MetricCategory.GENERATIVE_MULTI_TURN,
+        category=MetricCategory.LLM_AS_JUDGE_MULTI_TURN,
         use_case=MetricUseCase.SUMMARIZATION,
         sample_level_fn=JudgeLLM(
             judge_model_name="gpt-3.5-turbo",
@@ -243,7 +243,7 @@ class Metrics(Enum):
     llm_judge = SampleLevelMetricGrouping(
         metric=["judge_score"],
         higher_is_better=True,
-        category=MetricCategory.GENERATIVE,
+        category=MetricCategory.LLM_AS_JUDGE,
         use_case=MetricUseCase.SUMMARIZATION,
         sample_level_fn=JudgeLLM(
             judge_model_name="gpt-3.5-turbo",

src/lighteval/metrics/utils.py

@@ -28,7 +28,8 @@ class MetricCategory(Enum):
     TARGET_PERPLEXITY = auto()
     PERPLEXITY = auto()
     GENERATIVE = auto()
-    GENERATIVE_MULTI_TURN = auto()
+    LLM_AS_JUDGE_MULTI_TURN = auto()
+    LLM_AS_JUDGE = auto()
     GENERATIVE_LOGPROB = auto()
     MULTICHOICE = auto()
     MULTICHOICE_ONE_TOKEN = auto()

src/lighteval/tasks/lighteval_task.py

@@ -34,7 +34,7 @@
 from lighteval.metrics import (
     apply_generative_logprob_metric,
     apply_generative_metric,
-    apply_generative_multi_turn_metric,
+    apply_llm_as_judge_metric,
     apply_multichoice_metric,
     apply_multichoice_metric_one_token,
     apply_perplexity_metric,
@@ -412,8 +412,10 @@ def get_request_type(self) -> list[RequestType]:
             request_types.append(RequestType.LOGLIKELIHOOD_ROLLING)
         if self.has_metric_category[MetricCategory.GENERATIVE]:
             request_types.append(RequestType.GREEDY_UNTIL)
-        if self.has_metric_category[MetricCategory.GENERATIVE_MULTI_TURN]:
+        if self.has_metric_category[MetricCategory.LLM_AS_JUDGE_MULTI_TURN]:
             request_types.append(RequestType.GREEDY_UNTIL_MULTI_TURN)
+        if self.has_metric_category[MetricCategory.LLM_AS_JUDGE]:
+            request_types.append(RequestType.GREEDY_UNTIL)
         if self.has_metric_category[MetricCategory.GENERATIVE_LOGPROB]:
             request_types.append(RequestType.GREEDY_UNTIL_WITH_LOGITS)
         if self.has_metric_category[MetricCategory.MULTICHOICE]:
@@ -504,7 +506,7 @@ def construct_requests(
                     choices=formatted_doc.choices,
                 )
             ]
-        if self.has_metric_category[MetricCategory.GENERATIVE_MULTI_TURN]:
+        if self.has_metric_category[MetricCategory.LLM_AS_JUDGE_MULTI_TURN]:
             requests[RequestType.GREEDY_UNTIL_MULTI_TURN] += [
                 GreedyUntilMultiTurnRequest(
                     task_name=current_task_name,
@@ -561,8 +563,11 @@ def process_results(self, formatted_doc: Doc, results: list[ModelReturn]) -> dic
                 results=results, formatted_doc=formatted_doc, metrics=self.metrics
             )
             outputs.update(cur_outputs)
-        if self.has_metric_category[MetricCategory.GENERATIVE_MULTI_TURN]:
-            results, cur_outputs = apply_generative_multi_turn_metric(
+        if (
+            self.has_metric_category[MetricCategory.LLM_AS_JUDGE_MULTI_TURN]
+            or self.has_metric_category[MetricCategory.LLM_AS_JUDGE]
+        ):
+            results, cur_outputs = apply_llm_as_judge_metric(
                 results=results, formatted_doc=formatted_doc, metrics=self.metrics
             )
             outputs.update(cur_outputs)