Parametrizing the sampling evals from the CLI

community_tasks/aimo_evals.py

@@ -26,6 +26,7 @@
 """
 
 from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import math_normalizer
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
 
@@ -49,7 +50,9 @@ def aimo_prompt(line, task_name: str = None):
     evaluation_splits=["train"],
     few_shots_split="train",
     few_shots_select="sequential",
-    metrics=[Metrics.quasi_exact_match_math],
+    metrics=[
+        Metrics.exact_match(sample_params={"normalize_gold": math_normalizer, "normalize_pred": math_normalizer})
+    ],
     generation_size=2048,
     stop_sequence=None,
 )

community_tasks/arabic_evals.py

@@ -31,8 +31,10 @@
 import re
 from typing import Any, Dict, List, Optional, Union
 
-from lighteval.metrics.llm_as_judge import JudgeLM
-from lighteval.metrics.metrics import Metric, Metrics
+from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import LogProbCharNorm
+from lighteval.metrics.utils.llm_as_judge import JudgeLM
+from lighteval.metrics.utils.metric_utils import Metric
 from lighteval.tasks.default_prompts import LETTER_INDICES
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc, SamplingMethod
@@ -103,7 +105,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=arabic_mmlu_pfn,
             hf_repo="MBZUAI/ArabicMMLU",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test"],
             evaluation_splits=["test"],
             few_shots_split=["dev"],
@@ -164,7 +166,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=arabic_mmlu_ht_pfn,
             hf_repo="MBZUAI/human_translated_arabic_mmlu",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test"],
             evaluation_splits=["test"],
             few_shots_split=None,
@@ -228,7 +230,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=arabic_mmlu_mt_pfn,
             hf_repo="OALL/Arabic_MMLU",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test", "dev"],
             evaluation_splits=["test"],
             few_shots_split="dev",
@@ -283,7 +285,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=acva_pfn,
             hf_repo="OALL/ACVA",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test", "validation"],
             evaluation_splits=["test"],
             few_shots_split="validation",
@@ -339,7 +341,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=aratrust_pfn,
             hf_repo="asas-ai/AraTrust-categorized",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["train"],
             evaluation_splits=["train"],
             few_shots_split=None,
@@ -387,7 +389,7 @@ def arabic_exams_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -437,7 +439,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=alghafa_pfn,
             hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test", "validation"],
             evaluation_splits=["test"],
             few_shots_split="validation",
@@ -463,7 +465,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -479,7 +481,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -495,7 +497,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -511,7 +513,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -527,7 +529,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -543,7 +545,7 @@ def __init__(
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -580,7 +582,7 @@ def boolq_arabic_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -614,7 +616,7 @@ def copa_arabic_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -657,7 +659,7 @@ def hellaswag_arabic_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -693,7 +695,7 @@ def toxigen_arabic_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -743,7 +745,7 @@ def sciq_arabic_pfn(line, task_name: str = None):
     evaluation_splits=["test"],
     few_shots_split="validation",
     few_shots_select="sequential",
-    metrics=[Metrics.loglikelihood_acc_norm],
+    metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
     version=0,
 )
 
@@ -800,7 +802,7 @@ def __init__(
             hf_subset=hf_subset,
             prompt_function=madinah_qa_pfn,
             hf_repo="MBZUAI/MadinahQA",
-            metrics=[Metrics.loglikelihood_acc_norm],
+            metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})],
             hf_avail_splits=["test"],
             evaluation_splits=["test"],
             few_shots_split=["dev"],

community_tasks/french_evals.py

@@ -33,6 +33,7 @@
 import random
 
 from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import math_normalizer
 from lighteval.tasks.default_prompts import LETTER_INDICES
 from lighteval.tasks.extended.ifeval.main import ifeval_metrics
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
@@ -136,7 +137,10 @@ def prompt_bac_fr(line, task_name: str = None):
     few_shots_split=None,
     few_shots_select="random_sampling",
     generation_size=1,
-    metrics=[Metrics.quasi_exact_match_math, Metrics.exact_match],
+    metrics=[
+        Metrics.exact_match(sample_params={"normalize_gold": math_normalizer, "normalize_pred": math_normalizer}),
+        Metrics.exact_match,
+    ],
     stop_sequence=["\n"],
     version=0,
 )