Adding support for Arabic benchmarks : AceGPT benchmarking suite

community_tasks/arabic_evals.py

@@ -0,0 +1,179 @@
+# ruff: noqa: F405, F403, F401
+"""
+Custom evaluation tasks for lighteval
+
+This file generally create just a TASKS_TABLE and TASKS_GROUPS which are then imported by LightEval.
+"""
+from lighteval.tasks.lighteval_task import LightevalTaskConfig
+from lighteval.tasks.requests import Doc
+from lighteval.tasks.tasks_prompt_formatting import LETTER_INDICES
+
+
+# fmt: off
+LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"]
+# fmt: on
+
+## ARABIC MMLU ##
+# fmt: off
+ARABIC_MMLU_SUBSETS = [
+    "abstract_algebra", "anatomy", "astronomy", "business_ethics", "clinical_knowledge", "college_biology", "college_chemistry", "college_computer_science",
+    "college_mathematics", "college_medicine", "college_physics", "computer_security", "conceptual_physics", "econometrics", "electrical_engineering",
+    "elementary_mathematics", "formal_logic", "global_facts", "high_school_biology", "high_school_chemistry",  "high_school_computer_science",
+    "high_school_european_history", "high_school_geography", "high_school_government_and_politics", "high_school_macroeconomics", "high_school_mathematics",
+    "high_school_microeconomics", "high_school_physics", "high_school_psychology", "high_school_statistics", "high_school_us_history", "high_school_world_history",
+    "human_aging", "human_sexuality", "international_law", "jurisprudence", "logical_fallacies", "machine_learning", "management", "marketing", "medical_genetics",
+    "miscellaneous", "moral_disputes", "moral_scenarios", "nutrition", "philosophy", "prehistory", "professional_accounting", "professional_law",
+    "professional_medicine", "professional_psychology", "public_relations", "security_studies", "sociology", "us_foreign_policy", "virology", "world_religions"
+]
+# fmt: on
+
+
+class CustomArabicMMLUTask(LightevalTaskConfig):
+    def __init__(
+        self,
+        name,
+        hf_subset,
+    ):
+        super().__init__(
+            name=name,
+            hf_subset=hf_subset,
+            prompt_function="mmlu_arabic",
+            hf_repo="OALL/Arabic_MMLU",
+            metric=["loglikelihood_acc"],
+            hf_avail_splits=["test", "dev"],
+            evaluation_splits=["test"],
+            few_shots_split="dev",
+            few_shots_select="sequential",
+            suite=["community"],
+            generation_size=-1,
+            stop_sequence=None,
+            output_regex=None,
+            frozen=False,
+        )
+
+
+ARABIC_MMLU_TASKS = [CustomArabicMMLUTask(name=f"mmlu:{subset}", hf_subset=subset) for subset in ARABIC_MMLU_SUBSETS]
+
+
+def mmlu_arabic(line, task_name: str = None):
+    topic = line["subject"]
+    instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n"
+    choices = [line["A"], line["B"], line["C"], line["D"]]
+    # Answers are provided with roman letters - we look for the correct index in LETTER_INDICES,
+    # it will then be applied to arabic letters
+    gold_ix = LETTER_INDICES.index(line["answer"])
+
+    query = f"{instruction}{line['question']}\n"
+    query += "".join([f"{key}. {choice}\n" for key, choice in zip(LETTER_INDICES_AR[:4], choices)])
+    query += "الإجابة:"
+
+    return Doc(
+        task_name=task_name,
+        query=query,
+        choices=LETTER_INDICES_AR[:4],
+        gold_index=gold_ix,
+        instruction=instruction,
+        target_for_fewshot_sorting=LETTER_INDICES_AR[gold_ix],
+    )
+
+
+## ACVA ##
+# fmt: off
+ACVA_SUBSETS = [
+    "Algeria", "Ancient_Egypt", "Arab_Empire", "Arabic_Architecture", "Arabic_Art", "Arabic_Astronomy", "Arabic_Calligraphy", "Arabic_Ceremony",
+    "Arabic_Clothing", "Arabic_Culture", "Arabic_Food", "Arabic_Funeral", "Arabic_Geography", "Arabic_History", "Arabic_Language_Origin",
+    "Arabic_Literature", "Arabic_Math", "Arabic_Medicine", "Arabic_Music", "Arabic_Ornament", "Arabic_Philosophy", "Arabic_Physics_and_Chemistry",
+    "Arabic_Wedding", "Bahrain", "Comoros", "Egypt_modern", "InfluenceFromAncientEgypt", "InfluenceFromByzantium", "InfluenceFromChina",
+    "InfluenceFromGreece", "InfluenceFromIslam", "InfluenceFromPersia", "InfluenceFromRome", "Iraq", "Islam_Education", "Islam_branches_and_schools",
+    "Islamic_law_system", "Jordan", "Kuwait", "Lebanon", "Libya", "Mauritania", "Mesopotamia_civilization", "Morocco", "Oman", "Palestine", "Qatar",
+    "Saudi_Arabia", "Somalia", "Sudan", "Syria", "Tunisia", "United_Arab_Emirates", "Yemen",
+    "communication", "computer_and_phone", "daily_life", "entertainment"
+]
+# fmt: on
+
+
+class CustomACVATask(LightevalTaskConfig):
+    def __init__(
+        self,
+        name,
+        hf_subset,
+    ):
+        super().__init__(
+            name=name,
+            hf_subset=hf_subset,
+            prompt_function="acva",
+            hf_repo="OALL/ACVA",
+            metric=["loglikelihood_acc"],
+            hf_avail_splits=["test", "validation"],
+            evaluation_splits=["test"],
+            few_shots_split="validation",
+            few_shots_select="sequential",
+            suite=["community"],
+            generation_size=-1,
+            stop_sequence=None,
+            output_regex=None,
+            frozen=False,
+        )
+
+
+ACVA_TASKS = [CustomACVATask(name=f"acva:{subset}", hf_subset=subset) for subset in ACVA_SUBSETS]
+
+
+def acva(line, task_name: str = None):
+    question = line["question"]
+    answer = line["answer"]
+
+    return Doc(
+        task_name=task_name,
+        query=f"السؤال: {question}\nالإجابة:",
+        choices=["صح", "خطأ"],
+        gold_index=["صح", "خطأ"].index(answer),
+    )
+
+
+## ARABIC EXAMS ##
+arabic_exams_task = LightevalTaskConfig(
+    name="arabic_exams",
+    prompt_function="arabic_exams",
+    suite=["community"],
+    hf_repo="OALL/Arabic_EXAMS",
+    hf_subset="default",
+    hf_avail_splits=["test", "validation"],
+    evaluation_splits=["test"],
+    few_shots_split="validation",
+    few_shots_select="sequential",
+    metric=["loglikelihood_acc"],
+)
+
+
+def arabic_exams(line, task_name: str = None):
+    topic = line["subject"]
+    question = line["question"]
+    choices = [line["A"], line["B"], line["C"], line["D"]]
+    choices_formatted = [f" {LETTER_INDICES_AR[i]}) {choice}\n" for i, choice in enumerate(choices)]
+    answer = line["answer"]
+    answer_index = LETTER_INDICES.index(answer)
+
+    instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n"
+    query = f"{instruction}السؤال: {question}\n"
+    query += "\n".join(choices_formatted)
+    query += "\nالإجابة:"
+
+    return Doc(
+        task_name=task_name,
+        query=query,
+        choices=LETTER_INDICES_AR[:4],
+        gold_index=answer_index,
+        instruction=instruction,
+        target_for_fewshot_sorting=choices[answer_index],
+    )
+
+
+_TASKS = ARABIC_MMLU_TASKS + ACVA_TASKS + [arabic_exams_task]
+
+# Convert to dict for lighteval
+TASKS_TABLE = [task.as_dict() for task in _TASKS]
+
+if __name__ == "__main__":
+    print(t["name"] for t in TASKS_TABLE)
+    print(len(TASKS_TABLE))

tasks_examples/OALL_tasks.txt

@@ -0,0 +1,117 @@
+community|arabic_mmlu:abstract_algebra|5|1
+community|arabic_mmlu:anatomy|5|1
+community|arabic_mmlu:astronomy|5|1
+community|arabic_mmlu:business_ethics|5|1
+community|arabic_mmlu:clinical_knowledge|5|1
+community|arabic_mmlu:college_biology|5|1
+community|arabic_mmlu:college_chemistry|5|1
+community|arabic_mmlu:college_computer_science|5|1
+community|arabic_mmlu:college_mathematics|5|1
+community|arabic_mmlu:college_medicine|5|1
+community|arabic_mmlu:college_physics|5|1
+community|arabic_mmlu:computer_security|5|1
+community|arabic_mmlu:conceptual_physics|5|1
+community|arabic_mmlu:econometrics|5|1
+community|arabic_mmlu:electrical_engineering|5|1
+community|arabic_mmlu:elementary_mathematics|5|1
+community|arabic_mmlu:formal_logic|5|1
+community|arabic_mmlu:global_facts|5|1
+community|arabic_mmlu:high_school_biology|5|1
+community|arabic_mmlu:high_school_chemistry|5|1
+community|arabic_mmlu:high_school_computer_science|5|1
+community|arabic_mmlu:high_school_european_history|5|1
+community|arabic_mmlu:high_school_geography|5|1
+community|arabic_mmlu:high_school_government_and_politics|5|1
+community|arabic_mmlu:high_school_macroeconomics|5|1
+community|arabic_mmlu:high_school_mathematics|5|1
+community|arabic_mmlu:high_school_microeconomics|5|1
+community|arabic_mmlu:high_school_physics|5|1
+community|arabic_mmlu:high_school_psychology|5|1
+community|arabic_mmlu:high_school_statistics|5|1
+community|arabic_mmlu:high_school_us_history|5|1
+community|arabic_mmlu:high_school_world_history|5|1
+community|arabic_mmlu:human_aging|5|1
+community|arabic_mmlu:human_sexuality|5|1
+community|arabic_mmlu:international_law|5|1
+community|arabic_mmlu:jurisprudence|5|1
+community|arabic_mmlu:logical_fallacies|5|1
+community|arabic_mmlu:machine_learning|5|1
+community|arabic_mmlu:management|5|1
+community|arabic_mmlu:marketing|5|1
+community|arabic_mmlu:medical_genetics|5|1
+community|arabic_mmlu:miscellaneous|5|1
+community|arabic_mmlu:moral_disputes|5|1
+community|arabic_mmlu:moral_scenarios|5|1
+community|arabic_mmlu:nutrition|5|1
+community|arabic_mmlu:philosophy|5|1
+community|arabic_mmlu:prehistory|5|1
+community|arabic_mmlu:professional_accounting|5|1
+community|arabic_mmlu:professional_law|5|1
+community|arabic_mmlu:professional_medicine|5|1
+community|arabic_mmlu:professional_psychology|5|1
+community|arabic_mmlu:public_relations|5|1
+community|arabic_mmlu:security_studies|5|1
+community|arabic_mmlu:sociology|5|1
+community|arabic_mmlu:us_foreign_policy|5|1
+community|arabic_mmlu:virology|5|1
+community|arabic_mmlu:world_religions|5|1
+community|arabic_exams|5|1
+community|acva:Algeria|5|1
+community|acva:Ancient_Egypt|5|1
+community|acva:Arab_Empire|5|1
+community|acva:Arabic_Architecture|5|1
+community|acva:Arabic_Art|5|1
+community|acva:Arabic_Astronomy|5|1
+community|acva:Arabic_Calligraphy|5|1
+community|acva:Arabic_Ceremony|5|1
+community|acva:Arabic_Clothing|5|1
+community|acva:Arabic_Culture|5|1
+community|acva:Arabic_Food|5|1
+community|acva:Arabic_Funeral|5|1
+community|acva:Arabic_Geography|5|1
+community|acva:Arabic_History|5|1
+community|acva:Arabic_Language_Origin|5|1
+community|acva:Arabic_Literature|5|1
+community|acva:Arabic_Math|5|1
+community|acva:Arabic_Medicine|5|1
+community|acva:Arabic_Music|5|1
+community|acva:Arabic_Ornament|5|1
+community|acva:Arabic_Philosophy|5|1
+community|acva:Arabic_Physics_and_Chemistry|5|1
+community|acva:Arabic_Wedding|5|1
+community|acva:Bahrain|5|1
+community|acva:Comoros|5|1
+community|acva:Egypt_modern|5|1
+community|acva:InfluenceFromAncientEgypt|5|1
+community|acva:InfluenceFromByzantium|5|1
+community|acva:InfluenceFromChina|5|1
+community|acva:InfluenceFromGreece|5|1
+community|acva:InfluenceFromIslam|5|1
+community|acva:InfluenceFromPersia|5|1
+community|acva:InfluenceFromRome|5|1
+community|acva:Iraq|5|1
+community|acva:Islam_Education|5|1
+community|acva:Islam_branches_and_schools|5|1
+community|acva:Islamic_law_system|5|1
+community|acva:Jordan|5|1
+community|acva:Kuwait|5|1
+community|acva:Lebanon|5|1
+community|acva:Libya|5|1
+community|acva:Mauritania|5|1
+community|acva:Mesopotamia_civilization|5|1
+community|acva:Morocco|5|1
+community|acva:Oman|5|1
+community|acva:Palestine|5|1
+community|acva:Qatar|5|1
+community|acva:Saudi_Arabia|5|1
+community|acva:Somalia|5|1
+community|acva:Sudan|5|1
+community|acva:Syria|5|1
+community|acva:Tunisia|5|1
+community|acva:United_Arab_Emirates|5|1
+community|acva:Yemen|5|1
+community|acva:communication|5|1
+community|acva:computer_and_phone|5|1
+community|acva:daily_life|5|1
+community|acva:entertainment|5|1
+lighteval|xstory_cloze:ar|0|0