correct_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from fuzzywuzzy import fuzz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_answer_from_choices(choices, index):\n",
    "    choice_list = eval(choices)\n",
    "    index = int(float(index))\n",
    "    return choice_list[index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_answers(correct_index, correct_answer, ai_answer):\n",
    "    # Preprocess answers\n",
    "    correct_answer = correct_answer.lower().strip()\n",
    "    ai_answer = str(ai_answer).lower().strip().replace('*','').replace('  ',' ')\n",
    "    \n",
    "    # Calculate fuzzy string similarity\n",
    "    similarity_ratio = fuzz.ratio(correct_answer, ai_answer)\n",
    "    \n",
    "    # Check if the AI's answer contains the correct answer\n",
    "    contains_correct = correct_answer in ai_answer\n",
    "    \n",
    "    # Check if the correct answer contains the AI's answer (for short AI answers)\n",
    "    contains_ai = ai_answer in correct_answer\n",
    "    \n",
    "    # Combine checks (you can adjust these conditions)\n",
    "    return similarity_ratio > 80 or contains_correct or contains_ai or str(correct_index) in ai_answer.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL = \"gemma_9b\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load your data\n",
    "df = pd.read_excel(f'{MODEL}/result_inference_mmlu.xlsx')  # Replace with your actual file path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Correct Answer'] = df.apply(lambda row: extract_answer_from_choices(row['Choices'], row['Answer Dataset']), axis=1)\n",
    "df['Good Answer'] = df.apply(lambda row: compare_answers(row['Answer Dataset'], row['Correct Answer'], row['Answer Json']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Comparison completed. Results saved to 'updated_data.csv'.\n"
     ]
    }
   ],
   "source": [
    "df.to_excel(f'{MODEL}/updated_data.xlsx', index=False)\n",
    "\n",
    "print(\"Comparison completed. Results saved to 'updated_data.csv'.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                Subject  Average Good Answer\n",
      "0                      abstract_algebra             0.272727\n",
      "1                               anatomy             0.571429\n",
      "2                             astronomy             0.375000\n",
      "3                       business_ethics             0.272727\n",
      "4                    clinical_knowledge             0.344828\n",
      "5                       college_biology             0.500000\n",
      "6                     college_chemistry             0.625000\n",
      "7              college_computer_science             0.272727\n",
      "8                   college_mathematics             0.181818\n",
      "9                      college_medicine             0.409091\n",
      "10                      college_physics             0.272727\n",
      "11                    computer_security             0.363636\n",
      "12                   conceptual_physics             0.423077\n",
      "13                         econometrics             0.333333\n",
      "14               electrical_engineering             0.375000\n",
      "15               elementary_mathematics             0.341463\n",
      "16                         formal_logic             0.214286\n",
      "17                         global_facts             0.200000\n",
      "18                  high_school_biology             0.312500\n",
      "19                high_school_chemistry             0.181818\n",
      "20         high_school_computer_science             0.111111\n",
      "21         high_school_european_history             0.555556\n",
      "22                high_school_geography             0.545455\n",
      "23  high_school_government_and_politics             0.476190\n",
      "24           high_school_macroeconomics             0.348837\n",
      "25              high_school_mathematics             0.241379\n",
      "26           high_school_microeconomics             0.269231\n",
      "27                  high_school_physics             0.117647\n",
      "28               high_school_psychology             0.483333\n",
      "29               high_school_statistics             0.391304\n",
      "30               high_school_us_history             0.409091\n",
      "31            high_school_world_history             0.230769\n",
      "32                          human_aging             0.347826\n",
      "33                      human_sexuality             0.250000\n",
      "34                    international_law             0.230769\n",
      "35                        jurisprudence             0.181818\n",
      "36                    logical_fallacies             0.277778\n",
      "37                     machine_learning             0.454545\n",
      "38                           management             0.636364\n",
      "39                            marketing             0.640000\n",
      "40                     medical_genetics             0.454545\n",
      "41                        miscellaneous             0.500000\n",
      "42                       moral_disputes             0.236842\n",
      "43                      moral_scenarios             0.360000\n",
      "44                            nutrition             0.393939\n",
      "45                           philosophy             0.470588\n",
      "46                           prehistory             0.342857\n",
      "47              professional_accounting             0.193548\n",
      "48                     professional_law             0.270588\n",
      "49                professional_medicine             0.290323\n",
      "50              professional_psychology             0.333333\n",
      "51                     public_relations             0.250000\n",
      "52                     security_studies             0.296296\n",
      "53                            sociology             0.409091\n",
      "54                    us_foreign_policy             0.636364\n",
      "55                             virology             0.166667\n",
      "56                      world_religions             0.578947\n"
     ]
    }
   ],
   "source": [
    "result = df.groupby('Subject')['Good Answer'].mean().reset_index()\n",
    "result.columns = ['Subject', 'Average Good Answer']\n",
    "\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_excel('test.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}