llm_evaluation_mmlu_phi_2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mqLTcQ1wM7LH"
      },
      "source": [
        "# LLM Evaluation for mmlu dataset\n",
        "\n",
        "Google Colab Environment: T4 GPU (free tier can use this.)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qROaxTorqUB0",
        "outputId": "0761d510-b8a7-4b42-c3db-bfea5dbc1d2d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'llm_model_evaluation'...\n",
            "remote: Enumerating objects: 94, done.\u001b[K\n",
            "remote: Counting objects: 100% (94/94), done.\u001b[K\n",
            "remote: Compressing objects: 100% (54/54), done.\u001b[K\n",
            "remote: Total 94 (delta 50), reused 79 (delta 35), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (94/94), 31.71 KiB | 4.53 MiB/s, done.\n",
            "Resolving deltas: 100% (50/50), done.\n"
          ]
        }
      ],
      "source": [
        "# Git clone the evaluation code\n",
        "\n",
        "!git clone https://github.com/LiuYuWei/llm_model_evaluation.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NEFcbdU8qdMD",
        "outputId": "9a6ad0d8-68a2-481c-e0c2-255a29c30d22"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2024-01-17 09:06:07--  https://people.eecs.berkeley.edu/~hendrycks/data.tar\n",
            "Resolving people.eecs.berkeley.edu (people.eecs.berkeley.edu)... 128.32.244.190\n",
            "Connecting to people.eecs.berkeley.edu (people.eecs.berkeley.edu)|128.32.244.190|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 166184960 (158M) [application/x-tar]\n",
            "Saving to: ‘data.tar’\n",
            "\n",
            "data.tar            100%[===================>] 158.49M  38.7MB/s    in 4.5s    \n",
            "\n",
            "2024-01-17 09:06:12 (34.9 MB/s) - ‘data.tar’ saved [166184960/166184960]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Download mmlu dataset\n",
        "\n",
        "!wget https://people.eecs.berkeley.edu/~hendrycks/data.tar"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NUi3dL0QrGQg",
        "outputId": "09c9bdfc-9acd-4b3c-c8d1-b1335fcfbdf5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "data/\n",
            "data/README.txt\n",
            "data/val/\n",
            "data/val/high_school_geography_val.csv\n",
            "data/val/astronomy_val.csv\n",
            "data/val/conceptual_physics_val.csv\n",
            "data/val/medical_genetics_val.csv\n",
            "data/val/high_school_computer_science_val.csv\n",
            "data/val/business_ethics_val.csv\n",
            "data/val/anatomy_val.csv\n",
            "data/val/global_facts_val.csv\n",
            "data/val/philosophy_val.csv\n",
            "data/val/high_school_government_and_politics_val.csv\n",
            "data/val/college_physics_val.csv\n",
            "data/val/miscellaneous_val.csv\n",
            "data/val/machine_learning_val.csv\n",
            "data/val/high_school_mathematics_val.csv\n",
            "data/val/high_school_biology_val.csv\n",
            "data/val/abstract_algebra_val.csv\n",
            "data/val/college_medicine_val.csv\n",
            "data/val/clinical_knowledge_val.csv\n",
            "data/val/electrical_engineering_val.csv\n",
            "data/val/econometrics_val.csv\n",
            "data/val/world_religions_val.csv\n",
            "data/val/us_foreign_policy_val.csv\n",
            "data/val/formal_logic_val.csv\n",
            "data/val/college_mathematics_val.csv\n",
            "data/val/high_school_macroeconomics_val.csv\n",
            "data/val/high_school_statistics_val.csv\n",
            "data/val/high_school_physics_val.csv\n",
            "data/val/international_law_val.csv\n",
            "data/val/nutrition_val.csv\n",
            "data/val/computer_security_val.csv\n",
            "data/val/moral_scenarios_val.csv\n",
            "data/val/human_sexuality_val.csv\n",
            "data/val/management_val.csv\n",
            "data/val/human_aging_val.csv\n",
            "data/val/prehistory_val.csv\n",
            "data/val/virology_val.csv\n",
            "data/val/professional_medicine_val.csv\n",
            "data/val/logical_fallacies_val.csv\n",
            "data/val/public_relations_val.csv\n",
            "data/val/marketing_val.csv\n",
            "data/val/college_computer_science_val.csv\n",
            "data/val/high_school_chemistry_val.csv\n",
            "data/val/elementary_mathematics_val.csv\n",
            "data/val/professional_law_val.csv\n",
            "data/val/high_school_microeconomics_val.csv\n",
            "data/val/professional_psychology_val.csv\n",
            "data/val/college_biology_val.csv\n",
            "data/val/college_chemistry_val.csv\n",
            "data/val/high_school_psychology_val.csv\n",
            "data/val/jurisprudence_val.csv\n",
            "data/val/sociology_val.csv\n",
            "data/val/high_school_world_history_val.csv\n",
            "data/val/high_school_european_history_val.csv\n",
            "data/val/high_school_us_history_val.csv\n",
            "data/val/moral_disputes_val.csv\n",
            "data/val/professional_accounting_val.csv\n",
            "data/val/security_studies_val.csv\n",
            "data/possibly_contaminated_urls.txt\n",
            "data/auxiliary_train/\n",
            "data/auxiliary_train/aux_law_90s.csv\n",
            "data/auxiliary_train/science_middle.csv\n",
            "data/auxiliary_train/arc_hard.csv\n",
            "data/auxiliary_train/mc_test.csv\n",
            "data/auxiliary_train/science_elementary.csv\n",
            "data/auxiliary_train/race.csv\n",
            "data/auxiliary_train/arc_easy.csv\n",
            "data/auxiliary_train/obqa.csv\n",
            "data/test/\n",
            "data/test/high_school_government_and_politics_test.csv\n",
            "data/test/security_studies_test.csv\n",
            "data/test/global_facts_test.csv\n",
            "data/test/sociology_test.csv\n",
            "data/test/high_school_european_history_test.csv\n",
            "data/test/college_biology_test.csv\n",
            "data/test/high_school_psychology_test.csv\n",
            "data/test/astronomy_test.csv\n",
            "data/test/electrical_engineering_test.csv\n",
            "data/test/logical_fallacies_test.csv\n",
            "data/test/nutrition_test.csv\n",
            "data/test/high_school_biology_test.csv\n",
            "data/test/high_school_macroeconomics_test.csv\n",
            "data/test/virology_test.csv\n",
            "data/test/machine_learning_test.csv\n",
            "data/test/jurisprudence_test.csv\n",
            "data/test/professional_psychology_test.csv\n",
            "data/test/abstract_algebra_test.csv\n",
            "data/test/econometrics_test.csv\n",
            "data/test/high_school_mathematics_test.csv\n",
            "data/test/high_school_computer_science_test.csv\n",
            "data/test/philosophy_test.csv\n",
            "data/test/college_chemistry_test.csv\n",
            "data/test/human_sexuality_test.csv\n",
            "data/test/high_school_chemistry_test.csv\n",
            "data/test/human_aging_test.csv\n",
            "data/test/anatomy_test.csv\n",
            "data/test/management_test.csv\n",
            "data/test/college_medicine_test.csv\n",
            "data/test/computer_security_test.csv\n",
            "data/test/marketing_test.csv\n",
            "data/test/conceptual_physics_test.csv\n",
            "data/test/medical_genetics_test.csv\n",
            "data/test/public_relations_test.csv\n",
            "data/test/world_religions_test.csv\n",
            "data/test/high_school_us_history_test.csv\n",
            "data/test/international_law_test.csv\n",
            "data/test/professional_law_test.csv\n",
            "data/test/high_school_physics_test.csv\n",
            "data/test/moral_disputes_test.csv\n",
            "data/test/high_school_world_history_test.csv\n",
            "data/test/professional_medicine_test.csv\n",
            "data/test/miscellaneous_test.csv\n",
            "data/test/high_school_microeconomics_test.csv\n",
            "data/test/business_ethics_test.csv\n",
            "data/test/clinical_knowledge_test.csv\n",
            "data/test/formal_logic_test.csv\n",
            "data/test/college_physics_test.csv\n",
            "data/test/high_school_statistics_test.csv\n",
            "data/test/professional_accounting_test.csv\n",
            "data/test/elementary_mathematics_test.csv\n",
            "data/test/college_mathematics_test.csv\n",
            "data/test/moral_scenarios_test.csv\n",
            "data/test/college_computer_science_test.csv\n",
            "data/test/high_school_geography_test.csv\n",
            "data/test/us_foreign_policy_test.csv\n",
            "data/test/prehistory_test.csv\n",
            "data/dev/\n",
            "data/dev/professional_accounting_dev.csv\n",
            "data/dev/clinical_knowledge_dev.csv\n",
            "data/dev/college_medicine_dev.csv\n",
            "data/dev/college_mathematics_dev.csv\n",
            "data/dev/high_school_european_history_dev.csv\n",
            "data/dev/logical_fallacies_dev.csv\n",
            "data/dev/anatomy_dev.csv\n",
            "data/dev/human_aging_dev.csv\n",
            "data/dev/international_law_dev.csv\n",
            "data/dev/high_school_chemistry_dev.csv\n",
            "data/dev/formal_logic_dev.csv\n",
            "data/dev/public_relations_dev.csv\n",
            "data/dev/nutrition_dev.csv\n",
            "data/dev/high_school_geography_dev.csv\n",
            "data/dev/high_school_government_and_politics_dev.csv\n",
            "data/dev/high_school_macroeconomics_dev.csv\n",
            "data/dev/marketing_dev.csv\n",
            "data/dev/business_ethics_dev.csv\n",
            "data/dev/high_school_computer_science_dev.csv\n",
            "data/dev/college_biology_dev.csv\n",
            "data/dev/college_physics_dev.csv\n",
            "data/dev/us_foreign_policy_dev.csv\n",
            "data/dev/philosophy_dev.csv\n",
            "data/dev/virology_dev.csv\n",
            "data/dev/professional_medicine_dev.csv\n",
            "data/dev/abstract_algebra_dev.csv\n",
            "data/dev/machine_learning_dev.csv\n",
            "data/dev/sociology_dev.csv\n",
            "data/dev/elementary_mathematics_dev.csv\n",
            "data/dev/management_dev.csv\n",
            "data/dev/medical_genetics_dev.csv\n",
            "data/dev/moral_disputes_dev.csv\n",
            "data/dev/high_school_biology_dev.csv\n",
            "data/dev/moral_scenarios_dev.csv\n",
            "data/dev/security_studies_dev.csv\n",
            "data/dev/prehistory_dev.csv\n",
            "data/dev/high_school_mathematics_dev.csv\n",
            "data/dev/global_facts_dev.csv\n",
            "data/dev/high_school_statistics_dev.csv\n",
            "data/dev/college_computer_science_dev.csv\n",
            "data/dev/high_school_world_history_dev.csv\n",
            "data/dev/human_sexuality_dev.csv\n",
            "data/dev/econometrics_dev.csv\n",
            "data/dev/high_school_us_history_dev.csv\n",
            "data/dev/professional_psychology_dev.csv\n",
            "data/dev/computer_security_dev.csv\n",
            "data/dev/world_religions_dev.csv\n",
            "data/dev/electrical_engineering_dev.csv\n",
            "data/dev/jurisprudence_dev.csv\n",
            "data/dev/high_school_microeconomics_dev.csv\n",
            "data/dev/college_chemistry_dev.csv\n",
            "data/dev/professional_law_dev.csv\n",
            "data/dev/astronomy_dev.csv\n",
            "data/dev/miscellaneous_dev.csv\n",
            "data/dev/conceptual_physics_dev.csv\n",
            "data/dev/high_school_psychology_dev.csv\n",
            "data/dev/high_school_physics_dev.csv\n"
          ]
        }
      ],
      "source": [
        "# Unzip the data.tar file.\n",
        "\n",
        "!tar xvf data.tar"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pxIIBQtqqdHe",
        "outputId": "fbbc7055-b2a2-4bc4-f4ba-1699332c02d5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Git LFS initialized.\n",
            "Cloning into 'phi-2'...\n",
            "remote: Enumerating objects: 87, done.\u001b[K\n",
            "remote: Counting objects: 100% (76/76), done.\u001b[K\n",
            "remote: Compressing objects: 100% (76/76), done.\u001b[K\n",
            "remote: Total 87 (delta 38), reused 0 (delta 0), pack-reused 11\u001b[K\n",
            "Unpacking objects: 100% (87/87), 1.14 MiB | 1.90 MiB/s, done.\n",
            "Filtering content: 100% (2/2), 1.17 GiB | 10.23 MiB/s, done.\n",
            "Encountered 1 file(s) that may not have been copied correctly on Windows:\n",
            "\tmodel-00001-of-00002.safetensors\n",
            "\n",
            "See: `git lfs help smudge` for more details.\n"
          ]
        }
      ],
      "source": [
        "# Git clone the huggingface model\n",
        "## It might cost lots of time.\n",
        "\n",
        "!git lfs install\n",
        "!git clone https://huggingface.co/microsoft/phi-2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TGq51fSUvvVj",
        "outputId": "49952d81-8039-41fd-9b39-925f8e89f406"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/270.9 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m122.9/270.9 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m270.9/270.9 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "# python package installation\n",
        "\n",
        "!pip install -q accelerate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u-ZKy3fVrnZc",
        "outputId": "144f68b3-4ce2-493b-e720-6931a6980584"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Found existing installation: transformers 4.35.2\n",
            "Uninstalling transformers-4.35.2:\n",
            "  Successfully uninstalled transformers-4.35.2\n",
            "Collecting git+https://github.com/huggingface/transformers\n",
            "  Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-29lyveqv\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers /tmp/pip-req-build-29lyveqv\n",
            "  Resolved https://github.com/huggingface/transformers to commit f4f57f9dfa68948a383c352a900d588f63f6290a\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.0.dev0) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.0.dev0) (0.20.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.0.dev0) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.0.dev0) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.0.dev0) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.0.dev0) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.0.dev0) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.0.dev0) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.0.dev0) (0.4.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.0.dev0) (4.66.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.37.0.dev0) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.37.0.dev0) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.37.0.dev0) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.37.0.dev0) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.37.0.dev0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.37.0.dev0) (2023.11.17)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.37.0.dev0-py3-none-any.whl size=8340729 sha256=1c412a9613c6ae5704f2bee04ecd0fdf27f1febd0f35878989fc4e286c93017b\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-3e3siaxg/wheels/c0/14/d6/6c9a5582d2ac191ec0a483be151a4495fe1eb2a6706ca49f1b\n",
            "Successfully built transformers\n",
            "Installing collected packages: transformers\n",
            "Successfully installed transformers-4.37.0.dev0\n"
          ]
        }
      ],
      "source": [
        "# Need to use the latest transformers\n",
        "\n",
        "!pip uninstall -y transformers\n",
        "!pip install git+https://github.com/huggingface/transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WwdPAUOoqdJy",
        "outputId": "65a9b381-3eac-4c04-c597-82d0dbdf6d19"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wed, 17 Jan 2024 09:09:54 [INFO][evaluate_hf - main] The type of Category: mmlu\n",
            "Wed, 17 Jan 2024 09:09:56 [INFO][modeling - get_balanced_memory] We will use 90% of the memory on device 0 for storing the model, and 10% for the buffer to avoid OOM. You can set `max_memory` in to a higher value to use more memory (at your own risk).\n",
            "Loading checkpoint shards: 100% 2/2 [00:30<00:00, 15.44s/it]\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
            "Wed, 17 Jan 2024 09:10:27 [INFO][evaluate_hf - main] Start: abstract_algebra\n",
            "Wed, 17 Jan 2024 09:10:43 [INFO][evaluate_hf - eval] Average accuracy 0.340 - abstract_algebra\n",
            "Wed, 17 Jan 2024 09:10:43 [INFO][evaluate_hf - main] Start: anatomy\n",
            "Wed, 17 Jan 2024 09:11:02 [INFO][evaluate_hf - eval] Average accuracy 0.452 - anatomy\n",
            "Wed, 17 Jan 2024 09:11:02 [INFO][evaluate_hf - main] Start: astronomy\n",
            "Wed, 17 Jan 2024 09:11:43 [INFO][evaluate_hf - eval] Average accuracy 0.605 - astronomy\n",
            "Wed, 17 Jan 2024 09:11:43 [INFO][evaluate_hf - main] Start: business_ethics\n"
          ]
        }
      ],
      "source": [
        "# Run the script the evaluate the dataset\n",
        "\n",
        "!python3 /content/llm_model_evaluation/evaluation_hf_testing.py \\\n",
        "    --category_type mmlu \\\n",
        "    --model /content/phi-2 \\\n",
        "    --data_dir \"/content/data/\" \\\n",
        "    --save_dir \"/content/result/\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!python /content/llm_model_evaluation/catogories_result_eval.py \\\n",
        "    --category_type \"mmlu\" \\\n",
        "    --model \"/content/phi-2\" \\\n",
        "    --save_dir \"/content/result/results_phi-2\""
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}