llm_evaluation_mmlu.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mqLTcQ1wM7LH"
      },
      "source": [
        "# LLM Evaluation for mmlu dataset\n",
        "\n",
        "Google Colab Environment: T4 GPU in large RAM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qROaxTorqUB0",
        "outputId": "c189032a-953a-4aae-a7d2-9e68bd473a2c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'llm_evaluation_4_mmlu'...\n",
            "remote: Enumerating objects: 33, done.\u001b[K\n",
            "remote: Counting objects: 100% (33/33), done.\u001b[K\n",
            "remote: Compressing objects: 100% (27/27), done.\u001b[K\n",
            "remote: Total 33 (delta 8), reused 29 (delta 4), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (33/33), 1.05 MiB | 13.47 MiB/s, done.\n",
            "Resolving deltas: 100% (8/8), done.\n"
          ]
        }
      ],
      "source": [
        "# Git clone the evaluation code\n",
        "\n",
        "!git clone https://github.com/LiuYuWei/llm_model_evaluation.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NEFcbdU8qdMD",
        "outputId": "82eb5ece-f7a6-4ee8-e55d-d6b7d53e97a6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2024-01-08 17:02:32--  https://people.eecs.berkeley.edu/~hendrycks/data.tar\n",
            "Resolving people.eecs.berkeley.edu (people.eecs.berkeley.edu)... 128.32.244.190\n",
            "Connecting to people.eecs.berkeley.edu (people.eecs.berkeley.edu)|128.32.244.190|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 166184960 (158M) [application/x-tar]\n",
            "Saving to: ‘data.tar’\n",
            "\n",
            "data.tar            100%[===================>] 158.49M  40.6MB/s    in 4.4s    \n",
            "\n",
            "2024-01-08 17:02:36 (36.4 MB/s) - ‘data.tar’ saved [166184960/166184960]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Download mmlu dataset\n",
        "\n",
        "!wget https://people.eecs.berkeley.edu/~hendrycks/data.tar"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NUi3dL0QrGQg",
        "outputId": "bfa82d1d-32b3-4948-a2c6-b68731094a81"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "data/\n",
            "data/README.txt\n",
            "data/val/\n",
            "data/val/high_school_geography_val.csv\n",
            "data/val/astronomy_val.csv\n",
            "data/val/conceptual_physics_val.csv\n",
            "data/val/medical_genetics_val.csv\n",
            "data/val/high_school_computer_science_val.csv\n",
            "data/val/business_ethics_val.csv\n",
            "data/val/anatomy_val.csv\n",
            "data/val/global_facts_val.csv\n",
            "data/val/philosophy_val.csv\n",
            "data/val/high_school_government_and_politics_val.csv\n",
            "data/val/college_physics_val.csv\n",
            "data/val/miscellaneous_val.csv\n",
            "data/val/machine_learning_val.csv\n",
            "data/val/high_school_mathematics_val.csv\n",
            "data/val/high_school_biology_val.csv\n",
            "data/val/abstract_algebra_val.csv\n",
            "data/val/college_medicine_val.csv\n",
            "data/val/clinical_knowledge_val.csv\n",
            "data/val/electrical_engineering_val.csv\n",
            "data/val/econometrics_val.csv\n",
            "data/val/world_religions_val.csv\n",
            "data/val/us_foreign_policy_val.csv\n",
            "data/val/formal_logic_val.csv\n",
            "data/val/college_mathematics_val.csv\n",
            "data/val/high_school_macroeconomics_val.csv\n",
            "data/val/high_school_statistics_val.csv\n",
            "data/val/high_school_physics_val.csv\n",
            "data/val/international_law_val.csv\n",
            "data/val/nutrition_val.csv\n",
            "data/val/computer_security_val.csv\n",
            "data/val/moral_scenarios_val.csv\n",
            "data/val/human_sexuality_val.csv\n",
            "data/val/management_val.csv\n",
            "data/val/human_aging_val.csv\n",
            "data/val/prehistory_val.csv\n",
            "data/val/virology_val.csv\n",
            "data/val/professional_medicine_val.csv\n",
            "data/val/logical_fallacies_val.csv\n",
            "data/val/public_relations_val.csv\n",
            "data/val/marketing_val.csv\n",
            "data/val/college_computer_science_val.csv\n",
            "data/val/high_school_chemistry_val.csv\n",
            "data/val/elementary_mathematics_val.csv\n",
            "data/val/professional_law_val.csv\n",
            "data/val/high_school_microeconomics_val.csv\n",
            "data/val/professional_psychology_val.csv\n",
            "data/val/college_biology_val.csv\n",
            "data/val/college_chemistry_val.csv\n",
            "data/val/high_school_psychology_val.csv\n",
            "data/val/jurisprudence_val.csv\n",
            "data/val/sociology_val.csv\n",
            "data/val/high_school_world_history_val.csv\n",
            "data/val/high_school_european_history_val.csv\n",
            "data/val/high_school_us_history_val.csv\n",
            "data/val/moral_disputes_val.csv\n",
            "data/val/professional_accounting_val.csv\n",
            "data/val/security_studies_val.csv\n",
            "data/possibly_contaminated_urls.txt\n",
            "data/auxiliary_train/\n",
            "data/auxiliary_train/aux_law_90s.csv\n",
            "data/auxiliary_train/science_middle.csv\n",
            "data/auxiliary_train/arc_hard.csv\n",
            "data/auxiliary_train/mc_test.csv\n",
            "data/auxiliary_train/science_elementary.csv\n",
            "data/auxiliary_train/race.csv\n",
            "data/auxiliary_train/arc_easy.csv\n",
            "data/auxiliary_train/obqa.csv\n",
            "data/test/\n",
            "data/test/high_school_government_and_politics_test.csv\n",
            "data/test/security_studies_test.csv\n",
            "data/test/global_facts_test.csv\n",
            "data/test/sociology_test.csv\n",
            "data/test/high_school_european_history_test.csv\n",
            "data/test/college_biology_test.csv\n",
            "data/test/high_school_psychology_test.csv\n",
            "data/test/astronomy_test.csv\n",
            "data/test/electrical_engineering_test.csv\n",
            "data/test/logical_fallacies_test.csv\n",
            "data/test/nutrition_test.csv\n",
            "data/test/high_school_biology_test.csv\n",
            "data/test/high_school_macroeconomics_test.csv\n",
            "data/test/virology_test.csv\n",
            "data/test/machine_learning_test.csv\n",
            "data/test/jurisprudence_test.csv\n",
            "data/test/professional_psychology_test.csv\n",
            "data/test/abstract_algebra_test.csv\n",
            "data/test/econometrics_test.csv\n",
            "data/test/high_school_mathematics_test.csv\n",
            "data/test/high_school_computer_science_test.csv\n",
            "data/test/philosophy_test.csv\n",
            "data/test/college_chemistry_test.csv\n",
            "data/test/human_sexuality_test.csv\n",
            "data/test/high_school_chemistry_test.csv\n",
            "data/test/human_aging_test.csv\n",
            "data/test/anatomy_test.csv\n",
            "data/test/management_test.csv\n",
            "data/test/college_medicine_test.csv\n",
            "data/test/computer_security_test.csv\n",
            "data/test/marketing_test.csv\n",
            "data/test/conceptual_physics_test.csv\n",
            "data/test/medical_genetics_test.csv\n",
            "data/test/public_relations_test.csv\n",
            "data/test/world_religions_test.csv\n",
            "data/test/high_school_us_history_test.csv\n",
            "data/test/international_law_test.csv\n",
            "data/test/professional_law_test.csv\n",
            "data/test/high_school_physics_test.csv\n",
            "data/test/moral_disputes_test.csv\n",
            "data/test/high_school_world_history_test.csv\n",
            "data/test/professional_medicine_test.csv\n",
            "data/test/miscellaneous_test.csv\n",
            "data/test/high_school_microeconomics_test.csv\n",
            "data/test/business_ethics_test.csv\n",
            "data/test/clinical_knowledge_test.csv\n",
            "data/test/formal_logic_test.csv\n",
            "data/test/college_physics_test.csv\n",
            "data/test/high_school_statistics_test.csv\n",
            "data/test/professional_accounting_test.csv\n",
            "data/test/elementary_mathematics_test.csv\n",
            "data/test/college_mathematics_test.csv\n",
            "data/test/moral_scenarios_test.csv\n",
            "data/test/college_computer_science_test.csv\n",
            "data/test/high_school_geography_test.csv\n",
            "data/test/us_foreign_policy_test.csv\n",
            "data/test/prehistory_test.csv\n",
            "data/dev/\n",
            "data/dev/professional_accounting_dev.csv\n",
            "data/dev/clinical_knowledge_dev.csv\n",
            "data/dev/college_medicine_dev.csv\n",
            "data/dev/college_mathematics_dev.csv\n",
            "data/dev/high_school_european_history_dev.csv\n",
            "data/dev/logical_fallacies_dev.csv\n",
            "data/dev/anatomy_dev.csv\n",
            "data/dev/human_aging_dev.csv\n",
            "data/dev/international_law_dev.csv\n",
            "data/dev/high_school_chemistry_dev.csv\n",
            "data/dev/formal_logic_dev.csv\n",
            "data/dev/public_relations_dev.csv\n",
            "data/dev/nutrition_dev.csv\n",
            "data/dev/high_school_geography_dev.csv\n",
            "data/dev/high_school_government_and_politics_dev.csv\n",
            "data/dev/high_school_macroeconomics_dev.csv\n",
            "data/dev/marketing_dev.csv\n",
            "data/dev/business_ethics_dev.csv\n",
            "data/dev/high_school_computer_science_dev.csv\n",
            "data/dev/college_biology_dev.csv\n",
            "data/dev/college_physics_dev.csv\n",
            "data/dev/us_foreign_policy_dev.csv\n",
            "data/dev/philosophy_dev.csv\n",
            "data/dev/virology_dev.csv\n",
            "data/dev/professional_medicine_dev.csv\n",
            "data/dev/abstract_algebra_dev.csv\n",
            "data/dev/machine_learning_dev.csv\n",
            "data/dev/sociology_dev.csv\n",
            "data/dev/elementary_mathematics_dev.csv\n",
            "data/dev/management_dev.csv\n",
            "data/dev/medical_genetics_dev.csv\n",
            "data/dev/moral_disputes_dev.csv\n",
            "data/dev/high_school_biology_dev.csv\n",
            "data/dev/moral_scenarios_dev.csv\n",
            "data/dev/security_studies_dev.csv\n",
            "data/dev/prehistory_dev.csv\n",
            "data/dev/high_school_mathematics_dev.csv\n",
            "data/dev/global_facts_dev.csv\n",
            "data/dev/high_school_statistics_dev.csv\n",
            "data/dev/college_computer_science_dev.csv\n",
            "data/dev/high_school_world_history_dev.csv\n",
            "data/dev/human_sexuality_dev.csv\n",
            "data/dev/econometrics_dev.csv\n",
            "data/dev/high_school_us_history_dev.csv\n",
            "data/dev/professional_psychology_dev.csv\n",
            "data/dev/computer_security_dev.csv\n",
            "data/dev/world_religions_dev.csv\n",
            "data/dev/electrical_engineering_dev.csv\n",
            "data/dev/jurisprudence_dev.csv\n",
            "data/dev/high_school_microeconomics_dev.csv\n",
            "data/dev/college_chemistry_dev.csv\n",
            "data/dev/professional_law_dev.csv\n",
            "data/dev/astronomy_dev.csv\n",
            "data/dev/miscellaneous_dev.csv\n",
            "data/dev/conceptual_physics_dev.csv\n",
            "data/dev/high_school_psychology_dev.csv\n",
            "data/dev/high_school_physics_dev.csv\n"
          ]
        }
      ],
      "source": [
        "# Unzip the data.tar file.\n",
        "\n",
        "!tar xvf data.tar"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pxIIBQtqqdHe",
        "outputId": "c079ed06-58fe-40cd-8ef0-883ebfb9aa3d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Git LFS initialized.\n",
            "Cloning into 'Mistral-7B-v0.1'...\n",
            "remote: Enumerating objects: 87, done.\u001b[K\n",
            "remote: Counting objects: 100% (83/83), done.\u001b[K\n",
            "remote: Compressing objects: 100% (82/82), done.\u001b[K\n",
            "remote: Total 87 (delta 43), reused 0 (delta 0), pack-reused 4\u001b[K\n",
            "Unpacking objects: 100% (87/87), 473.17 KiB | 3.20 MiB/s, done.\n",
            "Filtering content: 100% (5/5), 3.46 GiB | 10.58 MiB/s, done.\n",
            "Encountered 4 file(s) that may not have been copied correctly on Windows:\n",
            "\tmodel-00002-of-00002.safetensors\n",
            "\tpytorch_model-00002-of-00002.bin\n",
            "\tpytorch_model-00001-of-00002.bin\n",
            "\tmodel-00001-of-00002.safetensors\n",
            "\n",
            "See: `git lfs help smudge` for more details.\n"
          ]
        }
      ],
      "source": [
        "# Git clone the huggingface model\n",
        "## It might cost lots of time.\n",
        "\n",
        "!git lfs install\n",
        "!git clone https://huggingface.co/mistralai/Mistral-7B-v0.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TGq51fSUvvVj",
        "outputId": "eefa9a8d-88bb-4836-8b4f-85f2f7ac4e8e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/265.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.2/265.7 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m265.7/265.7 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "# python package installation\n",
        "\n",
        "!pip install -q accelerate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WwdPAUOoqdJy",
        "outputId": "555ef1cc-5e8d-43c1-ab70-cb5bd3a5d78e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loading checkpoint shards: 100% 2/2 [00:25<00:00, 12.91s/it]\n",
            "WARNING:root:Some parameters are on the meta device device because they were offloaded to the cpu.\n",
            "2024-01-08 17:09:01.940169: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-08 17:09:01.940270: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-08 17:09:02.090192: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-08 17:09:04.906540: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Average accuracy 0.280 - abstract_algebra\n",
            "Average accuracy 0.622 - anatomy\n",
            "Average accuracy 0.658 - astronomy\n",
            "Average accuracy 0.570 - business_ethics\n",
            "Average accuracy 0.698 - clinical_knowledge\n",
            "Average accuracy 0.729 - college_biology\n",
            "Average accuracy 0.500 - college_chemistry\n",
            "Average accuracy 0.520 - college_computer_science\n",
            "Average accuracy 0.410 - college_mathematics\n",
            "Average accuracy 0.647 - college_medicine\n",
            "Average accuracy 0.392 - college_physics\n",
            "Average accuracy 0.770 - computer_security\n",
            "Average accuracy 0.570 - conceptual_physics\n",
            "Average accuracy 0.491 - econometrics\n",
            "Average accuracy 0.579 - electrical_engineering\n",
            "Average accuracy 0.376 - elementary_mathematics\n",
            "Average accuracy 0.413 - formal_logic\n",
            "Average accuracy 0.360 - global_facts\n",
            "Average accuracy 0.768 - high_school_biology\n",
            "Average accuracy 0.532 - high_school_chemistry\n",
            "Average accuracy 0.680 - high_school_computer_science\n",
            "Average accuracy 0.794 - high_school_european_history\n",
            "Average accuracy 0.768 - high_school_geography\n",
            "Average accuracy 0.865 - high_school_government_and_politics\n",
            "Average accuracy 0.662 - high_school_macroeconomics\n",
            "Average accuracy 0.337 - high_school_mathematics\n",
            "Average accuracy 0.664 - high_school_microeconomics\n",
            "Average accuracy 0.318 - high_school_physics\n",
            "Average accuracy 0.824 - high_school_psychology\n",
            "Average accuracy 0.574 - high_school_statistics\n",
            "Average accuracy 0.789 - high_school_us_history\n",
            "Average accuracy 0.776 - high_school_world_history\n",
            "Average accuracy 0.700 - human_aging\n",
            "Average accuracy 0.786 - human_sexuality\n",
            "Average accuracy 0.777 - international_law\n",
            "Average accuracy 0.778 - jurisprudence\n",
            "Average accuracy 0.791 - logical_fallacies\n",
            "Average accuracy 0.491 - machine_learning\n",
            "Average accuracy 0.825 - management\n",
            "Average accuracy 0.880 - marketing\n",
            "Average accuracy 0.740 - medical_genetics\n",
            "Average accuracy 0.816 - miscellaneous\n",
            "Average accuracy 0.711 - moral_disputes\n",
            "Average accuracy 0.322 - moral_scenarios\n",
            "Average accuracy 0.755 - nutrition\n",
            "Average accuracy 0.695 - philosophy\n",
            "Average accuracy 0.728 - prehistory\n",
            "Average accuracy 0.489 - professional_accounting\n",
            "Average accuracy 0.449 - professional_law\n",
            "Average accuracy 0.688 - professional_medicine\n",
            "Average accuracy 0.680 - professional_psychology\n",
            "Average accuracy 0.673 - public_relations\n",
            "Average accuracy 0.727 - security_studies\n",
            "Average accuracy 0.831 - sociology\n",
            "Average accuracy 0.860 - us_foreign_policy\n",
            "Average accuracy 0.548 - virology\n",
            "Average accuracy 0.830 - world_religions\n",
            "Average accuracy 0.400 - math\n",
            "Average accuracy 0.683 - health\n",
            "Average accuracy 0.503 - physics\n",
            "Average accuracy 0.796 - business\n",
            "Average accuracy 0.756 - biology\n",
            "Average accuracy 0.521 - chemistry\n",
            "Average accuracy 0.612 - computer science\n",
            "Average accuracy 0.636 - economics\n",
            "Average accuracy 0.579 - engineering\n",
            "Average accuracy 0.533 - philosophy\n",
            "Average accuracy 0.698 - other\n",
            "Average accuracy 0.766 - history\n",
            "Average accuracy 0.768 - geography\n",
            "Average accuracy 0.779 - politics\n",
            "Average accuracy 0.748 - psychology\n",
            "Average accuracy 0.813 - culture\n",
            "Average accuracy 0.492 - law\n",
            "Average accuracy 0.525 - STEM\n",
            "Average accuracy 0.564 - humanities\n",
            "Average accuracy 0.736 - social sciences\n",
            "Average accuracy 0.704 - other (business, health, misc.)\n",
            "Average accuracy: 0.625\n"
          ]
        }
      ],
      "source": [
        "# Run the script the evaluate the dataset\n",
        "\n",
        "!python3 /content/llm_model_evaluation/evaluation_hf_testing.py \\\n",
        "    --category_type mmlu \\\n",
        "    --model /content/Mistral-7B-v0.1 \\\n",
        "    --data_dir \"/content/data/\" \\\n",
        "    --save_dir \"/content/result\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7PdppQnaqct_"
      },
      "outputs": [],
      "source": [
        "!python /content/llm_model_evaluation/catogories_result_eval.py \\\n",
        "    --catogory \"mmlu\" \\\n",
        "    --model \"/content/Mistral-7B-v0.1\" \\\n",
        "    --save_dir \"/content/result/results_Mistral-7B-v0.1\""
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}