SFT-llm-experiments-2023-12-21.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e38893cc-dc2d-42b0-b728-f22e954a7e46",
   "metadata": {},
   "source": [
    "# SFT LLM and other alignment methods\n",
    "\n",
    "- SFT to collect clear decision\n",
    "- alignment to help the reasoning\n",
    "- Use gpt4 API to generate prompt and completion\n",
    "\n",
    "```json\n",
    "[\n",
    "  {\n",
    "    \"prompt\": \"she no went to market\",\n",
    "    \"completion\": \"She didn't go to the market.\"\n",
    "  }\n",
    "]\n",
    "```\n",
    "\n",
    "- `stabilityai/stablelm-zephyr-3b`\n",
    "\n",
    "```\n",
    "<|user|>\n",
    "List 3 synonyms for the word \"tiny\"<|endoftext|>\n",
    "<|assistant|>\n",
    "1. Dwarf\n",
    "2. Little\n",
    "3. Petite<|endoftext|>\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3356ecb6-8773-496d-a69e-9430b387c5fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'llama2-fine-tune'...\n",
      "remote: Enumerating objects: 12, done.\u001b[K\n",
      "remote: Counting objects: 100% (12/12), done.\u001b[K\n",
      "remote: Compressing objects: 100% (11/11), done.\u001b[K\n",
      "remote: Total 12 (delta 1), reused 10 (delta 1), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (12/12), 6.36 KiB | 6.36 MiB/s, done.\n",
      "Resolving deltas: 100% (1/1), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone --depth 1 https://github.com/mzbac/llama2-fine-tune.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3a054ff6-d32a-4418-91f6-3804fad7b81f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: datasets in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (2.14.5)\n",
      "Collecting datasets\n",
      "  Downloading datasets-2.15.0-py3-none-any.whl.metadata (20 kB)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (1.23.5)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (12.0.1)\n",
      "Requirement already satisfied: pyarrow-hotfix in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (0.6)\n",
      "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (0.3.7)\n",
      "Requirement already satisfied: pandas in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (2.1.0)\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (2.31.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (4.66.1)\n",
      "Requirement already satisfied: xxhash in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (3.3.0)\n",
      "Requirement already satisfied: multiprocess in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (0.70.15)\n",
      "Collecting fsspec<=2023.10.0,>=2023.1.0 (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets)\n",
      "  Using cached fsspec-2023.10.0-py3-none-any.whl.metadata (6.8 kB)\n",
      "Requirement already satisfied: aiohttp in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (3.8.5)\n",
      "Requirement already satisfied: huggingface-hub>=0.18.0 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (0.19.4)\n",
      "Requirement already satisfied: packaging in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from datasets) (6.0.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from aiohttp->datasets) (23.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from aiohttp->datasets) (2.1.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.3)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: filelock in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from huggingface-hub>=0.18.0->datasets) (3.12.4)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from huggingface-hub>=0.18.0->datasets) (4.9.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2.1.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from pandas->datasets) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from pandas->datasets) (2023.3)\n",
      "Requirement already satisfied: six>=1.5 in /home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
      "Downloading datasets-2.15.0-py3-none-any.whl (521 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hUsing cached fsspec-2023.10.0-py3-none-any.whl (166 kB)\n",
      "Installing collected packages: fsspec, datasets\n",
      "  Attempting uninstall: fsspec\n",
      "    Found existing installation: fsspec 2023.12.2\n",
      "    Uninstalling fsspec-2023.12.2:\n",
      "      Successfully uninstalled fsspec-2023.12.2\n",
      "  Attempting uninstall: datasets\n",
      "    Found existing installation: datasets 2.14.5\n",
      "    Uninstalling datasets-2.14.5:\n",
      "      Successfully uninstalled datasets-2.14.5\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "llmtuner 0.3.2 requires transformers<4.35.0,>=4.31.0, but you have transformers 4.36.1 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed datasets-2.15.0 fsspec-2023.10.0\n"
     ]
    }
   ],
   "source": [
    "!pip install -U datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9bdd1fa7-989c-4875-9f0a-61c8aa8e27b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-02 23:21:07.278831: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-01-02 23:21:07.306499: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: SSE4.1 SSE4.2 AVX AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/trl/trainer/ppo_config.py:141: UserWarning: The `optimize_cuda_cache` arguement will be deprecated soon, please use `optimize_device_cache` instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "from peft import (\n",
    "    AutoPeftModelForCausalLM,\n",
    "    LoraConfig,\n",
    "    get_peft_model,\n",
    "    prepare_model_for_kbit_training,\n",
    ")\n",
    "from transformers import (\n",
    "    AutoModelForCausalLM,\n",
    "    AutoTokenizer,\n",
    "    BitsAndBytesConfig,\n",
    "    TrainingArguments,\n",
    ")\n",
    "from trl import SFTTrainer\n",
    "\n",
    "from utils import find_all_linear_names, print_trainable_parameters\n",
    "\n",
    "output_dir = \"./results\"\n",
    "model_name = \"NousResearch/Llama-2-7b-hf\"\n",
    "\n",
    "# dataset = load_dataset(\"./\", data_files=\"conversations.json\", split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d48c8958-00e3-46fb-bc14-653a1f8fffda",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset_builder, list_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e5aef469-1ee1-4aa6-8374-50bdac4a337e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to data/FashionMNIST/raw/train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26421880/26421880 [00:18<00:00, 1425503.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/FashionMNIST/raw/train-images-idx3-ubyte.gz to data/FashionMNIST/raw\n",
      "\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to data/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 29515/29515 [00:00<00:00, 102390.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/FashionMNIST/raw/train-labels-idx1-ubyte.gz to data/FashionMNIST/raw\n",
      "\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4422102/4422102 [00:04<00:00, 1025982.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz to data/FashionMNIST/raw\n",
      "\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5148/5148 [00:00<00:00, 5991197.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz to data/FashionMNIST/raw\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "from torchvision import datasets\n",
    "from torchvision.transforms import ToTensor\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "training_data = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=ToTensor()\n",
    ")\n",
    "\n",
    "test_data = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=False,\n",
    "    download=True,\n",
    "    transform=ToTensor()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "046e4def-fa1a-4858-84de-01267ad09fda",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_18017/2270578651.py:1: FutureWarning: list_datasets is deprecated and will be removed in the next major version of datasets. Use 'huggingface_hub.list_datasets' instead.\n",
      "  dataset_list = list_datasets()\n"
     ]
    }
   ],
   "source": [
    "dataset_list = list_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3c25c6b4-2ef1-4e77-93ac-a8833e296389",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "88153"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0e735f29-c6b9-47eb-ae27-1b816117b263",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['huggingnft/alpacadabraz',\n",
       " 'tatsu-lab/alpaca',\n",
       " 'crumb/stanford_alpaca_full_prompts',\n",
       " 'nlpcloud/instructions-dataset-adapted-from-stanford-alpaca-for-gpt-j',\n",
       " 'dominguesm/alpaca-data-pt-br',\n",
       " 'Yasbok/Alpaca_arabic_instruct',\n",
       " 'tbboukhari/Alpaca-in-french',\n",
       " 'tbboukhari/Alpaca_french_instruct',\n",
       " 'Bingsu/ko_alpaca_data',\n",
       " 'bertin-project/alpaca-spanish',\n",
       " 'NbAiLab/norwegian-alpaca',\n",
       " 'AndreiMuresanu/alpaca_flan-format',\n",
       " 'blip-solutions/SlovAlpaca',\n",
       " 'IlyaGusev/ru_turbo_alpaca',\n",
       " 'HiTZ/alpaca_mt',\n",
       " 'cahya/alpaca-id',\n",
       " 'dvilasuero/somos-alpaca-es',\n",
       " 'dvilasuero/somos-alpaca-es-rg',\n",
       " 'somosnlp/somos-alpaca-es',\n",
       " 'cariai/somos-alpaca-es',\n",
       " 'dvilasuero/somos-alpaca-es-validations',\n",
       " 'somosnlp/somos-clean-alpaca-es',\n",
       " 'cariai/somos-alpaca-es-2',\n",
       " 'yahma/alpaca-cleaned',\n",
       " 'nataliaElv/somos-clean-alpaca-es-validations',\n",
       " 'pszemraj/fleece2instructions-codealpaca',\n",
       " 'pszemraj/fleece2instructions-inputs-alpaca-cleaned',\n",
       " 'shibing624/alpaca-zh',\n",
       " 'QingyiSi/Alpaca-CoT',\n",
       " 'aia8mason/alpaca',\n",
       " 'alarcon7a/somos-clean-alpaca-es-validations',\n",
       " 'alarcon7a/somos-clean-alpaca-es',\n",
       " 'dvilasuero/somos-clean-alpaca-es-test',\n",
       " 'sahil2801/CodeAlpaca-20k',\n",
       " 'dvilasuero/somos-clean-alpaca-es-herrius',\n",
       " 'dariolopez/somos-clean-alpaca-es-validations',\n",
       " 'vietgpt/alpaca_vi',\n",
       " 'Furuhata-du/alpaca-classify-dataset',\n",
       " 'mserras/alpaca-es-autoclean',\n",
       " 'yuekai/belle_1M_and_alpaca_cleaned',\n",
       " 'tsdocode/vi_alpaca_clean',\n",
       " 'dvilasuero/somos-alpaca-es-intro',\n",
       " 'HuggingFaceH4/CodeAlpaca_20K',\n",
       " 'gbharti/finance-alpaca',\n",
       " 'gbharti/finance-alpaca-csv',\n",
       " 'vietgpt/alpaca_en',\n",
       " 'monicaeme/somos-alpaca-es',\n",
       " 'johnrobinsn/alpaca-cleaned',\n",
       " 'frascuchon/somos-clean-alpaca-es',\n",
       " 'whitefox44/AlpacaGPT3.5Customized',\n",
       " 'argilla/alpaca_data_cleaned',\n",
       " 'joecodecreations/alpaca_data_52k',\n",
       " 'argilla/alpaca_data_cleaned_validations',\n",
       " 'Lbuk/alpaca_data_pl.json',\n",
       " 'royboy0416/ko-alpaca',\n",
       " 'datacrunch/finnish_alpaca',\n",
       " 'TotoB12/alpaca-win',\n",
       " 'argilla/alpaca-gigo-detector',\n",
       " 'LEL-A/translated_german_alpaca',\n",
       " 'jeffwan/BELLE-Alpaca-CN',\n",
       " 'magicgh/alpaca-cleaned-random-50',\n",
       " 'magicgh/alpaca-cleaned-random-25',\n",
       " 'dvilasuero/alpaca-german-validation',\n",
       " 'maga12/somos-clean-alpaca-es-validations',\n",
       " 'elcuazcode/somos-clean-alpaca-es-validations',\n",
       " 'beta3/somos-clean-alpaca-es-validations',\n",
       " 'Aspik101/translated_polish_alpaca',\n",
       " 'emre/stanford-alpaca-cleaned-turkish-translated',\n",
       " 'gbharti/wealth-alpaca_lora',\n",
       " 'mmosiolek/pl_alpaca_data_cleaned',\n",
       " 'SkyHuReal/DrugBank-Alpaca',\n",
       " 'abrazador/somos-alpaca-es-mario',\n",
       " 'hackathon-somos-nlp-2023/alpaca-es-auto-filter',\n",
       " 'Korakoe/Vicuna-Uncleaned-Alpaca-Format',\n",
       " 'mserras/alpaca-es-hackaton-backup',\n",
       " 'cahya/alpaca-id-cleaned',\n",
       " 'medalpaca/medical_meadow_cord19',\n",
       " 'mserras/alpaca-es-hackaton-test',\n",
       " 'medalpaca/medical_meadow_health_advice',\n",
       " 'medalpaca/medical_meadow_mediqa',\n",
       " 'medalpaca/medical_meadow_medqa',\n",
       " 'medalpaca/medical_meadow_pubmed_causal',\n",
       " 'medalpaca/medical_meadow_wikidoc',\n",
       " 'medalpaca/medical_meadow_wikidoc_patient_information',\n",
       " 'medalpaca/medical_meadow_medical_flashcards',\n",
       " 'medalpaca/medical_meadow_mmmlu',\n",
       " 'medalpaca/medical_meadow_usmle_self_assessment',\n",
       " 'Sebastian77/somos-alpaca-es',\n",
       " 'lopezjm96/somos-clean-alpaca-es-validations',\n",
       " 'magicgh/alpaca-cleaned-random-75',\n",
       " 'vicgalle/alpaca-gpt4',\n",
       " 'c-s-ale/alpaca-gpt4-data',\n",
       " 'c-s-ale/alpaca-gpt4-data-zh',\n",
       " 'ehartford/leet10k-alpaca',\n",
       " 'alexl83/AlpacaDataCleaned',\n",
       " 'hackathon-somos-nlp-2023/alpaca-es-agentes',\n",
       " 'japneets/Alpaca_instruction_fine_tune_Punjabi',\n",
       " 'magicgh/alpaca-cleaned',\n",
       " 'nihalbaig/alpaca_bangla',\n",
       " 'Yairama/alpaca_miner_dataset',\n",
       " 'AliEdalat/Persian_ChatBot_dataset_Fine_Tuning_Alpaca_Model',\n",
       " 'mesolitica/chatgpt-alpaca-clean',\n",
       " 'argilla/alpaca_bangla',\n",
       " 'larrylawl/alpaca-cleaned-indon',\n",
       " 'terrycm/alpaca_gpt4_data_es',\n",
       " 'BramVanroy/alpaca-cleaned-dutch',\n",
       " 'BramVanroy/alpaca-cleaned-dutch-baize',\n",
       " 'LEL-A/translated_german_alpaca_validation',\n",
       " 'HuggingFaceH4/testing_alpaca_small',\n",
       " 'HuggingFaceH4/testing_codealpaca_small',\n",
       " 'c-s-ale/dolly-15k-instruction-alpaca-format',\n",
       " 'kuleshov/alpaca-data',\n",
       " 'ChobPT/gradio_docs_alpaca',\n",
       " 'efederici/alpaca-gpt4-it',\n",
       " 'couchpotato888/dolly-and-alpaca-lora-data-format',\n",
       " 'mserras/alpaca-es-hackaton-validated',\n",
       " 'Honkware/oasst1-alpaca-json',\n",
       " 'japneets/Alpaca_instruction_fine_tune_Punjabi_small',\n",
       " 'nihalbaig/alpaca-bangla',\n",
       " 'couchpotato888/alpacagpt4dolly',\n",
       " 'iamketan25/alpaca-instructions-dataset',\n",
       " 'sam-mosaic/vicuna_alpaca_hc3_chatml',\n",
       " 'leocnj/alpaca_dschat',\n",
       " 'nihalbaig/alpaca-bangla_validation',\n",
       " 'nRuaif/Vietnamese_x_Alpaca',\n",
       " 'theblackcat102/audio-alpaca',\n",
       " 'thisisanshgupta/CodeAlpaca',\n",
       " 'thisisanshgupta/CodeAlpacaSmall',\n",
       " 'Miranda2023/alpaca13-belle-eval',\n",
       " 'TenzinGayche/AlpacaCleanedGPT4',\n",
       " 'DevAibest/alpaca_json_data',\n",
       " 'DevAibest/alpaca_prompt_data',\n",
       " 'lucasmccabe-lmi/codex_math_qa_alpaca_style',\n",
       " 'lucasmccabe-lmi/openai_humaneval_alpaca_style',\n",
       " 'lucasmccabe-lmi/instruct_to_code_alpaca_style',\n",
       " 'ehartford/WizardLM_alpaca_evol_instruct_70k_unfiltered',\n",
       " 'Shiroisha/alpaca-test',\n",
       " 'pvduy/sharegpt_alpaca_oa_gpt4all_vicuna_format',\n",
       " 'somosnlp/somos_alpaca_validation_disagreement',\n",
       " 'somosnlp/somos_alpaca_validation_agreement',\n",
       " 'pvduy/sharegpt_alpaca_oa_vicuna_format',\n",
       " 'pvduy/oa_vicuna_dolly_grademath_alpaca_leetcode',\n",
       " 'shawmoon/ekattor_alpaca2',\n",
       " 'OdiaGenAI/Odia_Alpaca_instructions_52k',\n",
       " 'LLMs/Alpaca-ShareGPT',\n",
       " 'DevAibest/alpaca-geotherm-data',\n",
       " 's3nh/alpaca-dolly-instruction-only-polish',\n",
       " 'PocketDoc/Alpaca_Evol_Instruct_Cleaned',\n",
       " 'spdenisov/udt_alpaca',\n",
       " 'winglian/alpaca-gpt4-split',\n",
       " 'innermost47/alpaca-fr',\n",
       " 'jeremyc/Alpaca-Lora-GPT4-Swedish',\n",
       " 'No-22/chinese-alpaca-7b-quantized',\n",
       " 'Lajonbot/alpaca-dolly-chrisociepa-instruction-only-polish',\n",
       " 'winglian/alpaca-reflect',\n",
       " 'Thaweewat/alpaca-cleaned-52k-th',\n",
       " 'Thaweewat/alpaca-finance-43k-th',\n",
       " 'ewof/alpaca-instruct-unfiltered',\n",
       " 'silk-road/Vanilla-chinese-alpaca-luotuo',\n",
       " 'ewof/code-alpaca-instruct-unfiltered',\n",
       " 'lucasmccabe-lmi/sql-create-context_alpaca_style',\n",
       " 'theoer/alpaca',\n",
       " 'zh-tw-llm-dv/zh-tw-pythia-a12k-alpaca',\n",
       " 'tatsu-lab/alpaca_farm',\n",
       " 'lucasmccabe-lmi/FLAN_CoT_alpaca_style',\n",
       " 'sam-liu-lmi/databricks-dolly-15k-alpaca-style',\n",
       " 'Laurie/alpaca_chinese_dataset',\n",
       " 'nikes64/ualpaca-gpt4',\n",
       " 'shi3z/alpaca_cleaned_ja_json',\n",
       " 'Koyd111/alpaca-jp-eng',\n",
       " 'ReehaKhan/AlpacaTraining',\n",
       " 'ReehaKhan/AlpacaTesting',\n",
       " 'PocketDoc/QED-Alpaca',\n",
       " 'fujiki/japanese_alpaca_data',\n",
       " 'Norquinal/WizardLM_alpaca_claude_evol_instruct_70k',\n",
       " 'cgulse/alpaca-cleaned-tr',\n",
       " 'PocketDoc/RUCAIBox-Story-Generation-Alpaca',\n",
       " 'lucasmccabe-lmi/CodeAlpaca-20k',\n",
       " 'Fredithefish/ShareGPT-unfiltered-alpaca-lora-format',\n",
       " 'Dampish/sharegpt-alpaca-unfiltered-94k',\n",
       " 'IlyaGusev/ru_turbo_alpaca_evol_instruct',\n",
       " 'JoostVanGils/alpaca_data_translated_nl.json',\n",
       " 'silk-road/alpaca-data-gpt4-chinese',\n",
       " 'next-social/zhihu_beautiful_super_alpaca',\n",
       " 'Rickxz06/alpaca-gpt4-cleaned-20mb',\n",
       " 'beomi/KoAlpaca-v1.1a',\n",
       " 'tatsu-lab/alpaca_eval',\n",
       " 'Bk9x/alpaca_prompt',\n",
       " 'TigerResearch/tigerbot-alpaca-en-50k',\n",
       " 'TigerResearch/tigerbot-alpaca-zh-0.5m',\n",
       " 'Abzu/CodeAlpacaPython',\n",
       " 'orderofmagnitude/alpaca_dataset.json',\n",
       " 'wakenedo/wakenedo-webui-alpaca',\n",
       " 'squaredev/translated_alpaca_tasks_gr',\n",
       " 'Amirkid/stanford_alpaca',\n",
       " 'AlekseyKorshuk/wizardlm-alpaca-evol-instruct-chatml',\n",
       " 'ttbui/html_alpaca',\n",
       " 'hamishivi/alpaca-farm-davinci-003-2048-token',\n",
       " 'eastwind/semeval-2016-absa-reviews-english-translated-stanford-alpaca',\n",
       " 'Amirkid/stanford_alpaca_new',\n",
       " 'ttbui/alpaca_data_with_html_output',\n",
       " 'ttbui/alpaca_webgen_html',\n",
       " 'linpang/alpaca_html',\n",
       " 'binhgiangnguyendanh/reddit_casual_conversation_for_alpaca_lora',\n",
       " 'practical-dreamer/RPGPT_PublicDomain-alpaca',\n",
       " 'scu-kdde/alpaca-law',\n",
       " 'HachiML/databricks-dolly-15k-ja-alpaca-format',\n",
       " 'SLTP/HLT-AA-C21-Alpaca',\n",
       " 'jondurbin/airoboros-gpt4-1.2-alpaca-eval',\n",
       " 'd0rj/alpaca-cleaned-ru',\n",
       " 'vivienmeally/alpaca-test-de',\n",
       " 'Guilherme34/alpaca_chat_data',\n",
       " 'causal-lm/cot_alpaca',\n",
       " 'causal-lm/cot_alpaca_gpt4',\n",
       " 'pankajmathur/alpaca_orca',\n",
       " 'causal-lm/code_alpaca',\n",
       " 'FreedomIntelligence/alpaca-gpt4-arabic',\n",
       " 'FreedomIntelligence/alpaca-gpt4-chinese',\n",
       " 'FreedomIntelligence/alpaca-gpt4-deutsch',\n",
       " 'FreedomIntelligence/alpaca-gpt4-french',\n",
       " 'FreedomIntelligence/alpaca-gpt4-hindi',\n",
       " 'FreedomIntelligence/alpaca-gpt4-indonesian',\n",
       " 'FreedomIntelligence/alpaca-gpt4-italian',\n",
       " 'FreedomIntelligence/alpaca-gpt4-japanese',\n",
       " 'FreedomIntelligence/alpaca-gpt4-korean',\n",
       " 'FreedomIntelligence/alpaca-gpt4-portuguese',\n",
       " 'FreedomIntelligence/alpaca-gpt4-spanish',\n",
       " 'psymon/namuwiki_alpaca_dataset',\n",
       " 'BramVanroy/alpaca-dolly-dutch',\n",
       " 'reciprocate/alpaca-eval',\n",
       " 'boapps/alpaca-hu',\n",
       " 'HoangHa/alpaca_vi',\n",
       " 'kz919/alpaca',\n",
       " 'nRuaif/wizard_alpaca_dolly_orca',\n",
       " 'ssbuild/alpaca_chinese_dataset',\n",
       " 'sieu-n/alpaca_eval_multilingual',\n",
       " 'ssbuild/alpaca_sharegpt',\n",
       " 'ssbuild/alpaca_firefly',\n",
       " 'ssbuild/alpaca_belle',\n",
       " 'Abdelkareem/rwkv_Alpaca_arabic_instruct_version',\n",
       " 'ssbuild/alpaca_medical',\n",
       " 'ssbuild/alpaca_gpt4',\n",
       " 'ssbuild/alpaca_dolly',\n",
       " 'ssbuild/alpaca_finance_en',\n",
       " 'ssbuild/alpaca_convai2',\n",
       " 'ssbuild/alpaca_gpt4all',\n",
       " 'ssbuild/alpaca_rlhf',\n",
       " 'ssbuild/alpaca_tabular',\n",
       " 'ssbuild/alpaca_auto_cot',\n",
       " 'ssbuild/alpaca_coig',\n",
       " 'ssbuild/alpaca_csl',\n",
       " 'ssbuild/alpaca_baize',\n",
       " 'ssbuild/alpaca_gpteacher',\n",
       " 'ssbuild/alpaca_guanaco',\n",
       " 'ssbuild/alpaca_hc3',\n",
       " 'ssbuild/alpaca_instinwild',\n",
       " 'ssbuild/alpaca_oig',\n",
       " 'ssbuild/alpaca_pclue',\n",
       " 'ssbuild/alpaca_prosocial-dialog',\n",
       " 'ssbuild/alpaca_webgpt',\n",
       " 'ssbuild/alpaca_thoughtsource',\n",
       " 'ssbuild/alpaca_xp3',\n",
       " 'ssbuild/alpaca_flan-muffin',\n",
       " 'Vezora/Mini_Orca_Uncencored_Alpaca',\n",
       " 'Vezora/Mini_Orca_Code_Uncencored_alpaca_Format',\n",
       " 'UCL-DARK/alpaca-farm-id-test',\n",
       " 'DAMO-NLP-MT/multialpaca',\n",
       " 'freQuensy23/ru-alpaca-cleaned',\n",
       " 'freQuensy23/turbo-alpaca-cleaned',\n",
       " 'Andyrasika/alpaca-bitcoin-sentiment-dataset',\n",
       " 'alaeddine-13/alpaca_rated',\n",
       " 'datatab/alpaca-cleaned-serbian-10000',\n",
       " 'datatab/alpaca-cleaned-serbian-30000',\n",
       " 'datatab/alpaca-cleaned-serbian-full',\n",
       " 'prognosis/alpaca_cardaio_qa',\n",
       " 'khalidalt/alpaca',\n",
       " 'gpt4life/alpaca_claud_filtered',\n",
       " 'TokenBender/code_instructions_122k_alpaca_style',\n",
       " 'xzuyn/open-instruct-uncensored-alpaca',\n",
       " 'baylitoo/alpacaa',\n",
       " 'agie-ai/alpaca',\n",
       " 'NebulaByte/alpaca-gpt4-hindi-hinglish',\n",
       " 'mhenrichsen/alpaca_2k_test',\n",
       " 'ashercn97/awesome-prompts-alpaca',\n",
       " 'theblackcat102/evol-codealpaca-v1',\n",
       " 'lamini/alpaca',\n",
       " 'iamtarun/code_instructions_120k_alpaca',\n",
       " 'kaleinaNyan/ehartford__WizardLM_alpaca_evol_instruct_70k_unfiltered',\n",
       " 'iamtarun/python_code_instructions_18k_alpaca',\n",
       " 'zhengxuanzenwu/alpaca-no-context',\n",
       " 'Nekochu/novel17_train_alpaca_format',\n",
       " 'Vezora/news_seniment_gpt_alpacaformat',\n",
       " 'EnigmaOfTheWorld/wikisql-alpaca',\n",
       " 'sankethgadadinni/alpaca-cleaned',\n",
       " 'HydraLM/CodeAlpaca-20k_standardized',\n",
       " 'shyam-incedoinc/alpaca-output_col-200rows',\n",
       " 'HydraLM/GPTeacher-General-Instruct_alpaca',\n",
       " 'HydraLM/airoboros-gpt4-1.4_alpaca',\n",
       " 'HydraLM/GPT4-LLM-Cleaned_alpaca',\n",
       " 'HydraLM/WizardLM_evol_instruct_V2_196k_alpaca',\n",
       " 'HydraLM/CodeAlpaca-20k_list_dict',\n",
       " 'arbml/alpaca_arabic',\n",
       " 'sankethgadadinni/alpaca',\n",
       " 'sankethgadadinni/alpaca-data',\n",
       " 'iamtarun/code_contest_python3_alpaca',\n",
       " 'HydraLM/CodeAlpaca-20k_alpaca',\n",
       " 'HydraLM/unnatural-instructions_alpaca',\n",
       " 'HydraLM/biology_dataset_alpaca',\n",
       " 'HydraLM/chemistry_dataset_alpaca',\n",
       " 'HydraLM/math_dataset_alpaca',\n",
       " 'HydraLM/physics_dataset_alpaca',\n",
       " 'HydraLM/GPTeacher_codegen_alpaca',\n",
       " 'HydraLM/GPTeacher_roleplay_alpaca',\n",
       " 'HydraLM/GPTeacher_toolformer_alpaca',\n",
       " 'flpelerin/openorca-alpaca-15k',\n",
       " 'msplkhh/KoAlpaca-v1.1a_modified',\n",
       " 'Yijia-Xiao/alpaca',\n",
       " 'lnutiu/alpaca-top',\n",
       " 'HydraLM/partitioned_v2_alpaca_1500words',\n",
       " 'iamshnoo/alpaca-cleaned-hindi',\n",
       " 'iamshnoo/alpaca-cleaned-chinese',\n",
       " 'iamshnoo/alpaca-cleaned-persian',\n",
       " 'iamshnoo/alpaca-cleaned-swahili',\n",
       " 'flpelerin/openorca-alpaca-50k',\n",
       " 'xzuyn/tulu-uncensored-alpaca',\n",
       " 'xzuyn/manythings-translations-alpaca',\n",
       " 'plncmm/spanish-alpaca',\n",
       " 'Vezora/Gorilla_Alpaca_Format',\n",
       " 'thanhnew2001/alpaca_vn',\n",
       " 'Laurie/HC3-Chinese-AlpacaFormat',\n",
       " 'arbml/alpaca_arabic_v2',\n",
       " 'arbml/alpaca_arabic_v3',\n",
       " 'xzuyn/tv-alpaca',\n",
       " 'xzuyn/tv-alpaca-open-instruct-uncensored-blend',\n",
       " 'Vazbeek/alpaca-cs',\n",
       " 'rombodawg/code_instruct_alpaca_vicuna_wizardlm_56k_backup',\n",
       " 'AnnasBlackHat/alpaca-indonesia-for-llama',\n",
       " 'AnnasBlackHat/alpaca-indonesia-llama',\n",
       " 'pourmand1376/alpaca-fa-instruction',\n",
       " 'pourmand1376/alpaca-fa-multi',\n",
       " 'az1/alpaca-test',\n",
       " 'arazd/tulu_stanford_alpaca',\n",
       " 'arazd/tulu_code_alpaca',\n",
       " 'arazd/tulu_gpt4_alpaca',\n",
       " 'rombodawg/alpaca_840k',\n",
       " 'caiobd/alpaca-data-pt-br-autotrain',\n",
       " 'sinarashidi/alpaca-persian',\n",
       " 'sinarashidi/alpaca-persian-llama2',\n",
       " 'hari560/alpaca_gpt4',\n",
       " 'voidful/spoken-alpaca-gpt4',\n",
       " 'poonehmousavi/alpaca_instruction',\n",
       " 'klima7/alpaca-polish-tales-dataset',\n",
       " 'aboonaji/alpaca_micro_demo',\n",
       " 'dball/GerAlpacaDataCleaned_transformer.wmt19.en-de',\n",
       " 'Braddy/alpaca_customised',\n",
       " 'GSQA/speech-alpaca-gpt4-unit',\n",
       " 'Vazbeek/alpaca-cs-subset-1k',\n",
       " 'flpelerin/ChatAlpaca-10k',\n",
       " 'HydraLM/alpaca_data_cleaned_standardized',\n",
       " 'HydraLM/code_alpaca_standardized',\n",
       " 'disham993/alpaca-train-validation-test-split',\n",
       " 'fengtc/alpaca_data_chinese_51k',\n",
       " 'wesley7137/dataset_alpaca_counseling',\n",
       " 'boapps/alpaca-filtered-hu',\n",
       " 'HachiML/hh-rlhf-49k-ja-alpaca-format',\n",
       " 'aongwachi/alpaca-cleaned-1k-th',\n",
       " 'wesley7137/cnslgjsonalpaca',\n",
       " 'Vezora/Puffin-Alpaca',\n",
       " 'Vezora/Wizard_Math_Alpaca',\n",
       " 'Xilabs/PIPPA-alpaca',\n",
       " 'arbml/alpaca_arabic_test',\n",
       " 'flpelerin/PIPPA_-_Alpaca',\n",
       " 'TFLai/Turkish-Alpaca',\n",
       " 'botp/silk-road_alpaca-data-gpt4-chinese',\n",
       " 'open-llm-leaderboard/details_jxhong__CAlign-alpaca-7b',\n",
       " 'open-llm-leaderboard/details_shibing624__chinese-alpaca-plus-13b-hf',\n",
       " 'open-llm-leaderboard/details_shibing624__chinese-alpaca-plus-7b-hf',\n",
       " 'open-llm-leaderboard/details_layoric__llama-2-13b-code-alpaca',\n",
       " 'open-llm-leaderboard/details_vicgalle__alpaca-7b',\n",
       " 'open-llm-leaderboard/details_vicgalle__gpt2-alpaca',\n",
       " 'open-llm-leaderboard/details_vicgalle__gpt2-alpaca-gpt4',\n",
       " 'open-llm-leaderboard/details_HiTZ__alpaca-lora-65b-en-pt-es-ca',\n",
       " 'open-llm-leaderboard/details_Aeala__GPT4-x-AlpacaDente-30b',\n",
       " 'open-llm-leaderboard/details_Aeala__GPT4-x-AlpacaDente2-30b',\n",
       " 'open-llm-leaderboard/details_GeorgiaTechResearchInstitute__galpaca-30b',\n",
       " 'open-llm-leaderboard/details_jordiclive__gpt4all-alpaca-oa-codealpaca-lora-13b',\n",
       " 'open-llm-leaderboard/details_Rachneet__gpt2-xl-alpaca',\n",
       " 'open-llm-leaderboard/details_LLMs__AlpacaGPT4-7B-elina',\n",
       " 'open-llm-leaderboard/details_beomi__KoAlpaca-Polyglot-5.8B',\n",
       " 'open-llm-leaderboard/details_TehVenom__Pygmalion_AlpacaLora-7b',\n",
       " 'open-llm-leaderboard/details_TFLai__llama-13b-4bit-alpaca',\n",
       " 'open-llm-leaderboard/details_TFLai__llama-7b-4bit-alpaca',\n",
       " 'open-llm-leaderboard/details_TFLai__gpt-neox-20b-4bit-alpaca',\n",
       " 'open-llm-leaderboard/details_TFLai__llama-2-13b-4bit-alpaca-gpt4',\n",
       " 'ideepankarsharma2003/warewe_alpaca_style_organic_keywords',\n",
       " 'open-llm-leaderboard/details_chansung__gpt4-alpaca-lora-13b-decapoda-1024',\n",
       " 'open-llm-leaderboard/details_WeOpenML__Alpaca-7B-v1',\n",
       " 'open-llm-leaderboard/details_WeOpenML__PandaLM-Alpaca-7B-v1',\n",
       " 'open-llm-leaderboard/details_NbAiLab__nb-gpt-j-6B-alpaca',\n",
       " 'open-llm-leaderboard/details_medalpaca__medalpaca-7b',\n",
       " 'open-llm-leaderboard/details_TheBloke__Chinese-Alpaca-33B-SuperHOT-8K-fp16',\n",
       " 'open-llm-leaderboard/details_TheBloke__VicUnlocked-alpaca-65B-QLoRA-fp16',\n",
       " 'open-llm-leaderboard/details_TheBloke__gpt4-alpaca-lora-13B-HF',\n",
       " 'open-llm-leaderboard/details_TheBloke__gpt4-alpaca-lora_mlp-65B-HF',\n",
       " 'open-llm-leaderboard/details_TheBloke__alpaca-lora-65B-HF',\n",
       " 'open-llm-leaderboard/details_ziqingyang__chinese-alpaca-2-7b',\n",
       " 'open-llm-leaderboard/details_lxe__Cerebras-GPT-2.7B-Alpaca-SP',\n",
       " 'open-llm-leaderboard/details_dsvv-cair__alpaca-cleaned-llama-30b-bf16',\n",
       " 'ramadita/alpaca-id-gptq',\n",
       " 'open-llm-leaderboard/details_TFLai__gpt-neo-1.3B-4bit-alpaca',\n",
       " 'open-llm-leaderboard/details_minlik__chinese-alpaca-33b-merged',\n",
       " 'open-llm-leaderboard/details_bertin-project__bertin-gpt-j-6B-alpaca',\n",
       " 'kranthigv/code_instructions_122k_alpaca_style_standardized',\n",
       " 'HydraLM/code_instructions_122k_alpaca_style_standardized',\n",
       " 'Ichsan2895/alpaca-gpt4-indonesian',\n",
       " 'arazd/llama_features_alpaca',\n",
       " 'jasonkstevens/PIPPA-Alpaca',\n",
       " 'LeoLM/AlpacaEval_de',\n",
       " 'dim/ru_turbo_alpaca_evol_instruct_3k',\n",
       " 'mir-hossain/conll2003_named_entities_alpaca_format',\n",
       " 'wesley7137/neuroalpaca_autotrain',\n",
       " 'FinchResearch/OpenPlatypus-Alpaca',\n",
       " 'botp/alpaca-taiwan-dataset',\n",
       " 'kranthigv/alpaca-gpt4',\n",
       " 'HydraLM/alpaca-gpt4-standardized',\n",
       " 'prateeky2806/bge_base_features_alpaca',\n",
       " 'RuterNorway/Fleurs-Alpaca-EN-NO',\n",
       " 'laampt/alpaca-train',\n",
       " 'xzuyn/lima-alpaca',\n",
       " 'Vezora/Dolphin1m_gpt4_Alpaca_format',\n",
       " 'xzuyn/lima-multiturn-alpaca',\n",
       " 'Toflamus/alpaca_data_raw',\n",
       " 'Toflamus/alpaca_data_split',\n",
       " 'NobodyExistsOnTheInternet/LIMAalpaca',\n",
       " 'mangostin2010/KakaoChatData-alpaca',\n",
       " 'open-llm-leaderboard/details_TFLai__pythia-2.8b-4bit-alpaca',\n",
       " 'nadiamaqbool81/java_code_instructions_1k_alpaca',\n",
       " 'liaaron1/bibile_trivia_alpaca',\n",
       " 'prognosis/medalpaca_dataset_v1',\n",
       " 'mariogiordano/Alpaca_dataset_mapped',\n",
       " 'nayohan/KoAlpaca-v1.1a_ppl',\n",
       " 'vikp/evol_codealpaca_filtered_87k',\n",
       " 'open-llm-leaderboard/details_CHIH-HUNG__llama-2-13b-alpaca-test',\n",
       " 'open-llm-leaderboard/details_DevaMalla__llama7b_alpaca_1gpu_bf16',\n",
       " 'sudiptabasak/alpaca-gpt4-csv',\n",
       " 'HydraLM/python-code-instructions-18k-alpaca-standardized',\n",
       " 'sudiptabasak/alpaca-vectors',\n",
       " 'dim/dolphin_flan1m_alpaca_uncensored_3k',\n",
       " 'YoungPhlo/juyongjiang-codeup_master_data_new_codealpaca_standardized',\n",
       " 'nadiamaqbool81/java_code_instructions_1.178k_alpaca',\n",
       " 'iamshnoo/alpaca-cleaned-bengali',\n",
       " 'iamshnoo/alpaca-cleaned-greek',\n",
       " 'mariogiordano/alpaca-dataset',\n",
       " 'nguyenthanhdo/dummy_alpaca',\n",
       " 'sarahlintang/Alpaca_indo_instruct',\n",
       " 'VMXVMX/alpacatest',\n",
       " 'HusainMehdi/alpaca-shortened',\n",
       " 'HusainMehdi/alpaca-short',\n",
       " 'HusainMehdi/alpaca-reduced',\n",
       " 'KingAlpaca/your-dataset-name',\n",
       " 'mesolitica/google-translate-chatalpaca',\n",
       " 'khalidalt/evol-codealpaca-v1-standardized',\n",
       " 'khalidalt/python_code_instructions_18k_alpaca-standardized',\n",
       " 'jtatman/wizard_alpaca_dolly_orca_uncensored_masked',\n",
       " 'dribar/Dan_Alpaca1',\n",
       " 'victor-buhl/COLREGS_test_bank_ALPACA',\n",
       " 'victor-buhl/COLREGS_ALPACA_SHORT',\n",
       " 'HydraLM/med_alpaca_standardized',\n",
       " 'TokenBender/roleplay_alpaca',\n",
       " 'Taegyuu/KoAlpaca-v1.1a',\n",
       " 'hynky/czech-justice-summ-alpaca-long',\n",
       " 'hynky/czech-justice-summ-alpaca-short',\n",
       " 'voidful/alpaca-gpt4',\n",
       " 'prognosis/medquad-alpaca',\n",
       " 'maximegmd/medqa_alpaca_format',\n",
       " 'maximegmd/medmcqa_alpaca_format',\n",
       " 'Taegyuu/KoAlpaca_hira_v1.1a',\n",
       " 'yvelos/python_code_instructions_18k_alpaca',\n",
       " 'Gryphe/CoEdit-Alpaca',\n",
       " 'totally-not-an-llm/alpacamix',\n",
       " 'atmallen/truth-tagged-oasst-alpaca',\n",
       " 'gangkongkong/koalpaca-llama2',\n",
       " 'nguyenphuthien/vi-alpaca-data',\n",
       " 'maximegmd/MedText-alpaca',\n",
       " 'DialogueCharacter/chinese_alpaca_unfiltered',\n",
       " 'open-llm-leaderboard/details_chavinlo__alpaca-native',\n",
       " 'open-llm-leaderboard/details_TheBloke__gpt4-alpaca-lora-30b-HF',\n",
       " 'chenqile09/alpaca-2-13B-chinese-couplet-val-4k-predictions',\n",
       " 'chenqile09/alpaca-2-7B-chinese-couplet-val-4k-predictions',\n",
       " 'ZhongshengWang/Alpaca-pubmed-summarization',\n",
       " 'ZhongshengWang/Alpaca-cnn-dailymail',\n",
       " 'ZhongshengWang/alpaca-booksum',\n",
       " 'FreedomIntelligence/Arabic-AlpacaEval',\n",
       " 'sam2ai/hindi_alpaca_dolly_67k',\n",
       " 'open-llm-leaderboard/details_chavinlo__gpt4-x-alpaca',\n",
       " 'xzuyn/beavertails-alpaca',\n",
       " 'rovi27/somos-clean-alpaca-es',\n",
       " 'DavidLanz/alpaca-tw-input-output-52k',\n",
       " 'dim/ru_turbo_alpaca_evol_instruct',\n",
       " 'DavidLanz/alpaca-gpt4-tw-input-output-48k',\n",
       " 'erhwenkuo/alpaca-data-gpt4-chinese-zhtw',\n",
       " 'mabryCodes/tiny-cot-alpaca',\n",
       " 'nitinbhayana/alpaca-review-input',\n",
       " 'hcho22/code_instructions_120k_alpaca_filtered',\n",
       " 'Akash092003/ABSA-alpaca-SemEval2014Task4',\n",
       " 'Rodr16020/code_instructions_7_5k_alpaca_spanish',\n",
       " 'Photolens/alpaca-cleaned',\n",
       " 'Photolens/alpaca-cleaned-airoboros-2.1-no-code-oasst1-en-merged',\n",
       " 'sam2ai/bengali_alpaca_dolly_67k',\n",
       " 'Lumos23/alpaca_farm',\n",
       " 'ZhongshengWang/PARARULE-Plus-Alpaca',\n",
       " 'sam2ai/telgu_alpaca_dolly_67k',\n",
       " 'hyejungg/KG-KoAlpaca',\n",
       " 'ASIDS/alpaca-cleaned-ru',\n",
       " 'NobodyExistsOnTheInternet/AlpacaToxicQA',\n",
       " 'open-llm-leaderboard/details_beomi__KoAlpaca-KoRWKV-6B',\n",
       " 'AntoineBlanot/alpaca-llama2-chat',\n",
       " 'freddyaboulton/alpaca-csv',\n",
       " 'sam2ai/odia_alpaca_dolly_67k',\n",
       " 'layoric/tiny-codes-alpaca',\n",
       " 'layoric/tiny-codes-alpaca-csharp',\n",
       " 'Ronal999/finance-alpaca-demo',\n",
       " 'open-llm-leaderboard/details_Aeala__Alpaca-elina-65b',\n",
       " 'Yukang/LongAlpaca-12k',\n",
       " 'open-llm-leaderboard/details_Yukang__LongAlpaca-13B',\n",
       " 'jamesargent/alpaca-small',\n",
       " 'ostapeno/stanford_alpaca',\n",
       " 'ostapeno/code_alpaca',\n",
       " 'ostapeno/tulu_v2_gpt4_alpaca_subset',\n",
       " 'ostapeno/tulu_v2_code_alpaca_subset',\n",
       " 'unoooo/ko-alpaca',\n",
       " 'infCapital/finance-alpaca_vi',\n",
       " 'open-llm-leaderboard/details_Yukang__LongAlpaca-7B',\n",
       " 'Abira1/finance-alpaca-v2',\n",
       " 'open-llm-leaderboard/details_TFLai__bloom-560m-4bit-alpaca',\n",
       " 'PiyushLavaniya/Small_Alpaca_Instruct',\n",
       " 'casperhansen/longalpaca_1k_unlimited_test',\n",
       " 'casperhansen/longalpaca_1k_test',\n",
       " 'abhinand/tamil-alpaca',\n",
       " 'berkouille/Baize_Alpaca_Golf',\n",
       " 'open-llm-leaderboard/details_ziqingyang__chinese-alpaca-2-13b',\n",
       " 'PHNG/cai-alpaca-en',\n",
       " 'alexrs/alpaca-cleaned-10-clusters',\n",
       " 'alexrs/alpaca-cleaned-5-clusters',\n",
       " 'alexrs/alpaca-cleaned-15-clusters',\n",
       " 'alexrs/alpaca-cleaned-30-clusters',\n",
       " 'ksaw008/finance_alpaca',\n",
       " 'qureshiu/embeddings_alpaca_aci',\n",
       " 'onyou611/ko-alpaca-nms',\n",
       " 'open-llm-leaderboard/details_Aeala__VicUnlocked-alpaca-30b',\n",
       " 'OdiaGenAI/hindi_alpaca_dolly_67k_formatted',\n",
       " 'gayathrimanoj/dataset_shell_alpaca',\n",
       " 'antolin/codealpaca-filtered',\n",
       " 'Lohse/alpaca-dummy',\n",
       " 'nguyenthanhdo/alpaca_en-vi',\n",
       " 'irlab-udc/alpaca_data_galician',\n",
       " 'umd-zhou-lab/recycled_alpaca_v1',\n",
       " 'ZhongshengWang/Chinese-WikiMatrix-Alpaca',\n",
       " 'OdiaGenAI/alpaca_dolly_eval_100',\n",
       " 'abhinand/tamil-alpaca-orca',\n",
       " 'bbokyeong/koalpaca-final',\n",
       " 'open-llm-leaderboard/details_TFLai__bloomz-1b7-4bit-alpaca',\n",
       " 'umd-zhou-lab/claude2_alpaca',\n",
       " 'AdapterOcean/physics_dataset_standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/physics_dataset_standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/physics_dataset_standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/physics_dataset_standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/physics_dataset_standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/biology_dataset_standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/biology_dataset_standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/biology_dataset_standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/biology_dataset_standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/biology_dataset_standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/math_dataset_standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/math_dataset_standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/math_dataset_standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/math_dataset_standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/math_dataset_standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/chemistry_dataset_standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/chemistry_dataset_standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/chemistry_dataset_standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/chemistry_dataset_standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/chemistry_dataset_standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/gorilla_16k_standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/gorilla_16k_standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/gorilla_16k_standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/gorilla_16k_standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/gorilla_16k_standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_5_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_6_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_7_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_8_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_9_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_10_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_11_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_12_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_13_alpaca',\n",
       " 'AdapterOcean/Open_Platypus_standardized_cluster_14_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_5_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_6_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_7_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_8_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_9_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_10_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_11_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_12_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_13_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_14_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_15_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_16_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_17_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_18_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_19_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_20_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_21_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_22_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_23_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_5_alpaca',\n",
       " 'AdapterOcean/python3-standardized_cluster_24_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_6_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_7_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_8_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_9_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_10_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_11_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_12_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_13_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_14_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_15_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_16_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_17_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_18_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_19_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_20_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_21_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_22_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_23_alpaca',\n",
       " 'AdapterOcean/data-standardized_cluster_24_alpaca',\n",
       " 'datajuicer/alpaca-cot-zh-refined-by-data-juicer',\n",
       " 'datajuicer/alpaca-cot-en-refined-by-data-juicer',\n",
       " 'AdapterOcean/datasci-standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/datasci-standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/augmentatio-standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/augmentatio-standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/augmentatio-standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/augmentatio-standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/augmentatio-standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/augmentatio-standardized_cluster_5_alpaca',\n",
       " 'AdapterOcean/augmentatio-standardized_cluster_6_alpaca',\n",
       " 'AdapterOcean/augmentatio-standardized_cluster_7_alpaca',\n",
       " 'AdapterOcean/augmentatio-standardized_cluster_8_alpaca',\n",
       " 'AdapterOcean/augmentatio-standardized_cluster_9_alpaca',\n",
       " 'AdapterOcean/gptindex-standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/langchain-standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_unified',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_embedded',\n",
       " 'AdapterOcean/med_alpaca_standardized_unified',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_0_std',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_0',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_1_std',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_1',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_2_std',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_2',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_3_std',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_3',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_4_std',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_4',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_5_std',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_5_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_5',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_6_std',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_6_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_6',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_7_std',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_7_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_7',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_8_std',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_8_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_8',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_9_std',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_9_alpaca',\n",
       " 'AdapterOcean/python-code-instructions-18k-alpaca-standardized_cluster_9',\n",
       " 'AdapterOcean/GPTeacher_roleplay_standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/GPTeacher_roleplay_standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/GPTeacher_roleplay_standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_embedded',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_5_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_6_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_7_alpaca',\n",
       " 'AdapterOcean/dollyaug-standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_8_alpaca',\n",
       " 'AdapterOcean/dollyaug-standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/dollyaug-standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/dollyaug-standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/dollyaug-standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_9_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_10_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_11_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_12_alpaca',\n",
       " 'Lucianopacheco/alpaca_1col_1000',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_13_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_14_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_15_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_16_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_17_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_18_alpaca',\n",
       " 'AdapterOcean/code_instructions_standardized_cluster_19_alpaca',\n",
       " 'AdapterOcean/pythonbook-standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_0_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_0_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_0',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_1_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_1_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_1',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_2_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_2_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_2',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_3_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_3_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_3',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_4_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_4_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_4',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_5_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_5_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_5',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_6_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_6_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_6',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_7_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_7_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_7',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_8_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_8_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_8',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_9_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_9_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_9',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_10_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_10_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_10',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_11_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_11_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_11',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_12_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_12_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_12',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_13_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_13_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_13',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_14_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_14_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_14',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_15_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_15_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_15',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_16_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_16_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_16',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_17_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_17_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_17',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_18_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_18_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_18',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_19_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_19_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_19',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_20_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_20_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_20',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_21_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_21_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_21',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_22_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_22_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_22',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_23_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_23_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_23',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_24_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_24_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_24',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_25_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_25_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_25',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_26_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_26_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_26',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_27_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_27_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_27',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_28_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_28_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_28',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_29_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_29_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_29',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_30_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_30_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_30',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_31_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_31_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_31',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_32_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_32_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_32',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_33_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_33_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_33',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_34_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_34_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_34',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_35_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_35_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_35',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_36_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_36_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_36',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_37_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_37_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_37',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_38_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_38_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_38',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_39_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_39_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_39',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_40_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_40_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_40',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_41_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_41_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_41',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_42_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_42_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_42',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_43_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_43_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_43',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_44_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_44_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_44',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_45_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_45_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_45',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_46_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_46_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_46',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_47_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_47_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_47',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_48_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_48_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_48',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_49_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_49_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_49',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_50_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_50_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_50',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_51_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_51_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_51',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_52_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_52_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_52',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_53_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_53_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_53',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_54_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_54_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_54',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_55_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_55_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_55',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_56_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_56_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_56',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_57_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_57_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_57',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_58_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_58_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_58',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_59_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_59_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_59',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_60_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_60_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_60',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_61_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_61_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_61',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_62_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_62_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_62',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_63_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_63_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_63',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_64_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_64_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_64',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_65_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_65_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_65',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_66_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_66_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_66',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_67_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_67_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_67',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_68_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_68_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_68',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_69_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_69_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_69',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_70_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_70_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_70',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_71_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_71_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_71',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_72_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_72_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_72',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_73_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_73_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_73',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_74_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_74_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_74',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_75_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_75_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_75',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_76_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_76_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_76',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_77_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_77_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_77',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_78_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_78_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_78',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_79_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_79_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_79',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_80_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_80_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_80',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_81_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_81_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_81',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_82_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_82_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_82',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_83_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_83_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_83',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_84_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_84_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_84',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_85_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_85_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_85',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_86_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_86_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_86',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_87_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_87_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_87',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_88_std',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_88_alpaca',\n",
       " 'AdapterOcean/med_alpaca_standardized_cluster_88',\n",
       " ...]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(filter(lambda n: 'alpaca' in n.lower(), dataset_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "95c8a727-11a6-4716-9a09-69991a9b14dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset FashionMNIST\n",
       "    Number of datapoints: 60000\n",
       "    Root location: data\n",
       "    Split: Train\n",
       "    StandardTransform\n",
       "Transform: ToTensor()"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "170f15d5-3191-44e9-bfa9-cade5f6ee5fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import torch\n",
    "# from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "# model_name = \"Deci/DeciLM-7B\"\n",
    "# device = \"cuda\" # for GPU usage or \"cpu\" for CPU usage\n",
    "\n",
    "# tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "# model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=\"auto\", trust_remote_code=True).to(device)\n",
    "\n",
    "# inputs = tokenizer.encode(\"In a shocking finding, scientists discovered a herd of unicorns living in\", return_tensors=\"pt\").to(device)\n",
    "# outputs = model.generate(inputs, max_new_tokens=100, do_sample=True, top_p=0.95)\n",
    "# print(tokenizer.decode(outputs[0]))\n",
    "\n",
    "# # The model can also be used via the text-generation pipeline interface\n",
    "# from transformers import pipeline\n",
    "# generator = pipeline(\"text-generation\", \"Deci/DeciLM-7B\", torch_dtype=\"auto\", trust_remote_code=True, device=device)\n",
    "# outputs = generator(\"In a shocking finding, scientists discovered a herd of unicorns living in\", max_new_tokens=100, do_sample=True, top_p=0.95)\n",
    "# print(outputs[0][\"generated_text\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8fb2e1bd-9811-4796-8b78-ef06a6bdc0a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://0.0.0.0:7861\n",
      "\n",
      "To create a public link, set `share=True` in `launch()`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"http://localhost:7861/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-15 (generate_and_signal_complete):\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/threading.py\", line 1016, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/threading.py\", line 953, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/tmp/ipykernel_33615/3155043630.py\", line 181, in generate_and_signal_complete\n",
      "NameError: name 'ov_model' is not defined\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/gradio/routes.py\", line 488, in run_predict\n",
      "    output = await app.get_blocks().process_api(\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/gradio/blocks.py\", line 1431, in process_api\n",
      "    result = await self.call_function(\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/gradio/blocks.py\", line 1117, in call_function\n",
      "    prediction = await utils.async_iteration(iterator)\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/gradio/utils.py\", line 350, in async_iteration\n",
      "    return await iterator.__anext__()\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/gradio/utils.py\", line 343, in __anext__\n",
      "    return await anyio.to_thread.run_sync(\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/anyio/to_thread.py\", line 56, in run_sync\n",
      "    return await get_async_backend().run_sync_in_worker_thread(\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/anyio/_backends/_asyncio.py\", line 2134, in run_sync_in_worker_thread\n",
      "    return await future\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/anyio/_backends/_asyncio.py\", line 851, in run\n",
      "    result = context.run(func, *args)\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/gradio/utils.py\", line 326, in run_sync_iterator_async\n",
      "    return next(iterator)\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/gradio/utils.py\", line 695, in gen_wrapper\n",
      "    yield from f(*args, **kwargs)\n",
      "  File \"/tmp/ipykernel_33615/3155043630.py\", line 189, in bot\n",
      "    for new_text in streamer:\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/site-packages/transformers/generation/streamers.py\", line 223, in __next__\n",
      "    value = self.text_queue.get(timeout=self.timeout)\n",
      "  File \"/home/syd/mambaforge/envs/torch-py310/lib/python3.10/queue.py\", line 179, in get\n",
      "    raise Empty\n",
      "_queue.Empty\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from threading import Event, Thread\n",
    "from uuid import uuid4\n",
    "from typing import List, Tuple\n",
    "import gradio as gr\n",
    "from transformers import (\n",
    "    AutoTokenizer,\n",
    "    StoppingCriteria,\n",
    "    StoppingCriteriaList,\n",
    "    TextIteratorStreamer,\n",
    ")\n",
    "\n",
    "\n",
    "model_name = \"stabilityai/stablelm-zephyr-3b\"\n",
    "\n",
    "tok = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)\n",
    "\n",
    "DEFAULT_SYSTEM_PROMPT = \"\"\"\\\n",
    "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n",
    "If a question does not make any sense or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.\\\n",
    "\"\"\"\n",
    "\n",
    "model_configuration = {\n",
    "    \"start_message\": f\"<|system|>\\n {DEFAULT_SYSTEM_PROMPT }<|endoftext|>\",\n",
    "    \"history_template\": \"<|user|>\\n{user}<|endoftext|><|assistant|>\\n{assistant}<|endoftext|>\",\n",
    "    \"current_message_template\": '<|user|>\\n{user}<|endoftext|><|assistant|>\\n{assistant}',\n",
    "}\n",
    "history_template = model_configuration[\"history_template\"]\n",
    "current_message_template = model_configuration[\"current_message_template\"]\n",
    "start_message = model_configuration[\"start_message\"]\n",
    "stop_tokens = model_configuration.get(\"stop_tokens\")\n",
    "tokenizer_kwargs = model_configuration.get(\"tokenizer_kwargs\", {})\n",
    "\n",
    "examples = [\n",
    "    [\"Hello there! How are you doing?\"],\n",
    "    [\"What is OpenVINO?\"],\n",
    "    [\"Who are you?\"],\n",
    "    [\"Can you explain to me briefly what is Python programming language?\"],\n",
    "    [\"Explain the plot of Cinderella in a sentence.\"],\n",
    "    [\"What are some common mistakes to avoid when writing code?\"],\n",
    "    [\n",
    "        \"Write a 100-word blog post on “Benefits of Artificial Intelligence and OpenVINO“\"\n",
    "    ],\n",
    "]\n",
    "\n",
    "max_new_tokens = 256\n",
    "\n",
    "\n",
    "class StopOnTokens(StoppingCriteria):\n",
    "    def __init__(self, token_ids):\n",
    "        self.token_ids = token_ids\n",
    "\n",
    "    def __call__(\n",
    "        self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs\n",
    "    ) -> bool:\n",
    "        for stop_id in self.token_ids:\n",
    "            if input_ids[0][-1] == stop_id:\n",
    "                return True\n",
    "        return False\n",
    "\n",
    "\n",
    "if stop_tokens is not None:\n",
    "    if isinstance(stop_tokens[0], str):\n",
    "        stop_tokens = tok.convert_tokens_to_ids(stop_tokens)\n",
    "\n",
    "    stop_tokens = [StopOnTokens(stop_tokens)]\n",
    "\n",
    "\n",
    "def default_partial_text_processor(partial_text: str, new_text: str):\n",
    "    \"\"\"\n",
    "    helper for updating partially generated answer, used by de\n",
    "\n",
    "    Params:\n",
    "      partial_text: text buffer for storing previosly generated text\n",
    "      new_text: text update for the current step\n",
    "    Returns:\n",
    "      updated text string\n",
    "\n",
    "    \"\"\"\n",
    "    partial_text += new_text\n",
    "    return partial_text\n",
    "\n",
    "\n",
    "text_processor = model_configuration.get(\n",
    "    \"partial_text_processor\", default_partial_text_processor\n",
    ")\n",
    "\n",
    "def convert_history_to_text(history: List[Tuple[str, str]]):\n",
    "    \"\"\"\n",
    "    function for conversion history stored as list pairs of user and assistant messages to string according to model expected conversation template\n",
    "    Params:\n",
    "      history: dialogue history\n",
    "    Returns:\n",
    "      history in text format\n",
    "    \"\"\"\n",
    "    text = start_message + \"\".join(\n",
    "        [\n",
    "            \"\".join(\n",
    "                [history_template.format(num=round, user=item[0], assistant=item[1])]\n",
    "            )\n",
    "            for round, item in enumerate(history[:-1])\n",
    "        ]\n",
    "    )\n",
    "    text += \"\".join(\n",
    "        [\n",
    "            \"\".join(\n",
    "                [\n",
    "                    current_message_template.format(\n",
    "                        num=len(history) + 1,\n",
    "                        user=history[-1][0],\n",
    "                        assistant=history[-1][1],\n",
    "                    )\n",
    "                ]\n",
    "            )\n",
    "        ]\n",
    "    )\n",
    "    return text\n",
    "\n",
    "\n",
    "def user(message, history):\n",
    "    \"\"\"\n",
    "    callback function for updating user messages in interface on submit button click\n",
    "\n",
    "    Params:\n",
    "      message: current message\n",
    "      history: conversation history\n",
    "    Returns:\n",
    "      None\n",
    "    \"\"\"\n",
    "    # Append the user's message to the conversation history\n",
    "    return \"\", history + [[message, \"\"]]\n",
    "\n",
    "\n",
    "def bot(history, temperature, top_p, top_k, repetition_penalty, conversation_id):\n",
    "    \"\"\"\n",
    "    callback function for running chatbot on submit button click\n",
    "\n",
    "    Params:\n",
    "      history: conversation history\n",
    "      temperature:  parameter for control the level of creativity in AI-generated text.\n",
    "                    By adjusting the `temperature`, you can influence the AI model's probability distribution, making the text more focused or diverse.\n",
    "      top_p: parameter for control the range of tokens considered by the AI model based on their cumulative probability.\n",
    "      top_k: parameter for control the range of tokens considered by the AI model based on their cumulative probability, selecting number of tokens with highest probability.\n",
    "      repetition_penalty: parameter for penalizing tokens based on how frequently they occur in the text.\n",
    "      conversation_id: unique conversation identifier.\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    # Construct the input message string for the model by concatenating the current system message and conversation history\n",
    "    messages = convert_history_to_text(history)\n",
    "\n",
    "    # Tokenize the messages string\n",
    "    input_ids = tok(messages, return_tensors=\"pt\", **tokenizer_kwargs).input_ids\n",
    "    if input_ids.shape[1] > 2000:\n",
    "        history = [history[-1]]\n",
    "        messages = convert_history_to_text(history)\n",
    "        input_ids = tok(messages, return_tensors=\"pt\", **tokenizer_kwargs).input_ids\n",
    "    streamer = TextIteratorStreamer(\n",
    "        tok, timeout=30.0, skip_prompt=True, skip_special_tokens=True\n",
    "    )\n",
    "    generate_kwargs = dict(\n",
    "        input_ids=input_ids,\n",
    "        max_new_tokens=max_new_tokens,\n",
    "        temperature=temperature,\n",
    "        do_sample=temperature > 0.0,\n",
    "        top_p=top_p,\n",
    "        top_k=top_k,\n",
    "        repetition_penalty=repetition_penalty,\n",
    "        streamer=streamer,\n",
    "    )\n",
    "    if stop_tokens is not None:\n",
    "        generate_kwargs[\"stopping_criteria\"] = StoppingCriteriaList(stop_tokens)\n",
    "\n",
    "    stream_complete = Event()\n",
    "\n",
    "    def generate_and_signal_complete():\n",
    "        \"\"\"\n",
    "        genration function for single thread\n",
    "        \"\"\"\n",
    "        global start_time\n",
    "        ov_model.generate(**generate_kwargs)\n",
    "        stream_complete.set()\n",
    "\n",
    "    t1 = Thread(target=generate_and_signal_complete)\n",
    "    t1.start()\n",
    "\n",
    "    # Initialize an empty string to store the generated text\n",
    "    partial_text = \"\"\n",
    "    for new_text in streamer:\n",
    "        partial_text = text_processor(partial_text, new_text)\n",
    "        history[-1][1] = partial_text\n",
    "        yield history\n",
    "\n",
    "\n",
    "def get_uuid():\n",
    "    \"\"\"\n",
    "    universal unique identifier for thread\n",
    "    \"\"\"\n",
    "    return str(uuid4())\n",
    "\n",
    "\n",
    "with gr.Blocks(\n",
    "    theme=gr.themes.Soft(),\n",
    "    css=\".disclaimer {font-variant-caps: all-small-caps;}\",\n",
    ") as demo:\n",
    "    conversation_id = gr.State(get_uuid)\n",
    "    gr.Markdown(f\"\"\"<h1><center>OpenVINO {model_name} Chatbot</center></h1>\"\"\")\n",
    "    chatbot = gr.Chatbot(height=500)\n",
    "    with gr.Row():\n",
    "        with gr.Column():\n",
    "            msg = gr.Textbox(\n",
    "                label=\"Chat Message Box\",\n",
    "                placeholder=\"Chat Message Box\",\n",
    "                show_label=False,\n",
    "                container=False,\n",
    "            )\n",
    "        with gr.Column():\n",
    "            with gr.Row():\n",
    "                submit = gr.Button(\"Submit\")\n",
    "                stop = gr.Button(\"Stop\")\n",
    "                clear = gr.Button(\"Clear\")\n",
    "    with gr.Row():\n",
    "        with gr.Accordion(\"Advanced Options:\", open=False):\n",
    "            with gr.Row():\n",
    "                with gr.Column():\n",
    "                    with gr.Row():\n",
    "                        temperature = gr.Slider(\n",
    "                            label=\"Temperature\",\n",
    "                            value=0.1,\n",
    "                            minimum=0.0,\n",
    "                            maximum=1.0,\n",
    "                            step=0.1,\n",
    "                            interactive=True,\n",
    "                            info=\"Higher values produce more diverse outputs\",\n",
    "                        )\n",
    "                with gr.Column():\n",
    "                    with gr.Row():\n",
    "                        top_p = gr.Slider(\n",
    "                            label=\"Top-p (nucleus sampling)\",\n",
    "                            value=1.0,\n",
    "                            minimum=0.0,\n",
    "                            maximum=1,\n",
    "                            step=0.01,\n",
    "                            interactive=True,\n",
    "                            info=(\n",
    "                                \"Sample from the smallest possible set of tokens whose cumulative probability \"\n",
    "                                \"exceeds top_p. Set to 1 to disable and sample from all tokens.\"\n",
    "                            ),\n",
    "                        )\n",
    "                with gr.Column():\n",
    "                    with gr.Row():\n",
    "                        top_k = gr.Slider(\n",
    "                            label=\"Top-k\",\n",
    "                            value=50,\n",
    "                            minimum=0.0,\n",
    "                            maximum=200,\n",
    "                            step=1,\n",
    "                            interactive=True,\n",
    "                            info=\"Sample from a shortlist of top-k tokens — 0 to disable and sample from all tokens.\",\n",
    "                        )\n",
    "                with gr.Column():\n",
    "                    with gr.Row():\n",
    "                        repetition_penalty = gr.Slider(\n",
    "                            label=\"Repetition Penalty\",\n",
    "                            value=1.1,\n",
    "                            minimum=1.0,\n",
    "                            maximum=2.0,\n",
    "                            step=0.1,\n",
    "                            interactive=True,\n",
    "                            info=\"Penalize repetition — 1.0 to disable.\",\n",
    "                        )\n",
    "    gr.Examples(\n",
    "        examples, inputs=msg, label=\"Click on any example and press the 'Submit' button\"\n",
    "    )\n",
    "\n",
    "    submit_event = msg.submit(\n",
    "        fn=user,\n",
    "        inputs=[msg, chatbot],\n",
    "        outputs=[msg, chatbot],\n",
    "        queue=False,\n",
    "    ).then(\n",
    "        fn=bot,\n",
    "        inputs=[\n",
    "            chatbot,\n",
    "            temperature,\n",
    "            top_p,\n",
    "            top_k,\n",
    "            repetition_penalty,\n",
    "            conversation_id,\n",
    "        ],\n",
    "        outputs=chatbot,\n",
    "        queue=True,\n",
    "    )\n",
    "    submit_click_event = submit.click(\n",
    "        fn=user,\n",
    "        inputs=[msg, chatbot],\n",
    "        outputs=[msg, chatbot],\n",
    "        queue=False,\n",
    "    ).then(\n",
    "        fn=bot,\n",
    "        inputs=[\n",
    "            chatbot,\n",
    "            temperature,\n",
    "            top_p,\n",
    "            top_k,\n",
    "            repetition_penalty,\n",
    "            conversation_id,\n",
    "        ],\n",
    "        outputs=chatbot,\n",
    "        queue=True,\n",
    "    )\n",
    "    stop.click(\n",
    "        fn=None,\n",
    "        inputs=None,\n",
    "        outputs=None,\n",
    "        cancels=[submit_event, submit_click_event],\n",
    "        queue=False,\n",
    "    )\n",
    "    clear.click(lambda: None, None, chatbot, queue=False)\n",
    "\n",
    "demo.queue(max_size=2)\n",
    "# if you are launching remotely, specify server_name and server_port\n",
    "#  demo.launch(server_name='your server name', server_port='server port in int')\n",
    "# if you have any issue to launch on your platform, you can pass share=True to launch method:\n",
    "# demo.launch(share=True)\n",
    "# it creates a publicly shareable link for the interface. Read more in the docs: https://gradio.app/docs/\n",
    "demo.launch(server_name='0.0.0.0')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1d3ddcf-1cd7-440e-93f7-d32e5f2505f3",
   "metadata": {},
   "source": [
    "# Build SFT trademark datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5127cf87-e58b-45d3-8aa2-73e2bf9f93fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "met_df = pd.read_pickle('../../data-augmentation/train_ds/CODE_cleaned_s41_met_before_cut-off.pickle')\n",
    "not_met_df = pd.read_pickle('../../data-augmentation/train_ds/CODE_cleaned_s41_not_met_before_cut-off.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fbfe1d4a-897e-4f2d-ad8b-40b02bdb0ba4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((36977, 11), (21598, 11))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "met_df.shape, not_met_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "96b4b117-7eea-4696-8829-8a807306bb3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>appl_number</th>\n",
       "      <th>s_41_requirement_outcome</th>\n",
       "      <th>sub_requirement_name</th>\n",
       "      <th>assessment_summary_txt</th>\n",
       "      <th>word_text_value</th>\n",
       "      <th>date_of_assessment</th>\n",
       "      <th>class_gsi_jsonish</th>\n",
       "      <th>tm_phrase</th>\n",
       "      <th>gs</th>\n",
       "      <th>gs-text-list</th>\n",
       "      <th>gs-nums</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>1739647</td>\n",
       "      <td>MET</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1. Clear,Exp. Headstart</td>\n",
       "      <td>CROWN RESORTS FOUNDATION</td>\n",
       "      <td>2016-01-04 00:00:00.000</td>\n",
       "      <td>[[{\"41\" : \", Entertainment, Charitable service...</td>\n",
       "      <td>crown resorts foundation</td>\n",
       "      <td>[{'class': '41', 'text': 'Entertainment'}, {'c...</td>\n",
       "      <td>[Entertainment, Charitable services, namely ed...</td>\n",
       "      <td>(35, 36, 41, 43)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>1718503</td>\n",
       "      <td>MET</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1. Adv.,S44: 1611785, 1651439 2. Adv., deferme...</td>\n",
       "      <td>INFLUX SYSTEMS</td>\n",
       "      <td>2016-01-05 00:00:00.000</td>\n",
       "      <td>[[{\"42\" : \", Outsource service providers in th...</td>\n",
       "      <td>influx systems</td>\n",
       "      <td>[{'class': '42', 'text': 'Outsource service pr...</td>\n",
       "      <td>[Outsource service providers in the field of i...</td>\n",
       "      <td>(42,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>1722991</td>\n",
       "      <td>MET</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1. Clear</td>\n",
       "      <td>NUTRIBLITZ</td>\n",
       "      <td>2016-01-07 00:00:00.000</td>\n",
       "      <td>[[{\"7\" : \", Electric blenders for household pu...</td>\n",
       "      <td>nutriblitz</td>\n",
       "      <td>[{'class': '7', 'text': 'Electric blenders for...</td>\n",
       "      <td>[Electric blenders for household purposes, Ele...</td>\n",
       "      <td>(7,)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     appl_number s_41_requirement_outcome sub_requirement_name  \\\n",
       "170      1739647                      MET                  NaN   \n",
       "171      1718503                      MET                  NaN   \n",
       "172      1722991                      MET                  NaN   \n",
       "\n",
       "                                assessment_summary_txt  \\\n",
       "170                            1. Clear,Exp. Headstart   \n",
       "171  1. Adv.,S44: 1611785, 1651439 2. Adv., deferme...   \n",
       "172                                           1. Clear   \n",
       "\n",
       "              word_text_value       date_of_assessment  \\\n",
       "170  CROWN RESORTS FOUNDATION  2016-01-04 00:00:00.000   \n",
       "171            INFLUX SYSTEMS  2016-01-05 00:00:00.000   \n",
       "172                NUTRIBLITZ  2016-01-07 00:00:00.000   \n",
       "\n",
       "                                     class_gsi_jsonish  \\\n",
       "170  [[{\"41\" : \", Entertainment, Charitable service...   \n",
       "171  [[{\"42\" : \", Outsource service providers in th...   \n",
       "172  [[{\"7\" : \", Electric blenders for household pu...   \n",
       "\n",
       "                    tm_phrase  \\\n",
       "170  crown resorts foundation   \n",
       "171            influx systems   \n",
       "172                nutriblitz   \n",
       "\n",
       "                                                    gs  \\\n",
       "170  [{'class': '41', 'text': 'Entertainment'}, {'c...   \n",
       "171  [{'class': '42', 'text': 'Outsource service pr...   \n",
       "172  [{'class': '7', 'text': 'Electric blenders for...   \n",
       "\n",
       "                                          gs-text-list           gs-nums  \n",
       "170  [Entertainment, Charitable services, namely ed...  (35, 36, 41, 43)  \n",
       "171  [Outsource service providers in the field of i...             (42,)  \n",
       "172  [Electric blenders for household purposes, Ele...              (7,)  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "met_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f609345e-7831-4156-b4f3-be9acef2a72a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>appl_number</th>\n",
       "      <th>s_41_requirement_outcome</th>\n",
       "      <th>sub_requirement_name</th>\n",
       "      <th>assessment_summary_txt</th>\n",
       "      <th>word_text_value</th>\n",
       "      <th>date_of_assessment</th>\n",
       "      <th>class_gsi_jsonish</th>\n",
       "      <th>tm_phrase</th>\n",
       "      <th>gs</th>\n",
       "      <th>gs-text-list</th>\n",
       "      <th>gs-nums</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1067476</td>\n",
       "      <td>NOT_MET</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.s41.5-desc 2.with s41;clr</td>\n",
       "      <td>REJUVENATE</td>\n",
       "      <td>2005-10-05 00:00:00.000</td>\n",
       "      <td>[[{\"20\" : \", bolsters, mattresses, pillows, cu...</td>\n",
       "      <td>rejuvenate</td>\n",
       "      <td>[{'class': '20', 'text': 'bolsters'}, {'class'...</td>\n",
       "      <td>[bolsters, mattresses, pillows, cushions, Furn...</td>\n",
       "      <td>(20,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>1382040</td>\n",
       "      <td>NOT_MET</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1. s41(5); Section 44: 457529, 1379819</td>\n",
       "      <td>KENT</td>\n",
       "      <td>2010-12-20 00:00:00.000</td>\n",
       "      <td>[[{\"32\" : \", Beer, \"}]]</td>\n",
       "      <td>kent</td>\n",
       "      <td>[{'class': '32', 'text': 'Beer'}]</td>\n",
       "      <td>[Beer]</td>\n",
       "      <td>(32,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>1446020</td>\n",
       "      <td>NOT_MET</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1. Exp. S41(5) sesc. Form; Classn, s/s. Sectio...</td>\n",
       "      <td>FASTCARD</td>\n",
       "      <td>2011-09-23 00:00:00.000</td>\n",
       "      <td>[[{\"38\" : \", Point of sale communication servi...</td>\n",
       "      <td>fastcard</td>\n",
       "      <td>[{'class': '38', 'text': 'Point of sale commun...</td>\n",
       "      <td>[Point of sale communication services, telepho...</td>\n",
       "      <td>(38,)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    appl_number s_41_requirement_outcome sub_requirement_name  \\\n",
       "10      1067476                  NOT_MET                  NaN   \n",
       "51      1382040                  NOT_MET                  NaN   \n",
       "63      1446020                  NOT_MET                  NaN   \n",
       "\n",
       "                               assessment_summary_txt word_text_value  \\\n",
       "10                        1.s41.5-desc 2.with s41;clr      REJUVENATE   \n",
       "51             1. s41(5); Section 44: 457529, 1379819            KENT   \n",
       "63  1. Exp. S41(5) sesc. Form; Classn, s/s. Sectio...        FASTCARD   \n",
       "\n",
       "         date_of_assessment  \\\n",
       "10  2005-10-05 00:00:00.000   \n",
       "51  2010-12-20 00:00:00.000   \n",
       "63  2011-09-23 00:00:00.000   \n",
       "\n",
       "                                    class_gsi_jsonish   tm_phrase  \\\n",
       "10  [[{\"20\" : \", bolsters, mattresses, pillows, cu...  rejuvenate   \n",
       "51                            [[{\"32\" : \", Beer, \"}]]        kent   \n",
       "63  [[{\"38\" : \", Point of sale communication servi...    fastcard   \n",
       "\n",
       "                                                   gs  \\\n",
       "10  [{'class': '20', 'text': 'bolsters'}, {'class'...   \n",
       "51                  [{'class': '32', 'text': 'Beer'}]   \n",
       "63  [{'class': '38', 'text': 'Point of sale commun...   \n",
       "\n",
       "                                         gs-text-list gs-nums  \n",
       "10  [bolsters, mattresses, pillows, cushions, Furn...   (20,)  \n",
       "51                                             [Beer]   (32,)  \n",
       "63  [Point of sale communication services, telepho...   (38,)  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_met_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c5c98464-dbe6-4abd-ae9b-7317ecd25c29",
   "metadata": {},
   "outputs": [],
   "source": [
    "s1_template = \"\"\" \n",
    "trademark \"{tm:s}\" for goods and services: \n",
    "\"{gs:s}\"\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "62ae58fb-43e2-491d-995b-282c1b15fd55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "s_41_requirement_outcome\n",
       "MET        1000\n",
       "NOT_MET    1000\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "met_1k = met_df.sample(frac=1000/len(met_df))\n",
    "not_met_1k = not_met_df.sample(frac=1000/len(not_met_df))\n",
    "\n",
    "merged = pd.concat([met_1k, not_met_1k]).fillna('') # use empty string as the value for nan\n",
    "\n",
    "merged['s_41_requirement_outcome'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "66b509c0-892e-450b-925e-2bf92290021a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "tm_conversations = []\n",
    "\n",
    "for r in merged.sample(frac=1.0).itertuples():\n",
    "    gs_text = '; '.join([item['text'] for item in r.gs])\n",
    "    prompt = s1_template.format(tm=r.word_text_value, gs=gs_text)\n",
    "\n",
    "    reason = r.sub_requirement_name if r.sub_requirement_name else 'other'\n",
    "    completion = f' {r.s_41_requirement_outcome} </s> {reason} '\n",
    "    tm_conversations.append({'prompt': prompt, 'completion': completion})\n",
    "\n",
    "with open('tm_conversations.json', 'w') as fh:\n",
    "    json.dump(tm_conversations, fh, indent=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5093a191-3538-463d-b170-fafc6caf3faa",
   "metadata": {},
   "source": [
    "## SFT out of memory errors notes\n",
    "\n",
    "```plaintext\n",
    "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.00 GiB. GPU 0 has a total capacty of 23.67 GiB of which 1.38 GiB is free. Including non-PyTorch memory, this process has 22.10 GiB memory in use. Of the allocated memory 18.35 GiB is allocated by PyTorch, and 3.42 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n",
    "  1%|          | 12/1875 [01:32<3:59:11,  7.70s/it]\n",
    "```\n",
    "\n",
    "### solution\n",
    "use `model_name = 'TinyLlama/TinyLlama-1.1B-Chat-v1.0'`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8dafaf06-9bb9-4914-a1e4-032c0278f289",
   "metadata": {},
   "source": [
    "## Test datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "62c0144c-79e6-4614-8db3-e22eaec63865",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_pickle('../train_ds/test_cases_Phrase.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "499dbde2-4834-4ace-b5a1-4a8bbeccc761",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ext_id</th>\n",
       "      <th>tm_number</th>\n",
       "      <th>generic</th>\n",
       "      <th>gs</th>\n",
       "      <th>tm_phrase</th>\n",
       "      <th>Non distinct reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AMCZ-2312189204</td>\n",
       "      <td>2352983</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'class': '43', 'text': 'Hotel accommodation ...</td>\n",
       "      <td>Casa Renoir</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AMCZ-2312189296</td>\n",
       "      <td>2352988</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'class': '44', 'text': 'Massage'}, {'class':...</td>\n",
       "      <td>Somatic Being</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AMCZ-2312189726</td>\n",
       "      <td>2353008</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'class': '12', 'text': 'Wiper blades for veh...</td>\n",
       "      <td>Dryline</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AMCZ-2312190303</td>\n",
       "      <td>2353038</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'class': '16', 'text': 'Art prints'}, {'clas...</td>\n",
       "      <td>Monsieur Finger</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AMCZ-2312190532</td>\n",
       "      <td>2353062</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'class': '3', 'text': 'Cosmetics for childre...</td>\n",
       "      <td>Ooku and Daimyo</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            ext_id  tm_number  generic  \\\n",
       "0  AMCZ-2312189204    2352983    False   \n",
       "1  AMCZ-2312189296    2352988    False   \n",
       "2  AMCZ-2312189726    2353008    False   \n",
       "3  AMCZ-2312190303    2353038    False   \n",
       "4  AMCZ-2312190532    2353062    False   \n",
       "\n",
       "                                                  gs        tm_phrase  \\\n",
       "0  [{'class': '43', 'text': 'Hotel accommodation ...     Casa Renoir    \n",
       "1  [{'class': '44', 'text': 'Massage'}, {'class':...    Somatic Being   \n",
       "2  [{'class': '12', 'text': 'Wiper blades for veh...          Dryline   \n",
       "3  [{'class': '16', 'text': 'Art prints'}, {'clas...  Monsieur Finger   \n",
       "4  [{'class': '3', 'text': 'Cosmetics for childre...  Ooku and Daimyo   \n",
       "\n",
       "  Non distinct reason  \n",
       "0                 NaN  \n",
       "1                 NaN  \n",
       "2                 NaN  \n",
       "3                 NaN  \n",
       "4                 NaN  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1b2f9a7d-047b-4ed0-a199-7e4125dd9582",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = []\n",
    "\n",
    "for r in test_df.itertuples():\n",
    "    prompts.append(s1_template.format(tm=r.tm_phrase, gs='; '.join([item['text'] for item in r.gs])))\n",
    "test_df['prompts'] = prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "480b0186-4529-4e7a-b97d-908f57fa4d56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ext_id</th>\n",
       "      <th>tm_number</th>\n",
       "      <th>generic</th>\n",
       "      <th>gs</th>\n",
       "      <th>tm_phrase</th>\n",
       "      <th>Non distinct reason</th>\n",
       "      <th>prompts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AMCZ-2312189204</td>\n",
       "      <td>2352983</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'class': '43', 'text': 'Hotel accommodation ...</td>\n",
       "      <td>Casa Renoir</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\ntrademark \"Casa Renoir \" for goods and serv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AMCZ-2312189296</td>\n",
       "      <td>2352988</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'class': '44', 'text': 'Massage'}, {'class':...</td>\n",
       "      <td>Somatic Being</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\ntrademark \"Somatic Being\" for goods and ser...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AMCZ-2312189726</td>\n",
       "      <td>2353008</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'class': '12', 'text': 'Wiper blades for veh...</td>\n",
       "      <td>Dryline</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\ntrademark \"Dryline\" for goods and services:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AMCZ-2312190303</td>\n",
       "      <td>2353038</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'class': '16', 'text': 'Art prints'}, {'clas...</td>\n",
       "      <td>Monsieur Finger</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\ntrademark \"Monsieur Finger\" for goods and s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AMCZ-2312190532</td>\n",
       "      <td>2353062</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'class': '3', 'text': 'Cosmetics for childre...</td>\n",
       "      <td>Ooku and Daimyo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\ntrademark \"Ooku and Daimyo\" for goods and s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            ext_id  tm_number  generic  \\\n",
       "0  AMCZ-2312189204    2352983    False   \n",
       "1  AMCZ-2312189296    2352988    False   \n",
       "2  AMCZ-2312189726    2353008    False   \n",
       "3  AMCZ-2312190303    2353038    False   \n",
       "4  AMCZ-2312190532    2353062    False   \n",
       "\n",
       "                                                  gs        tm_phrase  \\\n",
       "0  [{'class': '43', 'text': 'Hotel accommodation ...     Casa Renoir    \n",
       "1  [{'class': '44', 'text': 'Massage'}, {'class':...    Somatic Being   \n",
       "2  [{'class': '12', 'text': 'Wiper blades for veh...          Dryline   \n",
       "3  [{'class': '16', 'text': 'Art prints'}, {'clas...  Monsieur Finger   \n",
       "4  [{'class': '3', 'text': 'Cosmetics for childre...  Ooku and Daimyo   \n",
       "\n",
       "  Non distinct reason                                            prompts  \n",
       "0                 NaN   \\ntrademark \"Casa Renoir \" for goods and serv...  \n",
       "1                 NaN   \\ntrademark \"Somatic Being\" for goods and ser...  \n",
       "2                 NaN   \\ntrademark \"Dryline\" for goods and services:...  \n",
       "3                 NaN   \\ntrademark \"Monsieur Finger\" for goods and s...  \n",
       "4                 NaN   \\ntrademark \"Ooku and Daimyo\" for goods and s...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "905b7334-c8d5-437e-96ea-70568947f405",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from peft import AutoPeftModelForCausalLM\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from sklearn.metrics import classification_report, ConfusionMatrixDisplay \n",
    "\n",
    "DEV = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# model_name =\"./merged_peft/final_merged_checkpoint\"\n",
    "adapter_path = \"./results/final_checkpoint\"\n",
    "# adapter_path = \"./dpo_results/final_checkpoint\"\n",
    "\n",
    "model = AutoPeftModelForCausalLM.from_pretrained(\n",
    "    adapter_path,\n",
    "    torch_dtype=torch.bfloat16,\n",
    "    load_in_4bit=True,\n",
    ")\n",
    "\n",
    "# model = AutoModelForCausalLM.from_pretrained(\n",
    "#     model_name,\n",
    "#     torch_dtype=torch.bfloat16,\n",
    "#     load_in_4bit=True,\n",
    "# )\n",
    "\n",
    "def generate_output(prompt):\n",
    "    tokenizer = AutoTokenizer.from_pretrained(adapter_path)\n",
    "    # tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "    \n",
    "    inputs = tokenizer.encode(\n",
    "        f\"An AI tool that corrects and rephrase user text grammar errors delimited by triple backticks to standard English.\\n### Input: ```{prompt}```\\n### Output:\",\n",
    "        return_tensors=\"pt\",\n",
    "    ).to(DEV)\n",
    "    \n",
    "    generate_kwargs = dict(\n",
    "        input_ids=inputs,\n",
    "        temperature=0.2,\n",
    "        top_p=0.95,\n",
    "        top_k=40,\n",
    "        max_new_tokens=500,\n",
    "        repetition_penalty=1.3,\n",
    "    )\n",
    "    outputs = model.generate(**generate_kwargs)\n",
    "    return tokenizer.decode(outputs[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5c760fa3-cef6-426d-ba2f-6ddfd127ed18",
   "metadata": {},
   "outputs": [],
   "source": [
    "p0 = test_df['prompts'].iloc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "82d675e5-6c7c-4bd2-8abb-6aa6e367070d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' MET 1  ###  NOT_MET  Not_Met  MET 2  ###  Compliance  MET 3  ###  Non_Compliant  MET 4  ###  Invalid  MET 5  ###  Unchanged  MET 6  ###  Meet_Base Standard  MET 7  ###  Meet_Basic  MET 8  ###  Meet the Base  MET 9  ###  Meet the Minimum  MET 10  ###  No_Changes  MET 11  ###  Conformance  MET 12  ###  Conform_To  MET 13  ###  Reported as is (no editing)  MET 14  ###  Original  MET 15  ###  Translation  MET 16  ###  Localisation  MET 17  ###  Generated  MET 18  ###  Language  MET 19  ###  Preparation of legal documents including writing; drafting or recording forms; MET 20  ###  Legal document creation from information provided  MET 21  ###  Provide searchable index of web site contents  MET 22  ###  Indexing of data using online search facilities; Provision of location information via internet website portal  MET 23  ###  Online provision of information relating to entertainment; Providing live performances via video stream; Production of virtual content; Live performance activities; Video game technology consultancy services; Electronic publication of information on a wide range of topics; providing it through a blog; Webcasting; Entertainment technologies; Technological advisory services; Advisory services relating to publishing; Publication of magazines; Audio book production; Educational seminars; Production of audio recordings; not downloadable; Game design; Producing of motion pictures; videos and other digital media; Publishing of news stories; Recording of music; Arranging and conducting of workshops (training); Distribution of sound tracks; Production of pod casts; Providing online electronic publications (not published by us); such as newspapers; magazines and books; Production of radio programmes; Production of television programs; Musical recitals; Providing online videos;'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_output(p0).split('Output:')[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77c35c94-3bd7-4cc5-ab4d-4e39374a3abb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}