plot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "QYXqCM9Elqgl",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QYXqCM9Elqgl",
    "outputId": "3e68fe7a-3967-40b7-c322-ce808ad71fc2"
   },
   "outputs": [],
   "source": [
    "# from google.colab import drive\n",
    "# drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "tQqLttAwTzZC",
   "metadata": {
    "id": "tQqLttAwTzZC"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import pickle\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "IajHAZGJoBFm",
   "metadata": {
    "id": "IajHAZGJoBFm",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0Gcs8KcpoDK0",
   "metadata": {
    "id": "0Gcs8KcpoDK0"
   },
   "outputs": [],
   "source": [
    "\n",
    "# rm saved_models/Transformers/ReverseTask/ -rf\n",
    "\n",
    "def smooth(y, box_pts):\n",
    "    box = np.ones(box_pts)/box_pts\n",
    "    y_smooth = np.convolve(y, box, mode='same')\n",
    "    return y_smooth\n",
    "\n",
    "\n",
    "def compute_avg(results):\n",
    "    # compute the averages \n",
    "    results_avg = {}\n",
    "\n",
    "    for key, _ in results.keys():\n",
    "        results_avg[key] = {}\n",
    "\n",
    "    for key in results.keys():\n",
    "          for x,y in results[key].items():\n",
    "              # import pdb; pdb.set_trace()\n",
    "              if x in results_avg[key[0]]:\n",
    "                results_avg[key[0]][x] += np.array(y)\n",
    "              else: \n",
    "                results_avg[key[0]][x] = np.array(y)\n",
    "        \n",
    "    # num_runs = len(results.keys())//3\n",
    "    num_runs =  max(j for i,j in list(results.keys()) if i=='kv') + 1\n",
    "    # compute the avg\n",
    "    for key,val in results_avg.items():\n",
    "          for x, y in val.items():\n",
    "              y = np.array(y)\n",
    "              results_avg[key][x] = y / num_runs\n",
    "\n",
    "    results_avg = {k: results_avg[k] for k in desired_order_list if k in results_avg}                \n",
    "    return results_avg\n",
    "\n",
    "\n",
    "def compute_grand_avg(grand_res):\n",
    "\n",
    "    from copy import deepcopy\n",
    "\n",
    "    grand_res_avg = deepcopy(grand_res[0])\n",
    "\n",
    "    for y in grand_res[1:]:\n",
    "      for norm in y:\n",
    "          for k,v in y[norm].items():\n",
    "            grand_res_avg[norm][k] += v\n",
    "\n",
    "\n",
    "    for norm in grand_res_avg:\n",
    "        for k,v in grand_res_avg[norm].items():\n",
    "          grand_res_avg[norm][k] /= len(grand_res)\n",
    "\n",
    "        \n",
    "    grand_res_avg = {k: grand_res_avg[k] for k in desired_order_list if k in grand_res_avg}                    \n",
    "        \n",
    "    # compute std for accs\n",
    "    acc_k = config[which_db]['acc_key']\n",
    "    if acc_k is None:\n",
    "        return grand_res_avg, None\n",
    "    \n",
    "    res = {}\n",
    "    for norm in grand_res[0].keys():\n",
    "        accs = [y[norm][acc_k] for y in grand_res]\n",
    "        res[norm] = [np.mean(accs), np.std(accs)]\n",
    "            \n",
    "    res = {k: res[k] for k in desired_order_list if k in res}\n",
    "    \n",
    "    return grand_res_avg, res\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2M_SZYCHoN_A",
   "metadata": {
    "id": "2M_SZYCHoN_A"
   },
   "outputs": [],
   "source": [
    "def do_plot(results_avg):\n",
    "\n",
    "    fig, axs = plt.subplots(1,2) #, sharex=True, sharey=True)\n",
    "\n",
    "    # results_avg = {k: results_avg[k] for k in desired_order_list if k in results_avg}    \n",
    "\n",
    "    idx = 0\n",
    "    for key in results_avg.keys():\n",
    "          t_time = results_avg[key]['time_spent'] \n",
    "          box_size = config[which_db]['smooth_factor_train'] \n",
    "          zz = results_avg[key]['train_loss_history']\n",
    "          tt = np.log10(zz) if log_plot else zz\n",
    "          ss = smooth(tt,box_size)[box_size:-box_size]\n",
    "\n",
    "          axs[0].plot(t_time[box_size:-box_size], ss, color_dict[key], lw=2)\n",
    "          axs[0].grid('on')\n",
    "\n",
    "          axs[0].set(title='train', xlabel='Training time (sec)', ylabel='Loss (log 10)' if log_plot else 'Loss')\n",
    "\n",
    "\n",
    "          idx += 1\n",
    "\n",
    "\n",
    "\n",
    "    idx = 0\n",
    "    for key in results_avg.keys():\n",
    "          jj = '' if which_db == 'gpt' else config[which_db]['key_name']+'_'\n",
    "          t_time = results_avg[key][f'{jj}time_spent'] \n",
    "          box_size = config[which_db]['smooth_factor_val']           \n",
    "          xx = config[which_db]['key_name']\n",
    "          zz = results_avg[key][f'{xx}_loss_history']\n",
    "          tt = np.log10(zz) if log_plot else zz\n",
    "          ss = smooth(tt,box_size)[box_size:-box_size]\n",
    "\n",
    "          axs[1].plot(t_time[box_size:-box_size], ss, color_dict[key], lw=2)\n",
    "          axs[1].grid('on')\n",
    "\n",
    "          axs[1].set(title='val', xlabel='Training time (sec)')\n",
    "          \n",
    "          leg = list(results_avg.keys())\n",
    "          if 'sqrt' in leg:\n",
    "              leg[leg.index('sqrt')] = 'qkv'\n",
    "          leg = [map_key_names[l] for l in leg]    \n",
    "          axs[1].legend(leg) \n",
    "\n",
    "          idx += 1\n",
    "\n",
    "\n",
    "    fig.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "# fig.savefig(f'drive/MyDrive/KV_Transformer_Res/{d_dim}_{d_depth}_{h_heads}_{num_epochs}_{lr}_{sequence_length}_{which_task}_avg.png', dpi=150, format=\"png\")\n",
    "\n",
    "\n",
    "def do_plot_all(results):\n",
    "    fig, axs = plt.subplots(1,2, figsize=(10,7)) #, sharex=True, sharey=True)\n",
    "\n",
    "    idx = 0\n",
    "    for key, c in results.keys():\n",
    "          t_time = results[(key, c)]['time_spent'] \n",
    "          box_size = smooth_factor_train \n",
    "          tt = results[(key, c)]['train_loss_history']\n",
    "          ss = smooth(tt,box_size)[box_size:-box_size]\n",
    "\n",
    "          axs[0].plot(t_time[box_size:-box_size], ss, colors[idx//num_runs], lw=3)\n",
    "          axs[0].grid('on')\n",
    "\n",
    "          axs[0].set(title='train', xlabel='Training time (sec)', ylabel= 'Loss (log 10)' if log_plot else 'Loss')\n",
    "\n",
    "          idx += 1\n",
    "\n",
    "\n",
    "\n",
    "    idx = 0\n",
    "    for key, c in results.keys():\n",
    "          t_time = results[(key, c)]['val_time_spent'] \n",
    "          box_size = smooth_factor_val \n",
    "          tt = results[(key, c)]['val_loss_history']\n",
    "          ss = smooth(tt,box_size)[box_size:-box_size]      \n",
    "\n",
    "          axs[1].plot(t_time[box_size:-box_size], ss, colors[idx//num_runs], lw=3)\n",
    "          axs[1].grid('on')\n",
    "\n",
    "          axs[1].set(title='val', xlabel='Training time (sec)')\n",
    "          leg = list(results_avg.keys())\n",
    "          if 'sqrt' in leg:\n",
    "              leg[leg.index('sqrt')] = 'qkv'\n",
    "          leg = [map_key_names[l] for l in leg]    \n",
    "          axs[1].legend(leg) \n",
    "\n",
    "          idx += 1\n",
    "\n",
    "\n",
    "    fig.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "    # fig.savefig(f'drive/MyDrive/KV_Transformer_Res/{d_dim}_{d_depth}_{h_heads}_{num_epochs}_{lr}_{sequence_length}_{which_task}_all.png', dpi=150, format=\"png\")\n",
    "\n",
    "\n",
    "def plot_acc(results_avg):\n",
    "\n",
    "  # colors = ['g-.', 'b:', 'k']\n",
    "\n",
    "  fig = plt.figure(figsize=(2,3))\n",
    "\n",
    "  accs = []\n",
    "  for (key,val), c in zip(results_avg.items(), colors):\n",
    "      if np.max(results_avg[key][config[which_db]['acc_key']]) <=1: \n",
    "          accs.append(results_avg[key][config[which_db]['acc_key']]*100)\n",
    "      else:\n",
    "          accs.append(results_avg[key][config[which_db]['acc_key']])\n",
    "        \n",
    "\n",
    "    \n",
    "  bars = plt.bar(np.arange(len(accs)), accs)\n",
    "  plt.xticks(np.arange(len(accs)), list(results_avg.keys()), rotation=90)\n",
    "\n",
    "  # for bar,c in zip(bars, colors):\n",
    "  #   bar.set_color(c)\n",
    "\n",
    "  fig.gca().yaxis.grid(True, zorder=0,)\n",
    "  plt.ylabel('Acc (%)')\n",
    "  plt.ylim(0,100)\n",
    "  plt.show()\n",
    "\n",
    "  \n",
    "\n",
    "\n",
    "\n",
    "  # fig.savefig(f'drive/MyDrive/KV_Transformer_Res/{d_dim}_{d_depth}_{h_heads}_{num_epochs}_{lr}_{sequence_length}_{which_task}_acc.png', dpi=150, format=\"png\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44e1e74f-9790-4672-a443-f5f3fb44bd37",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "V04vXGwioE2G",
   "metadata": {
    "id": "V04vXGwioE2G"
   },
   "source": [
    "# Main part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "172b206c",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "172b206c",
    "outputId": "25087f67-bbdf-436f-e859-418a2e8945fc"
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[WinError 3] The system cannot find the path specified: './results_new/synthetic/'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[4], line 69\u001b[0m\n\u001b[0;32m     45\u001b[0m config \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m     46\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mreverse\u001b[39m\u001b[38;5;124m'\u001b[39m: {\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msmooth_factor_train\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;241m15\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msmooth_factor_val\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;241m15\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkey_name\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mval\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124macc_key\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtest_acc\u001b[39m\u001b[38;5;124m'\u001b[39m},        \n\u001b[0;32m     47\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msub\u001b[39m\u001b[38;5;124m'\u001b[39m: {\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msmooth_factor_train\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;241m15\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msmooth_factor_val\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;241m15\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkey_name\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mval\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124macc_key\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtest_acc\u001b[39m\u001b[38;5;124m'\u001b[39m},        \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     59\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtranslation_en_de\u001b[39m\u001b[38;5;124m'\u001b[39m: {\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msmooth_factor_train\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;241m1\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msmooth_factor_val\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;241m1\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkey_name\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mval\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124macc_key\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtest_acc\u001b[39m\u001b[38;5;124m'\u001b[39m}        \n\u001b[0;32m     60\u001b[0m }\n\u001b[0;32m     65\u001b[0m accs_res \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros((\u001b[38;5;28mlen\u001b[39m(desired_order_list), \u001b[38;5;28mlen\u001b[39m(DBS)))\n\u001b[1;32m---> 69\u001b[0m files \u001b[38;5;241m=\u001b[39m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlistdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43msource_folder\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     70\u001b[0m files\u001b[38;5;241m.\u001b[39msort()\n\u001b[0;32m     76\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m which_db \u001b[38;5;129;01min\u001b[39;00m DBS:\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 3] The system cannot find the path specified: './results_new/synthetic/'"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "show_plt = 0\n",
    "log_plot = 0\n",
    "\n",
    "# colors = ['y-.', 'b:', 'k', 'c', 'k--', 'r', 'r--', 'b--']\n",
    "colors = ['y-.', 'b--', 'k', 'c', 'k--', 'r', 'r--']\n",
    "# desired_order_list = ['L2','sqrt','std', 'layernorm']#, 'no-sm-sqrt', 'no-sm-std', 'no-sm-layernorm']\n",
    "desired_order_list = ['kv','sqrt']#, 'no-sm-sqrt', 'no-sm-std', 'no-sm-layernorm']\n",
    "color_dict  = dict(zip(desired_order_list,colors))\n",
    "\n",
    "map_key_names = {'qkv':'qkv', 'kv':'kv+pos', 'kv-nopos':'kv'}\n",
    "\n",
    "\n",
    "# ---------------------- synthetics ----------------------\n",
    "DBS = ['reverse', 'sort', 'copy', 'swap', 'sub']\n",
    "desired_order_list = ['qkv', 'kv', 'kv-nopos']\n",
    "color_dict  = dict(zip(desired_order_list,colors))\n",
    "source_folder = './results_new/synthetic/'\n",
    "which_db_add_legend = DBS[-1]\n",
    "\n",
    "# # ---------------------- vision ----------------------\n",
    "# DBS = ['_mnist', '_fashionmnist', 'cifar10.', 'cifar100.', 'anomaly']\n",
    "# # DBS = ['anomaly']\n",
    "# desired_order_list = ['sqrt', 'kv', 'kv-nopos']\n",
    "# color_dict  = dict(zip(desired_order_list,colors))\n",
    "# # which_db_add_legend = DBS[0]\n",
    "# source_folder = './results_new/vision/'\n",
    "\n",
    "# # ---------------------- NLP ----------------------\n",
    "# DBS = ['gpt'] #, 'translation_task', '']\n",
    "# desired_order_list = ['qkv', 'kv', 'kv-nopos']\n",
    "# color_dict  = dict(zip(desired_order_list,colors))\n",
    "# which_db_add_legend = DBS[0]\n",
    "# source_folder = './results_new/nlp/'\n",
    "\n",
    "# # # ---------------------- NLP ----------------------\n",
    "# DBS = ['translation_de_en', 'translation_en_de'] #, 'translation_task', '']\n",
    "# desired_order_list = ['kv', 'kv-nopos', 'qkv']\n",
    "# color_dict  = dict(zip(desired_order_list,colors))\n",
    "# # which_db_add_legend = 'translation_de_en'\n",
    "# # source_folder = './results/nlp/'\n",
    "# source_folder = './results_new/nlp/'\n",
    "\n",
    "\n",
    "\n",
    "config = {\n",
    "'reverse': {'smooth_factor_train':15, 'smooth_factor_val':15, 'key_name':'val', 'acc_key': 'test_acc'},        \n",
    "'sub': {'smooth_factor_train':15, 'smooth_factor_val':15, 'key_name':'val', 'acc_key': 'test_acc'},        \n",
    "'sort': {'smooth_factor_train':15, 'smooth_factor_val':15, 'key_name':'val', 'acc_key': 'test_acc'},        \n",
    "'swap': {'smooth_factor_train':15, 'smooth_factor_val':15, 'key_name':'val', 'acc_key': 'test_acc'},        \n",
    "'copy': {'smooth_factor_train':15, 'smooth_factor_val':15, 'key_name':'val', 'acc_key': 'test_acc'},            \n",
    "'_mnist': {'smooth_factor_train':500, 'smooth_factor_val':100, 'key_name':'val', 'acc_key': 'test_acc'},    \n",
    "'_fashionmnist': {'smooth_factor_train':500, 'smooth_factor_val':100, 'key_name':'val', 'acc_key': 'test_acc'},\n",
    "'cifar10.': {'smooth_factor_train':700, 'smooth_factor_val':500, 'key_name':'val', 'acc_key': 'test_acc'},\n",
    "'cifar100.': {'smooth_factor_train':700, 'smooth_factor_val':500, 'key_name':'val', 'acc_key': 'test_acc'},    \n",
    "'anomaly': {'smooth_factor_train':500, 'smooth_factor_val':500, 'key_name':'val', 'acc_key': 'test_acc'},\n",
    "'numbers': {'smooth_factor_train':200, 'smooth_factor_val':200, 'key_name':'val', 'acc_key': 'acc_history'},        \n",
    "'gpt': {'smooth_factor_train':1, 'smooth_factor_val':1, 'key_name':'test', 'acc_key': None},    \n",
    "'translation_de_en': {'smooth_factor_train':1, 'smooth_factor_val':1, 'key_name':'val', 'acc_key': 'test_acc'},    \n",
    "'translation_en_de': {'smooth_factor_train':1, 'smooth_factor_val':1, 'key_name':'val', 'acc_key': 'test_acc'}        \n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "accs_res = np.zeros((len(desired_order_list), len(DBS)))\n",
    "\n",
    "\n",
    "\n",
    "files = os.listdir(source_folder)\n",
    "files.sort()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "for which_db in DBS:\n",
    "\n",
    "    if which_db == 'anomaly':\n",
    "        desired_order_list = ['qkv', 'kv', 'kv-nopos']\n",
    "        color_dict  = dict(zip(desired_order_list,colors))    \n",
    "    \n",
    "    print(which_db)\n",
    "\n",
    "    \n",
    "    grand_res = []\n",
    "\n",
    "    for f in files:\n",
    "      # print(f)\n",
    "    \n",
    "      # if not f.endswith('.pkl') or f.startswith('512')  or which_db not in f: continue\n",
    "      if not f.endswith('.pkl') or which_db not in f: continue\n",
    "\n",
    "      with open(f'{source_folder}/{f}', 'rb') as g:\n",
    "          results = pickle.load(g)\n",
    "      \n",
    "      # import pdb; pdb.set_trace()   \n",
    "      # some modification for translation\n",
    "      if 'translation' in which_db:\n",
    "            for k,v in results.items():\n",
    "                v['time_spent'] = np.cumsum(v['time_spent'])\n",
    "                v['val_time_spent'] = np.cumsum(v['val_time_spent'])\n",
    "         \n",
    "        \n",
    "      # print(results.keys())\n",
    "            \n",
    "      grand_res.append(compute_avg(results))\n",
    "\n",
    "      \n",
    "      if show_plt:  \n",
    "          print(f)\n",
    "          do_plot(compute_avg(results))  \n",
    "\n",
    "\n",
    "    print(len(grand_res))\n",
    "\n",
    "\n",
    "    grand_res_avg, res_acc = compute_grand_avg(grand_res)\n",
    "    do_plot(grand_res_avg)\n",
    "    \n",
    "\n",
    "    \n",
    "    if not res_acc: continue    \n",
    "    \n",
    "    plot_acc(grand_res_avg)\n",
    "    \n",
    "    x = []\n",
    "    for i,(m,s) in res_acc.items():\n",
    "        print(f'{i} & {m:3.4} ({s:3.2}) \\n ')\n",
    "        x.append(m)\n",
    "    print(f'{np.mean(x):3.2}')\n",
    "    \n",
    "\n",
    "    accs_res[:,DBS.index(which_db)] = np.array(x)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "12a29a0d-90b6-4bde-8f00-2d70d0e1c977",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'time_spent': [2.429320812225342,\n",
       "  6.739947080612183,\n",
       "  11.021318435668945,\n",
       "  15.301215171813965,\n",
       "  19.565826892852783,\n",
       "  23.865971326828003,\n",
       "  28.117332458496094,\n",
       "  32.377562522888184,\n",
       "  36.635693311691284,\n",
       "  40.972492933273315,\n",
       "  45.261197566986084,\n",
       "  49.5072226524353,\n",
       "  53.777650594711304,\n",
       "  58.02485489845276,\n",
       "  62.37654685974121,\n",
       "  66.74114322662354,\n",
       "  71.14817476272583,\n",
       "  75.67902660369873,\n",
       "  79.88920831680298,\n",
       "  84.20834136009216,\n",
       "  88.49276852607727,\n",
       "  92.96220588684082,\n",
       "  97.27837204933167,\n",
       "  101.77230834960938,\n",
       "  106.21382594108582,\n",
       "  110.55776619911194,\n",
       "  114.8761637210846,\n",
       "  119.2555034160614,\n",
       "  123.79969263076782,\n",
       "  128.4488046169281,\n",
       "  132.87604355812073,\n",
       "  137.23995733261108,\n",
       "  141.55237340927124,\n",
       "  146.40762209892273,\n",
       "  150.92448234558105,\n",
       "  155.7985281944275,\n",
       "  160.52265977859497,\n",
       "  165.2111508846283,\n",
       "  169.64895391464233,\n",
       "  174.40864825248718,\n",
       "  178.94571590423584,\n",
       "  183.38846039772034,\n",
       "  188.14138174057007,\n",
       "  192.65029287338257,\n",
       "  197.20613551139832,\n",
       "  201.7533667087555,\n",
       "  206.4047336578369,\n",
       "  210.76075792312622,\n",
       "  215.32786965370178,\n",
       "  219.7882330417633,\n",
       "  224.38141679763794],\n",
       " 'train_loss_history': [tensor(4.3003),\n",
       "  tensor(2.2415),\n",
       "  tensor(2.1325),\n",
       "  tensor(2.0862),\n",
       "  tensor(2.0553),\n",
       "  tensor(2.0206),\n",
       "  tensor(2.0074),\n",
       "  tensor(1.9883),\n",
       "  tensor(1.9625),\n",
       "  tensor(1.9390),\n",
       "  tensor(1.9245),\n",
       "  tensor(1.9195),\n",
       "  tensor(1.9084),\n",
       "  tensor(1.8960),\n",
       "  tensor(1.8908),\n",
       "  tensor(1.8770),\n",
       "  tensor(1.8871),\n",
       "  tensor(1.8804),\n",
       "  tensor(1.8668),\n",
       "  tensor(1.8658),\n",
       "  tensor(1.8502),\n",
       "  tensor(1.8595),\n",
       "  tensor(1.8462),\n",
       "  tensor(1.8485),\n",
       "  tensor(1.8605),\n",
       "  tensor(1.8551),\n",
       "  tensor(1.8465),\n",
       "  tensor(1.8414),\n",
       "  tensor(1.8517),\n",
       "  tensor(1.8362),\n",
       "  tensor(1.8205),\n",
       "  tensor(1.8546),\n",
       "  tensor(1.8376),\n",
       "  tensor(1.8142),\n",
       "  tensor(1.8280),\n",
       "  tensor(1.8355),\n",
       "  tensor(1.8199),\n",
       "  tensor(1.8384),\n",
       "  tensor(1.8379),\n",
       "  tensor(1.8314),\n",
       "  tensor(1.8333),\n",
       "  tensor(1.8341),\n",
       "  tensor(1.8366),\n",
       "  tensor(1.8704),\n",
       "  tensor(1.8297),\n",
       "  tensor(1.8200),\n",
       "  tensor(1.8239),\n",
       "  tensor(1.8241),\n",
       "  tensor(1.8294),\n",
       "  tensor(1.8266),\n",
       "  tensor(1.8193)],\n",
       " 'test_loss_history': [tensor(4.3033),\n",
       "  tensor(2.2659),\n",
       "  tensor(2.1800),\n",
       "  tensor(2.1471),\n",
       "  tensor(2.1445),\n",
       "  tensor(2.1098),\n",
       "  tensor(2.0954),\n",
       "  tensor(2.0832),\n",
       "  tensor(2.0702),\n",
       "  tensor(2.0456),\n",
       "  tensor(2.0299),\n",
       "  tensor(2.0299),\n",
       "  tensor(2.0217),\n",
       "  tensor(2.0245),\n",
       "  tensor(2.0266),\n",
       "  tensor(2.0124),\n",
       "  tensor(2.0162),\n",
       "  tensor(2.0162),\n",
       "  tensor(2.0115),\n",
       "  tensor(2.0071),\n",
       "  tensor(1.9982),\n",
       "  tensor(1.9987),\n",
       "  tensor(1.9955),\n",
       "  tensor(1.9852),\n",
       "  tensor(1.9967),\n",
       "  tensor(1.9988),\n",
       "  tensor(1.9932),\n",
       "  tensor(1.9829),\n",
       "  tensor(1.9965),\n",
       "  tensor(1.9736),\n",
       "  tensor(1.9696),\n",
       "  tensor(1.9974),\n",
       "  tensor(1.9772),\n",
       "  tensor(1.9652),\n",
       "  tensor(1.9623),\n",
       "  tensor(1.9743),\n",
       "  tensor(1.9603),\n",
       "  tensor(1.9862),\n",
       "  tensor(1.9873),\n",
       "  tensor(1.9708),\n",
       "  tensor(1.9762),\n",
       "  tensor(1.9579),\n",
       "  tensor(1.9661),\n",
       "  tensor(2.0082),\n",
       "  tensor(1.9779),\n",
       "  tensor(1.9655),\n",
       "  tensor(1.9521),\n",
       "  tensor(1.9700),\n",
       "  tensor(1.9775),\n",
       "  tensor(1.9666),\n",
       "  tensor(1.9656)]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[('kv', 0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "269b87e1-7e51-4d45-b2d3-da39fd150d42",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([('kv', 0), ('kv', 1), ('qkv', 0), ('qkv', 1), ('kv-nopos', 0), ('kv-nopos', 1)])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad9ecdac-3905-47b1-a446-f3b43f6b18e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(accs_res)\n",
    "print(accs_res.mean(axis=1))\n",
    "\n",
    "for x in accs_res.mean(axis=1):\n",
    "    print(f'{x:3.2} & ', end='\\n')\n",
    "\n",
    "# print(accs_res)\n",
    "\n",
    "# DBS.index('sub')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "JyaYbgh2j4xj",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "JyaYbgh2j4xj",
    "outputId": "8fc9e920-7aad-485b-941e-31c7666560df"
   },
   "outputs": [],
   "source": [
    "# for x in grand_res:\n",
    "#   do_plot(x)\n",
    "#   plot_acc(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "f5573892-d5c0-4c43-bcbf-0d019a553a0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# files.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "ffd9aab2-690f-4c54-8fdc-22f31d4536cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "697f4cb9-474d-4656-8679-68015f5087bc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2-SC1xk1LQla",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 354
    },
    "id": "2-SC1xk1LQla",
    "outputId": "f8b5e6bf-d706-45d4-8e8b-d4e6e19a9de8"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "FwhrlhjSo4Ml",
   "metadata": {
    "id": "FwhrlhjSo4Ml"
   },
   "outputs": [],
   "source": [
    "# grand_res_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "Qnhu7UmUtP6w",
   "metadata": {
    "id": "Qnhu7UmUtP6w"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5KYy3DM4K4E8",
   "metadata": {
    "id": "5KYy3DM4K4E8"
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "yD5whREAVqjt",
   "metadata": {
    "id": "yD5whREAVqjt"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "jupytext": {
   "cell_metadata_filter": "colab,colab_type,id,-all",
   "formats": "ipynb,py:percent",
   "main_language": "python"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 187.334594,
   "end_time": "2023-03-24T16:10:04.696311",
   "environment_variables": {},
   "exception": null,
   "input_path": "course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.ipynb",
   "output_path": ".notebooks/course_UvA-DL/05-transformers-and-MH-attention.ipynb",
   "parameters": {},
   "start_time": "2023-03-24T16:06:57.361717",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}