notebookes/transformer/transformer_positional_encoding_graph.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "transformer positional encoding graph.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPsV1xgE7KIo3Yoa3Gacsfy",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jalammar/jalammar.github.io/blob/master/notebookes/transformer/transformer_positional_encoding_graph.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JaLFyOLDKr56",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMvYBo3RKfmA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# Code from https://www.tensorflow.org/tutorials/text/transformer\n",
        "def get_angles(pos, i, d_model):\n",
        "  angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
        "  return pos * angle_rates\n",
        "\n",
        "def positional_encoding(position, d_model):\n",
        "  angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
        "                          np.arange(d_model)[np.newaxis, :],\n",
        "                          d_model)\n",
        "  \n",
        "  # apply sin to even indices in the array; 2i\n",
        "  angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
        "  \n",
        "  # apply cos to odd indices in the array; 2i+1\n",
        "  angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
        "    \n",
        "  pos_encoding = angle_rads[np.newaxis, ...]\n",
        "    \n",
        "  return pos_encoding\n",
        "\n"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SB9-56JWKv6T",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 518
        },
        "outputId": "379c3f5a-07c7-49c3-e7ad-36de8080b2a1"
      },
      "source": [
        "tokens = 10\n",
        "dimensions = 64\n",
        "\n",
        "pos_encoding = positional_encoding(tokens, dimensions)\n",
        "print (pos_encoding.shape)\n",
        "\n",
        "plt.figure(figsize=(12,8))\n",
        "plt.pcolormesh(pos_encoding[0], cmap='viridis')\n",
        "plt.xlabel('Embedding Dimensions')\n",
        "plt.xlim((0, dimensions))\n",
        "plt.ylim((tokens,0))\n",
        "plt.ylabel('Token Position')\n",
        "plt.colorbar()\n",
        "plt.show()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1, 10, 64)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqQAAAHkCAYAAADo9j1YAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5wlVX3v/c93ehjuylUkXOTqJWoEnRCNiTEKStQjPvECJiaYR0UTb7mYEzzmqOHE19GTE415YpQ5SiTReCOaTJSgiHI0UZFBEQREhtHgjCACKigwzHT/nj92jW7G6do907t7dU9/3nnVq3fVqlpr7ZKe/Pq3atVKVSFJkiS1sqx1ByRJkrS0GZBKkiSpKQNSSZIkNWVAKkmSpKYMSCVJktSUAakkSZKaahKQJjk5ybVJ1iY5s0UfJEmSlqok5yS5OclXpylPkr/uYrUrkjxyqOz0JNd12+lj6c98v4c0yQTwdeAkYD1wKfDcqrp6XjsiSZK0RCV5HPBD4O+r6mHbKH8K8HLgKcAvAG+tql9Ish+wBlgJFHAZ8Kiq+t5s+tMiQ3oCsLaq1lXVPcD7gVMa9EOSJGlJqqrPALf1nHIKg2C1quoLwD5JDgaeDFxYVbd1QeiFwMmz7U+LgPQQ4FtD++u7Y5IkSVoYpovX5iSOWz7bCuZKkjOAMwCyYsWjdjnoftOe+/B9v9tb15XfO7C3fNT1C6WOUdePow7vxcyvH0cd8/E9xlGH92Lm14+jjp3le4yjDu/FzK8fRx1L6V4AXHbFxluqanRlc+jJv7pn3Xrb5NjrveyKjVcBdw8dWlVVq8be0Ji0CEg3AIcN7R/aHbuX7qatAtj18MPqkD/6g2kr/OKp7+ht8JgPvKS3fNT1C6WOUdePow7vxcyvH0cd8/E9xlGH92Lm14+jjp3le4yjDu/FzK8fRx1L6V4ATBx83X+OPGmO3XrbJF/8+OFjr3fi4OvurqqVs6hiunhtA/D4rY5fPIt2gDZD9pcCxyY5MskK4DRgdYN+SJIkNVXA1Bz83xisBn67m23/aOAHVXUj8HHgSUn2TbIv8KTu2KzMe4a0qjYneRmDzk8A51TVVfPdD0mSpKUqyfsYZDoPSLIeeB2wC0BVvQM4n8EM+7XAncDvdGW3JfkfDBKMAGdVVd/kqBlp8gxpVZ3P4ItKkiQtYcVkjSWjuX2tVj13RHkBL52m7BzgnHH2x5WaJEmS1NSCnWUvSZK0sxs8Qzq/ixQtRAakkiRJDY1pEtKi5pC9JEmSmjJDKkmS1EhRTJZD9mZIJUmS1JQZUkmSpIac1GRAKkmS1EwBkwakDtlLkiSpLTOkkiRJDTlkb4ZUkiRJjZkhlSRJaqTA1z5hQCpJktSU6zQ5ZC9JkqTGzJBKkiQ1UpSvfcIMqSRJkhozQypJktRKwaQJUjOkkiRJassMqSRJUiOFs+zBgFSSJKmhMElad6I5h+wlSZLUlBlSSZKkRgqYclKTGVJJkiS1ZYZUkiSpIZ8hNSCVJElqpjAgBYfsJUmS1JgZUkmSpIamygypGVJJkiQ1ZYZUkiSpEZ8hHTAglSRJaqQIkw5YewckSZLUlhlSSZKkhpzUZIZUkiRJjZkhlSRJasRJTQMGpJIkSc2EyXLA2jsgSZKkpsyQSpIkNVLAlPlB74AkSZLaWhQZ0qP2uZm//3/+etry39vwy73Xv+rkf+0tf/8P9x3ZhxMf+5Xe8svvuXtkHcced0Nv+YbJO3rL9zn2tpFt3D51V2/58kN/1Fu+sTaNbIMD+7/rpto8soqpffrbmWKqt3xyr9FtjOzDbv1tzKiOFbOrY2p5zboPNTGGOpa1r2McfSA7UR1z7JgPvIS1p76jdTck4aQmMEMqSUuSwaikhWRRZEglSZJ2RlXOsgcDUkmSpKamHLJ3yF6SJEltmSGVJElqZLBSk/lB74AkSZKaMkMqSZLUjJOawIBUkiSpmVYrNSU5GXgrMAG8s6reuFX5W4Bf7Xb3AO5XVft0ZZPAlV3ZDVX19Nn2x4BUkiRpCUkyAbwNOAlYD1yaZHVVXb3lnKr6g6HzXw4cP1TFXVV13Dj7ZEAqSZLU0GTN+2ufTgDWVtU6gCTvB04Brp7m/OcCr5vLDvnQgiRJ0tJyCPCtof313bGfkuQBwJHAp4YO75ZkTZIvJHnGODpkhlSSJKmRInP12qcDkqwZ2l9VVat2oJ7TgPOqanLo2AOqakOSo4BPJbmyqq6fTWcNSCVJkhqamptZ9rdU1cppyjYAhw3tH9od25bTgJcOH6iqDd3PdUkuZvB86awCUofsJUmSlpZLgWOTHJlkBYOgc/XWJyV5MLAv8PmhY/sm2bX7fADwWKZ/9nTGzJBKkiQ10mKlpqranORlwMcZvPbpnKq6KslZwJqq2hKcnga8v6pq6PKHAGcnmWKQ2Hzj8Oz8HWVAKkmStMRU1fnA+Vsde+1W+6/fxnWfAx4+7v4YkEqSJDVSpMVrnxYcnyGVJElSU2ZIJUmSGmqxdOhCY0AqSZLUSBVMzs1rnxYV74AkSZKaMkMqSZLUTJjCSU1mSCVJktSUGVJJkqRGCp8hBQNSSZKkpuZ7paaFyDsgSZKkpsyQSpIkNVKEKVdqMkMqSZKktsyQSpIkNeQzpAakkiRJzRQw5Sx7Q3JJkiS1ZYZUkiSpmTDpSk1mSCVJktSWGVJJkqRGfIZ0wDsgSZKkpsyQSpIkNeQzpAakkiRJzVTFIXscspckSVJjZkglSZIamjRDOv8Z0iSHJfl0kquTXJXklfPdB0mSJC0cLTKkm4E/qqovJdkbuCzJhVV1dYO+SJIkNVPAlJOa5j8graobgRu7z3ckuQY4BDAglSRJS0wcsqfxM6RJjgCOBy7pO2+XTHH/iY3Tln/5r47vbedv/6K3eo76lzN6ywEufupf9pa/dN2zR9bxwkM/21t+3u0P7y1/0qFfG9nGV+7Zvbf84Qff2Fu+fnL6+7zFQQfc3lt+e42uY4/73tVbvrE29ZYv33PzyDY21Yhzdu8vn2JqZBu16+hz+qx75tkc9a8vmlUdtbxmdT1ATcy+joXwB36NoQ/jqGPWMo7/PfrrOOaDL2btc86efTuSNAbNAtIkewH/BPx+Vf1UhJPkDOAMgEMO8S8H7ZxmG4xKO8pgVFoYBis1LYS/hNtqEukl2YVBMPreqvrwts6pqlVVtbKqVu63vwGpJEnSzmreM6RJArwLuKaq3jzf7UuSJC0kk74WvsmQ/WOB3wKuTHJ5d+y/VdX5DfoiSZLUTBGH7Gkzy/7fWRDTHyRJkrQQuFKTJElSQ1MO2XsHJEmS1JYZUkmSpEaqYNJnSM2QSpIkqS0zpJIkSQ05y96AVJIkqZnBa58csPYOSJIkqSkzpJIkSQ1N+np2M6SSJElqywypJElSI4WTmsCAVJIkqSEnNYFD9pIkSWrMDKkkSVJDU05qMkMqSZK01CQ5Ocm1SdYmOXMb5c9P8t0kl3fbC4fKTk9yXbedPo7+mCGVJElqpMVa9kkmgLcBJwHrgUuTrK6qq7c69QNV9bKtrt0PeB2wksGcrMu6a783mz6ZIZUkSWpoqpaNfRvhBGBtVa2rqnuA9wOnzLC7TwYurKrbuiD0QuDkHf7yHQNSSZKkpeUQ4FtD++u7Y1t7ZpIrkpyX5LDtvHa7GJBKkiQ1MljLfvwbcECSNUPbGdvZtX8Fjqiqn2OQBT133N99mM+QSpIk7XxuqaqV05RtAA4b2j+0O/ZjVXXr0O47gf81dO3jt7r24tl0FMyQSpIkNTVFxr6NcClwbJIjk6wATgNWD5+Q5OCh3acD13SfPw48Kcm+SfYFntQdmxUzpJIkSUtIVW1O8jIGgeQEcE5VXZXkLGBNVa0GXpHk6cBm4Dbg+d21tyX5HwyCWoCzquq22fbJgFSSJKmRVmvZV9X5wPlbHXvt0OdXA6+e5tpzgHPG2R8DUkmSpIZcy95nSCVJktSYGVJJkqRWfvKapiXNDKkkSZKaMkMqSZLUSMFMXtO00zMglSRJasghe4fsJUmS1JgZUkmSpEZavYd0oTFDKkmSpKbMkEqSJDVkhtSAVJIkqZnC95CCQ/aSJElqzAypJElSQ76H1AypJEmSGjNDKkmS1Eo5qQnMkEqSJKmxRZEhvfb2g/jli14+bfmx7/187/UXnLWit/yYf7xnZB8OP2Xv3vK1//fIkXU8+QX/3Fv+uK89pbf8rQ/9wMg2PvaDR/SWP2bf63vLr9x40Mg2HrjPd3vLb5oc/ZfeQfe5o7f8B1Obesv32uuukW1srM295Sv26G9jU02ObGPZiv5zppjqr2CXEeUzqKOW18g6RpqYfRU1Mbt+1Dj+PF4gSYbFkOw45oMvZu1zzu4/KWP4b2scdUg7MV+MP7AoAlJJ0niNDEYlzRsDUofsJUmS1JgZUkmSpEZ8Mf6AGVJJkiQ1ZYZUkiSpoTJDakAqSZLUkis1OWQvSZKkxsyQSpIkNVKu1ASYIZUkSVJjZkglSZIaclKTAakkSVJDvocUHLKXJElSY2ZIJUmSGnLI3gypJEmSGjNDKkmS1Ejha5/ADKkkSZIaM0MqSZLUSg1ejr/UGZBKkiQ15Fr2DtlLkiSpMTOkkiRJjRS+9gnMkEqSJKkxM6SSJEnNuHQoGJBKkiQ15Sx7h+wlSZLUmBlSSZKkhpzUZIZUkiRJjZkhlSRJaqTKDCkYkEqSJDXlLHuH7CVJktSYAakkSVJDg2H78W6jJDk5ybVJ1iY5cxvlf5jk6iRXJLkoyQOGyiaTXN5tq8dxDxyylyRJWkKSTABvA04C1gOXJlldVVcPnfZlYGVV3Znkd4H/BZzald1VVceNs09mSCVJkhqqyti3EU4A1lbVuqq6B3g/cMq9+1Sfrqo7u90vAIeO/YsPMSCVJElqpBh/MDqDgPQQ4FtD++u7Y9N5AfBvQ/u7JVmT5AtJnrFj3/zeHLKXJEna+RyQZM3Q/qqqWrW9lSR5HrAS+JWhww+oqg1JjgI+leTKqrp+Np01IJUkSWpojpayv6WqVk5TtgE4bGj/0O7YvSQ5EXgN8CtVtXHL8ara0P1cl+Ri4HhgVgGpQ/aSJElLy6XAsUmOTLICOA2412z5JMcDZwNPr6qbh47vm2TX7vMBwGOB4clQO8QMqSRJUisNVmqqqs1JXgZ8HJgAzqmqq5KcBaypqtXAXwB7AR9KAnBDVT0deAhwdpIpBonNN241O3+HGJBKkiQtMVV1PnD+VsdeO/T5xGmu+xzw8HH3x4BUkiSppTl6iHQxaRaQdi9lXQNsqKqnteqHJElSS/M9ZL8QtZzU9ErgmobtS5IkaQFoEpAmORR4KvDOFu1LkiQtFC3Wsl9oWg3Z/xXwX4G9Z3LybjdN8pA3/mDa8s2PeUTv9b/32elewzVw7GfX9JYDXHDXit7ywz5518g69nrRbr3ld31pv97y44+/e2Qbr1z/4N7ytz70A73lH/tB/70EePje63vLr7vnwJF1HL7X93vLb53q/1tpvz3u7C0H+GFt7i3ffbd7ess31eTINnbZrb+NUXVk+dTINkaZSR1T9J9TE2P412u2f94um30fxjLqtQBGzuZj9O6YD76Ytc85e+4bmq0swv/PKmm7zXuGNMnTgJur6rIR553RLUu15p7No4MPSdLMLYpgVFoCiiZr2S84LTKkjwWenuQpwG7AfZK8p6qeN3xSt7zVKoD77n6wfyJLkqSdTzE/wyIL3LxnSKvq1VV1aFUdwWBlgE9tHYxKkiRp6fA9pJIkSQ0txklI49Y0IK2qi4GLW/ZBkiRJbZkhlSRJaskMqQGpJElSO4tzVvy4tVypSZIkSTJDKkmS1JRD9mZIJUmS1NaMMqRJDgEeMHx+VX1mrjolSZK0JBQ+Q8oMAtIkbwJOBa4GtizMXYABqSRJkmZtJhnSZwAPqqqNc90ZSZKkJcdnSGcUkK4DdgEMSCVJksbOIfuZBKR3ApcnuYihoLSqXjFnvZIkSdKSMZOAdHW3SZIkadwcsh8dkFbVuUlWAA/sDl1bVZvmtluSJElaKmYyy/7xwLnANxk85HBYktN97ZMkSdIYmCGd0ZD9XwJPqqprAZI8EHgf8Ki57JgkSdJOrwDfQzqjlZp22RKMAlTV1xnMupckSZJmbSYZ0jVJ3gm8p9v/TWDN3HVJkiRp6SiH7GcUkP4u8FJgy2uePgv87Zz1SJIkSUvKTGbZbwTe3G2SJEkaJzOk0wekST5YVc9JciXbuFVV9XNz2jNJkqSlwElNvRnSV3Y/nzYfHZEkSdLSNO0s+6q6sfv4e1X1n8Mb8Hvz0z1JkqSdW2r822Izk9c+nbSNY7827o5IkiRpaep7hvR3GWRCj0pyxVDR3sB/zHXHJEmSdnqFk5rof4b0H4F/A/4ncObQ8Tuq6rY57ZUkSZKWjL6AtKrqm0leunVBkv0MSiVJkmYrzrJndIb0acBlDJLJw3ergKPmsF+SJElLg0P20wekVfW07ueR89cdSZIkLTUjZ9kneWySPbvPz0vy5iSHz33XJEmSloCag22Rmclrn94O3JnkEcAfAdcD/zCnvZIkSdKSMZOAdHNVFXAK8DdV9TYGr36SJEnSbJkh7Z3UtMUdSV4N/Bbwy0mWAbvMbbckSZKWgMJZ9swsQ3oqsBH4f6vqJuBQ4C/mtFeSJElaMkYGpF0Q+l7gvkmeBtxdVX8/5z2TJElaAlzLfmaz7J8DfBF4NvAc4JIkz5rrjkmSJGlpmMkzpK8Bfr6qbgZIciDwSeC8ueyYJEnSkrAIM5rjNpNnSJdtCUY7t87wOkmSJGmkmQSWFyT5eJLnJ3k+8DHg/LntliRJkuZKkpOTXJtkbZIzt1G+a5IPdOWXJDliqOzV3fFrkzx5HP0ZOWRfVX+c5NeBX+oOraqqj4yjcUmSpKVuvichJZkA3gacBKwHLk2yuqquHjrtBcD3quqYJKcBbwJOTfKzwGnAQ4GfAT6Z5IFVNTmbPk0bkCY5FvjfwNHAlcCrqmrDbBrbUbXxHibXfnPa8pvOO7b3+iPf1v/a1OWHHzqyD6/7Wn8b+33xmpF1XHnPXb3l97tsc2/5Xi/abWQbt1+3b2/5g477YW/5f7/liJFtnHnUv/WWf/nO0XUcucctveXf3nyf3vL773HHyDbumOov33u3jb3lGxn9u7Xriv7/zTaN+P2cWDG6jcka8UUmZv8vWcZQB8tmWcc4XsM3hn/Vd5bXAY76Hkd/6MVc/+yz56czkhaaE4C1VbUOIMn7GSyANByQngK8vvt8HvA3SdIdf39VbQS+kWRtV9/nZ9OhviH7c4CPAs8ELgP+v9k0JElaOAxGpQWkMv4NDkiyZmg7Y6jFQ4BvDe2v746xrXOqajPwA2D/GV673fqG7Peuqv/Tfb42yZdm25gkSZLmxS1VtbJ1J2aqLyDdLcnx/GQgbffh/aoyQJUkSZqNNmvPbwAOG9o/tDu2rXPWJ1kO3JfBm5Zmcu126wtIbwTePLR/09B+AU+YbeOSJElL3vwHpJcCxyY5kkEweRrwG1udsxo4ncGzoc8CPlVVlWQ18I9J3sxgUtOxDBZQmpVpA9Kq+tXZVi5JkqSFpao2J3kZ8HFgAjinqq5KchawpqpWA+8C/qGbtHQbg6CV7rwPMpgAtRl46Wxn2MPMVmqSJEnSHGmx9nxVnc9W75WvqtcOfb6bwbLx27r2DcAbxtkfV1ySJElSU2ZIJUmSWnIt+5kFpEkOAR4wfH5VfWauOiVJkrRkGJCODkiTvAk4lcHDq1seWi3AgFSSJEmzNpMM6TOAB3VLREmSJGlMUm0mNS00M5nUtA7oXwxekiRJ2kEzyZDeCVye5CLgx1nSqnrFnPVKkiRpqaiMPmcnN5OAdHW3SZIkadwcsh8dkFbVuUl2Bw6vqmvnoU+SJElaQkY+Q5rkvwCXAxd0+8d165hKkiRplrZMbBrnttjMZFLT64ETgO8DVNXlwFFz2CdJkiQtITN5hnRTVf0gudcDt1Nz1B9JkqSlZRFmNMdtJgHpVUl+A5hIcizwCuBzc9stSZIkLRUzGbJ/OfBQBq98+kfgduCVc9kpSZKkJWEOnh9djM+QziRD+tyqeg3wmi0HkrwROHPOeiVJkrRULMIActxmEpA+M8ndVfVegCR/A+w+t92SJEnSUjGjgBRYnWQKOBn4flW9YG67JUmStESYIZ0+IE2y39DuC4F/Bv4D+LMk+1XVbXPdOUmSJO38+jKklzGI2TP086ndVvguUkmSpFlbjJOQxm3agLSqjpzPjkiSJGlpGvkMaZJdgN8FHtcduhg4u6o2zWG/JEmStETMZFLT24FdgL/t9n+rO/bCueqUJEnSkuGQfe+kpuVVtRn4+ap6xFDRp5J8Ze67JkmSpKWgb6WmL3Y/J5McveVgkqOAyTntlSRJ0lLgSk1A/5B9up+vAj6dZF23fwTwO3PZKUmSpCVjEQaQ49YXkB6Y5A+7z2cDE93nSeB44NNz2TFJkiQtDX0B6QSwFz/JlA5fs/ec9UiSJGkpMUPaG5DeWFVnzVtPJEmStCTN5BlSSZIkzYGwOCchjVvfLPsnzlsvJEmStGT1LR1623x2RJIkaUkyQzqjlZokSZI0Fxbpe0PHrW/IXpIkSZpzTQLSJPskOS/J15Jck+QxLfohSZLUXM3Btsi0GrJ/K3BBVT0ryQpgj0b9kCRJUmPzHpAmuS/wOOD5AFV1D3DPfPdDkiRpQViEGc1xa5EhPRL4LvB3SR4BXAa8sqp+NN0Fmw/cg++c9vPTVvgfK9/S2+AzP9n/RMANr3p0bznAsov6y2vzupF1/MP3frG3fK+v3NhbfvvUXSPbuO/X+18fe8BEfzL6hg37j2zj6Af2v4DhH+7o/54AJ+//1d7yb95zYG/5/Xa9Y2Qbt07t1lu+76799/NHU1Mj29htl0295Zvor2P58tFtbGayt3zZDOqYrBHnTPT/azg14nsA1Ig6Rlo2hn+Rx/EQ0jhmFyyCtzgf/aEXc/2zz+49pxbB95gRZ4xogfM/0TbPkC4HHgm8vaqOB34EnLn1SUnOSLImyZrNd00bq0qSdsCoYFSS5lOLgHQ9sL6qLun2z2MQoN5LVa2qqpVVtXL57nvOawclSZLmjZOa5j8graqbgG8leVB36InA1fPdD0mSJC0MrWbZvxx4bzfDfh3wO436IUmS1M4izWiOW5OAtKouB1a2aFuSJGkhcVKTKzVJkiSpMdeylyRJaskMqRlSSZIktWVAKkmS1FBq/Nus+pPsl+TCJNd1P/fdxjnHJfl8kquSXJHk1KGydyf5RpLLu+24UW0akEqSJGnYmcBFVXUscBHbWMAIuBP47ap6KHAy8FdJ9hkq/+OqOq7bLh/VoAGpJElSSwvvxfinAOd2n88FnvFTXa76elVd133+NnAz0L/udw8DUkmSpFbmIhidfUB6UFXd2H2+CTio7+QkJwArgOuHDr+hG8p/S5JdRzXoLHtJkqSdzwFJ1gztr6qqVVt2knwSuP82rnvN8E5VVTL9U6lJDgb+ATi9qqa6w69mEMiuAFYBfwKc1ddZA1JJkqRG0m1z4JaqmnYRoqo6cbqyJN9JcnBV3dgFnDdPc959gI8Br6mqLwzVvSW7ujHJ3wGvGtVZh+wlSZI0bDVwevf5dOBftj6hW/79I8DfV9V5W5Ud3P0Mg+dPvzqqQQNSSZKklhbeM6RvBE5Kch1wYrdPkpVJ3tmd8xzgccDzt/F6p/cmuRK4EjgA+PNRDTpkL0mS1NBCW8u+qm4FnriN42uAF3af3wO8Z5rrn7C9bZohlSRJUlNmSCVJklpaYBnSFsyQSpIkqSkzpJIkSS2ZITUglSRJaqYW3qSmFhyylyRJUlNmSCVJkloyQ2qGVJIkSW2ZIZUkSWrIZ0jNkEqSJKkxM6SSJEktmSE1IJUkSWrJIXuH7CVJktSYGVJJkqRWCofsMUMqSZKkxsyQSpIktWSG1IBUkiSpleCkJnDIXpIkSY2ZIZUkSWrJDKkZUkmSJLVlhlSSJKmhlClSA1JJkqRWfA8p4JC9JEmSGjNDKkmS1JCvfTJDKkmSpMbMkEqSJLVkhnRxBKT7H3g7v/WSC6Yt/+Rd+/dev/x+B/SWn/CsK0b24dtnHN5/wiMePLKOD1+9orf86Bu+0lv+pXv2GNnGvtdu7C1fNiIpvuJbu45s46CJid7yr3/vwJF1vOj+t/aWf+L2h/eWH7Lr90e28d3J+/SW77Pirt7yO6r/ewLsueKe3vJNNdVbvmKXzSPbmBwx+3JieX8bAFMj/rXLstF1jJJls/sXtcYxXpOFUUfNto5xfI8Rjv7Qi7n+2WfPfUM7C8dUNYf8z8she0lakgxGJS0kiyJDKkmStNMyQ2qGVJIkSW2ZIZUkSWqlfIYUzJBKkiSpMTOkkiRJLZkhNSCVJElqJThkDw7ZS5IkqTEzpJIkSS2NWABlKTBDKkmSpKbMkEqSJDXkM6QGpJIkSe0UzrLHIXtJkiQ1ZoZUkiSpoUy17kF7ZkglSZLUlBlSSZKklnyG1IBUkiSpJWfZO2QvSZKkxsyQSpIktVK4UhNmSCVJkjQkyX5JLkxyXfdz32nOm0xyebetHjp+ZJJLkqxN8oEkK0a1aUAqSZLUUGr82yydCVxUVccCF3X723JXVR3XbU8fOv4m4C1VdQzwPeAFoxo0IJUkSdKwU4Bzu8/nAs+Y6YVJAjwBOG97rjcglSRJaqnmYJudg6rqxu7zTcBB05y3W5I1Sb6QZEvQuT/w/ara3O2vBw4Z1aCTmiRJkhoJc/bapwOSrBnaX1VVq37cbvJJ4P7buO41wztVVcm0PXxAVW1IchTwqSRXAj/Ykc4akEqSJO18bqmqldMVVtWJ05Ul+U6Sg6vqxiQHAzdPU8eG7ue6JBcDxwP/BOyTZHmXJT0U2DCqsw7ZS5IktVI1N9vsrAZO7z6fDvzL1ick2TfJrt3nA4DHAldXVQGfBp7Vd/3WDEglSZI07I3ASUmuA07s9kmyMsk7u3MeAqxJ8hUGAegbq+rqruxPgD9MspbBM6XvGtWgQ/aSJEkNLbSlQ6vqVuCJ2zi+Bnhh9/lzwMOnuX4dcML2tGlAKkmS1NICC0hbcMhekiRJTZkhlSRJamihDfpOWHoAABAxSURBVNm3YIZUkiRJTZkhlSRJaqWAKVOkBqSSJEktGY86ZC9JkqS2zJBKkiQ15KQmM6SSJElqzAypJElSS7Nfe37RM0MqSZKkppoEpEn+IMlVSb6a5H1JdmvRD0mSpNZS498Wm3kPSJMcArwCWFlVDwMmgNPmux+SJEnN1Rxti0yrIfvlwO5JlgN7AN9u1A9JkiQ1Nu+TmqpqQ5L/DdwA3AV8oqo+0XfN/SY28vv7Xj9t+cPe/tLeNnd9Tn+fPnDIW/pPAJ55xWN6y7/9qkePrGPPy/rLsyy95ef/4LiRbey67ube8tun7uot33PDyCbYa9muveW3fnfvkXUc8rA7esv/8879essfsv/ov2Fu2nTf3vL9Vvyot/yOqRUj29h7xcbe8rtHPKi+6/LNI9vYxFRv+cREfznA1Ig6MtHfz8ka3caoP29H9YFlY/iTfhzjVP2/hjOsY+GnJ47+0Iu5/tlnz3k7NY77Ke3EAsRJTU2G7PcFTgGOBH4G2DPJ87Zx3hlJ1iRZc8utk/PdTUnaqc1HMCpJM9ViyP5E4BtV9d2q2gR8GPjFrU+qqlVVtbKqVh6w/8S8d1KSJGleTM3Btsi0eA/pDcCjk+zBYMj+icCaBv2QJElqziH7BhnSqroEOA/4EnBl14dV890PSZIkLQxNVmqqqtcBr2vRtiRJ0oKxSF/TNG6u1CRJkqSmXMtekiSpmXItewxIJUmSmloEry6ecw7ZS5IkqSkzpJIkSS05ZG+GVJIkSW2ZIZUkSWqlIItwZaVxM0MqSZKkpsyQSpIkteQzpAakkiRJTRmPOmQvSZKktsyQSpIkNRSH7M2QSpIkqS0zpJIkSS2ZITUglSRJaqYA30PqkL0kSZLaMkMqSZLUSCgnNWGGVJIkSY2ZIZUkSWrJDKkBqSRJUlMGpA7ZS5IkqS0zpJIkSa342ifADKkkSZIaM0MqSZLUkK99MkMqSZKkxsyQSpIktWSG1AypJElSOzUISMe9zUKS/ZJcmOS67ue+2zjnV5NcPrTdneQZXdm7k3xjqOy4UW0akEqSJGnYmcBFVXUscFG3fy9V9emqOq6qjgOeANwJfGLolD/eUl5Vl49q0IBUkiSplWLBZUiBU4Bzu8/nAs8Ycf6zgH+rqjt3tEEDUkmSpJ3PAUnWDG1nbMe1B1XVjd3nm4CDRpx/GvC+rY69IckVSd6SZNdRDTqpSZIkqaW5eTH+LVW1crrCJJ8E7r+NotcM71RVJZk25ZrkYODhwMeHDr+aQSC7AlgF/AlwVl9nDUglSZIaavEe0qo6cbqyJN9JcnBV3dgFnDf3VPUc4CNVtWmo7i3Z1Y1J/g541aj+OGQvSZKkYauB07vPpwP/0nPuc9lquL4LYkkSBs+ffnVUgwakkiRJLS28SU1vBE5Kch1wYrdPkpVJ3rnlpCRHAIcB/3er69+b5ErgSuAA4M9HNeiQvSRJkn6sqm4FnriN42uAFw7tfxM4ZBvnPWF72zQglSRJaqWAKVdqMiCVJElqZixD7Iuez5BKkiSpqUWRIV1793156rVPm7b8iL++qvf63T/a/z7WW6Y29ZYDLNutv469n9D3RoSBPd90n97yiSMO7y3/xA27jWzj4Buv7y1ftzm95ff51uaRbSwb8XfMxHdXjKxjv4n+ftxwx08tm3sv9z/o+yPbuORHx/SW77/Lj3rLvz+1x8g29tplY2/5ndV/r3ZfMfq/vU3V/4K6XZZPjqxjcsRf38smxvASvOlfUzezy/v/k5hhJQukjlmqefgeR5/3Yq5/1tljaEgzNsvfEe3EzJCaIZWkpchgVNJCsigypJIkSTstM6RmSCVJktSWGVJJkqRWfO0TYEAqSZLUUMGIyatLgUP2kiRJasoMqSRJUktOajJDKkmSpLbMkEqSJLXipCbAgFSSJKkth+wdspckSVJbZkglSZJaMkNqhlSSJEltmSGVJElqpsyQYkAqSZLUTgFTrtTkkL0kSZKaMkMqSZLUkkP2ZkglSZLUlhlSSZKklsyQmiGVJElSW2ZIJUmSminXsseAVJIkqZ2CKl/75JC9JEmSmjJDKkmS1JJD9mZIJUmS1JYZUkmSpJZ87ZMBqSRJUjNVrmWPQ/aSJElqzAypJElSSw7ZmyGVJElSW2ZIJUmSGiqfITUglSRJaqccssche0mSJDVmhlSSJKmVwpWaMEMqSZKkxsyQSpIktVROajJDKkmSpKbMkEqSJDVSQPkMqQGpJElSM1UO2TOHQ/ZJzklyc5KvDh3bL8mFSa7rfu47V+1LkiRpcZjLZ0jfDZy81bEzgYuq6ljgom5fkiRpyaqpGvu22MxZQFpVnwFu2+rwKcC53edzgWfMVfuSJElaHOZ7lv1BVXVj9/km4KB5bl+SBBx93otbd0HSFjU1/m2RSc3h+qlJjgA+WlUP6/a/X1X7DJV/r6q2+RxpkjOAM7rdhwFf3dZ52iEHALe07sROwns5Xt7P8fJ+jo/3crwWyv18QFUd2LIDSS5gcD/G7Zaq2vrRyQVrvmfZfyfJwVV1Y5KDgZunO7GqVgGrAJKsqaqV89XJnZ33c3y8l+Pl/Rwv7+f4eC/Hy/v5E4spaJxL8z1kvxo4vft8OvAv89y+JEmSFpi5fO3T+4DPAw9Ksj7JC4A3AicluQ44sduXJEnSEjZnQ/ZV9dxpip64A9Wtmk1f9FO8n+PjvRwv7+d4eT/Hx3s5Xt5P3cucTmqSJEmSRpnvZ0glSZKke1nQAWmSk5Ncm2RtEld12k4u3zpeSQ5L8ukkVye5Kskru+Pe0+2UZLckX0zyle5e/ll3/Mgkl3S/8x9IsqJ1XxeTJBNJvpzko92+93MHJflmkiuTXJ5kTXfM3/UdkGSfJOcl+VqSa5I8xnuprS3YgDTJBPA24NeAnwWem+Rn2/Zq0Xk3Lt86TpuBP6qqnwUeDby0+2/Se7r9NgJPqKpHAMcBJyd5NPAm4C1VdQzwPeAFDfu4GL0SuGZo3/s5O79aVccNvZ7I3/Ud81bggqp6MPAIBv+Nei91Lws2IAVOANZW1bqqugd4P4OlRzVDLt86XlV1Y1V9qft8B4N/VA/Be7rdauCH3e4u3VbAE4DzuuPey+2Q5FDgqcA7u/3g/Rw3f9e3U5L7Ao8D3gVQVfdU1ffxXmorCzkgPQT41tD++u6YZsflW8egW4XseOASvKc7pBtevpzBAhkXAtcD36+qzd0p/s5vn78C/iuwZc3A/fF+zkYBn0hyWbdyIPi7viOOBL4L/F33OMk7k+yJ91JbWcgBqeZYDV6x4GsWtlOSvYB/An6/qm4fLvOezlxVTVbVccChDEZEHty4S4tWkqcBN1fVZa37shP5pap6JIPHxl6a5HHDhf6uz9hy4JHA26vqeOBHbDU8770ULOyAdANw2ND+od0xzc53umVbGbV8q35akl0YBKPvraoPd4e9p7PQDd99GngMsE+SLe9H9nd+5h4LPD3JNxk83vQEBs/teT93UFVt6H7eDHyEwR9N/q5vv/XA+qq6pNs/j0GA6r3UvSzkgPRS4NhulugK4DQGS49qdly+dQd1z+S9C7imqt48VOQ93U5JDkyyT/d5d+AkBs/kfhp4Vnea93KGqurVVXVoVR3B4N/KT1XVb+L93CFJ9kyy95bPwJOAr+Lv+narqpuAbyV5UHfoicDVeC+1lQX9YvwkT2HwXNQEcE5VvaFxlxaVbvnWxwMHAN8BXgf8M/BB4HDgP4HnVNXWE5+0DUl+CfgscCU/eU7vvzF4jtR7uh2S/ByDiQwTDP4w/mBVnZXkKAYZvv2ALwPPq6qN7Xq6+CR5PPCqqnqa93PHdPftI93ucuAfq+oNSfbH3/XtluQ4BpPtVgDrgN+h+73He6nOgg5IJUmStPNbyEP2kiRJWgIMSCVJktSUAakkSZKaMiCVJElSUwakkiRJasqAVFoikkwmuXxoO3P0VT++9vFJPjqLtqe9Psk3kxzQff7cjraxjfZ+0C1VeG2Sz3SrGW0pf0mS3x5HW9vZr5VJ/nq+25WkhW756FMk7STu6pbqXLCq6hfHWN1nq+pp8OP3IP5zkruq6qKqescY25mxqloDrGnRtiQtZGZIpSWuy1D+zy5ruibJI5N8PMn1SV4ydOp9knysyzi+I8my7vonJfl8ki8l+VCSvbrjJyf5WpIvAb8+1N7+ST6R5Kok7wQyVPbD7ufjk1yc5Lyujvd2K2WR5CndscuS/PVMMrdVdTlwFvCyro7XJ3lV9/niJG/pvvs1SX4+yYeTXJfkz4f69rwkX+zu09lJJrb0OckbknwlyReSHNQdf3aSr3bHPzP0vT7afd4vyT8nuaK77ueG+nZO1691SV7RHd+zu/9f6eo9dbv+h5akBcyAVFo6dt9qyH44oLmhy55+Fng3g+UmHw382dA5JwAvB34WOBr49W6o/U+BE6vqkQyyf3+YZDfg/wD/BXgUcP+hel4H/HtVPZTBajiHT9Pf44Hf79o7CnhsV+/ZwK9V1aOAA7fj+38JePA0ZfdU1UrgHQyWMHwp8DDg+V0A/RDgVOCx3X2aBH6zu3ZP4AtV9QjgM8CLuuOvBZ7cHX/6Ntr8M+DLVfVzDFb8+vuhsgcDT2Zwz1+XZBfgZODbVfWIqnoYcMF2fHdJWtAcspeWjr4h+9XdzyuBvarqDuCOJBvTrTkPfLGq1sGPl6X9JeBuBgHjf3QJzBXA5xkEVN+oquu6898DnNHV8zi6jGlVfSzJ96bp0xeran13/eXAEcAPgXVV9Y3unPcN1TtKesqGv/9VVXVj1+464LDuuz4KuLT7nrsDN3fX3ANsydJeBpzUff4P4N1JPgh8eBtt/hLwTICq+lQX+N6nK/tYt8TnxiQ3Awd1ffvLJG8CPlpVn53h95akBc+AVBLAlvXNp4Y+b9nf8u/E1usMF4Mg78Kqeu5wQffM5rj6BIOM5Gz/vToeuGZEW9N9/wDnVtWrt3HtpvrJGsw/7mdVvSTJLwBPBS5L8qjt6OtPffeq+nqSRwJPAf48yUVVddZ21ClJC5ZD9pJm6oQkR3bPjp4K/DvwBQZD6cfAj59zfCDwNeCIJEd31w4HrJ8BfqM7/9eAfbejD9cCRyU5otuf0XOU3fOZ/x1423a0Newi4FlJ7tfVt1+SB4xo8+iquqSqXgt8l0Gmddhn6Yb9kzweuKWqbu+p72eAO6vqPcBfAI/cwe8iSQuOGVJp6di9G/re4oKqmvGrn4BLgb8BjgE+DXykqqaSPB94X5Jdu/P+tMvmnQF8LMmdDIKvvbvyP+vOvwr4HHDDTDtQVXcl+T3ggiQ/6vo0nV9O8mVgDwbD66+oqotm2tZW7V6d5E+BT3QB+SYGz5n+Z89lf5HkWAbZ1YuArwC/MlT+euCcJFcAdwKnj+jGw7s6p7r2f3dHvoskLUT5yUiTJC18Sfaqqh92s+7fBlxXVW9p3S9J0o5zyF7SYvOiLtN7FXBfBrPuJUmLmBlSSZIkNWWGVJIkSU0ZkEqSJKkpA1JJkiQ1ZUAqSZKkpgxIJUmS1JQBqSRJkpr6/wGD6hOmmRYsKwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 864x576 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GdyrTa8vMZLs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}