training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Source code for ICLR Reproducibility Challenge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\gaha\\Workspace\\Repositories\\SNIP-pruning\\datasets.py:49: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  tmnist_x = TransformStandarize()(torch.tensor(mnist_full_x, device=DEVICE, dtype=torch.float))\n",
      "C:\\Users\\gaha\\Workspace\\Repositories\\SNIP-pruning\\datasets.py:50: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  tmnist_y = torch.tensor(mnist_full_y, device=DEVICE, dtype=torch.long)\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from torchvision import datasets\n",
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "DEVICE = 'cuda'\n",
    "\n",
    "from datasets import RamLoader\n",
    "from pruning import Pruner\n",
    "from models import LeNet5\n",
    "from network import NN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of weights to prune: [150, 6, 2400, 16, 48000, 120, 10080, 84, 840, 10]\n",
      "weights left:  [150, 6, 2385, 16, 40034, 107, 5982, 62, 613, 10]\n",
      "sparsities:  [0.0, 0.0, 0.62, 0.0, 16.6, 10.83, 40.65, 26.19, 27.02, 0.0]\n",
      "TRAINING:\n",
      "epoch      | time       | error rate   - mistakes...\n",
      "\n",
      "Epoch    0 | time 1.500 | lr 0.10000 | tr 100.000% | eval 95.100% - 8559 mis\n",
      "Epoch    5 | time 1.04e+02 | lr 0.10000 | tr   5.357% | eval  4.300% - 387 mis\n",
      "Epoch   10 | time 1.04e+02 | lr 0.10000 | tr   5.357% | eval  4.156% - 374 mis\n",
      "Epoch   15 | time 1.02e+02 | lr 0.10000 | tr   1.786% | eval  4.533% - 408 mis\n",
      "Epoch   20 | time 1.09e+02 | lr 0.10000 | tr   0.000% | eval  2.744% - 247 mis\n",
      "Epoch   25 | time 1.09e+02 | lr 0.10000 | tr   3.571% | eval  3.156% - 284 mis\n",
      "Epoch   30 | time 1.04e+02 | lr 0.10000 | tr   7.143% | eval  2.467% - 222 mis\n",
      "Epoch   35 | time 1.02e+02 | lr 0.10000 | tr   1.786% | eval  3.278% - 295 mis\n",
      "Epoch   40 | time 1.06e+02 | lr 0.10000 | tr   3.571% | eval  3.167% - 285 mis\n",
      "Epoch   45 | time 1.08e+02 | lr 0.01000 | tr   0.000% | eval  1.489% - 134 mis\n",
      "Epoch   50 | time 1.02e+02 | lr 0.01000 | tr   0.000% | eval  1.533% - 138 mis\n",
      "Epoch   55 | time 1.05e+02 | lr 0.01000 | tr   1.786% | eval  1.444% - 130 mis\n",
      "Epoch   60 | time 1.09e+02 | lr 0.01000 | tr   0.000% | eval  1.522% - 137 mis\n",
      "Epoch   61 | time 16.60 | lr 0.01000 | tr   2.344% | eval  1.533% - 138 mis\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4IAAAEGCAYAAAAwtyuGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3yV5f3/8dcnOyEhZAKywggJyBLCUEABF46KVVzVVq2ttVpH10/tcND222Ht8KtYcbbVurdV24qggEAFBQQSTNhLMggQyM65fn/kwBdJQk7GyTkneT8fj/M459z3dV/351wEwudcy5xziIiIiIiISNcRFugAREREREREpGMpERQREREREelilAiKiIiIiIh0MUoERUREREREuhglgiIiIiIiIl1MRKADaKnU1FSXkZHR6us9Hg8AYWHKgY+mdmlIbdKQ2qQhtUlD7dkmK1euLHbOpbW5oi6irb8jQ5n+Lrac2qx11G6to3ZrnabarT1+P4ZcIpiRkcGKFStafX1FRQUAsbGx7RVSp6B2aUht0pDapCG1SUPt2SZmtrXNlXQhbf0dGcr0d7Hl1Gato3ZrHbVb6zTVbu3x+1EpuYiIiIiISBejRFBERERERKSLUSIoIiIiIiLSxYTcHEEREREREQkONTU17Nixg8rKyuOWc84BYGYdEVanERkZSc+ePf1StxJBERERERFplR07dpCQkEBGRsZxkzytGtpyzjmKiorYs2cPCQkJ7V6/3/4kzOwJMys0s7VNnDcze8DMCsxsjZmN9VcsIiIiIiLS/iorK0lJSVFPnx+YGSkpKVRXV/ulfn+m5E8BM49z/hwg0/u4HnjYj7GIiIiIiIgfKAn0H3+2rd+GhjrnPjSzjOMUmQX8zdUPGF5mZj3MrLdzbvfx6vV4PEf202iN5sYvd1Vql4bUJg2pTRpSmzSkNvGNmYUDK4Cdzrnzjzl3DXAfsNN76EHn3GMdG2HXsXBDIanx0YzokxjoUEREOkwgB+n2AbYf9X6H91gDZna9ma0wsxXFxcUdEpyIiIif3QrkHuf88865Md6HkkA/qK71cNfra7nmyY+545U1gQ5HRAIsIyOD1uYar732GuvXrz/y/q677uK9995rr9D8IpCLxTTWz+kaK+icmwfMA8jJyXGxsbFtvnl71NEZqV0aUps0pDZpSG3SkNqkaWbWFzgP+BXwg/aqt62jZkJZS3uivzhQyW0vrmX1jgMMTuvGup0H2FWyn6S4KD9FGHzUe986arcvc84dWQjmeHwpEww8Hk+rYn311Vc577zzyM7OBuCee+45Ul9b43HO+eXf9kD2CO4A+h31vi+wK0CxiIiIdKQ/Af8PON7/EC72Lqb2kpn1a6qQRs203PLNpVz8yMfkFx7iT5eM4FcXZOOApZtKAx2aiLTS008/zaRJkxg7diw33HADc+fO5fbbbz9y/qmnnuKWW24B4Ktf/Srjx49n5MiRzJs3r0FdW7ZsYdSoUUfe33///dx7770APProo0ycOJGTTjqJ2bNnU15ezkcffcSbb77J7bffztixY9m4cSPXXnstL730EgDz589n3LhxjB49muuuu46qqioABg0axD333ENOTg6jR48mLy/Pb+3TmED2CL4BfM/MngMmAvubmx8oIiIS6szsfKDQObfSzKY1UexN4FnnXJWZ3QD8FZjRWEF/jJoJZcf7/M45HvlwE797N49BafH85apxDEmPp87j6B4TwX+3HuDi8RkdF2yQ6Oo/M62ldqtnZke2hLj3zXWs33WgXesffkJ37v7Kicctk5uby4svvsiSJUuIjIzkxhtvJCEhgVdffZX77rsPgBdffJGf/vSnhIWF8eSTT5KcnExFRQXjx4/nkksuISUlBajf3uLw5zn8bGZHPufs2bP5zne+A8DPfvYznnzySW6++WYuuOACzj//fGbPnv2ldqmuruab3/wm8+fPZ+jQoXzjG9/gkUce4bbbbgMgLS2NTz75hLlz5/KHP/yBxx5rOBPAzPzy8+bP7SOeBZYCWWa2w8yuM7MbvL/QAN4GNgEFwKPAjf6KRUREJIhMBi4wsy3Ac8AMM3v66ALOuRLnXJX37aPAuI4NsfMpq6zhhqdX8pt38jhnZG9ev2kyQ9LjAQgPM04ZnMqi/KIjm16LSOiYP38+K1euZPz48YwZM4b58+ezefNmBg0axLJlyygpKWHDhg1MnjwZgAceeIDRo0czadIktm/fTn5+vs/3Wrt2LVOnTmXkyJE888wzrFu37rjlN2zYwMCBAxk6dCgAV199NR9++OGR8xdddBEA48aNY8uWLS385G3jz1VDr2jmvANu8tf9RUREgpFz7k7gTgBvj+CPnHNXHV3mmFW0L+D4i8pIMz7fU8YNf1/J1r3l/Oy8YVw3ZWCDJdmnDk3l3XVfsKn4EIPT4gMUqUhoO17PnT83lHfOcfXVV/PrX//6S8cff/xxXnjhBbKzs/nqV7+KmbFw4ULee+89li5dSlxcHNOmTWsw7zMiIuJLc/uOPn/NNdfw2muvMXr0aJ566ikWLlzYbGzHEx0dDUB4eDi1tbW+fNx2E8g5giIiIuJlZnPM7ALv21vMbJ2ZrQZuAa4JXGSh7fVVO5n14BLKqmr5x7cm8q2pgxrdl2vqkDQAFudrnqVIqDn99NN56aWXKCwsBGDv3r1s3bqViy66iNdee41nn32Wyy67DID9+/eTlJREXFwceXl5LFu2rEF9PXv2pLCwkJKSEqqqqnjrrbeOnCsrK6N3797U1NTwzDPPHDmekJBAWVlZg7qys7PZsmULBQUFAPz973/ntNNOa9fP31pKBEVERALEObfw8B6Czrm7nHNveF/f6Zw70Tk32jk33TnXsSsIdALVtR7ueWMdtz63ihF9uvPPm6cwcVBKk+X7p8QxICWORflFHRiliLSH4cOH88tf/pKzzjqLUaNGceaZZ7J7926SkpIYPnw4W7duZcKECQDMnDmT2tpaRo0axc9//nMmTZrUoL7IyEjuuusuJk6cyPnnn39kJVCAX/ziF0ycOJEzzzzzS8cvv/xy7rvvPk466SQ2btx45HhMTAxPPvkkl1xyCSNHjiQsLIwbbriBYGChNhY+JyfHrVixotXXH156VRN8v0zt0pDapCG1SUNqk4bas03MbKVzLqfNFXURbf0dGcqO/rnbc6CSm575hBVbS/nm5IHceW42keHNf/f901c/47VPd7Lq7rN8Kh/q9O9X66jdviw3N5dhw4Y1W86fQ0M7M4/HQ15eHsOHD//S8fb4/ag/CREREek0lm0q4bwHFrN+9wH+94qTuOsrw31O6qZmpnKouo5Pt+3zc5QiIoGnRFBERERCnnOOJz/axpWPLad7bASv3TSZr4w+oUV1nDw4lTCDxRoeKiJdgBJBERERCWkHq2r5/ktr+d1/CjhzWE9ev2kyQ3smtLiexNhIRvfrwaICLRgjIp2fEkEREREJWQWFZcx6cDH/yS3iR2cM5uGrxpIQE9nq+qYOSWX19n3sL69pxyhFRIKPEkEREREJSf9cs5tZDy5hf0UNT3z9JK6bPKDRrSFaYurQNDwOlm5Sr6CIdG5+21BeREREQsv/zs+nW3QEpw9LZ0BKt0CH06SaOg+/fSePxxZvZmz/Hsy9chyJUe2zCvqYfj2Ij45gUX4xM0f0bpc6RUSCkXoERUREBOcc728oZM5b6zntvoWcfv9Cfv12Lss3lVBb5wl0eEcUllVy5WPLeWzxZq4+eQDPXX8yvRJj2q3+yPAwJg1KZpE2lhcJGfv27WPu3Lktvu7cc89l377jrxJ811138d5777U2tKCmHkERERHBzHj1xslsLTnE+3mFvJ9XyBNLNvPIh5voHhPBtKx0Th+WzrSh6STGtX4OXlt8vGUvNz3zCQcqa/jTZWO48KQ+frnP1Mw03sstZFtJOf1T4vxyDxFpP4cTwRtvvPFLx+vq6ggPD2/yurfffrvZuufMmdPm+IKVEkERERE5YkBKN66dPJBrJw+krLKGxfnFzM8rZEFeIW+s3kV4mDFuQBKnZ6dz+rCeDE7r1uZ5ec1xzvHkki38z9u59E2K5W/XTSC7V3e/3W9KZioAiwqKuDJlgN/uIyLt44477mDjxo2MGTOGyMhI4uPj6d27N6tWrWL9+vVceOGFbN++ncrKSm699Vauv/56ADIyMlixYgUHDx7knHPOYcqUKXz00Uf06dOH119/ndjYWK655hrOP/98Zs+eTUZGBldffTVvvvkmNTU1vPjii2RnZ1NUVMTXvvY1SkpKGD9+PO+++y4rV64kNTU1wC1zfEoERUREpFEJMZGcM7I354zsjcfjWLVjH+/nFjI/r5Bfv5PHr9/JY0BKHDOy0zljWE/GZyQTFdG+s04OVdVyxyuf8ebqXZw5vCf3Xzqa7m1YFdQXg1K70adHLIs+L+bKiUoERXz2zh3wxWeNnjLckVct0msknPOb4xb5zW9+w9q1a1m1ahULFy7kvPPOY+3atQwcOBCAJ554guTkZCoqKhg/fjwXX3wxKSkpX6ojPz+fZ599lkcffZRLL72Ul19+mauuuqrBvVJTU/nkk0+YO3cuv//973nssce49957mTFjBnfeeSfvvvsu8+bNa9lnDBAlgiIiItKssDBjbP8kxvZP4kdnZ7FzX0X9ENLcPTyzfBtPLtlCfHQEpw5NZUZ2T6ZnpZESH92me24sOsgNf1/JxqKD/PjsLL572mDCwvzb+wj1w2SnDEnlnbW7qa3zEBGuJRVEQsmECROOJIEADzzwAK+++ioA27dvJz8/v0EiOHDgQMaMGQPAuHHj2LJlS6N1X3TRRUfKvPLKKwAsXrz4SP0zZ84kKSmpXT+PvygRFBERkRbr0yOWr08awNcnDaC8upYlBSW8n7eH+bmFvP3ZF5jBSf16cPqwnpw+LJ2sngktGkL67trd/OjFNURFhPG3b048Mlyzo0zJTOX5FdtZs3M/Y/uHxn/qRALuOD13zlO/6JSF+f+LlW7d/m/V44ULF/Lee++xdOlS4uLimDZtGpWVlQ2uiY7+vy+uwsPDqaioaLTuw+XCw8Opra0F6oevhyIlgiIiItImcVERnDm8J2cO74nH41i36wDz8/bwfl4h9/1rA/f9awN9esQyI7t+wZlJg1KIiWx8AYfaOg/3/XsDj3ywidH9evDwlWM5oUdsB38imDwkFTNYnF+sRFAkyCUkJFBWVtbouf3795OUlERcXBx5eXksW7as3e8/ZcoUXnjhBW6//Xb+/e9/U1pa2u738AclgiIiItJuwsKMkX0TGdk3kdvOGErhgUoWbCjkvdxCXlq5g78v20psZDhTMlM5PTudGdnppHev3/6hqKyKW579lKWbSrhyYn/u+spwoiOaXvHPn5K7RTHihEQW5xdzy+mZAYlBRHyTkpLC5MmTGTFiBLGxsfTs2fPIuZkzZ/KXv/yFUaNGkZWVxaRJk9r9/nfffTdXXHEFzz//PKeddhq9e/cmISGh3e/T3izUujJzcnLcihUrWn394W7e2NiO/3YxmKldGlKbNKQ2aUht0lB7tomZrXTO5bS5oi6irb8j/a2ypo5lm0qYn1u/PcXOffU/K6P6JjJlSCqvfLKT0vJqfvXVkcwe17dFdfvj7+Jv383j0Q83serus4iP7nzfnevfr9ZRu31Zbm4uw4YNa7acxzs0NKwDhoZ2tKqqKsLDw4mIiGDp0qV897vfZdWqVe1St8fjIS8vj+HDh3/peHv8fux8/6qJiIhIUIqJDGdaVjrTstKZ4xwb9pQxP7eQ+bl7ePiDjfRLiuOVG0/hxBMSAx0qAFMzU3l44UaWbSzhjOE9m79ARLqkbdu2cemll+LxeIiKiuLRRx8NdEg+USIoIiIiHc7MyO7Vnexe3blp+hD2V9QQFxVOZBCt0DluQBKxkeEsLihWIigiTcrMzOTTTz8NdBgtpkRQREREAi4x1r97A7ZGdEQ4Ewcl82F+UaBDEQlqzrkWrQosvvPnNL7g+dpNREREJMhMGZLKpqJDR+YzisiXxcTEUFJSErJbKAQz5xwlJSVERUX5pX71CIqIiIg0YWpmGpDL4vwiLhvfP9DhiASdvn37smPHDoqKjt9zfjhRVM9hy0RGRn5pFdT2pERQREREpAlDe8aTnhDNovxiJYIijYiMjGTgwIHNltNqq63T1Mb27UFDQ0VERESaYGZMyUxlSUExHo+GvolI56FEUEREJADMLNzMPjWztxo5F21mz5tZgZktN7OMjo9QDpuamUppeQ3rdh0IdCgiIu1GiaCIiEhg3ArkNnHuOqDUOTcE+CPw2w6LShqYPCQVgEUFWj1URDoPzREUERHpYGbWFzgP+BXwg0aKzALu8b5+CXjQzMw1syyfx+Px63ySYFZZWem3uhMiIKtnPB9uKOTaiX38dp+O5s8268zUbq2jdmsdf7abegRFREQ63p+A/wd4mjjfB9gO4JyrBfYDKY0VNLPrzWyFma0oLi72R6wCTB6UzMpt+6ioqQt0KCIi7UI9giIiIh3IzM4HCp1zK81sWlPFGjnWaG+gc24eMA8gJyfHdfUV+fz1+acN68UTS7exZnc507LS/XKPQOnqPzOtpXZrHbVb6/ij3dQjKCIi0rEmAxeY2RbgOWCGmT19TJkdQD8AM4sAEoG9HRmkfNmEgclERYSxOF+9riLSOfg1ETSzmWa2wbvq2R2NnO9vZgu8q6atMbNz/RmPiIhIoDnn7nTO9XXOZQCXA+875646ptgbwNXe17O9ZbR3QQDFRIYzISOZRUoERaST8FsiaGbhwEPAOcBw4AozG35MsZ8BLzjnTqL+l+Fcf8UjIiISzMxsjpld4H37OJBiZgXULybT4MtU6XhTMlPZsKeMwgNa9EJEQp8/5whOAAqcc5sAzOw56ldBW39UGQd0975OBHY1V2lbV0TTikWNU7s0pDZpSG3SkNqkIbWJ75xzC4GF3td3HXW8ErgkMFFJU6Yc3kYiv5iLx/UNcDQiIm3jz6GhR1Y889rhPXa0e4CrzGwH8DZwc2MVaUU0ERERCbThvbuT0i2KxQX6v4iIhD5/9gj6suLZFcBTzrn7zexk4O9mNsI596XltP2xIppWLGqc2qUhtUlDapOG1CYNqU2kswkLMyYPSWVRfjHOOcwa+6+OiEho8GeP4JEVz7z60nDo53XACwDOuaVADJDqx5hEREREWm1KZirFB6vI+6Is0KGIiLSJPxPBj4FMMxtoZlHULwbzxjFltgGnA5jZMOoTwSI/xiQiIiLSalMz67+v1jYSIhLq/JYIOudqge8B/wJyqV8ddN0xq6L9EPi2ma0GngWu0fLYIiIiEqx6J8YyJD2eRZonKCIhzp9zBHHOvU39IjBHHzt6VbT11G+sKyIiIhISpmam8o/l26isqSMmMjzQ4YiItIpfN5QXERER6WymZqZSVeth5dbSQIciItJqSgRFREREWmDiwBQiw41FmicoIiFMiaCIiIhIC3SLjmBs/yQW5Wt9OxEJXUoERURERFpoamYq63YdoORgVaBDERFpFSWCIiIiIi00JTMNgMVaPVREQpQSQREREZEWGtknkcTYSO0nKCIhS4mgiIiISAuFhxmTh6SwuKAYbYEsIqFIiaCIiIhIK0zNTGP3/ko2Fh0MdCgiIi2mRFBERESkFaYMSQXQNhIiEpKUCIqIiIi0Qr/kODJS4jRPUERCkhJBERERkVaampnG0k0lVNd6Ah2KiEiLKBEUERERaaUpmamUV9fx6bbSQIciItIiSgRFREREWunkwSmEh5nmCYpIyFEiKCIiItJK3WMiGdOvB4u0sbyIhBglgiIiIiJtMGVIKp/t2Me+8upAhyIi4jMlgiIiIiJtcOrQVDwOPtpYEuhQRER8pkRQREREpA1G9+1BQnSE5gmKSEhRIigiItKBzCzGzP5rZqvNbJ2Z3dtImWvMrMjMVnkf3wpErOKbiPAwJg1OYVF+Ec65QIcjIuITJYIiIiIdqwqY4ZwbDYwBZprZpEbKPe+cG+N9PNaxIUpLnZqZyo7SCraWlAc6FBERn0QEOgAREZGuxNV3GR30vo30PtqlG8nj8VBRUdEeVYWcysrKgN4/p18CAO+v38UV4/sGNBZfBbrNQpXarXXUbq3jz3ZTj6CIiEgHM7NwM1sFFAL/cc4tb6TYxWa2xsxeMrN+x6nrejNbYWYrios1Ry1QBiTHckJiDB9t0sbyIhIa1CMoIiLSwZxzdcAYM+sBvGpmI5xza48q8ibwrHOuysxuAP4KzGiirnnAPICcnBwXGxvr5+iDWyA//2lZaby1ejeRUdFEhIfOd+1d/WemtdRuraN2ax1/tFvo/CslIiLSyTjn9gELgZnHHC9xzlV53z4KjOvg0KQVpgxJo6yqltU79gc6FBGRZikRFBER6UBmlubtCcTMYoEzgLxjyvQ+6u0FQG7HRSitNXlICmawKL8o0KGIiDRLiaCIiEjH6g0sMLM1wMfUzxF8y8zmmNkF3jK3eLeWWA3cAlwToFilBXrERTGqTyKLtZ+giISAZucImtnvgF8CFcC7wGjgNufc036OTUREpNNxzq0BTmrk+F1Hvb4TuLMj45L2MSUzlb98sImyyhoSYiIDHY6ISJN86RE8yzl3ADgf2AEMBX7s16hEREREQtDUzDTqPI6lG0sCHYqIyHH5kgge/jrrXOpXMNvrx3hEREREQtbY/knERYWzuEDDQ0UkuPmyfcSbZpZH/dDQG80sDdCOkCIiIiLHiIoIY+LAZM0TFJGg12yPoHPuDuBkIMc5VwMcAmb5OzARERGRUDQ1M41NxYfYUVoe6FBERJrUbCJoZpcAtc65OjP7GfA0cIIvlZvZTDPbYGYFZnZHE2UuNbP13tXR/tGi6EVERESCzNTMVAD1CopIUPNljuDPnXNlZjYFOBv4K/BwcxeZWTjwEHAOMBy4wsyGH1Mmk/pV0SY7504Ebmth/CIiIiJBZUh6PD27R7NI8wRFJIj5Mkewzvt8HvCwc+51M7vHh+smAAXOuU0AZvYc9UNK1x9V5tvAQ865UgDnXGFzlXo8HioqKny4feMqKzW9sTFql4bUJg2pTRpSmzSkNpGuzsyYmpnGe7l7qPM4wsMs0CGJiDTgS4/gTjN7BLgUeNvMon28rg+w/aj3O7zHjjYUGGpmS8xsmZnNbKwiM7vezFaY2YriYn27JiIiIsFtamYq+8prWLdrf6BDkWNU1tSxv7wm0GGIBJwvPYKXAjOB3zvn9plZb3zbR7Cxr79cI/fPBKYBfYFFZjbCObfvSxc5Nw+YB5CTk+NiY2N9uP3xtUcdnZHapSG1SUNqk4bUJg2pTaQrmzykfp7govxiRvXtEeBo5DCPx3HlY8tZubWUwWndGDcgiZwByYzLSGJQajfM1HsrXUeziaBzrtzMNgJnm9nZwCLn3L99qHsH0O+o932BXY2UWeZdjXSzmW2gPjH82KfoRURERIJQanw0w3t3Z1F+ETdNHxLocMTrxZXbWbm1lEvG9WXvoWr+vX4PL6zYAUBSXCTjBiQxbkAy4wYkMapvIjGR4QGOWMR/mk0EzexW6ufyveI99LSZzXPO/W8zl34MZJrZQGAncDnwtWPKvAZcATxlZqnUDxXd1IL4RURERILS1MxUnliymfLqWuKifBmEJf60r7ya3767gZwBSfxu9ijMDOccG4sOsXLrXlZuLWXF1lLey61fsiIy3BjRJ5Fx/ZPIyahPENMSogP8KUTajy//Kl0HTHTOHQIws98CS4HjJoLOuVoz+x7wLyAceMI5t87M5gArnHNveM+dZWbrqV+U5sfOuZLWfxwRERGR4DA1M41HPtzE8k17mZ6dHuhwurz7//05+8qrmTNr4pEhoGbGkPR4hqTHc9n4/gDsPVTNyq2l3sde/rZsK48t3gxA/+Q4xvTtzth+iUzKTGdoegJhWgxIQpQviaDxfyuH4n3t00+8c+5t4O1jjt111GsH/MD7EBEREek0cjKSiI4IY1F+sRLBAFu7cz/PLN/K1ycNYPgJ3Y9bNrlbFGcO78mZw3sCUFVbx7pdB1i5pZQVW/eyZGMJb6z5AthAQkwEY/sneecaJjGmfw/1/krI8OUn9UlguZm96n1/IfC4/0ISERERCX0xkeFMGJjM4oKiQIfSpXk8jrteX0tSXBQ/OCurxddHR4Qztn8SY/sn8W0GUV5ezvbSCtZ+UcHKbaWs3FLKH9/7HOcgPMwY1juhfgGaAfUJ4gk9tHCWBCdfFov5g5ktBKZQ3xN4rXPuU38HJiIiIhLqpmam8j9v5/HF/kp6JcYEOpwu6eVPdvDJtn38bvYoEmMj21yfmdE/OY6sPilcPK4vAPsravh0W/1w0hVbSnn+4+089dEWAE5IjGFcRjLj+vcgJyOZ7F4JRIT7shObiH81mQiaWfJRb7d4H0fOOef2+i8sERGR0GJmSUA/59yaQMciwWPKkDQgj8UFxcz2Jg3BYkdpBWFhxuBOvNXL/ooafvNOHif178Hssf5r/8TYSKZlpTMtq34IcE2dh7zdZazYupcVW0v5ePNe3lxdv3j+2Sf25JGv5/gtFhFfHa9HcCX1+/4dng94eA9A874e5Me4REREgp53xMwF1P8+XQUUmdkHzjnNfRcAsnslkBofzaL8oqBKBN9d+wXff/5TEmIi+Ndtp5HULSrQIfnFH//zOXvLq/nrNyd06KIukeFhjOybyMi+iVw7eSAAO/dV8Of3PufFlTvYe6ia5E7a5hI6muyXds4NdM4N8j4ffn34vZJAERERSHTOHQAuAp50zo0DzghwTBJEwsKMKUNSWFJQjMfjmr/Az5xzPPLBRr77zEoyUuIoPVTDT179jPr1+zqX9bsO8LelW7hyYn9G9EkMdDj06RHL1yYOwDlYlK95oxJ4GqAsIiLSehFm1hu4FHgr0MFIcJqSmUbxwWryvigLaBzVtR7uePkzfv1OHueN7M0/vjmOW2cM4p21X/DSyh0Bja29Oee4+421JMZG8qNWLBDjL6P6JJLSLYoFeYWBDkVEiaCIiEgb3Ev9nrgFzrmPzWwQkB/gmCTITM1MBQLbC7S/vIarn/gvz6/Yzs0zhvDA5ScRE1wj1VkAACAASURBVBnONSf3Z9KgZO55Yx3bSsoDFl97e/XTnXy8pZTbZ2bTIy54hmCGhRmnDU3jg8+LqAuCHmLp2pQIioiItIKZhVO/OMwo59yNAM65Tc65iwMcmgSZnt1jGNoznsUFxQG5/5biQ3x17hJWbi3lD5eO5odnZR2ZLxceZtx/6RjCwozvv7CK2jpPQGJsTwcqa/ift/MY3a8Hl+b0C3Q4DZyWlUZpeQ2rd+wLdCjSxTWbCJpZciOPtq+9KyIiEsKcc3XULxQj0qwpQ9L47+a9VNbUdeh9l28q4cK5Sygtr+aZb0/kokZWzuzTI5ZfXjiClVtLeXjhxg6Nzx/+9J98Sg5V8YtZJ3boAjG+OjUzjTCDhRoeKgHmS4/gJ0AR8Dn1w12KgM1m9omZjfNncCIiIkHuIzN70MymmtnYw49AByXBZ+rQVKpqPXy8peN233pp5Q6uenw5Kd2ieO2myYzPSG6y7KwxfZg15gT+ND+fVdtDt6cq74sD/HXpFi4f359RfXsEOpxGJXWL4qT+SSzYoAVjJLB8SQTfBc51zqU651KAc4AXgBuBuf4MTkREJMidApwIzAHu9z5+39xFZhZjZv81s9Vmts7M7m2kTLSZPW9mBWa23Mwy2jl26UATByYTFR7G4nz/Dw/1eBy/ezePH724mgkDk3nlxskMSOnW7HVzZo2gV/cYvv/8Ksqra/0eZ3tzznHX6+tIiIng/50dPAvENGZ6Vhqf7dxPUVlVoEORLsyXRDDHOfevw2+cc/8GTnXOLQOi/RaZiIhIkHPOTW/kMcOHS6uAGc650cAYYKaZTTqmzHVAqXNuCPBH4LftG710pLioCMYNSGKRnxPBiuo6vvfsJ8xduJErJvTnqWsnkBjr24yexNhI7r90NFtKDvGLt3L9Gqc/vLF6F//dvJcfn50V9PsiHt54/oPP1SsogXO8DeUP22tmtwPPed9fBpR6J8mH/oxiERGRVjKzROBu4FTvoQ+AOc65/ce7ztVv2nbQ+zbS+zh2CcFZwD3e1y8BD5qZueNs+ObxeKioqGjRZ+gsKisrAx1CsyZlJPLH9zexvWg/qfHtn6gUllXxvefWsHZXGbefNYSrJ/WjtrqKpvr2Gmuz0b3juO6U/jy2ZBtTBiUyIyut3eP0h4NVtfzyrfWc2DuBWSPS/Pr3oD1+1gYlRZIWH8V763dz3vCUdogq+IXC39Fg5M9286VH8GtAX+A14HWgv/dYOPX7JomIiHRVTwBl1P8+vBQ4ADzpy4VmFm5mq4BC4D/OueXHFOkDbAdwztUC+4EG/2M0s+vNbIWZrSguDsyqlOKbUwbXz9Fbtrn95wnmfVHG5Y+tYGNROQ9ePpJrTu6PWesWSrl5+iCG9YrnZ2/kUXQwNIYuzv1gM0UHq/n5uUMJD8IFYo5lZkwdksKSgr3UetSvIoHRbI+gc64YuLmJ0wXtG46IiEhIGXzMdhH3epO7ZnlXHR1jZj2AV81shHNu7VFFGvvfbIPeQOfcPGAeQE5OjouNjfU9+k4omD//2IExJMVFsmzLAS6ZMLDd6n0/bw83/+NTEmIieeGGkxnRJ7FF1x/bZrHAA1eM5fz/Xczdb33OE9eMb3VS2RHy95Tx9+U7uCynH5Mye3XYfdv6s3bmib15ZdVucgurmDCw6YV8Optg/jsazPzRbr5sHzHUzOaZ2b/N7P3Dj3aPREREJPRUmNmUw2/MbDLQojFpzrl9wEJg5jGndgD9vPVGAIlAxy05Ke0uPMw4ZUgqiwuKOM4IX58553hi8Wa+9dcVDEqL5/XvTW5xEtiUzJ4J/OTcYSzYUMTTy7e1S53+cHiBmLiocP7fzOBeIOZYkzNTiQgzFmzQNhISGL4MDX0R+BT4GfDjox4iIiJd3Q3AQ2a2xcy2AA8C32nuIjNL8/YEYmaxwBlA3jHF3gCu9r6eDbx/vPmBEhqmDkllz4EqCgoPNl/4OGrrPPz89bXMeWs9ZwzryfPfmUTP7jHtFGW9b5w8gNOGpvGrf65vc7z+8taa3SzdVMKPz84iJT601jDsHhNJTkYSC7SfoASIL4lgrXPuYefcf51zKw8//B6ZiIhIEDOzMCDLu/LnKGCUc+4k59waHy7vDSwwszXAx9TPEXzLzOaY2eFN6h8HUsysAPgBcIcfPoZ0sCmZqQB82IbVQw9U1nDtUx/z9LJtfOe0QfzlqnHERfmy/l/LmBn3zR5FbGQ4tz3/KdW1wTWX7VBVLb/6Zy4nntCdr00cEOhwWmV6Vjp5X5Sxe3/XXORJAsuXRPBNM7vRzHqbWfLhh98jExERCWLOOQ/wPe/rA865Ay24do03aRzlnBvhnJvjPX6Xc+4N7+tK59wlzrkhzrkJzrlNfvkg0qH6JsUxKLUbi/Nbt23A9r3lXDz3I5ZuLOG3F4/kznOGEebHxVHSu8fwm4tHsXbnAf48/3O/3ac1Hng/ny8OVDJn1oiQWCCmMdOz67eRWKjN5SUAfEkEr6Z+KOhHwErvY4U/gxIREQkR/zGzH5lZP31ZKr6akpnKsk17qaqta9F1K7fu5cKHllBYVsXfrpvAZeP7+ynCLzv7xF5cPr4fcxdu5L9+WPG0NQoKD/L4os3MHteXcQOSAh1Oq2Wmx9OnR6yGh0pANJsIOucGNvIY1BHBiYiIBLlvAjcBH6IvS8VHUzPTqKip45Ot+3y+5vVVO7ni0eUkxETw6o2ncMrgVD9G2NDPzx9O/+Q4vv/8Kg5U1nTovY/lnOOeN9YRGxXOHedkBzSWtjIzpmWlsaSguMVfDIi0VZOJoJnN8D5f1Nij40IUEREJPt45glfpy1JpqUmDkgkPMxYXND8c0DnHn977nFufW8WYfj149cbJDEqL74Aov6xbdAR/vGwMXxyo5J7X13X4/Y/2ztovWFxQzA/PHEpqiC0Q05jpWekcqq5jxZbSQIciXczxegRP8z5/pZHH+X6OS0REJKh55wj+PtBxSOhJiIlkbP8eLG5mwZjKmjpue34Vf3ovn4vH9uXv100gqVtUB0XZ0Nj+Sdw8YwivfLqTN1fvCkgM5dW1/PKt9WT3SuCqSaG5QMyxThmSQlR4mIaHSodrMhF0zt3tfb62kcc3Oy5EERGRoPVvM7vYgnm3bQlKU4aksWbnfkoPVTd6vvhgFVc+tpzXV+3ix2dn8ftLRhEdEd7BUTb0velDGNOvBz999bOArHT54PsF7NpfyS8uHEFEuC9LXQS/uKgIJg5K1n6C0uF82VA+2sy+ZmY/MbO7Dj86IjgREZEg9wPgBaDKzA6YWZmZ+bx6qHRdUzJTcQ4+2ljS4Fz+njIufGgJa3fuZ+6VY7lp+hCC5buGiPAw/nTZGGo9jh++sBqPp+O2ttxUdJBHF23iopP6MD6jc63JND0rnY1Fh9hWUh7oUKQL8eWrlNeBWUAtcOioh4iISFeXCFwD/NI51x04ETgzoBFJSBjdN5GEmAgWHbONxIefF3HR3I+oqvXwwndO5tyRvQMUYdMyUrtx91eG89HGEp5YsrlD7umc45431xMTEc4d54b2AjGNObKNxOfqFZSO48vuo32dczP9HomIiEjoeQjwADOAOUAZ8DIwPpBBSfCLCA/jlMEpLMovxjmHmfH3ZVu55411ZKbH8/g14+nTIzbQYTbp0px+zM8t5HfvbmDykFSG9e7u1/v9a90ePvy8iJ+fP5z0hBi/3isQBqZ2IyMljgV5hXzj5IxAhyNdhC89gh+Z2Ui/RyIiIhJ6JjrnbgIqAZxzpUDgVvOQkDIlM42d+yrYWHSIe99cx89fW8tpQ9N46bunBHUSCPXbHvzm4lEkxkVy23OrqKzx39YHFdV1/OKt9WT1TODqkzvHAjGNmZaVzkcbS/zaliJH8yURnAKsNLMNZrbGzD4zszX+DkxERCQE1JhZOOAAzCyN+h5CkWadmlm/F+CVjy3jySVbuHZyBo9+I4f4aF8GbAVecrcofn/JaDbsKeN3727w233mLixg574K5sw6sdMsENOY6dnpVNV6WLqp4bxREX/w5V+ac/wehYiISGh6AHgVSDezXwGzgZ8FNiQJFQNSutE/OY6d+yr4xYUj+HoIbodw2tA0rjklgyeWbGZ6dhpTM9Patf4txYd45INNzBpzAhMHpbRr3cFm4sBkYiLDWJhXyPSs9ECHI11Ak4mgmXV3zh2gfr6DiIiIHMM594yZrQROBwy40DmXG+CwJITMvXIstR7HmH49Ah1Kq91xTjaLC4r50YureffWU9ttr0PnHPe+uY7IcOMn5w5rlzqDWUxkOJMHp7JgQxH3eOeNivjT8frX/+F9Xgms8D6vPOp9s8xspndIaYGZ3XGccrPNzJlZjo9xi4iIBAXnXJ5z7iHn3INKAqWlRvRJDOkkEOoTmD9dNoa9h6r5yauf4Vz7bCkxP7eQBRuKuO2MofTs3vkWiGnMtOx0tu0tZ1OxFugX/zvehvLne58HOucGeZ8PPwY1V7F3zsRD1A8tHQ5cYWbDGymXANwCLG/thxARERGRwBnRJ5EfnpXFO2u/4OVPdra5vsqaOu59q34F1WsmZ7Q9wBAxbWj90NoFedpGQvzPp9nIZpYEZAJHvo5xzn3YzGUTgALn3CZvHc9Rvx/h+mPK/QL4HfAjX2LxeDxUVFT4UrRRlZWVrb62M1O7NKQ2aUht0pDapCG1iUjX9O2pg1iQV8jdr69lQkYy/VPiWl3XXz7YyPa9Ffzj2xOJ7MQLxByrX3IcmenxLNxQxLemNtvvItImzf7NMrNvAR8C/wLu9T7f40PdfYDtR73f4T12dN0nAf2cc281E8P1ZrbCzFYUFxf7cGsRERER6UjhYcYfLhtDWJjx/RdWUVvXugV0t+8t5+GFGzl/VG9OGZzazlEGv2lZaSzfXMKhqtpAhyKdnC89grdSvzHuMufcdDPLpj4hbE5jM1yPDBo3szDgj8A1zVXknJsHzAPIyclxsbFt31unPerojNQuDalNGlKbNKQ2aUhtItL19OkRyy8vHMGtz63i4YUbufn0zBbXce+b6wkPM356XudfIKYx07PSeXTRZpYUFHPWib0CHY50Yr70tVc65yoBzCzaOZcHZPlw3Q6g31Hv+wK7jnqfAIwAFprZFmAS8IYWjBEREREJXbPG9GHWmBP40/x8Vm3f16JrF+QV8l7uHm45PZPeiV3zy6ScjGS6RYWz8POiQIcinZwvieAOM+sBvAb8x8xe58sJXVM+BjLNbKCZRQGXA28cPumc2++cS3XOZTjnMoBlwAXOOZ9WJBURERGR4DRn1gh6JkTz/edXUV7t2xDHypo67nlzHYPTuvHNyQP9HGHwiooIY0pmKgvzCtttBVaRxjSbCDrnvuqc2+ecuwf4OfA4cKEP19UC36N+TmEu8IJzbp2ZzTGzC9oWtoiIiIgEq8TYSO6/dAxbSg7xy3/6tqvKox9uYmtJOfdeMIKoiK6zQExjpmels2t/JZ/vORjoUKQTO+4cQe88vjXOuREAzrkPWlK5c+5t4O1jjt3VRNlpLalbRERERILXyYNTuP7UQTzywSZmZKVzxvCeTZbdUVrOQwsLOHdkL6Zkdr0FYo41LSsdgAUbCsnqlRDgaKSzOu7XLc45D7DazPp3UDwiIiIi0kn84MyhDO/dndtfXkNRWVWT5X7x1noM42fnNdhyukvqlRjDsN7dtZ+g+JUv/e69gXVmNt/M3jj88HdgIiIiIhLaoiPC+fPlYzhYVcvtL69pdM7bB58X8a91e/jejCGc0KNrLhDTmOlZaazYWsqByppAhyKdlC+J4L3A+cAc4P6jHiIiItJCZtbPzBaYWa6ZrTOzWxspM83M9pvZKu+j0WkVIqEgs2cCd56Tzft5hTy9fNuXzlXV1nHPG+sYmNqNb03tugvENGZ6djp1HsfifO2hLf7hSyJ4rnPug6MfwLn+DkxERKSTqgV+6JwbRv3WSTeZWWPj4RY558Z4H3M6NkSR9nX1KRmcNjSNX/1zPQWF/7cAymOLNrO5+BD3XHAi0RHhAYww+JzUrwfdYyI0PFT8xpcN5c8Ebj/m2DmNHBMREZFmOOd2A7u9r8vMLBfoA6xva90ej4eKioq2VhOSKisrAx1CyOnoNptz/lBmPfxfbnn2E569bhzFB6t58P18zshOY0K/+JD52e3Idps8OJkFGwo5VF5OmFmH3dcf9He0dfzZbk32CJrZd83sMyDLzNYc9dgMrPFbRCIiIl2EmWUAJwHLGzl9spmtNrN3zOzE49RxvZmtMLMVxcUaQibBKz0hmjlfyWb97jLmLtzM7/6dj3Nwx9lDAh1a0Do1M4Xig9XkfqFtJKT9Ha9H8B/AO8CvgTuOOl7mnNvr16hEREQ6OTOLB14GbnPOHTjm9CfAAOfcQTM7F3gNyGysHufcPGAeQE5OjouN7dqLbXT1z98aHdlmF4ztz5JN+5i3ZCvOwQ/PHMqQ3skddv/21BHtduaIPtz5Wi5LN+8nZ1C63+/XEfR3tHX80W5N9gg65/Y757Y4565wzm096qEkUEREpA3MLJL6JPAZ59wrx553zh1wzh30vn4biDQzba4mncJdXxnOgOQ4BqZ249unDgp0OEEtNT6a0X0TWbBB8wSl/fkyR1BERETaiZkZ8DiQ65z7QxNlegF7nHPOzCZQ/8VtSQeGKeI33aIjePPmKXgcxERqgZjmTMtK54H389l7qJrkblGBDkc6EV9WDRUREZH2Mxn4OjDjqO0hzjWzG8zsBm+Z2cBaM1sNPABc7hrbgE0kRCXERJIYGxnoMELC9Ox0nINF+UWBDkU6GfUIioiIdCDn3GLguMv/OeceBB7smIhEJJiN6pNISrcoFuQVMmtMn0CHI52IegRFRERERIJUWJhx2tA0Pvi8iDqPBgZI+1EiKCIiIiISxKZlp1NaXsPqHfsCHYp0IkoERURERESC2KmZqYQZLMzT6qHSfpQIioiIiIgEsR5xUYztn8SCDVowRtqPEkERERERkSA3PTudz3bup7CsMtChSCehRFBEREREJMhNy0oD4AP1Cko7USIoIiIiIhLkhvfuTnpCNAuVCEo7USIoIiIiIhLkzIzpWel8mF9ETZ0n0OFIJ6BEUEREREQkBEzPTqOsspZPtpYGOhTpBJQIioiIiIiEgMlDUokIM60eKu1CiaCIiIiISAhIiIkkJyOJhRu0n6C0nRJBEREREZEQMT0rnbwvyti9vyLQoUiIUyIoIiIiIhIipmenA2j1UGkzJYIiIiIiIiEiMz2ePj1iWZCn4aHSNkoERURERERChJkxLSuNJQXFVNXWBTocCWFKBEVEREREQsj0rHQOVdexYou2kZDWUyIoIiIiIhJCThmSQlR4mIaHSpsoERQRERERCSFxURFMHJTMAm0jIW2gRFBEREREJMRMz0pnY9EhtpWUBzoUCVF+TQTNbKaZbTCzAjO7o5HzPzCz9Wa2xszmm9kAf8YjIiIiItIZHNlG4nP1Ckrr+C0RNLNw4CHgHGA4cIWZDT+m2KdAjnNuFPAS8Dt/xSMiIiIi0lkMTO1GRkqc5glKq0X4se4JQIFzbhOAmT0HzALWHy7gnFtwVPllwFXNVerxeKioqGh1UJWVla2+tjNTuzSkNmlIbdKQ2qQhtYmISMeYlpXOs//dRmVNHTGR4YEOR0KMP4eG9gG2H/V+h/dYU64D3mnshJldb2YrzGxFcXFxO4YoIiIiIhKapmenU1XrYemmkkCHIiHInz2C1sgx12hBs6uAHOC0xs475+YB8wBycnJcbGxsm4Nrjzo6I7VLQ2qThtQmDalNGlKbNM7M+gF/A3oBHmCec+7Px5Qx4M/AuUA5cI1z7pOOjlVEgtvEgcnERIaxMK+Q6VnpgQ5HQow/ewR3AP2Oet8X2HVsITM7A/gpcIFzrsqP8YiIiASDWuCHzrlhwCTgpkbm0J8DZHof1wMPd2yIIhIKYiLDmTw4lQUbinCu0f4WkSb5s0fwYyDTzAYCO4HLga8dXcDMTgIeAWY65zTTVUREOj3n3G5gt/d1mZnlUj91Yv1RxWYBf3P1/7NbZmY9zKy399omtXUefSjT3NSWU5u1TrC12+RBPZifV0jujhIGpnYLdDhNCrZ2CxX+bDe/9Qg652qB7wH/AnKBF5xz68xsjpld4C12HxAPvGhmq8zsDX/FIyIiEmzMLAM4CVh+zCmf59lrHr1I13ZqZgoAH+RrnqC0jD97BHHOvQ28fcyxu456fYY/7y8iIhKszCweeBm4zTl34NjTjVzS6Lgvf8yjD2Vd/fO3htqsdYKl3YbExpKZHs+STfv47oysQIfTrGBpt1Djj3bz64byIiIi0pCZRVKfBD7jnHulkSI+zbMXEYH61UOXby7hUFVtoEOREKJEUEREpAN5VwR9HMh1zv2hiWJvAN+wepOA/c3NDxSRrmtaVho1dY4lBRoeLr7z69BQERERaWAy8HXgMzNb5T32E6A/gHPuL9RPqzgXKKB++4hrAxCniISInAHJxEdHsGBDEWed2CvQ4UiIUCIoIiLSgZxzi2l8DuDRZRxwU8dEJCKhLioijClDUlm4oRDnHPUDD0SOT0NDRURERERC3PTsNHbvr2TDnrJAhyIhQomgiIiIiEiIm5aVDsCCvKIARyKhQomgiIiIiEiI69k9huG9u7NgQ2GgQ5EQoURQRERERKQTmJ6dxsqtpRyorAl0KBIClAiKiIiIiHQC07PSqfM4FudrGwlpnhJBEREREZFOYEy/HnSPiWBBnoaHSvOUCIqIiIiIdAIR4WGcOjSNhZ8X4fG4QIcjQU6JoIiIiIhIJzE9K52isirW7z4Q6FAkyCkRFBERERHpJE7LSgPQ8FBplhJBEREREZFOIjU+mtF9E7WNhDRLiaCIiIiISCcyLSudT7fvY++h6kCHIkFMiaCIiIiISCcyPTsd52BRflGgQ5EgpkRQRERERKQTGdUnkZRuUZonKMelRFBEREREpBMJCzNOG5rGB58XUadtJKQJSgRFRERERDqZadnplJbXsHrHvkCHIkFKiaCIiIiISCdzamYqYQYLNTxUmqBEUERERESkk+kRF8XY/kks2KAFY6RxSgRFRERERDqh6dnpfLZzP4VllYEORYKQEkERERERkU5oWlYaAB+oV1AaoURQRERERKQTGt67O+kJ0SxUIiiNUCIoIiIiItIJmRnTs9L5ML+ImjpPoMORIKNEUERERESkk5qenUZZZS2fbC0NdCgSZJQIioiIiIh0UpOHpBIRZlo9VBqICHQAIiIiXY2ZPQGcDxQ650Y0cn4a8Dqw2XvoFefcnI6LUEQ6i4SYSMZnJLNwQyF3nJPt9/tV13ooOVRFcVk1RQcrvc9VeOpqyeoZz5gBqaQlRGNmfo8lWFRU17G5+BCbig+yqegQY/r14NShaYEOS4mgiIhIADwFPAj87ThlFjnnzu+YcESkM5uencb/vJ3Hrn0VnNAjtsXX19Z52HuomsKyKooPVlFUVkXxwWrvc9X/PR+sYl95TbP1pXSLYljv7gzrneB97s7gtHiiIkJ7sGLJwSoKCg9SUHSw/rmwPvHbua/iSBkzuHHaYCWCIiIiXZFz7kMzy2jvej0eDxUVFc0X7IQqK7VPWkupzVonFNvt5IxEAP6zdieXjusDQJ3HUVpeQ8mhaooP1id2xQerve+rKfY+lxysprS8BtdIvXGR4aTGR5ESH0VGSizj+ieSGh9FareoI8fT4qNI6RbF/oPl5BeVs2lvNRv2HGTDnoP89aO9VHsXsYkMMwandSOrVzzZPePJ6hlPdq94kuKiOqqZfOKcY/eBKjYVHWJjcbn3+RCbisspPSoJjo0MY2BqN8b2685FY3oxMDWOgSlxDEiJIzYy3Od/q/3586ZEUEREJDidbGargV3Aj5xz6xorZGbXA9cD9OvXrwPDE5FQMTg1jhMSY3jog808+/FOig9Ws7e8Gk8j2V1MRNiRJK5/Uixj+yWS8qXELpqUbpGkxEfRLaoFqURsJDn9E5kyNObIoVqPhy0lFWz4oow8b3L40ca9vL76iyNl0hOiyDqcGPaMJ6tXAhkpsUSE+bf3sKbOw/bSCjYWlbOp2JvsFZWzubic8pq6I+V6xEYyOC2OM7LTGJQax+C0bgxKjaN3YgxhQT781a+JoJnNBP4MhAOPOed+c8z5aOqHxYwDSoDLnHNb/BmTiIhICPgEGOCcO2hm5wKvAZmNFXTOzQPmAeTk5LjY2JYP++pMuvrnbw21WeuEWrvdMG0wL3+yk7T4KE4akERqfDRpCdENnrtFhft1/t6x7TayWzdG9k/90rGSg1Xk7i4jd/cBcncfYP3uAyzbvJ2auvrM9f+3d/9Bctf1Hcefr9wll4RLQnLhaExiEpJQjTpCiikZKZNWGpVxjDpUgjphKlMLLbbaOp20zLSM/UdLa6cdaCkqozIIKlS56USRYigzaEgihBCCMQem9QSTIBAS+RES3v3j+7njm9u9y95ld797+309Znb2u9/97N7n+7rP7uf72f18v9vVOYmzz5xxwtTSN8+byaxpk8dcn5eOHueJg0d4Ijeds//AEfb96tdDfw9g3qypLOvtZtVZPSzr7WbZGd0s6+2mp7trHCmMXSPaW8MGgpI6gBuA3wcGgG2S+iJid67YFcBzEbFM0nrg88CljaqTmZnZRBARL+SWN0n6N0lzI+KZIutlZhPXhtWL2bB6cdHVqElPdxcXLO/iguWvDxCPHnuN/gNH+MkvX0gDxMPc+/gBvrl9YKjM/NOnVQwOF82ZzqRJ4vkXj54w0Bs8ju8Xz79EpPFexySxaM50lvZ2c9GKM4cGe0t7u+nuar+JlI3colVAf0Q8CSDpdmAdkB8IrgOuTct3ANdLUkRUm4YMnPrxDxNxXnczOJdKzqSSM6nkTCo5k1Mn6TeA/RERklaR/dzTrwqulplZYaZ0TmLFG2ay4g0zh9ZFBAcPv8LuihVteQAAC2JJREFUNDAc/AZx856DHE/zXqdP6WD6lA6eOXJ06HFdnZNYekY3K984mw+ftzD7hq+3m0U90+nq7Gj6thWlkQPB+cDPc7cHgN8eqUxEHJN0COgBTvjE08c/mJlZO5F0G7AGmCtpAPg7YDJARNwIXAJcJekY8BKwfrQPSc3MykgSvTOn0jtzKmt+s3do/cuvHmfv/iND00pfOnqcpb2npSmdM5g/exodk1r7+L1maORAsFq6wzuxWso05PiHiTavu1mcSyVnUsmZVHImlZzJyCLispPcfz3Zz0uYmdkYTZ3cwdsWzOJtC2YVXZWW1sjT7QwA+a/vFpCd+axqGUmdwCzg2QbWyczMzMzMrPQaORDcBiyXtETSFGA90DesTB9weVq+BPiBp76YmZmZmZk1VsOmhqZj/q4G7ib7+YibI+IxSZ8FtkdEH/Bl4BZJ/WTfBK5vVH3MzMzMzMws09DzoEbEJmDTsHV/m1t+GfiDRtbBzMzMzMzMTtTIqaFmZmZmZmbWgjwQNDMzMzMzKxkPBM3MzMzMzErGA0EzMzMzM7OS0UT7tQZJB4H/PcWnmQs8U4fqtBvnUsmZVHImlZxJpXplsigizqjD85RCnfrIicyvxbFzZuPj3MbHuY1PtdxOuX+ccAPBepC0PSLOK7oerca5VHImlZxJJWdSyZlYEdzuxs6ZjY9zGx/nNj6Nys1TQ83MzMzMzErGA0EzMzMzM7OSKetA8KaiK9CinEslZ1LJmVRyJpWciRXB7W7snNn4OLfxcW7j05DcSnmMoJmZmZmZWZmV9RtBMzMzMzOz0vJA0MzMzMzMrGTaeiAo6T2S9kjql7Sxyv1dkr6R7n9Q0uLm17K5asjkLyTtlrRT0r2SFhVRz2Y6WSa5cpdICkmlOO1xLblI+nBqL49J+nqz69hsNbx+3ihps6SH02vo4iLq2SySbpZ0QNKuEe6XpH9Nee2UtLLZdbSJT9I+SY9K2iFpe1o3R9I9kvam69lp/YhtTtLlqfxeSZcXtT2NUu31WM+cJP1W+j/0p8equVvYGCPkdq2kX6Q2tyP/Xi7pr1MGeyS9O7e+av8gaUnax9yb9jmnNG/rGkPSwtTXPZ76/z9P693eRjFKbsW1t4hoywvQATwBnAVMAR4BVgwr8yfAjWl5PfCNouvdApn8LjA9LV/lTIbKzQDuB7YA5xVd71bIBVgOPAzMTrd7i653C2RyE3BVWl4B7Cu63g3O5EJgJbBrhPsvBr4LCDgfeLDoOvsy8S7APmDusHX/AGxMyxuBz6flqm0OmAM8ma5np+XZRW9bnXOqeD3WMydgK7A6Pea7wHuL3uYG5nYt8JkqZVek9/4uYEnqEzpG6x+AbwLr0/KNg33ERL4A84CVaXkG8NOUjdvb+HIrrL218zeCq4D+iHgyIo4CtwPrhpVZB3w1Ld8BvKsdPnEYxUkziYjNEfFiurkFWNDkOjZbLe0E4O/J3uBebmblClRLLn8E3BARzwFExIEm17HZaskkgJlpeRbwVBPr13QRcT/w7ChF1gFfi8wW4HRJ85pTO2tz+f77q8AHcuurtbl3A/dExLPpPese4D3NrnQjjfB6rEtO6b6ZEfGjyPYwv5Z7rgmthvexvHXA7RHxSkT8DOgn6xuq9g9pn/L3yPYx4cT/wYQVEU9HxENp+TDwODAft7dRjZLbSBre3tp5IDgf+Hnu9gCVYQ+ViYhjwCGgpym1K0YtmeRdQfYpTDs7aSaSzgUWRsR/NbNiBaulrZwNnC3pAUlbJLXVTlUVtWRyLfAxSQPAJuCTzalayxrre45ZNQF8X9KPJX0irTszIp6GbOcK6E3rR2pzZW2L9cppfloevr6dXZ2mMd48OMWRsefWAzyf9jHz69uGssOqzgUexO2tZsNyg4LaWzsPBKt9szf8tzJqKdNOat5eSR8DzgOua2iNijdqJpImAf8M/GXTatQaamkrnWTTQ9cAlwFfknR6g+tVpFoyuQz4SkQsIJsKc0tqQ2VVtvdYa4x3RsRK4L3An0q6cJSyI7U5t8UTjTWnsuX378BS4BzgaeCf0nrnliOpG7gT+FREvDBa0SrrnNvruRXW3tp5B2UAWJi7vYDKaVpDZSR1kk3lqnV6wERUSyZIugi4Bnh/RLzSpLoV5WSZzADeCtwnaR/Z3PY+tf8JY2p9/dwVEa+mKQt7yAaG7aqWTK4gm59PRPwImArMbUrtWlNN7zlmo4mIp9L1AeDbZNOi9g9OM07Xg1PTR2pzZW2L9cppgBMPFWnr/CJif0Qcj4jXgC+StTkYe27PkE2D7By2fsKTNJlsMHNrRPxnWu32dhLVciuyvbXzQHAbsDydPWcK2clg+oaV6QMGz1B0CfCDNBe5XZ00kzQN8j/IBoHtfswXnCSTiDgUEXMjYnFELCY7bvL9EbG9mOo2TS2vn++QnVwISXPJpoo+2dRaNlctmfwf8C4ASW8mGwgebGotW0sfsCGdMe584NDgtCGzWkg6TdKMwWVgLbCLE/vvy4G70vJIbe5uYK2k2Wna1dq0rt3VJad032FJ56fjkDbknqvtDDuW+YNkbQ6y3NYrO+v8ErIPP7cyQv+Q9ik3k+1jwon/gwkrtYEvA49HxBdyd7m9jWKk3Aptb6OdSWaiX8imZv2U7Mw616R1nyXbkYdsJ+1bZAdfbgXOKrrOLZDJfwP7gR3p0ld0nYvOZFjZ+yjBWUNrbCsCvgDsBh4lnaWqnS81ZLICeIDsDF47gLVF17nBedxGNo3lVbJPKK8ArgSuzLWRG1Jej5blteNL/S5kZ8V7JF0ey73ueoB7gb3pek5aP2KbAz6e+vt+4A+L3rYGZFXt9Vi3nMgOF9mVHnM9oKK3uYG53ZJy2Um2Mz4vV/6alMEecmeyrNY/5Nrw1pTnt4Cuore5DpldQDblcCev7y9e7PY27twKa29KDzIzMzMzM7OSaOepoWZmZmZmZlaFB4JmZmZmZmYl44GgmZmZmZlZyXggaGZmZmZmVjIeCJqZmZmZmZWMB4JmdSTph+l6saSP1Pm5/6ba3zIzM2t17h/NWo9/PsKsASStAT4TEe8bw2M6IuL4KPcfiYjuetTPzMysCO4fzVqHvxE0qyNJR9Li54DfkbRD0qcldUi6TtI2STsl/XEqv0bSZklfJ/sxUSR9R9KPJT0m6RNp3eeAaen5bs3/LWWuk7RL0qOSLs09932S7pD0E0m3StLg80naneryj83MyMzMysf9o1nr6Sy6AmZtaiO5TzxTh3UoIt4hqQt4QNL3U9lVwFsj4mfp9scj4llJ04Btku6MiI2Sro6Ic6r8rQ8B5wBvB+amx9yf7jsXeAvwFPAA8E5Ju4EPAm+KiJB0et233szMrDr3j2Ytwt8ImjXHWmCDpB3Ag0APsDzdtzXXyQH8maRHgC3Awly5kVwA3BYRxyNiP/A/wDtyzz0QEa8BO4DFwAvAy8CXJH0IePGUt87MzGx83D+aFcQDQbPmEPDJiDgnXZZExOAnnr8eKpQdO3ERsDoi3g48DEyt4blH8kpu+TjQGRHHyD5lvRP4APC9MW2JmZlZ/bh/NCuIB4JmjXEYmJG7fTdwlaTJAJLOlnRalcfNAp6LiBclvQk4P3ffq4OPH+Z+4NJ0nMUZwIXA1pEqJqkbmBURm4BPkU2bMTMzawb3j2YtwscImjXGTuBYmsLyFeBfyKadPJQOSD9I9mnjcN8DrpS0E9hDNv1l0E3ATkkPRcRHc+u/DawGHgEC+KuI+GXqKKuZAdwlaSrZp6WfHt8mmpmZjZn7R7MW4Z+PMDMzMzMzKxlPDTUzMzMzMysZDwTNzMzMzMxKxgNBMzMzMzOzkvFA0MzMzMzMrGQ8EDQzMzMzMysZDwTNzMzMzMxKxgNBMzMzMzOzkvl/E2UXnbhalTAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = LeNet5()\n",
    "nn = NN(model, DEVICE)\n",
    "\n",
    "prune = Pruner(model, RamLoader['train'], DEVICE, silent=False)\n",
    "prune.snip(0.2)\n",
    "\n",
    "loop_operation=[prune.indicate]\n",
    "\n",
    "nn.optim = torch.optim.SGD(model.parameters(), lr = 0.1, momentum=0.9, weight_decay=0.0005)\n",
    "nn.scheduler = torch.optim.lr_scheduler.StepLR(nn.optim, step_size=40, gamma=0.1)\n",
    "\n",
    "hist = nn.fit(RamLoader, torch.nn.CrossEntropyLoss(), s_report=float('inf'),\n",
    "              v_report=5, max_epochs=140, silent=False, loop_operation=loop_operation)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}