optimizers_dropout_initializations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 2: Experiment with Initialization, Regularization and Optimizers\n",
    "***\n",
    "**Implementation specifics:**\n",
    "- Validation data kept separate in this task as well, in order to keep parity between Network 2 of Task 1 and the models being trained in this task\n",
    "- All models run for 50 epochs\n",
    "- Direct use of APIs done for the optimizers\n",
    "- Learning rate kept at 0.001 as before\n",
    "- Accuracies printed while training\n",
    "- Varying parameters of the initializers could significantly change the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from zipfile import ZipFile\n",
    "import numpy as np\n",
    "import mxnet as mx\n",
    "\n",
    "display_step=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/bt3/15CS10057/dl\n"
     ]
    }
   ],
   "source": [
    "class DataLoader(object):\n",
    "    DIR=None\n",
    "    import os\n",
    "    print(os.getcwd())\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.DIR = './'\n",
    "    \n",
    "    # Returns images and labels corresponding for training and testing. Default mode is train. \n",
    "    # For retrieving test data pass mode as 'test' in function call.\n",
    "    def load_data(self, mode = 'train'):\n",
    "        label_filename = mode + '_labels'\n",
    "        image_filename = mode + '_images'\n",
    "        label_zip = self.DIR + label_filename + '.zip'\n",
    "        image_zip = self.DIR + image_filename + '.zip'\n",
    "        with ZipFile(label_zip, 'r') as lblzip:\n",
    "            labels = np.frombuffer(lblzip.read(label_filename), dtype=np.uint8, offset=8)\n",
    "        with ZipFile(image_zip, 'r') as imgzip:\n",
    "            images = np.frombuffer(imgzip.read(image_filename), dtype=np.uint8, offset=16).reshape(len(labels), 784)\n",
    "        return images, labels\n",
    "    \n",
    "    \n",
    "    def create_batches(self,train_size):\n",
    "        '''\n",
    "        Handles the train-validation split \n",
    "        Params:\n",
    "            train_size: ratio of train_size : float\n",
    "        Returns:\n",
    "            images, labels, validation images, validation labels in that order\n",
    "        '''\n",
    "        img,labels=self.load_data()\n",
    "        n=labels.shape[0]\n",
    "        #split into train and validation data\n",
    "        validation_img=img[int(train_size*n):]\n",
    "        img=img[:int(train_size*n)]\n",
    "        validation_labels=labels[int(train_size*n):]\n",
    "        labels=labels[:int(train_size*n)]\n",
    "        return img, labels, validation_img, validation_labels        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_loader=DataLoader()\n",
    "data,label,val_data,val_labels=data_loader.create_batches(train_size=0.7)\n",
    "batch_size = 100\n",
    "ntrain = int(data.shape[0]*0.8)\n",
    "train_iter = mx.io.NDArrayIter(data, label, batch_size, shuffle=True)\n",
    "val_iter = mx.io.NDArrayIter(val_data, val_labels,batch_size=val_data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs=50\n",
    "data_2 = mx.symbol.Variable('data')\n",
    "fc1_2  = mx.symbol.FullyConnected(data = data_2, num_hidden=1024)\n",
    "act1_2 = mx.symbol.Activation(data = fc1_2, act_type=\"relu\")\n",
    "fc2_2  = mx.symbol.FullyConnected(data = act1_2, num_hidden = 512)\n",
    "act2_2 = mx.symbol.Activation(data = fc2_2, act_type=\"relu\")\n",
    "fc3_2  = mx.symbol.FullyConnected(data = act2_2, num_hidden = 256)\n",
    "act3_2 = mx.symbol.Activation(data = fc3_2, act_type=\"relu\")\n",
    "\n",
    "#output layer\n",
    "fc4_2  = mx.symbol.FullyConnected(data = act3_2, num_hidden=10)\n",
    "mlp_2 = mx.symbol.SoftmaxOutput(data = fc4_2,name='softmax')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Experiment 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Normal Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Training ('accuracy', 0.66466666666666663)\n",
      "Epoch 11, Training ('accuracy', 0.86850000000000005)\n",
      "Epoch 21, Training ('accuracy', 0.89069047619047614)\n",
      "Epoch 31, Training ('accuracy', 0.90835714285714286)\n",
      "Epoch 41, Training ('accuracy', 0.92528571428571427)\n",
      "Epoch 50, Training ('accuracy', 0.93778571428571433)\n"
     ]
    }
   ],
   "source": [
    "normal_init=[]\n",
    "mod = mx.mod.Module(symbol=mlp_2,\n",
    "                    context=mx.cpu(),\n",
    "                    data_names=['data'],\n",
    "                    label_names=['softmax_label'])\n",
    "\n",
    "# allocate memory given the input data and label shapes\n",
    "mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)\n",
    "mod.init_params(mx.initializer.Normal(0.01))\n",
    "learn_rate=0.001\n",
    "mod.init_optimizer(optimizer='sgd', optimizer_params=(('learning_rate', learn_rate), ))\n",
    "# use accuracy as the metric\n",
    "metric = mx.metric.CrossEntropy()\n",
    "metric2= mx.metric.create('acc')\n",
    "for epoch in range(n_epochs):\n",
    "    train_iter.reset()\n",
    "    metric.reset()\n",
    "    metric2.reset()\n",
    "    for batch in train_iter:\n",
    "        mod.forward(batch, is_train=True)       # compute predictions\n",
    "        mod.update_metric(metric, batch.label)  # accumulate prediction loss\n",
    "        mod.update_metric(metric2, batch.label)  # accumulate prediction accuracy\n",
    "        mod.backward()                          # compute gradients\n",
    "        mod.update()                            # update parameters\n",
    "    loss=metric.get()\n",
    "    acc=metric2.get()\n",
    "    normal_init.append(loss)\n",
    "    if epoch%display_step==0 or epoch==n_epochs-1:\n",
    "        print('Epoch %d, Training %s' % (epoch+1,acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "#### Xavier Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Training ('accuracy', 0.76530952380952377)\n",
      "Epoch 11, Training ('accuracy', 0.91880952380952385)\n",
      "Epoch 21, Training ('accuracy', 0.95002380952380949)\n",
      "Epoch 31, Training ('accuracy', 0.96999999999999997)\n",
      "Epoch 41, Training ('accuracy', 0.98371428571428576)\n",
      "Epoch 50, Training ('accuracy', 0.99152380952380947)\n"
     ]
    }
   ],
   "source": [
    "xavier_init=[]\n",
    "mod = mx.mod.Module(symbol=mlp_2,\n",
    "                    context=mx.cpu(),\n",
    "                    data_names=['data'],\n",
    "                    label_names=['softmax_label'])\n",
    "\n",
    "# allocate memory given the input data and label shapes\n",
    "mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)\n",
    "mod.init_params(mx.initializer.Xavier(factor_type=\"in\", magnitude=2.34))\n",
    "learn_rate=0.001\n",
    "mod.init_optimizer(optimizer='sgd', optimizer_params=(('learning_rate', learn_rate), ))\n",
    "# use accuracy as the metric\n",
    "metric = mx.metric.CrossEntropy()\n",
    "metric2= mx.metric.create('acc')\n",
    "for epoch in range(n_epochs):\n",
    "    train_iter.reset()\n",
    "    metric.reset()\n",
    "    metric2.reset()\n",
    "    for batch in train_iter:\n",
    "        mod.forward(batch, is_train=True)       # compute predictions\n",
    "        mod.update_metric(metric, batch.label)  # accumulate prediction loss\n",
    "        mod.update_metric(metric2, batch.label)  # accumulate prediction accuracy\n",
    "        mod.backward()                          # compute gradients\n",
    "        mod.update()                            # update parameters\n",
    "    loss=metric.get()\n",
    "    acc=metric2.get()\n",
    "    xavier_init.append(loss)\n",
    "    if epoch%display_step==0 or epoch==n_epochs-1:\n",
    "        print('Epoch %d, Training %s' % (epoch+1,acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "#### Orthogonal Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Training ('accuracy', 0.68757142857142861)\n",
      "Epoch 11, Training ('accuracy', 0.87447619047619052)\n",
      "Epoch 21, Training ('accuracy', 0.89728571428571424)\n",
      "Epoch 31, Training ('accuracy', 0.9103809523809524)\n",
      "Epoch 41, Training ('accuracy', 0.92026190476190473)\n",
      "Epoch 50, Training ('accuracy', 0.92792857142857144)\n"
     ]
    }
   ],
   "source": [
    "ortho_init=[]\n",
    "mod = mx.mod.Module(symbol=mlp_2,\n",
    "                    context=mx.cpu(),\n",
    "                    data_names=['data'],\n",
    "                    label_names=['softmax_label'])\n",
    "\n",
    "# allocate memory given the input data and label shapes\n",
    "mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)\n",
    "mod.init_params(mx.initializer.Orthogonal())\n",
    "learn_rate=0.001\n",
    "mod.init_optimizer(optimizer='sgd', optimizer_params=(('learning_rate', learn_rate), ))\n",
    "# use accuracy as the metric\n",
    "metric = mx.metric.CrossEntropy()\n",
    "metric2= mx.metric.create('acc')\n",
    "for epoch in range(n_epochs):\n",
    "    train_iter.reset()\n",
    "    metric.reset()\n",
    "    metric2.reset()\n",
    "    for batch in train_iter:\n",
    "        mod.forward(batch, is_train=True)       # compute predictions\n",
    "        mod.update_metric(metric, batch.label)  # accumulate prediction loss\n",
    "        mod.update_metric(metric2, batch.label)  # accumulate prediction accuracy\n",
    "        mod.backward()                          # compute gradients\n",
    "        mod.update()                            # update parameters\n",
    "    loss=metric.get()\n",
    "    acc=metric2.get()\n",
    "    ortho_init.append(loss)\n",
    "    if epoch%display_step==0 or epoch==n_epochs-1:\n",
    "        print('Epoch %d, Training %s' % (epoch+1,acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading network 2 data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "train_2_loss=pickle.load(open('train_2_loss','rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting relevant data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAFNCAYAAACuWnPfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xl0VdX5xvHvm5kpc4AECIRRhgBiACcUVBS1BbXaqjggrdZatVqL+rO2tWq1Dq1Da6utilUpWq3zUFsHROoAoSCCSJghBEhCQiAhCRn2749zuSQhEyE3N4Tns1YWuefsc+57bu6qT/feZx9zziEiIiIiwRMS7AJEREREjnQKZCIiIiJBpkAmIiIiEmQKZCIiIiJBpkAmIiIiEmQKZCIiIiJBpkAmIiKHxMweN7NfBLsOkcOZAplIkJjZxWaWaWbFZrbVzN41sxODWE8/M3O+emr+fK+ZxzszGxjoOpvDzGaY2YJg13EwzCzCzO4ws9VmVmJmG8zsaTPrF+zamuKcu9o5d1ew6xA5nCmQiQSBmf0UeBi4B+gBpAJ/AqY10D6s7aoj1jnXtcbPi61x0ja+hsPRy8BU4GIgBhgFLAZODWZRTTGz0GDXINIRKJCJtDEziwHuBH7snHvFOVfinKtwzr3pnJvla3OHmb1sZs+b2S5ghplFmtnDZpbj+3nYzCJ97RPN7C0z22lmBWb2iZmF+PbdYmZbzGy3ma0ysxb9B97MnjGzx8zsbd+5vjCzAb59833NvtzXq2ZmE80s2/f+24DZvrZXmtkaX51vmFlKjfdwZna9ma0zs3wze8DMQnzXXmBm6TXadjezUjNLOsjrSPG9b4Gvjitr7Bvn67XcZWbbzez3vu1Rvr/FDt9nvMjMevj2xZjZU75ezi1mdve+kGJmA83sYzMr8l1PveHWzE4DJgPTnHOLnHOVzrki59xjzrmnmlH3HWb2kq/G3Wb2lZkNNrP/M7NcM9tsZqfXaD/PzO41s4W+2l43s/ga+18ys22+ffPNbHid78GfzewdMysBJvm23e3b39h3cajvvXea2Qozm1rnvPV+v0SOBApkIm3vOCAKeLWJdtPwek1igTnAz4FjgdF4vSfjgNt9bW8CsoEkvB632wBnZkOAa4GxzrluwBnAhkOo/SLg10AcsAb4DYBz7iTf/lF1etV6AvFAX+AqMzsFuBf4LpAMbAReqPMe5wIZwBjfZzDTOVfua3dJnVred87lHeQ1zMX7rFKA84F7aoTUR4BHnHPRwADgH77tl+P1WvUBEoCrgVLfvr8BlcBA4GjgdOAHvn13Af/G+7x6A39ooKbTgIXOuc0trBvg28BzvvdaAryH97/xvfD+D8ATdc53GTDTd75K4NEa+94FBgHdgf/hff9quhjvb98NqDs03NB3MRx4E+/z6A5cB8zxfUf3qff7JXIkUCATaXsJQL5zrrKJdp85515zzlU750qB6cCdzrlcXwj5NXCpr20FXsDp6+tt+8R5D6qtAiKBYWYW7pzb4Jxb28T75vt6MPb9DK2x7xXn3EJf7XPwwmFjqoFfOefKa1zD0865//lC1v8Bx1nteVL3OecKnHOb8IZ1L/Jt/xtw8b7eFt+1P9fE+9diZn2AE4FbnHNlzrmlwJPU/hwHmlmic67YOfd5je0JwEDnXJVzbrFzbpevl+xM4AZfT2cu8BBwYY3j+gIpvvdraF5bArD1EOoG+MQ5957vb/MSXiD6rXOuAi/M9jOz2Brtn3POLXfOlQC/AL67r2fPOfe0c2637290BzDKvJ7dfV53zv3X990sq1NuQ9/FY4Guvpr2Ouc+BN5i/98XDv77JdJhKJCJtL0dQKI1Paeqbm9JCl6P0j4bfdsAHsDrUfi3b7jvVgDn3BrgBrz/qOaa2Qv7hgit9sT91BrnTXTOxdb4WVlj37Yav+/B+w9sY/Lq/Ae71jU454rxPo9eDVy3/xqdc18AJcDJZnYUXo/UG028f10pQIFzbned99j3/t8HBgPf+IYlv+Xb/hxej9ML5g0X3+/r8ekLhANb9wVYvJ6o7r7jbgYMWOgbopvZQF078EJMS+sG2F7j91K80F9V4zXU/nvV/ZzD8b6XoWb2WzNba95w+QZfm8QGjq2r3u+i7xo2O+eqG7mGg/1+iXQYCmQibe8zoAw4p4l2rs7rHLwAsE+qbxu+3oybnHP98YaufrpvOMs593fn3Im+Yx1wn297zYn7mw71olpyDWbWBa93aEuNNn1q/O6/Rp+/4Q1bXgq8XE/vTFNygHgz61bnPbYAOOdWO+cuwgtU9wEvm1kXX0/Pr51zw4DjgW/hDfltBsqpHWKjnXPDfefb5py70jmXAvwQ+JPVfyfq+8A4M+vdkrpbqO7nXAHk4w1HTsMbRo0B+vnaWI32df+u+3c0/F3MAfrU6OFsjWsQ6TAUyETamHOuCPgl8JiZnWNmnc0s3MzONLP7Gzl0LnC7mSWZWaLvHM8DmNm3fBPIDdiFN1RZZWZDzOwU8yb/l+H1lFQ1cP5DtR3o30SbvwNXmNloX033AF845zbUaDPLzOJ8w3Q/AWpOhH8Ob47ZJcCzTbyXmTcZ3//jm6P1KXCvb9tIvF6xOb4DLjGzJF8vzk7fearMbJKZpfuG9HbhhZcq59xWvDlRvzOzaPNuQBhgZif7zndBjZBViBdkDvj8nXPvA/8BXjWzY8wszMy6mdnVZjazqbpb6BIzG2ZmnfHmmL3s61HrhhcydwCd8f5GzdbQdxHY18N5s+/7PhEvsNWdQyhyRFIgEwkC59zvgZ/iTcrPw+tpuRZ4rZHD7gYygWXAV3iTre/27RuE18tSjNcD9yfn3Dy8+WO/xev52IbX83NbE+XtrDOc+dNmXtYdwN98Q3ffra+Bc+4DvPlK/8SbMzWA/fOt9nkdb7mHpcDbwFM1js/Gu24HfNJEPcfjBVD/j2+Y+CK8Xp8cvBsrfuWc+4/vmCnACjMrxpvgf6GvF64n3g0Wu4CVwMf4wjBeT1kE8DVe6HqZ/cOPY4EvfOd7A/iJc259A/WeD7yDF0CLgOV4Nze879vfWN0t8RzwDN73Igq43rf9WbyhxC2+a/q8voMbUe930Tm3F29ZjzPxvo9/Ai5zzn1zCNcg0mGYN9dSRCT4zMwBg3xz3xpq8zSQ45y7vaE20jgzmwc875x7Mti1iIhHCzWKyGHDdzfmeXjLS4iIdBgashSRw4KZ3YU3jPdAI8N+IiKHJQ1ZioiIiASZeshEREREgkyBTERERCTIDrtJ/YmJia5fv37BLkNERESkSYsXL853ziU11e6wC2T9+vUjMzMz2GWIiIiINMnMNjbdSkOWIiIiIkGnQCYiIiISZApkIiIiIkF22M0hExERaW0VFRVkZ2dTVlYW7FLkMBUVFUXv3r0JDw9v0fEKZCIicsTLzs6mW7du9OvXDzMLdjlymHHOsWPHDrKzs0lLS2vROTRkKSIiR7yysjISEhIUxqRFzIyEhIRD6mFVIBMREQGFMTkkh/r9USATERFpB8yMm266yf/6wQcf5I477mjTGmbMmMHLL7/cpu8pHgUyERGRdiAyMpJXXnmF/Pz8Fh1fWVnZyhVJW9Kk/jreWf0Ozy97njMHnsmloy4NdjkiInKECAsL46qrruKhhx7iN7/5Ta19GzduZObMmeTl5ZGUlMTs2bNJTU1lxowZxMfHs2TJEsaMGUO3bt1Yv349W7duJSsri9///vd8/vnnvPvuu/Tq1Ys333yT8PBw7rzzTt58801KS0s5/vjjeeKJJzRkG2TqIatj9Y7VzF0+l4VbFga7FBEROcL8+Mc/Zs6cORQVFdXafu2113LZZZexbNkypk+fzvXXX+/fl5WVxfvvv8/vfvc7ANauXcvbb7/N66+/ziWXXMKkSZP46quv6NSpE2+//bb/fIsWLWL58uWUlpby1ltvtd1FSr0UyOqI6xQHQGFZYZArERGRoDALzE8zREdHc9lll/Hoo4/W2v7ZZ59x8cUXA3DppZeyYMEC/74LLriA0NBQ/+szzzyT8PBw0tPTqaqqYsqUKQCkp6ezYcMGAD766CPGjx9Peno6H374IStWrDiUT0xagQJZHXFRCmQiIhI8N9xwA0899RQlJSUNtqk5vNilS5da+yIjIwEICQkhPDzc3zYkJITKykrKysq45pprePnll/nqq6+48sortSBuO6BAVoe/h6xUgUxE5IjkXGB+mik+Pp7vfve7PPXUU/5txx9/PC+88AIAc+bM4cQTT2zx5e0LX4mJiRQXF+uuynZCgawO9ZCJiEiw3XTTTbXutnz00UeZPXs2I0eO5LnnnuORRx5p8bljY2O58sorSU9P55xzzmHs2LGtUbIcInMHkdrbg4yMDJeZmRmw8+fszqHX73vRo0sPtv1sW8DeR0RE2o+VK1cydOjQYJchh7n6vkdmttg5l9HUsQHrITOzp80s18yWN9JmopktNbMVZvZxoGo5GLFRsYDXQ3a4hVURERE5PAVyyPIZYEpDO80sFvgTMNU5Nxy4IIC1NFunsE5EhEawt2ovZZWa5CgiIiKBF7BA5pybDxQ00uRi4BXn3CZf+9xA1XIwzEzzyERERKRNBXNS/2AgzszmmdliM7usoYZmdpWZZZpZZl5eXsAL052WIiIi0paCGcjCgGOAs4EzgF+Y2eD6Gjrn/uKcy3DOZSQlJQW8MPWQiYiISFsK5rMss4F851wJUGJm84FRQFYQawLUQyYiIiJtK5g9ZK8DE8wszMw6A+OBlUGsx089ZCIi0pY2b95MWloaBQXe1OvCwkLS0tLYuHHjQZ0nJyeH888/PxAlSoAFctmLucBnwBAzyzaz75vZ1WZ2NYBzbiXwL2AZsBB40jnX4BIZbcm/9IV6yEREpA306dOHH/3oR9x6660A3HrrrVx11VX07dv3oM6TkpJyUCvvV1VVHdT5JXACeZflRc65ZOdcuHOut3PuKefc4865x2u0ecA5N8w5N8I593CgajlY+3rIdpbtDHIlIiJypLjxxhv5/PPPefjhh1mwYAE33XQTxcXFnHrqqYwZM4b09HRef/11AG655Rb+9Kc/+Y+94447+N3vfseGDRsYMWIE4IWtWbNmMXbsWEaOHMkTTzwBwLx585g0aRIXX3wx6enpbX+hUq9gziFrl77+GlZ/cCJsH68hSxERaTPh4eE88MADTJkyhX//+99EREQQEhLCq6++SnR0NPn5+Rx77LFMnTqVCy+8kBtuuIFrrrkGgH/84x/861//orq62n++p556ipiYGBYtWkR5eTknnHACp59+OgALFy5k+fLlpKWlBeVa5UAKZHW88w68eM8ZcOzXFH5rabDLERGRNjZvngXkvBMnNv30l3fffZfk5GSWL1/O5MmTcc5x2223MX/+fEJCQtiyZQvbt2/n6KOPJjc3l5ycHPLy8oiLiyM1NZUNGzb4z/Xvf/+bZcuW+Ycwi4qKWL16NREREYwbN05hrJ1RIKsjIcH3S2m85pCJiEibWbp0Kf/5z3/4/PPPOfHEE7nwwgt57733yMvLY/HixYSHh9OvXz/KyrynyJx//vm8/PLLbNu2jQsvvPCA8znn+MMf/sAZZ5xRa/u8efPo0qVLm1yTNJ8CWR3+QLYnQUOWIiJHoOb0ZLU25xw/+tGPePjhh0lNTWXWrFn87Gc/Y9y4cXTv3p3w8HA++uijWnddXnjhhVx55ZXk5+fz8ccHPg76jDPO4M9//jOnnHIK4eHhZGVl0atXr7a8LDkIwVz2ol3a30OWoB4yERFpE3/9619JTU1l8uTJAFxzzTV88803jB49mszMTDIyMpgzZw5HHXWU/5jhw4eze/duevXqRXJy8gHn/MEPfsCwYcMYM2YMI0aM4Ic//CGVlZVtdk1ycMy5tv9/AociIyPDZWZmBuz8K1fCsGFAfBYpP5/Elp9uCdh7iYhI+7By5UqGDh0a7DLkMFff98jMFjvnMpo6Vj1kddTsIdOyFyIiItIWFMjqiI/3/VIWx57yUvZW7Q1qPSIiItLxKZDVERYGMTGAC4GyWM0jExERkYBTIKuHv5esVHdaioiISOApkNWj1tIX6iETERGRAFMgq0etpS/UQyYiIiIBpkBWD63WLyIibS07O5tp06YxaNAgBgwYwE9+8hP27j3wxrINGzbw97//3f/6mWee4dprr23LUpula9euwS7hsKJAVg+t1i8iIm3JOcd5553HOeecw+rVq8nKyqK4uJif//zntdpVVlYeEMikY9Cjk+pRc1K/1iITEZFA+/DDD4mKiuKKK64AIDQ0lIceeoi0tDTS0tL46KOPKCsro6SkhD179rBy5UpGjx7N5ZdfTlxcHDk5OUyZMoW1a9dy7rnncv/99wMwd+5c7rnnHpxznH322dx3330APPXUU9x3332kpKQwaNAgIiMj+eMf/8jGjRuZOXMmeXl5JCUlMXv2bFJTU5kxYwbR0dFkZmaybds27r//fs4//3yKi4uZNm0ahYWFVFRUcPfddzNt2rSgfY6HMwWyetSe1L82qLWIiEjHt2LFCo455pha26Kjo0lNTaWyspLPPvuMZcuWER8fz7x583jwwQd56623AG/IcunSpSxZsoTIyEiGDBnCddddR2hoKLfccguLFy8mLi6O008/nddee41x48Zx11138b///Y9u3bpxyimnMGrUKACuvfZaLrvsMi6//HKefvpprr/+el577TUAtm7dyoIFC/jmm2+YOnUq559/PlFRUbz66qtER0eTn5/Psccey9SpUzGztv0AOwAFsnrUmkNWFrjHNImISPtjvw5MmHC/avhRhc65ekPMvu2TJ08m3j98c6BTTz2VmJgYAIYNG8bGjRvZsWMHEydOJCkpCYDp06czf/58AE4++WT/+S644AKysrIA+Oyzz3jllVcAuPTSS7n55pv973HOOecQEhLCsGHD2L59u7++2267jfnz5xMSEsKWLVvYvn07PXv2bPbnIh4FsnroLksREWlLw4cP55///Getbbt27WLz5s2EhobSpUuXRo+PjIz0/x4aGkplZSUNPav6YJ5hXTMk1nyPfeeYM2cOeXl5LF68mPDwcPr160dZWVmzzy/7KZDVw/9/QrQOmYjIEaexnqxAOfXUU7n11lt59tlnueyyy6iqquKmm25ixowZdO7cuVbbbt26sXv37ibPOX78eH7yk5+Qn59PXFwcc+fO5brrriMjI4Mbb7yRwsJCunXrxj//+U/S09MBOP7443nhhRe49NJLmTNnDieeeGKj71FUVET37t0JDw/no48+YuPGjS3/EI5wusuyHuohExGRtmRmvPrqq7z00ksMGjSIwYMHExUVxT333HNA25EjRxIWFsaoUaN46KGHGjxncnIy9957L5MmTWLUqFGMGTOGadOm0atXL2677TbGjx/PaaedxrBhw/zDnY8++iizZ89m5MiRPPfcczzyyCON1j19+nQyMzPJyMhgzpw5HHXUUYf2QRzB7GC6LtuDjIwMl5kZ2HldRUUQGwuEF9PnvmFsunFTQN9PRESCa+XKlQwdOjTYZbSZ4uJiunbtSmVlJeeeey4zZ87k3HPPDXZZh736vkdmttg5l9HUsQHrITOzp80s18yWN9FurJlVmdn5garlYEVHQ1iYg4quFBbvCXY5IiIireqOO+5g9OjRjBgxgrS0NM4555xgl3TEC+QcsmeAPwLPNtTAzEKB+4D3AljHQTPz5pHl5kJxUTiV1ZWEhWi6nYiIdAwPPvhgsEuQOgLWQ+acmw8UNNHsOuCfQG6g6mip+HjfnSV7tDisiIiIBFbQJvWbWS/gXODxYNXQmFoT+3WnpYiIiARQMO+yfBi4xTlX1VRDM7vKzDLNLDMvL68NSqu7OKwCmYiIiAROMCdGZQAv+BadSwTOMrNK59xrdRs65/4C/AW8uyzborjaj09SIBMREZHACVogc86l7fvdzJ4B3qovjAVLzQeMq4dMREREAimQy17MBT4DhphZtpl938yuNrOrA/WeralmD5km9YuISCBt2LCBESNG1Np2xx13NHk3ZGZmJtdffz0A5eXlnHbaaYwePZoXX3yx1WssLS3l5JNPpqqq8ZlGs2bNYvjw4cyaNeuQ37O+hXFbQ79+/cjPz29w/549ezj77LM56qijGD58OLfeeqt/3x//+Edmz57d6jUFrIfMOXfRQbSdEag6Wqr2pP52dxOoiIgIGRkZZGR4a44uWbKEiooKli5d2uzjq6qqCA0NbVbbp59+mvPOO6/J9k888QR5eXm1nn3ZUvfccw+33XbbIZ+npqYC5T4/+9nPmDRpEnv37uXUU0/l3Xff5cwzz2TmzJmccMIJXHHFFa1alx6d1ABN6hcRkfZi4sSJ3HLLLYwbN47BgwfzySefADBv3jy+9a1vkZubyyWXXMLSpUsZPXo0a9eu5YMPPuDoo48mPT2dmTNnUl5eDni9Q3feeScnnngiL730EhMnTuTGG2/kpJNOYujQoSxatIjzzjuPQYMGcfvtt/trmDNnDtOmTQO8h4vPmjWLESNGkJ6e7u+Rmzp1KiUlJYwfP/6AXro77riDmTNnMnHiRPr378+jjz7q3/f8888zbtw4Ro8ezQ9/+EOqqqq49dZbKS0tZfTo0UyfPp3777/ff8yNN97IKaecAsAHH3zAJZdcAsDcuXNJT09nxIgR3HLLLf7zd+3alV/+8peMHz+ezz77zL+9tLSUKVOm8Ne//rVWrZ07d2bSpEkAREREMGbMGLKzs/37+vXrx8KFC1v0t2yIAlkDNKlfROTIZBaYn0NVWVnJwoULefjhh/n1r39da1/37t158sknmTBhAkuXLqVXr17MmDGDF198ka+++orKykr+/Oc/+9tHRUWxYMECLrzwQsALHfPnz+fqq69m2rRpPPbYYyxfvpxnnnmGHTt2sHfvXtatW0e/fv0AeOWVV1i6dClffvkl77//PrNmzWLr1q288cYbdOrUiaVLl/K9733vgGv45ptveO+991i4cCG//vWvqaioYOXKlbz44ov897//ZenSpYSGhjJnzhx++9vf+s81Z84cTjrpJH8QzczMpLi4mIqKChYsWMCECRPIycnhlltu4cMPP2Tp0qUsWrSI117zpqaXlJQwYsQIvvjiC/8D04uLi/n2t7/NxRdfzJVXXtng575z507efPNNTj31VP+2jIwMfy2tRYGsAZrULyIibcUaSGw1t5933nkAHHPMMWzYsKHR861atYq0tDQGDx4MwOWXX878+fP9++uGpalTpwKQnp7O8OHDSU5OJjIykv79+7N582by8/OJjY31t1+wYAEXXXQRoaGh9OjRg5NPPplFixY1eZ1nn302kZGRJCYm0r17d7Zv384HH3zA4sWLGTt2LKNHj+aDDz5g3bp1Bxx7zDHHsHjxYnbv3k1kZCTHHXccmZmZfPLJJ0yYMIFFixYxceJEkpKSCAsLY/r06f5rDg0N5Tvf+U6t802bNo0rrriCyy67rMF6Kysrueiii7j++uvp37+/f3v37t3Jyclp8noPhp4H1IBaPWQKZCIiRwzXJosr1ZaQkEBhYe3/1hQUFJCW5l+QwD8nKzQ0lMrKykbP55q4iC5dutR6ve/cISEhteZ+hYSEUFlZSadOnSgrK2v2+RtS89z7rsM5x+WXX869997b6LHh4eH069eP2bNnc/zxxzNy5Eg++ugj1q5dy9ChQ8nKymrw2KioqAPmvp1wwgm8++67XHzxxQ0G4quuuopBgwZxww031NpeVlZGp06dmrrcg6IesgbUnENWsEeBTEREAqdr164kJyfzwQcfAF4Y+9e//uUfXjtYRx11FBs2bGDNmjUAPPfcc5x88sktri8uLo6qqip/KDvppJN48cUXqaqqIi8vj/nz5zNu3LgWnfvUU0/l5ZdfJjfXu4GuoKCAjRs3Al4Iq6io8Lc96aSTePDBBznppJOYMGECjz/+OKNHj8bMGD9+PB9//DH5+flUVVUxd+7cRq/5zjvvJCEhgWuuuabe/bfffjtFRUU8/PDDB+zLyso64K7YQ6VA1oCoKOjUuRqqIygsqmj6ABERkUPw7LPPcvfddzN69GhOOeUUfvWrXzFgwIAWnSsqKorZs2dzwQUXkJ6eTkhICFdffWirTp1++uksWLAAgHPPPZeRI0cyatQoTjnlFO6//3569uzZovMOGzaMu+++m9NPP52RI0cyefJktm7dCng9VCNHjmT69OkATJgwga1bt3LcccfRo0cPoqKimDBhAgDJycnce++9TJo0iVGjRjFmzBj/TQgNefjhhykrK+Pmm2+utT07O5vf/OY3fP3114wZM4bRo0fz5JNP+vf/97//5bTTTmvR9TbEWtrtGCwZGRkuMzOzTd6rV+8qcraE0u3mkey6b1mbvKeIiLS9lStXMnTo0GCX0a4tWbKE3//+9zz33HPBLiWoGvsc6vsemdli51xGU+dVD1kjkhK9j2f3zgiqXXWQqxEREQmeo48+mkmTJjV7Ha+OKj8/n7vuuqvVz6tJ/Y1ISPBN8tsTT1FZEXGd4oJbkIiISBDNnDkz2CUE3eTJkwNyXvWQNUKLw4qIHDkOtyk80r4c6vdHgawRtR+fpEAmItJRRUVFsWPHDoUyaRHnHDt27CAqKqrF59CQZSP8i8NqLTIRkQ6td+/eZGdnk5eXF+xS5DAVFRVF7969W3y8Alkj1EMmInJkCA8Pr7UIq0hb05BlI2rOIdtZtjOotYiIiEjHpUDWCD0+SURERNqCAlkjaj1gXEOWIiIiEiAKZI1QD5mIiIi0BQWyRtSa1K9AJiIiIgGiQNaIuDgwc1AWR0FJUbDLERERkQ5KgawRoaHQLdp7ZlfejsogVyMiIiIdlQJZE2LjvIeKFxbooxIREZHAUMpoQkKi94DxnQWhQa5EREREOqqABTIze9rMcs1seQP7p5vZMt/Pp2Y2KlC1HIoeiV4QKy6K0DPOREREJCAC2UP2DDClkf3rgZOdcyOBu4C/BLCWFktM9D6i6j2x7N67O8jViIiISEcUsEDmnJsPFDSy/1Pn3L61JD4HWv5EzgCq9YBxLQ4rIiIiAdBe5pB9H3g32EXUR2uRiYiISKCFBbs7rJFGAAAgAElEQVQAM5uEF8hObKTNVcBVAKmpqW1UmafWav3qIRMREZEACGoPmZmNBJ4EpjnndjTUzjn3F+dchnMuIykpqe0KpGYPWTw7y3a26XuLiIjIkSFogczMUoFXgEudc1nBqqMpGrIUERGRQAvYkKWZzQUmAolmlg38CggHcM49DvwSSAD+ZGYAlc65jEDV01K1J/UvDWotIiIi0jEFLJA55y5qYv8PgB8E6v1bi3rIREREJNDay12W7VbNOWSa1C8iIiKBoEDWhK5dITSsCiq6kL+rONjliIiISAekQNYEM+gWWwFAbn5VkKsRERGRjkiBrBli46oB2NHgwhwiIiIiLadA1gwJCd5DxXcWhAa5EhEREemIFMiaISnRC2JFOxXIREREpPUpkDVDzyRvdZCSokicc0GuRkRERDoaBbK6qqshOxu2bfNvSkr0AllVSSx7KvYEqzIRERHpoBTI6rrrLujTB/7wB/+mWg8Y1+KwIiIi0soUyOrq39/7d+1a/yYtDisiIiKBpEBW14AB3r9r1vg36fFJIiIiEkgKZHUNHOj9W6OHrOYDxneW7Wz7mkRERKRDUyCrKynJe17Szp1QUADU6SHTkKWIiIi0MgWyusz295L5hi01qV9EREQCSYGsPvvmkfmGLWtO6i/Yo0AmIiIirUuBrD51JvZHREBk573gwtheUBrEwkRERKQjUiCrTz0T+7tG7wVge15lMCoSERGRDkyBrD51hiwBYuK9IJafr0cniYiISOtSIKtPnUn9APHxXhArLLRgVCQiIiIdmAJZfXr18iaObdsGJSUAJCZ4QayoMCyYlYmIiEgHpEBWDxcSQvWAft4L37DlvgeM794ZEaSqREREpKNSIKsjO/sRFiyIZuMlvrlivkCW3N0LYiVFUcEqTURERDqogAUyM3vazHLNbHkD+83MHjWzNWa2zMzGBKqWgxES0pmqqmLKevmGJn3zyHomhQNQVRJDWWVZsMoTERGRDiiQPWTPAFMa2X8mMMj3cxXw5wDW0mxRUX0BKE+o8Db4esgSE32T+Uvj9fgkERERaVUBC2TOuflAQSNNpgHPOs/nQKyZJQeqnubaF8jKOu/2NtRdrV+PTxIREZFWFsw5ZL2AzTVeZ/u2HcDMrjKzTDPLzMvLC2hRkZGpAJSH7MCF4B+yjI/3NShNYGfZzoDWICIiIkeWYAay+hb0qnfVVefcX5xzGc65jKSkpIAWFRraifDw7jgqKU802LQJ9u6t3UOmIUsRERFpRcEMZNlAnxqvewM5QaqllqgoXy9ZeneoroYNG2o9YFxDliIiItKaghnI3gAu891teSxQ5JzbGsR6/CIjffPIhid6G9auJTYWsGoojyV/t4YsRUREpPUEbNl5M5sLTAQSzSwb+BUQDuCcexx4BzgLWAPsAa4IVC0Hyz+xv39nb8PatYSEQKeuZZTu7kxOnpa9EBERkdYTsEDmnLuoif0O+HGg3v9Q+Je+6BnqbfBN7O8S6wWy7bkVwSpNREREOiCt1F8Pfw9ZXLm3wbf0RXRsJQC5+dVBqUtEREQ6JgWyevjnkEUVeRt8PWRx8V4QK2hsdTURERGRg6RAVg9/DxnbvXU41q+H6mr/nZY7C0ODVpuIiIh0PM0KZGb2EzOL9t0R+ZSZ/c/MTg90ccESFhZLaGg3qqtLqByQBOXlsGULSQnex7WrMDzIFYqIiEhH0twespnOuV3A6UAS3h2Rvw1YVUFmZvt7yY72Pc1pzRp6do8AoKQoMliliYiISAfU3EC2b1X9s4DZzrkvqX+l/Q7DP4/sKN8zk9auJaWHF8RKd3cOVlkiIiLSATU3kC02s3/jBbL3zKwb0KFvNfT3kPWL8jasWUOv7t7vlcXRVFRp6QsRERFpHc0NZN8HbgXGOuf2ABG0o4VcA8G/FlkP34a1a0lI8HUKlibo8UkiIiLSapobyBwwDLje97oLEBWQitqJyEjveZZl0aXehjVr9IBxERERCYjmBrI/AccB+1bf3w08FpCK2gn/kGWEL3itXUtCvPN+L01gZ5meZykiIiKto7mBbLxz7sdAGYBzrhBv2LLD8geyyi0QEwO7d5Pg8r2dpfEashQREZFW09xAVmFmoXhDl5hZEh18Un9ERE/MIqis3EHV0H4AdM5ZQ0j4XqjsxLaCXcEtUERERDqM5gayR4FXge5m9htgAXBPwKpqB8xCiIzsA0DZSG8tMlu3lqhuewDYklsWtNpERESkYwlrTiPn3BwzWwycirf+2DnOuZUBrawdiIrqS1nZWsoGR9MFYM0aukSXsacAtm7fG+zyREREpINo7qOTBgDrnXOPAcuByWYWG9DK2gH/0hepvpX5166lW6wXxHJ3VAarLBEREelgmjtk+U+gyswGAk8CacDfA1ZVO+Gf2J/kmy63di0xcVUA5Oe7YJUlIiIiHUxzA1m1c64SOA94xDl3I5AcuLLaB//jk7qWeBvWrCHet/RFYWFzPzoRERGRxh3MXZYXAZcBb/m2hQempPbD30MWkgdRUZCXR2KM10NWVNCs6XciIiIiTWpuILsCb2HY3zjn1ptZGvB84MpqH/xzyMo3Qv/+APQI2w1AcVFk0OoSERGRjqVZgcw597Vz7nrn3FwziwO6Oed+G+Dagi4ysjdglJfnUD3YC2Qpzluhf8+uDv3kKBEREWlDzb3Lcp6ZRZtZPPAlMNvMfh/Y0oIvJCSCiIhkoJry4d0B6F22DYDy3V2CWJmIiIh0JM0dsoxxzu3Cm9Q/2zl3DHBaUweZ2RQzW2Vma8zs1nr2p5rZR2a2xMyWmdlZB1d+4PnnkQ3oCkDvws0AVBbHUFVdFbS6REREpONobiALM7Nk4Lvsn9TfKN+jlh4DzgSGAReZ2bA6zW4H/uGcOxq4EO8h5u2Kfx5Zb+8ehqTcb7wdesC4iIiItJLmBrI7gfeAtc65RWbWH1jdxDHjgDXOuXXOub3AC8C0Om0cEO37PQbIaWY9bca/9EVCBQAJ2V96O/Yk6AHjIiIi0iqa++ikl4CXarxeB3ynicN6AZtrvM4GxtdpcwfwbzO7DuhCM4ZB25p/yDJqN4SGEp+z3NtRFkfBng0QH7zaREREpGNo7qT+3mb2qpnlmtl2M/unmfVu6rB6ttVd3v4i4BnnXG/gLOA5MzugJjO7yswyzSwzLy+vOSW3Gv+QZcVm6NuXcCoIjSoBF0p27u42rUVEREQ6puYOWc4G3gBS8Hq+3vRta0w20KfG694cOCT5feAfAM65z4AoILHuiZxzf3HOZTjnMpKSkppZcuvw95CVbYQBAwCI7FwEQPb20jatRURERDqm5gayJOfcbOdcpe/nGaCpZLQIGGRmaWYWgTdp/406bTYBpwKY2VC8QNa2XWBN8M8hK9uEG+gFss6dvECWs708aHWJiIhIx9HcQJZvZpeYWajv5xJgR2MH+J59eS3ezQAr8e6mXGFmd5rZVF+zm4ArzexLYC4wwznXrp7aHRbWlbCweJwrZ+8Qby2yruHe3ZXb8yuCWZqIiIh0EM19IONM4I/AQ3jzwD7Fe5xSo5xz7wDv1Nn2yxq/fw2c0NxigyUqqi/FxQWUD+hKJBAdkg9AXl51cAsTERGRDqG5j07a5Jyb6pxLcs51d86dg7dI7BHBP48s2btPIa4qF4CCgqCVJCIiIh1Ic4cs6/PTVquinfPPI4v15owllnr3JuwsbG4Ho4iIiEjDDiWQ1besRYcUFZUKQFnVVkhJoXuVd9/B7p0RwSxLREREOohDCWTtavJ9IPnXIiv3lr7oWe3dz1CyKzKYZYmIiEgH0eiYm5ntpv7gZUCngFTUDu1f+mIjDMwgZclW7/WuzsEsS0RERDqIRgOZc65bWxXSntVeHPZ7pFZ4j0/aW6yPR0RERA7doQxZHjHCwxMJCelEVdUuKgcm03evN2RZVRxLtdPSFyIiInJoFMiawcz295L160QP51sTtzSeXeW7gliZiIiIdAQKZM3kn0fWA2IoAquEvdHk7toZ5MpERETkcKdA1kz+HrKwHVh8PKGRhQBs3Lo7mGWJiIhIB6BA1kx1l76IiPCGLTdv2xPMskRERKQDUCBrptp3Wg6gU7gXyLZsLwtmWSIiItIBKJA1U+21yAbSxRfItuXtDWZZIiIi0gEokDVT3R6ybiHek8Vz86uCWZaIiIh0AApkzRQRkQyEUlGxnaoBfYg1r4dsx44j5glSIiIiEiAKZM0UEhJGZGRvAMpTI0nAC2SFBaHBLEtEREQ6AAWyg+Aftuy6h6TQIgB2FVgwSxIREZEOQIHsIPgDWfkmesR6j0wqLggPZkkiIiLSASiQHYT9a5FtIjkxEoDSXZ2DWZKIiIh0AApkB6Hm0hd9+sQAUF7cLZgliYiISAegQHYQai59MWhwCgB7i3rz/pJVwSxLREREDnMKZAeh5uOTho0dQErif2BvN6ZOhYKdFUGuTkRERA5XAQ1kZjbFzFaZ2Rozu7WBNt81s6/NbIWZ/T2Q9RyqyMhUAMrLs2FgGv/d9T3C4lZRmj2EY89cT5XWiBUREZEWCFggM7NQ4DHgTGAYcJGZDavTZhDwf8AJzrnhwA2Bqqc1hIZGER7eA+cqKe8ZTr8Tj+aZzmdDpx2s/nwwl/xwe7BLFBERkcNQIHvIxgFrnHPrnHN7gReAaXXaXAk85pwrBHDO5QawnlZRc+kL5sxhevUeLkg/B0L28sJTPXjoUT3bUkRERA5OIANZL2BzjdfZvm01DQYGm9l/zexzM5sSwHpaRc15ZPTsCS++yN++/JTkU74PwE03hvGvfwWzQhERETncBDKQ1beEfd0HP4YBg4CJwEXAk2YWe8CJzK4ys0wzy8zLy2v1Qg/GvnlkZWUbvQ0TJtDp7vt4/evnsQl34qpD+M4FlSxfHsQiRURE5LASyECWDfSp8bo3kFNPm9edcxXOufXAKryAVotz7i/OuQznXEZSUlLACm6Omktf+N10E2PHncPPQ38Fw19gT3EYZ51VzbZtQSpSREREDiuBDGSLgEFmlmZmEcCFwBt12rwGTAIws0S8Icx1AazpkNUbyMxg9mx+saU/o469Anp/xubNIUybBqWlQSpUREREDhsBC2TOuUrgWuA9YCXwD+fcCjO708ym+pq9B+wws6+Bj4BZzrkdgaqpNdSaQ1ZTbCwRL73C8+9VE37+ORC7noUL4fLLobo6CIWKiIjIYSOg65A5595xzg12zg1wzv3Gt+2Xzrk3fL8759xPnXPDnHPpzrkXAllPa9j/+KRNOFdnStyoUYy483F+szAXLv4WFrmLl16CX/wiCIWKiIjIYUMr9R+k8PBYQkOjqa7eQ0VFPZ15V1zBT4dewYmlX+MuOB8LqeKee+CZZ9q8VBERETlMKJC1QIPDlj6hf3iMZ1YNpUvqf3BnXQPAlVfCAw+g1fxFRETkAApkLVDvxP6aOnViwLNv8uAnUZDxF6Im3E9lJdx8M5x0EqxZ04bFioiISLunQNYC++eRNRDIAAYM4Ic/ncsZa6Ds1FsYe/VNJCc7Pv0URo2CP/5Rk/1FRETEo0DWAk32kPnYOefwVK8fEVsKi3r+nvGzvsP3Lqpkzx647jqYPBk2Nn4KEREROQIokLVAU3PIaup196O8kDWSbuXw2q5XWTUwhT//JZvERPjwQ0hPh6efhro3bIqIiMiRQ4GsBZrbQwZAWBhnPP8Zn1fOYOAOWBqaxy/W9eWv9/+Z886D3bvh+9+Hb30Lcuo+x0BERESOCApkLdCsOWQ1de7MsPtns/CiDzhje1fyo6q5YP01TEo5juceLyY2Ft55B0aMgL//Xb1lIiIiRxoFshaIiOiOWQSVlQVUVhY3+7i4407h7YfymBUygcpQuC7xc+Z9lMTi377OmWdCYSFMnw4nnwxvvaVJ/yIiIkcKBbIWMAshKioVaN48sppCI6O4/xfzmXPsA0RVhfDU0DIuWXQOT3Y5i78+sJOYGPjkE/j2t735ZbNnQ3l5IK5CRERE2gsFshY66GHLOi4+42cs+OHn9AmJ47M+MDb1XUb+tTeb7vobv3vQ0bs3fP01zJwJ/ft7i8oWFbXmFYiIiEh7oUDWQvsn9m9q8TmO6TWWzJ9+w4Qe48iJhpO+W8KrT83gp08NZ+1Nf+LZx/cwYoQ32f/mm6FPH+/fLVta6ypERESkPVAga6GDWfqiMd27dOf9Kz/hRxlXUx4GM86FC4evZNNdP+bSm7qz7Pireeex9Uya5N2R+cADkJYGV1wBS5fqBgAREZGOQIGshQ5q6YsmRIRG8Kez/8zjZz9OZGgkL46AodcaV08qYevcJzjzx/35sGICi+5+j++eX0VVlfew8qOPhuHD4c47ISvrkMsQERGRIFEga6FDnUNWnx9m/JCs67K4YvQVVIcaT2TAwJ+GcetZERRmLiDj9im8OD+FrKt/z3UzdpOQACtXwq9+BUOGwJgxcP/9sGFDq5UkIiIibcDcYTbmlZGR4TIzM4NdBqWlG/jiizRCQ7ty9NGf0bXriFY9/8q8ldz+0e28svIVAGKsE7csi+b6N7bTpQIICaFi8ll8OORHvLB9Iq+825ldu/Yff9xxcOGFcMEFkJzcqqWJiIhIM5nZYudcRpPtFMhaxjnHV199i4KCdwgLS2DUqH/TrduYVn+fhVsWctsHt/HB+g8A6BERzy83pfGD2V8SUV7pNTKjbOwE3htyPS/smMwb86LZs8e/i5NOgqlT4eyzYfBgb5uIiIgEngJZG6iqKmPFivMpKHib0NAYRo58l5iY4wLyXu+ve5//++D/yMzxrr1/dF9+HjKR7324nS7vfVRrsbKSgaN4a+gsXtg5hXe+iGfv3v0JbMAA7zFNZ5/tBbXIyICUKyIiIiiQtZnq6r2sXDmdvLyXCQnpQnr6m8TFTQrIeznnePWbV/n5hz/nm/xvAOga0ZXvDT6PmXuGcNz732BvvQ0FBf5jihIH8M7wWbxdcTrvfp1Kwc5Q/76uXeG007yAdtZZGtoUERFpbQpkbai6upJVq2ayfftzhIREMXz4qyQkTAnY+1VWV/L3r/7OE4uf4NPNn/q3D0kYwsxRM7h0zyCS/7UAXn8d1q/3768ihM+Tz+PtxMt5e+fxLNscX+u8Y8bA5MkwcSKccAJ06xawSxARETkiKJC1Meeqycq6hq1bn8AsnGHDXiQp6dyAv+83+d8we8lsnl32LNuKtwEQaqGcNegsZo6+grPLUwn/9wfw8cewYAE1Z/5vog/vxF3C250v4IO8dEr3hvn3hYbC2LFeONsX0Lp2DfjliIiIdCjtIpCZ2RTgESAUeNI599sG2p0PvASMdc41mrbaayADb0hx7dqbyM5+CAhl6NBn6dHj4jZ578rqSv615l88veRp3sx6k8pqb8J/UuckpqdP57yh53FcyjjCli33wtnHH3sPzSwsBKCUKD7mZOZ1Oot5EaeTuWsQVW7/8GbdgHb88epBExERaUrQA5mZhQJZwGQgG1gEXOSc+7pOu27A20AEcO3hHMjAC2UbNvySjRvvBozBg/9CSsoP2rSG3JJcnl/2PE8veZoVeSv82xM6JXDWoLOYOmQqpw84nejwrvDVV7UDWl4eALvpyn85gY+YxLzw01lcObJWQAsJ8RalHT9+/8+wYV5wExEREU97CGTHAXc4587wvf4/AOfcvXXaPQy8D/wM+NnhHsj22bjxXtavvw2AgQMfoXfv69u8BuccmTmZzF0+lzez3mRNwRr/vvCQcCalTeLbg7/Ntwd/m76xfb3nMG3cCIsW7f9ZvBh272YX3fgvJzCPiXzEJJZwNJWE13q/rl0dGRnGscfuD2m6UUBERI5k7SGQnQ9Mcc79wPf6UmC8c+7aGm2OBm53zn3HzObRgQIZQHb2I6xZcwMAaWn3kpp6CxakRcCcc6zasYo3Vr3Bm1lv8unmT6l21f79I3uM5NuDv83k/pMZ12scncI7eTuqq73nMtUMaUuWUFpuLOFoPudYvmA8XzCejfQ74H17p1Qx6ugQ0tONkSMhPd17qkB4+AFNRUREOpz2EMguAM6oE8jGOeeu870OAT4EZjjnNjQWyMzsKuAqgNTU1GM2bmy9xxUFWk7Ok2RlXQU4YmNPYeDAR1p9Vf+WyN+Tzzur3+GNVW/w3tr3KN5b7N8XERrB2JSxnNT3JCakTuD4PscTExWz/+CKCli1yhvu/OorWLYMvvqK7ZvK/OHsC8aziLHsIuaA9w4Pq2bowArSx4STPjLEH9R69dKitSIi0rG0h0DW6JClmcUAa4F9SaAnUABMbayX7HDqIdsnN/clsrKuprKyAAilV68f06/fHYSHxwW7NADKK8uZt2Eeb69+m/kb57Ns+zIc+78XIRbC6J6jmZA6wR/SkrokHXiinTth+XJ/SKtetpzVX+7hq5I0ljGSr0jnK9JZy8B664jutJej+u9l6Mhwho6KZOhQGDoU0tIgLKzeQ0RERNq19hDIwvAm9Z8KbMGb1H+xc25FA+3n0cGGLGuqqChg/fpfkpPzZ6Ca8PBE0tLuITl5Jt79D+1HYWkhn27+lPkb5zN/03wyczL9d23uMyh+EMekHENGcgbHpBzD0T2Prt2Lto9zsG0bfPON9yT0lSspXr6BFSvgq7we/qC2jJEUkFBvPRGhlQxOKWbokGqGjunMUaOjGDzYewyU7vQUEZH2LOiBzFfEWcDDeMtePO2c+42Z3QlkOufeqNN2Hh04kO1TXLyM1auvp6joYwC6dh3DoEF/ICbm+CBX1rCSvSV8seULPtn4CfM3zeezzZ9RWll6QLt9Ie2Y5GPISMloOKTts3u3P6i5r1eSt3y7l9k2d2VlxQBWMpSVDCWbPg2eIrnrLgb32sPgwY7Bozoz+JhuDD4qhP79ISKiNa5eRESk5dpFIAuEwz2QgTfBPi/vJdauvYny8mwAevS4hP797yMyMiXI1TVtb9VeVuSuYPHWxSzOWczirYv5cvuX7K3ae0DbgfEDSe+ezojuIxieNJwR3UcwKGEQEaGNpCXnYOtWWL0aVq9m9/KNrPqyjJWrw1i5LY5VVQPIYjCrGUQ5UfWeItSq6BddQP8ee0jrW03akAj6j44mLb0raf2NhATNVxMRkcBTIDsMVFWVsGnTfWzadD/OlRMS0oV+/X5Br14/ITS0/qDRXtUX0pZtX0Z5VfkBbcNCwhiSMITh3YczImmEF9a6D6d/XH/CQpqYLFZd7YW1tWupWrOe7CV5rFpeQdb6cLK2x5BV1ocsBrOBfjhCGjxNt7A9pMUUkNajlLS+1fQbHEGf4d1IHRlHn36hdO+uwCYiIodOgewwUlq6nrVrbyI//1UAwsMTSU6+kpSUHxEV1fBwXXtXUVXByvyVrMhdwfLc5SzPW86K3BWsK1xX66aBfcJDwhkQP4AhCUMYnDB4/7+JQ0jqnNS8JUN27YJ16yhbuZ51iwtZ/0056zcY67d1Yn1RHOsqU1lPWr13f9YUaeX06VJAn7gSUpP30ic1hNTBUfQZEeP99A0hOrqln4yIiBwpFMgOQwUF/2HdulspLv6fb0soiYnn0Lv3dcTEnBS0NcxaW8neknqD2uZdmxs8JiYyhiGJXkAbHD+YtLg0+sf1p39cf3p06dG8z8Y5KCzEbdhI4Yoc1i0pYv2qvazfaGzK7cSmohg27e3BZvo0eINBTd1CS/yhrXePCvr0gT6Doug9tBt9RsXTKy2Cbt3U0yYiciRTIDtMOefYtetTtmz5I3l5L+Ocd3djly7p9Op1HT16TCc0tHOQqwyMkr0lrC5YTdaOLFblryKrwPt31Y5V7Crf1eBxncI6kRaXRlrs/pC27/e+sX2JjjyIrqyyMtiyhZKsLWxeVsimb/awaX0Vm3NC2JTfhU2748iu9EJbKU3/HbqE7CElqpCU6GJ6JZaTkuxISQ0jZWBnUo6KptfwWJJ7h9KpU/NLFBGRw4cCWQdQXp5DTs4T5OQ8TkVFLgBhYXEkJ3+flJRr6NQpLcgVtg3nHLkluV5Q27GKtQVrWbdzHesK17G+cD07Snc0enxMZAx9Y/uSGpNK35ja/6bGpJLcLZkQa3i+2QFKS3FbcihcuY3Ny4vYnFVK9sYqNueEsnlHZ7J3R7O5vDs5pLCHLs06ZUzobpI77SQ5uoTkhApSkqtJ7hNOcv9OJA+JJnmoF9zU4yYicnhRIOtAqqvLyc19iS1b/sDu3Qt9W434+Cl0734hiYnTCAtrfE5UR1ZUVsT6netZX7iedYW+oLbT+31T0aZ6l+ioKTwknF7Rvegd3dv76da79uvo3vTs2rPpGw5qqqrC5eaxK2sbOSsKyVldQs76cnK2OLbkhpNT1Jmckhi2VPZgGz3ZS2SzThtlZfSIKqJHlxJ6xpXTI6mKHj1D6NEngh4DutBzcAw90jrTowdERyu8iYgEmwJZB7Vr10K2bPkjubkv4py3zIRZBPHxZ5CU9F0SE6cSFqbZ5vs459hRuoONOzeyqWgTG4vq/LtzI3l78po8T4iFkNw1mV7RvUjplkJK1xTv3zo/8f/f3r3HSJaW9x3/PnW/V1f1ZWaY2WWXZS+ssdl18IINMRcTs9gOWJFJwDgiCRZygi1bDrFwIjkJkmVbjuxEiRMLYQKKnJib7awcY0BcDEI2sJiF3WXZXfY+Oz19r/s5p87lzR/nVNWp6uqZ6enuqerp5yO9et/znlPVb/XZqf31e275+v7O9bNtzOpFth/dYPWRBqtP9LjwjMvqBcPqZprVZp7VbpUL3jKrnLmiw6QDWXFYybVYKfU4teCwshhw6kyClXNpTt1UYOXFFVZemGd5GZaX9fmiSil1FDSQXef6/Q02Nj7JxsbHaDS+CNFViyJZFhffxPLyP2Zx8R+SSpVmOs7jwHItnm8/z/nW+bES71vrrE29MnRSJpkZhrPTpdOcKZ3hdOn0WPtM+QwrxZX9zbj1+3DxIp2nNlh7tMHaE6LR9GEAABziSURBVB0uPuOwdsFnbSPB2k6atXaRNavCRbPCOit02N9jDBbSHZbzXVYqNst1j5VlWD6TYvmGHMs3F1m8oUh9UajXoV6HahUS+zjSq5RSJ5EGshPEcS6yuflJ1tc/RrP5ZQbhLJHIUa//JCsrb6VWeyPp9MJsB3qM9f0+q+1Vnm8/z4X2hT1L02le0fsJwlJhiTPlM5wqnmKluDIsk8srxRXy6Ss869+Y8NYfa2v0ntlg/bEG6091WXuuz/pqGN7Wd9KstQus2RU2gkXWWWGTJQL29wivhATUsj3qxT71qsdizVBfTrB0Os3SuRxLZ7MsLQtLSwzL4qI+l1QpdbJoIDuhHOcCGxufZH39o7RaX4mtSVKpvJLFxTdRr99LqXQ3sp8T2dUV6fa7rHZWudC+wMXORS52LrLaXuVi9+LY8np3/Ypm3AbKmTLLxWWWC8ujurDMUmFpV/9SYYlSpnT5Q6fGhI+vWlsjWF1j56kG60+02XjOZuOCy/o6bGwl2Ghl2OgW2fbKbFMfliZXF/BreYulSp/FBZ96XagtJ6mfzlA/nR2bgRuUWi0sGuSUUseRBjKFbZ9nY+MTbG7+Oa3WV4a30ABIp1eo199IvX4vtdqPk8kszXCkJ48XeGz2NofhbK27xnp3fay91hn1uYG7r/fPJDMs5hdZLCyO19P6CovU83Xq+fqlD6M6DmxswPo6rK/jrm7SeLbF9nNdtlcdttZ9trZgq5Fks51l06uyydJY2WJx3zNxA5V8n3rFp14z1BYT1E+lqC+nhqEtHt7iRa9MVUrNkgYyNcbzWuzsfJ7t7U+xvf0pHCd+E1ahXP6hKJy9nnL5HpJJvTHWvDDG0LAbrHfX2extstHbYKO7MWr3onZ31O65vav6WZVsZSykDULcILBNK7V8bXqQ6/XCADcIcRsb+GubNM532HjOZmfDZXsjYHtH2G6l2HYKYzNwg7LFIg0WLvkorEtJJgJqFT8MaIuJsNSEWg0WFrhkXa1C8uryo1JKARrI1CUYY+j1vhuFs7+i0fjr4RWbEF61WancQ7X696lWf5Rq9Uf0ys1jxnIttq1ttqwttnpbbFlbbPY2h+1Bf3ybHXuHwARX9fMq2Qq1XG0Y0BZyC9RyNWq5qJ0fby/kFljILVDNVsmlcuHhVceBra0wwG1ujsLcxgbB5jbN1R476y7bGz7b27DdSLDjjx9G3aHGDrWx9pXeC27Pz1YOWKgJCwvCwgKXLNXq7mUNdEqdbBrI1BXz/S6NxhfZ3v40jcaX6Ha/DWPnNyUole5iYeFHo4D2ajKZ5VkNVx2RwAQ07ebUsLZlbbFj7bBtb7NthWXH2gnrAwQ5CO8DV81VqWarYUiL2sM6aleyFarZqM5VqWTKVP00lU6fUsNCtrbCQLe5OVb31xvsrLvsbPrs7MCOW2SHGg0WLls3qV71zNxAuTwe1qrVsFQqV1aqVcjn9bCrUseVBjJ11Vy3Qav1FRqNL9Fsfpl2++tj558B5PO3U6ncQ7l8D5XKPZRKLyORuLKbm6rrS2ACWk5rLKg17AY7dlRbO6N2rK9hN2g6Tfp+//I/5DIEGQW1KLgNAt34coWq5Kk4ULUMla5Hpe1QadpUdiyy280wzEXF39yhve3SaEDDVGiwMLXsUKNJdVd/i8qBAx2EFzQMwlm8XGlftQqlkt6mRKlZ0ECmDo3v92i1/pZm88s0Gl+i1fobgmD87vciaUqllw0DWrl8D4XC7Xolp7os27Np2k2aTnNX3bAbNO0mLadFy2nRdJrjdbSu63YPZSzZZJZKtjJWytky5XSJsmQp+ynKXoJyH8qWoWx5lNp9yi2bctOmvNOjtNWmvNmisL4DWw3aQWFXUGtTpkVlahmsa1KlJVWaVLFN7sCfTSScratUwvpK23sVPRSr1JXRQKaOTBD06XYfpNX6Gu3212m1vkav9x2YuI1DMlmmXP57lEo/SKl0F6XS3RQKt5NI6C3h1eHyAm8U2qIwF28PglvTGa2LB7rBOi/wLv/D9qGUKVFKFSmn8pTIUiIzDHUlVyjbhrLlU+66lDr9MNQ1LMrbHUotm7ID5T5knDR+v0rXVIdhba8Srl+gmarTlBpNKjSDMl3/cC/UKRRG4axU2t2eVk8GvcGyzt6p65kGMnVNeV6bdvsbtNtfi4La1yau5AyJZCkWX0q5fPcwpBWLP6BPFFAzZ4zB8Z2xkNZyWrT7bdpOe3oda3f6HTr9Dm0nbB/WrF1cTtKUyVIyacp+kpIbztaV7IByz6fU7VNq9yl1XUp9xkrOSSJOBdMvY/plfLeM65WxggodyrRjpZVeop2u004t0E6EIa8dlGj7edpenrZz+KcnFIujgFYshiFtWj3ZFw98kyWTOfRhKrVvGsjUzDnORdrt++l0HqDT+SadzgPY9pNTthTy+RdTLH4/xeJLKRa/j2Lx+8jnb9PZNHVs+YFPz+0Nw9ogqA1CXLw93GYi9A1DXrT+IBdP7CVphKInFF2h6BhKdkDRhYILxT4Uo7rgjtr5vpByiiTsMvTL0C9h+iV8t4zvV3DTy7ipJZzUElaqRidZo50ID8e2gyJtr0DLzdF2srTtFB3raO76m8lMD2rTSjzUxcPeZMnl9AILtT8ayNRc8rwmnc63xkJat/swxuy+8alIinz+9mFAC8tLyeVuIbGf50AqdR0wxmB51liYiwe9PYvbGZu967pduv3usN7vTYf3KxmEYa4QC3mDcFeIBby0XSRlVUjaZZKmBrKIUIdkjYBFgkQNnxqeVPFkgT4V+qZEzytieXk6/QwdO03HStLpCu02eId7BBoID60OZuripVC49PJeM37xOpvVsHc90kCmjo3wnLTv0O0+RK/3MN3uw3S7D2HbT03dXiRDPn8rhcIdFIsvoVC4g0LhJRQKt5NMHuyeU0qdNK7v7gppnX6Hntvb1b+rzw374qXrdOhF2/Y8C9ccQSqaIh8FvLwLBU/I+0lyQYGsWyPl10n7NVJBjWRQJ2HqiKlhgjrGVAmCBfyggudVcP0irlvAsbM4vRR2L0Wvl6TXFbodwXGOLjElEnsfmp3sKxRGoW/QnrY8KPm8Br5Z0UCmjj3f79LtPrIrqE07N20gm70hCmd3ROV28vlbyWbP6hWfSs2A67vDIDcMbf1YO+ofBj6njdVr0us26PVa9Ow2PadNrx+FPt+i5zt0jYMlHpb4OInDP5S7l4SXpuAukHWrZN0FMsECaX+BtKmRCqokggWSpooEVcSrgFvA9IsETh7fyeE7OVwri2tn6FsZHDuN00th9ZK4/aP9jhIZhbN4HZ/Nu9xMXrG4+/XxOq1nmewyF4FMRO4F/guQBD5ojPntifW/Cvw84AEbwL8wxjxzqffUQKY8r41lPUa3+wi93nej8giW9fjUQ58AiUSefP7F5PO3USjcFs2whXU6vXz5B3ErpeaWH/jYnk3P7WG5PXqdHayddXqNTazWJr3WNlZnJxbyWvTsDr1oJrDnhSGvZxws49ITDysFVhp6acba/lFmJj8JbhH6JehHtVsk5dXIBDUyXp2Ut0DSL5P0yyS8EgmviLhFTL8Ibp6gnyfoZ/H7WTw7i9dP49pp+nYKzz36e5WkUpcObPEZu8m+/SzncsfnytyZBzIRSQKPAf8AOA98HXi7MeY7sW1eB3zVGNMTkX8JvNYY808u9b4ayNRegsDDtp+i1xsFNct6nF7vMVx3fc/XJZPVKKzdQj7/InK5W4btbPYc4X/KSqkTwxjodqHZ3FXcxjZWcxOrtYXV3sHqNMIZPauJZbWx7A5Wv4vV72ElAqw02KlRqBu07YllayL4DfsOc8YpSICbB7cQFi9q94tRXYoFwhIpb4GUXyHhVkm4ZcQtg1tE3DzGy2H6OQI3FwXADJ6TwQTX7vsyl9s74OXzV1aWl+HNbz7acc5DIPth4D8YY94YLf86gDHmt/bY/m7gvxljXnWp99VApq6G5zXp9R7Hsh4bhjTLeoxe7zF8v7Xn60Qy5HI3kc/fQi73oiiw3RyVm0inF67hp1BKHRvGgGWFQa7VGq8HZXK53R7fttUi6LRxklFImwxrqen98badAicZ1fk0di6Fk0liZxM46QR2FA7tRICdCLDwsHBx8a/uc/upUegbBL5pIXBqX2FsncRmAPHCdcbNEfRzGPfgN0sGuPlWmycfO5z32suVBrKjvFTtLBA/2ec88IpLbP8u4FNHOB51gqVSVSqVl1OpjP+bMMbguhtY1hNY1hPY9hNY1pPDdr9/MQpxj+3xvgvkcjeNhbRR/UJSqfK1+HhKqXkzOGGrUIAzZ676bRJBQL7TIR8PcO327qA3Gfom614PcKNyeb4wDGuD8DdoO8UcdqWAU85jl3LYxSx2MYuTz2DnU9i5FHYpiZNJYI+FPh870cKWHWw87KCP5Vk4noPt2diejeOH7b7fx8DesTAQ8HJ7hLp8tHz5urnsAe++6v1zmI4ykE07KWfqdJyI/BzwcuA1e6x/N9Fv7MYbbzys8SmFiJDJrJDJrFCt/vCu9b7fxbKewrYHIe1JbPvpqO8pPK8R3cLjganvn0rVyGZvJJd7IbncjcP2oM5kTunFBkqpvSUSoyfNH4TnjQe3yRm6wexcVCfbbYqtFsWJfjbaYGzAPvhnGzyktVKB8kKsXSaolOlXitiVPE4xDH5WMYOdT2Pl09i5JFY2iZUR7LRgGRfLtbA8Kwx2UchzfAfHc3D8rYnlsL6xeiMnIZCdB26ILZ8DLkxuJCJvAP4d8BpjjDPtjYwxHwA+AOEhy8MfqlLTJZNFSqWXUiq9dNe6cHZtE9t+Ctt+Gtt+KgpqYdtxnsXzdvC8Hbrdb019f5EM2exZstkbyGbPxepz5HJhO7zoQEObUuoAUimo18NyEEEQzrbFZ+amlSkhb6zdboPjwPZ2WCYkgFxUrkguN3rUQ6Uy/jyvchnKS+PLpRLUyrC0dLDfxyE6ynPIUoQn9f8Y8DzhSf0/a4x5OLbN3cAngHuNMY9fyfvqOWTquBgcDrXtZ3GcZ7DtZ7HtZ3CcUe26m5d9H5F0FNrOkcmcJZt9Adns2Yn2C0gmD/dZhUopdaQcJwxm0wLb5ZYnX+Nf5Tlvt90Gjz56uJ9rwszPITPGeCLyi8CnCW978SFjzMMi8n7gfmPMfcDvAiXg49FtB541xhzx9Q5KXRvxw6HhEfndfL+L4zyP45zHcZ6L6vPY9qjteVvRrNvTl/x54eHRMJxlsy8gkzkTtcM6kzlDNnuGROLwn0OolFL7ls2G5aCzVIMLKCZDWrsNnc6oPVk6nQOd33fY9MawSs0537eG4azfvxAFuOdj7Qv0+xf2vAfbpFSqHgtsg3KaTOZ0FN5Ok8mcIZks6/3ZlFLqgGY+Q6aUOhzJZJ5C4VYKhVv33MaYANfdjMLZ8zjOKv3+Bfr91ahvUF/E87bxvG263Ycu+XMTifwwnGUyp6L2KdLpU9HyqeFyKlU67I+tlFInigYypa4DIonY4dG79tzOGD8KboPAdpF+fzWqL0b9YV8Q9KILFqY/UzQukSjEQtrpWGg7PRbcMpnTGt6UUmoKDWRKnSAiyWFAulRwg/ARVYOgFoa2Nfr9NVx3bdgeLF9teEunl4clk1meupxMFg7p0yul1PzSQKaUmiqVKpNKlS95qBTCq0l9vx0FtItTQlu87+K+whuEAS4MaUux0La0qx6sS6UW9DYhSqljRwOZUupARIRUqkIqVdlXeHPddfr9DVx3VKYtB0EPx3kGx3nmCkeUIJ1enBLcBqEtbKdS9Wi7RZLJil7AoJSaKQ1kSqlrJh7e4NLhDUYBznU3o5C2GQttm7HwNujfxPebw/4rH1dqLKCNtxdJp+tR36hOpxdJJAoa5JRSh0IDmVJqbsUDXD7/oit6TRD0cd2tKLBtjoW5UXtrWDxvC9/v4LrruO76PseXmQhrtahd27MvXK6SSKSv5leilLpOaSBTSl1XEokM2Wx4E9wrFQQOrrs9DGjjgW0Hz9uOlrejdlgHgTW88GH/4yySSi1MLWF4i/fVojJoV/Q8OaWuMxrIlFInXiKR3XeIg/CmvaOAtjMMaqP2qI5v53lNgqBLv9+l33/+KkYspFLViZA2Geym9S2QSlVJJkt6qFWpOaOBTCmlrlIymSeZPEs2e3ZfrwvPjevgeY0pZWeibuC68b4dfL89XHd1EqRSFZLJahTsqlFQiy8vTPQtjK3T8+eUOlwayJRS6hoLz40LbysCN+z79UHg4fvNibA2LdRNK+Hs3GDZca72UyRjQa4Snes3ao/q6sRyeWxZg51SIQ1kSil1zCQSKRKJ8CrQfH7/rw8CF99v4XnNYQkD3rTlxkTfINRZw8dwHfDTDINaPLSNgtvl2uWoXYoOxeq5dep40kCmlFInTCKRHga6qxUE/SiktfC8VizgjdqT68JDrYPlNr7fioLdQQ6/Tn62YhTSSsPQFg9syWQptn6ylHfViURWZ/DUNaGBTCml1L4lEhkymWVg+UDvE87WtaeGtbBvst2ObRMvHXy/M7xY4rCIpCaCWjgrl0gUSSaLJJMFEolCVI8vJ5PFqD3YthR7XUlvfaLGaCBTSik1M+FsXXjPtoMyJsD3u7GAFg9x3bHgNt6Ob9sZe70x7vBiisMmkp4S1EbBLh7oRsFuMvjlY4FwVCcSeZLJPCLJQx+3OhoayJRSSl0XRBKxiyUORxA4E0EtHvB6BEFvjzpc7/vdqD0onWEdhr3DO1w7TSKRmxrYpge48ZA36hvVYX8u2iZe9NDuQWkgU0oppfaQSGTJZLLA0qG+rzEGY/pjQS0Mc90pgW68nh4AranhMAhsgsA+hIsvLi8Mf4OAlhvO0sWXB+3d/dmoPahziIzao/78sC98jxwimesiDGogU0oppa4xEYkCR/ZQDtdOY4yJAtl4UBuFu14s7FkEgTUMd+Mhz5oIfoNiR+ssjOkPwx8c/uHdS5NYcMtPCXHTQl5YZ7NnufHGX7vG451OA5lSSil1HRKR6ObF+QNdUXsljPEJAicKc6PANgp648uDtu9bGOMMw1z4HpNtJ/ZaJ/Z6OwqD7vB99xsGC4U7NZAppZRS6vogkowuRCiQvsYXj47C4GTIu1zQs0mlqtd2sJeggUwppZRSx1Y8DB5nektjpZRSSqkZO9JAJiL3isijIvI9EXnflPVZEflotP6rInLTUY5HKaWUUmoeHVkgk/BudH8AvAm4E3i7iNw5sdm7gB1jzIuB3wd+56jGo5RSSik1r45yhuwe4HvGmCeNMX3gT4C3TGzzFuAjUfsTwI/J9XAzEaWUUkqpfTjKQHYWeC62fD7qm7qNMcYDmsDRXpurlFJKKTVnjjKQTZvpMlexDSLybhG5X0Tu39jYOJTBKaWUUkrNi6MMZOeBG2LL54ALe20jIimgCux6voMx5gPGmJcbY16+vLx8RMNVSimllJqNowxkXwduFZGbRSQDvA24b2Kb+4B3Ru2fAT5vjNk1Q6aUUkopdT07shvDGmM8EflF4NNAEviQMeZhEXk/cL8x5j7gj4D/JSLfI5wZe9tRjUcppZRSal4d6Z36jTF/CfzlRN9vxNo28NajHINSSiml1LyT43aEUEQ2gGeu8uVLwOYhDkcdLt0/80v3zXzT/TO/dN/Mt2uxf15ojLnsCfDHLpAdhIjcb4x5+azHoabT/TO/dN/MN90/80v3zXybp/2jz7JUSimllJoxDWRKKaWUUjN20gLZB2Y9AHVJun/ml+6b+ab7Z37pvplvc7N/TtQ5ZEoppZRS8+ikzZAppZRSSs2dExPIROReEXlURL4nIu+b9XhOOhH5kIisi8hDsb66iHxWRB6P6tosx3hSicgNIvIFEXlERB4WkV+O+nX/zJiI5ETkayLyrWjf/Meo/2YR+Wq0bz4aPR1FzYCIJEXkmyLyF9Gy7ps5ISJPi8iDIvKAiNwf9c3N99qJCGQikgT+AHgTcCfwdhG5c7ajOvE+DNw70fc+4HPGmFuBz0XL6trzgH9tjHkJ8ErgPdG/F90/s+cArzfGvAy4C7hXRF4J/A7w+9G+2QHeNcMxnnS/DDwSW9Z9M19eZ4y5K3ari7n5XjsRgQy4B/ieMeZJY0wf+BPgLTMe04lmjPkSux8k/xbgI1H7I8BPX9NBKQCMMavGmL+L2m3C/7mcRffPzJlQJ1pMR8UArwc+EfXrvpkRETkH/CTwwWhZ0H0z7+bme+2kBLKzwHOx5fNRn5ovp4wxqxCGAmBlxuM58UTkJuBu4Kvo/pkL0SGxB4B14LPAE0DDGONFm+j32+z8Z+DXgCBaXkT3zTwxwGdE5Bsi8u6ob26+1470WZZzRKb06eWlSl2CiJSATwK/YoxphX/sq1kzxvjAXSKyAPwZ8JJpm13bUSkR+Slg3RjzDRF57aB7yqa6b2bnVcaYCyKyAnxWRL476wHFnZQZsvPADbHlc8CFGY1F7W1NRM4ARPX6jMdzYolImjCM/bEx5k+jbt0/c8QY0wC+SHie34KIDP7A1u+32XgV8GYReZrwtJjXE86Y6b6ZE8aYC1G9TvjHzD3M0ffaSQlkXwduja52yQBvA+6b8ZjUbvcB74za7wT+7wzHcmJF5738EfCIMeb3Yqt0/8yYiCxHM2OISB54A+E5fl8AfibaTPfNDBhjft0Yc84YcxPh/2M+b4x5B7pv5oKIFEWkPGgDPw48xBx9r52YG8OKyE8Q/rWSBD5kjPnNGQ/pRBOR/wO8FlgC1oB/D/w58DHgRuBZ4K3GmMkT/9URE5FXA18GHmR0Lsy/JTyPTPfPDInIDxCeeJwk/IP6Y8aY94vIiwhnZerAN4GfM8Y4sxvpyRYdsnyvMeandN/Mh2g//Fm0mAL+tzHmN0VkkTn5XjsxgUwppZRSal6dlEOWSimllFJzSwOZUkoppdSMaSBTSimllJoxDWRKKaWUUjOmgUwppZRSasY0kCmljjUR+S0Rea2I/LSIXJMHA4vI0yKydC1+llLqZNBAppQ67l5BeI+01xDeP00ppY4dDWRKqWNJRH5XRL4N/BDwN8DPA/9DRH5DRG4Rkb+KHiL8ZRG5I3rNh0XkD6O+x6LnDyIiORH5nyLyoIh8U0ReF/UnReQ/Rf3fFpFfig3hl0Tk76J1g/d/jYg8EJVvDu4MrpRSl3NSHi6ulLrOGGP+jYh8HPinwK8CXzTGvApARD4H/IIx5nEReQXw3wmfLQhwE+Fs2i3AF0TkxcB7ovf8/ihcfUZEbgP+OXAzcLcxxhORemwIm8aYHxSRfwW8lzAQvhd4jzHmK9HD2e2j/B0opa4fGsiUUsfZ3cADwB3AdwCiIPQjwMfDx3ICkI295mPGmAB4XESejF77auC/AhhjvisizwC3ET4r8g+NMV60Lv5IlcFD178B/KOo/RXg90Tkj4E/NcacP8TPqpS6jmkgU0odOyJyF/Bh4BywCRTCbnmAcParYYy5a4+XTz4vzgAybcOof6/nyw2eR+gTfZcaY35bRP4f8BPA34rIG4wx3738J1JKnXR6DplS6tgxxjwQBa7HgDuBzwNvNMbcZYxpAk+JyFshTGki8rLYy98qIgkRuQV4EfAo8CXgHdH2txE+aPhR4DPAL4hIKloXP2S5i4jcYox50BjzO8D9hLNvSil1WRrIlFLHkogsAzvR4cc7jDHfia1+B/AuEfkW8DDwlti6R4G/Bj5FeJ6ZTXiOWVJEHgQ+CvwzY4wDfBB4Fvh29F4/e5lh/YqIPBRta0U/QymlLkuM2Ws2Ximlri8i8mHgL4wxn5j1WJRSKk5nyJRSSimlZkxnyJRSSimlZkxnyJRSSimlZkwDmVJKKaXUjGkgU0oppZSaMQ1kSimllFIzpoFMKaWUUmrGNJAppZRSSs3Y/wchKH3TSakqfwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.figure(figsize=(10, 5))\n",
    "\n",
    "l1,l2,l3,l4=plt.plot(range(1,51),[v[1] for v in normal_init],range(1,51),[v[1] for v in xavier_init],\\\n",
    "              range(1,51),[v[1] for v in ortho_init],range(1,51),[v[1] for v in train_2_loss])\n",
    "plt.setp(l1,linewidth=2,color='r')\n",
    "plt.setp(l2,linewidth=2,color='y')\n",
    "plt.setp(l3,linewidth=2,color='g')\n",
    "plt.setp(l4,linewidth=2,color='b')\n",
    "plt.legend([l1,l2,l3,l4],['Normal','Xavier','Orthogonal','Uniform(of network 2)'])\n",
    "plt.xlabel('#epochs')\n",
    "plt.ylabel('Losses')\n",
    "plt.title('Cross-Entropy Losses Comparison')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Experiment 2 : Batch Normalization\n",
    "\n",
    "**Applying BN layer after second hiddern layer:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Training ('accuracy', 0.59114285714285719)\n",
      "Epoch 3, Training ('accuracy', 0.78261904761904766)\n",
      "Epoch 5, Training ('accuracy', 0.80840476190476196)\n",
      "Epoch 7, Training ('accuracy', 0.82269047619047619)\n",
      "Epoch 9, Training ('accuracy', 0.83388095238095239)\n",
      "Epoch 10, Training ('accuracy', 0.83797619047619043)\n"
     ]
    }
   ],
   "source": [
    "n_epochs=10\n",
    "display_step=2\n",
    "data_3 = mx.symbol.Variable('data')\n",
    "fc1_3  = mx.symbol.FullyConnected(data = data_3, num_hidden=1024)\n",
    "act1_3 = mx.symbol.Activation(data = fc1_3, act_type=\"relu\")\n",
    "#bn1_3 = mx.symbol.BatchNorm(data=act1_3)\n",
    "fc2_3  = mx.symbol.FullyConnected(data = act1_3, num_hidden = 512)\n",
    "act2_3 = mx.symbol.Activation(data = fc2_3, act_type=\"relu\")\n",
    "bn2_3 = mx.symbol.BatchNorm(data=act2_3)\n",
    "fc3_3  = mx.symbol.FullyConnected(data = bn2_3, num_hidden = 256)\n",
    "act3_3 = mx.symbol.Activation(data = fc3_3, act_type=\"relu\")\n",
    "#bn3_3 = mx.symbol.BatchNorm(data=act3_3)\n",
    "#output layer\n",
    "fc4_3  = mx.symbol.FullyConnected(data = act3_3, num_hidden=10)\n",
    "mlp_3 = mx.symbol.SoftmaxOutput(data = fc4_3,name='softmax')\n",
    "\n",
    "\n",
    "bn=[]\n",
    "mod_3 = mx.mod.Module(symbol=mlp_3,\n",
    "                    context=mx.cpu(),\n",
    "                    data_names=['data'],\n",
    "                    label_names=['softmax_label'])\n",
    "\n",
    "# allocate memory given the input data and label shapes\n",
    "mod_3.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)\n",
    "mod_3.init_params(mx.initializer.Uniform(scale=0.1))\n",
    "learn_rate=0.001\n",
    "mod_3.init_optimizer(optimizer='sgd', optimizer_params=(('learning_rate', learn_rate), ))\n",
    "# use accuracy as the metric\n",
    "metric = mx.metric.CrossEntropy()\n",
    "metric2= mx.metric.create('acc')\n",
    "for epoch in range(n_epochs):\n",
    "    train_iter.reset()\n",
    "    metric.reset()\n",
    "    metric2.reset()\n",
    "    for batch in train_iter:\n",
    "        mod_3.forward(batch, is_train=True)       # compute predictions\n",
    "        mod_3.update_metric(metric, batch.label)  # accumulate prediction loss\n",
    "        mod_3.update_metric(metric2, batch.label)  # accumulate prediction accuracy\n",
    "        mod_3.backward()                          # compute gradients\n",
    "        mod_3.update()                            # update parameters\n",
    "    loss=metric.get()\n",
    "    acc=metric2.get()\n",
    "    bn.append(loss)\n",
    "    if epoch%display_step==0 or epoch==n_epochs-1:\n",
    "        print('Epoch %d, Training %s' % (epoch+1,acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VOXZ+PHvnRD2sIWdAAFlXxIgArIIilVcissLKqKIG/Wt1lZ/xVptLa++1t22tvW11A2RuuCCaC1WLAiKIkFARUBQQSJb2EH25P798ZyZTIaZZLJMTpK5P9d1rpk558yZeyaTc8+znOcRVcUYY4wBSPI7AGOMMVWHJQVjjDFBlhSMMcYEWVIwxhgTZEnBGGNMkCUFY4wxQZYUjDG+E5EnROS3fsdhLClUayJyuYjkiMgBEdkiIv8SkWE+xpMhIurFE7pcGuPzVUROjnecsRCRSSLygd9xlIaI1BaRqSKyTkR+EJENIvK0iGT4HVtJVPUGVb3H7ziMJYVqS0RuBf4I/B5oBXQAHgcuiLJ/rcqLjiaq2jBkeakiDlrJ76E6egUYA1wONAYygWXAKD+DKomIJPsdgwmhqrZUswX3D38AGFfMPlNxJ4nngX3AdUAdXCLZ7C1/BOp4+zcH3gL2ALuARUCSt+1XwPfAfmAtMCrKa2YACtSKsv1Z4K/AP71jLQFO8rYt9J77g/feLgVGArne628FZnj7Xg+s9+KcA7QNeQ0Fbga+AXYAD+F+/NTx9u8Tsm9L4BDQIkKsk4APoryPtt7r7vLiuD5k20Agx/vMtwGPeuvren+Lnd5nvBRoFfL3fArY4n3O/wske9tOBt4H9nrv56UoMZ3pvZf2xXwniot7KjDLi3E/8DnQFfg1sB3YBJwVsv8C4D7gEy+2N4BmIdtneX+zvd7ftlfY9+D/gLe9v/eZ3rr/jeG72MN77T3AKmBMLN8vW0pxfvE7AFvK8EeD0cBxopx8vX2mAseAC72TYj3gbuBj72TYAlgM3OPtfx/wBJDiLcMBAbp5J4S23n4Z0f7RiC0p7MKdOGsBM4EXQ7YrcHLI45He+3wAd1KvB5zhnRz7e+v+DCwMO8Z8oBmu9PQVcJ237XHggZB9fw68GSXWSURPCu97x6oLZAF5eIkS+Ai40rvfEBjs3f8J8CZQH0gGBgCNvG2zgb8BDby/zSfAT7xtLwB3en/DusCwKDHdD7xfwvemuLinAoeBs72/zXPAt95rp+AS8bchx1qAS2C9vbhfBZ4P2X4NkErhD5EVYd+DvcDQkPf1LIVJIdp3MQWXzO4Aanvfhf1At1i+X7bEeH7xOwBbyvBHgwnA1hL2mRp6svTWfQ2cG/L4bGCDd/9u3K+9k8OeczLul+KZQEoJr5mBOynvCVt6eNufBZ4M2f9cYE3I40hJ4ShQN2TdU8CDIY8b4pJfRsgxRods/ynwnnd/EC7BBX515gCXRHkvk4iQFID2QD6QGrLuPuBZ7/5C4H+A5mHPuwaXhPuGrW8FHAHqhawbD8z37j8HTAPSS/js/17cCTCGuKcC74Zs+zGuxBYosaR6n20T7/EC4P6Q/Xt6f6vkCK/dxHtu45DvwXNh+zxLYVKI9l0cjit9JIWsewGYGsv3y5bYFmtTqJ52As1jqGPfFPa4LbAx5PFGbx24apb1wL9F5BsRuR1AVdcDv8CdNLaLyIsi0hYgrDG5Q8hxm6tqk5Bldci2rSH3D+JO6sXJU9XD0d6Dqh7AfR7torzv4HtU1SW46ooRItIdl/DmlPD64doCu1R1f9hrBF7/Wly1yxoRWSoi53vrZwDvAC+KyGYReVBEUoCOuF/AW0Rkj4jswZUaWnrPuw33K/kTEVklItdEiWsn0KYccYOr7go4BOxQ1fyQx1D07xX+OafgvpfJInK/iHwtIvuADd4+zaM8N1zE76L3HjapakEx76G03y8TxpJC9fQRrqh/YQn7hQ+Buxl3Egro4K1DVfer6v9T1c64X4m3isgob9s/VHWY91zFVeegRRuTvyvvmyrLexCRBkAariojoH3I/eB79EwHrgCuBF4JSzix2Aw0E5HUsNf4HkBV16nqeNxJ/QHgFRFpoKrHVPV/VLUnMAQ4H5iIOzkeoWgibaSqvbzjbVXV61W1La4K6vEoPbTmAQNFJL0scZdR+Od8DFe1dzmuw8OZuPaSDG8fCdk/6vDMxXwXNwPtRST0vFXe92DCWFKohlR1L3AX8FcRuVBE6otIioicIyIPFvPUF4DfiEgLEWnuHeN5ABE5X0ROFhHBNZLmA/ki0k1EzhCROrhEdMjbFg/bgM4l7PMP4GoRyfJi+j2wRFU3hOwzRUSaikh7XLtBaO+nGcBFuMTwXAmvJSJSN3RR1U24aqD7vHV9caWDmd4TrhCRFt6v2T3ecfJF5HQR6eP1tNmHO4Hmq+oW4N/AIyLSSESSROQkERnhHW9cyIl+N+5kesLnr6rzgHeB10VkgIjUEpFUEblBRK4pKe4yukJEeopIfVyVzyteySIVl+h24tpQfl+ag0b7LuIajn8AbvO+7yNxSePFcrwHE8aSQjWlqo8CtwK/wTUYbgJuwjVaRvO/uHr0z3C9Sz711gF0wf3aPIAriTyuqgtwDYX3434BbsX9Ar6jhPD2hFUt3Rrj25oKTPeqUS6JtIOqvgf8FtewuQU4CbgsbLc3cF0xV+B6ojwV8vxc3PtWXK+W4gzBJcHg4lXZjcf9+t0MvA78TlXf9Z4zGlglIgeAPwGXeaWR1rjeYPuA1bhG3+e950zENZx+iTvxv0JhVdApwBLveHOAn6vqt1HiHYvr0fMSriH3CyAb93elhLjLYgauHn8rrrH4Zm/9c7hqne+99/RxKY8b8buoqkdxXW7PwX0fHwcmquqacrwHE0ZUbZIdU3OIiAJdvLaQaPs8DWxW1d9UXmQ1i4gswPU2etLvWEzFsouBTELxru69GOjnbyTGVE1WfWQShojcg6tSeaiYKhhjEppVHxljjAmKW0nBG4hru4h8Ucw+I0Vkhdf/+v14xWKMMSY2cSspiMhpuN4Dz6lq7wjbm+C6yI1W1e9EpKWqbi/puM2bN9eMjIwKj9cYY2qyZcuW7VDVFiXtF7eGZlVdWMKQvZcDrwUueoolIQBkZGSQk5NT/gCNMSaBiMjGkvfyt6G5K9BURBaIyDIRmRhtRxGZLG7egJy8vLxKDNEYYxKLn0mhFm6kyPNwA7P9VkS6RtpRVaeparaqZrdoUWLpxxhjTBn5eZ1CLm7ArR+AH0RkIW5SkK98jMkYYxKan0nhDeAv3rABtXHDGv/Bx3iMqdKOHTtGbm4uhw+Xdgw/k0jq1q1Leno6KSkpZXp+3JKCiLyAGw+/uYjkAr/DDa2Lqj6hqqtFZC5uHJ4C3DjoUbuvGpPocnNzSU1NJSMjAzdWnDFFqSo7d+4kNzeXTp06lekY8ex9ND6GfR7CjZ1ujCnB4cOHLSGYYokIaWlplKdDjg1zYUw1YgnBlKS835GESQp/+AMMGADvlmegYGOMqeESJink5sKnn8LixX5HYowxVVfCJIVBg9ztkiX+xmFMdZacnExWVhaZmZn079+fxSX8ytqzZw+PP/54iccdOXJkzCMVLFiwABHhzTffDK47//zzWbBgQUzPrygZGRns2LEDgCFDhpTpGL//fdFJ6cp6nIqUkEnBBoY1pmzq1avHihUrWLlyJffddx+//vWvi90/1qRQWunp6dx7771lfn5+fsXOKFtScowmPCmU9TgVKWGSQocO0Lo17NoF66POyWVMNSESn6UU9u3bR9OmTQE4cOAAo0aNon///vTp04c33ngDgNtvv52vv/6arKwspkyZAsCDDz5Inz59yMzM5Pbbbw8eb9asWQwcOJCuXbuyaFHxM6VmZmbSuHFj3o3QSPjee+/Rr18/+vTpwzXXXMORI0cA98v+7rvvZtiwYcyaNYuRI0dyyy23cNppp9GjRw+WLl3KxRdfTJcuXfjNbwon5bvwwgsZMGAAvXr1Ytq0aRHjadiwIQB33XUXWVlZZGVl0a5dO66++uqox7j99ts5dOgQWVlZTJgwochxVJUpU6bQu3dv+vTpw0svuWnGFyxYwMiRIxk7dizdu3dnwoQJVPigpqparZYBAwZoWV1wgSqozphR5kMY45svv/yy8IEr8Fb8UoKkpCTNzMzUbt26aaNGjTQnJ0dVVY8dO6Z79+5VVdW8vDw96aSTtKCgQL/99lvt1atX8Plvv/22nnrqqfrDDz+oqurOnTtVVXXEiBF66623qqrqP//5Tx01alTUGObPn6/nnXeeLly4UE877TRVVT3vvPN0/vz5eujQIU1PT9e1a9eqquqVV16pf/jDH1RVtWPHjvrAAw8EjzNixAi97bbbVFX1j3/8o7Zp00Y3b96shw8f1nbt2umOHTuKxHjw4EHt1atXcH3Hjh01Ly9PVVUbNGhQJMY9e/Zonz59gp9PtGOEPy/w+JVXXtEzzzxTjx8/rlu3btX27dvr5s2bdf78+dqoUSPdtGmT5ufn6+DBg3XRokUnfEZFviseIEdjOMcmTEkBCquQPi7tNOLGVDXxSgslCFQfrVmzhrlz5zJx4sTgyeSOO+6gb9++nHnmmXz//fds27bthOfPmzePq6++mvr16wPQrFmz4LaLL74YgAEDBrBhw4YSYxk+fDhAkVLF2rVr6dSpE127umHUrrrqKhYuXBjcfumllxY5xpgxYwDo06cPvXr1ok2bNtSpU4fOnTuzadMmAB577DEyMzMZPHgwmzZtYt26dcXGpapMmDCBW265hQEDBpTpGB988AHjx48nOTmZVq1aMWLECJYuXQrAwIEDSU9PJykpiaysrJg+q9JIqDmaBw92t9bYbEz5nXrqqezYsYO8vDzefvtt8vLyWLZsGSkpKWRkZEQcjkNVo/ajr1OnDuAas48fPx5TDHfeeSf33nsvtWrVCh6/OA0aNIj4mklJScH7gcfHjx9nwYIFzJs3j48++oj69eszcuTIEocZmTp1Kunp6cGqo7Ico7j3ERpnaT6rWCVUSSE721WbrlgBhw75HY0x1duaNWvIz88nLS2NvXv30rJlS1JSUpg/fz4bN7qh+1NTU9m/f3/wOWeddRZPP/00Bw8eBGDXrl3liuGss85i9+7drFy5EoDu3buzYcMG1nsNhzNmzGDEiBFlPv7evXtp2rQp9evXZ82aNXxcQjXDW2+9xbvvvstjjz0W0zFSUlI4duzYCcc57bTTeOmll8jPzycvL4+FCxcycODAMr+P0kiopJCaCr16wfHjsHy539EYU/0EGkazsrK49NJLmT59OsnJyUyYMIGcnByys7OZOXMm3bt3ByAtLY2hQ4fSu3dvpkyZwujRoxkzZgzZ2dlkZWXx8MMPlzumO++8k9zcXMANBvfMM88wbtw4+vTpQ1JSEjfccEOZjz169GiOHz9O3759+e1vf8vgQHVDFI888gibN29m4MCBZGVlcddddxV7jMmTJ9O3b99gQ3PARRddRN++fcnMzOSMM87gwQcfpHXr1mV+H6URt+k44yU7O1vLM/Pa9dfDk0/Co4/CLbdUYGDGxNnq1avp0aOH32GYaiDSd0VElqlqdknPTaiSAthFbMYYU5yEamgG64FkTHXxzjvv8Ktf/arIuk6dOvH666/7FFFiSLik0LMnNGwIGzfCtm3QqpXfERljIjn77LM5++yz/Q4j4SRc9VFyMpxyirtvVUjGGFNUwiUFsCokY4yJJiGTgl3EZowxkSVkUgiUFJYuhQoeLNEYE2L27Nl8+eWXFX7cqVOnlniNw6OPPkrPnj3p27cvo0aNCl5QZ4qXkEmhdWs3aur+/bB6td/RGFNzxSMpxDqsQ79+/cjJyeGzzz5j7Nix3HbbbRUaR02VkEkBrArJmNLasGEDPXr04Prrr6dXr16cddZZHPLGi/n6668ZPXo0AwYMYPjw4axZs4bFixczZ84cpkyZQlZWFkuWLAkOELdy5UpEhO+++w6Ak046iYMHD7Jx40ZGjRoV/HUf2D5p0iRuvfVWTj/99BO6qf7973/nnHPOCcYScPrppwcH3hs8eHDwqmdTvIRNCnYRm6nO/JpOYd26ddx4442sWrWKJk2a8OqrrwJuuIY///nPLFu2jIcffpif/vSnDBkyhDFjxvDQQw+xYsUKBg0axOHDh9m3bx+LFi0iOzubRYsWsXHjRlq2bEn9+vW56aabmDhxIp999hkTJkzg5ptvDr72V199xbx583jkkUeC6/7yl7/w5ptvMnv2bOrVqxc17qeeeopzzjmn7B94Akm46xQCrAeSMaXXqVMnsrKygMIhrg8cOMDixYsZN25ccL/AxDbhhgwZwocffsjChQu54447mDt3LqoaHAb7o48+4rXXXgPgyiuvLFLlM27cOJKTk4OPZ8yYQXp6OrNnzyYlJSVqzM8//zw5OTm8//77ZX/jCSRhk0L//lCrFqxaBQcOuAvajKku/BqyLHzY5kOHDlFQUECTJk1YsWJFic8fPnx4sHRwwQUX8MADDyAinH/++RH3Dx1mO3zY6969e7NixQpyc3Pp1KlTxOfPmzePe++9l/fff79I7Ca6hK0+qlcPMjOhoADKMb6eMQmvUaNGdOrUiVmzZgFuLoDAUNbhQ2efdtppPP/883Tp0oWkpCSaNWvG22+/zdChQwFXknjxxRcBmDlzJsOGDYv6uv369eNvf/sbY8aMYfPmzSdsX758OT/5yU+YM2cOLVu2rLD3W9PFLSmIyNMisl1Evihhv1NEJF9ExsYrlmisCsmYijFz5kyeeuopMjMz6dWrV3CO5ssuu4yHHnqIfv368fXXX5ORkQG45AAwbNgwmjRpEpzr+bHHHuOZZ56hb9++zJgxgz/96U/Fvu6wYcN4+OGHOe+889ixY0eRbVOmTOHAgQOMGzeOrKys4CxrpnhxGzpbRE4DDgDPqWrvKPskA+8Ch4GnVfWVko5b3qGzQ82YARMnwoUXgo2xZao6GzrbxKpKDp2tqguBkqZV+hnwKrA9XnEUJ7SkUM2mlTDGmLjwrU1BRNoBFwFPxLDvZBHJEZGcvLy8CouhSxdo2hS2bgVvjm5jjElofjY0/xH4laqWONCEqk5T1WxVzW7RokWFBSBi1yuY6qW6zZRoKl95vyN+JoVs4EUR2QCMBR4XkQsrOwhLCqa6qFu3Ljt37rTEYKJSVXbu3EndunXLfAzfrlNQ1WDHYhF5FnhLVWdXdhzWA8lUF+np6eTm5lKRVaim5qlbty7p6ellfn7ckoKIvACMBJqLSC7wOyAFQFVLbEeoLAMHuttly+DYMSjmwkhjfJWSkhL1Ii1jKkrckoKqji/FvpPiFUdJ0tJcg/O6dfD55+5KZ2OMSVQJe0VzKKtCMsYYx5ICNoy2McYEWFLAeiAZY0yAJQWgb1+oUwfWroXdu/2Oxhhj/GNJAahdG7wJofjkE39jMcYYP1lS8FgVkjHGJFpSOHjQTaAQgfVAMsaYREoKv/0ttG4NCxZE3BzogfTJJzZiqjEmcSVOUhCB/fth+vSImzt0gFatYOdO+PrrSo7NGGOqiMRJChMnuttXX3WTMocJHTHVqpCMMYkqcZLCySfD0KHwww8uMURgF7EZYxJd4iQFgEmT3O2zz0bcbD2QjDGJLrGSwrhxULeua2zesOGEzdnZrhppxQo4fLjSozPGGN8lVlJo3BguusjdnzHjhM2NGkGvXm4I7eXLKzk2Y4ypAhIrKUBhFdL06RH7nloVkjEmkSVeUhg1Ctq1c/1OP/zwhM3WA8kYk8gSLykkJ8MVV7j7Ea5ZsB5IxphElnhJAeCqq9ztSy+5oS9C9OwJDRu6duht2yo/NGOM8VNiJoUePdzkzPv3w+zZRTYlJ7teSGClBWNM4knMpACFpQWrQjLGmKDETQqXXeYmUpg3D77/vsgm64FkjElUiZsUmjWDMWPcUNph1ywEksInn0B+vg+xGWOMTxI3KUDRKqSQaxbatHGjpu7fD2vW+BSbMcb4ILGTwtlnQ8uW7sy/dGmRTVaFZIxJRImdFFJSCq9ZCBskL9DYbBexGWMSSdySgog8LSLbReSLKNsniMhn3rJYRDLjFUuxAlVIL74IR44EV1tJwRiTiOJZUngWGF3M9m+BEaraF7gHmBbHWKLr2xeysmD3bnjzzeDq/v2hVi344ouIc/IYY0yNFLekoKoLgV3FbF+sqru9hx8D6fGKpUQR5lmoVw8yM13npJwcX6IyxphKV1XaFK4F/hVto4hMFpEcEcnJy8ur+Fe//HJXLJg7t8jYFlaFZIxJNL4nBRE5HZcUfhVtH1WdpqrZqprdokWLig+iRQs491x3UcLMmcHVlhSMMYnG16QgIn2BJ4ELVHWnn7EUqULyrlkI7YEUYeoFY4ypcXxLCiLSAXgNuFJVv/IrjqDzzoO0NPj8czcfJ9ClCzRtClu2QG6uz/EZY0wliGeX1BeAj4BuIpIrIteKyA0icoO3y11AGvC4iKwQEX+bc2vXhvHj3X1vkDwRN5gqWBWSMSYxxLP30XhVbaOqKaqarqpPqeoTqvqEt/06VW2qqlnekh2vWGIWuGZh5kw4ehSwi9iMMYnF94bmKmXAAOjVC3bsgH+5zlDW2GyMSSSWFEKJnDDPQqD6KCcHjh3zKS5jjKkklhTCXXEFJCXBW2/Bjh2kpbkG58OHXRu0McbUZJYUwrVp40ZPPXYMXngBsCokY0zisKQQSVgVUiApWGOzMaams6QQyQUXQOPGsGwZrFplczYbYxKGJYVI6tZ1czgDTJ9O375Qpw6sXesGUzXGmJrKkkI0gSqkGTOonXSc/v3dw08+8S8kY4yJN0sK0Qwe7Lodbd0K775rVUjGmIRgSSGasGsWrAeSMSYRWFIozpVXuuQwezaDuu8FXFKwEVONMTWVJYXidOgAZ5wBR47Q8aMXadUKdu6Er7/2OzBjjIkPSwol8aqQ5DmrQjLG1HyWFEpy8cXQsCF89BGDOrupQO0iNmNMTWVJoSQNGsC4cQAM3j4HsJKCMabmsqQQC68KKfv9RxBRVqxwA+QZY0xNY0khFsOHQ6dONPp+NT07/sCxY7B8ud9BGWNMxbOkEIukJJg4EYDByW7WUKtCMsbURJYUYuUlhUGbZgGWFIwxNZMlhVh17gzDhzPo6CLAeiAZY2qmmJKCiPxcRBqJ85SIfCoiZ8U7uCpn0iR6sYoGSQfZsAG2b/c7IGOMqVixlhSuUdV9wFlAC+Bq4P64RVVVjR1Lcr06nFLg6o6sCskYU9PEmhTEuz0XeEZVV4asSxyNGsHFFzMIlw2sCskYU9PEmhSWici/cUnhHRFJBQriF1YVNmkSg3HZYMkSGxnPGFOzxJoUrgVuB05R1YNAbVwVUuI5/XQGtdkEwCcf5ZOf73M8xhhTgWJNCgr0BG72HjcA6hb3BBF5WkS2i8gXUbaLiDwmIutF5DMR6R9z1H5KTqbNpLNpz3fsP1iLNWv8DsgYYypOrEnhceBUYLz3eD/w1xKe8ywwupjt5wBdvGUy8H8xxuK/q64qrEJaeMTnYIwxpuLEmhQGqeqNwGEAVd2Nq0KKSlUXAruK2eUC4Dl1PgaaiEibGOPxV7duDOqwFYAls77zORhjjKk4sSaFYyKSjKtGQkRaUP6G5nbAppDHud66E4jIZBHJEZGcvLy8cr5sxRg0rgMAHy+16/+MMTVHrGe0x4DXgZYici/wAfD7cr52pC6tEbvzqOo0Vc1W1ewWLVqU82UrRv9bRlCLY3xxIIMDa3L9DscYYypETElBVWcCtwH3AVuAC1V1VjlfOxdoH/I4HdhczmNWmvrtmtK36SYKSGbZIwv8DscYYypErMNcnAR8q6p/Bb4AfiQiTcr52nOAiV4vpMHAXlXdUs5jVqpBpyYD8PHsraB2zYIxpvqLtfroVSBfRE4GngQ6Af8o7gki8gLwEdBNRHJF5FoRuUFEbvB2eRv4BlgP/B34aVnegJ8Gj00HYMmOzjbmhTGmRqgV434FqnpcRC4G/qSqfxaRYqeZUdXxJWxX4MYYX79KGjTEKykwGH3mbmTwYJ8jMsaY8ilN76PxwETgLW9dSnxCqj66dIEmqflsoS25L35gc3QaY6q9WJPC1biL1+5V1W9FpBPwfPzCqh6SkgpLC0v2dYc5c3yOyBhjyifW3kdfqurNqvqCiDQFUlU18YbOjmDQIHe7hEEwfbq/wRhjTDnF2vtogTfJTjNgJfCMiDwa39Cqh0BS+FhOhblzYUu16kBljDFFxFp91NibZOdi3HwKA4Az4xdW9RFICsuSTuFYQRLMnOlvQMYYUw6xJoVa3rhEl1DY0GyAtDQ4+WQ4lF+HL+jtqpDsmgVjTDUVa1K4G3gH+FpVl4pIZ2Bd/MKqXoJVSA3OhC++gE8/9TcgY4wpo1gbmmepal9V/W/v8Teq+l/xDa36CFyesKTDWHfHGpyNMdVUrA3N6SLyujdpzjYReVVE0uMdXHUR7IF0KNPd+cc/4OhR/wIyxpgyirX66BncWEVtccNbv+mtM0BmJtSpA2s21GV3zyGwcyf8859+h2WMMaUWa1JooarPqOpxb3kWqBpjWFcBtWtDf28y0aVDb3F3rArJGFMNxZoUdojIFSKS7C1XADvjGVh1E6xCanI2JCe7kkIVmRDIGGNiFWtSuAbXHXUrbj6FsbihL4wn2ANpVSqMHg3Hj8MLL/gblDHGlFKsvY++U9UxqtpCVVuq6oW4C9mMJ9gDaQnoxKvcg2ef9S0eY4wpi/JMMHxrhUVRA3TsCC1bujbmb3qPgSZNYPly+Pxzv0MzxpiYlScpRJpjOWGJhFQhLa8D473pJKzB2RhTjZQnKdhYDmFCq5C4yqtCev55175gjDHVQLFJQUT2i8i+CMt+3DULJkSwB9ISYOBA6NYNtm2Dd97xNS5jjIlVsUlBVVNVtVGEJVVVY53KM2GccoqrRlq+HA4fkcLSglUhGWOqifJUH5kwjRpBz55w7BisWAFceaXLEm+8Abt2+R2eMcaUyJJCBStShZSeDmee6cZBeuklX+MyxphYWFKoYMEeSB97K6wKyRhTjVhSqGBFeiABXHQRpKa6FWvW+BaXMcYORaRgAAAWsUlEQVTEwpJCBevVCxo0gG+/he3bgfr14ZJL3EYrLRhjqjhLChUsORmys939YGkhUIU0Ywbk5/sSlzHGxCKuSUFERovIWhFZLyK3R9jeQUTmi8hyEflMRM6NZzyV5YQqpGHDoHNn+P57+M9/fIvLGGNKErekICLJwF+Bc4CewHgR6Rm222+Al1W1H3AZ8Hi84qlMRXoggeuWepUNkmeMqfriWVIYCKz35nM+CrwIXBC2jwKNvPuNgc1xjKfSBJLCJ59AQYG3cuJEd/v667Bvny9xGWNMSeKZFNoBm0Ie53rrQk0FrhCRXOBt4GeRDiQik0UkR0Ry8qrBxDVt20L79u7cH+xwlJEBI0bAoUMwa5af4RljTFTxTAqRRlENH0RvPPCsqqYD5wIzROSEmFR1mqpmq2p2ixbVYxbQE65XAJg0yd1aFZIxpoqKZ1LIBdqHPE7nxOqha4GXAVT1I6Au0DyOMVWaE9oVAP7rv1wX1Q8+gN//Hg4f9iU2Y4yJJp5JYSnQRUQ6iUhtXEPynLB9vgNGAYhID1xSqPr1QzE4oQcSuIvYfvlLd//OO6F7d3j5ZVAbhdwYUzXELSmo6nHgJuAdYDWul9EqEblbRMZ4u/0/4HoRWQm8AExSrRlnyP793TULn38OBw6EbPif/4F586BPH9i4ES69FIYPh5wc32I1xpgAqW7n4OzsbM2pJifQ/v3dMNoLFrg25iLy8+Gpp+A3v4FA4/nEia5aqV14e7wxxpSPiCxT1eyS9rMrmuMoYhVSQHIyTJ4M69bBbbdB7drw3HPQtSvcfTccPFipsRpjDFhSiKuIPZDCNW4MDzwAq1e7huiDB+F3v3Ozts2cGXKhgzHGxJ8lhTiK2AMpms6d4ZVXXF1Tv36QmwtXXAFDhpSQVYwxpuJYUoijrl2hSRPYvNmd42MyYgQsXeraG1q3dhnl1FPh8svhu+/iGq8xxlhSiKOkJBg40N0v1Y/95GS45hr46iu44w6oUwdeeMFVKd11V1h3JmOMqTiWFOKsVFVI4VJT4d573VgZl17qLna75x5XBJk+3dobjDEVzpJCnBXbAylWGRnw4ovuSuhTToEtW9yQGQMHunXGGFNBLCnEWaD6KCcHjh0r58GGDnX1UM8950bdW7bMXfh2ySVuqjdjjCknSwpx1rw5nHSSGxz1iy8q4IBJSXDlla694Xe/g3r13KirPXrAr39tw3IbY8rFkkIlqJAqpHANGsDUqbB2LUyYAEeOwP33u/aGJ5+0aT+NMWViSaESxHQRW1m1bw/PP+8OPngwbNsG118PAwbA/PlxeEFjTE1mSaESlKsHUmleZPFi+Mc/XKJYuRLOOAMuugjWr4/jCxtjahJLCpUgK8tdarBmDezZE8cXEoHx490L3XOPm7th9mzo2ROmTIG9e+P44saYmsCSQiWoXduNXAFu3ua4q1/fjb66bh1cdZXr9vTww3DyyfDEE3D8eCUEYYypjiwpVJJKqUIK17atm/pz6VIYNgx27ID//m+Xod59txIDMcZUF5YUKklceiDFKjsbFi50XVczMlzf2LPOcm0Ojz9uYyoZY4IsKVSS0B5IvsxrJAJjx7ohuu+7Dxo2dL2TbrwROnaEzExX5bRkiQ2fYUwCs5nXKokqtGrlJllbv95d0OarnTthzhx4803497/hhx8Kt7VsCeedBz/+MfzoRy6BGGOqNZt5rYoR8bkKKVxaGlx9Nbz2mksQc+e6UkOHDrB9OzzzDFx8sdtv9Gj461/dnNLGmBrNkkIliutFbOVRpw6cfTb85S+wYQN89pkbnfXUU13PpXfegZtucu0RffvCnXe6N2FXTRtT41j1USWaN8/VxgwcWEVKC7HYvh3efruwmil0LocWLYpWM6Wm+henMaZYsVYfWVKoRHv3QtOmUKuWG7eubl2/IyqlI0fg/fddgnjzzaLVSbVrw8iRLkH8+Meu8doYU2VYUqiieveGVavcOfO669wEa23b+h1VGai6N/LWWy5BfPRR0W5VvXsXJoiBA91scsYY31hSqKLmzIFf/KJw+oPkZDj/fJg82VXrV9tzZ16eq2Z66y3XBrF/f+G2Fi3g3HNdgjjrLKtmMsYHlhSqsIICeO89mDbNDU0UGHWiQ4fC0kO7dv7GWC5Hj7pqpkApInQCoJSUotVMGRl+RWlMQqkSSUFERgN/ApKBJ1X1/gj7XAJMBRRYqaqXF3fMmpAUQm3b5np//v3v8M03bl1SUmHpYfToalx6AFel9OWXRauZQi+O69XLXVk9YIAbfqNHD5c4jDEVyvekICLJwFfAj4BcYCkwXlW/DNmnC/AycIaq7haRlqq6vbjj1rSkEFBQAP/5jys9vP56YekhPb2w9NC+vb8xVogdO+Bf/3IJYu7cotVM4LrH9u0L/fsXLr17V8NWeWOqlqqQFE4Fpqrq2d7jXwOo6n0h+zwIfKWqT8Z63JqaFEJt2wbTp7vSQ2AqhKQkVy0/eTKcc47rwVTtHT0KH3zgho799FO3fP31ifvVquWG/w5NFJmZdqW1MaVQFZLCWGC0ql7nPb4SGKSqN4XsMxtXmhiKq2KaqqpzIxxrMjAZoEOHDgM2JsiVtQUFsGCBKz289pq7jgxce8O117qlQwdfQ6x4e/bAihWFSeLTT92Uo+HjMYlAt26uyimQKPr1c31+jTEnqApJYRxwdlhSGKiqPwvZ5y3gGHAJkA4sAnqratSpaBKhpBBJXp4rPUyb5qZJAHdePOccV3o477waUnqI5Icf3FXWoYniiy8izwvRqVPRJNG/vxt0ypgEVxWSQizVR08AH6vqs97j94DbVXVptOMmalIIUHUde6ZNg1dfdTUw4K51uOYaV3pIiA49R4646yRCE8XKlXD48In7tm1btOqpXz/XQCNS+XEb45OqkBRq4aqGRgHf4xqaL1fVVSH7jMY1Pl8lIs2B5UCWqu6MdtxETwqhduyA555zCWLtWrdOxF3vMHmy68GUUB15jh93U5EuX16YKJYvP7ExG9xAf+GJonPnat7Vy5jofE8KXhDnAn/EtRc8rar3isjdQI6qzhERAR4BRgP5wL2q+mJxx7SkcCJVWLTIJYdXXnE/ogFat3alh+uuc7UqCamgwDVehyaJZctg164T961d241p3qVL4dK1q7tt29a19htTTVWJpBAPlhSKt3MnzJgBf/ub+9EMrvTwox+50sOYMQlWeohEFTZtKlr1tGIFfP999OfUq+fmuA5PFl26uDYLq4oyVZwlhQSn6np7TpvmZuEMlB5atXLTKFx3XRWY6KeqOXDA9QH+6ivXmh+65OVFf15qavSEkZZmCcNUCZYUTNCuXa70MG2au7g4YOhQGD7c3Q4ZAs2a+Rdjlbdnz4mJYt06l0D2RO0sB02aRE4WXbq4bcZUEksK5gSqsHixSw4vv3xiR50ePVyCCCwnn2w/ckuk6ursIiWMdesiN3IHNG8eOWF07Oiut7AP31QgSwqmWHv2uMbpDz90y9KlhVVMAS1buhJEIEn07+9GoTAxUnWXp0dLGIcORX9ugwbuysRoS3q6axg3JkaWFEypHDni2lsDSeLDD0+sRq9TB045pbC6acgQ92PXlEFBAWzefGKiWL/eTV4UOsNdJCKue1lxicPaM0wISwqmXFTd+enDD12V04cfFm2PCOjWrWiVU9eudh4qN1U3Td9330Vfvv/+xKE/wtWrV3JpwwYaTBiWFEyF27XLjXwdKEl88smJ7RLNmxetchowwM47cXH8uCtpREsamzYV3wAe0KpV9ITRpo3bbtVUNYIlBRN3R4+6a8FCq5y2bSu6T+3akJ1dWOU0dKibiM1Ugn37XHKIljhycyOPHxWuaVNXVdW6tUsSgfvhj1u0sCvCqzBLCqbSqbpJ1kKTxKpVRaduBtfBJrTKqVs3u1jYF/n5sHVr9JLGtm1uyc+P7XhJSa6oGEsCadbM6hkrmSUFUyXs2VO0ymnJkhM73dSv7xJD9+6FS7durn2iXj1/4jaeggJXb7h1a+GybVvkxzt2nPgLIJqUFNe9raQE0qoVNGpkCaQCWFIwVdKxY25EiUDj9eLF0UeXEHFd9kOTRWBp2dLOE1XOsWOuy1po0oiWQGJp7wioVcuVQEKXFi1OXBe61K8fv/dZTVlSMNXG7t1ulNc1a4ou69dHr7lo0qSwRBGaLE46ycZ2qhYOHy6sniougWzbVnL33Ejq1y8+aYQnlrS0Gv/FsaRgqr2jR+Gbb1yCCE0aq1e7HpuR1KrlEkN4yaJbN5uUrdo6fNhdNb5jR/FLXl7hbWCikdJo3Dh64mjWrHBp2rTwtmHDalNktaRgaixV2L79xJLFmjXuuq9oX+mWLSNXRXXoYJ1mahRVN1tfSckjdNm5M/YG9VC1ahVNEpESR7TbSu7qa0nBJKRDh9yFweHJYu1aOHgw8nPq1nU9ojp3dhOyhXfZb93akkaNV1Dgip/RSh67d7sG9127Cu/v3u2ST1k1aBBbAgm9n5bmGt7LwJKCMSEKCly3/Eiliy1bin9urVrQrl3ka7wCSaRx48p5H6aKOXq0aJIIvY20LvQ2lmtEwmVluYuDyiDWpFBTp3o3poikpMIT+VlnFd22d68bAXvjxsIu+qFd9rdvd9s2box+/EaNIieLwNKuXY1vx0xMtWu7brOtWpXueaquAT3WBBK4bd8+Pu8jhJUUjCnBoUOulBHp4uDAumhVUwEibtSI4kobNn6diScrKRhTQerVK5zqIBJV90MuPFGELlu2uKGKNm+Gjz+O/jrt27tSRZs2blrotm0L7wduGzSI33s1xpKCMeUk4n7lp6VBv36R9zl2LPL4daEJJFCN9dVXxb9eamrRJBEpcbRp43pLGlNalhSMqQQpKe7q7I4do++zd69LElu2FC1ZBO4Hbvfvd72p1q4t/jXDk0f4rSUPE4klBWOqiMaN3dK7d/R9VN0IEdESRmgyKU3yiJQ4WrZ0S4sWhbc2817NZ0nBmGpExHVZb9oUevWKvl9o8igucWzZ4pLH/v0lV1uBS1qBJBGeNMIfp6W57rymerE/mTE1UGmTR6TEkZfnuuNu3154f+9et6xfH1sMaWnFJ5HQ+02a2BDqVYElBWMSWGjy6Nmz+H0LClwCCU0S4fdDH4cOV7R6dcmxBAZDDU0ggbHq0tIKL+gNfZyaat14K1pck4KIjAb+BCQDT6rq/VH2GwvMAk5RVbsIwZgqKCmpcNSF7t1L3v/4cddVN1LCiPR4797CAVJjlZJSGFOk5BEpkaSl2RSxxYlbUhCRZOCvwI+AXGCpiMxR1S/D9ksFbgaWxCsWY0zlq1Wr8Bd/LI4ccaWK0ISRl+dKHLt2udvAEnj8ww+FI2yXRv36pUskTZu66q1EaGiPZ0lhILBeVb8BEJEXgQuAL8P2uwd4EPhlHGMxxlRxdeq4C/fatYv9OUeORE8YxSWTgwfdkptb+hibNCnsKVbS/fB1jRpV/cEV45kU2gGbQh7nAoNCdxCRfkB7VX1LRKImBRGZDEwG6NChQxxCNcZUR3XquO6zbdrE/pzAsEOxJI/AsmePW44cKVvJJFTDhtGTRkkJpkmT+F/RHs+kEKn5JzjQkogkAX8AJpV0IFWdBkwDN/ZRBcVnjElAIq6BOjW1+IsJw6m6cbD27nUJItATq6T7oev27XMJ6cCB0pdSoFyDpMYsnkkhFwgd0i8d2BzyOBXoDSwQ132gNTBHRMZYY7MxpqoRcW0R9euXrmQSqqDAXRMSSwKJdL+sr1sa8UwKS4EuItIJ+B64DLg8sFFV9wLNA49FZAHwS0sIxpiaKimpsFqoqtaEx+1SEVU9DtwEvAOsBl5W1VUicreIjInX6xpjjCm7uF6noKpvA2+Hrbsryr4j4xmLMcaYktlF5cYYY4IsKRhjjAmypGCMMSbIkoIxxpggSwrGGGOCLCkYY4wJEtXqNWqEiOQBG/2Oo5yaAzv8DqIKsc+jKPs8CtlnUVR5Po+OqtqipJ2qXVKoCUQkR1Wz/Y6jqrDPoyj7PArZZ1FUZXweVn1kjDEmyJKCMcaYIEsK/pjmdwBVjH0eRdnnUcg+i6Li/nlYm4IxxpggKykYY4wJsqRgjDEmyJJCJRKR9iIyX0RWi8gqEfm53zH5TUSSRWS5iLzldyx+E5EmIvKKiKzxviOn+h2Tn0TkFu//5AsReUFE6vodU2USkadFZLuIfBGyrpmIvCsi67zbphX9upYUKtdx4P+pag9gMHCjiPT0OSa//Rw3CZOBPwFzVbU7kEkCfy4i0g64GchW1d5AMm72xkTyLDA6bN3twHuq2gV4z3tcoSwpVCJV3aKqn3r39+P+6dv5G5V/RCQdOA940u9Y/CYijYDTgKcAVPWoqu7xNyrf1QLqiUgtoD5F53iv8VR1IbArbPUFwHTv/nTgwop+XUsKPhGRDKAfsMTfSHz1R+A2oMDvQKqAzkAe8IxXnfakiDTwOyi/qOr3wMPAd8AWYK+q/tvfqKqEVqq6BdyPTKBlRb+AJQUfiEhD4FXgF6q6z+94/CAi5wPbVXWZ37FUEbWA/sD/qWo/4AfiUDVQXXh15RcAnYC2QAMRucLfqBKDJYVKJiIpuIQwU1Vf8zseHw0FxojIBuBF4AwRed7fkHyVC+SqaqDk+AouSSSqM4FvVTVPVY8BrwFDfI6pKtgmIm0AvNvtFf0ClhQqkYgIrs54tao+6nc8flLVX6tquqpm4BoQ/6OqCftLUFW3AptEpJu3ahTwpY8h+e07YLCI1Pf+b0aRwA3vIeYAV3n3rwLeqOgXqFXRBzTFGgpcCXwuIiu8dXeo6ts+xmSqjp8BM0WkNvANcLXP8fhGVZeIyCvAp7hee8tJsCEvROQFYCTQXERygd8B9wMvi8i1uMQ5rsJf14a5MMYYE2DVR8YYY4IsKRhjjAmypGCMMSbIkoIxxpggSwrGGGOCLCmYhCci94nISBG5UEQq5SpiEdkgIs0r47WMKQ1LCsbAINwYVCOART7HYoyvLCmYhCUiD4nIZ8ApwEfAdcD/ichdInKSiMwVkWUiskhEunvPeVZEnvDWfeWN4YSI1BWRZ0Tkc29Au9O99cki8rC3/jMR+VlICD8TkU+9bYHjjxCRFd6yXERSK/VDMQnPrmg2CUtVp4jILNxV5rcCC1R1KICIvAfcoKrrRGQQ8DhwhvfUDFyp4iRgvoicDNzoHbOPd4L/t4h0xV2V3Anop6rHRaRZSAg7VLW/iPwU+CUuKf0SuFFVP/QGTjwcz8/AmHCWFEyi6wesALrjjTXknYyHALPcsDsA1Al5zsuqWgCsE5FvvOcOA/4MoKprRGQj0BU3sNsTqnrc2xY6Pn5gQMRlwMXe/Q+BR0VkJvCaquZW4Hs1pkSWFExCEpEs3MxW6cAO3CQu4o1JNQLYo6pZUZ4ePjaMAhJpR299tLFkjni3+Xj/i6p6v4j8EzgX+FhEzlTVNSW/I2MqhrUpmISkqiu8k/5XQE/gP8DZqpqlqnuBb0VkHLhMISKZIU8fJyJJInISbnKctcBCYIK3f1egg7f+38AN3uxhhFUfnUBETlLVz1X1ASAHVwoxptJYUjAJS0RaALu9qqDuqho6VPUE4FoRWQmswk34ErAWeB/4F67d4TCuzSFZRD4HXgImqeoR3FSj3wGfece6vISwfuFNVL8SOOS9hjGVxkZJNaYURORZ4C1VfcXvWIyJByspGGOMCbKSgjHGmCArKRhjjAmypGCMMSbIkoIxxpggSwrGGGOCLCkYY4wJ+v8AJDUh2+++3wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "l1,l2=plt.plot(range(1,11),[v[1] for v in bn],range(1,11),[v[1] for v in train_2_loss[:10]])\n",
    "plt.setp(l1,linewidth=2,color='r')\n",
    "plt.setp(l2,linewidth=2,color='b')\n",
    "plt.legend([l1,l2],['Batch_Normalization','network 2'])\n",
    "plt.xlabel('#epochs')\n",
    "plt.ylabel('Losses')\n",
    "plt.title('Cross-Entropy Losses Comparison')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "mod_3.save_params('batch_norm')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Experiment 3: Regularizations\n",
    "\n",
    "- Dropouts applied on middle layer only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Training ('accuracy', 0.66302380952380957)\n",
      "Epoch 6, Training ('accuracy', 0.79440476190476195)\n",
      "Epoch 10, Training ('accuracy', 0.82199999999999995)\n",
      "Epoch 1, Training ('accuracy', 0.63352380952380949)\n",
      "Epoch 6, Training ('accuracy', 0.76245238095238099)\n",
      "Epoch 10, Training ('accuracy', 0.78711904761904761)\n",
      "Epoch 1, Training ('accuracy', 0.53900000000000003)\n",
      "Epoch 6, Training ('accuracy', 0.67002380952380958)\n",
      "Epoch 10, Training ('accuracy', 0.69714285714285718)\n"
     ]
    }
   ],
   "source": [
    "regs=[0.1,0.4,0.6]\n",
    "n_epochs=10\n",
    "display_step=5\n",
    "drop_loss=[]\n",
    "\n",
    "for reg in regs:\n",
    "    drop=[]\n",
    "    data_4 = mx.symbol.Variable('data')\n",
    "    fc1_4  = mx.symbol.FullyConnected(data = data_4, num_hidden=1024)\n",
    "    act1_4 = mx.symbol.Activation(data = fc1_4, act_type=\"relu\")\n",
    "    #dr1_4 = mx.symbol.Dropout(data=act1_4,p=reg)\n",
    "    fc2_4  = mx.symbol.FullyConnected(data = act1_4, num_hidden = 512)\n",
    "    act2_4 = mx.symbol.Activation(data = fc2_4, act_type=\"relu\")\n",
    "    dr2_4 = mx.symbol.Dropout(data=act2_4,p=reg)\n",
    "    fc3_4  = mx.symbol.FullyConnected(data = dr2_4, num_hidden = 256)\n",
    "    act3_4 = mx.symbol.Activation(data = fc3_4, act_type=\"relu\")\n",
    "    #dr3_4 = mx.symbol.Dropout(data=act3_4,p=reg)\n",
    "    #output layer\n",
    "    fc4_4  = mx.symbol.FullyConnected(data = act3_4, num_hidden=10)\n",
    "    mlp_4 = mx.symbol.SoftmaxOutput(data = fc4_4,name='softmax')\n",
    "\n",
    "\n",
    "    bn=[]\n",
    "    mod_4 = mx.mod.Module(symbol=mlp_4,\n",
    "                        context=mx.cpu(),\n",
    "                        data_names=['data'],\n",
    "                        label_names=['softmax_label'])\n",
    "\n",
    "    # allocate memory given the input data and label shapes\n",
    "    mod_4.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)\n",
    "    mod_4.init_params(mx.initializer.Uniform(scale=0.1))\n",
    "    learn_rate=0.001\n",
    "    mod_4.init_optimizer(optimizer='sgd', optimizer_params=(('learning_rate', learn_rate), ))\n",
    "    # use accuracy as the metric\n",
    "    metric = mx.metric.CrossEntropy()\n",
    "    metric2= mx.metric.create('acc')\n",
    "    for epoch in range(n_epochs):\n",
    "        train_iter.reset()\n",
    "        metric.reset()\n",
    "        metric2.reset()\n",
    "        for batch in train_iter:\n",
    "            mod_4.forward(batch, is_train=True)       # compute predictions\n",
    "            mod_4.update_metric(metric, batch.label)  # accumulate prediction loss\n",
    "            mod_4.update_metric(metric2, batch.label)  # accumulate prediction accuracy\n",
    "            mod_4.backward()                          # compute gradients\n",
    "            mod_4.update()                            # update parameters\n",
    "        loss=metric.get()\n",
    "        acc=metric2.get()\n",
    "        drop.append(loss)\n",
    "        if epoch%display_step==0 or epoch==n_epochs-1:\n",
    "            print('Epoch %d, Training %s' % (epoch+1,acc))\n",
    "    drop_loss.append(drop)\n",
    "    mod_4.save_params('dropout_'+str(reg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzsnXd4XNW1t98ljXp3l5vkBhj3gjG4ggkhmJ5QAwkBAgRISOA6JNxLzf0uxJBG6MFgOg41YJwEAzYuGGwZTHGJbVxly9iSLVm9ru+PMxqNRjPSqIxGZb3Ps5+z55x99lnnaHR+s/fae21RVQzDMAwDICLcBhiGYRgdBxMFwzAMw4OJgmEYhuHBRMEwDMPwYKJgGIZheDBRMAzDMDyYKBiGEXZE5HERuSPcdhgmCp0aEblMRLJEpEhEckTknyIyPYz2ZIqIuu3xThcHeb6KyPBQ2xkMInKliKwKtx3NQUSiReRuEdkmIsUisktEnhaRzHDb1hSqer2q/i7cdhgmCp0WEbkF+DPwf0BfYDDwKHBugPKu9rOOVFVN9EqL2qLSdr6HzshrwDnAZUAKMA5YD8wJp1FNISKR4bbB8EJVLXWyhPMPXwRc2EiZu3FeEi8AR4FrgBgcIdnvTn8GYtzlewGLgXzgMLASiHAfuw3YBxQC/wHmBLhmJqCAK8DxhcAjwLvuuj4FhrmPrXCfW+y+t4uB2UC2+/oHgOfdZX8KbHfb+TbQ3+saCvwC2AHkAg/g/PiJcZcf41W2D1AK9PZj65XAqgD30d993cNuO37qdWwKkOV+5t8Cf3Tvj3X/LfLcz3gd0Nfr77kAyHE/5/8FIt3HhgMfAQXu+1kUwKbT3PcyqJHvRGN23w286raxEPgKOAb4LXAQ2Auc7lV+OXAfsNZt2z+AHl7HX3X/zQrcf9tRPt+Dx4Al7r/3ae59/xvEd3Gk+9r5wEbgnGC+X5aa8X4JtwGWWvBHgzOAKgK8fN1l7gYqgfPcL8U44F7gE/fLsDfwMfA7d/n7gMeBKHeaAQhwrPuF0N9dLjPQPxrBicJhnBenC3gReMXruALDvT7Pdt/n73Fe6nHAqe6X40T3vr8CK3zqWAb0wGk9bQWucR97FPi9V9mbgXcC2HolgUXhI3ddscB44BBuoQTWAFe484nAVHf+OuAdIB6IBCYBye5jbwFPAAnuv81a4Dr3sZeB/3b/DWOB6QFsuh/4qInvTWN23w2UAd91/22eA3a6rx2FI8Q7vepajiNgo912vw684HX8KiCJuh8iG3y+BwXANK/7WkidKAT6LkbhiNntQLT7u1AIHBvM98tSkO+XcBtgqQV/NPghcKCJMnd7vyzd+74BzvT6/F1glzt/L86vveE+5wzH+aV4GhDVxDUzcV7K+T5ppPv4QuApr/JnAlu8PvsThQog1mvfAmC+1+dEHPHL9KrjDK/jNwAfuPMn4ghc7a/OLOCiAPdyJX5EARgEVANJXvvuAxa68yuAe4BePuddhSPCY3329wXKgTivfZcCy9z554AngYFNPPu/NfYCDMLuu4GlXsfOxmmx1bZYktzPNtX9eTlwv1f5491/q0g/1051n5vi9T14zqfMQupEIdB3cQZO6yPCa9/LwN3BfL8sBZfMp9A5yQN6BdHHvtfnc39gt9fn3e594HSzbAfeE5EdIvIbAFXdDvwS56VxUEReEZH+AD7O5MFe9fZS1VSvtNnr2AGvfAnOS70xDqlqWaB7UNUinOcxIMB9e+5RVT/F6a6YJSLH4Qje201c35f+wGFVLfS5Ru31r8bpdtkiIutE5Cz3/ueBfwOviMh+EZkvIlFABs4v4BwRyReRfJxWQx/3eb/G+ZW8VkQ2ishVAezKA9JbYTc43V21lAK5qlrt9Rnq/718n3MUzvcyUkTuF5FvROQosMtdpleAc33x+11038NeVa1p5B6a+/0yfDBR6JyswWnqn9dEOd8QuPtxXkK1DHbvQ1ULVfVWVR2K8yvxFhGZ4z72kqpOd5+rON05aH1n8p7W3lRL7kFEEoCeOF0ZtQzyynvu0c2zwOXAFcBrPoITDPuBHiKS5HONfQCquk1VL8V5qf8eeE1EElS1UlXvUdXjgZOBs4Af4bwcy6kvpMmqOspd3wFV/amq9sfpgno0wAit94EpIjKwJXa3EN/nXInTtXcZzoCH03D8JZnuMuJVPmB45ka+i/uBQSLi/d5q7T0YPpgodEJUtQC4E3hERM4TkXgRiRKR74nI/EZOfRn4HxHpLSK93HW8ACAiZ4nIcBERHCdpNVAtIseKyKkiEoMjRKXuY6HgW2BoE2VeAn4iIuPdNv0f8Kmq7vIqM09E0kRkEI7fwHv00/PA+TjC8FwT1xIRifVOqroXpxvoPve+sTitgxfdJ1wuIr3dv2bz3fVUi8gpIjLGPdLmKM4LtFpVc4D3gD+ISLKIRIjIMBGZ5a7vQq8X/RGcl2mD56+q7wNLgTdFZJKIuEQkSUSuF5GrmrK7hVwuIseLSDxOl89r7pZFEo7Q5eH4UP6vOZUG+i7iOI6LgV+7v++zcUTjlVbcg+GDiUInRVX/CNwC/A+Ow3AvcBOO0zIQ/4vTj/4lzuiSz9z7AEbg/NoswmmJPKqqy3Echffj/AI8gPML+PYmzMv36Vq6Jcjbuht41t2NcpG/Aqr6AXAHjmMzBxgGXOJT7B84QzE34IxEWeB1fjbOfSvOqJbGOBlHBD3J3WV3Kc6v3/3Am8BdqrrUfc4ZwEYRKQL+Alzibo30wxkNdhTYjOP0fcF9zo9wHKebcF78r1HXFXQC8Km7vreBm1V1ZwB7f4AzomcRjiP3a2Ayzt+VJuxuCc/j9OMfwHEW/8K9/zmcbp197nv6pJn1+v0uqmoFzpDb7+F8Hx8FfqSqW1pxD4YPomqL7BhdBxFRYITbFxKozNPAflX9n/azrGshIstxRhs9FW5bjLbFJgMZ3Qr37N4LgAnhtcQwOibWfWR0G0TkdzhdKg800gVjGN0a6z4yDMMwPFhLwTAMw/DQ6XwKvXr10szMzHCbYRiG0alYv359rqr2bqpcpxOFzMxMsrKywm2GYRhGp0JEdjddyrqPDMMwDC9MFAzDMAwPJgqGYRiGh07nUzCMjkxlZSXZ2dmUlTU3zp5htA2xsbEMHDiQqKioFp1vomAYbUh2djZJSUlkZmbixHMzjPZDVcnLyyM7O5shQ4a0qA7rPjKMNqSsrIyePXuaIBhhQUTo2bNnq1qqJgqG0caYIBjhpLXfv24jCn/+5M9MeGICH+78MNymGIZhdFi6jSjsKdjDhgMbWLZzWbhNMYyQEhkZyfjx4xk1ahTjxo3jj3/8IzU1NU2fGCLeeustNm3a1Kxz7rvvPoYPH86xxx7Lv//9b79lHn74YYYPH46IkJub2xamGnQjUTgl8xQAlu0yUTC6NnFxcWzYsIGNGzeydOlSlixZwj333NOgXFVVVbvY01xR2LRpE6+88gobN27kX//6FzfccAPV1Q0X+5s2bRrvv/8+GRkZfmoxWkq3EYUZGTOIkAjW7ltLcUVxuM0xjHahT58+PPnkkzz88MOoKgsXLuTCCy/k7LPP5vTTT0dVmTdvHqNHj2bMmDEsWuSsXLp8+XJmzpzJ+eefz/HHH8/111/vaW28/PLLjBkzhtGjR3Pbbbd5rpWYmOjJv/baa1x55ZV8/PHHvP3228ybN4/x48fzzTffNGnzP/7xDy655BJiYmIYMmQIw4cPZ+3atQ3KTZgwAYuD1vZ0myGpqbGpTOg3gfU56/l478d8Z9h3wm2S0dUJlcO5meHuhw4dSk1NDQcPHgRgzZo1fPnll/To0YPXX3+dDRs28MUXX5Cbm8sJJ5zAzJkzAVi7di2bNm0iIyODM844gzfeeIOTTz6Z2267jfXr15OWlsbpp5/OW2+9xXnnnef32ieffDLnnHMOZ511Fj/4wQ8AeOCBB3jxxYZLQ8+cOZOHHnqIffv2MXXqVM/+gQMHsm/fvmbds9Fyuo0ogNOFtD5nPct2LTNRMLoV3uumfOc736FHjx4ArFq1iksvvZTIyEj69u3LrFmzWLduHcnJyUyZMoWhQ4cCcOmll7Jq1SqioqKYPXs2vXs7wTZ/+MMfsmLFioCi4I958+Yxb968oGytxUZ0tR/dShRmZ87mwTUPsnzX8nCbYnQHOsgCVjt27CAyMpI+ffoAkJCQ4DnW2CJbvi9iEQm6fGPj5JtqKQwcOJC9e/d69mdnZ9O/f/+A9RltS7fxKUCdX2Hd/nUUVRSF2xzDCDmHDh3i+uuv56abbvL7a3vmzJksWrSI6upqDh06xIoVK5gyZQrgdB/t3LmTmpoaFi1axPTp0znxxBP56KOPyM3Npbq6mpdffplZs2YB0LdvXzZv3kxNTQ1vvvmm5xpJSUkUFhZ6Ps+bN48NGzY0SA899BAA55xzDq+88grl5eXs3LmTbdu2eWwyQk+3EoXkmGQmpU+iqqaK1XtWh9scwwgJpaWlniGpp512Gqeffjp33XWX37Lnn38+Y8eOZdy4cZx66qnMnz+ffv36AXDSSSfxm9/8htGjRzNkyBDOP/980tPTue+++zjllFMYN24cEydO5NxzzwXg/vvv56yzzuLUU08lPT3dc41LLrmEBx54gAkTJgTlaB41ahQXXXQRxx9/PGeccQaPPPIIkZGRAJx55pns378fwNOqyM7OZuzYsVxzzTWtem6GQ6dbo3ny5MnamkV2blt6G/M/ns9t027j/tPub0PLDAM2b97MyJEjw21Gq1m+fDkPPvggixcvDrcpRgvw9z0UkfWqOrmpc0PWUhCRWBFZKyJfiMhGEWkwUFpEYkRkkYhsF5FPRSQzVPbUMjtzNoD5FQzDMPwQyu6jcuBUVR0HjAfOEJGpPmWuBo6o6nDgT8DvQ2gPANMHTydSIsnan0VheWHTJxhGN2T27NnWSuimhEwU1KHWmxvlTr59VecCz7rzrwFzJMRjz5JikpjcfzLVWs2qPatCeSnDMIxOR0gdzSISKSIbgIPAUlX91KfIAGAvgKpWAQVAz1DaBBbywjAMIxAhFQVVrVbV8cBAYIqIjPYp4q9V0MDzLSLXikiWiGQdOnSo1XaZX8EwDMM/7TIkVVXzgeXAGT6HsoFBACLiAlKAw37Of1JVJ6vq5NqZlK1h2uBpuCJcrM9ZT0FZQavrMwzD6CqEcvRRbxFJdefjgNOALT7F3gZ+7M7/APhQ22GMbGJ0Iif0P4EarTG/gtHl6C6hs2v5+c9/Xi8Yn9E6QtlSSAeWiciXwDocn8JiEblXRM5xl1kA9BSR7cAtwG9CaE89zK9gdFW6S+hsgKysLPLz89vKVIPQjj76UlUnqOpYVR2tqve699+pqm+782WqeqGqDlfVKaq6I1T2+GJ+BaM70JVDZ1dXVzNv3jzmz5/f2sdkeNFtAuLl539Ebu7bpKdfQ0LCSE4edDJREVF8fuBz8svySY1NDbeJRhdj+fLQjK6ePdtCZ4Oz8to555xTL6SG0Xq6jSgcOLCQAwcWEhPTn4SEkSREJzBlwBRW713Nyt0rOfvYs8NtomGEjK4WOnv//v28+uqrLF++POjrGsHRbUQhLe00DhxYyOHDSxk06FbA8Sus3ruaZbuWmSgYbU5zf9GHiq4YOvvzzz9n+/btDB8+HICSkhKGDx/O9u3bA17XCI5uEyU1Le00AAoKVlBTUw6YX8Ho+nTV0Nlz587lwIED7Nq1i127dhEfH2+C0EZ0G1GIju5LQsIYampKKShYA8BJg04iOjKaDQc2cLi0wfQIw+iUdJfQ2UZo6Fahs7dvv4Xs7D8xePB/M3To/wIw85mZrNyzkrcufotzjzu3LU01uiEWOtvoCHTI0NkdkdoupCNH3vfss/kKhmEYdXQrUUhJmYmIi8LCdVRWOhNezK9gGA2x0Nndl24lCi5XIsnJJwE15OcvBxy/QkxkDF98+wV5JXlhtc8wDCPcdCtRAEhL+w4AR44sBSDWFcvUgc5EmRW7V4TNLsMwjI5ANxQF8ysYhmEEotuJQlLSCURGJlFaupWysj2A+RUMwzBq6XaiEBHhIjXVaRkcOfIBACcOPJFYVyxfHfyKQ8WtX8THMMJJdwmdrar893//N8cccwwjR470TH4zWke3EwVo2IUU64rlpIEnAeZXMDo/3SV09sKFC9m7dy9btmxh8+bNXHLJJW1pdrel24tC7eQ98ysYXZGuHDr7scce48477yQiIsJzr0br6TYB8byJjz+O6Oj+VFTsp7j4KxITx5pfwWhz5J7QhM7Wuyx0NsA333zDokWLePPNN+nduzcPPfQQI0aMaNazMRrSLUVBREhL+w7ffvssR468T2LiWKYMmEKcK46NhzZysPggfRLsV4fRdehqobMBysvLiY2NJSsrizfeeIOrrrqKlStXBm2H4Z9uKQrgdCHVisKgQbcQ44rh5EEn88HOD/ho10dcOOrCcJtodHKa+4s+VHTF0NngtCC+//3vA05gv5/85CcBr2kET7f0KQCkpc0BnBXZamoqAPMrGF2Prho6G+C8887jww8/BOCjjz7imGOOabsH143ptqIQE5NOfPwoampKOHr0E8DmKxhdg+4SOvs3v/kNr7/+OmPGjOG3v/0tTz31VKuem+HQrUJn+7Jt2y/Zt+8vZGTcwZAh91JRXUHa79MoqSwh59Yc+iX2a5PrGN0HC51tdAQsdHYL8Z2vEB0ZzbRB0wD4aNdHYbPLMAwjXHRrUUhNnYWIi6NH11JVVQCYX8EwwEJnd2e6tSi4XEkkJ08Fqj2htM2vYBhGd6ZbiwI07EKa3H8yCVEJ/CfvP+wvtLVgDcPoXpgo+IhCVGQU0wdPB8yvYBhG96Pbi0JS0hQiIxMpKdlCWVk2YH4FwzC6LyETBREZJCLLRGSziGwUkZv9lJktIgUissGd7gyVPYGIiIgiNXU2APn5Tiht8ysYnRkR4dZbb/V8fvDBB7n77rtbXF9mZia5ubltYFnz2bBhA0uWLKm376233uLee+9t9LxDhw5x4oknMmHChFaHvvBnQ1uwfPlyzjrrrEbLLF26lEmTJjFmzBgmTZrkmawHcNppp3HkyJE2tyuULYUq4FZVHQlMBW4UkeP9lFupquPdqfG/dIio7UI6fNhZonNS/0kkRiey7fA29h1tGIjLMDoyMTExvPHGGyF9katqu6zR4O+FPH/+fG644YZGz/vggw847rjj+Pzzz5kxY0ab29Bagg1b3qtXL9555x2++uornn32Wa644grPsSuuuIJHH320Te2CEIqCquao6mfufCGwGRgQquu1hrp1m51Q2q4IFzMGO18kay0YnQ2Xy8W1117Ln/70pwbHdu/ezZw5cxg7dixz5sxhz549Dcrk5eVx+umnM2HCBK677jpPvKNdu3YxcuRIbrjhBiZOnMjevXsbDaN96623MnHiRObMmcOhQ87iVRs2bGDq1KmMHTuW888/3/NLd/bs2dROSs3NzSUzM5OKigruvPNOFi1axPjx41m0aBFbt24lJiaGXr16BbyfDRs28Otf/5olS5Ywfvx4SktL691fZmYmd911FxMnTmTMmDFs2bIFgOLiYq666ipOOOEEJkyYwD/+8Q+/NowZM4b8/HxUlZ49e/Lcc88Bzkv6/fffp6ysjJ/85CeMGTOGCRMmsGyZ0w3tG7bcm3Xr1jFhwgR27NhRb/+ECRM8cZ9GjRpFWVkZ5eXlgBMO5OWXX278y9ASVDXkCcgE9gDJPvtnA3nAF8A/gVEBzr8WyAKyBg8erG1NTU2Nrl6drsuWoYWFX6mq6vxV85W70av/cXWbX8/oumzatMmTh9CkpkhISNCCggLNyMjQ/Px8feCBB/Suu+5SVdWzzjpLFy5cqKqqCxYs0HPPPbfB+T//+c/1nnvuUVXVxYsXK6CHDh3SnTt3qojomjVrVFV13759OmjQID148KBWVlbqKaecom+++ab73tEXXnhBVVXvuecevfHGG1VVdcyYMbp8+XJVVb3jjjv05ptvVlXVWbNm6bp161RV9dChQ5qRkaGqqs8884znXFXVp59+Wm+55RbP50D343ueNxkZGfrQQw+pquojjzyiV1/t/I//9re/1eeff15VVY8cOaIjRozQoqKiBnVdd911unjxYv3qq6908uTJes0116iq6vDhw7WwsFAffPBBvfLKK1VVdfPmzTpo0CAtLS3VZ555RgcMGKB5eXmqqrps2TKdO3eurl69WidOnKi7d+/2a28tr776qs6ZM6fevuHDh2tubm6Dst7fw1qALA3ifR1yR7OIJAKvA79U1aM+hz8DMlR1HPBX4C1/dajqk6o6WVUn14bsbWMbG4xCMr+C0ZlJTk7mRz/6UYMlKtesWcNll10GOL9sV61a1eDcFStWcPnllwMwd+5c0tLSPMcyMjI8ax2sW7fOE0bb5XJ5wmgDREREcPHFFwNw+eWXs2rVKgoKCsjPz/cE0Pvxj3/sKR8sOTk5eL8Dgrkff1xwwQUATJo0iV27dgHw3nvvcf/99zN+/Hhmz55NWVmZ35bUjBkzWLFiBStWrOBnP/sZX331Ffv27aNHjx4kJiayatUqTzfPcccdR0ZGBlu3bgXqhy0HJxzFtddeyzvvvMPgwYMD2rtx40Zuu+02nnjiiXr7+/Tp44kF1VaEVBREJApHEF5U1Td8j6vqUVUtcueXAFEi0iuUNgXCVxQmpE8gOSaZb458w96CvY2dahh+CVVbIVh++ctfsmDBAoqLiwOW8Rc5tbH9wYbdDra+Wlwul8dH0VjY7bi4uEaPN3WdWmJiYgBnPeva/n1V9Sw6tGHDBvbs2eM3jtXMmTNZuXIlK1eu9Ijia6+95vFdNPZcvJ8fQHp6OrGxsXz++ecBz8nOzub888/nueeeY9iwYfWOlZWVERcXF9Q9B0soRx8JsADYrKp/DFCmn7scIjLFbU9eqGxqjLpQ2supqak0v4LR6enRowcXXXQRCxYs8Ow7+eSTeeWVVwB48cUXmT59eoPzZs6c6Vnv4J///GfAES6NhdGuqanhtddeA+Cll15i+vTppKSkkJaW5hkN9Pzzz3vKZ2Zmsn79egDPedAw7PbIkSPZvn17s+4nWL773e/y17/+1fNSr31R+9owaNAgcnNz2bZtG0OHDmX69Ok8+OCDHlHwfn5bt25lz549HHvssX6vmZqayrvvvsvtt9/O8uXLGxzPz89n7ty53HfffUybNq3eMVXlwIEDZGZmtvie/RHKlsI04ArgVK8hp2eKyPUicr27zA+Ar0XkC+Ah4BJtzs+PNiQmZgDx8SOpqSn2hNK2+QpGZ+fWW2+tNwrpoYce4plnnmHs2LE8//zz/OUvf2lwzl133cWKFSuYOHEi7733XsBujcbCaCckJLBx40bPMMo773RGmz/77LPMmzePsWPHsmHDBs/+//qv/+Kxxx7j5JNPrmfvKaecwqZNmzxO3pkzZ/L55597XtzB3E+w3HHHHVRWVjJ27FhGjx7NHXfc4dcGcASxdv2GGTNmsG/fPo8g3XDDDVRXVzNmzBguvvhiFi5c6GmZ+KNv376888473HjjjXz66af1jj388MNs376d3/3ud4wfP57x48d7llVdv349U6dOxeVq27XSunXobF+2bbuZffseIiPjToYMuYf1+9cz+W+TGZI6hB0372i6AqPb01VCZ7eWxMREioqKQlL3zTffzNlnn81pp50Wkvo7CzfffDPnnHMOc+bMaXDMQme3Eb5+hfH9xpMSk8LO/J3szt8dTtMMw3Bz++23U1JSEm4zws7o0aP9CkJrMVHwIjV1FhDJ0aOfUlV1lMiISGZmzATMr2AYzSFUrQRwulvOOeeckNXfWfjpT38aknpNFLxwuZJJTj4RJ5S2EwzP/ApGc+lsXbJG16K13z8TBR9svoLRGmJjY8nLyzNhMMKCqpKXl0dsbGyL62hbt3UXIC3tNHbvvtcjCuP6jSMtNo3dBbvZeWQnQ9KGhNlCoyMzcOBAsrOzPWEdDKO9iY2NZeDAgS0+30TBh+TkE4mISKCkZBPl5fuIiRnArMxZvLXlLZbvWm6iYDRKVFQUQ4bYd8TovFj3kQ8REdGeUNpHjrhDaWc4n82vYBhGV8dEwQ++foVThjjO5uW7lltfsWEYXRoTBT94i4KqMrrPaHrG9WTv0b3sOGKT2AzD6LqYKPghIWEUUVF9qajIoaRkMxESwaxMJ0aLjUIyDKMrY6LgB7+htM2vYBhGN8BEIQB1ouAs0Wl+BcMwugMmCgGoFYXaUNrH9z6eXvG92Fe4j+2HtzdxtmEYRufERCEAsbEDiY8/jurqIgoL1xIhETa72TCMLo+JQiOYX8EwjO6GiUIj2HwFwzC6GyYKjeDMbI7g6NFPqKoqZGSvkfRJ6ENOUQ5b87aG2zzDMIw2x0ShEVyuFJKTp6BaRUHBCkTE/AqGYXRpTBSaoLYL6fBhZ2iq+RUMw+jKmCg0QVradwDzKxiG0T0wUWiC5OSpRETEU1KykfLyHI7teSz9EvvxbfG3bMndEm7zDMMw2hQThSZwQmk7cY+OHPmgnl/BupAMw+hqmCgEQaD5CuZsNgyjq2GiEAS+obTNr2AYRlfFRCEIEhJGExXVh4qKfZSUbGFEjxGkJ6ZzqOQQmw5tCrd5hmEYbYaJQhCIRJCWNgdwWgsi4mktmF/BMIyuRMhEQUQGicgyEdksIhtF5GY/ZUREHhKR7SLypYhMDJU9rcV3aKr5FQzD6Iq4Qlh3FXCrqn4mIknAehFZqqre/S3fA0a404nAY+5th6O2pZCfv4yamqp6foUarSFCrNFlGEbnJ2RvMlXNUdXP3PlCYDMwwKfYucBz6vAJkCoi6aGyqTXExg4mLu4YqqsLKSxcx7C0YQxIGkBeaR4bD24Mt3mGYRhtQrv8vBWRTGAC8KnPoQHAXq/P2TQUjg6D9ygk8ysYhtEVCbkoiEgi8DrwS1U96nvYzykNxniKyLUikiUiWYcOHQqFmUHhu0Sn+RUMw+hqhFQURCQKRxBeVNU3/BTJBgZ5fR4I7PctpKpPqupkVZ3cu3fv0BgbBKmpp+CE0l5DVVWRp6Xw0e6PqNGasNllGIbRVgQlCiJys4gku0fvaxg9AAAgAElEQVQLLRCRz0Tk9CbOEWABsFlV/xig2NvAj9z1TgUKVDWnWXfQjkRFpZKUdIInlPaQ1CEMSh7E4dLDfPXtV+E2zzAMo9UE21K4yt31czrQG/gJcH8T50wDrgBOFZEN7nSmiFwvIte7yywBdgDbgb8BNzT7DtoZ8ysYhtGVCVYUavv+zwSeUdUv8O8P8KCqq1RVVHWsqo53pyWq+riqPu4uo6p6o6oOU9UxqprV8ltpHywOkmEYXZlgRWG9iLyHIwr/ds876Fyd6G+9BbNmwbZtraomJeUkIiLiKC7+ivLyA/X8CtU11W1hqWEYRtgIVhSuBn4DnKCqJUA0ThdS52HxYlixAh59tFXVRETEkJIyE4D8/A/JTM0kIyWD/LJ8vvz2y7aw1DAMI2wEKwoKHA/8wv05AYgNiUWh4sYbne0zz0BRUauq8h2aan4FwzC6CsGKwqPAScCl7s+FwCMhsShUTJgA06ZBQQG8+GKrqvINpW1+BcMwugrBisKJqnojUAagqkdwupA6Fzfd5GwffhhasQ5CYuJYoqJ6U16eTWnpVs9KbCt2rzC/gmEYnZpgRaFSRCJxzzYWkd50NkczwAUXQL9+8PXXsHJli6vxDaWdkZrBkNQhFJQXsOHAhray1jAMo90JVhQeAt4E+ojI/wNWAf8XMqtCRXQ0XHedk3/44VZV5Ts09ZRM8ysYhtH5CUoUVPVF4NfAfUAOcJ6qvhpKw0LGtdeCywVvvAH79rW4mjpRcEJp13YhmV/BMIzOTLBhLoYBO1X1EeBr4DsikhpSy0JF//5ON1J1NTzxRIuriY3NIC5uONXVBRQWZtXzK1TVVLWRsYZhGO1LsN1HrwPVIjIceAoYArwUMqtCTa3D+YknoLy8xdV4dyENShnEsLRhFFYU8nnO521hpWEYRrsTrCjUqGoVcAHwF1X9FdAhF8MJiunTYexYOHgQXn+9xdX4LtFpfgXDMDo7zRl9dCnwI2Cxe19UaExqB0TqD09tIU4obeHo0Y+pri42v4JhGJ2eYEXhJziT1/6fqu4UkSHAC6Ezqx247DJITYU1a2D9+hZVERWVRlLSZFQryc9f6RGFlXtWUlld2YbGGoZhtA/Bjj7apKq/UNWXRSQNSFLVpkJnd2wSEuCqq5z8Iy2fnO3tVxiQPIARPUZQVFHEZzmftYWVhmEY7Uqwo4+WuxfZ6QF8ATwjIoEWzuk8/OxnTlfSSy9BXl6LqrD5CoZhdCWC7T5KcS+ycwHOegqTgNNCZ1Y7MXw4fO97zgikBQtaVEVy8slERMRSXPwFFRXfml/BMIxOTbCi4BKRdOAi6hzNXYNah/OjjzpzF5pJZGQsKSkzADhy5EOPKKzas8r8CoZhdDqCFYV7gX8D36jqOhEZCrRutZqOwne/C8OGwe7d8O67LarCe2hqelI6x/Y8luLKYrL2d/iF5AzDMOoRrKP5Vfeymj9zf96hqt8PrWntRERE3VoLLRye6r2+gqqaX8EwjE5LsI7mgSLypogcFJFvReR1ERkYauPajSuvhPh4WLoUtmxp9umJieNwuXpSXr6X0tLt5lcwDKPTEmz30TPA20B/YADwjntf1yAtDS6/3Mm3YLlO31DataKweu9qKqor2spKwzCMkBOsKPRW1WdUtcqdFgK9Q2hX+1PbhbRwIRQWNvt07y6kvol9GdlrJCWVJazbt64NjTQMwwgtwYpCrohcLiKR7nQ50LKB/R2VsWNh5kxHEJ5/vtmn14nCh6hWm1/BMIxOSbCicBXOcNQDOOsp/AAn9EXXohXLdcbFDSE2dpg7lPZ68ysYhtEpCXb00R5VPUdVe6tqH1U9D2ciW6eiyff8eec56y1s3gzLmv8L33t286zMWYDjVyivanl4bsMwjPYk2JaCP25pMyvagVdfhTFjYFtjsyuiouD66518C4aneotCn4Q+jOo9irKqMtbuW9sCiw3DMNqf1oiCtJkV7cC//gUbN8I99zRR8Kc/dcThH/+APXuadY20NCeUdkHBaqqrS8yvYBhGp6M1otBoZ4yIPO2e1/B1gOOzRaRARDa4052tsKVJ7rjDede/9JIjDgHp1w8uvBBqauDxx5t1jaioniQmTkS1goKCVeZXMAyj09GoKIhIoYgc9ZMKceYsNMZC4IwmyqxU1fHudG8z7G42mZlw7bWOX+HOpuSn1uH8t79BWVmzruM9NLXWr/Dx3o8pq2pePYZhGOGgUVFQ1SRVTfaTklTV1cS5K4DDbWptK7n9doiNhTfeaGJdnalTYeJEyM2Fv/+9Wdfw9iv0iu/F2L5jKa8u59PsT1thuWEYRvvQmu6jtuAkEflCRP4pIqMCFRKRa0UkS0SyDh061OKL9e9f1wi4445GCrZiuc6UlOlERMRSVLSBiopDzM6YDZhfwTCMzkE4ReEzIENVxwF/Bd4KVFBVn1TVyao6uXfv1k2kvu02SEyEf/4TVq9upOAll0CPHrBuHawNfvSQE0p7OgD5+R9yyhDH2Wx+BcMwOgNhEwVVPaqqRe78EiBKRHqF+rq9esGvfuXk/+d/Gpm7EBcH11zj5JvZWvDuQpqZMRNBWJO9htLK0hZabRiG0T6ETRREpJ+IiDs/xW1Lu4TOuOUWSE2F5cvhww8bKVi7XOeiRXDwYND114rC4cNLSYtNY1y/cVRUV/BJ9ietM9wwDCPEhEwURORlYA1wrIhki8jVInK9iLhnh/ED4GsR+QJ4CLhEtZmxJVpIairMm+fkG20tZGbC2WdDRQU89VTQ9Scmjsfl6kF5+W5KS78xv4JhGJ2GkImCql6qqumqGqWqA1V1gao+rqqPu48/rKqjVHWcqk5V1Y9DZYs/fvEL6N0bPvmkiQXXah3Ojz0GVVVB1S0SSVraqYDThWR+BcMwOgvhHn0UNhIT4be/dfJ33OHMVfPLnDlw7LGQnQ1vvx10/d5LdM4YPANB+CT7E0oqS1ppuWEYRujotqIAjstgwADYsAFefz1AoRYu11nrV8jP/5DU2GQmpE+gsqaSNXvXtNJqwzCM0NGtRSE21vEpgDPLubo6QMEf/9hpWixb1kSMjDri4oYSGzuEqqojFBZ+bn4FwzA6Bd1aFACuusrxJ2/Z4sRF8ktyMvzoR07+kUeCrtt7aKr5FQzD6Ax0e1GIjoa773byd98NlZUBCtZ2IT33HBQUBFW3dxykGYNnECERrN23luKK4lbZbBiGESq6vSgA/PCHji95xw545pkAhY4/Hk49FYqL4dlng6o3NdUZgVRQsIrEqCgmpk+ksqaSj/e260ArwzCMoDFRAFwuuNcdo/V3v2skMGrt8NRHHmlkuFId0dG9SEyc4A6lvdr8CoZhdHhMFNz84Acwbpwz8vSJJwIUOvtsGDQItm6F998Pql7voanmVzAMo6NjouAmIsJpJQD83/85vUQNcLmccawQ9PBUb2fz9MHTiZRI1u1fx+o9q6muCTTcyTAMIzyYKHhx1lkwZYoT5uivfw1Q6JprHO/04sWwc2eTdaakTEckhqKiz4mVCqYMmEJVTRXTn5lOvz/044o3r+CVr1/hSOmRtr0ZwzCMFmCi4IUI/O//Ovn58wMMMurd2wmrreqEvmiCyMg4UlKmAUp+/jIWnreQn0/5OUNSh5BbkssLX77Apa9fSu8HejNr4Szmr57PxoMbaacwUIZhGPWQzvbymTx5smZlZYWsflU45RT46CNnQts99/gptG6d06RIS3OcEPHxjda5e/d97Nx5O+npP+XYY590X0fZkruFd7e9y7vb3mXVnlVU1dTFVspMzWTuiLnMHTGX2ZmziYuKa8vbNAyjmyEi61V1cpPlTBQasmoVzJgBSUnOMNVe/lZ5OPFEZ/GdBQucGXCNcPToOj77bAqxsUOYOnWH3zIFZQW89817LN62mH9u+yeHSupWmItzxTFn6BzOGnEWZ444k0Epg1pze4ZhdENMFFrJ974H//qXE2J7/nw/BZ5/3pnlPGGCs+CzszSEX1SrWb26N1VVRzjxxG+Iixva6LVrtIZ1+9bx7rZ3Wbx1MZ8f+Lze8bF9xzJ3xFzOOuYsThxwIpERkS25RcMwuhEmCq0kKwtOOMFZgO2bbyA93adAWRkMHgyHDsHHH8NJJzVa39df/4Dc3Nc55pgn6N//2mbZsr9wP0u2LeHdbe+y9JulFFfWDY3qGdeTM4afwdwRc/nu8O/SI65Hs+o2DKN7EKwomKM5AJMnw/nnQ2mpM0S1AbGx8NOfOvkghqd6D01tLv2T+nPNxGt48+I3yft1Hv++/N/8YsovGJY2jLzSPF786kUue+Myej/QmxnPzOD3q37P1we/Nme1YRjNxloKjfDVV86ENpcLtm2DjAyfAnv3OtH0IiNhzx7o1y9gXSUl21m7dgQuVw+mTTuESOv1WFXZmreVxVsX8+62d1m5Z2U9Z/XglMGebqZTMk8xZ7VhdGOs+6iNuOwyePllZ3rC3/7mp8D3vw9vvOHEybjjjoD1qCqffDKE8vLdTJqURVLSpDa3taCsgKU7lvLutndZsm0JB4vr1pWOc8Vx6pBTnRFNx8xlcMrgNr++YRgdFxOFNmLrVicWHsDmzTBihE+BZcucQHn9+8OuXRAVFbCuLVuu4cCBBQwdej+DB98WMpvBcVZn7c/i3a3OkNf1OevrHR/dZzSnDz2dyf0nM6n/JIb3GE5EG7ReDMPomJgotCFXXw1PP+1EU33hBZ+DqjB6NGzaBH//O1x4YcB6vv32FTZvvpS0tNMYN25paI32Iacwp85ZvWMpRRVF9Y4nxyQzMX0ik9InOcmEwjC6FCYKbcju3U4LoaoKvvzS0YB6PPYY3HADzJzpzHoLQEXFIT7+uA8iMZx8cg5RUWmhNTwA5VXlrNyzktV7VrM+Zz3rc9azv3B/g3LeQjG5/2QmpU9iWI9hJhSG0QkxUWhjbrrJiZh9wQV+1nMuKnIWez56FL74AsaODVhPVtZkiorWExmZRHr6Txk48GZiY8Pfv59TmOMIxH5HJLL2Z5FTlNOgXK1QTE53up1MKAyjc2Ci0Mbk5MDQoc70hKwsmOTrJ775ZnjoIbj22kZib0NR0dds3/5z8vOXu/dE0qfPhQwceCvJyU3+vdoVb6HIysli/f71AYXCu9vJhMIwOh4mCiFg3jx48EFntvOSJT4H//MfOO44Jw5SdrYTF6kRCgvXs3fvHzl4cBHghNBOSZnJoEG30rPnWW0yZDUU1ApF1v4sj2D4E4qUmJQ6H4UJhWGEHROFEJCbC0OGOL1Fq1bBtGk+BU4/HZYuhT/+EX71q6DqLCvby759D7F//5NUVx8FIC7uGAYO/BX9+v2IyMjGg+11BPYX7vd0OzVHKCb3n8ywtGFIIyFCDMNoG0wUQsSddzqL8cyeDR9+6BPy6O234dxzYdgwZyxrRPC/iquqjpKTs4Ds7D9TXr4HAJerJwMG3MCAATcSHd23bW8kxPgKRdb+LA4UHWhQLiUmhZG9R5KRkkFmambdNjWDjJQMEqITwmC9YXQ9wi4KIvI0cBZwUFV9x+sgzs/DvwBnAiXAlar6WVP1hlsU8vOd1kJ+vrMi55w5Xgerqx1B2L3b6V/63veaXX9NTRW5ua+zd+8fKCxcB4BIDH37Xs6gQbeQkHB8G91J+xOsUHjTK75XQMHITM0kJTalnaw3jM5NRxCFmUAR8FwAUTgT+DmOKJwI/EVVT2yq3nCLAsB998HttzvRs9es8WktzJ8Pt90GZ54J777b4muoKgUFq9i79w/k5b0NOH+nHj2+x6BBt5KaemqX6HbJKcxh++Ht7Mrfxe6C3fW2ewr2UFFd0ej5KTEpHqHITKkvGBmpGfSM69klnpNhtJawi4LbiExgcQBReAJYrqovuz//B5itqg07o73oCKJQVOSMRDp0CN55x1nG00NeHgwcCOXlThfS8OGtvl5JyVays//MgQMLqakpBSAxcTwDB95Cnz4XExER3eprdERqtIZvi76tLxj5u9lV4Gx3F+ympLKk0ToSohLqC4WXYGSmZtInoY85v41uQWcQhcXA/aq6yv35A+A2VW30jd8RRAHgT3+CW26B8eOd5RTquQ+uugqeecYp8Ic/tNk1Kypy2b//cfbte5jKym8BiI7uz8CBvyA9/dqwTYYLF6pKbkluPcHwbW0cLT/aaB0xkTEMThlMRmoGA5IGkJ6YTnpSOv0S+9XLJ0YnttNdGUZo6Ayi8C5wn48o/FpV1/spey1wLcDgwYMn7d69O2Q2B0tZmdMI2LfPT3SLzz5zJjKkpjrDUxPa1llaXV3GwYMvsXfvHykp2QhAREQC6elXM3DgL4mLG9Km1+vM5JflBxSM3fm7ySvNC6qexOhE0hPdYpGUXpf3EZGe8T2t5WF0SDqDKHTa7qNaHn8cfvYzZ3rC1187EbQ9nHyy43B48sm6dRfaGFXl8OF/k539B691GiLo3fsCBg68lZSUqSG5bleiqKLIIxg5hTnkFOWQU5jDgeID9T6XV5cHVZ8rwkXfhL4NhMNXTPol9iPGFRPiuzOMOjqDKMwFbqLO0fyQqk5pqs6OJAoVFY4g7NwJzz7rrM7p4aWXnAh6Y8fChg2NLtfZFhQVfeGeDPcyqpUAJCefzKBBt9Kr17mI2JKdLUVVKSgvcMSi6ECdcNTmi9z5whyOlB0Jut4ecT3qtzYSHOHoEdeDHnE9SItNc7ZxzjbWFRvCuzS6OmEXBRF5GZgN9AK+Be4CogBU9XH3kNSHgTNwhqT+pCl/AnQsUQBHDK680hmm+p//eEXOrqhwluv89ltYsQJmzGgXe8rL97Nv31/Zv/9xqqryAYiNHcbAgb8kPf0nREbauP9QUl5V7hGLWqHw5L3E5EDRAaq1ull1x7pi/YqF53OA/amxqbaOtxF+UQgVHU0UqqudqKlbtjjdSddd53WwdqbbRRfBokXtaldVVREHDjxDdvafKSvbAYDLlUb//tczYMDPiYnxXXTaaE9qtIa8krwGrY4DRQc4XHqYI2VHnG2psz1cepjKmsoWXy8lJqVJEfHd1zOuJ/FR8Takt4tgotCO/P3vcPHFTqDU7dud5ZsBxwudkeF0He3a5RRoZ1Sryc19i717/8DRo2sAEImiT5/L6N//WpKSJhERYX3bHR1VpaSypIFYNBCPsobHCsoKUFr2fx4dGU3PuJ70iOtBz3j3Nq6n333en62rq+NhotCO1NTAxIlO1Ow//9kJmOrhoovg1VedVsM994TNRoCCgjXs3fsHcnPfBGoARyASE8eRlHQCSUknkJw8hfj448wH0YWorqmmoLwgoJB4PnvtzyvN43DpYcqqylp0zfio+DoB8RWOAILSI64HUZGBVy40WoeJQjvzzjtwzjnQpw/s2OE1CnXFCpg1C/r2hT17IDr8E81KS3ewb99fOXz4PUpKNoPPr8iIiASSkia5ReIEkpKmEBubad0I3ZDSylKPQOSVuLelefXy9bYleeSV5lFVU9Wi6yXHJNcTitTYVBKiE4h3xRMfFe/ko9z5KK98I/tdEa42fiqdExOFdkYVpk6FtWudMBi/+Y3XgXHj4KuvnBFJl14aVjt9qao6SmHhZxQWrqOwcB1Hj66lvLzhPBCXq6dbIOpSTEy/MFhsdHRUlaKKogZiUU9Qyg43EJbDpYep0Zo2tycqIqppQXGLTqByCdEJJEUnkRidWC8lRCd0mnkpJgph4P334TvfcZZS2LHDmbsGOHMVrrvOmbuwenVYbQyGioqDFBZmeUSisHAdlZWHGpSLiRlUr9spKWkSLpcFqDNaRo3WcLT8aD2xyC/Lp7SylOLKYkoqSyipLKG4wp2v8spXlvgtU1xZHBKh8SYhKsEjEkkxDYXDn5j43ec+Nz4qPiRCY6IQBlThlFOcZZrruRCKix0nc0GBExNj4sSw2tlcVJXy8j0cPbrO3aJYS2HheqqrCxuUjYs71qtFMYXExPFERprT0QgPqkpFdUVA0Qhmf3FlMcUVxRRVFNVLhRWFTcbeaim1QuMrMqN6j+LB0x9sUZ0mCmFi1SpnSkJSktNa6NXLfeCWW5yASVddBQsWhNXGtkC1hpKS/9RrTRQVbUC1flRTERcJCWN8HNnHE2H9vEYXoLqmmpLKkgZi0UBAyr32VQbY707FlcUBr3fSwJP4+OqPW2SriUIY+d734F//cpbvnD/fvXP7dhgxwhmvmp0NPXuG1cZQUFNTQXHxV+4WhSMUxcWbqB3pVEtERByJiRM9LYqEhDHExg7B5bKgc4ZRozUBWyYJUQnMypzVonpNFMJIVhaccALExcE330B67TyxM8+Ef/7TUYp588JqY3tRVVVEUdHnFBau9XQ/1U6m8yUqqjexsUOJixvi3g4lNtbJx8QMtNaFYbQCE4Uwc8EF8OabcNNN8Ne/uncuWQJz5zoT2l580RmVlNj9fh1XVOR6HNmFhVmUlm6ltHQnqoGDzom4iIkZXE8ovMXD5ephQ2YNoxFMFMLM1187sfBcLti2zdEBamrgmGOc5gM4M52POQYmTHCczxMmOKkLdi01hWoNFRU5lJbupKxsB6WlOygrq83vpKJiX6PnR0Ym1WtdeItHbGymObuNbo+JQgfghz90piZcfTU89ZR7Z1YWPPYYfP65oxyVfuLZZGTUF4qJE50+qG78S7i6uoyysl1eQuGIhrPd4XcklDfR0f19Whl14hEdnY50krHmhtFSTBQ6AFu3wvHHO/nNmx0/cz0qKmDjRkcgPvvMSV98ASV+hrn16VNfJCZOdEKzdmOhqEVVqao63EAo6vK7gcARSUViiI3NJCZmANHR/QKmqKieJh5Gp8VEoYNw9dXw9NNOq+GFF4I4obraUZNaoajd5uc3LJuSUtflVCsYxx7r9FkZHmpqqigvz/YRih2erip/E/P8E0l0dN9GhaM2RUYmmo/D6FCYKHQQdu92WghVVfDll06Y7Waj6kRZ9RWKAwcalo2Lc5wZ3q2K0aMhxiKhBqKqqoiysp1UVORQUXEgYKqqCn4BnYiI+KDEIzq6LxER4Y+HZXR9TBQ6EDfdBI884oxIev31Nqw4J6ehUOza1bCcywWjRtUXim468qk11NSUU1FxsFHhcFIONTWlQdfrcvWoJxIuVxouVzKRkSm4XHUpMjLZK5+Cy5Vk0WyNoDFR6EDk5MDQoVBW5viZJ00K4cUOH3aW//QWiy1bnNaGNyKOQzsjw1khzl8y0WgRqkp1dVEQ4nGAiopvaczf0RSRkYlBiEeyV75hGWupdA9MFDoY8+bBgw86s52XLGnnixcVOX1X3kIRaOSTN2lpgQVj8GBnRFSk/VJtDao1VFbm1ROJqqp8qqsLqKo6SlVVgTtf4M4f9coX4hv2vCVERMT6EY9UoqJ64HL18LPt6clHRsa1/iEY7YKJQgcjN9cZLFRU5MRHmjYtzAaVlzvrOzSWyppYYMXlcgL9NSYcycntcz/dENUaqqsL/YpHQwGp3X+0QbnWtFQiImK9BKOnXyGJiurZYF9EhC3z2d6YKHRAapdsnjHDGYk0aFAHHlGq6ihZY6Lhz9HtS0pK46LRv7+NlgojqkpNTakf8cinquoIlZWHqao6TGVlnle+bp9vAMRgEYn2Kxa+QuJ0b8USERHn3tblIyOdrflVgsNEoQOSn++0FmpHlyYmOvMYRo1yUm2+Q4uFN+XlTnC/QKKxezeUNuFwjYyE3r2dWdy9ejlb77y/bUoKRNh8gXBTKyiVlXk+YlG3DXSsOY74phCJaiAY3iJSKx7+hKU55ZzjcURExBMZGdfpxMhEoYPy4YdOa2HjRjgUYHh8pxeLWlQdx3djrY2cnIZO8KaIiAhOPLy3aWnm/+hAVFeXBiUk1dUF1NSUeVJ1dak7X7dtC79KS3DEKL6BWDj5OCIj45uVb/xYTKu720wUOgGHDsGmTY5A1G67hVh4U1HhdFPl5QW/LSho/nVEHGFoTDhSU51WiG+Ki+vED7hro6qoVtYTijrhqC8e9bf+Bcb3PKdM3fHq6hL35/YWIyEiIo7k5KmMH/9By2owUei81IqFt1Bs2gQHD/ov3yXFojEqK50WSHPE5EjwE88a4HL5F4tAImLC0uVxus7KPQLhLRbe+caOBc6XUlNTUi+v6owUTE6exsSJq1pks4lCFyQ3t36ronZrYhEEVVWOMDQmHvn5TivEN5UHDukdNIGExV9KSnL+eP5SQgJERbXeHqNToVpNdXUpqlVERaU2fYIfTBS6Ebm5DVsVwYjFiBHOVIN+/epv09Odd1O3E45AlJf7F4vmpKaG9zaHmJiGQhFIRIJNsbH2B+/imCgY9cTCWzQCiYU3sbENxcJbNGrzffrYiNKgaI6wFBc7E1oCpeqWzysISESE/1ZJ7dY739TWOx8fb2LTQegQoiAiZwB/ASKBp1T1fp/jVwIPALUrqDysqk/RCCYKradWLHbtcgb/HDhQf5uTA4WNL0/gQcQZUdqUePTrZ1Ez2gRVR2AaEw3f1JTIFBW1bUvGGxFHGFoiMrUpNtZ/iotztjExJjxBEHZREGcQ71bgO0A2sA64VFU3eZW5EpisqjcFW6+JQvtQXOxfLHz3HTwY/IjSxET/4tGrlzMwKC3N8d3W5pOTbTpCu1FVVV88Cgudz7X7fPOBtr77mpqn0lbExAQWj0Bi0pwUF1eX4uPr8p1IkIIVhVA2/KcA21V1h9ugV4BzgU2NnmV0CBISYNgwJzVGVZUzWiqQaHjni4qcpUm3bQvOBhHHt+ErFsHmoy3OW/B4O8LbkupqZ9Go1ohLebnTkmkslZfXddG1JyL1BcNXNIL53Jxz2mGQQShFYQCw1+tzNnCin3LfF5GZOK2KX6nqXt8CInItcC3A4MGDQ2Cq0VJcrrquosZQhaNHGwpFTk7dwJ8jR5xUmz961Mn7W18oGOLjmycmyclOa6Z28I91h7cBkZHOA01KCt01VJ35LqWlTYuHv9TUeVD/mTkAAAgDSURBVLXHS0sdgSstrUvl5c4+f6slhoITToC1a0N6iVCKgr9/J9+OhneAl1W1XESuB54FTm1wkuqTwJPgdB+1taFG6Kn91Z+SAscdF9w5VVXODz9/guGb93es9n91//6W2+wtEt4jRX33Bbu10aQhQMTpxgnHQlLV1XWC4U802vJzSUm7NH9DKQrZwCCvzwOBev+eqprn9fFvwO9DaI/RyXC56qJZNBdVp/ehOUJSWFi/S72szNkG63QPhtrRpIFEw9fHGh8fXN7mxoWJyMi6P0R7EIqRZz6EUhTWASNEZAjO6KJLgMu8C4hIuqrmuD+eA2wOoT1GN0KkrteipT2OVVX1RcJ7629fU2UKC+u6vvPymr5+c+83Pj44EQlGbPx1aVsrpwPQDvG7QiYKqlolIjcB/8YZkvq0qm4UkXuBLFV9G/iFiJwDVAGHgStDZY9hNBeXy/E3pLZsAmkDVJ3WR1PiUutnrfXP+ub9HSsrq/scKHZWa4mMbOgfDZT8+VKbe050tLV+woFNXjOMLkDtIJ+mxCNYkSkurt+1XVraLj0X9agd2FM7KjTQqNPm7g/2nJiYrhVYtyMMSTUMo51oj0E+lZWB/aHBpuacV1nZvgN7/BEVVScQ0dFOqs2Hal9jx+Li2n7UsC8mCoZhBEVUlJPaa4XVqqq6KQj+piUEs6+151dWOqktBxu0hnYYkWqiYBhGx8TlqhuRFQ5UHWEqLXWmQZSX+9+257EePUJ/3yYKhmEYfhCpax11JyyyjGEYhuHBRMEwDMPwYKJgGIZheDBRMAzDMDyYKBiGYRgeTBQMwzAMDyYKhmEYhgcTBcMwDMNDpwuIJyKHgN3htqOV9AJyw21EB8KeR33sedRhz6I+rXkeGarau6lCnU4UugIikhVMtMLugj2P+tjzqMOeRX3a43lY95FhGIbhwUTBMAzD8GCiEB6eDLcBHQx7HvWx51GHPYv6hPx5mE/BMAzD8GAtBcMwDMODiYJhGIbhwUShHRGRQSKyTEQ2i8hGEbk53DaFGxGJFJHPRWRxuG0JNyKSKiKvicgW93fkpHDbFE5E5Ffu/5OvReRlEYkNt03tiYg8LSIHReRrr309RGSpiGxzb9Pa+romCu1LFXCrqo4EpgI3isjxYbYp3NwMbA63ER2EvwD/UtXjgHF04+ciIgOAXwCTVXU0EAlcEl6r2p2FwBk++34DfKCqI4AP3J/bFBOFdkRVc1T1M3e+EOeffkB4rQofIjIQmAs8FW5bwo2IJAMzgQUAqlqhqvnhtSrsuIA4EXEB8cD+MNvTrqjqCuCwz+5zgWfd+WeB89r6uiYKYUJEMoEJwKfhtSSs/Bn4NVATbkM6AEOBQ8Az7u60p0QkIdxGhQtV3Qc8COwBcoACVX0vvFZ1CPqqag44PzKBPm19AROFMCAiicDrwC9V9Wi47QkHInIWcFBV14fblg6CC5gIPKaqE4BiQtA10Flw95WfCwwB+gMJInJ5eK3qHpgotDMiEoUjCC+q6hvhtieMTAPOEZFdwCvAqSLyQnhNCivZQLaq1rYcX8MRie7KacBOVT2kqpXAG8DJYbapI/CtiKQDuLcH2/oCJgrtiIgITp/xZlX9Y7jtCSeq+ltVHaiqmTgOxA9Vtdv+ElTVA8BeETnWvWsOsCmMJoWbPcBUEYl3/9/MoRs73r14G/ixO/9j4B9tfQFXW1doNMo04ArgKxHZ4N53u6ouCaNNRsfh58CLIhIN7AB+EmZ7woaqfioirwGf4Yza+5xuFvJCRF4GZgO9RCQbuAu4H/i7iFyNI5wXtvl1LcyFYRiGUYt1HxmGYRgeTBQMwzAMDyYKhmEYhgcTBcMwDMODiYJhGIbhwUTB6PaIyH0iMltEzhORdplFLCK7RKRXe1zLMJqDiYJhwIk4MahmASvDbIthhBUTBaPbIiIPiMiXwAnAGuAa4DER+f/t3TtoFFEYxfH/MYU2NoKdiLISgiAmNoIKQREESyGNaQSbYAiIxNZaQWwESWcaC41YRZSAbwIWRvNAyAMUxTKgViYQPRZzsyzREIQkFnt+sOxwZ+7snYGdjzvDfN8VSTVJjyWNSXolqa30GZQ0UNpmSw4nJG2TdFvSVElod7y0t0i6XtonJfU1DKFP0tuybnn/nZLGy+edpO2belKi6eWN5mhati9LGqJ6y/wS8Nz2UQBJT4Ae23OSDgO3gBOl6x6qWUUNeCZpH9Bb9nmgXOBHJLVSvZW8F+iwvSRpR8MQ5m0fknQB6KcKSv1Ar+3RkjhxYSPPQcRKCQrR7DqAcaCNkmuoXIyPAENV2h0Atjb0uWf7FzAn6UPpewy4CWB7WtInoJUqsduA7aWyrjE//nJCxDHgTFkeBW5IugM8sP1lHY81Yk0JCtGUJLVTVbbaBcxTFXFRyUnVCXyz3b5K95W5YQzobxuW9tVyySyW75+U/6Ltq5IeAqeB15JO2p5e+4gi1keeKURTsj1eLvqzwH7gKXDKdrvt78BHSV1QRQpJBxu6d0naIqlGVRxnBngJdJftW4HdpX0E6CnVw1hx++gPkmq2p2xfA95QzUIiNk2CQjQtSTuBr+VWUJvtxlTV3cB5SRPAe6qCL8tmgBfAI6rnDgtUzxxaJE0Bd4FzthepSo1+BibLvs6uMayLpVD9BPCj/EbEpkmW1Ih/IGkQGLZ9/3+PJWIjZKYQERF1mSlERERdZgoREVGXoBAREXUJChERUZegEBERdQkKERFR9xvY4xL70YCjoAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "l1,l2,l3,l4=plt.plot(range(1,11),[v[1] for v in drop_loss[0]],range(1,11),[v[1] for v in drop_loss[1]],\\\n",
    "              range(1,11),[v[1] for v in drop_loss[2]],range(1,11),[v[1] for v in train_2_loss[:10]])\n",
    "plt.setp(l1,linewidth=2,color='r')\n",
    "plt.setp(l2,linewidth=2,color='y')\n",
    "plt.setp(l3,linewidth=2,color='g')\n",
    "plt.setp(l4,linewidth=2,color='b')\n",
    "plt.legend([l1,l2,l3,l4],['Dropout=0.1','Dropout=0.4','Dropout=0.6','No dropout(of network 2)'])\n",
    "plt.xlabel('#epochs')\n",
    "plt.ylabel('Losses')\n",
    "plt.title('Cross-Entropy Losses Comparison')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Experiment 4: Optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Training ('accuracy', 0.75983333333333336)\n",
      "Epoch 6, Training ('accuracy', 0.87764285714285717)\n",
      "Epoch 10, Training ('accuracy', 0.88804761904761909)\n",
      "Epoch 1, Training ('accuracy', 0.70021428571428568)\n",
      "Epoch 6, Training ('accuracy', 0.84319047619047616)\n",
      "Epoch 10, Training ('accuracy', 0.86304761904761906)\n",
      "Epoch 1, Training ('accuracy', 0.67138095238095241)\n",
      "Epoch 6, Training ('accuracy', 0.84159523809523806)\n",
      "Epoch 10, Training ('accuracy', 0.85576190476190472)\n",
      "Epoch 1, Training ('accuracy', 0.53680952380952385)\n",
      "Epoch 6, Training ('accuracy', 0.78890476190476189)\n",
      "Epoch 10, Training ('accuracy', 0.83161904761904759)\n",
      "Epoch 1, Training ('accuracy', 0.76226190476190481)\n",
      "Epoch 6, Training ('accuracy', 0.87385714285714289)\n",
      "Epoch 10, Training ('accuracy', 0.90359523809523812)\n"
     ]
    }
   ],
   "source": [
    "opts=['adam','nag','rmsprop','adadelta','adagrad']\n",
    "n_epochs=10\n",
    "display_step=5\n",
    "optm_loss=[]\n",
    "data_5 = mx.symbol.Variable('data')\n",
    "fc1_5  = mx.symbol.FullyConnected(data = data_5, num_hidden=1024)\n",
    "act1_5 = mx.symbol.Activation(data = fc1_5, act_type=\"relu\")\n",
    "fc2_5  = mx.symbol.FullyConnected(data = act1_5, num_hidden = 512)\n",
    "act2_5 = mx.symbol.Activation(data = fc2_5, act_type=\"relu\")\n",
    "fc3_5  = mx.symbol.FullyConnected(data = act2_5, num_hidden = 256)\n",
    "act3_5 = mx.symbol.Activation(data = fc3_5, act_type=\"relu\")\n",
    "#output layer\n",
    "fc4_5  = mx.symbol.FullyConnected(data = act3_5, num_hidden=10)\n",
    "mlp_5 = mx.symbol.SoftmaxOutput(data = fc4_5,name='softmax')\n",
    "\n",
    "learn_rate=0.001\n",
    "\n",
    "for opt in opts:\n",
    "    mod_5 = mx.mod.Module(symbol=mlp_5,\n",
    "                    context=mx.cpu(),\n",
    "                    data_names=['data'],\n",
    "                    label_names=['softmax_label'])\n",
    "\n",
    "    # allocate memory given the input data and label shapes\n",
    "    mod_5.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)\n",
    "    mod_5.init_params(mx.initializer.Uniform(scale=0.1))\n",
    "    o_loss=[]\n",
    "    mod_5.init_optimizer(optimizer=opt, optimizer_params=(('learning_rate', learn_rate), ))\n",
    "    # use accuracy as the metric\n",
    "    metric = mx.metric.CrossEntropy()\n",
    "    metric2= mx.metric.create('acc')\n",
    "    for epoch in range(n_epochs):\n",
    "        train_iter.reset()\n",
    "        metric.reset()\n",
    "        metric2.reset()\n",
    "        for batch in train_iter:\n",
    "            mod_5.forward(batch, is_train=True)       # compute predictions\n",
    "            mod_5.update_metric(metric, batch.label)  # accumulate prediction loss\n",
    "            mod_5.update_metric(metric2, batch.label)  # accumulate prediction accuracy\n",
    "            mod_5.backward()                          # compute gradients\n",
    "            mod_5.update()                            # update parameters\n",
    "        loss=metric.get()\n",
    "        acc=metric2.get()\n",
    "        o_loss.append(loss)\n",
    "        if epoch%display_step==0 or epoch==n_epochs-1:\n",
    "            print('Epoch %d, Training %s' % (epoch+1,acc))\n",
    "    optm_loss.append(o_loss)\n",
    "    mod_5.save_params('optimizer_'+opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3XlYVGX7wPHvDSi4gbmLqGBiLsgm4orhbqWluVtpq2mLpW2+VmaLli1atlnWq72VSGq2+Gt1wR0FFFdcUBBU3FdElOX5/XGGEZBlBIYBeT7XNZczc55zzj3MOPecZxWlFJqmaZoGYGfrADRN07SyQycFTdM0zUwnBU3TNM1MJwVN0zTNTCcFTdM0zUwnBU3TNM1MJwVN02xOROaKyOu2jkPTSaFcE5FRIhIpIskikiQif4pIVxvG4y4iyhRP9ttwC/dXItLc2nFaQkQeFpH1to7jZohIZRGZJiIHROSyiMSLyH9FxN3WsRVGKTVOKfW2rePQdFIot0RkEvAxMAOoDzQBvgDuy6e8Q+lFR02lVPVst9CSOGgpv4byaAlwLzAKcAF8gCigpy2DKoyI2Ns6Bi0bpZS+lbMbxn/4ZGBoAWWmYXxJ/ABcBB4HHDESyTHT7WPA0VS+DrAcOA+cBdYBdqZtrwBHgUvAPqBnPud0BxTgkM/2BcDnwP+ZjrUZuN20ba1p38um1zYcCAaOmM5/HPjeVPYJINYU52+Aa7ZzKGACcAg4DXyA8ePH0VS+bbay9YArQN08Yn0YWJ/P63A1nfesKY4nsm0LBCJNf/MTwCzT806m9+KM6W8cAdTP9n5+CySZ/s7vAPambc2BNcAF0+sJzSemXqbX0riAz0RBcU8DFptivATsBFoA/wFOAolAn2zlw4B3gS2m2H4FamXbvtj0nl0wvbdtcn0OvgT+ML3fvUzPvWPBZ7GV6dzngd3AvZZ8vvTtJr5fbB2AvhXhTYN+QDr5fPmaykwD0oCBpi/FKsBbQLjpy7AusBF421T+XWAuUMl0CwIEuMP0heBqKuee3380LEsKZzG+OB2AH4FF2bYroHm2x8Gm1zkT40u9CtDD9OXob3ruU2BtrmOsBmphXD3tBx43bfsCmJmt7HPA7/nE+jD5J4U1pmM5Ab7AKUyJEtgEPGS6Xx3oaLr/JPA7UBWwB9oBzqZtvwBfAdVM780W4EnTthDgVdN76AR0zSem94A1hXxuCop7GpAK9DW9N/8D4kznroSRiOOyHSsMI4F5meJeCvyQbfujQA2u/xCJzvU5uAB0yfa6FnA9KeT3WayEkcymAJVNn4VLwB2WfL70zcLvF1sHoG9FeNPgAeB4IWWmZf+yND13ELg72+O+QLzp/lsYv/aa59qnOcYvxV5ApULO6Y7xpXw+162VafsC4Jts5e8G9mZ7nFdSuAY4ZXvuW+D9bI+rYyQ/92zH6Jdt+1PAStP9DhgJLutXZyQwLJ/X8jB5JAWgMZAB1Mj23LvAAtP9tcCbQJ1c+z2KkYS9cz1fH7gKVMn23Ehgten+/4CvAbdC/vbzCvoCtCDuacC/2bYNwLhiy7piqWH629Y0PQ4D3stWvrXpvbLP49w1Tfu6ZPsc/C9XmQVcTwr5fRaDMK4+7LI9FwJMs+TzpW+W3XSbQvl0BqhjQR17Yq7HrsDhbI8Pm54Do5olFvhHRA6JyGQApVQs8DzGl8ZJEVkkIq4AuRqTm2Q7bh2lVM1st5hs245nu5+C8aVekFNKqdT8XoNSKhnj79Eon9dtfo1Kqc0Y1RV3ikhLjIT3WyHnz80VOKuUupTrHFnnfwyj2mWviESISH/T898DfwOLROSYiLwvIpWAphi/gJNE5LyInMe4aqhn2u9ljF/JW0Rkt4g8mk9cZ4CGxYgbjOquLFeA00qpjGyPIef7lfvvXAnjc2kvIu+JyEERuQjEm8rUyWff3PL8LJpeQ6JSKrOA13Czny8tF50UyqdNGJf6Awspl3sK3GMYX0JZmpieQyl1SSn1glKqGcavxEki0tO0baFSqqtpX4VRnYPK2ZicUNwXVZTXICLVgNoYVRlZGme7b36NJt8BDwIPAUtyJRxLHANqiUiNXOc4CqCUOqCUGonxpT4TWCIi1ZRSaUqpN5VSrYHOQH9gNMaX41VyJlJnpVQb0/GOK6WeUEq5YlRBfZFPD60VQKCIuBUl7iLK/XdOw6jaG4XR4aEXRnuJu6mMZCuf7/TMBXwWjwGNRST791ZxX4OWi04K5ZBS6gIwFfhcRAaKSFURqSQid4nI+wXsGgK8JiJ1RaSO6Rg/AIhIfxFpLiKC0UiaAWSIyB0i0kNEHDES0RXTNms4ATQrpMxC4BER8TXFNAPYrJSKz1bmJRG5TUQaY7QbZO/99D0wCCMx/K+Qc4mIOGW/KaUSMaqB3jU9541xdfCjaYcHRaSu6dfsedNxMkSku4i0NfW0uYjxBZqhlEoC/gE+EhFnEbETkdtF5E7T8YZm+6I/h/FlesPfXym1AvgXWCYi7UTEQURqiMg4EXm0sLiL6EERaS0iVTGqfJaYrixqYCS6MxhtKDNu5qD5fRYxGo4vAy+bPu/BGEljUTFeg5aLTgrllFJqFjAJeA2jwTAReAaj0TI/72DUo+/A6F2y1fQcgCfGr81kjCuRL5RSYRgNhe9h/AI8jvELeEoh4Z3PVbU0ycKXNQ34zlSNMiyvAkqplcDrGA2bScDtwIhcxX7F6IoZjdET5dts+x/BeN0Ko1dLQTpjJEHzzVRlNxLj1+8xYBnwhlLqX9M+/YDdIpIMfAKMMF2NNMDoDXYRiMFo9P3BtM9ojIbTPRhf/Eu4XhXUHthsOt5vwHNKqbh84h2C0aMnFKMhdxcQgPG+UkjcRfE9Rj3+cYzG4gmm5/+HUa1z1PSawm/yuHl+FpVS1zC63N6F8Xn8AhitlNpbjNeg5SJK6UV2tFuHiCjA09QWkl+Z/wLHlFKvlV5ktxYRCcPobfSNrWPRSpYeDKRVKKbRvfcDfraNRNPKJl19pFUYIvI2RpXKBwVUwWhahaarjzRN0zQzfaWgaZqmmZW7NoU6deood3d3W4ehaZpWrkRFRZ1WStUtrFy5Swru7u5ERkbaOgxN07RyRUQOF15KVx9pmqZp2eikoGmappnppKBpmqaZlbs2BU3TrCMtLY0jR46QmnqzcwRqZYmTkxNubm5UqlSpSPvrpKBpGgBHjhyhRo0auLu7Y8xFp5U3SinOnDnDkSNH8PDwKNIxdPWRpmkApKamUrt2bZ0QyjERoXbt2sW62qtQSSFrZSFN0/KmE0L5V9z3sMIkhU8//ZSWLVuyZcsWW4eiaZpWZlWYpBAXF8f+/ftZuHChrUPRNK0Ay5YtQ0TYu9dYJiE+Pp4qVarg5+dHq1atCAwM5Lvvvrthv/vuu49OnTrleG7atGmICLGx12dSnz17NiKiB8Hmo8IkhVGjRgEQGhpKenq6jaPRNC0/ISEhdO3alUWLri+odvvtt7Nt2zZiYmJYtGgRs2fPZv78+ebt58+fZ+vWrZw/f564uJwT4LZt2zbHsZYsWULr1q2t/0LKqQqTFNq1a4enpycnTpxg1apVtg5H07Q8JCcns2HDBr799tscX+TZNWvWjFmzZjFnzhzzc0uXLmXAgAGMGDHihv0GDhzIr7/+CsChQ4dwcXGhbt1CpwCqsCpMUhARHnjgAQBdhaRphRGxzq0Qv/zyC/369aNFixbUqlWLrVu35lnO39/fXL0ExtXFyJEjGTlyJCEhITnKOjs707hxY3bt2kVISAjDhw8v3t/mFldhkgJcr0L6+eefuXLlio2j0TQtt5CQEEaMMJbcHjFixA1f8Fmy9yI8ceIEsbGxdO3alRYtWuDg4MCuXbtylM+6gvjll18YNGiQ9V7ALcDqSUFE7EVkm4gsz2Obo4iEikisiGw2LZVoNZ6enrRv355Lly6xfPkN4WialkUp69wKcObMGVatWsXjjz+Ou7s7H3zwAaGhoXl2I9+2bRutWrUCjHbCc+fO4eHhgbu7O/Hx8TdUIQ0YMIDvv/+eJk2a4OzsXHJ/p1tQaVwpPAfE5LPtMeCcUqo5MBuYae1gsq4WdBWSppUtS5YsYfTo0Rw+fJj4+HgSExPx8PDgyJEjOcrFx8fz4osv8uyzzwLG1cVff/1FfHw88fHxREVF3ZAUqlSpwsyZM3n11VdL7fWUV1ZNCiLiBtwDfJNPkfuArL5lS4CeYuXRM8OHD8fOzo4//viDc+fOWfNUmqbdhJCQkBuqdgYPHsyMGTM4ePCguUvqsGHDePbZZ3nkkUeIj48nISGBjh07mvfx8PDA2dmZzZs35zjWiBEj8Pf3L5XXUp5ZdY1mEVkCvAvUAF5USvXPtX0X0E8pdcT0+CDQQSl1Ole5scBYgCZNmrQ7fNiitSLy1bt3b1asWMG8efN4/PHHi3UsTbtVxMTEmKtktPItr/dSRKKUUgGF7Wu1KwUR6Q+cVEpFFVQsj+duyFJKqa+VUgFKqYCS6Eqmq5A0TdPyZs3qoy7AvSISDywCeojID7nKHAEaA4iIA+ACnLViTADcf//9ODo6EhYWxtGjR619Ok3TtHLDaklBKfUfpZSbUsodGAGsUko9mKvYb8AY0/0hpjJWn7HOxcWF/v37o5TKd4CMpmlaRVTq4xRE5C0Rudf08FugtojEApOAyaUVh65C0jRNu1GpLLKjlAoDwkz3p2Z7PhUYWhox5Hb33Xfj4uLC1q1b2bt3Ly1btrRFGJqmaWVKhRrRnJ2TkxODBw8G9NWCpmlalgqbFOB6FdKPP/6oF9/RtDJARHjhhRfMjz/88EOmTZuWo4yPjw8jR468Yd9Zs2bRsmVL2rZti4+PD5MmTSItLc3aId9yKnRSCA4OpmHDhhw6dEgvvqNpZYCjoyM///wzp0+fznN7TEwMmZmZrF27lsuXL5ufnzt3Lv/88w/h4eHs3LmTiIgI6tWrp+c4K4IKnRTs7e3Nk2/pKiRNsz0HBwfGjh3L7Nmz89y+cOFCHnroIfr06cNvv/1mfn769Ol8+eWX1KxZE4DKlSszefJkPc9REZRKQ3NZNmrUKGbPns2iRYv46KOPcHCo8H8STSMszDqzzQQHF15N+/TTT+Pt7c3LL798w7bQ0FD+/fdf9u3bx2effcbIkSO5dOkSycnJeHh4WCPkCqdCXymAsfhOixYtOHnypF58R9PKAGdnZ0aPHp1jER2AiIgI6tatS9OmTenZsydbt27l3LlzKKVyLFb/999/4+vri7u7Oxs3bizt8Mu9Cv+zWEQYNWoU06ZNY+HChfTp08fWIWmazVnyi96ann/+efz9/XnkkUfMz4WEhLB3717c3d0BuHjxIkuXLuXxxx+nWrVqxMXF4eHhQd++fenbty/9+/fn2rVrNnoF5VeFv1IAvfiOppU1tWrVYtiwYXz77bcAZGZmsnjxYnbs2GGeIvvXX381L8Lzn//8h/Hjx3P+/HnAWIQnNTXVZvGXZzopoBff0bSy6IUXXjD3Qlq7di2NGjWiUaNG5u3dunVjz549JCUlMX78eHr16kWHDh3w9vamS5cu+Pn54efnZ6vwy60KX32UZdSoUURERLBw4UKGDrXJIGtNq/CSk5PN9+vXr09KSor5cXh4eI6y9vb2JCUlmR+/+OKLvPjii9YP8hanrxRM9OI7mqZpOimYNWzYkB49enDt2jWWLl1q63A0TdNsosIkhX2n9/F11NfEnYvLt0z2aS80TdMqogqTFN5a+xZPLn+Sv2L/yrdM1uI7a9asuWGxcE3TtIqgwiSFTm6dANh0ZFO+ZbIvvhMaGlpaoWmappUZFSYpdG7cGSg4KYCuQtI0rWKzWlIQEScR2SIi20Vkt4i8mUeZh0XklIhEm26PWyse7/reVK1UldizsZy8fDLfclmL72zbto2YmBhrhaNpWh7s7e3x9fXFy8uLAQMGmAejxcfHIyK8/vrr5rKnT5+mUqVKPPPMMwDs27eP4OBgfH19adWqFWPHjgUgLCwMFxcX/Pz8aNWqFW++ecNXkZaNNa8UrgI9lFI+gC/QT0Q65lEuVCnla7p9Y61gHOwcaO/aHoDwI+H5lsu++E7WaElN00pHlSpViI6OZteuXdSqVYvPP//cvK1Zs2Y5BpcuXryYNm3amB9PmDCBiRMnEh0dTUxMDM8++6x5W1BQENu2bSMyMpIffviBqKioHOdNT0+34qsqX6yWFJQhayRKJdPNphOqmNsVEi2vQtKL72iabXTq1ImjR4+aH1epUoVWrVoRGRkJGDOmDhs2zLw9KSkJNzc38+O2bdvecMxq1arRrl07Dh48yIIFCxg6dCgDBgygT58+KKV46aWX8PLyom3btuZ2xbCwMLp168agQYNo3bo148aNIzMz01ov2+asOqJZROyBKKA58LlSanMexQaLSDdgPzBRKZVorXgsbVfIvfhOhw4drBWSppVJ8qZ1ps5Wb1j2IysjI4OVK1fy2GOP5Xh+xIgRLFq0iAYNGmBvb4+rqyvHjh0DYOLEifTo0YPOnTvTp08fHnnkEfP6ClnOnDlDeHg4r7/+OhEREWzatIkdO3ZQq1Ytli5dSnR0NNu3b+f06dO0b9+ebt26AbBlyxb27NlD06ZN6devHz///DNDhgwpgb9I2WPVhmalVIZSyhdwAwJFxCtXkd8Bd6WUN7AC+C6v44jIWBGJFJHIU6dOFTmejm5G7dWWo1tIy8h/mb7si+/oBmdNKz1XrlzB19eX2rVrc/bsWXr37p1je79+/fj3338JCQlh+PDhObY98sgjxMTEMHToUMLCwujYsSNXr14FYN26dfj5+dGnTx8mT55srnbq3bs3tWrVAmD9+vWMHDkSe3t76tevz5133klERAQAgYGBNGvWDHt7e0aOHMn69eut/aewmVKZ+0gpdV5EwoB+wK5sz5/JVmweMDOf/b8GvgYICAgocn1O3Wp1aV6rObFnY9lxYgftXNvlW/aBBx5g9uzZhIaGMmvWLL34jlahWPqLvqRltSlcuHCB/v378/nnnzNhwgTz9sqVK9OuXTs++ugjdu/eze+//55jf1dXVx599FEeffRRvLy82LXL+LoJCgrKc7LLatWqme8XVFWcfb2GvB7fSqzZ+6iuiNQ03a8C9AL25irTMNvDewGrd/exZLwCgL+/v158R9NsxMXFhTlz5vDhhx+Slpbzqv6FF15g5syZ1K5dO8fzf/31l7ns8ePHOXPmTI5ZVQvTrVs3QkNDycjI4NSpU6xdu5bAwEDAqD6Ki4sjMzOT0NBQunbtWsxXWHZZs/qoIbBaRHYAEcC/SqnlIvKWiNxrKjPB1F11OzABeNiK8QDX2xU2Jha8IlPW4jugq5A0zRb8/Pzw8fFh0aJFOZ5v06YNY8aMuaH8P//8g5eXFz4+PvTt25cPPviABg0aWHy+QYMG4e3tjY+PDz169OD9998379+pUycmT56Ml5cXHh4eDBo0qHgvrgyT8ta7JiAgQGX1PrgZGRmwbRtcqbGTbou8ca/pTtxz+c+DBHDgwAFatGhB9erVOXnyJFWqVClq2JpW5sXExNCqVStbh1HmhIWF8eGHH5artVbyei9FJEopFVDYvhVmRPOIEdC+PcRvaUP1ytWJPx/P8eTjBe6TtfhOcnJyufpAaJqmFVWFSQqdjKYEVq60o0Mjo4tpYeMVQE97oWkVXXBwcIX6UVhhkkKvXsa/K1ZAx0ZGhiisXQH04juaplUsFSYptG0L9erB0aPgds3IEIX1QILri++kpaXpxXc0TbvlVZikIHL9auF8jNHWEnkskmsZ1wrdV1chaZpWUVSYpACQNThyY1g1WtZpydWMq0Qfjy50P734jqZpFUWFSgpZVwphYdChgTH4xJJ2heyL7+TuM61pWslatmwZIsLevXvz3P7www+zZMmSAo/x8MMP4+HhgY+PDy1atGD06NE5JtfLT3BwsHnCvRkzZtx88LeACpUU3NygZUu4dAnqnx8AWNauAMa0FwALFy60WnyaphlT1nft2rXYP8A++OADtm/fzr59+/Dz86N79+5cu1Z4dXEWnRQqiKyrheR9lndLBbjrrrv04juaZmXJycls2LCBb7/91pwUlFI888wztG7dmnvuuYeTJ68vkvXWW2/Rvn17vLy8GDt2bJ7zF4kIEydOpEGDBvz555+AMfq5U6dO+Pv7M3ToUJKTk3PsM3nyZPPkfFk/CAcOHEi7du1o06YNX3/9tbX+BDZX4ZJCVrvC9o31cHZ0JvFiIkcuFt5OkH3xHX21oN3qRKxzK8wvv/xCv379aNGiBbVq1WLr1q0sW7aMffv2sXPnTubNm8fGjderfJ955hkiIiLYtWsXV65cKXA8gb+/P3v37uX06dO88847rFixgq1btxIQEMCsWbNylH3vvffMk/NldTD573//S1RUFJGRkcyZM4czZ87kdZpyr8IlhTvvBHt7CA8XAmr3ACy/WshehVTepgfRtPIgJCTEPG39iBEjCAkJYe3ateYprV1dXenRo4e5/OrVq+nQoQNt27Zl1apV7N69O99jZ/2fDQ8PZ8+ePXTp0gVfX1++++47Dh8+XGhsc+bMwcfHh44dO5KYmMiBAweK+WrLpgo3H7SLC3ToABs3Qr1TQ4Ff2HRkE0PbDC103zvvvFMvvqNVCLb4zXPmzBlWrVrFrl27EBEyMjIQEQYNGpTnVNWpqak89dRTREZG0rhxY6ZNm0Zqamq+x9+2bRs9e/ZEKUXv3r1varndsLAwVqxYwaZNm6hatSrBwcEFnqs8q3BXCnC9Cil1fxfA8sZmvfiOplnPkiVLGD16NIcPHyY+Pp7ExEQ8PDyoVasWixYtIiMjg6SkJFavXg1g/lKuU6cOycnJ+fZIUkoxZ84ckpKS6NevHx07dmTDhg3ExsYCkJKSwv79+2/Yr1KlSuapuC9cuMBtt91G1apV2bt3L+Hh+a/zXt5VyKSQ1dgcs9kNQYg6FkVqumVZP6sKKTQ0VC/2rWklKCQk5IYpqQcPHszx48fx9PSkbdu2jB8/njvvvBOAmjVr8sQTT9C2bVsGDhxI+/btc+z70ksvmbukRkREsHr1aipXrkzdunVZsGABI0eOxNvbm44dO+bZ/XXs2LF4e3vzwAMP0K9fP9LT0/H29ub111+nY8eO1vtD2FiFmTo7u7Q0qFULkpOhxTs92Z++ig2PbjCvtVAQpRQtW7Zk//79/PXXX/Tt27dYsWhaWaGnzr516Kmzb1KlShAcbNxvcMr45W9pY3P2xXd0LyRN0241FTIpwPV2hWv7jUtRS9sV4PpcSD///DNXrlwp8dg0TdNsxZprNDuJyBYR2W5acvPNPMo4ikioiMSKyGYRcbdWPLlltSvERjYFZUx3YWlVWvbFd3IvHK5pmlaeWfNK4SrQQynlA/gC/UQkd+vMY8A5pVRzYDYw04rx5NCqFbi6wulTDjhf6ExSchIJFxIs3l9Pe6Fp2q3IaklBGbLGjlcy3XL/FL8P+M50fwnQU/LqkGwF2afSdj1lLAJ+M1VIw4YN04vvaJp2y7Fqm4KI2ItINHAS+FcptTlXkUZAIoBSKh24ANTO4zhjRSRSRCJPnTpVYvFltStkxN7cyGbIufhOYTM2apqmlRdWTQpKqQyllC/gBgSKiFeuInldFdxQsa+U+lopFaCUCqhbt26Jxdezp/Fv4g4PSK/ExiOFT6Odna5C0rSSVxJTZwPMmjWLli1b0rZtW3x8fJg0aZJ5MFpRxMfH4+WV+yvs1lMqvY+UUueBMKBfrk1HgMYAIuIAuABnSyMmgIYNwcsLUq/YI0e7EH08mitplvcmGjRokF58R9NKWElMnT137lz++ecfwsPD2blzJxEREdSrVy/P3oIZGRnFCfeWY83eR3VFpKbpfhWgF5A79f8GjDHdHwKsUqU8mi6rXaHu8ZGkZ6YTeczygXF68R1NK1klNXX29OnT+fLLL6lZsyYAlStXZvLkyTg7OwNQvXp1pk6dSocOHdi0aVO+x4mKisLHx4dOnTrx+eefl+afwmaseaXQEFgtIjuACIw2heUi8paI3Gsq8y1QW0RigUnAZCvGk6esdgUOGXcsWYktO12FpN2KRMQqt8KUxNTZly5dIjk5GQ8Pj3zPc/nyZby8vNi8eTNdu3bNdwruRx55hDlz5rBpk+XtjeWdNXsf7VBK+SmlvJVSXkqpt0zPT1VK/Wa6n6qUGqqUaq6UClRKHbJWPPnp1s0Y4XzqQFO44nJTPZBAL76jaSWpJKbOVkrlSEB///03vr6+uLu7mxOKvb29eX2U/I5z4cIFzp8/b55r6aGHHiqNP4HNVdgRzVmqV4dOnUBl2kF8dzYd2XRTayU4OTkxZMgQQF8taLcOpZRVbgXJmjr78ccfx93dnQ8++IDQ0NAbvuSzZE2dvWTJEnbu3MkTTzxBamoqzs7OVKtWjbi4OAD69u1LdHQ0Xl5e5uU4nZycsLe3L/A4+Z33VlfhkwJcb1dwShjAycsniTsfd1P7Z58LqbxNMKhpZUVJTp39n//8h/Hjx3P+/HnASHL5rX+Q33Fq1qyJi4sL69evByrOdPkVbpGdvPTuDVOngn2cMePpxsSNNLutmcX7Z198Z/Pmzbf0tLqaZi0hISFMnpyzWXHw4MHExMSYp85u0aJFnlNnu7u755g6e/z48aSkpNChQwccHR2pXr06Xbp0wc/P74bzFnSc+fPn8+ijj1K1atUKMyNyhZw6O7f0dKhTBy5cAJ5vylO9+vP5PTfX0+CFF15g1qxZPPvss8yZM6dE49O00qCnzr516Kmzi8nBAbp3Nz041OumG5vhehWSXnxH07TyTCcFk6x2BTnUmx0ndpB8LbngHXLx9/enRYsWnDx5kpUrV1ohQk3TNOvTScEka7yCfXxfMjIyiTgacVP7i4ges6BpWrmnk4KJpyc0bgzpybfBCZ8iVSGNHDkS0IvvaJpWfumkYCKSfXRz0doV9OI7mqaVdxUrKShl3PKR1a7Awd5sSry5QWxZdBWSpmnlWcVJCp98Ai1bQgFzmGRNpU1CEGcuJnPg7IGbPs3w4cPNi++cPVtqE75qmqaViIqTFBISYP9+WLw43yL16oGPD5BeBRK73NSiO1kaNGh4N9nFAAAgAElEQVRAz549SUtLY+nSpcUIWNMqnunTp9OmTRu8vb3x9fVl82ZjXa709HSmTJmCp6cnvr6++Pr6Mn36dPN+9vb2+Pr60qZNG3x8fJg1axaZmZnm7du2bePxxx8vclwzZswo+osqgLu7O6dPn853e0pKCvfccw8tW7akTZs2OQb3ffbZZ8yfP7/kg7LWHCfWurVr104VyaZNRuWRm5tSGRn5FnvxRVMdU9cZ6snfnyzSqebPn68AFRwcXLRYNc0G9uzZY9Pzb9y4UXXs2FGlpqYqpZQ6deqUOnr0qFJKqVdeeUWNGTNGXblyRSml1MWLF9Ubb7xh3rdatWrm+ydOnFA9e/ZUU6dONT83ZMgQFR0dXeTYsh+/pKSnp6umTZuqU6dO5Vvm8uXLatWqVUoppa5evaq6du2q/vjjD/M2X1/fPPfL670EIpUF37EVZ5qLwEBwc4MjR2DLFshnKopeveDDD4GDvdmY+GiRTjVo0CDGjRtnXnzHzc2tGIFrWumTsDCrHFcFB+e7LSkpiTp16uDo6AgYcxGB8Wt53rx5xMfH4+TkBECNGjWYNm1ansepV68eX3/9Ne3bt2fatGkkJyezY8cOfHx8AJg2bRoJCQkcOnSIhIQEnn/+eSZMmADADz/8wJw5c7h27RodOnTgiy++4NVXX+XKlSs5rkScnJyYMGECEydOZPv27axatYqVK1cyf/58fvjhB0JCQpgxYwZKKe655x5mzpwJGOs4TJo0ib///puPPvrIHPOVK1cYNGgQgwcP5oknnjA/X7VqVbqbRtZWrlwZf39/84JeVatWxd3dnS1bthAYGHizb0W+Kk71kZ0dmGYzpYCl/IKCoHJlBcf92Rl/jItXL970qVxcXBgwYIBefEfTbkKfPn1ITEykRYsWPPXUU6xZswaA2NhYmjRpQo0aNSw+VrNmzcjMzOTkyZNERkbesIzm3r17+fvvv9myZQtvvvkmaWlpxMTEEBoayoYNG4iOjsbe3p4ff/yR9957jypVqhAdHc2PP/5It27dWLduHQCRkZEkJyeTlpbG+vXrCQoK4tixY7zyyiusWrWK6OhoIiIi+OWXX4Ab13EAY2GhAQMGMGrUqBwJIbfz58/z+++/09Pc+AkBAQHmWEpKxblSACMpfPyxkRQ++MDoh5pL1arQpYuwerVAXHe2HN1Cr2a98jhYwUaNGsWSJUtYuHAhL774YklEr2mlpqBf9NZSvXp1oqKiWLduHatXr2b48OG89957+Pv75yg3f/58PvnkE86cOcPGjRtp3LhxnsdTpt6DSUlJ5F7b/Z577sHR0RFHR0fq1avHiRMnWLlyJVFRUeYJ8a5cuUK9evVuOG67du2Iiori0qVLODo64u/vT2RkJOvWrWPOnDlEREQQHBxsPucDDzzA2rVrGThw4A3rOADcd999vPzyy+aei3lJT09n5MiRTJgwgWbNrk/WWa9evXzXsi6qinOlAMbCCa6ucPgwFDCpXo7xCkVobAa9+I6mFYW9vT3BwcG8+eabfPbZZyxdupTmzZuTkJDApUuXAGM1tOjoaFxcXPJdX/nQoUPY29tTr149qlSpcsO02VlVVFnnTE9PRynFmDFjiI6OJjo6mn379uVZRVWpUiXc3d2ZP38+nTt3JigoiNWrV3Pw4EFatWpVYFf27Os4ZOnSpQt//vlngfuNHTsWT09Pnn/++RzPp6amUqVKlXz3KwprrtHcWERWi0iMiOwWkefyKBMsIhdEJNp0m2qteACjCikrSxdQhZR9vMLGIze3PGcWvfiOpt2cffv2ceDA9W7g0dHRNG3alKpVq/LYY4/xzDPPmL/cMzIyzAvm5Hbq1CnGjRvHM888g4jQqlUrYmNjCz1/z549WbJkiXkN6LNnz3L48GHASARpaWnmst26dePDDz+kW7duBAUFMXfuXHx9fREROnTowJo1azh9+jQZGRmEhISYp/vOy1tvvUXt2rV56qmn8tz+2muvceHCBT7++OMbtu3fv/+GqrHisuaVQjrwglKqFdAReFpEWudRbp1Sytd0e8uK8RiGDjX+Xbw434Fs/v7gUjMDzjdjw/YkMlVmnuUKoxff0TTLJScnM2bMGFq3bo23tzd79uwx/1KfPn06DRs2xMvLCz8/P4KCghgzZgyurq4AORqCe/XqRZ8+fXjjjTcAaNmyJRcuXDBfaeSndevWvPPOO/Tp0wdvb2969+5NUlISYPxS9/b2NlfxBAUFkZSURKdOnahfvz5OTk4EBQUB0LBhQ9599126d++Oj48P/v7+3HfffQWe++OPPyY1NZWXX345x/NHjhxh+vTp7NmzB39/f3x9ffnmm2/M2zds2ECvXjdfvV0gS7oolcQN+BXoneu5YGD5zRynyF1Ss6SnK9WggdHvNCoq32KDB2caXVP7j1V7Thatq156erpydXVVgNq0aVNRI9a0UmHrLqnWNGvWLDVv3jxbh1Gitm7dqh588ME8txWnS2qptCmIiDvgB2zOY3MnEdkuIn+KSJt89h8rIpEiEnnq1KniBWNvf70KqYCBbL17mxqhizgPknEqe/Mi5LoKSdNsZ/z48TnaEW4Fp0+f5u233y7x41qUFETkORFxFsO3IrJVRPpYuG91YCnwvFIqd//OrUBTpZQP8CnwS17HUEp9rZQKUEoF5O5FUCTZu6bmU61zvbG5J+vji5YUQC++o5Uv6hat5nRycuKhhx6ydRglqnfv3ri7u9/wfHHfQ0uvFB41faH3AeoCjwDvFbaTiFTCSAg/KqV+zr1dKXVRKZVsuv8HUElE6lgafJEFBRlzWsTGwvbteRZp1gxcG6dCai3Cwi8U+VT+/v7ccccdevEdrcxzcnLizJkzt2xiqAiUUpw5c8Y8yK8oLB2nkNWh/25gvlJqu0genfyz72Bs/xaIUUrNyqdMA+CEUkqJSCBGkjpjYUxFZ28P998Pc+caVwu+vnkWu6tvJb79BuKibud86nlqOtW86VOJCKNGjeKNN95g4cKFFWbxb638cXNz48iRIxS7ilazKScnp2LNoiCW/CoQkflAI8AD8AHsgTClVLsC9ukKrAN2Alndd6YATQCUUnNF5BlgPEZPpSvAJKVUgX1AAwICVGQBYwwstnKl0fe0RQvYuzfPgWyLF8OwYYDHSv78J41+zfsV6VSxsbF4enpSvXp1Tpw4QdWqVYsZvKZp2s0RkSilVEBh5SytPnoMmAy0V0qlAJUxqpDypZRar5QSpZS3ut7l9A+l1Fyl1FxTmc+UUm2UUj5KqY6FJYQSdeedUKeOMXPqrl15FunRA5BMSOjK2gNFT0TNmzcnMDCQ5ORkli9fXuTjaJqmWZulSUEBrYEJpsfVgKJXWpUFDg4waJBxP59eSLVrQ7PW5yHDkb9WXS7W6bKPWdA0TSurLE0KXwCdgJGmx5eAz60SUWnKGshWwOjmu/tWAmD35oZkZOY9pN4SevEdTdPKA0uTQgel1NNAKoBS6hxGFVL5FhwMtWpBTAzs3p1nkYF3GzMzXtvfjT2n9hT5VHrxHU3TygNLk0KaiNhjVCMhInW53nhcflWqdL0KKZ+rhS5dwL7SNTjhy987thXrdFlVSD/++GOxjqNpmmYtliaFOcAyoJ6ITAfWA9ZZn660FbLGgpMTNPc7DsDvf6UU61T3338/jo6OrFmzhrVr1xbrWJqmadZgUVJQSv0IvAy8CyQBA5VS+c8RUZ706AE1axo9kPKZl7yPacqL7RuLN67O2dmZhx9+GIBevXoxb968Yh1P0zStpFk6zcXtQJxS6nNgF9BbRG5+JFdZVLkyDBxo3M/nauHBgQ0AuLAnkNOXize27rPPPmPixImkpaUxduxYnn766RxT8mqaptmSpdVHS4EMEWkOfIMxiO3W6VuZVYWUT9fUAP9KOFQ/Dxeb8PP6ncU6lYODA7NmzWLBggVUrlyZL774gt69e+tRpJqmlQmWJoVMpVQ6cD/wiVJqItDQemGVsl69wMUFduwwBrPlYmcHzdsZi238/H8Fz8luqTFjxrB27VoaNmzImjVraN++PdvzmYdJ0zSttNxM76ORwGgga0huJeuEZAOOjnDvvcb9fKqQevYyOltFbbitxE7boUMHIiMjCQwM5PDhw3Tu3JklBYyZ0DRNszZLk8IjGIPXpiul4kTEA/jBemHZQCED2R6+35hg6vTutqReK7kpsF1dXVmzZg2jR48mJSWFoUOHMnXqVDIzy3+PX03Tyh9Lex/tUUpNUEqFiMhtQA2lVKFTZ5crvXtDjRqwbRscPHjD5oDWdXGoGwdXXfjp30MlemonJycWLFjArFmzsLOz4+233+b+++8vdPlATdO0kmZp76Mw0yI7tYDtwHwRyXM67HLLyanQKiQPfyNZLP79fImfXkSYOHEif/75JzVr1uTXX3+lU6dOHMwjQWmaplmLpdVHLqZFdu7HWE+hHVDCq0WXAYX0QgrucQ2ALeucrRZCnz592LJlC61atWL37t20b9+eFStWWO18mqZp2VmaFBxEpCEwjOsNzbeevn2henWIioK4uBs2P3BvI5AMTu5rRnKy9cLw9PQkPDycAQMGcO7cOfr27cvHH3+sV8TSNM3qLE0KbwF/AweVUhEi0gw4YL2wbKRKFejf37ifRxVSlxZtsHOLgozK/PZPyVchZefs7Mwvv/zCq6++SmZmJhMnTuTRRx/l6tWrVj2vpmkVm6UNzYtNi+WMNz0+pJQabN3QbKSAuZAc7Bxw890HwKLfrL9qqJ2dHe+88w6hoaFUqVKFBQsWEBwcTFJSktXPrWlaxWRpQ7ObiCwTkZMickJElopIgYuAikhjEVktIjEisltEnsujjIjIHBGJFZEdIuJf1BdSYu66C6pWhS1b4PDhGzYHBacCsGlNtVILadiwYWzcuJEmTZoQHh5OQEAAW7ZsKbXza5pWcVhafTQf+A1wxVir+XfTcwVJB15QSrUCOgJPi0jrXGXuAjxNt7HAlxbGYz1Vq8I99xj381j34P7erlDpMqfjG1CaP9h9fX2JiIggKCiIY8eO0a1bN77//vvSC0DTtArB0qRQVyk1XymVbrotAOoWtINSKkkptdV0/xIQg5FQsrsP+J8yhAM1TQ3atlXAQLZut3eApmsA+PvfkhvEZol69eqxYsUKxo0bx9WrVxk9ejQvvvgi6emlG4emabcuS5PCaRF5UETsTbcHAYsr1UXEHfADNufa1AhIzPb4CDcmDkRkrIhEikhkqUwcd/fdRqPzpk2QmJhjU52qdajjZcxRtOT3C9aPJZfKlSvz5Zdf8uWXX+Lg4MBHH33EPffcw7lz50o9Fk3Tbj2WJoVHMbqjHsdYT2EIxtQXhRKR6hizrD5vGuuQY3Meu9zQ71Ip9bVSKkApFVC3boEXKCWjWjUjMQD8/PMNmzsFG/1R14U5YqteouPGjWPlypXUqVOHf/75h8DAQGJiYmwTjKZptwxLex8lKKXuVUrVVUrVU0oNxBjIViARqYSREH5USt347WpcGTTO9tgNOGZJTFZXwEC2uzo3hmrHuXi6Orb8Hu7WrRuRkZH4+PgQGxtLhw4dWL781h1Gomma9Vl6pZCXSQVtFBEBvgVilFL5TYnxGzDa1AupI3BBKVU2+lvec48x9cWGDXD0aI5NXZp0hmbGKON//7VFcNc1bdqUDRs2MGzYMC5dusS9997LjBkz9EA3TdOKpDhJIa+qn+y6AA8BPUQk2nS7W0TGicg4U5k/gENALDAPeKoY8ZSsGjWgXz/jfq4qpDZ12+DUYj0Ay/9KLe3IblCtWjUWLVrE9OnTAXj11VcZOXIkKSnFW1Na07SKpzhJocCfokqp9UopMQ168zXd/lBKzVVKzTWVUUqpp5VStyul2iqlIosRT8nLZyCbvZ097boaI5rXr7WnLKymKSJMmTKFX3/9lRo1ahAaGkrXrl1JSEiwdWiappUjBSYFEbkkIhfzuF3CGLNwaxswwFjDed06cg9K6O7tCXViSE2pxObcfapsaMCAAYSHh3P77bezbds2AgICWLduna3D0jStnCgwKSilaiilnPO41VBKOZRWkDbj7GxMkqcULFuWY1Pnxp2hmdGgYOt2hdxat27Nli1bzGs/9+jRg6+++srWYWmaVg4Up/qoYshnIFtHt47ZGpvL3ipptWrV4o8//mDSpEmkp6czbtw4nnrqKa5du2br0DRNK8N0UijMgAFQqRKsWQMnT5qfvq3KbXi2OwaSzpYtwoXSH8dWqKzBbd999x2Ojo58+eWX5qsHTdO0vOikUJiaNaFPH8jMvKEKKcjTB9zCycgQ1qyxUXwWGD16NGvWrKFhw4asXbuWgIAAoqOjbR2WpmllkE4KlshnIFunxp3KzHiFwnTo0IHIyEgCAwNJSEigS5cu/PDDD2Rmlr2qL03TbEcnBUvcdx84OEBYGGSreunk1gluN7JBeVgx09XVlTVr1jBmzBhSUlJ46KGH8Pb25rvvvtNtDZqmATopWOa226BXL8jIgF9+MT/dqm4rnJvtg8oX2bsXjhyxYYwWcnJyYv78+Xz11Vc0atSI3bt38/DDD3P77bcza9YsLl26ZOsQNU2zIZ0ULJVHLyQ7saNT0wBwDwPKx9UCGAPdxo4dy6FDh1iwYAGtW7fmyJEjvPDCCzRu3JgpU6Zw4sQJW4epaZoN6KRgqfvuA3t7WLkSzlyfNbyTW/lpV8itcuXKjBkzhp07d/L777/TtWtXLly4wLvvvkvTpk158sknOXDg1luKW9O0/OmkYKnataFHD6MK6ddfzU93apyzXaE8zkNnZ2dH//79WbduHRs2bGDgwIFcvXqVr7/+mjvuuIMhQ4YQERFh6zA1TSsFFSYpZGZe49ixr1Eqo+gHyapCytYLqUOjDlBnH9Q4ysmTsHNnMQO1sc6dO7Ns2TJiYmJ47LHHcHBwYOnSpQQGBtK9e3f++usvPQOrpt3CKkxS2LVrIPv3P8mRI58W/SADBxpVSCtWgGmlMxcnF7zqtzFPeVFe2hUK07JlS7755hvi4+N5+eWXcXZ2JiwsjLvuugtfX19+/PFH0srCTICappWoCpMUXF2NWbnj4l7lypX4oh2kbl0IDob09JxVSOW4XaEwrq6uzJw5k4SEBGbOnEnDhg3ZsWMHDz74IM2bN2fOnDlcvnzZ1mFqmlZCKkxSqFOnP3XrDiczM4X9+58sehVIHtNpZ08Ka9bA1avFjbbscXFx4eWXXyYuLo5vvvmGO+64g4SEBJ577jmaNGnC1KlT9fQZmnYLqDBJAUDcPsTBoRbnzv3DiRM/FO0ggwaBnR388w9ZEx51btwZapzAoUEMV67Apk0lGHQZ4+joyGOPPcaePXtYtmwZHTt25OzZs7z99ts0adKEp59+mkOHDtk6TE3TiqjCJIX/HDpE222xHGn4BQCxsc9z7drJQvbKQ/360K0bpKXBb78B0KJ2C2pVqUW6+5/ArVeFlBc7OzsGDhzIxo0bWbt2Lf379yc1NZUvvvgCT09PRowYwdatW20dpqZpN8lqSUFE/isiJ0VkVz7bg0XkQralOqdaKxaA6vb2ZAITTzQGl3tJTz9LbOzzRTtYroFsIpJjKu1bpbHZEiJCUFAQv//+Ozt37mTMmDHY2dkRGhpKu3bt6N27N//++6/usaRp5YQ1rxQWAP0KKbMu21Kdb1kxFiY3aUKQiwvHr13jI7spiFTl5MkQzpz5v5s/2P33gwj8/TdcvAiY2hWarsXOIZ3ISHPnpArFy8uLBQsWcOjQISZNmkT16tVZsWIFffr0oV27dixatIj09HRbh6lpWgGslhSUUmuBs9Y6/s2yF+H7Vq1wsbdn+bkrbKpjVCPt3z+e9PSbnO+nQQMICjJalJcvB0ztCo6Xqeqxg8xMWL26pF9B+dG4cWM++ugjEhISmD59OvXq1WPbtm2MHDmSFi1a8Pnnn5OSkmLrMDVNy4Ot2xQ6ich2EflTRNrkV0hExopIpIhEFqeHS1MnJ+a2aAHAW2c8OFP1bq5eTSQubsrNHyzXdNqBjQKxEztSGhsT5lWEdoXC3HbbbUyZMoXDhw8zd+5cmjdvTlxcHM888wxNmzblrbfe4ky2KUM0TbM9sWZdr4i4A8uVUl55bHMGMpVSySJyN/CJUsqzsGMGBASoyMjIYsU1OiaG70+cwLeqPR+k9MGBq/j5rcfFpbPlBzl6FNzcwNHRmE67Rg185/qyPdIJvg2neXPQ0wbllJGRwbJly5g5cyZZ72HVqlV54IEHGD58OHfeeScODrf+0t+aZgsiEqWUCiisnM2uFJRSF5VSyab7fwCVRKROaZz7M09PPJyciE7JYHGNLwDFvn2Pk5l5EwMMGjWCLl2MKqT/M9olOrl1AtdInKqnEhsL8fFWCb/csre3Z8iQIWzZsoVVq1bRr18/UlJSmDdvHr169cLV1ZVx48axatUqMjKKMR2JpmlFZrOkICINRERM9wNNsZRKXYKzgwM/tGqFHTDvkge7Kw8gJSWGw4dn3NyBcg1k69y4M9hnULOlsdRlReqFdDNEhO7du/Pnn3+ya9cuXnvtNTw9PTl16hRfffUVPXv2xNXVlaeeeoqwsDCdIDStFFmt+khEQoBgoA5wAngDqASglJorIs8A44F04AowSSm1sbDjlkT1UZZpcXG8efgwrpXgy7R7cZFU2rXbSvXqN9R25S0xEZo0gSpV4NQpYq8m4fmpJ9WjXyb5l5kMGwahoSUS6i1PKcWOHTv46aef+Omnn4iNjTVvq1+/PkOGDGHYsGF06dIFe3t7G0aqaeWTpdVHVm1TsIaSTArpmZl0i45m08WL9HVK5JXU0TjX6IC//wZELPzi6dQJwsPhp59QQ4ZQ78N6nE64DT7bT+3acPKkMQBas5xSiujoaBYvXkxoaGiOEdINGzY0J4jOnTtjp/+4mmaRMt+mUBY42NnxQ6tW1LC35+/UxqywH8GlS5s5evRzyw+SbTptETGqkGofoHbDZM6cgeho68R+KxMR/Pz8mDFjBrGxsURFRfHKK6/g4eFBUlISn376KUFBQTRu3Jjnn3+eDRs2kJmZaeuwNe2WUKGTAkCzKlX43NPo9PSxGstRXDl0aAqpqYctO8Dgwca///d/kJJiNDYL1PXaAeh2heISEfz9/Xnvvfc4ePAgERERvPTSSzRt2pRjx47xySef0LVrV5o0acLEiRPZtGmTThCaVgwVPikAPFi/PiPq1SMlU5jpMItrmans3z/OsqkZmjaFwEBISYG//jKSAnCliTEvkh6vUHJEhICAAN5//33i4uLYvHkzL774Ik2aNOHo0aN8/PHHdO7cGXd3d1544QU2b96sp9fQtJukkwLGl82Xnp40cXRkZ3p9vpcnOXv2L06eXGjZAbINZAtwDcBe7EmotQCAdevgyhXrxF2RiQiBgYF88MEHxMfHEx4ezqRJk3BzcyMxMZFZs2bRsWNHPDw8eOmll4iIiNAJQtMsUKEbmnNbe/48wdHRgGI2z9PO4SiBgTFUrly34B3j4qBZM6heHU6eJOD7IKKSomj+00Vi99Tg33+hVy+rhKzlkpmZyebNm/npp59YvHgxR48eNW9zd3dn2LBhDB06lHbt2mHqEa1pFYJuaC6CbjVr8p8mTVAI78mbnE9PJTZ2YuE7enhAu3aQnAx//22uQmrgvRvQ7Qqlyc7Ojk6dOjF79mwSEhJYv349zz33HK6ursTHx/P+++/Tvn17br/9diZPnszWrVv1FYSmZaOTQi7T3N0JqFGD46oms3mREyd/5MyZPwvfMdtAtk6NjaRwrakx0lm3K9iGnZ0dXbp04eOPPyYxMZF169bx7LPP0qBBA+Li4pg5cybt2rXD09OTKVOmsGHDBq5du2brsDXNpnT1UR4OpKTgFxnJ5cxMpjCd/o77ad9+Fw4ONfLfKTYWPD2hRg3i9m+h2VetqGXvRvLbCaSlCSdPQp1SmcRDK0xGRgYbNmzgp59+YsmSJZw4ccK8rWrVqgQFBdG9e3d69OiBn5+fno9JuyXowWvF9N+kJB7bt49qpDKPRwhoNAxPz08K3snPD6KjUb/+iuuBJzmefJyOKy8Tvq4qoaEwbJjVw9ZuUkZGBuvWrWPp0qWsWrWKPXv25Nju7OxMt27d6NGjB927d8fb21sPmNPKJd2mUEyPNGjA4Dp1uIwT03mVhKOfc+FCeME7mQayyZIl5naFRj4xgK5CKqvs7e0JDg7m008/Zffu3SQlJRESEsITTzxB8+bNuXjxIsuXL2fSpEn4+flRt25dBg8ezGeffcaePXt0e4R2y9FXCgU4m5aGd0QER69d42HmM75qFAEBW7Gzq5z3Dvv3wx13gIsLH/z2Ci+vnsKgGtNZ9sIUmjY1OinpDi/lS2JiIqtXr2b16tWsWrWKhISEHNvr169P9+7dzbfmzZvrXk1amaSrj0rIqnPn6LV9O3Zk8AkTuMd9OO7uBSwn7eMDO3awIeR9uu57Ga86Phx9PZpz54z1FZo3L7XQtRKmlCIuLo5Vq1aZk8Tx48dzlHFzczO3R3Tv3p2mTZvaKFpNy0knhRL08sGDfJCYSEOO8Q1PcWf7jVSr1jrvwm+/DVOnkvrIgzh7hJKemc69O67y68+V+PJLGDeuVEPXrEgpxb59+8xJIiwsjNOnT+co4+HhYU4Q3bt3x9XV1UbRahWdTgol6GpmJp22bmVbcjJ9+Jt3ndfg57cekTyaZPbuhVatoGZNOsz0ZEtSBM857OaT11pz//2wdGmphq6VoszMTHbt2mW+ilizZg0XLlzIUeaOO+4wX0kEBwdTt24hAyM1rYTopFDCYi5fpl1UJFcyFa/zFk96DqZRo6fzLuzlBbt38/zc+/jk+K881+ITPhk1gZo14fRp0MsBVAwZGRlER0ebryTWrl3L5cuXc5Rp27atOUl069aN2267zUbRarc6nRSsYO7Ro4w/cIBqJDIVvyQAABtySURBVLPAbgL9A8NwcmpyY8Fp0+DNN/npme4Mr7Oafs37sX/qnxw6BJs3G/PnaRVPWloakZGR5iSxYcMGUlNTzduzpgwPDAzE398ff39/vLy8cHR0tGHU2q3C5klBRP4L9AdOKqVuWMrMtBTnJ8DdQArwsFJqa2HHtWVSUEoxcNcufjtzBm+2s+C2f/H1/v3G3ia7d4OXF4lNXGjy6AVqOtVk6MGzzJsnTJ8OU6bYJHytjLl69Srh4eHm6qbw8HDS0tJylHFwcKBNmzbmJOHv74+Pjw/VqlWzUdRaeVUWkkI3IBn4Xz5J4W7gWYyk0AH4RCnVobDj2jIpAJy6dg3viM0cT8vgcebxTqv+1K8/MmchpaB1a9i7F7cZdTh67TSzGicw6bHGBAfD6tU2CV0r41JSUti8eTNbt2413/bt23fDWAgRoWXLlvj7++Pn52f+t2bNmjaKXCsPbJ4UTEG4A8vzSQpfAWFKqRDT431AsFIqqaBj2jopAPx99iz9duzAnnTm2r/G6A7LqVw51xwWU6fC228z9BUPllSJ4+Og75nY60EqVYKzZ0H/0NMskZyczPbt29m2bZs5UezevZv09PQbyjZr1izHFYWfnx/16tWzQdRaWVQeksJy4D2l1HrT45XAK0qpAr/xy0JSAHj+wAE+OXqURhzhl7qrCGjz35wFduwAHx9m96zGpKDLPOb3GNvf/obISPjzT+jXzzZxa+Vfamoqu3btynFFsWPHDq5evXpD2UaNGuVIFP7/396ZR9l11Af6+721F3VLallIslo7WqxBsloWtmQTA8YEBgdj5pjESSDAMXE4A5gkLPEQDj6TTAYICTMQkpCMh5iJOQibLbYZAoztgI8jb5Is27JkyVq7tbRlqdXrW+7ymz/ufWu/15u6+z2pf985dapu3br3Vb/T732vqm5Vbd7M4sWLbYLdDGSsUqjlSl+V/isrGkpE7gDuAFi6tMLAbg340sqV/L9z3exNtXP3mXn887mf0db2jkKBDRtgzRq2vXwAfg12dO3g5hvh2WeDpbRNCsZEaWhoYMuWLWzZUvh8O47D/v37S0Sxe/duTpw4wYkTJ3jooYfyZefPn1/S9bR582ZWrlxpojAA6z66IF4cGGDLzqfJaIS/iH2Dz269l1hsVqHAn/4pmS//d1o/HyErPj++uo9b3tXCxo2wZ0/t6m3MDHzf5+DBg3lB5GTR09MzrOzs2bNLJLFp0yZWr15NIlFlSRfjouNi6D66Cfg4hYHmr6vqqA9r1pMUAL7eeZxPHjpMC338ZOEOfm3dFwsnn3sOOjq47g/i/Psih3+59Wf81pZfJ52GT30Ktm2DrVth8eLa1d+YWagqx44dK2lR7Nq1q2T58BzRaJRVq1axbt26fLjiiitYu3atzae4CKm5FETku8BbgMuAbuBuIA6gqt8MH0n9BvBOgkdSPzzaeALUnxRUlXfsfoJf9Ll0sItHO65lzuytuZOwejWfXnWIv74WvnD9F9j7t/912KzmJUsCOeTC5s3Q0DD9f4sxczl16lSJJPbs2cPRo0errgK7YMGCYbJYt24dS5YssaXF65SaS2GqqDcpAHRns6x/8jHO+UnuTPyI/7H1K4WVVO+6ix88+GVu/S24ceWNPPybv+CRR+DJJ4Pw1FPQ11d6v3g82JohJ4lt22DZMlth1ZheUqkUBw8eZP/+/cNCKpWqeE1jYyNr164dJozVq1fT2Ng4zX+BUYxJYZp58MxJ3rP3ADEcfrzoRW5aG+7tvHMnJ9+6hcWfgpZECz1/0kM0UljnwveD5ZJ27CiIYu/eoJFRzIIFpa2JN77RHms1aoPv+3R2dg4Txb59+yp2Q0Ewt2L58uUVWxeXXXaZDXJPAyaFGvD7LzzGPWeFpRzn2c1vZH7rFcG3+8qVLL/lKMfmwPMffZ4NCzaMeJ++PnjmmVJRnD1bWiYSgY0bS0WxZo21Joza0tPTw8svvzxMGK+88gqe51W8pq2tbZgo1q1bx/Lly20r1EnEpFADUp7Hhn9/iEPeHG5NPM392z4drKT62c/y24e/wvYN8A+/8Q/ccdUd47qvKhw6FMghJ4o9e6D8MzZ3bqkkrrkGZs+exD/QMCZINpvl0KFDFVsX/f39Fa9JJBKsWrWKFStWsGLFCpYvX55Pr1ixwga7x4lJoUY803OS6/bsxSHOtxZ18uG1H4Cnn+brd17DJ/8jfHDj73Hve799wa8zNAQ7dxYksWMHlO33gkiwinexKNavt1VajfpBVTl16lTFcYvOzs4Rr509e/YwUeSOly9fzqxZs0a8fqZhUqghf7bvJ9zd3cxsetm9+UpWtCzjmS2LuPrmbtY0LeHlzxwf/SbjRBU6O0u7nHbtgmy2tFxLS7BK69atcNVVsHo1rFoFNgZo1BsDAwMcOnSII0eOcOTIEY4ePZpPHzlyZNgy5OXMnz+/qjSWLVs241afNSnUEM/3edOO+3jSWcq18aP8atvv4X/mj5nd+DVScXjtM68xr2nelNcjkwmmShSL4tix4eVEoL09EMSaNUGcCytXgs1fMuoNVeXs2bMlkiiWxtGjR8mW/yIqQkS4/PLLq0qjvb39khvPMCnUmCP9x+jYuYdeWvnzBf18vnc+1//jNh5fBg/f9iA3rX13Tep16lTwGOyOHfDii8G+0UeOQIX11YBgQHvZslJR5MSxfDlcYp8b4xLB931Onz5dVRqdnZ1VB74hmLi3dOnSkpZFe3s7S5Ysob29nfb2dlpaWqbxL7pwTAp1wLcOfIfbTy4mjsOOjeu4/yPr+csrB/jcyg/yFx+4t9bVy+M4cPRoIIjycOxY8NhsJWIxWLGiVBi5sHSpjV0Y9YvrunR1dVXtmjp58uSo92htbR0miuKwZMkSWltb6+ZxW5NCHaDqc+uOr/LD7BZeH+vhvz38f7it8ce81V3Co38++eMKU0EmA4cPVxbGSOOAiUQwVlFJGIsXBy0Qw6hX0uk0x48fL2lZdHV15eOurq6qE/iKmTVr1jBRlMtj7ty50yIOk0KdcKZ/P1ftfJZO2nm/f4L7Hn8/zY7Qc3eKePziHuhKpYJHZQ8cGC6MUyMsa9jYGAgj1w21cGEwAD5rVhCXp1tagmvq5AeXYaCq9PT0DBNFLnR2dtLZ2cnQ0NCo92pqaqra0sil582bd8HiMCnUEQ8f+Dq3nFyPR4yFT3+O06kdzMoKV0Yvp2PVdXRs+HU6Fm1m/fz1JGMXtyhyDAzAK68UJFEsjjNnxn+/SKQgipHkUe1cebq52VorxtSiqvT29paIopI8qs3TKCaZTNLe3s7WrVu57777JlQfk0Id4fsOd+64i7913k2z10/LYx/mdPzssHJxibH+df+BjkUddCwMwpULr6Q12VqDWk8dvb2lrYqzZ6G/vxAGBoani/a3nxREAjEUC2Pu3CC0tQVhpHRzs7VcjMmhr6+voiyKj8+fPw/A9ddfzy9/+csJvY5Joc7o6X2GG3Y/wXNs4i0tEe7oG6T/3x6ge9dDvJR8jd2L4MA80ApfNKvmrioRRceiDhbOWjj9f0QNcRwYHBxZHKOdK06PoVU/IvH46OLIpYuP58yxJ7aM8TMwMEBXVxeu6/KGNwzbiWBMmBTqkMf3f553nb6GAUofZWtTZVlPD4tfOUhjTydetpteujnRcp5D8S4cd/imKAtnLWTTwk0lolg5dyURsT6RseB5BckMDATrTZ0/H+yf3dMTxOXp4uMxjDFWpbV1ZJHMnRvII9dyyYXWVuvyMiaOSaEO8bwh7nvqJh7IXsVpFtLNArpZgMPIs8MaMinaBnuIyyBZOcu5vgNkBo9DuhvSpyF7DvBpSbTkRbFp4SY6FnWwfv56ElGbfTbZpNPV5TGSVM6fH74C7lgRCdayKhZFJXlUyrcWimFSqFMymdN0d/8z6fRR0umjDKWOcyrdwyltzUuiPAwy8houoh6R7Fm81IlQFN2QCeK4c471rZdx1YKN+S6ojQs20pK8uCbeXCr4fjCmMppIzp8P4uIwhvHIEZk1a3R55PLmzAnGTZqboampENvs9osXk8JFhKriOGfzosiFTOYY6fRRXh16lVPaUlUaPbSN8gJ+0JoIRUG6m/lRn6WNTcyORmiLxZgXTzA/0UBbchatyVZaki1BnGgZdhyPxqfnjTFKcN1AKMWiqCSP8rzz54NQbRLieIjFSiUx2bE9ejx11IUUROSdwNeAKHCPqn6p7PyHgK8AJ8Ksb6jqPSPd81KUwmioKq57LpDFuZdI7/4p6cM7SGeOkX6d0rsowcmmysI4pQs4y3x8GeP0Yi8Nbj84faVxUTrup2nCoyWitEaFObEocxMNtCaqi6TacSxifRrTge8HLY2xyqS3NxhzGRoqjasthzKZlEsiFgsG9ovDRPMmel0yGWyRWx6SyYtn5n7NpSAiUeAA8HagC3gG+G1VfamozIeALar68bHedyZKoSrnzsGPfoRu/y7uM4+Sfp2SXgDp9hjprctIr5tLek6agUwn3X5DXhQn/QUc9hZwxp/LgLQyJC2kIq1kIi34MsFWgO+C2wdOKJB8ug/cgeGSCeMGfJriSRLRxPhDpPq5eDQ+oXsmo0mSsWQ+nYgm6maZgnog9xRYuSxGi8dadrIfPZ4O4vHKwhhraGwce9mWlmDxyokwVilM5c+0q4FXVPVwWKHtwHuAl0a8yhg7bW1w++3I7bcT7+4m/oMf0LJ9O2x/HLYfCso0N6M3vxv3tneTftPrSXsnw+6pI2SzL+C6PbjuORynB8fpYcBL0UcL/WHoo7Uk3act9NLKeQ2OB2hhUFrIRhoh0RaEcZBWn7SXAi8F3lDQUskfFwV3CDK54/7KZXLBn9xvlngkTjKWHCaMMaVDseSur5YuvqYp3kRTvInmRHM+3RRvIhlN1lxQ8XhhzGEq8Lzgya6cJFKpoHXiOKWhPG+yj8vzstlAWOUhlSqUudAxn7GwZUuwK+NUMpUthVuBd6rqR8LjDwDXFLcKwpbCF4EzBK2KP1LVYSvqiMgdwB0AS5cuvepYpfWfjQJdXfDAA7B9Ozz9dCF/9mx473vhttvghhuCT3gZqj6u2xfKogfHOVcxXX48mO2j19e8RCrFA8wqlYy2MiBTMOCtShyXuLpE1SGmDlHNEPWziJ9GvAziFySi7iC+O4TnDeE5Q7juIK47SNYZwHOHwM8EwcuAnwV1Jr/OYyAikRJJNMdLpZEXSaypqljKr6lH8VxMqAZCqCSMkcLgkM/AkMvgkMtgymco5TGY8kmllFTKJxWWy6QhnRGyaSGTibBy3SBP/HTxhOpaD91H7wPeUSaFq1X1E0Vl5gEDqpoRkY8Cv6mqN4x0X+s+GieHD8P998P3vhdsrpBj3jx4+9uDZU6XLSuEpUuDDt0J4PsunteL4wStj0AcPSWtkfJ0xumh33UZ8D1SJEnROOYwRFM+naahKD21OwZFUJJCGISEQFyUBD4xfGJ4xPCJqhsEXMR3EM0iXhbxs+BnUC+DagZ1U/heEFx3iKwzQCbbTyrbRzrbRyrby1C2D8ervj/ApP1toXgaYg1EJUo0EiUqUSISyaejkfC4QvpCylY6lwsigiCjxmMtG5HImO6XK+v6Lhk3Q9pNk/HCuOi4Ul7uuNI515/Y4MzW9q3suH3HhK6th+6jLmBJ0XE7ULIeraoWr/Xwv4AvT2F9ZiYrV8JddwVh//5ADtu3B+nt2ytfM29eqSjKpTFvXsVHRCKRGJHIPOLx8W8gpKqoZvG8gVHCYFG6u+L5rDvIkOcw4HkMqIxZMmkayJIgQzIfio9zaZc4KYVUye8pIXieIgqMMC6TKzJOBGiMQCKUUC6OoYGA8ImqR0RdRB1EHfAd8DP4XiAgz0/ju4F4HHcoFNAgWaefjDNAxunHdQYZ8LMMqAOeA34qaBn5YWASHmEyEIRkLElDrIFkNIzDrsRcutK51W2rp75uU9hSiBF0Cb2N4OmiZ4DfUdW9RWUWqeqpMP1e4E9UdetI97WWwiSgGuyws2tXsGFCcTh+fPgenuU0NwdyqCaORYvq5pEMVS/oFqoqlkLw/SF8P4XnpfD9NL6fyocgLwhZL8OQ55H2PVK+kla/qkgqHY8knFzaIU6WRD54U/r7bexEUOISaC8uYVokjCFGkI6JhsJSYqJE8/IK4gg+MTQQGT5RPCLqE8Ejoh5R/GD+DR5R9YmoRwQNzuOHZX2i6iPkzgd5qj6KBj80Ron9MZT11ScWiY35i7vaudxxMpYkHolPezddzVsKquqKyMeBnxH8NvqWqu4VkT8DnlXVB4E7ReRmwAXOAR+aqvoYRYjAhg1BKMf3obu7IIhyaRw7FqwJsW9fECoRiwWPSFSTxpIlwaMU04BIlFishVhs6ibrqSq+n6kqkeHH6bK8c2XHGXw/jWomLJvB8bJkfI+075P2fTKqZHwlQ6SiRIpD+bmxHjvEhwWfKBmFDFReqKvwzodh+omJkohIXlgJkSCOCAmJEI9ESEiERCRCIhItSkfy5fPpSCR/fUyEeJgflaBryRfBEYFIBE+ErAjpCuVjKsRdIe5B3HWIiRvKNCwXls2nw/vXApu8Zoyf8+erC+PYsUAqo7FwYaG1MW/e8AfBi49HOlepbCw2Y2ZAqXr4fjYvj3KZ5PJKz5XmVTunmg3vXUi7vkPG98iqT8bXfByklawKDpGKQskFl1heOpXSLrFh5XN5uXTu2CNaln/pTKwMWmWB3KKh4DY3J/lZx4idKVWpeUvBuITJPZO4cWPl8+l0sC1bNWl0dcHp00EofjpqsohExi6QSucqlRlvXjI5LWISiRKNNhKNTu3g+njIiSonkiDOlOWVHheLJ1e+cDwYxk6Y55Slg3PBfRyyvoejHllVMr6H4yuO+mR8cNUnC2R9xVXIKjhERhRNLi937BEddlwtv/jcSPnDr43hI0GrLP+7XTndvx+YmBTGiknBmHwaGgp7b1bC84Kt2XKS6O0N9v0sfmZvosepVNAFNjR04etjXyiJxMSEEo8H1+am01ZKj3Z+rNdNRqtKNXjPPQ88D/E8or4Png+egBcLYj+RL5MPRddVzIPC9OKR4vK8aHTMf5eqj6o7gnhy+W4oH6fkuJBXOC7NGyzKd4uur35P33dwfBdHfRxVXPVwfJ9ZzeuY6l52k4Ix/USjwZhDeztcd93k3991xy+V4rzyeCJ52Wwh1DuxWGVZjPULfDIWVZoKKsmigkgkFkPicSLjlc+Y0k0TvzZRIT859TszmhSMS4/ch2iC8y0mBd8vTIMdr1ByU2iLp9NOJD2Wsq5bCBeCSCD74hCJXHgeFOpXPM24Ulyc9rxC3sW4dkY1rr4annpqSl/CpGAYU0HxuEY9k5uSW0kW4/kCr7eB/VyrZjSRVBLKWMpMZ7o4r218y8hMBJOCYcxkRILuokSiti2rySYSCUKFpVyMkbHN/QzDMIw8JgXDMAwjj0nBMAzDyGNSMAzDMPKYFAzDMIw8JgXDMAwjj0nBMAzDyGNSMAzDMPJcdEtni8gZ4GLfpPky4LVaV6KOsPejFHs/Cth7UcqFvB/LVHX+aIUuOilcCojIs2NZ13ymYO9HKfZ+FLD3opTpeD+s+8gwDMPIY1IwDMMw8pgUasM/1roCdYa9H6XY+1HA3otSpvz9sDEFwzAMI4+1FAzDMIw8JgXDMAwjj0lhGhGRJSLymIjsE5G9IvLJWtep1ohIVER2i8jDta5LrRGROSLyfRHZH/6PbKt1nWqJiPxR+Dl5UUS+KyJ1vo3d5CIi3xKRV0XkxaK8NhH5hYgcDOO5k/26JoXpxQU+papXAFuBj4nI+hrXqdZ8EthX60rUCV8D/lVV1wFXMoPfFxFZDNwJbFHVNwBR4Lba1mrauRd4Z1neXcAjqroaeCQ8nlRMCtOIqp5S1V1hup/gQ7+4trWqHSLSDtwE3FPrutQaEWkFrgf+N4CqZlX1fG1rVXNiQKOIxIAm4GSN6zOtqOqvgHNl2e8Bvh2mvw3cMtmva1KoESKyHOgAnqptTWrK/wQ+C/i1rkgdsBI4A/xT2J12j4hcQpsmjw9VPQH8FXAcOAX0qurPa1urumCBqp6C4Ecm8LrJfgGTQg0QkVnAD4A/VNW+WtenFojIbwCvqurOWtelTogBm4G/V9UOYJAp6Bq4WAj7yt8DrAAuB5pF5P21rdXMwKQwzYhInEAI31HVH9a6PjXkOuBmETkKbAduEJH7alulmtIFdKlqruX4fQJJzFRuBI6o6hlVdYAfAtfWuE71QLeILAII41cn+wVMCtOIiAhBn/E+Vf1qretTS1T1v6hqu6ouJxhAfFRVZ+wvQVU9DXSKyNow623ASzWsUq05DmwVkabwc/M2ZvDAexEPAh8M0x8E/mWyXyA22Tc0RuQ64APACyLyXJj3OVX9vzWsk1E/fAL4jogkgMPAh2tcn5qhqk+JyPeBXQRP7e1mhi15ISLfBd4CXCYiXcDdwJeA+0XkdgJxvm/SX9eWuTAMwzByWPeRYRiGkcekYBiGYeQxKRiGYRh5TAqGYRhGHpOCYRiGkcekYMx4ROSLIvIWEblFRKZlFrGIHBWRy6bjtQxjPJgUDAOuIViD6s3A4zWui2HUFJOCMWMRka+IyPPAG4EdwEeAvxeRL4jIKhH5VxHZKSKPi8i68Jp7ReSbYd6BcA0nRKRBRP5JRF4IF7R7a5gfFZG/CvOfF5FPFFXhEyKyKzyXu/+bReS5MOwWkZZpfVOMGY/NaDZmLKr6GRF5gGCW+R8D/6aq1wGIyCPAR1X1oIhcA/wdcEN46XKCVsUq4DEReT3wsfCeG8Iv+J+LyBqCWckrgA5VdUWkragKr6nqZhH5z8CnCaT0aeBjqvpEuHBieirfA8Mox6RgzHQ6gOeAdYRrDYVfxtcCDwTL7gCQLLrmflX1gYMicji89k3A3wCo6n4ROQasIVjY7Zuq6obnitfHzy2IuBP4T2H6CeCrIvId4Ieq2jWJf6thjIpJwZiRiMgmgp2t2oHXCDZxkXBNqjcD51V1U5XLy9eGUUAqFQzzq60lkwljj/CzqKpfEpGfAO8CnhSRG1V1/+h/kWFMDjamYMxIVPW58Ev/ALAeeBR4h6puUtVe4IiIvA8CU4jIlUWXv09EIiKyimBznJeBXwG/G5ZfAywN838OfDTcPYyy7qNhiMgqVX1BVb8MPEvQCjGMacOkYMxYRGQ+0BN2Ba1T1eKlqn8XuF1E9gB7CTZ8yfEy8EvgpwTjDmmCMYeoiLwAfA/4kKpmCLYaPQ48H97rd0ap1h+GG9XvAVLhaxjGtGGrpBrGOBCRe4GHVfX7ta6LYUwF1lIwDMMw8lhLwTAMw8hjLQXDMAwjj0nBMAzDyGNSMAzDMPKYFAzDMIw8JgXDMAwjz/8HoSZZ2tBoSl4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "l1,l2,l3,l4,l5,l6=plt.plot(range(1,11),[v[1] for v in optm_loss[0]],range(1,11),[v[1] for v in optm_loss[1]],\\\n",
    "              range(1,11),[v[1] for v in optm_loss[2]],range(1,11),[v[1] for v in optm_loss[3]],\\\n",
    "                     range(1,11),[v[1] for v in optm_loss[4]],range(1,11),[v[1] for v in train_2_loss[:10]])\n",
    "plt.setp(l1,linewidth=2,color='r')\n",
    "plt.setp(l2,linewidth=2,color='y')\n",
    "plt.setp(l3,linewidth=2,color='g')\n",
    "plt.setp(l4,linewidth=2,color='b')\n",
    "plt.setp(l5,linewidth=2,color='k')\n",
    "plt.setp(l6,linewidth=2,color='c')\n",
    "plt.legend([l1,l2,l3,l4,l5,l6],['ADAM','NAG','RMSProp','AdaDelta','AdaGrad','SGD(network 2)'])\n",
    "plt.xlabel('#epochs')\n",
    "plt.ylabel('Losses')\n",
    "plt.title('Cross-Entropy Losses Comparison')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py36]",
   "language": "python",
   "name": "conda-env-py36-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}