CNN+Bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import time\n",
    "from platform import python_version\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import transformers\n",
    "import math\n",
    "import datetime\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import bert ## Bert\n",
    "from bert import BertModelLayer\n",
    "from bert.loader import StockBertConfig, map_stock_config_to_params, load_stock_weights\n",
    "from bert.tokenization.bert_tokenization import FullTokenizer ##Bert's own tokenizer\n",
    "import seaborn as sns\n",
    "from pylab import rcParams\n",
    "from matplotlib.ticker import MaxNLocator\n",
    "from matplotlib import rc\n",
    "from sklearn.metrics import confusion_matrix, classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.getLogger(\"transformers.tokenization_utils\").setLevel(logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4000, 2) (978, 2)\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv('atis_intents.csv')[:4000]\n",
    "test = pd.read_csv('atis_intents.csv')[4000:]\n",
    "print(train.shape,test.shape) ##using roughly 10 % for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>intent</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>atis_flight</td>\n",
       "      <td>i want to fly from boston at 838 am and arriv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>atis_flight</td>\n",
       "      <td>what flights are available from pittsburgh to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>atis_flight_time</td>\n",
       "      <td>what is the arrival time in san francisco for...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>atis_airfare</td>\n",
       "      <td>cheapest airfare from tacoma to orlando</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>atis_airfare</td>\n",
       "      <td>round trip fares from pittsburgh to philadelp...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             intent                                               text\n",
       "0       atis_flight   i want to fly from boston at 838 am and arriv...\n",
       "1       atis_flight   what flights are available from pittsburgh to...\n",
       "2  atis_flight_time   what is the arrival time in san francisco for...\n",
       "3      atis_airfare            cheapest airfare from tacoma to orlando\n",
       "4      atis_airfare   round trip fares from pittsburgh to philadelp..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>intent</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4000</th>\n",
       "      <td>atis_flight</td>\n",
       "      <td>list all flights from seattle on continental ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4001</th>\n",
       "      <td>atis_flight</td>\n",
       "      <td>i'd like the earliest flight information from...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4002</th>\n",
       "      <td>atis_flight</td>\n",
       "      <td>what is the cheapest flight from pittsburgh t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4003</th>\n",
       "      <td>atis_flight</td>\n",
       "      <td>show me the flights from atlanta to washingto...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4004</th>\n",
       "      <td>atis_flight</td>\n",
       "      <td>flights from milwaukee to orlando one way</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           intent                                               text\n",
       "4000  atis_flight   list all flights from seattle on continental ...\n",
       "4001  atis_flight   i'd like the earliest flight information from...\n",
       "4002  atis_flight   what is the cheapest flight from pittsburgh t...\n",
       "4003  atis_flight   show me the flights from atlanta to washingto...\n",
       "4004  atis_flight          flights from milwaukee to orlando one way"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAF2CAYAAAB5xX8IAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOydd5xWxdX4v2eXKkgTRJqAigo2RFCDvYKIgoq9gCVYsEWNvSBqNFasKDbAbuwFC/YYKxqTqImRqPmprwVb4vvmfU3U+f1xznDnuTy7+9zdZRfkfD+f/exz5947M3fuzDlnzpQrIQQcx3Ecp1KqmjsDjuM4zpKFKw7HcRynEK44HMdxnEK44nAcx3EK4YrDcRzHKYQrDsdxHKcQrjicpQ4RWU1E3hCRb0XkyObOT3MhIleLyGnNnQ9nycMVh9MsiMgHIvK5iLRLwg4SkWeaIPnjgadDCMuGEC4rk7dnROSgxkhIRCaLyM2NEVdjE0I4JIRwViXXisgMETm7sdK29791Y8XnNC2uOJzmpBo4qhnS7Qu81QzpNhsi0qK58+D8hAgh+J//Nfkf8AFwIvAV0MnCDgKeSa4ZDrwK/MP+Dy8Q/46ocvgGeAYYaOFPAT8A/wf8N7Bq7r5zcuevsPDVgTmW33eA3Sy8FfAGcIQdVwO/A04HRgL/Bv5jcf3BrpkAvAd8C7wP7F3DM0wG7gLusGtfB9ZJzvcE7gbmWzxHlrn3ZuCfwEFl4p8BnG2/Nwc+Ao4FPgc+Afa3cxPtGf5tz/FghenfCcyyvL8FDLVzNwE/Av9r8R3f3PXR/4r9NXsG/G/p/DPFsTVwTyK8FigOoAvwNbAv0ALY046XqyDuVYH/AbYBWqKuqXlAKzv/TDlBmtxfch5oB3wI7G95WRf4Ahhk59e0vA0ETgFeAqrt3GTg5lxc/wRWs+MewBo15GOyCexx9hzHmYBuiXoLXkMVVCtgJVQZjcjdO9aubVsm/rzi+B6YYvGPAv4FdM5fa8eVpP9/Fk81cC7wUv79N3c99L/6/bmrymluTgeOEJFuufDtgXdDCDeFEL4PIdwG/AXYoYI4dwceDiHMCSH8B7gQaIv2YOrDaOCDEMKNlpffo5b2rgAhhDeBs4H7UOG+bwjhh1ri+xFYU0TahhA+CSHU5jZ7LYRwlz3HxUAbYENgGNAthDAlhPDvEMJ7wLXAHsm9L4YQ7gsh/BhC+N8KnvM/wJQQwn9CCLPR3sBqNVxbSfrPhxBmW1ncBKxTQR6cJQBXHE6zYkL3IdRtldIT+Hsu7O9ArwqiLbk3hPAj2mOo5N5y9AU2EJFv4h+wN7BCcs1Mu252COHdmiIKIfwPqtgOAT4RkYdFZPVa0v4w9xwfoc/XF+iZy9PJQPdy91bIlyGE75PjfwHta7i2kvQ/zcXVxsdafhr4S3QWB85A/fcXJWH/hQqnlBWBRyuI77+AteKBiAjQB/i4wvzkt4z+EHg2hLBNLfdchSrAESKycQjh+RriIoTwGPCYiLRFeyrXApvUEG+f+ENEqoDe6PN9D7wfQhhQ4DkaQrkyqSv9IvE5SxDe43CanRDCPHQAOF1TMRtYVUT2EpEWIrI7MAgVznVxJ7C9iGwlIi3RAd/vgBcqzNJnqM8+8pDlZV8RaWl/w0RkIICI7Aushw56HwnMFJH2SVz9TOgjIt1FZIxNQ/4OdQf9WEte1hORnc1SP9rueQl4BfhWRE4QkbYiUi0ia4rIsAqfsSj5Mmlo+vn4nCUIVxzO4sIUdOAYgBDCl+jYwrHAl+gA9+gQwhewYPHa1eUiCiG8A+wDXI4OYu8A7BBC+HeFebkUGCciX4vIZSGEb4FtUf/9f6EumF8DrUVkRWAqsF8I4b9DCLcCc4FLLK7f2P8vReR1tM0dY/F8BWwGHFpLXu5HXVtxosDONgbxg5XPYHTA/AvgOqBjhc9YlOuBQeaWuq8R0j8XONXiO26R5NhZZEgI3mN0nMUREZkMrBJC2Ke58+I4Kd7jcBzHcQrhisNxHMcphLuqHMdxnEJ4j8NxHMcphCsOx3EcpxA/+QWAXbt2Df369WvubDiO4yxRvPbaa1+EEPJbAQFLgeLo168fc+fObe5sOI7jLFGISH7LnwW4q8pxHMcphCsOx3EcpxBNojhEpI2IvCIifxCRt0TkTAvvLyIvi8g8EblDRFpZeGs7nmfn+yVxnWTh74jIiKbIv+M4jpPRVD2O74AtQwjroHvbjBSRDdH9fi4JIayC7sVzoF1/IPC1hV9i1yEig9D9gtZAv652lYhUN9EzOI7jODSR4gjKf9thS/sLwJbo5y1Bv2cw1n6PsWPs/Fa2NfYY4PYQwnchhPfRr7qt3wSP4DiO4xhNNsZh2y6/gX7PeA7wN+Cb5MMxH5F9aKcX9hEaO/8PYLk0vMw9aVoTRWSuiMydP3/+ongcx3GcpZYmUxwhhB9CCIPRD9GsD9T21bOGpjU9hDA0hDC0W7ey05Adx3GcetLks6pCCN8ATwM/Azoln5LsTfaFto+xL5/Z+Y7oNxkWhJe5x3Ecx2kCmmQBoIh0A/4TQvjGPpe5DTrg/TQwDrgdGI9+tAbgATt+0c4/FUIIIvIAcKuIXIx+d3kA+iWyOpk/7eZCee52qH8CwXEcpxxNtXK8B/o5zWq0l3NnCOEhEXkbuF1EzgZ+j35lDPt/k4jMQ7+StgdACOEtEbkTeBv95vIk+xKZ4ziO00Q0ieIIIfwRWLdM+HuUmRUVQvg/YNca4joHOKex8+g4juNUhq8cdxzHcQrhisNxHMcphCsOx3EcpxCuOBzHcZxCuOJwHMdxCuGKw3EcxymEKw7HcRynEK44HMdxnEK44nAcx3EK4YrDcRzHKYQrDsdxHKcQrjgcx3GcQrjicBzHcQrhisNxHMcphCsOx3EcpxCuOBzHcZxCuOJwHMdxCuGKw3EcxymEKw7HcRynEK44HMdxnEK44nAcx3EK4YrDcRzHKYQrDsdxHKcQrjgcx3GcQrjicBzHcQrhisNxHMcpRJMoDhHpIyJPi8jbIvKWiBxl4ZNF5GMRecP+RiX3nCQi80TkHREZkYSPtLB5InJiU+TfcRzHyWjRROl8DxwbQnhdRJYFXhOROXbukhDChenFIjII2ANYA+gJPCEiq9rpK4FtgI+AV0XkgRDC203yFI7jOE7TKI4QwifAJ/b7WxH5M9CrllvGALeHEL4D3heRecD6dm5eCOE9ABG53a51xeE4jtNENPkYh4j0A9YFXragw0XkjyJyg4h0trBewIfJbR9ZWE3h+TQmishcEZk7f/78Rn4Cx3GcpZsmVRwi0h64Gzg6hPBPYBqwMjAY7ZFc1BjphBCmhxCGhhCGduvWrTGidBzHcYymGuNARFqiSuOWEMI9ACGEz5Lz1wIP2eHHQJ/k9t4WRi3hjuM4ThPQVLOqBLge+HMI4eIkvEdy2U7Am/b7AWAPEWktIv2BAcArwKvAABHpLyKt0AH0B5riGRzHcRylqXocGwH7An8SkTcs7GRgTxEZDATgA+BggBDCWyJyJzro/T0wKYTwA4CIHA48BlQDN4QQ3mqiZ3Acx3FoullVzwNS5tTsWu45BzinTPjs2u5zHMdxFi2+ctxxHMcphCsOx3EcpxCuOBzHcZxCuOJwHMdxCuGKw3EcxymEKw7HcRynEK44HMdxnEK44nAcx3EK4YrDcRzHKYQrDsdxHKcQrjgcx3GcQrjicBzHcQrhisNxHMcphCsOx3EcpxCuOBzHcZxCuOJwHMdxCuGKw3EcxymEKw7HcRynEK44HMdxnEK44nAcx3EK4YrDcRzHKYQrDsdxHKcQrjgcx3GcQrjicBzHcQrhisNxHMcphCsOx3EcpxBNojhEpI+IPC0ib4vIWyJylIV3EZE5IvKu/e9s4SIil4nIPBH5o4gMSeIab9e/KyLjmyL/juM4TkZT9Ti+B44NIQwCNgQmicgg4ETgyRDCAOBJOwbYDhhgfxOBaaCKBjgD2ABYHzgjKhvHcRynaWgSxRFC+CSE8Lr9/hb4M9ALGAPMtMtmAmPt9xhgVlBeAjqJSA9gBDAnhPBVCOFrYA4wsimewXEcx1GafIxDRPoB6wIvA91DCJ/YqU+B7va7F/BhcttHFlZTeD6NiSIyV0Tmzp8/v1Hz7ziOs7TTpIpDRNoDdwNHhxD+mZ4LIQQgNEY6IYTpIYShIYSh3bp1a4woHcdxHKPJFIeItESVxi0hhHss+DNzQWH/P7fwj4E+ye29LaymcMdxHKeJaKpZVQJcD/w5hHBxcuoBIM6MGg/cn4TvZ7OrNgT+YS6tx4BtRaSzDYpva2GO4zhOE9GiidLZCNgX+JOIvGFhJwPnAXeKyIHA34Hd7NxsYBQwD/gXsD9ACOErETkLeNWumxJC+KppHsFxHMeBJlIcIYTnAanh9FZlrg/ApBriugG4ofFy5ziO4xTBV447juM4hXDF4TiO4xTCFYfjOI5TCFccjuM4TiFccTiO4ziFcMXhOI7jFMIVh+M4jlMIVxyO4zhOIVxxOI7jOIVwxeE4juMUwhWH4ziOUwhXHI7jOE4hXHE4juM4hXDF4TiO4xSikOIQkScrCXMcx3F+ulT0PQ4RaQMsA3S1L+/Fb2t0AHotorw5juM4iyGVfsjpYOBooCfwGpni+CdwxSLIl+M4jrOYUpHiCCFcClwqIkeEEC5fxHlyHMdxFmMKfTo2hHC5iAwH+qX3hhBmNXK+HMdxnMWUQopDRG4CVgbeAH6w4AC44nAcx1lKKKQ4gKHAoBBCWBSZcRzHcRZ/iq7jeBNYYVFkxHEcx1kyKNrj6Aq8LSKvAN/FwBDCjo2aK8dxHGexpajimLwoMuE4juMsORSdVfXsosqI4ziOs2RQdFbVt+gsKoBWQEvgf0IIHRo7Y47jOM7iSaHB8RDCsiGEDqYo2gK7AFfVdZ+I3CAin4vIm0nYZBH5WETesL9RybmTRGSeiLwjIiOS8JEWNk9ETiySd8dxHKdxqPfuuEG5DxhR58UwAxhZJvySEMJg+5sNICKDgD2ANeyeq0SkWkSqgSuB7YBBwJ52reM4jtOEFHVV7ZwcVqHrOv6vrvtCCM+JSL8KkxkD3B5C+A54X0TmAevbuXkhhPcsL7fbtW9XGK/jOI7TCBSdVbVD8vt74ANUeNeXw0VkP2AucGwI4Wt0t92Xkms+ItuB98Nc+AblIhWRicBEgBVXXLEB2XMcx3HyFJ1VtX8jpj0NOAsdbD8LuAg4oDEiDiFMB6YDDB061Fe5O47jNCJFP+TUW0TutYHuz0XkbhHpXZ+EQwifhRB+CCH8CFxL5o76GOiTXNrbwmoKdxzHcZqQooPjNwIPoN/l6Ak8aGGFEZEeyeFO6HYmWPx7iEhrEekPDABeAV4FBohIfxFphQ6gP1CftB3HcZz6U3SMo1sIIVUUM0Tk6LpuEpHbgM3RLwh+BJwBbC4ig1FX1Qfox6IIIbwlIneig97fA5NCCD9YPIcDjwHVwA0hhLcK5t9xHMdpIEUVx5cisg9wmx3vCXxZ100hhD3LBF9fy/XnAOeUCZ8NzK4sq47jOM6ioKir6gBgN+BT4BNgHDChkfPkOI7jLMYU7XFMAcbbtFlEpAtwIY00G8pxHMdZ/Cna41g7Kg2AEMJXwLqNmyXHcRxncaao4qgSkc7xwHocRXstjuM4zhJMUaF/EfCiiPzGjnelzCC24ziO89Ol6MrxWSIyF9jSgnYOIfheUY7jOEsRhd1MpihcWTiO4yyl1HtbdcdxHGfpxBWH4ziOUwhXHI7jOE4hXHE4juM4hXDF4TiO4xTCFYfjOI5TCFccjuM4TiFccTiO4ziFcMXhOI7jFMIVh+M4jlMIVxyO4zhOIVxxOI7jOIVwxeE4juMUwhWH4ziOUwhXHI7jOE4hXHE4juM4hXDF4TiO4xTCFYfjOI5TCFccjuM4TiGaRHGIyA0i8rmIvJmEdRGROSLyrv3vbOEiIpeJyDwR+aOIDEnuGW/Xvysi45si747jOE4pTdXjmAGMzIWdCDwZQhgAPGnHANsBA+xvIjANVNEAZwAbAOsDZ0Rl4ziO4zQdTaI4QgjPAV/lgscAM+33TGBsEj4rKC8BnUSkBzACmBNC+CqE8DUwh4WVkeM4jrOIac4xju4hhE/s96dAd/vdC/gwue4jC6spfCFEZKKIzBWRufPnz2/cXDuO4yzlLBaD4yGEAIRGjG96CGFoCGFot27dGitax3Ech+ZVHJ+ZCwr7/7mFfwz0Sa7rbWE1hTuO4zhNSHMqjgeAODNqPHB/Er6fza7aEPiHubQeA7YVkc42KL6thTmO4zhNSIumSEREbgM2B7qKyEfo7KjzgDtF5EDg78BudvlsYBQwD/gXsD9ACOErETkLeNWumxJCyA+4O47jOIuYJlEcIYQ9azi1VZlrAzCphnhuAG5oxKw5juM4BVksBscdx3GcJQdXHI7jOE4hXHE4juM4hXDF4TiO4xTCFYfjOI5TCFccjuM4TiFccTiO4ziFcMXhOI7jFMIVh+M4jlMIVxyO4zhOIVxxOI7jOIVwxeE4juMUwhWH4ziOUwhXHI7jOE4hXHE4juM4hXDF4TiO4xTCFYfjOI5TCFccjuM4TiFccTiO4ziFcMXhOI7jFMIVh+M4jlMIVxyO4zhOIVxxOI7jOIVwxeE4juMUwhWH4ziOUwhXHI7jOE4hml1xiMgHIvInEXlDROZaWBcRmSMi79r/zhYuInKZiMwTkT+KyJDmzb3jOM7SR7MrDmOLEMLgEMJQOz4ReDKEMAB40o4BtgMG2N9EYFqT59RxHGcpZ3FRHHnGADPt90xgbBI+KygvAZ1EpEdzZNBxHGdpZXFQHAF4XEReE5GJFtY9hPCJ/f4U6G6/ewEfJvd+ZGEliMhEEZkrInPnz5+/qPLtOI6zVNKiuTMAbBxC+FhElgfmiMhf0pMhhCAioUiEIYTpwHSAoUOHFrrXcRzHqZ1m73GEED62/58D9wLrA59FF5T9/9wu/xjok9ze28Icx3GcJqJZFYeItBORZeNvYFvgTeABYLxdNh64334/AOxns6s2BP6RuLQcx3GcJqC5XVXdgXtFJObl1hDCoyLyKnCniBwI/B3Yza6fDYwC5gH/AvZv+iw7juMs3TSr4gghvAesUyb8S2CrMuEBmNQEWXMcx3FqoNnHOBzHcZwlC1ccjuM4TiFccTiO4ziFcMXhOI7jFMIVh+M4jlMIVxyO4zhOIZp7HcdPnr9dPqbQ9SsfcX/dFzmO4zQj3uNwHMdxCuGKw3EcxymEKw7HcRynEK44HMdxnEK44nAcx3EK4YrDcRzHKYQrDsdxHKcQrjgcx3GcQvgCwJ8wD92wXaHrRx/wyCLKieM4PyW8x+E4juMUwhWH4ziOUwhXHI7jOE4hXHE4juM4hXDF4TiO4xTCZ1U5jc4lt44odP0v9nqsUdLd7v5DCt/zyJirGyVtx1ma8B6H4ziOUwhXHI7jOE4hXHE4juM4hfAxjsWYF6aPLnzP8IkPLYKcOI7jZCyRikNERgKXAtXAdSGE8xZlep9NKxZ990NPXEQ5cRZXtr/7mkLXP7zLwYsoJ46z6FniXFUiUg1cCWwHDAL2FJFBzZsrx3GcpYclscexPjAvhPAegIjcDowB3m7WXP3EmDlj20LXj5/weKOke8zdIwtdf/EujzZKugCj7i3Ws5y9U+P0LEffdUuh6x8at3ejpNtQdr37zULX/2aXNRdRTpymRkIIzZ2HQojIOGBkCOEgO94X2CCEcHhyzURgoh2uBrxTS5RdgS/qmZ0l8d7mTNufecm4tznT9mdefO7tG0LoVu7EktjjqJMQwnRgeiXXisjcEMLQ+qSzJN7bnGn7My8Z9zZn2v7MS8a9S9wYB/Ax0Cc57m1hjuM4ThOwJCqOV4EBItJfRFoBewAPNHOeHMdxlhqWOFdVCOF7ETkceAydjntDCOGtBkRZkUvrJ3Rvc6btz7xk3NucafszLwH3LnGD447jOE7zsiS6qhzHcZxmxBWH4ziOUwhXHI7jOE4hXHHUgohIc+ehqRGRliLSrgH3L3Vl1hw0Rjk3x7sSkU4ispn9rm7q9Jc0RKR7c+ehHD9pxWFCcMV63CciIiE3c6DShtbQBiEi7URkjeS4ZYF76/1ORWRZYCtgsB0PrPRZRKSqXJkVSLveZdbAe6saIkCbSfhW17ecU9I4CtTtBpUXMBR42tL/ociN+fdcJB8NbBcNqV9S37RFpAcwQURWFJFBIrJuwfsXmWL+SSsOYAXgPBHpKCJHpcK4NoIhIuuKyHkiMiKGV3j/DwAisqOIbCkiHe241oouIr1EZDq6NmULETlYRC4BKqowJrh/tN99RaSF/a71Pcd8hRC+RcvsNBF5H9gaqKhxhhB+tDLbQEROFJEhlebZ7v9BRFqZRVqVnqvkXjte1xpbRYIilpfleyUR6VxJnlPs3ioR2VZE2lZyT/65RGSPWMcqEYZWVq1F5GgRGSYi3ezeip45d3ySiPSqpG7nymvlSssrpmn3PwHMFpGpBfJcBQueu4O1qdaVtkcRqUraRYd8vuoiqV/9RaRNpfdauiGE8KOItK8kLbsvLpP4AugFPI/uBv6/lcaRy/fKIrJMkXvr4ienOKJFZJX0Q6An8DfU0vmylvsWalDoy3oJOF5ELqoj3erkd28ReQYYD2wKXCwiPWur6KLW/mTgNWAmunHjFKAqhPBKbWlHrEGvKiL3AZcBt4jIqrHR1JDugl6CiAwERqL7e90TQrg8hPB9JWlbuV8EnA98BFwgut6mzjzb/QcAbwCno7sf16mok3tXFZE5wFXAYyLStY5njoIoiEgXEbkeuAmYJiK71PWcZYJHAfuGEP63NkGYKLr8c62IGQflnrlM3dwErZfrAwcCN9u9NT5zpEz8qwDr1XZPrry6isgM4EbgGhEZU8e9ae8ols0k4AARWdGEaq1COBH6ewIvAGcCM6VCl6qlsYaI3AHcICKHikgXe56F0i5T3juKyJ+AE9A21aoSpZXk+0zgfhH5uYgML5eGhcVy/l7Uy1AFLA98DUwNIfxFCvQiRGSEiLwDXAA8ICI9K723Ln4yikO0O9czWkRA1PBzgH8Dp4QQPs037Foa83fAOLu3JzVsBpazhpYzy2IwMC2EsAuwKtAfqFXjm7XfDxgInAj8Dl0R/0wtz1yuEh0P3BRCGGP5OL6OdIOItBeRfYBrgOuAXwLficgWNaRbrt60AeaHEDYDfkS3gllI6ZhOz7+DjYFNUCV7L3CwiAyuIe3qJB4RkV2Bc4C7Qgg/A14Bym5zm7yrVFiNB94KIWyEtodjRaRLmXu7iUibRBj0kcx18AqwoYgsX054i/akBqALVrHynpJc0h747/T50jyXqZsDgVtCCHuFEA4BOonIcekzJvd3FJGVk+O+InKi/W4DtMYMqjL3puUVz+0P/D6EsCnQCjhGRDrlnzli7aK9aM95fxEZEEL4OzALuLbcPfk6ItrLOAY4EtgqhLAJ0BnY1Z4hf/+COmL/26OG1NOoUbIuauAs1O7T8rZ0OwC7AmOB04CdgHG1Cf40HyLyK7ReHQpsAxwmNfSWkro1EXgX2AL4OaooJ4jIsjW598rUmw6W5sEhhJ2BD4CJIrJSufuL8pNQHGZ5HAmcZMeXA3eLyIgQwjnAJcA0KHk5IlJibe8g6s7qbRVtd+BhYD9gjxDCuVKma57EtwvwLLAGarGfKiIvovtobRtCmCdJd9WSjxU8jmH8CBwMPB9CmIxaV5uIyPrlnjvpiu4pIkNEpCvwLbCcpT0HOKpcmjmOAI5GhdETmA8a+Flyb+yip93+USKyg13SFTjU0h0F7BBCuFpElsvlOXbde4nIoRY8EPg92jB/jZb3G+l9qYK2oA727r4BeqBCDOBYYKiIbF2mvGK+twIeEZFRqEJfRkSeQhXdriGEr5LyioJpJHCGhZ+MWvrnisjwEMLnwP3A2vk8i8gqQDtgN2CEiJyLuh82EJGzRF1rj6H1LX2nqdBeUUSOl2y8bhgq8CMnoEK0RfKMnUVkeVSB7yoio0XkPFR57S0iB6H17XVUeS7UY0ni2hZ4WPQDaq2BZUXkaeB/7F19k5RXXnjujLpaPgY6AL+xuA8HBovINma8RPdkdVJHupuQ/SfwOaoselvU09F6lirFfB2JZTQI+E8I4eoQwtOoYbSWiCy0wZ+l20ZELgZ+hbaB/4e2o4eBw0IIt5ZROKmbOPaE2qH76j2IGnBdgAtCCN/VUl77o0pqZAjhUTMoXwf+Cext1/ROrk8N12WtrlSj77YVmXvrXFQ2dcw/c70IISyxf6gbB9QPvwFwG3CfvaT9gUfQwV4B/ghsZ9cPSe5rA2yPVu5rLY4+qNX/QpLWILSy9ozpJueOAeYn8W6C9hhGJdfsBWwc003COwMr2O9V0Mp5gh2vjHYzJwF9UQHfO7l3DeBl4B5gkIW9BvwW6Jlct2OZshsN7It2hTsAt6PKt52d3w6YamXyBDAmuXcl4Cx037AngVPQRno18GBy3Xqou61DLu0TgbeAk+14F7SH9/Pcsw0G1s/dOwZVMrcAV1rYyajS6WvHk1Clu0xSR6pQy/4U4C5gMwu/HPgT8LMkjXFAS2AIcJGFtUcb43uoWwrgAItrb+CO5P1XoY12T1QAYPn9BrjXjnsCh6PKfVfgQmD5Mu9pO1SxzAZmoMJyMPAZ0Mqu+RlwYa5OHYYaMK2AucAnwCV2fgNU2TyCCqkpdp3kymtZK9e7geEWfhXqUlw/SW9XdPuiqiSs2v4PRXvdawAPAf8ATrRzBwGfWllvltzbAjUg/oy250NRw+RM4PTkuuvR3ma+fg2393+TlVVLYB4wNHm2q4AdWLgtb2Lnbkja4DuWn9YW1gvYJuY1ubcj2mu/H9jIwm5Dld4eyXXrYO3MjrsCbe33OWj93Rs1Jk5ADaMRqEx5EG2XHXP53g91ET8A/NrCrkZlWxs7vpukjTVI9jZGJE39l1bw2Ajt//HAh0n4iWiXtD3axXzKKv1MVGFMRV0z12CN1l7UFahl9pL9vh79UNShSdwrAl3td39UKGyXHJ9olXeEVaQXgFVzz3EC2iW9nkxArQm8D6xsx6OtIn+JunH6J/efDKWL6jgAACAASURBVEzKxTkBtVAGWKW/HbgTWM7Ot7PK/AxwMap41kEF8jRgndjwUR/6VVaZY0Poijboq+14Q7RHtzcqDN+zfF1rZbZ3Ln8rWgXulAt/A+319EQF96to7+GXZMpsJVQpbg50suc8AFXql6dpATcAPex3myT8PHvmdRPB9iIq/FZGx7VeB9aw8y3RhrsFKsT+mMv39lY+P6KCTciEZitUMe+AKoBH0d5nev/+9qwfY4ogl9d/AgPteG90PKONvavbUcE+F61LaZuoQhXHtui3ae4G1srFf769o9/GdlWmvC5A28Hayft+wd7RSlbuc5M8rooaWL9Iyn89K/P1UaX1BZmSfwit4xOTNPcG7rPfw1GBuSGqNK+OZWhxHZ6U9zKoS+oqtId4JuqaWh3tIb5i76cfKgsGJml2sv/rosbQ5bm6dDGwkdWDl+w4Le8uqAF5Iir470ddr3uh7a2bXXeGvbeeyfGbqJw5wN7ZdHuGY6xuHGfXjgOOzr3D3mg9mYW2neVQ5bwqsLPl/Sgrx+ew9t1gGdxcwr9emU0s9aTQDgN+QAXi8qhAHJ9U4quB41CrbbAV4KaolT7FKtRHVmGqrJL9BtjSKuL2qIV7qsXZHhUufwBuTdI6Hng8l79DgIuscqe9jG4W742odbUb8B9gczt/GXBzfGarDLER7gNsYb9vAY63322T+E9DleGL6NhOvsxuIrMsJwJv2O9pltcudt2DwJp2bnNgP/v96+SeKtQPOxXojiqsPdEGsUzyHgbY71XQhhIVdbxmHSurB9EezsZJnveych9mDaFFIpDeRQX0ofbMU4Cxyb3HoQ3mKFTBdEcb8o5kFvuuaFf+CbTnsJmF97D8vp6Uw7vAAbky7YL2Bk4nE2JREB2MCoplLZ2rgfVy96+FKsRxaT1H6/S3qF8fK9vzUKEsqHV8AdrjSIVYd/t/Flmv7FQ7Tg2PNqgAfA9YPanHz6G9z9VRYXQ7Ktxb2jW7Wz6eRHtKsRy3t3t3RI2JOah1PoKsPvdGBdtFmAVv4Z3JeiKTgHOSc3HCRDVat69L702ua4f2HmfYcX+79xQ7vhW1yOdjPRfUTXon2pZ2IhuPuCQpk25o/bkFVZqjkzQ3Av6K9gjnJuEXokbPQNSF/iSquG4jMwonkPUOjkeNz7xBdSw6MJ5/1qrkmW9CFVUXCzsFeM5+b4XWzdnATo0mixtTsC/KP7RLvWNyfAza/TzC/k+z8H1Qy7CjHf/CKsGqZFb3icBXZA30OlTQtUQF+YGon78t6ka5AxVO1VYJzrT7pgIfoq6ttmhDmVDLM/S3Sjodde10RIX1bNRC/51d1x5tXKk7YEvUYjs8uW4SamlGQSFklmErYFn7fag9R09Ueb5u4bHrPRdVqKOscaxi4TeiM72q0B7JB5a39mgD2t2uW83K4nRKBVg7VMn+FXWhjUOV4IWUCveVLVzQKcCnWfgWqLK4FTUQVkQtsA7Ju3sctTxXRRX+7faeuqDCaToq/GehDWgZ1EVyKdarSMqus6X3nMUVy2kacL79Hg18krynaDmOA+6036ehhsWKqLA/FxXELdG6dgCqwHZO0p8CbJocR+V4CvBqEj4SdWcucK0l+V8eVZ5PWlrDUYE7Cq2jt6I9kF72nBdY2GWo8pqKWroD0J7N9ahyORhtQwNz6XVKjqtRg2F3tJ6+gvrzQXtbN1j5P27v8kBUGLew+1ZBxxJWszguJevRd0cVQmvUWBgc85D+t9+j0PrWLnlfV5DNFnwLHX8D2MzKagcyj8Tu9l5nobIkdSltjLa59paXU638trI8vom5gqx8Z6DtRtAe8lpkbbXK3vkBqDJ5HHMHJs/xiOVpw3LPS6Y8trG0Nkru/wDzkADtG10eN7dCqDOD+llYUIu0msxCvQJraGiX+Rt7MdHXPsMKdBm0oSyLuhNWsN+PogNdoP7XBVauXbOe/d4PVRxPoJZvjO8h1OX1EHC9Xbsrar1VY5Zn0sh+ZhXhvPTZgAeS4/+gjUlI3AqoEDgeOCYR9OOsTC5F/ewboQroKrSxV9n581AleInd1wrtah+UxH8tKniryHoCHVHB/TSZBX4XmYW0J6bAkgaaCuJhlpdr7HgntAfUCxXcj6NjLMdafqKbrws6yeD/odbw8vYeH0YVxqWocFzbwp8AOtu9x6A9yRNQF9Ta9hxXoIprlpVva7TBH4nVpyTfPTD3TRLvymhPNvb0HreyToXQKagAOB+1AFPLfiwqhAfYe7oS9e3/FVWYF6BW8FY1tIF3yHq2PdGeRt4t1cae/Wyy3k47tEd5NWrY7IMaA1+jAvBEVBF/hQqqtVABNw2tK7Psurb2bJMo7dlube9jrB2fhLoxZ1Nq9HRA29ilZOMMa1g+5gH7WNiJaFurRuvaFNRin4K6xHbBxvjIeuBSprxuIBvP6YQaP4eg7ehNtCfwc7R9dEPr+VxL83arA2MtL8PIDM4osEeS9X6eJ+uNjkZ7p3G84lf219HexQ2oQXGR1YVD0LG9rZO8b2HXj0DXc20KjLBzg0jciLlnvsjKf0U73pnM2In5ri53b73kcnMrhhozplbLeqhgiD7Nw6wSiVX4ocn104An7Pf2aOP8Byo4H0cb1zXAb+yacahbJA5MT7Xzafe5GyqMvgAuTsL3I1MW66IKKSqxdXPPcRTa0FdC/cwzksa+nVXOtVEhOofMSuhgYfGlTwbOSgTRG6jl0wXtLd2KuVBQd8Bw1M1xbZKX68gsyjgv/Q6rzH2tXFugVtLrqMA8DVXCYs/wHjZWgwr4k8i54ez/qqiwuSk5dzeZO2J7VMjeBvRLrumNWqsvk1mN3VGr/UwyS+9eVJiPTspoKjqL5LqkjI8Gzk4E09P2HNuQjDdYfkaiDXwI6tocQmaonIIK3ZZow96VzE9fTeYKe5lMqMUeXU8rp4usHLvYtc+gdes+THHm6k50e+0C/G8UAuSUhv1fi9KeScsk/EIrhxZor7eXleOVwL/Q2XSxvI4FptjxqahF3hetq9Fl1g4Vro+hBs3LqFJaF21rcXyohz3bFpTWkVXQuv0M8HAS3hntyW6F9tZOQ9v/DKsDu6Jrsi5HhfBC7iqLZ3VUgcXJCl3tHa2AKujPsTpnYb+xvHdGDYbYzn6BKsfYroagAv0mVH50QdvdWDJ33U1k3o9OVs4T0LpzkpXJCeiUZlA3+WgLPwZVYKmbdltUEb9pz503dOL7XxdtX7uQG/BfJPJ5USdQOENaec9Arbm1USvnJjsXB3t6W4V9OrnvILTXMcIqSWfUUns/uaYFOsg8HG2Al6GWeJUVeM/kujFog9gSHcC8IKmIE1ELsh/qQ38WFURpg14ZtUYeQhtvFWp1TweGJcL1WHvWe8msqeNQofo4cIWFrYnODIszJB5EG/4OZMIt5vv3aMM7H1Omdr49mZW7NqqIT8KsWbvmSiujaBm2tXzsZcdREYs9YwcLXwcVEndYvG3QBjWVrCc3ELUwYzlGd0xrVDHsSql//vwkX+tZWW4dhVJyLiqOrVDlkM7seobMor0AtShTF9FCM8Qs/ES0JxbLdhm7ZvsaGm7Mw0xyM1fsXexiZbEimQvxFOC/gD5pXLl7Y7zr5o4Ho4bAkfaO2qC9sk3LxHEmanBFRbwXKmROtfd/X3Lt88Bu9vtiVLGnM+pORuvhTcCeFrYlOmtLUGt+NurqehcTwkmen0UVQU9UCf8O2CW5ZnfUqInP2T05tynaxh+qUIbsWSZ8N9R4GGVhmwN/S97T/WgbiL3NtVDl9S6qgFdApy9fgxoQ49Ee7Rr2/EPRNt6SrK3ui/auRiZ5eRmt78OtXjxlZd0/l+f1UOVyG5khsVAvy8IPw3o/+frZ6HJ6UUTaoAyppX2lvYDN0K79DLTb1oJSy/slVOBcYxV5KllPoD2qhb/CBrks/EjgtaTCX4IqoglJ5brS7ovCsyUqRKJPdjDqEvgA7X63T+Lva41iD+Ck3LP1RoXF2UlaUQBHd9ZD6ODzMvZ3q6W1jT17dN2tjgqOCUn8UehHxdQOFdRx6uAyaIPuQqkF2BEVLu3QXtb/xXTs/N6oYlveyuJ+slk0cfbV46jFvjFqNU6ya89DFWFsiFcBjyRx74j2nqbYu3nRwmNZxGnGg6z8X0WVlFj5VeeepQpthAcnjfb3qKvqPkqnKdc0Q2wvVBA/iRokv7Hy75N7n/nJGi2trG4hs7rPsngutzxfbmUV3Q+/pBZBaHEumCJr/0+xMh6JCpTpaB06AlVc1agyvgY1eo6yeDrYvc+QjWO1QIV5tKoPsLL6PapcYo98E9RYOcHe8T32bNFweJBsZuAq6Fjcacn9ra38J+aebwI27R1VgMujbfkqMuWxCWqkLYNa+p/WJBhRC/2WMu8mrSO7U9o7e9Hy9hG5MUq0h/YB6maORsSaqFEW6/gstLcQFfNgtJ2dTWYMPkgiD+y9pF6MnbDJNWgdPtnebxdUjv0aXWNUVlE2uZxu6gQrypRaLF+igrCTvegH0LGJYajA3BS1Bsah1kVrtBdyYGxw9v8kktkOFvYpainkhedkahaeI9DezubJyz0gqUwtUWt/LmpZXw1Mt3PpANvWqFW+fRLWPmlwP2LWvYWthFoS76ENfuWC+T4c7d4fiwqwu8lcKLF7LailOdGOz8dmZSTxPIEOsrdBxzeOQYXQwZanzvZsr6NK9jkLH44Kgp3IhH1nMkWypT33qqhC+gGbcoha/c+jjfZQVLCcBdxRR/0Zjbq7osW3DqUDzwPIGvkFlJ8h1gMVAL9GBXD0W49DjZXuNaTdwd7Hk6iSi66MuMboBJI1RnZPuTVG+XURQ8h6aFugSm8Uap2/ZeW7Atqjuhv4C+oaa2Nx7WfPdzyqqAclcY9FjbDYZgYDmyTnBbWofyRTeLtb/DHfPez8Srny2MDKuxc6hhDLsWVS5o+gyvR51FDcCm1rh1j5vWLPGHuuD2GzxZJ0qsi26HiDrJ22KPOOWqCTVM6w426oPOmeiy8qrjXQiR+/SM7vZHkcaO9jews/FHV3bUs2/tgNlVXvWbl1QnuHB+Xe8Vuoe+wKdAz2KjK36y9QI2zBtPp8PuOzNImMbopECmVIG8QFqACK3fMVUCUSp54ea43jN2SCcw5qaQ4qE+dHaMM5AlUAvSmdax8tubqE59X2ApdBLYHxZMIw7+IZao1hpeT+jdGGvDUqBDuiFugt2FQ5S3dOmWc4Fx0wjo0nCv2qCvL9DGqJpgOW56ACfowdj7D89rXj95Nzgg262fH+qG/8ZTKLqhMqHON6jyfIen97kM2QWQZ1CXxINh6yEWrlbooqmn+QCf1fUurK2BEdvI69qLIDfqhwOTcXNpJs0He2hdU2Q+zqJJ12qBCbgwnPGtKN9WEFVGFVssZoLKVrjFol122N9opeRy34OFtwJyvjuLXE66jwbmnlm9a71VH3zA5o+7oS2D85X40KqillnmfBRA9UUMb1FR0tP2cmdWZoUgbbor29C+3ZW9s775eLvzUqWI8kG3eIM9/uw6x0qwczUTdrT3Qsayhq1W+YxNcfrdd/p5Y1C2id+wQ1blIXUnvK9FbQOjw9livqQjodc3smZTTMnmcUahD8Djg2aZvvkM1ea5tLZwQqq+K6jRVRxbMbOsYT75tt16aLD9dHZeLYmp65Mf+aU0GU9dMl549AG3gULpuig3EDrUA3IBtcnAMcVSaOaKFtRuZ/T1deVyw87Xh5KrT27dzZaDd4d4v3QTKLq5tViomo1f0SOkVR0AG8OIMnCtA2SeUsmu9tUGumysrvUXS2x2YWHpXWtWRTjScAX+XKcxXUCuyP9vpOJpvBM5zM5TAAdS/ehllIUbBY3jugkxni+M1uZFM310a3/rgil/YmqMV2CSpkn6qtLlkdSdNeGVUmQ1DB9D9kLsPdKZ0hNgq1MnuQWZ3LoD2PQajFuhKlbq+0Xgila4zak60ximth4hqj6CIaTOl0zFVQq/gP2Mp91Eq9wN7heODG5P1+jvrMYx3ZD1Xwcf3Mvqh7qRXqTruITNBXoy7WzjW0xWhYtUFduMOTcrqKUoOkBzbOhLp0dkTHA3ujwv85Cx+IehHG5tLa397Rcva+4kBza9RQjC7IQ9B6fIm9j2p7P/8PXUz5SVK2NRkXQ9F2eqGVx3XkjILk/fdFFeW99h52sXt3QGXL6bEcUKX0CNrGd0QVyPqonPhLmgba+55INg53B1kvoxVqVDxq9/a295/usNAVrUcPYe7HJpHfTZVQ8qC1TgnLVdJHUX/ziqglOAVtQBUPLibxdkh+FxaelE6vTWde1WXt74sOMsaBxNiQB1hFXxNtZHcmzzER+LjMMzRE6M+xsuyMWku9UAvur3ZvX1SYvYoK6THYSmxUEP4am4llYYPRrng6rfc+tHF/hlnwsQHY/9FkM99WQRvUQEtrFtkCwP1Itt+w9H8BHJKU3f1kllz6bqLLIjb4s8gWci2LCs3XUQX2Ptm062dQ6zldwdyKbAyhpeVvHjqp4hXUBbcxtsaIrO5WusboaFTwdSPXLsjGhv5M1tMegNb5Q1GhfT9qZT6JbWmBKvWn7T1MQOtnH1R53YD65dugPd3TKK3LNbrhyIywY4AP7HdnVOmvQ7Z49lq0JxrXHuTH9c5CjYrfk6yCRgXxpWjvb3YSdg2Z+25HVInG3Q1SF/CG6MD5CfaudrM0Frh0yskgsskfX2D1qxYZ0gVVEPsm+XkEbcMXoz2Bfvaen7JrNrB3OMme51RsfMXy+ixaH19ClcCqqIHUPynjW0kWRCb5WRdtL9vWlu9F8dfkiqPCSlqVXPOcVcbLrUIUHVxskY+bCoWnXb927v5KrP04rz21Qgeg1soctNvZ19L5E6UWSJzq+pBdk9/XqhKhny4EjL7kVEAsiw7eTSBb1xFnE/0atZg2s7JqYXm/p0zZ7o9aa0NQRbKm/Y8rWJdBx5iOSO55kWxdyEnAbfZ7Y9Tii+XeDbXYo0/3DrJJES3QRjiXbKwkv3ZmOQubhm4aiT3rnWRCLW7oGKcad0AFwe1W/mPRnsqXZOtb1rL//VDFsTYqOFuQjXfdjzbozpRfY/SrpHzy7ooDUWMjCo6zUUEbe6onooor7s92aixvOz8EHSOosus+wsaE0J7JHFSR3Ee2jqkiN1ySxiPYnm2oInkatfS3RuvOA+gMyNiON0cVQHTrbG/vNvamh6JGxDaocfMSajy0QhXN5CTtM8ncRduTraMZhPY0Ypyt0Z5BXCFeZdcPSeLa3P6PRN130UWUuoCqSNy0uXI4xvK5O2rExcV//VC5dR/qgt0TlV1xp9z90Mkfv7U60Bmtb4fb/ReRzSQVtP2lhlQs12VoxLUZRf6aSklcTekc8Ep8xbFwLkAt0rIbGNo1ZQcXc/Gtl6sQtQnP4yhdhSk0zNpfBnWzHZaEdUOF9JQkjWswoVJLudSW71/m811DHJ3QcYIoDJ9EhdPeqKUaXRjHkG338QXqcpiKCtypqKD9Jdr9fjyJbzl0wLAKtYzvQd1acbbIzkk+/kIyhmHhO6HC6Fl71g1RhfQuWW9ze9SPfQmlyrWTvae3yJT5R6gSqEbr3t6o62AWKvzTWXG/Rhdi7pOETcOME1Qg7YIaAV+TWfqHoRbzTaii2S53f7rG6HyLJ7qV4r5Sz6FKawLaU+iKKqbpwJF27f7Yosoa2kvcH+tptK4ub2UcXZ/no8ZO6oZrSy1uuCSN/IB9HIv4lGRrG9S4mkw2ZtEJbVNXWdnHXuGGVg4TsU0a7R1tgwrc1mhbu4tS4yo/hfpku/ZGSqdwz7B30Y/MaDgStf6/RSeN3GrX/gxVNOvlnu8MVDmL5XMa2qOotnjmo+0/9iI7ovV/FbSXPINMQXfAZlLZs99H5urdCa23gyzdkp0jamvPzfHXVIqj4kpKMs3SjosOLs5i4cHFF1EhmBcw5YRn3xqeobC1n9y7MtpT+gVqbZyOdnO3RIXxA+gCnytJdr2k/Lz+OvNNHTN/7JoLrcxetrx1tYp9GmbhWRlfaGW8EWpZDUP9/39FLdcqkrnj6NTPd6xsppJttvgIOsPrMUp3Cl2LUmF0DKULtE5AjYeu6GyTB9GZXI+glnmsH8vadc/Y+z4hObcXtp4Htf5mWxnGhaUt7N4T7NrLgJnpe0B95ptauX5jZZauMZqO+v9HU8sao/x7JRMcw1Bl2wIVsB+TjWEcYenPQsc8osEU20p+MHcY2fhRW9S4eAsVfNHlVJEbrlxdRJXc7WS7FExABXk6VXwG6i5qbfm/0t7/dFQebG5xDEN7HPOS+AdYfmMPYFuymYDdUEUY3X8/Q91E+6BusY/R+nGNPUuqcEajbqOPydYlLdh7DO3NTEfb80GokbuHlc3lqLF7l73LoZavq8l6PVujBtQm9vzHoj3IP5G5zP5hee1jaaUD+38jMyRL1mMsbn+LSlF0pB6+4lxDTQcXl6X44OLhqDC+h5qnTqbC8zJs5k8tz1XW2kcthDfItoUo15j3IdvC4QqLZ2u0YQ0EVrPrigr9P5H1fCp2OaAW0/7AlknYSagfPK6a7mUNZiKZ4u+ATll9hIXXRHQDrrLjbVBLNM5WGmLv5EdUUUpSVp3IFFQ/1EqMQnZ9O7cvKijHoco2tehPtTx/jiq4NdCZanFGlKC7zMYZOn2Se4fbezwDFXSvoULha0oHfY9AezhXosJqM0trJipceqOCp841Rslzn2b1ZgCqMHqgvY7JqCvoXdQd1RkVjtFaXZNkbKnMu90YNSgmoS6QSfZMRdxw56O9nWpKx4x+buV8GJlrrr/l76gkD5Mtjj3sfbdD2+gUqwfx+Sej7WoG2Xqa7dC2/QzZKvydyabap5tsVpNNoe5qaRyN1tvo3muLGmyXoe3jebJ1Nttg60Ls/ntR5TLO7qu29/gJtsM12Y4LXVFvyJ/QsabXUMNjsKXRFq17tyTvbiQ2fonW2+tQBbSPvbOnKDUeF5teRkkda9TItJBiBStSSe/B/OBkftN5ZNuGxGmdNQ0uzmBhV9j7lJlpVYPwTPeKqfFFUd7an4n2OsquLygXHyoIHrHKVsiFl8v3llhvzv4qdTnkF021tgo8C3UtPYY2spZoL+NiVID1sEYyE+hl9/ZFXQDXogL7PlRQPE1m0bdM0roY26CP0jGJV7F58qil9lhy7gDL2+Aa8j0TbcCPoO6iatSFNhVVWNuihksUTJKUYy+0J9nfrvkM7TXF7cSHkDX6gVS2xmgjdCC17Boji+tcVKH0S8KGA5clde01tB2sjSq121FX4mvorKKS7UdyZTMSNVDStQeVuOEG2DPkv+myluX3WVSgd0eF7nh06vEIVFjfhCr26HIZjU5JPRsd1N+dbN+udci+i9EVFZqPWV0Ya3mLExvGoIZgd9QwfYFshfvq9t5PI6lrSV26DjVQN7P3fRtaz+P432NkWwF1QdtO3KjyKLI1VOOTeH9LZsT2oLTn0A+tix+gCnYAKo/iszxF5treH3VRzUANp4fIeSwWx7/GVBr5WSGVVNJx9lJfRy22aMHMtJd5B2rd1Tq4SKkrrB31EJ4WFrfXrtTaX9BLoYL1BfaMp6Nd7ROon5+5Ew3szeXLwRrLG2SrgIejQmBnK89LyL41MSC5dzjJxo2oNfYIyfoJCxvJwmNOa6I++LjuY2OrB33tvT5B1ptcBR0byA9a1pTvsWjP6Fh0AP05apmqaO/lcVQ4rYA28q3Jel9RQFS6xugmMuOmL2oYLVhjhCqZdE+rOOssujoORxXUsdh6CyvHc7APQdXyLPmebqVuuE2sbt2EfULAzm+O1usvyFaGP4pa5uegxtPvUCW8MWpArE9Wr3dFjb8H7LgHqkyj4JyEGhN9LI6V0Hp2NKooTiYT8LeQ1bU9sEkPdrwDangNSp77Iiv7rVHlfg3azg5B63TsdfRDjZ8WVh5RqQ9CDaZWqDH7q+SeXa1c2ufKu9rSe9/eYez1TAXuTtL7D9kgf3u0Dl5h93Ss7R0vDn+NG1nllXQiqv0/IdP0cQPDw9DFZeOT+/KDi1Ptujgg2JrirrB0wL499XTxUGB9ASpErifbhqFioU/p19nq43LIb463LdrI46ykBXssoX7rm9HG1BW1ulJltZKVYS90BtHMpFyjn3h3tIG+Tem0y56Wl+gGO4Kscd2IbcqINsz3KbPyNy3bMvm+1co09ojyH88qp9BXpVQIxS++9a4h7UrWGMUNI7emdLFdjXtaWfhotFcRZ9h0QQeWd0CFy9dWvpeg1v95uXjTehLdcJOp2w33gb2/OJbQAq1XD6OKf0t05tQ2lM7Qa4eOO3XLPUfMz0pWF94gU5BxbGATe6bpZIP3q6M98dvQHsplZLOf+lO619mzaE9mDNp+2qFKeVmrCy+SCeehaPsaT/Yxs8NYWPDnlXqcpbUGKtuOTJ5ttVrk4BpoW4i7ILRCx21G2/FUsu+vxCnBv2xMebwo/xpTadTpK7aK+TDwb1SYHUU2uLgrauENQRXP00nc6QaG9XGFnU82PTU2qh5kwrMHlVn75YTOMdSxvoDyQv/hSvON+ZmTNO9AxwpSYV6py6EvqpDuIBOurdAxhFlkH5m5DlUeG6PK7lQrrz9YJZ+DNrKxVnZximtXdIbIlZbGwCTdu9DGfgsqBLZBXQzRpTXUnmtTe+aSjf3KlH25fMct9X9Z17tLzi1nz3AP6ps+k9LFg/m9ohasMbLjjmRrjOqcHkn5Pa1OR3sYa6GK4mC0d7s+KhzPRuvmNuisrnXRmTl/JpuynPbIVrE4h1GhGy659xR0TGAPK4vd0TURcYxIyFYyv43N+MqXV/ruUIv/vKS8j7f31IJs+vYEe5470d6M2P9fkrWPR1EDrS0qa85C61U05B4jc3leSjbAHo3F+y397Ulc1Ln2WpNSPxDzSCTPtdA3QZIy2huVL7EeH8PCX5CM97fKp7c4/zWm4qjEV3wFI0BjUAAAIABJREFUKlDGUXwDw0spHVws6q9djWyhXl+00f4dFY51WftVZEKjG/YNZDuua31BS0p9+ZcUzHe6Y+81qJvkRVSYF3E5jEMb1VvkpvnZ+c5oI3wOnd48g6xBD7X4biJbo3IH2nDXJFngleRlwX5YZI32bLRBnYttGocKtytQS/NQbKfhXL5r2xsqn+8bUUF3C8l+SxXU386oW2Sz2tJM6sE41O8dLdq8b70+e1r1I9sC5VMyl+z6aO8srTPLoQL1TkrXcbRGXSq/Q5VQRW44u3cTVMBfgNbbgei01VvJZn+1tnfeG623sX52oYyLJSmvAehMvDgJZAjZZpzL2vnfoYZSb7QtjkSV4zSrG2ujgvg21PCK7sGzyb6bvi3aA1odNUZvTNK5EFUK+5R7J0l7zSv1MzAPB6VKcRi1Lynoh9bvdIpwXKcVxyUXy8HvOttLo0ZWQSWlYYOLP6ce/lpKeylz0Rk2v6Ruaz+uUI2Vfye0y/0JtpCMWtYXJPmKLryH7dobK8l3cn64leWn6MynulwOV2P775B1y+egU1FrGkyNUzsnkfUQW5ApgMMonQraG1Vsq6PjUHdiW1Unce5iz7wVqujS76f83upJXI37V3Tgffsk30UWpa1BNh20C2qx17kFAzWvMRpZwb0XoEZI4XVKSZmvkNSzThZ2KNoO0pljh6OryddEjaC/WrrpV/h2tjLcnazOV+yGQ3vcP5LN0uuCKqz4UaRh6NjTZbn7Trb6OYLyvfLYfq4F7krCO1nYOajxODU5txPZLtbbop6FP5LNtrsT7fFMsvM3J/m+HpUXK6BjJa+iBtd01Pi6DlXqPfJ5tfvzSv1OVPbEnl1n1J34AvZVUmruFY+18l6BGnonS+JfYyuOWispDRtcnIsKrjOowF9rv9OvlbUk66Xsm4RX6uKZgu5F9SDau6ptfcE+ZJbYcLSRv49aO5Xku3Uu3/G7zStR2cyf+sy0EnT84V8Wx47ovPLr0cZZjVpheyTv5mY7twyqHOJYRUfU6r2XbCbJIyTbQlv+L0+Oeyb5riqS7wbW2fpMUMhPBCkcR5p3dPzgKVQRHGRhP0d7wbGdrIQK8dhO8t9tWBa10PMr0St1w6UbGd6f1Iludt+taN09AXWlxbwv1Osp85xxwV8LsvG9UWg7m2rHW1kZpO6258mMueVy+Xwe7Q3F8joGbYc9UDn0BNr2rkZdYGuiSv0DtNddiTESN6pcHVujZeEHkdTdOt5tu7rSWRL/GjeyyitpnYOLFr5gcJGC0yaTNOs1q4TS9QW/QH25/6DY+oJqsm9ZDC6Y75JuLLX35p4EbkiuLTzoTtYgz7R8XYIquGFoY9sEFUx3o1bcuhZXOg1xWWwVONqIf0u2dcZOqDtpFDqD5n60B9SYM8TqVCI0zhqjBXHYO650T6tqdHA2/+XBx62sd0INpLFoW7oUFVL97fxuJDu+xndXwTN3Rt2Jm9VUVpSO33yVe6/xi4Xph7dOtt9noAtFJ1leTyUZNCYzYOK9w1Ahvh7qmjozufYh1OBYlWz9R+s0v2T1dCdU1sTPo3ZDe/JRkZyPKo0+lv6eqOJ7jcqU+vKUTpXuY8/WFzXiPkYN0QvQcZcJufyVW8vV5N/NWFR/ja04qkl8xflKSgWDi3XEX8RfKxRb3PVB0ijiy2+JdlVPt4pQ0fqC3DO3KprvGp69rt5cvRd3Je8kThZ4g9JPvu6GfdME7aE9i45xpLOFTrHneQm10la1/KRrZA5DreZXyaZjNmiSg/0umSFWQ/nVd41RSZpJHK+gPbLHKoxjgr3z3VHf+4GoK+9mtLe1L+qKmYaugWlv189E62uNfvmCbbTSjQzfT8JHkI3j7YMace+g40EDUAV3nt13n9WF1lbmsS1taO/oKbK1Kiej41PRuFgNbWuPoPJhozJ5TI2plazM4kaVe1j6Uyz9jmRKPX6IrCJjBJ3ddrS9ozNQZXMX2QaHh6LtYhPUIJ2bxJGOaa5NDR9gWpL/Fk2kSeVkYauwosHFGuItOm2ycC+FRlhf0NB81/As+d7cFEp7cw1Z3HU5OvbyWyuzcajySKc7P0L2saadknvboD2tGWhvYjbZIrwTrCxXTq5vSalfviH57ktuhlgNZVefNUYlaZIJwJjmnwvE8byluTbaS/0YdZfcbeXdH/WZr4T22uZiu6GirpdadzSo5bnrNfZi18eNDAXtcXyKjt3FPbP2JanTyX2/IreTK9o7j1ud34D2Nre3+G9BFW+6SDRd2R+nxdY0NncSqhRiz+ZYVDlFw2BnKjcM0na8P/Df6FT7uDh5P1T5rZ1c183i+FXu/i6oC/5JCrTzJeWv4RE0rHJeQG5vKGpfuV2RKyx3T6WLuxplfUFj5buGeEp6czR8cddAVIB9ho7PpBs3Pm4Vvxe6HmN2/t2QLcJ7nWwvnuGom2qsNaq7UfdMtGbj+oZ659uuiYP1C80Qq6HsGiPNw9HtNi4vGgdqSc9DF39+R2mPdxz2OV3Uip+Fur3akXPRFGybDd7I0MJ+hvaCXknCWqPW/XHJM7yAWuX9kut2tGeei1r/HdDe13TL089Ro2MopR8LWwE1SJ4mt9Yil7cu6Bhi+v2ZhhojN6Nt9RrLZ+zVt0Ll3eFWttuhvc7zKVUah6GTP5p8u/Om+mt4BI0wuFgwvRpdYTVcX5eLpy8F1hfYca3rCxoj3xXEVy83XBLe0vI0AlWucePG6+x4DXT22dVWPhuXyUN87zPIfMvpIrxlUNfM8MbMd1LvKlW4jVFWw+2+yXXEcRS2GSeZf34vdFHZm6g7Jn6j5Tiy/ZAeRy3k97EFb/WoEw0ev0l+r44K273Iep4PU7of1cZWZ5ZDFd6Wufx0R42u7dFeQZwwsg1qbOyFKscTrOzjrsyd0Z790VQwuIxOO24Mw2A62su4jqytX2XvtLsdb422h21QpZX/Dv2KaC+nTg/KkvzXrJXTjhuiROocHKQGa59sFWu91hc0qNArHNSs4/6K3HB2bW2D7uU2boxW8jnkptiWyUtchDeTzO14FapMFhqzqjTfdm2N+S5YXg1Os0B590EVQEdU8K2EWdLo9h2HogrsU+DSJK3OqEunVrdnLfWhvuM3g1FhWEXWK7wR3S58vMV1o4WPAP6QpLsC2mY2T9syC7sGV0ddOXFdyhDUNTeNbD1UFWrEbWxl/SQ6VrYjOui+RZnnrqJ0LHMyla2Qjz2d+F5aoyvC56BG7xRUSa5mcc+y+FZBjaqjKP00b8n6jqXhr6iwapRV2/a7zgHNuvJT4Nq8i6fo+oLLqaW7vKjyXUFcjTHoXtM27anboJJFeGejivktVIDOIvuCYF6QNDjfzVRWlbo9n0H3droFm35q4aeiPvNHsUV2NaRTsTFVpmyLuGn+YL/nWvh6qKH3DNrj3Nrawxdke7DNQmfH/c7KQdBew8XogPJCLly7ZpDFdT86PnQ6OkkifqN7I3R2VnQJX4T2UiZbnv5EJugXDLrb8VAqU+pxVtswS2tdtHdQbflriyr7t1FDKG5Nfxzqcv2KxGOwNP81ReWs14BmozzcwtZ1vdYXJL8XK6uCCgbdaaJt2u36NS1PVSYgxtU3381RVvWI42/oOFB0e9b4SWN0OujfyMYy7sI25GuEZ6vvlPP70N73VHRNxHyyxZ/7oL3sjuiAc9zGfBDqcouLFkegEykupO7PEqyACnlBewZ/Rg2NP2ILONFe/m5l7n2Yhb9z08Hy/iAqb+pafLw82rtui3ogPkQVRXR/boUqxjb2XK+SfXRsDUo/VbvEL+JrUJ1b1JUzF0ehAc0GNqbUIlkW7WUIla0vGJOLq0GupUX4jHk33Fwa+G0O6rl4MIlnIuq/Pq5Avus1WaCBZVU4zVwcL9hftJBr/aQx2XjH7ajPv18jPVe9x28sjy+gQvZaSj0Cx5DN7NoBHeOLa3Q6o0JY0GnFx6Nu6i6oAI8KdKHvfKO9glvRMYx70YHzOEuxLeoyvcbqWmt0uu4fUbdpdS5/L1nehAoNA0v7XbRn/Vru3K/J9s6bgCr343PpNsunWhe3v0VeOe134QHNRnvAeq4vWFL+KN1jqT5Cv7G3ad+I3OrluvKdhC1SK64x0kSFYyzvDaj8k8ZxUWvftG00wjPVd6zrRLSHcYO176PsfcdB4YPQsY5b0Z5H/DJdVyuDH1GLf0dUCcRv0/wuXpuklQrePdHFpf1QY3NX1JiLm1TG8ZCTUUPkV5QuKGyFLjacTOmXPvOGwTmYV8HOt0S/sfE62U7bx9pzx0+79kGVyuOoW21Ifd/LT/2vUSunXdsoA5oNfrDi6wuqKV1fsNh3RWnARAUauE27HacW5UhLt1M9nqPJe3RF08xfTzZ9+wQq+6TxTJKpro1Zvyi+MFbs2rFkA98vW12JO8t2QQfzL0bHBL7BJo/Y+VuA6+z3qqhCqbZ4J6MyI34boxodx/gr6nGYgE6zHYYaO1PJ1od0RN1KZ2Pf54l1jaxnW9PYUGpErYf2GOLeX8ughmN7dKHgaHsnH9g9Me6VWXjX3MXO29Dcf4uscjb7g9VjfUG+gi6uf9CwiQos4m3aiz5Lc5Rffa+l9JPG7Sj4SeNF9DyVjHVtgC6WG5K793FUsc0F7rOwVEhX2TO+D7yT3NcO9TgMS473Qnvt56GDzqfYuX1RL8UVFt+p6GLCZ+38WNTgeZD/396Zh9lVVXn7/SUhQkIIU+wWkCBTE8GPFhQDMYoooAgqiiCCArbQGgRBaDEtAiqgyCSDgIHGFiRE4JOpZTBACFO+QIMROiKgDDIrgzQyBEzW98dvH+rkWqmqW9Otm6z3efZTdc49wz7nnrvX2Wv0y866DX3s0ffF4jMbYTvTk6U/K9U++08sxK6jCw0DqZZa8r3uz4dz0Du/ZO+XpuML2qU1Psz00lGBPqZpL9sMms2qRff6E5Tsp2W5Kml8QPl7Zlm/pJLGjcWNBuTNlS7sN+V7PBGrm6q61l8tnx2JbQwXYuFwNIvPwquEmSOxq/ALLB7cdxgwp/y/F7YPHopT8Uwq9+Sv5Xe3UvndXYRVYCvjWfDeZf+PYg3Gys3eL/xSVDdcL49fHM8FXqitH4WF1h8olRSxwBzyFfeGWuuXh7OZL7lfOt2zYjlNxRe0U6PvEeNNpWkv/w8Jm9Ug3d/KHrEpnp1VJY1Pp8MGsC7dlDQe5D53msiw9OUbZfmz+IXvX8vyR/BM5GCs2nl/Wf8P5XczBwuDKqfcd4DbGs77AhawlfH7w9jD6tbyDG5T23YdbISv7ucVeHY8uuGYzagQ18cvRXtiu95PyjmquhcXUtRxZblKrHodjtOovCuHtJZhqLXePJyDatDspj/9Gl/QDo3+iYJuKk17/btemn9g+K168zKoVFkDqpLGwmqYesxPY0njH2AhsljFwEHqe131uAMddSLWxmqh27ArajWgVjOSjctn9VxR36fD5nA+dr/dpCw/jQ3ah2BbQj3eZ3n8YvkiFlI/xDMP4RnH2njm8xVsf3hD0FFTjfXweitBNRrPcM7EguggPAufiWcdq2PhVkV+Vznp3tHq562dW18e1EE1aNK3nFg9ii9oh0Y/OSrQZjarAb6nw7Agno9tQPvT4Zb5yXJP1sKeU7Nq+71R0rhF/f4nSpW6srwudl+9A6ukqiy1ZwFX1LbbvAiHSp15MX6pqmp9LF+er1l0lBD+Ufnsg2WAvpDaC1t5vt6HBcYR2M19QzyrnUdH3MfWpY/H9PKaF7NBYoH0j+Ua69/N+XTU+z4KG//vAg5r9fO2NLTePrCtMGj2JidWU/EF7dL6Y9BnCNqsWng/V8KBfO+i+ZLGP2TxksYD/jJFh0PHrnRkcV4dB9SdVZYnYjvLHtgD7EGsLTgbR0bvgWdSB2PPrweoldrFQvGM8v9WuIjZjmV5TG27j2FhUveAWh2rqu7As5kdu7iWXt0vrKL6d0riRTzL+iWwZVnejlIHpyzvRFFBZut7G0YviPJNDCSSxkrar7bqWWC6pGnA37DR7Qr8pjQDOE7Se2v7D6v6GREL8dvkpIg4YaD7PgiMwj/UyyPiKeA1nHTtiIjYPiLOqzbs4rt6Fpgv6ReSbsH38AcR8Rj4/g3sJQwdIuJ/8Rv2tXgwno+91T6LX1J+CbxF0iQ8QN2JPXa+WP7eVjvWooHsq6S1gEslbRIRFwHLSfp8RDyDfw8Ty6a3YwHxbuwBtj1+wXgUC8irsAfelLLdKRFxs6QRZf9XgHUlbYhtITPx7JaIeFHSWpJuxEbxycBpkt4qaQ3sHTUW294uAP7U+Dyp0N39krScpF0lbVKWh0k6Fc+SxgArS9oHp3G5HQsyIuJXeIzYUdLwiLgyIuaW0y4zz/aA0WrJ1djo/5xYvY4vGKqNAUrTXt3/Vl9fC+5nX0oa30A/pQ7ppo/1okg/AY4q/38cxyKsWNptwG7ls3/CM6IjG5+J8pu4iY5Kfsfjl7Az8IvJ+lildB+eYVWznNXKeXasnWd6uQ9rYu+sz+PZzgQ8a9u0l9c8AdvkqtijEaVvM+goebAz9uwbgw3904DPlM/eTBPlD7I18d20ugMND0pb5sRq0b3q7zTtQzKtSj/fsy7vD70oaTxI/a6/CI0FPoDtD+8v6y4Bjiv/7w7cWtt+B4odpPwmqsp8VXboS3HakPvKALxiw7nH1f7fBRvKJ2JBeg/2vDq+JlhWKn8/R/HG6sX1blEEwoX4ZVHYXrMT9vh6gg4BPhLPcKrI9qm4tkw9wHWpfq5b0VregU4emrbKiTUUWl8H/b4InHZodOM9VxtgelXSeID6PKwMlJXr6gg8u7irDKbfwnYYYRvfg3R4TM0GptaOtSGOqbgcv0hV261SBMEDOJfbNbVzD2voy9dwipLNyrrJ2I6xQ227z1LclXt5zStgN+CLsNA+pggoYe/I75bv6AxK5tqy3+k4vcvbWv2sLSut5R1oeHDaNidWC+/ZUj3o9/O96qredq9LGg9QX1cF9qot/wiraatiRytgB4lKyH2/LA8vgqUeKX0OxZMQB/ydU/Y/GgfEnYzzNH0V2KC239p01Mt4G/Yg+0ht+RtYNbZ9EUq31fdv9vnEAvBKPCuaiGdP07BX5EblHnwazzoexZUpf0xHaqEv9+a82ZpvQ81I9Ah2hTwPT1XXwh4u5wCnStpM0kkRcRo22BERC2qGttfLuidjgI2UQ4Uov5JkcSSdJemD5f/RkqZjFcaNEfF04/bV8xIRl2BvoIVl+fVyDA1Cn1WdJyKeAy6TdJSk0dizazQWDETEK/jt/NOS3oxnIH/GM5Q/AB+VtLmk1XFcxaqS5uAYlYMj4pWIOBzbQJ7HKuCREfGApBUlnYIH8VMl7RURD+H8bgeX8z8UEd/Hv9XtgJkRsVVEPFC/pmaez4h4EAuJR7Fb+Rzgj1g4/g6rxiZi9fVOWPg9hr/XBXjW0fR5k17QasnV2Mj4gmz90Bjkksb90N96nqV61tdbKDnEsAPITQ373QH8W215Yxyz8As6kgzeyd/bAOs1uvfGKqyRWC00lZLDCQuWR/GMZAXsXbV3T66jF/dgY6ymeri2bhs86/gQdmL4MU5rMry2zzXYVvOPvT13tubaUJtxQP+4mibLGP3kvr2wtjx8cHpuqnNLOgb4kaSPl4++C+wiaXxEfB14a+0zsFvtJbXlT+DgxU9GxG/LutPwbGW0pPUkzQD2kDSuzHAW1Y61O84iPU3Sf2E7yG+wF9creBA/ovH+1GZKC+klETE/InYFbi4ut2Cbzq+x6vplLBB/FRELJa2ES83+R0TsXMaLZDBoteRqbAyhnFjZhn5jiJU07sN1TMBvzsfiIMT5dBTlOpuOGcDeuIRpPXp6T0pOKBw3UbkQr1Db5lvldzWHxfOWjcOG72coGQfK+s9TAhtxidVFdOTpemcvr7E7J4XKM2s8NvZX308VzLhJq5+3bOW7anUHOu1Uxhdk60FrHIhoY/ft8sxvjmMhforTfp9d+rUhVklNLttWA+o22HnkKxQXXGwjPIyO3EzCaVSGYbthJURHlOW55Tj7YG/GymtqP2x0XgerhmZT6o738vrqgu6L2POxs/rklQrqO3SkXR9OTX2XrfVtKKqqiIjnI+LYiJhdRXpGeYKSpCKsrhgh6XhJh2GVyplYtVmxP7C5pMlYR38E1qFPjognACRNwWqs0yJit4h4fCD625WBPSKex8LiOByjsQnOvLsndpetaqYQEXdLWg4LjfdFxOnAmyTtgmfpa2B11yQc9f4l7Ik0NiKqKO5TsGDYPyJuwPmoVgYml+jx27Hb+43YvrFDRMxs8npHFieF9SIiJL1F0kw8G/xNRPytk92qazwCeELSysCiiHhtMBwUkh7SasnVVSNnGdm6aLSJ+zY9NBjjgfu3dMwKrsdupuPL8ko4sK5SzR1FRw6tT+BkgitiV94jcUT3F2rHH0uHd9Y4nH/qPbXPt8dOKFvX1o2v/d/TjAT12cXatfu8ORaMK5fltegk/TwZ7T3kW/UAJknbIWlN7Hn3HLYNbFP+roUD1KZgtdXXJE2IiHvLfpUhd1Af/jIj2Bz4YXTiEly2OQG7oo7GaqTvRMQzkg7FKTT+Gbg/Ir5S8jdNx0LyVUlXAn+MiP1LfqbK4D4S+FtELCp5yc6LiGmSfgBMjIj31c5/Fo4i/3FEvFzWVXakbl3cJb0Fpwi5uCyPxvbKn2F32ak4FcpsHNU+A5gWtRxpPTlP0lpScCRtTUn6dy4ODpuLjb/74ijmNYELonjilVifQXngywB8cURcXwbPs/Fb/gkRcW0X+43As4pHsMpqIk63MR87ioBtNA/iwXcSrgM+V9JGeHZ1YCzupfVmbOe5XNL2OBp734h4RNJDOP345WX7N0fEn/pw3cJu8/Px/T8ZC/IDcTT4W8p9+B/sEbUbtsm8BB3xNJJ2x7acqb3tSzKAtHrKky1bXxpDND08fYwjoeSMwp5hi6jlfSr7T8HC40Zq5V5r2zTlpdXJ/s1U4av3+01Y4D1PhxfWcJzN9siyPBLbcGZgV+J62ddN8Axl0D3bsjXxfLe6A9my9aUxRNy3sf1gv9rySJykbxrO8HoirlN+KjY8n0/J+lrvZznOaditthrof4AjsxvP+T0cWf3ehvXD6IWXVi+uuW7LGENHPfuJODfWlFp/3oVjMtbEqd6vAb7ScLzvl+9wo1Y/V9m6bqmqStoeSasAX8YuqbPLukFRS9XtJZKuxmVKV8TqpHuwkJgQ9mZ6R0TcI2kdPGOYHhHzKr2+pHHA/8UutMfTUXjpXOBPwK4RMUvS8mGbxvLA67GEoDtJYyjlbbEt5Cbgyog4pthM5kTErdV1NHO/6ttLOgh7r80GXo6IA0tw5Y9wdcSnynan4Rxae0kaEcWrqvpf0rrAQ4PxvSV9pNWSK1u2/mwMYnp4+imOpKwbj9/Sf4+N/Ffi9BtvLdvsBzy+hPN26rVFN15avbjeYVgArVJbtwuePY3GXlmLKKV08UzwHDzbOBDbNuo1yjOYt01bzjiSpYbBNH7XzjkCq4yewTmdJuJ4ib3K58OAx7G78D1YDXVfRBwtaQOshtoID7hTcYT38rjE8bXlGBtGxP0lBcj+2HMq+uKl1YvrPBLHukzAcTJP4iC9hVggfgfHnTyABcVWWH01Hc/AjoiIWeVYg/49Jf3LiO43SZL2oAVCYyuc++pqnIzzMJypdaakLSLi9rAK6licP2odSV8MZ3QehVU5l2G11dPYXnMDsCAiri1qsLOAZyWtCpwc9oQaLanupdWp0CgchtOHPBIO9Gtq4K659c7FM4pxuMb4TyPi1bLNBFzjY9uy/BJOe3KspD3CmX7fIIVG+5OCI0l6T1UGoIojaSwDMAVHfh+C7QwUoSHsdXUfflvfFqf2mIddWfeQdAX2npqNVWCjgKfKDGYRTpV+NHB/sQ28GiUSvkIdiRt/0rCux3ES0WE/mYwTIN4JXBgR36pttjLwiqTNsRfYzTi4kEpo1ONKkvZnSKYcSZJ2IJya5Gmsy58BbAochAf/+3C22WfLG/Z9dUN6uGbGXJxAcAyOtdgV/yan4Ky4P4+I/SPiBSycpuMZSE+z/S6qL5dzdyk0ymxm49rySEm7AR/FhutPAwskHS7pFEnrAX/A3lDnYvfeKRFxUsO9SqGxFJE2jiTpA5I2BM6NiPeW5ftwNbxTsdpqw4jYrWGfv1MVFc+w6TiX1n8XG0azXlo/C+exekNoSNoOV837ejgf1pKuY02cpmQujjt5HdteLgTujYgXJR2BbRrvBrbGsSVX1Y6xXhGITUWbJ+1HzjiSpG88C8yX9IuSzmMGcFw4hcYdwDskVbr/qnrfG0JD0lplQJ6Ds8HeUfv8blwLfEZEXFYG5Yvwmz1YTbUBLv+6JTbQU+wq4yWdD/wLrqXRldAYg3NY3YljPj6Ojd3Dip3mxbLpGKx2m1e2e7zsX81mKqExvMyqUmgspeSMI0n6SGMciZyJ96d45rEpQERsU7ZVg+BYBdgRB/g91Rcvrdoxp+DZzr4RcXsPr2Emji5/Bts+1wSuiohLy+cHYQ+uh7CKbDPgfyLitiZvV7IUkIIjSfpAJ7YE4brcL0fEWWVGcAIu+XqiFk8+2LjvZBz4dzVwLzUvLRwbcXvZ7gDgkOKl9aaIWFDWLxcRr8uJBp9e0ht/6eOwcFr6ap9rcS6pj0TEjZL2we61F0fEnKrfko7D6qtHI+LZ/ruTSTuRqqok6QV1Y7NcgvXLklYrs4n34Dd28Bv6ecDuklYpg+/wMhBXdoixcn2NB7GX1nm4yFKjl9Zmkk6KiNOAj5TzL1AhIl4v657sSmgUNdLCMttZrXy0P3YF3rIs34TtGZtJGo9dcMFZc+el0Fi2ScGRJE3Q6J0kaWdsd/gMcIqkiVjV9BlJY8JpNV7F8Q9HlH0X1mYdh2IvrDW68dL6bTnuvHKMexu8tLpUHdT6XaUJOQxUy8nHAAAJE0lEQVQb2o+RdGJE/B7PcParGblnAxvjHFMTyv4P9uH2JUsJqapKkl4i6WAc7b1FRDxcBuPVcdzFUTiNSFX//ALghuioOrgLLvl6OXB6NVtYgpfWZbg64WO97Gc9r9Q4YAucKmRf4JOlb9sWFdWpwKoRsWcRTKvisq1P9ubcydJJCo4k6YZq4C1v7SthL6ajsJC4E7ulXitpC2y0/g0uXPQprFK6JCKuLscajgP+dsEuss81nGs1HEw4Dsd2XA+cWqmGmg3gqx13PLaf/AXX7FgeZ6Mdj43tb4+ISZJWLMvb1g3rpd+LupvZJMsGKTiSpAc0GLXvwJltT5Z0CLBdRGxfPvsCjnE4KSLmNRyjEkAjI+K1Ls7Vp2y/aojSlrQlVpP9JiK+Uda9B/hmRHysLL+O82OdUsWH9OjGJMskaeNIkgYqe0BteRPgu5ImlVUHA58rb/FnAsOLwAAblX+OK9wtdrxq4O9KaJTPn4+IY4trr8oso8dveDUB99XiqfU08DIus1v1Z1WcJuT/SPocLgj1Wtn/nrKdenrOZNkiBUeS1Chv9pXhe42y+nkc/LaZpBUi4hasjjo8XJf7x8C35LoSv4+IXxajONB9mo9u+tKjQLq6sJO0nhyMuC1WTT2MVWevSXp3Od4fsHH8Qmzn2CcizqwfM9VSyZJIVVWSsLjtoMwkTsR2hsewTWN9HKj3y4j4laR34YF364i4SdI7I+LXzaiUBqjfr+N4jLdFxPdq260F7AWsEBGH19Y3pglJgZF0S844kmWahniMkWX1VOB32FbxR+AM4FacXuRjkjbC+ZpuxtltiYhfl7+DUXVwmKSdJI0q/R4h6eO4euDY0u+3lW1Hl349hvNQbSBph+pY8fdpQlJoJN2SgiNZpqm9rX8KuFTSB3GN8MvKODoV2wa2BU7HJVyvAN4OfDQiZrag2ytjl9mXy/IpOH/VlyLiXhwwuI6kdSPiJQA5c+4tuBTtrMYDRmavTZogVVXJMkWjOkbSWBz4NgGYGhG/k7PSnhsRF5dtvgqsHxEHlOU1avEYvXKP7ad+H4xdbEfhvFbvj4i55bN/w9HeP8SJDhfgmuWvDHRfk6WfnHEkywx1dYykMZI+Fa51sRKOyVhQNp0GHCBpB0lvxTXAr6uOExFP1LydBkNo1Ps9svThBeBDwB4R8Wecxv346rNixzgDpz/5SUTslEIj6S9yxpEsc0j6Jq5U93acRPAVnEb8nIi4rmwzBc9CJuIssUe2qLtvIOkYHBT4XxFxuaTtcZrzfcMlZR8CDo6Iyzoz0jfGdyRJb8kZR7LMIGl5Sf8BbABcgqvqfQG7pv4a2EauaEdEnIHzRG1bCY1WxTVImiDpGkDYrfZYSTtHxLW4fG0VQ/JtSq2OutCoqblSaCT9QtYcT5YlRuFSrZMj4iVJfwE+C+yEB9yzgPdKeqTEYSyKiL8Uz6tWehw9BXyz/D0WG+93kHQXVk9dIOm6iPjPsm4x0lMq6W9yxpEsVSxpVlBUN8/hqnq7l9V34Qjq3YCXcJbbB6rgvVqk94DnaOpqNhOu3nc/Tp44C9gE18rYMyLux5Hqi8q2d2fEdzLQpOBIlgrUSVnWOjXD8vXA5OKq+iqOrF6AVVI/j0GuaNddv2sMx1X3rgqnLHkJx2SMj4gTIuLWasOcYSQDTQqOZKkgOvIz7SLpe5L+oZNtXsNpzh8FzpM0H9fKGAasWPYfPni97lm/y3Z/Aa4CZkiai8u8HhoRjwxeb5PEpI0jaVsknYVLm15fIqTPxmlCToiIpzvbp6h9Di+JC18Dfg9cSnHFHQwDcm/6XTgM+DzwSETcUI41qClOkgTSHTdpY+Ta2k8VNdQKONjtFGwPWBt4tQrUq+1TpTYfjgPjvoRTpJ8wxPv9dzEjgxVHkiSNpKoqaRvk2tz71VY9C0yXNA34G/BXnA7kJFx69biSaqPa/4305GVmMR+YNNBCo5/6vai+XK4hhUbSEnLGkQx56uk2SjqQF7BN4hzgHpyldkJE/EmlCJGkdYApwM+Kp1E9i+yHcSrxrxfbQbv0ezvg06Xfzw9Uv5OkO3LGkQxpOsnaejcuyTojIi4LZ3e9iBL4BtwvaQOcCn1L4Bl4I/vteEnnA/sA3x5goTEQ/f4X4KgUGkmryRlHMuSRNAL4Hh5MH8VpQMZGxF7l82G4Tvau+E3+NOC+iDi6dowpOL3IvlGrpZ39TpLmScGRDGkkbYV1/1cD92LPon8FZgLbV4OppAOAQyJiHUlviogFZf1yEfF6MUg/PVh2gXbtd5L0hBQcyZBG0pq4HsZzON3GNuXvWsBkbA/YMyK+JmlCuB5Fy6vZtWu/k6QnpI0jGdJExOPA07iu9wxgU5x88Frgt1gVNK9se2994G3l4Nuu/U6SnpABgEk7MAoYFRGXA0h6DVfkOyJcEvUNhtig2679TpIuSVVVMuSRtBpW84zD9SiuB06NiGfL50MyEK5d+50k3ZGCI2kLJK0CfBm4NSJml3VDPt1Gu/Y7SboiBUfSdhR7gNrtbb1d+50kjaSNI2kram/rbfXG0679TpLOyBlHkiRJ0hTpjpskSZI0RQqOJEmSpClScCRJkiRNkYIjSfoBSd3WKpd0kKRRfTjH1iUHVpK0lBQcSdIPRERPBvSDcDR5b9kaSMGRtJwUHEnSD0j6a/m7taQbJV0i6XeSLpA5EFgDmCVpVtl2O0lzJN0l6WJJK5b1D0v6dll/j6SNSoGnLwEHS5onaXJrrjRJUnAkyUDwTjy7eDuwLi5PeyrwBPCBiPiApNVx0aYPRcRmwH8DX6sd45my/kzg0Ih4GDgLODki/jkibh68y0mSxckAwCTpf26vkhhKmgesA9zSsM1ELFhuLYlxRwJzap//ovy9E5e5TZIhQwqOJOl/FtT+X0jnvzMBMyNi926OsaT9k6RlpKoqSQaPF4Ex5f//B0yStD6ApNGSNmxi/yRpGSk4kmTwmAZcI2lWRPwZ2Bu4UNLdWE21UTf7XwnsnMbxpNVkrqokSZKkKXLGkSRJkjRFCo4kSZKkKVJwJEmSJE2RgiNJkiRpihQcSZIkSVOk4EiSJEmaIgVHkiRJ0hQpOJIkSZKm+P+8YmPC3UdWvwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "chart = sns.countplot(train.intent)\n",
    "plt.title(\"No. of texts per intent\")\n",
    "chart.set_xticklabels(chart.get_xticklabels(), rotation = 30, horizontalalignment='right');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-06-17 12:38:04--  https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip\n",
      "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.165.144\n",
      "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.165.144|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 407727028 (389M) [application/zip]\n",
      "Saving to: ‘uncased_L-12_H-768_A-12.zip’\n",
      "\n",
      "uncased_L-12_H-768_ 100%[===================>] 388.84M  10.8MB/s    in 41s     \n",
      "\n",
      "2020-06-17 12:38:45 (9.51 MB/s) - ‘uncased_L-12_H-768_A-12.zip’ saved [407727028/407727028]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_titles = [\"text\",\"intent\"]\n",
    "train=train.reindex(columns=columns_titles)\n",
    "test=test.reindex(columns=columns_titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>intent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4000</th>\n",
       "      <td>list all flights from seattle on continental ...</td>\n",
       "      <td>atis_flight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4001</th>\n",
       "      <td>i'd like the earliest flight information from...</td>\n",
       "      <td>atis_flight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4002</th>\n",
       "      <td>what is the cheapest flight from pittsburgh t...</td>\n",
       "      <td>atis_flight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4003</th>\n",
       "      <td>show me the flights from atlanta to washingto...</td>\n",
       "      <td>atis_flight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4004</th>\n",
       "      <td>flights from milwaukee to orlando one way</td>\n",
       "      <td>atis_flight</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text       intent\n",
       "4000   list all flights from seattle on continental ...  atis_flight\n",
       "4001   i'd like the earliest flight information from...  atis_flight\n",
       "4002   what is the cheapest flight from pittsburgh t...  atis_flight\n",
       "4003   show me the flights from atlanta to washingto...  atis_flight\n",
       "4004          flights from milwaukee to orlando one way  atis_flight"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  uncased_L-12_H-768_A-12.zip\n",
      "   creating: uncased_L-12_H-768_A-12/\n",
      "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.meta  \n",
      "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.data-00000-of-00001  \n",
      "  inflating: uncased_L-12_H-768_A-12/vocab.txt  \n",
      "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.index  \n",
      "  inflating: uncased_L-12_H-768_A-12/bert_config.json  \n"
     ]
    }
   ],
   "source": [
    "!unzip uncased_L-12_H-768_A-12.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(\"model\", exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mv uncased_L-12_H-768_A-12/ model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_model_name=\"uncased_L-12_H-768_A-12\"\n",
    "\n",
    "bert_checkpnt_dir = os.path.join(\"model/\", bert_model_name)\n",
    "bert_checkpnt_file = os.path.join(bert_checkpnt_dir, \"bert_model.ckpt\")\n",
    "bert_config_file = os.path.join(bert_checkpnt_dir, \"bert_config.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'model/uncased_L-12_H-768_A-12/bert_model.ckpt'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_checkpnt_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class IntentDetectionData:\n",
    "    DATA_COLUMN,  LABEL_COLUMN  = \"text\",\"intent\"\n",
    "\n",
    "    def __init__(self, train, test, tokenizer: FullTokenizer, classes, max_seq_len=192):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_seq_len = 0\n",
    "        self.classes = classes\n",
    "\n",
    "        ((self.train_x, self.train_y), (self.test_x, self.test_y)) = map(self._prepare, [train, test])\n",
    "\n",
    "        self.max_seq_len = min(self.max_seq_len, max_seq_len)\n",
    "        self.train_x, self.test_x = map(self._pad, [self.train_x, self.test_x])\n",
    "\n",
    "    def _prepare(self, df):\n",
    "        x, y = [], []\n",
    "    \n",
    "        for non, row in tqdm(df.iterrows()):\n",
    "            text, label =\\\n",
    "                row[IntentDetectionData.DATA_COLUMN], row[IntentDetectionData.LABEL_COLUMN]\n",
    "\n",
    "            tokens = self.tokenizer.tokenize(text)\n",
    "            tokens = [\"[CLS]\"] + tokens + [\"[SEP]\"] ## Tokens beigning and ending specified by separation of tokens.\n",
    "\n",
    "            token_ids = self.tokenizer.convert_tokens_to_ids(tokens) ## Convert Tokens to IDs\n",
    "\n",
    "            self.max_seq_len = max(self.max_seq_len, len(token_ids))\n",
    "\n",
    "            x.append(token_ids)\n",
    "            y.append(self.classes.index(label))\n",
    "\n",
    "        return np.array(x), np.array(y)\n",
    "\n",
    "    def _pad(self, ids):\n",
    "        x = []\n",
    "        for input_ids in ids:\n",
    "            input_ids = input_ids[:min(len(input_ids), self.max_seq_len - 2)] ## -2 as ignoring tokens provided by bert\n",
    "            input_ids = input_ids + [0] * (self.max_seq_len - len(input_ids)) ## padding by zeros\n",
    "            x.append(np.array(input_ids))\n",
    "        \n",
    "        return np.array(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = FullTokenizer(vocab_file=os.path.join(bert_checkpnt_dir, 'vocab.txt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['hello', 'don', \"'\", 't', 'know', 'if', 'this', 'is', 'bert', '!', 'token', '##izing', 'random', 'hello', '!']\n"
     ]
    }
   ],
   "source": [
    "tokens = tokenizer.tokenize(\"Hello don't know if this is BERT! tokenizing random hello!\")\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[7592,\n",
       " 2123,\n",
       " 1005,\n",
       " 1056,\n",
       " 2113,\n",
       " 2065,\n",
       " 2023,\n",
       " 2003,\n",
       " 14324,\n",
       " 999,\n",
       " 19204,\n",
       " 6026,\n",
       " 6721,\n",
       " 7592,\n",
       " 999]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_tokens_to_ids(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = train.intent.unique().tolist() ## Retreiving all the classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(max_seq_len, bert_checkpnt_file):\n",
    "\n",
    "    with tf.io.gfile.GFile(bert_config_file, \"r\") as reader:\n",
    "        bc = StockBertConfig.from_json_string(reader.read()) ## Reading bert config\n",
    "        bert_params = map_stock_config_to_params(bc) ## Mapping parameters \n",
    "        bert_params.adapter_size = None # Adapter size helps tune Bert model faster\n",
    "        bert = BertModelLayer.from_params(bert_params, name=\"bert\")\n",
    "\n",
    "    ## Specifying input\n",
    "    input_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32', name=\"input_ids\")\n",
    "    bert_output = bert(input_ids)\n",
    "    \n",
    "    print(\"bert shape\", bert_output.shape)\n",
    "\n",
    "    cls_out = keras.layers.Lambda(lambda seq: seq[:, 0, :])(bert_output) ## Preserve the first dimension; Done to get rid of higher dimension\n",
    "    cls_out = keras.layers.Dropout(0.5)(cls_out)\n",
    "    logits = keras.layers.Dense(units=768, activation=\"tanh\")(cls_out) ##Flattening\n",
    "    logits = keras.layers.Dropout(0.5)(logits)\n",
    "    logits = keras.layers.Dense(units=len(classes), activation=\"softmax\")(logits)\n",
    "\n",
    "    model = keras.Model(inputs=input_ids, outputs=logits)\n",
    "    model.build(input_shape=(None, max_seq_len))\n",
    "\n",
    "    load_stock_weights(bert, bert_checkpnt_file) ##Loading the weights from bert chckpoint file\n",
    "        \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4000it [00:02, 1697.74it/s]\n",
      "978it [00:00, 1939.00it/s]\n"
     ]
    }
   ],
   "source": [
    "data = IntentDetectionData(train, test, tokenizer, classes, max_seq_len=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert shape (None, 52, 768)\n",
      "Done loading 196 BERT weights from: model/uncased_L-12_H-768_A-12/bert_model.ckpt into <bert.model.BertModelLayer object at 0x7ff657de7510> (prefix:bert). Count of weights not found in the checkpoint was: [0]. Count of weights with mismatched shape: [0]\n",
      "Unused weights from checkpoint: \n",
      "\tbert/embeddings/token_type_embeddings\n",
      "\tbert/pooler/dense/bias\n",
      "\tbert/pooler/dense/kernel\n",
      "\tcls/predictions/output_bias\n",
      "\tcls/predictions/transform/LayerNorm/beta\n",
      "\tcls/predictions/transform/LayerNorm/gamma\n",
      "\tcls/predictions/transform/dense/bias\n",
      "\tcls/predictions/transform/dense/kernel\n",
      "\tcls/seq_relationship/output_bias\n",
      "\tcls/seq_relationship/output_weights\n"
     ]
    }
   ],
   "source": [
    "model = create_model(data.max_seq_len, bert_checkpnt_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_ids (InputLayer)       [(None, 52)]              0         \n",
      "_________________________________________________________________\n",
      "bert (BertModelLayer)        (None, 52, 768)           108890112 \n",
      "_________________________________________________________________\n",
      "lambda (Lambda)              (None, 768)               0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 768)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 768)               590592    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 768)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 22)                16918     \n",
      "=================================================================\n",
      "Total params: 109,497,622\n",
      "Trainable params: 109,497,622\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "113/113 [==============================] - 1588s 14s/step - loss: 2.4319 - acc: 0.7339 - val_loss: 2.4612 - val_acc: 0.7050\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer='Adamax',\n",
    "  loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "  metrics=[keras.metrics.SparseCategoricalAccuracy(name=\"acc\")]\n",
    ")##optimizer=keras.optimizers.Adam(1e-5)\n",
    "histo = model.fit(x=data.train_x, y=data.train_y,validation_split=0.1,verbose = 1,batch_size=32,shuffle=True,epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31/31 [==============================] - 148s 5s/step - loss: 2.4433 - acc: 0.7229\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7229039072990417"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non, test_acc = model.evaluate(data.test_x, data.test_y)\n",
    "test_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22,)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict(data.test_x).argmax(axis = -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TEXT:  Tell me the flights that I can get now\n",
      "INTENT:  atis_flight\n",
      " Next Line\n"
     ]
    }
   ],
   "source": [
    "tsting_sentence = [\n",
    "    \n",
    "    'Tell me the flights that I can get now'\n",
    "]\n",
    "tokens = map(tokenizer.tokenize, tsting_sentence)\n",
    "tokens = map(lambda tok:[\"[CLS]\"] + tok + [\"[SEP]\"] , tokens)\n",
    "token_ids = list(map(tokenizer.convert_tokens_to_ids, tokens))\n",
    "\n",
    "token_ids = map(lambda toids: toids + [0]*(data.max_seq_len - len(toids)), token_ids)\n",
    "token_ids = np.array(list(token_ids))\n",
    "\n",
    "\n",
    "pred = model.predict(token_ids).argmax(-1)\n",
    "\n",
    "for i, j in zip(tsting_sentence, pred):\n",
    "    print(\"TEXT: \", i)  \n",
    "    print(\"INTENT: \", classes[j])\n",
    "    print(\" Next Line\")\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}