FastAI/Twitter US Airline Sentiment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Twitter US Airline Sentiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table align=\"left\"><td>\n",
    "  <a target=\"_blank\"  href=\"https://colab.research.google.com/github/TannerGilbert/Tutorials/blob/master/FastAI/Twitter%20US%20Airline%20Sentiment.ipynb\">\n",
    "    <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab\n",
    "  </a>\n",
    "</td><td>\n",
    "  <a target=\"_blank\"  href=\"https://github.com/TannerGilbert/Tutorials/blob/master/FastAI/Twitter%20US%20Airline%20Sentiment.ipynb\">\n",
    "    <img width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
    "</td></table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('/media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset/Link.txt'),\n",
       " PosixPath('/media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset/models'),\n",
       " PosixPath('/media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset/Tweets.csv')]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = Path('/media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset')\n",
    "path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>airline_sentiment</th>\n",
       "      <th>airline_sentiment_confidence</th>\n",
       "      <th>negativereason</th>\n",
       "      <th>negativereason_confidence</th>\n",
       "      <th>airline</th>\n",
       "      <th>airline_sentiment_gold</th>\n",
       "      <th>name</th>\n",
       "      <th>negativereason_gold</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>tweet_coord</th>\n",
       "      <th>tweet_created</th>\n",
       "      <th>tweet_location</th>\n",
       "      <th>user_timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>570306133677760513</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cairdin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica What @dhepburn said.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:35:52 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eastern Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>570301130888122368</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.3486</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica plus you've added commercials t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:59 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>570301083672813571</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.6837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yvonnalynn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica I didn't today... Must mean I n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:48 -0800</td>\n",
       "      <td>Lets Play</td>\n",
       "      <td>Central Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>570301031407624196</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Bad Flight</td>\n",
       "      <td>0.7033</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica it's really aggressive to blast...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:36 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>570300817074462722</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Can't Tell</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica and it's a really big bad thing...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:14:45 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id airline_sentiment  airline_sentiment_confidence  \\\n",
       "0  570306133677760513           neutral                        1.0000   \n",
       "1  570301130888122368          positive                        0.3486   \n",
       "2  570301083672813571           neutral                        0.6837   \n",
       "3  570301031407624196          negative                        1.0000   \n",
       "4  570300817074462722          negative                        1.0000   \n",
       "\n",
       "  negativereason  negativereason_confidence         airline  \\\n",
       "0            NaN                        NaN  Virgin America   \n",
       "1            NaN                     0.0000  Virgin America   \n",
       "2            NaN                        NaN  Virgin America   \n",
       "3     Bad Flight                     0.7033  Virgin America   \n",
       "4     Can't Tell                     1.0000  Virgin America   \n",
       "\n",
       "  airline_sentiment_gold        name negativereason_gold  retweet_count  \\\n",
       "0                    NaN     cairdin                 NaN              0   \n",
       "1                    NaN    jnardino                 NaN              0   \n",
       "2                    NaN  yvonnalynn                 NaN              0   \n",
       "3                    NaN    jnardino                 NaN              0   \n",
       "4                    NaN    jnardino                 NaN              0   \n",
       "\n",
       "                                                text tweet_coord  \\\n",
       "0                @VirginAmerica What @dhepburn said.         NaN   \n",
       "1  @VirginAmerica plus you've added commercials t...         NaN   \n",
       "2  @VirginAmerica I didn't today... Must mean I n...         NaN   \n",
       "3  @VirginAmerica it's really aggressive to blast...         NaN   \n",
       "4  @VirginAmerica and it's a really big bad thing...         NaN   \n",
       "\n",
       "               tweet_created tweet_location               user_timezone  \n",
       "0  2015-02-24 11:35:52 -0800            NaN  Eastern Time (US & Canada)  \n",
       "1  2015-02-24 11:15:59 -0800            NaN  Pacific Time (US & Canada)  \n",
       "2  2015-02-24 11:15:48 -0800      Lets Play  Central Time (US & Canada)  \n",
       "3  2015-02-24 11:15:36 -0800            NaN  Pacific Time (US & Canada)  \n",
       "4  2015-02-24 11:14:45 -0800            NaN  Pacific Time (US & Canada)  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(path/'Tweets.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12001, 2640)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df, valid_df = df.loc[:12000,:], df.loc[12000:,:]\n",
    "len(train_df), len(valid_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = TextLMDataBunch.from_df(Path(path), train_df, valid_df, text_cols=10, bs=32)\n",
    "data_clas = TextClasDataBunch.from_df(Path(path), train_df, valid_df, text_cols=10, label_cols=1, bs=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>  <col width='5%'>  <col width='95%'>  <tr>\n",
       "    <th>idx</th>\n",
       "    <th>text</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>about it xxbos @virginamerica seriously would pay $ 30 a flight for seats that did n't have this playing . \\n it 's really the only bad thing about flying xxup va xxbos @virginamerica yes , nearly every time i fly xxup vx this “ xxunk xxunk ” wo n’t go away :) xxbos @virginamerica xxmaj really missed a xxunk opportunity for xxmaj xxunk xxmaj without xxmaj xxunk xxunk ,</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>you guys were starting those flights at the end of the month ! xxmaj guess i ’m still road xxunk on xxmaj feb 27 … xxbos @virginamerica thanks for that . xxmaj been needing a way to make those xxmaj austin trips from xxup dca , and now you 've come through ! xxbos @virginamerica announced xxmaj new xxmaj route to xxmaj austin w / fares from $ 39 /</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>xxmaj delhi call xxunk . xxmaj poor xxmaj united staff spread thin and stressed . xxmaj you name it . xxbos @united xxmaj yes . xxmaj does n't make that mistake any less absurd . xxmaj does n't change the fact that i 'm very xxunk missing my luggage tonight . xxbos @united i 'm constantly having xxunk with upgrades & & charges . xxmaj in order to prevent xxup</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>xxbos @united , and now while waiting for new pilot the door on plane xxup broke . why am i global services status on an airline that 's xxunk ? xxup xxunk xxbos @united xxmaj how do i get reimbursed for hotel and taxis for the xxmaj cancelled xxmaj flightation ? xxbos @united xxmaj awesome flight crew on xxup xxunk , re the plane , the xxmaj xxunk period called</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>is 2:30 hrs delayed so far b / c of xxmaj late xxmaj flight crew . xxmaj now we are literally waiting while they have dinner xxunk . to honest gate agent . xxbos @united many people on xxmaj xxunk are going to miss connections due to long wait time in xxup sfo tarmac . xxmaj seriously u ca n't find an empty gate ? xxbos @united xxmaj you 're</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_lm.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>  <col width='90%'>  <col width='10%'>  <tr>\n",
       "    <th>text</th>\n",
       "    <th>target</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>xxbos @united xxmaj hi have a question re future xxmaj flight xxmaj booking xxmaj problems . xxup xxunk - xxup jac 29 / 9 xxup jac - xxup lax 8 / 10 xxup lax - xxup xxunk 13 / 10 . i 'm * xxup xxunk xxmaj what is checked bag allowance for xxup jac - xxup lax ?</th>\n",
       "    <th>neutral</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>xxbos @virginamerica xxmaj many xxmaj thanks for the xxmaj follow . \\n  # xxunk # xxunk # xxup usa # xxmaj xxunk \\n xxmaj looking for xxmaj any xxmaj help 😍 \\n 💗 🇬 🇧 💗 🇺 🇸 💗 \\n 🎀 xxunk 🎀 http : / / t.co / xxunk</th>\n",
       "    <th>neutral</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>xxbos @united xxmaj united xxmaj club team is xxup a+ & & got me a seat xxmaj late xxmaj flightr . xxmaj still , not sure why a last min xxup ual xxmaj cancelled xxmaj flightlation costs me $ yet overbooked folks get $ ?</th>\n",
       "    <th>negative</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>xxbos @jetblue xxmaj flight 136 departs xxunk xxup am xxmaj finally ! ! ! xxmaj arrived xxup jfk with no xxunk at xxunk xxup pm . xxmaj that 's a long flight . xxup aka xxup bus http : / / t.co / xxunk</th>\n",
       "    <th>negative</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>xxbos @usairways xxmaj my hubby has a 5hr layover in xxup dfw xxunk the xxunk flt . xxmaj pls help get him home sooner if you can ! xxmaj any room on s / by 2 xxup bwi or xxup phl today ?</th>\n",
       "    <th>neutral</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas.show_batch()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data_lm, pretrained_model=URLs.WT103, drop_mult=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XecnGW5//HPNTvb+yab3gMJhJK2VEHgICWoIIoIeJQiYiwclXM84PF39By7YgFEwAiCBRvNo6gUC0WabCqQEEjvySbbkm2zs3P9/pjZZVg2ySbZZ8ru9/16zYuZ+7nnea4dZvfKXZ77NndHREQEIJTuAEREJHMoKYiISA8lBRER6aGkICIiPZQURESkh5KCiIj0UFIQEZEeSgoiItJDSUFERHqE0x3AgRo+fLhPmjQp3WGIiGSVhQsX7nT36v3Vy7qkMGnSJGpra9MdhohIVjGz9f2pp+4jERHpoaQgIiI9lBRERKSHkoKIiPRQUhARkR5KCiIi0kNJQUREeigpiIhkgZv+8hpPv14X+HWUFEREMlws5tzy19d5YU194NdSUhARyXAtkSgxh/LC3MCvpaQgIpLhmto6gSxPCmY23cyWJD2azewzveqUm9kfzGypmb1iZlcGFY+ISLbqTgplhcEvVxfYFdx9JTALwMxygM3AQ72qfRJY7u7vNrNqYKWZ3evukaDiEhHJNm8khSxuKfRyJrDa3Xuv0udAqZkZUALUA9EUxSQikhWa2+J/FlPRfZSqpbMvAX7VR/mtwO+BLUAp8AF3j6UoJhGRrNA8GMYUuplZHnA+cF8fh88BlgBjiHc13WpmZX2c4xozqzWz2rq64OfpiohkksHWfTQPWOTu2/s4diXwoMetAtYCR/Su5O4L3L3G3Wuqq/e7cZCIyKDS1NZJyKAkL/jOnVQkhUvpu+sIYAPx8QbMbCQwHViTgphERLJGc3snZYW5hEIW+LUCTTtmVgycBXwsqWw+gLvfAXwFuMfMXgIMuN7ddwYZk4hItmlq60zJeAIEnBTcvQUY1qvsjqTnW4Czg4xBRCTbNbV1UlaQmqSgO5pFRDJcKlsKSgoiIhmuWUlBRES6NbVFUzIdFZQUREQymrvT3NaZknWPQElBRCSjtXfGiHTF1H0kIiLxexQgNUtcgJKCiEhGS+VeCqCkICKS0XrWPdJ9CiIi0tSqloKIiCRoTEFERHpoTEFERHp0J4XSAt2nICIy5DW3RSnJDxPOSc2fayUFEZEMlsrF8EBJQUQkozW1daZs3SMIMCmY2XQzW5L0aDazz/RR7/TE8VfM7Mmg4hERyUbNbZ2UpWg8AQLcZMfdVwKzAMwsB9gMPJRcx8wqgNuAc919g5mNCCoeEZFs1NzeyYSqopRdL1XdR2cCq919fa/yy4AH3X0DgLvvSFE8IiJZYbCOKVwC/KqP8mlApZk9YWYLzezDKYpHRCQrpDopBN5RZWZ5wPnA5/dy/bnEWxKFwHNm9ry7v9brHNcA1wBMmDAh2IBFRDJEZ1eM1kjX4BhoTjIPWOTu2/s4tgl41N1b3H0n8BQws3cld1/g7jXuXlNdXR1wuCIimaE5xXczQ2qSwqX03XUE8H/AKWYWNrMi4ARgRQpiEhHJeKle4gIC7j4ys2LgLOBjSWXzAdz9DndfYWaPAMuAGHCnu78cZEwiItli0CUFd28BhvUqu6PX6xuBG4OMQ0QkG/XspZCi/ZlBdzSLiGSs5vYoMPjGFERE5CC80VJQUhARGfKaU7wVJygpiIhkrKa2TvLDIQpyc1J2TSUFEZEM1Zziu5lBSUFEJGOleokLUFIQEclYqd5LAZQUREQylloKIiLSo7ldSUFERBKaWpUUREQEiMWc3R3RlG7FCUoKIiIZaXd7FPfU3s0MSgoiIhmpuT31K6SCkoKISEZKx7LZoKQgIpKR0rEYHigpiIhkpEHXUjCz6Wa2JOnRbGaf2Uvd48wsamYXBRWPiEg2Scf+zBDgzmvuvhKYBWBmOcBm4KHe9RLHvgU8FlQsIiLZZtC1FHo5E1jt7uv7OHYt8ACwI0WxiIhkvKa2TnJCRlFe6pbNhtQlhUuAX/UuNLOxwIXA7SmKQ0QkK3Sve2RmKb1u4EnBzPKA84H7+jh8E3C9u8f2c45rzKzWzGrr6uqCCFNEJKM0tnZSUZTariMIcEwhyTxgkbtv7+NYDfDrRCYcDpxnZlF3/11yJXdfACwAqKmp8YDjFRFJu4bWCFVFeSm/biqSwqX00XUE4O6Tu5+b2T3Aw70TgojIUFTfEmFcZVHKrxto95GZFQNnAQ8mlc03s/lBXldEJNs1tnZSVTzIuo/cvQUY1qvsjr3UvSLIWEREsoW7U98aoTIN3Ue6o1lEJMO0dXYRicaoUFIQEZGG1viNa+noPlJSEBHJMA0tEQC1FEREJD4dFaCqWElBRGTIq0+0FCrTcPOakoKISIZpTIwpaPaRiIj0tBRSvUIqKCmIiGScxtYIZQVhwjmp/xOtpCAikmEaWjvTMsgMSgoiIhmnoTWSlumooKQgIpJxGlojaimIiEhcQ0t69lIAJQURkYyTrr0UQElBRCSjtHd20RrpolLdRyIi0n3j2qDrPjKz6Wa2JOnRbGaf6VXng2a2zMxeMrNnzWxmUPGIiGSDnnWP0tR9FNgmO+6+EpgFYGY5wGbgoV7V1gKnuXuDmc0jvg/zCUHFJCKS6dK5QiqkZo9mgDOB1e6+PrnQ3Z9Nevk8MC5F8YiIZKQ39lIY3GMKlwC/2k+djwB/TkEsIiIZq741fSukQgpaCmaWB5wPfH4fdc4gnhRO2cvxa4BrACZMmBBAlCIimaExzd1HqWgpzAMWufv2vg6a2bHAncAF7r6rrzruvsDda9y9prq6OsBQRUTSq6G1k5L8MHnh9EwOTcVVL2UvXUdmNgF4EPiQu7+WglhERDJafN2j9HQdQcDdR2ZWDJwFfCypbD6Au98BfBEYBtxmZgBRd68JMiYRkUyWznWPIOCk4O4txP/oJ5fdkfT8auDqIGMQEckmDS3pWyEVdEeziEhGaWjtpCqN3UdKCiIiGUQtBRERAaCzK8bujmhaxxSUFEREMkT3YnjpunENlBRERDJG92J4Gd99ZGZTzSw/8fx0M/s3M6sINjQRkaGlezG8bOg+egDoMrPDiK9kOh74ZWBRiYgMQW+0FDK/+yjm7lHgQuAH7v45YHRwYYmIDD3pXiEV+p8UOs3sUuBy4OFEWfpSmYjIIFTf0r1CauYnhSuBk4CvuftaM5sM/Dy4sEREhp7G1giFuTkU5OakLYZ+LXPh7suBfwMws0qg1N2/FWRgIiJDTUNrZ1qno0L/Zx89YWZlZlYFLAJ+bGbfCzY0EZGhJd13M0P/u4/K3b0ZeC/wM3c/AXhHcGGJiAw96V4hFfqfFMJmNhq4mDcGmkVEZAA1tHamdToq9D8pfBl4FFjt7i+a2RTg9eDCEhEZejKhpdDfgeb7gPuSXq8B3hdUUCIiQ01XzGlq68yOMQUzG2dmD5nZjsTjATMbt5/3TDezJUmPZjP7TK86Zma3mNkqM1tmZnMO5YcREclWTW2duJPWvRSg/91HdwO/B8YkHn9IlO2Vu69091nuPguYC7QCD/WqNg84PPG4Bri9/6GLiAwe3UtcVGbJQHO1u9/t7tHE4x6g+gCucybx8Yj1vcovID6byd39eaAiMaAtIjKk7NqT/ruZof97NO8ys38FfpV4fSmw6wCuc0nSe5ONBTYmvd6UKNuaXMnMriHekmDChAkHcNk3LN3YyK1/X8VRY8o4akw5M8aUMaa8ADM7qPOJiAykbc3tAIwuL0hrHP1NClcBPwC+DzjwLHBFf95oZnnA+cDnDyI+ANx9AfHVWampqfGDOUd9a4TVdXv4y4rteOIMo8oKOGvGSM45ahQnTKkiN+fNDaeGlggPv7SVPyzZQmtnlAlVRUyoKmZcZSH54RAhM3JCRmlBmMNGlDC+sohQSElGRA7c9qZ4UhiZDUkh0e1zfnJZYtD4pn68fR6wyN2393FsM/FluLuNS5QNuDOmj+CM6SNojURZsXU3y7c08Y9VO7lv4UZ+/vx6SgvCTBxWRFVxPsOK82hu6+TJ1+qIxpxpI0sYXV7Iq1t38/jy7XR29Z2XCnJDHDaihOkjyzhydClHji5jxuiytPcRikjm29rUTnFeDqX5/f23ejAO5erX0b+kcCl9dx1BfPD6U2b2a+AEoMndt+6l7oAoygszd2IlcydW8qGTJtEW6eIfq3by95U72NrYRn1LhDV1ewiZcdUpk3nPrLEcObq0p5upK+bs3NNBJBrDHbrcqW+JsGrHbl7bvofXtu/mqdfreGDRJgDM4ITJVZw/cyzzjh6lBCEifdre3M7IDOjSNveD6o3BzDa6+/j91CkGNgBT3L0pUTYfwN3vsPhPfytwLvHZSVe6e+2+zllTU+O1tfuskhF27ulgxdZmXlzXwMPLtrCmroVwyDj18OHMO2Y0Z88YmfL5yLGYs6G+laqSPMoK3jrtrbMrRmukC3enK+Y4kB8OUZCb85auNREZWO+97RkK83K49+oTAzm/mS1095r91juEpLDB3Q9u1PcQZEtSSObuvLKlmT8s3cLDy7ayubGNcMg4ccowSgvCtES6aOmIEnNnRGk+o8oKGFleQCQaY2tjO1ua2qjb3UFRXg4VRXlUFOZSUZTHsJI8qkvyqSrOIxqL0dwWpamtk7bOLorycigtCFOSn8vWpjZeWFvPi+vqezYGH1Gaz2EjShheks/WpjY2NbSxrbmdvX0dwiGjMC+HsoJcSgvClBXk0uVOW6SL9s4uYu6MqShk4rD4uMthI0qYOa6cEWXp7R8VyRYnf+OvnDh1GN+7eFYg5+9vUthn95GZ7SY+sPyWQ0DhQcY25JgZR48t5+ix5dww7whe2tzEn17axhMrd7C9uZ3i/DDF+TkYxpq6Fp5dtYvdHVEg/sd7dEUh4yqLaO/sYsfudl7bvpv6lgitka5+xzBxWBFnzxjJnAmVNLZ18vr2Payq28PGjQ2MKS/k5KnDGVtZSFlBuGcA3Qwi0RhtkS7aOrtojXTR3N7J7vYou9s7yQ2FqCzK7Vn7fVNDG4+9sp1diY1CID6Yf8y4ciZWFTG6opDR5QWMLCtgWHEelUV5lBaE6XKnoSXCzj0RGlsjRLpiRLucaMxp64xS39JJQ0uEhtYIIbOe1kthXg5VifMMK8kjFnN27olQt7udhtZOivNzqCzKo6o4j6K8MB3RLjqiMdo7u9jdHk+gzW2dtHd2cfjIUmaNr2D6qFK1iiTlumLOjt0daZ95BPtJCu5emqpAhgoz49hxFRw7roIb5h2x13otHVHCOUZ+eO+bbbRFuti5p4NdLRFyc4yyglzKi3Ipys2hNfGHb3d7JxWFeYxK4Zdtd3snK7ftZummJpZtauTlzU08/Xod7Z2xt9TNCRldsf23VkMGFUV5uDvtnTHao117bdUciLxwiLycEHsSSTgvHOLoMWXUTKpizoT42FN1af6hX0hkH3bt6SAac0ZlQMs6vcPcslfF/ZiBUJiXw/iqIsZXFb3lWFlOKDFukPoGXWlBLjWTqqiZVNVT5u40tnaytamd7c3tNLRGqG+J0NjaSW5OiKqSPIYX51FRlEd+bohwyAiHQvHWQKJFkTzd193piMaob4n0PMygujSf4SX5VBbl0RqJ0tDSSX1rhNaOKPm5OYlWRojSglzKC+OtHHdnU0MbSzY2smxTI4s3NHLPM+tY8NQaAEryw1SX5lNdkk91aT4jywoYVR7/79TqEo4YVUpYrQs5BN33KIwqT38HjJKCpISZUVmcR2VxHjPGlA3I+QpycxhTUciYir5/kUoLciktyGXCsLcmzd7n6k6u7545BoCOaBcvb25i8YZGNjfGx3TqdscnD/x95Y43dd0V5eUwa3wFcyZUcvjIEiYNK2bS8GLKC7WNufTP1sQ9CmopiGSo/HAOcydWMXdi1VuOuTu7O6Jsa2pnxdZmFq1vYOGGBm5/cvWbusJGlxdwzlGjOO+Y0dRMrNSNjbJX23taCkoKIlnHLD5+U1aQy7SRpVwwaywA7Z1dbKhvZe3OFtbtbGHh+gZ++c8N3PPsOkaU5lMzqZKJw4qZnGhJTBtZkvZlkiUzbG1qJzfHGJYB9zEpKYgMkILcHKaNLGXayDfmZ+zpiPK3V3fw6MvbWLG1+S13xI8ozWf6qFKOGFXKjMS6XFOGF2uMYojZ3tTOiNKCjGhNKimIBKgkP8z5M8dwfmKsItoVY0tjO2t2xu9+X7ltDyu3N/PT59YTicZnZxXkhjh56nDOO2Y0Z80YqbGJIWBrU3tGdB2BkoJISoVzQkwYVsSEYUWcPn1ET3m0K8bquhZe2dLEsk1NPL58O397dQe5OcZp06r55BmHMXtCZRojlyBtb27nyAGYgDEQlBREMkA4J8T0UaVMH1XKe+eM40vvnsGSjY386aWtPLBoMxfe9izvOHIE1501fUBmb0nmcHe2NrVzxhEj9l85BZQURDKQmTF7QiWzJ1Ty6XdM455n1vKjp9Zw3i1Pc9q0as49ehRnHjmCEaWZ0eUgB6+5PUpbZ1dG3M0MSgoiGa8kP8yn/uVwPnTiJO76xxp+t2QLn3/wJcxgzoRKPnrqZM45alTaV9eUg7Otex+FDLhHAZQURLJGeVEu1509nc+eNY2V23fz2Cvb+d3izcz/xSJmjq/g+nOnc/LU4ekOUw5Qpuy41k1JQSTLmBlHjCrjiFFlfOL0qTy4eDM3Pf4al/34BU6aMoyLjxvH2TNG9WupFEm/bU1tgFoKIjIAwjkhLq4Zz/kzx/CL59dzz7Pr+OxvllKY+zLnHDWSq0+dwtFjy9MdpuzDtqYOIHOSQqB3yJhZhZndb2avmtkKMzup1/FyM/uDmS01s1fM7Mog4xEZrApyc7j61Ck8/Z9ncP/8k3jvnLH8fWUdF/zwGb772MqeeyAk82xrbmN4SR554cy4YTHoKG4GHnH3I4CZwIpexz8JLHf3mcDpwHfNLP33eYtkKTOjZlIVX7vwGJ763Bm8Z9ZYfvC3VZx/6z94eXNTusOTPmTSjWsQYFIws3Lg7cBdAO4ecffGXtUcKE1sy1kC1APRoGISGUrKi3L57sUzuevyGupbIlzww2f46sPL2d3eme7QJMm2pvaMWB21W5AthclAHXC3mS02szsTezYnuxU4EtgCvAR82t3VzhUZQGceOZLHPvt2Lq4Zx13PrOWM7zzJg4s2cbBb8crA2tY8RFoKxAex5wC3u/tsoAW4oVedc4AlwBhgFnCrmb3ldk0zu8bMas2stq6uLsCQRQaniqI8vvHeY/ndJ97G2MpCrvvtUj6w4Hk2NbSmO7Qhrb2zi8bWziHTUtgEbHL3FxKv7yeeJJJdCTzocauAtcBb9qh09wXuXuPuNdXV1QGGLDK4zRxfwUMfP5lvvvcYlm9pZt5NT/N/SzanO6whq/vGtUzYca1bYEnB3bcBG81seqLoTGB5r2obEuWY2UhgOrAmqJhEBEIh45LjJ/DnT5/KtFGlfPrXS/j0rxfTrLGGlMu0G9cg+NlH1wL3mtky4t1DXzez+WY2P3H8K8DJZvYS8FfgenffGXBMIgKMryriN9ecyHVnTePhZVu58IfPsKZuT7rDGlIybYkLCPjmNXdfAtT0Kr4j6fgW4OwgYxCRvQvnhPi3Mw/n+MlVfOLeRVzww2f4waWz37SstwRnWwZtw9ktM+6WEJG0OnHKMH7/qbcxrrKIq+55kR89uVqzk1JgW1M7pflhSjJoSRIlBREBYFxlEQ98/CTmHTOab/z5Vb7551eVGAK2LcNuXAOtfSQiSYrywvzgktlUFeXxo6fW0BGN8aV3z9Cy3AHZmmH3KICSgoj0EgoZX77gKPLCIe76x1o6ojG+9p6jM2JT+cFmY30rZ88Yme4w3kRJQUTewsz4f+88kvxwiNueWE0kGuPbFx1LjhLDgGlsjVDfEmFKde+FHtJLSUFE+mRmfO6c6eSFQ9z0l9eJufOd989UYhggq+taAJgyvCTNkbyZkoKI7JWZ8Zl3TCPHjO8+/hpdMed7F88knKM5Kodq7c5EUlBLQUSyzbVnHk4oZNz46Epi7tz0gVlKDIdoTd0ewiFjfFVRukN5EyUFEemXT55xGDkh45t/fpWYOzdfMptcJYaDtqauhQnDijLuM1RSEJF+m3/aVMIh46t/XEEstphbLp2dMTuGZZs1O/dk3HgC6OY1ETlAV586hS++awaPvLKNT/5ykbb6PAhdMWfdrlamZth4AigpiMhBuOqUyXz5gqN4fPl2PnHvIqJdSgwHYnNDG5FoLOMGmUFJQUQO0odPmsT/nn8Uf1mxna//6dV0h5NVVu+Mr0Y7pTrzuo80piAiB+3ykyexblcLP3lmLYePLOHS4yekO6SssKbnHgW1FERkkPnCeUdy2rRq/vt3L/Pc6l3pDicrrKnbQ3lhLlXFeekO5S2UFETkkIRzQvzgstlMGl7Mx+9dyPpdLekOKeOtqWthSnVxRi40GGhSMLMKM7vfzF41sxVmdlIfdU43syVm9oqZPRlkPCISjLKCXO66PL6f1tU/rWW3tvbcp0ydjgrBtxRuBh5x9yOAmcCK5INmVgHcBpzv7kcB7w84HhEJyMRhxdx22RzW7Gzhs79ZQiymvRj6sqcjyvbmjoyceQQBJgUzKwfeDtwF4O4Rd2/sVe0y4EF335CosyOoeEQkeCcfNpwvvXsGf1mxg+8+vjLd4WSktRk8yAzBthQmA3XA3Wa22MzuNLPen8I0oNLMnjCzhWb24b5OZGbXmFmtmdXW1dUFGLKIHKoPnTiRS48fzw//vprfL92S7nAyzpoMno4KwSaFMDAHuN3dZwMtwA191JkLvBM4B/hvM5vW+0TuvsDda9y9prq6OsCQReRQmRn/e/7RHDepks/dt5Rlm3p3EAxta+paMIOJwzJrIbxuQSaFTcAmd38h8fp+4kmid51H3b3F3XcCTxEfexCRLJYXDnH7v85leEk+H/1ZLdua2tMdUsZYs7OFcZWFFOTmpDuUPgWWFNx9G7DRzKYnis4Elveq9n/AKWYWNrMi4AR6DUaLSHYaXpLPXVfUsKc9ytU/e5HWSDTdIWWENXWZO/MIgp99dC1wr5ktA2YBXzez+WY2H8DdVwCPAMuAfwJ3uvvLAcckIilyxKgyfnDZbJZvaea63ywd8jOS3J21O1syduYRBLzMhbsvAWp6Fd/Rq86NwI1BxiEi6fMvR4zkv847kq/+cQXfeWwl/3nuEekOKW22NbfTGunK2EFm0NpHIpICHzllMqvrWrjtidUcPrKEC2ePS3dIadG95tHUDJ2OClrmQkRSwMz48gVHceKUKq6//yUWrm9Id0hpsaYus6ejgpKCiKRIbk6I2z84l9EVBXzs57VsamhNd0gpt2RjE8OK8xhZlp/uUPZKSUFEUqayOI+7Lj+OjmiMq39aS0vH0JqR9OK6emomVWbkQnjdlBREJKUOG1HCDy+bw2vbd3P1T2tpi3SlO6SU2N7czob6Vo6bVJXuUPZJSUFEUu7t06r53sWzeGHtLj7y0xeHRGKoXRcfR1FSEBHpw3tmj+W7F8/kuTW7+OjPamnvHNyJ4cV19RTm5jBjTFm6Q9knJQURSZsLZ4/jOxfN5JnVOwd9YqhdX8/sCRXk5mT2n93Mjk5EBr33zR3Ht993LP9YNXgTw56OKMu3NFOT4V1HoKQgIhng/TXjufGimYM2MSze0EDM4bhJlekOZb+UFEQkI1w0d1xPYhhss5JeXNdAyGD2BCUFEZF+604Mz6zeycd+sZBINJbukAZE7bp6ZowpoyQ/81cWUlIQkYxy0dxxfPO9x/DUa3Vc/8CyrF9ZtbMrxuINjdRMzPzxBNCCeCKSgT5w3ATqdnfwncdeY0RpPp8/78h0h3TQXtnSTFtnV8bfn9BNSUFEMtInzziMHbs7+NFTa6guzefqU6ekO6SDUruuHsiOQWYIuPvIzCrM7H4ze9XMVpjZSXupd5yZRc3soiDjEZHsYWZ86d1HMe/oUXz1jyt45OVt6Q7poLy4rp6Jw4oYUVaQ7lD6JegxhZuBR9z9COJ7L79lq00zywG+BTwWcCwikmVyQsb3PzCLmeMr+I/7lvYsPZ0t3J3adQ1ZM54AASYFMysH3g7cBeDuEXdv7KPqtcADwI6gYhGR7FWQm8NtH5xDXjjE/F8szKq9nhdtaGRXS4QTpygpAEwG6oC7zWyxmd1pZm/absjMxgIXArcHGIeIZLmxFYXccslsVu3Yww0PvIR7dsxI+uULGyjJDzPvmNHpDqXfgkwKYWAOcLu7zwZagBt61bkJuN7d9zkZ2cyuMbNaM6utq6sLJloRyWinHD6cfz97Or9fuoW7n1mX7nD2q7E1wsPLtvCe2WOy4v6EbkEmhU3AJnd/IfH6fuJJIlkN8GszWwdcBNxmZu/pfSJ3X+DuNe5eU11dHWDIIpLJPn7aVM6aMZKv/nE5Dy3elO5w9umBRZvpiMa47PiJ6Q7lgASWFNx9G7DRzKYnis4ElveqM9ndJ7n7JOJJ4xPu/rugYhKR7BYKGbdcMpsTpwzjut8u5YGFmZkY3J17X1jPnAkVGb9Udm9Bzz66FrjXzJYBs4Cvm9l8M5sf8HVFZJAqzMvhrsuP421Th/Mf9y/l/gxMDM+vqWdNXQsfPCG7WgkQ8M1r7r6EeBdRsjv2UveKIGMRkcGjMC+HOy+v4aM/q+Vz9y8l2hXjkuMnpDusHve+sJ7ywlzeeWz2DDB309pHIpKVCnJz+PGHazhtWjU3PPgSdz69Jt0hAVC3u4NHX9nGRXPHUZCbk+5wDpiSgohkrYLcHBZ8qIZ3HjOar/5xBd97bGXap6v+tnYjnV3OZSdkTsvlQGTPPCkRkT7khUPcculsSvLD3PK3VTS3R/niu2YQClnKY2npiHL3M2s55bDhTK0uSfn1B4KSgohkvZyQ8c33HUNpQZg7/7GWbU3tfP8DsyjMS233zZ1Pr2XnngjXnT0tpdcdSOo+EpFBwcz4wjuP5L/fNYNHl2/jAwueY0dze8quv2tPBwueWs25R41iThbssLY3SgoiMmiYGR+8oCgQAAALZElEQVQ5ZTI//lANq3bs4YIfPsPyLc0pufYP/raK9miMz507ff+VM5iSgogMOu+YMZL75sdX6r/4R8/x9OvBLo+zYVcr976wnotrxmXtWEI3JQURGZSOGlPOQ594G+MqC7ny7hcDXRbje4+vJCdkfPrM7B1L6KakICKD1qjyAn47/ySOm1TFZ3+zlNufWD3gU1YXrq/nd0u2cOXbJjOqPDs20tkXJQURGdTKCnK556rjePfMMXzrkVf52M8XsqmhdUDOvWrHHq7+aS3jqwqZf9rUATlnuikpiMiglx/O4eYPzOL6c4/g6dd38o7vPckP/76KSHSfq/bv09amNi7/yT/JCRm/+MgJlBfmDmDE6aOkICJDQihkfPz0qfzl30/jtGnV3PjoSs656Sn+b8lmumIH1qXU2Brh8p/8k6a2Tu658ngmDive/5uyhJKCiAwpYysK+dGHarj7yuPIywnx6V8v4azvP8nvFu8/Obg7z63exb/e9QLrdray4MNzOXpseYoiTw1L9zohB6qmpsZra2vTHYaIDAKxmPPoK9u4+a+v8+q23Ywozee8Y0bzrmNHM2dCZc9SGV0x56nX67j1b6tYuL6B6tJ8vn7hMZw1Y2Saf4L+M7OF7t571eq31lNSEJGhLhZz/rJiOw8u2szfV+6gIxpjeEke4VCI3e2dtES6gHgrY/5pU3h/zfisWwG1v0kh0LWPzKwCuBM4GnDgKnd/Lun4B4HrAQN2Ax9396VBxiQi0lsoZJx91CjOPmoUezqi/HXFdp58rY5wyCgtyKW0IMzk4cXMO3o0eeHB3ese9IJ4NwOPuPtFZpYHFPU6vhY4zd0bzGwesAA4IeCYRET2qiQ/zAWzxnLBrLHpDiUtAksKZlYOvB24AsDdI0AkuY67P5v08nlgXFDxiIjI/gXZDpoM1AF3m9liM7vTzPY1b+sjwJ8DjEdERPYjyKQQBuYAt7v7bKAFuKGvimZ2BvGkcP1ejl9jZrVmVltXF+zCViIiQ1mQSWETsMndX0i8vp94kngTMzuW+GD0Be6+q68TufsCd69x95rq6urAAhYRGeoCSwruvg3YaGbdi4ufCSxPrmNmE4AHgQ+5+2tBxSIiIv0T9Oyja4F7EzOP1gBXmtl8AHe/A/giMAy4zcwAov2ZRysiIsEINCm4+xKg9x/5O5KOXw1cHWQMIiLSf4P7LgwRETkgWbfMhZnVAY1AU69D5fsp29/z7v8OB3YeRGh9Xb8/x3uX7+t171iTyw4m7lTGnPw8HZ+1vh/6fuzreDZ+Pw4kZoDD3X3/q/e5e9Y9gAUHWra/50n/rR2omPpzvHf5vl73jvVQ405lzOn+rPX90PdjsH0/DiTm/lyj+5Gt3Ud/OIiy/T3v6/2HGlN/jvcu39frvmI9lLhTGXPy83R81vp+HDh9P/r/PNNj7s81gCzsPgqamdV6Fs6Aysa4FXPqZGPcijk9srWlEKQF6Q7gIGVj3Io5dbIxbsWcBmopiIhID7UURESkx6BOCmb2EzPbYWYvH8R755rZS2a2ysxuscQt14lj15rZq2b2ipl9e2CjDiZuM/sfM9tsZksSj/MyPeak4/9uZm5mwwcu4sA+56+Y2bLEZ/yYmY3JgphvTHyfl5nZQ4nNsQZUQHG/P/E7GDOzAevHP5RY93K+y83s9cTj8qTyfX7v0+Zgpk9ly4P4fg5zgJcP4r3/BE4kvivcn4F5ifIzgL8A+YnXI7Ik7v8B/iObPuvEsfHAo8B6YHimxwyUJdX5N+COLIj5bCCceP4t4FvZ8P0AjgSmA08ANemONRHHpF5lVcSX+KkCKhPPK/f1c6X7MahbCu7+FFCfXGZmU83sETNbaGZPm9kRvd9nZqOJ/3I/7/H/ez8D3pM4/HHgm+7ekbjGjiyJO1ABxvx94D+Jb+ea8TG7e3NS1eKBjjugmB9z92iiaiCbXQUU9wp3X5kpse7FOcDj7l7v7g3A48C56fxd3Z9BnRT2YgFwrbvPBf4DuK2POmOJL/3dbVOiDGAacKqZvWBmT5rZcYFG+4ZDjRvgU4kugp+YWWVwofY4pJjN7AJgs6d23+5D/pzN7GtmthH4IPFFH4M2EN+NbleRus2uBjLuoPUn1r6MBTYmve6OP1N+rrcIepXUjGJmJcDJwH1J3Xf5B3iaMPGm4InAccBvzWxKItsHYoDivh34CvF/uX4F+C7xPwCBONSYzawI+C/iXRspMUCfM+7+BeALZvZ54FPAlwYsyF4GKubEub4ARIF7Bya6fV5rwOIO2r5iNbMrgU8nyg4D/mRmEWCtu1+Y6lgHwpBKCsRbRo3uPiu50MxygIWJl78n/gc0uQk9DticeL4JeDCRBP5pZjHi650EuSXcIcft7tuT3vdj4OEA44VDj3kq8S1dlyZ+EccBi8zseI/v1ZGJMfd2L/AnAkwKDFDMZnYF8C7gzCD/gZNkoD/rIPUZK4C73w3cDWBmTwBXuPu6pCqbgdOTXo8jPvawmfT/XH1L96BG0A9gEkkDRsCzwPsTzw2YuZf39R4EOi9RPh/4cuL5NOJNQ8uCuEcn1fks8OtMj7lXnXUM8EBzQJ/z4Ul1rgXuz4KYzyW+AVb1QMeaiu8HAzzQfLCxsveB5rXEB5krE8+r+vu9T8cj7QEE+sPBr4CtQCfxf+F/hPi/Ph8BliZ+Eb64l/fWAC8Dq4FbeeNGvzzgF4lji4B/yZK4fw68BCwj/i+w0Zkec6866xj42UdBfM4PJMqXEV9rZmwWxLyK+D9uliQeAzpjKsC4L0ycqwPYDjyazljpIykkyq9KfMargCsP5HufjofuaBYRkR5DcfaRiIjshZKCiIj0UFIQEZEeSgoiItJDSUFERHooKcigYGZ7Uny9O81sxgCdq8viq6q+bGZ/2N8qpWZWYWafGIhri/SmKakyKJjZHncvGcDzhf2NReIClRy7mf0UeM3dv7aP+pOAh9396FTEJ0OLWgoyaJlZtZk9YGYvJh5vS5Qfb2bPmdliM3vWzKYnyq8ws9+b2d+Av5rZ6Wb2hJndb/H9Bu7tXvM+UV6TeL4nsQjeUjN73sxGJsqnJl6/ZGZf7Wdr5jneWBCwxMz+amaLEue4IFHnm8DUROvixkTdzyV+xmVm9r8D+DHKEKOkIIPZzcD33f044H3AnYnyV4FT3X028VVMv570njnARe5+WuL1bOAzwAxgCvC2Pq5TDDzv7jOBp4CPJl3/Znc/hjeviNmnxLo/ZxK/4xygHbjQ3ecQ38fju4mkdAOw2t1nufvnzOxs4HDgeGAWMNfM3r6/64n0ZagtiCdDyzuAGUkrW5YlVrwsB35qZocTXzU2N+k9j7t78lr6/3T3TQBmtoT4mjj/6HWdCG8sMLgQOCvx/CTeWCP/l8B39hJnYeLcY4EVxNfch/iaOF9P/IGPJY6P7OP9ZyceixOvS4gniaf2cj2RvVJSkMEsBJzo7u3JhWZ2K/B3d78w0T//RNLhll7n6Eh63kXfvzOd/sbg3N7q7Eubu89KLBf+KPBJ4Bbi+zFUA3PdvdPM1gEFfbzfgG+4+48O8Loib6HuIxnMHiO+UikAZta99HE5byxTfEWA13+eeLcVwCX7q+zurcS38Px3MwsTj3NHIiGcAUxMVN0NlCa99VHgqkQrCDMba2YjBuhnkCFGSUEGiyIz25T0uI74H9iaxODrcuLLngN8G/iGmS0m2NbyZ4DrzGwZ8Q1Ymvb3BndfTHyF1UuJ78dQY2YvAR8mPhaCu+8CnklMYb3R3R8j3j31XKLu/bw5aYj0m6akigQk0R3U5u5uZpcAl7r7Bft7n0g6aUxBJDhzgVsTM4YaCXD7U5GBopaCiIj00JiCiIj0UFIQEZEeSgoiItJDSUFERHooKYiISA8lBRER6fH/AXXTW0LNvl/NAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 00:36 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>4.608401</th>\n",
       "    <th>4.314568</th>\n",
       "    <th>0.216563</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmQnPV95/H3t4+ZnntGo9EtJMCAjB3AIK7Y8YJxSMwSiNdOCmpTDji7rBPHTjaJU3ZS5WSddRI7m03sUAtLvPHGie0kxiHBKozBNgRnbYIFSAKEAIF1jI65pDl7ps/v/vE802qGkTTS9NPHzOdV9dQ8/fTT/Xx/avV853c+5u6IiIgAxGodgIiI1A8lBRERKVFSEBGREiUFEREpUVIQEZESJQURESlRUhARkRIlBRERKVFSEBGRkkStAzhTK1eu9M2bN9c6DBGRhvL0008Pu3vf6c5ruKSwefNmtm/fXuswREQaipntX8h5aj4SEZESJQURESlRUhARkRIlBRERKVFSEBGREiUFEREpUVIQEZESJQURkQbw599+me+9MhT5dZQURETqXLHofP47r/DUj45Ffi0lBRGROjeRyVN06GpJRn4tJQURkTo3ls4B0N3aFPm1lBREROrc6HQWgG7VFEREZLRUU1BSEBFZ9kanlRRERCQ0lg6aj7pa1KcgIrLsHQ+bjzT6SEREGE3naG9O0JSI/le2koKISJ0bnc5WpZYASgoiInVvLJ2rSiczRJgUzOwiM9tRto2b2a/POcfM7PNmttfMdpnZ5VHFIyLSqEanq5cUElG9sbu/BFwGYGZx4BDwwJzT3gNcEG5XA/eEP0VEJDSazrJlTWdVrlWt5qMbgFfdff+c47cCX/LAk0C3ma2tUkwiIg1hbDpHV6M3H81xG/DVeY6vBw6WPe4Pj4mICODujKZzVVniAqqQFMysCbgF+Noi3uMuM9tuZtuHhqJfT1xEpF5MZvLki974Hc1l3gM84+4D8zx3CNhY9nhDeOx13P0+d9/q7lv7+voiClNEpP6MVnGFVKhOUrid+ZuOAB4EPhCOQroGGHP3I1WISUSkIYzNrntUpeajyEYfAZhZG/CTwH8pO/YhAHe/F3gIuAnYC6SBO6OMR0Sk0VS7phBpUnD3KaB3zrF7y/Yd+HCUMYiINLLSvRSWUJ+CiIicpVJNYamMPhIRkbM326fQqaQgIiKj6SwtyTipZLwq11NSEBGpY8eruBgeKCmIiNS10XSuastmg5KCiEhdG5vO0lOl4aigpCAiUtdG1XwkIiKzqnkvBVBSEBGpW+7OWDpHV4uaj0RElr3pXIFsoaiagoiIVH82MygpiIjUrROL4SkpiIgse6PpYDE89SmIiAij06opiIhIaLb5SJPXRESk6vdSACUFEZG6NZbO0ZyIVW2FVFBSEBGpW9Ve4gKUFERE6tbodJbuKo48AiUFEZG6NZrO0aWagoiIQNh8VMXZzKCkICJSt0ans+pTEBGRQNDRrD4FEZFlbyZXIJOv7gqpEHFSMLNuM7vfzPaY2Ytmdu2c568zszEz2xFun4wyHhGRRnFihdTq1hQSEb//54CH3f39ZtYEtM5zzvfc/eaI4xARaSi1mM0MESYFM+sC3gncAeDuWSAb1fVERJaSWtxLAaJtPjoXGAK+aGbPmtkXzKxtnvOuNbOdZvZNM3tLhPGIiDSM2aSwlOYpJIDLgXvc/W3AFPDxOec8A2xy90uBvwD+ab43MrO7zGy7mW0fGhqKMGQRkfowey+FpTT6qB/od/d/Cx/fT5AkStx93N0nw/2HgKSZrZz7Ru5+n7tvdfetfX19EYYsIlIfSvdSWCrNR+5+FDhoZheFh24AdpefY2ZrzMzC/avCeEaiiklEpFGMpnMk40ZrU/VWSIXoRx99BPhyOPLoNeBOM/sQgLvfC7wf+GUzywPTwG3u7hHHJCJS98ams3S1NBH+3Vw1kSYFd98BbJ1z+N6y5+8G7o4yBhGRRjSaztFT5U5m0IxmEZG6VIt7KYCSgohIXRqdztFV5dnMoKQgIlKXxtLVXyEVlBREROqOuzM8lWVFm2oKIiLL3th0jmy+yKqO5qpfW0lBRKTODIxnAFjdmar6tZUURETqzODEDKCkICIinKgpqPlIRERKNYVVnUoKIiLL3uB4ho5UgtamqFcieiMlBRGROjMwPlOTpiNQUhARqTuDE5madDKDkoKISN1RTUFERIBgNvPguGoKIiJCOJu5UGSVkoKIiNRyjgIoKYiI1JVazmYGJQURkbpyYt0j1RRERJa9gfFwNnOHagoiIsve0EQwm7mlKV6T6yspiIjUkYHxmZr1J4CSgohIXanlxDVQUhARqSu1XOIClBREROrG7GzmWiyZPSvSpGBm3WZ2v5ntMbMXzezaOc+bmX3ezPaa2S4zuzzKeERE6llpNnONRh4BRL1Y9+eAh939/WbWBLTOef49wAXhdjVwT/hTRGTZqfUcBYiwpmBmXcA7gf8D4O5Zdx+dc9qtwJc88CTQbWZro4pJRKSe1XqOAkTbfHQuMAR80cyeNbMvmFnbnHPWAwfLHveHx17HzO4ys+1mtn1oaCi6iEVEamhwYgnXFAiapi4H7nH3twFTwMfP5o3c/T533+ruW/v6+ioZo4hI3VjqNYV+oN/d/y18fD9Bkih3CNhY9nhDeExEZNkZHJ+p6WxmiDApuPtR4KCZXRQeugHYPee0B4EPhKOQrgHG3P1IVDGJiNSzWs9RgOhHH30E+HI48ug14E4z+xCAu98LPATcBOwF0sCdEccjIlK3giUuatefABEnBXffAWydc/jesucd+HCUMYiINIrBiQxXbl5R0xg0o1lEpA7Uw2xmUFIQEakLo+lgNvPqGo48AiUFEZG6MDtHQTUFEREpzVGo9eijBSUFMzvfzJrD/evM7KNm1h1taCIiy0cpKTRI89HXgYKZvQm4j2DC2Vcii0pEZJlptOajorvngfcCf+HuHwO0cJ2ISIUMjs/QmUqQStZuNjMsPCnkzOx24BeBbeGxZDQhiYgsP/UwmxkWnhTuBK4FPu3uPzKzc4G/iS4sEZHlZWB8puZNR7DAGc3uvhv4KICZ9QAd7v6ZKAMTEVlOBsYzXH1ubWczw8JHHz1uZp1mtgJ4BvhLM/uf0YYmIrI8ZPIFDo9Ns3HF3JtTVt9Cm4+63H0c+A8Ed0q7Gnh3dGGJiCwf/cencYdNvY2TFBLhbTJ/nhMdzSIiUgEHRtJAYyWFTwHfAl519x+a2XnAK9GFJSKyfOwbmQJgU+/cOxZX30I7mr8GfK3s8WvA+6IKSkRkOdk/kqatKU5vW1OtQ1lwR/MGM3vAzAbD7etmtiHq4EREloMDx9Kc09uGmdU6lAU3H32R4NaZ68LtG+ExERFZpH0jU2yug/4EWHhS6HP3L7p7Ptz+L9AXYVwiIstCoej0H5vmnAZLCiNm9gtmFg+3XwBGogxMRGQ5ODo+Q7ZQZNOK2ncyw8KTwgcJhqMeBY4A7wfuiCgmEZFlY/9wMPKooZqP3H2/u9/i7n3uvsrdfxaNPhIRWbT9x4I5Co3WfDSf36hYFCIiy9T+kTTJuLG2q6XWoQCLSwq1HzslItLg9o9MsXFFK/FYffxKXUxS8IpFISKyTO0fSbOpDhbCm3XKGc1mNsH8v/wNOG1dx8z2ARNAAci7+9Y5z18H/DPwo/DQP7r7p04btYjIEuDu7B+Z4qo6WDJ71imTgrt3VOAa17v78Cme/56731yB64iINJSRqSxT2UJdLIQ3azHNRyIisgj762h11FlRJwUHHjGzp83srpOcc62Z7TSzb5rZW+Y7wczuMrPtZrZ9aGgoumhFRKpof7g66jl1MnENFrhK6iK8w90Pmdkq4FEz2+PuT5Q9/wywyd0nzewm4J+AC+a+ibvfB9wHsHXrVnVwi8iSsH8kjRlsXFEfw1Eh4pqCux8Kfw4CDwBXzXl+3N0nw/2HgKSZrYwyJhGRenHgWJp1XS00J+K1DqUksqRgZm1m1jG7D9wIPD/nnDUWrhVrZleF8WhNJRFZFvaNTHFOHQ1HhWibj1YDD4S/8xPAV9z9YTP7EIC730uwhtIvm1kemAZuc3c1D4nIsnBgJM2Nb1ld6zBeJ7KkEN6d7dJ5jt9btn83cHdUMYiI1KuJmRwjU9m66mQGDUkVEamJehyOCkoKIiI1ceCYkoKIiIRO1BTUfCQisuztH5mit62J9uaop4udGSUFEZEa2D+SrrumI1BSEBGpOnfnlcEJzutrr3Uob6CkICJSZYdGpxmezHLphq5ah/IGSgoiIlW2q38MgEs2dNc4kjdSUhARqbKd/aMk48aWtZW4ZU1lKSmIiFTZroNjvHltZ10thDdLSUFEpIqKRee5Q2NcUof9CaCkICJSVa8NTzKZyXNpHfYngJKCiEhV7TwYdDJfulFJQURk2dvVP0prU5zz63COAigpiIhU1c7+Md66vot4zGodyryUFEREqiSbL7L7yDiX1WnTESgpiIhUzcsDE2TzxbodeQRKCiIiVbPj4ChA3Y48AiUFEZGq2dU/Sk9rkg09LbUO5aSUFEREqmRX/xiXbOjGrD47mUFJQUSkKtLZPC8PTNTt/IRZSgoiIlXwwuFxik5dLpddTklBRKQKdoadzPW4XHa5SJOCme0zs+fMbIeZbZ/neTOzz5vZXjPbZWaXRxmPiEit7OwfY11Xir6O5lqHckrVuGP09e4+fJLn3gNcEG5XA/eEP0VElox8ocgPXh3m2vNX1jqU06p189GtwJc88CTQbWZraxyTiEhFPfnaMYYns9z01jW1DuW0ok4KDjxiZk+b2V3zPL8eOFj2uD88JiKyZGzbdZi2pjjXb1lV61BOK+rmo3e4+yEzWwU8amZ73P2JM32TMKHcBXDOOedUOkYRkcjkCkUefuEo7754Nalk/d1pba5Iawrufij8OQg8AFw155RDwMayxxvCY3Pf5z533+ruW/v6+qIKV0Sk4v517zCj6Rw3X7Ku1qEsSGRJwczazKxjdh+4EXh+zmkPAh8IRyFdA4y5+5GoYhIRqbZtO4/QkUrwzgvrv5MZom0+Wg08EE7nTgBfcfeHzexDAO5+L/AQcBOwF0gDd0YYj4hIVWXyBR7ZfZQbL15Dc6L+m44gwqTg7q8Bl85z/N6yfQc+HFUMIiK19MTLw0zM5Ln50sYZVFnrIakiIkvWtl2H6W5N8o43NUbTESgpiIhEYiZX4Nu7B/jpt6whGW+cX7WNE6mISAN5bM8gU9lCw4w6mqWkICISgW/sOkxvWxPXnLei1qGcESUFEZEKG5nM8OjuAW65bB2JBmo6AiUFEZGK+8dnDpErOLdf1XgrMCgpiIhUkLvzdz88wOXndHPh6o5ah3PGlBRERCpo+/7jvDo0xW1XNl4tAZQUREQq6qtPHaC9OdFQE9bKKSmIiFTI2HSOh547wi2XraO1qRr3MKs8JQURkQr55x2HmMkVub1Bm45ASUFEpCLcna8+dZCL13by1vWdtQ7nrCkpiIhUwHOHxnjxyDi3X7WRcHXohqSkICJSAX//w4OkkjFufVtj31FYSUFEZJHcnUd2D/CuLavoTCVrHc6iKCmIiCzSC4fHGZrIcP1Fq2odyqIpKYiILNJ39wwCcJ2SgoiIPPbSIJdu6KKvo7nWoSyakoKIyCKMTGbYcXCU67c0fi0BlBRERBblX14ewp0l0Z8ASgoiIovy2EtDrGxv5sfWd9U6lIpQUhAROUv5QpF/eWmQ6y7qIxZr3Alr5ZQURETO0jMHRhmfyfOuJdKfAEoKIiJn7bGXBknEjHdcsLLWoVRM5EnBzOJm9qyZbZvnuTvMbMjMdoTbf4o6HhGRSnlszyBbN/c0/CzmctWoKfwa8OIpnv97d78s3L5QhXhERBbt0Og0e45OLKmmI4g4KZjZBuDfA/plLyJLyuMvBbOYlRTOzJ8Dvw0UT3HO+8xsl5ndb2YbI45HRKQiHt09wMYVLZzf117rUCoqsqRgZjcDg+7+9ClO+waw2d0vAR4F/vok73WXmW03s+1DQ0MRRCsisnDHp7L86yvD3PTWtQ1974T5RFlTeDtwi5ntA/4OeJeZ/W35Ce4+4u6Z8OEXgCvmeyN3v8/dt7r71r6+vghDFhE5vW+9cJR80fmZS9fVOpSKiywpuPsn3H2Du28GbgO+6+6/UH6Oma0te3gLp+6QFhGpC9t2HWFzbytvWde4t908marPUzCzT5nZLeHDj5rZC2a2E/gocEdU152YyfGj4SkKRY/qEiKyDAxPZvj+q8PcfMm6Jdd0BJCoxkXc/XHg8XD/k2XHPwF8ohoxfO+VYX7ly8+QSsa4cHUHF63u4ILV7azpamFNZ4rVnc2s7kyRSsarEY6INKhvPn+UosPNl649/ckNqCpJoR5ctrGbz77vEvYcneClgXEee2mIrz3d/7pzknHjmvN6efebV3PDm1exoae1RtGKSL3atvMwb1rVzkWrO2odSiSWTVJY193Cz1/5+hGvY+kcAxMzDIzPMDCe4eWBCb794gC/9+AL/N6DL7Cpt5XuliRtzQnamhN0tyTZ1NvKpt620s+ulqUzk1FETm1gfIan9h3j1264YEk2HcEySgrz6WpN0tWa5MKyjP87N72Z14Ym+c6Lg+w4OMpkJs9UJs+xqTQ7prIMTWRe9x6dqQQbV7SysaeVNV0pOlIJ2psTtKcStCTjxGMWbGb0tDWxZU0H3a1N1S6qiFTAQ88dwR1uvmTpjTqatayTwsmc19fOeSeZkJLO5jlwLM2+4TQHjk1x8Ng0/cfT7B2a5PuvDjOZyXO6vuy1XSm2rOmgsyXJ2HSO0XSOsekczYkYa7pSrO1KsaazhZUdTfS0NtHdmqSntYlEzMjki+QKRXIFp6+jmY09LSTiWtdQpBq27TrCljUdvGnV0pqwVk5J4Qy1NiXYsqaTLWvmH4rm7kznCkzM5JnOFii4Uyw6uYIzODET9GkcneDFI+O8OjRFd2uSrpYkG3payOSLHB2b4flDYwxPZhcUTzJubOpt4/y+Njb3tgW1lhWtQbKIxcjkC2TyRbKFIl0tSVZ1NNPenFiyVV+RqBwanebp/cf52E9dVOtQIqWkUGFmRmtTgtamN/7TXkwn1y3wln2ZfIFjU1mOT+UYnc4yms5RdCcZj9GUiJGIGQPjGV4dmuTVwUn2Dk7y2EtDZPOnWlEkkErGWNWRoqetiRVhLWRFWxPnr2rnwtUdXLi6nY4ltOqjyGJl8gXueXwvADdfsjRHHc1SUqhTzYk4a7taWNvVsuDXFIvO4ESGA8fS9B9P4w7NyRjNiTiJuDGWzjE4McPgeIahyQzHprIMTWZ4eWCS4ckMmbKEsrqzmY5UkuZEjOZEjEQ8RjZfZCYX1DyK7mzsaeW8vjbOW9nGOb2ttDYlSCXjtCTjOM7h0Wn6jwfbwPgM09kC07kCmVyRXLFIKhGnpSlOKhkjlYwHWyJ4nAybxGZb4tydQvHElkzE6G5JhjWtJnpak6xoa6KnrYnulqSa1KQi3J1tu47wmYf30H98mvddvoFNvW21DitSSgpLSCxmrOlKsaYrxVXnrjij1xaLzqHRaV4emOClgQleG5piOls40fyUL9KRStDX0Vyay3FgZIoHnjnERCZ/yvduSsRY05mitSleShrNyQQzuQKDEzlmcsXStWZyQeLJz+mYMYNEzIhZ0HGfzRffcE75uW2zCaopRksyTjIeI2ZGzGZrc3F6Wpvoak3S3XKitrSivYnetiY6UknamuN0NCdJJWNqbltmikXniVeG+Nx3XuHZA6NsWdPB3/7S1UvqZjono6QgQJBQZvsjbnjz6gW/zt0ZnszSfzzNdLbATL7AdLaI46zrbmFDTwsr25rP+P617n7KX8TuTjpbYHQ6x/GpoHntWDrLaDrLyGSWyUye6VyBmWyBdLZAvlik6FAMaxzpbIE9R8dLHf0nSzAAMYNkPGiymx1NFjPDwiQTM6O1OU5HKklHc4LOlgS9bc30dTSzqqOZ3vbmMCEGtbaWpjgdZSPUlHDqx9BEhn/YfpCvPnWA/uPTrO5s5rPvv4T3Xb6B+BK5B/PpKCnIopgZfR3BL8BKv+/pnp+dP7K+e+FNbPNxd8Zn8hybynJsKlNKKlOZPJOZAlOZPLlCkULRyRedfLGIO+EosyDJTGWDwQUTMzmOjE0zMjXCaDp32mvHY0Z7c6JUiwpqUjHamoOhzW3NCdqa4qSaghpWSzJOeyrB2q4U67pbWNfdsqTu+lULk5k83949wLZdh3n8pSHyRefa83r5+Hu2cOPFa2hKLK+mSCUFWfbMjK6WYBTYuSsr116cyRcYmcwyPJkJa1FB09hMODptMhMkkcmZfNCElgv6XKbDBHN0bCZMTHlmwia8+aSSsTCxBMmlvTlBd2uSzpYk3S1NdKQSpWSTSsZpDSdizpZ5RXsTHctsRNp0tsB39wyybddhvrtnkEy+yJrOFHe+fTO3XXXOkrtHwplQUhCJSHMiXvprvhIKRWcmV2B8JseRsRkOj05zZHSGwYkZ0mEz2VQmz8RMnsOjM7x4ZILRdJapbOG0793aFA/XAEvR295UShhdLUlWtjezrruF9d0trOlKlf5ydneKTqmfpp65B4MwdhwcZduuI3znxQHS2QIr25u57cqN3HzpOq44p+eMmzmXIiUFkQYRj51oMlvb1cLl5/Qs6HWFor+uE38qk2dsOlfqTxmZyjAwnuHo+AwDYzPsPjzOaPj83FWFzSAZi1EI+2ZmxQwSsVjQHJZK0NvWVOq8b2uO09qUoCVsAguaxoJj7c0JmhMxkolgxFlTPEZHKkFnS9A/c6pf0tl8MahphYlwKpNnfCbP8GSG4YkMw5NBufaNTLF/JM10LkiOPa1Jbr1sPT9zyVquPq932fQVLJSSgsgSF4/Nzp05s9e5O5OZPEMTGY6MzXBodJrDo9PM5IrEYxCPxYib4WG/Sr7o5AtFJkr9M1lePDrOVCaYyDmTCyZRLlTMCJJGMk4yZiQTwfUmM3nGZ4JRa6fSkUqwqqOZTb1t/Pj5K9m8spU3rWrnys0rSkOe5Y2UFERkXmYWjKhKJU+67MuZyhWKpDMFJrP5Un9JtrR0S5FMrshEJs/4dC7YZvJlS7sEw5Dbm4IRXp2pJB2pBB2pJO2pRGlEV297M71tTVoG/ywpKYhI1STjMbpaY3S1asRUvVIdSkRESpQURESkRElBRERKlBRERKRESUFEREqUFEREpERJQURESpQURESkxNxPc5f5OmNmQ8D+eZ7qAsbO8vHs/uzPlcDwWYY49zpncs58xxcSd/l++bEoyxFlGcr3l/tnUesylO/Xy2eh7/bZlWOTu/ed9ix3XxIbcN/ZPp7dL/u5vVJxnMk58x1fSNzzlSHqckRZBn0W9VOGevws9N1eXDlOty2l5qNvLOLxN05yTiXiOJNz5ju+kLjL9ytRhoW8T5RlWMj1F2IpfBa1LsNCYzidSpZD3+0INVzzUTWY2XZ331rrOBZrKZRjKZQBlkY5VIb6EWU5llJNoZLuq3UAFbIUyrEUygBLoxwqQ/2IrByqKYiISIlqCiIiUrLkk4KZ/ZWZDZrZ82fx2ivM7Dkz22tmn7eyG9Ga2UfMbI+ZvWBmn61s1G+Io+JlMLPfN7NDZrYj3G6qfORviCWSzyJ8/jfNzM1sZeUinjeOKD6LPzCzXeHn8IiZrat85G+IJYpy/En4ndhlZg+YWXflI39dHFGU4efC73TRzCLre1hM7Cd5v180s1fC7RfLjp/yezOvqIY11csGvBO4HHj+LF77FHANYMA3gfeEx68Hvg00h49XNWAZfh/4rUb/LMLnNgLfIpi/srLRygB0lp3zUeDeRvwsgBuBRLj/GeAzDViGNwMXAY8DW+st9jCuzXOOrQBeC3/2hPs9pyrnqbYlX1Nw9yeAY+XHzOx8M3vYzJ42s++Z2Za5rzOztQRf1ic9+Nf9EvCz4dO/DPyxu2fCaww2YBmqLsJy/Bnw20DkHWRRlMHdx8tObaNxy/GIu+fDU58ENjRgGV5095eijHsxsZ/ETwGPuvsxdz8OPAr89Nl+/5d8UjiJ+4CPuPsVwG8B/2uec9YD/WWP+8NjABcCP2Fm/2Zm/2JmV0Ya7fwWWwaAXw2r+n9lZj3RhXpKiyqHmd0KHHL3nVEHegqL/izM7NNmdhD4j8AnI4z1VCrxf2rWBwn+Mq22Spah2hYS+3zWAwfLHs+W56zKuezu0Wxm7cCPA18ra15rPsO3SRBU1a4BrgT+wczOC7Nx5CpUhnuAPyD4q/QPgD8l+CJXzWLLYWatwO8QNFvURIU+C9z9d4HfNbNPAL8K/F7FglyASpUjfK/fBfLAlysT3YKvW7EyVNupYjezO4FfC4+9CXjIzLLAj9z9vZWOZdklBYLa0ai7X1Z+0MziwNPhwwcJfmmWV383AIfC/X7gH8Mk8JSZFQnWIhmKMvAyiy6Duw+Uve4vgW1RBnwSiy3H+cC5wM7wi7QBeMbMrnL3oxHHPqsS/5/KfRl4iConBSpUDjO7A7gZuKFafySVqfRnUU3zxg7g7l8EvghgZo8Dd7j7vrJTDgHXlT3eQND3cIizKWdUHSn1tAGbKevQAb4P/Fy4b8ClJ3nd3E6am8LjHwI+Fe5fSFB1swYrw9qyc/4r8HeN+FnMOWcfEXc0R/RZXFB2zkeA+xvxswB+GtgN9FUj/ij/PxFxR/PZxs7JO5p/RNDJ3BPur1hIOeeNq1ofXq024KvAESBH8Bf+LxH8dfkwsDP8T/zJk7x2K/A88CpwNycm+zUBfxs+9wzwrgYsw98AzwG7CP56WhtlGaIqx5xz9hH96KMoPouvh8d3Eaxvs74RPwtgL8EfSDvCLdJRVBGV4b3he2WAAeBb9RQ78ySF8PgHw3//vcCdZ/K9mbtpRrOIiJQs19FHIiIyDyUFEREpUVIQEZESJQURESlRUhARkRIlBVkSzGyyytf7gpldXKH3KliwQurzZvaN060uambdZvYrlbi2yFwakipLgplNunt7Bd8v4ScWd4tUeexm9tfAy+7+6VOcvxnY5u5vrUZ8sryopiBLlpn1mdnXzeyH4fb28PhVZvYDM3s61xI0AAACoUlEQVTWzL5vZheFx+8wswfN7LvAd8zsOjN73Mzut+A+AV+eXY8+PL413J8MF7TbaWZPmtnq8Pj54ePnzOy/L7A28wNOLPbXbmbfMbNnwve4NTznj4Hzw9rFn4Tnfiws4y4z+28V/GeUZUZJQZayzwF/5u5XAu8DvhAe3wP8hLu/jWBF0j8se83lwPvd/d+Fj98G/DpwMXAe8PZ5rtMGPOnulwJPAP+57Pqfc/cf4/WrVc4rXKPnBoIZ5gAzwHvd/XKCe3j8aZiUPg686u6XufvHzOxG4ALgKuAy4Aoze+fpricyn+W4IJ4sH+8GLi5bdbIzXI2yC/hrM7uAYJXYZNlrHnX38nXun3L3fgAz20GwXs2/zrlOlhMLCj4N/GS4fy0n1q//CvA/ThJnS/je64EXCdbDh2C9mj8Mf8EXw+dXz/P6G8Pt2fBxO0GSeOIk1xM5KSUFWcpiwDXuPlN+0MzuBh5z9/eG7fOPlz09Nec9MmX7Beb/zuT8ROfcyc45lWl3vyxcCvxbwIeBzxPcW6EPuMLdc2a2D0jN83oD/sjd//cZXlfkDdR8JEvZIwSrjgJgZrPLEndxYgnhOyK8/pMEzVYAt53uZHdPE9yO8zfNLEEQ52CYEK4HNoWnTgAdZS/9FvDBsBaEma03s1UVKoMsM0oKslS0mll/2fYbBL9gt4adr7sJljwH+CzwR2b2LNHWln8d+A0z20Vwc5Sx073A3Z8lWC31doJ7K2w1s+eADxD0heDuI8D/C4ew/om7P0LQPPWD8Nz7eX3SEFkwDUkViUjYHDTt7m5mtwG3u/utp3udSC2pT0EkOlcAd4cjhkap8u1ORc6GagoiIlKiPgURESlRUhARkRIlBRERKVFSEBGREiUFEREpUVIQEZGS/w+4qanMFkogAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 07:26 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>4.164592</th>\n",
       "    <th>4.145639</th>\n",
       "    <th>0.235074</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>3.985004</th>\n",
       "    <th>3.965798</th>\n",
       "    <th>0.258884</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>3.813833</th>\n",
       "    <th>3.862134</th>\n",
       "    <th>0.268512</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>3.683870</th>\n",
       "    <th>3.809419</th>\n",
       "    <th>0.272961</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>5</th>\n",
       "    <th>3.519063</th>\n",
       "    <th>3.764102</th>\n",
       "    <th>0.279018</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>6</th>\n",
       "    <th>3.378885</th>\n",
       "    <th>3.740636</th>\n",
       "    <th>0.281429</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>7</th>\n",
       "    <th>3.258823</th>\n",
       "    <th>3.758763</th>\n",
       "    <th>0.281190</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>8</th>\n",
       "    <th>3.164661</th>\n",
       "    <th>3.759633</th>\n",
       "    <th>0.280997</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>9</th>\n",
       "    <th>3.110600</th>\n",
       "    <th>3.772064</th>\n",
       "    <th>0.281116</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>10</th>\n",
       "    <th>3.072069</th>\n",
       "    <th>3.777004</th>\n",
       "    <th>0.280923</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(10, 1e-3, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('twitter-sentiment-lm')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing our language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = \"I liked \"\n",
    "N_WORDS = 40\n",
    "N_SENTENCES = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I liked  was a piece of paper , and it was very difficult to locate if it was correct . xxbos @united , first time flying with you , then u like my next flight .. i always fly with you guys\n",
      "I liked  the organization for public access . xxup ua xxbos @southwestair xxmaj random promo code , the most annoying words form - words are always used when you write a letter that does nt answer the phone . xxbos @united xxmaj\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n\".join(learn.predict(TEXT, N_WORDS, temperature=0.75) for _ in range(N_SENTENCES)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder('twitter-sentiment-enc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = text_classifier_learner(data_clas, pretrained_model=URLs.WT103, drop_mult=0.3)\n",
    "learn.load_encoder('twitter-sentiment-enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt81PWV//HXSSYXEnIBEu73i4BXxIhavGDtxbrbqr1o6cWqbd3e7O62tbvb3V/trrttXXtz11pKXbT2Ylu1brVra6kKWBUxqCAqd0SChIRbSAKZycyc3x8zGQImIUC+c0nez8djHs58v5+Z75mRzJnP3dwdERERgLxMByAiItlDSUFERFKUFEREJEVJQUREUpQUREQkRUlBRERSlBRERCRFSUFERFKUFEREJCWU6QCOVVVVlU+cODHTYYiI5JSVK1fucvfqo5XLuaQwceJEamtrMx2GiEhOMbOtvSmn5iMREUlRUhARkRQlBRERSVFSEBGRFCUFERFJUVIQEZEUJQUREUlRUhARyQE/+PN6lq1vDPw6SgoiIjngjic2snzz7sCvo6QgIpLlYnEnGneKQvmBX0tJQUQky0WicQAKQ8F/ZSspiIhkuXA0BkCRkoKIiISTNYWiAiUFEZEBr6P5KKf7FMxskZk1mNmabs7fZGYvJW9rzCxmZkODikdEJFd1NB/lep/CPcCl3Z1099vcfZa7zwL+CVjq7nsCjEdEJCe1tXfUFHI4Kbj7MqC3X/LzgfuCikVEJJdFYv0gKfSWmZWQqFE8mOlYRESyUbi9H/QpHIP3Ak/31HRkZjeYWa2Z1TY2Bj/NW0Qkm/SXPoXe+jBHaTpy94XuXuPuNdXVR913WkSkX0kNSe3vScHMKoCLgN9lMg4RkWzWMSS1OA3zFEJBvbCZ3QfMA6rMrA64GSgAcPcFyWJXAn9y99ag4hARyXUdNYXC/OD7FAJLCu4+vxdl7iExdFVERLqRWuZCM5pFRCTcH+YpiIhI3zg0T2FgDEkVEZEedNQUBsqQVBER6UE4GiOUZ+TnWeDXUlIQEclykWg8Lf0JoKQgIpL1wtE4RQXB9yeAkoKISNYLR2MU5qumICIidNQUlBRERAT1KYiISCfhaDwtw1FBSUFEJOuFo7G0TFwDJQURkawXblfzkYiIJEViSgoiIpIUblefgoiIJKlPQUREUsIakioiIh0imrwmIiIdwtF4WrbihACTgpktMrMGM1vTQ5l5ZvaSmb1iZkuDikVEJJeFo7F+UVO4B7i0u5NmVgncCbzP3U8BPhRgLCIiOSked9pjnvt9Cu6+DNjTQ5GPAL919zeS5RuCikVEJFd1bMU5EIakngQMMbMlZrbSzK7prqCZ3WBmtWZW29jYmMYQRUQyq2MrzoEwJDUEnAX8FfBu4P+Z2UldFXT3he5e4+411dXV6YxRRCSjwtEYQNqaj0JpuUrX6oDd7t4KtJrZMuAMYH0GYxIRySrhaEdNof83H/0OON/MQmZWApwDvJbBeEREsk5HUkhXn0JgNQUzuw+YB1SZWR1wM1AA4O4L3P01M/sjsBqIA3e5e7fDV0VEBqJDzUfp6VMILCm4+/xelLkNuC2oGEREcl2q+agfzFMQEZETFBlAfQoiInIUA6mjWUREjiLcnt4+BSUFEZEs1jGjWTUFERFJzWgeCMtciIjIURzqU1DzkYjIgJfuZS6UFEREslhE8xRERKRDapmLfCUFEZEBLxyNkZ9nhJQUREQk3B5PW38CKCmIiGS1SExJQUREksLt8bTNUQAlBRGRrBaOxtI2RwGUFEREslo4quYjERFJikTVfCQiIkn9pqZgZovMrMHMutxi08zmmVmTmb2UvH09qFhERHJVuvsUAtuOE7gHuAO4t4cyT7n7XwcYg4hITotE45SWBvlVfbjAagruvgzYE9Tri4gMBOFoPG1LXEDm+xTOM7NVZvYHMzslw7GIiGSdcDROUUH/aD46mheACe7eYmaXAf8LTOuqoJndANwAMH78+PRFKCKSYeH2WP/oaD4ad9/v7i3J+48CBWZW1U3Zhe5e4+411dXVaY1TRCSTBswyF2Y20swseX9OMpbdmYpHRCQbpXuZi8Caj8zsPmAeUGVmdcDNQAGAuy8APgh81syiwEHgw+7uQcUjIpKLEvMU+kGfgrvPP8r5O0gMWRURkS7E4z5wmo9ERKRnkVhy1zUlBRER6diKUzUFEREhHI0BpHWegpKCiEiWCrerpiAiIkkdfQpKCiIiopqCiIgckupT0HacIiIS0egjERHp0DEkVfMURESk0zwFNR+JiAx4h+YpqKYgIjLgdfQpDKSd10REpBup5iPVFEREJNyuIakiIpKkBfFERCQloiGpIiLSIRyNk2cQyrO0XTOwpGBmi8yswczWHKXc2WYWNbMPBhWLiEguCkdjFIXySW5nnxZB1hTuAS7tqYCZ5QO3An8KMA4RkZwUicbTOvIIAkwK7r4M2HOUYjcCDwINQcUhIpKrwtF4WucoQAb7FMxsDHAl8KNMxSAiks3C/amm0As/AP7B3eNHK2hmN5hZrZnVNjY2piE0EZHM6+hTSKdeJQUzm2JmRcn788zsi2ZWeYLXrgF+ZWavAx8E7jSzK7oq6O4L3b3G3Wuqq6tP8LIiIrkhksXNRw8CMTObCiwExgG/PJELu/skd5/o7hOBB4DPufv/nshrioj0J5loPgr1slzc3aNmdiXw3+7+32b2Yk9PMLP7gHlAlZnVATcDBQDuvuAEYhYRGRDC7fG0zmaG3ieFdjObD3wCeG/yWEFPT3D3+b0Nwt2v7W1ZEZGBIhyNUVlSmNZr9jYFXQecB/yHu28xs0nAz4ILS0REwtF4Wpe4gF7WFNz9VeCLAGY2BChz91uDDExEZKCLRNPffNTb0UdLzKzczIYCLwA/MbPvBRuaiMjAFo7Gs3NIKlDh7vuB9wP3uvs5wDuCC0tERMLRWNZOXguZ2SjgKuD3AcYjIiJJ2bzMxb8BjwGb3P15M5sMbAguLBERydp5Cu5+P3B/p8ebgQ8EFZSIyEDn7smO5izsUzCzsWb2UHJ/hAYze9DMxgYdnIjIQBWJpX8rTuh989HdwMPA6OTtkeQxEREJQCb2Z4beJ4Vqd7/b3aPJ2z2AVqYTEQlIuD27k8JuM/uYmeUnbx8DdgcZmIjIQBaOxgCys08BuJ7EcNR6YAeJpa6vDSgmEZEBL5JsPkr3Mhe9upq7b3X397l7tbsPd/cr0OgjEZHAZHufQle+1GdRiIjIYVJJIUtnNHfF+iwKERE5TLg9u/sUuuJ9FoWIiBymY55CVi2dbWbNdP3lb8CgQCISEZGMDUntMSm4e1m6AhERkUMOdTTnTvNRj8xsUXJJjDXdnL/czFab2UtmVmtm5wcVi4hIronEEn0KWTkk9TjdA1zaw/nHgTPcfRaJeRB3BRiLiEhOyfYZzcfM3ZcBe3o43+LuHf0VpajjWkQkJRfnKZwwM7vSzNYC/0eittBduRuSTUy1jY2N6QtQRCRDUstcFPSTPoXecPeH3H0GcAVwSw/lFrp7jbvXVFdrHT4R6f9Sy1xk6c5rgUo2NU02s6pMxyIikg3C0ThmUJCf3nnCGUsKZjbVzCx5fzZQhFZeFREBkltxhvJIfk2mTa+24zweZnYfMA+oMrM64GagAMDdF5BYUO8aM2sHDgJXd+p4FhEZ0MLtsbTPUYAAk4K7zz/K+VuBW4O6vohILovE4mmfowBZ0qcgIiKHC7fH0z4cFZQURESyUkefQropKYiIZKFwNEZhBvoUlBRERLKQagoiIpKipCAiIinhaDztS1yAkoKISFYKt8fSvsQFKCmIiGSl3a0RhpUWpv26SgoiIlkmEo2zqyXMyIritF9bSUFEJMs0NLfhDqOUFEREpL6pDUA1BRERgR3JpDCqYlDar62kICKSZVRTEBGRlB1NbZQU5lNeHNhC1t1SUhARyTL1+w8ysqI47RvsgJKCiEjW2dHUlpGRR6CkICKSdeqb2hhZnv5OZggwKZjZIjNrMLM13Zz/qJmtNrOXzewZMzsjqFhERHJFNBanoTnM6Mr+V1O4B7i0h/NbgIvc/TTgFmBhgLGIiOSEXS0RYnHPyMgjCHaP5mVmNrGH8890ergcGBtULCIiuWJH00EgM7OZIXv6FD4J/CHTQYiIZFrHxLVM9SmkfxDsEczsYhJJ4fweytwA3AAwfvz4NEUmIpJ+h2YzD8CagpmdDtwFXO7uu7sr5+4L3b3G3Wuqq6vTF6CISJrVNx2kKJRHZUlBRq6fsZqCmY0Hfgt83N3XZyqOXNfYHGb7voNUlxUxvKyIggxsyiEifadjjkImJq5BgEnBzO4D5gFVZlYH3AwUALj7AuDrwDDgzuSbj7p7TVDx5JJ43DHjsH8Ube0xNje2sqGhmfU7m3nlzf28+uZ+GprDqTJmUDW4iKElhXT+91SQn0dpUT6Di0KUFoUYVJBPUSiPwlAeRaF8KksKqBpcRNXgIoYNLqS4IJ9QnlGQn8eggnwqMvSLRWQgqm9qy9jIIwh29NH8o5z/FPCpoK6fTdydpoPtbN93kDf3tfHmvoO4O9NHljNjZBlDSgtpDUdZsq6Rx16p58m1DbREogwqyKekMJ+C/Dx27m8j7onXy88zpg0fzPnTqjh5VDnjh5awpzVC/f426pva2Hsgctj1I9E4reEYb+5rozUSpa09RjgaJ9wepy0aw73n+KePKOPiGcO5ZOZwTh1dwabGFtbWN/Pajv3U728jz4w8gzwzRlUUM2/6cGaPrySkWovIMdvR1MacSUMzdv2MdzSnywtv7OXup19n34EIe1oj7DvQTlEoj+vPn8RVNeMoDPXdF9ie1giLX63ntR3NrKtvZt3OZva0RrotP6K8iL0H2olE4wwtLeQ9p41kZHkxByIxDrbHaGuPM6aymGkjyjhpRBkTq0ooCvXNht7uzv6DURpbwuxqCbO7JUIkFiMac6JxZ++BCE+t38VdT21mwdJNhz23KJTHmMrECIm4OzF3duxr484lmygvDnHhSdVMrh5MKM/IzzPyzGhua2fvgQi7WyI0t0WZVF3KrHGVnDmukinVg8nLO7zKHI87tVv38tCL29nU2MKw0kKGDS6kanARk6pKmTNpaEaWFxYJQjzu7NzfT2sK2Wb/wXZerttHZUkhI8qLmT6yjC27WvmX/13Dj5Zs4vMXT+WDZ409oeSwrr6Zu5/ewkMvbiccjVNSmM9JI8p458wRTB0+mLFDBjFmyCBGVw4iHnfW1jeztn4/a+ubqRxUyLtPGUHNxKHk56WvLdHMqCgpoKKkgKnDB3dZ5nPzprK/rZ2/bNjF+p3NTB0+mBkjy5lUVfqWWDvKPbm2gSXrG/n96h2Hnc/PM4aUFDKstJCSonweeelNfvncGwCUFuYzsaqUiVWlTK4qJRZ3Hl71JnV7DzKoIJ9TRpezoaGF5ZvD7D3QnnrNcUMHcc6kYUwcVkJZcQGDi0KUDypg9vhKhg0u6uNPTCQ4u1rDROOesZFHAOZHazvIMjU1NV5bW9snr+XuLNuwi+8vXs9L2/YxoryIq2vGcdXZ4xg7pASArbtb+fXz23joxe3k5xmnj63g9LGVnDq6ggORKNv3HaRu70HWbG/iuS17KC7I4/2zx/LxcycwfUTZW375DjTuTtwhFndicae4IO+wvpJ43Nm8q5WXtu1jzfYmtuxq5fXdrWzbcwCAuVOruPLMMbz7lJGUFh36DdMei7OuvpkVW/awYssenn99D7uPqI3l5xlvmzKM954xmnefMpKKQeobkey2um4f77vjaX5yTQ3vPHlEn762ma3sTb/tgE4KHdydpesbueeZ11m6vhGAC6dVE43HeXrjbvIMLp4+nOLCfFbX7WPbnoOHPX9QQT4ThpXwvlmjmX/2eIaUFvZpfANRJJro7ygv7v0XeSQap7mtnZZwlF0tEZ5Yu5NHVu3gjT0HKMg3Th5dwZnjKjlzfCUzR5XTGo6y90CEPa3txOJxplQPZtqIMiUPyZjHXqnnb362kt/feD6njqno09dWUjhOdXsP8JvaOu6v3UZ+nnFVzTg+VDP2sHbrPa0R1u7Yz+DiEGOHlDCkpCBjw8ekZ+7O6rom/vhKPS9s3cvquiYOtsd6fM6I8iKGlxWnOuTb2mOUDypg+sgyZowoY/rIMvLM2N0aZldLhKaD7Vw8fTjnTRmWpncl/dVPn3mdmx9+hdp/eQdVfdz0qaQg0oVoLM7a+mY2NbZQXlxAZUkBQ0sLMYyNjc2s39nC+uTAgOJQPsUFeRQX5LOrJcK6nfvfUksECOUZ0bhz7uSh/N07TuLcyUoOcny+/Ye1LPrLFtbecmmfNz33NikMmI5mEYBQfh6njqnosmo+flgJb5/RcztuSzjKhp3N5JlRVVbEsGRT4X0r3uDOJZv48MLlzJk4lFPGlDOstJChpUUMKSmgqCCPwvx8CkN5DC0tZEp1qWqX8hb1TQcZUVGU0b5IJQWRYzC4KMSZ44e85fh1cycxf854fvncG/xyxRvcX1tHSzja7euMKC/igmnVXHhSNRdNq9YEQQGSs5kztBBeByUFkT5SXJDP9edP4vrzJwGJWegdc2IisTiRaOJWt/cAT23cxeJXd/LAyjpKCvO5bu5EPn3BZCpLNEhhIKvf38YZYyszGoOSgkhAigvyGV2ZmJdypA/PGU8s7qyq28fdT7/OnUs2ce8zW/nkBYmkciyjrqR/cHd2NLVx6SmZm6MASgoiGZOfZ8weP4TZ44fw+Yun8P3F6/nBnzfwk2WbuXL2GD5+7kSmjyzLdJiSJntaI0Si8YzOZgYlBZGsMGNkOT/+eA1rtjdxzzOv85vaOn6+/A3mTBrK31w4mbfPGK6O6X4u0/sodNCKZSJZ5NQxFXznQ2ew/J8u4R/fM4Ptew/yyZ/WcuWdz7BsfSO5NoRceq++Y8e1DK/lpaQgkoWGlhbymYumsOSmeXzr/afRsL+Naxat4KofP8sLb+zNdHgSgB37E0lhtGoKItKdgvw85s8Zz5M3zePfLj+F13cf4P13PsOXfv0SO5NfItI/1DcdJJRnGV/EUX0KIjmgKJTPNedN5AOzx/LDJzdy11Nb+OMr9Xz+4ql86oJJfbaUuqTP717azq1/WEtBKI+iUB6NzWFGlBendZXkrqimIJJDSotCfPXSGSz+0oWcP7WK2x5bx2W3P8WKLXsyHZoco1889wbRuDNrXCWTqwYza1wl182dmOmwVFMQyUUThpWy8JoanlzXwL88tIarfvws8+eM4x8vnanZ0Tlg34EIK7fu5bMXTeEr756e6XAOE1hNwcwWmVmDma3p5vwMM3vWzMJm9pWg4hDpzy6ePpzFX7qQGy6czG9q63jn95eyqbEl02HJUSxd30gs7rx95vBMh/IWQTYf3QNc2sP5PcAXge8EGINIv1dSGOJrl83kd5+fSyzufGLRChrUCZ3VnljbwLDSQmZleEmLrgSWFNx9GYkv/u7ON7j780B7d2VEpPdOHVPB3dedzZ7WCJ+4+3n2t+lPKxtFY3GWrGvk4hnDs3JnxpzoaDazG8ys1sxqGxsbMx2OSNY6fWwlCz52Fht2NvM3964kHO15Q6EO7bE4j768g6c2NHIw0rvnyPFZuXUvTQfbuWRG9jUdQY50NLv7QmAhJDbZyXA4IlntwpOque1Dp/P3v17FDfeu5LPzpjBn4tAuf5XG4s7vXtrOD/68gTeS+2IX5ucxe0Ilc6dU8bapVZwxtoJQfk78fswJT6xtoCDfOH9aVaZD6VJOJAUROTZXnjmWpgPt3PrHdSxd38iI8iL+6rTRnDGugvaYE4nGORCJ8uvnt7GhoYWTR5Xzk2tqCOUbz27azdMbd/G9P6/nu4vXM7goxDmThnLelGGcO3kYM0eVZ3wsfS57fG0D50waRlmWroSrpCDST107dxJXnT2Ox19r4JFVb/Lz5VtZ9HT8sDJTqkv54Udm855TR6ZqEhdPTzRr7G2N8OzmRIJ4ZtNuHl/bAEBZcYizJw5l7tQqrqoZm7Vfbtlo6+5WNja08JE54zMdSrcC26PZzO4D5gFVwE7gZqAAwN0XmNlIoBYoB+JAC3Cyu+/v6XW1R7PI8dnf1s7OpjYKQ3kUhRJbgw4pKej16qs7mg6yYsselm/ew3NbdrO5sZWKQQV88vxJXDt3ovaA6IW7n97Cvz7yKktvmseEYaVpvXZv92gOLCkERUlBJDusrtvHfz2+kT+/tpOy4hDXz53E9XMnafJcDz7+P8/x5r6DPP7leWm/dm+TgnqPROS4nD62krs+UcPvbzyf8yYP4/bHN3D+rU/wncfWsbc1kunwsk5LOMryzbu5ZOaITIfSI/UpiMgJOXVMBQuvqeG1Hfu544mN/HDJRu5+egufuWgKn7t46oDvlG4NR/nza4n9uNtjnrVDUTsoKYhIn5g5qpwffnQ263c28/3FiZFLz2/dy+1Xz2JIaWGmw0u7va0Rvv7wKyx+tZ629jgjyov44tuncvbEoZkOrUfqUxCRPufu3LdiG994+BWqy4pY8LGzOG1sRabDSptoLM61dz/Pii17uPrscbz3jNHUTBiS0RnM6lMQkYwxMz5yznju/8x5uDsfWPAMD66sy3RYfW5jQwu7WsJvOX7bn9bxl427+PcrTuWWK05lzqSuJw9mIyUFEQnMGeMq+f0XL6BmwhC+fP8qfvjkxn6zz3Q0FufKO5/m7d9ZwoMr61Lv6/9W7+DHSzfz0XPGc9XZ4zIc5bFTUhCRQA0tLeSe6+ZwxazR3PbYOr7x8CvE4ocSQ0s4yoote3JuzaX1O1tobotSXJDPl+9fxad+WstTGxq56YFVzB5fyc3vPSXTIR4XdTSLSOAKQ3l876pZDC8vZuGyzTQ0hzl9bCVL1zdQ+/peonFnaGkh175tItecN4HKkuzvmF5dtw+A+244lyfXNnDbY+t4fG0D1WVF/OhjZ1EYys3f3EoKIpIWeXnG1y6byfCyIv79/17jD2vqmTmqnE9dMJlTRpfz0Ivb+d7i9SxYuomPzBnPjZdMo2JQ9k6EW1XXRHlxiMlVpUy5YDKXzBzBfz+xgWvOm8iI8uJMh3fcNPpIRNJuc2MLpUWht3x5rq3fz4+XbuZ3L21neFkx3/7Aacybnp3j+i+7/SmGDS7kZ588J9Oh9IpGH4lI1ppcPbjLX9MzRpbz/atn8dDn5lJWHOLau5/nqw+syroNg9raY6zb2czp/XCYrZKCiGSdM8ZV8siN5/PZeVN4YGUd7/reMha/ujPTYaW88uZ+YnHn9CzcTvNEKSmISFYqLsjnHy6dwW8/N5eKQQV8+t5aPv+LF2hozvz+06u2JTqZZ41TUhARSatZyVrDV951Eotf28k7vruUXz73xmHDWtNtdd0+RpQX5XSHcnc0+khEsl5hKI8vvH0al502in/67ct87aGX+eWKrXzjvadQk1xLKByN8ejLO/j189soDOVz7uShnDd5GKeN6fvtRFfXNfXLpiNQUhCRHDK5ejC/uuFcHl71Jt96dC0fXPAsV8wazdghJdy34g12t0aYVFVKQb7xn39cB0BpYT6XnzmGz1w4hfHDSk44hqaD7Wze1coHzhp7wq+VjZQURCSnmBmXzxrDO08ewZ1PbmLhU5tpj8W5ZMYIPvG2CcydUkVenrGrJcyKLXt4cm0DD9TW8evnt/He00fx2XlTmT6y7Liv/3JdE0C/HHkEwW7HuQj4a6DB3U/t4rwBtwOXAQeAa939haO9ruYpiEhnDc1txOLOqIpB3ZbZub+N//nLFn6+fCsHIjFmjCzjounVXHRSNTUThh7T7OMfPrmR2x5bx6qvvyundpnr7TyFIGsK9wB3APd2c/49wLTk7RzgR8n/ioj02vCyo3f2jigv5muXzeSzFyWGuD6xtoFFf9nCj5dupqQwn/MmD+PCk6q5YFoVk6pKe9y3enXdPiYOK8mphHAsAksK7r7MzCb2UORy4F5PVFWWm1mlmY1y9x1BxSQiA9uQ0kI+feFkPn3hZFrCUZ7dtJul6xt4asMuHl/bAMDU4YP55pWnMWdS15vhrNrWxDmTs3ujnBORyT6FMcC2To/rkseUFEQkcIOLQrzz5BG88+TEnslbd7eybMMufrJsM1cvfJbr507ipndPp7ggP/Wchv1t1O9v67cjjyBH5imY2Q1mVmtmtY2NjZkOR0T6oQnDSvn4uRP4w99ewMfOmcD//GULl93+FM9u2p3aK2FVspP5jH7ayQyZrSlsBzrvQDE2eewt3H0hsBASHc3BhyYiA1VpUYhbrjiVS08dyVcfWM38nyxn6vDBXFUzljf2HCA/zzhltJJCEB4GvmBmvyLRwdyk/gQRyRZzp1ax+EsX8siqN/n189v45qNrAZg5qpxBhflHeXbuCiwpmNl9wDygyszqgJuBAgB3XwA8SmI46kYSQ1KvCyoWEZHjUVIY4uqzx3P12ePZ2NDMb1/Y3i/XO+pM+ymIiAwA2k9BRESOmZKCiIikKCmIiEiKkoKIiKQoKYiISIqSgoiIpCgpiIhIipKCiIik5NzkNTNrAjZ0caoCaOrh2JHnOx53VaYK2HWcIXYVR2/OHy3+Ix93dV/xZ0f8cPzv4Wjx91Smp3iPfNwf4+98Pxvi7ynOzo/T9R00wd2rj/psd8+pG7Cwt8c7HzvyfMfjrsoAtX0d34nG39P7OfK9KP7Mxn8i7+Fo8R/Lexho8ffFv6G+jL+nOHv43AP/GzjaLRebjx45huOP9HD+kV6UOR5He43jjf/Ix13dV/z9P/6eyvQU75GP+2P8vb1+T/oy/iOPZct3UI9yrvkoHcys1nuxRki2UvyZl+vvQfFnVibjz8WaQjoszHQAJ0jxZ16uvwfFn1kZi181BRERSVFNQUREUvp9UjCzRWbWYGZrjuO5Z5nZy2a20cz+y8ys07kbzWytmb1iZv/Zt1EfFkOfx29m3zCz7Wb2UvJ2Wd9HnoohkM8/ef7LZuZmVtV3Eb8lhiA+/1vMbHXys/+TmY3u+8hTMQQR/23Jf/urzewhMwt015mA3sOHkn+7cTPr87b7E4m5m9f7hJltSN4+0el4j38jx+VEhi7lwg24EJgNrDmO5674R4egAAAGcklEQVQAzgUM+APwnuTxi4E/A0XJx8NzLP5vAF/J1c8/eW4c8BiwFajKpfiB8k5lvggsyLH43wWEkvdvBW7NtX9DwExgOrAEqMmWmJPxTDzi2FBgc/K/Q5L3h/T0/k7k1u9rCu6+DNjT+ZiZTTGzP5rZSjN7ysxmHPk8MxtF4o93uSc+/XuBK5KnPwt8293DyWs05Fj8aRNg/N8HvgoE2ikWRPzuvr9T0VICfA8Bxf8nd48miy4HxgYVf4Dv4TV3X5dtMXfj3cBid9/j7nuBxcClQf2N9/uk0I2FwI3ufhbwFeDOLsqMAeo6Pa5LHgM4CbjAzJ4zs6Vmdnag0b7VicYP8IVk9X+RmQ0JLtQunVD8ZnY5sN3dVwUdaDdO+PM3s/8ws23AR4GvBxhrV/ri30+H60n8Qk23vnwP6dKbmLsyBtjW6XHH+wjk/YVO9AVyjZkNBt4G3N+p+a3oGF8mRKIqdy5wNvAbM5uczNaB6qP4fwTcQuIX6i3Ad0n8cQfuROM3sxLgaySaMNKujz5/3P2fgX82s38CvgDc3GdB9qCv4k++1j8DUeAXfRNdr6/bZ+8hXXqK2cyuA/42eWwq8KiZRYAt7n5lumMdcEmBRO1on7vP6nzQzPKBlcmHD5P44uxcLR4LbE/erwN+m0wCK8wsTmKtksYgA0864fjdfWen5/0E+H2QAR/hROOfAkwCViX/uMYCL5jZHHevDzh26Jt/P539AniUNCUF+ih+M7sW+GvgknT8GDpCX/8/SIcuYwZw97uBuwHMbAlwrbu/3qnIdmBep8djSfQ9bCeI99fXHSzZeAMm0qnDB3gG+FDyvgFndPO8IztxLkse/wzwb8n7J5Go2lkOxT+qU5m/B36VS5//EWVeJ8CO5oA+/2mdytwIPJBj8V8KvApUBxl3Ov4NEVBH8/HGTPcdzVtIdDIPSd4f2pv3d1xxp+t/aqZuwH3ADqCdxC/8T5L4pflHYFXyH/fXu3luDbAG2ATcwaHJfoXAz5PnXgDenmPx/wx4GVhN4hfVqFyK/4gyrxPs6KMgPv8Hk8dXk1inZkyOxb+RxA+hl5K3wEZPBfgerky+VhjYCTyWDTHTRVJIHr8++blvBK47lr+RY71pRrOIiKQM1NFHIiLSBSUFERFJUVIQEZEUJQUREUlRUhARkRQlBekXzKwlzde7y8xO7qPXillixdQ1ZvbI0VYdNbNKM/tcX1xb5Egakir9gpm1uPvgPny9kB9a9C1QnWM3s58C6939P3ooPxH4vbufmo74ZGBRTUH6LTOrNrMHzez55G1u8vgcM3vWzF40s2fMbHry+LVm9rCZPQE8bmbzzGyJmT1gif0DftGxXn3yeE3yfktygbtVZrbczEYkj09JPn7ZzP69l7WZZzm08N9gM3vczF5IvsblyTLfBqYkaxe3JcvelHyPq83sX/vwY5QBRklB+rPbge+7+9nAB4C7ksfXAhe4+5kkVij9ZqfnzAY+6O4XJR+fCfwdcDIwGZjbxXVKgeXufgawDPh0p+vf7u6ncfhqll1Krt1zCYlZ5gBtwJXuPpvEHh7fTSalfwQ2ufssd7/JzN4FTAPmALOAs8zswqNdT6QrA3FBPBk43gGc3GlVyvLkapUVwE/NbBqJlWILOj1nsbt3Xgd/hbvXAZjZSyTWs/nLEdeJcGhRwZXAO5P3z+PQ+va/BL7TTZyDkq89BniNxHr5kFjP5pvJL/h48vyILp7/ruTtxeTjwSSSxLJurifSLSUF6c/ygHPdva3zQTO7A3jS3a9Mts8v6XS69YjXCHe6H6Prv5l2P9Q5112Znhx091nJZcEfAz4P/BeJvRaqgbPcvd3MXgeKu3i+Ad9y9x8f43VF3kLNR9Kf/YnEKqQAmFnHssUVHFpi+NoAr7+cRLMVwIePVtjdD5DYnvPLZhYiEWdDMiFcDExIFm0Gyjo99THg+mQtCDMbY2bD++g9yACjpCD9RYmZ1XW6fYnEF2xNsvP1VRJLngP8J/AtM3uRYGvLfwd8ycxWk9g8peloT3D3F0msnjqfxF4LNWb2MnANib4Q3H038HRyCOtt7v4nEs1TzybLPsDhSUOk1zQkVSQgyeagg+7uZvZhYL67X36054lkkvoURIJzFnBHcsTQPtK05anIiVBNQUREUtSnICIiKUoKIiKSoqQgIiIpSgoiIpKipCAiIilKCiIikvL/AQgJCtLrXr2SAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 00:45 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.657963</th>\n",
       "    <th>0.483424</th>\n",
       "    <th>0.806061</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 00:53 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.601170</th>\n",
       "    <th>0.430959</th>\n",
       "    <th>0.832197</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.freeze_to(-2)\n",
    "learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 01:18 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.549314</th>\n",
       "    <th>0.427016</th>\n",
       "    <th>0.838636</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.freeze_to(-3)\n",
    "learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 03:18 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.479139</th>\n",
       "    <th>0.382407</th>\n",
       "    <th>0.853788</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>0.433286</th>\n",
       "    <th>0.377737</th>\n",
       "    <th>0.850758</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RNNLearner(data=TextClasDataBunch;\n",
      "\n",
      "Train: LabelList\n",
      "y: CategoryList (12001 items)\n",
      "[Category neutral, Category positive, Category neutral, Category negative, Category negative]...\n",
      "Path: /media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset\n",
      "x: TextList (12001 items)\n",
      "[Text xxbos @virginamerica xxmaj what xxunk said ., Text xxbos @virginamerica plus you 've added commercials to the experience ... xxunk ., Text xxbos @virginamerica i did n't today ... xxmaj must mean i need to take another trip !, Text xxbos @virginamerica it 's really aggressive to xxunk xxunk \" entertainment \" in your guests ' faces & & they have little recourse, Text xxbos @virginamerica and it 's a really big bad thing about it]...\n",
      "Path: /media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset;\n",
      "\n",
      "Valid: LabelList\n",
      "y: CategoryList (2640 items)\n",
      "[Category negative, Category neutral, Category positive, Category positive, Category neutral]...\n",
      "Path: /media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset\n",
      "x: TextList (2640 items)\n",
      "[Text xxbos @americanair xxmaj you xxmaj cancelled xxmaj flight my flight and there ’s no way to rebook on the website or app ? i have to wait 35 minutes on hold ? # fail cc @delta, Text xxbos @americanair i thought all those planes were xxunk ? # xxup xxunk, Text xxbos xxmaj just got off the phone @americanair customer service . xxmaj only 8 minutes to get my issue xxunk . xxmaj you guys are awesome ., Text xxbos @americanair thanks for the info xxmaj is there a number i can call to speak to a person ? xxmaj it 's going to take an hour to type it out, Text xxbos @americanair we are off to xxmaj xxunk premium . xxmaj hoping this flight is better food , xxup tv now i know how to work it and service . xxmaj xxunk]...\n",
      "Path: /media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset;\n",
      "\n",
      "Test: None, model=SequentialRNN(\n",
      "  (0): MultiBatchRNNCore(\n",
      "    (encoder): Embedding(4184, 400, padding_idx=1)\n",
      "    (encoder_dp): EmbeddingDropout(\n",
      "      (emb): Embedding(4184, 400, padding_idx=1)\n",
      "    )\n",
      "    (rnns): ModuleList(\n",
      "      (0): WeightDropout(\n",
      "        (module): LSTM(400, 1150, batch_first=True)\n",
      "      )\n",
      "      (1): WeightDropout(\n",
      "        (module): LSTM(1150, 1150, batch_first=True)\n",
      "      )\n",
      "      (2): WeightDropout(\n",
      "        (module): LSTM(1150, 400, batch_first=True)\n",
      "      )\n",
      "    )\n",
      "    (input_dp): RNNDropout()\n",
      "    (hidden_dps): ModuleList(\n",
      "      (0): RNNDropout()\n",
      "      (1): RNNDropout()\n",
      "      (2): RNNDropout()\n",
      "    )\n",
      "  )\n",
      "  (1): PoolingLinearClassifier(\n",
      "    (layers): Sequential(\n",
      "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (1): Dropout(p=0.12)\n",
      "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
      "      (3): ReLU(inplace)\n",
      "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (5): Dropout(p=0.1)\n",
      "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
      "    )\n",
      "  )\n",
      "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f7add4d6510>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset'), model_dir='models', callback_fns=[<class 'fastai.basic_train.Recorder'>], callbacks=[RNNTrainer\n",
      "learn: RNNLearner(data=TextClasDataBunch;\n",
      "\n",
      "Train: LabelList\n",
      "y: CategoryList (12001 items)\n",
      "[Category neutral, Category positive, Category neutral, Category negative, Category negative]...\n",
      "Path: /media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset\n",
      "x: TextList (12001 items)\n",
      "[Text xxbos @virginamerica xxmaj what xxunk said ., Text xxbos @virginamerica plus you 've added commercials to the experience ... xxunk ., Text xxbos @virginamerica i did n't today ... xxmaj must mean i need to take another trip !, Text xxbos @virginamerica it 's really aggressive to xxunk xxunk \" entertainment \" in your guests ' faces & & they have little recourse, Text xxbos @virginamerica and it 's a really big bad thing about it]...\n",
      "Path: /media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset;\n",
      "\n",
      "Valid: LabelList\n",
      "y: CategoryList (2640 items)\n",
      "[Category negative, Category neutral, Category positive, Category positive, Category neutral]...\n",
      "Path: /media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset\n",
      "x: TextList (2640 items)\n",
      "[Text xxbos @americanair xxmaj you xxmaj cancelled xxmaj flight my flight and there ’s no way to rebook on the website or app ? i have to wait 35 minutes on hold ? # fail cc @delta, Text xxbos @americanair i thought all those planes were xxunk ? # xxup xxunk, Text xxbos xxmaj just got off the phone @americanair customer service . xxmaj only 8 minutes to get my issue xxunk . xxmaj you guys are awesome ., Text xxbos @americanair thanks for the info xxmaj is there a number i can call to speak to a person ? xxmaj it 's going to take an hour to type it out, Text xxbos @americanair we are off to xxmaj xxunk premium . xxmaj hoping this flight is better food , xxup tv now i know how to work it and service . xxmaj xxunk]...\n",
      "Path: /media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset;\n",
      "\n",
      "Test: None, model=SequentialRNN(\n",
      "  (0): MultiBatchRNNCore(\n",
      "    (encoder): Embedding(4184, 400, padding_idx=1)\n",
      "    (encoder_dp): EmbeddingDropout(\n",
      "      (emb): Embedding(4184, 400, padding_idx=1)\n",
      "    )\n",
      "    (rnns): ModuleList(\n",
      "      (0): WeightDropout(\n",
      "        (module): LSTM(400, 1150, batch_first=True)\n",
      "      )\n",
      "      (1): WeightDropout(\n",
      "        (module): LSTM(1150, 1150, batch_first=True)\n",
      "      )\n",
      "      (2): WeightDropout(\n",
      "        (module): LSTM(1150, 400, batch_first=True)\n",
      "      )\n",
      "    )\n",
      "    (input_dp): RNNDropout()\n",
      "    (hidden_dps): ModuleList(\n",
      "      (0): RNNDropout()\n",
      "      (1): RNNDropout()\n",
      "      (2): RNNDropout()\n",
      "    )\n",
      "  )\n",
      "  (1): PoolingLinearClassifier(\n",
      "    (layers): Sequential(\n",
      "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (1): Dropout(p=0.12)\n",
      "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
      "      (3): ReLU(inplace)\n",
      "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (5): Dropout(p=0.1)\n",
      "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
      "    )\n",
      "  )\n",
      "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f7add4d6510>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/media/gilbert/948A92E98A92C760/Local_Programming/Datasets/Other/Twitter US Airline Sentiment Dataset'), model_dir='models', callback_fns=[<class 'fastai.basic_train.Recorder'>], callbacks=[...], layer_groups=[Sequential(\n",
      "  (0): Embedding(4184, 400, padding_idx=1)\n",
      "  (1): EmbeddingDropout(\n",
      "    (emb): Embedding(4184, 400, padding_idx=1)\n",
      "  )\n",
      "), Sequential(\n",
      "  (0): WeightDropout(\n",
      "    (module): LSTM(400, 1150, batch_first=True)\n",
      "  )\n",
      "  (1): RNNDropout()\n",
      "), Sequential(\n",
      "  (0): WeightDropout(\n",
      "    (module): LSTM(1150, 1150, batch_first=True)\n",
      "  )\n",
      "  (1): RNNDropout()\n",
      "), Sequential(\n",
      "  (0): WeightDropout(\n",
      "    (module): LSTM(1150, 400, batch_first=True)\n",
      "  )\n",
      "  (1): RNNDropout()\n",
      "), Sequential(\n",
      "  (0): PoolingLinearClassifier(\n",
      "    (layers): Sequential(\n",
      "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (1): Dropout(p=0.12)\n",
      "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
      "      (3): ReLU(inplace)\n",
      "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (5): Dropout(p=0.1)\n",
      "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
      "    )\n",
      "  )\n",
      ")])\n",
      "bptt: 70\n",
      "alpha: 2.0\n",
      "beta: 1.0], layer_groups=[Sequential(\n",
      "  (0): Embedding(4184, 400, padding_idx=1)\n",
      "  (1): EmbeddingDropout(\n",
      "    (emb): Embedding(4184, 400, padding_idx=1)\n",
      "  )\n",
      "), Sequential(\n",
      "  (0): WeightDropout(\n",
      "    (module): LSTM(400, 1150, batch_first=True)\n",
      "  )\n",
      "  (1): RNNDropout()\n",
      "), Sequential(\n",
      "  (0): WeightDropout(\n",
      "    (module): LSTM(1150, 1150, batch_first=True)\n",
      "  )\n",
      "  (1): RNNDropout()\n",
      "), Sequential(\n",
      "  (0): WeightDropout(\n",
      "    (module): LSTM(1150, 400, batch_first=True)\n",
      "  )\n",
      "  (1): RNNDropout()\n",
      "), Sequential(\n",
      "  (0): PoolingLinearClassifier(\n",
      "    (layers): Sequential(\n",
      "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (1): Dropout(p=0.12)\n",
      "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
      "      (3): ReLU(inplace)\n",
      "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (5): Dropout(p=0.1)\n",
      "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
      "    )\n",
      "  )\n",
      ")])\n"
     ]
    }
   ],
   "source": [
    "print(learn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Category positive, tensor(2), tensor([0.1067, 0.0125, 0.8808]))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.predict(\"I really loved the flight\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}