analysis-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RHgbIJZrwU0v"
   },
   "source": [
    "# 0. Preparation\n",
    "First, import related packages and load the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 187,
     "output_extras": [
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1963,
     "status": "ok",
     "timestamp": 1518147927711,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "pB4NLfEevnKz",
    "outputId": "8d24fc65-beba-45d6-b52d-0f3e1c5c8d8f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\ChuanLi\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.preprocessing import text as ktxt\n",
    "import nltk\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.stem.wordnet import WordNetLemmatizer \n",
    "nltk.download('wordnet')\n",
    "\n",
    "\n",
    "def hint(message):\n",
    "    \"\"\"\n",
    "    erase previous ipynb output and show new message\n",
    "    \"\"\"\n",
    "    clear_output()\n",
    "    print(message)\n",
    "    \n",
    "\n",
    "np.random.seed()\n",
    "\n",
    "train = pd.read_csv('data/train.csv')\n",
    "test = pd.read_csv('data/test.csv')\n",
    "\n",
    "train['comment_text'].fillna(\"unknown\", inplace=True)\n",
    "test['comment_text'].fillna(\"unknown\", inplace=True)\n",
    "\n",
    "labels = [\n",
    "    'toxic', \n",
    "    'severe_toxic', \n",
    "    'obscene', \n",
    "    'threat', \n",
    "    'insult', \n",
    "    'identity_hate'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Overview\n",
    "A quick glance at the training data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 408,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 261,
     "status": "ok",
     "timestamp": 1518147946911,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "TmH8WkF_ByCe",
    "outputId": "ce7b18ae-4187-492b-dcc1-a252d1791662"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set size:\t159571\n",
      "Test set size:\t153164\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000997932d777bf</td>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000103f0d9cfb60f</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000113f07ec002fd</td>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0001b41b1c6bb37e</td>\n",
       "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0001d958c54c6e35</td>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  toxic  \\\n",
       "0  0000997932d777bf  Explanation\\nWhy the edits made under my usern...      0   \n",
       "1  000103f0d9cfb60f  D'aww! He matches this background colour I'm s...      0   \n",
       "2  000113f07ec002fd  Hey man, I'm really not trying to edit war. It...      0   \n",
       "3  0001b41b1c6bb37e  \"\\nMore\\nI can't make any real suggestions on ...      0   \n",
       "4  0001d958c54c6e35  You, sir, are my hero. Any chance you remember...      0   \n",
       "\n",
       "   severe_toxic  obscene  threat  insult  identity_hate  \n",
       "0             0        0       0       0              0  \n",
       "1             0        0       0       0              0  \n",
       "2             0        0       0       0              0  \n",
       "3             0        0       0       0              0  \n",
       "4             0        0       0       0              0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Train set size:\\t%d\\nTest set size:\\t%d' % (len(train), len(test)))\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NcAsK6goC5Y_"
   },
   "source": [
    "Here are a few random samples of the comments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 428,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 253,
     "status": "ok",
     "timestamp": 1518035202721,
     "user": {
      "displayName": "Hongyu Zheng",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104216196693750147602"
     },
     "user_tz": 480
    },
    "id": "zHp7EGB-C5i4",
    "outputId": "f83c78ee-193e-4307-c6e8-6430014ffa90"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "----- Sample 1 start -----\n",
      "\n",
      "License to Wed\n",
      "\n",
      "John was actually the main star of License to Wed.  He was not a supporting character.\n",
      "\n",
      "------ Sample 1 end ------\n",
      "\n",
      "\n",
      "----- Sample 2 start -----\n",
      "\n",
      "Oh wait.. But what about an aircraft and fire? Empire state building had that happen.. oh look it still stands today!\n",
      "\n",
      "------ Sample 2 end ------\n",
      "\n",
      "\n",
      "----- Sample 3 start -----\n",
      "\n",
      "Whenever you have a chance, could you please take a look at this. I've made a number of changes per peer review but would still appreciate a second set of eyes on it. Thanks.\n",
      "\n",
      "------ Sample 3 end ------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "    show_count = 3\n",
    "    for i in range(1, show_count+1):\n",
    "        print('\\n----- Sample %d start -----\\n' % i)\n",
    "        print(train['comment_text'][np.random.randint(len(train))])\n",
    "        print('\\n------ Sample %d end ------\\n' % i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Gu3X2jZrDxtU"
   },
   "source": [
    "# 2. Original Data Statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_kr-j0oVFnKe"
   },
   "source": [
    "## 2.1 Distribution of the Labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 170,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 260,
     "status": "ok",
     "timestamp": 1517536980733,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "u7MssVClFvjf",
    "outputId": "4880bbc8-a9e3-4d41-e06a-7b95f2fe057c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label distribution: \n",
      "toxic            0.0958\n",
      "severe_toxic     0.0100\n",
      "obscene          0.0529\n",
      "threat           0.0030\n",
      "insult           0.0494\n",
      "identity_hate    0.0088\n",
      "none             0.8983\n",
      "Name: mean, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "train['none'] = 1 - train[labels].max(axis=1)\n",
    "print('label distribution: ')\n",
    "print(train.describe().loc['mean'].round(4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Vocabulary Size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set vocabulary size: 210337\n"
     ]
    }
   ],
   "source": [
    "tr_tkzr = ktxt.Tokenizer()\n",
    "tr_tkzr.fit_on_texts(train['comment_text'])\n",
    "print(\"Train set vocabulary size: %d\" % (len(tr_tkzr.word_index)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set vocabulary size: 273192\n"
     ]
    }
   ],
   "source": [
    "te_tkzr = ktxt.Tokenizer()\n",
    "te_tkzr.fit_on_texts(test['comment_text'])\n",
    "print(\"Test set vocabulary size: %d\" % (len(te_tkzr.word_index)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Token-Count Distribution\n",
    "One way to preprocess the data is to build a dictionary and map the words to their indices. The amount of tokens these comments would yield could be important when designing the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tMin\tMean\tMax\n",
      "Train\t1\t68\t1403\n",
      "Test\t0\t59\t2142\n"
     ]
    }
   ],
   "source": [
    "train_dirty_seq = tr_tkzr.texts_to_sequences(train['comment_text'])\n",
    "test_dirty_seq = tr_tkzr.texts_to_sequences(test['comment_text'])\n",
    "train_dirty_word_counts = [ len(seq) for seq in train_dirty_seq ]\n",
    "test_dirty_word_counts = [ len(seq) for seq in test_dirty_seq ]\n",
    "print(\"\\tMin\\tMean\\tMax\")\n",
    "print(\"Train\\t%d\\t%d\\t%d\" % (\n",
    "    np.min(train_dirty_word_counts), \n",
    "    np.mean(train_dirty_word_counts), \n",
    "    np.max(train_dirty_word_counts)\n",
    "))\n",
    "print(\"Test\\t%d\\t%d\\t%d\" % (\n",
    "    np.min(test_dirty_word_counts), \n",
    "    np.mean(test_dirty_word_counts), \n",
    "    np.max(test_dirty_word_counts)\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs8AAAE/CAYAAAC5CC4zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3X2cXVV97/HPrwQIkmAwatAEmoDYFrDGmGIQq6MiKLVFr4KpT1yEm/q6qJSW9mLvbYOgL/S2NdVitanEUqtEFL1ii9WIGauVZwjIQ4EUNIyEp0QeoiCE/O4fe81wMpnJrDnzcGYyn/frldfsvfY6e699zsqZ76yz9j6RmUiSJEka2q90ugGSJEnSZGF4liRJkioZniVJkqRKhmdJkiSpkuFZkiRJqmR4liRJkioZniWNqoh4fUSs73Q7JrKI2DMitkTE80dpfx+KiPPK8q9HxNbR2G/Z3wsj4qHR2t9UFxGfj4g/6nQ7JLXP8CxpByXY9f7bFhGPtay/o9Pt66SIuDciXrGT7a8vz1nv83V3RFwYES/prZOZv8zMGZl5zxDHqvpDJDOXZ+b7hncmgx5zu/PLzNszc9Zo7LuNtlwREY+X5/GBiLgoIp4zjsf/x5bX8YmIeLJl/Wvt7DMz35WZHx/ttkoaP4ZnSTsowW5GZs4ANgC/21L2hU63bxK4szx3+wAvB+4CfhgRvz3aB4qIaaO9zwnmlPJc/hrwXOCj43XgzPzvLf8PPg5c0PL/4M3960+B10IShmdJbYiIvSLiUxGxMSJ6IuIvI2L3Qer+SUTcGBH7lfU3l/WHIuL7EXFIS917I+L0iLgpIh6OiC9ExB47acf8iPh6RDxY/v11Kd+tTGXYEBH3RcSqiJhZtu0wmts62hoRHy3HvTAiHi1tXVi2fZkmwH27jD5+YGfPU2Zuy8y7M/PPgC8A55b9TI+IjIh5Zf24iPjPcry7I+IDETEb+BpwYMto5+zSvi9GxJci4lFgaSn7bL9zem95fe6JiPe3lK+OiP/Tst73fAx0fv2ngUTEARFxaURsjojbI+LElm2DPncjlZmbgUuAvv1FxJERcWXpK/dExIreABsRH4uIvyzLe0XELyPi7LK+TxnRnjGSNkXEYWU/742IHuCSiNgjIr5a+t1DEXFZRBzc8pivRMSZZfmN5XX/89J/eyJi6UjaJGnsGZ4lteNDwG8CLwJeCnQBf9q/UkR8BHgr0JWZ90bEEuDvgJOA2cDngf/Xb8TurcBrgRcALwPePlADSlj/JnArcACwP3Bx2fwHwAnAbwMH0wTC4XxU/mZgFTALuAz4G4DMPB64Hzi6jD5+chj7/CqwZJA/MlYB787MmTTh8PuZuam0486W0c5Npf5bgAuAZ/L0ObfaDTgCOBD4HeBDsZOpJr0qz+/LwG3A82hemxURcWTL9gGfu5GKZrrGm4DWP3yeBN4HPIvmtf5d4JSy7Xs0/RKa56IHeFVZfwVwQ2ZuGYWm7QG8hKafvQUImtf6IJrn6L+Az+3k8QeV89gPOB34+4h4xii0S9IYMTxLasc7gOWZ+WBm3gd8GHhXy/aIiE/RTFk4qowaQhNqz8vMazPzqcxcCexJE8B7rcjM+zLzAeBSWkYa+3kFzbSIP8vMX2TmY5n5w5b2/WVm/iQzHwH+N/COiIjK8/tuZq7JzKdoAv5ojJ7eQxNq9xlg21bg0IiYmZmbMvP6Ifb1vcy8tIxsPzZIneXlObke+Gfg99tveqOMoL6Y5jn/ZWZeQxPiW1/70X7u/j4iHqEJ9XvRBEwAMvOqzLy69KX/Aj7L0wH5B8BvRsQ+wCuBTwMvjIjppc73RtiuXgH8eXmuHyvPyz9n5pby2pwNHLGTT1AeBT6WmVsz88vA7jR/9EiaoAzPkoalBND9gJ+0FP8EmNuy/lya0eUPZ+ajLeW/CvxZ+Tj7oWju4vCcfo+9t2X5F8CMctzvtkxfeAvNSPNdmbltgGY+f4D27UUzQlljwDaM0FzgKeCRAba9iWbUckM5z8VD7OvuiuO11vkJzXMyUs8HHugX2Pu/9lXPXZlW0/t67mx0+g8ycx9gEU2/6zuPiDgkIr5Zpkg8AvwF8GyA8kfTj2j+yHolsBa4mubTjNEMz09k5v0tbdq9TB+5q7TpJprftfsO8vj7MzNb1kerv0kaI4ZnScNSftHfSxOEex0A/LRl/T6aQPjFiDi8pfxu4C8yc1bLv2dk5lcrjvualukLF5d9zY+Igd7H7hmgfY8Bm4GfA30fi5dpFLWhGiCHrjKgNwNXZOaTO+ww8/LMfCMwB/g2cOEQx6ppw/4tywfQPCfQ7/xpAmntvu8BnhMRe/Xb908HqT+ocoeQ3tfzDyvqXw/8X+C8luJ/AK4DDioB+2yakeBe3wNeB/wGsK6s/w7NaPgPhtvmwZrWb/0UmukiryxtOqyU137qIWmCMzxLaseFwPJyAdtzaaZF/HNrhcz8NvAe4Bvx9G3aVgLvj4jF0ZgREb/X5hzPH9B85H1ORDyjXBT28pb2nVEubptJM63kiyX43wo8KyJeW4Lzhxjee+F9VH6sXs5xXkScA7yT5nnqX2fviFhaphc8Wc7pqZZjPbfNC9uWl+fkxTTTKr5UytcBb4yIWRExF3h/v8ft7PzWAzcCH47mXtWLgBNpLoYcD58FDoqIY8r6TODhzNwSEYcC/6Nf/e8BJwPXlWkk3cB7gZsz8+HeStFcMDpaF+rNBB4HftbS9yTtQgzPktrxF8AtwM00Yew/aEYFt5OZ/0oTVr4ZEb+Zmf8BfAD4e+Ah4Haai86GPZpbRnCPpZmD20NzS73/VjZ/muairR/SXLC1Gfij8rgHgdNoAl8PzSj6g8M49EeAj5RpJ4PdW/nAiNgCbAGupLnN2isyc7CpAu+hmf7wMPBumkAKcAPNHSZ+Uo5XO0L+VDnuXcC/AWdn5r+XbatoQvAG4F94epR7yPMrf3ycABxC87x9CfiTzPx+ZbtGpEwXOQ/481J0OnBKea4/xdN/IPT6PrA30Hvu64BtLeuUP9xmAleNUjNX0vTte2lev9GaHiJpgojtp1pJkjR1RMRRwDsy86ROt0XS5GB4liRJkio5bUOSJEmqZHiWJEmSKhmeJUmSpEqGZ0mSJKnStE43YGee/exn5/z58zty7J///OfsvffeHTm2Ji/7jdphv1E77Ddqh/1mcNdee+2DmfmcoepN6PA8f/58rrnmmo4cu7u7m66uro4cW5OX/UbtsN+oHfYbtcN+M7iI+ElNPadtSJIkSZUMz5IkSVIlw7MkSZJUaULPeZYkSdLYefLJJ+np6eHxxx/vdFPGzfTp05k3bx677757W483PEuSJE1RPT09zJw5k/nz5xMRnW7OmMtMNm3aRE9PDwsWLGhrH07bkCRJmqIef/xxZs+ePSWCM0BEMHv27BGNtBueJUmSprCpEpx7jfR8Dc+SJEnqiE2bNrFw4UIWLlzIfvvtx9y5c/vWn3jiiap9nHTSSdx2221j3NKnOedZkiRJAKxYc/uo7u/0171wp9tnz57NunXrADjrrLOYMWMGZ5xxxnZ1MpPM5Fd+ZeAx38997nOj09hKjjxLkiRpQlm/fj2HHXYY733ve1m0aBEbN25k2bJlLF68mEMPPZSzzz67r+4rXvEK1q1bx9atW5k1axZnnnkmL37xizniiCO4//77R71thmdJkiRNOLfccgsnn3wy119/PXPnzuWjH/0o11xzDTfccANr1qzhlltu2eExDz/8MK961au44YYbOOKII1i1atWot8vwvDNrz+10CyRJkqakgw46iN/6rd/qW7/wwgtZtGgRixYt4tZbbx0wPO+111684Q1vAOClL30pP/7xj0e9Xc55liRJ0oSz99579y3fcccdfOITn+Cqq65i1qxZvPOd7xzwdnN77LFH3/Juu+3G1q1bR71djjxLkiRpQnvkkUeYOXMm++yzDxs3buRb3/pWx9riyLMkSZImtEWLFnHIIYdw2GGHceCBB3LkkUd2rC2GZ0mSJAFD31puLJ111ll9yy94wQv6bmEHzRebfP7znx/wcT/4wQ/6lh966KG+5aVLl7J06dJRb6fTNiRJkqRKhmdJkiSpkuFZkiRJqmR4liRJkioZniVJkqRKhmdJkiSpkuFZkiRJHbFp0yYWLlzIwoUL2W+//Zg7d27f+hNPPFG9n1WrVnHvvfeOYUuf5n2eJUmS1Fh77uju79Uf3Onm2bNn993P+ayzzmLGjBmcccYZwz7MqlWrWLRoEfvtt19bzRwOw7MkSZImnAsuuIBPfepTPPHEE7z85S/nvPPOY9u2bZx00kmsW7eOzGTZsmXMmTOHdevW8ba3vY299tqLq666ij322GPM2mV4liRJ0oRy00038bWvfY0f/vCHTJs2jWXLlrF69WoOOuggHnzwQX70ox8BzTcKzpo1i7/927/lvPPOY+HChWPeNsOzJEmSJpTvfOc7XH311SxevBiAxx57jP33359jjjmG2267jdNOO41jjz2Wo48+etzbZniWJEnShJKZvOc97+Gcc87ZYduNN97IN7/5TT75yU9y8cUXs3LlynFtm3fbkCRJ0oRy1FFHcdFFF/Hggw8CzV05NmzYwAMPPEBmcvzxx/OhD32I6667DoCZM2fy6KOPjkvbHHmWJEnShPKiF72I5cuXc9RRR7Ft2zZ23313PvOZz7Dbbrtx8sknk5lEBB/72McAOOmkkzjllFO8YFCSJEnjaIhby42ls846a7v1t7/97bz97W/fod7111+/Q9kJJ5zACSecMFZN247TNiRJkqRKhmdJkiSpkuFZkiRJqmR4liRJmsIys9NNGFcjPd+q8BwRp0fEzRFxU0RcGBHTI2JBRFwZEXdExJciYo9Sd8+yvr5sn9+ynw+W8tsi4pgRtVySJEkjMn36dDZt2jRlAnRmsmnTJqZPn972Poa820ZEzAU+ABySmY9FxEXAUuBYYEVmro6IzwAnA58uP3+WmS+IiKXAx4C3RcQh5XGHAs8HvhMRL8zMp9puvSRJkto2b948enp6eOCBBzrdlHEzffp05s2b1/bja29VNw3YKyKeBJ4BbAReA/TeP+QC4Cya8HxcWQb4CnBeREQpX52ZvwTuioj1wOHA5W23XpIkSW3bfffdWbBgQaebMakMOW0jM38K/BWwgSY0PwxcCzyUmVtLtR5gblmeC9xdHru11J/dWj7AYyRJkqQJr2baxr40o8YLgIeALwNvGKBq72SZGGTbYOX9j7cMWAYwZ84curu7h2rimNiyZQvduQA6dHxNTlu2bOlYn9XkZb9RO+w3aof9ZuRqpm0cBdyVmQ8ARMRXgZcDsyJiWhldngfcU+r3APsDPRExDXgmsLmlvFfrY/pk5kpgJcDixYuzq6urjdMaue7ubrryR9C1tCPH1+TU3d1Np/qsJi/7jdphv1E77DcjV3O3jQ3Akoh4Rpm7/FrgFmAt8NZS50Tg62X5krJO2f7dbC7hvARYWu7GsQA4GLhqdE5DkiRJGntDjjxn5pUR8RXgOmArcD3NyPC/Aqsj4sOl7PzykPOBz5cLAjfT3GGDzLy53KnjlrKfU73ThiRJkiaTqrttZOZyYHm/4jtp7pbRv+7jwPGD7OcjwEeG2UZJkiRpQvAbBiVJkqRKhmdJkiSpkuFZkiRJqmR4liRJkioZniVJkqRKhmdJkiSpkuFZkiRJqmR4rrX23E63QJIkSR1meJYkSZIqGZ4lSZKkSoZnSZIkqZLhWZIkSapkeJYkSZIqGZ4lSZKkSoZnSZIkqZLhWZIkSapkeJYkSZIqGZ4lSZKkSoZnSZIkqZLhWZIkSapkeJYkSZIqGZ4lSZKkSoZnSZIkqZLhWZIkSapkeJYkSZIqGZ4lSZKkSoZnSZIkqZLhWZIkSapkeJYkSZIqGZ4lSZKkSoZnSZIkqZLhWZIkSapkeJYkSZIqGZ4lSZKkSobnwTx6b6dbIEmSpAnG8CxJkiRVMjxLkiRJlQzPkiRJUiXDsyRJklTJ8CxJkiRVMjwPZe25nW6BJEmSJgjDsyRJklTJ8CxJkiRVMjxLkiRJlQzPkiRJUiXDsyRJklTJ8CxJkiRVMjxLkiRJlQzPkiRJUqWq8BwRsyLiKxHxnxFxa0QcERHPiog1EXFH+blvqRsR8cmIWB8RN0bEopb9nFjq3xERJ47VSUmSJEljoXbk+RPAv2XmrwMvBm4FzgQuy8yDgcvKOsAbgIPLv2XApwEi4lnAcuBlwOHA8t7ALUmSJE0GQ4bniNgHeCVwPkBmPpGZDwHHAReUahcAbyrLxwH/lI0rgFkR8TzgGGBNZm7OzJ8Ba4DXj+rZSJIkSWNoWkWdA4EHgM9FxIuBa4HTgDmZuREgMzdGxHNL/bnA3S2P7yllg5VvJyKW0YxYM2fOHLq7u4dzPqNmy7Y96d6yoFnp7oYtC5qf0k5s2bKlY31Wk5f9Ru2w36gd9puRqwnP04BFwPsz88qI+ARPT9EYSAxQljsp374gcyWwEmDx4sXZ1dVV0cTR1/2N1XTNuKtZ6VoKa89tfko70d3dTaf6rCYv+43aYb9RO+w3I1cz57kH6MnMK8v6V2jC9H1lOgbl5/0t9fdvefw84J6dlEuSJEmTwpDhOTPvBe6OiF8rRa8FbgEuAXrvmHEi8PWyfAnw7nLXjSXAw2V6x7eAoyNi33Kh4NGlTJIkSZoUaqZtALwf+EJE7AHcCZxEE7wvioiTgQ3A8aXupcCxwHrgF6Uumbk5Is4Bri71zs7MzaNyFpIkSdI4qArPmbkOWDzAptcOUDeBUwfZzypg1XAaKEmSJE0UfsOgJEmSVMnwLEmSJFUyPEuSJEmVDM+SJElSJcOzJEmSVMnwLEmSJFUyPEuSJEmVDM+SJElSJcOzJEmSVMnwLEmSJFUyPEuSJEmVDM+SJElSJcNzjbXndroFkiRJmgAMz5IkSVIlw7MkSZJUyfAsSZIkVTI8S5IkSZUMz5IkSVIlw7MkSZJUyfAsSZIkVTI8S5IkSZUMz5IkSVIlw7MkSZJUyfAsSZIkVTI8S5IkSZUMz5IkSVIlw7MkSZJUyfDcrrXndroFkiRJGmeGZ0mSJKmS4VmSJEmqZHiWJEmSKhmeJUmSpEqGZ0mSJKmS4VmSJEmqZHiWJEmSKhmeJUmSpEqGZ0mSJKmS4VmSJEmqZHiWJEmSKhmeJUmSpEqGZ0mSJKmS4Xk41p7b6RZIkiSpgwzPkiRJUiXDsyRJklTJ8CxJkiRVMjxLkiRJlQzPkiRJUiXDsyRJklSpOjxHxG4RcX1E/EtZXxARV0bEHRHxpYjYo5TvWdbXl+3zW/bxwVJ+W0QcM9onM268ZZ0kSdKUNJyR59OAW1vWPwasyMyDgZ8BJ5fyk4GfZeYLgBWlHhFxCLAUOBR4PfB3EbHbyJovSZIkjZ+q8BwR84DfAT5b1gN4DfCVUuUC4E1l+biyTtn+2lL/OGB1Zv4yM+8C1gOHj8ZJSJIkSeOhduT5b4A/BbaV9dnAQ5m5taz3AHPL8lzgboCy/eFSv698gMdIkiRJE960oSpExBuB+zPz2ojo6i0eoGoOsW1nj2k93jJgGcCcOXPo7u4eqoljYsu2PenesmDHDd3d0FveobZp4tqyZUvH+qwmL/uN2mG/UTvsNyM3ZHgGjgR+LyKOBaYD+9CMRM+KiGlldHkecE+p3wPsD/RExDTgmcDmlvJerY/pk5krgZUAixcvzq6urjZOa+S6v7Garhl37biha+nTFwx2LR3fRmnC6+7uplN9VpOX/UbtsN+oHfabkRty2kZmfjAz52XmfJoL/r6bme8A1gJvLdVOBL5eli8p65Tt383MLOVLy904FgAHA1eN2pmMF++0IUmSNGXVjDwP5n8BqyPiw8D1wPml/Hzg8xGxnmbEeSlAZt4cERcBtwBbgVMz86kRHF+SJEkaV8MKz5nZDXSX5TsZ4G4Zmfk4cPwgj/8I8JHhNlKSJEmaCPyGQUmSJKmS4VmSJEmqZHiWJEmSKhmeJUmSpEqGZ0mSJKmS4VmSJEmqZHiWJEmSKhmed+LyOzd1ugmSJEmaQAzPkiRJUiXDsyRJklTJ8CxJkiRVMjxLkiRJlQzPkiRJUiXDsyRJklTJ8CxJkiRVMjxLkiRJlQzPkiRJUiXDsyRJklTJ8CxJkiRVMjxLkiRJlQzPkiRJUiXDsyRJklTJ8CxJkiRVMjxLkiRJlQzPkiRJUiXD80isPbfTLZAkSdI4MjxLkiRJlQzPQ7j8zk2dboIkSZImCMOzJEmSVMnwLEmSJFUyPEuSJEmVDM+SJElSJcOzJEmSVMnwXME7bkiSJAkMzyPnF6VIkiRNGYZnSZIkqZLhWZIkSapkeK7kvGdJkiQZniVJkqRKhmdJkiSpkuFZkiRJqmR4HoZB5z17uzpJkqQpwfAsSZIkVTI8jxZHnyVJknZ5hmdJkiSpkuFZkiRJqmR4liRJkioZnofJbxqUJEmaugzPkiRJUqUhw3NE7B8RayPi1oi4OSJOK+XPiog1EXFH+blvKY+I+GRErI+IGyNiUcu+Tiz174iIE8futMaWo8+SJElTU83I81bgjzPzN4AlwKkRcQhwJnBZZh4MXFbWAd4AHFz+LQM+DU3YBpYDLwMOB5b3Bm5JkiRpMhgyPGfmxsy8riw/CtwKzAWOAy4o1S4A3lSWjwP+KRtXALMi4nnAMcCazNycmT8D1gCvH9WzkSRJksbQsOY8R8R84CXAlcCczNwITcAGnluqzQXubnlYTykbrFySJEmaFKbVVoyIGcDFwB9m5iMRMWjVAcpyJ+X9j7OMZroHc+bMobu7u7aJo2rLtj35+TMXDrq9e8sAT903VsPM/cawVZrotmzZ0rE+q8nLfqN22G/UDvvNyFWF54jYnSY4fyEzv1qK74uI52XmxjIt4/5S3gPs3/LwecA9pbyrX3l3/2Nl5kpgJcDixYuzq6urf5Vx0f2N1ez58LpBtx9x4OyBN3QtHaMWaTLo7u6mU31Wk5f9Ru2w36gd9puRq7nbRgDnA7dm5sdbNl0C9N4x40Tg6y3l7y533VgCPFymdXwLODoi9i0XCh5dyiRJkqRJoWbO85HAu4DXRMS68u9Y4KPA6yLiDuB1ZR3gUuBOYD3wD8D/BMjMzcA5wNXl39mlbNey9txOt0CSJEljZMhpG5n5Awaerwzw2gHqJ3DqIPtaBawaTgMlSZKkicJvGJQkSZIqGZ7b5LcMSpIkTT2GZ0mSJKmS4VmSJEmqZHiWJEmSKhmex4K3q5MkSdolGZ5HwIsGJUmSphbD81hx9FmSJGmXY3iWJEmSKhmex5Oj0ZIkSZOa4XmEnPcsSZI0dRiex5IjzZIkSbsUw7MkSZJUyfA8Cpy6IUmSNDUYnseaUzckSZJ2GYZnSZIkqZLheZQ4dUOSJGnXZ3iWJEmSKhmeR5Gjz5IkSbs2w/MoM0BLkiTtugzP48E7bkiSJO0SDM+dZKiWJEmaVAzPkiRJUiXDsyRJklTJ8DxenKIhSZI06Rmex4h33ZAkSdr1GJ7HwE6DsyPQkiRJk5bhWZIkSapkeB7Ez3+5tdNNkCRJ0gRjeJYkSZIqTet0A3ZlvXOfjzhw9vYbnPcsSZI0KTnyLEmSJFUyPHeao9CSJEmThuFZkiRJqmR4Hgd+YYokSdKuwfA8Ti6/c9PgIdqpG5IkSZOC4XmiMEBLkiRNeIbniWTtuYZoSZKkCczwPM52On2jlwFakiRpQjI8S5IkSZUMzx1SNfo80Ai0o9KSJEkdY3juIKdwSJIkTS6G5wlgyBBdAvTl558BwIo1t7Nize3bbZMkSdLYMzxPFgOE5L4ALUmSpHFheJ5AWkegq6Z0lHq9IdowLUmSNLYMzxPQUKF5yYaVLNmwcruyFWtuZ8mGldtN6XBqhyRJ0ugyPE9wvSPQAwXq3gDdP0gDOwRoR6UlSZJGzvC8i2gN0v1DdW/wHnQk2pFpSZKkKtM63QANT808aNhxVLr354o1y1iyYRNXbL29pe7T66e/7oWj2VxJkqRdyriPPEfE6yPitohYHxFnjvfxp7r+I9Q7husyZ7qMRreu95/64VQQSZI01YzryHNE7AZ8Cngd0ANcHRGXZOYt49kODax17vTlAHeewZLW9QOeDsxLNqyEA5axYs3tnP66F3L5+WdwxQHL+kaue8slSZJ2JeM98nw4sD4z78zMJ4DVwHHj3Aa1aaDR6iUbVvZ9eUvvcu+dP/pGq3vnVPcfvW4pH5BzsSVJ0gQz3nOe5wJ3t6z3AC8b5zZojLVeqLiElX2j2ECzfn5L5VLe97O/wcqH6YgDZ2+3fvmdmzjiwNlcfucmrjhgGUs2rOxb7193xda3cPq0iwd87BEn/xWXn39G3zoHvbFvfcD6g/1s2c+KrW9hyYaVfe3qfx699XtH93v/IOld7v0koPUxA+2z75OCtefCqz/Y97P1fFrbxas/2DwfvZ8q9P5xU8r79tN/uVf/+pIkTUKRmeN3sIjjgWMy85Sy/i7g8Mx8f0udZcCysvprwG3j1sDtPRt4sEPH1uRlv1E77Ddqh/1G7bDfDO5XM/M5Q1Ua75HnHmD/lvV5wD2tFTJzJbDjjYvHWURck5mLO90OTS72G7XDfqN22G/UDvvNyI33nOergYMjYkFE7AEsBS4Z5zZIkiRJbRnXkefM3BoR7wO+BewGrMrMm8ezDZIkSVK7xv1LUjLzUuDS8T5uGzo+dUSTkv1G7bDfqB32G7XDfjNC43rBoCRJkjSZjfs3DEqSJEmTleG5H78+XDsTET+OiB9FxLqIuKaUPSsi1kTEHeXnvqU8IuKTpS/dGBGLOtt6jaeIWBUR90fETS1lw+4rEXFiqX9HRJzYiXPR+Bmk35wVET8t7zvrIuLYlm0fLP3mtog4pqXc32VTRETsHxFrI+LWiLg5Ik4r5b7fjBHDc4uWrw9/A3AI8PsRcUhnW6UJ6NWZubDlVj9nApdl5sHAZWUdmn50cPm3DPj0uLdUnfSPwOv7lQ2rr0TEs4DlNF8mdTiwvPcXoHZZ/8iO/QZgRXnfWViuHaL8floKHFoe83cRsZu/y6acrcAfZ+ZvAEuAU8vr7fvNGDE8b8+vD1c7jgMuKMsXAG9qKf86/5NpAAACS0lEQVSnbFwBzIqI53WigRp/mfnvwOZ+xcPtK8cAazJzc2b+DFjDwMFKu4hB+s1gjgNWZ+YvM/MuYD3N7zF/l00hmbkxM68ry48Ct9J8o7PvN2PE8Ly9gb4+fG6H2qKJKYFvR8S15dswAeZk5kZo3sSA55Zy+5P6G25fsQ+p1/vKR+yrWkYD7TfaTkTMB14CXInvN2PG8Ly9GKDM25Go1ZGZuYjmY69TI+KVO6lrf1KtwfqKfUjQfKx+ELAQ2Aj8dSm336hPRMwALgb+MDMf2VnVAcrsN8NgeN7ekF8frqktM+8pP+8Hvkbz8eh9vdMxys/7S3X7k/obbl+xD4nMvC8zn8rMbcA/0LzvgP1GRUTsThOcv5CZXy3Fvt+MEcPz9vz6cA0qIvaOiJm9y8DRwE00faT3quQTga+X5UuAd5crm5cAD/d+hKYpa7h95VvA0RGxb/mo/uhSpimk37USb6Z534Gm3yyNiD0jYgHNBWBX4e+yKSUiAjgfuDUzP96yyfebMTLu3zA4kfn14RrCHOBrzfsU04AvZua/RcTVwEURcTKwATi+1L8UOJbmIp5fACeNf5PVKRFxIdAFPDsiemiuYv8ow+grmbk5Is6hCUMAZ2dm7cVkmoQG6TddEbGQ5iP0HwN/AJCZN0fERcAtNHdcODUznyr78XfZ1HEk8C7gRxGxrpT9Gb7fjBm/YVCSJEmq5LQNSZIkqZLhWZIkSapkeJYkSZIqGZ4lSZKkSoZnSZIkqZLhWZIkSapkeJYkSZIqGZ4lSZKkSv8flv9AKq8+63QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x149292b3c18>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(2, figsize=(12, 5))\n",
    "plt.hist(train_dirty_word_counts, bins=1000, alpha=0.5, label='Train')\n",
    "plt.hist(test_dirty_word_counts, bins=1000, alpha=0.5, label='Test')\n",
    "plt.title(\"Token-count Distribution - Raw, Train\")\n",
    "plt.grid(True); plt.legend(); plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Cleaned Data Statistics\n",
    "The original data came with a lot of noise such as usernames and IP addresses. They should be factored during the learning process for many reasons. Now we will attempt to clean up the origianl data and see how the numbers change. Note that this is just a basic demo. There are most likely more and better ways to clean the data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "tkzr = TweetTokenizer(preserve_case=False)\n",
    "eng_stopwords = (\n",
    "    'what', 'which', 'who', 'whom', \n",
    "    'this', 'that', 'these', 'those', \n",
    "    'am', 'is', 'are', 'was', 'were', \n",
    "    'be', 'been', 'being', \n",
    "    'have', 'has', 'had', 'having', \n",
    "    'do', 'does', 'did', 'doing', \n",
    "    'a', 'an', 'the', \n",
    "    'and', 'but', 'if', 'or', \n",
    "    'because', 'as', 'until', 'while', \n",
    "    'of', 'at', 'by', 'for', 'with', \n",
    "    'about', 'against', 'between', \n",
    "    'into', 'through', 'during', 'before', 'after', \n",
    "    'above', 'below', 'to', 'from', \n",
    "    'up', 'down', 'in', 'out', 'on', 'off', \n",
    "    'over', 'under', 'again', 'further', \n",
    "    'then', 'once', 'here', \n",
    "    'there', 'when', 'where', 'why', \n",
    "    'how', 'all', 'any', 'both', 'each', \n",
    "    'few', 'more', 'most', 'other', 'some', \n",
    "    'such', 'no', 'nor', 'not', 'only', \n",
    "    'own', 'same', 'so', 'than', 'too', 'very', \n",
    "    'can', 'will', 'just', 'don', 'should', 'now'\n",
    ")\n",
    "lmtzr = WordNetLemmatizer()\n",
    "appos = {\n",
    "    \"aren't\" : \"are not\",\n",
    "    \"can't\" : \"cannot\",\n",
    "    \"couldn't\" : \"could not\",\n",
    "    \"didn't\" : \"did not\",\n",
    "    \"doesn't\" : \"does not\",\n",
    "    \"don't\" : \"do not\",\n",
    "    \"hadn't\" : \"had not\",\n",
    "    \"hasn't\" : \"has not\",\n",
    "    \"haven't\" : \"have not\",\n",
    "    \"he'd\" : \"he would\",\n",
    "    \"he'll\" : \"he will\",\n",
    "    \"he's\" : \"he is\",\n",
    "    \"i'd\" : \"I would\",\n",
    "    \"i'd\" : \"I had\",\n",
    "    \"i'll\" : \"I will\",\n",
    "    \"i'm\" : \"I am\",\n",
    "    \"isn't\" : \"is not\",\n",
    "    \"it's\" : \"it is\",\n",
    "    \"it'll\":\"it will\",\n",
    "    \"i've\" : \"I have\",\n",
    "    \"let's\" : \"let us\",\n",
    "    \"mightn't\" : \"might not\",\n",
    "    \"mustn't\" : \"must not\",\n",
    "    \"shan't\" : \"shall not\",\n",
    "    \"she'd\" : \"she would\",\n",
    "    \"she'll\" : \"she will\",\n",
    "    \"she's\" : \"she is\",\n",
    "    \"shouldn't\" : \"should not\",\n",
    "    \"that's\" : \"that is\",\n",
    "    \"there's\" : \"there is\",\n",
    "    \"they'd\" : \"they would\",\n",
    "    \"they'll\" : \"they will\",\n",
    "    \"they're\" : \"they are\",\n",
    "    \"they've\" : \"they have\",\n",
    "    \"we'd\" : \"we would\",\n",
    "    \"we're\" : \"we are\",\n",
    "    \"weren't\" : \"were not\",\n",
    "    \"we've\" : \"we have\",\n",
    "    \"what'll\" : \"what will\",\n",
    "    \"what're\" : \"what are\",\n",
    "    \"what's\" : \"what is\",\n",
    "    \"what've\" : \"what have\",\n",
    "    \"where's\" : \"where is\",\n",
    "    \"who'd\" : \"who would\",\n",
    "    \"who'll\" : \"who will\",\n",
    "    \"who're\" : \"who are\",\n",
    "    \"who's\" : \"who is\",\n",
    "    \"who've\" : \"who have\",\n",
    "    \"won't\" : \"will not\",\n",
    "    \"wouldn't\" : \"would not\",\n",
    "    \"you'd\" : \"you would\",\n",
    "    \"you'll\" : \"you will\",\n",
    "    \"you're\" : \"you are\",\n",
    "    \"you've\" : \"you have\",\n",
    "    \"'re\": \" are\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'll\":\" will\",\n",
    "    \"didn't\": \"did not\"\n",
    "}\n",
    "\n",
    "def preprocess(comment):\n",
    "  \n",
    "    # credit to the author of this post:\n",
    "    # https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda\n",
    "\n",
    "    # remove special format\n",
    "    comment = re.sub('\\n\\t', '', comment)\n",
    "\n",
    "    # remove IP addresses\n",
    "    comment = re.sub('\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}', ' specipaddress ', comment)\n",
    "\n",
    "    # remove username\n",
    "    comment = re.sub(\"\\[\\[User.*\\]\", ' specusername ', comment)\n",
    "    comment = re.sub(\"\\[\\[User.*\\|\", ' specusername ', comment)\n",
    "\n",
    "    # tokenization \n",
    "    tokens = tkzr.tokenize(comment)\n",
    "\n",
    "    # aphostophe replacement\n",
    "    tokens = [ appos[token] if token in appos else token for token in tokens]\n",
    "\n",
    "    # remove stopwords\n",
    "    tokens = [ token for token in tokens if not token in eng_stopwords ]\n",
    "    \n",
    "    # stemming\n",
    "    tokens = [ lmtzr.lemmatize(token, 'v') for token in tokens]\n",
    "\n",
    "    return \" \".join(tokens)\n",
    "  \n",
    "\n",
    "hint('Cleaning train set...')\n",
    "train_clean = train['comment_text'].apply(lambda c: preprocess(c))\n",
    "hint('Cleaning test set...')\n",
    "test_clean = test['comment_text'].apply(lambda c: preprocess(c))\n",
    "hint('Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Token-Count Distribution:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tMin\tMean\tMax\n",
      "Train\t0\t43\t1387\n",
      "Test\t0\t38\t2142\n"
     ]
    }
   ],
   "source": [
    "train_clean_seq = tr_tkzr.texts_to_sequences(train_clean)\n",
    "test_clean_seq = tr_tkzr.texts_to_sequences(test_clean)\n",
    "train_clean_word_counts = [ len(seq) for seq in train_clean_seq ]\n",
    "test_clean_word_counts = [ len(seq) for seq in test_clean_seq ]\n",
    "print(\"\\tMin\\tMean\\tMax\")\n",
    "print(\"Train\\t%d\\t%d\\t%d\" % (\n",
    "    np.min(train_clean_word_counts), \n",
    "    np.mean(train_clean_word_counts), \n",
    "    np.max(train_clean_word_counts)\n",
    "))\n",
    "print(\"Test\\t%d\\t%d\\t%d\" % (\n",
    "    np.min(test_clean_word_counts), \n",
    "    np.mean(test_clean_word_counts), \n",
    "    np.max(test_clean_word_counts)\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtUAAAE/CAYAAACJhD8SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XucXWV97/HPzwQIQjAQJdAkmqDYFrHGmGIiVkeh3GoLVsEUL2mMr5QequgptujpabjIAVs11eLlpCUaFUEULdiiGCHT1sodwr1ADmgYE24JBAa5GPidP9YzYWeyJ5nsNTN7Nvm8X695zV7PetZaz9rzzJ7vfuZZa0dmIkmSJKl1L2p3AyRJkqROZ6iWJEmSajJUS5IkSTUZqiVJkqSaDNWSJElSTYZqSZIkqSZDtaSWRMQREbGq3e0YzSJil4jojYjfGKL9nRYR55THvxURG4div2V/r46IR4dqf0MpIk6IiJ+0ux39RcT9EfHmNhx3t9Kv9h7pY0samKFa2oGVP8x9X89FxJMNy+9td/vaaVuBqbypeK7h+bovIs6PiNf31cnMpzNz98xcs41jDeoNSmYuysy/2L4zGfCYm51fZt6VmROGYt8ttucPIuKnEfF4RDwYEVdExJHtas9QaXhj1fh79quG5Xdt7z4z84nSrx4cjjZLao2hWtqBlT/Mu2fm7sBq4A8bys5rd/s6wD3ludsDeBNwL/CziPi9oT5QRIwd6n2OFuUN3LeAfwImA/sCZwJHt7NdQ6HhjVXf79mDwGENZRf13+aF/LOWXsgM1ZIGFBG7RsQXI2JtRPRExN9HxE4D1P14RNwcEfuU5XeW5Ucj4j8j4oCGuvdHxMci4taI2BAR50XEzltpx7SIuDgiHi5fny3lY8qUiNUR8UBELI2I8WXdFqO/jaOzEXF2Oe75ZXT05oiYUdZ9B9gb+HEZTfzI1p6nzHwuM+/LzE8C5wFnlf2Mi4iMiCll+eiI+O9yvPsi4iMRMRH4PrBfw+jlxNK+b0XEtyPicWBuKfvnfud0Qvn5rImIDzeUXxARf9OwvOn5aHZ+/aeTRMTLI+LSiFgfEXdFxLyGdQM+d9urBMjPAn+Tmcsy87HMfDYzL8/MEwbY5sAykv1IRNwREcc0rHtnRNwUEY9FxC8i4pMN634rIjZGxPzSnx+KiI83rB8TEf87Iu4p/ey8iJjQsH5B6WubbVdXRHwmIr4REd8pP+t3R8RbIuKa8vuxJiI+GxFjSv3dS7/q+137bkR8LiKWl5/HTyNi6lC1T9LgGKolbc1pwO8ArwXeAHQBf9W/UkScCbwb6MrM+yNiNvAlYD4wEfgG8C+x+Qjcu4FDgFcBbwSOb9aAqEL8D4E7gJcDU4G+0b0/A44Dfg/Ynyoofm47zu+dwFJgAnA58A8AmXksm48ofmE79vk9YHY0f/OxFPhAZo4HZgD/mZnrSjvuaRi9XFfqvwtYBryE58+50RhgDrAf8AfAaTGIOb6DPL/vAHdSjRofDyyOiIMb1jd97lpwIDAJ+O5gKkfEHsBy4FzgpcAHgKUR8apS5bHS3gmljSdHxBENuxgDzKLqd0cBZ0bEfmXdx4HDgDcDU4BfA4vLcWdQneN7yrpp5fhD5dhyTnsAFwPPAP8D2At4azmX+VvZ/vjS/onAQ8CiIWybpEEwVEvamvcCizLz4cx8APgU8P6G9RERX6Sa+nBoZq4v5X8GnJOZ15dRxyXALlTBvM/izHwgMx8CLqUKmc28mSpofDIzf5WZT2bmzxra9/eZ+YvMfAz4X8B7IyIGeX5XZObyzHyWKvi3NNrazxqq4LZHk3UbgddExPjMXJeZN25jX/+emZeWkfAnB6izqDwnNwLfBP6k9aZXImJ/4HVUz/nTmXkdVbhv/NkP1XM3EUjggUHWfydwa2aeV/rWtcAPqN6AUEa4byvP2Q3AhVShtNGizHyqbPvfVG8coeq3p2Tmmsx8iupN5XtKfzoOuCgzr8zMp4FPMrR/Q6/IzB9l5cnMvCozryvneDfVG5j+59HogsxcmZnPAOczNH1Z0nYwVEtqqgSJfYBfNBT/gmrOa5+9qUbPPpWZjzeUvwL4ZFRTPx6N6q4SL+u37f0Nj38F7F6Oe0VsfhHXVODezHyuSTN/o0n7dqUa3RuMpm2oaTLwLNWIaX/HUIW/1eU8Z21jX/cN4niNdX5B9ZzU9RvAQ/2CfP+f/aCeu6im5/T9PJuNZq8Dgmq0ejBeAbylX996F9WIOhFxcET8e5misQH4UzYfUX42Mx/u3/bS36cClzbs90aqv5MTqZ6TTc91Zm4ANgyyzYOx2c+6THH5UVTTmh6jCvFbGxkfjr4saTsYqiU1lZlJ9Yf6FQ3FLwd+2bD8AFVQ/FZEHNRQfh/wt5k5oeHrxZn5vUEc9+39LuK6D5gWEc1er9Y0ad+TwHrgCeDFfSvKdIzBhm2oRk9b8U7gqsz89RY7rEY530EVIH9MNaK4tWMNpg2Nc2dfTvWcQL/zp3qDNNh9rwFeFhG79tv3LweoP6Byx5K+n+dHm1S5laofDfYuGPcBP+7Xtxr3fSHwbWBqZr4E+BpVaN9WO5Pq/N7eb9/jSghfS8NzHREvoZqWM1T6/zzOBa4B9svMPYD/wyDOQ1L7GKolbc35wKKoLpzbm2p6xTcbK2Tmj4EPAj+I528ntwT4cETMisruEfFHEfFitt9PgceBMyLixVFdPPmmhvadHNVFdeOppqd8qwSkO4C9IuKQEqhPY/te8x6gmqu8TeUcp0TEGcD7qJ6n/nV2i4i5ZU7wr8s5PdtwrL0jopXRxUXlOXkd1fSMb5fylcA7ImJCREwGPtxvu62d3yrgZuBTUd0SbiYwj+oizCGVmRuBk8ux3h8R4yPiRRHx1oj4UpNN/gV4fUS8JyJ2ioidI2J2VPfZDqoR2nWZ+VTpJ8duR3O+Apzdd5FfROwdEX9Y1l0I/HFEvDEidqHqa5v+exLVhaBPbfcTMLDxwKOZ+UREvBb40BDuW9IwMFRL2pq/BW4HbqMKaf8F/F3/Spn5b8AJwA8j4ncy87+AjwD/F3gUuIvqQqrtHv0tI75HUc3x7aG69d8fl9Vfprow8GfA/6Maof6fZbuHgZOogmAP1ah747/9t+VMqovYHo2Ige4NvV9E9AK9wNXAbwJvzsx/H6D+B6mmUWygusCu744aNwGXAL8oxxvsiPqz5bj3Aj8CTs/M/yjrllKF49XAv/L8qPg2z6+8KTkOOIDqefs28PHM/M9Btmu7ZOY3qd6MnEA1Inw/1YV2Fzep+whwONW0o7VUo+qfAnYq7T4B+ExUd9H4K6oLLgfr74CfAFeU7X8GzCzHvRH4S6oLKvv6YWN/mkr1+zFUPgr8eelfnwcuGMJ9SxoGUb0GSZKkVkXEN4F/2sobKkkvcIZqSZIkqSanf0iSJEk1GaolSZKkmgzVkiRJUk2GakmSJKmmse1uQKte+tKX5rRp00b8uE888QS77bbbiB9Xnc1+o1bYb9QK+41aYb8Z2PXXX/9wZr5sW/U6NlRPmzaN6667bsSP293dTVdX14gfV53NfqNW2G/UCvuNWmG/GVhE/GIw9Zz+IUmSJNVkqJYkSZJqMlRLkiRJNXXsnGpJkiQNn1//+tf09PTw1FNPtbspI2LcuHFMmTKFnXbaqaXtDdWSJEnaQk9PD+PHj2fatGlERLubM6wyk3Xr1tHT08P06dNb2ofTPyRJkrSFp556iokTJ77gAzVARDBx4sRao/KGakmSJDW1IwTqPnXP1VAtSZKkUWfdunXMmDGDGTNmsM8++zB58uRNy88888yg9jF//nzuvPPOYW5pxTnVkiRJ2qbFy+8a0v197PdfvdX1EydOZOXKlQCceuqp7L777px88smb1clMMpMXvaj5OPFXv/rVoWnsIDhSLUmSpI6xatUqDjzwQE444QRmzpzJ2rVrWbhwIbNmzeI1r3kNp59++qa6b37zm1m5ciUbN25kwoQJnHLKKbzuda9jzpw5PPjgg0PaLkO1JEmSOsrtt9/OggULuPHGG5k8eTJnn3021113HTfddBPLly/n9ttv32KbDRs28Na3vpWbbrqJOXPmsHTp0iFtk6G6FSvOancLJEmSdlivfOUr+d3f/d1Ny+effz4zZ85k5syZ3HHHHU1D9a677sqRRx4JwBve8AZ+/vOfD2mbnFMtSZKkjrLbbrttenz33Xfz+c9/nmuuuYYJEybwvve9r+mt8XbeeedNj8eMGcPGjRuHtE2OVEuSJKljPfbYY4wfP5499tiDtWvXctlll7WlHY5US5IkqWPNnDmTAw44gAMPPJD99tuPgw8+uC3tMFRLkiRpm7Z1C7zhdOqpp256/KpXvWrTrfag+tCWb3zjG023++lPf7rp8aOPPrrp8dy5c5k7d+6QttHpH5IkSVJNhmpJkiSpJkO1JEmSVJOhWpIkSarJUC1JkiTVZKiWJEmSajJUS5IkadRZt24dM2bMYMaMGeyzzz5Mnjx50/Izzzwz6P0sXbqU+++/fxhbWvE+1ZIkSdq2FWcN7f7e9omtrp44ceKm+1Gfeuqp7L777px88snbfZilS5cyc+ZM9tlnn5aaOViGakmSJHWUZcuW8cUvfpFnnnmGN73pTZxzzjk899xzzJ8/n5UrV5KZLFy4kEmTJrFy5Ure8573sOuuu3LNNdew8847D0ubDNWSJEnqGLfeeivf//73+dnPfsbYsWNZuHAhF1xwAa985St5+OGHueWWW4DqExQnTJjAP/7jP3LOOecwY8aMYW2XoVqSJEkd4yc/+QnXXnsts2bNAuDJJ59k6tSpHH744dx5552cdNJJHHXUURx22GEj2i5DtSRJkjpGZvLBD36QM844Y4t1N998Mz/84Q/5whe+wEUXXcSSJUtGrF3e/UOSJEkd49BDD+XCCy/k4YcfBqq7hKxevZqHHnqIzOTYY4/ltNNO44YbbgBg/PjxPP7448PeLkeqJUmS1DFe+9rXsmjRIg499FCee+45dtppJ77yla8wZswYFixYQGYSEXz6058GYP78+XzoQx/yQkVJkiSNAtu4Bd5wOvXUUzdbPv744zn++OO3qHfjjTduUXbcccdx3HHHDVfTNnH6hyRJklSToVqSJEmqyVAtSZIk1WSoliRJUlOZ2e4mjJi652qoliRJ0hbGjRvHunXrdohgnZmsW7eOcePGtbyPbd79IyKWAu8AHszMA0vZXsC3gWnAz4HjMvORiAjg88BRwK+AP83MG8o284C/Kbv9VGYuK+VvAL4G7ApcCpyUO8JPT5IkaRSbMmUKPT09PPTQQ+1uyogYN24cU6ZMaXn7wdxS72vAOcDXG8pOAS7PzLMj4pSy/NfAkcD+5euNwJeBN5YQvgiYBSRwfURckpmPlDoLgauoQvURwA9bPiNJkiTVttNOOzF9+vR2N6NjbHP6R2b+B7C+X/HRwLLyeBlwTEP517NyFTAhIvYFDgeWZ+b6EqSXA0eUdXtk5pVldPrrDfuSJEmSOkKrc6onZeZagPJ971I+GbivoV5PKdtaeU+TckmSJKljDPUnKkaTsmyhvPnOIxZSTRVh0qRJdHd3t9DEenp7e+nO6dCGY6tz9fb2tqW/qrPZb9QK+41aYb+pr9VQ/UBE7JuZa8sUjgdLeQ8wtaHeFGBNKe/qV95dyqc0qd9UZi4BlgDMmjUru7q6Bqo6bLq7u+nKW6Br7ogfW52ru7ubdvRXdTb7jVphv1Er7Df1tTr94xJgXnk8D7i4ofwDUZkNbCjTQy4DDouIPSNiT+Aw4LKy7vGImF3uHPKBhn1JkiRJHWEwt9Q7n2qU+aUR0UN1F4+zgQsjYgGwGji2VL+U6nZ6q6huqTcfIDPXR8QZwLWl3umZ2Xfx45/z/C31foh3/pAkSVKH2Waozsw/GWDVIU3qJnDiAPtZCixtUn4dcOC22iFJkiSNVn6ioiRJklSTobquFWe1uwWSJElqM0O1JEmSVJOhWpIkSarJUC1JkiTVZKiWJEmSajJUS5IkSTUZqiVJkqSaDNWSJElSTYZqSZIkqSZDtSRJklSToVqSJEmqyVAtSZIk1WSoliRJkmoyVEuSJEk1GaolSZKkmgzVkiRJUk2GakmSJKkmQ7UkSZJUk6FakiRJqslQLUmSJNVkqJYkSZJqMlRLkiRJNRmqJUmSpJoM1ZIkSVJNhmpJkiSpJkO1JEmSVJOhWpIkSarJUC1JkiTVZKhu1Yqz2t0CSZIkjRKGakmSJKkmQ7UkSZJUk6FakiRJqslQLUmSJNVkqJYkSZJqMlRLkiRJNRmqJUmSpJpqheqI+FhE3BYRt0bE+RExLiKmR8TVEXF3RHw7InYudXcpy6vK+mkN+/lEKb8zIg6vd0qSJEnSyGo5VEfEZOAjwKzMPBAYA8wFPg0szsz9gUeABWWTBcAjmfkqYHGpR0QcULZ7DXAE8KWIGNNquyRJkqSRVnf6x1hg14gYC7wYWAu8HfhuWb8MOKY8ProsU9YfEhFRyi/IzKcz815gFXBQzXZJkiRJI6blUJ2ZvwQ+A6ymCtMbgOuBRzNzY6nWA0wujycD95VtN5b6ExvLm2wjSZIkjXpjW90wIvakGmWeDjwKfAc4sknV7NtkgHUDlTc75kJgIcCkSZPo7u7evkYPgd7eXrpzerXQ3Q2906vv0lb09va2pb+qs9lv1Ar7jVphv6mv5VANHArcm5kPAUTE94A3ARMiYmwZjZ4CrCn1e4CpQE+ZLvISYH1DeZ/GbTaTmUuAJQCzZs3Krq6uGs1vTXd3N115S7XQNRdWnFV9l7aiu7ubdvRXdTb7jVphv1Er7Df11ZlTvRqYHREvLnOjDwFuB1YA7y515gEXl8eXlGXK+isyM0v53HJ3kOnA/sA1Ndo1clac1e4WSJIkaRRoeaQ6M6+OiO8CNwAbgRupRpH/DbggIj5Vys4tm5wLfCMiVlGNUM8t+7ktIi6kCuQbgRMz89lW2yVJkiSNtDrTP8jMRcCifsX30OTuHZn5FHDsAPs5EzizTlskSZKkdvETFSVJkqSaDNWSJElSTYZqSZIkqSZDtSRJklSToVqSJEmqyVAtSZIk1WSoliRJkmoyVEuSJEk1GaolSZKkmgzVkiRJUk2GakmSJKkmQ7UkSZJUk6FakiRJqslQLUmSJNVkqJYkSZJqMlRLkiRJNRmqJUmSpJoM1ZIkSVJNhmpJkiSpJkO1JEmSVJOhWpIkSarJUC1JkiTVZKiWJEmSajJUS5IkSTUZqiVJkqSaDNWSJElSTYZqSZIkqSZDtSRJklSToVqSJEmqyVAtSZIk1WSoliRJkmoyVEuSJEk1GaolSZKkmgzVkiRJUk2GakmSJKkmQ7UkSZJUk6FakiRJqqlWqI6ICRHx3Yj474i4IyLmRMReEbE8Iu4u3/csdSMivhARqyLi5oiY2bCfeaX+3RExr+5JSZIkSSOp7kj154EfZeZvAa8D7gBOAS7PzP2By8sywJHA/uVrIfBlgIjYC1gEvBE4CFjUF8QlSZKkTtByqI6IPYC3AOcCZOYzmfkocDSwrFRbBhxTHh8NfD0rVwETImJf4HBgeWauz8xHgOXAEa22S5IkSRppdUaq9wMeAr4aETdGxD9HxG7ApMxcC1C+713qTwbua9i+p5QNVC5JkiR1hLE1t50JfDgzr46Iz/P8VI9moklZbqV8yx1ELKSaOsKkSZPo7u7ergYPhd7eXrpz+uaFbWiHOktvb29b+qs6m/1GrbDfqBX2m/rqhOoeoCczry7L36UK1Q9ExL6ZubZM73iwof7Uhu2nAGtKeVe/8u5mB8zMJcASgFmzZmVXV1ezasOqu7ubrrxl88KuuSPeDnWW7u5u2tFf1dnsN2qF/UatsN/U1/L0j8y8H7gvIn6zFB0C3A5cAvTdwWMecHF5fAnwgXIXkNnAhjI95DLgsIjYs1ygeFgpkyRJkjpCnZFqgA8D50XEzsA9wHyqoH5hRCwAVgPHlrqXAkcBq4Bflbpk5vqIOAO4ttQ7PTPX12yXJEmSNGJqherMXAnMarLqkCZ1EzhxgP0sBZbWaYskSZLULn6ioiRJklSToVqSJEmqyVAtSZIk1WSoliRJkmoyVEuSJEk1GaolSZKkmgzVkiRJUk2G6u31+P3tboEkSZJGGUO1JEmSVJOhWpIkSarJUC1JkiTVZKiWJEmSajJUS5IkSTUZqiVJkqSaDNWSJElSTYbqobDirHa3QJIkSW1kqJYkSZJqMlRLkiRJNRmqJUmSpJoM1ZIkSVJNhuoWXHnPunY3QZIkSaOIoVqSJEmqyVAtSZIk1WSoHiorzmLx8rva3QpJkiS1gaFakiRJqslQLUmSJNVkqJYkSZJqMlRLkiRJNRmqJUmSpJoM1ZIkSVJNhmpJkiSpJkO1JEmSVJOhWpIkSarJUC1JkiTVZKiWJEmSajJUS5IkSTUZqiVJkqSaaofqiBgTETdGxL+W5ekRcXVE3B0R346InUv5LmV5VVk/rWEfnyjld0bE4XXbNBKuvGddu5sgSZKkUWIoRqpPAu5oWP40sDgz9wceARaU8gXAI5n5KmBxqUdEHADMBV4DHAF8KSLGDEG7JEmSpBFRK1RHxBTgD4B/LssBvB34bqmyDDimPD66LFPWH1LqHw1ckJlPZ+a9wCrgoDrtkiRJkkZS3ZHqfwD+CniuLE8EHs3MjWW5B5hcHk8G7gMo6zeU+pvKm2wjSZIkjXpjW90wIt4BPJiZ10dEV19xk6q5jXVb26b/MRcCCwEmTZpEd3f39jR5SPQ+twtPvGQGAP+2Dnbb5fmncDL30t29ZsTbpNGvt7e3Lf1Vnc1+o1bYb9QK+019LYdq4GDgjyLiKGAcsAfVyPWEiBhbRqOnAH0psweYCvRExFjgJcD6hvI+jdtsJjOXAEsAZs2alV1dXTWa35ruH1zALhtWblqes9/ETY8Xb5zJcV2vHvE2afTr7u6mHf1Vnc1+o1bYb9QK+019LU//yMxPZOaUzJxGdaHhFZn5XmAF8O5SbR5wcXl8SVmmrL8iM7OUzy13B5kO7A9c02q7JEmSpJFWZ6R6IH8NXBARnwJuBM4t5ecC34iIVVQj1HMBMvO2iLgQuB3YCJyYmc8OQ7skSZKkYTEkoTozu4Hu8vgemty9IzOfAo4dYPszgTOHoi3tNHv1EuAz7W6GJEmSRpifqChJkiTVZKiWJEmSajJUD7UVZ7W7BZIkSRphhmpJkiSpJkO1JEmSVJOhWpIkSarJUC1JkiTVZKiWJEmSajJU13TlPeva3QRJkiS1maFakiRJqslQLUmSJNVkqJYkSZJqMlRLkiRJNRmqJUmSpJoM1ZIkSVJNhmpJkiSpJkO1JEmSVJOhegj4ATCSJEk7NkP1cFhxVrtbIEmSpBFkqJYkSZJqMlRLkiRJNRmqJUmSpJoM1ZIkSVJNhmpJkiSpJkO1JEmSVJOhWpIkSarJUC1JkiTVZKiWJEmSajJUDxc/VVGSJGmHYaiWJEmSajJUD5Er71nX7iZIkiSpTQzVkiRJUk2GakmSJKkmQ7UkSZJUk6F6OHkHEEmSpB2CoXqYLV5+V7ubIEmSpGFmqJYkSZJqajlUR8TUiFgREXdExG0RcVIp3ysilkfE3eX7nqU8IuILEbEqIm6OiJkN+5pX6t8dEfPqn5YkSZI0cuqMVG8E/jIzfxuYDZwYEQcApwCXZ+b+wOVlGeBIYP/ytRD4MlQhHFgEvBE4CFjUF8QlSZKkTtByqM7MtZl5Q3n8OHAHMBk4GlhWqi0DjimPjwa+npWrgAkRsS9wOLA8M9dn5iPAcuCIVtvVbn4IjCRJ0o5nSOZUR8Q04PXA1cCkzFwLVfAG9i7VJgP3NWzWU8oGKpckSZI6wti6O4iI3YGLgI9m5mMRMWDVJmW5lfJmx1pINXWESZMm0d3dvd3trav3uV144iUztlqnu/f5p/XVT19Dd/ea4W6WRrne3t629Fd1NvuNWmG/USvsN/XVCtURsRNVoD4vM79Xih+IiH0zc22Z3vFgKe8BpjZsPgVYU8q7+pV3NzteZi4BlgDMmjUru7q6mlUbVt0/uIBdNqzcap05+03c9PjKB9cx593vG+5maZTr7u6mHf1Vnc1+o1bYb9QK+019de7+EcC5wB2Z+bmGVZcAfXfwmAdc3FD+gXIXkNnAhjI95DLgsIjYs1ygeFgpkyRJkjpCnZHqg4H3A7dERN/Q7SeBs4ELI2IBsBo4tqy7FDgKWAX8CpgPkJnrI+IM4NpS7/TMXF+jXZIkSdKIajlUZ+ZPaT4fGuCQJvUTOHGAfS0FlrbaFkmSJKmd/ERFSZIkqSZD9QhZvPyudjdBkiRJw8RQPQz8ABhJkqQdi6FakiRJqslQPRJWnLX1ZUmSJHU0Q7UkSZJUk6F6hMxevaTdTZAkSdIwMVRvpyee3jioel6sKEmStOMwVEuSJEk1GapHmPerliRJeuExVEuSJEk1GaqHkfOqJUmSdgyG6pHk/aklSZJekAzVkiRJUk2GakmSJKkmQ/UI6/sQGO8CIkmS9MJhqB4NnGstSZLU0QzVkiRJUk2G6jZyCogkSdILg6F6BHi/akmSpBc2Q3WbeMGiJEnSC4ehehRwJFuSJKmzGapHkOFZkiTphclQ3UZ9U0AkSZLU2QzVbWawliRJ6nyGakmSJKkmQ/Uo4p1AJEmSOpOhug2aXrBYPqrcYC1JktR5DNWjkMFakiSpsxiqR5HGixb7B2uDtiRJ0ug1tt0N0Ob6gvVVL19okJYkSeoQjlSPEv3nWXurPUmSpM5hqB7FZq9esumrj6PXkiRJo4+hulOVu4VIkiSp/QzVHWL26iWbBemmt+WTJElSWxiqO8wWU0FWnLXpuyRJktrDUN1BNo1OlwDd/2JG51tLkiS1x6gJ1RFxRETcGRGrIuKUdrdnNBvoTiF9U0QM15IkSSNrVITqiBgDfBE4EjgA+JOIOKC9reoMzeZWb5p/3fcFzaeHOGVEkiRpSIyWD385CFiVmfcARMQFwNHA7W1tVQfpC9f9vwNwz8mbfwfm7DexetA/WL/tE1XZ2z6xqWjx8rv42O+/uulxt7ZOkiRpRzFaQvVk4L6G5R7gjW1qyw5hwLuHNAngs4GWynMPAAAEiElEQVQrz21efWvroArvW7tTyVUvX8js1Uu46uUL+djYi7ao27d94/f+59Fs3ablvjcJ/cv76V++eOO7NrVnzoLPcOW5J292Lo37XrzxXVucw5wFn3l+5yvOgpizxRuQvmXfmEiS1PkiM9vdBiLiWODwzPxQWX4/cFBmfrhfvYXAwrL4m8CdI9rQykuBh9twXHU2+41aYb9RK+w3aoX9ZmCvyMyXbavSaBmp7gGmNixPAdb0r5SZS4C2fn53RFyXmbPa2QZ1HvuNWmG/USvsN2qF/aa+UXGhInAtsH9ETI+InYG5wCVtbpMkSZI0KKNipDozN0bEXwCXAWOApZl5W5ubJUmSJA3KqAjVAJl5KXBpu9sxCG2dfqKOZb9RK+w3aoX9Rq2w39Q0Ki5UlCRJkjrZaJlTLUmSJHUsQ/V28KPUNZCI+HlE3BIRKyPiulK2V0Qsj4i7y/c9S3lExBdKP7o5Ima2t/UaSRGxNCIejIhbG8q2u69ExLxS/+6ImNeOc9HIGaDfnBoRvyyvOysj4qiGdZ8o/ebOiDi8ody/YzuQiJgaESsi4o6IuC0iTirlvuYMA0P1IPlR6hqEt2XmjIZbEp0CXJ6Z+wOXl2Wo+tD+5Wsh8OURb6na6WvAEf3KtquvRMRewCKqD8k6CFjU90dRL1hfY8t+A7C4vO7MKNcmUf42zQVeU7b5UkSM8e/YDmkj8JeZ+dtUn9d2YvmZ+5ozDAzVg7fpo9Qz8xmg76PUpYEcDSwrj5cBxzSUfz0rVwETImLfdjRQIy8z/wNY3694e/vK4cDyzFyfmY8Ay2keuPQCMUC/GcjRwAWZ+XRm3gusovob5t+xHUxmrs3MG8rjx4E7qD7F2tecYWCoHrxmH6U+uU1t0eiTwI8j4vryyZ8AkzJzLVQvbMDepdy+pP62t6/Yh9TnL8q/6Zc2jBzab7SFiJgGvB64Gl9zhoWhevCiSZm3TlGfgzNzJtW/zk6MiLdspa59SYM1UF+xDwmqf82/EpgBrAU+W8rtN9pMROwOXAR8NDMf21rVJmX2nUEyVA/eoD5KXTumzFxTvj8IfJ/q36wP9E3rKN8fLNXtS+pve/uKfUhk5gOZ+WxmPgf8E9XrDthv1CAidqIK1Odl5vdKsa85w8BQPXh+lLqaiojdImJ832PgMOBWqv7Rd4X0PODi8vgS4APlKuvZwIa+f8Nph7W9feUy4LCI2LP8y/+wUqYdSL9rMd5J9boDVb+ZGxG7RMR0qovOrsG/YzuciAjgXOCOzPxcwypfc4bBqPlExdHOj1LXVkwCvl+9djEW+FZm/igirgUujIgFwGrg2FL/UuAoqouHfgXMH/kmq10i4nygC3hpRPRQXVF/NtvRVzJzfUScQRWSAE7PzMFexKYONEC/6YqIGVT/hv858GcAmXlbRFwI3E5194cTM/PZsh//ju1YDgbeD9wSEStL2SfxNWdY+ImKkiRJUk1O/5AkSZJqMlRLkiRJNRmqJUmSpJoM1ZIkSVJNhmpJkiSpJkO1JEmSVJOhWpIkSarJUC1JkiTV9P8BEXIG06UOF68AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14941725860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(3, figsize=(12, 5))\n",
    "plt.hist(train_clean_word_counts, bins=1000, alpha=0.5, label='Train')\n",
    "plt.hist(test_clean_word_counts, bins=1000, alpha=0.5, label='Test')\n",
    "plt.title(\"Token-count Distribution - Cleaned, Train\")\n",
    "plt.grid(True); plt.legend(); plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 Vocabulary size:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaned train set vocabulary size: 184644\n"
     ]
    }
   ],
   "source": [
    "tr_tkzr = ktxt.Tokenizer()\n",
    "tr_tkzr.fit_on_texts(train_clean)\n",
    "print(\"Cleaned train set vocabulary size: %d\" % (len(tr_tkzr.word_index)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaned test set vocabulary size: 244197\n"
     ]
    }
   ],
   "source": [
    "te_tkzr = ktxt.Tokenizer()\n",
    "te_tkzr.fit_on_texts(test_clean)\n",
    "print(\"Cleaned test set vocabulary size: %d\" % (len(te_tkzr.word_index)))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "default_view": {},
   "name": "analysis.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}