Deeper RCNN performance analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we will try out a deep RCNN model based on RCNN v2."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dXsRFoKJSNrH"
   },
   "source": [
    "# 1. Preparation\n",
    "We need to first import the required library, download the data, and load the data into the memory."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "swLpnk4u4XKo"
   },
   "source": [
    "## 1.1 Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 394,
     "output_extras": [
      {
       "item_id": 5
      },
      {
       "item_id": 6
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4509,
     "status": "ok",
     "timestamp": 1518570583550,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "9_NbEq2FSFLe",
    "outputId": "58bb96d9-3ffa-4ccf-b90a-c2db87f537b0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing required packages...\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\ChuanLi\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\ChuanLi\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "print('Importing required packages...')\n",
    "\n",
    "from IPython.display import clear_output\n",
    "import re\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "np.random.seed()\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "nltk.download('stopwords')\n",
    "from nltk.stem.wordnet import WordNetLemmatizer \n",
    "nltk.download('wordnet')\n",
    "from keras.preprocessing import sequence\n",
    "from keras.preprocessing import text as ktxt\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, GRU, SpatialDropout1D, Bidirectional, Dropout\n",
    "from keras.layers import Concatenate, GlobalAveragePooling1D, GlobalMaxPooling1D\n",
    "from keras.layers.convolutional import Conv1D, MaxPooling1D\n",
    "from keras.callbacks import EarlyStopping\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn.utils import class_weight\n",
    "\n",
    "\n",
    "def hint(message):\n",
    "    \"\"\"\n",
    "    erase previous ipynb output and show new message\n",
    "    \"\"\"\n",
    "    clear_output()\n",
    "    print(message)\n",
    "\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0oMhFbcOS7ch"
   },
   "source": [
    "## 1.2 Loading the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 153,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 128,
     "status": "ok",
     "timestamp": 1518570590592,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "C1nwDbfiS-vX",
    "outputId": "d69716f3-d89c-49a1-91d3-7cfe42286b4b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label distribution between training and validation set:\n",
      "           label     train  validation\n",
      "0          toxic  0.095648    0.096632\n",
      "1   severe_toxic  0.009886    0.010434\n",
      "2        obscene  0.053033    0.052608\n",
      "3         threat  0.003047    0.002789\n",
      "4         insult  0.049297    0.049632\n",
      "5  identity_hate  0.008695    0.009243\n"
     ]
    }
   ],
   "source": [
    "hint('loading data...')\n",
    "train = pd.read_csv('data/train.csv')\n",
    "train, valid = train_test_split(train, test_size=0.2)\n",
    "\n",
    "labels = [\n",
    "    'toxic', \n",
    "    'severe_toxic', \n",
    "    'obscene', \n",
    "    'threat', \n",
    "    'insult', \n",
    "    'identity_hate'\n",
    "]\n",
    "\n",
    "Ytr = train[labels].values\n",
    "Yva = valid[labels].values\n",
    "\n",
    "hint('Label distribution between training and validation set:')\n",
    "print(pd.DataFrame({\n",
    "    'label': labels,\n",
    "    'train': [np.mean(train[label]) for label in labels],\n",
    "    'validation' : [np.mean(valid[label]) for label in labels],\n",
    "}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KA8OAJOJWAoe"
   },
   "source": [
    "# 2. Pre-processing the Input\n",
    "There are many ways to pre-process the raw strings into valid input for the model. Here we will do it by building a dictionary with all the comments from the training set, mapping the words to their index in the dictionary, and pad/crop the resulting sequences so that they have the same length."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZmEOAxm4wdNn"
   },
   "source": [
    "## 2.1 Cleaning Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 127,
     "status": "ok",
     "timestamp": 1518570683382,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "Q5jQUn7MxSud",
    "outputId": "73b337ab-1654-422d-cad2-a82158bd8880"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "tkzr = TweetTokenizer(preserve_case=False)\n",
    "eng_stopwords = (\n",
    "    'what', 'which', 'who', 'whom', \n",
    "    'this', 'that', 'these', 'those', \n",
    "    'am', 'is', 'are', 'was', 'were', \n",
    "    'be', 'been', 'being', \n",
    "    'have', 'has', 'had', 'having', \n",
    "    'do', 'does', 'did', 'doing', \n",
    "    'a', 'an', 'the', \n",
    "    'and', 'but', 'if', 'or', \n",
    "    'because', 'as', 'until', 'while', \n",
    "    'of', 'at', 'by', 'for', 'with', \n",
    "    'about', 'against', 'between', \n",
    "    'into', 'through', 'during', 'before', 'after', \n",
    "    'above', 'below', 'to', 'from', \n",
    "    'up', 'down', 'in', 'out', 'on', 'off', \n",
    "    'over', 'under', 'again', 'further', \n",
    "    'then', 'once', 'here', \n",
    "    'there', 'when', 'where', 'why', \n",
    "    'how', 'all', 'any', 'both', 'each', \n",
    "    'few', 'more', 'most', 'other', 'some', \n",
    "    'such', 'no', 'nor', 'not', 'only', \n",
    "    'own', 'same', 'so', 'than', 'too', 'very', \n",
    "    'can', 'will', 'just', 'don', 'should', 'now'\n",
    ")\n",
    "lmtzr = WordNetLemmatizer()\n",
    "appos = {\n",
    "    \"aren't\" : \"are not\",\n",
    "    \"can't\" : \"cannot\",\n",
    "    \"couldn't\" : \"could not\",\n",
    "    \"didn't\" : \"did not\",\n",
    "    \"doesn't\" : \"does not\",\n",
    "    \"don't\" : \"do not\",\n",
    "    \"hadn't\" : \"had not\",\n",
    "    \"hasn't\" : \"has not\",\n",
    "    \"haven't\" : \"have not\",\n",
    "    \"he'd\" : \"he would\",\n",
    "    \"he'll\" : \"he will\",\n",
    "    \"he's\" : \"he is\",\n",
    "    \"i'd\" : \"I would\",\n",
    "    \"i'd\" : \"I had\",\n",
    "    \"i'll\" : \"I will\",\n",
    "    \"i'm\" : \"I am\",\n",
    "    \"isn't\" : \"is not\",\n",
    "    \"it's\" : \"it is\",\n",
    "    \"it'll\":\"it will\",\n",
    "    \"i've\" : \"I have\",\n",
    "    \"let's\" : \"let us\",\n",
    "    \"mightn't\" : \"might not\",\n",
    "    \"mustn't\" : \"must not\",\n",
    "    \"shan't\" : \"shall not\",\n",
    "    \"she'd\" : \"she would\",\n",
    "    \"she'll\" : \"she will\",\n",
    "    \"she's\" : \"she is\",\n",
    "    \"shouldn't\" : \"should not\",\n",
    "    \"that's\" : \"that is\",\n",
    "    \"there's\" : \"there is\",\n",
    "    \"they'd\" : \"they would\",\n",
    "    \"they'll\" : \"they will\",\n",
    "    \"they're\" : \"they are\",\n",
    "    \"they've\" : \"they have\",\n",
    "    \"we'd\" : \"we would\",\n",
    "    \"we're\" : \"we are\",\n",
    "    \"weren't\" : \"were not\",\n",
    "    \"we've\" : \"we have\",\n",
    "    \"what'll\" : \"what will\",\n",
    "    \"what're\" : \"what are\",\n",
    "    \"what's\" : \"what is\",\n",
    "    \"what've\" : \"what have\",\n",
    "    \"where's\" : \"where is\",\n",
    "    \"who'd\" : \"who would\",\n",
    "    \"who'll\" : \"who will\",\n",
    "    \"who're\" : \"who are\",\n",
    "    \"who's\" : \"who is\",\n",
    "    \"who've\" : \"who have\",\n",
    "    \"won't\" : \"will not\",\n",
    "    \"wouldn't\" : \"would not\",\n",
    "    \"you'd\" : \"you would\",\n",
    "    \"you'll\" : \"you will\",\n",
    "    \"you're\" : \"you are\",\n",
    "    \"you've\" : \"you have\",\n",
    "    \"'re\": \" are\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'll\":\" will\",\n",
    "    \"didn't\": \"did not\"\n",
    "}\n",
    "\n",
    "def preprocess(comment):\n",
    "  \n",
    "    # credit to the author of this post:\n",
    "    # https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda\n",
    "\n",
    "    # remove special format\n",
    "    comment = re.sub('\\n\\t', '', comment)\n",
    "\n",
    "    # remove IP addresses\n",
    "    comment = re.sub('\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}', ' specipaddress ', comment)\n",
    "\n",
    "    # remove username\n",
    "    comment = re.sub(\"\\[\\[User.*\\]\", ' specusername ', comment)\n",
    "    comment = re.sub(\"\\[\\[User.*\\|\", ' specusername ', comment)\n",
    "\n",
    "    # tokenization \n",
    "    tokens = tkzr.tokenize(comment)\n",
    "\n",
    "    # aphostophe replacement\n",
    "    tokens = [ appos[token] if token in appos else token for token in tokens]\n",
    "\n",
    "    # remove stopwords\n",
    "    tokens = [ token for token in tokens if not token in eng_stopwords ]\n",
    "\n",
    "    # stemming\n",
    "    tokens = [ lmtzr.lemmatize(token, 'v') for token in tokens]\n",
    "\n",
    "    return \" \".join(tokens)\n",
    "  \n",
    "\n",
    "hint('Cleaning train set...')\n",
    "Xtr = train['comment_text'].apply(lambda c: preprocess(c))\n",
    "hint('Cleaning test set...')\n",
    "Xva = valid['comment_text'].apply(lambda c: preprocess(c))\n",
    "hint('Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "btLphIdFwmts"
   },
   "source": [
    "## 2.2 Transforming Comments to Sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 144,
     "status": "ok",
     "timestamp": 1518570698059,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "JmEQQZs-kHK2",
    "outputId": "073af561-39f5-4c18-acee-ab2794e9e3a6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "vocab_max = 100000\n",
    "\n",
    "hint('Fitting the tokenizer...')\n",
    "tokenizer = ktxt.Tokenizer(num_words=vocab_max)\n",
    "tokenizer.fit_on_texts(Xtr)\n",
    "\n",
    "hint('Tokenizing...')\n",
    "Xtr = tokenizer.texts_to_sequences(Xtr)\n",
    "Xva = tokenizer.texts_to_sequences(Xva)\n",
    "\n",
    "hint('padding the sequences...')\n",
    "max_comment_length = 200  # padded/cropped comment length\n",
    "Xtr = sequence.pad_sequences(Xtr, maxlen=max_comment_length)\n",
    "Xva = sequence.pad_sequences(Xva, maxlen=max_comment_length)\n",
    "\n",
    "hint('Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OWHeGxuz3Wfn"
   },
   "source": [
    "# 3. Training Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "hint(\"Loading pre-embedding file...\")\n",
    "emb = pd.read_table(\n",
    "    'preembedding/glove.6B.300d.txt', \n",
    "    sep=\" \", index_col=0, header=None, quoting=csv.QUOTE_NONE\n",
    ")\n",
    "\n",
    "hint(\"Preparing embedding matrix...\")\n",
    "embedding_dimension = 300\n",
    "embedding_matrix = np.random.normal(\n",
    "    emb.mean(axis=0), \n",
    "    emb.std(axis=0), \n",
    "    (vocab_max, embedding_dimension)\n",
    ")\n",
    "hint(\"Constructing embedding matrix\")\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    if i < vocab_max and word in emb.index:\n",
    "        embedding_matrix[i] = emb.loc[word].as_matrix()\n",
    "\n",
    "hint(\"Done\")\n",
    "# optional: free memory:\n",
    "emb = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 374,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 389,
     "status": "ok",
     "timestamp": 1518570698678,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "GoawhjBA3nyc",
    "outputId": "e814cec6-a98e-4cce-da53-d2579963c70f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 200, 300)          30000000  \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_2 (Spatial (None, 200, 300)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_3 (Conv1D)            (None, 200, 256)          230656    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1 (None, 100, 256)          0         \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_3 (Spatial (None, 100, 256)          0         \n",
      "_________________________________________________________________\n",
      "bidirectional_3 (Bidirection (None, 100, 128)          123264    \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_4 (Spatial (None, 100, 128)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_4 (Conv1D)            (None, 100, 64)           24640     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1 (None, 50, 64)            0         \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_5 (Spatial (None, 50, 64)            0         \n",
      "_________________________________________________________________\n",
      "bidirectional_4 (Bidirection (None, 50, 64)            18624     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_2 (Glob (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 6)                 390       \n",
      "=================================================================\n",
      "Total params: 30,397,574\n",
      "Trainable params: 30,397,574\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = None\n",
    "model = Sequential()\n",
    "model.add(Embedding(\n",
    "    vocab_max, \n",
    "    embedding_dimension, \n",
    "    weights=[embedding_matrix],\n",
    "    input_length=max_comment_length\n",
    "))\n",
    "model.add(SpatialDropout1D(0.5))\n",
    "model.add(Conv1D(filters=256, kernel_size=3, padding='same', activation='relu'))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(SpatialDropout1D(0.2))\n",
    "model.add(Bidirectional(GRU(units=64, return_sequences=True)))\n",
    "model.add(SpatialDropout1D(0.5))\n",
    "model.add(Conv1D(filters=64, kernel_size=3, padding='same', activation='relu'))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(SpatialDropout1D(0.2))\n",
    "model.add(Bidirectional(GRU(units=32, return_sequences=True)))\n",
    "model.add(GlobalMaxPooling1D())\n",
    "model.add(Dense(len(labels), activation='sigmoid'))\n",
    "model.compile(\n",
    "    optimizer='adam', \n",
    "    loss='binary_crossentropy', \n",
    "    metrics=['accuracy']\n",
    ")\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1S6kDCqDFGUN"
   },
   "source": [
    "Now training the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 122,
     "output_extras": [
      {
       "item_id": 373
      },
      {
       "item_id": 1155
      },
      {
       "item_id": 1418
      },
      {
       "item_id": 2339
      },
      {
       "item_id": 2340
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 630277,
     "status": "ok",
     "timestamp": 1518571329571,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "QVU9nuDzFtlf",
    "outputId": "e0ea2b97-fe62-42bf-a1b2-46c7d5f08532"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/3\n",
      "127656/127656 [==============================] - 474s 4ms/step - loss: 0.1732 - acc: 0.9764 - val_loss: 0.0502 - val_acc: 0.9817\n",
      "Epoch 2/3\n",
      "127656/127656 [==============================] - 472s 4ms/step - loss: 0.1238 - acc: 0.9817 - val_loss: 0.0483 - val_acc: 0.9822\n",
      "Epoch 3/3\n",
      "127656/127656 [==============================] - 472s 4ms/step - loss: 0.1143 - acc: 0.9828 - val_loss: 0.0500 - val_acc: 0.9820\n"
     ]
    }
   ],
   "source": [
    "epochs = 3\n",
    "batch_size = 64\n",
    "\n",
    "def get_class_weight(x):\n",
    "    k = 100\n",
    "    return 3.32*np.log(k/x + 1)\n",
    "    \n",
    "\n",
    "history = model.fit(\n",
    "    Xtr, Ytr, \n",
    "    epochs=epochs, \n",
    "    batch_size=batch_size,\n",
    "    validation_data=(Xva, Yva),\n",
    "    class_weight={\n",
    "        0: get_class_weight(98),\n",
    "        1: get_class_weight(10),\n",
    "        2: get_class_weight(53),\n",
    "        3: get_class_weight(2),\n",
    "        4: get_class_weight(49),\n",
    "        5: get_class_weight(8),\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nAFiq9QuFI7R"
   },
   "source": [
    "Making prediction on the validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 121,
     "status": "ok",
     "timestamp": 1518571356080,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "hgiIMLjZE3Rr",
    "outputId": "f00c0add-a7ab-43f7-b75d-c47a54e12890"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "hint(\"Making prediction...\")\n",
    "Yva_ = model.predict(Xva)\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RfRNkLv7v04I"
   },
   "source": [
    "# 4. Result Analysis\n",
    "## 4.1 Global Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 255,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 275,
     "status": "ok",
     "timestamp": 1518571356419,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "vakvDUO31-oo",
    "outputId": "5b7acf10-a987-41f7-801c-0f291245b687"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation set sample count: 31915\n",
      "\n",
      "accuracy for threshold 0.1: 96.31%\n",
      "accuracy for threshold 0.2: 97.25%\n",
      "accuracy for threshold 0.3: 97.67%\n",
      "accuracy for threshold 0.4: 97.96%\n",
      "accuracy for threshold 0.5: 98.14%\n",
      "accuracy for threshold 0.6: 98.24%\n",
      "accuracy for threshold 0.7: 98.26%\n",
      "accuracy for threshold 0.8: 98.19%\n",
      "accuracy for threshold 0.9: 97.87%\n",
      "\n",
      "best threshold: 0.7\n",
      "best accuracy: 98.26%\n"
     ]
    }
   ],
   "source": [
    "total_sample = Xva.shape[0]\n",
    "print(\"validation set sample count: %d\\n\" % total_sample)\n",
    "prediction_total = total_sample*Yva.shape[1]\n",
    "best_t = None\n",
    "best_accuracy = 0\n",
    "for t in [i*0.1 for i in range(1, 10)]:\n",
    "    accuracy = np.sum(Yva == (Yva_ >= t))/prediction_total\n",
    "    if accuracy > best_accuracy: \n",
    "        best_t = t\n",
    "        best_accuracy = accuracy\n",
    "    print(\"accuracy for threshold %.1f: %.2f%%\" % (t, accuracy*100))\n",
    "Yva_T = Yva_ >= best_t\n",
    "correct = Yva == Yva_T\n",
    "print(\"\\nbest threshold: %.1f\" % best_t)\n",
    "print(\"best accuracy: %.2f%%\" % (best_accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YSptfDNc58dZ"
   },
   "source": [
    "## 4.2 Accuracy by Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "gLRH9AayowXA"
   },
   "outputs": [],
   "source": [
    "overview = pd.DataFrame(index=[\n",
    "    'label‰ of all',\n",
    "    'total wrong', \n",
    "    'P->N', \n",
    "    'N->P', \n",
    "    'P->N %', \n",
    "    'N->P %',\n",
    "    'avg len',\n",
    "])\n",
    "\n",
    "def analyze_class(i):\n",
    "    wrong = valid[correct[:, i] != 1]\n",
    "    total_class_error = len(wrong)\n",
    "    print(\"%d predicted incorrectly (%.2f%% of all samples)\" % (\n",
    "        total_class_error, \n",
    "        100*total_class_error/total_sample\n",
    "    ))\n",
    "        \n",
    "    wrong_seqs = Xva[correct[:, i] != 1]\n",
    "    lens = [ len(seq[seq != 0]) for seq in wrong_seqs]\n",
    "    avg_len = np.mean(lens)\n",
    "    print(\"Falsely predicted sequences have an average length of %d\" % avg_len)\n",
    "\n",
    "    PpN = valid[(Yva[:, i] == 1) & (Yva_T[:, i] == 0)]\n",
    "    PpN_count = len(PpN)\n",
    "    print(\"\\n%d (%.2f%%) positive label were predicted to be negative\" % (\n",
    "        PpN_count, \n",
    "        100*PpN_count/total_class_error \n",
    "    ))\n",
    "    if PpN_count > 4:\n",
    "        print(\"Samples:\")\n",
    "        for sample in PpN.sample(5)['comment_text']:\n",
    "            display(sample)\n",
    "  \n",
    "    NpP = valid[(Yva[:, i] == 0) & (Yva_T[:, i] == 1)]\n",
    "    NpP_count = len(NpP)\n",
    "    print(\"\\n%d (%.2f%%) negative label were predicted to be positive\" % (\n",
    "        NpP_count, \n",
    "        100*NpP_count/total_class_error \n",
    "    ))\n",
    "    if NpP_count > 4:\n",
    "        print(\"Samples:\")\n",
    "        for sample in NpP.sample(5)['comment_text']:\n",
    "            display(sample)\n",
    "  \n",
    "    overview[labels[i]] = [\n",
    "        np.mean(Yva[:, i]*1000),\n",
    "        total_class_error, \n",
    "        PpN_count,  \n",
    "        NpP_count,\n",
    "        100*PpN_count/total_class_error,\n",
    "        100*NpP_count/total_class_error,\n",
    "        avg_len\n",
    "    ]\n",
    "  \n",
    "    print('\\n')\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SSY9H7knpE5U"
   },
   "source": [
    "### 4.2.1 Toxic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 360,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      },
      {
       "item_id": 8
      },
      {
       "item_id": 9
      },
      {
       "item_id": 10
      },
      {
       "item_id": 11
      },
      {
       "item_id": 12
      },
      {
       "item_id": 13
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 251,
     "status": "ok",
     "timestamp": 1518573461778,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "NNGw690DpLMy",
    "outputId": "18e4b55e-63d1-42b2-9c93-4b75a8d5ad39"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1186 predicted incorrectly (3.72% of all samples)\n",
      "Falsely predicted sequences have an average length of 36\n",
      "\n",
      "926 (78.08%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\"\\n\\n Warning about your behavior \\n\\nCush, you had a conniption when I used an off-color term to describe you on my own talk page.  I was blocked for it for 24 hours.  But as the following diffs show, you are regularly abusive, foul-mouthed, and bigoted, and your refusal to WP:AGF and denigration of other editors has been going on for too long.  I\\'m asking you to stop.  Accept the fact that your opinion about religion is your opinion, and doesn\\'t have any special standing on Wikipedia.\\n\\nThese diffs are a partial list.  I got tired of going back through your history.  But it\\'s easy enough to extend the list.  Change your behavior and your attitude, please.\\n\\n 20:25, 19 August 2009 \"\"Lisa only suggests this name change to keep out material she does not like due to her religionist disposition\"\"\\n 07:08, 20 August 2009 \"\"Lisa is widely known to manipulate articles to render a religion-friendly POV. She has been warned about that so many times it ain\\'t even funny anymore.\"\" (this was a fabrication)\\n 15:40, 20 August 2009 \"\"when it comes to historicity religiously motivated publishers are in a COI\"\"\\n 20:37, 20 August 2009 \"\"Reliable sources for the historicity of anything are scientific sources. That excludes all publications by religious institutions or religiously motivated individuals\"\"\\n 18:10, 9 September 2009 \"\"anyone who has really read the bible and has an IQ above 5 will instantly recognize that it is a big fat lie\"\", \"\"religionists like you who may have \"\"reliable sources\"\" on their side but just no common sense or logic\"\"\\n 05:09, 10 September 2009 \"\"you fail to admit your COI when it comes to biblical history\"\"\\n 18:34, 18 September 2009 \"\"Religious people are in a COI when it comes to historical accuracy\"\"\\n 14:18, 27 September 2009 \"\"nutjob who ever wrote about this\"\"\\n 12:42, 4 October 2009 \"\"there is no such thing as a \\'Jewish people\\'\"\"\\n 16:37, 4 October 2009 \"\"religionist POV\"\"\\n 20:31, 9 October 2009 Personal attack\\n 09:34, 19 October 2009 \"\"Welcome to Wikipedia and its many endorsements of Jewish POVs\"\"\\n 17:09, 26 October 2009 \"\"Divine intolerance towards dissent\"\"\\n 14:44, 30 October 2009 \"\"digging up very old shit\"\"\\n 21:25, 2 November 2009 \"\"Read the fucking Bible\"\"\\n 13:42, 3 November 2009 \"\"Lisa is always Lisa\"\", \"\"Your Lisa-fundamentalism shows. As usual\"\"\\n\\n-  \"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Current event???? \\n\\nwtf is the current event about?'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\n Yeah, report me \\n\\nHave fun reporting me, then I won\\'t need to do it. Please be so kind and cite the following diffs to the admins, will you:  (\"\"leave the real work to real men, not clowns\"\");  (\"\"You think your funny, pair of Clowns, probably responsible for the Hitler redirect vandalism\"\");  (\"\"Just because you get easily confused by logic...\"\");  (\"\"sheer ignorance\"\");  (\"\"stop being such a proud moron.\"\"). BTW, I\\'m not currently an administrator.  ☼ \\n\\n(editconflict)Regardless, I am listing the diffs of the context because that doesn\\'t give you the right to abuse your authority and (no, \"\"Capital of Macedonia\"\" does not redirect here. Why would it?) lie, changing the redirect so that your statement would then be true. I can show the history with the diffs that you have been needlessly impeding and useless, basically provoking and abusing your wrongly given authority. So laugh all your want, adminship isn\\'t permanent it comes under review and lies count.  \\n\\n Oh, and BTW: some people might consider that our dispute over the dablink falls under the scope of this 1RR restriction, so we probably ought to be careful about reverting. I don\\'t think either of us has broken it though, at least not today.  ☼ \"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\nStupid Vikings.  A creative, new way to lose. Instead of a run play to center the ball and kick the winning FG with time expiring, let\\'s have the ol\\' man throw across his body to his blind side and get picked off and force OT and hand the game to the Aints.  Creative.  As I\\'ve always maintain, the vikings always find a way to lose.   | 76 \"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"I think he's a prick.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "260 (21.92%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'/* Im gay and im Proud */  \\n\\nListen every one i recently came out of the closet and came out as a homosexual and i have just told my parents this ....HOOORRAY BEER!!!!!!!!\\n\\n The VaNdAl'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\n\\nWould you just do me a favor though and stop harassing me! I have reported you, and there will be something coming to you soon. I don\\'t care that you have the \"\"Stupid Original Barnstar\"\", but that doesn\\'t mean that you can harass me.\"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'PHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!\\nPHUQ OFF YOU KNOB OF CHEESE!\\n\\nSALT!'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'And you should also wait for people to tell what you are known for. After all what we are known for is something that belongs to others thoughts not ours. You should make a survey. Maybe it turns out that you are not known for viciously attacking stupidity but for your stupid vicious attacks. 99.231.81.164'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'dude you guys must be on crack! instead of blocking those idiots for trying to delete a page without legitimate reason you blocked me out? with morons like you Wikipedia will not survive mark my words!  \\n\\ni could not care less for my account any more as i will boycott this madness of a site'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3cxSNKFepEg1"
   },
   "source": [
    "### 4.2.2 Severe Toxic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 360,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      },
      {
       "item_id": 8
      },
      {
       "item_id": 9
      },
      {
       "item_id": 10
      },
      {
       "item_id": 11
      },
      {
       "item_id": 12
      },
      {
       "item_id": 13
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 254,
     "status": "ok",
     "timestamp": 1518573463811,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "1cCGMbZKpV1H",
    "outputId": "428d69ea-64a9-4f28-e09c-f9ddd6d7516c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "321 predicted incorrectly (1.01% of all samples)\n",
      "Falsely predicted sequences have an average length of 28\n",
      "\n",
      "319 (99.38%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"by the way\\nthis is a university computer.. so go ahead and block it cock sucker. i'll just use my home computer to let the world know what a stupid asshole you are. FUCK OFFFF DICKBAG.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'DAT CHICK \\n\\nOMG I DIDNT VANDALISE NUFFINK YH??? JUST EXPRESSIN MY OPINION AND MAKIN CONVERSATION INNIT. GET A LIFE, GET A JOB, FK U ENNIT.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"You Are a fuck head I'm not a vandal so suck cock ass master.  We'll talk after your done with your Mom.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'oh, and armarkov, you fuck and are a cocksucker.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'DOOSH DOOSH DOOSH DOOSH DOOSH FUCKING COCK SUCKER QUEIR BATE ASS WIPE DONT TELL ME WHAT THE FUCK TO WRITE'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2 (0.62%) negative label were predicted to be positive\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "s0_zIe7m1A79"
   },
   "source": [
    "### 4.2.3 Obscene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 360,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      },
      {
       "item_id": 8
      },
      {
       "item_id": 9
      },
      {
       "item_id": 10
      },
      {
       "item_id": 11
      },
      {
       "item_id": 12
      },
      {
       "item_id": 13
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 258,
     "status": "ok",
     "timestamp": 1518573465290,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "Qt9vOoNK1EFl",
    "outputId": "cfeec26f-71ad-4668-840e-c183a862ba74"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "622 predicted incorrectly (1.95% of all samples)\n",
      "Falsely predicted sequences have an average length of 34\n",
      "\n",
      "419 (67.36%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"Why do you keep blocking me? \\n\\nYou are a big as idiot as DMacks now.  You cant block me as I just change my IP.  Why don't you try and negotiate with me and i will stop editing your page?   I WANT DMACKS PRIVELAGES REMOVED!!!\\n\\nBTW there is no threat so dont even try to turn it on me\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'complete bullshit - your past is catching up with you kurt  after you had defamed as many users as possible and spammed out the usenet groups it appears you moved over to wikipedia where hitherto you have remained unchallenged - that is up until now.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Piss off you evil dwarf'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////IDIOT/////\\n\\nGHATS WHAT YOU ARE'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'A proposal to cut down on AIDS \\n\\nLess AIDS would be spread if you were to stop inserting your minuscule penis into little boys.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "203 (32.64%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"Junior Barranquilla?\\n\\nWTF? how did you come up with that name Junior Barranquilla? should we call REAL MADRID... ROYAL MADRID??????????????  pleasse push for a revert!!! don't come up with that made up bullshit name! \\n\\nAtlético Junior is the common name!!! no one calls it Junior Barranquilla... maybe Junior de Barranquilla, but never something such name OMG.. that was just product of pure ignorance.. Cheers 76.19.149.244\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Because it gives you diarrhea and YOU SPELLS DIARRHEA WRONG!!!!\\n):):):('"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'YOU NEED SUM PUSSY GO PAY A GIRL SHE WILL TAKE CARE OF YOU, I SAY THIS BECUASE MOST PPL HERE ARE DORKS AND VIRGINS THATS NEVER EVEN TOUCHED THE A FEMALE'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"Another idiot admin who doesn't know the policy of Wikipedia. Good job, Swarm, you stupidly blocked me for returning the page to the last consensus. That's what we're supposed to do in the case of a content dispute. Yes, I know, that is so hard to learn! Another moron example of how admins are the sickness of Wikipedia.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\n\\n \"\"What\\'s a ballsy?\"\" \\n\\nShe really can\\'t be that stupid....can she???\"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Lwpco7aB1GrZ"
   },
   "source": [
    "### 4.2.4 Threat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 258,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 257,
     "status": "ok",
     "timestamp": 1518573466922,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "gQgcRgOq1GLO",
    "outputId": "69893ed6-1588-41db-eff3-e86766fac8e6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87 predicted incorrectly (0.27% of all samples)\n",
      "Falsely predicted sequences have an average length of 26\n",
      "\n",
      "69 (79.31%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'(http://wiki.uschallengecoins.com/Main_Page)\\n\\nIF YOU REVERT MY FUCKIN EDITS METROS I WILL KILL YOU!'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"Fuck you, like I give a shit. Point to the word where I cast suspicion on her. Point to it. Point to it you stupid asshole. But you can't. Go fuck yourself. I hope you die.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Fuck you cunt \\n\\n \\nHow fucking dare u judge me cunt. Dont fucking stand up for telstra fucker. You small cock fag. ILl cum on your grave and hope that shit rains down it. Fucker. Fuck I hope you die. Fuck i want you dead. WHy lefty. Do you use your left hand to wank cunt? Fucking wanker. Oww i just came.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"Personal Attack Number 2 \\n\\nThis is another personal attack about you being a massive donkey dick sucking homosexual. This is a concerned plea that you should at once drown yourself in a sewer. Fucking fuck off you fag! And if you threaten me with legal action again, I'll fucking come over there, tie you to your computer chair and stick my cock up your arse until it bleeds. You fucker!!\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'YOU ARE A FUCKING NERD. GO KILL YOURSELF YOU NERD.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "18 (20.69%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"blow me\\n\\nIts my right of freedom of speech to edit wikipedia. So fuck off or I'll rape you anally and vaginally. Good day. 78.148.200.177\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'I am going to shoot you in the head and laugh as your brains splatter onto the ground. \\n\\nI am going to shoot you in the head and laugh as your brains splatter onto the ground.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'I hope you die!==\\nFind a pit and jump into it. <3\\n\\n=='"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Hi! \\n\\nI wanna rape you!'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"I'ma smack ya upside da head wit a shovel.  \\n\\nI'm takin ya down, boi.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6lkRScLJ1fGN"
   },
   "source": [
    "### 4.2.5 Insult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 360,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      },
      {
       "item_id": 8
      },
      {
       "item_id": 9
      },
      {
       "item_id": 10
      },
      {
       "item_id": 11
      },
      {
       "item_id": 12
      },
      {
       "item_id": 13
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 270,
     "status": "ok",
     "timestamp": 1518573480301,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "zJ2VKxgK1hRC",
    "outputId": "68a3f353-831b-426f-85dc-9d65a39561cd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "856 predicted incorrectly (2.68% of all samples)\n",
      "Falsely predicted sequences have an average length of 31\n",
      "\n",
      "450 (52.57%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"You must have the lowest IQ of anyone on wikipedia, I can't converse with a fool like you\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'amelia earhart was a prostitute that live on broadway.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\nThank you , and the same to you. worm \"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Red herrings to continue to excuse Breivik.  Go away, nasty person.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"Considering the fact that I already had a warning about the nigger article, you're acting like I called you a 'cunt' or something. Seriously, it's funny the way you act.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "406 (47.43%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"As far as Disputive Edits are concerned You Idiot I didn't made any. Just made a request to delete the Images. Keep your temper to yourself.\\nThank You.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'WA NKER ALERT WA NKER ALERT\\n\\nif i find out where you or james whale are i promise you i will fucking smack the shit out of you'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'to munch pussy and not know how to spell good in English, because I am fucking foreigner for who with which English is the hard language I am dyke.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\n\\n\\nAre you some sort of idiot??\\n\"\"If you don\\'t want the warning messages, then stop adding unsourced information.\"\"\\nI just told you i didnt!\\nNow remove that rubbish attached to my ip because the orange box wont go away, like you\\n83.71.70.5  \"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\nIf you continue acting like a racist fool, you\\'ll be called a racist fool. You\\'re demonstrating abject anti-semitism, linking up to absurd \"\"proof\"\" of your racist theories. Being a racist fool isn\\'t civil, and you are definitely behaving like one right now.\"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ybjmae651jxY"
   },
   "source": [
    "### 4.2.6 Identity Hate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 258,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 440,
     "status": "ok",
     "timestamp": 1518573482230,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "6ojSnXP51jPm",
    "outputId": "d1ee8ad8-5407-43c8-cab2-a5f77112fa2c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "252 predicted incorrectly (0.79% of all samples)\n",
      "Falsely predicted sequences have an average length of 30\n",
      "\n",
      "158 (62.70%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Haha. Go ahead. Stupid American.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Jewish \\n\\nPlease dont think that jews are everyone just based on your claims. Also dont think jews are smart. if you think so, you are making a clown out of yourself..'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Faggot \\n\\nEnough said'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"American mouths are way too faster than American brain ) You Americans need to learn to listen if you truly want to get out of the mess you're in.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"Hey Harry, You are a fucking fag. There is nothing wrong with that. You delete all my edits because you know they are correct but are much smarter than anything you can come up with. Ps, you're a cunt. 144.132.144.57\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "94 (37.30%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'He is not gay in Rugrats, but on the internet, he is.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'a crazy nerd who dislikes women'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Is it gay if you bang an animal of the same secks?'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'sup \\n\\nThanks for the anal sex yesterday!\\n\\nLove,\\nGeorge (your homosexual lover)'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'By the way, are there any Azeris looking like this guy. Having lived among the Jews for a long time I can declare that this guy is a Jew. He is a typical Gorsky Jew.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GYOBKlWc12Rc"
   },
   "source": [
    "### 4.2.7 Overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 235,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 247,
     "status": "ok",
     "timestamp": 1518573484262,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "w9g39VJ4n00f",
    "outputId": "8e4f9efb-795c-4253-de18-d7b2be471bb0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>label‰ of all</th>\n",
       "      <td>96</td>\n",
       "      <td>10</td>\n",
       "      <td>52</td>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total wrong</th>\n",
       "      <td>1186</td>\n",
       "      <td>321</td>\n",
       "      <td>622</td>\n",
       "      <td>87</td>\n",
       "      <td>856</td>\n",
       "      <td>252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P-&gt;N</th>\n",
       "      <td>926</td>\n",
       "      <td>319</td>\n",
       "      <td>419</td>\n",
       "      <td>69</td>\n",
       "      <td>450</td>\n",
       "      <td>158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>N-&gt;P</th>\n",
       "      <td>260</td>\n",
       "      <td>2</td>\n",
       "      <td>203</td>\n",
       "      <td>18</td>\n",
       "      <td>406</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P-&gt;N %</th>\n",
       "      <td>78</td>\n",
       "      <td>99</td>\n",
       "      <td>67</td>\n",
       "      <td>79</td>\n",
       "      <td>52</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>N-&gt;P %</th>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>20</td>\n",
       "      <td>47</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg len</th>\n",
       "      <td>36</td>\n",
       "      <td>28</td>\n",
       "      <td>34</td>\n",
       "      <td>26</td>\n",
       "      <td>31</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
       "label‰ of all     96            10       52       2      49              9\n",
       "total wrong     1186           321      622      87     856            252\n",
       "P->N             926           319      419      69     450            158\n",
       "N->P             260             2      203      18     406             94\n",
       "P->N %            78            99       67      79      52             62\n",
       "N->P %            21             0       32      20      47             37\n",
       "avg len           36            28       34      26      31             30"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overview.astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Conclusion\n",
    "The model did not perform as well as RCNN v2. Adding more layers is difficult to be done right. Since other approaches proved to work better, I will leave this approach for later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "default_view": {},
   "name": "RCNN performance analysis.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}