Pooled RCNNv2 submission.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is made for producing submission for Kaggle. Analysis steps will be skipped. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "\n",
    "def hint(message):\n",
    "    \"\"\"\n",
    "    erase previous ipynb output and show new message\n",
    "    \"\"\"\n",
    "    clear_output()\n",
    "    print(message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "hint(\"loading data...\")\n",
    "train = pd.read_csv('data/train.csv')\n",
    "test = pd.read_csv('data/test.csv')\n",
    "\n",
    "labels = [\n",
    "    'toxic', \n",
    "    'severe_toxic', \n",
    "    'obscene', \n",
    "    'threat', \n",
    "    'insult', \n",
    "    'identity_hate'\n",
    "]\n",
    "\n",
    "Y = train[labels].values\n",
    "\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Pre-processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gather resources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\ChuanLi\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "import nltk\n",
    "\n",
    "nltk.download('wordnet')\n",
    "\n",
    "lmtzr = WordNetLemmatizer()\n",
    "eng_stopwords = (\n",
    "    'what', 'which', 'who', 'whom', \n",
    "    'this', 'that', 'these', 'those', \n",
    "    'am', 'is', 'are', 'was', 'were', \n",
    "    'be', 'been', 'being', \n",
    "    'have', 'has', 'had', 'having', \n",
    "    'do', 'does', 'did', 'doing', \n",
    "    'a', 'an', 'the', \n",
    "    'and', 'but', 'if', 'or', \n",
    "    'because', 'as', 'until', 'while', \n",
    "    'of', 'at', 'by', 'for', 'with', \n",
    "    'about', 'against', 'between', \n",
    "    'into', 'through', 'during', 'before', 'after', \n",
    "    'above', 'below', 'to', 'from', \n",
    "    'up', 'down', 'in', 'out', 'on', \n",
    "    'over', 'under', 'again', 'further', \n",
    "    'then', 'once', 'here', \n",
    "    'there', 'when', 'where', 'why', \n",
    "    'how', 'all', 'any', 'both', 'each', \n",
    "    'few', 'more', 'most', 'other', 'some', \n",
    "    'such', 'nor', 'only', \n",
    "    'own', 'same', 'so', 'than', 'too', 'very', \n",
    "    'can', 'will', 'just', 'don', 'should', 'now'\n",
    ")\n",
    "appos = {\n",
    "    \"aren't\" : \"are not\",\n",
    "    \"can't\" : \"cannot\",\n",
    "    \"couldn't\" : \"could not\",\n",
    "    \"didn't\" : \"did not\",\n",
    "    \"doesn't\" : \"does not\",\n",
    "    \"don't\" : \"do not\",\n",
    "    \"hadn't\" : \"had not\",\n",
    "    \"hasn't\" : \"has not\",\n",
    "    \"haven't\" : \"have not\",\n",
    "    \"he'd\" : \"he would\",\n",
    "    \"he'll\" : \"he will\",\n",
    "    \"he's\" : \"he is\",\n",
    "    \"i'd\" : \"I would\",\n",
    "    \"i'd\" : \"I had\",\n",
    "    \"i'll\" : \"I will\",\n",
    "    \"i'm\" : \"I am\",\n",
    "    \"isn't\" : \"is not\",\n",
    "    \"it's\" : \"it is\",\n",
    "    \"it'll\":\"it will\",\n",
    "    \"i've\" : \"I have\",\n",
    "    \"let's\" : \"let us\",\n",
    "    \"mightn't\" : \"might not\",\n",
    "    \"mustn't\" : \"must not\",\n",
    "    \"shan't\" : \"shall not\",\n",
    "    \"she'd\" : \"she would\",\n",
    "    \"she'll\" : \"she will\",\n",
    "    \"she's\" : \"she is\",\n",
    "    \"shouldn't\" : \"should not\",\n",
    "    \"that's\" : \"that is\",\n",
    "    \"there's\" : \"there is\",\n",
    "    \"they'd\" : \"they would\",\n",
    "    \"they'll\" : \"they will\",\n",
    "    \"they're\" : \"they are\",\n",
    "    \"they've\" : \"they have\",\n",
    "    \"we'd\" : \"we would\",\n",
    "    \"we're\" : \"we are\",\n",
    "    \"weren't\" : \"were not\",\n",
    "    \"we've\" : \"we have\",\n",
    "    \"what'll\" : \"what will\",\n",
    "    \"what're\" : \"what are\",\n",
    "    \"what's\" : \"what is\",\n",
    "    \"what've\" : \"what have\",\n",
    "    \"where's\" : \"where is\",\n",
    "    \"who'd\" : \"who would\",\n",
    "    \"who'll\" : \"who will\",\n",
    "    \"who're\" : \"who are\",\n",
    "    \"who's\" : \"who is\",\n",
    "    \"who've\" : \"who have\",\n",
    "    \"won't\" : \"will not\",\n",
    "    \"wouldn't\" : \"would not\",\n",
    "    \"you'd\" : \"you would\",\n",
    "    \"you'll\" : \"you will\",\n",
    "    \"you're\" : \"you are\",\n",
    "    \"you've\" : \"you have\",\n",
    "    \"'re\": \" are\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'll\":\" will\",\n",
    "    \"didn't\": \"did not\"\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define cleaning method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import TweetTokenizer\n",
    "import re\n",
    "\n",
    "tkzr = TweetTokenizer(preserve_case=False)\n",
    "\n",
    "def preprocess(comment):\n",
    "  \n",
    "    # credit to the author of this post:\n",
    "    # https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda\n",
    "    \n",
    "    # remove urls\n",
    "    comment = re.sub(r'https?://en.wikipedia.org/[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]', ' speclinkaddress ', comment)\n",
    "    comment = re.sub(r'https?://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]', ' speclinkaddress ', comment)\n",
    "    comment = re.sub(r'www.[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]', ' speclinkaddress ', comment)\n",
    "\n",
    "    # remove IP addresses\n",
    "    comment = re.sub(r'\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}', ' specipaddress ', comment)\n",
    "\n",
    "    # remove username\n",
    "    comment = re.sub(r\"\\[\\[User.*\\]\", ' specusername ', comment)\n",
    "    comment = re.sub(r\"\\[\\[User.*\\|\", ' specusername ', comment)\n",
    "    \n",
    "    # remove special characters\n",
    "    comment = re.sub(r'[^A-Za-z\\d\\' ]', '', comment)\n",
    "\n",
    "    # tokenization \n",
    "    tokens = comment.split()\n",
    "\n",
    "    # aphostophe replacement\n",
    "    tokens = [ appos[token] if token in appos else token for token in tokens]\n",
    "\n",
    "    # remove stopwords\n",
    "    tokens = [ token for token in tokens if not token in eng_stopwords ]\n",
    "\n",
    "    # stemming\n",
    "    tokens = [ lmtzr.lemmatize(token, 'v') for token in tokens]\n",
    "\n",
    "    return \" \".join(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "hint(\"Cleaning train set...\")\n",
    "X = train['comment_text'].apply(lambda c: preprocess(c))\n",
    "hint(\"Cleaning test set...\")\n",
    "X_ = test['comment_text'].apply(lambda c: preprocess(c))\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Making Sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing import text as ktxt, sequence\n",
    "\n",
    "vocab_max = 100000\n",
    "\n",
    "hint(\"Fitting the tokenizer...\")\n",
    "tokenizer = ktxt.Tokenizer(num_words=vocab_max)\n",
    "tokenizer.fit_on_texts(X)\n",
    "\n",
    "hint(\"Tokenizing...\")\n",
    "X = tokenizer.texts_to_sequences(X)\n",
    "X_ = tokenizer.texts_to_sequences(X_)\n",
    "\n",
    "hint(\"Padding the sequences...\")\n",
    "max_comment_length = 150  # padded/cropped comment length\n",
    "X = sequence.pad_sequences(X, maxlen=max_comment_length)\n",
    "X_ = sequence.pad_sequences(X_, maxlen=max_comment_length)\n",
    "\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Prepare Embedding Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "emb_file = 'preembedding/glove.840B.300d.txt'\n",
    "\n",
    "hint(\"Loading pre-embedding file...\")\n",
    "emb = pd.read_table(emb_file, \" \", index_col=0, header=None, quoting=csv.QUOTE_NONE)\n",
    "\n",
    "hint(\"Preparing embedding matrix...\")\n",
    "embedding_dimension = 300\n",
    "embedding_matrix = np.random.normal(\n",
    "    emb.mean(axis=0), \n",
    "    emb.std(axis=0), \n",
    "    (vocab_max, embedding_dimension)\n",
    ")\n",
    "\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "hint(\"Constructing embedding matrix\")\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    if i < vocab_max and word in emb.index:\n",
    "        embedding_matrix[i] = emb.loc[word].as_matrix()\n",
    "\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "del emb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 150, 300)     30000000    input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_3 (SpatialDro (None, 150, 300)     0           embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 150, 256)     230656      spatial_dropout1d_3[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1D)  (None, 75, 256)      0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_4 (SpatialDro (None, 75, 256)      0           max_pooling1d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 75, 128)      123648      spatial_dropout1d_4[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_2 (Glo (None, 128)          0           bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 128)          0           bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 256)          0           global_average_pooling1d_2[0][0] \n",
      "                                                                 global_max_pooling1d_2[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 256)          0           concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 6)            1542        dropout_2[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 30,355,846\n",
      "Trainable params: 30,355,846\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Input, Dense, Embedding, CuDNNGRU, Dropout\n",
    "from keras.layers import SpatialDropout1D, Bidirectional, concatenate\n",
    "from keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D\n",
    "from keras.layers.convolutional import Conv1D, MaxPooling1D\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "sequence_input = Input(shape=(max_comment_length, ))\n",
    "x = Embedding(vocab_max, embedding_dimension, weights=[embedding_matrix])(sequence_input)\n",
    "x = SpatialDropout1D(0.5)(x)\n",
    "x = Conv1D(filters=256, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = MaxPooling1D(pool_size=2)(x)\n",
    "x = SpatialDropout1D(0.2)(x)\n",
    "x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n",
    "avg_pool = GlobalAveragePooling1D()(x)\n",
    "max_pool = GlobalMaxPooling1D()(x)\n",
    "x = concatenate([avg_pool, max_pool])\n",
    "preds = Dense(6, activation=\"sigmoid\")(x)\n",
    "model = Model(sequence_input, preds)\n",
    "model.compile(\n",
    "    loss='binary_crossentropy', \n",
    "    optimizer=Adam(lr=0.0004),\n",
    "    metrics=['accuracy']\n",
    ")\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "159571/159571 [==============================] - 65s 410us/step - loss: 0.0640 - acc: 0.9782\n",
      "Epoch 2/5\n",
      "159571/159571 [==============================] - 63s 393us/step - loss: 0.0475 - acc: 0.9822\n",
      "Epoch 3/5\n",
      "159571/159571 [==============================] - 64s 403us/step - loss: 0.0437 - acc: 0.9833\n",
      "Epoch 4/5\n",
      "159571/159571 [==============================] - 65s 409us/step - loss: 0.0409 - acc: 0.9842\n",
      "Epoch 5/5\n",
      "159571/159571 [==============================] - 66s 415us/step - loss: 0.0388 - acc: 0.9848\n"
     ]
    }
   ],
   "source": [
    "epochs = 5\n",
    "batch_size = 64\n",
    "\n",
    "history = model.fit(\n",
    "    X, Y, \n",
    "    epochs=epochs, \n",
    "    batch_size=batch_size\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "hint(\"Making prediction...\")\n",
    "Y_ = model.predict(X_)\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "hint(\"Uploading...\")\n",
    "file_name = 'submission_' + time.strftime(\"%Y%m%d-%H%M%S\") + '.csv'\n",
    "sumbit_id = pd.DataFrame({'id': test['id']})\n",
    "sumbit_labels = pd.DataFrame(Y_, columns=labels)\n",
    "submission = pd.concat([sumbit_id, sumbit_labels], axis=1)\n",
    "submission.to_csv(file_name, index=False)\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}