BiLstm_+_crf_for_chunking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "W4ECTMlQrjtp",
    "outputId": "c97b8cf5-a8e1-4206-f034-0b28d8c69fcf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow==1.15.0 in /usr/local/lib/python3.6/dist-packages (1.15.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /home/naveen/.local/lib/python3.6/site-packages (from tensorflow==1.15.0) (0.8.1)\n",
      "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0) (1.18.1)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /home/naveen/.local/lib/python3.6/site-packages (from tensorflow==1.15.0) (1.11.2)\n",
      "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0) (1.0.8)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /home/naveen/.local/lib/python3.6/site-packages (from tensorflow==1.15.0) (0.9.0)\n",
      "Requirement already satisfied: gast==0.2.2 in /home/naveen/.local/lib/python3.6/site-packages (from tensorflow==1.15.0) (0.2.2)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /home/naveen/.local/lib/python3.6/site-packages (from tensorflow==1.15.0) (3.1.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /home/naveen/.local/lib/python3.6/site-packages (from tensorflow==1.15.0) (1.14.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.6 in /home/naveen/.local/lib/python3.6/site-packages (from tensorflow==1.15.0) (0.1.8)\n",
      "Requirement already satisfied: tensorboard<1.16.0,>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0) (1.15.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /home/naveen/.local/lib/python3.6/site-packages (from tensorflow==1.15.0) (1.1.0)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /home/naveen/.local/lib/python3.6/site-packages (from tensorflow==1.15.0) (1.26.0)\n",
      "Requirement already satisfied: tensorflow-estimator==1.15.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0) (1.15.1)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0) (0.33.6)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15.0) (1.1.0)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /home/naveen/.local/lib/python3.6/site-packages (from tensorflow==1.15.0) (3.11.2)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow==1.15.0) (2.10.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /home/naveen/.local/lib/python3.6/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15.0) (3.1.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /home/naveen/.local/lib/python3.6/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15.0) (0.16.0)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /home/naveen/.local/lib/python3.6/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15.0) (45.1.0)\n",
      "\u001b[33mWARNING: You are using pip version 19.3.1; however, version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: keras==2.2.4 in /usr/local/lib/python3.6/dist-packages (2.2.4)\n",
      "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from keras==2.2.4) (1.4.1)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from keras==2.2.4) (1.1.0)\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from keras==2.2.4) (1.0.8)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras==2.2.4) (2.10.0)\n",
      "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from keras==2.2.4) (1.18.1)\n",
      "Requirement already satisfied: six>=1.9.0 in /home/naveen/.local/lib/python3.6/site-packages (from keras==2.2.4) (1.14.0)\n",
      "Requirement already satisfied: pyyaml in /usr/lib/python3/dist-packages (from keras==2.2.4) (3.12)\n",
      "\u001b[33mWARNING: You are using pip version 19.3.1; however, version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#! pip install tensorflow==1.15.0\n",
    "#! pip install keras==2.2.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 338
    },
    "colab_type": "code",
    "id": "zh9Sb_IMsrui",
    "outputId": "4065c105-62d8-4431-e7a9-3702643bf2a1"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import keras\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import copy\n",
    "\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Model, Input\n",
    "from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional\n",
    "from keras_contrib.layers import CRF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version:  1.15.0\n",
      "Keras version:  2.2.4\n"
     ]
    }
   ],
   "source": [
    "print(\"Tensorflow version: \", tf.__version__)\n",
    "print(\"Keras version: \", keras.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7vZwHzNyszq4"
   },
   "outputs": [],
   "source": [
    "# Hyperparams\n",
    "BATCH_SIZE = 512  # Number of examples used in each iteration\n",
    "EPOCHS = 5  # Number of passes through entire dataset\n",
    "MAX_LEN = 80  # Max length of review (in words)\n",
    "EMBEDDING = 200  # Dimension of word embedding vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "T-Fj-UyuuNDD",
    "outputId": "6a8a4e62-41f7-4920-8ee6-ad37c417aac4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Training sentences:\t 8936\n",
      "Number of Testing sentences:\t 2012\n"
     ]
    }
   ],
   "source": [
    "def transform_dataset(fileName):\n",
    "  f_new = open(\"updated_\" + fileName,\"w\")\n",
    "  f = open(fileName, \"r\")\n",
    "  sentenceNumber = 1\n",
    "  writeLineNumber = True\n",
    "  for x in f:\n",
    "    if x == \"\\n\":\n",
    "      sentenceNumber = sentenceNumber + 1\n",
    "    else:\n",
    "      x_list = x.split(\" \")\n",
    "      # uncomment the below block if you only want to train the model for Noun Phrase.\n",
    "      '''if x_list[-1] not in [\"B-NP\\n\", \"I-NP\\n\"]:\n",
    "        x_list[-1] = \"O\\n\"'''\n",
    "      x_new = \" \".join(x_list)\n",
    "      f_new.write(\"Sentence:\" + str(sentenceNumber) + \" \" + x_new )\n",
    "  return(sentenceNumber-1) # the file ends with a \\n\n",
    "\n",
    "\n",
    "print(\"Number of Training sentences:\\t\", transform_dataset(\"train.txt\"))\n",
    "print(\"Number of Testing sentences:\\t\", transform_dataset(\"test.txt\"))\n",
    "# open updated_train.txt and updated_test.txt to see the format to which it has been transformed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "EPDkg3Yzs2Nj",
    "outputId": "f7e91a78-ccbe-49fb-cf30-4a2b6e627bf2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words in the training dataset:\t19122\n",
      "Number of tags in the training dataset:\t 22\n",
      "The tags are  ['I-NP', 'B-CONJP', 'I-UCP', 'I-VP', 'I-ADJP', 'I-SBAR', 'O', 'B-VP', 'I-PP', 'I-INTJ', 'I-CONJP', 'B-NP', 'B-PP', 'B-ADJP', 'B-ADVP', 'B-LST', 'B-INTJ', 'I-PRT', 'B-PRT', 'B-UCP', 'B-SBAR', 'I-ADVP']\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv('updated_train.txt', sep=\" \", header=None)\n",
    "data = data.fillna(method=\"ffill\")\n",
    "\n",
    "data.columns = [\"sentence#\", \"word\", \"ner\", \"chunk\"]\n",
    "\n",
    "words = list(set(data[\"word\"].values))\n",
    "n_words = len(words)\n",
    "print(\"Number of words in the training dataset:\\t\" + str(n_words))\n",
    "tags = list(set(data[\"chunk\"].values))\n",
    "n_tags = len(tags)\n",
    "print(\"Number of tags in the training dataset:\\t\", n_tags)\n",
    "print(\"The tags are \", tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "ankjYUjmZt6F",
    "outputId": "42c0f5ad-77b3-409a-db80-0fe64061396d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words in the testing dataset: 8118\n",
      "Number of tags in the testing dataset:  19\n",
      "The tags are  ['I-NP', 'B-NP', 'B-PP', 'B-INTJ', 'B-LST', 'I-ADJP', 'I-LST', 'I-SBAR', 'B-ADJP', 'B-PRT', 'B-CONJP', 'O', 'B-ADVP', 'B-VP', 'B-SBAR', 'I-PP', 'I-ADVP', 'I-VP', 'I-CONJP']\n"
     ]
    }
   ],
   "source": [
    "data_test = pd.read_csv('updated_test.txt', sep=\" \", header=None)\n",
    "data_test = data_test.fillna(method=\"ffill\")\n",
    "\n",
    "data_test.columns = [\"sentence#\", \"word\", \"ner\", \"chunk\"]\n",
    "\n",
    "words_test = list(set(data_test[\"word\"].values))\n",
    "n_words_test = len(words_test)\n",
    "print(\"Number of words in the testing dataset: \" + str(n_words_test))\n",
    "tags_test = list(set(data_test[\"chunk\"].values))\n",
    "n_tags_test = len(tags_test)\n",
    "print(\"Number of tags in the testing dataset: \", n_tags_test)\n",
    "print(\"The tags are \", tags_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 400
    },
    "colab_type": "code",
    "id": "x3jqcW7bz2KL",
    "outputId": "7cd7f97a-03d9-4565-8f52-fbd08fa677b9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8936\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAdcklEQVR4nO3de5gdVZnv8e+PJBDuIaTNxCTaoBEOz1EC5nAZGEUCDhclKPdhIHJyJqLIRUclyjyCM54Z8DggIA9MADVwEOQmRO5MCIIyXDokhEAINJCY5CSkuSTcBobAe/6o1UXRdKd3d1J7V3f/Ps+zn65atarq3Zfe716rqlYpIjAzMwPYqNEBmJlZdTgpmJlZzknBzMxyTgpmZpZzUjAzs5yTgpmZ5ZwUrE+QdICk1kbHYdbfOSlY3Uh6vfB4T9J/FuaPbXR8fZWkHSWtbXQc1j8MbnQANnBExBbt05IWA/8rIv69cRGVS9LgiPCXtfUpbilYZUjaVNJFklZIWibp/0ga0kXd70maL+kv0vxX0vxqSfdL2qlQd6Wkb0taIGmNpKskbdzFdk+UdI+kf5P0qqQnJX2usHy4pCvSNpdKOlPSRh3WvUjSK8C0Tra/l6S5adsrJf1LYdlfSXooPYdHJe1VWPZg2teDad3bJG2TFt8HDCq0unZJ63xd0iJJL0u6VdLoVD5UUkiaKulZSa9IOq9DnN+U9JSk1yQ9LunTqXyspJslvSjpOUknrvNNtb4nIvzwo+4PYDGwX4eynwL3AyOAkcAjwBlp2QFAa5r+Z+AhYHia3wNYAXwWGARMBZ4GBqflK4E/pW02Aa3A17qI60RgLfBNYAhwPPAysFVafjtwIbAZMAqYC0zusO7fpTg27WT7c4Ej0vSWwO5puhl4CdiP7MfaQUAbsE1a/iCwCPgEsDnwAHBWWrYjsLbDfo4CFgKfSs/jJ8DstGwoEMCNwFbAdsBqYJ+0/DhgCbALIGAHYEx6To8DpwMbp23/Gfh8oz9PfmzA/81GB+DHwHx0kRSWA/sW5icBT6XpA4BngYuA2cCWhXq/ak8ehbIlhS/clcDhhWUXAD/vIq4Tgec7lM0HjgA+DrwBDCksOwG4vbDu090874eBM4BtO5SfCVzaoewPwFFp+kHgu4Vl3wFuStOdJYXZwLGF+SHAO2SJsT0pTCgsnwmcVtjv1zuJ/fPAMx3Kfgxc3OjPkx8b7uFjClYJkgT8BdmXebslwOjC/EfIvoS/HBGvFco/Dhwp6XuFso07rLuyMP0mWWukK8s6zC8BPpr2MxRoy8IFsl/1xbOilq5juwCTgbOAp9PZVD+KiDvTto+RdESh7pC0366ewxZ07ePAJZIuKpStJfvFv6ab7Y0lS8CdbbNZ0upC2SCg3x4XGoicFKwSIiIkrST74mn/QvoYWeuh3Qtk3Tq/kfSliHgklS8Fbo2If91A4YzpMP8x4P+l/bxO1qXT1fDC6xx2OCIWAkdJGgQcDdyYjg0sBS6LiJN7EW9n+1wKfC8ibui4QNLQbra3lKybquOX/VKyltunexGj9RE+0GxVcjVwpqRtJX2ErJvl/xYrRMRdwP8Eft9+QBWYDpwsaYIyW0g6RNJmvYxjbDpoPFjS35L9cr4rIp4n68b5qaQtJW0kaZykvWvdsKTjJW0bEe+S/WKP9JgBHCFpoqRB6aD7xPYD6d1YRXag+WOFskuAf5C0Q9rvNpIOqzHMy4BpknZOr+enJI0B/pi2dVo6WD1Y0mck7Vrjdq0PcFKwKvkR8CTwBDCP7ODwTztWiohbgW8At0v6TET8CTgF+DeyA6ZPA39DN7/a1+E+soOsL5Mlpq9GRHuXyzHAMOCptPy3ZP30tfoSsEjSa8C/AEdGxDsR8RxwGFkf/YtkXVanUsP/aES8QvY6zUlnLo2PiKuBX5C1RF4lez33ryXAiLgSOBe4Hngt/R0WEe+QHQD/yxRfG3Ax6+7Gsj5GXbeCzQaedIrl4RGxX6NjMWsEtxTMzCznpGBmZjl3H5mZWc4tBTMzy/Xp6xRGjBgRzc3NjQ7DzKxPmTNnzosR0dTZsj6dFJqbm2lpaWl0GGZmfYqkJV0tc/eRmZnlnBTMzCznpGBmZjknBTMzyzkpmJlZzknBzMxyTgpmZpZzUjAzs5yTgpmZ5fr0Fc1WruZpt3Zavvjsg+sciZnVi1sKZmaWc0vBNii3Lsz6NrcUzMws56RgZmY5JwUzM8s5KZiZWc4Hmq3Lg8NmNvC4pWBmZjknBTMzyzkpmJlZzknBzMxyTgpmZpZzUjAzs5yTgpmZ5UpNCpKGSbpe0lOSFkraU9JwSXdLeib93SbVlaQLJLVKmi9p1zJjMzOzDyu7pXA+cEdE7AjsDCwEpgGzImIcMCvNAxwIjEuPqcDFJcdmZmYdlJYUJG0NfA64HCAi/isiVgOTgBmp2gzg0DQ9CbgiMg8CwySNKis+MzP7sDJbCtsBbcCvJM2VdJmkzYGREbEi1VkJjEzTo4GlhfWXpbIPkDRVUouklra2thLDNzMbeMpMCoOBXYGLI2IX4A3e7yoCICICiJ5sNCKmR8SEiJjQ1NS0wYI1M7Nyk8IyYFlEPJTmrydLEi+0dwulv6vS8uXA2ML6Y1KZmZnVSWlJISJWAksl7ZCKJgJPAjOByalsMnBzmp4JHJ/OQtoDWFPoZjIzszooe+jsk4GrJG0MPAecQJaIrpU0BVgCHJnq3gYcBLQCb6a61gu+T7KZ9VapSSEi5gETOlk0sZO6AZxUZjxmZrZuvsmONZRbNWbV4mEuzMws56RgZmY5JwUzM8s5KZiZWc5JwczMck4KZmaWc1IwM7Ock4KZmeWcFMzMLOekYGZmOScFMzPLOSmYmVnOScHMzHJOCmZmlnNSMDOznJOCmZnlnBTMzCznpGBmZjknBTMzyzkpmJlZzknBzMxypSYFSYslPS5pnqSWVDZc0t2Snkl/t0nlknSBpFZJ8yXtWmZsZmb2YfVoKXwhIsZHxIQ0Pw2YFRHjgFlpHuBAYFx6TAUurkNsZmZWMLgB+5wE7JOmZwD3Aqen8isiIoAHJQ2TNCoiVjQgxkppnnZrp+WLzz64zpGYWX9XdkshgLskzZE0NZWNLHzRrwRGpunRwNLCustS2QdImiqpRVJLW1tbWXGbmQ1IZbcU9o6I5ZI+Atwt6aniwogISdGTDUbEdGA6wIQJE3q0rpmZrVupSSEilqe/qyT9DtgNeKG9W0jSKGBVqr4cGFtYfUwqM8u5K82sXKV1H0naXNKW7dPAF4EFwExgcqo2Gbg5Tc8Ejk9nIe0BrPHxBDOz+iqzpTAS+J2k9v38JiLukPQIcK2kKcAS4MhU/zbgIKAVeBM4ocTYzMysE6UlhYh4Dti5k/KXgImdlAdwUlnxmJlZ93xFs5mZ5ZwUzMws56RgZmY5JwUzM8s5KZiZWc5JwczMck4KZmaWc1IwM7Ock4KZmeWcFMzMLOekYGZmOScFMzPLOSmYmVnOScHMzHJOCmZmlnNSMDOznJOCmZnlnBTMzCzXbVKQ9AlJm6TpfSSdImlY+aGZmVm91dJSuAF4V9IngenAWOA3pUZlZmYNUUtSeC8i1gJfAS6MiO8Bo8oNy8zMGqGWpPCOpGOAycAtqWxIeSGZmVmj1JIUTgD2BP53RDwvaTvgynLDMjOzRug2KUTEk8DpwKNp/vmIOKfWHUgaJGmupFvS/HaSHpLUKum3kjZO5Zuk+da0vLk3T8jMzHqvlrOPvgzMA+5I8+MlzezBPk4FFhbmzwHOi4hPAq8AU1L5FOCVVH5eqmdmZnVUS/fRWcBuwGqAiJgHbF/LxiWNAQ4GLkvzAvYFrk9VZgCHpulJaZ60fGKqb2ZmdVLTgeaIWNOh7L0at/9z4PuF+tsCq9PZTADLgNFpejSwFCAtX5Pqf4CkqZJaJLW0tbXVGIaZmdWilqTwhKS/AQZJGifpQuCB7laS9CVgVUTMWd8giyJiekRMiIgJTU1NG3LTZmYD3uAa6pwMnAG8DVwN3An8Uw3r7QUcIukgYCiwFXA+MEzS4NQaGAMsT/WXk10Yt0zSYGBr4KUePBezD2medmun5YvPPrjOkZj1DbWcffRmRJwREf8j/UI/IyLeqmG9H0TEmIhoBo4G7omIY4HZwOGp2mTg5jQ9M82Tlt8TEdHD52NmZuuhy5aCpN8DXX4pR8Qhvdzn6cA1kn4CzAUuT+WXA1dKagVeJkskZmZWR+vqPvrZhtpJRNwL3JumnyM7m6ljnbeAIzbUPs3MrOe6TAoR8Yf26XSB2Y5kLYdFEfFfdYjNzMzqrNsDzZIOBi4BngUEbCfp6xFxe9nBmZlZfdVy9tG/Al+IiFbI7q8A3Ao4KZiZ9TO1XKfwWntCSJ4DXispHjMza6BaWgotkm4DriU7pnAE8IikrwJExI0lxmdmZnVUS1IYCrwAfD7NtwGbAl8mSxJOCmZm/US3SSEiTqhHIGZm1ni1nH20HdlQF83F+utx8ZqZmVVULd1HN5Fdbfx7ah8d1czM+qBaksJbEXFB6ZGYmVnD1ZIUzpd0JnAX2UipAETEo6VFZWZmDVFLUvg0cBzZHdPau48izZuZWT9SS1I4Atje4x2ZmfV/tVzRvAAYVnYgZmbWeLW0FIYBT0l6hA8eU/ApqWZm/UwtSeHM0qMwM7NKqOWK5j90V8fMzPqHWq5o3gO4EPhvwMbAIOCNiNiq5Nj6Ld9M3syqqpbuo1+Q3S/5OmACcDzwqTKDsnJ0lYzMzNrVcvYR6X4KgyLi3Yj4FXBAuWGZmVkj1NJSeDPdo3mepJ8CK6gxmVj/5BaHWf9Vy5f7canet4A3gLHAYWUGZWZmjdFtUoiIJRHxVkS8ClwA/LrD7Tk7JWmopIclPSbpCUk/TuXbSXpIUquk36ZWCJI2SfOtaXnz+j01MzPrqW6TgqR7JW0laTjwKHCppHNr2PbbwL4RsTMwHjggncl0DnBeRHwSeAWYkupPAV5J5eelemZmVke1dB9tnVoJXwWuiIjdgf26Wykyr6fZIenRPpDe9al8BnBomp6U5knLJ0pSTc/CzMw2iFqSwmBJo4AjgVt6snFJgyTNA1YBdwPPAqsjYm2qsgwYnaZHA0sB0vI1wLadbHOqpBZJLW1tbT0Jx8zMulFLUvhH4E6gNSIekbQ98EwtG0+nsI4HxgC7ATv2OtL3tzk9IiZExISmpqb13ZyZmRXUMszFdWQXrrXPP0cPzz6KiNWSZgN7AsMkDU6tgTHA8lRtOdmZTcskDQa2Bl7qyX7MzGz91HKdQq9IagLeSQlhU2B/soPHs4HDgWuAycDNaZWZaf4/0vJ7IiLKis+sMx6CxAa60pICMAqYIWkQWTfVtRFxi6QngWsk/QSYC1ye6l8OXCmpFXiZbGgNMzOro9KSQkTMB3bppPw5suMLHcvfIrvLm5mZNUgt1yn8Q2F6k3LDMTOzRuoyKUg6XdKeZP377f6j/JDMzKxR1tV99BRZd872ku5P89tK2iEiFtUlOlsnD0xnZhvaurqPVgM/BFqBfYDzU/k0SQ+UHJeZmTXAuloKfw38CPgEcC4wn+yOayfUIzAzM6u/LpNCRPwQQNJjwJXArkCTpD+SDVz35fqEaAORu8bMGqOWU1LvjIgWoEXSNyJib0kjyg7M+hd/yZv1DbUMc/H9wuzXUtmLZQVk1htOOmYbRo9uqxkRj5UViJmZNZ7vtWxmZjknBTMzyzkpmJlZzknBzMxyTgpmZpZzUjAzs5yTgpmZ5ZwUzMws56RgZmY5JwUzM8uVdo/mgaSrcXcWn31wnSMxM1s/Tgpm68E/CKy/cfeRmZnlnBTMzCxXWlKQNFbSbElPSnpC0qmpfLikuyU9k/5uk8ol6QJJrZLmS9q1rNjMzKxzZR5TWAv8fUQ8KmlLYI6ku8lu1DMrIs6WNA2YBpwOHAiMS4/dgYvT3wHDN4oxs0YrraUQESsi4tE0/RqwEBgNTAJmpGozgEPT9CTgisg8CAyTNKqs+MzM7MPqckxBUjOwC/AQMDIiVqRFK4GRaXo0sLSw2rJU1nFbUyW1SGppa2srLWYzs4Go9KQgaQvgBuC0iHi1uCwiAoiebC8ipkfEhIiY0NTUtAEjNTOzUpOCpCFkCeGqiLgxFb/Q3i2U/q5K5cuBsYXVx6QyMzOrkzLPPhJwObAwIs4tLJoJTE7Tk4GbC+XHp7OQ9gDWFLqZzMysDso8+2gv4DjgcUnzUtkPgbOBayVNAZYAR6ZltwEHAa3Am8AJJcZmVqp1nUnmq52tykpLChHxR0BdLJ7YSf0ATiorHjMz656vaDYzs5yTgpmZ5TxKqg1IvnrcrHNuKZiZWc5JwczMck4KZmaWc1IwM7Ock4KZmeWcFMzMLOekYGZmOScFMzPLOSmYmVnOScHMzHJOCmZmlvPYR2Y18FhJNlA4KZhVRFeJxzflsXpy95GZmeXcUiiRuxzMrK9xS8HMzHJuKXTgfl0zG8jcUjAzs5yTgpmZ5UpLCpJ+KWmVpAWFsuGS7pb0TPq7TSqXpAsktUqaL2nXsuIyM7OuldlS+DVwQIeyacCsiBgHzErzAAcC49JjKnBxiXGZmVkXSksKEXEf8HKH4knAjDQ9Azi0UH5FZB4EhkkaVVZsZmbWuXofUxgZESvS9EpgZJoeDSwt1FuWyj5E0lRJLZJa2trayovUzGwAatiB5ogIIHqx3vSImBARE5qamkqIzMxs4Kr3dQovSBoVEStS99CqVL4cGFuoNyaVmVkXfE2NlaHeLYWZwOQ0PRm4uVB+fDoLaQ9gTaGbyczM6qS0loKkq4F9gBGSlgFnAmcD10qaAiwBjkzVbwMOAlqBN4ETyorLzMy6VlpSiIhjulg0sZO6AZxUVixmZlYbX9FsZmY5JwUzM8s5KZiZWc5JwczMcr6fQo18FzUzGwgGbFLwl7yZ2YcN2KRg1ij+QWJV5mMKZmaWc1IwM7Ocu4/MBggPoGe1cEvBzMxybimYVVyjDky7ZTEwuaVgZmY5JwUzM8s5KZiZWc7HFMz6GV8cZ+vDLQUzM8s5KZiZWc5JwczMck4KZmaW84FmswGupweme1q/Nxe7+cK5xnFSMLOG8ZlS1ePuIzMzy1WqpSDpAOB8YBBwWUSc3eCQzGw9uTXQt1QmKUgaBFwE7A8sAx6RNDMinmxsZGZWdT4GseFUJikAuwGtEfEcgKRrgEmAk4KZAeUfFN9QepqMehNnWQmvSklhNLC0ML8M2L1jJUlTgalp9nVJi2rc/gjgxfWKsFxVjq/KsUG146tybOD41keXsemc8ndewz7W9dp9vKuVqpQUahIR04HpPV1PUktETCghpA2iyvFVOTaodnxVjg0c3/qocmzQ+/iqdPbRcmBsYX5MKjMzszqpUlJ4BBgnaTtJGwNHAzMbHJOZ2YBSme6jiFgr6VvAnWSnpP4yIp7YgLvocZdTnVU5virHBtWOr8qxgeNbH1WODXoZnyJiQwdiZmZ9VJW6j8zMrMGcFMzMLDcgkoKkAyQtktQqaVoF4vmlpFWSFhTKhku6W9Iz6e82DYptrKTZkp6U9ISkU6sSn6Shkh6W9FiK7cepfDtJD6X397fpRIWGkTRI0lxJt1QpPkmLJT0uaZ6kllTW8Pe1EN8wSddLekrSQkl7ViU+STuk16398aqk0yoU37fT/8QCSVen/5Vefe76fVIoDJ9xILATcIyknRobFb8GDuhQNg2YFRHjgFlpvhHWAn8fETsBewAnpderCvG9DewbETsD44EDJO0BnAOcFxGfBF4BpjQgtqJTgYWF+SrF94WIGF84f70K72u784E7ImJHYGey17AS8UXEovS6jQc+C7wJ/K4K8UkaDZwCTIiI/052os7R9PZzFxH9+gHsCdxZmP8B8IMKxNUMLCjMLwJGpelRwKJGx5hiuZlsPKpKxQdsBjxKdtX7i8Dgzt7vBsQ1huzLYV/gFkBViQ9YDIzoUFaJ9xXYGniedPJL1eLrENMXgT9VJT7eHw1iONkZpbcAf93bz12/bynQ+fAZoxsUy7qMjIgVaXolMLKRwQBIagZ2AR6iIvGlrpl5wCrgbuBZYHVErE1VGv3+/hz4PvBemt+W6sQXwF2S5qThYqAi7yuwHdAG/Cp1vV0mafMKxVd0NHB1mm54fBGxHPgZ8GdgBbAGmEMvP3cDISn0OZGl9oaeKyxpC+AG4LSIeLW4rJHxRcS7kTXhx5ANorhjI+LojKQvAasiYk6jY+nC3hGxK1lX6kmSPldc2ODP3WBgV+DiiNgFeIMOXTEV+b/YGDgEuK7jskbFl45jTCJLrB8FNufD3dM1GwhJoa8Mn/GCpFEA6e+qRgUiaQhZQrgqIm6sWnwAEbEamE3WLB4mqf1CzEa+v3sBh0haDFxD1oV0PhWJL/2iJCJWkfWH70Z13tdlwLKIeCjNX0+WJKoSX7sDgUcj4oU0X4X49gOej4i2iHgHuJHss9irz91ASAp9ZfiMmcDkND2ZrC+/7iQJuBxYGBHnFhY1PD5JTZKGpelNyY51LCRLDoc3MjaAiPhBRIyJiGayz9k9EXFsFeKTtLmkLdunyfrFF1CB9xUgIlYCSyXtkIomkg2bX4n4Co7h/a4jqEZ8fwb2kLRZ+v9tf+1697lr9EGbOh2IOQh4mqz/+YwKxHM1Wd/fO2S/kKaQ9T3PAp4B/h0Y3qDY9iZrAs8H5qXHQVWID/gMMDfFtgD4USrfHngYaCVr1m9Sgfd4H+CWqsSXYngsPZ5o/z+owvtaiHE80JLe35uAbSoW3+bAS8DWhbJKxAf8GHgq/V9cCWzS28+dh7kwM7PcQOg+MjOzGjkpmJlZzknBzMxyTgpmZpZzUjAzs5yTgjWUpNdL2KYk3SNpqw297Q77uVdS6Tdul3RKGjX0qg7l4yUdVMP6Z0n67gaIo0nSHeu7Has2JwXrjw4CHosOw3NUSeFK01p8E9g/sgvhisaTPde6iIg2YIWkveq1T6s/JwWrnPSL9AZJj6THXqn8LGX3orhX0nOSTuliE8eSrt6U1Jx+ZV+axpu/K10N/YFf+pJGpOEpkPQ1STel8fEXS/qWpO+kgdoelDS8sK/j0vj6CyTtltbfPMX5cFpnUmG7MyXdQ3bBU8fn/Z20nQWSTktll5BdhHS7pG8X6m4M/CNwVNr/UcrG9r9J0vwU52c62cffSbpd0qaSPiHpjjRA3v2Sdkx1fi3pAkkPpNf58MImbkqvr/VXjbo60A8/IgLg9U7KfkM2eBvAx8iG3AA4C3iA7GrNEWRXlw7pZP0lwJZpupnsHhHj0/y1wN+m6XvJxqAnbW9xmv4a2VWgWwJNZKNOnpiWnUc2SGD7+pem6c+RhkIH/rmwj2FkV9Nvnra7jE6ueiUbo//xVG8LsquOd0nLFtNhyOtCnL8ozF8InJmm9wXmFV637wLfIkuWm6TyWcC4NL072bAckN3v4zqyH407Aa2FfYwGHm/058aP8h49acKa1ct+wE7ZMC4AbJVGbQW4NSLeBt6WtIpsqOJlHdYfHhGvFeafj4h5aXoOWaLozuy0jdckrQF+n8ofJxtuo93VABFxn6St0thMXyQbGK+9H38oWXIDuDsiXu5kf3sDv4uINwAk3Qj8FdmwHrXaGzgsxXOPpG0Lx1WOJxtC/tCIeCe9nn8JXFd4nTcpbOumiHgPeFJScTjoVWQjcVo/5aRgVbQRsEdEvFUsTF9ebxeK3qXzz/BaSRulL7XO1tm0vR7vd6EO7bCN4jrvFebf67DPjuPEBNmNdQ6LiEUd4t+dbEjoRnic7BjEGLKb2WxENt7++C7qF5+/CtNDgf8sJUKrBB9TsCq6Czi5fUZSV19cXVlE1g/fncVk3Tbw/miSPXUUgKS9gTURsQa4Ezg5jViJpF1q2M79wKFppMvNga+ksnV5jayLq7iNY9M+9wFejPcPts8Fvg7MlPTRVP68pCNSfUnauYY4P0U26Jr1U04K1mibSVpWeHyHdL/ZdMD0SeDEHm7zVrJRSrvzM+AbkuaSHVPojbfS+pfw/j1w/wkYAsyX9ESaX6eIeJSsL/9hsjvdXRYR3XUdzSbrZpsn6SiyYweflTQfOJv3h3Ru38cfyY4t3CppBFkCmSKpfeTUSd0/Xb5A9vpaP+VRUq3fUXazkysiYv9Gx9LfSLoPmBQRrzQ6FiuHWwrW70R2z9xLy754baCR1ASc64TQv7mlYGZmObcUzMws56RgZmY5JwUzM8s5KZiZWc5JwczMcv8f46gibeRvhU4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "class SentenceGetter(object):\n",
    "    \"\"\"Class to Get the sentence in this format:\n",
    "    [(Token_1, Part_of_Speech_1, Tag_1), ..., (Token_n, Part_of_Speech_1, Tag_1)]\"\"\"\n",
    "    def __init__(self, data):\n",
    "        \"\"\"Args:\n",
    "            data is the pandas.DataFrame which contains the above dataset\"\"\"\n",
    "        self.n_sent = 1\n",
    "        self.data = data\n",
    "        self.empty = False\n",
    "        agg_func = lambda s: [(w, p, t) for w, p, t in zip(s[\"word\"].values.tolist(),\n",
    "                                                           s[\"ner\"].values.tolist(),\n",
    "                                                           s[\"chunk\"].values.tolist())]\n",
    "        self.grouped = self.data.groupby(\"sentence#\", sort=False).apply(agg_func)\n",
    "        #self.grouped.sort_values('sentence#', ascending=True)\n",
    "\n",
    "        self.sentences = [s for s in self.grouped]\n",
    "    \n",
    "    def get_next(self):\n",
    "        \"\"\"Return one sentence\"\"\"\n",
    "        try:\n",
    "            s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n",
    "            self.n_sent += 1\n",
    "            return s\n",
    "        except:\n",
    "            return None\n",
    "\n",
    "getter = SentenceGetter(data)\n",
    "sent = getter.get_next()\n",
    "sentences = getter.sentences\n",
    "\n",
    "print(len(sentences))\n",
    "plt.hist([len(s) for s in sentences], bins=50)\n",
    "plt.title('Token per sentence')\n",
    "plt.xlabel('Len (number of token)')\n",
    "plt.ylabel('# samples')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "vaoOnrqZZkwf",
    "outputId": "20131abd-6151-4233-de58-681241032bb3"
   },
   "outputs": [],
   "source": [
    "getter_test = SentenceGetter(data_test)\n",
    "sent_test = getter_test.get_next()\n",
    "\n",
    "sentences_test = getter_test.sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "zV_oUq1a0Zek",
    "outputId": "e057aced-78a5-4165-ee00-12ac82e01e6f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The word commitment is identified by the index: 8441\n"
     ]
    }
   ],
   "source": [
    "# Vocabulary Key:word -> Value:token_index\n",
    "# The first 2 entries are reserved for PAD and UNK\n",
    "word2idx = {w: i + 2 for i, w in enumerate(words)}\n",
    "word2idx[\"UNK\"] = 1 # Unknown words\n",
    "word2idx[\"PAD\"] = 0 # Padding\n",
    "# Vocabulary Key:token_index -> Value:word\n",
    "idx2word = {i: w for w, i in word2idx.items()}\n",
    "print(\"The word commitment is identified by the index: {}\".format(word2idx[\"Confidence\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "usHCRHBt0iC3",
    "outputId": "861b9cdf-de4f-4334-abb0-29a4c1a9e941"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The labels B-NP is identified by the index: 12\n"
     ]
    }
   ],
   "source": [
    "# Vocabulary Key:Label/Tag -> Value:tag_index\n",
    "# The first entry is reserved for PAD\n",
    "tag2idx = {t: i+1 for i, t in enumerate(tags)}\n",
    "tag2idx[\"PAD\"] = 0\n",
    "# Vocabulary Key:tag_index -> Value:Label/Tag\n",
    "idx2tag = {i: w for w, i in tag2idx.items()}\n",
    "print(\"The labels B-NP is identified by the index: {}\".format(tag2idx[\"B-NP\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "rPZctZZf0q4L",
    "outputId": "2c2e2e00-8992-466f-b665-e1bcaa695ecc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "Raw Sample:  Rockwell International Corp. 's Tulsa unit said it signed a tentative agreement extending its contract with Boeing Co. to provide structural parts for Boeing 's 747 jetliners .\n",
      "Raw Label:  B-NP I-NP I-NP B-NP I-NP I-NP B-VP B-NP B-VP B-NP I-NP I-NP B-VP B-NP I-NP B-PP B-NP I-NP B-VP I-VP B-NP I-NP B-PP B-NP B-NP I-NP I-NP O\n",
      "After processing, sample: [ 8441  1435 17095 17882 17075  2073  8335 17649  5540 14390 18133  2352\n",
      " 16452  4626  2081   363  5815 13860  5408   363  7352  3429 13860  9347\n",
      " 17649 15744 11471 11726 13430 13743 16656  2078 17421  7125  8334 10037\n",
      " 13680     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0]\n",
      "After processing, labels: [[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "# Convert each sentence from list of Token to list of word_index\n",
    "X_tr= [[word2idx[w[0]] for w in s] for s in sentences]\n",
    "# Padding each sentence to have the same lenght\n",
    "X_tr = pad_sequences(maxlen=MAX_LEN, sequences=X_tr, padding=\"post\", value=word2idx[\"PAD\"])\n",
    "print(type(X_tr[0]))\n",
    "# Convert Tag/Label to tag_index\n",
    "y_tr = [[tag2idx[w[2]] for w in s] for s in sentences]\n",
    "# Padding each sentence to have the same lenght\n",
    "y_tr = pad_sequences(maxlen=MAX_LEN, sequences=y_tr, padding=\"post\", value=tag2idx[\"PAD\"])\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "# One-Hot encode\n",
    "y_tr = [to_categorical(i, num_classes=n_tags+1) for i in y_tr]  # n_tags+1(PAD)\n",
    "\n",
    "X_te = []\n",
    "\n",
    "for s in sentences_test:\n",
    "    temp = []\n",
    "    for w in s:\n",
    "        if w[0] in word2idx:\n",
    "            temp.append(word2idx[w[0]])\n",
    "        else:\n",
    "            temp.append(word2idx[\"UNK\"])\n",
    "    temp = np.array(temp)\n",
    "    X_te.append(temp)\n",
    "\n",
    "    \n",
    "X_te = np.array(X_te)\n",
    "#X_te = [[word2idx[w[0]] for w in s] for s in sentences_test]\n",
    "X_te = pad_sequences(maxlen=MAX_LEN, sequences=X_te, padding=\"post\", value = word2idx[\"PAD\"])\n",
    "\n",
    "y_te = []\n",
    "\n",
    "for s in sentences_test:\n",
    "    temp = []\n",
    "    for w in s:\n",
    "        if w[2] in tag2idx:\n",
    "            temp.append(tag2idx[w[2]])\n",
    "        else:\n",
    "            temp.append(tag2idx[\"O\"])\n",
    "    #temp = np.array(temp)\n",
    "    y_te.append(temp)\n",
    "    \n",
    "#y_te = [[tag2idx[w[2]] for w in s] for s in sentences_test]\n",
    "# Padding each sentence to have the same lenght\n",
    "y_te = pad_sequences(maxlen=MAX_LEN, sequences=y_te, padding=\"post\", value=tag2idx[\"PAD\"])\n",
    "y_te = [to_categorical(i, num_classes=n_tags+1) for i in y_te]  # n_tags+1(PAD)\n",
    "\n",
    "print('Raw Sample: ', ' '.join([w[0] for w in sentences_test[0]]))\n",
    "print('Raw Label: ', ' '.join([w[2] for w in sentences_test[0]]))\n",
    "print('After processing, sample:', X_tr[0])\n",
    "print('After processing, labels:', y_tr[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 394
    },
    "colab_type": "code",
    "id": "1UUp4dZW2cSn",
    "outputId": "d69d8152-458f-4fcb-b408-725e61d0f4ac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:133: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2974: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/keras_contrib/layers/crf.py:346: UserWarning: CRF.loss_function is deprecated and it might be removed in the future. Please use losses.crf_loss instead.\n",
      "  warnings.warn('CRF.loss_function is deprecated '\n",
      "/usr/local/lib/python3.6/dist-packages/keras_contrib/layers/crf.py:353: UserWarning: CRF.accuracy is deprecated and it might be removed in the future. Please use metrics.crf_accuracy\n",
      "  warnings.warn('CRF.accuracy is deprecated and it '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 80)                0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 80, 200)           3824800   \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 80, 100)           100400    \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, 80, 50)            5050      \n",
      "_________________________________________________________________\n",
      "crf_1 (CRF)                  (None, 80, 23)            1748      \n",
      "=================================================================\n",
      "Total params: 3,931,998\n",
      "Trainable params: 3,931,998\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Model definition\n",
    "input = Input(shape=(MAX_LEN,))\n",
    "model = Embedding(input_dim=n_words+2, output_dim=EMBEDDING, # n_words + 2 (PAD & UNK)\n",
    "                  input_length=MAX_LEN, mask_zero=True)(input)  # default: 20-dim embedding\n",
    "model = Bidirectional(LSTM(units=50, return_sequences=True,\n",
    "                           recurrent_dropout=0.1))(model)  # variational biLSTM\n",
    "model = TimeDistributed(Dense(50, activation=\"relu\"))(model)  # a dense layer as suggested by neuralNer\n",
    "crf = CRF(n_tags+1)  # CRF layer, n_tags+1(PAD)\n",
    "out = crf(model)  # output\n",
    "\n",
    "model = Model(input, out)\n",
    "model.compile(optimizer=\"rmsprop\", loss=crf.loss_function, metrics=[crf.accuracy])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 544
    },
    "colab_type": "code",
    "id": "UJckh8yZ2g57",
    "outputId": "b3e1086c-63da-432f-c45b-fdbf531c605a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:986: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:973: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2741: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "Epoch 1/20\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:199: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:206: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
      "\n",
      " - 11s - loss: 14.3715 - crf_viterbi_accuracy: 0.2419\n",
      "Epoch 2/20\n",
      " - 10s - loss: 13.6363 - crf_viterbi_accuracy: 0.3854\n",
      "Epoch 3/20\n",
      " - 9s - loss: 13.2868 - crf_viterbi_accuracy: 0.5237\n",
      "Epoch 4/20\n",
      " - 9s - loss: 12.8571 - crf_viterbi_accuracy: 0.7205\n",
      "Epoch 5/20\n",
      " - 9s - loss: 12.5485 - crf_viterbi_accuracy: 0.8109\n",
      "Epoch 6/20\n",
      " - 9s - loss: 12.3678 - crf_viterbi_accuracy: 0.8604\n",
      "Epoch 7/20\n",
      " - 10s - loss: 12.2589 - crf_viterbi_accuracy: 0.8883\n",
      "Epoch 8/20\n",
      " - 9s - loss: 12.1865 - crf_viterbi_accuracy: 0.9050\n",
      "Epoch 9/20\n",
      " - 9s - loss: 12.1355 - crf_viterbi_accuracy: 0.9185\n",
      "Epoch 10/20\n",
      " - 9s - loss: 12.0983 - crf_viterbi_accuracy: 0.9283\n",
      "Epoch 11/20\n",
      " - 9s - loss: 12.0675 - crf_viterbi_accuracy: 0.9370\n",
      "Epoch 12/20\n",
      " - 9s - loss: 12.0415 - crf_viterbi_accuracy: 0.9438\n",
      "Epoch 13/20\n",
      " - 9s - loss: 12.0237 - crf_viterbi_accuracy: 0.9483\n",
      "Epoch 14/20\n",
      " - 9s - loss: 12.0056 - crf_viterbi_accuracy: 0.9538\n",
      "Epoch 15/20\n",
      " - 9s - loss: 11.9927 - crf_viterbi_accuracy: 0.9571\n",
      "Epoch 16/20\n",
      " - 9s - loss: 11.9806 - crf_viterbi_accuracy: 0.9603\n",
      "Epoch 17/20\n",
      " - 9s - loss: 11.9703 - crf_viterbi_accuracy: 0.9630\n",
      "Epoch 18/20\n",
      " - 9s - loss: 11.9601 - crf_viterbi_accuracy: 0.9655\n",
      "Epoch 19/20\n",
      " - 9s - loss: 11.9517 - crf_viterbi_accuracy: 0.9679\n",
      "Epoch 20/20\n",
      " - 10s - loss: 11.9442 - crf_viterbi_accuracy: 0.9698\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_tr, np.array(y_tr), batch_size=BATCH_SIZE, epochs=20, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_cat = model.predict(X_te)\n",
    "pred = np.argmax(pred_cat, axis=-1)\n",
    "y_te_true = np.argmax(y_te, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing  2012  sentences\n",
      "49389\n"
     ]
    }
   ],
   "source": [
    "def writeToPredictedFile():\n",
    "    f_new = open(\"predicted_test.txt\",\"w\")\n",
    "    count = 0\n",
    "    print(\"Writing \", len(sentences_test), \" sentences\")\n",
    "    for index,s in enumerate(sentences_test):\n",
    "        for word_index, word in enumerate(s):\n",
    "            #print(word)\n",
    "            f_new.write(word[0] + \" \" + word[1] + \" \" + idx2tag[pred[index][word_index]] + \"\\n\")\n",
    "            count = count + 1\n",
    "        f_new.write(\"\\n\")\n",
    "        count = count + 1\n",
    "    print(count)\n",
    "writeToPredictedFile()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have obtained the predicted tags in BIO format, we will aggregate the results and find the F score on the NPs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createNPs(fileName):\n",
    "    with open(fileName) as fp:\n",
    "        line = fp.readline()\n",
    "        count = 0\n",
    "        sentenceList = []\n",
    "        sentCount = 0\n",
    "        NPstarted = False\n",
    "        NPList = []\n",
    "        listWithWordAndPhraseTag = []\n",
    "        sentenceLevelListWithWordAndPhraseTag = []\n",
    "        sentence = \"\"\n",
    "        runningNP = \"\"\n",
    "        while line:\n",
    "            lineList = line.split(\" \")\n",
    "            if line.strip() == \"\":\n",
    "                if runningNP != \"\":\n",
    "                    NPList.append(runningNP)\n",
    "\n",
    "                tempObj = {\"sentence\" : sentence.strip(), \"NPList\" : copy.deepcopy(NPList)}\n",
    "                sentenceList.append(tempObj)\n",
    "                sentCount = sentCount + 1\n",
    "                NPstarted = False\n",
    "                NPList = []\n",
    "                runningNP = \"\"\n",
    "                sentence = \"\"\n",
    "                line = fp.readline()\n",
    "                listWithWordAndPhraseTag.append(copy.deepcopy(sentenceLevelListWithWordAndPhraseTag))\n",
    "                sentenceLevelListWithWordAndPhraseTag = []\n",
    "                count = count + 1\n",
    "                continue\n",
    "\n",
    "            word = lineList[0]\n",
    "            POS = lineList[1]\n",
    "            POS = POS.strip()\n",
    "            NP = lineList[2]\n",
    "            NP = NP.strip()\n",
    "            sentenceLevelListWithWordAndPhraseTag.append([word, NP])\n",
    "            if NP == \"B-NP\" and NPstarted == False:\n",
    "                #print(\"came here\")\n",
    "                NPstarted = True\n",
    "                runningNP = word\n",
    "            else:\n",
    "                if NPstarted == True and (NP == \"I-NP\" or (NP == \"B-NP\" and POS == \"POS\")):\n",
    "                    #print(\"came here \")\n",
    "                    if POS == \"POS\" or POS == \".\" or POS == \",\":\n",
    "                        runningNP = runningNP + word\n",
    "                    else:\n",
    "                        runningNP = runningNP + \" \" + word\n",
    "\n",
    "                elif NPstarted == True and NP != \"I-NP\":\n",
    "                    #print(\"came in elif\")\n",
    "                    NPstarted = False\n",
    "                    NPList.append(runningNP.strip())\n",
    "                    runningNP = \"\"\n",
    "                    if NP == \"B-NP\":\n",
    "                        NPstarted = True\n",
    "                        runningNP = word\n",
    "\n",
    "            if POS == \"POS\" or POS == \".\" or POS == \",\":\n",
    "                sentence = sentence + word\n",
    "            else:\n",
    "                sentence = sentence + \" \" + word\n",
    "\n",
    "            count = count + 1\n",
    "            line = fp.readline()\n",
    "        print(\"Processed \", count, \" lines\")\n",
    "        return (listWithWordAndPhraseTag, sentenceList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed  49389  lines\n"
     ]
    }
   ],
   "source": [
    "listWithWordAndPhraseTag_ground, sentenceList_ground = createNPs(\"test.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed  49389  lines\n"
     ]
    }
   ],
   "source": [
    "listWithWordAndPhraseTag_eval, sentenceList_eval = createNPs(\"predicted_test.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2012 2012\n"
     ]
    }
   ],
   "source": [
    "print(len(sentenceList_ground), len(sentenceList_eval))\n",
    "assert len(sentenceList_ground) == len(sentenceList_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "totalChunksGroundTruth = 0\n",
    "totalChunksPredicted = 0\n",
    "totalCorrectChunksPredicted = 0\n",
    "deviations = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index,item in enumerate(sentenceList_ground):\n",
    "    NPList_ground = item[\"NPList\"]\n",
    "    NPList_eval = sentenceList_eval[index][\"NPList\"]\n",
    "    totalChunksGroundTruth = totalChunksGroundTruth + len(NPList_ground)\n",
    "    totalChunksPredicted = totalChunksPredicted + len(NPList_eval)\n",
    "    \n",
    "    for predictedNP in NPList_eval:\n",
    "        if predictedNP in NPList_ground:\n",
    "            totalCorrectChunksPredicted = totalCorrectChunksPredicted + 1\n",
    "    \n",
    "    if NPList_ground != NPList_eval:\n",
    "        deviations.append(index)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: \t86.6644412918301\n",
      "Recall: \t86.60662163289133\n",
      "F Score: \t86.63552181529991\n"
     ]
    }
   ],
   "source": [
    "precision = (totalCorrectChunksPredicted/totalChunksPredicted)*100\n",
    "print(\"Precision: \\t\" + str(precision))\n",
    "\n",
    "recall = (totalCorrectChunksPredicted/totalChunksGroundTruth)*100\n",
    "print(\"Recall: \\t\" + str(recall))\n",
    "\n",
    "FScore = 2 * ((precision * recall)/(precision + recall))\n",
    "print(\"F Score: \\t\" + str(FScore))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print the below line if you want a list of indexes of sentences that have deviated from the ground truth.\n",
    "# You can use this to analyze sentences as done in the next block.\n",
    "#print(deviations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in testIndex, feed the index of a sentence that you would like to evaluate.\n",
    "testIndex = 2\n",
    "print(\"Ground Truth:\")\n",
    "print(sentenceList_ground[testIndex][\"NPList\"])\n",
    "\n",
    "print(\"Predicted:\")\n",
    "print(sentenceList_eval[testIndex][\"NPList\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine grained evaluation of the tags assigned."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This part is optional and charts out the fine grained accuracies of the labels. You could use this to obtain a finer understanding of which tags have not been predicted correctly the most times and other analysis that could help you better the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gUpQGitV2rOT"
   },
   "outputs": [],
   "source": [
    "#! pip -q install git+https://www.github.com/keras-team/keras-contrib.git sklearn-crfsuite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''from sklearn_crfsuite.metrics import flat_classification_report\n",
    "\n",
    "# Convert the index to tag\n",
    "pred_tag = [[idx2tag[i] for i in row] for row in pred]\n",
    "y_te_true_tag = [[idx2tag[i] for i in row] for row in y_te_true] \n",
    "\n",
    "report = flat_classification_report(y_pred=pred_tag, y_true=y_te_true_tag)\n",
    "print(report)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "BiLstm + crf for chunking",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}