15CS10057_Assignment_4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assignment 4: LSTM from Scratch using MxNet\n",
    "***\n",
    "Submitted By: Suyash Damle\n",
    "\n",
    "Roll : 15CS10057\n",
    "\n",
    "***\n",
    "\n",
    "### NOTE:\n",
    "- The code requires **pre-trained FastText embeddings**. These are available here: https://dl.fbaipublicfiles.com/fasttext/vectors-english/wiki-news-300d-1M.vec.zip  & it should be unzipped\n",
    "- The file location of the train,test data & embeddings are to be specified as a global variables in the \"Global Variables\" section\n",
    "- **As the parameter files were too large, I have uploaded them on google drive. They are available on these links:**\n",
    "    - for one-hot encoding model: https://drive.google.com/file/d/182DnuEdSaso1Q03-ZzXhOpg09rhitTiY/view?usp=sharing\n",
    "    - for embeddings model: https://drive.google.com/file/d/1NkTpkSyFm0bgKHEKuJDpxH0FZaMfdohi/view?usp=sharing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "from mxnet import nd\n",
    "from mxnet import autograd\n",
    "import numpy as np\n",
    "import re\n",
    "import io\n",
    "import pickle\n",
    "from zipfile import ZipFile\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Global Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = \"data/train.txt\"\n",
    "test_file = \"data/test.txt\"\n",
    "embeddings_file = \"data/wiki-news-300d-1M.vec\"\n",
    "short_embeddings_file = \"data/short_embeddings_file.pkl\"\n",
    "\n",
    "### Global Parameters\n",
    "embedding_matrix = None\n",
    "embedding_size = 300\n",
    "vocab_dict = None\n",
    "ctx = mx.cpu()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to get FastText word embeddings for our specific vocab\n",
    "def load_vectors(fname, vocab_dict):\n",
    "    '''\n",
    "    Since our vocab is expected to much smaller than the fasttext vocab, we only...\n",
    "    ... retain the embeddings required for our purpose in the main memory.\n",
    "    Special token </s>(the start symbol) is assigned all-zero embeddings.\n",
    "    \n",
    "    Params:\n",
    "        fname : the name of the zip of the file\n",
    "        vocab : the vocab dict of our dataset: <token : idx>\n",
    "    Returns:\n",
    "        np array: row at the idx of token (from dict) contains the embedding\n",
    "    '''\n",
    "    \n",
    "    fin = io.open(fname, 'r', encoding='utf-8', newline='\\n', errors='ignore')\n",
    "    n, d = map(int, fin.readline().split())\n",
    "    data = np.zeros((len(vocab_dict),embedding_size))\n",
    "    for line in fin:\n",
    "        tokens = line.rstrip().split(' ')\n",
    "        if (tokens[0] in vocab_dict.keys()) or tokens[0]=='</s>':\n",
    "            if tokens[0] == '</s>':\n",
    "                data[vocab_dict['</s>']]=np.zeros(embedding_size)\n",
    "            elif tokens[0] =='/e':\n",
    "                data[vocab_dict['/e']]=np.ones(embedding_size)\n",
    "            else:\n",
    "                data[vocab_dict[tokens[0]]] = np.array(list(map(float, tokens[1:])))\n",
    "    pickle.dump(data,open(short_embeddings_file,'wb'))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(sentences):\n",
    "    sents=[]\n",
    "    for sent in sentences:\n",
    "        sent = re.sub(r\"^[a-zA-Z]\",\" \",sent)\n",
    "        sent=sent.lower().split()\n",
    "#         sent.insert(0,'</s>')\n",
    "#         sent.append('</e>')\n",
    "        sents.append(sent)\n",
    "        \n",
    "    return sents    \n",
    "        \n",
    "    \n",
    "def build_vocab(sentences):\n",
    "    \"\"\"\n",
    "    Builds a vocabulary mapping from token to index based on the sentences.\n",
    "    Returns vocabulary mapping and inverse vocabulary mapping.\n",
    "    \"\"\"\n",
    "    tokens=[]\n",
    "    # Mapping from index to word\n",
    "    for sent in sentences:\n",
    "        tokens.extend(sent)\n",
    "        \n",
    "    vocabulary_inv=list(set(tokens))\n",
    "    \n",
    "    # Mapping from word to index\n",
    "    vocabulary = {vocabulary_inv[i]: i for i in range(len(vocabulary_inv))}\n",
    "    \n",
    "    return vocabulary, vocabulary_inv\n",
    "\n",
    "def create_sent_representation(sent,type='one-hot'):\n",
    "    '''\n",
    "    Returns an embedding matrix for the string\n",
    "    Params:\n",
    "        sent: the string to be converted to matrix\n",
    "        type: 'one-hot' / 'pre-trained'\n",
    "    Returns:\n",
    "        list of vectors representing the sentence\n",
    "    '''\n",
    "    global embedding_matrix\n",
    "    #sent_matrix = None\n",
    "    sent_matrix=[]\n",
    "    if (type == 'pre-trained') and (embedding_matrix is None):\n",
    "        embedding_matrix = load_vectors(embeddings_file,vocab_dict)\n",
    "        #sent_matrix = np.zeros((len(sent), embedding_size))\n",
    "    #else:\n",
    "        #sent_matrix = np.zeros((len(sent), len(vocab_dict)))\n",
    "        \n",
    "    for idx,word in enumerate(sent):\n",
    "        if type=='one-hot':\n",
    "            vect = np.zeros(len(vocab_dict))\n",
    "            vect[vocab_dict[word]] = 1\n",
    "            #sent_matrix[idx] = vect\n",
    "            sent_matrix.append(vect)\n",
    "        else:\n",
    "            #sent_matrix[idx] = embedding_matrix[vocab_dict[word]]\n",
    "            sent_matrix.append(embedding_matrix[vocab_dict[word]])\n",
    "    return sent_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(train_file,'r') as f:\n",
    "    train_data=f.readlines()\n",
    "with open(test_file,'r') as f:\n",
    "    test_data=f.readlines()\n",
    "l_train = len(train_data)\n",
    "all_data = train_data+test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = preprocess(all_data)\n",
    "vocab_dict , inv_vocab = build_vocab(all_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The LSTM model from scratch\n",
    "***\n",
    "Following are the formulae:\n",
    "\n",
    "$$g_t = \\text{tanh}(X_t W_{xg} + h_{t-1} W_{hg} + b_g)$$\n",
    "$$i_t = \\sigma(X_t W_{xi} + h_{t-1} W_{hi} + b_i)$$\n",
    "$$f_t = \\sigma(X_t W_{xf} + h_{t-1} W_{hf} + b_f)$$\n",
    "$$o_t = \\sigma(X_t W_{xo} + h_{t-1} W_{ho} + b_o)$$\n",
    "$$c_t = f_t \\odot c_{t-1} + i_t \\odot g_t$$\n",
    "$$h_t = o_t \\odot \\text{tanh}(c_t)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the One-Hot encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(vocab_dict)\n",
    "num_inputs = vocab_size\n",
    "num_hidden = 100\n",
    "num_outputs = vocab_size\n",
    "\n",
    "#  Weights connecting the inputs to the hidden layer\n",
    "Wxg = nd.random_normal(shape=(num_inputs,num_hidden), ctx=ctx) * .01\n",
    "Wxi = nd.random_normal(shape=(num_inputs,num_hidden), ctx=ctx) * .01\n",
    "Wxf = nd.random_normal(shape=(num_inputs,num_hidden), ctx=ctx) * .01\n",
    "Wxo = nd.random_normal(shape=(num_inputs,num_hidden), ctx=ctx) * .01\n",
    "\n",
    "#  Recurrent weights connecting the hidden layer across time steps\n",
    "Whg = nd.random_normal(shape=(num_hidden,num_hidden), ctx=ctx)* .01\n",
    "Whi = nd.random_normal(shape=(num_hidden,num_hidden), ctx=ctx)* .01\n",
    "Whf = nd.random_normal(shape=(num_hidden,num_hidden), ctx=ctx)* .01\n",
    "Who = nd.random_normal(shape=(num_hidden,num_hidden), ctx=ctx)* .01\n",
    "\n",
    "#  Bias vector for hidden layer\n",
    "bg = nd.random_normal(shape=num_hidden, ctx=ctx) * .01\n",
    "bi = nd.random_normal(shape=num_hidden, ctx=ctx) * .01\n",
    "bf = nd.random_normal(shape=num_hidden, ctx=ctx) * .01\n",
    "bo = nd.random_normal(shape=num_hidden, ctx=ctx) * .01\n",
    "\n",
    "# Weights to the output nodes\n",
    "Why = nd.random_normal(shape=(num_hidden,num_outputs), ctx=ctx) * .01\n",
    "by = nd.random_normal(shape=num_outputs, ctx=ctx) * .01\n",
    "\n",
    "\n",
    "# The gradients attached\n",
    "params = [Wxg, Wxi, Wxf, Wxo, Whg, Whi, Whf, Who, bg, bi, bf, bo, Why, by]\n",
    "for param in params:\n",
    "    param.attach_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(y_linear):\n",
    "    lin = (y_linear-nd.max(y_linear))\n",
    "    exp = nd.exp(lin)\n",
    "    partition = nd.sum(exp, axis=0, exclude=True).reshape((-1,1))\n",
    "    return exp / partition\n",
    "\n",
    "def cross_entropy(yhat, y):\n",
    "    return - nd.mean(nd.sum(y * nd.log(yhat), axis=0, exclude=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_rnn(inputs, h, c):\n",
    "    outputs = []\n",
    "    for X in inputs:\n",
    "        #X = nd.array(X, ctx=ctx)\n",
    "        g = nd.tanh(nd.dot(X, Wxg) + nd.dot(h, Whg) + bg)\n",
    "        i = nd.sigmoid(nd.dot(X, Wxi) + nd.dot(h, Whi) + bi)\n",
    "        f = nd.sigmoid(nd.dot(X, Wxf) + nd.dot(h, Whf) + bf)\n",
    "        o = nd.sigmoid(nd.dot(X, Wxo) + nd.dot(h, Who) + bo)\n",
    "        c = f * c + i * g\n",
    "        h = o * nd.tanh(c)\n",
    "        yhat_linear = nd.dot(h, Why) + by\n",
    "        yhat = softmax(yhat_linear)\n",
    "        outputs.append(yhat)\n",
    "    return (outputs, h, c)\n",
    "\n",
    "def average_ce_loss(outputs, labels):\n",
    "    #labels = nd.array(labels, ctx=ctx)\n",
    "    assert(len(outputs) == len(labels))\n",
    "    total_loss = 0.\n",
    "    for (output, label) in zip(outputs,labels):\n",
    "        total_loss = total_loss + cross_entropy(output, label)\n",
    "    return total_loss / len(outputs)\n",
    "\n",
    "def SGD(params, lr):\n",
    "    for param in params:\n",
    "        # with gradient clipping\n",
    "        param[:] = param - lr * param.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Other specifics for the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_vects = []\n",
    "for sent in all_data:\n",
    "    all_vects.append(create_sent_representation(sent))\n",
    "train_data_org = all_vects[:l_train]\n",
    "test_data_org = all_vects[l_train:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_cont=[]\n",
    "for sent in train_data_org:\n",
    "    train_data_cont.extend(sent)\n",
    "\n",
    "batch_size = 128\n",
    "seq_length = 1\n",
    "# -1 here so we have enough characters for labels later\n",
    "num_samples = (len(train_data_cont) - 1) // seq_length\n",
    "dataset = train_data_cont[:seq_length*num_samples]\n",
    "dataset = np.array(dataset).reshape((num_samples, seq_length, vocab_size))\n",
    "num_batches = len(dataset) // batch_size\n",
    "train_data = dataset[:num_batches*batch_size].reshape((num_batches, batch_size, seq_length, vocab_size))\n",
    "# swap batch_size and seq_length axis to make later access easier\n",
    "train_data = np.swapaxes(train_data, 1, 2)\n",
    "labels = (train_data_cont[1:seq_length*num_samples+1])[:num_batches*batch_size]\n",
    "train_label = np.array(labels).reshape((num_batches, batch_size, seq_length, vocab_size))\n",
    "train_label = np.swapaxes(train_label, 1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33788\n",
      "(33787, 1, 7161)\n",
      "(263, 1, 128, 7161)\n"
     ]
    }
   ],
   "source": [
    "print(len(train_data_cont))\n",
    "print(dataset.shape)\n",
    "print(train_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_fn(test_data, type=1,encoding='one-hot'):\n",
    "    total_pred=0.0\n",
    "    correct_pred=0.0\n",
    "    for sent in test_data:\n",
    "        h = nd.zeros(shape=(1, num_hidden), ctx=ctx)\n",
    "        c = nd.zeros(shape=(1, num_hidden), ctx=ctx)\n",
    "        if type==1:\n",
    "            total_pred +=1\n",
    "            input_sequence = nd.array(sent[:-1], ctx=ctx)\n",
    "            outputs, h, c = lstm_rnn(input_sequence, h, c)\n",
    "            if np.argmax(sent[-1])==np.argmax(outputs[-1].asnumpy()):\n",
    "                correct_pred+=1\n",
    "        else:\n",
    "            input_sequence = nd.array(sent[:int(len(sent)/2)],ctx=ctx)\n",
    "            yhat,h,c = lstm_rnn(input_sequence, h, c)\n",
    "            # run on first half of sentence:\n",
    "            for word in sent[int(len(sent)/2):]:\n",
    "                total_pred+=1\n",
    "                # feed previous output as input this time\n",
    "                X=yhat[-1]\n",
    "                if encoding=='pre-trained':\n",
    "                    #print(np.argmax(yhat[-1].asnumpy()))\n",
    "                    X=nd.array(embedding_matrix[np.argmax(yhat[-1].asnumpy())],ctx=ctx)\n",
    "                g = nd.tanh(nd.dot(X, Wxg) + nd.dot(h, Whg) + bg)\n",
    "                i = nd.sigmoid(nd.dot(X, Wxi) + nd.dot(h, Whi) + bi)\n",
    "                f = nd.sigmoid(nd.dot(X, Wxf) + nd.dot(h, Whf) + bf)\n",
    "                o = nd.sigmoid(nd.dot(X, Wxo) + nd.dot(h, Who) + bo)\n",
    "                c = f * c + i * g\n",
    "                h = o * nd.tanh(c)\n",
    "                yhat_linear = nd.dot(h, Why) + by\n",
    "                yhat = softmax(yhat_linear)\n",
    "                if np.argmax(yhat[-1].asnumpy)==np.argmax(word):\n",
    "                    correct_pred+=1\n",
    "            \n",
    "    acc = correct_pred/total_pred\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0. Loss: 6.69921\n",
      "Epoch 5. Loss: 5.89201\n",
      "Epoch 10. Loss: 5.52554\n",
      "Epoch 15. Loss: 5.38315\n",
      "Epoch 20. Loss: 5.30624\n",
      "Epoch 25. Loss: 5.27942\n"
     ]
    }
   ],
   "source": [
    "epochs = 30\n",
    "learning_rate = 4\n",
    "\n",
    "train_data = nd.array(train_data, ctx=ctx)\n",
    "train_label = nd.array(train_label, ctx=ctx)\n",
    "\n",
    "h = nd.zeros(shape=(batch_size, num_hidden), ctx=ctx)\n",
    "c = nd.zeros(shape=(batch_size, num_hidden), ctx=ctx)\n",
    "\n",
    "# state = nd.zeros(shape=(batch_size, num_hidden), ctx=ctx)\n",
    "epoch_data=[]\n",
    "error_data=[]\n",
    "for e in range(epochs):\n",
    "    # Attenuate the learning rate by a factor of 2 every 5 epochs\n",
    "    if ((e+1) % 5 == 0):\n",
    "        learning_rate = learning_rate / 2.0\n",
    "\n",
    "    for i in range(num_batches):\n",
    "        data_one_hot = train_data[i]\n",
    "        label_one_hot = train_label[i]\n",
    "        with autograd.record():\n",
    "            outputs, h, c = lstm_rnn(data_one_hot, h, c)\n",
    "            loss = average_ce_loss(outputs, label_one_hot)\n",
    "            loss.backward()\n",
    "        SGD(params, learning_rate)\n",
    "        op_loss = nd.mean(loss).asscalar()\n",
    "    epoch_data.append(e+1)\n",
    "    error_data.append(op_loss)\n",
    "    if e%5==0:\n",
    "        print(\"Epoch %s. Loss: %s\" % (e, op_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VPW9//HXZzJZICQESAghLCEgYV/DJohwVdzqWru4Uqui1ette+1ie29/rfXe+7C111avWxHrVreK0mK1iAuKioJBdsIaWQJkYU8CJCT5/v7IaEMIZEKGnJnJ+/l48EjmnG9mPseRdw6f+Z7vMeccIiISXXxeFyAiIqGncBcRiUIKdxGRKKRwFxGJQgp3EZEopHAXEYlCCncRkSikcBcRiUIKdxGRKOT36oVTU1NdVlaWVy8vIhKRli5duts5l9bUOM/CPSsri7y8PK9eXkQkIpnZ1mDGqS0jIhKFFO4iIlFI4S4iEoUU7iIiUUjhLiIShRTuIiJRSOEuIhKFIi7cNxSXce/f11JZXeN1KSIiYSviwr1w3yGe/OgLlnyx1+tSRETCVsSF+4TsVOL8PhasK/W6FBGRsBVx4d4uLoYJ2V14f32J16WIiIStiAt3gKk5aRTsrmDL7gqvSxERCUsRGe5TcroC6OxdROQEIjLcs1ITyU5NZMF69d1FRBoTkeEOdWfvnxTs4XCVpkSKiDQUseE+dUAaVdW1fFKw2+tSRETCTsSG+9g+nWkXG6MpkSIijYjYcI/3xzCxXyoL1pfgnPO6HBGRsBJUuJtZipnNNrN1ZpZvZhMaGTPFzJab2Roz+yD0pR5v6oA0CvcdZnNpeWu8nIhIxAj2HqoPAvOcc1eZWRzQvv5OM0sBHgUucM5tM7OuIa6zUV9OiVywrpR+XZNa4yVFRCJCk2fuZpYMTAaeBHDOVTnn9jcYdg3wmnNuW2BMq0xAz0xpR056Egs0311E5BjBtGWygVLgKTNbZmazzCyxwZj+QCcze9/MlprZDSGv9ASmDEjjsy17KTtytLVeUkQk7AUT7n5gFPCYc24kUAHc3ciY0cDFwPnAL8ysf8MnMrMZZpZnZnmlpaGZ5TI1pytHaxwfb9oTkucTEYkGwYR7IVDonFsceDyburBvOGaec67CObcbWAgMb/hEzrmZzrlc51xuWlpaS+r+yujenUiK92spAhGRepoMd+dcEbDdzHICm84B1jYY9jfgLDPzm1l7YByQH9JKTyA2xsdZ/TUlUkSkvmDnud8JPG9mK4ERwP+Y2W1mdhuAcy4fmAesBJYAs5xzq09HwY2ZktOV4oOV5O8qa62XFBEJa0FNhXTOLQdyG2x+vMGY+4H7Q1RXs0zpX9fiWbC+hEHdk70oQUQkrETsFar1dU1OYEhmsvruIiIBURHuUDdrZunWfRw4pCmRIiJRE+5TcrpS62DhRi0kJiISNeE+omcKKe1jdbWqiAhRFO4xPuPs/ml8sL6U2lpNiRSRti1qwh3q+u57KqpYteOA16WIiHgqqsJ9cv80zFBrRkTavKgK986JcYzomaIbZ4tImxdV4Q51rZmVhfvZXV7pdSkiIp6JynB3DhZu0Nm7iLRdURfug7snk9ohXq0ZEWnToi7cfT5jSk4aCzeUUl1T63U5IiKeiLpwh7rWzIHDR1m+veHdAEVE2oaoDPdJZ6QS4zNNiRSRNisqw71ju1hG9+7EgnXqu4tI2xSV4Q51rZm1uw5SfPCI16WIiLS66A33AXU38PhAs2ZEpA2K2nDPSU8iM6UdTy/awpGjNV6XIyLSqoIKdzNLMbPZZrbOzPLNbMIJxo0xsxozuyq0ZTafmXHv5YPJLzrI3a+u1M2zRaRNCfbM/UFgnnNuADAcyG84wMxigN8Ab4WuvJb5lwHp/GhaDn9dvpMnPizwuhwRkVbTZLibWTIwGXgSwDlX5ZxrbAL5ncCrQFjNP7x9Sl8uHprBff9Yp3usikibEcyZezZQCjxlZsvMbJaZJdYfYGaZwBXA46ehxhYxM+7/xjByuiVz54vLKCgt97okEZHTLphw9wOjgMeccyOBCuDuBmP+APzUOXfSTy7NbIaZ5ZlZXmlp681iaR/nZ+b1o4mN8XHLs3mUHdFNtEUkugUT7oVAoXNuceDxbOrCvr5c4CUz2wJcBTxqZpc3fCLn3EznXK5zLjctLa0FZTdfz87teeSaUWzZc4gfvLRct+ITkajWZLg754qA7WaWE9h0DrC2wZg+zrks51wWdeF/u3Pur6EutqUm9O3CLy8ZxLvrSnjg7Q1elyMictr4gxx3J/C8mcUBBcCNZnYbgHMu7PrsJ3P9+N6s3XmQhxdsYmBGMhcPy/C6JBGRkAsq3J1zy6lrvdTXaKg7577TwppOKzPjnssGs7GknB+9soI+qYkM6p7sdVkiIiEVtVeonky8P4bHrhtFx3ax3PJsHnsrqrwuSUQkpNpkuAN0TUpg5g2jKS2v5Pbnl3JUN/YQkSjSZsMdYFiPFO67ciifFuzlv/6+tukfEBGJEG063AGuHNWDmyf14ZlPtrLki71elyMiEhJtPtwB7pqWQ+fEOGYu3Ox1KSIiIaFwB9rFxXD9+N68k1/CppIyr8sREWkxhXvADRN6E+/38cTCL7wuRUSkxRTuAV06xPON3B7MWbaDEt2aT0QinMK9npsnZXO0tpZnPtnidSkiIi2icK8nKzWRCwZ348+fbqOistrrckRETpnCvYEZk7M5cPgoL3+23etSREROmcK9gZG9OjE2qzNPfvQF1bpqVUQilMK9EbdMzmbH/sO8sWqX16WIiJwShXsjzhnQlb5picxcWIBzuqmHiEQehXsjfD7jlrOyWbPzIIs27/G6HBGRZlO4n8DlIzNJ7RDPHxcWeF2KiEizKdxPICE2hhsnZrFwQyn5uw56XY6ISLMo3E/iunG9aR8XwxM6exeRCBNUuJtZipnNNrN1ZpZvZhMa7L/WzFYG/iwys+Gnp9zW1bF9LN8a05O5K3ayc/9hr8sREQlasGfuDwLznHMDgOFAfoP9XwBnO+eGAfcCM0NXordumtQHBzz1sRYUE5HI0WS4m1kyMBl4EsA5V+Wc219/jHNukXNuX+Dhp0CPUBfqlR6d2nPx0AxeXLKdg0eOel2OiEhQgjlzzwZKgafMbJmZzTKzxJOMvwn4R0iqCxMzJmdTXlnNC4u3eV2KiEhQggl3PzAKeMw5NxKoAO5ubKCZTaUu3H96gv0zzCzPzPJKS0tPseTWNySzIxP7deGpj7+gqlpLEohI+Asm3AuBQufc4sDj2dSF/THMbBgwC7jMOdfolT/OuZnOuVznXG5aWtqp1uyJW87KpvhgJXNX7PS6FBGRJjUZ7s65ImC7meUENp0DrK0/xsx6Aa8B1zvnNoS8yjBwdv80BnRL4gktSSAiESDY2TJ3As+b2UpgBPA/Znabmd0W2P//gC7Ao2a23MzyTkOtnjKrW5JgfXEZ72+InJaSiLRN5tVZaG5ursvLi6zfAVXVtZx9/wKyuiTy4ozxXpcjIm2QmS11zuU2NU5XqDZDnN/HjROz+KRgD6sKD3hdjojICSncm+nqsb1Iivfzx4WbvS5FROSEFO7NlJQQyzXjevHmql1s33vI63JERBqlcD8FN07sQ4zPePIjLUkgIuFJ4X4KunVM4NLhmbz82Xb2VVR5XY6IyHEU7qdoxuRsDh+t4fnFW70uRUTkOAr3U5TTLYkpOWk8vWgrR47WeF2OiMgxFO4tMGNyNrvLK5mzbIfXpYiIHEPh3gITsrswJDOZJz4soLZWSxKISPhQuLeAmTFjcl8KSit4J7/Y63JERL6icG+hi4Z0o0endszUfVZFJIwo3FvIH+Pjpkl9yNu6j6Vb9zX9AyIirUDhHgLfzO1Jx3axzNSSBCISJhTuIZAY7+f68b2Zv7aYgtJyr8sREVG4h8r0M7OIjfFpSQIRCQsK9xBJS4rn66Mymb20kN3llV6XIyJtnMI9hG4+K5uqmlqe/URLEoiItxTuIdQ3rQPnDkznuU+2cLhKSxKIiHeCCnczSzGz2Wa2zszyzWxCg/1mZg+Z2SYzW2lmo05PueHv1snZ7Dt0lFeWbve6FBFpw4I9c38QmOecGwAMB/Ib7L8QOCPwZwbwWMgqjDC5WZ0Z1SuFWR9+QY2WJBARjzQZ7maWDEwGngRwzlU55/Y3GHYZ8Kyr8ymQYmYZIa82QsyY3Jdtew8xb3WR16WISBsVzJl7NlAKPGVmy8xslpklNhiTCdTvQxQGtrVJ5w1Kp09qIjMXbsY5nb2LSOsLJtz9wCjgMefcSKACuLvBGGvk545LNTObYWZ5ZpZXWlra7GIjRYzPuOWsbFYUHuDVz7UcsIi0vmDCvRAodM4tDjyeTV3YNxzTs97jHsDOhk/knJvpnMt1zuWmpaWdSr0R41tjejIhuwu/+OtqNhaXeV2OiLQxTYa7c64I2G5mOYFN5wBrGwybC9wQmDUzHjjgnNsV2lIjS4zPePDbI0iMj+GOFz7X1EgRaVXBzpa5E3jezFYCI4D/MbPbzOy2wP43gQJgE/AEcHvIK41AXZMT+MO3RrKxpJxfzl3tdTki0ob4gxnknFsO5DbY/Hi9/Q64I4R1RY1JZ6Ry59R+PPTeJsb16cLXR/fwuiQRaQN0hWor+P65/RnXpzP/+dfVbCpR/11ETj+FeyuI8RkPXT2S9nEx3PH8MvXfReS0U7i3kvTkBH7/rRFsKCnjV3PXeF2OiEQ5hXsrmtw/jdun9OXlvO3MWVbodTkiEsUU7q3sh+f2Z2xWZ/5jzmo2leiuTSJyeijcW5k/xsdDV48kITaGf33hc44cVf9dREJP4e6Bbh0TeOCbw1lXVMY9r6v/LiKhp3D3yJScrnxvSl9eXLKdvy3X+jMiEloKdw/ddV5/cnt34uevrVL/XURCSuHuIX+Mj/+7ZiRxfh+XP/Ixj72/mcpq9eBFpOUU7h7L6NiOObdPZHx2F34zbx3nPbCQeauLtA68iLSIwj0MZKUmMmt6Ls/dNJaEWB+3/Xkp1zyxmPxdB70uTUQilMI9jJx1Rhpv/ttZ3HvZYNYVHeTihz7kZ6+tYk95pdeliUiEUbiHGX+Mj+snZPH+j6Yy/cwsXsnbzpT73+eJhQVUVdd6XZ6IRAiFe5jq2D6WX14ymHk/mMzorE7895v5nP+HhbybX+x1aSISARTuYa5f1w48feNYnrpxDD6Dm57J47bnllJ88IjXpYlIGFO4R4ipOV2Z94PJ/PSCASxYX8K5D3zAC4u3UVurWTUicjyFewSJjfHxvSl9eesHkxnSvSM/n7OKbz/xKZtLdQGUiBwrqHA3sy1mtsrMlptZXiP7O5rZ62a2wszWmNmNoS9VvpSVmsgLt4zjt18fxrpdB7nwwQ95+L2N+sBVRL7SnDP3qc65Ec65hvdShbr7p651zg0HpgD/a2ZxoShQGmdmfHNMT96562zOG5TO7+Zv4NKHP2LZtn1elyYiYSBUbRkHJJmZAR2AvUB1iJ5bTqJrUgKPXDOKWTfkcuDwUa58bBH3vL6Gikr95xdpy4INdwfMN7OlZjajkf0PAwOBncAq4PvOOfUIWtG5g9KZ/8PJXD++N08v2sK03y/k1aWFHKpSyIu0RRbMGiZm1t05t9PMugJvA3c65xbW238VMBH4d6BvYMxw59zBBs8zA5gB0KtXr9Fbt24N2YHIPy3dupefv7aa9cVlJMbFcNHQDL4+ugdjszrj85nX5YlIC5jZ0hO0x48d19wFqszsV0C5c+539ba9AdznnPsw8Pg94G7n3JITPU9ubq7Lyzvus1kJEeccn23Zx+yl23lzVRHlldX06NSOK0f14OujMundJdHrEkXkFIQs3M0sEfA558oC378N/No5N6/emMeAYufcr8wsHficujP33Sd6XoV76zlcVcNba4p49fNCPtq0G+dgTFYnrhrdg4uGZpCUEOt1iSISpFCGezYwJ/DQD7zgnPtvM7sNwDn3uJl1B54GMgCj7iz+zyd7XoW7N3buP8ycZTt49fNCCkorSIj1ccXITH592RBiY3TZg0i4O21tmVBRuHvLOcfy7ft57tOtvPb5DmbdkMu5g9K9LktEmhBsuOtUrY0yM0b26sR9Vw4jOcHPm6t2eV2SiISQwr2Ni/P7mDa4G2+vLdYt/kSiiMJduHhoBmWV1Xy08YSff4tIhFG4CxP7pZKc4OeNlWrNiEQLhbuoNSMShRTuAqg1IxJtFO4C1GvNaNaMSFRQuAtQ15o5b5BaMyLRQuEuX/nasAzKjqg1IxINFO7yFbVmRKKHwl2+otaMSPRQuMsxLh7WjbIj1Xy8Sa0ZkUimcJdjTOqXRlKCnzdWFnldioi0gMJdjhHn9zFtUDfmry1Sa0Ykginc5ThqzYhEPoW7HEetGZHIp3CX49TNmknn7bVFVFXXel2OiJwChbs06uKhGRxUa0YkYgUV7ma2xcxWmdlyM2v03nhmNiWwf42ZfRDaMqW1TTojlaR4P3/XMsAiEcnfjLFTnXONnsaZWQrwKHCBc26bmXUNSXXimXh/DOcN/rI1M5Q4v/6RJxJJQvU39hrgNefcNgDnXEmInlc8pNaMSOQKNtwdMN/MlprZjEb29wc6mdn7gTE3hK5E8cqXrRmtNSMSeYIN94nOuVHAhcAdZja5wX4/MBq4GDgf+IWZ9W/4JGY2w8zyzCyvtLS0JXVLK4j3x3DeoHTmr9GsGZFIE1S4O+d2Br6WAHOAsQ2GFALznHMVgb78QmB4I88z0zmX65zLTUtLa1nl0iou+rI1s1mtGZFI0mS4m1mimSV9+T0wDVjdYNjfgLPMzG9m7YFxQH6oi5XWd1b/QGtGs2ZEIkowZ+7pwEdmtgJYArzhnJtnZreZ2W0Azrl8YB6wMjBmlnOu4S8AiUBqzYhEpianQjrnCmi8xfJ4g8f3A/eHrjQJFxcNzeC1ZTv4ePNupuZolqtIJNDkZWnSl62ZN9WaEYkYCndpUrw/hnMHpTN/bbFaMyIRQuEuQbl4aAYHDh8N2QVNzjn2lFfy+bZ9lFdWh+Q5ReSfmrP8gLRhX7ZmvvvMZ3RLTqBn5/b07tye3l3a133fJZFendvTqX0sZvbVz1VW17B1zyEKSsvZXFpBQWkFBbvLKSit4MDhowD0TUvk5VsnkNoh3qvDE4k65pzz5IVzc3NdXl6ja5BJmMrbspePN+1h694Ktu89xNY9hygpqzxmTFK8n56d29M5MY5tew9RuO8QtfX+F0tPjic7tQPZaYlkp3UgMS6GX72+huzUDrw4Yzwd28W28lGJRBYzW+qcy21qnM7cJWi5WZ3Jzep8zLbDVTVs31cX9Fv3BEJ/7yH2VVQxrEdHLh+ZSd+0RLJTO9AnLZEO8cf/L9etYwK3PJvHd5/+jOduGkv7OP1vKdJSOnOXsPCPVbu444XPmdgvlVnTc4n3x3hdkkhYCvbMXR+oSli4cGgGv/n6MD7cuJs7X1hGdY1m5QhsLi3ngbc3UFvrzUloJFO4S9j4Rm5PfnXJIOavLebHs1fqL7Qw84MCHnp3I3lb93ldSsRRc1PCyncm9qGiqob731pPYnwM91425JjZN9J21NQ63skvBmDuih2M7dO5iZ+Q+nTmLmHn9il9ufXsbP786TZ++9Z6r8sRjyzduo89FVV0TozjzVVFatU1k8Jdwo6ZcfcFA7h2XC8ee38zjyzY5HVJ4oH5a4qIi/HxHxcNZG9FFR9v3uN1SRFF4S5hycy497IhXDEyk/vfWs8zi7Z4XZK0Iuccb60tYmK/Llw8LIOkeD9zl+/0uqyIop67hC2fz7j/qmGUV1bzy7lr8BkMyezIoaoaKiqrOVRVQ3llNYeqqqmorKn7WlXD4aoaLh+Zydn9dUOYSLWuqIztew9z+5R+JMTGcP6Qbry1uogjR4eQEKtpssFQuEtY88f4+L+rR3LzM3n84m9rTjq2fVwM7eP8VNfW8sbKXTx/yzjGZOlDuEj01poizODcgekAXDq8O7OXFvL++lIuGNLN4+oig8Jdwl5CbAyzpufy0cbdxMQYiXF+2sfFkBjvJzHwtV1sDD5f3ayafRVVXPnYImY8m8ec2yeSlZro8RFIc81fU8zoXp1IS6pbb+jMvl3okhjH6yt2KtyDpJ67RISE2Lplh6fmdGVsn84MyexIn9REuiYnkBjv/yrYATolxvHUd8ZgZtz49Gfsq6jysHJpru17D7F210HOH/zPEPfH+LhoaAbv5BdrFdEgKdwlKmWlJvLEDaPZsf8wM57L48jRGq9LkiDNX1s3t/28QenHbL90RHcqq2t5J7BfTi6ocDezLWa2ysyWm9kJF4QxszFmVmNmV4WuRJFTM7p3Zx745nA+27KPn+iK14gxf00ROelJx7XTRvfqRPeOCcxdoVkzwWjOmftU59yIEy1YY2YxwG+At0JSmUgIfG1Yd35yQQ5zV+zkgbc3eF2ONGFPeSWfbdnL+YPTj9vn8xmXDO/Owg2larUFIZRtmTuBV4GSED6nSIt97+y+fHtMTx5esIm/fLbd63LkJN5dV0Ktg2mDG//Q9JLh3amudfxjdVErVxY6BaXlrXK1bbDh7oD5ZrbUzGY03GlmmcAVwOOhLE4kFMyMey8fwllnpPLzOav4aGNobhUooTd/TRGZKe0Y3D250f2DuyeTnZrI6xHamllXdJArHl3Ef72Rf9pfK9hwn+icGwVcCNxhZpMb7P8D8FPn3Ek/tTKzGWaWZ2Z5paWlp1CuyKmJjfHxyLWj6JvWge/9eSkbisu8LkkaqKisZuHG3Zw3KP2Ei8WZ1bVmPv1iD8UHj7RyhS2zZXcF181aQkKsj5sm9TntrxdUuDvndga+lgBzgLENhuQCL5nZFuAq4FEzu7yR55npnMt1zuWmpenqQWldyQmx/OnGMSTExXDjU59RUtZ0OFRV15K/6yB/W76DLbsrWqHKtmvhhlKqqmuPmQLZmEuGd8c5+PvKXa1UWcvt3H+Ya2ctptY5/nzTOHp2bn/aX7PJi5jMLBHwOefKAt9PA35df4xzrk+98U8Df3fO/TXEtYq0WGZKO/40fQzf/OMn3PxMHi/NGP/Vbf12l1eSv+sg+bsOsm5XGWt3HWRzaTlHa+pm2cT7ffz4/By+O7HPMfPqJTTmry0mpX0sY7I6nXRcv64dGJSRzNwVO1vlDLildpdXct2Tizl4+CgvzhjPGelJrfK6wVyhmg7MCfwzyQ+84JybZ2a3ATjn1GeXiDK0R0ceunokM57L47pZi0mM95O/q4zd5f+82Xd6cjwDM5KZktOVgRlJ9O6SyMPvbeS/3sjn7bXF/O4bw1vl7KutOFpTy7v5xZw3qBv+mKYbCpeO6M59/1jHtj2H6NUlfN+HA4ePcsOTS9i5/zDP3TSOIZkdW+21dQ9VabOe+3Qr/zt/Pd07tmNgRjIDM5IYlJHMgIxkOifGHTfeOcfspYX8+vW11DrHf35tEN8e01M3EwmBjzbu5ronFzPz+tEnnClT3479h5l433v8+Pwc7pjarxUqbL5DVdVc/+QSVhbuZ9b0MSFbyC7Ye6hqbRlps64f35vrx/cOeryZ8Y3cnpzZL5Ufv7KCn722ivlrirjv68NIT044jZVGv7fWFJEQ6+OsM4ILwMyUduT27sTc5TvDMtyPHK1hxrNLWbZtH49cM8qTFUq1/IBIM2WmtOPPN43jnksH80nBHqb9fqGummyB2lrH22uLObt/Gu3igl/O95Lh3VlfXMb6ovCa+VRdU8u/vbiMjzbt5rdXDefCoRme1KEzd5FT4PMZ08/M4qwzUrnrlRX824vLeGtNEfdeNqTRls7eiio2l5azqaTuz+bScgpKK+iaFM+ZfbtwZr9URvZKId7f9tYqX7XjAEUHj/DjQTnN+rmLhmZwz+treH3FTnK6Ne9nT5faWsdPZq9k/tpi7rl0MFeN7uFZLQp3kRbITuvAK7dO4I8LC/jDOxtYXLCXu6b1p6Kyms2lFWwuKWdTaTl7610uH+/3kZ3WgaGZHSncf5iHF2ziofc2kRDrY0xWZyb07cLEvqkMyexITBuYlfPWmiJifMY5A7s26+fSkuKZ2C+VuSt2cte0/p5/9uGc45dz1/Dash38aFp/pp+Z5Wk9CneRFvLH+Lhjaj/+ZUBXfvjycn722ioAOrWPpV/XDkwblE6/rh3o27UD/dI6kJnS7piplAcOH2XJF3tZtHk3izbt4bfz1gPrSUrwM65PFyb268KwHh2pqnaUV1ZTUVlNWeBr+ZFqyiurv9peXllNz87tmdg3lQl9uzT6r4hwM39tMeP6dCalffNrvWR4d34yeyUrCg8womfKaaguePe/tZ7nPt3KrWdnh8XnAAp3kRAZmJHM3H+dxPqiMrqnJNClQ3xQP9exXSznDUr/aonb0rJKPinYwyebd/Pxpj28k3/iJW59BonxfpLi/STG193EZO7ynbyweBsAgzKSObNvFyb2S2Vsn84kxofXX/kvW1XXjet1Sj9//uBu/Oec1by+YmdQ4V504AgPvruR1TsO8Oi1o0I2nfXR9zfx6PubuXZcL+6+YIDn/4oAhbtISMX5fQzt0bK5zGlJ8Vw6vDuXDu8O1N28YmNJGQmxMSTFx5IYH0OHBD8dAnegahgk1TW1rCg88NUvh2c/2cqsj77A7zNG9EzhzH6pTOzbhRFh0OOfv6buF1cw0x8b07FdLGfnpPH3lTv5+UUDT9jG2ldRxeMfbObpRVuodY64GB/XPbmYV26dQNcWznR67pMt/Hbeei4b0Z17LxsSFsEOmucuEvWOHK0hb8s+Fm3ezceb97CqcD+1DjrE+7l0RHeuGdurVS+uqe/yRz6mptbx+p2TTvk5Xl+xkztfXMaLt4xnQt8ux+yrqKzmTx99wcyFBZRXVXPFiEx+eF7/uqtGZy2me0o7Xr51wim3r15dWshdr6zgvEHpPHrtKGKDuACrpTTPXUSAulsUTjojlUlnpAJ1Pf7FBXuYt6aIV5cW8sLibQzvmcK1Y3vxteEZXy3H0By7yytZtHkPJQePcOWoHkGFZfHBIyzfvp8fTevf7Ner79yB6bSPi+H1lTu/CvfK6hpeXLyNhxdsYnd5FecNSufi9YdZAAAIcklEQVRH03LI6VZ36X/Pzu2ZNX0M33lqCdP/tITnbxlHckJss1533uoifjx7BRP7deH/rh7ZKsHeHDpzF2nDDhw6yqufF/LCkm1sKiknKd7PFaMyuWZcLwZ0a3zZXYDyymqWfLGHjzft4eNNu1lXb655YlwM08/M4pazsul0kpB/7tOt/OKvq5n/w8n0b+F6K99/aRkLN5Tyyc/O4e8rd/H7tzewY/9hxmd35icXDGBUr8bXq3lvXTEznl3KyF4pPPvdcUHPs1+4oZSbn8ljSGYyz900rlU/ywj2zF3hLiI45/hsyz5eWLyVN1cVUVVTy+jenbhmbC8uHpaBGSzbtp9Fm+paOyu276e61hHn9zEmqxNn9k1lYr9UEmJ9PPzeJt5YtYv2sTF8Z2JdyDc2E+b6JxdTuO8w7911dov71O+sLebmZ/NIT46n+GAlQzM78uPzczjrjNQmn/uNlbu488XPmdgvlVnTc5v8HOKzLXu5/snF9EntwEu3jKdj++ad8beUwl1ETsneiipeXVrIi0u2UbC7gqQEP9U1jsNHa/AZDO2RwsTADJzRvTuREHt8GK4vKuOhdzfyxqpddIj3c+PELG6elP1VEB44fJTR977NTZP68LOLBra45qrqWs6+fwHt4mL40bQcLhzSrVm/MP7y2XZ+8upKzh+cziPXjDrh4mWrdxzg6pmfkpYcz19unUBqkDOiQknhLiIt4pzjk4I9vPb5DhLjYjizXyrjs7vQsV3wZ6rrig7y0LsbeXNVEUmBkL9pUjbvbyjh+y8t59Xvncno3idf4jdYR47WEBvjO+ULv576+AvueX0tV47M5HffGH7css6bSsr45h8/pV1sDK/cNoHuKe1CUXazKdxFJGzk76oL+X+srgv51KR4yo5Us+Tn54TV2vgPv7eR383fwHXjex0zrXH73kNc9fgiah28cusEslITPatRs2VEJGwMzEjmsetGs3bnQR58dwNvrSlm+oTeYRXsAHdM7UdZZTV//KCAxHg/d18wgJKySq6dtZgjR2t5+dbxngZ7cyjcRaTVDOqezB+vz6Vw3yFP+tVNMTPuvmAAFYGAN4x384vZU17J87eMP+kMonCjcBeRVtejU/jePcnM+PWlQzhUWcPjH2wm3u/j6RvHer52TXMp3EVEGvD5jN9eNYwendoxPrvLcVe+RoKgwt3MtgBlQA1Q3bCZb2bXAj8NPCwHvuecWxHCOkVEWpU/xse/TwuPdeJPRXPO3Kc653afYN8XwNnOuX1mdiEwExjX4upEROSUhKQt45xbVO/hp4B3tx8REZGg76HqgPlmttTMZjQx9ibgHy0rS0REWiLYM/eJzrmdZtYVeNvM1jnnFjYcZGZTqQv3RtfvDPximAHQq9epLc4vIiJNC+rM3Tm3M/C1BJgDjG04xsyGAbOAy5xze07wPDOdc7nOudy0tLRTr1pERE6qyXA3s0QzS/rye2AasLrBmF7Aa8D1zrkNp6NQEREJXjBtmXRgTmCNBT/wgnNunpndBuCcexz4f0AX4NHAuOOmS4qISOtpMtydcwXA8Ea2P17v+5uBm0NbmoiInCrPVoU0s1Jga4PNqcCJ5tJHomg7Hoi+Y4q244HoO6ZoOx5o2TH1ds41+aGlZ+HeGDPLi6Z2TrQdD0TfMUXb8UD0HVO0HQ+0zjGF1x1dRUQkJBTuIiJRKNzCfabXBYRYtB0PRN8xRdvxQPQdU7QdD7TCMYVVz11EREIj3M7cRUQkBMIi3M3sAjNbb2abzOxur+sJBTPbYmarzGy5mUXkncDN7E9mVmJmq+tt62xmb5vZxsDX0Ny6vhWc4Hh+ZWY7Au/TcjO7yMsam8PMeprZAjPLN7M1Zvb9wPZIfo9OdEwR+T6ZWYKZLTGzFYHjuSewvY+ZLQ68Ry+bWVzIX9vrtoyZxQAbgPOAQuAz4Grn3FpPC2uhwA1Ock+yBn7YM7PJ1N185Vnn3JDAtt8Ce51z9wV+EXdyzv30ZM8TLk5wPL8Cyp1zv/OytlNhZhlAhnPu88ASIUuBy4HvELnv0YmO6ZtE4PtkdZfsJzrnys0sFvgI+D7w78BrzrmXzOxxYIVz7rFQvnY4nLmPBTY55wqcc1XAS8BlHtckQGDlz70NNl8GPBP4/hnq/uJFhBMcT8Ryzu1yzn0e+L4MyAcyiez36ETHFJFcnfLAw9jAHwf8CzA7sP20vEfhEO6ZwPZ6jwuJ4DeznuasgR9J0p1zu6DuLyLQ1eN6QuFfzWxloG0TMS2M+swsCxgJLCZK3qMGxwQR+j6ZWYyZLQdKgLeBzcB+51x1YMhpybxwCHdrZFs0TOGZ6JwbBVwI3BFoCUj4eQzoC4wAdgH/6205zWdmHYBXgR845w56XU8oNHJMEfs+OedqnHMjqLtD3VhgYGPDQv264RDuhUDPeo97ADs9qiVkglkDP0IVB/qiX/ZHSzyup0Wcc8WBv3y1wBNE2PsU6OO+CjzvnHstsDmi36PGjinS3ycA59x+4H1gPJBiZl8u3HhaMi8cwv0z4IzAp8dxwLeBuR7X1CLBrIEfweYC0wPfTwf+5mEtLfZlCAZcQQS9T4EP654E8p1zD9TbFbHv0YmOKVLfJzNLM7OUwPftgHOp+xxhAXBVYNhpeY88ny0DEJjW9AcgBviTc+6/PS6pRcwsm7qzdfjnGvgRd0xm9iIwhboV7IqBXwJ/Bf4C9AK2Ad9wzkXEh5QnOJ4p1P1T3wFbgFu/7FeHOzObBHwIrAJqA5t/Tl2POlLfoxMd09VE4PtkdXeoe4a6bPMBf3HO/TqQES8BnYFlwHXOucqQvnY4hLuIiIRWOLRlREQkxBTuIiJRSOEuIhKFFO4iIlFI4S4iEoUU7iIiUUjhLiIShRTuIiJR6P8DJr4fqPsgTQkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(epoch_data,error_data)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "nd.save(\"data/saved_params\",params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best\n"
     ]
    }
   ],
   "source": [
    "sent = \"this is the\" #....\n",
    "sent = create_sent_representation(sent.split())\n",
    "h = nd.zeros(shape=(1, num_hidden), ctx=ctx)\n",
    "c = nd.zeros(shape=(1, num_hidden), ctx=ctx)\n",
    "outputs,_,_ = lstm_rnn(nd.array(sent,  ctx=ctx),h,c)\n",
    "print(inv_vocab[np.argmax(outputs[-1].asnumpy())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy for predicting last word is: 0.02046783625730994\n",
      "accuracy for predicting half the sentence is: 0.0\n"
     ]
    }
   ],
   "source": [
    "print (\"accuracy for predicting last word is:\" ,test_fn(test_data_org))\n",
    "print (\"accuracy for predicting half the sentence is:\" ,test_fn(test_data_org,type=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using FastText Word Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(vocab_dict)\n",
    "num_inputs = embedding_size\n",
    "num_hidden = 100\n",
    "num_outputs = vocab_size\n",
    "\n",
    "#  Weights connecting the inputs to the hidden layer\n",
    "Wxg = nd.random_normal(shape=(num_inputs,num_hidden), ctx=ctx) * .01\n",
    "Wxi = nd.random_normal(shape=(num_inputs,num_hidden), ctx=ctx) * .01\n",
    "Wxf = nd.random_normal(shape=(num_inputs,num_hidden), ctx=ctx) * .01\n",
    "Wxo = nd.random_normal(shape=(num_inputs,num_hidden), ctx=ctx) * .01\n",
    "\n",
    "#  Recurrent weights connecting the hidden layer across time steps\n",
    "Whg = nd.random_normal(shape=(num_hidden,num_hidden), ctx=ctx)* .01\n",
    "Whi = nd.random_normal(shape=(num_hidden,num_hidden), ctx=ctx)* .01\n",
    "Whf = nd.random_normal(shape=(num_hidden,num_hidden), ctx=ctx)* .01\n",
    "Who = nd.random_normal(shape=(num_hidden,num_hidden), ctx=ctx)* .01\n",
    "\n",
    "#  Bias vector for hidden layer\n",
    "bg = nd.random_normal(shape=num_hidden, ctx=ctx) * .01\n",
    "bi = nd.random_normal(shape=num_hidden, ctx=ctx) * .01\n",
    "bf = nd.random_normal(shape=num_hidden, ctx=ctx) * .01\n",
    "bo = nd.random_normal(shape=num_hidden, ctx=ctx) * .01\n",
    "\n",
    "# Weights to the output nodes\n",
    "Why = nd.random_normal(shape=(num_hidden,num_outputs), ctx=ctx) * .01\n",
    "by = nd.random_normal(shape=num_outputs, ctx=ctx) * .01\n",
    "\n",
    "\n",
    "# The gradients attached\n",
    "params = [Wxg, Wxi, Wxf, Wxo, Whg, Whi, Whf, Who, bg, bi, bf, bo, Why, by]\n",
    "for param in params:\n",
    "    param.attach_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_vects = []\n",
    "for sent in all_data:\n",
    "    all_vects.append(create_sent_representation(sent,type=\"pre-trained\"))\n",
    "train_data_org = all_vects[:l_train]\n",
    "test_data_org = all_vects[l_train:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_cont=[]\n",
    "for sent in train_data_org:\n",
    "    train_data_cont.extend(sent)\n",
    "    \n",
    "# Splitting into batches\n",
    "\n",
    "batch_size = 128\n",
    "seq_length = 1\n",
    "# -1 here so we have enough characters for labels later\n",
    "num_samples = (len(train_data_cont) - 1) // seq_length\n",
    "dataset = train_data_cont[:seq_length*num_samples]\n",
    "dataset = np.array(dataset).reshape((num_samples, seq_length, embedding_size))\n",
    "num_batches = len(dataset) // batch_size\n",
    "train_data = dataset[:num_batches*batch_size].reshape((num_batches, batch_size, seq_length, embedding_size))\n",
    "# swap batch_size and seq_length axis to make later access easier\n",
    "train_data = np.swapaxes(train_data, 1, 2)\n",
    "# labels = (train_data_cont[1:seq_length*num_samples+1])[:num_batches*batch_size]\n",
    "# train_label = np.array(labels).reshape((num_batches, batch_size, seq_length, embedding_size))\n",
    "# train_label = np.swapaxes(train_label, 1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0. Loss: 6.48249\n",
      "Epoch 5. Loss: 5.76998\n",
      "Epoch 10. Loss: 5.25046\n",
      "Epoch 15. Loss: 4.97071\n",
      "Epoch 20. Loss: 4.8946\n",
      "Epoch 25. Loss: 4.87144\n"
     ]
    }
   ],
   "source": [
    "epochs = 30\n",
    "learning_rate = 4\n",
    "\n",
    "train_data = nd.array(train_data, ctx=ctx)\n",
    "train_label = nd.array(train_label, ctx=ctx)\n",
    "\n",
    "h = nd.zeros(shape=(batch_size, num_hidden), ctx=ctx)\n",
    "c = nd.zeros(shape=(batch_size, num_hidden), ctx=ctx)\n",
    "epoch_data=[]\n",
    "error_data=[]\n",
    "# state = nd.zeros(shape=(batch_size, num_hidden), ctx=ctx)\n",
    "for e in range(epochs):\n",
    "    # Attenuate the learning rate by a factor of 2 every 2 epochs\n",
    "    if ((e+1) % 5 == 0):\n",
    "        learning_rate = learning_rate / 2.0\n",
    "    \n",
    "    for i in range(num_batches):\n",
    "        data_one_hot = train_data[i]\n",
    "        label_one_hot = train_label[i]\n",
    "        with autograd.record():\n",
    "            outputs, h, c = lstm_rnn(data_one_hot, h, c)\n",
    "            loss = average_ce_loss(outputs, label_one_hot)\n",
    "            loss.backward()\n",
    "        SGD(params, learning_rate)\n",
    "        op_loss = nd.mean(loss).asscalar()\n",
    "    epoch_data.append(e+1)\n",
    "    error_data.append(op_loss)\n",
    "    if e%5==0:\n",
    "        print(\"Epoch %s. Loss: %s\" % (e, op_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VeW59/HvnQlIgCRACBkYBJGZIEQEcaIOSBWxrbV6arWe9qVY9a3t28HTY9XaanvaWqvWoVSttdVqa4tQxQHnESVomEcZJIQhEAhJGEPu94+98cQYYGdcO3v/Pte1r+y9hr3v51ryW8tnrfUsc3dERCR+JARdgIiItC0Fv4hInFHwi4jEGQW/iEicUfCLiMQZBb+ISJxR8IuIxBkFv4hInFHwi4jEmaSgC2hIjx49vF+/fkGXISLSbixYsGC7u2dFsmxUBn+/fv0oKioKugwRkXbDzDZEuqy6ekRE4oyCX0QkzkQU/GaWYWZPmdkKM1tuZuPrzT/TzCrMrDj8uqnOvPPMbKWZrTGzG1q6ASIi0jiR9vHfBTzv7hebWQqQ2sAyb7r7BXUnmFkicC9wDlACzDez2e6+rDlFi4hI0x3ziN/MugKnAw8BuPsBd98V4fePBda4+1p3PwA8AUxtarEiItJ8kXT19AfKgD+Z2Ydm9qCZpTWw3HgzW2hmz5nZsPC0PGBjnWVKwtNERCQgkQR/EjAauN/dTwSqgfp99R8Afd29ALgHeDo83Rr4vgYf+WVm08ysyMyKysrKIipeREQaL5LgLwFK3P298OenCO0IPuHuu929Kvx+DpBsZj3C6/aus2g+UNrQj7j7DHcvdPfCrKyI7kH4lIOHanng9Y9YsGFno9cVEYknxwx+d98CbDSzQeFJZwGfOjlrZr3MzMLvx4a/dwcwHxhoZseFTwpfCsxuwfo/caCmlj+/s57/nrmYg4dqW+MnRERiQqTX8V8HPGZmi4BRwO1mNt3MpofnXwwsMbOFwN3ApR5SA1wLvAAsB/7u7ktbtgkhaR2SuOXCYazYUsmf3l7XGj8hIhITzL3BLvdAFRYWelOHbPjmn4t4e812Xvp/Z5CX0amFKxMRiU5mtsDdCyNZNubu3L3lwqEA3DyrVf7HQkSk3Yu54M/PTOW75wzkpeVbeXHplqDLERGJOjEX/ABXTTiOwb26cMvspVTvrwm6HBGRqBKTwZ+cmMBtXxhBacU+7py7KuhyRESiSkwGP8CYvplcNrYPf3pnPUtLK4IuR0QkasRs8APccN5gMjol898zl3CoNvquXhIRCUJMB396ajI3XjCE4o27+Nv7HwddjohIVIjp4Ae4aFQepwzozv88v4JtlfuCLkdEJHAxH/xmxs8uGs7+g7Xc9uzyoMsREQlczAc/wICszlx95gBmFZfy5mqN/Cki8S0ugh/g6jMHcFyPNH7y9BL2HTwUdDkiIoGJm+DvmJzIz6YOZ/2OPdz32kdBlyMiEpi4CX6AUwf2YOqoXB547SM+KqsKuhwRkUDEVfAD3Hj+UDokJ3DjzCVE48ikIiKtLe6CP6tLB3503mDeXbuD2QsbfBiYiEhMi7vgB7hsbB9G5qdz27PLqdx3MOhyRETaVFwGf2KC8bOpwymr2s9dL60OuhwRkTYVl8EPUNA7g0tPCg3itnJLZdDliIi0mbgNfoAfThpEl45J/GSWTvSKSPyI6+DPTEvhh5MG8/66cmYV60SviMSHuA5+gK+c1JuC/HRum6MTvSISHyIKfjPLMLOnzGyFmS03s/H15n/VzBaFX++YWUGdeevNbLGZFZtZUUs3oLkSE4xbpw5ne9V+fqcTvSISByI94r8LeN7dBwMFQP1hLtcBZ7j7SOBnwIx68ye6+yh3L2xWta2koHcGl43twyPvrGfFlt1BlyMi0qqOGfxm1hU4HXgIwN0PuPuuusu4+zvuvjP8cR6Q39KFtrYfnDuIrh2TuOnppTrRKyIxLZIj/v5AGfAnM/vQzB40s7SjLP8N4Lk6nx140cwWmNm0ZtTaqjLTUvjReYN5f305TxdvCrocEZFWE0nwJwGjgfvd/USgGrihoQXNbCKh4P9RnckT3H00MBm4xsxOP8K608ysyMyKysqCGTP/ksLeFPTO4LZnV7BbJ3pFJEZFEvwlQIm7vxf+/BShHcGnmNlI4EFgqrvvODzd3UvDf7cBM4GxDf2Iu89w90J3L8zKympcK1pIQoLx86nD2VG9nzvnrgqkBhGR1nbM4Hf3LcBGMxsUnnQWsKzuMmbWB/gX8DV3X1VnepqZdTn8HjgXWNJCtbeKEfnp/MfYPvz5nfUsK9WJXhGJPZFe1XMd8JiZLQJGAbeb2XQzmx6efxPQHbiv3mWb2cBbZrYQeB941t2fb8H6W8UPJg0ivVMyN+mOXhGJQRaNwVZYWOhFRcFe8v/k/I/50T8Xc8eXC/jSmHZ3kZKIxBkzWxDpJfNxf+fukXx5TG9G9c7gF88tp2KvTvSKSOxQ8B9BQoLx84uGs6P6ALfM1rX9IhI7FPxHMTwvne+efQIzP9zEQ2+tC7ocEZEWoeA/hmsnHs/k4b24fc5yXl8VzP0FIiItScF/DAkJxm++XMAJ2V247vEPWLe9OuiSRESaRcEfgbQOSfzxikISE4z/82iRhm8WkXZNwR+h3t1Sue+rY1i/vZrrnyjmUK1O9opI+6Tgb4TxA7pz85ShvLxiG3e8uDLockREmiQp6ALam8vH9WXZ5t3c99pHDMnpypSC3KBLEhFpFB3xN5KZ8dMLh3NSv0x+8NRClmyqCLokEZFGUfA3QUpSAvdfPoZuqSlMe7SIssr9QZckIhIxBX8T9ejcgRlXFFK+5wDffmwBB2pqgy5JRCQiCv5mGJ6Xzq8vLmD++p3crGEdRKSd0MndZppSkMvy8MneoTld+Nr4fkGXJCJyVDribwHfP3cQnxvck1ufWcaOKvX3i0h0U/C3gIQE4/vnDuLgIeeFpVuDLkdE5KgU/C1kSE4X+mel8cyi0qBLERE5KgV/CzEzLhiRw7y1O3R5p4hENQV/C7qgIJdah+eXbA66FBGRI1Lwt6ATsrswsGdn/r1IwS8i0UvB38IuGJnL/PXlbN29L+hSREQaFFHwm1mGmT1lZivMbLmZja8338zsbjNbY2aLzGx0nXlXmtnq8OvKlm5AtDl/ZA7uMGexjvpFJDpFesR/F/C8uw8GCoDl9eZPBgaGX9OA+wHMrBtwM3AyMBa42cwyW6DuqHV8z84M7tWFZ9TdIyJR6pjBb2ZdgdOBhwDc/YC776q32FTgUQ+ZB2SYWQ4wCZjr7uXuvhOYC5zXoi2IQlMKclmwYSelu/YGXYqIyGdEcsTfHygD/mRmH5rZg2aWVm+ZPGBjnc8l4WlHmh7Tzh+RA6i7R0SiUyTBnwSMBu539xOBauCGestYA+v5UaZ/hplNM7MiMysqKyuLoKzo1a9HGsPzuurqHhGJSpEEfwlQ4u7vhT8/RWhHUH+Z3nU+5wOlR5n+Ge4+w90L3b0wKysrktqj2gUjc1m4cRcby/cEXYqIyKccM/jdfQuw0cwGhSedBSyrt9hs4Irw1T3jgAp33wy8AJxrZpnhk7rnhqfFvMPdPc+qu0dEokykV/VcBzxmZouAUcDtZjbdzKaH588B1gJrgD8C3wZw93LgZ8D88OvW8LSY17tbKgW9MzR2j4hEnYjG43f3YqCw3uQH6sx34JojrPsw8HBTC2zPpozM4efPLmf99mr69ah/PlxEJBi6c7cVfV7dPSIShRT8rSg3oxNj+mby74Xq7hGR6KHgb2Xnj8hhxZZK1myrCroUERFAwd/qzh+Zgxk8q2v6RSRKKPhbWXbXjpzUr5uu7hGRqKHgbwMXjMxh9bYqVm6pDLoUEREFf1uYPDyHBINnddQvIlFAwd8Gsrp0YFz/7jyzaDOhWx5ERIKj4G8j54/MYe32apZt3h10KSIS5xT8beS8Yb1ITDBd3SMigVPwt5HunTtwygB194hI8BT8beiCkTl8XL6HJZvU3SMiwVHwt6FJw3qRlGC6pl9EAqXgb0MZqSmcOrCHuntEJFAK/jZ2wchcNu3aS/HG+s+rFxFpGwr+NnbusGw6Jifw3SeLeW/tjqDLEZE4pOBvY107JvOnr4/lkDtfmTGPG59eTOW+g0GXJSJxRMEfgPEDuvPC9afznxOO47H3PmbSnW/w2sptQZclInFCwR+Q1JQkbpoylKemn0JqhyS+/qf5fO/vxezacyDo0kQkxin4AzambybPXHcq1048nlnFpZz92zd4To9qFJFWpOCPAh2TE/n+pEHMvnYC2V07cPVjH3D1XxewrXJf0KWJSAyKKPjNbL2ZLTazYjMramD+D8Lzis1siZkdMrNukawr/2tYbjpPXzOBH0waxMvLt3HOb9/g6Q83BV2WiMSYpEYsO9Hdtzc0w91/DfwawMymAN919/JI1pVPS05M4JqJxzNpWC9+9M9FXP9kMdsq9zHt9AFBlyYiMaI1unouA/7WCt8bV47v2Zknp43j/BE53D5nBTPe+CjokkQkRkR6xO/Ai2bmwB/cfUZDC5lZKnAecG1j15XPSkpM4K5LRwFw+5wVADryF5FmizT4J7h7qZn1BOaa2Qp3f6OB5aYAb9fr5oloXTObBkwD6NOnTyObEbvqh787fOsMhb+INF1EXT3uXhr+uw2YCYw9wqKXUq+bJ9J13X2Guxe6e2FWVlZk1ceJw+F//sgcfvHcCv7wurp9RKTpjhn8ZpZmZl0OvwfOBZY0sFw6cAYwq7HryrElJSZw11cU/iLSfJF09WQDM83s8PKPu/vzZjYdwN0fCC/3BeBFd68+1rotVXy8ORz+AL94bgUOTFe3j4g00jGD393XAgUNTH+g3udHgEciWVea7nD4G/DL50InfBX+ItIYjbmOX6JEUmICvwsf+Sv8RaSxFPztlMJfRJpKwd+O1Q//PQcO8d2zBxI+pyIi0iAFfzt3OPw7Jidy98urWVtWxW++XEDH5MSgSxORKKXgjwFJiQn8+uKRDMjqzK9eWMHGnXv54xVj6NmlY9CliUgU0rDMMcLMuPrMATxw+RhWbankot+/zbLS3UGXJSJRSMEfYyYN68U/po/HgYsfeIe5y7YGXZKIRBkFfwwanpfOrGsmMLBnZ6b9pYgHXv8Idw+6LBGJEgr+GNWza0ee/NZ4Pj8ih18+t4IfPrWIAzW1QZclIlFAJ3djWMfkRO659EQGZHXm7pdXs6F8Dw9cPoZuaSlBlyYiAdIRf4xLSDC+d84J3HXpKIo37uKie99mzbbKoMsSkQAp+OPE1FF5PDFtHHsOHOKie9/hmUWlQZckIgFR8MeR0X0ymXXtBE7I7sy1j3/Ij2cuZt/BQ0GXJSJtTMEfZ/IyOvHkt8Yz/YwBPP7ex+Gun6qgyxKRNqTgj0PJiQncMHkwj1x1Etsq9zPlnrd4akFJ0GWJSBtR8MexMwf15LnvnEZB73S+/4+FfO/vxVTvrwm6LBFpZQr+OJfdtSOPfXMc1589kJkfbmLK79/SUA8iMU7BLyQmGNeffQKPffNkqvbVcNF9b/PXeRt0t69IjFLwyydOGdCDOd85jXH9u3Pj00u45vEPqNx3MOiyRKSFKfjlU3p07sAjXz+JGyYP5oWlW7lz7uqgSxKRFqbgl89ISDCmnzGAc4ZkM3thKTWHNMaPSCyJKPjNbL2ZLTazYjMramD+mWZWEZ5fbGY31Zl3npmtNLM1ZnZDSxYvrWvqqFy2V+3n3bU7gi5FRFpQYwZpm+ju248y/013v6DuBDNLBO4FzgFKgPlmNtvdlzW+VGlrEwf3pEuHJGYVl3LawKygyxGRFtLaXT1jgTXuvtbdDwBPAFNb+TelhXRMTmTS8F48v2SLhnYQiSGRBr8DL5rZAjObdoRlxpvZQjN7zsyGhaflARvrLFMSnibtxNRRuVTtr+HVFduCLkVEWkikwT/B3UcDk4FrzOz0evM/APq6ewFwD/B0eLo18F0NXhxuZtPMrMjMisrKyiIsS1rb+P7d6dG5A7OKNZqnSKyIKPjdvTT8dxswk1AXTt35u929Kvx+DpBsZj0IHeH3rrNoPtBggrj7DHcvdPfCrCz1J0eLpMQEphTk8MrKbVTs1TX9IrHgmMFvZmlm1uXwe+BcYEm9ZXqZmYXfjw1/7w5gPjDQzI4zsxTgUmB2yzZBWtvUUXkcqKnlhaVbgi5FRFpAJFf1ZAMzw7meBDzu7s+b2XQAd38AuBi42sxqgL3ApR6637/GzK4FXgASgYfdfWkrtENaUUF+On27pzK7uJRLCnsfewURiWrHDH53XwsUNDD9gTrvfw/8/gjrzwHmNKNGCZiZMbUgl9+/uoZtu/fRs2vHoEsSkWbQnbsSkQtH5VLr8MyizUGXIiLNpOCXiBzfswvDcrsya6Gu7hFp7xT8ErGpo3JZuHEX67dXB12KiDSDgl8iNqUgFzOYraN+kXZNwS8Ry0nvxNh+3Xi6eJMe0iLSjin4pVGmjspjbVk1S/V4RpF2S8EvjTJ5eC+SE03dPSLtmIJfGiUzLYUzTshidnEptbXq7hFpjxT80mgXjspjy+59vLeuPOhSRKQJFPzSaGcP6UlqSiKzF24KuhQRaQIFvzRaakoS5w7NZs7iLeyv0QNaRNobBb80ydRReVTsPcgbq472NE4RiUYKfmmSUwf2oFtaCrOK1d0j0t4o+KVJkhMTOH9EDi8t30rV/pqgyxGRRlDwS5NNHZXLvoO1zF2mB7SItCcKfmmy0X0yycvopOfxirQzCn5psoQE48JRuby5ejs7qvYHXY6IREjBL80ydVQuh2qdOYv1gBaR9kLBL80yuFdXBmV34akFJew7qGv6RdoDBb802+Xj+rCwpILTfvUqD765lr0HtAMQiWYKfmm2r43vxxPTxjGwZ2d+/uxyTvvVK/zh9Y+o1mWeIlHJInmghpmtByqBQ0CNuxfWm/9V4Efhj1XA1e6+MJJ1G1JYWOhFRUWRt0Kixvz15dz98mreXL2dbmkpfPO047hifD86d0gKujSRmGZmCyLJV2hc8Be6e4P355vZKcByd99pZpOBW9z95EjWbYiCv/1bsGEn97yymtdWlpGRmsw3JhzHlRP60bVjctClicSkxgR/i3T1uPs77r4z/HEekN8S3yvt15i+mTxy1VhmXTOBwr6Z3DF3FRN++Qp3zl1Fxd6DQZcnEtciDX4HXjSzBWY27RjLfgN4ronrSowp6J3Bg1eexDPXncopA7pz18urOfV/XuHul1dTuU87AJEgRNrVk+vupWbWE5gLXOfubzSw3ETgPuBUd9/RyHWnAdMA+vTpM2bDhg3NaZdEqWWlu7nzpVXMXbaVjNRkvnX6AK48pS+pKToHINIcLd7HX+/LbwGq3P039aaPBGYCk919VWPWrU99/LFvUckufjt3Fa+tLKNH5xSmnzGAy8f1pWNyYtClibRLLdrHb2ZpZtbl8HvgXGBJvWX6AP8CvlY39CNZV+LTyPwMHrlqLP+8ejwnZHfh588u54xfv8pf3l2vh7uItLJjHvGbWX9CR/IAScDj7n6bmU0HcPcHzOxB4EvA4f6ZGncvPNK6xypKR/zx592PdvDbuSuZv34neRmduO5zx/OlMfkkJ+pWE5FItGpXT1tQ8Mcnd+fN1du5Y+4qFm7cRX5mJ64Y35dLCnuTkZoSdHkiUU3BL+2au/PKim384fW1vL++nA5JCVw0Ko+vje/L8Lz0oMsTiUqNCX5dSiFRx8w4a0g2Zw3JZvnm3Tz67gZmfljCk0UbKeybydfG92Xy8BxSktQNJNIUOuKXdqFiz0H+sWAjf5m3gQ079tCjcwf+4+Q+fPXkPmR37Rh0eSKBU1ePxKzaWuf11WU8+s56XltVRqIZk4b34qpT+jGmbyZmFnSJIoFQV4/ErIQEY+Kgnkwc1JP126v567wN/L1oI88u2kxB7wy+cepxTB7eS1cDiRyFjvil3aveX8M/Pyjh4bfWsX7HHnLTO3LlKf24dGwf0jtpUDiJD+rqkbhUWxu6GujBt9Yyb205qSmJXFLYm6sm9KNv97SgyxNpVQp+iXtLNlXw8Nvr+PfCUmpqnXOGZPPN0/pzUj+dB5DYpOAXCdu6ex9/eXcDf31vA7v2HGRc/248ctVYjQkkMafNx+MXiVbZXTvy/UmDePeGs7jx/CHMWxt6QphIPFPwS1zolJLIN0/rzyWF+Tzw+kcs3Lgr6JJEAqPgl7jy3+cPpWeXjvzgqYUaBVTiloJf4kp6p2R+8cURrNpaxT0vrwm6HJFAKPgl7kwc3JOLx+Rz/+sfsbikIuhyRNqcgl/i0k/OH0qPzil8/x8LOVBTG3Q5Im1KwS9xKT011OWzcmslv39FV/lIfFHwS9z63OBsvjg6j3tf+4glm9TlI/FDwS9x7eYLhtE9TV0+El8U/BLX0lOTuf0LI1ixpZJ7X9VVPhIfFPwS984ems0XTszj3lfXsLRUXT4S+xT8IsDNU4aSkZrC9/+xiIOH1OUjsS2i4Dez9Wa22MyKzewzo6dZyN1mtsbMFpnZ6DrzrjSz1eHXlS1ZvEhLyUhN4fYvDGf55t3c9+pHQZcj0qoac8Q/0d1HHWH0t8nAwPBrGnA/gJl1A24GTgbGAjebWWbzShZpHecO68XUUbnc88pqlpXuDrockVbTUl09U4FHPWQekGFmOcAkYK67l7v7TmAucF4L/aZIi7tlyjAyUpP5wVML1eUjMSvS4HfgRTNbYGbTGpifB2ys87kkPO1I0z/DzKaZWZGZFZWVlUVYlkjLykxL4ecXjWBp6W4m3fkGv5iznPfXlVOjnYDEkEgftj7B3UvNrCcw18xWuPsbdeY39EgjP8r0z050nwHMgNCDWCKsS6TFnTe8F3d8uYCnizfx8Nvr+MMba8lMTWbioJ6cPTSb00/IonOHSP/piESfiP7rdffS8N9tZjaTUH993eAvAXrX+ZwPlIann1lv+mtNL1ekbXxpTD5fGpNP5b6DvLFqOy8t38orK7fxrw83kZxojOvfnbOHZHPWkJ7kZ6YGXe4xle7ay8byPZzcv3vQpUgUOOajF80sDUhw98rw+7nAre7+fJ1lzgeuBT5P6ETu3e4+NnxydwFw+CqfD4Ax7l5+tN/UoxclGtUcqmXBhp28vGIbLy3bytrt1QAMyu7C+AHdGde/Oycf143MtJSAK/20WcWbuHHmEir31/DD8wZx9RkD9NzhGNSYRy9GcsSfDcwM/4eSBDzu7s+b2XQAd38AmEMo9NcAe4CrwvPKzexnwPzwd916rNAXiVZJiQmc3L87J/fvzo8/P4S1ZVW8vHwbr68q44n5H/PIO+sBGNyrC+P6B78j2L3vID95egmziksZ0zeTXl078qvnV1JedYAff34ICQkK/3ilh62LtIADNbUsKtnFvLU7mLe2nKIN5ew7GDohXHdHMKp3BtldO7T6Eff768r57pPFbNm9j++cNZBvnzmABDNufWYZj7yzni+emMf/XDyS5ETdwxkrGnPEr+AXaQV1dwTvrt3Bgg07P9kRdEtLYWhOV4bkdGFobleG5qTTPyutRUL44KFa7nppNfe9tobe3VK58yujGN3nf2+dcXd+/8oa7pi7is8N7sm9/zGaTimJzf5dCZ6CXyTK7K85xOKSCpZsqmD55kqWbd7Nyq2Vn4wImpKUwKDsLqGdQU5XhuelMzwvnY7JkYfyuu3VXP/EhywsqeCSwnxumjLsiFcf/XXeBn4yawlj+mTy0JUnkZ6a3CLtlOAo+EXagZpDtazdXs2y0t0s27z7k7/l1QcASEowhuV25cQ+mYzum8noPhnkZXT6TDeRu/P3oo389N/LSE5M4JdfHMHkETnH/P1nF23m+ic/pH+Pzjz6jbFkd+3YKu2UtqHgF2mn3J1tlftZVFLBBx/v5IMNO1lUUsHeg4cA6NmlA6P7ZDK6bwaj+2SSn5nKLbOX8vzSLZwyoDt3XFJATnqniH/vrdXbmfaXIrqlpfCXb5zMcT3SWqtp0soU/CIxpOZQLSu2VH6yI/jg4118XL7nk/nJicYPJw3mG6ce16QrdRZu3MVVj8wnweCRq8YyPC+9JcuXNqLgF4lx26v28+HHu1ixeTdnDclmaG7XZn3fR2VVXPHQ+1TsPciMK8ZwyoAeLVSptBUFv4g02uaKvVzx0Pts2LGHqyb048JRuQzN6aqbvdoJBb+INMnO6gP8178W89LyrdTUOgOy0riwII8LR+VGVf+/u7N4UwWGkZ/ZiYzU5LjfQSn4RaRZyqsPMGfxZmYvLOX9daGb7Ufmp3NhQS4XjMylV3pwVwBtLN/DzbOX8sqKbZ9MS01JJC+jE/mZncjL7ER+ZuqnPmd1bv2b5oKm4BeRFrO5Yi/PLAztBBZvqsAMxvbrxoWjcjlvWC+6d+7QJnXsrznEH99Yyz2vrCExwfi/Zw2kX/c0SnbuYdOuvWzauZeSnXvZtGsvFXsPfmrdjskJ9O2WRr8eqfTrkUa/7qHXcT3S2uRO6rag4BeRVrG2rIrZC0uZvbCUtWWhQer6dk9lRF46BfkZjMwP3XiW1sLDVr+zZjs3zlrC2rJqJg/vxU1Thh71stXKfQc/tTP4uHwPG3ZUs257NRvL93KgzvMVOiUn0rd7amhn0CONHp1T6NopmfROyXTtGPqbnhr6m5aSGLU7CQW/iLQqd2dp6W7eWF3Goo0VLN5UwaZdewEwg+OzOjMyP4OC3umMzM9gcK8ujboL+bBtlfu47dnlzCoupU+3VH46dRgTB/VsVu2Hap3SXXtZv6Oa9durWb9jD+u3V7NuRzUby/dw8NCRMzEpwT7ZKaR3SqZH5xS6p3Wge+cUunfuQPe0lND7tA706JxCZlpKm42HpOAXkTZXVrmfxZt2saikIvzaxfaq0F3IyYnGCdldGJ6bzvD8dIbndmVITtcj7gwO1Tp/nbeB37ywkv01tUw/cwDfPnNAk3YejVFb61QdqKFiz0Eq9h5k997Q34Zeu/YcZEf1AXZU7ae8+gA1tQ1naUZqMsdndWZ4XjojwkNxDMhKI6mFdwgKfhEJnLuzuWIfi0p2sTA8TtGSTRXs3BPqf09MMAb2rBuIoQHrVm4BVoksAAAGKElEQVSt5ManF7Nk025OPb4Ht04dRv+szgG35ujcnd17a9heHdoJ7Kjaz/aqA+yoOsC2yn2s2FLJstLdn9yB3TE5gaE5XT/ZEQzPS2dgz87N2hko+EUkKrk7m3btZcmm3SzZFOoiWrKpgh3h8YkSLPRs1qzOHbhpylDOH5ETtX3qjXWo1llbVhVuc6j9S0srqD4Q2hl0SEqgID+DJ781rkltbukHsYiItAgzIz8zlfzMVM4b3gsI7Qy27N73yeiliQkJ/Oep/ejSMbZGDE1MMAZmd2Fgdhe+GH4mYW2ts25HdWgnWFJB9YGaNtnR6YhfRCQGNOaIX4/fERGJMwp+EZE4o+AXEYkzCn4RkTgT8VU9ZpYIFAGb3P2CevPuBCaGP6YCPd09IzzvELA4PO9jd7+w2VWLiEiTNeZyzu8Ay4HPPPHB3b97+L2ZXQecWGf2Xncf1eQKRUSkRUXU1WNm+cD5wIMRLH4Z8LfmFCUiIq0n0j7+3wE/BGqPtpCZ9QWOA16pM7mjmRWZ2Twzu+go604LL1dUVlYWYVkiItJYx+zqMbMLgG3uvsDMzjzG4pcCT7n7oTrT+rh7qZn1B14xs8Xu/lH9Fd19BjAj/JtlZrahzuwewPZj1drOxFqbYq09EHttirX2QOy1qTnt6RvpgpH08U8ALjSzzwMdga5m9ld3v7yBZS8Frqk7wd1Lw3/XmtlrhPr/PxP89dbJqvvZzIoivSOtvYi1NsVaeyD22hRr7YHYa1NbteeYXT3u/l/unu/u/QgF+ysNhb6ZDQIygXfrTMs0sw7h9z0I7USWtVDtIiLSBE0epM3MbgWK3H12eNJlwBP+6cF/hgB/MLNaQjuZX7q7gl9EJECNCn53fw14Lfz+pnrzbmlg+XeAEU2u7n/NaIHviDax1qZYaw/EXptirT0Qe21qk/ZE5eicIiLSejRkg4hInIn64Dez88xspZmtMbMbgq6nucxsvZktNrNiM2uXDx0ws4fNbJuZLakzrZuZzTWz1eG/mUHW2BhHaM8tZrYpvJ2Kw1e1tRtm1tvMXjWz5Wa21My+E57eLrfTUdrTbreTmXU0s/fNbGG4TT8NTz/OzN4Lb6MnzSylxX87mrt6wuMDrQLOAUqA+cBl7fkEsZmtBwrdvd1ee2xmpwNVwKPuPjw87VdAubv/MryDznT3HwVZZ6SO0J5bgCp3/02QtTWVmeUAOe7+gZl1ARYAFwFfpx1up6O05xLa6Xay0KO20ty9ysySgbcIDY3zPeBf7v6EmT0ALHT3+1vyt6P9iH8ssMbd17r7AeAJYGrANcU9d38DKK83eSrw5/D7PxP6R9kuHKE97Zq7b3b3D8LvKwmNs5VHO91OR2lPu+UhVeGPyeGXA58DngpPb5VtFO3BnwdsrPO5hHa+sQlt2BfNbIGZTQu6mBaU7e6bIfSPFOgZcD0t4VozWxTuCmoXXSINMbN+hG6cfI8Y2E712gPteDuZWaKZFQPbgLmEbm7d5e414UVaJfOiPfgbeupw9PZNRWaCu48GJgPXhLsZJPrcDwwARgGbgTuCLadpzKwz8E/genffHXQ9zdVAe9r1dnL3Q+HRi/MJ9XAMaWixlv7daA/+EqB3nc/5QGlAtbSIOkNYbANmEtrYsWBruB/2cH/stoDraRZ33xr+R1kL/JF2uJ3C/cb/BB5z93+FJ7fb7dRQe2JhOwG4+y5C90iNAzLM7PA9Vq2SedEe/POBgeGz3CmEhoyYfYx1opaZpYVPTGFmacC5wJKjr9VuzAauDL+/EpgVYC3Ndjgcw75AO9tO4ROHDwHL3f23dWa1y+10pPa05+1kZllmdviBVZ2Aswmdu3gVuDi8WKtso6i+qgcgfHnW74BE4GF3vy3gkprMQiOUzgx/TAIeb4/tMbO/AWcSGklwK3Az8DTwd6AP8DHwZXdvFydMj9CeMwl1HziwHvjW4b7x9sDMTgXeJPT0u8PDqf+YUL94u9tOR2nPZbTT7WRmIwmdvE0kdBD+d3e/NZwTTwDdgA+By919f4v+drQHv4iItKxo7+oREZEWpuAXEYkzCn4RkTij4BcRiTMKfhGROKPgFxGJMwp+EZE4o+AXEYkz/x+cPoMOChtJqAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(epoch_data,error_data)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "nd.save(\"data/saved_params_2\",params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy for predicting last word is: 0.0\n",
      "accuracy for predicting half the sentence is: 0.030256251929607902\n"
     ]
    }
   ],
   "source": [
    "print (\"accuracy for predicting last word is:\" ,test_fn(test_data_org,encoding='pre-trained'))\n",
    "print (\"accuracy for predicting half the sentence is:\" ,test_fn(test_data_org,type=2,encoding='pre-trained'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}