6_lstm-Problem3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8tQJd2YSCfWR"
   },
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "D7tqLMoKF6uq"
   },
   "source": [
    "Deep Learning\n",
    "=============\n",
    "\n",
    "Assignment 6\n",
    "------------\n",
    "\n",
    "After training a skip-gram model in `5_word2vec.ipynb`, the goal of this notebook is to train a LSTM character model over [Text8](http://mattmahoney.net/dc/textdata) data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "MvEblsgEXxrd"
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "from __future__ import print_function\n",
    "import os\n",
    "import numpy as np\n",
    "import random\n",
    "import string\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 5993,
     "status": "ok",
     "timestamp": 1445965582896,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "RJ-o3UBUFtCw",
    "outputId": "d530534e-0791-4a94-ca6d-1c8f1b908a9e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified text8.zip\n"
     ]
    }
   ],
   "source": [
    "url = 'http://mattmahoney.net/dc/'\n",
    "\n",
    "def maybe_download(filename, expected_bytes):\n",
    "  \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "  if not os.path.exists(filename):\n",
    "    filename, _ = urlretrieve(url + filename, filename)\n",
    "  statinfo = os.stat(filename)\n",
    "  if statinfo.st_size == expected_bytes:\n",
    "    print('Found and verified %s' % filename)\n",
    "  else:\n",
    "    print(statinfo.st_size)\n",
    "    raise Exception(\n",
    "      'Failed to verify ' + filename + '. Can you get to it with a browser?')\n",
    "  return filename\n",
    "\n",
    "filename = maybe_download('text8.zip', 31344016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 5982,
     "status": "ok",
     "timestamp": 1445965582916,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "Mvf09fjugFU_",
    "outputId": "8f75db58-3862-404b-a0c3-799380597390"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size 100000000\n"
     ]
    }
   ],
   "source": [
    "def read_data(filename):\n",
    "  f = zipfile.ZipFile(filename)\n",
    "  for name in f.namelist():\n",
    "    return tf.compat.as_str(f.read(name))\n",
    "  f.close()\n",
    "  \n",
    "text = read_data(filename)\n",
    "print('Data size %d' % len(text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ga2CYACE-ghb"
   },
   "source": [
    "Create a small validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 6184,
     "status": "ok",
     "timestamp": 1445965583138,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "w-oBpfFG-j43",
    "outputId": "bdb96002-d021-4379-f6de-a977924f0d02"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99999000 ons anarchists advocate social relations based upon voluntary as\n",
      "1000  anarchism originated as a term of abuse first used against earl\n"
     ]
    }
   ],
   "source": [
    "valid_size = 1000\n",
    "valid_text = text[:valid_size]\n",
    "train_text = text[valid_size:]\n",
    "train_size = len(train_text)\n",
    "print(train_size, train_text[:64])\n",
    "print(valid_size, valid_text[:64])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Zdw6i4F8glpp"
   },
   "source": [
    "Utility functions to map characters to vocabulary IDs and back."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 6276,
     "status": "ok",
     "timestamp": 1445965583249,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "gAL1EECXeZsD",
    "outputId": "88fc9032-feb9-45ff-a9a0-a26759cc1f2e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unexpected character: ï\n",
      "1 26 0 None\n",
      "a z  \n"
     ]
    }
   ],
   "source": [
    "vocabulary_size = len(string.ascii_lowercase) + 1 # [a-z] + ' '\n",
    "first_letter = ord(string.ascii_lowercase[0])\n",
    "\n",
    "def char2id(char):\n",
    "  if char in string.ascii_lowercase:\n",
    "    return ord(char) - first_letter + 1\n",
    "  elif char == ' ':\n",
    "    return 0\n",
    "  else:\n",
    "    print('Unexpected character: %s' % char)\n",
    "    return \n",
    "  \n",
    "def id2char(dictid):\n",
    "  if dictid > 0:\n",
    "    return chr(dictid + first_letter - 1)\n",
    "  else:\n",
    "    return ' '\n",
    "\n",
    "print(char2id('a'), char2id('z'), char2id(' '), char2id('ï'))\n",
    "print(id2char(1), id2char(26), id2char(0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lFwoyygOmWsL"
   },
   "source": [
    "Function to generate a training batch for the LSTM model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 6473,
     "status": "ok",
     "timestamp": 1445965583467,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "d9wMtjy5hCj9",
    "outputId": "3dd79c80-454a-4be0-8b71-4a4a357b3367"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ons anarchi', 'when milita', 'lleria arch', ' abbeys and', 'married urr', 'hel and ric', 'y and litur', 'ay opened f', 'tion from t', 'migration t', 'new york ot', 'he boeing s', 'e listed wi', 'eber has pr', 'o be made t', 'yer who rec', 'ore signifi', 'a fierce cr', ' two six ei', 'aristotle s', 'ity can be ', ' and intrac', 'tion of the', 'dy to pass ', 'f certain d', 'at it will ', 'e convince ', 'ent told hi', 'ampaign and', 'rver side s', 'ious texts ', 'o capitaliz', 'a duplicate', 'gh ann es d', 'ine january', 'ross zero t', 'cal theorie', 'ast instanc', ' dimensiona', 'most holy m', 't s support', 'u is still ', 'e oscillati', 'o eight sub', 'of italy la', 's the tower', 'klahoma pre', 'erprise lin', 'ws becomes ', 'et in a naz', 'the fabian ', 'etchy to re', ' sharman ne', 'ised empero', 'ting in pol', 'd neo latin', 'th risky ri', 'encyclopedi', 'fense the a', 'duating fro', 'treet grid ', 'ations more', 'appeal of d', 'si have mad']\n",
      "['ists advoca', 'ary governm', 'hes nationa', 'd monasteri', 'raca prince', 'chard baer ', 'rgical lang', 'for passeng', 'the nationa', 'took place ', 'ther well k', 'seven six s', 'ith a gloss', 'robably bee', 'to recogniz', 'ceived the ', 'icant than ', 'ritic of th', 'ight in sig', 's uncaused ', ' lost as in', 'cellular ic', 'e size of t', ' him a stic', 'drugs confu', ' take to co', ' the priest', 'im to name ', 'd barred at', 'standard fo', ' such as es', 'ze on the g', 'e of the or', 'd hiver one', 'y eight mar', 'the lead ch', 'es classica', 'ce the non ', 'al analysis', 'mormons bel', 't or at lea', ' disagreed ', 'ing system ', 'btypes base', 'anguages th', 'r commissio', 'ess one nin', 'nux suse li', ' the first ', 'zi concentr', ' society ne', 'elatively s', 'etworks sha', 'or hirohito', 'litical ini', 'n most of t', 'iskerdoo ri', 'ic overview', 'air compone', 'om acnm acc', ' centerline', 'e than any ', 'devotional ', 'de such dev']\n",
      "[' a']\n",
      "['an']\n"
     ]
    }
   ],
   "source": [
    "batch_size=64\n",
    "num_unrollings=10\n",
    "\n",
    "class BatchGenerator(object):\n",
    "  def __init__(self, text, batch_size, num_unrollings):\n",
    "    self._text = text\n",
    "    self._text_size = len(text)\n",
    "    self._batch_size = batch_size\n",
    "    self._num_unrollings = num_unrollings\n",
    "    segment = self._text_size // batch_size\n",
    "    self._cursor = [ offset * segment for offset in range(batch_size)]\n",
    "    self._last_batch = self._next_batch()\n",
    "  \n",
    "  def _next_batch(self):\n",
    "    \"\"\"Generate a single batch from the current cursor position in the data.\"\"\"\n",
    "    batch = np.zeros(shape=(self._batch_size, vocabulary_size), dtype=np.float)\n",
    "    for b in range(self._batch_size):\n",
    "      batch[b, char2id(self._text[self._cursor[b]])] = 1.0\n",
    "      self._cursor[b] = (self._cursor[b] + 1) % self._text_size\n",
    "    return batch\n",
    "  \n",
    "  def next(self):\n",
    "    \"\"\"Generate the next array of batches from the data. The array consists of\n",
    "    the last batch of the previous array, followed by num_unrollings new ones.\n",
    "    \"\"\"\n",
    "    batches = [self._last_batch]\n",
    "    for step in range(self._num_unrollings):\n",
    "      batches.append(self._next_batch())\n",
    "    self._last_batch = batches[-1]\n",
    "    return batches\n",
    "\n",
    "def characters(probabilities):\n",
    "  \"\"\"Turn a 1-hot encoding or a probability distribution over the possible\n",
    "  characters back into its (most likely) character representation.\"\"\"\n",
    "  return [id2char(c) for c in np.argmax(probabilities, 1)]\n",
    "\n",
    "def batches2string(batches):\n",
    "  \"\"\"Convert a sequence of batches back into their (most likely) string\n",
    "  representation.\"\"\"\n",
    "  s = [''] * batches[0].shape[0]\n",
    "  for b in batches:\n",
    "    s = [''.join(x) for x in zip(s, characters(b))]\n",
    "  return s\n",
    "\n",
    "train_batches = BatchGenerator(train_text, batch_size, num_unrollings)\n",
    "valid_batches = BatchGenerator(valid_text, 1, 1)\n",
    "\n",
    "print(batches2string(train_batches.next()))\n",
    "print(batches2string(train_batches.next()))\n",
    "print(batches2string(valid_batches.next()))\n",
    "print(batches2string(valid_batches.next()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "KyVd8FxT5QBc"
   },
   "outputs": [],
   "source": [
    "def logprob(predictions, labels):\n",
    "  \"\"\"Log-probability of the true labels in a predicted batch.\"\"\"\n",
    "  predictions[predictions < 1e-10] = 1e-10\n",
    "  return np.sum(np.multiply(labels, -np.log(predictions))) / labels.shape[0]\n",
    "\n",
    "def sample_distribution(distribution):\n",
    "  \"\"\"Sample one element from a distribution assumed to be an array of normalized\n",
    "  probabilities.\n",
    "  \"\"\"\n",
    "  r = random.uniform(0, 1)\n",
    "  s = 0\n",
    "  for i in range(len(distribution)):\n",
    "    s += distribution[i]\n",
    "    if s >= r:\n",
    "      return i\n",
    "  return len(distribution) - 1\n",
    "\n",
    "def sample(prediction):\n",
    "  \"\"\"Turn a (column) prediction into 1-hot encoded samples.\"\"\"\n",
    "  p = np.zeros(shape=[1, vocabulary_size], dtype=np.float)\n",
    "  p[0, sample_distribution(prediction[0])] = 1.0\n",
    "  return p\n",
    "\n",
    "def random_distribution():\n",
    "  \"\"\"Generate a random column of probabilities.\"\"\"\n",
    "  b = np.random.uniform(0.0, 1.0, size=[1, vocabulary_size])\n",
    "  return b/np.sum(b, 1)[:,None]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "K8f67YXaDr4C"
   },
   "source": [
    "Simple LSTM Model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pl4vtmFfa5nn"
   },
   "source": [
    "---\n",
    "Problem 1\n",
    "---------\n",
    "\n",
    "You might have noticed that the definition of the LSTM cell involves 4 matrix multiplications with the input, and 4 matrix multiplications with the output. Simplify the expression by using a single matrix multiply for each, and variables that are 4 times larger.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4eErTCTybtph"
   },
   "source": [
    "---\n",
    "Problem 2\n",
    "---------\n",
    "\n",
    "We want to train a LSTM over bigrams, that is pairs of consecutive characters like 'ab' instead of single characters like 'a'. Since the number of possible bigrams is large, feeding them directly to the LSTM using 1-hot encodings will lead to a very sparse representation that is very wasteful computationally.\n",
    "\n",
    "a- Introduce an embedding lookup on the inputs, and feed the embeddings to the LSTM cell instead of the inputs themselves.\n",
    "\n",
    "b- Write a bigram-based LSTM, modeled on the character LSTM above.\n",
    "\n",
    "c- Introduce Dropout. For best practices on how to use Dropout in LSTMs, refer to this [article](http://arxiv.org/abs/1409.2329).\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "See https://github.com/vanhuyz/udacity-dl/blob/master/6_lstm-Problem2.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Y5tapX3kpcqZ"
   },
   "source": [
    "---\n",
    "Problem 3\n",
    "---------\n",
    "\n",
    "(difficult!)\n",
    "\n",
    "Write a sequence-to-sequence LSTM which mirrors all the words in a sentence. For example, if your input is:\n",
    "\n",
    "    the quick brown fox\n",
    "    \n",
    "the model should attempt to output:\n",
    "\n",
    "    eht kciuq nworb xof\n",
    "    \n",
    "Refer to the lecture on how to put together a sequence-to-sequence model, as well as [this article](http://arxiv.org/abs/1409.3215) for best practices.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try to mirror fix-length sequences. So no need `<pad>` symbol."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unexpected character: ï\n",
      "3 28 0 None 1 2\n",
      "<go> x  \n",
      "[22, 10, 7, 0, 19, 23, 11, 5, 13, 0, 4, 20, 17, 25, 16, 0, 8, 17, 26]\n",
      "[7, 10, 22, 0, 13, 5, 11, 23, 19, 0, 16, 25, 17, 20, 4, 0, 26, 17, 8]\n"
     ]
    }
   ],
   "source": [
    "vocabulary_size = len(string.ascii_lowercase) + 3 # [a-z] + ' ' + <go> + <eos>\n",
    "first_letter = ord(string.ascii_lowercase[0])\n",
    "GO_ID = 1\n",
    "EOS_ID = 2\n",
    "\n",
    "def char2id(char):\n",
    "  if char in string.ascii_lowercase:\n",
    "    return ord(char) - first_letter + 3\n",
    "  elif char == ' ':\n",
    "    return 0\n",
    "  elif char == '<go>':\n",
    "    return 1\n",
    "  elif char == '<eos>':\n",
    "    return 2\n",
    "  else:\n",
    "    print('Unexpected character: %s' % char)\n",
    "    return \n",
    "  \n",
    "def id2char(dictid):\n",
    "  if dictid > 2:\n",
    "    return chr(dictid + first_letter - 3)\n",
    "  elif dictid == GO_ID:\n",
    "    return '<go>'\n",
    "  elif dictid == EOS_ID:\n",
    "    return '<eos>'\n",
    "  else:\n",
    "    return ' '\n",
    "\n",
    "print(char2id('a'), char2id('z'), char2id(' '), char2id('ï'), char2id('<go>'), char2id('<eos>'))\n",
    "print(id2char(1), id2char(26), id2char(0))\n",
    "\n",
    "def str2id(str):\n",
    "  return [char2id(c) for c in str]\n",
    "\n",
    "print(str2id('the quick brown fox'))\n",
    "print(str2id('eht kciuq nworb xof'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sno ihcrana\n"
     ]
    }
   ],
   "source": [
    "def reverse_word(word):\n",
    "  return word[::-1]\n",
    "\n",
    "def mirror_sequence(seq):\n",
    "  return ' '.join(list(map(lambda word: reverse_word(word), seq.split(' '))))\n",
    "\n",
    "print(mirror_sequence('ons anarchi'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ons anarchists advoc', 'when military govern', 'lleria arches nation', ' abbeys and monaster', 'married urraca princ', 'hel and richard baer', 'y and liturgical lan', 'ay opened for passen', 'tion from the nation', 'migration took place', 'new york other well ', 'he boeing seven six ', 'e listed with a glos', 'eber has probably be', 'o be made to recogni', 'yer who received the', 'ore significant than', 'a fierce critic of t', ' two six eight in si', 'aristotle s uncaused', 'ity can be lost as i', ' and intracellular i', 'tion of the size of ', 'dy to pass him a sti', 'f certain drugs conf', 'at it will take to c', 'e convince the pries', 'ent told him to name', 'ampaign and barred a', 'rver side standard f', 'ious texts such as e', 'o capitalize on the ', 'a duplicate of the o', 'gh ann es d hiver on', 'ine january eight ma', 'ross zero the lead c', 'cal theories classic', 'ast instance the non', ' dimensional analysi', 'most holy mormons be', 't s support or at le', 'u is still disagreed', 'e oscillating system', 'o eight subtypes bas', 'of italy languages t', 's the tower commissi', 'klahoma press one ni', 'erprise linux suse l', 'ws becomes the first', 'et in a nazi concent', 'the fabian society n', 'etchy to relatively ', ' sharman networks sh', 'ised emperor hirohit', 'ting in political in', 'd neo latin most of ', 'th risky riskerdoo r', 'encyclopedic overvie', 'fense the air compon', 'duating from acnm ac', 'treet grid centerlin', 'ations more than any', 'appeal of devotional', 'si have made such de']\n",
      "['sno stsihcrana covda', 'nehw yratilim nrevog', 'airell sehcra noitan', ' syebba dna retsanom', 'deirram acarru cnirp', 'leh dna drahcir reab', 'y dna lacigrutil nal', 'ya denepo rof nessap', 'noit morf eht noitan', 'noitargim koot ecalp', 'wen kroy rehto llew ', 'eh gnieob neves xis ', 'e detsil htiw a solg', 'rebe sah ylbaborp eb', 'o eb edam ot ingocer', 'rey ohw deviecer eht', 'ero tnacifingis naht', 'a ecreif citirc fo t', ' owt xis thgie ni is', 'eltotsira s desuacnu', 'yti nac eb tsol sa i', ' dna ralullecartni i', 'noit fo eht ezis fo ', 'yd ot ssap mih a its', 'f niatrec sgurd fnoc', 'ta ti lliw ekat ot c', 'e ecnivnoc eht seirp', 'tne dlot mih ot eman', 'ngiapma dna derrab a', 'revr edis dradnats f', 'suoi stxet hcus sa e', 'o ezilatipac no eht ', 'a etacilpud fo eht o', 'hg nna se d revih no', 'eni yraunaj thgie am', 'ssor orez eht dael c', 'lac seiroeht cissalc', 'tsa ecnatsni eht non', ' lanoisnemid isylana', 'tsom yloh snomrom eb', 't s troppus ro ta el', 'u si llits deergasid', 'e gnitallicso metsys', 'o thgie sepytbus sab', 'fo ylati segaugnal t', 's eht rewot issimmoc', 'amohalk sserp eno in', 'esirpre xunil esus l', 'sw semoceb eht tsrif', 'te ni a izan tnecnoc', 'eht naibaf yteicos n', 'yhcte ot ylevitaler ', ' namrahs skrowten hs', 'desi rorepme tihorih', 'gnit ni lacitilop ni', 'd oen nital tsom fo ', 'ht yksir oodreksir r', 'cidepolcycne eivrevo', 'esnef eht ria nopmoc', 'gnitaud morf mnca ca', 'teert dirg nilretnec', 'snoita erom naht yna', 'laeppa fo lanoitoved', 'is evah edam hcus ed']\n",
      "[' anarchism originate']\n",
      "['ed as a term of abus']\n",
      "['se first used agains']\n"
     ]
    }
   ],
   "source": [
    "batch_size=64\n",
    "sequence_length = 20\n",
    "\n",
    "class BatchGenerator(object):\n",
    "  def __init__(self, text, batch_size, sequence_length):\n",
    "    self._text = text\n",
    "    self._text_size = len(text)\n",
    "    self._batch_size = batch_size\n",
    "    self._sequence_length = sequence_length\n",
    "    segment = self._text_size // batch_size\n",
    "    self._cursor = [ offset * segment for offset in range(batch_size)]\n",
    "\n",
    "  def next(self):\n",
    "    \"\"\"Generate the next array of batches from the data.\n",
    "    \"\"\"\n",
    "    batches = [None] * self._batch_size\n",
    "    for i in range(self._batch_size):\n",
    "      batches[i] = self._text[self._cursor[i]:self._cursor[i]+self._sequence_length].ljust(self._sequence_length)\n",
    "      self._cursor[i] = (self._cursor[i] + self._sequence_length - 1) % self._text_size\n",
    "    return batches\n",
    "\n",
    "def characters(probabilities):\n",
    "  \"\"\"Turn a 1-hot encoding or a probability distribution over the possible\n",
    "  characters back into its (most likely) character representation.\"\"\"\n",
    "  return [id2char(c) for c in np.argmax(probabilities, 1)]\n",
    "\n",
    "def mirror_batches(batches):\n",
    "   return [mirror_sequence(seq) for seq in batches]\n",
    "\n",
    "train_batches = BatchGenerator(train_text, batch_size, sequence_length)\n",
    "valid_batches = BatchGenerator(valid_text, 1, sequence_length)\n",
    "\n",
    "batches = train_batches.next()\n",
    "print(batches)\n",
    "print(mirror_batches(batches))\n",
    "print(valid_batches.next())\n",
    "print(valid_batches.next())\n",
    "print(valid_batches.next())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of weight: 21\n",
      "shape of weight[0]: (?,)\n",
      "(1,)\n"
     ]
    }
   ],
   "source": [
    "lstm_size = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "  encoder_inputs = list()\n",
    "  decoder_inputs = list()\n",
    "  train_labels = list()  \n",
    "\n",
    "  for _ in range(sequence_length):\n",
    "    encoder_inputs.append(tf.placeholder(tf.int32, shape=(None,)))\n",
    "  for _ in range(sequence_length+1):\n",
    "    decoder_inputs.append(tf.placeholder(tf.int32, shape=(None,)))\n",
    "    train_labels.append(tf.placeholder(tf.int32, shape=(None,)))\n",
    "\n",
    "  weights = [tf.ones_like(label, dtype=tf.float32) for label in train_labels]\n",
    "\n",
    "  print('length of weight:', len(weights))\n",
    "  print('shape of weight[0]:', weights[0].get_shape())\n",
    "\n",
    "  # Use LSTM cell\n",
    "  cell = tf.nn.rnn_cell.BasicLSTMCell(lstm_size)\n",
    "  #outputs, states = tf.nn.seq2seq.basic_rnn_seq2seq(encoder_inputs, decoder_inputs, cell)\n",
    "  with tf.variable_scope(\"train_test\"):\n",
    "    outputs, states = tf.nn.seq2seq.embedding_rnn_seq2seq(encoder_inputs,\n",
    "                                                          decoder_inputs,\n",
    "                                                          cell,\n",
    "                                                          vocabulary_size, # num_encoder_symbols\n",
    "                                                          vocabulary_size, # num_decoder_symbols\n",
    "                                                          vocabulary_size, # embedding_size\n",
    "                                                         )\n",
    "\n",
    "  loss = tf.nn.seq2seq.sequence_loss(outputs, train_labels, weights) \n",
    "  optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)\n",
    "\n",
    "  # Predictions.\n",
    "  train_predictions = tf.pack([tf.nn.softmax(output) for output in outputs])\n",
    "\n",
    "  # Validation eval\n",
    "  valid_encoder_inputs = list()\n",
    "  valid_decoder_inputs = list()\n",
    "  valid_labels = list()  \n",
    "\n",
    "  for _ in range(sequence_length):\n",
    "    valid_encoder_inputs.append(tf.placeholder(tf.int32, shape=(1,)))\n",
    "  for _ in range(sequence_length+1):\n",
    "    valid_decoder_inputs.append(tf.placeholder(tf.int32, shape=(1,)))\n",
    "    valid_labels.append(tf.placeholder(tf.int32, shape=(1,)))\n",
    "  valid_weights = [tf.ones_like(label, dtype=tf.float32) for label in valid_labels]\n",
    "  with tf.variable_scope(\"train_test\", reuse = True):\n",
    "    valid_outputs, valid_states = tf.nn.seq2seq.embedding_rnn_seq2seq(valid_encoder_inputs,\n",
    "                                                                     valid_decoder_inputs,\n",
    "                                                                     cell,\n",
    "                                                                     vocabulary_size, # num_encoder_symbols\n",
    "                                                                     vocabulary_size, # num_decoder_symbols\n",
    "                                                                     vocabulary_size, # embedding_size\n",
    "                                                                     feed_previous=True\n",
    "                                                                     )\n",
    "  valid_predictions = tf.pack([output for output in valid_outputs])\n",
    "  valid_loss = tf.nn.seq2seq.sequence_loss(valid_outputs, valid_labels, valid_weights) \n",
    "\n",
    "  print(valid_encoder_inputs[0].get_shape())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0:\n",
      "Training set:\n",
      "  Perplexity       :  3.37315\n",
      "  Input            :  cate social relation\n",
      "  Correct output   :  etac laicos noitaler\n",
      "  Generated output :  ghh<eos> ddddgrrluuuwuuup\n",
      "Valid set:\n",
      "  Perplexity       :  3.3565\n",
      "  Input            :  st early working cla\n",
      "  Correct output   :  ts ylrae gnikrow alc\n",
      "  Generated output :  kxxxvnpnpwkyyyyewpkkx\n",
      "====================================================================================================\n",
      "Step 1000:\n",
      "Training set:\n",
      "  Perplexity       :  2.66839\n",
      "  Input            :  ictory but in one ni\n",
      "  Correct output   :  yrotci tub ni eno in\n",
      "  Generated output :         e   ei eei e <eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.88864\n",
      "  Input            :  ass radicals includi\n",
      "  Correct output   :  ssa slacidar idulcni\n",
      "  Generated output :   ee e ee ee ee et <eos><eos><eos>\n",
      "====================================================================================================\n",
      "Step 2000:\n",
      "Training set:\n",
      "  Perplexity       :  2.39611\n",
      "  Input            :   institutions that d\n",
      "  Correct output   :   snoitutitsni taht d\n",
      "  Generated output :   nii          ee   <eos><eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.01884\n",
      "  Input            :  ing the diggers of t\n",
      "  Correct output   :  gni eht sreggid fo t\n",
      "  Generated output :  e et et eto etit eti<eos>\n",
      "====================================================================================================\n",
      "Step 3000:\n",
      "Training set:\n",
      "  Perplexity       :  2.11983\n",
      "  Input            :  cle control unusual \n",
      "  Correct output   :  elc lortnoc lausunu \n",
      "  Generated output :  sl  lalaaa  sall  o <eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.0211\n",
      "  Input            :  the english revoluti\n",
      "  Correct output   :  eht hsilgne itulover\n",
      "  Generated output :  et eht satas sis eht<eos>\n",
      "====================================================================================================\n",
      "Step 4000:\n",
      "Training set:\n",
      "  Perplexity       :  2.04686\n",
      "  Input            :  me talent in a certa\n",
      "  Correct output   :  em tnelat ni a atrec\n",
      "  Generated output :  et tnih   di eretae<eos><eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.3375\n",
      "  Input            :  ion and the sans cul\n",
      "  Correct output   :  noi dna eht snas luc\n",
      "  Generated output :  no ni sa saita da eht\n",
      "====================================================================================================\n",
      "Step 5000:\n",
      "Training set:\n",
      "  Perplexity       :  1.76379\n",
      "  Input            :  ter prospecting duri\n",
      "  Correct output   :  ret gnitcepsorp irud\n",
      "  Generated output :  ee  siisaarr r  draa<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.7638\n",
      "  Input            :  lottes of the french\n",
      "  Correct output   :  settol fo eht hcnerf\n",
      "  Generated output :  tno tnoc eht elorec<eos><eos>\n",
      "====================================================================================================\n",
      "Step 6000:\n",
      "Training set:\n",
      "  Perplexity       :  1.71327\n",
      "  Input            :  festival spirit of a\n",
      "  Correct output   :  lavitsef tirips fo a\n",
      "  Generated output :  setitsis sisi   oo a<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  4.28222\n",
      "  Input            :  h revolution whilst \n",
      "  Correct output   :  h noitulover tslihw \n",
      "  Generated output :   oretilolp dnicelo <eos><eos>\n",
      "====================================================================================================\n",
      "Step 7000:\n",
      "Training set:\n",
      "  Perplexity       :  1.6358\n",
      "  Input            :  tion overstepping th\n",
      "  Correct output   :  noit gnippetsrevo ht\n",
      "  Generated output :  no   eeeeeecoee o it<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.82507\n",
      "  Input            :   the term is still u\n",
      "  Correct output   :   eht mret si llits u\n",
      "  Generated output :   eht es dnals suo t<eos><eos>\n",
      "====================================================================================================\n",
      "Step 8000:\n",
      "Training set:\n",
      "  Perplexity       :  1.45502\n",
      "  Input            :  ca president buchana\n",
      "  Correct output   :  ac tnediserp anahcub\n",
      "  Generated output :  d  tiiiilara anacci<eos><eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.08229\n",
      "  Input            :  used in a pejorative\n",
      "  Correct output   :  desu ni a evitarojep\n",
      "  Generated output :  desi na eht lopnaerp<eos>\n",
      "====================================================================================================\n",
      "Step 9000:\n",
      "Training set:\n",
      "  Perplexity       :  1.31401\n",
      "  Input            :   general general uly\n",
      "  Correct output   :   lareneg lareneg ylu\n",
      "  Generated output :   nrnenen nnnen   lllb\n",
      "Valid set:\n",
      "  Perplexity       :  4.77581\n",
      "  Input            :  e way to describe an\n",
      "  Correct output   :  e yaw ot ebircsed na\n",
      "  Generated output :  e oi saeh saleht ort<eos>\n",
      "====================================================================================================\n",
      "Step 10000:\n",
      "Training set:\n",
      "  Perplexity       :  1.43934\n",
      "  Input            :  e nine five two spec\n",
      "  Correct output   :  e enin evif owt ceps\n",
      "  Generated output :  e eni  enif sstssess<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  4.17777\n",
      "  Input            :  ny act that used vio\n",
      "  Correct output   :  yn tca taht desu oiv\n",
      "  Generated output :  ta ni atac evi sim p<eos>\n",
      "====================================================================================================\n",
      "Step 11000:\n",
      "Training set:\n",
      "  Perplexity       :  1.19806\n",
      "  Input            :  plies the ascent fro\n",
      "  Correct output   :  seilp eht tnecsa orf\n",
      "  Generated output :  eeitt ttt tsecs  frf<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.45349\n",
      "  Input            :  olent means to destr\n",
      "  Correct output   :  tnelo snaem ot rtsed\n",
      "  Generated output :  notse ale natse fos<eos><eos>\n",
      "====================================================================================================\n",
      "Step 12000:\n",
      "Training set:\n",
      "  Perplexity       :  1.08396\n",
      "  Input            :  damascus a final cat\n",
      "  Correct output   :  sucsamad a lanif tac\n",
      "  Generated output :  saaaadad a latit tac<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.94134\n",
      "  Input            :  roy the organization\n",
      "  Correct output   :  yor eht noitazinagro\n",
      "  Generated output :  eht ori noilagnirtah<eos>\n",
      "====================================================================================================\n",
      "Step 13000:\n",
      "Training set:\n",
      "  Perplexity       :  1.04489\n",
      "  Input            :  to our successors to\n",
      "  Correct output   :  ot ruo srosseccus ot\n",
      "  Generated output :  os soosssessecuus ot<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  4.73758\n",
      "  Input            :  n of society it has \n",
      "  Correct output   :  n fo yteicos ti sah \n",
      "  Generated output :  n ot seicif co saht <eos>\n",
      "====================================================================================================\n",
      "Step 14000:\n",
      "Training set:\n",
      "  Perplexity       :  0.936414\n",
      "  Input            :  nduism also promote \n",
      "  Correct output   :  msiudn osla etomorp \n",
      "  Generated output :  dsiddn osaa ecomomp <eos>\n",
      "Valid set:\n",
      "  Perplexity       :  4.14475\n",
      "  Input            :   also been taken up \n",
      "  Correct output   :   osla neeb nekat pu \n",
      "  Generated output :   suo enale na evap s<eos>\n",
      "====================================================================================================\n",
      "Step 15000:\n",
      "Training set:\n",
      "  Perplexity       :  0.946605\n",
      "  Input            :  n activities testimo\n",
      "  Correct output   :  n seitivitca omitset\n",
      "  Generated output :  nieeititatca ttttset<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.17102\n",
      "  Input            :   as a positive label\n",
      "  Correct output   :   sa a evitisop lebal\n",
      "  Generated output :   sa eht yliloisaep s<eos>\n",
      "====================================================================================================\n",
      "Step 16000:\n",
      "Training set:\n",
      "  Perplexity       :  0.850466\n",
      "  Input            :  rnia allan dwan has \n",
      "  Correct output   :  ainr nalla nawd sah \n",
      "  Generated output :  ennranalaa dadd sah <eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.83638\n",
      "  Input            :  l by self defined an\n",
      "  Correct output   :  l yb fles denifed na\n",
      "  Generated output :  l yf selil dneedn f<eos><eos>\n",
      "====================================================================================================\n",
      "Step 17000:\n",
      "Training set:\n",
      "  Perplexity       :  0.846162\n",
      "  Input            :  ncessions including \n",
      "  Correct output   :  snoissecn gnidulcni \n",
      "  Generated output :  ssosssec  dniduooni <eos>\n",
      "Valid set:\n",
      "  Perplexity       :  1.18641\n",
      "  Input            :  narchists the word a\n",
      "  Correct output   :  stsihcran eht drow a\n",
      "  Generated output :  sstihcnar eht drom o<eos>\n",
      "====================================================================================================\n",
      "Step 18000:\n",
      "Training set:\n",
      "  Perplexity       :  0.749664\n",
      "  Input            :  rs taggart is an exp\n",
      "  Correct output   :  sr traggat si na pxe\n",
      "  Generated output :  r  taagtat ti aa eee<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.43113\n",
      "  Input            :  anarchism is derived\n",
      "  Correct output   :  msihcrana si devired\n",
      "  Generated output :  hcisramsn ia devider<eos>\n",
      "====================================================================================================\n",
      "Step 19000:\n",
      "Training set:\n",
      "  Perplexity       :  0.7119\n",
      "  Input            :   use of the word tod\n",
      "  Correct output   :   esu fo eht drow dot\n",
      "  Generated output :   esu fo eht dooo oot<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.18389\n",
      "  Input            :  d from the greek wit\n",
      "  Correct output   :  d morf eht keerg tiw\n",
      "  Generated output :  d morf eht ergit emo<eos>\n",
      "====================================================================================================\n",
      "Step 20000:\n",
      "Training set:\n",
      "  Perplexity       :  0.728638\n",
      "  Input            :  s them to support le\n",
      "  Correct output   :  s meht ot troppus el\n",
      "  Generated output :  s meht ot tcoppus eb<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  4.28979\n",
      "  Input            :  thout archons ruler \n",
      "  Correct output   :  tuoht snohcra relur \n",
      "  Generated output :  tohtu snarocref ura <eos>\n",
      "====================================================================================================\n",
      "Step 21000:\n",
      "Training set:\n",
      "  Perplexity       :  0.706293\n",
      "  Input            :   seem to be disappea\n",
      "  Correct output   :   mees ot eb aeppasid\n",
      "  Generated output :   sess ot ab appaasid<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.76924\n",
      "  Input            :   chief king anarchis\n",
      "  Correct output   :   feihc gnik sihcrana\n",
      "  Generated output :   feihc niht dracnido<eos>\n",
      "====================================================================================================\n",
      "Step 22000:\n",
      "Training set:\n",
      "  Perplexity       :  0.616789\n",
      "  Input            :  ed centers and labor\n",
      "  Correct output   :  de sretnec dna robal\n",
      "  Generated output :  de seetnec nna rolal<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.93312\n",
      "  Input            :  sm as a political ph\n",
      "  Correct output   :  ms sa a lacitilop hp\n",
      "  Generated output :  sm sa la citalopi hs<eos>\n",
      "====================================================================================================\n",
      "Step 23000:\n",
      "Training set:\n",
      "  Perplexity       :  0.703025\n",
      "  Input            :  ed kennewick man is \n",
      "  Correct output   :  de kciwennek nam si \n",
      "  Generated output :  decccinnnnem ma  si <eos>\n",
      "Valid set:\n",
      "  Perplexity       :  6.04495\n",
      "  Input            :  hilosophy is the bel\n",
      "  Correct output   :  yhposolih si eht leb\n",
      "  Generated output :  doicifods a esme avo \n",
      "====================================================================================================\n",
      "Step 24000:\n",
      "Training set:\n",
      "  Perplexity       :  0.663782\n",
      "  Input            :  article pseudoarchae\n",
      "  Correct output   :  elcitra eahcraoduesp\n",
      "  Generated output :  llciraa ecccioppeesp<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  5.75395\n",
      "  Input            :  lief that rulers are\n",
      "  Correct output   :  feil taht srelur era\n",
      "  Generated output :  efit sal nracer erp e\n",
      "====================================================================================================\n",
      "Step 25000:\n",
      "Training set:\n",
      "  Perplexity       :  0.636882\n",
      "  Input            :  ient for either or b\n",
      "  Correct output   :  tnei rof rehtie ro b\n",
      "  Generated output :  tnii rof eehtie robb<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.57646\n",
      "  Input            :  e unnecessary and sh\n",
      "  Correct output   :  e yrassecennu dna hs\n",
      "  Generated output :  e seuradnicne sdna r<eos>\n",
      "====================================================================================================\n",
      "Step 26000:\n",
      "Training set:\n",
      "  Perplexity       :  0.599096\n",
      "  Input            :  he use of hermetic t\n",
      "  Correct output   :  eh esu fo citemreh t\n",
      "  Generated output :  eh esu fo citemreh t<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  4.02423\n",
      "  Input            :  hould be abolished a\n",
      "  Correct output   :  dluoh eb dehsiloba a\n",
      "  Generated output :  deloh sub daiforeb a<eos>\n",
      "====================================================================================================\n",
      "Step 27000:\n",
      "Training set:\n",
      "  Perplexity       :  0.613242\n",
      "  Input            :  een fulfilled by ivf\n",
      "  Correct output   :  nee dellifluf yb fvi\n",
      "  Generated output :  eee slllffluf bb fii<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.83446\n",
      "  Input            :  although there are d\n",
      "  Correct output   :  hguohtla ereht era d\n",
      "  Generated output :  hguonamec ereht arud<eos>\n",
      "====================================================================================================\n",
      "Step 28000:\n",
      "Training set:\n",
      "  Perplexity       :  0.544118\n",
      "  Input            :  space australia is e\n",
      "  Correct output   :  ecaps ailartsua si e\n",
      "  Generated output :  ecapa ailartsua si e<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  4.65177\n",
      "  Input            :  differing interpreta\n",
      "  Correct output   :  gnireffid aterpretni\n",
      "  Generated output :  gnifreib rehtaitrebs<eos>\n",
      "====================================================================================================\n",
      "Step 29000:\n",
      "Training set:\n",
      "  Perplexity       :  0.527932\n",
      "  Input            :   catholic church whi\n",
      "  Correct output   :   cilohtac hcruhc ihw\n",
      "  Generated output :   cilohcuc hcruhw hhw<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  1.1151\n",
      "  Input            :  ations of what this \n",
      "  Correct output   :  snoita fo tahw siht \n",
      "  Generated output :  snoita fo atht fihs <eos>\n",
      "====================================================================================================\n",
      "Step 30000:\n",
      "Training set:\n",
      "  Perplexity       :  0.493637\n",
      "  Input            :  y contrast federal l\n",
      "  Correct output   :  y tsartnoc laredef l\n",
      "  Generated output :  y tsartnoc seeedef m<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.71655\n",
      "  Input            :   means anarchism als\n",
      "  Correct output   :   snaem msihcrana sla\n",
      "  Generated output :   snam ehtilarhsm sar<eos>\n",
      "====================================================================================================\n",
      "Step 31000:\n",
      "Training set:\n",
      "  Perplexity       :  0.495817\n",
      "  Input            :  e is more popular th\n",
      "  Correct output   :  e si erom ralupop ht\n",
      "  Generated output :  e si erom rauupop ht<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  4.81076\n",
      "  Input            :  so refers to related\n",
      "  Correct output   :  os srefer ot detaler\n",
      "  Generated output :  so resfo era tedletn<eos>\n",
      "====================================================================================================\n",
      "Step 32000:\n",
      "Training set:\n",
      "  Perplexity       :  0.459188\n",
      "  Input            :  ion the ascii charac\n",
      "  Correct output   :  noi eht iicsa carahc\n",
      "  Generated output :  noi eht iccaacraracc<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.91338\n",
      "  Input            :  d social movements t\n",
      "  Correct output   :  d laicos stnemevom t\n",
      "  Generated output :  d laitom sesnocemi g<eos>\n",
      "====================================================================================================\n",
      "Step 33000:\n",
      "Training set:\n",
      "  Perplexity       :  0.457276\n",
      "  Input            :   and africa as europ\n",
      "  Correct output   :   dna acirfa sa porue\n",
      "  Generated output :   dna ccirfa sa uorue<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.55039\n",
      "  Input            :  that advocate the el\n",
      "  Correct output   :  taht etacovda eht le\n",
      "  Generated output :  taht etapocde a noc <eos>\n",
      "====================================================================================================\n",
      "Step 34000:\n",
      "Training set:\n",
      "  Perplexity       :  0.52947\n",
      "  Input            :  ges spoken in much o\n",
      "  Correct output   :  seg nekops ni hcum o\n",
      "  Generated output :  seg nekoss fi ccum o<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  1.9627\n",
      "  Input            :  limination of author\n",
      "  Correct output   :  noitanimil fo rohtua\n",
      "  Generated output :  noitainmil fo rotahu<eos>\n",
      "====================================================================================================\n",
      "Step 35000:\n",
      "Training set:\n",
      "  Perplexity       :  0.466982\n",
      "  Input            :  of talented individu\n",
      "  Correct output   :  fo detnelat udividni\n",
      "  Generated output :  no detnalu  ydigidni<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.71935\n",
      "  Input            :  ritarian institution\n",
      "  Correct output   :  nairatir noitutitsni\n",
      "  Generated output :  nairatir suoitnitnit<eos>\n",
      "====================================================================================================\n",
      "Step 36000:\n",
      "Training set:\n",
      "  Perplexity       :  0.460935\n",
      "  Input            :   in the cave he foun\n",
      "  Correct output   :   ni eht evac eh nuof\n",
      "  Generated output :   ni eht evac en noof<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  0.261705\n",
      "  Input            :  ns particularly the \n",
      "  Correct output   :  sn ylralucitrap eht \n",
      "  Generated output :  sn ylralucitrap eht <eos>\n",
      "====================================================================================================\n",
      "Step 37000:\n",
      "Training set:\n",
      "  Perplexity       :  0.420296\n",
      "  Input            :  ed by a third degree\n",
      "  Correct output   :  de yb a driht eerged\n",
      "  Generated output :  de yb d dreht eerged<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.16959\n",
      "  Input            :   state the word anar\n",
      "  Correct output   :   etats eht drow rana\n",
      "  Generated output :   etats eht rodna raw<eos>\n",
      "====================================================================================================\n",
      "Step 38000:\n",
      "Training set:\n",
      "  Perplexity       :  0.384416\n",
      "  Input            :  guists traditionally\n",
      "  Correct output   :  stsiug yllanoitidart\n",
      "  Generated output :  stsibb ylaanoitidrrt<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.41903\n",
      "  Input            :  rchy as most anarchi\n",
      "  Correct output   :  yhcr sa tsom ihcrana\n",
      "  Generated output :  yhcr sa stah lohcrih<eos>\n",
      "====================================================================================================\n",
      "Step 39000:\n",
      "Training set:\n",
      "  Perplexity       :  0.421272\n",
      "  Input            :  licism culture andor\n",
      "  Correct output   :  msicil erutluc rodna\n",
      "  Generated output :  miicil ecutcuc dodna<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.71032\n",
      "  Input            :  ists use it does not\n",
      "  Correct output   :  stsi esu ti seod ton\n",
      "  Generated output :  stis esu ts edot iso<eos>\n",
      "====================================================================================================\n",
      "Step 40000:\n",
      "Training set:\n",
      "  Perplexity       :  0.385288\n",
      "  Input            :  on as vice president\n",
      "  Correct output   :  no sa eciv tnediserp\n",
      "  Generated output :  no sa ecig tnediserp<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.51074\n",
      "  Input            :  t imply chaos nihili\n",
      "  Correct output   :  t ylpmi soahc ilihin\n",
      "  Generated output :  t ylpmi saoci hinimo<eos>\n",
      "====================================================================================================\n",
      "Step 41000:\n",
      "Training set:\n",
      "  Perplexity       :  0.377384\n",
      "  Input            :  for a target audienc\n",
      "  Correct output   :  rof a tegrat cneidua\n",
      "  Generated output :  rof a teraac eeeidua<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.21531\n",
      "  Input            :  ism or anomie but ra\n",
      "  Correct output   :  msi ro eimona tub ar\n",
      "  Generated output :  smi ro ehtilo aba re<eos>\n",
      "====================================================================================================\n",
      "Step 42000:\n",
      "Training set:\n",
      "  Perplexity       :  0.349671\n",
      "  Input            :  a dinner party for t\n",
      "  Correct output   :  a rennid ytrap rof t\n",
      "  Generated output :  a neniid ytrap nof t<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  3.86851\n",
      "  Input            :  ather a harmonious a\n",
      "  Correct output   :  rehta a suoinomrah a\n",
      "  Generated output :  rehta sa oirahtnou m<eos>\n",
      "====================================================================================================\n",
      "Step 43000:\n",
      "Training set:\n",
      "  Perplexity       :  0.446204\n",
      "  Input            :  als to develop lungs\n",
      "  Correct output   :  sla ot poleved sgnul\n",
      "  Generated output :  sla ot eolebed dgnul<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  4.19471\n",
      "  Input            :  anti authoritarian s\n",
      "  Correct output   :  itna nairatirohtua s\n",
      "  Generated output :  nita inarotairacs da<eos>\n",
      "====================================================================================================\n",
      "Step 44000:\n",
      "Training set:\n",
      "  Perplexity       :  0.365398\n",
      "  Input            :  r consumption within\n",
      "  Correct output   :  r noitpmusnoc nihtiw\n",
      "  Generated output :  r noitpsusnoc tihtiw<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  0.874939\n",
      "  Input            :  society in place of \n",
      "  Correct output   :  yteicos ni ecalp fo \n",
      "  Generated output :  ytiecos ni ecabl fo <eos>\n",
      "====================================================================================================\n",
      "Step 45000:\n",
      "Training set:\n",
      "  Perplexity       :  0.51223\n",
      "  Input            :  rops and transport f\n",
      "  Correct output   :  spor dna tropsnart f\n",
      "  Generated output :  spor dna trossnart f<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  0.779399\n",
      "  Input            :   what are regarded a\n",
      "  Correct output   :   tahw era dedrager a\n",
      "  Generated output :   tahw eda redraged o<eos>\n",
      "====================================================================================================\n",
      "Step 46000:\n",
      "Training set:\n",
      "  Perplexity       :  0.334563\n",
      "  Input            :  ather s well equippe\n",
      "  Correct output   :  rehta s llew eppiuqe\n",
      "  Generated output :  rehta s leew epiiupe<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  1.70143\n",
      "  Input            :  as authoritarian pol\n",
      "  Correct output   :  sa nairatirohtua lop\n",
      "  Generated output :  sa nairatioracnu aso<eos>\n",
      "====================================================================================================\n",
      "Step 47000:\n",
      "Training set:\n",
      "  Perplexity       :  0.334985\n",
      "  Input            :  alled phycology or a\n",
      "  Correct output   :  della ygolocyhp ro a\n",
      "  Generated output :  della ygolopphp oo a<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.8294\n",
      "  Input            :  litical structures a\n",
      "  Correct output   :  lacitil serutcurts a\n",
      "  Generated output :  lacitil scutresurt a<eos>\n",
      "====================================================================================================\n",
      "Step 48000:\n",
      "Training set:\n",
      "  Perplexity       :  0.346598\n",
      "  Input            :  te easily within the\n",
      "  Correct output   :  et ylisae nihtiw eht\n",
      "  Generated output :  et ylasae nihtiw eht<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.22328\n",
      "  Input            :  and coercive economi\n",
      "  Correct output   :  dna evicreoc imonoce\n",
      "  Generated output :  dna eirpicel owonoce<eos>\n",
      "====================================================================================================\n",
      "Step 49000:\n",
      "Training set:\n",
      "  Perplexity       :  0.346538\n",
      "  Input            :  are an important ind\n",
      "  Correct output   :  era na tnatropmi dni\n",
      "  Generated output :  era na tnarropmi dni<eos>\n",
      "Valid set:\n",
      "  Perplexity       :  2.60853\n",
      "  Input            :  ic instituti        \n",
      "  Correct output   :  ci itutitsni        \n",
      "  Generated output :  ci tiutistin s a  <eos><eos><eos>\n",
      "====================================================================================================\n",
      "Step 50000:\n",
      "Training set:\n",
      "  Perplexity       :  0.379369\n",
      "  Input            :  ing in the criminal \n",
      "  Correct output   :  gni ni eht lanimirc \n",
      "  Generated output :  gni ni eht laiimirc <eos>\n",
      "Valid set:\n",
      "  Perplexity       :  0.152589\n",
      "  Input            :  ism originated as a \n",
      "  Correct output   :  msi detanigiro sa a \n",
      "  Generated output :  msi detanigiro sa a <eos>\n",
      "====================================================================================================\n"
     ]
    }
   ],
   "source": [
    "def sample_prediction(predictions):\n",
    "  return ''.join([characters(onehot)[0] for onehot in predictions])\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "  sess.run(tf.initialize_all_variables())\n",
    "\n",
    "  for step in range(50001):\n",
    "    feed_dict = dict()\n",
    "    \n",
    "    input_batches = train_batches.next()\n",
    "    output_batches = mirror_batches(input_batches)\n",
    "       \n",
    "    feed_dict[decoder_inputs[0]] = [GO_ID] * batch_size\n",
    "    for i in range(sequence_length):\n",
    "      feed_dict[encoder_inputs[i]] = [char2id(seq[i]) for seq in input_batches]\n",
    "      feed_dict[decoder_inputs[i+1]] = [char2id(seq[i]) for seq in output_batches]\n",
    "      feed_dict[train_labels[i]] = [char2id(seq[i]) for seq in output_batches]\n",
    "    feed_dict[train_labels[sequence_length]] = [EOS_ID] * batch_size\n",
    "    \n",
    "    _, l, predictions = sess.run([optimizer, loss, train_predictions], feed_dict=feed_dict)\n",
    "\n",
    "    if step % 1000 == 0:\n",
    "      print('Step %d:' % step)\n",
    "      print('Training set:')\n",
    "      print('  Perplexity       : ', l)\n",
    "      print('  Input            : ', input_batches[0])\n",
    "      print('  Correct output   : ', output_batches[0])\n",
    "      print('  Generated output : ', first_prediction(predictions))\n",
    "          \n",
    "      valid_feed_dict = dict()\n",
    "      valid_input_batches = valid_batches.next()\n",
    "      valid_output_batches = mirror_batches(valid_input_batches)\n",
    "\n",
    "      valid_feed_dict[valid_decoder_inputs[0]] = [GO_ID]\n",
    "      for i in range(sequence_length):\n",
    "        valid_feed_dict[valid_encoder_inputs[i]] = [char2id(valid_input_batches[0][i])]\n",
    "        valid_feed_dict[valid_labels[i]] = [char2id(valid_output_batches[0][i])]\n",
    "      valid_feed_dict[valid_labels[sequence_length]] = [EOS_ID]\n",
    "\n",
    "      valid_l, valid_p = sess.run([valid_loss, valid_predictions], feed_dict=valid_feed_dict)\n",
    "     \n",
    "        \n",
    "      print('Valid set:')\n",
    "      print('  Perplexity       : ', valid_l)\n",
    "      print('  Input            : ', valid_input_batches[0])\n",
    "      print('  Correct output   : ', valid_output_batches[0])\n",
    "      print('  Generated output : ', sample_prediction(valid_p))\n",
    "      print(\"=\"*100)\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "6_lstm.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}