lab-12-0-rnn_basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# http://www.wildml.com/2016/08/rnns-in-tensorflow-a-practical-guide-and-undocumented-features/\n",
    "# http://learningtensorflow.com/index.html\n",
    "# http://suriyadeepan.github.io/2016-12-31-practical-seq2seq/\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.contrib import rnn\n",
    "import pprint\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# One hot encoding for each char in 'hello'\n",
    "h = [1, 0, 0, 0]\n",
    "e = [0, 1, 0, 0]\n",
    "l = [0, 0, 1, 0]\n",
    "o = [0, 0, 0, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image](https://cloud.githubusercontent.com/assets/901975/23348727/cc981856-fce7-11e6-83ea-4b187473466b.png)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 2\n",
      "array([[[ 1.,  0.,  0.,  0.]]], dtype=float32)\n",
      "array([[[ 0.67200625, -0.14664482]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('one_cell') as scope:\n",
    "    # One cell RNN input_dim (4) -> output_dim (2)\n",
    "    hidden_size = 2\n",
    "    cell = tf.contrib.rnn.BasicRNNCell(num_units=hidden_size)\n",
    "    print(cell.output_size, cell.state_size)\n",
    "\n",
    "    x_data = np.array([[h]], dtype=np.float32) # x_data = [[[1,0,0,0]]]\n",
    "    pp.pprint(x_data)\n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)\n",
    "\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image](https://cloud.githubusercontent.com/assets/901975/23383634/649efd0a-fd82-11e6-925d-8041242743b0.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 5, 4)\n",
      "array([[[ 1.,  0.,  0.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  0.,  1.]]], dtype=float32)\n",
      "array([[[ 0.68234169, -0.00615374],\n",
      "        [-0.64476734, -0.42795229],\n",
      "        [ 0.73443085,  0.44646609],\n",
      "        [ 0.60938346,  0.11492527],\n",
      "        [-0.54910451,  0.20318383]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('two_sequances') as scope:\n",
    "    # One cell RNN input_dim (4) -> output_dim (2). sequence: 5\n",
    "    hidden_size = 2\n",
    "    cell = tf.contrib.rnn.BasicRNNCell(num_units=hidden_size)\n",
    "    x_data = np.array([[h, e, l, l, o]], dtype=np.float32)\n",
    "    print(x_data.shape)\n",
    "    pp.pprint(x_data)\n",
    "    outputs, states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image](https://cloud.githubusercontent.com/assets/901975/23383681/9943a9fc-fd82-11e6-8121-bd187994e249.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[ 1.,  0.,  0.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  0.,  1.]],\n",
      "\n",
      "       [[ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  1.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.]],\n",
      "\n",
      "       [[ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.]]], dtype=float32)\n",
      "array([[[-0.14501694, -0.07943568],\n",
      "        [ 0.0712046 , -0.11884281],\n",
      "        [ 0.16517045, -0.05914218],\n",
      "        [ 0.24221966,  0.00742693],\n",
      "        [ 0.30892155,  0.05031026]],\n",
      "\n",
      "       [[ 0.17852989, -0.01393668],\n",
      "        [ 0.27475667,  0.02382141],\n",
      "        [ 0.33198762,  0.09683739],\n",
      "        [ 0.38116094,  0.16157949],\n",
      "        [ 0.42101568,  0.21835908]],\n",
      "\n",
      "       [[ 0.09538922,  0.03974426],\n",
      "        [ 0.1808964 ,  0.08554014],\n",
      "        [ 0.30597433,  0.07592102],\n",
      "        [ 0.39893293,  0.08319239],\n",
      "        [ 0.45521817,  0.15857427]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('3_batches') as scope:\n",
    "    # One cell RNN input_dim (4) -> output_dim (2). sequence: 5, batch 3\n",
    "    # 3 batches 'hello', 'eolll', 'lleel'\n",
    "    x_data = np.array([[h, e, l, l, o],\n",
    "                       [e, o, l, l, l],\n",
    "                       [l, l, e, e, l]], dtype=np.float32)\n",
    "    pp.pprint(x_data)\n",
    "    \n",
    "    hidden_size = 2\n",
    "    cell = rnn.BasicLSTMCell(num_units=hidden_size, state_is_tuple=True)\n",
    "    outputs, _states = tf.nn.dynamic_rnn(\n",
    "        cell, x_data, dtype=tf.float32)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[ 1.,  0.,  0.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  0.,  1.]],\n",
      "\n",
      "       [[ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  1.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.]],\n",
      "\n",
      "       [[ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.]]], dtype=float32)\n",
      "array([[[-0.17904168, -0.08053244],\n",
      "        [-0.01294809,  0.01660814],\n",
      "        [-0.05754048, -0.1368292 ],\n",
      "        [-0.08655578, -0.20553185],\n",
      "        [ 0.07297077, -0.21743253]],\n",
      "\n",
      "       [[ 0.10272847,  0.06519825],\n",
      "        [ 0.20188759, -0.05027055],\n",
      "        [ 0.09514933, -0.16452041],\n",
      "        [ 0.        ,  0.        ],\n",
      "        [ 0.        ,  0.        ]],\n",
      "\n",
      "       [[-0.04893036, -0.14655617],\n",
      "        [-0.07947272, -0.20996611],\n",
      "        [ 0.06466491, -0.02576563],\n",
      "        [ 0.15087658,  0.05166111],\n",
      "        [ 0.        ,  0.        ]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('3_batches_dynamic_length') as scope:\n",
    "    # One cell RNN input_dim (4) -> output_dim (5). sequence: 5, batch 3\n",
    "    # 3 batches 'hello', 'eolll', 'lleel'\n",
    "    x_data = np.array([[h, e, l, l, o],\n",
    "                       [e, o, l, l, l],\n",
    "                       [l, l, e, e, l]], dtype=np.float32)\n",
    "    pp.pprint(x_data)\n",
    "    \n",
    "    hidden_size = 2\n",
    "    cell = rnn.BasicLSTMCell(num_units=hidden_size, state_is_tuple=True)\n",
    "    outputs, _states = tf.nn.dynamic_rnn(\n",
    "        cell, x_data, sequence_length=[5,3,4], dtype=tf.float32)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[ 1.,  0.,  0.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  0.,  1.]],\n",
      "\n",
      "       [[ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  1.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.]],\n",
      "\n",
      "       [[ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.]]], dtype=float32)\n",
      "array([[[-0.01767019,  0.11875235],\n",
      "        [ 0.08330756,  0.17304452],\n",
      "        [ 0.07657852,  0.04430496],\n",
      "        [ 0.09675486, -0.05384296],\n",
      "        [ 0.13961229, -0.08712364]],\n",
      "\n",
      "       [[ 0.11577919,  0.09888638],\n",
      "        [ 0.12283466, -0.00558388],\n",
      "        [ 0.11969706, -0.09668337],\n",
      "        [ 0.14534688, -0.16223255],\n",
      "        [ 0.17191799, -0.21179219]],\n",
      "\n",
      "       [[ 0.03162946, -0.07984579],\n",
      "        [ 0.06664791, -0.1425948 ],\n",
      "        [ 0.1997031 , -0.02399314],\n",
      "        [ 0.28559196,  0.06579842],\n",
      "        [ 0.22943661, -0.06061804]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('initial_state') as scope:\n",
    "    batch_size = 3\n",
    "    x_data = np.array([[h, e, l, l, o],\n",
    "                      [e, o, l, l, l],\n",
    "                      [l, l, e, e, l]], dtype=np.float32)\n",
    "    pp.pprint(x_data)\n",
    "    \n",
    "    # One cell RNN input_dim (4) -> output_dim (5). sequence: 5, batch: 3\n",
    "    hidden_size=2\n",
    "    cell = rnn.BasicLSTMCell(num_units=hidden_size, state_is_tuple=True)\n",
    "    initial_state = cell.zero_state(batch_size, tf.float32)\n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data,\n",
    "                                         initial_state=initial_state, dtype=tf.float32)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[  0.,   1.,   2.],\n",
      "        [  3.,   4.,   5.],\n",
      "        [  6.,   7.,   8.],\n",
      "        [  9.,  10.,  11.],\n",
      "        [ 12.,  13.,  14.]],\n",
      "\n",
      "       [[ 15.,  16.,  17.],\n",
      "        [ 18.,  19.,  20.],\n",
      "        [ 21.,  22.,  23.],\n",
      "        [ 24.,  25.,  26.],\n",
      "        [ 27.,  28.,  29.]],\n",
      "\n",
      "       [[ 30.,  31.,  32.],\n",
      "        [ 33.,  34.,  35.],\n",
      "        [ 36.,  37.,  38.],\n",
      "        [ 39.,  40.,  41.],\n",
      "        [ 42.,  43.,  44.]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# Create input data\n",
    "batch_size=3\n",
    "sequence_length=5\n",
    "input_dim=3\n",
    "\n",
    "x_data = np.arange(45, dtype=np.float32).reshape(batch_size, sequence_length, input_dim)\n",
    "pp.pprint(x_data)  # batch, sequence_length, input_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[  5.73196858e-02,   5.41342497e-02,   8.11929479e-02,\n",
      "           2.89054047e-02,   1.34020135e-01],\n",
      "        [  1.06766215e-02,   8.97800848e-02,   8.12651366e-02,\n",
      "           3.42822880e-01,   1.52226299e-01],\n",
      "        [  7.44317716e-04,   1.06552474e-01,   3.69244218e-02,\n",
      "           4.27359670e-01,   9.71041247e-02],\n",
      "        [  4.12664740e-05,   1.24697894e-01,   1.62402373e-02,\n",
      "           4.06710804e-01,   4.76750508e-02],\n",
      "        [  2.34956156e-06,   1.43548846e-01,   7.30458880e-03,\n",
      "           3.81038815e-01,   2.30577681e-02]],\n",
      "\n",
      "       [[  1.39306891e-07,   2.81301272e-05,   3.51365912e-03,\n",
      "           3.63229781e-01,   1.03324354e-02],\n",
      "        [  8.43908499e-09,   3.37266210e-05,   1.34868850e-03,\n",
      "           3.39024425e-01,   4.67472617e-03],\n",
      "        [  4.95971986e-10,   3.69007030e-05,   5.85811620e-04,\n",
      "           3.14222544e-01,   2.09378102e-03],\n",
      "        [  2.90123793e-11,   3.91298818e-05,   2.53556267e-04,\n",
      "           2.89835870e-01,   9.21602652e-04],\n",
      "        [  1.68882313e-12,   4.08560663e-05,   1.09498724e-04,\n",
      "           2.66193688e-01,   4.00958583e-04]],\n",
      "\n",
      "       [[  9.84404005e-14,   1.78621107e-09,   5.17079461e-05,\n",
      "           2.42278278e-01,   1.75901252e-04],\n",
      "        [  5.65664653e-15,   2.07072182e-09,   2.03506170e-05,\n",
      "           2.21973091e-01,   7.43764613e-05],\n",
      "        [  3.25962366e-16,   2.14338081e-09,   8.75680507e-06,\n",
      "           2.01670796e-01,   3.18452840e-05],\n",
      "        [  1.87466043e-17,   2.17641327e-09,   3.76725689e-06,\n",
      "           1.82663798e-01,   1.36062417e-05],\n",
      "        [  1.07650567e-18,   2.19793783e-09,   1.61981211e-06,\n",
      "           1.64988294e-01,   5.80482811e-06]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('generated_data') as scope:\n",
    "    # One cell RNN input_dim (3) -> output_dim (5). sequence: 5, batch: 3\n",
    "    cell = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n",
    "    initial_state = cell.zero_state(batch_size, tf.float32)\n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data,\n",
    "                                         initial_state=initial_state, dtype=tf.float32)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dynamic rnn:  Tensor(\"MultiRNNCell/rnn/transpose:0\", shape=(3, 5, 5), dtype=float32)\n",
      "array([[[ -3.10879829e-03,  -9.10558592e-05,  -7.01715879e-04,\n",
      "           1.35251693e-03,   2.53533502e-03],\n",
      "        [ -1.14160208e-02,  -2.80480108e-05,  -1.63684087e-03,\n",
      "           4.57381038e-03,   9.74608120e-03],\n",
      "        [ -2.22180523e-02,   3.08097660e-04,  -2.13090540e-03,\n",
      "           8.49605352e-03,   1.99551620e-02],\n",
      "        [ -3.23539115e-02,   7.02519726e-04,  -1.67509622e-03,\n",
      "           1.15907416e-02,   3.06729674e-02],\n",
      "        [ -3.89163308e-02,   1.22858479e-03,   8.57470091e-04,\n",
      "           1.18361041e-02,   3.99656408e-02]],\n",
      "\n",
      "       [[  8.56597035e-04,   7.93908432e-04,   5.59686450e-03,\n",
      "          -4.41643409e-03,   6.43887848e-04],\n",
      "        [  1.67058269e-03,   1.36145751e-03,   1.66291557e-02,\n",
      "          -1.41483489e-02,   7.71593652e-04],\n",
      "        [  1.91514462e-03,   1.40461128e-03,   3.01372949e-02,\n",
      "          -2.76954118e-02,  -1.30095705e-03],\n",
      "        [  1.12191879e-03,   1.12981093e-03,   4.35832031e-02,\n",
      "          -4.29573022e-02,  -6.18922431e-03],\n",
      "        [ -9.90379252e-04,   7.79668044e-04,   5.57502434e-02,\n",
      "          -5.82473576e-02,  -1.37054017e-02]],\n",
      "\n",
      "       [[  3.20020434e-03,   7.29400141e-04,   6.98260404e-03,\n",
      "          -6.08149497e-03,  -1.22098892e-03],\n",
      "        [  6.36928435e-03,   1.16217067e-03,   1.90642942e-02,\n",
      "          -1.75039843e-02,  -3.88378790e-03],\n",
      "        [  8.19770992e-03,   1.25892612e-03,   3.25303152e-02,\n",
      "          -3.17953900e-02,  -8.54605623e-03],\n",
      "        [  8.19627941e-03,   1.17743609e-03,   4.53238562e-02,\n",
      "          -4.69417013e-02,  -1.52633339e-02],\n",
      "        [  6.38724072e-03,   1.10207184e-03,   5.66427037e-02,\n",
      "          -6.15584217e-02,  -2.38218009e-02]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('MultiRNNCell') as scope:\n",
    "    # Make rnn\n",
    "    cell = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n",
    "    cell = rnn.MultiRNNCell([cell] * 3, state_is_tuple=True) # 3 layers\n",
    "\n",
    "    # rnn in/out\n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)\n",
    "    print(\"dynamic rnn: \", outputs)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())  # batch size, unrolling (time), hidden_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dynamic rnn:  Tensor(\"dynamic_rnn/rnn/transpose:0\", shape=(3, 5, 5), dtype=float32)\n",
      "array([[[  1.02596119e-01,   8.13936442e-02,  -2.21952677e-01,\n",
      "           3.14354822e-02,   1.33584484e-01],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00]],\n",
      "\n",
      "       [[  7.15273619e-01,   3.41551058e-05,  -4.73012865e-01,\n",
      "           5.27261022e-07,   1.08660650e-04],\n",
      "        [  9.16665077e-01,   3.21273251e-06,  -5.50870597e-01,\n",
      "           2.17522498e-07,   1.91098352e-05],\n",
      "        [  9.69454646e-01,   4.34344145e-07,  -5.64205647e-01,\n",
      "           6.60358168e-08,   3.42170779e-06],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00]],\n",
      "\n",
      "       [[  7.57702351e-01,   2.50661802e-09,  -5.41027546e-01,\n",
      "           5.73374681e-13,   2.49524046e-08],\n",
      "        [  9.47589517e-01,   2.21001203e-10,  -6.10944688e-01,\n",
      "           2.41092899e-13,   4.25206892e-09],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('dynamic_rnn') as scope:\n",
    "    cell = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32,\n",
    "                                         sequence_length=[1, 3, 2])\n",
    "    # lentgh 1 for batch 1, lentgh 2 for batch 2\n",
    "    \n",
    "    print(\"dynamic rnn: \", outputs)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())  # batch size, unrolling (time), hidden_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(   array([[[  7.86920730e-03,  -1.04616687e-01,   2.39413321e-01,\n",
      "          -4.35516179e-01,   1.06723167e-01],\n",
      "        [ -1.21955043e-02,  -1.02602579e-01,   3.60045642e-01,\n",
      "          -8.40017617e-01,   2.97072291e-01],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00]],\n",
      "\n",
      "       [[ -1.20136920e-05,  -1.13076554e-03,   8.96362439e-02,\n",
      "          -7.61588812e-01,   2.08835021e-01],\n",
      "        [ -1.21170262e-06,  -5.25127922e-04,   6.47772551e-02,\n",
      "          -7.77604699e-01,   2.05319718e-01],\n",
      "        [ -1.23689546e-07,  -1.78960283e-04,   4.34194990e-02,\n",
      "          -7.70763099e-01,   1.86754748e-01],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00]],\n",
      "\n",
      "       [[ -2.41870191e-10,  -4.46524245e-06,   9.42816399e-03,\n",
      "          -7.61594176e-01,   1.42177656e-01],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00]]], dtype=float32),\n",
      "    array([[[ -1.04614452e-01,   1.66081160e-01,  -1.20360456e-01,\n",
      "          -8.23983699e-02,   3.81366670e-01],\n",
      "        [ -5.40922545e-02,   9.12286118e-02,  -4.00333591e-02,\n",
      "          -5.27230799e-02,   5.28500080e-01],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00]],\n",
      "\n",
      "       [[ -1.22155820e-03,   3.80537799e-03,   6.14994962e-04,\n",
      "          -9.74841474e-04,   7.74027526e-01],\n",
      "        [ -2.85826449e-04,   1.63079472e-03,   2.20349306e-04,\n",
      "          -3.30164738e-04,   7.66294062e-01],\n",
      "        [ -5.20601061e-05,   9.61752550e-04,   7.25701248e-05,\n",
      "          -1.08131149e-04,   7.61572838e-01],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00]],\n",
      "\n",
      "       [[ -9.32446483e-07,   6.96141724e-05,   2.49875643e-06,\n",
      "          -4.20590504e-06,   7.61594057e-01],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00],\n",
      "        [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
      "           0.00000000e+00,   0.00000000e+00]]], dtype=float32))\n",
      "(   LSTMStateTuple(c=array([[ -6.15294650e-02,  -1.23345828e+00,   9.50983405e-01,\n",
      "         -1.25812840e+00,   1.12825119e+00],\n",
      "       [ -2.34156236e-04,  -2.98752785e+00,   2.52021313e+00,\n",
      "         -1.02220488e+00,   1.22982037e+00],\n",
      "       [ -8.33474860e-06,  -9.99853134e-01,   9.40003037e-01,\n",
      "         -1.00000000e+00,   1.00000000e+00]], dtype=float32), h=array([[ -1.21955043e-02,  -1.02602579e-01,   3.60045642e-01,\n",
      "         -8.40017617e-01,   2.97072291e-01],\n",
      "       [ -1.23689546e-07,  -1.78960283e-04,   4.34194990e-02,\n",
      "         -7.70763099e-01,   1.86754748e-01],\n",
      "       [ -2.41870191e-10,  -4.46524245e-06,   9.42816399e-03,\n",
      "         -7.61594176e-01,   1.42177656e-01]], dtype=float32)),\n",
      "    LSTMStateTuple(c=array([[ -2.37466425e-01,   4.69284594e-01,  -3.70027304e-01,\n",
      "         -1.26245111e-01,   9.96336579e-01],\n",
      "       [ -4.68969867e-02,   7.25073993e-01,   5.45056105e-01,\n",
      "         -1.32846390e-03,   1.03086507e+00],\n",
      "       [ -1.13657396e-03,   9.16120887e-01,   8.83642733e-01,\n",
      "         -5.73368561e-06,   9.99999881e-01]], dtype=float32), h=array([[ -1.04614452e-01,   1.66081160e-01,  -1.20360456e-01,\n",
      "         -8.23983699e-02,   3.81366670e-01],\n",
      "       [ -1.22155820e-03,   3.80537799e-03,   6.14994962e-04,\n",
      "         -9.74841474e-04,   7.74027526e-01],\n",
      "       [ -9.32446483e-07,   6.96141724e-05,   2.49875643e-06,\n",
      "         -4.20590504e-06,   7.61594057e-01]], dtype=float32)))\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('bi-directional') as scope:\n",
    "    # bi-directional rnn\n",
    "    cell_fw = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n",
    "    cell_bw = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n",
    "\n",
    "    outputs, states = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, x_data,\n",
    "                                                      sequence_length=[2, 3, 1],\n",
    "                                                      dtype=tf.float32)\n",
    "\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(sess.run(outputs))\n",
    "    pp.pprint(sess.run(states))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[  0.,   1.,   2.],\n",
      "        [  3.,   4.,   5.],\n",
      "        [  6.,   7.,   8.],\n",
      "        [  9.,  10.,  11.],\n",
      "        [ 12.,  13.,  14.]],\n",
      "\n",
      "       [[ 15.,  16.,  17.],\n",
      "        [ 18.,  19.,  20.],\n",
      "        [ 21.,  22.,  23.],\n",
      "        [ 24.,  25.,  26.],\n",
      "        [ 27.,  28.,  29.]],\n",
      "\n",
      "       [[ 30.,  31.,  32.],\n",
      "        [ 33.,  34.,  35.],\n",
      "        [ 36.,  37.,  38.],\n",
      "        [ 39.,  40.,  41.],\n",
      "        [ 42.,  43.,  44.]]], dtype=float32)\n",
      "array([[  0.,   1.,   2.],\n",
      "       [  3.,   4.,   5.],\n",
      "       [  6.,   7.,   8.],\n",
      "       [  9.,  10.,  11.],\n",
      "       [ 12.,  13.,  14.],\n",
      "       [ 15.,  16.,  17.],\n",
      "       [ 18.,  19.,  20.],\n",
      "       [ 21.,  22.,  23.],\n",
      "       [ 24.,  25.,  26.],\n",
      "       [ 27.,  28.,  29.],\n",
      "       [ 30.,  31.,  32.],\n",
      "       [ 33.,  34.,  35.],\n",
      "       [ 36.,  37.,  38.],\n",
      "       [ 39.,  40.,  41.],\n",
      "       [ 42.,  43.,  44.]], dtype=float32)\n",
      "array([[[   25.,    28.,    31.,    34.,    37.],\n",
      "        [   70.,    82.,    94.,   106.,   118.],\n",
      "        [  115.,   136.,   157.,   178.,   199.],\n",
      "        [  160.,   190.,   220.,   250.,   280.],\n",
      "        [  205.,   244.,   283.,   322.,   361.]],\n",
      "\n",
      "       [[  250.,   298.,   346.,   394.,   442.],\n",
      "        [  295.,   352.,   409.,   466.,   523.],\n",
      "        [  340.,   406.,   472.,   538.,   604.],\n",
      "        [  385.,   460.,   535.,   610.,   685.],\n",
      "        [  430.,   514.,   598.,   682.,   766.]],\n",
      "\n",
      "       [[  475.,   568.,   661.,   754.,   847.],\n",
      "        [  520.,   622.,   724.,   826.,   928.],\n",
      "        [  565.,   676.,   787.,   898.,  1009.],\n",
      "        [  610.,   730.,   850.,   970.,  1090.],\n",
      "        [  655.,   784.,   913.,  1042.,  1171.]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# flattern based softmax\n",
    "hidden_size=3\n",
    "sequence_length=5\n",
    "batch_size=3\n",
    "num_classes=5\n",
    "\n",
    "pp.pprint(x_data) # hidden_size=3, sequence_length=4, batch_size=2\n",
    "x_data = x_data.reshape(-1, hidden_size)\n",
    "pp.pprint(x_data)\n",
    "\n",
    "softmax_w = np.arange(15, dtype=np.float32).reshape(hidden_size, num_classes)\n",
    "outputs = np.matmul(x_data, softmax_w)\n",
    "outputs = outputs.reshape(-1, sequence_length, num_classes) # batch, seq, class\n",
    "pp.pprint(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:  0.646595\n"
     ]
    }
   ],
   "source": [
    "# [batch_size, sequence_length]\n",
    "y_data = tf.constant([[1, 1, 1]])\n",
    "\n",
    "# [batch_size, sequence_length, emb_dim ]\n",
    "prediction = tf.constant([[[0, 1], [1, 0], [0, 1]]], dtype=tf.float32)\n",
    "\n",
    "# [batch_size * sequence_length]\n",
    "weights = tf.constant([[1, 1, 1]], dtype=tf.float32)\n",
    "\n",
    "sequence_loss = tf.contrib.seq2seq.sequence_loss(prediction, y_data, weights)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "print(\"Loss: \", sequence_loss.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss1:  0.313262 Loss2:  1.31326 Loss3:  0.646595\n"
     ]
    }
   ],
   "source": [
    "# [batch_size, sequence_length]\n",
    "y_data = tf.constant([[1, 1, 1]])\n",
    "\n",
    "# [batch_size, sequence_length, emb_dim ]\n",
    "prediction1 = tf.constant([[[0, 1], [0, 1], [0, 1]]], dtype=tf.float32)\n",
    "prediction2 = tf.constant([[[1, 0], [1, 0], [1, 0]]], dtype=tf.float32)\n",
    "prediction3 = tf.constant([[[0, 1], [1, 0], [0, 1]]], dtype=tf.float32)\n",
    "\n",
    "# [batch_size * sequence_length]\n",
    "weights = tf.constant([[1, 1, 1]], dtype=tf.float32)\n",
    "\n",
    "sequence_loss1 = tf.contrib.seq2seq.sequence_loss(prediction1, y_data, weights)\n",
    "sequence_loss2 = tf.contrib.seq2seq.sequence_loss(prediction2, y_data, weights)\n",
    "sequence_loss3 = tf.contrib.seq2seq.sequence_loss(prediction3, y_data, weights)\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "print(\"Loss1: \", sequence_loss1.eval(),\n",
    "      \"Loss2: \", sequence_loss2.eval(),\n",
    "      \"Loss3: \", sequence_loss3.eval())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}