2_fullyconnected-keras.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kR-4eNdK6lYS"
   },
   "source": [
    "Deep Learning\n",
    "=============\n",
    "\n",
    "Assignment 2\n",
    "------------\n",
    "\n",
    "Previously in `1_notmnist.ipynb`, we created a pickle with formatted datasets for training, development and testing on the [notMNIST dataset](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html).\n",
    "\n",
    "The goal of this assignment is to progressively train deeper and more accurate models using TensorFlow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attempting to download: notMNIST_large.tar.gz\n",
      "0%....5%....10%....15%....20%....25%....30%....35%....40%....45%....50%....55%....60%....65%....70%....75%....80%....85%....90%....95%....100%\n",
      "Download Complete!\n",
      "Found and verified notMNIST_large.tar.gz\n",
      "Attempting to download: notMNIST_small.tar.gz\n",
      "0%....5%....10%....15%....20%....25%....30%....35%....40%....45%....50%....55%....60%....65%....70%....75%....80%....85%....90%....95%....100%\n",
      "Download Complete!\n",
      "Found and verified notMNIST_small.tar.gz\n",
      "Extracting data for notMNIST_large. This may take a while. Please wait.\n",
      "['notMNIST_large/A', 'notMNIST_large/B', 'notMNIST_large/C', 'notMNIST_large/D', 'notMNIST_large/E', 'notMNIST_large/F', 'notMNIST_large/G', 'notMNIST_large/H', 'notMNIST_large/I', 'notMNIST_large/J']\n",
      "Extracting data for notMNIST_small. This may take a while. Please wait.\n",
      "['notMNIST_small/A', 'notMNIST_small/B', 'notMNIST_small/C', 'notMNIST_small/D', 'notMNIST_small/E', 'notMNIST_small/F', 'notMNIST_small/G', 'notMNIST_small/H', 'notMNIST_small/I', 'notMNIST_small/J']\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import tarfile\n",
    "from IPython.display import display, Image\n",
    "from scipy import ndimage\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "from six.moves import cPickle as pickle\n",
    "\n",
    "url = 'http://commondatastorage.googleapis.com/books1000/'\n",
    "last_percent_reported = None\n",
    "\n",
    "def download_progress_hook(count, blockSize, totalSize):\n",
    "  \"\"\"A hook to report the progress of a download. This is mostly intended for users with\n",
    "  slow internet connections. Reports every 1% change in download progress.\n",
    "  \"\"\"\n",
    "  global last_percent_reported\n",
    "  percent = int(count * blockSize * 100 / totalSize)\n",
    "\n",
    "  if last_percent_reported != percent:\n",
    "    if percent % 5 == 0:\n",
    "      sys.stdout.write(\"%s%%\" % percent)\n",
    "      sys.stdout.flush()\n",
    "    else:\n",
    "      sys.stdout.write(\".\")\n",
    "      sys.stdout.flush()\n",
    "      \n",
    "    last_percent_reported = percent\n",
    "        \n",
    "def maybe_download(filename, expected_bytes, force=False):\n",
    "  \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "  if force or not os.path.exists(filename):\n",
    "    print('Attempting to download:', filename) \n",
    "    filename, _ = urlretrieve(url + filename, filename, reporthook=download_progress_hook)\n",
    "    print('\\nDownload Complete!')\n",
    "  statinfo = os.stat(filename)\n",
    "  if statinfo.st_size == expected_bytes:\n",
    "    print('Found and verified', filename)\n",
    "  else:\n",
    "    raise Exception(\n",
    "      'Failed to verify ' + filename + '. Can you get to it with a browser?')\n",
    "  return filename\n",
    "\n",
    "train_filename = maybe_download('notMNIST_large.tar.gz', 247336696)\n",
    "test_filename = maybe_download('notMNIST_small.tar.gz', 8458043)\n",
    "\n",
    "num_classes = 10\n",
    "np.random.seed(133)\n",
    "\n",
    "def maybe_extract(filename, force=False):\n",
    "  root = os.path.splitext(os.path.splitext(filename)[0])[0]  # remove .tar.gz\n",
    "  if os.path.isdir(root) and not force:\n",
    "    # You may override by setting force=True.\n",
    "    print('%s already present - Skipping extraction of %s.' % (root, filename))\n",
    "  else:\n",
    "    print('Extracting data for %s. This may take a while. Please wait.' % root)\n",
    "    tar = tarfile.open(filename)\n",
    "    sys.stdout.flush()\n",
    "    tar.extractall()\n",
    "    tar.close()\n",
    "  data_folders = [\n",
    "    os.path.join(root, d) for d in sorted(os.listdir(root))\n",
    "    if os.path.isdir(os.path.join(root, d))]\n",
    "  if len(data_folders) != num_classes:\n",
    "    raise Exception(\n",
    "      'Expected %d folders, one per class. Found %d instead.' % (\n",
    "        num_classes, len(data_folders)))\n",
    "  print(data_folders)\n",
    "  return data_folders\n",
    "  \n",
    "train_folders = maybe_extract(train_filename)\n",
    "test_folders = maybe_extract(test_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pickling notMNIST_large/A.pickle.\n",
      "notMNIST_large/A\n",
      "Could not read: notMNIST_large/A/RnJlaWdodERpc3BCb29rSXRhbGljLnR0Zg==.png : cannot identify image file 'notMNIST_large/A/RnJlaWdodERpc3BCb29rSXRhbGljLnR0Zg==.png' - it's ok, skipping.\n",
      "Could not read: notMNIST_large/A/SG90IE11c3RhcmQgQlROIFBvc3Rlci50dGY=.png : cannot identify image file 'notMNIST_large/A/SG90IE11c3RhcmQgQlROIFBvc3Rlci50dGY=.png' - it's ok, skipping.\n",
      "Could not read: notMNIST_large/A/Um9tYW5hIEJvbGQucGZi.png : cannot identify image file 'notMNIST_large/A/Um9tYW5hIEJvbGQucGZi.png' - it's ok, skipping.\n",
      "Full dataset tensor: (52912, 28, 28)\n",
      "Mean: -0.128243\n",
      "Standard deviation: 0.443109\n",
      "Pickling notMNIST_large/B.pickle.\n",
      "notMNIST_large/B\n",
      "Could not read: notMNIST_large/B/TmlraXNFRi1TZW1pQm9sZEl0YWxpYy5vdGY=.png : cannot identify image file 'notMNIST_large/B/TmlraXNFRi1TZW1pQm9sZEl0YWxpYy5vdGY=.png' - it's ok, skipping.\n",
      "Full dataset tensor: (52912, 28, 28)\n",
      "Mean: -0.0075629\n",
      "Standard deviation: 0.454487\n",
      "Pickling notMNIST_large/C.pickle.\n",
      "notMNIST_large/C\n",
      "Full dataset tensor: (52912, 28, 28)\n",
      "Mean: -0.142258\n",
      "Standard deviation: 0.439806\n",
      "Pickling notMNIST_large/D.pickle.\n",
      "notMNIST_large/D\n",
      "Could not read: notMNIST_large/D/VHJhbnNpdCBCb2xkLnR0Zg==.png : cannot identify image file 'notMNIST_large/D/VHJhbnNpdCBCb2xkLnR0Zg==.png' - it's ok, skipping.\n",
      "Full dataset tensor: (52912, 28, 28)\n",
      "Mean: -0.0573667\n",
      "Standard deviation: 0.455643\n",
      "Pickling notMNIST_large/E.pickle.\n",
      "notMNIST_large/E\n",
      "Full dataset tensor: (52912, 28, 28)\n",
      "Mean: -0.069899\n",
      "Standard deviation: 0.452942\n",
      "Pickling notMNIST_large/F.pickle.\n",
      "notMNIST_large/F\n",
      "Full dataset tensor: (52912, 28, 28)\n",
      "Mean: -0.125583\n",
      "Standard deviation: 0.44709\n",
      "Pickling notMNIST_large/G.pickle.\n",
      "notMNIST_large/G\n",
      "Full dataset tensor: (52912, 28, 28)\n",
      "Mean: -0.0945814\n",
      "Standard deviation: 0.44624\n",
      "Pickling notMNIST_large/H.pickle.\n",
      "notMNIST_large/H\n",
      "Full dataset tensor: (52912, 28, 28)\n",
      "Mean: -0.0685221\n",
      "Standard deviation: 0.454232\n",
      "Pickling notMNIST_large/I.pickle.\n",
      "notMNIST_large/I\n",
      "Full dataset tensor: (52912, 28, 28)\n",
      "Mean: 0.0307862\n",
      "Standard deviation: 0.468899\n",
      "Pickling notMNIST_large/J.pickle.\n",
      "notMNIST_large/J\n",
      "Full dataset tensor: (52911, 28, 28)\n",
      "Mean: -0.153358\n",
      "Standard deviation: 0.443656\n",
      "Pickling notMNIST_small/A.pickle.\n",
      "notMNIST_small/A\n",
      "Could not read: notMNIST_small/A/RGVtb2NyYXRpY2FCb2xkT2xkc3R5bGUgQm9sZC50dGY=.png : cannot identify image file 'notMNIST_small/A/RGVtb2NyYXRpY2FCb2xkT2xkc3R5bGUgQm9sZC50dGY=.png' - it's ok, skipping.\n",
      "Full dataset tensor: (1873, 28, 28)\n",
      "Mean: -inf\n",
      "Standard deviation: inf\n",
      "Pickling notMNIST_small/B.pickle.\n",
      "notMNIST_small/B\n",
      "Full dataset tensor: (1873, 28, 28)\n",
      "Mean: 0.00535609\n",
      "Standard deviation: 0.457115\n",
      "Pickling notMNIST_small/C.pickle.\n",
      "notMNIST_small/C\n",
      "Full dataset tensor: (1873, 28, 28)\n",
      "Mean: -0.141521\n",
      "Standard deviation: 0.44269\n",
      "Pickling notMNIST_small/D.pickle.\n",
      "notMNIST_small/D\n",
      "Full dataset tensor: (1873, 28, 28)\n",
      "Mean: -0.0492167\n",
      "Standard deviation: 0.459759\n",
      "Pickling notMNIST_small/E.pickle.\n",
      "notMNIST_small/E\n",
      "Full dataset tensor: (1873, 28, 28)\n",
      "Mean: -0.0599148\n",
      "Standard deviation: 0.45735\n",
      "Pickling notMNIST_small/F.pickle.\n",
      "notMNIST_small/F\n",
      "Could not read: notMNIST_small/F/Q3Jvc3NvdmVyIEJvbGRPYmxpcXVlLnR0Zg==.png : cannot identify image file 'notMNIST_small/F/Q3Jvc3NvdmVyIEJvbGRPYmxpcXVlLnR0Zg==.png' - it's ok, skipping.\n",
      "Full dataset tensor: (1873, 28, 28)\n",
      "Mean: -0.118211\n",
      "Standard deviation: 0.452261\n",
      "Pickling notMNIST_small/G.pickle.\n",
      "notMNIST_small/G\n",
      "Full dataset tensor: (1872, 28, 28)\n",
      "Mean: -0.0925503\n",
      "Standard deviation: 0.449006\n",
      "Pickling notMNIST_small/H.pickle.\n",
      "notMNIST_small/H\n",
      "Full dataset tensor: (1872, 28, 28)\n",
      "Mean: -0.0586893\n",
      "Standard deviation: 0.458759\n",
      "Pickling notMNIST_small/I.pickle.\n",
      "notMNIST_small/I\n",
      "Full dataset tensor: (1872, 28, 28)\n",
      "Mean: 0.0526451\n",
      "Standard deviation: 0.471894\n",
      "Pickling notMNIST_small/J.pickle.\n",
      "notMNIST_small/J\n",
      "Full dataset tensor: (1872, 28, 28)\n",
      "Mean: -0.151689\n",
      "Standard deviation: 0.448014\n"
     ]
    }
   ],
   "source": [
    "image_size = 28  # Pixel width and height.\n",
    "pixel_depth = 255.0  # Number of levels per pixel.\n",
    "\n",
    "def load_letter(folder, min_num_images):\n",
    "  \"\"\"Load the data for a single letter label.\"\"\"\n",
    "  image_files = os.listdir(folder)\n",
    "  dataset = np.ndarray(shape=(len(image_files), image_size, image_size),\n",
    "                         dtype=np.float32)\n",
    "  print(folder)\n",
    "  for image_index, image in enumerate(image_files):\n",
    "    image_file = os.path.join(folder, image)\n",
    "    try:\n",
    "      image_data = (ndimage.imread(image_file).astype(float) - \n",
    "                    pixel_depth / 2) / pixel_depth\n",
    "      if image_data.shape != (image_size, image_size):\n",
    "        raise Exception('Unexpected image shape: %s' % str(image_data.shape))\n",
    "      dataset[image_index, :, :] = image_data\n",
    "    except IOError as e:\n",
    "      print('Could not read:', image_file, ':', e, '- it\\'s ok, skipping.')\n",
    "    \n",
    "  num_images = image_index + 1\n",
    "  dataset = dataset[0:num_images, :, :]\n",
    "  if num_images < min_num_images:\n",
    "    raise Exception('Many fewer images than expected: %d < %d' %\n",
    "                    (num_images, min_num_images))\n",
    "    \n",
    "  print('Full dataset tensor:', dataset.shape)\n",
    "  print('Mean:', np.mean(dataset))\n",
    "  print('Standard deviation:', np.std(dataset))\n",
    "  return dataset\n",
    "        \n",
    "def maybe_pickle(data_folders, min_num_images_per_class, force=False):\n",
    "  dataset_names = []\n",
    "  for folder in data_folders:\n",
    "    set_filename = folder + '.pickle'\n",
    "    dataset_names.append(set_filename)\n",
    "    if os.path.exists(set_filename) and not force:\n",
    "      # You may override by setting force=True.\n",
    "      print('%s already present - Skipping pickling.' % set_filename)\n",
    "    else:\n",
    "      print('Pickling %s.' % set_filename)\n",
    "      dataset = load_letter(folder, min_num_images_per_class)\n",
    "      try:\n",
    "        with open(set_filename, 'wb') as f:\n",
    "          pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)\n",
    "      except Exception as e:\n",
    "        print('Unable to save data to', set_filename, ':', e)\n",
    "  \n",
    "  return dataset_names\n",
    "\n",
    "train_datasets = maybe_pickle(train_folders, 45000)\n",
    "test_datasets = maybe_pickle(test_folders, 1800)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training: (200000, 28, 28) (200000,)\n",
      "Validation: (10000, 28, 28) (10000,)\n",
      "Testing: (10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "def make_arrays(nb_rows, img_size):\n",
    "  if nb_rows:\n",
    "    dataset = np.ndarray((nb_rows, img_size, img_size), dtype=np.float32)\n",
    "    labels = np.ndarray(nb_rows, dtype=np.int32)\n",
    "  else:\n",
    "    dataset, labels = None, None\n",
    "  return dataset, labels\n",
    "\n",
    "def merge_datasets(pickle_files, train_size, valid_size=0):\n",
    "  num_classes = len(pickle_files)\n",
    "  valid_dataset, valid_labels = make_arrays(valid_size, image_size)\n",
    "  train_dataset, train_labels = make_arrays(train_size, image_size)\n",
    "  vsize_per_class = valid_size // num_classes\n",
    "  tsize_per_class = train_size // num_classes\n",
    "    \n",
    "  start_v, start_t = 0, 0\n",
    "  end_v, end_t = vsize_per_class, tsize_per_class\n",
    "  end_l = vsize_per_class+tsize_per_class\n",
    "  for label, pickle_file in enumerate(pickle_files):       \n",
    "    try:\n",
    "      with open(pickle_file, 'rb') as f:\n",
    "        letter_set = pickle.load(f)\n",
    "        # let's shuffle the letters to have random validation and training set\n",
    "        np.random.shuffle(letter_set)\n",
    "        if valid_dataset is not None:\n",
    "          valid_letter = letter_set[:vsize_per_class, :, :]\n",
    "          valid_dataset[start_v:end_v, :, :] = valid_letter\n",
    "          valid_labels[start_v:end_v] = label\n",
    "          start_v += vsize_per_class\n",
    "          end_v += vsize_per_class\n",
    "                    \n",
    "        train_letter = letter_set[vsize_per_class:end_l, :, :]\n",
    "        train_dataset[start_t:end_t, :, :] = train_letter\n",
    "        train_labels[start_t:end_t] = label\n",
    "        start_t += tsize_per_class\n",
    "        end_t += tsize_per_class\n",
    "    except Exception as e:\n",
    "      print('Unable to process data from', pickle_file, ':', e)\n",
    "      raise\n",
    "    \n",
    "  return valid_dataset, valid_labels, train_dataset, train_labels\n",
    "            \n",
    "            \n",
    "train_size = 200000\n",
    "valid_size = 10000\n",
    "test_size = 10000\n",
    "\n",
    "valid_dataset, valid_labels, train_dataset, train_labels = merge_datasets(\n",
    "  train_datasets, train_size, valid_size)\n",
    "_, _, test_dataset, test_labels = merge_datasets(test_datasets, test_size)\n",
    "\n",
    "print('Training:', train_dataset.shape, train_labels.shape)\n",
    "print('Validation:', valid_dataset.shape, valid_labels.shape)\n",
    "print('Testing:', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle_file = 'notMNIST.pickle'\n",
    "\n",
    "try:\n",
    "  f = open(pickle_file, 'wb')\n",
    "  save = {\n",
    "    'train_dataset': train_dataset,\n",
    "    'train_labels': train_labels,\n",
    "    'valid_dataset': valid_dataset,\n",
    "    'valid_labels': valid_labels,\n",
    "    'test_dataset': test_dataset,\n",
    "    'test_labels': test_labels,\n",
    "    }\n",
    "  pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)\n",
    "  f.close()\n",
    "except Exception as e:\n",
    "  print('Unable to save data to', pickle_file, ':', e)\n",
    "  raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "JLpLa8Jt7Vu4"
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "# from __future__ import print_function\n",
    "import numpy as np\n",
    "# import tensorflow as tf\n",
    "from six.moves import cPickle as pickle\n",
    "# from six.moves import range\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Embedding, Dropout\n",
    "from keras.regularizers import l2\n",
    "\n",
    "# For sake of annoying error\n",
    "import theano\n",
    "theano.config.compute_test_value = 'ignore'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1HrCK6e17WzV"
   },
   "source": [
    "First reload the data we generated in `1_notmnist.ipynb`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 19456,
     "status": "ok",
     "timestamp": 1449847956073,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "y3-cj1bpmuxc",
    "outputId": "0ddb1607-1fc4-4ddb-de28-6c7ab7fb0c33"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 28, 28) (200000,)\n",
      "Validation set (10000, 28, 28) (10000,)\n",
      "Test set (10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "pickle_file = 'notMNIST.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "    save = pickle.load(f)\n",
    "    train_dataset = save['train_dataset']\n",
    "    train_labels = save['train_labels']\n",
    "    valid_dataset = save['valid_dataset']\n",
    "    valid_labels = save['valid_labels']\n",
    "    test_dataset = save['test_dataset']\n",
    "    test_labels = save['test_labels']\n",
    "    del save  # hint to help gc free up memory\n",
    "    print('Training set', train_dataset.shape, train_labels.shape)\n",
    "    print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "    print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.imshow(train_dataset[1])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "L7aHrm6nGDMB"
   },
   "source": [
    "Reformat into a shape that's more adapted to the models we're going to train:\n",
    "- data as a flat matrix,\n",
    "- labels as float 1-hot encodings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 19723,
     "status": "ok",
     "timestamp": 1449847956364,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "IRSyYiIIGIzS",
    "outputId": "2ba0fc75-1487-4ace-a562-cf81cae82793"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 784) (200000, 10)\n",
      "Validation set (10000, 784) (10000, 10)\n",
      "Test set (10000, 784) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "image_size = 28\n",
    "num_labels = 10\n",
    "\n",
    "def reformat(dataset, labels):\n",
    "    dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)\n",
    "    # Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...]\n",
    "    labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n",
    "    return dataset, labels\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
    "test_dataset, test_labels = reformat(test_dataset, test_labels)\n",
    "print('Training set', train_dataset.shape, train_labels.shape)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 784)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.]], dtype=float32)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sklearn part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "train_len = 5000\n",
    "train_sample = train_dataset[0:train_len,:]\n",
    "train_sample_labels = train_labels[0:train_len]\n",
    "train_sample_labels = np.argmax(train_sample_labels,axis=1)\n",
    "\n",
    "logreg = LogisticRegression()\n",
    "logreg.fit(train_sample, train_sample_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9506"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Z = logreg.predict(train_sample)\n",
    "sum(Z==train_sample_labels)/len(train_sample_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8518"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Z = logreg.predict(test_dataset)\n",
    "sum(Z==np.argmax(test_labels,axis=1))/len(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep Learning part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Dense(input_dim=train_dataset.shape[1],output_dim=10,activation=\"softmax\"))\n",
    "model.compile(optimizer='adagrad', loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "# model.fit(train_dataset,train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "dense_1 (Dense)                  (None, 10)            7850        dense_input_1[0][0]              \n",
      "====================================================================================================\n",
      "Total params: 7850\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 784)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 200000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "200000/200000 [==============================] - 0s - loss: 0.6895 - acc: 0.8194 - val_loss: 0.6910 - val_acc: 0.8199\n",
      "Epoch 2/10\n",
      "200000/200000 [==============================] - 0s - loss: 0.6779 - acc: 0.8223 - val_loss: 0.6819 - val_acc: 0.8228\n",
      "Epoch 3/10\n",
      "200000/200000 [==============================] - 0s - loss: 0.6706 - acc: 0.8239 - val_loss: 0.6763 - val_acc: 0.8244\n",
      "Epoch 4/10\n",
      "200000/200000 [==============================] - 0s - loss: 0.6653 - acc: 0.8252 - val_loss: 0.6724 - val_acc: 0.8255\n",
      "Epoch 5/10\n",
      "200000/200000 [==============================] - 0s - loss: 0.6613 - acc: 0.8262 - val_loss: 0.6692 - val_acc: 0.8265\n",
      "Epoch 6/10\n",
      "200000/200000 [==============================] - 0s - loss: 0.6581 - acc: 0.8272 - val_loss: 0.6668 - val_acc: 0.8271\n",
      "Epoch 7/10\n",
      "200000/200000 [==============================] - 0s - loss: 0.6554 - acc: 0.8280 - val_loss: 0.6645 - val_acc: 0.8270\n",
      "Epoch 8/10\n",
      "200000/200000 [==============================] - 0s - loss: 0.6531 - acc: 0.8284 - val_loss: 0.6633 - val_acc: 0.8288\n",
      "Epoch 9/10\n",
      "200000/200000 [==============================] - 0s - loss: 0.6511 - acc: 0.8291 - val_loss: 0.6610 - val_acc: 0.8286\n",
      "Epoch 10/10\n",
      "200000/200000 [==============================] - 0s - loss: 0.6495 - acc: 0.8296 - val_loss: 0.6595 - val_acc: 0.8294\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x158831c50>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_dataset,train_labels,validation_data=(valid_dataset,valid_labels),batch_size=4096,nb_epoch=10,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.89490000000000003"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Z = np.argmax(model.predict(test_dataset),axis=1)\n",
    "sum(Z==np.argmax(test_labels,axis=1))/len(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7omWxtvLLxik"
   },
   "source": [
    "---\n",
    "Problem\n",
    "-------\n",
    "\n",
    "Turn the logistic regression example with SGD into a 1-hidden layer neural network with rectified linear units [nn.relu()](https://www.tensorflow.org/versions/r0.7/api_docs/python/nn.html#relu) and 1024 hidden nodes. This model should improve your validation / test accuracy.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Dense(input_dim=train_dataset.shape[1],output_dim=1024,activation=\"relu\"))\n",
    "model.add(Dense(output_dim=10,activation=\"softmax\"))\n",
    "model.compile(optimizer='adagrad', loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "dense_4 (Dense)                  (None, 1024)          803840      dense_input_3[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dense_5 (Dense)                  (None, 10)            10250       dense_4[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 814090\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 200000 samples, validate on 10000 samples\n",
      "Epoch 1/2\n",
      "200000/200000 [==============================] - 16s - loss: 0.4710 - acc: 0.8651 - val_loss: 0.4142 - val_acc: 0.8753\n",
      "Epoch 2/2\n",
      "103040/200000 [==============>...............] - ETA: 7s - loss: 0.3664 - acc: 0.8934"
     ]
    }
   ],
   "source": [
    "model.fit(train_dataset,train_labels,validation_data=(valid_dataset,valid_labels),batch_size=128,nb_epoch=2,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "colab": {
   "default_view": {},
   "name": "2_fullyconnected.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "latex_envs": {
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 0
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}