Bad Machine Learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is an example of **how not** to use Deep Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from six.moves.urllib.request import urlretrieve\n",
    "\n",
    "filename = \"BlogFeedback.zip\"\n",
    "url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00304/BlogFeedback.zip\"\n",
    "urlretrieve(url,filename)\n",
    "\n",
    "import zipfile\n",
    "\n",
    "zip_ref = zipfile.ZipFile(filename, 'r')\n",
    "zip_ref.extractall(\"blogData\")\n",
    "zip_ref.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df_train = pd.read_csv(\"blogData/blogData_train.csv\",header=None)\n",
    "x_train = df_train.values[:,:-1]\n",
    "y_train = df_train.values[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = df_train.values[:,:-1]\n",
    "y_train = df_train.values[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>271</th>\n",
       "      <th>272</th>\n",
       "      <th>273</th>\n",
       "      <th>274</th>\n",
       "      <th>275</th>\n",
       "      <th>276</th>\n",
       "      <th>277</th>\n",
       "      <th>278</th>\n",
       "      <th>279</th>\n",
       "      <th>280</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40.30467</td>\n",
       "      <td>53.845657</td>\n",
       "      <td>0.0</td>\n",
       "      <td>401.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.52416</td>\n",
       "      <td>32.44188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40.30467</td>\n",
       "      <td>53.845657</td>\n",
       "      <td>0.0</td>\n",
       "      <td>401.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.52416</td>\n",
       "      <td>32.44188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>40.30467</td>\n",
       "      <td>53.845657</td>\n",
       "      <td>0.0</td>\n",
       "      <td>401.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.52416</td>\n",
       "      <td>32.44188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>40.30467</td>\n",
       "      <td>53.845657</td>\n",
       "      <td>0.0</td>\n",
       "      <td>401.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.52416</td>\n",
       "      <td>32.44188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40.30467</td>\n",
       "      <td>53.845657</td>\n",
       "      <td>0.0</td>\n",
       "      <td>401.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.52416</td>\n",
       "      <td>32.44188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>40.30467</td>\n",
       "      <td>53.845657</td>\n",
       "      <td>0.0</td>\n",
       "      <td>401.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.52416</td>\n",
       "      <td>32.44188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>40.30467</td>\n",
       "      <td>53.845657</td>\n",
       "      <td>0.0</td>\n",
       "      <td>401.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.52416</td>\n",
       "      <td>32.44188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>40.30467</td>\n",
       "      <td>53.845657</td>\n",
       "      <td>0.0</td>\n",
       "      <td>401.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.52416</td>\n",
       "      <td>32.44188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>40.30467</td>\n",
       "      <td>53.845657</td>\n",
       "      <td>0.0</td>\n",
       "      <td>401.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.52416</td>\n",
       "      <td>32.44188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>40.30467</td>\n",
       "      <td>53.845657</td>\n",
       "      <td>0.0</td>\n",
       "      <td>401.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.52416</td>\n",
       "      <td>32.44188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 281 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0          1    2      3     4         5         6    7      8    9    \\\n",
       "0  40.30467  53.845657  0.0  401.0  15.0  15.52416  32.44188  0.0  377.0  3.0   \n",
       "1  40.30467  53.845657  0.0  401.0  15.0  15.52416  32.44188  0.0  377.0  3.0   \n",
       "2  40.30467  53.845657  0.0  401.0  15.0  15.52416  32.44188  0.0  377.0  3.0   \n",
       "3  40.30467  53.845657  0.0  401.0  15.0  15.52416  32.44188  0.0  377.0  3.0   \n",
       "4  40.30467  53.845657  0.0  401.0  15.0  15.52416  32.44188  0.0  377.0  3.0   \n",
       "5  40.30467  53.845657  0.0  401.0  15.0  15.52416  32.44188  0.0  377.0  3.0   \n",
       "6  40.30467  53.845657  0.0  401.0  15.0  15.52416  32.44188  0.0  377.0  3.0   \n",
       "7  40.30467  53.845657  0.0  401.0  15.0  15.52416  32.44188  0.0  377.0  3.0   \n",
       "8  40.30467  53.845657  0.0  401.0  15.0  15.52416  32.44188  0.0  377.0  3.0   \n",
       "9  40.30467  53.845657  0.0  401.0  15.0  15.52416  32.44188  0.0  377.0  3.0   \n",
       "\n",
       "   ...   271  272  273  274  275  276  277  278  279   280  \n",
       "0  ...   0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   1.0  \n",
       "1  ...   1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  \n",
       "2  ...   1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  \n",
       "3  ...   0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   1.0  \n",
       "4  ...   0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  27.0  \n",
       "5  ...   1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  \n",
       "6  ...   1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  \n",
       "7  ...   0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  27.0  \n",
       "8  ...   0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   9.0  \n",
       "9  ...   0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   9.0  \n",
       "\n",
       "[10 rows x 281 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(52397, 281)\n"
     ]
    }
   ],
   "source": [
    "print(df_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "test_files = glob.glob(\"blogData/blogData_test*.csv\")\n",
    "\n",
    "dfs = []\n",
    "for file in test_files:\n",
    "    dfs.append(pd.read_csv(file,header=None))\n",
    "    \n",
    "df_test = pd.concat(dfs)    \n",
    "\n",
    "x_test = df_test.values[:,:-1]\n",
    "y_test = df_test.values[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>271</th>\n",
       "      <th>272</th>\n",
       "      <th>273</th>\n",
       "      <th>274</th>\n",
       "      <th>275</th>\n",
       "      <th>276</th>\n",
       "      <th>277</th>\n",
       "      <th>278</th>\n",
       "      <th>279</th>\n",
       "      <th>280</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.630660</td>\n",
       "      <td>17.882992</td>\n",
       "      <td>1.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.018276</td>\n",
       "      <td>10.396790</td>\n",
       "      <td>0.0</td>\n",
       "      <td>235.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>43.435825</td>\n",
       "      <td>75.590485</td>\n",
       "      <td>0.0</td>\n",
       "      <td>634.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.998589</td>\n",
       "      <td>44.560870</td>\n",
       "      <td>0.0</td>\n",
       "      <td>473.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.733333</td>\n",
       "      <td>3.043390</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>1.526070</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27.230215</td>\n",
       "      <td>45.970950</td>\n",
       "      <td>0.0</td>\n",
       "      <td>371.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>10.784173</td>\n",
       "      <td>24.209942</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.500000</td>\n",
       "      <td>6.677075</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>156.402980</td>\n",
       "      <td>246.055980</td>\n",
       "      <td>0.0</td>\n",
       "      <td>970.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>76.149254</td>\n",
       "      <td>131.900830</td>\n",
       "      <td>0.0</td>\n",
       "      <td>725.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>139.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10.509316</td>\n",
       "      <td>36.593983</td>\n",
       "      <td>0.0</td>\n",
       "      <td>191.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.602485</td>\n",
       "      <td>20.633831</td>\n",
       "      <td>0.0</td>\n",
       "      <td>179.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>123.869190</td>\n",
       "      <td>129.566220</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1065.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>43.328970</td>\n",
       "      <td>62.774147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>491.0</td>\n",
       "      <td>19.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>22.463415</td>\n",
       "      <td>42.184900</td>\n",
       "      <td>0.0</td>\n",
       "      <td>188.0</td>\n",
       "      <td>7.5</td>\n",
       "      <td>8.219512</td>\n",
       "      <td>25.020493</td>\n",
       "      <td>0.0</td>\n",
       "      <td>174.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.155508</td>\n",
       "      <td>0.668326</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.075594</td>\n",
       "      <td>0.411378</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>16.593575</td>\n",
       "      <td>19.671364</td>\n",
       "      <td>1.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.512450</td>\n",
       "      <td>11.051215</td>\n",
       "      <td>0.0</td>\n",
       "      <td>111.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.378698</td>\n",
       "      <td>1.081757</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.035503</td>\n",
       "      <td>0.214655</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>49.442368</td>\n",
       "      <td>112.620125</td>\n",
       "      <td>1.0</td>\n",
       "      <td>849.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>20.445482</td>\n",
       "      <td>62.619390</td>\n",
       "      <td>0.0</td>\n",
       "      <td>506.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>122.812930</td>\n",
       "      <td>109.961100</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1069.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>44.894543</td>\n",
       "      <td>74.547530</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1046.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>56.512093</td>\n",
       "      <td>77.442830</td>\n",
       "      <td>0.0</td>\n",
       "      <td>438.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>19.296530</td>\n",
       "      <td>49.221344</td>\n",
       "      <td>0.0</td>\n",
       "      <td>432.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>43.435825</td>\n",
       "      <td>75.590485</td>\n",
       "      <td>0.0</td>\n",
       "      <td>634.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.998589</td>\n",
       "      <td>44.560870</td>\n",
       "      <td>0.0</td>\n",
       "      <td>473.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>10.630660</td>\n",
       "      <td>17.882992</td>\n",
       "      <td>1.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.018276</td>\n",
       "      <td>10.396790</td>\n",
       "      <td>0.0</td>\n",
       "      <td>235.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>122.812930</td>\n",
       "      <td>109.961100</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1069.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>44.894543</td>\n",
       "      <td>74.547530</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1046.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>122.812930</td>\n",
       "      <td>109.961100</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1069.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>44.894543</td>\n",
       "      <td>74.547530</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1046.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 281 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0           1    2       3     4          5           6    7    \\\n",
       "0    10.630660   17.882992  1.0   259.0   5.0   4.018276   10.396790  0.0   \n",
       "1    43.435825   75.590485  0.0   634.0  20.0  15.998589   44.560870  0.0   \n",
       "2     1.733333    3.043390  0.0     9.0   0.0   0.733333    1.526070  0.0   \n",
       "3    27.230215   45.970950  0.0   371.0  14.0  10.784173   24.209942  0.0   \n",
       "4     4.500000    6.677075  0.0    18.0   0.5   3.000000    4.000000  0.0   \n",
       "5   156.402980  246.055980  0.0   970.0  28.0  76.149254  131.900830  0.0   \n",
       "6    10.509316   36.593983  0.0   191.0   1.0   3.602485   20.633831  0.0   \n",
       "7   123.869190  129.566220  0.0  1065.0  87.0  43.328970   62.774147  0.0   \n",
       "8    22.463415   42.184900  0.0   188.0   7.5   8.219512   25.020493  0.0   \n",
       "9     0.000000    0.000000  0.0     0.0   0.0   0.000000    0.000000  0.0   \n",
       "10    0.155508    0.668326  0.0     7.0   0.0   0.075594    0.411378  0.0   \n",
       "11   16.593575   19.671364  1.0   144.0  10.0   6.512450   11.051215  0.0   \n",
       "12    0.378698    1.081757  0.0     4.0   0.0   0.035503    0.214655  0.0   \n",
       "13   49.442368  112.620125  1.0   849.0   9.0  20.445482   62.619390  0.0   \n",
       "14  122.812930  109.961100  0.0  1069.0  89.0  44.894543   74.547530  0.0   \n",
       "15   56.512093   77.442830  0.0   438.0  32.0  19.296530   49.221344  0.0   \n",
       "16   43.435825   75.590485  0.0   634.0  20.0  15.998589   44.560870  0.0   \n",
       "17   10.630660   17.882992  1.0   259.0   5.0   4.018276   10.396790  0.0   \n",
       "18  122.812930  109.961100  0.0  1069.0  89.0  44.894543   74.547530  0.0   \n",
       "19  122.812930  109.961100  0.0  1069.0  89.0  44.894543   74.547530  0.0   \n",
       "\n",
       "       8     9    ...    271  272  273  274  275  276  277  278  279    280  \n",
       "0    235.0   1.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    4.0  \n",
       "1    473.0   2.0  ...    0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0    0.0  \n",
       "2      5.0   0.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    1.0  \n",
       "3    228.0   4.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    5.0  \n",
       "4     10.0   0.5  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0  \n",
       "5    725.0  16.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  139.0  \n",
       "6    179.0   0.0  ...    0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0    1.0  \n",
       "7    491.0  19.5  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    5.0  \n",
       "8    174.0   1.5  ...    0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0    2.0  \n",
       "9      0.0   0.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0  \n",
       "10     5.0   0.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0  \n",
       "11   111.0   2.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    1.0  \n",
       "12     2.0   0.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0  \n",
       "13   506.0   2.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    5.0  \n",
       "14  1046.0  12.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    6.0  \n",
       "15   432.0   0.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0  \n",
       "16   473.0   2.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   22.0  \n",
       "17   235.0   1.0  ...    0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0    0.0  \n",
       "18  1046.0  12.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0  \n",
       "19  1046.0  12.0  ...    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0  \n",
       "\n",
       "[20 rows x 281 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7624, 281)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ -5.16689953e-01   1.39052440e-01  -1.21287604e+10  -8.26500036e-03\n",
      "    6.07796765e-02  -2.14362619e+05  -1.94530395e-01  -1.84347110e-02\n",
      "    3.50181746e-02  -6.99542083e-02   2.12103171e+05   1.34653649e-01\n",
      "   -1.98930637e+10  -6.01333846e-02   1.19197986e-01   1.41589606e-01\n",
      "   -1.19700199e-01   1.21287604e+10   2.19469150e-02   1.65173850e-02\n",
      "    1.51843607e+04   6.05153523e-02  -7.95234974e-03   3.60551366e-03\n",
      "    4.59733783e-02  -3.54977397e-02   2.78596544e-03  -8.83281211e+08\n",
      "   -6.21900377e-03  -1.01636537e-03  -4.97407631e+04   3.77590045e-02\n",
      "    1.26823795e+10  -8.54396820e-03   2.68959999e-03   4.73144057e+04\n",
      "   -4.54580784e-02   1.49226958e+08   1.95808411e-02  -5.30740619e-03\n",
      "    1.75871849e-02   1.49297714e-03   8.83281211e+08   3.98468971e-03\n",
      "   -1.39629841e-03   1.13904941e+04  -6.15143776e-03   6.75678253e-04\n",
      "   -6.63185120e-03   6.83605671e-04  -6.05580807e-02  -2.22909550e+08\n",
      "    2.22909550e+08  -4.21763659e-02   4.19207708e+08  -1.88636333e-02\n",
      "   -8.35921082e+09   8.35921082e+09   1.66872144e-02   1.49769194e+10\n",
      "   -7.65019655e-03   4.85855341e-03   1.56763941e-03  -4.21524048e-04\n",
      "    7.87603455e+09   6.59599900e-04  -2.60353088e-04   2.48908997e-04\n",
      "    1.06525421e-03  -2.15496868e-03  -1.83072686e-03   1.77812576e-03\n",
      "    6.90204434e+09  -4.36188281e-03  -9.45687294e-04   2.82925222e+08\n",
      "    3.45373154e-03  -1.57225132e-03  -6.67572021e-05   5.16835249e+09\n",
      "   -3.29752763e+09  -2.99156457e-03  -5.12704253e-04  -8.57171044e-03\n",
      "    2.09601372e-02  -7.62939453e-05  -1.07409153e-02   1.77609921e-03\n",
      "   -8.04305077e-04   2.13124938e-02  -3.92339518e-03   2.10016146e-02\n",
      "    1.08009577e-03  -7.13960310e+08  -4.59448108e-03   6.65187836e-04\n",
      "   -9.64536406e+09   6.99539110e-03  -2.05399469e-03  -4.09550965e-03\n",
      "   -3.32832336e-04   5.32150269e-04   3.95178795e-05   4.61578369e-04\n",
      "    1.42425299e-03  -1.15330517e-03  -2.93838978e-03  -9.23871994e-04\n",
      "    2.14947388e-04  -9.56535339e-04   4.82622534e-04  -2.55947933e-03\n",
      "    2.09584832e-03  -7.71045685e-04  -7.99894333e-05  -1.23977661e-03\n",
      "   -1.05491281e-03   4.59134579e-04  -8.57353210e-04  -1.58071518e-04\n",
      "    5.24520874e-04  -2.81333923e-04  -1.21159852e-03   1.60360970e-02\n",
      "   -1.17158890e-03   3.38935107e-03  -2.23350525e-03   9.47713852e-05\n",
      "    7.95673579e-04   4.31522083e+09  -6.83307648e-04  -1.88583322e+09\n",
      "    2.62123346e-03   2.60591507e-04  -6.60896301e-04   1.57833099e-04\n",
      "    3.26091051e-03   9.98616219e-04   1.55448914e-04  -1.71422958e-04\n",
      "    3.68595123e-04   8.76732171e-04  -4.83512878e-04   6.11543655e-04\n",
      "   -3.97264957e-03  -2.01252103e-03  -4.89398837e-03  -4.21827286e-03\n",
      "   -2.34989415e+09   7.43180513e-04   3.19480896e-04  -2.09453702e-03\n",
      "   -7.16984272e-04   1.54352188e-03  -1.27524994e-02   1.31209460e+09\n",
      "    3.44753265e-04   7.32421875e-04   5.79833984e-04  -5.12534007e-03\n",
      "    2.09436463e+08  -3.90887260e-04   5.26428223e-04  -1.52634829e-03\n",
      "   -3.34136188e-04  -1.34787212e+09   7.31287524e-04  -1.66827440e-03\n",
      "   -1.16005359e+09  -3.84271145e-04   1.11995004e-02  -1.91124113e+08\n",
      "   -3.05766612e-03  -1.02424622e-03   9.40531492e-04  -3.41424346e-03\n",
      "   -6.80208206e-04  -1.06555223e-03  -2.07079405e+08  -2.76565552e-04\n",
      "   -4.67300415e-04   2.72035599e-04  -4.91380692e-04   4.81605530e-04\n",
      "    2.48499215e-04   8.44478607e-04  -8.22067261e-04  -1.34363770e-03\n",
      "   -9.25183296e-04  -1.75088387e+09  -2.67744064e-04   7.27176666e-04\n",
      "    3.43322754e-05  -4.95433807e-04   1.98173523e-03  -9.98854637e-04\n",
      "   -6.82950020e-04  -5.73730748e+08   2.78203189e-03  -1.81617358e+08\n",
      "   -1.76006556e-03   4.55856323e-04  -2.79115140e-03   1.85775757e-03\n",
      "    4.91499901e-04  -8.60899687e-04   2.79426575e-04   1.12080574e-03\n",
      "   -1.09478831e-03   1.76429749e-04  -2.12579966e-03  -8.75188740e+08\n",
      "    6.96182251e-05   3.89985368e-03  -1.44869089e-04   7.86185265e-04\n",
      "   -2.06546932e-02   3.46343219e-03  -1.47938728e-03  -6.75320625e-05\n",
      "    1.30748749e-03  -7.22724199e-03  -1.10150129e-03  -3.01454216e-03\n",
      "   -1.41814351e-03   8.83102417e-04   6.13331795e-04  -2.47001648e-04\n",
      "   -6.42284751e-04   1.58190727e-04  -1.30385160e-03   5.19752502e-04\n",
      "    2.47955322e-05  -4.12225723e-04  -1.10825896e-03  -6.04461023e+08\n",
      "   -1.92517042e-03  -2.27563083e-03  -8.71159136e-05  -6.78658485e-04\n",
      "    1.93476677e-03  -1.32441521e-03  -6.39386175e+08   3.49022448e-04\n",
      "    1.54137611e-04  -5.02109528e-04  -5.67436218e-04   3.40461731e-04\n",
      "   -3.22461128e-05  -6.42131994e+08  -4.76300716e-04   2.11162865e-03\n",
      "    9.97483730e-05  -1.42097473e-04  -1.29842758e-03   1.84671581e-03\n",
      "    6.73770905e-04   3.19421291e-04   5.33521175e-04   2.75218487e-03\n",
      "    1.99490786e-03  -1.72661431e-03   2.44370429e+09   2.44370429e+09\n",
      "    2.44370429e+09   2.44370429e+09   2.44370429e+09   2.44370429e+09\n",
      "    2.44370429e+09  -1.58652579e+10  -1.58652579e+10  -1.58652579e+10\n",
      "   -1.58652579e+10  -1.58652579e+10  -1.58652579e+10  -1.58652579e+10\n",
      "    2.56768614e-03   0.00000000e+00  -3.09541263e-03   7.77932443e-03]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn import linear_model\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "# Scale xs and ys to lie between 0 and 1\n",
    "scaler_x = MinMaxScaler()\n",
    "x_train = scaler_x.fit_transform(x_train)\n",
    "x_test = scaler_x.transform(x_test)\n",
    "\n",
    "scaler_y = MinMaxScaler()\n",
    "y_train = scaler_y.fit_transform(y_train[:,None])\n",
    "y_test = scaler_y.transform(y_test[:,None])\n",
    "\n",
    "lin_reg = linear_model.LinearRegression()\n",
    "lin_reg.fit(x_train,y_train)\n",
    "print(lin_reg.coef_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_train_pred = lin_reg.predict(x_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "View the predicted value against the true value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.00573158,  0.00070225],\n",
       "       [ 0.00325584,  0.        ],\n",
       "       [ 0.00325584,  0.        ],\n",
       "       ..., \n",
       "       [ 0.00100708,  0.        ],\n",
       "       [-0.00126457,  0.        ],\n",
       "       [-0.00328445,  0.        ]])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.hstack([y_train_pred,y_train])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.36475478021118291"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e = y_train - y_train_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_train)\n",
    "r2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the same thing against test case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.30384571496440282"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred = lin_reg.predict(x_test)\n",
    "e = y_test - y_test_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_test)\n",
    "r2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use one more linear method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  0.00000000e+00   0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "   8.23962902e-04   8.48123978e-02   0.00000000e+00   0.00000000e+00\n",
      "  -0.00000000e+00   2.64222047e-02   0.00000000e+00   7.73978460e-03\n",
      "   0.00000000e+00  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   0.00000000e+00   0.00000000e+00  -3.08295507e-03   0.00000000e+00\n",
      "   4.43591517e-02   0.00000000e+00  -0.00000000e+00  -3.11868369e-03\n",
      "   0.00000000e+00  -0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   0.00000000e+00  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "   0.00000000e+00   2.14877332e-03   3.30251140e-03  -0.00000000e+00\n",
      "  -0.00000000e+00   0.00000000e+00   7.64202780e-05   0.00000000e+00\n",
      "  -1.80160683e-03   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "  -1.17142537e-02  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "   0.00000000e+00   0.00000000e+00  -5.69064565e-02   1.61435902e-01\n",
      "  -0.00000000e+00  -2.22719861e-02   7.81006859e-02  -0.00000000e+00\n",
      "   0.00000000e+00  -0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "  -8.21726790e-03   0.00000000e+00   0.00000000e+00  -3.54030749e-04\n",
      "  -0.00000000e+00  -0.00000000e+00  -1.53778629e-04   9.21828498e-05\n",
      "   9.51404346e-04  -0.00000000e+00  -0.00000000e+00   8.85430373e-04\n",
      "  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "   2.48963853e-03  -0.00000000e+00  -4.80354510e-05  -0.00000000e+00\n",
      "  -0.00000000e+00  -2.49485494e-04  -0.00000000e+00  -0.00000000e+00\n",
      "   0.00000000e+00  -0.00000000e+00  -0.00000000e+00   1.24547043e-03\n",
      "  -7.22800208e-04   0.00000000e+00  -0.00000000e+00   0.00000000e+00\n",
      "   0.00000000e+00  -0.00000000e+00  -0.00000000e+00   5.82711705e-04\n",
      "  -0.00000000e+00   0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "  -2.73840001e-04   5.36459407e-04   0.00000000e+00   1.86737296e-04\n",
      "   0.00000000e+00  -0.00000000e+00  -0.00000000e+00  -2.83627809e-04\n",
      "  -0.00000000e+00  -4.78409328e-04  -0.00000000e+00  -0.00000000e+00\n",
      "  -0.00000000e+00  -6.45974648e-04  -0.00000000e+00  -6.87509675e-04\n",
      "  -0.00000000e+00  -0.00000000e+00  -5.69380153e-04  -0.00000000e+00\n",
      "   1.48024834e-04  -2.14040287e-04  -0.00000000e+00   0.00000000e+00\n",
      "  -2.17092199e-04  -0.00000000e+00  -7.36605762e-04  -0.00000000e+00\n",
      "  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "   0.00000000e+00   0.00000000e+00  -5.45618789e-04   0.00000000e+00\n",
      "   1.57789251e-04   6.62843108e-04   1.12064872e-04  -0.00000000e+00\n",
      "   2.45340941e-04   0.00000000e+00  -3.92444626e-04   0.00000000e+00\n",
      "  -3.46028914e-04  -0.00000000e+00  -2.95618652e-03  -0.00000000e+00\n",
      "  -0.00000000e+00  -0.00000000e+00   2.99506727e-04  -0.00000000e+00\n",
      "  -2.76037973e-04   1.44908176e-03  -0.00000000e+00  -0.00000000e+00\n",
      "  -0.00000000e+00   6.10598670e-04   3.10128476e-04  -0.00000000e+00\n",
      "  -0.00000000e+00  -4.19727026e-05   2.65361238e-04  -0.00000000e+00\n",
      "  -0.00000000e+00  -0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "  -0.00000000e+00  -3.75575627e-04   1.53221895e-04  -0.00000000e+00\n",
      "  -0.00000000e+00  -4.71678856e-04  -0.00000000e+00  -0.00000000e+00\n",
      "  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "  -2.74487573e-04   0.00000000e+00  -2.47147067e-04   4.86503179e-04\n",
      "  -0.00000000e+00   1.31846288e-04  -3.29624689e-05  -0.00000000e+00\n",
      "  -1.00670479e-04  -0.00000000e+00  -2.79915324e-04   1.23679453e-04\n",
      "   0.00000000e+00  -3.14504996e-04   1.49350311e-03  -4.02260825e-04\n",
      "  -4.43244530e-04  -0.00000000e+00   0.00000000e+00  -0.00000000e+00\n",
      "  -0.00000000e+00   4.17009032e-04  -0.00000000e+00  -0.00000000e+00\n",
      "   0.00000000e+00  -0.00000000e+00   1.49166508e-04   5.41077443e-04\n",
      "  -0.00000000e+00   1.10154937e-04  -0.00000000e+00  -0.00000000e+00\n",
      "  -0.00000000e+00   0.00000000e+00  -0.00000000e+00   4.21761601e-04\n",
      "  -1.08777504e-03   0.00000000e+00  -7.98020121e-04  -0.00000000e+00\n",
      "   1.12895339e-03  -1.37749980e-04  -0.00000000e+00  -0.00000000e+00\n",
      "  -2.46413841e-04   6.54263042e-04   0.00000000e+00  -2.30290131e-04\n",
      "  -0.00000000e+00   9.73729157e-05  -0.00000000e+00   3.96559636e-04\n",
      "   2.00514736e-05  -2.34491641e-04  -0.00000000e+00  -0.00000000e+00\n",
      "  -8.77996904e-05  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00\n",
      "   1.37356308e-03  -4.88124085e-04  -0.00000000e+00  -0.00000000e+00\n",
      "   0.00000000e+00  -2.43080184e-04  -2.25503664e-04   3.46631099e-04\n",
      "  -0.00000000e+00  -0.00000000e+00  -0.00000000e+00   4.51674159e-04\n",
      "  -0.00000000e+00  -0.00000000e+00  -8.42666667e-04  -0.00000000e+00\n",
      "   2.68414126e-04   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   7.00246801e-04  -0.00000000e+00   6.11381384e-04   3.29940132e-05\n",
      "  -3.06945668e-04  -4.89109490e-05  -2.93296713e-04  -0.00000000e+00\n",
      "   7.93875915e-04  -1.65779539e-04   2.26977199e-04   0.00000000e+00\n",
      "  -2.51260118e-04  -1.59178439e-05   2.87919881e-04   0.00000000e+00\n",
      "   0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00]\n"
     ]
    }
   ],
   "source": [
    "lin_reg = linear_model.Lasso(alpha=1e-5)\n",
    "lin_reg.fit(x_train,y_train)\n",
    "print(lin_reg.coef_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.35916815567241667"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.score(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.32755997269319675"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.score(x_test,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep Learning model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Embedding, Dropout\n",
    "from keras.regularizers import l2\n",
    "\n",
    "import theano\n",
    "theano.config.compute_test_value = 'ignore'\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Dense(input_dim=x_train.shape[1],output_dim=500,activation=\"relu\"))\n",
    "# model.add(Dense(10,activation=\"relu\"))\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adagrad', loss='mean_squared_error')\n",
    "#                     metrics=['mean_squared_error'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "dense_4 (Dense)                  (None, 500)           140500      dense_input_3[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dense_5 (Dense)                  (None, 1)             501         dense_4[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 141001\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "52397/52397 [==============================] - 1s - loss: 0.0280     \n",
      "Epoch 2/5\n",
      "52397/52397 [==============================] - 1s - loss: 0.0011     \n",
      "Epoch 3/5\n",
      "52397/52397 [==============================] - 1s - loss: 8.5390e-04     \n",
      "Epoch 4/5\n",
      "52397/52397 [==============================] - 1s - loss: 7.1216e-04     \n",
      "Epoch 5/5\n",
      "52397/52397 [==============================] - 1s - loss: 6.3427e-04     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x6675459e8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train, y_train, nb_epoch=5, batch_size=128,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51200/52397 [============================>.] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.20490720465289669"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = model.predict(x_train,batch_size=1024,verbose=1)\n",
    "e = y_train - y_train_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_train)\n",
    "r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.96978917429947709"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred = model.predict(x_test)\n",
    "e = y_test - y_test_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_test)\n",
    "r2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reguralise the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "# model.add(Dropout(0.5,input_shape=(x_train.shape[1],)))\n",
    "model.add(Dense(input_shape=(x_train.shape[1],),output_dim=1024,activation=\"relu\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(100,activation=\"relu\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adagrad', loss='mean_squared_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "dense_13 (Dense)                 (None, 1024)          287744      dense_input_5[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)              (None, 1024)          0           dense_13[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_14 (Dense)                 (None, 100)           102500      dropout_3[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)              (None, 100)           0           dense_14[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_15 (Dense)                 (None, 1)             101         dropout_4[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 390345\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "52397/52397 [==============================] - 4s - loss: 0.0471     \n",
      "Epoch 2/10\n",
      "52397/52397 [==============================] - 4s - loss: 5.8423e-04     \n",
      "Epoch 3/10\n",
      "52397/52397 [==============================] - 4s - loss: 5.8757e-04     \n",
      "Epoch 4/10\n",
      "52397/52397 [==============================] - 4s - loss: 5.4283e-04     \n",
      "Epoch 5/10\n",
      "52397/52397 [==============================] - 4s - loss: 5.5087e-04     \n",
      "Epoch 6/10\n",
      "52397/52397 [==============================] - 4s - loss: 5.3384e-04     \n",
      "Epoch 7/10\n",
      "52397/52397 [==============================] - 4s - loss: 5.1492e-04     \n",
      "Epoch 8/10\n",
      "52397/52397 [==============================] - 4s - loss: 5.1893e-04     \n",
      "Epoch 9/10\n",
      "52397/52397 [==============================] - 4s - loss: 4.9942e-04     \n",
      "Epoch 10/10\n",
      "52397/52397 [==============================] - 4s - loss: 5.1645e-04     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x135b57128>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train, y_train, nb_epoch=10, batch_size=128,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3903834247657394"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = model.predict(x_train)\n",
    "e = y_train - y_train_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_train)\n",
    "r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.18852326468969605"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred = model.predict(x_test)\n",
    "e = y_test - y_test_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_test)\n",
    "r2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add more layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "# reg = 1e-5\n",
    "model.add(Dense(input_shape=(x_train.shape[1],),output_dim=1024,activation=\"relu\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(100,activation=\"relu\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(100,activation=\"relu\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(100,activation=\"relu\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adagrad', loss='mean_squared_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "dense_16 (Dense)                 (None, 1024)          287744      dense_input_6[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)              (None, 1024)          0           dense_16[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_17 (Dense)                 (None, 100)           102500      dropout_5[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)              (None, 100)           0           dense_17[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_18 (Dense)                 (None, 100)           10100       dropout_6[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)              (None, 100)           0           dense_18[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_19 (Dense)                 (None, 100)           10100       dropout_7[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)              (None, 100)           0           dense_19[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_20 (Dense)                 (None, 1)             101         dropout_8[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 410545\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "52397/52397 [==============================] - 4s - loss: 0.0022     \n",
      "Epoch 2/20\n",
      "52397/52397 [==============================] - 4s - loss: 7.1031e-04     \n",
      "Epoch 3/20\n",
      "52397/52397 [==============================] - 4s - loss: 6.8128e-04     \n",
      "Epoch 4/20\n",
      "52397/52397 [==============================] - 4s - loss: 6.8266e-04     \n",
      "Epoch 5/20\n",
      "52397/52397 [==============================] - 5s - loss: 6.5111e-04     \n",
      "Epoch 6/20\n",
      "52397/52397 [==============================] - 5s - loss: 6.3426e-04     \n",
      "Epoch 7/20\n",
      "52397/52397 [==============================] - 5s - loss: 6.1734e-04     \n",
      "Epoch 8/20\n",
      "52397/52397 [==============================] - 4s - loss: 6.1130e-04     \n",
      "Epoch 9/20\n",
      "52397/52397 [==============================] - 4s - loss: 5.9954e-04     \n",
      "Epoch 10/20\n",
      "52397/52397 [==============================] - 4s - loss: 5.7958e-04     \n",
      "Epoch 11/20\n",
      "52397/52397 [==============================] - 5s - loss: 5.7617e-04     \n",
      "Epoch 12/20\n",
      "52397/52397 [==============================] - 5s - loss: 5.4242e-04     \n",
      "Epoch 13/20\n",
      "52397/52397 [==============================] - 5s - loss: 5.3811e-04     \n",
      "Epoch 14/20\n",
      "52397/52397 [==============================] - 5s - loss: 5.3952e-04     \n",
      "Epoch 15/20\n",
      "52397/52397 [==============================] - 5s - loss: 5.1508e-04     \n",
      "Epoch 16/20\n",
      "52397/52397 [==============================] - 5s - loss: 5.0522e-04     \n",
      "Epoch 17/20\n",
      "52397/52397 [==============================] - 5s - loss: 4.9099e-04     \n",
      "Epoch 18/20\n",
      "52397/52397 [==============================] - 5s - loss: 4.8340e-04     \n",
      "Epoch 19/20\n",
      "52397/52397 [==============================] - 5s - loss: 4.9205e-04     \n",
      "Epoch 20/20\n",
      "52397/52397 [==============================] - 5s - loss: 4.7842e-04     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x134494208>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train, y_train, nb_epoch=20, batch_size=128,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.28476809719426499"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = model.predict(x_train)\n",
    "e = y_train - y_train_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_train)\n",
    "r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.12500588987879147"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred = model.predict(x_test)\n",
    "e = y_test - y_test_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_test)\n",
    "r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "reg = 1e-5\n",
    "model.add(Dropout(p=0.4,input_shape=(x_train.shape[1],),))\n",
    "model.add(Dense(output_dim=10,activation=\"relu\"))\n",
    "model.add(Dense(10,activation=\"relu\",W_regularizer=l2(reg)))\n",
    "model.add(Dense(10,activation=\"relu\",W_regularizer=l2(reg)))\n",
    "model.add(Dense(10,activation=\"relu\",W_regularizer=l2(reg)))\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adagrad', loss='mean_squared_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "52397/52397 [==============================] - 0s - loss: 0.0012     \n",
      "Epoch 2/100\n",
      "52397/52397 [==============================] - 0s - loss: 8.7392e-04     \n",
      "Epoch 3/100\n",
      "52397/52397 [==============================] - 0s - loss: 8.1624e-04     \n",
      "Epoch 4/100\n",
      "52397/52397 [==============================] - 0s - loss: 8.0237e-04     \n",
      "Epoch 5/100\n",
      "52397/52397 [==============================] - 0s - loss: 7.8455e-04     \n",
      "Epoch 6/100\n",
      "52397/52397 [==============================] - 0s - loss: 7.6285e-04     \n",
      "Epoch 7/100\n",
      "52397/52397 [==============================] - 0s - loss: 7.5285e-04     \n",
      "Epoch 8/100\n",
      "52397/52397 [==============================] - 0s - loss: 7.4226e-04     \n",
      "Epoch 9/100\n",
      "52397/52397 [==============================] - 0s - loss: 7.2958e-04     \n",
      "Epoch 10/100\n",
      "52397/52397 [==============================] - 0s - loss: 7.1638e-04     \n",
      "Epoch 11/100\n",
      "52397/52397 [==============================] - 0s - loss: 7.0561e-04     \n",
      "Epoch 12/100\n",
      "52397/52397 [==============================] - 0s - loss: 7.0125e-04     \n",
      "Epoch 13/100\n",
      "52397/52397 [==============================] - 0s - loss: 7.0066e-04     \n",
      "Epoch 14/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.8770e-04     \n",
      "Epoch 15/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.8021e-04     \n",
      "Epoch 16/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.9363e-04     \n",
      "Epoch 17/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.7140e-04     \n",
      "Epoch 18/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.6396e-04     \n",
      "Epoch 19/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.5543e-04     \n",
      "Epoch 20/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.5355e-04     \n",
      "Epoch 21/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.4777e-04     \n",
      "Epoch 22/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.4258e-04     \n",
      "Epoch 23/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.5436e-04     \n",
      "Epoch 24/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.2869e-04     \n",
      "Epoch 25/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.3631e-04     \n",
      "Epoch 26/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.2833e-04     \n",
      "Epoch 27/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.3975e-04     \n",
      "Epoch 28/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.2115e-04     \n",
      "Epoch 29/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.2330e-04     \n",
      "Epoch 30/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.2715e-04     \n",
      "Epoch 31/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.1366e-04     \n",
      "Epoch 32/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.1495e-04     \n",
      "Epoch 33/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.1057e-04     \n",
      "Epoch 34/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.1931e-04     \n",
      "Epoch 35/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.0405e-04     \n",
      "Epoch 36/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.1044e-04     \n",
      "Epoch 37/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.0617e-04     \n",
      "Epoch 38/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.0606e-04     \n",
      "Epoch 39/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.9897e-04     \n",
      "Epoch 40/100\n",
      "52397/52397 [==============================] - 0s - loss: 6.0198e-04     \n",
      "Epoch 41/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.9965e-04     \n",
      "Epoch 42/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.9345e-04     \n",
      "Epoch 43/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.8485e-04     \n",
      "Epoch 44/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.9389e-04     \n",
      "Epoch 45/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.8160e-04     \n",
      "Epoch 46/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.8647e-04     \n",
      "Epoch 47/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.8192e-04     \n",
      "Epoch 48/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.8488e-04     \n",
      "Epoch 49/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.7300e-04     \n",
      "Epoch 50/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.7559e-04     \n",
      "Epoch 51/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.7358e-04     \n",
      "Epoch 52/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.7715e-04     \n",
      "Epoch 53/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.7175e-04     \n",
      "Epoch 54/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.7375e-04     \n",
      "Epoch 55/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.7058e-04     \n",
      "Epoch 56/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.7069e-04     \n",
      "Epoch 57/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.7222e-04     \n",
      "Epoch 58/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6374e-04     \n",
      "Epoch 59/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6460e-04     \n",
      "Epoch 60/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6284e-04     \n",
      "Epoch 61/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6099e-04     \n",
      "Epoch 62/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6692e-04     \n",
      "Epoch 63/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6571e-04     \n",
      "Epoch 64/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.5789e-04     \n",
      "Epoch 65/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6044e-04     \n",
      "Epoch 66/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.5016e-04     \n",
      "Epoch 67/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.5963e-04     \n",
      "Epoch 68/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.5605e-04     \n",
      "Epoch 69/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4948e-04     \n",
      "Epoch 70/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6158e-04     \n",
      "Epoch 71/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6464e-04     \n",
      "Epoch 72/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.3718e-04     \n",
      "Epoch 73/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6057e-04     \n",
      "Epoch 74/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.5612e-04     \n",
      "Epoch 75/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4205e-04     \n",
      "Epoch 76/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4891e-04     \n",
      "Epoch 77/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4236e-04     \n",
      "Epoch 78/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.3784e-04     \n",
      "Epoch 79/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4834e-04     \n",
      "Epoch 80/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4735e-04     \n",
      "Epoch 81/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.6024e-04     \n",
      "Epoch 82/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.5348e-04     \n",
      "Epoch 83/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4324e-04     \n",
      "Epoch 84/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4717e-04     \n",
      "Epoch 85/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4046e-04     \n",
      "Epoch 86/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4041e-04     \n",
      "Epoch 87/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.3472e-04     \n",
      "Epoch 88/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.3959e-04     \n",
      "Epoch 89/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.3386e-04     \n",
      "Epoch 90/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.4003e-04     \n",
      "Epoch 91/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.3854e-04     \n",
      "Epoch 92/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.3712e-04     \n",
      "Epoch 93/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.2642e-04     \n",
      "Epoch 94/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.2990e-04     \n",
      "Epoch 95/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.2961e-04     \n",
      "Epoch 96/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.3201e-04     \n",
      "Epoch 97/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.2893e-04     \n",
      "Epoch 98/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.2603e-04     \n",
      "Epoch 99/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.3573e-04     \n",
      "Epoch 100/100\n",
      "52397/52397 [==============================] - 0s - loss: 5.2314e-04     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x15bd5e390>"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train, y_train, nb_epoch=100, batch_size=1000,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.42548711177836751"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = model.predict(x_train)\n",
    "e = y_train - y_train_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_train)\n",
    "r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21659633764895159"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred = model.predict(x_test)\n",
    "e = y_test - y_test_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_test)\n",
    "r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "reg = 1e-5\n",
    "# model.add(Dropout(p=0.4,input_shape=(x_train.shape[1],),))\n",
    "model.add(Dense(input_dim=x_train.shape[1],output_dim=10,activation=\"relu\"))\n",
    "model.add(Dense(10,activation=\"relu\"))\n",
    "model.add(Dense(1))\n",
    "model.compile(optimizer='adagrad', loss='mean_squared_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 20.7 s, sys: 3.28 s, total: 24 s\n",
      "Wall time: 12.1 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x15da02b38>"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "model.fit(x_train, y_train, nb_epoch=40, batch_size=1000,verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.39323777229045509"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_pred = model.predict(x_train)\n",
    "e = y_train - y_train_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_train)\n",
    "r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21421963437184965"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred = model.predict(x_test)\n",
    "e = y_test - y_test_pred\n",
    "r2 = 1 - np.var(e)/np.var(y_test)\n",
    "r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "latex_envs": {
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 0
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}