08-grid-search-v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Activation, BatchNormalization, Dropout, Input\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "import keras\n",
    "from datetime import datetime\n",
    "from sklearn import preprocessing\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import plot_model\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_dataframe = pd.read_csv('dataset/1. istanbul/train_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize some training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_dataframe.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_x = train_dataframe.drop(['Subject ID', 'Class information', 'UPDRS'], axis=1)\n",
    "train_x = train_x.as_matrix()\n",
    "print(train_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare training labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_y = train_dataframe['Class information']\n",
    "train_y = train_y.as_matrix()\n",
    "print(train_y.shape)\n",
    "train_y = np_utils.to_categorical(train_y)\n",
    "print(train_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_dataframe = pd.read_csv('dataset/1. istanbul/test_data.csv')\n",
    "test_x = test_dataframe.drop(['Subject ID', 'Class information'], axis=1)\n",
    "test_x = test_x.as_matrix()\n",
    "print(test_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_y = test_dataframe['Class information']\n",
    "test_y = test_y.as_matrix()\n",
    "print(test_y.shape)\n",
    "test_y = np_utils.to_categorical(test_y)\n",
    "print(test_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine train and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x = np.vstack([train_x, test_x])\n",
    "train_y = np.vstack([train_y, test_y])\n",
    "print(train_x.shape)\n",
    "print(train_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mins = np.min(train_x, axis=0)\n",
    "maxs = np.max(train_x, axis=0)\n",
    "rng = maxs - mins\n",
    "train_x = 1.0 - (((1.0 - 0.0) * (maxs - train_x)) / rng)\n",
    "test_x = 1.0 - (((1.0 - 0.0) * (maxs - test_x)) / rng)\n",
    "pd.DataFrame(train_x).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shuffle data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, train_y = shuffle(train_x, train_y)\n",
    "pd.DataFrame(train_y).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "root_directory = 'results/results-gridsearch-02/'\n",
    "model_directory = root_directory + \"model/\"\n",
    "tensorboard_directory = root_directory + \"tensorboard/\"\n",
    "validation_split = 0.172;\n",
    "def create_model(layers, optimizer):\n",
    "    global date\n",
    "    x = Input(shape=(train_x.shape[1],))\n",
    "\n",
    "    y = Dense(units = layers[0], activation='relu')(x)\n",
    "    y = Dropout(0.5)(y)\n",
    "\n",
    "    y = Dense(units = layers[1], activation='relu')(y)\n",
    "    y = Dropout(0.5)(y)\n",
    "    \n",
    "    y = Dense(units = layers[2], activation='relu')(y)\n",
    "    y = Dropout(0.5)(y)\n",
    "    \n",
    "    if (layers[3] > 0):\n",
    "        y = Dense(units = layers[3], activation='relu')(y)\n",
    "        y = Dropout(0.5)(y)\n",
    "\n",
    "    y = Dense(units = train_y.shape[1], activation='softmax')(y)\n",
    "    model = Model(x, y)\n",
    "    \n",
    "    # Create directory\n",
    "    directory = model_directory + date + '/'\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    \n",
    "    # Write model hyper-parameters\n",
    "    file = open(directory + \"params.txt\", \"a\")\n",
    "    file.write(\"optimizer: %s, layer 1: %d, layer 2: %d, layer 3: %d, layer 4: %d\" % (optimizer[0], layers[0], layers[1], layers[2], layers[3]))\n",
    "    file.close()\n",
    "    \n",
    "    # Write model summary\n",
    "    file2 = open(directory + \"summary.txt\", \"a\")\n",
    "    model.summary(print_fn=lambda x: file2.write(x + '\\n'))\n",
    "    file2.close()\n",
    "\n",
    "    # Write model diagram\n",
    "    plot_model(model, to_file=directory + 'model.png', show_shapes=True, show_layer_names=False)\n",
    "    \n",
    "    # Compile the model\n",
    "    model.compile(optimizer=optimizer[1], loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KerasClassifierTensorBoard(KerasClassifier):\n",
    "    def fit(self, x, y, **kwargs):\n",
    "        global date\n",
    "        date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "        tensorboard_callback = keras.callbacks.TensorBoard(log_dir=tensorboard_directory + date + '/')\n",
    "        csv_logger = keras.callbacks.CSVLogger(model_directory + date + '/epochs.csv')\n",
    "        callbacks = [tensorboard_callback, csv_logger]\n",
    "        return super(KerasClassifierTensorBoard, self).fit(x, y, callbacks=callbacks, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid search hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KerasClassifierTensorBoard(build_fn=create_model, epochs=6000, batch_size=20)\n",
    "\n",
    "layers = [\n",
    "    [4, 8, 16, 0],\n",
    "    [4, 32, 16, 0],\n",
    "    [4, 64, 8, 0],\n",
    "    [4, 64, 8, 6],\n",
    "    [4, 64, 32, 0],\n",
    "    [6, 6, 32, 0],\n",
    "    [6, 6, 32, 64],\n",
    "    [6, 6, 32, 16],\n",
    "    [6, 6, 32, 8],\n",
    "    [6, 6, 64, 0]\n",
    "]\n",
    "optimizers = [\n",
    "    ['Adam(lr=0.0001, decay=0)', keras.optimizers.Adam(lr=0.0001, decay=0)],\n",
    "    ['Adam(lr=0.0001, decay=0.00001)', keras.optimizers.Adam(lr=0.0001, decay=0.00001)],\n",
    "    ['Adam(lr=0.0001, decay=0.000001)', keras.optimizers.Adam(lr=0.0001, decay=0.000001)],\n",
    "    ['Adam(lr=0.00001, decay=0.000001)', keras.optimizers.Adam(lr=0.00001, decay=0.000001)],\n",
    "    ['RMSprop(lr=0.0001, decay=0)', keras.optimizers.RMSprop(lr=0.0001, decay=0)],\n",
    "    ['RMSprop(lr=0.0001, decay=0.00001)', keras.optimizers.RMSprop(lr=0.0001, decay=0.00001)],\n",
    "    ['RMSprop(lr=0.0001, decay=0.000001)', keras.optimizers.RMSprop(lr=0.0001, decay=0.000001)],\n",
    "    ['RMSprop(lr=0.00001, decay=0.000001)', keras.optimizers.RMSprop(lr=0.00001, decay=0.000001)]\n",
    "]\n",
    "param_grid = dict(\n",
    "    layers=layers, \n",
    "    optimizer = optimizers\n",
    ")\n",
    "\n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, error_score=0, verbose=2, cv=[(slice(None), slice(None))], n_jobs=1, fit_params=dict(validation_split=validation_split))\n",
    "grid_result = grid.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Print results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"Mean %f, Std %f with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The End"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}