60-oxford-autoencoder-2-30-01.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers import Dense, Activation, BatchNormalization, Dropout, Input\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "import keras\n",
    "from datetime import datetime\n",
    "from sklearn import preprocessing\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import plot_model\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.utils import shuffle\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load processed dataset\n",
    "The data we load here are already shuffled and normalized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train: (136, 22)\n",
      "y_train: (136, 2)\n",
      "x_test: (59, 22)\n",
      "y_test: (59, 2)\n"
     ]
    }
   ],
   "source": [
    "data_directory = 'dataset/8. oxford-normalized-30/'\n",
    "x_train = pickle.load(open(data_directory + \"x_train.p\", \"rb\"))\n",
    "y_train = pickle.load(open(data_directory + \"y_train.p\", \"rb\"))\n",
    "x_test = pickle.load(open(data_directory + \"x_test.p\", \"rb\"))\n",
    "y_test = pickle.load(open(data_directory + \"y_test.p\", \"rb\"))\n",
    "print('x_train: ' + str(x_train.shape))\n",
    "print('y_train: ' + str(y_train.shape))\n",
    "print('x_test: ' + str(x_test.shape))\n",
    "print('y_test: ' + str(y_test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load autoencoder 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 22)                0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 12)                276       \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 22)                286       \n",
      "=================================================================\n",
      "Total params: 562\n",
      "Trainable params: 562\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "autoencoder_path = 'results/59-oxford-autoencoder-1-30-01/model/2018-03-15-19-13-06/weights.hdf5'\n",
    "weights = load_model(autoencoder_path)\n",
    "weights.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Copy autoencoder's weights except the last layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(136, 12)\n",
      "(59, 12)\n"
     ]
    }
   ],
   "source": [
    "x = Input(shape=(x_train.shape[1],))\n",
    "y = Dense(units = 12, activation='relu')(x)\n",
    "\n",
    "auto_1 = Model(x, y)\n",
    "\n",
    "for i in range(2):\n",
    "    auto_1.layers[i].set_weights(weights.layers[i].get_weights())\n",
    "    \n",
    "x_train = auto_1.predict(x_train)\n",
    "x_test = auto_1.predict(x_test)\n",
    "print(x_train.shape)\n",
    "print(x_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tunable parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 10000\n",
    "batch_size = 20\n",
    "learning_rate = 0.0001\n",
    "decay = 0.0001\n",
    "loss = keras.losses.categorical_crossentropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Autoencoder #2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "root_directory = 'results/60-oxford-autoencoder-2-30-01/'\n",
    "model_directory = root_directory + \"model/\"\n",
    "tensorboard_directory = root_directory + \"tensorboard/\"\n",
    "def create_model(layer_size, dropout):\n",
    "    global date\n",
    "    x = Input(shape=(x_train.shape[1],))\n",
    "\n",
    "    y = Dense(units = layer_size, activation='relu')(x)\n",
    "    y = Dropout(dropout)(y)\n",
    "\n",
    "    y = Dense(units = x_train.shape[1], activation='sigmoid')(y)\n",
    "    model = Model(x, y)\n",
    "    \n",
    "    # Create directory\n",
    "    directory = model_directory + date + '/'\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    \n",
    "    # Write model hyper-parameters\n",
    "    df = pd.DataFrame({\n",
    "        'Hidden layer units': layer_size,\n",
    "        'Dropout': dropout,\n",
    "        'Learning Rate': learning_rate,\n",
    "        'Decay': decay,\n",
    "        'Epochs': epochs,\n",
    "        'Batch Size': batch_size,\n",
    "        'Loss': loss,\n",
    "        'Test size': 30\n",
    "    }, index=[0])\n",
    "    df.to_csv(model_directory + date + '/params.csv')\n",
    "    \n",
    "    # Write model summary\n",
    "    file2 = open(directory + \"summary.txt\", \"a\")\n",
    "    model.summary(print_fn=lambda line: file2.write(line + '\\n'))\n",
    "    file2.close()\n",
    "\n",
    "    # Write model diagram\n",
    "    plot_model(model, to_file=directory + 'model.png', show_shapes=True, show_layer_names=False)\n",
    "    \n",
    "    # Compile the model\n",
    "    model.compile(optimizer=keras.optimizers.Adam(lr=learning_rate, decay=decay), loss=loss, metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KerasClassifierTensorBoard(KerasClassifier):\n",
    "    def fit(self, x, y, **kwargs):\n",
    "        global date\n",
    "        date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "        tensorboard_callback = keras.callbacks.TensorBoard(log_dir=tensorboard_directory + date + '/')\n",
    "        csv_logger = keras.callbacks.CSVLogger(model_directory + date + '/epochs.csv')\n",
    "        model_checkpoint = keras.callbacks.ModelCheckpoint(model_directory + date + '/weights.hdf5', save_best_only=True)\n",
    "        callbacks = [tensorboard_callback, csv_logger, model_checkpoint]\n",
    "        return super(KerasClassifierTensorBoard, self).fit(x, y, callbacks=callbacks, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KerasClassifierTensorBoard(build_fn=create_model, epochs=epochs, batch_size=batch_size)\n",
    "\n",
    "layers = [10, 8, 7, 6]\n",
    "dropouts = [0, 0.4]\n",
    "param_grid = dict(\n",
    "    layer_size=layers, \n",
    "    dropout=dropouts\n",
    ")\n",
    "\n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, error_score=0, verbose=2, cv=[(slice(None), slice(None))], n_jobs=1, fit_params=dict(validation_data=(x_test, x_test)))\n",
    "grid_result = grid.fit(x_train, x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}