ssl_stacked_M1_M2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Picking GPU 0\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "from __future__ import unicode_literals\n",
    "from __future__ import division\n",
    "from __future__ import absolute_import\n",
    "from __future__ import print_function\n",
    "\n",
    "import notebook_util\n",
    "notebook_util.setup_one_gpu()\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import random\n",
    "import logpdf\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "random.seed(31415)\n",
    "np.random.seed(31415)\n",
    "tf.set_random_seed(31415)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow.contrib.layers as tcl\n",
    "class SSL:\n",
    "    def __init__(self, x_dim = 50):\n",
    "        self.name = 'SSL'\n",
    "        \n",
    "        # classifier params\n",
    "        self.hidden_size = 500\n",
    "        self.num_labels = 10\n",
    "        \n",
    "        # encode & decoder params\n",
    "        self.z_dim = 50\n",
    "        self.x_dim = x_dim\n",
    "        self.batch_size   = 500\n",
    "        self.dataset_size = 50000\n",
    "        \n",
    "        # training\n",
    "        self.lr_decay_factor = 0.95\n",
    "        self.learning_rate = 1e-3\n",
    "        \n",
    "        with tf.variable_scope(self.name):\n",
    "            self._build_annealing()\n",
    "            self._build_graph()\n",
    "            self._build_train_op()\n",
    "        self.check_parameters()\n",
    "    \n",
    "    def global_vars(self):\n",
    "        var_list = [var for var in tf.global_variables() if self.name in var.name]\n",
    "        return var_list\n",
    "    \n",
    "    def trainable_vars(self):\n",
    "        var_list = [var for var in tf.trainable_variables() if self.name in var.name]\n",
    "        return var_list\n",
    "    \n",
    "    def check_parameters(self):\n",
    "        for var in tf.trainable_variables():\n",
    "            print('%s: %s' % (var.name, var.get_shape()))\n",
    "        print()\n",
    "    \n",
    "    def get_collection(self, collections):\n",
    "        return [var for var in tf.get_collection(collections)]\n",
    "    \n",
    "    def classify(self, x, reuse = False):\n",
    "        with tf.variable_scope('classifier', reuse = reuse):\n",
    "            h = tcl.fully_connected(x, self.hidden_size, activation_fn = tf.nn.softplus)\n",
    "            y = tcl.fully_connected(h, self.num_labels,  activation_fn = tf.nn.softplus)\n",
    "            return y\n",
    "    \n",
    "    def reparameterize(self, mu, logvar):\n",
    "        std = tf.exp(logvar * 0.5)\n",
    "        eps = tf.random_normal(tf.shape(mu))\n",
    "        z = mu + eps * std\n",
    "        return z\n",
    "    \n",
    "    def encode(self, x, y, reuse = False):\n",
    "        with tf.variable_scope('encoder', reuse = reuse):\n",
    "            concat = tf.concat([x, y], 1)\n",
    "            h = tcl.fully_connected(concat, self.hidden_size, activation_fn = tf.nn.softplus)\n",
    "            mu     = tcl.fully_connected(h, self.z_dim, activation_fn = None)\n",
    "            logvar = tcl.fully_connected(h, self.z_dim, activation_fn = None)\n",
    "            z = self.reparameterize(mu, logvar)\n",
    "            return z, mu, logvar\n",
    "    \n",
    "    def decode(self, z, y, reuse = False):\n",
    "        with tf.variable_scope('decoder', reuse = reuse):\n",
    "            concat = tf.concat([z, y], 1)\n",
    "            h = tcl.fully_connected(concat, self.hidden_size, activation_fn = tf.nn.softplus)\n",
    "            mu     = tcl.fully_connected(h, self.x_dim, activation_fn = None)\n",
    "            logvar = tcl.fully_connected(h, self.x_dim, activation_fn = None)\n",
    "            return mu, logvar\n",
    "        \n",
    "    def likelihood(self, x, mu_x, logvar_x, y, z, mu_z, logvar_z):\n",
    "        # uniform prior\n",
    "        prior_y = (1. / self.num_labels) * tf.ones([tf.shape(x)[0], 10], tf.float32)\n",
    "        logpy = - tf.nn.softmax_cross_entropy_with_logits(logits = prior_y, labels = y)\n",
    "        \n",
    "        kld = tf.reduce_sum(logpdf.KLD(mu_z, logvar_z), 1)\n",
    "        logpx = tf.reduce_sum(logpdf.gaussian(x, mu_x, logvar_x), 1)\n",
    "        likelihood = logpx + logpy - kld  \n",
    "        return likelihood\n",
    "    \n",
    "    def prior_likelihood(self):\n",
    "        likelihood = 0\n",
    "        vars = self.trainable_vars()\n",
    "        for var in vars:\n",
    "            likelihood += tf.reduce_sum(logpdf.std_gaussian(var))\n",
    "        return likelihood\n",
    "    \n",
    "    def _build_graph(self, reuse = False):\n",
    "        self.x_l = tf.placeholder(tf.float32, shape = (None, self.x_dim))\n",
    "        self.y_l = tf.placeholder(tf.int32, shape = (None, ))\n",
    "        self.x_u = tf.placeholder(tf.float32, shape = (None, self.x_dim))\n",
    "        \n",
    "        self.y_l_onehot = tcl.one_hot_encoding(self.y_l, num_classes = self.num_labels)\n",
    "                \n",
    "        '''\n",
    "            classifier, labelled\n",
    "        '''\n",
    "        scores_l = self.classify(self.x_l, reuse = reuse)\n",
    "        # loss of classifier\n",
    "        self.loss_clf = tf.nn.softmax_cross_entropy_with_logits(\\\n",
    "                        logits = scores_l, labels = self.y_l_onehot )\n",
    "\n",
    "        '''\n",
    "            labelled data, encoder & decoder\n",
    "        '''\n",
    "        z_l, mu_z_l, logvar_z_l = self.encode(self.x_l, self.y_l_onehot, reuse = reuse)\n",
    "        mu_x_l, logvar_x_l = self.decode(z_l, self.y_l_onehot, reuse = reuse)\n",
    "        \n",
    "        # loss of labelled data, refered as L(x, y)\n",
    "        likelihood_l = self.likelihood(self.x_l, mu_x_l, logvar_x_l, \\\n",
    "                                       self.y_l_onehot, z_l, mu_z_l, logvar_z_l)\n",
    "        \n",
    "        '''\n",
    "            unlabelled data, encoder & decoder\n",
    "        '''\n",
    "        for i in range(self.num_labels):\n",
    "            y_us = i*tf.ones([tf.shape(self.x_u)[0]], tf.int32)\n",
    "            y_us = tcl.one_hot_encoding(y_us, num_classes = self.num_labels)\n",
    "            \n",
    "            z_u, mu_u, logvar_u = self.encode(self.x_u, y_us, reuse = True)\n",
    "            mu_recon_u, logvar_recon_u = self.decode(z_u, y_us, reuse = True)\n",
    "            \n",
    "            _likelihood_u = self.likelihood(self.x_u, mu_recon_u, logvar_recon_u,\\\n",
    "                                y_us, z_u, mu_u, logvar_u)\n",
    "            _likelihood_u = tf.expand_dims(_likelihood_u, 1)\n",
    "            \n",
    "            if i == 0:\n",
    "                likelihood_u = tf.identity( _likelihood_u )\n",
    "            else:\n",
    "                likelihood_u = tf.concat([likelihood_u, _likelihood_u], 1)\n",
    "            \n",
    "        # with x & clf, give the dist over y\n",
    "        scores_u = self.classify(self.x_u, reuse = True)\n",
    "        y_u_prob = tf.nn.softmax(scores_u, dim=-1)\n",
    "        \n",
    "        # add the H(q(y|x))\n",
    "        likelihood_u = tf.multiply(y_u_prob, likelihood_u + -tf.log(y_u_prob)) \n",
    "        likelihood_u = tf.reduce_sum(likelihood_u, 1)\n",
    "\n",
    "        alpha = 0.1 * self.batch_size\n",
    "        self.loss_clf = tf.reduce_sum(self.loss_clf, 0)\n",
    "        self.loss_l = - tf.reduce_sum(likelihood_l, 0)\n",
    "        self.loss_u = - tf.reduce_sum(likelihood_u, 0)\n",
    "        self.loss = (self.loss_l + alpha* self.loss_clf + self.loss_u)/self.batch_size\n",
    "\n",
    "        print('loss_u  : '+str(self.loss_u.shape))\n",
    "        print('loss_l  : '+str(self.loss_l.shape))\n",
    "        print('loss_clf: '+str(self.loss_clf.shape))\n",
    "        \n",
    "        prior_weight = 1./(self.dataset_size) \n",
    "        self.loss_prior = - self.prior_likelihood()\n",
    "        self.loss += prior_weight * self.loss_prior\n",
    "        \n",
    "        self.pred_y = tf.argmax(scores_u, 1)\n",
    "    \n",
    "    def _build_train_op(self):\n",
    "        self.global_step = tf.Variable(0, name=\"global_step\", trainable = False)\n",
    "        self.lr = tf.Variable(self.learning_rate, trainable=False, \n",
    "                    dtype=tf.float32)\n",
    "        optimizer = tf.train.AdamOptimizer(self.lr)\n",
    "        grads_and_vars = optimizer.compute_gradients(self.loss)\n",
    "        def ClipIfNotNone(grad):\n",
    "            if grad is None:\n",
    "                return grad\n",
    "            return tf.clip_by_value(grad, -1, 1)\n",
    "        capped_gvs = [(ClipIfNotNone(grad), var) for grad, var in grads_and_vars]\n",
    "        self.train_op = optimizer.apply_gradients(capped_gvs, self.global_step)\n",
    "        self.lr_decay_op = self.lr.assign(\n",
    "                self.lr * self.lr_decay_factor)\n",
    "    \n",
    "    def lr_decay(self, sess):\n",
    "        _ = sess.run([self.lr_decay_op])\n",
    "        \n",
    "    def _build_annealing(self):\n",
    "        self.kld_weight = tf.Variable(float(0.0), trainable=False, \n",
    "                                    dtype=tf.float32)\n",
    "        kld_anneal_factor = 0.95\n",
    "        self.anneal_decay_op = self.kld_weight.assign(\n",
    "                self.kld_weight * kld_anneal_factor)\n",
    "\n",
    "    def kld_anneal(self, sess):\n",
    "        _ = sess.run([self.anneal_decay_op])\n",
    "    \n",
    "    def predict(self, x, sess):\n",
    "        feed_dict = {\n",
    "            self.x_u: x,\n",
    "        }\n",
    "        pred = sess.run([self.pred_y], feed_dict = feed_dict)[0]\n",
    "        return pred\n",
    "    \n",
    "    def optimize(self, sess, x_l, y_l, x_u):\n",
    "        feed_dict = {\n",
    "            self.x_l: x_l,\n",
    "            self.y_l: y_l,\n",
    "\n",
    "            self.x_u: x_u,\n",
    "        }\n",
    "        eval_train = [self.train_op, self.loss]\n",
    "        eval_loss  = [self.loss_clf, self.loss_l, self.loss_u, self.loss_prior]\n",
    "        eval_vars = eval_train + eval_loss\n",
    "        _, loss, loss_clf, loss_l, loss_u, loss_p = sess.run(eval_vars, feed_dict = feed_dict)\n",
    "        return loss, loss_clf, loss_l, loss_u, loss_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VAE/encoder/fully_connected/weights:0: (784, 600)\n",
      "VAE/encoder/fully_connected/biases:0: (600,)\n",
      "VAE/encoder/fully_connected_1/weights:0: (600, 600)\n",
      "VAE/encoder/fully_connected_1/biases:0: (600,)\n",
      "VAE/encoder/fully_connected_2/weights:0: (600, 50)\n",
      "VAE/encoder/fully_connected_2/biases:0: (50,)\n",
      "VAE/encoder/fully_connected_3/weights:0: (600, 50)\n",
      "VAE/encoder/fully_connected_3/biases:0: (50,)\n",
      "VAE/decoder/fully_connected/weights:0: (50, 600)\n",
      "VAE/decoder/fully_connected/biases:0: (600,)\n",
      "VAE/decoder/fully_connected_1/weights:0: (600, 600)\n",
      "VAE/decoder/fully_connected_1/biases:0: (600,)\n",
      "VAE/decoder/fully_connected_2/weights:0: (600, 784)\n",
      "VAE/decoder/fully_connected_2/biases:0: (784,)\n",
      "\n",
      "loss_u  : ()\n",
      "loss_l  : ()\n",
      "loss_clf: ()\n",
      "SSL/classifier/fully_connected/weights:0: (8, 500)\n",
      "SSL/classifier/fully_connected/biases:0: (500,)\n",
      "SSL/classifier/fully_connected_1/weights:0: (500, 10)\n",
      "SSL/classifier/fully_connected_1/biases:0: (10,)\n",
      "SSL/encoder/fully_connected/weights:0: (18, 500)\n",
      "SSL/encoder/fully_connected/biases:0: (500,)\n",
      "SSL/encoder/fully_connected_1/weights:0: (500, 50)\n",
      "SSL/encoder/fully_connected_1/biases:0: (50,)\n",
      "SSL/encoder/fully_connected_2/weights:0: (500, 50)\n",
      "SSL/encoder/fully_connected_2/biases:0: (50,)\n",
      "SSL/decoder/fully_connected/weights:0: (60, 500)\n",
      "SSL/decoder/fully_connected/biases:0: (500,)\n",
      "SSL/decoder/fully_connected_1/weights:0: (500, 8)\n",
      "SSL/decoder/fully_connected_1/biases:0: (8,)\n",
      "SSL/decoder/fully_connected_2/weights:0: (500, 8)\n",
      "SSL/decoder/fully_connected_2/biases:0: (8,)\n",
      "\n",
      "time: 0 m 4 s\n",
      "epoch: 0\n",
      "  labelled loss: 14.25\n",
      "unlabelled loss: 5726.38\n",
      "classifier loss: 2.16\n",
      "     prior loss: 99092.91\n",
      "     total loss: 13.68\n",
      " valid accuracy: 0.446\n",
      "  test accuracy: 0.445\n",
      "\n",
      "time: 0 m 39 s\n",
      "epoch: 10\n",
      "  labelled loss: 9.72\n",
      "unlabelled loss: 5680.91\n",
      "classifier loss: 0.79\n",
      "     prior loss: 99166.34\n",
      "     total loss: 13.44\n",
      " valid accuracy: 0.849\n",
      "  test accuracy: 0.843\n",
      "\n",
      "time: 1 m 13 s\n",
      "epoch: 20\n",
      "  labelled loss: 8.75\n",
      "unlabelled loss: 5658.08\n",
      "classifier loss: 0.07\n",
      "     prior loss: 99191.83\n",
      "     total loss: 13.32\n",
      " valid accuracy: 0.879\n",
      "  test accuracy: 0.873\n",
      "\n",
      "time: 1 m 48 s\n",
      "epoch: 30\n",
      "  labelled loss: 9.54\n",
      "unlabelled loss: 5548.80\n",
      "classifier loss: 0.03\n",
      "     prior loss: 99203.27\n",
      "     total loss: 13.10\n",
      " valid accuracy: 0.883\n",
      "  test accuracy: 0.878\n",
      "\n",
      "time: 2 m 23 s\n",
      "epoch: 40\n",
      "  labelled loss: 11.99\n",
      "unlabelled loss: 5661.44\n",
      "classifier loss: 0.06\n",
      "     prior loss: 99218.05\n",
      "     total loss: 13.34\n",
      " valid accuracy: 0.884\n",
      "  test accuracy: 0.881\n",
      "\n",
      "time: 2 m 58 s\n",
      "epoch: 50\n",
      "  labelled loss: 12.61\n",
      "unlabelled loss: 5610.22\n",
      "classifier loss: 0.05\n",
      "     prior loss: 99237.84\n",
      "     total loss: 13.24\n",
      " valid accuracy: 0.893\n",
      "  test accuracy: 0.884\n",
      "\n",
      "time: 3 m 33 s\n",
      "epoch: 60\n",
      "  labelled loss: 9.69\n",
      "unlabelled loss: 5677.47\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99258.94\n",
      "     total loss: 13.36\n",
      " valid accuracy: 0.898\n",
      "  test accuracy: 0.891\n",
      "\n",
      "time: 4 m 8 s\n",
      "epoch: 70\n",
      "  labelled loss: 12.28\n",
      "unlabelled loss: 5564.34\n",
      "classifier loss: 0.06\n",
      "     prior loss: 99281.54\n",
      "     total loss: 13.14\n",
      " valid accuracy: 0.905\n",
      "  test accuracy: 0.896\n",
      "\n",
      "time: 4 m 43 s\n",
      "epoch: 80\n",
      "  labelled loss: 11.42\n",
      "unlabelled loss: 5507.36\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99304.60\n",
      "     total loss: 13.02\n",
      " valid accuracy: 0.905\n",
      "  test accuracy: 0.898\n",
      "\n",
      "time: 5 m 18 s\n",
      "epoch: 90\n",
      "  labelled loss: 8.34\n",
      "unlabelled loss: 5529.04\n",
      "classifier loss: 0.04\n",
      "     prior loss: 99327.88\n",
      "     total loss: 13.07\n",
      " valid accuracy: 0.912\n",
      "  test accuracy: 0.905\n",
      "\n",
      "time: 5 m 53 s\n",
      "epoch: 100\n",
      "  labelled loss: 11.62\n",
      "unlabelled loss: 5547.00\n",
      "classifier loss: 0.03\n",
      "     prior loss: 99350.75\n",
      "     total loss: 13.11\n",
      " valid accuracy: 0.914\n",
      "  test accuracy: 0.907\n",
      "\n",
      "time: 6 m 28 s\n",
      "epoch: 110\n",
      "  labelled loss: 9.04\n",
      "unlabelled loss: 5535.62\n",
      "classifier loss: 0.12\n",
      "     prior loss: 99373.09\n",
      "     total loss: 13.09\n",
      " valid accuracy: 0.915\n",
      "  test accuracy: 0.909\n",
      "\n",
      "time: 7 m 3 s\n",
      "epoch: 120\n",
      "  labelled loss: 12.18\n",
      "unlabelled loss: 5490.02\n",
      "classifier loss: 0.05\n",
      "     prior loss: 99393.28\n",
      "     total loss: 13.00\n",
      " valid accuracy: 0.916\n",
      "  test accuracy: 0.911\n",
      "\n",
      "time: 7 m 38 s\n",
      "epoch: 130\n",
      "  labelled loss: 12.42\n",
      "unlabelled loss: 5566.96\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99411.50\n",
      "     total loss: 13.15\n",
      " valid accuracy: 0.918\n",
      "  test accuracy: 0.912\n",
      "\n",
      "time: 8 m 13 s\n",
      "epoch: 140\n",
      "  labelled loss: 11.27\n",
      "unlabelled loss: 5625.75\n",
      "classifier loss: 0.07\n",
      "     prior loss: 99428.58\n",
      "     total loss: 13.27\n",
      " valid accuracy: 0.919\n",
      "  test accuracy: 0.914\n",
      "\n",
      "time: 8 m 48 s\n",
      "epoch: 150\n",
      "  labelled loss: 10.58\n",
      "unlabelled loss: 5607.02\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99442.93\n",
      "     total loss: 13.23\n",
      " valid accuracy: 0.920\n",
      "  test accuracy: 0.915\n",
      "\n",
      "time: 9 m 23 s\n",
      "epoch: 160\n",
      "  labelled loss: 9.63\n",
      "unlabelled loss: 5457.35\n",
      "classifier loss: 0.06\n",
      "     prior loss: 99455.80\n",
      "     total loss: 12.93\n",
      " valid accuracy: 0.922\n",
      "  test accuracy: 0.919\n",
      "\n",
      "time: 9 m 58 s\n",
      "epoch: 170\n",
      "  labelled loss: 12.07\n",
      "unlabelled loss: 5580.75\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99467.69\n",
      "     total loss: 13.18\n",
      " valid accuracy: 0.924\n",
      "  test accuracy: 0.918\n",
      "\n",
      "time: 10 m 33 s\n",
      "epoch: 180\n",
      "  labelled loss: 9.60\n",
      "unlabelled loss: 5596.14\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99477.95\n",
      "     total loss: 13.20\n",
      " valid accuracy: 0.924\n",
      "  test accuracy: 0.918\n",
      "\n",
      "time: 11 m 8 s\n",
      "epoch: 190\n",
      "  labelled loss: 9.88\n",
      "unlabelled loss: 5579.90\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99486.61\n",
      "     total loss: 13.17\n",
      " valid accuracy: 0.924\n",
      "  test accuracy: 0.918\n",
      "\n",
      "time: 11 m 43 s\n",
      "epoch: 200\n",
      "  labelled loss: 13.34\n",
      "unlabelled loss: 5613.80\n",
      "classifier loss: 0.05\n",
      "     prior loss: 99494.79\n",
      "     total loss: 13.25\n",
      " valid accuracy: 0.925\n",
      "  test accuracy: 0.919\n",
      "\n",
      "time: 12 m 18 s\n",
      "epoch: 210\n",
      "  labelled loss: 8.05\n",
      "unlabelled loss: 5542.00\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99502.39\n",
      "     total loss: 13.09\n",
      " valid accuracy: 0.926\n",
      "  test accuracy: 0.921\n",
      "\n",
      "time: 12 m 53 s\n",
      "epoch: 220\n",
      "  labelled loss: 9.28\n",
      "unlabelled loss: 5595.12\n",
      "classifier loss: 0.04\n",
      "     prior loss: 99509.35\n",
      "     total loss: 13.20\n",
      " valid accuracy: 0.925\n",
      "  test accuracy: 0.921\n",
      "\n",
      "time: 13 m 28 s\n",
      "epoch: 230\n",
      "  labelled loss: 10.12\n",
      "unlabelled loss: 5555.82\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99514.41\n",
      "     total loss: 13.12\n",
      " valid accuracy: 0.926\n",
      "  test accuracy: 0.919\n",
      "\n",
      "time: 14 m 3 s\n",
      "epoch: 240\n",
      "  labelled loss: 11.08\n",
      "unlabelled loss: 5479.02\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99520.33\n",
      "     total loss: 12.97\n",
      " valid accuracy: 0.928\n",
      "  test accuracy: 0.921\n",
      "\n",
      "time: 14 m 38 s\n",
      "epoch: 250\n",
      "  labelled loss: 12.41\n",
      "unlabelled loss: 5456.30\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99525.71\n",
      "     total loss: 12.93\n",
      " valid accuracy: 0.926\n",
      "  test accuracy: 0.921\n",
      "\n",
      "time: 15 m 13 s\n",
      "epoch: 260\n",
      "  labelled loss: 12.53\n",
      "unlabelled loss: 5546.49\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99531.23\n",
      "     total loss: 13.11\n",
      " valid accuracy: 0.928\n",
      "  test accuracy: 0.921\n",
      "\n",
      "time: 15 m 48 s\n",
      "epoch: 270\n",
      "  labelled loss: 13.09\n",
      "unlabelled loss: 5636.31\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99536.70\n",
      "     total loss: 13.29\n",
      " valid accuracy: 0.927\n",
      "  test accuracy: 0.922\n",
      "\n",
      "time: 16 m 23 s\n",
      "epoch: 280\n",
      "  labelled loss: 10.81\n",
      "unlabelled loss: 5557.62\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99541.76\n",
      "     total loss: 13.13\n",
      " valid accuracy: 0.928\n",
      "  test accuracy: 0.922\n",
      "\n",
      "time: 16 m 58 s\n",
      "epoch: 290\n",
      "  labelled loss: 10.96\n",
      "unlabelled loss: 5497.80\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99546.44\n",
      "     total loss: 13.01\n",
      " valid accuracy: 0.927\n",
      "  test accuracy: 0.921\n",
      "\n",
      "time: 17 m 33 s\n",
      "epoch: 300\n",
      "  labelled loss: 9.03\n",
      "unlabelled loss: 5572.58\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99551.50\n",
      "     total loss: 13.15\n",
      " valid accuracy: 0.927\n",
      "  test accuracy: 0.922\n",
      "\n",
      "time: 18 m 9 s\n",
      "epoch: 310\n",
      "  labelled loss: 8.41\n",
      "unlabelled loss: 5493.39\n",
      "classifier loss: 0.10\n",
      "     prior loss: 99556.34\n",
      "     total loss: 13.00\n",
      " valid accuracy: 0.929\n",
      "  test accuracy: 0.921\n",
      "\n",
      "time: 18 m 44 s\n",
      "epoch: 320\n",
      "  labelled loss: 11.51\n",
      "unlabelled loss: 5586.64\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99560.95\n",
      "     total loss: 13.19\n",
      " valid accuracy: 0.928\n",
      "  test accuracy: 0.922\n",
      "\n",
      "time: 19 m 19 s\n",
      "epoch: 330\n",
      "  labelled loss: 10.74\n",
      "unlabelled loss: 5503.99\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99565.46\n",
      "     total loss: 13.02\n",
      " valid accuracy: 0.927\n",
      "  test accuracy: 0.922\n",
      "\n",
      "time: 19 m 54 s\n",
      "epoch: 340\n",
      "  labelled loss: 9.78\n",
      "unlabelled loss: 5521.74\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99570.00\n",
      "     total loss: 13.05\n",
      " valid accuracy: 0.929\n",
      "  test accuracy: 0.922\n",
      "\n",
      "time: 20 m 29 s\n",
      "epoch: 350\n",
      "  labelled loss: 9.53\n",
      "unlabelled loss: 5519.24\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99574.89\n",
      "     total loss: 13.05\n",
      " valid accuracy: 0.929\n",
      "  test accuracy: 0.922\n",
      "\n",
      "time: 21 m 5 s\n",
      "epoch: 360\n",
      "  labelled loss: 13.39\n",
      "unlabelled loss: 5410.78\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99579.77\n",
      "     total loss: 12.84\n",
      " valid accuracy: 0.927\n",
      "  test accuracy: 0.922\n",
      "\n",
      "time: 21 m 40 s\n",
      "epoch: 370\n",
      "  labelled loss: 9.26\n",
      "unlabelled loss: 5562.02\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99584.12\n",
      "     total loss: 13.14\n",
      " valid accuracy: 0.929\n",
      "  test accuracy: 0.924\n",
      "\n",
      "time: 22 m 15 s\n",
      "epoch: 380\n",
      "  labelled loss: 7.13\n",
      "unlabelled loss: 5547.62\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99588.48\n",
      "     total loss: 13.10\n",
      " valid accuracy: 0.929\n",
      "  test accuracy: 0.924\n",
      "\n",
      "time: 22 m 50 s\n",
      "epoch: 390\n",
      "  labelled loss: 9.27\n",
      "unlabelled loss: 5498.85\n",
      "classifier loss: 0.05\n",
      "     prior loss: 99593.27\n",
      "     total loss: 13.01\n",
      " valid accuracy: 0.930\n",
      "  test accuracy: 0.924\n",
      "\n",
      "time: 23 m 26 s\n",
      "epoch: 400\n",
      "  labelled loss: 15.62\n",
      "unlabelled loss: 5613.29\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99597.46\n",
      "     total loss: 13.25\n",
      " valid accuracy: 0.930\n",
      "  test accuracy: 0.924\n",
      "\n",
      "time: 24 m 1 s\n",
      "epoch: 410\n",
      "  labelled loss: 17.28\n",
      "unlabelled loss: 5526.20\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99601.88\n",
      "     total loss: 13.08\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.924\n",
      "\n",
      "time: 24 m 36 s\n",
      "epoch: 420\n",
      "  labelled loss: 8.03\n",
      "unlabelled loss: 5459.42\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99605.82\n",
      "     total loss: 12.93\n",
      " valid accuracy: 0.930\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 25 m 11 s\n",
      "epoch: 430\n",
      "  labelled loss: 10.76\n",
      "unlabelled loss: 5496.24\n",
      "classifier loss: 0.03\n",
      "     prior loss: 99610.59\n",
      "     total loss: 13.01\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 25 m 47 s\n",
      "epoch: 440\n",
      "  labelled loss: 10.58\n",
      "unlabelled loss: 5464.92\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99614.76\n",
      "     total loss: 12.94\n",
      " valid accuracy: 0.930\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 26 m 23 s\n",
      "epoch: 450\n",
      "  labelled loss: 11.70\n",
      "unlabelled loss: 5573.72\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99618.80\n",
      "     total loss: 13.16\n",
      " valid accuracy: 0.929\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 26 m 58 s\n",
      "epoch: 460\n",
      "  labelled loss: 10.11\n",
      "unlabelled loss: 5547.12\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99622.91\n",
      "     total loss: 13.11\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.924\n",
      "\n",
      "time: 27 m 33 s\n",
      "epoch: 470\n",
      "  labelled loss: 9.78\n",
      "unlabelled loss: 5458.82\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99627.46\n",
      "     total loss: 12.93\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.924\n",
      "\n",
      "time: 28 m 8 s\n",
      "epoch: 480\n",
      "  labelled loss: 12.19\n",
      "unlabelled loss: 5531.95\n",
      "classifier loss: 0.05\n",
      "     prior loss: 99631.80\n",
      "     total loss: 13.09\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 28 m 43 s\n",
      "epoch: 490\n",
      "  labelled loss: 23.29\n",
      "unlabelled loss: 5545.62\n",
      "classifier loss: 0.21\n",
      "     prior loss: 99635.70\n",
      "     total loss: 13.15\n",
      " valid accuracy: 0.930\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 29 m 18 s\n",
      "epoch: 500\n",
      "  labelled loss: 8.37\n",
      "unlabelled loss: 5544.17\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99639.46\n",
      "     total loss: 13.10\n",
      " valid accuracy: 0.930\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 29 m 54 s\n",
      "epoch: 510\n",
      "  labelled loss: 9.40\n",
      "unlabelled loss: 5541.61\n",
      "classifier loss: 0.07\n",
      "     prior loss: 99643.61\n",
      "     total loss: 13.10\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.926\n",
      "\n",
      "time: 30 m 29 s\n",
      "epoch: 520\n",
      "  labelled loss: 13.87\n",
      "unlabelled loss: 5578.08\n",
      "classifier loss: 0.03\n",
      "     prior loss: 99647.54\n",
      "     total loss: 13.18\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 31 m 5 s\n",
      "epoch: 530\n",
      "  labelled loss: 9.42\n",
      "unlabelled loss: 5497.53\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99651.13\n",
      "     total loss: 13.01\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 31 m 40 s\n",
      "epoch: 540\n",
      "  labelled loss: 8.62\n",
      "unlabelled loss: 5466.03\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99655.12\n",
      "     total loss: 12.94\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.926\n",
      "\n",
      "time: 32 m 16 s\n",
      "epoch: 550\n",
      "  labelled loss: 11.10\n",
      "unlabelled loss: 5475.03\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99658.98\n",
      "     total loss: 12.97\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.926\n",
      "\n",
      "time: 32 m 51 s\n",
      "epoch: 560\n",
      "  labelled loss: 9.22\n",
      "unlabelled loss: 5513.41\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99662.97\n",
      "     total loss: 13.04\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 33 m 26 s\n",
      "epoch: 570\n",
      "  labelled loss: 9.91\n",
      "unlabelled loss: 5605.87\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99666.54\n",
      "     total loss: 13.23\n",
      " valid accuracy: 0.929\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 34 m 1 s\n",
      "epoch: 580\n",
      "  labelled loss: 13.43\n",
      "unlabelled loss: 5536.23\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99669.94\n",
      "     total loss: 13.09\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.926\n",
      "\n",
      "time: 34 m 36 s\n",
      "epoch: 590\n",
      "  labelled loss: 11.68\n",
      "unlabelled loss: 5551.68\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99673.49\n",
      "     total loss: 13.12\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.926\n",
      "\n",
      "time: 35 m 11 s\n",
      "epoch: 600\n",
      "  labelled loss: 9.91\n",
      "unlabelled loss: 5468.13\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99677.05\n",
      "     total loss: 12.95\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 35 m 46 s\n",
      "epoch: 610\n",
      "  labelled loss: 10.91\n",
      "unlabelled loss: 5554.36\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99680.55\n",
      "     total loss: 13.12\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.926\n",
      "\n",
      "time: 36 m 21 s\n",
      "epoch: 620\n",
      "  labelled loss: 9.86\n",
      "unlabelled loss: 5480.40\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99683.84\n",
      "     total loss: 12.97\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.925\n",
      "\n",
      "time: 36 m 56 s\n",
      "epoch: 630\n",
      "  labelled loss: 12.65\n",
      "unlabelled loss: 5490.03\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99687.16\n",
      "     total loss: 13.00\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 37 m 31 s\n",
      "epoch: 640\n",
      "  labelled loss: 8.97\n",
      "unlabelled loss: 5524.39\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99690.69\n",
      "     total loss: 13.06\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.926\n",
      "\n",
      "time: 38 m 6 s\n",
      "epoch: 650\n",
      "  labelled loss: 8.69\n",
      "unlabelled loss: 5445.98\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99693.59\n",
      "     total loss: 12.90\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 38 m 41 s\n",
      "epoch: 660\n",
      "  labelled loss: 9.58\n",
      "unlabelled loss: 5487.04\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99697.24\n",
      "     total loss: 12.99\n",
      " valid accuracy: 0.931\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 39 m 16 s\n",
      "epoch: 670\n",
      "  labelled loss: 8.04\n",
      "unlabelled loss: 5365.22\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99700.15\n",
      "     total loss: 12.74\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.926\n",
      "\n",
      "time: 39 m 51 s\n",
      "epoch: 680\n",
      "  labelled loss: 8.00\n",
      "unlabelled loss: 5458.96\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99703.20\n",
      "     total loss: 12.93\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 40 m 26 s\n",
      "epoch: 690\n",
      "  labelled loss: 10.98\n",
      "unlabelled loss: 5496.21\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99706.29\n",
      "     total loss: 13.01\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 41 m 1 s\n",
      "epoch: 700\n",
      "  labelled loss: 10.70\n",
      "unlabelled loss: 5506.50\n",
      "classifier loss: 0.09\n",
      "     prior loss: 99709.19\n",
      "     total loss: 13.04\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 41 m 36 s\n",
      "epoch: 710\n",
      "  labelled loss: 14.74\n",
      "unlabelled loss: 5426.56\n",
      "classifier loss: 0.03\n",
      "     prior loss: 99712.65\n",
      "     total loss: 12.88\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 42 m 11 s\n",
      "epoch: 720\n",
      "  labelled loss: 10.61\n",
      "unlabelled loss: 5557.11\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99715.47\n",
      "     total loss: 13.13\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 42 m 46 s\n",
      "epoch: 730\n",
      "  labelled loss: 9.59\n",
      "unlabelled loss: 5567.11\n",
      "classifier loss: 0.05\n",
      "     prior loss: 99718.51\n",
      "     total loss: 13.15\n",
      " valid accuracy: 0.930\n",
      "  test accuracy: 0.926\n",
      "\n",
      "time: 43 m 21 s\n",
      "epoch: 740\n",
      "  labelled loss: 8.63\n",
      "unlabelled loss: 5506.14\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99721.34\n",
      "     total loss: 13.02\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.929\n",
      "\n",
      "time: 43 m 56 s\n",
      "epoch: 750\n",
      "  labelled loss: 13.70\n",
      "unlabelled loss: 5508.01\n",
      "classifier loss: 0.10\n",
      "     prior loss: 99724.11\n",
      "     total loss: 13.05\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 44 m 31 s\n",
      "epoch: 760\n",
      "  labelled loss: 10.96\n",
      "unlabelled loss: 5569.17\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99726.53\n",
      "     total loss: 13.15\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 45 m 6 s\n",
      "epoch: 770\n",
      "  labelled loss: 10.10\n",
      "unlabelled loss: 5473.88\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99729.43\n",
      "     total loss: 12.96\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 45 m 41 s\n",
      "epoch: 780\n",
      "  labelled loss: 8.73\n",
      "unlabelled loss: 5505.33\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99732.31\n",
      "     total loss: 13.02\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 46 m 16 s\n",
      "epoch: 790\n",
      "  labelled loss: 11.48\n",
      "unlabelled loss: 5485.58\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99734.86\n",
      "     total loss: 12.99\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 46 m 51 s\n",
      "epoch: 800\n",
      "  labelled loss: 12.21\n",
      "unlabelled loss: 5541.09\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99737.50\n",
      "     total loss: 13.10\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 47 m 26 s\n",
      "epoch: 810\n",
      "  labelled loss: 10.20\n",
      "unlabelled loss: 5440.64\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99740.72\n",
      "     total loss: 12.90\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 48 m 1 s\n",
      "epoch: 820\n",
      "  labelled loss: 14.59\n",
      "unlabelled loss: 5472.21\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99743.73\n",
      "     total loss: 12.97\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 48 m 36 s\n",
      "epoch: 830\n",
      "  labelled loss: 11.72\n",
      "unlabelled loss: 5537.84\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99746.60\n",
      "     total loss: 13.09\n",
      " valid accuracy: 0.935\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 49 m 11 s\n",
      "epoch: 840\n",
      "  labelled loss: 9.03\n",
      "unlabelled loss: 5565.09\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99749.29\n",
      "     total loss: 13.14\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 49 m 46 s\n",
      "epoch: 850\n",
      "  labelled loss: 12.66\n",
      "unlabelled loss: 5557.44\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99752.08\n",
      "     total loss: 13.14\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 50 m 21 s\n",
      "epoch: 860\n",
      "  labelled loss: 13.34\n",
      "unlabelled loss: 5440.29\n",
      "classifier loss: 0.04\n",
      "     prior loss: 99755.00\n",
      "     total loss: 12.91\n",
      " valid accuracy: 0.935\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 50 m 56 s\n",
      "epoch: 870\n",
      "  labelled loss: 13.71\n",
      "unlabelled loss: 5453.15\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99757.41\n",
      "     total loss: 12.93\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 51 m 30 s\n",
      "epoch: 880\n",
      "  labelled loss: 14.75\n",
      "unlabelled loss: 5526.34\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99759.96\n",
      "     total loss: 13.08\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 52 m 5 s\n",
      "epoch: 890\n",
      "  labelled loss: 12.12\n",
      "unlabelled loss: 5517.29\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99762.80\n",
      "     total loss: 13.06\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 52 m 40 s\n",
      "epoch: 900\n",
      "  labelled loss: 15.01\n",
      "unlabelled loss: 5478.35\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99765.55\n",
      "     total loss: 12.98\n",
      " valid accuracy: 0.935\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 53 m 15 s\n",
      "epoch: 910\n",
      "  labelled loss: 12.47\n",
      "unlabelled loss: 5417.35\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99768.25\n",
      "     total loss: 12.86\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 53 m 50 s\n",
      "epoch: 920\n",
      "  labelled loss: 15.19\n",
      "unlabelled loss: 5554.46\n",
      "classifier loss: 0.04\n",
      "     prior loss: 99770.70\n",
      "     total loss: 13.14\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 54 m 25 s\n",
      "epoch: 930\n",
      "  labelled loss: 11.70\n",
      "unlabelled loss: 5562.97\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99773.45\n",
      "     total loss: 13.14\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 55 m 0 s\n",
      "epoch: 940\n",
      "  labelled loss: 11.73\n",
      "unlabelled loss: 5551.93\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99776.38\n",
      "     total loss: 13.12\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 55 m 35 s\n",
      "epoch: 950\n",
      "  labelled loss: 10.56\n",
      "unlabelled loss: 5433.90\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99778.95\n",
      "     total loss: 12.89\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 56 m 9 s\n",
      "epoch: 960\n",
      "  labelled loss: 10.50\n",
      "unlabelled loss: 5537.09\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99781.45\n",
      "     total loss: 13.09\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.929\n",
      "\n",
      "time: 56 m 44 s\n",
      "epoch: 970\n",
      "  labelled loss: 12.70\n",
      "unlabelled loss: 5478.94\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99784.13\n",
      "     total loss: 12.98\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 57 m 18 s\n",
      "epoch: 980\n",
      "  labelled loss: 11.82\n",
      "unlabelled loss: 5399.49\n",
      "classifier loss: 0.04\n",
      "     prior loss: 99786.91\n",
      "     total loss: 12.82\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 57 m 53 s\n",
      "epoch: 990\n",
      "  labelled loss: 12.78\n",
      "unlabelled loss: 5511.17\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99789.87\n",
      "     total loss: 13.04\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 58 m 28 s\n",
      "epoch: 1000\n",
      "  labelled loss: 7.89\n",
      "unlabelled loss: 5536.62\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99792.09\n",
      "     total loss: 13.09\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 59 m 2 s\n",
      "epoch: 1010\n",
      "  labelled loss: 9.32\n",
      "unlabelled loss: 5490.49\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99795.27\n",
      "     total loss: 13.00\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 59 m 37 s\n",
      "epoch: 1020\n",
      "  labelled loss: 8.17\n",
      "unlabelled loss: 5488.97\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99798.06\n",
      "     total loss: 12.99\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 60 m 11 s\n",
      "epoch: 1030\n",
      "  labelled loss: 15.53\n",
      "unlabelled loss: 5491.15\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99800.84\n",
      "     total loss: 13.01\n",
      " valid accuracy: 0.935\n",
      "  test accuracy: 0.929\n",
      "\n",
      "time: 60 m 46 s\n",
      "epoch: 1040\n",
      "  labelled loss: 9.47\n",
      "unlabelled loss: 5483.80\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99803.70\n",
      "     total loss: 12.98\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 61 m 21 s\n",
      "epoch: 1050\n",
      "  labelled loss: 10.60\n",
      "unlabelled loss: 5534.35\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99806.20\n",
      "     total loss: 13.09\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 61 m 56 s\n",
      "epoch: 1060\n",
      "  labelled loss: 11.70\n",
      "unlabelled loss: 5468.20\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99808.76\n",
      "     total loss: 12.96\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 62 m 30 s\n",
      "epoch: 1070\n",
      "  labelled loss: 8.98\n",
      "unlabelled loss: 5450.22\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99811.45\n",
      "     total loss: 12.91\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.930\n",
      "\n",
      "time: 63 m 5 s\n",
      "epoch: 1080\n",
      "  labelled loss: 10.32\n",
      "unlabelled loss: 5535.20\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99814.29\n",
      "     total loss: 13.09\n",
      " valid accuracy: 0.935\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 63 m 40 s\n",
      "epoch: 1090\n",
      "  labelled loss: 9.53\n",
      "unlabelled loss: 5533.42\n",
      "classifier loss: 0.01\n",
      "     prior loss: 99816.86\n",
      "     total loss: 13.08\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 64 m 14 s\n",
      "epoch: 1100\n",
      "  labelled loss: 16.66\n",
      "unlabelled loss: 5563.32\n",
      "classifier loss: 0.16\n",
      "     prior loss: 99819.45\n",
      "     total loss: 13.17\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.926\n",
      "\n",
      "time: 64 m 48 s\n",
      "epoch: 1110\n",
      "  labelled loss: 11.52\n",
      "unlabelled loss: 5510.17\n",
      "classifier loss: 0.00\n",
      "     prior loss: 99822.59\n",
      "     total loss: 13.04\n",
      " valid accuracy: 0.935\n",
      "  test accuracy: 0.928\n",
      "\n",
      "time: 65 m 23 s\n",
      "epoch: 1120\n",
      "  labelled loss: 10.07\n",
      "unlabelled loss: 5471.03\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99825.40\n",
      "     total loss: 12.96\n",
      " valid accuracy: 0.935\n",
      "  test accuracy: 0.930\n",
      "\n",
      "time: 65 m 57 s\n",
      "epoch: 1130\n",
      "  labelled loss: 8.63\n",
      "unlabelled loss: 5563.67\n",
      "classifier loss: 0.03\n",
      "     prior loss: 99828.16\n",
      "     total loss: 13.14\n",
      " valid accuracy: 0.933\n",
      "  test accuracy: 0.927\n",
      "\n",
      "time: 66 m 32 s\n",
      "epoch: 1140\n",
      "  labelled loss: 8.14\n",
      "unlabelled loss: 5432.82\n",
      "classifier loss: 0.04\n",
      "     prior loss: 99831.04\n",
      "     total loss: 12.88\n",
      " valid accuracy: 0.934\n",
      "  test accuracy: 0.929\n",
      "\n",
      "time: 67 m 7 s\n",
      "epoch: 1150\n",
      "  labelled loss: 9.54\n",
      "unlabelled loss: 5535.34\n",
      "classifier loss: 0.02\n",
      "     prior loss: 99833.30\n",
      "     total loss: 13.09\n",
      " valid accuracy: 0.932\n",
      "  test accuracy: 0.927\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from utils import (load_mnist_split, create_semisupervised, \\\n",
    "                   ssl_batch_gen, time_since, get_dims, sample)\n",
    "from vae import VAE\n",
    "\n",
    "'''\n",
    "    here are the config params for the experiment.\n",
    "'''\n",
    "data_size = 50000\n",
    "batch_size = 500\n",
    "n_labelled = 100\n",
    "n_epoch = 1000\n",
    "\n",
    "# load data from mnist\n",
    "train_x, train_y, valid_x, valid_y, test_x, test_y = load_mnist_split()\n",
    "# split training set\n",
    "X_labelled, Y_labelled, X_unlabelled, Y_unlabelled = create_semisupervised(\\\n",
    "                                                        train_x, train_y, n_labelled)\n",
    "\n",
    "np.random.seed(31415)\n",
    "tf.set_random_seed(31415)\n",
    "\n",
    "# Set config for tensorflow session.\n",
    "tf_config = tf.ConfigProto(\n",
    "    device_count = {'GPU': 1}, # single gpu\n",
    ")\n",
    "tf_config.gpu_options.allow_growth=True\n",
    "with tf.Session(config = tf_config) as sess:\n",
    "    model = VAE()\n",
    "    model.saver.restore(sess, tf.train.latest_checkpoint('models/vae/'))\n",
    "\n",
    "    mu_labelled,   logvar_labelled   = model.get_repr(sess, X_labelled, stats_only = True)\n",
    "    mu_unlabelled, logvar_unlabelled = model.get_repr(sess, X_unlabelled, stats_only = True)\n",
    "    mu_valid,      logvar_valid      = model.get_repr(sess, valid_x, stats_only = True)\n",
    "    mu_test,       logvar_test       = model.get_repr(sess, test_x, stats_only = True)\n",
    "    \n",
    "min_std = 0.1\n",
    "selected_dims = np.std(mu_unlabelled, axis = 0) > min_std\n",
    "\n",
    "def sample_dims(mu, logvar, selected_dims):\n",
    "    mu     = get_dims(mu, selected_dims)\n",
    "    logvar = get_dims(logvar, selected_dims)\n",
    "    return sample(mu, logvar)\n",
    "\n",
    "data_x_l = sample_dims(mu_labelled, logvar_labelled, selected_dims)\n",
    "data_x_u = sample_dims(mu_unlabelled, logvar_unlabelled, selected_dims)\n",
    "valid_x  = sample_dims(mu_valid, logvar_valid, selected_dims)\n",
    "test_x   = sample_dims(mu_test, logvar_test, selected_dims)\n",
    "\n",
    "data_y_l = Y_labelled\n",
    "\n",
    "x_dim = data_x_u.shape[1]\n",
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "\n",
    "    with tf.Session(config = tf_config) as sess:\n",
    "        model = SSL(x_dim = x_dim)\n",
    "        var_list = model.global_vars()\n",
    "        init_op = tf.variables_initializer(var_list)\n",
    "        sess.run(init_op)\n",
    "\n",
    "        decay_cnt = 0\n",
    "        start = time.time()\n",
    "        for epoch in range(n_epoch):\n",
    "            l_batch_gen, u_batch_gen = ssl_batch_gen(data_x_l, data_y_l, data_x_u, 500, 1)\n",
    "\n",
    "            for l_batch, u_batch in zip(l_batch_gen, u_batch_gen):\n",
    "                x_l, y_l = zip(*l_batch)\n",
    "                x_u = zip(*u_batch)[0]\n",
    "                loss, loss_clf, loss_l, loss_u, loss_p = model.optimize(sess, x_l, y_l, x_u)\n",
    "            \n",
    "            if epoch % 10 == 0:\n",
    "                pred_valid = model.predict(valid_x, sess)\n",
    "                pred_test  = model.predict(test_x,  sess)\n",
    "                accuracy_valid = accuracy_score(valid_y, pred_valid)\n",
    "                accuracy_test  = accuracy_score(test_y,  pred_test)\n",
    "\n",
    "                print('time: %s' % time_since(start))\n",
    "                print('epoch: %d' % epoch)\n",
    "                print('  labelled loss: %.2f' % loss_l)\n",
    "                print('unlabelled loss: %.2f' % loss_u)\n",
    "                print('classifier loss: %.2f' % loss_clf)\n",
    "                print('     prior loss: %.2f' % loss_p)\n",
    "                print('     total loss: %.2f' % loss)\n",
    "                print(' valid accuracy: %.3f' % accuracy_valid)\n",
    "                print('  test accuracy: %.3f' % accuracy_test)\n",
    "                print()\n",
    "            \n",
    "            if epoch % 10 == 0 and decay_cnt < 23:\n",
    "                model.lr_decay(sess)\n",
    "                decay_cnt += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}