Neural_Nets_Project_Starter_Code-2-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "\n",
    "import skimage.io\n",
    "import skimage.viewer\n",
    "from skimage.io import imread\n",
    "from skimage.transform import rescale, resize\n",
    "\n",
    "from scipy import ndimage as ndi\n",
    "from skimage import feature\n",
    "\n",
    "from skimage.transform import rotate, AffineTransform\n",
    "from skimage import transform\n",
    "\n",
    "from skimage.util import random_noise\n",
    "import random\n",
    "\n",
    "import datetime\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "cnn = 0\n",
    "img_nums = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "id": "zkUK4KDkJMLp"
   },
   "outputs": [],
   "source": [
    "np.random.seed(30)\n",
    "import random as rn\n",
    "rn.seed(30)\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "np.random.seed(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xA110drUXUFo",
    "outputId": "b7e18ce4-3f13-487b-b0e6-71434c858550"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.4.1\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GOrFz83aJMLp",
    "outputId": "e5fe1da0-e5d0-4352-cee4-27b8bd4a3bc6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size = 90, \n",
      "Train length = 100, \n",
      "Validation length = 100\n",
      "Training Length = 663, Validation Length = 100\n"
     ]
    }
   ],
   "source": [
    "# 0: Left Swipe, 1: Right Swipe, 2: Stop, 3: Thumbs Down, 4: Thumbs Up\n",
    "train_doc = np.random.permutation(open('C:\\\\Users\\\\Vrishank\\\\Downloads\\\\Hackathon\\\\Project_data\\\\train.csv').readlines())\n",
    "val_doc = np.random.permutation(open('C:\\\\Users\\\\Vrishank\\\\Downloads\\\\Hackathon\\\\Project_data\\\\val.csv').readlines())\n",
    "train_len = len(train_doc)\n",
    "val_len = len(val_doc)\n",
    "train_dirs = []\n",
    "val_dirs = []\n",
    "for i in range(train_doc.shape[0]):\n",
    "    train_dir = train_doc[i].split(';')[0]\n",
    "    if(i < 100):\n",
    "        val_dir = val_doc[i].split(';')[0]\n",
    "        val_dirs.append(val_dir)\n",
    "        train_dirs.append(train_dir)\n",
    "batch_size = 90                                        #experiment with the batch size\n",
    "print(f'Batch Size = {batch_size}, \\nTrain length = {len(train_dirs)}, \\nValidation length = {len(val_dirs)}')\n",
    "print(f'Training Length = {train_len}, Validation Length = {val_len}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generator(source_path, folder_list, batch_size, lenth, aug_aff = 0, aug_edg = 0, aug_nois = 0):\n",
    "    print( 'Source path = ', source_path, '; batch size =', batch_size)\n",
    "    \n",
    "    if(img_nums == 30):\n",
    "        img_idx = range(0, 30)                                                                     #create a list of image numbers you want to use for a particular video\n",
    "    else:\n",
    "        img_idx = range(0, 30, 2)\n",
    "    while True:\n",
    "        t = np.random.permutation(folder_list)\n",
    "        num_batches = lenth // batch_size                                                     # calculate the number of batches\n",
    "        num_imgs = lenth - num_batches*batch_size\n",
    "        for batch in range(num_batches):                                                          # we iterate over the number of batches\n",
    "            x = len(img_idx)\n",
    "            y = 96\n",
    "            z = 96\n",
    "            batch_data = np.zeros((batch_size,x,y,z,3))                                           # x is the number of images you use for each video, (y,z) is the final size of the input images and 3 is the number of channels RGB\n",
    "            batch_labels = np.zeros((batch_size,5))                                               # batch_labels is the one hot representation of the output\n",
    "            if(aug_aff):\n",
    "                aug_batch_data = np.zeros((batch_size,x,y,z,3))\n",
    "                aug_batch_labels = np.zeros((batch_size,5))\n",
    "            if(aug_nois):\n",
    "                nois_batch_data = np.zeros((batch_size,x,y,z,3))\n",
    "                nois_batch_labels = np.zeros((batch_size,5))\n",
    "            if(aug_edg):\n",
    "                edg_batch_data = np.zeros((batch_size,x,y,z,3))\n",
    "                edg_batch_labels = np.zeros((batch_size,5))\n",
    "\n",
    "\n",
    "\n",
    "            for folder in range(batch_size):                                                      # iterate over the batch_size\n",
    "                imgs = os.listdir(source_path+'/'+ t[folder + (batch*batch_size)].split(';')[0])  # read all the images in the folder\n",
    "                for idx,item in enumerate(img_idx):                                               #  Iterate over the frames/images of a folder to read them in\n",
    "                    image = imread(source_path+'/'+ t[folder + (batch*batch_size)].strip().split(';')[0]+'/'+imgs[item]).astype(np.float32)\n",
    "                    \n",
    "                    #crop the images and resize them. Note that the images are of 2 different shape \n",
    "                    #and the conv3D will throw error if the inputs in a batch have different shapes\n",
    "\n",
    "                    image = resize(image = image, output_shape=(y, z))\n",
    "\n",
    "                    if(aug_edg):\n",
    "                        img1 = image.astype('float64')\n",
    "                        img1[:, :, 0] += 0.2 * np.random.random(img1[:, :, 0].shape)\n",
    "                        img1[:, :, 1] += 0.2 * np.random.random(img1[:, :, 1].shape)\n",
    "                        img1[:, :, 2] += 0.2 * np.random.random(img1[:, :, 2].shape)\n",
    "                        edges10 = feature.canny(img1[:, :, 0])\n",
    "                        edges20 = feature.canny(img1[:, :, 0], sigma=3)\n",
    "                        edges11 = feature.canny(img1[:, :, 1])\n",
    "                        edges21 = feature.canny(img1[:, :, 1], sigma=3)\n",
    "                        edges12 = feature.canny(img1[:, :, 2])\n",
    "                        edges22 = feature.canny(img1[:, :, 2], sigma=3)\n",
    "                        edg_batch_data[folder, idx, :, :, 0] = edges20\n",
    "                        edg_batch_data[folder, idx, :, :, 1] = edges21\n",
    "                        edg_batch_data[folder, idx, :, :, 2] = edges22\n",
    "\n",
    "                    \n",
    "                    if(aug_nois):\n",
    "                        image_noise_aug = random_noise(image, var = 0.1)\n",
    "                        nois_batch_data[folder, idx, :, :, 0] = image_noise_aug[:, :, 0]\n",
    "                        nois_batch_data[folder, idx, :, :, 1] = image_noise_aug[:, :, 1]\n",
    "                        nois_batch_data[folder, idx, :, :, 2] = image_noise_aug[:, :, 2]\n",
    "\n",
    "\n",
    "                    image[:, :, 0] = (image [:, :, 0] - np.min(image[:, :, 0]))/(np.max(image[:, :, 0]) - np.min(image[:, :, 0]))\n",
    "                    image[:, :, 1] = (image [:, :, 1] - np.min(image[:, :, 1]))/(np.max(image[:, :, 1]) - np.min(image[:, :, 1]))\n",
    "                    image[:, :, 2] = (image [:, :, 2] - np.min(image[:, :, 2]))/(np.max(image[:, :, 2]) - np.min(image[:, :, 2]))\n",
    "\n",
    "                    if(aug_aff):\n",
    "                        shear_val = random.uniform(0.0, -0.5)\n",
    "                        tf = AffineTransform(shear = shear_val)\n",
    "                        img_aff = transform.warp(image, tf, order=1, preserve_range=True, mode='wrap')\n",
    "                        aug_batch_data[folder, idx, :, :, 0] = img_aff[:, :, 0]\n",
    "                        aug_batch_data[folder, idx, :, :, 1] = img_aff[:, :, 1]\n",
    "                        aug_batch_data[folder, idx, :, :, 2] = img_aff[:, :, 2]\n",
    "\n",
    "                    batch_data[folder,idx,:,:,0] = image[:, :, 0]       #normalise and feed in the image\n",
    "                    batch_data[folder,idx,:,:,1] = image[:, :, 1]       #normalise and feed in the image\n",
    "                    batch_data[folder,idx,:,:,2] = image[:, :, 2]       #normalise and feed in the image\n",
    "\n",
    "\n",
    "                    \n",
    "                batch_labels[folder, int(t[folder + (batch*batch_size)].strip().split(';')[2])] = 1\n",
    "                if aug_aff:\n",
    "                    aug_batch_labels[folder, int(t[folder + (batch*batch_size)].strip().split(';')[2])] = 1\n",
    "\n",
    "                if aug_nois:\n",
    "                    nois_batch_labels[folder, int(t[folder + (batch*batch_size)].strip().split(';')[2])] = 1\n",
    "                    \n",
    "                if(aug_edg):\n",
    "                    edg_batch_labels[folder, int(t[folder + (batch*batch_size)].strip().split(';')[2])] = 1\n",
    "\n",
    "            if aug_aff:\n",
    "                batch_data = np.append(batch_data, aug_batch_data, axis = 0)\n",
    "                batch_labels = np.append(batch_labels, aug_batch_labels, axis = 0) \n",
    "\n",
    "            if aug_nois:\n",
    "                batch_data = np.append(batch_data, nois_batch_data, axis = 0) \n",
    "                batch_labels = np.append(batch_labels, nois_batch_labels, axis = 0)\n",
    "                \n",
    "            if aug_edg:\n",
    "                batch_data = np.append(batch_data, edg_batch_data, axis = 0) \n",
    "                batch_labels = np.append(batch_labels, edg_batch_labels, axis = 0) \n",
    "\n",
    "\n",
    "\n",
    "            yield batch_data, batch_labels #you yield the batch_data and the batch_labels, remember what does yield do\n",
    "\n",
    "\n",
    "        # write the code for the remaining data points which are left after full batches\n",
    "        x = len(img_idx)\n",
    "        y = 96\n",
    "        z = 96\n",
    "        batch_data = np.zeros((num_imgs,x,y,z,3))                                               # x is the number of images you use for each video, (y,z) is the final size of the input images and 3 is the number of channels RGB\n",
    "        batch_labels = np.zeros((num_imgs,5))\n",
    "\n",
    "        if(aug_aff):\n",
    "            aug_batch_data = np.zeros((num_imgs,x,y,z,3))\n",
    "            aug_batch_labels = np.zeros((num_imgs,5))\n",
    "        if(aug_nois):\n",
    "            nois_batch_data = np.zeros((num_imgs,x,y,z,3))\n",
    "            nois_batch_labels = np.zeros((num_imgs,5))\n",
    "        if(aug_edg):\n",
    "            edg_batch_data = np.zeros((num_imgs,x,y,z,3))\n",
    "            edg_batch_labels = np.zeros((num_imgs,5))\n",
    "\n",
    "        for folder in range(num_imgs):\n",
    "          imgs = os.listdir(source_path+'/'+ t[folder + (num_batches*batch_size)].split(';')[0])\n",
    "          for idx, item in enumerate(img_idx):\n",
    "            image = imread(source_path+'/'+ t[folder + (num_batches*batch_size)].strip().split(';')[0]+'/'+imgs[item]).astype(np.float32)\n",
    "            #crop the images and resize them. Note that the images are of 2 different shape \n",
    "            #and the conv3D will throw error if the inputs in a batch have different shapes\n",
    "\n",
    "            image = resize(image = image, output_shape=(y, z))\n",
    "\n",
    "            if(aug_edg):\n",
    "                img1 = image.astype('float64')\n",
    "                img1[:, :, 1] += 0.2 * np.random.random(img1[:, :, 1].shape)\n",
    "                img1[:, :, 2] += 0.2 * np.random.random(img1[:, :, 2].shape)\n",
    "                edges10 = feature.canny(img1[:, :, 0])\n",
    "                edges20 = feature.canny(img1[:, :, 0], sigma=3)\n",
    "                edges11 = feature.canny(img1[:, :, 1])\n",
    "                edges21 = feature.canny(img1[:, :, 1], sigma=3)\n",
    "                edges12 = feature.canny(img1[:, :, 2])\n",
    "                edges22 = feature.canny(img1[:, :, 2], sigma=3)\n",
    "                edg_batch_data[folder, idx, :, :, 0] = edges20\n",
    "                edg_batch_data[folder, idx, :, :, 1] = edges21\n",
    "                edg_batch_data[folder, idx, :, :, 2] = edges22\n",
    "\n",
    "                    \n",
    "            if(aug_nois):\n",
    "                image_noise_aug = random_noise(image, var = 0.1)\n",
    "                nois_batch_data[folder, idx, :, :, 0] = image_noise_aug[:, :, 0]\n",
    "                nois_batch_data[folder, idx, :, :, 1] = image_noise_aug[:, :, 1]\n",
    "                nois_batch_data[folder, idx, :, :, 2] = image_noise_aug[:, :, 2]\n",
    "\n",
    "            image[:, :, 0] = (image [:, :, 0] - np.min(image[:, :, 0]))/(np.max(image[:, :, 0]) - np.min(image[:, :, 0]))\n",
    "            image[:, :, 1] = (image [:, :, 1] - np.min(image[:, :, 1]))/(np.max(image[:, :, 1]) - np.min(image[:, :, 1]))\n",
    "            image[:, :, 2] = (image [:, :, 2] - np.min(image[:, :, 2]))/(np.max(image[:, :, 2]) - np.min(image[:, :, 2]))\n",
    "\n",
    "            if(aug_aff):\n",
    "                shear_val = random.uniform(0.0, -0.5)\n",
    "                tf = AffineTransform(shear = shear_val)\n",
    "                img_aff = transform.warp(image, tf, order=1, preserve_range=True, mode='wrap')\n",
    "                aug_batch_data[folder, idx, :, :, 0] = img_aff[:, :, 0]\n",
    "                aug_batch_data[folder, idx, :, :, 1] = img_aff[:, :, 1]\n",
    "                aug_batch_data[folder, idx, :, :, 2] = img_aff[:, :, 2]\n",
    "                \n",
    "            batch_data[folder,idx,:,:,0] = image [:, :, 0]\n",
    "            batch_data[folder,idx,:,:,1] = image [:, :, 1]\n",
    "            batch_data[folder,idx,:,:,2] = image [:, :, 2]\n",
    "\n",
    "\n",
    "            batch_labels[folder, int(t[folder + (num_batches*batch_size)].strip().split(';')[2])] = 1\n",
    "\n",
    "            if aug_aff:\n",
    "                aug_batch_labels[folder, int(t[folder + (num_batches*batch_size)].strip().split(';')[2])] = 1\n",
    "\n",
    "            if aug_nois:\n",
    "                nois_batch_labels[folder, int(t[folder + (num_batches*batch_size)].strip().split(';')[2])] = 1\n",
    "\n",
    "            if aug_edg:\n",
    "                edg_batch_labels[folder, int(t[folder + (num_batches*batch_size)].strip().split(';')[2])] = 1\n",
    "\n",
    "\n",
    "                \n",
    "        if aug_aff:\n",
    "            batch_data = np.append(batch_data, aug_batch_data, axis = 0)\n",
    "            batch_labels = np.append(batch_labels, aug_batch_labels, axis = 0) \n",
    "\n",
    "        if aug_nois:\n",
    "            batch_data = np.append(batch_data, nois_batch_data, axis = 0)\n",
    "            batch_labels = np.append(batch_labels, nois_batch_labels, axis = 0) \n",
    "\n",
    "        if aug_edg:\n",
    "            batch_data = np.append(batch_data, edg_batch_data, axis = 0)\n",
    "            batch_labels = np.append(batch_labels, edg_batch_labels, axis = 0) \n",
    "\n",
    "        print(batch_data.shape, batch_labels.shape)     \n",
    "        yield batch_data, batch_labels                                                            #you yield the batch_data and the batch_labels, remember what does yield do"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "69MRL7veJMLr"
   },
   "source": [
    "Note here that a video is represented above in the generator as (number of images, height, width, number of channels). Take this into consideration while creating the model architecture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VCbWCgDvJMLr",
    "outputId": "62100853-6de0-47ed-8e63-2679cf19ae79"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# training sequences = 663\n",
      "# validation sequences = 100\n",
      "# epochs = 15\n"
     ]
    }
   ],
   "source": [
    "curr_dt_time = datetime.datetime.now()\n",
    "train_path = 'C:\\\\Users\\\\Vrishank\\\\Downloads\\\\Hackathon\\\\Project_data\\\\train'               # '/notebooks/storage/Final_data/Collated_training/train'\n",
    "val_path = 'C:\\\\Users\\\\Vrishank\\\\Downloads\\\\Hackathon\\\\Project_data\\\\val'                   # '/notebooks/storage/Final_data/Collated_training/val'\n",
    "num_train_sequences = len(train_doc)\n",
    "print('# training sequences =', num_train_sequences)\n",
    "num_val_sequences = len(val_doc)\n",
    "print('# validation sequences =', num_val_sequences)\n",
    "num_epochs = 15                                                          # choose the number of epochs\n",
    "print ('# epochs =', num_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xwdd2MmbJMLs"
   },
   "source": [
    "## Model\n",
    "Here you make the model using different functionalities that Keras provides. Remember to use `Conv3D` and `MaxPooling3D` and not `Conv2D` and `Maxpooling2D` for a 3D convolution model. You would want to use `TimeDistributed` while building a Conv2D + RNN model. Also remember that the last layer is the softmax. Design the network in such a way that the model is able to give good accuracy on the least number of parameters so that it can fit in the memory of the webcam."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "id": "Ah1o-bMNJMLs"
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, GRU, Flatten, TimeDistributed, Flatten, BatchNormalization, Activation, Dropout\n",
    "from keras.layers.convolutional import Conv3D, MaxPooling3D\n",
    "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n",
    "from keras import optimizers\n",
    "\n",
    "#write your model here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DYd3s_yNFs3r",
    "outputId": "a0b0e2f3-a2d1-4f55-8e01-a3d020cdb9e2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "mobilenetv2_1.00_96 (Functio (None, 3, 3, 1280)        2257984   \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 11520)             0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 64)                737344    \n",
      "=================================================================\n",
      "Total params: 2,995,328\n",
      "Trainable params: 737,344\n",
      "Non-trainable params: 2,257,984\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "if(cnn == 0):\n",
    "    input_shape = (96, 96, 3)\n",
    "\n",
    "    mobv2_model = keras.applications.MobileNetV2(input_shape=input_shape, weights = 'imagenet', include_top=False)\n",
    "\n",
    "    mobv2_model.trainable = False\n",
    "#     num_layers = len(mobv2_model.layers)     \n",
    "#     print(num_layers)\n",
    "    \n",
    "#     layer_split = 4*num_layers//5\n",
    "#     print(layer_split)\n",
    "#     for layer in range(layer_split):\n",
    "#         mobv2_model.layers[layer].trainable = False\n",
    "#     for layer in range(layer_split, num_layers):\n",
    "#         mobv2_model.layers[layer].trainable = True\n",
    "        \n",
    "    cnn_model = Sequential([\n",
    "        mobv2_model,\n",
    "        keras.layers.Flatten(),\n",
    "        keras.layers.Dense(64),\n",
    "#        keras.layers.BatchNormalization(),\n",
    "#        keras.layers.Dropout(0.25)\n",
    "    ])\n",
    "    cnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "id": "ccMxtOtcfYVL"
   },
   "outputs": [],
   "source": [
    "# from keras.layers import Lambda\n",
    "from keras import regularizers\n",
    "\n",
    "def rnn_gru(input_dims, n_classes, cnn_model):\n",
    "    model = Sequential()\n",
    "    model.add(TimeDistributed(cnn_model, input_shape = input_dims))\n",
    "#    model.add(GRU(64, input_shape = input_dims, return_sequences=False, kernel_regularizer = regularizers.l2(1e-02), dropout = 0.25))\n",
    "#    model.add(GRU(64, input_shape = input_dims, return_sequences=False, dropout = 0.25))\n",
    "    model.add(GRU(64, input_shape = input_dims, return_sequences=False))\n",
    "\n",
    "    model.add(Dense(64, activation = 'relu'))\n",
    "#    model.add(BatchNormalization())\n",
    "#    model.add(Dropout(0.25))\n",
    "    model.add(Dense(n_classes, activation='softmax'))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "id": "1sTqEPmdfm_4"
   },
   "outputs": [],
   "source": [
    "if(cnn == 0):\n",
    "    if(img_nums == 30):\n",
    "        input_dimens = (30, 96, 96, 3)\n",
    "    else:\n",
    "        input_dimens = (15, 96, 96, 3)\n",
    "    model = rnn_gru(input_dims = input_dimens, n_classes = 5, cnn_model = cnn_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WFI5DkWkJMLs"
   },
   "source": [
    "Now that you have written the model, the next step is to `compile` the model. When you print the `summary` of the model, you'll see the total number of parameters you have to train."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PFm4sKa_o-3O"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Y5OcVz_RJMLt",
    "outputId": "c4466d01-0aa5-4d14-d3cc-d1bedf316e1c",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "time_distributed_2 (TimeDist (None, 15, 64)            2995328   \n",
      "_________________________________________________________________\n",
      "gru_2 (GRU)                  (None, 64)                24960     \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 3,024,773\n",
      "Trainable params: 766,789\n",
      "Non-trainable params: 2,257,984\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "if(cnn == 0):\n",
    "    #optimiser = keras.optimizers.Adam(learning_rate = 0.01)                            #write your optimizer\n",
    "    #optimiser = 'Adam'\n",
    "    #model.compile(optimizer=keras.optimizers.Adam(lr = 0.1), loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "    #model.compile(optimizer=tf.train.AdamOptimizer(0.01), loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "    optimiser = 'Adam'\n",
    "    model.compile(optimizer=optimiser, loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "\n",
    "\n",
    "    print (model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CCAMysyeJMLt"
   },
   "source": [
    "Let us create the `train_generator` and the `val_generator` which will be used in `.fit_generator`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "id": "hssZ8JAxJMLt"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<generator object generator at 0x00000247A57F4F48>\n",
      "<generator object generator at 0x0000024756983F48>\n"
     ]
    }
   ],
   "source": [
    "if(cnn == 0):\n",
    "    aug_aff = 0\n",
    "    aug_edg = 0\n",
    "    aug_nois = 0\n",
    "    train_generator = generator(train_path, train_doc, batch_size, train_len, aug_aff = 1, aug_edg = 0, aug_nois = 1)\n",
    "    val_generator = generator(val_path, val_doc, batch_size, val_len, aug_aff = 0, aug_edg = 0, aug_nois = 0)\n",
    "    print(train_generator)\n",
    "    print(val_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TcGvP3wwJMLt",
    "outputId": "1916d56b-d52b-4354-c3e1-bd79787aa444"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`period` argument is deprecated. Please use `save_freq` to specify the frequency in number of batches seen.\n"
     ]
    }
   ],
   "source": [
    "if(cnn == 0):\n",
    "    model_name = 'model_init' + '_' + str(curr_dt_time).replace(' ','').replace(':','_') + '/'\n",
    "    \n",
    "    if not os.path.exists(model_name):\n",
    "        os.mkdir(model_name)\n",
    "\n",
    "    filepath = model_name + 'model-{epoch:05d}-{loss:.5f}-{categorical_accuracy:.5f}-{val_loss:.5f}-{val_categorical_accuracy:.5f}.h5'\n",
    "\n",
    "    checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=False, mode='auto', period=1)\n",
    "\n",
    "    LR = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, cooldown=1, verbose=1)     # write the REducelronplateau code here\n",
    "    callbacks_list = [checkpoint, LR]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wckZnt9SJMLu"
   },
   "source": [
    "The `steps_per_epoch` and `validation_steps` are used by `fit_generator` to decide the number of next() calls it need to make."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Qb1bJ_uLJMLv",
    "outputId": "ad215d76-8c81-493a-e38b-e20b31e1ada5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps per epoch = 8, validation_steps = 2\n"
     ]
    }
   ],
   "source": [
    "if (num_train_sequences%batch_size) == 0:\n",
    "    steps_per_epoch = int(num_train_sequences/batch_size)\n",
    "else:\n",
    "    steps_per_epoch = (num_train_sequences//batch_size) + 1\n",
    "\n",
    "if (num_val_sequences%batch_size) == 0:\n",
    "    validation_steps = int(num_val_sequences/batch_size)\n",
    "else:\n",
    "    validation_steps = (num_val_sequences//batch_size) + 1\n",
    "\n",
    "print(f'Steps per epoch = {steps_per_epoch}, validation_steps = {validation_steps}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "txflnsPtJMLv"
   },
   "source": [
    "Let us now fit the model. This will start training the model and with the help of the checkpoints, you'll be able to save the model at the end of each epoch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "j0rtIqZoJMLv",
    "outputId": "9799ee3c-7602-4dd8-e1fe-40b5ec281a0c",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\train ; batch size = 90\n",
      "Epoch 1/15\n",
      "7/8 [=========================>....] - ETA: 26s - loss: 1.5667 - categorical_accuracy: 0.2890(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 1.5558 - categorical_accuracy: 0.2977 Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\val ; batch size = 90\n",
      "(10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 224s 29s/step - loss: 1.5473 - categorical_accuracy: 0.3044 - val_loss: 1.1580 - val_categorical_accuracy: 0.6100\n",
      "\n",
      "Epoch 00001: saving model to model_init_2022-04-1400_29_49.369108\\model-00001-1.47961-0.35797-1.15804-0.61000.h5\n",
      "Epoch 2/15\n",
      "7/8 [=========================>....] - ETA: 25s - loss: 1.1257 - categorical_accuracy: 0.5899(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 1.1173 - categorical_accuracy: 0.5935 (10, 15, 96, 96, 3) (10, 5)\n",
      "(10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 189s 25s/step - loss: 1.1108 - categorical_accuracy: 0.5962 - val_loss: 0.7536 - val_categorical_accuracy: 0.7900\n",
      "\n",
      "Epoch 00002: saving model to model_init_2022-04-1400_29_49.369108\\model-00002-1.05864-0.61840-0.75358-0.79000.h5\n",
      "Epoch 3/15\n",
      "7/8 [=========================>....] - ETA: 26s - loss: 0.8314 - categorical_accuracy: 0.7076(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.8298 - categorical_accuracy: 0.7074 (10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 208s 28s/step - loss: 0.8285 - categorical_accuracy: 0.7073 - val_loss: 0.5469 - val_categorical_accuracy: 0.8200\n",
      "\n",
      "Epoch 00003: saving model to model_init_2022-04-1400_29_49.369108\\model-00003-0.81857-0.70639-0.54690-0.82000.h5\n",
      "Epoch 4/15\n",
      "7/8 [=========================>....] - ETA: 25s - loss: 0.7054 - categorical_accuracy: 0.7292(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.7036 - categorical_accuracy: 0.7306 (10, 15, 96, 96, 3) (10, 5)\n",
      "(10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 192s 26s/step - loss: 0.7022 - categorical_accuracy: 0.7317 - val_loss: 0.4450 - val_categorical_accuracy: 0.8300\n",
      "\n",
      "Epoch 00004: saving model to model_init_2022-04-1400_29_49.369108\\model-00004-0.69098-0.74007-0.44501-0.83000.h5\n",
      "Epoch 5/15\n",
      "7/8 [=========================>....] - ETA: 26s - loss: 0.5946 - categorical_accuracy: 0.7730(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.5929 - categorical_accuracy: 0.7730 (10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 214s 29s/step - loss: 0.5916 - categorical_accuracy: 0.7729 - val_loss: 0.5140 - val_categorical_accuracy: 0.8000\n",
      "\n",
      "Epoch 00005: saving model to model_init_2022-04-1400_29_49.369108\\model-00005-0.58120-0.77275-0.51403-0.80000.h5\n",
      "Epoch 6/15\n",
      "7/8 [=========================>....] - ETA: 25s - loss: 0.5176 - categorical_accuracy: 0.7899(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.5158 - categorical_accuracy: 0.7912 (10, 15, 96, 96, 3) (10, 5)\n",
      "(10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 196s 26s/step - loss: 0.5143 - categorical_accuracy: 0.7922 - val_loss: 0.3908 - val_categorical_accuracy: 0.8600\n",
      "\n",
      "Epoch 00006: saving model to model_init_2022-04-1400_29_49.369108\\model-00006-0.50279-0.80040-0.39082-0.86000.h5\n",
      "Epoch 7/15\n",
      "7/8 [=========================>....] - ETA: 26s - loss: 0.4684 - categorical_accuracy: 0.8035(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.4669 - categorical_accuracy: 0.8043 (10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 216s 28s/step - loss: 0.4658 - categorical_accuracy: 0.8049 - val_loss: 0.3378 - val_categorical_accuracy: 0.8700\n",
      "\n",
      "Epoch 00007: saving model to model_init_2022-04-1400_29_49.369108\\model-00007-0.45662-0.80995-0.33782-0.87000.h5\n",
      "Epoch 8/15\n",
      "7/8 [=========================>....] - ETA: 25s - loss: 0.4340 - categorical_accuracy: 0.8257(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.4327 - categorical_accuracy: 0.8260 (10, 15, 96, 96, 3) (10, 5)\n",
      "(10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 190s 26s/step - loss: 0.4317 - categorical_accuracy: 0.8263 - val_loss: 0.4559 - val_categorical_accuracy: 0.8400\n",
      "\n",
      "Epoch 00008: saving model to model_init_2022-04-1400_29_49.369108\\model-00008-0.42353-0.82856-0.45589-0.84000.h5\n",
      "Epoch 9/15\n",
      "7/8 [=========================>....] - ETA: 26s - loss: 0.3978 - categorical_accuracy: 0.8362(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.3976 - categorical_accuracy: 0.8361 (10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 207s 27s/step - loss: 0.3975 - categorical_accuracy: 0.8360 - val_loss: 0.3247 - val_categorical_accuracy: 0.8500\n",
      "\n",
      "Epoch 00009: saving model to model_init_2022-04-1400_29_49.369108\\model-00009-0.39621-0.83560-0.32467-0.85000.h5\n",
      "Epoch 10/15\n",
      "7/8 [=========================>....] - ETA: 25s - loss: 0.3875 - categorical_accuracy: 0.8211(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.3873 - categorical_accuracy: 0.8220 (10, 15, 96, 96, 3) (10, 5)\n",
      "(10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 192s 26s/step - loss: 0.3872 - categorical_accuracy: 0.8226 - val_loss: 0.3362 - val_categorical_accuracy: 0.8700\n",
      "\n",
      "Epoch 00010: saving model to model_init_2022-04-1400_29_49.369108\\model-00010-0.38593-0.82805-0.33617-0.87000.h5\n",
      "Epoch 11/15\n",
      "7/8 [=========================>....] - ETA: 25s - loss: 0.3772 - categorical_accuracy: 0.8303(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.3761 - categorical_accuracy: 0.8304 (10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 206s 27s/step - loss: 0.3753 - categorical_accuracy: 0.8305 - val_loss: 0.2937 - val_categorical_accuracy: 0.8900\n",
      "\n",
      "Epoch 00011: saving model to model_init_2022-04-1400_29_49.369108\\model-00011-0.36857-0.83107-0.29366-0.89000.h5\n",
      "Epoch 12/15\n",
      "7/8 [=========================>....] - ETA: 25s - loss: 0.3614 - categorical_accuracy: 0.8401(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.3610 - categorical_accuracy: 0.8404 (10, 15, 96, 96, 3) (10, 5)\n",
      "(10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 190s 25s/step - loss: 0.3607 - categorical_accuracy: 0.8407 - val_loss: 0.3924 - val_categorical_accuracy: 0.8700\n",
      "\n",
      "Epoch 00012: saving model to model_init_2022-04-1400_29_49.369108\\model-00012-0.35817-0.84263-0.39236-0.87000.h5\n",
      "Epoch 13/15\n",
      "7/8 [=========================>....] - ETA: 26s - loss: 0.3588 - categorical_accuracy: 0.8442(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.3580 - categorical_accuracy: 0.8445 (10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 205s 28s/step - loss: 0.3573 - categorical_accuracy: 0.8448 - val_loss: 0.3358 - val_categorical_accuracy: 0.8600\n",
      "\n",
      "Epoch 00013: saving model to model_init_2022-04-1400_29_49.369108\\model-00013-0.35228-0.84666-0.33579-0.86000.h5\n",
      "\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 14/15\n",
      "7/8 [=========================>....] - ETA: 25s - loss: 0.3526 - categorical_accuracy: 0.8560(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.3521 - categorical_accuracy: 0.8555 (10, 15, 96, 96, 3) (10, 5)\n",
      "(10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 192s 26s/step - loss: 0.3517 - categorical_accuracy: 0.8551 - val_loss: 0.3147 - val_categorical_accuracy: 0.8800\n",
      "\n",
      "Epoch 00014: saving model to model_init_2022-04-1400_29_49.369108\\model-00014-0.34853-0.85219-0.31465-0.88000.h5\n",
      "Epoch 15/15\n",
      "7/8 [=========================>....] - ETA: 25s - loss: 0.3399 - categorical_accuracy: 0.8498(99, 15, 96, 96, 3) (99, 5)\n",
      "8/8 [==============================] - ETA: 0s - loss: 0.3402 - categorical_accuracy: 0.8501 (10, 15, 96, 96, 3) (10, 5)\n",
      "8/8 [==============================] - 210s 27s/step - loss: 0.3405 - categorical_accuracy: 0.8503 - val_loss: 0.4003 - val_categorical_accuracy: 0.8500\n",
      "\n",
      "Epoch 00015: saving model to model_init_2022-04-1400_29_49.369108\\model-00015-0.34244-0.85219-0.40029-0.85000.h5\n",
      "\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n"
     ]
    }
   ],
   "source": [
    "if(cnn == 0):\n",
    "    model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=num_epochs, verbose=1, \n",
    "                    callbacks=callbacks_list, validation_data=val_generator, \n",
    "                    validation_steps=validation_steps, class_weight=None, workers=1, initial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "m2 = keras.models.load_model('C:\\\\Users\\\\Vrishank\\\\Downloads\\\\Hackathon\\\\model_init_2022-04-1221_57_19.709516\\\\model-00015-0.38932-0.84515-0.33530-0.87000.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\train ; batch size = 1\n",
      "SHAPE =  (96, 96, 3)\n",
      "***************** batch_data.shape = *************** (1, 15, 96, 96, 3)\n",
      "[0.016418015584349632, 0.014704793691635132, 0.4727494418621063, 0.047185689210891724, 0.4489421248435974]\n",
      "Stop\n",
      "Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\train ; batch size = 1\n",
      "SHAPE =  (96, 96, 3)\n",
      "***************** batch_data.shape = *************** (1, 15, 96, 96, 3)\n",
      "[0.020466944202780724, 0.01646864041686058, 0.6570666432380676, 0.2425665706396103, 0.06343112885951996]\n",
      "Stop\n",
      "2\n",
      "Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\train ; batch size = 1\n",
      "SHAPE =  (96, 96, 3)\n",
      "***************** batch_data.shape = *************** (1, 15, 96, 96, 3)\n",
      "[0.13690337538719177, 0.31875020265579224, 0.297463059425354, 0.06378047168254852, 0.18310284614562988]\n",
      "Right Swipe\n",
      "Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\train ; batch size = 1\n",
      "SHAPE =  (96, 96, 3)\n",
      "***************** batch_data.shape = *************** (1, 15, 96, 96, 3)\n",
      "[0.3050120174884796, 0.004623477812856436, 0.0021535479463636875, 0.007647660095244646, 0.6805632710456848]\n",
      "Thumbs Up\n",
      "4\n",
      "Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\train ; batch size = 1\n",
      "SHAPE =  (96, 96, 3)\n",
      "***************** batch_data.shape = *************** (1, 15, 96, 96, 3)\n",
      "[0.6576446294784546, 0.015817517414689064, 0.08906981348991394, 0.06112709268927574, 0.17634087800979614]\n",
      "Left Swipe\n",
      "0\n",
      "Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\train ; batch size = 1\n",
      "SHAPE =  (96, 96, 3)\n",
      "***************** batch_data.shape = *************** (1, 15, 96, 96, 3)\n",
      "[0.16428852081298828, 0.0055117374286055565, 0.4239259958267212, 0.026426469907164574, 0.3798471987247467]\n",
      "Stop\n",
      "Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\train ; batch size = 1\n",
      "SHAPE =  (96, 96, 3)\n",
      "***************** batch_data.shape = *************** (1, 15, 96, 96, 3)\n",
      "[0.03219179809093475, 0.0045724171213805676, 0.004233644809573889, 0.010590806603431702, 0.9484113454818726]\n",
      "Thumbs Up\n",
      "4\n",
      "Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\train ; batch size = 1\n",
      "SHAPE =  (96, 96, 3)\n",
      "***************** batch_data.shape = *************** (1, 15, 96, 96, 3)\n",
      "[0.008329532109200954, 0.010158209130167961, 0.07455534487962723, 0.0074223424308001995, 0.8995345830917358]\n",
      "Thumbs Up\n",
      "4\n",
      "Source path =  C:\\Users\\Vrishank\\Downloads\\Hackathon\\Project_data\\train ; batch size = 1\n",
      "SHAPE =  (96, 96, 3)\n",
      "***************** batch_data.shape = *************** (1, 15, 96, 96, 3)\n",
      "[0.0012700564693659544, 0.00034943578066304326, 0.001304214121773839, 0.9937824606895447, 0.0032938721124082804]\n",
      "Thumbs Down\n",
      "3\n"
     ]
    }
   ],
   "source": [
    "import shutil\n",
    "import os\n",
    "\n",
    "mnp = 0;\n",
    "flag_2 = 0\n",
    "exit = 0\n",
    "\n",
    "\n",
    "while(True):\n",
    "\n",
    "    directory = 'imgs3'\n",
    "\n",
    "    # Parent Directory path\n",
    "    parent_dir = location\n",
    "\n",
    "    # Path\n",
    "    path2 = os.path.join(parent_dir, directory)\n",
    "\n",
    "    # Create the directory\n",
    "    # 'GeeksForGeeks' in\n",
    "    # '/home / User / Documents'\n",
    "    os.mkdir(path)\n",
    "\n",
    "\n",
    "    import cv2\n",
    "    from time import *\n",
    "\n",
    "    # Opens the inbuilt camera of laptop to capture video.\n",
    "    cap = cv2.VideoCapture(0)\n",
    "    i = 0\n",
    "    if(mnp==0):\n",
    "        sleep(2)\n",
    "    mnp+=1\n",
    "\n",
    "    while(cap.isOpened()):\n",
    "        ret, frame = cap.read()\n",
    "        sleep(0.01)\n",
    "        # This condition prevents from infinite looping\n",
    "        # incase video ends.\n",
    "        if ret == False:\n",
    "            break\n",
    "        #cv2.imshow('frame', frame)\n",
    "        # Save Frame by Frame into disk using imwrite method\n",
    "        #cv2.imwrite('Frame'+str(i)+'.jpg', frame)\n",
    "        path = 'C:\\\\Users\\\\Vrishank\\\\Downloads\\\\Hackathon\\\\Project_data\\\\train\\\\imgs3'\n",
    "        cv2.imwrite(os.path.join(path ,  str(i) + '.png'), frame)\n",
    "        i += 1\n",
    "        if (i == 45):\n",
    "            break\n",
    "\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    def pred_creator(source_path, folder_list, batch_size, lenth ):\n",
    "        print( 'Source path = ', source_path, '; batch size =', batch_size)\n",
    "\n",
    "        l = []\n",
    "        i = 0\n",
    "\n",
    "        if(img_nums == 30):\n",
    "            img_idx = range(0, 30)                                                                     #create a list of image numbers you want to use for a particular video\n",
    "        else:\n",
    "            img_idx = range(0, 30, 2)\n",
    "\n",
    "        t = np.random.permutation(folder_list)\n",
    "        #t = folder_list\n",
    "        num_batches = lenth // batch_size                                                     # calculate the number of batches\n",
    "        num_batches = 1\n",
    "        num_imgs = lenth - num_batches*batch_size\n",
    "        for batch in range(num_batches):                                                          # we iterate over the number of batches\n",
    "            x = len(img_idx)\n",
    "            y = 96\n",
    "            z = 96\n",
    "            batch_data = np.zeros((batch_size,x,y,z,3))                                           # x is the number of images you use for each video, (y,z) is the final size of the input images and 3 is the number of channels RGB\n",
    "    #           batch_labels = np.zeros((batch_size,5))                                               # batch_labels is the one hot representation of the output\n",
    "            batch_labels = np.zeros((30, 5))                                               # batch_labels is the one hot representation of the output\n",
    "\n",
    "\n",
    "    # WIN_20180925_17_44_26_Pro_Right_Swipe_new;Right_Swipe_new;1\n",
    "\n",
    "\n",
    "            for folder in range(batch_size):                                                      # iterate over the batch_size\n",
    "    #            imgs = os.listdir(source_path+'/'+ 'WIN_20180925_17_08_43_Pro_Left_Swipe_new')\n",
    "                imgs = os.listdir(source_path+'/'+ 'WIN_20180925_17_44_26_Pro_Right_Swipe_new')\n",
    "                imgs = os.listdir(source_path+'/'+ 'imgs3')\n",
    "                #imgs = sorted(imgs)\n",
    "                #print(imgs)                                                                       # read all the images in the folder\n",
    "                for idx,item in enumerate(img_idx):                                               #  Iterate over the frames/images of a folder to read them in\n",
    "    #                    print(\"This Img Path is: \")\n",
    "    #                    print(source_path+'/'+ 'WIN_20180925_17_08_43_Pro_Left_Swipe_new'+'/'+imgs[item])\n",
    "    #                image = imread(source_path+'/'+ 'WIN_20180925_17_44_26_Pro_Right_Swipe_new'+'/'+imgs[item]).astype(np.float32)\n",
    "                    image = imread(source_path+'/'+ 'imgs3'+'/'+imgs[item]).astype(np.float32)\n",
    "\n",
    "                    #crop the images and resize them. Note that the images are of 2 different shape \n",
    "                    #and the conv3D will throw error if the inputs in a batch have different shapes\n",
    "\n",
    "                    image[:, :, 0] = (image [:, :, 0] - np.min(image[:, :, 0]))/(np.max(image[:, :, 0]) - np.min(image[:, :, 0]))\n",
    "                    image[:, :, 1] = (image [:, :, 1] - np.min(image[:, :, 1]))/(np.max(image[:, :, 1]) - np.min(image[:, :, 1]))\n",
    "                    image[:, :, 2] = (image [:, :, 2] - np.min(image[:, :, 2]))/(np.max(image[:, :, 2]) - np.min(image[:, :, 2]))\n",
    "\n",
    "\n",
    "                    image = resize(image = image, output_shape=(y, z))\n",
    "\n",
    "                    batch_data[folder,idx,:,:,0] = image[:, :, 0]       #normalise and feed in the image\n",
    "                    batch_data[folder,idx,:,:,1] = image[:, :, 1]       #normalise and feed in the image\n",
    "                    batch_data[folder,idx,:,:,2] = image[:, :, 2]       #normalise and feed in the image\n",
    "\n",
    "    #                l.append(image)\n",
    "    #               batch_labels[folder, int(t[folder + (batch*batch_size)].strip().split(';')[2])] = 1\n",
    "                batch_labels[folder, 0]= 1\n",
    "        print(\"SHAPE = \" , image.shape)\n",
    "    #        l.append(batch_labels)\n",
    "            #return l\n",
    "    #        l1 = np.array(l)\n",
    "        print(\"***************** batch_data.shape = ***************\", batch_data.shape)\n",
    "        yield batch_data, batch_labels\n",
    "    #        yield l1, batch_labels\n",
    "    #        return l1, batch_labels\n",
    "    #        return batch_data, batch_labels\n",
    "\n",
    "    p2 = pred_creator(train_path, train_doc, 1, 15)\n",
    "    pred_mat1 = m2.predict(p2)\n",
    "    d = ['Left Swipe' , 'Right Swipe' , 'Stop' , 'Thumbs Down' , 'Thumbs Up']\n",
    "    # 0: Left Swipe, 1: Right Swipe, 2: Stop, 3: Thumbs Down, 4: Thumbs Up\n",
    "\n",
    "    l_pred = pred_mat1.tolist()\n",
    "    l_pred = l_pred[0]\n",
    "    v = 0\n",
    "    flag_2 = 0\n",
    "    print(l_pred)\n",
    "    for i in range(len(l_pred)):\n",
    "        if l_pred[i] == max(l_pred):\n",
    "            print (d[i])\n",
    "            v = i\n",
    "            break\n",
    "    flag = 0\n",
    "    for i in l_pred:\n",
    "        if i>0.5:\n",
    "            flag = 1\n",
    "    if(flag):\n",
    "        from pynput.keyboard import Key, Controller\n",
    "        from time import *\n",
    "\n",
    "        keyboard = Controller()\n",
    "\n",
    "        def spacebar():\n",
    "            keyboard.press(Key.space)\n",
    "            keyboard.release(Key.space)\n",
    "\n",
    "        def right():\n",
    "            keyboard.press(Key.right)\n",
    "            keyboard.release(Key.right)\n",
    "\n",
    "        def left():\n",
    "            keyboard.press(Key.left)\n",
    "            keyboard.release(Key.left)\n",
    "\n",
    "        def up():\n",
    "            keyboard.press(Key.up)\n",
    "            keyboard.release(Key.up)\n",
    "\n",
    "        def down():\n",
    "            keyboard.press(Key.down)\n",
    "            keyboard.release(Key.down)\n",
    "        def escp():\n",
    "            keyboard.press(Key.esc)\n",
    "            keyboard.press(Key.esc)\n",
    "        def enter():\n",
    "            keyboard.press(Key.f5)\n",
    "            keyboard.press(Key.f5)\n",
    "\n",
    "        x = v\n",
    "        print(x)\n",
    "        if x == 0:\n",
    "            sleep(0.5)\n",
    "            left()\n",
    "        if x == 1:\n",
    "            sleep(0.5)\n",
    "            right()\n",
    "        if x == 2:\n",
    "            sleep(10)\n",
    "            #left()\n",
    "        if x == 3:\n",
    "            sleep(0.5)\n",
    "            flag_2 = 1\n",
    "            break\n",
    "        if x == 4:\n",
    "            sleep(0.5)\n",
    "            if (exit == 0):\n",
    "                escp()\n",
    "                exit += 1\n",
    "            else:\n",
    "                enter()\n",
    "                exit-=1\n",
    "                \n",
    "              \n",
    "\n",
    "    # Python program to demonstrate\n",
    "    # shutil.rmtree()\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "    # location\n",
    "    location = \"C:\\\\Users\\\\Vrishank\\\\Downloads\\\\Hackathon\\\\Project_data\\\\train\"\n",
    "\n",
    "    # directory\n",
    "    dir = \"imgs3\"\n",
    "\n",
    "    # path\n",
    "    path = os.path.join(location, dir)\n",
    "\n",
    "    # removing directory\n",
    "    shutil.rmtree(path)\n",
    "    if(flag_2):\n",
    "        break\n",
    "\n",
    "if(flag_2 == 1):\n",
    "    dir = \"imgs3\"\n",
    "\n",
    "    # path\n",
    "    path = os.path.join(location, dir)\n",
    "\n",
    "    # removing directory\n",
    "    shutil.rmtree(path)\n",
    "    \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting opencv-python\n",
      "  Downloading opencv_python-4.5.5.64-cp36-abi3-win_amd64.whl (35.4 MB)\n",
      "Requirement already satisfied: numpy>=1.14.5; python_version >= \"3.7\" in c:\\users\\vrishank\\anaconda3\\lib\\site-packages (from opencv-python) (1.19.5)\n",
      "Installing collected packages: opencv-python\n",
      "Successfully installed opencv-python-4.5.5.64\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install opencv-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pynput\n",
      "  Downloading pynput-1.7.6-py2.py3-none-any.whl (89 kB)\n",
      "Requirement already satisfied: six in c:\\users\\vrishank\\anaconda3\\lib\\site-packages (from pynput) (1.15.0)\n",
      "Installing collected packages: pynput\n",
      "Successfully installed pynput-1.7.6\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install pynput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "location = \"C:\\\\Users\\\\Vrishank\\\\Downloads\\\\Hackathon\\\\Project_data\\\\train\"\n",
    "\n",
    "    # directory\n",
    "dir = \"imgs3\"\n",
    "\n",
    "    # path\n",
    "path = os.path.join(location, dir)\n",
    "\n",
    "    # removing directory\n",
    "shutil.rmtree(path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Neural_Nets_Project_Starter_Code.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3.9.5 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}