tutorials/RNN_tutorials_toy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we quickly demonstrate how to utilize different mudules in our package.\n",
    "We will go through how to use our packages for 2 major sections:\n",
    "1. Process data through our dataloader \n",
    "2. Specify parameters and train\n",
    "***\n",
    "* The parts where you can take the most control (modify to suit your needs) will have explanations highlighted in **bold**. \n",
    "* Hyperparameters should be self-explanatory with details in options() function. \n",
    "* You can also find commments at the beginning of each cell for their functionalities in gerneral\n",
    "\n",
    "### Step0. Load libraries and our modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip3.7 install cupy pynvrtc git+https://github.com/salesforce/pytorch-qrnn --user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Created on Wed Nov 28 12:57:40 2018\n",
    "@author: ginnyzhu\n",
    "Last reviewed and updated Lrasmy Feb 21 2020\n",
    "\"\"\"\n",
    "from __future__ import print_function, division\n",
    "from io import open\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "\n",
    "import os\n",
    "import argparse\n",
    "import time\n",
    "import math\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "import numpy as np\n",
    "\n",
    "try:\n",
    "    import cPickle as pickle\n",
    "except:\n",
    "    import pickle\n",
    "    \n",
    "#import self-defined modules\n",
    "#models, utils, and Dataloader\n",
    "#sys.path.insert() only for jupyter notebook imports\n",
    "import sys\n",
    "sys.path.insert(0, '../ehr_pytorch')\n",
    "import models as model \n",
    "from EHRDataloader import EHRdataFromPickles,EHRdataFromLoadedPickles, EHRdataloader \n",
    "import utils as ut #:)))) \n",
    "from EHREmb import EHREmbeddings\n",
    "\n",
    "#silly ones\n",
    "from termcolor import colored\n",
    "from tqdm import tqdm\n",
    "# check GPU availability\n",
    "use_cuda = torch.cuda.is_available()\n",
    "#device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#args, slightly modified from main.py file to be more compatible with jupyter notebook \n",
    "#all args provide default values, so you can run the whole notebook without changing/providing any args\n",
    "#args ordered by dataloader, model, and training sections\n",
    "def options():\n",
    "    parser = argparse.ArgumentParser(description='Predictive Analytics on EHR with Pytorch')\n",
    "    \n",
    "    #EHRdataloader \n",
    "    parser.add_argument('-root_dir', type = str, default = '../data/' , \n",
    "                        help='the path to the folders with pickled file(s)')\n",
    "    parser.add_argument('-file', type = str, default = 'toy.train' , \n",
    "                        help='the name of pickled files')\n",
    "    parser.add_argument('-test_ratio', type = float, default = 0.2, \n",
    "                        help='test data size [default: 0.2]')\n",
    "    parser.add_argument('-valid_ratio', type = float, default = 0.1, \n",
    "                        help='validation data size [default: 0.1]')\n",
    "    \n",
    "    #EHRmodel\n",
    "    parser.add_argument('-which_model', type = str, default = 'DRNN', \n",
    "                        help='choose from {\"RNN\",\"DRNN\",\"QRNN\",\"LR\"}') \n",
    "    parser.add_argument('-cell_type', type = str, default = 'GRU', \n",
    "                        help='For RNN based models, choose from {\"RNN\", \"GRU\", \"LSTM\"}')\n",
    "    parser.add_argument('-input_size', type = list, default =[15817], \n",
    "                        help='''input dimension(s), decide which embedding types to use. \n",
    "                        If len of 1, then  1 embedding; \n",
    "                        len of 3, embedding medical, diagnosis and others separately (3 embeddings) \n",
    "                        [default:[15817]]''') ###multiple embeddings not effective in this release\n",
    "    parser.add_argument('-embed_dim', type=int, default=128, \n",
    "                        help='number of embedding dimension [default: 128]')\n",
    "    parser.add_argument('-hidden_size', type=int, default=128, \n",
    "                        help='size of hidden layers [default: 128]')\n",
    "    parser.add_argument('-dropout_r', type=float, default=0.1, \n",
    "                        help='the probability for dropout[default: 0.1]')\n",
    "    parser.add_argument('-n_layers', type=int, default=3, \n",
    "                        help='''number of Layers, \n",
    "                        for Dilated RNNs, dilations will increase exponentialy with mumber of layers [default: 1]''')\n",
    "    parser.add_argument('-bii', type=bool, default=False, \n",
    "                        help='indicator of whether Bi-directin is activated. [default: False]')\n",
    "    parser.add_argument('-time', type=bool, default=False, \n",
    "                        help='indicator of whether time is incorporated into embedding. [default: False]')\n",
    "    parser.add_argument('-preTrainEmb', type= str, default='', \n",
    "                        help='path to pretrained embeddings file. [default:'']')\n",
    "    parser.add_argument(\"-output_dir\",type=str, default= '../models/', \n",
    "                        help=\"The output directory where the best model will be saved and logs written [default: we will create'../models/'] \")\n",
    "    \n",
    "    # training \n",
    "    parser.add_argument('-lr', type=float, default=10**-4, \n",
    "                        help='learning rate [default: 0.0001]')\n",
    "    parser.add_argument('-L2', type=float, default=10**-4, \n",
    "                        help='L2 regularization [default: 0.0001]')\n",
    "    parser.add_argument('-epochs', type=int, default= 100, \n",
    "                        help='number of epochs for training [default: 100]')\n",
    "    parser.add_argument('-patience', type=int, default= 20, \n",
    "                        help='number of stagnant epochs to wait before terminating training [default: 20]')\n",
    "    parser.add_argument('-batch_size', type=int, default=128, \n",
    "                        help='batch size for training, validation or test [default: 128]')\n",
    "    parser.add_argument('-optimizer', type=str, default='adam', \n",
    "                        choices=  ['adam','adadelta','adagrad', 'adamax', 'asgd','rmsprop', 'rprop', 'sgd'], \n",
    "                        help='Select which optimizer to train [default: adam]. Upper/lower case does not matter') \n",
    "    #parser.add_argument('-cuda', type= bool, default=True, help='whether GPU is available [default:True]')\n",
    "    args = parser.parse_args([])\n",
    "    return args "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### StepX: You can modify parameters here to suit your own need\n",
    "\n",
    "* All parameters have explanations in the cell above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Namespace(L2=0.0001, batch_size=128, bii=False, cell_type='GRU', dropout_r=0.2, embed_dim=128, epochs=100, file='toy.train', hidden_size=128, input_size=[30000], lr=0.0001, n_layers=2, optimizer='adam', output_dir='../models/', patience=3, preTrainEmb='', root_dir='../data/', test_ratio=0.2, time=False, valid_ratio=0.1, which_model='RNN')\n"
     ]
    }
   ],
   "source": [
    "args = options()\n",
    "##Update the args here if you dont want to use the default ones\n",
    "##start an example\n",
    "args.which_model = 'RNN'\n",
    "args.cell_type = 'GRU'\n",
    "args.embed_dim = 128\n",
    "args.hidden_size = 128\n",
    "args.dropout_r = 0.2\n",
    "args.n_layers = 2\n",
    "args.input_size=[30000]\n",
    "args.patience=3\n",
    "##end\n",
    "print(args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step1. Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\n",
      "Loading and preparing data...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "####Step1. Data preparation\n",
    "#By default, prevent sort (on visit length) before splitting, if splitting\n",
    "#Gotta specify your split ratios here if intend to split on non-default split ratios\n",
    "#First load your data\n",
    "print(colored(\"\\nLoading and preparing data...\", 'green'))    \n",
    "data = EHRdataFromPickles(root_dir = args.root_dir, \n",
    "                          file = args.file, \n",
    "                          sort= False,\n",
    "                          test_ratio = args.test_ratio, \n",
    "                          valid_ratio = args.valid_ratio) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| data_description   | data                                |\n",
      "|--------------------+-------------------------------------|\n",
      "| patient_id         | 24                                  |\n",
      "| label              | 1                                   |\n",
      "| visit_time         | [list([9358])]                      |\n",
      "| visit_codes        | [list([16422, 17589, 8301, 17447])] |\n",
      "[24, 1, [[[9358], [16422, 17589, 8301, 17447]]]]\n",
      "\u001b[32m\n",
      "Sample data after split:\u001b[0m\n",
      "train: [215, 0, [[[2094], [2709, 12635, 17202, 12404, 7548, 16477, 18066, 8872, 11111, 18996, 9296, 14756, 7263, 8968]]]]\n",
      "test: [5733, 1, [[[10213], [7940, 6237, 12609, 8603, 758, 2066, 3914, 10758, 16268, 17491, 10553, 16013, 17513, 14205, 13604, 17346, 8807, 1558, 9898, 9635, 2356, 11765, 10720, 12938, 7196, 17190, 17493, 15507, 10740, 17982, 17527, 9718, 13312, 2447, 16315, 5174, 18163, 10339, 8880, 2782, 4949, 10292, 19569, 7362, 17950, 18022, 12886]]]]\n",
      "validation: [6237, 1, [[[5678], [1645, 15544, 6377, 11201, 3330, 19591, 14144, 8792, 18870, 6087, 3935, 5814, 8663, 16352, 4739, 4403]]]]\n",
      "\u001b[32m\n",
      "Sample data lengths for train, test and validation:\u001b[0m\n",
      "7000 2000 1000\n"
     ]
    }
   ],
   "source": [
    "#see an example of our pickle data\n",
    "#40 is the index\n",
    "#it will print out a formatted table of what each value mean and how they are organized in the file\n",
    "print(data.__getitem__(24, seeDescription = True)) \n",
    "\n",
    "# Dataloader splits\n",
    "train, test, valid = data.__splitdata__()\n",
    "# can comment out this part if you dont want to know what's going on here\n",
    "print(colored(\"\\nSample data after split:\", 'green'))\n",
    "# an example from train, test, and valiation\n",
    "print(\n",
    "  \"train: {}\".format(train[-1]),\n",
    "  \"test: {}\".format(test[-1]),\n",
    "  \"validation: {}\".format(valid[-1]), sep='\\n')\n",
    "print(colored(\"\\nSample data lengths for train, test and validation:\", 'green'))\n",
    "print(len(train), len(test), len(valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step2. Model loading "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#depending on different models, model parameters might have different choices.\n",
    "#e.g. if you set bi = True for DRNN or QRNN, it will throw you warnings and implement correct bi =False instead\n",
    "if args.which_model == 'RNN': \n",
    "    ehr_model = model.EHR_RNN(input_size= args.input_size, \n",
    "                              embed_dim=args.embed_dim, \n",
    "                              hidden_size= args.hidden_size,\n",
    "                              n_layers= args.n_layers,\n",
    "                              dropout_r=args.dropout_r,\n",
    "                              cell_type=args.cell_type,\n",
    "                              bii= args.bii,\n",
    "                              time= args.time,\n",
    "                              preTrainEmb= args.preTrainEmb) \n",
    "    pack_pad = True\n",
    "elif args.which_model == 'DRNN': \n",
    "    ehr_model = model.EHR_DRNN(input_size= args.input_size, \n",
    "                              embed_dim=args.embed_dim, \n",
    "                              hidden_size= args.hidden_size,\n",
    "                              n_layers= args.n_layers,\n",
    "                              dropout_r=args.dropout_r, #default =0 \n",
    "                              cell_type=args.cell_type, #default ='DRNN'\n",
    "                              bii= False,\n",
    "                              time = args.time, \n",
    "                              preTrainEmb= args.preTrainEmb)     \n",
    "    pack_pad = False\n",
    "elif args.which_model == 'QRNN': \n",
    "    ehr_model = model.EHR_QRNN(input_size= args.input_size, \n",
    "                              embed_dim=args.embed_dim, \n",
    "                              hidden_size= args.hidden_size,\n",
    "                              n_layers= args.n_layers,\n",
    "                              dropout_r=args.dropout_r, #default =0.1\n",
    "                              cell_type= 'QRNN', #doesn't support normal cell types\n",
    "                              bii= False, #QRNN doesn't support bi\n",
    "                              time = args.time,\n",
    "                              preTrainEmb= args.preTrainEmb)  \n",
    "    pack_pad = False\n",
    "elif args.which_model == 'TLSTM': \n",
    "    ehr_model = model.EHR_TLSTM(input_size= args.input_size, \n",
    "                              embed_dim=args.embed_dim, \n",
    "                              hidden_size= args.hidden_size,\n",
    "                              n_layers= args.n_layers,\n",
    "                              dropout_r=args.dropout_r, #default =0.1\n",
    "                              cell_type= 'TLSTM', #doesn't support normal cell types\n",
    "                              bii= False, \n",
    "                              time = args.time, \n",
    "                              preTrainEmb= args.preTrainEmb)  \n",
    "    pack_pad = False\n",
    "elif args.which_model == 'RETAIN': \n",
    "    ehr_model = model.RETAIN(input_size= args.input_size, \n",
    "                              embed_dim=args.embed_dim, \n",
    "                              hidden_size= args.hidden_size,\n",
    "                              n_layers= args.n_layers) \n",
    "    pack_pad = False\n",
    "else: \n",
    "    ehr_model = model.EHR_LR_emb(input_size = args.input_size,\n",
    "                                 embed_dim = args.embed_dim,\n",
    "                                 preTrainEmb= args.preTrainEmb)\n",
    "    pack_pad = False\n",
    "\n",
    "\n",
    "#make sure cuda is working\n",
    "if use_cuda:\n",
    "    ehr_model = ehr_model.cuda() \n",
    "#model optimizers to choose from. Upper/lower case dont matter\n",
    "if args.optimizer.lower() == 'adam':\n",
    "    optimizer = optim.Adam(ehr_model.parameters(), \n",
    "                           lr=args.lr, \n",
    "                           weight_decay=args.L2)\n",
    "elif args.optimizer.lower() == 'adadelta':\n",
    "    optimizer = optim.Adadelta(ehr_model.parameters(), \n",
    "                               lr=args.lr, \n",
    "                               weight_decay=args.L2)\n",
    "elif args.optimizer.lower() == 'adagrad':\n",
    "    optimizer = optim.Adagrad(ehr_model.parameters(), \n",
    "                              lr=args.lr, \n",
    "                              weight_decay=args.L2) \n",
    "elif args.optimizer.lower() == 'adamax':\n",
    "    optimizer = optim.Adamax(ehr_model.parameters(), \n",
    "                             lr=args.lr, \n",
    "                             weight_decay=args.L2)\n",
    "elif args.optimizer.lower() == 'asgd':\n",
    "    optimizer = optim.ASGD(ehr_model.parameters(), \n",
    "                           lr=args.lr, \n",
    "                           weight_decay=args.L2)\n",
    "elif args.optimizer.lower() == 'rmsprop':\n",
    "    optimizer = optim.RMSprop(ehr_model.parameters(), \n",
    "                              lr=args.lr, \n",
    "                              weight_decay=args.L2)\n",
    "elif args.optimizer.lower() == 'rprop':\n",
    "    optimizer = optim.Rprop(ehr_model.parameters(), \n",
    "                            lr=args.lr)\n",
    "elif args.optimizer.lower() == 'sgd':\n",
    "    optimizer = optim.SGD(ehr_model.parameters(), \n",
    "                          lr=args.lr, \n",
    "                          weight_decay=args.L2)\n",
    "else:\n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 55/55 [00:07<00:00,  7.07it/s]\n",
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " creating the list of valid minibatches\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:01<00:00,  7.87it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " creating the list of test minibatches\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:02<00:00,  6.73it/s]\n"
     ]
    }
   ],
   "source": [
    "##### separate loader for train, test, validation\n",
    "#if you have different files, you need to load them separately into EHRdataFromPickles()\n",
    "#and then use EHRdataloader() on each\n",
    "#dataloader's default will sort data based on length of visits and then split into batches with default batch_size/of your choice\n",
    "#new in this release is the creation of minibatches lists once before the epochs run, then will shuffle within the epochs\n",
    "train_mbs = list(tqdm(EHRdataloader(train, batch_size = args.batch_size, packPadMode = pack_pad)))\n",
    "print (' creating the list of valid minibatches')\n",
    "valid_mbs = list(tqdm(EHRdataloader(valid, batch_size = args.batch_size, packPadMode = pack_pad)))\n",
    "print (' creating the list of test minibatches')\n",
    "test_mbs = list(tqdm(EHRdataloader(test, batch_size = args.batch_size, packPadMode = pack_pad)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step3. Train, validation and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\n",
      " Epoch (0): Train_auc (0.5578475101672433), Valid_auc (0.5103293868854034) ,Training Average_loss (0.6936373829841613), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[33m\n",
      " Test_AUC (0.482506255630067) , Test_eval_time (0m 0s) \u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (1): Train_auc (0.5946469756966579), Valid_auc (0.5133092487864273) ,Training Average_loss (0.6881005471402948), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[33m\n",
      " Test_AUC (0.484342908617756) , Test_eval_time (0m 0s) \u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (2): Train_auc (0.6275864439150787), Valid_auc (0.5170380813534341) ,Training Average_loss (0.6836279966614464), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[33m\n",
      " Test_AUC (0.4873571214092683) , Test_eval_time (0m 0s) \u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (3): Train_auc (0.6587956501447891), Valid_auc (0.5232100802640222) ,Training Average_loss (0.6783660476857964), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[33m\n",
      " Test_AUC (0.48973676308677805) , Test_eval_time (0m 0s) \u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (4): Train_auc (0.6870815213766601), Valid_auc (0.5284729008795399) ,Training Average_loss (0.6699316544966264), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[33m\n",
      " Test_AUC (0.4912371134020619) , Test_eval_time (0m 0s) \u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (5): Train_auc (0.7170872055144382), Valid_auc (0.5351896057290249) ,Training Average_loss (0.6600027311931957), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[33m\n",
      " Test_AUC (0.49302572315083576) , Test_eval_time (0m 0s) \u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (6): Train_auc (0.7467720701567746), Valid_auc (0.5409170284688957) ,Training Average_loss (0.6463938897306268), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[33m\n",
      " Test_AUC (0.495264738264438) , Test_eval_time (0m 0s) \u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (7): Train_auc (0.7791326346978611), Valid_auc (0.5448300998093529) ,Training Average_loss (0.629902078888633), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[33m\n",
      " Test_AUC (0.4954308877990191) , Test_eval_time (0m 0s) \u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (8): Train_auc (0.813342197094355), Valid_auc (0.5465042695333151) ,Training Average_loss (0.6028947992758318), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[33m\n",
      " Test_AUC (0.4992453207887098) , Test_eval_time (0m 0s) \u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (9): Train_auc (0.8504341144920696), Valid_auc (0.5472932921065701) ,Training Average_loss (0.5710894974795256), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[33m\n",
      " Test_AUC (0.4987849064157742) , Test_eval_time (0m 0s) \u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (10): Train_auc (0.8883171825615586), Valid_auc (0.5458674442076932) ,Training Average_loss (0.5318197385831313), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (11): Train_auc (0.9229746269330498), Valid_auc (0.5433642000032042) ,Training Average_loss (0.48344862677834255), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (12): Train_auc (0.9503537545146364), Valid_auc (0.5416299524183341) ,Training Average_loss (0.4266189293427901), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[32m\n",
      " Epoch (13): Train_auc (0.9686907927825247), Valid_auc (0.5358945192970089) ,Training Average_loss (0.3668542482636191), Train_time (0m 0s), Eval_time (0m 0s)\u001b[0m\n",
      "\u001b[32mBestValidAuc 0.547293 has a TestAuc of 0.498785 at epoch 9 \u001b[0m\n",
      "\u001b[32mDetails see ../models/dhf.trainEHRmodel.log\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#Notes: default: sort data based on visit length \n",
    "#default: （batch）shuffle = true\n",
    "#allows for keyboard interrupt\n",
    "#saving best model in the directory specified in args.output_dir\n",
    "try:\n",
    "    ut.epochs_run(args.epochs, \n",
    "                  train = train_mbs, \n",
    "                  valid = valid_mbs, \n",
    "                  test = test_mbs, \n",
    "                  model = ehr_model, \n",
    "                  optimizer = optimizer,\n",
    "                  shuffle = True, \n",
    "                  which_model = args.which_model, \n",
    "                  patience = args.patience,\n",
    "                  output_dir = args.output_dir)\n",
    "#we can keyboard interupt now \n",
    "except KeyboardInterrupt:\n",
    "    print(colored('-' * 89, 'green'))\n",
    "    print(colored('Exiting from training early','green'))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EHR_RNN(\n",
       "  (embed): Embedding(30000, 128, padding_idx=0)\n",
       "  (rnn_c): GRU(128, 128, num_layers=2, batch_first=True, dropout=0.2)\n",
       "  (out): Linear(in_features=128, out_features=1, bias=True)\n",
       "  (sigmoid): Sigmoid()\n",
       ")"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#if you want to use previous trained models, use\n",
    "best_model= torch.load(args.output_dir + 'dhf.trainEHRmodel.pth')\n",
    "best_model.load_state_dict(torch.load(args.output_dir + 'dhf.trainEHRmodel.st'))\n",
    "best_model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### StepExtra: Singly use our dataloader for data preparation purposes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from EHRDataloader import EHRdataFromPickles, EHRdataloader, iter_batch2\n",
    "data2 = EHRdataFromPickles(root_dir = args.root_dir, \n",
    "                          file = args.file, \n",
    "                          sort= False,\n",
    "                          test_ratio = args.test_ratio, \n",
    "                          valid_ratio = args.valid_ratio) \n",
    "loader2 =  EHRdataloader(data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EOF\n"
     ]
    }
   ],
   "source": [
    "#if you want to shuffle batches before using them, add this line \n",
    "#(options are achieved in utils by setting shuffle = True)\n",
    "loader2 = iter_batch2(loader2, len(loader2))\n",
    "\n",
    "#otherwise, directly call \n",
    "for i, batch in enumerate(loader2): \n",
    "    #feed the batch to do things\n",
    "       # print('EOF')\n",
    "       # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py_37_env",
   "language": "python",
   "name": "py_37_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}