DenseNet_Cifar100.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h1>DenseNet: Cifar100</h1></center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division,print_function\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.nn.init as init\n",
    "from torch.autograd import Variable, grad\n",
    "from torchvision import datasets, transforms\n",
    "from torch.nn.parameter import Parameter\n",
    "\n",
    "import calculate_log as callog\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.set_device(2) #Select the GPU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "def conv3x3(in_planes, out_planes, stride=1):\n",
    "    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "\n",
    "\n",
    "class BottleneckBlock(nn.Module):\n",
    "    def __init__(self, in_planes, out_planes, dropRate=0.0):\n",
    "        super(BottleneckBlock, self).__init__()\n",
    "        inter_planes = out_planes * 4\n",
    "        self.bn1 = nn.BatchNorm2d(in_planes)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.conv1 = nn.Conv2d(in_planes, inter_planes, kernel_size=1, stride=1,\n",
    "                               padding=0, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(inter_planes)\n",
    "        self.conv2 = nn.Conv2d(inter_planes, out_planes, kernel_size=3, stride=1,\n",
    "                               padding=1, bias=False)\n",
    "        self.droprate = dropRate\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        out = self.conv1(self.relu(self.bn1(x)))\n",
    "        \n",
    "        torch_model.record(out)\n",
    "        \n",
    "        if self.droprate > 0:\n",
    "            out = F.dropout(out, p=self.droprate, inplace=False, training=self.training)\n",
    "        \n",
    "        out = self.conv2(self.relu(self.bn2(out)))\n",
    "        torch_model.record(out)\n",
    "        \n",
    "        if self.droprate > 0:\n",
    "            out = F.dropout(out, p=self.droprate, inplace=False, training=self.training)\n",
    "        return torch.cat([x, out], 1)\n",
    "\n",
    "class TransitionBlock(nn.Module):\n",
    "    def __init__(self, in_planes, out_planes, dropRate=0.0):\n",
    "        super(TransitionBlock, self).__init__()\n",
    "        self.bn1 = nn.BatchNorm2d(in_planes)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1,\n",
    "                               padding=0, bias=False)\n",
    "        self.droprate = dropRate\n",
    "    \n",
    "    def forward(self, x):\n",
    "        out = self.conv1(self.relu(self.bn1(x)))\n",
    "        torch_model.record(out)\n",
    "        \n",
    "        if self.droprate > 0:\n",
    "            out = F.dropout(out, p=self.droprate, inplace=False, training=self.training)\n",
    "        return F.avg_pool2d(out, 2)\n",
    "\n",
    "class DenseBlock(nn.Module):\n",
    "    def __init__(self, nb_layers, in_planes, growth_rate, block, dropRate=0.0):\n",
    "        super(DenseBlock, self).__init__()\n",
    "        self.layer = self._make_layer(block, in_planes, growth_rate, nb_layers, dropRate)\n",
    "    \n",
    "    def _make_layer(self, block, in_planes, growth_rate, nb_layers, dropRate):\n",
    "        layers = []\n",
    "        for i in range(int(nb_layers)):\n",
    "            layers.append(block(in_planes+i*growth_rate, growth_rate, dropRate))\n",
    "        return nn.Sequential(*layers)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        t = self.layer(x)\n",
    "        torch_model.record(t)\n",
    "        return t\n",
    "\n",
    "\n",
    "class DenseNet3(nn.Module):\n",
    "    def __init__(self, depth, num_classes, growth_rate=12,\n",
    "                 reduction=0.5, bottleneck=True, dropRate=0.0):\n",
    "        super(DenseNet3, self).__init__()\n",
    "        \n",
    "        self.collecting = False\n",
    "        \n",
    "        \n",
    "        \n",
    "        in_planes = 2 * growth_rate\n",
    "        n = (depth - 4) / 3\n",
    "        if bottleneck == True:\n",
    "            n = n/2\n",
    "            block = BottleneckBlock\n",
    "        else:\n",
    "            block = BasicBlock\n",
    "        # 1st conv before any dense block\n",
    "        self.conv1 = nn.Conv2d(3, in_planes, kernel_size=3, stride=1,\n",
    "                               padding=1, bias=False)\n",
    "        # 1st block\n",
    "        self.block1 = DenseBlock(n, in_planes, growth_rate, block, dropRate)\n",
    "        in_planes = int(in_planes+n*growth_rate)\n",
    "        self.trans1 = TransitionBlock(in_planes, int(math.floor(in_planes*reduction)), dropRate=dropRate)\n",
    "        in_planes = int(math.floor(in_planes*reduction))\n",
    "        # 2nd block\n",
    "        self.block2 = DenseBlock(n, in_planes, growth_rate, block, dropRate)\n",
    "        in_planes = int(in_planes+n*growth_rate)\n",
    "        self.trans2 = TransitionBlock(in_planes, int(math.floor(in_planes*reduction)), dropRate=dropRate)\n",
    "        in_planes = int(math.floor(in_planes*reduction))\n",
    "        # 3rd block\n",
    "        self.block3 = DenseBlock(n, in_planes, growth_rate, block, dropRate)\n",
    "        in_planes = int(in_planes+n*growth_rate)\n",
    "        # global average pooling and classifier\n",
    "        self.bn1 = nn.BatchNorm2d(in_planes)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.fc = nn.Linear(in_planes, num_classes)\n",
    "        self.in_planes = in_planes\n",
    "\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
    "                m.weight.data.normal_(0, math.sqrt(2. / n))\n",
    "            elif isinstance(m, nn.BatchNorm2d):\n",
    "                m.weight.data.fill_(1)\n",
    "                m.bias.data.zero_()\n",
    "            elif isinstance(m, nn.Linear):\n",
    "                m.bias.data.zero_()\n",
    "                    \n",
    "    def forward(self, x):\n",
    "        out = self.conv1(x)\n",
    "        out = self.trans1(self.block1(out))\n",
    "        out = self.trans2(self.block2(out))\n",
    "        out = self.block3(out)\n",
    "        out = self.relu(self.bn1(out))\n",
    "        out = F.avg_pool2d(out, 8)\n",
    "        out = out.view(-1, self.in_planes)\n",
    "        return self.fc(out)\n",
    "    \n",
    "    def load(self, path=\"densenet_cifar100.pth\"):\n",
    "        tm = torch.load(path,map_location=\"cpu\")\n",
    "        self.load_state_dict(tm.state_dict(),strict=False)\n",
    "    \n",
    "    def record(self, t):\n",
    "        if self.collecting:\n",
    "            self.gram_feats.append(t)\n",
    "    \n",
    "    def gram_feature_list(self,x):\n",
    "        self.collecting = True\n",
    "        self.gram_feats = []\n",
    "        self.forward(x)\n",
    "        self.collecting = False\n",
    "        temp = self.gram_feats\n",
    "        self.gram_feats = []\n",
    "        return temp\n",
    "    \n",
    "    def get_min_max(self, data, power):\n",
    "        mins = []\n",
    "        maxs = []\n",
    "        \n",
    "        for i in range(0,len(data),64):\n",
    "            batch = data[i:i+64].cuda()\n",
    "            feat_list = self.gram_feature_list(batch)\n",
    "            for L,feat_L in enumerate(feat_list):\n",
    "                if L==len(mins):\n",
    "                    mins.append([None]*len(power))\n",
    "                    maxs.append([None]*len(power))\n",
    "                \n",
    "                for p,P in enumerate(power):\n",
    "                    g_p = G_p(feat_L,P)\n",
    "                    \n",
    "                    current_min = g_p.min(dim=0,keepdim=True)[0]\n",
    "                    current_max = g_p.max(dim=0,keepdim=True)[0]\n",
    "                    \n",
    "                    if mins[L][p] is None:\n",
    "                        mins[L][p] = current_min\n",
    "                        maxs[L][p] = current_max\n",
    "                    else:\n",
    "                        mins[L][p] = torch.min(current_min,mins[L][p])\n",
    "                        maxs[L][p] = torch.max(current_max,maxs[L][p])\n",
    "        \n",
    "        return mins,maxs\n",
    "    \n",
    "    def get_deviations(self,data,power,mins,maxs):\n",
    "        deviations = []\n",
    "        \n",
    "        for i in range(0,len(data),64):            \n",
    "            batch = data[i:i+64].cuda()\n",
    "            feat_list = self.gram_feature_list(batch)\n",
    "            batch_deviations = []\n",
    "            for L,feat_L in enumerate(feat_list):\n",
    "                dev = 0\n",
    "                for p,P in enumerate(power):\n",
    "                    g_p = G_p(feat_L,P)\n",
    "                    \n",
    "                    dev +=  (F.relu(mins[L][p]-g_p)/torch.abs(mins[L][p]+10**-6)).sum(dim=1,keepdim=True)\n",
    "                    dev +=  (F.relu(g_p-maxs[L][p])/torch.abs(maxs[L][p]+10**-6)).sum(dim=1,keepdim=True)\n",
    "                batch_deviations.append(dev.cpu().detach().numpy())\n",
    "            batch_deviations = np.concatenate(batch_deviations,axis=1)\n",
    "            deviations.append(batch_deviations)\n",
    "        deviations = np.concatenate(deviations,axis=0)\n",
    "        \n",
    "        return deviations\n",
    "\n",
    "torch_model = DenseNet3(100, num_classes=100)\n",
    "torch_model.load()\n",
    "torch_model.cuda()\n",
    "torch_model.params = list(torch_model.parameters())\n",
    "torch_model.eval()\n",
    "print(\"Done\")  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>In-distribution Datasets</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "mean = np.array([[125.3/255, 123.0/255, 113.9/255]]).T\n",
    "\n",
    "std = np.array([[63.0/255, 62.1/255.0, 66.7/255.0]]).T\n",
    "normalize = transforms.Normalize((125.3/255, 123.0/255, 113.9/255), (63.0/255, 62.1/255.0, 66.7/255.0))\n",
    "\n",
    "transform_train = transforms.Compose([\n",
    "        transforms.RandomCrop(32, padding=4),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        normalize\n",
    "        \n",
    "    ])\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.CenterCrop(size=(32, 32)),\n",
    "        transforms.ToTensor(),\n",
    "        normalize\n",
    "    ])\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    datasets.CIFAR100('data', train=True, download=True,\n",
    "                   transform=transform_train),\n",
    "    batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.CIFAR100('data', train=False, transform=transform_test),\n",
    "    batch_size=batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "data_train = list(torch.utils.data.DataLoader(\n",
    "        datasets.CIFAR100('data', train=True, download=True,\n",
    "                       transform=transform_test),\n",
    "        batch_size=1, shuffle=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "data = list(torch.utils.data.DataLoader(\n",
    "    datasets.CIFAR100('data', train=False, download=True,\n",
    "                   transform=transform_test),\n",
    "    batch_size=1, shuffle=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.7763\n"
     ]
    }
   ],
   "source": [
    "torch_model.eval()\n",
    "correct = 0\n",
    "total = 0\n",
    "for x,y in test_loader:\n",
    "    x = x.cuda()\n",
    "    y = y.numpy()\n",
    "    correct += (y==np.argmax(torch_model(x).detach().cpu().numpy(),axis=1)).sum()\n",
    "    total += y.shape[0]\n",
    "print(\"Accuracy: \",correct/total)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Out-of-distribution Datasets</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "cifar10 = list(torch.utils.data.DataLoader(\n",
    "    datasets.CIFAR10('data', train=False, download=True,\n",
    "                   transform=transform_test),\n",
    "    batch_size=1, shuffle=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using downloaded and verified file: data/test_32x32.mat\n"
     ]
    }
   ],
   "source": [
    "svhn = list(torch.utils.data.DataLoader(\n",
    "    datasets.SVHN('data', split=\"test\", download=True,\n",
    "                   transform=transform_test),\n",
    "    batch_size=1, shuffle=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "isun = list(torch.utils.data.DataLoader(\n",
    "    datasets.ImageFolder(\"iSUN/\",transform=transform_test),batch_size=1,shuffle=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "lsun_c = list(torch.utils.data.DataLoader(\n",
    "    datasets.ImageFolder(\"LSUN/\",transform=transform_test),batch_size=1,shuffle=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "lsun_r = list(torch.utils.data.DataLoader(\n",
    "    datasets.ImageFolder(\"LSUN_resize/\",transform=transform_test),batch_size=1,shuffle=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tinyimagenet_c = list(torch.utils.data.DataLoader(\n",
    "    datasets.ImageFolder(\"Imagenet/\",transform=transform_test),batch_size=1,shuffle=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tinyimagenet_r = list(torch.utils.data.DataLoader(\n",
    "    datasets.ImageFolder(\"Imagenet_resize/\",transform=transform_test),batch_size=1,shuffle=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Code for Detecting OODs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b> Extract predictions for train and test data </b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "train_preds = []\n",
    "train_confs = []\n",
    "train_logits = []\n",
    "for idx in range(0,len(data_train),128):\n",
    "    batch = torch.squeeze(torch.stack([x[0] for x in data_train[idx:idx+128]]),dim=1).cuda()\n",
    "    \n",
    "    logits = torch_model(batch)\n",
    "    confs = F.softmax(logits,dim=1).cpu().detach().numpy()\n",
    "    preds = np.argmax(confs,axis=1)\n",
    "    logits = (logits.cpu().detach().numpy())\n",
    "    \n",
    "    train_confs.extend(np.max(confs,axis=1))    \n",
    "    train_preds.extend(preds)\n",
    "    train_logits.extend(logits)\n",
    "print(\"Done\")\n",
    "\n",
    "test_preds = []\n",
    "test_confs = []\n",
    "test_logits = []\n",
    "\n",
    "for idx in range(0,len(data),128):\n",
    "    batch = torch.squeeze(torch.stack([x[0] for x in data[idx:idx+128]]),dim=1).cuda()\n",
    "    \n",
    "    logits = torch_model(batch)\n",
    "    confs = F.softmax(logits,dim=1).cpu().detach().numpy()\n",
    "    preds = np.argmax(confs,axis=1)\n",
    "    logits = (logits.cpu().detach().numpy())\n",
    "\n",
    "    test_confs.extend(np.max(confs,axis=1))    \n",
    "    test_preds.extend(preds)\n",
    "    test_logits.extend(logits)\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b> Code for detecting OODs by identifying anomalies in correlations </b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import calculate_log as callog\n",
    "def detect(all_test_deviations,all_ood_deviations, verbose=True, normalize=True):\n",
    "    average_results = {}\n",
    "    for i in range(1,11):\n",
    "        random.seed(i)\n",
    "        \n",
    "        validation_indices = random.sample(range(len(all_test_deviations)),int(0.1*len(all_test_deviations)))\n",
    "        test_indices = sorted(list(set(range(len(all_test_deviations)))-set(validation_indices)))\n",
    "\n",
    "        validation = all_test_deviations[validation_indices]\n",
    "        test_deviations = all_test_deviations[test_indices]\n",
    "\n",
    "        t95 = validation.mean(axis=0)+10**-7\n",
    "        if not normalize:\n",
    "            t95 = np.ones_like(t95)\n",
    "        test_deviations = (test_deviations/t95[np.newaxis,:]).sum(axis=1)\n",
    "        ood_deviations = (all_ood_deviations/t95[np.newaxis,:]).sum(axis=1)\n",
    "        \n",
    "        results = callog.compute_metric(-test_deviations,-ood_deviations)\n",
    "        for m in results:\n",
    "            average_results[m] = average_results.get(m,0)+results[m]\n",
    "    \n",
    "    for m in average_results:\n",
    "        average_results[m] /= i\n",
    "    if verbose:\n",
    "        callog.print_results(average_results)\n",
    "    return average_results\n",
    "\n",
    "\n",
    "def cpu(ob):\n",
    "    for i in range(len(ob)):\n",
    "        for j in range(len(ob[i])):\n",
    "            ob[i][j] = ob[i][j].cpu()\n",
    "    return ob\n",
    "    \n",
    "def cuda(ob):\n",
    "    for i in range(len(ob)):\n",
    "        for j in range(len(ob[i])):\n",
    "            ob[i][j] = ob[i][j].cuda()\n",
    "    return ob\n",
    "\n",
    "class Detector:\n",
    "    def __init__(self):\n",
    "        self.all_test_deviations = None\n",
    "        self.mins = {}\n",
    "        self.maxs = {}\n",
    "        self.classes = range(100)\n",
    "    \n",
    "    def compute_minmaxs(self,data_train,POWERS=[10]):\n",
    "        for PRED in tqdm(self.classes):\n",
    "            train_indices = np.where(np.array(train_preds)==PRED)[0]\n",
    "            train_PRED = torch.squeeze(torch.stack([data_train[i][0] for i in train_indices]),dim=1)\n",
    "            mins,maxs = torch_model.get_min_max(train_PRED,power=POWERS)\n",
    "            self.mins[PRED] = cpu(mins)\n",
    "            self.maxs[PRED] = cpu(maxs)\n",
    "            torch.cuda.empty_cache()\n",
    "    \n",
    "    def compute_test_deviations(self,POWERS=[10]):\n",
    "        all_test_deviations = None\n",
    "        for PRED in tqdm(self.classes):\n",
    "            test_indices = np.where(np.array(test_preds)==PRED)[0]\n",
    "            test_PRED = torch.squeeze(torch.stack([data[i][0] for i in test_indices]),dim=1)\n",
    "            test_confs_PRED = np.array([test_confs[i] for i in test_indices])\n",
    "            mins = cuda(self.mins[PRED])\n",
    "            maxs = cuda(self.maxs[PRED])\n",
    "            test_deviations = torch_model.get_deviations(test_PRED,power=POWERS,mins=mins,maxs=maxs)/test_confs_PRED[:,np.newaxis]\n",
    "            cpu(mins)\n",
    "            cpu(maxs)\n",
    "            if all_test_deviations is None:\n",
    "                all_test_deviations = test_deviations\n",
    "            else:\n",
    "                all_test_deviations = np.concatenate([all_test_deviations,test_deviations],axis=0)\n",
    "            torch.cuda.empty_cache()\n",
    "        self.all_test_deviations = all_test_deviations\n",
    "    \n",
    "    def compute_ood_deviations(self,ood,POWERS=[10]):\n",
    "        ood_preds = []\n",
    "        ood_confs = []\n",
    "        \n",
    "        for idx in range(0,len(ood),128):\n",
    "            batch = torch.squeeze(torch.stack([x[0] for x in ood[idx:idx+128]]),dim=1).cuda()\n",
    "            logits = torch_model(batch)\n",
    "            confs = F.softmax(logits,dim=1).cpu().detach().numpy()\n",
    "            preds = np.argmax(confs,axis=1)\n",
    "            \n",
    "            ood_confs.extend(np.max(confs,axis=1))\n",
    "            ood_preds.extend(preds)  \n",
    "            torch.cuda.empty_cache()\n",
    "        print(\"Done\")\n",
    "        \n",
    "        all_ood_deviations = None\n",
    "        for PRED in tqdm(self.classes):\n",
    "            ood_indices = np.where(np.array(ood_preds)==PRED)[0]\n",
    "            if len(ood_indices)==0:\n",
    "                continue\n",
    "            ood_PRED = torch.squeeze(torch.stack([ood[i][0] for i in ood_indices]),dim=1)\n",
    "            ood_confs_PRED =  np.array([ood_confs[i] for i in ood_indices])\n",
    "            mins = cuda(self.mins[PRED])\n",
    "            maxs = cuda(self.maxs[PRED])\n",
    "            ood_deviations = torch_model.get_deviations(ood_PRED,power=POWERS,mins=mins,maxs=maxs)/ood_confs_PRED[:,np.newaxis]\n",
    "            cpu(self.mins[PRED])\n",
    "            cpu(self.maxs[PRED])            \n",
    "            if all_ood_deviations is None:\n",
    "                all_ood_deviations = ood_deviations\n",
    "            else:\n",
    "                all_ood_deviations = np.concatenate([all_ood_deviations,ood_deviations],axis=0)\n",
    "            torch.cuda.empty_cache()\n",
    "        average_results = detect(self.all_test_deviations,all_ood_deviations)\n",
    "        return average_results, self.all_test_deviations, all_ood_deviations\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h1> Results </h1></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d38db591504460ea13535698dea877c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffae25d812914b6c95aa3c1c7fd392a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "iSUN\n",
      "Done\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "716da00211014f8988f1c9937cf1d35c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 95.867 99.042 95.632 98.990 99.083\n",
      "LSUN (R)\n",
      "Done\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0397655250134e9d853d8598410ced8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 97.234 99.349 96.393 99.264 99.407\n",
      "LSUN (C)\n",
      "Done\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36483d6fcf034a8887df000863cb187a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 65.517 91.391 83.644 89.553 92.749\n",
      "TinyImgNet (R)\n",
      "Done\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9aa5b886321c4ba793127afe191bb73c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 95.748 98.973 95.522 98.768 99.126\n",
      "TinyImgNet (C)\n",
      "Done\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ab175d00f4e44688f226557051e413a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 89.013 97.687 92.452 97.303 98.018\n",
      "SVHN\n",
      "Done\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "999c654468da4044bae289c360b2de38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 89.341 97.322 92.364 91.652 99.080\n",
      "CIFAR-10\n",
      "Done\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebf38974d3bf4350aa44ea06f629e369",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 10.596 64.227 60.404 61.350 64.092\n"
     ]
    }
   ],
   "source": [
    "def G_p(ob, p):\n",
    "    temp = ob.detach()\n",
    "    \n",
    "    temp = temp**p\n",
    "    temp = temp.reshape(temp.shape[0],temp.shape[1],-1)\n",
    "    temp = ((torch.matmul(temp,temp.transpose(dim0=2,dim1=1)))).sum(dim=2) \n",
    "    temp = (temp.sign()*torch.abs(temp)**(1/p)).reshape(temp.shape[0],-1)\n",
    "    \n",
    "    return temp\n",
    "\n",
    "detector = Detector()\n",
    "detector.compute_minmaxs(data_train,POWERS=range(1,11))\n",
    "\n",
    "detector.compute_test_deviations(POWERS=range(1,11))\n",
    "\n",
    "print(\"iSUN\")\n",
    "isun_results = detector.compute_ood_deviations(isun,POWERS=range(1,11))\n",
    "print(\"LSUN (R)\")\n",
    "lsunr_results = detector.compute_ood_deviations(lsun_r,POWERS=range(1,11))\n",
    "print(\"LSUN (C)\")\n",
    "lsunc_results = detector.compute_ood_deviations(lsun_c,POWERS=range(1,11))\n",
    "print(\"TinyImgNet (R)\")\n",
    "timr_results = detector.compute_ood_deviations(tinyimagenet_r,POWERS=range(1,11))\n",
    "print(\"TinyImgNet (C)\")\n",
    "timc_results = detector.compute_ood_deviations(tinyimagenet_c,POWERS=range(1,11))\n",
    "print(\"SVHN\")\n",
    "svhn_results = detector.compute_ood_deviations(svhn,POWERS=range(1,11))\n",
    "print(\"CIFAR-10\")\n",
    "c10_results = detector.compute_ood_deviations(cifar10,POWERS=range(1,11))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}