run_tests_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import numpy as np\n",
    "from pgd import PGD\n",
    "from dknn import AdvDKNN, DKNN\n",
    "from models.cnn import CNN\n",
    "from torchvision.datasets import MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN(\n",
       "  (f1): Sequential(\n",
       "    (0): Conv2d(1, 64, kernel_size=(8, 8), stride=(2, 2), padding=(3, 3))\n",
       "    (1): ReLU()\n",
       "    (2): Dropout(p=0.5, inplace=False)\n",
       "  )\n",
       "  (f2): Sequential(\n",
       "    (0): Conv2d(64, 128, kernel_size=(6, 6), stride=(2, 2))\n",
       "    (1): ReLU()\n",
       "    (2): Dropout(p=0.5, inplace=False)\n",
       "  )\n",
       "  (f3): Sequential(\n",
       "    (0): Conv2d(128, 128, kernel_size=(5, 5), stride=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): Dropout(p=0.5, inplace=False)\n",
       "  )\n",
       "  (classifier): Sequential(\n",
       "    (0): Flatten(start_dim=1, end_dim=-1)\n",
       "    (1): Linear(in_features=128, out_features=10, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root = \"./datasets\"\n",
    "model_weights = \"./model_weights/mnist_cnn_at.pt\"\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "download = not os.path.exists(os.path.join(root,\"MNIST\"))\n",
    "random_seed = 1234\n",
    "class_samp_size = 2000\n",
    "\n",
    "trainset = MNIST(root=root, train=True, download=download)\n",
    "testset = MNIST(root=root, train=False, download=download)\n",
    "\n",
    "np.random.seed(random_seed)\n",
    "subset = [\n",
    "    np.random.choice(\n",
    "        np.where(np.array(trainset.targets)==i)[0],\n",
    "        size=class_samp_size,\n",
    "        replace=False\n",
    "    ) for i in range(10)\n",
    "]\n",
    "train_data = torch.cat([\n",
    "    torch.FloatTensor(trainset.data[inds].unsqueeze(1)/255) for inds in subset\n",
    "], dim=0)\n",
    "train_targets = torch.arange(10).repeat_interleave(class_samp_size)\n",
    "\n",
    "model = CNN()\n",
    "model.load_state_dict(torch.load(model_weights, map_location=device))\n",
    "model.eval()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pgd = PGD(eps=80 / 255., step_size=20 / 255., max_iter=10, random_init=True, batch_size=128)\n",
    "\n",
    "x, y = (\n",
    "    torch.FloatTensor(testset.data.unsqueeze(1) / 255.)[:1000],\n",
    "    torch.LongTensor(testset.targets)[:1000]\n",
    ")\n",
    "\n",
    "x_adv = pgd.generate(model, x, y, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00,  2.22it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00,  2.23it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00,  2.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The benign accuracy by DKNN is 0.99\n",
      "The adversarial accuracy by DKNN is 0.588\n",
      "The benign accuracy by Adv-DKNN is 0.992\n",
      "The adversarial accuracy by Adv-DKNN is 0.6\n"
     ]
    }
   ],
   "source": [
    "dknn = DKNN(\n",
    "    model,\n",
    "    train_data,\n",
    "    train_targets,\n",
    "    hidden_layers=[2,],\n",
    "    device=device\n",
    ")\n",
    "advdknn = AdvDKNN(\n",
    "    model,\n",
    "    train_data,\n",
    "    train_targets,\n",
    "    hidden_layers=[2,],\n",
    "    device=device,\n",
    "    eps=60 / 255.,\n",
    "    step_size=20 / 255.,\n",
    "    max_iter=10,\n",
    "    random_init=True\n",
    ")\n",
    "\n",
    "pred_benign_dknn = dknn(x.to(device)).argmax(axis=1)\n",
    "acc_benign_dknn = (pred_benign_dknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The benign accuracy by DKNN is {acc_benign_dknn}\")\n",
    "\n",
    "pred_adv_dknn = dknn(x_adv.to(device)).argmax(axis=1)\n",
    "acc_adv_dknn = (pred_adv_dknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The adversarial accuracy by DKNN is {acc_adv_dknn}\")\n",
    "\n",
    "pred_benign_advdknn = advdknn(x.to(device)).argmax(axis=1)\n",
    "acc_benign_advdknn = (pred_benign_advdknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The benign accuracy by Adv-DKNN is {acc_benign_advdknn}\")\n",
    "\n",
    "pred_adv_advdknn = advdknn(x_adv.to(device)).argmax(axis=1)\n",
    "acc_adv_advdknn = (pred_adv_advdknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The adversarial accuracy by Adv-DKNN is {acc_adv_advdknn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00,  2.21it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00,  2.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The benign accuracy by Adv-DKNN is 0.99\n",
      "The adversarial accuracy by Adv-DKNN is 0.596\n"
     ]
    }
   ],
   "source": [
    "advdknn = AdvDKNN(\n",
    "    model,\n",
    "    train_data,\n",
    "    train_targets,\n",
    "    hidden_layers=[2,],\n",
    "    device=device,\n",
    "    eps=40 / 255.,\n",
    "    step_size=20 / 255.,\n",
    "    max_iter=10,\n",
    "    random_init=True\n",
    ")\n",
    "pred_benign_advdknn = advdknn(x.to(device)).argmax(axis=1)\n",
    "acc_benign_advdknn = (pred_benign_advdknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The benign accuracy by Adv-DKNN is {acc_benign_advdknn}\")\n",
    "\n",
    "pred_adv_advdknn = advdknn(x_adv.to(device)).argmax(axis=1)\n",
    "acc_adv_advdknn = (pred_adv_advdknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The adversarial accuracy by Adv-DKNN is {acc_adv_advdknn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:13<00:00,  6.57s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The benign accuracy by DKNN is 0.985\n",
      "The adversarial accuracy by DKNN is 0.683\n"
     ]
    }
   ],
   "source": [
    "dknn = DKNN(\n",
    "    model,\n",
    "    train_data,\n",
    "    train_targets,\n",
    "    hidden_layers=[1,2],\n",
    "    device=device\n",
    ")\n",
    "pred_benign_dknn = dknn(x.to(device)).argmax(axis=1)\n",
    "acc_benign_dknn = (pred_benign_dknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The benign accuracy by DKNN is {acc_benign_dknn}\")\n",
    "\n",
    "pred_adv_dknn = dknn(x_adv.to(device)).argmax(axis=1)\n",
    "acc_adv_dknn = (pred_adv_dknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The adversarial accuracy by DKNN is {acc_adv_dknn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:12<00:00,  6.10s/it]\n",
      "100%|██████████| 2/2 [00:11<00:00,  5.98s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The benign accuracy by Adv-DKNN is 0.983\n",
      "The adversarial accuracy by Adv-DKNN is 0.697\n"
     ]
    }
   ],
   "source": [
    "advdknn = AdvDKNN(\n",
    "    model,\n",
    "    train_data,\n",
    "    train_targets,\n",
    "    hidden_layers=[1,2],\n",
    "    device=device,\n",
    "    eps=60 / 255.,\n",
    "    step_size=20 / 255.,\n",
    "    max_iter=10,\n",
    "    random_init=True\n",
    ")\n",
    "pred_benign_advdknn = advdknn(x.to(device)).argmax(axis=1)\n",
    "acc_benign_advdknn = (pred_benign_advdknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The benign accuracy by Adv-DKNN is {acc_benign_advdknn}\")\n",
    "\n",
    "pred_adv_advdknn = advdknn(x_adv.to(device)).argmax(axis=1)\n",
    "acc_adv_advdknn = (pred_adv_advdknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The adversarial accuracy by Adv-DKNN is {acc_adv_advdknn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00,  2.24it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00,  2.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The benign accuracy by Adv-DKNN is 0.986\n",
      "The adversarial accuracy by Adv-DKNN is 0.598\n"
     ]
    }
   ],
   "source": [
    "advdknn = AdvDKNN(\n",
    "    model,\n",
    "    train_data,\n",
    "    train_targets,\n",
    "    hidden_layers=[2,],\n",
    "    weight=[0,0],\n",
    "    device=device,\n",
    "    eps=60 / 255.,\n",
    "    step_size=20 / 255.,\n",
    "    max_iter=10,\n",
    "    random_init=True\n",
    ")\n",
    "pred_benign_advdknn = advdknn(x.to(device)).argmax(axis=1)\n",
    "acc_benign_advdknn = (pred_benign_advdknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The benign accuracy by Adv-DKNN is {acc_benign_advdknn}\")\n",
    "\n",
    "pred_adv_advdknn = advdknn(x_adv.to(device)).argmax(axis=1)\n",
    "acc_adv_advdknn = (pred_adv_advdknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The adversarial accuracy by Adv-DKNN is {acc_adv_advdknn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00,  2.24it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00,  2.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The benign accuracy by Adv-DKNN is 0.989\n",
      "The adversarial accuracy by Adv-DKNN is 0.602\n"
     ]
    }
   ],
   "source": [
    "advdknn = AdvDKNN(\n",
    "    model,\n",
    "    train_data,\n",
    "    train_targets,\n",
    "    hidden_layers=[2,],\n",
    "    weight=[0,0],\n",
    "    device=device,\n",
    "    eps=40 / 255.,\n",
    "    step_size=20 / 255.,\n",
    "    max_iter=10,\n",
    "    random_init=True\n",
    ")\n",
    "pred_benign_advdknn = advdknn(x.to(device)).argmax(axis=1)\n",
    "acc_benign_advdknn = (pred_benign_advdknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The benign accuracy by Adv-DKNN is {acc_benign_advdknn}\")\n",
    "\n",
    "pred_adv_advdknn = advdknn(x_adv.to(device)).argmax(axis=1)\n",
    "acc_adv_advdknn = (pred_adv_advdknn == y.numpy()).astype(\"float\").mean()\n",
    "print(f\"The adversarial accuracy by Adv-DKNN is {acc_adv_advdknn}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}