basic/MNIST From Scratch.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python385jvsc74a57bd031f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6",
   "display_name": "Python 3.8.5 64-bit"
  },
  "metadata": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.data.core import DataLoaders\n",
    "from fastai.learner import Learner\n",
    "from fastai.metrics import accuracy\n",
    "from fastai.vision.all import torch, tensor, untar_data, Image, URLs, DataLoader, Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_param(size):\n",
    "    return torch.randn(size).requires_grad_()"
   ]
  },
  {
   "source": [
    "# Layers"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear:\n",
    "    def __init__(self, in_, out_):\n",
    "        self.w = init_param((in_, out_))\n",
    "        self.b = init_param(1)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        return x @ self.w + self.b\n",
    "\n",
    "    @property\n",
    "    def params(self):\n",
    "        return [self.w, self.b]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReLU:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def __call__(self, x):\n",
    "        return torch.where(x > 0.0, x, torch.zeros_like(x))"
   ]
  },
  {
   "source": [
    "# Sequential API"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sequential:\n",
    "    def __init__(self, *args):\n",
    "        self.layers = list(args)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        y = x\n",
    "        for lay in self.layers:\n",
    "            y = lay(y)\n",
    "        return y\n",
    "\n",
    "    @property\n",
    "    def params(self):\n",
    "        p_ = []\n",
    "\n",
    "        for layer in self.layers:\n",
    "            if hasattr(layer, \"params\"):\n",
    "                p_ += layer.params\n",
    "\n",
    "        return p_"
   ]
  },
  {
   "source": [
    "# Optimizer"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SGD:\n",
    "    def __init__(self, params, lr):\n",
    "        self.params = list(params)\n",
    "        self.lr = lr\n",
    "\n",
    "    def step(self):\n",
    "        for param in self.params:\n",
    "            param.data -= param.grad.data * self.lr\n",
    "\n",
    "    def zero_grad(self):\n",
    "        for param in self.params:\n",
    "            param.grad = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mnist_loss(preds, targets):\n",
    "    preds = preds.sigmoid()\n",
    "    return torch.where(targets == 1, 1 - preds, preds).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def acc(preds, targets):\n",
    "    preds = preds.sigmoid()\n",
    "    return ((preds > 0.5) == targets).float().mean().item()"
   ]
  },
  {
   "source": [
    "# Trainer"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Trainer:\n",
    "    def __init__(self, train_dl, val_dl, model, lr, optim_cls, loss_fn):\n",
    "        self.train_dl = train_dl\n",
    "        self.val_dl = val_dl\n",
    "        self.model = model\n",
    "        self.optim = optim_cls(self.model.params, lr)\n",
    "        self.loss_fn = loss_fn\n",
    "\n",
    "    def _calc_grad(self, x, y):\n",
    "        out = self.model(x)\n",
    "        loss = self.loss_fn(out, y)\n",
    "        loss.backward()\n",
    "\n",
    "        return loss.detach().cpu().item()\n",
    "\n",
    "    def _train_step(self, *args, **kwargs):\n",
    "        losses = []\n",
    "        for x, y in self.train_dl:\n",
    "            losses.append(self._calc_grad(x, y))\n",
    "            self.optim.step()\n",
    "            self.optim.zero_grad()\n",
    "\n",
    "        return tensor(losses).mean().item()\n",
    "\n",
    "    def _val_step(self, *args, **kwargs):\n",
    "        preds = []\n",
    "        targets = []\n",
    "\n",
    "        with torch.no_grad():\n",
    "            for x, y in self.val_dl:\n",
    "                preds.append(self.model(x))\n",
    "                targets.append(y)\n",
    "\n",
    "        preds = torch.cat(preds).float()\n",
    "        targets = torch.cat(targets).float()\n",
    "\n",
    "        return (\n",
    "            self.loss_fn(preds, targets).item(),\n",
    "            acc(preds.reshape(-1), targets.reshape(-1)),\n",
    "        )\n",
    "\n",
    "    def fit(self, epochs):\n",
    "        for eph in range(epochs):\n",
    "            train_loss = self._train_step()\n",
    "            val_loss, val_acc = self._val_step()\n",
    "            print(\n",
    "                f\"Epoch [{eph}]: train_loss: {round(train_loss, 4)}, val_loss: {round(val_loss, 4)}, val_acc:{round(val_acc, 4)}\"\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "/home/himanshu/.fastai/data/mnist_sample\n",
      "Epoch [0]: train_loss: 0.5055, val_loss: 0.5044, val_acc:0.4975\n",
      "Epoch [1]: train_loss: 0.5053, val_loss: 0.5044, val_acc:0.4971\n",
      "Epoch [2]: train_loss: 0.5054, val_loss: 0.5044, val_acc:0.4961\n",
      "Epoch [3]: train_loss: 0.5053, val_loss: 0.5043, val_acc:0.4975\n",
      "Epoch [4]: train_loss: 0.5053, val_loss: 0.5039, val_acc:0.4877\n",
      "Epoch [5]: train_loss: 0.5065, val_loss: 0.5043, val_acc:0.499\n",
      "Epoch [6]: train_loss: 0.5053, val_loss: 0.5044, val_acc:0.4995\n",
      "Epoch [7]: train_loss: 0.5052, val_loss: 0.5043, val_acc:0.501\n",
      "Epoch [8]: train_loss: 0.5047, val_loss: 0.5042, val_acc:0.5044\n",
      "Epoch [9]: train_loss: 0.5053, val_loss: 0.5043, val_acc:0.5015\n",
      "Epoch [10]: train_loss: 0.5055, val_loss: 0.5043, val_acc:0.502\n",
      "Epoch [11]: train_loss: 0.5052, val_loss: 0.5044, val_acc:0.4985\n",
      "Epoch [12]: train_loss: 0.5051, val_loss: 0.5043, val_acc:0.5005\n",
      "Epoch [13]: train_loss: 0.5051, val_loss: 0.5043, val_acc:0.502\n",
      "Epoch [14]: train_loss: 0.5047, val_loss: 0.5041, val_acc:0.5049\n",
      "Epoch [15]: train_loss: 0.497, val_loss: 0.4995, val_acc:0.6609\n",
      "Epoch [16]: train_loss: 0.487, val_loss: 0.4965, val_acc:0.5952\n",
      "Epoch [17]: train_loss: 0.4923, val_loss: 0.4959, val_acc:0.5324\n",
      "Epoch [18]: train_loss: 0.4909, val_loss: 0.497, val_acc:0.6541\n",
      "Epoch [19]: train_loss: 0.4886, val_loss: 0.4974, val_acc:0.6914\n",
      "Epoch [20]: train_loss: 0.4926, val_loss: 0.4957, val_acc:0.5123\n",
      "Epoch [21]: train_loss: 0.4887, val_loss: 0.4977, val_acc:0.7233\n",
      "Epoch [22]: train_loss: 0.4812, val_loss: 0.4982, val_acc:0.7753\n",
      "Epoch [23]: train_loss: 0.4783, val_loss: 0.4991, val_acc:0.8499\n",
      "Epoch [24]: train_loss: 0.4794, val_loss: 0.5001, val_acc:0.8871\n",
      "Epoch [25]: train_loss: 0.4762, val_loss: 0.4998, val_acc:0.8935\n",
      "Epoch [26]: train_loss: 0.4741, val_loss: 0.4997, val_acc:0.8989\n",
      "Epoch [27]: train_loss: 0.475, val_loss: 0.5001, val_acc:0.9097\n",
      "Epoch [28]: train_loss: 0.4741, val_loss: 0.5008, val_acc:0.8783\n",
      "Epoch [29]: train_loss: 0.4731, val_loss: 0.5001, val_acc:0.922\n",
      "Epoch [30]: train_loss: 0.4731, val_loss: 0.5001, val_acc:0.9249\n",
      "Epoch [31]: train_loss: 0.4721, val_loss: 0.5, val_acc:0.9269\n",
      "Epoch [32]: train_loss: 0.4735, val_loss: 0.5, val_acc:0.9298\n",
      "Epoch [33]: train_loss: 0.4725, val_loss: 0.4999, val_acc:0.9333\n",
      "Epoch [34]: train_loss: 0.4731, val_loss: 0.4998, val_acc:0.9367\n",
      "Epoch [35]: train_loss: 0.4719, val_loss: 0.5, val_acc:0.9387\n",
      "Epoch [36]: train_loss: 0.4741, val_loss: 0.5002, val_acc:0.9342\n",
      "Epoch [37]: train_loss: 0.4721, val_loss: 0.5002, val_acc:0.9392\n",
      "Epoch [38]: train_loss: 0.4738, val_loss: 0.4999, val_acc:0.9495\n",
      "Epoch [39]: train_loss: 0.4711, val_loss: 0.5001, val_acc:0.946\n"
     ]
    }
   ],
   "source": [
    "# download the data\n",
    "path = untar_data(URLs.MNIST_SAMPLE)\n",
    "print(path)\n",
    "\n",
    "Path.BASE_PATH = path\n",
    "\n",
    "# load and convert the images to float tensors\n",
    "train_3 = (\n",
    "    torch.stack(\n",
    "        [tensor(Image.open(pth)) for pth in (path / \"train\" / \"3\").ls()]\n",
    "    ).float()\n",
    "    / 255\n",
    ")\n",
    "train_7 = (\n",
    "    torch.stack(\n",
    "        [tensor(Image.open(pth)) for pth in (path / \"train\" / \"7\").ls()]\n",
    "    ).float()\n",
    "    / 255\n",
    ")\n",
    "\n",
    "val_3 = (\n",
    "    torch.stack(\n",
    "        [tensor(Image.open(pth)) for pth in (path / \"valid\" / \"3\").ls()]\n",
    "    ).float()\n",
    "    / 255\n",
    ")\n",
    "val_7 = (\n",
    "    torch.stack(\n",
    "        [tensor(Image.open(pth)) for pth in (path / \"valid\" / \"7\").ls()]\n",
    "    ).float()\n",
    "    / 255\n",
    ")\n",
    "\n",
    "# training and validation dataset\n",
    "train_X = torch.cat([train_3, train_7]).reshape(-1, 28 * 28)\n",
    "train_y = tensor([1.0] * len(train_3) + [0.0] * len(train_7))\n",
    "\n",
    "val_X = torch.cat([val_3, val_7]).reshape(-1, 28 * 28)\n",
    "val_y = tensor([1.0] * len(val_3) + [0.0] * len(val_7))\n",
    "\n",
    "# dataloaders\n",
    "train_dl = DataLoader(\n",
    "    list(zip(train_X, train_y)), bs=16, num_workers=4, shuffle=True\n",
    ")\n",
    "val_dl = DataLoader(list(zip(val_X, val_y)), bs=32, num_workers=4)\n",
    "\n",
    "dls = DataLoaders(train_dl, val_dl)\n",
    "\n",
    "model = Sequential(\n",
    "    Linear(28 * 28, 32),\n",
    "    ReLU(),\n",
    "    Linear(32, 64),\n",
    "    ReLU(),\n",
    "    Linear(64, 1),\n",
    ")\n",
    "\n",
    "lrn = Trainer(train_dl, val_dl, model, 9e-2, SGD, mnist_loss)\n",
    "lrn.fit(40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}