experiments/SuperResolution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bf2bedf9-edc9-4514-b7a1-2b10a58832e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/vvrbeast/Desktop/Giannis/Code/SPVD\n"
     ]
    }
   ],
   "source": [
    "# Add the parent directory to the path so that we can import the necessary modules\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b2d7c587-40e5-43a0-848f-fd8dd311fc81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Jupyter environment detected. Enabling Open3D WebVisualizer.\n",
      "[Open3D INFO] WebRTC GUI backend enabled.\n",
      "[Open3D INFO] WebRTCWindowSystem: HTTP handshake server disabled.\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "from datasets.shapenet_pointflow import ShapeNet15kPointClouds\n",
    "from datasets.utils import NoiseSchedulerDDPM\n",
    "from torchsparse.utils.quantize import sparse_quantize\n",
    "from torchsparse import SparseTensor\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "from pclab.utils import DataLoaders\n",
    "from torchsparse.utils.collate import sparse_collate_fn\n",
    "\n",
    "from pclab.learner import *\n",
    "\n",
    "from functools import partial\n",
    "from utils.visualization import quick_vis_batch\n",
    "vis_batch = partial(quick_vis_batch, x_offset = 8, y_offset=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f4132cc6-b444-4c85-999f-55223cbef423",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ShapaNet15kNoisySparseSuperRes(ShapeNet15kPointClouds):\n",
    "\n",
    "    def __init__(self, root_dir='./data/ShapeNetCore.v2.PC15k', categories=['chair'], pres=1e-5, #, 'car', 'airplane', 'table', 'bag'\n",
    "                 noise_params={'beta_min':0.0001 ,'beta_max':0.02 , 'n_steps':1000},\n",
    "                 high_res = 2048, low_res = 1024,\n",
    "                 split='train',\n",
    "                 random_subsample=False):\n",
    "        super().__init__(root_dir=root_dir,\n",
    "                 categories=categories,\n",
    "                 tr_sample_size=high_res, \n",
    "                 te_sample_size=low_res, # no using this\n",
    "                 split=split, scale=1., normalize_per_shape=False,\n",
    "                 normalize_std_per_axis=False,\n",
    "                 random_subsample=random_subsample)\n",
    "\n",
    "        self.high_res, self.low_res = high_res, low_res\n",
    "        self.pres = pres\n",
    "        self.noise_scheduler = NoiseSchedulerDDPM(beta_min=noise_params['beta_min'],\n",
    "                                                  beta_max=noise_params['beta_max'],\n",
    "                                                  n_steps =noise_params['n_steps'])\n",
    "\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        res = super().__getitem__(idx)\n",
    "\n",
    "        # get original points\n",
    "        pts = res['train_points']\n",
    "\n",
    "        # add noise to points\n",
    "        noisy_pts, t, noise = self.noise_scheduler(pts)\n",
    "\n",
    "        # keep the 'low_res_size' points without noise\n",
    "        real_pts = pts[:self.low_res]\n",
    "        noisy_pts = noisy_pts[self.low_res:]\n",
    "        all_pts = torch.cat([real_pts, noisy_pts], dim=0)\n",
    "        # create a mask to points at real points\n",
    "        mask = torch.ones(len(all_pts), dtype=torch.bool)\n",
    "        mask[self.low_res:] = False\n",
    "\n",
    "        # turn to numpy to pass through sparse_quantize #TODO: Use GPU quantize\n",
    "        all_pts = all_pts.numpy()\n",
    "\n",
    "        # calculate sparse coordinates\n",
    "        coords = all_pts - np.min(all_pts, axis=0, keepdims=True)\n",
    "        coords, indices = sparse_quantize(coords, self.pres, return_index=True)\n",
    "\n",
    "        # back to torch tensors\n",
    "        coords = torch.tensor(coords, dtype=torch.int)\n",
    "        all_pts = torch.tensor(all_pts)\n",
    "        all_pts = torch.cat([all_pts, mask.unsqueeze(-1)], dim=-1) # add the information of which points are from the original pc\n",
    "        feats = all_pts[indices]\n",
    "        noise = noise[indices]\n",
    "        mask = mask[indices]\n",
    "        \n",
    "        # to sparse tensors\n",
    "        noisy_pc = SparseTensor(coords=coords, feats=feats)\n",
    "        \n",
    "        return {\n",
    "            'noisy_pc':noisy_pc,\n",
    "            't': t,\n",
    "            'noise': noise,\n",
    "            'mask': mask\n",
    "        }\n",
    "\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6c60f9f5-8397-4967-abc0-fc4112aa06dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 1, 1)\n",
      "Total number of data:4612\n",
      "Min number of points: (train)2048 (test)512\n",
      "(1, 1, 1)\n",
      "Total number of data:662\n",
      "Min number of points: (train)2048 (test)512\n"
     ]
    }
   ],
   "source": [
    "path = '/home/vvrbeast/Desktop/Giannis/Data/ShapeNetCore.v2.PC15k'\n",
    "\n",
    "tr_dataset = ShapaNet15kNoisySparseSuperRes(root_dir=path, low_res=512, split='train')\n",
    "vl_dataset = ShapaNet15kNoisySparseSuperRes(root_dir=path, low_res=512, split='val')\n",
    "\n",
    "tr_dl = DataLoader(tr_dataset, batch_size=32, collate_fn=sparse_collate_fn, drop_last=True, shuffle=True)\n",
    "vl_dl = DataLoader(vl_dataset, batch_size=32, collate_fn=sparse_collate_fn, drop_last=True, shuffle=True)\n",
    "\n",
    "dls = DataLoaders(tr_dl, vl_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "edc37a20-3026-4c60-b581-528dc69b908e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['noisy_pc', 't', 'noise', 'mask'])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch = next(iter(tr_dl))\n",
    "batch.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a43c1294-d7ce-413d-98ee-a2fbbd87d1dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 512, 3])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_real = batch['noisy_pc'].F[batch['mask'].view(-1)][:, :3].reshape(32, -1, 3)\n",
    "batch_real.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "331a2706-74ab-4d0b-a603-79e2d48d6ba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "vis_batch(batch_real)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15951e41-4c56-46dc-adc3-79c79b792162",
   "metadata": {},
   "source": [
    "# Train Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "88a014ab-5733-43b7-85c0-9de36d43aab1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DDPMCB(Callback):\n",
    "\n",
    "    def before_batch(self, learn):\n",
    "        pts = learn.batch['noisy_pc']\n",
    "        t = torch.tensor(learn.batch['t'])\n",
    "        inp = (pts, t)\n",
    "        target = learn.batch['noise'], learn.batch['mask'].view(-1)\n",
    "        learn.batch = (inp, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0573dcf1-53c5-42f7-ac21-875ff5474672",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pclab.utils import def_device\n",
    "import fastcore.all as fc\n",
    "from typing import Mapping\n",
    "from copy import copy\n",
    "from torcheval.metrics import Mean\n",
    "from utils.callbacks import GradientClipCB\n",
    "from utils.callbacks import CheckpointCB\n",
    "import torchsparse\n",
    "\n",
    "def to_device(x, device=def_device):\n",
    "    if isinstance(x, (torch.Tensor, torchsparse.SparseTensor)): return x.to(device)\n",
    "    if isinstance(x, Mapping): return {k:v.to(device) for k,v in x.items()}\n",
    "    return type(x)(to_device(o, device) for o in x)\n",
    "\n",
    "class DeviceCBSparse(Callback):\n",
    "    order = DDPMCB.order + 1\n",
    "    def __init__(self, device=def_device): fc.store_attr()\n",
    "    def before_fit(self, learn):\n",
    "        if hasattr(learn.model, 'to'): learn.model.to(self.device)\n",
    "    def before_batch(self, learn): learn.batch = to_device(learn.batch, device=self.device)\n",
    "\n",
    "\n",
    "class LossCB(Callback):\n",
    "    def __init__(self, *ms, **metrics):\n",
    "        for o in ms: metrics[type(o).__name__] = o\n",
    "        self.metrics = metrics\n",
    "        self.all_metrics = copy(metrics)\n",
    "        self.all_metrics['loss'] = self.loss = Mean()\n",
    "\n",
    "    def _log(self, d): print(d)\n",
    "    def before_fit(self, learn): learn.metrics = self\n",
    "    def before_epoch(self, learn): [o.reset() for o in self.all_metrics.values()]\n",
    "\n",
    "    def after_epoch(self, learn):\n",
    "        log = {k:f'{v.compute():.3f}' for k,v in self.all_metrics.items()}\n",
    "        log['epoch'] = learn.epoch\n",
    "        log['train'] = 'train' if learn.model.training else 'eval'\n",
    "        self._log(log)\n",
    "\n",
    "    def after_batch(self, learn):\n",
    "        x,y,*_ = learn.batch\n",
    "        #for m in self.metrics.values(): m.update(to_cpu(learn.preds), y)\n",
    "        self.loss.update(to_cpu(learn.loss), weight=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7a266f71-6307-4401-a190-1af6a1dc7a3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomMSELoss(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.loss_fn = nn.MSELoss()\n",
    "    \n",
    "    def forward(self, preds, target):\n",
    "\n",
    "        noise, mask = target\n",
    "\n",
    "        # calculate loss only for the noisy points\n",
    "        preds = preds[~mask, :3]\n",
    "        \n",
    "        noise = noise.view(-1, 3)[~mask]\n",
    "\n",
    "        return self.loss_fn(preds, noise)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bf03a89-2c11-483f-a172-0ac3c326b1a3",
   "metadata": {},
   "source": [
    "# Training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3576aa69-ab4b-4ac2-8a26-afa9f156710a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.spvd import SPVUnet\n",
    "get_model = partial(SPVUnet, point_channels=4, \n",
    "                    down_blocks = [[(64, 128, 192, 256, 256), (True, True, True, False), (None, None, None, None)]], # only one point skip connection during downsampling\n",
    "                    up_blocks   = [[(256, 256, 192), (True, True), (None, None), (3, 3)], \n",
    "                                    [(192, 128, 64), (True, False), (None, None), (3, 3)]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2445ebda-a0f5-43a3-9c1c-1c86783daf64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGhCAYAAACzurT/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABQfUlEQVR4nO3deXhTZfYH8O9N0nTf95aWUqAUKLSslUVZrIAobuMy6siio47iMjJujCOMOoo66PibEcVBEXXGEVdEQRAKiEiVteyl0FJa6L6le9Ik9/dHlhZpS5Mmvbnp9/M8fZ4hvUlO8U5zeN9zziuIoiiCiIiISCIKqQMgIiKivo3JCBEREUmKyQgRERFJiskIERERSYrJCBEREUmKyQgRERFJiskIERERSUoldQDdYTQaUVxcDH9/fwiCIHU4RERE1A2iKKK+vh4xMTFQKDpf/5BFMlJcXIy4uDipwyAiIiI7FBUVoV+/fp1+XxbJiL+/PwDTDxMQECBxNERERNQddXV1iIuLs36Od0YWyYhlayYgIIDJCBERkcxcqsSCBaxEREQkKSYjREREJCkmI0RERCQpJiNEREQkKSYjREREJCkmI0RERCQpJiNEREQkKSYjREREJCkmI0RERCQpJiNEREQkKSYjREREJCkmI0RERCSpPp2MaPUGbDpaInUYREREfVqfTUZaWg24+v9+xB/+cwBZeVVSh0NERNRn9dlkxMtDiUkDwwAAz31zDAajKHFEREREfVOfTUYAYNFVSQj09kBOaT0+2VsodThERER9Up9ORoJ91fhjxmAAwGvf50LT3CpxRERERH1Pn05GAOB3l/XHoAg/VDfq8M/MU1KHQ0RE1Of0+WTEQ6nAs9cOAwB8sLsAeRUNEkdERETUt/T5ZAQApiSF48rkCOiNIv727XGpwyEiIupTmIyYPXPNUHgoBWw/WYHtJ8ulDoeIiKjPYDJilhjuh/kTEwAAL3x7HK0Go7QBERER9RFMRtp5+MrBCPVVI7+iER9mnZU6HCIioj6ByUg7AV4eeHzmEADAG1tzUdWglTgiIiIi98dk5FduHRuHYdEBqG/R4/UtuVKHQ0RE5PaYjPyKUiFg6RxTq+//9hTiREmdxBERERG5NyYjHUhPDMU1I6JhFIHnvzkOUeS5NURERM7CZKQTT1+dDLVKgaz8Kmw+ViZ1OERERG6LyUgn4kJ8cP8ViQCAFzceR0urQeKIiIiI3BOTkS48MHUgogK8UFTdjPd2nZE6HCIiIrfEZKQLPmoVnrra1Oq7YvtplNW1SBwRERGR+2EycgnXp8ZiVHwQmnQGvLrppNThEBERuR0mI5egUAhYOmc4AOCLA+dwqKhW2oCIiIjcDJORbkiLC8JNo2MBAM99c4ytvkRERA7EZKSbnpqVDB+1EgcKa7H+ULHU4RAREbkNJiPdFBnghYXTBgEAlm3MQZNOL3FERERE7oHJiA3umTwA/YK9UVrXgpU/5EsdDhERkVtgMmIDLw8lnpk9FADwzg95OFfTJHFERERE8sdkxEazUqKQPiAEWr0Ry77LkTocIiIi2WMyYiNBELBkzjAoBGDD4RLsOVMtdUhERESyxmTEDsNjAvHb8fEATK2+BiNbfYmIiOzFZMROf7oqCf5eKhwrrsPavUVSh0NERCRbTEbsFOrniT9mJAEAXt2cg+pGncQRERERyROTkR6YN6E/kqP8UdvUilc3sZiViIjIHkxGekClVOCFG1IAAJ/sLcKBwhqJIyIiIpIfJiM9NC4hBL8Z3Q8A8Oy6oyxmJSIishGTEQdYPDsZAeZi1v/8fFbqcIiIiGSFyYgDhPl54omZQwAAy78/iYp6rcQRERERyQeTEQe5I70/RsQGor5Fj2UbT0gdDhERkWwwGXEQpULACzekQBCALw+exy/5VVKHREREJAtMRhwoLS4Ivx1nmsz67NdH0WowShwRERFR1259JwvTl+9AdlGtZDEwGXGwJ2cOQbCPB3LLGrDmpwKpwyEiIurS2apG5Fc2QqUQJIuByYiDBfuq8fTVyQCAN7bmolTTInFEREREnattagUABHp7SBYDkxEnuGVMHEbHB6FRZ8ALG45LHQ4REVGHmnUGaPWmkoJgX7VkcTAZcQKFuZhVIQAbDpdg16lKqUMiIiK6SG2z6Vw1lUKAr1opWRxMRpxkeEwg5k5IAAAs+footHqDtAERERH9imWLJsjHA4Igo5qRnTt3Ys6cOYiJiYEgCFi3bt0ln7Njxw6MHj0anp6eGDRoENasWWNHqPKzaEYSwv09kV/ZiHd/PCN1OERERBeoaTKtjAT5SLdFA9iRjDQ2NiI1NRUrVqzo1vVnzpzBNddcg2nTpiE7Oxt//OMf8fvf/x6bN2+2OVi5CfDywDOzhwIA/rXtFIqqmySOiIiIqI3GsjIiYfEqAKhsfcLVV1+Nq6++utvXr1y5EgMGDMBrr70GABg6dCh27dqFf/zjH5g5c6atby8716fF4JO9hfg5vxrPf3scq+aOlTokIiIiAEBtc9s2jZScXjOSlZWFjIyMCx6bOXMmsrKyOn2OVqtFXV3dBV9yJQgCXrg+BSqFgC3Hy7Atp0zqkIiIiADIeJvGVqWlpYiMjLzgscjISNTV1aG5ubnD5yxbtgyBgYHWr7i4OGeH6VSDI/1xz+QBAICl64+hpZXFrEREJD1X2aZxyW6axYsXQ6PRWL+KioqkDqnHHrlyMKIDvVBU3Yy3duRJHQ4REdEF3TRScnoyEhUVhbKyC7cmysrKEBAQAG9v7w6f4+npiYCAgAu+5M7XU4Vnrx0GAFj5Qx4KKhsljoiIiPq6PrNNM2HCBGRmZl7w2JYtWzBhwgRnv7XLuTolCpcPDoNOb8TS9ccgiqLUIRERUR8m2wLWhoYGZGdnIzs7G4CpdTc7OxuFhYUATFssc+fOtV7/hz/8Afn5+XjyySeRk5ODt956C59++ikee+wxx/wEMiIIAp6/PgVqpQI/5FZg87FSqUMiIqI+rK1mRGYrI/v27cOoUaMwatQoAMCiRYswatQoLFmyBABQUlJiTUwAYMCAAdiwYQO2bNmC1NRUvPbaa3j33Xf7RFtvRwaE+eL+KYkAgOe/OY4mnV7iiIiIqK9q26aRdmVEEGWwV1BXV4fAwEBoNBq3qB9p1hlw1T9+wLmaZvxhykDrKb9ERES9RRRFDHl2E3R6I3Y9NQ39gn0c/h7d/fx2yW4ad+etVuKvc4YDAN79MR+ny+sljoiIiPqallYjdOYTe92+gJU6ljEsEhlDI6A3ili+OVfqcIiIqI+xbNF4KKU9sRdgMiKpp2YlQxCATcdKcbxYvlNmiYhIfiwzRgK91ZKe2AswGZHU4Eh/XDsyBgDwz8xTEkdDRER9SW2zaxSvAkxGJPfI9EHW1ZETJVwdISKi3lHrIqPgASYjkhsc6Y9rRkQD4OoIERH1nrZR8NIWrwJMRlzCI1cOhiAA3x3l6ggREfUObtPQBZK4OkJERL2M2zR0Ea6OEBFRb6o1t/YG+3KbhsySIv0x27w68q9tXB0hIiLnamvt5coItfPIdNPqyMYjpcgp5eoIERE5T1sBK5MRamdIVNvqCGtHiIjImSwFrMHspqFf4+oIERH1Bm7TUKeGRPljdgpXR4iIyHlEUURtM7dpqAuPXDkYgGl15GQpT/QlIiLHam41WE/s5TYNdWhIFOeOEBGR81i2aDyUAnwkPrEXYDLisiyrIxuOlHB1hIiIHMqVTuwFmIy4LFNnTRQAro4QEZFjWQeeuUC9CMBkxKVZa0eOcnWEiIgcx5WKVwEmIy4tOSoAs0dEQRSBf3IqKxEROUj7bRpXwGTExbV11nB1hIiIHKOG2zRki+SoAFydwtURIiJyHA23achW7VdHcsu4OkJERD1jKWANcoEZIwCTEVkYGt1udYSdNURE1EM1LnRIHsBkRDbazx3h6ggREfWExpKMsICVbDE0OgCzhnN1hIiIes5yYi9XRshm7VdHTnF1hIiI7MRtGrLbsJh2qyPbTksdDhERyZAoim3bNCxgJXtYVke+PVzM1REiIrJZc6sBOoPpxN4gb66MkB2GxQRg5vBIro4QEZFdLFs0aqXCJU7sBZiMyFL71ZHT5VwdISKi7rPMGAn08XCJE3sBJiOyNDwmsG11JJOrI0RE1H1tbb2usUUDMBmRLcvqyDdcHSEiIhtYtmmCXaR4FWAyIlvDYwIxY5hpdeTfO/OlDoeIiGTCMmMk0EXaegEmI7J2z+QBAIDvjpSipdUgcTRERCQHtdymIUcalxCC2CBv1Gv12HqiTOpwiIhIBiwFrMG+3KYhB1AoBNwwKgYAsO7geYmjISIiObCsjARyZYQc5cZRsQCAHScrUNWglTgaIiJydbXNrjUKHmAyInuDIvwxIjYQeqOIDUdKpA6HiIhcnHWbht005Eg3mFdHvuJWDRERXQILWMkprkuNgVIh4GBhLQoqG6UOh4iIXJhlm4atveRQ4f6emDwoDABXR4iIqHOiKHKbhpzHUsi6Lvs8RFGUOBoiInJFTToDWg2mzwgWsJLDzRgeCR+1EmermnCwqFbqcIiIyAVZtmjUSgW8PVzjxF6AyYjb8FGrMGt4FADgqwPcqiEioovVNJq2aIJc6MRegMmIW7F01Xx7uBg6vVHiaIiIyNVoXHDGCMBkxK1MHBiKcH9P1DS1YmduhdThEBGRi2lr63Wd4lWAyYhbUSkVuD7VNB6eXTVERPRrNU1t2zSuhMmIm7Fs1Ww5UYa6llaJoyEiIlfCbRrqFcNjAjA4wg86vRGbjpRKHQ4REbmQWuvKCLdpyIkEQcCNozkenoiILlbTxJUR6iXXp5mSkZ/PVKG4tlniaIiIyFWwgJV6TWyQN9IHhEAUga+zi6UOh4iIXISmmQWs1ItutJ7ke47j4YmICEC7bRoXOrEXYDLitq4eEQ21SoHcsgYcL6mTOhwiInIB1m0aFrBSbwj09kDG0AgAwDoWshIR9XmiKHKbhnrfDeZC1q+zi2EwcquGiKgva3TRE3sBJiNubeqQCAT5eKC8XovdeZVSh0NERBKyzBhRq1zrxF6AyYhbU6sUuHZkNADOHCEi6utq2xWvutKJvQCTEbdn6arZfLQUTTq9xNEQEZFUal104BnAZMTtjY4PRnyIDxp1Bmw5XiZ1OEREJJHaZtccBQ8wGXF7giBYD8/jVg0RUd9V66IzRgAmI33CDWkxAIAfT1Wiol4rcTRERCQFVz2xF2Ay0ickhvshNS4IBqOIbw9zPDwRUV9U02japgnmNg1J5SZu1RAR9Wm15pWRQK6MkFSuHRkNpULA4XManC5vkDocIiLqZa56Yi9gZzKyYsUKJCQkwMvLC+np6dizZ0+X17/xxhsYMmQIvL29ERcXh8ceewwtLS12BUz2CfXzxJSkcADA19lcHSEi6mssQ8+C3WFlZO3atVi0aBGWLl2KAwcOIDU1FTNnzkR5eXmH13/88cd4+umnsXTpUpw4cQLvvfce1q5diz//+c89Dp5sc2O7rRqe5EtE1Le41TbN66+/jnvvvRcLFizAsGHDsHLlSvj4+GD16tUdXr97925MmjQJd9xxBxISEjBjxgzcfvvtl1xNIcfLGBoJP08VztU0Y9/ZGqnDISKiXuQ22zQ6nQ779+9HRkZG2wsoFMjIyEBWVlaHz5k4cSL2799vTT7y8/OxceNGzJ49u9P30Wq1qKuru+CLes5brcSslCgALGQlIupLRFFs26bxlfnKSGVlJQwGAyIjIy94PDIyEqWlpR0+54477sDzzz+PyZMnw8PDAwMHDsTUqVO73KZZtmwZAgMDrV9xcXG2hEldsGzVbDhcAq3eIHE0RETUGxp1BujNp7fLfmXEHjt27MBLL72Et956CwcOHMCXX36JDRs24IUXXuj0OYsXL4ZGo7F+FRUVOTvMPuOyxFBEBXhB09yK7TkVUodDRES9oP2JvV4ertdIq7Ll4rCwMCiVSpSVXXjGSVlZGaKiojp8zrPPPou77roLv//97wEAI0aMQGNjI+677z4888wzUCgu/kvx9PSEp6enLaFRNykVAq5Pi8E7O/Ox7uB567YNERG5L0u9SLCP653YC9i4MqJWqzFmzBhkZmZaHzMajcjMzMSECRM6fE5TU9NFCYdSqQQAdnRIxHJWzbaccmjMNygREbkvVy5eBezYplm0aBFWrVqFDz74ACdOnMADDzyAxsZGLFiwAAAwd+5cLF682Hr9nDlz8Pbbb+OTTz7BmTNnsGXLFjz77LOYM2eONSmh3jU0OgDJUf7QGYzYcKRE6nCIiMjJLCf2umJbL2DjNg0A3HbbbaioqMCSJUtQWlqKtLQ0bNq0yVrUWlhYeMFKyF/+8hcIgoC//OUvOH/+PMLDwzFnzhy8+OKLjvspyGY3jorFsu9ysO7gedyRHi91OERE5EQ17bZpXJEgymCvpK6uDoGBgdBoNAgICJA6HLdQomnGxJe3QRSBH5+chrgQH6lDIiIiJ3lz2yks/z4Xt42Nwys3j+y19+3u57frldRSr4gO9MbEgaEAgPWHeJIvEZE7s9aMuOjKCJORPuyGNI6HJyLqC2qsyYibFLCS+5iVEgW1SoHT5Q04XsIpt0RE7kpjLmDlygi5HH8vD2QMjQAAfJ3NrRoiInfV1trLZIRc0HWppq2a9dnFMBq5VUNE5I5qmiwrI9ymIRc0LTkc/l4qlNa1YE9BtdThEBGRE2iaWcBKLsxTpcTslGgAwNfZPMmXiMjdmE7sZTJCLu76tBgAwMYjpTzJl4jIzTRo9dYTe4O5TUOuKj0xFJEBntA0t2JnbqXU4RARkQNZVkU8VQp4ebjmMSxMRghKhYA5I02rI+u4VUNE5FZcvV4EYDJCZtebB6BtPV6GBq1e4miIiMhRLJ00rrpFAzAZIbOU2AAkhvtCqzfi+2OlUodDREQOYtmmCXTRGSMAkxEyEwQB15tnjqzjADQiIrdRy20akhNLV81PpytRUa+VOBoiInKE2kZu05CMJIT5IjUuCAajiI1HSqQOh4iIHMCyMhLIlRGSi+tT2VVDRORO2s6l4coIycS1qdFQCMDBwloUVjVJHQ4REfVQrbWbhisjJBMR/l6YNCgMAMfDExG5Axawkixd126rRhR5ki8RkZxZVkYCuU1DcjIrJQpqlQJ5FY04XlIndThERNQDlpqRYF+ujJCM+Ht5IGNoBADga84cISKSLVEU27ZpuDJCcnOdeQDa+uxiGI3cqiEikqMGrR4G8+9w1oyQ7ExLDoe/lwqldS3YU1AtdThERGQHyxaNl4frntgLMBmhTniqlJidEg2AXTVERHIlhxkjAJMR6oJlPPzGI6XQ6g0SR0NERLaqbTZ10rjyFg3AZIS6kJ4YisgAT2iaW7Ezt1LqcIiIyEY1MjixF2AyQl1QKgTMGcnx8EREcqVpcv1D8gAmI3QJ16eZumq2Hi9Dg1YvcTRERGQLa80It2lIzlJiA5AY7gut3ojNR0ulDoeIiGxg3aZhMkJyJggCrjfPHPn6EAegERHJiaWAlds0JHuWrpqfTleiol4rcTRERNRdGmtrL1dGSOYSwnyRGhcEg1HEhsNcHSEikouaJrb2khu53nySL7dqiIjkw3ouDbdpyB1cmxoNhQAcLKxFYVWT1OEQEVE3aNhNQ+4kwt8LkwaFAeB4eCIiOZDLib0AkxGywXWpbQPQRJEn+RIRubJ6mZzYCzAZIRvMSomCWqVAXkUjjpfUSR0OERF1QSOTE3sBJiNkA38vD2QMjQAAfJ3NQlYiIlcmlxN7ASYjZKPrzAPQ1mcXw2jkVg0RkauSS1svwGSEbDQtORz+XiqU1rVgT0G11OEQEVEn2tp6mYyQm/FUKTE7JRoAu2qIiFyZ5cRebtOQW7KMh994pBRavUHiaIiIqCOWQ/KCfbkyQm4oPTEUkQGe0DS34oeTFVKHQ0REHbAUsAZyZYTckVIhYM5IjocnInJllhN7WTNCbuv6NFNXzdbjZWjQ6iWOhoiIfs2yMhLMZITcVUpsABLDfaHVG7H5aKnU4RAR0a/UmgtYuU1DbksQBFxvnjny4c9nodMbJY6IiIjaY2sv9Qk3j+0HH7USh4pq8afPDnEIGhGRC2nbpuHKCLmx2CBvvHPXGHgoBXxzqBjPf3ucB+gREbkAo1G0btNwZYTc3uWDw7H8llQAwJrdBXj7hzyJIyIiogadHpbF6kBvJiPUB1yfFotnrx0GAHh100l8uq9I4oiIiPq22kbTFo23h9LlT+wFmIyQg9wzeQD+MGUgAGDxl0ew9XiZxBEREfVdcpoxAjAZIQd6atYQ/GZ0PxiMIhZ+fAD7z/IgPSIiKbRNX2UyQn2MIAh4+TcjMD05Alq9EXev2YfcsnqpwyIi6nNqzMWrcuikAZiMkIN5KBVYccdojIoPgqa5FfNW70FxbbPUYRER9SkaGc0YAZiMkBN4q5VYPW8cBkX4oUTTgrmr96CmUSd1WEREfYZlm4bJCPVpwb5qfHj3eEQHeuF0eQPu/mAvmnUGqcMiIuoTaqwzRrhNQ31cTJA3Prh7PAK8VDhYWIuFHx9Aq4Fj44mInE1jWRlhASsRkBTpj9Xzx8FTpcC2nHIs/vIIp7QSETmZnM6lAZiMUC8YmxCCFXeMhlIh4PP95/Dq5pNSh0RE5Na4TUPUgYxhkVh24wgAwNs78vDerjMSR0RE5L64TUPUiVvHxeGJmUMAAC98exxfZ5+XOCIiIvfUtk3DlRGiizw4dSDmT0wAADz+2SHszK2QNiAiIjfT/sTeYNaMEF1MEAQsuXYYrh0ZjVaDiD/8Zz8OFdVKHRYRkduo17ad2BvAbRqijikUAl67NRWTBoWiSWfA7z/cx6FoREQOYqkXkcuJvQCTEZKIp0qJd+4ai4Hhvqio12LJ+mNSh0RE5BZqZLZFA9iZjKxYsQIJCQnw8vJCeno69uzZ0+X1tbW1WLhwIaKjo+Hp6YmkpCRs3LjRroDJffh5qvD6rWlQKgR8c6gY3x4uljokIiLZsxSvBsqkeBWwIxlZu3YtFi1ahKVLl+LAgQNITU3FzJkzUV5e3uH1Op0OV111FQoKCvD555/j5MmTWLVqFWJjY3scPMlfalwQFk4dCAD4y7qjKK9rkTgiIiJ5sxSvyqWtF7AjGXn99ddx7733YsGCBRg2bBhWrlwJHx8frF69usPrV69ejerqaqxbtw6TJk1CQkICpkyZgtTU1E7fQ6vVoq6u7oIvcl8PTR+M4TEBqG1qxdOc0EpE1COWQ/KCfd00GdHpdNi/fz8yMjLaXkChQEZGBrKysjp8zvr16zFhwgQsXLgQkZGRSElJwUsvvQSDofND05YtW4bAwEDrV1xcnC1hksyoVQq8fmsa1ErTyPjP9p2TOiQiol6hNxjx239nYf77exz2DzFLMhLo7abbNJWVlTAYDIiMjLzg8cjISJSWlnb4nPz8fHz++ecwGAzYuHEjnn32Wbz22mv429/+1un7LF68GBqNxvpVVFRkS5gkQ0Oi/LFoRhIA4Plvj6OoukniiIiInC+vohE/51djx8kKnCpvcMhr1jZbRsG76cqIPYxGIyIiIvDvf/8bY8aMwW233YZnnnkGK1eu7PQ5np6eCAgIuOCL3N+9lydibP9gNGj1eOLzQzAauV1DRO7tZFm99X/vPl3pkNe0btO4azISFhYGpVKJsrKyCx4vKytDVFRUh8+Jjo5GUlISlMq2XuehQ4eitLQUOh1nS1AbpULA8ltS4e2hxM/51Vizu0DqkIiInCq3tC0Z+SmvyiGv2VbA6qbbNGq1GmPGjEFmZqb1MaPRiMzMTEyYMKHD50yaNAmnT5+G0Wi0Ppabm4vo6Gio1fL5i6LekRDmiz9fMxQA8MqmHORVOGbZkojIFbVfGfk5vwoGB6wIt7X2uunKCAAsWrQIq1atwgcffIATJ07ggQceQGNjIxYsWAAAmDt3LhYvXmy9/oEHHkB1dTUeffRR5ObmYsOGDXjppZewcOFCx/0U5FZ+lx6PyweHQas3YtGnh6A3GC/9JCIiGcptl4zUt+hx9Lymx69ZK7MTewE7kpHbbrsNy5cvx5IlS5CWlobs7Gxs2rTJWtRaWFiIkpIS6/VxcXHYvHkz9u7di5EjR+KRRx7Bo48+iqefftpxPwW5FUEQ8OrNI+HvpcKholqs/CFP6pCIiByuWWdAoblYf3R8EABgtwO2aqyH5PnKZ/dBEGUw1KGurg6BgYHQaDQsZu1DvjxwDos+PQQPpYB1CydheEyg1CERETnMkXMazHlzF0J91Xho+iA8981xXD44DB/dk273axqNIgY9sxFGEdjz5ysREeDlwIht193Pb55NQy7rxlGxmDEsEq0GEX/69BC0+s5n0xARyY2lXiQp0h8TB4YBAPYWVPfod119S9uJvW5dM0LUWwRBwEs3jUCorxo5pfV4Y+spqUMiInIYS73IkCh/JEX6IcxPjZZWI7ILa+1+TcuMER+1Ep4qeZzYCzAZIRcX5ueJF28cAQB454c87D9bLXFERESOcbK0bWVEEARMMK+O9KTFV47FqwCTEZKBWSlRuGlULIwi8KdPD6FJp5c6JCKiHmtbGfEDAEwcGAoAyMqzf/hZjbl4VU4n9gJMRkgmll43HFEBXiioasIr3+VIHQ4RUY9omltRojGdUj440h8AMMm8MnKwsBaNWvv+0aVplt/0VYDJCMlEoLcHXr15JADgg6yz+MlBY5OJiKRwuty0KhId6IUAL1PiEBfijdggb+iNIvYW2Lclbd2mYTJC5BxXJIXjd5fFAwCe+OwQ6lpaJY6IiMg+J0tN06WTzKsigKlof9Ig01aNvfNGrNs0MhoFDzAZIZlZfPVQxIf4oFjTgufWH5c6HCIiu7TvpGnP0uK72866ETkekgcwGSGZ8fVU4bVbUyEIwBcHzuH7Y6VSh0REZLP2nTTtWYpYjxXXWSep2sJSM8JtGiInG5cQgvsuTwQA/PmrI6hq0EocERGRbawrI79KRiICvDAowg+iaDo4z1ZyPLEXYDJCMvXYVUlIivRDZYMOf/7qCGRwqgEREQCgskGLqkYdBAEYFOF30fcnDbS/bqSGBaxEvcfLQ4nXb02DSiFg87EyvLr5pNQhERF1i2VVJD7EB97qi6ekWoef2dE12LZNw5URol6REhuIl24yTWd9e0ce3v/pjMQRERFdWm4n9SIWlyWGQBCAvIpGlNW12PTa1m0arowQ9Z5bx8bhiZlDAADPf3sc3x4uljgiIqKunSwztfX+ul7EIshHjRTzKeVZNmzVGI0iC1iJpPLg1IGYO6E/RBFYtPYQdnMgGhG5MMs2TVJUx8kI0NZVY8tWzQUn9vJsGqLeJQgCls4ZjtkjoqAzGHHfR/txrFgjdVhERBcRRdG6TdPZyggATBxkmTdS1e0Cfbme2AswGSE3oVQIeP3WNKQPCEGDVo/57+9FUXWT1GEREV2gRNOCeq0eKoWAAWG+nV43LiEYHkoB52ubUdjN32U11oFn8ipeBZiMkBvx8lBi1byxSI7yR0W9FnNX7+EMEiJyKZYtmgFhvlCrOv8I9lGrMCouGED3W3xrraPg5bVFAzAZITcT4OWBD+4ej9ggb5ypbMTda/baffolEZGjdadexGKCjXUjci1eBZiMkBuKDPDCh/eMR7CPBw6d0+CB/x5Aq8EodVhERNYD8rqqF7GwFLFmdbNupKbRtDLCbRoiFzEw3A+r54+Dt4cSO3Mr8NTnh2E0ckorEUnLujLSjWRkVHwwvDwUqGrUIdfcDtyVWvPKSCBXRohcx6j4YLx152goFQK+PHger2zOkTokIurDDEYRp8o7Pq23I2qVAuMSQgB0b6vGcmJvEGtGiFzLtOQIvGye0vrOD/l4bxentBKRNIqqm9DSaoSnSoH4EJ9uPWdSuxbfS7EUsHKbhsgF3TI2Dk/OMk1pfeHb41h/iFNaiaj3WbZoBkX4QakQuvUcS93IL/lV0F+i9o3bNEQu7oEpAzF/YgIA4E+fZmPXKU5pJaLeZUlGulO8ajE8JhABXirUa/U4WlzX5bXcpiFycYIgYMm1w3DNyGi0GkTc/9E+HD3PKa1E1HssZ9J0p63XQqkQcFli91p8rds0vtymIXJZCoWA129NxcSBoWjUGTD//T04W9UodVhE1Ed0Zwx8Ryx1I5c6NM+yTcOVESIX56lS4p27xmBodAAqG3SYt3oPKjmllYicTKc3Iq/C9pURoK1uZG9BNbR6Q4fXtD+xlzUjRDLg7+WBDxaMQ79gbxRUNeHuNXuh03MoGhE5T0FVI/RGEX6eKsQEetn03EERfgj394RWb8SBs7UdXlPX0grLXLQgb27TEMlCRIAXPrzbNKX18DkNPswqkDokInJjluLVwZF+EITuddJYCILQbhprx3UjluJVX7WyyzNvXJX8IiZykMRwPzx9dTIA4J+Zp6yjlImIHM3eehELSzLyUyd1I9Z6ERnOGAGYjFAfd/OYOCRH+aOuRY9/bjsldThE5KZO2jAGviMTB5qKWA8V1aKhg8M/a8ydNHI8JA9gMkJ9nFIh4C/XDAMAfJR1FvkVlz7/gYjIVpazZbozBr4jcSE+iAvxht4oYm9B9UXf1zTJ98RegMkIESYPDsO0IeHQG0W8/B3PryEix2ppNaDAPEbA3pURAJhkXh3Z3cG8EcuMETkWrwJMRogAAH+ePRRKhYDvj5fh5/xLnwFBRNRdp8sbIIpAiK8aYX72JwsTzHUjHZ1TU8OVESL5Gxzpj9vHxwEA/rbhOIxGUeKIiMhdWDtpImzvpGnPkowcL6m7qOBe08xkhMgt/DEjCf6eKhw9X4evDp6XOhwichOW4lV760UsIvy9kBTpB1HERSu43KYhchNhfp54cNogAMDfN59Es67jSYdERLawtPX2pF7EwtJV89Ov5o1wm4bIjSyYlIDYIG+U1rVg1Y/5UodDRG6gp5007U3spG6Ec0aI3IiXhxJPmQehrfwhD+V1LRJHRERyVt/SivO1zQCApIieJyPpiaFQCEB+RSNKNW2/nzScM0LkXuaMjMao+CA06Qx47ftcqcMhIhmzrIpEBXg55AC7QG8PjIgNBADsbrdVY9mmCWYyQuQeBEHAX64ZCgD4dH8RjhfXSRwREcnVqXZn0jjKBMu8EfNWjcEooq7FfGIvC1iJ3MeY/iG4ZmQ0RBF4aeMJiCJbfYnIdtZOGgcUr1pY60ZOV0IURdS3O7E30JsrI0Ru5elZyVArFdh1uhI7TlZIHQ4RyZBlxkiSA4pXLcYlhMBDKaBY04KzVU3WLRo/T5UsT+wFmIwQdSouxAcLJiUAAF7ceAJ6g1HagIhIdk6WmjtpHLgy4q1WYlR8MABTi69lxohcV0UAJiNEXXpw2iAE+3jgdHkD/re3SOpwiEhGqhq0qGzQAnBszQjQ7pyavKp2bb1MRojcUqC3Bx67KgkA8I8tudYiMSKiS7F00sSH+MBHrXLoa08cZKob+TmvyjoanskIkRu7fXw8EsN9Ud2ow1vb86QOh4hk4lS5ZfKqY1dFACC1XxB81EpUNerwS341APkOPAOYjBBdkodSgWdmm1p9V+86g6LqJokjIiI5OOnAMfC/plYpMC4hBACw+XgpACCINSNE7m16cgQmDgyFzmDEq5tPSh0OEclAroMOyOvMJPNWTa3Mz6UBmIwQdYsgCHjmmqEQBOCbQ8U4UFgjdUhE5MJEUXTqygjQdmieRTC3aYjc3/CYQNw8uh8A4G/fHucgNCLqVFmdFnUteigVAhLDfZ3yHkOjAy5o52VrL1Ef8fjMIfD2UOJAYS02HCmROhwiclGWyasDwnzhqVI65T2UCgETEkOtf2YBK1EfERnghfunJAIAXtmUg5ZWg8QREZErspxJ44xOmvYsLb6AfA/JA5iMENnsvisSERngiaLqZnywu0DqcIjIBTm7XsSifd0IC1iJ+hAftQqPzxgCAHhz22lUmScsEhFZ5DrhgLyODAz3RUpsACL8PdEv2Mep7+VMTEaI7PCb0f0wPCYA9Vo9/i/zlNThEJELMRpF6/RVRx6Q1xFBELDuwUnY+eQ0eHk4pzalNzAZIbKDQmFq9QWA//5SiKPnNRJHRH3VL/lV+GfmKbTyIEeXca6mGc2tBqhVCvQPcf5qhUqpkHUiAjAZIbLbxIFhuDolCgajiAf/e4Dn1lCvq23S4b6P9uP1LblYy4McXYalk2ZQuB9USn7Mdgf/loh6YNlNIxAb5I3C6iY8+dlhzh6hXvXmttPQmE9s/TCrgPefi8jtpU4ad8JkhKgHgnzUWHHnaHgoBWw6Vor3fyqQOiTqIwqrmvBBVgEAQKUQkFvWgKy8KmmDIgDtkhEn14u4EyYjRD2UFhdkPUjvpY0nOCqeesUrm3LQahBxRVI4bh8fDwBYw1Zzl2Bp63V2J407YTJC5ADzJiZg9ogo6I0iHv74IGoadVKHRG5s/9kabDhSAoUA/Hl2MuZN7A8A2HqiDOdqevdU6U/3FWHos5tw68osvPNDHk6X1/fp7aJWgxH5FY0AnD9jxJ0wGSFyAEEQ8PJvRiIh1Afna5ux6NNsGI199xcyOY8oinhxw3EAwK1j45AcFYBBEf6YPCgMRhH46OezvRZLs86AVzfloLnVgD0F1Vj2XQ4yXt+Jqct34PlvjmP36co+1+VztqoROoMRvmolYoO8pQ5HNpiMEDlIgJcHVtw5GmqVAttPVuCdnflSh0Ru6LujpThQWAtvDyUWXZVkfXzexAQAwNq9Rb12TMEnewtR2aBDv2BvPH/9cFyRFA61UoGzVU1Y/dMZ3PHuLxj9whY89PEBfHXwXJ9YMTxZapovMjjSHwqFIHE08mFXMrJixQokJCTAy8sL6enp2LNnT7ee98knn0AQBNxwww32vC2RyxseE4jnrhsOAFj+/Un8ks+CQnIcrd6Al7/LAQDcPyUREQFe1u9NT45Av2Bv1Da1Yn12ca/E8s4PpoT7gakDMXdCAj68ezwOLLkKK383GjeP6YdQXzXqW/T49nAJHlt7CGP+tqXddk6DW27n9NbkVXdjczKydu1aLFq0CEuXLsWBAweQmpqKmTNnory8vMvnFRQU4PHHH8fll19ud7BEcvDbcXG4cVQsDEYRD//vICo5Lp4c5KOssyisbkKEvyfuuyLxgu8pFQLuusxUO7Jmt/PbfL/Yfx6ldS2IDPDEzWP6WR/381RhVko0lt+Sij3PZODLBydi4bSBSI7yh1FEu+2cHzDNsp2TV+k2iYklGRnMtl6b2JyMvP7667j33nuxYMECDBs2DCtXroSPjw9Wr17d6XMMBgPuvPNOPPfcc0hMTOz0OiJ3IAgC/nZDCgZF+KG8Xos/fpINA+tHqIdqm3T417bTAIA/zUiCj1p10TW3jYuDl4cCx0vqsO+s87q6Wg1GvLXDFMv9VwyEp6rj6Z9KhYDR8cF4YmYyNv3xCvz45DQ8d13bdk6BZTtn1S9Ysf200+LtTZaBZ0PY1msTm5IRnU6H/fv3IyMjo+0FFApkZGQgKyur0+c9//zziIiIwD333NOt99Fqtairq7vgi0hOfD1VePvO0fD2UGLX6Ur8axvPr6GesQw4S47yx81j4jq8JshHjRvSYgE4t813fXYxztU0I9RXbW0r7o64EB/Mm3jhdk7G0EgAploYuWtpNaCg0tRJw20a29iUjFRWVsJgMCAyMvKCxyMjI1Fa2vGNtGvXLrz33ntYtWpVt99n2bJlCAwMtH7FxXX8fzwiVzY40h8v3pgCAPi/zFPYdapS4ohIrs5WNVoHnP159lAouyiMtBSybjpailJNi8NjMRhF66rI7y9PhLfavjNRLNs5L91k+v/I8ZI6VMu8wDWvogFGEQjy8UC4v6fU4ciKU7tp6uvrcdddd2HVqlUICwvr9vMWL14MjUZj/Soq4pkLJE83je6H346LgygCj35yEGV1jv9wIPf36qaT1gFnVySFd3nt0OgAjB8QAoNRxH9/cXyb76ajpciraESgtwd+d1n3V0U6E+HvhaRIP4giZD9Btm0MvD8EgZ00trApGQkLC4NSqURZWdkFj5eVlSEqKuqi6/Py8lBQUIA5c+ZApVJBpVLhww8/xPr166FSqZCXl9fh+3h6eiIgIOCCLyK5+ut1w5Ec5Y+qRh0e/vgg9H1s7gL1zK8HnHXHfPPqyP/2FEKrd1ybryiK1i3HBZMS4O/l4ZDXnTTI9I/Vn/LkvXqYW2Zq6+UWje1sSkbUajXGjBmDzMxM62NGoxGZmZmYMGHCRdcnJyfjyJEjyM7Otn5dd911mDZtGrKzs7n9Qn2Cl4cSb905Gn6eKuwpqMZrW3KlDsmtaJpae22uRm/raMBZd8wYFonoQC9UNuiw4XCJw+LJPFGOnNJ6+HmqrAmPI0waaE5GTss8GSnlAXn2snmbZtGiRVi1ahU++OADnDhxAg888AAaGxuxYMECAMDcuXOxePFiAICXlxdSUlIu+AoKCoK/vz9SUlKgVqsd+9MQuajEcD+8/JsRAIC3d+RhW07ZJZ5B3ZFf0YCJL2ci/aVMvLE1F7VN8q45+LWNRzoecHYpKqUCvzO3+X7goEJWURTxL3PHy10T+iPIx3G/v9MTQ6BUCDhb1YSi6t4dZ+9IJ9tt05BtbE5GbrvtNixfvhxLlixBWloasrOzsWnTJmtRa2FhIUpKHJeJE7mLa0fGYN4E0wfEY2sP9foZIu5o9U9n0KgzQNPcije2nsKkl7dh2XcnUFEv/9kuWr0Br2zqeMBZd9w2Lg5qpQKHzmlw0AGHN+46XYlDRbXw8lDgnskDevx67fl7eSC1XyAAYLdMt2oatHqcq2kGwGTEHnYVsD700EM4e/YstFotfvnlF6Snp1u/t2PHDqxZs6bT565Zswbr1q2z522JZO/P1wzFyH6B0DS34qGPD0KnZ/2IvTTNrfhi/3kAwEPTBiE5yh+NOtNU0MmvbMNf1x9DcW2zxFHar6sBZ90R5ueJa1OjAThmdcQy4+SO8f0R5uf4TpHJ5rqRXaflWcR6yrwqEuHviWBfrvrbimfTEPUiT5USK+4YjQAvFbKLaq2jvcl2n+0rQnOrAUMi/fGnGUn47tHL8e7csUiNC4JWb8Sa3QWY8vfteOrzw9bZD3LRnQFn3WGp69hwpKRHq0W/5Fdhz5lqqJUKuxKj7phoTkZ2n66U5SGTuRx21iNMRoh6WVyID167NQ2AaZth01Fua9rKYBStczfmTUyAIAgQBAEZwyKx7sGJ+M896bgsMQStBhFr9xVh+ms78OgnB3HSXGDo6roz4Kw7RvYLwqj4ILQaRPxvT6H98ZhrRW4Z2w9RgbZtF3XXqPggeHsoUdWos9Ze9AZNcyt+/8E+LPn6KMrr7W+9t3TScIvGPkxGiCRw1bBI3G/+F+YTnx3GsWKNxBHZT9PU2uvnimzPKUdRdTMCvT1ww6iYC74nCAImDw7DJ/dNwBcPTMC0IeEwisDX2cWY+cZO3PfhPhw+V9ur8drClgFn3WFZHfnvL2fRakdbeXZRLX48VQmlQsAfpgzsUSxd8VQpMX5ACIDe7ar5Yv85bD1Rhg+zzmLq33fg/7aeQpNOb/PrtM0YYSeNPZiMEEnk8ZlDMC4hGPVaPW5dmYUfciukDslm23LKkPbC972+3WQZdf7bcXFdbmGM6R+C9xeMx7cPT8bsEVEQBOD742W47s2fcNd7v7jkqcq2DDjrjqtTohHu74myOi022TFy/U3zdtGNo2IRF+LT43i6MmlQKIDeTUY2HzP9nYT6qtGkM+AfW3Mx5e878L89hTbNBDpZyk6anmAyQiQRD6UC784bhwmJoWjUGXD3mr34dK+8pg2v3JEPUTRtN/VWS+apsnrsOl0JhQBr++qlpMQG4q07x2DLY1fgplGxUCoE/HiqErf9+2fcsnI3Djig28QR7BlwdilqlQJ3mM+PsbWQ9XhxHbaeKIMgAA9Mdd6qiIVl+NkvZ6rtWsWxVVWDFnsLqgEA6xZOwpt3jEJ8iA8q6rVY/OURzP7nj9ieU37Jlb+aRh3KzTU5g5mM2IXJCJGEAr098MHd43HjqFgYjCKe/OIwXt+SK4vj1E+W1mOP+Rd5q0HstcMALasiGUMjbf6X+qAIf7x+Wxq2/2kq7kiPh1qpwN6CGtz17i+Sn4ti74Cz7rgzPR4qhYB9Z2tw9Hz3twRXmM+guWZENAaGO3/7YWhUAELMKxTZRbVOf7+tJ8pgFIHhMQGIC/HBtSNjsGXRFXj22mEI8vFAblkDFqzZizvf/aXLvzfLFk2/YG/4edpXbNzXMRkhkphapcDrt6bioWmDAAD/zDyFP312yOXbfj82n3syOML0IfXFgfNO71rRNLfiywOmdt75kxLsfp34UB+8dOMI7HxymrUlePWuMw6K0j72DjjrjogAL8weYVub7+nyBmw8YiquXmi+N51NoRAwYWDvbdVsPmYaPjhreNtxJp4qJe6ZPAA/PD4N91+RCLVSgd15Vbj2X7vw2NrsDucDWTtpuCpiNyYjRC5AEAQ8PnMIlt00AkqFgC8PnMfda/airqVV6tA61KTTW5OCZ68dhmlDwmEwivhnpnNXR9q3805IDO3x60UFeuGPGYMBAB9kFUj2993TAWfdMW+iaUvr60PF3VoFemvHaYiiqdh6aHTvnQ/WW6PhG7R660naM1MuPlst0McDi2cPReafpuD6NFOR9FcHz2P6az9g2cYT0DS33SvWThq29dqNyQiRC7l9fDzenTcWPmoldp2uxK0rs1Cicb3BXd8cKka9Vo/+oT6YPCgMj5n/Jb8u+zxOlzc45T3bt/POn5TgsFNRZwyLwuAIP9S36PFRluNPue2Ong44647R8cFIiQ2ATm/EJ3u7bvMtrGrC19nFAGBdsestluFnBwtr0ai1vaulu7bnlENnMGJAmK91da8jcSE++L/fjsL6hybhssQQ6PRGvLMzH1P+vh2rd52BTm9sNwaenTT2YjJC5GKmDYnAp/dPQLi/J3JK63Hjit04XlwndVgX+M/Ppg+zO8bHQ6EQMLJfEK4aFgmjCLyx1TkHAW5r386bFuuw11UoBOs2xLs/5tvV1tkTjhpwdimCIGDehAQAwH+yznbZKfL2D3kwGE0dPalxQU6JpzPxoT7oF+wNvVHEnjPVTnsfSxfNzOFR3UpsR/YLwv/uvQyr54/F4Ag/1Da14vlvjyPj9R9wzFxPwk4a+zEZIXJBKbGB+OrBiRgU4YfSuhbc+k4WfjzlGq2/h8/V4sh5DdRKBW4e08/6uKXOYcOREuSUOj55+qBdO6+3WunQ1752ZDTiQ3xQ09SKj3+xfziYPf7loAFn3TEnNQYhvmoUa1qw9UTHhzWWaJrx+X5TV9fD03t3VcSibTS8c7ZqWloN2J5TDgCYOTyy288TBAHTkyPx3aOXY9lNIxDm54nC6iY06gxQCOiVIl93xWSEyEX1C/bBF3+YiPQBIWjQ6rHg/b34bJ/0rb//Na+KzB4RhdB2Z5QMjQ7ANSOiIYrAG1scWztiTzuvLVRKBR40t66u+jEfLa0Gh79HR/IqGvCheetpsQMGnF2Kl4cSvx1nSng+2N3xltQ7P+Sj1SAifUAIxiWEODWezlhGwzurbmR3XiUadQZEBXghtV+Qzc9XKRW4fXw8fnhiKv6YMRg+aiUuHxwOLw/HJsl9CZMRIhcW6OOBD+8Zj+vTYqA3inji88N4Y6t0rb+a5lasP2SqJbizg6Tg0YzBEARg07FSm1pIL8XSznvVMNvbebvrptH9EB3ohbI6LT7ff84p7/FrL3x7HK0GEdOTIzDFAQPOuuN3l/WHUiEgK7/qovH4FfVa69j4h6cP7pV4OjLR3FGTU1qPygbHn8C8+ahpVWjG8EgoepAA+nqq8MeMJGQvmYHV88c5Krw+ickIkYvzVCnxj1vTrP9yf2PrKTzx+eFeGQr1a18dOIfmVgOSIv0wtn/wRd9PivTHdammzoN/bHFM7Yimqa2dd555tLkzqFVth8Ct/CHP6X+/23LKsONkBTyUAp69dphT36u9mCBvzBhm2pqwFARbvLsrH1q9EWlxQdZpqFII8/O0dvDsznPslFyDUcQW8xbVzOEXd9HYQ61SOH1Vy90xGSGSAYVCwJOzkvHijSlQCMDn+8/h7jV7Ud+LraiiKOK/5nqKO9P7d1r09+iVg6EQgMyccocMrvpsv2Pbebvy23HxCPNT41xNM9abu0mcQac34oVvTwAA7p40AAPCfJ32Xh2xJHVfHTgPTZPpHqpp1OE/5m6ih6cPcli3kr0mWeaNnHLsVs3egmpUN+oQ5ONhPQuHpMdkhEhG7kzvj3fnjYW3hxI/nqrELSuzUKqx/6RRW+wtqMGp8gZ4eyhx4+jOu1kSw/1w02hTYevrPVwdcVY7b2e81UrcM9m0OrJix2kYnHSU/fs/ncGZykaE+3viIQmKRNMHhCA5yh/NrQZ8Zi5WfX93ARp1BgyNDsD05Ihej+nXJg1uK2J15LakpYvmyuRIeCj5Eegq+F+CSGamJ0di7f2XIczP3Pr71k84W+XcyaeA6dRXALg+LQYBXh5dXvvI9MFQKQTszK3AvgL72zOd1c7bld9dFo8ALxXyKxrtOljuUsrrWqzD4Z6alQz/S/xdOoMgCNbVkQ+zzkLT1Io1P5km0LrCqggAjE8IgUoh4HxtMwoddO6RKIr4/phli6b7XTTkfExGiGRoZL8gfPXgRAwM90WJpgVPfXHYqUWtVQ1afHfE9MF8Z/qlu1niQ31wy1jT6shr39u/OmJt5x3v+Hbezvh7eWDBpAEAgDe3n3b43+srm06iUWdAalwQbhrVOwlWR0xJpQqF1U34w3/2o65Fj0ERfheMRpeSr6cKo+NNdUk/nXZM3cjR83U4X9sMbw+lQ05EJsdhMkIkU3EhPlizYDy8PBT4Ob8aXx0877T3+mz/OegMRozsF4gR/QK79ZyHpg+Gh9LUtbE7z/Z9//btvHc5oZ23KwsmJcBXrcSJkjpsM8+jcISDhTX44oCpU+evc4b1qJOjp3zUKtxmbvPNyjd92C+cNlDSmH5t4iDHnlNj2aKZOoRtuK6GyQiRjMWF+OCRK00tmC9uOIHaJsefPGs0itZBYL/rxqqIRWyQN347znR0/T/sOIm4fTtvv2DntPN2JshHbZ1n4qjVEaNRxF+/MZ3K+5vR/TAq/uJupN5212UJsOzIxIf4YM7IGGkD+hXL8LPdeZUwOqB+p/3UVXItTEaIZO73kxMxOMIPVY06vLLppMNff9fpShRWN8HfS4VrU6Nteu7CaYOgVimwt6AGP9rQFdG+nXf+xAE2vaej3HP5AHiqFDhYWIssB7SXfnnwPA4V1cLPU4WnZg1xQIQ9Fx/qg9kppv+mj1w5GCoXK+hMjQuCr1qJmqZWHC/p2VTfvIoGnCpvgIdSwDQXKNClC7nWnUdENlOrFPjbDSkAgP/tKcT+szUOff3//GwqXP3N6H42n5sSFehlXU15zYbVEUs7b3KUPy5LlKb9MsLfyzqt1HJ2jL3qW1rx8nemU3kfnj7IKafy2uvvt4zE1wsnXTDa31V4KBVIT3TMVo1lVWTCwDAEevd+0TB1jckIkRtITwzFLeYPk2e+OtLlIWi2KNE0I9NcM3FHerxdr/HA1IHw8lDgUFFtt+ov2rfzzpvo/Hbertw3ZSBU5mmlPUny3tx2GpUNWgwI87UWx7oKH7Wq1w/Ds4VlGutPPVyd2nzUskXDLhpXxGSEyE0snj0UQT4eyCmtt9Zb9NTavUUwGEWMHxBi94mk4f6e1tNiX+/G6ogU7bydiQ3yxm/MM1NWbLdvdSS/ogGrzW2zz147FGoVf+3aYrJ53sieM1XQ6u07M6hE04xD5zQQBFMNErke/r+CyE2E+Kqx+OpkAKYP/eLa5h69nt5gxCd7TAOx7rRzVcTi/ikD4atW4lhxHTYf6/i0WIs1u00f3L3ZztuVB6YOhEIwJUn2nLfztw0n0GoQMXVIOKYn84PQVkMi/RHmp0ZLqxEHC2vteg3LbJEx8cGI8HedLTJqw2SEyI3cMiYOY/sHo0lnwHPfHOvRa2XmlKO0rgUhvmrMSulZ90GIr9q6PfGPLbmddkacKqvHT6erJGnn7UxCmC+uNXeZvLXDttWR7Tnl2JZT3uvnz7gTQRAwcWDPTvHddJRdNK6OyQiRG1EoBPztxhSoFAI2HyvD1uNdr0J0xXIOzS1j+8FT1fMVinsvT4S/pwony+qx4UhJh9dI2c7blYXTTCPbvztaitPl9Ze42sR0/oyplXfBpAEYGO7ntPjcnaXF155kpKZRhz3mKcBMRlwXkxEiN5McFYB7LjetQixdfwxNOr3Nr1FY1YSduRUAgDvHO2aFItDHwxrXG1tzLzr3xRXaeTszJMofM4ZFQhSBt7bndes5a3afQX5lI8L8PPGwBOfPuBPL8LND5zQ2Hw659UQZDEYRQ6MDEB/qOgkuXYjJCJEbevTKwYgN8sb52mb8M9P2wsv/7jG1816RFO7QX+B3Tx6AQG8P5FU0Yv2hCyfGfrpP+nberlgOtPv6UDEKq7o+K6W8vsX69/7krCGSnD/jTvoF+yAh1AcGo4hf8m0766ht0BnrdVwZkxEiN+SjVuG564YDAN79MR8nS7u3tQAAWr0Bn+0zjSzvaeHqrwV4eeC+K0yn4v7f1lPWFmSDUcSHPxcAAOZL3M7bmZH9gnBFUjgMRhErd3a9OvL3TSfRoNUjtV8gbh7tevM75GjioLZTfLurUavHTvOwvZ7WPZFzMRkhclMZwyIxY1gk9EYRf1l3pNvjtDcdLUV1ow5RAV640gmTKudPTECIrxoFVU3WbRlLO2+Qjweul7idtysPmWtHPt93DqWalg6vyS6qxWf7Tcnc0uuGu9RZL3LWfjR8d/2QWwGd3oj+oT4YYmdrOvUOJiNEbmzpdcPho1Zib0ENPjd/QF6KpXD1t+PjnDIe3NdThT9MMa2O/HPbKej0Rms7723jXKOdtzPjB4RgfEIIdAYj/r0z/6LvG40i/rre1MV00+hY66mz1HMTEkMhCEBuWQPK6zpOBH/N0kUza3iUS662URsmI0RuLDbIG49lJAEAln13AtWNXR+kl1tWjz1nqqFUCNZD7pzhrssSEO7viXM1zXhp4wmXa+ftiqV25OM9Z1HVoL3ge18dPI/solr4qpV4elayFOG5rWBfNYbHBAAAdndjGqtOb8R288TfGeyicXlMRojc3PxJCUiO8kdNUyte/u5El9daTue9MjkCUYHOGw7lrVbiwakDAbS1884YFuVS7byduXxwGEb2C0RLqxHv7TpjfbxBq8fLm0znzzw0fbBLnT/jLiYN7H7dyO68StRr9Yjw98QoFx53TyZMRojcnIdSgRdvHAEA+HTfOew503E3QpNOjy8OmLZyftcLKxS3j49HVLsP7PmTEpz+no4gCIJ17shHWWehaTa1mr657TQq6rVICPXB3ZMTJIzQfU1qN2/kUscKWLpoZgyPZN2ODDAZIeoDxvQPxu3jTdsuf1l3BDr9xQfpfXuoBPUtesSH+FiLBZ3Jy0Np3fIYGh2A9AGu187bmauGRmJIpD/qtXp8uLsAZyobsXqX5fyZYQ4ZEkcXG5cQArVSgRJNC85UNnZ6ncEoYot54B8HnckDkxGiPuKpWUMQ6qtGblnDBdsLFv/5xTRb5I70+F77l+Sd6fFYccdorJo7RlYFhgqFgAenmbaZ3vvpDJZ8fRQ6gxFTksIx3QkdSGTirVZidP8gAF1PYz1QWIPKBh0CvFS4LDG0l6KjnmAyQtRHBPmo8efZQwEA/5eZi6LqtsFdh8/V4vA5DdRKBW4Z03tzMQRBwDUjo2VRK/Jr146MQUKoD2qbWvHjqUqoFKbzZ+SUVMlR22j4zotYLV00GUMj4eGEjjByPP5XIupDbhodi/QBIWhpNWLp+mPWfXdL4erVI6IQ6ucpZYiyoVQIeHBq25j3+RMTMCiC588428R280Z+faQAAIii2K5ehFs0csFkhKgPEQQBL96YAg+lgG055dh8rAx1La34OrsYAHBnuuu31rqSG0bFYnhMABLDfPFIxmCpw+kTRsYGwt9ThboWPY4Vay76/vGSOpyraYaXhwJTksIliJDswWSEqI8ZFOGP+68w1Ts8980x/Ofns2huNWBwhB/GJXBIly3UKgW+fXgyti6aggCeP9MrVEoF0s11IB21+G42b9FMSQp36QF6dCEmI0R90EPTByE+xAclmhb8ffNJAKZiUtY72E4QBLaO9rLJ5lN8d3dQN7L5GLto5IjJCFEf5OWhxPPXmw7SE0XA20OJm3qxcJWoJyzzRvYWVKOl1WB9/ExlI06W1UOlEHBlMk/plRMmI0R91NQhEbhmRDQA4IZRMdxmINkYFOGHCH9PaPVGHDhbY33cUrg6YWAoAn14P8sJkxGiPuzvt4zEq78ZiWeuGSZ1KETdJgiCdXWkfd0Iu2jki8kIUR/mo1bh1nFx8PNUSR0KkU3aj4YHgLK6FhwsrAUAzBjGLRq5YTJCRESyM8lcxHrkvAaaplZ8b14VGR0fhEgeUig7TEaIiEh2ogO9kRjuC6MIZOVXsYtG5piMEBGRLFlGw288UoKf801tvkxG5InJCBERydLEgaZkZP2hYuiNIpKj/JEQ5itxVGQPJiNERCRLExJD0X7eHLto5IvJCBERyVKgjwdGxAZa/zyLyYhsMRkhIiLZsrT4xoV4Y2i0v8TRkL2YjBARkWzdPj4eI2ID8VhGEs9WkjFOOiIiItmKC/HBNw9PljoM6iGujBAREZGkmIwQERGRpJiMEBERkaSYjBAREZGkmIwQERGRpJiMEBERkaSYjBAREZGkmIwQERGRpJiMEBERkaSYjBAREZGkmIwQERGRpOxKRlasWIGEhAR4eXkhPT0de/bs6fTaVatW4fLLL0dwcDCCg4ORkZHR5fVERETUt9icjKxduxaLFi3C0qVLceDAAaSmpmLmzJkoLy/v8PodO3bg9ttvx/bt25GVlYW4uDjMmDED58+f73HwREREJH+CKIqiLU9IT0/HuHHj8OabbwIAjEYj4uLi8PDDD+Ppp5++5PMNBgOCg4Px5ptvYu7cuR1eo9VqodVqrX+uq6tDXFwcNBoNAgICbAmXiIiIJFJXV4fAwMBLfn6rbHlRnU6H/fv3Y/HixdbHFAoFMjIykJWV1a3XaGpqQmtrK0JCQjq9ZtmyZXjuuecueryurs6WcImIiEhCls/tS6172JSMVFZWwmAwIDIy8oLHIyMjkZOT063XeOqppxATE4OMjIxOr1m8eDEWLVpk/fP58+cxbNgwxMXF2RIuERERuYD6+noEBgZ2+n2bkpGeevnll/HJJ59gx44d8PLy6vQ6T09PeHp6Wv/s5+eHgQMHYv/+/RAEodPnjRs3Dnv37u329yzbP0VFRbLY/unq53Ol97D3NWx5XnevvdR1tt4zgLzuG7ncM/a+jq3P6c71vGfc+56x9XmOuGcudY07fz6Jooj6+nrExMR0+XybkpGwsDAolUqUlZVd8HhZWRmioqK6fO7y5cvx8ssvY+vWrRg5cqQtbwuFQgG1Wt1lVgUASqWy0/9oXX0vICBAFv+xu/oZXOk97H0NW57X3WsvdZ299wwgj/tGLveMva9j63O6cz3vGfe+Z2x9niPumUtd4+6fT5f67AZs7KZRq9UYM2YMMjMzrY8ZjUZkZmZiwoQJnT7v1VdfxQsvvIBNmzZh7Nixtryl1cKFC3t0TXee7+p642dwxHvY+xq2PK+7117qOt4zrvMe9ryOrc/p6e+RS32f90zvvoer/K7h51PPfwabu2nWrl2LefPm4Z133sH48ePxxhtv4NNPP0VOTg4iIyMxd+5cxMbGYtmyZQCAV155BUuWLMHHH3+MSZMmWV/Hz88Pfn5+PQq+p7pb5UvUHu8bshXvGbJVX7tnbK4Zue2221BRUYElS5agtLQUaWlp2LRpk7WotbCwEApF24LL22+/DZ1Oh5tvvvmC11m6dCn++te/9iz6HvL09MTSpUsvqE8huhTeN2Qr3jNkq752z9i8MkJERETkSDybhoiIiCTFZISIiIgkxWSEiIiIJMVkhIiIiCTFZISIiIgkxWTEBgkJCRg5ciTS0tIwbdo0qcMhmWhqakL//v3x+OOPSx0Kubja2lqMHTsWaWlpSElJwapVq6QOiWSgqKgIU6dOxbBhwzBy5Eh89tlnUodkM7b22iAhIQFHjx6VfFgbycszzzyD06dPIy4uDsuXL5c6HHJhBoMBWq0WPj4+aGxsREpKCvbt24fQ0FCpQyMXVlJSgrKyMqSlpaG0tBRjxoxBbm4ufH19pQ6t27gyQuREp06dQk5ODq6++mqpQyEZUCqV8PHxAQBotVqIonjJo9eJoqOjkZaWBgCIiopCWFgYqqurpQ3KRm6TjOzcuRNz5sxBTEwMBEHAunXrLrpmxYoVSEhIgJeXF9LT07Fnzx6b3kMQBEyZMgXjxo3Df//7XwdFTlLpjXvm8ccftx6NQPLXG/dMbW0tUlNT0a9fPzzxxBMICwtzUPQkld64byz2798Pg8GAuLi4Hkbdu9wmGWlsbERqaipWrFjR4ffXrl2LRYsWYenSpThw4ABSU1Mxc+ZMlJeXW6+x7NP++qu4uBgAsGvXLuzfvx/r16/HSy+9hMOHD/fKz0bO4ex75uuvv0ZSUhKSkpJ660ciJ+uN3zNBQUE4dOgQzpw5g48//viiU9JJfnrjvgGA6upqzJ07F//+97+d/jM5nOiGAIhfffXVBY+NHz9eXLhwofXPBoNBjImJEZctW2bXezz++OPi+++/34MoyZU44555+umnxX79+on9+/cXQ0NDxYCAAPG5555zZNgkod74PfPAAw+In332WU/CJBfjrPumpaVFvPzyy8UPP/zQUaH2KrdZGemKTqfD/v37kZGRYX1MoVAgIyMDWVlZ3XqNxsZG1NfXAwAaGhqwbds2DB8+3CnxkvQccc8sW7YMRUVFKCgowPLly3HvvfdiyZIlzgqZJOaIe6asrMz6e0aj0WDnzp0YMmSIU+Il1+CI+0YURcyfPx/Tp0/HXXfd5axQncrmU3vlqLKyEgaDwXqysEVkZCRycnK69RplZWW48cYbAZgq3u+9916MGzfO4bGSa3DEPUN9iyPumbNnz+K+++6zFq4+/PDDGDFihDPCJRfhiPvmp59+wtq1azFy5EhrPcpHH30kq3unTyQjjpCYmIhDhw5JHQbJ1Pz586UOgWRg/PjxyM7OljoMkpnJkyfDaDRKHUaP9IltmrCwMCiVyosKwcrKyhAVFSVRVOTKeM+QrXjPkD1435j0iWRErVZjzJgxyMzMtD5mNBqRmZmJCRMmSBgZuSreM2Qr3jNkD943Jm6zTdPQ0IDTp09b/3zmzBlkZ2cjJCQE8fHxWLRoEebNm4exY8di/PjxeOONN9DY2IgFCxZIGDVJifcM2Yr3DNmD9003SNzN4zDbt28XAVz0NW/ePOs1//rXv8T4+HhRrVaL48ePF3/++WfpAibJ8Z4hW/GeIXvwvrk0nk1DREREkuoTNSNERETkupiMEBERkaSYjBAREZGkmIwQERGRpJiMEBERkaSYjBAREZGkmIwQERGRpJiMEBERkaSYjBAREZGkmIwQERGRpJiMEBERkaSYjBAREZGk/h+PQ3ov0yqaogAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ddpm_cb = DDPMCB()\n",
    "model = get_model()\n",
    "learn = TrainLearner(model, dls, CustomMSELoss(), cbs=[ddpm_cb, DeviceCBSparse(), GradientClipCB()], opt_func=torch.optim.Adam)\n",
    "learn.lr_find(max_mult=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "61a2f895-812e-4006-9b44-d6cf73e11add",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='16' class='' max='20' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      80.00% [16/20 05:33&lt;01:23]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>loss</th>\n",
       "      <th>epoch</th>\n",
       "      <th>train</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0.486</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.301</td>\n",
       "      <td>0</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.270</td>\n",
       "      <td>1</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.233</td>\n",
       "      <td>1</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.245</td>\n",
       "      <td>2</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.234</td>\n",
       "      <td>2</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.219</td>\n",
       "      <td>3</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.234</td>\n",
       "      <td>3</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.234</td>\n",
       "      <td>4</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.222</td>\n",
       "      <td>4</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.229</td>\n",
       "      <td>5</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.233</td>\n",
       "      <td>5</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.224</td>\n",
       "      <td>6</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.224</td>\n",
       "      <td>6</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.221</td>\n",
       "      <td>7</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.210</td>\n",
       "      <td>7</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.229</td>\n",
       "      <td>8</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.220</td>\n",
       "      <td>8</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.219</td>\n",
       "      <td>9</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.210</td>\n",
       "      <td>9</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.215</td>\n",
       "      <td>10</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.210</td>\n",
       "      <td>10</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.219</td>\n",
       "      <td>11</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.210</td>\n",
       "      <td>11</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.218</td>\n",
       "      <td>12</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.226</td>\n",
       "      <td>12</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.221</td>\n",
       "      <td>13</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.228</td>\n",
       "      <td>13</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.220</td>\n",
       "      <td>14</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.222</td>\n",
       "      <td>14</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.215</td>\n",
       "      <td>15</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0.233</td>\n",
       "      <td>15</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "      <progress value='26' class='' max='144' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      18.06% [26/144 00:03&lt;00:16 0.218]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[13], line 16\u001b[0m\n\u001b[1;32m     13\u001b[0m cbs \u001b[38;5;241m=\u001b[39m [ddpm_cb, DeviceCBSparse(), ProgressCB(plot\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m), LossCB(), GradientClipCB(), checkpoint_cb, BatchSchedCB(sched)]\n\u001b[1;32m     15\u001b[0m learn \u001b[38;5;241m=\u001b[39m TrainLearner(model, dls, CustomMSELoss(), lr\u001b[38;5;241m=\u001b[39mlr, cbs\u001b[38;5;241m=\u001b[39mcbs, opt_func\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39moptim\u001b[38;5;241m.\u001b[39mAdam)\n\u001b[0;32m---> 16\u001b[0m \u001b[43mlearn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mepochs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Desktop/Giannis/Code/pclab/pclab/learner.py:110\u001b[0m, in \u001b[0;36mLearner.fit\u001b[0;34m(self, n_epochs, train, valid, cbs, lr)\u001b[0m\n\u001b[1;32m    108\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m lr \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m: lr \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlr\n\u001b[1;32m    109\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mopt_func: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mopt \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mopt_func(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39mparameters(), lr)\n\u001b[0;32m--> 110\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalid\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    111\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    112\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m cb \u001b[38;5;129;01min\u001b[39;00m cbs: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcbs\u001b[38;5;241m.\u001b[39mremove(cb)\n",
      "File \u001b[0;32m~/Desktop/Giannis/Code/pclab/pclab/learner.py:61\u001b[0m, in \u001b[0;36mwith_cbs.__call__.<locals>._f\u001b[0;34m(o, *args, **kwargs)\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     60\u001b[0m     o\u001b[38;5;241m.\u001b[39mcallback(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbefore_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnm\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 61\u001b[0m     \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     62\u001b[0m     o\u001b[38;5;241m.\u001b[39mcallback(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnm\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     63\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;28mglobals\u001b[39m()[\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCancel\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnm\u001b[38;5;241m.\u001b[39mtitle()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124mException\u001b[39m\u001b[38;5;124m'\u001b[39m]: \u001b[38;5;28;01mpass\u001b[39;00m\n",
      "File \u001b[0;32m~/Desktop/Giannis/Code/pclab/pclab/learner.py:98\u001b[0m, in \u001b[0;36mLearner._fit\u001b[0;34m(self, train, valid)\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[38;5;129m@with_cbs\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfit\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     96\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_fit\u001b[39m(\u001b[38;5;28mself\u001b[39m, train, valid):\n\u001b[1;32m     97\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mepoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mepochs:\n\u001b[0;32m---> 98\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m train: \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mone_epoch\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m     99\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m valid: torch\u001b[38;5;241m.\u001b[39mno_grad()(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mone_epoch)(\u001b[38;5;28;01mFalse\u001b[39;00m)\n",
      "File \u001b[0;32m~/Desktop/Giannis/Code/pclab/pclab/learner.py:93\u001b[0m, in \u001b[0;36mLearner.one_epoch\u001b[0;34m(self, training)\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39mtrain(training)\n\u001b[1;32m     92\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdl \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdls\u001b[38;5;241m.\u001b[39mtrain \u001b[38;5;28;01mif\u001b[39;00m training \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdls\u001b[38;5;241m.\u001b[39mvalid\n\u001b[0;32m---> 93\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_one_epoch\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Desktop/Giannis/Code/pclab/pclab/learner.py:61\u001b[0m, in \u001b[0;36mwith_cbs.__call__.<locals>._f\u001b[0;34m(o, *args, **kwargs)\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     60\u001b[0m     o\u001b[38;5;241m.\u001b[39mcallback(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbefore_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnm\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 61\u001b[0m     \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     62\u001b[0m     o\u001b[38;5;241m.\u001b[39mcallback(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnm\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     63\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;28mglobals\u001b[39m()[\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCancel\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnm\u001b[38;5;241m.\u001b[39mtitle()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124mException\u001b[39m\u001b[38;5;124m'\u001b[39m]: \u001b[38;5;28;01mpass\u001b[39;00m\n",
      "File \u001b[0;32m~/Desktop/Giannis/Code/pclab/pclab/learner.py:88\u001b[0m, in \u001b[0;36mLearner._one_epoch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[38;5;129m@with_cbs\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mepoch\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     87\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_one_epoch\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m---> 88\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miter,\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdl): \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_one_batch\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Desktop/Giannis/Code/pclab/pclab/learner.py:61\u001b[0m, in \u001b[0;36mwith_cbs.__call__.<locals>._f\u001b[0;34m(o, *args, **kwargs)\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     60\u001b[0m     o\u001b[38;5;241m.\u001b[39mcallback(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbefore_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnm\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 61\u001b[0m     \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     62\u001b[0m     o\u001b[38;5;241m.\u001b[39mcallback(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnm\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     63\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;28mglobals\u001b[39m()[\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCancel\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnm\u001b[38;5;241m.\u001b[39mtitle()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124mException\u001b[39m\u001b[38;5;124m'\u001b[39m]: \u001b[38;5;28;01mpass\u001b[39;00m\n",
      "File \u001b[0;32m~/Desktop/Giannis/Code/pclab/pclab/learner.py:80\u001b[0m, in \u001b[0;36mLearner._one_batch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_loss\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     79\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining:\n\u001b[0;32m---> 80\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     81\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mafter_backward\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     82\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstep()\n",
      "File \u001b[0;32m~/Desktop/Giannis/Code/pclab/pclab/learner.py:127\u001b[0m, in \u001b[0;36mTrainLearner.backward\u001b[0;34m(self)\u001b[0m\n\u001b[0;32m--> 127\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mbackward\u001b[39m(\u001b[38;5;28mself\u001b[39m): \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/sparse/lib/python3.10/site-packages/torch/_tensor.py:487\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    478\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    479\u001b[0m         Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m    480\u001b[0m         (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    485\u001b[0m         inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m    486\u001b[0m     )\n\u001b[0;32m--> 487\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    488\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m    489\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/sparse/lib/python3.10/site-packages/torch/autograd/__init__.py:200\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    195\u001b[0m     retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m    197\u001b[0m \u001b[38;5;66;03m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m    198\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    199\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 200\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    201\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    202\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "lr = 0.0001 \n",
    "epochs = 20\n",
    "\n",
    "model = get_model()\n",
    "\n",
    "# scheduler\n",
    "total_steps = epochs * len(dls.train)\n",
    "sched = partial(torch.optim.lr_scheduler.OneCycleLR, max_lr=lr, total_steps = total_steps)\n",
    "\n",
    "# Callbacks\n",
    "ddpm_cb = DDPMCB()\n",
    "checkpoint_cb = CheckpointCB(1000, 'SuperRes', run_params={'msg':model.msg})\n",
    "cbs = [ddpm_cb, DeviceCBSparse(), ProgressCB(plot=False), LossCB(), GradientClipCB(), checkpoint_cb, BatchSchedCB(sched)]\n",
    "\n",
    "learn = TrainLearner(model, dls, CustomMSELoss(), lr=lr, cbs=cbs, opt_func=torch.optim.Adam)\n",
    "learn.fit(epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45130d19-ffa6-4e91-82bd-76615865e309",
   "metadata": {},
   "source": [
    "# Test Super-Resolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eff39065-9a25-44d9-a92a-4ad6d243f16d",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'utils'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtorchsparse\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SparseTensor\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mschedulers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SchedulerBase, SchedulingStrategy, DDPM\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mabc\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m abstractmethod\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmodels\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msparse_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m batch_sparse_quantize_torch\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'utils'"
     ]
    }
   ],
   "source": [
    "#export\n",
    "import torch\n",
    "from torchsparse import SparseTensor\n",
    "from utils.schedulers import SchedulerBase, SchedulingStrategy, DDPM\n",
    "from abc import abstractmethod\n",
    "from models.sparse_utils import batch_sparse_quantize_torch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cf9d68c-e5e6-4189-8313-af87c593e6a7",
   "metadata": {},
   "source": [
    "For completion a tensor with 4 features is required\n",
    "- `points[..., :3]` : x,y,z coordinates\n",
    "- `points[..., -1]` : Indicates if the point is from the original point cloud or if it padded noise. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "1b3a5a02-f0bc-409b-ae2e-0f8b05f2f4dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class CompletionScheduler(SchedulerBase):\n",
    "\n",
    "    @abstractmethod\n",
    "    def get_pc(self, x_t, shape):\n",
    "        pass\n",
    "\n",
    "    @abstractmethod\n",
    "    def pad_noise(x0, n_points):\n",
    "        pass\n",
    "    \n",
    "    @torch.no_grad()\n",
    "    def complete(self, x0, model, n_points, emb=None, save_process=False):\n",
    "        # get the model device\n",
    "        device = next(model.parameters()).device\n",
    "\n",
    "        # take x_0 to device\n",
    "        x0 = x0.to(device)\n",
    "\n",
    "        shape = torch.Size((x0.shape[0], n_points, x0.shape[-1] + 1))\n",
    "        \n",
    "        # pad noise to point cloud\n",
    "        x_t = self.pad_noise(x0, n_points)\n",
    "        if save_process: preds = [self.get_pc(x_t, shape)] \n",
    "        \n",
    "        for i, t in enumerate(self.strategy.steps):\n",
    "            x_t = self.sample_step(model, x_t, t, i, emb, shape, device)\n",
    "            if save_process: preds.append(self.get_pc(x_t, shape))\n",
    "        \n",
    "        return preds if save_process else self.get_pc(x_t, shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "f94b3b8d-5c9b-4d9a-8295-1baa4dfb3c0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class SparseCompletionScheduler(CompletionScheduler):\n",
    "\n",
    "    def __init__(self, strategy:SchedulingStrategy, save_process=False, pres=1e-5):\n",
    "        super().__init__(strategy, save_process)\n",
    "        self.pres = pres\n",
    "\n",
    "    def pad_noise(self, x0, n_points):\n",
    "        B, N, F = x0.shape\n",
    "        num_pad = n_points - N\n",
    "        noise = torch.randn(B, num_pad, F).to(x0.device)\n",
    "        padded_pc = torch.cat([x0, noise], dim=1)\n",
    "\n",
    "        mask = torch.zeros(B, n_points, 1).to(x0.device)\n",
    "        mask[:, :N] = 1\n",
    "        # print(N)\n",
    "        # print(x0.shape)\n",
    "        # print(n_points)\n",
    "        # print(mask.sum())\n",
    "        # print(mask.shape)\n",
    "\n",
    "        padded_pc = torch.cat([padded_pc, mask], dim=-1)\n",
    "\n",
    "        return self.torch2sparse(padded_pc, padded_pc.shape)       \n",
    "\n",
    "    def update_rule(self, x_t, noise_pred, t, i, shape, device):\n",
    "        mask = x_t.F[:, -1].bool()\n",
    "\n",
    "        # print(mask.sum())\n",
    "        \n",
    "        x_t_new = self.strategy.update_rule(x_t.F[:, :3], noise_pred[:, :3], t, i, shape, device)\n",
    "        \n",
    "        x_t = x_t.F[:, :3]* mask.unsqueeze(-1) + x_t_new * ~mask.unsqueeze(-1)\n",
    "        \n",
    "        x_t = torch.cat([x_t, mask.unsqueeze(-1)], dim=-1)\n",
    "\n",
    "        return self.torch2sparse(x_t, shape).to(device)\n",
    "\n",
    "    def get_pc(self, x_t, shape):\n",
    "        return x_t.F.detach().cpu().reshape(shape)[:, :, :3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "fd224939-334c-433f-81be-8608a6d23cc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class SparseCompletionSchedulerGPU(SparseCompletionScheduler):\n",
    "\n",
    "    def torch2sparse(self, pts:torch.Tensor, shape):\n",
    "        pts = pts.reshape(shape)\n",
    "        \n",
    "        coords = pts[..., :3] # In case points have additional features\n",
    "        coords = coords - coords.min(dim=1, keepdim=True).values\n",
    "        coords, indices = batch_sparse_quantize_torch(coords, voxel_size=self.pres, return_index=True, return_batch_index=False)\n",
    "        feats = pts.view(-1, shape[-1])[indices]\n",
    "\n",
    "        return SparseTensor(coords=coords, feats=feats).to(coords.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "1e65ae54-ea9a-4dff-a2bb-4e11f0a48956",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class DDPMSparseCompletionSchedulerGPU(SparseCompletionSchedulerGPU):\n",
    "    def __init__(self, beta_min=0.0001, beta_max=0.02, n_steps=1000, mode='linear', sigma='bt', pres=1e-5, save_process=False):\n",
    "        strategy = DDPM(beta_min=beta_min, beta_max=beta_max, n_steps=n_steps, mode=mode, sigma=sigma)\n",
    "        super().__init__(strategy, save_process=save_process, pres=pres)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "db9724bc-47a9-405b-ba8f-f1c816f84f9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = get_model().eval().cuda()\n",
    "model.load_state_dict(torch.load('./checkpoints/SuperRes.pt')['state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "55d2b896-b158-41e1-b6a5-f18e343b880d",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = next(iter(vl_dl))\n",
    "pc = batch['noisy_pc'].F\n",
    "mask = pc[:, -1].bool()\n",
    "pc = pc[mask, :3].reshape(32, 512, 3)[:2]\n",
    "quick_vis_batch(pc, grid=(2, 1), x_offset=8, y_offset= 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc1f60e9-72b1-4b1f-8988-66e0c9dae8bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad(t, np):\n",
    "    B, N, F = t.shape\n",
    "    padded = torch.zeros(B, np, F).to(t)\n",
    "    padded[:, :N, :] = t\n",
    "\n",
    "    return padded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "30496d00-5e6e-4b1a-801d-b66006ad2e6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "c = DDPMSparseCompletionSchedulerGPU()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "a62c642e-9734-4fef-ae08-71663a809fe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = c.complete(pc, model, n_points=4096, save_process=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "d76ef7d9-9893-455f-9cbf-408ace38f271",
   "metadata": {},
   "outputs": [],
   "source": [
    "vis = torch.cat([preds, pad(pc, 4096)], dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "d0f43c2f-0fac-4c91-8697-232b03302456",
   "metadata": {},
   "outputs": [],
   "source": [
    "quick_vis_batch(vis, grid=(2, 2), x_offset=8, y_offset= 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f46f68d-f984-46d8-b1de-490a8a038eba",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}