diff --git a/Rewrite VI, why not.ipynb b/Rewrite VI, why not.ipynb
new file mode 100644
index 0000000000..ec601bcabc
--- /dev/null
+++ b/Rewrite VI, why not.ipynb
@@ -0,0 +1,672 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "628a2479-a1a5-428c-9c43-73dc3a91fb59",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "9f946eb4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "\n",
+ "from tqdm import trange\n",
+ "\n",
+ "import pymc as pm\n",
+ "\n",
+ "from pymc.variational.autoguide import (\n",
+ " AutoDiagonalNormal,\n",
+ " compile_svi_training_fn,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "e746bc33",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Sampling: [X, alpha, beta, sigma, y]\n"
+ ]
+ }
+ ],
+ "source": [
+ "with pm.Model() as m:\n",
+ " X = pm.Normal(\"X\", 0, 1, size=(100, 3))\n",
+ " alpha = pm.Normal(\"alpha\", 0, 10)\n",
+ " beta = pm.Normal(\"beta\", 0, 5, size=(3,))\n",
+ "\n",
+ " mu = alpha + X @ beta\n",
+ " sigma = pm.HalfNormal(\"sigma\", 1)\n",
+ " y = pm.Normal(\"y\", mu=mu, sigma=sigma)\n",
+ "\n",
+ " prior = pm.sample_prior_predictive(random_seed=38)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "a8ca0161",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "draw = 123\n",
+ "true_params = {}\n",
+ "true_params[\"alpha\"] = prior.prior.alpha.sel(chain=0, draw=draw).values\n",
+ "true_params[\"beta\"] = prior.prior.beta.sel(chain=0, draw=draw).values\n",
+ "true_params[\"sigma\"] = prior.prior.sigma.sel(chain=0, draw=draw).values\n",
+ "\n",
+ "X_data = prior.prior.X.sel(chain=0, draw=draw).values\n",
+ "y_data = prior.prior.y.sel(chain=0, draw=draw).values"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "dea42437-a68f-4602-9239-d5f60faab70d",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'alpha': array(11.82213038),\n",
+ " 'beta': array([-0.92518728, 0.27270752, -0.20081106]),\n",
+ " 'sigma': array(0.40007044)}"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "true_params"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "b89f4031",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "m_obs = pm.observe(pm.do(m, {X: X_data}), {\"y\": y_data})"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "015bdc56-309d-40e4-8e89-4f75691e2301",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Initializing NUTS using jitter+adapt_diag...\n",
+ "Multiprocess sampling (4 chains in 4 jobs)\n",
+ "NUTS: [sigma, beta, alpha]\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "04bbc4b8d6db4c8190e19fed7e7c8bcb",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Output()"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n"
+ ],
+ "text/plain": []
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Sampling 4 chains for 1_000 tune and 1_000 draw iterations (4_000 + 4_000 draws total) took 17 seconds.\n"
+ ]
+ }
+ ],
+ "source": [
+ "with m_obs:\n",
+ " idata = pm.sample(mp_ctx=\"spawn\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "4a4b3b07-0ef9-4387-93bc-622ae98aacb9",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "[autoreload of cutils_ext failed: Traceback (most recent call last):\n",
+ " File \"/home/ricardo/miniforge3/envs/pymc-dev/lib/python3.12/site-packages/IPython/extensions/autoreload.py\", line 276, in check\n",
+ " superreload(m, reload, self.old_objects)\n",
+ " File \"/home/ricardo/miniforge3/envs/pymc-dev/lib/python3.12/site-packages/IPython/extensions/autoreload.py\", line 475, in superreload\n",
+ " module = reload(module)\n",
+ " ^^^^^^^^^^^^^^\n",
+ " File \"/home/ricardo/miniforge3/envs/pymc-dev/lib/python3.12/importlib/__init__.py\", line 130, in reload\n",
+ " raise ModuleNotFoundError(f\"spec not found for the module {name!r}\", name=name)\n",
+ "ModuleNotFoundError: spec not found for the module 'cutils_ext'\n",
+ "]\n"
+ ]
+ }
+ ],
+ "source": [
+ "guide = AutoDiagonalNormal(m_obs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "69469eda-ae7d-4259-bb17-f69640d378dd",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{sigma_loc: array(-0.81661984),\n",
+ " sigma_scale: array(0.1),\n",
+ " beta_loc: array([ 0.92426301, -0.29670779, 0.8050404 ]),\n",
+ " beta_scale: array([0.1, 0.1, 0.1]),\n",
+ " alpha_loc: array(-0.78312419),\n",
+ " alpha_scale: array(0.1)}"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "guide.params_init_values"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "e735cd44-49e0-452e-a000-b56b27e60bca",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'sigma_loc': array(0.40007044),\n",
+ " 'sigma_scale': array(0.1),\n",
+ " 'beta_loc': array([-0.92518728, 0.27270752, -0.20081106]),\n",
+ " 'beta_scale': array([0.1, 0.1, 0.1]),\n",
+ " 'alpha_loc': array(11.82213038),\n",
+ " 'alpha_scale': array(0.1)}"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "true_loc_dict = {k.name: v for k, v in guide.params_init_values.items()}\n",
+ "for key, value in true_params.items():\n",
+ " true_loc_dict[f\"{key}_loc\"] = value\n",
+ "true_loc_dict"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "8ef58408-639a-460e-bcbf-c2d17b58ad09",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "f_loss_dloss = compile_svi_training_fn(m_obs, guide)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "b9f1edb0-b70f-4130-9a89-16287738330f",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[array(267.06378459),\n",
+ " array(-145.85250709),\n",
+ " array(154.00018737),\n",
+ " array([-11.96743328, 2.79924744, -2.35017322]),\n",
+ " array([39.80956106, 53.99061549, 28.58763406]),\n",
+ " array(0.96201239),\n",
+ " array(43.20248001)]"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "f_loss_dloss(np.array(500), **true_loc_dict)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "6086f2cc",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Loss: 461.90: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2000/2000 [00:03<00:00, 626.57it/s]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "{'sigma_loc': array(2.23715394),\n",
+ " 'sigma_scale': array(-1.810869),\n",
+ " 'beta_loc': array([ 0.75573259, -0.25781575, 0.7929361 ]),\n",
+ " 'beta_scale': array([0.09952254, 0.0982991 , 0.10187045]),\n",
+ " 'alpha_loc': array(-0.26682673),\n",
+ " 'alpha_scale': array(0.0957252)}"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "init_param_values = {k.name: v for k, v in guide.params_init_values.items()}\n",
+ "opt_param_values = list(init_param_values.values())\n",
+ "learning_rate = 1e-5\n",
+ "n_iter = 2_000\n",
+ "loss_history = np.empty(n_iter)\n",
+ "progress_bar = trange(n_iter)\n",
+ "draws = np.array(500, dtype=int)\n",
+ "for i in progress_bar:\n",
+ " loss, *grads = f_loss_dloss(draws, *opt_param_values)\n",
+ " loss_history[i] = loss\n",
+ " opt_param_values = [\n",
+ " np.asarray(value - learning_rate * grad) for value, grad in zip(opt_param_values, grads)\n",
+ " ]\n",
+ " if i % 50 == 0:\n",
+ " progress_bar.set_description(f\"Loss: {loss:.2f}\")\n",
+ " if i % 5_000 == 0 and i > 0:\n",
+ " learning_rate = max(learning_rate / 5, 1e-5)\n",
+ "\n",
+ "optimized_params = dict(zip(init_param_values, opt_param_values))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "id": "7f78e19d-1b1d-4b48-91b7-7711f35704e7",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[]"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjEAAAGdCAYAAADjWSL8AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAMlVJREFUeJzt3X90VPWd//HX5CcQk4EkTYapAdFFpSZFjRpIXUHBADXELVvRho10y6JWBVPAHxzXiv22RGkL7jGrol9qqOim57srHLduI9Dlhyw/DUYFEWFN+aEZgjRMEgiTkPl8/4BcMkkAJ0ycueH5OGcOufe+753PZ+5w8sqdz/2MwxhjBAAAYDNR4W4AAABAdxBiAACALRFiAACALRFiAACALRFiAACALRFiAACALRFiAACALRFiAACALcWEuwE9xe/368svv1RiYqIcDke4mwMAAL4GY4waGhrkdrsVFXXuay29NsR8+eWXysjICHczAABANxw4cECXXnrpOWt6bYhJTEyUdOpFSEpKCnNrAADA11FfX6+MjAzr9/i59NoQ0/YRUlJSEiEGAACb+TpDQRjYCwAAbIkQAwAAbIkQAwAAbIkQAwAAbIkQAwAAbIkQAwAAbIkQAwAAbIkQAwAAbIkQAwAAbIkQAwAAbIkQAwAAbIkQAwAAbKnXfgFkT9lb26hlm/fJ5eyjB0ZdEe7mAABw0eJKTJC+PNqkso1/0dtVX4a7KQAAXNQIMQAAwJYIMQAAwJYIMQAAwJYIMd1kwt0AAAAucoSYIDkc4W4BAACQCDEAAMCmCDEAAMCWCDEAAMCWCDHdZAxDewEACCdCTJAcYmQvAACRgBADAABsiRADAABsiRADAABsiRADAABsiRATJGbsBQAgMhBiAACALRFiAACALRFiAACALRFiuokJewEACC9CTJAY1wsAQGQgxAAAAFsixAAAAFsixAAAAFsixHSTESN7AQAIJ0JMsBjZCwBARCDEAAAAWyLEAAAAWyLEAAAAWwo6xKxfv14TJ06U2+2Ww+HQihUrOtXs2rVLBQUFcjqdSkxM1IgRI7R//35ru8/n04wZM5SamqqEhAQVFBTo4MGDAceoq6tTUVGRnE6nnE6nioqKdPTo0aA72FOYsRcAgPAKOsQcO3ZMw4cPV2lpaZfb//d//1c333yzrr76aq1du1YffvihnnrqKfXp08eqKS4u1vLly1VeXq4NGzaosbFR+fn5am1ttWoKCwtVVVWliooKVVRUqKqqSkVFRd3oYmg5GNkLAEBEiAl2hwkTJmjChAln3f7kk0/q+9//vhYsWGCtu/zyy62fvV6vlixZotdff11jx46VJC1btkwZGRlavXq1xo0bp127dqmiokKbN29WTk6OJOnVV1/VyJEjtXv3bl111VXBNhsAAPQyIR0T4/f79c477+jKK6/UuHHjlJaWppycnICPnCorK9XS0qK8vDxrndvtVmZmpjZu3ChJ2rRpk5xOpxVgJGnEiBFyOp1WTUc+n0/19fUBDwAA0HuFNMTU1taqsbFRzz77rMaPH6+VK1fqBz/4gSZNmqR169ZJkjwej+Li4jRgwICAfdPT0+XxeKyatLS0TsdPS0uzajoqKSmxxs84nU5lZGSEsmsAACDChPxKjCTdeeed+tnPfqZrr71WTzzxhPLz8/Xyyy+fc19jjByOM+NN2v98tpr25s6dK6/Xaz0OHDhwAT05P8b1AgAQXiENMampqYqJidF3vvOdgPXDhg2z7k5yuVxqbm5WXV1dQE1tba3S09OtmkOHDnU6/uHDh62ajuLj45WUlBTw6AlnyVAAAOAbFtIQExcXpxtvvFG7d+8OWP/ZZ59p8ODBkqTs7GzFxsZq1apV1vaamhrt2LFDubm5kqSRI0fK6/Vq69atVs2WLVvk9XqtGgAAcHEL+u6kxsZG7d2711qurq5WVVWVkpOTNWjQID366KO6++67dcstt+jWW29VRUWF/vM//1Nr166VJDmdTk2bNk2zZ89WSkqKkpOTNWfOHGVlZVl3Kw0bNkzjx4/X9OnTtXjxYknSfffdp/z8fO5MAgAAkroRYt5//33deuut1vKsWbMkSVOnTlVZWZl+8IMf6OWXX1ZJSYlmzpypq666Sv/xH/+hm2++2dpn0aJFiomJ0eTJk9XU1KQxY8aorKxM0dHRVs0bb7yhmTNnWncxFRQUnHVuGgAAcPFxGNM7556tr6+X0+mU1+sN6fiYzZ8f0T2vbNYV30rQn2ePDtlxAQBAcL+/+e6kIDGuFwCAyECIAQAAtkSIAQAAtkSIAQAAtkSI6aZeORoaAAAbIcQE6WxfewAAAL5ZhBgAAGBLhBgAAGBLhBgAAGBLhJjuYmQvAABhRYgJEuN6AQCIDIQYAABgS4QYAABgS4QYAABgS4SYbmJcLwAA4UWICRLjegEAiAyEGAAAYEuEGAAAYEuEGAAAYEuEmG4yhqG9AACEEyEmSMzYCwBAZCDEAAAAWyLEAAAAWyLEAAAAWyLEdBPDegEACC9CTNAY2QsAQCQgxAAAAFsixAAAAFsixAAAAFsixHQTE/YCABBehJggMWMvAACRgRADAABsiRADAABsiRADAABsiRDTTYY5ewEACCtCTJAY1wsAQGQgxAAAAFsixAAAAFsixAAAAFsixHQTM/YCABBehJggOZiyFwCAiECIAQAAthR0iFm/fr0mTpwot9sth8OhFStWnLX2/vvvl8Ph0PPPPx+w3ufzacaMGUpNTVVCQoIKCgp08ODBgJq6ujoVFRXJ6XTK6XSqqKhIR48eDba5AACglwo6xBw7dkzDhw9XaWnpOetWrFihLVu2yO12d9pWXFys5cuXq7y8XBs2bFBjY6Py8/PV2tpq1RQWFqqqqkoVFRWqqKhQVVWVioqKgm0uAADopWKC3WHChAmaMGHCOWu++OILPfzww3r33Xd1xx13BGzzer1asmSJXn/9dY0dO1aStGzZMmVkZGj16tUaN26cdu3apYqKCm3evFk5OTmSpFdffVUjR47U7t27ddVVVwXb7JBjYC8AAOEV8jExfr9fRUVFevTRR3XNNdd02l5ZWamWlhbl5eVZ69xutzIzM7Vx40ZJ0qZNm+R0Oq0AI0kjRoyQ0+m0asKFYb0AAESGoK/EnM9zzz2nmJgYzZw5s8vtHo9HcXFxGjBgQMD69PR0eTweqyYtLa3TvmlpaVZNRz6fTz6fz1qur6/vbhcAAIANhPRKTGVlpf7lX/5FZWVlQd+KbIwJ2Ker/TvWtFdSUmINAnY6ncrIyAiu8QAAwFZCGmLee+891dbWatCgQYqJiVFMTIz27dun2bNn67LLLpMkuVwuNTc3q66uLmDf2tpapaenWzWHDh3qdPzDhw9bNR3NnTtXXq/Xehw4cCCUXQMAABEmpCGmqKhIH330kaqqqqyH2+3Wo48+qnfffVeSlJ2drdjYWK1atcrar6amRjt27FBubq4kaeTIkfJ6vdq6datVs2XLFnm9Xqumo/j4eCUlJQU8AABA7xX0mJjGxkbt3bvXWq6urlZVVZWSk5M1aNAgpaSkBNTHxsbK5XJZdxQ5nU5NmzZNs2fPVkpKipKTkzVnzhxlZWVZdysNGzZM48eP1/Tp07V48WJJ0n333af8/Pyw35nEhL0AAESGoEPM+++/r1tvvdVanjVrliRp6tSpKisr+1rHWLRokWJiYjR58mQ1NTVpzJgxKisrU3R0tFXzxhtvaObMmdZdTAUFBeedmwYAAFw8HMb0zhlP6uvr5XQ65fV6Q/rR0kcHj6qg9H/07f599T9P3Bay4wIAgOB+f/PdSQAAwJYIMd3USy9gAQBgG4SYIDmYsxcAgIhAiAEAALZEiAEAALZEiAEAALZEiOkmhvUCABBehJggMWMvAACRgRADAABsiRADAABsiRADAABsiRDTTUzYCwBAeBFiAACALRFiAACALRFiAACALRFiAACALRFiuskwZy8AAGFFiAEAALZEiAkSXzsAAEBkIMQAAABbIsQAAABbIsR0EzP2AgAQXoQYAABgS4SYIDnEyF4AACIBIQYAANgSIQYAANgSIaabGNcLAEB4EWIAAIAtEWKCxIy9AABEBkIMAACwJUIMAACwJUJMNzFjLwAA4UWIAQAAtkSICRIDewEAiAyEGAAAYEuEGAAAYEuEGAAAYEuEmG7j9iQAAMKJEBMkhxjZCwBAJCDEAAAAWyLEAAAAWyLEAAAAWyLEdBNfOwAAQHgFHWLWr1+viRMnyu12y+FwaMWKFda2lpYWPf7448rKylJCQoLcbrfuvfdeffnllwHH8Pl8mjFjhlJTU5WQkKCCggIdPHgwoKaurk5FRUVyOp1yOp0qKirS0aNHu9XJUGLGXgAAIkPQIebYsWMaPny4SktLO207fvy4tm/frqeeekrbt2/XW2+9pc8++0wFBQUBdcXFxVq+fLnKy8u1YcMGNTY2Kj8/X62trVZNYWGhqqqqVFFRoYqKClVVVamoqKgbXQQAAL1RTLA7TJgwQRMmTOhym9Pp1KpVqwLWvfDCC7rpppu0f/9+DRo0SF6vV0uWLNHrr7+usWPHSpKWLVumjIwMrV69WuPGjdOuXbtUUVGhzZs3KycnR5L06quvauTIkdq9e7euuuqqYJsNAAB6mR4fE+P1euVwONS/f39JUmVlpVpaWpSXl2fVuN1uZWZmauPGjZKkTZs2yel0WgFGkkaMGCGn02nVdOTz+VRfXx/wAAAAvVePhpgTJ07oiSeeUGFhoZKSkiRJHo9HcXFxGjBgQEBtenq6PB6PVZOWltbpeGlpaVZNRyUlJdb4GafTqYyMjBD3JhDjegEACK8eCzEtLS2655575Pf79eKLL5633hgjR7tRs44uRtB2rGlv7ty58nq91uPAgQPdb/w5MK4XAIDI0CMhpqWlRZMnT1Z1dbVWrVplXYWRJJfLpebmZtXV1QXsU1tbq/T0dKvm0KFDnY57+PBhq6aj+Ph4JSUlBTwAAEDvFfIQ0xZg9uzZo9WrVyslJSVge3Z2tmJjYwMGANfU1GjHjh3Kzc2VJI0cOVJer1dbt261arZs2SKv12vVAACAi1vQdyc1NjZq79691nJ1dbWqqqqUnJwst9utH/7wh9q+fbv++Mc/qrW11RrDkpycrLi4ODmdTk2bNk2zZ89WSkqKkpOTNWfOHGVlZVl3Kw0bNkzjx4/X9OnTtXjxYknSfffdp/z8fO5MAgAAkroRYt5//33deuut1vKsWbMkSVOnTtW8efP09ttvS5KuvfbagP3WrFmj0aNHS5IWLVqkmJgYTZ48WU1NTRozZozKysoUHR1t1b/xxhuaOXOmdRdTQUFBl3PThIthyl4AAMLKYXrpb+P6+no5nU55vd6Qjo/ZW9ugsQvXa0C/WH3w87zz7wAAAL62YH5/891JAADAlggxAADAlggxAADAlggx3dQrBxIBAGAjhJigMWcvAACRgBADAABsiRADAABsiRADAABsiRDTTb1zikAAAOyDEBMkB+N6AQCICIQYAABgS4QYAABgS4QYAABgS4SYbuqlX/4NAIBtEGKCxLheAAAiAyEGAADYEiEGAADYEiEGAADYEiGmmxjWCwBAeBFiguRgyl4AACICIQYAANgSIQYAANgSIQYAANgSIaa7GNkLAEBYEWKCxLBeAAAiAyEGAADYEiEGAADYEiEGAADYEiGmmxjXCwBAeBFigsSEvQAARAZCDAAAsCVCDAAAsCVCDAAAsCVCTDcZw9BeAADCiRATJAdz9gIAEBEIMQAAwJYIMQAAwJYIMQAAwJYIMd3EsF4AAMKLEBMkZuwFACAyEGIAAIAtEWIAAIAtBR1i1q9fr4kTJ8rtdsvhcGjFihUB240xmjdvntxut/r27avRo0dr586dATU+n08zZsxQamqqEhISVFBQoIMHDwbU1NXVqaioSE6nU06nU0VFRTp69GjQHQQAAL1T0CHm2LFjGj58uEpLS7vcvmDBAi1cuFClpaXatm2bXC6Xbr/9djU0NFg1xcXFWr58ucrLy7VhwwY1NjYqPz9fra2tVk1hYaGqqqpUUVGhiooKVVVVqaioqBtd7BlM2AsAQJiZCyDJLF++3Fr2+/3G5XKZZ5991lp34sQJ43Q6zcsvv2yMMebo0aMmNjbWlJeXWzVffPGFiYqKMhUVFcYYYz755BMjyWzevNmq2bRpk5FkPv3006/VNq/XayQZr9d7IV3sZP+RY2bw4380V//zn0J6XAAAENzv75COiamurpbH41FeXp61Lj4+XqNGjdLGjRslSZWVlWppaQmocbvdyszMtGo2bdokp9OpnJwcq2bEiBFyOp1WTUc+n0/19fUBDwAA0HuFNMR4PB5JUnp6esD69PR0a5vH41FcXJwGDBhwzpq0tLROx09LS7NqOiopKbHGzzidTmVkZFxwfwAAQOTqkbuTHB0mUzHGdFrXUcearurPdZy5c+fK6/VajwMHDnSj5QAAwC5CGmJcLpckdbpaUltba12dcblcam5uVl1d3TlrDh061On4hw8f7nSVp018fLySkpICHj3JMGcvAABhFdIQM2TIELlcLq1atcpa19zcrHXr1ik3N1eSlJ2drdjY2ICampoa7dixw6oZOXKkvF6vtm7datVs2bJFXq/XqgkXZuwFACAyxAS7Q2Njo/bu3WstV1dXq6qqSsnJyRo0aJCKi4s1f/58DR06VEOHDtX8+fPVr18/FRYWSpKcTqemTZum2bNnKyUlRcnJyZozZ46ysrI0duxYSdKwYcM0fvx4TZ8+XYsXL5Yk3XfffcrPz9dVV10Vin4DAACbCzrEvP/++7r11lut5VmzZkmSpk6dqrKyMj322GNqamrSgw8+qLq6OuXk5GjlypVKTEy09lm0aJFiYmI0efJkNTU1acyYMSorK1N0dLRV88Ybb2jmzJnWXUwFBQVnnZsGAABcfBzG9M5p2+rr6+V0OuX1ekM6PuZg3XHd/Nwa9YmN0qf/Z0LIjgsAAIL7/c13J3VT74x+AADYByEmSOe7VRwAAHwzCDEAAMCWCDEAAMCWCDEAAMCWCDHdxLheAADCixATJIb1AgAQGQgxAADAlggxAADAlggxAADAlggx3cXIXgAAwooQEyQm7AUAIDIQYgAAgC0RYgAAgC0RYgAAgC0RYrrJMLIXAICwIsQEycGcvQAARARCDAAAsCVCDAAAsCVCDAAAsCVCTDcZxvUCABBWhJggMWMvAACRgRADAABsiRADAABsiRADAABsiRDTTYzrBQAgvAgxQWJcLwAAkYEQAwAAbIkQAwAAbIkQAwAAbIkQ002GKXsBAAgrQkywGNkLAEBEIMQAAABbIsQAAABbIsQAAABbIsR0E8N6AQAIL0JMkByM7AUAICIQYgAAgC0RYgAAgC0RYgAAgC0RYrqJCXsBAAgvQkyQHIzrBQAgIoQ8xJw8eVL//M//rCFDhqhv3766/PLL9Ytf/EJ+v9+qMcZo3rx5crvd6tu3r0aPHq2dO3cGHMfn82nGjBlKTU1VQkKCCgoKdPDgwVA3FwAA2FTIQ8xzzz2nl19+WaWlpdq1a5cWLFigX//613rhhResmgULFmjhwoUqLS3Vtm3b5HK5dPvtt6uhocGqKS4u1vLly1VeXq4NGzaosbFR+fn5am1tDXWTAQCADcWE+oCbNm3SnXfeqTvuuEOSdNlll+nf/u3f9P7770s6dRXm+eef15NPPqlJkyZJkpYuXar09HS9+eabuv/+++X1erVkyRK9/vrrGjt2rCRp2bJlysjI0OrVqzVu3LhQNxsAANhMyK/E3Hzzzfrzn/+szz77TJL04YcfasOGDfr+978vSaqurpbH41FeXp61T3x8vEaNGqWNGzdKkiorK9XS0hJQ43a7lZmZadV05PP5VF9fH/AAAAC9V8ivxDz++OPyer26+uqrFR0drdbWVv3qV7/Sj370I0mSx+ORJKWnpwfsl56ern379lk1cXFxGjBgQKeatv07Kikp0TPPPBPq7nTCuF4AACJDyK/E/OEPf9CyZcv05ptvavv27Vq6dKl+85vfaOnSpQF1jg63+RhjOq3r6Fw1c+fOldfrtR4HDhy4sI4AAICIFvIrMY8++qieeOIJ3XPPPZKkrKws7du3TyUlJZo6dapcLpekU1dbBg4caO1XW1trXZ1xuVxqbm5WXV1dwNWY2tpa5ebmdvm88fHxio+PD3V3AABAhAr5lZjjx48rKirwsNHR0dYt1kOGDJHL5dKqVaus7c3NzVq3bp0VULKzsxUbGxtQU1NTox07dpw1xAAAgItLyK/ETJw4Ub/61a80aNAgXXPNNfrggw+0cOFC/eQnP5F06mOk4uJizZ8/X0OHDtXQoUM1f/589evXT4WFhZIkp9OpadOmafbs2UpJSVFycrLmzJmjrKws626lSPB1PgIDAAA9I+Qh5oUXXtBTTz2lBx98ULW1tXK73br//vv185//3Kp57LHH1NTUpAcffFB1dXXKycnRypUrlZiYaNUsWrRIMTExmjx5spqamjRmzBiVlZUpOjo61E0OCqEFAIDI4DCmd34LUH19vZxOp7xer5KSkkJ23L8ea9b1/+fUx1zVJd8n1AAAEELB/P7mu5MAAIAtEWIAAIAtEWIuQO/8IA4AAHsgxASJETAAAEQGQgwAALAlQgwAALAlQgwAALAlQswFYFwvAADhQ4gJEnPbAQAQGQgxAADAlggxAADAlggxAADAlggxF6CXfncmAAC2QIgJkoM5ewEAiAiEGAAAYEuEGAAAYEuEGAAAYEuEmAvAsF4AAMKHEBMsxvUCABARCDEAAMCWCDEAAMCWCDEAAMCWCDEXgAl7AQAIH0JMkBwM7AUAICIQYgAAgC0RYgAAgC0RYgAAgC0RYi6AYc5eAADChhATJMb1AgAQGQgxAADAlggxAADAlggxAADAlggxF4AZewEACB9CTJAcTNkLAEBEIMQAAABbIsQAAABbIsQAAABbIsQAAABbIsQEiWG9AABEBkIMAACwJUIMAACwJUIMAACwpR4JMV988YX+4R/+QSkpKerXr5+uvfZaVVZWWtuNMZo3b57cbrf69u2r0aNHa+fOnQHH8Pl8mjFjhlJTU5WQkKCCggIdPHiwJ5rbbczYCwBA+IQ8xNTV1el73/ueYmNj9ac//UmffPKJfvvb36p///5WzYIFC7Rw4UKVlpZq27Ztcrlcuv3229XQ0GDVFBcXa/ny5SovL9eGDRvU2Nio/Px8tba2hrrJQWHCXgAAIkNMqA/43HPPKSMjQ6+99pq17rLLLrN+Nsbo+eef15NPPqlJkyZJkpYuXar09HS9+eabuv/+++X1erVkyRK9/vrrGjt2rCRp2bJlysjI0OrVqzVu3LhQNxsAANhMyK/EvP3227rhhht01113KS0tTdddd51effVVa3t1dbU8Ho/y8vKsdfHx8Ro1apQ2btwoSaqsrFRLS0tAjdvtVmZmplUDAAAubiEPMZ9//rleeuklDR06VO+++64eeOABzZw5U7///e8lSR6PR5KUnp4esF96erq1zePxKC4uTgMGDDhrTUc+n0/19fUBDwAA0HuF/OMkv9+vG264QfPnz5ckXXfdddq5c6deeukl3XvvvVZdx2+DNsac9xuiz1VTUlKiZ5555gJbHxwjRvYCABAuIb8SM3DgQH3nO98JWDds2DDt379fkuRyuSSp0xWV2tpa6+qMy+VSc3Oz6urqzlrT0dy5c+X1eq3HgQMHQtKfjhzM2QsAQEQIeYj53ve+p927dwes++yzzzR48GBJ0pAhQ+RyubRq1Spre3Nzs9atW6fc3FxJUnZ2tmJjYwNqampqtGPHDqumo/j4eCUlJQU8AABA7xXyj5N+9rOfKTc3V/Pnz9fkyZO1detWvfLKK3rllVcknfoYqbi4WPPnz9fQoUM1dOhQzZ8/X/369VNhYaEkyel0atq0aZo9e7ZSUlKUnJysOXPmKCsry7pbCQAAXNxCHmJuvPFGLV++XHPnztUvfvELDRkyRM8//7ymTJli1Tz22GNqamrSgw8+qLq6OuXk5GjlypVKTEy0ahYtWqSYmBhNnjxZTU1NGjNmjMrKyhQdHR3qJgMAABtyGNM7552tr6+X0+mU1+sN6UdLTc2tGvbzCknSR/PylNQnNmTHBgDgYhfM72++OylIfWKj5ErqI0laufNQmFsDAMDFixATJIfDoXtzTw1S/r/vfa5eeiELAICIR4jphik3DVa/uGh96mnQe3u+CndzAAC4KBFiusHZL1Z335ghSfqXP+/hagwAAGFAiOmmn466QvExUarcV6e1nx0Od3MAALjoEGK6KS2pj6bmXiZJ+nXFbrX6uRoDAMA3iRBzAR4YdYWS+sTok5p6Ldu8L9zNAQDgokKIuQDJCXF6dPzVkqTfvLtbNd6mMLcIAICLByHmAhXeNEjXZvRXg++k5r71MYN8AQD4hhBiLlB0lEO/ueu7iouJ0trdh7Vsy/5wNwkAgIsCISYE/iYtUY+f/ljpl3/8RJ966sPcIgAAej9CTIj8Y+5lGnXlt+Q76de0svdVW38i3E0CAKBXI8SESFSUQ8/ffa0uT03QF0eb9JOl23TMdzLczQIAoNcixITQgIQ4vfaPNyolIU47vqjXvb/bqqPHm8PdLAAAeiVCTIgNTknQkh/fqKQ+MarcV6cfvrxJXxzl1msAAEKNENMDrs3or//3QK5cSX20t7ZRf//iRgb7AgAQYoSYHnKVK1FvPZiroWmXyFN/QpNe3Kh/rzzIPDIAAIQIIaYHufv31b8/kKvv/U2Kjje3as7/+1A/XbZdB/56PNxNAwDA9ggxPczZL1a//0mO5uRdqegohyp2ejTmt+tU8qddajjREu7mAQBgWw7TSz/fqK+vl9PplNfrVVJSUribI0n61FOvX72zS+/t+UqSlJIQp5/dfqXuuuFSxcdEh7l1AACEXzC/vwkx3zBjjNbsrtUv39mlzw8fkySlXhKnwpzB+oecQUpL6hPmFgIAED6EGEVuiGnT0urXG5v3afH6z1XjPTW7b2y0Q3dkDdQPszM04vJkxUTzaR8A4OJCiFHkh5g2La1+vbvTo9f+5y+q3FdnrU9JiNP4TJfu+O5A3XQZgQYAcHEgxMg+Iaa9jw4e1b9t3a+KHR7VHT8z6LdfXLSuzeiv6wcN0PWD++u6jAEakBAXxpYCANAzCDGyZ4hp09Lq1+bPj+idj2pUsdOjo8c738U0KLmfrnYl6uqBSfqbtEs0KLmfBif3U/9+sXI4HGFoNQAAF44QI3uHmPZa/UZ7axu1fX+dKvfVafv+OmtAcFcS+8RocEo/DU5O0KCUfnIl9VHqJfFKvSROKZfE61uXxCupbwxBBwAQkQgx6j0hpit1x5q1y1OvT2sa9KmnXtVfHdO+I8dV2+D7WvvHRUcp5ZI4JfWJ1SV9YpTYJ0aXxJ/595L4M+sT42N0Sbvt/eJiFB8TpfjYaMXHRCkmykEgAgCEDCFGvTvEnE1Tc6sO1B3XviPHte/IMR3466lg81WjT181NuurBp8afCdD+pxRDik+JlrxsVGnwk1MtOJiohQbHaW4aIf1c2x0lOJiohQXHaWYaIdioqIUF3Pq35hoh2KjTwWimCiHYqKjFB3lUGy0Q9FRUYp2SFFRDjkkORwOORxSlMOhKIfk0KllR9vy6W2n2uZQlFUvSW01Z2odbTWn6x1t6zvVnnneLmujzmw7V21Uh21d1kYpYL/2/QnoO+ERQC8UzO/vmG+oTfgG9I2L1pXpiboyPfGsNSdaWnXkWLOONPpU33RSjb4WNZw4qUbfSTWe/rfeWm5Ro++ktb3hxEkdbz6pltYzuddvpKaWVjW1tH4TXUQHp0JU+6Dk6BSArJqojqHqzHL7oNYxVOlrhMa2QBYQ/HSW8GYd+0ztmYB5JnwqoB3t2ne6NqAfp4Nq+/AZ0I9OIfjctQp4DQP73antHfp7vnDd/vVpH6Y7BdcO57bL4Hu6pmMfHO3aoHbL1s/t23h6ndpv15k2WcfoeLwunqvtMIHHDqyTo+vnaH98dfG8bfVtLW1rAy5uhJiLTJ/YaH27f199u3/fbh/D7zdqbvXL1+KX72SrfCf9px+tOtHiV0urX80nT/3b0npqW0ursZabTy+fbPWrxX9q/clWv076jU62Gp30G7X6/TrZatTiN/IbIxnJb4xM2786NXGg37T79/Q607HWSEbnrm3b1rH29FOfrjm9HHCcttq2dafa5ve3tbGr2o796P75bN++02u6fzDAproO3h2CqmQFw4ArrDoT4tqHuk5BrMvwF3jcM8GyfeBrF9TatUkBAa7rsNfxj4Fzhul24T46qvNV23Y5tVOAbf8cat9267Xp0CfrdXfoirRLVDRicGhPaBAIMQhaVJRDfaKi1Sc2WlJsuJvTK3QMZG2hpFNw858JQ2dClbGCTEBwa7ffBdX6z4SwrmrbAmb79qtduAsMnW3P21Vo7FDrb3uOc9S2P97pbeoQGjuH4PZ9afdaqqsQ3Lm2fZ2Rkd8f2K6uXuNT57jrIB74enZov7+LcB7w/F2dy8BwbLXfWnemPdaxrPdhu2WjduvbhfJ2x5PpePwLD+bdYYzUSpAPi1FXfosQA1zsHA6Hotv/qQTYXPug1VUIUlfL7fZtWzZG5wxL5nTaOls4D7gC2na8DsGu/RXbrtrW6TnP9nO7tnTcvy1Mt/9joHPw66Jd7V+PgLZ28UdEu762+jtfYT7Tr67ORbtw2r4/7faxFtqdq8EpCd16f4QKIQYAEHJtH8GcXgpnU9CLMZc9AACwJUIMAACwJUIMAACwJUIMAACwJUIMAACwJUIMAACwJUIMAACwJUIMAACwJUIMAACwJUIMAACwpR4PMSUlJXI4HCouLrbWGWM0b948ud1u9e3bV6NHj9bOnTsD9vP5fJoxY4ZSU1OVkJCggoICHTx4sKebCwAAbKJHQ8y2bdv0yiuv6Lvf/W7A+gULFmjhwoUqLS3Vtm3b5HK5dPvtt6uhocGqKS4u1vLly1VeXq4NGzaosbFR+fn5am1t7ckmAwAAm+ixENPY2KgpU6bo1Vdf1YABA6z1xhg9//zzevLJJzVp0iRlZmZq6dKlOn78uN58801Jktfr1ZIlS/Tb3/5WY8eO1XXXXadly5bp448/1urVq3uqyQAAwEZ67FusH3roId1xxx0aO3asfvnLX1rrq6ur5fF4lJeXZ62Lj4/XqFGjtHHjRt1///2qrKxUS0tLQI3b7VZmZqY2btyocePGdXo+n88nn89nLXu9XklSfX19T3QPAAD0gLbf28aY89b2SIgpLy/X9u3btW3btk7bPB6PJCk9PT1gfXp6uvbt22fVxMXFBVzBaatp27+jkpISPfPMM53WZ2RkdKsPAAAgfBoaGuR0Os9ZE/IQc+DAAT3yyCNauXKl+vTpc9Y6h8MRsGyM6bSuo3PVzJ07V7NmzbKW/X6//vrXvyolJeW8xw1WfX29MjIydODAASUlJYX02JGGvvZeF1N/6WvvdTH192LpqzFGDQ0Ncrvd560NeYiprKxUbW2tsrOzrXWtra1av369SktLtXv3bkmnrrYMHDjQqqmtrbWuzrhcLjU3N6uuri7gakxtba1yc3O7fN74+HjFx8cHrOvfv3+outWlpKSkXv1Gao++9l4XU3/pa+91MfX3Yujr+a7AtAn5wN4xY8bo448/VlVVlfW44YYbNGXKFFVVVenyyy+Xy+XSqlWrrH2am5u1bt06K6BkZ2crNjY2oKampkY7duw4a4gBAAAXl5BfiUlMTFRmZmbAuoSEBKWkpFjri4uLNX/+fA0dOlRDhw7V/Pnz1a9fPxUWFko6lcCmTZum2bNnKyUlRcnJyZozZ46ysrI0duzYUDcZAADYUI/dnXQujz32mJqamvTggw+qrq5OOTk5WrlypRITE62aRYsWKSYmRpMnT1ZTU5PGjBmjsrIyRUdHh6PJAeLj4/X00093+viqN6KvvdfF1F/62ntdTP29mPr6dTnM17mHCQAAIMLw3UkAAMCWCDEAAMCWCDEAAMCWCDEAAMCWCDFBevHFFzVkyBD16dNH2dnZeu+998LdpKCVlJToxhtvVGJiotLS0vR3f/d31iSEbX784x/L4XAEPEaMGBFQ4/P5NGPGDKWmpiohIUEFBQU6ePDgN9mV85o3b16nfrhcLmu7MUbz5s2T2+1W3759NXr0aO3cuTPgGHboZ5vLLrusU38dDoceeughSfY+r+vXr9fEiRPldrvlcDi0YsWKgO2hOpd1dXUqKiqS0+mU0+lUUVGRjh492sO9C3Suvra0tOjxxx9XVlaWEhIS5Ha7de+99+rLL78MOMbo0aM7net77rknoCYS+iqd/9yG6n0bCf09X1+7+v/rcDj061//2qqx07ntaYSYIPzhD39QcXGxnnzySX3wwQf627/9W02YMEH79+8Pd9OCsm7dOj300EPavHmzVq1apZMnTyovL0/Hjh0LqBs/frxqamqsx3/9138FbC8uLtby5ctVXl6uDRs2qLGxUfn5+Wptbf0mu3Ne11xzTUA/Pv74Y2vbggULtHDhQpWWlmrbtm1yuVy6/fbb1dDQYNXYpZ+StG3btoC+tk0Yedddd1k1dj2vx44d0/Dhw1VaWtrl9lCdy8LCQlVVVamiokIVFRWqqqpSUVFRj/evvXP19fjx49q+fbueeuopbd++XW+99ZY+++wzFRQUdKqdPn16wLlevHhxwPZI6Kt0/nMrheZ9Gwn9PV9f2/expqZGv/vd7+RwOPT3f//3AXV2Obc9zuBru+mmm8wDDzwQsO7qq682TzzxRJhaFBq1tbVGklm3bp21burUqebOO+886z5Hjx41sbGxpry83Fr3xRdfmKioKFNRUdGTzQ3K008/bYYPH97lNr/fb1wul3n22WetdSdOnDBOp9O8/PLLxhj79PNsHnnkEXPFFVcYv99vjOk951WSWb58ubUcqnP5ySefGElm8+bNVs2mTZuMJPPpp5/2cK+61rGvXdm6dauRZPbt22etGzVqlHnkkUfOuk8k9tWYrvsbivdtJPb365zbO++809x2220B6+x6bnsCV2K+pubmZlVWViovLy9gfV5enjZu3BimVoWG1+uVJCUnJwesX7t2rdLS0nTllVdq+vTpqq2ttbZVVlaqpaUl4PVwu93KzMyMuNdjz549crvdGjJkiO655x59/vnnkqTq6mp5PJ6APsTHx2vUqFFWH+zUz46am5u1bNky/eQnPwn4EtTecl7bC9W53LRpk5xOp3JycqyaESNGyOl0RnT/vV6vHA5Hp++Le+ONN5SamqprrrlGc+bMCbgqZbe+Xuj71m79laRDhw7pnXfe0bRp0zpt603n9kKEZcZeO/rqq6/U2tpqfUllm/T0dHk8njC16sIZYzRr1izdfPPNAV8XMWHCBN11110aPHiwqqur9dRTT+m2225TZWWl4uPj5fF4FBcXF/AFnVLkvR45OTn6/e9/ryuvvFKHDh3SL3/5S+Xm5mrnzp1WO7s6p/v27ZMk2/SzKytWrNDRo0f14x//2FrXW85rR6E6lx6PR2lpaZ2On5aWFrH9P3HihJ544gkVFhYGfCnglClTNGTIELlcLu3YsUNz587Vhx9+aH3EaKe+huJ9a6f+tlm6dKkSExM1adKkgPW96dxeKEJMkNr/RSudCgEd19nJww8/rI8++kgbNmwIWH/33XdbP2dmZuqGG27Q4MGD9c4773T6D9VepL0eEyZMsH7OysrSyJEjdcUVV2jp0qXWwMDunNNI62dXlixZogkTJgR8nX1vOa9nE4pz2VV9pPa/paVF99xzj/x+v1588cWAbdOnT7d+zszM1NChQ3XDDTdo+/btuv766yXZp6+het/apb9tfve732nKlCnq06dPwPredG4vFB8nfU2pqamKjo7ulGJra2s7/fVnFzNmzNDbb7+tNWvW6NJLLz1n7cCBAzV48GDt2bNHkuRyudTc3Ky6urqAukh/PRISEpSVlaU9e/ZYdymd65zatZ/79u3T6tWr9U//9E/nrOst5zVU59LlcunQoUOdjn/48OGI639LS4smT56s6upqrVq1KuAqTFeuv/56xcbGBpxru/S1o+68b+3W3/fee0+7d+8+7/9hqXed22ARYr6muLg4ZWdnW5fr2qxatUq5ublhalX3GGP08MMP66233tJ///d/a8iQIefd58iRIzpw4IAGDhwoScrOzlZsbGzA61FTU6MdO3ZE9Ovh8/m0a9cuDRw40Loc274Pzc3NWrdundUHu/bztddeU1pamu64445z1vWW8xqqczly5Eh5vV5t3brVqtmyZYu8Xm9E9b8twOzZs0erV69WSkrKeffZuXOnWlparHNtl752pTvvW7v1d8mSJcrOztbw4cPPW9ubzm3QwjGa2K7Ky8tNbGysWbJkifnkk09McXGxSUhIMH/5y1/C3bSg/PSnPzVOp9OsXbvW1NTUWI/jx48bY4xpaGgws2fPNhs3bjTV1dVmzZo1ZuTIkebb3/62qa+vt47zwAMPmEsvvdSsXr3abN++3dx2221m+PDh5uTJk+HqWiezZ882a9euNZ9//rnZvHmzyc/PN4mJidY5e/bZZ43T6TRvvfWW+fjjj82PfvQjM3DgQNv1s73W1lYzaNAg8/jjjwest/t5bWhoMB988IH54IMPjCSzcOFC88EHH1h35ITqXI4fP95897vfNZs2bTKbNm0yWVlZJj8/P2L62tLSYgoKCsyll15qqqqqAv4P+3w+Y4wxe/fuNc8884zZtm2bqa6uNu+88465+uqrzXXXXRdxfT1ff0P5vo2E/p7vfWyMMV6v1/Tr18+89NJLnfa327ntaYSYIP3rv/6rGTx4sImLizPXX399wG3JdiGpy8drr71mjDHm+PHjJi8vz3zrW98ysbGxZtCgQWbq1Klm//79AcdpamoyDz/8sElOTjZ9+/Y1+fn5nWrC7e677zYDBw40sbGxxu12m0mTJpmdO3da2/1+v3n66aeNy+Uy8fHx5pZbbjEff/xxwDHs0M/23n33XSPJ7N69O2C93c/rmjVrunzfTp061RgTunN55MgRM2XKFJOYmGgSExPNlClTTF1d3TfUy1PO1dfq6uqz/h9es2aNMcaY/fv3m1tuucUkJyebuLg4c8UVV5iZM2eaI0eORFxfz9ffUL5vI6G/53sfG2PM4sWLTd++fc3Ro0c77W+3c9vTHMYY06OXegAAAHoAY2IAAIAtEWIAAIAtEWIAAIAtEWIAAIAtEWIAAIAtEWIAAIAtEWIAAIAtEWIAAIAtEWIAAIAtEWIAAIAtEWIAAIAtEWIAAIAt/X/2ud8E3WdqSQAAAABJRU5ErkJggg==",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "window_size = 100\n",
+ "kernel = np.full(window_size, 1 / window_size)\n",
+ "plt.plot(np.convolve(loss_history, kernel, mode=\"valid\"))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "id": "38e1a57f-4644-412d-b046-999db6a59205",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Loss: 56.615: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2000/2000 [00:03<00:00, 639.06it/s]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "{'sigma_loc': array(-1.06275392),\n",
+ " 'sigma_scale': array(-2.61505505),\n",
+ " 'beta_loc': array([-0.91056208, 0.2716077 , -0.15967444]),\n",
+ " 'beta_scale': array([-3.28925422, -3.30663365, -3.19186797]),\n",
+ " 'alpha_loc': array(11.78263119),\n",
+ " 'alpha_scale': array(-3.30535793)}"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "n_iter = 2_000\n",
+ "loss_history = np.empty(n_iter)\n",
+ "progress_bar = trange(n_iter)\n",
+ "draws = np.array(500, dtype=int)\n",
+ "learning_rate = 1e-3\n",
+ "for i in progress_bar:\n",
+ " loss, *grads = f_loss_dloss(draws, *opt_param_values)\n",
+ " loss_history[i] = loss\n",
+ " if any(np.isnan(d_loss).any() for d_loss in grads):\n",
+ " print(\"Got nan, getting out\")\n",
+ " break\n",
+ " opt_param_values = [\n",
+ " np.asarray(value - learning_rate * grad) for value, grad in zip(opt_param_values, grads)\n",
+ " ]\n",
+ "\n",
+ " if i % 50 == 0:\n",
+ " progress_bar.set_description(f\"Loss: {loss:.3f}\")\n",
+ "\n",
+ "optimized_params = dict(zip(init_param_values, opt_param_values))\n",
+ "optimized_params"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "id": "650c5e39",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[]"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAQ3lJREFUeJzt3XlcVOe9P/DPmRXEYWSRGUaQEJeYOGgMxq257qJecYk2mmhSba3NojbcaGNNbm7MbSqJ+UWTXhvb5Fq3LNj2hiStS4JRSawaEUUBE8WICsKIGpgBxGF7fn8gEwcRHRg4s3zer9d5yZzzzJnvw0H5eM5zniMJIQSIiIiIPIhC7gKIiIiImmJAISIiIo/DgEJEREQehwGFiIiIPA4DChEREXkcBhQiIiLyOAwoRERE5HEYUIiIiMjjqOQuoDXq6+tRVFQEnU4HSZLkLoeIiIjugBAC5eXlMJlMUChaPkfilQGlqKgI0dHRcpdBRERErVBQUICoqKgW23hlQNHpdAAaOhgcHCxzNURERHQnbDYboqOjHb/HW+KVAaXxsk5wcDADChERkZe5k+EZHCRLREREHocBhYiIiDwOAwoRERF5HAYUIiIi8jgMKERERORxGFCIiIjI47QpoCQnJ0OSJCQlJTnWzZs3D5IkOS1Dhgxxep/dbsfixYsRHh6OoKAgTJkyBYWFhW0phYiIiHxIqwNKRkYG3n33XfTr1++mbRMmTEBxcbFj2b59u9P2pKQkpKamIiUlBfv27UNFRQUSExNRV1fX2nKIiIjIh7QqoFRUVGDOnDl47733EBISctN2rVYLo9HoWEJDQx3brFYr1q9fjzfffBNjx47FgAED8P777yM7Oxu7du1qfU+IiIjIZ7QqoCxcuBCTJk3C2LFjm92+d+9eREREoHfv3liwYAFKSkoc2zIzM1FTU4OEhATHOpPJBLPZjP3797emHCIiIvIxLk91n5KSgiNHjiAjI6PZ7RMnTsQjjzyCmJgY5Ofn46WXXsLo0aORmZkJrVYLi8UCjUZz05kXg8EAi8XS7D7tdjvsdrvjtc1mc7VsIiIi8iIuBZSCggI8++yz+OKLLxAQENBsm1mzZjm+NpvNGDhwIGJiYrBt2zZMnz79lvsWQtxybv7k5GS88sorrpRKREREXsylSzyZmZkoKSlBfHw8VCoVVCoV0tPT8Yc//AEqlarZQa6RkZGIiYlBXl4eAMBoNKK6uhqlpaVO7UpKSmAwGJr93OXLl8NqtTqWgoICV8q+Y/X1Ai+kZuOjQ+fbZf9ERER0Z1w6gzJmzBhkZ2c7rfv5z3+OPn36YNmyZVAqlTe958qVKygoKEBkZCQAID4+Hmq1GmlpaZg5cyYAoLi4GDk5OVi1alWzn6vVaqHVal0ptVW25xTjw28awokQwOzB3dv9M4mIiOhmLgUUnU4Hs9nstC4oKAhhYWEwm82oqKjAihUrMGPGDERGRuLs2bN44YUXEB4ejocffhgAoNfrMX/+fCxZsgRhYWEIDQ3F0qVLERcXd8tBtx1lUlwkjvykDH/5Vz5eSM2GgMCcwTGy1kREROSPXB4k2xKlUons7Gxs3rwZZWVliIyMxKhRo7B161bodDpHuzVr1kClUmHmzJmoqqrCmDFjsHHjxmbPwHQkSZLwUuK9kCRg/b58vJiag/p6gSeG3iVrXURERP5GEkIIuYtwlc1mg16vh9VqRXBwsNv3L4TAyu3f4r2v8wEAL0++Dz//SazbP4eIiMifuPL7m8/iaYYkSXjh3+/F0yN7AABe+ccJvPfVGZmrIiIi8h8MKLcgSRKeH38Pfj26JwDg99u/xTt7T8tcFRERkX9gQGmBJEl4LuEe/MfY3gCAVTtP4g9f5slcFRERke9jQLkDz47thd+MvwcAsDrtFFannYIXDt0hIiLyGgwod2jhqJ544d/7AAD+8GUe3vj8JEMKERFRO2FAccGvhvfAS4n3AQDe2fs9knd8x5BCRETUDhhQXDT/oVi8MqUvAODdr87gv/95giGFiIjIzRhQWmHusLvw+4cbZtTd8K+z+K9Pc1Ffz5BCRETkLgworTRncAxenxEHSQK2HDyHFz/JZkghIiJyEwaUNpj1YHe88dP+kCTgo0MFWPZ/x1HHkEJERNRmDCht9NP4KLw1634oJOBvmYX47f8d55kUIiKiNnLrwwL91dT7u0EhSUjamoW/ZRYCAF6f0Q8KhSRzZURERN6JAcVNJvc3AQCeTTnKkEJERNRGDChuxJBCRETkHhyD4maT+5vw1qMDfhyT8jHHpBAREbmKZ1DawZTrZ1KSUo7ir4cbzqS8Np1nUoiIiO4Uz6C0kyn9TVhz/e6evx4uxPKPOU8KERHRneIZlHY09f5uAID/2JqFrYcLAADJ0+N4JoWIiOg2GFDaWdOQIknAyocZUoiIiFrCgNIBbgwpKRkNZ1IYUoiIiG6NAaWDNA0pkgT8fhpDChERUXMYUDrQ1Pu7QQjgub9m4aNDDWdSGFKIiIhuxoDSwaYNaDiT8mNIkfD7aWaGFCIiohswoMhg2oBuEBBY8tdj+OjQeQBgSCEiIroBA4pMHh4QBQCOkCJJwKtTGVKIiIgABhRZNYaU5/56DB9+03AmhSGFiIiIAUV2DClEREQ3Y0DxAE1DigTgdwwpRETkx/gsHg/x8IAovPlIf0gS8ME35/Hf/zwBIfjsHiIi8k8MKB5k+gNR+H8/7Q8A2Lj/LN7alSdzRURERPJgQPEwM+Kj8MqUvgCAt7/Mw/p9+TJXRERE1PEYUDzQ3GF3Ycm43gCA3/3zBP56/UnIRERE/qJNASU5ORmSJCEpKcmxTgiBFStWwGQyITAwECNHjkRubq7T++x2OxYvXozw8HAEBQVhypQpKCwsbEspPmfR6J5Y8G+xAIDf/t9x7MgulrkiIiKijtPqgJKRkYF3330X/fr1c1q/atUqrF69GmvXrkVGRgaMRiPGjRuH8vJyR5ukpCSkpqYiJSUF+/btQ0VFBRITE1FXV9f6nvgYSZLwwr/fi1kDo1EvgF+nHMVXpy7JXRYREVGHaFVAqaiowJw5c/Dee+8hJCTEsV4Igbfeegsvvvgipk+fDrPZjE2bNuHq1av48MMPAQBWqxXr16/Hm2++ibFjx2LAgAF4//33kZ2djV27drmnVz5CkiSsnB6HSXGRqKkTeHJLJjLP/SB3WURERO2uVQFl4cKFmDRpEsaOHeu0Pj8/HxaLBQkJCY51Wq0WI0aMwP79+wEAmZmZqKmpcWpjMplgNpsdbehHSoWENbPux/DeXVFVU4efb8jAiSKb3GURERG1K5cDSkpKCo4cOYLk5OSbtlksFgCAwWBwWm8wGBzbLBYLNBqN05mXpm2astvtsNlsTos/0agU+NPjD2BgTAhs12rxs798g/zLlXKXRURE1G5cCigFBQV49tln8f777yMgIOCW7STJeQZUIcRN65pqqU1ycjL0er1jiY6OdqVsn9BJo8L6eQ/ivshgXK6oxuP/+w2KyqrkLouIiKhduBRQMjMzUVJSgvj4eKhUKqhUKqSnp+MPf/gDVCqV48xJ0zMhJSUljm1GoxHV1dUoLS29ZZumli9fDqvV6lgKCvzztlt9oBqbfjEIseFBuFBWhcfXf4MrFXa5yyIiInI7lwLKmDFjkJ2djaysLMcycOBAzJkzB1lZWbj77rthNBqRlpbmeE91dTXS09MxbNgwAEB8fDzUarVTm+LiYuTk5DjaNKXVahEcHOy0+KuuOi3e/+VgmPQBOHOpEnM3HILtWo3cZREREbmVSw8L1Ol0MJvNTuuCgoIQFhbmWJ+UlISVK1eiV69e6NWrF1auXIlOnTph9uzZAAC9Xo/58+djyZIlCAsLQ2hoKJYuXYq4uLibBt1S87p1CcSWXw7GzD8dQM4FGxZsOozN8wdBq1LKXRoREZFbuP1pxs8//zyqqqrwzDPPoLS0FIMHD8YXX3wBnU7naLNmzRqoVCrMnDkTVVVVGDNmDDZu3Ailkr9g71SPrp2x6ReD8Ni7B/FN/g9Y+rfjeHvW/XwCMhER+QRJeOEjc202G/R6PaxWq19f7gGAfXmXMW/DIdTWCzw9sgeWTegjd0lERETNcuX3N5/F4+Ue6hWO12Y0zOa7bu/3eP/gOZkrIiIiajsGFB/w0/go/MfYhocL/tenOdj93UWZKyIiImobBhQf8esxPTFzYBTqBbD4w6OcbZaIiLwaA4qPkCQJr06Lw7AeYaisrsP8TRkosV2TuywiIqJWYUDxIRqVAuvmxOPurkEotl7D/E2HcbW6Vu6yiIiIXMaA4mP0ndTYMO9BhHRSI/uCFUv/dgxeeKMWERH5OQYUHxQTFoR3fzYQaqWE7dkW/M/u03KXRERE5BIGFB/14F2h+N3Uhtl9V6edwue5zT8pmoiIyBMxoPiwRwd1x9yhMQCA57Zm4aSlXOaKiIiI7gwDio/7z8T7MPTuhjt7frk5A6WV1XKXREREdFsMKD5OrVTgnTkPIDo0EAU/VOGZD46gpq5e7rKIiIhaxIDiB0KCNPjfnz2III0SB85cwav/PCF3SURERC1iQPET9xh1WDPrfgDApgPnkHLovLwFERERtYABxY8k9DViybiGZ/a89GkOMs7+IHNFREREzWNA8TOLRvfEpLhI1NQJPLUlExfKquQuiYiI6CYMKH5GkiS88Ug/3BcZjCuV1fjV5sOoqq6TuywiIiInDCh+qJNGhXd/Fo+wIA1yi2z4zd85HT4REXkWBhQ/FRXSCesej4daKeGfx4vx56/OyF0SERGRAwOKHxsUG4qXJ/cFALzx+UkcyuegWSIi8gwMKH5uzuDueHhAN9TVCyz+6AguV9jlLomIiIgBxd9JkoRXp5nRM6IzLtrsSErJQl09x6MQEZG8GFAIQVoV1s15AIFqJfadvoz/2Z0nd0lEROTnGFAIANDLoMPK6WYAwNtf5uHrvEsyV0RERP6MAYUcHh4QhccGdYcQQFJKFizWa3KXREREfooBhZy8PPk+xyRuz6Yc5XgUIiKSBQMKOQlQK/HOnAcQpFHim/wf8Kf07+UuiYiI/BADCt3krvAgvDK1YTzKmrRTyCook7cgIiLyOwwo1KwZD3RDYr9I1NYLPJtyFBX2WrlLIiIiP8KAQs2SJAm/fzgO3boE4tyVq1jxWa7cJRERkR9hQKFb0geqsWbW/VBIwN8zC/GPY0Vyl0RERH6CAYVaNCg2FAtH9QQAvJCajcLSqzJXRERE/oABhW7r12N64f7oLii/Vovnth7jrcdERNTuXAoo69atQ79+/RAcHIzg4GAMHToUO3bscGyfN28eJElyWoYMGeK0D7vdjsWLFyM8PBxBQUGYMmUKCgsL3dMbahdqpQJvP3o/gjRKHDr7A97Zc1rukoiIyMe5FFCioqLw2muv4fDhwzh8+DBGjx6NqVOnIjf3xwGUEyZMQHFxsWPZvn270z6SkpKQmpqKlJQU7Nu3DxUVFUhMTERdXZ17ekTtIiYsCL+b1nDr8Vtf5uHI+VKZKyIiIl8mCSHadL4+NDQUb7zxBubPn4958+ahrKwMn3zySbNtrVYrunbtii1btmDWrFkAgKKiIkRHR2P79u0YP378HX2mzWaDXq+H1WpFcHBwW8onFwgh8GxKFj47VoS7wjphx7PDEahRyl0WERF5CVd+f7d6DEpdXR1SUlJQWVmJoUOHOtbv3bsXERER6N27NxYsWICSkhLHtszMTNTU1CAhIcGxzmQywWw2Y//+/a0thTqIJEn43TQzjMEBOHvlKlannZS7JCIi8lEuB5Ts7Gx07twZWq0WTz31FFJTU3HfffcBACZOnIgPPvgAu3fvxptvvomMjAyMHj0adrsdAGCxWKDRaBASEuK0T4PBAIvFcsvPtNvtsNlsTgvJQx+odjz1eP2+fF7qISKiduFyQLnnnnuQlZWFgwcP4umnn8bcuXNx4sQJAMCsWbMwadIkmM1mTJ48GTt27MCpU6ewbdu2FvcphIAkSbfcnpycDL1e71iio6NdLZvcaHQfA6YP6IZ6ATz/9+O4VsPxQ0RE5F4uBxSNRoOePXti4MCBSE5ORv/+/fH222832zYyMhIxMTHIy8sDABiNRlRXV6O01Pl/3SUlJTAYDLf8zOXLl8NqtTqWgoICV8smN/uvyfchvLMWp0sq8Icv8+Quh4iIfEyb50ERQjgu4TR15coVFBQUIDIyEgAQHx8PtVqNtLQ0R5vi4mLk5ORg2LBht/wMrVbruLW5cSF5demkwavX7+r581dnkF1olbkiIiLyJS4FlBdeeAFff/01zp49i+zsbLz44ovYu3cv5syZg4qKCixduhQHDhzA2bNnsXfvXkyePBnh4eF4+OGHAQB6vR7z58/HkiVL8OWXX+Lo0aN4/PHHERcXh7Fjx7ZLB6n9TDAbkdgvEnX1Ar/5+zFU19bLXRIREfkIlSuNL168iCeeeALFxcXQ6/Xo168fdu7ciXHjxqGqqgrZ2dnYvHkzysrKEBkZiVGjRmHr1q3Q6XSOfaxZswYqlQozZ85EVVUVxowZg40bN0Kp5O2q3uiVKX2x//sr+M5Sjve+PuOYFp+IiKgt2jwPihw4D4pnST1aiP/YegwBagXS/mMEokM7yV0SERF5oA6ZB4Wo0bT7u2Ho3WG4VlOPlz/LhRdmXiIi8jAMKNRmjRO4qZUSdn9Xgs9zL8pdEhEReTkGFHKLnhGd8eTwHgCAV/6Ri0p7rcwVERGRN2NAIbdZNLonuod2QrH1Gt7adUrucoiIyIsxoJDbBKiVeGVqXwDAX/51Ft8W85EERETUOgwo5Faj7onARLMRdfUCL32SwwGzRETUKgwo5Hb/Nfk+BKqVOHyuFNuzb/0QSCIiolthQCG3i9QH4skRdwMAknd8y4cJEhGRyxhQqF38avjdMAYHoLC0Chv+dVbucoiIyMswoFC76KRR4Tfj7wEA/HHPaVyuaP6BkkRERM1hQKF28/CAbugXpUeFvRar03jbMRER3TkGFGo3CoWE/5x0HwAg5dB5fGfhbcdERHRnGFCoXQ2KDcW/xxlRL4Dfb/uWtx0TEdEdYUChdvfbCfdCo1Tg67zL2HvyktzlEBGRF2BAoXbXPawTfv6TuwAAK7d/i7p6nkUhIqKWMaBQh3hmVE8EB6iQV1KBfx4vkrscIiLycAwo1CH0gWr8anjD5G1v7cpDbV29zBUREZEnY0ChDjPvJ7EIDdIg/3IlUo9ekLscIiLyYAwo1GE6a1V46voU+G9/mYfqWp5FISKi5jGgUId6Yshd6KrTorC0Cn/LLJC7HCIi8lAMKNShAjVKPDOyBwBg7e7TfJAgERE1iwGFOtxjg7ojUh+AYus1pBw6L3c5RETkgRhQqMMFqJVYNLonAGDtnu9RVc2zKERE5IwBhWTxSHw0okICcbnCjpQMnkUhIiJnDCgkC41KgadGNIxF+d+v8zkvChEROWFAIdn8ND4KYUEaXCirwrbsYrnLISIiD8KAQrIJUCsxb9hdAIA/p5/hk46JiMiBAYVk9cTQGASqlThRbMO+05flLoeIiDwEAwrJqksnDR4dFA2g4SwKERERwIBCHmD+Q7FQKiTsO30ZORescpdDREQegAGFZBcV0gmT+0UCAP6U/r3M1RARkSdgQCGPsGB4w0MEd+RYYLFek7kaIiKSGwMKeYS+Jj0GxYairl7gg2/OyV0OERHJzKWAsm7dOvTr1w/BwcEIDg7G0KFDsWPHDsd2IQRWrFgBk8mEwMBAjBw5Erm5uU77sNvtWLx4McLDwxEUFIQpU6agsLDQPb0hrzZ36F0AgI8OnYe9ltPfExH5M5cCSlRUFF577TUcPnwYhw8fxujRozF16lRHCFm1ahVWr16NtWvXIiMjA0ajEePGjUN5ebljH0lJSUhNTUVKSgr27duHiooKJCYmoq6Ov5D8XUJfA4zBAbhcUY1txzlxGxGRP5NEG2fHCg0NxRtvvIFf/OIXMJlMSEpKwrJlywA0nC0xGAx4/fXX8eSTT8JqtaJr167YsmULZs2aBQAoKipCdHQ0tm/fjvHjx9/RZ9psNuj1elitVgQHB7elfPIwa3fn4f99cQr9o7vg04U/kbscIiJyI1d+f7d6DEpdXR1SUlJQWVmJoUOHIj8/HxaLBQkJCY42Wq0WI0aMwP79+wEAmZmZqKmpcWpjMplgNpsdbZpjt9ths9mcFvJNjw7qDo1SgWMFZcgqKJO7HCIikonLASU7OxudO3eGVqvFU089hdTUVNx3332wWCwAAIPB4NTeYDA4tlksFmg0GoSEhNyyTXOSk5Oh1+sdS3R0tKtlk5cI76xF4vVbjjftPytvMUREJBuXA8o999yDrKwsHDx4EE8//TTmzp2LEydOOLZLkuTUXghx07qmbtdm+fLlsFqtjqWgoMDVssmLzL3+fJ5tx4tRWlktbzFERCQLlwOKRqNBz549MXDgQCQnJ6N///54++23YTQaAeCmMyElJSWOsypGoxHV1dUoLS29ZZvmaLVax51DjQv5rv7RXdDXFIzqunqkHr0gdzlERCSDNs+DIoSA3W5HbGwsjEYj0tLSHNuqq6uRnp6OYcOGAQDi4+OhVqud2hQXFyMnJ8fRhggAHn2w4TLe1owCPuWYiMgPqVxp/MILL2DixImIjo5GeXk5UlJSsHfvXuzcuROSJCEpKQkrV65Er1690KtXL6xcuRKdOnXC7NmzAQB6vR7z58/HkiVLEBYWhtDQUCxduhRxcXEYO3Zsu3SQvNOU+7vh1W3f4uTFchwrtOL+6C5yl0RERB3IpYBy8eJFPPHEEyguLoZer0e/fv2wc+dOjBs3DgDw/PPPo6qqCs888wxKS0sxePBgfPHFF9DpdI59rFmzBiqVCjNnzkRVVRXGjBmDjRs3QqlUurdn5NX0gWpMiovEx0cvYGvGeQYUIiI/0+Z5UOTAeVD8wzdnrmDWuwcRpFHi0ItjEaR1KU8TEZGH6ZB5UIja26DYUMSGB6Gyug7bszmzLBGRP2FAIY8lSRKmD+gGAPg0q0jmaoiIqCMxoJBHm3p/Q0DZ//1lXLRdk7kaIiLqKAwo5NG6h3XCwJgQ1AvgH8d4FoWIyF8woJDHm3r9Mg8nbSMi8h8MKOTxEuMioVJIyC2yIe9iudzlEBFRB2BAIY8XEqTByHu6AgA+42UeIiK/wIBCXiGxnwkAsC27mFPfExH5AQYU8gpj7o2ARqXAmUuVOHWxQu5yiIionTGgkFfQBagxvFfDZZ5tnLSNiMjnMaCQ15jUzwgAnFWWiMgPMKCQ1xhzrwEapQKnSyp4Nw8RkY9jQCGvERygxr/1CgfAyzxERL6OAYW8ynhzw2WeXd9elLkSIiJqTwwo5FVG94mAJAE5F2ywWPlsHiIiX8WAQl4lvLMW90d3AQDs/q5E3mKIiKjdMKCQ1xl7rwEA8CUv8xAR+SwGFPI6o/tEAAD2nb6Mquo6mashIqL2wIBCXqePUYduXQJhr63H/u8vy10OERG1AwYU8jqSJGHMvQ1nUXZ9y3EoRES+iAGFvFLjZZ7d313kwwOJiHwQAwp5pSF3h6GTRomLNjtyi2xyl0NERG7GgEJeKUCtxNC7wwAAX+dxHAoRka9hQCGv9dD1ae//dZoBhYjI1zCgkNdqfC7PobM/4FoNbzcmIvIlDCjktXp07QxjcACqa+uRcfYHucshIiI3YkAhryVJkuMyzz6OQyEi8ikMKOTVGi/zcKAsEZFvYUAhr/aTng0B5USxDZcr7DJXQ0RE7sKAQl4tvLMW90YGAwAOfH9F5mqIiMhdGFDI6zXOh/JNPgMKEZGvYEAhrzf47lAAwKF83slDROQrXAooycnJePDBB6HT6RAREYFp06bh5MmTTm3mzZsHSZKcliFDhji1sdvtWLx4McLDwxEUFIQpU6agsLCw7b0hv/TgXQ0B5dTFCvxQWS1zNURE5A4uBZT09HQsXLgQBw8eRFpaGmpra5GQkIDKykqndhMmTEBxcbFj2b59u9P2pKQkpKamIiUlBfv27UNFRQUSExNRV8fJtsh1oUEa9DZ0BsCzKEREvkLlSuOdO3c6vd6wYQMiIiKQmZmJ4cOHO9ZrtVoYjcZm92G1WrF+/Xps2bIFY8eOBQC8//77iI6Oxq5duzB+/HhX+0CEwbFhOHWxAt/kX8EEc/M/e0RE5D3aNAbFarUCAEJDQ53W7927FxEREejduzcWLFiAkpISx7bMzEzU1NQgISHBsc5kMsFsNmP//v3Nfo7dbofNZnNaiG40KJbjUIiIfEmrA4oQAs899xweeughmM1mx/qJEyfigw8+wO7du/Hmm28iIyMDo0ePht3eMEeFxWKBRqNBSEiI0/4MBgMsFkuzn5WcnAy9Xu9YoqOjW1s2+ajB1wPKiWIbbNdqZK6GiIjaqtUBZdGiRTh+/Dg++ugjp/WzZs3CpEmTYDabMXnyZOzYsQOnTp3Ctm3bWtyfEAKSJDW7bfny5bBarY6loKCgtWWTj4oIDkBseBCEAA7zuTxERF6vVQFl8eLF+Oyzz7Bnzx5ERUW12DYyMhIxMTHIy8sDABiNRlRXV6O0tNSpXUlJCQwGQ7P70Gq1CA4OdlqImhp0/W6eb3iZh4jI67kUUIQQWLRoET7++GPs3r0bsbGxt33PlStXUFBQgMjISABAfHw81Go10tLSHG2Ki4uRk5ODYcOGuVg+0Y8a50P55gwDChGRt3PpLp6FCxfiww8/xKeffgqdTucYM6LX6xEYGIiKigqsWLECM2bMQGRkJM6ePYsXXngB4eHhePjhhx1t58+fjyVLliAsLAyhoaFYunQp4uLiHHf1ELVG43woOResqKquQ6BGKXNFRETUWi4FlHXr1gEARo4c6bR+w4YNmDdvHpRKJbKzs7F582aUlZUhMjISo0aNwtatW6HT6Rzt16xZA5VKhZkzZ6KqqgpjxozBxo0boVTyFwq1XlRIICJ0WpSU25FTZHUEFiIi8j6SEELIXYSrbDYb9Ho9rFYrx6OQkye3HMbnuRexfGIfPDmih9zlEBHRDVz5/c1n8ZBPeaB7w+3rR8+XyVsIERG1CQMK+ZQB1wPKkfOl8MKTg0REdB0DCvmUuG56qBQSSsrtKLJek7scIiJqJQYU8imBGiXujWy4rnnkXOltWhMRkadiQCGf80D3LgA4DoWIyJsxoJDPuXEcChEReScGFPI5jXfy5BZZca2mTuZqiIioNRhQyOdEhwYiLEiDmjqB3CKr3OUQEVErMKCQz5EkCQOuj0M5VsCAQkTkjRhQyCfFdesCADheWCZrHURE1DoMKOST+kXrAQDHL/AMChGRN2JAIZ/Ur1tDQDlzqRK2azUyV0NERK5iQCGfFNZZi25dAgEAOTyLQkTkdRhQyGf1b7zMU8iAQkTkbRhQyGdxoCwRkfdiQCGf1T+KZ1CIiLwVAwr5LPP1gFJYWoUrFXaZqyEiIlcwoJDPCg5Q4+7wIABANgfKEhF5FQYU8mn9eJmHiMgrMaCQT+sX1QUAAwoRkbdhQCGf9uMZlDJ5CyEiIpcwoJBP62vSQyEBJeV2WKzX5C6HiIjuEAMK+bRAjRK9DToAPItCRORNGFDI53GgLBGR92FAIZ/nGCjLW42JiLwGAwr5vBsHygohZK6GiIjuBAMK+bw+xmBolAqUXa1BwQ9VcpdDRER3gAGFfJ5GpUCfyIaBspxRlojIOzCgkF8wd2u4zMOAQkTkHRhQyC/0cwSUMnkLISKiO8KAQn4h7vpA2exCKwfKEhF5AQYU8gu9DTpoVArYrtXi/A9X5S6HiIhuw6WAkpycjAcffBA6nQ4RERGYNm0aTp486dRGCIEVK1bAZDIhMDAQI0eORG5urlMbu92OxYsXIzw8HEFBQZgyZQoKCwvb3huiW1ArFbg3MhgAJ2wjIvIGLgWU9PR0LFy4EAcPHkRaWhpqa2uRkJCAyspKR5tVq1Zh9erVWLt2LTIyMmA0GjFu3DiUl5c72iQlJSE1NRUpKSnYt28fKioqkJiYiLq6Ovf1jKiJuG4NASWHA2WJiDyeJNpwQf7SpUuIiIhAeno6hg8fDiEETCYTkpKSsGzZMgANZ0sMBgNef/11PPnkk7BarejatSu2bNmCWbNmAQCKiooQHR2N7du3Y/z48bf9XJvNBr1eD6vViuDg4NaWT37mrxkFeP7/jmPo3WH46FdD5C6HiMjvuPL7u01jUKzWhv+JhoaGAgDy8/NhsViQkJDgaKPVajFixAjs378fAJCZmYmamhqnNiaTCWaz2dGmKbvdDpvN5rQQuapxoGzOBSvq6zlQlojIk7U6oAgh8Nxzz+Ghhx6C2WwGAFgsFgCAwWBwamswGBzbLBYLNBoNQkJCbtmmqeTkZOj1escSHR3d2rLJj/WK6AytSoFyey3OcaAsEZFHa3VAWbRoEY4fP46PPvropm2SJDm9FkLctK6pltosX74cVqvVsRQUFLS2bPJjKqUC95kaB8qWyVsMERG1qFUBZfHixfjss8+wZ88eREVFOdYbjUYAuOlMSElJieOsitFoRHV1NUpLS2/ZpimtVovg4GCnhag14rr9eJmHiIg8l0sBRQiBRYsW4eOPP8bu3bsRGxvrtD02NhZGoxFpaWmOddXV1UhPT8ewYcMAAPHx8VCr1U5tiouLkZOT42hD1F4aAwpvNSYi8mwqVxovXLgQH374IT799FPodDrHmRK9Xo/AwEBIkoSkpCSsXLkSvXr1Qq9evbBy5Up06tQJs2fPdrSdP38+lixZgrCwMISGhmLp0qWIi4vD2LFj3d9Dohv0i+oCAMgtsqG+XkChaPnSIxERycOlgLJu3ToAwMiRI53Wb9iwAfPmzQMAPP/886iqqsIzzzyD0tJSDB48GF988QV0Op2j/Zo1a6BSqTBz5kxUVVVhzJgx2LhxI5RKZdt6Q3QbPboGIUCtQIW9FvlXKtGja2e5SyIioma0aR4UuXAeFGqLGev2I/NcKd6adT+mDegmdzlERH6jw+ZBIfJGcY4nG3McChGRp2JAIb/jCCgcKEtE5LEYUMjvOGaULbKijjPKEhF5JAYU8js9unZGoFqJq9V1yL9cIXc5RETUDAYU8jtKhQRzt8YZZXmZh4jIEzGgkF8yc6AsEZFHY0Ahv9QvigNliYg8GQMK+aXGO3lyi2wcKEtE5IEYUMgvxYZ3RpBGiaqaOnx/iQNliYg8DQMK+SWlQkJfPjiQiMhjMaCQ3/pxwrYyeQshIqKbMKCQ33IMlOWdPEREHocBhfxW463GJ4ptqK2rl7kaIiK6EQMK+a3YsCB01qpwraYepzlQlojIozCgkN9ScEZZIiKPxYBCfo1PNiYi8kwMKOTX4qK6AOBAWSIiT8OAQn4t7oaBsjUcKEtE5DEYUMivxYR2gi5AheraeuRd5EBZIiJPwYBCfk2hkBxnUY5zwjYiIo/BgEJ+L+76hG3HOFCWiMhjMKCQ3xsQ3QUAcPR8qbyFEBGRAwMK+b0B3UMAAKculqPSXitzNUREBDCgEMEQHIBIfQDqBW83JiLyFAwoRADuv36ZJ6ugTNY6iIioAQMKEYAB3bsA4DgUIiJPwYBCBOD+6IZxKDyDQkTkGRhQiNAwo6xSIeGizY5ia5Xc5RAR+T0GFCIAgRol7jHoAABHz5fJWwwRETGgEDVqHIfCyzxERPJjQCG6znEnD8+gEBHJjgGF6LrGMyjHL5TxycZERDJzOaB89dVXmDx5MkwmEyRJwieffOK0fd68eZAkyWkZMmSIUxu73Y7FixcjPDwcQUFBmDJlCgoLC9vUEaK2uju8M3QBKlyrqcdJS7nc5RAR+TWXA0plZSX69++PtWvX3rLNhAkTUFxc7Fi2b9/utD0pKQmpqalISUnBvn37UFFRgcTERNTV1bneAyI3USgkTthGROQhVK6+YeLEiZg4cWKLbbRaLYxGY7PbrFYr1q9fjy1btmDs2LEAgPfffx/R0dHYtWsXxo8f72pJRG5zf3QXfJ13GVkFZXh8SIzc5RAR+a12GYOyd+9eREREoHfv3liwYAFKSkoc2zIzM1FTU4OEhATHOpPJBLPZjP379ze7P7vdDpvN5rQQtQfOKEtE5BncHlAmTpyIDz74ALt378abb76JjIwMjB49Gna7HQBgsVig0WgQEhLi9D6DwQCLxdLsPpOTk6HX6x1LdHS0u8smAgD0j+oCAPj+UiWsVTXyFkNE5MfcHlBmzZqFSZMmwWw2Y/LkydixYwdOnTqFbdu2tfg+IQQkSWp22/Lly2G1Wh1LQUGBu8smAgCEddaie2gnAMDxwjJ5iyEi8mPtfptxZGQkYmJikJeXBwAwGo2orq5GaanzKfSSkhIYDIZm96HVahEcHOy0ELWXxoGynFGWiEg+7R5Qrly5goKCAkRGRgIA4uPjoVarkZaW5mhTXFyMnJwcDBs2rL3LIbotzihLRCQ/l+/iqaiowOnTpx2v8/PzkZWVhdDQUISGhmLFihWYMWMGIiMjcfbsWbzwwgsIDw/Hww8/DADQ6/WYP38+lixZgrCwMISGhmLp0qWIi4tz3NVDJKcbbzVu6dIjERG1H5cDyuHDhzFq1CjH6+eeew4AMHfuXKxbtw7Z2dnYvHkzysrKEBkZiVGjRmHr1q3Q6XSO96xZswYqlQozZ85EVVUVxowZg40bN0KpVLqhS0Rtc58pGFqVAj9UVuPM5Ur06NpZ7pKIiPyOJIQQchfhKpvNBr1eD6vVyvEo1C5m/vkADuX/gNemx+HRQd3lLoeIyCe48vubz+Ihasbg2FAAwKH8H2SuhIjIPzGgEDXjwbuuB5SzDChERHJgQCFqxgMxIVBIQGFpFYqtVXKXQ0TkdxhQiJrRWatCX5MeAC/zEBHJgQGF6BYaL/Nk8DIPEVGHY0AhuoVBsQ3Pi8rI54MDiYg6GgMK0S00nkE5ebEcpZXVMldDRORfGFCIbiGssxa9IhomaTt45orM1RAR+RcGFKIW/KRnOABg//cMKEREHYkBhagFw3qEAQD+9f1lmSshIvIvDChELRh8dxgUEnDmUiUs1mtyl0NE5DcYUIhaoA9UI65bw3wo/zrNsyhERB2FAYXoNoZdH4fCyzxERB2HAYXoNn7S4/pA2dNX4IUP/yYi8koMKES3MfCuEGhUClhs13DmcqXc5RAR+QUGFKLbCFAr8eBdDbPKpp+8JHM1RET+gQGF6A6MuicCALDnZInMlRAR+QcGFKI7MKpPQ0D55swPqLTXylwNEZHvY0AhugN3hwehe2gnVNfV83ZjIqIOwIBCdAckScLoPrzMQ0TUURhQiO5Q42WePd9d4u3GRETtjAGF6A4Njg1FoFoJi+0acotscpdDROTTGFCI7lCAWomR93QFAOzIKZa5GiIi38aAQuSCCWYjAGBHtoWXeYiI2hEDCpELRveJgEalwJnLlTh1sULucoiIfBYDCpELdAFqDO/VcJlnezYv8xARtRcGFCIXTbx+meefx4t4mYeIqJ0woBC5aFxfA7QqBb6/VInsC1a5yyEi8kkMKEQuCg5QI6Fvw1mUj49ckLkaIiLfxIBC1AozHugGAPjsWBGqa+tlroaIyPcwoBC1wkM9w9FVp8UPldVIP3VJ7nKIiHwOAwpRK6iUCky73wQA+NvhApmrISLyPS4HlK+++gqTJ0+GyWSCJEn45JNPnLYLIbBixQqYTCYEBgZi5MiRyM3NdWpjt9uxePFihIeHIygoCFOmTEFhYWGbOkLU0R4ZGA0A2PXtRRRbq2SuhojIt7gcUCorK9G/f3+sXbu22e2rVq3C6tWrsXbtWmRkZMBoNGLcuHEoLy93tElKSkJqaipSUlKwb98+VFRUIDExEXV1da3vCVEH623QYXBsKOoF8NE35+Uuh4jIp0iiDRM5SJKE1NRUTJs2DUDD2ROTyYSkpCQsW7YMQMPZEoPBgNdffx1PPvkkrFYrunbtii1btmDWrFkAgKKiIkRHR2P79u0YP378bT/XZrNBr9fDarUiODi4teUTtdm248VY+OERdNVp8a9lo6FR8aopEdGtuPL7263/mubn58NisSAhIcGxTqvVYsSIEdi/fz8AIDMzEzU1NU5tTCYTzGazo01TdrsdNpvNaSHyBAl9DYjQaXGp3I7Pcy1yl0NE5DPcGlAsloZ/oA0Gg9N6g8Hg2GaxWKDRaBASEnLLNk0lJydDr9c7lujoaHeWTdRqaqUCjw3qDgDYfOCsvMUQEfmQdjkfLUmS02shxE3rmmqpzfLly2G1Wh1LQQHvmiDPMXtwd2iUCmScLUXmuR/kLoeIyCe4NaAYjQ2zazY9E1JSUuI4q2I0GlFdXY3S0tJbtmlKq9UiODjYaSHyFIbgAEy/PnHbO3u+l7kaIiLf4NaAEhsbC6PRiLS0NMe66upqpKenY9iwYQCA+Ph4qNVqpzbFxcXIyclxtCHyNk+O6AFJAr78rgTfWThGioiorVwOKBUVFcjKykJWVhaAhoGxWVlZOH/+PCRJQlJSElauXInU1FTk5ORg3rx56NSpE2bPng0A0Ov1mD9/PpYsWYIvv/wSR48exeOPP464uDiMHTvWrZ0j6iix4UH4d3MkAGDdXp5FISJqK5Wrbzh8+DBGjRrleP3cc88BAObOnYuNGzfi+eefR1VVFZ555hmUlpZi8ODB+OKLL6DT6RzvWbNmDVQqFWbOnImqqiqMGTMGGzduhFKpdEOXiOTx9Mge2JZdjH8cK8KScfege1gnuUsiIvJabZoHRS6cB4U81c/+cghfnbqEmQOjsOqn/eUuh4jIo8g2DwqRv3t2TC8AwP8duYAzlypkroaIyHsxoBC5UXxMCMbeG4G6eoE1u/LkLoeIyGsxoBC52XPj7gEA/ONYEY6cL71NayIiag4DCpGb3WcKxk/jowAA/5mag9q6epkrIiLyPgwoRO1g+cQ+0AeqcaLYhs0HzsldDhGR12FAIWoHYZ21WDahDwBgddopXLRdk7kiIiLvwoBC1E4efTAaA7p3QYW9Fv/9zxNyl0NE5FUYUIjaiUIh4dVpZigkYNvxYuz5rkTukoiIvAYDClE76mvS4xc/iQUA/Obvx3Cp3C5zRURE3oEBhaidLR1/D+4x6HC5ohq/+fsx1Nd73eTNREQdjgGFqJ0FqJX4n9kDoFUpsPfkJfxhNydwIyK6HQYUog7Q26DD76aaAQBv7crDJ0cvyFwREZFnY0Ah6iAzH4zGkyPuBgA8//fjOJT/g8wVERF5LgYUog60bHwfTOhrRHVdPX615TBOWsrlLomIyCMxoBB1IIVCwppZ96N/dBeUXa3BY+8dZEghImoGAwpRBwvUKLH554Ng7haMHyqr8ei7B3i5h4ioCQYUIhnoO6nxwfwh6B+lR+nVGsz534P42+ECucsiIvIYDChEMtF3UiPlV0Mx0WxETZ3Ab/5+HP/1aQ6u1dTJXRoRkewYUIhkFKhR4o+zH8Di0T0BAJsPnEPi/+zD4bO85ENE/o0BhUhmCoWEJQn3YMO8BxHeWYPTJRX46Z8OYPnHx/kUZCLyW5IQwuvm3bbZbNDr9bBarQgODpa7HCK3KbtajZXbv8VfDxcCALQqBX42NAZPj+yJ0CCNzNUREbWNK7+/GVCIPFDG2R+waud3yDhbCgAI0ijx+JAYzB7cHTFhQTJXR0TUOgwoRD5ACIG9py7hzS9OIueCzbF+eO+umNrfhLH3GaAPVMtYIRGRaxhQiHyIEAJffluCLQfP4au8S2j8G6tWSvhJz3CM7hOBYT3C0aNrECRJkrdYIqIWMKAQ+ajzV67i46OF2JFtwcmLzjPQGoMDMKxHGAbFhsLcTY9ehs7QqpQyVUpEdDMGFCI/cLqkAp/nWvCv05dx+FwpqmvrnbarlRJ6Rehg7haM3gYdYsODEBsehOjQTlAreQMfEXU8BhQiP3Otpg6Z50rxr9OXcaywDLlFNpRdrWm2rVIhISokEHeFBaFbSCC6dtaiq06L8Ot/Ruga/gxQ8+wLEbkXAwqRnxNC4EJZFXIu2HCiyIrTlyqQf/kqzl6uRNUdzlSr06ocwSWsswbBAWp0DlBBF6BCZ63qpte6ADV0118HqpUcD0NEN2FAIaJmCSFQUm7HmUuVOHulEhbrNVyqsONyuR2XKuy4VN6w2JtcLnKVUiGhk0YJrUoJrUoBrVrx49cqBQLUjet/XKdVKa+3U0CjUkClkKCQJKgUEpTKhtdKhQSlJEGlbPi6YZ0CSgWgVPzYRqWQoHBsl6BSKG5oLzl9rVIooFDAqY1CwXBF1B5c+f2t6qCaiMgDSJIEQ3AADMEBGNojrNk2QgiU22sdYeVyhR1XKqpRfq0G5fZalF+rRcW1WpRfq0HF9dfl12qvf12DegHU1YuG9ajt4B66hyQBSsk57CgkCQqp4XvYkF8aXwMKSYKEhm2O107rf3yfBMn59Q3t4Gjz4z6k69sUTfbdcv0/NmjatOl7Jadt0i23NffeG1u0tN+m26UmW296bwttW3rpav2uvBeu1N/itlu/t6Xvr1s/p+mn3OIHqqtOi4Wjeja7rSMwoBCRE0mSEBygRnCAGj26dnbpvUIIVNXUofxaLSrttbDX1jcsNXU/fl1bB3vNDV/X1uNa4/aahnXVtfWoEwJ19QK19QL11/+sc/xZjzqn1wK1dQL14sZ29airE077qau7/uf1dXX1zZ9AFgKovb4vuzu+qURe6O6uQQwoROQbJElCJ40KnTTe8U+LuDEEiSYhpjHI1AnU1NdDiIb2AkC9EBDixz+FAAQE6q+3acg9ja9vbNfM+6/X0fC6hfejcZto0oemfWryukl/b7WtuRUCt/6spu+9+XPv/L1N33zbfd+w4vZ13Pq9tymjxT403ber7735s2/o0237cOvPut17Xfleh8j8eA23/yuyYsUKvPLKK07rDAYDLBYLgIaD8Morr+Ddd99FaWkpBg8ejD/+8Y/o27evu0shImqRdH08C6eLIfI87TIZQt++fVFcXOxYsrOzHdtWrVqF1atXY+3atcjIyIDRaMS4ceNQXl7ewh6JiIjIn7RLQFGpVDAajY6la9euABrOnrz11lt48cUXMX36dJjNZmzatAlXr17Fhx9+2B6lEBERkRdql4CSl5cHk8mE2NhYPProozhz5gwAID8/HxaLBQkJCY62Wq0WI0aMwP79+2+5P7vdDpvN5rQQERGR73J7QBk8eDA2b96Mzz//HO+99x4sFguGDRuGK1euOMahGAwGp/fcOEalOcnJydDr9Y4lOjra3WUTERGRB3F7QJk4cSJmzJiBuLg4jB07Ftu2bQMAbNq0ydGm6T3XQogWZ51cvnw5rFarYykoKHB32URERORB2v2JYUFBQYiLi0NeXh6MRiMA3HS2pKSk5KazKjfSarUIDg52WoiIiMh3tXtAsdvt+PbbbxEZGYnY2FgYjUakpaU5tldXVyM9PR3Dhg1r71KIiIjIS7h9HpSlS5di8uTJ6N69O0pKSvDqq6/CZrNh7ty5kCQJSUlJWLlyJXr16oVevXph5cqV6NSpE2bPnu3uUoiIiMhLuT2gFBYW4rHHHsPly5fRtWtXDBkyBAcPHkRMTAwA4Pnnn0dVVRWeeeYZx0RtX3zxBXQ6nbtLISIiIi/FpxkTERFRh3Dl93e7j0EhIiIichUDChEREXkcBhQiIiLyON7xTPQmGofNcMp7IiIi79H4e/tOhr96ZUBpfPIxp7wnIiLyPuXl5dDr9S228cq7eOrr61FUVASdTtfiFPmtYbPZEB0djYKCAp+/Q8if+gr4V3/ZV9/lT/1lX32PEALl5eUwmUxQKFoeZeKVZ1AUCgWioqLa9TP8aUp9f+or4F/9ZV99lz/1l331Lbc7c9KIg2SJiIjI4zCgEBERkcdhQGlCq9Xi5ZdfhlarlbuUdudPfQX8q7/sq+/yp/6yr/7NKwfJEhERkW/jGRQiIiLyOAwoRERE5HEYUIiIiMjjMKAQERGRx2FAucE777yD2NhYBAQEID4+Hl9//bXcJbksOTkZDz74IHQ6HSIiIjBt2jScPHnSqc28efMgSZLTMmTIEKc2drsdixcvRnh4OIKCgjBlyhQUFhZ2ZFdua8WKFTf1w2g0OrYLIbBixQqYTCYEBgZi5MiRyM3NddqHN/Sz0V133XVTfyVJwsKFCwF493H96quvMHnyZJhMJkiShE8++cRpu7uOZWlpKZ544gno9Xro9Xo88cQTKCsra+feOWuprzU1NVi2bBni4uIQFBQEk8mEn/3sZygqKnLax8iRI2861o8++qhTG0/oK3D7Y+uun1tP6O/t+trc319JkvDGG2842njTsW1vDCjXbd26FUlJSXjxxRdx9OhR/Nu//RsmTpyI8+fPy12aS9LT07Fw4UIcPHgQaWlpqK2tRUJCAiorK53aTZgwAcXFxY5l+/btTtuTkpKQmpqKlJQU7Nu3DxUVFUhMTERdXV1Hdue2+vbt69SP7Oxsx7ZVq1Zh9erVWLt2LTIyMmA0GjFu3DjHs5wA7+knAGRkZDj1NS0tDQDwyCOPONp463GtrKxE//79sXbt2ma3u+tYzp49G1lZWdi5cyd27tyJrKwsPPHEE+3evxu11NerV6/iyJEjeOmll3DkyBF8/PHHOHXqFKZMmXJT2wULFjgd6z//+c9O2z2hr8Dtjy3gnp9bT+jv7fp6Yx+Li4vxl7/8BZIkYcaMGU7tvOXYtjtBQgghBg0aJJ566imndX369BG//e1vZarIPUpKSgQAkZ6e7lg3d+5cMXXq1Fu+p6ysTKjVapGSkuJYd+HCBaFQKMTOnTvbs1yXvPzyy6J///7NbquvrxdGo1G89tprjnXXrl0Ter1e/OlPfxJCeE8/b+XZZ58VPXr0EPX19UII3zmuAERqaqrjtbuO5YkTJwQAcfDgQUebAwcOCADiu+++a+deNa9pX5tz6NAhAUCcO3fOsW7EiBHi2WefveV7PLGvQjTfX3f83Hpif+/k2E6dOlWMHj3aaZ23Htv2wDMoAKqrq5GZmYmEhASn9QkJCdi/f79MVbmH1WoFAISGhjqt37t3LyIiItC7d28sWLAAJSUljm2ZmZmoqalx+n6YTCaYzWaP+37k5eXBZDIhNjYWjz76KM6cOQMAyM/Ph8ViceqDVqvFiBEjHH3wpn42VV1djffffx+/+MUvnB6Y6SvH9UbuOpYHDhyAXq/H4MGDHW2GDBkCvV7v0f23Wq2QJAldunRxWv/BBx8gPDwcffv2xdKlS53OJnlbX9v6c+tt/QWAixcvYtu2bZg/f/5N23zp2LaFVz4s0N0uX76Muro6GAwGp/UGgwEWi0WmqtpOCIHnnnsODz30EMxms2P9xIkT8cgjjyAmJgb5+fl46aWXMHr0aGRmZkKr1cJisUCj0SAkJMRpf572/Rg8eDA2b96M3r174+LFi3j11VcxbNgw5ObmOups7pieO3cOALymn8355JNPUFZWhnnz5jnW+cpxbcpdx9JisSAiIuKm/UdERHhs/69du4bf/va3mD17ttMD5ObMmYPY2FgYjUbk5ORg+fLlOHbsmOOynzf11R0/t97U30abNm2CTqfD9OnTndb70rFtKwaUG9z4P1Gg4Rd803XeZNGiRTh+/Dj27dvntH7WrFmOr81mMwYOHIiYmBhs27btpr8sN/K078fEiRMdX8fFxWHo0KHo0aMHNm3a5Bhk15pj6mn9bM769esxceJEmEwmxzpfOa634o5j2Vx7T+1/TU0NHn30UdTX1+Odd95x2rZgwQLH12azGb169cLAgQNx5MgRPPDAAwC8p6/u+rn1lv42+stf/oI5c+YgICDAab0vHdu24iUeAOHh4VAqlTelz5KSkpv+1+YtFi9ejM8++wx79uxBVFRUi20jIyMRExODvLw8AIDRaER1dTVKS0ud2nn69yMoKAhxcXHIy8tz3M3T0jH11n6eO3cOu3btwi9/+csW2/nKcXXXsTQajbh48eJN+7906ZLH9b+mpgYzZ85Efn4+0tLSnM6eNOeBBx6AWq12Otbe0temWvNz6239/frrr3Hy5Mnb/h0GfOvYuooBBYBGo0F8fLzjFFqjtLQ0DBs2TKaqWkcIgUWLFuHjjz/G7t27ERsbe9v3XLlyBQUFBYiMjAQAxMfHQ61WO30/iouLkZOT49HfD7vdjm+//RaRkZGOU6Q39qG6uhrp6emOPnhrPzds2ICIiAhMmjSpxXa+clzddSyHDh0Kq9WKQ4cOOdp88803sFqtHtX/xnCSl5eHXbt2ISws7Lbvyc3NRU1NjeNYe0tfm9Oan1tv6+/69esRHx+P/v3737atLx1bl8kxMtcTpaSkCLVaLdavXy9OnDghkpKSRFBQkDh79qzcpbnk6aefFnq9Xuzdu1cUFxc7lqtXrwohhCgvLxdLliwR+/fvF/n5+WLPnj1i6NCholu3bsJmszn289RTT4moqCixa9cuceTIETF69GjRv39/UVtbK1fXbrJkyRKxd+9ecebMGXHw4EGRmJgodDqd45i99tprQq/Xi48//lhkZ2eLxx57TERGRnpdP29UV1cnunfvLpYtW+a03tuPa3l5uTh69Kg4evSoACBWr14tjh496rhzxV3HcsKECaJfv37iwIED4sCBAyIuLk4kJiZ6TF9ramrElClTRFRUlMjKynL6O2y324UQQpw+fVq88sorIiMjQ+Tn54tt27aJPn36iAEDBnhcX2/XX3f+3HpCf2/3cyyEEFarVXTq1EmsW7fupvd727FtbwwoN/jjH/8oYmJihEajEQ888IDTrbneAkCzy4YNG4QQQly9elUkJCSIrl27CrVaLbp37y7mzp0rzp8/77SfqqoqsWjRIhEaGioCAwNFYmLiTW3kNmvWLBEZGSnUarUwmUxi+vTpIjc317G9vr5evPzyy8JoNAqtViuGDx8usrOznfbhDf280eeffy4AiJMnTzqt9/bjumfPnmZ/bufOnSuEcN+xvHLlipgzZ47Q6XRCp9OJOXPmiNLS0g7qZYOW+pqfn3/Lv8N79uwRQghx/vx5MXz4cBEaGio0Go3o0aOH+PWvfy2uXLnicX29XX/d+XPrCf293c+xEEL8+c9/FoGBgaKsrOym93vbsW1vkhBCtOspGiIiIiIXcQwKEREReRwGFCIiIvI4DChERETkcRhQiIiIyOMwoBAREZHHYUAhIiIij8OAQkRERB6HAYWIiIg8DgMKEREReRwGFCIiIvI4DChERETkcRhQiIiIyOP8f6wnZs0fWkQoAAAAAElFTkSuQmCC",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "window_size = 100\n",
+ "kernel = np.full(window_size, 1 / window_size)\n",
+ "plt.plot(np.convolve(loss_history, kernel, mode=\"valid\"))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "id": "859eb2c5-bdc0-4376-a250-05f1ff68e491",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([-0.92518728, 0.27270752, -0.20081106])"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "true_params[\"beta\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "id": "6ab67d23-5bb2-4f1e-b69c-2b955823f035",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[array(336.70736025),\n",
+ " array(-268.97581555),\n",
+ " array(395.78744606),\n",
+ " array([-25.6507373 , -16.77466337, -20.70216968]),\n",
+ " array([65.89290602, 59.15876422, 40.72710202]),\n",
+ " array(-2.30179032),\n",
+ " array(84.43618552)]"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "f_loss_dloss(np.array(500), **true_loc_dict)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "id": "ab2a24a2-d4d2-4791-b1e8-549f03ecc2af",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[array(56.60708962),\n",
+ " array(0.12550054),\n",
+ " array(0.0109625),\n",
+ " array([-2.48193195, -0.68754083, 0.4438946 ]),\n",
+ " array([0.08497766, 0.17830603, 0.1258238 ]),\n",
+ " array(0.92784876),\n",
+ " array(0.04297923)]"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "f_loss_dloss(np.array(500, dtype=int), *opt_param_values)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "id": "f9e889a5-b509-4208-9648-ded17cf1e5d7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def inverse_softplus(x):\n",
+ " return np.log(np.expm1(x))\n",
+ "\n",
+ "\n",
+ "mcmc_param_values = {}\n",
+ "mcmc_param_values[\"sigma_loc\"] = np.log(idata.posterior[\"sigma\"]).mean((\"chain\", \"draw\")).values\n",
+ "mcmc_param_values[\"sigma_scale\"] = inverse_softplus(\n",
+ " np.log(idata.posterior[\"sigma\"]).std((\"chain\", \"draw\"))\n",
+ ").values\n",
+ "\n",
+ "for param in (\"beta\", \"alpha\"):\n",
+ " mcmc_param_values[f\"{param}_loc\"] = idata.posterior[param].mean((\"chain\", \"draw\")).values\n",
+ " mcmc_param_values[f\"{param}_scale\"] = inverse_softplus(\n",
+ " idata.posterior[param].std((\"chain\", \"draw\"))\n",
+ " ).values"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "id": "9eb47148-f517-4a00-924d-4a8269b9587c",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'sigma_loc': array(-1.06275392),\n",
+ " 'sigma_scale': array(-2.61505505),\n",
+ " 'beta_loc': array([-0.91056208, 0.2716077 , -0.15967444]),\n",
+ " 'beta_scale': array([-3.28925422, -3.30663365, -3.19186797]),\n",
+ " 'alpha_loc': array(11.78263119),\n",
+ " 'alpha_scale': array(-3.30535793)}"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "optimized_params"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "id": "8986aa65-5837-4d30-abf7-5aa70776d438",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'sigma_loc': array(-1.06716543),\n",
+ " 'sigma_scale': array(-2.58522658),\n",
+ " 'beta_loc': array([-0.90909634, 0.273244 , -0.15972646]),\n",
+ " 'beta_scale': array([-3.30488966, -3.31799041, -3.20806874]),\n",
+ " 'alpha_loc': array(11.78161401),\n",
+ " 'alpha_scale': array(-3.32904032)}"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "mcmc_param_values"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "id": "653c599e-8db3-4aa7-939c-1ef08bf7cf74",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[array(56.61987761),\n",
+ " array(-0.04343654),\n",
+ " array(0.07714657),\n",
+ " array([-2.03604289, -2.60214627, 0.07962086]),\n",
+ " array([ 0.10942038, 0.11859711, -0.01547745]),\n",
+ " array(-0.80121942),\n",
+ " array(-0.00921298)]"
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "f_loss_dloss(np.array(500, dtype=int), **mcmc_param_values)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0cf321e6",
+ "metadata": {},
+ "source": [
+ "## Todo:\n",
+ "\n",
+ "- Does this \"two models\" frameworks fits into what we already have?\n",
+ "- rsample --> stochastic gradients? Or automatic reparameterization?\n",
+ "- figure out guide param initalization\n",
+ "- More flexible optimizers..."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "77786d86",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "pymc-dev",
+ "language": "python",
+ "name": "pymc-dev"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.8"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/VI_Overview.ipynb b/VI_Overview.ipynb
new file mode 100644
index 0000000000..63f1b514d3
--- /dev/null
+++ b/VI_Overview.ipynb
@@ -0,0 +1,623 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "c51c3c1a-553c-45e4-a92f-d75063187863",
+ "metadata": {},
+ "source": [
+ "# Variational Inference overview"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c0777ef6-dd90-4452-88af-69a53f0f7713",
+ "metadata": {},
+ "source": [
+ "## Existing Variational Inference implementation\n",
+ "\n",
+ "The best way to get a sense for the current implementation is to walk backwards from how it's used"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "3d5724fe-72db-4908-a464-46f7fac97309",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "import pymc as pm\n",
+ "import arviz as az"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "33437d00-60e6-4505-8b8e-8ebe64473c5b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data = np.random.normal(size=10_000)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "894f9e31-90a1-4f13-b2bc-b75bdf996f78",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with pm.Model() as model:\n",
+ " d = pm.Data(\"data\", data)\n",
+ " batched_data = pm.Minibatch(d, batch_size=100)\n",
+ " x = pm.Normal(\"x\", 0., 1.)\n",
+ " y = pm.Normal(\"y\", x, total_size=len(data), observed=batched_data)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "15bc2997-8974-4d61-88ce-9423b215f84f",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "a695972a8ca3415f9f8dd118ae6288dc",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Output()"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n"
+ ],
+ "text/plain": []
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Finished [100%]: Average Loss = 144.77\n"
+ ]
+ }
+ ],
+ "source": [
+ "with model:\n",
+ " idata = pm.fit(n=10_000, method=\"advi\") "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d311e2f2-f264-4cb2-9287-21d6f5aad3e3",
+ "metadata": {},
+ "source": [
+ "But what does fit do? It roughly dispatches on the method. So the above is roughly equalivalent to:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "ec3b637d-c6a2-46cb-99bc-87fc952bda3e",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "43686ad598a649b09a88b84480985eac",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Output()"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n"
+ ],
+ "text/plain": []
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Finished [100%]: Average Loss = 143.83\n"
+ ]
+ }
+ ],
+ "source": [
+ "with model:\n",
+ " advi = pm.ADVI()\n",
+ " idata = advi.fit(n=100_000)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bfbd2a63-b5da-41d3-a1d1-254934ad4923",
+ "metadata": {},
+ "source": [
+ "But what is this `ADVI` object? Well, if you look at it's implementation with the documentation removed, you see it's a type of `KLqp`\n",
+ "\n",
+ "````python\n",
+ "class ADVI(KLqp):\n",
+ " def __init__(self, *args, **kwargs):\n",
+ " super().__init__(MeanField(*args, **kwargs))\n",
+ "````\n",
+ "\n",
+ "So what's a `Klqp`? Look at it's implementation with the documentation removed, you see it's an Inference object\n",
+ "\n",
+ "````python\n",
+ "class KLqp(Inference):\n",
+ " def __init__(self, approx, beta=1.0):\n",
+ " super().__init__(KL, approx, None, beta=beta)\n",
+ "````\n",
+ "\n",
+ "So what's an `Inference` object? Look at it's implementation with the documentation removed we finally get a sense for what are the main abstraction we will be working with."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "ce19d0bd-8a6b-4877-a4ee-5ee1fa481da8",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "\u001b[0;31mInit signature:\u001b[0m \u001b[0mpm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInference\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapprox\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mDocstring:\u001b[0m \n",
+ "**Base class for Variational Inference**.\n",
+ "\n",
+ "Communicates Operator, Approximation and Test Function to build Objective Function\n",
+ "\n",
+ "Parameters\n",
+ "----------\n",
+ "op : Operator class #:class:`~pymc.variational.operators`\n",
+ "approx : Approximation class or instance #:class:`~pymc.variational.approximations`\n",
+ "tf : TestFunction instance #?\n",
+ "model : Model\n",
+ " PyMC Model\n",
+ "kwargs : kwargs passed to :class:`Operator` #:class:`~pymc.variational.operators`, optional\n",
+ "\u001b[0;31mFile:\u001b[0m ~/upstream/pymc/pymc/variational/inference.py\n",
+ "\u001b[0;31mType:\u001b[0m type\n",
+ "\u001b[0;31mSubclasses:\u001b[0m KLqp, ImplicitGradient"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "pm.Inference?"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "73873c2f-11ed-43a3-b256-2e65816343b9",
+ "metadata": {},
+ "source": [
+ "Now things are falling into place. The `Inference` class is the way we perform variational inference. This is where the actual fit machinery lives. It also highlights what we need to do variational inference. We need a `Model`, an `Operator`, and an `Approximation`. We already know for `ADVI`, that the `Operator` is `KL` and the `Approximation` is `MeanField`.\n",
+ "\n",
+ "But what do these things mean? And how are they combined to perform inference?\n",
+ "\n",
+ "Well the `__init__` method of `Inference` makes it where we can find our answer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "0a77e663-8982-4a8a-bafc-290da5f45838",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "\u001b[0;31mSignature:\u001b[0m \u001b[0mpm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInference\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapprox\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mDocstring:\u001b[0m Initialize self. See help(type(self)) for accurate signature.\n",
+ "\u001b[0;31mSource:\u001b[0m \n",
+ " \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapprox\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobjective\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mapprox\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mFile:\u001b[0m ~/upstream/pymc/pymc/variational/inference.py\n",
+ "\u001b[0;31mType:\u001b[0m function"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "pm.Inference.__init__??"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0564cc40-2d42-476d-8fa4-91a063bff433",
+ "metadata": {},
+ "source": [
+ "Alright, so let's go ahead and explore the operator `KL`\n",
+ "\n",
+ "````python\n",
+ "class KL(Operator):\n",
+ " def __init__(self, approx, beta=1.0):\n",
+ " super().__init__(approx)\n",
+ " self.beta = pm.floatX(beta)\n",
+ "\n",
+ " def apply(self, f):\n",
+ " return -self.datalogp_norm + self.beta * (self.logq_norm - self.varlogp_norm)\n",
+ "````\n",
+ "\n",
+ "We see no `__call__` but we see a call to the `__init__` of `Operator`. For the `apply` method we see what looks like the ELBO. Let's for now inline for `ADVI` case and see what we get"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "1752272d-b32d-4bea-9c3c-1e331b7fda9a",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "objective = pm.operators.KL(pm.MeanField(model=model))(None)\n",
+ "objective"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cb70a473-6ca3-471c-b051-9a3cee666bd6",
+ "metadata": {},
+ "source": [
+ "So how'd that happen? Well if you look in the `Objective` class you see\n",
+ "\n",
+ "````python\n",
+ " objective_class = ObjectiveFunction\n",
+ "\n",
+ " def __call__(self, f=None):\n",
+ " if self.has_test_function:\n",
+ " if f is None:\n",
+ " raise ParametrizationError(f\"Operator {self} requires TestFunction\")\n",
+ " else:\n",
+ " if not isinstance(f, TestFunction):\n",
+ " f = TestFunction.from_function(f)\n",
+ " else:\n",
+ " if f is not None:\n",
+ " warnings.warn(f\"TestFunction for {self} is redundant and removed\", stacklevel=3)\n",
+ " else:\n",
+ " pass\n",
+ " f = TestFunction()\n",
+ " f.setup(self.approx)\n",
+ " return self.objective_class(self, f)\n",
+ "````\n",
+ "\n",
+ "Which finally brings us to `ObjectiveFunction`\n",
+ "\n",
+ "This is the function that sets up the actual loss functions and does the updates on it."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "339f72bf-a40d-4f84-9161-99a20ad090cb",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "\u001b[0;31mSignature:\u001b[0m\n",
+ "\u001b[0mpm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopvi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mObjectiveFunction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mobj_n_mc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mtf_n_mc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mobj_optimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m<\u001b[0m\u001b[0mfunction\u001b[0m \u001b[0madagrad_window\u001b[0m \u001b[0mat\u001b[0m \u001b[0;36m0x70ee648da480\u001b[0m\u001b[0;34m>\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mtest_optimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m<\u001b[0m\u001b[0mfunction\u001b[0m \u001b[0madagrad_window\u001b[0m \u001b[0mat\u001b[0m \u001b[0;36m0x70ee648da480\u001b[0m\u001b[0;34m>\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mmore_obj_params\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mmore_tf_params\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mmore_updates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mmore_replacements\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mtotal_grad_norm_constraint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mscore\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mcompile_kwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m \u001b[0mfn_kwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
+ "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mDocstring:\u001b[0m\n",
+ "Step function that should be called on each optimization step.\n",
+ "\n",
+ "Generally it solves the following problem:\n",
+ "\n",
+ ".. math::\n",
+ "\n",
+ " \\mathbf{\\lambda^{\\*}} = \\inf_{\\lambda} \\sup_{\\theta} t(\\mathbb{E}_{\\lambda}[(O^{p,q}f_{\\theta})(z)])\n",
+ "\n",
+ "Parameters\n",
+ "----------\n",
+ "obj_n_mc: `int`\n",
+ " Number of monte carlo samples used for approximation of objective gradients\n",
+ "tf_n_mc: `int`\n",
+ " Number of monte carlo samples used for approximation of test function gradients\n",
+ "obj_optimizer: function (grads, params) -> updates\n",
+ " Optimizer that is used for objective params\n",
+ "test_optimizer: function (grads, params) -> updates\n",
+ " Optimizer that is used for test function params\n",
+ "more_obj_params: `list`\n",
+ " Add custom params for objective optimizer\n",
+ "more_tf_params: `list`\n",
+ " Add custom params for test function optimizer\n",
+ "more_updates: `dict`\n",
+ " Add custom updates to resulting updates\n",
+ "total_grad_norm_constraint: `float`\n",
+ " Bounds gradient norm, prevents exploding gradient problem\n",
+ "score: `bool`\n",
+ " calculate loss on each step? Defaults to False for speed\n",
+ "compile_kwargs: `dict`\n",
+ " Add kwargs to pytensor.function (e.g. `{'profile': True}`)\n",
+ "fn_kwargs: dict\n",
+ " arbitrary kwargs passed to `pytensor.function`\n",
+ "\n",
+ " .. warning:: `fn_kwargs` is deprecated and will be removed in future versions\n",
+ "\n",
+ "more_replacements: `dict`\n",
+ " Apply custom replacements before calculating gradients\n",
+ "\n",
+ "Returns\n",
+ "-------\n",
+ "`pytensor.function`\n",
+ "\u001b[0;31mFile:\u001b[0m ~/upstream/pymc/pymc/variational/opvi.py\n",
+ "\u001b[0;31mType:\u001b[0m function"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "pm.opvi.ObjectiveFunction.step_function?"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "83b5504c-6a99-40eb-8b45-b485a8618ecd",
+ "metadata": {},
+ "source": [
+ "## Proposed Improvements\n",
+ "\n",
+ "There is a lot to like here, but there is also a lot of indirection. Further, much of it isn't used for the `ADVI` case. This is all in service of `SVGD` and `ASVGD`\n",
+ "\n",
+ "Further, the `Inference` class has to be aware of too many of these details. Ideally the `Inference` should be reworked to only take in a step function. It could be re-named `Trainer` to match what's in PyTorch Lightning. I think forcing all `VI` through `OPVI` makes it more challenging to write and port new `VI` algorithms to `pymc`"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f7e83647-ac36-4043-8bb5-8fcf5581ffa3",
+ "metadata": {},
+ "source": [
+ "### PyTorch Lightning and Optax optimization"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ece80fe0-e332-444d-b1e3-761aca6a02e8",
+ "metadata": {},
+ "source": [
+ "How would this look? One possibility is having each Variational Inference technique encapsulated into an object that takes a model and optimizer as inputs and provides a step function as a method.\n",
+ "\n",
+ "````python\n",
+ "class ADVI(Inference):\n",
+ " def __init__(self, model=None, optimizers=None):\n",
+ " ...\n",
+ "\n",
+ " def step(self, batch):\n",
+ " ...\n",
+ " return loss\n",
+ "````\n",
+ "\n",
+ "This is then passed to a `Trainer` object for fitting\n",
+ "\n",
+ "````python\n",
+ "with model:\n",
+ " trainer = Trainer(method=ADVI(), dataloader= ...)\n",
+ " advi_fit = trainer.fit(n=10_000)\n",
+ "````\n",
+ "\n",
+ "Under this setup most of the optimization logic moves into the `__init__` and `step` methods. As for how those should happen. I think this can be handled separately. But something like optax might not be so bad. So we could end with code that resembles the below\n",
+ "\n",
+ "````python\n",
+ "class ADVI(Inference):\n",
+ " def __init__(self, model=None, optimizers=None):\n",
+ " if model is None:\n",
+ " model = modelcontext(None)\n",
+ " if optimizers is None:\n",
+ " optimizers = [pm.opt.Adam(1e-3)]\n",
+ " self.optimizer = optimizers[0]\n",
+ " self.params = self.optimizer.init(model.basic_RVs)\n",
+ "\n",
+ " def step(self, batch):\n",
+ " loss = self.loss_function(self.params, batch)\n",
+ " grads = grad(loss)\n",
+ " self.params = self.optimizer.update(grads, self.params)\n",
+ " return loss\n",
+ "````"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d5572e2b-6a32-492d-9cb0-d003a14f490d",
+ "metadata": {},
+ "source": [
+ "### Model and Guide programs\n",
+ "\n",
+ "Additionally it would be nice if we could easily suppose variational inference with guide programs ala pyro/numpyro\n",
+ "\n",
+ "The way this could look is we define both as `pymc` models and then pass them to a `SVI` method\n",
+ "\n",
+ "````python\n",
+ "with pm.Model() as model:\n",
+ " data = pm.Data(\"data\", ...)\n",
+ " x = pm.Normal(\"x\", 0, 1)\n",
+ " y = pm.Normal(\"y\", x, 1, observed=data)\n",
+ "\n",
+ "with pm.Model() as guide:\n",
+ " mu = pt.tensor(\"mu\", param=True)\n",
+ " sd = pt.tensor(\"sd\", param=True)\n",
+ " x = pm.Normal(\"x\", mu, sd)\n",
+ "\n",
+ "\n",
+ "with model:\n",
+ " trainer = Trainer(method=SVI(model, guide), dataloader= ...)\n",
+ " trainer.fit(n=10_000)\n",
+ "````\n",
+ "\n",
+ "Naturally, `SVI` is a very general inference method, and in fact we can re-define `ADVI` in terms of it. Following the lead of pyro/numpyro we can have a guide generation\n",
+ "\n",
+ "````python\n",
+ "with model:\n",
+ " guide = AutoGuide(model)\n",
+ " trainer = Trainer(method=SVI(model, guide), dataloader= ...)\n",
+ " trainer.fit(n=10_000)\n",
+ "````"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7f97c341-e9bb-4301-b452-d006d6408cec",
+ "metadata": {},
+ "source": [
+ "### Reworking Minibatch\n",
+ "\n",
+ "Another small change we should consider is moving `pm.Minibatch` out of the model. Max already has a [proposal](https://github.com/pymc-devs/pymc/issues/7496) that I think can be adopted with only a few changes.\n",
+ "\n",
+ "I think where before we explicitly minibatch the data, instead we have dataloaders that stream in updates to the model.\n",
+ "\n",
+ "````python\n",
+ "with pm.Model() as model:\n",
+ " data = pm.Data(\"data\", None)\n",
+ " x = pm.Normal(\"x\", 0, 1)\n",
+ " y = pm.Normal(\"y\", x, 1, observed=data)\n",
+ "\n",
+ "dataloader = pm.Dataloader(np.random.normal(10_000, 2), batch_size=64)\n",
+ "\n",
+ "with model:\n",
+ " trainer = Trainer(method=ADVI(), dataloader=dataloader)\n",
+ " trainer.fit(n=10_000)\n",
+ "````\n",
+ "\n",
+ "Importantly, the model doesn't need to know about the dataloader. We will need to tweak the inference object, but it's not so bad.\n",
+ "\n",
+ "````python\n",
+ "class ADVI(Inference):\n",
+ " def step(self, batch):\n",
+ " self.model.set_data(\"data\", batch)\n",
+ " ...\n",
+ "````"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d241d205-ad9c-4afe-b5da-33ea5d2b880b",
+ "metadata": {},
+ "source": [
+ "Dataloader:\n",
+ "\n",
+ "__iter__ # loading the minibatches\n",
+ "__len__ # total_size\n",
+ "\n",
+ "\n",
+ "````python\n",
+ "with pm.Model(coords=coords) as model:\n",
+ " x = pm.Normal(\"x\", dims=\"documents\")\n",
+ " y = pm.Normal(\"y\", dims=\"documents\")\n",
+ "\n",
+ "with model:\n",
+ " ...\n",
+ " mf_fit = advi.fit(..)\n",
+ "\n",
+ "with model:\n",
+ " mf_fit.sample() # Error: fitted variable `x` has size `batch_size` but model variable `x` has size `full_size`\n",
+ "````"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0cc747c0-4b8f-40d6-ab05-788bd1d03bb3",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "pymc-dev",
+ "language": "python",
+ "name": "pymc-dev"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/pymc/variational/autoguide.py b/pymc/variational/autoguide.py
new file mode 100644
index 0000000000..de252dc648
--- /dev/null
+++ b/pymc/variational/autoguide.py
@@ -0,0 +1,170 @@
+# Copyright 2025 - present The PyMC Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Protocol
+
+import numpy as np
+import pytensor.tensor as pt
+
+from pytensor import config
+from pytensor.graph.basic import Variable
+from pytensor.graph.fg import FunctionGraph
+from pytensor.graph.replace import graph_replace, vectorize_graph
+from pytensor.graph.traversal import ancestors
+from pytensor.tensor import TensorLike, TensorVariable
+from pytensor.tensor.basic import infer_shape_db
+from pytensor.tensor.random.op import RandomVariable
+from pytensor.tensor.rewriting.shape import ShapeFeature
+
+from pymc.distributions import Normal
+from pymc.distributions.distribution import SymbolicRandomVariable
+from pymc.distributions.shape_utils import change_dist_size
+from pymc.logprob.basic import conditional_logp
+from pymc.model.core import Deterministic, Model
+from pymc.pytensorf import compile, rewrite_pregrad
+
+
+def vectorize_random_graph(
+ graph: Sequence[TensorVariable], batch_draws: TensorLike
+) -> tuple[TensorVariable]:
+ # Find the root random nodes
+ rvs = tuple(
+ var
+ for var in ancestors(graph)
+ if (
+ var.owner is not None
+ and isinstance(var.owner.op, RandomVariable | SymbolicRandomVariable)
+ )
+ )
+ rvs_set = set(rvs)
+ root_rvs = tuple(rv for rv in rvs if not (set(rv.owner.inputs) & rvs_set))
+
+ # Vectorize graph by vectorizing root RVs
+ batch_draws = pt.as_tensor(batch_draws, dtype=int)
+ vectorized_replacements = {
+ root_rv: change_dist_size(root_rv, new_size=batch_draws, expand=True)
+ for root_rv in root_rvs
+ }
+ return vectorize_graph(graph, replace=vectorized_replacements)
+
+
+@dataclass(frozen=True)
+class AutoGuideModel:
+ model: Model
+ params_init_values: dict[Variable, np.ndarray]
+
+ @property
+ def params(self) -> tuple[Variable]:
+ return tuple(self.params_init_values.keys())
+
+ def stochastic_logq(self):
+ """Returns a graph representing the logp of the guide model, evaluated under draws from its random variables."""
+ # This allows arbitrary
+ logp_terms = conditional_logp(
+ {rv: rv for rv in self.model.deterministics},
+ warn_rvs=False,
+ )
+ return pt.sum([logp_term.sum() for logp_term in logp_terms.values()])
+
+
+def get_symbolic_rv_shapes(
+ rvs: Sequence[Variable], raise_if_rvs_in_graph: bool = True
+) -> tuple[TensorVariable]:
+ # TODO: Move me to pytensorf, this is needed often
+
+ rv_shapes = [rv.shape for rv in rvs]
+ shape_fg = FunctionGraph(outputs=rv_shapes, features=[ShapeFeature()], clone=True)
+ with config.change_flags(optdb__max_use_ratio=10, cxx=""):
+ infer_shape_db.default_query.rewrite(shape_fg)
+ rv_shapes = shape_fg.outputs
+
+ if raise_if_rvs_in_graph and (overlap := (set(rvs) & set(ancestors(rv_shapes)))):
+ raise ValueError(f"rv_shapes still depend the following rvs {overlap}")
+
+ return tuple(rv_shapes)
+
+
+def AutoDiagonalNormal(model) -> AutoGuideModel:
+ coords = model.coords
+ free_rvs = model.free_RVs
+
+ free_rv_shapes = dict(zip(free_rvs, get_symbolic_rv_shapes(free_rvs)))
+ params_init_values = {}
+
+ with Model(coords=coords) as guide_model:
+ for rv in free_rvs:
+ loc = pt.tensor(f"{rv.name}_loc", shape=rv.type.shape)
+ scale = pt.tensor(f"{rv.name}_scale", shape=rv.type.shape)
+ # TODO: Make these customizable
+ params_init_values[loc] = pt.random.uniform(-1, 1, size=free_rv_shapes[rv]).eval()
+ params_init_values[scale] = pt.full(free_rv_shapes[rv], 0.1).eval()
+
+ z = Normal(
+ f"{rv.name}_z",
+ mu=0,
+ sigma=1,
+ shape=free_rv_shapes[rv],
+ )
+ Deterministic(
+ rv.name,
+ loc + pt.softplus(scale) * z,
+ dims=model.named_vars_to_dims.get(rv.name, None),
+ )
+
+ return AutoGuideModel(guide_model, params_init_values)
+
+
+def get_logp_logq(model: Model, guide: AutoGuideModel):
+ inputs_to_guide_rvs = {
+ model_value_var: guide.model[rv.name]
+ for rv, model_value_var in model.rvs_to_values.items()
+ if rv not in model.observed_RVs
+ }
+
+ logp = graph_replace(model.logp(), inputs_to_guide_rvs)
+ logq = guide.stochastic_logq()
+
+ return logp, logq
+
+
+def advi_objective(logp: TensorVariable, logq: TensorVariable):
+ negative_elbo = logq - logp
+ return negative_elbo
+
+
+class TrainingFn(Protocol):
+ def __call__(self, draws: int, *params: np.ndarray) -> tuple[np.ndarray, ...]: ...
+
+
+def compile_svi_training_fn(model: Model, guide: AutoGuideModel, **compile_kwargs) -> TrainingFn:
+ draws = pt.scalar("draws", dtype=int)
+ params = guide.params
+
+ logp, logq = get_logp_logq(model, guide)
+
+ scalar_negative_elbo = advi_objective(logp, logq)
+ [negative_elbo_draws] = vectorize_random_graph([scalar_negative_elbo], batch_draws=draws)
+ negative_elbo = negative_elbo_draws.mean(axis=0)
+
+ negative_elbo_grads = pt.grad(rewrite_pregrad(negative_elbo), wrt=params)
+
+ if "trust_input" not in compile_kwargs:
+ compile_kwargs["trust_input"] = True
+
+ f_loss_dloss = compile(
+ inputs=[draws, *params], outputs=[negative_elbo, *negative_elbo_grads], **compile_kwargs
+ )
+
+ return f_loss_dloss
diff --git a/pymc/variational/trainer.py b/pymc/variational/trainer.py
new file mode 100644
index 0000000000..ab5b151075
--- /dev/null
+++ b/pymc/variational/trainer.py
@@ -0,0 +1,14 @@
+from dataclasses import dataclasses
+from typing import Iterable
+from .inference import Inference
+
+@dataclasses
+class Trainer:
+ method: Inference
+ dataloader: Iterable # Dataloader
+
+ def fit(self, n: int):
+ """Fit the inference method for `n` iterations"""
+ for i in range(n):
+ batch = self.dataloader.next()
+ method.step(batch)
\ No newline at end of file
diff --git a/tests/variational/test_autoguide.py b/tests/variational/test_autoguide.py
new file mode 100644
index 0000000000..9ff53a38e7
--- /dev/null
+++ b/tests/variational/test_autoguide.py
@@ -0,0 +1,137 @@
+# Copyright 2025 - present The PyMC Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+import pytensor.tensor as pt
+import pytest
+
+import pymc as pm
+
+from pymc.variational.autoguide import AutoDiagonalNormal, AutoFullRankNormal, get_logp_logq
+
+Parameter = pt.tensor
+
+
+@pytest.fixture(scope="module")
+def X_y_params():
+ """Generate synthetic data for testing."""
+
+ rng = np.random.default_rng(sum(map(ord, "autoguide_test")))
+
+ alpha = rng.normal(loc=100, scale=10)
+ beta = rng.normal(loc=0, scale=1, size=(10,))
+
+ true_params = {
+ "alpha": alpha,
+ "beta": beta,
+ }
+
+ X_data = rng.normal(size=(100, 10))
+ y_data = alpha + X_data @ beta
+
+ return X_data, y_data, true_params
+
+
+@pytest.fixture(scope="module")
+def model(X_y_params):
+ X_data, y_data, _ = X_y_params
+
+ with pm.Model() as model:
+ X = pm.Data("X", X_data)
+ alpha = pm.Normal("alpha", 100, 10)
+ beta = pm.Normal("beta", 0, 5, size=(10,))
+
+ mu = alpha + X @ beta
+ sigma = pm.Exponential("sigma", 1)
+ y = pm.Normal("y", mu=mu, sigma=sigma, observed=y_data)
+
+ return model
+
+
+@pytest.fixture(scope="module")
+def target_guide_model(X_y_params):
+ X_data, *_ = X_y_params
+
+ draws = pt.tensor("draws", shape=(), dtype="int64")
+
+ with pm.Model() as guide_model:
+ X = pm.Data("X", X_data)
+
+ alpha_loc = Parameter("alpha_loc", shape=())
+ alpha_scale = Parameter("alpha_scale", shape=())
+ alpha_z = pm.Normal("alpha_z", mu=0, sigma=1, shape=(draws,))
+ alpha = pm.Deterministic("alpha", alpha_loc + alpha_scale * alpha_z)
+
+ beta_loc = Parameter("beta_loc", shape=(10,))
+ beta_scale = Parameter("beta_scale", shape=(10,))
+ beta_z = pm.Normal("beta_z", mu=0, sigma=1, shape=(draws, 10))
+ beta = pm.Deterministic("beta", beta_loc + beta_scale * beta_z)
+
+ sigma_loc = Parameter("sigma_loc", shape=())
+ sigma_scale = Parameter("sigma_scale", shape=())
+ sigma_z = pm.Normal(
+ "sigma_z", 0, 1, shape=(draws,), transform=pm.distributions.transforms.log
+ )
+ sigma = pm.Deterministic("sigma", sigma_loc + sigma_scale * sigma_z)
+
+ return guide_model
+
+
+def test_diagonal_normal_autoguide(model, target_guide_model, X_y_params):
+ guide_model = AutoDiagonalNormal(model)
+
+ logp, logq = get_logp_logq(model, guide_model)
+ logp_target, logq_target = get_logp_logq(model, target_guide_model)
+
+ inputs = pm.inputvars(logp)
+ target_inputs = pm.inputvars(logp_target)
+
+ expected_locs = [f"{var}_loc" for var in ["alpha", "beta", "sigma"]]
+ expected_scales = [f"{var}_scale" for var in ["alpha", "beta", "sigma"]]
+
+ expected_inputs = expected_locs + expected_scales + ["draws"]
+ name_to_input = {input.name: input for input in inputs}
+ name_to_target_input = {input.name: input for input in target_inputs}
+
+ assert all(input.name in expected_inputs for input in inputs), (
+ "Guide inputs do not match expected inputs"
+ )
+
+ negative_elbo = (logq - logp).mean()
+ negative_elbo_target = (logq_target - logp_target).mean()
+
+ fn = pm.compile(
+ [name_to_input[input] for input in expected_inputs], negative_elbo, random_seed=69420
+ )
+ fn_target = pm.compile(
+ [name_to_target_input[input] for input in expected_inputs],
+ negative_elbo_target,
+ random_seed=69420,
+ )
+
+ test_inputs = {
+ "alpha_loc": np.zeros(()),
+ "alpha_scale": np.ones(()),
+ "beta_loc": np.zeros(10),
+ "beta_scale": np.ones(10),
+ "sigma_loc": np.zeros(()),
+ "sigma_scale": np.ones(()),
+ "draws": 100,
+ }
+
+ np.testing.assert_allclose(fn(**test_inputs), fn_target(**test_inputs))
+
+
+def test_full_mv_normal_guide(model, X_y_params):
+ guide_model = AutoFullRankNormal(model)
+ logp, logq = get_logp_logq(model, guide_model)