perplexity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reconstruction_replacement(value, hook, autoencoder):\n",
    "    # Rearrange to fit autoencoder\n",
    "    int_val = rearrange(value, 'b s h -> (b s) h')\n",
    "    # Run through the autoencoder\n",
    "    reconstruction = autoencoder.predict(int_val)\n",
    "    batch, seq_len, hidden_size = value.shape\n",
    "    reconstruction = rearrange(reconstruction, '(b s) h -> b s h', b=batch, s=seq_len)\n",
    "    return reconstruction\n",
    "\n",
    "def reconstruction_no_outliers(value, hook, autoencoder, outliers):\n",
    "    # Rearrange to fit autoencoder\n",
    "    int_val = rearrange(value, 'b s h -> (b s) h')\n",
    "    # Run through the autoencoder\n",
    "    reconstruction = autoencoder.predict(int_val)\n",
    "    batch, seq_len, hidden_size = value.shape\n",
    "    reconstruction = rearrange(reconstruction, '(b s) h -> b s h', b=batch, s=seq_len)\n",
    "    # Replace outliers with original values\n",
    "    reconstruction[:, :, outliers] = value[:, :, outliers]\n",
    "    return reconstruction\n",
    "\n",
    "def find_outliers(dataset, cache_name, model, num_outlier_dims=2):\n",
    "    device = model.cfg.device\n",
    "    batch_size = 32\n",
    "    with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "        dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "        # Just take the first batch\n",
    "        first_batch = next(iter(dl))\n",
    "        _, cache = model.run_with_cache(first_batch.to(device))\n",
    "        layer_activations = rearrange(cache[cache_name], \"b s h -> (b s) h\").cpu()\n",
    "        outlier_dims = layer_activations.abs().max(0).values.topk(num_outlier_dims).indices\n",
    "    return outlier_dims\n",
    "\n",
    "def get_original_perplexity(model, dataset):\n",
    "    total_loss = 0\n",
    "    batch_size = 32\n",
    "    with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "        dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "        for i, batch in enumerate(dl):\n",
    "            loss = model(batch.to(device), return_type=\"loss\")\n",
    "            total_loss += loss.item()\n",
    "        # Average\n",
    "        avg_neg_log_likelihood = total_loss / len(dl)\n",
    "\n",
    "        # Convert to tensor\n",
    "        avg_neg_log_likelihood = torch.tensor(avg_neg_log_likelihood, device=device)\n",
    "\n",
    "        # Exponentiate to compute perplexity\n",
    "        perplexity = torch.exp(avg_neg_log_likelihood)\n",
    "        print(f\"Perplexity for original model: {perplexity.item():.2f}\")\n",
    "    return perplexity.item()\n",
    "\n",
    "def get_perplexity(autoencoder, model, dataset, forward_hook):\n",
    "    device = model.cfg.device\n",
    "    model = model.eval()\n",
    "    batch_size = 32\n",
    "    autoencoder.to_device(device)\n",
    "    total_loss = 0\n",
    "    with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "        dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "        for i, batch in enumerate(dl):\n",
    "            loss = model.run_with_hooks(\n",
    "                batch.to(device), \n",
    "                return_type=\"loss\",\n",
    "                fwd_hooks=[forward_hook]\n",
    "                )\n",
    "            total_loss += loss.item()\n",
    "    # Average\n",
    "    avg_neg_log_likelihood = total_loss / len(dl)\n",
    "\n",
    "    # Convert to tensor\n",
    "    avg_neg_log_likelihood = torch.tensor(avg_neg_log_likelihood, device=device)\n",
    "\n",
    "    # Exponentiate to compute perplexity\n",
    "    perplexity = torch.exp(avg_neg_log_likelihood)\n",
    "    print(f\"Perplexity {perplexity.item():.2f}\")\n",
    "    return perplexity.item()\n",
    "\n",
    "def download_dataset(dataset_name, max_length=256, num_datapoints=None):\n",
    "    dataset = load_dataset(dataset_name, split=\"train\")\n",
    "    if(num_datapoints):\n",
    "        split_text = f\"train[:{num_datapoints}]\"\n",
    "    else:\n",
    "        split_text = \"train\"\n",
    "    dataset = load_dataset(dataset_name, split=split_text).map(\n",
    "        lambda x: model.tokenizer(x['text']),\n",
    "        batched=True,\n",
    "    ).filter(\n",
    "        lambda x: len(x['input_ids']) > max_length\n",
    "    ).map(\n",
    "        lambda x: {'input_ids': x['input_ids'][:max_length]}\n",
    "    )\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mchorse/logan/t/logan/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "#Change these settings to load the correct autoencoder\n",
    "from transformer_lens import HookedTransformer\n",
    "import torch\n",
    "import numpy as np\n",
    "from torch.utils.data import DataLoader\n",
    "from datasets import Dataset, load_dataset\n",
    "from tqdm.auto import tqdm\n",
    "from functools import partial\n",
    "from einops import rearrange\n",
    "layer = 2\n",
    "setting = \"residual\"\n",
    "device = torch.device(\"cuda:5\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "if setting == \"residual\":\n",
    "    cache_name = f\"blocks.{layer}.hook_resid_post\"\n",
    "elif setting == \"mlp\":\n",
    "    cache_name = f\"blocks.{layer}.mlp.hook_post\"\n",
    "else:\n",
    "    raise NotImplementedError\n",
    "gpt2_autoencoders = torch.load(\"/mnt/ssd-cluster/gpt2small/tied_residual_l2_r8/_31/learned_dicts.pt\")\n",
    "pythia_autoencoders = torch.load(\"/mnt/ssd-cluster/longrun2408/tied_residual_l2_r6/_31/learned_dicts.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained model gpt2 into HookedTransformer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset parquet (/home/mchorse/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n",
      "Found cached dataset parquet (/home/mchorse/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n",
      "Loading cached processed dataset at /home/mchorse/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-3fdc5f8176a0d569.arrow\n",
      "Loading cached processed dataset at /home/mchorse/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-fe8c1d66ae63294c.arrow\n",
      "Loading cached processed dataset at /home/mchorse/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-ab6a44fadfb19740.arrow\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Perplexity for original model: 27.56\n",
      "Perplexity 27.60\n",
      "Perplexity 27.61\n",
      "Perplexity 27.65\n",
      "Perplexity 27.97\n",
      "Perplexity 28.64\n",
      "Perplexity 30.66\n",
      "Perplexity 32.77\n",
      "Perplexity 39.18\n",
      "Perplexity 77.80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained model EleutherAI/pythia-70m-deduped into HookedTransformer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset parquet (/home/mchorse/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n",
      "Found cached dataset parquet (/home/mchorse/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n",
      "Loading cached processed dataset at /home/mchorse/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-6814e00ee585e5e7.arrow\n",
      "Loading cached processed dataset at /home/mchorse/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-95d5ac1abb14d087.arrow\n",
      "Loading cached processed dataset at /home/mchorse/.cache/huggingface/datasets/NeelNanda___parquet/NeelNanda--pile-10k-72f566e9f7c464ab/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-2b36cec300a8056d.arrow\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Perplexity for original model: 27.52\n",
      "Perplexity 27.51\n",
      "Perplexity 27.76\n",
      "Perplexity 28.43\n",
      "Perplexity 31.45\n",
      "Perplexity 41.50\n",
      "Perplexity 61.40\n",
      "Perplexity 99.80\n",
      "Perplexity 300.33\n",
      "Perplexity 4183.59\n"
     ]
    }
   ],
   "source": [
    "all_the_aes = [gpt2_autoencoders, pythia_autoencoders]\n",
    "models = [\"gpt2\", \"EleutherAI/pythia-70m-deduped\"]\n",
    "all_model_perplexities2 = []\n",
    "all_model_original_perplexities = []\n",
    "all_model_l1s = []\n",
    "for i, model_name in enumerate(models):\n",
    "    model = HookedTransformer.from_pretrained(model_name, device=device)\n",
    "    all_autoencoders = all_the_aes[i]\n",
    "    # Get dataset\n",
    "    dataset = download_dataset(\"NeelNanda/pile-10k\", max_length=256, num_datapoints=100)\n",
    "    # Get top-N outlier dims\n",
    "    outlier_dims = find_outliers(dataset, cache_name, model, num_outlier_dims=2)\n",
    "    # Get original model perplexity \n",
    "    original_perplexity = get_original_perplexity(model, dataset)\n",
    "    # Get perplexity for all autoencoders\n",
    "    \n",
    "    num_autoencoders = len(all_autoencoders)\n",
    "    all_perplexities = np.zeros(num_autoencoders)\n",
    "    for i, (autoencoder, hyperparam) in enumerate(all_autoencoders):\n",
    "        forward_hook = (cache_name, partial(reconstruction_no_outliers, autoencoder=autoencoder, outliers = outlier_dims))\n",
    "        # forward_hook = (cache_name, partial(reconstruction_replacement, autoencoder=autoencoder))\n",
    "        all_perplexities[i] = get_perplexity(autoencoder, model, dataset, forward_hook)\n",
    "\n",
    "    all_model_original_perplexities.append(original_perplexity)\n",
    "    all_model_perplexities2.append(all_perplexities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtYAAAHHCAYAAACbch9lAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABwj0lEQVR4nO3dd1hT1/8H8HdACBsUAUFZTnAh4vg60Yp7VivOijiqLa46Wm2riIvaqtU667bWWXcduOr+uQXrnmhxT0CgApLz+4PmlhCCINEb4P16njwk596b+7kzH07OOVEIIQSIiIiIiChPjOQOgIiIiIioIGBiTURERESkB0ysiYiIiIj0gIk1EREREZEeMLEmIiIiItIDJtZERERERHrAxJqIiIiISA+YWBMRERER6QETayIiIiIiPcj3ifXBgwehUChw8ODB97aORo0aoVGjRu/t/d+md+/e8PDw0ChTKBQYP368Rtnp06dRt25dWFpaQqFQICoqCgAQERGBatWqwczMDAqFArGxsR8kbkPUu3dvWFlZyR0GUY5kde2T4crpvbowkPtzk96//HJue3h4oHfv3u+07LtsY64S6+XLl0OhUEgPMzMzlC9fHoMGDcLjx49zteL87MGDBxg/fryUuBqC1NRUdO7cGS9evMBPP/2ElStXwt3dHc+fP0dgYCDMzc0xd+5crFy5EpaWlnKHW6DMmzcPy5cvlzsMegeGcC0bQgxZ2blzp+wfmvv370efPn1Qvnx5WFhYoHTp0ujXrx8ePnwoW0yGerzI8KxevRozZ84s9DEUNkXeZaEJEybA09MTr1+/xtGjRzF//nzs3LkTFy9ehIWFhb5jlN2ePXs0Xj948ABhYWHw8PBAtWrVZInpn3/+QZEi/x2+W7du4e7du1i0aBH69esnlUdERODVq1eYOHEiAgIC5Ai1wJs3bx6KFy/+zv8Rk3wM4VrOLoZFixZBpVLJEtfOnTsxd+5cWZPrr7/+Gi9evEDnzp1Rrlw53L59G3PmzMH27dsRFRWFEiVKfPCYcnvOZL5XU+GxevVqXLx4EcOGDSvUMRQ273S1t2zZEjVq1AAA9OvXD/b29pgxYwa2bt2Kbt265SmgpKQkg0vOTU1N5Q5Bi5mZmcbrJ0+eAADs7OxyVJ4XiYmJrPUu5ArrOfCh708mJiYfbF2GaMaMGahfvz6MjP77crVFixbw9/fHnDlzMGnSJBmjy5nM9+q8eP36NUxNTTX2R0GjUqmQkpKi1/2WHxSGY1tY6OUIfvTRRwCA6Ohoqey3336Dn58fzM3NUaxYMXTt2hUxMTEayzVq1AiVK1fG2bNn0bBhQ1hYWOCbb74BkN4mpk2bNtizZ4/UPrhixYrYtGlTjmI6efIkWrRoAVtbW1hYWMDf3x/Hjh2Tpl+5cgXm5ubo1auXxnJHjx6FsbExvv76a4041W3FDh48iJo1awIAgoODpWYxy5cvR2hoKExMTPD06VOteD777DPY2dnh9evX2ca9ZcsWVK5cGWZmZqhcuTI2b96c5XwZ2/307t0b/v7+AIDOnTtDoVBIMQcFBQEAatasCYVCoVGr+rZ9BADjx4+HQqHA5cuX0b17dxQtWhT169eXpufmOF++fBmNGzeGhYUFSpYsiR9++EFru16/fo3x48ejfPnyMDMzg7OzMzp27Ihbt25J86hUKsycOROVKlWCmZkZnJycMGDAALx8+TLbfZvR7du30bx5c1haWsLFxQUTJkyAEEJjnpysx8PDA5cuXcKhQ4ekc6FRo0aIjY2FsbExfv75Z2neZ8+ewcjICPb29hrr+vzzz7Vq3nJybADg/v376NOnD5ycnKBUKlGpUiUsXbpUYx51P4T169dj8uTJKFWqFMzMzNCkSRPcvHnzrftKH+eAeptatWqFokWLwtLSElWrVsWsWbM05vnzzz/RoEEDWFpaws7ODu3bt8eVK1eyjOfmzZvo3bs37OzsYGtri+DgYCQlJWnMu3fvXtSvXx92dnawsrJChQoVpHtMdtcykP39SVe7u6za8sXGxuLLL7+Eh4cHlEolSpUqhV69euHZs2dvjSGrNruJiYkYMWIEXF1doVQqUaFCBUybNk3r/FUoFBg0aJB0T1GfHxEREVpxZ9a7d2/MnTtXeh/1411jWLVqFSpUqAAzMzP4+fnh8OHDb40BABo2bKiVaDRs2BDFihXTOi+yM2/ePFSqVAlKpRIuLi4ICQnR6muiqx1mTu//umR1ruTmul27di2+++47lCxZEhYWFoiPj0dqairCwsJQrlw5mJmZwd7eHvXr18fevXuz3Q8vXrzAyJEjUaVKFVhZWcHGxgYtW7bE+fPns1x3Tu8ZCxcuRJkyZWBubo5atWrhyJEj2caRef+ozxH1MVKfoznZT0DOPjf0fd28evUKw4YNk65rR0dHNG3aFOfOnQOQft7s2LEDd+/elc4T9bWc3bFV398yUzfFvXPnjkb5rl274O/vD2tra9jY2KBmzZpYvXr1W2MAgOTkZISGhqJs2bJQKpVwdXXFV199heTkZI11JCcn48svv4SDgwOsra3Rrl073Lt3L5uj+p+M51JYWBhKliwJa2trfPLJJ4iLi0NycjKGDRsGR0dHWFlZITg4WGv9b968wcSJE1GmTBkolUp4eHjgm2++0ZpPCIFJkyahVKlSsLCwQOPGjXHp0qUs44qNjcWwYcOk86Fs2bKYOnWqXr4h1Mv3U+qT197eHgAwefJkjB07FoGBgejXrx+ePn2K2bNno2HDhoiMjNSoPX3+/DlatmyJrl27omfPnnBycpKm3bhxA126dMHAgQMRFBSEZcuWoXPnzoiIiEDTpk11xvPnn3+iZcuW8PPzQ2hoKIyMjLBs2TJ89NFHOHLkCGrVqgVvb29MnDgRo0aNwieffIJ27dohMTERvXv3hpeXFyZMmJDle3t7e2PChAkYN24cPvvsMzRo0AAAULduXdSvXx8TJkzAunXrMGjQIGmZlJQUbNiwAZ06dcr2v/A9e/agU6dOqFixIsLDw/H8+XMEBwejVKlS2e7/AQMGoGTJkpgyZQqGDBmCmjVrSvuxQoUKWLhwodR8p0yZMjneRxmpv4qdMmWKdCPKzXF++fIlWrRogY4dOyIwMBAbNmzA119/jSpVqqBly5YAgLS0NLRp0wb79+9H165dMXToULx69Qp79+7FxYsXpdgHDBiA5cuXIzg4GEOGDEF0dDTmzJmDyMhIHDt27K21fGlpaWjRogX+97//4YcffkBERARCQ0Px5s0bjeOek/XMnDkTgwcPhpWVFb799lsAgJOTE+zs7FC5cmUcPnwYQ4YMAZD+T5tCocCLFy9w+fJlVKpUCQBw5MgR6TzKzbF5/Pgx/ve//0kfBA4ODti1axf69u2L+Ph4ra/+vv/+exgZGWHkyJGIi4vDDz/8gB49euDkyZPZ7i99nAN79+5FmzZt4OzsjKFDh6JEiRK4cuUKtm/fjqFDhwIA9u3bh5YtW6J06dIYP348/vnnH8yePRv16tXDuXPntBLMwMBAeHp6Ijw8HOfOncPixYvh6OiIqVOnAgAuXbqENm3aoGrVqpgwYQKUSiVu3rwp/YOS3bWslt39KScSEhLQoEEDXLlyBX369EH16tXx7NkzbNu2Dffu3ctRDBkJIdCuXTscOHAAffv2RbVq1bB7926MGjUK9+/fx08//aQx/9GjR7Fp0yZ88cUXsLa2xs8//4xOnTrh77//lu7XWRkwYAAePHiAvXv3YuXKlXmK4dChQ1i3bh2GDBkCpVKJefPmoUWLFjh16hQqV66cq/2p3qcJCQkoXrx4juYfP348wsLCEBAQgM8//xzXrl3D/Pnzcfr06RzdLzLK7fHKSm6v24kTJ8LU1BQjR45EcnIyTE1NMX78eISHh6Nfv36oVasW4uPjcebMGZw7dy7bz8bbt29jy5Yt6Ny5Mzw9PfH48WP88ssv8Pf3x+XLl+Hi4qIxf07uGUuWLMGAAQNQt25dDBs2DLdv30a7du1QrFgxuLq65mif/Pnnn1i/fj0GDRqE4sWLw8PDI8f7KSefG+/juhk4cCA2bNiAQYMGoWLFinj+/DmOHj2KK1euoHr16vj2228RFxeHe/fuSe+fueN8Vsc2N5YvX44+ffqgUqVKGDNmDOzs7BAZGYmIiAh079492xhUKhXatWuHo0eP4rPPPoO3tzcuXLiAn376CdevX8eWLVuk9fTr1w+//fYbunfvjrp16+LPP/9E69atcxVreHg4zM3NMXr0aNy8eROzZ8+GiYkJjIyM8PLlS4wfPx4nTpzA8uXL4enpiXHjxmmsf8WKFfjkk08wYsQInDx5EuHh4bhy5YpG5eO4ceMwadIktGrVCq1atcK5c+fQrFkzpKSkaMSSlJQEf39/3L9/HwMGDICbmxv+7//+D2PGjMHDhw/z3iZd5MKyZcsEALFv3z7x9OlTERMTI9auXSvs7e2Fubm5uHfvnrhz544wNjYWkydP1lj2woULokiRIhrl/v7+AoBYsGCB1rrc3d0FALFx40apLC4uTjg7OwtfX1+p7MCBAwKAOHDggBBCCJVKJcqVKyeaN28uVCqVNF9SUpLw9PQUTZs2lcrS0tJE/fr1hZOTk3j27JkICQkRRYoUEadPn9aIxd/fX/j7+0uvT58+LQCIZcuWacVdp04dUbt2bY2yTZs2acSoS7Vq1YSzs7OIjY2Vyvbs2SMACHd3d415AYjQ0FCt/fD7779rzKc+Zhm3KTf7KDQ0VAAQ3bp103jfdznOv/76q1SWnJwsSpQoITp16iSVLV26VAAQM2bM0No36jiPHDkiAIhVq1ZpTI+IiMiyPLOgoCABQAwePFjjvVu3bi1MTU3F06dPc72eSpUqaZwfaiEhIcLJyUl6PXz4cNGwYUPh6Ogo5s+fL4QQ4vnz50KhUIhZs2ZJseT02PTt21c4OzuLZ8+eaay3a9euwtbWViQlJQkh/js3vL29RXJysjTfrFmzBABx4cKFbPdZXs+BN2/eCE9PT+Hu7i5evnypMW/GbaxWrZpwdHQUz58/l8rOnz8vjIyMRK9evbTi6dOnj8Z7ffzxx8Le3l56/dNPPwkA0jHNSnbXcnb3p8zXn5q7u7sICgqSXo8bN04AEJs2bdKaV73t2cUQFBSkce1v2bJFABCTJk3SmO+TTz4RCoVC3Lx5UyNGU1NTjbLz588LAGL27Nla68osJCREZPURkdsYAIgzZ85IZXfv3hVmZmbi448/fmsMWZk4caIAIPbv3//WeZ88eSJMTU1Fs2bNRFpamlQ+Z84cAUAsXbpUKst87NRyc//PfLyE0D5Xcnvdli5dWipT8/HxEa1bt37L1mt7/fq1xn4QQojo6GihVCrFhAkTpLKc3jNSUlKEo6OjqFatmsZ8CxcuFACyvC9mBkAYGRmJS5cuaZTndD/l5HPjfVw3tra2IiQkJNtta926tdb5IET2x1Z9f8tM/VkeHR0thBAiNjZWWFtbi9q1a4t//vkny+3OLoaVK1cKIyMjceTIEY3yBQsWCADi2LFjQgghoqKiBADxxRdfaMzXvXt3nffBrLa1cuXKIiUlRSrv1q2bUCgUomXLlhrz16lTRyNe9fr79eunMd/IkSMFAPHnn38KIf671lu3bq2x/d98840AoHFtT5w4UVhaWorr169rvOfo0aOFsbGx+Pvvv6WynGxjZu/UFCQgIAAODg5wdXVF165dYWVlhc2bN6NkyZLYtGkTVCoVAgMD8ezZM+lRokQJlCtXDgcOHNB4L6VSieDg4CzX4+Ligo8//lh6bWNjg169eiEyMhKPHj3KcpmoqCjcuHED3bt3x/Pnz6X1JyYmokmTJjh8+LBU1W9kZITly5cjISEBLVu2xLx58zBmzBip/fi76NWrF06ePKnxFdSqVavg6uoqNdfIysOHDxEVFYWgoCDY2tpK5U2bNkXFihXfOZ6s5GYfqQ0cOFDjdW6Ps5WVFXr27Cm9NjU1Ra1atXD79m2pbOPGjShevDgGDx6sFbP6q7Hff/8dtra2aNq0qcZ6/fz8YGVlpbVeXTJ+o6CuEUlJScG+ffv0tp4GDRrg8ePHuHbtGoD0mumGDRuiQYMG0lelR48ehRBCqvnK6bERQmDjxo1o27YthBAaMTZv3hxxcXHSV5JqwcHBGjUi6nVmPAbZeddzIDIyEtHR0Rg2bJhWW3/1cVWf/71790axYsWk6VWrVkXTpk2xc+fOt8bToEEDPH/+HPHx8QD+61ewdevWd/56L7v7U05s3LgRPj4+Gvcxtay+7n2bnTt3wtjYWPoWRG3EiBEQQmDXrl0a5QEBAdI3PUD6/rSxscnxMddHDHXq1IGfn5/02s3NDe3bt8fu3buRlpaWq3UfPnwYYWFhCAwMlJogZmffvn1ISUnBsGHDNJqU9O/fHzY2NtixY0eu1p9X73LdBgUFwdzcXKPMzs4Oly5dwo0bN3K1fqVSKe2HtLQ0PH/+XGoilXm9wNvvGWfOnMGTJ08wcOBAjfl69+6t8Tn2Nv7+/hqfc7nZTzn53Hgf142dnR1OnjyJBw8e5Hg7M8vq2ObU3r178erVK4wePVrrm/Cc3Ft+//13eHt7w8vLS2P/qq8r9f1bfe/NvO9y2xmyV69eGt8O1a5dG0II9OnTR2O+2rVrIyYmBm/evNFY//DhwzXmGzFiBABI17D6Wh88eLDG9mcV5++//44GDRqgaNGiGtseEBCAtLS0HDdV0+WdmoLMnTsX5cuXR5EiReDk5IQKFSpIF+uNGzcghEC5cuWyXDbz124lS5bU+fVH2bJltU6Q8uXLAwDu3LmTZY9w9Y1G3bY4K3FxcShatCgAoEyZMhg/fjxGjRqFypUrY+zYsTqXy4kuXbpg2LBhWLVqFcaNG4e4uDhs374dX375ZbYn+927dwEgy/2m66b3rnK7jwDA09NT6z1yc5xLlSqltf1FixbFX3/9Jb2+desWKlSokG0P+hs3biAuLg6Ojo5ZTld31syOkZERSpcurVGW8bzS13rUH0JHjhxBqVKlEBkZiUmTJsHBwQHTpk2TptnY2MDHx0daL/D2Y5OamorY2FgsXLgQCxcuzFGMbm5uGq/VxzenbdPf9RxQ/5OZ3df+6vO/QoUKWtO8vb2xe/durQ6T2W2PjY0NunTpgsWLF6Nfv34YPXo0mjRpgo4dO+KTTz7JcQeh7O5POXHr1i106tTpnZfP7O7du3BxcYG1tbVGube3tzQ9o8z7CEjfT+pjnpKSghcvXmhMd3BwgLGxsd5iyOr8KF++PJKSkvD06VMUK1YsRzFcvXoVH3/8MSpXrozFixdrTHvx4oXG173m5uawtbXVeV6ZmpqidOnSWrG+b0+fPs31dZv5ugPSR+Zq3749ypcvj8qVK6NFixb49NNPUbVq1WzXr1KpMGvWLMybNw/R0dEa/9hk1TTobfcMXZ9bJiYmWvfY7GTextzsp5x8buj7ugGAH374AUFBQXB1dYWfnx9atWqFXr165Wm7cyMn99Xs3LhxA1euXIGDg0OW09X79+7duzAyMtL4RwPI+l6dncz7VP2PV+bmQra2tlCpVIiLi4O9vb20/rJly2rMV6JECdjZ2UnHTte56ODgoJHLAOnb/tdff71129/VOyXWtWrV0lmrq1KpoFAosGvXrixvzpnbGL3rf2u6qGumfvzxR51DIWWOQT2c3oMHD/D8+fM8DeFUtGhRtGnTRkqsN2zYgOTkZI3aWrm9yz7KfJxye5x1fVCLTB1H3kalUsHR0RGrVq3KcrquCyW39LEeFxcXeHp64vDhw/Dw8IAQAnXq1IGDgwOGDh2Ku3fv4siRI6hbt66U6OX02Dx//hwA0LNnT51JeOYP2bweg7yeA/r2tu0xNzfH4cOHceDAAezYsQMRERFYt24dPvroI+zZsyfb5FEtt/en3NbAvm9v20f/93//h8aNG2tMi46O/qA/SpOTGGJiYtCsWTPY2tpi586dWglSx44dcejQIel1UFBQrseW11XxkZaWlqNzJSfU13durtuszsGGDRvi1q1b2Lp1K/bs2YPFixfjp59+woIFCzSGW81sypQpGDt2LPr06YOJEyeiWLFiMDIywrBhw7L8Vkdf9+23yereAuRuP+lTTrY7MDAQDRo0wObNm7Fnzx78+OOPmDp1KjZt2iT1G3qbrI5tduehPqlUKlSpUgUzZszIcnpO28fnlK59mtNz7F2+4dNFpVKhadOm+Oqrr7Kcrq5oe1d6H1xT3VHA09Mzz8HdvHkTQgiNHXr9+nUA0HnjV/9XZWNjk6NxmxcsWIC9e/di8uTJCA8Px4ABA7B169Zsl3nbAe7Vqxfat2+P06dPY9WqVfD19ZU6quni7u4OAFl+taduSqAvud1Hut5DX8c543uePHkSqampOjsUlSlTBvv27UO9evXe+Z8ylUqF27dva8Sd+bzKzXqyOx8aNGiAw4cPw9PTE9WqVYO1tTV8fHxga2uLiIgInDt3DmFhYRrbB7z92Kh7Z6elpck2PnlOzwH1Nl28eFFnrOrzP6tz/erVqyhevPg7De9nZGSEJk2aoEmTJpgxYwamTJmCb7/9FgcOHEBAQMA736yLFi2qNapESkqK1g+XlClTBhcvXsz2vXITg7u7O/bt24dXr15pJJdXr16VpueGj4+P1kgS6ooFXXHlNoas7mnXr1+HhYUFHBwcoFQqdcYApHcgbdasGZKTk7F//344Oztrvd/06dM1ahPVnfAynlcZaxJTUlIQHR2tcT5mdUyB9JqwjMvm5QNen9dtsWLFEBwcjODgYCQkJKBhw4YYP358ton1hg0b0LhxYyxZskSjPDY2NsedQTPK+LmVsWlOamoqoqOjpW/icis3+yknnxv6vm7UnJ2d8cUXX+CLL77AkydPUL16dUyePFlKrN/lXFHXrsbGxmo0nctcq57xvpq5NjcjXTGUKVMG58+fR5MmTbKN093dHSqVSvpmQE3fecnb1n/jxg3pGwYgvRNwbGysdOwynosZr9enT59qfStbpkwZJCQkvLfPTr0PmNixY0cYGxsjLCxM6z8OIYRU05YTDx480OjxGR8fj19//RXVqlXTWavs5+eHMmXKYNq0aUhISNCannEovOjoaIwaNQqdOnXCN998g2nTpmHbtm349ddfs41L/QGv66fBW7ZsieLFi2Pq1Kk4dOhQjmqrnZ2dUa1aNaxYsQJxcXFS+d69e3H58uW3Lp8budlHuujzOKt16tQJz549w5w5c7SmqdcRGBiItLQ0TJw4UWueN2/e5Pjn2jOuQwiBOXPmwMTEBE2aNMn1eiwtLXWut0GDBrhz5w7WrVsnNQ0xMjJC3bp1MWPGDKSmpmqMCJLTY2NsbIxOnTph48aNWSZuOTmGeZXTc6B69erw9PTEzJkztfaTermM53/GeS5evIg9e/agVatWuY4vc/MCANK3AOphmt52LetSpkwZrXZ4Cxcu1KpV6tSpE86fP5/lsJnqbc9NDK1atUJaWprWNfLTTz9BoVDkuKZMrWjRoggICNB4qNtr6oortzEcP35coylbTEwMtm7dimbNmsHY2DjbGBITE9GqVSvcv38fO3fu1NnsyM/PT2N5dXvdgIAAmJqa4ueff9Y4R5csWYK4uDiNkQ3KlCmDEydOaDQp2b59u9bQke96zgD6u24z31+trKxQtmxZreHHslp/5mv1999/x/3793O03sxq1KgBBwcHLFiwQGO/LV++/J32T8Y4c7qfcvK5oe/rJi0tTeNzGgAcHR3h4uKicQwsLS215nsbdcKc8f6SmJiIFStWaMzXrFkzWFtbIzw8XGsY34zHWFcMgYGBuH//PhYtWqQ17Z9//kFiYiIASPsm49CxAD7Yrzmq7/2Z16euaVdfwwEBATAxMcHs2bM1tj+rOAMDA3H8+HHs3r1ba1psbKzUvvtdvZca60mTJmHMmDG4c+cOOnToAGtra0RHR2Pz5s347LPPMHLkyBy9V/ny5dG3b1+cPn0aTk5OWLp0KR4/foxly5bpXMbIyAiLFy9Gy5YtUalSJQQHB6NkyZK4f/8+Dhw4ABsbG/zxxx9So3lzc3PMnz8fQPoQUxs3bsTQoUMREBCgNfRQxm20s7PDggULYG1tDUtLS9SuXVtqL2ViYoKuXbtizpw5MDY2zvGP5oSHh6N169aoX78++vTpgxcvXmD27NmoVKlSlknWu8rpPsqOPo+zWq9evfDrr79i+PDhOHXqFBo0aIDExETs27cPX3zxBdq3bw9/f38MGDAA4eHhiIqKQrNmzWBiYoIbN27g999/x6xZs/DJJ59kux4zMzNEREQgKCgItWvXxq5du7Bjxw588803UhOP3KzHz88P8+fPx6RJk1C2bFk4OjpKtTfqpPnatWuYMmWKFEPDhg2xa9cuKJVKaVzc3B6b77//HgcOHEDt2rXRv39/VKxYES9evMC5c+ewb9++LBNLfcrpOWBkZIT58+ejbdu2qFatGoKDg+Hs7IyrV6/i0qVL0s3txx9/RMuWLVGnTh307dtXGm7P1tb2nX79b8KECTh8+DBat24Nd3d3PHnyBPPmzUOpUqWkcbjfdi3r0q9fPwwcOBCdOnVC06ZNcf78eezevVur1m/UqFHYsGEDOnfujD59+sDPzw8vXrzAtm3bsGDBAvj4+OQqhrZt26Jx48b49ttvcefOHfj4+GDPnj3YunUrhg0bptUOMi/UHQ6HDBmC5s2bw9jYGF27ds11DJUrV0bz5s01htsDoPFNjS49evTAqVOn0KdPH1y5ckVj7GorKyt06NAh2+UdHBwwZswYhIWFoUWLFmjXrh2uXbuGefPmoWbNmhqVHv369cOGDRvQokULBAYG4tatW/jtt9+0tuddzxk1fVy3FStWRKNGjeDn54dixYrhzJkz0tBv2WnTpg0mTJiA4OBg1K1bFxcuXMCqVaty1S44IxMTE0yaNAkDBgzARx99hC5duiA6OhrLli175/dUy+l+ysnnhr6vm1evXqFUqVL45JNP4OPjAysrK+zbtw+nT5/G9OnTpfn8/Pywbt06DB8+HDVr1oSVlRXatm2b7Xs3a9YMbm5u6Nu3L0aNGgVjY2MsXboUDg4O+Pvvv6X5bGxs8NNPP6Ffv36oWbOm9BsD58+fR1JSkpSI64rh008/xfr16zFw4EAcOHAA9erVQ1paGq5evYr169dj9+7dqFGjBqpVq4Zu3bph3rx5iIuLQ926dbF///4c/QaCPvj4+CAoKAgLFy5EbGws/P39cerUKaxYsQIdOnSQmpE5ODhg5MiRCA8PR5s2bdCqVStERkZi165dWd6Xt23bhjZt2qB3797w8/NDYmIiLly4gA0bNuDOnTvv9A2OJDdDiGQ1dJsuGzduFPXr1xeWlpbC0tJSeHl5iZCQEHHt2jVpHn9/f1GpUqUsl3d3dxetW7cWu3fvFlWrVhVKpVJ4eXlpDSeXebg9tcjISNGxY0dhb28vlEqlcHd3F4GBgdIQTephgzIO5yeEEH///bewsbERrVq10ogz87BBW7duFRUrVhRFihTJcuilU6dOCQCiWbNmb91XGW3cuFF4e3sLpVIpKlasKDZt2pSjIZxyM9ye2tv2kRD/Df2ja8iyvBznrLYrKSlJfPvtt8LT01OYmJiIEiVKiE8++UTcunVLY76FCxcKPz8/YW5uLqytrUWVKlXEV199JR48eJBlnBnXaWlpKW7duiWaNWsmLCwshJOTkwgNDdUahiqn63n06JFo3bq1sLa2znKIKUdHRwFAPH78WCo7evSoACAaNGiQZZw5OTZCCPH48WMREhIiXF1dpf3VpEkTsXDhQmkeXedGdHS0zmHDMtLHOaDe5qZNmwpra2thaWkpqlatqjXs2759+0S9evWEubm5sLGxEW3bthWXL1/OUTyZh6Pav3+/aN++vXBxcRGmpqbCxcVFdOvWTWuIJV3Xcnb3p7S0NPH111+L4sWLCwsLC9G8eXNx8+bNLIdse/78uRg0aJAoWbKkMDU1FaVKlRJBQUEaw4jpiiGra+TVq1fiyy+/FC4uLsLExESUK1dO/PjjjxpDTAmRfo/IajgwXcPKZfbmzRsxePBg4eDgIBQKhcYQYLmN4bfffhPlypUTSqVS+Pr6vnXo0Yyx4t8h+zI/shpCTJc5c+YILy8vYWJiIpycnMTnn3+uNfSjEEJMnz5dlCxZUiiVSlGvXj1x5syZXN3/c3KvFiJv160QQkyaNEnUqlVL2NnZCXNzc+Hl5SUmT56sMZxZVl6/fi1GjBghnJ2dhbm5uahXr544fvy41jbm9p4xb9484enpKZRKpahRo4Y4fPhwlvstK7rOUyFytp+EyNnnhj6vm+TkZDFq1Cjh4+Mj3c98fHzEvHnzNJZJSEgQ3bt3F3Z2dhrnbHbHVgghzp49K2rXri1MTU2Fm5ubmDFjhtb9TW3btm2ibt260j2zVq1aYs2aNW+NQYj04RKnTp0qKlWqJJRKpShatKjw8/MTYWFhIi4uTprvn3/+EUOGDBH29vbC0tJStG3bVsTExORquL2c5iZZ3d9TU1NFWFiYdHxdXV3FmDFjxOvXrzWWTUtLE2FhYdL53ahRI3Hx4sUs73mvXr0SY8aMEWXLlhWmpqaiePHiom7dumLatGka11FOtjEzxb8LGhwPDw9UrlwZ27dvlzuUd3L+/HlUq1YNv/76Kz799FO5wyEi+uAUCgVCQkKy/JqeiKgg4o/SvyeLFi2ClZUVOnbsKHcoRERERPQB6L2NdWH3xx9/4PLly1i4cCEGDRr0TiMZEBEREVH+w8RazwYPHozHjx+jVatWOeqcQ0REREQFg8E2Bblz506+bF99584d/PPPP9iyZYs0Xubhw4fRtm1buLi4QKFQYMuWLRrLCCEwbtw4ODs7w9zcHAEBAbn+qVoiIkMj/h3KkoiosDDYxLogSUxMhI+PD+bOnZvl9B9++AE///wzFixYgJMnT8LS0hLNmzfXGpuSiIiIiAyXwY4KUlApFAps3rxZGn9VCAEXFxeMGDFCGvc5Li4OTk5OWL58Obp27SpjtERERESUU2xjLbPo6Gg8evRI46c1bW1tUbt2bRw/flxnYp2cnKzxC08qlQovXryAvb19nn5yl4iIiD4cIQRevXoFFxcXGBmxIUF+x8RaZo8ePQIAODk5aZQ7OTlJ07ISHh7OzpFEREQFRExMDEqVKiV3GJRHTKzzqTFjxmD48OHS67i4OLi5uSEmJgY2NjYyRkZERIXOo4vAshaApSMw5Jzc0eQr8fHxcHV1lQY8oPyNibXMSpQoAQB4/PgxnJ2dpfLHjx+jWrVqOpdTKpVQKpVa5TY2NkysiYjow0q0BJQKwLwIwM+gd8JmnAUDG/PIzNPTEyVKlMD+/fulsvj4eJw8eRJ16tSRMTIiIqIcksZBYHJIhRtrrD+AhIQE3Lx5U3odHR2NqKgoFCtWDG5ubhg2bBgmTZqEcuXKwdPTE2PHjoWLi4s0cggREVG+wFpXKuSYWH8AZ86cQePGjaXX6rbRQUFBWL58Ob766iskJibis88+Q2xsLOrXr4+IiAiYmZnJFTIREVEucOReIoDjWBcY8fHxsLW1RVxcXLZtrNPS0pCamvoBIyOi/MzU1JRDgNHb3T8HLGoM2JQChl+SO5p8Jaef35Q/sMa6kBBC4NGjR4iNjZU7FCLKR4yMjODp6QlTU1O5QyGD9m8dHZuCUCHHxLqQUCfVjo6OsLCwYO9jInorlUqFBw8e4OHDh3Bzc+N9g3KA5wgVbkysC4G0tDQpqba3t5c7HCLKRxwcHPDgwQO8efMGJiYmcodDhoqNSokAcLi9QkHdptrCwkLmSIgov1E3AUlLS5M5EjJs6qYg8kZBJDcm1oUIv8YlotzifYNyhONYEwFgYk1ERER5xs6LRAATayIiItIbJtZUuDGxpgJPoVBgy5YtGmWbNm1C06ZN4eDgABsbG9SpUwe7d++WJ0AiovyOP4lBBICJNRVShw8fRtOmTbFz506cPXsWjRs3Rtu2bREZGSl3aERE+RCbghABTKzJwL169Qo9evSApaUlnJ2d8dNPP6FRo0YYNmwYAMDDwwMTJ05Et27dYGlpiZIlS2Lu3LnS8h4eHgCAjz/+GAqFQno9c+ZMfPXVV6hZsybKlSuHKVOmoFy5cvjjjz8+8BYSERUA7LxIBICJdaElhEBSypsP/hC5/Lpw+PDhOHbsGLZt24a9e/fiyJEjOHfunMY8P/74I3x8fBAZGYnRo0dj6NCh2Lt3LwDg9OnTAIBly5bh4cOH0uvMVCoVXr16hWLFir3D3iQiIgCssaZCjz8QU0j9k5qGiuM+fJviyxOaw8I0Z6fdq1evsGLFCqxevRpNmjQBkJ4gu7i4aMxXr149jB49GgBQvnx5HDt2DD/99JPUhhoA7OzsUKJECZ3rmjZtGhISEhAYGPgum0VEVMixjTURwBprMmC3b99GamoqatWqJZXZ2tqiQoUKGvPVqVNH6/WVK1dyvJ7Vq1cjLCwM69evh6OjY96CJiIqjNgUhAgAa6wLLXMTY1ye0FyW9RqStWvXol+/fvj9998REBAgdzhERPkUOy8SAUysCy2FQpHjJhlyKV26NExMTHD69Gm4ubkBAOLi4nD9+nU0bNhQmu/EiRMay504cQLe3t7SaxMTkyx/jnnNmjXo06cP1q5di9atW7+nrSAiKkyYWFPhZtiZFRVq1tbWCAoKwqhRo1CsWDE4OjoiNDQURkZGGj+zfOzYMfzwww/o0KED9u7di99//x07duyQpnt4eGD//v2oV68elEolihYtitWrVyMoKAizZs1C7dq18ejRIwCAubk5bG1tP/i2EhHlaxzHmggA21iTgZsxYwbq1KmDNm3aICAgAPXq1YO3tzfMzMykeUaMGIEzZ87A19cXkyZNwowZM9C8+X/NXKZPn469e/fC1dUVvr6+AICFCxfizZs3CAkJgbOzs/QYOnToB99GIqL8j01BiADWWJOBs7a2xqpVq6TXiYmJCAsLw2effSaV2djYYP369Trfo23btmjbtq1G2cGDB/UeKxFRocXOi0QAmFiTgYuMjMTVq1dRq1YtxMXFYcKECQCA9u3byxwZERH9hzXWRAATa8oHpk2bhmvXrsHU1BR+fn44cuQIihcvLndYRESkhYk1FW5MrMmg+fr64uzZszqn37lz58MFQ0REWWPnRSIA7LxIREREecamIEQAE2siIiLKK3ZeJALAxJqIiIj0hXk1FXJMrImIiCiP2MaaCGBiTURERHkl5dWssqbCjYk1ERER5RE7LxIBTKwpn1MoFNiyZct7X8/y5cthZ2f33tdjqHr37o0OHTpkO4+Hhwdmzpz5QeKRU072xdscPHgQCoUCsbGxeompoMRC+Rg7LxIBYGJNBq53795QKBRajxYtWry3dcqZIHp5eUGpVOLRo0da0xo1aoRhw4ZplTdu3BiLFy/Wy/rv3LkDhUKBqKioXC97+vRpjZ+afxe6jnelSpU05ps7dy48PDxgZmaG2rVr49SpU3laLxHpCWusqZBjYk0Gr0WLFnj48KHGY82aNXKHpRepqanS86NHj+Kff/7BJ598ghUrVuRo+RcvXuDYsWNo27bt+woxxxwcHGBhYZGn95g1a5bGcY6JiUGxYsXQuXNnaZ5169Zh+PDhCA0Nxblz5+Dj44PmzZvjyZMned0EInpn7LxIBDCxpnxAqVSiRIkSGo+iRYtmOW9MTAwCAwNhZ2eHYsWKoX379hq/zphVrW+HDh3Qu3dvafrdu3fx5ZdfSrWlGe3evRve3t6wsrKSEv6MFi9eDG9vb5iZmcHLywvz5s2Tpqlrg9etWwd/f3+YmZlh1apV0vQlS5age/fu+PTTT7F06dIc7ZsdO3agevXqcHJykr7S37FjB6pWrQozMzP873//w8WLFwEAiYmJsLGxwYYNGzTeY8uWLbC0tMSrV6/g6ekJIP0XLxUKBRo1aqQx77Rp0+Ds7Ax7e3uEhIRo/GOQuaZ/xowZqFKlCiwtLeHq6oovvvgCCQkJ2W6Pra2txnE+c+YMXr58ieDgYI337d+/P4KDg1GxYkUsWLAAFhYWGvtMoVDgl19+QZs2bWBhYQFvb28cP34cN2/eRKNGjWBpaYm6devi1q1b2caTlpaG4cOHw87ODvb29vjqq68gMv3CnEqlQnh4ODw9PWFubg4fHx+tfbxz506UL18e5ubmaNy4sdYvho4fPx7VqlXTKJs5cyY8PDyk1+omKGFhYXBwcICNjQ0GDhyIlJQUvcZC9E7YFIQIABPrwksIICXxwz/e48/epqamonnz5rC2tsaRI0dw7NgxKQHOmHxkZ9OmTShVqhQmTJgg1ZqqJSUlYdq0aVi5ciUOHz6Mv//+GyNHjpSmr1q1CuPGjcPkyZNx5coVTJkyBWPHjtWqfR49ejSGDh2KK1euoHnz5gCAV69e4ffff0fPnj3RtGlTxMXF4ciRI2+Nd9u2bWjfvr1G2ahRozB9+nScPn0aDg4OaNu2LVJTU2FpaYmuXbti2bJlGvMvW7YMn3zyCaytraUmFfv27cPDhw+xadMmab4DBw7g1q1bOHDgAFasWIHly5dj+fLlOmMzMjLCzz//jEuXLmHFihX4888/8dVXX711mzJasmQJAgIC4O7uDgBISUnB2bNnERAQoLGegIAAHD9+XGPZiRMnolevXoiKioKXlxe6d++OAQMGYMyYMThz5gyEEBg0aFC2658+fTqWL1+OpUuX4ujRo3jx4gU2b96sMU94eDh+/fVXLFiwAJcuXcKXX36Jnj174tChQwDS/9nr2LEj2rZti6ioKPTr1w+jR4/O1X5Q279/P65cuYKDBw9izZo12LRpE8LCwmSJhUgTOy8SAUARuQMgmaQmAVNcPvx6v3kAmFrmapHt27fDyspK822++QbffPONRtm6deugUqmwePFiqaZ52bJlsLOzw8GDB9GsWbO3rqtYsWIwNjaGtbU1SpQooTEtNTUVCxYsQJkyZQAAgwYNwoQJE6TpoaGhmD59Ojp27AgA8PT0xOXLl/HLL78gKChImm/YsGHSPGpr165FuXLlpLbEXbt2xZIlS9CgQQOdsSYnJyMiIgLjx4/XKA8NDUXTpk0BACtWrECpUqWwefNmBAYGol+/fqhbty4ePnwIZ2dnPHnyBDt37sS+ffsApDfnAAB7e3ut7S9atCjmzJkDY2NjeHl5oXXr1ti/fz/69++fZXwZvxnw8PDApEmTMHDgQI1a/Ow8ePAAu3btwurVq6WyZ8+eIS0tDU5OThrzOjk54erVqxplwcHBCAwMBAB8/fXXqFOnDsaOHSv9MzN06FCNmvCszJw5E2PGjJGO14IFC7B7925penJyMqZMmYJ9+/ahTp06AIDSpUvj6NGj+OWXX+Dv74/58+ejTJkymD59OgCgQoUKuHDhAqZOnZqj/ZCRqakpli5dCgsLC1SqVAkTJkzAqFGjMHHiRKSmpn7QWIiyxsSaCjcm1mTwGjdujPnz52uUFStWTGu+8+fP4+bNm7C2ttYof/369Vu/8s8JCwsLKakGICWmQHozi1u3bqFv374aieabN29ga2ur8T41atTQeu+lS5eiZ8+e0uuePXvC398fs2fP1toetT///BOOjo5aHfvUSRWQvp8qVKiAK1euAABq1aqFSpUqYcWKFRg9ejR+++03uLu7o2HDhm/d/kqVKsHY2Fhj+y9cuKBz/n379iE8PBxXr15FfHw83rx5g9evXyMpKQkWFhYa/yz17NkTCxYs0Fh+xYoVsLOze+cROKpWrSo9VyfiVapU0Sh7/fo14uPjERsbi4oVK0rTvvnmG4SEhODhw4eoXbu2VF6kSBHUqFFDag5y8+ZNJCUlSf/IqKWkpMDX1xcAcOXKFY33ADSPUW74+PhotGOvU6cOEhISEBMTg4SEhA8aC5GG9/htJFF+wsS6sDKxSK89lmO9uWRpaYmyZcu+db6EhAT4+flptFtWU9fEGhkZabWRzdhOODsmJiYarxUKhfRe6rbDixYt0kpcMiajQPr2ZHT58mWcOHECp06dwtdffy2Vp6WlYe3atTprhLdt24Z27drlKPaM+vXrh7lz52L06NFYtmwZgoODtdqSZyWr7VepVFnOe+fOHbRp0waff/45Jk+ejGLFiuHo0aPo27cvUlJSYGFhoTHyiI2NjcbyQggsXboUn376KUxNTaXy4sWLw9jYGI8fP9aY//Hjx1o17BnjVW9fVmUqlQouLi4a8WT1j1tW1Md9x44dKFmypMY0pVKZo/cA8nZe6jsWonfDpiBEABPrwkuhyHWTDENXvXp1rFu3Do6OjlqJmpqDg4NGu+m0tDRcvHgRjRs3lspMTU2RlpaWq3U7OTnBxcUFt2/fRo8ePXK17JIlS9CwYUPMnTtXo3zZsmVYsmRJlom1EAJ//PEHfvvtN61pJ06cgJubGwDg5cuXuH79Ory9vaXpPXv2xFdffYWff/4Zly9f1mimok5ic7v9mZ09exYqlQrTp0+HkVF6V47169drzJPdP0uHDh3CzZs30bdvX41yU1NT+Pn5Yf/+/VJNtkqlwv79+9/aXjo7RYoUyTIeZ2dnnDx5UqrRf/PmDc6ePYvq1asDACpWrAilUom///4b/v7+Wb63t7c3tm3bplF24sQJjdcODg549OgRhBBSwp/VkIfnz5/HP//8A3Nzc+l9rKys4OrqimLFiuklFqJ3ws6LRADYeZHygeTkZDx69Ejj8ezZM635evTogeLFi6N9+/Y4cuQIoqOjcfDgQQwZMgT37t0DAHz00UfYsWMHduzYgatXr+Lzzz/X+mEMDw8PHD58GPfv389yPbqEhYUhPDwcP//8M65fv44LFy5g2bJlmDFjhs5lUlNTsXLlSnTr1g2VK1fWePTr1w8nT57EpUuXtJY7e/YskpKSUL9+fa1pEyZMwP79+3Hx4kX07t0bxYsX12hOUbRoUXTs2BGjRo1Cs2bNUKpUKWmao6MjzM3NERERgcePHyMuLi7H259R2bJlkZqaitmzZ+P27dtYuXKlVlOP7CxZsgS1a9dG5cqVtaYNHz4cixYtwooVK3DlyhV8/vnnSExMfGt76XcxdOhQfP/999iyZQuuXr2KL774QuN8sba2xsiRI/Hll19ixYoVuHXrFs6dO4fZs2dLnVYHDhyIGzduYNSoUbh27RpWr16t1emzUaNGePr0KX744QfcunULc+fOxa5du7TiSUlJQd++fXH58mXs3LkToaGhGDRoEIyMjPQWC1GesMaaCjkm1mTwIiIi4OzsrPHIKqG0sLDA4cOH4ebmho4dO8Lb2xt9+/bF69evpRrsPn36ICgoCL169YK/vz9Kly6tUVsNpCemd+7cQZkyZaQmJDnRr18/LF68GMuWLUOVKlXg7++P5cuXS0PYZWXbtm14/vw5Pv74Y61p3t7e8Pb2xpIlS7Smbd26Fa1atUKRItpfOn3//fcYOnQo/Pz88OjRI/zxxx8azSkASE0y+vTpo1FepEgR/Pzzz/jll1/g4uKiNeJITvn4+GDGjBmYOnUqKleujFWrViE8PDxHy8bFxWHjxo1atdVqXbp0wbRp0zBu3DhUq1YNUVFRiIiI0OrQqA8jRozAp59+iqCgINSpUwfW1tZax2rixIkYO3YswsPD4e3tjRYtWmDHjh3ScXdzc8PGjRuxZcsW+Pj4YMGCBZgyZYrGe3h7e2PevHmYO3cufHx8cOrUKY0RZ9SaNGmCcuXKoWHDhujSpQvatWun0XlVH7EQvRu2sSYCAIXI3LCP8qX4+HjY2toiLi5OqxnE69evER0dDU9PT5iZmckUIelT1apV8d1330mjXgDpP03duHFjvHz58q0/v75y5Up8+eWXePDggVbSTYapd+/eiI2NxZYtWz7oenn/oBy5vA1Y/yng+j+g7+63z0+S7D6/Kf9hG2uifCYlJQWdOnVCy5Ytc71sUlISHj58iO+//x4DBgxgUk1EesLOi0QAm4IQ5TumpqYIDQ3VOQxfdn744Qd4eXmhRIkSGDNmzHuIjogKJXZeJALAGmuiAqNRo0ZaQ7ZlNn78eK0flKH8gZ0MKV9gjTUVcqyxJiIiojxidy0igIl1ocJ+qkSUW7xvUI6wKQgRACbWhYL61+aSkpJkjoSI8puUlBQA2r8gSqSJnReJALaxLhSMjY1hZ2eHJ0+eAEgf7zknP2FNRIWbSqXC06dPYWFhkeWY6UREpIl3ykKiRIkSACAl10REOWFkZAQ3Nzf+M07ZY5MhIgBMrAsNhUIBZ2dnODo6IjU1Ve5wiCifMDU1hZERWw1SDvEfMCrkmFgXMsbGxmwrSURE+sXOi0QA2HmRiIiI8oydF4kAJtZERERERHrBxJqIiIjyhk1BiAAwsSYiIqI8Y1MQIoCJNREREeUVa6yJADCxJiIiIn1hjTUVckysiYiIKI/4AzFEABNrIiIiyis2BSECwMSaiIiI8oydF4kAJtZERESkN0ysqXBjYk1ERER5I9jGmghgYk1ERER5xqYgRAATayIiIsordl4kAsDEmoiIiPKMNdZEABNrg5CWloaxY8fC09MT5ubmKFOmDCZOnAjBNmtERJSvMLGmwq2I3AEQMHXqVMyfPx8rVqxApUqVcObMGQQHB8PW1hZDhgyROzwiIqLssSKICAATa4Pwf//3f2jfvj1at24NAPDw8MCaNWtw6tQpmSMjIiLKCTYFIQLYFMQg1K1bF/v378f169cBAOfPn8fRo0fRsmVLncskJycjPj5e40FERCQL1lgTAWCNtUEYPXo04uPj4eXlBWNjY6SlpWHy5Mno0aOHzmXCw8MRFhb2AaMkIiJ6C9ZYUyHHGmsDsH79eqxatQqrV6/GuXPnsGLFCkybNg0rVqzQucyYMWMQFxcnPWJiYj5gxERERBmxxpoIYI21QRg1ahRGjx6Nrl27AgCqVKmCu3fvIjw8HEFBQVkuo1QqoVQqP2SYREREWeM41kQAWGNtEJKSkmBkpHkojI2NoVKpZIqIiIjoHbApCBVyrLE2AG3btsXkyZPh5uaGSpUqITIyEjNmzECfPn3kDo2IiCgXmFhT4cbE2gDMnj0bY8eOxRdffIEnT57AxcUFAwYMwLhx4+QOjYiI6O04KggRACbWBsHa2hozZ87EzJkz5Q6FiIjoHXAcayKAbayJiIgor9h5kQgAE2siIiLKM9ZYEwFMrImIiEhvmFhT4cbEmoiIiPKGnReJADCxJiIiojxjUxAigIk1ERER5RU7LxIBYGJNRERE+sIaayrkmFgTERFRHrGNNRHAxJqIiIjyik1BiAAwsSYiIqI8Y+dFIoCJNREREeUVa6yJADCxJiIiIiLSCybWRERElEdsCkIEMLEmIiKivGJTECIATKyJiIgoz9Q11vJGQSQ3JtZERESkJ8ysqXBjYk1ERER5I/gDMUQAE2siIiLKM3ZeJAKYWBMREVFeSRXWTKypcGNiTURERPrBGmsq5JhYExERUR6xjTURwMSaiIiI8orjWBMBYGJNREREecbOi0QAE2siIiLKK9ZYEwFgYk1ERET6whprKuSYWBMREVEesfMiEcDEmoiIiPKKTUGIADCxJiIiojxj50UigIk1ERER6Q0TayrcmFgTERFR3gi2sSYCmFgTERFRnrEpCBHAxJqIiIjyip0XiQAwsSYiIqI8Y401EcDEmoiIiIhIL5hYZ+PIkSPo2bMn6tSpg/v37wMAVq5ciaNHj8ocGRERkQFhUxAiAEysddq4cSOaN28Oc3NzREZGIjk5GQAQFxeHKVOmyBwdERGRIWFTECKAibVOkyZNwoIFC7Bo0SKYmJhI5fXq1cO5c+dkjIyIiMjAcLg9IgBMrHW6du0aGjZsqFVua2uL2NjYDx8QERGRoWONNRVyTKx1KFGiBG7evKlVfvToUZQuXVqGiIiIiAwVa6yJACbWOvXv3x9Dhw7FyZMnoVAo8ODBA6xatQojR47E559/Lnd4REREhoOdF4kAAEXkDsBQjR49GiqVCk2aNEFSUhIaNmwIpVKJkSNHYvDgwXKHR0REZHjYFIQKOSbWOigUCnz77bcYNWoUbt68iYSEBFSsWBFWVlZyh0ZERGSgmFhT4cbEWoe4uDikpaWhWLFiqFixolT+4sULFClSBDY2NjJGR0REZEA4KggRALax1qlr165Yu3atVvn69evRtWtXGSIiIiIyVBzHmghgYq3TyZMn0bhxY63yRo0a4eTJkzJEREREZKDYeZEIABNrnZKTk/HmzRut8tTUVPzzzz8yRERERGSoWGNNBDCx1qlWrVpYuHChVvmCBQvg5+cnQ0REREREZMjYeVGHSZMmISAgAOfPn0eTJk0AAPv378fp06exZ88emaMjIiIyIGwKQgSANdY61atXD8ePH4erqyvWr1+PP/74A2XLlsVff/2FBg0ayB0eERGRAWFTECKANdbZqlatGlatWiV3GERERIaNNdZEAJhYZ0ulUuHmzZt48uQJVCqVxrSGDRvKFBUREZGBYo01FXJMrHU4ceIEunfvjrt370JkGvheoVAgLS1NpsiIiIgMDX8ghghgYq3TwIEDUaNGDezYsQPOzs5Q8L9wIiKirLEpCBEAJtY63bhxAxs2bEDZsmXlDoWIiMjAsfMiEcBRQXSqXbs2bt68KXcYRERE+QgTayrcWGOtw+DBgzFixAg8evQIVapUgYmJicb0qlWryhQZERGRgRFsY00EMLHWqVOnTgCAPn36SGUKhQJCCHZeJCIi0sCmIEQAE2udoqOj5Q6BiIgof2DnRSIATKx1cnd3lzsEIiKifEJdYy1vFERyY+fFbKxcuRL16tWDi4sL7t69CwCYOXMmtm7dKnNkREREhoiZNRVuTKx1mD9/PoYPH45WrVohNjZWalNtZ2eHmTNnyhscERGRIWHnRSIATKx1mj17NhYtWoRvv/0WxsbGUnmNGjVw4cIFGSMjIiIyNOy8SAQwsdYpOjoavr6+WuVKpRKJiYkyRERERGSgpAprJtZUuDGx1sHT0xNRUVFa5REREfD29tb7+u7fv4+ePXvC3t4e5ubmqFKlCs6cOaP39RAREb03rLGmQo6jgugwfPhwhISE4PXr1xBC4NSpU1izZg3Cw8OxePFiva7r5cuXqFevHho3boxdu3bBwcEBN27cQNGiRfW6HiIioveDbayJACbWOvXr1w/m5ub47rvvkJSUhO7du8PFxQWzZs1C165d9bquqVOnwtXVFcuWLZPKPD099boOIiKi94bjWBMBYFOQLL158wa//vorAgICcOPGDSQkJODRo0e4d+8e+vbtq/f1bdu2DTVq1EDnzp3h6OgIX19fLFq0KNtlkpOTER8fr/EgIiKSBzsvEgFMrLNUpEgRDBw4EK9fvwYAWFhYwNHR8b2t7/bt25g/fz7KlSuH3bt34/PPP8eQIUOwYsUKncuEh4fD1tZWeri6ur63+IiIiLLFGmsiAEysdapVqxYiIyM/yLpUKhWqV6+OKVOmwNfXF5999hn69++PBQsW6FxmzJgxiIuLkx4xMTEfJFYiIiIiyhrbWOvwxRdfYMSIEbh37x78/PxgaWmpMb1q1ap6W5ezszMqVqyoUebt7Y2NGzfqXEapVEKpVOotBiIionfHpiBEABNrndQdFIcMGSKVKRQKCCGgUCikX2LUh3r16uHatWsaZdevX4e7u7ve1kFERPTesCkIEQAm1jpFR0d/sHV9+eWXqFu3LqZMmYLAwECcOnUKCxcuxMKFCz9YDERERO+ONdZEABNrnT5kbXHNmjWxefNmjBkzBhMmTICnpydmzpyJHj16fLAYiIiI8o6JNRVu7LyYjZUrV6JevXpwcXHB3bt3AQAzZ87E1q1b9b6uNm3a4MKFC3j9+jWuXLmC/v37630dRERE74XgD8QQAUysdZo/fz6GDx+OVq1aITY2VmpTbWdnh5kzZ8obHBERkUFhUxAigIm1TrNnz8aiRYvw7bffwtjYWCqvUaMGLly4IGNkREREBoadF4kAMLHWKTo6Gr6+vlrlSqUSiYmJMkRERERk4FhjTYUcE2sdPD09ERUVpVUeEREBb2/vDx8QERGRwWIbayKAo4LoNHz4cISEhOD169cQQuDUqVNYs2YNwsPDsXjxYrnDIyIiMhxsCkIEgIm1Tv369YO5uTm+++47JCUloXv37nBxccGsWbOkH48hIiIigJ0XidKxKUgG27ZtQ2pqqvS6R48euHHjBhISEvDo0SPcu3cPffv2lTFCIiIiA8Th9ogAMLHW8PHHHyM2NhYAYGxsjCdPngAALCws4OjoKGNkRERERGTomFhn4ODggBMnTgAAhBBQ8CstIiKiHGBTECKAbaw1DBw4EO3bt4dCoYBCoUCJEiV0zqv+wRgiIqJCj50XiQAwsdYwfvx4dO3aFTdv3kS7du2wbNky2NnZyR0WERFR/sAaayrkmFhnsG3bNrRs2RJeXl4IDQ1F586dYWFhIXdYRERE+QQTayrc2MY6g4ydFydMmICEhAR5AyIiIsoPOCoIEQAm1hrYeZGIiOhdsPMiEcCmIBrYeZGIiOgdsPMiEQAm1hrYeZGIiOhdsMaaCGBircXLy4udF4mIiIgo15hY6xAaGip3CERERPkDm4IQAWBiraF69erYv38/ihYtCl9f32w7L547d+4DRkZERGTI2BSECGBiraF9+/ZQKpUAgA4dOsgbDBERUX7BGmsiAEysNWRs/sGmIERERLnEGmsq5DiONREREeURfyCGCGCNtYaiRYvm+EdhXrx48Z6jISIiyifYFIQIABNrDTNnzpSeP3/+HJMmTULz5s1Rp04dAMDx48exe/dujB07VqYIiYiIDBE7LxIBTKw1BAUFSc87deqECRMmYNCgQVLZkCFDMGfOHOzbtw9ffvmlHCESEREZMCbWVLixjbUOu3fvRosWLbTKW7RogX379skQERERkYESbGNNBDCx1sne3h5bt27VKt+6dSvs7e1liIiIiMhQsSkIEcCmIDqFhYWhX79+OHjwIGrXrg0AOHnyJCIiIrBo0SKZoyMiIjIg7LxIBICJtU69e/eGt7c3fv75Z2zatAkA4O3tjaNHj0qJNhEREQGssSZKx8Q6G7Vr18aqVavkDoOIiIiI8gG2sSYiIqK8YedFIgBMrImIiEhf2BSECjkm1kRERJQ37LxIBICJNREREekLa6ypkGNircOyZcuQlJQkdxhERET5ANtYEwFMrHUaPXo0SpQogb59++L//u//5A6HiIjIcLEpCBEAJtY63b9/HytWrMCzZ8/QqFEjeHl5YerUqXj06JHcoRERERkYjmNNBDCx1qlIkSL4+OOPsXXrVsTExKB///5YtWoV3Nzc0K5dO2zduhUqlUruMImIiOTHGmsiAEysc8TJyQn169dHnTp1YGRkhAsXLiAoKAhlypTBwYMH5Q6PiIiIiAwAE+tsPH78GNOmTUOlSpXQqFEjxMfHY/v27YiOjsb9+/cRGBiIoKAgucMkIiKSGZuCEAFMrHVq27YtXF1dsXz5cvTv3x/379/HmjVrEBAQAACwtLTEiBEjEBMTI3OkREREMmNTECIAQBG5AzBUjo6OOHToEOrUqaNzHgcHB0RHR3/AqIiIiAwRa6yJANZY6+Tv74/q1atrlaekpODXX38FACgUCri7u3/o0IiIiIjIADGx1iE4OBhxcXFa5a9evUJwcLAMERERERkoNgUhAsDEWichBBRZfKV179492NrayhARERGRoWJTECKAbay1+Pr6QqFQQKFQoEmTJihS5L9dlJaWhujoaLRo0ULGCImIiAwMa6yJADCx1tKhQwcAQFRUFJo3bw4rKytpmqmpKTw8PNCpUyeZoiMiIjJgrLGmQo6JdSahoaEAAA8PD3Tp0gVmZmYyR0RERGToxNtnISoEmFjrwB9+ISIiyiE2BSECwMRaQ7FixXD9+nUUL14cRYsWzbLzotqLFy8+YGRERESGjJ0XiQAm1hp++uknWFtbS8+zS6yJiIjoX6yxJgLAxFpDxuYfvXv3li8QIiIiIsp3OI61DsuXL8+y/M2bNxgzZsyHDYaIiMigqZuCyBsFkdyYWOswZMgQdO7cGS9fvpTKrl27htq1a2PNmjUyRkZERGRgpEFBmFlT4cbEWofIyEjcu3cPVapUwd69ezF37lxUr14dXl5eOH/+vNzhERERGRB2XiQC2MZapzJlyuDYsWMYNmwYWrRoAWNjY6xYsQLdunWTOzQiIiIDxcSaCjfWWGdjx44dWLt2LerUqQM7OzssWbIEDx48kDssIiIiwyL4AzFEABNrnQYMGIDOnTvj66+/xpEjR/DXX3/B1NQUVapUwfr16+UOj4iIyICwKQgRwKYgOh07dgwnT56Ej48PAKBEiRLYuXMn5s6diz59+iAwMFDmCImIiAwEx7EmAsDEWqezZ89CqVRqlYeEhCAgIECGiIiIiAwca6ypkGNTEB2USiVu3bqF7777Dt26dcOTJ08AALt27cKbN29kjo6IiMiQsI01EcDEWqdDhw6hSpUqOHnyJDZt2oSEhAQAwPnz5xEaGipzdERERAaETUGIADCx1mn06NGYNGkS9u7dC1NTU6n8o48+wokTJ2SMjIiIyNCw8yIRwMRapwsXLuDjjz/WKnd0dMSzZ8/e67q///57KBQKDBs27L2uh4iISC9YY00EgIm1TnZ2dnj48KFWeWRkJEqWLPne1nv69Gn88ssvqFq16ntbBxERERHpHxNrHbp27Yqvv/4ajx49gkKhgEqlwrFjxzBy5Ej06tXrvawzISEBPXr0wKJFi1C0aNH3sg4iIiL9Y1MQIoCJtU5TpkyBl5cXXF1dkZCQgIoVK6Jhw4aoW7cuvvvuu/eyzpCQELRu3TpHw/klJycjPj5e40FERCQLNgUhAsBxrHUyNTXFokWLMHbsWFy8eBEJCQnw9fVFuXLl3sv61q5di3PnzuH06dM5mj88PBxhYWHvJRYiIqLcYY01EcDE+q3c3Nzg5ub2XtcRExODoUOHYu/evTAzM8vRMmPGjMHw4cOl1/Hx8XB1dX1fIRIREeUAE2sq3JhYZ5AxUX2bGTNm6G29Z8+exZMnT1C9enWpLC0tDYcPH8acOXOQnJwMY2NjjWWUSmWWvwxJRET0wQn+QAwRwMRaQ2RkZI7mU+j5q64mTZrgwoULGmXBwcHw8vLC119/rZVUExERGRY2BSECmFhrOHDggCzrtba2RuXKlTXKLC0tYW9vr1VORERkcNh5kQgARwXJkZiYGMTExMgdBhERkYFijTURwMRapzdv3mDs2LGwtbWFh4cHPDw8YGtri++++w6pqanvff0HDx7EzJkz3/t6iIiIiEg/2BREh8GDB2PTpk344YcfUKdOHQDA8ePHMX78eDx//hzz58+XOUIiIiIDIfVdZI01FW5MrHVYvXo11q5di5YtW0plVatWhaurK7p168bEmoiISKJuCiJvFERyY1MQHZRKJTw8PLTKPT09YWpq+uEDIiIiMlTsvEgEgIm1ToMGDcLEiRORnJwslSUnJ2Py5MkYNGiQjJERERERkSFiUxAdIiMjsX//fpQqVQo+Pj4AgPPnzyMlJQVNmjRBx44dpXk3bdokV5hEREQGgKOCEAFMrHWys7NDp06dNMr4k+FERERZYFMQIgBMrLMkhEBYWBgcHBxgbm4udzhEREQGjjXWRADbWGdJCIGyZcvi3r17codCRESUjzCxpsKNiXUWjIyMUK5cOTx//lzuUIiIiAyf1BSEqHBjYq3D999/j1GjRuHixYtyh0JERGTg2BSECGAba5169eqFpKQk+Pj4wNTUVKut9YsXL2SKjIiIyMCw8yIRACbWOs2cOVPuEIiIiPIJ1lgTAUysdQoKCpI7BCIiIiLKR9jGOhu3bt3Cd999h27duuHJkycAgF27duHSpUsyR0ZERGRA2BSECAATa50OHTqEKlWq4OTJk9i0aRMSEhIApP/6YmhoqMzRERERGRI2BSECmFjrNHr0aEyaNAl79+6FqampVP7RRx/hxIkTMkZGRERkYFhjTQSAibVOFy5cwMcff6xV7ujoiGfPnskQERERkYFjjTUVckysdbCzs8PDhw+1yiMjI1GyZEkZIiIiIjJU/IEYIoCJtU5du3bF119/jUePHkGhUEClUuHYsWMYOXIkevXqJXd4REREhoNNQYgAMLHWacqUKfDy8oKrqysSEhJQsWJFNGzYEHXr1sV3330nd3hEREQGhJ0XiQCOY62TqakpFi1ahHHjxuHChQtISEiAr68vypUrJ3doREREhkVqCcLEmgo3JtaZqFQq/Pjjj9i2bRtSUlLQpEkThIaGav2kORERERFRRmwKksnkyZPxzTffwMrKCiVLlsSsWbMQEhIid1hEREQGjE1BiAAm1lp+/fVXzJs3D7t378aWLVvwxx9/YNWqVVCpVHKHRkREZJgERwUhAphYa/n777/RqlUr6XVAQAAUCgUePHggY1RERESGjDXWRAATay1v3ryBmZmZRpmJiQlSU1NlioiIiIiI8gN2XsxECIHevXtDqVRKZa9fv8bAgQNhaWkplW3atEmO8IiIiAzLq0dAalL6cxPL7OclKuCYWGcSFBSkVdazZ08ZIiEiIsoHolYDQgW4/g+wdpI7GiJZMbHOZNmyZXKHQERElD8IAZz7Nf159U/ljYXIALCNNREREb2bO0eBl9GAqTVQsYPc0RDJjok1ERERvZvIlel/q3QClFbyxkJkAJhYExERUe79Ewtc3pr+3LeXrKEQGQom1kRERJR7F34H3rwGHCsBJavLHQ2RQWBiTURERLmXsdMifxiGCAATayIiIsqtB1HAo78AY1Ogahe5oyEyGEysiYiIKHfUnRa92wIWxeSNhciAMLEmIiKinEv9B/jr9/Tnvhy7migjJtZERESUc5e3AclxgJ0b4OkvdzREBoWJNREREeWcutOi76eAEdMIoox4RRAREVHOPL8F3D0KKIyAat3ljobI4DCxJiIiopxRd1osGwDYlpI3FiIDxMSaiIiI3i7tDRC1Ov05Oy0SZYmJNREREb3djT1AwmPA0gEo30LuaIgMEhNrIiIiejt1p0WfrkARU3ljITJQTKyJiIgoe/EPgRu705/79pI3FiIDxsSaiIiIsnd+NSBUgFsdwKG83NEQGSwm1kRERKSbSgWc+3c0EHZaJMoWE2siIiLS7e4x4GU0YGoNVOogdzREBo2JNREREemm7rRYpRNgailvLEQGjok1ERERZe2fl8DlrenPq7PTItHbMLEmIiKirF3YAKQlA06VAZfqckdDZPCYWBMREVHWzq1I/+v7KaBQyBsLUT7AxJqIiIi0PYgCHl0AjJVA1UC5oyHKF5hYExERkTZ1p0XvNoBFMXljIconmFgTERGRppQk4MLv6c/ZaZEox5hYExERkaYr24DkeMDOHfBoKHc0RPkGE2siIiLSpG4G4vspYMRUgSineLUQERHRf57dTP+1RYURUK273NEQ5StMrImIiOg/kSvT/5YNAGxLyhsLUT7DxJqIiIjSpaUC59ekP2enRaJcY2JNRERE6W7sARIeA5YOQPkWckdDlO8wsSYiIqJ06k6LPt0AYxN5YyHKh5hYExERERD/IL3GGmAzEKJ3xMTaQISHh6NmzZqwtraGo6MjOnTogGvXrskdFhERFRZRqwGhAtzqAMXLyR0NUb7ExNpAHDp0CCEhIThx4gT27t2L1NRUNGvWDImJiXKHRkREBZ1KBUT+lv6ctdVE76yI3AFQuoiICI3Xy5cvh6OjI86ePYuGDfmrV0RE9B7dPQq8jAaUNkDF9nJHQ5RvMbE2UHFxcQCAYsWKZTk9OTkZycnJ0uv4+PgPEhcRERVA6k6LlTsBppbyxkKUj7EpiAFSqVQYNmwY6tWrh8qVK2c5T3h4OGxtbaWHq6vrB46SiIgKhH9eApe3pT9nMxCiPGFibYBCQkJw8eJFrF27Vuc8Y8aMQVxcnPSIiYn5gBESEVGB8dfvQFoy4FQZcPGVOxqifI1NQQzMoEGDsH37dhw+fBilSpXSOZ9SqYRSqfyAkRERUYEjxH/NQKr3AhQKeeMhyueYWBsIIQQGDx6MzZs34+DBg/D09JQ7JCIiKugeRgGPLwDGSqBKZ7mjIcr3mFgbiJCQEKxevRpbt26FtbU1Hj16BACwtbWFubm5zNEREVGBpK6t9m4LWGTdWZ6Ico5trA3E/PnzERcXh0aNGsHZ2Vl6rFu3Tu7QiIioIEpJAi5sSH/OTotEesEaawMhhJA7BCIiKkwubwWS4wE7d8CjgdzREBUIrLEmIiIqjCJXpv+t/ilgxHSASB94JRERERU2z24Cd48BCiOgWg+5oyEqMJhYExERFTaR/3ZaLNsUsHGRNxaiAoSJNRERUWGSlgpErUl/zk6LRHrFxJqIiKgwub4bSHwCWDoC5ZvLHQ1RgcLEmoiIqDBRd1qs1g0wNpE3FqIChok1ERFRYRH/ALixJ/25L5uBEOkbE2siIqLCImoVIFSAW12geFm5oyEqcJhYExERFQYqFRD5W/pzdlokei+YWBMRERUGd44AL+8AShugYnu5oyEqkJhYExERFQbqTotVPgFMLeSNhaiAYmJNRERU0CW9AC5vS3/OZiBE7w0TayIiooLuwu9AWjLgVAVwriZ3NEQFFhNrIiKigkwI4Ny/P2FevRegUMgbD1EBxsSaiIioIHsQCTy+CBgrgaqd5Y6GqEBjYk1ERFSQqTstVmwHmBeVNxaiAo6JNRERUUGVkghc2JD+nJ0Wid47JtZEREQF1eWtQHI8UNQDcK8vdzREBR4TayIiooLq3L/NQHw/BYz4kU/0vvEqIyIiKoie3QD+/j9AYQRU6y53NESFAhNrIiKigkjdabFcM8DGRd5YiAoJJtZEREQFTVoqELU6/Tk7LRJ9MEysiYiICprrEUDiU8DSMb3Gmog+CCbWREREBY2602K17oCxibyxEBUiTKyJiIgKkvgHwM296c99P5U3FqJChok1ERFRQRK1ChAqwL0eULys3NEQFSpMrImIiAoKleq/ZiDstEj0wTGxJiIiKijuHAZi7wJKG8C7ndzREBU6TKyJiIgKCnVtdZXOgKmFvLEQFUJMrImIiAqCpBfAlT/Sn1dnp0UiOTCxJiIiKggu/A6kJQMlqgDO1eSOhqhQYmJNRESU3wkBnF2R/rx6EKBQyBsPUSHFxJqIiCi/e3AOeHIJMFYCVT6ROxqiQouJNRERUX6n7rRYsT1gXlTeWIgKMSbWRERE+VlKInBhQ/pzdlokkhUTayIiovzs8lYg5RVQ1BNwry93NESFGhNrIiKi/Ozcr+l/q38KGPFjnUhOvAKJiIjyq6fXgb+PAwojwKe73NEQFXpMrImIiPKryH87LZZrDtg4yxsLETGxJiIiypfSUoHza9Kfs9MikUFgYk1ERJQfXY8AEp8CVk5AuWZyR0NEYGJNRESUP6k7LVbrDhibyBsLEQFgYk1ERJT/xN0Hbu5Lf+7LZiBEhoKJNRERUX4TtRoQqvRxq+3LyB0NEf2LiTUREVF+olIBkRnGriYig8HEmoiIKD+5cxiI/RtQ2gLe7eSOhogyYGJNRESUn6g7LVbtDJhayBsLEWlgYk1ERJRfJL0ArvyR/pydFokMDhNrIiKi/OKv9UBaClCiKuBSTe5oiCgTJtZERET5gRD/NQOp3kveWIgoS0ysiYiI8oMH54Anl4AiZkCVT+SOhoiywMSaiIgoP1DXVldsD5gXlTcWIsoSE2siIiJDl5IIXNiY/pydFokMFhNrIiIiQ3dpC5DyCihWGvCoL3c0RKQDE2siIiJDp24G4tsTUCjkjYWIdGJiTUREZMieXgdiTgAKY8Cnu9zREFE2mFgTEREZssh/a6vLNwdsnOWNhYiyxcSaiIjIUL1JAaLWpD9np0Uig8fEmoiIyFBdjwCSngFWJYByzeSOhojegok1ERGRoVJ3WqzWDTAuIm8sRPRWTKyJiIgMUdw94Nb+9OdsBkKULzCxJiIiMkRRqwGhAjwaAPZl5I6GiHKAibUBmTt3Ljw8PGBmZobatWvj1KlTcodERERyUKmAyJXpz1lbTZRvMLE2EOvWrcPw4cMRGhqKc+fOwcfHB82bN8eTJ0/kDo2IiD606ENA7N+A0hao2E7uaIgoh5hYG4gZM2agf//+CA4ORsWKFbFgwQJYWFhg6dKlcodGZNiESH8QFSTqTotVOwMm5vLGQkQ5xi7GBiAlJQVnz57FmDFjpDIjIyMEBATg+PHjMkYGPLz1F2L/vgxA9W8CAwBpEEJAIQSEEAAEINL+naaSyhVQSfNBqCAAKIQq03OR4b3VywAQKiggICAyzfffNPz7vv8WppdLSdZ/ZenvKzL8VWnMo/j3ufSewL+vIT0HoDFdgf8SueyWU2QoT5+m+DeujO+T8f0BKCDFpHv9/5VL25U53kzvoX5/Bf7bfmmdyLBvMsX933qz2lfq8v+O/X/7/r/tU0jHCRrzKDK9r3rfKDLGnVVMGePIQKUoAqEwglAYaz6MMr1WGEGo5zVK/wtpWhEIowzTFcaAkbHG/Ph3eSiMpXkhvVf6+iAtl74sMs33398M8yuMAXU8xhnWY2QMlYkFUpTFkKq0R5qxWfreEoD4d18LACohpMtUCKE+tOlXkgBU4r9y8e/E/+ZPn089DzKUqS8r9ToyvqfIPJ963ZnWpS6HVpz/bgMABRRQKAAjxX/PFQrFv68BI6P0KwhS2b9//51PAcBIoX4PhfTL3xpl+HfeDK+NjNLfCxnLFOr1Z1zPf+tD5jL8N7/2+tLnU+/TjNep+rXI8Fzx+gXKXtkOIwC33Trh9YP4f+cRWv9D/rd8Vu8nMqwp63VnnpbVMur3zvRHa902ZiYo72QNc1NjEBVWTKwNwLNnz5CWlgYnJyeNcicnJ1y9ejXLZZKTk5GcnCy9jouLAwDEx8frNbZLuxah1v0Ven1Pyr9Epr+GJzXXSyig/kcl/0gUpngpbPAC1ngh0h8vYYMXwurf5+ry9HlSYCp3yJQLPYz3YoxJMi6r3BC49D6A+3KHlGNGCsDd3gIVnKxRwdkaFUrYwNvJGsWtldI/F6RJ/bktMv/XRPkSE+t8Kjw8HGFhYVrlrq6uMkRDRB/ec7kDoPfk+38fwCUAgbLG8i7uAjgsdxD50KtXr2Brayt3GJRHTKwNQPHixWFsbIzHjx9rlD9+/BglSpTIcpkxY8Zg+PDh0muVSoUXL17A3t5e77UC8fHxcHV1RUxMDGxsbPT63oaA25f/FfRt5PblfwV9G7l9704IgVevXsHFxUWv70vyYGJtAExNTeHn54f9+/ejQ4cOANIT5f3792PQoEFZLqNUKqFUKjXK7Ozs3mucNjY2BfKGqcbty/8K+jZy+/K/gr6N3L53w5rqgoOJtYEYPnw4goKCUKNGDdSqVQszZ85EYmIigoOD5Q6NiIiIiHKAibWB6NKlC54+fYpx48bh0aNHqFatGiIiIrQ6NBIRERGRYWJibUAGDRqks+mHnJRKJUJDQ7WanhQU3L78r6BvI7cv/yvo28jtI0qnEBzfhYiIiIgoz/jLi0REREREesDEmoiIiIhID5hYExERERHpARNrIiIiIiI9YGJN2Zo7dy48PDxgZmaG2rVr49SpU3KHpDeHDx9G27Zt4eLiAoVCgS1btsgdkl6Fh4ejZs2asLa2hqOjIzp06IBr167JHZbezJ8/H1WrVpV+sKFOnTrYtWuX3GG9N99//z0UCgWGDRsmdyh6M378eCgUCo2Hl5eX3GHp1f3799GzZ0/Y29vD3NwcVapUwZkzZ+QOS288PDy0jqFCoUBISIjcoelFWloaxo4dC09PT5ibm6NMmTKYOHEiOO4D6cLEmnRat24dhg8fjtDQUJw7dw4+Pj5o3rw5njx5IndoepGYmAgfHx/MnTtX7lDei0OHDiEkJAQnTpzA3r17kZqaimbNmiExMVHu0PSiVKlS+P7773H27FmcOXMGH330Edq3b49Lly7JHZrenT59Gr/88guqVq0qdyh6V6lSJTx8+FB6HD16VO6Q9Obly5eoV68eTExMsGvXLly+fBnTp09H0aJF5Q5Nb06fPq1x/Pbu3QsA6Ny5s8yR6cfUqVMxf/58zJkzB1euXMHUqVPxww8/YPbs2XKHRgaKw+2RTrVr10bNmjUxZ84cAOk/s+7q6orBgwdj9OjRMkenXwqFAps3b5Z+Ur4gevr0KRwdHXHo0CE0bNhQ7nDei2LFiuHHH39E37595Q5FbxISElC9enXMmzcPkyZNQrVq1TBz5ky5w9KL8ePHY8uWLYiKipI7lPdi9OjROHbsGI4cOSJ3KB/MsGHDsH37dty4cQMKhULucPKsTZs2cHJywpIlS6SyTp06wdzcHL/99puMkZGhYo01ZSklJQVnz55FQECAVGZkZISAgAAcP35cxsjoXcXFxQFITz4LmrS0NKxduxaJiYmoU6eO3OHoVUhICFq3bq1xLRYkN27cgIuLC0qXLo0ePXrg77//ljskvdm2bRtq1KiBzp07w9HREb6+vli0aJHcYb03KSkp+O2339CnT58CkVQDQN26dbF//35cv34dAHD+/HkcPXoULVu2lDkyMlT85UXK0rNnz5CWlqb1k+pOTk64evWqTFHRu1KpVBg2bBjq1auHypUryx2O3ly4cAF16tTB69evYWVlhc2bN6NixYpyh6U3a9euxblz53D69Gm5Q3kvateujeXLl6NChQp4+PAhwsLC0KBBA1y8eBHW1tZyh5dnt2/fxvz58zF8+HB88803OH36NIYMGQJTU1MEBQXJHZ7ebdmyBbGxsejdu7fcoejN6NGjER8fDy8vLxgbGyMtLQ2TJ09Gjx495A6NDBQTa6JCICQkBBcvXixQ7VcBoEKFCoiKikJcXBw2bNiAoKAgHDp0qEAk1zExMRg6dCj27t0LMzMzucN5LzLW+lWtWhW1a9eGu7s71q9fXyCa86hUKtSoUQNTpkwBAPj6+uLixYtYsGBBgUyslyxZgpYtW8LFxUXuUPRm/fr1WLVqFVavXo1KlSohKioKw4YNg4uLS4E8hpR3TKwpS8WLF4exsTEeP36sUf748WOUKFFCpqjoXQwaNAjbt2/H4cOHUapUKbnD0StTU1OULVsWAODn54fTp09j1qxZ+OWXX2SOLO/Onj2LJ0+eoHr16lJZWloaDh8+jDlz5iA5ORnGxsYyRqh/dnZ2KF++PG7evCl3KHrh7Oys9U+et7c3Nm7cKFNE78/du3exb98+bNq0Se5Q9GrUqFEYPXo0unbtCgCoUqUK7t69i/DwcCbWlCW2saYsmZqaws/PD/v375fKVCoV9u/fX+DasBZUQggMGjQImzdvxp9//glPT0+5Q3rvVCoVkpOT5Q5DL5o0aYILFy4gKipKetSoUQM9evRAVFRUgUuqgfSOmrdu3YKzs7PcoehFvXr1tIa4vH79Otzd3WWK6P1ZtmwZHB0d0bp1a7lD0aukpCQYGWmmSsbGxlCpVDJFRIaONdak0/DhwxEUFIQaNWqgVq1amDlzJhITExEcHCx3aHqRkJCgUTMWHR2NqKgoFCtWDG5ubjJGph8hISFYvXo1tm7dCmtrazx69AgAYGtrC3Nzc5mjy7sxY8agZcuWcHNzw6tXr7B69WocPHgQu3fvljs0vbC2ttZqD29paQl7e/sC005+5MiRaNu2Ldzd3fHgwQOEhobC2NgY3bp1kzs0vfjyyy9Rt25dTJkyBYGBgTh16hQWLlyIhQsXyh2aXqlUKixbtgxBQUEoUqRgpRVt27bF5MmT4ebmhkqVKiEyMhIzZsxAnz595A6NDJUgysbs2bOFm5ubMDU1FbVq1RInTpyQOyS9OXDggACg9QgKCpI7NL3IatsAiGXLlskdml706dNHuLu7C1NTU+Hg4CCaNGki9uzZI3dY75W/v78YOnSo3GHoTZcuXYSzs7MwNTUVJUuWFF26dBE3b96UOyy9+uOPP0TlypWFUqkUXl5eYuHChXKHpHe7d+8WAMS1a9fkDkXv4uPjxdChQ4Wbm5swMzMTpUuXFt9++61ITk6WOzQyUBzHmoiIiIhID9jGmoiIiIhID5hYExERERHpARNrIiIiIiI9YGJNRERERKQHTKyJiIiIiPSAiTURERERkR4wsSYiIiIi0gMm1kREJLlz5w4UCgWioqLkDoWIKN9hYk1Ehd7x48dhbGyM1q1b53rZ8ePHo1q1avoPioiI8h0m1kRU6C1ZsgSDBw/G4cOH8eDBA7nDyfdSUlLkDoGISBZMrImoUEtISMC6devw+eefo3Xr1li+fLk0bfny5bCzs9OYf8uWLVAoFNL0sLAwnD9/HgqFAgqFQlr+77//Rvv27WFlZQUbGxsEBgbi8ePHGu+1detWVK9eHWZmZihdujTCwsLw5s0babpCocDixYvx8ccfw8LCAuXKlcO2bds03uPSpUto06YNbGxsYG1tjQYNGuDWrVsAAJVKhQkTJqBUqVJQKpWoVq0aIiIiNJY/deoUfH19YWZmhho1aiAyMlJrH128eBEtW7aElZUVnJyc8Omnn+LZs2fS9EaNGmHQoEEYNmwYihcvjubNm+ds5xMRFTBMrImoUFu/fj28vLxQoUIF9OzZE0uXLoUQIkfLdunSBSNGjEClSpXw8OFDPHz4EF26dIFKpUL79u3x4sULHDp0CHv37sXt27fRpUsXadkjR46gV69eGDp0KC5fvoxffvkFy5cvx+TJkzXWERYWhsDAQPz1119o1aoVevTogRcvXgAA7t+/j4YNG0KpVOLPP//E2bNn0adPHyk5nzVrFqZPn45p06bhr7/+QvPmzdGuXTvcuHEDQPo/FW3atEHFihVx9uxZjB8/HiNHjtRYf2xsLD766CP4+vrizJkziIiIwOPHjxEYGKgx34oVK2Bqaopjx45hwYIFuTsIREQFhSAiKsTq1q0rZs6cKYQQIjU1VRQvXlwcOHBACCHEsmXLhK2trcb8mzdvFhlvnaGhocLHx0djnj179ghjY2Px999/S2WXLl0SAMSpU6eEEEI0adJETJkyRWO5lStXCmdnZ+k1APHdd99JrxMSEgQAsWvXLiGEEGPGjBGenp4iJSUly21zcXERkydP1iirWbOm+OKLL4QQQvzyyy/C3t5e/PPPP9L0+fPnCwAiMjJSCCHExIkTRbNmzTTeIyYmRgAQ165dE0II4e/vL3x9fbOMgYioMGGNNREVWteuXcOpU6fQrVs3AECRIkXQpUsXLFmyJE/ve+XKFbi6usLV1VUqq1ixIuzs7HDlyhUAwPnz5zFhwgRYWVlJj/79++Phw4dISkqSlqtatar03NLSEjY2Nnjy5AkAICoqCg0aNICJiYlWDPHx8Xjw4AHq1aunUV6vXj0phitXrqBq1aowMzOTptepU0dj/vPnz+PAgQMacXp5eQGA1OQEAPz8/HK3k4iICqAicgdARCSXJUuW4M2bN3BxcZHKhBBQKpWYM2cOjIyMtJqFpKam6mXdCQkJCAsLQ8eOHbWmZUx0MyfNCoUCKpUKAGBubq6XWLKTkJCAtm3bYurUqVrTnJ2dpeeWlpbvPRYiIkPHxJqICqU3b97g119/xfTp09GsWTONaR06dMCaNWvg7u6OV69eITExUUocM4/vbGpqirS0NI0yb29vxMTEICYmRqq1vnz5MmJjY1GxYkUAQPXq1XHt2jWULVv2nbehatWqWLFiBVJTU7UScBsbG7i4uODYsWPw9/eXyo8dO4ZatWpJca5cuRKvX7+WkvkTJ05ovE/16tWxceNGeHh4oEgRfmQQEWWHTUGIqFDavn07Xr58ib59+6Jy5coaj06dOmHJkiWoXbs2LCws8M033+DWrVtYvXq1xqghAODh4YHo6GhERUXh2bNnSE5ORkBAAKpUqYIePXrg3LlzOHXqFHr16gV/f3/UqFEDADBu3Dj8+uuvCAsLw6VLl3DlyhWsXbsW3333XY63YdCgQYiPj0fXrl1x5swZ3LhxAytXrsS1a9cAAKNGjcLUqVOxbt06XLt2DaNHj0ZUVBSGDh0KAOjevTsUCgX69++Py5cvY+fOnZg2bZrGOkJCQvDixQt069YNp0+fxq1bt7B7924EBwdr/UNBRFTYMbEmokJpyZIlCAgIgK2trda0Tp064cyZM7h37x5+++037Ny5E1WqVMGaNWswfvx4rXlbtGiBxo0bw8HBAWvWrIFCocDWrVtRtGhRNGzYEAEBAShdujTWrVsnLde8eXNs374de/bsQc2aNfG///0PP/30E9zd3XO8Dfb29vjzzz+RkJAAf39/+Pn5YdGiRVLt9ZAhQzB8+HCMGDECVapUQUREBLZt24Zy5coBAKysrPDHH3/gwoUL8PX1xbfffqvV5ENd652WloZmzZqhSpUqGDZsGOzs7GBkxI8QIqKMFCJzA0IiIiIiIso1VjcQEREREekBE2siIiIiIj1gYk1EREREpAdMrImIiIiI9ICJNRERERGRHjCxJiIiIiLSAybWRERERER6wMSaiIiIiEgPmFgTEREREekBE2siIiIiIj1gYk1EREREpAdMrImIiIiI9OD/AfIn7pitfBmCAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plot all_model_perplexities - all_model_perplexities2\n",
    "import matplotlib.pyplot as plt\n",
    "model_perplexity_diff = np.array(all_model_perplexities) - np.array(all_model_perplexities2)\n",
    "plt.plot(model_perplexity_diff.T)\n",
    "plt.legend(models)\n",
    "plt.title(\"Perplexity difference between reconstruction-top-2-outliers and reconstructed model\")\n",
    "plt.xlabel(\"Autoencoder\")\n",
    "plt.ylabel(\"Perplexity difference\")\n",
    "# set ylimit to 10\n",
    "plt.ylim(0, 10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAHHCAYAAACyWSKnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACNxElEQVR4nOzdd3gU1dfA8e+m9wRCCgESQpFA6EWlIyBFQBAE9EUBRUUNIKKo2AAVEESwUcSfAlJUUIqgoIB06QKCQGihptGSkEDq3vePSRaWFJKQZLKb83mefebuzOzsmd3N7sncZlBKKYQQQgghrIiN3gEIIYQQQhQ1SXCEEEIIYXUkwRFCCCGE1ZEERwghhBBWRxIcIYQQQlgdSXCEEEIIYXUkwRFCCCGE1ZEERwghhBBWRxIcIYQQQlgdSXBKwKZNmzAYDGzatKnYnqNdu3a0a9eu2I5fGHv27KFFixa4urpiMBg4cOCA3iFZhcGDB1O1alW9wzCZN28eBoOBvXv33nXf0vA5PXPmDAaDgXnz5uV736lTpxZ/YPeoOF/bgnyHZe37888/F0ssQl9Zf+9nzpwxrSsNf9c5sboEJ+vFz7o5OTlx3333MWzYMGJiYvQOr8RERkYybtw43ZKKtLQ0+vbty9WrV5k+fToLFiwgKCgoz8fExsby1ltvUa9ePdzc3HBycqJGjRo888wzbNu2zWzf/LzPVatWNdsnt9u8efO4cuUKn3zyCW3atMHHxwcvLy8efPBBfvrpp2J7jYpbu3btzM6zfPnyNGvWjO+++w6j0Vjg482cOTNfSYEl+v333xk3bpzeYViUxYsX89lnn+kdhpkbN24wbty4fP8zeeTIEcaNG2f2Y13Sjh07xhtvvEHDhg1xd3enYsWKdOvWLV//MBSniRMnsmLFCl1juFd2egdQXD744AOCg4NJTk5m27ZtzJo1i99//53Dhw/j4uKid3hF7s8//zS7HxkZyfjx46latSoNGzYs8XhOnTrF2bNn+eabb3juuefuuv/u3bvp1q0b169f54knnuDFF1/E0dGRiIgIVqxYwbx589i8eTNt2rQxe1xe7/Nnn31GYmKiad/ff/+dH374genTp1OhQgXT+hYtWrBjxw7eeecdHnnkEd59913s7Oz45ZdfeOKJJzhy5Ajjx48vuhenBFWuXJlJkyYBcOnSJb7//nuGDBnC8ePH+fjjjwt0rJkzZ1KhQgUGDx5c6Hju/JzqISgoiJs3b2Jvb29a9/vvvzNjxgxJcnLRpk0bbt68iYODg2nd4sWLOXz4MCNHjtQvsDvcuHHD9LeanysKWX/b7dq10+2q6P/+9z++/fZb+vTpw8svv0x8fDxff/01Dz74IGvXrqVjx466xDVx4kQef/xxevXqddd9S8PfdU6sNsHp2rUrTZs2BeC5557D29ubadOmsXLlSp588sl7OvaNGzdKXZJ0+xdPaRAbGwuAl5fXXfe9du0avXr1ws7OjgMHDhASEmK2/aOPPuLHH3/E2dk522ML8j5HR0fzww8/0KtXr2xfZgaDgRMnTphdZXr55Zfp2LEjkydP5o033sDV1TU/p16qeHp68tRTT5nuDx06lFq1avHVV1/x4Ycfmv3Il4TS8DnNuuIn8s/GxkZes2Ly5JNPMm7cONzc3Ezrnn32WWrXrs24ceN0S3AKoij/ro1GI6mpqUXyebO6KqrctG/fHoCIiAjTuoULF9KkSROcnZ0pX748TzzxBOfPnzd7XLt27ahbty779u2jTZs2uLi48PbbbwNaFUj37t35888/adiwIU5OTtSpU4dly5blK6Zdu3bRpUsXPD09cXFxoW3btmzfvt20/ejRozg7OzNw4ECzx23btg1bW1vefPNNsziz/mPZtGkTzZo1A+CZZ54xq4oZO3Ys9vb2XLp0KVs8L7zwAl5eXiQnJ+cZ919//UXr1q1xdXXFy8uLnj17cvToUdP2wYMH07ZtWwD69u2LwWDI87+p2bNnExUVxWeffZYtuQHtB+nJJ580nVNecnqf8yM4ODhbFZrBYKBXr16kpKRw+vTpPB+fmprK+++/T5MmTfD09MTV1ZXWrVuzceNGs/1ub9MxZ84cqlevjqOjI82aNWPPnj3ZjrtixQrq1q2Lk5MTdevWZfny5QU6rzu5uLjw4IMPkpSUxKVLl/L9eahatSr//fcfmzdvNn2e7nxPU1JSGDVqFD4+Pri6uvLYY49lO+6ddfX5fd1yMmrUKLy9vVFKmdYNHz4cg8HAF198YVoXExODwWBg1qxZQPY2OIMHD2bGjBkAZlV6d8rP+3Wnq1ev8vrrr5uqXT08POjatSsHDx402y+r3cqSJUuYMGEClStXxsnJiQ4dOnDy5MlcY3F2dub+++9n69atd40FoHfv3jRu3NhsXY8ePTAYDPz666+mdbt27cJgMLBmzRqz+LKqftq1a8dvv/3G2bNnTa/Xnf80GI3GfJ3L0qVLTd/DFSpU4KmnnuLixYtm++TWxuP29mhnzpzBx8cHgPHjx5viyu2q3Lx58+jbty8ADz30kGn/26u3Zs6cSWhoKI6OjgQEBBAWFkZcXFy22LJ+I1q0aIGzszPBwcHMnj07x+e9U5MmTcySGwBvb29at25t9r16N3f7Xobc2++NGzfO7DNvMBhISkpi/vz5ptclryu3Ob0/KSkpjB07lho1auDo6EiVKlV44403SElJMdvPYDAwbNgwFi1aZHqt165dC8CPP/5IkyZNcHd3x8PDg3r16vH555/n+zUpMwnOqVOnAO2DAzBhwgQGDhxIzZo1mTZtGiNHjmTDhg20adMm2wf4ypUrdO3alYYNG/LZZ5/x0EMPmbadOHGC/v3707VrVyZNmoSdnR19+/Zl3bp1ecbz119/0aZNGxISEhg7diwTJ04kLi6O9u3bs3v3bgBq167Nhx9+yIIFC0xfPklJSQwePJiQkBA++OCDHI9du3Zt07YXXniBBQsWsGDBAtq0acPTTz9Nenp6trYlqamp/Pzzz/Tp0yfPzHn9+vV07tyZ2NhYxo0bx6hRo/j7779p2bKlqR576NChpiRwxIgRLFiwgHfeeSfXY65atQpnZ2d69+6d52uWH3e+z/cqOjoawKxKKycJCQn873//o127dkyePJlx48Zx6dIlOnfunGM7qMWLF/PJJ58wdOhQPvroI86cOUPv3r1JS0sz7fPnn3/Sp08fDAYDkyZNolevXjzzzDP3XDd/+vRpbG1t8fLyyvfn4bPPPqNy5cqEhISYPk93vqfDhw/n4MGDjB07lpdeeolVq1YxbNiwPGMp6Ot2u9atW3P16lX+++8/07qtW7diY2Nj9oOfVb6zejPL0KFDefjhhwFM57ZgwQKzffLzfuXk9OnTrFixgu7duzNt2jRGjx7NoUOHaNu2LZGRkdn2//jjj1m+fDmvv/46Y8aMYefOnQwYMMBsn2+//ZahQ4fi7+/PlClTaNmyJY8++mi2f85y0rp1aw4ePEhCQgIASim2b9+e42tmY2NDy5YtczzOO++8Q8OGDalQoYLp9bqzPU5+zmXevHn069cPW1tbJk2axPPPP8+yZcto1apVtu/hu/Hx8TElsY899pgprty+V9q0acOIESMAePvtt037165dG9B+9MPCwggICODTTz+lT58+fP3113Tq1Cnb+37t2jUeeeQRmjRpwpQpU6hcuTIvvfQS3333XYHO4XbR0dF3/d7Jkp/v5YJYsGABjo6OtG7d2vS6DB06NN+PNxqNPProo0ydOpUePXrw5Zdf0qtXL6ZPn07//v2z7f/XX3/x6quv0r9/fz7//HOqVq3KunXrePLJJylXrhyTJ0/m448/pl27dmYXAe5KWZm5c+cqQK1fv15dunRJnT9/Xv3444/K29tbOTs7qwsXLqgzZ84oW1tbNWHCBLPHHjp0SNnZ2Zmtb9u2rQLU7Nmzsz1XUFCQAtQvv/xiWhcfH68qVqyoGjVqZFq3ceNGBaiNGzcqpZQyGo2qZs2aqnPnzspoNJr2u3HjhgoODlYPP/ywaV1GRoZq1aqV8vPzU5cvX1ZhYWHKzs5O7dmzxyyWtm3bqrZt25ru79mzRwFq7ty52eJu3ry5euCBB8zWLVu2zCzG3DRs2FD5+vqqK1eumNYdPHhQ2djYqIEDB2Y756VLl+Z5PKWUKleunGrYsGG29QkJCerSpUumW2Jiomlbft7nO33yyScKUBEREXeNSSmlrly5onx9fVXr1q3vum96erpKSUkxW3ft2jXl5+ennn32WdO6iIgIBShvb2919epV0/qVK1cqQK1atcq0rmHDhqpixYoqLi7OtO7PP/9UgAoKCrprTG3btlUhISGm1+/o0aNqxIgRClA9evQw7Zffz0NoaKjZZyxL1nvRsWNHs8/zq6++qmxtbc3iv/Nzmt/XLSexsbEKUDNnzlRKKRUXF6dsbGxU3759lZ+fn2m/ESNGqPLly5tiy3oPbv/bCAsLUzl9HRbk/cpJcnKyysjIyHZMR0dH9cEHH5jWZf291K5d2+z1+PzzzxWgDh06pJRSKjU1Vfn6+qqGDRua7TdnzhwF5Pj+3C7re+H3339XSin177//KkD17dvX7DPw6KOP5vkdppRS3bp1y/FzWNBzqVu3rrp586Zpv9WrVytAvf/++6Z1d35usgwaNMgshkuXLilAjR07Ns/XIcvSpUtz/N6LjY1VDg4OqlOnTmbv31dffaUA9d1335nFBqhPP/3UtC4lJcX0XZmampqvWG63ZcsWZTAY1HvvvZev/fP7vXzn65Vl7Nix2T7/rq6uatCgQdn2zfp7v/179M73Z8GCBcrGxkZt3brV7LGzZ89WgNq+fbtpHaBsbGzUf//9Z7bvK6+8ojw8PFR6enpep54nq72C07FjR3x8fKhSpQpPPPEEbm5uLF++nEqVKrFs2TKMRiP9+vXj8uXLppu/vz81a9bMdnnc0dGRZ555JsfnCQgI4LHHHjPd9/DwYODAgezfv9/03/+dDhw4wIkTJ/i///s/rly5Ynr+pKQkOnTowJYtW0y9XGxsbJg3bx6JiYl07dqVmTNnMmbMGFO7k8IYOHAgu3btMl3tAFi0aBFVqlQxVS3lJCoqigMHDjB48GDKly9vWl+/fn0efvhhfv/990LFk5CQkO0SLcDTTz+Nj4+P6XZ7lVyWvN7ne2E0GhkwYABxcXF8+eWXd93f1tbWVA9tNBq5evUq6enpNG3alH/++Sfb/v3796dcuXKm+61btwYwVYVlvdaDBg3C09PTtN/DDz9MnTp18n0ex44dM71+tWvX5ssvv6Rbt25m/1kW9vNwpxdeeMHsMnfr1q3JyMjg7NmzuT6moK/b7Xx8fAgJCWHLli0AbN++HVtbW0aPHk1MTAwnTpwAtKsRrVq1yrHaKb/u9n7lxtHRERsb7Ws2IyODK1eu4ObmRq1atXI8v2eeecasPcOdz7N3715iY2N58cUXzfYbPHiw2eckN40aNcLNzc30mm3dupXKlSszcOBA/vnnH27cuIFSim3btpmeu7Dyey4vv/yy2VXjbt26ERISwm+//XZPz38v1q9fT2pqKiNHjjS9fwDPP/88Hh4e2WKzs7Mzu8Lh4ODA0KFDiY2NZd++fQV67tjYWP7v//6P4OBg3njjjbvuX1zfy/di6dKl1K5dm5CQELPf2KwmBHf+xrZt2zbb95qXlxdJSUl3rQ3Ji9UmODNmzGDdunVs3LiRI0eOcPr0aTp37gxo1UpKKWrWrGn2A+rj48PRo0dNDWSzVKpUKddGVDVq1Mj2xXnfffcB5HppMOuLd9CgQdme/3//+x8pKSnEx8eb9q9evTrjxo1jz549hIaG8t577xXqNcnSv39/HB0dWbRoEQDx8fGsXr2aAQMG5PkjkPVDVatWrWzbateubUrSCsrd3d2st1OWDz74gHXr1uX5Ac/rfb4Xw4cPZ+3atfzvf/+jQYMG+XrM/PnzqV+/Pk5OTnh7e+Pj48Nvv/1m9l5mCQwMNLuf9eN57do14NZrXbNmzWyPzen1z03Wpd7169ezbds2oqOjWb16tdml78J+Hgp6TrkpyOt2p9atW5uqVrZu3UrTpk1p2rQp5cuXZ+vWrSQkJHDw4MF7/rEu7LkZjUamT59OzZo1cXR0pEKFCvj4+PDvv/8W6efC3t6eatWq3fU8bG1tad68udlr1rp1a1q1akVGRgY7d+7kyJEjXL16tdhfs7y+T0JCQvJMjItbbrE5ODhQrVq1bLEFBARk64Rw5+9AdHS02e3mzZvZnjcpKYnu3btz/fp1Vq5cafaPX2Jiotnjs9q3Fdf38r04ceIE//33X7bft6zX5M7f2ODg4GzHePnll7nvvvvo2rUrlStX5tlnnzW1zckvq+1Fdf/99+d6lcNoNJoa0Nna2mbbfufVhJx679yLrKszn3zySa5duO+MIasbXmRkJFeuXMHf37/Qz1+uXDm6d+/OokWLeP/99/n5559JSUkx621TkkJCQjh48CBpaWlmvXrq169/18fm9T4X1vjx45k5cyYff/wxTz/9dL4es3DhQgYPHkyvXr0YPXo0vr6+pnYFt18ZyZLT5w4wazBbFFxdXe/aC6OoPg+FOaeCvm53atWqFd988w2nT582/VgbDAZatWrF1q1bCQgIwGg03vOPdWHfr4kTJ/Lee+/x7LPP8uGHH1K+fHlsbGwYOXJkjmMRlcTnolWrVkyYMIHk5GS2bt3KO++8g5eXF3Xr1mXr1q34+fkB6Paa5cRgMOT4uIyMjAIfSy8VK1Y0uz937lyzhrupqan07t2bf//9lz/++IO6deua7T916lSz4SqCgoIK3L4mt39Yivp1NBqN1KtXj2nTpuW4vUqVKmb3c/qN9fX15cCBA/zxxx+sWbOGNWvWMHfuXAYOHMj8+fPzFYfVJjh5qV69OkopgoODTRllYZ08eRKllNkH5/jx4wC5jqtQvXp1QKvOyk8XwNmzZ7Nu3TomTJjApEmTGDp0KCtXrszzMXf7z3vgwIH07NmTPXv2sGjRIho1akRoaGiej8nqZRQeHp5t27Fjx6hQoUKhulJ3796dnTt3snz5cvr161fgxxelrLFQRo4cmWOVWG5+/vlnqlWrxrJly8xe+7FjxxYqjqzXOutq3+1yev3vVX4+D/dSxZObe33dsn6E161bx549e3jrrbcArQHprFmzTP9ZN2nSJM/jFMe5gXZ+Dz30EN9++63Z+ri4uHw3IL3d7Z+LrMv9oA2sGRERka+rja1btyY1NZUffviBixcvml7DNm3amBKc++67z5To5OZeX7Pbv09uP5esdbf3aixXrlyO1YF3XkkpaEy57X97bLdfGUtNTSUiIiLb93ZkZCRJSUlm3393/g7ceSX69r8vo9HIwIED2bBhA0uWLMmxanjgwIG0atXKdD8rKSjI93K5cuVybLyd09Wye3l/q1evzsGDB+nQocM9HcfBwYEePXrQo0cPjEYjL7/8Ml9//TXvvfceNWrUuOvjrbaKKi+9e/fG1taW8ePHZ/uvQCnFlStX8n2syMhIs667CQkJfP/99zRs2DDXqyxNmjShevXqTJ06Nceqmdu71kZERDB69Gj69OnD22+/zdSpU/n111/5/vvv84wr6wOdW0+Erl27UqFCBSZPnszmzZvz9d96xYoVadiwIfPnzzc77uHDh/nzzz955JFH7nqMnLz00kv4+fnx6quvmr4UblfUVzVy89NPPzFixAgGDBiQ638eucn6b/X2WHft2sWOHTsKFcvtr/XtVRnr1q3jyJEjhTpmXvLzeXB1dS1wz5a7udfXLTg4mEqVKjF9+nTS0tJMvX5at27NqVOn+Pnnn3nwwQexs8v7f7m7/b0Ulq2tbbbP79KlS7N1g86vpk2b4uPjw+zZs0lNTTWtnzdvXr5jf+CBB7C3t2fy5MmUL1/e9EPbunVrdu7cyebNm/N19cbV1TVf1Yi5adq0Kb6+vsyePdus6/CaNWs4evQo3bp1M62rXr06x44dM/tuPHjwYLYeNVnjk+X3tcjtfe/YsSMODg588cUXZu/ft99+S3x8vFlsAOnp6Xz99dem+6mpqXz99df4+PiYkuuOHTua3W6/ojN8+HB++uknZs6cmWuvr2rVqpk9PuuzXpDv5erVqxMfH8+///5rWhcVFZXj8BP38vfer18/Ll68yDfffJNt282bN/NVZXbn77CNjY3pqv6dXc1zU2av4Hz00UeMGTOGM2fO0KtXL9zd3YmIiGD58uW88MILvP766/k61n333ceQIUPYs2cPfn5+fPfdd8TExDB37txcH2NjY8P//vc/unbtSmhoKM888wyVKlXi4sWLbNy4EQ8PD1atWoVSimeffRZnZ2dT98ehQ4fyyy+/8Morr9CxY0cCAgJyPUcvLy9mz56Nu7s7rq6uPPDAA6a6Tnt7e5544gm++uorbG1t8z344SeffELXrl1p3rw5Q4YM4ebNm3z55Zd4enoWehTY8uXLs3z5cnr06EGDBg144oknaNasGfb29pw/f56lS5cC2ev0i9Lu3bsZOHAg3t7edOjQwdQeJUuLFi3ybOPQvXt3li1bxmOPPUa3bt2IiIhg9uzZ1KlTJ8ckNj8mTZpEt27daNWqFc8++yxXr17lyy+/JDQ0tNDHzE1+Pg9NmjRh1qxZfPTRR9SoUQNfX99s/3kXVFG8bq1bt+bHH3+kXr16pnYejRs3xtXVlePHj/N///d/dz1G1o/QiBEj6Ny5M7a2tjzxxBOFP7FM3bt354MPPuCZZ56hRYsWHDp0iEWLFuWrvUxO7O3t+eijjxg6dCjt27enf//+REREMHfu3Hwf08XFhSZNmrBz507TGDigXcFJSkoiKSkpXwlOkyZN+Omnnxg1ahTNmjXDzc2NHj16FOhcJk+ezDPPPEPbtm158skniYmJMXUTfvXVV037Pvvss0ybNo3OnTszZMgQYmNjmT17NqGhoaYu76Bd1ahTpw4//fQT9913H+XLl6du3brZqnuyNGzYEFtbWyZPnkx8fDyOjo60b98eX19fxowZw/jx4+nSpQuPPvoo4eHhzJw5k2bNmmX7ByAgIIDJkydz5swZ7rvvPn766ScOHDjAnDlz7jqY5meffcbMmTNp3rw5Li4uLFy40Gz7Y489dtcr4/n9Xn7iiSd48803eeyxxxgxYgQ3btxg1qxZ3HfffdkavTdp0oT169czbdo0AgICCA4O5oEHHsgzjixPP/00S5Ys4cUXX2Tjxo20bNmSjIwMjh07xpIlS/jjjz/u2rTgueee4+rVq7Rv357KlStz9uxZvvzySxo2bGjqyn9Xhe5/VUpldWG7sxt1Tn755RfVqlUr5erqqlxdXVVISIgKCwtT4eHhpn3atm2rQkNDc3x8UFCQ6tatm/rjjz9U/fr1laOjowoJCcnWNTqnLpZKKbV//37Vu3dv5e3trRwdHVVQUJDq16+f2rBhg1LqVrfK27uhK6XUuXPnlIeHh3rkkUfM4ryzG+XKlStVnTp1lJ2dXY5dxnfv3q0A1alTp7u+Vrdbv369atmypXJ2dlYeHh6qR48e6siRIzmec366iWeJiopSo0ePVnXq1FHOzs7K0dFRVatWTQ0cOFBt2bLFbN+CvM9Z8uomnnW83G45dbe/ndFoVBMnTlRBQUHK0dFRNWrUSK1evTpbt8ysbseffPJJtmOQQ/fWX375RdWuXVs5OjqqOnXqqGXLluXa1fNOeX12c3K3z0N0dLTq1q2bcnd3N+uSnNt7kdPn/s7PaX5ft7zMmDFDAeqll14yW9+xY0cFmP6esuTUTTw9PV0NHz5c+fj4KIPBYOoyW9D3607JycnqtddeUxUrVlTOzs6qZcuWaseOHdleh9z+XnKKVSmlZs6cqYKDg5Wjo6Nq2rSp2rJlS65dqXMyevRoBajJkyebra9Ro4YC1KlTp8zW5/ReJiYmqv/7v/9TXl5eZkMXFPRcfvrpJ9WoUSPl6OioypcvrwYMGJDjMA8LFy5U1apVUw4ODqphw4bqjz/+yPFz8vfff6smTZooBweHfL1H33zzjapWrZqytbXNdo5fffWVCgkJUfb29srPz0+99NJL6tq1a2aPz/o727t3r2revLlycnJSQUFB6quvvsrzebMMGjQoz++e/A5rkZ/vZaW0oSbq1q2rHBwcVK1atdTChQtz7CZ+7Ngx1aZNG+Xs7KwAU5fx/HQTV0obBmDy5MkqNDRUOTo6qnLlyqkmTZqo8ePHq/j4eNN+gAoLC8sW588//6w6deqkfH19lYODgwoMDFRDhw5VUVFR+Xo9lFLKkPkEohCqVq1K3bp1Wb16td6hFMrBgwdp2LAh33//fb4b0wrrJZ8HIQquXbt2XL58mcOHD+sdirhDmWyDIzTffPMNbm5uRTKCsLB88nkQQliTMtkGp6xbtWoVR44cYc6cOQwbNswiJ5EURUc+D0IIayQJThk0fPhwYmJieOSRR8zGVRBlk3wehBDWSNc2OBkZGYwbN46FCxcSHR1NQEAAgwcP5t133zW17FdKMXbsWL755hvi4uJo2bIls2bNynGEVyGEEEII0LkNzuTJk5k1axZfffUVR48eZfLkyUyZMsVs7p8pU6bwxRdfMHv2bHbt2oWrqyudO3cmOTlZx8iFEEIIUZrpegWne/fu+Pn5mY3y2adPH5ydnVm4cCFKKQICAnjttddM49LEx8fj5+fHvHnzimScCiGEEEJYH13b4LRo0YI5c+Zw/Phx7rvvPg4ePMi2bdtMo8hGREQQHR1tNiy2p6cnDzzwADt27MgxwUlJSTEb5TBrhmJvb+9iG45dCCGEEEVLKcX169cJCAgwm9U9v3RNcN566y0SEhIICQnB1taWjIwMJkyYwIABAwBt9lUg25wofn5+pm13mjRpkjSUFEIIIazE+fPnqVy5coEfp2uCs2TJEhYtWsTixYsJDQ3lwIEDjBw5koCAAAYNGlSoY44ZM4ZRo0aZ7sfHxxMYGMj58+fx8PAoqtCFEMKqNGsGx4/DF1/Ak0+Cg4PeEYmyLiEhgSpVquDu7l6ox+ua4IwePZq33nrLVNVUr149zp49y6RJkxg0aJBpssqYmBizicliYmJo2LBhjsd0dHTE0dEx23oPDw9JcIQQIgeXL2vJDcD//R94e+sbjxC3K2zzEl17Ud24cSNbvZqtrS1GoxHQZgr29/dnw4YNpu0JCQns2rWL5s2bl2isQghhrbZt05Z16khyI6yHrldwevTowYQJEwgMDCQ0NJT9+/czbdo0nn32WUDL2kaOHMlHH31EzZo1CQ4O5r333iMgIIBevXrpGboQQliNrVu1ZWIinD0LQUH6xiNEUdA1wfnyyy957733ePnll4mNjSUgIIChQ4fy/vvvm/Z54403SEpK4oUXXiAuLo5WrVqxdu1anJycdIxcCCGsx/r12vLCBbmCI6yH1c8mnpCQgKenJ/Hx8Xm2wcnIyCAtLa0EIxNCP/b29tja2uodhigFEhPBwwOUgoYNYf9+vSMSQpPf3+/clPm5qJRSREdHExcXp3coQpQoLy8v/P39ZXyoMm7nTi25AbhtyDEhLF6ZT3CykhtfX19cXFzky15YPaUUN27cIDY2FsCsh6Ioe7La3wC0aaNfHEIUtTKd4GRkZJiSG2+peBZliLOzMwCxsbH4+vpKdVUZltVJ1WCAVq30jUWIoqRrN3G9ZbW5cXFx0TkSIUpe1ude2p6VXampsGePVq5ZE8qV0zceIYpSmU5wski1lCiL5HMv/vlHS3Ls7ODRR/WORoiiVaarqIQQoizLan/TrRt88om+sQgLFf0X3IwCv4fAJUDvaMzIFRwhisGZM2cwGAwcOHAAgE2bNmEwGKS3nihVshKc1q31jUNYsBMzYMdTEPG93pFkIwmOBYuOjuaVV16hRo0aODk54efnR8uWLZk1axY3btwAoGrVqhgMBgwGA66urjRu3JilS5dm25bTbfDgwZw5c4YhQ4YQHByMs7Mz1atXZ+zYsaSmpuYZ27hx43KdL6w4TJo0iWbNmuHu7o6vry+9evUiPDy8RJ578ODBdx1Zu0WLFkRFReHp6VkiMQlxN0YjbNmilSXBEYWiFFzKnOfDt/R9iKSKykKdPn2ali1b4uXlxcSJE6lXrx6Ojo4cOnSIOXPmUKlSJR7NrFT/4IMPeP7550lISODTTz+lf//+VKpUiT179pCRkQHA33//TZ8+fQgPDzcNqOTs7MyOHTswGo18/fXX1KhRg8OHD/P888+TlJTE1KlTdTv/O23evJmwsDCaNWtGeno6b7/9Np06deLIkSO4urrqHR4ODg6myWMLKzU1FQeZ4lkUkSNHID5eK69cCfffr288wgJdPwHJsYAtOFbQO5rslJWLj49XgIqPj8+27ebNm+rIkSPq5s2bOkR2bzp37qwqV66sEhMTc9xuNBqVUkoFBQWp6dOnm9anpaUpFxcX9dZbb5ntv3HjRgWoa9eu3fW5p0yZooKDg3PdPnfuXAWY3ebOnauUUurs2bPq0UcfVa6ursrd3V317dtXRUdHmx47duxY1aBBAzV79mxVuXJl5ezsrPr27avi4uLuGtftYmNjFaA2b96c535Xr15VTz/9tPLy8lLOzs6qS5cu6vjx49niud306dNVUFCQafud57px40YVERGhALV//36lVM6v79atW1WrVq2Uk5OTqly5sho+fLjZ+xkUFKQ++OAD9fTTTyt3d3c1aNAglZKSosLCwpS/v79ydHRUgYGBauLEiQV6bbJY8udf3LuZM5XS/gVXasECvaMRFunk/5RahFI/OCl17d8iP3xev9/5IVVUt1MK0pP0uRVgxowrV67w559/EhYWluvVidx6yNjZ2WFvb3/XKqa8xMfHU758+Vy39+/fn9dee43Q0FCioqKIioqif//+GI1GevbsydWrV9m8eTPr1q3j9OnT9O/f3+zxJ0+eZMmSJaxatYq1a9eyf/9+Xn755QLHCOQZJ2jVS3v37uXXX39lx44dKKV45JFH8t11+vXXX6dfv3506dLFdK4tWrS46+NOnTpFly5d6NOnD//++y8//fQT27ZtY9iwYWb7TZ06lQYNGrB//37ee+89vvjiC3799VeWLFlCeHg4ixYtomrVqvmKVYjb/fXXrbIM8CcKJat6qvYo8Kqnbyw5kCqq22XcgCVu+jx3v0Swy19VysmTJ1FKUatWLbP1FSpUIDk5GYCwsDAmT55stj01NZVPP/2U+Ph42rdvX6gwT548yZdffpln9ZSzszNubm7Y2dmZVcusW7eOQ4cOERERQZUqVQD4/vvvCQ0NZc+ePTRr1gyA5ORkvv/+eypVqgRok7J269aNTz/9NF/VPEajkZEjR9KyZUvq1q2b634nTpzg119/Zfv27aakZNGiRVSpUoUVK1bQt2/fuz6Xm5sbzs7OpKSkFKgKatKkSQwYMICRI0cCULNmTb744gvatm3LrFmzTJPJtm/fntdee830uHPnzlGzZk1atWqFwWAgSKZ9FoWgFGzapJX9/CAwUNdwhKWKzWyl7lM6R4iUKzhWZPfu3Rw4cIDQ0FBSUlJM6998803c3NxwcXFh8uTJfPzxx3Tr1q3Ax7948SJdunShb9++PP/886b1bm5uptuLL76Y6+OPHj1KlSpVTMkNQJ06dfDy8uLo0aOmdYGBgabkBqB58+YYjUbCw8PZunWr2fMtWrQo2/OEhYVx+PBhfvzxR9O6F1980exxWfHY2dnxwAMPmPbz9vamVq1aZvEUh4MHDzJv3jyzmDp37ozRaCQiIsK0X9OmTc0eN3jwYA4cOECtWrUYMWIEf/75Z7HGKazT2bNw+bJWLuT/OqKsuxkFiae0coW7X7XWg1zBuZ2ti3YlRa/nzqcaNWpgMBiy9RKqVq0acGsY/iyjR49m8ODBuLm54efnV6gB3iIjI3nooYdo0aIFc+bMMduW1RUaKNSMrwXRtGlTs+fz8/Mz2z5s2DBWr17Nli1bqFy5smn9Bx98wOuvv17g57OxsUHdUX1YFCP/JiYmMnToUEaMGJFtW+Bt/07fWQXZuHFjIiIiWLNmDevXr6dfv3507NiRn3/++Z5jEmXH7fNPSYIjCiU2swseNlqy41D6eohKgnM7gyHf1UR68vb25uGHH+arr75i+PDhd+0lVKFCBWrUqFHo57t48SIPPfQQTZo0Ye7cudjYmF/4y+nYDg4Oph5aWWrXrs358+c5f/686SrOkSNHiIuLo06dOqb9zp07R2RkJAEB2qBRO3fuxMbGhlq1auHs7Jzj8ymlGD58OMuXL2fTpk0EBwebbff19cXX1zdbPOnp6ezatctURXXlyhXCw8NN8fj4+BAdHY1SypQY3p5g5Xaud9O4cWOOHDlSqPfFw8OD/v37079/fx5//HG6dOnC1atX79reSIgsGzfeKrdtq18cwoJd+FVbGmzArZq+seRCqqgs1MyZM0lPT6dp06b89NNPHD16lPDwcBYuXMixY8eKbPLEixcv0q5dOwIDA5k6dSqXLl0iOjqa6OjoPB9XtWpVIiIiOHDgAJcvXyYlJYWOHTtSr149BgwYwD///MPu3bsZOHAgbdu2NauKcXJyYtCgQRw8eJCtW7cyYsQI+vXrl2cbl7CwMBYuXMjixYtxd3c3xXjz5s1cH1OzZk169uzJ888/z7Zt2zh48CBPPfUUlSpVomfPngC0a9eOS5cuMWXKFE6dOsWMGTNYs2ZNtnP9999/CQ8P5/Lly/m6wvPmm2/y999/M2zYMA4cOMCJEydYuXJltkbGd5o2bRo//PADx44d4/jx4yxduhR/f3+8vLzu+pxCZNm+XVt26wb38L+PKMsuZV7B8agFtqV0+Iqi69BVOllrN3GllIqMjFTDhg1TwcHByt7eXrm5uan7779fffLJJyopKUkplb2beG5y6yaeU5fvrFtekpOTVZ8+fZSXl1ehuonPnDlTBQQEKCcnJ/X444+rq1ev5vl8ucWY9by5yeom7unpqZydnVXnzp3NuokrpdSsWbNUlSpVlKurqxo4cKCaMGGCqZu4UlqX9Icffli5ubkVqJv47t27TY9zdXVV9evXVxMmTDBtz+m9mzNnjmrYsKFydXVVHh4eqkOHDuqff/7J8xxzY+mff1E4sbG3uodfvqx3NMIipcRp3cMXodTeV4vtae61m7hBqQL0T7ZACQkJeHp6Eh8fn619SHJyMhEREQQHB5t6rQh9jRs3jhUrVmSrBhJFTz7/ZdPy5dC7N4SGwuHDekcjLFLkWtjUVSu3Xwf+HYvlafL6/c4PqaISQogyJKt7eEiINl2DEAV2cXVmwQDeD+oaSl4kwRFCiDIka2SBtWv1jUNYsJj12tI1COx1GjsuHyTBEaXKuHHjpHpKiGKSmAhZo0u0aAE28gsgCiojBa6f1srVBusayt3Ix1sIIcqIHTtuzQrTubO+sQgLdXUfqDRw9IG67+sdTZ4kwRFCiDJi8+ZbZZl/ShTKpdumZyjEoLElSRIcIYQoI/74Q1s6OkKjRvrGIizUxd+0ZfnS/wGSBEcIIcqA1FTIat7WrBnYyTj2oqCUEa7s1spXD+gaSn5IgiOEEGXAvn2Qnq6Vu3TRNxZhoeL/A2PmRM6Ve+obSz5IgiOEEGVA1gSbLVvCgAH6xiIsVNS6W2W/drqFkV+S4AghRBmQleD07g1Vq+oairBUF1dpS3tPcA3UN5Z8kATHgkVHR/PKK69Qo0YNnJyc8PPzo2XLlsyaNYsbN24A2kSQBoMBg8GAq6srjRs3ZunSpdm25XQbPHgwZ86cYciQIQQHB+Ps7Ez16tUZO3Ysqampep66EKIAjMZbE2y2bq1vLMJCKaV1EQco3zTvfUsJaWZmoU6fPk3Lli3x8vJi4sSJ1KtXD0dHRw4dOsScOXOoVKkSjz76KAAffPABzz//PAkJCXz66af079+fSpUqsWfPHjIyMgD4+++/6dOnD+Hh4aY5P5ydndmxYwdGo5Gvv/6aGjVqcPjwYZ5//nmSkpKYOnWqbucvhMi///6Da9e0hsWOjnpHIyxS0llIv66VKz+qbyz5JAmOhXr55Zexs7Nj7969uLq6mtZXq1aNnj17cvscqu7u7vj7++Pv78+MGTNYuHAhq1atYtKkSaZ9ypcvD4Cvry9eXl6m9V26dKHLbS0Sq1WrRnh4OLNmzZIERwgLkVU9lZ4Oly7pG4uwUDGbbpWLaXLNoiYJTk7Sk3LfZrAFW6f87YsN2DnffV8715zX5+LKlSv8+eefTJw40Sy5MQszlwGY7OzssLe3v6cqpvj4eFNCJIQo/das0ZY2NvBg6Z0bUZRml//WlhW7gkdtfWPJJ0lwcrIkj8nDAh6Bdr/duv+LL2TcyHlf37bQcdOt+yurQsrl7Pv9n8q+Lg8nT55EKUWtWrXM1leoUIHk5GQAwsLCmDx5stn21NRUPv30U+Lj42nfvn2BnvP25/7yyy/l6o0QFkKpW+1vQkIgl/+JhMjbpW3asubQUj+CcRZpZGxFdu/ezYEDBwgNDSUlJcW0/s0338TNzQ0XFxcmT57Mxx9/TLdu3Qp8/IsXL9KlSxf69u3L888/X5ShCyGKyZkzWvsbkPmnRCElX4aEo1q5Qkt9YykAuYKTk36JuW8z2Jrf7xObx4HuyB97nilsRGZq1KiBwWAgPGta4EzVqlUDtMbBtxs9ejSDBw/Gzc0NPz+/XKuv8hIZGclDDz1EixYtmDNnTuGDF0KUqKz2NwAdOugXh7BgWfNPOfqCnYu+sRSAJDg5KUibmOLaNw/e3t48/PDDfPXVVwwfPjzXdjhZKlSoQI0aNQr9fBcvXuShhx6iSZMmzJ07FxsbufAnhKXImn8KtEH+hCiw8yu0ZepVsHHQNZSCkF8qCzVz5kzS09Np2rQpP/30E0ePHiU8PJyFCxdy7NgxbG1t736QfLh48SLt2rUjMDCQqVOncunSJaKjo4mOji6S4wshite2zKYTwcFwWwdJIfLv0hZt6VELbCznuojlRCrMVK9enf379zNx4kTGjBnDhQsXcHR0pE6dOrz++uu8/PLLRfI869at4+TJk5w8eZLKlSubbbu9K7oQovSJjYVz57Tyhg36xiIsVHqSNgYOgH8nfWMpIIOy8l+phIQEPD09iY+PNw1glyU5OZmIiAiCg4NxcnLK5QhCWCf5/Fu/ZcugTx+oWxcOHdI7GmGRov+CvzIbb7XfAP6F64FbGHn9fueHVFEJIYSV2rxZW8r0DKLQLv6aWbCBCpY1iJIkOEIIYaV+yxyy68wZXcMQliw6cwZx1yCL6kEFkuAIIYRVun4dTp3Syt7e+sYiLJQxHa6f0Mp+bfWNpRAkwRFCCCu0Y8etcteu+sUhLNi1A2BMAztPaPiJ3tEUmCQ4Qghhhdavv1Vu00a/OIQFyxrgz7cVOFXQN5ZCkARHCCGs0Nq12rJCBbhjhAch8ic2K8GxzFbqkuAIIYSVSUmBI0e0svSgEoWiFERlZsnGVH1jKSRJcIQQwsrs2wcZGVq5e3d9YxEW6vpxyLiplV2r6hpKYUmCI4QQViZrgs3y5aGt5XV+EaVB1LpbZb+SG9yvKEmCI0wGDx5Mr1698tynatWqfPbZZyUSj7U6c+YMBoOBAwcOALBp0yYMBgNxcXG6xiWsR1aC8+67UL26vrEIC3Vhpba09wKXSrqGUliS4FiowYMHYzAYMBgMODg4UKNGDT744APS09Pv+tg7f2ALYs+ePbzwwguFiPiWcePG0bBhw3s6RkFMmjSJZs2a4e7ujq+vL7169SI8PLxEnjs/SWOLFi2IiorC09OzRGIS1s1ohO3btbK0vxGFdnWvtizfRN847oGuCU7VqlVNP9K338LCwgBtrpywsDC8vb1xc3OjT58+xMTE6BlyqdKlSxeioqI4ceIEr732GuPGjeOTT4p3rAIfHx9cXCxrNMvNmzcTFhbGzp07WbduHWlpaXTq1ImkpCS9QwPAwcEBf39/DAZDoY+RmmqZjQBF0Tt8GOLiwNUVSvD/CGFNbkRCWpxWrtxT11Duha4Jzp49e4iKijLd1q3T6vz69u0LwKuvvsqqVatYunQpmzdvJjIykt69e+sZcqni6OiIv78/QUFBvPTSS3Ts2JElS5bg4eHBzz//bLbvihUrcHV15fr16wQHBwPQqFEjDAYD7dq1M9t36tSpVKxYEW9vb8LCwkhLSzNtu7OKatq0adSrVw9XV1eqVKnCyy+/TGJiYq4xz5s3j/Hjx3Pw4EFTQjtv3jwAzp07R8+ePXFzc8PDw4N+/fqZJbRZV36+/vprqlSpgouLC/369SM+Pj7P12nt2rUMHjyY0NBQGjRowLx58zh37hz79u3L83HXrl1j4MCBlCtXDhcXF7p27cqJEyeyxXO7zz77jKpVq5q2z58/n5UrV5rOddOmTdmeJ6cqqm3bttG6dWucnZ2pUqUKI0aMMEvIqlatyocffsjAgQPx8PDghRdeIDU1lWHDhlGxYkWcnJwICgpi0qRJeZ6jsD4bN2rLpCSZokEUUszGW+WKljWD+O10TXB8fHzw9/c33VavXk316tVp27Yt8fHxfPvtt0ybNo327dvTpEkT5s6dy99//83OnTuLJR6ltC8FPW5FMae7s7MzNjY2PPHEE8ydO9ds29y5c3n88cdxd3dn9+7dAKxfv56oqCiWLVtm2m/jxo2cOnWKjRs3Mn/+fObNm2dKQHJiY2PDF198wX///cf8+fP566+/eOONN3Ldv3///rz22muEhoaaEtv+/ftjNBrp2bMnV69eZfPmzaxbt47Tp0/Tv39/s8efPHmSJUuWsGrVKtauXcv+/ft5+eWXC/Q6ZSVE5cuXz3O/wYMHs3fvXn799Vd27NiBUopHHnnELOHLy+uvv06/fv1MV9qioqJo0aLFXR936tQpunTpQp8+ffj333/56aef2LZtG8OGDTPbb+rUqTRo0ID9+/fz3nvv8cUXX/Drr7+yZMkSwsPDWbRokSnZEmXHqlXa0sUFqlXTNxZhoWL+0paOPuB+n76x3AtVSqSkpChvb281YcIEpZRSGzZsUIC6du2a2X6BgYFq2rRp+T5ufHy8AlR8fHy2bTdv3lRHjhxRN2/eVEoplZiolJZqlPwtMbFgr9egQYNUz549lVJKGY1GtW7dOuXo6Khef/11tWvXLmVra6siIyOVUkrFxMQoOzs7tWnTJqWUUhEREQpQ+/fvz3bMoKAglZ6eblrXt29f1b9/f9P9oKAgNX369FzjWrp0qfL29s4z9rFjx6oGDRqYrfvzzz+Vra2tOnfunGndf//9pwC1e/du0+NsbW3VhQsXTPusWbNG2djYqKioqDyfM0tGRobq1q2batmyZZ77HT9+XAFq+/btpnWXL19Wzs7OasmSJbmex/Tp01VQUJDp/u3vU5Y7X/+NGzeafdaHDBmiXnjhBbPHbN26VdnY2Jg+q0FBQapXr15m+wwfPly1b99eGY3GPM8ty52ff2H5jEal3N2175RWrfSORlis3xsqtQilzvykaxh5/X7nR6lpZLxixQri4uIYPHgwANHR0Tg4OODl5WW2n5+fH9HR0bkeJyUlhYSEBLObtVq9ejVubm44OTnRtWtX+vfvz7hx47j//vsJDQ1l/vz5ACxcuJCgoCDa5GO89tDQUGxtbU33K1asSGxsbK77r1+/ng4dOlCpUiXc3d15+umnuXLlCjdu3ADAzc3NdHvxxRdzPc7Ro0epUqUKVapUMa2rU6cOXl5eHD161LQuMDCQSpVutehv3rw5RqOR8PBwtm7davZ8ixYtyvY8YWFhHD58mB9//NG07sUXXzR7XFY8dnZ2PPDAA6b9vL29qVWrllk8xeHgwYPMmzfPLKbOnTtjNBqJiIgw7de0aVOzxw0ePJgDBw5Qq1YtRowYwZ9//lmscYrSJyJCm2QT4NFH9Y1FWKjUeLh2UCtb6AjGWez0DiDLt99+S9euXQkICLin40yaNInx48cX6rEuLpBH85FiVZh2uw899BCzZs3CwcGBgIAA7OxuvZ3PPfccM2bM4K233mLu3Lk888wz+WrEam9vb3bfYDBgNBpz3PfMmTN0796dl156iQkTJlC+fHm2bdvGkCFDSE1NxcXFxaynloeHR8FPsgCaNm1q9nx+fn5m24cNG8bq1avZsmULlW8bu/6DDz7g9ddfL/Dz2djYoO6oW8xv9VVeEhMTGTp0KCNGjMi2LTAw0FR2dXU129a4cWMiIiJYs2YN69evp1+/fnTs2DFbeyxhvTZvvlXu0EG/OIQFu7QNUOBaDZwr6h3NPSkVCc7Zs2dZv369WVsQf39/UlNTiYuLM7uKExMTg7+/f67HGjNmDKNGjTLdT0hIMLsqkBeDQet5YClcXV2pUaNGjtueeuop3njjDb744guOHDnCoEGDTNscHBwAyMga6rSQ9u3bh9Fo5NNPP8XGRrsYuGTJErN9corPwcEh23PXrl2b8+fPc/78edP7deTIEeLi4qhTp45pv3PnzhEZGWlKhHfu3ImNjQ21atXC2dk5x+dTSjF8+HCWL1/Opk2bTI2ss/j6+uLr65stnvT0dHbt2mVqN3PlyhXCw8NN8fj4+BAdHY1SypQ83tn1PqdzvZvGjRtz5MiRXN/bvHh4eNC/f3/69+/P448/TpcuXbh69epd2xsJ67Ayc+gSBwdo0EDfWISFOr9cW6bEai0o7qF3p95KRRXV3Llz8fX1pVu3bqZ1TZo0wd7eng0bNpjWhYeHc+7cOZo3b57rsRwdHfHw8DC7lUXlypWjd+/ejB49mk6dOpldsfD19cXZ2Zm1a9cSExNz115IualRowZpaWl8+eWXnD59mgULFjB79uy7Pq5q1apERERw4MABLl++TEpKCh07dqRevXoMGDCAf/75h927dzNw4EDatm1rVhXj5OTEoEGDOHjwIFu3bmXEiBH069cvz6Q3LCyMhQsXsnjxYtzd3YmOjiY6OpqbN2/m+piaNWvSs2dPnn/+ebZt28bBgwd56qmnqFSpEj17at0m27Vrx6VLl5gyZQqnTp1ixowZrFmzJtu5/vvvv4SHh3P58uV8XeF58803+fvvvxk2bBgHDhzgxIkTrFy5Mlsj4ztNmzaNH374gWPHjnH8+HGWLl2Kv79/tmpeYb3+/ltb1qsHt9U0C5F/sZk9qFyrWnRyA6UgwTEajcydO5dBgwaZVbF4enoyZMgQRo0axcaNG9m3bx/PPPMMzZs358EHH9QxYsuRVVX07LPPmq23s7Pjiy++4OuvvyYgIMD0g11QDRo0YNq0aUyePJm6deuyaNGifHVL7tOnD126dOGhhx7Cx8eHH374AYPBwMqVKylXrhxt2rShY8eOVKtWjZ9++snssTVq1KB379488sgjdOrUifr16zNz5sw8n2/WrFnEx8fTrl07KlasaLrdeew7zZ07lyZNmtC9e3eaN2+OUorff//dVI1Xu3ZtZs6cyYwZM2jQoAG7d+/OVtX1/PPPU6tWLZo2bYqPjw/bs0Zgy0P9+vXZvHkzx48fp3Xr1jRq1Ij333//rtW37u7uTJkyhaZNm9KsWTPOnDnD77//brq6JqxbTAxcuqSVX3tN31iEhcpIgcQzWtn/YV1DKQoGdWcjghL2559/0rlzZ8LDw7nvPvPuaMnJybz22mv88MMPpKSk0LlzZ2bOnJnnf+t3SkhIwNPTk/j4+GxXc5KTk4mIiCA4OBgnJ6ciOZ/SZMGCBbz66qtERkaaqqUs2bhx41ixYkWhRmAW2Vn757+s+eUXePxx7erNv//qHY2wSLHbYH1mw+IOm8Hv7h1TilNev9/5oXsbnE6dOmVrqJnFycmJGTNmMGPGjBKOyrLduHGDqKgoPv74Y4YOHWoVyY0QIm9Z80/J9Ayi0C6syCzYQIX79YykSMi1ays0ZcoUQkJC8Pf3Z8yYMXqHI4QoAb/9pi3vs+Bx2YTOojNnEHcNAlvLv6orCY4VGjduHGlpaWzYsME0ros1GDdunFRPCZGDhAQ4eVIrnzunbyzCQikjxB/Tyr5t9Y2liEiCI4QQFi6r9xRA9+76xSEsWNxhUKlgsIfqQ/SOpkhIgiOEEBYua/4pGxu4bfBtIfLv0jZt6dcWfFvpG0sRkQRHCCEs3Pr12jI4uHCjogvBpcxW6j7W00pdEhwhhLBgKSm32t+0b69vLMJCKQVRmXPXlW+kbyxFSBIcIYSwYHv3QtZ0cb166RqKsFRJZyH1qlZOLdzI9qWRJDhCCGHBbp8dpGVL/eIQFixq7a2yf0f94ihikuAIk8GDB9PrLv8CVq1alc8++6xE4hFC3N3+/dpy+HDw9NQ3FmGhzq/Qlg7lwDn/MwWUdpLgWKjBgwdjMBgwGAw4ODhQo0YNPvjgA9LT0+/62DNnzmAwGAo1psyePXt44YUXChGxEKKoZWRA1vRmgwbpG4uwYFf3astyjfWNo4jpPlWDKLwuXbowd+5cUlJS+P333wkLC8Pe3r5YRy/28fEptmMLIQrm8GGIjwc3N2jQQO9ohEVKvgypV7Ry5V66hlLU5AqOBXN0dMTf35+goCBeeuklOnbsyJIlS/Dw8ODnn38223fFihW4urpy/fp1goODAWjUqBEGg4F27dqZ7Tt16lQqVqyIt7c3YWFhpKWlmbbdWUU1bdo06tWrh6urK1WqVOHll18mMTGx2M5ZCHHL779rywoVtDFwhCiwmL9ulQO66hdHMZA/iRwkJeV+S07O/743b+Zv36Li7OyMjY0NTzzxBHPnzjXbNnfuXB5//HHc3d3ZvXs3AOvXrycqKoply5aZ9tu4cSOnTp1i48aNzJ8/n3nz5jFv3rxcn9PGxoYvvviC//77j/nz5/PXX3/xxhtvFN1JCSFytXq1trxxQxIcUUjnM7//bV3ArZq+sRQxqaLKQV7TNz3yyK1J7QB8fbUvl5y0bQubNt26X7UqXL6cfb9cJlPPN6UUGzZs4I8//mD48OH07duXFi1aEBUVRcWKFYmNjeX3339nfeZoYFnVTN7e3vj7mzcoK1euHF999RW2traEhITQrVs3NmzYwPPPP5/jc48cOfK286vKRx99xIsvvsjMmTPv7aSEEHlSCg4e1MotWugbi7Bgiae15X3DwWDQN5YiJjm/BVu9ejVubm44OTnRtWtX+vfvz7hx47j//vsJDQ1l/vz5ACxcuJCgoCDatGlz12OGhoZia2trup+VIOVm/fr1dOjQgUqVKuHu7s7TTz/NlStXuJFb1ieEKBKnT9+6Aty7t76xCAuVngTXMrvh3feSvrEUA7mCk4O8mpDc9tsPQB6//dkuGZ85U+iQcvTQQw8xa9YsHBwcCAgIwM7u1tv53HPPMWPGDN566y3mzp3LM888gyEf2bm9vb3ZfYPBgDFrFLE7nDlzhu7du/PSSy8xYcIEypcvz7Zt2xgyZAipqam4yJjxQhSbP/64Ve7QQb84hAW7vAtUOrhUBpdAvaMpcpLg5MDVVf9983c8V2rUqJHjtqeeeoo33niDL774giNHjjDotj6kDg4OAGRkZNzT8+/btw+j0cinn36KTWY2t2TJkns6phAif1as0JblykFAgK6hCEt1bqm2dK5kddVTIFVUVqtcuXL07t2b0aNH06lTJypXrmza5uvri7OzM2vXriUmJob4+MINzV2jRg3S0tL48ssvOX36NAsWLGD27NlFdQpCiDzszRy6pGlTfeMQFixmg7ZMt86er5LgWLGsqqJnn33WbL2dnR1ffPEFX3/9NQEBAfTs2bNQx2/QoAHTpk1j8uTJ1K1bl0WLFjFp0qSiCF0IkYfoaLh2TSs/9pi+sQgLZUy/1cDYiqZnuJ1BqXvtw1O6JSQk4OnpSXx8PB4eHmbbkpOTiYiIIDg4GCcnJ50iLD4LFizg1VdfJTIy0lQtJUQWa//8W7Off4a+faFePfjnH7CTxgaioC7vhj8f0Modt4Bva33jyUFev9/5IX8WVujGjRtERUXx8ccfM3ToUEluhLAyW7dqyzZtJLkRhXQha/wzG/C+X9dQiotUUVmhKVOmEBISgr+/f7FO2yCE0EfW+FqtS98/3cJSRGZ2w3MNAltHfWMpJpLgWKFx48aRlpbGhg0bcMtr1EIhhMVJSIB//9XK1t3AQBQbpSDhqFb2vfv4aJZKEhwhhLAgt4+OLhNsikK5fhyMKVo58HF9YylGkuCgTXUgRFkjn3vLlDWPrpMThIToG4uwULGZjbgqtAT/TvrGUozKdIKTNWqvTCsgyqKsz/2do1eL0i2rgXFoqFWOzSZKwqXMD5FfO7C13k4oZbr9va2tLV5eXqa5llxcXPI1nYEQlkwpxY0bN4iNjcXLy8ts7jFRuqWkwNmzWrlLF31jERYs6wqOTyt94yhmZTrBAUyzaec1oaQQ1sjLyyvbbPKidNux41bD4j599I1FWKgbkZAUoZUdvHQNpbiV+QTHYDBQsWJFfH19SUtL0zscIUqEvb29XLmxQFntb+ztoX59fWMRFipyza2yWzX94igBZT7ByWJraytf+EKIUu3QIW3ZqhXI15UolAvLtaW9Fzj56hpKcSvTjYyFEMJSZGTAgQNaeepUXUMRluzKbm1ZvrG+cZQASXCEEMICHDqkDfLn7i7j34hCSo2HlEtauXLhJlm2JJLgCCGEBVi9Wlvef79UT4lCivnrVrlSD/3iKCGS4AghhAVYljk34pkzuoYhLNm5pdrS1gVcq+oaSkmQBEcIIUo5peBo5tRBbdvqG4uwYPFHtGWFB8vEKJGS4AghRCl38iQkJ2vlJ57QNxZhoTJSIOGYVm42S99YSogkOEIIUcr98ou2NBi0LuJCFNiVPdoEm06+4F5T72hKhCQ4QghRymU1MK5UCZyd9Y1FWKisAf4qtCwT1VMgCY4QQpR6//6rLVu00DcOYcEuZLZSTzqnbxwlSBIcIYQoxaKj4fp1rdyvn76xCAtlzIDrJ7Wyb0t9YylBkuAIIUQptmWLtqxUCTp10jcWYaHiDoFK18pV+uobSwmSBEcIIUqxbdu05WOPaaMYC1Fg5zNnacUGvO/XNZSSJAmOEEKUYlu3asvWrfWNQ1iwyLXa0jUIbB30jaUESYIjhBClVHz8rQk2GzbUMxJhsZSChMwB/nzLVpYsCY4QQpRSv/9+q+zjo18cwoIlnYGMm1o5sOy0vwFJcIQQotRamjl1ULly2k2IAovNrON0qgi+7XQNpaRJgiOEEKWQUrfa3zRqpG8swoJdymylHjwA7N30jaWESYIjhBCl0Pr1cPmyVv6//9M3FmHBLmVmyT5lq/0NSIIjhBCl0uuva0tnZ0lwRCElX7o1wWb5pvrGogNJcIQQopQ5ePDW9AwvvSTzT4lCypp/CsDWUb84dCIJjhBClDKTJmlLOzt45x19YxEW7HzmNPQO5cDRW99YdKB7gnPx4kWeeuopvL29cXZ2pl69euzdu9e0XSnF+++/T8WKFXF2dqZjx46cOHFCx4iFEKL4nDsHv2T+Lq1dC+XL6xuPsGBXdmvLcmWzlbquCc61a9do2bIl9vb2rFmzhiNHjvDpp59S7rb+kFOmTOGLL75g9uzZ7Nq1C1dXVzp37kxycrKOkQshRPH47DNIT4f27aFDB72jERYrPQmSo7Vy5Z76xqITg1JK6fXkb731Ftu3b2drVl/IOyilCAgI4LXXXuP1zBZ38fHx+Pn5MW/ePJ544om7PkdCQgKenp7Ex8fj4eFRpPELIURRunYNKlaElBTt6k3nznpHJCzWhdWwpYdW7nkOXKvoG08h3Ovvt65XcH799VeaNm1K37598fX1pVGjRnzzzTem7REREURHR9OxY0fTOk9PTx544AF27NiR4zFTUlJISEgwuwkhhCWYMEFLbgBCQ/WNRVi4c0u0pa2rRSY3RUHXBOf06dPMmjWLmjVr8scff/DSSy8xYsQI5s+fD0B0tHZ5zc/Pz+xxfn5+pm13mjRpEp6enqZblSpl840VQliW5GSYOVMrN2gAlSvrG4+wcJe2a0uvspsp65rgGI1GGjduzMSJE2nUqBEvvPACzz//PLNnzy70MceMGUN8fLzpdv78+SKMWAghisfMmXAzc8qgTz/VNxZh4YxpcDNKK9cZo28sOtI1walYsSJ16tQxW1e7dm3OnTsHgL+/PwAxMTFm+8TExJi23cnR0REPDw+zmxBClGZGo1Y9BRAYqDUwFqLQrh0A402te3jlR/WORje6JjgtW7YkPDzcbN3x48cJCgoCIDg4GH9/fzZs2GDanpCQwK5du2jevHmJxiqEEMXlp5/g6lWtPHEiGAz6xiMsXNYEmxVagkH30WB0Y6fnk7/66qu0aNGCiRMn0q9fP3bv3s2cOXOYM2cOAAaDgZEjR/LRRx9Rs2ZNgoODee+99wgICKBXr156hi6EEEVmTGYtQvnykI/OoULk7cwCbensl/d+Vk7XBKdZs2YsX76cMWPG8MEHHxAcHMxnn33GgAEDTPu88cYbJCUl8cILLxAXF0erVq1Yu3YtTk5OOkYuhBBFY/t2OHtWK48ZA7a2+sYjLJxSEP+fVrZ10TcWnek6Dk5JkHFwhBClWc+e8OuvMHAgzJkDjmVvyiBRlOKOwO+ZPace/ht8LLc5h0WPgyOEEGXZ0aNacmMwwNtvS3IjikDW+DcGW/Bupm8sOpMERwghdPLmm9ry0UehVi19YxFWIipzBnGXQLDRtRWK7iTBEUIIHURGwurVWjmz46gQ9y4us/2NTyt94ygFJMERQggdvPmm1h7UYIA33tA7GmEVki5ARpJWDuqvbyylgCQ4QghRwhIS4McftXL79lCpkr7xCCsR/WdmwQB+MlqkJDhCCFHCJkyA9HSt/Pnn+sYirMjVf7RljRfBzlnfWEoBSXCEEKIEpabCV19p5YYNZdZwUYQubdOW/g/pG0cpIQmOEEKUoNmz4cYNrTx9ur6xCCuSGgdxB7WyT2tdQyktJMERQogSohR89plWDgqCdu30jEZYlYuZXfJsnMDRW99YSglJcIQQooSsWQMREeDmBn/8oXc0wqqcX6Yt7ZzBxl7fWEoJSXCEEKKETJmiLV98UQb2E0Xsyk5tWa6RvnGUIpLgCCFECdi2DTZv1ibTfOUVvaMRViUjGW5Ga+VKPfSNpRSRBEcIIUrAyJHa0stLxr0RRSxmM5A5b3aVvrqGUppIgiOEEMXs+HHYt08rDx6sjV4sRJE5mzlqpK0ruEr2nEUSHCGEKGZZVVJ2dvD++/rGIqxQ7FZt6VlH3zhKGUlwhBCiGMXG3uox1a8feHjoG4+wMsYMSI7UylV66RpKaSMJjhBCFKM33rg1qebUqXpHI6xO/GHIuAl2blBbZm29nSQ4QghRTJKSYPFirdy+PVSsqG88wgplTc9QoQXY2OkbSykjCY4QQhSTb76BtDSt/OWX+sYirNSFVdqyQkt94yiFJMERQohikJ5+a6bwCROgdm194xFWSCmI3aSV0+L0jKRUkgRHCCGKwc8/w5kz4OMDr76qdzTCKiWeAmOKVg7so28spZAkOEIIUcSUgnfe0crDh4Ozs77xCCt15gdtabAF7wf1jaUUkgRHCCGK2MqVcPq0VpYZw0WxifxdW7pUARtbfWMphSTBEUKIIjZ6tLb09oZWrfSNRVix+P+0pY98yHJSqARn7ty53Lhxo6hjEUIIi7drF5w8qZXfflumZRDF5GYspF/XykH99Y2llCpUgvPWW2/h7+/PkCFD+Pvvv4s6JiGEsFgjRmhLFxet/Y0QxeL80syCAfw76hpKaVWoBOfixYvMnz+fy5cv065dO0JCQpg8eTLR0dFFHZ8QQliM06dh926t/NJLYG+vbzzCiiUc05YVmoOtk76xlFKFSnDs7Ox47LHHWLlyJefPn+f5559n0aJFBAYG8uijj7Jy5UqMRmNRxyqEEKVa1hUbOzsYN07XUIS1u7xLW9Z8Wd84SrF7bmTs5+dHq1ataN68OTY2Nhw6dIhBgwZRvXp1Nm3aVAQhCiFE6XflCqxbp5X79QM3N33jEVYsLRGu/aOVfVvrG0spVugEJyYmhqlTpxIaGkq7du1ISEhg9erVREREcPHiRfr168egQYOKMlYhhCi1Zs3SpmWoWxe++krvaIRVi1oLKgMcfcA1UO9oSi2DUkoV9EE9evTgjz/+4L777uO5555j4MCBlC9f3myf2NhY/P39da+qSkhIwNPTk/j4eDw8PHSNRQhhnW7ehKpVITYWFi2C//s/vSMSVm3r43D+F7AvB32v6h1NsbnX3+9CTT3q6+vL5s2bad68ea77+Pj4EBERUZjDCyGERfnkEy25CQqCvn31jkZYvcs7tWW5BvrGUcoVqoqqbdu2NG7cONv61NRUvv/+ewAMBgNBQUH3Fp0QQpRyGRkwdapWDgmRnlOimBnT4GakVq7UXd9YSrlCJTjPPPMM8fHx2dZfv36dZ5555p6DEkIIS/HNN3A9c7y1rPmnhCg2MVuAzJYlQU/qGkppV6gERymFIYfhOS9cuICnp+c9ByWEEJZAqVvdwatWhdbSoUUUt7OZE2zauoJLgL6xlHIFaoPTqFEjDAYDBoOBDh06YGd36+EZGRlERETQpUuXIg9SCCFKo9WrISZGK0+erG8sooy4tEVbetbWNw4LUKAEp1evXgAcOHCAzp0743bbQA8ODg5UrVqVPn36FGmAQghRWr3+urb09pbGxaIEKAWJZ7Ryxc66hmIJCpTgjB07FoCqVavSv39/nJxkeGghRNm0bx8cP66V33lHJtUUJSAhHFQa2DhAzTC9oyn1CtVNXAbwE0KUdR9+qC1lUk1RYi5t1ZYVmoNLRX1jsQD5TnDKly/P8ePHqVChAuXKlcuxkXGWq1etd+AhIYS4eBF+/10rz5+vzT0lRLGLzUxwfKQ1e37k+89y+vTpuLu7m8p5JThCCGHNPv9cm5ahTRt4/HG9oxFlxoXl2tJFpmfIj0JN1WBJZKoGIURRiouDihUhOVnrRdWtm94RiTIh8Sz8WlUrP7wNfFrqGk5JuNff70KNgzNv3rwc16enpzNmzJjCHFIIISzCG29oyY2DAzz8sN7RiDLjzEJtabAF7wf1jcVCFCrBGTFiBH379uXatWumdeHh4TzwwAP88MMPRRacEEKUJikpkDkbDa1ba0mOECUico22dKkCNrb6xmIhCpXg7N+/nwsXLlCvXj3WrVvHjBkzaNy4MSEhIRw8eLCoYxRCiFJh0iQtyQGYOVPfWEQZE39IW1ZooW8cFqRQbf+rV6/O9u3bGTlyJF26dMHW1pb58+fz5JMyL4YQwjoZjTB9ulZu2BDuu0/XcERZknIN0hK0clB/fWOxIIW6ggPw22+/8eOPP9K8eXO8vLz49ttviYyMLMrYhBCi1PjuO0jI/I358kt9YxFlzLklmQWDjGBcAIVKcIYOHUrfvn1588032bp1K//++y8ODg7Uq1ePJUuW3P0AQghhYd5/X1tWrQqtWukaiihrsgb4c/IDW0d9Y7Eghaqi2r59O7t27aJBgwYA+Pv78/vvvzNjxgyeffZZ+vXrV6RBCiGEnnbsgKgorSyTaooSl3RGW9b7QNcwLE2hEpx9+/bh6Jg9iwwLC6Njx473HJQQQpQmU6dqy549ZVJNUcIykuHKHq3s107XUCxNoaqoHB0dOXXqFO+++y5PPvkksbGxAKxZs4b09PQiDVAIIfR0/DgszxxAduJEmVRTlLBL28GYqlVPudfQOxqLUqgEZ/PmzdSrV49du3axbNkyEhMTATh48KBpxvH8GDduHAaDwewWEhJi2p6cnExYWBje3t64ubnRp08fYmJiChOyEEIUyujRoBT06AF16ugdjShzTn6jLW1dJbsuoEIlOG+99RYfffQR69atw+G2ka7at2/Pzp07C3Ss0NBQoqKiTLdt27aZtr366qusWrWKpUuXsnnzZiIjI+ndu3dhQhZCiAK7cAF+/VUry6jFQheXd2hL1yr6xmGBCtUG59ChQyxevDjbel9fXy5fvlywAOzs8Pf3z7Y+Pj6eb7/9lsWLF9O+fXsA5s6dS+3atdm5cycPPihDVQshitewYdrS3h6GDNE3FlEGGTPgxgWtHNBd31gsUKGu4Hh5eRGV1aXgNvv376dSpUoFOtaJEycICAigWrVqDBgwgHPnzgFaQ+a0tDSzRsshISEEBgayY8eOXI+XkpJCQkKC2U0IIQrq+nVYtUor9+0LLi76xiPKoEvbAKNWrjpA11AsUaESnCeeeII333yT6OhoDAYDRqOR7du38/rrrzNw4MB8H+eBBx5g3rx5rF27llmzZhEREUHr1q25fv060dHRODg44OXlZfYYPz8/oqOjcz3mpEmT8PT0NN2qVJHLekKIghs9Whu92GCAzz7TOxpRJp3JrCmxdQWXivrGYoEKVUU1ceJEwsLCqFKlChkZGdSpU4eMjAz+7//+j3fffTffx+nataupXL9+fR544AGCgoJYsmQJzs7OhQmNMWPGMGrUKNP9hIQESXKEEAWSlgbz5mnl9u3Bx0fXcERZFbtZW3rW1jcOC1WoBMfBwYFvvvmG9957j8OHD5OYmEijRo2oWbPmPQXj5eXFfffdx8mTJ3n44YdJTU0lLi7O7CpOTExMjm12sjg6OuY4Ro8QQuTX7ZNqzpqlbyyijFIKEk9rZX9p4V4YhUpwsgQGBhIYGFhUsZCYmMipU6d4+umnadKkCfb29mzYsIE+ffoAEB4ezrlz52jevHmRPacQQtxOKfgms2duo0Zwj/+3CVE4SRGg0gADBA/SOxqLlO8E5/Zqn7uZNm1avvZ7/fXX6dGjB0FBQURGRjJ27FhsbW158skn8fT0ZMiQIYwaNYry5cvj4eHB8OHDad68ufSgEkIUm3XrtO7hLi7w/fd6RyPKrNjM+acqPAietfSNxULlO8HZv39/vvYzFGAgogsXLvDkk09y5coVfHx8aNWqFTt37sQns8J7+vTp2NjY0KdPH1JSUujcuTMzZ87M9/GFEKKgpkzRli+8AHXr6huLKMMuZY4J5yMzuxaWQSml9A6iOCUkJODp6Ul8fDweHh56hyOEKMU2bICOHcHWFk6fhiKsgReiYFZU0cbAafMrVO6hdzS6uNff70J1E7/d+fPnOX/+/L0eRgghdDdihLasUUOSG6GjGxdvDfDnKF34CqtQCU56ejrvvfcenp6eVK1alapVq+Lp6cm7775LWlpaUccohBDF7tAhOHJEKw8dqm8soow7s0hbGmyhwv36xmLBCtWLavjw4SxbtowpU6aYejTt2LGDcePGceXKFWZJv0ohhIV56SVt6eJy60qOELq4uFpbulQBwz1XtJRZhUpwFi9ezI8//phtoL4qVarw5JNPSoIjhLAokZHw999a+cUXtTY4Qugm7pC2rNBC3zgsXKFSQ0dHR6pWrZptfXBwsNns4kIIYQmGDdPGv7Gzg48+0jsaUaalJkBanFYO7KdrKJauUAnOsGHD+PDDD0nJGuoTbZLLCRMmMCxr+l0hhLAAiYnw669auW9fKOQsMUIUjXNLMgsGqPSIrqFYukJVUe3fv58NGzZQuXJlGjRoAMDBgwdJTU2lQ4cO9O7d27TvsmXLiiZSIYQoBtOnQ0aGNqnmF1/oHY0o884v15ZOvmBjr28sFq5QCY6Xl5dp+oQsMqGlEMLSZGTcmlRz9GioUEHXcISA9ARtWaVP3vuJuypwgqOUYvz48fj4+BR6xm8hhCgNli3TBvTz9ob339c7GlHmGdPg6j9auebL+sZiBQrcBkcpRY0aNbhw4UJxxCOEECVCKXjvPa08bBi4uuobjxBc3Q8ZN8ChHHjW1jsai1fgBMfGxoaaNWty5cqV4ohHCCFKxIIFEB6utb0ZJJM1i9Lg9Hfa0rOejH9TBAr1Cn788ceMHj2aw4cPF3U8QghRIt55R1sGBUFwsL6xCAFA1Fq9I7AqhWpkPHDgQG7cuEGDBg1wcHDI1hbn6tWrRRKcEEIUh7/+gqxa9smT9Y1FCECrM82afyqgm76xWIlCJTifffZZEYchhBAlJ2sqBm9v6CdjqYnS4NLfoDK0cvDT+sZiJQqV4AySCmshhIX67z/tBjBmjL6xCGFyZqG2tHUBl4r6xmIlCt2K6dSpU7z77rs8+eSTxMbGArBmzRr+y/rmEEKIUuj2STVffVXfWIQwid2sLT1C9I3DihQqwdm8eTP16tVj165dLFu2jMTEREAbzXjs2LFFGqAQQhSVuDjYsUMrv/AC2EhHFVFaJJ7SlhUf1jcOK1KoP++33nqLjz76iHXr1plNrtm+fXt27txZZMEJIURRmj0b0tOhWjWYNEnvaITIdP0UGFO1crA0ASkqhUpwDh06xGOPPZZtva+vL5cvX77noIQQoqglJ8Pnn2vlcePAyUnXcIS45coebeleSwb4K0KFSnC8vLyIiorKtn7//v1UqlTpnoMSQoiiNmUKREdD5crwxBN6RyPEbS5t1ZYBXfSNw8oUKsF54oknePPNN4mOjsZgMGA0Gtm+fTuvv/46AwcOLOoYhRDinhiNt6qkHn4Y7GWSZlGaZCU4Pq31jcPKFCrBmThxIrVr1yYwMJDExETq1KlDmzZtaNGiBe+++25RxyiEEPdk8mStigrg7bf1jUUIM8kxEHdIK5drpG8sVqZA4+AYjUY++eQTfv31V1JTU3n66afp06cPiYmJNGrUiJo1axZXnEIIUWiffKItGzSAGjX0jUUIM6cXaEuDLbjJnCFFqUAJzoQJExg3bhwdO3bE2dmZxYsXo5Tiu+++K674hBDinixcCNeuaeWvvtI3FiGyiVytLZ0razO/iiJToCqq77//npkzZ/LHH3+wYsUKVq1axaJFizAajcUVnxBC3JOsKqmqVaFVK11DESK7uH+1ZYXm+sZhhQqU4Jw7d45HHnnEdL9jx44YDAYiIyOLPDAhhLhXmzbB+fNa+eOPdQ1FiOzSb0Bq5uXFIJkUragVKMFJT0/H6Y7BI+zt7UlLSyvSoIQQoih8+KG29PaG/v31jUWIbM4uySwYIKC7rqFYowK1wVFKMXjwYBwdHU3rkpOTefHFF3F1dTWtW7ZsWdFFKIQQhRAVBdu2aeWsAf6EKFUuZP5WOvqArYxdUNQKlODkNIv4U089VWTBCCFEUfniC0hNhZYtYcAAvaMRIgfXM+ef8m6qbxxWqkAJzty5c4srDiGEKDKXLt3qMTV6tL6xCJEjYwbcvKCV643XNxYrJXPpCiGsTlgYJCaCpyf06KF3NELkIP4QpCWAnbsM8FdMJMERQliVmzdh+XKt3KUL2Mi3nCiNIv/Qlj4twMZW31islPzpCyGsyptvQnq6NmbajBl6RyNELk5+rS1tZFr74iIJjhDCahiN8L//aeV27bTu4UKUOkrBjXNaucID+sZixSTBEUJYjalTtSoqgK+/1jcWIXJ1eReoDK0cnL13sigakuAIIazG5Mnasn59kLl/Ral1ZqG2tHUBlwB9Y7FikuAIIazCX3/B1ataWSbVFKVa7EZt6VFL3zisnCQ4QgirMHu2tnz4YWjdWt9YhMhT1gB//g/rG4eVkwRHCGHxTp2CX37Ryp9+qm8sQuTp+ikwpmjl4IH6xmLlJMERQli8t9/WelB17Qr16ukdjRB5uJQ5QZq9F3iF6hqKtZMERwhh0cLDYUnmpMzPPqtvLELc1bUD2rLqk7qGURZIgiOEsGgvvqgtnZ2hd299YxHirrKu4Pi00jeOMkASHCGExbp8GTZv1srPPy/TMohSLvkyXP1HK/tIS/jiJl8HQgiL9fLL2qCwdnYwZYre0QhxF2cWAEbABlwq6x2N1ZMERwhhka5duzWpZu/e4OiobzxC3NWFVdrSpZI2WZooVpLgCCEsTkoKhIbemlRTBvYTFiHuoLb0flDfOMoISXCEEBZFKXjpJYiK0u5/+in4+OgbkxB3lZ4MqZlDbQf21TeWMkISHCGERXn3XZg7V7tyM20avPqq3hEJkQ/nltwqV+6lWxhliZ3eAQghRH6FhcHMmVp5zhx47jl94xEi384v05aOvmBrr28sZYRcwRFCWIRhw24lN+PHS3IjLMzVPdqyfGN94yhDSk2C8/HHH2MwGBg5cqRpXXJyMmFhYXh7e+Pm5kafPn2IiYnRL0ghhC4mTIAZM7Ry/fpaNZUQFkMZIeOGVq42WNdQypJSkeDs2bOHr7/+mvr165utf/XVV1m1ahVLly5l8+bNREZG0luGKhWiTJk791ZCExgI+/bJgH7CwiQcg9Q4sHWGyo/pHU2ZofvXRGJiIgMGDOCbb76hXLlypvXx8fF8++23TJs2jfbt29OkSRPmzp3L33//zc6dO3WMWAhRUn7/HYYM0cre3vDff9qgfkJYlKzpGbwfAFsHfWMpQ3RPcMLCwujWrRsdO3Y0W79v3z7S0tLM1oeEhBAYGMiOHTtKOkwhRAnbswcefVTrFu7qCocPg5ub3lEJUQgnv9GWXjLVfUnS9X+hH3/8kX/++Yc9e/Zk2xYdHY2DgwNeXl5m6/38/IiOjs71mCkpKaSkpJjuJyQkFFm8QoiScfUq9O0LGRlgbw9794K/v95RCVFIWTOIO8mHuCTpdgXn/PnzvPLKKyxatAgnJ6ciO+6kSZPw9PQ03apUqVJkxxZCFL8bN6BHDzh7FipUgI0bISRE76iEKKTLe0Cla+Vqg/SNpYzRLcHZt28fsbGxNG7cGDs7O+zs7Ni8eTNffPEFdnZ2+Pn5kZqaSlxcnNnjYmJi8M/jX7kxY8YQHx9vup0/f76Yz0QIUVSykpu//wYvLy25adlS76iEuAdnFmhLW2dtDipRYnSrourQoQOHDh0yW/fMM88QEhLCm2++SZUqVbC3t2fDhg306dMHgPDwcM6dO0fz5s1zPa6joyOOMuueEBYnIwPq1YPTp8HBAVavhrp19Y5KiHsUs1FbutfSN44ySLcEx93dnbp3fHu5urri7e1tWj9kyBBGjRpF+fLl8fDwYPjw4TRv3pwHH5SJyoSwJkrBgw9qyQ3AmDFy5UZYiesntaV/B33jKINKdYfL6dOnY2NjQ58+fUhJSaFz587MzBrKVAhhNbp31xoSgzZC8bhxuoYjxL1LjYd1bcCYrN0PlvY3Jc2glFJ6B1GcEhIS8PT0JD4+Hg8PD73DEULc4bnn4NtvtXK3blrVlBAWLXYL7BgISWe1+xW7wkO/6xuTBbrX3+9SfQVHCGHd3n//VnLTrBmsWqVvPELck9RE2DMUzv4AKHCrBo2nQ+VH9Y6sTJIERwihi+3btTmmAKpXhx07wGDQNyYhCu3Cr7D9Cci4qd2vPkRLbuzd9Y2rDJMERwhR4v77T+sObjRCzZpw8CDY2uodlRCFYMyAbf3hwi+ZKwxQ9z2oP17XsIQkOEKIEnbsGHTuDNeuaT2n1q8HZ2e9oxKiEK7sg42dIfWKdt+xAjy0Dso31DUsoZEERwhRYo4cgYYNIS1NG5149WptnikhLM4/o+HYp0BmP53AftDiB5nqvhSRBEcIUSIiI7WGxGlpWnXUzz9rM4QLYVFS42Hv8NtGKHaBVkugUjd94xLZSIIjhCh2CQnaKMU3bmgNiVetgtBQvaMSooCiN8CuIZndvw1QsQu0Wgr2chmyNJIERwhRrNLStCkXrl7V7s+dC1276huTEAWSmgibOsHlHdp9t2rQfAH4tNA3LpEnSXCEEMXGaIQmTSBrzttJk2CQDOgqLMmFVbC9/63u3z5toN1q6f5tASTBEUIUm5degqw5dUeMgLfe0jceIfLNmKGNa3P+58wVBgh5HRpP0TUskX+S4AghisW338KcOVr58cfh88/1jUeIfLu6H/56+Fb3bwdvaP8nlG+sb1yiQKQ/mxCiyK1YAS+8oJXffBOWLtU1HCHy79xS+POBW8lNlcehd4wkNxZIEhwhRJGaPh1699ba3zzzjNbuRohSLzUedgyCbf3AmAZ27tB2FbReCjYyzLYlkioqIUSRWbwYRo3SytWra1VUMr+UKPWOz4KDYyAtHgw2UOdtqPc+2NjrHZm4B5LgCCGKxIYN8PTTWtnLC/buBTv5hhGlWVoSbOwEl//W7jv5Q+tfpPu3lZCvHyHEPTt4ELp00aqlnJ21+15eekclRB4u/qZVR2Xc0O67VYeOm8Clsq5hiaIjCY4Q4p6cPatNmpmerl2x2bULAgP1jkqIXOTY/fs1aPyJrmGJoicJjhCi0FJStPmlkpO1OQb/+EObkkGIUulGJKxtCslR2n3p/m3VpBeVEKJQMjJg4EC4dElLbhYtgvbt9Y5KiFyc+xl+r3cruZHu31ZPruAIIQpMKXjlFViyBOzt4bff4OGH9Y5KiBwkXYA9QyHyd+1++SbQaBr4tdE3LlHsJMERQhRYu3awZYvWBXzBAkluRCl1fBbsGwEqHTBA6DvS/bsMkQRHCFEgffpoyQ3A++9D//76xiNENnd2/zbYwv3fQPVn9I1LlChJcIQQ+TZsGCxbppU7dIBx43QNR4jspPu3yCQJjhAiXyZMgBkztHL9+rBunb7xCGFGGWH7k3BuSeYKA4SMgsZTdQ1L6EcSHCHEXX33Hbz7rlYODIR9+2QKBlGK3IiEnc9A9J/afen+LZAERwhxF3/9Bc89p5W9veG//2QKBlGKnPkB9g6D1Ktg4wRVekPz72WCTCEJjhAid9HRWnKjFLi4wKFD4Oamd1RCADcjYV1bSDyp3S/fBJovBM8QfeMSpYYM9CeEyFF8PHTtChEREBysJTcVK+odlRBo3b9XBN1Kbqo+DZ12SHIjzMgVHCFENpcuwf33w5kz4OurNSiuVk3vqESZl1P378bTodZwfeMSpZIkOEIIMzduQGioluTY2cHatVC9ut5RiTLv4u+wre9t3b+rQYdN4FpFz6hEKSZVVEIIk/R0bbLMS5e0+19+CY0a6RuTKOOUEY59AZu7ZyY3md2/Hz0lyY3Ik1zBEUIAWkPi5s3h9Gnt/vvvw4sv6huTKOPu7P7tWAEeWqs1KBbiLiTBEUIA0K0b7N2rlZ97DsaP1zceUcbtfwuOf6ldtbF1gkafQs2XZAAmkW+S4AghGDIE1qzRyt26wTff6BuPKMNuRsL6dnD9hHbfoza0XiY9pESBSYIjRBm3bh3Mn6+VmzWDVav0jUeUYSe+1gbtU+nafe8H4KE/wcFD37iERZIER4gybO9eeOwxyMiARx6BlSulBkDoIO1GZvfv7dp96f4tioAkOEKUUcuXa1VTSUnazODLlskUDEIHcYfhz+aQnqjdd60GHf8C1yB94xIWT77OhCiDtm6Fxx8HoxHq1NGSHUdHvaMSZULSOTj9PSRHQ/whuLwTjKmAAWq9Ak2m6x2hsBKS4AhRxhw5ol2xMRq1pGbVKnB31zsqYZWMRri0Fc7+ALFbIfE0GJOz7+ffGRp9DOUalniIwnpJgiNEGRIZqTUkTksDW1vtSo5MwSCKTEYqXPsHLm2HyDUQsxEwZt/PYKeNZVPjeajQEjxqSeMvUeQkwRGijEhIgLp1takYDAbtyk2zZnpHJSxa4hmImA9Rf0L8f1o7GpWRfT87V3APAf/2UHUAlGtQ4qGKskcSHCHKgNRULbm5dk27/9132kzhQuSbUpAQDsemadVOiRFgTMm+n6M3VGgBPi3BwRuqPKatE6KESYIjRBnw5ptw/rxWnjQJBg/WNRxhCVIT4dyPcGkbpMVrM3gnx2bfz2AProHg0wKq9IVK3aW6SZQKkuAIYeU++0y7gZbovPWWntGIUivhBEQsgOh1kHAE0hKy72PjAI6+4OwP/h2h6lPgFVrysQqRD5LgCGHFJk6Ed965VR4zRt94RClhNELiKW1gvUvb4NzSnBMaDODkq80B5d9Raxhs61Ti4QpRGJLgCGFlLl+GTz+FxYvh3Dlt3YgRcuWmTEuNh7M/wsVf4eo/kBwDqOz72ThoA+xVaAlB/bTu2zY2JR6uEEVBEhwhrMClSzBlCixZciupyVKnDkyfLs0iypTky1p107kfIf4opF/Pvo/BDio8oCUznqHavE+etUo+ViGKiSQ4QliotDTYuBF++gnmztU6uWSxtYV69WDoUHjhBfkn3KoZMyDqDzi3BDJuQty/kHAshx0N4OQP5RtDpUch6AmZxFJYNUlwhLAgkZEweTKsXatVRV29emubrS00bAgvvgiDBoG9vW5hiuKUcg3OLIbI1XBtf2bPphyqm1yDtSonn1YQ1B/8OkimK8oUSXCEKOUuXNCSml9+gago820+PtC7N/TsCQ8/LJNlWqUb0XB1tzY6cNQaiDuU8372nuDXDqoNgQrNwalCiYYpRGkjX4dClELJyTB1KsyYAdHR5tvs7KBJExg9Gnr10q7cCCuRdgMuLIeLK+HKHrhxAVR6DjvaaF21yzeFSj21BsH2biUerhClma4JzqxZs5g1axZnzpwBIDQ0lPfff5+umUOsJicn89prr/Hjjz+SkpJC586dmTlzJn5+fjpGLUTxiIjQ2tRs2KBNo3D9tnah9vbQtCkMGwb9+0tSYzWSYyF2Cxz9RGs3k2NXbcAjRBsZuEJLcK+hLaW6SYg86ZrgVK5cmY8//piaNWuilGL+/Pn07NmT/fv3Exoayquvvspvv/3G0qVL8fT0ZNiwYfTu3Zvt27frGbYQRebUKW1k4V9/1XpC3S4gQJsIc/hwePxx+T2zeGmJ2ngzF1dB0hltdODE0znsaNCmNvCsC/4PQ9X/A7eqJRysEJbPoJTKoXWafsqXL88nn3zC448/jo+PD4sXL+bxxx8H4NixY9SuXZsdO3bw4IMP5ut4CQkJeHp6Eh8fj4eH9BgQ+jt+XEtqVq/WGgrfzsVFayTcty/cf78kNRbt8h6tm3bsZkg4nnNXbQDPOmDvpc2oXaWPNvaMrbQeEOJef79LzV9RRkYGS5cuJSkpiebNm7Nv3z7S0tLo2LGjaZ+QkBACAwPzTHBSUlJISbk1AVxCQi6XfIUoQdevw2+/aePULF9uvs3REZo3h1dfhR49ZLwai5QaDxdWQOpVuLxTu904l8OOBnCsAF71IWSUNn+Tg1cJBytE2aB7gnPo0CGaN29OcnIybm5uLF++nDp16nDgwAEcHBzw8vIy29/Pz4/oO1td3mbSpEmMHz++mKMW4u4OH9au1Pz1lzaL9215N46O0LIljBoFjzwiSY1FMRrhyk44+5PWfibxBKQn5byvrTO4BWttZgIfz+yqLQ2ohCgJuic4tWrV4sCBA8THx/Pzzz8zaNAgNm/eXOjjjRkzhlGjRpnuJyQkUKVKlaIIVYi7+vdfLalZuxbi4sy31aypVT099pjWC0qSGguRfhOu7oPLO+DMQog7DBhz2NEAPq0hoDN4P6j1cJKB9ITQje4JjoODAzVq1ACgSZMm7Nmzh88//5z+/fuTmppKXFyc2VWcmJgY/P39cz2eo6Mjjo6OxR22ECbXrmlJzZw5EB9vvs3ZGdq21eaBatNGkppSz2iES1u0xsCXtsL1U5CRAmRk39fWBdyqg29rCOwLPm2k0ZQQpYjuCc6djEYjKSkpNGnSBHt7ezZs2ECfPn0ACA8P59y5czRv3lznKEVZt2ePVvW0aROsXw/ptw1V4uqqJTVvvCFJTamXlqhNQHn6e4g7mPsklE7+2uB5nnXAuSIEPgFO3iUerhAi/3RNcMaMGUPXrl0JDAzk+vXrLF68mE2bNvHHH3/g6enJkCFDGDVqFOXLl8fDw4Phw4fTvHnzfPegEqIo7dypjSi8YYP5GDUAdetC9erw2mvQurU+8Ym7MBohZgOc+xlSr2ltZ+L+BZVDdZOdK7jX1KqcggdCealTFMLS6JrgxMbGMnDgQKKiovD09KR+/fr88ccfPPzwwwBMnz4dGxsb+vTpYzbQnxAlZft2LanZuBESE823lSsHr78OffpALZmEufS5cVGbsynqD216g5RL5Hh1xrkiYAvlGkGl7tqowNKzSQiLV+rGwSlqMg6OKKjoaFi2TOvSfWd7dw8P6NgR3nxTG6dGlBLGDLj2j5bIXN6h9W66fjznfe3cwfsBqDkUKjwILpVLNlYhRL5YzTg4QuhFKe0KzSefwN9/w51DJ3l6QqdOWkPhxo31ibFMy0iF1CuQcgViNsPNC3AzSrsik3JZGw045Qo5Xp0x2IJzJa2KqVIPqPI4OLiX+CkIIUqeJDiiTMnI0OZ82rMHzp/X2tNs3Qo3b5rv9+CD2vQIffpA1aq6hGp9jBmQdE5ryGswaElJ6hW4sFKbkyn1mjYXU3oiZNwEY6r2OJVDD6YcGcC3jXZVxvtBbf4mz5BiOx0hROkmCY6wSMnJ2u36dbhyBQ4dgi1bICZGu3/tmnYlJilJ269cOW0ZF6ddsclJuXLQtSu8/TaEhpbo6Vie5MuQeEqbUynpPGQkg1MFLWlJuaLNt5SWABk3wJgCxnRyvMKSXwYbcCgP6Te0sq2zVtXk4Kl11a7cEyr3BnuXojpDIYSFkwRH6EopuHoVTp+GM2fAzU3rcn3lChw4oF1diY/XEpkbN7TRgNPTc09SchMVZX7fYAAHB61NTZcuMGYM1K5dVGdlQdJvalU8iWfgxlm4eVGr/kmO1RIJ54pawpJ8Ga7u0qqLchzkriBswLUKOHhrk0omnQUbe3Aop61z8tOe16WKNgpw+UZg76nFI4QQ+SQJjigyaWlatc/p03D2LFy4AJGRkDWQ9JUr2ki/+/drVUJpaVqV0b2wtwdvb22Syvh4LUHy8IDy5aFCBfD312blrlcPatTQ9i1XTnucRTJmaFU5qdcgNQ7S4gAj2JfTqnUybkLMRm1ix/QkbZyX1CuZ+8ffSiRSM5OW9Hudq80m85he2jgxjt5aknL9JNi7aePHuFQC58xkxa2ajO4rhCgRkuAI4NYVEaUgNVVLRi5e1G7nzmnLqCiIjdWuuAQHa/teuQInT2rJzL32xzMYwMcHAgO1RMTeXqty8vYGX18tWalUCYKCtOevWhXc3XUeniQjVauCybgJadch/piWdKQlaElG2nWtTUlaIth7aFcusvaN/lMbJTcjJbMaJxWMaaDSwNZVSxYybmrVMsnR3FMVz10ZtAa5to7aCL12rlrj3IqdbiUtN6PByQdcg7RExclfRu4VQpRakuAUVnIsJEZoP1xZl+xNA4Yp7dfepRLYOGj3U65qP3xZ+yhFWpqBhOu2XE+yxcHDD6PBiZRUAxfO3OTcmVSSbtiSdNOOpBs23Ey25UayLTeT7QgNNeDgZE9KioHwY/DfMXtSU21JTbPJvBlIS7chLc2GWjVTcHKy0Y570ZZzF5zIMBowGg0YFShlMCUmtraKjIz8/WDt3ZvXVoWtjcLBPgNnxzRq10wgpGYq3uVSseMGVy8lEeCTQBXfOIIqXqVa5csEVIjH3i4DXILApaL2+qXFa6PLKpX5mhpvlW8Y4UyQljCgIPU6XNtrem2190Td2t+lCrgGa+X0JLiyS9um0rUEIiMZjDe1ZMOxAjj7a9U3qXEQd0Br6Gq6ZR67OKUlQHLU3fcDsHHU4rV11m6JEVp1jsEebB0y26p4aeflGgz+HbSkxdEbsAWXQLBzKM6zEUKIEicJTiEtmbaSeT9V4szlqqRm2JOWbk96hj1pRjsyMuxIN9pRzfcUShlJSXckJj6E+BteWkJB1iWH4r/0cDHGNd/7ZmRkj8eAETvbdBztk3FxuIG703W8XONpGryH0MpH8Ha7gpPDTZQyEOwTQbBPBF6u8QW7qhKfebN0No6ZPX8yr4YYbMBgBzZ2WjWOkx941deSEBsnuLoHbJ0yr5g4a1dt7Fy1qh3nSlCuwa2k5Wa01qDWwUurYrIvD3Yy55oQQuRGEpxC2rA3lDX/tshznwNnCzZoir1dKo52qSgFyWlO2BiM2s3GiK1NBrY26djZZFCnynEqeMThaJ/KtetuXLzih4NdKg52qTjap+Jol4KTfTKO9sk0rHacit5xONilkXjDjktXXXFxvIGrUyKujjdwc0rCzTERN6dEfCqk4+Fu1I5hiMMhI0pLVAxgnowZwMlXq3KBzCqUqNv2cTbf37mi1gPGYNCuniSdu3Ucw23HBHAO0BIBg0E77vUT5tsNtyWHzpW0q2QYtKqc+P9y2DfruJXALShz3xS4diBzn6xqGUct6bB11qpfyjXUkg6DPcQfAXt37UqIvYfWhsTeE+y9tCobRx/t8dIIVgghSg1JcAqp68AWJDhqo946OZnfnJ21W7NmWmNXR8dbXZbd3LTJGN3ctPYjHh7azcUFbGwcgPxUFVQo7tMTQgghLJokOIXUq5d2E0IIIUTpI9fUhRBCCGF1JMERQgghhNWRBEcIIYQQVkcSHCGEEEJYHUlwhBBCCGF1JMERQgghhNWRBEcIIYQQVkcSHCGEEEJYHUlwhBBCCGF1JMERQgghhNWRBEcIIYQQVkcSHCGEEEJYHUlwhBBCCGF1JMERQgghhNWRBEcIIYQQVkcSHCGEEEJYHUlwhBBCCGF1JMERQgghhNWRBEcIIYQQVkcSHCGEEEJYHUlwhBBCCGF1JMERQgghhNWRBEcIIYQQVkcSHCGEEEJYHUlwhBBCCGF1JMERQgghhNWRBEcIIYQQVkcSHCGEEEJYHUlwhBBCCGF1JMERQgghhNWRBEcIIYQQVkcSHCGEEEJYHUlwhBBCCGF1JMERQgghhNWRBEcIIYQQVkcSHCGEEEJYHUlwhBBCCGF1JMERQgghhNWRBEcIIYQQVkfXBGfSpEk0a9YMd3d3fH196dWrF+Hh4Wb7JCcnExYWhre3N25ubvTp04eYmBidIhZCCCGEJdA1wdm8eTNhYWHs3LmTdevWkZaWRqdOnUhKSjLt8+qrr7Jq1SqWLl3K5s2biYyMpHfv3jpGLYQQQojSzqCUUnoHkeXSpUv4+vqyefNm2rRpQ3x8PD4+PixevJjHH38cgGPHjlG7dm127NjBgw8+eNdjJiQk4OnpSXx8PB4eHsV9CkIIIYQoAvf6+12q2uDEx8cDUL58eQD27dtHWloaHTt2NO0TEhJCYGAgO3bs0CVGIYQQQpR+dnoHkMVoNDJy5EhatmxJ3bp1AYiOjsbBwQEvLy+zff38/IiOjs7xOCkpKaSkpJjuZyVNCQkJxRO4EEIIIYpc1u92YSuaSk2CExYWxuHDh9m2bds9HWfSpEmMHz8+2/oqVarc03GFEEIIUfKuX7+Op6dngR9XKhKcYcOGsXr1arZs2ULlypVN6/39/UlNTSUuLs7sKk5MTAz+/v45HmvMmDGMGjXKdN9oNHL16lW8vb0xGAxFFnNCQgJVqlTh/PnzVtu2x9rP0drPD6z/HOX8LJ+1n6OcX+Eppbh+/ToBAQGFeryuCY5SiuHDh7N8+XI2bdpEcHCw2fYmTZpgb2/Phg0b6NOnDwDh4eGcO3eO5s2b53hMR0dHHB0dzdbdWcVVlDw8PKzyQ3s7az9Haz8/sP5zlPOzfNZ+jnJ+hVOYKzdZdE1wwsLCWLx4MStXrsTd3d3UrsbT0xNnZ2c8PT0ZMmQIo0aNonz58nh4eDB8+HCaN2+erx5UQgghhCibdE1wZs2aBUC7du3M1s+dO5fBgwcDMH36dGxsbOjTpw8pKSl07tyZmTNnlnCkQgghhLAkuldR3Y2TkxMzZsxgxowZJRBR/jk6OjJ27Nhs1WHWxNrP0drPD6z/HOX8LJ+1n6Ocn35K1UB/QgghhBBFoVQN9CeEEEIIURQkwRFCCCGE1ZEERwghhBBWRxIcIYQQQlgdSXAKacaMGVStWhUnJyceeOABdu/erXdIRWbLli306NGDgIAADAYDK1as0DukIjVp0iSaNWuGu7s7vr6+9OrVi/DwcL3DKjKzZs2ifv36poG3mjdvzpo1a/QOq9h8/PHHGAwGRo4cqXcoRWbcuHEYDAazW0hIiN5hFamLFy/y1FNP4e3tjbOzM/Xq1WPv3r16h1Vkqlatmu09NBgMhIWF6R1akcjIyOC9994jODgYZ2dnqlevzocffljoeaOKgyQ4hfDTTz8xatQoxo4dyz///EODBg3o3LkzsbGxeodWJJKSkmjQoEGp65pfVDZv3kxYWBg7d+5k3bp1pKWl0alTJ5KSkvQOrUhUrlyZjz/+mH379rF3717at29Pz549+e+///QOrcjt2bOHr7/+mvr16+sdSpELDQ0lKirKdLvXefpKk2vXrtGyZUvs7e1Zs2YNR44c4dNPP6VcuXJ6h1Zk9uzZY/b+rVu3DoC+ffvqHFnRmDx5MrNmzeKrr77i6NGjTJ48mSlTpvDll1/qHdotShTY/fffr8LCwkz3MzIyVEBAgJo0aZKOURUPQC1fvlzvMIpVbGysAtTmzZv1DqXYlCtXTv3vf//TO4widf36dVWzZk21bt061bZtW/XKK6/oHVKRGTt2rGrQoIHeYRSbN998U7Vq1UrvMErUK6+8oqpXr66MRqPeoRSJbt26qWeffdZsXe/evdWAAQN0iig7uYJTQKmpqezbt4+OHTua1tnY2NCxY0d27NihY2SisOLj4wEoX768zpEUvYyMDH788UeSkpJynb/NUoWFhdGtWzezv0VrcuLECQICAqhWrRoDBgzg3LlzeodUZH799VeaNm1K37598fX1pVGjRnzzzTd6h1VsUlNTWbhwIc8++2yRTvqspxYtWrBhwwaOHz8OwMGDB9m2bRtdu3bVObJbSsVs4pbk8uXLZGRk4OfnZ7bez8+PY8eO6RSVKCyj0cjIkSNp2bIldevW1TucInPo0CGaN29OcnIybm5uLF++nDp16ugdVpH58ccf+eeff9izZ4/eoRSLBx54gHnz5lGrVi2ioqIYP348rVu35vDhw7i7u+sd3j07ffo0s2bNYtSoUbz99tvs2bOHESNG4ODgwKBBg/QOr8itWLGCuLg40xRE1uCtt94iISGBkJAQbG1tycjIYMKECQwYMEDv0EwkwRFlWlhYGIcPH7aq9g0AtWrV4sCBA8THx/Pzzz8zaNAgNm/ebBVJzvnz53nllVdYt24dTk5OeodTLG7/L7h+/fo88MADBAUFsWTJEoYMGaJjZEXDaDTStGlTJk6cCECjRo04fPgws2fPtsoE59tvv6Vr164EBAToHUqRWbJkCYsWLWLx4sWEhoZy4MABRo4cSUBAQKl5DyXBKaAKFSpga2tLTEyM2fqYmBj8/f11ikoUxrBhw1i9ejVbtmyhcuXKeodTpBwcHKhRowYATZo0Yc+ePXz++ed8/fXXOkd27/bt20dsbCyNGzc2rcvIyGDLli189dVXpKSkYGtrq2OERc/Ly4v77ruPkydP6h1KkahYsWK2ZLt27dr88ssvOkVUfM6ePcv69etZtmyZ3qEUqdGjR/PWW2/xxBNPAFCvXj3Onj3LpEmTSk2CI21wCsjBwYEmTZqwYcMG0zqj0ciGDRusro2DtVJKMWzYMJYvX85ff/1FcHCw3iEVO6PRSEpKit5hFIkOHTpw6NAhDhw4YLo1bdqUAQMGcODAAatLbgASExM5deoUFStW1DuUItGyZctsQzMcP36coKAgnSIqPnPnzsXX15du3brpHUqRunHjBjY25imEra0tRqNRp4iykys4hTBq1CgGDRpE06ZNuf/++/nss89ISkrimWee0Tu0IpGYmGj2n2JERAQHDhygfPnyBAYG6hhZ0QgLC2Px4sWsXLkSd3d3oqOjAfD09MTZ2Vnn6O7dmDFj6Nq1K4GBgVy/fp3FixezadMm/vjjD71DKxLu7u7Z2ku5urri7e1tNe2oXn/9dXr06EFQUBCRkZGMHTsWW1tbnnzySb1DKxKvvvoqLVq0YOLEifTr14/du3czZ84c5syZo3doRcpoNDJ37lwGDRqEnZ11/dz26NGDCRMmEBgYSGhoKPv372fatGk8++yzeod2i97duCzVl19+qQIDA5WDg4O6//771c6dO/UOqchs3LhRAdlugwYN0ju0IpHTuQFq7ty5eodWJJ599lkVFBSkHBwclI+Pj+rQoYP6888/9Q6rWFlbN/H+/furihUrKgcHB1WpUiXVv39/dfLkSb3DKlKrVq1SdevWVY6OjiokJETNmTNH75CK3B9//KEAFR4erncoRS4hIUG98sorKjAwUDk5Oalq1aqpd955R6WkpOgdmolBqVI07KAQQgghRBGQNjhCCCGEsDqS4AghhBDC6kiCI4QQQgirIwmOEEIIIayOJDhCCCGEsDqS4AghhBDC6kiCI4QQQgirIwmOEELkw5kzZzAYDBw4cEDvUIQQ+SAJjhAiVzt27MDW1rZQ8+iMGzeOhg0bFn1QQgiRD5LgCCFy9e233zJ8+HC2bNlCZGSk3uFYvNTUVL1DEKLMkARHCJGjxMREfvrpJ1566SW6devGvHnzTNvmzZuHl5eX2f4rVqzAYDCYto8fP56DBw9iMBgwGAymx587d46ePXvi5uaGh4cH/fr1IyYmxuxYK1eupHHjxjg5OVGtWjXGjx9Penq6abvBYOB///sfjz32GC4uLtSsWZNff/3V7Bj//fcf3bt3x8PDA3d3d1q3bs2pU6cAbRLEDz74gMqVK+Po6EjDhg1Zu3at2eN3795No0aNcHJyomnTpuzfvz/ba3T48GG6du2Km5sbfn5+PP3001y+fNm0vV27dgwbNoyRI0dSoUIFOnfunL8XXwhxzyTBEULkaMmSJYSEhFCrVi2eeuopvvvuO/I7dV3//v157bXXCA0NJSoqiqioKPr374/RaKRnz55cvXqVzZs3s27dOk6fPk3//v1Nj926dSsDBw7klVde4ciRI3z99dfMmzePCRMmmD3H+PHj6devH//++y+PPPIIAwYM4OrVqwBcvHiRNm3a4OjoyF9//cW+fft49tlnTUnS559/zqeffsrUqVP5999/6dy5M48++ignTpwAtOSue/fu1KlTh3379jFu3Dhef/11s+ePi4ujffv2NGrUiL1797J27VpiYmLo16+f2X7z58/HwcGB7du3M3v27IK9CUKIwtN5sk8hRCnVokUL9dlnnymllEpLS1MVKlRQGzduVEopNXfuXOXp6Wm2//Lly9XtXyljx45VDRo0MNvnzz//VLa2turcuXOmdf/9958C1O7du5VSSnXo0EFNnDjR7HELFixQFStWNN0H1Lvvvmu6n5iYqAC1Zs0apZRSY8aMUcHBwSo1NTXHcwsICFATJkwwW9esWTP18ssvK6WU+vrrr5W3t7e6efOmafusWbMUoPbv36+UUurDDz9UnTp1MjvG+fPnzWaPbtu2rWrUqFGOMQghipdcwRFCZBMeHs7u3bt58sknAbCzs6N///58++2393Tco0ePUqVKFapUqWJaV6dOHby8vDh69CgABw8e5IMPPsDNzc10e/7554mKiuLGjRumx9WvX99UdnV1xcPDg9jYWAAOHDhA69atsbe3zxZDQkICkZGRtGzZ0mx9y5YtTTEcPXqU+vXr4+TkZNrevHlzs/0PHjzIxo0bzeIMCQkBMFWFATRp0qRgL5IQokjY6R2AEKL0+fbbb0lPTycgIMC0TimFo6MjX331FTY2Ntmqq9LS0orkuRMTExk/fjy9e/fOtu32hOPO5MVgMGA0GgFwdnYukljykpiYSI8ePZg8eXK2bRUrVjSVXV1diz0WIUR2kuAIIcykp6fz/fff8+mnn9KpUyezbb169eKHH34gKCiI69evk5SUZPoBv3N8GAcHBzIyMszW1a5dm/Pnz3P+/HnTVZwjR44QFxdHnTp1AGjcuDHh4eHUqFGj0OdQv3595s+fT1paWrZEyMPDg4CAALZv307btm1N67dv3879999vinPBggUkJyebkqqdO3eaHadx48b88ssvVK1aFTs7+SoVotTRu45MCFG6LF++XDk4OKi4uLhs29544w3VtGlTdeXKFeXq6qpGjBihTp48qRYtWqQCAgLM2uAsWrRIubq6qv3796tLly6p5ORkZTQaVcOGDVXr1q3Vvn371K5du1STJk1U27ZtTY9bu3atsrOz+//27dgltSgO4Ph9g8EttAvZ1OrgZpOD0NoQEmi1Ofgf6Obg4n/Q7iK05BJ3CvwPopoEDbnY0BA0tLU0qN+GxxOEB+/x3vB4l+9nPQfOgbN8h9+h1+sxmUx4enri+vqabre73hMEAXEcb9xtd3eXwWAAwPv7O3t7e9TrdR4fH0mShKurK2azGQCXl5fkcjmGwyGz2YxOp0MmkyFJEgA+Pj7I5/M0Gg2m0ym3t7cUCoWNGZzX11f29/c5Pz/n4eGB+XzOaDSi2WyyWCyA7zM4rVbrL19E0p8wcCRtqFarnJyc/HTt/v6eIAgYj8fEcUyhUCAMQ6rVKv1+fyNwPj8/OTs7I4oigiBYx8fLywunp6fs7OyQzWa5uLjg7e1t45zRaESlUiEMQ3K5HOVymX6/v17/VeAAjMdjjo+P2d7eJpvNcnR0xPPzMwDL5ZJer8fBwQGZTIZSqbQeUP7h7u6OUqnE1tYWh4eH3NzcbAQOQJIk1Go1oigiDEOKxSLtdpvVagUYONK/9A1+89+nJEnSf8JfVJIkKXUMHEmSlDoGjiRJSh0DR5IkpY6BI0mSUsfAkSRJqWPgSJKk1DFwJElS6hg4kiQpdQwcSZKUOgaOJElKHQNHkiSlzhfV+p51fNrOnQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plot the pythia perplexities & gpts\n",
    "plt.plot(all_model_perplexities2[0], label=\"GPT2-top-2-outliers\", color=\"orange\")\n",
    "plt.plot(all_model_perplexities[0], label=\"GPT2\", linestyle=\"--\", color=\"orange\")\n",
    "plt.plot(all_model_perplexities2[1], label=\"Pythia-top-2-outliers\", color=\"blue\")\n",
    "plt.plot(all_model_perplexities[1], label=\"Pythia\", linestyle=\"--\", color=\"blue\")\n",
    "plt.legend()\n",
    "plt.title(\"Perplexity of GPT2 and Pythia with and without top-2-outliers\")\n",
    "plt.xlabel(\"Autoencoder\")\n",
    "plt.ylabel(\"Perplexity\")\n",
    "plt.ylim(20, 80)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.67083740e-03,  2.05039978e-03,  6.40869141e-04,\n",
       "         1.07574463e-03, -2.60925293e-03,  4.00543213e-03,\n",
       "        -4.59289551e-03, -1.39236450e-03, -2.84675598e-01],\n",
       "       [-4.90188599e-04,  7.83538818e-03,  1.17053986e-02,\n",
       "         1.84993744e-02, -2.21633911e-03, -1.29508972e-01,\n",
       "        -8.25691223e-01,  3.09411621e+00,  5.08766113e+02]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_perplexity_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([27.59947968, 27.61652374, 27.64967537, 27.97204018, 28.64097404,\n",
       "        30.66466331, 32.76556015, 39.17483521, 77.51404572]),\n",
       " array([  27.50660706,   27.76299477,   28.44542885,   31.47120667,\n",
       "          41.49584579,   61.27206421,   98.97418976,  303.42489624,\n",
       "        4692.35351562])]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_model_perplexities # 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([27.59962654, 27.61610031, 27.64871979, 27.9709053 , 28.63993073,\n",
       "        30.66494179, 32.76259995, 39.17511368, 77.8780365 ]),\n",
       " array([  27.50711823,   27.75564194,   28.44044304,   31.42954636,\n",
       "          41.33651352,   61.02084732,   98.85749817,  294.18612671,\n",
       "        4537.15478516])]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_model_perplexities # 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([27.60115051, 27.61447334, 27.6490345 , 27.97096443, 28.6435833 ,\n",
       "        30.66065788, 32.77015305, 39.17622757, 77.79872131]),\n",
       " array([  27.50709724,   27.75515938,   28.43372345,   31.45270729,\n",
       "          41.49806213,   61.40157318,   99.79988098,  300.33078003,\n",
       "        4183.58740234])]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_model_perplexities # 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([27.59186172, 27.61104393, 27.64015579, 27.89469147, 28.61002541,\n",
       "        30.1843586 , 31.86545753, 35.27019119, 55.05447388]),\n",
       " array([  27.50718307,   27.75325966,   28.41473007,   31.3962326 ,\n",
       "          41.19856262,   60.96576309,   97.82530212,  288.10852051,\n",
       "        3631.71118164])]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_model_perplexities# 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outlier_dims = find_outliers(dataset, cache_name, model, num_outlier_dims=2)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Change these settings to load the correct autoencoder\n",
    "layer = 2\n",
    "setting = \"residual\"\n",
    "model_name = \"EleutherAI/pythia-70m-deduped\"\n",
    "device = torch.device(\"cuda:5\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = HookedTransformer.from_pretrained(model_name, device=device)\n",
    "# filename = \"outputs/20230720-192301-EleutherAI/pythia-70m-deduped-2/minirun9/autoencoders.pkl\"\n",
    "filename = \"outputs/20230720-192301-EleutherAI/pythia-70m-deduped-2/minirun5/autoencoders.pkl\"\n",
    "# filename = \"outputs/20230721-161919-EleutherAI/pythia-70m-deduped-2/minirun9/autoencoders.pkl\"\n",
    "# filename = \"outputs/20230721-161919-EleutherAI/pythia-70m-deduped-2/minirun3/autoencoders.pkl\"\n",
    "# filename = \"/home/mchorse/logan_sparse_coding/sparse_coding/ae2.pkl\"\n",
    "\n",
    "\n",
    "if setting == \"residual\":\n",
    "    cache_name = f\"blocks.{layer}.hook_resid_post\"\n",
    "    neurons = model.cfg.d_model\n",
    "elif setting == \"mlp\":\n",
    "    cache_name = f\"blocks.{layer}.mlp.hook_post\"\n",
    "    neurons = model.cfg.d_mlp\n",
    "else:\n",
    "    raise NotImplementedError\n",
    "\n",
    "\n",
    "# Load the pickle file\n",
    "with open(filename, 'rb') as file:\n",
    "    autoencoders = pickle.load(file)\n",
    "\n",
    "# Index for l1 value, usually only 1 value is available\n",
    "l1_index = -1\n",
    "dictionaries = [autoencoder.decoder.weight.data.T for autoencoder in autoencoders[l1_index]]\n",
    "for d in dictionaries:\n",
    "    print(d.shape)\n",
    "print(\"len of autoencoders: \", len(autoencoders))\n",
    "\n",
    "dict_index = 0\n",
    "smaller_dict, larger_dict = dictionaries[dict_index], dictionaries[dict_index+1]\n",
    "smaller_auto_encoder, larger_auto_encoder = autoencoders[l1_index][dict_index], autoencoders[l1_index][dict_index+1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformer_lens import HookedTransformer\n",
    "from torch import nn\n",
    "import pickle\n",
    "import numpy as np\n",
    "from torch.utils.data import DataLoader\n",
    "from datasets import Dataset, load_dataset\n",
    "from tqdm.auto import tqdm\n",
    "from einops import rearrange\n",
    "# autoencoder_filename = \"/home/mchorse/logan/sparse_coding/output_sweep_tied_mlpout_l1_r8/_9/learned_dicts.pt\"\n",
    "# autoencoder_filename = \"/home/mchorse/logan/sparse_coding/output_sweep_tied_mlpout_l2_r4/_19/learned_dicts.pt\"\n",
    "# autoencoder_filename = \"/home/mchorse/sparse_coding_hoagy/tiedlong_tied_residual_l5_r4/_10/learned_dicts.pt\"\n",
    "# autoencoder_filename = \"/home/mchorse/logan/sparse_coding/output_hoagy_dense_sweep_tied_residual_l4_r6/_99/learned_dicts.pt\"\n",
    "# autoencoder_filename = \"/home/mchorse/logan/sparse_coding/output_hoagy_dense_sweep_tied_residual_l4_r6/_0/learned_dicts.pt\"\n",
    "layer = 4\n",
    "\n",
    "autoencoder_filename = \"/home/mchorse/logan/sparse_coding/output_hoagy_dense_sweep_tied_residual_l4_r6/_0/learned_dicts.pt\"\n",
    "auto_num = 0 # Selects which specific autoencoder to use\n",
    "all_autoencoders = torch.load(autoencoder_filename)\n",
    "num_dictionaries = len(all_autoencoders)\n",
    "print(f\"Loaded {num_dictionaries} autoencoders\")\n",
    "autoencoder, hyperparams = all_autoencoders[auto_num]\n",
    "l1_alpha = hyperparams['l1_alpha']\n",
    "# autoencoder2, hyperparams2 = all_autoencoders[auto_num+1]\n",
    "smaller_dict = autoencoder.get_learned_dict()\n",
    "# larger_dict = autoencoder2.get_learned_dict()\n",
    "\n",
    "#Change these settings to load the correct autoencoder\n",
    "setting = \"residual\"\n",
    "# setting = \"attention\"\n",
    "# setting = \"mlp\"\n",
    "# setting = \"mlp_out\"\n",
    "model_name = \"EleutherAI/pythia-70m-deduped\"\n",
    "# model_name = \"EleutherAI/pythia-160m\"\n",
    "\n",
    "device = torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = HookedTransformer.from_pretrained(model_name, device=device)\n",
    "\n",
    "if setting == \"residual\":\n",
    "    cache_name = f\"blocks.{layer}.hook_resid_post\"\n",
    "    neurons = model.cfg.d_model\n",
    "elif setting == \"mlp\":\n",
    "    cache_name = f\"blocks.{layer}.mlp.hook_post\"\n",
    "    neurons = model.cfg.d_mlp\n",
    "elif setting == \"attention\":\n",
    "    cache_name = f\"blocks.{layer}.hook_attn_out\"\n",
    "    neurons = model.cfg.d_model\n",
    "elif setting == \"mlp_out\":\n",
    "    cache_name = f\"blocks.{layer}.hook_mlp_out\"\n",
    "    neurons = model.cfg.d_model\n",
    "else:\n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformer_lens import HookedTransformer\n",
    "import torch\n",
    "import numpy as np\n",
    "from torch.utils.data import DataLoader\n",
    "from datasets import Dataset, load_dataset\n",
    "from tqdm.auto import tqdm\n",
    "from einops import rearrange\n",
    "layer = 2\n",
    "setting = \"residual\"\n",
    "model_name = \"EleutherAI/pythia-70m-deduped\"\n",
    "device = torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = HookedTransformer.from_pretrained(model_name, device=device)\n",
    "\n",
    "if setting == \"residual\":\n",
    "    cache_name = f\"blocks.{layer}.hook_resid_post\"\n",
    "    neurons = model.cfg.d_model\n",
    "elif setting == \"mlp\":\n",
    "    cache_name = f\"blocks.{layer}.mlp.hook_post\"\n",
    "    neurons = model.cfg.d_mlp\n",
    "else:\n",
    "    raise NotImplementedError\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MCS\n",
    "Max cosine similarity between one dictionary & another one. If they learned the same feature, then they'll have high cosine similarity. \n",
    "\n",
    "If two dictionaries learned it, it's probably a real feature."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model activations & Dictionary Activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Downnload dataset\n",
    "dataset_name = \"NeelNanda/pile-10k\"\n",
    "token_amount= 256\n",
    "dataset = load_dataset(dataset_name, split=\"train\").map(\n",
    "    lambda x: model.tokenizer(x['text']),\n",
    "    batched=True,\n",
    ").filter(\n",
    "    lambda x: len(x['input_ids']) > token_amount\n",
    ").map(\n",
    "    lambda x: {'input_ids': x['input_ids'][:token_amount]}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_autoencoders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "def mlp_ablation_hook(value, hook, autoencoder):\n",
    "    # Rearrange to fit autoencoder\n",
    "    int_val = rearrange(value, 'b s h -> (b s) h')\n",
    "    # Run through the autoencoder\n",
    "    reconstruction = autoencoder.predict(int_val)\n",
    "    batch, seq_len, hidden_size = value.shape\n",
    "    reconstruction = rearrange(reconstruction, '(b s) h -> b s h', b=batch, s=seq_len)\n",
    "    return reconstruction\n",
    "n_chunks = 12\n",
    "chunk_size = 2\n",
    "l1_values = [1e-3, 9e-4, 8e-4, 7e-4]\n",
    "all_perplexities = np.zeros(4, (n_chunks//chunk_size))\n",
    "for chunk_trained in range(0,12,2): \n",
    "    autoencoder_filename = f\"/home/mchorse/logan/sparse_coding/output_hoagy_dense_sweep_tied_residual_l4_r6/_{chunk_trained}/learned_dicts.pt\"\n",
    "    all_autoencoders = torch.load(autoencoder_filename)\n",
    "    model = model.eval()\n",
    "    batch_size = 128\n",
    "    num_dictionaries = len(all_autoencoders)\n",
    "    l1_perplexity = np.zeros(num_dictionaries + 1)\n",
    "    all_l1s = np.zeros(num_dictionaries + 1)\n",
    "    for l1_index in range(len(all_autoencoders)):\n",
    "        # print(f\"L1 index: {l1_index}\")\n",
    "        autoencoder, hyperparams = all_autoencoders[l1_index]\n",
    "        autoencoder.to_device(device)\n",
    "        print(hyperparams)\n",
    "        # for dict_index in range(num_dictionaries + 1):\n",
    "        #     if(dict_index != num_dictionaries):\n",
    "        #         autoencoder, hyperparams = all_autoencoders[dict_index]\n",
    "        #         autoencoder.to_device(device)\n",
    "        #         print(hyperparams)\n",
    "        #     else:\n",
    "        #         continue\n",
    "        total_loss = 0\n",
    "        total_tokens = 0\n",
    "        with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "            dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "            for i, batch in enumerate(dl):\n",
    "                # Get model predictions\n",
    "                if(l1_index == num_dictionaries): # Full model\n",
    "                    loss = model(batch.to(device), return_type=\"loss\")\n",
    "                else:\n",
    "                    loss = model.run_with_hooks(batch.to(device), \n",
    "                        return_type=\"loss\",\n",
    "                        fwd_hooks=[(\n",
    "                            cache_name, \n",
    "                            partial(mlp_ablation_hook, autoencoder=autoencoder),\n",
    "                            )]\n",
    "                        )\n",
    "                total_loss += loss.item()\n",
    "        # Average\n",
    "        avg_neg_log_likelihood = total_loss / len(dl)\n",
    "\n",
    "        # Convert to tensor\n",
    "        avg_neg_log_likelihood = torch.tensor(avg_neg_log_likelihood, device=device)\n",
    "\n",
    "        # Exponentiate to compute perplexity\n",
    "        perplexity = torch.exp(avg_neg_log_likelihood)\n",
    "        if(l1_index == num_dictionaries):\n",
    "            print(\"Perplexity for full model: \", perplexity.item())\n",
    "            original_perplexity = perplexity.item()\n",
    "        else:\n",
    "            # print(f\"Perplexity for dict_size={dictionaries[dict_index].shape[0]}: {perplexity.item():.2f}\")\n",
    "            print(f\"{perplexity.item():.2f}\")\n",
    "        # l1_perplexity[l1_index] = perplexity.item()\n",
    "        all_perplexities[chunk_trained, l1_index] = perplexity.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_autoencoders[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "def mlp_ablation_hook(value, hook, autoencoder):\n",
    "    # Rearrange to fit autoencoder\n",
    "    int_val = rearrange(value, 'b s h -> (b s) h')\n",
    "    # Run through the autoencoder\n",
    "    reconstruction = autoencoder.predict(int_val)\n",
    "    batch, seq_len, hidden_size = value.shape\n",
    "    reconstruction = rearrange(reconstruction, '(b s) h -> b s h', b=batch, s=seq_len)\n",
    "    return reconstruction\n",
    "\n",
    "model = model.eval()\n",
    "batch_size = 128\n",
    "# for l1_index in range(len(autoencoders)):\n",
    "#     print(f\"L1 index: {l1_index}\")\n",
    "#     all_autoencoders = autoencoders[l1_index]\n",
    "all_autoencoders = all_autoencoders[3:]\n",
    "num_dictionaries = len(all_autoencoders)\n",
    "all_perplexities = np.zeros(num_dictionaries + 1)\n",
    "all_l1s = np.zeros(num_dictionaries + 1)\n",
    "for dict_index in range(num_dictionaries + 1):\n",
    "    if(dict_index != num_dictionaries):\n",
    "        autoencoder, hyperparams = all_autoencoders[dict_index]\n",
    "        autoencoder.to_device(device)\n",
    "        print(hyperparams)\n",
    "    else:\n",
    "        continue\n",
    "    total_loss = 0\n",
    "    total_tokens = 0\n",
    "    with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "        dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "        for i, batch in enumerate(dl):\n",
    "            # Get model predictions\n",
    "            if(dict_index == num_dictionaries): # Full model\n",
    "                loss = model(batch.to(device), return_type=\"loss\")\n",
    "            else:\n",
    "                loss = model.run_with_hooks(batch.to(device), \n",
    "                    return_type=\"loss\",\n",
    "                    fwd_hooks=[(\n",
    "                        cache_name, \n",
    "                        partial(mlp_ablation_hook, autoencoder=autoencoder),\n",
    "                        )]\n",
    "                    )\n",
    "            total_loss += loss.item()\n",
    "    # Average\n",
    "    avg_neg_log_likelihood = total_loss / len(dl)\n",
    "\n",
    "    # Convert to tensor\n",
    "    avg_neg_log_likelihood = torch.tensor(avg_neg_log_likelihood, device=device)\n",
    "\n",
    "    # Exponentiate to compute perplexity\n",
    "    perplexity = torch.exp(avg_neg_log_likelihood)\n",
    "    if(dict_index == num_dictionaries):\n",
    "        print(\"Perplexity for full model: \", perplexity.item())\n",
    "        original_perplexity = perplexity.item()\n",
    "    else:\n",
    "        # print(f\"Perplexity for dict_size={dictionaries[dict_index].shape[0]}: {perplexity.item():.2f}\")\n",
    "        print(f\"{perplexity.item():.2f}\")\n",
    "    all_perplexities[dict_index] = perplexity.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Run_9                                   Run_5 (~converged on reconstruction loss. This is check connection between recon & perplexity and MMCS & perplexity)      \n",
    "Perplexity for full model:  25.12\n",
    "L1 index: 0 (3e-3)                                Sparsity\n",
    "Perplexity for dict_size=2048: 169.76   176.08    24\n",
    "Perplexity for dict_size=4096: 138.02   171.45    26\n",
    "Perplexity for dict_size=8192: 127.86   124.92    26\n",
    "L1 index: 1 (1e-3)\n",
    "Perplexity for dict_size=2048: 59.14    59.96    73\n",
    "Perplexity for dict_size=4096: 53.14    55.28    72\n",
    "Perplexity for dict_size=8192: 51.90    51.16    71\n",
    "L1 index: 2 (8e-4)\n",
    "Perplexity for dict_size=2048: 48.80    48.25    96\n",
    "Perplexity for dict_size=4096: 47.09    45.67    95\n",
    "Perplexity for dict_size=8192: 55.75    44.87    93\n",
    "\n",
    "L1 index: 3 (6e-4)                      Run_3 (converged on reconstruction loss)\n",
    "Perplexity for dict_size=2048: 41.49    39.21    127\n",
    "Perplexity for dict_size=4096: 42.53    38.97    127\n",
    "Perplexity for dict_size=8192: 39.87    39.46    124\n",
    "L1 index: 4 (4e-4)\n",
    "Perplexity for dict_size=2048: 30.51    31.06    245\n",
    "Perplexity for dict_size=4096: 49.06    32.59    197 #NOTE: LOGAN DO A DIFF DICT HERE YOU LOSER\n",
    "Perplexity for dict_size=8192: 33.24    32.81    181\n",
    "L1 index: 5 (2e-4)\n",
    "Perplexity for dict_size=2048: 26.19    26.35    387\n",
    "Perplexity for dict_size=4096: 26.23    26.27    407\n",
    "Perplexity for dict_size=8192: 26.27    27.28    377\n",
    "'''\n",
    "\n",
    "'''\n",
    "L1 index   Perplexity  Layer  n_chunks   Size\n",
    "1e-3            38.8     4      100       3k\n",
    "1e-3            38.8     4      80        3k\n",
    "1e-3            38.8     4      60        3k\n",
    "1e-3            38.3     4      40        3k\n",
    "1e-3            38.82    4      20        3k\n",
    "1e-3            50.61    4      1         3k\n",
    "1e-3            49       4      80        2k\n",
    "7e-4            38       4      80        2k\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Run_3 (~converged on reconstruction loss. This is check connection between recon & perplexity and MMCS & perplexity)\n",
    "Run_9\n",
    "Perplexity for full model:  25.12\n",
    "L1 index: 0 (3e-3)\n",
    "L1 index: 1 (1e-3)\n",
    "L1 index: 2 (8e-4)\n",
    "L1 index: 3 (6e-4)\n",
    "Perplexity for dict_size=2048: 41.49\n",
    "Perplexity for dict_size=4096: 42.53\n",
    "Perplexity for dict_size=8192: 39.87\n",
    "L1 index: 4 (4e-4)\n",
    "Perplexity for dict_size=2048: 30.51\n",
    "Perplexity for dict_size=4096: 49.06\n",
    "Perplexity for dict_size=8192: 33.24\n",
    "L1 index: 5 (2e-4)\n",
    "Perplexity for dict_size=2048: 26.19\n",
    "Perplexity for dict_size=4096: 26.23\n",
    "Perplexity for dict_size=8192: 26.27\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "import torch\n",
    "from transformer_lens import HookedTransformer\n",
    "import numpy as np \n",
    "from torch import nn\n",
    "import pickle\n",
    "\n",
    "def mlp_ablation_hook(value, hook, autoencoder):\n",
    "    # Rearrange to fit autoencoder\n",
    "    int_val = rearrange(value, 'b s h -> (b s) h')\n",
    "    # Run through the autoencoder\n",
    "    reconstruction = autoencoder.predict(int_val)\n",
    "    batch, seq_len, hidden_size = value.shape\n",
    "    reconstruction = rearrange(reconstruction, '(b s) h -> b s h', b=batch, s=seq_len)\n",
    "    return reconstruction\n",
    "\n",
    "all_autoencoders = torch.load(\"/home/mchorse/sparse_coding_aidan_new/output_4_rd_deep/_7/learned_dicts.pt\")\n",
    "num_dictionaries = len(all_autoencoders)\n",
    "all_perplexities = np.zeros(num_dictionaries + 1)\n",
    "all_l1s = np.zeros(num_dictionaries + 1)\n",
    "model = model.eval()\n",
    "batch_size = 32\n",
    "for dict_index in range(num_dictionaries + 1):\n",
    "    autoencoder, hyperparams = all_autoencoders[dict_index]\n",
    "    l1_alpha = hyperparams['l1_alpha']\n",
    "    autoencoder.to_device(device)\n",
    "    total_loss = 0\n",
    "    total_tokens = 0\n",
    "    with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "        dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "        for i, batch in enumerate(dl):\n",
    "            # Get model predictions\n",
    "            if(dict_index == num_dictionaries): # Full model\n",
    "                loss = model(batch.to(device), return_type=\"loss\")\n",
    "            else:\n",
    "                loss = model.run_with_hooks(batch.to(device), \n",
    "                    return_type=\"loss\",\n",
    "                    fwd_hooks=[(\n",
    "                        cache_name, \n",
    "                        partial(mlp_ablation_hook, autoencoder=autoencoder),\n",
    "                        )]\n",
    "                    )\n",
    "            total_loss += loss.item()\n",
    "    # Average\n",
    "    avg_neg_log_likelihood = total_loss / len(dl)\n",
    "\n",
    "    # Convert to tensor\n",
    "    avg_neg_log_likelihood = torch.tensor(avg_neg_log_likelihood, device=device)\n",
    "\n",
    "    # Exponentiate to compute perplexity\n",
    "    perplexity = torch.exp(avg_neg_log_likelihood)\n",
    "    if(dict_index == num_dictionaries):\n",
    "        print(\"Perplexity for full model: \", perplexity.item())\n",
    "    else:\n",
    "        print(f\"Perplexity for l1={l1_alpha:.2E}: {perplexity.item():.2f}\")\n",
    "    all_perplexities[dict_index] = perplexity.item()\n",
    "    all_l1s[dict_index] = l1_alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "from transformer_lens import HookedTransformer\n",
    "import torch\n",
    "import numpy as np\n",
    "from torch.utils.data import DataLoader\n",
    "from datasets import Dataset, load_dataset\n",
    "from tqdm.auto import tqdm\n",
    "from einops import rearrange\n",
    "\n",
    "# Load model\n",
    "model_name = \"EleutherAI/pythia-70m-deduped\"\n",
    "device = torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = HookedTransformer.from_pretrained(model_name, device=device)\n",
    "model = model.eval()\n",
    "\n",
    "# Load autoencoders\n",
    "all_autoencoders = torch.load(\"/home/mchorse/sparse_coding_aidan_new/output_4_rd_deep/_7/learned_dicts.pt\")\n",
    "num_dictionaries = len(all_autoencoders)\n",
    "\n",
    "# Set layer & cache name\n",
    "layer = 2\n",
    "setting = \"residual\"\n",
    "if setting == \"residual\":\n",
    "    cache_name = f\"blocks.{layer}.hook_resid_post\"\n",
    "    neurons = model.cfg.d_model\n",
    "elif setting == \"mlp\":\n",
    "    cache_name = f\"blocks.{layer}.mlp.hook_post\"\n",
    "    neurons = model.cfg.d_mlp\n",
    "else:\n",
    "    raise NotImplementedError\n",
    "\n",
    "# Download dataset\n",
    "batch_size = 32\n",
    "dataset_name = \"NeelNanda/pile-10k\"\n",
    "token_amount= 256\n",
    "dataset = load_dataset(dataset_name, split=\"train\").map(\n",
    "    lambda x: model.tokenizer(x['text']),\n",
    "    batched=True,\n",
    ").filter(\n",
    "    lambda x: len(x['input_ids']) > token_amount\n",
    ").map(\n",
    "    lambda x: {'input_ids': x['input_ids'][:token_amount]}\n",
    ")\n",
    "\n",
    "# Define function to replace activations with reconstruction\n",
    "def replace_with_reconstruction(value, hook, autoencoder):\n",
    "    # Rearrange to fit autoencoder\n",
    "    int_val = rearrange(value, 'b s h -> (b s) h')\n",
    "    # Run through the autoencoder\n",
    "    reconstruction = autoencoder.predict(int_val)\n",
    "    batch, seq_len, hidden_size = value.shape\n",
    "    reconstruction = rearrange(reconstruction, '(b s) h -> b s h', b=batch, s=seq_len)\n",
    "    return reconstruction\n",
    "\n",
    "with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "    dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "    # Calculate Original Perplexity ie no intervention/no dictionary\n",
    "    total_loss = 0\n",
    "    for i, batch in enumerate(dl):\n",
    "        loss = model(batch.to(device), return_type=\"loss\")\n",
    "        total_loss += loss.item()\n",
    "    # Average\n",
    "    avg_neg_log_likelihood = total_loss / len(dl)\n",
    "    # Convert to tensor\n",
    "    avg_neg_log_likelihood = torch.tensor(avg_neg_log_likelihood, device=device)\n",
    "    # Exponentiate to compute perplexity\n",
    "    original_perplexity = torch.exp(avg_neg_log_likelihood)\n",
    "    print(\"Perplexity for original model: \", original_perplexity.item())\n",
    "\n",
    "    # Compute perplexity for each dictionary\n",
    "    all_perplexities = np.zeros(num_dictionaries)\n",
    "    all_l1s = np.zeros(num_dictionaries)\n",
    "    # Calculate Perplexity for each dictionary\n",
    "    for dict_index in range(num_dictionaries):\n",
    "        autoencoder, hyperparams = all_autoencoders[dict_index]\n",
    "        l1_alpha = hyperparams['l1_alpha']\n",
    "        autoencoder.to_device(device)\n",
    "        total_loss = 0\n",
    "        for i, batch in enumerate(dl):\n",
    "            # Perplexity with reconstructed activations\n",
    "            loss = model.run_with_hooks(batch.to(device), \n",
    "                return_type=\"loss\",\n",
    "                fwd_hooks=[(\n",
    "                    cache_name, # intermediate activation that we intervene on\n",
    "                    partial(replace_with_reconstruction, autoencoder=autoencoder), # function to apply to cache_name\n",
    "                    )]\n",
    "                )\n",
    "            total_loss += loss.item()\n",
    "        # Average\n",
    "        avg_neg_log_likelihood = total_loss / len(dl)\n",
    "        # Convert to tensor\n",
    "        avg_neg_log_likelihood = torch.tensor(avg_neg_log_likelihood, device=device)\n",
    "        # Exponentiate to compute perplexity\n",
    "        perplexity = torch.exp(avg_neg_log_likelihood)\n",
    "        print(f\"Perplexity for l1={l1_alpha:.2E}: {perplexity.item():.2f}\")\n",
    "        all_perplexities[dict_index] = perplexity.item()\n",
    "        all_l1s[dict_index] = l1_alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot perplexities against their l1s\n",
    "import matplotlib.pyplot as plt\n",
    "plt.scatter(all_l1s, all_perplexities)\n",
    "plt.xlabel('l1')\n",
    "plt.ylabel('perplexity')\n",
    "plt.yscale('log')\n",
    "plt.title('Pythia-70m Residual Layer 2')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_perplexities[:32].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot perplexities against their l1s\n",
    "N = 15\n",
    "import matplotlib.pyplot as plt\n",
    "plt.scatter(all_l1s[:N], all_perplexities[:N])\n",
    "plt.xlabel('l1')\n",
    "plt.ylabel('perplexity')\n",
    "# plt.yscale('log')\n",
    "plt.title('Pythia-70m Residual Layer 2')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_l1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_perplexities.squeeze().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "# Now we can use the model to get the activations\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm.auto import tqdm\n",
    "from einops import rearrange\n",
    "\n",
    "\n",
    "outlier_dimensions = torch.tensor([216, 117, 437, 341, 215,  58, 477, 113, 265,  41, 145, 298, 61, 359, 355, 290])\n",
    "outlier_dimensions = torch.tensor([111, 156,  71,  47, 369, 473, 396, 197,  48, 258])\n",
    "perplexities_individual = torch.zeros(len(outlier_dimensions))\n",
    "l1_index = 1\n",
    "num_dictionaries = len(autoencoders[l1_index])\n",
    "# for dict_index in range(num_dictionaries):\n",
    "for dict_index in range(len(outlier_dimensions)):\n",
    "    def ablate_residual_dimension(value, hook):\n",
    "        # Rearrange to fit autoencoder\n",
    "        value[:, :, outlier_dimensions[dict_index:dict_index+1]] = 0.0\n",
    "        return value\n",
    "    model = model.eval()\n",
    "    total_loss = 0\n",
    "    total_tokens = 0\n",
    "    batch_size = 128\n",
    "    with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "        dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "        for i, batch in enumerate(dl):\n",
    "            # Get model predictions\n",
    "            loss = model.run_with_hooks(batch.to(device), \n",
    "                return_type=\"loss\",\n",
    "                fwd_hooks=[(\n",
    "                    cache_name, \n",
    "                    ablate_residual_dimension\n",
    "                    )]\n",
    "                )\n",
    "            total_loss += loss.item()\n",
    "\n",
    "    # Average\n",
    "    avg_neg_log_likelihood = total_loss / len(dl)\n",
    "\n",
    "    # Convert to tensor\n",
    "    avg_neg_log_likelihood = torch.tensor(avg_neg_log_likelihood, device=device)\n",
    "\n",
    "    # Exponentiate to compute perplexity\n",
    "    perplexity = torch.exp(avg_neg_log_likelihood)\n",
    "    print(f\"Perplexity for Outlier_dimension={dict_index}: {perplexity.item():.2f}\")\n",
    "    perplexities_individual[dict_index] = perplexity.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot perplexities: y-axis perplexity, x-axis: number of outlier dimensions ablated\n",
    "# Shift perplexities_individual to the right by 1 & assign it's first value to perplexities[0]\n",
    "perplexities_new = torch.zeros(len(outlier_dimensions)+1)\n",
    "perplexities_new[1:] = perplexities_individual\n",
    "perplexities_new[0] = perplexities[0]\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.plot(perplexities.cpu().numpy(), label=\"Cumulative Ablation\")\n",
    "plt.plot(perplexities_new[:-1].cpu().numpy(), label=\"Individual Ablation\")\n",
    "plt.xlabel(\"Number of outlier dimensions ablated\")\n",
    "plt.ylabel(\"Perplexity\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we can use the model to get the activations\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm.auto import tqdm\n",
    "from einops import rearrange\n",
    "# neurons = model.W_in.shape[-1]\n",
    "neurons = model.cfg.d_model\n",
    "datapoints = dataset.num_rows\n",
    "batch_size = 64\n",
    "neuron_activations = torch.zeros((datapoints*token_amount, neurons))\n",
    "dictionary_activations = torch.zeros((datapoints*token_amount, smaller_dict_features))\n",
    "smaller_auto_encoder = smaller_auto_encoder.to(device)\n",
    "\n",
    "with torch.no_grad(), dataset.formatted_as(\"pt\"):\n",
    "    dl = DataLoader(dataset[\"input_ids\"], batch_size=batch_size)\n",
    "    for i, batch in enumerate(tqdm(dl)):\n",
    "        _, cache = model.run_with_cache(batch.to(device))\n",
    "        batched_neuron_activations = rearrange(cache[cache_name], \"b s n -> (b s) n\" )\n",
    "        neuron_activations[i*batch_size*token_amount:(i+1)*batch_size*token_amount,:] = batched_neuron_activations.cpu()\n",
    "        reconstruction, batched_dictionary_activations = smaller_auto_encoder(batched_neuron_activations)\n",
    "        dictionary_activations[i*batch_size*token_amount:(i+1)*batch_size*token_amount,:] = batched_dictionary_activations.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary_activations[:10000,:].count_nonzero(axis=1).float().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Activation Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Useful Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from circuitsvis.activations import text_neuron_activations\n",
    "# Get the activations for the best dict features\n",
    "def get_feature_datapoints(feature_index, dictionary_activations, dataset, k=10, setting=\"max\"):\n",
    "    best_feature_activations = dictionary_activations[:, feature_index]\n",
    "    # Sort the features by activation, get the indices\n",
    "    if setting==\"max\":\n",
    "        found_indices = torch.argsort(best_feature_activations, descending=True)[:k]\n",
    "    elif setting==\"uniform\":\n",
    "        min_value = torch.min(best_feature_activations)\n",
    "        max_value = torch.max(best_feature_activations)\n",
    "\n",
    "        # Define the number of bins\n",
    "        num_bins = k\n",
    "\n",
    "        # Calculate the bin boundaries as linear interpolation between min and max\n",
    "        bin_boundaries = torch.linspace(min_value, max_value, num_bins + 1)\n",
    "\n",
    "        # Assign each activation to its respective bin\n",
    "        bins = torch.bucketize(best_feature_activations, bin_boundaries)\n",
    "\n",
    "        # Initialize a list to store the sampled indices\n",
    "        sampled_indices = []\n",
    "\n",
    "        # Sample from each bin\n",
    "        for bin_idx in torch.unique(bins):\n",
    "            # Get the indices corresponding to the current bin\n",
    "            bin_indices = torch.nonzero(bins == bin_idx, as_tuple=False).squeeze(dim=1)\n",
    "            \n",
    "            # Randomly sample from the current bin\n",
    "            sampled_indices.extend(np.random.choice(bin_indices, size=1, replace=False))\n",
    "\n",
    "        # Convert the sampled indices to a PyTorch tensor & reverse order\n",
    "        found_indices = torch.tensor(sampled_indices).long().flip(dims=[0])\n",
    "    else: # random\n",
    "        # get nonzero indices\n",
    "        nonzero_indices = torch.nonzero(best_feature_activations)[:, 0]\n",
    "        # shuffle\n",
    "        shuffled_indices = nonzero_indices[torch.randperm(nonzero_indices.shape[0])]\n",
    "        found_indices = shuffled_indices[:k]\n",
    "    datapoint_indices =[np.unravel_index(i, (datapoints, token_amount)) for i in found_indices]\n",
    "    text_list = []\n",
    "    full_text = []\n",
    "    token_list = []\n",
    "    full_token_list = []\n",
    "    for md, s_ind in datapoint_indices:\n",
    "        md = int(md)\n",
    "        s_ind = int(s_ind)\n",
    "        full_tok = torch.tensor(dataset[md][\"input_ids\"])\n",
    "        full_text.append(model.tokenizer.decode(full_tok))\n",
    "        tok = dataset[md][\"input_ids\"][:s_ind+1]\n",
    "        text = model.tokenizer.decode(tok)\n",
    "        text_list.append(text)\n",
    "        token_list.append(tok)\n",
    "        full_token_list.append(full_tok)\n",
    "    return text_list, full_text, token_list, full_token_list\n",
    "\n",
    "def get_neuron_activation(token, feature, model, setting=\"dictionary_basis\"):\n",
    "    with torch.no_grad():\n",
    "        _, cache = model.run_with_cache(token.to(model.cfg.device))\n",
    "        neuron_act_batch = cache[cache_name]\n",
    "        if setting==\"dictionary_basis\":\n",
    "            _, act = smaller_auto_encoder(neuron_act_batch)\n",
    "            return act[0, :, feature].tolist()\n",
    "        else: # neuron/residual basis\n",
    "            return neuron_act_batch[0, :, feature].tolist()\n",
    "\n",
    "def ablate_text(text, feature, model, setting=\"plot\"):\n",
    "    if isinstance(text, str):\n",
    "        text = [text]\n",
    "    display_text_list = []\n",
    "    activation_list = []\n",
    "    for t in text:\n",
    "        # Convert text into tokens\n",
    "        if isinstance(t, str): # If the text is a list of tokens\n",
    "            split_text = model.to_str_tokens(t, prepend_bos=False)\n",
    "            tokens = model.to_tokens(t, prepend_bos=False)\n",
    "        else: # t equals tokens\n",
    "            tokens = t\n",
    "            split_text = model.to_str_tokens(t, prepend_bos=False)\n",
    "        seq_size = tokens.shape[1]\n",
    "        if(seq_size == 1): # If the text is a single token, we can't ablate it\n",
    "            continue\n",
    "        original = get_neuron_activation(tokens, feature, model)[-1]\n",
    "        changed_activations = torch.zeros(seq_size, device=device).cpu()\n",
    "        for i in range(seq_size):\n",
    "            # Remove the i'th token from the input\n",
    "            ablated_tokens = torch.cat((tokens[:,:i], tokens[:,i+1:]), dim=1)\n",
    "            changed_activations[i] += get_neuron_activation(ablated_tokens, feature, model)[-1]\n",
    "        changed_activations -= original\n",
    "        display_text_list += [x.replace('\\n', '\\\\newline') for x in split_text] + [\"\\n\"]\n",
    "        activation_list += changed_activations.tolist() + [0.0]\n",
    "    activation_list = torch.tensor(activation_list).reshape(-1,1,1)\n",
    "    if setting == \"plot\":\n",
    "        return text_neuron_activations(tokens=display_text_list, activations=activation_list)\n",
    "    else:\n",
    "        return display_text_list, activation_list\n",
    "def visualize_text(text, feature, model, setting=\"dictionary_basis\", max_activation = None):\n",
    "    if isinstance(text, str):\n",
    "        text = [text]\n",
    "    if isinstance(feature, int):\n",
    "        feature = [feature]\n",
    "    display_text_list = []\n",
    "    act_list = []\n",
    "    for t in text:\n",
    "        if isinstance(t, str): # If the text is a list of tokens\n",
    "            split_text = model.to_str_tokens(t, prepend_bos=False)\n",
    "            token = model.to_tokens(t, prepend_bos=False)\n",
    "        else: # t are tokens\n",
    "            token = t\n",
    "            split_text = model.to_str_tokens(t, prepend_bos=False)\n",
    "        for f in feature:\n",
    "            display_text_list += [x.replace('\\n', '\\\\newline') for x in split_text] + [\"\\n\"]\n",
    "            act_list += get_neuron_activation(token, f, model, setting) + [0.0]\n",
    "    act_list = torch.tensor(act_list).reshape(-1,1,1)\n",
    "    if(max_activation is not None):\n",
    "        act_list = torch.clamp(act_list, max=max_activation)\n",
    "    return text_neuron_activations(tokens=display_text_list, activations=act_list)\n",
    "# Ablate the feature direction of the tokens\n",
    "# token_list is a list of tokens, convert to tensor of shape (batch_size, seq_len)\n",
    "from einops import rearrange\n",
    "def ablate_feature_direction(tokens, feature, model, autoencoder):\n",
    "    def mlp_ablation_hook(value, hook):\n",
    "        # Rearrange to fit autoencoder\n",
    "        int_val = rearrange(value, 'b s h -> (b s) h')\n",
    "\n",
    "        # Run through the autoencoder\n",
    "        _, act = autoencoder(int_val)\n",
    "        feature_to_ablate = feature # TODO: bring this out of the function\n",
    "\n",
    "        # Subtract value with feature direction*act_of_feature\n",
    "        feature_direction = torch.outer(act[:, feature_to_ablate].squeeze(), autoencoder.decoder.weight[:, feature_to_ablate].squeeze())\n",
    "        batch, seq_len, hidden_size = value.shape\n",
    "        feature_direction = rearrange(feature_direction, '(b s) h -> b s h', b=batch, s=seq_len)\n",
    "        value -= feature_direction\n",
    "        return value\n",
    "\n",
    "    return model.run_with_hooks(tokens, \n",
    "        fwd_hooks=[(\n",
    "            cache_name, \n",
    "            mlp_ablation_hook\n",
    "            )]\n",
    "        )\n",
    "def add_feature_direction(tokens, feature, model, autoencoder, scalar=1.0):\n",
    "    def residual_add_hook(value, hook):\n",
    "        feature_direction = autoencoder.decoder.weight[:, feature].squeeze()\n",
    "        value += scalar*feature_direction\n",
    "        return value\n",
    "\n",
    "    return model.run_with_hooks(tokens, \n",
    "        fwd_hooks=[(\n",
    "            cache_name,\n",
    "            residual_add_hook\n",
    "            )]\n",
    "        )\n",
    "def ablate_feature_direction_display(text, features=None, setting=\"true_tokens\", verbose=False):\n",
    "\n",
    "    if features==None:\n",
    "        features = torch.tensor([best_feature])\n",
    "    if isinstance(features, int):\n",
    "        features = torch.tensor([features])\n",
    "    if isinstance(features, list):\n",
    "        features = torch.tensor(features)\n",
    "    if isinstance(text, str):\n",
    "        text = [text]\n",
    "    text_list = []\n",
    "    logit_list = []\n",
    "    for t in text:\n",
    "        tokens = model.to_tokens(t, prepend_bos=False)\n",
    "        with torch.no_grad():\n",
    "            original_logits = model(tokens).log_softmax(-1).cpu()\n",
    "            ablated_logits = ablate_feature_direction(tokens, features, model, smaller_auto_encoder).log_softmax(-1).cpu()\n",
    "        diff_logits = ablated_logits  - original_logits# ablated > original -> negative diff\n",
    "        tokens = tokens.cpu()\n",
    "        if setting == \"true_tokens\":\n",
    "            split_text = model.to_str_tokens(t, prepend_bos=False)\n",
    "            gather_tokens = rearrange(tokens[:,1:], \"b s -> b s 1\") # TODO: verify this is correct\n",
    "            # Gather the logits for the true tokens\n",
    "            diff = rearrange(diff_logits[:, :-1].gather(-1,gather_tokens), \"b s n -> (b s n)\")\n",
    "        elif setting == \"max\":\n",
    "            # Negate the diff_logits to see which tokens have the largest effect on the neuron\n",
    "            val, ind = (-1*diff_logits).max(-1)\n",
    "            diff = rearrange(val[:, :-1], \"b s -> (b s)\")\n",
    "            diff*= -1 # Negate the values gathered\n",
    "            split_text = model.to_str_tokens(ind, prepend_bos=False)\n",
    "            gather_tokens = rearrange(ind[:,1:], \"1 s -> 1 s 1\")\n",
    "        split_text = split_text[1:] # Remove the first token since we're not predicting it\n",
    "        if(verbose):\n",
    "            text_list += [x.replace('\\n', '\\\\newline') for x in split_text] + [\"\\n\"]\n",
    "            text_list += [x.replace('\\n', '\\\\newline') for x in split_text] + [\"\\n\"]\n",
    "            orig = rearrange(original_logits[:, :-1].gather(-1, gather_tokens), \"b s n -> (b s n)\")\n",
    "            ablated = rearrange(ablated_logits[:, :-1].gather(-1, gather_tokens), \"b s n -> (b s n)\")\n",
    "            logit_list += orig.tolist() + [0.0]\n",
    "            logit_list += ablated.tolist() + [0.0]\n",
    "        text_list += [x.replace('\\n', '\\\\newline') for x in split_text] + [\"\\n\"]\n",
    "        logit_list += diff.tolist() + [0.0]\n",
    "    logit_list = torch.tensor(logit_list).reshape(-1,1,1)\n",
    "    if verbose:\n",
    "        print(f\"Max & Min logit-diff: {logit_list.max().item():.2f} & {logit_list.min().item():.2f}\")\n",
    "    return text_neuron_activations(tokens=text_list, activations=logit_list)\n",
    "def generate_text(input_text, num_tokens, model, autoencoder, feature, temperature=0.7, setting=\"add\", scalar=1.0):\n",
    "    # Convert input text to tokens\n",
    "    input_ids = model.tokenizer.encode(input_text, return_tensors='pt').to(device)\n",
    "\n",
    "    for _ in range(num_tokens):\n",
    "        # Generate logits\n",
    "        with torch.no_grad():\n",
    "            if(setting==\"add\"):\n",
    "                logits = add_feature_direction(input_ids, feature, model, autoencoder, scalar=scalar)\n",
    "            else:\n",
    "                logits = model(input_ids)\n",
    "\n",
    "        # Apply temperature\n",
    "        logits = logits / temperature\n",
    "\n",
    "        # Sample from the distribution\n",
    "        probs = torch.nn.functional.softmax(logits[:, -1, :], dim=-1)\n",
    "        predicted_token = torch.multinomial(probs, num_samples=1)\n",
    "\n",
    "        # Append predicted token to input_ids\n",
    "        input_ids = torch.cat((input_ids, predicted_token), dim=-1)\n",
    "\n",
    "    # Decode the tokens to text\n",
    "    output_text = model.tokenizer.decode(input_ids[0])\n",
    "\n",
    "    return output_text\n",
    "\n",
    "# Logit Lens\n",
    "def logit_lens(model, best_feature, smaller_dict, layer):\n",
    "    with torch.no_grad():\n",
    "        # There are never-used tokens, which have high norm. We want to ignore these.\n",
    "        bad_ind = (model.W_U.norm(dim=0) > 20)\n",
    "        feature_direction = smaller_dict[best_feature].to(device)\n",
    "        # feature_direction = torch.matmul(feature_direction, model.W_out[layer]) # if MLP\n",
    "        logits = torch.matmul(feature_direction, model.W_U).cpu()\n",
    "    # Don't include bad indices\n",
    "    logits[bad_ind] = -1000\n",
    "    topk_values, topk_indices = torch.topk(logits, 20)\n",
    "    top_text = model.to_str_tokens(topk_indices)\n",
    "    print(f\"{top_text}\")\n",
    "    print(topk_values)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate Text\n",
    "You can use the functions below to find interesting features to then add here to \"feature\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = \" for\"\n",
    "temp = 0.7\n",
    "tokens_to_generate = 20\n",
    "feature = 10 \n",
    "scalar = 100.0\n",
    "# Using the function:\n",
    "print(\"Normal:\\n\" + generate_text(sentence, tokens_to_generate, model, smaller_auto_encoder, feature=feature, temperature=temp, scalar=scalar, setting=\"normal\"))\n",
    "print(\"Add:\\n\" + generate_text(sentence, tokens_to_generate, model, smaller_auto_encoder, feature=feature, temperature=temp, scalar=scalar, setting=\"add\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Search\n",
    "Type in a sentence & see which features activate \n",
    "Note: Some features may be outliers, which will typically show up as high activations for the first word & first period or \\n (or high positive bias)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = \" I do like a\"\n",
    "split_text = model.to_str_tokens(t, prepend_bos=False)\n",
    "token = model.to_tokens(t, prepend_bos=False)\n",
    "_, cache = model.run_with_cache(token.to(model.cfg.device))\n",
    "neuron_act_batch = cache[cache_name]\n",
    "_, act = smaller_auto_encoder(neuron_act_batch)\n",
    "v, i = act[0, -1, :].topk(10)\n",
    "\n",
    "print(\"Activations:\",[round(val,2) for val in v.tolist()])\n",
    "print(\"Feature_ids\", i.tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Interp\n",
    "Investigate the example sentences the activate this feature.\n",
    "\n",
    "Max: show max activating (tokens,contexts)\n",
    "\n",
    "Uniform: Show range of activations from each bin (e.g. sample an example from 1-2, 2-3, etc). \n",
    "[Note: if a feature is monosemantic, then the full range of activations should be that feature, not just max-activating ones]\n",
    "\n",
    "Full_text: shows the full text example\n",
    "\n",
    "Text_list: shows up to the most activating example (try w/ max activating on a couple of examples to see)\n",
    "\n",
    "ablate_text: remove the context one token at a time, and show the decrease/increase in activation of that feature\n",
    "\n",
    "ablate_feature_direction: removes feature direction from model's activation mid-inference, showing the logit diff in the output for every token.\n",
    "\n",
    "logit_lens: show the logit lens for that feature. If matches ablate_feature_direction, then the computation path is through the residual stream, else, it's through future layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# N = 5\n",
    "# best_feature = int(max_indices[N])\n",
    "best_feature = 1461\n",
    "print(\"bias:\", smaller_auto_encoder.encoder_bias.detach().cpu().numpy()[best_feature])\n",
    "print(f\"Feature index: {best_feature}\")\n",
    "print(f\"MCS: {max_cosine_similarities[best_feature]}\")\n",
    "text_list, full_text, token_list, full_token_list = get_feature_datapoints(best_feature, dictionary_activations, dataset, setting=\"uniform\")\n",
    "# text_list, full_text, token_list, full_token_list = get_feature_datapoints(best_feature, dictionary_activations, dataset, setting=\"max\")\n",
    "visualize_text(full_text, best_feature, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ablate_text(text_list, best_feature, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ablate_feature_direction_display(full_text, best_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logit_lens(model,best_feature, smaller_dict, layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_text = [\n",
    "    \" Your text here\",\n",
    "]\n",
    "visualize_text(custom_text, best_feature, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Centric Viewpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Go through datapoints & see if the features that activate on them make sense.\n",
    "d_point = 0\n",
    "# text = tokens_dataset[d_point]\n",
    "data_ind, sequence_pos = np.unravel_index(d_point, (datapoints, token_amount))\n",
    "feature_val, feature_ind = dictionary_activations[d_point].topk(10)\n",
    "data_ind = int(data_ind)\n",
    "sequence_pos = int(sequence_pos)\n",
    "full_tok = torch.tensor(dataset[data_ind][\"input_ids\"])\n",
    "full_text = []\n",
    "full_text.append(model.tokenizer.decode(full_tok))\n",
    "visualize_text(full_text, feature_ind, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check the neuron/residual basis\n",
    "When we look at the weights of a feature, we are seeing the literal dimensions from the residual stream/neurons being read from the feature. \n",
    "\n",
    "Here I'm visualizing the weight values for the residual stream. If there are outliers, then it's mainly reading from that dimension."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check features non-zero weights in decoder\n",
    "# Plot a histogram of the weights\n",
    "max_activation = dictionary_activations[:, best_feature].max()\n",
    "weights = smaller_dict[best_feature]\n",
    "plt.hist(weights, bins=100)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(weights*max_activation).topk(20), (weights*max_activation).topk(20, largest=False).values, (weights*max_activation > 0.2).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepend/Append tokens\n",
    "We can iterate over all tokens to check which ones activate a feature a lot to more rigorously test a hypothesis on what a feature means.\n",
    "\n",
    "Note: I'm literately running the model through all 50k tokens prepended to the text here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepend_all_tokens_and_get_feature_activation(model, minimal_activating_example, feature, setting=\"prepend\"):\n",
    "    tokens = model.to_tokens(minimal_activating_example, prepend_bos=False)\n",
    "\n",
    "    # Run through every number up to vocab size\n",
    "    vocab_size = model.cfg.d_vocab\n",
    "    batch_size = 256*2 # Define your desired batch size\n",
    "\n",
    "    dollar_feature_activations = torch.zeros(vocab_size)\n",
    "    for start in range(0, vocab_size, batch_size):\n",
    "        end = min(start + batch_size, vocab_size)\n",
    "\n",
    "        token_prep = torch.arange(start, end).to(device)\n",
    "        token_prep = token_prep.unsqueeze(1)  # Add a dimension for concatenation\n",
    "\n",
    "        # 1. Prepend to the tokens\n",
    "        if setting == \"prepend\":\n",
    "            tokens_catted = torch.cat((token_prep, tokens.repeat(end - start, 1)), dim=1).long()\n",
    "        elif setting == \"append\":\n",
    "            tokens_catted = torch.cat((tokens.repeat(end - start, 1), token_prep), dim=1).long()\n",
    "        else:\n",
    "            raise ValueError(f\"Unknown setting: {setting}\")\n",
    "\n",
    "        # 2. Run through the model\n",
    "        with torch.no_grad():\n",
    "            _, cache = model.run_with_cache(tokens_catted.to(device))\n",
    "            neuron_act_batch = cache[cache_name]\n",
    "            _, act = smaller_auto_encoder(neuron_act_batch)\n",
    "\n",
    "        # 3. Get the feature\n",
    "        dollar_feature_activations[start:end] = act[:, -1, feature].cpu().squeeze()\n",
    "\n",
    "    k = 20\n",
    "    k_increasing_val, k_increasing_ind = dollar_feature_activations.topk(k)\n",
    "    k_decreasing_val, k_decreasing_ind = dollar_feature_activations.topk(k, largest=False)\n",
    "    if(setting == \"prepend\"):\n",
    "        print(f\"[token]{minimal_activating_example}\")\n",
    "    elif(setting == \"append\"):\n",
    "        print(f\"{minimal_activating_example}[token]\")\n",
    "    else:\n",
    "        raise ValueError(f\"Unknown setting: {setting}\")\n",
    "    # Print indices converted to tokens\n",
    "    print(f\"Top-{k} increasing: {model.to_str_tokens(k_increasing_ind)}\")\n",
    "    # Print values\n",
    "    print(f\"Top-{k} increasing: {[f'{val:.2f}' for val in k_increasing_val]}\")\n",
    "    print(f\"Top-{k} decreasing: {model.to_str_tokens(k_decreasing_ind)}\")\n",
    "    print(f\"Top-{k} decreasing: {[f'{val:.2f}' for val in k_decreasing_val]}\")\n",
    "    print(f\"Number of 0 activations: {torch.sum(dollar_feature_activations == 0)}\")\n",
    "    if(setting == \"prepend\"):\n",
    "        best_text = \"\".join(model.to_str_tokens(dollar_feature_activations.argmax()) + [minimal_activating_example])\n",
    "    else:\n",
    "        best_text = \"\".join([minimal_activating_example] + model.to_str_tokens(dollar_feature_activations.argmax()))\n",
    "    return best_text\n",
    "\n",
    "best_text = \"\"\n",
    "for x in range(3):\n",
    "    # best_text = prepend_all_tokens_and_get_feature_activation(model, best_text, best_feature, setting=\"prepend\")\n",
    "    best_text = prepend_all_tokens_and_get_feature_activation(model, best_text, best_feature, setting=\"append\")\n",
    "# prepend_all_tokens_and_get_feature_activation(model, \" for all $\", best_feature, setting=\"prepend\")\n",
    "# prepend_all_tokens_and_get_feature_activation(model, \" tree\", best_feature, setting=\"prepend\")\n",
    "# prepend_all_tokens_and_get_feature_activation(model, \" tree\", best_feature, setting=\"append\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_text = \"\"\n",
    "for x in range(3):\n",
    "    best_text = prepend_all_tokens_and_get_feature_activation(model, best_text, best_feature, setting=\"prepend\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prepend_all_tokens_and_get_feature_activation(model, \" for all $\", best_feature, setting=\"prepend\")\n",
    "# prepend_all_tokens_and_get_feature_activation(model, \"The\", best_feature, setting=\"append\")\n",
    "# prepend_all_tokens_and_get_feature_activation(model, \" tree\", best_feature, setting=\"append\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}