MeidumCNN_Val_Acc_Ensembles.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MeidumCNN Val Acc Ensembles",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyM9zxjLnLc8e8RioXIffJHX",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4f868cdd95e54f6b82bf51dce4c6935d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_184d7b5dc6b44f3b98773697f7e972da",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_094da48e36414b8dab58cf6359d495b3",
              "IPY_MODEL_de52547f2c69421b9c44fc1c28369156"
            ]
          }
        },
        "184d7b5dc6b44f3b98773697f7e972da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "094da48e36414b8dab58cf6359d495b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_66345248e48c48f195e8719de48c20a0",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_58c204c8c73f44068eacced4b9cc0df6"
          }
        },
        "de52547f2c69421b9c44fc1c28369156": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0f0f573ade7d42cbb0f7650545dc7b4c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5/5 [03:12&lt;00:00, 38.47s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_142d1c536cfd48d2bfa2cafc1a1a8fbb"
          }
        },
        "66345248e48c48f195e8719de48c20a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "58c204c8c73f44068eacced4b9cc0df6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0f0f573ade7d42cbb0f7650545dc7b4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "142d1c536cfd48d2bfa2cafc1a1a8fbb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ayulockin/LossLandscape/blob/master/MeidumCNN_Val_Acc_Ensembles.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kWXU2ICvCDN1",
        "colab_type": "text"
      },
      "source": [
        "# Setups, Imports and Installations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rQAggd-btWJK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 129
        },
        "outputId": "5abb5ff4-3f33-43ef-f5a4-48e77a157d74"
      },
      "source": [
        "## This is so that I can save my models.\n",
        "from google.colab import drive\n",
        "drive.mount('gdrive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sAZBTFNQq0ys",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!pip install wandb"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9-APP7-6C5uK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.datasets import cifar10\n",
        "from tensorflow.keras.applications import resnet50"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Pr3HrMxq8SF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "43d507e1-0f31-4b85-cf5f-e54ec868a0b3"
      },
      "source": [
        "import os\n",
        "os.environ[\"TF_DETERMINISTIC_OPS\"] = \"1\"\n",
        "\n",
        "import numpy as np\n",
        "from numpy.linalg import norm\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from mpl_toolkits.mplot3d import Axes3D\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "sns.set()\n",
        "\n",
        "from tqdm.notebook import tqdm_notebook\n",
        "from sklearn.manifold import TSNE\n",
        "\n",
        "from sklearn.metrics import accuracy_score"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XmVkQbw_q_07",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        },
        "outputId": "4524f3ce-07db-4174-f570-f247a0b79748"
      },
      "source": [
        "import wandb\n",
        "from wandb.keras import WandbCallback\n",
        "\n",
        "wandb.login()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://app.wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Not authenticated.  Copy a key from https://app.wandb.ai/authorize\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "API Key: ··········\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FhhwqSIdCu9V",
        "colab_type": "text"
      },
      "source": [
        "# Get Trained Models"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qDLJ-R6jCz1I",
        "colab_type": "text"
      },
      "source": [
        "(For now I am using models trained from SmallCNN.)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WWCudbf8CgoH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ROOT_PATH = 'gdrive/My Drive/LossLandscape/'\n",
        "## SmallCNN models with different initialization(total 10 models)\n",
        "MODEL_PATH = ROOT_PATH+'MediumIndependentSolutions/'"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pZjcYJYJDqHA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "68a782d2-aa6d-4bf8-e3e7-62c6095af923"
      },
      "source": [
        "model_ckpts = os.listdir(MODEL_PATH)\n",
        "\n",
        "print(len(model_ckpts))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fzS6bWLyEwU2",
        "colab_type": "text"
      },
      "source": [
        "# Get Dataset and Prepare"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LMFYLweasySz",
        "colab_type": "text"
      },
      "source": [
        "#### CIFAR-10"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Posjlu7lDJBd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "ac8391ac-8a78-4c21-9adb-219ac83fad42"
      },
      "source": [
        "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
        "\n",
        "y_train = y_train.flatten()\n",
        "y_test = y_test.flatten()\n",
        "\n",
        "CLASS_NAMES = (\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \"dog\", \"frog\", \"horse\", \"ship\", \"truck\")\n",
        "print(x_train.shape, y_train.shape, x_test.shape, y_test.shape)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
            "170500096/170498071 [==============================] - 2s 0us/step\n",
            "(50000, 32, 32, 3) (50000,) (10000, 32, 32, 3) (10000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QZzY01xwxS-f",
        "colab_type": "text"
      },
      "source": [
        "#### Dataloader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6TqcgM67xxDF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "AUTO = tf.data.experimental.AUTOTUNE\n",
        "BATCH_SIZE = 128\n",
        "IMG_SHAPE = 32\n",
        "\n",
        "testloader = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
        "\n",
        "def preprocess_image(image, label):\n",
        "  img = tf.cast(image, tf.float32)\n",
        "  img = img/255.\n",
        "\n",
        "  return img, label\n",
        "\n",
        "testloader = (\n",
        "    testloader\n",
        "    .map(preprocess_image, num_parallel_calls=AUTO)\n",
        "    .batch(BATCH_SIZE)\n",
        "    .prefetch(AUTO)\n",
        ")"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IZ1-EmWrLRVa",
        "colab_type": "text"
      },
      "source": [
        "# Deep Ensemble "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4EGLJG5tLRBi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "6028116b-0fe3-41a3-a068-358dd9b59bd6"
      },
      "source": [
        "# Choose a single model randomly and load it up\n",
        "model_ckpt = model_ckpts[int(np.random.choice(len(model_ckpts), 1))]\n",
        "single_model = tf.keras.models.load_model(MODEL_PATH + model_ckpt)\n",
        "\n",
        "# Evaluate it\n",
        "accuracy_single_model = single_model.evaluate(testloader)[1] * 100\n",
        "accuracy_single_model"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "79/79 [==============================] - 6s 75ms/step - loss: 0.8009 - accuracy: 0.7817\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "78.17000150680542"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "avosm4PfLQ3M",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "bba9beb1-22d0-42c3-b686-75b7efee553e"
      },
      "source": [
        "# Create list to models with different solutions\n",
        "members = [tf.keras.models.load_model(MODEL_PATH + model_ckpts[i]) \n",
        "                            for i in range(len(model_ckpts))]\n",
        "len(members)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8TzKOLvMzwCO",
        "colab_type": "text"
      },
      "source": [
        "Reference: https://machinelearningmastery.com/model-averaging-ensemble-for-deep-learning-neural-networks/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "coGl-4WMxBGH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# make an ensemble prediction for multi-class classification\n",
        "def ensemble_predictions(members):\n",
        "\t# make predictions\n",
        "\tyhats = [model.predict(x_test/255.) for model in members]\n",
        "\tyhats = np.array(yhats)\n",
        "\t# sum across ensemble members\n",
        "\tsummed = np.sum(yhats, axis=0)\n",
        "\t# argmax across classes\n",
        "\tresult = np.argmax(summed, axis=1)\n",
        "\treturn result\n",
        " \n",
        "# evaluate a specific number of members in an ensemble\n",
        "def evaluate_n_members(members):\n",
        "    accuracy_list = []\n",
        "    for n_members in tqdm_notebook(range(2, 12, 2)):\n",
        "        # select a subset of members\n",
        "        subset = members[:n_members]\n",
        "        print(len(subset))\n",
        "        # make prediction\n",
        "        yhat = ensemble_predictions(subset)\n",
        "        # calculate accuracy\n",
        "        accuracy_list.append(100*accuracy_score(y_test, yhat))\n",
        "\n",
        "    return accuracy_list"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nP3sUVfO1F72",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 159,
          "referenced_widgets": [
            "4f868cdd95e54f6b82bf51dce4c6935d",
            "184d7b5dc6b44f3b98773697f7e972da",
            "094da48e36414b8dab58cf6359d495b3",
            "de52547f2c69421b9c44fc1c28369156",
            "66345248e48c48f195e8719de48c20a0",
            "58c204c8c73f44068eacced4b9cc0df6",
            "0f0f573ade7d42cbb0f7650545dc7b4c",
            "142d1c536cfd48d2bfa2cafc1a1a8fbb"
          ]
        },
        "outputId": "b73c4095-3162-4b4e-d158-8b54fa3fefd5"
      },
      "source": [
        "# Run the ensembles\n",
        "accuracy_list = evaluate_n_members(members)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4f868cdd95e54f6b82bf51dce4c6935d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2\n",
            "4\n",
            "6\n",
            "8\n",
            "10\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R7men6lu1ZoW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "accuracy_list.insert(0, accuracy_single_model)\n",
        "rng = [i for i in range(0, 12, 2)]"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c4ZxtCgUKoxt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 518
        },
        "outputId": "8eaf6dff-eb63-4683-841d-089218c7c897"
      },
      "source": [
        "plt.figure(figsize=(9,8))\n",
        "plt.plot(rng, accuracy_list, label='deep ensemble')\n",
        "plt.plot(rng, [accuracy_single_model]*len(rng), '--', label='single model')\n",
        "plt.title(\"Test accuracy as a function of ensemble size\")\n",
        "plt.xlabel(\"Ensemble size\")\n",
        "plt.ylabel(\"Test accuracy\")\n",
        "plt.legend();\n",
        "plt.savefig('ensemble_func.png')"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAAH1CAYAAAA6bM6tAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxM5/4H8M9M9lX2zZKQJmqLREKIoFXlUkksVaWq2murXaneH61uKPeqUrVTrdJF7VtbXVAUtYQSYskq+y6TyTKZmef3R2QqTcRIMjNZPu/X676uzDlzzneepPHxnOd8j0QIIUBERERUz0kNXQARERGRNhhaiIiIqEFgaCEiIqIGgaGFiIiIGgSGFiIiImoQGFqIiIioQWBoISKdiY2NRUREBAICArBt2za9nTclJQUBAQFQqVR6OycAZGVl4aWXXkJAQACWLl2q13PXhaSkJLRt2xZKpbLK7atXr8bcuXPr9Jzr16/HggUL6vSY1HgZG7oAogcFBARo/lxUVARTU1MYGRkBAN5//32Eh4c/1vFefvllhIeHY8SIEXVaJ2ln8+bNCA4Oxv79+3V6nr59+2LRokUICQkBAHh4eCAyMlKn56zKd999B3t7e1y6dAkSiUTv52+IJk+ebOgSqAFhaKF65cG/aP75F1FjpFQqYWzceP8zTElJwXPPPWfoMvQmJSUF3t7eDCxEOsLLQ9QgqNVqbNy4Ef369UNwcDBmzpyJvLw8AEBJSQnmzp2L4OBgBAUFYfjw4cjKysInn3yCCxcu4IMPPkBAQAA++OCDKo89Y8YM9OzZE4GBgXjppZdw+/Ztzbbi4mIsXboUTz/9NAIDAzFq1CgUFxcDAC5cuIAXX3wRQUFB6NOnD/bs2QOgbHbn+++/1xxjz549GDVqlObrtm3bYseOHejfvz/69+8PAFi0aBH69OmDLl26YNiwYbhw4YJmf5VKhfXr16Nfv34ICAjAsGHDkJqaivfff7/SJYjJkyfjiy++qPJzVneOv/76C8OGDUOXLl0QEhKCjz76qMpj3Lt3D5MmTUL37t3RtWtXTJo0CWlpaVXuO3bsWJw7d04z/nFxcVqNzTfffIP+/fsjKCgI77//Ph5s2r1z504MHDgQAQEBGDRoEKKiovDmm28iJSUFkydPRkBAADZt2lTpMkd6ejomT56Mbt264dlnn8XOnTs1x1y9ejVmzpyJefPmISAgAM899xyuXr1a5WcCgEuXLmH48OEIDAzE8OHDcenSJQDAf/7zH+zbtw9btmxBQEAA/vjjj0rvVSgUWLZsGZ566imEhIRg4cKFmp+nc+fOoXfv3vj888/Ro0cPhIaGYvfu3Zr3njhxAoMGDUJAQAB69eqFLVu2aLYdO3YMERERCAoKwosvvojo6GjNtr59+2Lz5s0ICwuDv78/5s+fj6ysLIwfPx4BAQEYN24c7t27V6HO3bt3IzQ0FKGhoRXO80+XL1/W/DcQHh6Oc+fOPXTfjRs3olevXggICMCAAQNw5swZzfiXX3Iq/1kp/1/79u2xevVqAGXfw+nTp6N79+7o27evXi83Uj0iiOqpp59+Wpw+fVoIIcQXX3whRowYIVJTU0VJSYl45513xOzZs4UQQnzzzTdi0qRJorCwUCiVSnH16lUhk8mEEEKMGTNG7Ny5s9rzfP/990Imk4mSkhKxaNEiER4ertn23nvviTFjxoi0tDShVCrFxYsXRUlJiUhKShL+/v7i4MGDQqFQiJycHHH9+vUqz7l7927x4osvar729fUV48aNE7m5uaKoqEgIIcS+fftETk6OKC0tFVu2bBEhISGiuLhYCCHEpk2bxODBg0VMTIxQq9Xixo0bIicnR1y5ckX07NlTqFQqIYQQ2dnZws/PT2RmZlb5Oas7xwsvvCD27t0rhBCioKBAREZGVnmMnJwc8eOPP4rCwkIhk8nE9OnTxeuvv/7Qsf3nWGgzNhMnThT37t0TycnJIjg4WJw4cUIIIcSRI0dEaGiouHLlilCr1SI+Pl4kJSUJISr+rAghxN27d4Wvr68oLS0VQggxevRo8e6774ri4mJx/fp1ERwcLP744w8hhBCffvqp6Nixozh+/LhQKpVi+fLlYsSIEVV+ntzcXBEUFCT27t0rSktLxcGDB0VQUJDIyckRQgjx1ltviRUrVjx0PBYvXiwmTZokcnNzhUwmE5MmTRLLly8XQghx9uxZ0a5dO7Fy5UqhUCjE8ePHhZ+fn8jLyxNCCNGzZ09x/vx5IYQQeXl54tq1a0IIIaKiokT37t3F5cuXhVKpFHv27BFPP/20KCkp0YzNiBEjRGZmpkhLSxPdu3cXQ4YMEVFRUaK4uFi8/PLLYvXq1RXGbfbs2UIul4vo6GgRHBysGdtPP/1UzJkzRwghRFpamujWrZs4fvy4UKlU4tSpU6Jbt24iOzu70ueOiYkRvXv3FmlpaZrzJCQkVDrmg8q/T1FRUUKlUomhQ4eK1atXi5KSEpGYmCj69u0rfv/994eONTVOnGmhBuHbb7/F7Nmz4ebmBlNTU0ybNg0//fST5vJKXl4eEhISYGRkhI4dO8La2lrrYz///POwtraGqakppk+fjujoaMhkMqjVauzevRsLFiyAq6srjIyM0KVLF5iamuLQoUMICQnB4MGDYWJiAnt7e7Rr107rc06cOBF2dnYwNzcHAERERMDe3h7GxsZ47bXXoFAoEBcXBwD4/vvvMXPmTLRp0wYSiQRPPvkk7O3t4efnBxsbG82/WI8cOYJu3brBycmpynNWdw5jY2MkJiYiJycHVlZW8Pf3r/IY9vb2GDBgACwsLGBtbY3XX38d58+f1/pza2PChAmwtbWFh4cHgoODNbMGu3btwvjx4+Hn5weJRAJPT080b978kcdLTU3FpUuXMHfuXJiZmaFdu3YYMWJEhXU2gYGB6NOnD4yMjBAREVFhpuJBx48fh6enJ4YMGQJjY2MMHjwYbdq0wbFjxx5ZhxACO3fuxPz582FnZwdra2tMmjQJhw8f1uxjbGyMqVOnwsTEBH369IGlpWWF79GdO3dQUFCAZs2aoUOHDgDK1tGMHDkSnTt3hpGREYYOHQoTExNcvnxZc9wxY8bAyckJrq6uCAoKgp+fH9q3bw8zMzM8++yzuH79eoVap06dCktLS7Rt2xbDhg3DoUOHKn2e/fv3o3fv3ujTpw+kUil69uyJjh074sSJE5X2NTIygkKhQExMDEpLS9GiRQu0atXqoWOVk5ODqVOn4p133kH79u1x9epV5OTkYNq0aTA1NUXLli3xwgsv4MiRI48cd2pcGu/FdGpUUlJSMHXqVEilf+dsqVSK7OxsREREIC0tDW+88Qby8/MRHh6O2bNnw8TE5JHHValU+OSTT/Djjz8iJydHc/zc3FwoFAqUlJSgZcuWld6Xmppa7S/dR3F3d6/w9ZYtW7Br1y5kZGRAIpGgoKAAubm5AIC0tLSHnmvo0KE4cOAAevbsiQMHDmDs2LEPPWd151i8eDE+/fRTDBw4EC1atMC0adPw9NNPVzpGUVERPvroI5w8eVJzSUEul0OlUmkWTNeWs7Oz5s8WFhaQy+UAaj7mGRkZaNasWYUg6+HhgWvXrmm+fjDomZubo6SkpMr1RhkZGfDw8KjwmoeHB9LT0x9ZR05ODoqKijBs2DDNa0IIqNVqzdd2dnYVzmlhYYHCwkIAwKeffop169bh448/Rtu2bTFnzhwEBAQgJSUF+/btw/bt2zXvKy0tRUZGRpWfz8zMrNLnLT9HuQd/Pps3b45bt25V+jwpKSn48ccfKwQ2pVKJ4ODgSvt6enpi/vz5WL16Ne7cuYPQ0FD85z//gaura6V9S0tLMWPGDAwePFizHio5ORkZGRkICgrS7KdSqSp8TU0DQws1CG5ubliyZAkCAwOr3D5t2jRMmzYNSUlJmDhxIlq3bq3VHUMHDx7Er7/+iq1bt6JFixaQyWTo2rUrhBCwt7eHmZkZ7t69iyeffLLC+9zd3fHXX39VeUwLCwsUFRVpvs7Kyqq0z4MLNS9cuIDNmzfjiy++gI+PD6RSqaaG8s+emJgIX1/fSscJDw/H4MGDER0djZiYGPTr16/Kmh51Di8vL6xYsQJqtRpHjx7FjBkzcO7cOVhaWlY4zueff464uDjs3LkTzs7OuHHjBoYMGVJh3Ul1tBmbh3F3d0diYqLW+5dzcXHBvXv3UFBQoAkuqampVf6Fqc2xUlJSKryWmpqKXr16PfK99vb2MDc3x+HDh2t0bj8/P6xbtw6lpaXYsWMHZs2ahRMnTsDd3R2TJ0/G66+//tjHfJjU1FR4e3sDKAsnLi4ulfZxd3dHREQEFi1apNUxw8LCEBYWhoKCAixcuBDLly/H//73v0r7ffjhh7C2tsasWbMqnKtFixY4evRoDT8RNRa8PEQNwqhRo7By5UokJycDKPtX6y+//AIAOHv2LG7evAmVSgVra2sYGxtrZkycnJxw9+7dhx5XLpfD1NQU9vb2KCoqwooVKzTbpFIphg8fjo8++gjp6elQqVSIjIyEQqFAWFgY/vjjDxw5cgRKpRK5ubm4ceMGAKBdu3b4+eefUVRUhISEBOzatavazyaXy2FkZAQHBwcolUp89tlnKCgo0GwfMWIEVq1ahfj4eAghEB0drZkhcXNzQ6dOnfDmm2+if//+mstNj3uO/fv3a2aabG1tNZ+/quOYmZnB1tYWeXl5+Oyzz6r9bP/0uGPzoOeffx6ff/45rl27BiEEEhISND8P1X2f3d3dERAQgBUrVqCkpATR0dHYtWvXY98+DwB9+vRBfHw8Dh48CKVSiSNHjuDOnTt46qmnHvleqVSKESNGYMmSJcjOzgZQtrj05MmTj3yvQqHAgQMHIJPJYGJiAisrK833Z8SIEfj2229x5coVCCFQWFiI48ePV/j+Pq61a9eiqKgIt2/fxp49ezBo0KBK+4SHh+PYsWM4efIkVCoVSkpKcO7cuSoXZsfGxuLMmTNQKBQwNTWFmZlZlT9f3377Lc6fP4/ly5dX2O7n5wcrKyts3LgRxcXFUKlUuHXr1kP/4UCNF0MLNQhjx45F37598dprryEgIAAvvPCC5hdWVlYWZsyYgcDAQAwaNAjdunVDRESE5n0//fQTunbtWuW/CIcMGQIPDw/06tULzz33XKW1HG+99RZ8fX3x/PPPo1u3bli+fDnUajU8PDywadMmbN26Fd26dcOQIUM06yBeeeUVmJiYICQkBG+99RbCwsKq/WyhoaHo1asXBgwYgL59+8LMzKzC9Pyrr76KgQMH4rXXXkOXLl2wYMEClJSUVPgMt27d0nzmmpzj5MmTeO655xAQEIDFixfjk08+qTIAvfLKKygpKUH37t0xcuRIrWYY/vn+xxmbBw0cOBCTJ0/GnDlz0KVLF0ydOlVziWrixIlYt24dgoKCqrzbZcWKFUhOTkavXr0wbdo0TJ8+vUa30tvb22P9+vXYunUrgoODsXnzZqxfvx4ODg5avf/NN9+Ep6cnXnjhBXTp0gXjxo3TrFl5lP3796Nv377o0qULvv32W80sRadOnfDhhx/igw8+QNeuXdG/f3/NnWw1VX6X1bhx4/Daa68hNDS00j7u7u5Yu3YtNmzYgB49eqBPnz7YsmVLhctd5RQKBT7++GMEBwcjNDQUOTk5eOONNyrtd/jwYdy9e1dzl1FAQADWr18PIyMjrF+/HtHR0XjmmWfQvXt3vP3227UKZtQwSYS287pEVC+dP38eb775Jo4dO8b+IETUqHGmhagBKy0txbZt2/D8888zsBBRo8fQQtRAxcTEoGvXrsjMzMS4ceMMXQ4Rkc7x8hARERE1CJxpISIiogaBoYWIiIgahEbRXC43Vw61uu6vcjk6WiM7m7fU6QPHWn841vrDsdYvjrf+6GqspVIJ7O2tHrq9UYQWtVroJLSUH5v0g2OtPxxr/eFY6xfHW38MMda8PEREREQNAkMLERERNQgMLURERNQgNIo1Lf+kUimRm5sJpVJRq+NkZEirfI4G1b3ajLWxsSns7Z1hZNQof5yJiOi+RvlbPjc3E+bmlrCycqtVa3NjYymUSoYWfajpWAshIJfnIzc3E05O7o9+AxERNViN8vKQUqmAlZUtn8XSBEgkElhZ2dZ6Vo2IiOq/RhlaADCwNCH8XhMRNQ2NNrQQERFR48LQYgChoUEoLCw0dBkGs2XLBnz22crH3kZERE0bQwsRERE1CI3y7qF/On01Faf+Sn3s90kkgHhEl+JQP3f07FT9XSsnTvyGDRvWwNTUDE891bfCtqioa1i/fjXkcjkAYPz4yQgJCQUAnDlzCtu2fY6SEgVMTEwwffob6NixEy5duoBVqz7GE0/44ObNaFhYmGP+/PfQunWbSufOysrCypX/RXp6GkpKStCv3wCMHfsaAOD558Pwr389h/PnzyE7OwujRo3B8OEjoVarsWLFf3Hp0nmYmJjC0tIC69Z9rlVN7dt3QFTUVRgbG+Pttz/A1q2bEBcXAxcXVyxe/D9YWFgAANLT0zBjxmRkZWWides2eOed92BuXvl5E9u3f4ETJ36DSqWCk5ML3nprARwdnar/phARUaPUJEKLIeXkZGPZssVYv34LWrXywo4dX2q2yWQyLF++BP/736dwcnJCVlYWJkwYi23bvkN+/j188cUWrFixGlZW1oiNjcHcuTOwZ89hAEBMzG3MmjUX77zzAX744RAWLXoXW7Z8Ven8ixYtxLhx4+Hv3wWlpaWYOfN1tGvXHl27dgcAFBcXY8OGrUhNTcHYsSMxcGAYkpISERl5Adu3fw+pVIr8/HwAQHJyUrU1xcfH4u2338Nbb72Njz9ehjlzpmPDhq1wcXHF3Lkz8MsvPyEsbAgA4K+/IrF169dwcHDEkiXv4/PPN2PKlJkVav/ppyNITk7Ghg1fQCqVYu/eXfjss5V4991Fdf+NIiKieq9JhJaenR49G1KVuujTcv36Nfj6tkWrVl4AgPDwYVi3bjUA4Nq1K0hNTcHcuTM0+0skEiQn38X161FITk7C1KkTNdtUKhVycrIBAC1atERAQCAAYMCAQfjvfxdDLi+AlZW1Zv+ioiJERl5EXl6e5rXCQjni4+M1oaVfv/4AAHd3D9jY2CIzMwMeHi2gVCqxdOmH6NIlCCEhvQAA586dqbamVq084ePTFgDQtm1bpKenwsXF9f7X7ZCUdFfzvpCQXnBwcAQADB4cgZUrl1cau1Onfkd09A289tqY++dSwtrautJ+RETUNDSJ0FJfCQF4e/tgzZpNlbZFRV1DcHAPvPPOB5W2xcfHaXl8NSQSCTZv3gZj46q/1aamppo/S6VSTTD46qudiIy8iAsX/sS6davx+efbIYSotiZTU7MHjmVUxbFVWtX9d/0Cr7zyGgYPjnis9xERUePEhbg61qFDJ9y+fRN37yYCAA4e3KfZ1rGjH5KSEnHp0gXNazduREEIgW7duuPcuTOIjY2psK1ccnISrlyJBAD8/POPaNPmiQqzLABgaWmFzp0DsH37F5rX0tPTkJ2dVW3Nubm5KC4uRnBwD0yePA3W1tZISUl+ZE2P448/TiE3NxcAcOTIQQQFda20T2hob+zdu0tzeUqhUOD27Vs1Oh8RETV8nGnRMXt7B8ybtwBvvTUbZmZm6NPn74W4tra2WLp0BdasWYVVqz6GUlkKD4/mWLbsE7Rs2QoLF36IpUs/RElJCZTKUnTq1Bnt2nUAALRp8wQOHtyH5cs/grm5Od5++/0qz79w4Yf49NMVGDt2JICyIPN//7ew2sWsGRnpWLZsEVQqFVQqFbp3D0GHDp0glUqrrelxdO4cgPfem4/MzAx4ebXBrFlvVNrnX/96Dvfu5WH69LLLUWq1GkOHjoCPj+9jn4+IiBo+iRCPuj+m/svOLoBa/ffHSEtLgJubZ62PW1+fPXTp0gWsWbOqyoW3DVVtx7quvudNgbOzDTIzZYYuo0ngWOsXx1v3cvKLcflOFny8HNHSwaLOjy+VSuDo+PC1i5xpISIioioJIZCUKUfkrUxE3s5CQnpZKBwU4oWWvSu32dA1hpYGqEuXoEY1y0JERPWHSq3GnaR7uHQrC5G3M5F1rxgSAN7Nm2HE094I8HFGp7auBpnVYmghIiJq4koUKlyLy0Hk7UxcuZMFebESxkZSdPCyx+AQL3R+wgnNrEwffSAdY2ghIiJqgvLlCly+k4XLt7MQFZ+DUqUaVubG8PN2QhdfJ3Ro7QBz0/oVE+pXNURERKQz6TmFiLydhUu3MxGTdA8CgKOtOfr4eyDAxxm+LZvBSFp/u6EwtBARETVSaiEQnypD5O2yhbQpWWXPuWvlao3w0NYI8HFCSxdrSCQSA1eqHYYWIiKiRqRUqUZ0Yi4ib5ctpL1XoIBUIkHbVnZ4yt8D/j5OcGpW97cr6wNDi4FER1/Hd999XauH/6WmpmD8+Jdx+PCvdViZbmtpjD1miIgMrbC4FH/FZiPyVhauxmajWKGCmYkROrVxKLvbx9sR1hYmhi6z1hhaDOTJJ9vzacVERFRjOfnFiLydhcu3MxGdmAeVWsDWyhTd2rmii68T2nnaw8TYyNBl1qkmE1oKD35U6TXjNt1g2uEZCGUJin5YUWm7WbveMHqiJ9TFMhT//Fml7Sbt+8LEO7ja8xYXF2PRoncRHx8LIyNjtGrliQ8/XFphxqF8liI8fBjOnj2N4uJi/Oc/C9G5sz8AYPfu7/D999/C2toGPXr0xJ49O6uc0YiKuob161dDLi+7Zjl+/GSEhIRW2m/x4vdgYmKCpKS7SE5OQp8+T6Nnz97YsmUDMjLS8cILo/HCC6MAlD1baOXK5SguLoK5uQVmzZqradu/e/dO7Nz5NaysrNCjR8XznDlzCtu2fY6SEgVMTEwwffob6NixU7VjRUREDyeEQHKmHJG3M3HpdhYS0sr6pLg7WqJ/t5YI8HFGGw9bSBvI+pSaaDKhxVDOnTuDwkI5tm//HgA0D//7p3v37qFjRz9MmjQVR4/+gPXrP8W6dZ/jzp3b+OqrL7B169ewt7fHypXLq3y/TCbD8uVL8L//fQonJydkZWVhwoSx2LbtO9jY2FTaPy4uFqtWrYNarcbzz4ehoKAAn322EdnZWRg9ejgGD46AiYkJFiyYh/nz30VQUDecP38OCxbMw3ff7UNCQjy2bfscW7fugIODI5YvX6o5dnJyEr74YgtWrFgNKytrxMbGYO7cGdiz53AdjCgRUdNR3uitfH1KZl5Zo7c2zW0x4ilv+Ps4wd3RytBl6k2TCS2WYf/30G0SY7Mqt5c/D0dqblPt+6vzxBM+iI+Pw8cfL0NAQGCVMx8AYGFhiZ49ewEoezL0Z5+tBABERl5Ejx49YW9vDwB47rlw/PzzD5Xef+3aFaSmpmDu3Bl/fy6JBMnJd/Hkk+0r7d+r11MwNS1rFNSqlSd69OgJqVQKZ2cX2NjYIjMzA0qlEiYmJggK6gYA6No1GCYmJkhMTEBk5EWEhITCwcERABARMRTHjv0MoCyoJScnYerUiZrzqVQq5ORkP97gERE1QSWlKkRpGr1lo6CoFMZGUrT3sseg7p7wf8IJzazNDF2mQegttBw7dgyrVq2CEAJCCEybNg39+/fHlClTkJSUBKlUCktLS7zzzjto166dvsrSuebNW2D79p24cOE8zp49jY0b1+DLL7+ttJ+p6d8LpKRSKVQq5WOdRwjA29sHa9Zs0mp/M7O/OxtKpVKYmppV+Ppxz1+xFoHg4B54550PKm2Lj4+r8XGJiBqr/EIFrtzOQuTtLFyPz4FC0+jNEQE+zujYpv41ejMEvYyAEALz5s3Djh074Ovri+joaIwaNQr9+vXDsmXLNJcvfvnlF8yfPx979+7VR1l6kZGRDlvbZujd+yl069YdQ4b8CzJZ1ZeIquLv3wVff70NeXl5sLOzw48/Hqpyv44d/ZCUlIhLly6gS5cgAGXrUZ58sn2N779v1coTpaWlmmNevHgeSqUSrVp5QgiBHTu+RG5uDuztHXDo0H7N+7p1646tWzchNjYGbdp4a2opXwtDRERAem4hIu8/3+dO8j0IATjamqF3Zw8E+DjBp6UdjI3qb6M3Q9BbbJNKpZDJyhYNyWQyuLi4QCqVVlhvUVBQ0GAa3GgrJuYO1q8vW8SrVqswZsw4ODk5IzExQav3+/j4YvTosZg8+VVYWlohKKgrrKwqP7bb1tYWS5euwJo1q7Bq1cdQKkvh4dEcy5Z9UuMxNTExweLF/62wEHfRomUwMTHBE0/44OWXX8Xrr/8blpZW6NGjp+Z9LVu2wsKFH2Lp0g9RUlICpbIUnTp1ZmghoiZNLQQS0mS4dCsTl29nIbm80ZuLNcJCvNDF17lBNXozBIkQQujjRGfOnMGsWbNgaWkJuVyOjRs3wt+/7O6YBQsW4PTp0xBCYPPmzfDx8anVuaKirsPDw7Muyq4X5HI5rKzKFlpt2rQeSUl38f77iw1cVf2SkpKADh0qr90hIjKkUqUaV+9k4WxUKs5dS0NOfjGkUgk6tnFEcEc3BHdwh6uDpaHLbDD0ElqUSiXGjx+P6dOnIzAwEBcvXsScOXNw+PBhzV/GALBv3z4cPnwYmzZpty6jXHZ2AdTqvz9GWloC3NxqH1rKF+Ia2scfL8PVq1c0syfz5i2Ak5OzocuqU7Ud67r6njcFzs42BnmkfFPEsdav+jLehcVKXI3NRuTtTFyNzUZRSVmjt45tHBDg4wQ/b6cG3+hNV2MtlUrg6Fj5akI5vVweunHjBjIyMhAYGAgACAwMhIWFBWJiYuDn56fZb8iQIVi4cCFyc3M1d8sQMGfOW4YugYiIqpGTX4zLd8oW0kYn5JY1erM0QdcnXRDg44z2Xo2v0Zsh6CW0uLm5IS0tDbGxsWjTpg1iYmKQnZ0NJycnpKamwt3dHQDw22+/oVmzZrCzs6v1OYUQvC7YROjpCicRkYYQAslZ8rL+KbcyEX+/0ZurgyX6d32g0ZuUfw/VJb2EFmdnZ7z33nuYOXOmJkgsWbIEpqammDJlCoqKiiCVStGsWTOsX7++1mHD2NgUcnk+rKxsGVwaOSEE5PJ8GBubPnpnIqJaUKsF7iTf05L41A4AACAASURBVCykzcgrAgB4e9hieJ826OLr3KQavRmC3hbi6tI/17SoVErk5mZCqVTU6rhSqRRqteHXtDQFtRlrY2NT2Ns7w8iIPQy0UV+u+zcFHGv90sV4l5SqcD0up+wZP3ey7jd6k6C9lwP8fZzg/4QT7Jpgo7dGvaZF34yMjOHk5F7r4/AXjv5wrImovpAVKnDlTtlC2qi4skZvlmbG8HvCEV18nNGhtQMszBrlX5/1HkediIiavIzcwvvP98nC7aQ8CAE42Jqh1/1Gb75s9FYvMLQQEVGTI4RAfJoMkbczEXk7C8mZZY3eWt5v9Bbg44xWrmz0Vt8wtBARUZOgVKkRnZhbtj7ldhZyZSWQSiTwbdkMo57xgb+PE5ztLAxdJlWDoYWIiBqtopKyRm+Xbv3d6M3URIpOrR3h39sJnZ9o+I3emhKGFiIialRyZSW4fP+yz40HGr0FtXVBgK8z2nvaw9SEjd4aIoYWIiJq0FRqNZIz5Th2JRWnLichLvV+ozd7CzzbtSUCfJzg7dGMjd4aAYYWIiJqMJQqNVKy5EhIkyE+XYbENBnuZhRAcf/ZZW3uN3oL8HGGu6MlF9I2MgwtRERULylVZTMoCekyxKfJkJCWj7sZcihVZQHF3NQInq42eCqgOTzdbNAzoAXUCqWBqyZdYmghIiKDK1WqkZxVcD+clIWU5MwCKFVl3c4tzIzh6WqNfoEt0MrNGl5utnCxt4D0gZkUx2YWbFLZyDG0EBGRXpUqVUjKlGtmT8oCihyq+49jsTQzhqebDZ4NaglPNxt4utnA2a5iQKGmiaGFiIh0RlGqwt3MAs3sSUKaDClZfwcUK3NjeLnZYEC3VvBys0ErNxs4NzPnWhSqEkMLERHViZJSFe5mlAeU/PsBpRDq+8/ltbYwgZebDfy8HeHlZgNPVxs4MqDQY2BoISKix1asUCIxvQAJ6WWzJwlpMqRky3E/n8DW0gSebrbw93HWBBQHWzMGFKoVhhYiIqpWUYkSiekyJKQXaNagpGUX4n4+QTMrU3i62SCwrTM83Wzg5WYLO2tTBhSqcwwtRESkUVhcFlDi02Sa/0/P+Tug2FmbwsvNFt3auZYtknW1gb2NmUFrpqaDoYWIqIkqLC7VNGkrv8STnluk2W5vYwYvNxt07+CqucTTzJoBhQyHoYWIqAkoKCqtsP4kIU2GjLy/A4qjrRk83WwR0sldE1BsrUwNWDFRZQwtRESNTEFRqebunfLbjLPuFWu2OzUzh6ebDXp1dtdc4rGxZECh+o+hhYioAcsvVFQIJwlpMmTn/x1QXOws4OVuq2l17+lqA2sLEwNWTFRzDC1ERA3EPblCc/dOeVDJlZVotrvaW8C7uS36BjaHl2tZozYrcwYUajwYWoiI6qG8gpIKsyfxafnIK1AAACQAXB0s4dvSDp6uNmWdZF1tYGnOX+nUuPEnnIjIgIQQyCtQVFqDck/+d0Bxc7REO097eLqWPYenlasNLMz465uaHv7UExHpiRACOfklSEh/cA1KPvILSwEAEgng4WiFDq0dHggo1jA35a9qIoChhYhIZzJyC3HpZkaFNSgFRWUBRSqRwMPJEp28He9f4rFFSxdrmJkaGbhqovqLoYWIqI4lZxZg94lYXL6TBQAwkkrg4WQFfx8nzRqUFi7WMDNhQCF6HAwtRER1JPteMfadisUf19JgbmqE0f3boo2bDVq6WMHEmAGFqLYYWoiIaklWqMDhMwn47VIyAKB/15Z4rocXWrdyQGamzMDVETUeDC1ERDVUolDh6PlE/PhnIooVKvTs6I6I0NZwbGZu6NKIGiWGFiKix6RUqfH7lRQcOB2PfLkCAT5OGNa7DZo7Wxu6NKJGjaGFiEhLaiFw/kYG9v4ei4y8Ivi2aIZpQzvhiRbNDF0aUZPA0EJE9AhCCETF52DX8RgkpheghbMVZj7vBz9vR0gkEkOXR9RkMLQQEVUjLjUfu47H4EZCLpyamWPC4PYIbu8KqZRhhUjfGFqIiKqQmi3H3t9jceFmJmwsTTCqnw+e8m8OE2OpoUsjarIYWoiIHpArK8H+U3E49VcqTEykCO/phQHdWvFZP0T1AP8rJCICIC8uxZGzCfjlQhLUaoGnuzRHWIgXbK1MDV0aEd3H0EJETZqiVIVfLybh8JkEFJUo0b2DKyJ6tYGLnYWhSyOif2BoIaImSaVW4/TVNOw/FYdcWQn8vB0xrHcbtHK1MXRpRPQQDC1E1KQIIXDxZib2/B6LtJxCeHvYYmJYe7RtZW/o0ojoERhaiKjJuJGQi13HYxCXmg93R0tMG9YJAT5O7LVC1EAwtBBRo5eQJsPuEzG4FpcDexszvDrwSYR0coORlLcvEzUkDC1E1Ghl5BZi78k4nLueDitzY7zw9BPo26U5TE2MDF0aEdUAQwsRNTr3Ckpw8I94nLicAiOpBM/18MTA4FawNDcxdGlEVAsMLUTUaBSVKPHDuUT8fP4uSpVq9Pb3QHhPL9hZmxm6NCKqAwwtRNTglSrVOHYpCYfOJKCgqBRdn3TBsN5t4OpgaejSiKgOMbQQUYOlVguciUrDvpOxyM4vQXsvezz/lDe83GwNXRoR6QBDCxE1OEIIXLmTjd2/xyA5Uw5PNxuMG9QOHbwcDF0aEekQQwsRNSi37uZh14kY3Em6B1d7C7w+pCMC2zpDyl4rRI0eQwsRNQhJmQXYfTwGV2Ky0czaFGMHtEWonzuMjdhrhaipYGghonot614R9p2Mw5lraTA3M8bwPm3QL6glzNhrhajJYWghonpJVqjAoT8ScCwyCYAEA7q1wqAenrC2YK8VoqaKoYWI6pVihRJHz9/Fj+cSUVKqQs9O7hgS2hoOtuaGLo2IDIyhhYjqBaVKjROXU3DwdBzyC0vRxdcZw3q3gYeTlaFLI6J6gqGFiAxKLQT+vJ6OvSdjkZlXjLYt7TB9uDe8mzczdGlEVM8wtBCRQQghcC0uB7uPxyAxowAtXawxa0RndGrjAAlvXyaiKjC0EJHexaTcw+7jMYhOzINTM3NMCGuP4Pau7LVCRNViaCEivUnNlmP3iVhcupUJG0sTjO7ng6cCmrPXChFphaGFiHQuJ78Y+0/F4dTVVJiaGCEitDX6d20JCzP+CiIi7fE3BhHpTEFRKY6cTcCvF5OgVgs8E9gCg0O8YGtpaujSiKgBYmghojpXUqrCLxfu4oeziSgqUaJ7BzcM7dUaTnYWhi6NiBowhhYiqjNKlRqnrqbiwKk45BUo4OftiOF9vNHSxdrQpRFRI8DQQkS1JoTAxZuZ2P17LNJzCuHd3BaTIzrCt6WdoUsjokaEoYWIauV6fA52HY9BfJoMHk5WmD68E/yfcGKvFSKqcwwtRFQjCWky7Dp+B1HxuXCwNcNrg9ohpKMbpFKGFSLSDYYWInos6bmF2Pt7LP68kQErc2OM7PsE+nZpDhNjI0OXRkSNHEMLEWnlXkEJDpyOx+9XUmBkJMHgEE/8q5snLM35a4SI9IO/bYioWoXFSvz4ZwKOnr8LlUqgd2cPhPX0gp21maFLI6ImhqGFiKpUqlTh14vJOHwmHvJiJbq1c8HQ3m3gam9p6NKIqIliaCGiCtRqgdPXUrH/VBxy8kvQobUDnu/jDU83G0OXRkRNHEMLEQEo67Vy+XYWdv8ei5QsOVq72+Dfg9qhnZeDoUsjIgKgx9By7NgxrFq1CkIICCEwbdo0dO3aFfPmzUNiYiJMTU3h6emJDz74AA4O/CVJpE+37uZh1/EY3Em+B1cHS0wZ0hGBbZ3Za4WI6hW9hBYhBObNm4cdO3bA19cX0dHRGDVqFI4dO4bx48cjODgYALBs2TIsX74cS5Ys0UdZRE3e3YwC7D4Rg79ismFnbYpX/tUWoX7uMJJKDV0aEVEleptpkUqlkMlkAACZTAYXFxfY2dlpAgsA+Pv745tvvtFXSURNVmZeEfadjMXZqHRYmBnj+ae88UxgC5iZsNcKEdVfEiGE0MeJzpw5g1mzZsHS0hJyuRwbN26Ev7+/ZrtarcZrr72Gvn37YuzYsfooiajJkRUq8M3Rm/jhjzhIJRKE9WqD4X19YGNpaujSiIgeSS8zLUqlEhs2bMDatWsRGBiIixcvYtasWTh8+DCsrKwAAB9++CEsLS0xZsyYxz5+dnYB1Oq6z17OzjbIzJTV+XGpMo617pUqVfho+yUkpssQ6ueO8J6t4WBrjmJ5CYrlJYYur1Hiz7V+cbz1R1djLZVK4Oj48KfC6yW03LhxAxkZGQgMDAQABAYGwsLCAjExMfDz88OyZcuQkJCA9evXQ8pr6UQ6sePn24hPk2H+uG54wu3hvxSIiOorvSQENzc3pKWlITY2FgAQExOD7OxstGrVCitWrMC1a9ewZs0amJpyippIF07+lYLfr6TguR6e6NHJ3dDlEBHViF5mWpydnfHee+9h5syZmlsolyxZgszMTGzYsAFeXl548cUXAQAtWrTAmjVr9FEWUZOQkCbD9qO30M7THkN7tTF0OURENaa3u4fCw8MRHh5e6fWbN2/qqwSiJqegqBRr9l6FtYUJJkV0gFTKvitE1HBxAQlRI6UWApsPXUeurARThnaELe8QIqIGjqGFqJE6dDoef8VkY3Q/H3h7NDN0OUREtcbQQtQIXY3Nxv5TcejRwQ1PBTQ3dDlERHWCoYWokcnKK8LGA1Fo7myNsf9qy+cHEVGjwdBC1IiUKlVYs+8a1AKYOqwj2/ITUaPC0ELUiOz4+RYS0mQYP7gdXO0tDV0OEVGdYmghaiROXknB71dS8VwPTwT4OBu6HCKiOsfQQtQIJKTJ8NXRW2jvxQZyRNR4MbQQNXDlDeRsrUwwMZwN5Iio8WJoIWrA1EJg08GyBnKvD2EDOSJq3BhaiBqwg6fjcTWWDeSIqGlgaCFqoK7GZuPAqTiEdGQDOSJqGhhaiBqgBxvIvTyADeSIqGlgaCFqYB5sIDeNDeSIqAlhaCFqYMobyE0Y3B4ubCBHRE0IQwtRA/L7Aw3k/H2cDF0OEZFeMbQQNRDxafnYzgZyRNSEMbQQNQAFRaVYu/cabK1MMIkN5IioiWJoIarnyhvI5RWUYMqQTrBhAzkiaqIYWojqufIGcqP6+aKNh62hyyEiMhiGFqJ67K+YBxrI+XsYuhwiIoNiaCGqpzLzirDpYBRauLCBHBERwNBCVC+VKlVYu7esgdzUoWwgR0QEMLQQ1Uvbj95CQroME8LYQI6IqBxDC1E98/uVFJz8KxWDQzzh/wQbyBERlWNoIapHyhvIdfCyx5BQNpAjInoQQwtRPVFQVIo1e66hmZUJJrKBHBFRJQwtRPWAWi2w8WAU7slLMGUoG8gREVWFoYWoHjhwOg7XYnMwup8vWruzgRwRUVUYWogM7K+YLBw8HY+eHd3Qhw3kiIgeiqGFyIDKGshdRwsXa4xhAzkiomoxtBAZiKJUhTV7r0KwgRwRkVYYWogMZPvPt5CYXoDxbCBHRKQVhhYiA/j9SgpO/ZWKwSFebCBHRKQlhhYiPYtLvd9ArrUDhoS2NnQ5REQNBkMLkR4VFJVi7d77DeTC2rOBHBHRY2BoIdITtVpg4wE2kCMiqimGFiI9OXA6DtficjD6WTaQIyKqCYYWIj34KyYLB07Ho2cnN/TpzAZyREQ1wdBCpGMZeUXYeOA6WrlY4+X+bCBHRFRTDC1EOqQoVWHtnqsAgCnDOsGUDeSIiGqMoYVIR4QQ2H70FhIzCjAhrD1c7CwMXRIRUYPG0EKkI79fScGpq6kIC/FCZzaQIyKqNYYWIh2IS83Hjp/LGshFsIEcEVGdYGghqmOyQgXW7r2KZlammBTegQ3kiIjqCEMLUR1SqwU2HryOe3IFpgztBGsLE0OXRETUaDC0ENWh/afiEBWXg5fYQI6IqM4xtBDVkSt3snDwj3iEdnJHbzaQIyKqcwwtRHUgI68Imw5eRytXa4zp78sGckREOsDQQlRLFRrIDWUDOSIiXWFoIaoFIQS+OnoTiRkFmBjOBnJERLrE0EJUCyeupOD01TSE9/SCnzcbyBER6RJDC1ENxaXm4+ufb6FjaweE92QDOSIiXWNoIaqBvxvImWEiG8gREekFQwvRY6rYQK4jG8gREekJQwvRY9p3v4HcmP5t2UCOiEiPGFqIHsPlO1k49Ec8Qv3YQI6ISN8YWoi0lJFXhM3lDeSe9TV0OURETQ5DC5EWyhvISSTAVDaQIyIyCK1Cy5dffomcnBxd10JULwkh8NVPN3E3owATwtrDmQ3kiIgMQqvQcvbsWTzzzDOYNGkSjhw5AoVCoeu6iOqNE5dTcPpaGsLYQI6IyKC0Ci3r1q3Db7/9ht69e+PLL79Ez549sWDBApw/f17X9REZVFxqPr7+5RY6tnFAeCgbyBERGZLWa1rs7e3x0ksv4bvvvsNXX32Fq1evYuzYsejbty/WrVsHuVyuyzqJ9E5WqMCa8gZyYR0g5ZObiYgMyvhxdj5z5gwOHDiAX3/9FR07dsT48ePh4eGBbdu2YcKECfj66691VSeRXqnVAhsPRCFfXor5L3dhAzkionpAq9CybNkyHD58GDY2NoiIiMDBgwfh6uqq2d65c2d069ZNZ0US6du+U3GIis/FuIFPwsuNDeSIiOoDrUJLSUkJPvvsM/j5+VW53cTEBLt27arTwogM5fLtsgZyvdhAjoioXtEqtEyaNAnm5uYVXrt37x6Ki4s1My7e3t51Xx2RnmXkFmLToevwdLXBmP5sIEdEVJ9otRB3ypQpSEtLq/BaWloapk2bppOiiAyhpFSFNXuvQSoBpgztCBNjNpAjIqpPtAotcXFxaNu2bYXX2rZti9jYWJ0URaRvQghs/+kmkjIKMCGsAxvIERHVQ1qFFkdHRyQkJFR4LSEhAXZ2dlqf6NixYxgyZAgiIiIQHh6Oo0ePAihb5Nu3b1+0bdsWt27deozSiepOeQO58NDW8PN2NHQ5RERUBa3WtAwfPhzTp0/H7Nmz0bJlSyQmJmLVqlUYMWKEVicRQmDevHnYsWMHfH19ER0djVGjRqFfv3545plnMHbsWLz00ku1+iBENRWbUtZArlMbR4T19DJ0OURE9BBahZaJEyfC2NgYy5YtQ1paGtzc3DBixAi8+uqrWp9IKpVCJpMBAGQyGVxcXCCVShEUFFSzyonqgKxQgbX7rsLO2gwTwtqzgRwRUT2mVWiRSqUYP348xo8fX6OTSCQSrFy5ElOmTIGlpSXkcjk2btxYo2MR1RW1WmDD/QZyC14OZAM5IqJ6TuuOuAqFAnFxccjNzYUQQvN6jx49HvlepVKJDRs2YO3atQgMDMTFixcxa9YsHD58GFZWVjWr/AGOjta1PsbDODvb6OzYVJG+x/qrH27genwupr/gj6BOTasfC3+u9YdjrV8cb/0xxFhrFVouXLiAWbNmQaFQoKCgANbW1pDL5XBzc8Ovv/76yPffuHEDGRkZCAwMBAAEBgbCwsICMTExD21Y9ziyswugVotH7/iYnJ1tkJkpq/PjUmX6HuvLt7Ow85db6N3ZHQFtHJrU95k/1/rDsdYvjrf+6GqspVJJtRMRWt099NFHH2H8+PH4888/YWVlhT///BOvv/46Ro8erVURbm5uSEtL09wiHRMTg+zsbLRq1Uqr9xPVpfTyBnJuNnjpWTaQIyJqKLSaaYmPj8fYsWMrvDZx4kQ888wz+Pe///3I9zs7O+O9997DzJkzIbm/0HHJkiWws7PDokWLcPToUWRlZeHVV1+FnZ0dDh8+XIOPQvRoJaUqrNlT1kBu6hA2kCMiaki0Ci02NjYoKCiAra0tnJ2dcefOHdjZ2aGwsFDrE4WHhyM8PLzS62+//Tbefvtt7SsmqiEhBL766SaSMwsw64XOcGIDOSKiBkWr0PLss8/ixIkTCAsLw/DhwzF27FgYGxtjwIABuq6PqM4cv5yCP66lISK0NTq1YQM5IqKGRqvQsmDBAs2f//3vf6Nz586Qy+Xo1auXzgojqksxKffw9c+34OfNBnJERA3VIxfiqlQq9OvXDwqFQvNaUFAQ+vTpA6lUq3W8RAaVX6jA2r3XYG9jhvGD2UCOiKihemTqMDIygpGREUpKSvRRD1GdUqsFNh6IgqywFFOHdmIDOSKiBkyry0Njx47FrFmzMGnSJLi5uWnuAAKAli1b6qw4otraezIW1+Nz8erAJ+HpxqZTREQNmVah5cMPPwQAnD59usLrEokEN27cqPuqiOpA5O1MHD6TgN6d3dGrc9PqeEtE1BhpFVqio6N1XQdRnUrPLcRmNpAjImpUuJKWGp2/G8hJMHUoG8gRETUWWs20jB49usI6lgft2LGjTgsiqg0hBLb9WNZAbvYLneHUjA3kiIgaC61Cy4gRIyp8nZmZid27dyMsLEwnRRHV1PHIZJyJSsOQ0NboyAZyRESNilahZejQoZVeGzBgAP7v//4P06ZNq/OiiGoiJuUevv7lNvy8HTGYDeSIiBqdGq9pcXV1xc2bN+uyFqIaYwM5IqLGT6uZll27dlX4uri4GEePHoW/v79OiiJ6HGq1wIb9USgoKsX8MYFsIEdE1EhpFVr2799f4WtLS0sEBARg3LhxuqiJ6LHsPRmLGwm5eHUQG8gRETVmWoWWr776Std1ENVI5K3yBnIe6OXHBnJERI2ZVmta9u3bV6nBXHR0NPbt26eTooi0kZ5TiM2HyxvI+Ri6HCIi0jGtQsuqVavg7u5e4TU3NzesWrVKJ0URPUqJQoU1e6+ygRwRUROiVWgpKCiAtbV1hddsbGyQn5+vk6KIqiOEwLafopGcKcekiA5sIEdE1ERoFVq8vb3x008/VXjt559/hre3t06KIqrOschknIlKR0Sv1ujYmg3kiIiaCq0W4s6dOxcTJ07EDz/8gJYtWyIxMRFnzpzBxo0bdV0fUQUxyffwTXkDuRAvQ5dDRER6pNVMS1BQEA4dOoROnTqhqKgIfn5+OHToEAIDA3VdH5FGvlyBtfvKGshNCGMDOSKipkarmRaFQgFnZ2dMnDhR81ppaSkUCgVMTU11VhxROZVajQ0H/m4gZ2XOBnJERE2NVjMtr776KqKioiq8FhUVhX//+986KYron/b+HocbCbl4uX9bNpAjImqitAott27dQufOnSu85ufnV6l3C5EuRN7KxJGzCejj74FQP/dHv4GIiBolrUKLjY0NsrKyKryWlZUFCwveakq6Vd5AzsvNBqP7sYEcEVFTplVo6d+/P+bMmYNbt26hqKgIN2/exFtvvYWBAwfquj5qwsobyBlJpZjCBnJERE2eVqFl9uzZ8Pb2xogRI9ClSxeMHDkSrVu3xhtvvKHr+qiJEkLgy/sN5CaGt2cDOSIi0u7uITMzM7z77rtYuHAhcnNzYW9vDwlvNyUd+u1SMs5GpWMoG8gREdF9WoWWcnK5XPO/ci1btqzzoqhpi0m+h29/vY3O3o54jg3kiIjoPq1Cy507dzB37lxER0dDIpFACKGZablx44ZOC6SmpbyBnIOtGcazgRwRET1AqzUt77//PoKDg/Hnn3/C2toa58+fx8iRI7F06VJd10dNiEqtxvr911BQVIqpQzuxgRwREVWgVWiJjo7G3LlzYWtrCyEEbGxsMG/ePKxatUrX9VETsuf3WEQn5mHsgLZo5coGckREVJFWocXMzAxKpRIAYG9vj5SUFKjVauTl5em0OGo6zlxNxQ9nE/GUvwd6dmIDOSIiqkyrNS2BgYH44YcfMGzYMAwYMAATJkyAqakpunfvruv6qAlIzynEym8vobW7DUb18zV0OUREVE9pFVoevAz0xhtvwMfHB3K5HEOGDNFZYdQ0qNUC6/ZdK2sgN6QTTIy1mvwjIqIm6LFueQYAqVSKiIgIXdRCTdD56AwkZhTgzTGBcGxmbuhyiIioHuM/a8lg1GqBA6fj4OFkhdDOzQ1dDhER1XMMLWQw56MzkJpdiPCeXpBK2Y+FiIiqx9BCBvHgLEvQky6GLoeIiBoArULLokWLqnx98eLFdVoMNR0VZlnY9ZaIiLSgVWjZs2dPla8fOHCgTouhpoGzLEREVBPV3j20a9cuAIBKpdL8udzdu3dhZ2enu8qo0SqfZZkc0YGzLEREpLVqQ8v+/fsBAKWlpZo/A4BEIoGTkxOWLVum2+qo0eEsCxER1VS1oeWrr74CAHzyySeYPXu2Xgqixu3CTc6yEBFRzWi1puWVV16BXC4HUHapaPfu3di3bx/UarVOi6PGpWyWJZ6zLEREVCNahZZJkyYhISEBALBixQp8/vnn2Lp1K5YuXarT4qhxuXAzAylZct4xRERENaJVaImPj0e7du0AAAcPHsSmTZvw5Zdf4siRIzotjhoPzrIQEVFtaRVapFIpSktLcfPmTdjY2MDDwwO2traaS0ZEj8JZFiIiqi2tHpjYu3dvzJw5E3l5eRg0aBAA4M6dO3B1ddVpcdQ4cJaFiIjqglahZfHixdi7dy+MjY01T3jOzc3F9OnTdVocNQ7lsyy8Y4iIiGpDq9BiamqKkSNHQq1WIysrCy4uLggODtZ1bdQIcJaFiIjqilZrWvLz8zFnzhz4+fmhf//+AIBff/0Vn3zyiU6Lo4aPa1mIiKiuaBVa3n33XVhbW+O3336DiYkJACAgIAA//PCDToujhq3CLEtbzrIQEVHtaHV56MyZMzh58iRMTEwguf+vZQcHB2RnZ+u0OGrYKqxlkXKWhYiIakermRYbGxvk5uZWeC0lJQXOzs46KYoaPs6yEBFRXas2tBw6dAgAMGLECMyYMQNnz56FWq1GZGQk3nrrLbz44ot6KZIangprWTjLQkREdaDa0LJw4UIAwIQJEzBw4EB88MEHUCqVmD9/Pp555hm88soreimSGhbOshARkS5Uu6ZFCAEAkEgkeOWVVxhSSCtcy0JERLpQbWhRq9U4e/asJrxUpUePHnVeFDVcasFZ4CiuUwAAGMNJREFUFiIi0o1qQ4tCocCCBQseGlokEgl+/fVXnRRGDdOFaM6yEBGRblQbWiwsLBhKSGucZSEiIl3S6pZnIm2Uz7LwjiEiItKFakNLdWtZiB7EWRYiItK1akNLZGSkvuqgBo6zLEREpGu8PES1xlkWIiLSB4YWqrXyWZawEM6yEBGR7jC0UK2Uz7K4O1qi65OcZSEiIt1haKFa+XstS2vOshARkU4xtFCNcZaFiIj0iaGFaoyzLEREpE96Cy3Hjh3DkCFDEBERgfDwcBw9ehQAEBcXh5EjR2LAgAEYOXIk4uPj9VUS1QJnWYiISN+qbeNfV4QQmDdvHnbs2AFfX19ER0dj1KhR6NevH959912MHj0aERER2L9/PxYuXIht27bpoyyqhfJZlknhfMYQERHph95mWqRSKWQyGQBAJpPBxcUFubm5uH79OgYPHgwAGDx4MK5fv46cnBx9lUU1oBYCBznLQkREeqaXmRaJRIKVK1diypQpsLS0hFwux8aNG5GamgpXV1cYGRkBAIyMjODi4oLU1FQ4ODhofXxHR2tdlQ5nZxudHbuhOnUlGclZcrw5JhCurrZ1dlyOtf5wrPWHY61fHG/9McRY6yW0KJVKbNiwAWvXrkVgYCAuXryIWbNm4b///W+dHD87uwBqdd0/J8nZ2QaZmbI6P25DphYC2/+/vXsPjrK+9zj+yYVAAsSES0gQQU0FA6dySQgaBErEIHMgluFYUCtzPFhLRZuMOgXLaFMogXTmQBgFkZbqdIpilTt4wR6gFo4CVS4DBIGA3AK5gZAEyW1/54+cbIkLAiH7PPs8+37NMOPus9n95udOfh8+z8PmgwIldIxSr67RLbY+rLV1WGvrsNbWYr2t46+1Dg0N+d4iwpLTQwUFBSopKVFycrIkKTk5WZGRkWrdurWKi4tVX18vSaqvr1dJSYkSEhKsGAvN8MVXpTrFvxgCANjAktASHx+vM2fO6MiRI5KkwsJClZeXq0ePHkpKStK6deskSevWrVNSUtINnRqCdTzGaM2Wo1zLAgCwhSWnhzp37qycnBxlZWUpJKThb+e5ubmKiYlRTk6Opk2bpoULFyo6Olp5eXlWjIRmaGxZ+BdDAAA7WBJaJCkzM1OZmZk+9ycmJuq9996zagw0Ey0LAMBufCIurgvXsgAA7EZowTXRsgAAAgGhBdfU2LKMGXw7LQsAwDaEFnyvy1uW1Lu72D0OACCIEVrwvWhZAACBgtCCq6JlAQAEEkILroqWBQAQSAgtuCJaFgBAoCG04IpoWQAAgYbQAh8eY7RmKy0LACCwEFrg48uvSnWqlJYFABBYCC1owmOMVtOyAAACEKEFTdCyAAACFaEFXrQsAIBARmiBFy0LACCQEVogiZYFABD4CC2QRMsCAAh8hBbQsgAAHIHQgn+1LGm0LACAwEVoCXKNLUt8hyilJtGyAAACF6ElyDW2LJlcywIACHCEliBGywIAcBJCSxCjZQEAOAmhJUjRsgAAnIbQEqRoWQAATkNoCUIeY7SGlgUA4DCEliD05VelOknLAgBwGEJLkKFlAQA4FaElyNCyAACcitASRGhZAABORmgJIrQsAAAnI7QECVoWAIDTEVqCRGPLMoaWBQDgUISWINDYsnTpEKVBtCwAAIcitAQBrmUBALgBocXlaFkAAG5BaHE5WhYAgFsQWlyMlgUA4CaEFhejZQEAuAmhxaVoWQAAbkNocamdB2lZAADuQmhxIY8xWr3la1oWAICrEFpcqKFlqaRlAQC4CqHFZWhZAABuRWhxGVoWAIBbEVpchJYFAOBmhBYXoWUBALgZocUlaFkAAG5HaHEJb8uSRssCAHAnQosLXN6ypPaOs3scAAD8gtDiApe3LGGh/C8FALgTO5zD0bIAAIIFocXhaFkAAMGCXc7BaFkAAMGE0OJgtCwAgGDCTudQHmO0ZistCwAgeBBaHGrnwTKdKKFlAQAED3Y7B2poWY7SsgAAggqhxYFoWQAAwYgdz2FoWQAAwYrQ4jC0LACAYMWu5yC0LACAYEZocZDGlmVMWg9aFgBA0GHncwhvyxIbqUG9u9g9DgAAliO0OIS3ZRnMtSwAgODE7ucAtCwAABBaHIGWBQAAQkvAo2UBAKABoSXA0bIAANCAXTCA0bIAAPAv4Va8yMmTJzVlyhTv7YqKClVWVmr79u3avHmz5s+fr7q6Ot1yyy2aPXu2brvtNivGCni7DjW0LE+NTqJlAQAEPUtCS7du3bR69Wrv7VmzZqm+vl7nz5/X1KlTtWzZMt1xxx1avXq1cnJytGTJEivGCmjGGK3ZQssCAEAjy//6XlNTo7Vr12rcuHE6duyYOnXqpDvuuEOSNGzYMG3ZskVnz561eqyAs/NQmY5zLQsAAF6WNC2X27hxo7p06aI+ffqooqJCZWVl2rNnj+655x6tXbtWknT69Gl16NDhup+zY8d2/hpXnTu399tzX40xRh/8+Qt17dRWo4f+QGFhwRFa7FjrYMVaW4e1thbrbR071try0LJ8+XKNGzdOktS+fXvNmzdPs2fPVnV1tYYOHaro6GiFhYXd0HOWl1fK4zEtPmvnzu1VWlrR4s97LV8eLNWRovN6anSSzp6tsvz17WDXWgcj1to6rLW1WG/r+GutQ0NDvreIsDS0FBcXa8eOHfr973/vvS8tLU1paWmSpLKyMi1ZskTdu3e3cqyAwrUsAABcmaXnHVauXKlhw4YpNjbWe19paakkyePxaO7cuZowYYKioqKsHCugcC0LAABXZnloaTw11Cg/P1+jRo1SRkaGWrVqpRdffNHKkQJKY8sSR8sCAIAPS08Pffzxxz73zZo1y8oRAlpjyzLp3/lcFgAAvoudMUBc3rLc24eWBQCA7yK0BAjvtSxpXMsCAMCVsDsGAFoWAACujdASAGhZAAC4NnZIm9GyAABwfQgtNqNlAQDg+rBL2oiWBQCA60dosdEuWhYAAK4bO6VNjDFaTcsCAMB1I7TYhJYFAIAbw25pA1oWAABuHKHFBrQsAADcOHZMi9GyAADQPIQWi9GyAADQPOyaFqJlAQCg+QgtFqJlAQCg+dg5LeJtWWJoWQAAaA5Ci0W8LctgWhYAAJqD3dMCtCwAANw8QosFaFkAALh57KB+RssCAEDLILT4GS0LAAAtg13Uj4wxWr2VlgUAgJZAaPGjXYfLdLyYlgUAgJbATuonXMsCAEDLIrT4CS0LAAAti93UD2hZAABoeYQWP6BlAQCg5bGjtjBaFgAA/IPQ0sIaW5bR/CZnAABaFLtqC7q8Zbnv32hZAABoSYSWFkTLAgCA/7CzthBaFgAA/IvQ0kJoWQAA8C921xZAywIAgP8RWloALQsAAP7HDnuTaFkAALAGoeUm7T5cTssCAIAF2GVvAi0LAADWIbTchN2Hy3WsuIKWBQAAC7DTNhMtCwAA1iK0NBMtCwAA1mK3bQZaFgAArEdoaQZaFgAArMeOe4MaW5bOMW1oWQAAsBCh5QbRsgAAYA923RvQpGXpE2/3OAAABBVCyw24vGUJD2PpAACwEjvvdaJlAQDAXoSW60TLAgCAvdh9rwMtCwAA9iO0XAdaFgAA7McOfA3GGK3eSssCAIDdCC3XsLuwXMfO0LIAAGA3duHvwbUsAAAEDkLL99hRUEzLAgBAgGAnvgpjjN75+AAtCwAAAYLQchW7C8t1+OR5WhYAAAIEu/EVNF7LEt8xipYFAIAAEW73AIHoXEW1jp2p0JzErar5YJNqLjsWfmeqIvo8IFNXrW8/nOvzta163q9WvYbIc6lClz55zfd473S1ShwkT2W5Lm1a7HM84p6HFN6jvzzfnNalf7zle7x/psK79VF92TFVf/a2z/HWA/9DYfF3qf7MIVXveN/3+H2PKaxTD9Wd3KeanWt8jrcZ8p8KjUlQ3bGdqtnzke/x4U8rtF1H1RZuU+3+jb7HH3xWoW3aq/arf6j24Baf45GjnldIeGvV7Psf1R3Z7r2/qFW4amvrFDXmJUlSze4PVXd8V9MvDo9Q1KgXJEnVX65W/an9TQ6HtG6nyIznGo5vf0/1xYebHm/bQZHpP5ckXfrfpfKUH29yPPSWeLUZ+mTD8U/flOf8mabHO3ZXm7THJUnfbnxDpupsk+NhXX6g1qmPNBzf8KpMdWXT47f2VusBD0uSLn7431JdTZPj4d37KaLvqIbja2fru1rqvVd3oUwX1/p+fbC+9xr5473X+L7mvWfNz71LJw/o4oY/+x4Pwvee97i/3nv/levzfViBpuUKOkS30eyf36vY9m3sHgUAAPy/EGOMsXuIm1VeXimPp+W/jc6d26u0tKLFnxe+WGvrsNbWYa2txXpbx19rHRoaoo4d2139eIu/IgAAgB8QWgAAgCMQWgAAgCMQWgAAgCMQWgAAgCMQWgAAgCMQWgAAgCMQWgAAgCNY8jH+J0+e1JQpU7y3KyoqVFlZqe3bt2vTpk2aP3++jDEyxujZZ59VRkaGFWMBAAAHsSS0dOvWTatXr/benjVrlurr62WM0a9+9SstXbpUPXv21IEDB/Too49qxIgRCg2lBAIAAP9ieTKoqanR2rVrNW7cuIYBQkNVUdHwUcAVFRWKi4sjsAAAAB+W/+6hjz76SK+//rq3efnss8+UnZ2tqKgoVVVVafHixerXr5+VIwEAAAew5PTQ5ZYvX+5tWerq6vTGG29o4cKFSk5O1hdffKHs7GytX79ebdu2ve7n5BcmOh9rbR3W2jqstbVYb+sExS9MLC4u1o4dOzRmzBhJUkFBgUpKSpScnCxJSk5OVmRkpAoLC60cCwAAOICloWXlypUaNmyYYmNjJUnx8fE6c+aMjhw5IkkqLCxUeXm5unfvbuVYAADAASw9PbRy5UpNnz7de7tz587KyclRVlaWQkJCJEm5ubmKiYmxciwAAOAAll+I6w/nzlX55ZqWjh3bqby8ssWfF75Ya+uw1tZhra3FelvHX2sdGhqi2NirX9PqitACAADcjw9EAQAAjkBoAQAAjkBoAQAAjkBoAQAAjkBoAQAAjkBoAQAAjkBoAQAAjkBoAQAAjkBoAQAAjkBoAQAAjkBouYKjR49q/PjxGjlypMaPH6+vv/7a7pFc6dy5c/rZz36mkSNHasyYMXr22Wd19uxZu8dyvddee029evXSwYMH7R7F1aqrq/Wb3/xGGRkZGjNmjF5++WW7R3KtTZs26cc//rEefvhhZWZmasOGDXaP5Bp5eXlKT0/3+Zlh2z5p4OOJJ54wq1atMsYYs2rVKvPEE0/YPJE7nTt3znz++efe23PmzDEvvfSSjRO53969e82kSZPM8OHDzVdffWX3OK42c+ZMM2vWLOPxeIwxxpSWlto8kTt5PB6TkpLifT8XFBSYfv36mfr6epsnc4cdO3aYoqIin58Zdu2TNC3fUV5erv3792v06NGSpNGjR2v//v00AH4QExOjQYMGeW/369dPRUVFNk7kbjU1NZoxY4ZycnLsHsX1qqqqtGrVKmVlZSkkJESS1KlTJ5uncq/Q0FBVVFRIkioqKhQXF6fQULa3lpCSkqKEhIQm99m5T4b7/RUc5vTp0+rSpYvCwsIkSWFhYYqLi9Pp06fVoUMHm6dzL4/Ho3feeUfp6el2j+Ja8+fPV2Zmprp162b3KK534sQJxcTE6LXXXtO2bdvUtm1bZWVlKSUlxe7RXCckJET5+fl65plnFBUVpaqqKi1evNjusVzNzn2SKIqAMHPmTEVFRemnP/2p3aO40s6dO7V371499thjdo8SFOrr63XixAn17t1bK1as0IsvvqjnnntOlZWVdo/mOnV1dXrjjTe0cOFCbdq0Sa+//rqys7NVVVVl92jwA0LLdyQkJKi4uFj19fWSGn74lJSU+NRjaDl5eXk6duyY8vPzqXT9ZMeOHSosLNQDDzyg9PR0nTlzRpMmTdKWLVvsHs2VEhISFB4e7q3P+/btq9jYWB09etTmydynoKBAJSUlSk5OliQlJycrMjJShYWFNk/mXnbuk+wQ39GxY0clJSVp3bp1kqR169YpKSmJU0N+MnfuXO3du1cLFixQRESE3eO41tNPP60tW7Zo48aN2rhxo+Lj47VkyRLdf//9do/mSh06dNCgQYO0detWSQ3/0qK8vFw9evSweTL3iY+P15kzZ3TkyBFJUmFhocrLy9W9e3ebJ3MvO/fJEGOM8furOExhYaGmTZumCxcuKDo6Wnl5ebrzzjvtHst1Dh06pNGjR+v2229XmzZtJEndunXTggULbJ7M/dLT07Vo0SL17NnT7lFc68SJE/r1r3+tb775RuHh4crOztawYcPsHsuV1qxZoz/84Q/ei55/+ctfasSIETZP5Q6/+93vtGHDBpWVlSk2NlYxMTFav369bfskoQUAADgCp4cAAIAjEFoAAIAjEFoAAIAjEFoAAIAjEFoAAIAjEFoABLRt27Zp6NChVz0+bdo0zZs3r0Vf85VXXuGf3gMBiN89BAS59PR0lZWVeX+PiCSNHTtWr7zyio1T2WvGjBl2jwDgCggtALRo0SKlpaXZPQYAfC9ODwG4qhUrVujRRx9VXl6eBg4cqPT0dP39739vcvyBBx5Q//79lZ6erjVr1niPvf/++xo1apQGDhyoSZMm6dSpU95jvXr10tKlS5WRkaH+/fsrPz9fx48f14QJEzRgwABlZWWppqamySyLFi3SoEGDfF7nuzZt2qSHH35YKSkpmjBhgg4cOHDFxxljlJubq/vuu08DBgzQmDFjdPDgQUlNTzlNnjxZ/fv39/65++67tWLFCkkNn5795JNPKjU1VSNHjtQHH3xwgysM4IYYAEFt+PDhZuvWrVc8tnz5ctO7d2/z7rvvmrq6OrN06VIzePBg4/F4TFVVlenfv78pLCw0xhhTXFxsDh48aIwx5pNPPjEjRowwhw8fNrW1tWbBggVm/Pjx3uft2bOnmTx5sqmoqDAHDx40ffr0MRMnTjTHjx83Fy5cMKNGjTIrVqwwxhjz+eefm6SkJJObm2uqq6vNtm3bTN++fb2vO3XqVDN37lxjjDH79u0z9957r9m1a5epq6szK1asMMOHDzfV1dU+39unn35qxo4da86fP288Ho85fPiwKS4u9nnOy23evNkMHjzYFBUVmaqqKjN06FDz/vvvm9raWrNv3z6TmppqDh061Nz/FQCugaYFgKZMmaKUlBTvn7/+9a/eY127dtVPfvIThYWFaezYsSotLVVZWZkkKTQ0VIcOHdKlS5cUFxenu+66S5K0bNkyPf3000pMTFR4eLgmT56sgoKCJm3LU089pXbt2umuu+5Sz549NXjwYN12221q3769hg4dqv379zeZMSsrSxEREUpNTdWwYcP04Ycf+nwf7777rsaPH6++fft6523VqpV27drl89jw8HBVVVXpyJEjMsYoMTFRcXFxV12jo0ePatq0acrPz1dCQoI2b96sW2+9VePGjVN4eLh69+6tkSNH6qOPPrqxxQdw3bimBYAWLFhw1WtaOnXq5P3vyMhISdLFixfVuXNnzZs3T3/60580ffp0DRgwQFOnTlViYqKKioqUm5urvLw879caY1RcXKxbb73V53lbt27tc7sxGElSdHS0oqKivLe7du2qkpISn1mLioq0atUq/eUvf/HeV1tbe8XH3nfffXr88cc1Y8YMnTp1ShkZGZo6daratWvn89iKigo988wzys7OVkpKiiTp1KlT2rNnj/e2JNXX1yszM/NKywigBRBaADTbkCFDNGTIEF26dEn5+fl6+eWX9fbbbyshIUGTJ09usQ38woULunjxoje4nD592tvqXK7xdX/xi19c1/NOnDhREydOVHl5ubKzs/XHP/5R2dnZTR7j8Xj0wgsvaNCgQRo/fnyT1xo4cKDefPPNm/jOANwITg8BaJaysjL97W9/08WLFxUREaGoqCiFhjb8SJkwYYIWL16sQ4cOSWpoKq50OudGvPrqq6qpqdE///lPbd68WQ899JDPYx555BEtW7ZMu3fvljFGFy9e1ObNm1VZWenz2D179mj37t2qra1VZGSkIiIivPNfbt68efr22281ffr0Jvf/6Ec/0tdff61Vq1aptrZWtbW12rNnjwoLC2/q+wRwdTQtADR58uQmn9OSlpZ2zQ9X83g8euuttzR16lSFhIQoKSlJOTk5kqQHH3xQVVVVev7553Xq1Cm1b99eaWlpGjVqVLPm69Spk6KjozVkyBBFRkYqJydHiYmJPo/74Q9/qJkzZ2rGjBk6duyY2rRpowEDBjQ5hdOoqqpKubm5OnnypCIiInT//fdr0qRJPo9bv369SktLlZqa6r3vt7/9rTIzM7VkyRLNmTNHc+bMkTFGvXr10ksvvdSs7xHAtYUYY4zdQwAAAFwLp4cAAIAjEFoAAIAjEFoAAIAjEFoAAIAjEFoAAIAjEFoAAIAjEFoAAIAjEFoAAIAj/B+IkmWsBgbB+wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 648x576 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s21FDW4qBbA1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "74b850a4-b1b5-4c64-b51c-a3d623ed880a"
      },
      "source": [
        "wandb.init(entity='authors', project='loss-landscape', id='medium_cnn_ensemble')\n",
        "wandb.log({'ensemble_function': wandb.Image('ensemble_func.png')})"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/authors/loss-landscape\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/authors/loss-landscape/runs/medium_cnn_ensemble\" target=\"_blank\">https://app.wandb.ai/authors/loss-landscape/runs/medium_cnn_ensemble</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}