Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Estimation of Confidence Intervals for Machine Learning\n",
    "This notebook presents the usage of python functions to perform empirical\n",
    "estimation of confidence intervals for machine learning models. This kind of confidence interval estimation is often\n",
    "called \"Bootstrap Confidence Interval\" or \"Monte Carlo Confidence Interval\" estimation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris, load_breast_cancer\n",
    "\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier \n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.naive_bayes import GaussianNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import utils as cie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic Usage\n",
    "The basic usage of the functions is presented by computing different confidence intervals for a random-forest classifier over the iris dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data loading and presentation\n",
    "\n",
    "iris_dataset = load_iris()\n",
    "iris_X, iris_y = iris_dataset['data'], iris_dataset['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(array([5.1, 3.5, 1.4, 0.2]), 0),\n",
       " (array([4.9, 3. , 1.4, 0.2]), 0),\n",
       " (array([4.7, 3.2, 1.3, 0.2]), 0),\n",
       " (array([4.6, 3.1, 1.5, 0.2]), 0),\n",
       " (array([5. , 3.6, 1.4, 0.2]), 0)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(a, b) for (a, b) in zip(iris_X, iris_y)][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(array([6.7, 3. , 5.2, 2.3]), 2),\n",
       " (array([6.3, 2.5, 5. , 1.9]), 2),\n",
       " (array([6.5, 3. , 5.2, 2. ]), 2),\n",
       " (array([6.2, 3.4, 5.4, 2.3]), 2),\n",
       " (array([5.9, 3. , 5.1, 1.8]), 2)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(a, b) for (a, b) in zip(iris_X, iris_y)][-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parametric Estimation of Confidence Intervals\n",
    "In order to perform parametric estimation, we need to fit a classifier to the data and evaluate it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = RandomForestClassifier(n_estimators=20, max_depth=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 50)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(iris_X, iris_y, test_size=0.33, random_state=24)\n",
    "len(X_train), len(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=3, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=20,\n",
       "                       n_jobs=None, oob_score=False, random_state=None,\n",
       "                       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 1, 1, 2])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = clf.predict(X_test)\n",
    "preds[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is 0.98\n"
     ]
    }
   ],
   "source": [
    "accuracy = sum(preds == y_test)/len(y_test)\n",
    "print(f\"Accuracy is {accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 90%, 95% and 99% Parametric Confidence Intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_size = len(X_test)\n",
    "sample_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing C.I. assuming the data follow a standard normal distribution\n",
      "With 90.0% confidence the accuracy of the model is in 0.977 +/-0.023.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9773132867658845, 0.02268671323411553)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_90 = cie.parametric_confidence_interval(confidence_level=.9, mean=accuracy, sample_size=sample_size, verbose=True)\n",
    "param_90"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing C.I. assuming the data follow a standard normal distribution\n",
      "With 95.0% confidence the accuracy of the model is in 0.974 +/-0.026.\n"
     ]
    }
   ],
   "source": [
    "param_95 = cie.parametric_confidence_interval(confidence_level=.95, mean=accuracy, sample_size=sample_size, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing C.I. assuming the data follow a standard normal distribution\n",
      "With 99.0% confidence the accuracy of the model is in 0.967 +/-0.033.\n"
     ]
    }
   ],
   "source": [
    "param_99 = cie.parametric_confidence_interval(confidence_level=.99, mean=accuracy, sample_size=sample_size, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAacAAAEYCAYAAAD4czk4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de7xcVX338c+XcPXCzcSgCTc1KkEt6Gm8tJIoogEvIFI0glzkMfoSrFW8QL0Q01pbq1WpiEUfRCyCyFNrHsWmFgjaCm1O5A4NxmhNAuhRQLRQIPDtH2sdHIZDMpA5mT0z3/frNa/M3nvtxZpzfpzf7LXXXku2iYiIaJItet2AiIiIdklOERHROElOERHROElOERHROElOERHROFv2ugGPxNSpU73HHnv0uhkREQGsWLHil7anTUbdfZWc9thjD0ZHR3vdjIiIACT912TVnW69iIhonCSniIhonCSniIhonCSniIhonI6Sk6QzJf1C0rUPc1ySTpW0StLVkp7bcuxoST+qr6Nb9j9P0jX1nFMladM/TkREDIJOr5zOAuZv4PiBwKz6WgicDiBpZ+AU4PnAHOAUSTvVc04H3tJy3obqj4iIIdJRcrL9PeDWDRQ5GDjbxeXAjpKeBLwC+K7tW23fBnwXmF+PbW/7cpdp0c8GDtmkTxIREQOjW/ecZgBrWrbX1n0b2r92gv0RERHNHxAhaaGkUUmjY2Njj/j8RYsWIekhr0WLFnW/sREbkFiMJuiXOFSniw1K2gP4lu1nTXDs74Blts+t2yuBeeMv229tLVdfl9h+Zt2/oLXcwxkZGfGjnSFi3rx5ACxbtuxRnR/RLYnFaIJuxKGkFbZHutOiB+vWldMS4Kg6au8FwK9t3wwsBV4uaac6EOLlwNJ67A5JL6ij9I4CvtmltkRERJ/raG49SedSroKmSlpLGYG3FYDtzwMXAgcBq4A7gWPrsVsl/RmwvFa12Pb4wIq3U0YBbgd8p74iIiI6S062F2zkuIHjH+bYmcCZE+wfBR7SRRgREdH4ARERETF8kpwiIqJxkpwiIqJxkpwiIqJxkpwiIqJxkpwiIqJxkpwiIqJxkpwiIqJxkpwiIqJxkpwiIqJxkpwiIqJxkpwiIqJxkpwiIqJxOpqVvJ/tsssu/PznP39guywfVUyfPp1bbrmlF82KiIgNGPgrp9bE9EiORURE73SUnCTNl7RS0ipJJ01wfHdJF0m6WtIySTPr/pdIurLl9T+SDqnHzpL0k5Zj+3T3o0VERL/aaLeepCnAacABwFpguaQltq9vKfYJ4GzbX5b0UuBjwJtsXwLsU+vZmbJS7j+3nPde2xd056NENFu6mCM618mV0xxgle3Vtu8BzgMObiszG7i4vr9kguMAhwHfsX3no21sRD9LF3NE5zpJTjOANS3ba+u+VlcBh9b3rwUeL+kJbWXeAJzbtu+jtSvwU5K2meg/LmmhpFFJo2NjYx00NyIi+l23BkS8B5gr6QpgLrAOuG/8oKQnAc8GlracczLwTOD3gZ2B909Use0zbI/YHpk2bVqXmhsREU3WSXJaB+zasj2z7nuA7ZtsH2p7X+ADdd/tLUUOB75h+96Wc252cTfwJUr3YddNnz79UR2LiIje6SQ5LQdmSdpT0taU7rklrQUkTZU0XtfJwJltdSygrUuvXk2hclf4EODaR978jbvllluwzdy5c5k7dy62H3jlBnRERDNtdLSe7fWSTqB0yU0BzrR9naTFwKjtJcA84GOSDHwPOH78fEl7UK68Lm2r+hxJ0wABVwJv2+RPExERG9Qvo0Y7miHC9oXAhW37Ptzy/gJgwiHhtn/KQwdQYPulj6ShEf1u+vTpDzsqL13Msbn0y6jRgZ++KKIpxr+Rzps3D4Bly5b1rjERDTfw0xdFRET/SXKKiIjGSXKKiIjGSXKKiIjGSXKKiBgi/TIxQUbrRUQMkX4ZNZorp4iIaJwkp4iIaJwkp4iIaJwkp4iIaJwkp4iIaJwkp4iIaJyBT06LFi1CEpdeeimXXnopkpDEokWLet20iIh4GB0lJ0nzJa2UtErSSRMc313SRZKulrRM0syWY/dJurK+lrTs31PSv9c6v1YXMuy6RYsWPWiBwfFXklNERHNtNDlJmgKcBhwIzAYWSJrdVuwTwNm2nwMsBj7Wcuwu2/vU12ta9v8V8CnbTwNuA47bhM8REREDpJMrpznAKturbd8DnAcc3FZmNnBxfX/JBMcfpC7N/lJ+t0DhlylLtUdERHSUnGYAa1q21/LQlW2vAg6t718LPF7SE+r2tpJGJV0uaTwBPQG43fb6DdQJgKSF9fzRsbGxDpobERH9rlsDIt4DzJV0BTAXWAfcV4/tbnsEeCPwaUlPfSQV2z7D9ojtkWnTpnWpuRER0WSdJKd1wK4t2zPrvgfYvsn2obb3BT5Q991e/11X/10NLAP2BX4F7Chpy4erM2LQZORoROc6SU7LgVl1dN3WwBuAJa0FJE2VNF7XycCZdf9OkrYZLwP8AXC9bVPuTR1Wzzka+OamfpiIJsvI0YjObTQ51ftCJwBLgRuA821fJ2mxpPHRd/OAlZJuBKYDH6379wJGJV1FSUZ/afv6euz9wLslraLcg/q/XfpMERHR5zpaz8n2hcCFbfs+3PL+An438q61zA+AZz9MnaspIwEjIiIeZOBniIiIiP6T5BQREY2T5BQREY2T5BQREY2T5BQREY2T5BQREY2T5BQREY2T5BQREY2T5BQREY2T5BQRMUT6ZQJilTlY+8PIyIhHR0d73YyIiAAkrahLInVdrpwiIqJxkpwiIqJxkpwiIqJxOkpOkuZLWilplaSTJji+u6SLJF0taZmkmXX/PpIuk3RdPfb6lnPOkvQTSVfW1z7d+1gREdHPNpqcJE0BTgMOBGYDCyTNbiv2CeBs288BFgMfq/vvBI6yvTcwH/i0pB1bznuv7X3q68pN/CwRETEgOrlymgOssr3a9j3AecDBbWVmAxfX95eMH7d9o+0f1fc3Ab8ApnWj4RERMbg6SU4zgDUt22vrvlZXAYfW968FHi/pCa0FJM0BtgZ+3LL7o7W771OStpnoPy5poaRRSaNjY2MdNDciIvpdtwZEvAeYK+kKYC6wDrhv/KCkJwFfAY61fX/dfTLwTOD3gZ2B909Use0zbI/YHpk2LRddERHDYMsOyqwDdm3Znln3PaB22R0KIOlxwOts3163twe+DXzA9uUt59xc394t6UuUBBcREdHRldNyYJakPSVtDbwBWNJaQNJUSeN1nQycWfdvDXyDMljigrZznlT/FXAIcO2mfJCIiBgcG01OttcDJwBLgRuA821fJ2mxpNfUYvOAlZJuBKYDH637Dwf2A46ZYMj4OZKuAa4BpgJ/3q0PFRER/S1z60VExKOSufUiImKoJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjdJScJM2XtFLSKkknTXB8d0kXSbpa0jJJM1uOHS3pR/V1dMv+50m6ptZ5al10MCIiYuPJSdIU4DTgQGA2sEDS7LZin6CsdvscYDHwsXruzsApwPOBOcApknaq55wOvAWYVV/zN/nTRETEQOjkymkOsMr2atv3AOcBB7eVmQ1cXN9f0nL8FcB3bd9q+zbgu8D8ukT79rYvd1nt8GzKUu0REREdJacZwJqW7bV1X6urgEPr+9cCj5f0hA2cO6O+31CdAEhaKGlU0ujY2FgHzY2IiH7XrQER7wHmSroCmAusA+7rRsW2z7A9Yntk2rRp3agyIiIabssOyqwDdm3Znln3PcD2TdQrJ0mPA15n+3ZJ64B5becuq+fPbNv/oDojImJ4dXLltByYJWlPSVsDbwCWtBaQNFXSeF0nA2fW90uBl0vaqQ6EeDmw1PbNwB2SXlBH6R0FfLMLnyciIgbARpOT7fXACZREcwNwvu3rJC2W9JpabB6wUtKNwHTgo/XcW4E/oyS45cDiug/g7cAXgVXAj4HvdOtDRUREf1MZLNcfRkZGPDo62utmREQEIGmF7ZHJqDszREREROMkOUVEROMkOUVEROMkOUVEROMkOUVEROMkOUVEROMkOUVEROMkOUVEROMkOUVEROMkOUVEROMkOUVEROMkOUVEROMkOUVEROMkOUVERON0lJwkzZe0UtIqSSdNcHw3SZdIukLS1ZIOqvuPkHRly+t+SfvUY8tqnePHntjdjxYREf1qo8u0S5oCnAYcAKwFlktaYvv6lmIfpCxCeLqk2cCFwB62zwHOqfU8G/hH21e2nHeE7SzQFBERD9LJldMcYJXt1bbvAc4DDm4rY2D7+n4H4KYJ6llQz42IiNigTpLTDGBNy/bauq/VIuBISWspV03vmKCe1wPntu37Uu3S+5AkddbkiIgYdN0aELEAOMv2TOAg4CuSHqhb0vOBO21f23LOEbafDby4vt40UcWSFkoalTQ6NjbWpeZGRESTdZKc1gG7tmzPrPtaHQecD2D7MmBbYGrL8TfQdtVke1399zfAVyndhw9h+wzbI7ZHpk2b1kFzIyKi33WSnJYDsyTtKWlrSqJZ0lbmZ8D+AJL2oiSnsbq9BXA4LfebJG0paWp9vxXwKuBaIiIi6GC0nu31kk4AlgJTgDNtXydpMTBqewlwIvAFSe+iDI44xrZrFfsBa2yvbql2G2BpTUxTgH8BvtC1TxUREX1Nv8shzTcyMuLR0Yw8j4hoAkkrbI9MRt2ZISIiIhonySkiIhonySkiIhonySkiIhonySkiIhonySkiIhonySkiIhonySkiIhonySkiIhonySkiIhonySkiIhonySkiIhonySkiIhonySkiIhonySkiIhqno+Qkab6klZJWSTppguO7SbpE0hWSrpZ0UN2/h6S7JF1ZX59vOed5kq6pdZ4qSd37WBER0c82mpwkTQFOAw4EZgMLJM1uK/ZB4Hzb+1KWcf9cy7Ef296nvt7Wsv904C3ArPqa/+g/RkREDJJOrpzmAKtsr7Z9D3AecHBbGQPb1/c7ADdtqEJJTwK2t315Xc79bOCQR9TyiIgYWJ0kpxnAmpbttXVfq0XAkZLWAhcC72g5tmft7rtU0otb6ly7kToBkLRQ0qik0bGxsQ6aGxER/a5bAyIWAGfZngkcBHxF0hbAzcButbvv3cBXJW2/gXoewvYZtkdsj0ybNq1LzY2IiCbbsoMy64BdW7Zn1n2tjqPeM7J9maRtgam2fwHcXfevkPRj4On1/JkbqTMiIoZUJ1dOy4FZkvaUtDVlwMOStjI/A/YHkLQXsC0wJmlaHVCBpKdQBj6stn0zcIekF9RRekcB3+zKJ4qIiL630Ssn2+slnQAsBaYAZ9q+TtJiYNT2EuBE4AuS3kUZHHGMbUvaD1gs6V7gfuBttm+tVb8dOAvYDvhOfUVERKAyWK4/jIyMeHR0tNfNiIgIQNIK2yOTUXdmiIiIiMZJcoqIiMZJcoqIiMZJcoqIiMZJcoqIiMZJcoqIiMZJcoqIiMZJcoqIiMZJcoqIiMZJcoqIiMZJcoqIiMZJcoqIiMZJcoqIiMZJcoqIiMbpKDlJmi9ppaRVkk6a4Phuki6RdIWkqyUdVPcfIGmFpGvqvy9tOWdZrfPK+npi9z5WRET0s40uNlhXsj0NOABYCyyXtMT29S3FPgicb/t0SbOBC4E9gF8Cr7Z9k6RnURYsnNFy3hG2s0BTREQ8SCdXTnOAVbZX274HOA84uK2Mge3r+x2AmwBsX2H7prr/OmA7SdtserMjImKQdZKcZgBrWrbX8uCrH4BFwJGS1lKumt4xQT2vA35o++6WfV+qXXofkqSJ/uOSFkoalTQ6NjbWQXMjIqLfdWtAxALgLNszgYOAr0h6oG5JewN/Bby15ZwjbD8beHF9vWmiim2fYXvE9si0adO61NyIiGiyTpLTOmDXlu2ZdV+r44DzAWxfBmwLTAWQNBP4BnCU7R+Pn2B7Xf33N8BXKd2HERERHSWn5cAsSXtK2hp4A7CkrczPgP0BJO1FSU5jknYEvg2cZPvfxgtL2lLSePLaCngVcO2mfpiIiBgMG01OttcDJ1BG2t1AGZV3naTFkl5Ti50IvEXSVcC5wDG2Xc97GvDhtiHj2wBLJV0NXEm5EvtCtz9cRET0J5Uc0h9GRkY8OpqR5xERTSBphe2Ryag7M0RERETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjJDlFRETjdJScJM2XtFLSKkknTXB8N0mXSLpC0tWSDmo5dnI9b6WkV3RaZ0REDK+NJidJU4DTgAOB2cACSbPbin2QsgjhvpSVcj9Xz51dt/cG5gOfkzSlwzojImJIdXLlNAdYZXu17XuA84CD28oY2L6+3wG4qb4/GDjP9t22fwKsqvV1UmdERAypLTsoMwNY07K9Fnh+W5lFwD9LegfwWOBlLede3nbujPp+Y3UCIGkhsLBu/lbSyg7a/HCmAr/chPMjuiWxGE2wqXG4e7ca0q6T5NSJBcBZtj8p6YXAVyQ9qxsV2z4DOKMbdUkanawlhSMeicRiNEGT47CT5LQO2LVle2bd1+o4yj0lbF8maVtKRt7QuRurMyIihlQn95yWA7Mk7Slpa8oAhyVtZX4G7A8gaS9gW2CslnuDpG0k7QnMAv6jwzojImJIbfTKyfZ6SScAS4EpwJm2r5O0GBi1vQQ4EfiCpHdRBkccY9vAdZLOB64H1gPH274PYKI6J+HztetK92BEFyQWowkaG4cqOSQiIqI5MkNEREQ0TpJTREQ0TpJTREQ0TpJTF0hSr9sQAYnFaI5NjcVuPYQ7tCSpjkxE0nOBW4B7bOfp/9is2mLxRcCvgdts37ThMyO6qxuxmCunTdTyCzgeOBV4K3CepJ172rAYOm2x+AngEODfJD2xpw2LodONWExy6gJJ+wGvA15Bmfj2DuA2Sfn5xmYlaQ5lEuUDKM8c3gj8Mt19sbltaizmj+ejMMEP97+BbwDHAHsBC+o3hwPqVE4Rk2KCWPwV8B3geGAucLDt+4HDJD1+c7cvhke3YzEP4T5CbX2pB1CWB7kNuAz4re2967E3AwcBb7Z9R6/aG4OrLRYPAW6lzMbyA2BL20+px94EHAEcYftXvWpvDK7JiMUkp0dJ0ruBw4Bjba+U9BrgfcA3gfspM7Ufa/uaHjYzhkCdNuyPgLfUqcV+D/hH4Ku1yCtILMZm0M1YTHJ6FOoP/LPAq2z/uu4T8FxK194vKSsD39CzRsZQkPRUyvxor7N9u6StbN8raRZwKHAPcKHtTVkHLWKjuh2LGUregdZL1upOyqCHrSVNsX2fbUu60fY7etTMGAITxOLdwHbAk4Dbbd9b999h+682ewNjaEx2LGZAxEa09aVuK2kbyrNMdwCzKcuDIOkI4E8lbdezxsZAa4vFnSQ9zvZa4N+B50maWY8dAfxFBkDEZNkcsZhuvQ1o+wWcCPwB8BvgA8C+lEUWbwZupwwlP8T29T1qbgywtlh8L2V47nbA/wEeR3m+bgZluO4rSSzGJNlcsZjk1AFJfwgsBk6hDIl8PWXl38cALwJ2Af7B9o961sgYCpLmAh8E3gK8FjiZEoPrKAt+TgW+Z3t1zxoZQ2GyYzH3nDZC0oHAu4Fv2P4+8H1J91DG77/R9pd72sAYGvVL0onAVbZ/CnxKkoHvA6+2/a1eti+Gx+aIxdxzajPBg2Q/pHTbvUjSNADbHwe+DnypLkGfn2N03QSxeC3l2ZE9JM2p3SufBj4NfD2xGJOlF7GYbr0WbX2ph1NGn9xGucl3LvCfwKdt/6KW2dn2rb1qbwyutlg8ChBwr+2vSvoE5Vm6C2z/Ry2zk+3betfiGFS9isV8y2rR8gt4J3ACZZ68syiDH94OPAP4UMsVVBJTTIqWWDyBcqN5DfAFSa8E/oLy/+6bJT2vnnJ7TxoaA69XsZjkBEiaOj4EXNIuwItt7wfsDlwD/ND2LcDbgB0p3xwiuk7Srqoz2kvaCXghZTTUPsDFwHfrl6KPAGOUPxQP/AGJ6JZex+LQd+tJ2gP4MGWKjYsog0S+SPlBP4Uyietdko6lDIL4RZ28MKKr6hejDwCrga9Q5ic7G/gtZeTTUbbvrN9gl9m+tmeNjYHWhFgc+iunOtLkGuBlwH51OqJVwFGU+aHuqv2s7wC2SGKKyVKvzi+mPCNyeI21K4CFwAn1j8EbKV0rmUw4Jk0TYnFoh5KPjz6pl6A/p8z9tL+k+yjzQwFcLOmfKGP2j3JWFI1J0nLTeUfKzCMHSVoPnEa593mRpH+lzN94pO2f9a61MciaEotD2603/guoV0XHA28G3glMoYw8+Y6kgyiXsWts/6SHzY0hIOkwykONhwJHAtOA62x/vt5svg+4NYkpJlsTYnHorpwkvQK4y/b36q7nUGZ3uA5YKOnDwJ9LmgJcZPuuXrU1Bpukl1DWuvlu3fUUYGl9on6xpGOA4+t8jl/J6NCYLE2MxaG65zQ+GSGwVtKM+n4F8FSV6d6xvRjYCngJ5SoqouskPY3ybfRGSbvV3ZcDe6ksb43ts4BfA7tRlrmO6LqmxuLQXDnVaYg+y+9mEr9NZYHAb1EGQxwiaQXweGAt8Bnbv+1Ve2NwSXoZ8DnKVftWwBpJxwP/QLnp/Ecqa+DcWY9/Kg/YxmRociwOzT0nSe8Dtqf8wK8CngWcCRwM/AQ4ljKO/zGU0SgZphuTQtJC4AXApZQpYKYCn6HM4fhDygCc11P69T9k++oeNTUGXJNjcZiS0x8CHwN2BQ6yfb2k1wJfpkzpfnF9EHcb23naPiaNpF0p02E9DZhv+8o6+OazwIm2vyFpK2Ar23f2sq0x2JociwPdrdc6JxRljrwtKN8Q9pK0rv7gDfyLpFfb/jaQARDRdW2xCGVdsH8CXiFpre0LJb0d+JrK6soXAPdOWFnEJuiXWBzYK6e2yQqfQJnv6TGU6Tf2B0aB823/d50j6se2/7NnDY6B1RaLM4Ff1EPPoDzUeBvwSdu/rvcAfmp7VW9aG4Osn2JxIJNT2y/gHcAxwD8DFwL/ChwB/D5llvEvp+skJoukB2YVqRMKv5Xy5P0K21+S9CLgMMoIqI/YzswPMSn6LRYHsluvJTEdBDyfcnNvHuUHv73tv5e0LfB0ygiUiEnR8sfgJZR+/aOAWcBLJT3W9mdrn/6BwDa9a2kMun6LxYG8cgKQ9CxgGfBntj8j6cmUp51nUSYq/Iak7Xv97SAGm8qCa3tR5m/8uO2T6kzPLwQOAVba/oSkbW3/Ty/bGoOt32JxYB7CHZ8rb1wdCv53wHslPaPOi3c+5RmmF9ZvCklM0XWtsWj7/jr7yAnAH0t6Vn26/nuUbuY9VBZn6/kfgxg8/RyLA3Hl1HaP6WXAk4HllGUvjqE8w3Sk7RtUFgpcn4caYzK0xeLBlCv1K23/i6SjKctYv6QO2X08cL/t/+5hk2NA9XssDsQ9p5ZfwJ9QktEVwOuAfwPOoXzOb0k60PaNvWpnDL6WWHwncDRljbCj6jN176NMifVDSc/Jg94xmfo9Fvs6OUnaHbjb9i0qKzXOAw6wPSZpf+CVwHNsf7r2t+a5kZgUdYoX2b6x3lR+LvAa22slzQb+CDjG9ml1ME5iMSbFoMRi395zkvRy4KuUNZh2pyxt8VjKdETYvoiydPCCuv03zrIXMQlUZrr/KrCfpKdTpnp5MqU7GdvXUxawfHHd/pztlT1qbgywQYrFvkxOkuZT+ks/YPsc2/9l+17gLMpNvfm16E+BuyRt3ZuWxqCTdADwKcpUL1+0fWMdsrsYmCHp2Fr0bmALSY/tVVtjsA1aLPbVgIg68mQryoSt/2j7grabfrtSLlkPpEzvvjdlieFretXmGEw1FregTJJ5eX12rjUWt6fMRnIi5Sn8Z5BYjEkwqLHYV/ec6g/7Hkl3AzfX3VOA9fX4GkmXAl+jPGB7o+11PWlsDLQai/dJ+h/KN1GosVj/MNwh6UbKH4XdgV/Z/nmPmhsDbFBjsS+79ShXT+N9qOvbuu32AX5t+5IkptgMtqAsY/1ALPp33REvBXa0fX0//DGIvjdQsdhXyanlgbJFwI6S3gVg+556/FjgTUDuMcXmchKwraRPwoNi8WjgjcA9PWxbDJeBisW+uOfUOmFh3d4GmE+5evo1cB6lG+9twGH1KeiIrmubPHML2/dLeg7wN5QulW8CO1JiM7EYk2bQY7HRyUnSsx/upl0dv78L8EHKGkyPoSyt3le/gOgPkva1fcUGjk8BTqEM3d2OMtv9DZurfTE8hiUWm56cPk65KrrCG2moyqJY922elsWwkfQ3wCXAt9pjUdKWtte3bLcv5hbRNcMSi02/53QH8MyWIZE71W8F1O0pLWXvbz85ootWALu0xOIuknaEB24+N/3/pRgcQxGLjbtykvRCyuXof1G66o62vUjSKcBM4AbK1O7f7mEzYwhImkuJwZ8DPwH+0vZbayzuRpnh/mbbn+9hM2MIDGMsNirDSpoD/D/KnHjvpKw9srekXWqR9wDXU8bqR0waSc+lLLHyNMpAmz8E7pc0HfgNZWTUWcDUXrUxhoOkfRnCWGxEcmoZIv5EyiJYpwC3U66Svgc8G3gW8CLKtO+H1BF7EZNFlFj8W+D7lG+sVwGvoowMfSllgbb5kh7Ts1bGwGr5uzgF+Othi8VGJCfKDx/gSuDo+rzSccDvUWbM3ZOyQNYrKTPsvtf23RNVFLEpJD2lznD/C+CVkt4KfBh4KqU75bHAyZQvSXsCx9u+s1ftjYE2/ndxDSXxvJ0hisWe33OSdCAlEV0BXAb8EPgDYBvb/yBpB+BPbb+/Xi3dXyd5jegqSQdRHk14o+2fSnoK5Wp+B9tLa/ydCvwxZcqs+/t1JFQ0W1114SjK38OrgR9Q5gqdavs7wxCLPZ1br95jOpUy44Pr+6/b/kg9viVltoenSnq6s1BgTJK61MAngTfXxLSl7dXA6np8K0pX3xRglhu4OFsMhjr44TOUvxeslK0AAALcSURBVIu/BP4WONv2X9bjQxGLvZ74dWtgme1zACRdAvygzgn1gTpef0zS14FGrGsfg6d24x0DXGb7sjos948l3QGM1WVZ7gXulfT/gcYsZR0D6cnAF21/DUDSOcCH6ywQfzEssdjre053A7tI2hnA9s3AC4EDJR0+Xsj212z/rEdtjAFn+zbg74FbJH2M0oXyOMoEw++UtLCl7DedRStjcj0WOLL2HEF5hvP9lL+L+40XGvRY3OxXTipLCO9ge9T2ckkrge8CzwNwWXL9NGD65m5bDJcaizvZ/g/b35b0G+BPgM/Z/mwtsw54Zi/bGYOvxuKOtpfb/mK95XF5XeriybbnSXoisENvW7r5bNbkJOkQ4CPAKkk3Az+0/R5JfydpFHiV7Vso4/Vn1yedPWg3+qL32mLxJuAayiKWNwK/aim6GzC9n6eBiWZri8VbgFHbCyXtDWxLGTZOfT80z3hutuQk6QnAW4EFtq+X9GZKv/7O9UnnjwOnqywk+HuUWXQzJVF03cPE4gnANMrkwffWckcCr6eM3ktiiq7bwN/FJwGftX1HLXcccDiwf+9au3ltzntO6yn9+LsA2D4T+BGwp6TX2n4f8CHgr4EDM7t4TKKJYvHHlGHjrwaQ9HzK2mBH9uOMztE3JorFVZTbGq+CB7r89gZeaXtVj9q52W3W55wkjU+9sZTSj7875YnnF9k+erM1JIbeBmJxju3japmd6mCJiEnTYSxuZ/uu3rVy89vcAyLOpcwFtT9wu+0jASS9WtL245ewEZvBhmJxJ9u3JTHFZrKhWNzR9u3DlpigRzNE6MErOB5FmczwANsDO2Y/mimxGE2RWHywnjyE2/ILeDNlpvHXD+svIHorsRhNkVh8sJ7OrSdpd2CrYbrJF82UWIymSCwWPZ/4NSIiol2vpy+KiIh4iCSniIhonCSniIhonCSniIhonCSniIhonCSniIhonCSniIhonP8FCOvfErUnF+wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cie.plot_util([param_90, param_95, param_99], ylim=(.8, 1.01))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parametric Estimation: t-student Distribution\n",
    "If the size of the sample for the evaluation of the model has less than 30 datapoints,the function performs the estimation of the confidence intervals under the assumption that the data was drawn from a t-student distribution. Confidence interval estimates with a small sample size tend to be much more unreliable and you should try, if possible, to avoid to use them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = RandomForestClassifier(n_estimators=20, max_depth=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(121, 29)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(iris_X, iris_y, test_size=0.19, random_state=42)\n",
    "len(X_train), len(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=3, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=20,\n",
       "                       n_jobs=None, oob_score=False, random_state=None,\n",
       "                       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is 1.0%\n"
     ]
    }
   ],
   "source": [
    "preds = clf.predict(X_test)\n",
    "accuracy = sum(preds == y_test)/len(y_test)\n",
    "print(f\"Accuracy is {accuracy}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_size = len(X_test)\n",
    "sample_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing C.I. assuming the data follow a t-distribution (sample_size < 30)\n",
      "With 90.0% confidence the accuracy of the model is in 1.0 +/-0.0.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0, 0.0)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cie.parametric_confidence_interval(confidence_level=.9, mean=accuracy, sample_size=sample_size, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing C.I. assuming the data follow a t-distribution (sample_size < 30)\n",
      "With 95.0% confidence the accuracy of the model is in 1.0 +/-0.0.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.0, 0.0)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cie.parametric_confidence_interval(confidence_level=.95, mean=accuracy, sample_size=sample_size, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Non-Parametric Estimation of Confidence Intervals\n",
    "In order to perform non-parametric estimation, we need to fit several classifier to resamplings of the original dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These parameters will be passed to the constructor of RandomForestClassifier for the creation of each new instance\n",
    "# Notice that the values are the same of the classifier instantiated for parametric estimation.\n",
    "\n",
    "rfc_params_dict = {\"n_estimators\":20, \"max_depth\":3}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating model on 100 samples of (X, y).\n",
      "Each sample will contain 150 elements from (X, y), out of which 100 elements\n",
      "will be used to train the model, while the remaining 50 elements to test its accuracy.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05c238a298fd4013a7df4571259919d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# The accuracy of a random-forest classifier is evaluated on 100 samples of the iris dataset. Each sample is sampled\n",
    "# with replacement from the original dataset.\n",
    "\n",
    "accs=cie.get_accuracy_samples(RandomForestClassifier, iris_X, iris_y,\n",
    "                             model_params_dict=rfc_params_dict,\n",
    "                             n_iterations=100, sample_ratio=1,\n",
    "                             train_ratio=0.67, random_seed=None,\n",
    "                             sample_with_replacement=True, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0.98), (1, 0.96), (2, 0.96), (3, 1.0), (4, 1.0)]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(k, accs[k]) for k in range(5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the given data, with 90.0% confidence the accuracy of the model is in 0.95 +/-0.05.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.95, 0.04999999999999993)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Confidence intervals are estimated from the performance of the models in the 100 evaluations.\n",
    "\n",
    "nonparam_90=cie.nonparametric_confidence_interval(accs, .9)\n",
    "nonparam_90"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the given data, with 95.0% confidence the accuracy of the model is in 0.95 +/-0.05.\n"
     ]
    }
   ],
   "source": [
    "nonparam_95=cie.nonparametric_confidence_interval(accs, .95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the given data, with 99.0% confidence the accuracy of the model is in 0.94 +/-0.06.\n"
     ]
    }
   ],
   "source": [
    "nonparam_99=cie.nonparametric_confidence_interval(accs, .99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparison of the Estimates\n",
    "The following plot compares nonparametric and parametric estimates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZhU1Z3/8feHBkRxQQXBgAImTMIioCwmJqY7EhWNImrcgiDGBB1FE5cY/JlowySacUxQE5fRjOIWlzg6YRIyjkaaLMKEVnHDSHCJNKI2uEVR2b6/P+7tTlk0UA0ldav683qeeqx77rm3z6WO9a177lkUEZiZmWVNu1IXwMzMrCUOUGZmlkkOUGZmlkkOUGZmlkkOUGZmlkntS12A1ujatWv06dOn1MUwM7MievTRR5dHRLf89LIKUH369KG+vr7UxTAzsyKS9LeW0t3EZ2ZmmeQAZWZmmeQAZWZmmeQAZWZmmVRQgJJ0k6TXJT29gf2SdLWkxZKelLRvzr6TJf01fZ2ckz5M0lPpMVdL0pZfjpmZVYpC76BmAKM3sv9QoF/6mgRcByBpF+ASYD9gJHCJpJ3TY64Dvplz3MbOb2ZmbUxBASoifg+8sZEsRwK3RmIe0EXS7sAhwIMR8UZEvAk8CIxO9+0YEfMimU79VmDsFl2JmZlVlGI9g+oJLMnZbkjTNpbe0EK6mZkZUAYDdSVNImk2ZM8999ysc9TW1jJ16tT10i+55BJqa2u3pHibVFdXx5w5c9ZLr66upqam5mP927b1ua7Z1tIW6poKXbBQUh/g1xExqIV9/w7URcSd6fZzQE3TKyJOy82XvmZHxGfS9BNz823I8OHDY0tmkmj6h6urq9vsc2yuGTNmADBx4sSt/rdt63Nds62lEuqapEcjYnh+erGa+GYCE9LefJ8F3o6IZcADwMGSdk47RxwMPJDue0fSZ9PeexOAXxWpLGZmVgEKauKTdCfJ3VBXSQ0kPfM6AETE9cAs4DBgMbASOCXd94akfwHmp6eaFhFNnS3OIOkduC3w2/RlZmYGFBigIuLETewP4MwN7LsJuKmF9HpgveZCMzMz8EwSZmaWUQ5QZmaWSQ5QZmaWSQ5QZmaWSQ5QZmaWSQ5QZmaWSQ5QZmaWSQ5QZmaWSQ5QZmaWSQ5QZmaWSQ5QZmaWSQ5QZmaWSQ5QZmaWSZlfUbcYevTowWuvvda8nSxBlejevTuvvvpqKYplZmYb0SbuoHKDU2v2mZlZ6RQUoCSNlvScpMWSprSwv7ek30l6UlKdpF5p+pckLch5fSBpbLpvhqQXc/YNLe6lmZlZOdtkE5+kKuAa4CCgAZgvaWZELMzJdgVwa0TcIulA4DJgfETMBoam59mFZMXd/8057jsRcW9xLsWs9NycbFY8hdxBjQQWR8QLEbEKuAs4Mi/PAODh9P3sFvYDfBX4bUSs3NzCmmWdm5PNiqeQANUTWJKz3ZCm5XoCODp9fxSwg6Rd8/KcANyZl/bDtFlwuqRtWvrjkiZJqpdU39jYWEBxzcysEhSrk8T5QLWkx4FqYCmwtmmnpN2BvYEHco65EPgMMALYBfhuSyeOiBsiYnhEDO/WrVuRimtmZllXSIBaCuyRs90rTWsWEa9ExNERsQ9wUZr2Vk6W44D7I2J1zjHLIvEhcDNJU+LHonv37pu1z8zMSqeQADUf6Cepr6SOJE11M3MzSOoqqelcFwI35Z3jRPKa99K7KpQ8RR4LPN364hfm1VdfJSKorq6murqaiGh++aG1mVk2bbIXX0SskTSZpHmuCrgpIp6RNA2oj4iZQA1wmaQAfg+c2XS8pD4kd2Bz8k59h6RugIAFwOlbfDVmZm1EW+gxWtBMEhExC5iVl3Zxzvt7gRa7i0fES6zfqYKIOLA1BTUrB927d99gbz03J1sxtYUeo21iqiOzraXpV2tNTQ0AdXV1pSuMWZlrE1MdmZlZ+XGAMjOzTHKAMjOzTHKAMjOzTHKAMjMrQ21hAgL34jMzK0Ntoceo76DMzCyTHKDMzCyTHKDMzCyTHKDMzCyTHKDMzCyTHKDMzCyT2kSAqq2tRRJz5sxhzpw5SEIStbW1pS6amZltQEHjoCSNBq4iWQ/q5xHxo7z9vUkWKewGvAGcFBEN6b61wFNp1pcjYkya3he4C9gVeBQYHxGrtviKWlBbW1uyYHTFFVfw3nvvNW9PnTq1+X3nzp05//zzS1Esq0Cua1ZpNnkHJakKuAY4FBgAnChpQF62K4BbI2IwMA24LGff+xExNH2NyUn/V2B6RHwKeBM4dQuuI7NyvzBas8+stVzXrNIU0sQ3ElgcES+kdzh3AUfm5RkAPJy+n93C/o9Il3k/kH8scngLybLvZmZmQGEBqiewJGe7gfVXyH0CODp9fxSwg6Rd0+1OkuolzZPUFIR2Bd6KiDUbOScAkialx9c3NjYWUFwzM6sExeokcT5QLelxoBpYCqxN9/WOiOHA14ArJX2yNSeOiBsiYnhEDO/WrVuRimtmZllXSIBaCuyRs90rTWsWEa9ExNERsQ9wUZr2Vvrfpel/XwDqgH2AFUAXSe03dE6zcuQeo2bFU0iAmg/0k9RXUkfgBGBmbgZJXSU1netCkh59SNpZ0jZNeYDPAwsjIkieVX01PeZk4FdbejFmpVZbW0tErPdygDJrvU0GqPQ50WTgAeBZ4J6IeEbSNElNvfJqgOckLQK6Az9M0/sD9ZKeIAlIP4qIhem+7wLnSlpM8kzqP4p0TZnSuXPnzdpn1lqua1ZpChoHFRGzgFl5aRfnvL+Xf/TIy83zCLD3Bs75AkkPwYrWNPZkxowZAEycOLF0hbGK5rpmlcYLFpqZWatsrUHhbWKqIzMzK56tNSjcAcrMzDLJAcrMzDLJAcrMzDLJAcrMzDLJAcrMzDLJAcrMzFplaw0K9zgoMzNrla01KNx3UGZmZagtTEzsOygzszJUW1tbUcGoJb6DMjOzTHKAMjOzTHKAMjOzTCooQEkaLek5SYslTWlhf29Jv5P0pKQ6Sb3S9KGS5kp6Jt13fM4xMyS9KGlB+hpavMsyM7Nyt8kAJakKuAY4FBgAnChpQF62K4BbI2IwMA24LE1fCUyIiIHAaOBKSV1yjvtORAxNXwu28FrMzKyCFHIHNRJYHBEvRMQq4C7gyLw8A4CH0/ezm/ZHxKKI+Gv6/hXgdaBbMQpuZmaVrZAA1RNYkrPdkKblegI4On1/FLCDpF1zM0gaCXQEns9J/mHa9Ddd0jYt/XFJkyTVS6pvbGwsoLhmZlYJitVJ4nygWtLjQDWwFFjbtFPS7sBtwCkRsS5NvhD4DDAC2AX4bksnjogbImJ4RAzv1s03X2ZmbUUhA3WXAnvkbPdK05qlzXdHA0jaHjgmIt5Kt3cEfgNcFBHzco5Zlr79UNLNJEHOzMwMKOwOaj7QT1JfSR2BE4CZuRkkdZXUdK4LgZvS9I7A/SQdKO7NO2b39L8CxgJPb8mFmJlZZdlkgIqINcBk4AHgWeCeiHhG0jRJY9JsNcBzkhYB3YEfpunHAV8EJrbQnfwOSU8BTwFdgR8U66LMzKz8FTQXX0TMAmblpV2c8/5e4N4WjrsduH0D5zywVSU1M7M2xTNJmJlZJjlAmZlZJjlAmZlZJjlAmZlZJjlAmZlZJjlAmZlZJjlAmZlZJjlAmZlZJjlAmZlZJjlAmZlZJjlAmZlZJjlAmZlZJjlAmZlZJjlAmZlZJhUUoCSNlvScpMWSprSwv7ek30l6UlKdpF45+06W9Nf0dXJO+jBJT6XnvDpduNDMzAwoIEBJqgKuAQ4FBgAnShqQl+0KklVzBwPTgMvSY3cBLgH2A0YCl0jaOT3mOuCbQL/0NXqLr8bMzCpGIXdQI4HFEfFCRKwC7gKOzMszAHg4fT87Z/8hwIMR8UZEvAk8CIxOl3vfMSLmRUQAt5Is+25mZgYUtqJuT2BJznYDyR1RrieAo4GrgKOAHSTtuoFje6avhhbS1yNpEjAJYM899yyguNlSV1fHnDlzmrenTp0KQHV1NTU1NSUqlVUi1zWrNAUt+V6A84GfSZoI/B5YCqwtxokj4gbgBoDhw4dHMc65NdXU1PjLwbYK1zWrNIUEqKXAHjnbvdK0ZhHxCskdFJK2B46JiLckLQVq8o6tS4/vlZf+kXOamVnbVsgzqPlAP0l9JXUETgBm5maQ1FVS07kuBG5K3z8AHCxp57RzxMHAAxGxDHhH0mfT3nsTgF8V4XrMzKxCbDJARcQaYDJJsHkWuCcinpE0TdKYNFsN8JykRUB34IfpsW8A/0IS5OYD09I0gDOAnwOLgeeB3xbroszMrPwV9AwqImYBs/LSLs55fy9w7waOvYl/3FHlptcDg1pTWDMzazs8k4SZmWWSA5SZmWWSA5SZmWWSA5SZmWWSA5SZmWWSA5SZmWWSA5SZmWWSA5SZmWWSA5SZmWWSA5SZmWWSA5SZmWWSA5SZmWWSA5SZmWVSsVbUNTOzNqKuro45c+Y0b0+dOhWA6urqoq7qXFCAkjQauAqoAn4eET/K278ncAvQJc0zJSJmSRoHfCcn62Bg34hYIKkO2B14P913cES8viUXY2ZmH7+ampqiBqIN2WSAklQFXAMcBDQA8yXNjIiFOdm+R7KQ4XWSBpCsHdUnIu4A7kjPszfwXxGxIOe4cem6UGZmZh9RyDOokcDiiHghIlYBdwFH5uUJYMf0/U7AKy2c58T0WDMzs00qJED1BJbkbDekablqgZMkNZDcPZ3VwnmOB+7MS7tZ0gJJ35ekwopsZmZtQbF68Z0IzIiIXsBhwG2Sms8taT9gZUQ8nXPMuIjYGzggfY1v6cSSJkmql1Tf2NhYpOKamVnWFRKglgJ75Gz3StNynQrcAxARc4FOQNec/SeQd/cUEUvT//4d+AVJU+J6IuKGiBgeEcO7detWQHHNzKwSFBKg5gP9JPWV1JEk2MzMy/MyMApAUn+SANWYbrcDjiPn+ZOk9pK6pu87AIcDT2NmZpbaZC++iFgjaTLwAEkX8psi4hlJ04D6iJgJnAfcKOkckg4TEyMi0lN8EVgSES/knHYb4IE0OFUBDwE3Fu2qzMys7OkfcST7hg8fHvX17pVuZlZJJD0aEcPz0z3VkZmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZZIDlJmZZVJBAUrSaEnPSVosaUoL+/eUNFvS45KelHRYmt5H0vuSFqSv63OOGSbpqfScV0tS8S7LzMzK3SYDlKQq4BrgUGAAcKKkAXnZvgfcExH7kCwJf23OvucjYmj6Oj0n/Trgm0C/9DV68y/DzMwqTSF3UCOBxRHxQkSsAu4CjszLE8CO6fudgFc2dkJJuwM7RsS8dGn4W4GxrSq5mZlVtEICVE9gSc52Q5qWqxY4SVIDMAs4K2df37Tpb46kA3LO2bCJcwIgaZKkekn1jY2NBRTXzMwqQbE6SZwIzIiIXsBhwG2S2gHLgD3Tpr9zgV9I2nEj51lPRNwQEcMjYni3bt2KVFwzM8u69gXkWQrskbPdK03LdSrpM6SImCupE9A1Il4HPkzTH5X0PPBP6fG9NnFOMzNrwwq5g5oP9JPUV1JHkk4QM/PyvAyMApDUH+gENErqlnayQNJeJJ0hXoiIZcA7kj6b9t6bAPyqKFdkZmYVYZN3UBGxRtJk4AGgCrgpIp6RNA2oj4iZwHnAjZLOIekwMTEiQtIXgWmSVgPrgNMj4o301GcAM4Btgd+mLzMzMwCUdKIrD8OHD4/6+vpSF8PMzIpI0qMRMTw/3TNJmJlZJjlAmZlZJhXSiy/TVq9eTUNDAx988EGpi2KbqVOnTvTq1YsOHTqUuihmliFlH6AaGhrYYYcd6NOnD57Or/xEBCtWrKChoYG+ffuWujhmliFl38T3wQcfsOuuuzo4lSlJ7Lrrrr4DNrP1lH2AAhycypw/PzNrSUUEqELV1tYiab1XbW1tqYtmZmZ52lyAigiqq6uprq4mIoiILQ5QVVVVDB06lEGDBnHssceycuXK4hR4C1166aUb3X/YYYfx1ltvbaXSmJm1TpsKUD169EASc+bMYc6cOR+5i+rRo8dmn3fbbbdlwYIFPP3003Ts2JHrr79+0wel1q5du9l/d1M2FKAignXr1jFr1iy6dOnysf19M7Mt0aYC1GuvvbZZ+1rjgAMOYPHixQCMHTuWYcOGMXDgQG644YbmPNtvvz3nnXceQ4YMYe7cuUybNo0RI0YwaNAgJk2aRNPsHjU1NZxzzjkMHz6c/v37M3/+fI4++mj69evH9773vebz3X777YwcOZKhQ4dy2mmnsXbtWqZMmcL777/P0KFDGTduHC+99BKf/vSnmTBhAoMGDWLJkiX06dOH5cuXA3DrrbcyePBghgwZwvjx44vyb2FmtkWamrnK4TVs2LDIt3DhwvXSNoRknsANvjZX586dIyJi9erVMWbMmLj22msjImLFihUREbFy5coYOHBgLF++vLkcd999d/PxTfkiIk466aSYOXNmRERUV1fHBRdcEBERV155Zey+++7xyiuvxAcffBA9e/aM5cuXx8KFC+Pwww+PVatWRUTEP//zP8ctt9zykXJFRLz44oshKebOnduc1rt372hsbIynn346+vXrF42NjeuVZ2tpzedoZpWFZF7X9b7zy34cVBY03alAcgd16qmnAnD11Vdz//33A7BkyRL++te/suuuu1JVVcUxxxzTfPzs2bO5/PLLWblyJW+88QYDBw7kiCOOAGDMmDEA7L333gwcOJDdd98dgL322oslS5bwxz/+kUcffZQRI0Y0l2W33XZrsZy9e/fms5/97HrpDz/8MMceeyxdu3YFYJdddtnifxMzsy3lAFUETc+gctXV1fHQQw8xd+5ctttuO2pqaprH+nTq1ImqqiogGcd1xhlnUF9fzx577EFtbe1HxgRts802ALRr1675fdP2mjVriAhOPvlkLrvssk2Ws3Pnzlt8rWZmW0ubega1Nb399tvsvPPObLfddvzlL39h3rx5LeZrCkZdu3bl3Xff5d57723V3xk1ahT33nsvr7/+OgBvvPEGf/vb3wDo0KEDq1ev3uQ5DjzwQH75y1+yYsWK5nOYmZVaQQFK0mhJz0laLGlKC/v3lDRb0uOSnpR0WJp+kKRHJT2V/vfAnGPq0nMuSF8tt0uVqdGjR7NmzRr69+/PlClTWmxaA+jSpQvf/OY3GTRoEIccckhzU12hBgwYwA9+8AMOPvhgBg8ezEEHHcSyZcsAmDRpEoMHD2bcuHEbPcfAgQO56KKLqK6uZsiQIZx77rmtKoOZ2cdhk+tBpSviLgIOAhpIVtg9MSIW5uS5AXg8Iq6TNACYFRF9JO0DvBYRr0gaBDwQET3TY+qA8yOi4AWeWloP6tlnn6V///4FHd+jR48N9tbr3r07r776aqFFsSJrzedoZpVlS9aDGgksjogXImIVcBdwZF6eAHZM3+8EvAIQEY9HxCtp+jPAtpK2oUROP/30FtMvueQSByczs4wpJED1BJbkbDekablqgZMkNQCzgLNaOM8xwGMR8WFO2s1p8973tYEJ2SRNklQvqb6xsbGA4m5Y00wS+S9PdWRmlj3F6iRxIjAjInoBhwG3SWo+t6SBwL8Cp+UcMy4i9gYOSF8tjg6NiBsiYnhEDO/WrVuRimtmZllXSIBaCuyRs90rTct1KnAPQETMBToBXQEk9QLuByZExPNNB0TE0vS/fwd+QdKUaGZmBhQWoOYD/ST1ldQROAGYmZfnZWAUgKT+JAGqUVIX4DfAlIj4U1NmSe0lNQWwDsDhwNNbejFmZlY5NjlQNyLWSJoMPABUATdFxDOSppFMTzETOA+4UdI5JB0mJkZEpMd9CrhY0sXpKQ8G3gMeSINTFfAQcGOxLy5fXV0dc+bMWS+9urqampqaj/vPm5lZK2yym3mWbGk38yYzZswAYOLEiUUplyTOPfdcfvzjHwNwxRVX8O677xal80VtbS033ngj3bp1Y82aNVx66aXN0x+VUl1dHR07dmT//fdvcf/MmTNZuHAhU6asN2yuRe5mbtZ2baibeZua6uiKK67gvffea96eOnVq8/vOnTtz/vnnb9Z5t9lmG+677z4uvPDC5vnsiumcc87h/PPP59lnn+WAAw7g9ddfp127TbfOrlmzhvbtP56PuK6uju23377FALVmzRrGjBmTiUBqZuWrTU11lBucWrNvU9q3b8+kSZOYPn36evteeuklDjzwQAYPHsyoUaN4+eWXgeTu7eyzz2b//fdnr732KmiKo/79+9O+fXuWL1/Of//3f7Pffvuxzz778OUvf7l5AHJtbS3jx4/n85//POPHj+ell17igAMOYN9992XfffflkUceAZIAU11dzZFHHslee+3FlClTuOOOOxg5ciR77703zz+f9GdpbGzkmGOOYcSIEYwYMYI//elPvPTSS1x//fVMnz6doUOH8oc//IGJEydy+umns99++3HBBRcwY8YMJk+eDCRLmRx11FEMGTKEIUOGNJfBzGxj2lSA+jideeaZ3HHHHbz99tsfST/rrLM4+eSTefLJJxk3bhxnn312875ly5bxxz/+kV//+tcFNYX93//9H+3ataNbt2584QtfYN68eTz++OOccMIJXH755c35Fi5cyEMPPcSdd97JbrvtxoMPPshjjz3G3Xff/ZG//8QTT3D99dfz7LPPctttt7Fo0SL+/Oc/841vfIOf/vSnAHzrW9/inHPOYf78+fznf/4n3/jGN+jTpw+nn34655xzDgsWLOCAAw4AoKGhgUceeYSf/OQnHyn32WefTXV1NU888QSPPfYYAwcObP0/sJm1OW2qie/jtOOOOzJhwgSuvvpqtt122+b0uXPnct999wEwfvx4LrjgguZ9Y8eOpV27dgwYMGCjCyZOnz6d22+/nR122IG7774bSTQ0NHD88cezbNkyVq1aRd++fZvzjxkzprkMq1evZvLkySxYsICqqioWLVrUnG/EiBHNy3d88pOf5OCDDwaSpT1mz54NwEMPPcTChc2zWvHOO+/w7rvvtljOY489tnmW9lwPP/wwt956KwBVVVXstNNOG7xWM7MmDlBF9O1vf5t9992XU045paD8uctnNHVWueiii/jNb34D0LyER9MzqFxnnXUW5557LmPGjKGuru4jHTJyl9WYPn063bt354knnmDdunV06tSpxb+fu5xH01IeAOvWrWPevHkfOW5DvJyHmRWTm/iKaJddduG4447jP/7jP5rT9t9/f+666y4A7rjjjubmsA354Q9/yIIFC9ZbXyrf22+/Tc+eyYxTt9xyy0bz7b777rRr147bbruNtWvXFno5ABx88MHNzX3wj6C5ww478Pe//72gc4waNYrrrrsOgLVr167XDGpm1hIHqCI777zzWL58efP2T3/6U26++WYGDx7MbbfdxlVXXVWUv1NbW8uxxx7LsGHDNtpz8IwzzuCWW25hyJAh/OUvf2n1Xc7VV19NfX09gwcPZsCAAVx//fUAHHHEEdx///3NnSQ25qqrrmL27NnsvffeDBs27CNNhmZmG9KmxkHldzPPtSXdzG3LeRyUWdu1JcttVIzhw9e7fiCZScLBycwsW9pUJ4mamhpPaWRmViYq4g6qnJopbX3+/MysJWUfoDp16sSKFSv8JVemIoIVK1YU1I3dzNqWsm/i69WrFw0NDWzpartWOp06daJXr16lLoaZZUzZB6gOHTp8ZBYFMzOrDAU18UkaLek5SYslrTdpnKQ9Jc2W9LikJyUdlrPvwvS45yQdUug5zcysbdtkgJJUBVwDHAoMAE6UNCAv2/eAeyJiH5IVd69Njx2Qbg8ERgPXSqoq8JxmZtaGFXIHNRJYHBEvRMQq4C7gyLw8AeyYvt8JeCV9fyRwV0R8GBEvAovT8xVyTjMza8MKeQbVE1iSs90A7JeXpxb4X0lnAZ2BL+ccOy/v2J7p+02dEwBJk4BJ6ea7kp4roMwb0xVYvslcZlvOdc22lnKva71bSixWJ4kTgRkR8WNJnwNukzSoGCeOiBuAG4pxLgBJ9S1NqWFWbK5rtrVUal0rJEAtBfbI2e6VpuU6leQZExExV1Inkoi+sWM3dU4zM2vDCnkGNR/oJ6mvpI4knR5m5uV5GRgFIKk/0AloTPOdIGkbSX2BfsCfCzynmZm1YZu8g4qINZImAw8AVcBNEfGMpGlAfUTMBM4DbpR0DkmHiYmRTO3wjKR7gIXAGuDMiFgL0NI5P4bra0nRmgvNNsF1zbaWiqxrZbXchpmZtR1lPxefmZlVJgcoMzPLJAcoMzPLJAeoIpGkUpfB2gbXNdtaSl3Xyn428yyQpLTXIpL2BV4FVkVEOY/stgzKq2v7A28Db0bEKxs/0qx1slDXfAdVBDkf4pnA1cBpwF2Sdilpwazi5NW1K4CxwJ8k7VbSglnFyUJdc4AqEklfBI4BDiGZMPcd4E1J/je2opI0kmRy5YNIxh0uApaXujnGKk+p65q/PDdTCx/Qe8D9wESgP3Bi+gvkoHTqJ7PN0kJdWwH8FjgTqAaOjIh1wFcl7bC1y2eVI2t1zQN1N0Ne2+xBJMuLvAnMBd6NiIHpvq8DhwFfj4h3SlVeK195dW0s8AbJzCyPAO0jYq9033hgHDAuIlaUqrxWvrJY1xygtoCkc4GvAqdExHOSxgAXAL8C1pHM8n5KRDxVwmJaBUinETsW+GY61dgQ4L+AX6RZDsF1zYogS3XNAWozpR/az4DDI+LtNE3AviTNfMtJVhl+tmSFtIog6ZMkc60dExFvSeoQEasl9QOOBlYBsyJiS9dKszYua3XN3cwLlHv7m1pJ0hGio6SqiFgbESFpUUScVaJiWgVooa59CGwL7A68FRGr0/R3IuJft3oBrWJkva65k0QB8tpmO0nahmSs0zvAAJLlRZA0Dvh/krYtWWGtrOXVtZ0lbR8RDcD/AcMk9Ur3jQMudacI21zlUNfcxLcJeR/iecDngb8DFwH7kCzWuAx4i6Sb+diIWFii4loZy6tr3yHp2qRCEmAAAA08SURBVLst8A1ge5LxdT1Juvp+Bdc120zlUtccoAok6QvANOASku6Wx5OsIrwdsD/QA7gvIv5askJaRZBUDXwP+CZwFHAhSR1bSrIwaFfg9xHxQskKaRUh63XNz6AKIOlQ4Fzg/oj4A/AHSatIxgd8LSJuKWkBrWKkP4TOA56IiJeA6ZIC+ANwRET8upTls8pRDnXNz6Ba0MJgtcdImvD2l9QNICIuB34J3Jwuae9/S2u1Fura0yRjT/pIGpk2xVwJXAn80nXNNlc51jU38eXJa5s9jqRXy5skDw7vBP4CXBkRr6d5domIN0pVXitfeXVtAiBgdUT8QtIVJGPp7o2IP6d5do6IN0tXYitX5VrX/EssT86H+C1gMsm8ejNIOkScAXwa+H7OnZSDk22WnLo2meTh9BLgRklfAS4l+f/z65KGpYe8VZKCWtkr17rmAJWS1LWpe7ikHsABEfFFoDfwFPBYRLwKnA50IfkFYtZqkvZQOtO9pJ2Bz5H0ohoKPAw8mP7wmQo0knyZNH/JmBWq3Ouam/gASX2Ai0mm8/gdSeeRn5N8WHuRTPz6vqRTSDpGvJ5OmGjWKumPn4uAF4DbSOY7uxV4l6TH1ISIWJn+0q2LiKdLVlgra5VQ13wHBaQ9WJ4Cvgx8MZ26aDEwgWQ+qvfTdtuzgHYOTra50rvwh0nGmByX1qXHgUnA5PQL42skzTCeYNg2WyXUtTbdzbypV0t6O/sayVxToyStJZmPCuBhSf9DMiZgQnjlUttMOQ+qu5DMQHKYpDXANSTPOn8n6Y8k8zmeFBEvl660Vs4qpa616Sa+pg8xvTs6E/g68C2giqRHy28lHUZyS7wkIl4sYXGtAkj6KsnAyKOBk4BuwDMRcX36gHot8EZWvzCsfFRCXWuTd1CSDgHej4jfp0mDSWaBeAaYJOli4AeSqoDfRcT7pSqrlTdJXyJZS+fBNGkv4IF0ZP40SROBM9P5HW9zr1DbXJVY19rcM6imCRCBBkk90/ePAp9UMtU8ETEN6AB8ieRuyqzVJH2K5FfrIkl7psnzgP5KltImImYAbwN7kiypbdZqlVrX2tQdVDpl0c/4xwzkbypZZPDXJB0kxkp6FNgBaACuioh3S1VeK1+SvgxcS3J33gFYIulM4D6SB9XHKlljZ2W6f3oWBkZa+ankutamnkFJugDYkeRDewIYBNwEHAm8CJxCMk5gO5JeLpnrdmnlQdIk4LPAHJLpZLoCV5HM6fgYSaeb40meA3w/Ip4sUVGtzFVyXWtrAeoLwGXAHsBhEbFQ0lHALSTTyT+cDtbdJiIyMZLaypOkPUimxvoUMDoiFqQdbn4GnBcR90vqAHSIiJWlLKuVt0quaxXfxJc7BxXJnHrtSH5p9Je0NP3wAnhI0hER8RvAnSKs1fLqGiTrhv0PcIikhoiYJekM4G4lqzDfC6xu8WRmG9FW6lpF30HlTZC4K8n8UtuRTPUxCqgH7omI99I5qZ6PiL+UrMBWtvLqWi/g9XTXp0kGRr4J/Dgi3k6fGbwUEYtLU1orZ22prlVsgMr7EM8CJgL/C8wC/giMA0aQzE5+S7nd+lp2SGqeXSSdZPg0khH8j0bEzZL2B75K0nNqakRkctS+ZV9bq2sV28SXE5wOA/YjeWBYQ/Lh7RgRt0vqBPwTSc8Ws82S84XxJZLnABOAfsCBkjpHxM/SZwCHAtuUrqRW7tpaXavYOygASYOAOuBfIuIqSZ8gGVXdj2RyxPsl7VjuvzKstJQs6tafZD7HyyNiSjqD9OeAscBzEXGFpE4R8UEpy2rlra3VtYoaqNs0t16TtJv4vwPfkfTpdB69e0jGOH0u/cXh4GStllvXImJdOgvJZOBsSYPSUfq/J2lS7qNkAbiy/8Kwra8t17WKuYPKe+b0ZeATwHySJTMmkoxxOikinlWy2OCachmsZtmSV9eOJLkjXxARD0k6mWTJ7C+l3X13ANZFxHslLLKVqbZe1yrmGVTOh/htkoD0OHAM8CfgDpJr/bWkQyNiUanKaeUvp659CziZZA2xCemYugtIpsd6TNJgD/a2LdHW61rZByhJvYEPI+JVJStG1gAHRUSjpFHAV4DBEXFl2n5bdmMBLBvS6WIUEYvSB9H7AmMiokHSAOBYYGJEXJN2wHFds83iupYo62dQkg4GfkGyhlNvkmUxOpNMXURE/I5kGeMT0+2fhJfMsM2gZAb8XwBflPRPJNPGfIKk6ZiIWEiyyOUB6fa1EfFciYprZcx17R/KNkBJGk3S/npRRNwREX+LiNXADJIHhaPTrC8B70vqWJqSWrmTdBAwnWTamJ9HxKK0u+80oKekU9KsHwLtJHUuVVmtvLmufVTZdZJIe7R0IJnk9b8i4t68B4l7kNz+HkoytfxAkuWOnypVma08pXWtHcnEm/PSsXO5dW1HkllJziMZzf9pXNdsM7iutazsnkGlH9gqSR8Cy9LkKmBNun+JpDnA3SSDcBdFxNKSFNbKWlrX1kr6gOQXK6R1Lf3yeEfSIpIvjt7Aioh4rUTFtTLmutaysm3iI7mLamqTXZPXhDcUeDsiZjs4WRG0I1kyu7muxT+aHg4EukTEwrbwhWEfO9e1HGUXoHIGrdUCXSSdAxARq9L9pwDjAT9zsmKZAnSS9GP4SF07GfgasKqEZbPK4rqWo2yeQeVOkphubwOMJrmLehu4i6RJ73Tgq+loa7NWy5uQs11ErJM0GPgJSfPLr4AuJHXPdc02m+vaxmU+QEnae0MPAtPxAT2A75Gs4bQdyTLtbepDtOKQtE9EPL6R/VXAJSTdfrclmQX/2a1VPqscrmuFKYcAdTnJ3dHjsYnCKlmYa+3WKZlVGkk/AWYDv86va5LaR8SanO38BePMCua6VphyeAb1DvCZnO6WO6e/Lki3q3Lyrss/2KwVHgV65NS1HpK6QPMD63L4/8XKg+taATJ5ByXpcyS3tn8jabY7OSJqJV0C9AKeJZlW/jclLKZVAEnVJHXsNeBF4EcRcVpa1/Ykmfl+WURcX8JiWgVwXWu9zEVpSSOB/ySZQ+9bJGufDJTUI81yPrCQZCyA2WaTtC/J8iufIulc8wVgnaTuwN9JelTNALqWqoxWGSTtg+taq2UmQOV0H9+NZCGuS4C3SO6Wfg/sDQwC9ieZcn5s2pPPbHOJpK79FPgDyS/bJ4DDSXqEHkiyCNxoSduVrJRWtnK+16qAf3Nda53MBCiSDxBgAXByOp7pVGAIyUy9fUkW6foKycy+34mID1s6kdnGSNornfn+deArkk4DLgY+SdL00hm4kOSHUF/gzIhYWaryWllr+l5bQhJ8zsB1rWCZeAYl6VCSYPQ4MBd4DPg8sE1E3CdpJ+D/RcR307umdenEsGatIukwkmEJX4uIlyTtRXLXvlNEPJDWr6uBs0mmz1rXVntQ2ZZJV1uYQPJ99iTwCMncoF0j4reua5tW8rn40mdOV5PMDBHp+19GxNR0f3uSWSE+KemfwosN2mZKlzH4MfD1NDi1j4gXgBfS/R1Imv2qgH5RgQvA2daRdoi4iuR7bTnwU+DWiPhRut91rQAlD1AkwacuIu4AkDQbeCSdg+qidDxAo6RfAh+UsqBWvtImvYnA3IiYm3bpPVvSO0BjumTLamC1pP8GKmbZbCuJTwA/j4i7ASTdAVyczhZxqetaYbLwDOpDoIekXQAiYhnwOeBQScc1ZYqIuyPi5RKV0cpcRLwJ3A68KukykuaW7UkmHf6WpEk5eX8VXtjStkxn4KS0BQiSMZrfJfle+2JTJte1jSvJHZSS5Yx3ioj6iJgv6TngQWAYQCTLt18DdC9F+axypHVt54j4c0T8RtLfgW8D10bEz9I8S4HPlLKcVv7SutYlIuZHxM/Txxfz0mUyPhERNZJ2A3YqbUnLx1YPUJLGAlOBxZKWAY9FxPmS/l1SPXB4RLxKMh5gQDqiOvzw0Forr669AjxFstDlImBFTtY9ge5teUoZ2zJ5de1VoD4iJkkaCHQi6VJO+t5jOAu0VQOUpF2B04ATI2KhpK+TPAfYJR1RfTlwnZLFCIeQzN7r6Yus1TZQ1yYD3UgmFF6d5jsJOJ6kV5+Dk7XaRr7Xdgd+FhHvpPlOBY4DRpWutOVlaz+DWkPS7t8DICJuAv4K9JV0VERcAHwf+DfgUM9Kblugpbr2PEmX8iMAJO1HsnbYSW1xpmgrmpbq2mKSRxSHQ3Pz30DgKxGxuETlLDtbfRyUpKZpPh4gaffvTTKyev+IOHmrFsYq2kbq2siIODXNs3PagcJssxVY17aNiPdLV8ryU4pOEneSzD01CngrIk4CkHSEpB2bbofNimBjdW3niHjTwcmKZGN1rUtEvOXg1Holm0lCH11JcgLJBIoHRYTHBFhRua7Z1uK6VlwlG6ib8yF+nWSG8uP9IdrHwXXNthbXteIq+Vx8knoDHfzg0D5urmu2tbiuFUfJA5SZmVlLsjDVkZmZ2XocoMzMLJMcoMzMLJMcoMzMLJMcoMzMLJMcoMzMLJMcoMzMLJP+P2qrqzJxNNJbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cie.plot_util(intervals_A=[param_90, param_95, param_99],\n",
    "              intervals_B=[nonparam_90, nonparam_95, nonparam_99],\n",
    "              label_A=\"Parametric\", label_B=\"Non-Parametric\", ylim=(0.8, 1.01))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Networks and One-Hot-Encoding\n",
    "The following example shows how to use the functions with simple neural networks and one hot encoding of labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "iris_y_onehot = OneHotEncoder().fit_transform(iris_y.reshape(-1, 1)).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 50)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train_onehot, y_test_onehot = train_test_split(iris_X, iris_y_onehot, test_size=0.33, random_state=42)\n",
    "len(X_train), len(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 1., 0.],\n",
       "       [0., 0., 1.],\n",
       "       [0., 1., 0.],\n",
       "       [1., 0., 0.],\n",
       "       [0., 0., 1.]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_onehot[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The following function will return instances of the model\n",
    "\n",
    "def get_model(hidden_units,  n_output_units, hidden_activation=\"relu\",\n",
    "              output_activation=\"softmax\", loss=\"binary_crossentropy\",\n",
    "              optimizer=\"adam\", metric=\"accuracy\"):\n",
    "    model = Sequential()\n",
    "    \n",
    "    for u in hidden_units:\n",
    "        model.add(Dense(u, activation=hidden_activation))\n",
    "    model.add(Dense(n_output_units, activation=output_activation))\n",
    "    \n",
    "    model.compile(loss=loss, optimizer=optimizer, metrics=[metric])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parametric Estimation\n",
    "The parametric estimation is shown to give an example of how the model works and to compare it to the nonparametric estimate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = get_model(hidden_units=[30, 20, 10],  n_output_units=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "model.fit(X_train, y_train_onehot, epochs = 50, verbose=0, batch_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4.6857153e-03, 9.6951354e-01, 2.5800724e-02],\n",
       "       [4.4403583e-07, 8.2842354e-03, 9.9171537e-01],\n",
       "       [1.8619797e-03, 8.0265957e-01, 1.9547836e-01]], dtype=float32)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(X_train[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100/100 [==============================] - 0s 295us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.06419484227895737, 0.986666738986969]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_train, y_train_onehot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50/50 [==============================] - 0s 54us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.06085592210292816, 0.986666738986969)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(loss, acc) = model.evaluate(X_test, y_test_onehot)\n",
    "loss, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing C.I. assuming the data follow a standard normal distribution\n",
      "With 90.0% confidence the accuracy of the model is in 0.983 +/-0.017.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9829395655094056, 0.017060434490594356)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_90 = cie.parametric_confidence_interval(confidence_level=.9, mean=acc, sample_size=50, verbose=True)\n",
    "param_90"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing C.I. assuming the data follow a standard normal distribution\n",
      "With 95.0% confidence the accuracy of the model is in 0.98 +/-0.02.\n"
     ]
    }
   ],
   "source": [
    "param_95 = cie.parametric_confidence_interval(confidence_level=.95, mean=acc, sample_size=50, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing C.I. assuming the data follow a standard normal distribution\n",
      "With 99.0% confidence the accuracy of the model is in 0.974 +/-0.026.\n"
     ]
    }
   ],
   "source": [
    "param_99 = cie.parametric_confidence_interval(confidence_level=.99, mean=acc, sample_size=50, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Non-Parametric Estimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "dnn_params_dict = {'hidden_units':[30, 20, 10], 'hidden_activation':\"relu\", 'n_output_units':3,\n",
    "                   'output_activation':\"softmax\", 'loss':'binary_crossentropy', 'optimizer':'adam'}\n",
    "\n",
    "# These parameters will be passed to the network on the calls of the fit method.\n",
    "dnn_fit_params_dict={'epochs':50, 'batch_size':10, 'verbose':0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating model on 20 samples of (X, y).\n",
      "Each sample will contain 150 elements from (X, y), out of which 100 elements\n",
      "will be used to train the model, while the remaining 50 elements to test its accuracy.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87f934c8d12e4587b023cad5afc3844b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=20.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "accs=cie.get_accuracy_samples(get_model, iris_X, iris_y_onehot, model_params_dict=dnn_params_dict,\n",
    "                            fit_params_dict=dnn_fit_params_dict, n_iterations=20, sample_ratio=1,\n",
    "                            train_ratio=0.67, random_seed=42, sample_with_replacement=True,\n",
    "                             verbose=True, is_one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the given data, with 90.0% confidence the accuracy of the model is in 0.95 +/-0.05.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.95, 0.04999999999999993)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nonparam_90=cie.nonparametric_confidence_interval(accs, .9)\n",
    "nonparam_90"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the given data, with 95.0% confidence the accuracy of the model is in 0.95 +/-0.05.\n"
     ]
    }
   ],
   "source": [
    "nonparam_95=cie.nonparametric_confidence_interval(accs, .95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the given data, with 99.0% confidence the accuracy of the model is in 0.95 +/-0.05.\n"
     ]
    }
   ],
   "source": [
    "nonparam_99=cie.nonparametric_confidence_interval(accs, .99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparison Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZxU1Z338c+XBkRxQQXBgAImTMIioDSYmJgmEhWN4ha3oIgxQUfRxCUGH5PYMlEzPiaoictoRnGLS4xOmMQ8jkYgizChUdwwEmKMNKI2uEVR2X7PH/d2pywauropu25Vf9+vV72sOnfpU9TP+tU599xzFBGYmZllTadSV8DMzKw5TlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJnUtdgdbo2bNnDBgwoNTVMDOzIlq4cOHKiOiVX15WCWrAgAHU1dWVuhpmZlZEkv7eXLm7+MzMLJOcoMzMLJOcoMzMLJOcoMzMLJMKSlCSbpb0mqRnNrFdkq6RtFTSU5L2ztl2sqS/pI+Tc8pHSXo6PeYaSdryt2NmZpWi0BbUTGD8ZrYfDAxKH1OA6wEk7QRcDOwDjAEulrRjesz1wNdzjtvc+c3MrIMpKEFFxO+A1zezy+HAbZGYD/SQtCtwEPBwRLweEW8ADwPj023bR8T8SKZTvw04YoveiZmZVZRiXYPqCyzLeV2flm2uvL6ZcjMzM6AMbtSVNIWk25Ddd9+9Teeora3lkksu2aj84osvpra2dkuq16I5c+Ywd+7cjcpramoYO3bsR/q3rf051qy9dIRYU6ELFkoaAPwqIoY1s+0/gDkRcVf6+nlgbOMjIk7L3S99zI6IT6XlJ+TutynV1dWxJTNJNP7DzZkzp83naKuZM2cCMHny5Hb/29b+HGvWXioh1iQtjIjq/PJidfHNAialo/k+DbwVESuAh4ADJe2YDo44EHgo3fa2pE+no/cmAb8sUl3MzKwCFNTFJ+kuktZQT0n1JCPzugBExA3Ag8AhwFJgNXBKuu11Sf8GLEhPNT0iGgdbnEEyOnBr4Dfpw8zMDCgwQUXECS1sD+DMTWy7Gbi5mfI6YKPuQjMzM/BMEmZmllFOUGZmlklOUGZmlklOUGZmlklOUGZmlklOUGZmlklOUGZmlkmZn4uvGPr06cOrr77a9Dp36anevXvzyiuvlKJaZma2GR2iBZWbnFqzzczMSqdDJCgzMys/HaKLz6y9uDvZrHjcgjIrIncnmxWPE5SZmWWSE5SZmWVSh0hQvXv3btM2MzMrnUIXLBwPXA1UAT+NiB/kbe9PsuZTL+B14MSIqJf0BWBGzq6fAo6PiP+SNBOoAd5Kt02OiEVb8mY2pfHCdCmXRjYzK6aOMCCnxQQlqQq4FjgAqAcWSJoVEYtzdrsSuC0ibpW0P3A5cFJEzAZGpufZiWTF3f/JOe5bEXFfcd6KmVnH0REG5BTSxTcGWBoRL0TEGuBu4PC8fYYAj6bPZzezHeDLwG8iYnVbK2uWde5ONiueQhJUX2BZzuv6tCzXk8BR6fMjge0k7Zy3z/HAXXlll0p6StIMSVs198clTZFUJ6muoaGhgOqalc4rr7xCRFBTU0NNTQ0R0fSohC4Xs/ZUrEES5wM1kp4gua60HFjfuFHSrsCewEM5x1xIck1qNLAT8O3mThwRN0ZEdURU9+rVq0jVNTOzrCtkkMRyYLec1/3SsiYR8TJpC0rStsDREfFmzi7HAg9ExNqcY1akTz+QdAtJkjMzMwMKa0EtAAZJGiipK0lX3azcHST1lNR4rgtJRvTlOoG87r20VYWSoSdHAM+0vvpmZlapWkxQEbEOmErSPfcccG9EPCtpuqQJ6W5jgeclLQF6A5c2Hi9pAEkLbG7eqe+U9DTwNNAT+P4WvRMzsw6kIwzIKeg+qIh4EHgwr+x7Oc/vA5odLh4RL7LxoAoiYv/WVNTMzP6pI9zf2SFmkjAzs/LjBGVmZpnUIRJUbW0tkpg7dy5z585FEpKora0tddXMzGwTOsSChbW1tU5GZmZlpkO0oMzMrPw4QZmZWSY5QZmZWSY5QZmZWSY5QZkVkUeMmhVPhxjFZ9ZePGLUrHjcgjIzs0xygjIzs0xygjIzs0xygjIzs0xSRLS8kzQeuBqoAn4aET/I296fZJHCXsDrwIkRUZ9uW0+y5hPASxExIS0fCNwN7AwsBE6KiDWbq0d1dXXU1dUV/u4y4Morr+Tdd99tdlv37t05/3wvJGzF4VjrmEqx3EaxY03Swoiozi9vsQUlqQq4FjgYGAKcIGlIfn2B2yJiODAduDxn23sRMTJ9TMgp/3dgRkR8AngDOLVV76hMbOpDbGmbWWs51qy9tFesFdLFNwZYGhEvpC2cu4HD8/YZAjyaPp/dzPYPSZd5359/LnJ4K8my72ZmZkBhCaovsCzndT0br5D7JHBU+vxIYDtJO6evu0mqkzRfUmMS2hl4M11OflPnBEDSlPT4uoaGhgKqa2ZmlaBYgyTOB2okPQHUAMuB9em2/mnf4leAqyR9vDUnjogbI6I6Iqp79epVpOqamZW3jjBrSSEzSSwHdst53S8taxIRL5O2oCRtCxwdEW+m25an/31B0hxgL+AXQA9JndNW1EbnNDOzTesIs5YU0oJaAAySNFBSV+B4YFbuDpJ6Smo814UkI/qQtKOkrRr3AT4LLI5k6OBs4MvpMScDv9zSN2NmZpWjxQSVtnCmAg8BzwH3RsSzkqZLahyVNxZ4XtISoDdwaVo+GKiT9CRJQvpBRCxOt30bOFfSUpJrUv9ZpPeUKd27d2/TNrPWcqxZe2mvWCvoPqisKMf7oBrNnDkTgMmTJ5e0Hlb5HGvWXooVa22+D8rMzKwUnKDMzCyTnKDMzCyTnKDMzCyTnKDMzCyTnKDMzCyTnKDMzCyTnKDMzCyTnKDMzCyTnKDMzCyTnKDMzCyTnKDMzCyTnKDMzCyTnKDMzCyTCkpQksZLel7SUknTmtneX9JvJT0laY6kfmn5SEnzJD2bbjsu55iZkv4maVH6GFm8t2VmZuWuxQQlqQq4FjgYGAKcIGlI3m5XArdFxHBgOnB5Wr4amBQRQ4HxwFWSeuQc962IGJk+Fm3hezEzswpSSAtqDLA0Il6IiDXA3cDhefsMAR5Nn89u3B4RSyLiL+nzl4HXgF7FqLiZmVW2QhJUX2BZzuv6tCzXk8BR6fMjge0k7Zy7g6QxQFfgrznFl6ZdfzMkbdXcH5c0RVKdpLqGhoYCqmtmZpWgWIMkzgdqJD0B1ADLgfWNGyXtCtwOnBIRG9LiC4FPAaOBnYBvN3fiiLgxIqojorpXLze+zMw6is4F7LMc2C3ndb+0rEnafXcUgKRtgaMj4s309fbAr4GLImJ+zjEr0qcfSLqFJMmZmZkBhbWgFgCDJA2U1BU4HpiVu4OknpIaz3UhcHNa3hV4gGQAxX15x+ya/lfAEcAzW/JGzMyssrSYoCJiHTAVeAh4Drg3Ip6VNF3ShHS3scDzkpYAvYFL0/Jjgc8Dk5sZTn6npKeBp4GewPeL9abMzKz8FdLFR0Q8CDyYV/a9nOf3Afc1c9wdwB2bOOf+raqpmZl1KJ5JwszMMskJyszMMskJyszMMskJyszMMskJyszMMskJyszMMskJyszMMskJyszMMskJyszMMskJyszMMskJyszMMskJyszMMskJyszMMskJyszMMqmgBCVpvKTnJS2VNK2Z7f0l/VbSU5LmSOqXs+1kSX9JHyfnlI+S9HR6zmvShQvNzMyAAhKUpCrgWuBgYAhwgqQhebtdSbJq7nBgOnB5euxOwMXAPsAY4GJJO6bHXA98HRiUPsZv8bsxM7OKUUgLagywNCJeiIg1wN3A4Xn7DAEeTZ/Pztl+EPBwRLweEW8ADwPj0+Xet4+I+RERwG0ky76bmZkBha2o2xdYlvO6nqRFlOtJ4CjgauBIYDtJO2/i2L7po76Z8o1ImgJMAdh9990LqG62zJkzh7lz5za9vuSSSwCoqalh7NixJaqVVSLHmrWX9oq1gpZ8L8D5wE8kTQZ+BywH1hfjxBFxI3AjQHV1dRTjnO1p7Nix/nKwduFYs/bSXrFWSIJaDuyW87pfWtYkIl4maUEhaVvg6Ih4U9JyYGzesXPS4/vllX/onGZm1rEVcg1qATBI0kBJXYHjgVm5O0jqKanxXBcCN6fPHwIOlLRjOjjiQOChiFgBvC3p0+novUnAL4vwfszMrEK0mKAiYh0wlSTZPAfcGxHPSpouaUK621jgeUlLgN7ApemxrwP/RpLkFgDT0zKAM4CfAkuBvwK/KdabMjOz8qdkEF15qK6ujrq6ulJXw8zMikjSwoiozi/3TBJmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJBSUoSeMlPS9pqaRpzWzfXdJsSU9IekrSIWn5REmLch4bJI1Mt81Jz9m4bZfivjUzMytnLS75LqkKuBY4AKgHFkiaFRGLc3b7DslChtdLGgI8CAyIiDuBO9Pz7An8V0QsyjluYkR4gSczM9tIIS2oMcDSiHghItYAdwOH5+0TwPbp8x2Al5s5zwnpsWZmZi0qJEH1BZblvK5Py3LVAidKqidpPZ3VzHmOA+7KK7sl7d77riQVVmUzM+sIijVI4gRgZkT0Aw4BbpfUdG5J+wCrI+KZnGMmRsSewH7p46TmTixpiqQ6SXUNDQ1Fqq6ZmWVdIQlqObBbzut+aVmuU4F7ASJiHtAN6Jmz/XjyWk8RsTz97z+An5F0JW4kIm6MiOqIqO7Vq1cB1TUzs0pQSIJaAAySNFBSV5JkMytvn5eAcQCSBpMkqIb0dSfgWHKuP0nqLKln+rwLcCjwDGZmZqkWR/FFxDpJU4GHgCrg5oh4VtJ0oC4iZgHnATdJOodkwMTkiIj0FJ8HlkXECzmn3Qp4KE1OVcAjwE1Fe1dmZlb29M88kn3V1dVRV+dR6WZmlUTSwoiozi/3TBJmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJTlBmZpZJBSUoSeMlPS9pqaRpzWzfXdJsSU9IekrSIWn5AEnvSVqUPm7IOWaUpKfTc14jScV7W2ZmVu5aTFCSqoBrgYOBIcAJkobk7fYd4N6I2ItkSfjrcrb9NSJGpo/Tc8qvB74ODEof49v+NszMrNIU0oIaAyyNiBciYg1wN3B43j4BbJ8+3wF4eXMnlLQrsH1EzE+Xhr8NOKJVNTczs4pWSILqCyzLeV2fluWqBU6UVA88CJyVs21g2vU3V9J+Oeesb+GcAEiaIqlOUl1DQ0MB1TUzs0pQrEESJwAzI6IfcAhwu6ROwApg97Tr71zgZ5K238x5NhIRN0ZEdURU9+rVq0jVNTOzrOtcwD7Lgd1yXvdLy3KdSnoNKSLmSeoG9IyI14AP0vKFkv4K/Et6fL8WzmlmZh1YIS2oBcAgSQMldSUZBDErb5+XgHEAkgYD3YAGSb3SQRZI2oNkMMQLEbECeFvSp9PRe5OAXxblHZmZWUVosQUVEeskTQUeAqqAmyPiWUnTgbqImAWcB9wk6RySAROTIyIkfR6YLmktsAE4PSJeT099BjAT2Br4TfowMzMDQMkguvJQXV0ddXV1pa6GmZkVkaSFEVGdX+6ZJMzMLJOcoMzMLJMKGcWXaWvXrqW+vp7333+/1FWxNurWrRv9+vWjS5cupa6KmWVI2Seo+vp6tttuOwYMGICn8ys/EcGqVauor69n4MCBpa6OmWVI2Xfxvf/+++y8885OTmVKEjvvvLNbwGa2kbJPUICTU5nz52dmzamIBFWo2tpaJG30qK2tLXXVzMwsT4dLUBFBTU0NNTU1RAQRscUJqqqqipEjRzJs2DCOOeYYVq9eXZwKb6HLLrtss9sPOeQQ3nzzzXaqjZlZ63SoBNWnTx8kMXfuXObOnfuhVlSfPn3afN6tt96aRYsW8cwzz9C1a1duuOGGlg9KrV+/vs1/tyWbSlARwYYNG3jwwQfp0aPHR/b3zcy2RIdKUK+++mqbtrXGfvvtx9KlSwE44ogjGDVqFEOHDuXGG29s2mfbbbflvPPOY8SIEcybN4/p06czevRohg0bxpQpU2ic3WPs2LGcc845VFdXM3jwYBYsWMBRRx3FoEGD+M53vtN0vjvuuIMxY8YwcuRITjvtNNavX8+0adN47733GDlyJBMnTuTFF1/kk5/8JJMmTWLYsGEsW7aMAQMGsHLlSgBuu+02hg8fzogRIzjppJOK8m9hZrZFGru5yuExatSoyLd48eKNyjaFZJ7ATT7aqnv37hERsXbt2pgwYUJcd911ERGxatWqiIhYvXp1DB06NFauXNlUj3vuuafp+Mb9IiJOPPHEmDVrVkRE1NTUxAUXXBAREVdddVXsuuuu8fLLL8f7778fffv2jZUrV8bixYvj0EMPjTVr1kRExL/+67/Grbfe+qF6RUT87W9/C0kxb968prL+/ftHQ0NDPPPMMzFo0KBoaGjYqD7tpTWfo5lVFpJ5XTf6zi/7+6CyoLGlAkkL6tRTTwXgmmuu4YEHHgBg2bJl/OUvf2HnnXemqqqKo48+uun42bNnc8UVV7B69Wpef/11hg4dymGHHQbAhAkTANhzzz0ZOnQou+66KwB77LEHy5Yt4w9/+AMLFy5k9OjRTXXZZZddmq1n//79+fSnP71R+aOPPsoxxxxDz549Adhpp522+N/EzGxLOUEVQeM1qFxz5szhkUceYd68eWyzzTaMHTu26V6fbt26UVVVBST3cZ1xxhnU1dWx2267UVtb+6F7grbaaisAOnXq1PS88fW6deuICE4++WQuv/zyFuvZvXv3LX6vZmbtpUNdg2pPb731FjvuuCPbbLMNf/7zn5k/f36z+zUmo549e/LOO+9w3333tervjBs3jvvuu4/XXnsNgNdff52///3vAHTp0oW1a9e2eI7999+fn//856xatarpHGZmpVZQgpI0XtLzkpZKmtbM9t0lzZb0hKSnJB2Slh8gaaGkp9P/7p9zzJz0nIvSR/P9UmVq/PjxrFu3jsGDBzNt2rRmu9YAevTowde//nWGDRvGQQcd1NRVV6ghQ4bw/e9/nwMPPJDhw4dzwAEHsGLFCgCmTJnC8OHDmThx4mbPMXToUC666CJqamoYMWIE5557bqvqYGb2UWhxPah0RdwlwAFAPckKuydExOKcfW4EnoiI6yUNAR6MiAGS9gJejYiXJQ0DHoqIvukxc4DzI6LgBZ6aWw/queeeY/DgwQUd36dPn02O1uvduzevvPJKoVWxImvN52hmlWVL1oMaAyyNiBciYg1wN3B43j4BbJ8+3wF4GSAinoiIl9PyZ4GtJW1FiZx++unNll988cVOTmZmGVNIguoLLMt5XZ+W5aoFTpRUDzwInNXMeY4GHo+ID3LKbkm7976rTUzIJmmKpDpJdQ0NDQVUd9MaZ5LIf3iqIzOz7CnWIIkTgJkR0Q84BLhdUtO5JQ0F/h04LeeYiRGxJ7Bf+mj27tCIuDEiqiOiulevXkWqrpmZZV0hCWo5sFvO635pWa5TgXsBImIe0A3oCSCpH/AAMCki/tp4QEQsT//7D+BnJF2JZmZmQGEJagEwSNJASV2B44FZefu8BIwDkDSYJEE1SOoB/BqYFhF/bNxZUmdJjQmsC3Ao8MyWvhkzM6scLd6oGxHrJE0FHgKqgJsj4llJ00mmp5gFnAfcJOkckgETkyMi0uM+AXxP0vfSUx4IvAs8lCanKuAR4KZiv7l8c+bMYe7cuRuV19TUMHbs2I/6z5uZWSu0OMw8S7Z0mHmjmTNnAjB58uSi1EsS5557Lj/84Q8BuPLKK3nnnXeKMviitraWm266iV69erFu3Touu+yypumPSmnOnDl07dqVfffdt9nts2bNYvHixUybttFtc83yMHOzjmtTw8w71FRHV155Je+++27T60suuaTpeffu3Tn//PPbdN6tttqK+++/nwsvvLBpPrtiOuecczj//PN57rnn2G+//Xjttdfo1Knl3tl169bRufNH8xHPmTOHbbfdttkEtW7dOiZMmJCJRGpm5atDTXWUm5xas60lnTt3ZsqUKcyYMWOjbS+++CL7778/w4cPZ9y4cbz00ktA0no7++yz2Xfffdljjz0KmuJo8ODBdO7cmZUrV/Lf//3f7LPPPuy111588YtfbLoBuba2lpNOOonPfvaznHTSSbz44ovst99+7L333uy999489thjQJJgampqOPzww9ljjz2YNm0ad955J2PGjGHPPffkr39NxrM0NDRw9NFHM3r0aEaPHs0f//hHXnzxRW644QZmzJjByJEj+f3vf8/kyZM5/fTT2WeffbjggguYOXMmU6dOBZKlTI488khGjBjBiBEjmupgZrY5HSpBfZTOPPNM7rzzTt56660PlZ911lmcfPLJPPXUU0ycOJGzzz67aduKFSv4wx/+wK9+9auCusL+93//l06dOtGrVy8+97nPMX/+fJ544gmOP/54rrjiiqb9Fi9ezCOPPMJdd93FLrvswsMPP8zjjz/OPffc86G//+STT3LDDTfw3HPPcfvtt7NkyRL+9Kc/8bWvfY0f//jHAHzjG9/gnHPOYcGCBfziF7/ga1/7GgMGDOD000/nnHPOYdGiRey3334A1NfX89hjj/GjH/3oQ/U+++yzqamp4cknn+Txxx9n6NChrf8HNrMOp0N18X2Utt9+eyZNmsQ111zD1ltv3VQ+b9487r//fgBOOukkLrjggqZtRxxxBJ06dWLIkCGbXTBxxowZ3HHHHWy33Xbcc889SKK+vp7jjjuOFStWsGbNGgYOHNi0/4QJE5rqsHbtWqZOncqiRYuoqqpiyZIlTfuNHj26afmOj3/84xx44IFAsrTH7NmzAXjkkUdYvLhpVivefvtt3nnnnWbrecwxxzTN0p7r0Ucf5bbbbgOgqqqKHXbYYZPv1cyskRNUEX3zm99k77335pRTTilo/9zlMxoHq1x00UX8+te/BmhawqPxGlSus846i3PPPZcJEyYwZ86cDw3IyF1WY8aMGfTu3Zsnn3ySDRs20K1bt2b/fu5yHo1LeQBs2LCB+fPnf+i4TfFyHmZWTO7iK6KddtqJY489lv/8z/9sKtt33325++67AbjzzjubusM25dJLL2XRokUbrS+V76233qJv32TGqVtvvXWz++2666506tSJ22+/nfXr1xf6dgA48MADm7r74J9Jc7vttuMf//hHQecYN24c119/PQDr16/fqBvUzKw5TlBFdt5557Fy5cqm1z/+8Y+55ZZbGD58OLfffjtXX311Uf5ObW0txxxzDKNGjdrsyMEzzjiDW2+9lREjRvDnP/+51a2ca665hrq6OoYPH86QIUO44YYbADjssMN44IEHmgZJbM7VV1/N7Nmz2XPPPRk1atSHugzNzDalQ90HlT/MPNeWDDO3Lef7oMw6ri1ZbqNiVFdv9P6BZCYJJyczs2zpUIMkxo4d6ymNzMzKREW0oMqpm9I25s/PzJpT9gmqW7durFq1yl9yZSoiWLVqVUHD2M2sYyn7Lr5+/fpRX1/Plq62a6XTrVs3+vXrV+pqmFnGlH2C6tKly4dmUTAzs8pQUBefpPGSnpe0VNJGk8ZJ2l3SbElPSHpK0iE52y5Mj3te0kGFntPMzDq2FhOUpCrgWuBgYAhwgqQhebt9B7g3IvYiWXH3uvTYIenrocB44DpJVQWe08zMOrBCWlBjgKUR8UJErAHuBg7P2yeA7dPnOwAvp88PB+6OiA8i4m/A0vR8hZzTzMw6sEKuQfUFluW8rgf2ydunFvgfSWcB3YEv5hw7P+/Yvunzls4JgKQpwJT05TuSni+gzpvTE1jZ4l5mW86xZu2l3GOtf3OFxRokcQIwMyJ+KOkzwO2ShhXjxBFxI3BjMc4FIKmuuSk1zIrNsWbtpVJjrZAEtRzYLed1v7Qs16kk15iIiHmSupFk9M0d29I5zcysAyvkGtQCYJCkgZK6kgx6mJW3z0vAOABJg4FuQEO63/GStpI0EBgE/KnAc5qZWQfWYgsqItZJmgo8BFQBN0fEs5KmA3URMQs4D7hJ0jkkAyYmRzK1w7OS7gUWA+uAMyNiPUBz5/wI3l9zitZdaNYCx5q1l4qMtbJabsPMzDqOsp+Lz8zMKpMTlJmZZZITlJmZZZITVJFIUqnrYB2DY83aS6ljrexnM88CSUpHLSJpb+AVYE1ElPOd3ZZBebG2L/AW8EZEvLz5I81aJwux5hZUEeR8iGcC1wCnAXdL2qmkFbOKkxdrVwJHAH+UtEtJK2YVJwux5gRVJJI+DxwNHEQyYe7bwBuS/G9sRSVpDMnkygeQ3He4BFhZ6u4YqzyljjV/ebZRMx/Qu8ADwGRgMHBC+gvkgHTqJ7M2aSbWVgG/Ac4EaoDDI2ID8GVJ27V3/axyZC3WfKNuG+T1zR5AsrzIG8A84J2IGJpu+ypwCPDViHi7VPW18pUXa0cAr5PMzPIY0Dki9ki3nQRMBCZGxKpS1dfKVxZjzQlqC0g6F/gycEpEPC9pAnAB8EtgA8ks76dExNMlrKZVgHQasWOAr6dTjY0A/gv4WbrLQTjWrAiyFGtOUG2Ufmg/AQ6NiLfSMgF7k3TzrSRZZfi5klXSKoKkj5PMtXZ0RLwpqUtErJU0CDgKWAM8GBFbulaadXBZizUPMy9QbvM3tZpkIERXSVURsT4iQtKSiDirRNW0CtBMrH0AbA3sCrwZEWvT8rcj4t/bvYJWMbIeax4kUYC8vtlukrYiudfpbWAIyfIiSJoI/B9JW5esslbW8mJtR0nbRkQ98L/AKEn90m0Tgcs8KMLaqhxizV18Lcj7EM8DPgv8A7gI2ItkscYVwJskw8yPiIjFJaqulbG8WPsWydDerYGvAduS3F/Xl2So75dwrFkblUusOUEVSNLngOnAxSTDLY8jWUV4G2BfoA9wf0T8pWSVtIogqQb4DmOao34AAAz7SURBVPB14EjgQpIYW06yMGhP4HcR8ULJKmkVIeux5mtQBZB0MHAu8EBE/B74vaQ1JPcHfCUibi1pBa1ipD+EzgOejIgXgRmSAvg9cFhE/KqU9bPKUQ6x5mtQzWjmZrXHSbrw9pXUCyAirgB+DtySLmnvf0trtWZi7RmSe08GSBqTdsVcBVwF/NyxZm1VjrHmLr48eX2zx5KManmD5MLhXcCfgasi4rV0n50i4vVS1dfKV16sTQIErI2In0m6kuReuvsi4k/pPjtGxBulq7GVq3KNNf8Sy5PzIX4DmEoyr95MkgERZwCfBL6b05JycrI2yYm1qSQXp5cBN0n6EnAZyf+fX5U0Kj3kzZJU1MpeucaaE1RKUs/G4eGS+gD7RcTngf7A08DjEfEKcDrQg+QXiFmrSdpN6Uz3knYEPkMyimok8CjwcPrD5xKggeTLpOlLxqxQ5R5r7uIDJA0AvkcyncdvSQaP/JTkw9qDZOLX9ySdQjIw4rV0wkSzVkl//FwEvADcTjLf2W3AOyQjpiZFxOr0l+6ciHimZJW1slYJseYWFJCOYHka+CLw+XTqoqXAJJL5qN5L+23PAjo5OVlbpa3wR0nuMTk2jaUngCnA1PQL4ysk3TCeYNjarBJirUMPM28c1ZI2Z18lmWtqnKT1JPNRATwq6f+R3BMwKbxyqbVRzoXqHiQzkBwiaR1wLcm1zt9K+gPJfI4nRsRLpautlbNKibUO3cXX+CGmraMzga8C3wCqSEa0/EbSISRN4mUR8bcSVtcqgKQvk9wYeRRwItALeDYibkgvUK8HXs/qF4aVj0qItQ7ZgpJ0EPBeRPwuLRpOMgvEs8AUSd8Dvi+pCvhtRLxXqrpaeZP0BZK1dB5Oi/YAHkrvzJ8uaTJwZjq/4+0eFWptVYmx1uGuQTVOgAjUS+qbPl8IfFzJVPNExHSgC/AFktaUWatJ+gTJr9YlknZPi+cDg5UspU1EzATeAnYnWVLbrNUqNdY6VAsqnbLoJ/xzBvI3lCwy+CuSARJHSFoIbAfUA1dHxDulqq+VL0lfBK4jaZ13AZZJOhO4n+RC9TFK1thZnW6fkYUbI638VHKsdahrUJIuALYn+dCeBIYBNwOHA38DTiG5T2AbklEumRt2aeVB0hTg08BckulkegJXk8zp+DjJoJvjSK4DfDcinipRVa3MVXKsdbQE9TngcmA34JCIWCzpSOBWkunkH01v1t0qIjJxJ7WVJ0m7kUyN9QlgfEQsSgfc/AQ4LyIekNQF6BIRq0tZVytvlRxrFd/FlzsHFcmcep1IfmkMlrQ8/fACeETSYRHxa8CDIqzV8mINknXD/h9wkKT6iHhQ0hnAPUpWYb4PWNvsycw2o6PEWkW3oPImSNyZZH6pbUim+hgH1AH3RsS76ZxUf42IP5eswla28mKtH/BauumTJDdGvgH8MCLeSq8ZvBgRS0tTWytnHSnWKjZB5X2IZwGTgf8BHgT+AEwERpPMTn5ruTV9LTskNc0ukk4yfBrJHfwLI+IWSfsCXyYZOXVJRGTyrn3Lvo4WaxXbxZeTnA4B9iG5YDiW5MPbPiLukNQN+BeSkS1mbZLzhfEFkusAk4BBwP6SukfET9JrAAcDW5WuplbuOlqsVWwLCkDSMGAO8G8RcbWkj5HcVT2IZHLEByRtX+6/Mqy0lCzqNphkPscrImJaOoP0Z4AjgOcj4kpJ3SLi/VLW1cpbR4u1irpRt3FuvUbpMPH/AL4l6ZPpPHr3ktzj9Jn0F4eTk7VabqxFxIZ0FpKpwNmShqV36f+OpEt5gJIF4Mr+C8PaX0eOtYppQeVdc/oi8DFgAcmSGZNJ7nE6MSKeU7LY4LpyuVnNsiUv1g4naZEviohHJJ1MsmT2F9LhvtsBGyLi3RJW2cpUR4+1irkGlfMhfpMkIT0BHA38EbiT5L3+StLBEbGkVPW08pcTa98ATiZZQ2xSek/dBSTTYz0uabhv9rYt0dFjrewTlKT+wAcR8YqSFSPHAgdERIOkccCXgOERcVXaf1t29wJYNqTTxSgilqQXovcGJkREvaQhwDHA5Ii4Nh2A41izNnGsJcr6GpSkA4Gfkazh1J9kWYzuJFMXERG/JVnG+IT09Y/CS2ZYGyiZAf9nwOcl/QvJtDEfI+k6JiIWkyxyuV/6+rqIeL5E1bUy5lj7p7JNUJLGk/S/XhQRd0bE3yNiLTCT5ELh+HTXF4H3JHUtTU2t3Ek6AJhBMm3MTyNiSTrcdzrQV9Ip6a4fAJ0kdS9VXa28OdY+rOwGSaQjWrqQTPL6XxFxX96FxN1Imr8Hk0wtP5RkueOnS1VnK09prHUimXhzfnrvXG6sbU8yK8l5JHfzfxLHmrWBY615ZXcNKv3A1kj6AFiRFlcB69LtyyTNBe4huQl3SUQsL0llraylsbZe0vskv1ghjbX0y+NtSUtIvjj6A6si4tUSVdfKmGOteWXbxUfSimrsk12X14U3EngrImY7OVkRdCJZMrsp1uKfXQ/7Az0iYnFH+MKwj5xjLUfZJaicm9ZqgR6SzgGIiDXp9lOAkwBfc7JimQZ0k/RD+FCsnQx8BVhTwrpZZXGs5Siba1C5kySmr7cCxpO0ot4C7ibp0jsd+HJ6t7VZq+VNyNkpIjZIGg78iKT75ZdAD5LYc6xZmznWNi/zCUrSnpu6EJjeH9AH+A7JGk7bkCzT3qE+RCsOSXtFxBOb2V4FXEwy7Hdrklnwn2uv+lnlcKwVphwS1BUkraMnooXKKlmYa3371MwqjaQfAbOBX+XHmqTOEbEu53X+gnFmBXOsFaYcrkG9DXwqZ7jljumvC9LXVTn7bsg/2KwVFgJ9cmKtj6Qe0HTBuhz+f7Hy4FgrQCZbUJI+Q9K0/TtJt93JEVEr6WKgH/AcybTyvy5hNa0CSKohibFXgb8BP4iI09JY251k5vsVEXFDCatpFcCx1nqZy9KSxgC/IJlD7xska58MldQn3eV8YDHJvQBmbSZpb5LlVz5BMrjmc8AGSb2Bf5CMqJoJ9CxVHa0ySNoLx1qrZSZB5Qwf34VkIa6LgTdJWku/A/YEhgH7kkw5f0Q6ks+srUQSaz8Gfk/yy/ZJ4FCSEaH7kywCN17SNiWrpZWtnO+1KuD/OtZaJzMJiuQDBFgEnJzez3QqMIJkpt6BJIt0fYlkZt9vRcQHzZ3IbHMk7ZHOfP8a8CVJpwHfAz5O0vXSHbiQ5IfQQODMiFhdqvpaWWv8XltGknzOwLFWsExcg5J0MEkyegKYBzwOfBbYKiLul7QD8H8i4ttpq2lDOjGsWatIOoTktoSvRMSLkvYgabXvEBEPpfF1DXA2yfRZGzrqCCrbMulqC5NIvs+eAh4jmRu0Z0T8xrHWspLPxZdec7qGZGaISJ//PCIuSbd3JpkV4uOS/iW82KC1UbqMwQ+Br6bJqXNEvAC8kG7vQtLtVwUMigpcAM7aRzog4mqS77WVwI+B2yLiB+l2x1oBSp6gSJLPnIi4E0DSbOCxdA6qi9L7ARok/Rx4v5QVtfKVdulNBuZFxLx0SO/Zkt4GGtIlW9YCayX9N1Axy2ZbSXwM+GlE3AMg6U7ge+lsEZc51gqThWtQHwB9JO0EEBErgM8AB0s6tnGniLgnIl4qUR2tzEXEG8AdwCuSLifpbtmWZNLhb0iakrPvL8MLW9qW6Q6cmPYAQXKP5rdJvtc+37iTY23zStKCUrKc8Q4RURcRCyQ9DzwMjAKIZPn2a4HepaifVY401naMiD9FxK8l/QP4JnBdRPwk3Wc58KlS1tPKXxprPSJiQUT8NL18MT9dJuNjETFW0i7ADqWtaflo9wQl6QjgEmCppBXA4xFxvqT/kFQHHBoRr5DcDzAkvaM6fPHQWisv1l4GniZZ6HIJsCpn192B3h15ShnbMnmx9gpQFxFTJA0FupEMKSd97ns4C9SuCUrSzsBpwAkRsVjSV0muA+yU3lF9BXC9ksUIR5DM3uvpi6zVNhFrU4FeJBMKr033OxE4jmRUn5OTtdpmvtd2BX4SEW+n+50KHAuMK11ty0t7X4NaR9Lv3wcgIm4G/gIMlHRkRFwAfBf4v8DBnpXctkBzsfZXkiHlhwFI2odk7bATO+JM0VY0zcXaUpJLFIdCU/ffUOBLEbG0RPUsO+1+H5Skxmk+HiLp9+9Pcmf1vhFxcrtWxiraZmJtTEScmu6zYzqAwqzNCoy1rSPivdLVsvyUYpDEXSRzT40D3oyIEwEkHSZp+8bmsFkRbC7WdoyIN5ycrEg2F2s9IuJNJ6fWK9lMEvrwSpKTSCZQPCAifE+AFZVjzdqLY624Snajbs6H+FWSGcqP84doHwXHmrUXx1pxlXwuPkn9gS6+cGgfNceatRfHWnGUPEGZmZk1JwtTHZmZmW3ECcrMzDLJCcrMzDLJCcrMzDLJCcrMzDLJCcrMzDLJCcrMzDLp/wMfOnsrBymAfgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cie.plot_util(intervals_A=[param_90, param_95, param_99],\n",
    "              intervals_B=[nonparam_90, nonparam_95, nonparam_99],\n",
    "              label_A=\"Parametric\", label_B=\"Non-Parametric\", ylim=(0.8, 1.01))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter Tuning and Plots\n",
    "The following example shows how to compute confidence intervals during the tuning of hyperparameters and how to plot them.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(array([1.799e+01, 1.038e+01, 1.228e+02, 1.001e+03, 1.184e-01]), 0),\n",
       " (array([2.057e+01, 1.777e+01, 1.329e+02, 1.326e+03, 8.474e-02]), 0)]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data loading and presentation\n",
    "\n",
    "cancer_dataset = load_breast_cancer()\n",
    "cancer_X, cancer_y = cancer_dataset['data'], cancer_dataset['target']\n",
    "[(a[:5], b) for (a, b) in zip(cancer_X, cancer_y)][:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(array([2.060e+01, 2.933e+01, 1.401e+02, 1.265e+03, 1.178e-01]), 0),\n",
       " (array([7.760e+00, 2.454e+01, 4.792e+01, 1.810e+02, 5.263e-02]), 1)]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(a[:5], b) for (a, b) in zip(cancer_X, cancer_y)][-2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "neigbhors= [1, 3, 5, 7, 10, 15, 20, 50, 75, 100, 150, 200]\n",
    "nn_params_dict = {\"n_neighbors\":None}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae00cf54f4704ceb92cfc4c5bcdb0cf3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=12.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# As an example we will test different hyperparameters for KNN classifiers.\n",
    "\n",
    "confidence = {90:[], 95:[], 99:[]}\n",
    "for k in tqdm(neigbhors):\n",
    "    nn_params_dict[\"n_neighbors\"]=k\n",
    "    accs=cie.get_accuracy_samples(KNeighborsClassifier, cancer_X, cancer_y,\n",
    "                             model_params_dict=nn_params_dict,\n",
    "                             n_iterations=100, sample_ratio=0.7,\n",
    "                             train_ratio=0.7, random_seed=42,\n",
    "                             sample_with_replacement=True,\n",
    "                             verbose=False, progress_bar=False)\n",
    "    \n",
    "    confidence[90].append(cie.nonparametric_confidence_interval(accs, .9, verbose=False))\n",
    "    confidence[95].append(cie.nonparametric_confidence_interval(accs, .95, verbose=False))\n",
    "    confidence[99].append(cie.nonparametric_confidence_interval(accs, .99, verbose=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy with 90% confidence:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dfbxVZZ338c9XFLEHE+UEJYhYkGKa1glLx6C7RDJHKqcCc9SeyEorzSadTI5YkzVNz5hhkVkpMTY5NDflTSbUy3TkkE9BqYiTgGLH1Bx7EMXf/cd1HV1uD7AP7rX3Yu/v+/Xar7PWtdbav2sdDvu3r4e1liICMzOzqtmh1RUwMzMbiBOUmZlVkhOUmZlVkhOUmZlVkhOUmZlVkhOUmZlVUmkJStJ8SX+Q9JvNbJekr0haLelmSS8vbDtR0u35dWJZdTQzs+oqswV1MTBtC9vfAIzPr1nA1wEk7Q7MBg4BJgGzJQ0vsZ5mZlZBO5b1xhHxC0l7b2GX6cAlka4Uvk7SbpJeAEwBlkTE/QCSlpAS3WVbijdixIjYe+8thTMzsypasWLFfRHRVVteWoKqw57A2sL6uly2ufKnkTSL1Ppir732ore3t5yamplZaST9fqDy7XqSRETMi4juiOju6npa8jUzs+1YKxPUemBMYX10LttcuZmZdZBWJqhFwAl5Nt+rgD9FxD3AlcBUScPz5IipuczMzDpIaWNQki4jTXgYIWkdaWbeTgARcSGwGDgKWA38BXhn3na/pPOA5fmt5vRPmDAzs85R5iy+mVvZHsAHN7NtPjC/jHqZmdn2YbueJGFmZu3LCcrMzCrJCcrMzCqp4xNUT08Pkp726unpaXXVzMw6mtJche1fd3d3PJM7SUyZMgWApUuXNqZCZmZWF0krIqK7trzjW1BmZlZNTlBmZlZJTlBmZlZJTlBmZlZJTlBmZlZJTlBmZlZJTlAt0onXX/mcO+OczRrF10FlrboOqhOvv/I5m1mRr4MyM7PtihOUmZlVkhNUB+rEcZFOO+dOO19rT6WOQUmaBnwZGAJ8MyLOr9k+lvRgwi7gfuD4iFiXt20Cbsm73hURx2wplsegtp/YPuf2j2s2GJsbgyrzke9DgLnAEcA6YLmkRRGxqrDb54FLIuI7kv4P8BngH/O2v0bEQWXVz8zMqq20BAVMAlZHxBoASQuA6UAxQU0ETs/LVwNXlFifAY0aNYp77733iXVJTyyPHDmSDRs2NLtKZmZGuWNQewJrC+vrclnRTcBb8vKbgedK2iOvD5PUK+k6SW8aKICkWXmf3r6+vm2qZDE5DWabmQ3M41/WKK2eJHEGMFnSDcBkYD2wKW8bm/skjwO+JOlFtQdHxLyI6I6I7q6urqZV2sw2r6enh4hg8uTJTJ48mYggIpygbNDK7OJbD4wprI/OZU+IiLvJLShJzwGOjYgH87b1+ecaSUuBg4E7SqyvmZlVSJktqOXAeEnjJA0FZgCLijtIGiGpvw5nkWb0IWm4pJ379wEO46ljV2Zm1uZKS1AR8RhwCnAl8FtgYUSslDRHUv+U8SnArZJuA0YCn87l+wG9km4iTZ44v2b2n5mZtbkyu/iIiMXA4pqycwrLlwOXD3Dcr4ADyqybmZlVW6snSbTcyJEjt2mbmZmVq9QW1Pag/zonX3FvZlYtHd+CapVRo0YhiWXLlrFs2bKnXC8yatSoVlevFJ14ztY8vv6q/XR8C6pVOvEC4U48Z2uenp4eenp63BvSRpygzErkW2mZbTt38ZmVyK1Gs23nBNWBOnEsqNPOudPOt5U89lUed/F1oE78Vt9p59xp59tKHvsqj1tQZmZWSU5QLdKJFwh34jmb2bZzF1+LdOIFwp14zma27dyCMiuRW41m284tKLMSdWKr0dd+WaO4BdWBOvFbfaedcyvP1zMIrVHcgupAnfitvtPOudPO19pTx7eg+i+yq72g0RfZmZm1VqkJStI0SbdKWi3pzAG2j5V0laSbJS2VNLqw7URJt+fXiWXVsaenh4h42ssJysystUpLUJKGAHOBNwATgZmSJtbs9nngkog4EJgDfCYfuzswGzgEmATMljS8rLqamVn1lDkGNQlYHRFrACQtAKYDqwr7TAROz8tXA1fk5SOBJRFxfz52CTANuKzE+prZdsyzB9tPmV18ewJrC+vrclnRTcBb8vKbgedK2qPOY5E0S1KvpN6+vr6GVdzMtl2rZhB24uzBdr9RbasnSZwBTJZ0AzAZWA9sqvfgiJgXEd0R0d3V1VVWHc1sEDZs2EBEMHnyZCZPnvyUsV23Yhqrfwy99nfdLgmqzC6+9cCYwvroXPaEiLib3IKS9Bzg2Ih4UNJ6YErNsUtLrKuZmVVMmS2o5cB4SeMkDQVmAIuKO0gaIam/DmcB8/PylcBUScPz5IipuczMzDpEaQkqIh4DTiEllt8CCyNipaQ5ko7Ju00BbpV0GzAS+HQ+9n7gPFKSWw7M6Z8w0S468fqrTjxnM9t2pd5JIiIWA4trys4pLF8OXL6ZY+fzZIuq7fQ/5KyTdOI5W/vz7MHytHqShJlZQ3j2YPvxvfjMrC34/oPtxy2oDtSJY0GtOudO/F2bNYoiotV1aIju7u7o7e1tdTXMKqGnp4dzzz33aeWzZ89uWnJsVUum2XGLY04DacZn7PbeapS0IiK6a8vdxWfWhjwhxdqBu/jMzKySnKDMzJ6BTntaczO5i8/M7Bnw7MHyuAVlZmaV5ARlZmaV5ARlZmaV5ARlZg3li5PbX7MelOhJEmbWUL4Gq/31/xuXPTHELSgzM6skJygzM6skJygzawse+2o/pY5BSZoGfBkYAnwzIs6v2b4X8B1gt7zPmRGxWNLepKfw3pp3vS4iTi6zrma2fevEsa92f1hiaQlK0hBgLnAEsA5YLmlRRKwq7HY26VHwX5c0kfT03b3ztjsi4qCy6mdmtr1r94clltnFNwlYHRFrImIjsACYXrNPALvm5ecBd5dYHzMz246UmaD2BNYW1tflsqIe4HhJ60itp1ML28ZJukHSMkmHDxRA0ixJvZJ6+/r6Glh1MzNrtVZPkpgJXBwRo4GjgO9K2gG4B9grIg4GTgculbRr7cERMS8iuiOiu6urq6kVNzOzcpWZoNYDYwrro3NZ0buBhQARcS0wDBgREY9ExB9z+QrgDmBCiXU1M7OKKTNBLQfGSxonaSgwA1hUs89dwOsAJO1HSlB9krryJAsk7QOMB9aUWFczM6uY0hJURDwGnAJcSZoyvjAiVkqaI+mYvNtHgfdKugm4DDgpIgJ4DXCzpBuBy4GTI+L+supqZratWnn9Vbs/LFEpH2z/uru7o7e3t9XVMDNrulY9LLFRcSWtiIju2vJWT5IwMzMbkBOUmZlVkhOUmZlVkhOUmZlVkhOUmZlVkhOUmZlVkhOUmZlVkhOUmZlVkhOUmZlV0lYTlKRTJQ1vRmXMzMz61dOCGkl6Gu5CSdNUfKawmZlZSbaaoCLibNLdxL8FnATcLulfJL2o5LqZmVkHq2sMKt9hfEN+PQYMBy6X9LkS62ZmZh1sx63tIOnDwAnAfcA3gY9FxKP5ybe3A/9UbhXNzKwTbTVBAbsDb4mI3xcLI+JxSUeXUy0zM+t09XTx/QR44mGBknaVdAhARPx2SwfmSRW3Slot6cwBtu8l6WpJN0i6WdJRhW1n5eNulXRk/adkZmbtoJ4E9XXg4cL6w7lsi/Ij2+cCbwAmAjMlTazZ7WzSk3YPJj0S/oJ87MS8vj8wDbig/xHwZmbWGepJUIrCY3cj4nHq6xqcBKyOiDURsRFYAEyv2SeAXfPy84C78/J0YEFEPBIRdwKr8/uZmVmHqCdBrZH0IUk75deHgTV1HLcnsLawvi6XFfUAx0taBywGTh3EsWZm1sbqSVAnA4cC60mJ4hBgVoPizwQujojRwFHAd/PswLpImiWpV1JvX19fg6pkZrZ96OnpQRLLli1j2bJlSEISPT09ra5aQ2y1qy4i/kAaDxqs9cCYwvroXFb0btIYExFxraRhwIg6jyUi5gHzALq7u6N2u5lZO+vp6WmbZDSQeq6DGkZKJPsDw/rLI+JdWzl0OTBe0jhScpkBHFezz13A64CLJe2X378PWARcKukLwAtJd7K4vp4TMjOz9lBPd9p3gVHAkcAyUmvmf7d2UEQ8BpwCXAn8ljRbb6WkOZKOybt9FHivpJuAy4CTIlkJLARWAT8FPhgRmwZ3amZmtj2rZzbeiyPirZKmR8R3JF0K/LKeN4+IxaTJD8WycwrLq4DDNnPsp4FP1xPHzMzaTz0tqEfzzwclvZQ0Hfz55VXJzMysvhbUvPw8qLNJY0PPAT5Zaq3MzKzjbTFB5SnfD0XEA8AvgH2aUiszM+t4W+ziy3eN8N3Kzcys6eoZg/qZpDMkjZG0e/+r9JqZmVlHq2cM6u355wcLZYG7+8zMrET13EliXDMqYmZmVlTPnSROGKg8Ii5pfHXMzMySerr4XllYHka6NdGvAScoMzMrTT1dfKcW1yXtRnq2k5mZWWnqfrRFwZ8Bj0uZmVmp6hmD+jFp1h6khDaRdCNXMzOz0tQzBvX5wvJjwO8jYl1J9TEzMwPqS1B3AfdExN8AJO0iae+I+J9Sa2ZmZh2tnjGofwceL6xvymVmZmalqSdB7RgRG/tX8vLQ8qpkZmZWX4LqKzwBF0nTgfvqeXNJ0yTdKmm1pDMH2P5FSTfm122SHixs21TYtqieeGZm1j7qGYM6Gfi+pK/l9XXAgHeXKJI0BJgLHJGPWS5pUX6KLgARcVph/1OBgwtv8deIOKiO+pmZWRuq50LdO4BXSXpOXn+4zveeBKyOiDUAkhYA04FVm9l/JjC7zvc2M7M2t9UuPkn/Imm3iHg4Ih6WNFzSp+p47z2BtYX1dblsoBhjSRf//rxQPExSr6TrJL1pM8fNyvv09vX11VElMzPbXtQzBvWGiHhibCg/XfeoBtdjBnB5RGwqlI2NiG7gOOBLkl5Ue1BEzIuI7ojo7urqanCVzMyslepJUEMk7dy/ImkXYOct7N9vPTCmsD46lw1kBnBZsSAi1uefa4ClPHV8yszM2lw9Cer7wFWS3i3pPcAS4Dt1HLccGC9pnKShpCT0tNl4kvYFhgPXFsqG9ydFSSOAw9j82JWZmbWheiZJfFbSTcDrSffkuxIYW8dxj0k6Je8/BJgfESslzQF6I6I/Wc0AFkREFA7fD/iGpMdJSfT84uw/MzNrf/VMMwe4l5Sc3grcCfywnoMiYjGwuKbsnJr1ngGO+xVwQJ11MzOzNrTZBCVpAmnq90zShbk/ABQRr21S3czMrINtqQX1O+CXwNERsRpA0mlb2N/MzKxhtjRJ4i3APcDVki6S9DpAzamWmZl1us0mqIi4IiJmAPsCVwMfAZ4v6euSpjargmZm1pm2Os08Iv4cEZdGxN+TrmW6Afh46TUzM7OOVs91UE+IiAfy3RteV1aFzMzMoP5p5mZmZgCMGjWKe++994l16cnpCSNHjmTDhg0NiTOoFpSZmVkxOQ1m22A5QZmZWSU5QZmZWSU5QZmZWSU5QZmZWSU5QZmZWSU5QZmZ2aCMHDlym7YNlq+DMjOzQem/zmnKlCkALF26tJQ4bkGZmVkllZqgJE2TdKuk1ZLOHGD7FyXdmF+3SXqwsO1ESbfn14ll1tPMzKqntC4+SUOAucARwDpguaRFxUe3R8Rphf1PBQ7Oy7sDs4Fu0pN8V+RjHyirvmZmVi1ltqAmAasjYk1EbAQWANO3sP9M4LK8fCSwJCLuz0lpCTCtxLqamVnFlJmg9gTWFtbX5bKnkTQWGAf8fDDHSpolqVdSb19fX0MqbWZm1VCVSRIzgMsjYtNgDsqP/uiOiO6urq6SqmZmZq1QZoJaD4wprI/OZQOZwZPde4M91szM2lCZCWo5MF7SOElDSUloUe1OkvYFhgPXFoqvBKZKGi5pODA1l5mZWYcobRZfRDwm6RRSYhkCzI+IlZLmAL0R0Z+sZgALIiIKx94v6TxSkgOYExH3l1VXMzOrnlLvJBERi4HFNWXn1Kz3bObY+cD80ipnZmaVVpVJEmZmZk/hBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpXkBGVmZpVUaoKSNE3SrZJWSzpzM/u8TdIqSSslXVoo3yTpxvx62qPizcysvZX2RF1JQ4C5wBHAOmC5pEURsaqwz3jgLOCwiHhA0vMLb/HXiDiorPqZmVm1ldmCmgSsjog1EbERWABMr9nnvcDciHgAICL+UGJ9zMxsO1JmgtoTWFtYX5fLiiYAEyRdI+k6SdMK24ZJ6s3lbxoogKRZeZ/evr6+xtbezMxaqrQuvkHEHw9MAUYDv5B0QEQ8CIyNiPWS9gF+LumWiLijeHBEzAPmAXR3d0dzq25mZmUqswW1HhhTWB+dy4rWAYsi4tGIuBO4jZSwiIj1+ecaYClwcIl1NTOziikzQS0HxksaJ2koMAOonY13Ban1hKQRpC6/NZKGS9q5UH4YsAozM+sYpXXxRcRjkk4BrgSGAPMjYqWkOUBvRCzK26ZKWgVsAj4WEX+UdCjwDUmPk5Lo+cXZf2Zm1v5KHYOKiMXA4pqycwrLAZyeX8V9fgUcUGbdzMys2nwnCTMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzqyQnKDMzq6RSE5SkaZJulbRa0pmb2edtklZJWinp0kL5iZJuz68Ty6ynmZlVT2lP1JU0BJgLHAGsA5ZLWlR8dLuk8cBZwGER8YCk5+fy3YHZQDcQwIp87ANl1dfMzKqlzBbUJGB1RKyJiI3AAmB6zT7vBeb2J56I+EMuPxJYEhH3521LgGkl1tXMzCqmzAS1J7C2sL4ulxVNACZIukbSdZKmDeJYJM2S1Cupt6+vr4FVNzOzVmv1JIkdgfHAFGAmcJGk3eo9OCLmRUR3RHR3dXWVVEUzM2uFMhPUemBMYX10LitaByyKiEcj4k7gNlLCqudYMzNrY2UmqOXAeEnjJA0FZgCLava5gtR6QtIIUpffGuBKYKqk4ZKGA1NzmZmZdYjSZvFFxGOSTiElliHA/IhYKWkO0BsRi3gyEa0CNgEfi4g/Akg6j5TkAOZExP1l1dXMzKqntAQFEBGLgcU1ZecUlgM4Pb9qj50PzC+zfmZmVl2tniRhZmY2ICcoMzOrJCcoMzOrJCcoMzOrJCcoMzOrJCcoMzOrJCcoMzMblJ6eHiSxbNkyli1bhiQk0dPT09A4Spcibf+6u7ujt7e31dUwM7NBkrQiIrpry92CMjOzSnKCMjOzSnKCMjOzSnKCMjOzSnKCMjOzSnKCMjOzSnKCMjOzSnKCMjOzSmqbC3Ul9QG/f4ZvMwK4rwHV2V7itjK2z7n947YydqfFbWXsRsQdGxFdtYVtk6AaQVLvQFczt2vcVsb2Obd/3FbG7rS4rYxdZlx38ZmZWSU5QZmZWSU5QT3VvA6L28rYPuf2j9vK2J0Wt5WxS4vrMSgzM6skt6DMzKySnKDMzKySnKCsqSSNkqRW18PMqs8JCpA0pAUx95c0WdIeLYj995I+3IK4RwI/AsY0O7Z1Dn8Bah8dnaAkTQCIiE3NTFKS3gBcBpwGXCJpVBNjTwXOA1Y1K2Yh7meBFwAfbWbsgTT7Q0zSLs2MV4g7RtJQSc/O6035P9/CuIcAhzYj1hbqoP6/r2b+neXf99BmxSvEnSLpbZKOa/R7d2yCknQ0cKOkS6F5SUrSFODLwHsi4k3ARuClZcfNsQ8FvgvMioglkp4naaykZ5Uc9/XABcA7gPHAfpJeU2bMAepwSG6xvhIgIqJZHx655XiKpGHNiFeI+0bgJ8DXgG9LeklEPF52smhh3COB7wB/KzPOVuowHZgPzJd0eDRpmrSkY4FLgf+S9EZJw5sU97WkL9t7AadLukDSCxsWICI67gU8G/gpMAu4GPheYduQkmPvB7w2L48C7gauAL4B/AN56n9JsV8CrAOmA3sAVwOLgYVlxgaOBA7Ny7sBXwXen9dLO99C/DcAt5Ou1/gR8K3CtlLj59g3AVMG2FbW71ukbtRbgCnASOAM4B5g/7zPDu0SN7/vq3Kc/v9bz8k/dykzbk0dXgb8DjgKOBn4DXBcf11KjDsB+C3wamAmsAj4MPCikuMK+BxwWl4fBnwf+Aowsn+fZxSj7H+0qr6AFwLPId3o8PJikmpiHT4BnJ2XTwIWAF0lx3wZsCYnqveSWtHvIn0L2r3k2Dvkn9OADcABTfgdD8m/13/M67sC1wCXF/YpK1FMBO4ktVghfSl4SfG8S4w9JCfkPftj5A+t9cCEkn/n8/L/r6bEBfbOifAy4ABgLKk1cSHw78D4Mn/XhXocCVxRWJ9GaknOzOtlJuelhfVXAxcBHwKeW/I5z8i/5/6E9Kz8/+2CRrx/x3bxRcTdEfFwRNwHvA/YRdL3ACS9XNK+TajDpyPiU3n5YtKHZ6kTCCLiJuBo4PyIuCgiHo+I+cBwUjO9zNiP558/JX2IHZ2760v7O4yITcANhfWHIuIwYKSkb+SysrphdiF9QD0uaRrwA2AO8AVJXy0jtqQX527M3YDnAe/ojxERXyZ1L/+zpGGN7OLME29Ok7QT6e/4pCbFPRL4FvBzYBlpXPca4DpSV9uvga9Jem6J/879bgQeyt3JO+S/868An5R0aP/ff6MUuuavB+7M40A7RsS1wLdJrfdXNzJmjjtG0s55XPVa4LnAgZJ2iYi/AO8EDpF0zDON1bEJqigi/khKUo9K+h3pg+ThMmPW/ifNfcgjSV1+pYqIVRHxtZrYXaQukma5CXgj6VtlQ//jwpMTYLL1wMclFRPwm4E9JO1fVuyIWEHq8tgfmEtqqc8gtVj3k3R4g+MeDfwH8Hng3Bz7A5LOKuy2EHgEeKRRH9jFiTcR8ShwJnCypI83Ie5nSeOax0bEhaQkcW5EfCUirieNua7PsRuuMLbZHRH3Av8DvB3YS9KQiPgJ8HVSF3oj4x4JfLAw+WYFMAn4O0k7RcSvSC2Z90jasYFx+8cXv0r6ArCR1HL9MHC4pBdExF+Bq4BNzzhgmc2/7e1F+vbVlK6nQsydgXcDK4GXNvl8RfqwXEUeI2hy/IXA3iW879HAX4AFhbLzgLXAXoWyBcCkJsSeBLy5Zr+LgVc1MO6hpHGIg/P6POBTpK62u4CzgReTupJ7geENjHtv/++R1GW+I/By0hee00ljJI2O+3pgNSn5DwV+BrwC2AkYVtjvHcBSYLcS/s6KY5uLgC/k8guBLwCH5/VTgLkNjvuUcU3ScMVs4Is82Z19AnAJDRhXZ+DxxX/K/6f2JH3ZvCS/PkMaQnjGXboN/Qfbnl+kLq4lwIFNjrsTaVD1JS04Z+U/tn2bHbfE966dAHNZYdt5+T/2+0jjf6uAcSXGvrSwbZfC8rHActJD2hoV+1BSt1r/ehfwf/PyPqRvuxeQvmk37AsYA0+8WQJ8Kf++v09qQSxvcNyBJt58oPj3RRpjvZESvnwx8NjmdcBFef2TwPfy38NK4GUNijvQuOZE8hc94Pgcd2lOJgc3+JxrxzVPJz0o9gWkL0PTgXMa9Xnmm8UWSBoWES2bomqNkae5PkSaVXQh8GhEzMzb3kyaPfkK4EsR8ZuSYz8SEe8obD+R9I36nY2MnS+ReHZEPJSXXwD8GDgqIu6RNJbU1fXsiPhTo+Lm2C8jzY4cSupa/BbwHtKEnPMjYq2k4RHxQCPj5tg7RJrCPo30peCIiLglX381G/h2RPy20XFz7I8Dd0fEdwtlvwKuiYiP5aneLwXujIh1DYr5ClKPy69JXwzOAP5I+kKyPCI+nvc7ALg3Iv7QgJgvJn2BX0P+khMRnytsP4vUzfr+iGhsV2qjv1n45VeVXqRvmD8kt6RIXUJjmxz7e3l9P+ADwD4lx92R1OVzVV4/nnQZwy4lxpwInFJTdiXw8rzcjMsJ5gBnkbu0KGcq/YTC8vGkqeTFbuMRpHHAiSXGPYzUlXcHaTp7f/fbVcBrGhz3aOBm0gSUrwHHkMbZzirsszepZdXwf2NPkrC2Fk9OgPmbpFuB/6QRg7eDi/1oIfaPImJNyXEfi4iHgbWSPkMaW/1apMHrsmIONPFmBKnVRuRPspL1T7xRjtnoWXP9F/cvyO//PVLL8Zr+CTiRZgVvJH1BKCvuNaSJCWdExIWRrCWNB21sYNxDgX8FToyIyaQW8iRSd/L7JZ2dW1dTSGOOuzUqdj8nKGt7+UPjZtK06zdHg7pbtiH2sRFR+kzJPHV/KHA4aZLAjIi4pey4hdjvIo0/nRBpZltTRMQPSbNgRzf6vXOX4SnAR0hfdi7LMT9J6lr8saT3SfoEcCDQV1Lc/jvfXE8a3+rf71hS70Cj/74+GxH9l2l8gjSmdTcpKe1DGoM6ldRl3fgu3OZ8sTFrnTwWsBD4aETc3EGxTyKNS6xsYkwBk4ENEfG7ZsYtu5XWqrHNThzXfKIOTlDWCVo5AaZVsZvxod2plJ5CMA/YGBEz8/V0D0fE75sU968Rcbyk/YDXAj8ts+s4X0s1DPjPiHidpONJLfSPlNl17ARlZrYNJI0gjdEcSpqCPaUZ3cc1cQVMbkbXcY59MakbcSrpsoZSu44bdoWxmVkniYj7JN1MunD2iGaNbQ4QtynjmqRrNg/PP18XEbeXHdcJysxsG+TxxaOAqc2ahNKquLmreKOk80jjmqUnJ3AXn5nZNmvh+GJHjGs6QZmZWSX5OigzM6skJygzM6skJygzM6skJygzM6skJyizAUgKSf9WWD9DUs8gjt9Z0s8k3Sjp7c+wLi+UdHkd+w34FGhJF0tq6BNdzZrBCcpsYI8Ab8lX7W+LgwEi4qCI+MEzqUhE3B0RLUkwjXxcuNlgOUGZDewx0j3PTtvSTpJ2l3SFpJslXSfpQEnPJz3V9JW5BfWimmOWSvqspOsl3Sbp8Fw+RNK/SrBpUfQAAAKNSURBVFqe3+99uXxvSb/Jy8+StFDSKkk/kvTfkroL7/1pSTfluowshH29pN4c7+i87zBJ35Z0i6QbJL02l58kaZGknwNXSXqBpF/kc/lNf33NyuYEZbZ5c4F3SHreFvY5F7ghIg4E/hm4JNJTTN8D/DK3oO4Y4LgdI2IS6TEKs3PZu4E/RcQrgVcC75U0rua4DwAPRMRE0mPFX1HY9mzguoh4GfAL0iPP++1NepbPG4ELJQ0DPki6ScABwEzgO7kc0vN9/iE/B+g44MqIOIj0lNwbt/D7MGsYJyizzYiIh4BLgA9tYbe/A76b9/85sIekXet4+//IP1eQkgekG3CeIOlG4L9JT+QdP0C8/gfX/Yb0rKl+G4H/GuB9ARZGxOP5FjVrgH3ze30vv9fvgN8DE/L+SyLi/ry8HHhnHoM7ICL+t47zM3vGnKDMtuxLpJbNsxv8vo/kn5t48p6YAk7Nra6DImJcRPy/Qbzno4Xb0BTfF6D2ljFbu4XMn5/YMeIXwGtIz/65WNIJg6iT2TZzgjLbgtyKWEhKUgP5JemptUiaAtyXW17b4krSo7R3yu83IT9Rtega4G15+0TggDrf+62SdsjjYfsAt9bUfQKwVy5/ivxgunsj4iLgm6TuP7PSeYaO2db9G+mJpQPpAebnxx/8BTjxGcT5Jqlb7tf58QZ9wJtq9rmANFa0CvgdsBKo52mmdwHXA7sCJ0fE3yRdAHxd0i2kSSEnRcQjKfRTTAE+JulR4GHALShrCt8s1mw7kh+9vVNOMC8Cfga8JCI2trhqZg3nFpTZ9uVZwNW5G1DAB5ycrF25BWVmZpXkSRJmZlZJTlBmZlZJTlBmZlZJTlBmZlZJTlBmZlZJ/x/xuky9qA9+aAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Accuracy with 90% confidence:\")\n",
    "cie.plot_util(confidence[90], x_ticks=neigbhors, ylim=None, xlabel=\"N of neighbors\", ylabel=\"Accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy with 95% confidence:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAcGElEQVR4nO3de5gkZXn38e+PBQTxAMo6azgtKqh4ijqiwZBdX0+IRKLmAOpLMCoaxRiMRk1URtQYc44BJBAR8UQ2JBqSl4R4gDUX0ciiiByVYJQFd12MxheNcrrzR9VAMwzLLNvVXTP9/VxXX9NVXVP3Mz0zfdfzPHdVpaqQJKlvthl3AyRJmo8JSpLUSyYoSVIvmaAkSb1kgpIk9ZIJSpLUS50lqCSnJvlOkkvu4vUkeV+Sq5JcnOQJXbVFkrT4bNvhvk8DjgdOv4vXnwPs0z6eDLy//bpZu+66a61cuXI4LZQkjd2FF154fVUtn7u+swRVVZ9LsnIzmxwKnF7NmcJfSLJzkgdX1bc3t9+VK1eybt26IbZUkjROSb453/pxzkHtBlwzsLy+XXcnSY5Ksi7Juk2bNo2kcZKk8VoURRJVdXJVTVfV9PLld+oFSpKWoHEmqGuBPQaWd2/XSZI01gR1FnBEW833FOC/727+SZI0OTorkkjycWA1sGuS9cCxwHYAVXUScDZwMHAV8CPgpV21RZK0+HRZxXf43bxewGu6ii9JWtwWRZGEJGnymKAkSb1kgpIk9dLEJKiZmRmS3OkxMzMz7qZJkuaRplZh8Zienq6tudTR6tWrATjvvPOG0yBJ0lZJcmFVTc9dPzE9KEnS4mKCkiT1kglKktRLJihJUi+ZoNSZcVZOWrUpLX5W8alz43zP/X1L/WcV3xjZk5gc/q6l4bEHNULGnpzYk/gzS/eUPShJ0qJigpIk9ZIJSpLUSyYoSVIvmaAkbTUrCNWFzm75LmlyzMzMMDMzYwWhhsoelCSpl0xQkqReMkFJknppYuagVqxYwcaNG29bTnLb86mpKTZs2DCOZkmS7sLE9KAGk9OWvCZJd8XqxW5NTA9KkobN6sVuTUwPStLSZC9m6bIHJWlRsxezdJmg1BkLUyRtDRPUCIzzg3qcsS1MkbQ1JiZBTU1N3eWH4tTUVKexx/lBPalJwt6btPhNTIKa/UBynHoyTGpilpYSq/gkSb1kgpIk9ZIJSpLUSxMzB6XRG2dhyrhMasWm1AV7UCOwuQ/jrj+oxxl7w4YNVBWrVq1i1apVVNVtj6X6YWnFpjQ89qBGYJwVhJNavTiJvTdpqTFBaUma1MQ8Lg4vqgsO8Unaag4vqgsmKElSL3WaoJIclOTKJFclefM8r++V5DNJLk5yXpLdu2yPJGnx6CxBJVkGnAA8B9gPODzJfnM2+yPg9Kp6LHAc8J6u2iONwqRWbGr0JuE+WF0WSewPXFVVVwMkOQM4FLhsYJv9gNe3z88FPtlhe6TOWbGpUZmE+2B1OcS3G3DNwPL6dt2grwAvaJ8/H7hvkgd22CZJ0iIx7jLzNwDHJzkS+BxwLXDL3I2SHAUcBbDnnnuOsn2SFmCc551Z4r50dZmgrgX2GFjevV13m6q6jrYHleQ+wAur6vtzd1RVJwMnA0xPT1dXDZZ0z4xzeNES96WryyG+C4B9kuydZHvgMOCswQ2S7Jpktg1vAU7tsD2SpEWkswRVVTcDRwPnAJcDa6rq0iTHJXleu9lq4MokXwOmgHd31R5J0uLS6RxUVZ0NnD1n3dsHnp8JnNllG2bNzMzwjne847bl2XHqY489dkmVZUoaHee/ujXuIomRmS3JlKRhcf6rW17qSJLUSyYoSYuaV9BYukxQIzB7SZK1a9eydu3akV6SZFJja3JM4o0xJ8XEzEGN0zjnvyY1tqTFzx6UJKmXTFCSdA85/9UtE5SWJOe/NArOf3XLOSgtSc5/SYufPShJUi+ZoCRJvWSCkiRtkVHdbt45KEnSFhnV7ebtQUmSeskEJUnqJROUNESTeu1DzztTF1JV427DFpmenq5169aNuxmSeqbr+RBjdxc7yYVVNT13vT0oSVIvmaAkSb1kgpK0qE3q/NeKFSvm/bmTsGLFinE3byg8D0rSojap113cuHHjPXptMbEHJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqJROUJC1CU1NT9+i1xcQyc0lahDZs2ACM91JHXbMHJUnqJROUJN1Dk3oVi1FxiE+S7qFJvYrFqNiDkiT1kglKktRLJihJUi+ZoCRJvXS3CSrJa5PsMorGSJI0ayE9qCnggiRrkhyUJF03SpKku01QVfVWYB/gA8CRwNeT/F6Sh3bcNknSBFvQHFRVFbChfdwM7AKcmeQPOmybJGmC3e2JukleBxwBXA/8FfDGqropyTbA14Hf7raJkqRJtJAe1AOAF1TVs6vqb6rqJoCquhU4ZHPf2M5ZXZnkqiRvnuf1PZOcm+TLSS5OcvA9+ikkSUvOQhLUPwH/NbuQ5H5JngxQVZff1TclWQacADwH2A84PMl+czZ7K7Cmqh4PHAacuGXNlyQtVQtJUO8HbhhYvqFdd3f2B66qqqur6kbgDODQOdsUcL/2+f2B6xawX0nSBFjIxWLTFkkAzdBekoV8327ANQPL64Enz9lmBviXJK8FdgKesYD9SpImwEJ6UFcn+Y0k27WP1wFXDyn+4cBpVbU7cDDw4bb44g6SHJVkXZJ1mzZtGlJoSVKfLSRBvQo4ALiW23tBRy3g+64F9hhY3r1dN+hlwBqAqvo8sAOw69wdVdXJVTVdVdPLly9fQGhJ0mJ3t0N1VfUdmgKGLXUBsE+SvWkS02HAi+Zs8y3g6cBpSR5Jk6DsIkmSFnQe1A40PZ1H0SQQAKrq1zb3fVV1c5KjgXOAZcCpVXVpkuOAdVV1FvBbwClJjqEpmDhycL5LkjS5FlLs8GHgCuDZwHHAi4G7LC8fVFVnA2fPWff2geeXAU9daGMlSZNjIXNQD6uqtwE/rKoPAc/lztV4kiQN1UIS1E3t1+8neTTN+UoP6q5JkiQtbIjv5PZ+UG8FzgLuA7yt01ZJkibeZntQ7TlJP6iq71XV56rqIVX1oKr6yxG1T5I0j5mZGZKwdu1a1q5dSxKSMDMzM+6mDc1mE1R7QVivVi5JPTMzM0NV3ekxMQmq9ekkb0iyR5IHzD46b5kkaaItZA7qV9qvrxlYV8BDht8cSZIaC7mSxN6jaIgkSYMWciWJI+ZbX1WnD785kiQ1FjLE96SB5zvQXDvvS4AJSpLUmYUM8b12cDnJzjQ3H5QkqTMLqeKb64eA81KSpE4tZA7qH2iq9qBJaPvR3sNJkqSuLGQO6o8Gnt8MfLOq1nfUHkmSgIUlqG8B366qHwMk2THJyqr6z05bJkmaaAuZg/ob4NaB5VvadZIkdWYhCWrbqrpxdqF9vn13TZIkaWEJalOS580uJDkUuL67JkmStLA5qFcBH01yfLu8Hpj36hKSJA3LQk7U/Q/gKUnu0y7f0HmrJEkT726H+JL8XpKdq+qGqrohyS5J3jWKxkmSJtdC5qCeU1Xfn12oqu8BB3fXJEmSFpagliW51+xCkh2Be21me0mSttpCiiQ+CnwmyQeBAEcCH+qyUZIkLaRI4r1JvgI8g+aafOcAe3XdMEnSZFvo1cw30iSnXwL+D3B5Zy2SJInN9KCS7Asc3j6uB/4aSFU9bURtkyRNsM0N8V0B/CtwSFVdBZDkmJG0SpI08TY3xPcC4NvAuUlOSfJ0miIJSZI6d5cJqqo+WVWHAY8AzgV+E3hQkvcnedaoGihJmkx3WyRRVT+sqo9V1c8DuwNfBt7UecskSRNtoVV8QHMViao6uaqe3lWDJEmChZ2oK0nSbVasWMHGjRtvW05uL0+Ymppiw4YNQ4mzRT0oSZIGk9OWvLalTFCSpF4yQUmSeskEJUnqJROUJKmXTFCSpF4yQUmStsjU1NQ9em1LdZqgkhyU5MokVyV58zyv/2mSi9rH15J8f779SJL6Y8OGDVQVq1atYtWqVVTVbY9hnQMFHZ6om2QZcALwTGA9cEGSs6rqstltquqYge1fCzy+q/ZIkhaXLntQ+wNXVdXVVXUjcAZw6Ga2Pxz4eIftkSQtIl0mqN2AawaW17fr7iTJXsDewGc7bI8kaRHpS5HEYcCZVXXLfC8mOSrJuiTrNm3aNOKmSZLGocsEdS2wx8Dy7u26+RzGZob32iuoT1fV9PLly4fYRElSX3WZoC4A9kmyd5LtaZLQWXM3SvIIYBfg8x22RZK0yHSWoKrqZuBo4BzgcmBNVV2a5LgkzxvY9DDgjKqqrtoiSVp8Or0fVFWdDZw9Z93b5yzPdNkGSdLi1JciCUmS7sAEJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqJROUJKmXTFCSpF4yQUmSeqnTBJXkoCRXJrkqyZvvYptfTnJZkkuTfKzL9kiSFo9tu9pxkmXACcAzgfXABUnOqqrLBrbZB3gL8NSq+l6SB3XVHknS4tJlD2p/4KqqurqqbgTOAA6ds80rgBOq6nsAVfWdDtsjSVpEukxQuwHXDCyvb9cN2hfYN8n5Sb6Q5KD5dpTkqCTrkqzbtGlTR82VJPXJuIsktgX2AVYDhwOnJNl57kZVdXJVTVfV9PLly0fcREnSOHSZoK4F9hhY3r1dN2g9cFZV3VRV3wC+RpOwJEkTrssEdQGwT5K9k2wPHAacNWebT9L0nkiyK82Q39UdtkmStEh0lqCq6mbgaOAc4HJgTVVdmuS4JM9rNzsH+G6Sy4BzgTdW1Xe7apMkafHorMwcoKrOBs6es+7tA88LeH37kCTpNuMukpAkaV4mKElSL5mgJEm9ZIKSJPWSCUqS1EsmKElSL5mgJEm9ZIKSJPWSCUqStEVmZmZIwtq1a1m7di1JSMLMzMxQ46S5mMPiMT09XevWrRt3MyRJQ5LkwqqanrveHpQkqZdMUJKkXjJBSZJ6yQQlSeolE5QkqZdMUJKkXjJBSZJ6yQQlSeqlRXeibpJNwDe3cje7AtcPoTnGNnYf4xrb2Ist9l5VtXzuykWXoIYhybr5zlo2trGXQlxjG3upxHaIT5LUSyYoSVIvTWqCOtnYxl7CcY1t7CUReyLnoCRJ/TepPShJUs+ZoCRJvWSCUmeSrEiScbdD0uI0UQkqybIxxX1UklVJHjjiuD+f5HWjjDkQ+9nAJ4A9xhFfk8eDoaVnIhJUkn0BquqWUSepJM8BPg4cA5yeZMWI4j4LeCdw2SjizRP7vcCDgd8adfz5jPrDK8mOo4w3J/YeSbZPslO7PLL/83HFTvJk4IBRxNpMGzL7dzaGv7ftk2w/ypgDsVcn+eUkLxr2vpd8gkpyCHBRko/BaJNUktXAnwMvr6pfAG4EHj2CuAcAHwaOqqpPJbl/kr2S3HsEsZ8BnAi8GNgHeGSSn+s67jzteHLba30SQFXVqD402t7j0Ul2GEW8ObGfC/wTcDzwwSQPr6pbR5EoxhW7fb8/BPy4yzh304ZDgVOBU5McWCMsj07yQuBjwD8meW6SXUYY+2k0B+B7Aq9PcmKSnxpagKpasg9gJ+CfgaOA04CPDLy2bATxHwk8rX2+ArgO+CTwl8Av0pb5dxD34cB64FDggcC5wNnAmi7jtrGfDRzQPt8Z+Avg19vlzuLOacNzgK/TnJ/xCeADA6912oY29leA1fO81uX7Hprh1K8Cq4Ep4A3At4FHtdtsswRjP6WNM/t/dp/2645dxp3ThscBVwAHA68CLgFeNNuWjmPvC1wO/AxwOHAW8DrgoSOIHeAPgGPa5R2AjwLvA6Zmt9mqGF3/EON+AD8F3IfmgoZnDiapEbfjd4G3ts+PBM4AlncY73HA1W2iegVNb/nXaI52HjCCn3eb9utBwAbgMSN6n5e17+3/bZfvB5wPnDmwTVcHBvsB36DpuUJzcPDwwZ+94yS1rE3Ku83GaT+srgX2HcF7f3L7/zaS2MDKNhF+HHgMsBdNT+Ik4G+Afbp+z9v9Pxv45MDyQTQ9ycPb5c6SZJugzxtY/hngFOA3gPuO4Hd+WPt+zyake7f/fycOY/9Lfoivqq6rqhuq6nrglcCOST4CkOQJSR4xona8u6re1T4/jeaDs7MCgqr6CnAI8PtVdUpV3VpVpwK70HTHO1VVt7Zf/5nmg+uQdoi+07+5qroF+PLA8g+q6qnAVJK/bNd1NfyyI80H061JDgL+GjgO+JMkf9FV7CQPa4cydwbuD7x4Nk5V/TnNMPPvJNlh2MOcbSHOMUm2o/mbPnIUsdthvQ8AnwXW0szxng98gWao7UvA8Unu2+Hve9ZFwA/aYeVt2r/59wFvS3LA7P/CMA0M138R+EY7B7RtVX0e+CBNT/5nhh23jb1Hknu186yfB+4LPDbJjlX1I+ClwJOTPG9rYy35BDWoqr5Lk6RuSnIFzQfIDV3HnfuP2Y4ZT9EM+XWmqi6rquPnxF1OMyQySl8BnktzJDn0f1a4vRCmdS3wpiSDifj5wAOTPKqr2FV1Ic0Qx6OAE2h67IfR9FwfmeTADmIfAvwd8EfAO9r4r07yloHN1gA/AX4yzA/rwUKcqroJeDPwqiRv6jL2QBHOPsALq+okmiTxjqp6X1V9kWYO9to29tANzHFOV9VG4D+BXwH2TLKsqv4JeD/NkPqwYz8beM1AIc6FwP7AzybZrqr+jaYX8/Ik2w459uw841/QHAjcSNODfR1wYJIHV9X/AJ8BbtnqgF13Afv4oDnaGtmw00DcewEvAy4FHj3CuKH5kLyMdk5gDO/5GmBlR/s+BPgRcMbAuncC1wB7Dqw7A9h/BLH3B54/Z7vTgKcMOfYBNPMPj2+XTwbeRTPM9i3grcDDaIaU1wG7DDn2xtn3k2YIfVvgCTQHQK+nmR8ZamzgGcBVNAcB2wOfBp4IbAfsMLDdi4HzgJ07+HsbnOM8C/iTdv1JwJ8AB7bLRwMndBD7DnOcNFMYxwJ/yu1D20cApzOkuXbmn2f87fZ/bDeaA9DT28d7aKYWtnpod6i/uMXwoBni+hTw2DHE3o5mIvXhI46b9o/qEWP4mbse/59bCPPxgdfe2f4zv5JmDvAyYO8OY39s4LUdB56/ELiA5qZsw/zZD6AZUptdXg78v/b5Q2iOcE+kOcIe6sEY8xfifAr4s/Z9/yhND+KCYcZm/iKcVw/+rdHMuV5EBwdjzD/H+QXglHb5bcBH2r+LS4HHDTH2fHOc+9Ee+AEvaWOf1yaSx3fws8+d43w9zQ1kH0xzYHQo8PZhfcZN5MVik+xQVWMrSdVwtWWtP6CpIjoJuKmqDm9fez5NBeUTgT+rqks6jv2TqnrxwOu/SnMk/dIOYi8DdqqqH7TPHwz8A3BwVX07yV40w1w7VdV/DzN2G/9xNFWS29MML34AeDlNgc7vV9U1SXapqu91EHubakrYD6I5OHhmVX21Pf/qWOCDVXX5sOO2sd8EXFdVHx5Y92/A+VX1xrbM+9HAN6pq/RDjPpFmBOZLNAcHbwC+S3NgckFVvand7jHAxqr6zpDiPozmwP5q2gOeqvqDgdffQjPc+utVNdwh1WEfYfjwMc4HzVHl39L2pGiGgvYaceyPtMuPBF4NPGQEsbelGer5TLv8EprTGXbsOO5+wNFz1p0DPKF93vmpBTSFKG+hHc6ig6o5Boar2vf2Eu44fLwrzVzgfh3HfirNUN5/0JS0zw69fQb4uQ5iHwJcTFOIcjzwPJr5trcMbLOSpmc19N/1RBVJaOmr2wthfpzkSuDvGcZk7ZbFvmkg9ieq6uoRxL65qm4ArknyHpp51uOrmbDuMu58hTi70vTcqPYTrGOzRThpYw61EGfgZP8z2v1/hKbneP5sIU41VcI30hwkdBn7fJqihDdU1UnVuIZmLujGIcc+APhD4FerahVNT3l/mqHlX0/y1rZ3tZpm7nHnYcaHCavi02RoPywupim5fn4NcZhlC2O/sKpGUjHZlvBvDxxIUyBwWFV9dRSxB+L/Gs380xHVVLaNRFX9LU1F7O7D3nc7ZHg08Js0Bz0fb2O+jWZo8R+SvDLJ7wKPBTZ1GHv2ajhfpJnjmt3uhTQjBV38rb23qmZP2/hdmnmt62iS0kNo5qBeSzOEPfyh3NEc4Eij084BrAF+q6ounpTYbfwjaeYjLh1x3ACrgA1VdcUo43bdS3OOczxznGCC0hI1zkKYMcfu/AN7kqW5I8HJwI1VdXh7Xt0NVfXNEcb+n6p6SZJHAk8D/rnrYeT2fKodgL+vqqcneQlNb/03uxxGNkFJ0hZIsivN3MwBNKXXq0c1jDwndoBVoxpGbuOfRjOU+CyaUxw6HUYe6lnGkrTUVdX1SS6mOWn2maOe45wTe2RznDTncR7Yfn16VX2967gmKEnaAu0848HAs0ZZiDLO2O2w8Y1J3kkzx9l5cgKH+CRpiznHOaJ4JihJUh95HpQkqZdMUJKkXjJBSZJ6yQQlSeolE5TUSlJJ/nhg+Q1JZrbg+++V5NNJLkryK1vZlp9KcuYCtpv3jtBJTksy9Lu5SqNkgpJu9xPgBe3Z+vfE4wGq6qer6q+3piFVdV1VjSXBDPs24dI9ZYKSbnczzbXOjtncRkkekOSTSS5O8oUkj03yIJq7mT6p7UE9dM73nJfkvUm+mORrSQ5s1y9L8odJLmj398p2/cokl7TP751kTZLLknwiyb8nmR7Y97uTfKVty9RA2GckWdfGO6TddockH0zy1SRfTvK0dv2RSc5K8lngM0kenORz7c9yyWx7pVEyQUl3dALw4iT338w27wC+XFWPBX4HOL2au5e+HPjXtgf1H/N837ZVtT/N7ROObde9DPjvqnoS8CTgFUn2nvN9rwa+V1X70dxS/IkDr+0EfKGqHgd8juZ257NW0ty/57nASUl2AF5Dc2GAxwCHAx9q10NzT59fbO/98yLgnKr6aZo75F60mfdD6oQJShpQVT8ATgd+YzOb/Szw4Xb7zwIPTHK/Bez+79qvF9IkD2guunlEkouAf6e5K+8+88SbvWHdJTT3m5p1I/CP8+wXYE1V3dpeluZq4BHtvj7S7usK4JvAvu32n6qq/2qfXwC8tJ2De0xV/f8F/HzSUJmgpDv7M5qezU5D3u9P2q+3cPt1MAO8tu11/XRV7V1V/7IF+7xp4NIzg/sFmHuZmLu7bMwPb9uw6nPAz9Hc7+e0JEdsQZukoTBBSXO0vYg1NElqPv9Kc9dakqwGrm97XvfEOTS3z96u3d++7Z1UB50P/HL7+n7AYxa4719Ksk07H/YQ4Mo5bd8X2LNdfwftzeg2VtUpwF/RDP9JI2W1jjS/P6a5U+l8ZoBT29se/Aj41a2I81c0w3Jfam9psAn4hTnbnEgzV3QZcAVwKbCQO5h+C/gicD/gVVX14yQnAu9P8lWaopAjq+onTeg7WA28MclNwA2APSiNnBeLlXquvd32dm2CeSjwaeDhVXXjmJsmdcoelNR/9wbObYcBA7za5KRJYA9KktRLFklIknrJBCVJ6iUTlCSpl0xQkqReMkFJknrpfwG45sN5l//GTwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Accuracy with 95% confidence:\")\n",
    "cie.plot_util(confidence[95], x_ticks=neigbhors, ylim=None, xlabel=\"N of neighbors\", ylabel=\"Accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy with 99% confidence:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAcFElEQVR4nO3debgkZXn38e/PAQRXUMZBARlUUFFcR1AMMsYNkUjUREGN4oYbLrhEjdtxi9EY4wJKICLihhMSzZgX5XWB0YtonEERYRBFjDIgOLgGjQ7LnT+qDjbHw8wZpqu7zunv57r6mq7qOnU/3XNO3/U8z11VqSokSeqbm4y7AZIkzcYEJUnqJROUJKmXTFCSpF4yQUmSeskEJUnqpc4SVJITkvw0ybk38HqSvC/JhUnOSXK/rtoiSZp/tupw3ycCRwMn3cDrjwb2aB/7Ah9s/92oHXfcsZYuXTqcFkqSxu6ss866oqoWz1zfWYKqqq8kWbqRTQ4BTqrmTOGvJ9k+ye2r6icb2+/SpUtZs2bNEFsqSRqnJD+abf0456B2Bi4eWF7XrvsjSY5IsibJmvXr14+kcZKk8ZoXRRJVdVxVLauqZYsX/1EvUJK0AI0zQV0C7DqwvEu7TpKksSaolcDT2mq+BwK/2tT8kyRpcnRWJJHkk8ByYMck64A3AlsDVNWxwKnAQcCFwG+BZ3TVFknS/NNlFd9hm3i9gBd2FV+SNL/NiyIJSdLkMUFJknppYhLU1NQUSf7oMTU1Ne6mdWpS37ek+S/z7Zbvy5Ytqy25ksTy5csBOOOMM4bToHliUt+3pP5LclZVLZu5fmJ6UBq9cfbexhXbHqs0PPagJsQ43/ckxp7U3zPpxrAHNUYeVUvS5uvydhtqTU1NMTU15VG1JG0Ge1DSAmFPXQuNPShpgbCnroXGHpQkqZdMUJKkXjJBSZJ6yQQlSeolE5QkqZdMUJK2mCXu6oJl5pK2mCXu6oI9KElSL5mgJM1rDi8uXA7xSZrXHF5cuOxBSZJ6yQQlSTeSw4vdcohPkm4khxe7ZQ9KktRLE9OD2mmnnbj88suvW05y3fMlS5Zw2WWXjaNZkqQbMDE9qMHktDmvSZLGY2ISlCRpfjFBSZJ6yQQlSeqliSmS0OhZmCJpS5igRmBSv6gtTJG0JSYmQS1ZsuQGvxSXLFnSaexxflFPanKc1PctLSQTk6Cmv5Am7YzvSe3FjOt9mxil4bFIQhqiST0g0OhNwnUAJ6YHJUkLySRcB9AelCSpl0xQ6szGik+6LkyRNP85xDcC46wgHKdJLUwZFws0tNDYgxqByy67jKrigAMO4IADDqCqrnt0/aUxqb2YSXzfFmhooek0QSU5MMkFSS5M8upZXt8tyZeSnJPkjCS7dNmeSTTO5DhO43rfk5gYpa50NsSXZBFwDPAIYB2wOsnKqlo7sNm7gJOq6iNJ/hR4O/BXXbVJ6tqkDms6vKgudNmD2ge4sKouqqoNwMnAITO22Qv4cvv89FlelzQPOLyoLnRZJLEzcPHA8jpg3xnbfBt4PPBe4HHALZPctqp+1mG7JC0g9t4WrnEXSbwCOCDJt4ADgEuAa2ZulOSIJGuSrFm/fv2o2yipx+y9LVxdJqhLgF0Hlndp112nqi6tqsdX1X2B17brfjlzR1V1XFUtq6plixcv7rDJ0vxlgYYWmi6H+FYDeyTZnSYxHQo8eXCDJDsCP6+qa4HXACd02B5pQZvUAg0tXJ31oKrqauBI4DTgfGBFVZ2X5M1JHttuthy4IMn3gCXA27pqjyRpfun0ShJVdSpw6ox1bxh4fgpwSpdtkNS9Sb1aigUa3fJSR5K22KQOL1qg0a1xV/FJ0haxOGT0RnUvKntQkua1Se29jdOo7kU1MT2o6Yy/atUqVq1atSDvPilJC8nE9KCmM74kaX6YmB6UJGl+MUFJ0o1kgUa3JmaIT5KGzQKNbtmDkiT1kglKktRLJihJUi+ZoEZgUs/BmtT3LWk4LJIYgUk9B2tS37ek4bAHJUnqJROUJKmXTFBakJz/kuY/56C0IDn/Jc1/9qAWOHsSo+XnLQ2PPagFzp7EaPl5a1Qm4Xbz9qAkaR6ahNvNm6AkSb1kgpIk9ZIJSpLUSyYoaYEYZwXhpMZWt1JV427DZlm2bFmtWbNm3M2QpOuM44aFg1V7sxnFd/uw3neSs6pq2cz19qAkaR6ahNvNex6UJM1Dk3C7eXtQkqReMkFJknrJBCVJ6iUTlCSpl0xQknQjeQ5Wt6zik6QbyavXd8selCSpl0xQkqReMkFJknppkwkqyYuS7DCKxkiSNG0uPaglwOokK5IcmE1doVCSpCHYZIKqqtcBewAfAg4Hvp/kb5PcueO2SZIm2JzmoKq5bvtl7eNqYAfglCTv7LBtkqQJNpc5qJckOQt4J3AmsHdVPR+4P/CETfzsgUkuSHJhklfP8vodk5ye5FtJzkly0I18H5KkBWYuJ+reBnh8Vf1ocGVVXZvk4Bv6oSSLgGOARwDraOaxVlbV2oHNXgesqKoPJtkLOBVYupnvQZK0AM1liO9zwM+nF5LcKsm+AFV1/kZ+bh/gwqq6qKo2ACcDh8zYpoBbtc9vDVw614ZLkha2uSSoDwJXDixf2a7blJ2BiweW17XrBk0BT02yjqb39KLZdpTkiCRrkqxZv379HEJLkua7uSSo1MDN7avqWoZ3Db/DgBOrahfgIOCjSf6oTVV1XFUtq6plixcvHlJoSVKfzSVBXZTkxUm2bh8vAS6aw89dAuw6sLxLu27Qs4AVAFX1NWBbYMc57FuStMDNJUE9D9iPJrmsA/YFjpjDz60G9kiye5JtgEOBlTO2+THwMIAkd6dJUI7hSZI2PVRXVT+lSS6bpaquTnIkcBqwCDihqs5L8mZgTVWtBF4OHJ/kKJqCicMHhxMlSZNrkwkqybY0Q3H3oOnhAFBVz9zUz1bVqTTFD4Pr3jDwfC3w4M1oryRpQsxliO+jwE7Ao4BVNHNJ/9NloyRJmkuCuktVvR74TVV9BHgMzTyUJEmdmUuCuqr995dJ7klzQu3tumuSJElzO5/puPZ+UK+jqcK7BfD6TlslSZp4G01Q7Umzv66qXwBfAe40klZJkibeRof42qtG/PWI2iJJ0nXmMgf1xSSvSLJrkttMPzpvmSRpos1lDupJ7b8vHFhXONwnSerQXG75vvssD5OTJI3R1NQUSVi1ahWrVq0iCUmYmpoad9OGZi5XknjabOur6qThN0eSNBdTU1MLKhnNZi5DfA8YeL4tzcVdvwmYoCRJnZnLxWKvdxPBJNvT3B1XkqTOzKWKb6bfALsPuyGSJA2ayxzUZ2mq9qBJaHvR3mRQkqSuzGUO6l0Dz68GflRV6zpqjyRJwNwS1I+Bn1TV7wCSbJdkaVX9d6ctkyRNtLnMQf0LcO3A8jXtOkmSOjOXBLVVVW2YXmifb9NdkyRJmluCWp/ksdMLSQ4BruiuSZIkzW0O6nnAx5Mc3S6vA2a9uoQkScMylxN1fwA8MMkt2uUrO2+VJGnibXKIL8nfJtm+qq6sqiuT7JDkraNonCRpcs1lDurRVfXL6YX27roHddckSZLmlqAWJbnp9EKS7YCbbmR7SZK22FyKJD4OfCnJh4EAhwMf6bJRkiTNpUjiHUm+DTyc5pp8pwG7dd0wSdJkm+vVzC+nSU5/CfwpcH5nLZIkiY30oJLsCRzWPq4APgWkqh46orZJknpop5124vLLL79uOcl1z5csWcJll102lDgbG+L7LvBV4OCqurBtxFFDiSpJmrcGk9PmvLa5NjbE93jgJ8DpSY5P8jCaIglJkjp3gwmqqj5TVYcCdwNOB14K3C7JB5M8clQNlCRNpk0WSVTVb6rqE1X1Z8AuwLeAV3XeMknSRJtrFR/QXEWiqo6rqod11SBJkmAzE5QkSaNigpIkbZYlS5bcqNc211wudSRJ0nWmz3Navnw5AGeccUYncexBSZJ6yQQlSeolE5QkqZc6TVBJDkxyQZILk7x6ltf/McnZ7eN7SX45234kSZOnsyKJJIuAY4BHAOuA1UlWVtXa6W2q6qiB7V8E3Ler9kiS5pcue1D7ABdW1UVVtQE4GThkI9sfBnyyw/ZIkuaRLhPUzsDFA8vr2nV/JMluwO7Al2/g9SOSrEmyZv369UNvqCSpf/pSJHEocEpVXTPbi+3llZZV1bLFixePuGmSpHHoMkFdAuw6sLxLu242h+LwniRpQJcJajWwR5Ldk2xDk4RWztwoyd2AHYCvddgWSdI801mCqqqrgSOB04DzgRVVdV6SNyd57MCmhwInV1V11RZJ0vzT6bX4qupU4NQZ694wY3mqyzZIkuanvhRJSJJ0PSYoSVIvmaAkSb1kgpIk9ZIJSpLUSyYoSVIvmaAkSb1kgpIk9ZIJSpLUSyYoSVIvmaAkSb1kgpIk9ZIJSpLUSyYoSVIvmaAkSb1kgpIk9ZIJSpLUSyYoSVIvmaAkSb1kgpIk9ZIJSpLUSyYoSVIvmaAkSb1kgpIk9ZIJSpLUSyYoSVIvmaAkSb1kgpIk9ZIJSpLUSyYoSVIvmaAkSb1kgpIk9ZIJSpLUSyYoSVIvmaAkSb1kgpIk9ZIJSpLUSyYoSVIvmaAkSb3UaYJKcmCSC5JcmOTVN7DNE5OsTXJekk902R5J0vyxVVc7TrIIOAZ4BLAOWJ1kZVWtHdhmD+A1wIOr6hdJbtdVeyRJ80uXPah9gAur6qKq2gCcDBwyY5vnAMdU1S8AquqnHbZHkjSPdJmgdgYuHlhe164btCewZ5Izk3w9yYGz7SjJEUnWJFmzfv36jporSeqTcRdJbAXsASwHDgOOT7L9zI2q6riqWlZVyxYvXjziJkqSxqHLBHUJsOvA8i7tukHrgJVVdVVV/RD4Hk3CkiRNuC4T1GpgjyS7J9kGOBRYOWObz9D0nkiyI82Q30UdtkmSNE90lqCq6mrgSOA04HxgRVWdl+TNSR7bbnYa8LMka4HTgVdW1c+6apMkaf7orMwcoKpOBU6dse4NA88LeFn7kCTpOuMukpAkaVYmKElSL5mgJEm9ZIKSJPWSCUqS1EsmKElSL5mgJEm9ZIKSJPWSCUqS1EsmKElSL5mgJEm9ZIKSJPWSCUqStFmmpqZIwqpVq1i1ahVJSMLU1NRQ46S5oPj8sWzZslqzZs24myFJGpIkZ1XVspnr7UFJknrJBCVJ6iUTlCSpl0xQkqReMkFJknrJBCVJ6iUTlCSpl0xQkqRemncn6iZZD/xoC3ezI3DFEJpjbGP3Ma6xjT3fYu9WVYtnrpx3CWoYkqyZ7axlYxt7IcQ1trEXSmyH+CRJvWSCkiT10qQmqOOMbewFHNfYxl4QsSdyDkqS1H+T2oOSJPWcCUqS1EsmKHUmyU5JMu52SJqfJipBJVk0prj3SHJAktuOOO6fJXnJKGMOxH4U8Glg13HE1+TxYGjhmYgElWRPgKq6ZtRJKsmjgU8CRwEnJdlpRHEfCbwFWDuKeLPEfgdwe+Dlo44/m1F/eSXZbpTxZsTeNck2SW7eLo/s73xcsZPsC+w3ilgbaUOmf8/G8Pu2TZJtRhlzIPbyJE9M8uRh73vBJ6gkBwNnJ/kEjDZJJVkOvBd4dlX9ObABuOcI4u4HfBQ4oqq+kOTWSXZLcrMRxH448AHgKcAewN2TPKTruLO0Y9+21/oAgKqqUX1ptL3HI5NsO4p4M2I/BvgccDTw4SR3raprR5EoxhW7/bw/AvyuyzibaMMhwAnACUn2rxGWRyd5AvAJ4D+SPCbJDiOM/VCaA/A7Ai9L8oEkdxhagKpasA/g5sDngSOAE4GPDby2aATx7w48tH2+E3Ap8Bngn4C/oC3z7yDuXYF1wCHAbYHTgVOBFV3GbWM/Ctivfb498H7g+e1yZ3FntOHRwPdpzs/4NPChgdc6bUMb+9vA8lle6/JzD81w6neA5cAS4BXAT4B7tNvcZAHGfmAbZ/rv7Bbtv9t1GXdGG+4NfBc4CHgecC7w5Om2dBx7T+B84EHAYcBK4CXAnUcQO8A7gaPa5W2BjwPvA5ZMb7NFMbp+E+N+AHcAbkFzQcNTBpPUiNvxWuB17fPDgZOBxR3GuzdwUZuonkPTW34mzdHObUbwfm/S/nsgcBmw94g+50XtZ/tX7fKtgDOBUwa26erAYC/ghzQ9V2gODu46+N47TlKL2qS883Sc9svqEmDPEXz2x7V/byOJDSxtE+Engb2B3Wh6EscC/wLs0fVn3u7/UcBnBpYPpOlJHtYud5Yk2wR9xsDyg4DjgRcDtxzB//mh7ec9nZBu1v79fWAY+1/wQ3xVdWlVXVlVVwDPBbZL8jGAJPdLcrcRteNtVfXW9vmJNF+cnRUQVNW3gYOBv6uq46vq2qo6AdiBpjveqaq6tv338zRfXAe3Q/Sd/s5V1TXAtwaWf11VDwaWJPmndl1Xwy/b0XwxXZvkQOBTwJuBdyd5f1exk9ylHcrcHrg18JTpOFX1Xpph5r9Jsu2whznbQpyjkmxN8zt9+Chit8N6HwK+DKyimeM9E/g6zVDbN4Gjk9yyw//vaWcDv26HlW/S/s6/D3h9kv2m/xaGaWC4/hvAD9s5oK2q6mvAh2l68g8adtw29q5JbtrOs34NuCVwryTbVdVvgWcA+yZ57JbGWvAJalBV/YwmSV2V5Ls0XyBXdh135h9mO2a8hGbIrzNVtbaqjp4RdzHNkMgofRt4DM2R5ND/WOEPhTCtS4BXJRlMxI8DbpvkHl3FrqqzaIY47gEcQ9NjP5Sm53r3JPt3EPtg4N+AdwFvauO/IMlrBjZbAfwe+P0wv6wHC3Gq6irg1cDzkryqy9gDRTh7AE+oqmNpksSbqup9VfUNmjnYS9rYQzcwx7msqi4H/ht4EnDHJIuq6nPAB2mG1Icd+1HACwcKcc4C9gH+JMnWVfWfNL2YZyfZasixp+cZ309zILCBpgf7EmD/JLevqv8FvgRcs8UBu+4C9vFBc7Q1smGngbg3BZ4FnAfcc4RxQ/MluZZ2TmAMn/kKYGlH+z4Y+C1w8sC6twAXA3ccWHcysM8IYu8DPG7GdicCDxxy7P1o5h/u2y4fB7yVZpjtx8DrgLvQDCmvAXYYcuzLpz9PmiH0rYD70RwAvYxmfmSosYGHAxfSHARsA3wRuD+wNbDtwHZPAc4Atu/g921wjnMl8O52/bHAu4H92+UjgWM6iH29OU6aKYw3Av/IH4a2nwacxJDm2pl9nvGv27+xnWkOQE9qH2+nmVrY4qHdof7HzYcHzRDXF4B7jSH21jQTqXcdcdy0v1R3G8N77nr8f2YhzCcHXntL+8f8XJo5wLXA7h3G/sTAa9sNPH8CsJrmpmzDfO/70QypTS8vBv5f+/xONEe4H6A5wh7qwRizF+J8AXhP+7l/nKYHsXqYsZm9COcFg79rNHOuZ9PBwRizz3F+HTi+XX498LH29+I84N5DjD3bHOdetAd+wFPb2Ge0ieS+Hbz3mXOcL6O5geztaQ6MDgHeMKzvuIm8WGySbatqbCWpGq62rPXXNFVExwJXVdVh7WuPo6mgvD/wnqo6t+PYv6+qpwy8/nSaI+lndBB7EXDzqvp1+/z2wGeBg6rqJ0l2oxnmunlV/WqYsdv496apktyGZnjxQ8CzaQp0/q6qLk6yQ1X9ooPYN6mmhP1AmoODR1TVd9rzr94IfLiqzh923Db2q4BLq+qjA+v+Ezizql7ZlnnfE/hhVa0bYtz704zAfJPm4OAVwM9oDkxWV9Wr2u32Bi6vqp8OKe5daA7sL6I94Kmqdw68/hqa4dbnV9Vwh1SHfYThw8c4HzRHlf9K25OiGQrabcSxP9Yu3x14AXCnEcTeimao50vt8lNpTmfYruO4ewFHzlh3GnC/9nnnpxbQFKK8hnY4iw6q5hgYrmo/23O5/vDxjjRzgXt1HPvBNEN5P6ApaZ8eevsS8JAOYh8MnENTiHI08Fia+bbXDGyzlKZnNfT/64kqktDCV38ohPldkguAf2cYk7WbF/uqgdifrqqLRhD76qq6Erg4ydtp5lmPrmbCusu4sxXi7EjTc6Pab7COTRfhpI051EKcgZP9T273/zGanuOZ04U41VQJb6A5SOgy9pk0RQmvqKpjq3ExzVzQhiHH3g/4e+DpVXUATU95H5qh5ecneV3bu1pOM/e4/TDjw4RV8WkytF8W59CUXD+uhjjMspmxn1BVI6mYbEv4twH2pykQOLSqvjOK2APxn0kz//S0airbRqKq/pWmInaXYe+7HTI8EngpzUHPJ9uYr6cZWvxskucmeS1wL2B9h7Gnr4bzDZo5runtnkAzUtDF79o7qmr6tI3X0sxrXUqTlO5EMwf1Ipoh7OEP5Y7mAEcanXYOYAXw8qo6Z1Jit/EPp5mPOG/EcQMcAFxWVd8dZdyue2nOcY5njhNMUFqgxlkIM+bYnX9hT7I0dyQ4DthQVYe159VdWVU/GmHs/62qpya5O/BQ4PNdDyO351NtC/x7VT0syVNpeusv7XIY2QQlSZshyY40czP70ZReLx/VMPKM2AEOGNUwchv/RJqhxEfSnOLQ6TDyUM8ylqSFrqquSHIOzUmzjxj1HOeM2COb46Q5j3P/9t+HVdX3u45rgpKkzdDOMx4EPHKUhSjjjN0OG29I8haaOc7OkxM4xCdJm805zhHFM0FJkvrI86AkSb1kgpIk9ZIJSpLUSyYoSVIvmaCkVpJK8g8Dy69IMrUZP3/TJF9McnaSJ21hW+6Q5JQ5bDfrHaGTnJhk6HdzlUbJBCX9we+Bx7dn698Y9wWoqvtU1ae2pCFVdWlVjSXBDPs24dKNZYKS/uBqmmudHbWxjZLcJslnkpyT5OtJ7pXkdjR3M31A24O684yfOSPJO5J8I8n3kuzfrl+U5O+TrG7399x2/dIk57bPb5ZkRZK1ST6d5L+SLBvY99uSfLtty5KBsA9PsqaNd3C77bZJPpzkO0m+leSh7frDk6xM8mXgS0lun+Qr7Xs5d7q90iiZoKTrOwZ4SpJbb2SbNwHfqqp7AX8DnFTN3UufDXy17UH9YJaf26qq9qG5fcIb23XPAn5VVQ8AHgA8J8nuM37uBcAvqmovmluK33/gtZsDX6+qewNfobnd+bSlNPfveQxwbJJtgRfSXBhgb+Aw4CPtemju6fMX7b1/ngycVlX3oblD7tkb+TykTpigpAFV9WvgJODFG9nsT4CPttt/GbhtklvNYff/1v57Fk3ygOaim09LcjbwXzR35d1jlnjTN6w7l+Z+U9M2AP8xy34BVlTVte1laS4C7tbu62Ptvr4L/AjYs93+C1X18/b5auAZ7Rzc3lX1P3N4f9JQmaCkP/Yemp7NzYe839+3/17DH66DGeBFba/rPlW1e1X9/83Y51UDl54Z3C/AzMvEbOqyMb+5bsOqrwAPobnfz4lJnrYZbZKGwgQlzdD2IlbQJKnZfJXmrrUkWQ5c0fa8bozTaG6fvXW7vz3bO6kOOhN4Yvv6XsDec9z3Xya5STsfdifgghlt3xO4Y7v+etqb0V1eVccD/0wz/CeNlNU60uz+geZOpbOZAk5ob3vwW+DpWxDnn2mG5b7Z3tJgPfDnM7b5AM1c0Vrgu8B5wFzuYPpj4BvArYDnVdXvknwA+GCS79AUhRxeVb9vQl/PcuCVSa4CrgTsQWnkvFis1HPt7ba3bhPMnYEvAnetqg1jbprUKXtQUv/dDDi9HQYM8AKTkyaBPShJUi9ZJCFJ6iUTlCSpl0xQkqReMkFJknrJBCVJ6qX/A3GruaJi2hfGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Accuracy with 99% confidence:\")\n",
    "cie.plot_util(confidence[99], x_ticks=neigbhors, ylim=None, xlabel=\"N of neighbors\", ylabel=\"Accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deterministic Resampling\n",
    "The following example shows the usage of the $\\texttt{random_seed}$ parameter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating model on 100 samples of (X, y).\n",
      "Each sample will contain 398 elements from (X, y), out of which 318 elements\n",
      "will be used to train the model, while the remaining 80 elements to test its accuracy.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5cf04bae6d1c4e20aca89466f181c217",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Here the random_seed param is set.\n",
    "\n",
    "accs_run_1=cie.get_accuracy_samples(GaussianNB, cancer_X, cancer_y,\n",
    "                             n_iterations=100, sample_ratio=0.7,\n",
    "                             train_ratio=0.8, random_seed=42,\n",
    "                             sample_with_replacement=True, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0.95), (1, 0.9), (2, 0.8875), (3, 0.9625), (4, 0.9625), (5, 0.8625)]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(k, accs_run_1[k]) for k in range(6)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the given data, with 95.0% confidence the accuracy of the model is in 0.934 +/-0.053.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.93421875, 0.05328125000000006)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cie.nonparametric_confidence_interval(accs_run_1, .95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating model on 100 samples of (X, y).\n",
      "Each sample will contain 398 elements from (X, y), out of which 318 elements\n",
      "will be used to train the model, while the remaining 80 elements to test its accuracy.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa2ca98efe504a57b4877c61a16fb044",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "accs_run_2=cie.get_accuracy_samples(GaussianNB, cancer_X, cancer_y,\n",
    "                             n_iterations=100, sample_ratio=0.7,\n",
    "                             train_ratio=0.8, random_seed=42,\n",
    "                             sample_with_replacement=True, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0.95), (1, 0.9), (2, 0.8875), (3, 0.9625), (4, 0.9625), (5, 0.8625)]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# As you can see, the results are identical to the previous run of the function. Be aware that the results\n",
    "# also depend on the model's characteristic. As an example, if the random seed is set, but the model has \n",
    "# some sort of random initialization (i.e. Neural Networks), you may see different results although the\n",
    "# resampling of the data was identical.\n",
    "\n",
    "[(k, accs_run_2[k]) for k in range(6)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the given data, with 95.0% confidence the accuracy of the model is in 0.934 +/-0.053.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.93421875, 0.05328125000000006)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cie.nonparametric_confidence_interval(accs_run_2, .95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating model on 100 samples of (X, y).\n",
      "Each sample will contain 398 elements from (X, y), out of which 318 elements\n",
      "will be used to train the model, while the remaining 80 elements to test its accuracy.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e1f7695f73a4abdb8461f71f9778f91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Here the random_seed param is set to None\n",
    "\n",
    "accs_run_3=cie.get_accuracy_samples(GaussianNB, cancer_X, cancer_y,\n",
    "                             n_iterations=100, sample_ratio=0.7,\n",
    "                             train_ratio=0.8, random_seed=None,\n",
    "                             sample_with_replacement=True, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0.9125), (1, 0.95), (2, 0.95), (3, 0.9), (4, 0.975), (5, 0.95)]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(k, accs_run_3[k]) for k in range(6)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the given data, with 95.0% confidence the accuracy of the model is in 0.931 +/-0.056.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.93125, 0.05625000000000002)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cie.nonparametric_confidence_interval(accs_run_3, .95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating model on 100 samples of (X, y).\n",
      "Each sample will contain 398 elements from (X, y), out of which 318 elements\n",
      "will be used to train the model, while the remaining 80 elements to test its accuracy.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "235976c91fa64376958dfb83c592adaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "accs_run_4=cie.get_accuracy_samples(GaussianNB, cancer_X, cancer_y, \n",
    "                             n_iterations=100, sample_ratio=0.7,\n",
    "                             train_ratio=0.8, random_seed=None,\n",
    "                             sample_with_replacement=True, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0.975), (1, 0.975), (2, 0.9625), (3, 0.925), (4, 0.95), (5, 1.0)]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(k, accs_run_4[k]) for k in range(6)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From the given data, with 95.0% confidence the accuracy of the model is in 0.937 +/-0.057.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9374999999999999, 0.05656249999999996)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cie.nonparametric_confidence_interval(accs_run_4, .95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}