Kaggle/Flower Classification with TPUs/bug-report-unavailableerror-socket-closed.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This notebook is just to reproduce the `UnavailableError: Socket closed` bug.\n",
    "\n",
    "#### Usually happens when using some combination of the below:\n",
    "- Long epochs\n",
    "- Heavy models\n",
    "- Some loops using too much memory\n",
    "\n",
    "##### Let me know if anyone else is getting similar issues."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-input": true,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "!pip install --quiet efficientnet\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import os, re, math, warnings, time\n",
    "from matplotlib import pyplot as plt\n",
    "from kaggle_datasets import KaggleDatasets\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "from tensorflow.keras.models import load_model\n",
    "from tensorflow.keras import optimizers, applications, Sequential, layers, metrics, losses\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\n",
    "import efficientnet.tfkeras as efn\n",
    "\n",
    "def seed_everything(seed=0):\n",
    "    np.random.seed(seed)\n",
    "    tf.random.set_seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n",
    "\n",
    "seed = 0\n",
    "seed_everything(seed)\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TPU configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on TPU  grpc://10.0.0.2:8470\n",
      "REPLICAS:  8\n"
     ]
    }
   ],
   "source": [
    "# TPU or GPU detection\n",
    "# Detect hardware, return appropriate distribution strategy\n",
    "try:\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "    print('Running on TPU ', tpu.master())\n",
    "except ValueError:\n",
    "    tpu = None\n",
    "\n",
    "if tpu:\n",
    "    tf.config.experimental_connect_to_cluster(tpu)\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "else:\n",
    "    strategy = tf.distribute.get_strategy()\n",
    "\n",
    "print(\"REPLICAS: \", strategy.num_replicas_in_sync)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 16 * strategy.num_replicas_in_sync\n",
    "EPOCHS = 15\n",
    "LEARNING_RATE = 3e-5 * strategy.num_replicas_in_sync\n",
    "HEIGHT = 512\n",
    "WIDTH = 512\n",
    "CHANNELS = 3\n",
    "N_CLASSES = 104\n",
    "N_FOLDS = 5\n",
    "FOLDS_USED = 5\n",
    "\n",
    "GCS_PATH = KaggleDatasets().get_gcs_path() + '/tfrecords-jpeg-%sx%s' % (HEIGHT, WIDTH)\n",
    "\n",
    "TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec') + tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\n",
    "TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')\n",
    "\n",
    "CLASSES = [\n",
    "    'pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', \n",
    "    'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', \n",
    "    'globe thistle', 'snapdragon', \"colt's foot\", 'king protea', 'spear thistle', \n",
    "    'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', \n",
    "    'balloon flower', 'giant white arum lily', 'fire lily', 'pincushion flower', \n",
    "    'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', \n",
    "    'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william', \n",
    "    'carnation', 'garden phlox', 'love in the mist', 'cosmos',  'alpine sea holly', \n",
    "    'ruby-lipped cattleya', 'cape flower', 'great masterwort',  'siam tulip', \n",
    "    'lenten rose', 'barberton daisy', 'daffodil',  'sword lily', 'poinsettia', \n",
    "    'bolero deep blue',  'wallflower', 'marigold', 'buttercup', 'daisy', \n",
    "    'common dandelion', 'petunia', 'wild pansy', 'primula',  'sunflower', \n",
    "    'lilac hibiscus', 'bishop of llandaff', 'gaura',  'geranium', 'orange dahlia', \n",
    "    'pink-yellow dahlia', 'cautleya spicata',  'japanese anemone', 'black-eyed susan', \n",
    "    'silverbush', 'californian poppy',  'osteospermum', 'spring crocus', 'iris', \n",
    "    'windflower',  'tree poppy', 'gazania', 'azalea', 'water lily',  'rose', \n",
    "    'thorn apple', 'morning glory', 'passion flower',  'lotus', 'toad lily', \n",
    "    'anthurium', 'frangipani',  'clematis', 'hibiscus', 'columbine', 'desert-rose', \n",
    "    'tree mallow', 'magnolia', 'cyclamen ', 'watercress',  'canna lily', \n",
    "    'hippeastrum ', 'bee balm', 'pink quill',  'foxglove', 'bougainvillea', \n",
    "    'camellia', 'mallow',  'mexican petunia',  'bromelia', 'blanket flower', \n",
    "    'trumpet creeper',  'blackberry lily', 'common tulip', 'wild rose']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_kg_hide-input": true,
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "# Datasets utility functions\n",
    "AUTO = tf.data.experimental.AUTOTUNE # instructs the API to read from multiple files if available.\n",
    "\n",
    "def decode_image(image_data):\n",
    "    image = tf.image.decode_jpeg(image_data, channels=3)\n",
    "    image = tf.cast(image, tf.float32) / 255.0\n",
    "    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n",
    "    return image\n",
    "\n",
    "def read_labeled_tfrecord(example):\n",
    "    LABELED_TFREC_FORMAT = {\n",
    "        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n",
    "        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n",
    "    }\n",
    "    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n",
    "    image = decode_image(example['image'])\n",
    "    label = tf.cast(example['class'], tf.int32)\n",
    "    return image, label\n",
    "\n",
    "def read_unlabeled_tfrecord(example):\n",
    "    UNLABELED_TFREC_FORMAT = {\n",
    "        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n",
    "        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n",
    "        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n",
    "    }\n",
    "    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n",
    "    image = decode_image(example['image'])\n",
    "    idnum = example['id']\n",
    "    return image, idnum # returns a dataset of image(s)\n",
    "\n",
    "def load_dataset(filenames, labeled=True, ordered=False):\n",
    "    ignore_order = tf.data.Options()\n",
    "    if not ordered:\n",
    "        ignore_order.experimental_deterministic = False # disable order, increase speed\n",
    "\n",
    "    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n",
    "    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n",
    "    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n",
    "    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n",
    "    return dataset\n",
    "\n",
    "def data_augment(image, label):\n",
    "    p_spatial = tf.random.uniform([1], minval=0, maxval=1, dtype='float32', seed=seed)\n",
    "    p_spatial2 = tf.random.uniform([1], minval=0, maxval=1, dtype='float32', seed=seed)\n",
    "    p_pixel = tf.random.uniform([1], minval=0, maxval=1, dtype='float32', seed=seed)\n",
    "    p_crop = tf.random.uniform([1], minval=0, maxval=1, dtype='float32', seed=seed)\n",
    "    \n",
    "    ### Spatial-level transforms\n",
    "    if p_spatial >= .2: # flips\n",
    "        image = tf.image.random_flip_left_right(image, seed=seed)\n",
    "        image = tf.image.random_flip_up_down(image, seed=seed)\n",
    "        \n",
    "    if p_crop >= .7: # crops\n",
    "        if p_crop >= .95:\n",
    "            image = tf.image.random_crop(image, size=[int(HEIGHT*.6), int(WIDTH*.6), CHANNELS], seed=seed)\n",
    "        elif p_crop >= .85:\n",
    "            image = tf.image.random_crop(image, size=[int(HEIGHT*.7), int(WIDTH*.7), CHANNELS], seed=seed)\n",
    "        elif p_crop >= .8:\n",
    "            image = tf.image.random_crop(image, size=[int(HEIGHT*.8), int(WIDTH*.8), CHANNELS], seed=seed)\n",
    "        else:\n",
    "            image = tf.image.random_crop(image, size=[int(HEIGHT*.9), int(WIDTH*.9), CHANNELS], seed=seed)\n",
    "        image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n",
    "    \n",
    "    ## Pixel-level transforms\n",
    "    if p_pixel >= .4: # pixel transformations\n",
    "        if p_pixel >= .85:\n",
    "            image = tf.image.random_saturation(image, lower=0, upper=2, seed=seed)\n",
    "        elif p_pixel >= .65:\n",
    "            image = tf.image.random_contrast(image, lower=.8, upper=2, seed=seed)\n",
    "        elif p_pixel >= .5:\n",
    "            image = tf.image.random_brightness(image, max_delta=.2, seed=seed)\n",
    "        else:\n",
    "            image = tf.image.adjust_gamma(image, gamma=.6)\n",
    "\n",
    "    return image, label\n",
    "\n",
    "def get_training_dataset(filenames):\n",
    "    dataset = load_dataset(filenames, labeled=True)\n",
    "    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n",
    "    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n",
    "    dataset = dataset.shuffle(2048)\n",
    "#     dataset = dataset.batch(BATCH_SIZE, drop_remainder=True) # slighly faster with fixed tensor sizes\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n",
    "    return dataset\n",
    "\n",
    "def get_validation_dataset(filenames, ordered=True, repeated=False):\n",
    "    dataset = load_dataset(filenames, labeled=True, ordered=ordered)\n",
    "    if repeated:\n",
    "        dataset = dataset.repeat()\n",
    "        dataset = dataset.shuffle(2048)\n",
    "#     dataset = dataset.batch(BATCH_SIZE, drop_remainder=repeated)\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    dataset = dataset.cache()\n",
    "    dataset = dataset.prefetch(AUTO)\n",
    "    return dataset\n",
    "\n",
    "def get_test_dataset(filenames=TEST_FILENAMES, ordered=True):\n",
    "    dataset = load_dataset(filenames, labeled=False, ordered=ordered)\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    dataset = dataset.prefetch(AUTO)\n",
    "    return dataset\n",
    "\n",
    "def count_data_items(filenames):\n",
    "    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n",
    "    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n",
    "    return np.sum(n)\n",
    "\n",
    "def int_div_round_up(a, b):\n",
    "    return (a + b - 1) // b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# # Visualization utility functions\n",
    "# np.set_printoptions(threshold=15, linewidth=80)\n",
    "\n",
    "# def plot_metrics(history, metric_list):\n",
    "#     fig, axes = plt.subplots(len(metric_list), 1, sharex='col', figsize=(24, 12))\n",
    "#     axes = axes.flatten()\n",
    "    \n",
    "#     for index, metric in enumerate(metric_list):\n",
    "#         axes[index].plot(history[metric], label='Train %s' % metric)\n",
    "#         axes[index].plot(history['val_%s' % metric], label='Validation %s' % metric)\n",
    "#         axes[index].legend(loc='best', fontsize=16)\n",
    "#         axes[index].set_title(metric)\n",
    "\n",
    "#     plt.xlabel('Epochs', fontsize=16)\n",
    "#     sns.despine()\n",
    "#     plt.show()\n",
    "\n",
    "    \n",
    "# def dataset_to_numpy_util(dataset, N):\n",
    "#     dataset = dataset.unbatch().batch(N)\n",
    "#     for images, labels in dataset:\n",
    "#         numpy_images = images.numpy()\n",
    "#         numpy_labels = labels.numpy()\n",
    "#         break;  \n",
    "#     return numpy_images, numpy_labels\n",
    "\n",
    "# def title_from_label_and_target(label, correct_label):\n",
    "#     label = np.argmax(label, axis=-1)\n",
    "#     correct = (label == correct_label)\n",
    "#     return \"{} [{}{}{}]\".format(CLASSES[label], str(correct), ', shoud be ' if not correct else '',\n",
    "#                                 CLASSES[correct_label] if not correct else ''), correct\n",
    "\n",
    "# def display_one_flower_eval(image, title, subplot, red=False):\n",
    "#     plt.subplot(subplot)\n",
    "#     plt.axis('off')\n",
    "#     plt.imshow(image)\n",
    "#     plt.title(title, fontsize=14, color='red' if red else 'black')\n",
    "#     return subplot+1\n",
    "\n",
    "# def display_9_images_with_predictions(images, predictions, labels):\n",
    "#     subplot=331\n",
    "#     plt.figure(figsize=(13,13))\n",
    "#     for i, image in enumerate(images):\n",
    "#         title, correct = title_from_label_and_target(predictions[i], labels[i])\n",
    "#         subplot = display_one_flower_eval(image, title, subplot, not correct)\n",
    "#         if i >= 8:\n",
    "#             break;\n",
    "              \n",
    "#     plt.tight_layout()\n",
    "#     plt.subplots_adjust(wspace=0.1, hspace=0.1)\n",
    "#     plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## About the datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of test images 7382\n"
     ]
    }
   ],
   "source": [
    "# # Train data\n",
    "# NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\n",
    "# train_dataset = get_training_dataset(TRAINING_FILENAMES)\n",
    "# y_train = next(iter(train_dataset.unbatch().map(lambda image, label: label).batch(NUM_TRAINING_IMAGES))).numpy()\n",
    "# print('Number of training images %d' % NUM_TRAINING_IMAGES)\n",
    "\n",
    "# Test data\n",
    "NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n",
    "print('Number of test images %d' % NUM_TEST_IMAGES)\n",
    "test_dataset = get_test_dataset(ordered=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(input_shape, N_CLASSES):\n",
    "    base_model = efn.EfficientNetB0(weights='noisy-student', \n",
    "                                    include_top=False,\n",
    "                                    input_shape=input_shape)\n",
    "\n",
    "    model = tf.keras.Sequential([\n",
    "            base_model,\n",
    "            layers.GlobalAveragePooling2D(),\n",
    "            layers.Dense(N_CLASSES, activation='softmax')\n",
    "            ])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning rate schedule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate schedule: 1e-08 to 0.00024 to 2.15e-05\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABX0AAAFpCAYAAADA54teAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd4lFXC/vH7zKRRE0gBAgkJXVoooUsTC6hgR6wQURT7umt73y2+u++uu+quqKuCCgGxICpqRBA7JaEklNCkhCRASAgJJYSSOuf3B7P7Y31BAkKeZPL9XFcuJ8+c58w9/oFyz5lzjLVWAAAAAAAAAADf4HI6AAAAAAAAAADg/KH0BQAAAAAAAAAfQukLAAAAAAAAAD6E0hcAAAAAAAAAfAilLwAAAAAAAAD4EEpfAAAAAAAAAPAhlL4AAAAAAAAA4EMofQEAAAAAAADAh1Sp9DXGjDTGbDXGZBhjnjrF84HGmA+8z680xsSc9NzT3utbjTFXnGlOY8y73usbjTEzjDH+3uvDjDFFxph13p/f/5I3DgAAAAAAAAC+6IylrzHGLelVSaMkdZZ0izGm80+GTZR00FrbTtKLkv7mvbezpHGSukgaKek1Y4z7DHO+K6mTpG6S6km6+6TXWWqt7eH9+eO5vGEAAAAAAAAA8GV+VRjTV1KGtTZTkowxcyRdI2nzSWOukfSM9/FHkv5pjDHe63OstaWSsowxGd75dLo5rbUL/jWpMWaVpFbn+N4UFhZmY2JizvV2AAAAAAAAAKgRVq9eXWitDa/K2KqUvi0l7T7p9xxJ/U43xlpbYYwpkhTqvb7iJ/e29D7+2Tm92zrcIemRky4PMMakS8qV9Btr7aafCx4TE6O0tLSfGwIAAAAAAAAANZ4xZmdVx1al9DWnuGarOOZ010+1rcRP53xN0hJr7VLv72sktbbWHjHGXCnpU0nt/09YYyZJmiRJ0dHRp3gZAAAAAAAAAPBdVTnILUdS1Em/t9KJlbanHGOM8ZMULOnAz9z7s3MaY/4gKVzSY/+6Zq09bK094n28QJK/MSbsp2GttW9Ya+OttfHh4VVa7QwAAAAAAAAAPqMqpW+qpPbGmFhjTIBOHMyW9JMxSZLGex/fKOk7a631Xh9njAk0xsTqxMrcVT83pzHmbklXSLrFWuv51wsYY5p79wmWMaavN/v+c3nTAAAAAAAAAOCrzri9g3eP3gclLZLkljTDWrvJGPNHSWnW2iRJ0yXN9h7UdkAnSlx5x83ViUPfKiQ9YK2tlKRTzel9yamSdkpa7u1451lr/6gTZfJkY0yFpOOSxnmLZQAAAAAAAACAl/Hl3jQ+Pt5ykBsAAAAAAACA2s4Ys9paG1+VsVXZ3gEAAAAAAAAAUEtQ+gIAAAAAAACAD6H0BQAAAAAAAAAfQukLAAAAAAAAAD6E0hcAAAAAAAAAfAilLwAAAAAAAAD4EEpfAPBReUXHte9widMxAAAAAABANaP0BQAftKPgiEa9tFTDX/hB76/aJWut05EAAAAAAEA1ofQFAB9TUFyqCYmr5Ocy6tYqWE/P26AJianaW8SqXwAAAAAA6gJKXwDwIcfKKjRxVqoKi8s0fXwfvXd3f/3PmC5ambVfl7+4WJ+szWHVLwAAAAAAPo7SFwB8REWlRw++t1Yb9xTpn7f2VFxUiFwuo/EDY7TwkSFq36yRfvVBuu57Z7UKj5Q6HRcAAAAAAFwglL4A4AOstfrdZ5v03ZZ9+tO1XTXiomb/8XxsWAPNvXeAnh7VSd9vKdDlLy7RlxvzHEoLAAAAAAAuJEpfAPABr/2wQ++v2qX7h7XVbf1an3KM22V079C2mv/wxYoMCdJ976zRo3PWquhYeTWnBQAAAAAAFxKlLwDUcp+szdHzi7bq2h6RevyKjmcc36FZI31y/yA9eml7zV+fp8unLNb3W/dVQ1IAAAAAAFAdKH0BoBZLySjUEx+t14A2oXruxjgZY6p0n7/bpUcv7aBPHxik4Hr+SkhM1VMfr1dxCat+AQAAAACo7Sh9AaCW2rL3sO6dvVqxYQ009Y7eCvA7+z/Su7YM1ucPXaz7hrbV3LTdGjllqVJ2FF6AtAAAAAAAoLpQ+gJALZRXdFwTZqSqfqBbMxP6Krie/znPFejn1lOjOunD+wbI321065sr9UzSJh0vqzyPiQEAAAAAQHWh9AWAWuZwSbkSElN1pLRCMxP6KjKk3nmZt3frplrwyGBNGBijmSnZuvLlpVq98+B5mRsAAAAAAFQfSl8AqEXKKjya/M5qZew7otdv76WLWjQ+r/PXD/DTM2O66L27+6mswqObpqborwu3qLSCVb8AAAAAANQWlL4AUEtYa/XUx+uVnLFff72huwa3D79grzWwXZi+fHSwxsZHaeriHRrzSrI27im6YK8HAAAAAADOH0pfAKgl/v7VNs1bu0e/vqyDbuzd6oK/XqMgf/31hu5KnNBHB4+V6dpXk/XSN9tVXum54K8NAAAAAADOHaUvANQC763cpX9+n6FxfaL04CXtqvW1h3eK0Fe/GqKrurfQi99s0/WvpWh7fnG1ZgAAAAAAAFVH6QsANdz3W/bpd59t1LCO4frfa7vKGFPtGULqB+ilcT312m29tOfQcV31yjK9sWSHKj222rMAAAAAAICfR+kLADXY+pxDuv/dNbqoRSO9emsv+bmd/WP7ym4ttOjRIRrWIVx/WbBFN09bruzCo45mAgAAAAAA/4nSFwBqqN0HjumumakKbRigGRP6qEGgn9ORJEnhjQI17Y7e+sfYOG3NL9aol5bq7eXZ8rDqFwAAAACAGoHSFwBqoINHyzQ+cZXKK61mJvRVRKMgpyP9B2OMru/VSl/9aoj6xDbV7z/bpDtmrNSeQ8edjgYAAAAAQJ1H6QsANUxJeaXueTtNOQeP680749UuoqHTkU6rRXA9zUroo79c101rdx3SyBeXaG7ablnLql8AAAAAAJxC6QsANYjHY/XY3HVK23lQ/xgbp76xTZ2OdEbGGN3aL1pfPjJEF0U21hMfrdfds9K073CJ09EAAAAAAKiTKH0BoAb584IftWDDXv32qot0dfdIp+OclejQ+ppzT3/97urOWpZRqMunLNHn6blOxwIAAAAAoM6h9AWAGmLGsixNX5alCQNjNPHiWKfjnBOXy2jixbFa8MhgxYQ20EPvr9UD767RgaNlTkcDAAAAAKDOoPQFgBpg4YY8/emLzbqiSzP97urOMsY4HekXaRveUB/dN0CPX9FRX23eq8tfXKyvN+c7HQsAAAAAgDqB0hcAHJaWfUCPfrBOPaNC9NK4nnK7anfh+y9+bpceGN5OSQ9erPBGQbrn7TT9em66io6XOx0NAAAAAACfRukLAA7aUXBEd7+dpsiQenprfB8F+budjnTeXdSisT57YJAeuqSdPl23RyOnLNGSbQVOxwIAAAAAwGdR+gKAQwqKSzUhcZXcxmhmQh81bRDgdKQLJsDPpV9f3lHzJg9U/QC37pyxSv/9yQYdLa1wOhoAAAAAAD6H0hcAHHCsrEITZ6WqoLhU0yf0UevQBk5HqhZxUSH64uHBumdwrN5btUujXlqqlZn7nY4FAAAAAIBPofQFgGpWUenRQ++t1cY9RfrnLb3UIyrE6UjVKsjfrf++qrM+mDRAkjTuzRX63/mbVVJe6XAyAAAAAAB8A6UvAFQja61+n7RJ327Zp/+5pqsu7dzM6UiO6RvbVAsfGazb+kXrrWVZuurlpVq3+5DTsQAAAAAAqPUofQGgGr32ww69t3KXJg9rqzv6t3Y6juMaBPrpf6/tptkT++pYWaVueD1Ff/9qq8oqPE5HAwAAAACg1qL0BYBq8unaPXp+0VZd0yNSj1/e0ek4Ncrg9uH68tEhuq5nS73yXYaueTVZP+YddjoWAAAAAAC1EqUvAFSDlIxCPf5Ruvq3aarnbuwul8s4HanGCa7nrxduitObd8aroLhUY/65TK9+n6GKSlb9AgAAAABwNih9AeAC27L3sO6dvVqxYQ007Y54Bfq5nY5Uo13WuZm++tUQXd65uZ5ftFU3TF2ujH1HnI4FAAAAAECtQekLABdQXtFxJSSmqn6gW4kJfRVcz9/pSLVC0wYBevW2Xnrllp7auf+ornp5qaYvy5LHY52OBgAAAABAjUfpCwAXSHFJuRISU1VcUqHECX3VMqSe05FqndFxkfrq0SG6uF2Y/jR/s255c4V2HzjmdCwAAAAAAGo0Sl8AuADKKjya/M4aZew7otdv76XOkY2djlRrRTQO0lvj4/Xcjd21KfewrpiyRO+t3CVrWfULAAAAAMCpUPoCwHlmrdVT89ZrWUahnr2+mwa3D3c6Uq1njNHY+Cgt+tUQ9YwO0X99skHjE1OVV3Tc6WgAAAAAANQ4lL4AcJ794+ttmrdmjx67rINuio9yOo5PaRlST7Pv6qc/XdNFqVkHdPmLSzRvTQ6rfgEAAAAAOAmlLwCcR++v2qVXvsvQuD5ReuiSdk7H8Ukul9EdA2K08JHB6tiskR6bm657Z69WQXGp09EAAAAAAKgRKH0B4Dz5fus+/fbTjRraIVx/urarjDFOR/JpMWEN9MG9A/RfV3bSD9sKdMWUJVqwIc/pWAAAAAAAOI7SFwDOgw05RXrg3TXq1LyRXr2tl/zd/PFaHdwuo0lD2uqLhy5Wy5B6uv/dNXr4/bU6dKzM6WgAAAAAADiGVgIAfqHdB44pYWaqmtQPUOKEPmoY6Od0pDqnfbNGmnf/QD12WQct2JCny19cou+25DsdCwAAAAAAR1D6AsAvcOhYmcYnrlJ5pUez7uqjiMZBTkeqs/zdLj08or0+fWCQmtQP0F0z0/TkR+tVXFLudDQAAAAAAKoVpS8AnKOS8krd83aacg4c15t3xqtdRCOnI0FS15bBSnpokCYPa6sPV+/WyClLlZJR6HQsAAAAAACqDaUvAJwDj8fq13PTlZp9UH8fG6e+sU2djoSTBPq59eTITvpo8kAF+rl061sr9UzSJh0vq3Q6GgAAAAAAF1yVSl9jzEhjzFZjTIYx5qlTPB9ojPnA+/xKY0zMSc897b2+1RhzxZnmNMa8672+0Rgzwxjj771ujDEve8evN8b0+iVvHAB+ib8s+FFfbMjTf195kUbHRTodB6fRK7qJvnh4sBIGxWhmSraufHmpVu884HQsAAAAAAAuqDOWvsYYt6RXJY2S1FnSLcaYzj8ZNlHSQWttO0kvSvqb997OksZJ6iJppKTXjDHuM8z5rqROkrpJqifpbu/1UZLae38mSXr9XN4wAPxSiclZemtZliYMjNHdg2OdjoMzqBfg1h9Gd9F79/RTWYVHN01drr8u3KLSClb9AgAAAAB8U1VW+vaVlGGtzbTWlkmaI+man4y5RtIs7+OPJI0wxhjv9TnW2lJrbZakDO98p53TWrvAeklaJanVSa/xtvepFZJCjDEtzvF9A8A5+XJjnv44f7Ou6NJMv7u6s078UYfaYGDbMC361RDd3CdKUxfv0OhXlmnjniKnYwEAAAAAcN5VpfRtKWn3Sb/neK+dcoy1tkJSkaTQn7n3jHN6t3W4Q9KXZ5EDAC6Y1TsP6JE569QjKkQvjespt4vCt7ZpGOinZ6/vrsSEPio6Xq5rX03WlG+2qbzS43Q0AAAAAADOm6qUvqdqNWwVx5zt9ZO9JmmJtXbpWeSQMWaSMSbNGJNWUFBwilsA4OxlFhzR3bPSFBlST9PH91GQv9vpSPgFhneM0FePDtXouEhN+Wa7rnstWdvyi52OBQAAAADAeVGV0jdHUtRJv7eSlHu6McYYP0nBkg78zL0/O6cx5g+SwiU9dpY5ZK19w1obb62NDw8Pr8LbA4CfV3ikVBMSU+UyRjMT+qhpgwCnI+E8CK7vrxdv7qGpt/dS3qESXf3yMk1bvEOVnv/zeSIAAAAAALVKVUrfVEntjTGxxpgAnTiYLeknY5Ikjfc+vlHSd949eZMkjTPGBBpjYnXiELZVPzenMeZuSVdIusVa6/nJa9xpTugvqcham3cO7xkAquxYWYUmzkzVvuISTZ/QR61DGzgdCefZyK4ttOhXQzS8U7ieXbhFY6ctV1bhUadjAQAAAABwzs5Y+nr36H1Q0iJJP0qaa63dZIz5ozFmjHfYdEmhxpgMnVid+5T33k2S5krarBN78z5gra083ZzeuaZKaiZpuTFmnTHm997rCyRl6sRhcG9Kuv+XvXUA+HkVlR499N5abdhTpFdu6aUeUSFOR8IFEtYwUFNv760pN/fQ9vxijXppiWalZMvDql8AAAAAQC1kTizI9U3x8fE2LS3N6RgAaiFrrX776Ua9u3KX/nRNF90xIMbpSKgme4tK9OTH67V4W4EGtg3Vczd2V6sm9Z2OBQAAAACo44wxq6218VUZW5XtHQCgznl98Q69u3KX7hvalsK3jmkeHKSZCX307PXdlL77kEZOWaq5qbvlyx+SAgAAAAB8C6UvAPzEZ+v26Lkvt2pMXKSeuKKj03HgAGOMbukbrS8fHaKuLRvriY/X666Zqco/XOJ0NAAAAAAAzojSFwBOkrKjUL/5MF392zTV8zd1l8tlnI4EB0U1ra/37u6vP4zurJQd+3X5i0v02bo9rPoFAAAAANRolL4A4LV1b7HufXu1YkIbaNod8Qr0czsdCTWAy2WUMChWCx4ZrDbhDfTInHV64L012n+k1OloAAAAAACcEqUvAOjE4V0TElepXoBbM+/qq+B6/k5HQg3TNryhPrx3gJ4Y2VFfb87XsOd/0KvfZ+h4WaXT0QAAAAAA+A+UvgDqvOKSck1IXKXDx8uVmNBHLUPqOR0JNZSf26X7h7XTwkcGq1+bpnp+0VYNf+EHfZC6S5UetnwAAAAAANQMlL4A6rSyCo8mv7NGGfuO6PXbe6tLZLDTkVALtItopLfG99EHk/qreXCQnvx4g0ZOWaJvf8xnv18AAAAAgOMofQHUWdZaPTVvvZZlFOrZ67tpSIdwpyOhlunXJlSf3D9Qr9/WSxUeq4mz0nTzGyu0dtdBp6MBAAAAAOowSl8AddaLX2/TvDV79KtLO+im+Cin46CWMsZoVLcW+upXQ/Sna7sqs+CIrnstRZPfWa3MgiNOxwMAAAAA1EF+TgcAACfMWbVLL3+XoZvjo/TwiHZOx4EP8He7dEf/1rquZ0u9uSRTby7N1Neb83VL32g9PKK9whsFOh0RAAAAAFBHGF/eezA+Pt6mpaU5HQNADfP91n26e1aaBrUL0/Tx8fJ386UHnH/7ikv08rfb9f6q3Qryc+meIW10z+A2ahDI560AAAAAgLNnjFltrY2v0lhKXwB1yYacIt38xnLFhjXQB/cOUEMKOFxgmQVH9PyirVq4ca/CGgbqkUvba1yfKD5sAAAAAACclbMpffkbJ4A6Y/eBY0qYmaom9QOUOKEPhS+qRZvwhnr99t6ad/9AtQlroN99ulFXvLhECzfkyZc/eAUAAAAAOIfSF0CdcOhYmSYkrlJZRaVm3dVHEY2DnI6EOqZXdBN9cG9/vXVnvNwuo8nvrtH1r6doVdYBp6MBAAAAAHwMpS8An1dSXqlJb6/W7gPH9ead8WoX0cjpSKijjDG6tHMzLXxksP52QzflHjqusdOW6+5ZqdqeX+x0PAAAAACAj6D0BeDTPB6rX3+YrlXZB/TC2Dj1axPqdCRAfm6Xbu4TrR9+M1yPX9FRKzMP6IopS/TUx+uVf7jE6XgAAAAAgFqO0heAT3t24Y/6Yn2e/uvKThoTF+l0HOA/1Atw64Hh7bT4ieEaPzBGH6/J0dDnv9fzi7bocEm50/EAAAAAALUUpS8AnzUzOUtvLs3S+AGtdc/gNk7HAU6raYMA/WF0F3372DBd3rm5Xv1+h4Y+971mLMtSaUWl0/EAAAAAALUMpS8An/Tlxr36n/mbdXnnZvr96C4yxjgdCTij6ND6evmWnvr8wYvVObKx/jh/sy79x2J9tm6PPB7rdDwAAAAAQC1B6QvA56zeeVCPzFmrHlEhemlcT7ldFL6oXbq1CtY7E/tp1l191TDQX4/MWadrXk1Wckah09EAAAAAALUApS8An5JZcER3z0pVi+AgvXVnvOoFuJ2OBJwTY4yGdgjXFw9drH+MjdOBo2W67a2VunPGKm3OPex0PAAAAABADUbpC8BnFB4p1YTEVBljNDOhr0IbBjodCfjFXC6j63u10re/Hqr/vvIipe8+pKteWarH5q7TnkPHnY4HAAAAAKiBKH0B+IRjZRWaOCtN+4pLNH18vGLCGjgdCTivgvzdumdIGy15fLgmDWmj+evzNPyFH/SXBT/q0LEyp+MBAAAAAGoQSl8AtV5FpUcPv79WG3IO6eVxPdUzuonTkYALJri+v54edZG+/80wje4eqTeXZmrIc99r2uIdKimvdDoeAAAAAKAGoPQFUKtZa/XM55v0zY/79MyYLrq8S3OnIwHVomVIPf19bJwWPDxYvVo30bMLt+iSF37QR6tzVOmxTscDAAAAADiI0hdArTZ1cabeWbFL9w5tozsHxDgdB6h2F7VorJkJffXePf0U1ihQv/kwXVe9vFTfb90nayl/AQAAAKAuovQFUGt9tm6P/vblFo2Oi9STV3RyOg7gqIFtw/Tp/YP0yi09daysUgmJqbr1zZVan3PI6WgAAAAAgGpG6QugVkrZUajffJiufrFN9cJN3eVyGacjAY5zuYxGx0Xqm8eG6pnRnbU1v1hj/pmsh95fq537jzodDwAAAABQTYwvf/UzPj7epqWlOR0DwHm2dW+xbpyaouaNg/TRfQMVXN/f6UhAjVRcUq43lmTqzaWZqvRY3davtR66pJ1CGwY6HQ0AAAAAcJaMMauttfFVGkvpC6A2yT9couteTVaFx2re/QPVqkl9pyMBNV7+4RJN+Wa75qbtVj1/t+4b2kZ3XRyr+gF+TkcDAAAAAFTR2ZS+bO8AoNYoLinXhMRUFR0vV2JCHwpfoIqaNQ7Ss9d306JHh2hg21C98NU2DXv+B72/apcqKj1OxwMAAAAAnGeUvgBqhfJKj+5/d4225Rfrtdt7q0tksNORgFqnXURDvXFnvD66b4BaNamnp+dt0MiXluqrTXvly9/8AQAAAIC6htIXQI1nrdVTH2/Q0u2Fevb6bhraIdzpSECtFh/TVB9PHqipt/eWx1pNmr1aY6ct1+qdB52OBgAAAAA4Dyh9AdR4L36zXR+vydGjl7bX2Pgop+MAPsEYo5Fdm+urR4foz9d1Vfb+Y7rh9RTdOztNOwqOOB0PAAAAAPALUPoCqNE+SN2ll7/drrHxrfTIiPZOxwF8jp/bpdv6tdYPvxmmxy7roGXbC3X5i0v0359s0L7iEqfjAQAAAADOgfHlPfzi4+NtWlqa0zEAnKMftu7TxFlpGtQuTNPHx8vfzedUwIVWeKRUr3y7Xe+u3KUAP5fuHtxGk4a0UcNAP6ejAQAAAECdZoxZba2Nr9JYSl8ANdHGPUUaO225YkIbaO59AyicgGqWXXhUzy/aqi825CmsYYAeHtFet/SN5sMXAAAAAHDI2ZS+/M0NQI2z+8AxJcxMVZP6AUpM6EPhCzggJqyBXr2tlz59YJDahjfU7z/bpMv+sVhfrM+TL39gDAAAAAC+gNIXQI1SdKxcCTNTVVpeqZkJfdSscZDTkYA6rUdUiOZM6q8ZE+IV6OfWA++t0bWvpWhF5n6nowEAAAAAToPSF0CNUVJeqXtmp2nX/mN64854tW/WyOlIACQZY3RJp2Za8MhgPXdjd+07XKJxb6zQXTNTtXVvsdPxAAAAAAA/QekLoEbweKx+/WG6VmUd0Atj49S/TajTkQD8hNtlNDY+St//ZpieHNlJqdkHNOqlJXrio3TlFR13Oh4AAAAAwIvSF0CN8Ncvt+iL9Xl6elQnjYmLdDoOgJ8R5O/W5GFtteTx4bprUKw+XZurYc//oL99uUVFx8udjgcAAAAAdR6lLwDHzUrJ1htLMnXngNaaNKSN03EAVFGTBgH67dWd9e2vh2pU1+Z6/YcdGvr893praaZKKyqdjgcAAAAAdRalLwBHLdq0V898vkmXdW6mP4zuImOM05EAnKWopvU1ZVxPzX/oYnVrGaz//eJHjfj7Yn26do88Hut0PAAAAACocyh9AThm9c6Devj9tYprFaKXx/WU20XhC9RmXVsGa/bEfpo9sa+C6/nr0Q/WafQ/l2np9gKnowEAAABAnULpC8ARWYVHdfesVDUPDtL08fGqF+B2OhKA82Rw+3B9/uDFmnJzDxUdL9cd01fpjukrtXFPkdPRAAAAAKBOoPQFUO0Kj5RqQuIqGWM0K6GvQhsGOh0JwHnmchld27Olvv31UP32qou0YU+Rrn5lmR6ds1a7DxxzOh4AAAAA+DRKXwDV6nhZpSbOSlP+4RK9NT5eMWENnI4E4AIK9HPr7sFttPjx4Zo8rK0WbtyrEX9frD/N36yDR8ucjgcAAAAAPonSF0C1qfRYPfT+Wq3POaSXxvVUr+gmTkcCUE2C6/nryZGd9MPjw3Rtz0glJmdpyPPf6/UfdqikvNLpeAAAAADgUyh9AVQLa62eSdqkb37M1zOju+iKLs2djgTAAS2C6+m5G+O08JEh6hvTVH/7couGv/CD5qbtVqXHOh0PAAAAAHwCpS+AajFtSaZmr9ipe4e00fiBMU7HAeCwjs0bafqEPpozqb8iGgfpiY/W68qXluq7LfmylvIXAAAAAH4JSl8AF9xn6/borwu3aHRcpJ4c2cnpOABqkP5tQvXp/QP16q29VFpRqbtmpum611L02bo9Kq/0OB0PAAAAAGol48uraeLj421aWprTMYA6bfmO/Ro/Y5V6RIdo9sS+CvRzOx0JQA1VXunRB6m7NX1ZlrIKj6p54yDdMaC1bukbraYNApyOBwAAAACOMsasttbGV2VslVb6GmNGGmO2GmMyjDFPneL5QGPMB97nVxpjYk567mnv9a3GmCvONKcx5kHvNWuMCTvp+jBjTJExZp335/dVyQ7AOdvyizVpdpqiQ+vrzTviKXwB/Cx/t0u392+tbx8bqhkT4tW+WUM9v2irBjz7rZ78aL227D3sdEQAAAAAqBX8zjTAGOOW9Kr6wLG/AAAgAElEQVSkyyTlSEo1xiRZazefNGyipIPW2nbGmHGS/ibpZmNMZ0njJHWRFCnpG2NMB+89p5szWdJ8ST+cIs5Sa+3V5/A+AVSz/MMlmjBjlYL83ZqZ0EfB9f2djgSglnC5jC7p1EyXdGqm7fnFSkzJ1rw1OfogbbcGtg1VwqBYXdIpQm6XcToqAAAAANRIVVnp21dShrU201pbJmmOpGt+MuYaSbO8jz+SNMIYY7zX51hrS621WZIyvPOddk5r7VprbfYvfF8AHHSktEIJiakqOl6uxAl91KpJfacjAail2jdrpL9c100rnh6hJ0d2UlbhUd3zdpqGv/CDZizLUnFJudMRAQAAAKDGqUrp21LS7pN+z/FeO+UYa22FpCJJoT9zb1XmPJUBxph0Y8xCY0yXKowHUM3KKz2a/M5qbc0v1mu391bXlsFORwLgA0LqB2jysLZa+sRwvXprL4U3CtQf52/WgGe/0zNJm5RdeNTpiAAAAABQY5xxewdJp/ru5E9PfzvdmNNdP1XZfKYT5dZIam2tPWKMuVLSp5La/3SQMWaSpEmSFB0dfYYpAZxP1lo9PW+Dlm4v1HM3dNfQDuFORwLgY/zcLl3VvYWu6t5C63MOKTE5W++u3KlZy7M1olOEEgbFamDbUJ34whEAAAAA1E1VWembIynqpN9bSco93RhjjJ+kYEkHfubeqsz5H6y1h621R7yPF0jyP/mgt5PGvWGtjbfWxoeHUzgB1WnKN9v10eocPTKivcb2iTrzDQDwC3RvFaIXb+6h5Ccv0UPD22ntrkO67a2VGjllqeas2qWS8kqnIwIAAACAI6pS+qZKam+MiTXGBOjEwWxJPxmTJGm89/GNkr6z1lrv9XHGmEBjTKxOrMxdVcU5/4Mxprl3n2AZY/p6s++vypsEcOHNTd2tl77drpt6t9Kjl/6fRfgAcMFENA7SY5d3VPJTl+j5G7vL5TJ6at4GDXj2Wz335RblFR13OiIAAAAAVKszbu9gra0wxjwoaZEkt6QZ1tpNxpg/Skqz1iZJmi5ptjEmQydW+I7z3rvJGDNX0mZJFZIesNZWStKp5vRef1jSE5KaS1pvjFlgrb1bJ8rkycaYCknHJY3zFssAHLZ4W4Ge/mSDBrcP01+u78bXqgE4IsjfrZvio3Rj71ZamXVAiclZmrp4h95YkqlR3VooYVCMekU3cTomAAAAAFxwxpd70/j4eJuWluZ0DMCnbdxTpJunLVd0aAPNvbe/GgX5Ox0JAP5t94FjmpWSrQ/Sdqu4pEJxUSG6a1CMRnVtoQC/qnzhCQAAAABqBmPMamttfJXGUvoCOFc5B4/putdS5O8y+uSBQWrWOMjpSABwSkdLK/TxmhzNTM5WZuFRNWscqDv6t9YtfaMV2jDQ6XgAAAAAcEaUvl6UvsCFU3SsXDdMTVH+4RJ9PHmgOjRr5HQkADgjj8dq8bYCzUjO0tLthQrwc+m6Hi2VcHGMOjVv7HQ8AAAAADitsyl9z7inLwD8VGlFpe6ZnaZd+49p1l19KXwB1Boul9HwThEa3ilC2/OLlZiSrXlrcvRB2m4NaBOqhEExGnFRM7ld7E0OAAAAoPZipS+As+LxWD08Z63mr8/TS+N66JoeLZ2OBAC/yKFjZZqTultvp2Qrt6hE0U3ra/zAGI2Nb8U+5QAAAABqDLZ38KL0Bc6/Zxf8qGlLMvXUqE66b2hbp+MAwHlTUenRok35SkzOUtrOg2oQ4NZN8VEaPzBGsWENnI4HAAAAoI5jewcAF8Tby7M1bUmm7ujfWvcOaeN0HAA4r/zcLl3VvYWu6t5CG3KKlJicpXdX7tSs5dm6pGOEEgbFalC7UBnD1g8AAAAAajZW+gKokkWb9uq+d1ZrRKdmmnZHb/a7BFAn7DtcondW7tJ7K3eq8EiZOjRrqIRBsbq2R0vVC3A7HQ8AAABAHcL2Dl6UvsD5sWbXQd3yxgp1atFYc+7pT9EBoM4pKa/U/PV5mrEsS5vzDiukvr9u6RutOwe0Vovgek7HAwAAAFAHUPp6UfoCv1x24VFd/3qKGgX56ePJAxXWMNDpSADgGGutVmUdUGJytr7avFfGGI3q2lwJg2LVKzqErR8AAAAAXDDs6QvgvNh/pFTjE1dJkmYm9KXwBVDnGWPUr02o+rUJ1e4Dx/T28mzNSd2t+evzFNcqWHddHKtRXVsowM/ldFQAAAAAdRgrfQGc0vGySt3y5gr9mHdY70/qr17RTZyOBAA10tHSCs1bk6PE5GxlFh5VRKNA3dG/tW7tF61QPiwDAAAAcJ6wvYMXpS9wbio9Vve9s1rf/Jiv12/rrZFdmzsdCQBqPI/HavH2AiUmZ2vJtgIF+Ll0bY9IJQyK1UUtGjsdDwAAAEAtx/YOAM6ZtVb/8/kmfb05X/8zpguFLwBUkctlNLxjhIZ3jFDGvmIlJmdr3po9mpuWo/5tmiphUKwuvaiZ3C72/QUAAABwYbHSF8B/mLZ4h55duEWThrTRf115kdNxAKBWKzpWrjmpu/T28p3ac+i4oprW0/gBMRrbJ0qNg/ydjgcAAACgFmF7By9KX+DsJKXn6uH31+rq7i308riecrEaDQDOi4pKj77anK/E5CylZh9UgwC3boqP0viBMYoNa+B0PAAAAAC1AKWvF6UvUHUrMvfrzumr1CM6RG/f1VdB/m6nIwGAT9qQU6TE5Cx9vj5XFR6r4R0jlDAoRhe3C5MxfNgGAAAA4NQofb0ofYGq2Z5frBteT1F4o0B9PHmgQuoHOB0JAHzevuISvbtil95duVOFR8rUoVlDTRgYq+t6tlS9AD54AwAAAPCfKH29KH2BM8s/XKLrX0tRWaVH8yYPVFTT+k5HAoA6pbSiUp+n5ykxOUubcg8rpL6/xvWJ1p0DWisypJ7T8QAAAADUEJS+XpS+wM87UlqhsVOXK3v/Uc29d4C6tgx2OhIA1FnWWq3KOqDE5Gx9tXmvjDEa2bW57hoUo17RTdj6AQAAAKjjzqb09bvQYQDUTOWVHt3/7hptzS/W9PHxFL4A4DBjjPq1CVW/NqHafeCYZq/YqfdX7dIX6/PUvVWw7hoUqyu7tVCAn8vpqAAAAABqOFb6AnWQtVZPfLReH67O0d9u6Kab+0Q7HQkAcApHSys0b02OElOylVlwVBGNAnV7/9a6tV+0whoGOh0PAAAAQDViewcvSl/g1KZ8s01Tvtmuh0e012OXdXA6DgDgDDweqyXbCzQjOVtLthUowM+la+IilTAoVp0jGzsdDwAAAEA1YHsHAKc1N223pnyzXTf2bqVfXdre6TgAgCpwuYyGdYzQsI4RythXrJkp2fp49R59uDpH/WKb6q6LY3XpRc3kdrHvLwAAAABW+gJ1yuJtBZo4M1UD2oZqxoQ+8nezLyQA1FZFx8o1J3WX3l6+U3sOHVdU03oaPyBGY/tEqXGQv9PxAAAAAJxnbO/gRekL/H8b9xTp5mnLFR3aQHPv7a9GFAIA4BMqKj36enO+ZiRnKTX7oOoHuHVT71YaPzBGbcIbOh0PAAAAwHlC6etF6QucsOfQcV33arL8XEbz7h+k5sFBTkcCAFwAG/cUaUZyluan56ms0qPhHcOVMChWg9uHyRi2fgAAAABqM0pfL0pf4MTXf2+cmqK9h0v08eSB6tCskdORAAAX2L7iEr27YpfeXblThUfK1D6ioSYMitG1PVqqQSBHOgAAAAC1EaWvF6Uv6rrSikrdOX2V1uw6qFl39dXAtmFORwIAVKPSikrNT8/TjOQsbco9rHr+bl3auZnGxEVqSIcwBfq5nY4IAAAAoIrOpvRlqQfgozweq998uF4rsw7opXE9KHwBoA4K9HPrht6tdH2vllq986A+WbtHCzbk6fP0XDUO8tOori00pkek+rcJldvF9g8AAACAr2ClL+Cjnl34o6YtztSTIztp8rC2TscBANQQ5ZUeLcso1OfrcrVo014dLatUeKNAXdXtRAHcMyqE/X8BAACAGoiVvkAd9/bybE1bnKnb+0frvqFtnI4DAKhB/N0uDe8YoeEdI1RSXqnvtuxT0rpcvbdql2amZCuqaT2N7h6pMT0i1al5Y6fjAgAAADgHrPQFfMxXm/bqvndW65JOEZp6e2/5uV1ORwIA1AKHS8r11aZ8JaXnKjmjUJUeqw7NGmpMXKRGx0WqdWgDpyMCAAAAdRoHuXlR+qKuWbvroG55c4U6Nmuk9yf1V/0AFvMDAM5e4ZFSLdyQp6T0XKVmH5QkxUWFaExcpK7u3kLNGgc5nBAAAACoeyh9vSh9UZdkFx7V9a+nqGGgn+bdP1BhDQOdjgQA8AF7Dh3X5+m5SlqXq815h2WM1D82VGN6RGpU1+YKqR/gdEQAAACgTqD09aL0RV2x/0ipbng9RUXHyzXv/kGKDeMruACA8y9j3xF9np6rz9NzlVl4VP5uoyHtwzWmR6QuvaiZGgTyDRMAAADgQqH09aL0RV1wvKxSt761QptzD+u9e/qrd+smTkcCAPg4a6025R5WkrcAzisqUT1/t0ZcFKExcZEa2jFcgX5up2MCAAAAPoXS14vSF76u0mN13zur9c2P+Xr9tt4a2bW505EAAHWMx2OVmn1ASem5WrAhTwePlatxkJ9Gdm2uMXEtNaBtqNwu43RMAAAAoNaj9PWi9IUvs9bqmaRNmrV8p54Z3VkTBsU6HQkAUMeVV3q0LKNQn6/L1aJNe3W0rFJhDQN1dfcWGh0XqV7RITKGAhgAAAA4F2dT+rLxGlBLvbk0U7OW79Q9g2MpfAEANYK/26XhHSM0vGOESsor9d2WfUpal6v3Vu3SzJRstWpST6PjIjUmLlKdmjeiAAYAAAAuEFb6ArVQUnquHn5/ra7q3kKvjOspF1+bBQDUYIdLyvXVpnwlpecqOaNQlR6r9hENNSYuUmN6RKp1KAeQAgAAAGfC9g5elL7wRSsy9+vO6avUIypEb0/sqyB/DsoBANQe+4+UasGGPCWl5yo1+6AkKa5VsEbHRWp0XKSaNQ5yOCEAAABQM1H6elH6wtdszy/WDa+nKLxRoD6ePFAh9QOcjgQAwDnbc+i45qfnKik9V5tyD8sYqV9sU42Ja6lRXZurSQP+OwcAAAD8C6WvF6UvfMm+wyW67rUUlVV6NG/yQEU1re90JAAAzpsdBUeUtC5Xn6fnKrPwqPxcRkM6hGtMXKQu69xMDQI5igIAAAB1G6WvF6UvfMWR0grdPG25sgqPau69A9S1ZbDTkQAAuCCstdqUe1hJ6ScK4LyiEgX5uzTiomYaExepYR3DFejH1kYAAACoeyh9vSh94QvKKz2aOCtNyRmFemt8vIZ3jHA6EgAA1cLjsUrbeVBJ6Xu0YMNeHThapkZBfhrZpbnG9IjUgDah8nO7nI4JAAAAVAtKXy9KX9R21lo9+fF6zU3L0V+v76ZxfaOdjgQAgCPKKz1KzihUUnquvtqUryOlFQprGKCrurXQmB6R6hXdRMYYp2MCAAAAF8zZlL5sjgbUYC9/m6G5aTl6+JJ2FL4AgDrN3+3SsI4RGtYxQiXllfp+yz4lpefq/dTdmrV8p1qG1NPouEiNiYvURS0aUQADAACgTmOlL1BDfZi2W49/tF439GqlF27qzl9eAQA4heKScn21KV9J6blallGoSo9Vu4iGGuMtgGPCGjgdEQAAADgv2N7Bi9IXtdWSbQW6a2aq+rcJ1YwJfRTgx36FAACcyf4jpVqwca8+X5erVdkHJEndWwVrTFykru4eqebBQQ4nBAAAAM4dpa8XpS9qo025RRo7dbmimtbXh/cNUKMgf6cjAQBQ6+QeOq7563OVlJ6rjXsOyxipb0xTjekRqSu7tlCTBgFORwQAAADOCqWvF6Uvaps9h47ruleT5XYZfXL/IFYkAQBwHuwoOKLP008UwJkFR+XnMhrcPkxjekTqss7N1TCQYy4AAABQ81H6elH6ojYpOlauG6emaO/hEn1030B1bN7I6UgAAPgUa6025R7W5+m5+jw9V7lFJQryd2lEp2YaHRepYR3DFeTvdjomAAAAcEpnU/pWaaNQY8xIY8xWY0yGMeapUzwfaIz5wPv8SmNMzEnPPe29vtUYc8WZ5jTGPOi9Zo0xYSddN8aYl73PrTfG9KpKdqA2KK2o1KTZacref1TT7uhN4QsAwAVgjFHXlsF6+sqLtOzJS/ThfQN0U+8orcjcr/veWa0+f/5Gv/kwXUu3F6ii0uN0XAAAAOCcnfG7bMYYt6RXJV0mKUdSqjEmyVq7+aRhEyUdtNa2M8aMk/Q3STcbYzpLGiepi6RISd8YYzp47zndnMmS5kv64SdRRklq7/3pJ+l17z+BWs3jsXr8w/VamXVAU27uoYFtw858EwAA+EVcLqM+MU3VJ6ap/jC6s5J37FfSulx9uXGvPlqdo7CGAbqqWwuN6RGpXtFNZIxxOjIAAABQZVXZwKyvpAxrbaYkGWPmSLpG0sml7zWSnvE+/kjSP82J/zO+RtIca22ppCxjTIZ3Pp1uTmvtWu+1n+a4RtLb9sR+FCuMMSHGmBbW2ryzecNATfPcoq1KSs/VEyM76tqeLZ2OAwBAnePndmloh3AN7RCuP5d31Q9b9ykpPVfvp+7WrOU71TKknkbHRWp0XAt1btGYAhgAAAA1XlVK35aSdp/0e47+7wrbf4+x1lYYY4okhXqvr/jJvf9qtc40Z1VytJRE6Ytaa/aKnZq6eIdu7x+tyUPbOh0HAIA6L8jfrZFdW2hk1xYqLinX15vzlZSeqzeXZmrq4h1qG95A1/RoqTFxkYoJa+B0XAAAAOCUqlL6nmopw09PfzvdmNNdP9Vewmc6Ua4qOWSMmSRpkiRFR0efYUrAOV9vztcfPtuoEZ0i9MzoLqwaAgCghmkU5K/re7XS9b1a6cDRMi3YkKek9Fz94+tt+sfX29S9VbDGxEXq6u6Rah4c5HRcAAAA4N+qUvrmSIo66fdWknJPMybHGOMnKVjSgTPce6Y5zyWHrLVvSHpDkuLj489UJAOOWLvroB56f426tQzWK7f2lJ+7SmcqAgAAhzRtEKDb+7fW7f1bK6/ouOan5+mz9D363y9+1J8X/KjurUI0tH2YhnYMV1yrEP7bDgAAAEeZE1vk/syAEyXuNkkjJO2RlCrpVmvtppPGPCCpm7X2Pu9Bbtdba8caY7pIek8n9vGNlPStThzEZqowZ7akeGttoff3qyQ9KOlKndgK4mVr7b/2Bz6l+Ph4m5aWVsV/FUD12Ln/qK5/LUUNAv308eSBCm8U6HQkAABwjjILjmj++jx9v3Wf0ncfksdKjYL8dHG7MA3pEK4hHcLVMqSe0zEBAADgA4wxq6218VUZe8aVvt49eh+UtEiSW9IMa+0mY8wfJaVZa5MkTZc023tQ2wFJ47z3bjLGzNWJQ98qJD1gra30hvw/c3qvPyzpCUnNJa03xiyw1t4taYFOFL4Zko5JSqjavw6g5jhwtEwTElPlsVYzE/pQ+AIAUMu1CW+oh0e018Mj2uvQsTIlZ+zX4m37tGRboRZu3CtJahfRUEPah2tIhzD1bxOqIH+3w6kBAADg68640rc2Y6UvapLjZZW69a0V2px7WO/d00+9Wzd1OhIAALhArLXavu+Ilmwr0OJtBVqZdUBlFR4F+LnUL7aphnpXAbePaMi+/gAAAKiSs1npS+kLVINKj9Xkd1br6x/z9fptvTSyawunIwEAgGp0vKxSK7P2a8m2Qi3ZXqCMfUckSS2CgzS4fZiGdojQxe3CFFzf3+GkAAAAqKnO6/YOAH4Za63+NH+zvtqcrz+M7kzhCwBAHVQvwK1hHSM0rGOEJGnPoeNasq1AS7YVaOHGvZqbliOXkeKiQrxbQYSrR1SI3C5WAQMAAODssdIXuMDeXJKpPy/4UXdfHKvfXt3Z6TgAAKCGqaj0KD3nkBZvK9TibQVan3NI1krB9fy9B8KdOBSuRTAHwgEAANRlbO/gRekLp32enquH3l+rq7q10Cu39JSL1ToAAOAMDh4t07KMwhMrgbcXKP9wqSSpQ7OG/14F3De2KQfCAQAA1DGUvl6UvnDSysz9umP6KsVFBWv2xH78xQwAAJw1a6225hd7t4Io1KqsAyqr9CjQz6V+bUI1tEO4hnYIU9twDoQDAADwdZS+XpS+cErGvmJd/1qKwhsF6uPJAxVSP8DpSAAAwAccL6vUiqz9Wrz1xCrgzIKjkqTI4CAN6XBiFfCgdmEKrseBcAAAAL6G0teL0hdO2He4RNe9lqLSCo8+uX+goprWdzoSAADwUTkHj2nJtkIt3rZPKRn7VVxaIbfLqMe/D4QLU/dWHAgHAADgCyh9vSh9Ud2OlFZo3BvLlVlwVB9MGqBurYKdjgQAAOqI8kqP1u0+5N0KokDr9xTJWimk/r8OhAvX0A7hatY4yOmoAAAAOAeUvl6UvqhO5ZUe3T0rTcsyCvXWnfEa3inC6UgAAKAOO3C0TEu3n9gLeMn2AhUUnzgQrlPzRie2gmgfrviYJpw7AAD/r707j667rvM//nrfLcvNvjRJ2zTd0g0QaEPLDuKouBwRdRQZlVYQUXEcddyOOud3RpyfjjPg/H6I/hBoAYGCqIjjOMgAiojSpgtb99I2XdI2SbPvyf38/rjfrE2alLb55t48H+f03Jvv/Xy/9x34njZ55Z33BwASBKGvh9AXE8U5p6//4lU9Wrlf//sD5+ijy2f5XRIAAEA/55y2HY5vCPfHHTWq3Fuvrt6YUsMBXehtCHf5gkLNLYiyIRwAAMAkRejrIfTFRPk/z+zU7U/v0Oevmq8vv2Oh3+UAAACcUFtXj/76Rl28C3hHjd6ojW8INyMnzRsDUaCL5xcoK5UN4QAAACaLkwl9Q2e6GCDZ/bxyv25/eoc+sHSGvvT2BX6XAwAAMKb0SEhXLSrSVYuKJEn7j7Xpj94s4N+8fEiPrKtSMGBaOqtvQ7hCnTMjWwE2hAMAAEgIdPoCp+D5HTX65Jr1unBuvu5beYEioYDfJQEAAJyS7t6YNu6r1/PePOBXDzZKkvKikf4N4S4vL9A0NoQDAACYUIx38BD64kzacqhJH/5/f9HM3DQ9dstF/PojAABISnUtnXphV63+uL1Gz++sVW3LwIZwVyws1BXlhVo2O1cpITaEAwAAOJMIfT2EvjhTDjW069q7/qyAmX712UtUnE2nCwAASH6xmNPWw039s4Ar9x1Td69TWjioi+bl6/LyAl2xcJpm56ezIRwAAMBpRujrIfTFmdDY3q2//cmLqm7o0M8/c5EWFWf5XRIAAIAvWjt79Jfddd4oiBrtrWuTJJXmpfXPAr54Xr4y+Y0oAACAU8ZGbsAZ0tnTq08/WKk9ta26f9VyAl8AADClRVNC+pslRfqbJfEN4fbVter5HTX6445aPbHpoB56qUqhgGlpWa6uWFCoy8sLddb0LDaEAwAAOMPo9AXGKRZz+uJjm/XrzYd0x0fO1bXnz/S7JAAAgEmrqyemjVX1Xghco9cPNUmS8qMRXVYe3xDusvJCFWam+FwpAABAYmC8g4fQF6fT9/97m378h936yjsX6nNvne93OQAAAAmlprlTL+yq0fM7avWnnTWqbemSJC0pydLlCwp1xYJCLSvLVSQU8LlSAACAyYnQ10Poi9PlZ3/dp2898ZquXzFL333/2WxMAgAAcApiMact1U364474LOAN++rVE3OKRrwN4bwQuCw/6nepAAAAkwahr4fQF6fD01uO6NMPVurKhdN098eXKRSk+wQAAOB0avE2hPvjjqN6fketqo7FN4Qry0/Xijl5WjorV0vLcjW/MIN5wAAAYMoi9PUQ+uJUbd7foOvu/osWFGVq7c0XKj3C3ocAAABn2t7aVj2/Mz4KYsO+Y6pv65YkZaaGdF5pTn8IfF5pjrLTwj5XCwAAMDEIfT2EvjgV++pa9YG7XlR6SlC//MwlbDICAADgA+ec9tS2amNVgzZW1WvjvnrtONKsmPdtTPm0DC8EjofB8+gGBgAASepkQl/aFoERHGvt0srV69XrnNasWk7gCwAA4BMz09zCDM0tzNCHls2UFB8H8cp+LwSuatBTWw7r0cr9kqSs1JDOm5Wr80tz6AYGAABTFqEvMExHd69uun+9Dja06+GbVmheYYbfJQEAAGCQjJSQLp5foIvnF0gauRv4/z67UzEnmUnzC+kGBgAAUwuhLzBIb8zpC2s3adP+Bt11/VJVzM7zuyQAAACMYbRu4Jf3N2jjvnptrKofsRt46ax4CHzerBxlpdINDAAAkgehL+Bxzuk7/7lFT71+RP/03iV61zklfpcEAACANykjJaRL5hfokkHdwG/UtnohcIM2VdXrP57ZKed1A/fPBvY6gucW0A0MAAASF6Ev4Ln3hT1a8+Je3XjpHH3y0jl+lwMAAIDTyMw0rzBD8woz9LcVpZKk5o5uvby/0ZsNXK/fvXZYa9cPdAOfPygEPq80R5l0AwMAgARB6AtI+s9XDum2327Vu88p1jffvdjvcgAAADABMlPDurS8QJeWx7uBYzGvG7iqXpuq6rVxX4N++MyO/m7gBdMytbQspz8MnlsQpRsYAABMSuac87uGM6aiosJVVlb6XQYmuXV7julj97ykt8zM1s9uWqHUcNDvkgAAADBJNHV0e7OBG/rD4KaOHklSdlpY53tzgZfOytW5pdl0AwMAgDPGzDY45yrGs5ZOX0xpu44261MPVGpmXpp++okKAl8AAAAMkZUa1mXlhbqsvFDSyN3Ad+wY6AZeWJSp82cNdAPPK4zKjG5gAAAwsej0xZR1tLlD1/7oRXX2xPSrz16s0rx0v0sCAABAAqIbGAAATAQ6fYExtHb26JNr1utYa5ce/fSFBL4AAAB400buBm7pD4E3VtXrjztqhnUD52rprBwtLYvPBqYbGAAAnE50+mLK6emN6aYHKvX8jhrdc0OFrlpU5DfAVmgAACAASURBVHdJAAAASHJNHd3aXNUXAjdoU1W9mr1u4Jz0sM4v9bqBy3J1bmmOMlLozwEAAEPR6QuMwjmnbz3xmv6wvUb/cu05BL4AAACYEFmpYV2+oFCXLxjoBt5d0xIPgb2O4Oe210iSAiYtKMrU0rJcbyxEjubQDQwAAE4CoS+mlDuf3aW16/fr1rfO1/UrZvldDgAAAKaoQMBUXpSp8qJMfeSC+Nelje3d2ry/QRv3xUdC/OblQ3r4pSpJUm56eGAkxKx4N3CUbmAAADAKvkrAlPH4hgP696d36APnz9CX37HA73IAAACAIbLTwrpiQaGuGNQNvKumRRv31WuTNxri2W1HJcW7gRcWZw3aJI5uYAAAMICZvpgS/rSzRqtWr9eKuXlavXK5IqGA3yUBAAAAJ214N/DmqgY1d8ZnA9MNDABAcmOmLzDIlkNN+szPNmr+tAz9+GPLCHwBAACQsE7UDdy3SdzwbuC+EHhpWa5m56fTDQwAwBRApy+S2qGGdl17159lMv3qcxerJDvN75IAAACAM6qxrVub9scD4E3DuoHzohGdX5qjpWW5Omt6lhaXZGlaZgpBMAAACYBOX0DxX31btXq92jp79fPPXETgCwAAgCkhOz2sKxdO05ULp0mSemNOu462xDuBvY7gZ7xuYEnKSQ9rUXGmFhVnxR9LsrSgKEPpEb5dBAAgUfGvOJJSV09Mtzy4QW/UtmjNquVaVJzld0kAAACAL4IB08LiTC0sztRHl8+SFO8G3na4SdsON3t/mvTzyv1q7eqVJJlJZXnpWjgsDJ6Vl65ggK5gAAAmO0JfJB3nnL76+Mv6yxt1uv3D5+qS+QV+lwQAAABMKtnpYa2Ym68Vc/P7j8ViTgfq2weFwfHHp7ccUcybCpgWDmpBUYYWFWfFA+GSeCicF4349JkAAICREPoi6fzgqe16YvMhfeWdC/WBpTP9LgcAAABICIGAaVZ+umblp+sdZxX3H+/o7tXOIy3aerhJ270w+H+2HtGjlfv710zLTNGiEq8j2Osqnj8tQymhoB+fCgAAUx6hL5LKQy/t011/2K2PLp+lz145z+9yAAAAgISXGg7qnJnZOmdm9pDjNc2d2uYFwVur42Hwmhfr1NUTkxQfKzG3IDokDF5UkqXp2alsHAcAwBlG6Iuk8czWI/r2E6/pqkXT9J1rzuILSQAAAOAMKsxMUWFmoS4rL+w/1tMb09661vh4iOr4vOBNVfX6zcuH+tdkpob6u4EXFWdpcUmmFhRlKjM17MenAQBAUjLnnN81nDEVFRWusrLS7zIwAV7e36Dr7v6ryosy9MinLlQ0hZ9nAAAAAJNFc0e3dhyJdwT3jYjYVt2s5s6e/jUzc9O8juD4vODFJZmanR9VKBjwsXIAACYPM9vgnKsYz1qSMSS8qro23Xj/ehVkRnTvDRcQ+AIAAACTTGZqWMvK8rSsLK//mHNOhxo7tK26b+O4Zm0/3KTntteo19s5LhIKqHxafOO4RYM2jivMTPHrUwEAICGQjiGh1bd2aeXqdeqJOa1ZtZwv/gAAAIAEYWaakZOmGTlpetviov7jnT292nW0xesIjv/5084a/WLjgf41+dGIFpVkamFRlhaVZGpxcZbKizKUGmbjOAAAJEJfJLCO7l7d9EClDjS06+GbVmheYYbfJQEAAAA4RSmhoM6anq2zpg/dOO5Ya1f/WIi+EREPr9unju74xnEBk2YXRIeOiCjO0szcNAUC7PcBAJhaCH2RkHpjTv+wdrM2VtXrruuXqmJ23tgnAQAAAEhYedGILp5XoIvnFfQf6405VR1r0/bDTdpaHQ+Ctxxq0u9eO6y+7WuikaAWeEFwPBCOP89OZ+M4AEDyIvRFQrrtt1v0368f1rffu0TvOqfE73IAAAAA+CAYMM0piGpOQVRXnz3wfUFbV492HGkZNC+4Sb97rVqPrKvqX1OSnapFxZlaWJylxSWZWlicqbkFGYqE2DgOAJD4CH2RcO750xta/ee9+uQlc3TjpXP8LgcAAADAJJMeCem80hydV5rTf8w5p6PNndpa3dQ/L3hrdZNe2FWr7t54W3A4aJpXmOFtGjcwIqIoK0VmjIgAACSOcYW+Zna1pP+QFJR0j3Pue8NeT5H0gKRlkuokfcQ5t9d77RuSbpTUK+nvnXNPneiaZjZH0lpJeZI2Svq4c67LzFZK+oGkg97b3umcu+fNfdpIVL99pVrf/a+tetfZxfrWexb7XQ4AAACABGFmKspKVVFWqq5cOK3/eHdvTG/UtMbnBR9u1rbqJq3bc0xPbD7UvyY7LTwwGqIkPiZiQVGmoin0UQEAJqcx/4Uys6CkH0l6u6QDktab2ZPOuS2Dlt0oqd45N9/MrpP0fUkfMbMlkq6TdJak6ZL+x8wWeOeMds3vS7rDObfWzH7iXfvH3jmPOuduPcXPGQlq/d5j+uJjm7VsVq7u+Mh5bMYAAAAA4JSFgwEtLI6Pd7hm0PHGtm5tP9I8JAx+fMMBtXb19q8py0/XwqJ4ELzYu0ZZflRBvlcBAPhsPD+WXC5pl3PuDUkys7WSrpE0OPS9RtL/8p4/LulOi//uyzWS1jrnOiXtMbNd3vU00jXNbKukqyRd762537tuX+iLKWrX0RbddH+lZuam6aefqFBqOOh3SQAAAACSWHZ6WMvn5Gn5nIFNo2Mxp4MN7UNGRGw73KT/2XpEMW/juNRwQLPzo/E/BVHNzk/3HqOMiQAATJjxhL4zJO0f9PEBSStGW+Oc6zGzRkn53vG/Djt3hvd8pGvmS2pwzvWMsF6SPmhml0vaIemLzrnB10CSOtrcoZWr1ykcNN2/arlyoxG/SwIAAAAwBQUCptK8dJXmpesdZxX3H+/o7tXOIy3adjgeBu+pbdXOo816dttRdfXG+tf1BcJlg4LgeDicrqLMVH6bEQBw2own9B3pXx03zjWjHR9pO9QTrZek30h6xDnXaWa3KN4FfNVxxZrdLOlmSZo1a9YIl0Miae3s0Y1rKlXX0qVHP32hSvPS/S4JAAAAAIZIDQd1zsxsnTMze8jx3pjToYZ27a1r1d66Nu2tbdW+ulbtrmnVc9tqjguEy/LiAXA8GB54XpxFIAwAODnjCX0PSCod9PFMSYdGWXPAzEKSsiUdG+PckY7XSsoxs5DX7du/3jlXN2j9TxWf/Xsc59zdku6WpIqKiuHhNBJIT29Mtz68Ua8fatQ9N1ToLTNzxj4JAAAAACaJ4KDO4MvKh77WG3OqbmzX3tq2eChcGw+GRwqEU0KBeHdw/8iIgbERBMIAgJGMJ/RdL6nczOZIOqj4xmzXD1vzpKQbJP1F0ockPeucc2b2pKSHzex2xTdyK5e0TvGO3uOu6Z3znHeNtd41fy1JZlbinKv23u99kra+yc8ZCcA5p2//+jU9t71G/3LtObpqUZHfJQEAAADAaRMMmGbmpmtmbrouLS8Y8lpfILyvrk17vO7gvd7zP+yoUVfPQCAcCQVUlpd+3Pzg2QVRlRAIA8CUNWbo683ovVXSU5KCku5zzr1uZv8sqdI596SkeyU96G3UdkzxEFfeuscU3/StR9LnnHO9kjTSNb23/JqktWZ2m6RN3rUl6e/N7H3edY5JWnnKnz0mrR89t0uPrNuvz711nq5fwZgOAAAAAFPH4ED4kvlDA+FYzKm6qUP7alu1p651SDD8/I4adY4QCJflRzWnIP7YN0O4JDtNQQJhAEha5lzyTkCoqKhwlZWVfpeBk/TLjQf0pcde1rXnz9DtHz6X3W0BAAAAYBxiMafDTR3euIg27atr9QLh+AiJ4YHwrLz0eHdwflRlBVHN8TaZm55DIAwAk5GZbXDOVYxn7XjGOwAT5oWdtfrq46/o4nn5+v4H30LgCwAAAADjFAiYpuekaXpOmi6eN/S1WMzpSHPHQAhc26q9XqfwC7tq1dE9KBAOBlSalzZohvDA2AgCYQBIDIS+mDS2Vjfplp9t0PxpGfrJx5cpEgr4XRIAAAAAJIVAwFSSnaaS7JED4aPNnf1jIvbUtWqft8Hcn3cPDYTDwfjmdPGu4PioiNn5fYFwqkJBvo8DgMmA0BeTQnVju1atXq+MlJBWr7pAWalhv0sCAAAAgCkhEDAVZ6eqODtVF83LH/Kac05Hmjq9ruBW7Rk0NuLF3XVq7+7tXxsOmkpz413BZfnpmlMQD4bnEAgDwIQj9IXvmjq6tfK+9Wrt7NFjt1ykkuw0v0sCAAAAAEgyGwiEL5x7fCB8tLmzf1TE3v6xEW366xt1ausaCIRDgXiH8Oz8vo3lBoLhGTlpBMIAcJoR+sJXXT0x3fLgBu2uadH9n1yuxSVZfpcEAAAAABgHM1NRVqqKslK1YoRAuKa5c1AQ3Nq/wdxLe46NGAiX5feNikjv31huRm6awgTCAHDSCH3hG+ecvvaLV/Ti7jrd/uFzdcn8Ar9LAgAAAACcBmamaVmpmpaVquVz8oa85pxTTUun9npzg/f2bS5X16r1e46pdVggPDM3LT4/OD9dM3Ljc4njG9alalpmKhvLAcAICH3hm3/7/Xb9atNB/eM7FugDS2f6XQ4AAAAAYAKYmaZlxgPbkQLh2pau/jB48NiIDfvq1dLZM2R9MGAqykxRSY4XBGenqiQ7Nf5xdppKclKVH43IjGAYwNRC6AtfPPxSlX703G59dHmpPvfW+X6XAwAAAACYBMxMhZkpKsxM0QWzjw+Emzp6VN3YruqGDh3qe2xo16HGdr1yoEFPvd6hrp7YkPNSQoF4EOyFwH1h8PRBwTCbiQNINoS+mHDPbD2ibz3xqt66sFDfueZsfuIKAAAAABiTmSk7LazstLAWFY+8H4xzTnWtXapu6NDBhvZ4QNwYD4arGzv01911OtzUoZgbel5GSqi/Q3hGjhcQZ8eD4b7H1HBwAj5LADg9CH0xoV7e36BbH96ks6Zn687rl7JDKwAAAADgtDEzFWSkqCAjRefMzB5xTU9vTEebO1Xd2K5DDR39j33B8JZDjapt6TruvLxopL9jeHrOwGNfMFyUlcqmcwAmDUJfTJiqujbdeP965WdEdO/KCkVTuP0AAAAAABMrFAx4G8GlaVnZyGs6unt1pMnrFu4Lhhs7VN3QrgP1bXppT52aO4bOFzaTpmWmDITB2WnebOHU/seCjBQF2HgOwAQgdcOEqG/t0srV69Td67T25uWalpnqd0kAAAAAAIwoNRxUWX5UZfnRUde0dPaoumEgDD7UP0aiXduqm/XstqPq6B46XzgcNBX3dQsPCoPjQXE8LM5OCzMGEcApI/TFGdfR3atPPVCpAw3teuimFZo/LcPvkgAAAAAAOCUZKSGVF2WqvChzxNedc2po69ahYWMk+jaiq9xXr8OvVKtn2IDhtHBwYMM5b57w4HESJdlp/OYsgDHxtwTOqFjM6YuPbtaGqnr96Pqlx+2+CgAAAABAMjIz5UYjyo1GdNb0kecL98acals6++cJ9z1WN7brYEOHduyoUU1Lp9ywjeeyUkP9IyoGbzhXkp2mGTlpKspOUUqIjeeAqYzQF2fUbb/dqt+9dljfes9ivfucEr/LAQAAAABg0ggGTEVZ8U3gzh9lTVdPTEeaOgaFwYPmDDd0aFNVverbuo87ryAjxesMHgiDSwZ1DE/LTFWQ+cJA0iL0xRlz7wt7dN+f92jVJbN102Vz/S4HAAAAAICEEwkFVJqXrtK89FHXtHf16pA3NqLvsW/zud01rXphZ61au3qHnBMMmIoyU+JdwjlpKsxIUWFmigoyIirITOn/OC8aUTgYONOfJoDTjNAXZ8TvXq3Wbb/doqvPKta33rPE73IAAAAAAEhaaZGg5hVmaF7hyHvoOOfU1NHTP084Pmd44PkrBxpU09yptmHBcJ+8aCQeBvcHwymDnsePT/MC4hABMTApEPritKvce0xfeHSzls7K1Q+vO49fFwEAAAAAwEdmpuy0sLLTwlpUnDXqurauHtU2d6mmpUM1zV2qbelUTXOnals6+59vqooHxO3dxwfEZlJuekSFGSkqyPRC4owUFWQODYkLMwiIgTON0Ben1e6aFt30QKVm5qTpnk9UKDXM4HgAAAAAABJBeiSkWfkhzcoffZREn9bOniFhcE1Ll2qbO1XT0qlaLyjeWFWv2uauUQPivPTIcR3D/Z3EmSn94XF+NIWGMuAkEfritKlp7tTK1esUCpjWrFqu3GjE75IAAAAAAMAZEE0JKZoSUll+dMy1rZ09x3UM17QM7STeV9WqmuZOdXTHjjvfTMqPRkYcKzF83EReNEJADIjQF6dJW1ePbrx/vWqbu7T25gvH9VNBAAAAAACQ/PoC4tkFJw6InXNq7eo9rmO4LyTuC4j31sUD4s6e4wPigPXNIB4IhEcLiQmIkcwIfXHKenpjuvXhTXrtYKN++okKnVua43dJAAAAAAAgwZiZMlJCyhhnQNzS2aPaQWHw4M7hmuYu1bR06o2aVtW2nCgg9uYMe+MkBsZLRFSYkdo/mzgvPaIAATESCKEvTolzTt/+9et6dttRfffas/W2xUV+lwQAAAAAAJKcmSkzNazM1LDmjCMgbu7s8TqHRwmJW7r0Rk2ralo61TVCQBwM2LAO4sjQkHjQ8VwCYkwChL44JXf9YbceWVelz145T3+3oszvcgAAAAAAAIYwM2WlhpWVGtbcwhOv7QuIa5o7B4XEHaodNoN499EW1TR3qqt35IC4fwZxZopy08PKTY8oJz2snLSwcqMR5aRH4s/TI8qJhpWZEpIZQTFOH0JfvGm/2nRAP3hqu95/3nR95Z0L/S4HAAAAAADglAwOiOcVZpxwrXNOTR09Q8LggXnEXkjc0qm9ta2qb+tSc0fPqNcKBkw5aeF4MJweUa73OBASh5WTNui4FySnRYKn+z8BkgShL96UP++q1Vcff0UXz8vXv37oXH4aBQAAAAAAphQzU3ZaWNlpYwfEUnxPpMb2btW3dauhrUsNbd2q9x4b2ruGHD/Y0KHXDzWpoa1b7d29o14zJRToD4D7g+FoeGgn8fAgOT2scDBwOv9TYBIi9MVJ23a4Sbc8uEFzCzL0k48vUyTEXxQAAAAAAAAnEgoGlJ+RovyMlJM6r6O7dyAYbvWC4fZBgXHbQGC8u6ZF9fviz3tibtRrZqSEvDC4LxjuC4mHdhIPDoyzUsPMKk4ghL44KdWN7Vp533pFU0JaveoCZaWG/S4JAAAAAAAgaaWGgyrODqo4O3Xc5zjn1NrVq/rWkTuJhwfG+4+1qaG9W43t3XKjZMVmUnbasPnEfYFx+uiBcTQS5DfEfUDoi3Fr6ujWqtXr1dLZo5/fcpGm56T5XRIAAAAAAACGMTNlpISUkRJSad74z+uNOTW1dw/qJO4LiY8PjGtaOrXjSIsa27vV0jn6vOJw0EYYNzF2YJwSYl7xqSD0xbh09cT0mZ9t0K6jLVqzarkWl2T5XRIAAAAAAABOo2DAlBuNKDca0RxFx31eV09MDe1davQC4vq2vufxTuJGbzRFfVuXqo616eUD8eNdPbFRr5kWDo7SPTwQGF86v+CkOqCnEkJfjMk5p6//4hX9eVed/v1vz9Wl5QV+lwQAAAAAAIBJIhIKaFpmqqZlntwIio7umBcMDwuM27tV3zooMG7r1tbDTWpsi3ch93rzih+8cTmh7ygIfTGmf//9Dv1y00F9+e0L9MFlM/0uBwAAAAAAAAnOzJQWCSotknZSI0RjMafmzh41tnWrIDNyBitMbIS+OKGHX6rSnc/t0nUXlOrWq+b7XQ4AAAAAAACmsEDAlJ0WVnZa2O9SJrWA3wVg8npu21F9+9ev6a0LC3Xb+89mp0UAAAAAAAAgARD6YkSvHGjQZx/aqCUlWbrz+qUKBblVAAAAAAAAgERAkofj7D/Wpk+uWa/8jIjuXVmhaApTQAAAAAAAAIBEQZqHIepbu3TD6nXq7nVae/Pyk9p1EQAAAAAAAID/CH3Rr6O7V596oFIH6tv10E0rNH9aht8lAQAAAAAAADhJjHeAJCkWc/rSY5u1oaped3z4PF0wO8/vkgAAAAAAAAC8CYS+kCR997+26r9ePaxvvnux3vOWEr/LAQAAAAAAAPAmEfpC972wR/e+sEcrL56tGy+d43c5AAAAAAAAAE4Boe8U97tXq/Wd327R1WcV69vvXSIz87skAAAAAAAAAKeA0HcKq9x7TP/w6GYtnZWrH153noIBAl8AAAAAAAAg0RH6TlG7a1p00wOVmp6Tpp9+okKp4aDfJQEAAAAAAAA4DQh9p6Ca5k6tXL1OQTOtWXWB8qIRv0sCAAAAAAAAcJqE/C4AE6utq0c33r9etc1dWnvzhSrLj/pdEgAAAAAAAIDTiE7fKaSnN6bPP7xJrx1s1J3Xn69zS3P8LgkAAAAAAADAaUan7xThnNM/Pfm6ntl2VLe9/2y9bXGR3yUBAAAAAAAAOAPo9J0i7vrDbj38UpU+c+U8fezCMr/LAQAAAAAAAHCGEPpOAU9sOqgfPLVd7z9vur7yjoV+lwMAAAAAAADgDCL0TXIv7qrVVx5/WRfNzde/fuhcBQLmd0kAAAAAAAAAziBC3yS27XCTPv3gBs0piOonH1+mSIj/3QAAAAAAAECyIwVMUtWN7Vq1er3SU4Jas2q5stPCfpcEAAAAAAAAYAKMK/Q1s6vNbLuZ7TKzr4/weoqZPeq9/pKZzR702je849vN7J1jXdPM5njX2OldMzLWe2Co5o5urVq9Xs0dPVq9crmm56T5XRIAAAAAAACACTJm6GtmQUk/kvQuSUskfdTMlgxbdqOkeufcfEl3SPq+d+4SSddJOkvS1ZLuMrPgGNf8vqQ7nHPlkuq9a4/6Hhiqqyemz/xso3YdbdGPP7ZUS6Zn+V0SAAAAAAAAgAk0nk7f5ZJ2OefecM51SVor6Zpha66RdL/3/HFJbzMz846vdc51Ouf2SNrlXW/Ea3rnXOVdQ9413z/Ge8DjnNPXf/mKXthVq+998C26rLzQ75IAAAAAAAAATLDxhL4zJO0f9PEB79iIa5xzPZIaJeWf4NzRjudLavCuMfy9RnsPeG5/eod+ufGgvvT2BfrQspl+lwMAAAAAAADAB+MJfUfqpnXjXHO6jo+3DpnZzWZWaWaVNTU1I5ySvNIiQX10eak+f9V8v0sBAAAAAAAA4JPQONYckFQ66OOZkg6NsuaAmYUkZUs6Nsa5Ix2vlZRjZiGvm3fw+tHeYwjn3N2S7pakioqK40LhZPbZK+fLOSemXgAAAAAAAABT13g6fddLKjezOWYWUXxjtieHrXlS0g3e8w9JetY557zj15lZipnNkVQuad1o1/TOec67hrxr/nqM98AgBL4AAAAAAADA1DZmp69zrsfMbpX0lKSgpPucc6+b2T9LqnTOPSnpXkkPmtkuxbtvr/POfd3MHpO0RVKPpM8553olaaRrem/5NUlrzew2SZu8a2u09wAAAAAAAAAADLBkbpatqKhwlZWVfpcBAAAAAAAAAKfEzDY45yrGs3Y84x0AAAAAAAAAAAmC0BcAAAAAAAAAkgihLwAAAAAAAAAkEUJfAAAAAAAAAEgihL4AAAAAAAAAkEQIfQEAAAAAAAAgiRD6AgAAAAAAAEASIfQFAAAAAAAAgCRC6AsAAAAAAAAASYTQFwAAAAAAAACSiDnn/K7hjDGzGkn7/K5jghVIqvW7COAkcd8iEXHfIhFx3yIRcd8iEXHfIhFx3yIRTbX7tsw5VziehUkd+k5FZlbpnKvwuw7gZHDfIhFx3yIRcd8iEXHfIhFx3yIRcd8iEXHfjo7xDgAAAAAAAACQRAh9AQAAAAAAACCJEPomn7v9LgB4E7hvkYi4b5GIuG+RiLhvkYi4b5GIuG+RiLhvR8FMXwAAAAAAAABIInT6AgAAAAAAAEASIfRNImZ2tZltN7NdZvZ1v+sBxmJmpWb2nJltNbPXzewLftcEjIeZBc1sk5n9p9+1AONhZjlm9riZbfP+zr3I75qAsZjZF72vD14zs0fMLNXvmoDhzOw+MztqZq8NOpZnZk+b2U7vMdfPGoHhRrlvf+B9nfCKmf3KzHL8rBEYbqT7dtBr/2hmzswK/KhtsiL0TRJmFpT0I0nvkrRE0kfNbIm/VQFj6pH0ZefcYkkXSvoc9y0SxBckbfW7COAk/Iek/3bOLZJ0rrh/McmZ2QxJfy+pwjl3tqSgpOv8rQoY0RpJVw879nVJzzjnyiU9430MTCZrdPx9+7Sks51zb5G0Q9I3JrooYAxrdPx9KzMrlfR2SVUTXdBkR+ibPJZL2uWce8M51yVpraRrfK4JOCHnXLVzbqP3vFnxEGKGv1UBJ2ZmMyW9R9I9ftcCjIeZZUm6XNK9kuSc63LONfhbFTAuIUlpZhaSlC7pkM/1AMdxzj0v6diww9dIut97fr+k909oUcAYRrpvnXO/d871eB/+VdLMCS8MOIFR/r6VpDskfVUSm5YNQ+ibPGZI2j/o4wMiPEMCMbPZks6X9JK/lQBj+qHiX1TE/C4EGKe5kmokrfbGktxjZlG/iwJOxDl3UNK/Kd61Uy2p0Tn3e3+rAsatyDlXLcWbHCRN87ke4GR9UtLv/C4CGIuZvU/SQefcy37XMhkR+iYPG+EYP+VAQjCzDEm/kPQPzrkmv+sBRmNm75V01Dm3we9agJMQkrRU0o+dc+dLahW/aoxJzpuBeo2kOZKmS4qa2cf8rQoAkp+ZfVPxMXwP+V0LcCJmli7pm5L+ye9aJitC3+RxQFLpoI9nil+BQwIws7Dige9Dzrlf+l0PMIZLJL3PzPYqPkbnKjP7mb8lAWM6IOmAc67vNykeVzwEBiazv5G0xzlX45zrlvRLSRf7XBMwXkfMrESSvMejPtcDjIuZ3SDpvZL+zjlHExkmu3mK/3D4Ze/7s5mSNppZsa9VTSKEvsljvaRyM5tjZhHFN7p40ueagBMyJduNbQAAAV1JREFUM1N8xuRW59ztftcDjMU59w3n3Ezn3GzF/5591jlH5xkmNefcYUn7zWyhd+htkrb4WBIwHlWSLjSzdO/rhbeJDQiROJ6UdIP3/AZJv/axFmBczOxqSV+T9D7nXJvf9QBjcc696pyb5pyb7X1/dkDSUu9rX4jQN2l4A9dvlfSU4l8QP+ace93fqoAxXSLp44p3S272/rzb76IAIAl9XtJDZvaKpPMk/YvP9QAn5HWmPy5po6RXFf++5W5fiwJGYGaPSPqLpIVmdsDMbpT0PUlvN7Odiu8o/z0/awSGG+W+vVNSpqSnve/LfuJrkcAwo9y3OAGjYx8AAAAAAAAAkgedvgAAAAAAAACQRAh9AQAAAAAAACCJEPoCAAAAAAAAQBIh9AUAAAAAAACAJELoCwAAAAAAAABJhNAXAAAAAAAAAJIIoS8AAAAAAAAAJBFCXwAAAAAAAABIIv8fZbcd2ugetUoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1728x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "LR_START = 0.00000001\n",
    "LR_MIN = 0.000001\n",
    "LR_MAX = LEARNING_RATE\n",
    "LR_RAMPUP_EPOCHS = 3\n",
    "LR_SUSTAIN_EPOCHS = 0\n",
    "LR_EXP_DECAY = .8\n",
    "\n",
    "@tf.function\n",
    "def lrfn(epoch):\n",
    "    if epoch < LR_RAMPUP_EPOCHS:\n",
    "        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n",
    "    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n",
    "        lr = LR_MAX\n",
    "    else:\n",
    "        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n",
    "    return lr\n",
    "    \n",
    "rng = [i for i in range(EPOCHS)]\n",
    "y = [lrfn(x) for x in rng]\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(24, 6))\n",
    "plt.plot(rng, y)\n",
    "print(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimized training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "##### FOLD 0 #####\n",
      "Downloading data from https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b0_noisy-student_notop.h5\n",
      "16703488/16696600 [==============================] - 0s 0us/step\n",
      "\n",
      "EPOCH 1/15\n",
      "time: 148.4s loss: 4.6343 sparse_categorical_accuracy: 0.0160 val_loss: 4.6362 val_sparse_categorical_accuracy: 0.0073\n",
      "LearningRate: 1e-08\n",
      "\n",
      "EPOCH 2/15\n",
      "time: 38.9s loss: 3.8180 sparse_categorical_accuracy: 0.2229 val_loss: 3.0170 val_sparse_categorical_accuracy: 0.3315\n",
      "LearningRate: 8.001e-05\n",
      "\n",
      "EPOCH 3/15\n",
      "time: 38.0s loss: 2.0872 sparse_categorical_accuracy: 0.5362 val_loss: 2.3926 val_sparse_categorical_accuracy: 0.5349\n",
      "LearningRate: 0.00016\n",
      "\n",
      "EPOCH 4/15\n",
      "time: 36.6s loss: 1.0505 sparse_categorical_accuracy: 0.7580 val_loss: 1.0426 val_sparse_categorical_accuracy: 0.7689\n",
      "LearningRate: 0.00024\n",
      "\n",
      "EPOCH 5/15\n",
      "time: 31.0s loss: 0.6469 sparse_categorical_accuracy: 0.8455 val_loss: 0.4293 val_sparse_categorical_accuracy: 0.8990\n",
      "LearningRate: 0.0001922\n",
      "\n",
      "EPOCH 6/15\n",
      "time: 27.0s loss: 0.4884 sparse_categorical_accuracy: 0.8842 val_loss: 0.3393 val_sparse_categorical_accuracy: 0.9111\n",
      "LearningRate: 0.000154\n",
      "\n",
      "EPOCH 7/15\n",
      "time: 28.3s loss: 0.4161 sparse_categorical_accuracy: 0.8996 val_loss: 0.3090 val_sparse_categorical_accuracy: 0.9211\n",
      "LearningRate: 0.0001234\n",
      "\n",
      "EPOCH 8/15\n",
      "time: 29.3s loss: 0.3554 sparse_categorical_accuracy: 0.9176 val_loss: 0.2640 val_sparse_categorical_accuracy: 0.9329\n",
      "LearningRate: 9.889e-05\n",
      "\n",
      "EPOCH 9/15\n",
      "time: 28.8s loss: 0.3068 sparse_categorical_accuracy: 0.9283 val_loss: 0.2574 val_sparse_categorical_accuracy: 0.9349\n",
      "LearningRate: 7.932e-05\n",
      "\n",
      "EPOCH 10/15\n",
      "time: 29.4s loss: 0.2887 sparse_categorical_accuracy: 0.9316 val_loss: 0.2475 val_sparse_categorical_accuracy: 0.9382\n",
      "LearningRate: 6.365e-05\n",
      "\n",
      "EPOCH 11/15\n",
      "time: 30.3s loss: 0.2650 sparse_categorical_accuracy: 0.9404 val_loss: 0.2449 val_sparse_categorical_accuracy: 0.9395\n",
      "LearningRate: 5.112e-05\n",
      "\n",
      "EPOCH 12/15\n",
      "time: 29.0s loss: 0.2543 sparse_categorical_accuracy: 0.9409 val_loss: 0.2316 val_sparse_categorical_accuracy: 0.9397\n",
      "LearningRate: 4.11e-05\n",
      "\n",
      "EPOCH 13/15\n",
      "time: 28.7s loss: 0.2425 sparse_categorical_accuracy: 0.9428 val_loss: 0.2299 val_sparse_categorical_accuracy: 0.9435\n",
      "LearningRate: 3.308e-05\n"
     ]
    },
    {
     "ename": "UnavailableError",
     "evalue": "Socket closed\nAdditional GRPC error information:\n{\"created\":\"@1584660215.390218670\",\"description\":\"Error received from peer\",\"file\":\"external/grpc/src/core/lib/surface/call.cc\",\"file_line\":1039,\"grpc_message\":\"Socket closed\",\"grpc_status\":14}",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mUnavailableError\u001b[0m                          Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-f43d1cdf3898>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m                 \u001b[0;31m# compute metrics\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m                 \u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'sparse_categorical_accuracy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_accuracy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m                 \u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_sparse_categorical_accuracy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_accuracy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m                 \u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mepoch_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py\u001b[0m in \u001b[0;36mnumpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    940\u001b[0m     \"\"\"\n\u001b[1;32m    941\u001b[0m     \u001b[0;31m# TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 942\u001b[0;31m     \u001b[0mmaybe_arr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    943\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_arr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py\u001b[0m in \u001b[0;36m_numpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    908\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    909\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 910\u001b[0;31m       \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    911\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    912\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/six.py\u001b[0m in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
      "\u001b[0;31mUnavailableError\u001b[0m: Socket closed\nAdditional GRPC error information:\n{\"created\":\"@1584660215.390218670\",\"description\":\"Error received from peer\",\"file\":\"external/grpc/src/core/lib/surface/call.cc\",\"file_line\":1039,\"grpc_message\":\"Socket closed\",\"grpc_status\":14}"
     ]
    }
   ],
   "source": [
    "model_path_list = []\n",
    "kfold = KFold(N_FOLDS, shuffle=True, random_state=seed)\n",
    "for n_fold, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n",
    "    if n_fold < FOLDS_USED:\n",
    "        print(f'##### FOLD {n_fold} #####')    \n",
    "        ### Data\n",
    "        fold_train_filenames = np.asarray(TRAINING_FILENAMES)[trn_ind]\n",
    "        fold_valid_filenames = np.asarray(TRAINING_FILENAMES)[val_ind]\n",
    "        train_size = count_data_items(fold_train_filenames)\n",
    "        validation_size = count_data_items(fold_valid_filenames)\n",
    "        \n",
    "        # distribute the datset according to the strategy\n",
    "        train_dist_ds = strategy.experimental_distribute_dataset(get_training_dataset(fold_train_filenames))\n",
    "        # Hitting End Of Dataset exceptions is a problem in this setup. Using a repeated validation set instead.\n",
    "        # This will introduce a slight inaccuracy because the validation dataset now has some repeated elements.\n",
    "        valid_dist_ds = strategy.experimental_distribute_dataset(get_validation_dataset(fold_valid_filenames, repeated=True))\n",
    "        \n",
    "        train_data_iter = iter(train_dist_ds) # the training data iterator is repeated and it is not reset\n",
    "                                              # for each validation run (same as model.fit)\n",
    "        valid_data_iter = iter(valid_dist_ds) # the validation data iterator is repeated and it is not reset\n",
    "                                              # for each validation run (different from model.fit whre the  \n",
    "                                              # recommendation is to use a non-repeating validation dataset)\n",
    "                                              # recommendation is to use a non-repeating validation dataset)\n",
    "\n",
    "        STEPS_PER_TPU_CALL = STEPS_PER_EPOCH = train_size//BATCH_SIZE\n",
    "        VALIDATION_STEPS_PER_TPU_CALL = validation_size//BATCH_SIZE\n",
    "        \n",
    "        # Step functions (must be called everytime for each fold)\n",
    "        @tf.function\n",
    "        def train_step(data_iter):\n",
    "            def train_step_fn(images, labels):\n",
    "                with tf.GradientTape() as tape:\n",
    "                    probabilities = model(images, training=True)\n",
    "                    loss = loss_fn(labels, probabilities)\n",
    "                grads = tape.gradient(loss, model.trainable_variables)\n",
    "                optimizer.apply_gradients(zip(grads, model.trainable_variables))\n",
    "                train_accuracy.update_state(labels, probabilities)\n",
    "                train_loss.update_state(loss)\n",
    "            for _ in tf.range(STEPS_PER_TPU_CALL):\n",
    "                strategy.experimental_run_v2(train_step_fn, next(data_iter))\n",
    "\n",
    "        @tf.function\n",
    "        def valid_step(data_iter):\n",
    "            def valid_step_fn(images, labels):\n",
    "                probabilities = model(images, training=False)\n",
    "                loss = loss_fn(labels, probabilities)\n",
    "                valid_accuracy.update_state(labels, probabilities)\n",
    "                valid_loss.update_state(loss)\n",
    "            for _ in tf.range(VALIDATION_STEPS_PER_TPU_CALL):\n",
    "                strategy.experimental_run_v2(valid_step_fn, next(data_iter))\n",
    "        \n",
    "        ### Model\n",
    "        model_path = 'model_%sx%s_fold_%d.h5' % (HEIGHT, WIDTH, (n_fold+1))\n",
    "        with strategy.scope():\n",
    "            model = create_model((None, None, CHANNELS), N_CLASSES)\n",
    "            \n",
    "            # Instiate optimizer with learning rate schedule\n",
    "            class LRSchedule(optimizers.schedules.LearningRateSchedule):\n",
    "                def __call__(self, step):\n",
    "                    return lrfn(epoch=step//STEPS_PER_EPOCH)\n",
    "            optimizer = optimizers.Adam(learning_rate=LRSchedule())\n",
    "            \n",
    "            train_accuracy = metrics.SparseCategoricalAccuracy()\n",
    "            valid_accuracy = metrics.SparseCategoricalAccuracy()\n",
    "            train_loss = metrics.Sum()\n",
    "            valid_loss = metrics.Sum()\n",
    "\n",
    "            loss_fn = losses.sparse_categorical_crossentropy\n",
    "\n",
    "        step = 0\n",
    "        epoch = 0\n",
    "        epoch_steps = 0\n",
    "        epoch_start_time = time.time()\n",
    "        history = {'loss': [], 'val_loss': [], 'sparse_categorical_accuracy': [], 'val_sparse_categorical_accuracy': []}\n",
    "        \n",
    "        ### Train model\n",
    "        while True:\n",
    "            # run training step\n",
    "            train_step(train_data_iter)\n",
    "            epoch_steps += STEPS_PER_TPU_CALL\n",
    "            step += STEPS_PER_TPU_CALL\n",
    "\n",
    "            # validation run at the end of each epoch\n",
    "            if (step // STEPS_PER_EPOCH) > epoch:\n",
    "                # validation run\n",
    "                valid_epoch_steps = 0\n",
    "                for _ in range(int_div_round_up(validation_size, BATCH_SIZE*VALIDATION_STEPS_PER_TPU_CALL)):\n",
    "                    valid_step(valid_data_iter)\n",
    "                    valid_epoch_steps += VALIDATION_STEPS_PER_TPU_CALL\n",
    "\n",
    "                # compute metrics\n",
    "                history['sparse_categorical_accuracy'].append(train_accuracy.result().numpy())\n",
    "                history['val_sparse_categorical_accuracy'].append(valid_accuracy.result().numpy())\n",
    "                history['loss'].append(train_loss.result().numpy() / (BATCH_SIZE*epoch_steps))\n",
    "                history['val_loss'].append(valid_loss.result().numpy() / (BATCH_SIZE*valid_epoch_steps))\n",
    "\n",
    "                # report metrics\n",
    "                epoch_time = time.time() - epoch_start_time\n",
    "                print('\\nEPOCH {:d}/{:d}'.format(epoch+1, EPOCHS))\n",
    "                print('time: {:0.1f}s'.format(epoch_time),\n",
    "                      'loss: {:0.4f}'.format(history['loss'][-1]),\n",
    "                      'sparse_categorical_accuracy: {:0.4f}'.format(history['sparse_categorical_accuracy'][-1]),\n",
    "                      'val_loss: {:0.4f}'.format(history['val_loss'][-1]),\n",
    "                      'val_sparse_categorical_accuracy: {:0.4f}'.format(history['val_sparse_categorical_accuracy'][-1]))\n",
    "                print('LearningRate: {:0.4g}'.format(lrfn(epoch)))\n",
    "\n",
    "                # set up next epoch\n",
    "                epoch = step // STEPS_PER_EPOCH\n",
    "                epoch_steps = 0\n",
    "                epoch_start_time = time.time()\n",
    "                train_accuracy.reset_states()\n",
    "                valid_accuracy.reset_states()\n",
    "                valid_loss.reset_states()\n",
    "                train_loss.reset_states()\n",
    "                if epoch >= EPOCHS:\n",
    "                    break\n",
    "                    \n",
    "        model_path_list.append(model_path)\n",
    "        model.save(model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [],
   "source": [
    "# Test predictions\n",
    "x_test = test_dataset.map(lambda image, idnum: image)\n",
    "test_preds = np.zeros((NUM_TEST_IMAGES, N_CLASSES))\n",
    "\n",
    "for model_path in model_path_list:\n",
    "    print(model_path)\n",
    "    with strategy.scope():\n",
    "        model = load_model(model_path)\n",
    "                    \n",
    "    test_preds += model.predict(x_test) / FOLDS_USED\n",
    "    \n",
    "test_preds = np.argmax(test_preds, axis=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test set predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "_kg_hide-input": true
   },
   "outputs": [
    {
     "ename": "InvalidArgumentError",
     "evalue": "Unable to find a context_id matching the specified one (8264415813418159559). Perhaps the worker was restarted, or the context was GC'd?",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-e64250af481a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtest_ids_ds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_dataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mimage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midnum\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0midnum\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtest_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_ids_ds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mNUM_TEST_IMAGES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'U'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0msubmission\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'id'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0msubmission\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'label'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_preds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py\u001b[0m in \u001b[0;36mbatch\u001b[0;34m(self, batch_size, drop_remainder)\u001b[0m\n\u001b[1;32m   1379\u001b[0m       \u001b[0mDataset\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mA\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1380\u001b[0m     \"\"\"\n\u001b[0;32m-> 1381\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mBatchDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop_remainder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1382\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1383\u001b[0m   def padded_batch(self,\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, input_dataset, batch_size, drop_remainder)\u001b[0m\n\u001b[1;32m   3601\u001b[0m         drop_remainder, dtype=dtypes.bool, name=\"drop_remainder\")\n\u001b[1;32m   3602\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3603\u001b[0;31m     \u001b[0mconstant_drop_remainder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtensor_util\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_drop_remainder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3604\u001b[0m     \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3605\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mconstant_drop_remainder\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/tensor_util.py\u001b[0m in \u001b[0;36mconstant_value\u001b[0;34m(tensor, partial)\u001b[0m\n\u001b[1;32m    820\u001b[0m   \"\"\"\n\u001b[1;32m    821\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEagerTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 822\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    823\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    824\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py\u001b[0m in \u001b[0;36mnumpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    940\u001b[0m     \"\"\"\n\u001b[1;32m    941\u001b[0m     \u001b[0;31m# TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 942\u001b[0;31m     \u001b[0mmaybe_arr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    943\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_arr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py\u001b[0m in \u001b[0;36m_numpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    908\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    909\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 910\u001b[0;31m       \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    911\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    912\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/six.py\u001b[0m in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: Unable to find a context_id matching the specified one (8264415813418159559). Perhaps the worker was restarted, or the context was GC'd?"
     ]
    }
   ],
   "source": [
    "test_ids_ds = test_dataset.map(lambda image, idnum: idnum).unbatch()\n",
    "test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n",
    "\n",
    "submission = pd.DataFrame(test_ids, columns=['id'])\n",
    "submission['label'] = test_preds\n",
    "submission.to_csv('submission.csv', index=False)\n",
    "display(submission.head(10))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}