CogALex_XLM_RoBERTa_Updated_Task_Version.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/Text2TCS/Transrelation/blob/master/CogALex_XLM_RoBERTa_Updated_Task_Version.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WF-aoH7jZLTn"
   },
   "source": [
    "# Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3w3ZNBmItp_p",
    "outputId": "6b629027-783d-4ac3-dab9-97a86114f71a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/0c/7d5950fcd80b029be0a8891727ba21e0cd27692c407c51261c3c921f6da3/transformers-4.1.1-py3-none-any.whl (1.5MB)\n",
      "\u001b[K     |████████████████████████████████| 1.5MB 7.9MB/s \n",
      "\u001b[?25hCollecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 28.9MB/s \n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.4)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.8)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
      "Collecting tokenizers==0.9.4\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
      "\u001b[K     |████████████████████████████████| 2.9MB 50.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=e36db39f3fe6fe180e08f12a98a4819adee0292cc3b798c15ec46a46b99ff5d3\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sacremoses, tokenizers, transformers\n",
      "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.1.1\n",
      "Collecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 9.2MB/s \n",
      "\u001b[?25hInstalling collected packages: sentencepiece\n",
      "Successfully installed sentencepiece-0.1.94\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers\n",
    "!pip install sentencepiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "fM0lku9lt9As"
   },
   "outputs": [],
   "source": [
    "import torch                                              #for training the model\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "from torch.utils.data import TensorDataset\n",
    "import pandas as pd                                       #for handling the data\n",
    "from transformers import XLMRobertaTokenizer              #for loading the pretrained model and tokenizer\n",
    "from transformers import XLMRobertaForSequenceClassification\n",
    "from transformers import AdamW                            \n",
    "from transformers import get_linear_schedule_with_warmup\n",
    "from sklearn import preprocessing                         #for label encoding\n",
    "from sklearn.metrics import classification_report         #for showing performance on validation/test sets\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import ParameterGrid         #for gridsearch\n",
    "import sentencepiece\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import datetime\n",
    "import random\n",
    "import numpy as np\n",
    "% matplotlib inline\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vXVdDSzPWpuQ"
   },
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4XjnVL2AfMrV"
   },
   "source": [
    "Load data from text files "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "3uHTNIY9xM08"
   },
   "outputs": [],
   "source": [
    "# sk learn label encoder for changing the labels to integers\n",
    "labels=[\"ANT\", \"HYP\", \"RANDOM\", \"SYN\"]\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(labels)\n",
    "\n",
    "#chinese \n",
    "data_train_zh = pd.read_csv('data/train/train_chinese_data.txt', sep=\"\\t\", header=None)\n",
    "data_train_zh.columns = [\"Word1\", \"Word2\", \"Label\"]\n",
    "\n",
    "data_valid_zh = pd.read_csv('data/valid/validgold_chinese_data.txt', sep=\"\\t\", header=None)\n",
    "data_valid_zh.columns = [\"Word1\", \"Word2\", \"Label\"]\n",
    "\n",
    "data_train_zh[\"Label\"]=le.transform(data_train_zh[\"Label\"])\n",
    "data_valid_zh[\"Label\"]=le.transform(data_valid_zh[\"Label\"])\n",
    "\n",
    "#english \n",
    "data_train_en = pd.read_csv('data/train/train_english_data.txt', sep=\"\\t\", header=None)\n",
    "data_train_en.columns = [\"Word1\", \"Word2\", \"Label\"]\n",
    "\n",
    "data_valid_en = pd.read_csv('data/valid/validgold_english_data.txt', sep=\"\\t\", header=None)\n",
    "data_valid_en.columns = [\"Word1\", \"Word2\", \"Label\"]\n",
    "\n",
    "data_train_en[\"Label\"]=le.transform(data_train_en[\"Label\"])\n",
    "data_valid_en[\"Label\"]=le.transform(data_valid_en[\"Label\"])\n",
    "\n",
    "#german \n",
    "data_train_de = pd.read_csv('data/train/train_german_data.txt', sep=\"\\t\", header=None)\n",
    "data_train_de.columns = [\"Word1\", \"Word2\", \"Label\"]\n",
    "\n",
    "data_valid_de = pd.read_csv('data/valid/validgold_german_data.txt', sep=\"\\t\", header=None)\n",
    "data_valid_de.columns = [\"Word1\", \"Word2\", \"Label\"]\n",
    "\n",
    "data_train_de[\"Label\"]=le.transform(data_train_de[\"Label\"])\n",
    "data_valid_de[\"Label\"]=le.transform(data_valid_de[\"Label\"])\n",
    "\n",
    "\n",
    "# all together\n",
    "data_train_all=pd.concat([data_train_zh, data_train_en, data_train_de])\n",
    "data_train_all=data_train_all.reset_index(drop=True)\n",
    "data_valid_all=pd.concat([data_valid_zh, data_valid_en, data_valid_de])\n",
    "data_valid_all=data_valid_all.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "id": "ccHj8gEmqA88",
    "outputId": "cd5f46a0-5589-45f4-ae97-86206c620047"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f3264718e80>"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEFCAYAAAAPCDf9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAURklEQVR4nO3dfbCedX3n8fdHIvi4JMjZFBM07Jrq4nQBN+VhtI4rsyFgu+EPZamuRoZOZnagu51uH7A6hiJ06c7sKozKTKakBlGBUi3ZysqmWMe6LQ+hPImApAibpECOJIBKhUa/+8f9O3iTnpNzDpyck8Pv/Zo5c67re/2u6/5ed5LPfZ3ffd0nqSokSX142Vw3IEmaPYa+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1pmpKMJLkvySvnupfxJDmk9Tcy173owGPo64CU5P1JtiT5YZJHkvzvJO+YhcetJG+aZNh5wOeq6h/aPt9I8mv7u7eJ7P34VfUMsIFBn9LzGPo64CT5TeBTwB8Ai4E3AJ8FVs9lXzC4igbWAFfO4DEXzNSxhnwRWNP6lZ5j6OuAkuRQ4ALgnKr6clX9qKr+sar+V1X9dhtzSJJPJfn79vWpsXBL8uEk39rrmM9dvSf5XJLPJPlqkh8kuTnJv2zbvtl2ubP9hPEfxmnxBOCJqtre9rkI+CXg022fT7f6JUm2JXkqyW1Jfmmon/OTXJvkyiRPAR9OclSSb7ae/qL1eOXQPicm+eskTyS5M8m79vX4rb/dwIkv4o9DL0GGvg40JwGvAL6yjzEfZRBmxwLHAMcDH5vGY5wJ/D6wCNgKXARQVe9s24+pqtdU1dXj7PsLwP1jK1X1UeCvgHPbPue2Tbe2/g5jcNX9J0leMXSc1cC1wELgC23MLcDrgPOBD44NTLIE+CpwYTvebwF/mmRkH48PcC+D50d6jqGvA83rgO9X1Z59jPkAcEFV7ayqUQYB/sF9jN/bV6rqlvYYX2AQzlO1EPjBZIOq6sqqeryq9lTV/wAOAd48NORvqurPquqnwAjwi8DHq+rZqvoWsGlo7H8Erq+q66vqp1W1GdgCnDZJGz9o/UrPMfR1oHkcOHySee7XAw8PrT/calP16NDy08BrprHvbuC1kw1K8ltJ7k3yZJIngEOBw4eGbBtafj2wq6qenmD7G4H3tamdJ9rx3gEcMUkbrwWemKxX9cXQ14Hmb4BngNP3MebvGQThmDe0GsCPgFeNbUjyczPc313Az+9Ve96vqm3z978DnAEsqqqFwJNAJtjnEeCwJK8aqh05tLwN+HxVLRz6enVVXTze4w/5V8CdUzkp9cPQ1wGlqp4EPg58JsnpSV6V5OVJTk3y39uwLwEfa/fLH97Gj73peSfw1iTHtjn086fZwmPAv9jH9luAhW2efaJ9XgvsAUaBBUk+DvyziQ5YVQ8zmK45P8nBSU4CfmVoyJXAryQ5JclBSV6R5F1Jlk7Uc+vvMOCmfZyLOmTo64DT5sB/k8Gbs6MMrnTPBf6sDbmQQUjeBdwN/G2rUVXfZXD3z18ADwDPu5NnCs4HNrZplDPG6e1Z4HMM5tnHXAK8N8nuJJcCNwBfA77LYOrpxzx/umY8H2DwJvbj7VyuZvATD1W1jcEbv7/Hz56P3+Zn/373fnyA9wMb2z370nPif6IiTU/7pOtfAceNfUBrPzzG1cB9VbXuBex7CIOfeN5ZVTtnvDnNa4a+dABI8ovALuB7wEoGP9WcVFW3z2ljesnZH58ElDR9Pwd8mcEtq9uB/2Tga3/wSl+SOuIbuZLUEUNfkjpyQM/pH3744bVs2bK5bkOS5pXbbrvt+1U17v+ncECH/rJly9iyZctctyFJ80qShyfa5vSOJHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSMH9Iez9odl5311rluYkocufs9ctyDpJcgrfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktSRKYV+koVJrk1yX5J7k5yU5LAkm5M80L4vamOT5NIkW5PcleRtQ8dZ08Y/kGTN/jopSdL4pnqlfwnwtap6C3AMcC9wHnBjVS0HbmzrAKcCy9vXWuAygCSHAeuAE4DjgXVjLxSSpNkxaegnORR4J3A5QFU9W1VPAKuBjW3YRuD0trwauKIGbgIWJjkCOAXYXFW7qmo3sBlYNaNnI0nap6lc6R8FjAJ/nOT2JH+U5NXA4qp6pI15FFjclpcA24b2395qE9UlSbNkKqG/AHgbcFlVHQf8iJ9N5QBQVQXUTDSUZG2SLUm2jI6OzsQhJUnNVEJ/O7C9qm5u69cyeBF4rE3b0L7vbNt3AEcO7b+01SaqP09Vra+qFVW1YmRk3P/MXZL0Ak0a+lX1KLAtyZtb6WTgO8AmYOwOnDXAdW15E/ChdhfPicCTbRroBmBlkkXtDdyVrSZJmiVT/S2bvw58IcnBwIPAWQxeMK5JcjbwMHBGG3s9cBqwFXi6jaWqdiX5BHBrG3dBVe2akbOQJE3JlEK/qu4AVoyz6eRxxhZwzgTH2QBsmE6DkqSZ4ydyJakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHZlS6Cd5KMndSe5IsqXVDkuyOckD7fuiVk+SS5NsTXJXkrcNHWdNG/9AkjX755QkSROZzpX+v62qY6tqRVs/D7ixqpYDN7Z1gFOB5e1rLXAZDF4kgHXACcDxwLqxFwpJ0ux4MdM7q4GNbXkjcPpQ/YoauAlYmOQI4BRgc1XtqqrdwGZg1Yt4fEnSNE019Av4P0luS7K21RZX1SNt+VFgcVteAmwb2nd7q01Uf54ka5NsSbJldHR0iu1JkqZiwRTHvaOqdiT558DmJPcNb6yqSlIz0VBVrQfWA6xYsWJGjilJGpjSlX5V7WjfdwJfYTAn/1ibtqF939mG7wCOHNp9aatNVJckzZJJQz/Jq5O8dmwZWAl8G9gEjN2Bswa4ri1vAj7U7uI5EXiyTQPdAKxMsqi9gbuy1SRJs2Qq0zuLga8kGRv/xar6WpJbgWuSnA08DJzRxl8PnAZsBZ4GzgKoql1JPgHc2sZdUFW7ZuxMJEmTmjT0q+pB4Jhx6o8DJ49TL+CcCY61Adgw/TYlSTPBT+RKUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1JEph36Sg5LcnuTP2/pRSW5OsjXJ1UkObvVD2vrWtn3Z0DE+0ur3Jzllpk9GkrRv07nS/y/AvUPrfwh8sqreBOwGzm71s4Hdrf7JNo4kRwNnAm8FVgGfTXLQi2tfkjQdUwr9JEuB9wB/1NYDvBu4tg3ZCJzelle3ddr2k9v41cBVVfVMVX0P2AocPxMnIUmamqle6X8K+B3gp239dcATVbWnrW8HlrTlJcA2gLb9yTb+ufo4+0iSZsGkoZ/kl4GdVXXbLPRDkrVJtiTZMjo6OhsPKUndmMqV/tuBf5/kIeAqBtM6lwALkyxoY5YCO9ryDuBIgLb9UODx4fo4+zynqtZX1YqqWjEyMjLtE5IkTWzS0K+qj1TV0qpaxuCN2K9X1QeAvwTe24atAa5ry5vaOm3716uqWv3MdnfPUcBy4JYZOxNJ0qQWTD5kQr8LXJXkQuB24PJWvxz4fJKtwC4GLxRU1T1JrgG+A+wBzqmqn7yIx5ckTdO0Qr+qvgF8oy0/yDh331TVj4H3TbD/RcBF021SkjQz/ESuJHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI5OGfpJXJLklyZ1J7kny+61+VJKbk2xNcnWSg1v9kLa+tW1fNnSsj7T6/UlO2V8nJUka31Su9J8B3l1VxwDHAquSnAj8IfDJqnoTsBs4u40/G9jd6p9s40hyNHAm8FZgFfDZJAfN5MlIkvZt0tCvgR+21Ze3rwLeDVzb6huB09vy6rZO235ykrT6VVX1TFV9D9gKHD8jZyFJmpIpzeknOSjJHcBOYDPwd8ATVbWnDdkOLGnLS4BtAG37k8Drhuvj7CNJmgVTCv2q+klVHQssZXB1/pb91VCStUm2JNkyOjq6vx5Gkro0rbt3quoJ4C+Bk4CFSRa0TUuBHW15B3AkQNt+KPD4cH2cfYYfY31VraiqFSMjI9NpT5I0iancvTOSZGFbfiXw74B7GYT/e9uwNcB1bXlTW6dt/3pVVauf2e7uOQpYDtwyUyciSZrcgsmHcASwsd1p8zLgmqr68yTfAa5KciFwO3B5G3858PkkW4FdDO7YoaruSXIN8B1gD3BOVf1kZk9HkrQvk4Z+Vd0FHDdO/UHGufumqn4MvG+CY10EXDT9NiVJM8FP5EpSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUkQVz3YDmt2XnfXWuW5iShy5+z1y3IB0QJg39JEcCVwCLgQLWV9UlSQ4DrgaWAQ8BZ1TV7iQBLgFOA54GPlxVf9uOtQb4WDv0hVW1cWZPR5rffBHV/jaV6Z09wH+tqqOBE4FzkhwNnAfcWFXLgRvbOsCpwPL2tRa4DKC9SKwDTgCOB9YlWTSD5yJJmsSkoV9Vj4xdqVfVD4B7gSXAamDsSn0jcHpbXg1cUQM3AQuTHAGcAmyuql1VtRvYDKya0bORJO3TtOb0kywDjgNuBhZX1SNt06MMpn9g8IKwbWi37a02UV2SZpxTZeOb8t07SV4D/CnwG1X11PC2qioG8/0vWpK1SbYk2TI6OjoTh5QkNVMK/SQvZxD4X6iqL7fyY23ahvZ9Z6vvAI4c2n1pq01Uf56qWl9VK6pqxcjIyHTORZI0iUlDv92Nczlwb1X9z6FNm4A1bXkNcN1Q/UMZOBF4sk0D3QCsTLKovYG7stUkSbNkKnP6bwc+CNyd5I5W+z3gYuCaJGcDDwNntG3XM7hdcyuDWzbPAqiqXUk+Adzaxl1QVbtm5CwkSVMyaehX1beATLD55HHGF3DOBMfaAGyYToOSpJnjr2GQpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1ZNLQT7Ihyc4k3x6qHZZkc5IH2vdFrZ4klybZmuSuJG8b2mdNG/9AkjX753QkSfsylSv9zwGr9qqdB9xYVcuBG9s6wKnA8va1FrgMBi8SwDrgBOB4YN3YC4UkafZMGvpV9U1g117l1cDGtrwROH2ofkUN3AQsTHIEcAqwuap2VdVuYDP/9IVEkrSfvdA5/cVV9UhbfhRY3JaXANuGxm1vtYnqkqRZ9KLfyK2qAmoGegEgydokW5JsGR0dnanDSpJ44aH/WJu2oX3f2eo7gCOHxi1ttYnq/0RVra+qFVW1YmRk5AW2J0kazwsN/U3A2B04a4DrhuofanfxnAg82aaBbgBWJlnU3sBd2WqSpFm0YLIBSb4EvAs4PMl2BnfhXAxck+Rs4GHgjDb8euA0YCvwNHAWQFXtSvIJ4NY27oKq2vvNYUnSfjZp6FfVr06w6eRxxhZwzgTH2QBsmFZ3kqQZ5SdyJakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHZn10E+yKsn9SbYmOW+2H1+SejaroZ/kIOAzwKnA0cCvJjl6NnuQpJ7N9pX+8cDWqnqwqp4FrgJWz3IPktStVNXsPVjyXmBVVf1aW/8gcEJVnTs0Zi2wtq2+Gbh/1hp84Q4Hvj/XTbyE+HzOLJ/PmTNfnss3VtXIeBsWzHYnk6mq9cD6ue5jOpJsqaoVc93HS4XP58zy+Zw5L4Xncrand3YARw6tL201SdIsmO3QvxVYnuSoJAcDZwKbZrkHSerWrE7vVNWeJOcCNwAHARuq6p7Z7GE/mVfTUfOAz+fM8vmcOfP+uZzVN3IlSXPLT+RKUkcMfUnqiKEvSR054O7Tnw+SvAVYAtxcVT8cqq+qqq/NXWfqXfu7uZrB308Y3BK9qarunbuu5q8kxwNVVbe2XxmzCrivqq6f49ZeMK/0pynJfwauA34d+HaS4V8j8Qdz09VLU5Kz5rqH+STJ7zL41SYBbmlfAb7kLzecviTrgEuBy5L8N+DTwKuB85J8dE6bexG8e2eaktwNnFRVP0yyDLgW+HxVXZLk9qo6bk4bfAlJ8v+q6g1z3cd8keS7wFur6h/3qh8M3FNVy+ems/mp/Vs/FjgEeBRYWlVPJXklg5/y//WcNvgCOb0zfS8bm9KpqoeSvAu4NskbGVxVaRqS3DXRJmDxbPbyEvBT4PXAw3vVj2jbND17quonwNNJ/q6qngKoqn9IMm+fT0N/+h5LcmxV3QHQrvh/GdgA/MLctjYvLQZOAXbvVQ/w17Pfzrz2G8CNSR4AtrXaG4A3AedOuJcm8mySV1XV08C/GSsmOZR5/CLq9M40JVnK4Arg0XG2vb2q/u8ctDVvJbkc+OOq+tY4275YVe+fg7bmrSQvY/ArzIffyL21XbFqGpIcUlXPjFM/HDiiqu6eg7ZeNENfkjri3TuS1BFDX5I6YuhLUkcMfUnqiKEvSR35/5tH52z6bSShAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# print class distribution \n",
    "data_train_all[\"Label\"].value_counts().plot(kind='bar', title='Count (target)')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "da8D7C2WZDAh"
   },
   "source": [
    "# Tokenize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zd6-b2-SiCiL"
   },
   "source": [
    "Tokenize Data\n",
    "\n",
    "- encode in the right format for XLM-Roberta ( <s\\> = sentence beginning, </s\\> end of sentence/sentence seperator)\n",
    "\n",
    "- Truncate/Padding so everything has the same length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 66,
     "referenced_widgets": [
      "ba89918f181649f2bd10ce5c6f1e6d48",
      "d79b07b8495c416c83cbfdfffb3d8e1c",
      "3753df9760b643dba4e0aa54313b1689",
      "ff08b72911f14f77a8c51fbf0287ef15",
      "0db3ca8db482422ca68db57d980c0643",
      "5bca0b4663d3475fb9ca4793783b2130",
      "b07f70c2f1e4473bb8a90fa65d281183",
      "b5e15dfc072a46ad9367eaafa17f6313"
     ]
    },
    "id": "qoUK_mvmh1im",
    "outputId": "4c69b179-a621-40fa-a479-ca6d88090dfb"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba89918f181649f2bd10ce5c6f1e6d48",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=5069051.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "xlmr_tokenizer = XLMRobertaTokenizer.from_pretrained(\"xlm-roberta-base\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "fCoGBdK6iput"
   },
   "outputs": [],
   "source": [
    "max_len=64\n",
    "\n",
    "def tokenizer_xlm(data, max_len):\n",
    "  labels_ = []\n",
    "  input_ids_ = []\n",
    "  attn_masks_ = []\n",
    "\n",
    "  # for each datasample:\n",
    "  for index, row in data.iterrows():\n",
    "\n",
    "      word1 = row['Word1']\n",
    "      word2 = row['Word2']\n",
    "\n",
    "      # create requiered input, i.e. ids and attention masks\n",
    "      encoded_dict = xlmr_tokenizer.encode_plus(word1, word2, \n",
    "                                                max_length=max_len, \n",
    "                                                padding='max_length',\n",
    "                                                truncation=True, \n",
    "                                                return_tensors='pt')\n",
    "\n",
    "      # add encoded sample to lists\n",
    "      input_ids_.append(encoded_dict['input_ids'])\n",
    "      attn_masks_.append(encoded_dict['attention_mask'])\n",
    "      labels_.append(row['Label'])\n",
    "      \n",
    "  # Convert each Python list of Tensors into a 2D Tensor matrix.\n",
    "  input_ids_ = torch.cat(input_ids_, dim=0)\n",
    "  attn_masks_ = torch.cat(attn_masks_, dim=0)\n",
    "\n",
    "  # labels to tensor\n",
    "  labels_ = torch.tensor(labels_)\n",
    "\n",
    "  print('Encoder finished. {:,} examples.'.format(len(labels_)))\n",
    "  return input_ids_, attn_masks_, labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rD54rHpy4OEm",
    "outputId": "8708454e-7a26-4b00-fb41-94ca9a7e64c5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All\n",
      "Encoder finished. 12,606 examples.\n",
      "Encoder finished. 4,201 examples.\n",
      "zh\n",
      "Encoder finished. 2,514 examples.\n",
      "Encoder finished. 838 examples.\n",
      "en\n",
      "Encoder finished. 5,210 examples.\n",
      "Encoder finished. 1,736 examples.\n",
      "de\n",
      "Encoder finished. 4,882 examples.\n",
      "Encoder finished. 1,627 examples.\n"
     ]
    }
   ],
   "source": [
    "# tokenize data\n",
    "\n",
    "#all\n",
    "print(\"All\")\n",
    "input_ids_train_all, attn_masks_train_all, labels_train_all = tokenizer_xlm(data_train_all, max_len)\n",
    "input_ids_valid_all, attn_masks_valid_all, labels_valid_all = tokenizer_xlm(data_valid_all, max_len)\n",
    "\n",
    "#zh\n",
    "print(\"zh\")\n",
    "input_ids_train_zh, attn_masks_train_zh, labels_train_zh = tokenizer_xlm(data_train_zh, max_len)\n",
    "input_ids_valid_zh, attn_masks_valid_zh, labels_valid_zh = tokenizer_xlm(data_valid_zh, max_len)\n",
    "\n",
    "#en\n",
    "print(\"en\")\n",
    "input_ids_train_en, attn_masks_train_en, labels_train_en = tokenizer_xlm(data_train_en, max_len)\n",
    "input_ids_valid_en, attn_masks_valid_en, labels_valid_en = tokenizer_xlm(data_valid_en, max_len)\n",
    "\n",
    "#de\n",
    "print(\"de\")\n",
    "input_ids_train_de, attn_masks_train_de, labels_train_de = tokenizer_xlm(data_train_de, max_len)\n",
    "input_ids_valid_de, attn_masks_valid_de, labels_valid_de = tokenizer_xlm(data_valid_de, max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "7K62ohYg00oH"
   },
   "outputs": [],
   "source": [
    "# Combine the training inputs into a TensorDataset.\n",
    "\n",
    "#all\n",
    "tensor_data_train_all = TensorDataset(input_ids_train_all, attn_masks_train_all, labels_train_all)\n",
    "tensor_data_valid_all = TensorDataset(input_ids_valid_all, attn_masks_valid_all, labels_valid_all)\n",
    "#zh\n",
    "tensor_data_train_zh = TensorDataset(input_ids_train_zh, attn_masks_train_zh, labels_train_zh)\n",
    "tensor_data_valid_zh = TensorDataset(input_ids_valid_zh, attn_masks_valid_zh, labels_valid_zh)\n",
    "#en\n",
    "tensor_data_train_en = TensorDataset(input_ids_train_en, attn_masks_train_en, labels_train_en)\n",
    "tensor_data_valid_en = TensorDataset(input_ids_valid_en, attn_masks_valid_en, labels_valid_en)\n",
    "#de\n",
    "tensor_data_train_de = TensorDataset(input_ids_train_de, attn_masks_train_de, labels_train_de)\n",
    "tensor_data_valid_de = TensorDataset(input_ids_valid_de, attn_masks_valid_de, labels_valid_de)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "SRjNGp6Y1R_h"
   },
   "outputs": [],
   "source": [
    "#prepare pytorch dataloaders\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "#all\n",
    "train_dataloader_all = DataLoader(tensor_data_train_all, sampler = RandomSampler(tensor_data_train_all), batch_size = batch_size) #random sampling\n",
    "validation_dataloader_all = DataLoader(tensor_data_valid_all, sampler = SequentialSampler(tensor_data_valid_all),batch_size = batch_size ) #sequential sampling\n",
    "#zh\n",
    "train_dataloader_zh = DataLoader(tensor_data_train_zh, sampler = RandomSampler(tensor_data_train_zh), batch_size = batch_size)\n",
    "validation_dataloader_zh = DataLoader(tensor_data_valid_zh, sampler = SequentialSampler(tensor_data_valid_zh),batch_size = batch_size)\n",
    "#en\n",
    "train_dataloader_en = DataLoader(tensor_data_train_en, sampler = RandomSampler(tensor_data_train_en), batch_size = batch_size)\n",
    "validation_dataloader_en = DataLoader(tensor_data_valid_en, sampler = SequentialSampler(tensor_data_valid_en),batch_size = batch_size)\n",
    "#de\n",
    "train_dataloader_de = DataLoader(tensor_data_train_de, sampler = RandomSampler(tensor_data_train_de), batch_size = batch_size)\n",
    "validation_dataloader_de = DataLoader(tensor_data_valid_de, sampler = SequentialSampler(tensor_data_valid_de),batch_size = batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OAm6GWa-6NJS"
   },
   "source": [
    "# Training the Classifier (Finetuning XLM-RoBERTa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 171,
     "referenced_widgets": [
      "c69e9514345944a18780f74fd55cc97f",
      "392d1b126ea549fea9b77e317929b5c9",
      "55513c30e3194e5b8b98793776098b14",
      "b96064f9c88b4d5fb605842e22ee5ec8",
      "3b82354faba14c37bc6419d1d033df03",
      "9fcfd2bc25434ab1b3b478f6528c577b",
      "745bdbc2dd454163bc3de08b80a19806",
      "0bca7a6b07ac4b7f9980679b06dcb798"
     ]
    },
    "id": "x-1l4-Ze6U5P",
    "outputId": "04eede80-3e1d-46b6-fb30-83c4ece2a9e4"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c69e9514345944a18780f74fd55cc97f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1115590446.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForSequenceClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
      "- This IS expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForSequenceClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "#load the pretrained model provided by HuggingFace with an added untrained classification head for 4 classes\n",
    "xlmr_model = XLMRobertaForSequenceClassification.from_pretrained(\"xlm-roberta-base\", num_labels=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hkxluZ1ulxeF",
    "outputId": "10923a25-8f8d-4a86-ddfd-ac2ed6507f0e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Samples: 12606\n"
     ]
    }
   ],
   "source": [
    "#choose training set used for this single training example (original training data for final model)\n",
    "train_dataloader = train_dataloader_all\n",
    "print(\"Training Samples:\",len(train_dataloader.dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rj2KvpZ46wJX",
    "outputId": "886453d7-509e-4fa9-cc28-68d2b59080ad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Connected to GPU: Tesla T4\n"
     ]
    }
   ],
   "source": [
    "# connect to GPU \n",
    "device = torch.device('cuda')\n",
    "# copy weights onto GPU\n",
    "desc = xlmr_model.to(device)\n",
    "\n",
    "print('Connected to GPU:', torch.cuda.get_device_name(0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8znTmQ-067np"
   },
   "source": [
    "Set parameters (learning rate & epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "zF-StfZ664Fj"
   },
   "outputs": [],
   "source": [
    "optimizer = AdamW(xlmr_model.parameters(),\n",
    "                  lr = 2e-5,   # do work well: 2e-5 with 5-7 epochs for trainall, 1e-5\n",
    "                  eps = 1e-8   # 1e-8\n",
    "                  # weight_decay = 0          \n",
    "                  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "e3n400X88kau",
    "outputId": "8898a820-2e67-4d76-c923-617e7998a2c4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total steps: 2758\n"
     ]
    }
   ],
   "source": [
    "# number of training epochs\n",
    "epochs = 7\n",
    "\n",
    "# number of batches x epochs\n",
    "total_steps = len(train_dataloader) * epochs\n",
    "print(\"total steps:\", total_steps)\n",
    "\n",
    "#scheduler for lr\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
    "                                            num_warmup_steps = 0,   #start low and increase learning rate during these steps\n",
    "                                            num_training_steps = total_steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ia0s_Z4I3KO7"
   },
   "source": [
    "Formatting functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "u-OAa6pi5FVQ"
   },
   "outputs": [],
   "source": [
    "# return time as hh:mm:ss\n",
    "def format_time(elapsed):\n",
    "    elapsed_rounded = int(round((elapsed)))\n",
    "    return str(datetime.timedelta(seconds=elapsed_rounded))  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Rfz7OkuwEAXa"
   },
   "source": [
    "Validation function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "-66H7RWhDtti"
   },
   "outputs": [],
   "source": [
    "def validate(validation_dataloader, xlmr_model, verbose): \n",
    "  \n",
    "  # put model in evaluation mode \n",
    "  xlmr_model.eval()\n",
    "\n",
    "  # variable for loss, predictions and labels\n",
    "  total_eval_loss = 0\n",
    "  predictions, true_labels = [], []\n",
    "\n",
    "  # Evaluate data for one epoch\n",
    "  for batch in validation_dataloader:\n",
    "          \n",
    "      # Unpack training batch and copy the tensors to the gpu\n",
    "      b_input_ids = batch[0].to(device)\n",
    "      b_input_mask = batch[1].to(device)\n",
    "      b_labels = batch[2].to(device)\n",
    "          \n",
    "      # no backprop needed\n",
    "      with torch.no_grad():        \n",
    "\n",
    "          # forward pass\n",
    "          output = xlmr_model(b_input_ids, \n",
    "                                    token_type_ids=None, \n",
    "                                    attention_mask=b_input_mask,\n",
    "                                    labels=b_labels)\n",
    "          loss=output.loss\n",
    "          logits=output.logits\n",
    "              \n",
    "      # add up loss\n",
    "      total_eval_loss += loss.item()\n",
    "\n",
    "      # on cpu\n",
    "      logits = logits.detach().cpu().numpy()\n",
    "      label_ids = b_labels.to('cpu').numpy()\n",
    "\n",
    "      # save preds/true labels\n",
    "      predictions.append(logits)\n",
    "      true_labels.append(label_ids)\n",
    "\n",
    "  # results of the whole validation set\n",
    "  flat_predictions = np.concatenate(predictions, axis=0)\n",
    "  flat_true_labels = np.concatenate(true_labels, axis=0)\n",
    "\n",
    "  # logit to label\n",
    "  predicted_labels = np.argmax(flat_predictions, axis=1).flatten()\n",
    "\n",
    "  # print classification report\n",
    "  if verbose:\n",
    "    print(classification_report(flat_true_labels, predicted_labels, target_names=labels))\n",
    "\n",
    "  # Calculate the validation accuracy, macro f1, and weighted f1 without RANDOM\n",
    "  val_accuracy = (predicted_labels == flat_true_labels).mean()\n",
    "  macroF1 = f1_score(flat_true_labels, predicted_labels, average='macro')\n",
    "  weightedF1_no_random=f1_score(flat_true_labels, predicted_labels, average='weighted', labels=[0,1,3])\n",
    "  print(\"\\t Weighted F1 (no random):\", weightedF1_no_random)\n",
    "\n",
    "  # Calculate the average loss over all of the batches.\n",
    "  avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
    "\n",
    "  # plot confusion matrix\n",
    "  if verbose:\n",
    "    print(confusion_matrix(flat_true_labels, predicted_labels, labels=[0,1,2,3]))\n",
    "\n",
    "  return avg_val_loss, val_accuracy, macroF1, weightedF1_no_random\n",
    "      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CrCKhOce9ASW"
   },
   "source": [
    "Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "BcgjT8Gw82UN"
   },
   "outputs": [],
   "source": [
    "def train_model(epochs, xlmr_model, train_dataloader, validation_dataloader_set, random_seed, verbose):\n",
    "\n",
    "  seed_val = random_seed\n",
    "\n",
    "  random.seed(seed_val)\n",
    "  np.random.seed(seed_val)\n",
    "  torch.manual_seed(seed_val)\n",
    "  torch.cuda.manual_seed_all(seed_val)\n",
    "\n",
    "  # mostly contains scores about how the training went for each epoch\n",
    "  training_stats = []\n",
    "\n",
    "  # total training time\n",
    "  total_t0 = time.time()\n",
    "\n",
    "  print('\\033[1m'+\"================ Model Training ================\"+'\\033[0m')\n",
    "\n",
    "  # For each epoch...\n",
    "  for epoch_i in range(0, epochs):\n",
    "\n",
    "      print(\"\")\n",
    "      print('\\033[1m'+'======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs)+'\\033[0m')\n",
    "\n",
    "      t0 = time.time()\n",
    "\n",
    "      # summed training loss of the epoch\n",
    "      total_train_loss = 0\n",
    "\n",
    "\n",
    "      # model is being put into training mode as mechanisms like dropout work differently during train and test time\n",
    "      xlmr_model.train()\n",
    "\n",
    "      # For each batch of training data...\n",
    "      for step, batch in enumerate(train_dataloader):\n",
    "\n",
    "          # unpack training batch at load it to gpu (device)  \n",
    "          b_input_ids = batch[0].to(device)\n",
    "          b_input_mask = batch[1].to(device)\n",
    "          b_labels = batch[2].to(device)\n",
    "\n",
    "          # clear gradients before calculating new ones\n",
    "          xlmr_model.zero_grad()        \n",
    "\n",
    "          # forward pass with current batch\n",
    "          output = xlmr_model(b_input_ids, \n",
    "                              token_type_ids=None, \n",
    "                              attention_mask=b_input_mask, \n",
    "                              labels=b_labels)\n",
    "          \n",
    "          loss=output.loss\n",
    "          logits=output.logits\n",
    "\n",
    "          # add up the loss\n",
    "          total_train_loss += loss.item()\n",
    "\n",
    "          # calculate new gradients\n",
    "          loss.backward()\n",
    "\n",
    "          # gradient clipping (not bigger than)\n",
    "          torch.nn.utils.clip_grad_norm_(xlmr_model.parameters(), 1.0)\n",
    "\n",
    "          # Update the networks weights based on the gradient as well as the optimiziers parameters\n",
    "          optimizer.step()\n",
    "\n",
    "          # lr update\n",
    "          scheduler.step()\n",
    "\n",
    "      # avg loss over all batches\n",
    "      avg_train_loss = total_train_loss / len(train_dataloader)            \n",
    "      \n",
    "      # training time of this epoch\n",
    "      training_time = format_time(time.time() - t0)\n",
    "\n",
    "      print(\"\")\n",
    "      print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
    "      print(\"  Training epoch took: {:}\".format(training_time))\n",
    "          \n",
    "  \n",
    "      # VALIDATION\n",
    "\n",
    "      #all\n",
    "      print(\"evaluate on all\")\n",
    "      avg_val_loss_all, val_accuracy_all, macroF1_all, weightedF1_no_random_all = validate(validation_dataloader_set[0], xlmr_model, verbose)   \n",
    "      #zh\n",
    "      print(\"evaluate on zh\")\n",
    "      avg_val_loss_zh, val_accuracy_zh, macroF1_zh, weightedF1_no_random_zh = validate(validation_dataloader_set[1], xlmr_model, verbose) \n",
    "      #en\n",
    "      print(\"evaluate on en\")\n",
    "      avg_val_loss_en, val_accuracy_en, macroF1_en, weightedF1_no_random_en = validate(validation_dataloader_set[2], xlmr_model, verbose) \n",
    "      #de\n",
    "      print(\"evaluate on de\")\n",
    "      avg_val_loss_de, val_accuracy_de, macroF1_de, weightedF1_no_random_de = validate(validation_dataloader_set[3], xlmr_model, verbose)  \n",
    "       \n",
    "\n",
    "      print('\\033[1m'+ \"  Validation Loss All: {0:.2f}\".format(avg_val_loss_all) + '\\033[0m')\n",
    "\n",
    "      training_stats.append(\n",
    "          {\n",
    "              'epoch': epoch_i + 1,\n",
    "              'Training Loss': avg_train_loss,\n",
    "              'Valid. Loss all': avg_val_loss_all,\n",
    "              'Valid. Accur. all': val_accuracy_all,\n",
    "              'Weigh_F1 all (no random)': weightedF1_no_random_all, \n",
    "              'Macro F1 all': macroF1_all,\n",
    "              'Weigh_F1 en': weightedF1_no_random_en,\n",
    "              'Weigh_F1 de': weightedF1_no_random_de,\n",
    "              'Weigh_F1 zh': weightedF1_no_random_zh,\n",
    "              'Training Time': training_time,\n",
    "          }\n",
    "      )\n",
    "\n",
    "  print(\"\\n\\nTraining complete!\")\n",
    "  print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))\n",
    "  \n",
    "  return training_stats\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "Dsue5wnCmQCr",
    "outputId": "1dede1ef-a6b6-42c7-d7a7-1094ca1ba6a0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m================ Model Training ================\u001b[0m\n",
      "\n",
      "\u001b[1m======== Epoch 1 / 7 ========\u001b[0m\n",
      "\n",
      "  Average training loss: 1.24\n",
      "  Training epoch took: 0:02:49\n",
      "evaluate on all\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.27      0.16      0.20       719\n",
      "         HYP       0.46      0.16      0.24       731\n",
      "      RANDOM       0.64      0.87      0.74      2091\n",
      "         SYN       0.34      0.36      0.35       660\n",
      "\n",
      "    accuracy                           0.54      4201\n",
      "   macro avg       0.43      0.39      0.38      4201\n",
      "weighted avg       0.50      0.54      0.50      4201\n",
      "\n",
      "\t Weighted F1 (no random): 0.2602448727180897\n",
      "[[ 112   45  386  176]\n",
      " [  76  119  342  194]\n",
      " [ 129   56 1812   94]\n",
      " [  96   41  284  239]]\n",
      "evaluate on zh\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.40      0.15      0.22       136\n",
      "         HYP       0.65      0.25      0.36       145\n",
      "      RANDOM       0.91      0.79      0.85       428\n",
      "         SYN       0.33      0.91      0.48       129\n",
      "\n",
      "    accuracy                           0.61       838\n",
      "   macro avg       0.57      0.52      0.48       838\n",
      "weighted avg       0.69      0.61      0.60       838\n",
      "\n",
      "\t Weighted F1 (no random): 0.34921310531134203\n",
      "[[ 20   1  14 101]\n",
      " [  7  36  11  91]\n",
      " [ 19  18 340  51]\n",
      " [  4   0   8 117]]\n",
      "evaluate on en\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.22      0.13      0.16       308\n",
      "         HYP       0.39      0.22      0.28       292\n",
      "      RANDOM       0.62      0.86      0.72       877\n",
      "         SYN       0.35      0.22      0.27       259\n",
      "\n",
      "    accuracy                           0.53      1736\n",
      "   macro avg       0.39      0.36      0.36      1736\n",
      "weighted avg       0.47      0.53      0.48      1736\n",
      "\n",
      "\t Weighted F1 (no random): 0.2346543909446359\n",
      "[[ 40  36 201  31]\n",
      " [ 37  65 142  48]\n",
      " [ 62  32 757  26]\n",
      " [ 46  35 122  56]]\n",
      "evaluate on de\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.29      0.19      0.23       275\n",
      "         HYP       0.47      0.06      0.11       294\n",
      "      RANDOM       0.58      0.91      0.71       786\n",
      "         SYN       0.36      0.24      0.29       272\n",
      "\n",
      "    accuracy                           0.52      1627\n",
      "   macro avg       0.43      0.35      0.33      1627\n",
      "weighted avg       0.48      0.52      0.45      1627\n",
      "\n",
      "\t Weighted F1 (no random): 0.20701293263907533\n",
      "[[ 52   8 171  44]\n",
      " [ 32  18 189  55]\n",
      " [ 48   6 715  17]\n",
      " [ 46   6 154  66]]\n",
      "\u001b[1m  Validation Loss All: 1.11\u001b[0m\n",
      "\n",
      "\u001b[1m======== Epoch 2 / 7 ========\u001b[0m\n",
      "\n",
      "  Average training loss: 1.03\n",
      "  Training epoch took: 0:02:52\n",
      "evaluate on all\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.51      0.39      0.44       719\n",
      "         HYP       0.47      0.46      0.47       731\n",
      "      RANDOM       0.79      0.74      0.77      2091\n",
      "         SYN       0.41      0.60      0.48       660\n",
      "\n",
      "    accuracy                           0.61      4201\n",
      "   macro avg       0.54      0.55      0.54      4201\n",
      "weighted avg       0.63      0.61      0.61      4201\n",
      "\n",
      "\t Weighted F1 (no random): 0.46249658218205847\n",
      "[[ 279   66  194  180]\n",
      " [  57  336  114  224]\n",
      " [ 149  209 1554  179]\n",
      " [  66  101   96  397]]\n",
      "evaluate on zh\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.86      0.61      0.71       136\n",
      "         HYP       0.79      0.82      0.81       145\n",
      "      RANDOM       0.93      0.90      0.92       428\n",
      "         SYN       0.64      0.88      0.74       129\n",
      "\n",
      "    accuracy                           0.84       838\n",
      "   macro avg       0.81      0.80      0.79       838\n",
      "weighted avg       0.85      0.84      0.84       838\n",
      "\n",
      "\t Weighted F1 (no random): 0.7547869267220005\n",
      "[[ 83   4  16  33]\n",
      " [  2 119   7  17]\n",
      " [  6  23 386  13]\n",
      " [  6   4   6 113]]\n",
      "evaluate on en\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.48      0.38      0.42       308\n",
      "         HYP       0.41      0.42      0.41       292\n",
      "      RANDOM       0.77      0.71      0.74       877\n",
      "         SYN       0.38      0.55      0.45       259\n",
      "\n",
      "    accuracy                           0.58      1736\n",
      "   macro avg       0.51      0.52      0.51      1736\n",
      "weighted avg       0.60      0.58      0.58      1736\n",
      "\n",
      "\t Weighted F1 (no random): 0.42774524912317574\n",
      "[[116  24 103  65]\n",
      " [ 31 124  44  93]\n",
      " [ 74 106 622  75]\n",
      " [ 22  52  42 143]]\n",
      "evaluate on de\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.38      0.29      0.33       275\n",
      "         HYP       0.36      0.32      0.34       294\n",
      "      RANDOM       0.75      0.69      0.72       786\n",
      "         SYN       0.33      0.52      0.40       272\n",
      "\n",
      "    accuracy                           0.53      1627\n",
      "   macro avg       0.45      0.46      0.45      1627\n",
      "weighted avg       0.55      0.53      0.53      1627\n",
      "\n",
      "\t Weighted F1 (no random): 0.35616834343218273\n",
      "[[ 80  38  75  82]\n",
      " [ 24  93  63 114]\n",
      " [ 69  80 546  91]\n",
      " [ 38  45  48 141]]\n",
      "\u001b[1m  Validation Loss All: 0.97\u001b[0m\n",
      "\n",
      "\u001b[1m======== Epoch 3 / 7 ========\u001b[0m\n",
      "\n",
      "  Average training loss: 0.87\n",
      "  Training epoch took: 0:02:52\n",
      "evaluate on all\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.61      0.42      0.50       719\n",
      "         HYP       0.59      0.44      0.50       731\n",
      "      RANDOM       0.75      0.84      0.79      2091\n",
      "         SYN       0.44      0.54      0.48       660\n",
      "\n",
      "    accuracy                           0.65      4201\n",
      "   macro avg       0.60      0.56      0.57      4201\n",
      "weighted avg       0.65      0.65      0.64      4201\n",
      "\n",
      "\t Weighted F1 (no random): 0.49665186740720385\n",
      "[[ 305   39  236  139]\n",
      " [  35  321  186  189]\n",
      " [ 118   86 1765  122]\n",
      " [  40   97  169  354]]\n",
      "evaluate on zh\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.82      0.73      0.77       136\n",
      "         HYP       0.91      0.80      0.85       145\n",
      "      RANDOM       0.93      0.93      0.93       428\n",
      "         SYN       0.69      0.86      0.77       129\n",
      "\n",
      "    accuracy                           0.87       838\n",
      "   macro avg       0.84      0.83      0.83       838\n",
      "weighted avg       0.87      0.87      0.87       838\n",
      "\n",
      "\t Weighted F1 (no random): 0.7990626700638204\n",
      "[[ 99   2  13  22]\n",
      " [  6 116   9  14]\n",
      " [  8   7 399  14]\n",
      " [  7   2   9 111]]\n",
      "evaluate on en\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.59      0.39      0.47       308\n",
      "         HYP       0.48      0.39      0.43       292\n",
      "      RANDOM       0.74      0.82      0.78       877\n",
      "         SYN       0.39      0.49      0.43       259\n",
      "\n",
      "    accuracy                           0.62      1736\n",
      "   macro avg       0.55      0.52      0.53      1736\n",
      "weighted avg       0.62      0.62      0.61      1736\n",
      "\n",
      "\t Weighted F1 (no random): 0.4448649006530869\n",
      "[[119  20 113  56]\n",
      " [ 15 113  73  91]\n",
      " [ 53  48 720  56]\n",
      " [ 13  53  65 128]]\n",
      "evaluate on de\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.49      0.32      0.38       275\n",
      "         HYP       0.51      0.31      0.39       294\n",
      "      RANDOM       0.68      0.82      0.74       786\n",
      "         SYN       0.37      0.42      0.39       272\n",
      "\n",
      "    accuracy                           0.58      1627\n",
      "   macro avg       0.51      0.47      0.48      1627\n",
      "weighted avg       0.56      0.58      0.56      1627\n",
      "\n",
      "\t Weighted F1 (no random): 0.38810878188776454\n",
      "[[ 87  17 110  61]\n",
      " [ 14  92 104  84]\n",
      " [ 57  31 646  52]\n",
      " [ 20  42  95 115]]\n",
      "\u001b[1m  Validation Loss All: 0.88\u001b[0m\n",
      "\n",
      "\u001b[1m======== Epoch 4 / 7 ========\u001b[0m\n",
      "\n",
      "  Average training loss: 0.76\n",
      "  Training epoch took: 0:02:52\n",
      "evaluate on all\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.57      0.53      0.55       719\n",
      "         HYP       0.52      0.53      0.52       731\n",
      "      RANDOM       0.77      0.83      0.80      2091\n",
      "         SYN       0.49      0.41      0.45       660\n",
      "\n",
      "    accuracy                           0.66      4201\n",
      "   macro avg       0.59      0.57      0.58      4201\n",
      "weighted avg       0.65      0.66      0.65      4201\n",
      "\n",
      "\t Weighted F1 (no random): 0.5079433355646169\n",
      "[[ 379   52  217   71]\n",
      " [  62  386  154  129]\n",
      " [ 142  140 1729   80]\n",
      " [  80  165  145  270]]\n",
      "evaluate on zh\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.84      0.82      0.83       136\n",
      "         HYP       0.90      0.81      0.85       145\n",
      "      RANDOM       0.93      0.93      0.93       428\n",
      "         SYN       0.75      0.84      0.79       129\n",
      "\n",
      "    accuracy                           0.88       838\n",
      "   macro avg       0.85      0.85      0.85       838\n",
      "weighted avg       0.88      0.88      0.88       838\n",
      "\n",
      "\t Weighted F1 (no random): 0.8255445512030877\n",
      "[[112   1  13  10]\n",
      " [  6 117   8  14]\n",
      " [  7   9 399  13]\n",
      " [  9   3   8 109]]\n",
      "evaluate on en\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.55      0.49      0.52       308\n",
      "         HYP       0.44      0.49      0.46       292\n",
      "      RANDOM       0.76      0.82      0.79       877\n",
      "         SYN       0.42      0.32      0.36       259\n",
      "\n",
      "    accuracy                           0.63      1736\n",
      "   macro avg       0.54      0.53      0.53      1736\n",
      "weighted avg       0.62      0.63      0.62      1736\n",
      "\n",
      "\t Weighted F1 (no random): 0.4527289252403182\n",
      "[[151  24 107  26]\n",
      " [ 30 142  61  59]\n",
      " [ 62  71 715  29]\n",
      " [ 32  85  59  83]]\n",
      "evaluate on de\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.46      0.42      0.44       275\n",
      "         HYP       0.44      0.43      0.43       294\n",
      "      RANDOM       0.70      0.78      0.74       786\n",
      "         SYN       0.38      0.29      0.33       272\n",
      "\n",
      "    accuracy                           0.58      1627\n",
      "   macro avg       0.49      0.48      0.48      1627\n",
      "weighted avg       0.56      0.58      0.56      1627\n",
      "\n",
      "\t Weighted F1 (no random): 0.40052407625843633\n",
      "[[116  27  97  35]\n",
      " [ 26 127  85  56]\n",
      " [ 73  60 615  38]\n",
      " [ 39  77  78  78]]\n",
      "\u001b[1m  Validation Loss All: 0.88\u001b[0m\n",
      "\n",
      "\u001b[1m======== Epoch 5 / 7 ========\u001b[0m\n",
      "\n",
      "  Average training loss: 0.67\n",
      "  Training epoch took: 0:02:52\n",
      "evaluate on all\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.59      0.51      0.55       719\n",
      "         HYP       0.50      0.57      0.53       731\n",
      "      RANDOM       0.79      0.82      0.81      2091\n",
      "         SYN       0.51      0.43      0.47       660\n",
      "\n",
      "    accuracy                           0.67      4201\n",
      "   macro avg       0.60      0.59      0.59      4201\n",
      "weighted avg       0.66      0.67      0.66      4201\n",
      "\n",
      "\t Weighted F1 (no random): 0.5189183911142061\n",
      "[[ 370   79  200   70]\n",
      " [  51  417  144  119]\n",
      " [ 137  147 1723   84]\n",
      " [  70  185  120  285]]\n",
      "evaluate on zh\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.82      0.82      0.82       136\n",
      "         HYP       0.83      0.85      0.84       145\n",
      "      RANDOM       0.93      0.95      0.94       428\n",
      "         SYN       0.84      0.76      0.80       129\n",
      "\n",
      "    accuracy                           0.88       838\n",
      "   macro avg       0.86      0.85      0.85       838\n",
      "weighted avg       0.88      0.88      0.88       838\n",
      "\n",
      "\t Weighted F1 (no random): 0.8207964161274266\n",
      "[[112   5  13   6]\n",
      " [  6 123   9   7]\n",
      " [  8   9 406   5]\n",
      " [ 10  12   9  98]]\n",
      "evaluate on en\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.57      0.50      0.53       308\n",
      "         HYP       0.44      0.53      0.48       292\n",
      "      RANDOM       0.78      0.80      0.79       877\n",
      "         SYN       0.43      0.36      0.39       259\n",
      "\n",
      "    accuracy                           0.64      1736\n",
      "   macro avg       0.56      0.55      0.55      1736\n",
      "weighted avg       0.63      0.64      0.63      1736\n",
      "\n",
      "\t Weighted F1 (no random): 0.4719665419813136\n",
      "[[153  32  93  30]\n",
      " [ 23 156  55  58]\n",
      " [ 67  71 703  36]\n",
      " [ 25  93  49  92]]\n",
      "evaluate on de\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.47      0.38      0.42       275\n",
      "         HYP       0.42      0.47      0.44       294\n",
      "      RANDOM       0.72      0.78      0.75       786\n",
      "         SYN       0.42      0.35      0.38       272\n",
      "\n",
      "    accuracy                           0.59      1627\n",
      "   macro avg       0.51      0.50      0.50      1627\n",
      "weighted avg       0.57      0.59      0.58      1627\n",
      "\n",
      "\t Weighted F1 (no random): 0.4163772057005051\n",
      "[[105  42  94  34]\n",
      " [ 22 138  80  54]\n",
      " [ 62  67 614  43]\n",
      " [ 35  80  62  95]]\n",
      "\u001b[1m  Validation Loss All: 0.93\u001b[0m\n",
      "\n",
      "\u001b[1m======== Epoch 6 / 7 ========\u001b[0m\n",
      "\n",
      "  Average training loss: 0.61\n",
      "  Training epoch took: 0:02:52\n",
      "evaluate on all\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.60      0.51      0.55       719\n",
      "         HYP       0.54      0.56      0.55       731\n",
      "      RANDOM       0.77      0.84      0.81      2091\n",
      "         SYN       0.50      0.43      0.46       660\n",
      "\n",
      "    accuracy                           0.67      4201\n",
      "   macro avg       0.60      0.58      0.59      4201\n",
      "weighted avg       0.66      0.67      0.66      4201\n",
      "\n",
      "\t Weighted F1 (no random): 0.5227786963019768\n",
      "[[ 369   63  219   68]\n",
      " [  44  409  153  125]\n",
      " [ 126  127 1753   85]\n",
      " [  78  162  138  282]]\n",
      "evaluate on zh\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.81      0.83      0.82       136\n",
      "         HYP       0.85      0.85      0.85       145\n",
      "      RANDOM       0.93      0.95      0.94       428\n",
      "         SYN       0.84      0.77      0.80       129\n",
      "\n",
      "    accuracy                           0.89       838\n",
      "   macro avg       0.86      0.85      0.85       838\n",
      "weighted avg       0.88      0.89      0.88       838\n",
      "\n",
      "\t Weighted F1 (no random): 0.8248199502679605\n",
      "[[113   4  13   6]\n",
      " [  6 123   8   8]\n",
      " [  7   9 407   5]\n",
      " [ 13   9   8  99]]\n",
      "evaluate on en\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.57      0.49      0.53       308\n",
      "         HYP       0.47      0.50      0.49       292\n",
      "      RANDOM       0.77      0.81      0.79       877\n",
      "         SYN       0.43      0.38      0.40       259\n",
      "\n",
      "    accuracy                           0.64      1736\n",
      "   macro avg       0.56      0.55      0.55      1736\n",
      "weighted avg       0.63      0.64      0.63      1736\n",
      "\n",
      "\t Weighted F1 (no random): 0.4762121798502134\n",
      "[[152  25 101  30]\n",
      " [ 20 147  59  66]\n",
      " [ 65  65 711  36]\n",
      " [ 29  75  57  98]]\n",
      "evaluate on de\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.49      0.38      0.43       275\n",
      "         HYP       0.46      0.47      0.46       294\n",
      "      RANDOM       0.71      0.81      0.75       786\n",
      "         SYN       0.40      0.31      0.35       272\n",
      "\n",
      "    accuracy                           0.59      1627\n",
      "   macro avg       0.51      0.49      0.50      1627\n",
      "weighted avg       0.57      0.59      0.58      1627\n",
      "\n",
      "\t Weighted F1 (no random): 0.41577476115920653\n",
      "[[104  34 105  32]\n",
      " [ 18 139  86  51]\n",
      " [ 54  53 635  44]\n",
      " [ 36  78  73  85]]\n",
      "\u001b[1m  Validation Loss All: 0.93\u001b[0m\n",
      "\n",
      "\u001b[1m======== Epoch 7 / 7 ========\u001b[0m\n",
      "\n",
      "  Average training loss: 0.56\n",
      "  Training epoch took: 0:02:52\n",
      "evaluate on all\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.61      0.53      0.56       719\n",
      "         HYP       0.56      0.54      0.55       731\n",
      "      RANDOM       0.79      0.82      0.81      2091\n",
      "         SYN       0.47      0.52      0.49       660\n",
      "\n",
      "    accuracy                           0.67      4201\n",
      "   macro avg       0.61      0.60      0.60      4201\n",
      "weighted avg       0.67      0.67      0.67      4201\n",
      "\n",
      "\t Weighted F1 (no random): 0.5369758154649746\n",
      "[[ 380   56  191   92]\n",
      " [  42  395  134  160]\n",
      " [ 138  119 1707  127]\n",
      " [  67  137  116  340]]\n",
      "evaluate on zh\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.83      0.84      0.83       136\n",
      "         HYP       0.87      0.84      0.86       145\n",
      "      RANDOM       0.94      0.94      0.94       428\n",
      "         SYN       0.82      0.82      0.82       129\n",
      "\n",
      "    accuracy                           0.89       838\n",
      "   macro avg       0.86      0.86      0.86       838\n",
      "weighted avg       0.89      0.89      0.89       838\n",
      "\n",
      "\t Weighted F1 (no random): 0.8363389455600133\n",
      "[[114   3  10   9]\n",
      " [  6 122   9   8]\n",
      " [  8  10 403   7]\n",
      " [ 10   5   8 106]]\n",
      "evaluate on en\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.59      0.50      0.54       308\n",
      "         HYP       0.47      0.47      0.47       292\n",
      "      RANDOM       0.78      0.80      0.79       877\n",
      "         SYN       0.41      0.46      0.43       259\n",
      "\n",
      "    accuracy                           0.64      1736\n",
      "   macro avg       0.57      0.56      0.56      1736\n",
      "weighted avg       0.64      0.64      0.64      1736\n",
      "\n",
      "\t Weighted F1 (no random): 0.48586880489148015\n",
      "[[153  24  92  39]\n",
      " [ 16 138  53  85]\n",
      " [ 67  60 701  49]\n",
      " [ 22  69  48 120]]\n",
      "evaluate on de\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ANT       0.49      0.41      0.45       275\n",
      "         HYP       0.49      0.46      0.47       294\n",
      "      RANDOM       0.73      0.77      0.75       786\n",
      "         SYN       0.39      0.42      0.40       272\n",
      "\n",
      "    accuracy                           0.59      1627\n",
      "   macro avg       0.52      0.51      0.52      1627\n",
      "weighted avg       0.59      0.59      0.59      1627\n",
      "\n",
      "\t Weighted F1 (no random): 0.44146533108538394\n",
      "[[113  29  89  44]\n",
      " [ 20 135  72  67]\n",
      " [ 63  49 603  71]\n",
      " [ 35  63  60 114]]\n",
      "\u001b[1m  Validation Loss All: 0.98\u001b[0m\n",
      "\n",
      "\n",
      "Training complete!\n",
      "Total training took 0:23:41 (h:mm:ss)\n"
     ]
    }
   ],
   "source": [
    "# start training\n",
    "validation_dataloader_set=[validation_dataloader_all, validation_dataloader_zh, validation_dataloader_en, validation_dataloader_de]\n",
    "training_stats=train_model(epochs=epochs, \n",
    "                           xlmr_model=xlmr_model, \n",
    "                           train_dataloader=train_dataloader, \n",
    "                           validation_dataloader_set=validation_dataloader_set,\n",
    "                           random_seed=42,\n",
    "                           verbose=True)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-fjiw0rymj7U"
   },
   "source": [
    "# Evaluation Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 331
    },
    "id": "ICm4zOH-Q5p2",
    "outputId": "aa1fbb7d-e97e-42ae-9ab2-35821dc0673c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Valid. Loss all</th>\n",
       "      <th>Valid. Accur. all</th>\n",
       "      <th>Weigh_F1 all (no random)</th>\n",
       "      <th>Macro F1 all</th>\n",
       "      <th>Weigh_F1 en</th>\n",
       "      <th>Weigh_F1 de</th>\n",
       "      <th>Weigh_F1 zh</th>\n",
       "      <th>Training Time</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epoch</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.24</td>\n",
       "      <td>1.11</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0:02:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.03</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0:02:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.87</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.57</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0:02:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.76</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0:02:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.67</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0:02:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.61</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0:02:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.56</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0:02:52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Training Loss  Valid. Loss all  ...  Weigh_F1 zh  Training Time\n",
       "epoch                                  ...                            \n",
       "1               1.24             1.11  ...         0.35        0:02:49\n",
       "2               1.03             0.97  ...         0.75        0:02:52\n",
       "3               0.87             0.88  ...         0.80        0:02:52\n",
       "4               0.76             0.88  ...         0.83        0:02:52\n",
       "5               0.67             0.93  ...         0.82        0:02:52\n",
       "6               0.61             0.93  ...         0.82        0:02:52\n",
       "7               0.56             0.98  ...         0.84        0:02:52\n",
       "\n",
       "[7 rows x 9 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Table with training stats\n",
    "pd.set_option('precision', 2)\n",
    "df_stats = pd.DataFrame(data=training_stats)\n",
    "df_stats = df_stats.set_index('epoch')\n",
    "df_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 418
    },
    "id": "lXGHvyQMQ8vB",
    "outputId": "d7dbf0fa-2630-4b36-d8fb-6bde207f5d0b"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAGaCAYAAACopj13AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdZ0AU19rA8f/uUqV3EKwgoDSxUxSxG7tiiS1RE0ssucnNNcUUk1zz5qYZNYkmauxd7GKvsWFQgxpbxIo0RQVBpe2+HwzEFVRAYFd8fl/u5czMmWfnbNZnzz5zRqHRaDQIIYQQQgghdEap6wCEEEIIIYR40UlSLoQQQgghhI5JUi6EEEIIIYSOSVIuhBBCCCGEjklSLoQQQgghhI5JUi6EEEIIIYSOSVIuhKi04uPj8fLyYtq0aaXu47333sPLy6sMo6q8Hne9vby8eO+994rVx7Rp0/Dy8iI+Pr7M41u1ahVeXl5ER0eXed9CCPGsDHQdgBDixVGS5HbHjh24ubmVYzTPn7t37zJjxgyioqJISUnB1taWhg0b8sYbb+Du7l6sPsaNG8eWLVtYs2YNdevWLXIfjUZD69atSU9PZ9++fZiYmJTlyyhX0dHRHD58mFdeeQVLS0tdh1NIfHw8rVu3ZsCAAXz88ce6DkcIoUckKRdCVJivvvpK6+8jR46wbNky+vbtS8OGDbW22draPvP5XF1dOX78OCqVqtR9fP7553z66afPHEtZ+PDDD9m4cSOdO3emSZMmXL9+nZ07dxIbG1vspDwiIoItW7YQGRnJhx9+WOQ+hw4d4tq1a/Tt27dMEvLjx4+jVFbMD7OHDx/mhx9+oEePHoWS8m7dutGpUycMDQ0rJBYhhCgJScqFEBWmW7duWn/n5eWxbNky6tevX2jbozIyMjA3Ny/R+RQKBcbGxiWO82H6ksDdu3ePzZs3ExoayrffflvQPmbMGLKzs4vdT2hoKC4uLqxfv57x48djZGRUaJ9Vq1YBDxL4svCsY1BWVCrVM31BE0KI8iQ15UIIvdOqVSsGDRrEqVOnGDZsGA0bNqRr167Ag+R88uTJ9O7dm6ZNm+Lr60vbtm355ptvuHfvnlY/RdU4P9y2a9cuevXqhZ+fH6Ghofzvf/8jNzdXq4+iasrz2+7cucMnn3xCUFAQfn5+9OvXj9jY2EKv59atW7z//vs0bdqUwMBABg8ezKlTpxg0aBCtWrUq1jVRKBQoFIoivyQUlVg/jlKppEePHty+fZudO3cW2p6RkcHWrVvx9PTE39+/RNf7cYqqKVer1fz888+0atUKPz8/OnfuzLp164o8Pi4ujokTJ9KpUycCAwMJCAigZ8+erFixQmu/9957jx9++AGA1q1b4+XlpTX+j6spv3nzJp9++ilhYWH4+voSFhbGp59+yq1bt7T2yz/+4MGDzJ49mzZt2uDr60v79u1ZvXp1sa5FSZw5c4bRo0fTtGlT/Pz8eOmll5g5cyZ5eXla+yUmJvL+++8THh6Or68vQUFB9OvXTysmtVrN3Llz6dKlC4GBgTRo0ID27dvzwQcfkJOTU+axCyFKTmbKhRB6KSEhgVdeeYUOHTrQrl077t69C0BycjIrV66kXbt2dO7cGQMDAw4fPsysWbM4ffo0s2fPLlb/e/bsYfHixfTr149evXqxY8cOfv31V6ysrBg5cmSx+hg2bBi2traMHj2a27dvM2fOHIYPH86OHTsKZvWzs7MZMmQIp0+fpmfPnvj5+XH27FmGDBmClZVVsa+HiYkJ3bt3JzIykg0bNtC5c+diH/uonj17Mn36dFatWkWHDh20tm3cuJH79+/Tq1cvoOyu96P+7//+j/nz59O4cWNeffVVUlNT+eyzz6hWrVqhfQ8fPkxMTAwtW7bEzc2t4FeDDz/8kJs3bzJixAgA+vbtS0ZGBtu2beP999/HxsYGePK9DHfu3OHll1/m8uXL9OrVi3r16nH69GmWLFnCoUOHWLFiRaFfaCZPnsz9+/fp27cvRkZGLFmyhPfee4/q1asXKsMqrRMnTjBo0CAMDAwYMGAA9vb27Nq1i2+++YYzZ84U/FqSm5vLkCFDSE5Opn///tSsWZOMjAzOnj1LTEwMPXr0AGD69OlMnTqV8PBw+vXrh0qlIj4+np07d5Kdna03vwgJ8ULTCCGEjkRGRmo8PT01kZGRWu3h4eEaT09PzfLlywsdk5WVpcnOzi7UPnnyZI2np6cmNja2oO3q1asaT09PzdSpUwu1BQQEaK5evVrQrlarNZ06ddKEhIRo9fvuu+9qPD09i2z75JNPtNqjoqI0np6emiVLlhS0LVy4UOPp6an56aeftPbNbw8PDy/0Wopy584dzeuvv67x9fXV1KtXT7Nx48ZiHfc4gwcP1tStW1eTnJys1d6nTx+Nj4+PJjU1VaPRPPv11mg0Gk9PT827775b8HdcXJzGy8tLM3jwYE1ubm5B+8mTJzVeXl4aT09PrbHJzMwsdP68vDzNwIEDNQ0aNNCKb+rUqYWOz5f/fjt06FBB23fffafx9PTULFy4UGvf/PGZPHlyoeO7deumycrKKmhPSkrS+Pj4aN56661C53xU/jX69NNPn7hf3759NXXr1tWcPn26oE2tVmvGjRun8fT01Bw4cECj0Wg0p0+f1nh6emp++eWXJ/bXvXt3TceOHZ8anxBCd6R8RQihl6ytrenZs2ehdiMjo4JZvdzcXNLS0rh58ybBwcEARZaPFKV169Zaq7soFAqaNm3K9evXyczMLFYfr776qtbfzZo1A+Dy5csFbbt27UKlUjF48GCtfXv37o2FhUWxzqNWq3nzzTc5c+YMmzZtokWLFrzzzjusX79ea7+PPvoIHx+fYtWYR0REkJeXx5o1awra4uLi+OOPP2jVqlXBjbZldb0ftmPHDjQaDUOGDNGq8fbx8SEkJKTQ/lWqVCn4/1lZWdy6dYvbt28TEhJCRkYGFy5cKHEM+bZt24atrS19+/bVau/bty+2trZs37690DH9+/fXKhlycnKiVq1aXLp0qdRxPCw1NZVjx47RqlUrvL29C9oVCgWjRo0qiBsoeA9FR0eTmpr62D7Nzc1JTk4mJiamTGIUQpQ9KV8RQuilatWqPfamvEWLFrF06VLOnz+PWq3W2paWllbs/h9lbW0NwO3btzEzMytxH/nlErdv3y5oi4+Px9HRsVB/RkZGuLm5kZ6e/tTz7Nixg3379vH111/j5ubGlClTGDNmDOPHjyc3N7egROHs2bP4+fkVq8a8Xbt2WFpasmrVKoYPHw5AZGQkQEHpSr6yuN4Pu3r1KgC1a9cutM3d3Z19+/ZptWVmZvLDDz+wadMmEhMTCx1TnGv4OPHx8fj6+mJgoP3PoYGBATVr1uTUqVOFjnnce+fatWuljuPRmAA8PDwKbatduzZKpbLgGrq6ujJy5Eh++eUXQkNDqVu3Ls2aNaNDhw74+/sXHPf2228zevRoBgwYgKOjI02aNKFly5a0b9++RPckCCHKjyTlQgi9ZGpqWmT7nDlz+PLLLwkNDWXw4ME4OjpiaGhIcnIy7733HhqNplj9P2kVjmfto7jHF1f+jYmNGzcGHiT0P/zwA6NGjeL9998nNzcXb29vYmNjmTRpUrH6NDY2pnPnzixevJijR48SEBDAunXrcHZ2pnnz5gX7ldX1fhb//ve/2b17N3369KFx48ZYW1ujUqnYs2cPc+fOLfRFobxV1PKOxfXWW28RERHB7t27iYmJYeXKlcyePZvXXnuN//znPwAEBgaybds29u3bR3R0NNHR0WzYsIHp06ezePHigi+kQgjdkaRcCPFcWbt2La6ursycOVMrOdq7d68Oo3o8V1dXDh48SGZmptZseU5ODvHx8cV6wE3+67x27RouLi7Ag8T8p59+YuTIkXz00Ue4urri6elJ9+7dix1bREQEixcvZtWqVaSlpXH9+nVGjhypdV3L43rnzzRfuHCB6tWra22Li4vT+js9PZ3du3fTrVs3PvvsM61tBw4cKNS3QqEocSwXL14kNzdXa7Y8NzeXS5cuFTkrXt7yy6rOnz9faNuFCxdQq9WF4qpWrRqDBg1i0KBBZGVlMWzYMGbNmsXQoUOxs7MDwMzMjPbt29O+fXvgwS8gn332GStXruS1114r51clhHga/fq6L4QQT6FUKlEoFFoztLm5ucycOVOHUT1eq1atyMvLY/78+Vrty5cv586dO8XqIywsDHiw6sfD9eLGxsZ89913WFpaEh8fT/v27QuVYTyJj48PdevWJSoqikWLFqFQKAqtTV4e17tVq1YoFArmzJmjtbzfn3/+WSjRzv8i8OiMfEpKSqElEeGf+vPiltW0adOGmzdvFupr+fLl3Lx5kzZt2hSrn7JkZ2dHYGAgu3bt4ty5cwXtGo2GX375BYC2bdsCD1aPeXRJQ2Nj44LSoPzrcPPmzULn8fHx0dpHCKFbMlMuhHiudOjQgW+//ZbXX3+dtm3bkpGRwYYNG0qUjFak3r17s3TpUr7//nuuXLlSsCTi5s2bqVGjRqF10YsSEhJCREQEK1eupFOnTnTr1g1nZ2euXr3K2rVrgQcJ1o8//oi7uzsdO3YsdnwRERF8/vnn/PbbbzRp0qTQDGx5XG93d3cGDBjAwoULeeWVV2jXrh2pqaksWrQIb29vrTpuc3NzQkJCWLduHSYmJvj5+XHt2jWWLVuGm5ubVv0+QEBAAADffPMNXbp0wdjYmDp16uDp6VlkLK+99hqbN2/ms88+49SpU9StW5fTp0+zcuVKatWqVW4zyCdPnuSnn34q1G5gYMDw4cOZMGECgwYNYsCAAfTv3x8HBwd27drFvn376Ny5M0FBQcCD0qaPPvqIdu3aUatWLczMzDh58iQrV64kICCgIDl/6aWXqF+/Pv7+/jg6OnL9+nWWL1+OoaEhnTp1KpfXKIQoGf38V0wIIR5j2LBhaDQaVq5cyaRJk3BwcKBjx4706tWLl156SdfhFWJkZMS8efP46quv2LFjB5s2bcLf35+5c+cyYcIE7t+/X6x+Jk2aRJMmTVi6dCmzZ88mJycHV1dXOnTowNChQzEyMqJv37785z//wcLCgtDQ0GL126VLF7766iuysrIK3eAJ5Xe9J0yYgL29PcuXL+err76iZs2afPzxx1y+fLnQzZVff/013377LTt37mT16tXUrFmTt956CwMDA95//32tfRs2bMg777zD0qVL+eijj8jNzWXMmDGPTcotLCxYsmQJU6dOZefOnaxatQo7Ozv69evH2LFjS/wU2eKKjY0tcuUaIyMjhg8fjp+fH0uXLmXq1KksWbKEu3fvUq1aNd555x2GDh1asL+Xlxdt27bl8OHDrF+/HrVajYuLCyNGjNDab+jQoezZs4cFCxZw584d7OzsCAgIYMSIEVorvAghdEehqYi7dIQQQmjJy8ujWbNm+Pv7l/oBPEIIISoPqSkXQohyVtRs+NKlS0lPTy9yXW4hhBAvHilfEUKIcvbhhx+SnZ1NYGAgRkZGHDt2jA0bNlCjRg369Omj6/CEEELoASlfEUKIcrZmzRoWLVrEpUuXuHv3LnZ2doSFhfHmm29ib2+v6/CEEELoAUnKhRBCCCGE0DGpKRdCCCGEEELHJCkXQgghhBBCx+RGz7/dupWJWl2xlTx2duakpmZU6DnF08m46B8ZE/0k46J/ZEz0k4yL/tHVmCiVCmxszIrcJkn539RqTYUn5fnnFfpHxkX/yJjoJxkX/SNjop9kXPSPvo2JlK8IIYQQQgihY5KUCyGEEEIIoWOSlAshhBBCCKFjkpQLIYQQQgihY5KUCyGEEEIIoWOy+ooQQgjxt9zcHDIz08nKuodanafrcEosJUWJWq3WdRjiETIu+qesx0SlMsTc3ApT06KXOywOScqFEEIIHiTkN28mU6WKBba2zqhUKhQKha7DKhEDAyW5uZL86RsZF/1TlmOi0WjIycni9u0bGBgYYmhoVKp+pHxFCCGEADIz06lSxQJzcysMDAyeu4RcCKEbCoUCIyMTzMysyMi4Xep+JCkXQgghgKyse5iYlP6nZyHEi83ExJScnOxSHy/lKzpw8M8kVu2J42Z6FraWxvQMcyfIx1nXYQkhxAtNrc5DpVLpOgwhxHNKqVQ9070okpRXsIN/JjFv0xmy/65jSk3PYt6mMwCSmAshhI5JyYoQorSe9fNDylcq2Ko9cQUJeb7sXDWr9sTpKCIhhBBCCKFrkpRXsNT0rBK1CyGEEPpuzJjhjBkzvMKPFaIykfKVCmZnaVxkAq4AtsdcJbyBKyqlfFcSQgjx7EJDGxVrvxUr1uHiUrWcoxFCPIlCo9FodB2EPkhNzUCtLv9L8WhNOYChgRJHa1Ou3cikuqM5g9p74e5qVe6xiKI5OFhw/fodXYchHiJjop8q27gkJV3G2bmGrsN4Jo+uvbxlS5TW9uXLl5CcnMjYsW9rtbdoEY6pqWmpz5uTkwOAoaFhhR77vJB1yvVPeY3J0z5HlEoFdnbmRcdU5tGIJ8q/mfPR1Vea1XPiyNnrLNnxF5MWHKG5vwsRLd2xqFK6BeiFEEKI9u1f0vp79+4dpKXdLtT+qPv372NiYlLs8zxLQl2Zk3EhSkKSch0I8nEmyMe50CxTI29HfGvbsm7/Jbb9fpWj567Tq6U7LQKqopQVAYQQQpSDMWOGk5GRwfjxHzBt2mTOnj3DgAGDGTZsBL/9tpt161Zz7txZ0tPTcHBw5KWXujBo0BCt5SPza8J/+OEXAI4ejWHcuJFMmvQVFy9eYM2aSNLT0/DzC+A///kAN7dqZXIsQGTkcpYuXURq6g3c3d0ZM+YtZs6crtWnEM8DnSblKSkpzJ8/n9jYWE6ePMndu3eZP38+TZs2feJxarWa1atXs23bNk6fPk1aWhpubm507tyZoUOHYmT0/M4umxgZ0CfcgxBfZxZuPcf8zWf5LTaBQe29qOlsqevwhBBClED+cylS07Ow0+PnUty+fYvx49+iXbsOdOjQCSenBzFGRW3A1LQKffsOoEoVU44ciWHWrBlkZmYyevSbT+133rzZKJUq+vcfzJ076SxZsoBPP/2QmTPnlcmxq1evZPLkr6hfvwF9+75MYmIi77//DhYWFjg4OJb+ggihAzpNyi9evMjMmTOpUaMGXl5eHDt2rFjH3bt3jw8++ID69evTr18/7OzsOHbsGFOmTOHQoUPMnTu3fAOvAK4O5ozvH0j0qWSW7TzP53NjaBnoSs+w2piZyE99Qgih756n51LcuHGd9977iM6du2m1T5z4X4yN/ylj6d49gq+//oLVq1fw+uujnjoJlpuby6+/zsPA4EG6YWlpxZQp33Dhwnlq1/Z4pmNzcnKYNWs6Pj5+fP/9TwX7eXjUYdKkiZKUi+eOTpNyHx8fDh06hI2NDdu3b2f06NHFOs7Q0JAlS5bQoEGDgrY+ffrg6urKtGnTiI6Ofups+/NAoVDQzMcZf3d71uy7wI4j8cScTaF3Sw+C/ZylpEUIISrA/hOJ7DueWOLj4hLSyM3TXkAgO1fNnKjT7P0jocT9hfq7EOLnUuLjisPExIQOHToVan84Ib97N5Ps7BwCAgJZu3YVly9fok4dzyf226lT14JkGSAgoD4ACQnXnpqUP+3YM2dOkZaWxhtv9NDar23bDkyd+t0T+xZCH+k0KTc3L/ru06cxMjLSSsjztW3blmnTphEXF1cpkvJ8VUwM6N/Gk1A/FxZuPcevUafZezyBQe28qOZYumsohBCifD2akD+tXZccHBy1Ett8Fy7EMXPmdI4e/Z3MzEytbZmZGU/tN78MJp+FxYMyzDt3nr5qz9OOTUp68EXp0RpzAwMDXFzK58uLEOWpUt3oeePGDQBsbGx0HEn5qO5kwXsDG7D/RCIrdsXx6Zzfad3Qje7Na2FqXKmGUggh9EaIX+lmqP/z0/4in0thZ2nMuwMKTyzp0sMz4vnu3LnD2LHDqVLFnGHDRuLq6oaRkRHnzp1h+vRpqNVPX05OqVQV2V6c1Zif5VghnkeVKpObNWsWFhYWhIaG6jqUcqNUKGjuX5XAOg6s2nuB7TFXOXw6mb6tPGhazwmFlLQIIYRe6BnmXui5FEYGSnqGueswquI7duwIaWlpTJr0NfXr//MlIjGx5KU35cHZ+cEXpfj4qwQEBBa05+bmkpiYiLv7k8tjhNA3lSYpnzFjBgcOHOCzzz7DwsKixMc/biH38ubgUPJYARyAfw+0pWuYO9Mjj/PL+lMcOp3CiB5+VJdVWp5ZacdFlB8ZE/1UmcYlJUWJgUHZPVG5eUBVVCoFK3bFkZp2HzsrE3qHuxPsW76lFU96DfkTNw/vo1AoUCgKH2do+GCmWqlUFGzLyclhzZqVAKhU/1yvR/tVqfL/V6HVb377w32W9lhfX1+srKxZv341nTp1Lii/2bJlC3fupKNQKMp0PJ+VPsUiHiiPMVEqlaX+XKwUSXlUVBTff/89ffv2pW/fvqXqo6Ke6PmwsnganrWJAe++HMje2AQi98Qx7tvdtGtcjS4hNTExqhTDW+Eq21MKKwMZE/1U2cZFrVaX+RP+mng70cTbSautPJ/s+LSnFOaXfjy8j0ajQaMpHFe9en5YWFjy2WcfExHRF4VCwZYtUQX/Vubl/XO9Hu03Ly//fzVa/ea3q9WaZz5WoVAxdOjrTJ78NWPGjCQ8vDWJiYls2rQeV1e3Qq9Tl+SJnvqnvMZErVY/8XPxSU/0fO6/tu3fv5/x48cTHh7OJ598outwdEKpVNAy0JVJw5sR5OvMpugrTJgZTcyZFKm9E0IIUSpWVtZ89dVk7OzsmTlzOkuWLKRRo6a88cY4XYdWoFevvvzrX++QlJTIjz9OITb2GF9++R3m5hYYGRnrOjwhSkSh0ZOsLX9JxOI8PChfbGwsr776Kt7e3syZM6dEjwR+1PM6U16U8/FpLNh6lqspGfjUsmVgW0+cbKuU+Xkqq8o2+1cZyJjop8o2LklJl3F2rqHrMJ6JzMg+mKns3LktYWHhvPvuh7oOB5Bx0UflNSZP+xx57mfKr1y5wpUrV7Ta4uLiGD58OK6ursyYMeOZEvLKxsPNio9fbcTLbepwISGNj2ZHs2rvBbJy8nQdmhBCCFFmsrIKr26zefNG0tPTCAxsqIOIhCg9nRcd//TTT8CDJBtg7dq1HDlyBEtLSwYOHAjAq6++CsDOnTsByMjIYNiwYaSnpzNs2DB2796t1aeXlxfe3t4V8wL0lEqppG2jajTxdmT5rvNsOHCJQ38m0b+NJ/Xr2Os6PCGEEOKZHT/+B9OnT6Nly1ZYWlpx7twZNm5cR+3a7oSHt9F1eEKUiM6T8ilTpmj9HRkZCYCrq2tBUv6o27dvk5j44KEB3377baHtY8aMeeGT8nxW5sa83sWHFgFVWbj1HFMjj1Pfw56X29TBwdpU1+EJIYQQpVa1qiv29g6sXLmM9PQ0LC2t6NChEyNHjsHQ0FDX4QlRInpTU65rlamm/HFy89Rsj4ln7b6LqDUaOgXVoGPT6hgaFP2AhhdVZauTrQxkTPRTZRsXqSkX5UXGRf/oY025zmfKRcUxUCnp0LQ6Teo6smznedb8dpEDJ5MY2NYT39p2ug5PCCGEEOKF9Vzc6CnKlq2lCaO6+/LvvvVRKBR8tzyWH1ef4Gb6fV2HJoQQQgjxQpKk/AXmU8uWz4Y2oWeL2pyIS2XCzGg2HbpMbp78xCaEEEIIUZGkfOUFZ2igpHNwTZrVc2LJjr9YsTuOfScSGdjOi7o1bHQdnhBCCCHEC0FmygUA9tamjO3lz5sR/uTmqfl6yTF+XvcntzMKrwErhBBCCCHKlsyUCy0BHvbUrWFD1KHLRB26Quz5G3RvXpvWDV1RKeU7nBBCCCFEeZAsSxRiZKiie/PafP5aE+q4WbN0x198OieGv+Jv6zo0IYQQQohKSZJy8VhONlX4V29/Rvfw415WDv+38CizN5wiPTNb16EJIYTQgaio9YSGNiIxMaGgLSKiC5MmTSzVsc/q6NEYQkMbcfRoTJn1KYSuSFIunkihUNDQy4H/vtaMTkE1OHQqmQ9+OcTOo/EV/rAlIYQQJTN+/Fu0aRPKvXv3HrvP22+PoX37MLKy9Pceou3bt7B8+WJdhyFEuZKkXBSLsZGKXmHufDasCTWcLVi49Ryfz4/hQkK6rkMTQgjxGG3btuf+/fvs27enyO23bt3kyJHfadEiHGNj41KdY/HiSN5998NnCfOpduzYyvLlSwq116/fgB079lO/foNyPb8QFUGSclEiLnZmvNOvPiO7+ZCWkcWk+THM23yGjHs5ug5NCCHEI5o3b4mpaRW2b99S5PadO7eTl5dHu3YdSn0OIyMjDAx0s26EUqnE2NgYpSxEICoBWX1FlJhCoaBJXSf8atuxdt9FtsfEc+TsdSJauhPq74JSodB1iEIIIQATExOaNw9j167tpKenY2lpqbV9+/Yt2NnZUa1aDb755kuOHDlMcnIyJiYmNGjQiNGj38TFpeoTzxER0YXAwIZMmDCxoO3ChTi+//5rTp48gZWVFd269cTe3qHQsb/9tpt161Zz7txZ0tPTcHBw5KWXujBo0BBUKhUAY8YM548/jgIQGtoIAGdnF1auXM/RozGMGzeSqVNn0KBBo4J+d+zYysKFc7l8+RJVqpgREtKcUaPGYW1tXbDPmDHDycjI4OOPP+O7777i9Ok/sbCwpHfvfgwY8ErJLrQQZUCSclFqpsYG9Gtdh1A/FxZuPcvcTWf4LTaBge28qOFsoevwhBBC5w4nHWVd3GZuZd3Gxtiaru4daOJcsaUWbdt2YOvWTezevYOuXXsUtCclJXLy5HEiIvpx+vSfnDx5nDZt2uPg4EhiYgJr1kQyduwIFi5cgYmJSbHPl5p6g3HjRqJWqxk48BVMTExZt251keUxUVEbMDWtQt++A6hSxZQjR2KYNWsGmZmZjB79JgCvvDKUe/fukZycyNixbwNgalrlseePilrPF198io+PH6NGjSMlJZnIyGWcPv0nM2fO14ojPT2Nf/97HOHhrWnduh27dm1n+vRp1K7tQVBQSLFfsxBlQZJy8czcHM15d0ADDv6ZxPOzmI0AACAASURBVPKd5/ls3u+0CnSjR4taVDEx1HV4QgihE4eTjrL4TCQ56gflfbeybrP4TCRAhSbmjRs3xdrahu3bt2gl5du3b0Gj0dC2bXvc3T0ID2+jdVxISAtGjhzC7t076NChU7HPt2jRPNLSbjNr1gK8vLwB6NixMy+/3KPQvhMn/hdj438S/u7dI/j66y9YvXoFr78+CiMjIxo3bsaqVStIS7tN+/YvPfHcubm5TJ8+DQ8PT6ZN+xkjIyMAvLy8mThxAuvXryYiol/B/ikpyXzyyX9p2/ZB+U7nzt2IiOjMxo1rJSkXFU6SclEmFAoFwb4u1PewZ/Xei+w8Fs/vZ5LpHe5BsK8zCilpEUI8p6ITj3Aw8fcSH3cx7Qq5mlytthx1DotOr+RAwuES9xfk0pimLg1LfJyBgQGtWrVhzZpIbty4gb29PQDbt2/Fza0a9er5au2fm5tLZmYGbm7VMDe34Ny5MyVKyg8e3I+fX0BBQg5gY2ND27YdWb16hda+Dyfkd+9mkp2dQ0BAIGvXruLy5UvUqeNZotd65swpbt26WZDQ52vVqi0//jiFAwf2ayXl5ubmtGnTvuBvQ0ND6tb1ISHhWonOK0RZkKRclKkqJoYMaOdJqL8LC7aeZfbG0w9KWtp74eZgruvwhBCiwjyakD+tvTy1bduBVatWsHPnVvr06c+lSxc5f/4cQ4a8DkBW1n0WLJhLVNR6rl9PQaP5Z8nbjIyMEp0rOTkJP7+AQu3Vq9co1HbhQhwzZ07n6NHfyczM1NqWmVmy88KDkpyizqVUKnFzq0ZycqJWu6OjU6FJIwsLS+Lizpf43EI8K0nKdSC/xvB21m2sdVRjWN5qOFvwwaCG7DueyMrdcUz89XfaNHKjW2gtTI3lbSeEeH40dWlYqhnqD/d/wa2swk9CtjG25l8NRpZFaMXm5xeAi4sr27Ztpk+f/mzbthmgoGxj8uSviYpaT+/eL+Pr64e5uTmgYOLED7QS9LJ0584dxo4dTpUq5gwbNhJXVzeMjIw4d+4M06dPQ61Wl8t5H6ZUqopsL6/XLMSTSHZUwfSlxrAiKBUKWgRUpYGnA5F74tj2+1UOn06mX+s6NPZ2lJIWIUSl1tW9g9bnPYCh0pCu7qVffvBZtGnTjgUL5hAff5UdO7bi5VW3YEY5v2587Ni3CvbPysoq8Sw5gJOTM/HxVwu1X7lyWevvY8eOkJaWxqRJX2utM170Ez+L9++Fs7NLwbke7lOj0RAff5VatdyL1Y8QuiALe1awdXGbtT6g4UGN4bq4zTqKqPyZmxrySgdvJgxuhJWZMTPW/sk3S/8gMTXz6QcLIcRzqolzA/p798LG+MEyfDbG1vT37qWzCZh27ToC8MMPk4mPv6q1NnlRM8aRkcvIy8sr8XmCgkI4cSKWs2fPFLTdunWLbds2ae2Xv7b4w7PSOTk5herOAUxNTYv1BcHbux42NrasWbOSnJx//q3dtWsH16+nEBwsN28K/SUz5RWsqJ8y89vXnI+iuWsQdqY2FRxVxahd1ZKPXmnE7j+usWrPBT6efZj2TarTJbgmxkZF/4QohBDPsybODfTmV9BatWrj4eHJvn17USqVtG79zw2OwcGhbNkShZmZOTVr1uLPP08QE3MYKyurEp+nf/9X2LIlirffHk1ERD+MjU1Yt241Tk4uZGT8VbCfn58/FhaWTJo0kYiIvigUCrZsiaKoyhEvL2+2bt3EtGnf4e1dD1PTKoSGtii0n4GBAaNGjeWLLz5l7NgRtGnTjpSUZFauXEbt2u506VJ4BRgh9IUk5RXMxti6yMTcUGnI9it72H5lD/4OPrR0C6aOtXulK/FQKhW0auBGIy9HVuw6T9Shy0SfSqJfa08aeNpXutcrhBD6pF27Dpw/f47AwIYFq7AAvPnmOyiVSrZt20RWVjZ+fgF8//2PvP322BKfw97enqlTf2by5K9YsGCu1sODvvzy84L9rKys+eqryfzww/fMnDkdCwtL2rXrSKNGTXj77TFafXbr1otz584QFbWBZcsW4+zsUmRSDvDSS10wMjJi0aJ5/PjjFMzMzGjbtgMjR44tcq10IfSFQiN3MwCQmpqBWl3+l+LRmnJ4kJD39+6Fh3Utfrt2iP0J0WTm3KWqmTNhbsE0dm6AscroCb0+v85dvc3CrWeJv56JX207BrStg6PN4x8KUREcHCy4fv2OTmMQ2mRM9FNlG5ekpMs4OxdeIeR5YmCgJDe3/G+QFCUj46J/ymtMnvY5olQqsLMrejU61cSJEyeWeUTPoXv3sov8yaysuZq7YGtiw5X0eLLy7mNjbE2EZ1eaODfA1MAUb9s6hLmF4GBqx5U78exPPMzeawfJyM7EoYo9VQxNyz/ICmRnZUKL+lUxMzbgwMkkth+5Rp5aTe2qlqhUurnlwczMmLt3s3VyblE0GRP9VNnGJSMjDXNz66fvqMeUSkWFTDCJkpFx0T/lNSZP+xxRKBRUqVL0RKvMlP+tombKH/a0WSaNRsOFtMvsid/Psesn0Gg0+Np709ItFC8bj0pX6nHrThbLd50n+lQyDtYm9G/jSYCH/dMPLGOVbfavMpAx0U+VbVxkplyUFxkX/aOPM+VSU67HFAoF7tY1cbeuye2sNH67doh91w5x4sZMnKs4EuYWTBPnhpgYVI4aORsLY0Z09aGFvwsLt51jysrjBNax5+U2dbC3qly/EAghhBBCPEzKV/5WUeUrDyvJT78mBiZ42XjQsloojqb2XM1I4EDiYfbGH+RO9h3sTe0wM9RtLXZZcbA2Jax+VYyNVOw7kcjOIw8ed1zLxRKVsvx/HahsP8lXBjIm+qmyjYuUr4jyIuOif/SxfEVmyp8zhkoDmro0pIlzAy6lX2F3/H72XjvIrvh9+Nh5E+YWQl3bOigVz/cS9AYqJS81q0HTuk4s3fkXq/ZeYP/JJAa288Snpq2uwxNCCCGEKFNSU/43fawpL660rHT2JUSz79oh0rPv4GhqTwu3YJq5NMLUwKQMItW9ExdSWbTtHCm37tHY25G+rTywtSyf11bZ6mQrAxkT/VTZxkVqykV5kXHRP/pYUy5J+d+e56Q8X646l2MpJ9gTv5+L6VcwVhnRzKURYa7BOJk5ltl5dCUnN49N0VfYePAySoWCbqG1aNPIDYMyXqWlsiUalYGMiX6qbOMiSbkoLzIu+kcfk3IpX6lEDJQGNHYOpLFzIJfTr7In/gD7r0WzJ/4AdW09CXMLxsfO+7ktbTE0UNE1pBZBPs4s3naO5bvOs/9EIgPbeeJVvXI+BVUIIYQQLwaZKf9bZZgpL8qd7Az2XYvmt2sHSctOx97UjjDXIJq5NH7u1zz/468bLN5+jhtp9wnycaJPuAdW5s++Ek1lm/2rDGRM9FNlGxeZKRflRcZF/8hMuahwFkbmdKzVmnY1WvLH9ZPsid9P5PkNrL+whSYuDWnpFoKLmZOuwyyV+nXsqVvTho0HL7M5+jJ/nL9Bj+a1CW/gikr5fP4aIIQQQogXkyTlLwiVUkVDpwAaOgVw9c41dsfv51BiDPuuHcLTxoOWbiH42dd97kpbjA1V9GxRm2BfZxZtO8fi7X+x73giA9t74eFqpevwhBBCCCGKRcpX/lZZy1eeJCM7kwMJh9l77SC3sm5jZ2JDc9cggqs2eS7XPNdoNBw5e50lO/7i1p0sQv1diGjpjuVj1gN9HF2PiyhMxkQ/VbZxkfIVUV5kXPSPPpavPF/ToqJMmRuZ0a5mOJ8GvctrvoOwNbFhTVwUE/ZPYvGZlVzLSNR1iCWiUCho5O3IpNeb0rFpdQ6eTGLCL4fYfeyaPLRBCCHKWWhoI2bP/rnUx06Z8m0ZR1QyERFdmDRpok5jeFhiYgLh4UGcOXOqxMdOmjSRiIguWn2FhjYiKmp9WYZYKT36Pvj990O0bduC27dvl/u5pXxFoFKqCHT0I9DRj2sZieyJ38/hpGPsTzhMHevahLmF4G9fD5VSpetQi8XEyIDe4R4E+7mwaOtZ5m85y2/HExjYzotaLpa6Dk8IISrMjh3b+OST9/nyy+8IDW2hta1Pn24kJFzj118X4unpXdCek5ND+/YtCQlpzueff1nRIT/RyZMniI4+QJ8+/bGwsNBJDKGhjYpsr169BosXRwJw6tRJoqI2cOrUSS5cOE9ubi779sWU6Dxz5szE3z8Qb+96zxyzKL3GjZtRrVo1Fi2ax+jRb5bruSQpF1pczV3o7x1BN/eXOJBwmN+uHWTWyQXYGFvT4u/SFnMjM12HWSyu9mb85+VAok8ls2znef47L4awQFd6tqiNuamhrsMTQohy5+8fAMDJk8e1kvLU1BskJFxDpVJx4sRxraT87NkzZGdn4e9fv0Tn2rFjPypV+U7enDp1gjlzZvLSS110lpQDNGkSRPv2HbXazMz+KUk4eHA/69evxt29Dq6ubly+fKlE/d+8mcqWLVF88smksghXPKOuXXvw449TGTp0OKam5bdynSTlokhmhlVoW6Mlrau34MSN0+yJ38/aC5vYeGkbjZzq09IthGoWrroO86kUCgXNfJzxd7dnzb4L7DgST8yZFHqHuxPi54JSodB1iEIIUW4cHBxxdnbhxIlYrfYTJ2IxMjImKCiYEydi6dWrj9Y2oMRJubHxsy9J+7yoUaMm7du/9NjtPXpEMHDgKxgbmzBlyrclTsq3bNmEkZERwcGhzxjp8+3evXvlmgQXV1hYKyZP/prdu3fQsWPncjuPJOXiiZQKJQEOPgQ4+JCQkcSeawc4nHiEQ4kx1LaqSUu3YOo7+Ol9aUsVEwP6t/Ek1M+FhVvPMSfqDL/FPnjwUHUn3c22CCFEefP3r8/u3TvJycnB0PDBr4QnTsTi5eVN/foNWbp0odb+J0/GYmpaBQ+POgCo1WqWLl3Exo1rSUi4hqWlJS1btmbEiDFUqfLPogChoY0YMuR1hg0bUdB29GgMP/44hYsX47C3d6B//8Gkpt5gzpyZRZZz7N69g1mzfubatau4ulZjzJh/0axZMACzZ//MnDkzAejdu2vBMStWrMPFpSoAGzeuIzJyGZcuXcLU1ISgoFDeeGMctrZ2BftrNBrmzZvN2rWrSE9Po149X956a/wzXeNHPXy+0ti/fy8+Pn6YmJhotcfGHmPFiqWcOnWSW7duYmNjS8uWrRgxYjTGxiaP6e3ZJCYm0Lt3V8aNexsjIyMWL17A9espuLvX4d//fpe6dX209o+JOczs2TM4d+4shoZGNGrUhDfeGEfVqk+eyIuKWs8XX3zKjz/OYsuWjezZs5MqVcxYsWIdSUmJLFw4jyNHDpOcnIyJiQkNGjRi9Og3C8b+4T5mzJjDjh1b2bZtE/fv36dx42aMHz8BG5t/HjSo0WiYO3dWsd4HNja2uLt7sG/fHknKhX6oau7My1496Va7I4cSf2dP/AF+/XMxVkaWNHcNItS1KRZGRd9RrC+qO1nw3sAGHDiRxIrd5/l07u+0buhG99DaxMbdYNWeOG6mZ2FraUzPMHeCfJx1HbIQ4jmWfugAN1ZFknszFQNbO+x79sLy7ySzovj5BbB16ybOnj2Dr68fAMePxxIY2AA/P3+mTEkiJSUZR8cHz6w4ceI4Pj6+BaUoX375Odu2baZTp6707v0y8fFXWbVqOZcuXeT7739C8ZhfHM+dO8M774zD3t6BoUOHo1armTt3FtbWRT+B+Y8/jrBr13Z69IjA1LQKK1cu5cMPxxMZuQErK2vCwlqRkBDPli2bGDfubaysrAEK+vv111+YO3cWbdq0p2vXHqSmprJixRLOnDnF7NkLCpLWWbNmMG/ebIKDm9O0aRDnzp3h7bfHkJubU+xrmp2dVejGPzMzs4IvPc8iNzeX06f/pFevvoW27dq1nays+/ToEYGlpRWnT/9JZORyUlJS+O9///fM536SzZs3cu/ePbp164lCoWDx4vlMmDCe5cvXYmDwIJ38/fdo3nlnHNWq1eC110Zx924mK1YsYdSoYcydu0QrKX6cr7/+Ajs7e4YOHUFeXi4Ap0//ycmTx2nTpj0ODo4kJiawZk0kY8eOYOHCFYW+vHz33ZdYWloxZMhwEhMTWLFiCZMnf8Vnn/1fwT6//DKdOXNmFft94OVVl/37fyvt5SsWScpFiVUxNKVV9Ra0rBbKqdSz7I7fz4aLW9h8aTsNnAJo6RZCDctqug7zsZQKBaH+LgR62rNqzwV2xMSz/3giOXlqcvMerNKSmp7FvE1nACQxF0KUSvqhAyTPn4smOxuA3JupJM+fC1Chibmf34O68hMnYvH19SMr6z5//XWWQYOG4OHhibGxMSdOxNK6dTuuXYvn5s1UunfvBTyYmY2KWs9///s/WrZsXdBn3br1+OSTD4iOPlgwk/2o2bN/RqUyYPr02djZ2QPQqlVbBgyIKHL/y5cvsXDhioIZ1QYNGvHqqy+zffsWevXqi4dHHby86rJlyyaaN2+pNUOamJjAvHmzGTVqHC+/PLCgvWnTYEaNGsqmTRvo3j2CW7dusXjxfJo3D+OLL74p+ELx888/smDBnGJf07VrV7F27Sqtts8//5Lw8DbF7uNxkpOTyMrKomrVqoW2jRo1VmtGvFu3nri6VuOXX34kKSkJZ+fy+/cqJSWFpUtXY27+YPKtevUavPfev4mOPkhISHMAfvppCtbW1kyfPrug5r9p02BGjhzCwoVzGTv2raeex9ramsmTf0D50EMAg4NDC13bkJAWjBw5hN27d9ChQ6dH+rDl22+nFoyvRqNm5cplZGRkYG5uzq1bt1i4cF6J3gdVq7py82Yq6elpWFqWz3NQJCkXpaZUKPG1r4uvfV2SM1PYc+0AhxJjOJx0lFqW1QlzCyHQ0Q8DpX6+zcxMDBnU3otQfxe+WHCEvEeWTczOVbNqT5wk5UK84NIP7Cdt394SH3f/Qhya3FytNk12NslzfyVt754S92cV2gLL4JASH1e7tjvm5uacOBHLyy8P5NSpP8nNzcXPzx8DAwO8vesVJOWP1pPv2rUDS0sr6tdvqDUzHBDQAJVKxbFjR4pMyvPy8oiJOUx4eJuChBzAza0azZoFFznj2KRJM60SBw+POpiZmZGQcO2pr3Hv3l1oNBrCwsK14nRzq4adnT3Hjh2he/cIYmKiycnJoVevvloz/H369C9RUh4WFk737tpfLvLLfZ5VWtqD+C0sCq8W9nBCfu/ePbKysvDz80ej0fDXX2fKNSlv06Z9QUIO4O8fCFAwPjdu3OCvv84xaNAQrZtwfX398PHx4+DBfcVKyrt27amVkIP2687NzSUzMwM3t2qYm1tw7tyZQkl5/mz+w7EuW7aYpKREPDzqlOp9kP+abt++LUm50G9OZo708exOl9odOJQYw974A8w9tYRV5zcQWrUpoa7NsDLWz+UIa7lYFkrI86WmZ1VwNEKIyuLRhPxp7eVFqVTi4+NfkHCfOBGLm1s1bGxsAfD19ef336MLtqlUKurV8wUgPv4K6elpdO5c9Azw7du3imy/desWWVlZuLkV/tW0qDYAJ6fCCaWFhSV37jz9AVVXr15FrVbTp0+3x8T5INFNSkr6O4bqWtttbGyKTIIfx9HRmcaNmxZ7/9Io6tmOSUlJzJ49g3379nLnTrrWtoyMjHKN59HxsbR8cL3y40hKevBsk+rVCz84p0aNmmzfvrVY5ynqF4KsrPssWDCXqKj1XL+eonVtinrdj8aan1D/E2vJ3wf5p3xcuVZZkKRclClTAxPCq4US5hbM6Zt/sSd+P1GXtrPl8i4CHf1o6RZCTcvq5fqmLg07S+MiE3BDlZKzV27hVf3pdXBCiMrJMjikVDPUF8b/m9ybqYXaDWztqDb+/bIIrdj8/QOIjj5AfPzVv8tY/Au2+fr6sXTpQu7evcuJE7F4eHgW3MCpVmuws7Pnww8/LbJfe3uHMotR+ZgFA4rz4HGNRo1KpeKbb6YAhf99KUnCrWv5tfKPfhnJy8vjrbfe4M6ddAYMGEyNGjUxMTHlxo3rTJo0sVjX6VmoVEU/b7Ksz1vUKj6TJ39NVNR6evd+GV9fv79n7BVMnPhBkecvj1jzE/r88SkPkpSLcqFUKPGx88LHzouUu9fZG3+Qg4kxxCT/QXULN1q6hdDAKQBDPSlt6RnmzrxNZ8h+6JG7KqUClUrB/xYfw9PNis4hNfGpaat3XyiEEPrJvmcvrZpyAIWREfY9e1V4LPl15bGxxzh58gQjRowu2ObrG0BeXh6HDx/k0qWLWjcYurq6cexYDAEBgRgZGRX7fDY2NhgZGRMff7XQtqLaiq/oz19XVzfy8vJwc6uuVWv+qPzyjvj4K1qlHrdu3So086wrTk7OGBsbk5ioXbZz4cJ5rl69woQJE7VWAPn990MVHWKRnJ1dALhy5XKhbVeuXH6m0pr8uvGHy1+ysrJK/etAad4HiYkJ2NraFfxCUB6K/iohRBlyrOJAhGdXJoVMoK9nd7Lzspl/ehkf7p/E+gtbuJ2VpusQCfJx5pWO3thZGqPgwcz50E51+X5sKAPaenI97T7fLYvlv/NjOPbX9XKfkRBCPP8smwXjNPhVDP5eHs/A1g6nwa9W+OorAPXq+WJgYMD69Wu4cye9IEmHBwm0q6sbS5cuQq1Wa20LD29NTk4OCxfOLdRndnY2mZlFJ0UqlYpGjZqwZ88uUlNvFLTHx1/l0KEDpX4d+WtWZ2RozyK3aBGOUqksWDLxYWq1mvT0B//ONGrUFAMDAyIjl2nts3z54lLHVNby6/zPnDmt1V7ULwkajYYVK5ZWVGhPZG9vT506nkRFrddKlk+dOsnJk8cJCir9mutFvfbIyGXk5eWVqr/SvA/Onj2t9QtTedCPaUrxQjAxMKaFWzDNXYM4e+s8u+P3s+XSTrZe3kV9B1/C3EJwt6qps5noIB9ngnyccXCw4Pr1fz7wWzd0I6x+VQ6cTGLjwUtMizyBm4M5nYNr0MjLEaVSZs6FEEWzbBaskyT8USYmJtSp48XJk8cxNzenVq3aWtt9ff3ZsiUK0H5oUIMGjejSpTu//voLZ8+epmHDJiiVCq5evcLOndv5+OPPH1tbPXTocEaNGsqoUcPo1q0narWayMjl1K7tzl9/nSvV6/DyevDk0V9++YnWrdthYGBASEgL3NyqMWzYCGbOnE5CwjWCg5tjamrKtWvx7Nmzk8GDh9KlS3dsbGx4+eVBLFgwh/Hj36JZs2DOnTvDoUMHsLYuu7KEpKRENm/eCDxYzg9g7txZAHh4eGo9XbUooaFhzJo1XevhOTVq1MTV1Y0ff/ye69dTMDMzY/funcWquS/K0aMxjBs3stDa8s/ijTfe5J13xjFq1FA6depKZmYmK1Ysxc7OnoEDXy11v8HBoWzZEoWZmTk1a9bizz9PEBNzGCur0t1waWNjw4ABg5g3r3jvg1u3bhIXd57evV8u9WsoDp0m5SkpKcyfP5/Y2FhOnjzJ3bt3mT9/Pk2bFu/mibi4OL744guOHj2KoaEh4eHhvPvuu9ja2pZz5OJZKBQKvG3r4G1bhxv3Utkbf5ADib9zNOU4buZVaekWQkOn+hipnn2917JioFLSIqAqIX7OHD6VwoaDl5ix9k+cbS/SKagGTes5YfCYGjYhhNAH/v4BnD79J/Xq+RZa3SI/Ka9a1RV7e3utbePHT8DLy5t161YzY8YPGBkZ4uJSlS5dulOnjtdjz+ftXZdvvpnKjz9+z6xZM3B0dOK110Zw6dIlLl8uXOJQHJ6e3owYMZpVq1YQHX0QtVrNihXrMDU15ZVXhuHmVp0VK5bw668/o1AocXJypnnzMK0vDq+/PgojIyPWrInkyJHD1Kvny3ff/cD48f8qVUxFSUi4xqxZM7Ta8v/u2LHzU5Py9u07MmPGNA4c+I3WrdsBD2bQ//e/yXz//dcsWDAXY2MjmjcPp1evPrz6asmTxXv37gForY7zrBo3bso330xl9uyf+eWX6RgZGf798KA3i7VG+eO8+eY7KJVKtm3bRFZWNn5+AXz//Y+8/fbYUvc5YsRoDAyK9z7Ys2cnRkZGhIW1KvX5ikOh0eHv8NHR0QwePJgaNWpga2vLsWPHip2UJyUl0b17dywtLRk4cCB3797l119/xdXVleXLl5d4Af/U1AzUj1mBo7w8OiP7IsvKy+b3pKPsjt9PYmYyZoZVCKnalOauzbA1qdibLIszLmqNhqNnr7P+wCWupmRgb2XCS0E1CPF1wdBAkvOyJv+t6KfKNi5JSZdxdi68csTzxMBASe5D98boo/ff/zcXL15g6dLVug6lwpRmXL744lOSk5OYMmV6ucQ0ffo0tm3bzNKlq0t0v0BlUZIxGTp0IIGBDYu1pOPTPkeUSgV2dkU/aFGnM+U+Pj4cOnQIGxsbtm/fzujRo59+0N9mzJhBVlYWCxYswMnpwVPI/P39GTJkCGvXriUiouiHEwj9ZKwyItS1GSFVm/LX7Th2xx9g2+XdbLu8mwAHX1q6BeNhXVtvbrJUKhQ08nakoZcDsXGpbDhwifmbz7J+/yU6NK1Oi4CqGBsWvZKAEEK8KLKysrRW07h69QoHD+4v10eVVxZDhrxO//69OHPmFN7e9cq8/2PHjvDKK8NeyIS8JH7//RBXr17m22+nlfu5dJqUP7wIfUlt3bqVVq1aFSTkAMHBwdSsWZNNmzZJUv6cUigUeNp44GnjQeq9W/x27SAHEg7zx/UTVDVzpqVbCI2dAzFS6ceHiEKhoL6HPQHudpy6fIsN+y+xZPtfbDxwifZNqtMy0BVTY7l1Qwjx4snNzaVPn2507NiZqlVdSUxMYO3aVRgaGtK//2Bdh6f3XFyqsmvXwXLr/5df5pZb35VJ48bN2Lat8MOuysNzmS0kJyeTmpqKr69voW3+/v7s379fB1GJsmZnakN3j5d4qVZbYpKPsTt+P4vPRrImLoqgqo0Jcw3GzlQ/+xfrmgAAIABJREFU7h9QKBT41LTFp6Yt567eZsOBS6zYHUfUocu0bVSN1o3cMDPRnxp5IYQobyqViiZNmrF9+xZu3kzF0NAQX98Ahg9/g2rVqj+9AyFeMM9lUp6SkgKAg0PhhxY4ODiQmppKXl4eKpWUD1QGRipDgqs2IcilMXFpl9gdv59dV/ex88pv+NnXI8wtGC8bD70pbfGsZs3bfetzMTGdDQcusWbfRTYfvkLrhm60bVwNyyr6McsvhBDlSaFQMGHCRF2HIcRz47lMyrOyHjx5sag6qPzatfv372NmZlbsPh9XdF/eHBwsdHLe55Wjoz9BdfxJvXuLref3sv3CPo7/8Sduli50qBNGixpNMTE0eebzlMW4ODhY0MTflYsJaazY8RdRhy6zLSaejkE16dHSHTsr02c+x4tE/lvRT5VpXFJSlBhUghu1K8NrqIxkXPRPeYyJUqks9efic5mU5yfe2Q89JS1ffsJuYlKyxExWX3neGNDGpRVhjs05khLLnvj9zDqylEWxawhyaUwL12AcqtiVqueyHhdzQyVDOnjRobEbUQcvs/63C2zcf4Hm/lXp2LQ69taSnD+N/LeinyrbuKjVar1fueRpnofVV15EMi76p7zGRK1WP/FzUW9XXyktR0dHAK5fv15o2/Xr17Gzs5PSlReEocqQZi6NaOrckIvpV9h9dV9BeYuPnRct3ULxsvVAqdD9DIWLnRnDOteja2gtNh26zG/HE9gbm0CQjzOdgmrgZFtF1yEK8cLTaDR6UwonhHi+POsq489lUu7k5IStrS0nT54stO348ePUrVtXB1EJXVIoFNS2qkFtqxrczkpj37Vo9iUc4ofYWThVcaCFWzDNnBtiYvDspS3PysHalMEdvOkcXJPNh6+w548E9p9MpEldJzoF1cDNQTelVEK86FQqQ3JysjAy0v3nhBCifGTmZHI7K508dR4qpQprY0vMDItf7vwkOTnZqFSlT62fi6T8ypUrAFSv/s/d2u3atWPdunUkJycXLIt48OBBLl26xGuvvaaTOIV+sDa2onPtdrSv2YpjKcfZHb+fFefWsj5uM01dGhHmFoxTlcI3CVc0W0sT+rfxpFNQTbb+foWdR68RfSqZBp4OdA6uQU1nS12HKMQLxdzcitu3b2D2/+zdeXQc5Z0u/qeq90Wt3lu7vGqxwca7DWHHYMAhYEOYJBNgJjAkk8kMJJkQ7r1nzj2XJATCjzjDhBAyWbAxEK/xgA0BDJjgFWzwgiXZFrZlW5bUau1bb1W/P3pRt1q2JVtSlaTnc06OpKqu7rf9TcsPr7/1vpZsGI0miKKGs+ZEY0hnuBOBnpbkjHZEiiLQ0wIAlxTMZVlGOBxCS4sfWVkXv+Ghojt6AsDzzz8PAKiursYbb7yB5cuXo6CgILlTJwDccENsW9P33nsved3Zs2dx5513wm63J3f0/P3vf4/c3FysXbt20Ivhs6d8bDvRVoMPTu3Avob9iMpRTHOW4tqCKzHNVZrR2qJUXTq6w3j3k1N495PT6ApGcPkkF7585QRMKcge8bGoDT8r6jQW6xIOh9DR0YJwOARJiio9nEETRRGSxN5ltWFdlCfJMlqDbZDkzDqIggiH8dL+rtVotLBa7TCZzh/uz9dTrngoLy0t7fd4fn5+MoT3F8oB4OjRo/j5z3+OvXv3QqfT4brrrsPjjz8Op3Pwa1czlI8PrcF2bK/dhY/O7EJrqB0ekwvXFFyJRblzcbCxAv9T/RZagi2wG+y4Y/ISzM+ZPeJj7A5G8N6+0/jrnlPo6A6jrMiOL185AWXFjnE7a8fPijqxLurDmqgT6zIyJFlCc08L6rr8qO9qQF1nA+q7GlDf6Ud7uOO81/76hqdHZIyqDuVqwVA+vkSkCD7zH8K209vxRetJaAQNZMhp/wWtE3X4etlyRYI5AARDUWzbX4s3d59Ea0cIk/Nt+PKVE3D5JNe4C+f8rKgT66I+rIk6sS5DKxQNob6rMR64G1Df5UddVwMauvwIS5Hk48xaE3IsXvjMXvjMHrxbsw0d4c6M53MY7PjJVf9rRMY+5lZfIbpUWlGLub4rMNd3BWraT+OXe19ASEpfYjMshfE/1W8pFsoNeg1unleI62fl4aODddiy8yRWrD2AIp8VX75yAmaVeCCOs3BORETjgyzLaAt1xIJ3fLa7risWwJt6mpOPEyDAZXTAZ/Gi1DEFOWYvfJZYCLfqLGmTWNkGG16pXI+wFE4e04k63DF5yYi+t3NhKKdxryirICOQJzQHW/DigZdQ7ipBubMUbtPgW6MulU6rwfWz8nH1jFzs+rwem3eewK83HkKe24Kli4oxr9wLjaj8ko9ERESDFZWi8HcHMoJ3fVcDuiM9ycfpRR18Fi8mZRfjytx5yeDtNbmh0+gG9FqJSTY1tKr2h+0rcWxfGd/+z/afoTnYknHcoNHDrDUnz3lNbpQ5SzDNVYKp9skwag0jPVRIkoyPKxvwxo4TONPYCa/DhNsXFmPRZTnQasZmOOdnRZ1YF/VhTdSJdQG6wt2xPu8uf7LlpL6rAf7uQFrraLbeBp/FixyzJ9Z2YvEgx+xFtsE2pHuOKFUT9pQPAEP5+Lanbl+//6T19bLlmOebhfouPyqajqCi6QiONlcjJIWhETSYlF2Mac5SlLtKkG/NHdFNiiRZxmdHG/H6jhM4WdcOl82AWxcW4+oZudBpx9bmWfysqBProj6siTqNl7rEbrRsjc92p/d7t4d6b7TUCBp4zO7e4G32IMfihdfsgWmE9hNhKFcxhnLaU7dvQP+kFZYiqG45ngzpZzrOAgCydFaUOUtQ7pyKclcJbPqsERm3LMs4dLwJr28/gWNnWpFt0WPJgiJcd0U+DPqxEc75WVEn1kV9WBN1Gmt1CUVDaIjfaJk+8+1Pm9wyaU3xHu/YbLfP7IHP4oXb6IRGVPbvJ4ZyFWMop4TB1qU12JYM6JVNR5N3dhdY81Aeb3WZlD0BWnF4b+GQZRlVNS14fccJVJxshtWkw83zCnHD7AKYjaP79hF+VtSJdVEf1kSdRmNdZFlGe7gD9Z0NySUG6ztjX5t6WiAjlpkECHAaHenB2+xFjsWbcaOlmjCUqxhDOSVcSl0kWcLp9locbjqCiqYqfNF6EpIsQa/Ro8Q+GeWuEkxzlsBjcg/rL6pjZ1rxxo4TOFAdgMmgxU1zCrB4XiGspoHdDKM2/KyoE+uiPqyJOqm5LlEpisbuQEbwruvyozvSnXycXtQlZ7pTg7fH5IZ+gDdaqglDuYoxlFPCUNalO9KDI83VsZn0QBUae5oAAC6jA+XOEpS7SlHqmAyT1jQkr9fXybp2vLHzBPZW+WHQaXD97HzcMq8Q2daRv0H1UvCzok6si/qwJuqkhrp0R7pRFw/c9fGWk7ouP/zdjX1utMyK32AZ7/WOt5/YDdkjet/UcGMoVzGGckoYzrr4uwKoaKrC4aYjONJ8DMFoCKIgYqKtCOXOUkxzlaAwK3/If/Gd8Xdg866T2H24HlqNiGtm5uHWBUVw2kbmhppLxc+KOrEu6sOaqNNI1UWSJbQEW9OWFqzrrEd9lx9tod7XFwURXpM7I3j7zJ5hmyRSG4ZyFWMop4SRqktEiuB468l4q8sRnGo/AwCw6Mwoc0yNz6SXwG7IHrLXrG/uwpadJ7HjUB0A4KrLc3HbomJ47er+JczPijqxLurDmqjTUNclFA3D393Yu418ys2WoYwbLdOXFlTLjZZKYyhXMYZySlCqLu2hjuTNooebqpLLR+VZcpIBfUr2xAFvknA+ja3deGt3DT7cfxaSJGPBNB9uX1SMPLflkp97OPCzok6si/qwJup0MXWRZRkd4c604F0X7/lu6mnuc6OlPRm8feb4Gt8WL7J0VtXeaKk0hnIVYyinBDXURZZlnOk4i4qmIzjcdARftBxHRI5CJ+ow1T4pvsNoCXLM3kv6hdvSEcRf99Tg/U/PIByWMKfMi6WLilHkG5nlHAdKDTWhTKyLegx0SVcaWQOpS1SKorGnKW1N78TNll0pN1rqEjdaxgN3Ygbca3ZDr9GP9Fsb9RjKVYyhnBLUWJdgNISjzdXJVV0auhoBAA6DPb4ueinKHFNg1pkv6vnbu0J455NT2Lr3NLqDUVwxxY3bryzG5Lyha525FGqsCbEuanG+zc8YzJXTX120ohYLc+bCrDMlW0783QFE5WjyMTZ9Vkrw7l3pxGEcWzdaKo2hXMUYyilhNNQl0N2UXBu9qvkYuiM9ECBggq0wuapLcVbBoHsGu3rC2Lr3NN7++BQ6eyKYPsGBpVdOQGmRY5jeycCMhpqMR6yLskLRMAI9TVix74Xk/gipdKIOl7nL0w/2+Ss/82+9C5y/wPVyP0fOc/kFXy/j+fr5azrzNfucv0DM6Xv9hd/zwJ7vRGsNIilhO5UoiPCY3Mk2E1/KzpZmnbrv8RkrGMpVjKGcEkZbXaJSFCfaTiVXdalpOw0ZMkxaE0odUzAt3o/uNA48WPeEIvjg01q8tacGbZ0hlBRkY+lVEzB9glOR/sTRVpPxgnUZXonNWxq7m9DYHYj/r/f71tCF/+x9Zi8yPrF9PsN9zwuZV/S5/ALncf7nv9Dr9z3S73iEvj8O8jkzB3VJ1/c3xqMtX2QcS/jP654c9zdaKo2hXMUYyilhtNelI9yJqqajqGg6ioqmI2gJtgKI/eWcCOhT7ZMG1IMYCkfxtwNnsWXXSTS3BzExNwtLr5yAmVPcEEcwnI/2moxVrMuli0gRNPU0w9/dhEB3AP7uAALdTfB3B9DY04RQNJT2eLshG26TE26TC26jCx6TE+uPvZG23F2Cw2DHT676XyP1VqiP/7P9Z2gOtmQcZ13UQY2hfHTvvU1EGaw6C+b4rsAc3xWQZRlnO+uTrS4f1e7C+6c/glbUYkr2xOQNo3mWnH5nv/Q6DW6cU4Brr8jDjkN12LzzBJ5bfxAFHiuWXlmMuaVeiCLv7Cc6n65wVyxkJ2e54197mtCcsl05EGs5iYVuJ0odU2LhOx7CXUZHv6svSZD77Sm/Y/KSEXl/1L87Ji9hXWhQOFMex5lyShjLdQlFwzjW8kVyVZe6znoAsR3cypwlmOYsQZmzBFZ9/0sjRiUJew434I2dJ3A20IUcpxm3LyrGgmk+aDXDdwPSWK7JaMa6xESlKFqCrWlhOzbjHYC/uyltq3IAyNJb4Ta6koHbY3LBFf9q02ddVIsYV19RJ9ZFvdQ4U85QHsdQTgnjqS7NPS3xNpcqVDYdRVekGwIEFGblx1tdSjHRVpTR+yhJMvYd8eP1HSdwqqED7mwjbltUjKsuy4VOO/ThfDzVZDQZT3XpifTEQndPU0Zvd6CnOW2bco2ggcvoSJvl7p3tdsKoNQzbOMdTTUYT1kV9GMpVjKGcEsZrXSRZwsm206hoqkJF0xEcb62BDBlGjQEljikod5ZgmqsEbpMreY0sy9h/LIDXd5zA8bNtcGQZsGRBEa6ZmQeDbuhuYhqvNVG7sVQXSZbQFmpHY3fqLHdvf3ff1U3MWlNG6PaYXHAZXYouXTeWajKWsC7qw1CuYgzllMC6xHSFu1HVfCy2qkvgSPKGJY/JhXJnKaa5SjDVPhlGrQGyLOPwyWa8vv0Ejpxqgc2swy3zi3DdrHyYDJd+6wprok6jrS6haBhN8daSvv3dgZ4mhKVI8rGJXRJdptjNlG6jC26zC25jrN/7YvcEGG6jrSbjBeuiPgzlKsZQTgmsSyZZltHQ5Y9vXnQER5urEZLC0AgaTMoujq+NXoICax6OnW7DGztO4NDxJliMWiyeW4gb5xbAYsy8QW2gWBN1UltdEtuSN/YJ3f546E6sRJRg0Ojjs9yJsN078+002qEVR99aCGqrCcWwLurDUK5iDOWUwLpcWFiKoLrleHJVlzMdZwEAWTorypxTUe4sgSWSi/d2N+KzY40w6mOruCyeVwibefDbQbMm6qREXaJSFIGe5oxVTBL93cF+lhB0GWOtJX37u606iyLr7g8nflbUiXVRHzWG8tE3DUBEitOJWpQ5p6LMORV34Xa0BttQ2XQUh+P96B/XfwoAKCjKw/UlE1BfY8GWXcfxzsencN2sfNwyvwiOrOG72Y1Gt65wd2/Y7gqgsSeQXMe7qc8SglpRm5zlnmqflNbf7TQ6oe9nCUEiIjViKCeiS5ZtsGFB7hwsyJ0DSZZwur023upShT2BXZDMErIW6GAK+/BezXG897kHXyqdgtsWFMNt55bS440kS2juaUWgJ9Bvf3dXnyUErToLPCYXJmYXY37O7Hifdyx82/RZit1USUQ0lBjKiWhIiYKIIlsBimwFWDLhBnRHenCkuTrW6hKogq74NIAK7Ap+jB1vuTE1eyqWz52PCR6n0kOnARrI2ss9kSACGcsHNiWXEIzK0eRjRUFMLiFYbCtM2bEydlOlUWsc6bdIRDTi2FMex55ySmBdhpe/K4CKpirsb6jEkeZqSEIYsizAEvVgTt40LCy6DEVZBRAFkRtvqNCeun0ZuxRqBA2mu8pg0Ojja3kH0B7qSLvOpDXBY3L2znIbnckNc+yG7Iy18Oni8PeXOrEu6qPGnnKG8jiGckpgXUZORIrgYH013q7Yi5NdX0AwtwEAjBoTcsxu1LSfgYSUTVmgxd9Pu3tAwVyWZciQIckSJDn2VUbv95IsQ4YEOeVnCRLktMenP7b32vh1kOOPlyDFH5t5ffwxKdfL8ccP5LXl+Lm0a9Pem5Qyjvix+PuSU35OjiP5WnKfa9NfO/WxvddKCKWE8b6cRkfGKiaJ/m61LiE41vD3lzqxLurDUK5iDOWUwLooo6M7jC0fH8G26gOIWOqhddUC/SyMIUBAlt6aGXrTgrGcdjPgaCIKIkQIEAQRgiBAhAhRECCm/Rw7JiBxvPcx5742di55bfx86rVC6uMhpF8bf/zWUx+ec+y/vuHpEfyTov7w95c6sS7qo8ZQzp5yIlIFq0mHr14zHUvnl+L9T09jc9fz/WVyyLKMy93l8ZDZN0TGw2hKABX6BFixv8Da99qUcNv32tjP4nnDbe+16eF2IK+tdvsaDiQ3kkrlMNgVGA0R0djBUE5EqmI2anH7ogl4400jBENPxnkpZMTXSpePufWlR4s7Ji/J6CnXiTrcMXmJgqMiIhr91D8tQ0TjkqFxGuRo+q8oOSoicqoEj/92FzZ8WI3T/o5zXE3DZX7ObHy9bDkcBjsExGbIv162nDfgEhFdIs6UE5Eq3Tv7OqzcFYWcVwVB3wM5ZIRcW4qrCmajsa0Hm3eexBs7TiLfY8H8ch8WlHvhdfBmwpEwP2c25ufMZp8sEdEQYignIlVaND0HwI3YsG0CmtqCcNoMWHbt5PhxoLUzhE8qG7C7oh4bP/wCGz/8AhNzbVhQ7sW8ch93DCUiolGFq6/EcfUVSmBd1OdCNWls7cbHFbGAXlPfAQFAaZEd88t9mFvmhdXErdaHAz8r6sOaqBProj5cfYWIaBi4s024dWExbl1YjLOBTuw+XI/dFQ1Y+dcqrH7nCKZPdGJ+uRezpnpgMvDXHhERqQ//diKiMSXXZcGdV0/CV740ETX1HdhdUY89FfU4UB2ATluFmZNdmF/uw4zJLuh13EWSiIjUgaGciMYkQRBQnJOF4pws3H3dZFSfacXuw/X4pLIBn1T5YdRrMLvEg/nlPkyb4IBWw8WoiIhIOQzlRDTmiYKAqQV2TC2w42s3TUXlyRbsrqjH3io/dhyqg9Wkw9wyLxaUezG10A6Ra6ATEdEIYygnonFFI4qYPtGJ6ROd+ObNpTj0RQC7K+qx4+BZfPDpGTiyDJhX5sWCaT5MyMniJkVERDQiGMqJaNzSaUXMKvFgVokHPaEIPjvWiD2HG7B172m8/fEpeB2m5Bro+Z7+75YnIiIaCgzlREQAjHotFk7LwcJpOejsCWNvlR+7D9dj884TeGPHCRR4LFgwzYd55T547Salh0tERGMMQzkRUR8Wow7XzMzDNTPz0NoRxMeVDdhT0YD1277A+m1fYFKeDQvKfZhX7oXdyk2KiIjo0jGUExGdR7bVgJvmFuKmuYVobOnGnsoG7Dlcj1e3HsVrW4+itMiOBdN8mFPKTYqIiOjiMZQTEQ2Q227CbQuLcdvCYtQ2dmJPRT12H67HS29V4eW3Y5sULZjmwxVT3NykiIiIBoV/axARXYQ8d59Nig7XY3d8kyK9VsSMKW4sKPdhxmQndFpuUkREROfHUE5EdAnSNim6fjKOnW7F7or4JkWVDTAZNJg91YP503woL+YmRURE1D9FQ3koFMKvfvUrbNq0CW1tbSgrK8Ojjz6KRYsWXfDaHTt24De/+Q2OHDkCSZIwadIk3H///bjttttGYORERJlEQUBJoR0lhXZ8/aapqDjZjD2HG7D3iB/b45sUJdZAn1KQzU2KiIgoSdFQ/uMf/xhvv/027rvvPhQXF2Pjxo146KGHsGrVKsyaNeuc173//vv4zne+g1mzZuF73/seAGDz5s149NFH0dnZiXvuuWek3gIRUb80oojLJrpw2UQXvnlLCQ5+0YTdh+ux/eBZvB/fpGh+eSygF/u4SRER0XgnyLIsK/HCBw4cwD333IPHH38cDzzwAAAgGAxi6dKl8Hq9WL169TmvffDBB1FVVYWtW7dCr9cDiM2633jjjSguLsbLL7886PEEAh2QpJH9o/B4suD3t4/oa9KFsS7qM5Zq0hOK4LOjjdh9uB6HjjchKsnwJTYpmuZDntui9BAHbCzVZaxgTdSJdVEfpWoiigJcrv43o1Nspvytt96CTqdLm9U2GAy4++678ctf/hINDQ3wer39XtvR0YHs7OxkIAcAvV6P7OxsGAxcM5iI1Muo12Lh9BwsnJ6Dju4w9h2JbVL0xs4TeH3HCRR4rFgwzYsF5T64uUkREdG4oVgor6iowMSJE2GxpM8KzZgxA7Iso6Ki4pyhfP78+fjtb3+LFStWYNmyZQCADRs24MSJE3j88ceHfexEREPBakrfpGhPZQP2VNQnNymanGfD/Gk+zCvjJkVERGPdkITySCSCrVu3orW1Fddffz08Hs8Fr/H7/fD5fBnHE9c2NDSc89pvf/vbqKmpwQsvvIDf/OY3AACz2Yznn38eV1111UW+CyIi5WRbDVg8txCL45sU7a6ox56KBrz6bmyTorIiBxZM82F2iYebFBERjUGDDuVPP/00du/ejfXr1wMAZFnGP/zDP+CTTz6BLMuw2+1Ys2YNioqKzvs8PT090Oky/2JJtJ8Eg8FzXqvX6zFhwgQsWbIEixcvRjQaxZo1a/DII4/gT3/6E2bMmDHYt3XO/p7h5vFkKfK6dH6si/qMp5p4PFkon+rFA3cANXVt+PCzM/jw0zP405uVePntKswq9eKaWQVYMD1H8U2KxlNdRgvWRJ1YF/VRW00G/dv8b3/7G6688srkz++99x4+/vhjPPjggygvL8cTTzyBF198ET/5yU/O+zxGoxHhcDjjeCKMn683/IknnsDBgwexbt06iGJszd9bb70VS5cuxc9+9jO89tprg31bvNGTklgX9RnPNTFpBNwypwA3z87Hyfp27D4cm0H/+HA99FoRM6e4sWCaD5dPckGnHdk10MdzXdSKNVEn1kV9xsSNnnV1dSguLk7+/P7776OgoAA//OEPAQBHjx7F66+/fsHn8Xg8/bao+P1+ADhnP3koFMK6devw8MMPJwM5AOh0Olx99dV49dVXEYlEoNVyXyQiGjsEQcCEHBsm5Nhwz/VTYpsUHa7Hx5UN+LiyASaDFrNLYgG9vNgBjchNioiIRpNBJ9dwOJwWeHfv3p02c15YWJgM1udTVlaGVatWobOzM+1mz/379yfP96elpQWRSATRaDTjXCQSQSQSgUKrPBIRjYi0TYoWT0XFiWbsPlyPfUf82H6wDllmHeaWxVZw4SZFRESjw6CnUnJycvDpp58CiM2Knzp1CvPmzUueDwQCMJvNF3yeJUuWIBwOY+3atcljoVAIGzZswOzZs5M3gdbW1qK6ujr5GJfLBZvNhnfeeSet/aWzsxPvv/8+SkpK+u1VJyIaizSiiMsmufCtpdOw4ntfwnfvuhylRQ58dOAsfr56H370mx1Y894xnKxr54QFEZGKDXqm/Pbbb8fzzz+PpqYmHD16FFarFddee23yfEVFxQVv8gSAmTNnYsmSJXjmmWfg9/tRVFSEjRs3ora2Fk8++WTycY899hj27NmDqqoqAIBGo8E//uM/YsWKFbj33ntxxx13QJIkrFu3DnV1dXjssccG+5aIiMYEnVaDOaUezCn1oDsYwWfHYpsUvfPJKby1pwY+pxkL4ruI5rpGzyZFRETjwaBD+cMPP4yzZ89i69atsFqteOqpp2Cz2QAA7e3teO+995I7dF7I008/jRUrVmDTpk1obW1FaWkpXnzxRcyZM+e8133nO99BQUEBVq5ciV//+tcIhUIoLS3Ff/3Xf2Hx4sWDfUtERGOOyaDFouk5WBTfpGhvVQP2VDTg9e0n8D/bT6DQa8WCaT7ML/fCnc1NioiIlCbIQ/jvmZIkobOzE0ajcdS1kHD1FUpgXdSHNRk6LR1BfFwR26SourYNADA534YF5T7MK/ch26K/wDP0Yl3UhzVRJ9ZFfcbE6ivnE4lEkJWlrjUfiYiol91qwOJ5hVg8rxD+lm7sqajH7sMNeOXdo3g1ZZOiOaUeWIyja3KFiGg0G/SNntu2bcNzzz2Xdmz16tWYPXs2rrjiCvzgBz/od/1xIiJSF4/dhNsXTcD/+9Z8PPHgAty+aAICrT3405uVeOQ/P8J/rjuAXYfrEAxlrnZFRERDa9Az5b///e/hcrmSP1dXV+NnP/sZCgsLUVBQgC1btuDyyy8fcF85EREpL99twbJrJuGuqyfiRF17cg30z441Qq8TccUUNxaU+3DZJBc+qWrAhm3VaGoLwmkzYNm1k7GS8QI3AAAgAElEQVRoeo7Sb4GIaFQbdCj/4osv0lZb2bJlCwwGA9atWwer1Yof/OAH+Mtf/sJQTkQ0CgmCgIm5NkzMteGrN0zB0VMt2F3RgE8qYzeK6jQCopKMxC04gbYgXnqzEgAYzImILsGg21daW1vhcDiSP+/YsQMLFy6E1RprWp8/fz5Onz49dCMkIiJFiIKA0iIH7rulFM/+y1V49KszIYoC+t4TH4pI+PN7xxCJSsoMlIhoDBh0KHc4HKitrQUAdHR04ODBg5g7d27y/Ll22yQiotFLqxFx+SQXguH+g3dbZwjf+9Xf8Oyaz7Bl10lU17YiKjGkExEN1KDbV6644gq89tprmDJlCj788ENEo1Fcc801yfMnT56E1+sd0kESEZE6uGwGBNqCGcetJh3ml3tRWdOCdR/EdmE26DUoKbCjrMiOsmIHinxWaMRBzwUREY0Lgw7l//qv/4r77rsPjzzyCADgrrvuwpQpUwAAsizj3XffxYIFC4Z2lEREpArLrp2Ml96sRCjSOwuu14r42k1Tkz3lrZ0hHDnVgsqTzaisacbaDwIAAJNBg6kFdpQVOVBaZEexLwuiKCjyPoiI1GbQoXzKlCnYsmUL9u3bh6ysLMybNy95rq2tDffffz9DORHRGJUI3udbfSXbose8Mi/mlcX+1bS1I4iqUy2orIkF9QPVvSG9pMCO0iIHyosdKPRaGdKJaNwa0h09RzPu6EkJrIv6sCbqdLF1aekIoqqmBZU1zaisaUF9UxcAwGTQorQw1u5SWsSQfjH4WVEn1kV9xtSOnjU1Ndi6dStOnToFACgsLMSNN96IoqKii31KIiIaB+xWAxZM82HBNB8AoLk9iKp4QK+sacZnxxoBABajFiWFsYBeVmRHgdcKUWBIJ6Kx6aJC+YoVK/C73/0uY5WVX/ziF3j44Yfxb//2b0MyOCIiGvscWQYsnJ6DhfEWmKa2nli7y8lmVNW04NOj6SG9rNiBsiIH8j0WhnQiGjMGHcrXrVuHF154AbNmzcKDDz6IqVOnAgCOHj2K3//+93jhhRdQWFiIZcuWDflgiYho7HPajFg0PSfZp97U1pNsdak82ZwM6VaTDqWFdpQWxW4ezWNIJ6JRbNA95cuWLYNOp8Pq1auh1aZn+kgkgm984xsIh8PYsGHDkA50uLGnnBJYF/VhTdRJqbo0tnYne9KralrQ2NoDIB7S4wG9rMiOPLcFwjgL6fysqBProj5joqe8uroa3//+9zMCOQBotVrcdtttePbZZwc/SiIiogFwZ5vgvtyEqy7PBQA0tnSjsqYl3pfejL1VfgBAllmX7EcvLXIgz2UedyGdiEaPQYdynU6Hrq6uc57v7OyETqe7pEERERENlNtuwpfsJnxpRi5kWUZja7zd5WRsNv2TygYAgC0lpJcVO5DjZEgnIvUYdCi//PLL8ec//xn33HMP3G532rlAIIA1a9Zg5syZQzZAIiKigRIEAR67CR67CVfPyIMsy/C39sRvGo31pX+cCOkWfSygxzczYkgnIiUNOpT/8z//Mx544AHcdtttWL58eXI3z2PHjmHDhg3o7OzEM888M+QDJSIiGixBEOC1m+C1m3DNzFhIb2iJ96SfbEZFTTP2VMRCerZVnwzo5UUOeB0mhnQiGjGDDuXz5s3Dc889hyeeeAJ//OMf087l5eXhqaeewty5c4dsgERERENFEAT4HGb4HObekN7cjYr4TaOVJ5ux+3A9AMAeD+llxbGg7rUzpBPR8LmodcpvuOEGXHfddTh06BBOnz4NILZ50PTp07FmzRrcdttt2LJly5AOlIiIaKgJggCf0wyf04zrrsiHLMuoa+pKru5y+GQzdsVDuiPLkLxptKzIDg9DOhENoYve0VMURcyYMQMzZsxIO97c3Izjx49f8sCIiIhGmiAIyHVZkOuy4LpZvSG98mSsH/3z403Y+XkspDttBpQW9t446s42MqQT0UW76FBOREQ01qWG9OtnF0CWZdQGupI3jR46HsDOz+sAAC6bId6THgvqbrtJ4dET0WjCUE5ERDRAgiAg321BvtuCGxIhvbEztttoTTP2Vwew/VAspLuzjSmbGTngyjYqPHoiUjOGciIiooskCALyPVbke6y4cU4BJFlGrb8zudvoZ0cbsf1gb0iP3TgaC+pOG0M6EfViKCciIhoioiCgwGtFgdeKm+YWQpJlnPF3xnvSm/HpUT8+OngWAOC1m3pn0osdcGQZFB49ESlpQKG879KH57Nv376LHgwREdFYIgoCCr1WFHqtWDwvFtJPN3SgsqYFVTXN2Fvlx98OxEO6w5SymRFDOtF4M6BQ/tRTTw3qSXn3ORERUSZREFDky0KRLws3zyuEJMk41dCRbHf5uNKPD/fHQrrPYUqukV5W5IDdypBONJYNKJSvXLlyuMdBREQ07oiigOKcLBTnZOGW+UWQJBk1De2oPBmbSd9TUY9tn9UCAHKc5uTyi6WFdmSnhPSdn9dhw7ZqNLUF4bQZsOzayVg0PUept0VEF2FAoXz+/PnDPQ4iIqJxTxQFTMixYUKODUsWFCEqSaip751J33W4Hh/EQ3quy4yyIgdEAfjwwFmEIxIAINAWxEtvVgIAgznRKMIbPYmIiFRKI4qYmGvDxFwbbl1QjKgk4WRdR3Kd9B2f1yEYimZcF4pIWP9BNUM50SjCUE5ERDRKaEQRk/JsmJRnw60LixGJSvinX3zQ72Ob2oP437/bhUKvFcW+LBT6rCjyZsFm0Y/soIloQBjKiYiIRimtRoTLZkCgLZhxzmTQwOcwo/pMK/ZUNCSPZ1v1KPJmocgXWxWmyJcFr8MEkYs0ECmKoZyIiGgUW3btZLz0ZiVC8Z5yANBrRfz9zaXJ9pWO7jBONXTgVH07TtZ34FRDOz4/3gRJlgEABp0mtnSjz4qieFDPd1ug12kUeU9E4xFDORER0SiWCN7nW33FatKhvNiB8mJH8lg4EkVtYxdq6ttREw/sOw/V4f14j7ogALkuC4oSYd2XhUKvFTYz21+IhgNDORER0Si3aHoOFk3PgceTBb+/fUDX6LSa5HKMCZIso7G1p3dGvb4dVadiq74kOLIM8baXWI96oc8Kj53tL0SXiqGciIiIAMQ2N/LaTfDaTZhT6k0eb+8K4VRDB2rirS81DR049EVv+4tRH2t/SYT0Ip8V+W4LdFq2vxANFEM5ERERnVeWWY9pE5yYNsGZPBaORHHa3xkP67Gg/tGhswjui7W/iIKAXLc51v4Sv7G0yJcFq0mn1NsgUjWGciIiIho0nVaTXEM9QZJl+Fu6caq+AzUN7fGNj1qw8/Pe9henzRCbUY+3wBT6suDJNkJg+wuNcwzlRERENCREQYDPYYbPYcbcst72l7bOePtLQ3s8sHdgf3Uj4t0vMBk0KPTEbyaN96rnuS3QaUWF3gnRyGMoJyIiomFls+gxfaIT0yf2tr+EwlGcaexMtr7U1LfjbwfOIhiOtb9oRCG2+kt8mcbC+OovbH+hsYqhnIiIiEacXtdP+4sko6GlGzX17ckbSz8/0YQdh+qSj3HZjGkbHxV5rXCx/YXGAIZyIiIiUgVRFJDjNCPHacb8cl/yeGtnCKfirS8n44H9s6ONiHe/wGTQJtdTL47PqOe5LdBq2P5CowdDOREREalatkWP7IkuXDbRlTwWDEVxurEj2aN+qr4dH+6vRSgc29lUIwrId1uSPeqJ2XWzke0vpE4M5URERDTqGPQaTM7LxuS87OQxSZJR39yFmvrem0oPVgew/WBv+4s725hse0kEdqfNwPYXUhxDOREREY0JYvzm0FyXBQumpbS/dASTN5MmetU/PeJPtr9YjNpkj3qhN9YCk+Mys/2FRhRDOREREY1p2VYDLrcacPmk9PaXU/5Y20tNPKi//+kZhCOx9hetRkC+OzGb3hvYTQZGJxoeiv4/KxQK4Ve/+hU2bdqEtrY2lJWV4dFHH8WiRYsGdP3rr7+Ol156CceOHYNer0dJSQl+9KMfYcaMGcM8ciIiIhrNDHoNpuRnY0p+b/tLVJJQ39Sdvp76sUZ8dOBs8jEeu7G3Rz3eBuPI6r/9ZefnddiwrRpNbUE4bQYsu3YyFk3PGZH3R6OPoqH8xz/+Md5++23cd999KC4uxsaNG/HQQw9h1apVmDVr1nmv/eUvf4n//u//xh133IF7770XXV1dqKyshN/vH6HRExER0ViiEUXkuS3Ic1uwcFrsmCzLaOmIrf5Sk3JT6d4jvXnDatIldygt8sY2QDpZ345Vb1UhFJ95D7QF8dKblQDAYE79UiyUHzhwAJs3b8bjjz+OBx54AABw5513YunSpXjmmWewevXqc167b98+/Pa3v8Vzzz2HxYsXj9CIiYiIaLwRBAGOLAMcWQbMmOxOHu8ORnDG34maRFivb8fWvWcQiUrnfK5QRML6bdUM5dQvxUL5W2+9BZ1Oh3vuuSd5zGAw4O6778Yvf/lLNDQ0wOv19nvtypUrcfnll2Px4sWQJAnd3d2wWCwjNXQiIiIa50wGLaYUZGNKQXr7S12gCzUNHfjd64f7va6pLYj/+4c9yHVbkOs0J7/6nCbotJqRGj6pkGKhvKKiAhMnTswI0zNmzIAsy6ioqDhnKN+5cyduv/12PPvss1i1ahW6urqQn5+PRx55BHfcccdIDJ+IiIgojUYUke+xIt9jxYZt1Qi0BTMeY9RrkG01oPpMK/Ycrk+uACMIgCfbhFyXOb6CTOxrjssMq4lrq48HioVyv98Pn8+Xcdzj8QAAGhoa+r2utbUVLS0t2Lx5MzQaDX74wx/Cbrdj9erV+Pd//3eYTCa2tBAREZGill07GS+9WZnsKQcAvVbEN28pTbavBMNR1Dd14WygC2cDnfGvXfj8RHNaG4zNrEOOy4I8lzn5NddlgcNmgMj11ccMxUJ5T08PdLrM//IzGAwAgGAw878uAaCrqwsA0NLSgjVr1mDmzJkAgMWLF2Px4sX49a9/fVGh3OWyDvqaoeDxZCnyunR+rIv6sCbqxLqoD2uiDndclwVblhEr36xAY3M33A4T7ru1HNfNKUx7XEGePePaqCTD39yFU/XtOB2/sfR0Qwf2HvGjvas2+TiDXoN8jxWF3iwU+OJfvVbkeSxshRkAtX1WFAvlRqMR4XA443gijCfCeV+J4wUFBclADgB6vR633HILVq5cic7OzkH3mAcCHZAk+cIPHEIeTxb8/vYRfU26MNZFfVgTdWJd1Ic1UZfpRXY89fCitLoMtD4aABM8FkzwWIDpsc4CWZbR3h3G2cZOnG3qwtnGLpxt6sSh6kZs+/R08lpBADx2E/Li7S+JVpg8lxlmI1thAOU+K6IonHMiWLFQ7vF4+m1RSSxpeK5+crvdDr1eD7fbnXHO7XZDlmV0dHTwxk8iIiIaUwRBgM2sh61Ij9IiR9q5YDiKukAspMfCeqwl5tDxACLR3klHm0WfbIPJjQf2PJflnGut08hRLJSXlZVh1apVGbPa+/fvT57vjyiKKC8vR319fca5uro6aDQaZGdn93MlERER0dhk0GlQnJOF4pz0lgxJkuFv7e7Tt96JPYfr0RWMpF2fOqueWBnG5zBBqxFH+u2MS4qF8iVLluAPf/gD1q5dm1ynPBQKYcOGDZg9e3byJtDa2lp0d3dj8uTJadc+9dRT2L59O6666ioAQEdHB958803MmjULRqNxxN8PERERkdqIogCfwwyfw4wrpvR2GciyjLauMOriQb020Im6QBeOnmrBrs97Jz5FQYDHbkxbESbXbUau0wKzUdE9KMccxf40Z86ciSVLluCZZ56B3+9HUVERNm7ciNraWjz55JPJxz322GPYs2cPqqqqkse+9rWvYe3atfje976HBx54ADabDevXr0d7ezu+//3vK/F2iIiIiEYNQRCQbdEj29JPK0woirp4+0ttoCsZ3A9+EUA05f67bKs+NqOeGthdZrbCXCRF/xPn6aefxooVK7Bp0ya0traitLQUL774IubMmXPe60wmE1auXImnn34aL7/8Mnp6ejB9+nT88Y9/vOC1RERERHRuBn3/rTBRSUJjS09GK8yuw/XoTm2F0WviYT09sHvZCnNegizLI7vkiEpx9RVKYF3UhzVRJ9ZFfVgTdRrrdZFlGW2doYywfrapC00pGyhpRAEeez8bJDnNI94Kw9VXiIiIiGhMEQQB2VYDsq0GlBWnt8L0hCKxVpjG9JVhDlSnt8LYrfqMNphclwV2q37ctMIwlBMRERHRsDDqtZiQY8OEHFva8UhUQmNrT8qa67GvOz+vQ3cwmnK9JiOo57rM8NjHXisMQzkRERERjSitRkSO04wcpxmzUo7LsozWzlBvWI+3wlScbMaOQ3XJx2lEAV6HCTlOM/LclrSvJsO54+3Oz+uwYVs1mtqCcNoMWHbtZCyanjOM73TgGMqJiIiISBUEQYDdaoDdakD5BGfaue5gJLkqTCysx77v2wrjyDLEQnp8R9PEZkkVJ5uw8q0qhCISACDQFsRLb1YCgCqCOUM5EREREameyaDFxFwbJuZmtsL4W/pukNSF7YfOoifU2wojAOi7pEcoImHDtmqGciIiIiKiS6HViPFecwsAT/K4LMto6Qglg/rqd470e30gZYUYJY2tDnkiIiIiIsRaYRxZBkyb4MSNcwrgshn6fdy5jo80hnIiIiIiGvOWXTsZem169NVrRSy7drJCI0rH9hUiIiIiGvMWTc+BvuJTYOtmWMMd6NBZgRtvxxwV9JMDnCknIiIionGgbdcO2N7biKxwBwQAWeEO2N7biLZdO5QeGgCGciIiIiIaY2RZhhQOIdrRgXAggGBtLfxr/ww5FEp/XCiExg3rFRplOravEBEREdGIkyUJUjAIORiMfQ3FvkqhUOxY/Ofe86HY+cT3od5zadcHQ5BDQUDuuwBi/yJNgWF+pwPDUE5EREREGWRZhhwOpwTkUErw7Q3BcjwIS6HEsf5Ddex8KHlMjkQGNyBRhGgwQNAbIBoMEA362PdGIzQ2W/xY6vnE97HH+V9bjWh7e8bTap2uIfoTuzQM5URERETDoG3XDjRuWI8jzU3QOpxwL1sO28Irh/Q15Gg0bRa5v9CcGqbTZ5v7PjZ00bPNCYJeD1FvgGDQpwVkrd2eci41NOszvs8M1bH/QaOBIAiX8IcloX7ln9JaWAS9Hu5lyy/+OYcQQzkREdEoNxLhjwanbdeOtAAYaQqg/qU/ItzcDEtZeb8zx8nZ5tSZ5dR2juSx3gA+6NlmjQaiXt8bjBMh2WSGNtvRG4zjM9FpIVpv6HPekHZe0OkgiOq9XTHxmWjcsB4RFX5WGMqJiIhGsX7D38o/AcCgwoYsy4AkQZYkQJIAOfG9nDwmpx1PHItfJ6f8LEtpzyWf4/nSr+v/+TKfR04+X8bYMp5P7nfcaa8l9/PcGX8O8nleI2Ucqa8RDmf+GYfDCKxfiwt1MKcF3/hMsRCfbe7bkpGcUU4NyH2uTwvY2vEd/WwLr4Rt4ZXweLLg92e2sihpfFeGiIhohMjRKORwCFI4HOvTDcW+SuFQ7OdwuPd8KJw81ns+FP85fj4ShhwKoavicMZsqRwKoe4P/x1bVaKfMJoWPuNhfLBtCooTBEAUYzOzia9C4uc+54TYsdhXsc91Qsp1IkSdLvYYIX7unK8Rf8601+g91/zWlnMOPe97j2T2RqfMRF9SiwaNWgzlREQ0KKO5VUKW5ViYPWcojv/cbygOp4TiUHqITj0fCsUDc8rjw6FY8L0Egl4PQauDoNdB1Okg6PQQdLpzty9IEsxlZWlhMj2opgbO9GCaGTSF9DCb8Vih/9foG4qTjxHOE3RTwvA5ng+CoPrg2r5nd7+remidLlhnXqHAiEjtGMqJiGjAhqxVQpKSwTY19EopM8AZoTh+LjUU9w3MyVDcN0SnPO6SaDTxQNwbigWdDqJeB0Grg2ixQGO3p4Xm3sfrYsG67zGdPvbzOc4JOh0ErfacIfSLH/3gnOEv5x8furT3SxfNvWy5qm8qJPVhKCciGsdiLRV9Z4dDacG2d+Y3BP/aNf1uvlG/6iV0HtifPosciYfs1NaLeLBGNHpJ484MtLr4LLI+dhObxZISmOMzzPHgmx56dRATwVefCMDxkN03MGu1EDSaSxr3cGD4Uye131RI6sNQTkSkAsn1gJOhuP+ALIdCvTO/iZnjUCg99Pbzfd/nTMwgX2o4To4/GETPyRPxQBuf8TWZobENJhT3+Tk1FKecF7Q61bcujCSGP/VS802FpD4M5USkWkr0LsuyDESjvTO+kdQg3E8oTp1l7hOQ5VB660XaDXuhlLaKUGjwy5r1kd5vrE8LwKLRCMFmy2i7SITnzLCsTzvX+70ep37+E0SamzNeX+t0YeJPn7qk90AXj+GPaPRjKCciVTrnGr9NTTCXTUvpO05drSIxQ9x/P3Lv9ymrWGSE69ClrUKRWAM4dRY45WeN2Zw5S5zaS5z8Xp/ekpEym9x3ZnkkZ47dy+9hqwQR0TBgKCciRciyDKmzE5HWFkRaWhBpaY5/jX3feeAAEO2zzFs4jMCGdRdc4xdAbHWGfgJusrXCbIYmOTscOy9qdWkBOvbYxKxy6o19KX3MfW7SU/PGGUOBrRJERMODoZyIhpzU09MbtFtbEGnuDdzR1t4A3t9KGKLFAq3dkRHIU+X96yMZLRrJtoz4z5e8HTOdE1sliIiGHkM5EQ2YFA7HQ3XmzHakpQXR+PdST0/GtbHd6BzQ2u0wTpqc/D7xVWO3Q5tth6jXAzj/Mm/WGVzjl4iIxhaGciKCHI0i0taGaGrQbo1/n5jlbm2B1NGRca2g1cYCtd0BfUEBzJddBm12InDboXXEvheNpkGNicu8ERHReMJQTjSGyZKEaGdHfAY7dXY7dZa7BdG21sybG0UR2uxsaLLt0Hk8ME0tSZvZTnwvWizD0ibC3mUiIhpPGMqJRiFZliF1d/f2aDc39wbtPu0l/a1DrcnKirWMZDtgKCrqDdrZKe0kNpviNy2yd5mIiMYLhnIilZGCwZRw3Zwyy50+0913V0UAEE2meKh2wFxSFm8rsafMcDugzc6GoOVHn4iISE34NzPRCJEjEURaW9NntJtTQne8h1vq6sq4VtDre2+SnDAB2uwrYoHbkQjaseAtGgwKvDMiIiK6VAzlRHEXu3ukLEmItrf1O5udOssdbW/LvFijSQZqfU4uzGXTkrPaqbPcosnM5f2IiIjGMIZyIpxj98iVf4LUE4RpypT0wJ3o2W5ujvVzt7YCkpT+hIIAjc0Wm8V2OmGcNCk5o62x26FzxEK3xmJVvG+biIiIlMdQToTYCh99e7TlUAgNL7+U8VjRak3ObhvyC/r0bNuhsTugtdkgaDQjNXwiIiIa5RjKadyTJanfTWoScr/9z8l1tzX2bIg6/QiOjoiIiMYDhnIa13qOf4H61avOeV7rdCFr7vwRHBERERGNRwzlNC5FOzrQuHEdWj/cBo3NBtv1N6B9+0fcPZKIiIgUwVBO44osSWjb8REa161FtKsT9hsXw/WVu6AxmWCePIW7RxIREZEiGMpp3OipOYmG1avQU30MxilTUfCNb8JQWJQ8z90jiYiISCkM5TTmRbu6ENi0ES3vvQuN1QrfP3wLtkVXcSlCIiIiUg2GchqzZFlG++6d8K/9M6Jtbci+9nq471oOjcWi9NCIiIiI0jCU05gUrD2DhtWr0F1VCcOEicj/3iMwTpio9LCIiIiI+sVQTmOK1NODwOub0Pzu2xANRni/eT+yr76WrSpERESkagzlNCbIsoyOvZ/A/+dXEWlugu1LV8O9/B5os2xKD42IiIjoghjKadQL1deh4ZWX0fX5IRgKC5H78HdgmjJV6WERERERDRhDOY1aUjCIpjffQPNbb0LQ6eD5u2/Afv0NEDQapYdGRERENCgM5TQqdXz2KRpeW41IYyOyFiyC5557obXblR4WERER0UVR9O63UCiEX/ziF/jSl76EGTNm4Ktf/Sp27tw56Od56KGHUFpaip/+9KfDMEpSk7DfjzPPrUDtf/0Kol6Pgh8+htyHHmYgJyIiolFN0ZnyH//4x3j77bdx3333obi4GBs3bsRDDz2EVatWYdasWQN6jg8++ACffPLJMI+UlCaFw2j+65to2vw6IIpw3/1VOG66GYKW/9hDREREo59iiebAgQPYvHkzHn/8cTzwwAMAgDvvvBNLly7FM888g9WrV1/wOUKhEJ588kl861vfwnPPPTfMIyaldB46iIZXXka4oR7WOXPhuffr0DmdSg+LiIiIaMgo1r7y1ltvQafT4Z577kkeMxgMuPvuu7F37140NDRc8DlWrlyJnp4efOtb3xrOoZJCwk1NqP3Nf+HMiv8PEID8R3+IvO/8CwM5ERERjTmKzZRXVFRg4sSJsPTZ8nzGjBmQZRkVFRXwer3nvN7v9+P555/Hf/zHf8BkMg33cGkEyZEImt99G4HXNwGSBNedy+C45VaIOp3SQyMiIiIaFoqFcr/fD5/Pl3Hc4/EAwAVnyp999llMnDgRX/nKV4ZlfKSMrsoKNLyyCqHaWlhmXgHv330Duvj/J4iIiIjGKsVCeU9PD3T9zHwaDAYAQDAYPOe1Bw4cwF/+8hesWrUKgiAMyXhcLuuQPM9geTxZiryu2oSam3Hijyvh3/YhDF4vyv/3j+GcP0+x8bAu6sOaqBProj6siTqxLuqjtpooFsqNRiPC4XDG8UQYT4TzvmRZxk9/+lPcfPPNmDt37pCNJxDogCTJQ/Z8A+HxZMHvbx/R11QbORpFy/tbEdi0EXI4DOfSL8N561JEDQbF/mxYF/VhTdSJdVEf1kSdWBf1UaomoiiccyJYsVDu8Xj6bVHx+/0AcM5+8nfeeQcHDhzAo48+itOnT6ed6+jowOnTp+F2u2E0God+0DSkuo8dRcPqlQieOgXz9AJAIw0AABPLSURBVMvg/frfQ+/LUXpYRERERCNOsVBeVlaGVatWobOzM+1mz/379yfP96e2thaSJOH+++/POLdhwwZs2LABv/vd73DNNdcMz8DpkkXa29C4bi3atv8NWocTud/5Lqyz5w5ZKxIRERHRaKNYKF+yZAn+8Ic/YO3atcl1ykOhEDZs2IDZs2cnbwKtra1Fd3c3Jk+eDAC44YYbUFBQkPF83/3ud3H99dfj7rvvxvTp00fsfdDAyZKE1g8/QOOG9ZCCPXAsuQ2upXdA5L9qEBER0TinWCifOXMmlixZgmeeeQZ+vx9FRUXYuHEjamtr8eSTTyYf99hjj2HPnj2oqqoCABQVFaGoqKjf5ywsLMRNN900IuOnwek5cRz1L69E8MRxmEpK4f37+2DIy1d6WERERESqoOge5U8//TRWrFiBTZs2obW1FaWlpXjxxRcxZ84cJYdFQyja2YnGDevQ+uEH0GRlIefBf0LWgkVsVSEiIiJKIciyPLJLjqgUV18ZWrIkoW3HdjSuW4NoZwfsN9wE11fugsZsVnpoFzSW6zJasSbqxLqoD2uiTqyL+nD1FRoXgqdqUL96FXqOHYVx8hTkf+OHMBYVKz0sIiIiItViKKchE+3uRmDTBrS8txUaswW+B74F25VXQRBFpYdGREREpGoM5XTJZFlG+55d8K95DdG2NmRfcx3cdy2HxqrMLqlEREREow1DOV2SYO0ZNLzyMrorK2CYMBH5//JvME6cpPSwiIiIiEYVhnK6KFJPDwJv/A+a3/krRIMR3r+/D9nXXMdWFSIiIqKLwFBOgyLLMjr2fQL/a68i0twE21VXw333PdBm2ZQeGhEREdGoxVBOAxaqr0PDKy+j6/ND0BcUovCfvgPT1KlKD4uIiIho1GMopwuSQiE0bXkDzW9tgaDVwvN3X4f9+hshaDRKD42IiIhoTGAop/Pq2P8Z/K+uRrjRj6wFC+G55++gtduVHhYRERHRmMJQTv0KN/rR8Nor6PzsU+hz81Dww8dgLitXelhEREREYxJDOaWRwmE0//VNNG15AwDgXv5VOBbfDEHL/6sQERERDRcmLUrq/PwQGl55GeH6OljnzIXn3q9B53QpPSwiIiKiMY+hnBBuaoJ/zavo+ORj6Lw+5D/yA1guu1zpYRERERGNGwzl45gciaD53bcReH0TIElwfeUuOJbcClGnV3poREREROMKQ/k41VVViYbVKxGqrYVlxkx4vvYN6D1epYdFRERENC4xlI8zkdYW+Nf8Ge27d0LrciHvX/4N1itmKT0sIiIionGNoXyckKNRtHzwHgJ/2QA5HIbz9i/DedtSiAaD0kMjIiIiGvcYyseB7upjaHh5JYKnamCeNh3er38T+pwcpYdFRERERHEM5WNYtL0d/vVr0PbR36B1OJD77X+Gdc48CIKg9NCIiIiIKAVD+RgkSxJa/7YNjevXQQr2wHHLEri+/BWIRpPSQyMiIiKifjCUjzE9J06g/uWXEDxxHKaSUni/cR8M+flKD4uIiIiIzoOhfIyIdnaiceN6tG57H5qsLOQ8+E/IWrCIrSpEREREowBD+SgnSxLadm5H47o1iHZ0wH7DTXB95S5ozGalh0ZEREREA8RQPooFT51C/eqV6Dl2FMZJk5H/6A9hLCpWelhERERENEgM5aNQtLsbgU0b0fLeuxDNZvju/wfYrroagigqPTQiIiIiuggM5aOILMto37Mb/jWvIdrWiuxrroX7rruhsVqVHhoRERERXQKG8lEiWFuLhldWobuyAobiCcj77r/CNGmS0sMiIiIioiHAUK5yUjCIwOub0PzOXyEaDPB+4z5kX3sdW1WIiIiIxhCGcpWSZRkd+/bC/+dXEGlqgu3KL8F991ehtdmUHhoRERERDTGGchUK1dej4dWX0XXoIPQFhSh86NswTS1RelhERERENEwYylVECoXQ9OZmNL+5GYJWC8+9X4P9hpsgaDRKD42IiIiIhhFDuQLadu1A44b1ONLcBK3DCfey5RDNZvhfWY1wox9Z8xfC89V7obU7lB4qEREREY0AhvIR1rZrB+pX/glyKAQAiDQFUPf73wGyDH1OLgp+8COYy6cpPEoiIiIiGkkM5SOsccP6ZCBPkmWIJjOK/+8TELQsCREREdF4w3X1RlikKdDvcam7i4GciIiIaJxiKB9hWqdrUMeJiIiIaOxjKB9h7mXLIej1accEvR7uZcsVGhERERERKY39EiPMtvBKALHe8kjK6iuJ40REREQ0/jCUK8C28ErYFl4JjycLfn+70sMhIiIiIoWxfYWIiIiISGEM5URERERECmMoJyIiIiJSGEM5EREREZHCGMqJiIiIiBTGUE5EREREpDCGciIiIiIihTGUExEREREpjKGciIiIiEhh3NEzThSFcfW6dH6si/qwJurEuqgPa6JOrIv6KFGT872mIMuyPIJjISIiIiKiPti+QkRERESkMIZyIiIiIiKFMZQTERERESmMoZyIiIiISGEM5URERERECmMoJyIiIiJSGEM5EREREZHCGMqJiIiIiBTGUE5EREREpDCGciIiIiIihWmVHsB409DQgJUrV2L//v04dOgQurq6sHLlSixYsEDpoY1bBw4cwMaNG7F7927U1tbCbrdj1qxZeOSRR1BcXKz08MalgwcP4oUXXsDhw4cRCASQlZWFsrIyfPe738X/3979x0Rd/3EAfwKiQWhxiM2AVKxx8WNwsihEyThMUhgsa2f88AeKKciEio20tn4ut8BV/HB01KZb09ZlntwsFWEgYTST0EloIOjdFORHCCcIB36+f6g3rqMff3zlTXyej40/Pu/P+7jn8dln9+J9r/t8Fi1aJDoe3aXVapGXlwelUgm9Xi86jizV1dVh7dq14+47cuQIFi5cOMGJaKyzZ8+isLAQ9fX1GBkZgY+PD9avX48XX3xRdDTZyc3NxXffffeX+6urq/HII49MYCJ7LMonWGtrK7RaLebNmwc/Pz/U19eLjiR7paWlOHPmDGJiYuDn54fOzk589dVXSEhIgE6n45uaAEajEaOjo3j55Zfh6emJ/v5+lJWVITk5GVqtFhEREaIjyl5nZyf27NkDV1dX0VEIwLp16xAQEGAzJrrAkLuqqipkZGQgLCwM27dvx7Rp09DW1oZr166JjiZLGo0G4eHhNmOSJOGdd96Bl5fXpDhfHCRJkkSHkBOz2QyLxQJ3d3eUl5cjIyODK+WCnTlzBoGBgZg+fbp1rK2tDXFxcVi1ahV27dolMB3dMzg4iOjoaAQGBqKkpER0HNnLzc3F1atXIUkS+vr6uFIuyL2V8qKiIkRHR4uOQ3f19/djxYoVWLlyJd566y3RcegvnD59GklJScjOzsaWLVtEx2FP+URzc3ODu7u76Bg0xqJFi2wKcgCYP38+nnjiCbS0tAhKRX/m4uIChUKBvr4+0VFk7+zZszh8+DDefPNN0VFoDLPZjJGREdExCEBZWRn6+vqwfft2AHeODddAJx+DwQAHBwfExsaKjgKARTnRuCRJQldXF/+BEsxsNqOnpweXLl3C7t27cfHiRbuPH2liSZKE999/HwkJCXjyySdFx6G7cnJyEBoaiuDgYKSmpuLChQuiI8naqVOn4Ovri6qqKjz77LMIDQ1FWFgY8vLyMDo6KjoeAbBYLPj++++hUqng7e0tOg4A9pQTjevw4cPo6OhAdna26CiytmPHDhw9ehQA4OzsjDVr1kyKjxjl7NChQ2hubkZRUZHoKIQ758WKFSsQGRkJd3d3XLhwAV9++SUSExOh0+mwYMEC0RFl6fLly2hvb0dubi42bdoEf39/VFZWQqvVYmhoCDt37hQdUfZqamrQ29uLuLg40VGsWJQT/UlLSwvee+89hIaGIj4+XnQcWcvIyIBGo0F7ezv0ej2Gh4dhsVjs2o1oYpjNZuTn52Pz5s2YM2eO6DiEO+13Y69IpFarERUVhdWrV6OwsBD5+fkC08nXwMAAbty4gddffx2bN28GADz//PMYGBjA/v37sXXrVigUCsEp5c1gMMDZ2RkvvPCC6ChWbF8hGqOzsxOvvvoqHnroIXz66adwdOQpIpKfnx8iIiKwevVqfPHFFzh//jz7mAXas2cPnJ2dsWHDBtFR6G8olUqEh4fjp59+Eh1Fth544AEAsOtVjouLg8Viwblz50TEortu3ryJEydOYMmSJZOqTZUVB9Fd/f39SEtLQ39/P0pLS+Hp6Sk6Eo3h7OwMtVqNY8eO4datW6LjyM7169exd+9eJCYmoqurCyaTCSaTCUNDQ7BYLDCZTLhx44bomHTX3LlzeTwEuvf+MXv2bJvxe9s8NmKVl5djcHBwUrWuACzKiQAAQ0ND2LJlC9ra2lBSUgJfX1/RkWgct27dgiRJuHnzpugostPd3Q2LxYK8vDyo1WrrT0NDA1paWqBWq6HVakXHpLuMRuOkWgGUm3vXjO/o6LAZb29vBwC2rghWVlYGV1dXREVFiY5igz3lJHujo6PIysrCr7/+iuLiYoSEhIiOJHs9PT12b1pmsxlHjx7F3Llz4eHhISiZfHl7e4/75c5PPvkEAwMD2LFjB+bPnz/xwWRuvHPl9OnTqKurQ0JCgqBUFBMTA61WC51OZ71ggCRJ+Oabb+Dq6sr3GYF6enpw6tQprFq1Ci4uLqLj2GBRLkBxcTEAWK+Brdfr8csvv2DWrFlITk4WGU2Wdu3ahYqKCjz33HPo7e21uQnKgw8+yBtyCJCVlYUZM2ZApVLB09MT165dw8GDB9He3o7du3eLjidLM2fOHPdc2Lt3L5ycnHieCJKVlQUXFxeoVCq4u7vj999/x9dffw13d3dkZmaKjidbgYGBSEhIQElJCbq7u+Hv74+qqirU1NQgJycHbm5uoiPK1pEjRzAyMjLpWlcA3tFTCD8/v3HHvby8UFFRMcFpKCUlBT///PO4+3hMxNDpdNDr9WhubkZfXx9mzpyJkJAQpKamIiwsTHQ8GiMlJYV39BRo3759KCsrw5UrV2A2m6FQKLBkyRJkZmbi0UcfFR1P1oaHh1FcXIxDhw6hq6sL3t7eWL9+PdasWSM6mqxpNBoYjUacPHkSTk5OouPYYFFORERERCQYv+hJRERERCQYi3IiIiIiIsFYlBMRERERCcainIiIiIhIMBblRERERESCsSgnIiIiIhKMRTkRERERkWAsyomISJiUlBRERUWJjkFEJNw00QGIiOj/q66uDmvXrv3L/U5OTmhsbJzARERE9E9YlBMRTVGxsbGIjIy0G3d05IekRESTDYtyIqIpyt/fH/Hx8aJjEBHRv8DlEiIimTKZTPDz80NBQQEMBgPi4uIQFBSEZcuWoaCgACMjI3aPaWpqQkZGBp5++mkEBQVh5cqV0Gq1GB0dtZvb2dmJDz74AGq1GoGBgQgPD8eGDRvw448/2s3t6OjAa6+9hqeeegrBwcHYuHEjWltb78vrJiKajLhSTkQ0RQ0ODqKnp8dufPr06XBzc7NuV1RUwGg0IikpCbNnz0ZFRQUKCwtx9epVfPTRR9Z5586dQ0pKCqZNm2adW1lZiby8PDQ1NSE/P98612Qy4ZVXXkF3dzfi4+MRGBiIwcFBNDQ0oLa2FhEREda5AwMDSE5ORnBwMLKzs2EymbBv3z6kp6fDYDDAycnpPv2FiIgmDxblRERTVEFBAQoKCuzGly1bhpKSEut2U1MTdDodAgICAADJycnYtm0bDh48CI1Gg5CQEADAhx9+iOHhYRw4cABKpdI6NysrCwaDAS+99BLCw8MBAO+++y6uX7+O0tJSLF261Ob5b9++bbP9xx9/YOPGjUhLS7OOKRQKfPzxx6itrbV7PBHRVMSinIhoitJoNIiJibEbVygUNtuLFy+2FuQA4ODggE2bNqG8vBzHjx9HSEgIuru7UV9fj+XLl1sL8ntzt27dih9++AHHjx9HeHg4ent7cfLkSSxdunTcgvrPXzR1dHS0u1rMM888AwC4fPkyi3IikgUW5UREU9S8efOwePHif5y3cOFCu7HHH38cAGA0GgHcaUcZOz6Wr68vHB0drXOvXLkCSZLg7+//r3LOmTMHM2bMsBl7+OGHAQC9vb3/6ncQEf3X8YueREQk1N/1jEuSNIFJiIjEYVFORCRzLS0tdmPNzc0AAB8fHwCAt7e3zfhYly5dwu3bt61zH3vsMTg4OOC33367X5GJiKYcFuVERDJXW1uL8+fPW7clSUJpaSkAIDo6GgDg4eEBlUqFyspKXLx40Wbu559/DgBYvnw5gDutJ5GRkaiurkZtba3d83H1m4jIHnvKiYimqMbGRuj1+nH33Su2AUCpVGLdunVISkqCp6cnTpw4gdraWsTHx0OlUlnn7dy5EykpKUhKSkJiYiI8PT1RWVmJmpoaxMbGWq+8AgBvv/02GhsbkZaWhoSEBAQEBGBoaAgNDQ3w8vJCTk7O/XvhRET/QSzKiYimKIPBAIPBMO6+Y8eOWXu5o6KisGDBApSUlKC1tRUeHh5IT09Henq6zWOCgoJw4MABfPbZZ9i/fz8GBgbg4+ODN954A6mpqTZzfXx88O2336KoqAjV1dXQ6/WYNWsWlEolNBrN/XnBRET/YQ4SP0ckIpIlk8kEtVqNbdu2ITMzU3QcIiJZY085EREREZFgLMqJiIiIiARjUU5EREREJBh7yomIiIiIBONKORERERGRYCzKiYiIiIgEY1FORERERCQYi3IiIiIiIsFYlBMRERERCcainIiIiIhIsP8BTfOw7NM6aGIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot Trainingloss, Validation loss and Weighted F1 score for each epoch\n",
    "\n",
    "sns.set(style='darkgrid')\n",
    "sns.set(font_scale=1.5)\n",
    "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
    "plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
    "plt.plot(df_stats['Valid. Loss all'], 'g-o', label=\"Validation\")\n",
    "plt.plot(df_stats['Weigh_F1 all (no random)'], 'r-o', label=\"Weighted F1 (all, no rand)\")\n",
    "plt.title(\"Training & Validation Loss\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vzwvY1JTfjv2"
   },
   "outputs": [],
   "source": [
    "# save model\n",
    "#PATH = \"xlmr_model2409.pt\"\n",
    "#torch.save(xlmr_model, PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aMlqA-L-gZum"
   },
   "outputs": [],
   "source": [
    "#load\n",
    "PATH = \"xlmr_model2409.pt\"\n",
    "xlmr_model=torch.load(PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Q12682pRYiFF"
   },
   "source": [
    "# CogALex Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vFcYm817535q"
   },
   "outputs": [],
   "source": [
    "# read in test data\n",
    "test_data_zh = pd.read_csv('data/test/test_chinese_data.txt', sep=\"\\t\", header=None)\n",
    "test_data_zh.columns = [\"Word1\", \"Word2\"]\n",
    "\n",
    "test_data_de = pd.read_csv('data/test/test_german_data.txt', sep=\"\\t\", header=None)\n",
    "test_data_de.columns = [\"Word1\", \"Word2\"]\n",
    "\n",
    "test_data_en = pd.read_csv('data/test/test_english_data.txt', sep=\"\\t\", header=None)\n",
    "test_data_en.columns = [\"Word1\", \"Word2\"]\n",
    "\n",
    "test_data_it = pd.read_csv('data/test/test_italian_data.txt', sep=\"\\t\", header=None)\n",
    "test_data_it.columns = [\"Word1\", \"Word2\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "G52kBcg8YlVL"
   },
   "outputs": [],
   "source": [
    "# annotates a file that was already parsed to a dataframe (uses tokenizer from before)\n",
    "\n",
    "def annotate_test_data(test_data, output_file_name):\n",
    "  file = open(output_file_name,\"w\") \n",
    "  for index, row in test_data.iterrows():\n",
    "      word1=row[\"Word1\"]\n",
    "      word2=row[\"Word2\"]  \n",
    "      encoded_dict = xlmr_tokenizer.encode_plus(word1, word2, \n",
    "                                                  max_length=max_len, \n",
    "                                                  padding='max_length',\n",
    "                                                  truncation=True, \n",
    "                                                  return_tensors='pt') \n",
    "      input_id=encoded_dict['input_ids'].to(device)\n",
    "      attn_mask=encoded_dict['attention_mask'].to(device)\n",
    "      label=torch.tensor(0).to(device)                    #0=antonym, 1=hypony, 2=random, 3=synonym\n",
    "      (loss, logits) = xlmr_model(input_id, \n",
    "                                    token_type_ids=None, \n",
    "                                    attention_mask=attn_mask,\n",
    "                                    labels=label)\n",
    "      logits = logits.detach().cpu().numpy()\n",
    "\n",
    "      #write to file \n",
    "      file.write(word1+\"\\t\"+word2+\"\\t\"+labels[logits[0].argmax(axis=0)]+\"\\n\") \n",
    "  file.close() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Gzjd8BSd57cT"
   },
   "outputs": [],
   "source": [
    "#annotate the unlabeled test sets\n",
    "annotate_test_data(test_data_it, \"xlmr_it_annotated.txt\")\n",
    "annotate_test_data(test_data_zh, \"xlmr_zh_annotated.txt\")\n",
    "annotate_test_data(test_data_en, \"xlmr_en_annotated.txt\")\n",
    "annotate_test_data(test_data_de, \"xlmr_de_annotated.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 153
    },
    "id": "lTRGaSP9-fB4",
    "outputId": "20ee8931-038c-4113-ef6d-9aba03baf08c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validgold_chinese_data.txt\n",
      "valid_chinese_xlmr.txt\n",
      "Loading the datasets ...\n",
      "Evaluating ...\n",
      "      SYN:  P = 0.862  R = 0.822  F1 = 0.841  sup = 129\n",
      "      HYP:  P = 0.938  R = 0.841  F1 = 0.887  sup = 145\n",
      "      ANT:  P = 0.877  R = 0.838  F1 = 0.857  sup = 136\n",
      "Overall:  P = 0.894  R = 0.834  F1 = 0.863\n"
     ]
    }
   ],
   "source": [
    "#zh\n",
    "annotate_test_data(data_valid_zh, \"valid_chinese_xlmr.txt\")\n",
    "#first arg: gold standard file, second arg: our results\n",
    "! python evaluation_grade.py validgold_chinese_data.txt valid_chinese_xlmr.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 153
    },
    "id": "ovje4QmYlGCN",
    "outputId": "4865ae21-420e-4cc5-812d-493cabea62ba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validgold_english_data.txt\n",
      "valid_english_xlmr.txt\n",
      "Loading the datasets ...\n",
      "Evaluating ...\n",
      "      SYN:  P = 0.492  R = 0.463  F1 = 0.477  sup = 259\n",
      "      HYP:  P = 0.597  R = 0.473  F1 = 0.528  sup = 292\n",
      "      ANT:  P = 0.801  R = 0.497  F1 = 0.613  sup = 308\n",
      "Overall:  P = 0.639  R = 0.478  F1 = 0.543\n"
     ]
    }
   ],
   "source": [
    "#en\n",
    "annotate_test_data(data_valid_en, \"valid_english_xlmr.txt\")\n",
    "! python evaluation_grade.py validgold_english_data.txt valid_english_xlmr.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 153
    },
    "id": "sP28N7uJlGLO",
    "outputId": "efabb2fb-be83-4b56-ea31-37fa83a360d2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validgold_german_data.txt\n",
      "valid_german_xlmr.txt\n",
      "Loading the datasets ...\n",
      "Evaluating ...\n",
      "      SYN:  P = 0.507  R = 0.419  F1 = 0.459  sup = 272\n",
      "      HYP:  P = 0.595  R = 0.459  F1 = 0.518  sup = 294\n",
      "      ANT:  P = 0.673  R = 0.411  F1 = 0.510  sup = 275\n",
      "Overall:  P = 0.592  R = 0.430  F1 = 0.496\n"
     ]
    }
   ],
   "source": [
    "#de\n",
    "annotate_test_data(data_valid_de, \"valid_german_xlmr.txt\")\n",
    "! python evaluation_grade.py validgold_german_data.txt valid_german_xlmr.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FueN-MqmmXGm"
   },
   "source": [
    "# GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "yOyTujm7VY0A"
   },
   "outputs": [],
   "source": [
    "def create_model(lr, eps, train_dataloader, epochs):\n",
    "  xlmr_model = XLMRobertaForSequenceClassification.from_pretrained(\"xlm-roberta-base\", num_labels=4)\n",
    "  device = torch.device('cuda')\n",
    "  desc = xlmr_model.to(device)\n",
    "  optimizer = AdamW(xlmr_model.parameters(),\n",
    "                  lr = lr,   \n",
    "                  eps = eps       \n",
    "                )\n",
    "  total_steps = len(train_dataloader) * epochs\n",
    "  scheduler = get_linear_schedule_with_warmup(optimizer, \n",
    "                                            num_warmup_steps = 0,   \n",
    "                                            num_training_steps = total_steps)\n",
    "  return xlmr_model, optimizer, scheduler\n",
    "  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "h14K_Kt3jIRj",
    "outputId": "ac1c96a0-1652-463d-cea8-dd5724922197"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grid size: 3\n",
      "[{'batch_size': 32, 'epochs': 3, 'eps': 1e-08, 'lr': 2e-05, 'max_len': 64, 'model_name': 'xlm-roberta-base', 'random_seed': 42, 'warmup_steps': 0, 'weight_decay': 0}, {'batch_size': 32, 'epochs': 3, 'eps': 1e-08, 'lr': 1e-05, 'max_len': 64, 'model_name': 'xlm-roberta-base', 'random_seed': 42, 'warmup_steps': 0, 'weight_decay': 0}, {'batch_size': 32, 'epochs': 3, 'eps': 1e-08, 'lr': 2e-06, 'max_len': 64, 'model_name': 'xlm-roberta-base', 'random_seed': 42, 'warmup_steps': 0, 'weight_decay': 0}]\n"
     ]
    }
   ],
   "source": [
    "#parameters checked in gridsearch\n",
    "\n",
    "parameter_grid=list(ParameterGrid({\n",
    "    \"model_name\":[\"xlm-roberta-base\"],\n",
    "    \"batch_size\":[32], \n",
    "    \"lr\":[2e-5, 1e-5, 2e-6], \n",
    "    \"eps\":[1e-8], \n",
    "    \"max_len\":[64], \n",
    "    \"epochs\":[3], \n",
    "    \"warmup_steps\":[0],\n",
    "    \"weight_decay\":[0],\n",
    "    \"random_seed\": [42]\n",
    "    #\"number_samples\": [100,500,1000,5000,10000,12606]\n",
    "    }))\n",
    "\n",
    "print(\"Grid size:\",len(parameter_grid))\n",
    "print(parameter_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "kHV1TX4WjIRk"
   },
   "outputs": [],
   "source": [
    "def gridsearch(parameter_grid, data_train):\n",
    "\n",
    "  validation_dataloader_set=[validation_dataloader_all, validation_dataloader_zh, validation_dataloader_en, validation_dataloader_de]\n",
    "\n",
    "  print(\"Starting Gridsearch\")\n",
    "\n",
    "  results=[]\n",
    "\n",
    "  for i, parameters in enumerate(parameter_grid):\n",
    "\n",
    "    #number_samples=parameters[\"number_samples\"]\n",
    "    #data_train=data_train_all.sample(number_samples)\n",
    "\n",
    "    model_name=parameters[\"model_name\"]\n",
    "    batch_size=parameters[\"batch_size\"]\n",
    "    lr=parameters[\"lr\"]\n",
    "    eps=parameters[\"eps\"]\n",
    "    max_len=parameters[\"max_len\"]\n",
    "    epochs=parameters[\"epochs\"]\n",
    "    warmup_steps=parameters[\"warmup_steps\"]\n",
    "    random_seed=parameters[\"random_seed\"]\n",
    "\n",
    "    print(\"\\n\"+\"\\033[1m\"+\"============== Iteration \"+str(i+1)+\"/\"+str(len(parameter_grid))+\" ==============\"+\"\\033[0m\"+\"\\n\")\n",
    "    print(parameters)\n",
    "    print(\"length training data:\", len(data_train))\n",
    "    #print(data_train)\n",
    "    \n",
    "    # prepare chosen trainings data\n",
    "    xlmr_tokenizer = XLMRobertaTokenizer.from_pretrained(model_name)\n",
    "    input_ids_train, attn_masks_train, labels_train = tokenizer_xlm(data_train, max_len)\n",
    "    tensor_data_train = TensorDataset(input_ids_train, attn_masks_train, labels_train)\n",
    "    train_dataloader = DataLoader(\n",
    "                tensor_data_train,  \n",
    "                sampler = RandomSampler(tensor_data_train), \n",
    "                batch_size = batch_size \n",
    "            )\n",
    "\n",
    "    \n",
    "    # prepare model\n",
    "    xlmr_model, optimizer, scheduler = create_model(lr, eps, train_dataloader, epochs) \n",
    "\n",
    "    # train and save stats\n",
    "    training_stats=train_model( epochs=epochs, \n",
    "                                xlmr_model=xlmr_model, \n",
    "                                train_dataloader=train_dataloader, \n",
    "                                validation_dataloader_set=validation_dataloader_set,\n",
    "                                random_seed=random_seed,\n",
    "                                verbose = False) \n",
    "  \n",
    "    df_stats = pd.DataFrame(data=training_stats)\n",
    "\n",
    "    #parameters, valid loss, min valid lost, macro f1\n",
    "    results.append((parameters,\n",
    "                    training_stats[len(training_stats)-1][\"Valid. Loss all\"],\n",
    "                    min(df_stats[\"Valid. Loss all\"]),\n",
    "                    training_stats[len(training_stats)-1]['Weigh_F1 all (no random)']))\n",
    "  return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lYikFsUAa2rs"
   },
   "outputs": [],
   "source": [
    "# run gridsearch\n",
    "results = gridsearch(parameter_grid, data_train_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MTxv3ep2y2qv"
   },
   "outputs": [],
   "source": [
    "# show results\n",
    "for r in results: print(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HHqMu3-NBFjr"
   },
   "outputs": [],
   "source": [
    "# sort results by best \n",
    "sorted(results, key=lambda tup: tup[1])"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "Q12682pRYiFF"
   ],
   "include_colab_link": true,
   "name": "CogALex XLM RoBERTa - Updated Task Version",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0bca7a6b07ac4b7f9980679b06dcb798": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0db3ca8db482422ca68db57d980c0643": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "3753df9760b643dba4e0aa54313b1689": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5bca0b4663d3475fb9ca4793783b2130",
      "max": 5069051,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0db3ca8db482422ca68db57d980c0643",
      "value": 5069051
     }
    },
    "392d1b126ea549fea9b77e317929b5c9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3b82354faba14c37bc6419d1d033df03": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "55513c30e3194e5b8b98793776098b14": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9fcfd2bc25434ab1b3b478f6528c577b",
      "max": 1115590446,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3b82354faba14c37bc6419d1d033df03",
      "value": 1115590446
     }
    },
    "5bca0b4663d3475fb9ca4793783b2130": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "745bdbc2dd454163bc3de08b80a19806": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9fcfd2bc25434ab1b3b478f6528c577b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b07f70c2f1e4473bb8a90fa65d281183": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b5e15dfc072a46ad9367eaafa17f6313": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b96064f9c88b4d5fb605842e22ee5ec8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0bca7a6b07ac4b7f9980679b06dcb798",
      "placeholder": "​",
      "style": "IPY_MODEL_745bdbc2dd454163bc3de08b80a19806",
      "value": " 1.12G/1.12G [00:19&lt;00:00, 56.5MB/s]"
     }
    },
    "ba89918f181649f2bd10ce5c6f1e6d48": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3753df9760b643dba4e0aa54313b1689",
       "IPY_MODEL_ff08b72911f14f77a8c51fbf0287ef15"
      ],
      "layout": "IPY_MODEL_d79b07b8495c416c83cbfdfffb3d8e1c"
     }
    },
    "c69e9514345944a18780f74fd55cc97f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_55513c30e3194e5b8b98793776098b14",
       "IPY_MODEL_b96064f9c88b4d5fb605842e22ee5ec8"
      ],
      "layout": "IPY_MODEL_392d1b126ea549fea9b77e317929b5c9"
     }
    },
    "d79b07b8495c416c83cbfdfffb3d8e1c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ff08b72911f14f77a8c51fbf0287ef15": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b5e15dfc072a46ad9367eaafa17f6313",
      "placeholder": "​",
      "style": "IPY_MODEL_b07f70c2f1e4473bb8a90fa65d281183",
      "value": " 5.07M/5.07M [00:02&lt;00:00, 2.26MB/s]"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}