06-train_HeterSUMGraph_GATv2.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train HeterSUMGraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import argparse\n",
    "from time import time\n",
    "import torch\n",
    "import numpy as np\n",
    "import random\n",
    "from sklearn.utils import shuffle\n",
    "import json\n",
    "import os\n",
    "import networkx as nx\n",
    "import torch_geometric\n",
    "from statistics import mean\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from rouge_score import rouge_scorer\n",
    "\n",
    "from utils.create_graph_dataset import create_graph_dataset\n",
    "from utils.EmbMgr import EmbMgr\n",
    "from utils.DataLoader import DataLoader\n",
    "from utils.accuracy_nb_sent_per_doc import accuracy_nb_sent_per_doc_fn\n",
    "from utils.split_all_docs import split_all_docs\n",
    "from utils.threshold_probs import threshold_probs_by_nb, threshold_probs_by_prop\n",
    "from utils.accuracy_prop_sent_per_doc import accuracy_prop_sent_per_doc_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_notebook() -> bool:\n",
    "    try:\n",
    "        shell = get_ipython().__class__.__name__\n",
    "        if shell == 'ZMQInteractiveShell':\n",
    "            return True   # Jupyter notebook or qtconsole\n",
    "        elif shell == 'TerminalInteractiveShell':\n",
    "            return False  # Terminal running IPython\n",
    "        else:\n",
    "            return False  # Other type (?)\n",
    "    except NameError:\n",
    "        return False      # Probably standard Python interpreter"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (Hyper-)parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse args if script mode\n",
    "parser = argparse.ArgumentParser(description='extractive summary')\n",
    "\n",
    "parser.add_argument('-is_graphic',type=int,default=1,choices=[0,1])\n",
    "parser.add_argument('-gpu_num',type=int,default=1)\n",
    "parser.add_argument('-batch_size',type=int,default=4)\n",
    "parser.add_argument('-epochs',type=int,default=10)\n",
    "parser.add_argument('-dataset',type=str,default=\"data/wiki_geo_ratio_sc_0.5\")\n",
    "parser.add_argument('-dataset_tfidf',type=str,default=\"./data/wiki_geo_ratio_sc_0.5_dataset_tfidf.json\")\n",
    "parser.add_argument('-sent_tfidf',type=str,default=\"data/wiki_geo_ratio_sc_0.5_sent_tfidf.json\")\n",
    "parser.add_argument('-doc_column_name',type=str,default=\"flat_contents\")\n",
    "parser.add_argument('-labels_column_name',type=str,default=\"labels_sentences\")\n",
    "parser.add_argument(\"-word_emb\",type=str,default=\"./data/cc.fr.300.vec\")\n",
    "\n",
    "args = None\n",
    "\n",
    "if is_notebook():\n",
    "    args = parser.parse_args(\"\")\n",
    "else:\n",
    "    args = parser.parse_args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parse:\n",
      "is_graphic: True\n",
      "cuda_num: 1\n",
      "epochs 100\n",
      "batch_size 4\n"
     ]
    }
   ],
   "source": [
    "# parameters\n",
    "is_graphic = args.is_graphic != 0\n",
    "cuda_num = args.gpu_num\n",
    "embed_name = \"cc.fr.300.vec\"\n",
    "\n",
    "# hyper-parameters\n",
    "vocab_size = 50000\n",
    "batch_size = args.batch_size\n",
    "epochs = args.epochs\n",
    "learning_rate = 5e-4\n",
    "early_stopping = 3\n",
    "word_blacklist_proportion = 0.1\n",
    "model_name = \"06-train_HeterSUMGraph_GATv2\"\n",
    "sub_folder_name = \"model_name__{}__time__{}__embed_name__{}__lr__{}__batch_size__{}__vocab_size__{}__cuda_num__{}__early_stopping__{}\".format(model_name, time(), embed_name, learning_rate, batch_size, vocab_size, cuda_num, early_stopping)\n",
    "checkpoints_folder = \"./checkpoints/\" + sub_folder_name\n",
    "average_number_of_sentences_per_document = 3\n",
    "\n",
    "# print\n",
    "print(\"parse:\")\n",
    "print(\"is_graphic:\", is_graphic)\n",
    "print(\"cuda_num:\", cuda_num)\n",
    "print(\"epochs\", epochs)\n",
    "print(\"batch_size\", batch_size)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch initialisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(0)\n",
    "random.seed(0)\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of available GPUs: 6\n",
      "GPU 0: NVIDIA GeForce GTX 1080 Ti\n",
      "GPU 1: NVIDIA GeForce GTX 1080 Ti\n",
      "GPU 2: NVIDIA GeForce GTX 1080 Ti\n",
      "GPU 3: NVIDIA GeForce GTX 1080\n",
      "GPU 4: NVIDIA GeForce GTX 1080\n",
      "GPU 5: NVIDIA GeForce GTX 1080\n"
     ]
    }
   ],
   "source": [
    "# Check if a GPU is available\n",
    "if torch.cuda.is_available():\n",
    "  # Display the number of available GPUs\n",
    "  print(f\"Number of available GPUs: {torch.cuda.device_count()}\")\n",
    "  # Display the name of each GPU\n",
    "  for i in range(torch.cuda.device_count()):\n",
    "    print(f\"GPU {i}: {torch.cuda.get_device_name(i)}\")\n",
    "elif torch.backends.mps.is_available():\n",
    "  print(\"MPS available.\")\n",
    "else:\n",
    "  print(\"No GPU available.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=1)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if torch.cuda.is_available():  \n",
    "  dev = \"cuda:\" + str(cuda_num) \n",
    "elif torch.backends.mps.is_available():\n",
    "  dev = torch.device(\"mps\")\n",
    "else:  \n",
    "  dev = \"cpu\" \n",
    "\n",
    "device = torch.device(dev)\n",
    "device"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "embmgr = EmbMgr(args.word_emb, vocab_size=vocab_size, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidfs_sent = pd.read_json(args.sent_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidfs_dataset = json.load(open(args.dataset_tfidf))\n",
    "tfidfs_dataset_sorted = sorted(tfidfs_dataset, key=tfidfs_dataset.get)\n",
    "word_blacklist = set(tfidfs_dataset_sorted[:int(word_blacklist_proportion*len(tfidfs_dataset_sorted))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_json(args.dataset + \"/test.json\")\n",
    "df_val = pd.read_json(args.dataset + \"/val.json\")\n",
    "df_train = pd.read_json(args.dataset + \"/train.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = create_graph_dataset(df=df_train, tfidfs_sent=tfidfs_sent, embmgr=embmgr, word_blacklist = word_blacklist, remove_unkn_words=False, doc_column_name=args.doc_column_name, labels_sum_column_name=args.labels_column_name, is_sep_n=False, remove_stop_word = True, stemming=False, trunc_sent=-1, padding_sent=-1, trunc_doc=100)\n",
    "val_dataset = create_graph_dataset(df=df_val, tfidfs_sent=tfidfs_sent, embmgr=embmgr, word_blacklist = word_blacklist, remove_unkn_words=False, doc_column_name=args.doc_column_name, labels_sum_column_name=args.labels_column_name, is_sep_n=False, remove_stop_word = True, stemming=False, trunc_sent=-1, padding_sent=-1, trunc_doc=100)\n",
    "test_dataset = create_graph_dataset(df=df_test, tfidfs_sent=tfidfs_sent, embmgr=embmgr, word_blacklist = word_blacklist, remove_unkn_words=False, doc_column_name=args.doc_column_name, labels_sum_column_name=args.labels_column_name, is_sep_n=False, remove_stop_word = True, stemming=False, trunc_sent=-1, padding_sent=-1, trunc_doc=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "val_loader =  DataLoader(val_dataset, batch_size=1, shuffle=False)\n",
    "test_loader =  DataLoader(test_dataset, batch_size=1, shuffle=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Graph stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average number of distinct word per doc: 35.23\n"
     ]
    }
   ],
   "source": [
    "# Average number of distinct word per doc\n",
    "print(\"Average number of distinct word per doc:\", mean([len(t[\"docs\"].x[0]) for t in train_dataset]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average sentence len: 14.165876777251185\n"
     ]
    }
   ],
   "source": [
    "def sents_len(doc):\n",
    "  return [torch.sum(torch.sign(s),dim=0).item() for s in doc]\n",
    "\n",
    "# Average sentence length\n",
    "lens = []\n",
    "for t in train_dataset:\n",
    "  lens = lens + sents_len(t[\"docs\"].x[1])\n",
    "\n",
    "print(\"Average sentence len:\", mean(lens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average density: 0.12510346157363167\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/raoufdine/anaconda3/lib/python3.9/site-packages/torch_geometric/data/storage.py:303: UserWarning: Unable to accurately infer 'num_nodes' from the attribute set '{'edge_attr', 'undirected', 'x', 'edge_index'}'. Please explicitly set 'num_nodes' as an attribute of 'data' to suppress this warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Average density\n",
    "def density(g):\n",
    "  nb_edges = g.num_edges / 2\n",
    "  nb_nodes = g.num_nodes\n",
    "  nb_poss_edges = nb_nodes * (nb_nodes - 1) / 2\n",
    "  return nb_edges / nb_poss_edges\n",
    "\n",
    "print(\"Average density:\", mean([density(t[\"docs\"]) for t in train_dataset]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average clustering: 0.0\n"
     ]
    }
   ],
   "source": [
    "# Average clustering\n",
    "def average_clustering(g):\n",
    "  G = nx.Graph()\n",
    "  edge_list = g.edge_index.t().tolist()\n",
    "  G.add_edges_from(edge_list)\n",
    "  return nx.average_clustering(G)\n",
    "\n",
    "print(\"Average clustering:\", mean([average_clustering(t[\"docs\"]) for t in train_dataset]))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "from torch_geometric.nn import GATv2Conv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Position Feed Forward\n",
    "class PositionwiseFeedForward(nn.Module):\n",
    "  def __init__(self, d_in, d_out, d_h, dropout=0.1) -> None:\n",
    "    super(PositionwiseFeedForward, self).__init__()\n",
    "    self.fc1 = nn.Linear(in_features=d_in, out_features=d_h)\n",
    "    self.fc2 = nn.Linear(in_features=d_h, out_features=d_out)\n",
    "    self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "  def forward(self, x):\n",
    "    x = torch.relu(self.fc1(x))\n",
    "    x = self.dropout(x)\n",
    "    return self.fc2(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HeterSUMGraph(torch.nn.Module):\n",
    "  def __init__(self, device, vocab_size, word_embed = None, dim_word_embed = 300, num_iter=3):\n",
    "    super(HeterSUMGraph, self).__init__()\n",
    "    self.device = device\n",
    "    self.num_iter = num_iter\n",
    "\n",
    "    self.word_emb = nn.Embedding(vocab_size+2, dim_word_embed, padding_idx=0)\n",
    "\n",
    "    # Load word embedding if specified\n",
    "    if word_embed is not None:\n",
    "      self.word_embedding = torch.nn.Embedding.from_pretrained(torch.from_numpy(word_embed).float())\n",
    "\n",
    "    self.word_emb.weight.requires_grad = False\n",
    "\n",
    "    # edge_attr embed\n",
    "    self.edge_attr_embed = nn.Embedding(101, 50)\n",
    "\n",
    "    # sent CNN\n",
    "    dim_out_cnn = 64\n",
    "    self.convs = nn.Sequential(\n",
    "                            nn.Conv1d(in_channels=300,out_channels=100,kernel_size=3,padding='same'),\n",
    "                            nn.BatchNorm1d(num_features=100),\n",
    "                            nn.ReLU(inplace=True),\n",
    "                            nn.Conv1d(in_channels=100,out_channels=100,kernel_size=4,padding='same'),\n",
    "                            nn.BatchNorm1d(num_features=100),\n",
    "                            nn.ReLU(inplace=True),\n",
    "                            nn.Conv1d(in_channels=100,out_channels=dim_out_cnn,kernel_size=5,padding='same'),\n",
    "                            nn.BatchNorm1d(num_features=dim_out_cnn),\n",
    "                            nn.ReLU(inplace=True))\n",
    "\n",
    "    # sent LSTM\n",
    "    dim_out_lstm = 32\n",
    "    self.sent_LSTM = nn.LSTM(input_size=300, hidden_size=dim_out_lstm, batch_first=True, bidirectional=True)\n",
    "\n",
    "    # sent to word GAT\n",
    "    self.s2wGAT = GATv2Conv(in_channels=[128, 300], out_channels=300, heads=8, edge_dim=50, fill_value=0, share_weights=False)\n",
    "\n",
    "    # word to sen GAT\n",
    "    self.w2sGAT = GATv2Conv(in_channels=[300, 128], out_channels=128, heads=8, edge_dim=50, fill_value=0, share_weights=False)\n",
    "\n",
    "    # ffn sent to word\n",
    "    self.ffns2w = PositionwiseFeedForward(d_in=8*300, d_out=300, d_h=512)\n",
    "\n",
    "    # ffn word to sent\n",
    "    self.ffnw2s = PositionwiseFeedForward(d_in=8*128, d_out=128, d_h=512)\n",
    "\n",
    "    self.wh = nn.Linear(128, 1)\n",
    "\n",
    "  def max_pool1d(self, x, seq_lens):\n",
    "    out = []\n",
    "    for index,t in enumerate(x):\n",
    "      if seq_lens[index] == 0:\n",
    "        t = t[:1]\n",
    "      else:\n",
    "        t = t[:seq_lens[index],:]\n",
    "      t = torch.t(t).unsqueeze(0)\n",
    "      out.append(torch.avg_pool1d(t, t.size(2)))\n",
    "\n",
    "    out = torch.cat(out).squeeze(2)\n",
    "    return out\n",
    "\n",
    "  def forward(self, data):# In the real implémentation remove nb_sent\n",
    "    # edge attr embeddings\n",
    "    edge_attr = torch.floor(data.edge_attr * 100).int()\n",
    "    edge_attr = self.edge_attr_embed(edge_attr)\n",
    "\n",
    "    # features\n",
    "    Xw = data.x[0]\n",
    "    Xs = data.x[1]\n",
    "    sent_lens = torch.sum(torch.sign(Xs),dim=1).data\n",
    "    \n",
    "    # embeddings\n",
    "    Xw = self.word_emb(Xw)\n",
    "    Xs = self.word_emb(Xs)\n",
    "\n",
    "    # CNN\n",
    "    XsCNN = self.convs(Xs.permute(0,2,1))\n",
    "    XsCNN = torch.max_pool1d(XsCNN, XsCNN.size(2)).squeeze(2)\n",
    "\n",
    "    # LSTM\n",
    "    # TO DO: ask \n",
    "    XsLSTM, _ = self.sent_LSTM(Xs)\n",
    "    XsLSTM = self.max_pool1d(XsLSTM, sent_lens)\n",
    "    Xs = torch.cat((XsCNN, XsLSTM), dim=1)\n",
    "\n",
    "    Hw = Xw\n",
    "    Hs = Xs\n",
    "\n",
    "    edge_index_w_i = data.edge_index[1][data.edge_index[1] < len(data.x[0])]\n",
    "    edge_index_s_j = data.edge_index[0][data.edge_index[1] < len(data.x[0])] - Hw.shape[0]\n",
    "    edge_attr_w = edge_attr[data.edge_index[1] < len(data.x[0])]\n",
    "    edge_index_w = torch.stack([edge_index_s_j, edge_index_w_i])\n",
    "\n",
    "    edge_index_s_i = data.edge_index[1][data.edge_index[1] >= len(data.x[0])] - Hw.shape[0]\n",
    "    edge_index_w_j = data.edge_index[0][data.edge_index[1] >= len(data.x[0])]\n",
    "    edge_attr_s = edge_attr[data.edge_index[1] >= len(data.x[0])]\n",
    "    edge_index_s = torch.stack([edge_index_w_j, edge_index_s_i])\n",
    "\n",
    "    Hw = torch.layer_norm(Hw, Hw.size()[1:])\n",
    "    Hs = torch.layer_norm(Hs, Hs.size()[1:])\n",
    "\n",
    "    for i in range(self.num_iter):\n",
    "      if i != 0: # H^1w = H^0w = Xw\n",
    "        Hw = self.s2wGAT(x=[Hs, Hw], edge_index=edge_index_w, edge_attr=edge_attr_w) + torch.cat([Hw] * 8, dim=1)\n",
    "        Hw = self.ffns2w(Hw)\n",
    "        Hw = torch.layer_norm(Hw, Hw.size()[1:])\n",
    "\n",
    "      Hs = self.w2sGAT(x=[Hw, Hs], edge_index=edge_index_s, edge_attr=edge_attr_s) + torch.cat([Hs] * 8, dim=1)\n",
    "      Hs = self.ffnw2s(Hs)\n",
    "      Hs = torch.layer_norm(Hs, Hs.size()[1:])\n",
    "\n",
    "    res = self.wh(Hs)\n",
    "\n",
    "    res = torch.sigmoid(res).squeeze(-1)\n",
    "\n",
    "    return res\n",
    "\n",
    "  def save(self, fname):\n",
    "    torch.save(self.state_dict(), fname)\n",
    "\n",
    "  def load(self, fname):\n",
    "    self.load_state_dict(torch.load(fname))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HeterSUMGraph(device=device, vocab_size=vocab_size, word_embed=embmgr.getEmbeddings(), dim_word_embed=300).to(device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.BCELoss()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(checkpoints_folder):\n",
    "  os.makedirs(checkpoints_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_val[\"doc_splitted\"] = split_all_docs(df_val[args.doc_column_name])\n",
    "val_set = df_val\n",
    "scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1:   0%|          | 0/25 [00:00<?, ?batch/s]/home/raoufdine/.local/lib/python3.9/site-packages/torch/nn/modules/conv.py:309: UserWarning: Using padding='same' with even kernel lengths and odd dilation may require a zero-padded copy of the input be created (Triggered internally at ../aten/src/ATen/native/Convolution.cpp:895.)\n",
      "  return F.conv1d(input, weight, bias, self.stride,\n",
      "Epoch 1: 100%|██████████| 25/25 [00:01<00:00, 12.92batch/s, accuracy=0.338, loss=0.537]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 : val loss = 0.424, val accuracy = 0.752, r1 = 0.224, r2 = 0.062, rL = 0.142\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|██████████| 25/25 [00:01<00:00, 18.02batch/s, accuracy=0.31, loss=0.47]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2 : val loss = 0.431, val accuracy = 0.754, r1 = 0.228, r2 = 0.063, rL = 0.142\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 100%|██████████| 25/25 [00:01<00:00, 19.30batch/s, accuracy=0.312, loss=0.534]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3 : val loss = 0.424, val accuracy = 0.766, r1 = 0.229, r2 = 0.062, rL = 0.142\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4: 100%|██████████| 25/25 [00:01<00:00, 19.23batch/s, accuracy=0.325, loss=0.492]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4 : val loss = 0.425, val accuracy = 0.764, r1 = 0.225, r2 = 0.062, rL = 0.142\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5: 100%|██████████| 25/25 [00:01<00:00, 18.75batch/s, accuracy=0.325, loss=0.484]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5 : val loss = 0.425, val accuracy = 0.770, r1 = 0.226, r2 = 0.061, rL = 0.141\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6: 100%|██████████| 25/25 [00:01<00:00, 20.32batch/s, accuracy=0.331, loss=0.481]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6 : val loss = 0.449, val accuracy = 0.782, r1 = 0.229, r2 = 0.062, rL = 0.140\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7: 100%|██████████| 25/25 [00:01<00:00, 18.67batch/s, accuracy=0.331, loss=0.501]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7 : val loss = 0.423, val accuracy = 0.770, r1 = 0.228, r2 = 0.062, rL = 0.141\n",
      "Training duration = 24.351954698562622\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "arr_train_loss = []\n",
    "arr_train_acc = []\n",
    "arr_val_loss = []\n",
    "arr_val_acc = []\n",
    "\n",
    "t1 = time()\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "  # Train\n",
    "  model.train()\n",
    "  nb_batch_train = 0\n",
    "  total_train_loss = 0\n",
    "  total_train_acc = 0\n",
    "  with tqdm(train_loader, unit=\"batch\", total=len(train_loader)) as tepoch:\n",
    "    for batch in tepoch:\n",
    "      tepoch.set_description(f\"Epoch {epoch}\")\n",
    "      #if dev != \"cpu\":\n",
    "      #  torch.cuda.empty_cache()\n",
    "      doc_lens = batch[\"doc_lens\"]\n",
    "      data = batch[\"docs\"].to(device)\n",
    "      targets = torch.tensor(batch[\"labels\"], dtype=torch.float).to(device)\n",
    "      y_pred = model(data)\n",
    "\n",
    "      loss = criterion(y_pred, targets)\n",
    "      optimizer.zero_grad()\n",
    "      loss.backward()\n",
    "      nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "      optimizer.step()\n",
    "      nb_batch_train += 1\n",
    "      total_train_loss += loss.item()\n",
    "\n",
    "      probs = y_pred.tolist() # compute_probs(y_pred)\n",
    "      probs_tensor = y_pred #torch.tensor(probs, dtype=torch.float).to(device)\n",
    "\n",
    "      total_train_acc += accuracy_nb_sent_per_doc_fn(probs=probs, targets=targets.cpu().detach().numpy(), doc_lens=doc_lens, average_number_of_sentences_per_document=average_number_of_sentences_per_document)\n",
    "\n",
    "      tepoch.set_postfix(loss=total_train_loss/nb_batch_train, accuracy=total_train_acc/nb_batch_train)\n",
    "\n",
    "  # Save model\n",
    "  model.save(checkpoints_folder + \"/\" + model_name + \"-\" + str(epoch) + \".pt\")\n",
    "\n",
    "  # Eval\n",
    "  model.eval()\n",
    "  nb_batch_val = 0\n",
    "  total_val_loss = 0\n",
    "  total_val_acc = 0\n",
    "  total_r1 = 0\n",
    "  total_r2 = 0\n",
    "  total_rl = 0\n",
    "\n",
    "  del loss\n",
    "  del y_pred\n",
    "\n",
    "  if dev != \"cpu\":\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "  for i, batch in enumerate(val_loader):\n",
    "    #if dev != \"cpu\":\n",
    "    #  torch.cuda.empty_cache()\n",
    "    doc_lens = batch[\"doc_lens\"]\n",
    "    data = batch[\"docs\"].to(device)\n",
    "    targets = torch.tensor(batch[\"labels\"], dtype=torch.float).to(device)\n",
    "    y_pred = model(data)\n",
    "\n",
    "    loss = criterion(y_pred, targets)\n",
    "    nb_batch_val += 1\n",
    "    total_val_loss += loss.item()\n",
    "\n",
    "    doc = val_set[\"doc_splitted\"].iloc[i]\n",
    "    summaries = val_set[\"summaries\"].iloc[i]\n",
    "\n",
    "    indices = torch.argsort(y_pred, descending=True)\n",
    "\n",
    "    y_pred_thresh = []\n",
    "\n",
    "    txt = \"\"\n",
    "\n",
    "    for j in range(min(len(doc), y_pred.shape[0])):\n",
    "      txt = txt + \". \" + doc[indices[j]]\n",
    "      y_pred_thresh.append(indices[j])\n",
    "      if len(txt) >= len(summaries):\n",
    "        break\n",
    "\n",
    "    y_pred_thresh.sort()\n",
    "\n",
    "    txt = \"\"\n",
    "\n",
    "    for j in y_pred_thresh:\n",
    "      txt = txt + \". \" + doc[j]\n",
    "\n",
    "    n = min(len(txt), len(summaries))\n",
    "\n",
    "    while n < len(txt) and txt[n].isalnum():\n",
    "      n += 1\n",
    "\n",
    "    txt = txt[:n]\n",
    "\n",
    "    scores = scorer.score(summaries, txt)\n",
    "    total_r1 += scores[\"rouge1\"].recall\n",
    "    total_r2 += scores[\"rouge2\"].recall\n",
    "    total_rl += scores[\"rougeL\"].recall\n",
    "\n",
    "    probs = y_pred.tolist() # compute_probs(y_pred)\n",
    "    probs_tensor = y_pred #torch.tensor(probs, dtype=torch.float).to(device)\n",
    "\n",
    "    total_val_acc += accuracy_nb_sent_per_doc_fn(probs=probs, targets=targets.cpu().detach().numpy(), doc_lens=doc_lens, average_number_of_sentences_per_document=average_number_of_sentences_per_document)\n",
    "\n",
    "  print(\"Epoch {} : val loss = {:.3f}, val accuracy = {:.3f}, r1 = {:.3f}, r2 = {:.3f}, rL = {:.3f}\".format(epoch, total_val_loss / nb_batch_val, total_val_acc / nb_batch_val, total_r1 / nb_batch_val, total_r2 / nb_batch_val, total_rl / nb_batch_val))\n",
    "\n",
    "  if len(arr_val_loss) >= early_stopping+1:\n",
    "    if min(arr_val_loss[-early_stopping:]) >= arr_val_loss[-(early_stopping+1)]:\n",
    "      break\n",
    "\n",
    "  del loss\n",
    "  del y_pred\n",
    "\n",
    "  if dev != \"cpu\":\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "  arr_train_loss.append(total_train_loss / nb_batch_train)\n",
    "  arr_train_acc.append(total_train_acc / nb_batch_train)\n",
    "\n",
    "  arr_val_loss.append(total_val_loss / nb_batch_val)\n",
    "  arr_val_acc.append(total_val_acc / nb_batch_val)\n",
    "\n",
    "t2 = time()\n",
    "print(\"Training duration =\", t2-t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_metrics = {}\n",
    "training_metrics[\"duration\"]   = t2 - t1\n",
    "training_metrics[\"train_loss\"] = arr_train_loss\n",
    "training_metrics[\"train_acc\"]  = arr_train_acc\n",
    "training_metrics[\"val_loss\"]   = arr_val_loss\n",
    "training_metrics[\"val_acc\"]    = arr_val_acc\n",
    "\n",
    "# Save to file in JSON format\n",
    "\n",
    "with open(checkpoints_folder + \"/training_metrics.json\", 'w') as fp:\n",
    "  json.dump(training_metrics, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAA9hAAAPYQGoP6dpAABm/UlEQVR4nO3dd3hUZd7G8e+UTAppJIGEQEhCkSK9hS4oig3RVUFFQFRcK+siFnzXhu6ia1ksCIoi61rAtaKsCCIgIL1IkQ4hoSQBQjppM/P+MUkgQCBAkpOZuT/XNZfMmXPO/E6EzD3PeYrJ6XQ6EREREfEiZqMLEBEREalpCkAiIiLidRSARERExOsoAImIiIjXUQASERERr6MAJCIiIl5HAUhERES8jgKQiIiIeB0FIBEREfE6CkAiIiLidRSARMStzJgxA5PJxJo1a4wuRUTcmAKQiIiIeB0FIBEREfE6CkAi4nHWr1/PNddcQ3BwMIGBgVxxxRWsWLGi3D5FRUW88MILNG/eHD8/P8LDw+nduzfz588v2yclJYVRo0bRqFEjfH19adCgAYMHDyYxMbGGr0hEqprV6AJERKrSli1b6NOnD8HBwTzxxBP4+Pjw3nvv0a9fPxYvXkxCQgIAzz//PBMnTuTee++lW7duZGVlsWbNGtatW8eVV14JwM0338yWLVt45JFHiIuLIy0tjfnz55OUlERcXJyBVykiF8vkdDqdRhchIlJZM2bMYNSoUaxevZouXbqc9vpNN93E//73P7Zu3UqTJk0AOHToEC1atKBjx44sXrwYgA4dOtCoUSN++OGHM75PRkYGdevW5dVXX2XcuHHVd0EiYgjdAhMRj2G325k3bx433nhjWfgBaNCgAXfccQdLly4lKysLgNDQULZs2cLOnTvPeC5/f39sNhuLFi3i2LFjNVK/iNQcBSAR8RiHDx8mLy+PFi1anPZaq1atcDgcJCcnAzBhwgQyMjK45JJLaNu2LY8//jgbN24s29/X15dXXnmFH3/8kcjISPr27cs///lPUlJSaux6RKT6KACJiFfq27cvu3fvZvr06bRp04YPPviATp068cEHH5Tt8+ijj7Jjxw4mTpyIn58fzzzzDK1atWL9+vUGVi4iVUEBSEQ8Rr169QgICGD79u2nvbZt2zbMZjMxMTFl28LCwhg1ahSff/45ycnJtGvXjueff77ccU2bNuWxxx5j3rx5bN68mcLCQl5//fXqvhQRqWYKQCLiMSwWC1dddRXfffdduaHqqampfPbZZ/Tu3Zvg4GAAjh49Wu7YwMBAmjVrRkFBAQB5eXnk5+eX26dp06YEBQWV7SMi7kvD4EXELU2fPp25c+eetv35559n/vz59O7dmwcffBCr1cp7771HQUEB//znP8v2a926Nf369aNz586EhYWxZs0avvzySx5++GEAduzYwRVXXMGQIUNo3bo1VquVb775htTUVG677bYau04RqR4aBi8ibqV0GHxFkpOTOXz4MOPHj2fZsmU4HA4SEhL4+9//To8ePcr2+/vf/87s2bPZsWMHBQUFxMbGMnz4cB5//HF8fHw4evQozz33HAsWLCA5ORmr1UrLli157LHHuPXWW2viUkWkGikAiYiIiNdRHyARERHxOgpAIiIi4nUUgERERMTrKACJiIiI11EAEhEREa+jACQiIiJeRxMhnoHD4eDgwYMEBQVhMpmMLkdEREQqwel0kp2dTXR0NGbz2dt4FIDO4ODBg+XWCxIRERH3kZycTKNGjc66jwLQGQQFBQGuH2DpukEiIiJSu2VlZRETE1P2OX42CkBnUHrbKzg4WAFIRETEzVSm+4o6QYuIiIjXUQASERERr1MrAtDkyZOJi4vDz8+PhIQEVq1aVeG+M2bMwGQylXv4+flVuP/999+PyWRi0qRJ1VC5iIiIuCPD+wDNmjWLsWPHMnXqVBISEpg0aRIDBw5k+/bt1K9f/4zHBAcHs3379rLnFd3r++abb1ixYgXR0dHVUrvdbqeoqKhazu3pbDbbOYcoioiIVBfDA9Abb7zB6NGjGTVqFABTp05lzpw5TJ8+naeeeuqMx5hMJqKios563gMHDvDII4/w008/cd1111VpzU6nk5SUFDIyMqr0vN7EbDYTHx+PzWYzuhQREfFChgagwsJC1q5dy/jx48u2mc1mBgwYwPLlyys8Licnh9jYWBwOB506deIf//gHl156adnrDoeD4cOH8/jjj5fbXpGCggIKCgrKnmdlZZ11/9LwU79+fQICAjRZ4nkqnWjy0KFDNG7cWD8/ERGpcYYGoCNHjmC324mMjCy3PTIykm3btp3xmBYtWjB9+nTatWtHZmYmr732Gj179mTLli1lkx698sorWK1WxowZU6k6Jk6cyAsvvFCpfe12e1n4CQ8Pr9Qxcrp69epx8OBBiouL8fHxMbocERHxMm7XCaNHjx6MGDGCDh06cNlll/H1119Tr1493nvvPQDWrl3Lm2++WdZZujLGjx9PZmZm2SM5ObnCfUv7/AQEBFz8xXix0ltfdrvd4EpERMQbGRqAIiIisFgspKamltuempp6zj4+pXx8fOjYsSO7du0CYMmSJaSlpdG4cWOsVitWq5V9+/bx2GOPERcXd8Zz+Pr6lk16WNnJD3Xb5uLo5yciIkYyNADZbDY6d+7MggULyrY5HA4WLFhAjx49KnUOu93Opk2baNCgAQDDhw9n48aNbNiwoewRHR3N448/zk8//VQt1yEiIiLuxfBRYGPHjmXkyJF06dKFbt26MWnSJHJzc8tGhY0YMYKGDRsyceJEACZMmED37t1p1qwZGRkZvPrqq+zbt497770XgPDw8NP65vj4+BAVFUWLFi1q9uI8WFxcHI8++iiPPvqo0aWIiIicN8MD0NChQzl8+DDPPvssKSkpdOjQgblz55Z1jE5KSio3X8yxY8cYPXo0KSkp1K1bl86dO/Pbb7/RunVroy7BbfTr148OHTpUyaSQq1evpk6dOhdflIiIiAFMTqfTaXQRtU1WVhYhISFkZmae1h8oPz+fvXv3Eh8ff9YZqCtyvLAYq8WMj6Xm7z6eKwA5nU7sdjtWa/Xn4ov9OYqIiJzqbJ/fp3K7UWDu7GDGcXam5XA0p+DcO1exu+66i8WLF/Pmm2+WLSFSOlLuxx9/pHPnzvj6+rJ06VJ2797N4MGDiYyMJDAwkK5du/Lzzz+XO19cXFy5IGUymfjggw+46aabCAgIoHnz5syePbuGr1JERKRyFICqgNPpJK+w+JwPkwnyi+wcyDhOdn5RpY452+N8Gu/efPNNevTowejRozl06BCHDh0iJiYGgKeeeoqXX36ZrVu30q5dO3Jycrj22mtZsGAB69ev5+qrr2bQoEEkJSWd9T1eeOEFhgwZwsaNG7n22msZNmwY6enpF/WzFRERqQ6G9wHyBMeL7LR+tuZHmP0xYSABtsr9LwwJCcFmsxEQEFA2xUDpZJMTJkzgyiuvLNs3LCyM9u3blz1/8cUX+eabb5g9ezYPP/xwhe9x1113cfvttwPwj3/8g7feeotVq1Zx9dVXn/e1iYiIVCe1AAldunQp9zwnJ4dx48bRqlUrQkNDCQwMZOvWredsAWrXrl3Zn+vUqUNwcDBpaWnVUrOIiMjFUAtQFfD3sfDHhIGV2tfhcLIjNZtih5OYugGEBFz4MhD+PpYLPvZkp47mGjduHPPnz+e1116jWbNm+Pv7c8stt1BYWHjW85y6pIXJZMLhcFRJjSIiIlVJAagKmEymSt+KAogODSAtO5/cQjtRIX41NiuyzWar1NITy5Yt46677uKmm24CXC1CiYmJ1VydiIhIzdEtMAOEB9owmUwlnZlrbi2suLg4Vq5cSWJiIkeOHKmwdaZ58+Z8/fXXbNiwgd9//5077rhDLTkiIuJRFIAM4GMxU7fk1tfh7JobEj9u3DgsFgutW7emXr16FfbpeeONN6hbty49e/Zk0KBBDBw4kE6dOtVYnSIiItVNEyGeQXVOhFh2niI7O1KzAWgRGYRvFfXncReaCFFERKqaJkJ0A34+FoL9XK1ARwyYGFFERMSbKQAZKCLQF4BjeUUU29XHRtxLQbGdkdNX8cjn6/X3V0TcjkaBGaiOrwV/HwvHi+wczS0kMli3gsR9fLX2AIt3HAagab06PDrgEoMrEhGpPLUAGchkMlEvyNUKdDSnEIdD3bHEPRTbHUxdvLvs+VsLdrJqr5Y9ERH3oQBksGB/H2wWM8UOB8eOn32iQZHaYs6mQySl51E3wIfr2zXA4YRHZ64nI09/h0XEPSgAGcxsMhFe0hfoSHbheS1wKmIEh8PJuwtdrT9394rn5ZvbERcewMHMfJ76apP+DouIW1AAqgXC6vhgMZkoKLaTnV9sdDkiZ/XLtjS2p2YT6GtlRI84An2tvH17J3wsJuZuSeHzVclGlygick4KQLWAxWwmLNAGwGENiZdazOl08s7CXQDc2T22bC27to1CeGJgSwBe+H5L2RxXIiK1lQJQLRFexxcTJnILiskrVCuQ1E7L9xxlQ3IGNquZu3vHlXvtnt7x9L2kHgXFDsZ8vp78oppb5kVE5HwpANUSNquZ0JJv00eya2dH0ri4OCZNmmR0GWKg0r4/Q7vEUD+o/LQNZrOJ129tT0SgjW0p2fzjf1uNKFFEpFIUgGqRiJLbYJnHiygs1rdnqV1+T85g6a4jWMwm7uvb5Iz71Avy5bVb2wPw8fJ9zNuSUpMliohUmgJQLeJvsxLoa8WJkyM5tbMVSLzXu4tcfX8Gd4gmJiygwv36tajP6D7xADzx1UYOZR6vkfpERM6HAlAtUzoxYnpuIcWOqlte4P333yc6OhrHKeccPHgwd999N7t372bw4MFERkYSGBhI165d+fnnn6vs/cW97UzN5qctqZhM8GC/pufc//GBLWnbMISMvCL+OmsDdk3yKSK1jAJQVXA6oTC3Sh6BpgL8KcBZmMuxjGNn3/885lu59dZbOXr0KAsXLizblp6ezty5cxk2bBg5OTlce+21LFiwgPXr13P11VczaNAgkpKSquMnJm5myiJX35+rWkfSrH7QOfe3Wc28dXtHAmwWVuxJZ0pJ65GISG2htcCqQlEe/CO6Sk5lAppXduenD4KtTqV2rVu3Ltdccw2fffYZV1xxBQBffvklERER9O/fH7PZTPv27cv2f/HFF/nmm2+YPXs2Dz/88PldhHiU5PQ8vvv9IAAP9mtW6ePiI+owYXAbxv33d/718056NA2nc2xYdZUpInJe1ALkRYYNG8ZXX31FQYFrrqFPP/2U2267DbPZTE5ODuPGjaNVq1aEhoYSGBjI1q1b1QIkvP/rHuwOJ32aR9A+JvS8jr25U0MGd4jG7nAy5vMNZB4vqp4iRUTOk1qAqoJPgKs1pgqlZeeTmlWAn4+FZvXqYDKZzvy+52HQoEE4nU7mzJlD165dWbJkCf/6178AGDduHPPnz+e1116jWbNm+Pv7c8stt1BYqM7Y3iwtO59Za1wzO59P608pk8nESze2YV3SMZLTj/P0N5t45/aOZ/77LCJSgxSAqoLJVOlbUZUVFupPWn42x51Ocp2+BPr6XPQ5/fz8+NOf/sSnn37Krl27aNGiBZ06dQJg2bJl3HXXXdx0000A5OTkkJiYeNHvKe7tw6V7KSx20KlxKN2bXNjtqyA/H966rSO3Tl3OnI2HuKx5PYZ0janiSkVEzo9ugdVSVouZunVKl8eoulaYYcOGMWfOHKZPn86wYcPKtjdv3pyvv/6aDRs28Pvvv3PHHXecNmJMvEtmXhGfLN8HuFp/LqbVpmPjujx2VQsAnpu9hV1pOVVSo4jIhVIAqsUiAm2YgOz8oipbVuDyyy8nLCyM7du3c8cdd5Rtf+ONN6hbty49e/Zk0KBBDBw4sKx1SLzTv5cnkltop2VUEJe3rH/R5/tz3yb0bhbB8SI7j2ipDBExmG6B1WK+VgvB/j5kHi/icHbBWSefqyyz2czBg6f3V4qLi+OXX34pt+2hhx4q91y3xLxHXmExHy3bC8AD/ZpiNl98nx2z2cQbQ9pz9ZtL2Hooi1fmbuO5QZde9HlFRC6EWoBquYhA18SIGceLKLLrlpTUjM9XJXMsr4jY8ACua9ugys5bP9iP125tB8BHyxL5ZVtqlZ1bROR8KADVcnV8rQTYrDidTo7mFBhdjniBgmI7037dA8D9lzXFaqnaXxOXt4xkVK84AMb9dyNpWflVen4RkcpQAHIDpctjHM0t1JICUu2+WXeAlKx8IoN9+VOnhtXyHk9d05LWDYJJzy3kr19swKG/1yJSwxSA3ECwnxVfqxm7w8mxPM3LI9XH7nAydbFr2YvRfZrga7VUy/v4Wi28dXtH/H0sLNt1lPdKWpxERGqKAtAFcp7HOlwXy2QylfUFOpJTUKPvXV084Ro80f82HSLxaB6hAT7c3q1xtb5Xs/qBPH9DawBen7ed9UnHqvX9REROpgB0nnx8XBMS5uXl1ej71g2wYTWbKCx2kOUBywmUzjBtsVRPC4OcP6fTyeSFrkVLR/WMp45v9Q8SHdIlhuvaNaDY4WTMzPVk5bv/320RcQ8aBn+eLBYLoaGhpKWlARAQEFBj0/oH+UB6biEpx+zYTDX3vlXN4XBw+PBhAgICsFr1V7C2WLg9jW0p2dSxWRjZM7ZG3tNkMvGPm9qyISmD5PTjPPPtZiYN7eC2f7dFxH3o0+cCREVFAZSFoJpidzg5nJWP0wnH033xtbpvA57ZbKZx48b6oKslXK0/rr4/w7rHEhpgq7H3DvH34a3bOzDkvRV8t+EgfZvX4+bOjWrs/UXEOykAXQCTyUSDBg2oX78+RUU122T/7U/b+XHzIXo1i2DC4DY1+t5VyWazYTa7b4DzNKv2prN23zFsFjP39o6v8ffvHBvGXwc057V5O3jmu810bBxKk3qBNV6HiHgPBaCLYLFYarwPy9DuTfhg+X7+uyGVB65oRXxE1S7CKt5p8iJX68+tXRpRP9jPkBoe6NeMpbuOsGJPOmNmrufrB3phc+NWThGp3fTbxc00j3Sty+R0wvSle40uRzzApv2Z/LrjMBaziT/3bWpYHRaziUlDOxIa4MPmA1m8+tM2w2oREc+nAOSG7u3jukXx37XJpOdqXiC5OO8uco38uqF9NI3DL369uYsRFeLHP292LZUxbcleFm2v2X52IuI9FIDcUI8m4bRpGEx+kYNPVuwzuhxxY7vScpi7JQVwLXpaG1x1aRQjerhGoY377+8cztYSMCJS9RSA3JDJZGJ0nyYAfLw8kfwiu8EVibuaung3Tidc2TqSSyKDjC6nzNPXtqJlVBBHcgp57L+/a6kMEalyCkBu6tq2DYgO8eNITiHfrj9gdDnihvYfyyv7u/NgLWn9KeXnY+Ht2zvi52Pm1x2H+VD93USkiikAuSkfi5lRvVx9gT5YulffkOW8Tft1D8UOJ72ahdOxcV2jyzlN88ggnrnetVTGP3/axsb9GcYWJCIeRQHIjd3WLYYgXyu70nJYtEOdRaXyDmcXMHN1MgAP9WtmcDUVu6NbY66+NIoiu5Mxn68np6DY6JJExEMoALmxID8fbk9wLVj5vlbTlvMwfdleCoodtI8JpUfTcKPLqZDJZOLlm9sSHeJH4tE8nvtui9EliYiHqBUBaPLkycTFxeHn50dCQgKrVq2qcN8ZM2ZgMpnKPfz8TkzcVlRUxJNPPknbtm2pU6cO0dHRjBgxgoMHD9bEpdS4u3rGYTWbWLEnnU37M40uR9xA5vEiPlnuGj34UL+mtX45ktAAG5Nu64jZBF+t268+byJSJQwPQLNmzWLs2LE899xzrFu3jvbt2zNw4MCzrrMVHBzMoUOHyh779p0YCp6Xl8e6det45plnWLduHV9//TXbt2/nhhtuqInLqXHRof5c364BANOWqBVIzu2TFfvILijmkshABrSKNLqcSukWH8aYK5oD8LdvN7PvaK7BFYmIuzM8AL3xxhuMHj2aUaNG0bp1a6ZOnUpAQADTp0+v8BiTyURUVFTZIzLyxC/xkJAQ5s+fz5AhQ2jRogXdu3fnnXfeYe3atSQlJdXEJdW4e0uGxM/ZdIgDGccNrkZqs+OF9rIRVQ/2a4bZXLtbf072cP9mdIsLI6egmDGfr6ew2GF0SSLixgwNQIWFhaxdu5YBAwaUbTObzQwYMIDly5dXeFxOTg6xsbHExMQwePBgtmw5e7+AzMxMTCYToaGhZ3y9oKCArKyscg930qZhCD2bhmN3OPlIw4XlLGauTiI9t5CYsBMth+7CajHzr9s6EOxn5ff9mbwxf4fRJYmIGzM0AB05cgS73V6uBQcgMjKSlJSUMx7TokULpk+fznfffccnn3yCw+GgZ8+e7N+//4z75+fn8+STT3L77bcTHBx8xn0mTpxISEhI2SMmJubiLswAo/u6WoFmrk4mK79mV6gX91BY7CjrLP/nvk2xWgxvAD5vDUP9eaVkqYz3ft3N0p1HDK5IRNyV2/0G7NGjByNGjKBDhw5cdtllfP3119SrV4/33nvvtH2LiooYMmQITqeTKVOmVHjO8ePHk5mZWfZITk6uzkuoFv0uqUfz+oHkFBQzc5Vn3uqTi/PthgMcysynXpAvt3RuZHQ5F+yatg24I6ExTif89YsNHM3RUhkicv4MDUARERFYLBZSU1PLbU9NTSUqKqpS5/Dx8aFjx47s2rWr3PbS8LNv3z7mz59fYesPgK+vL8HBweUe7ubk5TE+WpZIkV39I+QEu8PJ1EW7ARjdJx4/H4vBFV2cZ65rTfP6gRzOLmDcf3/H6dREoCJyfgwNQDabjc6dO7NgwYKybQ6HgwULFtCjR49KncNut7Np0yYaNDjRn6E0/OzcuZOff/6Z8PDaO89JVRrcMZqIQF8OZeYzZ+Mho8uRWmTu5hT2HMklxN+HOxJijS7novnbLLx9R0dsVjMLtx/mo2WJRpckIm7G8FtgY8eOZdq0afz73/9m69atPPDAA+Tm5jJq1CgARowYwfjx48v2nzBhAvPmzWPPnj2sW7eOO++8k3379nHvvfcCrvBzyy23sGbNGj799FPsdjspKSmkpKRQWFhoyDXWFF+rhbt6uj7c3v91j74VCwBOp5PJC10tpHf1jCPQ12pwRVWjZVQwf7uuFQAv/7iNzQc0D5aIVJ7hAWjo0KG89tprPPvss3To0IENGzYwd+7cso7RSUlJHDp0ojXj2LFjjB49mlatWnHttdeSlZXFb7/9RuvWrjWDDhw4wOzZs9m/fz8dOnSgQYMGZY/ffvvNkGusScMSYvHzMfPHoSyW7z5qdDlSCyzacZg/DmURYLNwV884o8upUsO7xzKgVSSFdgdjZq4nr1BLZYhI5ZicaiY4TVZWFiEhIWRmZrplf6Bnv9vMx8v30a9FPWaM6mZ0OWKwIVOXsyoxnXt7x/O3ksVFPcmx3EKueXMJKVn5DO0Swyu3tDO6JBExyPl8fhveAiRV757e8ZhMsGj7YXakZhtdjhho1d50ViWmY7OYyybM9DR169j419AOmEwwa00y3//umcveiEjVUgDyQLHhdRjY2jWK7gMtj+HV3l3k6vtzc+dGRIX4nWNv99WjaTgP93etav/015tITs8zuCIRqe0UgDxU6cSI364/SFp2vsHViBE2H8hk0fbDmE1w/2We2fpzsr9c0ZxOjUPJLihmzMz1mgpCRM5KAchDdY6tS6fGoRTaHXz8275zHyAeZ0rJvD/Xt4smNryOwdVUP6vFzJu3dSTIz8r6pAze/Hmn0SWJSC2mAOTB7itpBfrPin0aHeNl9hzO4X+bXaMnH+jX1OBqak5MWAD/uKktAJMX7dJISBGpkAKQB7uydRSx4QFkHi/iy7VnXitNPNPUxbtxOmFAq/q0auB+IxkvxqD20QztEoPTCY/OWk96rmfP/yUiF0YByINZzCbu6R0PwAdL9mJ3aMYDb3Ag4zhfrzsAwIMlHYO9zXM3tKZJvTqkZhXwxJcbNSmoiJxGAcjD3dK5EaEBPiSl5zFvS4rR5UgNmPbrHoodTno0CadT47pGl2OIAJuVt2/viM1i5uetqfxnhfrBiUh5CkAeLsBm5c6StZ+maUi8xzuaU8DM1UkAPNjfe/r+nMml0SE8dU1LAF6as5Wth7IMrkhEahMFIC8womcsNouZdUkZrN2XbnQ5Uo0+WpZIfpGDdo1C6N0swuhyDDeqVxyXt6xPYbGDMZ+v53ih3eiSRKSWUADyAvWD/LixYzQA037da3A1Ul2y8ov49/JEAB7s1wyTyWRsQbWAyWTi1VvaUS/Il51pObw45w+jSxKRWkIByEuULoPw0x8pJB7JNbgaqQ6frNhHdn4xzeoHclXrSKPLqTXCA32ZVLJUxmcrk/hx06FzHyQiHk8ByEtcEhlEvxb1cDph+jK1Anma44V2Plzi+v/6YL+mmM1q/TlZr2YR3H+Zq0/Uk19t5EDGcYMrEhGjKQB5kftKWoG+WJPMMc2N4lG+WJPM0dxCGob6M6h9tNHl1Epjr7yE9jGhZOUX8+jM9RRrqQwRr6YA5EV6NA3n0uhg8oscfKJhwR6jyO7g/V9dI/zuv6wJPhb9sz4TH4uZt2/rSKCvldWJx3j7l11GlyQiBtJvSi9iMpkYXdIK9O/l+8gv0ogYT/DdhoMcyDhORKAvt3aJMbqcWq1xeAB/v6kNAG//spNVezUqUsRbKQB5mevaNaBBiB9Hcgr4bsMBo8uRi2R3OHl3kasl494+8fj5WAyuqPYb3KEhN3dqhMMJj85cT0aebgeLeCMFIC/jYzEzqlccANOW7MWh5THc2rwtKew5nEuwn5VhCY2NLsdtvDD4UuIj6nAwM5+nvtqkpTJEvJACkBe6rVtjAn2t7ErLYfGOw0aXIxfI6XTy7qLdAIzsGUeQn4/BFbmPQF8rb93WER+LiblbUvhsVZLRJYlIDVMA8kLBfj7c1tXVV0TLY7ivJTuPsOlAJv4+Fkb1ije6HLfTtlEITwx0LZUx4fs/2JGabXBFIlKTFIC81Kje8VjMJn7bfZTNBzKNLkcuwOSFrr4/t3drTFgdm8HVuKd7esfT95J6FJQslaGBASLeQwHISzUM9ef6dg0AtQK5ozWJ6azcm46PxcTovmr9uVBms4nXb21PRKCNbSnZ/ON/W40uSURqiAKQFysdEv/DxkMc1My4bqW078/NnRrRIMTf4GrcW70gX14f0gGAj5fvY96WFGMLEpEaoQDkxdo0DKFHk3DsDicfaXkMt/HHwSx+2ZaG2QR/LlneQS7OZZfU476+ri8ET3y1kUOZ+kIg4ukUgLxc6e2Tz1clk5VfZHA1UhlTFrtaf65t24D4iDoGV+M5xl3VgrYNQ8jIK+LRmRuwa4oIEY+mAOTl+l1Sn2b1A8kpKGbWqmSjy5Fz2HsklzkbDwLwYL9mBlfjWWxWM2/d3pEAm4WVe9OZskhLZYh4MgUgL2c2mxjdx9UKNH3ZXoq0QGSt9t7i3TiccHnL+rSODja6HI8TH1GHFwe7lsr41887WbtPS2WIeCoFIGFwh4ZEBNo4lJnP/zYdMrocqcChzON8tW4/AA/1V9+f6vKnTg25sUM0doeTMZ9vIPO4bg2LeCIFIMHPx8LIHnEAvP/rHi0LUEt9sGQvRXYn3eLD6BwbZnQ5HstkMvHijW1oHBbAgYzjPP21lsoQ8UQKQALAnd1j8fMxs+VgFsv3HDW6HDlFem4hn610LdfwUH/1/aluQX4+vHV7R6xmE3M2HeKLNeofJ+JpFIAEgLp1bNzauWR5jF81MWJtM2PZXo4X2WnTMJi+zSOMLscrdIgJ5bGrWgDw/Ow/2JWWY3BFIlKVFICkzD294zGZYOH2w+zUuki1RnZ+ETN+SwTgoX7NMJlMxhbkRf7ctwm9m0VwvMjOI1oqQ8SjKABJmbiIOlzVOhJw9TeR2uHTlUlk5RfTtF4dBl4aZXQ5XsVsNvHGkPaE1bGx9VAWr8zdZnRJIlJFFICknNLZcL9Zf4C07HyDq5H8IntZGL3/sqaYzWr9qWn1g/14/db2AHy0LJEFW1MNrkhEqoICkJTTOTaMjo1DKbQ7+M/yfUaX4/X+u3Y/R3IKaBjqz40dGxpdjtfq37I+d/dyzZf1+JcbSc3SlwMRd6cAJKe5r2SR1P+s2EdeYbHB1XivIruD90qWvbivbxN8LPrnaqQnr2lB6wbBpOcWMvaLDTi0VIaIW9NvVDnNVZdG0TgsgIy8Ir5au9/ocrzW978fZP+x40QE2hjaNcbocryer9XC23d0xN/HwrJdR3lPoyVF3JoCkJzGYjZxT29Xc/8HS/dqUUgDOBxO3l3kav25u3c8fj4WgysSgKb1AnnhhksBeH3edtYnHTO4IhG5UApAcka3dmlEiL8P+47mMf8PdfqsafO3prIrLYcgXyt3do81uhw5ya1dGnF9uwYUO5yMmbmerHwtlSHijhSA5IwCbFbu7N4YgGlL1NRfk5xOJ+8udK1EPqJnLMF+PgZXJCczmUz8/aa2NKrrT3L6cf72zWYtlSHihhSApEIje8Rhs5hZu+8Ya/epqb+mLNt1lN/3Z+LnY2ZUycgjqV1C/H1487aOWMwmZv9+kK/WHTC6JBE5TwpAUqH6wX4M7hANwAdqBaoxk0taf27r2piIQF+Dq5GKdI6ty18HNAfg2e82s+ewlsoQcScKQHJWo0smRpy7JYV9R3MNrsbzrUs6xvI9R7GaTWWTUkrt9UC/ZnRvEkZeoZ0xM9dTWOwwuiQRqSQFIDmrSyKDuOySejidMH2plseobu8udI38uqljQ6JD/Q2uRs7FYjYxaWhHQgN82Hwgi1d/0lIZIu5CAUjOqbQl4os1+zmWW2hwNZ5rW0oWP29NxWSC+/s1NbocqaSoED9evcW1VMa0JXtZtD3N4IpEpDIUgOScejYNp1WDYI4X2fl0pZbHqC5TSub9ubZNA5rWCzS4GjkfV7aOZGQP13QF4/77O4ezCwyuSETORQFIzslkMnFfX9dopBm/7aOg2G5wRZ5n39Fcvv/9IAAPqPXHLY2/thUto4I4klPIY//9XUtliNRyCkBSKde3iyYq2I8jOQV8t/6g0eV4nKmL9+BwQr8W9WjTMMTocuQC+PlYePv2jvj5mPl1x2E+VJ85kVqtVgSgyZMnExcXh5+fHwkJCaxatarCfWfMmIHJZCr38PPzK7eP0+nk2WefpUGDBvj7+zNgwAB27txZ3Zfh0XwsZkb1igNcEyNq4reqk5qVX7bm2oP9mhlcjVyM5pFBPHu9a6mMf/60jY37M4wtSEQqZHgAmjVrFmPHjuW5555j3bp1tG/fnoEDB5KWVnFHwuDgYA4dOlT22LevfL+Uf/7zn7z11ltMnTqVlStXUqdOHQYOHEh+fn51X45Huz2hMYG+Vnam5bBox2Gjy/EYHyzZQ6HdQde4unSLDzO6HLlIt3eL4Zo2URTZnYz5fD05BcVGlyQiZ2B4AHrjjTcYPXo0o0aNonXr1kydOpWAgACmT59e4TEmk4moqKiyR2RkZNlrTqeTSZMm8be//Y3BgwfTrl07Pv74Yw4ePMi3335bA1fkuYL9fMpWJdfEiFXjWG4hn65MAuDB/mr98QQmk4mX/9SO6BA/Eo/m8ex3m40uSUTOwNAAVFhYyNq1axkwYEDZNrPZzIABA1i+fHmFx+Xk5BAbG0tMTAyDBw9my5YtZa/t3buXlJSUcucMCQkhISGhwnMWFBSQlZVV7iFnNqpXHBaziWW7jrL5QKbR5bi9Gb8lkldop3WDYPpdUs/ocqSKhAT48ObtHTGb4Ot1B/h2vZbKEKltDA1AR44cwW63l2vBAYiMjCQlJeWMx7Ro0YLp06fz3Xff8cknn+BwOOjZsyf797v6UJQedz7nnDhxIiEhIWWPmJiYi700j9WobgDXtW0AqBXoYuUUFDPjt0QAHurfDJPJZGxBUqW6xoUx5grXUhl/+3azZlIXqWUMvwV2vnr06MGIESPo0KEDl112GV9//TX16tXjvffeu+Bzjh8/nszMzLJHcnJyFVbseUb3cU2M+MPGQxzMOG5wNe7r85VJZB4voklEHa5uE2V0OVINHu7fjG5xYeQUFDPmcy2VIVKbGBqAIiIisFgspKamltuemppKVFTlPhB8fHzo2LEju3a5FpAsPe58zunr60twcHC5h1SsbaMQujcJo9jhLGvBkPOTX2RnWkkL2v2XNcViVuuPJ7JazPzrtg6E+Pvw+/5M3pi/w+iSRKSEoQHIZrPRuXNnFixYULbN4XCwYMECevToUalz2O12Nm3aRIMGrtsy8fHxREVFlTtnVlYWK1eurPQ55dxKW4E+X5lEdn6RwdW4n6/W7Sctu4AGIX7c2LGh0eVINWoY6s8rN7cFYOri3SzZqRGUIrWB4bfAxo4dy7Rp0/j3v//N1q1beeCBB8jNzWXUqFEAjBgxgvHjx5ftP2HCBObNm8eePXtYt24dd955J/v27ePee+8FXCMwHn30UV566SVmz57Npk2bGDFiBNHR0dx4441GXKJH6t+iPk3r1SG7oJhZq3XL8HwU2x1MXexa9uK+vk2wWQ3/ZyjV7Oo2DRiW0BiAsV/8ztEcLZUhYjSr0QUMHTqUw4cP8+yzz5KSkkKHDh2YO3duWSfmpKQkzOYTHxDHjh1j9OjRpKSkULduXTp37sxvv/1G69aty/Z54oknyM3N5b777iMjI4PevXszd+7c0yZMlAtnNpu4t08Txn+9iY+WJTKyZxw+Fn2QV8acTYdITj9OWB0bt3VtbHQ5UkOeub41qxPT2ZGaw7j//s70u7qq47uIgUxOTel7mqysLEJCQsjMzFR/oLPIL7LT+5VfOJJTyJu3dWBwB93KOReHw8k1by5he2o24666hIcvb250SVKDtqVkccM7yygsdvDs9a25u3e80SWJeJTz+fzWV3a5YH4+Fkb0iAO0PEZlLdiWxvbUbAJ9rQwv+dmJ92gZFcwz17UC4OUft2kuLREDKQDJRbmzeyx+PmY2H8hixZ50o8up1ZxOJ+8sdI1WHN4jlhB/H4MrEiPc2T2WK1tHUmh3MObz9eRqqQwRQygAyUUJq2Pjls6NAMqGdcuZLd99lN+TM/C1mrm7l259eCuTycQ/b25HVLAfe47k8sL3W859kIhUOQUguWj39G6CyQS/bEtjV1q20eXUWu8uco38uq1rDPWCfA2uRoxUt46NSbd1wGSCL9bs5/vfDxpdkojXUQCSixYfUYcrW7lG7X2wZK/B1dROG5IzWLrrCFazidF9mxhdjtQC3ZuE83DJArhPf72J5PQ8gysS8S4KQFIlSj/Uv153gMPZmuPkVO+W9P0Z3KEhjeoGGFyN1BZ/uaI5nRqHkl1QzJiZ6ymya6kMkZqiACRVoktsXTrEhFJod/Cf5YlGl1Or7EjNZt4fqZhM8EA/tf7ICVaLmTdv60iQn5X1SRm8+fNOo0sS8RoKQFIlTCYT95W0Av1nxT6OF9oNrqj2mFLS9+fqS6NoVj/I4GqktokJC+DlP7UDYPKiXfy2+4jBFYl4BwUgqTIDL40iJsyfY3lFfLluv9Hl1ArJ6XnMLung+mC/ZgZXI7XVde0acFvXGJxO+OusDaTnFhpdkojHUwCSKmMxm7inZHj3h0v2YHdoYsT3ft2N3eGkT/MI2jYKMbocqcWeHdSapvXqkJpVwBNfbtTEoiLVTAFIqtStXWII9rOSeDSPn7emGl2OodKy8vlijasl7KH+av2RswuwWXn79k7YLGZ+3prKf1bsM7okEY+mACRVqo6vlTu7xwIw7Vfvnhjxw6V7KSx20Dm2LgnxYUaXI26gdXQw469tCcBLc7ay9VCWwRWJeC4FIKlyrpXhTazZd4x1SceMLscQmXlFfFLyDf6h/k216rdU2l0947i8ZX0Kix088vl6DSgQqSYKQFLlIoP9ylaG/8BLl8f49/JEcgvttIwKon+L+kaXI27EZDLx6i3tqB/ky660HCb88IfRJYl4JAUgqRaj+7iGxM/dnELSUe+a4Ta3oJjpy1wzYj/Yv5laf+S8hQf68q+hrqUyPl+VxI+bDhldkojHUQCSatEiKoi+l9TD4aQsDHiLz1clkZFXRFx4ANe1bWB0OeKmejWL4P7LmgLw5FcbOZBx3OCKRDyLApBUm/tKWoFmrU4mI8875jUpKLYzreS23/2XNcViVuuPXLixV15C+5hQsvKL+fN/1vDV2v3sP+ZdLaoi1cVqdAHiuXo1C6dlVBDbUrL5dGWSVwwF/3rdAVKzCogK9uOmTg2NLkfcnI/FzNu3deS6t5aw+UAWj/33dwAahvqT0CSMhPgwEuLDiQ0P0K1WkfNkcmq2rdNkZWUREhJCZmYmwcHBRpfj1r5et5+xX/xOvSBflj7ZH1+rxeiSqk2x3cEVbyxm39E8nrm+Nff0jje6JPEQu9Ky+XLtAVbuPcqm/ZkUnzLJaGSwLwnx4XSLD6N7kzCa1gtUIBKvdD6f3wpAZ6AAVHUKix30+ecvpGYV8M9b2jGkS4zRJVWb2b8fZMzn66kb4MOypy4nwKYGVql6uQXFrEs6xso96azce5TfkzMpPGUV+YhAG93iw+gWF0ZCk3BaRAZh1u1Y8QLn8/mt39BSrWxWM6N6xfPyj9v4YMkebu3cyCO/mTqdTt5duAuAUb3iFX6k2tTxtdKneT36NK8HQH6RnfVJGazce5SVe9JZl3SMIzmF/G9TCv/blAJAaIAPXeNct8y6NwmnVYNg9U8Tr6ff0lLtbu/WmLcX7GRHag6LdxymnwfOi/PLtjS2pWRTx2ZhZI84o8sRL+LnY6FH03B6NA0HXB3xN+3PZOXedFbsOcrafcfIyCti/h+pzP/DtTxNkK+VLnF1SWgSTkJ8GG0ahuBj0ZgY8S4KQFLtQvx9GNq1MdOX7eWDJXs9LgA5nU4ml7T+3NkjlpAAH4MrEm/ma7XQJS6MLnFhPNS/GUV2B1sOZrFyz1FW7k1n9d50sguKWbj9MAu3HwYgwGYpW7IloUk47RqFeHR/PRFQH6AzUh+gqpecnke/1xZhdziZM6Y3l0Z7zsroK/Yc5bb3V2Czmln6ZH/qB/kZXZJIhewOJ1sPZbFybzor9xxlVWI6GXlF5fbxtZrp2DiUhPhwEpqE0alxXfx8FIik9lMfIKl1YsICuKZNFD9sPMQHS/byr6EdjC6pypS2/gzp0kjhR2o9i9lEm4YhtGkYwj2943E4nOxIyy7rVL1qbzpHcgpZsSedFXvSYQHYLGbax4TQrWTYfefYutTx1ceHuDe1AJ2BWoCqx8b9GdzwzjKsZhNLnuxPgxB/o0u6aKXXZDGbWDSuHzFhAUaXJHJRnE4nuw/nlnWqXrn3KKlZBeX2sZaEqNK5iLrEhRHsp1u/Yjy1AEmt1K5RKAnxYazcm86MZYmMv7aV0SVdtHcX7gZgcPtohR/xCCaTiWb1A2lWP5BhCbE4nU6S0vNYuSedFSWh6EDGcTYkZ7AhOYP3Fu/BbILW0cFlcxF1iwujbh2b0ZciclZqAToDtQBVn5//SOXej9cQ5Gvlt/GXE+TG3xp3pWUz4I1fAZj/1740jwwyuCKRmrH/WB6r9qaXtRAlnmHB45ZRQWWdqrvFhxER6GtApeJt1AIktdblLevTpF4d9hzOZdbqZO4tWS/MHU1Z5Frza+ClkQo/4lUa1Q2gUd0A/tSpEQCpWfllnapX7k1nV1oO21Ky2ZaSzb+X7wOgab06ZcPuuzcJJzJY/eXEWGoBOgO1AFWvz1Ym8fQ3m2gY6s/ix/thdcP5R04e1fbdQ71oHxNqdEkitcaRnAJW7U1nVclcRNtSsk/bJzY8oGwts4QmYTSqq1vIcvHUAiS12p86NeT1eds5kHGc/21O4Yb20UaXdN6mLdmD3eGkd7MIhR+RU0QE+nJt2wZc27YBABl5ha5bZiWhaMvBTPYdzWPf0Ty+WLMfKFngNT6spGO1FniV6qcWoDNQC1D1m/TzDib9vJO2DUOY/XAvt/pFl5adT+9XFlJY7OCz0Qn0bBphdEkibiUrv4i1icdYUTLsXgu8SlVRC5DUesO7xzJl0W42HXBN2d+9SbjRJVXa9KWJFBY76Ng4lB5uVLdIbRHs50P/lvXp39I1K/yZFnhNzSpg9u8Hmf37QUALvErVUwASQ4QH+nJL50Z8ujKJab/ucZsAlHm8iE9WuDp1PtSvmb6RilSBi13gNSE+nNbRWuBVzo8CkBjmnt7xfLYqiQXb0tiVlkOz+oFGl3RO/1meSE5BMS0ig7i8pWetaSZSW2iBV6kJ6gN0BuoDVHNGf7yG+X+kcnu3GCb+qZ3R5ZxVXmExvV9ZSHpuIW/e1oHBHRoaXZKIVyqyO9h8ILOsY3XpAq8n0wKv3ul8Pr8VgM5AAajmrNqbzpD3lmOzmvntqctr9WRp05fuZcIPf9A4LIBfHrvMLYfvi3ii0gVeV+xxdarWAq/eS52gxW10jatL+5hQfk/O4OPl+xh75SVGl3RGhcUOpi1xTXx4/2VNFX5EapGTF3i9t0+TSi3w6mMx0b5RaNmwey3w6n3UAnQGagGqWT9sPMjDn62nboAPvz11Bf622vet7IvVyTzx1UbqB/my5Mn+akoXcSPns8Br59i6NAz1JyrEj8hgPxqE+FEvyFf9idyEWoDErVx9aRSN6vqz/9hxvlq3nzu7xxpdUjl2h5Mpi12Lno7u00ThR8TNnO8Cr6cf75rcMSrYj6gQv7L/lgakyJLngWpBciv6vyWGs1rM3NM7nhe+/4MPl+7ljm6Na9X8Hj9uPsTeI7mE+PtwR0Jjo8sRkYtkMpmIDa9DbHgdhnSNAU4s8Lr5QBapWfmkZOWTkplPWnY+RXYnh7MLOJxdwKYDmRWeN9DXWhaQysJRaWAqCUnhdWy16vebN7ugAPTvf/+biIgIrrvuOgCeeOIJ3n//fVq3bs3nn39ObGzt+gYvtd+QLjH8a/4O9h7J5eetqVx1aZTRJQGupvPJC12tP6N6xamPgIiHOrHAa/ntDoeTo7mFrlCUeSIYpWTln9iWmU92QTE5BcXsSsthV1pOhe/jYzFRP8iPyGBfGoT4l7Qe+ZYEJn+igv2oH+yrDto14IL6ALVo0YIpU6Zw+eWXs3z5cgYMGMC//vUvfvjhB6xWK19//XV11Fpj1AfIGK/M3caURbvpGleX/97f0+hyAFi4PY1RH60mwGbht6cuJzTAZnRJIlIL5RYUu0JRZj6HTg1IJf89nFNAZT9x6wb4EBXiT1Sw7xlvt0UF+xHi76PJWE9R7X2AkpOTadasGQDffvstN998M/fddx+9evWiX79+F3JKEe7qGccHS/awOvEY65OO0bFxXaNL4t2FuwC4s3uswo+IVKiOr5Wm9QJpWq/iCV2L7Q4O5xRwKNMVlFJOutWWkplfdustv8jBsbwijuUVsfVQxe/p52Muu90WFXJS/6Rg1623BiF+1Av01ajVClxQAAoMDOTo0aM0btyYefPmMXbsWAD8/Pw4fvx4lRYo3iMy2I8b2jfkq3X7+WDJXiYPMzYArdqbzurEY9hK+iiJiFwMq8VMgxB/GoT4V7iP0+kk83hRuWB0cmvSoZKgdCyviPwiB4lH80g8mlfh+cylHbhP6bwddUofJW+8vX9BV3zllVdy77330rFjR3bs2MG1114LwJYtW4iLi6vK+sTL3Nsnnq/W7efHzYdITs8jJizAsFoml7T+3NKlEZHBfobVISLew2QyERpgIzTARsuoim/h5BfZScsq4FDm8ZMCUgEpWcdLWpMKSM3Kp9jhJC27gLTsAjZScQfuoNIO3CcFpFNDk6d14L6gADR58mT+9re/kZyczFdffUV4uGu9lrVr13L77bdXaYHiXVo1CKZP8wiW7DzCh0v38vwNlxpSx+YDmSzecRizCe7v29SQGkREKuLnY6FxeACNwyv+kuhwODmSW0BqZsFJt9uOk5JZUG6kW05BMdkFxWSn5bCzEh24Tw5GpbfbSluU6gf7us1UIZoI8QzUCdpYS3YeZviHqwiwWVj+1BWEBPjUeA0PfbqOOZsOcWOHaCbd1rHG319EpKbkFBSX9UEqvcV26oi3I+fRgTusjq18p+2SkW5RJaPcooL9CPa3VksH7mrvBD137lwCAwPp3bs34GoRmjZtGq1bt2by5MnUrVv5vhuTJ0/m1VdfJSUlhfbt2/P222/TrVu3cx43c+ZMbr/9dgYPHsy3335btj0nJ4ennnqKb7/9lqNHjxIfH8+YMWO4//77z/s6xRi9m0XQMiqIbSnZfLpqHw/2a1aj77/7cA7/2+zqefhADb+3iEhNC/S1lk0UWZEiu4PD2QXlAlJpYDq5j1JBsYP03ELScwvZeiirwvP5+ZgZ3acJj13VojouqVIuKAA9/vjjvPLKKwBs2rSJxx57jLFjx7Jw4ULGjh3LRx99VKnzzJo1i7FjxzJ16lQSEhKYNGkSAwcOZPv27dSvX7/C4xITExk3bhx9+vQ57bWxY8fyyy+/8MknnxAXF8e8efN48MEHiY6O5oYbbriQy5UaZjKZXP8w/vs7M5Ylck/v+BptUp26aDdOJwxoFUmLqKAae18RkdrKx2ImOtSf6NCzd+DOyCsqu92WelKn7ZNbkzJKOnDbDB6ddkG3wAIDA9m8eTNxcXE8//zzbN68mS+//JJ169Zx7bXXkpKSUqnzJCQk0LVrV9555x0AHA4HMTExPPLIIzz11FNnPMZut9O3b1/uvvtulixZQkZGRrkWoDZt2jB06FCeeeaZsm2dO3fmmmuu4aWXXqpUXboFZrzCYgd9/vkLqVkFvHpLO27tElMj73sg4ziX/XMhxQ4nXz/Yk061YCi+iIgnyS+yk5qVT4DNSr0g3yo99/l8fl9Q/LLZbOTluYbd/fzzz1x11VUAhIWFkZVVcZPXyQoLC1m7di0DBgw4UYzZzIABA1i+fHmFx02YMIH69etzzz33nPH1nj17Mnv2bA4cOIDT6WThwoXs2LGjrMYzKSgoICsrq9xDjGWzmrmrp2vo+QdL9lJTXdWm/bqHYoeTnk3DFX5ERKqBn4+F2PA6VR5+ztcFBaDevXszduxYXnzxRVatWlW2JMaOHTto1KhRpc5x5MgR7HY7kZGR5bZHRkZW2IK0dOlSPvzwQ6ZNm1bhed9++21at25No0aNsNlsXH311UyePJm+fftWeMzEiRMJCQkpe8TE1Exrg5zdHd0aE2CzsD01m193Hqn29zuSU8DM1UkAPNRffX9ERDzZBQWgd955B6vVypdffsmUKVNo2LAhAD/++CNXX311lRZYKjs7m+HDhzNt2jQiIiIq3O/tt99mxYoVzJ49m7Vr1/L666/z0EMP8fPPP1d4zPjx48nMzCx7JCcnV8clyHkKCfBhaMlChR8s2VPt7/fRsr3kFzloHxNKz6bh1f5+IiJinAvqBN24cWN++OGH07b/61//qvQ5IiIisFgspKamltuemppKVNTpC2Hu3r2bxMREBg0aVLbN4XAAYLVa2b59O9HR0Tz99NN88803Za1S7dq1Y8OGDbz22mvlbredzNfXF19fY5vi5Mzu7hXPv39LZMnOI/xxMIvW0dXTJysrv4iPf9sHwIP9mmp9HRERD3fBXbDtdjtfffUVL730Ei+99BLffPMNdru90sfbbDY6d+7MggULyrY5HA4WLFhAjx49Ttu/ZcuWbNq0iQ0bNpQ9brjhBvr378+GDRuIiYmhqKiIoqIizObyl2WxWMrCkriXmLAArmnbAIAPllZfK9B/lu8ju6CY5vUDubJV5LkPEBERt3ZBLUC7du3i2muv5cCBA7Ro4RrDP3HiRGJiYpgzZw5Nm1Zu5tyxY8cycuRIunTpQrdu3Zg0aRK5ubmMGjUKgBEjRtCwYUMmTpyIn58fbdq0KXd8aGgoQNl2m83GZZddxuOPP46/vz+xsbEsXryYjz/+mDfeeONCLlVqgfv6NGHOxkPM3nCQxwe2OOs6OhfieKGd6Uv3AvBg/6YeNdW7iIic2QUFoDFjxtC0aVNWrFhBWFgYAEePHuXOO+9kzJgxzJkzp1LnGTp0KIcPH+bZZ58lJSWFDh06MHfu3LKO0UlJSae15pzLzJkzGT9+PMOGDSM9PZ3Y2Fj+/ve/ayJEN9Y+JpRucWGsSkxnxm+JjL+mVZWe/4s1yRzNLaRRXX8GtYuu0nOLiEjtdEHzANWpU4cVK1bQtm3bctt///13evXqRU5OxWuJuAPNA1T7zP8jldEfryHIz8ry8VcQWEUrFxcWO+j36kIOZubz0o1tuLN7bJWcV0REal61zwPk6+tLdnb2adtzcnKw2WwXckqRs7qiZX2aRNQhO7+YWaurbpTedxsOcDAzn3pBvtzSuXJTOIiIiPu7oAB0/fXXc99997Fy5UqcTidOp5MVK1Zw//33a7kJqRZms4l7+rgmRpy+dC/F9ovv1G53OJmyeDcA9/aOx8/HPVYwFhGRi3dBAeitt96iadOm9OjRAz8/P/z8/OjZsyfNmjVj0qRJVVyiiMvNnRoRVsfGgYzj/Li5csutnM1PW1LYcziXYD8rw3TrS0TEq1xQR4rQ0FC+++47du3axdatWwFo1aoVzZpp9lypPn4+FoZ3j+XNBTuZtmQP17drcMHz9TidTt5dtAuAu3rFV1mfIhERcQ+V/q0/duzYs76+cOHCsj9ryLlUl+E9Ypm6eDcb92eyam86CU0ubMbmX3ceYfOBLAJsFkb1jKvaIkVEpNardABav359pfbTDLpSnSICfflTp0Z8viqJaUv2XHAAmrzQ1fpze7fG1K2jjvsiIt6m0gHo5BYeESPd2yeez1cl8fPWNHYfzqFpvcDzOn51Yjqr9qbjYzExuk+TaqpSRERqswteCkPEKE3rBTKgZLmKD5bsPe/j3y1p/bmlcyOiQvyqtDYREXEPCkDilkaXDIn/et1+juQUVPq4LQczWbj9MGYT/Llv5ZZsERERz6MAJG6pW3wY7RuFUFDs4D/L91X6uCmLXPP+XN8umriIOtVVnoiI1HIKQOKWTCYT95b03/nPin3kF9nPecyewznM2XQIgAf6qfVHRMSbKQCJ27qmTRQNQ/1Jzy3kq3X7z7n/e4v34HS6ltVo1UBrvImIeDMFIHFbVouZu3u7+gJ9uGQvDkfF6/oeyjzO1+tdIenB/pqwU0TE2ykAiVsb2jWGID8re47ksmBbWoX7Tft1L0V2J92bhNE5tm4NVigiIrWRApC4tUBfK8MSXOt4Tft1zxn3OZpTwOerkgB4SK0/IiKCApB4gLt6xmE1m1iVmM6G5IzTXp/xWyLHi+y0bRhC72YRNV+giIjUOgpA4vaiQvy4oUM0ANOWlG8Fys4vYsZviQA81L+plmoRERFAAUg8xL29XUPif9x0iOT0vLLtn6xIIju/mKb16nBV6yijyhMRkVpGAUg8QuvoYPo0j8DhhOnLXMtj5BfZ+XCp688P9muG2azWHxERcVEAEo9ROjHirNXJZOYV8d81yRzJKaBhqH/ZLTIRERFQABIP0rd5BC0ig8grtPPx8kSmLnb1B/rzZU3wseivuoiInKBPBfEYJpOJ0X1drUCTFuzkQMZxIgJtDOkSY3BlIiJS2ygAiUe5oX009YN8sZfMCn1P7yb4+VgMrkpERGobBSDxKDarmbt6xQEQ5Gflzu6NjS1IRERqJavRBYhUtbt6xpGSmU+f5vUI8vMxuhwREamFFIDE4wTYrEwY3MboMkREpBbTLTARERHxOgpAIiIi4nUUgERERMTrKACJiIiI11EAEhEREa+jACQiIiJeRwFIREREvI4CkIiIiHgdBSARERHxOgpAIiIi4nUUgERERMTrKACJiIiI11EAEhEREa+jACQiIiJeRwFIREREvI4CkIiIiHgdBSARERHxOgpAIiIi4nUUgERERMTrKACJiIiI11EAEhEREa+jACQiIiJex/AANHnyZOLi4vDz8yMhIYFVq1ZV6riZM2diMpm48cYbT3tt69at3HDDDYSEhFCnTh26du1KUlJSFVcuIiIi7srQADRr1izGjh3Lc889x7p162jfvj0DBw4kLS3trMclJiYybtw4+vTpc9pru3fvpnfv3rRs2ZJFixaxceNGnnnmGfz8/KrrMkRERMTNmJxOp9OoN09ISKBr16688847ADgcDmJiYnjkkUd46qmnzniM3W6nb9++3H333SxZsoSMjAy+/fbbstdvu+02fHx8+M9//nPBdWVlZRESEkJmZibBwcEXfB4RERGpOefz+W1YC1BhYSFr165lwIABJ4oxmxkwYADLly+v8LgJEyZQv3597rnnntNeczgczJkzh0suuYSBAwdSv359EhISygWkMykoKCArK6vcQ0RERDyXYQHoyJEj2O12IiMjy22PjIwkJSXljMcsXbqUDz/8kGnTpp3x9bS0NHJycnj55Ze5+uqrmTdvHjfddBN/+tOfWLx4cYW1TJw4kZCQkLJHTEzMhV+YiIiI1HqGd4KurOzsbIYPH860adOIiIg44z4OhwOAwYMH89e//pUOHTrw1FNPcf311zN16tQKzz1+/HgyMzPLHsnJydVyDSIiIlI7WI1644iICCwWC6mpqeW2p6amEhUVddr+u3fvJjExkUGDBpVtKw08VquV7du3ExMTg9VqpXXr1uWObdWqFUuXLq2wFl9fX3x9fS/mckRERMSNGNYCZLPZ6Ny5MwsWLCjb5nA4WLBgAT169Dht/5YtW7Jp0yY2bNhQ9rjhhhvo378/GzZsICYmBpvNRteuXdm+fXu5Y3fs2EFsbGy1X5OIiIi4B8NagADGjh3LyJEj6dKlC926dWPSpEnk5uYyatQoAEaMGEHDhg2ZOHEifn5+tGnTptzxoaGhAOW2P/744wwdOpS+ffvSv39/5s6dy/fff8+iRYtq6rJERESkljM0AA0dOpTDhw/z7LPPkpKSQocOHZg7d25Zx+ikpCTM5vNrpLrpppuYOnUqEydOZMyYMbRo0YKvvvqK3r17V8cliIiIiBsydB6g2krzAImIiLgft5gHSERERMQoCkAiIiLidRSARERExOsoAImIiIjXUQASERERr6MAJCIiIl5HAUhERES8jgKQiIiIeB0FIBEREfE6CkAiIiLidRSARERExOsoAImIiIjXUQASERERr6MAJCIiIl5HAUhERES8jgKQiIiIeB0FIBEREfE6CkAiIiLidRSARERExOsoAImIiIjXUQASERERr6MAJCIiIl5HAUhERES8jgKQiIiIeB0FIBEREfE6CkAiIiJSc4ryYfGrUHTc0DKshr67iIiIeI/M/TBrOBxcB8cS4cbJhpWiACQiIiLVb+8S+O9dkHcE/OtC25sNLUcBSERERKqP0wkr3oV5z4DTDlFtYegnUDfO0LIUgERERKR6FObC7DGw+UvX83a3wfX/AluAsXWhACQiIiLVIX0PzLwT0raA2QoD/wHd7gOTyejKAAUgERERqWo758NX90B+JtSpD0P+DbE9ja6qHAUgERERqRoOByx5HRb+HXBCo64w5GMIjja6stMoAImIiMjFy8+Ebx6A7XNcz7vcDVe/DFZfY+uqgAKQiIiIXJzD22HmHXB0F1hscN0b0Gm40VWdlQKQiIiIXLg/ZsO3D0BhDgQ3hKH/gYadja7qnBSARERE5Pw57PDLS7D0DdfzuD5wy0cQWM/YuipJAUhERETOT166a5TX7l9cz3s8DANeAIv7xAr3qVRERESMd2gjzBoGGUlg9YfB70DbW4yu6rwpAImIiEjlbPzCNbNz8XHXUhZDP4WoNkZXdUEUgEREROTs7EWutbxWTnE9b3Yl3DzNtaipm1IAEhERkYrlpLlWcd+3zPW87+PQbzyYLYaWdbEUgEREROTMklfDF8Mh+xDYguCmqdDqeqOrqhIKQCIiInK6NR/Bj0+AvRAiWsBtn0JEc6OrqjIKQCIiInJCUT78+Dis+9j1vNUguHEK+AYZW1cVUwASERERl8z98MUIOLAWMMEVz0Lvv4LJZHRlVU4BSERERCBxKXwxEvKOuEZ33fwhNLvC6KqqjQKQiIiIN3M6YcUUmPc3cNohqi0M/cQ1z48HMxtdAMDkyZOJi4vDz8+PhIQEVq1aVanjZs6ciclk4sYbb6xwn/vvvx+TycSkSZOqplgRERFPUZgHX4+Gn8a7wk+7oXD3PI8PP1ALAtCsWbMYO3Yszz33HOvWraN9+/YMHDiQtLS0sx6XmJjIuHHj6NOnT4X7fPPNN6xYsYLo6OiqLltERMS9pe+FD6+ETf8FsxWu+Sfc9B7YAoyurEYYHoDeeOMNRo8ezahRo2jdujVTp04lICCA6dOnV3iM3W5n2LBhvPDCCzRp0uSM+xw4cIBHHnmETz/9FB8fn+oqX0RExP3s/Bne7wepm6FOPRgxGxL+7JGdnStiaAAqLCxk7dq1DBgwoGyb2WxmwIABLF++vMLjJkyYQP369bnnnnvO+LrD4WD48OE8/vjjXHrppVVet4iIiFtyOODXV+HTWyA/Axp2gT//CnG9jK6sxhnaCfrIkSPY7XYiIyPLbY+MjGTbtm1nPGbp0qV8+OGHbNiwocLzvvLKK1itVsaMGVOpOgoKCigoKCh7npWVVanjRERE3EZ+Fnz7AGz7wfW8812u215WX0PLMopbjQLLzs5m+PDhTJs2jYiIiDPus3btWt58803WrVuHqZJNeRMnTuSFF16oylJFRERqj8M7YNYwOLIDLDa49jXoPNLoqgxlcjqdTqPevLCwkICAAL788styI7lGjhxJRkYG3333Xbn9N2zYQMeOHbFYTizA5nA4ANets+3bt/P9998zduxYzOYTd/fsdjtms5mYmBgSExNPq+NMLUAxMTFkZmYSHBxcRVcrIiJigK3fwzcPQGE2BDeEIf+BRp2NrqpaZGVlERISUqnPb0NbgGw2G507d2bBggVlAcjhcLBgwQIefvjh0/Zv2bIlmzZtKrftb3/7G9nZ2bz55pvExMQwfPjwcn2KAAYOHMjw4cMZNWrUGevw9fXF19c7mwBFRMRDOeyw8O+w5HXX89jecOsMCKxnaFm1heG3wMaOHcvIkSPp0qUL3bp1Y9KkSeTm5paFlREjRtCwYUMmTpyIn58fbdq0KXd8aGgoQNn28PBwwsPDy+3j4+NDVFQULVq0qP4LEhERMVpeOnx1L+xe4Hre/SG48gWwaFR0KcMD0NChQzl8+DDPPvssKSkpdOjQgblz55Z1jE5KSip3O0tERETOImUTzBwGGfvA6g83vA3tbjW6qlrH0D5AtdX53EMUERGpNTZ+AbPHQPFxCI2F2z51LW3hJdymD5CIiIhUAXsRzHsGVk5xPW96Bdz8AQSEGVtXLaYAJCIi4s5y0uC/o2DfUtfzPuOg/9Ngtpz9OC+nACQiIuKu9q+BWcMh+yDYguCmqdDqeqOrcgsKQCIiIu5o7Qz43+NgL4SIS2Dop1DvEqOrchsKQCIiIu6kuMAVfNb92/W85fVw4xTw06Cd86EAJCIi4i4yD8AXw+HAWsAEVzwDvcd61SruVUUBSERExB0kLoX/3gW5h8EvFG75EJoNONdRUgEFIBERkdrM6YSVU+Gn/wOnHSLbwtD/QFi80ZW5NQUgERGR2qowD77/C2z6wvW87RAY9CbYAoytywMoAImIiNRG6XtdQ9xTN4HJAgP/Dgn3q79PFVEAEhERqW12/Qxf3gP5GVCnnmsV97jeRlflURSAREREagunE5a+AQteBJzQsAsM+RhCGhpdmcdRABIREakN8rPg2wdg2w+u551GwrWvgtXX2Lo8lAKQiIiI0Q7vgFnD4MgOsNhcwafzXUZX5dEUgERERIy09Qf45n4ozIagaNcQ90ZdjK7K4ykAiYiIGMFhh4X/gCWvuZ7H9nJ1dg6sb2hZ3kIBSEREpKblpcPXo12jvQC6PwhXTgCLj7F1eREFIBERkZqUsglm3QnHEsHqDze8Be2GGF2V11EAEs/gdLo6DyYuca2Xs3+Na+6M+D4Q1wcadwffIKOrFBFvt+lL+O5hKD4OobEw9BNo0M7oqrySApC4J6cTju6Cvb+6Ak/iUshNK79PZjIcXAfL3nTNotqwkysMxfeBmASw1TGmdhHxPvYimP8crJjset70Crj5AwgIM7YuL6YAJO7B6YSju0+08CQuhZyU8vtYfCGmG8T3dbX4ZB2EvUsg8VfISIL9q12PpW+A2Qcadj7RQhTTDXz8jbk2EfFsOYddq7jvW+p63ucx6P9/YLYYWpa3MzmdTqfRRdQ2WVlZhISEkJmZSXBwsNHleCenE9L3lA882YfK71MaeOJ6u0JMw87g43fm8x3bd+Jce5dA1v5TzmWDRl1PtBA16qrJx0Tk4u1fC18Mh6wDYAuCm6ZAq0FGV+WxzufzWwHoDBSADOB0wrG9J8LO3iWQfbD8PieHlLjerj9XFHgq8157l7hC0d4lp7cmWf1c54/veyJcWW0Xfn0i4n3WfQxzHgN7IYQ3h9s+g3qXGF2VR1MAukgKQDXA6YSMfSUhpCT0nNoqY/YpCTy9T7TKVMdtqrLba7+eqOfU/kQ+Aa5+Q3G9XaEouqOGq4rImRUXwI9PwNoZructr4cbp4CfPk+qmwLQRVIAqibH9pWEnZKQkZlc/nWzj2v207jeJS083cAWUPN1lo4o2/vriVrzjpbfxxbo6mcUV9KHqEF7sKhLnYjXyzoIX4xw9TfEBJf/H/R+DMxmoyvzCgpAF0kBqIpkJJ/Uh2eJqyPyycxW162l0ltaMd1q58gshwMObyu5XfYr7FsGx4+V38c3GBr3ONGpOqqtOjiKeJt9v8EXI10tyH6hcPOH0HyA0VV5FQWgi6QAdIEy95/ov5O4xHWL62RmK0R3OtHC07h77Qw85+JwQNqWE9eZuAwKMsvv4xfimta+tFN1/Uv1DVDEUzmdsPI9mPd/4CiGyDau+X3C4o2uzOsoAF0kBaBKKhtmXtLKc2xv+ddNFldfmfjSFp7u4BtoTK3VyWGHlI0nwt++31yLGp7Mv64rEJV2qq7fCkwmY+oVkapTmAc/PAobZ7met7nFNbOzO3658wAKQBdJAagCWYdKbmeVTD6Yvqf86yazK/CUDkv31tmX7cVw6PcTP6d9y6Eot/w+AREQV9pC1BciLlEgEnE3xxJdS1qkbHJ94bvqJej+gP4tG0gB6CIpAJXITinfafnorvKvm8yuzr+lHYEbd9cohzOxF8HB9Sc6VSetdE2Df7LAyBO3BuP6QnhT/RIVqc12LYCv7nH1BwyIcK3iHt/H6Kq8ngLQRfLaAJSd6pqptHQo+NGd5V83mSGq3Ymh4I27u/q6yPkpLoQDa090qk5eBfaC8vsENTjRfyiuN9SNVyASqQ2cTtds8gteBJyugRxD/gMhDY2uTFAAumheE4By0k7MwZO4FI5sP2UHk2s0U3zfkk7LPcA/1IhKPVtRPhxYc6I/1f7VronTThbc6MQIs/g+ENrYmFpFvFlBNnz7AGz93vW80wi45tULm5BVqoUC0EXy2ACUe6T8La3D207ZwQRRbU7c0ort4eq8KzWr6LirVah0luoDa1wjS04W2th1q6w0FOnbp0j1OrITZg5zfVG02ODaV6HzXUZXJadQALpIHhOAco+6bmmVjk46vPX0fSLbnuh7EttTKxPXRoW5kLzyRAvRgXXgtJffJ6zJif5D8X0gKMqYWkU80bY58M39UJAFQdEw5GOI6Wp0VXIGCkAXyW0DUF66a5K+0j48aVtO36f+pSeWlojtpcDjjgqyIWnFiRaiQxvA6Si/T3jzE/2H4vpAYH1DShVxaw47LJoIv77qet64Jwz5t/491WIKQBfJbQJQXrprzpnS21qpm0/fp37rk1p4ekOd8JqvU6pXfqZrqH1iSQvRoY3AKf+s67U80X9Ifw9Ezu34MfhqNOya73qe8ABc9aLWAKzlFIAuUq0NQMePlf+gS9nMmT/oSr71x/aCwHqGlCoGOn7MFYxLb5mdMRhfeqL/kG591gx7sWuCzIIcVyteYcl/y/5cur2CfexFrg9fi4+rD4rZ6vpv6TZzyXZLyXazT8X7lzv2Ys/l43mznKdshlnDXPP8WP1h0JvQfqjRVUklKABdpFoTgI5nQNLykj48v7om2zo18ES0OGn+mN5qmpXT5aWfaCU8Y1+w0s7vJf2HYntqeoNSxYUVBJWsk7bnlISW0j+fun/J9lPnfvIkJkvVhKlT97/gY8/23qec69TpJTZ9CbMfgaI812CDoZ9Cg3bG/FzlvCkAXSTDAlB+pqtvx96SGYRTNp69b0dsbwiKrLn6xDPkHD5pvqclcGRH+ddL53uK7+MKRbE93GdGb6cTigtKgkfW2QPJyfucMdzknD4/U1Ww+LqWhPENAluQ67++gWAr2eYbdNKfS7cHuz6sHcWuKRLsRa6Ho6iC5yX7OYpOea30UXiWc53h2FP3P/WLmDsznxKO8o66tje93LWYqVpH3YoC0EWqsQCUn3WiM2viEtfyCacFnmYnbmnF9dboHql6J8/4vXcJpO8u/3rpmm6lnecb96jadY6cTtfQ/4qCysmB5Iy3iE7enn36lAFVwep/eiApF1pKttkCTwk3geUDjS0QrLaqr68mOZ2uzsHnDFuVDFOV2r+KgpyjqHLX2HssXP43MFuq92cpVU4B6CJVWwA6efRO4lI4uKGC4cwlfTPiekFwdNW9v0hlZB08cds1cYmrH8TJzFbX7LeloTyk0VluEZ0cVE5tkck5EWZO/XdQFWyBpwSSU1tYKgoqp4QbW6CrZUDcn9N57sDkH6rfu25MAegiVVsA+t8TsOq98tvqxp9YWiK2lya0k9onI/lEaN+7BDKTqumNTGcIJxXdFjr11tGpoaWOvr2LeKHz+fzW15qaFNcbdv5U/pZWSCOjqxI5u9AY6HCH6wGuFqHSuab2LXO17FSmL0tFt4VK/+wT4HmjiUSk1lIL0BlUWwuQ06kFLUVERKrJ+Xx+6+tWTVL4ERERqRUUgERERMTrKACJiIiI11EAEhEREa9TKwLQ5MmTiYuLw8/Pj4SEBFatWlWp42bOnInJZOLGG28s21ZUVMSTTz5J27ZtqVOnDtHR0YwYMYKDBw9WU/UiIiLibgwPQLNmzWLs2LE899xzrFu3jvbt2zNw4EDS0tLOelxiYiLjxo2jT58+5bbn5eWxbt06nnnmGdatW8fXX3/N9u3bueGGG6rzMkRERMSNGD4MPiEhga5du/LOO+8A4HA4iImJ4ZFHHuGpp5464zF2u52+ffty9913s2TJEjIyMvj2228rfI/Vq1fTrVs39u3bR+PGjc9ZU61ZDFVEREQqzW2GwRcWFrJ27VoGDBhQts1sNjNgwACWL19e4XETJkygfv363HPPPZV6n8zMTEwmE6GhoRdbsoiIiHgAQ2eCPnLkCHa7ncjI8iuaR0ZGsm3btjMes3TpUj788EM2bNhQqffIz8/nySef5Pbbb68wDRYUFFBQcGLV56ysrMpdgIiIiLglw/sAnY/s7GyGDx/OtGnTiIiIOOf+RUVFDBkyBKfTyZQpUyrcb+LEiYSEhJQ9YmJiqrJsERERqWUMbQGKiIjAYrGQmppabntqaipRUVGn7b97924SExMZNGhQ2TaHwwGA1Wpl+/btNG3aFDgRfvbt28cvv/xy1nuB48ePZ+zYsWXPs7KyFIJEREQ8mKEByGaz0blzZxYsWFA2lN3hcLBgwQIefvjh0/Zv2bIlmzZtKrftb3/7G9nZ2bz55ptloaU0/OzcuZOFCxcSHh5+1jp8fX3x9fWtmosSERGRWs/w1eDHjh3LyJEj6dKlC926dWPSpEnk5uYyatQoAEaMGEHDhg2ZOHEifn5+tGnTptzxpR2bS7cXFRVxyy23sG7dOn744QfsdjspKSkAhIWFYbPZau7iREREpFYyPAANHTqUw4cP8+yzz5KSkkKHDh2YO3duWcfopKQkzObKd1U6cOAAs2fPBqBDhw7lXlu4cCH9+vWrqtJFRETETRk+D1BtlJmZSWhoKMnJyZoHSERExE2U9uHNyMggJCTkrPsa3gJUG2VnZwOoI7SIiIgbys7OPmcAUgvQGTgcDg4ePEhQUBAmk6lKz12aTtW6VL30c64Z+jnXDP2ca4Z+zjWjOn/OTqeT7OxsoqOjz9l9Ri1AZ2A2m2nUqFG1vkdwcLD+gdUA/Zxrhn7ONUM/55qhn3PNqK6f87lafkq51USIIiIiIlVBAUhERES8jgJQDfP19eW5557TxIvVTD/nmqGfc83Qz7lm6OdcM2rLz1mdoEVERMTrqAVIREREvI4CkIiIiHgdBSARERHxOgpAIiIi4nUUgGrIr7/+yqBBg4iOjsZkMvHtt98aXZLHmThxIl27diUoKIj69etz4403sn37dqPL8khTpkyhXbt2ZROZ9ejRgx9//NHosjzayy+/jMlk4tFHHzW6FI/z/PPPYzKZyj1atmxpdFke6cCBA9x5552Eh4fj7+9P27ZtWbNmjSG1KADVkNzcXNq3b8/kyZONLsVjLV68mIceeogVK1Ywf/58ioqKuOqqq8jNzTW6NI/TqFEjXn75ZdauXcuaNWu4/PLLGTx4MFu2bDG6NI+0evVq3nvvPdq1a2d0KR7r0ksv5dChQ2WPpUuXGl2Sxzl27Bi9evXCx8eHH3/8kT/++IPXX3+dunXrGlKPlsKoIddccw3XXHON0WV4tLlz55Z7PmPGDOrXr8/atWvp27evQVV5pkGDBpV7/ve//50pU6awYsUKLr30UoOq8kw5OTkMGzaMadOm8dJLLxldjseyWq1ERUUZXYZHe+WVV4iJieGjjz4q2xYfH29YPWoBEo+VmZkJQFhYmMGVeDa73c7MmTPJzc2lR48eRpfjcR566CGuu+46BgwYYHQpHm3nzp1ER0fTpEkThg0bRlJSktEleZzZs2fTpUsXbr31VurXr0/Hjh2ZNm2aYfWoBUg8ksPh4NFHH6VXr160adPG6HI80qZNm+jRowf5+fkEBgbyzTff0Lp1a6PL8igzZ85k3bp1rF692uhSPFpCQgIzZsygRYsWHDp0iBdeeIE+ffqwefNmgoKCjC7PY+zZs4cpU6YwduxYnn76aVavXs2YMWOw2WyMHDmyxutRABKP9NBDD7F582bdx69GLVq0YMOGDWRmZvLll18ycuRIFi9erBBURZKTk/nLX/7C/Pnz8fPzM7ocj3Zy94R27dqRkJBAbGwsX3zxBffcc4+BlXkWh8NBly5d+Mc//gFAx44d2bx5M1OnTjUkAOkWmHichx9+mB9++IGFCxfSqFEjo8vxWDabjWbNmtG5c2cmTpxI+/btefPNN40uy2OsXbuWtLQ0OnXqhNVqxWq1snjxYt566y2sVit2u93oEj1WaGgol1xyCbt27TK6FI/SoEGD074gtWrVyrDbjWoBEo/hdDp55JFH+Oabb1i0aJGhneu8kcPhoKCgwOgyPMYVV1zBpk2bym0bNWoULVu25Mknn8RisRhUmefLyclh9+7dDB8+3OhSPEqvXr1Om5pkx44dxMbGGlKPAlANycnJKfdtYu/evWzYsIGwsDAaN25sYGWe46GHHuKzzz7ju+++IygoiJSUFABCQkLw9/c3uDrPMn78eK655hoaN25MdnY2n332GYsWLeKnn34yujSPERQUdFr/tTp16hAeHq5+bVVs3LhxDBo0iNjYWA4ePMhzzz2HxWLh9ttvN7o0j/LXv/6Vnj178o9//IMhQ4awatUq3n//fd5//31jCnJKjVi4cKETOO0xcuRIo0vzGGf6+QLOjz76yOjSPM7dd9/tjI2NddpsNme9evWcV1xxhXPevHlGl+XxLrvsMudf/vIXo8vwOEOHDnU2aNDAabPZnA0bNnQOHTrUuWvXLqPL8kjff/+9s02bNk5fX19ny5Ytne+//75htZicTqfTmOglIiIiYgx1ghYRERGvowAkIiIiXkcBSERERLyOApCIiIh4HQUgERER8ToKQCIiIuJ1FIBERETE6ygAiYhUwqJFizCZTGRkZBhdiohUAQUgERER8ToKQCIiIuJ1FIBExC04HA4mTpxIfHw8/v7+tG/fni+//BI4cXtqzpw5tGvXDj8/P7p3787mzZvLneOrr77i0ksvxdfXl7i4OF5//fVyrxcUFPDkk08SExODr68vzZo148MPPyy3z9q1a+nSpQsBAQH07NnztNWtRcQ9KACJiFuYOHEiH3/8MVOnTmXLli389a9/5c4772Tx4sVl+zz++OO8/vrrrF69mnr16jFo0CCKiooAV3AZMmQIt912G5s2beL555/nmWeeYcaMGWXHjxgxgs8//5y33nqLrVu38t577xEYGFiujv/7v//j9ddfZ82aNVitVu6+++4auX4RqVpaDFVEar2CggLCwsL4+eef6dGjR9n2e++9l7y8PO677z769+/PzJkzGTp0KADp6ek0atSIGTNmMGTIEIYNG8bhw4eZN29e2fFPPPEEc+bMYcuWLezYsYMWLVowf/58BgwYcFoNixYton///vz8889cccUVAPzvf//juuuu4/jx4/j5+VXzT0FEqpJagESk1tu1axd5eXlceeWVBAYGlj0+/vhjdu/eXbbfyeEoLCyMFi1asHXrVgC2bt1Kr169yp23V69e7Ny5E7vdzoYNG7BYLFx22WVnraVdu3Zlf27QoAEAaWlpF32NIlKzrEYXICJyLjk5OQDMmTOHhg0blnvN19e3XAi6UP7+/pXaz8fHp+zPJpMJcPVPEhH3ohYgEan1Wrduja+vL0lJSTRr1qzcIyYmpmy/FStWlP352LFj7Nixg1atWgHQqlUrli1bVu68y5Yt45JLLsFisdC2bVscDke5PkUi4rnUAiQitV5QUBDjxo3jr3/9Kw6Hg969e5OZmcmyZcsIDg4mNjYWgAkTJhAeHk5kZCT/93//R0REBDfeeCMAjz32GF27duXFF19k6NChLF++nHfeeYd3330XgLi4OEaOHMndd9/NW2+9Rfv27dm3bx9paWkMGTLEqEsXkWqiACQibuHFF1+kXr16TJw4kT179hAaGkqnTp14+umny25Bvfzyy/zlL39h586ddOjQge+//x6bzQZAp06d+OKLL3j22Wd58cUXadCgARMmTOCuu+4qe48pU6bw9NNP8+CDD3L06FEaN27M008/bcTlikg10ygwEXF7pSO0jh07RmhoqNHliIgbUB8gERER8ToKQCIiIuJ1dAtMREREvI5agERERMTrKACJiIiI11EAEhEREa+jACQiIiJeRwFIREREvI4CkIiIiHgdBSARERHxOgpAIiIi4nUUgERERMTr/D+0S3s6s6kgAgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# draw Losses\n",
    "if is_graphic:\n",
    "  plt.plot(list(range(1, len(arr_train_loss) + 1)), arr_train_loss, label=\"train\")\n",
    "  plt.plot(list(range(1, len(arr_val_loss) + 1)), arr_val_loss, label=\"val\")\n",
    "\n",
    "  plt.title(\"Loss\")\n",
    "  plt.xlabel(\"epoch\")\n",
    "  plt.ylabel(\"loss\")\n",
    "  plt.legend(loc=\"upper left\")\n",
    "\n",
    "  plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA98klEQVR4nO3de3gU5d3/8c/uJtmcDxCScAhBQEFEzhABz8bioalHRIuGQ9VfFSyS0kcRQYVKbKsUFRX1Ea0tCorYB4tiMYpWioAgKoggKCQKSYhAjpDD7vz+SLJkSQLJsskkw/t1XXMxe+89s99dLp0P99wzYzMMwxAAAIBF2M0uAAAAwJ8INwAAwFIINwAAwFIINwAAwFIINwAAwFIINwAAwFIINwAAwFIINwAAwFIINwAAwFIINwAAwFIINwD84tlnn5XNZlNycrLZpQA4zdl4thQAfxg5cqT27dunPXv26LvvvlPPnj3NLgnAaYqRGwCn7IcfftB///tfzZs3Tx06dNDixYvNLqleJSUlZpcAoAUQbgCcssWLFysmJkZXX321brzxxnrDzeHDhzV16lR169ZNTqdTXbp0UVpamvLz8z19jh49qocfflhnnXWWgoOD1bFjR11//fXavXu3JGnNmjWy2Wxas2aN17737Nkjm82mV155xdM2fvx4hYeHa/fu3brqqqsUERGhsWPHSpL+85//aPTo0erataucTqcSExM1depUHTlypE7d3377rW666SZ16NBBISEh6tWrl2bMmCFJ+uijj2Sz2fT222/X2e61116TzWbTunXrmvx7Ajg1AWYXAKDtW7x4sa6//noFBQXplltu0XPPPaeNGzdq6NChkqTi4mJdcMEF2r59uyZOnKhBgwYpPz9fK1as0I8//qjY2Fi5XC798pe/VGZmpm6++WZNmTJFRUVFWr16tbZu3aoePXo0ua7KykqNGjVK559/vh5//HGFhoZKkt58802VlpbqrrvuUvv27bVhwwY9/fTT+vHHH/Xmm296tv/qq690wQUXKDAwUHfeeae6deum3bt365133tGjjz6qiy++WImJiVq8eLGuu+66Or9Jjx49NHz48FP4ZQH4xACAU/D5558bkozVq1cbhmEYbrfb6NKlizFlyhRPn1mzZhmSjOXLl9fZ3u12G4ZhGIsWLTIkGfPmzWuwz0cffWRIMj766COv93/44QdDkvHyyy972saNG2dIMu6///46+ystLa3TlpGRYdhsNmPv3r2etgsvvNCIiIjwaqtdj2EYxvTp0w2n02kcPnzY05aXl2cEBAQYDz30UJ3PAdD8OC0F4JQsXrxY8fHxuuSSSyRJNptNY8aM0ZIlS+RyuSRJb731lvr3719ndKOmf02f2NhY3XPPPQ328cVdd91Vpy0kJMSzXlJSovz8fI0YMUKGYeiLL76QJB04cECffPKJJk6cqK5duzZYT1pamsrKyrRs2TJP29KlS1VZWalbb73V57oB+I5wA8BnLpdLS5Ys0SWXXKIffvhBu3bt0q5du5ScnKzc3FxlZmZKknbv3q2+ffuecF+7d+9Wr169FBDgv7PlAQEB6tKlS532rKwsjR8/Xu3atVN4eLg6dOigiy66SJJUUFAgSfr+++8l6aR19+7dW0OHDvWaZ7R48WKdd955XDEGmIQ5NwB89uGHH2r//v1asmSJlixZUuf9xYsX6xe/+IXfPq+hEZyaEaLjOZ1O2e32On0vv/xyHTx4UPfdd5969+6tsLAw/fTTTxo/frzcbneT60pLS9OUKVP0448/qqysTJ999pkWLFjQ5P0A8A/CDQCfLV68WHFxcXrmmWfqvLd8+XK9/fbbWrhwoXr06KGtW7eecF89evTQ+vXrVVFRocDAwHr7xMTESKq68qq2vXv3Nrrmr7/+Wjt37tTf/vY3paWledpXr17t1a979+6SdNK6Jenmm29Wenq6Xn/9dR05ckSBgYEaM2ZMo2sC4F+clgLgkyNHjmj58uX65S9/qRtvvLHOMnnyZBUVFWnFihW64YYb9OWXX9Z7ybRRfR/RG264Qfn5+fWOeNT0SUpKksPh0CeffOL1/rPPPtvouh0Oh9c+a9affPJJr34dOnTQhRdeqEWLFikrK6veemrExsbqyiuv1D/+8Q8tXrxYV1xxhWJjYxtdEwD/YuQGgE9WrFihoqIi/epXv6r3/fPOO89zQ7/XXntNy5Yt0+jRozVx4kQNHjxYBw8e1IoVK7Rw4UL1799faWlpevXVV5Wenq4NGzboggsuUElJiT744APdfffduuaaaxQVFaXRo0fr6aefls1mU48ePfSvf/1LeXl5ja67d+/e6tGjh6ZNm6affvpJkZGReuutt3To0KE6fZ966imdf/75GjRokO68806dccYZ2rNnj1auXKktW7Z49U1LS9ONN94oSZozZ07jf0gA/mfmpVoA2q7U1FQjODjYKCkpabDP+PHjjcDAQCM/P9/4+eefjcmTJxudO3c2goKCjC5duhjjxo0z8vPzPf1LS0uNGTNmGGeccYYRGBhoJCQkGDfeeKOxe/duT58DBw4YN9xwgxEaGmrExMQY/+///T9j69at9V4KHhYWVm9d33zzjZGSkmKEh4cbsbGxxh133GF8+eWXdfZhGIaxdetW47rrrjOio6ON4OBgo1evXsbMmTPr7LOsrMyIiYkxoqKijCNHjjTyVwTQHHi2FAD4QWVlpTp16qTU1FS99NJLZpcDnNaYcwMAfvDPf/5TBw4c8JqkDMAcjNwAwClYv369vvrqK82ZM0exsbHavHmz2SUBpz1GbgDgFDz33HO66667FBcXp1dffdXscgCIkRsAAGAxjNwAAABLIdwAAABLOe1u4ud2u7Vv3z5FRESc0pOGAQBAyzEMQ0VFRerUqVOdZ8bV19lUCxYsMJKSkgyn02kMGzbMWL9+/Qn7//WvfzXOOussIzg42OjSpYtx7733NumGWdnZ2YYkFhYWFhYWlja4ZGdnn/RYb+rIzdKlS5Wenq6FCxcqOTlZ8+fP16hRo7Rjxw7FxcXV6f/aa6/p/vvv16JFizRixAjt3LlT48ePl81m07x58xr1mREREZKk7OxsRUZG+vX7AACA5lFYWKjExETPcfxETL1aKjk5WUOHDvU8KM/tdisxMVH33HOP7r///jr9J0+erO3btyszM9PT9vvf/17r16/Xp59+2qjPLCwsVFRUlAoKCgg3AAC0EU05fps2obi8vFybNm1SSkrKsWLsdqWkpGjdunX1bjNixAht2rRJGzZskCR9//33evfdd3XVVVc1+DllZWUqLCz0WgAAgHWZdloqPz9fLpdL8fHxXu3x8fH69ttv693m17/+tfLz83X++efLMAxVVlbqt7/9rR544IEGPycjI0OPPPKIX2sHAACtV5u6FHzNmjWaO3eunn32WW3evFnLly/XypUrNWfOnAa3mT59ugoKCjxLdnZ2C1YMAABammkjN7GxsXI4HMrNzfVqz83NVUJCQr3bzJw5U7fddptuv/12SdK5556rkpIS3XnnnZoxY0a9l4Y5nU45nc4m1+dyuVRRUdHk7SAFBgbK4XCYXQYA4DRlWrgJCgrS4MGDlZmZqWuvvVZS1YTizMxMTZ48ud5tSktL6wSYmoOov+ZFG4ahnJwcHT582C/7O11FR0crISGBewkBAFqcqZeCp6ena9y4cRoyZIiGDRum+fPnq6SkRBMmTJAkpaWlqXPnzsrIyJAkpaamat68eRo4cKCSk5O1a9cuzZw5U6mpqX4bKagJNnFxcQoNDeXg3ESGYai0tFR5eXmSpI4dO5pcEQDgdGNquBkzZowOHDigWbNmKScnRwMGDNCqVas8k4yzsrK8RmoefPBB2Ww2Pfjgg/rpp5/UoUMHpaam6tFHH/VLPS6XyxNs2rdv75d9no5CQkIkSXl5eYqLi+MUFQCgRZ12TwU/0XXyR48e1Q8//KBu3bp5DtDwzZEjR7Rnzx6dccYZCg4ONrscAEAb1ybuc9OacSrq1PEbAgDMQrgBAACWQrhBHd26ddP8+fPNLgMAAJ+YOqEY/nPxxRdrwIABfgklGzduVFhY2KkXBQCACQg3pwnDMORyuRQQcPK/8g4dOrRARQAAy3FVSCUHqv6MSTKtDE5LWcD48eP18ccf68knn5TNZpPNZtMrr7wim82m9957T4MHD5bT6dSnn36q3bt365prrlF8fLzCw8M1dOhQffDBB177O/60lM1m0//+7//quuuuU2hoqM4880ytWLGihb8lAMAUrgqpcJ+0b4v03Wrpi39I/5knrZouLZsovfJL6Zlk6U/dpDmx0ryzpRX134y3pTBycxKGYehIhcuUzw4JdDTqqqMnn3xSO3fuVN++fTV79mxJ0rZt2yRJ999/vx5//HF1795dMTExys7O1lVXXaVHH31UTqdTr776qlJTU7Vjxw517dq1wc945JFH9Oc//1l/+ctf9PTTT2vs2LHau3ev2rVr558vCwBoOa4KqSRfKsmTiquXhtaPHGzavm0OyeS7zBBuTuJIhUt9Zr1vymd/M3uUQoNO/lcUFRWloKAghYaGep7LVfNk9dmzZ+vyyy/39G3Xrp369+/veT1nzhy9/fbbWrFiRYOPvZCqRoduueUWSdLcuXP11FNPacOGDbriiit8+m4AAD9zVUql+VJxrlR8oDqgNLBe+nPT9m1zSGEdpPAOUni8FBbX8HpIjFTPsx5bEuHG4oYMGeL1uri4WA8//LBWrlyp/fv3q7KyUkeOHFFWVtYJ99OvXz/PelhYmCIjIz2PWAAANBNPYDl+ZOVAdViptV56UFITRkxsDiksVgqPqw4ocQ2vh7QzPbA0BeHmJEICHfpm9ijTPvtUHX/V07Rp07R69Wo9/vjj6tmzp0JCQnTjjTeqvLz8hPsJDAz0em2z2eR2u0+5PgA47bhdtU4J1R5ZqeeUUOnPalpgsVeNsHiNpnSoDivHrbexwNIUhJuTsNlsjTo1ZLagoCC5XCefG7R27VqNHz9e1113naSqkZw9e/Y0c3UAYHFuV1UQOX40xWu9OsSU5KvJgSU09rjRlAZOCYW2k+w8z6/1H7XRKN26ddP69eu1Z88ehYeHNziqcuaZZ2r58uVKTU2VzWbTzJkzGYEBgPp4AssJJtt6Tgn9LBlN+X+prfqUUO3RlAZOCYW2J7A0EeHGIqZNm6Zx48apT58+OnLkiF5++eV6+82bN08TJ07UiBEjFBsbq/vuu0+FhYUtXC0AmMTtqpqbcqLJtp5TQvm+BZbGnBIisDQrngpeS81TwXmS9anjtwTgN4YhVR6Vyoql8qLqP4vreV1UtXjeO65PWZFvgSW0/QlGVmqfEmovORgzaC5NeSo4fwsAAP9zu+oJGkW1QsjJgkrNn4VV64Y/7zdmq5qb0uDISu1TQrEEljaIvzEAQNXoSMWR40LIcYGk3qDSwKhJ5ZHmqTMwTHKGS0Hh1X9GVP3pjKjbVl+fsA4EltMAf7vA6cwwpCOHqucYVF/ZUZxzbL2sqGpegD2g1nL86/raTmEbm+MU92vNS1vr5ar0PuXiSwip/adfR0eq2QPrDxzOiHpCSH0hpVafoDDmqaBRCDeAFVWW1QoruVJRTj0Bpvq168T3OGp7bKcYvJojzJ1kH5JUXtLAaZoTnLqpPNo8P2FQPYHDK3QcH0Jqv47wDjMBzuapETgBwg3QVtSMshTl1B9SinOlouo/jx5u2r6Do6WIhGNzD2qW4MiquRNul+SurLUc/7q+tmZ+3eAogyG5K6qW04UjqOHRjpOdpjk+qASGnV6jX7Akwg1gtooj3gGldkjxas9r2gHbEeQdVGqCS0R83fa2+K9rw2im0NVS29R6LaPqlEtDIaTBUznVoSQgyOy/DaBVIdwAzcHtrnqSbp1TQrWCSk17WUHT9h3S7riQUjPakuAdYIKjpUY8Vb7NstmqJoUyMRTAcfi/AtAU5aV1Q0q9ASavaZMzHc56RlTqCTBhcfwrHQBOgnAD1H4mTNHxwaVWaCnKrZrM2RShscfCSX1zWmreC46y9igLALQgwg0kVT2b6t5779W9995rdin+U1Z88om3Nc+GacooS0DIcSMqCfUHmLAOkiPw5PsDAPgV4QbmMYyq26C7XVV/Gu6qkOFpM4699upXfedTz7yV+gJMXtWlso1W8xC74+at1BdgnBGMsgBAK0a48Rd3ZdV8DBnVT7KveWSXUXWQ9qjnfenkfep9Xd/2jX193P7clVUPjcv/zrfPO76twpAKcqXnJ0ol2ceFluqwUnvb5hIYWh1UGjglVBNguGMpAFgG/zf3l8oy6eBuUz76hX+8pYfnPa8fP18le637U1wzYarax0Rpxu9+o/RH5umzzV+rpPSIzj7zDGXcf49SLkw+thPDLbnKmjjacQI1ozIVRU2fp3I8m73qrrU2e9WNz2z2qtASEd/wKaGaxRnun+8DAGgzCDcnYxhSRenJ+1XUPEfFduxPW2Ne12o7/nVgaPXpj5r3bfX0t2n0mF/rnpl/0Udf7NZlF18oSTp48JBWrVmnd99arGJbhK66+pd6dM5sOZ1OvfraG0qdMFU7vvhMXbt2keeOriHtpJhutQup5zMben3cdzxaJhUGSGPfkpyB3gGldkipvXjaavfjZmIAgKYh3JxMRak0t5M5n/3Avqobe51ETGQnXXnllXrtn+/rstSbJEnLXl+h2NhYXXLV9bLb7eo//BJP/zn9k/X2yve14oP/aPLkyVWNNrsUFCqFxPindpetajJtTJIUHOyffQIA0Aj8s9gixo4dq7feektlZWWSpMWLF+vmm2+W3W5XcXGxpk2bprPPPlvR0dEKDw/X9u3blZWVZXLVAAD4HyM3JxMYWjWCYtZnN1JqaqoMw9DKlSs1dOhQ/ec//9Ff//pXSdK0adO0evVqPf744+rZs6dCQkJ04403qrzcag9MBACAcHNyNlujTg2ZLTg4WNdff70WL16sXbt2qVevXho0aJAkae3atRo/fryuu+46SVJxcbH27NljYrUAADQfwo2FjB07Vr/85S+1bds23XrrrZ72M888U8uXL1dqaqpsNptmzpwpt9ttYqUAADQf5txYyKWXXqp27dppx44d+vWvf+1pnzdvnmJiYjRixAilpqZq1KhRnlEdAACshpEbC7Hb7dq3r+78oG7duunDDz/0aps0aZLXa05TAQCsgpEbAABgKYQbAABgKYQbAABgKYQbAABgKYSbehhGCzyt2uL4DQEAZiHc1BIYGChJKi1txIMycUI1v2HNbwoAQEvhUvBaHA6HoqOjlZeXJ0kKDQ2Vzeup1zgZwzBUWlqqvLw8RUdHy+FwmF0SAOA0Q7g5TkJCgiR5Ag58Ex0d7fktAQBoSYSb49hsNnXs2FFxcXGqqKgwu5w2KTAwkBEbAIBpCDcNcDgcHKABAGiDmFAMAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAspVWEm2eeeUbdunVTcHCwkpOTtWHDhgb7XnzxxbLZbHWWq6++ugUrBgAArZXp4Wbp0qVKT0/XQw89pM2bN6t///4aNWqU8vLy6u2/fPly7d+/37Ns3bpVDodDo0ePbuHKAQBAa2R6uJk3b57uuOMOTZgwQX369NHChQsVGhqqRYsW1du/Xbt2SkhI8CyrV69WaGgo4QYAAEgyOdyUl5dr06ZNSklJ8bTZ7XalpKRo3bp1jdrHSy+9pJtvvllhYWH1vl9WVqbCwkKvBQAAWJep4SY/P18ul0vx8fFe7fHx8crJyTnp9hs2bNDWrVt1++23N9gnIyNDUVFRniUxMfGU6wYAAK2X6aelTsVLL72kc889V8OGDWuwz/Tp01VQUOBZsrOzW7BCAADQ0gLM/PDY2Fg5HA7l5uZ6tefm5iohIeGE25aUlGjJkiWaPXv2Cfs5nU45nc5TrhUAALQNpo7cBAUFafDgwcrMzPS0ud1uZWZmavjw4Sfc9s0331RZWZluvfXW5i4TAAC0IaaO3EhSenq6xo0bpyFDhmjYsGGaP3++SkpKNGHCBElSWlqaOnfurIyMDK/tXnrpJV177bVq3769GWUDAIBWyvRwM2bMGB04cECzZs1STk6OBgwYoFWrVnkmGWdlZclu9x5g2rFjhz799FP9+9//NqNkAADQitkMwzDMLqIlFRYWKioqSgUFBYqMjDS7HAAA0AhNOX636aulAAAAjke4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlkK4AQAAlmJ6uHnmmWfUrVs3BQcHKzk5WRs2bDhh/8OHD2vSpEnq2LGjnE6nzjrrLL377rstVC0AAGjtAsz88KVLlyo9PV0LFy5UcnKy5s+fr1GjRmnHjh2Ki4ur07+8vFyXX3654uLitGzZMnXu3Fl79+5VdHR0yxcPAABaJZthGIZZH56cnKyhQ4dqwYIFkiS3263ExETdc889uv/+++v0X7hwof7yl7/o22+/VWBgoE+fWVhYqKioKBUUFCgyMvKU6gcAAC2jKcdv005LlZeXa9OmTUpJSTlWjN2ulJQUrVu3rt5tVqxYoeHDh2vSpEmKj49X3759NXfuXLlcrgY/p6ysTIWFhV4LAACwLtPCTX5+vlwul+Lj473a4+PjlZOTU+8233//vZYtWyaXy6V3331XM2fO1BNPPKE//vGPDX5ORkaGoqKiPEtiYqJfvwcAAGhdTJ9Q3BRut1txcXF64YUXNHjwYI0ZM0YzZszQwoULG9xm+vTpKigo8CzZ2dktWDEAAGhppk0ojo2NlcPhUG5urld7bm6uEhIS6t2mY8eOCgwMlMPh8LSdffbZysnJUXl5uYKCgups43Q65XQ6/Vs8AABotUwbuQkKCtLgwYOVmZnpaXO73crMzNTw4cPr3WbkyJHatWuX3G63p23nzp3q2LFjvcEGAACcfkw9LZWenq4XX3xRf/vb37R9+3bdddddKikp0YQJEyRJaWlpmj59uqf/XXfdpYMHD2rKlCnauXOnVq5cqblz52rSpElmfQUAANDKmHqfmzFjxujAgQOaNWuWcnJyNGDAAK1atcozyTgrK0t2+7H8lZiYqPfff19Tp05Vv3791LlzZ02ZMkX33XefWV8BAAC0Mqbe58YM3OcGAIC2p03c5wYAAKA5EG4AAIClEG4AAICl+BRuPvroI3/XAQAA4Bc+hZsrrrhCPXr00B//+Efu+AsAAFoVn8LNTz/9pMmTJ2vZsmXq3r27Ro0apTfeeEPl5eX+rg8AAKBJfAo3sbGxmjp1qrZs2aL169frrLPO0t13361OnTrpd7/7nb788kt/1wkAANAopzyheNCgQZo+fbomT56s4uJiLVq0SIMHD9YFF1ygbdu2+aNGAACARvM53FRUVGjZsmW66qqrlJSUpPfff18LFixQbm6udu3apaSkJI0ePdqftQIAAJyUT3covueee/T666/LMAzddtttuv3229W3b1+vPjk5OerUqZPXQy5bA+5QDABA29OU47dPz5b65ptv9PTTT+v666+X0+mst09sbCyXjAMAgBbHs6UAAECr1+zPlsrIyNCiRYvqtC9atEh/+tOffNklAACAX/gUbp5//nn17t27Tvs555yjhQsXnnJRAAAAvvIp3OTk5Khjx4512jt06KD9+/efclEAAAC+8incJCYmau3atXXa165dq06dOp1yUQAAAL7y6WqpO+64Q/fee68qKip06aWXSpIyMzP1P//zP/r973/v1wIBAACawqdw84c//EE///yz7r77bs/zpIKDg3Xfffdp+vTpfi0QAACgKU7pUvDi4mJt375dISEhOvPMMxu8501rwqXgAAC0Pc1+E78a4eHhGjp06KnsAgAAwK98Djeff/653njjDWVlZXlOTdVYvnz5KRcGAADgC5+ullqyZIlGjBih7du36+2331ZFRYW2bdumDz/8UFFRUf6uEQAAoNF8Cjdz587VX//6V73zzjsKCgrSk08+qW+//VY33XSTunbt6u8aAQAAGs2ncLN7925dffXVkqSgoCCVlJTIZrNp6tSpeuGFF/xaIAAAQFP4FG5iYmJUVFQkSercubO2bt0qSTp8+LBKS0v9Vx0AAEAT+TSh+MILL9Tq1at17rnnavTo0ZoyZYo+/PBDrV69Wpdddpm/awQAAGg0n8LNggULdPToUUnSjBkzFBgYqP/+97+64YYb9OCDD/q1QAAAgKZocriprKzUv/71L40aNUqSZLfbdf/99/u9MAAAAF80ec5NQECAfvvb33pGbgAAAFoTnyYUDxs2TFu2bPFzKQAAAKfOpzk3d999t9LT05Wdna3BgwcrLCzM6/1+/fr5pTgAAICm8unBmXZ73QEfm80mwzBks9nkcrn8Ulxz4MGZAAC0Pc3+4MwffvjBp8IAAACam0/hJikpyd91AAAA+IVP4ebVV1894ftpaWk+FQMAAHCqfJpzExMT4/W6oqJCpaWlCgoKUmhoqA4ePOi3Av2NOTcAALQ9TTl++3Qp+KFDh7yW4uJi7dixQ+eff75ef/11n4oGAADwB5/CTX3OPPNMPfbYY5oyZYq/dgkAANBkfgs3UtXdi/ft2+fPXQIAADSJTxOKV6xY4fXaMAzt379fCxYs0MiRI/1SGAAAgC98CjfXXnut12ubzaYOHTro0ksv1RNPPOGPugAAAHziU7hxu93+rgMAAMAv/DrnBgAAwGw+hZsbbrhBf/rTn+q0//nPf9bo0aNPuSgAAABf+RRuPvnkE1111VV12q+88kp98sknp1wUAACAr3wKN8XFxQoKCqrTHhgYqMLCwlMuCgAAwFc+hZtzzz1XS5curdO+ZMkS9enT55SLAgAA8JVPV0vNnDlT119/vXbv3q1LL71UkpSZmanXX39db775pl8LBAAAaAqfwk1qaqr++c9/au7cuVq2bJlCQkLUr18/ffDBB7rooov8XSMAAECj+fRU8LaMp4IDAND2NPtTwTdu3Kj169fXaV+/fr0+//xzX3YJAADgFz6Fm0mTJik7O7tO+08//aRJkyadclEAAAC+8incfPPNNxo0aFCd9oEDB+qbb7455aIAAAB85VO4cTqdys3NrdO+f/9+BQT4NEcZAADAL3wKN7/4xS80ffp0FRQUeNoOHz6sBx54QJdffrnfigMAAGgqn4ZZHn/8cV144YVKSkrSwIEDJUlbtmxRfHy8/v73v/u1QAAAgKbwKdx07txZX331lRYvXqwvv/xSISEhmjBhgm655RYFBgb6u0YAAIBG83mCTFhYmM4//3x17dpV5eXlkqT33ntPkvSrX/3KP9UBAAA0kU/h5vvvv9d1112nr7/+WjabTYZhyGazed53uVx+KxAAAKApfJpQPGXKFJ1xxhnKy8tTaGiotm7dqo8//lhDhgzRmjVr/FwiAABA4/kUbtatW6fZs2crNjZWdrtdDodD559/vjIyMvS73/2uyft75pln1K1bNwUHBys5OVkbNmxosO8rr7wim83mtQQHB/vyNQAAgAX5FG5cLpciIiIkSbGxsdq3b58kKSkpSTt27GjSvpYuXar09HQ99NBD2rx5s/r3769Ro0YpLy+vwW0iIyO1f/9+z7J3715fvgYAALAgn8JN37599eWXX0qSkpOT9ec//1lr167V7Nmz1b179ybta968ebrjjjs0YcIE9enTRwsXLlRoaKgWLVrU4DY2m00JCQmeJT4+3pevAQAALMincPPggw/K7XZLkmbPnq0ffvhBF1xwgd5991099dRTjd5PeXm5Nm3apJSUlGMF2e1KSUnRunXrGtyuuLhYSUlJSkxM1DXXXKNt27b58jUAAIAF+XS11KhRozzrPXv21LfffquDBw8qJibG66qpk8nPz5fL5aoz8hIfH69vv/223m169eqlRYsWqV+/fiooKNDjjz+uESNGaNu2berSpUud/mVlZSorK/O8LiwsbHR9AACg7fFp5KY+7dq1a1Kw8dXw4cOVlpamAQMG6KKLLtLy5cvVoUMHPf/88/X2z8jIUFRUlGdJTExs9hoBAIB5/BZufBEbGyuHw1HnIZy5ublKSEho1D4CAwM1cOBA7dq1q973a56BVbNkZ2efct0AAKD1MjXcBAUFafDgwcrMzPS0ud1uZWZmavjw4Y3ah8vl0tdff62OHTvW+77T6VRkZKTXAgAArMvnxy/4S3p6usaNG6chQ4Zo2LBhmj9/vkpKSjRhwgRJUlpamjp37qyMjAxJVROYzzvvPPXs2VOHDx/WX/7yF+3du1e33367mV8DAAC0EqaHmzFjxujAgQOaNWuWcnJyNGDAAK1atcozyTgrK0t2+7EBpkOHDumOO+5QTk6OYmJiNHjwYP33v/9Vnz59zPoKAACgFbEZhmGYXURLKiwsVFRUlAoKCjhFBQBAG9GU47epc24AAAD8jXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAshXADAAAspVWEm2eeeUbdunVTcHCwkpOTtWHDhkZtt2TJEtlsNl177bXNWyAAAGgzTA83S5cuVXp6uh566CFt3rxZ/fv316hRo5SXl3fC7fbs2aNp06bpggsuaKFKAQBAW2B6uJk3b57uuOMOTZgwQX369NHChQsVGhqqRYsWNbiNy+XS2LFj9cgjj6h79+4tWC0AAGjtTA035eXl2rRpk1JSUjxtdrtdKSkpWrduXYPbzZ49W3FxcfrNb35z0s8oKytTYWGh1wIAAKzL1HCTn58vl8ul+Ph4r/b4+Hjl5OTUu82nn36ql156SS+++GKjPiMjI0NRUVGeJTEx8ZTrBgAArZfpp6WaoqioSLfddptefPFFxcbGNmqb6dOnq6CgwLNkZ2c3c5UAAMBMAWZ+eGxsrBwOh3Jzc73ac3NzlZCQUKf/7t27tWfPHqWmpnra3G63JCkgIEA7duxQjx49vLZxOp1yOp3NUD0AAGiNTB25CQoK0uDBg5WZmelpc7vdyszM1PDhw+v07927t77++mtt2bLFs/zqV7/SJZdcoi1btnDKCQAAmDtyI0np6ekaN26chgwZomHDhmn+/PkqKSnRhAkTJElpaWnq3LmzMjIyFBwcrL59+3ptHx0dLUl12gEAwOnJ9HAzZswYHThwQLNmzVJOTo4GDBigVatWeSYZZ2VlyW5vU1ODAACAiWyGYRhmF9GSCgsLFRUVpYKCAkVGRppdDgAAaISmHL8ZEgEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuPETt9vQRzvydLi03OxSAAA4rQWYXYBV7DpQrAkvb5Qkde8QpkFdY6qWpGidGRchh91mcoUAAJweCDd+crCkXN1jw/R9fom+P1C1LNv0oyQp3BmgAYnRGtQ1WgOTYjQwMVrRoUEmVwwAgDXZDMMwzC6iJRUWFioqKkoFBQWKjIz0+/4PlZTri+xD2rz3sDZnHdKX2YdVUu6q04/RHQAAGq8px2/CTTNzuQ3tyCnS5qxD2px1SF9kHdYP+SV1+kU4A9S/1ujOoMQYRYUGNnt9AAC0BYSbE2jpcFOfgyXl+qI67Gzee1hf/nhYpfWM7vSoGd1JqhrhOTMuXHZGdwAArZxhGLLZ/Hu8ItycQGsIN8erdLm1I7dIm7MO64u9VaFnz8+ldfpFOAM0oGu0BnaNqRrhYXQHAOAjwzBU7nLrSLlLJeUuHSmvVGm5SyVlLh2pqFovLXOptLxSpRU161XvlRy3fqTcpdKKSk+fAYnRev3O8/xab1OO30wobgUCHHad0ylK53SK0m3nJUmSfi4u0xdZhz2ns77MLlBRWaX+812+/vNdvmfbnnHhGtQ12jPC07MDozsAYCUut6EjFS6VllUHjvLqwHHcelVIqawKGset19e/tNwll7t5xjdKyiubZb+NxchNG1HpcuvbnCJ9UT1vp8HRneCaK7Oqws6AxGhFhTC6AwDNyTAMlVW66w8SZS6VVlSNjFSNilS1HxvxqBVcaq0fqXCppKxSZZXuZq8/yGFXSJBDYUEOhQQ5FBoUoNAgR/VybD0kKMCrT5jToZDA6j7O6v6BAQoPDlC7MP9eFcxpqRNoq+GmPvWN7hypqDt358y4cM9VWYO6xqgHozsATlOVLnetUyzeIaTO6Zla63VDS3VYqR4xKS2vVDMNgnjYbFJYUEB1sDg+gFS9PhZQqt6rvV53m2P7CnS0/nv6Em5OwErh5ni1R3c2V4eevfWM7kQGB2hA9bydQV1jNKBrtCKDGd0BWoLbbehgablyC49WL2U6WFIut9uQ25AMVf0pw5AhyW0YMgx53pNRt80wqkYOavev2kVNv2NtbqPWPqo+ptZ6zb6832/cvmvq9t6f+7j+hgy53fW0GbVf1962qr+Ob6uuU7W2ddeq1TC8+9fUWuFq/kOeM8Beb5AIrR7xqB1QwpwB1SMfDoU6AxRaez2oalQkrHrdGWD3+yTdtoRwcwJWDjf1ya89urP3kL76se7ojs1WNbozMJHRHcBXhmGo8EilcouOKqegKrjkFZV5hZi86rbK5v4nPk7KYbc14rSL4wQjJd5hJTTo2Kka7lnWPAg3J3C6hZvj1Yzu1ISdzVmHlXWQ0R3gRErKKpVbeFQ5hUeVV1jmCSu5RUeVV7NeeLTRcyNsNql9mFPxkU7FRwarXViQAh02STbZbVXv2zzrNs9rm00Nt1X3l80mmyS77eT9bTXv6Vi/Bvchm2SrbpNkt9e3j2P95fks7/5e+/DUdazGE+6j+j15fVYD+6j+ne3VG9jtVd8tJLAqhAQ5Tu9RkLaIcHMCp3u4qc+BojKvU1lf/XhYRyu8/yddM7pT+67K3WMZ3UHbdrTCpQPVoys5tUZXvMNLmYrLGn/lR3RooOIjghVXHVwSIoMVH+lUXGSw4qvXY8OdbWKOA9CaEG5OgHBzchUut77d731X5fpGd6JCAmtdmRWtAYnRimB0B61Ahcut/OIyz4hK7VNDuTWjL0VHdbi0otH7DHcGVAWWiGDPiEt8rcASHxmsDhFOBQc6mvGbAacvws0JEG5809jRnbPiIjQoqeZGgzHqHhvG6A78xu029HNJefV8lqPKKSjzrOfWOl30c0mZGvt/NmeA3RNQ4iKDPeElISpYcRHH2sOd3BYMMBPh5gQIN/5x/OjO5qxDyj54pE6/qJBADay5yWDXGPVPjGJ0B3UYhqGCIxUNjrTkFlWdLjrQhMm4AXab4iJqTgfVP9ISHxGsyJAA5l4AbQDh5gQIN80nr+io58qsL6qfmXX8BEubTeoVH+F5hMSgpKrRHQ4u1lVcPRm35nRQTq31quBSFWLKmzAZNza8ejJuRLBXeEmIPDbXpV1oEKOGgIUQbk6AcNNyKlxubd9f6Lkqa3PWIf14iNEdqzha4fLMXfEaZTkuvJTU81DYhsSEBio+sjqwRDi9ThclVI+6xIYHKYDJuMBph3BzAoQbc+UVHdXmvYc9T0X/6scCRnf8wDAMudyGXDV/uqtufOYyDFW63Z51l+tYH7dhqNJV9afLbajSfWy9Ziktr/TcqyWnoKx6bktVkCk40vjJuBE1k3GrA8qxibnBSohyKi6CybgAToxwcwKEm9alvLJ6dKd6svIXDYzuRIcGamCtZ2Z1bRfa4AHZZRhy17xX/dqzXnNQd9d3gK8JAG65DB3bxwn2X3t/bnc9AeO4AOEyJJfb7Qkfle5jn3Wi/bvq2Xft98y6J5wzwK6EqGCvS59rThHFRQRXT8p1KozJuABOEeHmBAg3rV9e4VFP0GlodAdNE2C3yW63KcBuk8NWte6oWWy11qtvdBZgt8tutyk40O65eqj2fVoSqk8dRQYzGRdAy2jK8Zt/TqHViYsM1hV9E3RF3wRJdUd3Nu89pJ9LyqoOwDYpwGGX3WaTw66qA7Xj2AE8wG6rfs/mOcDXfs9R/X7t97wP9LXes1cf9Ks/6/iw0Kh92GwKcByrqeY9h71uvQ675LDbq/df9dkOu6q3sctuV51wUmcf1bUBwOmEkRsAANDqNeX4zSUHAADAUlpFuHnmmWfUrVs3BQcHKzk5WRs2bGiw7/LlyzVkyBBFR0crLCxMAwYM0N///vcWrBYAALRmpoebpUuXKj09XQ899JA2b96s/v37a9SoUcrLy6u3f7t27TRjxgytW7dOX331lSZMmKAJEybo/fffb+HKAQBAa2T6nJvk5GQNHTpUCxYskCS53W4lJibqnnvu0f3339+ofQwaNEhXX3215syZc9K+zLkBAKDtaTNzbsrLy7Vp0yalpKR42ux2u1JSUrRu3bqTbm8YhjIzM7Vjxw5deOGF9fYpKytTYWGh1wIAAKzL1HCTn58vl8ul+Ph4r/b4+Hjl5OQ0uF1BQYHCw8MVFBSkq6++Wk8//bQuv/zyevtmZGQoKirKsyQmJvr1OwAAgNbF9Dk3voiIiNCWLVu0ceNGPfroo0pPT9eaNWvq7Tt9+nQVFBR4luzs7JYtFgAAtChTb+IXGxsrh8Oh3Nxcr/bc3FwlJCQ0uJ3dblfPnj0lSQMGDND27duVkZGhiy++uE5fp9Mpp9Pp17oBAEDrZerITVBQkAYPHqzMzExPm9vtVmZmpoYPH97o/bjdbpWVlTVHiQAAoI0x/fEL6enpGjdunIYMGaJhw4Zp/vz5Kikp0YQJEyRJaWlp6ty5szIyMiRVzaEZMmSIevToobKyMr377rv6+9//rueee87MrwEAAFoJ08PNmDFjdODAAc2aNUs5OTkaMGCAVq1a5ZlknJWVJbv92ABTSUmJ7r77bv34448KCQlR79699Y9//ENjxowx6ysAAIBWxPT73LQ07nMDAEDb02bucwMAAOBvhBsAAGApps+5aWk1Z+G4UzEAAG1HzXG7MbNpTrtwU1RUJEncqRgAgDaoqKhIUVFRJ+xz2k0odrvd2rdvnyIiImSz2fy678LCQiUmJio7O5vJys2I37ll8Du3DH7nlsNv3TKa63c2DENFRUXq1KmT11XU9TntRm7sdru6dOnSrJ8RGRnJfzgtgN+5ZfA7twx+55bDb90ymuN3PtmITQ0mFAMAAEsh3AAAAEsh3PiR0+nUQw89xIM6mxm/c8vgd24Z/M4th9+6ZbSG3/m0m1AMAACsjZEbAABgKYQbAABgKYQbAABgKYQbAABgKYQbP/jkk0+UmpqqTp06yWaz6Z///KfZJVlSRkaGhg4dqoiICMXFxenaa6/Vjh07zC7Lcp577jn169fPcwOu4cOH67333jO7LMt77LHHZLPZdO+995pdiqU8/PDDstlsXkvv3r3NLsuSfvrpJ916661q3769QkJCdO655+rzzz83pRbCjR+UlJSof//+euaZZ8wuxdI+/vhjTZo0SZ999plWr16tiooK/eIXv1BJSYnZpVlKly5d9Nhjj2nTpk36/PPPdemll+qaa67Rtm3bzC7NsjZu3Kjnn39e/fr1M7sUSzrnnHO0f/9+z/Lpp5+aXZLlHDp0SCNHjlRgYKDee+89ffPNN3riiScUExNjSj2n3eMXmsOVV16pK6+80uwyLG/VqlVer1955RXFxcVp06ZNuvDCC02qynpSU1O9Xj/66KN67rnn9Nlnn+mcc84xqSrrKi4u1tixY/Xiiy/qj3/8o9nlWFJAQIASEhLMLsPS/vSnPykxMVEvv/yyp+2MM84wrR5GbtBmFRQUSJLatWtnciXW5XK5tGTJEpWUlGj48OFml2NJkyZN0tVXX62UlBSzS7Gs7777Tp06dVL37t01duxYZWVlmV2S5axYsUJDhgzR6NGjFRcXp4EDB+rFF180rR5GbtAmud1u3XvvvRo5cqT69u1rdjmW8/XXX2v48OE6evSowsPD9fbbb6tPnz5ml2U5S5Ys0ebNm7Vx40azS7Gs5ORkvfLKK+rVq5f279+vRx55RBdccIG2bt2qiIgIs8uzjO+//17PPfec0tPT9cADD2jjxo363e9+p6CgII0bN67F6yHcoE2aNGmStm7dyrnzZtKrVy9t2bJFBQUFWrZsmcaNG6ePP/6YgONH2dnZmjJlilavXq3g4GCzy7Gs2lMG+vXrp+TkZCUlJemNN97Qb37zGxMrsxa3260hQ4Zo7ty5kqSBAwdq69atWrhwoSnhhtNSaHMmT56sf/3rX/roo4/UpUsXs8uxpKCgIPXs2VODBw9WRkaG+vfvryeffNLssixl06ZNysvL06BBgxQQEKCAgAB9/PHHeuqppxQQECCXy2V2iZYUHR2ts846S7t27TK7FEvp2LFjnX/8nH322aadAmTkBm2GYRi655579Pbbb2vNmjWmTlY73bjdbpWVlZldhqVcdtll+vrrr73aJkyYoN69e+u+++6Tw+EwqTJrKy4u1u7du3XbbbeZXYqljBw5ss6tOXbu3KmkpCRT6iHc+EFxcbHXvwJ++OEHbdmyRe3atVPXrl1NrMxaJk2apNdee03/93//p4iICOXk5EiSoqKiFBISYnJ11jF9+nRdeeWV6tq1q4qKivTaa69pzZo1ev/9980uzVIiIiLqzBcLCwtT+/btmUfmR9OmTVNqaqqSkpK0b98+PfTQQ3I4HLrlllvMLs1Spk6dqhEjRmju3Lm66aabtGHDBr3wwgt64YUXzCnIwCn76KOPDEl1lnHjxpldmqXU9xtLMl5++WWzS7OUiRMnGklJSUZQUJDRoUMH47LLLjP+/e9/m13WaeGiiy4ypkyZYnYZljJmzBijY8eORlBQkNG5c2djzJgxxq5du8wuy5Leeecdo2/fvobT6TR69+5tvPDCC6bVYjMMwzAnVgEAAPgfE4oBAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AnPbWrFkjm82mw4cPm10KAD8g3AAAAEsh3AAAAEsh3AAwndvtVkZGhs444wyFhISof//+WrZsmaRjp4xWrlypfv36KTg4WOedd562bt3qtY+33npL55xzjpxOp7p166YnnnjC6/2ysjLdd999SkxMlNPpVM+ePfXSSy959dm0aZOGDBmi0NBQjRgxos5TjgG0DYQbAKbLyMjQq6++qoULF2rbtm2aOnWqbr31Vn388ceePn/4wx/0xBNPaOPGjerQoYNSU1NVUVEhqSqU3HTTTbr55pv19ddf6+GHH9bMmTP1yiuveLZPS0vT66+/rqeeekrbt2/X888/r/DwcK86ZsyYoSeeeEKff/65AgICNHHixBb5/gD8iwdnAjBVWVmZ2rVrpw8++EDDhw/3tN9+++0qLS3VnXfeqUsuuURLlizRmDFjJEkHDx5Uly5d9Morr+imm27S2LFjdeDAAf373//2bP8///M/WrlypbZt26adO3eqV69eWr16tVJSUurUsGbNGl1yySX64IMPdNlll0mS3n33XV199dU6cuSIgoODm/lXAOBPjNwAMNWuXbtUWlqqyy+/XOHh4Z7l1Vdf1e7duz39agefdu3aqVevXtq+fbskafv27Ro5cqTXfkeOHKnvvvtOLpdLW7ZskcPh0EUXXXTCWvr16+dZ79ixoyQpLy/vlL8jgJYVYHYBAE5vxcXFkqSVK1eqc+fOXu85nU6vgOOrkJCQRvULDAz0rNtsNklV84EAtC2M3AAwVZ8+feR0OpWVlaWePXt6LYmJiZ5+n332mWf90KFD2rlzp84++2xJ0tlnn621a9d67Xft2rU666yz5HA4dO6558rtdnvN4QFgXYzcADBVRESEpk2bpqlTp8rtduv8889XQUGB1q5dq8jISCUlJUmSZs+erfbt2ys+Pl4zZsxQbGysrr32WknS73//ew0dOlRz5szRmDFjtG7dOi1YsEDPPvusJKlbt24aN26cJk6cqKeeekr9+/fX3r17lZeXp5tuusmsrw6gmRBuAJhuzpw56tChgzIyMvT9998rOjpagwYN0gMPPOA5LfTYY49pypQp+u677zRgwAC98847CgoKkiQNGjRIb7zxhmbNmqU5c+aoY8eOmj17tsaPH+/5jOeee04PPPCA7r77bv3888/q2rWrHnjgATO+LoBmxtVSAFq1miuZDh06pOjoaLPLAdAGMOcGAABYCuEGAABYCqelAACApTByAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALOX/Azri8IgJqC4PAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# draw Accuracies\n",
    "if is_graphic:\n",
    "  plt.plot(list(range(1, len(arr_train_acc) + 1)), arr_train_acc, label=\"train\")\n",
    "  plt.plot(list(range(1, len(arr_val_acc) + 1)), arr_val_acc, label=\"val\")\n",
    "\n",
    "  plt.title(\"Accuracy\")\n",
    "  plt.xlabel(\"epoch\")\n",
    "  plt.ylabel(\"accuracy\")\n",
    "  plt.legend(loc=\"upper left\")\n",
    "\n",
    "  plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Eval model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HeterSUMGraph(device=device, vocab_size=vocab_size, word_embed=embmgr.getEmbeddings())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_epoch = np.argmin(np.array(arr_val_loss)) + 1\n",
    "model.load(checkpoints_folder + \"/\" + model_name + \"-\" + str(best_epoch) + \".pt\")\n",
    "model.to(device)\n",
    "model.eval()\n",
    "best_epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test[\"doc_splitted\"] = split_all_docs(df_test[args.doc_column_name], False)\n",
    "test_set = df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)\n",
    "criterion = nn.BCELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Eval model: 100%|██████████| 100/100 [00:02<00:00, 42.90batch/s, accuracy=0.737, loss=0.459, nb_nvalid_doc=0, rouge1=0.238, rouge2=0.0688, rougeL=0.151]\n"
     ]
    }
   ],
   "source": [
    "arr_rouge1 = []\n",
    "arr_rouge2 = []\n",
    "arr_rougeL = []\n",
    "arr_loss = []\n",
    "accuracy = []\n",
    "\n",
    "idx = 0\n",
    "\n",
    "counter = 0\n",
    "nb_invalid_doc = 0\n",
    "with tqdm(test_loader, unit=\"batch\", total=len(test_loader)) as tepoch:\n",
    "  for batch in tepoch:\n",
    "    tepoch.set_description(\"Eval model\")\n",
    "    data = batch[\"docs\"].to(device)\n",
    "    targets = torch.tensor(batch[\"labels\"], dtype=torch.float).to(device)\n",
    "    y_pred = model(data)\n",
    "\n",
    "    loss = criterion(y_pred, targets)\n",
    "\n",
    "    arr_loss.append(loss.item())\n",
    "\n",
    "    doc = test_set[\"doc_splitted\"].iloc[idx]\n",
    "    summaries = test_set[\"summaries\"].iloc[idx]\n",
    "\n",
    "    probs = np.array(y_pred.tolist()) # compute_probs(y_pred)\n",
    "    probs_tensor = y_pred #torch.tensor(probs, dtype=torch.float).to(device)\n",
    "\n",
    "    #probs = threshold_probs_by_nb(probs=probs, doc_lens=[probs.shape[0]], average_number_of_sentences_per_document=average_number_of_sentences_per_document)\n",
    "    #probs = threshold_probs_by_prop(probs=probs, doc_lens=[probs.shape[0]], average_proportion_of_sentences_per_document=average_proportion_of_sentences_per_document)\n",
    "    indices = torch.argsort(y_pred, descending=True)\n",
    "\n",
    "    y_pred_thresh = []\n",
    "\n",
    "    txt = \"\"\n",
    "\n",
    "    if len(doc) < y_pred.shape[0]:\n",
    "      nb_invalid_doc += 1\n",
    "    else:\n",
    "      for i in range(min(len(doc), y_pred.shape[0])):\n",
    "        txt = txt + \". \" + doc[indices[i]]\n",
    "        y_pred_thresh.append(indices[i])\n",
    "        if len(txt) >= len(summaries):\n",
    "          break\n",
    "\n",
    "      y_pred_thresh.sort()\n",
    "\n",
    "      txt = \"\"\n",
    "\n",
    "      for i in y_pred_thresh:#range(min(len(doc), y_pred.shape[0])):\n",
    "        txt = txt + \". \" + doc[i]\n",
    "\n",
    "      n = min(len(txt), len(summaries))\n",
    "\n",
    "      while n < len(txt) and txt[n].isalnum():\n",
    "        n += 1\n",
    "\n",
    "      txt = txt[:n]\n",
    "\n",
    "      # assert len(txt) - len(summaries) <= 20\n",
    "\n",
    "      scores = scorer.score(summaries, txt)\n",
    "      arr_rouge1.append(scores[\"rouge1\"].recall)\n",
    "      arr_rouge2.append(scores[\"rouge2\"].recall)\n",
    "      arr_rougeL.append(scores[\"rougeL\"].recall)\n",
    "\n",
    "      accuracy.append(accuracy_nb_sent_per_doc_fn(probs=probs, targets=targets.cpu().detach().numpy(), doc_lens=[len(probs)], average_number_of_sentences_per_document=average_number_of_sentences_per_document))\n",
    "      #accuracy.append(accuracy_prop_sent_per_doc_fn(probs=probs, targets=targets.cpu().detach().numpy(), doc_lens=[len(probs)], average_proportion_of_sentences_per_document=average_proportion_of_sentences_per_document))\n",
    "\n",
    "    tepoch.set_postfix(nb_nvalid_doc=nb_invalid_doc, loss=np.mean(arr_loss), rouge1 = np.mean(arr_rouge1), rouge2 = np.mean(arr_rouge2), rougeL = np.mean(arr_rougeL), accuracy=np.mean(accuracy))\n",
    "    idx += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_metrics = {}\n",
    "test_metrics[\"accuracy\"] = np.mean(accuracy)\n",
    "test_metrics[\"rouge1\"]   = np.mean(arr_rouge1)\n",
    "test_metrics[\"rouge2\"]   = np.mean(arr_rouge2)\n",
    "test_metrics[\"rougeL\"]   = np.mean(arr_rougeL)\n",
    "\n",
    "# Save to file in JSON format\n",
    "\n",
    "with open(checkpoints_folder + \"/test_metrics.json\", 'w') as fp:\n",
    "  json.dump(test_metrics, fp)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lead-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Lead-3: 100%|██████████| 100/100 [00:00<00:00, 115.89batch/s, rouge1=0.22, rouge2=0.0693, rougeL=0.149]\n"
     ]
    }
   ],
   "source": [
    "arr_rouge1 = []\n",
    "arr_rouge2 = []\n",
    "arr_rougeL = []\n",
    "mae = []\n",
    "accuracy = []\n",
    "\n",
    "idx = 0\n",
    "with tqdm(test_loader, unit=\"batch\", total=len(test_loader)) as tepoch:\n",
    "    for batch in tepoch:\n",
    "        tepoch.set_description(\"Lead-3\")\n",
    "        doc = test_set[\"doc_splitted\"].iloc[idx]\n",
    "\n",
    "        txt = \"\"\n",
    "\n",
    "        for i in range(min(len(doc), 3)):\n",
    "            txt = txt + doc[i]\n",
    "\n",
    "        summaries = test_set[\"summaries\"].iloc[idx]\n",
    "\n",
    "        n = min(len(txt), len(summaries))\n",
    "\n",
    "        while n < len(txt) and txt[n].isalnum():\n",
    "            n += 1\n",
    "\n",
    "        txt = txt[:n]\n",
    "\n",
    "        # assert len(txt) - len(summaries) <= 20\n",
    "\n",
    "        scores = scorer.score(summaries, txt)\n",
    "        arr_rouge1.append(scores[\"rouge1\"].recall)\n",
    "        arr_rouge2.append(scores[\"rouge2\"].recall)\n",
    "        arr_rougeL.append(scores[\"rougeL\"].recall)\n",
    "\n",
    "        idx += 1\n",
    "\n",
    "        tepoch.set_postfix(rouge1 = np.mean(arr_rouge1), rouge2 = np.mean(arr_rouge2), rougeL = np.mean(arr_rougeL))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First n char"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "First-n-char': 100%|██████████| 100/100 [00:01<00:00, 94.85batch/s, rouge1=0.246, rouge2=0.0751, rougeL=0.158]\n"
     ]
    }
   ],
   "source": [
    "arr_rouge1 = []\n",
    "arr_rouge2 = []\n",
    "arr_rougeL = []\n",
    "mae = []\n",
    "accuracy = []\n",
    "\n",
    "idx = 0\n",
    "with tqdm(test_loader, unit=\"batch\", total=len(test_loader)) as tepoch:\n",
    "    for batch in tepoch:\n",
    "        tepoch.set_description(\"First-n-char'\")\n",
    "        doc = test_set[\"doc_splitted\"].iloc[idx]\n",
    "\n",
    "        txt = \"\"\n",
    "\n",
    "        for i in range(len(doc)):\n",
    "            txt = txt + doc[i]\n",
    "\n",
    "        summaries = test_set[\"summaries\"].iloc[idx]\n",
    "\n",
    "        n = min(len(txt), len(summaries))\n",
    "\n",
    "        while n < len(txt) and txt[n].isalnum():\n",
    "            n += 1\n",
    "\n",
    "        txt = txt[:n]\n",
    "\n",
    "        scores = scorer.score(summaries, txt)\n",
    "        arr_rouge1.append(scores[\"rouge1\"].recall)\n",
    "        arr_rouge2.append(scores[\"rouge2\"].recall)\n",
    "        arr_rougeL.append(scores[\"rougeL\"].recall)\n",
    "\n",
    "        idx += 1\n",
    "\n",
    "        tepoch.set_postfix(rouge1 = np.mean(arr_rouge1), rouge2 = np.mean(arr_rouge2), rougeL = np.mean(arr_rougeL))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "ename": "SystemExit",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[0;31mSystemExit\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/raoufdine/anaconda3/lib/python3.9/site-packages/IPython/core/interactiveshell.py:3465: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "# Does not execute this cell if you want to execute the following cells.\n",
    "import sys\n",
    "sys.exit()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4f810f0bde7d7b5dc90f9e6832822547e13e234c53731a97fc91254323942f40"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}