metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IMDB\n",
    "df_gen = pd.read_json('/content/sst-2_train.json')\n",
    "obj = pd.read_json('/content/imdb_dev.jsonl', lines=True)\n",
    "obj.drop(columns='id')\n",
    "df_gen = df_gen.replace({'label':{0:'generated_0', 1:'generated_1'}})\n",
    "obj = obj.replace({'label':{0:'original_0', 1:'original_1'}})\n",
    "df = pd.concat([df_gen.sample(1000),obj.sample(1000)])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperpartisan Politics\n",
    "df_gen_true = pd.read_json('/content/hyperpartisan_news_true_500_sorted.json')\n",
    "df_gen_false = pd.read_json('/content/hyperpartisan_news_false_500_sorted.json')\n",
    "obj = pd.read_json('train.jsonl', lines=True)\n",
    "obj = obj.iloc[:, ::-1].drop(columns='id')\n",
    "df_gen = pd.concat([df_gen_true, df_gen_false])\n",
    "df_gen = df_gen.replace({'label':{'true':'generated_true', 'false':'generated_false'}})\n",
    "obj = obj.replace({'label':{'true':'original_true', 'false':'original_false'}})\n",
    "df = pd.concat([df_gen,obj])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperpartisan News\n",
    "df_gen_true = pd.read_json('/content/hyperpartisan_news_true_500_sorted (1).json')\n",
    "df_gen_false = pd.read_json('/content/hyperpartisan_news_false_100_sorted_v1.json')\n",
    "obj = pd.read_json('train.jsonl', lines=True)\n",
    "obj = obj.iloc[:, ::-1].drop(columns='id')\n",
    "df_gen = pd.concat([df_gen_true, df_gen_false])\n",
    "df_gen = df_gen.replace({'label':{'true':'generated_true', 'false':'generated_false'}})\n",
    "obj = obj.replace({'label':{'true':'original_true', 'false':'original_false'}})\n",
    "df = pd.concat([df_gen,obj])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RCT\n",
    "df_gen_conclusion = pd.read_json('/content/rct_Conclusions_500.json')\n",
    "df_gen_method =  pd.read_json('/content/rct_METHODS_100.json')\n",
    "df_rct = pd.read_json('/content/dev.jsonl', lines=True)\n",
    "df_rct = df_rct.iloc[:, ::-1].drop(columns='metadata')\n",
    "df_gen_rct = pd.concat([df_gen_conclusion.sample(100), df_gen_method])\n",
    "df_gen_rct = df_gen_rct.replace({'label':{'Conclusions':'generated_Conclusions', 'METHODS':'generated_Methods'}})\n",
    "df_rct = pd.concat([df_rct[df_rct['label'] == 'CONCLUSIONS'].sample(100), df_rct[df_rct['label'] == 'METHODS'].sample(100)])\n",
    "df = df_rct.replace({'label':{'CONCLUSIONS':'original_Conclusions', 'METHODS':'original_Methods'}})\n",
    "df_RCT = pd.concat([df_gen_rct, df_rct])\n",
    "df_RCT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Amazon\n",
    "\n",
    "obj = pd.read_json('/content/amazon_dev.jsonl', lines=True)\n",
    "obj = obj.drop(columns='id')\n",
    "df_gen = pd.read_json('/content/amazon_train.json')\n",
    "df_gen = df_gen.replace({'label':{'Helpful':'generated_helpful', 'Unhelpful':'generated_unhelpful'}})\n",
    "obj = obj.replace({'label':{'helpful':'original_helpful', 'unhelpful':'original_unhelpful'}})\n",
    "df = pd.concat([df_gen,obj.sample(600)])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def featurize(data_df, tokenizer, vocab, max_seq_length=128):\n",
    "    vocab_to_idx = {word: i for i, word in enumerate(vocab)}\n",
    "    text_data = []\n",
    "    label_data = []\n",
    "    for row in tqdm.tqdm_notebook(data_df.itertuples(), total=len(data_df)):\n",
    "        tokenized = tokenizer.tokenize(row.text.lower())\n",
    "        if len(tokenized) < max_seq_length:\n",
    "            # pad each input sequence to the maximum sequence length\n",
    "            tokenized += [PAD_TOKEN] * (max_seq_length - len(tokenized))\n",
    "        tokenized = tokenized[:max_seq_length]\n",
    "        ids = [vocab_to_idx.get(token, 0) for token in tokenized]\n",
    "        text_data.append(ids)\n",
    "        label_data.append(row.label)\n",
    "    return np.array(text_data), np.array(label_data)\n",
    "\n",
    "tokenizer = sacremoses.MosesTokenizer()\n",
    "text, labels = featurize(df, tokenizer, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_embed = []\n",
    "\n",
    "for sentence in text:\n",
    "    sentence_embed = []\n",
    "    for word in sentence:\n",
    "        word_embed = embeddings[word]\n",
    "        sentence_embed.append(word_embed)\n",
    "    text_embed.append(sentence_embed)\n",
    "text_embedding = np.array(text_embed)\n",
    "average = np.mean(text_embedding, axis=1)\n",
    "average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "count_vect = CountVectorizer()\n",
    "tfidf = TfidfVectorizer()\n",
    "\n",
    "X_train_counts = count_vect.fit_transform(x_df.text)\n",
    "X = tfidf.fit_transform(x_df.text)\n",
    "y=x_df.set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: yellowbrick in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (1.3.post1)\n",
      "Requirement already satisfied: cycler>=0.10.0 in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from yellowbrick) (0.10.0)\n",
      "Requirement already satisfied: scipy>=1.0.0 in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from yellowbrick) (1.4.1)\n",
      "Requirement already satisfied: numpy<1.20,>=1.16.0 in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from yellowbrick) (1.18.3)\n",
      "Requirement already satisfied: scikit-learn>=0.20 in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from yellowbrick) (0.22.2.post1)\n",
      "Requirement already satisfied: matplotlib!=3.0.0,>=2.0.2 in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from yellowbrick) (3.2.1)\n",
      "Requirement already satisfied: six in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from cycler>=0.10.0->yellowbrick) (1.14.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (2.8.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (2.4.7)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick) (1.2.0)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\users\\victor cui\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from scikit-learn>=0.20->yellowbrick) (0.14.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install yellowbrick"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'c' argument looks like a single numeric RGB or RGBA sequence, which should be avoided as value-mapping will have precedence in case its length matches with 'x' & 'y'.  Please use a 2-D array with a single row if you really want to specify the same RGB or RGBA value for all points.\n",
      "'c' argument looks like a single numeric RGB or RGBA sequence, which should be avoided as value-mapping will have precedence in case its length matches with 'x' & 'y'.  Please use a 2-D array with a single row if you really want to specify the same RGB or RGBA value for all points.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAckAAAFHCAYAAADKhbfZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU9bk/8M9ZZskkJGSAsAUEgmEpIrWWslO5Li1g9WprF+qtVtuKiq23WvQqLnXfrrVo0Vf1euvyU6y1VXupvS9brbGQC0pdkCUSLRLIRhKyzGTOnO33x3CGmcnMZJazz/N+vfpHmTjrOec53+/3eZ4vo6qqCkIIIYQMwVr9BgghhBC7oiBJCCGEZEBBkhBCCMmAgiQhhBCSAQVJQgghJAMKkoQQQkgGvNVvoJTcfvvt2LFjBwCgubkZEydOhN/vBwBs3rwZfX19uPPOO9Hc3AwA8Pv9+NGPfoTTTz8dALBixQp84QtfwH333Rd/zg8//BA//vGP8de//hUtLS0444wzUF9fP+S1f/vb38Lr9Sb924oVK+DxeOD3+8EwDERRxOLFi3HdddeBZXO/f3ruuefQ39+PH/7wh/l9IcfceOON+Na3voU5c+bghhtuwKpVq7Bo0aKCnitXb7/9NjZs2IBRo0bhmWeeif8OiR566CH09vbipptuAgBEo1HcfvvtaGxsRCAQwGmnnYZ169aBZVl0d3fjZz/7GQ4fPgyWZfHzn/8cp5xyypDnvPDCC3Ho0CGMGDECACBJEpYvX47LL78cFRUVhn5mvR08eBD33nsvNm7caPVbIcQ4KrHEaaedpn7wwQdJ//aDH/xAffLJJ+P//+OPP1ZPPfVUdf/+/fH/Zs6cOeof/vCH+N988MEH6mmnnaaqqqoePHhQnTdvXsHvQRAE9YILLlCffvrpQj5SwdJ9F0a77rrr1EceeSTtY62treq6devUk08+Wb311lvj//7QQw+p3//+99VIJKIqiqLecMMN8e/qqquuUjdt2qSqqqru3r1bXbJkiRoOh4c893e/+131T3/6U/z/R6NR9aabblJ/9KMf6fnxTNHY2KiuWrXK6rdBiKFoJGkjnZ2diEQiUBQFLMti+vTp2LRpEyorK+N/c/XVV+P222/HKaecgkmTJun6+l6vF1/4whfwySefoKWlBWvWrEFdXR0OHTqEp59+Gh988AEefvhhKIqC8vJyXH/99Zg7dy42btyInp4e3HTTTWhvb8fPf/5ztLa2QhRFrFq1CpdddhkA4I033sAvfvELKIqCQCCAW2+9FX/605/Q0dGBa665Bvfeey/uv/9+rFmzBl/5ylfw+uuvZ3y9Q4cOobOzE4cOHcLYsWNx3333oaamJunziKKIu+++G9u2bQPHcZg7dy6uv/56PP/88/jLX/4Cn8+H/v5+rF+/Pum/e/HFFzF//nzU1dWht7c3/u8fffQRVq1aBZ/PBwA4/fTT8cQTT+Bb3/oW3nzzTdx8880AgFmzZmHKlCloaGjAmWeemfU793g8uP7667F48WI0Nzejrq4OmzdvxtNPPw2WZTF69Ghs2LABU6dORSgUwu23346dO3eC4zicfvrpuPrqq3H99dfjxBNPxCWXXAIAuO666+L/f8WKFVi9ejUaGxvR29uLSy+9FDt37sRHH30EnuexadMmjB07NuPv1tLSgosuugjLly/H+++/j76+Plx77bVYsWIFbrzxRrS3t+OSSy7BY489httuuw07d+6Ex+NBbW0t7rrrLpSXlxd3UBJiMVqTtJGf/exneOaZZ7Bw4UKsXbsWjz/+OCZNmoQxY8bE/+aLX/wivvOd7+Caa66BJElDniMSieCcc85J+t+tt96a0+u3t7fjjTfewJe+9CUAQFtbGy6//HL8+c9/Rn9/P26++WZs3LgRr7zyCq666ipcfvnlGBgYSHqOa6+9Fueffz5eeuklvPjii9i6dSu2bNmCI0eO4Nprr8Vdd92FV199FZdccgnuv/9+XH311aipqcH999+Pk08+Of48zc3NWV/vnXfewUMPPYTXXnsNZWVleP7554d8nk2bNqGjowMvv/wyXn75ZSiKgnvvvReXXnopVqxYgYsuumhIgASAK6+8Et/97neHTDnPnTsXW7ZsQSgUQjQaxauvvoqOjg709PRAURQEg8H4344dOxZtbW05fe9+vx9TpkxBU1MTtm3bhscffxxPPfUUXnnlFaxevRpXXHEFVFXFL3/5SwiCgC1btuAPf/gDdu7cie3btw/7/IIg4IUXXsCPf/xj3HTTTfje976HV155BePHj8fvf//7rL8bEJtWXbJkCV588UX89Kc/xZ133gmO43D77bdj8uTJeOKJJ/Dee+9h+/bteOWVV/DSSy9h0qRJ2LdvX06fnxA7o5GkjSxcuBBvvvkm3nvvPbzzzjt444038Mgjj+A3v/kN5s6dG/+7devWYdu2bdi4cWN8vVLj9/vx8ssv5/ya11xzDfx+PxRFgcfjwTe+8Q2cddZZaGlpAc/zmDdvHgCgsbERCxYsiI9eFy5ciGAwiF27dsWfKxwOY8eOHejt7cVDDz0U/7e9e/eC53mceOKJmD17NgDgzDPPzDrKGu715s+fH1/Dmz17dtKIT/PWW2/h6quvhsfjARBbD7ziiity/m5S/eAHP8CDDz6Ib33rW6isrMTKlSvR1NQERVHAMEzS36qqCo7jcn5uhmFQVlaGhoYGrFy5Mh5wzzvvPNxxxx1oaWnB1q1bcf3114PjOHAch2eeeQYA4oEuE+17njRpEkaPHo2ZM2cCACZPnoze3t6sv9vcuXPh8XiwfPlyALHv+ujRo0Neo76+HhzH4Rvf+AaWLFmCs846K+mYJcSpKEjaRFdXFzZu3IgNGzbg1FNPxamnnorLLrsMN9xwA/7whz8kXXB4nscDDzyA8847DyNHjizqde+//36cdNJJaR/zer3g+dghkikQJI5mFUWBqqp4/vnnUVZWBgDo7u6Gz+dDY2Nj0n+vqir27dsXv2CnGu71EhNtGIaBmqYFcepzKIoCURTTvl4uent7cfHFF8dHn6+++iomT56MUaNGQVVVHD16NP57dHR0YOzYsTk97+DgIJqbmzF9+nQ0NjYOeVz73DzPJ32e1tbWeNJV4udP/YyJCVvaDUOibL9bT08PPB5PfFSd+ptoKisr8fLLL2Pnzp1obGzET37yE1xyySVYs2ZNTt8BIXZF0602UVVVha1bt+Kpp56KX/AGBwfx2WefxUdfiSZNmoQbbrgB//mf/2nK+1u4cCHefvttHDx4EACwbds2tLa2Jk2RVlRUYN68eXjyyScBAH19ffj2t7+Nv/zlLzj55JPR3NyMjz/+GADwl7/8Bddeey0AgOO4IVPHubzecJYuXYrnnnsOoihCURQ8++yzWLx4ccHfwV//+lfcdNNNUFUVoVAI//3f/42zzz4bPM/jy1/+Ml544QUAwN69e9Hc3Byfts4mEongzjvvxLJly1BbW4ulS5diy5Yt6O7uBgD87ne/w8iRI3HCCSdg4cKF+P3vfw9FURCNRnHVVVdhx44dqK6ujo+w29vbc5qCTZTtd8uG47h4QH7jjTdw0UUX4fOf/zzWrVuHc889N2mWgRCnopGkTfA8jyeeeAL33Xcfnn76aQQCATAMg3/913/F17/+9bT/zbnnnou3334bO3fujP+btiaZ6u6778asWbMKfn/Tp0/HzTffjCuvvBKyLMPv9+PRRx+NlzJo7r//ftx22204++yzEY1GsXr1anzta1+LP7Z+/XrIsoyKigo8+OCDAIAzzjgD1157LW655Za8Xy+btWvX4p577sG5554LSZIwd+5cbNiwoeDv4Pzzz8f777+P1atXQ5ZlXHDBBfjKV74CALj55ptx4403YvXq1WAYBvfee2/G93rvvfdi06ZNYFkWkiRh0aJFuOGGGwAAixcvxkUXXYTvfe978XXOxx57DCzL4sorr8Qdd9yBc845B7IsY+XKlTjzzDNx0kkn4ZprrsFZZ52F2tpaLFiwIO/Plul3a2lpyfjfTJ8+HT6fD1//+texefNmvPXWW1i9ejUCgQCqqqpw22235f0+CLEbRk03T0VIHu666y7wPB8fGRJCiFvQdCspyrXXXovXXnstPqIihBA3oZEkIYQQkgGNJAkhhJAMKEgSQgghGWTMblUUBaFQCB6PJ2NtFCGEEHOoqgpRFFFeXp7XBgSkOBmDZCgUQlNTk5nvhRBCyDDq6+vzKoUixckYJLXOHPX19UO2WCKEEGKuaDSKpqamtF2TiHEyBkltitXr9cZ3PSCEEGItWv4yF01sE0IIIRlQkCSEEEIyoCBJCCGEZEBBkhBCCMmAgiQhhBCSAQVJQgghJAMKkqRosiIhHO2DrEjD/zEhhDgIbbpMCqaoCva1NqK971MI4iB8njKMrZyKGeMXgGXo/osQ4nwUJEnB9rU2oqV7LxiGBcfykGQRLd17AQCzJiyy+N0RQkjx6HafFERWJLT1fgomZcTIMCzaej+lqVdCiCtQkCQFEaQwotJg2seiUgSCFDb5HRFCiP4oSJKC+PgAfJ6ytI95eT98fMDkd0QIIfqjIElykprByrE8xlZOhaoqSX+nqgrGVU0Fx9JyNyHE+ehKRrLKlsE6Y/wCAEBb76eIShF4eT/GVU2N/zshhDgdBUmS1XAZrLMmLEL9uPkQpDB8fIBGkIQQV6HpVpJRrhmsHMsj4K2kAEkIcR0KkiQjymAlhJQ6CpIkI8pgJYSUOgqSJCPKYCWElDq6ypGsKIOVEFLKKEiSrFiGpQxWQkjJoulWAwiSjNa+MARJtvqt6IYyWAkhpYiueDqSFQWPbm1Cwycd6A4LCAZ8WDqtBpctqgfH0v0IIYQ4DQVJHT26tQlb9hwCxzLweziERQlb9hwCAFyxZKbF744QQki+aHijE0GS0dDcAY5lkv6dYxk0NHfoPvXqxildQgixGxpJ6qQ7LKB7UIDfww15rGdQQHdYwPjK4usKaUqXEELMQ1dVnQQDPgQDvrSPVZdlfixf2pRuWJSSpnQf3dqky/MTQgg5joKkTnw8h6XTaiAratK/y4qKpXU18PFDR5j5MntKlxBCSh0FSR1dtqgeK2dNRIDnIYgyAjyPlbMm4rJF9bo8vzalm442pUsIIUQ/tCapI45lccWSmbh0wYnx9UI9RpAabUo3LEpDHtNzSpcQQkgMjSRzkG8mqY/nML4yoGuA1J7X6CldQgghx9FIMotMmaQXz69Db0TMa6QoSLIuo0tt6rahuQM9gwKqy3xYWlej25QuIYSQ4xhVVdV0DwiCgF27dmHOnDnw+UpzGu+Rt/fGmwMAgKoCn3b3w8dxGFXuy6n8wqiSDb2CLiHEGeiabA0aSWaQLpP0QM8AjgwI4FgG4yrLcuqoY1QXHm1Kl2QnKxI1ZieEFIzWJDNIzSRVVBXdoSgYhoGkqBDl2B6L2covqGTDOoqqYM/hrWho2oyGfS+goWkz9hzeCiVlb0xCCMmGgmQGqc0BRFmBqMQusDzLwMMd/+oylV9QyYZ19rU2oqV7LyRZBMfykGQRLd17sa+10eq3RghxEAqSGaRmkno4Fh6OhaqqCAZ8SaPDTOUXZnXhIclkRUJb76dgmOTDm2FYtPV+ClkZWkJDCCHpUJDMIrE5gCgpGDfCj9HlPkwJlsf/Jlv5hZNKNvRumG5lA3ZBCiMqDaZ9LCpFIEhhk98RIcSpKJMhi9TmAFV+D57c3pxX+YWZJRuFZLzqnX1rhwbsPj4An6cMkiwOeczL++HjKeGJEJIbKgEpQCHByMiSjWICU2qZS+z5VKycNbGg7Fu9n69Qew5vRUv33qQpV1VVUBuciVkTFpn2PqxCWb3uQ9dka9DZU4BCyi+MLNkotMxkuOzbSxecmFdA1/v5ijFj/AIAQFvvp4hKEXh5P8ZVTY3/u1spqoJ9rY1o7/sUgjgIn6cMY0ZMxgmjT0KZp4ICJiF5ojPG4YoJTHrvgWnWnpq5YBkWsyYsQv24+SU1otKyehmGBctwOBpqR9vRT7GvdTuCFeMwtjJ2o8AylI5ASC7oTHG4YspM9M6+tWM2L8fyCHgrSyJApmb1hoQeDIoDAKNClCMQ5SiVwRCSJwqSDldMYNI7+9ZJ2bxulJjVq6oKIlIYDBObYVBUBYoqF1wGIysSwtE+Kp8hJcf9t9c2l29CT+rfa4EpXbJMLoFJ7+xbasBuncSsXkVVoKpKPEiyx6ZfgeNlMAFv5bDPmW6Nk6ZsSSmhIGmRfDNSs/19MYFJ7z0wjd5Tk2TGsTzGVk5FS/desAx7bNpVBVTA5w2AQSxg5lMGk7jGmdi5CEBJZAkTQkHSIvlmpA7398UGJr2zb6kBuzUSs3o9XBlEaRB+bzkqfNUAYtOw46qm5rRGO1znovpx80tirZeUNpovsUC+jc9z/XujNnsmzqFl9S6b8U2cddIlOKl2OarKaiArMjjWg9rgzJzLYKhzESE0krREvqUSRpVW0J6U7sWxPCp8I/G52qWYWWBjAepcRAgFSUtoGalhcWimYLqM1Hz/fjh2aB1HzKOVwRTy32lrnKmdi3KdsiXE6eiKaIF8SyX0Lq3Q1jfDopS0vvno1qbCPhBxrRnjF6A2OBMc6yloypYQp6NbQYvkm5GqV2mFnVrHEfsr1c5FhGjoaLdIvqUSepVW2Kl1HHGOQqdsCXE6mm61WL4ZqcVmsNqxdVwqK/eiJISQRDSSLDHFdugxEiUUEULshq48abi9T+Vli+qxctZEBHgegigjwPNYOWtixvVNs0Z2WkJRvxArOegXRF0TimiESgjJF40kEzi5T2U+NY+5rm+aObITJBlv7W/HZz0hdIcFSLIKnmMQDPjw1v72ohKKaIRKCCkUBckETuxTWUwAGK51XKGbOReiOyzgg7Ye9ISjYBgGLMtAUYHOgQg+VJSiEorM/ByEEHeh2+hjhutTadepV6NqHvNtnVesci+PcFSO71qhYRgGoaiMcm9h93Nmfw5CiLtQkDzGiX0qjQwAxWzmXIhQVELAy0NVkxsmqKqKch+PULSwmxSzPwchxF0oSB6j9alMx659Ko0MAGaXigQDPsydMBI1I/zgGAaKqoJjGNSM8OOkcSMLfj0nlLwQQuyLguQxWp9KVVWS/t3OfSqNDAB6t8LL5fWWTRuLSSPLMa+2GvMmVmNebTUmjSzHsrqxBb+e2Z+DEOIuFCQTOK1PZa4BoNDSh3xLRYqlvV6FxwOoQIXHo8vrmf05CCHuwaipi0DHCIKAXbt2Yc6cOfD5SmtKSi5wayErxLNbM/R01aP0wewttYx6PdoajDhZKV+TrURB0iXSBYBH3t6btrPOylkTqfSBEIeha7I1aLrVIYabMk3t6UqlD4QQUjx7zyWSgpsF0G4fhBBSPBpJ2lyhzQLsVPpAPVMJIU5FI0kbK2aDZDvs9kE9UwkhTkdXKhsrtlmA1aUPRrXMI4QQs9BIUkd6l45oU6ZhcWhLtlymTLPt9mF0OUQxo2BCCLELCpI6MGqLLb2mTBN3+zBrCpQShwghbkBBUgdGbrGlTY1mahaQr3TbRv1xdwt6I1H89Muf0210V+womBBC7ICCZJGG22Krftz8oqZec90gORepU6CqChzoGUB3OIr3Wrrxj5YefHn6WF1GlXZIHCKEkGJR4k6RzNpiK7VZQCFSE4EO9Aygoz8CWVGhAOiNRHVNrLE6cchMVOZCiDvRSLJI2hZbkiwOecxuW2wlToEqqoruUDS+yTHPMvBwrK6JNXqOgu2KylwIcTdXncWyIiEc7YOsFLZBbyGM3GJL79FJ4q4hoqxAVJRj71VFMOCLT4vqvRlx4ijYbSMuKnMhxN1cMZI0Krs0V9pWWm29nyIqReDl/RhXNbXgLbaMHJ1oU51vNreDZQCWAYLlfkwJlsf/xojEGjeOuKjMhRD3c0WQNDK7NBcsw2LWhEWoHzdflzrJdBmoW/YcAoCid+9InAJ94M2P0NDcAQ9/PEgZlVhj5GeyCpW5EOJ+zryFTzBcdqnZU68Bb2XRU6xm7N7h4zmsXzEHqz9Xa3hiTbGfyYpp9FzYqT8uIcQYjh9Jatml6QKTll0a8FZa8M4KY+boxKzEmkI/k9XT6MOhMhdC3M/6K02RtOzSdOyWXZoLK0YnepSXZFPoZ9Km0SVZTJpG39faaMj7LEQplbkQUoocP5LUsku1NUmNHtmlVnDj6KSQz2R0kwa9lEKZCyGlzPqrjA70zi61WiGt6IxuWF6sfD+T06bRE/vjEkLcg1FVVU33gCAI2LVrF+bMmQOfzxkJCHrvwmG1XAKfWaUVegXhXJ9HViQ0NG1O26SBYz1YNuObrviNCcmVE6/JbuCqq4yWXeoWuYxOjC6tSA3CVX4vPl9bjZ8sm4WA15P38+U64jJ7Gt3uI3FCiDVcFSRLQeLFHIDhxexaEGYZoLVvEB+1HcXfmtvx0gef4eL50w1tBmDGNLobmxwQQvRDQdIh0l3M504Yia5wBGXeoT+jHuUiifWNn3YNoHMgAoZhwLEM2voi+OPuFgDGNQPQu0lDOm5sckAI0Q/dKjtEuh6hsYAZTfv3epSLaPWNsqKiOyzEm6EDgKgoUFRV1wYHmejRpCEdsxo3EEKci4KkA2S6mHs4FgwDiFJyc3W9ykW0+kZRViDJyfldHpaFh2N1b4ZuptStwxLZ9XPZtfsQIW5F060OkK1jTTAQK6X44NDRnMtFcqXVN/7xoxbwHAPlWJxUVRXBCh9YhnF0+7XErcNS2e1z2b37ECFuRUHSAbJdzINlPvz0y58DAEOyM7Vg2zEQwcGjIXg5DsEKH06ornB0gwPAWY0brG7iT0ipoltQB0jcBzJR4sXcqNZyWkeZhnVn4SfLZ2Hh1DEYP6IM5R53tF9zQls5OzXxJ6TU0EjSIQrpwqOngNeD/zh9ruvqCZ3QVs5p3YcycVuzD1Ia6Eh1CLtczN3afs3On0tr4p+u+5ATmvjTeipxMjpCHcboHTtIdoIko7UvbGp5iNZ9SFWTs5id0sTfCbu5EJKJvc8uQmzC6s48Tm3i75TdXAjJhI5OQnJgdWceM7oPGcEt66mkdNF0axGsmHoj5rNTZx6jug8ZxcxN0anRAjGCM840HeiRlak9R5Xfgye3N1NTbIPZJZM2WzMHPXrkupkZu7lQYhAxkuuDpB5rSanP0RUSIMgypgZHUFNsA1i9/pfKSZ157Mjo9VRqtECM5PogqcdaUuJzeDgWbf2DkBUVLMNgSrACgL7bU7nNcCPC1MetXv9L5aTOPJlYWaNo5HoqJQYRo7n66BluLSmXgJb6HFqzb5Zl0B2KYnJ1LFgCNPWWargRYbrHF5wwGtv+2WnoHpmFsLqZQ6HsNBVpxKbolBhEjObqIKnHWlLqc3g4Nt7sW1QUiLISv2jT1Fuy4UaE6R5/dXcLDvcO4sQxI4Y8n1E3IbmsfdqlmUO+3D4V6fRGC8T+XB0k9VhLSn0OjmUQDPjQORCJbxcFOGvqzQzDjeIvPHVa2sf9PIdwVIKiHh+ha/S+CZEVBQ837MNf97chLIoYFfAPu/Zp5848qUphKtKMxCBS2lx9BOmxlpTuOaYEy6GqKvweDqKkmD71lm3k45SM0E+6+tM+zrEMyr0cBFFGmff44an3TYisKPjO0w14+9MOSIoKD8ciGIigLxIbkbghAatUpiKd2miBOIOrgySgz1pSuudYu3gGLp5fh96IqGtAyhbksq3xAXBURui0USMyPn7SuGosmlqDbf/sNGz9b2PDXjR80gEVAMswkBUVHf0RAHBNAlapTEU6tdECcQbXH0l6rCVle46A16PL+8yl7CHbGh8AR2WEVvq9GR9fNmMsrlgyE5ctrjdkVCxIMt74uC2WoZzw2gwTS8bqdlgCVqbM1VKbijQiMYgQd50lWeixlmTketRwSS7Z1vje/LgdYOG4jNDhHjfq++4OCwhFpXgCViJRUVDm4RyRgJVL5ipNRRJSnJIJknaWS6lKd1jAkVAE7LFazcS/7QwNAgyDSv/QUa2VZSnDjeKtyhgNBnwYXeFHcCCCzoEImIQEIZ5hsGL6eEdMteaSuUpTkYQUh84WGxguyaVzIILfvf8ZPj7Sj4gog+diGbZTguVgGAZjysvAsEAkTQ9Rq8tSZEWCrIRRUxEAx6YPPGZnjGpTwX2RKIDY9y8pKjiGwZJpNbhy6Yy8n9PshKl8M1dpKpKQwlCQtIHhklx+9/5n+N+mw6gq80CQZCgq0DkQSzKZXF2OL88YCwC26ghjpyL2dBKnertCEQS8PE47cRzWLZ2ZV6JTprVkI5K6EpVK5iohVqMgaQPZklwWTDnegeaE6lgLvO5QFKKqoi8i4oz68UlZn3bpCGP3Ina9pnpT15JDUQmbtu7Df/3ffowq9xmWZVwqmauEWI2CpE1kSmI5Z84kbNlzCH4PB4YBpgQrMLlahSgrUFXgm5+fEr/42qUjjJOK2IuZ6k23lnygZwBHBgRwLINxlWWGZRmXWuYqIVahM8kmMo1sBEkeMhXLMgx8PIcAzw9Zb7RDR5hSmQpMXUtWVBXdoSgYhoGkxG5kOJYzLMuYMlcJMR4FSZtJDXJO3IGiVKYCU9eSRVmBqChgGQb8sSxkjRFZxpS5SojxrM+gIMO6bFE9Vs6aiADPQxBlBHgeK2dNtO0OFNpUoKoqSf+e61SgIMlo7QtDSJOtayfaDYx8rNjSw8V6+aqqimDAl3RTY2SWsZa5SgGSEP3RWeUATtyBopCpQLtttpyL1LXkcSP8iIgypgTL439j51G/xsr9JgmxM0ZVVTXdA4IgYNeuXZgzZw58Pvt3HyH2lM/F95G396adVl45a6LtG45rdZJVfg+e3N6cNsvYjoHe7qU65Di6JluDbhmJoXItYtdjg2wrJa4lO2nUb/dSHUKsRreKxBa0TNF0tKQXJ9GCpp0D5HClOrIytLkFIaWGgiSxBS1TNB2rW+u5lVaqk45WqkNIqaMgSWwhNVNUU2jSi1MyZIslKxLC0b6CRn1aqU462Up1inlNQpyG1iSJbeixQbYTM2QLoUfCTb5deyjJh5QiCpLENvQodRluX0630CvhJp9SHUryIaWIgiSxndxOh90AACAASURBVEJb6zk9QzZXevbGzbVrj5P68RKiJ5ojIa7htgzZTIxIuBmuaw8l+ZBSRUGSuEapZMgWmnDjtNckxA4oSBLX0DtDVi96Z9oW2xvXKa9JiB3QkU1cRY8MWb0YmWlrxTZZtDUXKUXUu5Vk5OSm11ovVSvbwpnRi9aK38jJx4WT0TXZGpYd4UZcxOxwYSyUnS48bqiHs3rzabMybRN745p1DOXaj5cQNzD9aiwrCh5u2Ie/7m9DWBQxKuAvegrKyQXkdgxIqfVwohzFga6PoCgyPle71JL35DRapq3fMzQQ6r0Bsx2PIULcwtQgKSsKvvN0A97+tAOSosLDsQgGIuiLxHawL3QKyskF5HYr0E6thxsQeiCIYSiqjL7BIwADzJqwmC6+w9AybcPi0NZtemfa2u0YIsRNTA2SGxv2ouGTDqgAWIaBrKjo6I8AQMFTUE4uILdjgbZWD8exPAaEHkSiAwADMAwDWRFxsGsPWIaji+8wtEzbdGuSemba2vEYItaRFQXNXQO6PmfdqArbz8gZybSzR5BkvPFxG2RFBZtw0WAYBt2hKLoLnIIyc1pLb4kBKZVWoG322o9WDyfKUQhiGEi499BGKnTxzY0ZmbZ2PIacrHNgEO8d6sa8iUGMqUhfF2pnzV0DmHX3y7o+557rzkH9GPseQ5s3b8Z5550Hj8cz7N8+99xzOHLkCNatW5fz85t2lesOCwhFJfAcg5QyNoiKgjIPV9AUlJnTWnrTApIki0Mes6pAW6uHO9D1ERRVBsPEoqSqqijzlINhWNtffO2SwKVHL9rh2PEYcqLBaBRLNv4Zezv6IckyeI7DzJoReHvdWSjzeq1+eySLxx57DOeee65hz29akAwGfBhd4UdwIILOgUj84gsAPMNgxfTxBV1AzJrWMkK+uzCYZcb4BVCU2BqkrIhgGBZlnnKU+6oB2Pfia9cELiMzbe16DDnNko1/xq7Wo2BZFizLQlFV7Go9iiUb/4x3f3q21W/P9iKRCH72s5+ho6MD48ePx44dO/DEE0/g9ttvBwCMHDkSd955J3bv3o1f//rX8Hg8aGlpwcqVK7F27Vq0trZiw4YNEAQBPp8Pt912G2RZxtq1azFy5EgsW7YMJ598Mh5++OH4691zzz1455130NnZiauvvhq/+tWv8MADD2DHjh1QVRUXXXQRvvrVr+Kdd97BnXfeiaqqKrAsi3nz5uX12Uw7g7Rg1heJAoiNLCVFBccwWDKtBlcunVHwc9upgDxfdizQZhk2lsXKAAe79oBj+fgF2M4XXycncBXDjseQk3QODGJvRz/YlBsplmWxt6MfnQODjpx6NdPmzZtRW1uLX/7yl2hubsbq1auxYcMG3HnnnZg+fTp++9vf4vHHH8eiRYtw+PBhvPLKK4hGo1i6dCnWrl2Le+65BxdeeCGWL1+Obdu24f7778fVV1+Nzs5O/O53v4PX68Wzzz6L++67D2PHjsWjjz6K1157DWvXrsWmTZvw4IMP4m9/+xtaWlrw/PPPQxAEXHDBBVi8eDHuuusuPPDAA5g6dSpuvvnmvD+bqVe6xGDWFYog4OVx2onjsG7pzKLu9M2Y1jJKrrswWCGWxco54uLr5ASuYtn5GHKC9w51Q5LlIUESACRZxnuHunHGjIkWvDPnaG5uxrJlywAAdXV1CAaDaG5uxq233goAEEURU6dOBQDU19eD53nwPA+/3w8AaGpqwmOPPYbHH38cqqrG1xdra2vhPTbdPXbsWNxxxx0IBAJob2/HKaeckvQempqa8NFHH+HCCy8EAEiShMOHD6O9vT3+2qeccgo+++yzvD6bqWeS0cHM6gLyYtixQNtJF18nJ3DpxY7HkBPMmxgEz3FQ0jQf4zkO8yYGLXhXzlJfX49//OMfOP300/HZZ5+hp6cHs2fPxj333IMJEybg3XffRWdnJwAkLbVppk2bhu9///s45ZRT0NzcjB07dgBA0o3LjTfeiNdffx0VFRVYv349tGZxDMNAURRMmzYNX/rSl3DbbbdBURT86le/Qm1tLcaMGYPm5mbU1dXhww8/RFVVVV6fzZIrnpODWSlywsXXyQlcxFpjKsows2ZEfE1SoygKZo8fSVOtOfj617+O6667DmvWrMGECRPg8/lwyy23YP369ZDlWGP/O+64Ax0dHWn/+/Xr1+OWW26BIAiIRCK44YYbhvzNOeecgwsuuACVlZUYPXp0/LlOPfVU/PCHP8RTTz2F7du34zvf+Q7C4TBOP/10VFRU4L777sP69etRXl6O8vLyvIMk9W61iJ3a0LmFGb1SiTs5Ibs1l2uyVXWSO3fuRDgcxpIlS/DPf/4Tl156KV5//XVd34dV6OpsMmohZhwnJ3ARa5V5vXj3p2c7vk6SY1lLahonTZqEf//3f8fDDz8MSZJw0003mf4ejEIjSZPtObw1bbp+bXAmdbHRiV3qJAnRE12TrUFDFxMN10JMVoaup7mF3hsPZ6OteVOAJIQUi6ZbTVSKLcTsWuBPCCG5oCBpolJsIaYV+DMMwDDAQFQsiQJ/Qog7UJA0Uam1EBMkGW81t+Pg0RC6w1GIsnJsezQv3mpud3WBPyFWUFQF/ZEuXZ9zhH9USScVuuuq7ACl1EKsOyzgg9aj6AkLYBgmaXs0SVFLosCfEDP1R7rw+3cf0PU5//ULP0VV2ZiinuOtt95Ca2srvvnNb6Z9/KWXXkJVVRX+5V/+Ja/nXbx4Mf7+978X9d6GQ0HSZE7qYlOsci+PcFQa0mGDYRiEBAnlXuM/N9WjEmI9rWVdJuedd55J7yR/dNWwiBO62OQiWxAKRSUEvByEsJwUKFVVRbmXQygqodJvTKE21aMSYh5RFPEf//EfOHjwIGRZxsUXX4znnnsO1dXV6Ovrw6pVq3DgwAFcc801eOSRR/D6668jGAxicHAQP/7xj7F9+3aMHj0a06ZNS7tLSFNTE+6++24oioK+vj7ceOONQ3q3GoWCJClILkEoGPBh7vhq7G7vje/6wrMMguV+zK6pMrRV3L7WxvjaL8fykGQRLd17AYDqUQnR2ebNm1FdXY377rsPAwMDOO+88+D1evFv//ZvOOOMM/DSSy8BAPbu3YuGhga8+OKLEEURZ589dBuydLuE7N+/H+vXr8eMGTPw6quv4qWXXqIgSewtlyDk4zksqxuLgaiEydXl8cQdAFg2faxhSTvD1aPWj5tPU6+E6Ki5uRmLFsXO+4qKCtTV1eHvf/97fPeNxL876aSTwHEcOI7DnDlzhjxXul1Campq8Ktf/Qp+vx+hUAgVFRXGf6hjaN7JIrIiIRztc2QDgXyaIly2qB4rZ03ECK8HDIARXg9WzppoaKs4rR41Ha0e1bjXNq9pAiF2UVdXh3feeQcAMDAwgKamJtTW1g7JR5g+fTo+/PBDKIqCaDSK3bt3D3mudLuE3HHHHbjqqqtwzz33oL6+HhkaxRmCbqdN5oa1snyaIlix16cV9ail2DSBkqKI5oILLsCGDRvw7W9/G4Ig4Morr4xPsSaaMWMGli9fjgsuuADV1dXweDzg+eGPna997Wu4/PLLMWrUKIwbNw49PT1GfIy0qHerydzQu1VWJDQ0bU4bhDjWg2Uzvmn6RTP1gm3291xKO5C44UbPiXK5Jtu9TrKrqwuvvfYa1qxZg2g0ilWrVuE3v/kNJkyYoMvzG4Fu/0xk9VqZXnf+dmqKkOmCfeK4+QDMqUcVJBkNzR1JARIAOJZBQ3OH65omUFKUfbEMW3RNo5Gqq6uxa9cunH/++WAYBt/4xjdsHSABCpKmyjZNKUiDOBruwMhAje5Bxog7f7s0RRjugm1GPWp3WED3oAC/Z2gg7BkUXNU0weobPeJsLMvirrvusvpt5IWOZhNlWisbEHoQlSLY8cn/wO8N6D51ZcSdvx2aIuR6wTa6HjUY8CEY8CEsDk3Cqi7zGVrqYrZSbNJPShstIJhIm6ZUVSX+bwNCDwaFfng5H3jOEw9g+1obdXlNo7fn0oKQFaMHK7NYE/l4Dkun1UBWkpf3ZUXF0roaV021ajd66bi1ST8pbRQkTTZj/ALUBmeCYz2QFAlRKYIybwXKfdXxv9Fzf0m7BBIj2OmCrZW6BHgegigjwPOGl7pYId2NHuDeJv2E0BFtssRpyqPhDuz45H/Ac54hf6fX1JXdt+cqJpnITglEVpS65EKQZN3fj13WowkxAwVJi3Asj5GBGvi9AUMDmJ0CSSK9konsdsH28ZwtknSMrNu0w3o0IWahIzsLo4ulzQpgdgskgH7JRHTBTk/b7JpjGfg9HMKipPtm125p0k9INnQ1SSN1lOPl/QiWT8DsiUvg4fTdtcKMAGZVIMk01WdEGQFdsI8rtbpNQoxEQTKNxFHOoNiPo+F2tB5txoGuXThx7Km6lmeYGcDMCiTDTfU5sYzAiLU9o5RS3SYhRqMgmSJxlDMg9CASHQCYWBFsONqHg12xhrx6dxZx00houKk+uycTJXJiT9ZSqtskxGj2PMstpI1yVKgQxDCQMGOlqgpUQLfyDDcabqpPkGRHlRFoAT8sSkkB/9GtTYa8nh67iJRS3SYhRrPP1cgmtFGOIA1CUeWkbVsYhgXLsKZMCTp1h4Vcp/rsmEyUysy1Pb1HrFp9ZkNzB3oGBVSX+bC0rsZ1dZtGcOq5R4xBR0AKbZRzsHsPWIaDithoR1VVlHnKwTAsvJzHsClBp++wkOtUnxOyUs1c29M7G9WudZt25vRzjxiDfvk0ZoxfgEnBWSjzjoCqKAAYlHliXXGMnhLUkoYkWUwqjdCrTV02emwEne9Un5Vt7YajBfx09Fzby2WKulBa3SYFyOFZee4R+7LflckGtFHO9LGnYveht9E1cAiiHAXPeQ2dErRqhwW976AvW1QPUVbwxsdtCEcljCr3O3KqTwv46faJ1HNtL5cRazDgoxGhgWh3E5IJ/epZeDgvTp68wrQ1CqtKI/TcJURbW2s8cARhSULA58HCKWNsnQ2ajRlre9mmqEf6vXjhHwew7UCnY7JrC2H1OqATy5KIOQw9Gq0+8PViVnmGFaURet9BJ66tlXl4qFDxv02HwXOMbp1ezGTG2l62ESt44H+bDhvaOcdKdlkHdFJZEjGXIUehoirYc3grGpo2o2HfC2ho2ow9h7dCSUn5J8nMLo2QFQlHwx2xUpc08t0lJNe1NT3WPs1m9Npeul1EzqgfDwCGrFXahV3WAZ1UlkTMZcgvb8Qmv6XCqNKIxFE9w7Dxu/eIGEZf5Ai8nA/lvuqkkpd876CHW1s7EoqgL/S+5aMGO0o3Yu0OC9iy55BrO+fYbR3QCWVJxHy6H4F2O/CdRu/SiHTTWaoKiFIELMuBZz3w8n4MCv0AgAp/EEBhd9DDlX8c6X0PHX376OYpi8RdRNzeOcdu64BOKEsi5tP99t3Nm/yaSa/SiNTpLFGOor33U4SjvfG/qfBVo8w3AlFZOPZ3HtQGZ+Z9B52t/GPxtNHoDv0z482Tk6ZezeL2zjl22jQ7kZ3Lkoj5dD8KaAHcPtKN6hVVhgoFESmMcnVk/LEKXzUkj4QvTl2JkYGagi8QmbJB/+3U8dj68VZbjBqc1KzczZ1z7LrXKSGJdD8K6cC3j3TTWSzDgWU4KKoMRVXAJfxGPr6sqAAJZM4GlRXJ8psnJzYrd3vnHFoHJHZnSMSiA98e0o3qGTDweQKIRENJyTJ638Qkrq0B5tw8DTdCNGMjYqOkfp9uQeuAxO4MORrpwLeHTIGp3FuFqrIaADD0JiY1aBmXuTv8CJE2IrY3N20VR9zF0MhFB771sgUmVVUMuYnJFrSMuHnKZYRIGxETQgpBwzub0qtbUdZRPcMachMzXNDS8+Yp1xGik8op9E4sckvnK0KsQGeMzRjVpsusUb3Z05q5jhDNalZeDL0Ti+zS8o3E0M2KM9EvZTNO71Zk9rRmPiNEu5dT6J1Y5PRjyS3oZsXZKEjaiBu6FZk9rZk4QmQYQJQVeDgWqoohI0Q7l1PoPQJ3w7HkFnSz4mx0G2MjbuhWZEWXmB8smI4KL48PDx/Fuwe78eHho6jw8vjBgukZ36PdNiLWRuDpaCPwfBh9LDmxSb0VhrtZoe/P/uhW0kbc0q3I7GnNXzfux0BUwpzxI+MjyYGohF837rd9/aNG7xG4UceSXaYOnbK+Z7f+tCR/9j26SpBbuhUlTmu29g0CUDG+MmBIV5vUaUqOPT46tKr+sZDsVL0Ti4w6lqyeOrRLkM6VW258S5kzrrp5cspdZjpu6VYkKwoeb/zY8BZwdqp/LDY7Ve8RuN7Hkh3WOa0O0vlyy41vKXPVL+S0u8x03NKtyKwWcNo05UBUjE+1ssf2xDS7/rHYz6x3YpHex5LVU4d2CNKFcMuNb6my3xFVBKfdZWZTbF2jlaNpM2sleZaBqqr4R0s3JEWFh2MRDHhRW1WOpTPMq3/U8zPr3adVrxpZq6cOrQ7ShXLLjW+pcsbwKgf5ZpG5NTtPURXsObwVDU2b0bDvBTQ0bcaew1uhqIpp70HvTM1sHt3ahH5BRDDgA88yEGUFRwYEjPDxptY/mvmZraJNHaopx5JZU4d23X8yV7RPpTO55tfK9S7TDVOy2dhhNG1WraQ2euM5FlNHVWBydXl8ypUBA0lRwZn0kzqp7V0xrJw6pPU9YgXXHFW5TgXZIYgYxS5rNma1gEtN2uFYJp7danbSjhPa3unB6qlDI4O0kxP+iHFccyTkcpdplyBiFDut2ZhRK2m30Zvd297pyaodfowI0m6fXSLFcW5ESGO4u0w7BREjWJ1YkciMFnB2G73Zue2dnRUygtMzSLt5dokUz1VBcri7TDsFESPYcc1G70zNVHYcvRn9md1CjxFcsVOkbp9dIsVz5a+f6S7TjkFEb2YmVui9hlNIpxq3jd5KZV1MViTsankL7b2fgGX5vEdwek2Run12iRTPvWdhBnYr7NX7omhGYoXeazh67KPo9NFbqayLaZ+zrfeTYyM4Bn4+gHJfNRiGyXkEp9cUaaGzS6VyM0NKMEhanZ2nMfqiaGRihd5rOGZ157GzUlkX0z6nAgVAbKeYQXEAAFDhDwIYfgSn5xRpvrNLpXIzQ44z7Fe1e7G+1YW92sVCksWki+K+1kZL3k+u9N76Z7hONYIkF/2e7a5UtlNK/Jwsw4FlYtPiDMMgIoXjTQqGyw/QexuwGeMXoDY4ExzrgazI4FgPaoMz084uOfW8JYXTPULQndbwnJwsoPcajp0alFulVNbFEj8nAwY+TwCR6ADAxEZuiqqABYbND9A7AS/X2SUnn7ekcLpHLbrTGp6TN1cutjWYIMlo7QvHR4harWM6bupUk02m71RVFbAsB571WvCu9CUrEmRFgpf3x/+twlcNv7cCDFgADDycL+MILpFR7fGGm11y8nlLCqfrbQ/daeXGyaUohWQIy4qE/kgvnvi/A9h+sA89YTEpOcdOtY6FZNgWK/U7VVUVIaEHETEML1+Grft/59jZmNSZJUEKQ1YkjDi2/ljhq0bAW4mxlVMxp3ZZztcHKxLwnHzeksLpGrFKZdqoWE4vRcn1AqWoCvYe3ob9He/i97sUvN9WBg/Ho9w3EuFodTw5x8paRy0oVvk9eHJ7s+H7X2aS+J12h9ogSoPwe8tR4at2dBJPakJSmWcEBoRuhKN98PHl8PJ+TKyqT7oByCVz1IoEPKeft6Qwuv6qdKcVk8tJbpdSlEJS2XO9QO1rbcS+tkb0R0LYd2QMWCb2egORHgBAhb86vo2U2bWOqWUnXSEBgixjanCEJRm22ndaV3MK3tr3PFSoYHB8ZG3kbIxR5QzpZpYYhsEI/yiwDIcvTluNgHdE/DULyWcwuz2eXc5bYh5dz7ZSv9PK5yQ36044lmXcD0BNWm/RI8Eq2wVKViS0Hm2GIA0iJPIIRTl4ODX+2oIYRrlvZFJyjpm1jollJx6ORVv/IGRFBcswmBKsOPb59N//cjiSEoWsSKbMxhidZJdtZkmUo+BYLukxJ5TB2KWEjJhH91+3lO+0CjnJjboTTpzqDEV7wagMAr5KTK/5AmZOWDjsey12dCFIYUTEEFRVQYVXRYVXhiBrF14VsipDUWVUl/nTJucYWaydWnYiygokWQXLMugORTG5OhYsASt2EzFvNsbooJTPZ3FaPoNVDd6J+XQ/6kr1TsuskzzX4BGb6vw/CGIIDAOAURESjmJfWyNUKOjo+yzDe/0Eiiqjs/+zokYXPj6AMm85+iIsPJyK+jERvN8aQCwuMeAYDqrKDknOMaOEKLXsxMOx4DkGigqIigJRVuLvyewMW7NmY8w4XvP5LJTPQOzKsOhVandaRp/k+QSP2AXwk1i6ekKNPsMwEKRBHO75GLIqg2c9Q17naKgdghgGz3mLGl1wLI9xVXU4Gu5ARAzhjLpYV5V9nX4MSh5UlZVj1ezaIck5Zky5pW6xxbEMggEfOgci8LAsPJyWQGJNhq0ZszFmBaVcPwvlMxC7cv8QzyRGn+SpwUOUozjQ9REURcbnapcm/a0ghTEoDkBRZTBMcicbVVUQlaLwefxIpUKFqERRkXLhLHR0MWP8Aqiqio/b30U42oszp4eweiaH0SNmY/60RSjzJAdps0bj6bbYmhIsh6qq8Hs4iJJi6W4iZszGmBWUcv0spZ7PQOyLjjydGHmSpwaPAaEHghiGosroGzwCMMCsCYvjI0ofH0CZpwL9TBdUJBdcMwyLMm85xlZOweGjHye911jCiGdIkAIKG12wDIvZExdjxvgvIRztA8AkZTOmMnPKLV3ZydrFM3Dx/Dr0RkRb7CZi5GyM3sfrcMsAuXyWUs5nIPZFQVJHRp3kicFjQOiJt/JiGAayIuJg1x6wDBefjoxNdU7D0XAHBDEUn3KNjZQCGD+yLjZNy3JJ73V8VR06+g+k7RVazOiCY/l48Xg2Zk65ZdtiK+AdOg3tRnocr3quIZuZz0C7eJBc0dGhI6NOci14iHIUghhOWWeMTb+mTkdqU537O95FSOhDrARkJE4c+4X4BSzde2UPc6aNLlJZMeXm9C22iqHH8WrEGrKRI2jqLU3yRUHSAHqf5FrwOND1UdI6o6qqKPOUg2HYIdORyVOdQ+skM71Xq0cXNOVmvkKPV6eVbQDOqMUk9mKvI5hkNGP8AihKbA1SVsTY2qKnHOW+agCZpyNjU53VOb+O1aOLXF8/tceq2T1XrejxajdOK9twYlAn1qMjwkB6rnuwDBvLYmWAg117YtsNHTvZjZiOtHp0ken1U9vJVZcd2yGDAXrCUcN7rqa+vtk9Xu3EaWUbTgvqxB5cHSStWpw3ct0jlsXK2XY60ugLUWI7Ob+Hw+72XnT0D2JsZRmmBCsM77ma+vpm93i1EzPXkPU4l50W1Ik9uDJIWr04b+S6h907Ghl5IUptJycrKrrDAliWTWonZ1TP1dTX11jR49UujF5D1vNcplpMUghXHhVWLs6bte5h145GRl6IUtvJJfZcTW0nZ0TP1dTXT2R2j1e7MPqmTe9zmRLDSL5cFyStXpwvpXWPTFNgRl2IUtvJJfZcTWwnBxjTczX19ROZ3ePVboy4aTPiXLb7TAyxH9cdHVYHqVJY9xhuCsy4etHkdnJaz9WO/kGMqSyL79xhVM/VdO3sjHy9UmfkuWzXmRhiP64LklYHqVJY98h1CkzvC5EgyThnziSIsoLGfx5Bz6CA2TVVmD22CgBwdDBqeM/VdO3srOrx6nZWn8uEAC4MknYIUm5e97BiOjtd2cXCKWNw/smTMabCb2qdZLZ2dk7hlJZsdjiXCXHlUWZ1kHLzuocV09kbG/bi1d0t8PNcvOzif5sOg+eYeNmF2e3lnNjOzuqs70JYfS4T4o4rdwq7BCk3rnuYOQUmKwoebtiHB/+2BxFRBs/F1iCnBMtLuuyiUE5syWaXc5mULnvePupEC1J0UulHmwJT1eQtuIyYAnt0axP+uPsgIpIMlo1lsXYORLD/SD96B6PoHBhEd1jQ7fXcbLhp8nQ7v9gJncvEKqYGSVmREI722f6EJNnNGL8AtcGZ4FgPZEUGx3pQG5yp6xSYVrjv83Dx0g5FUdAbEbG/sx+NB46g8bMurHtpO6KSO44nI88PbZo8HW2anBAylCm3ZU5cC7ELOyZZmDEFlli4Hwx40dEfQZ8gISorYBDbLYznGDQ0t+PCZ/+Ozd9bruvrm8mM84MyRQkpjClXXSeuhVjNCTcWRq65Jhbun1BdAUlRcCQkxAIkE0ucCXhih2/jgSPoi0RR6fca8l6MZsb5QZmihBTG8Kut09dCrKJdOCVZTLpw7mttNOT17DYVrhXuy4oKhgHGlPvBsQx4lkHAwyPgPX5RD0clfNLVb+G7LZyZ54cZ0+SEuI3ht49Wd8BxIjNrEe08Yk0s3A8xIng21oZOG0FqAl4e00aNsOItFs3M84MyRQnJn+FXQW0tJB1aC0nPzCQLs0es+dAK93+zZjGeuXApvjprAvxc8iGrKAoWnDDa1lOt2UbpVpwflClKSO4MD5Jmlgy4hVkXTiunwvsiUbx3qAt9keiwf6sV7v+/7y7Bl6ePg5djIckKvByLL08fh6fXLDbsfRZDURXsObwVDU2b0bDvBTQ0bcaew1uhJJwLdH4QYm+mnIHUNSM/ZiVZWDEVHpUkXPjs39F44AhCUQnlXh4LThiNp9cshpfP/rm8PI/N31uOvkgUn3T1Y9qoEbYeQeaakEPnByH2ZUqQpLWQ/Jlx4bSiLODCZ/+ON/e3gT22tVVUVvDm/ra8yjgq/V7MmzhK1/eld+/XfNaV6fwgxL5MPRPd2KbNKGZcOM0uC+iLRNF44AhYNjlwsCxrWRlHuubpS6fFdvXg2MJXIwoZpdP5QYj92KPgjmRkdJKFmWUBn3T1IxRNv85pVRnHo1ubsGXPIYRFKd48fcueQ3h0a1NRz0sJa4S4A83plDgzp/qmCuLZTQAACABJREFUjRqBci+PqKwMecyKMg6t9V3iBsoAdGmeTsX7hLgDjSQJAHPKAir9Xiw4YTQUJTlIWlXGobW+S6dnUCi6eToV7xPifHQ7S0z19JrF8ezWcFRCwMtjwbQaS8o4Elvfpaouiz1WDErIIcT56IwlprJTGYfW+m7LnkNJU66yomJpXY1u+1RSQg4hzkVB0kXsuGNIJkaUcRQisfVdz6CA6jIfltbVxP+dEFLa7H0lJTmxc/9Vu9Na31264ERd6yQJIe5AQdIFaCuy4mmt7wghJBENM7IQJBmtfWEIkmz1W8mItiIjhBDj0EgyDaO6sBiBtiIjVnPSWjgh+aIjOg2tCwvHMkldWADgiiUzLX53yazov0oIQGvhpDTQkZxiuC4sdpt6pa2WiFXsvBcpIXqhIJnC6C4sRqDOLsRstBZOSgUNM1IY3YXFCHbs7KL31lPEXmgtnJQKCpIpzOrCYgQ7dHZxUtJTKSWcZPushXwPdlgLL6Xfj1iHjqw0qAtL4ZyQ9FRKCSfZPiuAgr8HK3c5KaXfj1iPgmQa1IWlMIlJT7KiQpQVeDhWl62n9FRKzReyfVYARX0PWqBt6/0UUSkCL+/HuKqphq+Fl9LvR6xHQTIL6sKSn+6wgK5wBG39EXSHBUiyCp5jEAz4MH6EH91hwfLvc7iEk/px810zdZfts7YebQbDMEV9D1ashZfS70fsgeYmiG6CAR+6w1F0DkSgqADLMlBUoHMggq5w1BZJT1rCSTpawolbZPusETGMQXEg7WP5fg9m7EWqKaXfj9gDBUmiK4bJ79/NpiWcpOO25gvZPqvfE0CZpyLtY3b+Hkrp9yP2QEGS6EZbv60Z4QfHMFBUFRzDoGaE/9go0/oa01JqvpDts44fWYdxVdMc9z2U0u9H7KGkjyiq5dOXVmPq93CYXH08cYdlGAR43hbTrYB1CSdWyOWzOu17KKXfj1iPUVVVTfeAIAjYtWsX5syZA5/PHhc3vTipls9pHnl7b9oa05WzJtqmBERTSnV2etdJ2oFT33eh3HxNtjP3H1lpJNbyeTgW3WEBf/yoBYB9avmcykk1pnZovmCWbJ/Vqd+DU983cZaSC5JaLR/LAJ92DSSVKnQMRHDx/DoEvB6r36ZjUY0pIcRNSm5uUWtg/s/u0JBShYNHQ/jFW3usfouuoNWYUoAkhDhZyQXJYMCHKr8X3WEBTEpdgpfj8I9DPbbbDovkTpBktPaF6TckhOii5KZbfTyHz9dW42/N7UnJJaqqIljhQ99g1BadYUh+KBmLEGKEkrx6/GTZLEyuDgyp5TuhusK222GR7LRkrLAoJTVWf3Rrk9VvjRDiYCUZJANeDy6ePx0nTRiJeROrMa+2GlOCFVBU+2+HBdCUYqrExuqJtMbq9D0RQgpVctOtGieVKmhoSjE9LRnL7xl6c9MzKND0OSGkYCUbJJ1YqrCxYS9e3d0CP8/Zdq9GK2idfsKiNOQxmj5Pr9QK8QkpVMmfHU7YDktWFDzcsA8P/m0PIqIc335qSrDcdns1WsHHc1g6rSZtpx8nTJ+biTYsJiQ/dFY4wKNbm/DH3QcRkeSk7af+2R0CcHxKsZRdtqgeK2dNRIDnIYgyAjyPlbMm2nr63ArahsWSLCZtWLyvtRGyIiEc7YOsDB2RE1KqSn4kaXdaUorPw8HDsZCVWKtdhmHQHRYwubqcphThzOlzs2XasBhg0NT2Dtp6P0FUitDokpAEdAbYnJaUwjIMggEvEvvRS4qKiCSX9JRiaqYvdfrJLNOGxSGhByHhKKJyZMjokpBSRyNJm0tMSjmhOrZJbncoClFR4Oc5nD27tiSnFCnTN3/ahsWSLMb/TVUVRKQwOJYDyxy/sWAYFm29n6J+3HxK7CElja4mNqclpciKCoYBpgQrMK+2GnPHj8S/L5+NnyyfXZJB4dGtTfjjRy3oDgvwcCw1D8hBug2LFVWBqsjweQJgkFxnGpUiEKSw2W+TEFuhW0QHSFfT+dWZpZuUEo6K+K//24+2/sH4Di5atm8pZPoWU76RumGxh/Oh3F+ddsspL++Hj7d35jchRqMg6QCUlJLsF2/twcGjIXAsm5TtCwATKstc2zxAj/INlmExa8Ii1I+bHw+0TW3b0dK9NymhR1UVjKuaSlOtpOSV3jydg1FSSixR5x+HeuBN+Q60bN9Kn9e1mb7ZyjfypW1YzLE8ZoxfgNrgTHCsB7Iig2M9qA3OjI86CSlldJtIHKU7LKB3MIpgwIuO/kjSdmeirOLzk6pdeRORqXxDjwSbdKNLGkESEkNnAnEULdtXC4Rapq+HZTGxugw/WTbL4ndoDK18I13w0hJs0q0r5kMbXRJCjqMgSRwlsQXdlGAFJlerEGUFLMNg9exaBLweq9+iIdKVb2gowYYQ49CaJHGcxBZ0oqSg2u/DapfXi6Yr3wAowYYQo9GZRRynVLN9U8s3vLwf46qmUoINIQaiIEkcywk7uOiJEmwIMR+dYYQ4DCXYEGIeWpMkhBBCMqAgSQghhGRAQZIQQgjJgIIkIYQQkgEFSUIIISQDCpKEEEJIBhQkCSGEkAwy1kmqqgoAiEajpr0ZQggh6WnXYu3aTMyRMUiKYqyRclNTk2lvhhBCSHaiKMLv91v9NkoGo2a4LVEUBaFQCB6PJ2nPPkIIIeZTVRWiKKK8vBwsSytlZskYJAkhhJBSR7cjhBBCSAYUJAkhhJAMKEgSQgghGVCQJIQQQjL4/6aBJjD0FD4xAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a3adb735c0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from yellowbrick.text import TSNEVisualizer\n",
    "\n",
    "tsne = TSNEVisualizer()\n",
    "tsne.fit(average, labels)\n",
    "tsne.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.translate.bleu_score import SmoothingFunction, corpus_bleu, sentence_bleu\n",
    "\n",
    "\n",
    "def bleu(ref, gen):\n",
    "    ''' \n",
    "    calculate pair wise bleu score. uses nltk implementation\n",
    "    Args:\n",
    "        references : a list of reference sentences \n",
    "        candidates : a list of candidate(generated) sentences\n",
    "    Returns:\n",
    "        bleu score(float)\n",
    "    '''\n",
    "    ref_bleu = []\n",
    "    gen_bleu = []\n",
    "    for l in gen:\n",
    "        gen_bleu.append(l.split())\n",
    "    for i,l in enumerate(ref):\n",
    "        ref_bleu.append([l.split()])\n",
    "    cc = SmoothingFunction()\n",
    "    score_bleu = corpus_bleu(ref_bleu, gen_bleu, weights=(0, 1, 0, 0), smoothing_function=cc.method4)\n",
    "    return score_bleu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0017888189081306362"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bleu(Methods_df.text,gen_df.text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}