K-fold_on_test_set.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OK\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "SCRIPT PER PROVARE un classificatore con kfold sul test set\n",
    "'''\n",
    "\n",
    "import time\n",
    "t_orig = time.process_time()\n",
    "last_time = 0\n",
    "\n",
    "#import codecs\n",
    "#import csv\n",
    "#import argparse\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "#from time import time\n",
    "from dill import dill\n",
    "from sklearn.metrics import f1_score, classification_report, accuracy_score, recall_score, precision_score\n",
    "#from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import KFold\n",
    "#from sklearn.ensemble import RandomForestClassifier\n",
    "#from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer\n",
    "from sklearn.feature_selection import chi2, SelectKBest\n",
    "from sklearn import svm #cross_validation,\n",
    "from sklearn.svm import LinearSVC\n",
    "#from sklearn.svm import SVC\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "#from sklearn.tree.tree import DecisionTreeClassifier\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "import sqlite3\n",
    "import pandas as pd\n",
    "\n",
    "# modulo per confrontare il risultato delle regex\n",
    "# richiede entrambi i db già etichettati\n",
    "import tortellino_parmigiano \n",
    "import crea_db\n",
    "\n",
    "\n",
    "import evaluation_report # stampa confusion_matrix e altro\n",
    "from text_tokenizer import get_tokenizer \n",
    "from nltk.stem.snowball import ItalianStemmer\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(\"ignore\")\n",
    "print('OK')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time to load DB:  3.041657495\n",
      "Time to join conversations:  7.623455488\n",
      "Time to create validation set:  0.011302338000000134\n",
      "Items in the training set: (344, 11)\n",
      "Items in the validation set: (60, 11)\n",
      "ok\n"
     ]
    }
   ],
   "source": [
    "#import timeit\n",
    "# decorator to time the function\n",
    "# this function should be assigned to an object with parameters\n",
    "# and then passed to timeit\n",
    "#def wrapper(func, *args, **kwargs):\n",
    "#    def wrapped():\n",
    "#        return func(*args, **kwargs)\n",
    "#    return wrapped\n",
    "\n",
    "\n",
    "DATA_FOLDER = \"/home/backup/dati_e_csv/\" # DATI\n",
    "DB_NAME = 'test_manina.sqlite'\n",
    "\n",
    "# Load sqlite db training set annotato a mano\n",
    "\n",
    "con = sqlite3.connect(os.path.join(DATA_FOLDER, DB_NAME))\n",
    "training_pd = pd.read_sql_query(\"SELECT * from test\", con)\n",
    "\n",
    "ita_stopwords = set(stopwords.words('italian'))\n",
    "ita_stemmer = ItalianStemmer()\n",
    "ita_stemming = ita_stemmer.stem\n",
    "\n",
    "print('Time to load DB: ', time.process_time()-last_time)\n",
    "last_time = time.process_time()\n",
    "\n",
    "def get_tokenizer(doc): #(word_ngrams=None, char_ngrams=None, stopwords=None, additional_extractors=None):\n",
    "    doc = filter(lambda x: x not in ita_stopwords, word_tokenize(doc))\n",
    "    #doc = map(ita_stemming, doc)\n",
    "    #doc = [ita_stemmer.stem(a) for a in doc]\n",
    "    temp = ''\n",
    "    for par in doc:\n",
    "        temp = '{} {}'.format(temp, ita_stemming(par))\n",
    "    return temp\n",
    "#building data and trying some features selection\n",
    "\n",
    "def unisci_canali(o,c):\n",
    "    o = o.replace('O:', '')\n",
    "    c = c.replace('C:', '')\n",
    "    return '{} {}'.format(o,c)\n",
    "\n",
    "\n",
    "training_corretto = pd.DataFrame()\n",
    "# Unisce i canali, tokenizza, fa stemming, e mette la conversazione in un dataframe\n",
    "training_corretto['trascritto'] = pd.Series(map(get_tokenizer, [ unisci_canali(\n",
    "            row['testo_operatore'],row['testo_cliente']) for _,row in training_pd.iterrows() ]))\n",
    "\n",
    "print('Time to join conversations: ', time.process_time()-last_time)\n",
    "last_time = time.process_time()\n",
    "\n",
    "features = ['attivazione', 'canone_rai','cessazione','contratto','credito','domiciliazione','fatturazione',\n",
    "           'gr_invito_a_chiamare','non_di_competenza','richiamate']\n",
    "# aggiunge le colonne etichette al dataframe\n",
    "training_corretto[features] = training_pd[features]\n",
    "\n",
    "# Set seed\n",
    "generatore_random = np.random.RandomState(seed=5)\n",
    "#print('Rand seed:', generatore_random.seed)\n",
    "\n",
    "# separa il validation set\n",
    "msk = generatore_random.rand(len(training_corretto)) < 0.85 # soglia dello split: il 15% va nel validation\n",
    "validation_set = training_corretto[~msk]\n",
    "training_set = training_corretto[msk]\n",
    "\n",
    "print('Time to create validation set: ', time.process_time()-last_time)\n",
    "last_time = time.process_time()\n",
    "\n",
    "print('Items in the training set:', training_set.shape)\n",
    "# il validation set va usato solo alla fine\n",
    "print('Items in the validation set:', validation_set.shape)\n",
    "print('ok')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>attivazione</th>\n",
       "      <th>canone_rai</th>\n",
       "      <th>cessazione</th>\n",
       "      <th>contratto</th>\n",
       "      <th>credito</th>\n",
       "      <th>domiciliazione</th>\n",
       "      <th>fatturazione</th>\n",
       "      <th>gr_invito_a_chiamare</th>\n",
       "      <th>non_di_competenza</th>\n",
       "      <th>richiamate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    attivazione  canone_rai  cessazione  contratto  credito  domiciliazione  \\\n",
       "0             0           1           0          1        0               0   \n",
       "1             0           0           0          0        1               0   \n",
       "2             1           0           0          0        0               0   \n",
       "3             0           0           0          0        0               0   \n",
       "4             0           0           0          0        0               0   \n",
       "6             0           0           0          0        1               0   \n",
       "7             0           0           0          0        0               0   \n",
       "8             0           0           0          0        1               0   \n",
       "9             0           0           0          0        1               0   \n",
       "10            0           0           0          1        1               0   \n",
       "\n",
       "    fatturazione  gr_invito_a_chiamare  non_di_competenza  richiamate  \n",
       "0              1                     0                  0           0  \n",
       "1              1                     0                  0           0  \n",
       "2              1                     1                  0           0  \n",
       "3              1                     0                  0           0  \n",
       "4              0                     0                  1           0  \n",
       "6              0                     0                  0           0  \n",
       "7              0                     1                  0           0  \n",
       "8              0                     1                  0           0  \n",
       "9              0                     0                  1           0  \n",
       "10             0                     0                  0           0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#training_set[features].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score f1 del modello 0: 0.46180122907395627\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.67      0.29      0.40        14\n",
      "          canone_rai       1.00      0.44      0.62        18\n",
      "          cessazione       0.00      0.00      0.00         2\n",
      "           contratto       0.83      0.31      0.45        16\n",
      "             credito       0.67      0.80      0.73        25\n",
      "      domiciliazione       0.00      0.00      0.00         3\n",
      "        fatturazione       0.36      0.31      0.33        16\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         5\n",
      "   non_di_competenza       1.00      0.20      0.33        10\n",
      "          richiamate       0.00      0.00      0.00         1\n",
      "\n",
      "         avg / total       0.66      0.40      0.46       110\n",
      "\n",
      "Score f1 del modello 1: 0.5377602379318233\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.75      0.25      0.38        12\n",
      "          canone_rai       1.00      0.33      0.50         6\n",
      "          cessazione       0.00      0.00      0.00         1\n",
      "           contratto       0.43      0.38      0.40         8\n",
      "             credito       0.75      0.75      0.75        32\n",
      "      domiciliazione       0.00      0.00      0.00         2\n",
      "        fatturazione       0.67      0.27      0.39        22\n",
      "gr_invito_a_chiamare       1.00      0.50      0.67         4\n",
      "   non_di_competenza       0.80      0.57      0.67         7\n",
      "          richiamate       0.00      0.00      0.00         0\n",
      "\n",
      "         avg / total       0.71      0.47      0.54        94\n",
      "\n",
      "Score f1 del modello 2: 0.560189367413632\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.71      0.45      0.56        11\n",
      "          canone_rai       1.00      0.50      0.67        12\n",
      "          cessazione       0.00      0.00      0.00         1\n",
      "           contratto       1.00      0.31      0.47        13\n",
      "             credito       0.81      0.76      0.78        33\n",
      "      domiciliazione       0.00      0.00      0.00         3\n",
      "        fatturazione       0.47      0.41      0.44        17\n",
      "gr_invito_a_chiamare       0.33      0.25      0.29         4\n",
      "   non_di_competenza       0.50      0.20      0.29         5\n",
      "          richiamate       0.00      0.00      0.00         1\n",
      "\n",
      "         avg / total       0.71      0.49      0.56       100\n",
      "\n",
      "Score f1 del modello 3: 0.5352951815936207\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.38      0.50      0.43         6\n",
      "          canone_rai       1.00      0.30      0.46        10\n",
      "          cessazione       0.00      0.00      0.00         2\n",
      "           contratto       0.44      0.36      0.40        11\n",
      "             credito       0.92      0.76      0.83        29\n",
      "      domiciliazione       0.00      0.00      0.00         4\n",
      "        fatturazione       0.53      0.42      0.47        19\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         3\n",
      "   non_di_competenza       1.00      0.40      0.57        10\n",
      "          richiamate       0.00      0.00      0.00         0\n",
      "\n",
      "         avg / total       0.68      0.47      0.54        94\n",
      "\n",
      "Score f1 del modello 4: 0.4584867543855487\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.75      0.40      0.52        15\n",
      "          canone_rai       0.67      0.86      0.75         7\n",
      "          cessazione       0.00      0.00      0.00         3\n",
      "           contratto       0.50      0.29      0.36        14\n",
      "             credito       0.68      0.68      0.68        22\n",
      "      domiciliazione       0.00      0.00      0.00         3\n",
      "        fatturazione       0.50      0.27      0.35        22\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         3\n",
      "   non_di_competenza       0.75      0.38      0.50         8\n",
      "          richiamate       0.00      0.00      0.00         1\n",
      "\n",
      "         avg / total       0.56      0.41      0.46        98\n",
      "\n",
      "C: 25\n",
      "La media e': 0.51070655408\n",
      "Stdev: 0.0421987903202\n"
     ]
    }
   ],
   "source": [
    "# set up ML models\n",
    "n_splits = 5\n",
    "# setta il parametro C da testare\n",
    "combination = [25]\n",
    "\n",
    "class BorsoneModello:\n",
    "    'Common class for all models: saves count_vect, svm'\n",
    "    def __init__(self, count_vect):\n",
    "        # una lista per ogni fold\n",
    "        # una sottolista per ogni categoria\n",
    "        self.algo = []\n",
    "        self.count_vect = count_vect\n",
    "\n",
    "        \n",
    "# ricostruisce gli indici partendo da 0\n",
    "etichette_train_set = training_set[features].reindex()\n",
    "# fa la matrice document-term\n",
    "# questa deve essere la stessa matrice del training e del test\n",
    "count_vect = CountVectorizer()\n",
    "matrix_countvect = count_vect.fit_transform(training_set['trascritto'])\n",
    "\n",
    "# crea il modello da salvare in pickler\n",
    "zainone = BorsoneModello(count_vect) \n",
    "\n",
    "# crea un classificatore per ogni valore di c\n",
    "for c in combination:\n",
    "    algo = OneVsRestClassifier(LinearSVC(C=c), n_jobs=-1)\n",
    "    # crea una lista di classificatori a votazione\n",
    "    models = [algo]*n_splits\n",
    "\n",
    "    folds = KFold(n_splits=n_splits, shuffle=True, random_state=1)\n",
    "    #print(folds)\n",
    "    #print(training_set.head(2)['contratto'])\n",
    "    \n",
    "    #crea il vettore in cui salveremo gli score\n",
    "    average = np.zeros(n_splits)\n",
    "    # kfold: each classificato is evalutaed against its test set fold\n",
    "    # folds returns the row *indeces*\n",
    "    for i, (train, test) in enumerate(folds.split(training_set)):\n",
    "        # preparation\n",
    "        X_train = matrix_countvect[train]\n",
    "        Y_train = etichette_train_set.iloc[train]\n",
    "        X_test = matrix_countvect[test]\n",
    "        Y_test = etichette_train_set.iloc[test]\n",
    "\n",
    "        # TfIdf Transform\n",
    "        X_train = TfidfTransformer(use_idf=True).fit_transform(X_train)\n",
    "        X_test = TfidfTransformer(use_idf=True).fit_transform(X_test)\n",
    "    \n",
    "        # learns from data\n",
    "        models[i].fit(X_train, Y_train)\n",
    "        # predicts on test set\n",
    "        y = models[i].predict(X_test)\n",
    "        # evaluate\n",
    "        score = f1_score(Y_test, y, average='weighted')\n",
    "        #print('Score f1 del modello {}: {}'.format(i, score))\n",
    "        average[i] = score\n",
    "        \n",
    "        print('Score f1 del modello {}: {}'.format(i, score))\n",
    "        print(classification_report(Y_test, y, target_names=features))\n",
    "\n",
    "        \n",
    "    # salva il modello del fold\n",
    "    zainone.algo = models\n",
    "    print('C:', c)\n",
    "    print('La media e\\':', average.mean())\n",
    "    print('Stdev:', average.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time to create training set:  0.4327085730000064\n",
      "f1_score: 0.0\n",
      "Score f1 del modello 0: 0.0\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.00      0.00      0.00        14\n",
      "          canone_rai       0.00      0.00      0.00        18\n",
      "          cessazione       0.00      0.00      0.00         2\n",
      "           contratto       0.00      0.00      0.00        16\n",
      "             credito       0.00      0.00      0.00        25\n",
      "      domiciliazione       0.00      0.00      0.00         3\n",
      "        fatturazione       0.00      0.00      0.00        16\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         5\n",
      "   non_di_competenza       0.00      0.00      0.00        10\n",
      "          richiamate       0.00      0.00      0.00         1\n",
      "\n",
      "         avg / total       0.00      0.00      0.00       110\n",
      "\n",
      "f1_score: 0.0102127659574\n",
      "Score f1 del modello 1: 0.010212765957446808\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.00      0.00      0.00        12\n",
      "          canone_rai       0.09      1.00      0.16         6\n",
      "          cessazione       0.00      0.00      0.00         1\n",
      "           contratto       0.00      0.00      0.00         8\n",
      "             credito       0.00      0.00      0.00        32\n",
      "      domiciliazione       0.00      0.00      0.00         2\n",
      "        fatturazione       0.00      0.00      0.00        22\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         4\n",
      "   non_di_competenza       0.00      0.00      0.00         7\n",
      "          richiamate       0.00      0.00      0.00         0\n",
      "\n",
      "         avg / total       0.01      0.06      0.01        94\n",
      "\n",
      "f1_score: 0.213529411765\n",
      "Score f1 del modello 2: 0.2135294117647059\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.00      0.00      0.00        11\n",
      "          canone_rai       0.00      0.00      0.00        12\n",
      "          cessazione       0.00      0.00      0.00         1\n",
      "           contratto       0.00      0.00      0.00        13\n",
      "             credito       0.48      1.00      0.65        33\n",
      "      domiciliazione       0.00      0.00      0.00         3\n",
      "        fatturazione       0.00      0.00      0.00        17\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         4\n",
      "   non_di_competenza       0.00      0.00      0.00         5\n",
      "          richiamate       0.00      0.00      0.00         1\n",
      "\n",
      "         avg / total       0.16      0.33      0.21       100\n",
      "\n",
      "f1_score: 0.0\n",
      "Score f1 del modello 3: 0.0\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.00      0.00      0.00         6\n",
      "          canone_rai       0.00      0.00      0.00        10\n",
      "          cessazione       0.00      0.00      0.00         2\n",
      "           contratto       0.00      0.00      0.00        11\n",
      "             credito       0.00      0.00      0.00        29\n",
      "      domiciliazione       0.00      0.00      0.00         4\n",
      "        fatturazione       0.00      0.00      0.00        19\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         3\n",
      "   non_di_competenza       0.00      0.00      0.00        10\n",
      "          richiamate       0.00      0.00      0.00         0\n",
      "\n",
      "         avg / total       0.00      0.00      0.00        94\n",
      "\n",
      "f1_score: 0.0\n",
      "Score f1 del modello 4: 0.0\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.00      0.00      0.00        15\n",
      "          canone_rai       0.00      0.00      0.00         7\n",
      "          cessazione       0.00      0.00      0.00         3\n",
      "           contratto       0.00      0.00      0.00        14\n",
      "             credito       0.00      0.00      0.00        22\n",
      "      domiciliazione       0.00      0.00      0.00         3\n",
      "        fatturazione       0.00      0.00      0.00        22\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         3\n",
      "   non_di_competenza       0.00      0.00      0.00         8\n",
      "          richiamate       0.00      0.00      0.00         1\n",
      "\n",
      "         avg / total       0.00      0.00      0.00        98\n",
      "\n",
      "C: 20\n",
      "La media e': 0.0447484355444\n",
      "Stdev: 0.084483131877\n",
      "Time to create all models:  1.049786288\n"
     ]
    }
   ],
   "source": [
    "# USE VotingClassifier + CHI2\n",
    "\n",
    "# set up ML models\n",
    "\n",
    "n_splits = 5\n",
    "# c values to test\n",
    "combination = [20]\n",
    "\n",
    "# ricostruisce gli indici partendo da 0\n",
    "#etichette_train_set = training_set[features].reindex()\n",
    "\n",
    "# come deve essere: \n",
    "# OneVsRest \n",
    "# a votazione tanti quanti sono i folds\n",
    "\n",
    "# ricostruisce gli indici partendo da 0\n",
    "etichette_train_set = training_set[features].reindex()\n",
    "# fa la matrice document-term\n",
    "# questa deve essere la stessa matrice del training e del test\n",
    "count_vect = CountVectorizer()\n",
    "matrix_countvect = count_vect.fit_transform(training_set['trascritto'])\n",
    "\n",
    "def spurga_svm(n_svm, combination, n_splits):\n",
    "    for c in combination:\n",
    "        for s in range(n_splits):\n",
    "            yield [LinearSVC(C=c)] * n_svm\n",
    "\n",
    "def rapporto(divisore):\n",
    "    def soglia(l):\n",
    "        if l/divisore < 0.5:\n",
    "            return 0\n",
    "        else:\n",
    "            return 1\n",
    "    \n",
    "    return soglia\n",
    "\n",
    "giudice_supremo = rapporto(n_splits)\n",
    "\n",
    "model_dispenser = spurga_svm(etichette_train_set.shape[1], combination, n_splits)\n",
    "\n",
    "class BorsoneModello:\n",
    "    'Common class for all models: saves chi2, svm'\n",
    "    def __init__(self, count_vect):\n",
    "        # una lista per ogni fold\n",
    "        # una sottolista per ogni categoria\n",
    "        self.chi2 = []\n",
    "        self.algo = []\n",
    "        self.count_vect = count_vect\n",
    "        self.counter = 0\n",
    "    def chi2_populate(self, chi2_list):\n",
    "        self.chi2.append(chi2_list)\n",
    "        self.counter += 1\n",
    "    def algo_populate(self, algo_list):\n",
    "        self.algo.append(algo_list)\n",
    "        self.counter += 1\n",
    "    \n",
    "print('Time to create training set: ', time.process_time()-last_time)\n",
    "last_time = time.process_time()\n",
    "\n",
    "nomi_etichette = list(validation_set[features].columns.values)\n",
    "\n",
    "#il count_vect e' uguale per tutti i fold\n",
    "zainone = BorsoneModello(count_vect) \n",
    "\n",
    "# crea un classificatore per ogni valore di c\n",
    "for c in combination:\n",
    "    #algo = OneVsRestClassifier(LinearSVC(C=c), n_jobs=-1)\n",
    "    # crea una lista di classificatori a votazione\n",
    "    #models = [algo]*n_splits\n",
    "\n",
    "    folds = KFold(n_splits=n_splits, shuffle=True, random_state=1)\n",
    "    #print(folds)\n",
    "    #print(training_set.head(2)['contratto'])\n",
    "    \n",
    "    #crea il vettore in cui salveremo gli score\n",
    "    average = np.zeros(n_splits)\n",
    "    \n",
    "    #y_voted = np.zeros((training_set.shape[0], etichette_train_set.shape[1]))\n",
    "    # crea la lista dove salvare i modelli\n",
    "    # kfold: each classificato is evalutaed against its test set fold\n",
    "    # folds returns the row *indeces*\n",
    "    for i, (train, test) in enumerate(folds.split(training_set)):\n",
    "        # preparation\n",
    "        X_train = matrix_countvect[train]\n",
    "        Y_train = etichette_train_set.iloc[train]\n",
    "        X_test = matrix_countvect[test]\n",
    "        Y_test = etichette_train_set.iloc[test]\n",
    "        \n",
    "        # chi2\n",
    "        # la matrice filtrata, una per classe\n",
    "        X_chi2 = [] \n",
    "        X_test_chi2 = []\n",
    "        #k = (4000,1500,2500,5000,2500,2500,1500,3000,3000,5000)\n",
    "        #k = (300,300,300,300,300,300,300,300,300,300)\n",
    "        #scores = []\n",
    "        feat = [] # il chi2\n",
    "        # matrice dove salvare i predicted\n",
    "        y = np.zeros((Y_test.shape[0], etichette_train_set.shape[1]))\n",
    "        \n",
    "        for g in range(Y_train.shape[1]):\n",
    "            # oggetto chi2 che salvo in lista\n",
    "            feat.append(SelectKBest(chi2, k = 300))#k[g])) \n",
    "            # fitto e salvo nella lista la matrice filtrata\n",
    "            X_chi2.append(feat[g].fit_transform(X_train,Y_train.iloc[:,g])) \n",
    "            # test set\n",
    "            X_test_chi2.append(feat[g].transform(X_test[g]))\n",
    "            #scores.append(feat[i].pvalues_)\n",
    "            #chi_k.append(feat[i].get_support(indices=True))\n",
    "        \n",
    "        # TfIdf Transform # attenzione fit transform\n",
    "        for g in range(Y_train.shape[1]):\n",
    "            X_chi2[g] = TfidfTransformer(use_idf=True).fit_transform(X_chi2[g])\n",
    "            X_test_chi2[g] = TfidfTransformer(use_idf=True).fit_transform(X_test_chi2[g])\n",
    "        \n",
    "        # creo un svm per ogni classe\n",
    "        models = next(model_dispenser)\n",
    "        \n",
    "        \n",
    "        # per ogni classe\n",
    "        for g in range(len(models)):\n",
    "            # fitto\n",
    "            models[g].fit(X_chi2[g], Y_train.iloc[:,g]) #texts[g]\n",
    "            # predicts on test set\n",
    "            y[:,g] = models[g].predict(X_test_chi2[g])\n",
    "        \n",
    "        # evaluate\n",
    "        score = f1_score(Y_test, y, average='weighted')\n",
    "        print('f1_score:', score)\n",
    "        print('Score f1 del modello {}: {}'.format(i, score))\n",
    "        average[i] = score\n",
    "        print(classification_report(Y_test, y, target_names=nomi_etichette))\n",
    "\n",
    "        # at last, save all the objects\n",
    "        # chi2\n",
    "        zainone.chi2_populate(feat)\n",
    "        # svm\n",
    "        zainone.algo_populate(models)\n",
    "    \n",
    "    # votazione\n",
    "    #y_voted = pd.DataFrame(y.sum(axis=2), columns=nomi_etichette)\n",
    "    #y_voted = y_voted.applymap(giudice_supremo)\n",
    "    \n",
    "    print('C:', c)\n",
    "    print('La media e\\':', average.mean())\n",
    "    print('Stdev:', average.std())\n",
    "print('Time to create all models: ', time.process_time()-last_time)\n",
    "last_time = time.process_time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time to create training set:  0.3266428029999986\n",
      "f1_score: 0.461801229074\n",
      "Score f1 del modello 0: 0.46180122907395627\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.67      0.29      0.40        14\n",
      "          canone_rai       1.00      0.44      0.62        18\n",
      "          cessazione       0.00      0.00      0.00         2\n",
      "           contratto       0.83      0.31      0.45        16\n",
      "             credito       0.67      0.80      0.73        25\n",
      "      domiciliazione       0.00      0.00      0.00         3\n",
      "        fatturazione       0.36      0.31      0.33        16\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         5\n",
      "   non_di_competenza       1.00      0.20      0.33        10\n",
      "          richiamate       0.00      0.00      0.00         1\n",
      "\n",
      "         avg / total       0.66      0.40      0.46       110\n",
      "\n",
      "f1_score: 0.537760237932\n",
      "Score f1 del modello 1: 0.5377602379318233\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.75      0.25      0.38        12\n",
      "          canone_rai       1.00      0.33      0.50         6\n",
      "          cessazione       0.00      0.00      0.00         1\n",
      "           contratto       0.43      0.38      0.40         8\n",
      "             credito       0.75      0.75      0.75        32\n",
      "      domiciliazione       0.00      0.00      0.00         2\n",
      "        fatturazione       0.67      0.27      0.39        22\n",
      "gr_invito_a_chiamare       1.00      0.50      0.67         4\n",
      "   non_di_competenza       0.80      0.57      0.67         7\n",
      "          richiamate       0.00      0.00      0.00         0\n",
      "\n",
      "         avg / total       0.71      0.47      0.54        94\n",
      "\n",
      "f1_score: 0.563784138655\n",
      "Score f1 del modello 2: 0.5637841386554622\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.83      0.45      0.59        11\n",
      "          canone_rai       1.00      0.50      0.67        12\n",
      "          cessazione       0.00      0.00      0.00         1\n",
      "           contratto       1.00      0.31      0.47        13\n",
      "             credito       0.81      0.76      0.78        33\n",
      "      domiciliazione       0.00      0.00      0.00         3\n",
      "        fatturazione       0.47      0.41      0.44        17\n",
      "gr_invito_a_chiamare       0.33      0.25      0.29         4\n",
      "   non_di_competenza       0.50      0.20      0.29         5\n",
      "          richiamate       0.00      0.00      0.00         1\n",
      "\n",
      "         avg / total       0.73      0.49      0.56       100\n",
      "\n",
      "f1_score: 0.535295181594\n",
      "Score f1 del modello 3: 0.5352951815936207\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.38      0.50      0.43         6\n",
      "          canone_rai       1.00      0.30      0.46        10\n",
      "          cessazione       0.00      0.00      0.00         2\n",
      "           contratto       0.44      0.36      0.40        11\n",
      "             credito       0.92      0.76      0.83        29\n",
      "      domiciliazione       0.00      0.00      0.00         4\n",
      "        fatturazione       0.53      0.42      0.47        19\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         3\n",
      "   non_di_competenza       1.00      0.40      0.57        10\n",
      "          richiamate       0.00      0.00      0.00         0\n",
      "\n",
      "         avg / total       0.68      0.47      0.54        94\n",
      "\n",
      "f1_score: 0.458486754386\n",
      "Score f1 del modello 4: 0.4584867543855487\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.75      0.40      0.52        15\n",
      "          canone_rai       0.67      0.86      0.75         7\n",
      "          cessazione       0.00      0.00      0.00         3\n",
      "           contratto       0.50      0.29      0.36        14\n",
      "             credito       0.68      0.68      0.68        22\n",
      "      domiciliazione       0.00      0.00      0.00         3\n",
      "        fatturazione       0.50      0.27      0.35        22\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         3\n",
      "   non_di_competenza       0.75      0.38      0.50         8\n",
      "          richiamate       0.00      0.00      0.00         1\n",
      "\n",
      "         avg / total       0.56      0.41      0.46        98\n",
      "\n",
      "C: 20\n",
      "La media e': 0.511425508328\n",
      "Stdev: 0.0430576037776\n",
      "Time to create all models:  0.6513167789999983\n"
     ]
    }
   ],
   "source": [
    "# USE VotingClassifier WITHOUT CHI2\n",
    "\n",
    "# set up ML models\n",
    "\n",
    "n_splits = 5\n",
    "# c values to test\n",
    "combination = [20]\n",
    "\n",
    "# ricostruisce gli indici partendo da 0\n",
    "#etichette_train_set = training_set[features].reindex()\n",
    "\n",
    "# come deve essere: \n",
    "# OneVsRest \n",
    "# a votazione tanti quanti sono i folds\n",
    "\n",
    "# ricostruisce gli indici partendo da 0\n",
    "etichette_train_set = training_set[features].reindex()\n",
    "# fa la matrice document-term\n",
    "# questa deve essere la stessa matrice del training e del test\n",
    "count_vect = CountVectorizer()\n",
    "matrix_countvect = count_vect.fit_transform(training_set['trascritto'])\n",
    "\n",
    "def spurga_svm(n_svm, combination, n_splits):\n",
    "    for c in combination:\n",
    "        for s in range(n_splits):\n",
    "            yield [LinearSVC(C=c)] * n_svm\n",
    "\n",
    "def rapporto(divisore):\n",
    "    def soglia(l):\n",
    "        if l/divisore < 0.5:\n",
    "            return 0\n",
    "        else:\n",
    "            return 1\n",
    "    \n",
    "    return soglia\n",
    "\n",
    "giudice_supremo = rapporto(n_splits)\n",
    "\n",
    "model_dispenser = spurga_svm(etichette_train_set.shape[1], combination, n_splits)\n",
    "\n",
    "class BorsoneModello:\n",
    "    'Common class for all models: saves chi2, svm'\n",
    "    def __init__(self, count_vect):\n",
    "        # una lista per ogni fold\n",
    "        # una sottolista per ogni categoria\n",
    "        self.algo = []\n",
    "        self.count_vect = count_vect\n",
    "        self.counter = 0\n",
    "    def algo_populate(self, algo_list):\n",
    "        self.algo.append(algo_list)\n",
    "        self.counter += 1\n",
    "    \n",
    "print('Time to create training set: ', time.process_time()-last_time)\n",
    "last_time = time.process_time()\n",
    "\n",
    "nomi_etichette = list(validation_set[features].columns.values)\n",
    "\n",
    "#il count_vect e' uguale per tutti i fold\n",
    "zainone = BorsoneModello(count_vect) \n",
    "\n",
    "# crea un classificatore per ogni valore di c\n",
    "for c in combination:\n",
    "    #algo = OneVsRestClassifier(LinearSVC(C=c), n_jobs=-1)\n",
    "    # crea una lista di classificatori a votazione\n",
    "    #models = [algo]*n_splits\n",
    "\n",
    "    folds = KFold(n_splits=n_splits, shuffle=True, random_state=1)\n",
    "    #print(folds)\n",
    "    #print(training_set.head(2)['contratto'])\n",
    "    \n",
    "    #crea il vettore in cui salveremo gli score\n",
    "    average = np.zeros(n_splits)\n",
    "    \n",
    "    # crea la lista dove salvare i modelli\n",
    "    # kfold: each classificato is evalutaed against its test set fold\n",
    "    # folds returns the row *indeces*\n",
    "    for i, (train, test) in enumerate(folds.split(training_set)):\n",
    "        # preparation\n",
    "        X_train = matrix_countvect[train]\n",
    "        Y_train = etichette_train_set.iloc[train]\n",
    "        X_test = matrix_countvect[test]\n",
    "        Y_test = etichette_train_set.iloc[test]\n",
    "        \n",
    "        # TfIdf Transform # attenzione fit transform\n",
    "        X_train = TfidfTransformer(use_idf=True).fit_transform(X_train)\n",
    "        X_test = TfidfTransformer(use_idf=True).fit_transform(X_test)\n",
    "        \n",
    "        # creo un svm per ogni classe\n",
    "        models = next(model_dispenser)\n",
    "        y = np.zeros((Y_test.shape[0], etichette_train_set.shape[1]))\n",
    "        \n",
    "        # per ogni classe\n",
    "        for g in range(len(models)):\n",
    "            # fitto\n",
    "            models[g].fit(X_train, Y_train.iloc[:,g])\n",
    "            # predicts on test set\n",
    "            y[:,g] = models[g].predict(X_test)\n",
    "        \n",
    "        # evaluate\n",
    "        score = f1_score(Y_test, y, average='weighted')\n",
    "        print('f1_score:', score)\n",
    "        print('Score f1 del modello {}: {}'.format(i, score))\n",
    "        average[i] = score\n",
    "        print(classification_report(Y_test, y, target_names=nomi_etichette))\n",
    "\n",
    "        # at last, save all the objects\n",
    "        # svm\n",
    "        zainone.algo_populate(models)\n",
    "    \n",
    "    print('C:', c)\n",
    "    print('La media e\\':', average.mean())\n",
    "    print('Stdev:', average.std())\n",
    "print('Time to create all models: ', time.process_time()-last_time)\n",
    "last_time = time.process_time()\n",
    "\n",
    "    # votazione\n",
    "    #y_voted = pd.DataFrame(y.sum(axis=2), columns=nomi_etichette)\n",
    "    #y_voted = y_voted.applymap(giudice_supremo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "######################## learning curve\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.model_selection import learning_curve\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "\n",
    "def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,\n",
    "                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):\n",
    "    \"\"\"\n",
    "    Generate a simple plot of the test and training learning curve.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    estimator : object type that implements the \"fit\" and \"predict\" methods\n",
    "        An object of that type which is cloned for each validation.\n",
    "\n",
    "    title : string\n",
    "        Title for the chart.\n",
    "\n",
    "    X : array-like, shape (n_samples, n_features)\n",
    "        Training vector, where n_samples is the number of samples and\n",
    "        n_features is the number of features.\n",
    "\n",
    "    y : array-like, shape (n_samples) or (n_samples, n_features), optional\n",
    "        Target relative to X for classification or regression;\n",
    "        None for unsupervised learning.\n",
    "\n",
    "    ylim : tuple, shape (ymin, ymax), optional\n",
    "        Defines minimum and maximum yvalues plotted.\n",
    "\n",
    "    cv : int, cross-validation generator or an iterable, optional\n",
    "        Determines the cross-validation splitting strategy.\n",
    "        Possible inputs for cv are:\n",
    "          - None, to use the default 3-fold cross-validation,\n",
    "          - integer, to specify the number of folds.\n",
    "          - An object to be used as a cross-validation generator.\n",
    "          - An iterable yielding train/test splits.\n",
    "\n",
    "        For integer/None inputs, if ``y`` is binary or multiclass,\n",
    "        :class:`StratifiedKFold` used. If the estimator is not a classifier\n",
    "        or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.\n",
    "\n",
    "        Refer :ref:`User Guide <cross_validation>` for the various\n",
    "        cross-validators that can be used here.\n",
    "\n",
    "    n_jobs : integer, optional\n",
    "        Number of jobs to run in parallel (default 1).\n",
    "    \"\"\"\n",
    "    plt.figure()\n",
    "    plt.title(title)\n",
    "    if ylim is not None:\n",
    "        plt.ylim(*ylim)\n",
    "    plt.xlabel(\"Training examples\")\n",
    "    plt.ylabel(\"Score\")\n",
    "    train_sizes, train_scores, test_scores = learning_curve(\n",
    "        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes, scoring='f1_macro')\n",
    "    train_scores_mean = np.mean(train_scores, axis=1)\n",
    "    train_scores_std = np.std(train_scores, axis=1)\n",
    "    test_scores_mean = np.mean(test_scores, axis=1)\n",
    "    test_scores_std = np.std(test_scores, axis=1)\n",
    "    plt.grid()\n",
    "\n",
    "    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,\n",
    "                     train_scores_mean + train_scores_std, alpha=0.1,\n",
    "                     color=\"r\")\n",
    "    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,\n",
    "                     test_scores_mean + test_scores_std, alpha=0.1, color=\"g\")\n",
    "    plt.plot(train_sizes, train_scores_mean, 'o-', color=\"r\",\n",
    "             label=\"Training score\")\n",
    "    plt.plot(train_sizes, test_scores_mean, 'o-', color=\"g\",\n",
    "             label=\"Cross-validation score\")\n",
    "\n",
    "    plt.legend(loc=\"best\")\n",
    "    return plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(344, 5346)\n",
      "(344, 10)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEZCAYAAABiu9n+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXl4VOXVwH9nspE9AQXCFjC0KG64gAK2QK2KqLWt7acI\nVmxrbV3q8rm3CIha99qq/T5cPnFBsdW2arEuVaMoiggqLmyyhF1AtiSEJDNzvj/unWEymUlCkiEz\nl/N7nvvkLu+995x7J+95zznvfV9RVQzDMAwjFr6OFsAwDMNIXsxIGIZhGHExI2EYhmHExYyEYRiG\nERczEoZhGEZczEgYhmEYcTEjYaQ0IvKyiJzX0XIkEyIyUETmdcB954rIIfv6vkZiMSNhtAoRWSki\n3+toOVR1jKo+mYhri0i+iNwnIhUislNElonIvSLSORH3a0duBu4MbYjICSLynohsF5EtIjJbRI4R\nkeNEpEpEcqIvICILRORiESkVkaCIzI863kVE6kRkRcTuu4CpCdPK6BDMSBhJi4ikdeC9M4A3gUOA\nk1W1ABgKbAGGtOJ6+0QXEekOjARecLfzgZeAPwHFQE9gClCrqnOBNcBPoq5xGI7eT0fszhGRgRHb\n5wLLo27/EjBKRLq2lz5Gx2NGwmh3ROR0EflYRLaJyLsicnjEsetE5Cu3Zf65iPww4tj5bvl7RWQL\nMMndN1tE7hKRrSKyXERGR5zzloj8POL8psr2FZG3RWSHiLwmIg+ISDwv5HygF/BDVV0CoKpbVPU2\nVX3FvV5QRA6KuP5jInKzuz5CRNaIyLUisgH4PxH5UkTGRJRPE5FNIjLI3T7ebfFvc5/fiIiyE1x9\ndrp/x8aR+yRggarWudvfdkTXv6pDrar+R1U/d48/Afws6hrnAS+r6vaIfU8CEyK2f+aeG0ZVa4H5\nwClxZDNSEDMSRrsiIkcBjwIXAp2BacCLbssc4CtguNsynwI8JSLdIi5xnFumK3BrxL5FQBeckMaj\nTYgwpImyTwMfuMem4FSG8calORF4RVVrmrhXc2PadAeKgD7Ar9z7nxtxfDSwWVU/EZGewL+Am1W1\nGLgaeN4N6+TgeAKnuM9tGPBJnHseDiyJ2F4KBERkuoiMFpGiqPJPAt9174+IiCvj9Cg9nwLOEYeB\nQC7wYYz7LwKOjPtEjJTDjITR3lwI/K+qfuS2XJ8EaoHjAVT1eVX92l3/G7CMhuGbdar6F1UNui1T\ngFWq+n/qDDT2OFDSREijIlZZEekNHAtMUlW/qr4HvNiEHl2ADc3oKs0cD7j3q3d1eQb4gYh0co+P\ndfcBjANmqeqrAKr6BvARMCbiWoeLSCdV/VpVF8W5ZxFQGdpQ1UrgBCAIPARsEpEXQs9PVdcCb+MY\nTIDvA5nAy1HXXQssxvFUzsMxLrGodGUwPIIZCaO9KQX+2w33bBWRbThhmx4AIvKziFDUNuBQ4ICI\n89fEuObG0EpEyz4vzv3jle0BbFXV3c3cK8Q3QEkTx1vCZlWtj5BnOfAlcIaIZAM/AGa4h0uB/4p6\nbsOBElXdBZwN/AbYICIviciAOPfcBuRH7lDVJar6c1XtAxyG8yz+GFHkcfYYifHATFUNxLh2KOR0\nDvGNRD6wPc4xIwUxI2G0N2uAW1W1s7sUq2qeqj4rIn1wWrMXu/uLgS9o2CJP1LDEG4DOEa14gN5N\nlP8PcIpbmcdjFxDZM6h71PFYuszECeecCXyhqivd/WuAJ6KeW76q3gmgqq+r6snuPZYAD8eRaSFO\nHiImqroUJ5R0WMTuvwO9RGQk8GMcoxGL54HTgOWuBxKLQ4BP493fSD3MSBhtIVNEsiKWNJzK69ci\nMgRARHJFZIyI5OLEsYPAFhHxicgFNKysEoaqrsYJ30wWkQwRGQqc0cQpT+JU3M+LyAA3Ft9FRG6I\nSIZ/Apzr6jIaGBH3anuYCZyM4xVE9h56CsfDONm9Xic3+d3DDZf9wM1N1ANVOOGnWLwOHC0imQCu\n7FdF5Bx644S53o94NrtwDMBjOKG9BVHXlIhyo3BCio0QkSzgGFcGwyOYkTDawiyc1nSN+3eSqs7H\nqUQeEJGtOInT8wHcOPo9OMnjjTihpndbcV+Ns95c2XE4Sd8tON8SzMTJlzQ+yekd9H2cOPzrwA72\nJL3nusUuxwkZbcOpeP/RrOCqG3Eq6OOBZyP2r8XxLm4ENgMVOMlrn7tcBaxzZf8ujpGJdf1NOF13\nQ73GKnES/3NFpBKYg+NtXB116uM4CfZYXkT4GarqggjvJ5ofAG+5OhoeQWzSIWN/RURmAotUdUpH\ny9KeiPPV83RVPW4f3/d94Beq+uW+vK+RWMxIGPsNInIssBVYidOX/+/AUFW1GLphxCG9owUwjH1I\ndxzD0BmnS+evzUAYRtOYJ2EYhmHExRLXhmEYRlxSJtwkIubyGIZhtAJVbW50gLiklCehqp5dJk2a\n1OEymH6m3/6m2/6gX1tJKSPhZVatWtXRIiQU0y918bJu4H392ooZCcMwDCMuZiSShAkTJnS0CAnF\n9EtdvKwbeF+/tpIyXWBFRFNFVsMwjGRBRND9JXHtZcrLyztahIRi+qUuXtYNvK9fWzEjYRiGYcTF\nwk2GYRgexsJNhmEYRsIwI5EkeD0uavqlLl7WDbyvX1sxI2EYhmHExXIShmEYHsZyEoZhGEbCMCOR\nJHg9Lmr6pS5e1g28r19bSaiREJFHReRrEVnYRJk/i8gyEflERAYlUh7DSBiq4PdDIOCsG4ZHSGhO\nQkROAKqAJ1T1iBjHTwUuVdXTROQ44E+qenyca1lOwkg+6uqgqgq2b4f6ehA39CsCaWng8zl/09Od\n9ci/Is56U38No420NSeR0EmHVPVdESltosiZwBNu2bkiUigi3VT160TKZRhtor4edu2Cbdugttap\n0Dt1cpYQqs4SDDrL7t179oX2R6Pa0DBEG5roJYkNTcXKlUyfOJHgunX4evZkwtSplPbr1yGyJAup\n+kw6ema6nsCaiO117r79zkiUl5czcuTIjhYjYaS8fn6/Yxi2b4eaGqfy7dQJ8vMBKJ8zh5HDhu0p\nL7Knom4t0YbG72/e0ESeGzIQIe+llYam/O239+rdVaxcyf0nncSU5cvJBaqBSR98wGWvv56UleK+\n+G2m2jOJpKONhGEkL8GgYxC2b3dCSiKQmRk2DAmnow1NiDVrYMUKR5Z0t8qoqYHqamepqoLKSmfZ\nuZPpjz0WrgwBcoEpy5dz9/e/z6QRI1qvS6LYsAGeeCKht5j+9ttMWbGi8TOZOJFJTz2V0Hu3lY42\nEuuA3hHbvdx9MZkwYQJ9+/YFoKioiEGDBoVbAKEeCqm6HdqXLPLst/p997uwezflr7wCu3YxcsgQ\nyMig/LPPnOOut1A+Z06D7dC+eMc7dFuE8g8+cLaPOw4qKx19d+1i5EEHwc6dlC9Y4GwfcIBzfOlS\nqKlhZFYWI3fupPzGG53jdXWwaxflmZmQk+OUz8uj3O+HTp0YWVpKcNMm5rnPZKT7dx6wYvduGDjQ\nkW/5cud4WVmHb48cODDh91vx6qvMi3ge5e7f4Pr1znY7/p7Ly8uZPn06QLi+bAsJ/5hORPoCL6nq\n4TGOjQEucRPXxwP3WeLa2OeoOrmFnTthxw6ndZ2e7oSTWhDTr1i9mul33klw40Z83bsz4dprKe3T\np/3l9PsbtNgbrUf/jbWvpgZycx1vqKDA+Ru53tS+wkLIy3OeSzDo9OQKPb+MDMjKgpwcpvzmN1z9\n7LPhVjM44ZW7x41L+lZzopgyfjxXz5jRIc+krYnrRPduehrHeHbByTNMAjIBVdWH3DIPAKNxntkF\nqrogzrU8bSRSPmbfDEmpX22tEy4J9UxKT3cqur0I71SsXs3955zDiRUVnIobay4t5bKZMxsairZW\n8Dt3OsnvvLzWVfChv3l5LdfP7we/n/L33nM8qlD4q1MnyM52/qanOwYiLW3PM4kVfy8rS9r4e4fl\nJPbRM0n23k3ntqDMpYmUwTAaUF/vGIZt25zuq7F6Ju0F02+9lSkVFeHwSi4wpaKCu8eMYZIbuglX\n8KEKPF7FXlwMffrEr/Rzc9uWn4hHMOg8l8h8RSj/kpu7R66QMWjGuyrt14/LXn+duydOJLh+Pb4e\nPbgsRXryJIpUfiY2dpPhfaJ7Jvl8jseQ3oo20vr1MG8efPghfPghkxYvZkqM5O+kww9nyp/+1LCC\n7+jvHlQdYxAIOM8ksvdTyDvIzHSMQUZGx8trtAtJ7UkYRocRCOzpmVRd7VR4WVl71zMpGISlSx2D\nEDIMu3bBkCEweDCcdRa+Rx6h+oUXGsWaff37w4AB7a1Vy/H7HYMQ6tUU6iWVlbUnr5CR4RiIiFCR\nYURjnkSSkJQx+3Zkn+gX+mhtxw4nzKPqtIyzslp2/u7dsHBh2Etg/nwoKnKMQsgwlJU1aGG3OCeR\nKEJeQShUFCIz0/EMsrP3eAat8Zyw32aqY56EsX+j6lTulZWOcVB1KsOWhHe2b4ePPnK8hLlz4fPP\n4dvfdozB2WfD3XdD165NXqK0Tx8umzmTSddcwwfBIL5u3bgsEb2bIkNFoV5F4OgaSiKHQmgWKjLa\nEfMkjNSktnaPYfD7m++ZpArr1u3xEubNcz4SO+qoPV7C0Uc7oZiOJjJUFEokp6WFu5iSlbXHO0hE\nItvwFEndBbY9MSNhUFe3p2dSfb1TcXbqFLuiDARg8eIGSWb8fscYhMJHAwc6FW1HERkqCuUNwDEC\nIe+gjaEiwzAj4RG8HhdttX5+/x7DEBpML1bPpJoa+PRTJ2w0b56TTzjwwD1ewpAh0LdvwsIwjcZu\niiQUKgoZg1AX01CoKLJXUWgspSTCfpupjeUkDO/Rkp5JW7c6+YSQl/Dll3DwwY4xGD8e7rsPDjhg\n38qt6sgeyh2E9oU8nvz8hh+gWajISAHMkzCSg9Bgejt2OAPGgdO6zsx0KtrVqxt2Rd240ckhhLyE\no492WuSJJGQEAoGGw1KECH2AlpPTsIuphYqMDsTCTUbqEt0zKRjc81Xv4sUNk8ywJ5cwZIjjNbR3\n5RsyACEjEP178/kc+ULGKzOz8dDbSRYqMgwzEh7B63HRsH6hwfSqqhzDEAg424sWOcZg3jxYsABK\nShommXv3blsF3JQXEJkjCA1Ul5m5xwsIGYAmwkNefn9e1g28r5/lJIzUoL7eST5v2+aEij75xFk+\n+giWLHF6Gg0ZAhMmwAMPQOfOe3f9lnoBnTqZF2AYe4F5EkbiCA2m9/HH8P77zt8FC2DLFjj2WMdT\nOO44OOKIpvMJ0V5AaAkR8gJCHkDIC4g0AJYkNvZTLNxk7FOanad3927HILz1FnzwgWMUMjMdYxAK\nHw0Y0HC8IMsFGEbCMCPhEVIhLhpzTPx+/bhs0iRKlyyB2bMdb6FnT8cYHH+8Yxi6d6f83XedOQmi\njYDP57T6Iw1ACnoBqfD+WouXdQPv62c5CWOfMX3ixMZzF69cyd1XXcWkH/4Qxo6Fe+5xBsUL4fM5\nRiEry5mXINILSE93jpsXYBhJi3kSRtOoOj2PZs1i0q23MmXHjkZFJg0dypQXXkhZL8AwvIx5Ekb7\ns3u3k1P4179g1ixnOIkTT8Q3cCDV77/feO6Egw5yhsAwDMNzWDMvSSgvL+9YAdasgWnT4IwznOGx\nJ092Jr5/9FFnPKQHH2TCU08xqayMaveU0Dy9E6ZObfbyHa5fgvGyfl7WDbyvX1sxT2J/JRBweh/N\nmuV4DGvXwqhRcOqpcPvtjqEoLHRCSC6leXkpO0+vYRitw3IS+xNbt8IrrzhG4dVXoXt3OPFEZzn6\naGcu5vx8yyMYhoewLrBGfFThs8/2eAsLF8LQoY5R+N73nKEuCgsTPzCeYRgdRluNhDUZk4R2i4vu\n2uUYhF//Gvr0cXIMy5bBxRc78y3MnAlXXx3+fmFfGQivx329rJ+XdQPv69dWLCfhBVat2uMtvPsu\nHH44fP/78NRT0L+/891CTo59j2AYxl5j4aZUpL4e5szZYxg2b3aSzieeCCNGOJPtFBZ27NSchmEk\nBZaT2F/YvBn+/W/HKLz+upNPCCWdBw3ak3Q2b8EwjAgsJ+ERGsVFVZ1xkG65xRkDqazMySccdxy8\n+aZjKP7wBzj9dOjVyzESSWwgvB739bJ+XtYNvK9fW7GcRDJRVQX/+Y/jLbz8sjP3wYknwhVXOMah\nqMgxBpEjqBqGYSQQCzd1NF99tSe38P77cNRRTtL5xBPhoIOc3EJubvPXMQzDiIHlJFKNujpnSO2Q\nYdixw/lm4cQT4bvfdWZks6SzYRjthOUkUoGNG+Gxx+DHP3YGwrvmGidk9Oc/w/z58NBDlJeVwbe/\n7fRM8qCB8Hrc18v6eVk38L5+bcVyEokgGHQq/1mz4KWXYPly+M53HG9h8mTo1s3JL2Rl7TkniZPO\nhmHsvyQ83CQio4H7cLyWR1X1jqjjBcBTQB8gDbhHVafHuE5yh5t27oTXXnMMw8svOwnmE0908guD\nBzshpIICGxfJMIx9SlLnJETEBywFTgTWA/OAc1R1cUSZG4ACVb1BRA4AlgDdVNUfda19aiSanctZ\nFZYs2ZNbmDfPMQahbxf69nUMQ07OPpPZMAwjmmSfdGgIsExVKwBEZCZwJrA4oowC+e56PvBNtIHY\n18Scy/mDD7jsX/+itKJij2GoqXEMwvnnw8MPO9NzFhY6s7PtJV6fZ9f0S128rBt4X7+2kmgj0RNY\nE7G9FsdwRPIA8KKIrAfygLMTLFOzxJzLefly7j78cCYNGuQYhmnTYOBAJ4SUl2c5BcMwPEkyJK5P\nAT5W1e+JSBnwuogcoapV0QUnTJhA3759ASgqKmLQoEHhFkCoh0J7bAfXrWOee8+R7t95wIqyMseL\nKCyk/P33Ydmydrt/aF8i9EmGbdMvdbdHjhyZVPKYfk1vl5eXM336dIBwfdkWEp2TOB6YrKqj3e3r\nAY1MXovIv4A/qOp77vYbwHWq+lHUtfZZTmLK+PFcPWNGo7mc7x43jklPPbVPZDAMw2gPkv07iXlA\nfxEpFZFM4BzgxagyFcD3AUSkG/BtYEWC5WqSCVOntnou59YSagl4FdMvdfGybuB9/dpKQsNNqhoQ\nkUuB19jTBXaRiFzkHNaHgFuA6SKy0D3tWlXdmki5mqO0Xz+by9kwDAMblsMwDMPTJHu4yTAMw0hh\nzEgkCV6Pi5p+qYuXdQPv69dWzEgYhmEYcbGchGEYhoexnIRhGIaRMMxIJAlej4uafqmLl3UD7+vX\nVsxIGIZhGHGxnIRhGIaHsZyEYRiGkTDMSCQJXo+Lmn6pi5d1A+/r11bMSBiGYRhxsZyEYRiGh7Gc\nhGEYhpEwzEgkCV6Pi5p+qYuXdQPv69dWzEgYhmEYcbGchGEYhoexnIRhGIaRMMxIJAlej4uafqmL\nl3UD7+vXVsxIGIZhGHGxnIRhGIaHsZyEYRiGkTDMSCQJXo+Lmn6pi5d1A+/r11bMSBiGYRhxsZyE\nYRiGh7GchGEYhpEwzEgkCV6Pi5p+qYuXdQPv69dWzEgYhmEYcbGchGEYhoexnIRhGIaRMMxIJAle\nj4uafqmLl3UD7+vXVsxIGIZhGHGxnIRhGIaHSfqchIiMFpHFIrJURK6LU2akiHwsIp+LyFuJlskw\nDMNoGQk1EiLiAx4ATgEOBcaKyMFRZQqBB4HTVfUw4KeJlClZ8Xpc1PRLXbysG3hfv7aSaE9iCLBM\nVStUtR6YCZwZVeZc4HlVXQegqlsSLJNhGIbRQlqckxCRE4BvqepjInIgkKeqK5s55yzgFFX9lbs9\nHhiiqr+NKPNHIAPH08gD/qyqT8a4luUkDMMw9pK25iTSW3iTScCxwADgMZxK/SlgeGtvHCXD0cD3\ngFzgfRF5X1W/ii44YcIE+vbtC0BRURGDBg1i5MiRwB6X0bZt27Zte3/eLi8vZ/r06QDh+rIttMiT\nEJFPgKOABap6lLtvoaoe0cx5xwOTVXW0u309oKp6R0SZ64BOqjrF3X4E+LeqPh91LU97EuXl5eEX\n7kVMv9TFy7qB9/XbV72b6twaWt2b5rbwvHlAfxEpFZFM4BzgxagyLwAniEiaiOQAxwGLWnh9wzAM\nI4G01JO4GvgWcBLwB+DnwNOqen8Lzh0N/AnHID2qqreLyEU4HsVDEde/AAgAD8e6rtc9CcMwjETQ\nVk9ibxLXJwEnAwK8qqqvt/amrcGMhGEYxt6T8HCTGwZ6S1VfV9VrVPXqfW0g9gdCiSevYvqlLl7W\nDbyvX1tp1kioagAIuh+9GYZhGPsRLc1JvIDTu+l1oDq0P/J7h0Rj4SbDMIy9Z598JwH83V0MwzCM\n/YgWdYFV1ceBZ4D57vK0u89oJ7weFzX9Uhcv6wbe16+ttPSL65HA48AqnN5NvUXkfFV9J3GiGYZh\nGB1NS3MS84FzVXWJu/1t4BlVPSbB8kXKYDkJwzCMvWRffXGdETIQAKq6FGf8JsMwDMPDtNRIfCQi\nj7iTA40UkYeBjxIp2P6G1+Oipl/q4mXdwPv6tZWW9m76DXAJEOryOhv4S0IkMgzDMJKGluYkcoHd\n7od1iEgakKWquxIsX6QMlpMwDMPYS/ZVTuINIDtiOxv4T2tvahiGYaQGLTUSnVS1KrThruckRqT9\nE6/HRU2/1MXLuoH39WsrLTUS1SJydGhDRI4FahIjkmEYhpEstDQnMRiYCax3d5UAZ6vq/ATKFi2D\n5SQMwzD2koTmJERksIh0V9V5wMHAs0A98AqwsrU3NQzDMFKD5sJN04A6d30ocCPwILANeCiBcu13\neD0uavqlLl7WDbyvX1tp7juJNFXd6q6fDTykqs8Dz4vIJ4kVzTAMw+homsxJiMjnwCBV9YvIYuBX\noUH9RORzVT1sH8lpOQnDMIxWkOj5JJ4B3haRLTi9mWa7N+0P7GjtTQ3DMIzUoMmchKreCvw3MB04\nIaIp7wMuS6xo+xdej4uafqmLl3UD7+vXVpodu0lVP4ixb2lixDEMwzCSiRZ9J5EMWE7CMAxj79lX\nYzcZhmEY+yFmJJIEr8dFTb/Uxcu6gff1aytmJAzDMIy4WE7CMAzDw1hOwjAMw0gYZiSSBK/HRU2/\n1MXLuoH39WsrZiQMwzCMuFhOwjAMw8MkfU5CREaLyGIRWSoi1zVRbrCI1IvIjxMtk2EYhtEyEmok\nRMQHPACcAhwKjBWRg+OUux14NZHyJDNej4uafqmLl3UD7+vXVhLtSQwBlqlqharW40yBemaMcpcB\nzwGbEiyPYRiGsRckNCchImcBp6jqr9zt8cAQVf1tRJkewAxVHSUijwEvqerfY1zLchKGYRh7SdLn\nJFrAfUBkrqLVyhiGYRjtS7NDhbeRdUCfiO1e7r5IjgVmiogABwCniki9qr4YfbEJEybQt29fAIqK\nihg0aBAjR44E9sQVU3X7vvvu85Q+pl9yydeW7ciYfTLIY/o1r8/06dMBwvVlW0h0uCkNWAKcCGwA\nPgTGquqiOOX323BTeXl5+IV7EdMvdfGybuB9/doabkr4dxIiMhr4E05o61FVvV1ELgJUVR+KKvt/\nwL/2RyNhGIaRCJLeSLQXZiQMwzD2Hi8krg2831fb9EtdvKwbeF+/tmJGwjAMw4iLhZsMwzA8jIWb\nDMMwjIRhRiJJ8Hpc1PRLXbysG3hfv7ZiRsIwDMOIi+UkDMMwPIzlJAzDMIyEYUYiSfB6XNT0S128\nrBt4X7+2YkbCMAzDiIvlJAzDMDyM5SQMwzCMhGFGIknwelzU9EtdWqqbquIP+tnt3011XTU7du9g\nc/VmNldv5ptd37CtZhvbaraxY/cOKmsrqaytpKquiuq6aqrrqtlVv4ua+hp2+3ez27+bWn8tdYE6\n6gJ11Afq8Qf9+IN+AsEAQQ0S1CDtEV3w8rtrDxI96ZBhGB4iEAw4FbUGCAQD1PprqQ04iz/oByU8\nt6Qg+GRPO1TRcKWuqFMWt3wL1xVFRBqtA/h8Pnxuu1fEubeIhOUIyRK9XllbybaabQ32h86Lt+7M\nkUaL1kOsXLWSifdOZN3OdfQs6MnUq6bSr2+/vX4H+xrLSRiGESaowQaGoC5Q16BFH125p0kaab60\n8N+OItL47M16vGOCgBB3fW+N17rV6/jZDT9j9dGrIROog7JPy3j9gdcTbihsPgnDMFqMqhLQQDhs\nEwoP1QXqqA3UEgwGG1SIPp+vgSGIbh3HY3XFau588E42Vm2ke153rr3kWvqU9mn+xA4gqEFq/bXh\nMNdu/25qA7UNwl7hY4HdsfdHbYfOr/HXUOuvZfULq6keXO0YiBB1MK5yHE/9+amE6tdWI2HhpiTB\n61Momn77juiQUKQRCIWEQq1dQcIGoFN6pwbhoRBzZs9h2HeGtfj+qytWc86151BxVAV0B+pgwbUL\nmHnnzGYNRX2gvkEFHb3Eqogj99f4axqd39w5tV/V0ql/JzqlO0tWelbM9U7pnchKa3ysIKugwf5Y\n17hu9nV8nPlxQ2UzYf3O9S1+rh2FGQnDSDGCGgx7ApEhoVp/LfXB+rghoQxfBp3SOyVEJn/Qz9fV\nX7OhcgO33H6LYyBCreZMqDiqgh9e/0MO+tFBjSrryAobaLKCzk7PblwJuxV0XlYeB+Qe0KjCjlXB\nR15j/pz5DP/u8DY/g1CoLpRUDyfWBXoV9uLjuo8beRI9Cnq0+b6JxsJNhpFkRIeEQq3r9g4JtZTd\n/t1srNrIhsoNzt+qDWyo3BD+u7FqI9/UfEOXnC6U5JVQ8c8Kth6/tdF1DvnsEKZMmdKggs9Oz26w\nne5LrnZr6F2EDICiBIKBcCI7MkkfegeZaZmk+9LJTMsMv5PVFasZc8UYVhy5wnISicKMhOElIkNC\n/oC/gRFoKiSU5kuLGRJqLVV1VXsq/BiV/4aqDVTVVdEttxsl+SWU5LmLu949rzsl+SV0ze0aruAv\nvfZS/tH5H41azT/a+iMeuPOBdpO9tahquKUf0EDYEISORRoAn/hIl3Qy0jLCFX+6Lz38HtIkLdwr\nqjnjHOrdtH7nenoU9NhnvZvMSHiEZIppJ4L9Tb/WhoTaq5eQqrJt97awBxBpACK9AX/Qv6fyz3cr\n/bwSeuTcsLU5AAAgAElEQVT3CO9b8tGSvQrHNMhJuK3m0o9LW5STaAvhij/qO4qQsY00AKFKP92X\nzofvfcjIkSNJ96U7FX+EAWhvr6wjsMS1YewDIlufQQ2iaINKacfuHXxd9XWjkJBzMg1CQjlpOW2q\nfIIaZMuuLY1a/eur1u8xBJUbyUrPatDaL8kr4dgexzYwCIVZhc3KslSW7pV8fUr7MPPOmdz54J18\nXfU13fK6ce2drevdFHruoZBPZOUfHfJJF6eFHwpbpfscDyCyxR/LEyvsVEhhp8K9lm1/wTwJY78h\n1KqMrmxC65Et/8ivewPqxKIbXmzP6to1a7lv2n1sqtoU7u7Zt2/fVslYH6hnU/WmcIUfKweweddm\n8jPzG3gA0Z5ASV4JuZm5rX9YCSZWkjeowUYt/pBxjYzzx2rxtyTcs79i4SZjv6O5Vn1k989wZa8B\np3UfeZ3QR08uPvGFv5QNxZkj98Vib0IrNfU14Uo/Xhhoa83WcAI4Vg6gJL+EbrndyErPavfn2lYa\nhXrQ8DOPDvmEWvnRcf7oFn975l/2V8xIeIT9LWbfkla9P+hvsB6vVR/Z8hSRRkMxtDSx2BouufYS\n/tn5n7AOCOUg6+CQpYdw9NijG3gC1fXVdM/r3qC1H+0FRCaAO5rQ+3hv9nsMGT6kQYI3skzo2af7\n0klPSyfDtyfe39ok777E6/97lpMw9inR48/cfOXNlJaWxmzZR7bqv9n1DWt2rGlTqz4zPbPJVn0s\n6gP1VNfvGTyuxl/jrPtrnG13X4NjscpFlQ8dq1pUBaOibpoJO2t3cmjXQ/n+Qd8PG4HO2Z07vGXc\nVGs/jPt4QxV7mqSRm5HboOKPNL7t3ePKSC7MkzCaJahB6gP1LFuxjDOuPINVg1aFQyu9F/Tmsdse\no3ef3kDjVn30oGqRLUl/0N985dzGSlxVycnIITsjm+z0bHIycpz++RHbDdbTsxuWzegU91h2RjbX\n//56XujyQod194xO7La1tW8xfu9h4Saj3VBV6oPOkMx1/rpwxesP+lGUa353DS8d8FKjCvGwrw5j\n9C9GN1uJ76rfxe763eH1gAYaVL456U7FG6rEYx0Lb0cc65TeKW5ln+HLSGhll4junq1p7Yfi+tba\nN6KxcJNH2NdxUX/QHx6jP1SR1wZqAcdY+MRHui+d2kAt8zfMZ86aOby54k2IHkUgE77Z9Q3+oJ+C\nrAK65XZrUGFnZzjL0o+Wcvx3jm9Q2WemZaZ8azXU3fOaG68hWBiM2d2zta39TmmdkqK17/WYvdf1\naytmJDxOKFRUH6yn1l/rTOoS2B0eWiAUEspIyyAvM4+a+hrmbZjHe2veY86aOSzespgjux3JsN7D\nOLrn0cyum93Ikzi+1/FcM/yaJuWoL66nrLisVTpEe5CRievoIZ/jnRfvnNCxEYNGsHb12lbJF80/\n/vcf7XIdw9gbSktLWbVqVbtf18JNHqG5UJGINBjoLdQC3e3fzfz1jqcwZ+0cPt/0OYceeCjDeg9j\nWO9hHFNyDNkZ2YATWjn72rNZfdTqxjmJ3r33yBKRhG7w0RMQuRr96UFTx3y+PaESX+SEikKD60eG\nVBpMFhOV8I4s5xMf3fO7t8ssZ4bRUYS6GMfZbzmJ/YmWhopCIYpI6gJ1fLLxk7Cn8OnGT/l2l28z\nvPdwhvUexuCeg8nJyGl0Tl2gDlVl8/rN3D/tfr6u+pqSghImXz45/OFYZCUcWXFHh0TiHWtgTGKc\nl0ji/YMZRqqQskZCREYD9+HMp/2oqt4Rdfxc4Dp3sxL4jap+FuM6njYSseKiexMqite33h/08+nG\nT5mzdg5z1sxh/vr5HFR8EMN6D2N47+EM6TmE/Kz8RueFDAM4QzcXZRWRk5nT6j78yR73NSNhpDqJ\nMhIJzUmIiA94ADgRWA/ME5EXVHVxRLEVwHdVdYdrUB4Gjk+kXMmGqlIfqGdX/a5mQ0XZ6dlNtrAD\nwQCfb/rcCR+tmcO89fPoVdCLYb2HMeHICfzPaf9DUaeimOdGG4buud3bZBgMw0h9EupJiMjxwCRV\nPdXdvh7QaG8ionwR8Jmq9o5xzBOeRFtCRbEIapBFWxbx3monfPThug/plteNYb2cnMLQ3kPpnN05\n7vmRhiE7PZvCrML90jCYJ2GkOinpSQA9gTUR22uBIU2U/yXw74RKtI/Y215FLUVVWfrN0rCn8P7a\n9ynqVMTwPsP50SE/4q6T7uLA3AObvEakYchJz6Fzbuf90jDsrwSDQQoLC1m0aBG9evVqt7KGN0ma\nWkFERgEXACfEKzNhwoRwkrSoqIhBgwaF49zl5eUA+3x7xIgR+IN+3njrDfwBP8cMO4aa+hrem/0e\ninL8CceTJmnMmzOPNEkLj8s/Z/YcgPDcwQ8/+DCHHnFoeDt0fOgJQ1m+bTlPvfgUn2/6nGUFy8jJ\nyKFsRxmHHXgYN593MyX5JU75TXDggANjXv+dt9+hPlDPkOFDyM3I5Yv5X9ApvRMnfu/EffK87rvv\nvqR4X/G2k5n8/PxwiLG6upqsrCzS0py5DqZNm8bYsWP36no+n4/Kysp2L2skB+Xl5UyfPh2g1aMR\nR7Ivwk2TVXW0ux0z3CQiRwDPA6NVdXmca3V4uKm9Q0WRhCabV1UqdlTs8RTWvI/P5wt3SR3eezi9\nClrWogtPcKNKbkYuhZ0Kyc7I7hCPIZUT1xUrVzJ94kSC69bh69mTCVOnUtqv5TOKtfX8SA466CAe\nffRRRo2KHjBqD4FAgLS0tk9clOrsb88hUeGmRH+nPw/oLyKlIpIJnAO8GFlARPrgGIjz4hmIjqS6\nrpq1O9by1davWL51OWt2rGFj1UYqaysREfIy88jLzCM/K5/czFyy0rP22kCs3bmWNZ3XcPkrlzPk\nkSH86Nkf8d7q9xjaayjP/ddzfPjLD/nT6D9x9qFnN2sgav21VNVVUVlbSYYvgx75PSjrXEavwl7k\nZ+V3WEgpmQ1EU1SsXMn9J53E1TNmMKW8nKtnzOD+k06iYuXKfXJ+NKraqCKYOHEi55xzDueeey6F\nhYXMmDGDDz74gKFDh1JcXEzPnj25/PLLCQScr7wDgQA+n4/Vq1cDcN5553H55ZczZswYCgoKGD58\nOBUVFXtdFuDf//43AwYMoLi4mN/+9reccMIJPPHEEzF1mTt3LscccwyFhYWUlJRw3XXXhY+98847\nDB06lKKiIkpLS5kxYwYAO3bsYPz48XTt2pWDDjqI22+/PXzOo48+yogRI7j88svp0qULt956KwCP\nPPIIhxxyCF26dOG0005j7dr2+WhyvyH0o0vUAowGlgDLgOvdfRcBv3LXHwa+ARYAHwMfxrmOdgSb\nqjbp0i1Lde2Otbpu57p2Weavn6/3z71fxz43Vkv/WKpd7uiipz99ut42+zZ9e9Xbe32vFVtX6JIt\nS3Tx5sW6Zvsarayt1PpAfYc8r1Ql3u9r8rhxWgWqEUsV6ORx41p03baeH03fvn31jTfeaLDv97//\nvWZlZemsWbNUVXX37t360Ucf6YcffqjBYFBXrlypAwYM0AcffFBVVf1+v/p8Pq2oqFBV1fHjx+uB\nBx6oCxYsUL/fr2effbaed955e13266+/1vz8fH3ppZfU7/frvffeq5mZmfr444/H1GXw4ME6c+ZM\nVVWtqqrSDz/8UFVVV6xYoXl5efrcc89pIBDQb775Rj/99FNVVR07dqyeddZZWl1drStWrND+/fvr\nE088oaqqjzzyiKanp+u0adM0GAzq7t279bnnntODDz5Yly1bpoFAQKdMmaLf+c53WvXsk514v2F3\nf6vr8IQ3K1X1FWBA1L5pEesXAhcmWo620NYxcjZXbw5/pzBnzRy21mxlaK+hDOs9jF8e/UsGdBnA\n++++z7Ajh7X4mrX+2nDyOTcjlwNzDwxP25iMJHu4KR7BdeuInt8tFwjOmAFu67bJ893yjc5fv759\nBHQ54YQTGDNmDABZWVkcc8wx4WN9+/blwgsv5O233+biiy8GGg9N8pOf/ISjjjoKgHHjxvG73/0u\nfKylZWfNmsVRRx3F6aefDsCVV17JXXfdFVfmzMxMli1bxtatW+ncuTODBw8GYMaMGYwZM4azzjoL\ngM6dO9O5c2f8fj9/+9vfWLx4MTk5OfTr148rr7ySJ598kvPOOw9whqb41a9+FX4O06ZN48Ybb6R/\n//4A3Hjjjdx2221s2LCBkpKSlj3c/ZzkrFFSnK01W3l/zfvhoS42Vm1kSM8hDOs9jPOOOI+BBw5s\n1YicIcMgCDkZORyYeyDZ6dl7Hd4yWo6vZ0+qaVjRVwO+cePgqaeaP3/8eKpnzGh8fo/okRLbRuSw\nKABLlizhv//7v5k/fz67du0iEAhw3HHHxT2/e/fu4fWcnByqqqr2uuz69esbydFUj6jHHnuMm266\niQEDBlBWVsakSZM49dRTWbNmDWVljcf52rRpE8FgkD599gyeWFpayrp168Lb0fevqKjgkksu4fLL\nLwccg5eens7atWvNSLQQMxLtwI7dO5i7bm54qIvVO1YzuMdghvUexh9P+SOHdT2s2RZ+qBdSJKoa\n7q6a6oYhFb0IgAlTpzLpgw+Ysnw5uTgV/KSyMi6bOnWfnN9Soj3diy66iKFDh/K3v/2N7Oxs7rnn\nHmbNmtWu94ympKSE1157rcG+yAo8mm9961s888wzAPz1r3/lrLPOYvv27fTu3ZuFCxc2Kt+1a1fS\n0tKoqKgIewYVFRX07NkzXCb6OfTp04dbbrmFn/70p63Wa3/HjEQcQjOwrdq2ii65Xbjh0hvCwz9X\n1VUxd+3csKewfOtyji45mmG9h/GHE//Akd2OJCMto1X3jTYMuZm5KWsYvEBpv35c9vrr3D1xIsH1\n6/H16MFle9E7qa3nt5bKykoKCwvJzs5m0aJFTJs2LeHfOZx++ulcccUVzJo1i9GjR3P//fezZcuW\nuOWfeuopTj31VLp06UJBQQE+nw+fz8f48eM58sgj+cc//sEPfvADtm/fzrp16zjiiCP4yU9+wo03\n3shjjz3Gpk2buO+++5g4cWLce1x00UVMnTqVww8/nIMPPpjt27fzxhtvhENZRvOYkYjBylUrOenS\nk1h+5HLoAtTB3KvmcvLYk/m87nOWfLMkPHz25BGTGdR9UJsmpldV3nn7HY4eerRnDUOq5iTAqegn\ntSC0lKjzI2lpbuyee+7h17/+NbfddhtHH30055xzDu+++27M6zR3zZaW7dq1K88++yyXX34548eP\n52c/+xlHHXUUWVmx/zdefvllrrrqKmprayktLeWvf/0r6enp9O3bl5deeolrrrmGCRMmUFxczG23\n3cYRRxzBgw8+yKWXXkrfvn3Jzc3loosuCucjYvGTn/yEXbt28dOf/pQ1a9ZQVFTEKaecYkZiL7BR\nYGMw/rfjmZE/o9G8CQOWDODmKTc3GD67tYQ8hvpAPQALP1zI6O+P9pRhiCTZjYQNy9H+BINBevTo\nwfPPP8/w4cM7WhzPk6rDcqQk63auczyISDKhS3YXTugT94PwZlFVagO1+AN+APKy8uia25VO6Z34\n9phvt0Hi5CeZDYTRfrz66qscf/zxdOrUiT/84Q9kZmYyZEhTI/EYyY4ZiRj0LOgJdTTyJLrlddvr\na8UyDIW5hXRK7+RJj8HYv3n33Xc599xzCQQCHHroofzzn/8kI6N1+TkjObBwUwwa5CRaMbl9TMOQ\n1bRhSPZwTFtJdv0s3GSkOhZu2of069uP1x943endtH0VXXK6cMOdNzRpIEKGoT5QjyDmMRiG4QnM\nk2iGzdWb2Vm7M2aiOtIw+MRHbmZusx6DkZyYJ2GkOuZJJAmxDEO33G5mGAzD8CSJHgXWM+z276ay\ntpJd9bvITs+mV0EvyjqX0SO/B7mZuW02EKH5DbyK1/UzDK9inkQzCII/6Kcgq4Buud3Izshu1bhL\nhmEYqYjVds1QnF3Mt7p8i5L8EnIzcxNmIJK550974HX9jL1jypQp4S+l16xZQ0FBQdycUGTZ1nDY\nYYfxzjvvtPr8/R0zEs2Q5kszz8FICp5++mkGDx5Mfn4+PXv25LTTTuO9997raLFaTWiIj969e7Nz\n584mh/xo6XAkF1xwATfddFODfZ9//jnf/e53Wy/ofo7VfkmC12P2qazfylUrGf/b8YyaMIrxvx3P\nylV7N6tcW88HuPfee7nqqqv4/e9/z6ZNm1i9ejWXXHIJL730UszyoVnojNQlGAx2tAiAGQnDaJLQ\nh5Uz8mdQ3q+cGfkzOOnSk1pc0bf1fICdO3cyadIk/vKXv3DmmWeSnZ1NWloaY8aMCU/fOWXKFH76\n059y3nnnUVRUxOOPP05dXR1XXHEFPXv2pFevXlx55ZXU1ztjhX3zzTecccYZFBcX06VLF0aMGBG+\n3x133EGvXr0oKCjgkEMO4a233oop15gxY/jLX/7SYN+gQYP45z//CcAVV1xBnz59KCwsZPDgwQ0G\nGIykoqICn88XrhRXrVrFyJEjKSws5JRTTmk0kux//dd/UVJSQnFxMSNHjmTRokUAPPzww8yYMYM7\n77yTgoICzjzzTAD69evHm2++CdDkM3n77bfp3bs39957L926daNnz55Mnz497nuZPn06ZWVlFBQU\nUFZWFh72PCTLwIEDKSgo4LDDDuOTTz4BYPHixYwaNYri4mIOP/zwBkb+ggsu4OKLL+a0004jPz+f\n8vJy6urquPrqqyktLaWkpISLL76Y2trauDIlhLZMa7cvFzpo+lJj/yDe72vcZeOUG1EmRyw3ouMu\na9n0o209X1X1lVde0YyMDA0EAnHLTJ48WTMzM/XFF19UVdWamhqdOHGiDh06VLds2aJbtmzRYcOG\n6U033aSqqjfccIP+5je/0UAgoH6/X999911VVV2yZIn27t1bN27cqKqqFRUVumLFipj3fOKJJ3T4\n8OHh7S+++EKLi4u1rq5OVVVnzJih27Zt00AgoPfee692795da2trw/KGpj1dtWqV+ny+sH5Dhw7V\nq6++Wuvq6vSdd97R/Pz8cFlV1ccee0yrq6u1rq5Or7zySh00aFD42IQJE3TixIkN5Iyc8rWpZ1Je\nXq7p6ek6efJk9fv9+vLLL2tOTo5u3769ke7V1dVaUFCgy5YtU1XVjRs36pdffqmqqn/961+1V69e\nOn/+fFVVXb58ua5evVrr6+u1f//+evvtt2t9fb2++eabmp+fr0uXLg3LXlRUpO+//76qOtPQXnHF\nFXrmmWfq9u3btaqqSn/wgx/ojTfeGPN9xPsN08bpSzu88m+xoGYkjAQS7/c18vyRDSv40DIixr69\nKDfq/FEtlm3GjBlaUlLSZJnJkyfriBEjGuwrKyvTV155Jbz96quvar9+/VRV9aabbtIf/vCH+tVX\nXzU456uvvtJu3brpf/7zH62vb3qe9MrKSs3Ly9PVq1erqurvfvc7/cUvfhG3fHFxsS5cuDAsbywj\nUVFRoRkZGbpr167weeeee24DIxHJtm3bVER0586dqtq8kWjqmZSXl2tOTk4DY9y1a1edO3duo/tW\nV1drcXGx/v3vf9eampoGx0455RT985//3Oic2bNnN3qPY8eO1SlTpoRlP//88xscz83NbWCk58yZ\nE5Y3mkQZCQs3JQmpHLNvCamqX3iwx0jqYNwR49BJ2uwy7ohxMc/vUdDy6Uu7dOnCli1bmo1RR0/d\nuX79+kZTfa5359a+5pprKCsr4+STT6Z///7ccccdAJSVlXHfffcxefJkunXrxrnnnsvGjRsByM/P\np6CggIKCAtauXUteXh5jxoxh5syZADzzzDOMGzcufL+7776bgQMHUlxcTHFxMTt37mxyEiKADRs2\nUFxcTHb2nhEOSktLw+vBYJDrr7+e/v37U1RURL9+/RCRZq/bkmcCzrP2+fZUi/Gmcs3JyeHZZ5/l\nf/7nfygpKeGMM85g6dKlAHGnX401vWtT069u3ryZXbt2ccwxx4Tn+T711FP55ptvWqRre2FGwjCa\nYOpVUyn7tGxPRV8HZZ+WMfWqlk0/2tbzAYYOHUpWVlY41h+P6B5APXv2pKKiIrxdUVFBD3du7by8\nPO6++26WL1/Oiy++yL333hvOPZxzzjnMnj07fO51110HOLPd7dy5k507d4ZnuRs7dixPP/00H3zw\nAbW1tYwaNQpwRoO96667eO6559i2bRvbtm1rsptriJKSErZt20ZNTU143+rVq8PrM2bM4KWXXuLN\nN99k+/btrFq1KjLa0GwvqB49esR9JnvLSSedxGuvvcbGjRsZMGAAF154IeBU9MuXL4957zVr1jTY\nt3r16rjTrx5wwAHk5OTwxRdfsHXrVrZu3cr27dvZsWNHq+RtLWYkkgSvf0eQqvqFBnscVzmOUStH\nMa5yHK8/8Dr9+rZs+tG2ng9QUFDAlClTuOSSS3jhhReoqanB7/fz73//m+uvvz7ueeeccw633HIL\nW7ZsYcuWLUydOjX8vcGsWbPCFVl+fj7p6en4fD6WLl3KW2+9RV1dHZmZmWRnZzdoWUczZswYKioq\nuOmmmzj77LPD+ysrK8nIyKBLly7U1dVx8803U1lZGfc6oUq+T58+HHvssUyaNIn6+nrefffdBsnd\nqqoqsrKyKC4uprq6mhtuuKFBxdqtWzdWrFgR9z5jx46N+0z2hk2bNvHiiy+ya9cuMjIyyMvLCz+n\nX/7yl9x9990sWLAAgOXLl7NmzRqOO+44cnJyuPPOO/H7/ZSXl/Ovf/2LsWPHxryHiHDhhRdyxRVX\nsHnzZsCZMzx6HvGE05ZY1b5csJyEkUBS4ff19NNP67HHHqt5eXlaUlKip59+ejjJGRnjD7F79269\n/PLLtaSkRHv06KFXXHFFOHH8xz/+Ufv27at5eXnau3dvvfXWW1VVdeHChTpkyBAtKCjQLl266Bln\nnKEbNmxoUq5f/OIX6vP59KOPPgrvCwQC+vOf/1wLCgq0R48eetddd2m/fv3CuYGmEtcrV67U73zn\nO5qfn68nn3yyXnbZZeGyVVVVeuaZZ2p+fr727dtXn3zySfX5fLp8+XJVVV22bJkOGjRIi4uL9Uc/\n+pGqaoP7NvVMysvLtXfv3g10izw3kg0bNuiIESO0qKhIi4uLddSoUbpo0aLw8WnTpumAAQM0Pz9f\nDz/8cP3kk09UVfXLL7/UESNGaGFhoR566KH6wgsvhM+54IILGuVTamtr9cYbb9SDDjpICwsLdeDA\ngXr//ffHfA/xfsO0MSdho8AmCck+30JbSXb9bBRYI9VJ1CiwFm4yDMMw4mKehGFgnoSR+pgnYRiG\nYexzzEgkCan6HUFL8bp+huFVzEgYhmEYcbGchGFgOQkj9bE5rg0jgZSWlrZ4zgLDSEYihy9pTxIe\nbhKR0SKyWESWish1ccr8WUSWicgnIjIo0TIlI16P2Se7fpHDO7Rmeeuttzr8g9NELV7WzUv6rVq1\nKiH/Gwk1EiLiAx4ATgEOBcaKyMFRZU4FylT1W8BFwP8mUqZkJTTevFcx/VIXL+sG3tevrSTakxgC\nLFPVClWtB2YCZ0aVORN4AkBV5wKFItItwXIlHdu3b+9oERKK6Ze6eFk38L5+bSXRRqInEDns4Vp3\nX1Nl1sUoYxiGYXQA1gU2SUhUPDFZMP1SFy/rBt7Xr60ktAusiBwPTFbV0e729TgjEt4RUeZ/gbdU\n9Vl3ezEwQlW/jrqW9U80DMNoBZrEXWDnAf1FpBTYAJwDRA+e/iJwCfCsa1S2RxsIaJuShmEYRutI\nqJFQ1YCIXAq8hhPaelRVF4nIRc5hfUhVXxaRMSLyFVANXJBImQzDMIyWkzJfXBuGYRj7npRIXLfk\ng7xUQkRWicinIvKxiHzo7isWkddEZImIvCoihR0tZ0sRkUdF5GsRWRixL64+InKD+/HkIhE5uWOk\nbjlx9JskImtFZIG7jI44lmr69RKRN0XkCxH5TER+6+5P+XcYQ7fL3P2eeH8ikiUic9265DMRmeTu\nb79319FfCTa34Biyr4BSIAP4BDi4o+Vqo04rgOKofXcA17rr1wG3d7Sce6HPCcAgYGFz+gADgY9x\nQp193XcrHa1DK/SbBFwVo+whKahfd2CQu54HLAEO9sI7bEI3L72/HPdvGvABzvdp7fbuUsGTaMkH\neamG0NiLOxN43F1/HPjhPpWoDajqu8C2qN3x9PkBMFNV/aq6CliG846Tljj6gfMeozmT1NNvo6p+\n4q5XAYuAXnjgHcbRLfQdllfe3y53NQun8lfa8d2lgpFoyQd5qYYCr4vIPBH5pbuvm7q9ulR1I9C1\nw6RrH7rG0cdLH09e6o439kiEO5/S+olIXxyv6QPi/yZTUscI3ea6uzzx/kTEJyIfAxuB11V1Hu34\n7lLBSHiR4ap6NDAGuEREvoNjOCLxWo8Cr+nzF+AgVR2E8895TwfL02ZEJA94DrjcbXV75jcZQzfP\nvD9VDarqUTje3xAROZR2fHepYCTWAX0itnu5+1IWVd3g/t0M/BPH3fs6NGaViHQHNnWchO1CPH3W\nAb0jyqXk+1TVzeoGeYGH2eOyp6R+IpKOU4k+qaovuLs98Q5j6ea19wegqjuBcmA07fjuUsFIhD/I\nE5FMnA/yXuxgmVqNiOS4rRpEJBc4GfgMR6cJbrHzgRdiXiB5ERrGeOPp8yJwjohkikg/oD/w4b4S\nsg000M/9xwvxY+Bzdz1V9fs/4EtV/VPEPq+8w0a6eeX9icgBoVCZiGQDJ+HkXdrv3XV0Zr6F2fvR\nOL0SlgHXd7Q8bdSlH04PrY9xjMP17v7OwH9cPV8Dijpa1r3Q6WlgPVALrMb5ILI4nj7ADTi9KhYB\nJ3e0/K3U7wlgofsu/4kTA05V/YYDgYjf5QL3fy7ubzJVdGxCN0+8P+BwV6dPXH1+5+5vt3dnH9MZ\nhmEYcUmFcJNhGIbRQZiRMAzDMOJiRsIwDMOIixkJwzAMIy5mJAzDMIy4mJEwDMMw4mJGwkgaRKSz\nO3VzfV8AAAPTSURBVOTxAhHZ4A7lHNpu0QRZ7rDe32qmzMUiEj1DYkojIrNF5IiOlsPwHvadhJGU\niMhNQJWq3hvjmKj9cBsgIrOBS1R1YbOFDWMvME/CSFYih8AocyeNeUpEPge6i8g0EfnQnWjl9xFl\nZ4vIESKSJiLbROQP7kif74nIAW6ZqRET68x2y8x1J2E53t2fIyLPicjnIvI3d8TeRi11ETlWRMrd\n47NE5EARSReRj0RkmFvmrojJYCa791ooIn+Jkvse9zqfi8gxIvJ3d9KY0Lll7rFnRORLEZkpIlkx\nZBotInNcGZ5xh2sIyfG5+zz+0C5vyfA8ZiSMVGEAcI+qHqbOAInXqeoQnKGfTxaRg2OcUwi8pc5I\nnx8AP493cVU9DrgWZzIagMuADap6GDDVvU8D3LHE/gT8WFUHAzOAW1TVjzN0x0MichIwArjFPe0+\nVT1OVY8AikTklIhL7nKv8384Q0X8CjgC+JWIFLhlDgHuVdWBOMOEXBQl04HA9cD3VPVYnKFfLheR\nrsCp7vMbBJiRMFqEGQkjVViuqh9HbI8Tkfk449YcjDPjVjS7VPU1d30+zkxcsfh7RJlSd/0EnAmu\ncEM4X8Q47xDgUOA/7nj+1+GMqomqfgY8izOw2gWqGnDPOcn1JD4FvuueHyI0cOVnOLPgbVHVWmBl\n6LrASnXmCwB4ypUzkmE4z2KOK9O5rk5bgYCIPCQiPwR2YRgtoEXJQMNIAqpDKyLSH/gtcKyqVorI\nk0CnGOfURawHiP97r21BmVizmAnwqaqOiHPOYcB2oBvwhRv2uR9nOs2NIjI1Su6QHMGIdXDmAkiP\n2hd5LFqmf6vq+Y2EFTkWZ5TQnwK/AU6JLmMY0ZgnYaQKkZV0AbATqBKREuJXdrEq9pbyHnA2gIgc\njuM1RPMl0FNEBrvlMkRkoLt+NpALjAT+4g4Pn41jiL4RkXzgrFbI1U9EjnHXzwVmRx2fA4xwh4EO\n5Vb6u/cvVNWXgauIET4zjFiYJ2GkCuEWs6ouEJFFOEMdVwDvxipHy2bjilfmfuBxN1H+pbvsaHCi\nap2I/AS4380Z+IB7RGQzMAUYoapfi8j/An9U1QtF5AlX7vU4eZKWyBp5bBFwlYgchTM09MORZVR1\nk4j8AnjWzZkocCNQA/zdTXQLcGUT9zOMMNYF1jBiICJpQLqq1rrhrVeBb6lqsANlKgOeU2eqSsPY\nJ5gnYRixyQPeiPiI71cdaSAisFadsU8xT8IwDMOIiyWuDcMwjLiYkTAMwzDiYkbCMAzDiIsZCcMw\nDCMuZiQMwzCMuJiRMAzDMOLy/y9zN48bEMP+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1ed457b358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "estimators = OneVsRestClassifier(LinearSVC(C=c))\n",
    "#svm_all = VotingClassifier(estimators)\n",
    "\n",
    "title = \"Learning Curves (SVM)\"\n",
    "print(matrix_countvect.shape)\n",
    "print(etichette_train_set.shape)\n",
    "plot_learning_curve(estimators, title, matrix_countvect, etichette_train_set, (0, 1.1), cv=5, n_jobs=-1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                      precision    recall  f1-score   support\n",
      "\n",
      "         attivazione       0.57      0.36      0.44        11\n",
      "          canone_rai       0.75      0.67      0.71         9\n",
      "          cessazione       0.00      0.00      0.00         0\n",
      "           contratto       0.40      0.17      0.24        12\n",
      "             credito       0.50      0.67      0.57        15\n",
      "      domiciliazione       1.00      0.33      0.50         3\n",
      "        fatturazione       0.80      0.52      0.63        23\n",
      "gr_invito_a_chiamare       0.00      0.00      0.00         2\n",
      "   non_di_competenza       0.67      0.25      0.36         8\n",
      "          richiamate       0.00      0.00      0.00         0\n",
      "\n",
      "         avg / total       0.63      0.45      0.50        83\n",
      "\n",
      "#n of items predicted in each category\n",
      "attivazione 7\n",
      "canone_rai 8\n",
      "cessazione 0\n",
      "contratto 5\n",
      "credito 20\n",
      "domiciliazione 1\n",
      "fatturazione 15\n",
      "gr_invito_a_chiamare 0\n",
      "non_di_competenza 3\n",
      "richiamate 0\n",
      "\n",
      "\n",
      "accuracy 0.216666666667\n",
      "precision micro 0.627118644068\n",
      "recall micro 0.44578313253\n",
      "f1 micro 0.521126760563\n",
      "precision macro 0.46880952381\n",
      "recall macro 0.29687088274\n",
      "f1 macro 0.345226479747\n"
     ]
    }
   ],
   "source": [
    "# Now compare with the validatino set! NOT CHI2\n",
    "\n",
    "Y_validation = validation_set[features].reindex()\n",
    "matrix_countvect = zainone.count_vect.transform(validation_set['trascritto']) #possible problem on fit_transform\n",
    "\n",
    "X_validation = TfidfTransformer(use_idf=True).fit_transform(matrix_countvect)\n",
    "\n",
    "#create numpy matrix\n",
    "Y_predicted = np.zeros(Y_validation.shape)\n",
    "\n",
    "for i, model in enumerate(zainone.algo):\n",
    "    y = model.predict(X_validation)\n",
    "    Y_predicted = Y_predicted + y\n",
    "\n",
    "Y_predicted = Y_predicted / n_splits\n",
    "\n",
    "def soglia(l):\n",
    "    if l < 0.5:\n",
    "        return 0\n",
    "    else:\n",
    "        return 1\n",
    "applica_soglia = np.vectorize(soglia)\n",
    "\n",
    "Y_predicted = applica_soglia(Y_predicted)\n",
    "\n",
    "nomi_etichette = list(validation_set[features].columns.values)\n",
    "#evaluation_report.print_evaluation(Y_true, Y_predicted)\n",
    "print(classification_report(Y_validation, Y_predicted, target_names=nomi_etichette))\n",
    "print('#n of items predicted in each category')\n",
    "for i in range(Y_validation.shape[1]):\n",
    "    print(nomi_etichette[i], np.sum(Y_predicted[:,i]))\n",
    "    \n",
    "\n",
    "print('\\n')\n",
    "print('accuracy', accuracy_score(Y_validation, Y_predicted))\n",
    "print('precision micro',precision_score(Y_validation, Y_predicted, average='micro') )\n",
    "print('recall micro',recall_score(Y_validation, Y_predicted, average='micro'))\n",
    "print('f1 micro', f1_score(Y_validation, Y_predicted, average='micro'))\n",
    "print('precision macro', precision_score(Y_validation, Y_predicted, average='macro') )\n",
    "print('recall macro', recall_score(Y_validation, Y_predicted, average='macro'))\n",
    "print('f1 macro', f1_score(Y_validation, Y_predicted, average='macro'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Grid Search celle larghe:\n",
    "\n",
    "Best score: 0.464\n",
    "\n",
    "Best parameters set:\n",
    "\n",
    "clf__estimator__C: 1\n",
    "\n",
    "tfidf__use_idf: True\n",
    "\n",
    "vect__max_df: 0.5\n",
    "\n",
    "vect__max_features: 1000\n",
    "\n",
    "GRID SEARCH 2:\n",
    "\n",
    "done in 822.022s\n",
    "Best score: 0.488\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 1.5\n",
    "\tvect__max_df: 0.5\n",
    "\tvect__max_features: 1500\n",
    "    \n",
    "GRID SEARCH 3:\n",
    "done in 935.240s\n",
    "Best score: 0.480\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 10\n",
    "\tvect__max_features: 2500\n",
    "\tvect__min_df: 0\n",
    "    \n",
    "    \n",
    "GS 4:\n",
    "Best score: 0.488\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 9\n",
    "\tvect__max_df: 0.5\n",
    "\tvect__max_features: 1500\n",
    "\tvect__min_df: 0\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best score: 0.471\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 12\n",
    "    \n",
    "done in 1479.661s\n",
    "Best score: 0.473\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 18\n",
    "    \n",
    "done in 1254.545s\n",
    "Best score: 0.481\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "score_matrix = gs.cv_results_\n",
    "df = pd.DataFrame(score_matrix)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# controlla se esiste il modello per evitare di ricrearlo ogni volta\n",
    "if False: #os.path.exists(os.path.join(DATA_FOLDER, 'OneVsRestClassifier')):\n",
    "    with open(os.path.join(DATA_FOLDER, 'OneVsRestClassifier'), mode='rb') as modelfile:\n",
    "        model = dill.load(modelfile)\n",
    "\n",
    "else:\n",
    "    model = pipeline.fit(texts, Y)\n",
    "    print('Modello creato, verifico il test set')\n",
    "\n",
    "\n",
    "    with open(os.path.join(DATA_FOLDER, 'OneVsRestClassifier'), mode='wb') as modelfile:\n",
    "        dill.dump(model, modelfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# importo il test set per la valutazione del modello\n",
    "unito_test = []\n",
    "names_test = []\n",
    "test_texts = list()\n",
    "test_labels = list()\n",
    "#raise SystemExit\n",
    "\n",
    "# load test data\n",
    "# carica il test set\n",
    "with codecs.open(os.path.join(DATA_FOLDER, test_db_name), 'r', errors='ignore') as file:\n",
    "    reader = csv.reader(file, delimiter=';')\n",
    "    next(reader)\n",
    "    print(etichette)\n",
    "    for row in reader:\n",
    "        # load documents\n",
    "        test_texts.append(row[1]+row[2].replace('#O:', '').replace('#C:', ''))\n",
    "        # load names\n",
    "        names_test.append(row[0])\n",
    "        # load labels\n",
    "        test_labels.append(list(map(lambda x: int(x),row[-10:])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Y_test = np.array(test_labels)\n",
    "# apply model to data\n",
    "predicted_labels = model.predict(test_texts) #np_array\n",
    "\n",
    "# print evaluation report\n",
    "evaluation_report.print_evaluation(Y_test, predicted_labels)\n",
    "for i in range(predicted_labels.shape[1]):\n",
    "    print(etichette[i], np.sum(predicted_labels[:,i]))\n",
    "print('Decision function: ')\n",
    "\n",
    "scores = model.decision_function(test_texts) #np_array\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# score della cat credito \n",
    "cred_score = scores[:,4]\n",
    "cred_pred = predicted_labels[:,4]\n",
    "#print(len(cred_score), cred_pred)\n",
    "\n",
    "positive = cred_score[cred_pred == 1]\n",
    "neg = cred_score[cred_pred == 0]\n",
    "#print(positive)\n",
    "print('media score positivi predetti:   ', positive.mean())\n",
    "print('media score negativi predetti:   ', neg.mean())\n",
    "print('std score positivi predetti:     ', positive.std())\n",
    "print('std score negativi predetti:     ', neg.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# confronto gli score dei doc annotati a mano con quelli assegnati dal modello\n",
    "cred_score = scores[:,4]\n",
    "cred_ammano = Y_test[:,4]\n",
    "#print(len(cred_score), cred_ammano)\n",
    "\n",
    "positive_ammano = cred_score[cred_ammano == 1]\n",
    "neg_ammano = cred_score[cred_ammano == 0]\n",
    "print(sum(cred_ammano), len(positive_ammano))\n",
    "print('media score positivi:   ' , positive_ammano.mean())\n",
    "print('media score negativi:   ', neg_ammano.mean())\n",
    "print('std score positivi:     ', positive_ammano.std())\n",
    "print('std score negativi:     ', neg_ammano.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}