prove-SVM_stemming.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/utente_locale/anaconda3/lib/python3.5/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "SCRIPT PER PROVARE configurazioni SVM\n",
    "'''\n",
    "\n",
    "import time\n",
    "t_orig = time.process_time()\n",
    "last_time = 0\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(\"ignore\")\n",
    "\n",
    "import codecs\n",
    "import csv\n",
    "import argparse\n",
    "import os\n",
    "import numpy as np\n",
    "import pickle\n",
    "\n",
    "from time import time\n",
    "from dill import dill\n",
    "from sklearn.metrics import f1_score, accuracy_score, recall_score, precision_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "from sklearn import cross_validation, svm\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.tree.tree import DecisionTreeClassifier\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "# modulo per confrontare il risultato delle regex\n",
    "# richiede entrambi i db già etichettati\n",
    "import tortellino_parmigiano \n",
    "import crea_db\n",
    "\n",
    "\n",
    "import evaluation_report # stampa confusion_matrix e altro\n",
    "from text_tokenizer import get_tokenizer \n",
    "from nltk.stem.snowball import ItalianStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ita_stopwords = set(stopwords.words('italian'))\n",
    "ita_stemmer = ItalianStemmer()\n",
    "ita_stemming = ita_stemmer.stem\n",
    "def get_tokenizer1(doc): #(word_ngrams=None, char_ngrams=None, stopwords=None, additional_extractors=None):\n",
    "    # rimuove le stopwords\n",
    "    doc = filter(lambda x: x not in ita_stopwords, word_tokenize(doc))\n",
    "    # stemma le parole\n",
    "    doc = [ita_stemmer.stem(a) for a in doc]\n",
    "    return doc\n",
    "\n",
    "def get_tokenizer(doc): #(word_ngrams=None, char_ngrams=None, stopwords=None, additional_extractors=None):\n",
    "    doc = filter(lambda x: x not in ita_stopwords, word_tokenize(doc))\n",
    "    #doc = map(ita_stemming, doc)\n",
    "    #doc = [ita_stemmer.stem(a) for a in doc]\n",
    "    temp = ''\n",
    "    for par in doc:\n",
    "        temp = '{} {}'.format(temp, ita_stemming(par))\n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CestinoTraining:\n",
    "    def __init__(self, features, labels, categories):\n",
    "        self.stemmed = list(map(get_tokenizer, texts))\n",
    "        self.features = features\n",
    "        self.labels = np.array(labels)\n",
    "        self.categories = categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training sqlite e csv presenti!\n",
      "Test sqlite e csv presenti!\n",
      "Trovato salvataggio training set\n",
      "Salvataggio caricato\n"
     ]
    }
   ],
   "source": [
    "# list of possible algorithms\n",
    "algos = ['svm-lin', 'svm-rbf', 'nb', 'decisiontree', 'randomforest', 'regex', 'svm-multi']\n",
    "    # svm-rbf might not apply to text problems: https://charlesmartin14.wordpress.com/2012/02/06/kernels_part_1/\n",
    "\n",
    "DATA_FOLDER = \"/home/backup/dati_e_csv/\" # DATI\n",
    "# ci assicuriamo siano presenti i DB\n",
    "training_db_name = crea_db.build_training(OUTPUT=DATA_FOLDER)\n",
    "test_db_name = crea_db.build_test(OUTPUT=DATA_FOLDER) ################################\n",
    "\n",
    "    \n",
    "if os.path.exists('/home/backup/dati_e_csv/training_stemmed-bubu.pickle'):\n",
    "    print('Trovato salvataggio training set')\n",
    "    with open('/home/backup/dati_e_csv/training_stemmed-bubu.pickle', 'rb') as file:\n",
    "        salvataggio = pickle.load(file)\n",
    "    texts = salvataggio.features\n",
    "    texts_stemmed = salvataggio.stemmed\n",
    "    Y = salvataggio.labels\n",
    "    print('Salvataggio caricato')\n",
    "else:\n",
    "    print('Creo il training set')\n",
    "    \n",
    "    texts = list()\n",
    "    labels = list()\n",
    "    names= []\n",
    "\n",
    "    # load training dataset # possiamo usare sqlite\n",
    "    with codecs.open(os.path.join(DATA_FOLDER, training_db_name), 'r', errors='ignore') as file:\n",
    "        reader = csv.reader(file, delimiter=';')\n",
    "        etichette = next(reader)\n",
    "        for row in reader:\n",
    "            # load documents\n",
    "            #print('Nome {}, etichette {}, conversazione{}'.format(row[10], row[:10], row[11]+row[12]))\n",
    "            #raise SystemExit\n",
    "            unito = (row[11]+row[12]).replace('#O:', '').replace('#C:', '') #concatena il testo cliente con operatore\n",
    "            texts.append(unito)\n",
    "            # load row name\n",
    "            names.append(row[10])\n",
    "            # load labels\n",
    "            labels.append(list(map(lambda x: int(x), row[:10])))\n",
    "\n",
    "    print('dimensioni:',len(texts), len(labels))\n",
    "    # tokenization function # possiamo usare anche solo un canale cliente/operatore\n",
    "    print('Inizio lo stemming')\n",
    "\n",
    "    salvataggio = CestinoTraining(texts, labels, etichette)\n",
    "    texts_stemmed = salvataggio.stemmed\n",
    "    Y = salvataggio.labels\n",
    "    with open('/home/backup/dati_e_csv/training_stemmed-bubu.pickle', 'wb') as file:\n",
    "        pickle.dump(salvataggio, file, protocol=0)\n",
    "    print('Training set creato e salvato')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Algoritmo pronto per essere fittato\n"
     ]
    }
   ],
   "source": [
    "#analyzer = get_tokenizer(stopwords=stopwords.words('italian'), #word_ngrams=[1],\n",
    "#                         additional_extractors=None)\n",
    "\n",
    "\n",
    "\n",
    "#algo = OneVsRestClassifier(SVC(kernel='linear', max_iter=5000), n_jobs=1)\n",
    "algo = OneVsRestClassifier(LinearSVC(C=15), n_jobs=-1) #C=9\n",
    "#print('Multilabel ', algo.multilabel_)\n",
    "\n",
    "# pipeline definition\n",
    "pipeline = Pipeline([\n",
    "    ('vect', CountVectorizer(analyzer=get_tokenizer1,# tokenizer=lambda x: x.split(' '),\n",
    "                             max_df=0.6,min_df=0,max_features=1000)),#, max_df=0.5,min_df=0, max_features=1500)),# feature extraction,\n",
    "        #analyzer=get_tokenizer1, \n",
    "    ('tfidf', TfidfTransformer(use_idf=True)),#use_idf=True)), # weighting\n",
    "    #('chi2', SelectKBest(chi2, k=k)), # feature selection\n",
    "\n",
    "########### Per il chi2 multilabel\n",
    "# selected_features = [] \n",
    "# for label in labels:\n",
    "#     selector = SelectKBest(chi2, k='all')\n",
    "#     selector.fit(X, Y[label])\n",
    "#     selected_features.append(list(selector.scores_))\n",
    "\n",
    "# // MeanCS \n",
    "# selected_features = np.mean(selected_features, axis=0) > threshold\n",
    "# // MaxCS\n",
    "# selected_features = np.max(selected_features, axis=0) > threshold\n",
    "\n",
    "    ('clf', algo) # learning algorithm\n",
    "])\n",
    "\n",
    "parameters = {#'vect__ngram_range': ((1, 1),(1, 2)),\n",
    "              #'vect__max_df': (0.5,0.75,1.0),\n",
    "              #'vect__min_df': (0,0.25,0.4),\n",
    "              #'vect__max_features': (None,50,500,1000, 5000), # TO-DO: bisogna stemmare le features\n",
    "              #'tfidf__use_idf': (True,False),\n",
    "              #'tfidf__smooth_idf': (True,False)\n",
    "              'clf__estimator__C': (15,18,20),\n",
    "              #'clf__estimator__penalty': ('l1','l2')\n",
    "             } # vuole min e max --> facciamo unigrammi e bigrammi\n",
    "\n",
    "\n",
    "\n",
    "gs = GridSearchCV(pipeline, parameters, scoring='f1_macro', n_jobs=-1, verbose=10)\n",
    "\n",
    "\n",
    "#print('Pipeline creata, avvio il modello')\n",
    "\n",
    "print('Algoritmo pronto per essere fittato')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  sì  hanno hanno inviato documentazioni fonte signora  numero cliente vediamo signora  non ho il numero cliente  al  senza essere persone nella  il ministro con tutto toccata  perché allora nella documentazione che lei abbia inviato  risultava barche non vi era correttamente compilata  marcato alla Castelli fiscale su questa delicata  è molto  estremo non ce l' ho ancora signora questa anzi mi serve il venticinque di agosto  allora no signora lì di fronte a stare domani pomeriggio sul tasti perché i documenti lender lavorasse nel giro di quarantotto ore  quindi vedete ricontattare domani verso il tardi per vedere se sono pervenuti i documenti  va bene  io non ce l' ho ancora  non c'è ancora tutta  no con il nono bis signora non non non c'è proprio  non lo so signora io non lo vedo  grazie buongiorno  non c'è concorrenza  più o meno come fare la pratica perché è un po' di giorni ti aspetto  sì presa fatto tutto ha mandato il fax e avremo compilato un po' in  Como ottenendo Torino alle  il numero cliente scusi zero sette nove nove otto sei quattro nove sei  varato uno  vorrei sapere quando riescono riuscite ad attivare la luce  sì  testé  target nella  poi la  sì ma l' avevo già atto  eh sì però ieri mattina  o ringrazio di nuovo  ho capito  va beh ma ieri mattina è arrivato il fax giusto  il fax  e ma ieri ho parlato con giusto  collegai diceva che era arrivato il trentuno  e come ha fatto a vedere la sua collega mi scusi  va bene ok 24379\n",
      " sì invi document font signor numer client ved signor numer client senz esser person ministr tocc allor document invi risult barc corrett compil marc castell fiscal delic molt estrem ce ' ancor signor anzi serv venticinqu agost allor no signor lì front star doman pomerigg tast document lender lavor gir quarantott ore quind ved ricontatt doman vers tard ved perven document va ben ce ' ancor c'è ancor tutt no non bis signor c'è propr so signor ved graz buongiorn c'è concorrt men far pratic po ' giorn aspett sì pres fatt mand fax compil po ' com otten torin numer client scus zer sett nov nov otto quattr nov var vorre sap quand riesc riusc attiv luc sì test target poi sì ' già atto eh sì per ier mattin ringraz nuov cap va beh ier mattin arriv fax giust fax ier parl giust collega dic arriv trentun fatt ved colleg scus va ben ok 24379\n"
     ]
    }
   ],
   "source": [
    "print(texts[1], len(texts))\n",
    "print(texts_stemmed[1], len(texts_stemmed))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Grid Search celle larghe:\n",
    "\n",
    "Best score: 0.464\n",
    "\n",
    "Best parameters set:\n",
    "\n",
    "clf__estimator__C: 1\n",
    "\n",
    "tfidf__use_idf: True\n",
    "\n",
    "vect__max_df: 0.5\n",
    "\n",
    "vect__max_features: 1000\n",
    "\n",
    "GRID SEARCH 2:\n",
    "\n",
    "done in 822.022s\n",
    "Best score: 0.488\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 1.5\n",
    "\tvect__max_df: 0.5\n",
    "\tvect__max_features: 1500\n",
    "    \n",
    "GRID SEARCH 3:\n",
    "done in 935.240s\n",
    "Best score: 0.480\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 10\n",
    "\tvect__max_features: 2500\n",
    "\tvect__min_df: 0\n",
    "    \n",
    "    \n",
    "GS 4:\n",
    "Best score: 0.488\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 9\n",
    "\tvect__max_df: 0.5\n",
    "\tvect__max_features: 1500\n",
    "\tvect__min_df: 0\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best score: 0.471\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 12\n",
    "    \n",
    "done in 1479.661s\n",
    "Best score: 0.473\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 18\n",
    "    \n",
    "done in 1254.545s\n",
    "Best score: 0.481\n",
    "Best parameters set:\n",
    "\tclf__estimator__C: 18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print('grid search in action...')\n",
    "t0 = time()\n",
    "gs.fit(texts, Y)\n",
    "print(\"done in %0.3fs\" % (time() - t0))\n",
    "print(\"Best score: %0.3f\" % gs.best_score_)\n",
    "print(\"Best parameters set:\")\n",
    "best_parameters = gs.best_estimator_.get_params()\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'GridSearchCV' object has no attribute 'cv_results_'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-f389f6fa6ff7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mscore_matrix\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcv_results_\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscore_matrix\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'GridSearchCV' object has no attribute 'cv_results_'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "score_matrix = gs.cv_results_\n",
    "df = pd.DataFrame(score_matrix)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-b274744be3e7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtexts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Modello creato, verifico il test set'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/utente_locale/anaconda3/lib/python3.5/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    266\u001b[0m             \u001b[0mThis\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m         \"\"\"\n\u001b[0;32m--> 268\u001b[0;31m         \u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfit_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    269\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_final_estimator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    270\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_final_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/utente_locale/anaconda3/lib/python3.5/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    232\u001b[0m                 \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"fit_transform\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m                 \u001b[0mXt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params_steps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m                 \u001b[0mXt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params_steps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/utente_locale/anaconda3/lib/python3.5/site-packages/sklearn/feature_extraction/text.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, raw_documents, y)\u001b[0m\n\u001b[1;32m    822\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    823\u001b[0m         vocabulary, X = self._count_vocab(raw_documents,\n\u001b[0;32m--> 824\u001b[0;31m                                           self.fixed_vocabulary_)\n\u001b[0m\u001b[1;32m    825\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    826\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinary\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/utente_locale/anaconda3/lib/python3.5/site-packages/sklearn/feature_extraction/text.py\u001b[0m in \u001b[0;36m_count_vocab\u001b[0;34m(self, raw_documents, fixed_vocab)\u001b[0m\n\u001b[1;32m    750\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mdoc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mraw_documents\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    751\u001b[0m             \u001b[0mfeature_counter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 752\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mfeature\u001b[0m \u001b[0;32min\u001b[0m \u001b[0manalyze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    753\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    754\u001b[0m                     \u001b[0mfeature_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvocabulary\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeature\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-2-9d490f0f928f>\u001b[0m in \u001b[0;36mget_tokenizer1\u001b[0;34m(doc)\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mita_stopwords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;31m# stemma le parole\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mita_stemmer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-2-9d490f0f928f>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mita_stopwords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;31m# stemma le parole\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mita_stemmer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/utente_locale/anaconda3/lib/python3.5/site-packages/nltk/stem/snowball.py\u001b[0m in \u001b[0;36mstem\u001b[0;34m(self, word)\u001b[0m\n\u001b[1;32m   2179\u001b[0m         \u001b[0;31m# Every occurrence of 'u' and 'i'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2180\u001b[0m         \u001b[0;31m# between vowels is put into upper case.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2181\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2182\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mword\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__vowels\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mword\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__vowels\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2183\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mword\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"u\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# controlla se esiste il modello per evitare di ricrearlo ogni volta\n",
    "if False: #os.path.exists(os.path.join(DATA_FOLDER, 'OneVsRestClassifier')):\n",
    "    with open(os.path.join(DATA_FOLDER, 'OneVsRestClassifier'), mode='rb') as modelfile:\n",
    "        model = dill.load(modelfile)\n",
    "\n",
    "else:\n",
    "    model = pipeline.fit(texts, Y)\n",
    "    print('Modello creato, verifico il test set')\n",
    "\n",
    "\n",
    "    with open(os.path.join(DATA_FOLDER, 'OneVsRestClassifier'), mode='wb') as modelfile:\n",
    "        dill.dump(model, modelfile)\n",
    "        \n",
    "print('Time to load/create the model: ', time.process_time()-last_time)\n",
    "last_time = time.process_time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time to load create the model:  0.022760550000043622\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "print('Time to load create the model: ', time.process_time()-last_time)\n",
    "last_time = time.process_time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['attivazione', 'canone rai', 'cessazione', 'contratto', 'credito', 'domiciliazione', 'fatturazione', 'gr-invito a chiamare - focus', 'non di competenza', 'richiamate']\n"
     ]
    }
   ],
   "source": [
    "# importo il test set per la valutazione del modello\n",
    "unito_test = []\n",
    "names_test = []\n",
    "test_texts = list()\n",
    "test_labels = list()\n",
    "#raise SystemExit\n",
    "\n",
    "# load test data\n",
    "# carica il test set\n",
    "with codecs.open(os.path.join(DATA_FOLDER, test_db_name), 'r', errors='ignore') as file:\n",
    "    reader = csv.reader(file, delimiter=';')\n",
    "    etichette = next(reader)[3:]\n",
    "    print(etichette)\n",
    "    for row in reader:\n",
    "        # load documents\n",
    "        test_texts.append((row[1]+row[2]).replace('O:', '').replace('C:', ''))\n",
    "        # load names\n",
    "        names_test.append(row[0])\n",
    "        # load labels\n",
    "        test_labels.append(list(map(lambda x: int(x),row[-10:])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'test_labels' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-448990f596fa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mY_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_labels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;31m# apply model to data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mpredicted_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_texts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#np_array\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# print evaluation report\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'test_labels' is not defined"
     ]
    }
   ],
   "source": [
    "Y_test = np.array(test_labels)\n",
    "# apply model to data\n",
    "predicted_labels = model.predict(test_texts) #np_array\n",
    "\n",
    "# print evaluation report\n",
    "evaluation_report.print_evaluation(Y_test, predicted_labels)\n",
    "for i in range(predicted_labels.shape[1]):\n",
    "    print(etichette[i], np.sum(predicted_labels[:,i]))\n",
    "print('Decision function: ')\n",
    "\n",
    "scores = model.decision_function(test_texts) #np_array\n",
    "\n",
    "print('\\n')\n",
    "print('accuracy', accuracy_score(Y_test, predicted_labels))\n",
    "print('precision micro',precision_score(Y_test, predicted_labels, average='micro') )\n",
    "print('recall micro',recall_score(Y_test, predicted_labels, average='micro'))\n",
    "print('f1 micro', f1_score(Y_test, predicted_labels, average='micro'))\n",
    "print('precision macro', precision_score(Y_test, predicted_labels, average='macro') )\n",
    "print('recall macro', recall_score(Y_test, predicted_labels, average='macro'))\n",
    "print('f1 macro', f1_score(Y_test, predicted_labels, average='macro'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "media score positivi predetti:    1.80927241817\n",
      "media score negativi predetti:    -0.972798737\n",
      "std score positivi predetti:      1.41654818067\n",
      "std score negativi predetti:      0.720769286264\n"
     ]
    }
   ],
   "source": [
    "# score della cat credito \n",
    "cred_score = scores[:,4]\n",
    "cred_pred = predicted_labels[:,4]\n",
    "#print(len(cred_score), cred_pred)\n",
    "\n",
    "positive = cred_score[cred_pred == 1]\n",
    "neg = cred_score[cred_pred == 0]\n",
    "#print(positive)\n",
    "print('media score positivi predetti:   ', positive.mean())\n",
    "print('media score negativi predetti:   ', neg.mean())\n",
    "print('std score positivi predetti:     ', positive.std())\n",
    "print('std score negativi predetti:     ', neg.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "156 156\n",
      "media score positivi:    1.98479949207\n",
      "media score negativi:    -0.129678476263\n",
      "std score positivi:      1.72169001482\n",
      "std score negativi:      1.31721210827\n"
     ]
    }
   ],
   "source": [
    "# confronto gli score dei doc annotati a mano con quelli assegnati dal modello\n",
    "cred_score = scores[:,4]\n",
    "cred_ammano = Y_test[:,4]\n",
    "#print(len(cred_score), cred_ammano)\n",
    "\n",
    "positive_ammano = cred_score[cred_ammano == 1]\n",
    "neg_ammano = cred_score[cred_ammano == 0]\n",
    "print(sum(cred_ammano), len(positive_ammano))\n",
    "print('media score positivi:   ' , positive_ammano.mean())\n",
    "print('media score negativi:   ', neg_ammano.mean())\n",
    "print('std score positivi:     ', positive_ammano.std())\n",
    "print('std score negativi:     ', neg_ammano.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-837e4bc48485>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# confronto gli error rate per ciascuna classe\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# predico il training set\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mY_pred_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtexts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0maccuracy_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "# confronto gli error rate per ciascuna classe\n",
    "# predico il training set\n",
    "Y_pred_train = model.predict(texts)\n",
    "\n",
    "accuracy_test = []\n",
    "accuracy_train = []\n",
    "for i in range(predicted_labels.shape[1]):\n",
    "    accuracy_test.append(accuracy_score(Y_test[:,i], predicted_labels[:,i]))\n",
    "    accuracy_train.append(accuracy_score(Y[:,i], Y_pred_train[:,i]))\n",
    "    \n",
    "\n",
    "print ('error rate:')\n",
    "print('{:>35} {:>18}'.format('test','train'))\n",
    "for n in range(len(etichette[:10])):\n",
    "      print('{:>23}{:>20}{:>10}'.format(etichette[n], round(1-accuracy_test[n],3), round(1-accuracy_train[n],3)))\n",
    "    \n",
    "print('''\n",
    "    COME INTERPRETARE:\n",
    "    \n",
    "    Training error is low but is much lower than testing error - overfitting\n",
    "    Both errors are low - ok\n",
    "    Both errors are high - underfitting\n",
    "    Training error is high but testing is low - error in implementation or very small dataset\n",
    "''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# ============================================    CHI2     ===================================================\n",
    "# build training set\n",
    "# stopwords, tf-idf\n",
    "\n",
    "count_vect = CountVectorizer(texts, analyzer='word', stop_words=ita_stopwords)\n",
    "X = count_vect.fit_transform(texts)\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tf_transformer = TfidfTransformer(use_idf=True).fit(X)\n",
    "X_train_tf = tf_transformer.transform(X)\n",
    "# dictionary for explore features selected k:index, v:feature name\n",
    "inverse_vocab = { v:k for k,v in count_vect.vocabulary_.items() }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# statistical feature selection with chi2\n",
    "# rank features according to the average or the maximum Chi-squared score across all labels\n",
    "# led to most of the best classifiers while using less features. \n",
    "# fonte(http://ceur-ws.org/Vol-1094/bioasq2013_submission_8.pdf)\n",
    "from sklearn.feature_selection import chi2, SelectKBest\n",
    "\n",
    "def feature_selector(X,y):\n",
    "    '''\n",
    "    input:\n",
    "    X: DT matrix\n",
    "    y: labels vector\n",
    "    n_features_perc: float: e.g. default 1.0, all features are retained\n",
    "    \n",
    "    output:\n",
    "    X_reduced: DT matrix with only n_features\n",
    "    '''\n",
    "    n_features = 1500\n",
    "    features_scores = [] \n",
    "    for cat in range(y.shape[1]):\n",
    "        selector = SelectKBest(chi2, k='all')\n",
    "        selector.fit(X, y[:,cat])\n",
    "        features_scores.append(selector.scores_) # 10xn_feat matrix, score of each featrue for each class\n",
    "    #print('selected_features:\\n',len(features_scores), len(features_scores[0]))\n",
    "\n",
    "    sorted_scores = sorted([item for sublist in features_scores for item in sublist], reverse = True)\n",
    "    best_scores = sorted_scores[:n_features]\n",
    "    #print(len(best_scores))\n",
    "    \n",
    "    # retrieve selected feature indices\n",
    "    A = np.asarray(features_scores)\n",
    "    selected_features = np.where(A > min(best_scores)) #retrieve indices of the best feature\n",
    "    selected_features = selected_features[1]\n",
    "    print(len(selected_features))\n",
    "    #print('how many features we delete: \\n', len(selected_features) - len(set(selected_features)), 'su',len(selected_features))\n",
    "\n",
    "    # reduce DT matrix dimension\n",
    "    selected_features = list(selected_features)\n",
    "    X_reduced = X[:,selected_features] # cutting matrix features columns\n",
    "    return X_reduced, selected_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1499\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(24379, 1499)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_reduced, selected_feat = feature_selector(X_train_tf, Y)\n",
    "X_train_reduced.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[161, 1196, 1221, 1313, 2570, 2974, 2997, 3164, 3494, 4089]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_feat[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# building the model\n",
    "import time\n",
    "model = None\n",
    "start = time.process_time()    \n",
    "model = OneVsRestClassifier(LinearSVC(C=10), n_jobs=1)\n",
    "model.fit(X_train_reduced, Y)\n",
    "last_time = time.process_time()\n",
    "print('tempo per fittare il modello: ', last_time - start/60)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reduced:  (424, 49) original:  (424, 90906)\n"
     ]
    }
   ],
   "source": [
    "# build the test set to make predictions keeping attention for stemming\n",
    "#count_vect = CountVectorizer(test_texts, analyzer='word', stop_words=ita_stopwords)\n",
    "X_test = count_vect.transform(test_texts)\n",
    "y_test = np.asarray(test_labels)\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tf_transformer = TfidfTransformer(use_idf=True).fit(X_test)\n",
    "X_test_tf = tf_transformer.transform(X_test)\n",
    "\n",
    "X_test_reduced = X_test_tf[:,selected_feat]\n",
    "print('reduced: ', X_test_reduced.shape, 'original: ', X_test_tf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 0 1 0 0 1 0 0 0]\n",
      " [0 0 0 0 1 0 1 0 0 0]\n",
      " [1 0 0 0 0 0 1 1 0 0]\n",
      " [0 0 0 0 0 0 1 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 1 0]]\n",
      "[[0 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 1 0 1 0 0 0]\n",
      " [1 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 1 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 0]]\n",
      "Confusion matrix attivazione\n",
      "[[333  18]\n",
      " [ 59  14]]\n",
      "Confusion matrix canone rai\n",
      "[[359   2]\n",
      " [ 52  11]]\n",
      "Confusion matrix cessazione\n",
      "[[413   1]\n",
      " [  9   1]]\n",
      "Confusion matrix contratto\n",
      "[[329  21]\n",
      " [ 45  29]]\n",
      "Confusion matrix credito\n",
      "[[177  85]\n",
      " [ 29 133]]\n",
      "Confusion matrix domiciliazione\n",
      "[[398   5]\n",
      " [ 20   1]]\n",
      "Confusion matrix fatturazione\n",
      "[[275  25]\n",
      " [ 94  30]]\n",
      "Confusion matrix gr-invito a chiamare - focus\n",
      "[[260 141]\n",
      " [  8  15]]\n",
      "Confusion matrix non di competenza\n",
      "[[373   1]\n",
      " [ 48   2]]\n",
      "Confusion matrix richiamate\n",
      "[[417   4]\n",
      " [  3   0]]\n",
      "                              precision    recall  f1-score   support\n",
      "\n",
      "                 attivazione       0.44      0.19      0.27        73\n",
      "                  canone rai       0.85      0.17      0.29        63\n",
      "                  cessazione       0.50      0.10      0.17        10\n",
      "                   contratto       0.58      0.39      0.47        74\n",
      "                     credito       0.61      0.82      0.70       162\n",
      "              domiciliazione       0.17      0.05      0.07        21\n",
      "                fatturazione       0.55      0.24      0.34       124\n",
      "gr-invito a chiamare - focus       0.10      0.65      0.17        23\n",
      "           non di competenza       0.67      0.04      0.08        50\n",
      "                  richiamate       0.00      0.00      0.00         3\n",
      "\n",
      "                 avg / total       0.56      0.39      0.39       603\n",
      "\n",
      "Doc test set:  424\n",
      "attivazione 32\n",
      "canone rai 13\n",
      "cessazione 2\n",
      "contratto 50\n",
      "credito 218\n",
      "domiciliazione 6\n",
      "fatturazione 55\n",
      "gr-invito a chiamare - focus 156\n",
      "non di competenza 3\n",
      "richiamate 4\n",
      "Decision function: \n",
      "attivazione 32\n",
      "canone rai 13\n",
      "cessazione 2\n",
      "contratto 50\n",
      "credito 218\n",
      "domiciliazione 6\n",
      "fatturazione 55\n",
      "gr-invito a chiamare - focus 156\n",
      "non di competenza 3\n",
      "richiamate 4\n",
      "Decision function: \n"
     ]
    }
   ],
   "source": [
    "# predictions\n",
    "# apply model to data\n",
    "predicted_labels = model.predict(X_test_reduced) #np_array\n",
    "\n",
    "# print evaluation report\n",
    "evaluation_report.print_evaluation(y_test, predicted_labels)\n",
    "for i in range(predicted_labels.shape[1]):\n",
    "    print(etichette[i], np.sum(predicted_labels[:,i]))\n",
    "print('Decision function: ')\n",
    "\n",
    "# scores = model.decision_function(test_texts) #np_array\n",
    "\n",
    "# print('\\n')\n",
    "# print('accuracy', accuracy_score(Y_test, predicted_labels))\n",
    "# print('precision micro',precision_score(Y_test, predicted_labels, average='micro') )\n",
    "# print('recall micro',recall_score(Y_test, predicted_labels, average='micro'))\n",
    "# print('f1 micro', f1_score(Y_test, predicted_labels, average='micro'))\n",
    "# print('precision macro', precision_score(Y_test, predicted_labels, average='macro') )\n",
    "# print('recall macro', recall_score(Y_test, predicted_labels, average='macro'))\n",
    "# print('f1 macro', f1_score(Y_test, predicted_labels, average='macro'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}