p4_SentenceSentimentPrediction_10K_version 04MultipleRun_2018May.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentence sentiment prediction using three models\n",
    "\n",
    "\n",
    "Some portions of the program adapted from coursera Sequence Modeling course emojify homework.\n",
    "Change from previous version: use keras for both neural network models.\n",
    "\n",
    "First run: train/test proportions were incorrectly set to yield small training sets (300) and large test sets (700). Increasing the training sets may benefit the neural networks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/murataydogdu/Desktop/TextualAnalysis\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "from random import shuffle\n",
    "import csv\n",
    "import time\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy.random import seed\n",
    "from tensorflow import set_random_seed\n",
    "seed(1)\n",
    "set_random_seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, precision_recall_fscore_support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import TweetTokenizer, sent_tokenize\n",
    "tokenizer_words = TweetTokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model, Sequential, load_model\n",
    "from keras.layers import Dense, Input, Dropout, LSTM, Activation, Flatten\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.preprocessing import sequence\n",
    "from keras.initializers import glorot_uniform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GloVe: Global vectors for Word representation\n",
    "https://nlp.stanford.edu/projects/glove/\n",
    "\n",
    "Convert an input sentence into the word vector representation, which then get averaged together. We will use pretrained 50-dimensional GloVe embeddings. Run the following cell to load the `word_to_vec_map`, which contains all the vector representations.\n",
    "\n",
    "This will loaded:\n",
    "- `word_to_index`: dictionary mapping from words to their indices in the vocabulary (400,001 words, with the valid indices ranging from 0 to 400,000)\n",
    "- `index_to_word`: dictionary mapping from indices to their corresponding words in the vocabulary\n",
    "- `word_to_vec_map`: dictionary mapping words to their GloVe vector representation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_glove_vecs(glove_file):\n",
    "    with open(glove_file, 'r') as f:\n",
    "        words = set()\n",
    "        word_to_vec_map = {}\n",
    "        for line in f:\n",
    "            line = line.strip().split()\n",
    "            curr_word = line[0]\n",
    "            words.add(curr_word)\n",
    "            word_to_vec_map[curr_word] = np.array(line[1:], dtype=np.float64)\n",
    "        \n",
    "        i = 1\n",
    "        words_to_index = {}\n",
    "        index_to_words = {}\n",
    "        for w in sorted(words):\n",
    "            words_to_index[w] = i\n",
    "            index_to_words[i] = w\n",
    "            i = i + 1\n",
    "    return words_to_index, index_to_words, word_to_vec_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_to_index, index_to_word, word_to_vec_map = read_glove_vecs('data/glove.6B.50d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_sent(input_sent):\n",
    "    # This tokenization retains numbers and punctuation.\n",
    "    # It makes sure the tokens retained from each sentence are in the dictionary.\n",
    "    in0 = tokenizer_words.tokenize(input_sent)\n",
    "    inp = [x.encode('UTF8').lower() for x in in0]\n",
    "    # This happens with sentences that have commas in them.  \n",
    "    # In that case, the first and last character of the line is a quotation.\n",
    "    if inp[-1] == '\"':\n",
    "        inp = inp[:-1] \n",
    "    #print (\"Input:\", inp)\n",
    "\n",
    "    valid_sent = []\n",
    "    for word in inp:\n",
    "        try:\n",
    "            exists = word_to_index[word]\n",
    "            valid_sent.append(word)\n",
    "        except:\n",
    "            # Ignore this word \n",
    "            x = 0\n",
    "    valid_sent = ' '.join(valid_sent)        \n",
    "    #print (\"Output:\", valid_sent)\n",
    "    return valid_sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the labelled input data\n",
    "# which will be used for training and testing.\n",
    "# The sentences are already shuffled so sentences from one filing are not in sequence.\n",
    "# Not all of the sentences are labelled:\n",
    "# those that are labelled have the first field coded as 0/1/2\n",
    "# the rest are labelled 5.\n",
    "with open('new_trtest_10K_labelled.csv', 'rb') as f:\n",
    "    reader = csv.reader(f)\n",
    "    sentences = list(reader)\n",
    "    \n",
    "# Grab the labelled sentences and store them in arrays\n",
    "y, trfile, sent_cnt, sent_ind, word_cnt, full_s = [],[],[],[],[],[]\n",
    "for sent in sentences:\n",
    "    # This is for when the sentence has commas in it. In that case the sentence will be split.\n",
    "    full_sent = \",\".join([str(i) for i in sent])\n",
    "    \n",
    "    # Split the sentence using the split characters: ' _*_ '\n",
    "    inp = full_sent.split(' _*_ ')\n",
    "    y_ = int(inp[0]) \n",
    "    #print y_\n",
    "    if y_ >= 0 and y_ <= 2: # Not all sentences are labelled: grab the labelled ones only\n",
    "        trfile_ = inp[1].replace('\"', '')\n",
    "        sent_cnt_ = int(inp[2])\n",
    "        sent_ind_ = int(inp[3])\n",
    "        word_cnt_ = int(inp[4])\n",
    "        # Split the sentence into words, and keep the words that are in GloVe only\n",
    "        full_sent_ = parse_sent(inp[5])\n",
    "        #print y_, trfile_, sent_cnt_, sent_ind_, word_cnt_ , full_sent_\n",
    "\n",
    "        y.append(y_)\n",
    "        trfile.append(trfile_)\n",
    "        sent_cnt.append(sent_cnt_)\n",
    "        sent_ind.append(sent_ind_)\n",
    "        word_cnt.append(word_cnt_)\n",
    "        full_s.append([full_sent_])         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([0, 1, 2]), array([193, 479, 328]))\n"
     ]
    }
   ],
   "source": [
    "print np.unique(y, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Randomly select training and test observations\n",
    "# I used this instead of sckit_learn so that I can use the\n",
    "# indices on other data items if needed\n",
    "# returns arrays \n",
    "def TrTestSet(X, y, trlen):\n",
    "    all_ind = range(0, len(X))\n",
    "    #print all_ind\n",
    "    train_ind = random.sample(range(0, len(X)-1), trlen)\n",
    "    test_ind = [i for i in all_ind if i not in train_ind]\n",
    "\n",
    "    X_test= [X[i][0] for i in test_ind]\n",
    "    y_test= np.asarray([y[i] for i in test_ind])\n",
    "    X_train= [X[i][0] for i in train_ind]\n",
    "    X_train = np.asarray(X_train)\n",
    "    X_test = np.asarray(X_test)\n",
    "    y_train= np.asarray([y[i] for i in train_ind])\n",
    "    return X_test, X_train, y_test, y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_metrics(actual, predicted, cnt , m , t):   \n",
    "    res = {}\n",
    "    res['acc'] = accuracy_score(actual, predicted)\n",
    "    cls = classification_report(actual, predicted)\n",
    "    con = confusion_matrix(actual, predicted)\n",
    "    precision,recall,fscore,support = precision_recall_fscore_support(actual, predicted)\n",
    "    for i in range(3): # Rows: predicted\n",
    "        varname = 'prec_'+str(i)\n",
    "        res[varname] = precision[i]\n",
    "        varname = 'rec_'+str(i)\n",
    "        res[varname] = recall[i]\n",
    "        varname = 'f1_'+str(i)\n",
    "        res[varname] = fscore[i]\n",
    "        varname = 'sup_'+str(i)\n",
    "        res[varname] = support[i]        \n",
    "    precision,recall,fscore,support = precision_recall_fscore_support(actual, predicted, average='weighted')\n",
    "    res['prec_all'] = precision\n",
    "    res['rec_all'] = recall\n",
    "    res['f1_all'] = fscore\n",
    "    res['sup_all'] = res['sup_0']+res['sup_1']+res['sup_2']\n",
    "    for i in range(3): # Rows: predicted\n",
    "        for j in range(3): # Columns: actual\n",
    "            varname = 'A'+str(i)+'P'+str(j)\n",
    "            res[varname] = con[i,j]\n",
    "    res['rnd_ct'] = cnt\n",
    "    res['model'] = m\n",
    "    res['trtest'] = t\n",
    "    return pd.Series(res).to_frame().transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sentiment prediction based on Loughran - McDonald dictionary of financial words.\n",
    "# In this \"model\" there is no training.\n",
    "# Sentiment: Positive / Neutral / Negative\n",
    "positives = open('data/LoughranMcDonald_Positive.csv', \"r\").readlines()\n",
    "positive = [pos.strip().lower().split(',')[0] for pos in positives]\n",
    "negatives = open('data/LoughranMcDonald_Negative.csv', \"r\").readlines()\n",
    "negative = [neg.strip().lower().split(',')[0] for neg in negatives]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LM_predict(X):\n",
    "    bins = np.array([-0.1, 0.10, 1.0])\n",
    "    tot_, perc_ = [], []\n",
    "    for s in X:\n",
    "        words = s.split()\n",
    "        #print (words)\n",
    "        pos,neg = [], []\n",
    "        for word in words:\n",
    "            if word in (positive):\n",
    "                pos.append(word)\n",
    "            if word in (negative):\n",
    "                neg.append(word)  \n",
    "        pos_len = len(pos)\n",
    "        neg_len = len(neg)\n",
    "        tot =  pos_len + neg_len\n",
    "        if tot == 0:\n",
    "            perc = 0\n",
    "        else:\n",
    "            perc = 1.0 * (pos_len - neg_len) / tot\n",
    "        tot_.append(tot)\n",
    "        perc_.append(perc)\n",
    "    pred_ = np.digitize(perc_, bins, right = True)\n",
    "    return (tot_, perc_, pred_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_to_avg(sentences, word_to_vec_map):\n",
    "    avgs = []\n",
    "    for sentence in sentences:\n",
    "        words = sentence.split()\n",
    "        # Initialize the average word vector, should have the same shape as your word vectors.\n",
    "        avg = np.zeros((50,))\n",
    "        # Step 2: average the word vectors. You can loop over the words in the list \"words\".\n",
    "        for w in words:\n",
    "            avg += word_to_vec_map[w]\n",
    "        avg = avg / len(words)\n",
    "        avgs.append(avg)\n",
    "    np_avgs = np.array(avgs)    \n",
    "    return np_avgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Note: when using the categorical_crossentropy loss, your targets should be in categorical format (e.g. if you have 10 classes, the target for each sample should be a 10-dimensional vector that is all-zeros except for a 1 at the index corresponding to the class of the sample). In order to convert integer targets into categorical targets, you can use the Keras utility to_categorical:\n",
    "        \n",
    "#from keras.utils.np_utils import to_categorical\n",
    "\n",
    "#categorical_labels = to_categorical(int_labels, num_classes=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GV_predict(X_train, X_test, y_train, y_test):\n",
    "    # For the GV model, we need word vectors averaged into a single vector per sentence\n",
    "    X_train_avg = sentence_to_avg(X_train, word_to_vec_map)\n",
    "    X_test_avg = sentence_to_avg(X_test, word_to_vec_map)\n",
    "    Y_train_oh = np.eye(3)[np.asarray(y_train).reshape(-1)]\n",
    "    Y_test_oh = np.eye(3)[np.asarray(y_test).reshape(-1)]    \n",
    "\n",
    "    GV_model = Sequential([\n",
    "    Dense(50, input_shape=(50,)),\n",
    "    Dense(3),\n",
    "    Activation('softmax')\n",
    "    ])\n",
    "    #GV_model.summary()\n",
    "    GV_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])    \n",
    "\n",
    "    seed(1)\n",
    "    set_random_seed(1)\n",
    "    np.random.seed(1)\n",
    "    GV_model.fit(X_train_avg, Y_train_oh, epochs = 200, batch_size = 32, shuffle=True, verbose=0)  \n",
    "    \n",
    "    # Simple GloVe Neural Network (GV) Model \n",
    "    # performance on the training and test sets\n",
    "    #loss, acc = GV_model.evaluate(X_train_avg, Y_train_oh)\n",
    "    pred_train = np.argmax(GV_model.predict(X_train_avg), axis=1)\n",
    "    # Save the model so that it can be used on full Item 7s later\n",
    "    GV_model.save('GV_model.h5')  # creates an HDF5 file '\n",
    "    #print('GV model performance')\n",
    "    #print 'Training set accuracy:','%0.2f' % acc\n",
    "    #loss, acc = GV_model.evaluate(X_test_avg, Y_test_oh)\n",
    "    pred_test = np.argmax(GV_model.predict(X_test_avg), axis=1)\n",
    "    #print 'Test set accuracy:','%0.2f' % acc\n",
    "    from IPython.display import SVG\n",
    "    from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "    SVG(model_to_dot(GV_model).create(prog='dot', format='svg'))    \n",
    "    \n",
    "    return pred_train, pred_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentences_to_indices(X, word_to_index, max_len):\n",
    "    m = X.shape[0]                                   # number of training examples\n",
    "    # Initialize X_indices as a numpy matrix of zeros and the correct shape \n",
    "    X_indices = np.zeros((m,max_len))\n",
    "    \n",
    "    for i in range(m):                               # loop over training examples   \n",
    "        # Split the ith training sentence into a list of words.\n",
    "        sentence_words =X[i].split()\n",
    "        #print len(sentence_words), sentence_words\n",
    "        # Initialize j to 0\n",
    "        j = 0\n",
    "        \n",
    "        # Loop over the words of sentence_words\n",
    "        for w in sentence_words:\n",
    "            # Set the (i,j)th entry of X_indices to the index of the correct word.\n",
    "            X_indices[i, j] = word_to_index[w]\n",
    "            # Increment j to j + 1\n",
    "            j = j + 1    \n",
    "    return X_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pretrained_embedding_layer(word_to_vec_map, word_to_index):\n",
    "    vocab_len = len(word_to_index) + 1                  # adding 1 to fit Keras embedding (requirement)\n",
    "    emb_dim = word_to_vec_map[\"cucumber\"].shape[0]      # define dimensionality of your GloVe word vectors (= 50)\n",
    "    \n",
    "    # Initialize the embedding matrix as a numpy array of zeros of shape (vocab_len, dimensions of word vectors = emb_dim)\n",
    "    emb_matrix = np.zeros((vocab_len, emb_dim))\n",
    "    \n",
    "    # Set each row \"index\" of the embedding matrix to be the word vector representation of the \"index\"th word of the vocabulary\n",
    "    for word, index in word_to_index.items():\n",
    "        emb_matrix[index, :] = word_to_vec_map[word]\n",
    "\n",
    "    # Define Keras embedding layer with the correct output/input sizes, make it trainable. \n",
    "    # Use Embedding(...). Make sure to set trainable=False. \n",
    "    embedding_layer = Embedding(vocab_len, emb_dim, trainable=False)\n",
    "\n",
    "    # Build the embedding layer, it is required before setting the weights of the embedding layer. \n",
    "    # Do not modify the \"None\".\n",
    "    embedding_layer.build((None,))\n",
    "    \n",
    "    # Set the weights of the embedding layer to the embedding matrix. Your layer is now pretrained.\n",
    "    embedding_layer.set_weights([emb_matrix])\n",
    "    \n",
    "    return embedding_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(input_shape, word_to_vec_map, word_to_index):\n",
    "    \"\"\"\n",
    "    Function creating the Emojify-v2 model's graph.\n",
    "    \n",
    "    Arguments:\n",
    "    input_shape -- shape of the input, usually (max_len,)\n",
    "    word_to_vec_map -- dictionary mapping every word in a vocabulary into its 50-dimensional vector representation\n",
    "    word_to_index -- dictionary mapping from words to their indices in the vocabulary (400,001 words)\n",
    "\n",
    "    Returns:\n",
    "    model -- a model instance in Keras\n",
    "    \"\"\"\n",
    "    # Define sentence_indices as the input of the graph, it should be of shape input_shape and dtype 'int32' (as it contains indices).\n",
    "    sentence_indices = Input(input_shape, dtype = 'int32')\n",
    "    # Create the embedding layer pretrained with GloVe Vectors (≈1 line)\n",
    "    embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index)\n",
    "    # Propagate sentence_indices through your embedding layer, you get back the embeddings\n",
    "    embeddings = embedding_layer(sentence_indices)   \n",
    "    # Propagate the embeddings through an LSTM layer with 128-dimensional hidden state\n",
    "    # Be careful, the returned output should be a batch of sequences.\n",
    "    X = LSTM(128, return_sequences = True)(embeddings)\n",
    "    # Add dropout with a probability of 0.5\n",
    "    X = Dropout(0.5)(X)\n",
    "    # Propagate X trough another LSTM layer with 128-dimensional hidden state\n",
    "    # Be careful, the returned output should be a single hidden state, not a batch of sequences.\n",
    "    X = LSTM(128, return_sequences = False)(X)\n",
    "    # Add dropout with a probability of 0.5\n",
    "    X = Dropout(0.5)(X)\n",
    "    # Propagate X through a Dense layer with softmax activation to get back a batch of 3-dimensional vectors.\n",
    "    X = Dense(3)(X)\n",
    "    # Add a softmax activation\n",
    "    X = Activation('softmax')(X)\n",
    "    # Create Model instance which converts sentence_indices into X.\n",
    "    model = Model(inputs = sentence_indices, outputs = X)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def NN_predict(X_train, X_test, y_train, y_test):\n",
    "    seed(1)\n",
    "    set_random_seed(1)\n",
    "    np.random.seed(1)\n",
    "    NN_model = model((maxLen,), word_to_vec_map, word_to_index)\n",
    "    #NN_model.summary()\n",
    "    NN_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    X_train_indices = sentences_to_indices(X_train, word_to_index, maxLen)\n",
    "    Y_train_oh = np.eye(3)[np.asarray(y_train).reshape(-1)]\n",
    "    Y_test_oh = np.eye(3)[np.asarray(y_test).reshape(-1)]    \n",
    "\n",
    "    NN_model.fit(X_train_indices, Y_train_oh, epochs = 30, batch_size = 32, shuffle=True, verbose = 0)\n",
    "    # Save the model so that it can be used on full Item 7s later\n",
    "    NN_model.save('NN_model.h5')  # creates an HDF5 file '\n",
    "\n",
    "    #del model  # deletes the existing model\n",
    "    #model = load_model('model2.h5') # \n",
    "    # LSTM Neural Network (NN) Model\n",
    "    X_train_indices = sentences_to_indices(X_train, word_to_index, max_len = maxLen)\n",
    "    #loss, acc = NN_model.evaluate(X_train_indices, Y_train_oh)\n",
    "    pred_train = np.argmax(NN_model.predict(X_train_indices), axis=1)\n",
    "\n",
    "    X_test_indices = sentences_to_indices(X_test, word_to_index, max_len = maxLen)\n",
    "    #loss, acc = NN_model.evaluate(X_test_indices, Y_test_oh)\n",
    "    pred_test = np.argmax(NN_model.predict(X_test_indices), axis=1)\n",
    "    #print 'Test set accuracy:','%0.2f' % acc\n",
    "    #del NN_model    \n",
    "    return pred_train, pred_test    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial: 1 Tue May 29 19:55:13 2018\n",
      "Finished: 1 LM TR\n",
      "Finished: 1 LM TE\n",
      "Finished: 1 GV TR\n",
      "Finished: 1 GV TE\n",
      "Finished: 1 NN TR\n",
      "Finished: 1 NN TE\n",
      "Trial end: Tue May 29 19:57:51 2018\n",
      "Trial: 2 Tue May 29 19:57:51 2018\n",
      "Finished: 2 LM TR\n",
      "Finished: 2 LM TE\n",
      "Finished: 2 GV TR\n",
      "Finished: 2 GV TE\n",
      "Finished: 2 NN TR\n",
      "Finished: 2 NN TE\n",
      "Trial end: Tue May 29 20:00:29 2018\n",
      "Trial: 3 Tue May 29 20:00:29 2018\n",
      "Finished: 3 LM TR\n",
      "Finished: 3 LM TE\n",
      "Finished: 3 GV TR\n",
      "Finished: 3 GV TE\n",
      "Finished: 3 NN TR\n",
      "Finished: 3 NN TE\n",
      "Trial end: Tue May 29 20:02:56 2018\n",
      "Trial: 4 Tue May 29 20:02:56 2018\n",
      "Finished: 4 LM TR\n",
      "Finished: 4 LM TE\n",
      "Finished: 4 GV TR\n",
      "Finished: 4 GV TE\n",
      "Finished: 4 NN TR\n",
      "Finished: 4 NN TE\n",
      "Trial end: Tue May 29 20:05:23 2018\n",
      "Trial: 5 Tue May 29 20:05:23 2018\n",
      "Finished: 5 LM TR\n",
      "Finished: 5 LM TE\n",
      "Finished: 5 GV TR\n",
      "Finished: 5 GV TE\n",
      "Finished: 5 NN TR\n",
      "Finished: 5 NN TE\n",
      "Trial end: Tue May 29 20:07:50 2018\n",
      "Trial: 6 Tue May 29 20:07:50 2018\n",
      "Finished: 6 LM TR\n",
      "Finished: 6 LM TE\n",
      "Finished: 6 GV TR\n",
      "Finished: 6 GV TE\n",
      "Finished: 6 NN TR\n",
      "Finished: 6 NN TE\n",
      "Trial end: Tue May 29 20:10:19 2018\n",
      "Trial: 7 Tue May 29 20:10:19 2018\n",
      "Finished: 7 LM TR\n",
      "Finished: 7 LM TE\n",
      "Finished: 7 GV TR\n",
      "Finished: 7 GV TE\n",
      "Finished: 7 NN TR\n",
      "Finished: 7 NN TE\n",
      "Trial end: Tue May 29 20:12:50 2018\n",
      "Trial: 8 Tue May 29 20:12:50 2018\n",
      "Finished: 8 LM TR\n",
      "Finished: 8 LM TE\n",
      "Finished: 8 GV TR\n",
      "Finished: 8 GV TE\n",
      "Finished: 8 NN TR\n",
      "Finished: 8 NN TE\n",
      "Trial end: Tue May 29 20:15:19 2018\n",
      "Trial: 9 Tue May 29 20:15:19 2018\n",
      "Finished: 9 LM TR\n",
      "Finished: 9 LM TE\n",
      "Finished: 9 GV TR\n",
      "Finished: 9 GV TE\n",
      "Finished: 9 NN TR\n",
      "Finished: 9 NN TE\n",
      "Trial end: Tue May 29 20:17:50 2018\n",
      "Trial: 10 Tue May 29 20:17:50 2018\n",
      "Finished: 10 LM TR\n",
      "Finished: 10 LM TE\n",
      "Finished: 10 GV TR\n",
      "Finished: 10 GV TE\n",
      "Finished: 10 NN TR\n",
      "Finished: 10 NN TE\n",
      "Trial end: Tue May 29 20:20:22 2018\n",
      "Trial: 11 Tue May 29 20:20:22 2018\n",
      "Finished: 11 LM TR\n",
      "Finished: 11 LM TE\n",
      "Finished: 11 GV TR\n",
      "Finished: 11 GV TE\n",
      "Finished: 11 NN TR\n",
      "Finished: 11 NN TE\n",
      "Trial end: Tue May 29 20:22:58 2018\n",
      "Trial: 12 Tue May 29 20:22:58 2018\n",
      "Finished: 12 LM TR\n",
      "Finished: 12 LM TE\n",
      "Finished: 12 GV TR\n",
      "Finished: 12 GV TE\n",
      "Finished: 12 NN TR\n",
      "Finished: 12 NN TE\n",
      "Trial end: Tue May 29 20:25:31 2018\n",
      "Trial: 13 Tue May 29 20:25:31 2018\n",
      "Finished: 13 LM TR\n",
      "Finished: 13 LM TE\n",
      "Finished: 13 GV TR\n",
      "Finished: 13 GV TE\n",
      "Finished: 13 NN TR\n",
      "Finished: 13 NN TE\n",
      "Trial end: Tue May 29 20:28:03 2018\n",
      "Trial: 14 Tue May 29 20:28:03 2018\n",
      "Finished: 14 LM TR\n",
      "Finished: 14 LM TE\n",
      "Finished: 14 GV TR\n",
      "Finished: 14 GV TE\n",
      "Finished: 14 NN TR\n",
      "Finished: 14 NN TE\n",
      "Trial end: Tue May 29 20:30:39 2018\n",
      "Trial: 15 Tue May 29 20:30:39 2018\n",
      "Finished: 15 LM TR\n",
      "Finished: 15 LM TE\n",
      "Finished: 15 GV TR\n",
      "Finished: 15 GV TE\n",
      "Finished: 15 NN TR\n",
      "Finished: 15 NN TE\n",
      "Trial end: Tue May 29 20:33:14 2018\n",
      "Trial: 16 Tue May 29 20:33:14 2018\n",
      "Finished: 16 LM TR\n",
      "Finished: 16 LM TE\n",
      "Finished: 16 GV TR\n",
      "Finished: 16 GV TE\n",
      "Finished: 16 NN TR\n",
      "Finished: 16 NN TE\n",
      "Trial end: Tue May 29 20:35:53 2018\n",
      "Trial: 17 Tue May 29 20:35:53 2018\n",
      "Finished: 17 LM TR\n",
      "Finished: 17 LM TE\n",
      "Finished: 17 GV TR\n",
      "Finished: 17 GV TE\n",
      "Finished: 17 NN TR\n",
      "Finished: 17 NN TE\n",
      "Trial end: Tue May 29 20:38:33 2018\n",
      "Trial: 18 Tue May 29 20:38:33 2018\n",
      "Finished: 18 LM TR\n",
      "Finished: 18 LM TE\n",
      "Finished: 18 GV TR\n",
      "Finished: 18 GV TE\n",
      "Finished: 18 NN TR\n",
      "Finished: 18 NN TE\n",
      "Trial end: Tue May 29 20:41:15 2018\n",
      "Trial: 19 Tue May 29 20:41:15 2018\n",
      "Finished: 19 LM TR\n",
      "Finished: 19 LM TE\n",
      "Finished: 19 GV TR\n",
      "Finished: 19 GV TE\n",
      "Finished: 19 NN TR\n",
      "Finished: 19 NN TE\n",
      "Trial end: Tue May 29 20:43:59 2018\n",
      "Trial: 20 Tue May 29 20:43:59 2018\n",
      "Finished: 20 LM TR\n",
      "Finished: 20 LM TE\n",
      "Finished: 20 GV TR\n",
      "Finished: 20 GV TE\n",
      "Finished: 20 NN TR\n",
      "Finished: 20 NN TE\n",
      "Trial end: Tue May 29 20:46:51 2018\n",
      "Trial: 21 Tue May 29 20:46:51 2018\n",
      "Finished: 21 LM TR\n",
      "Finished: 21 LM TE\n",
      "Finished: 21 GV TR\n",
      "Finished: 21 GV TE\n",
      "Finished: 21 NN TR\n",
      "Finished: 21 NN TE\n",
      "Trial end: Tue May 29 20:49:47 2018\n",
      "Trial: 22 Tue May 29 20:49:47 2018\n",
      "Finished: 22 LM TR\n",
      "Finished: 22 LM TE\n",
      "Finished: 22 GV TR\n",
      "Finished: 22 GV TE\n",
      "Finished: 22 NN TR\n",
      "Finished: 22 NN TE\n",
      "Trial end: Tue May 29 20:52:29 2018\n",
      "Trial: 23 Tue May 29 20:52:29 2018\n",
      "Finished: 23 LM TR\n",
      "Finished: 23 LM TE\n",
      "Finished: 23 GV TR\n",
      "Finished: 23 GV TE\n",
      "Finished: 23 NN TR\n",
      "Finished: 23 NN TE\n",
      "Trial end: Tue May 29 20:55:09 2018\n",
      "Trial: 24 Tue May 29 20:55:09 2018\n",
      "Finished: 24 LM TR\n",
      "Finished: 24 LM TE\n",
      "Finished: 24 GV TR\n",
      "Finished: 24 GV TE\n",
      "Finished: 24 NN TR\n",
      "Finished: 24 NN TE\n",
      "Trial end: Tue May 29 20:57:53 2018\n",
      "Trial: 25 Tue May 29 20:57:53 2018\n",
      "Finished: 25 LM TR\n",
      "Finished: 25 LM TE\n",
      "Finished: 25 GV TR\n",
      "Finished: 25 GV TE\n",
      "Finished: 25 NN TR\n",
      "Finished: 25 NN TE\n",
      "Trial end: Tue May 29 21:00:33 2018\n",
      "Trial: 26 Tue May 29 21:00:33 2018\n",
      "Finished: 26 LM TR\n",
      "Finished: 26 LM TE\n",
      "Finished: 26 GV TR\n",
      "Finished: 26 GV TE\n",
      "Finished: 26 NN TR\n",
      "Finished: 26 NN TE\n",
      "Trial end: Tue May 29 21:03:18 2018\n",
      "Trial: 27 Tue May 29 21:03:18 2018\n",
      "Finished: 27 LM TR\n",
      "Finished: 27 LM TE\n",
      "Finished: 27 GV TR\n",
      "Finished: 27 GV TE\n",
      "Finished: 27 NN TR\n",
      "Finished: 27 NN TE\n",
      "Trial end: Tue May 29 21:06:12 2018\n",
      "Trial: 28 Tue May 29 21:06:12 2018\n",
      "Finished: 28 LM TR\n",
      "Finished: 28 LM TE\n",
      "Finished: 28 GV TR\n",
      "Finished: 28 GV TE\n",
      "Finished: 28 NN TR\n",
      "Finished: 28 NN TE\n",
      "Trial end: Tue May 29 21:09:07 2018\n",
      "Trial: 29 Tue May 29 21:09:07 2018\n",
      "Finished: 29 LM TR\n",
      "Finished: 29 LM TE\n",
      "Finished: 29 GV TR\n",
      "Finished: 29 GV TE\n",
      "Finished: 29 NN TR\n",
      "Finished: 29 NN TE\n",
      "Trial end: Tue May 29 21:12:03 2018\n",
      "Trial: 30 Tue May 29 21:12:03 2018\n",
      "Finished: 30 LM TR\n",
      "Finished: 30 LM TE\n",
      "Finished: 30 GV TR\n",
      "Finished: 30 GV TE\n",
      "Finished: 30 NN TR\n",
      "Finished: 30 NN TE\n",
      "Trial end: Tue May 29 21:14:54 2018\n",
      "Trial: 31 Tue May 29 21:14:54 2018\n",
      "Finished: 31 LM TR\n",
      "Finished: 31 LM TE\n",
      "Finished: 31 GV TR\n",
      "Finished: 31 GV TE\n",
      "Finished: 31 NN TR\n",
      "Finished: 31 NN TE\n",
      "Trial end: Tue May 29 21:17:47 2018\n",
      "Trial: 32 Tue May 29 21:17:47 2018\n",
      "Finished: 32 LM TR\n",
      "Finished: 32 LM TE\n",
      "Finished: 32 GV TR\n",
      "Finished: 32 GV TE\n",
      "Finished: 32 NN TR\n",
      "Finished: 32 NN TE\n",
      "Trial end: Tue May 29 21:20:34 2018\n",
      "Trial: 33 Tue May 29 21:20:34 2018\n",
      "Finished: 33 LM TR\n",
      "Finished: 33 LM TE\n",
      "Finished: 33 GV TR\n",
      "Finished: 33 GV TE\n",
      "Finished: 33 NN TR\n",
      "Finished: 33 NN TE\n",
      "Trial end: Tue May 29 21:23:25 2018\n",
      "Trial: 34 Tue May 29 21:23:25 2018\n",
      "Finished: 34 LM TR\n",
      "Finished: 34 LM TE\n",
      "Finished: 34 GV TR\n",
      "Finished: 34 GV TE\n",
      "Finished: 34 NN TR\n",
      "Finished: 34 NN TE\n",
      "Trial end: Tue May 29 21:26:16 2018\n",
      "Trial: 35 Tue May 29 21:26:16 2018\n",
      "Finished: 35 LM TR\n",
      "Finished: 35 LM TE\n",
      "Finished: 35 GV TR\n",
      "Finished: 35 GV TE\n",
      "Finished: 35 NN TR\n",
      "Finished: 35 NN TE\n",
      "Trial end: Tue May 29 21:29:07 2018\n",
      "Trial: 36 Tue May 29 21:29:07 2018\n",
      "Finished: 36 LM TR\n",
      "Finished: 36 LM TE\n",
      "Finished: 36 GV TR\n",
      "Finished: 36 GV TE\n",
      "Finished: 36 NN TR\n",
      "Finished: 36 NN TE\n",
      "Trial end: Tue May 29 21:31:57 2018\n",
      "Trial: 37 Tue May 29 21:31:57 2018\n",
      "Finished: 37 LM TR\n",
      "Finished: 37 LM TE\n",
      "Finished: 37 GV TR\n",
      "Finished: 37 GV TE\n",
      "Finished: 37 NN TR\n",
      "Finished: 37 NN TE\n",
      "Trial end: Tue May 29 21:34:47 2018\n",
      "Trial: 38 Tue May 29 21:34:47 2018\n",
      "Finished: 38 LM TR\n",
      "Finished: 38 LM TE\n",
      "Finished: 38 GV TR\n",
      "Finished: 38 GV TE\n",
      "Finished: 38 NN TR\n",
      "Finished: 38 NN TE\n",
      "Trial end: Tue May 29 21:37:35 2018\n",
      "Trial: 39 Tue May 29 21:37:35 2018\n",
      "Finished: 39 LM TR\n",
      "Finished: 39 LM TE\n",
      "Finished: 39 GV TR\n",
      "Finished: 39 GV TE\n",
      "Finished: 39 NN TR\n",
      "Finished: 39 NN TE\n",
      "Trial end: Tue May 29 21:40:24 2018\n",
      "Trial: 40 Tue May 29 21:40:24 2018\n",
      "Finished: 40 LM TR\n",
      "Finished: 40 LM TE\n",
      "Finished: 40 GV TR\n",
      "Finished: 40 GV TE\n",
      "Finished: 40 NN TR\n",
      "Finished: 40 NN TE\n",
      "Trial end: Tue May 29 21:43:15 2018\n",
      "Trial: 41 Tue May 29 21:43:15 2018\n",
      "Finished: 41 LM TR\n",
      "Finished: 41 LM TE\n",
      "Finished: 41 GV TR\n",
      "Finished: 41 GV TE\n",
      "Finished: 41 NN TR\n",
      "Finished: 41 NN TE\n",
      "Trial end: Tue May 29 21:46:05 2018\n",
      "Trial: 42 Tue May 29 21:46:05 2018\n",
      "Finished: 42 LM TR\n",
      "Finished: 42 LM TE\n",
      "Finished: 42 GV TR\n",
      "Finished: 42 GV TE\n",
      "Finished: 42 NN TR\n",
      "Finished: 42 NN TE\n",
      "Trial end: Tue May 29 21:48:58 2018\n",
      "Trial: 43 Tue May 29 21:48:58 2018\n",
      "Finished: 43 LM TR\n",
      "Finished: 43 LM TE\n",
      "Finished: 43 GV TR\n",
      "Finished: 43 GV TE\n",
      "Finished: 43 NN TR\n",
      "Finished: 43 NN TE\n",
      "Trial end: Tue May 29 21:51:53 2018\n",
      "Trial: 44 Tue May 29 21:51:53 2018\n",
      "Finished: 44 LM TR\n",
      "Finished: 44 LM TE\n",
      "Finished: 44 GV TR\n",
      "Finished: 44 GV TE\n",
      "Finished: 44 NN TR\n",
      "Finished: 44 NN TE\n",
      "Trial end: Tue May 29 21:54:53 2018\n",
      "Trial: 45 Tue May 29 21:54:53 2018\n",
      "Finished: 45 LM TR\n",
      "Finished: 45 LM TE\n",
      "Finished: 45 GV TR\n",
      "Finished: 45 GV TE\n",
      "Finished: 45 NN TR\n",
      "Finished: 45 NN TE\n",
      "Trial end: Tue May 29 21:57:53 2018\n",
      "Trial: 46 Tue May 29 21:57:53 2018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/py36/lib/python2.7/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished: 46 LM TR\n",
      "Finished: 46 LM TE\n",
      "Finished: 46 GV TR\n",
      "Finished: 46 GV TE\n",
      "Finished: 46 NN TR\n",
      "Finished: 46 NN TE\n",
      "Trial end: Tue May 29 22:00:53 2018\n",
      "Trial: 47 Tue May 29 22:00:53 2018\n",
      "Finished: 47 LM TR\n",
      "Finished: 47 LM TE\n",
      "Finished: 47 GV TR\n",
      "Finished: 47 GV TE\n",
      "Finished: 47 NN TR\n",
      "Finished: 47 NN TE\n",
      "Trial end: Tue May 29 22:04:02 2018\n",
      "Trial: 48 Tue May 29 22:04:02 2018\n",
      "Finished: 48 LM TR\n",
      "Finished: 48 LM TE\n",
      "Finished: 48 GV TR\n",
      "Finished: 48 GV TE\n",
      "Finished: 48 NN TR\n",
      "Finished: 48 NN TE\n",
      "Trial end: Tue May 29 22:07:05 2018\n",
      "Trial: 49 Tue May 29 22:07:05 2018\n",
      "Finished: 49 LM TR\n",
      "Finished: 49 LM TE\n",
      "Finished: 49 GV TR\n",
      "Finished: 49 GV TE\n",
      "Finished: 49 NN TR\n",
      "Finished: 49 NN TE\n",
      "Trial end: Tue May 29 22:10:09 2018\n",
      "Trial: 50 Tue May 29 22:10:09 2018\n",
      "Finished: 50 LM TR\n",
      "Finished: 50 LM TE\n",
      "Finished: 50 GV TR\n",
      "Finished: 50 GV TE\n",
      "Finished: 50 NN TR\n",
      "Finished: 50 NN TE\n",
      "Trial end: Tue May 29 22:13:16 2018\n",
      "Trial: 51 Tue May 29 22:13:16 2018\n",
      "Finished: 51 LM TR\n",
      "Finished: 51 LM TE\n",
      "Finished: 51 GV TR\n",
      "Finished: 51 GV TE\n",
      "Finished: 51 NN TR\n",
      "Finished: 51 NN TE\n",
      "Trial end: Tue May 29 22:16:27 2018\n",
      "Trial: 52 Tue May 29 22:16:27 2018\n",
      "Finished: 52 LM TR\n",
      "Finished: 52 LM TE\n",
      "Finished: 52 GV TR\n",
      "Finished: 52 GV TE\n",
      "Finished: 52 NN TR\n",
      "Finished: 52 NN TE\n",
      "Trial end: Tue May 29 22:19:37 2018\n",
      "Trial: 53 Tue May 29 22:19:37 2018\n",
      "Finished: 53 LM TR\n",
      "Finished: 53 LM TE\n",
      "Finished: 53 GV TR\n",
      "Finished: 53 GV TE\n",
      "Finished: 53 NN TR\n",
      "Finished: 53 NN TE\n",
      "Trial end: Tue May 29 22:22:50 2018\n",
      "Trial: 54 Tue May 29 22:22:50 2018\n",
      "Finished: 54 LM TR\n",
      "Finished: 54 LM TE\n",
      "Finished: 54 GV TR\n",
      "Finished: 54 GV TE\n",
      "Finished: 54 NN TR\n",
      "Finished: 54 NN TE\n",
      "Trial end: Tue May 29 22:26:13 2018\n",
      "Trial: 55 Tue May 29 22:26:13 2018\n",
      "Finished: 55 LM TR\n",
      "Finished: 55 LM TE\n",
      "Finished: 55 GV TR\n",
      "Finished: 55 GV TE\n",
      "Finished: 55 NN TR\n",
      "Finished: 55 NN TE\n",
      "Trial end: Tue May 29 22:29:29 2018\n",
      "Trial: 56 Tue May 29 22:29:29 2018\n",
      "Finished: 56 LM TR\n",
      "Finished: 56 LM TE\n",
      "Finished: 56 GV TR\n",
      "Finished: 56 GV TE\n",
      "Finished: 56 NN TR\n",
      "Finished: 56 NN TE\n",
      "Trial end: Tue May 29 22:32:47 2018\n",
      "Trial: 57 Tue May 29 22:32:47 2018\n",
      "Finished: 57 LM TR\n",
      "Finished: 57 LM TE\n",
      "Finished: 57 GV TR\n",
      "Finished: 57 GV TE\n",
      "Finished: 57 NN TR\n",
      "Finished: 57 NN TE\n",
      "Trial end: Tue May 29 22:36:06 2018\n",
      "Trial: 58 Tue May 29 22:36:06 2018\n",
      "Finished: 58 LM TR\n",
      "Finished: 58 LM TE\n",
      "Finished: 58 GV TR\n",
      "Finished: 58 GV TE\n",
      "Finished: 58 NN TR\n",
      "Finished: 58 NN TE\n",
      "Trial end: Tue May 29 22:39:26 2018\n",
      "Trial: 59 Tue May 29 22:39:26 2018\n",
      "Finished: 59 LM TR\n",
      "Finished: 59 LM TE\n",
      "Finished: 59 GV TR\n",
      "Finished: 59 GV TE\n",
      "Finished: 59 NN TR\n",
      "Finished: 59 NN TE\n",
      "Trial end: Tue May 29 22:42:49 2018\n",
      "Trial: 60 Tue May 29 22:42:49 2018\n",
      "Finished: 60 LM TR\n",
      "Finished: 60 LM TE\n",
      "Finished: 60 GV TR\n",
      "Finished: 60 GV TE\n",
      "Finished: 60 NN TR\n",
      "Finished: 60 NN TE\n",
      "Trial end: Tue May 29 22:46:12 2018\n",
      "Trial: 61 Tue May 29 22:46:12 2018\n",
      "Finished: 61 LM TR\n",
      "Finished: 61 LM TE\n",
      "Finished: 61 GV TR\n",
      "Finished: 61 GV TE\n",
      "Finished: 61 NN TR\n",
      "Finished: 61 NN TE\n",
      "Trial end: Tue May 29 22:49:36 2018\n",
      "Trial: 62 Tue May 29 22:49:36 2018\n",
      "Finished: 62 LM TR\n",
      "Finished: 62 LM TE\n",
      "Finished: 62 GV TR\n",
      "Finished: 62 GV TE\n",
      "Finished: 62 NN TR\n",
      "Finished: 62 NN TE\n",
      "Trial end: Tue May 29 22:53:01 2018\n",
      "Trial: 63 Tue May 29 22:53:01 2018\n",
      "Finished: 63 LM TR\n",
      "Finished: 63 LM TE\n",
      "Finished: 63 GV TR\n",
      "Finished: 63 GV TE\n",
      "Finished: 63 NN TR\n",
      "Finished: 63 NN TE\n",
      "Trial end: Tue May 29 22:56:28 2018\n",
      "Trial: 64 Tue May 29 22:56:28 2018\n",
      "Finished: 64 LM TR\n",
      "Finished: 64 LM TE\n",
      "Finished: 64 GV TR\n",
      "Finished: 64 GV TE\n",
      "Finished: 64 NN TR\n",
      "Finished: 64 NN TE\n",
      "Trial end: Tue May 29 22:59:56 2018\n",
      "Trial: 65 Tue May 29 22:59:56 2018\n",
      "Finished: 65 LM TR\n",
      "Finished: 65 LM TE\n",
      "Finished: 65 GV TR\n",
      "Finished: 65 GV TE\n",
      "Finished: 65 NN TR\n",
      "Finished: 65 NN TE\n",
      "Trial end: Tue May 29 23:03:27 2018\n",
      "Trial: 66 Tue May 29 23:03:27 2018\n",
      "Finished: 66 LM TR\n",
      "Finished: 66 LM TE\n",
      "Finished: 66 GV TR\n",
      "Finished: 66 GV TE\n",
      "Finished: 66 NN TR\n",
      "Finished: 66 NN TE\n",
      "Trial end: Tue May 29 23:06:57 2018\n",
      "Trial: 67 Tue May 29 23:06:57 2018\n",
      "Finished: 67 LM TR\n",
      "Finished: 67 LM TE\n",
      "Finished: 67 GV TR\n",
      "Finished: 67 GV TE\n",
      "Finished: 67 NN TR\n",
      "Finished: 67 NN TE\n",
      "Trial end: Tue May 29 23:10:29 2018\n",
      "Trial: 68 Tue May 29 23:10:29 2018\n",
      "Finished: 68 LM TR\n",
      "Finished: 68 LM TE\n",
      "Finished: 68 GV TR\n",
      "Finished: 68 GV TE\n",
      "Finished: 68 NN TR\n",
      "Finished: 68 NN TE\n",
      "Trial end: Tue May 29 23:14:03 2018\n",
      "Trial: 69 Tue May 29 23:14:03 2018\n",
      "Finished: 69 LM TR\n",
      "Finished: 69 LM TE\n",
      "Finished: 69 GV TR\n",
      "Finished: 69 GV TE\n",
      "Finished: 69 NN TR\n",
      "Finished: 69 NN TE\n",
      "Trial end: Tue May 29 23:17:38 2018\n",
      "Trial: 70 Tue May 29 23:17:38 2018\n",
      "Finished: 70 LM TR\n",
      "Finished: 70 LM TE\n",
      "Finished: 70 GV TR\n",
      "Finished: 70 GV TE\n",
      "Finished: 70 NN TR\n",
      "Finished: 70 NN TE\n",
      "Trial end: Tue May 29 23:21:15 2018\n",
      "Trial: 71 Tue May 29 23:21:15 2018\n",
      "Finished: 71 LM TR\n",
      "Finished: 71 LM TE\n",
      "Finished: 71 GV TR\n",
      "Finished: 71 GV TE\n",
      "Finished: 71 NN TR\n",
      "Finished: 71 NN TE\n",
      "Trial end: Tue May 29 23:24:52 2018\n",
      "Trial: 72 Tue May 29 23:24:52 2018\n",
      "Finished: 72 LM TR\n",
      "Finished: 72 LM TE\n",
      "Finished: 72 GV TR\n",
      "Finished: 72 GV TE\n",
      "Finished: 72 NN TR\n",
      "Finished: 72 NN TE\n",
      "Trial end: Tue May 29 23:28:30 2018\n",
      "Trial: 73 Tue May 29 23:28:30 2018\n",
      "Finished: 73 LM TR\n",
      "Finished: 73 LM TE\n",
      "Finished: 73 GV TR\n",
      "Finished: 73 GV TE\n",
      "Finished: 73 NN TR\n",
      "Finished: 73 NN TE\n",
      "Trial end: Tue May 29 23:32:07 2018\n",
      "Trial: 74 Tue May 29 23:32:07 2018\n",
      "Finished: 74 LM TR\n",
      "Finished: 74 LM TE\n",
      "Finished: 74 GV TR\n",
      "Finished: 74 GV TE\n",
      "Finished: 74 NN TR\n",
      "Finished: 74 NN TE\n",
      "Trial end: Tue May 29 23:35:51 2018\n",
      "Trial: 75 Tue May 29 23:35:51 2018\n",
      "Finished: 75 LM TR\n",
      "Finished: 75 LM TE\n",
      "Finished: 75 GV TR\n",
      "Finished: 75 GV TE\n",
      "Finished: 75 NN TR\n",
      "Finished: 75 NN TE\n",
      "Trial end: Tue May 29 23:39:35 2018\n",
      "Trial: 76 Tue May 29 23:39:35 2018\n",
      "Finished: 76 LM TR\n",
      "Finished: 76 LM TE\n",
      "Finished: 76 GV TR\n",
      "Finished: 76 GV TE\n",
      "Finished: 76 NN TR\n",
      "Finished: 76 NN TE\n",
      "Trial end: Tue May 29 23:43:21 2018\n",
      "Trial: 77 Tue May 29 23:43:21 2018\n",
      "Finished: 77 LM TR\n",
      "Finished: 77 LM TE\n",
      "Finished: 77 GV TR\n",
      "Finished: 77 GV TE\n",
      "Finished: 77 NN TR\n",
      "Finished: 77 NN TE\n",
      "Trial end: Tue May 29 23:47:03 2018\n",
      "Trial: 78 Tue May 29 23:47:03 2018\n",
      "Finished: 78 LM TR\n",
      "Finished: 78 LM TE\n",
      "Finished: 78 GV TR\n",
      "Finished: 78 GV TE\n",
      "Finished: 78 NN TR\n",
      "Finished: 78 NN TE\n",
      "Trial end: Tue May 29 23:50:49 2018\n",
      "Trial: 79 Tue May 29 23:50:49 2018\n",
      "Finished: 79 LM TR\n",
      "Finished: 79 LM TE\n",
      "Finished: 79 GV TR\n",
      "Finished: 79 GV TE\n",
      "Finished: 79 NN TR\n",
      "Finished: 79 NN TE\n",
      "Trial end: Tue May 29 23:54:38 2018\n",
      "Trial: 80 Tue May 29 23:54:38 2018\n",
      "Finished: 80 LM TR\n",
      "Finished: 80 LM TE\n",
      "Finished: 80 GV TR\n",
      "Finished: 80 GV TE\n",
      "Finished: 80 NN TR\n",
      "Finished: 80 NN TE\n",
      "Trial end: Tue May 29 23:58:31 2018\n",
      "Trial: 81 Tue May 29 23:58:31 2018\n",
      "Finished: 81 LM TR\n",
      "Finished: 81 LM TE\n",
      "Finished: 81 GV TR\n",
      "Finished: 81 GV TE\n",
      "Finished: 81 NN TR\n",
      "Finished: 81 NN TE\n",
      "Trial end: Wed May 30 00:02:19 2018\n",
      "Trial: 82 Wed May 30 00:02:19 2018\n",
      "Finished: 82 LM TR\n",
      "Finished: 82 LM TE\n",
      "Finished: 82 GV TR\n",
      "Finished: 82 GV TE\n",
      "Finished: 82 NN TR\n",
      "Finished: 82 NN TE\n",
      "Trial end: Wed May 30 00:06:10 2018\n",
      "Trial: 83 Wed May 30 00:06:10 2018\n",
      "Finished: 83 LM TR\n",
      "Finished: 83 LM TE\n",
      "Finished: 83 GV TR\n",
      "Finished: 83 GV TE\n",
      "Finished: 83 NN TR\n",
      "Finished: 83 NN TE\n",
      "Trial end: Wed May 30 00:10:03 2018\n",
      "Trial: 84 Wed May 30 00:10:03 2018\n",
      "Finished: 84 LM TR\n",
      "Finished: 84 LM TE\n",
      "Finished: 84 GV TR\n",
      "Finished: 84 GV TE\n",
      "Finished: 84 NN TR\n",
      "Finished: 84 NN TE\n",
      "Trial end: Wed May 30 00:13:57 2018\n",
      "Trial: 85 Wed May 30 00:13:57 2018\n",
      "Finished: 85 LM TR\n",
      "Finished: 85 LM TE\n",
      "Finished: 85 GV TR\n",
      "Finished: 85 GV TE\n",
      "Finished: 85 NN TR\n",
      "Finished: 85 NN TE\n",
      "Trial end: Wed May 30 00:17:52 2018\n",
      "Trial: 86 Wed May 30 00:17:52 2018\n",
      "Finished: 86 LM TR\n",
      "Finished: 86 LM TE\n",
      "Finished: 86 GV TR\n",
      "Finished: 86 GV TE\n",
      "Finished: 86 NN TR\n",
      "Finished: 86 NN TE\n",
      "Trial end: Wed May 30 00:21:49 2018\n",
      "Trial: 87 Wed May 30 00:21:49 2018\n",
      "Finished: 87 LM TR\n",
      "Finished: 87 LM TE\n",
      "Finished: 87 GV TR\n",
      "Finished: 87 GV TE\n",
      "Finished: 87 NN TR\n",
      "Finished: 87 NN TE\n",
      "Trial end: Wed May 30 00:25:56 2018\n",
      "Trial: 88 Wed May 30 00:25:56 2018\n",
      "Finished: 88 LM TR\n",
      "Finished: 88 LM TE\n",
      "Finished: 88 GV TR\n",
      "Finished: 88 GV TE\n",
      "Finished: 88 NN TR\n",
      "Finished: 88 NN TE\n",
      "Trial end: Wed May 30 00:29:56 2018\n",
      "Trial: 89 Wed May 30 00:29:56 2018\n",
      "Finished: 89 LM TR\n",
      "Finished: 89 LM TE\n",
      "Finished: 89 GV TR\n",
      "Finished: 89 GV TE\n",
      "Finished: 89 NN TR\n",
      "Finished: 89 NN TE\n",
      "Trial end: Wed May 30 00:33:55 2018\n",
      "Trial: 90 Wed May 30 00:33:55 2018\n",
      "Finished: 90 LM TR\n",
      "Finished: 90 LM TE\n",
      "Finished: 90 GV TR\n",
      "Finished: 90 GV TE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished: 90 NN TR\n",
      "Finished: 90 NN TE\n",
      "Trial end: Wed May 30 00:37:56 2018\n",
      "Trial: 91 Wed May 30 00:37:56 2018\n",
      "Finished: 91 LM TR\n",
      "Finished: 91 LM TE\n",
      "Finished: 91 GV TR\n",
      "Finished: 91 GV TE\n",
      "Finished: 91 NN TR\n",
      "Finished: 91 NN TE\n",
      "Trial end: Wed May 30 00:42:00 2018\n",
      "Trial: 92 Wed May 30 00:42:00 2018\n",
      "Finished: 92 LM TR\n",
      "Finished: 92 LM TE\n",
      "Finished: 92 GV TR\n",
      "Finished: 92 GV TE\n",
      "Finished: 92 NN TR\n",
      "Finished: 92 NN TE\n",
      "Trial end: Wed May 30 00:46:02 2018\n",
      "Trial: 93 Wed May 30 00:46:02 2018\n",
      "Finished: 93 LM TR\n",
      "Finished: 93 LM TE\n",
      "Finished: 93 GV TR\n",
      "Finished: 93 GV TE\n",
      "Finished: 93 NN TR\n",
      "Finished: 93 NN TE\n",
      "Trial end: Wed May 30 00:50:15 2018\n",
      "Trial: 94 Wed May 30 00:50:15 2018\n",
      "Finished: 94 LM TR\n",
      "Finished: 94 LM TE\n",
      "Finished: 94 GV TR\n",
      "Finished: 94 GV TE\n",
      "Finished: 94 NN TR\n",
      "Finished: 94 NN TE\n",
      "Trial end: Wed May 30 00:54:21 2018\n",
      "Trial: 95 Wed May 30 00:54:21 2018\n",
      "Finished: 95 LM TR\n",
      "Finished: 95 LM TE\n",
      "Finished: 95 GV TR\n",
      "Finished: 95 GV TE\n",
      "Finished: 95 NN TR\n",
      "Finished: 95 NN TE\n",
      "Trial end: Wed May 30 00:58:26 2018\n",
      "Trial: 96 Wed May 30 00:58:26 2018\n",
      "Finished: 96 LM TR\n",
      "Finished: 96 LM TE\n",
      "Finished: 96 GV TR\n",
      "Finished: 96 GV TE\n",
      "Finished: 96 NN TR\n",
      "Finished: 96 NN TE\n",
      "Trial end: Wed May 30 01:02:32 2018\n",
      "Trial: 97 Wed May 30 01:02:32 2018\n",
      "Finished: 97 LM TR\n",
      "Finished: 97 LM TE\n",
      "Finished: 97 GV TR\n",
      "Finished: 97 GV TE\n",
      "Finished: 97 NN TR\n",
      "Finished: 97 NN TE\n",
      "Trial end: Wed May 30 01:06:41 2018\n",
      "Trial: 98 Wed May 30 01:06:41 2018\n",
      "Finished: 98 LM TR\n",
      "Finished: 98 LM TE\n",
      "Finished: 98 GV TR\n",
      "Finished: 98 GV TE\n",
      "Finished: 98 NN TR\n",
      "Finished: 98 NN TE\n",
      "Trial end: Wed May 30 01:10:50 2018\n",
      "Trial: 99 Wed May 30 01:10:50 2018\n",
      "Finished: 99 LM TR\n",
      "Finished: 99 LM TE\n",
      "Finished: 99 GV TR\n",
      "Finished: 99 GV TE\n",
      "Finished: 99 NN TR\n",
      "Finished: 99 NN TE\n",
      "Trial end: Wed May 30 01:15:02 2018\n",
      "Trial: 100 Wed May 30 01:15:02 2018\n",
      "Finished: 100 LM TR\n",
      "Finished: 100 LM TE\n",
      "Finished: 100 GV TR\n",
      "Finished: 100 GV TE\n",
      "Finished: 100 NN TR\n",
      "Finished: 100 NN TE\n",
      "Trial end: Wed May 30 01:19:16 2018\n"
     ]
    }
   ],
   "source": [
    "# Main loop: Randomly generate train and test sets, run the three models\n",
    "# save model performances (both train and test sets) in a dataframe.\n",
    "# 70% of the observations will be in the test set\n",
    "trlen = int(len(full_s)*0.7)\n",
    "maxLen = 55\n",
    "\n",
    "f_ = 1   # Very first run will create a dataframe\n",
    "for i_ in range(1,101):\n",
    "    print 'Trial:', i_, time.ctime()\n",
    "    X_test, X_train, y_test, y_train = TrTestSet(full_s, y, trlen)\n",
    "        \n",
    "    tot_, perc_, pred_ = LM_predict(X_train)\n",
    "    res = eval_metrics(y_train, pred_, i_, 'LM','TR')\n",
    "    print 'Finished:', i_, 'LM','TR'\n",
    "    if f_ == 1:\n",
    "        res_df = res\n",
    "        f_ = 0\n",
    "    else:\n",
    "        res_df = res_df.append(res, ignore_index=True)    \n",
    "        \n",
    "    tot_, perc_, pred_ = LM_predict(X_test)\n",
    "    res = eval_metrics(y_test, pred_, i_, 'LM','TE')\n",
    "    print 'Finished:', i_, 'LM','TE'\n",
    "    if f_ == 1:\n",
    "        res_df = res\n",
    "        f_ = 0\n",
    "    else:\n",
    "        res_df = res_df.append(res, ignore_index=True)\n",
    "    \n",
    "    pred_train, pred_test = GV_predict(X_train, X_test, y_train, y_test)\n",
    "    res = eval_metrics(y_train, pred_train, i_, 'GV','TR')\n",
    "    print 'Finished:', i_, 'GV','TR'\n",
    "    if f_ == 1:\n",
    "        res_df = res\n",
    "        f_ = 0\n",
    "    else:\n",
    "        res_df = res_df.append(res, ignore_index=True)    \n",
    "    \n",
    "    res = eval_metrics(y_test, pred_test, i_, 'GV','TE')   \n",
    "    print 'Finished:', i_, 'GV','TE'\n",
    "    if f_ == 1:\n",
    "        res_df = res\n",
    "        f_ = 0\n",
    "    else:\n",
    "        res_df = res_df.append(res, ignore_index=True)    \n",
    "        \n",
    "    pred_train, pred_test = NN_predict(X_train, X_test, y_train, y_test)\n",
    "    res = eval_metrics(y_train, pred_train, i_, 'NN','TR')\n",
    "    print 'Finished:', i_, 'NN','TR'\n",
    "    if f_ == 1:\n",
    "        res_df = res\n",
    "        f_ = 0\n",
    "    else:\n",
    "        res_df = res_df.append(res, ignore_index=True)    \n",
    "    \n",
    "    res = eval_metrics(y_test, pred_test, i_, 'NN','TE') \n",
    "    print 'Finished:', i_, 'NN','TE'\n",
    "    if f_ == 1:\n",
    "        res_df = res\n",
    "        f_ = 0\n",
    "    else:\n",
    "        res_df = res_df.append(res, ignore_index=True)     \n",
    "    print 'Trial end:', time.ctime()     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A0P0</th>\n",
       "      <th>A0P1</th>\n",
       "      <th>A0P2</th>\n",
       "      <th>A1P0</th>\n",
       "      <th>A1P1</th>\n",
       "      <th>A1P2</th>\n",
       "      <th>A2P0</th>\n",
       "      <th>A2P1</th>\n",
       "      <th>A2P2</th>\n",
       "      <th>acc</th>\n",
       "      <th>f1_0</th>\n",
       "      <th>f1_1</th>\n",
       "      <th>f1_2</th>\n",
       "      <th>f1_all</th>\n",
       "      <th>model</th>\n",
       "      <th>prec_0</th>\n",
       "      <th>prec_1</th>\n",
       "      <th>prec_2</th>\n",
       "      <th>prec_all</th>\n",
       "      <th>rec_0</th>\n",
       "      <th>rec_1</th>\n",
       "      <th>rec_2</th>\n",
       "      <th>rec_all</th>\n",
       "      <th>rnd_ct</th>\n",
       "      <th>sup_0</th>\n",
       "      <th>sup_1</th>\n",
       "      <th>sup_2</th>\n",
       "      <th>sup_all</th>\n",
       "      <th>trtest</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>59</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>66</td>\n",
       "      <td>242</td>\n",
       "      <td>22</td>\n",
       "      <td>18</td>\n",
       "      <td>169</td>\n",
       "      <td>48</td>\n",
       "      <td>0.498571</td>\n",
       "      <td>0.42446</td>\n",
       "      <td>0.596794</td>\n",
       "      <td>0.308682</td>\n",
       "      <td>0.466835</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.412587</td>\n",
       "      <td>0.503119</td>\n",
       "      <td>0.631579</td>\n",
       "      <td>0.528785</td>\n",
       "      <td>0.437037</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>0.204255</td>\n",
       "      <td>0.498571</td>\n",
       "      <td>1</td>\n",
       "      <td>135</td>\n",
       "      <td>330</td>\n",
       "      <td>235</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>21</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>106</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>66</td>\n",
       "      <td>20</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.355932</td>\n",
       "      <td>0.600567</td>\n",
       "      <td>0.310078</td>\n",
       "      <td>0.463219</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.519608</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.497961</td>\n",
       "      <td>0.362069</td>\n",
       "      <td>0.711409</td>\n",
       "      <td>0.215054</td>\n",
       "      <td>0.49</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>149</td>\n",
       "      <td>93</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "      <td>51</td>\n",
       "      <td>13</td>\n",
       "      <td>274</td>\n",
       "      <td>43</td>\n",
       "      <td>18</td>\n",
       "      <td>64</td>\n",
       "      <td>153</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.403846</td>\n",
       "      <td>0.771831</td>\n",
       "      <td>0.634855</td>\n",
       "      <td>0.654878</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.575342</td>\n",
       "      <td>0.721053</td>\n",
       "      <td>0.619433</td>\n",
       "      <td>0.658836</td>\n",
       "      <td>0.311111</td>\n",
       "      <td>0.830303</td>\n",
       "      <td>0.651064</td>\n",
       "      <td>0.67</td>\n",
       "      <td>1</td>\n",
       "      <td>135</td>\n",
       "      <td>330</td>\n",
       "      <td>235</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>118</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>35</td>\n",
       "      <td>41</td>\n",
       "      <td>0.573333</td>\n",
       "      <td>0.273684</td>\n",
       "      <td>0.753994</td>\n",
       "      <td>0.427083</td>\n",
       "      <td>0.559792</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.351351</td>\n",
       "      <td>0.719512</td>\n",
       "      <td>0.414141</td>\n",
       "      <td>0.553669</td>\n",
       "      <td>0.224138</td>\n",
       "      <td>0.791946</td>\n",
       "      <td>0.44086</td>\n",
       "      <td>0.573333</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>149</td>\n",
       "      <td>93</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>109</td>\n",
       "      <td>0</td>\n",
       "      <td>312</td>\n",
       "      <td>18</td>\n",
       "      <td>4</td>\n",
       "      <td>37</td>\n",
       "      <td>194</td>\n",
       "      <td>0.744286</td>\n",
       "      <td>0.194805</td>\n",
       "      <td>0.904348</td>\n",
       "      <td>0.697842</td>\n",
       "      <td>0.69818</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.789474</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>0.604361</td>\n",
       "      <td>0.76372</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.945455</td>\n",
       "      <td>0.825532</td>\n",
       "      <td>0.744286</td>\n",
       "      <td>1</td>\n",
       "      <td>135</td>\n",
       "      <td>330</td>\n",
       "      <td>235</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>122</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>33</td>\n",
       "      <td>57</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.121212</td>\n",
       "      <td>0.77707</td>\n",
       "      <td>0.518182</td>\n",
       "      <td>0.570016</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.739394</td>\n",
       "      <td>0.448819</td>\n",
       "      <td>0.603033</td>\n",
       "      <td>0.0689655</td>\n",
       "      <td>0.818792</td>\n",
       "      <td>0.612903</td>\n",
       "      <td>0.61</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>149</td>\n",
       "      <td>93</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>54</td>\n",
       "      <td>70</td>\n",
       "      <td>9</td>\n",
       "      <td>75</td>\n",
       "      <td>241</td>\n",
       "      <td>18</td>\n",
       "      <td>15</td>\n",
       "      <td>168</td>\n",
       "      <td>50</td>\n",
       "      <td>0.492857</td>\n",
       "      <td>0.389892</td>\n",
       "      <td>0.592866</td>\n",
       "      <td>0.322581</td>\n",
       "      <td>0.464334</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.503132</td>\n",
       "      <td>0.649351</td>\n",
       "      <td>0.527457</td>\n",
       "      <td>0.406015</td>\n",
       "      <td>0.721557</td>\n",
       "      <td>0.214592</td>\n",
       "      <td>0.492857</td>\n",
       "      <td>2</td>\n",
       "      <td>133</td>\n",
       "      <td>334</td>\n",
       "      <td>233</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>26</td>\n",
       "      <td>32</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>107</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>67</td>\n",
       "      <td>18</td>\n",
       "      <td>0.503333</td>\n",
       "      <td>0.436975</td>\n",
       "      <td>0.609687</td>\n",
       "      <td>0.276923</td>\n",
       "      <td>0.469769</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.440678</td>\n",
       "      <td>0.519417</td>\n",
       "      <td>0.514286</td>\n",
       "      <td>0.502045</td>\n",
       "      <td>0.433333</td>\n",
       "      <td>0.737931</td>\n",
       "      <td>0.189474</td>\n",
       "      <td>0.503333</td>\n",
       "      <td>2</td>\n",
       "      <td>60</td>\n",
       "      <td>145</td>\n",
       "      <td>95</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>56</td>\n",
       "      <td>31</td>\n",
       "      <td>46</td>\n",
       "      <td>14</td>\n",
       "      <td>266</td>\n",
       "      <td>54</td>\n",
       "      <td>32</td>\n",
       "      <td>62</td>\n",
       "      <td>139</td>\n",
       "      <td>0.658571</td>\n",
       "      <td>0.476596</td>\n",
       "      <td>0.767677</td>\n",
       "      <td>0.588983</td>\n",
       "      <td>0.652892</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.54902</td>\n",
       "      <td>0.740947</td>\n",
       "      <td>0.58159</td>\n",
       "      <td>0.651438</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>0.796407</td>\n",
       "      <td>0.596567</td>\n",
       "      <td>0.658571</td>\n",
       "      <td>2</td>\n",
       "      <td>133</td>\n",
       "      <td>334</td>\n",
       "      <td>233</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "      <td>116</td>\n",
       "      <td>21</td>\n",
       "      <td>14</td>\n",
       "      <td>26</td>\n",
       "      <td>55</td>\n",
       "      <td>0.626667</td>\n",
       "      <td>0.343434</td>\n",
       "      <td>0.75817</td>\n",
       "      <td>0.564103</td>\n",
       "      <td>0.613768</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.435897</td>\n",
       "      <td>0.720497</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.609586</td>\n",
       "      <td>0.283333</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.578947</td>\n",
       "      <td>0.626667</td>\n",
       "      <td>2</td>\n",
       "      <td>60</td>\n",
       "      <td>145</td>\n",
       "      <td>95</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>24</td>\n",
       "      <td>13</td>\n",
       "      <td>96</td>\n",
       "      <td>4</td>\n",
       "      <td>304</td>\n",
       "      <td>26</td>\n",
       "      <td>12</td>\n",
       "      <td>44</td>\n",
       "      <td>177</td>\n",
       "      <td>0.721429</td>\n",
       "      <td>0.277457</td>\n",
       "      <td>0.87482</td>\n",
       "      <td>0.665414</td>\n",
       "      <td>0.691619</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.842105</td>\n",
       "      <td>0.591973</td>\n",
       "      <td>0.712847</td>\n",
       "      <td>0.180451</td>\n",
       "      <td>0.91018</td>\n",
       "      <td>0.759657</td>\n",
       "      <td>0.721429</td>\n",
       "      <td>2</td>\n",
       "      <td>133</td>\n",
       "      <td>334</td>\n",
       "      <td>233</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>39</td>\n",
       "      <td>2</td>\n",
       "      <td>114</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>27</td>\n",
       "      <td>61</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.207792</td>\n",
       "      <td>0.762542</td>\n",
       "      <td>0.544643</td>\n",
       "      <td>0.582591</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.470588</td>\n",
       "      <td>0.74026</td>\n",
       "      <td>0.472868</td>\n",
       "      <td>0.601651</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.786207</td>\n",
       "      <td>0.642105</td>\n",
       "      <td>0.61</td>\n",
       "      <td>2</td>\n",
       "      <td>60</td>\n",
       "      <td>145</td>\n",
       "      <td>95</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>52</td>\n",
       "      <td>74</td>\n",
       "      <td>6</td>\n",
       "      <td>72</td>\n",
       "      <td>237</td>\n",
       "      <td>29</td>\n",
       "      <td>15</td>\n",
       "      <td>164</td>\n",
       "      <td>51</td>\n",
       "      <td>0.485714</td>\n",
       "      <td>0.383764</td>\n",
       "      <td>0.583026</td>\n",
       "      <td>0.322785</td>\n",
       "      <td>0.459943</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.374101</td>\n",
       "      <td>0.498947</td>\n",
       "      <td>0.593023</td>\n",
       "      <td>0.506316</td>\n",
       "      <td>0.393939</td>\n",
       "      <td>0.701183</td>\n",
       "      <td>0.221739</td>\n",
       "      <td>0.485714</td>\n",
       "      <td>3</td>\n",
       "      <td>132</td>\n",
       "      <td>338</td>\n",
       "      <td>230</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "      <td>5</td>\n",
       "      <td>26</td>\n",
       "      <td>111</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>71</td>\n",
       "      <td>17</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.448</td>\n",
       "      <td>0.632479</td>\n",
       "      <td>0.274194</td>\n",
       "      <td>0.477928</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.4375</td>\n",
       "      <td>0.528571</td>\n",
       "      <td>0.653846</td>\n",
       "      <td>0.550977</td>\n",
       "      <td>0.459016</td>\n",
       "      <td>0.787234</td>\n",
       "      <td>0.173469</td>\n",
       "      <td>0.52</td>\n",
       "      <td>3</td>\n",
       "      <td>61</td>\n",
       "      <td>141</td>\n",
       "      <td>98</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>32</td>\n",
       "      <td>34</td>\n",
       "      <td>66</td>\n",
       "      <td>11</td>\n",
       "      <td>266</td>\n",
       "      <td>61</td>\n",
       "      <td>12</td>\n",
       "      <td>45</td>\n",
       "      <td>173</td>\n",
       "      <td>0.672857</td>\n",
       "      <td>0.342246</td>\n",
       "      <td>0.778917</td>\n",
       "      <td>0.65283</td>\n",
       "      <td>0.655145</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.581818</td>\n",
       "      <td>0.771014</td>\n",
       "      <td>0.576667</td>\n",
       "      <td>0.67148</td>\n",
       "      <td>0.242424</td>\n",
       "      <td>0.786982</td>\n",
       "      <td>0.752174</td>\n",
       "      <td>0.672857</td>\n",
       "      <td>3</td>\n",
       "      <td>132</td>\n",
       "      <td>338</td>\n",
       "      <td>230</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>113</td>\n",
       "      <td>27</td>\n",
       "      <td>9</td>\n",
       "      <td>34</td>\n",
       "      <td>55</td>\n",
       "      <td>0.586667</td>\n",
       "      <td>0.202532</td>\n",
       "      <td>0.743421</td>\n",
       "      <td>0.506912</td>\n",
       "      <td>0.556181</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.693252</td>\n",
       "      <td>0.462185</td>\n",
       "      <td>0.567179</td>\n",
       "      <td>0.131148</td>\n",
       "      <td>0.801418</td>\n",
       "      <td>0.561224</td>\n",
       "      <td>0.586667</td>\n",
       "      <td>3</td>\n",
       "      <td>61</td>\n",
       "      <td>141</td>\n",
       "      <td>98</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>121</td>\n",
       "      <td>0</td>\n",
       "      <td>297</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>219</td>\n",
       "      <td>0.738571</td>\n",
       "      <td>0.0150376</td>\n",
       "      <td>0.905488</td>\n",
       "      <td>0.716858</td>\n",
       "      <td>0.675596</td>\n",
       "      <td>NN</td>\n",
       "      <td>1</td>\n",
       "      <td>0.933962</td>\n",
       "      <td>0.574803</td>\n",
       "      <td>0.828406</td>\n",
       "      <td>0.00757576</td>\n",
       "      <td>0.878698</td>\n",
       "      <td>0.952174</td>\n",
       "      <td>0.738571</td>\n",
       "      <td>3</td>\n",
       "      <td>132</td>\n",
       "      <td>338</td>\n",
       "      <td>230</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>108</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>69</td>\n",
       "      <td>0.593333</td>\n",
       "      <td>0.031746</td>\n",
       "      <td>0.739726</td>\n",
       "      <td>0.563265</td>\n",
       "      <td>0.538126</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.715232</td>\n",
       "      <td>0.469388</td>\n",
       "      <td>0.591159</td>\n",
       "      <td>0.0163934</td>\n",
       "      <td>0.765957</td>\n",
       "      <td>0.704082</td>\n",
       "      <td>0.593333</td>\n",
       "      <td>3</td>\n",
       "      <td>61</td>\n",
       "      <td>141</td>\n",
       "      <td>98</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>51</td>\n",
       "      <td>76</td>\n",
       "      <td>9</td>\n",
       "      <td>66</td>\n",
       "      <td>231</td>\n",
       "      <td>23</td>\n",
       "      <td>19</td>\n",
       "      <td>177</td>\n",
       "      <td>48</td>\n",
       "      <td>0.471429</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.574627</td>\n",
       "      <td>0.296296</td>\n",
       "      <td>0.438824</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.477273</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.500182</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.721875</td>\n",
       "      <td>0.196721</td>\n",
       "      <td>0.471429</td>\n",
       "      <td>4</td>\n",
       "      <td>136</td>\n",
       "      <td>320</td>\n",
       "      <td>244</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>29</td>\n",
       "      <td>26</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>117</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>58</td>\n",
       "      <td>20</td>\n",
       "      <td>0.553333</td>\n",
       "      <td>0.467742</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.344828</td>\n",
       "      <td>0.529923</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.432836</td>\n",
       "      <td>0.58209</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.565746</td>\n",
       "      <td>0.508772</td>\n",
       "      <td>0.735849</td>\n",
       "      <td>0.238095</td>\n",
       "      <td>0.553333</td>\n",
       "      <td>4</td>\n",
       "      <td>57</td>\n",
       "      <td>159</td>\n",
       "      <td>84</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>42</td>\n",
       "      <td>27</td>\n",
       "      <td>67</td>\n",
       "      <td>6</td>\n",
       "      <td>258</td>\n",
       "      <td>56</td>\n",
       "      <td>29</td>\n",
       "      <td>53</td>\n",
       "      <td>162</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.394366</td>\n",
       "      <td>0.784195</td>\n",
       "      <td>0.612476</td>\n",
       "      <td>0.6486</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.763314</td>\n",
       "      <td>0.568421</td>\n",
       "      <td>0.653053</td>\n",
       "      <td>0.308824</td>\n",
       "      <td>0.80625</td>\n",
       "      <td>0.663934</td>\n",
       "      <td>0.66</td>\n",
       "      <td>4</td>\n",
       "      <td>136</td>\n",
       "      <td>320</td>\n",
       "      <td>244</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>32</td>\n",
       "      <td>8</td>\n",
       "      <td>123</td>\n",
       "      <td>28</td>\n",
       "      <td>12</td>\n",
       "      <td>23</td>\n",
       "      <td>49</td>\n",
       "      <td>0.613333</td>\n",
       "      <td>0.269663</td>\n",
       "      <td>0.773585</td>\n",
       "      <td>0.507772</td>\n",
       "      <td>0.603412</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.773585</td>\n",
       "      <td>0.449541</td>\n",
       "      <td>0.607122</td>\n",
       "      <td>0.210526</td>\n",
       "      <td>0.773585</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>0.613333</td>\n",
       "      <td>4</td>\n",
       "      <td>57</td>\n",
       "      <td>159</td>\n",
       "      <td>84</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>125</td>\n",
       "      <td>0</td>\n",
       "      <td>301</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>219</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.070922</td>\n",
       "      <td>0.923313</td>\n",
       "      <td>0.721582</td>\n",
       "      <td>0.687388</td>\n",
       "      <td>NN</td>\n",
       "      <td>1</td>\n",
       "      <td>0.906627</td>\n",
       "      <td>0.603306</td>\n",
       "      <td>0.819039</td>\n",
       "      <td>0.0367647</td>\n",
       "      <td>0.940625</td>\n",
       "      <td>0.897541</td>\n",
       "      <td>0.75</td>\n",
       "      <td>4</td>\n",
       "      <td>136</td>\n",
       "      <td>320</td>\n",
       "      <td>244</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>118</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>67</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>0</td>\n",
       "      <td>0.771242</td>\n",
       "      <td>0.567797</td>\n",
       "      <td>0.567741</td>\n",
       "      <td>NN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.802721</td>\n",
       "      <td>0.440789</td>\n",
       "      <td>0.548863</td>\n",
       "      <td>0</td>\n",
       "      <td>0.742138</td>\n",
       "      <td>0.797619</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>4</td>\n",
       "      <td>57</td>\n",
       "      <td>159</td>\n",
       "      <td>84</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>57</td>\n",
       "      <td>72</td>\n",
       "      <td>8</td>\n",
       "      <td>68</td>\n",
       "      <td>239</td>\n",
       "      <td>25</td>\n",
       "      <td>21</td>\n",
       "      <td>161</td>\n",
       "      <td>49</td>\n",
       "      <td>0.492857</td>\n",
       "      <td>0.402827</td>\n",
       "      <td>0.594527</td>\n",
       "      <td>0.313099</td>\n",
       "      <td>0.464137</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.390411</td>\n",
       "      <td>0.506356</td>\n",
       "      <td>0.597561</td>\n",
       "      <td>0.513762</td>\n",
       "      <td>0.416058</td>\n",
       "      <td>0.71988</td>\n",
       "      <td>0.212121</td>\n",
       "      <td>0.492857</td>\n",
       "      <td>5</td>\n",
       "      <td>137</td>\n",
       "      <td>332</td>\n",
       "      <td>231</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>23</td>\n",
       "      <td>30</td>\n",
       "      <td>3</td>\n",
       "      <td>30</td>\n",
       "      <td>109</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>74</td>\n",
       "      <td>19</td>\n",
       "      <td>0.503333</td>\n",
       "      <td>0.40708</td>\n",
       "      <td>0.605556</td>\n",
       "      <td>0.299213</td>\n",
       "      <td>0.469456</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.403509</td>\n",
       "      <td>0.511737</td>\n",
       "      <td>0.633333</td>\n",
       "      <td>0.530851</td>\n",
       "      <td>0.410714</td>\n",
       "      <td>0.741497</td>\n",
       "      <td>0.195876</td>\n",
       "      <td>0.503333</td>\n",
       "      <td>5</td>\n",
       "      <td>56</td>\n",
       "      <td>147</td>\n",
       "      <td>97</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>34</td>\n",
       "      <td>44</td>\n",
       "      <td>59</td>\n",
       "      <td>10</td>\n",
       "      <td>277</td>\n",
       "      <td>45</td>\n",
       "      <td>21</td>\n",
       "      <td>65</td>\n",
       "      <td>145</td>\n",
       "      <td>0.651429</td>\n",
       "      <td>0.336634</td>\n",
       "      <td>0.771588</td>\n",
       "      <td>0.604167</td>\n",
       "      <td>0.631212</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.523077</td>\n",
       "      <td>0.717617</td>\n",
       "      <td>0.582329</td>\n",
       "      <td>0.634898</td>\n",
       "      <td>0.248175</td>\n",
       "      <td>0.834337</td>\n",
       "      <td>0.627706</td>\n",
       "      <td>0.651429</td>\n",
       "      <td>5</td>\n",
       "      <td>137</td>\n",
       "      <td>332</td>\n",
       "      <td>231</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>117</td>\n",
       "      <td>28</td>\n",
       "      <td>12</td>\n",
       "      <td>34</td>\n",
       "      <td>51</td>\n",
       "      <td>0.593333</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.745223</td>\n",
       "      <td>0.495146</td>\n",
       "      <td>0.571923</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.700599</td>\n",
       "      <td>0.46789</td>\n",
       "      <td>0.572356</td>\n",
       "      <td>0.178571</td>\n",
       "      <td>0.795918</td>\n",
       "      <td>0.525773</td>\n",
       "      <td>0.593333</td>\n",
       "      <td>5</td>\n",
       "      <td>56</td>\n",
       "      <td>147</td>\n",
       "      <td>97</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>106</td>\n",
       "      <td>1</td>\n",
       "      <td>314</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>36</td>\n",
       "      <td>192</td>\n",
       "      <td>0.744286</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>0.899713</td>\n",
       "      <td>0.703297</td>\n",
       "      <td>0.696447</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.789474</td>\n",
       "      <td>0.857923</td>\n",
       "      <td>0.609524</td>\n",
       "      <td>0.762555</td>\n",
       "      <td>0.109489</td>\n",
       "      <td>0.945783</td>\n",
       "      <td>0.831169</td>\n",
       "      <td>0.744286</td>\n",
       "      <td>5</td>\n",
       "      <td>137</td>\n",
       "      <td>332</td>\n",
       "      <td>231</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>27</td>\n",
       "      <td>3</td>\n",
       "      <td>39</td>\n",
       "      <td>55</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.15625</td>\n",
       "      <td>0.747664</td>\n",
       "      <td>0.511628</td>\n",
       "      <td>0.560948</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.689655</td>\n",
       "      <td>0.466102</td>\n",
       "      <td>0.605304</td>\n",
       "      <td>0.0892857</td>\n",
       "      <td>0.816327</td>\n",
       "      <td>0.56701</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5</td>\n",
       "      <td>56</td>\n",
       "      <td>147</td>\n",
       "      <td>97</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>570</th>\n",
       "      <td>53</td>\n",
       "      <td>69</td>\n",
       "      <td>9</td>\n",
       "      <td>68</td>\n",
       "      <td>243</td>\n",
       "      <td>27</td>\n",
       "      <td>18</td>\n",
       "      <td>165</td>\n",
       "      <td>48</td>\n",
       "      <td>0.491429</td>\n",
       "      <td>0.392593</td>\n",
       "      <td>0.596319</td>\n",
       "      <td>0.304762</td>\n",
       "      <td>0.461979</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.381295</td>\n",
       "      <td>0.509434</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.505912</td>\n",
       "      <td>0.40458</td>\n",
       "      <td>0.718935</td>\n",
       "      <td>0.207792</td>\n",
       "      <td>0.491429</td>\n",
       "      <td>96</td>\n",
       "      <td>131</td>\n",
       "      <td>338</td>\n",
       "      <td>231</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>571</th>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>2</td>\n",
       "      <td>30</td>\n",
       "      <td>105</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>70</td>\n",
       "      <td>20</td>\n",
       "      <td>0.506667</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.601719</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.474846</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.421875</td>\n",
       "      <td>0.504808</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.555399</td>\n",
       "      <td>0.435484</td>\n",
       "      <td>0.744681</td>\n",
       "      <td>0.206186</td>\n",
       "      <td>0.506667</td>\n",
       "      <td>96</td>\n",
       "      <td>62</td>\n",
       "      <td>141</td>\n",
       "      <td>97</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572</th>\n",
       "      <td>35</td>\n",
       "      <td>43</td>\n",
       "      <td>53</td>\n",
       "      <td>4</td>\n",
       "      <td>288</td>\n",
       "      <td>46</td>\n",
       "      <td>16</td>\n",
       "      <td>75</td>\n",
       "      <td>140</td>\n",
       "      <td>0.661429</td>\n",
       "      <td>0.376344</td>\n",
       "      <td>0.774194</td>\n",
       "      <td>0.595745</td>\n",
       "      <td>0.640851</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.70936</td>\n",
       "      <td>0.585774</td>\n",
       "      <td>0.654916</td>\n",
       "      <td>0.267176</td>\n",
       "      <td>0.852071</td>\n",
       "      <td>0.606061</td>\n",
       "      <td>0.661429</td>\n",
       "      <td>96</td>\n",
       "      <td>131</td>\n",
       "      <td>338</td>\n",
       "      <td>231</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>573</th>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>37</td>\n",
       "      <td>3</td>\n",
       "      <td>121</td>\n",
       "      <td>17</td>\n",
       "      <td>8</td>\n",
       "      <td>35</td>\n",
       "      <td>54</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.128205</td>\n",
       "      <td>0.763407</td>\n",
       "      <td>0.526829</td>\n",
       "      <td>0.555638</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.3125</td>\n",
       "      <td>0.6875</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.549375</td>\n",
       "      <td>0.0806452</td>\n",
       "      <td>0.858156</td>\n",
       "      <td>0.556701</td>\n",
       "      <td>0.6</td>\n",
       "      <td>96</td>\n",
       "      <td>62</td>\n",
       "      <td>141</td>\n",
       "      <td>97</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>127</td>\n",
       "      <td>0</td>\n",
       "      <td>298</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>225</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.0300752</td>\n",
       "      <td>0.925466</td>\n",
       "      <td>0.722311</td>\n",
       "      <td>0.690859</td>\n",
       "      <td>NN</td>\n",
       "      <td>1</td>\n",
       "      <td>0.973856</td>\n",
       "      <td>0.57398</td>\n",
       "      <td>0.84679</td>\n",
       "      <td>0.0152672</td>\n",
       "      <td>0.881657</td>\n",
       "      <td>0.974026</td>\n",
       "      <td>0.75</td>\n",
       "      <td>96</td>\n",
       "      <td>131</td>\n",
       "      <td>338</td>\n",
       "      <td>231</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>52</td>\n",
       "      <td>1</td>\n",
       "      <td>98</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>76</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>0.03125</td>\n",
       "      <td>0.728625</td>\n",
       "      <td>0.569288</td>\n",
       "      <td>0.532982</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.765625</td>\n",
       "      <td>0.447059</td>\n",
       "      <td>0.607726</td>\n",
       "      <td>0.016129</td>\n",
       "      <td>0.695035</td>\n",
       "      <td>0.783505</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>96</td>\n",
       "      <td>62</td>\n",
       "      <td>141</td>\n",
       "      <td>97</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>576</th>\n",
       "      <td>57</td>\n",
       "      <td>77</td>\n",
       "      <td>6</td>\n",
       "      <td>60</td>\n",
       "      <td>258</td>\n",
       "      <td>24</td>\n",
       "      <td>20</td>\n",
       "      <td>152</td>\n",
       "      <td>46</td>\n",
       "      <td>0.515714</td>\n",
       "      <td>0.411552</td>\n",
       "      <td>0.622437</td>\n",
       "      <td>0.312925</td>\n",
       "      <td>0.483869</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.416058</td>\n",
       "      <td>0.529774</td>\n",
       "      <td>0.605263</td>\n",
       "      <td>0.53054</td>\n",
       "      <td>0.407143</td>\n",
       "      <td>0.754386</td>\n",
       "      <td>0.211009</td>\n",
       "      <td>0.515714</td>\n",
       "      <td>97</td>\n",
       "      <td>140</td>\n",
       "      <td>342</td>\n",
       "      <td>218</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577</th>\n",
       "      <td>23</td>\n",
       "      <td>25</td>\n",
       "      <td>5</td>\n",
       "      <td>38</td>\n",
       "      <td>90</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>83</td>\n",
       "      <td>22</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.386555</td>\n",
       "      <td>0.537313</td>\n",
       "      <td>0.30137</td>\n",
       "      <td>0.424167</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.348485</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.611111</td>\n",
       "      <td>0.493215</td>\n",
       "      <td>0.433962</td>\n",
       "      <td>0.656934</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.45</td>\n",
       "      <td>97</td>\n",
       "      <td>53</td>\n",
       "      <td>137</td>\n",
       "      <td>110</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>578</th>\n",
       "      <td>53</td>\n",
       "      <td>34</td>\n",
       "      <td>53</td>\n",
       "      <td>16</td>\n",
       "      <td>278</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>53</td>\n",
       "      <td>117</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.412451</td>\n",
       "      <td>0.786421</td>\n",
       "      <td>0.536697</td>\n",
       "      <td>0.633856</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.452991</td>\n",
       "      <td>0.761644</td>\n",
       "      <td>0.536697</td>\n",
       "      <td>0.629859</td>\n",
       "      <td>0.378571</td>\n",
       "      <td>0.812865</td>\n",
       "      <td>0.536697</td>\n",
       "      <td>0.64</td>\n",
       "      <td>97</td>\n",
       "      <td>140</td>\n",
       "      <td>342</td>\n",
       "      <td>218</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>579</th>\n",
       "      <td>20</td>\n",
       "      <td>12</td>\n",
       "      <td>21</td>\n",
       "      <td>5</td>\n",
       "      <td>102</td>\n",
       "      <td>30</td>\n",
       "      <td>15</td>\n",
       "      <td>32</td>\n",
       "      <td>63</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>0.430108</td>\n",
       "      <td>0.720848</td>\n",
       "      <td>0.5625</td>\n",
       "      <td>0.611423</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.69863</td>\n",
       "      <td>0.552632</td>\n",
       "      <td>0.610006</td>\n",
       "      <td>0.377358</td>\n",
       "      <td>0.744526</td>\n",
       "      <td>0.572727</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>97</td>\n",
       "      <td>53</td>\n",
       "      <td>137</td>\n",
       "      <td>110</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580</th>\n",
       "      <td>34</td>\n",
       "      <td>10</td>\n",
       "      <td>96</td>\n",
       "      <td>1</td>\n",
       "      <td>291</td>\n",
       "      <td>50</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>194</td>\n",
       "      <td>0.741429</td>\n",
       "      <td>0.365591</td>\n",
       "      <td>0.887195</td>\n",
       "      <td>0.695341</td>\n",
       "      <td>0.723125</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.73913</td>\n",
       "      <td>0.926752</td>\n",
       "      <td>0.570588</td>\n",
       "      <td>0.778308</td>\n",
       "      <td>0.242857</td>\n",
       "      <td>0.850877</td>\n",
       "      <td>0.889908</td>\n",
       "      <td>0.741429</td>\n",
       "      <td>97</td>\n",
       "      <td>140</td>\n",
       "      <td>342</td>\n",
       "      <td>218</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>581</th>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>97</td>\n",
       "      <td>35</td>\n",
       "      <td>5</td>\n",
       "      <td>22</td>\n",
       "      <td>83</td>\n",
       "      <td>0.623333</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.740458</td>\n",
       "      <td>0.619403</td>\n",
       "      <td>0.60059</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.411765</td>\n",
       "      <td>0.776</td>\n",
       "      <td>0.525316</td>\n",
       "      <td>0.619734</td>\n",
       "      <td>0.132075</td>\n",
       "      <td>0.708029</td>\n",
       "      <td>0.754545</td>\n",
       "      <td>0.623333</td>\n",
       "      <td>97</td>\n",
       "      <td>53</td>\n",
       "      <td>137</td>\n",
       "      <td>110</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>582</th>\n",
       "      <td>63</td>\n",
       "      <td>75</td>\n",
       "      <td>10</td>\n",
       "      <td>78</td>\n",
       "      <td>228</td>\n",
       "      <td>24</td>\n",
       "      <td>19</td>\n",
       "      <td>157</td>\n",
       "      <td>46</td>\n",
       "      <td>0.481429</td>\n",
       "      <td>0.409091</td>\n",
       "      <td>0.577215</td>\n",
       "      <td>0.304636</td>\n",
       "      <td>0.455222</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.39375</td>\n",
       "      <td>0.495652</td>\n",
       "      <td>0.575</td>\n",
       "      <td>0.499272</td>\n",
       "      <td>0.425676</td>\n",
       "      <td>0.690909</td>\n",
       "      <td>0.207207</td>\n",
       "      <td>0.481429</td>\n",
       "      <td>98</td>\n",
       "      <td>148</td>\n",
       "      <td>330</td>\n",
       "      <td>222</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>583</th>\n",
       "      <td>17</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>120</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>78</td>\n",
       "      <td>22</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.386364</td>\n",
       "      <td>0.641711</td>\n",
       "      <td>0.318841</td>\n",
       "      <td>0.489328</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.395349</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.6875</td>\n",
       "      <td>0.567108</td>\n",
       "      <td>0.377778</td>\n",
       "      <td>0.805369</td>\n",
       "      <td>0.207547</td>\n",
       "      <td>0.53</td>\n",
       "      <td>98</td>\n",
       "      <td>45</td>\n",
       "      <td>149</td>\n",
       "      <td>106</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>584</th>\n",
       "      <td>61</td>\n",
       "      <td>46</td>\n",
       "      <td>41</td>\n",
       "      <td>10</td>\n",
       "      <td>279</td>\n",
       "      <td>41</td>\n",
       "      <td>37</td>\n",
       "      <td>73</td>\n",
       "      <td>112</td>\n",
       "      <td>0.645714</td>\n",
       "      <td>0.476562</td>\n",
       "      <td>0.766484</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.63287</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.564815</td>\n",
       "      <td>0.701005</td>\n",
       "      <td>0.57732</td>\n",
       "      <td>0.632985</td>\n",
       "      <td>0.412162</td>\n",
       "      <td>0.845455</td>\n",
       "      <td>0.504505</td>\n",
       "      <td>0.645714</td>\n",
       "      <td>98</td>\n",
       "      <td>148</td>\n",
       "      <td>330</td>\n",
       "      <td>222</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>585</th>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>4</td>\n",
       "      <td>125</td>\n",
       "      <td>20</td>\n",
       "      <td>24</td>\n",
       "      <td>31</td>\n",
       "      <td>51</td>\n",
       "      <td>0.646667</td>\n",
       "      <td>0.395604</td>\n",
       "      <td>0.788644</td>\n",
       "      <td>0.53125</td>\n",
       "      <td>0.638742</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.391304</td>\n",
       "      <td>0.744048</td>\n",
       "      <td>0.593023</td>\n",
       "      <td>0.637774</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.838926</td>\n",
       "      <td>0.481132</td>\n",
       "      <td>0.646667</td>\n",
       "      <td>98</td>\n",
       "      <td>45</td>\n",
       "      <td>149</td>\n",
       "      <td>106</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>586</th>\n",
       "      <td>93</td>\n",
       "      <td>12</td>\n",
       "      <td>43</td>\n",
       "      <td>12</td>\n",
       "      <td>259</td>\n",
       "      <td>59</td>\n",
       "      <td>53</td>\n",
       "      <td>19</td>\n",
       "      <td>150</td>\n",
       "      <td>0.717143</td>\n",
       "      <td>0.607843</td>\n",
       "      <td>0.835484</td>\n",
       "      <td>0.632911</td>\n",
       "      <td>0.72311</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.588608</td>\n",
       "      <td>0.893103</td>\n",
       "      <td>0.595238</td>\n",
       "      <td>0.734258</td>\n",
       "      <td>0.628378</td>\n",
       "      <td>0.784848</td>\n",
       "      <td>0.675676</td>\n",
       "      <td>0.717143</td>\n",
       "      <td>98</td>\n",
       "      <td>148</td>\n",
       "      <td>330</td>\n",
       "      <td>222</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>587</th>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>8</td>\n",
       "      <td>108</td>\n",
       "      <td>33</td>\n",
       "      <td>32</td>\n",
       "      <td>13</td>\n",
       "      <td>61</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.380952</td>\n",
       "      <td>0.788321</td>\n",
       "      <td>0.552036</td>\n",
       "      <td>0.643728</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.864</td>\n",
       "      <td>0.530435</td>\n",
       "      <td>0.66654</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.724832</td>\n",
       "      <td>0.575472</td>\n",
       "      <td>0.63</td>\n",
       "      <td>98</td>\n",
       "      <td>45</td>\n",
       "      <td>149</td>\n",
       "      <td>106</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>52</td>\n",
       "      <td>65</td>\n",
       "      <td>5</td>\n",
       "      <td>77</td>\n",
       "      <td>240</td>\n",
       "      <td>26</td>\n",
       "      <td>22</td>\n",
       "      <td>169</td>\n",
       "      <td>44</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.380952</td>\n",
       "      <td>0.587515</td>\n",
       "      <td>0.283871</td>\n",
       "      <td>0.449577</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.344371</td>\n",
       "      <td>0.506329</td>\n",
       "      <td>0.586667</td>\n",
       "      <td>0.505073</td>\n",
       "      <td>0.42623</td>\n",
       "      <td>0.699708</td>\n",
       "      <td>0.187234</td>\n",
       "      <td>0.48</td>\n",
       "      <td>99</td>\n",
       "      <td>122</td>\n",
       "      <td>343</td>\n",
       "      <td>235</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>28</td>\n",
       "      <td>37</td>\n",
       "      <td>6</td>\n",
       "      <td>21</td>\n",
       "      <td>108</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>66</td>\n",
       "      <td>24</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.455285</td>\n",
       "      <td>0.622478</td>\n",
       "      <td>0.369231</td>\n",
       "      <td>0.504402</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.511848</td>\n",
       "      <td>0.648649</td>\n",
       "      <td>0.560555</td>\n",
       "      <td>0.394366</td>\n",
       "      <td>0.794118</td>\n",
       "      <td>0.258065</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>99</td>\n",
       "      <td>71</td>\n",
       "      <td>136</td>\n",
       "      <td>93</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590</th>\n",
       "      <td>38</td>\n",
       "      <td>32</td>\n",
       "      <td>52</td>\n",
       "      <td>12</td>\n",
       "      <td>290</td>\n",
       "      <td>41</td>\n",
       "      <td>29</td>\n",
       "      <td>64</td>\n",
       "      <td>142</td>\n",
       "      <td>0.671429</td>\n",
       "      <td>0.378109</td>\n",
       "      <td>0.79561</td>\n",
       "      <td>0.604255</td>\n",
       "      <td>0.658605</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.481013</td>\n",
       "      <td>0.751295</td>\n",
       "      <td>0.604255</td>\n",
       "      <td>0.654825</td>\n",
       "      <td>0.311475</td>\n",
       "      <td>0.845481</td>\n",
       "      <td>0.604255</td>\n",
       "      <td>0.671429</td>\n",
       "      <td>99</td>\n",
       "      <td>122</td>\n",
       "      <td>343</td>\n",
       "      <td>235</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>18</td>\n",
       "      <td>25</td>\n",
       "      <td>28</td>\n",
       "      <td>6</td>\n",
       "      <td>101</td>\n",
       "      <td>29</td>\n",
       "      <td>15</td>\n",
       "      <td>30</td>\n",
       "      <td>48</td>\n",
       "      <td>0.556667</td>\n",
       "      <td>0.327273</td>\n",
       "      <td>0.691781</td>\n",
       "      <td>0.484848</td>\n",
       "      <td>0.541365</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.647436</td>\n",
       "      <td>0.457143</td>\n",
       "      <td>0.544449</td>\n",
       "      <td>0.253521</td>\n",
       "      <td>0.742647</td>\n",
       "      <td>0.516129</td>\n",
       "      <td>0.556667</td>\n",
       "      <td>99</td>\n",
       "      <td>71</td>\n",
       "      <td>136</td>\n",
       "      <td>93</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>15</td>\n",
       "      <td>19</td>\n",
       "      <td>88</td>\n",
       "      <td>0</td>\n",
       "      <td>283</td>\n",
       "      <td>60</td>\n",
       "      <td>12</td>\n",
       "      <td>40</td>\n",
       "      <td>183</td>\n",
       "      <td>0.687143</td>\n",
       "      <td>0.201342</td>\n",
       "      <td>0.826277</td>\n",
       "      <td>0.646643</td>\n",
       "      <td>0.657054</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.827485</td>\n",
       "      <td>0.55287</td>\n",
       "      <td>0.6879</td>\n",
       "      <td>0.122951</td>\n",
       "      <td>0.825073</td>\n",
       "      <td>0.778723</td>\n",
       "      <td>0.687143</td>\n",
       "      <td>99</td>\n",
       "      <td>122</td>\n",
       "      <td>343</td>\n",
       "      <td>235</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593</th>\n",
       "      <td>5</td>\n",
       "      <td>18</td>\n",
       "      <td>48</td>\n",
       "      <td>4</td>\n",
       "      <td>92</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>67</td>\n",
       "      <td>0.546667</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>0.689139</td>\n",
       "      <td>0.540323</td>\n",
       "      <td>0.507753</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.357143</td>\n",
       "      <td>0.70229</td>\n",
       "      <td>0.432258</td>\n",
       "      <td>0.536895</td>\n",
       "      <td>0.0704225</td>\n",
       "      <td>0.676471</td>\n",
       "      <td>0.72043</td>\n",
       "      <td>0.546667</td>\n",
       "      <td>99</td>\n",
       "      <td>71</td>\n",
       "      <td>136</td>\n",
       "      <td>93</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>594</th>\n",
       "      <td>57</td>\n",
       "      <td>75</td>\n",
       "      <td>7</td>\n",
       "      <td>70</td>\n",
       "      <td>246</td>\n",
       "      <td>23</td>\n",
       "      <td>19</td>\n",
       "      <td>152</td>\n",
       "      <td>51</td>\n",
       "      <td>0.505714</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.605911</td>\n",
       "      <td>0.336634</td>\n",
       "      <td>0.479624</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.390411</td>\n",
       "      <td>0.520085</td>\n",
       "      <td>0.62963</td>\n",
       "      <td>0.529077</td>\n",
       "      <td>0.410072</td>\n",
       "      <td>0.725664</td>\n",
       "      <td>0.22973</td>\n",
       "      <td>0.505714</td>\n",
       "      <td>100</td>\n",
       "      <td>139</td>\n",
       "      <td>339</td>\n",
       "      <td>222</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>595</th>\n",
       "      <td>23</td>\n",
       "      <td>27</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>102</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>83</td>\n",
       "      <td>17</td>\n",
       "      <td>0.473333</td>\n",
       "      <td>0.414414</td>\n",
       "      <td>0.579545</td>\n",
       "      <td>0.248175</td>\n",
       "      <td>0.432738</td>\n",
       "      <td>LM</td>\n",
       "      <td>0.403509</td>\n",
       "      <td>0.481132</td>\n",
       "      <td>0.548387</td>\n",
       "      <td>0.490923</td>\n",
       "      <td>0.425926</td>\n",
       "      <td>0.728571</td>\n",
       "      <td>0.160377</td>\n",
       "      <td>0.473333</td>\n",
       "      <td>100</td>\n",
       "      <td>54</td>\n",
       "      <td>140</td>\n",
       "      <td>106</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>596</th>\n",
       "      <td>61</td>\n",
       "      <td>43</td>\n",
       "      <td>35</td>\n",
       "      <td>15</td>\n",
       "      <td>283</td>\n",
       "      <td>41</td>\n",
       "      <td>44</td>\n",
       "      <td>68</td>\n",
       "      <td>110</td>\n",
       "      <td>0.648571</td>\n",
       "      <td>0.471042</td>\n",
       "      <td>0.772169</td>\n",
       "      <td>0.539216</td>\n",
       "      <td>0.638494</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.508333</td>\n",
       "      <td>0.718274</td>\n",
       "      <td>0.591398</td>\n",
       "      <td>0.636348</td>\n",
       "      <td>0.438849</td>\n",
       "      <td>0.834808</td>\n",
       "      <td>0.495495</td>\n",
       "      <td>0.648571</td>\n",
       "      <td>100</td>\n",
       "      <td>139</td>\n",
       "      <td>339</td>\n",
       "      <td>222</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>597</th>\n",
       "      <td>24</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>112</td>\n",
       "      <td>15</td>\n",
       "      <td>25</td>\n",
       "      <td>35</td>\n",
       "      <td>46</td>\n",
       "      <td>0.606667</td>\n",
       "      <td>0.413793</td>\n",
       "      <td>0.746667</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.599594</td>\n",
       "      <td>GV</td>\n",
       "      <td>0.387097</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.589744</td>\n",
       "      <td>0.60472</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.433962</td>\n",
       "      <td>0.606667</td>\n",
       "      <td>100</td>\n",
       "      <td>54</td>\n",
       "      <td>140</td>\n",
       "      <td>106</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>598</th>\n",
       "      <td>103</td>\n",
       "      <td>10</td>\n",
       "      <td>26</td>\n",
       "      <td>9</td>\n",
       "      <td>312</td>\n",
       "      <td>18</td>\n",
       "      <td>78</td>\n",
       "      <td>35</td>\n",
       "      <td>109</td>\n",
       "      <td>0.748571</td>\n",
       "      <td>0.62614</td>\n",
       "      <td>0.896552</td>\n",
       "      <td>0.581333</td>\n",
       "      <td>0.742886</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.542105</td>\n",
       "      <td>0.87395</td>\n",
       "      <td>0.712418</td>\n",
       "      <td>0.756826</td>\n",
       "      <td>0.741007</td>\n",
       "      <td>0.920354</td>\n",
       "      <td>0.490991</td>\n",
       "      <td>0.748571</td>\n",
       "      <td>100</td>\n",
       "      <td>139</td>\n",
       "      <td>339</td>\n",
       "      <td>222</td>\n",
       "      <td>700</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599</th>\n",
       "      <td>34</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>108</td>\n",
       "      <td>21</td>\n",
       "      <td>37</td>\n",
       "      <td>30</td>\n",
       "      <td>39</td>\n",
       "      <td>0.603333</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.755245</td>\n",
       "      <td>0.438202</td>\n",
       "      <td>0.597279</td>\n",
       "      <td>NN</td>\n",
       "      <td>0.414634</td>\n",
       "      <td>0.739726</td>\n",
       "      <td>0.541667</td>\n",
       "      <td>0.611229</td>\n",
       "      <td>0.62963</td>\n",
       "      <td>0.771429</td>\n",
       "      <td>0.367925</td>\n",
       "      <td>0.603333</td>\n",
       "      <td>100</td>\n",
       "      <td>54</td>\n",
       "      <td>140</td>\n",
       "      <td>106</td>\n",
       "      <td>300</td>\n",
       "      <td>TE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>600 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    A0P0 A0P1 A0P2 A1P0 A1P1 A1P2 A2P0 A2P1 A2P2       acc       f1_0  \\\n",
       "0     59   70    6   66  242   22   18  169   48  0.498571    0.42446   \n",
       "1     21   32    5   32  106   11    7   66   20      0.49   0.355932   \n",
       "2     42   42   51   13  274   43   18   64  153      0.67   0.403846   \n",
       "3     13   11   34    7  118   24   17   35   41  0.573333   0.273684   \n",
       "4     15   11  109    0  312   18    4   37  194  0.744286   0.194805   \n",
       "5      4   10   44    1  122   26    3   33   57      0.61   0.121212   \n",
       "6     54   70    9   75  241   18   15  168   50  0.492857   0.389892   \n",
       "7     26   32    2   23  107   15   10   67   18  0.503333   0.436975   \n",
       "8     56   31   46   14  266   54   32   62  139  0.658571   0.476596   \n",
       "9     17   19   24    8  116   21   14   26   55  0.626667   0.343434   \n",
       "10    24   13   96    4  304   26   12   44  177  0.721429   0.277457   \n",
       "11     8   13   39    2  114   29    7   27   61      0.61   0.207792   \n",
       "12    52   74    6   72  237   29   15  164   51  0.485714   0.383764   \n",
       "13    28   28    5   26  111    4   10   71   17      0.52      0.448   \n",
       "14    32   34   66   11  266   61   12   45  173  0.672857   0.342246   \n",
       "15     8   16   37    1  113   27    9   34   55  0.586667   0.202532   \n",
       "16     1   10  121    0  297   41    0   11  219  0.738571  0.0150376   \n",
       "17     1   14   46    1  108   32    0   29   69  0.593333   0.031746   \n",
       "18    51   76    9   66  231   23   19  177   48  0.471429      0.375   \n",
       "19    29   26    2   32  117   10    6   58   20  0.553333   0.467742   \n",
       "20    42   27   67    6  258   56   29   53  162      0.66   0.394366   \n",
       "21    12   13   32    8  123   28   12   23   49  0.613333   0.269663   \n",
       "22     5    6  125    0  301   19    0   25  219      0.75   0.070922   \n",
       "23     0   12   45    1  118   40    0   17   67  0.616667          0   \n",
       "24    57   72    8   68  239   25   21  161   49  0.492857   0.402827   \n",
       "25    23   30    3   30  109    8    4   74   19  0.503333    0.40708   \n",
       "26    34   44   59   10  277   45   21   65  145  0.651429   0.336634   \n",
       "27    10   16   30    2  117   28   12   34   51  0.593333       0.25   \n",
       "28    15   16  106    1  314   17    3   36  192  0.744286   0.192308   \n",
       "29     5   15   36    0  120   27    3   39   55       0.6    0.15625   \n",
       "..   ...  ...  ...  ...  ...  ...  ...  ...  ...       ...        ...   \n",
       "570   53   69    9   68  243   27   18  165   48  0.491429   0.392593   \n",
       "571   27   33    2   30  105    6    7   70   20  0.506667   0.428571   \n",
       "572   35   43   53    4  288   46   16   75  140  0.661429   0.376344   \n",
       "573    5   20   37    3  121   17    8   35   54       0.6   0.128205   \n",
       "574    2    2  127    0  298   40    0    6  225      0.75  0.0300752   \n",
       "575    1    9   52    1   98   42    0   21   76  0.583333    0.03125   \n",
       "576   57   77    6   60  258   24   20  152   46  0.515714   0.411552   \n",
       "577   23   25    5   38   90    9    5   83   22      0.45   0.386555   \n",
       "578   53   34   53   16  278   48   48   53  117      0.64   0.412451   \n",
       "579   20   12   21    5  102   30   15   32   63  0.616667   0.430108   \n",
       "580   34   10   96    1  291   50   11   13  194  0.741429   0.365591   \n",
       "581    7    6   40    5   97   35    5   22   83  0.623333        0.2   \n",
       "582   63   75   10   78  228   24   19  157   46  0.481429   0.409091   \n",
       "583   17   27    1   20  120    9    6   78   22      0.53   0.386364   \n",
       "584   61   46   41   10  279   41   37   73  112  0.645714   0.476562   \n",
       "585   18   12   15    4  125   20   24   31   51  0.646667   0.395604   \n",
       "586   93   12   43   12  259   59   53   19  150  0.717143   0.607843   \n",
       "587   20    4   21    8  108   33   32   13   61      0.63   0.380952   \n",
       "588   52   65    5   77  240   26   22  169   44      0.48   0.380952   \n",
       "589   28   37    6   21  108    7    3   66   24  0.533333   0.455285   \n",
       "590   38   32   52   12  290   41   29   64  142  0.671429   0.378109   \n",
       "591   18   25   28    6  101   29   15   30   48  0.556667   0.327273   \n",
       "592   15   19   88    0  283   60   12   40  183  0.687143   0.201342   \n",
       "593    5   18   48    4   92   40    5   21   67  0.546667   0.117647   \n",
       "594   57   75    7   70  246   23   19  152   51  0.505714        0.4   \n",
       "595   23   27    4   28  102   10    6   83   17  0.473333   0.414414   \n",
       "596   61   43   35   15  283   41   44   68  110  0.648571   0.471042   \n",
       "597   24   13   17   13  112   15   25   35   46  0.606667   0.413793   \n",
       "598  103   10   26    9  312   18   78   35  109  0.748571    0.62614   \n",
       "599   34    8   12   11  108   21   37   30   39  0.603333        0.5   \n",
       "\n",
       "         f1_1      f1_2    f1_all model    prec_0    prec_1    prec_2  \\\n",
       "0    0.596794  0.308682  0.466835    LM  0.412587  0.503119  0.631579   \n",
       "1    0.600567  0.310078  0.463219    LM      0.35  0.519608  0.555556   \n",
       "2    0.771831  0.634855  0.654878    GV  0.575342  0.721053  0.619433   \n",
       "3    0.753994  0.427083  0.559792    GV  0.351351  0.719512  0.414141   \n",
       "4    0.904348  0.697842   0.69818    NN  0.789474  0.866667  0.604361   \n",
       "5     0.77707  0.518182  0.570016    NN       0.5  0.739394  0.448819   \n",
       "6    0.592866  0.322581  0.464334    LM     0.375  0.503132  0.649351   \n",
       "7    0.609687  0.276923  0.469769    LM  0.440678  0.519417  0.514286   \n",
       "8    0.767677  0.588983  0.652892    GV   0.54902  0.740947   0.58159   \n",
       "9     0.75817  0.564103  0.613768    GV  0.435897  0.720497      0.55   \n",
       "10    0.87482  0.665414  0.691619    NN       0.6  0.842105  0.591973   \n",
       "11   0.762542  0.544643  0.582591    NN  0.470588   0.74026  0.472868   \n",
       "12   0.583026  0.322785  0.459943    LM  0.374101  0.498947  0.593023   \n",
       "13   0.632479  0.274194  0.477928    LM    0.4375  0.528571  0.653846   \n",
       "14   0.778917   0.65283  0.655145    GV  0.581818  0.771014  0.576667   \n",
       "15   0.743421  0.506912  0.556181    GV  0.444444  0.693252  0.462185   \n",
       "16   0.905488  0.716858  0.675596    NN         1  0.933962  0.574803   \n",
       "17   0.739726  0.563265  0.538126    NN       0.5  0.715232  0.469388   \n",
       "18   0.574627  0.296296  0.438824    LM     0.375  0.477273       0.6   \n",
       "19       0.65  0.344828  0.529923    LM  0.432836   0.58209     0.625   \n",
       "20   0.784195  0.612476    0.6486    GV  0.545455  0.763314  0.568421   \n",
       "21   0.773585  0.507772  0.603412    GV     0.375  0.773585  0.449541   \n",
       "22   0.923313  0.721582  0.687388    NN         1  0.906627  0.603306   \n",
       "23   0.771242  0.567797  0.567741    NN         0  0.802721  0.440789   \n",
       "24   0.594527  0.313099  0.464137    LM  0.390411  0.506356  0.597561   \n",
       "25   0.605556  0.299213  0.469456    LM  0.403509  0.511737  0.633333   \n",
       "26   0.771588  0.604167  0.631212    GV  0.523077  0.717617  0.582329   \n",
       "27   0.745223  0.495146  0.571923    GV  0.416667  0.700599   0.46789   \n",
       "28   0.899713  0.703297  0.696447    NN  0.789474  0.857923  0.609524   \n",
       "29   0.747664  0.511628  0.560948    NN     0.625  0.689655  0.466102   \n",
       "..        ...       ...       ...   ...       ...       ...       ...   \n",
       "570  0.596319  0.304762  0.461979    LM  0.381295  0.509434  0.571429   \n",
       "571  0.601719      0.32  0.474846    LM  0.421875  0.504808  0.714286   \n",
       "572  0.774194  0.595745  0.640851    GV  0.636364   0.70936  0.585774   \n",
       "573  0.763407  0.526829  0.555638    GV    0.3125    0.6875       0.5   \n",
       "574  0.925466  0.722311  0.690859    NN         1  0.973856   0.57398   \n",
       "575  0.728625  0.569288  0.532982    NN       0.5  0.765625  0.447059   \n",
       "576  0.622437  0.312925  0.483869    LM  0.416058  0.529774  0.605263   \n",
       "577  0.537313   0.30137  0.424167    LM  0.348485  0.454545  0.611111   \n",
       "578  0.786421  0.536697  0.633856    GV  0.452991  0.761644  0.536697   \n",
       "579  0.720848    0.5625  0.611423    GV       0.5   0.69863  0.552632   \n",
       "580  0.887195  0.695341  0.723125    NN   0.73913  0.926752  0.570588   \n",
       "581  0.740458  0.619403   0.60059    NN  0.411765     0.776  0.525316   \n",
       "582  0.577215  0.304636  0.455222    LM   0.39375  0.495652     0.575   \n",
       "583  0.641711  0.318841  0.489328    LM  0.395349  0.533333    0.6875   \n",
       "584  0.766484  0.538462   0.63287    GV  0.564815  0.701005   0.57732   \n",
       "585  0.788644   0.53125  0.638742    GV  0.391304  0.744048  0.593023   \n",
       "586  0.835484  0.632911   0.72311    NN  0.588608  0.893103  0.595238   \n",
       "587  0.788321  0.552036  0.643728    NN  0.333333     0.864  0.530435   \n",
       "588  0.587515  0.283871  0.449577    LM  0.344371  0.506329  0.586667   \n",
       "589  0.622478  0.369231  0.504402    LM  0.538462  0.511848  0.648649   \n",
       "590   0.79561  0.604255  0.658605    GV  0.481013  0.751295  0.604255   \n",
       "591  0.691781  0.484848  0.541365    GV  0.461538  0.647436  0.457143   \n",
       "592  0.826277  0.646643  0.657054    NN  0.555556  0.827485   0.55287   \n",
       "593  0.689139  0.540323  0.507753    NN  0.357143   0.70229  0.432258   \n",
       "594  0.605911  0.336634  0.479624    LM  0.390411  0.520085   0.62963   \n",
       "595  0.579545  0.248175  0.432738    LM  0.403509  0.481132  0.548387   \n",
       "596  0.772169  0.539216  0.638494    GV  0.508333  0.718274  0.591398   \n",
       "597  0.746667       0.5  0.599594    GV  0.387097       0.7  0.589744   \n",
       "598  0.896552  0.581333  0.742886    NN  0.542105   0.87395  0.712418   \n",
       "599  0.755245  0.438202  0.597279    NN  0.414634  0.739726  0.541667   \n",
       "\n",
       "     prec_all       rec_0     rec_1     rec_2   rec_all rnd_ct sup_0 sup_1  \\\n",
       "0    0.528785    0.437037  0.733333  0.204255  0.498571      1   135   330   \n",
       "1    0.497961    0.362069  0.711409  0.215054      0.49      1    58   149   \n",
       "2    0.658836    0.311111  0.830303  0.651064      0.67      1   135   330   \n",
       "3    0.553669    0.224138  0.791946   0.44086  0.573333      1    58   149   \n",
       "4     0.76372    0.111111  0.945455  0.825532  0.744286      1   135   330   \n",
       "5    0.603033   0.0689655  0.818792  0.612903      0.61      1    58   149   \n",
       "6    0.527457    0.406015  0.721557  0.214592  0.492857      2   133   334   \n",
       "7    0.502045    0.433333  0.737931  0.189474  0.503333      2    60   145   \n",
       "8    0.651438    0.421053  0.796407  0.596567  0.658571      2   133   334   \n",
       "9    0.609586    0.283333       0.8  0.578947  0.626667      2    60   145   \n",
       "10   0.712847    0.180451   0.91018  0.759657  0.721429      2   133   334   \n",
       "11   0.601651    0.133333  0.786207  0.642105      0.61      2    60   145   \n",
       "12   0.506316    0.393939  0.701183  0.221739  0.485714      3   132   338   \n",
       "13   0.550977    0.459016  0.787234  0.173469      0.52      3    61   141   \n",
       "14    0.67148    0.242424  0.786982  0.752174  0.672857      3   132   338   \n",
       "15   0.567179    0.131148  0.801418  0.561224  0.586667      3    61   141   \n",
       "16   0.828406  0.00757576  0.878698  0.952174  0.738571      3   132   338   \n",
       "17   0.591159   0.0163934  0.765957  0.704082  0.593333      3    61   141   \n",
       "18   0.500182       0.375  0.721875  0.196721  0.471429      4   136   320   \n",
       "19   0.565746    0.508772  0.735849  0.238095  0.553333      4    57   159   \n",
       "20   0.653053    0.308824   0.80625  0.663934      0.66      4   136   320   \n",
       "21   0.607122    0.210526  0.773585  0.583333  0.613333      4    57   159   \n",
       "22   0.819039   0.0367647  0.940625  0.897541      0.75      4   136   320   \n",
       "23   0.548863           0  0.742138  0.797619  0.616667      4    57   159   \n",
       "24   0.513762    0.416058   0.71988  0.212121  0.492857      5   137   332   \n",
       "25   0.530851    0.410714  0.741497  0.195876  0.503333      5    56   147   \n",
       "26   0.634898    0.248175  0.834337  0.627706  0.651429      5   137   332   \n",
       "27   0.572356    0.178571  0.795918  0.525773  0.593333      5    56   147   \n",
       "28   0.762555    0.109489  0.945783  0.831169  0.744286      5   137   332   \n",
       "29   0.605304   0.0892857  0.816327   0.56701       0.6      5    56   147   \n",
       "..        ...         ...       ...       ...       ...    ...   ...   ...   \n",
       "570  0.505912     0.40458  0.718935  0.207792  0.491429     96   131   338   \n",
       "571  0.555399    0.435484  0.744681  0.206186  0.506667     96    62   141   \n",
       "572  0.654916    0.267176  0.852071  0.606061  0.661429     96   131   338   \n",
       "573  0.549375   0.0806452  0.858156  0.556701       0.6     96    62   141   \n",
       "574   0.84679   0.0152672  0.881657  0.974026      0.75     96   131   338   \n",
       "575  0.607726    0.016129  0.695035  0.783505  0.583333     96    62   141   \n",
       "576   0.53054    0.407143  0.754386  0.211009  0.515714     97   140   342   \n",
       "577  0.493215    0.433962  0.656934       0.2      0.45     97    53   137   \n",
       "578  0.629859    0.378571  0.812865  0.536697      0.64     97   140   342   \n",
       "579  0.610006    0.377358  0.744526  0.572727  0.616667     97    53   137   \n",
       "580  0.778308    0.242857  0.850877  0.889908  0.741429     97   140   342   \n",
       "581  0.619734    0.132075  0.708029  0.754545  0.623333     97    53   137   \n",
       "582  0.499272    0.425676  0.690909  0.207207  0.481429     98   148   330   \n",
       "583  0.567108    0.377778  0.805369  0.207547      0.53     98    45   149   \n",
       "584  0.632985    0.412162  0.845455  0.504505  0.645714     98   148   330   \n",
       "585  0.637774         0.4  0.838926  0.481132  0.646667     98    45   149   \n",
       "586  0.734258    0.628378  0.784848  0.675676  0.717143     98   148   330   \n",
       "587   0.66654    0.444444  0.724832  0.575472      0.63     98    45   149   \n",
       "588  0.505073     0.42623  0.699708  0.187234      0.48     99   122   343   \n",
       "589  0.560555    0.394366  0.794118  0.258065  0.533333     99    71   136   \n",
       "590  0.654825    0.311475  0.845481  0.604255  0.671429     99   122   343   \n",
       "591  0.544449    0.253521  0.742647  0.516129  0.556667     99    71   136   \n",
       "592    0.6879    0.122951  0.825073  0.778723  0.687143     99   122   343   \n",
       "593  0.536895   0.0704225  0.676471   0.72043  0.546667     99    71   136   \n",
       "594  0.529077    0.410072  0.725664   0.22973  0.505714    100   139   339   \n",
       "595  0.490923    0.425926  0.728571  0.160377  0.473333    100    54   140   \n",
       "596  0.636348    0.438849  0.834808  0.495495  0.648571    100   139   339   \n",
       "597   0.60472    0.444444       0.8  0.433962  0.606667    100    54   140   \n",
       "598  0.756826    0.741007  0.920354  0.490991  0.748571    100   139   339   \n",
       "599  0.611229     0.62963  0.771429  0.367925  0.603333    100    54   140   \n",
       "\n",
       "    sup_2 sup_all trtest  \n",
       "0     235     700     TR  \n",
       "1      93     300     TE  \n",
       "2     235     700     TR  \n",
       "3      93     300     TE  \n",
       "4     235     700     TR  \n",
       "5      93     300     TE  \n",
       "6     233     700     TR  \n",
       "7      95     300     TE  \n",
       "8     233     700     TR  \n",
       "9      95     300     TE  \n",
       "10    233     700     TR  \n",
       "11     95     300     TE  \n",
       "12    230     700     TR  \n",
       "13     98     300     TE  \n",
       "14    230     700     TR  \n",
       "15     98     300     TE  \n",
       "16    230     700     TR  \n",
       "17     98     300     TE  \n",
       "18    244     700     TR  \n",
       "19     84     300     TE  \n",
       "20    244     700     TR  \n",
       "21     84     300     TE  \n",
       "22    244     700     TR  \n",
       "23     84     300     TE  \n",
       "24    231     700     TR  \n",
       "25     97     300     TE  \n",
       "26    231     700     TR  \n",
       "27     97     300     TE  \n",
       "28    231     700     TR  \n",
       "29     97     300     TE  \n",
       "..    ...     ...    ...  \n",
       "570   231     700     TR  \n",
       "571    97     300     TE  \n",
       "572   231     700     TR  \n",
       "573    97     300     TE  \n",
       "574   231     700     TR  \n",
       "575    97     300     TE  \n",
       "576   218     700     TR  \n",
       "577   110     300     TE  \n",
       "578   218     700     TR  \n",
       "579   110     300     TE  \n",
       "580   218     700     TR  \n",
       "581   110     300     TE  \n",
       "582   222     700     TR  \n",
       "583   106     300     TE  \n",
       "584   222     700     TR  \n",
       "585   106     300     TE  \n",
       "586   222     700     TR  \n",
       "587   106     300     TE  \n",
       "588   235     700     TR  \n",
       "589    93     300     TE  \n",
       "590   235     700     TR  \n",
       "591    93     300     TE  \n",
       "592   235     700     TR  \n",
       "593    93     300     TE  \n",
       "594   222     700     TR  \n",
       "595   106     300     TE  \n",
       "596   222     700     TR  \n",
       "597   106     300     TE  \n",
       "598   222     700     TR  \n",
       "599   106     300     TE  \n",
       "\n",
       "[600 rows x 29 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#display(res_df)\n",
    "from IPython.display import display\n",
    "\n",
    "pd.options.display.max_columns = None\n",
    "display(res_df)\n",
    "res_df.to_csv('res_df.csv',  float_format='%.3f', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# STOP HERE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'NN_model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-1950ae2e07fb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mplot_model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mplot_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mNN_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mto_file\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'NN_model.png'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'NN_model' is not defined"
     ]
    }
   ],
   "source": [
    "from keras.utils import plot_model\n",
    "plot_model(NN_model, to_file='NN_model.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "SVG(model_to_dot(NN_model).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots(Y_test, pred_nn_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots(Y_test, pred_lm_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots(Y_test, pred_gv_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plots(y_act, y_pred, title='Confusion matrix', cmap=plt.cm.gray_r): \n",
    "    print(pd.crosstab(y_act, y_pred.reshape(len(y_pred),), rownames=['Actual'], colnames=['Predicted'], margins=True))\n",
    "    df_confusion = pd.crosstab(y_act, y_pred.reshape(y_pred.shape[0],), rownames=['Actual'], colnames=['Predicted'], margins=True)\n",
    "    df_conf_norm = df_confusion / df_confusion.sum(axis=1)\n",
    "    plt.matshow(df_confusion, cmap=cmap) \n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(df_confusion.columns))\n",
    "    plt.xticks(tick_marks, df_confusion.columns) \n",
    "    plt.yticks(tick_marks, df_confusion.index)\n",
    "    plt.ylabel(df_confusion.index.name)\n",
    "    plt.xlabel(df_confusion.columns.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_results(Y_test, pred_lm_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_results(Y_test, pred_nn_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_results(Y_test, pred_gv_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix#, accuracy_score\n",
    "# Inputs are 0/1/2, not the one-hot coded versions\n",
    "def show_results (y_true, y_pred):\n",
    "\n",
    "    correct = np.sum(y_true == y_pred)\n",
    "    total = len(y_true)\n",
    "    error = 100.0 - (100 * float(correct) / float(total))\n",
    "    print (\"Error rate: %.1f%%\" % (error))\n",
    "\n",
    "    print classification_report(y_true, y_pred)\n",
    "    print confusion_matrix(y_true, y_pred)\n",
    "    \n",
    "    confusions = np.zeros([3, 3], np.float32)\n",
    "    bundled = zip(y_true, y_pred)\n",
    "    for actual, predicted in bundled:\n",
    "        confusions[actual, predicted] += 1\n",
    "\n",
    "    plt.xlabel('Predicted')\n",
    "    plt.ylabel('Actual')\n",
    "    plt.grid(False)\n",
    "    plt.xticks(np.arange(3))\n",
    "    plt.yticks(np.arange(3))\n",
    "    plt.imshow(confusions, cmap=plt.cm.RdGy, interpolation='nearest')\n",
    "\n",
    "    for i, cas in enumerate(confusions):\n",
    "        for j, count in enumerate(cas):\n",
    "            if count >= 0:\n",
    "                xoff = .07 * len(str(count))\n",
    "                plt.text(j-xoff, i+.2, int(count), fontsize=9, color='blue')\n",
    "    plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This code allows you to see the mislabelled examples\n",
    "C = 5\n",
    "y_test_oh = np.eye(C)[Y_test.reshape(-1)]\n",
    "X_test_indices = sentences_to_indices(X_test, word_to_index, maxLen)\n",
    "pred = NN_model.predict(X_test_indices)\n",
    "for i in range(len(X_test)):\n",
    "    x = X_test_indices\n",
    "    num = np.argmax(pred[i])\n",
    "    if(num != Y_test[i]):\n",
    "        print('Expected label:', Y_test[i], ' prediction: ', num, X_test[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Consolidate predictions\n",
    "# First concatenate training and test sets, then merge the predictions with \n",
    "# the input data\n",
    "pred_tr = pd.DataFrame(np.column_stack([X_train, Y_train, pred_lm_tr, pred_gv_tr,pred_nn_tr, tot_lm_tr, perc_lm_tr]), \n",
    "                               columns=['sentence', 'actual', 'pred_lm', 'pred_gv', 'pred_nn', 'tot_lm', 'perc_lm'])\n",
    "pred_tr['trte'] = 'TR'\n",
    "#display(pred_tr)\n",
    "pred_te = pd.DataFrame(np.column_stack([X_test, Y_test, pred_lm_te, pred_gv_te,pred_nn_te, tot_lm_te, perc_lm_te]), \n",
    "                               columns=['sentence', 'actual', 'pred_lm', 'pred_gv', 'pred_nn', 'tot_lm', 'perc_lm'])\n",
    "pred_te['trte'] = 'TE'\n",
    "#display(pred_te)\n",
    "predictions = pd.concat([pred_tr, pred_te])\n",
    "predictions.index = range(len(predictions.index))\n",
    "display(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_cols = pd.DataFrame(np.column_stack([full_s, y, trfile, sent_cnt, sent_ind, word_cnt]), \n",
    "                               columns=['full_s', 'y', 'trfile', 'sent_cnt', 'sent_ind', 'word_cnt'])\n",
    "other_cols.info()\n",
    "other_cols.reset_index(drop=True)\n",
    "display(other_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df = pd.concat([other_cols, predictions], axis=1)\n",
    "final_df.info()\n",
    "display(final_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.to_csv('final_df.csv',  float_format='%.3f', index=False)"
   ]
  }
 ],
 "metadata": {
  "coursera": {
   "course_slug": "nlp-sequence-models",
   "graded_item_id": "RNnEs",
   "launcher_item_id": "acNYU"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}