receiptID_2_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>jQuery(function() {if (jQuery(\"body.notebook_app\").length == 0) { jQuery(\".input_area\").toggle(); jQuery(\".prompt\").toggle();}});</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<button onclick=\"jQuery('.input_area').toggle(); jQuery('.prompt').toggle();\">Toggle code</button>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display\n",
    "from IPython.display import HTML\n",
    "import IPython.core.display as di # Example: di.display_html('<h3>%s:</h3>' % str, raw=True)\n",
    "\n",
    "# This line will hide code by default when the notebook is exported as HTML\n",
    "di.display_html('<script>jQuery(function() {if (jQuery(\"body.notebook_app\").length == 0) { jQuery(\".input_area\").toggle(); jQuery(\".prompt\").toggle();}});</script>', raw=True)\n",
    "\n",
    "# This line will add a button to toggle visibility of code blocks, for use with the HTML export version\n",
    "di.display_html('''<button onclick=\"jQuery('.input_area').toggle(); jQuery('.prompt').toggle();\">Toggle code</button>''', raw=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Receipt.ID\n",
    "### Hierarchical text item classification\n",
    "Taxonomic classification, categorize items according to a pre-defined taxonomy. The goal is to assign one or more categories in the taxonomy to an item. It is a multi-class **and** multi-label classification problem with hierarchical relationships between each node in the tree.\n",
    "\n",
    "#### Items\n",
    "- Items come from a wide range for categories like Produce, Meat, Beverage, Supplies. \n",
    "- Example item to category mapping:\n",
    "\n",
    "\n",
    "|item|mapping|\n",
    "|---|---|\n",
    "|Kale  | \"Food/Produce/Kale\"  |\n",
    "|Vinegar white wine 50 grain  | \"Food/Dry-Grocery/Vinegars/White Wine Vinegar\"  |\n",
    "|Imported nat flank steak  | \"Food/Meats/Beef/Flank Steak\"  |\n",
    "\n",
    "To solve this problem, I will undertake the following course of action:\n",
    "1. Explore the dataset\n",
    "    - Explore the dataset to ensure its integrity and understand the context. \n",
    "2. Identify features that may be used. \n",
    "    - If possible, engineer features that might provide greater discrimination.\n",
    "3. Build k independent *text-based* classifiers for the text-based features and feed the output from these classifiers into the next layer classifier which takes in the other features. Explore a couple of classifiers that might be well suited for the problem at hand.\n",
    "    - Decision Trees\n",
    "    - SVM\n",
    "    - AdaBoost\n",
    "    - Random Forest\n",
    "4.  Select appropriate classifier based on evaluation metric and tune it for optimality.\n",
    "\n",
    "In this notebook I do processes 3 and 4.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "# Ignore warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import sys\n",
    "sys.path.append('tools/')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "# Graphing Libraries\n",
    "import matplotlib.pyplot as pyplt\n",
    "import seaborn as sns\n",
    "sns.set_style(\"white\")  \n",
    "\n",
    "# Use CPickle if available\n",
    "try:\n",
    "   import cPickle as pickle\n",
    "except:\n",
    "   import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read in a data file with 127108 datapoints\n"
     ]
    }
   ],
   "source": [
    "dataPath = '/Users/omojumiller/mycode/insight/PlateIQ/'\n",
    "df = pd.read_pickle(dataPath+'data/df_data_vectors.dat')\n",
    "df_catergory_lookup = pd.read_pickle(dataPath+'data/data_category_lookup.dat')\n",
    "print ('Read in a data file with {0} datapoints'.format(len(df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Define helper functions\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def shuffle_split_data(X, y):\n",
    "    \"\"\" Shuffles and splits data into 75% training and 25% testing subsets,\n",
    "        then returns the training and testing subsets. \"\"\"\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=num_train, random_state=42)\n",
    "    \n",
    "    # Return the training and testing data subsets\n",
    "    return X_train, y_train, X_test, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MeanEmbeddingVectorizer(object):\n",
    "    \"\"\"Transforms lists of feature-value mappings to vectors.\n",
    "    \n",
    "    Attributes\n",
    "    ----------\n",
    "    object : word2vec model \n",
    "    \"\"\"\n",
    "    def __init__(self, word2vec):\n",
    "        self.word2vec = word2vec\n",
    "        self.dim = len(word2vec.itervalues().next())\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        return self \n",
    "\n",
    "    def transform(self, X):\n",
    "        return np.array([\n",
    "            np.mean([self.word2vec[w] for w in words if w in self.word2vec] \n",
    "                    or [np.zeros(self.dim)], axis=0)\n",
    "            for words in X\n",
    "        ])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_categories(level, num_data = 1000):\n",
    "    \"\"\"Gets categories which at least num_data datapoints.\n",
    "    \n",
    "    Attributes\n",
    "    ----------\n",
    "    level : column name\n",
    "    num_data : int\n",
    "    \"\"\"    \n",
    "    the_categories = {}\n",
    "    counts = df.groupby(level).size()\n",
    "    total = df.groupby(level).size().sum()\n",
    "    \n",
    "    for i in counts.index:\n",
    "        if counts[i] > num_data:\n",
    "            the_categories[int(i)] = counts[i]\n",
    "\n",
    "\n",
    "    print(level, 'has', len(the_categories), 'categories with enough data.')  \n",
    "\n",
    "    for key in the_categories:\n",
    "        find_name = df_catergory_lookup.category_id == key\n",
    "        name = df_catergory_lookup[find_name].category_name\n",
    "        the_index = name.index[0]\n",
    "\n",
    "    return the_categories, total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_category_name(key):\n",
    "    \"\"\"\n",
    "    Get the name of a category\n",
    "    Attributes\n",
    "    ----------\n",
    "    key : int\n",
    "    \"\"\"\n",
    "    \n",
    "    if key in df_catergory_lookup.category_id.unique():\n",
    "        find_name = df_catergory_lookup.category_id == key\n",
    "        name = df_catergory_lookup[find_name].category_name\n",
    "        the_index = name.index[0]\n",
    "        return name.ix[the_index]\n",
    "    else:\n",
    "        print(str(key)+\" not in the dataframe\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Current approach\n",
    "The current approach used focuses exclusively on the item's name, for example, a data point in the dataset would have an object's name as \"mary's organic fryers\" or \"organic baby spinach.\" The first challenge with this approach lay in the fact that the item label was quite short, roughly about two to eight words.  Further, when modifiers like *gluten free*, *organic*, or *pesticide free* wherein the item's label, this added a layer of misinformation causing items like *organic milk* and *organic beer* to be classified in the same class.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## My Approach\n",
    "I took an entirely different approach. I got inspiration from the approach that Google, [YouTube](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/36411.pdf) used in organizing videos and decided to shift the unit of analysis from item *name* to the *categories* themselves. My approach combines information from both the text-based labels as well as the item's metadata.\n",
    "\n",
    "This method achieves two crucial things. First, by focusing on individual categories, each time a new category of item is added to the restaurant domain, instead of having to retrain the classifier on the entire dataset, all we have to do is gather enough data for that category, and train a classifier for it. This way, the approach can scale beautifully as the taxonomy grows. Second, moving the unit of analysis from text labels to categories, it becomes easier to correctly separate \"organic cream\" and \"organic beer.\" \n",
    "\n",
    "I chose to discard categories that had less than 300 datapoints. As the datapoints in the category grows, those categories can then been trained individually."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "level_0 has 6 categories with enough data.\n"
     ]
    }
   ],
   "source": [
    "the_level = {}\n",
    "num_data = 300\n",
    "\n",
    "level = 'level_0'\n",
    "the_level[level], total = get_categories(level, num_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "level_1 has 19 categories with enough data.\n"
     ]
    }
   ],
   "source": [
    "level = 'level_1'\n",
    "the_level[level], total = get_categories(level, num_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "level_2 has 65 categories with enough data.\n"
     ]
    }
   ],
   "source": [
    "level = 'level_2'\n",
    "df.loc[:,('label')] = df.loc[:, level]\n",
    "the_level[level], total = get_categories(level, num_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "level_3 has 42 categories with enough data.\n"
     ]
    }
   ],
   "source": [
    "level = 'level_3'\n",
    "df.loc[:,('label')] = df.loc[:, level]\n",
    "the_level[level], total = get_categories(level, num_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "level_4 has 9 categories with enough data.\n"
     ]
    }
   ],
   "source": [
    "level = 'level_4'\n",
    "df.loc[:,('label')] = df.loc[:, level]\n",
    "the_level[level], total = get_categories(level, num_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed 141 categories in all\n"
     ]
    }
   ],
   "source": [
    "k_categories = 0\n",
    "for key in the_level.keys():\n",
    "    k_categories += len(the_level[key].keys())\n",
    "   \n",
    "    \n",
    "print('processed {} categories in all'.format(k_categories))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Labeling\n",
    "For each data point, I assigned the deepest class node in the tree to which it belonged as its label. For example, an item named \"bacon ends\" belonged to both classes [Food], [Food, Meats] and [Food, Meats, Pork]. For such an item, I assigned it is label as \"Pork.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "null_examples = df['label'].isnull()\n",
    "df.ix[df[null_examples].index, 'label'] = df.loc[:, 'level_4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "null_examples = df['label'].isnull()\n",
    "df.ix[df[null_examples].index, 'label'] = df.loc[:, 'level_3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "null_examples = df['label'].isnull()\n",
    "df.ix[df[null_examples].index, 'label'] = df.loc[:, 'level_2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "null_examples = df['label'].isnull()\n",
    "df.ix[df[null_examples].index, 'label'] = df.loc[:, 'level_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "null_examples_ = df['label'].isnull()\n",
    "df.ix[df[null_examples_].index, 'label'] = df.loc[:, 'level_0']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve the *k* training sets for the *k* categories for each level in the tree   \n",
    "The aim of moving to a category-based solution is to embed knowledge of the taxonomy into classifiers. To do this, I had to figure out how to get positive and negative samples for each category. For every category node, I decided that itself, as well as all its descendants, were *positive* samples for that class. All other nodes that were neither the categories ancestor(s) or itself were set as *negative* samples.  The figure below gives a visual explanation of selecting category training set. I did this for each category node in the tree that had enough data.\n",
    "<img src=\"images/hc_5.png\" alt=\"Drawing\" style=\"width: 450px;\"/>\n",
    "\n",
    "One of the limitations of this method is that most nodes have un-balanced classes. Some more severe than other, especially as you go further down in the tree. There are some classes whose ratio of positive signals is as small as 0.02%. The more granular that subclasses get, the harder it is to classify them. The good news is that one can decide to focus on a few of these classes and use synthetic methods to rebalance the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a dict of positive and negative samples for the categories\n",
    "category_samples = {}\n",
    "category_positive_samples = {}\n",
    "category_negative_samples= {}\n",
    "category_test_samples = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing 1: Food\n",
      "processing 2: Supplies\n",
      "processing 3: Beverages\n",
      "processing 4: Other\n",
      "processing 495: Grocery\n",
      "processing 499: Protein\n",
      "Finished Processing\n"
     ]
    }
   ],
   "source": [
    "level = 'level_0'\n",
    "for key in the_level[level].keys():\n",
    "    print (\"processing {0}: {1}\".format(key, get_category_name(key)))\n",
    "    # create POSITIVE training set\n",
    "    \n",
    "    key_df = df[level] == key\n",
    "    null_examples = df['level_1'].isnull()\n",
    "    positive_examples = df['level_1'].notnull()\n",
    "    \n",
    "\n",
    "    # Create training set \n",
    "    try:\n",
    "        category_samples[key] = df[key_df & null_examples]\n",
    "        category_samples[key] = category_samples[key].append(df[key_df & positive_examples])\n",
    "        num_test = (len(category_samples[key]) // 10) * 2\n",
    "        category_positive_samples[key] = category_samples[key][num_test:]\n",
    "        category_test_samples[key] = category_samples[key][:num_test]        \n",
    "\n",
    "    except:\n",
    "        print(\"This category didn't generate any feature set {0}\".format(key))\n",
    "        continue\n",
    "        \n",
    "    # create NEGATIVE training set\n",
    "    level_0_item = df['level_0'] != key\n",
    "    \n",
    "    # Create training set \n",
    "    try:\n",
    "        category_negative_samples[key] = df[level_0_item]\n",
    "        \n",
    "    except:\n",
    "        print(\"This category didn't generate any label set {0}\".format(key))\n",
    "        continue\n",
    "        \n",
    "    \n",
    "    \n",
    "print(\"Finished Processing\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing 1281: SF Checkout Bag Fee\n",
      "processing 130: Cream\n",
      "processing 375: Sausages\n",
      "processing 132: Tomatoes\n",
      "processing 134: Peppers\n",
      "processing 391: Salt\n",
      "processing 392: Disposables & Packaging Supplies\n",
      "processing 267: Glasses\n",
      "processing 258: Keg Deposit\n",
      "processing 143: Tortillas\n",
      "processing 16: Beers\n",
      "processing 145: Cucumbers\n",
      "processing 18: Wines\n",
      "processing 531: Condiments\n",
      "processing 535: Fruits\n",
      "processing 152: Flour & Starch\n",
      "processing 537: Herbs\n",
      "processing 26: Juices\n",
      "processing 157: Garlic\n",
      "processing 1030: Seeds\n",
      "processing 40: Teas\n",
      "processing 44: Beans\n",
      "processing 482: Linens\n",
      "processing 177: Onions\n",
      "processing 51: Fuel & Freight/Delivery\n",
      "processing 1077: Breads\n",
      "processing 54: Liquor\n",
      "processing 55: Oils\n",
      "processing 56: Pork\n",
      "processing 1730: Leaves\n",
      "processing 325: Eggs\n",
      "processing 70: Beef\n",
      "processing 456: Sodas\n",
      "processing 329: Poultry\n",
      "processing 74: Fish\n",
      "processing 331: Coffee\n",
      "processing 76: Spices\n",
      "processing 226: Vinegars\n",
      "processing 78: Cheese\n",
      "processing 464: Canned\n",
      "processing 1272: Squash\n",
      "processing 84: Lettuce\n",
      "processing 1365: Spreads & Pastes\n",
      "processing 86: Asparagus\n",
      "processing 87: Syrup\n",
      "processing 216: Radishes\n",
      "processing 89: Kale\n",
      "processing 986: Cleaning & Janitorial\n",
      "processing 1039: Nuts & Grains\n",
      "processing 98: Beets\n",
      "processing 228: Sugar\n",
      "processing 1041: Medical Supplies\n",
      "processing 105: Lamb\n",
      "processing 235: Chocolate\n",
      "processing 109: Broccoli\n",
      "processing 111: Mushrooms\n",
      "processing 1960: Glass Cleaner\n",
      "processing 115: Butter \n",
      "processing 118: Potatoes\n",
      "processing 119: Cabbages\n",
      "processing 332: Pasta\n",
      "processing 1078: Shellfish\n",
      "processing 124: Milk\n",
      "processing 125: Carrots\n",
      "processing 126: Cauliflower\n",
      "Finished Processing\n"
     ]
    }
   ],
   "source": [
    "level = 'level_2'\n",
    "for key in the_level[level].keys():\n",
    "    print (\"processing {0}: {1}\".format(key, get_category_name(key)))\n",
    "    # create POSITIVE training set\n",
    "    \n",
    "    key_df = df[level] == key\n",
    "    null_examples = df['level_3'].isnull()\n",
    "    positive_examples = df['level_3'].notnull()\n",
    "    \n",
    "\n",
    "    # Create training set \n",
    "    try:\n",
    "        category_samples[key] = df[key_df & null_examples]\n",
    "        category_samples[key] = category_samples[key].append(df[key_df & positive_examples])\n",
    "        num_test = (len(category_samples[key]) // 10) * 2\n",
    "        category_positive_samples[key] = category_samples[key][num_test:]\n",
    "        category_test_samples[key] = category_samples[key][:num_test]    \n",
    "    \n",
    "    except:\n",
    "        print(\"This category didn't generate any feature set {0}\".format(key))\n",
    "        continue\n",
    "        \n",
    "    # create NEGATIVE training set\n",
    "    find_level_0 = df[level] == key\n",
    "    name_level_0 = df[find_level_0].level_0\n",
    "    the_index_level_0 = name_level_0.index[0]\n",
    "    \n",
    "    find_level_1 = df[level] == key\n",
    "    name_level_1 = df[find_level_1].level_1\n",
    "    the_index_level_1 = name_level_1.index[0]\n",
    "    \n",
    "    level_0_item = df['level_0'] != name_level_0.ix[the_index_level_0]\n",
    "    level_1_item = df['level_1'] != name_level_1.ix[the_index_level_1]\n",
    "    level_2_item = df['level_2'] != key\n",
    "    \n",
    "    # Create training set \n",
    "    try:\n",
    "        category_negative_samples[key] = df[level_0_item & level_1_item & level_2_item]\n",
    "        \n",
    "    except:\n",
    "        print(\"This category didn't generate any label set {0}\".format(key))\n",
    "        continue    \n",
    "    \n",
    "print(\"Finished Processing\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing 387: Oysters\n",
      "processing 136: Cilantro\n",
      "processing 1033: Ground Black Pepper\n",
      "processing 1802: Baby Carrot\n",
      "processing 1239: Lids\n",
      "processing 275: Containers\n",
      "processing 148: Olives\n",
      "processing 25: Vodka\n",
      "processing 282: Uniforms\n",
      "processing 286: Bags\n",
      "processing 32: Gin\n",
      "processing 289: Cups\n",
      "processing 164: Mints\n",
      "processing 39: Tequila\n",
      "processing 176: Oranges\n",
      "processing 1160: Red Wines\n",
      "processing 50: Iced Tea\n",
      "processing 52: Whiskey\n",
      "processing 437: Bacon\n",
      "processing 1209: Gloves\n",
      "processing 319: Red Onions\n",
      "processing 963: Mozzarella\n",
      "processing 69: Chicken\n",
      "processing 970: Extra Virgin Olive Oil\n",
      "processing 1200: Shrimp\n",
      "processing 1358: Rice\n",
      "processing 80: Apples\n",
      "processing 355: Duck\n",
      "processing 980: Dried Fruits\n",
      "processing 983: Cheddar\n",
      "processing 88: Avocados\n",
      "processing 1113: Kegs\n",
      "processing 95: Basil\n",
      "processing 99: Bell Peppers\n",
      "processing 229: Sauces\n",
      "processing 998: White Wines\n",
      "processing 363: Rum\n",
      "processing 403: Purees\n",
      "processing 247: Zucchini Squash\n",
      "processing 1019: Salmon\n",
      "processing 1148: Liqueur\n",
      "processing 1021: Tuna\n",
      "Finished Processing\n"
     ]
    }
   ],
   "source": [
    "level = 'level_3'\n",
    "for key in the_level[level].keys():\n",
    "    print (\"processing {0}: {1}\".format(key, get_category_name(key)))\n",
    "    # create POSITIVE training set\n",
    "    \n",
    "    key_df = df[level] == key\n",
    "    null_examples = df['level_4'].isnull()\n",
    "    positive_examples = df['level_4'].notnull()\n",
    "    \n",
    "\n",
    "    # Create training set \n",
    "    try:\n",
    "        category_samples[key] = df[key_df & null_examples]\n",
    "        category_samples[key] = category_samples[key].append(df[key_df & positive_examples])\n",
    "        num_test = (len(category_samples[key]) // 10) * 2\n",
    "        category_positive_samples[key] = category_samples[key][num_test:]\n",
    "        category_test_samples[key] = category_samples[key][:num_test]        \n",
    "  \n",
    "    except:\n",
    "        print(\"This category didn't generate any feature set {0}\".format(key))\n",
    "        continue\n",
    "        \n",
    "    # create NEGATIVE training set\n",
    "    find_level_0 = df[level] == key\n",
    "    name_level_0 = df[find_level_0].level_0\n",
    "    the_index_level_0 = name_level_0.index[0]\n",
    "    \n",
    "    find_level_1 = df[level] == key\n",
    "    name_level_1 = df[find_level_1].level_1\n",
    "    the_index_level_1 = name_level_1.index[0]\n",
    "    \n",
    "    find_level_2 = df[level] == key\n",
    "    name_level_2 = df[find_level_2].level_1\n",
    "    the_index_level_2 = name_level_2.index[0]\n",
    "    \n",
    "    level_0_item = df['level_0'] != name_level_0.ix[the_index_level_0]\n",
    "    level_1_item = df['level_1'] != name_level_1.ix[the_index_level_1]\n",
    "    level_2_item = df['level_2'] != name_level_2.ix[the_index_level_2]\n",
    "    level_3_item = df['level_3'] != key\n",
    "    \n",
    "    # Create training set \n",
    "    try:\n",
    "        category_negative_samples[key] = df[level_0_item & level_1_item & level_2_item & level_3_item]\n",
    "        \n",
    "    except:\n",
    "        print(\"This category didn't generate any label set {0}\".format(key))\n",
    "        continue    \n",
    "    \n",
    "print(\"Finished Processing\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing 35: Bourbon\n",
      "processing 999: Cabernet Sauvignon\n",
      "processing 1064: Chardonnay\n",
      "processing 1002: Sauvignon Blanc\n",
      "processing 1164: Brut\n",
      "processing 49: Rye\n",
      "processing 21: Rose Wine\n",
      "processing 1015: Chicken Breast\n",
      "processing 989: Pinot Noir\n",
      "Finished Processing\n"
     ]
    }
   ],
   "source": [
    "level = 'level_4'\n",
    "for key in the_level[level].keys():\n",
    "    print (\"processing {0}: {1}\".format(key, get_category_name(key)))\n",
    "    # create POSITIVE training set\n",
    "    \n",
    "    key_df = df[level] == key\n",
    "    null_examples = df['level_5'].isnull()\n",
    "    positive_examples = df['level_5'].notnull()\n",
    "    \n",
    "\n",
    "    # Create training set \n",
    "    try:\n",
    "        category_samples[key] = df[key_df & null_examples]\n",
    "        category_samples[key] = category_samples[key].append(df[key_df & positive_examples])\n",
    "        num_test = (len(category_samples[key]) // 10) * 2\n",
    "        category_positive_samples[key] = category_samples[key][num_test:]\n",
    "        category_test_samples[key] = category_samples[key][:num_test]        \n",
    "\n",
    "    except:\n",
    "        print(\"This category didn't generate any feature set {0}\".format(key))\n",
    "        continue\n",
    "        \n",
    "    # create NEGATIVE training set\n",
    "    find_level_0 = df[level] == key\n",
    "    name_level_0 = df[find_level_0].level_0\n",
    "    the_index_level_0 = name_level_0.index[0]\n",
    "    \n",
    "    find_level_1 = df[level] == key\n",
    "    name_level_1 = df[find_level_1].level_1\n",
    "    the_index_level_1 = name_level_1.index[0]\n",
    "    \n",
    "    find_level_2 = df[level] == key\n",
    "    name_level_2 = df[find_level_2].level_1\n",
    "    the_index_level_2 = name_level_2.index[0]\n",
    "    \n",
    "    find_level_3 = df[level] == key\n",
    "    name_level_3 = df[find_level_3].level_1\n",
    "    the_index_level_3 = name_level_3.index[0]\n",
    "    \n",
    "    level_0_item = df['level_0'] != name_level_0.ix[the_index_level_0]\n",
    "    level_1_item = df['level_1'] != name_level_1.ix[the_index_level_1]\n",
    "    level_2_item = df['level_2'] != name_level_2.ix[the_index_level_2]\n",
    "    level_3_item = df['level_3'] != name_level_3.ix[the_index_level_3]\n",
    "    level_4_item = df['level_4'] != key\n",
    "    \n",
    "    # Create training set \n",
    "    try:\n",
    "        category_negative_samples[key] = df[level_0_item & level_1_item & level_2_item & level_3_item & level_4_item]\n",
    "        \n",
    "    except:\n",
    "        print(\"This category didn't generate any label set {0}\".format(key))\n",
    "        continue    \n",
    "    \n",
    "print(\"Finished Processing\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling\n",
    "\n",
    "With the training sets done, I vectorized them using a mean embedding vectorizer. \n",
    "\n",
    "### Classifier Selection\n",
    "Based on how the positive and negative samples of the categories were generated, i.e., encoding the category hierarchy into the learning set, I inferred that the classes for each category would be linearly seperable. So for classifier selection, I experimented with an SVM, and AdaBoost and Random Forest. The SVM took too long to run for $K$ = 5 fold cross-validation. While AdaBoost ran faster, the score was lower than that of the Random Forest. Further, I could parallelize the Random Forest which significantly improved the running time.\n",
    "\n",
    "For each category, I created an Random Forest binary classifier. I train each classifier using a $K=5$ fold cross-validation scheme. I fitted the resulting classifier and retrieved the predicted probability for each data point in the dataset, which resulted in *$K$* vectors for *$K$* categories. For each classifier, I calibrated its predicted probability by using an *isotonic* calibration."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Process item label names using Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from gensim.models.word2vec import Word2Vec\n",
    "\n",
    "# Only use item labels as input into word2vec embeddings\n",
    "# train word2vec on all the item_labels \n",
    "\n",
    "w2v_model = Word2Vec(df['item_labels'], size=750, window=5, min_count=5, workers=4)\n",
    "w2v = dict(zip(w2v_model.index2word, w2v_model.syn0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "w2v_model.init_sims(replace=True) #to trim unneeded model memory = use (much) less RAM."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train *k* classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from time import time, gmtime, strftime\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import precision_recall_fscore_support as score\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.externals import joblib\n",
    "\n",
    "seed = 342 # For reproducability\n",
    "Ada_w2v =  Pipeline([(\"word2vec vectorizer\", MeanEmbeddingVectorizer(w2v)), \n",
    "                     (\"Ada\", AdaBoostClassifier(n_estimators=10))])\n",
    "\n",
    "RF_w2v = Pipeline([(\"word2vec vectorizer\", MeanEmbeddingVectorizer(w2v)), \n",
    "                     (\"RF\", RandomForestClassifier(n_estimators=100, n_jobs=-1, random_state=seed))])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    scores = joblib.load(dataPath+'data/df_category_scores.pkl')\n",
    "except:\n",
    "    scores = {}\n",
    "try:\n",
    "    pred_proba = joblib.load(dataPath+'data/df_category_pred_proba.pkl')\n",
    "except:\n",
    "    pred_proba = {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 0.002s\n"
     ]
    }
   ],
   "source": [
    "t0 = time()\n",
    "\n",
    "\n",
    "    \n",
    "try:\n",
    "    if (pred_proba <> {}) and (scores <> {}) :\n",
    "        target = open(dataPath+'data/train_log_'+strftime(\"%d_%b_%Y_%H_%M_%S\", gmtime())+'.txt', 'w')\n",
    "        print(\"done in %0.3fs\" % (time() - t0))\n",
    "        target.close()\n",
    "    \n",
    "except:\n",
    "    target = open(dataPath+'data/train_log_'+strftime(\"%d_%b_%Y_%H_%M_%S\", gmtime())+'.txt', 'w')\n",
    "    for key in category_positive_samples:\n",
    "        if str(key) in scores:\n",
    "            continue\n",
    "        if key in scores:\n",
    "            continue\n",
    "        else:\n",
    "        \n",
    "            print (\"processing {0}: {1}\".format(key, get_category_name(key)))\n",
    "            target.write(\"processing {0}: {1}\\n\".format(key, name.ix[the_index]))\n",
    "            category_positive_samples[key].loc[:,('is_category')] = 1\n",
    "            category_negative_samples[key].loc[:,('is_category')] = 0\n",
    "\n",
    "            data = category_positive_samples[key].append(category_negative_samples[key], ignore_index=True)\n",
    "            X = data['item_labels']\n",
    "            y = data['is_category']\n",
    "            target.write('{:5.3f}% positive samples\\n'.format(len(category_positive_samples[key])/len(X)))\n",
    "            target.write(\"Number of datapoints in set {}\\n\".format(len(X)))\n",
    "\n",
    "            scores[key] = cross_val_score(RF_w2v,  X, y, cv=5, scoring='f1').mean()\n",
    "            RF_w2v.fit(X, y)\n",
    "            pred_proba[key] = RF_w2v.predict_proba(X)\n",
    "\n",
    "            # persist model\n",
    "            joblib.dump(RF_w2v, dataPath+'data/category_classifier/catg_'+str(key)+'.pkl')    \n",
    "            print('Finished training category {0}: {1}\\n'.format(key, name.ix[the_index]))\n",
    "            target.write('Finished training category {0}: {1}\\n\\n'.format(key, name.ix[the_index]))\n",
    "\n",
    "    target.close()    \n",
    "    joblib.dump(scores, dataPath+'data/df_category_scores.pkl')\n",
    "    joblib.dump(pred_proba,dataPath+'data/df_category_pred_proba.pkl')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get *k* vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for key in category_positive_samples:\n",
    "    if 'catg_'+str(key) in df:\n",
    "        continue\n",
    "    else:\n",
    "        df_ = category_positiv_samples[key].append(category_negative_samples[key])\n",
    "        scores_ = pd.DataFrame(pred_proba[key], columns = ['Neg','catg_'+str(key)])\n",
    "        scores_.drop(['Neg'], axis = 1, inplace = True)\n",
    "        scores_ = scores_.set_index(df_.index)\n",
    "        df.ix[df_.index, 'catg_'+str(key)] = scores_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Write vectors to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_pickle('data/df_data_vectors.dat')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Model Architecture\n",
    "<figure>\n",
    "  <!---img src=\"images/hc_4.png\" style=\"width: 450px;\"--->\n",
    "</figure>\n",
    "After training the *k=141* classifiers, I extracted the *k* vectors; I carefully combined them with the engineered features, and the other raw metadata taking care to ensure that I assigned the right probabilities to the right data points in my training set. I feed these features and labels into a multi-label, multi-class Random Forest classifier. \n",
    "\n",
    "I took 75% of the data for training and 25% for testing. Once again I do a 5 fold cross-validation scheme, fit the final classifier, and retrieve the predicted class.\n",
    "\n",
    "\n",
    "### Get Data for Final Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Get Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    data = joblib.load(dataPath+'data/data_test.pkl') \n",
    "except:\n",
    "    # Merge all test datasets\n",
    "    data = pd.DataFrame()\n",
    "    for key in category_test_samples:\n",
    "        data = data.append(category_test_samples[key], ignore_index=False)\n",
    "    X_test_0 = data['item_labels']\n",
    "\n",
    "    for keys in category_positive_samples:\n",
    "        catg = 'catg_'+str(key)\n",
    "        clf = joblib.load(dataPath+'data/category_classifier/'+catg+'.pkl')\n",
    "        answer = clf.predict_proba(X_test_0)\n",
    "        scores_ = pd.DataFrame(answer, columns = ['Neg',catg])\n",
    "        scores_.drop(['Neg'], axis = 1, inplace = True)\n",
    "        data.ix[:, catg] = scores_\n",
    "        joblib.dump(clf, dataPath+'data/category_classifier/'+catg+'.pkl')\n",
    "\n",
    "    joblib.dump(data, dataPath+'data/data_test.pkl') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Drop duplicates\n",
    "data = data.drop_duplicates(['item_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Make sure you are testing on datapoints whose classifiers have been trained\n",
    "label_mask_test = data[['label']].isin([key for key in scores.keys()]).all(axis=1)\n",
    "data = data.ix[label_mask_test, ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get Training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Merge all training datasets\n",
    "\n",
    "the_df = pd.DataFrame()\n",
    "for key in category_positive_samples:\n",
    "    the_df = the_df.append(category_positive_samples[key], ignore_index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "the_df = the_df.drop_duplicates(['item_id'], keep='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# drop rows that have the overlap between training and testing data from the training data\n",
    "\n",
    "test_item_id = [x for x in data.item_id]\n",
    "training_item_id = [x for x in the_df.item_id]\n",
    "overlap = set(training_item_id).intersection(test_item_id)\n",
    "mask = the_df[['item_id']].isin(list(overlap)).all(axis=1)\n",
    "overlap_index = the_df.ix[mask].index \n",
    "the_df.drop(the_df.ix[mask].index, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train on present labels\n",
    "For both training and testing, we can only train and test on datapoints whose category labels we have trained classifiers for."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "label_mask_train = the_df[['label']].isin([key for key in scores.keys()]).all(axis=1)\n",
    "the_df = the_df.ix[label_mask_train, ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set training and  testing set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Drop categories that we didn't train for in this implementation\n",
    "col_name = ['label', u'item_id', u'price_stddev', u'primary_unit', u'price_mean',\n",
    " u'vendor_id', u'branch_lenght', u'item_name_match'] + ['catg_'+str(key) for key in scores.keys()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = the_df[col_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = X[X['label'].notnull()]\n",
    "y = X['label']\n",
    "X = X.fillna(0)\n",
    "X.drop(['label'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "category_mask = data[['label']].isin(y.unique()).all(axis=1)\n",
    "X_test = data[category_mask]\n",
    "X_test = X_test[col_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_test = X_test['label']\n",
    "X_test = X_test.fillna(0)\n",
    "X_test.drop(['label'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>price_stddev</th>\n",
       "      <th>primary_unit</th>\n",
       "      <th>price_mean</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>branch_lenght</th>\n",
       "      <th>item_name_match</th>\n",
       "      <th>catg_1</th>\n",
       "      <th>catg_2</th>\n",
       "      <th>catg_3</th>\n",
       "      <th>...</th>\n",
       "      <th>catg_989</th>\n",
       "      <th>catg_482</th>\n",
       "      <th>catg_998</th>\n",
       "      <th>catg_999</th>\n",
       "      <th>catg_1002</th>\n",
       "      <th>catg_495</th>\n",
       "      <th>catg_499</th>\n",
       "      <th>catg_1015</th>\n",
       "      <th>catg_1019</th>\n",
       "      <th>catg_1021</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>55507</th>\n",
       "      <td>349603</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>8738</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55508</th>\n",
       "      <td>352946</td>\n",
       "      <td>5.630275</td>\n",
       "      <td>3</td>\n",
       "      <td>31.800000</td>\n",
       "      <td>7280</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55509</th>\n",
       "      <td>355295</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>13.950000</td>\n",
       "      <td>9724</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55510</th>\n",
       "      <td>360428</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>7.500000</td>\n",
       "      <td>1146</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55511</th>\n",
       "      <td>362518</td>\n",
       "      <td>0.638381</td>\n",
       "      <td>3</td>\n",
       "      <td>9.128294</td>\n",
       "      <td>22</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 129 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       item_id  price_stddev  primary_unit  price_mean  vendor_id  \\\n",
       "55507   349603      0.000000             3   52.000000       8738   \n",
       "55508   352946      5.630275             3   31.800000       7280   \n",
       "55509   355295      0.000000             3   13.950000       9724   \n",
       "55510   360428      0.000000             3    7.500000       1146   \n",
       "55511   362518      0.638381             3    9.128294         22   \n",
       "\n",
       "       branch_lenght  item_name_match  catg_1  catg_2  catg_3    ...      \\\n",
       "55507              4              1.0     0.9     0.0     0.0    ...       \n",
       "55508              4              1.0     1.0     0.0     0.0    ...       \n",
       "55509              4              0.0     1.0     0.0     0.0    ...       \n",
       "55510              4              1.0     1.0     0.0     0.0    ...       \n",
       "55511              4              0.0     1.0     0.0     0.0    ...       \n",
       "\n",
       "       catg_989  catg_482  catg_998  catg_999  catg_1002  catg_495  catg_499  \\\n",
       "55507       0.0       0.0       0.0       0.0        0.0       0.0       0.0   \n",
       "55508       0.0       0.0       0.0       0.0        0.0       0.0       0.0   \n",
       "55509       0.0       0.0       0.0       0.0        0.0       0.0       0.0   \n",
       "55510       0.0       0.0       0.0       0.0        0.0       0.0       0.1   \n",
       "55511       0.0       0.0       0.0       0.0        0.0       0.0       0.0   \n",
       "\n",
       "       catg_1015  catg_1019  catg_1021  \n",
       "55507        0.0        0.2        0.0  \n",
       "55508        0.0        0.2        0.0  \n",
       "55509        0.0        0.0        0.0  \n",
       "55510        0.2        0.0        0.0  \n",
       "55511        0.0        0.3        0.0  \n",
       "\n",
       "[5 rows x 129 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>price_stddev</th>\n",
       "      <th>primary_unit</th>\n",
       "      <th>price_mean</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>branch_lenght</th>\n",
       "      <th>item_name_match</th>\n",
       "      <th>catg_1</th>\n",
       "      <th>catg_2</th>\n",
       "      <th>catg_3</th>\n",
       "      <th>...</th>\n",
       "      <th>catg_989</th>\n",
       "      <th>catg_482</th>\n",
       "      <th>catg_998</th>\n",
       "      <th>catg_999</th>\n",
       "      <th>catg_1002</th>\n",
       "      <th>catg_495</th>\n",
       "      <th>catg_499</th>\n",
       "      <th>catg_1015</th>\n",
       "      <th>catg_1019</th>\n",
       "      <th>catg_1021</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>106620</th>\n",
       "      <td>553484</td>\n",
       "      <td>0.6</td>\n",
       "      <td>7</td>\n",
       "      <td>31.29</td>\n",
       "      <td>1475</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.206667</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.685000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106900</th>\n",
       "      <td>562929</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>54.55</td>\n",
       "      <td>1244</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.060000</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106967</th>\n",
       "      <td>565698</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>35.90</td>\n",
       "      <td>1745</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.030000</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.936667</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106971</th>\n",
       "      <td>565756</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7</td>\n",
       "      <td>140.95</td>\n",
       "      <td>1475</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.020000</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.903333</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107166</th>\n",
       "      <td>573430</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>198.50</td>\n",
       "      <td>821</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.090000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.834000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 129 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_id  price_stddev  primary_unit  price_mean  vendor_id  \\\n",
       "106620   553484           0.6             7       31.29       1475   \n",
       "106900   562929           0.0             4       54.55       1244   \n",
       "106967   565698           0.0             1       35.90       1745   \n",
       "106971   565756           0.0             7      140.95       1475   \n",
       "107166   573430           0.0             1      198.50        821   \n",
       "\n",
       "        branch_lenght  item_name_match    catg_1  catg_2    catg_3    ...      \\\n",
       "106620              3              0.0  0.206667    0.01  0.685000    ...       \n",
       "106900              3              0.0  0.060000    0.01  0.880000    ...       \n",
       "106967              3              0.0  0.030000    0.01  0.936667    ...       \n",
       "106971              3              0.0  0.020000    0.02  0.903333    ...       \n",
       "107166              3              0.0  0.090000    0.00  0.834000    ...       \n",
       "\n",
       "        catg_989  catg_482  catg_998  catg_999  catg_1002  catg_495  catg_499  \\\n",
       "106620       0.0       0.0       0.0       0.0        0.0       0.0       0.0   \n",
       "106900       0.0       0.0       0.0       0.0        0.0       0.0       0.0   \n",
       "106967       0.0       0.0       0.0       0.0        0.0       0.0       0.0   \n",
       "106971       0.0       0.0       0.0       0.0        0.0       0.0       0.0   \n",
       "107166       0.0       0.0       0.0       0.0        0.0       0.0       0.0   \n",
       "\n",
       "        catg_1015  catg_1019  catg_1021  \n",
       "106620        0.0        0.0        0.0  \n",
       "106900        0.0        0.0        0.0  \n",
       "106967        0.0        0.0        0.0  \n",
       "106971        0.0        0.0        0.0  \n",
       "107166        0.0        0.0        0.0  \n",
       "\n",
       "[5 rows x 129 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train final classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(17015, 129)\n",
      "(9575, 129)\n"
     ]
    }
   ],
   "source": [
    "models = {\n",
    "          'DecisionTree': DecisionTreeClassifier(random_state=seed),\n",
    "          'RandomForest': RandomForestClassifier(n_estimators=10, n_jobs=-1, random_state=seed),\n",
    "         }\n",
    "\n",
    "scaler = RobustScaler()\n",
    "X_transform = scaler.fit_transform(X)\n",
    "X_test_transform  = scaler.fit_transform(X_test)\n",
    "\n",
    "print(X_transform.shape)\n",
    "print(X_test_transform.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CLASSIFICATION RESULTS OF BASELINE CLASSIFIERS\n",
      "\n",
      "CLASSIFIER           MEAN SCORE %  STD DEV %    TIME   \n",
      "DecisionTree             98.95        0.29         3.28secs\n",
      "RandomForest             99.10        0.19         2.38secs\n"
     ]
    }
   ],
   "source": [
    "print('CLASSIFICATION RESULTS OF BASELINE CLASSIFIERS\\n')\n",
    "print('{:20}{:^15}{:^10}{:^10}'.format('CLASSIFIER', 'MEAN SCORE %', 'STD DEV %', 'TIME'))\n",
    "\n",
    "\n",
    "for clf_name, clf in models.iteritems():\n",
    "    t0 = time()\n",
    "    results = cross_val_score(clf, X_transform, y, cv=5)\n",
    "    t1 = time() - t0\n",
    "    print('{:20}{:^15.2f}{:^10.2f}{:>10.2f}secs'.format(clf_name, results.mean()*100, results.std()*100, t1))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:     46.366\n",
      "Recall:   38.332  \n",
      "F Score  33.905  \n",
      "Support:  63.000  \n",
      "Chance:  1.587   \n",
      "182.90secs\n"
     ]
    }
   ],
   "source": [
    "t0 = time()\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=1000, n_jobs=-1, random_state=seed)\n",
    "calibrated_clf = CalibratedClassifierCV(clf, method='sigmoid', cv=5)\n",
    "calibrated_clf.fit(X_transform, y)\n",
    "final_preds = calibrated_clf.predict(X_test_transform)\n",
    "precision, recall, fbeta_score, support = score(y_test, final_preds)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print (\"Precision: {:10.3f}\\nRecall: {:^10.3f}\\nF Score{:^10.3f}\\nSupport:{:^10.3f}\\nChance:{:^10.3f}\".format(precision.mean()*100, \n",
    "                                                 recall.mean()*100, fbeta_score.mean()*100, len(support), (1/len(support))*100))\n",
    "\n",
    "print(\"{:<5.2f}secs\".format(time() - t0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results\n",
    "|Tree Level|mapping|Precision|Recall|Chance|\n",
    "|---|---|---|---|---|\n",
    "|One |92% |94% |90% | 25%|\n",
    "|Two | 66%| 80%| 64%| 7.0%|\n",
    "|Three | 47%|60% |48% | 2.5%|\n",
    "|Four|39% |44% |50% |3.0% |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Let's take a look at the results\n",
    "df_output = pd.DataFrame()\n",
    "df_output['truth'] = y_test\n",
    "df_output['predicted'] = final_preds\n",
    "df_output['predicted_name'] = ''\n",
    "df_output['item_name'] = data['item_name']\n",
    "df_output['mapped_level_0'] = data['mapped_level_0']\n",
    "df_output['level_0'] = data['level_0']\n",
    "df_output['preds_level_0'] = NaN\n",
    "df_output['mapped_level_1'] = data['mapped_level_1']\n",
    "df_output['level_1'] = data['level_1']\n",
    "df_output['preds_level_1'] = NaN\n",
    "df_output['mapped_level_2'] = data['mapped_level_2']\n",
    "df_output['level_2'] = data['level_2']\n",
    "df_output['preds_level_2'] = NaN\n",
    "df_output['mapped_level_3'] = data['mapped_level_3']\n",
    "df_output['level_3'] = data['level_3']\n",
    "df_output['preds_level_3'] = NaN\n",
    "df_output['mapped_level_4'] = data['mapped_level_4']\n",
    "df_output['level_4'] = data['level_4']\n",
    "df_output['preds_level_4'] = NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in list(df_output.index):\n",
    "    # find the a record where label is the predicted value\n",
    "    the_query = df.ix[:, 'label'] == df_output.ix[i, 'predicted']\n",
    "    answer = df[the_query].head(1)[['item_name', 'level_0', 'level_1', 'level_2', 'level_3', 'level_4']]\n",
    "    df_output.ix[i, 'predicted_name'] = answer.item_name.values[0]\n",
    "    df_output.ix[i, 'preds_level_0'] = answer.level_0.values[0]\n",
    "    df_output.ix[i, 'preds_level_1'] = answer.level_1.values[0]\n",
    "    df_output.ix[i, 'preds_level_2'] = answer.level_2.values[0]\n",
    "    df_output.ix[i, 'preds_level_3'] = answer.level_3.values[0]\n",
    "    df_output.ix[i, 'preds_level_4'] = answer.level_4.values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAFoCAYAAABNMRFVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XtcVXW+//E3omwsdUAu1njomDgNosRGJHMc4WQ8rDHR\nJtSZPDpCGY9HUTYzWumYOXgZOtFYpsDkZdTocix4TLeZk79spouXEoGAAs8M2AVMLtsjo84Am2D9\n/vDhGrei4hbYe+nr+Xjw0PX9rLX4LJd782btdfExDMMQAACABfXxdAMAAADuIsgAAADLIsgAAADL\nIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADL8oog8/XXX+vee+9VTEyMJk2apM2b\nN5u1VatWKSIiQiNHjjT/fOmll8z6nj17lJSUJLvdrpSUFNXU1Lise+vWrYqPj1dsbKyWLl2q1tbW\nXtsuAADQszweZAzDUFpamoKDg/XGG2/o17/+tXJzc/XHP/5RknTw4EEtWrRIu3bt0u7du7Vr1y7N\nmDFDknT48GGlp6crOTlZBQUFCgwMVHp6urnuHTt2KCcnRytXrtS2bdtUWlqqrKwsj2wnAADofh4P\nMg6HQ5GRkVq+fLmuu+46xcfHa/z48SoqKpIkVVdXKzIyUkFBQeaXzWaTJL322muKiopSSkqKwsPD\nlZmZqUOHDqmwsFCSlJeXp3nz5ikhIUGjR49WRkaG8vPzOSoDAMBlwuNBJiQkRGvWrNFVV10lSSoq\nKlJhYaHGjRunEydOqL6+XsOGDet02dLSUsXFxZnT/v7+ioyMVElJiTo6OlReXq6xY8eadbvdrra2\nNh04cKBHtwkAAPQOjweZ002aNElz5sxRTEyMJk+erOrqavn4+Cg3N1cJCQmaPn26Xn/9dXP+hoYG\nhYaGuqwjODhY9fX1OnbsmFpbW13qvr6+CggIUF1dXa9tEwAA6Dl9Pd3A6datWyeHw6Hly5dr9erV\nGj16tPr06aPw8HDNnTtX+/bt07JlyzRgwAAlJiaqpaVFfn5+Luvw8/OT0+lUS0uLOd1ZvSvGjh17\nVhgCAAAX1tDQIJvNpv379/fo9/GqIDNq1ChJ0pIlS/TII4/oscce06RJkzRo0CBJ0g033KAvv/xS\nr7zyihITE2Wz2c4KJU6nU4MGDTIDTGf1/v37d6kfp9Op9vb2S90sAACuOO3t7V0+cHApPB5kjhw5\nopKSEiUmJppjI0aMUFtbm/7xj38oICDAZf7hw4frk08+kSQNGTJEjY2NLnWHw6GRI0cqMDBQNptN\nDodD119/vaST/6hNTU0KCQnpUm+n5nvvvffc3j4AAK5Et956a698H4+fI1NbW6uHHnpIDQ0N5lh5\nebkGDx6sF154QampqS7zV1ZWmsEkOjpaxcXFZq25uVkVFRWKiYmRj4+PoqKizKufJKmkpET9+vVT\nRERED28VAADoDR4PMlFRURo9erR+9atfqbq6Wh988IGefvpp3X///brllltUWFioLVu2qKamRi+/\n/LLefPNNzZ8/X5KUnJys4uJibdy4UVVVVVqyZInCwsLMK5lmz56tzZs3a+fOnSorK1NGRoZmzZpl\nXr4NAACszccwDMPTTTQ2NmrlypXau3ev+vfvrzlz5igtLU2S9Oc//1lr167VV199paFDh+oXv/iF\ny8dQH330kVavXq36+nqNGTNGK1as0NChQ836xo0btXXrVrW1tem2227TsmXLzjoB+FxOHRbjoyUA\nAC5Ob/0M9Yog460IMgAAuKe3foZ6/KMlAAAAd3n8qiUr+eyzz3X06FFz2qdPH42/eZx8fX092BUA\nAFcugsxFeHTFep2w3WBO/7PxgP6wMUxhYWEe7AoAgCsXQeYiXD0wQL4Bw81p32//7sFuAAAA58gA\nAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADL\nIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgA\nAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADLIsgAAADL8oog8/XX\nX+vee+9VTEyMJk2apM2bN5u12tpapaamKiYmRlOnTtXu3btdlt2zZ4+SkpJkt9uVkpKimpoal/rW\nrVsVHx+v2NhYLV26VK2trb2yTQAAoOd5PMgYhqG0tDQFBwfrjTfe0K9//Wvl5ubqj3/8oyTpgQce\nUGhoqAoKCjRt2jQ9+OCDqqurkyQdPnxY6enpSk5OVkFBgQIDA5Wenm6ue8eOHcrJydHKlSu1bds2\nlZaWKisryyPbCQAAup/Hg4zD4VBkZKSWL1+u6667TvHx8Ro/fryKior08ccfq7a2VitWrNDw4cOV\nlpYmu92u/Px8SdKrr76qqKgopaSkKDw8XJmZmTp06JAKCwslSXl5eZo3b54SEhI0evRoZWRkKD8/\nn6MyAABcJjweZEJCQrRmzRpdddVVkqSioiLt379fN910k0pLSzVq1CjZbDZz/tjYWH366aeSpLKy\nMsXFxZk1f39/RUZGqqSkRB0dHSovL9fYsWPNut1uV1tbmw4cONBLWwcAAHqSx4PM6SZNmqQ5c+bI\nbrdr8uTJamxsVGhoqMs8QUFBqq+vlyQ1NDScVQ8ODlZ9fb2OHTum1tZWl7qvr68CAgLMj6YAAIC1\neVWQWbdunX73u9/pwIED+s1vfqPm5mb5+fm5zOPn5yen0ylJamlpOWe9paXFnD7X8gAAwNq8KsiM\nGjVKCQkJWrx4sbZv395p6HA6nfL395ck2Wy2c9ZPBZjO6v379+/BrQAAAL3F40HmyJEj2rlzp8vY\niBEj1NbWppCQEDU2NrrUHA6HQkJCJElDhgw5Zz0wMFA2m00Oh8Ostbe3q6mpyVweAABYm8eDTG1t\nrR566CE1NDSYY+Xl5QoKClJsbKw+//xzl6MqRUVFstvtkqTo6GgVFxebtebmZlVUVCgmJkY+Pj6K\niopSUVGRWS8pKVG/fv0UERHRC1sGAAB6mseDTFRUlEaPHq1f/epXqq6u1gcffKCnn35a999/v+Li\n4nTttddq8eLFqqqq0oYNG1ReXq4ZM2ZIkpKTk1VcXKyNGzeqqqpKS5YsUVhYmHkl0+zZs7V582bt\n3LlTZWVlysjI0KxZs1yuggIAANbl8SDTp08f5eTk6KqrrtJPf/pTLVu2TD/72c80Z84c9enTR7m5\nuWpsbFRycrLeeustZWdn65prrpEkDR06VOvWrVNBQYFmzpyp48ePKzs721z3lClTlJaWpuXLl2v+\n/Pmy2+1atGiRpzYVAAB0Mx/DMAxPN+Gtbr31VknSe++9J0maee8StQTcbNZPfFOiF566V2FhYR7p\nDwAAb3Xmz9Ce4vEjMgAAAO4iyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAA\nAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsi\nyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAA\nAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMsiyAAAAMvyeJCpr6/XggULNG7cOCUkJOjJ\nJ5+U0+mUJK1atUoREREaOXKk+edLL71kLrtnzx4lJSXJbrcrJSVFNTU1LuveunWr4uPjFRsbq6VL\nl6q1tbVXtw0AAPQsjweZBQsWqLW1VS+//LLWrFmjv/zlL1q7dq0k6eDBg1q0aJF27dql3bt3a9eu\nXZoxY4Yk6fDhw0pPT1dycrIKCgoUGBio9PR0c707duxQTk6OVq5cqW3btqm0tFRZWVke2UYAANAz\nPBpkDh48qLKyMmVmZio8PFyxsbFasGCB3n77bUlSdXW1IiMjFRQUZH7ZbDZJ0muvvaaoqCilpKQo\nPDxcmZmZOnTokAoLCyVJeXl5mjdvnhISEjR69GhlZGQoPz+fozIAAFxGPBpkQkJCtGnTJg0ePNgc\nMwxDx48f14kTJ1RfX69hw4Z1umxpaani4uLMaX9/f0VGRqqkpEQdHR0qLy/X2LFjzbrdbldbW5sO\nHDjQY9sDAAB6l0eDzMCBAzVhwgRz2jAMvfjii/rBD36ggwcPysfHR7m5uUpISND06dP1+uuvm/M2\nNDQoNDTUZX3BwcGqr6/XsWPH1Nra6lL39fVVQECA6urqen7DAABAr+jr6QZO99RTT+nAgQPKz8/X\nZ599pj59+ig8PFxz587Vvn37tGzZMg0YMECJiYlqaWmRn5+fy/J+fn5yOp1qaWkxpzurAwCAy4PX\nBJmsrCzl5eXp2Wef1YgRIzRixAhNmjRJgwYNkiTdcMMN+vLLL/XKK68oMTFRNpvtrFDidDo1aNAg\nM8B0Vu/fv3/vbBAAAOhxHr9qSZJ5ZVFWVpYSExPN8VMh5pThw4eroaFBkjRkyBA1Nja61B0Oh0JC\nQhQYGCibzSaHw2HW2tvb1dTUpJCQkB7cEgAA0Js8HmTWr1+v7du365lnntGPfvQjc/y5555Tamqq\ny7yVlZW6/vrrJUnR0dEqLi42a83NzaqoqFBMTIx8fHwUFRWloqIis15SUqJ+/fopIiKih7cIAAD0\nFo8GmerqauXm5iotLU0xMTFyOBzm1y233KLCwkJt2bJFNTU1evnll/Xmm29q/vz5kqTk5GQVFxdr\n48aNqqqq0pIlSxQWFmZeyTR79mxt3rxZO3fuVFlZmTIyMjRr1izz8m0AAGB9Hj1H5r333lNHR4dy\nc3OVm5sr6eSVSz4+PqqsrNRzzz2ntWvXau3atRo6dKh++9vf6sYbb5QkDR06VOvWrdPq1auVk5Oj\nMWPGKDs721z3lClTdOjQIS1fvlxtbW267bbbtGjRIo9sJwAA6Bk+hmEYnm7CW916662STgYuSZp5\n7xK1BNxs1k98U6IXnrpXYWFhHukPAABvdebP0J7i8XNkAAAA3EWQAQAAlkWQAQAAlkWQAQAAlkWQ\nAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAA\nlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQ\nAQAAlkWQAQAAlkWQAQAAlkWQAQAAlkWQAQAAluVWkJk5c6b++7//W8ePH+/ufgAAALrMrSBz8803\n63e/+51++MMf6pe//KV27dolwzC6uzcAAIDzcivILFy4UH/5y1+Uk5MjX19fPfTQQ/qP//gPPfPM\nM/riiy+6u0cAAIBO9XV3QR8fH02YMEETJkxQc3Oz8vLylJOTow0bNmjMmDGaN2+eJk+e3J29AgAA\nuLikk30bGhq0adMm/eQnP9GaNWsUGRmpFStW6Oabb9bjjz+u1atXX3Ad9fX1WrBggcaNG6eEhAQ9\n+eSTcjqdkqTa2lqlpqYqJiZGU6dO1e7du12W3bNnj5KSkmS325WSkqKamhqX+tatWxUfH6/Y2Fgt\nXbpUra2tl7K5AADAy7gVZN544w3dc889uuWWW7Rt2zYlJCTonXfe0csvv6yZM2fqoYce0qOPPqr8\n/PwLrmvBggVqbW3Vyy+/rDVr1ugvf/mL1q5dK0l64IEHFBoaqoKCAk2bNk0PPvig6urqJEmHDx9W\nenq6kpOTVVBQoMDAQKWnp5vr3bFjh3JycrRy5Upt27ZNpaWlysrKcmdzAQCAl3IryCxdulRXX321\nsrOz9cEHH2jhwoUaNmyYyzzDhw/XnDlzzruegwcPqqysTJmZmQoPD1dsbKwWLFigt99+Wx9//LFq\na2u1YsUKDR8+XGlpabLb7WY4evXVVxUVFaWUlBSFh4crMzNThw4dUmFhoSQpLy9P8+bNU0JCgkaP\nHq2MjAzl5+dzVAYAgMuIW0Hmww8/1HPPPafo6Gj16XNyFWVlZWpvbzfnGTNmjBYuXHje9YSEhGjT\npk0aPHiwy/jx48dVWlqqUaNGyWazmeOxsbH69NNPze8XFxdn1vz9/RUZGamSkhJ1dHSovLxcY8eO\nNet2u11tbW06cOCAO5sMAAC8kFtB5sSJE7r99tu1ceNGcywtLU3Tp0/X4cOHu7yegQMHasKECea0\nYRh68cUXNX78eDU2Nio0NNRl/qCgINXX10s6eX7OmfXg4GDV19fr2LFjam1tdan7+voqICDA/GgK\nAABYn1tB5je/+Y3+/d//XampqebYn/70J1177bXKzMx0u5mnnnpKlZWV+sUvfqHm5mb5+fm51P38\n/MwTgVtaWs5Zb2lpMafPtTwAALA+t4LM/v37tXjxYoWEhJhjgwcP1qOPPqqPP/7YrUaysrKUl5en\np59+WiNGjJDNZjsrdDidTvn7+0vSeeunAkxn9f79+7vVHwAA8D5uBZm+ffvq2LFjZ403Nze7dYff\nU1cWZWVlKTExUZI0ZMgQNTY2uszncDjM8HS+emBgoGw2mxwOh1lrb29XU1OTS/gCAADW5laQiY+P\n16pVq/T111+bYzU1NcrMzNTEiRMval3r16/X9u3b9cwzz+hHP/qROR4dHa2KigqXoypFRUWy2+1m\nvbi42Kw1NzeroqJCMTEx8vHxUVRUlIqKisx6SUmJ+vXrp4iIiIveXgAA4J3cCjKPPfaYnE6nbrvt\nNo0bN07jxo3T5MmT1dbWpiVLlnR5PdXV1crNzVVaWppiYmLkcDjMr5tuuknXXnutFi9erKqqKm3Y\nsEHl5eWaMWOGJCk5OVnFxcXauHGjqqqqtGTJEoWFhZlXMs2ePVubN2/Wzp07VVZWpoyMDM2aNcvl\nKigAAGBtbj2iICgoSH/4wx+0Z88e/e1vf1Pfvn01YsQIjR8/Xj4+Pl1ez3vvvaeOjg7l5uYqNzdX\n0skrl3x8fFRZWans7GwtXbpUycnJuu6665Sdna1rrrlGkjR06FCtW7dOq1evVk5OjsaMGaPs7Gxz\n3VOmTNGhQ4e0fPlytbW16bbbbtOiRYvc2VwAAOClfAweW31Ot956q6STgUuSZt67RC0BN5v1E9+U\n6IWn7lVYWJhH+gMAwFud+TO0p7h1RKaxsVHPPvusiouL1dbWdtYJvj3dNAAAgORmkFm2bJk+++wz\n3XHHHRo4cGB39wQAANAlbgWZjz/+WJs2bXJ5BAAAAEBvc+uqpauuukpBQUHd3QsAAMBFcSvITJ8+\nXZs2bXJ5SCQAAEBvc+ujpaamJr399tt6//33FRYWdtYzjV544YVuaQ4AAOB83AoykjR16tTu7AMA\nAOCiuRVkLuUJ1wAAAN3FrXNkJKmhoUHr16/XwoULdeTIEb3zzjs6ePBgd/YGAABwXm4Fma+++kpJ\nSUn6wx/+oB07duif//yn/vSnPyk5OVmlpaXd3SMAAECn3AoyTz75pBITE7Vz507169dPkrRmzRpN\nmjRJTz/9dLc2CAAAcC5uBZni4mKlpqa6PCCyb9++euCBB1RRUdFtzQEAAJyPW0Gmo6NDHR0dZ43/\n4x//kK+v7yU3BQAA0BVuBZkf/vCHev75513CTFNTk7KysnTzzTefZ0kAAIDu41aQWbx4sT777DP9\n8Ic/VGtrq+6//37dcsstqq2t1WOPPdbdPQIAAHTKrfvIDBkyRK+//rrefvttVVZWqqOjQ3fffbem\nT5+uAQMGdHePAAAAnXL7zr79+/fXzJkzu7MXAACAi+JWkPnZz3523jrPWgIAAL3BrSAzdOhQl+lv\nv/1WX331lf76179q3rx53dIYAADAhXTrs5ays7NVV1d3SQ0BAAB0ldvPWurM9OnT9T//8z/duUoA\nAIBz6tYgU1JSwg3xAABAr+m2k31PnDih//3f/9Xs2bMvuSkAAICucCvIfPe733V5zpIk9evXT3Pm\nzNG0adO6pTEAAIALcSvIPPnkk93dBwAAwEVzK8gUFhZ2ed64uDh3vgUAAMAFuRVk5s6da360ZBiG\nOX7mmI+PjyorKy+1RwAAgE65FWR+97vfadWqVXrkkUd00003yc/PT+Xl5VqxYoV+/OMfa8qUKd3d\nJwAAwFncuvw6MzNTTzzxhG677TYFBgbq6quv1s0336wVK1bolVde0dChQ80vAACAnuJWkGloaOg0\npAwYMEBHjx695KYAAAC6wq0gY7fbtWbNGp04ccIca2pqUlZWlsaPH99tzQEAAJyPW+fIPP744/rZ\nz36m+Ph4DRs2TIZh6Msvv1RISAhPvgYAAL3GrSMy4eHh+tOf/qSFCxfKbrcrJiZGS5cu1RtvvKFr\nrrnG7WacTqeSkpJcLu9etWqVIiIiNHLkSPPPl156yazv2bNHSUlJstvtSklJUU1Njcs6t27dqvj4\neMXGxmrp0qVqbW11uz8AAOBd3DoiI0nf+c53NHPmTNXW1iosLEzSybv7usvpdOqXv/ylqqqqXMYP\nHjyoRYsW6cc//rE5NmDAAEnS4cOHlZ6erocfflgTJ07U+vXrlZ6erjfffFOStGPHDuXk5CgrK0tB\nQUFavHixsrKy9Pjjj7vdJwAA8B5uHZExDENPP/204uLiNHXqVNXV1emxxx7T0qVL1dbWdtHrq66u\n1qxZs1RbW9tpLTIyUkFBQeaXzWaTJL322muKiopSSkqKwsPDlZmZqUOHDplHdPLy8jRv3jwlJCRo\n9OjRysjIUH5+PkdlAAC4TLgVZPLy8vTGG29o+fLl8vPzkyQlJiZq586dWr9+/UWvb9++fRo/fry2\nb9/ucoO9EydOqL6+XsOGDet0udLSUpc7B/v7+ysyMlIlJSXq6OhQeXm5xo4da9btdrva2tp04MCB\ni+4RAAB4H7eCzPbt2/XEE0/orrvuMu/mO2XKFK1atUpvvfXWRa/v7rvv1mOPPWYeaTnl4MGD8vHx\nUW5urhISEjR9+nS9/vrrZr2hoUGhoaEuywQHB6u+vl7Hjh1Ta2urS93X11cBAQGqq6u76B4BAID3\ncescmdraWo0cOfKs8YiICDU2Nl5yU6ccPHhQffr0UXh4uObOnat9+/Zp2bJlGjBggBITE9XS0mIe\nETrFz89PTqdTLS0t5nRndQAAYH1uBZmhQ4eqvLxc//Zv/+Yy/uGHH5on/naHO++8U5MmTdKgQYMk\nSTfccIO+/PJLvfLKK0pMTJTNZjsrlDidTg0aNMgMMJ3V+/fv3209AgAAz3EryNx7773KyMhQY2Oj\nDMPQ3r17tX37duXl5Wnx4sXd2uCpEHPK8OHD9cknn0iShgwZctYRIIfDoZEjRyowMFA2m00Oh0PX\nX3+9JKm9vV1NTU0KCQnp1h4BAIBnuBVkkpOT9e233yo3N1ctLS164oknNHjwYP385z/X3Xff3W3N\nPffccyopKdGWLVvMscrKSjOYREdHq7i42Kw1NzeroqJCCxYskI+Pj6KiolRUVGSeEFxSUqJ+/fop\nIiKi23oEAACe41aQefvtt3X77bfrJz/5if7v//5PhmEoKCiou3vTLbfcog0bNmjLli1KTEzURx99\npDfffFN5eXmSTgaq3//+99q4caNuueUWrV+/XmFhYWZwmT17tpYvX64RI0YoNDRUGRkZmjVr1lkn\nFQMAAGty66qlFStWmB/pDB48uFtDzKmroCQpKipKzz33nF5//XUlJSXppZde0m9/+1vdeOONkk6e\nq7Nu3ToVFBRo5syZOn78uLKzs83lp0yZorS0NC1fvlzz58+X3W7XokWLuq1XAADgWW4dkRk2bJj+\n+te/asSIEd3djyorK12mJ02apEmTJp1z/okTJ+qdd945Z/2+++7Tfffd1239AQAA7+FWkImIiNCi\nRYu0adMmDRs27KyPajIzM7ulOQAAgPNxK8h88cUXio2NlaRuvW8MAADAxehykHnqqaf04IMP6qqr\nrjJPtgUAAPCkLp/su2XLFjU3N7uMpaWlqaGhodubAgAA6IouB5nTH+Z4SmFhIU+SBgAAHuPW5dcA\nAADegCADAAAs66KCzOk3qwMAAPC0i7r8etWqVS73jGlra1NWVpauvvpql/m4jwwAAOgNXQ4ycXFx\nZ90zJiYmRkePHtXRo0e7vTEAAIAL6XKQ4d4xAADA23CyLwAAsCyCDAAAsCyCDAAAsCyCDAAAsCyC\nDAAAsCyCDAAAsCyCDAAAsCyCDAAAsCyCDAAAsCyCDAAAsCyCDAAAsCyCDAAAsCyCDAAAsCyCDAAA\nsCyCDAAAsCyCDAAAsCyCDAAAsCyCDAAAsCyCDAAAsCyCDAAAsCyCDAAAsCyvCjJOp1NJSUkqLCw0\nx2pra5WamqqYmBhNnTpVu3fvdllmz549SkpKkt1uV0pKimpqalzqW7duVXx8vGJjY7V06VK1trb2\nyrYAAICe5zVBxul06pe//KWqqqpcxtPT0xUaGqqCggJNmzZNDz74oOrq6iRJhw8fVnp6upKTk1VQ\nUKDAwEClp6eby+7YsUM5OTlauXKltm3bptLSUmVlZfXqdgEAgJ7jFUGmurpas2bNUm1trcv43r17\nVVNToxUrVmj48OFKS0uT3W5Xfn6+JOnVV19VVFSUUlJSFB4erszMTB06dMg8opOXl6d58+YpISFB\no0ePVkZGhvLz8zkqAwDAZcIrgsy+ffs0fvx4bd++XYZhmONlZWUaNWqUbDabORYbG6tPP/3UrMfF\nxZk1f39/RUZGqqSkRB0dHSovL9fYsWPNut1uV1tbmw4cONALWwUAAHpaX083IEl33313p+ONjY0K\nDQ11GQsKClJ9fb0kqaGh4ax6cHCw6uvrdezYMbW2trrUfX19FRAQoLq6OkVHR3fzVgAAgN7mFUdk\nzqW5uVl+fn4uY35+fnI6nZKklpaWc9ZbWlrM6XMtDwAArM2rg4zNZjsrdDidTvn7+1+wfirAdFbv\n379/D3YNAAB6i1cHmSFDhqixsdFlzOFwKCQk5IL1wMBA2Ww2ORwOs9be3q6mpiZzeQAAYG1eHWSi\no6NVUVEHBXx2AAARyElEQVThclSlqKhIdrvdrBcXF5u15uZmVVRUKCYmRj4+PoqKilJRUZFZLykp\nUb9+/RQREdF7GwEAAHqMVweZm266Sddee60WL16sqqoqbdiwQeXl5ZoxY4YkKTk5WcXFxdq4caOq\nqqq0ZMkShYWFmVcyzZ49W5s3b9bOnTtVVlamjIwMzZo1y+UqKAAAYF1eF2R8fHzMv/fp00c5OTlq\nbGxUcnKy3nrrLWVnZ+uaa66RJA0dOlTr1q1TQUGBZs6cqePHjys7O9tcfsqUKUpLS9Py5cs1f/58\n2e12LVq0qNe3CQAA9AyvuPz6dJWVlS7TYWFhysvLO+f8EydO1DvvvHPO+n333af77ruv2/oDAADe\nw+uOyAAAAHQVQQYAAFgWQQYAAFgWQQYAAFgWQQYAAFgWQQYAAFgWQQYAAFgWQQYAAFgWQQYAAFgW\nQQYAAFiW1z2iwEo62ttVWlqquro6cyw6Olp+fn4e7AoAgCsHQeYSnGiq0+Pr/58GBl0nSTp+5Gtt\nXCnz6dsAAKBnEWQu0cCg6xRwzfc83QYAAFckzpEBAACWRZABAACWRZABAACWRZABAACWRZABAACW\nRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACWRZAB\nAACWRZABAACWRZABAACWRZABAACWRZABAACWRZABAACW5fVBZufOnYqIiNDIkSPNPx9++GFJUm1t\nrVJTUxUTE6OpU6dq9+7dLsvu2bNHSUlJstvtSklJUU1NjSc2AQAA9BCvDzJVVVWaNGmSdu/erd27\nd2vXrl1avXq1JOmBBx5QaGioCgoKNG3aND344IOqq6uTJB0+fFjp6elKTk5WQUGBAgMDlZ6e7slN\nAQAA3ayvpxu4kOrqan3ve9/T4MGDXcb37t2r2tpavfbaa7LZbEpLS9PevXuVn5+vBx98UK+++qqi\noqKUkpIiScrMzNSECRNUWFiouLg4D2wJgEvldDpVWlp61nh0dLT8/Pw80BEAT7NEkJkwYcJZ42Vl\nZRo1apRsNps5Fhsbq08//dSsnx5Y/P39FRkZqZKSEoIMYFGlpaW6b1meBgZdZ44dP/K1Nq4Ur2vg\nCuX1QeaLL77QRx99pNzcXHV0dOj222/XggUL1NjYqNDQUJd5g4KCVF9fL0lqaGg4qx4cHGzWAVjT\nwKDrFHDN9zzdBgAv4dVB5ptvvlFLS4tsNpvWrl2r2tparV69Wi0tLWpubj7rULKfn5+cTqckqaWl\n5bx1AABgfV4dZL773e/qk08+0aBBgyRJERER6ujo0COPPKK77rpLx44dc5nf6XTK399fkmSz2c4K\nLU6n01wXAACwPq+/aunM4BEeHq7W1lYFBwersbHRpeZwOBQSEiJJGjJkyHnrAADA+rw6yOzatUvj\nxo1Ta2urOVZRUaHAwECNHTtWn3/+uctRl6KiItntdkknr2IoLi42a83NzaqoqDDrAADA+rw6yMTE\nxKh///5aunSpvvjiC33wwQfKysrSfffdp7i4OF177bVavHixqqqqtGHDBpWXl2vGjBmSpOTkZBUX\nF2vjxo2qqqrSkiVLdN111+mmm27y8FYBAIDu4tVB5uqrr9bmzZt19OhRzZgxQ8uWLdNPf/pT3XPP\nPerTp49yc3PV2Nio5ORkvfXWW8rOztY111wjSRo6dKjWrVungoICzZw5U8ePH9f69es9vEUAAKA7\nefXJvtLJc2I2b97caS0sLEx5eXnnXHbixIl65513eqo1AADgYV59RAYAAOB8CDIAAMCyCDIAAMCy\nCDIAAMCyCDIAAMCyCDIAAMCyvP7ya6tzOp0qLS11GYuOjj7rgZYAAODiEWR6WGlpqe5blqeBQddJ\nko4f+VobV0pxcXEe7gwAAOsjyPSCgUHXKeCa73m6DQAALjucIwMAACyLIAMAACyLIAMAACyLc2Rw\nls6utJK42goA4H0IMjjLmVdaSVxtBQDwTgQZL+DpIyBnfv+KigqutIJldLR/q4qKCpcxjh4CVw6C\nTC/r7E23oqJCz7xS5LEjIGcegak/WKghwznyAs/rLGSf6R9Nh/XMK99oYFCjJI4eAlcagkwvO/NN\nV/pXcDj9CEhv/5Z5+hGY40dqeuR7ABerqyGbI4jAlYsg4wFnvul2Fhz4LRM4iZAN4HwIMl7sSvgt\n09PnBwEArI0gA7d0VwDhCikAwKUgyMAt3RlAroQjT7h8nRnq29raJEn9+vVzmY+jjEDPIMjAbQQQ\noPMTkq/6zhCOMgK9hCBzhensI6HOLmkF0HVnnpA8MCiMkA/0EoLMFaazj4S4bwwAwKoIMlegrlz+\nDQCAFfD0awAAYFkckQFwWensrtidXUnUm1cR8TwooOcQZNBteLOGNzjXY0BOv5Kot68i4k7dQM8h\nyKDbnPlm/ffGL7TwP29SZGSky3yEG/S0zs4Du9gribr7rtOn99RZ6L+UdQNXMoKMhXX2RnvmG2FX\nnh7cFWe+8Z5rPWdehnryqd7/+s2Y30RhFT151+nOjhp1FvwvFGx4xAdAkLG0M99oO3uT7erTgy/k\nzDferq6Hm+bBynry/29nR41OD/5dCU084gMgyFhGZ4eiKyoquvRG211PD/bUU4j5rfPKwM0a3QtO\n/LJw+enK0fYruZ8zXfZBxul06te//rXeffdd+fv765577lFqaqqn27po5zqB8Uq4kR2/dV4ZuFkj\ncFJXjrZfyf2c6bIPMv/1X/+liooK5eXlqba2Vo899piGDh2qyZMne7q1i3Yl38iO3zqvDFfy/3Hg\ndN72nudt/Zzusr4hXnNzs/Lz8/X4448rIiJCiYmJmj9/vl588UVPtwYAALrBZR1kDhw4oPb2dtnt\ndnMsNjZWZWVlHuwKAAB0l8s6yDQ2NiogIEB9+/7rE7SgoCC1trbq6NGjHuwMAAB0Bx/DMAxPN9FT\n3njjDa1du1Z//vOfzbGamhpNnjxZ77//voYMGXLe5W+88UZ9++23uvbaayVJDY3/J8PX36x/6/yn\nJB/59DkZlIyOb/WdATaX26C3tbXp7ydazXnav21Vnz6+5nRXx7oyT3d9/56c58wez+zP3XlgfZ3t\n5576v9mV/z9d/X/XU6+x7uwR1nLmfvX0Pu2sn+DAAbLZbOdd7vDhw/L19VV5eXmP9ndZn+xrs9nk\ndDpdxk5N9+/f/4LL+/n56fScFxoy+Iw5rupSDwMGDLhwsz3E09//QrrSn7dvA7qHt+3nrvbjyb69\n7d8M3cPb9qu7/fTt27dXLtG+rIPMkCFD1NTUpI6ODvXpc/JTNIfDIX9/fw0aNOiCy+/fv7+nWwQA\nAJfgsj5HZuTIkerbt68+/fRTc2z//v0aPXq0B7sCAADd5bIOMv7+/po+fbqWL1+u8vJy7dy5U1u2\nbNG8efM83RoAAOgGl/XJvpLU0tKijIwM7dixQwMHDtT8+fM1d+5cT7cFAAC6wWUfZAAAwOXrsv5o\nCQAAXN4IMgAAwLIIMgAAwLIIMgAAwLIIMgAAwLIIMp1wOp361a9+pbi4OE2cOFFbtmzxdEu4gJ07\ndyoiIkIjR440/3z44YclSbW1tUpNTVVMTIymTp2q3bt3uyy7Z88eJSUlyW63KyUlRTU1NZ7YBOjk\nay8pKUmFhYXm2KXuv61btyo+Pl6xsbFaunSpWltbe2Vb0Pn+XLVq1Vmv1Zdeesmssz+9U319vRYs\nWKBx48YpISFBTz75pPnIH4+/Rg2cZcWKFcb06dONyspK49133zXGjBlj7Nixw9Nt4Txyc3ON+++/\n3zhy5IjhcDgMh8NhHD9+3DAMw0hKSjIeffRRo7q62nj++ecNu91uHD582DAMw/jmm28Mu91ubNmy\nxaiqqjJ+/vOfG0lJSZ7clCtWa2urkZ6ebkRERBj79u0zx6dNm+b2/nvnnXeMuLg44/333zfKy8uN\nO+64w1i5cmWvb9uV6Fz7MzU11di4caP5OnU4HEZLS4thGOxPbzZr1iwjLS3NqKqqMvbv329MnjzZ\neOqppwzDuLT32O7YpwSZM/zzn/80brzxRqOwsNAcy8nJMebOnevBrnAhixYtMtasWXPW+J49e4yY\nmBjzjdIwDCMlJcVYt26dYRiG8eyzz7rs2+bmZmPMmDEub7zoeVVVVcb06dON6dOnu/zgu9T995//\n+Z/G+vXrzfr+/fuN6Ohol/Wh+51rfxqGYcTHxxu7d+/udLm1a9eyP71QdXW1ERERYRw5csQce/vt\nt434+Hhj7969Hn+N8tHSGQ4cOKD29nbZ7XZzLDY2VmVlZR7sChdSXV2t66+//qzxsrIyjRo1yuVx\n87Gxsebzt8rKyhQXF2fW/P39FRkZqZKSkp5vGqZ9+/Zp/Pjx2r59u8sT5y9l/3V0dKi8vFxjx441\n63a7XW1tbTpw4EAvbNWV61z788SJE6qvr9ewYcM6Xa60tJT96YVCQkK0adMmDR482GX8+PHjKi0t\n9fhr9LJ++rU7GhsbFRAQoL59//VPExQUpNbWVh09elSBgYEe7A7n8sUXX+ijjz5Sbm6uOjo6dPvt\nt2vBggVqbGxUaGioy7xBQUGqr6+XJDU0NJxVDw4ONuvoHXfffXen45ey/44dO6bW1laXuq+vrwIC\nAlRXV6fo6Ohu3gqccq79efDgQfn4+Cg3N1cffvihAgIClJqaqjvvvFMS+9NbDRw4UBMmTDCnDcPQ\niy++qPHjx3vFa5Qgc4bm5mb5+fm5jJ2aPnViE7zLN998o5aWFtlsNq1du1a1tbVavXq1Wlpazrk/\nT+3LlpaW89bhWZey/1paWszpcy2P3nXw4EH16dNH4eHhmjt3rvbt26dly5ZpwIABSkxMZH9axFNP\nPaXKykrl5+dry5YtHn+NEmTOYLPZzvoHPDXdv39/T7SEC/jud7+rTz75RIMGDZIkRUREqKOjQ488\n8ojuuusuHTt2zGV+p9Mpf39/Sefe36fWBc+y2Wz6+9//7jLW1f13rl9AnE4nr2UPufPOOzVp0iTz\n9XXDDTfoyy+/1CuvvKLExET2pwVkZWUpLy9Pzz77rEaMGOEVr1HOkTnDkCFD1NTUpI6ODnPM4XDI\n39+fH25e7Mx9Ex4ertbWVgUHB6uxsdGl5nA4FBISIunk/j5fHZ51of1zvnpgYKBsNpscDodZa29v\nV1NTE/vXg858rQ4fPlwNDQ2S2J/ebuXKldq2bZuysrKUmJgoyTteowSZM4wcOVJ9+/Y1T1SSpP37\n92v06NEe7Arns2vXLo0bN87l3gMVFRUKDAzU2LFj9fnnn7sk/qKiIvNk7ujoaBUXF5u15uZmVVRU\nuJzsDc+Jjo5WRUXFRe+/mJgY+fj4KCoqSkVFRWa9pKRE/fr1U0RERO9tBEzPPfecUlNTXcYqKyvN\nE/XZn95r/fr12r59u5555hn96Ec/Mse94jXq3sVYl7cnnnjCmDp1qlFWVma8++67RmxsrPHuu+96\nui2cw4kTJ4yEhARj4cKFxsGDB43333/fmDhxorF582ajvb3duOOOO4xf/OIXxt/+9jfj+eefN8aM\nGWPe46C2ttaIjo42NmzYYPztb38zHn74YePOO+/08BZd2b7//e+bl2a2t7cbU6dOvaj9N336dHNd\nf/zjH42xY8ca7777rlFaWmpMnTrVWL16tUe260p1+v4sKyszRo0aZfz+9783vv76a+Oll14ybrzx\nRqO0tNQwDPant6qqqjIiIyONtWvXGo2NjS5f3vAaJch0orm52Vi8eLERExNjxMfHGy+88IKnW8IF\nVFVVGffcc48xZswYY+LEiUZ2drZZ+/rrr405c+YYN954ozF16lRj7969Lst++OGHxm233WbY7Xbj\nnnvuMWpra3u7fZzmzPuOXOr+27Bhg/GDH/zAiIuLMx5//HGjtbW1V7YDJ525P9977z1j2rRpRnR0\ntDFlypSzfklkf3qf559/3oiIiHD5+v73v29EREQYhmEYX331lUdfoz6GcdpF/gAAABbCOTIAAMCy\nCDIAAMCyCDIAAMCyCDIAAMCyCDIAAMCyCDIAAMCyCDIAAMCyCDIAAMCyCDIAAMCyCDIAAMCyCDIA\nAMCy/j8NkMaJ/IlqlgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x57067d550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = (6, 4)\n",
    "df_output['truth'].plot.hist(bins=100);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>truth</th>\n",
       "      <th>predicted</th>\n",
       "      <th>predicted_name</th>\n",
       "      <th>item_name</th>\n",
       "      <th>mapped_level_0</th>\n",
       "      <th>level_0</th>\n",
       "      <th>preds_level_0</th>\n",
       "      <th>mapped_level_1</th>\n",
       "      <th>level_1</th>\n",
       "      <th>preds_level_1</th>\n",
       "      <th>mapped_level_2</th>\n",
       "      <th>level_2</th>\n",
       "      <th>preds_level_2</th>\n",
       "      <th>mapped_level_3</th>\n",
       "      <th>level_3</th>\n",
       "      <th>preds_level_3</th>\n",
       "      <th>mapped_level_4</th>\n",
       "      <th>level_4</th>\n",
       "      <th>preds_level_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>432</th>\n",
       "      <td>95.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>noah's mill 114.3 pf bourbon</td>\n",
       "      <td>basil - pesticide free</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Produce</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>Herbs</td>\n",
       "      <td>537.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>Basil</td>\n",
       "      <td>95.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433</th>\n",
       "      <td>95.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>basil - pesticide free</td>\n",
       "      <td>basil-lb pest-free</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Produce</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Herbs</td>\n",
       "      <td>537.0</td>\n",
       "      <td>537.0</td>\n",
       "      <td>Basil</td>\n",
       "      <td>95.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>157.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>garlic super colossal</td>\n",
       "      <td>garlic super colossal</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Produce</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Garlic</td>\n",
       "      <td>157.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>176.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>oranges 56ct</td>\n",
       "      <td>oranges 56ct</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Produce</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Fruits</td>\n",
       "      <td>535.0</td>\n",
       "      <td>535.0</td>\n",
       "      <td>Oranges</td>\n",
       "      <td>176.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455</th>\n",
       "      <td>176.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>oranges 56ct</td>\n",
       "      <td>oranges 72ct</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Produce</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Fruits</td>\n",
       "      <td>535.0</td>\n",
       "      <td>535.0</td>\n",
       "      <td>Oranges</td>\n",
       "      <td>176.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     truth  predicted                predicted_name               item_name  \\\n",
       "432   95.0       35.0  noah's mill 114.3 pf bourbon  basil - pesticide free   \n",
       "433   95.0       95.0        basil - pesticide free      basil-lb pest-free   \n",
       "444  157.0      157.0         garlic super colossal   garlic super colossal   \n",
       "454  176.0      176.0                  oranges 56ct            oranges 56ct   \n",
       "455  176.0      176.0                  oranges 56ct            oranges 72ct   \n",
       "\n",
       "    mapped_level_0  level_0  preds_level_0 mapped_level_1  level_1  \\\n",
       "432           Food        1            3.0        Produce      9.0   \n",
       "433           Food        1            1.0        Produce      9.0   \n",
       "444           Food        1            1.0        Produce      9.0   \n",
       "454           Food        1            1.0        Produce      9.0   \n",
       "455           Food        1            1.0        Produce      9.0   \n",
       "\n",
       "     preds_level_1 mapped_level_2  level_2  preds_level_2 mapped_level_3  \\\n",
       "432            6.0          Herbs    537.0           54.0          Basil   \n",
       "433            9.0          Herbs    537.0          537.0          Basil   \n",
       "444            9.0         Garlic    157.0          157.0            NaN   \n",
       "454            9.0         Fruits    535.0          535.0        Oranges   \n",
       "455            9.0         Fruits    535.0          535.0        Oranges   \n",
       "\n",
       "     level_3  preds_level_3 mapped_level_4  level_4  preds_level_4  \n",
       "432     95.0           52.0            NaN      NaN           35.0  \n",
       "433     95.0           95.0            NaN      NaN            NaN  \n",
       "444      NaN            NaN            NaN      NaN            NaN  \n",
       "454    176.0          176.0            NaN      NaN            NaN  \n",
       "455    176.0          176.0            NaN      NaN            NaN  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_output.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_output = df_output.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## First Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:     94.489\n",
      "Recall:   90.365  \n",
      "F Score  92.267  \n",
      "Support:    4     \n",
      "Chance:  25.000  \n"
     ]
    }
   ],
   "source": [
    "precision, recall, fbeta_score, support = score(df_output.level_0, df_output.preds_level_0)\n",
    "print (\"Precision: {:10.3f}\\nRecall: {:^10.3f}\\nF Score{:^10.3f}\\nSupport:{:^10}\\nChance:{:^10.3f}\".format(precision.mean()*100, \n",
    "                                                 recall.mean()*100, fbeta_score.mean()*100, len(support), (1/len(support))*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Second Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:     79.803\n",
      "Recall:   64.363  \n",
      "F Score  66.517  \n",
      "Support:    14    \n",
      "Chance:  7.143   \n"
     ]
    }
   ],
   "source": [
    "precision, recall, fbeta_score, support = score(df_output.level_1, df_output.preds_level_1)\n",
    "print (\"Precision: {:10.3f}\\nRecall: {:^10.3f}\\nF Score{:^10.3f}\\nSupport:{:^10}\\nChance:{:^10.3f}\".format(precision.mean()*100, \n",
    "                                                 recall.mean()*100, fbeta_score.mean()*100, len(support), (1/len(support))*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Third Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:     60.151\n",
      "Recall:   48.520  \n",
      "F Score  47.016  \n",
      "Support:    39    \n",
      "Chance:  2.564   \n"
     ]
    }
   ],
   "source": [
    "precision, recall, fbeta_score, support = score(df_output.level_2, df_output.preds_level_2)\n",
    "print (\"Precision: {:10.3f}\\nRecall: {:^10.3f}\\nF Score{:^10.3f}\\nSupport:{:^10}\\nChance:{:^10.3f}\".format(precision.mean()*100, \n",
    "                                                 recall.mean()*100, fbeta_score.mean()*100, len(support), (1/len(support))*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Fourth Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:     44.091\n",
      "Recall:   50.211  \n",
      "F Score  39.591  \n",
      "Support:  33.000  \n",
      "Chance:  3.030   \n"
     ]
    }
   ],
   "source": [
    "precision, recall, fbeta_score, support = score(df_output.level_3, df_output.preds_level_3)\n",
    "print (\"Precision: {:10.3f}\\nRecall: {:^10.3f}\\nF Score{:^10.3f}\\nSupport:{:^10.3f}\\nChance:{:^10.3f}\".format(precision.mean()*100, \n",
    "                                                 recall.mean()*100, fbeta_score.mean()*100, len(support), (1/len(support))*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}