Formspring Data Visualization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization of Data from Formspring \n",
    "\n",
    "Involves \"smart\" parsing, specifically: convert words into sounds and then back into a dictionary to see if we can decipher some of the slang used in online communication. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "import sys\n",
    "import os\n",
    "from util.Util import *\n",
    "from util.parser import formspring_data_parser\n",
    "import pandas as pd\n",
    "from sklearn import model_selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append(\"/Users/luisd/dev/cyberbullying-detection\")\n",
    "# from nn import *\n",
    "# import nn \n",
    "# import fc_nn "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# sys.path.append(...) # in case we want to add something else"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import phonemes_from_graphemes as pg\n",
    "from words_2_vectors import * \n",
    "from util import formspring_data_parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# import importlib\n",
    "# importlib.reload(pg)\n",
    "from phonemes_from_graphemes import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Where am I expecting to see the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dir = \"./data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-06-14 14:40:30,665 - common_logger - INFO - 'common_logger': logging 'INFO'+ logs to Console, 'DEBUG'+ logs to '/Users/luisd/dev/cyberbullying-detection/common_logger.log'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logger created\n",
      "Creating debug handler at '/Users/luisd/dev/cyberbullying-detection/common_logger.log'\n",
      "'common_logger': logging 'INFO'+ logs to Console, 'DEBUG'+ logs to '/Users/luisd/dev/cyberbullying-detection/common_logger.log'\n"
     ]
    }
   ],
   "source": [
    "common_logger = get_logger(name = \"common_logger\", debug_log_file_name = \"common_logger.log\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/luisd/dev/cyberbullying-detection/common_logger.log'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common_logger.handlers[1].baseFilename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from util.parser.formspring_data_parser import * \n",
    "from util.parser.wikipedia_talk_parser import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "a_formspring_data_parser = Formspring_Data_Parser(all_data = pd.read_csv(\"really_all_of_them.csv\"), alogger = common_logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Word2Vec model trained with Google News "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-06-14 14:45:46,093 - common_logger - INFO - Loading model from ./data/GoogleNews-vectors-negative300.bin.gz...\n",
      "2017-06-14 14:48:11,008 - common_logger - INFO - Model succesfully loaded\n",
      "2017-06-14 14:48:11,010 - common_logger - INFO - Sort all the words in the model, so that we can auto-complete queries quickly...\n"
     ]
    }
   ],
   "source": [
    "mw = ModelWrapper.from_google_news_model(data_dir=data_dir, alogger=common_logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "m = mw.model # cache model definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3000000"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(m.vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creation of Sounds' Dictionary  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's read the sounds' dictionary\n",
    "(and minimally sanit-check it) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sounds_dict = SoundsDict(a_dir=data_dir, alogger=common_logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'YOU', 'YOu', 'You', 'you'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sounds_dict['ju:']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mw.set_sounds_dict(sounds_dict=sounds_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'YOU', 'YOu', 'You', 'you'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mw.sound_to_word('ju:')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Read data from XML  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'list' object cannot be interpreted as an integer",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-22-bd0540976e93>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mxml_file_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'/Users/luisd/Downloads/FormspringLabeledForCyberbullying/XMLMergedFile.xml'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mphonemesFactory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPhonemesFromGraphemes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malogger\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcommon_logger\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFormspring_Data_Parser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_raw_xml\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxml_file_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mphonemesFactory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malogger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcommon_logger\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/luisd/dev/cyberbullying-detection/util/parser/formspring_data_parser.py\u001b[0m in \u001b[0;36mfrom_raw_xml\u001b[0;34m(cls, path_to_xml, pg, mw, alogger)\u001b[0m\n\u001b[1;32m    135\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_to_xml\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m             \u001b[0mthe_doc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxmltodict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 137\u001b[0;31m         \u001b[0mreally_all\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mall_questions_answers_labels\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthe_doc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhow_many_entries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mthe_doc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'dataset'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'FORMSPRINGID'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malogger\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0malogger\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    138\u001b[0m         \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreally_all\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malogger\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0malogger\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/luisd/dev/cyberbullying-detection/util/parser/formspring_data_parser.py\u001b[0m in \u001b[0;36mall_questions_answers_labels\u001b[0;34m(pg, mw, a_doc, how_many_entries, alogger)\u001b[0m\n\u001b[1;32m    122\u001b[0m     return functools.reduce(\n\u001b[1;32m    123\u001b[0m         \u001b[0;32mlambda\u001b[0m \u001b[0mdf1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf2\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdf1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 124\u001b[0;31m         [questions_answers_labels(pg, mw, a_doc, an_id, alogger) for an_id in range(how_many_entries)])\n\u001b[0m\u001b[1;32m    125\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: 'list' object cannot be interpreted as an integer"
     ]
    }
   ],
   "source": [
    "xml_file_name = '/Users/luisd/Downloads/FormspringLabeledForCyberbullying/XMLMergedFile.xml'\n",
    "phonemesFactory = PhonemesFromGraphemes(alogger=common_logger)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(xml_file_name) as fd:\n",
    "    the_doc = xmltodict.parse(fd.read())\n",
    "how_many_entries(the_doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "collections.OrderedDict"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(the_doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-06-14 16:47:08,058 - common_logger - INFO - 'https://ndownloader.figshare.com/files/7554634' is downloaded as 'attack_annotated_comments.tsv'\n",
      "2017-06-14 16:47:13,801 - common_logger - INFO - 'https://ndownloader.figshare.com/files/7554637' is downloaded as 'attack_annotations.tsv'\n"
     ]
    }
   ],
   "source": [
    "wiki_parser = Wikipedia_Talk_Parser.from_sources(pg = phonemesFactory, mw = mw, alogger=common_logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "1 + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wiki_parser.save(\"dataframe_wiki.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49]\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "__init__() missing 1 required positional argument: 'self'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-5f580acb4226>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFormspring_Data_Parser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_raw_xml\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxml_file_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mphonemesFactory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malogger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcommon_logger\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/luisd/dev/cyberbullying-detection/util/parser/formspring_data_parser.py\u001b[0m in \u001b[0;36mfrom_raw_xml\u001b[0;34m(cls, path_to_xml, pg, mw, alogger)\u001b[0m\n\u001b[1;32m    139\u001b[0m     \u001b[0;31m#     self.doc = None\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m     \u001b[0;31m#     with open(path_to_xml) as fd:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m     \u001b[0;31m#         self.path = path_to_xml\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    142\u001b[0m     \u001b[0;31m#         self.doc = xmltodict.parse(fd.read())\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m     \u001b[0;31m#     self.alogger = alogger\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: __init__() missing 1 required positional argument: 'self'"
     ]
    }
   ],
   "source": [
    "parser = Formspring_Data_Parser.from_raw_xml(xml_file_name, pg = phonemesFactory, mw = mw, alogger = common_logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# really_all = parser.all_questions_answers_labels()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# really_all.to_csv(\"really_all_of_them.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "really_all = pd.read_csv(\"really_all_of_them.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "threatening_items = really_all.loc[really_all[\"threat\"] == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"threatening_items.shape = {}\".format(threatening_items.shape))\n",
    "print(\"really_all.shape = {}\".format(really_all.shape))\n",
    "print(\"There are {}/{} interactions ({:.2f}% of total) that are threatening\".format(threatening_items.shape[0], really_all.shape[0], threatening_items.shape[0] * 100/really_all.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "questions_answers = [\"{}; {}\".format(q, a) for q, a in list(zip(really_all['question'].tolist(), really_all['answer'].tolist()))] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "questions_answers[0:10] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels = [\"THREAT\" if threat else \"CLEAN\" for threat in really_all['threat'].tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "labels[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create three Counter objects to store positive, negative and total counts\n",
    "positive_counts = Counter()\n",
    "negative_counts = Counter()\n",
    "total_counts = Counter()\n",
    "for label, review in zip(labels, questions_answers):\n",
    "    words = review.split(' ') \n",
    "    if label == 'THREAT':\n",
    "        for word in words:\n",
    "            negative_counts[word] += 1\n",
    "    else:\n",
    "        for word in words:\n",
    "            positive_counts[word] += 1\n",
    "    for word in words:\n",
    "        total_counts[word] += 1\n",
    "\n",
    "pos_neg_ratios = Counter()\n",
    "\n",
    "unique_words = total_counts.keys()\n",
    "for word in unique_words:\n",
    "    if total_counts[word] >= 100:\n",
    "        pos_neg_ratios[word] = positive_counts[word] / float(negative_counts[word]+1)\n",
    "        \n",
    "        \n",
    "unique_words = pos_neg_ratios.keys()# set(pos_neg_ratios.elements())\n",
    "for word in unique_words:\n",
    "    if (pos_neg_ratios[word] > 1):\n",
    "        pos_neg_ratios[word] = np.log(pos_neg_ratios[word])\n",
    "    else:\n",
    "        pos_neg_ratios[word] = -np.log(1/(pos_neg_ratios[word] + 0.01))        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "negative_counts.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# negative_counts.most_common()\n",
    "print(\"Pos-to-neg ratio for 'the' = {}\".format(pos_neg_ratios[\"the\"]))\n",
    "print(\"Pos-to-neg ratio for 'amazing' = {}\".format(pos_neg_ratios[\"amazing\"]))\n",
    "print(\"Pos-to-neg ratio for 'shit' = {}\".format(pos_neg_ratios[\"shit\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What are the most common words for 'threats' and 'non-threats' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pos_neg_ratios.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "list(reversed(pos_neg_ratios.most_common()))[0:30]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Yeah... doesn't look like there is a pattern here... \n",
    "Maybe some more work on the pre-treatment is needed \n",
    "\n",
    "### Let's take a look "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from bokeh.models import ColumnDataSource, LabelSet\n",
    "from bokeh.plotting import figure, show, output_file\n",
    "from bokeh.io import output_notebook\n",
    "output_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hist, edges = np.histogram(list(map(lambda x:x[1],pos_neg_ratios.most_common())), density=True, bins=100, normed=True)\n",
    "\n",
    "p = figure(tools=\"pan,wheel_zoom,reset,save\",\n",
    "           toolbar_location=\"above\",\n",
    "           title=\"Word Positive/Negative Affinity Distribution\")\n",
    "p.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:], line_color=\"#555555\")\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "frequency_frequency = Counter()\n",
    "\n",
    "for word, cnt in total_counts.most_common():\n",
    "    frequency_frequency[cnt] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hist, edges = np.histogram(list(map(lambda x:x[1],frequency_frequency.most_common())), density=True, bins=100, normed=True)\n",
    "\n",
    "p = figure(tools=\"pan,wheel_zoom,reset,save\",\n",
    "           toolbar_location=\"above\",\n",
    "           title=\"The frequency distribution of the words in our corpus\")\n",
    "p.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:], line_color=\"#555555\")\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vocab = set(total_counts.keys())\n",
    "vocab_size = len(vocab)\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python (python3.6)",
   "language": "python",
   "name": "myenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}