notebooks/scrape-voter-power-index.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrape the \"Voter Power Index\" From 538\n",
    "\n",
    "This notebook scrapes the \"Voter Power Index\" for 538 at a given moment and puts the resulting CSV in the `data/fivethirtyeight` directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import lxml.html\n",
    "import cssselect\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetch 538 Election Forecast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "URL = \"http://projects.fivethirtyeight.com/2016-election-forecast/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "res = requests.get(URL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dom = lxml.html.fromstring(res.content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract \"Voter Power Index\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "table = dom.cssselect(\"table.tippingpointroi\")[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "states = []\n",
    "for row in table.cssselect(\"tr\"):\n",
    "    abbrev = row.attrib[\"data-state\"]\n",
    "    cells = row.cssselect(\"td\")\n",
    "    state = cells[0].text_content().strip()\n",
    "    power = cells[1].text_content().strip()\n",
    "    # Set any value less than 0.1 to 0\n",
    "    if power == \"<0.1\":\n",
    "        power = 0\n",
    "    else:\n",
    "        power = float(power)\n",
    "    state_dict = {\n",
    "        \"state_raw\": state,\n",
    "        \"power\": power,\n",
    "        \"abbrev\": abbrev\n",
    "    }\n",
    "    states.append(state_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "voter_power_index = pd.DataFrame(states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_state_names(state_str):\n",
    "    # Prepare to drop non-statewide electoral votes\n",
    "    if state_str in [\n",
    "        \"Maine 2nd District\", \"Nebraska 2nd District\", \n",
    "         \"Maine 1st District\", \"Nebraska 1st District\", \n",
    "         \"Nebraska 3rd District\" ]:\n",
    "        return None\n",
    "    elif state_str == \"Maine - statewide\":\n",
    "        return \"MAINE\"\n",
    "    elif state_str == \"Nebraska - statewide\":\n",
    "        return \"NEBRASKA\"\n",
    "    else:\n",
    "        return state_str.upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "voter_power_index[\"state\"] = voter_power_index[\"state_raw\"].apply(clean_state_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>abbrev</th>\n",
       "      <th>power</th>\n",
       "      <th>state_raw</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NH</td>\n",
       "      <td>5.6</td>\n",
       "      <td>New Hampshire</td>\n",
       "      <td>NEW HAMPSHIRE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NV</td>\n",
       "      <td>4.9</td>\n",
       "      <td>Nevada</td>\n",
       "      <td>NEVADA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NM</td>\n",
       "      <td>4.7</td>\n",
       "      <td>New Mexico</td>\n",
       "      <td>NEW MEXICO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MI</td>\n",
       "      <td>3.1</td>\n",
       "      <td>Michigan</td>\n",
       "      <td>MICHIGAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CO</td>\n",
       "      <td>3.1</td>\n",
       "      <td>Colorado</td>\n",
       "      <td>COLORADO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  abbrev  power      state_raw          state\n",
       "0     NH    5.6  New Hampshire  NEW HAMPSHIRE\n",
       "1     NV    4.9         Nevada         NEVADA\n",
       "2     NM    4.7     New Mexico     NEW MEXICO\n",
       "3     MI    3.1       Michigan       MICHIGAN\n",
       "4     CO    3.1       Colorado       COLORADO"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voter_power_index.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "voter_power_index.to_csv(\"../data/fivethirtyeight/voter-power-index.csv\", index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "---\n",
    "\n",
    "---"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}