review-3-traffic-accidents.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "71fd8080c77935a0e21d4944691108b4280ed8f6"
   },
   "source": [
    "# Road Accident Prediction And Classification\n",
    "###### Abdul Wahed and Abrar Hashmi        \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "4f3c71060eb916343706bb2d8a500ba979e331dc"
   },
   "source": [
    "### Introduction\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "28c6e782278d576238ac2fd5fbddfc88adc61796"
   },
   "source": [
    "## Importing Data and cleaning\n",
    "- We import three files to perform analysis on this data. This data is consist of three files that are accidents, casualities and vehicles. However, we have one more file which is general information about the traffic count for year 2000 to 2015. We can use general traffic information data for machine learning part."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_uuid": "43f2cbb55a0f5e95b02c7ee99d9e416f83836dfd"
   },
   "outputs": [],
   "source": [
    "import datetime as dt\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "#from mpl_toolkits.basemap import Basemap\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "plt.style.use('ggplot')\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_uuid": "0b93ca1a20c07af055065343e30fb64c37dcadd1"
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "File b'Accidents0515.csv' does not exist",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-06456ee12c1a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0maccidents\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Accidents0515.csv'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mindex_col\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'Accident_Index'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mcasualties\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Casualties0515.csv'\u001b[0m \u001b[1;33m,\u001b[0m \u001b[0merror_bad_lines\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mindex_col\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'Accident_Index'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mwarn_bad_lines\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mvehicles\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Vehicles0515.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merror_bad_lines\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mindex_col\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'Accident_Index'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mwarn_bad_lines\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;31m#general_info = pd.read_csv('ukTrafficAADF.csv')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[0;32m    676\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[0;32m    677\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 678\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    679\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    680\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    438\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    439\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 440\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    441\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    442\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m    785\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'has_index_names'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'has_index_names'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    786\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 787\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    788\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    789\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m   1012\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'c'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1013\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'c'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1014\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1015\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1016\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'python'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m   1706\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'usecols'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1707\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1708\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1709\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1710\u001b[0m         \u001b[0mpassed_names\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnames\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: File b'Accidents0515.csv' does not exist"
     ]
    }
   ],
   "source": [
    "accidents = pd.read_csv('Accidents0515.csv',index_col='Accident_Index')\n",
    "casualties=pd.read_csv('Casualties0515.csv' , error_bad_lines=False,index_col='Accident_Index',warn_bad_lines=False)\n",
    "vehicles=pd.read_csv('Vehicles0515.csv', error_bad_lines=False,index_col='Accident_Index',warn_bad_lines=False)\n",
    "#general_info = pd.read_csv('ukTrafficAADF.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "d7ceba9967f9674832cb2d88396be5477b718ec4"
   },
   "outputs": [],
   "source": [
    "accidents.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "06b5266e977f07acd3ad0018bd698fce03a17cda"
   },
   "outputs": [],
   "source": [
    "accidents = accidents.join(vehicles, how='outer')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "033c3bd61ac666e3fc87316f2642aff6dcab1719"
   },
   "source": [
    "## Identifying Missing Values\n",
    "\n",
    "In this particular dataset, there are two types of missing values '-1' and 'Nan'. We will invesitigate each column with total missing values.\n",
    "We will not be imputing any mean or median value since the dataset is big enough to perform analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "292691d5d1b20bf1664cb13dbda2c982f5563590"
   },
   "outputs": [],
   "source": [
    "accidents.drop(['Location_Easting_OSGR', 'Location_Northing_OSGR','LSOA_of_Accident_Location',\n",
    "                'Junction_Control' ,'2nd_Road_Class'], axis=1, inplace=True)\n",
    "accidents['Date_time'] =  accidents['Date'] +' '+ accidents['Time']\n",
    "\n",
    "for col in accidents.columns:\n",
    "    accidents = (accidents[accidents[col]!=-1])\n",
    "    #print(col ,' ' , x)\n",
    "for col in casualties.columns:\n",
    "    casualties = (casualties[casualties[col]!=-1])\n",
    "\n",
    "    accidents['Date_time'] = pd.to_datetime(accidents.Date_time)\n",
    "accidents.drop(['Date','Time'],axis =1 , inplace=True)\n",
    "accidents.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "c010b516516f74ec01f0347bee12c2dfe11f8edd"
   },
   "source": [
    "Our dataset is clean to do some analysis. We would be using very few columns to do analysis since the dataset is fairly large."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "190d4df6e3bfef24aab5176a410ae1890ef95108"
   },
   "source": [
    "# Data Visualization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "9f62ed3d0e6aeb5dd14df572d477357978c58da2"
   },
   "source": [
    "#### The first thing we can do is to find out about accidents time to get intution and some driver's age who are involved in the accident.\n",
    "- We can find out the number of accidents on the days of a week.\n",
    "- We can find out about the accidents number using hours of the day.\n",
    "- Finding out about the age of driver can tell us more about the accidents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "210383c1cf401845104e000c33815343ed939d91"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,6))\n",
    "accidents.Date_time.dt.dayofweek.hist(bins=7,rwidth=0.55,alpha=0.5, color= 'orange')\n",
    "plt.title('Accidents on the day of a week' , fontsize= 30)\n",
    "plt.grid(False)\n",
    "plt.ylabel('Accident count' , fontsize = 20)\n",
    "plt.xlabel('0 - Sunday ,  1 - Monday  ,2 - Tuesday , 3 - Wednesday , 4 - Thursday , 5 - Friday , 6 - Saturday' , fontsize = 13)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "9a2e5d2440529e7a60e4e689dbf8fc7714a7d044"
   },
   "source": [
    "As we can see that thursday has the highest amount of accidents in this dataset from 2005 to 2015. We have to keep in mind that accidents numbers could be depending on traffic amount on particular day."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "f2b4c64532386f9263b7f9bdd26cbac22ea8054a"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,6))\n",
    "accidents.Date_time.dt.hour.hist(rwidth=0.75,alpha =0.50, color= 'orange')\n",
    "plt.title('Time of the day/night',fontsize= 30)\n",
    "plt.grid(False)\n",
    "plt.xlabel('Time 0-23 hours' , fontsize = 20)\n",
    "plt.ylabel('Accident count' , fontsize = 15)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "654850f800234a8784e81e447142b44c6b99d278"
   },
   "source": [
    "We found out that the most of accidents happened around after noon. We can assume that this time of the day has the most traffic moving such as people leaving from work.\n",
    "\n",
    "\n",
    "#### Age band of casualities\n",
    "\n",
    "In this dataset, age band is grouped in 11 different codes. We will create the labels and pass it to the plot as xticks so we can have idea about the bins representation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "0eb24ee1383bc02820ac4af76e07203b3cc96db3"
   },
   "outputs": [],
   "source": [
    "objects = ['0','0-5','6-10','11-15','16-20','21-25','26-35',\n",
    "          '36-45', '46-55','56-65','66-75','75+']\n",
    "\n",
    "plt.figure(figsize=(12,6))\n",
    "casualties.Age_Band_of_Casualty.hist(bins = 11,alpha=0.5,rwidth=0.90, color= 'red',)\n",
    "plt.title('Age of people involved in the accidents', fontsize = 25)\n",
    "plt.grid(False)\n",
    "y_pos = np.arange(len(objects))\n",
    "plt.xticks(y_pos , objects)\n",
    "plt.ylabel('Accident count' , fontsize = 15)\n",
    "plt.xlabel('Age of Drivers', fontsize = 15,)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "6ef96ceff3694498d0e12ed5c214519e3a9f56f3"
   },
   "source": [
    "This is very interesting fact about this dataset. Most of the drivers age is around 225 to 35 who are involved in the accident. However, we do not know the number of drivers with age 25 to 35 on the road compare to other ages. Intutively, I would assume that the driver with age 25 to 35 are more in the number of drivers with different age."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "465462cdac777a4ba6b1351808a1f81eb7658fca"
   },
   "outputs": [],
   "source": [
    "speed_zone_accidents = accidents.loc[accidents['Speed_limit'].isin(['20' ,'30' ,'40' ,'50' ,'60' ,'70'])]\n",
    "speed  = speed_zone_accidents.Speed_limit.value_counts()\n",
    "\n",
    "explode = (0.0, 0.0, 0.0 , 0.0 ,0.0,0.0) \n",
    "plt.figure(figsize=(10,8))\n",
    "plt.pie(speed.values,  labels=None, \n",
    "        autopct='%.1f',pctdistance=0.8, labeldistance=1.9 ,explode = explode, shadow=False, startangle=160,textprops={'fontsize': 15})\n",
    " \n",
    "plt.axis('equal')\n",
    "plt.legend(speed.index, bbox_to_anchor=(1,0.7), loc=\"center right\", fontsize=15, \n",
    "           bbox_transform=plt.gcf().transFigure)\n",
    "# plt.figtext(.5,.9,'Accidents percentage in Speed Zone', fontsize=25, ha='center')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "510d4d4fcf5618e3db3fea5fcd125dbe07d06d5d"
   },
   "source": [
    "Most of the accidents occured on the road where the speed limit is 30. I was expecting more accidents on highway or major roadways. Some of the accidents could be cause of stop sign, changing lanes or turning into parking lot etc."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "cbbb531cc52aea344a7f0e34f7158feedcfecf05"
   },
   "source": [
    "## Co-relation between variables\n",
    "\n",
    "Since our dataset is in numeric values. We can findout correlation between columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "741ba44e0e1154e24ae0b8a0d9ed2e42710bb5f5"
   },
   "outputs": [],
   "source": [
    "corr =  accidents.corr()\n",
    "plt.subplots(figsize=(20,9))\n",
    "sns.heatmap(corr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "ebfdadae90f44fd7783071d7c895e3d492a0142e"
   },
   "source": [
    "As we see that there is not so much strong correlations between any variables. I was expecting weather condition to be strong correlation with any of the variable. \n",
    "- There is only one postiive strong correlation between speed limit and Urban or Rural Area. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "996760b6548e85fa80d61fee849647190aa946de"
   },
   "outputs": [],
   "source": [
    "accidents_2014 = accidents[accidents.Date_time.dt.year ==2014]\n",
    "accidents_2014_01 = accidents_2014[accidents_2014.Accident_Severity == 1]\n",
    "accidents_2014_02 = accidents_2014[accidents_2014.Accident_Severity == 2]\n",
    "accidents_2014_03 = accidents_2014[accidents_2014.Accident_Severity == 3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "a8e320ab1318e7bb3a0bd1a64d3389dd9227ad22"
   },
   "source": [
    "As we can see that most of fatal accidents happened locally within cities instead on highways. It could be the reason of the traffic is more congested locally than on highways."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "525e94433411de32b1c6b52f778abf07bc1f7883"
   },
   "source": [
    "## Machine Learning\n",
    "\n",
    "We will be looking at different columns to figure out predicting about the accidents severity. After we can predict the accident severity, we can make some recommendation to law enforcement for looking into this and be prepared for the future. We can also have more emergency medical services available for those situations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "5626d86298cd35b6ce5367d1331a9dbf279b96fd"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "6f440c02334d1e37d67a6b246effd17d4425de10"
   },
   "source": [
    "## Normalize the Data\n",
    "There are few columns that we will standarize, so it would not effect negatively on our machine learning algorithms. Age of driver is from 18 to 88 in the dataset and we can normalize it. Also, the age of vehicle is also from 0 to 100 and it can skew the performance of your machine learning algorithm and we will normalize this predictor too."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "baabdc534c038c27b30123553e6b2ac2680879d4"
   },
   "outputs": [],
   "source": [
    "sns.distplot(accidents['Age_of_Driver']);\n",
    "fig = plt.figure()\n",
    "sns.distplot(accidents['Age_of_Vehicle']);\n",
    "fig = plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "ac0701bcf6d5d46bc1804ca6a2065ffad1e574aa"
   },
   "outputs": [],
   "source": [
    "accidents['Age_of_Driver'] = np.log(accidents['Age_of_Driver'])\n",
    "accidents['Age_of_Vehicle'] = np.log(accidents['Age_of_Vehicle'])\n",
    "sns.distplot(accidents['Age_of_Driver']);\n",
    "fig = plt.figure()\n",
    "sns.distplot(accidents['Age_of_Vehicle']);\n",
    "fig = plt.figure()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "8996af4753d28de07eb8b80690cf813b526ffa8e"
   },
   "source": [
    "## Spliting the data into training data and test data\n",
    "We will also consider few features as predictors for machine learning algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "4ff41e45b4ddcfbf1dc8e3741e5db4e4bd0188c6"
   },
   "outputs": [],
   "source": [
    "accident_ml = accidents.drop('Accident_Severity' ,axis=1)\n",
    "accident_ml = accident_ml[['Did_Police_Officer_Attend_Scene_of_Accident' , 'Age_of_Driver' ,'Vehicle_Type', 'Age_of_Vehicle','Engine_Capacity_(CC)','Day_of_Week' , 'Weather_Conditions' , 'Road_Surface_Conditions'\n",
    "                          , 'Light_Conditions', 'Sex_of_Driver' ,'Speed_limit']]\n",
    "\n",
    "accidents_ml.head()\n",
    "\n",
    "# Split the data into a training and test set.\n",
    "X_train, X_test, y_train, y_test = train_test_split(accident_ml.values, \n",
    "                                              accidents['Accident_Severity'].values,test_size=0.20, random_state=99)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "e88f3ee992a1e169244b33a4fd38968ed3b54433"
   },
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "ddd13f370e56bbde94e945aa600b6feaedf3ffeb"
   },
   "outputs": [],
   "source": [
    "random_forest = RandomForestClassifier(n_estimators=200)\n",
    "random_forest.fit(X_train,y_train)\n",
    "Y_pred = random_forest.predict(X_test)\n",
    "random_forest.score(X_test, y_test)\n",
    "acc_random_forest1 = round(random_forest.score(X_test, y_test) * 100, 2)\n",
    "\n",
    "sk_report = classification_report(\n",
    "    digits=6,\n",
    "    y_true=y_test, \n",
    "    y_pred=Y_pred)\n",
    "print(\"Accuracy\" , acc_random_forest1)\n",
    "print(sk_report)\n",
    "pd.crosstab(y_test, Y_pred, rownames=['Actual'], colnames=['Predicted'], margins=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "ae20ccb692186d35f9f574b66c3265025d25e13c"
   },
   "source": [
    "## Logistic Regression "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "96978ce02fb5095b38972f85042ec4e66dce8d38"
   },
   "outputs": [],
   "source": [
    "lr = LogisticRegression()\n",
    "# Fit the model on the trainng data.\n",
    "lr.fit(X_train, y_train)\n",
    "y_pred = lr.predict(X_test)\n",
    "sk_report = classification_report(\n",
    "    digits=6,\n",
    "    y_true=y_test, \n",
    "    y_pred=y_pred)\n",
    "print(\"Accuracy\", round(accuracy_score(y_pred, y_test)*100,2))\n",
    "print(sk_report)\n",
    "pd.crosstab(y_test, y_pred, rownames=['Actual'], colnames=['Predicted'], margins=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "714781275052e5512f4f6fb654fc006c322fde65"
   },
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "b66350c57786ddac65f80ec4682ee28bf68250cb"
   },
   "outputs": [],
   "source": [
    "decision_tree = DecisionTreeClassifier()\n",
    "decision_tree.fit(X_train, y_train)\n",
    "Y_pred = decision_tree.predict(X_test)\n",
    "acc_decision_tree1 = round(decision_tree.score(X_test, y_test) * 100, 2)\n",
    "sk_report = classification_report(\n",
    "    digits=6,\n",
    "    y_true=y_test, \n",
    "    y_pred=Y_pred)\n",
    "print(\"Accuracy\", acc_decision_tree1)\n",
    "print(sk_report)\n",
    "### Confusion Matrix \n",
    "pd.crosstab(y_test, Y_pred, rownames=['Actual'], colnames=['Predicted'], margins=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "be6bd6b139f10cbf1434aee4f3ecb7e58ac14fee"
   },
   "source": [
    "As we can see that Logistic regression did pretty well in terms of number. If we look carefully at the confusion matrix. We can definitely tell that Decision tree algorithm did much better. It predicted more fatal and serious injuries as true positive. The accuracy score is lower compare to another algorithm because other algorithm predicted majority of slightly accidents and those numbers are really high overall in the dataset. Confusion matrix helps us to understand what algorithm actually worked better in terms of looking at all different prediction of each class."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "20e2285ebc104e46ead88fdf348d3c729106999f"
   },
   "source": [
    "# Hyperparameters tuning for the models\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "5bdaee73e2ed61b9649792f057c9fc05e2b30108"
   },
   "source": [
    "### Logistic Regression with Hyperparameter tuning\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "422c67476c024b857edf2ba9cffb4108bba4d4e9"
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "lr = LogisticRegressionCV(cv=3, random_state=0,multi_class='multinomial')\n",
    "# Fit the model on the trainng data.\n",
    "lr.fit(X_train, y_train)\n",
    "y_pred = lr.predict(X_test)\n",
    "sk_report = classification_report(\n",
    "    digits=6,\n",
    "    y_true=y_test, \n",
    "    y_pred=y_pred)\n",
    "print(\"Accuracy\", round(accuracy_score(y_pred, y_test)*100,2))\n",
    "print(sk_report)\n",
    "pd.crosstab(y_test, y_pred, rownames=['Actual'], colnames=['Predicted'], margins=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "95f441cfef4974bc8a44bdbf07d7b8a85bc362ad"
   },
   "source": [
    "As we can see that Logistic regression still didn't predict two classes of accident severity out of 3. Even though it is showing the 86.2% accuracy. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "5365138a8ce876cd45b43a05281cdb863424bad8"
   },
   "source": [
    "### Decision Tree hyperparameters tuning\n",
    "\n",
    "All we are going to do is find the best values for mininum sample leaf and maximum features to get the best score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "c87f509414eac285236e01fca0ecd8a99d2bdbf8"
   },
   "outputs": [],
   "source": [
    "decision_tree = DecisionTreeClassifier(min_samples_leaf=12, max_features=4)\n",
    "decision_tree.fit(X_train, y_train)\n",
    "Y_pred = decision_tree.predict(X_test)\n",
    "acc_decision_tree1 = round(decision_tree.score(X_test, y_test) * 100, 2)\n",
    "sk_report = classification_report(\n",
    "    digits=6,\n",
    "    y_true=y_test, \n",
    "    y_pred=Y_pred)\n",
    "print(\"Accuracy\", acc_decision_tree1)\n",
    "print(sk_report)\n",
    "### Confusion Matrix \n",
    "pd.crosstab(y_test, Y_pred, rownames=['Actual'], colnames=['Predicted'], margins=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "e9052958b1ac2236b1789355a5a11a750f8e4aa1"
   },
   "source": [
    "We really didn't see much difference in Accident severity 1 and 2. However we did improve the accuracy of Accident severity 3. It jumped the accuracy from 75.1% to 85.8%."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "7f9440cd53d5f18ef91c80e19fd7593d1a7590f4"
   },
   "source": [
    "## Feature importance\n",
    "We can use Sklearn's random forest library to find out the most important features. We will be plotting in  ascending order so we know what features are most important to predict the accident severity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "8fcb8231e0a2baf72e89f5985c5f20f05da9382c"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,6))\n",
    "feat_importances = pd.Series(random_forest.feature_importances_, index=accident_ml.columns)\n",
    "feat_importances.nlargest(5).plot(kind='barh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "8a6e8de82ae44d10b8c2ccb687bba2c947be8f95"
   },
   "outputs": [],
   "source": [
    "Y_pred = grid_search.predict(X_test)\n",
    "acc_random_forest1 = round(grid_search.score(X_test, y_test) * 100, 2)\n",
    "\n",
    "sk_report = classification_report(\n",
    "    digits=6,\n",
    "    y_true=y_test, \n",
    "    y_pred=Y_pred)\n",
    "print(\"Accuracy\" , acc_random_forest1)\n",
    "print(sk_report)\n",
    "pd.crosstab(y_test, Y_pred, rownames=['Actual'], colnames=['Predicted'], margins=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "bfceeaaa72237771033a5fb480018723e2011192"
   },
   "source": [
    "Random forest took lots of time to tune the hyperparameter. Most of the algorithm works well only with default values except decision tree."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "1dc94a15c09d173b17e5e890d27355884a588ffb"
   },
   "source": [
    "## Conclusion\n",
    "\n",
    "\n",
    "#### Machine Learning Conclusion\n",
    "\n",
    "\n",
    "#### Recommendation for Public ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "330f32e479a1c183e1faa763240fcf91b7c491b9"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "61f3e804e120de9092d0f11ae857671e6629be94"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}